diff --git a/kernel/x86_64/cgemm_kernel_4x2_bulldozer.S b/kernel/x86_64/cgemm_kernel_4x2_bulldozer.S
index 97958a88f..2675f71fb 100644
--- a/kernel/x86_64/cgemm_kernel_4x2_bulldozer.S
+++ b/kernel/x86_64/cgemm_kernel_4x2_bulldozer.S
@@ -1,1897 +1,1897 @@
-/*********************************************************************/
-/* Copyright 2009, 2010 The University of Texas at Austin.           */
-/* All rights reserved.                                              */
-/*                                                                   */
-/* Redistribution and use in source and binary forms, with or        */
-/* without modification, are permitted provided that the following   */
-/* conditions are met:                                               */
-/*                                                                   */
-/*   1. Redistributions of source code must retain the above         */
-/*      copyright notice, this list of conditions and the following  */
-/*      disclaimer.                                                  */
-/*                                                                   */
-/*   2. Redistributions in binary form must reproduce the above      */
-/*      copyright notice, this list of conditions and the following  */
-/*      disclaimer in the documentation and/or other materials       */
-/*      provided with the distribution.                              */
-/*                                                                   */
-/*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */
-/*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */
-/*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */
-/*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */
-/*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */
-/*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */
-/*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */
-/*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */
-/*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */
-/*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */
-/*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */
-/*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */
-/*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */
-/*    POSSIBILITY OF SUCH DAMAGE.                                    */
-/*                                                                   */
-/* The views and conclusions contained in the software and           */
-/* documentation are those of the authors and should not be          */
-/* interpreted as representing official policies, either expressed   */
-/* or implied, of The University of Texas at Austin.                 */
-/*********************************************************************/
-
-
-#define ASSEMBLER
-#include "common.h"
- 
-#define OLD_M	%rdi
-#define OLD_N	%rsi
-#define M	%r13
-#define J	%r14
-#define OLD_K	%rdx
-
-#define A	%rcx
-#define B	%r8
-#define C	%r9
-#define LDC	%r10
-	
-#define I	%r11
-#define AO	%rdi
-#define BO	%rsi
-#define	CO1	%r15
-#define K	%r12
-#define BI	%rbp
-#define	SP	%rbx
-
-#define BO1	%rdi
-#define BO2	%r15
-
-#ifndef WINDOWS_ABI
-
-#define STACKSIZE 96
-
-#else
-
-#define STACKSIZE 320
-
-#define OLD_ALPHA_I     40 + STACKSIZE(%rsp)
-#define OLD_A           48 + STACKSIZE(%rsp)
-#define OLD_B           56 + STACKSIZE(%rsp)
-#define OLD_C           64 + STACKSIZE(%rsp)
-#define OLD_LDC         72 + STACKSIZE(%rsp)
-#define OLD_OFFSET      80 + STACKSIZE(%rsp)
-
-#endif
-
-#define L_BUFFER_SIZE 8192
-
-#define Ndiv6	 24(%rsp)
-#define Nmod6	 32(%rsp)
-#define N	 40(%rsp)
-#define ALPHA_R  48(%rsp)
-#define ALPHA_I  56(%rsp)
-#define OFFSET   64(%rsp)
-#define KK       72(%rsp)
-#define KKK      80(%rsp)
-#define BUFFER1	           128(%rsp)
-#define BUFFER2	LB2_OFFSET+128(%rsp)
-
-#if defined(OS_WINDOWS)
-#if   L_BUFFER_SIZE > 16384
-#define STACK_TOUCH \
-        movl    $0,  4096 * 4(%rsp);\
-        movl    $0,  4096 * 3(%rsp);\
-        movl    $0,  4096 * 2(%rsp);\
-        movl    $0,  4096 * 1(%rsp);
-#elif L_BUFFER_SIZE > 12288
-#define STACK_TOUCH \
-        movl    $0,  4096 * 3(%rsp);\
-        movl    $0,  4096 * 2(%rsp);\
-        movl    $0,  4096 * 1(%rsp);
-#elif L_BUFFER_SIZE > 8192
-#define STACK_TOUCH \
-        movl    $0,  4096 * 2(%rsp);\
-        movl    $0,  4096 * 1(%rsp);
-#elif L_BUFFER_SIZE > 4096
-#define STACK_TOUCH \
-        movl    $0,  4096 * 1(%rsp);
-#else
-#define STACK_TOUCH
-#endif
-#else
-#define STACK_TOUCH
-#endif
-
-
-#if   defined(NN) || defined(NT) || defined(TN) || defined(TT)
-#define VFMADD_R    vfmaddps
-#define VFMADD_I    vfmaddps
-#elif defined(RN) || defined(RT) || defined(CN) || defined(CT)
-#define VFMADD_R    vfnmaddps
-#define VFMADD_I    vfmaddps
-#elif defined(NR) || defined(NC) || defined(TR) || defined(TC)
-#define VFMADD_R    vfmaddps
-#define VFMADD_I    vfnmaddps
-#else
-#define VFMADD_R    vfnmaddps
-#define VFMADD_I    vfnmaddps
-#endif
-
-
-
-#define	A_PR1	384
-#define	B_PR1	192
-
-#define KERNEL4x2_1(xx) \
-        prefetcht0      A_PR1(AO,%rax,SIZE)        ;\
-        vmovups         -16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss         -8 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        vmovups         -12 * SIZE(AO, %rax, SIZE), %xmm1 ;\
-        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
-        vbroadcastss         -7 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-        VFMADD_I        %xmm13,%xmm5,%xmm1,%xmm13 ;\
-        vbroadcastss         -6 * SIZE(BO, BI, SIZE), %xmm6 ;\
-        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
-        VFMADD_R        %xmm14,%xmm6,%xmm1,%xmm14 ;\
-        vbroadcastss         -5 * SIZE(BO, BI, SIZE), %xmm7 ;\
-        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
-        VFMADD_I        %xmm15,%xmm7,%xmm1,%xmm15 ;\
-
-#define KERNEL4x2_2(xx) \
-        vmovups          -8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss         -4 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        vmovups          -4 * SIZE(AO, %rax, SIZE), %xmm1 ;\
-        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
-        vbroadcastss         -3 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-        VFMADD_I        %xmm13,%xmm5,%xmm1,%xmm13 ;\
-        vbroadcastss         -2 * SIZE(BO, BI, SIZE), %xmm6 ;\
-        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
-        VFMADD_R        %xmm14,%xmm6,%xmm1,%xmm14 ;\
-        vbroadcastss         -1 * SIZE(BO, BI, SIZE), %xmm7 ;\
-        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
-        VFMADD_I        %xmm15,%xmm7,%xmm1,%xmm15 ;\
-
-#define KERNEL4x2_3(xx) \
-	prefetcht0      A_PR1+64(AO,%rax,SIZE)     ;\
-        vmovups           0 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss          0 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        vmovups           4 * SIZE(AO, %rax, SIZE), %xmm1 ;\
-        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
-        vbroadcastss          1 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-        VFMADD_I        %xmm13,%xmm5,%xmm1,%xmm13 ;\
-        vbroadcastss          2 * SIZE(BO, BI, SIZE), %xmm6 ;\
-        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
-        VFMADD_R        %xmm14,%xmm6,%xmm1,%xmm14 ;\
-        vbroadcastss          3 * SIZE(BO, BI, SIZE), %xmm7 ;\
-        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
-        VFMADD_I        %xmm15,%xmm7,%xmm1,%xmm15 ;\
-
-#define KERNEL4x2_4(xx) \
-        vmovups           8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss          4 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        vmovups          12 * SIZE(AO, %rax, SIZE), %xmm1 ;\
-        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
-        vbroadcastss          5 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-        VFMADD_I        %xmm13,%xmm5,%xmm1,%xmm13 ;\
-        vbroadcastss          6 * SIZE(BO, BI, SIZE), %xmm6 ;\
-        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
-        VFMADD_R        %xmm14,%xmm6,%xmm1,%xmm14 ;\
-        vbroadcastss          7 * SIZE(BO, BI, SIZE), %xmm7 ;\
-        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
-        VFMADD_I        %xmm15,%xmm7,%xmm1,%xmm15 ;\
-        addq    $16, BI                            ;\
-        addq    $32, %rax                          ;\
-
-
-#define KERNEL4x2_SUB(xx) \
-        vmovups         -16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss         -8 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        vmovups         -12 * SIZE(AO, %rax, SIZE), %xmm1 ;\
-        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
-        vbroadcastss         -7 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-        VFMADD_I        %xmm13,%xmm5,%xmm1,%xmm13 ;\
-        vbroadcastss         -6 * SIZE(BO, BI, SIZE), %xmm6 ;\
-        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
-        VFMADD_R        %xmm14,%xmm6,%xmm1,%xmm14 ;\
-        vbroadcastss         -5 * SIZE(BO, BI, SIZE), %xmm7 ;\
-        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
-        VFMADD_I        %xmm15,%xmm7,%xmm1,%xmm15 ;\
-        addq    $4, BI                            ;\
-        addq    $8, %rax                          ;\
-
-/************************************************************************************************/
-
-#define KERNEL2x2_1(xx) \
-        prefetcht0      A_PR1(AO,%rax,SIZE)        ;\
-        vmovups         -16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss         -8 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        vbroadcastss         -7 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-        vbroadcastss         -6 * SIZE(BO, BI, SIZE), %xmm6 ;\
-        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
-        vbroadcastss         -5 * SIZE(BO, BI, SIZE), %xmm7 ;\
-        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
-
-#define KERNEL2x2_2(xx) \
-        vmovups         -12 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss         -4 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
-        vbroadcastss         -3 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-        vbroadcastss         -2 * SIZE(BO, BI, SIZE), %xmm6 ;\
-        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
-        vbroadcastss         -1 * SIZE(BO, BI, SIZE), %xmm7 ;\
-        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
-
-#define KERNEL2x2_3(xx) \
-        vmovups          -8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss          0 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        vbroadcastss          1 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-        vbroadcastss          2 * SIZE(BO, BI, SIZE), %xmm6 ;\
-        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
-        vbroadcastss          3 * SIZE(BO, BI, SIZE), %xmm7 ;\
-        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
-
-#define KERNEL2x2_4(xx) \
-        vmovups          -4 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss          4 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        vbroadcastss          5 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-        vbroadcastss          6 * SIZE(BO, BI, SIZE), %xmm6 ;\
-        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
-        vbroadcastss          7 * SIZE(BO, BI, SIZE), %xmm7 ;\
-        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
-        addq    $16, BI                            ;\
-        addq    $16, %rax                          ;\
-
-
-#define KERNEL2x2_SUB(xx) \
-        vmovups         -16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss         -8 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        vbroadcastss         -7 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-        vbroadcastss         -6 * SIZE(BO, BI, SIZE), %xmm6 ;\
-        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
-        vbroadcastss         -5 * SIZE(BO, BI, SIZE), %xmm7 ;\
-        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
-        addq    $4, BI                            ;\
-        addq    $4, %rax                          ;\
-
-/************************************************************************************************/
-
-#define KERNEL1x2_1(xx) \
-        vmovsd          -16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss         -8 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        vbroadcastss         -7 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-        vbroadcastss         -6 * SIZE(BO, BI, SIZE), %xmm6 ;\
-        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
-        vbroadcastss         -5 * SIZE(BO, BI, SIZE), %xmm7 ;\
-        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
-
-#define KERNEL1x2_2(xx) \
-        vmovsd          -14 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss         -4 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
-        vbroadcastss         -3 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-        vbroadcastss         -2 * SIZE(BO, BI, SIZE), %xmm6 ;\
-        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
-        vbroadcastss         -1 * SIZE(BO, BI, SIZE), %xmm7 ;\
-        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
-
-#define KERNEL1x2_3(xx) \
-        vmovsd          -12 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss          0 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        vbroadcastss          1 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-        vbroadcastss          2 * SIZE(BO, BI, SIZE), %xmm6 ;\
-        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
-        vbroadcastss          3 * SIZE(BO, BI, SIZE), %xmm7 ;\
-        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
-
-#define KERNEL1x2_4(xx) \
-        vmovsd          -10 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss          4 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        vbroadcastss          5 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-        vbroadcastss          6 * SIZE(BO, BI, SIZE), %xmm6 ;\
-        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
-        vbroadcastss          7 * SIZE(BO, BI, SIZE), %xmm7 ;\
-        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
-        addq    $16, BI                            ;\
-        addq    $8, %rax                          ;\
-
-
-#define KERNEL1x2_SUB(xx) \
-        vmovsd         -16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss         -8 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        vbroadcastss         -7 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-        vbroadcastss         -6 * SIZE(BO, BI, SIZE), %xmm6 ;\
-        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
-        vbroadcastss         -5 * SIZE(BO, BI, SIZE), %xmm7 ;\
-        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
-        addq    $4, BI                            ;\
-        addq    $2, %rax                          ;\
-
-
-
-/************************************************************************************************/
-
-#define KERNEL4x1_1(xx) \
-        prefetcht0      A_PR1(AO,%rax,SIZE)        ;\
-        vmovups         -16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss         -4 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        vmovups         -12 * SIZE(AO, %rax, SIZE), %xmm1 ;\
-        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
-        vbroadcastss         -3 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-        VFMADD_I        %xmm13,%xmm5,%xmm1,%xmm13 ;\
-
-#define KERNEL4x1_2(xx) \
-        vmovups          -8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss         -2 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        vmovups          -4 * SIZE(AO, %rax, SIZE), %xmm1 ;\
-        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
-        vbroadcastss         -1 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-        VFMADD_I        %xmm13,%xmm5,%xmm1,%xmm13 ;\
-
-#define KERNEL4x1_3(xx) \
-	prefetcht0      A_PR1+64(AO,%rax,SIZE)     ;\
-        vmovups           0 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss          0 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        vmovups           4 * SIZE(AO, %rax, SIZE), %xmm1 ;\
-        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
-        vbroadcastss          1 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-        VFMADD_I        %xmm13,%xmm5,%xmm1,%xmm13 ;\
-
-#define KERNEL4x1_4(xx) \
-        vmovups           8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss          2 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        vmovups          12 * SIZE(AO, %rax, SIZE), %xmm1 ;\
-        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
-        vbroadcastss          3 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-        VFMADD_I        %xmm13,%xmm5,%xmm1,%xmm13 ;\
-        addq    $8, BI                            ;\
-        addq    $32, %rax                          ;\
-
-
-#define KERNEL4x1_SUB(xx) \
-        vmovups         -16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss         -4 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        vmovups         -12 * SIZE(AO, %rax, SIZE), %xmm1 ;\
-        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
-        vbroadcastss         -3 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-        VFMADD_I        %xmm13,%xmm5,%xmm1,%xmm13 ;\
-        addq    $2, BI                            ;\
-        addq    $8, %rax                          ;\
-
-
-/************************************************************************************************/
-
-#define KERNEL2x1_1(xx) \
-        prefetcht0      A_PR1(AO,%rax,SIZE)        ;\
-        vmovups         -16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss         -4 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        vbroadcastss         -3 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-
-#define KERNEL2x1_2(xx) \
-        vmovups         -12 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss         -2 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        vbroadcastss         -1 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-
-#define KERNEL2x1_3(xx) \
-        vmovups          -8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss          0 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        vbroadcastss          1 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-
-#define KERNEL2x1_4(xx) \
-        vmovups          -4 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss          2 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        vbroadcastss          3 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-        addq    $8, BI                            ;\
-        addq    $16, %rax                          ;\
-
-
-#define KERNEL2x1_SUB(xx) \
-        vmovups         -16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss         -4 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        vbroadcastss         -3 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-        addq    $2, BI                            ;\
-        addq    $4, %rax                          ;\
-
-
-/************************************************************************************************/
-
-#define KERNEL1x1_1(xx) \
-        vmovsd         -16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss         -4 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        vbroadcastss         -3 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-
-#define KERNEL1x1_2(xx) \
-        vmovsd         -14 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss         -2 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        vbroadcastss         -1 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-
-#define KERNEL1x1_3(xx) \
-        vmovsd         -12 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss          0 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        vbroadcastss          1 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-
-#define KERNEL1x1_4(xx) \
-        vmovsd          -10 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss          2 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        vbroadcastss          3 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-        addq    $8, BI                            ;\
-        addq    $8, %rax                          ;\
-
-
-#define KERNEL1x1_SUB(xx) \
-        vmovsd         -16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss         -4 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        vbroadcastss         -3 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-        addq    $2, BI                            ;\
-        addq    $2, %rax                          ;\
-
-
-/************************************************************************************************/
-
-
-
-
-	PROLOGUE
-	PROFCODE
-	
-	subq	$STACKSIZE, %rsp
-	movq	%rbx,   (%rsp)
-	movq	%rbp,  8(%rsp)
-	movq	%r12, 16(%rsp)
-	movq	%r13, 24(%rsp)
-	movq	%r14, 32(%rsp)
-	movq	%r15, 40(%rsp)
-
-	vzeroupper
-
-#ifdef WINDOWS_ABI
-	movq	%rdi,    48(%rsp)
-	movq	%rsi,    56(%rsp)
-	vmovups	%xmm6,   64(%rsp)
-	vmovups	%xmm7,   80(%rsp)
-	vmovups	%xmm8,   96(%rsp)
-	vmovups	%xmm9,  112(%rsp)
-	vmovups	%xmm10, 128(%rsp)
-	vmovups	%xmm11, 144(%rsp)
-	vmovups	%xmm12, 160(%rsp)
-	vmovups	%xmm13, 176(%rsp)
-	vmovups	%xmm14, 192(%rsp)
-	vmovups	%xmm15, 208(%rsp)
-
-	movq	ARG1,      OLD_M
-	movq	ARG2,      OLD_N
-	movq	ARG3,      OLD_K
-	movq	OLD_A,     A
-	movq	OLD_B,     B
-	movq	OLD_C,     C
-	movq	OLD_LDC,   LDC
-#ifdef TRMMKERNEL
-	vmovsd	OLD_OFFSET, %xmm12
-#endif
-	vmovaps	%xmm3, %xmm0
-	vmovsd   OLD_ALPHA_I, %xmm1
-
-#else
-	movq	STACKSIZE +  8(%rsp), LDC
-#ifdef TRMMKERNEL
-	vmovsd	STACKSIZE + 16(%rsp), %xmm12
-#endif
-
-#endif
-
-	movq    %rsp, SP      # save old stack
-        subq    $128 + L_BUFFER_SIZE, %rsp
-        andq    $-4096, %rsp    # align stack
-
-        STACK_TOUCH
-
-	cmpq	$0, OLD_M
-	je	.L999
-
-	cmpq	$0, OLD_N
-	je	.L999
-
-	cmpq	$0, OLD_K
-	je	.L999
-
-	movq	OLD_M, M
-	movq	OLD_N, N
-	movq	OLD_K, K
-
-	vmovss	 %xmm0, ALPHA_R
-	vmovss	 %xmm1, ALPHA_I
-
-	salq	$ZBASE_SHIFT, LDC
-
-	movq    N, %rax
-        xorq    %rdx, %rdx
-        movq    $2,  %rdi
-        divq    %rdi                    //    N / 2
-        movq    %rax, Ndiv6             //    N / 2
-        movq    %rdx, Nmod6             //    N % 2
-
-	
-
-#ifdef TRMMKERNEL
-	vmovsd	%xmm12, OFFSET
-	vmovsd	%xmm12, KK
-#ifndef LEFT
-	negq	KK
-#endif	
-#endif
-
-.L2_0:
-
-	movq	Ndiv6,  J
-	cmpq	$0, J
-	je	.L1_0
-	ALIGN_4
-
-
-
-.L2_01:
-	// copy to sub buffer
-	movq	B, BO1
-	leaq    BUFFER1, BO		// first buffer to BO
-	movq	K, %rax
-	ALIGN_4
-
-.L2_02b:
-
-	vmovups	(BO1), %xmm0
-	vmovups	%xmm0,       (BO)
-	addq	$4*SIZE,BO1
-	addq	$4*SIZE,BO
-	decq	%rax
-	jnz	.L2_02b
-
-.L2_02c:
-
-	movq	BO1, B			// next offset of B
-
-.L2_10:
-	movq	C, CO1
-	leaq	(C, LDC, 2), C		// c += 2 * ldc
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        movq    OFFSET, %rax
-        movq    %rax, KK
-#endif
-	
-	movq	A, AO		 	// aoffset = a
-	addq	$16 * SIZE, AO
-
-	movq	M,  I
-	sarq	$2, I			// i = (m >> 2)
-	je	.L2_20
-
-	ALIGN_4
-
-.L2_11:
-
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-	leaq	BUFFER1, BO		// first buffer to BO
-	addq	$8 * SIZE, BO
-#else
-        movq    KK, %rax
-	leaq	BUFFER1, BO			// first buffer to BO
-	addq	$8 * SIZE, BO
-	movq    %rax, BI                        //  Index for BO
-        leaq    (,BI,4), BI                     //  BI = BI * 4 ; number of values
-        leaq    (BO, BI, SIZE), BO
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $4, %rax        // number of values in AO
-#else
-        addq    $2, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-
-	andq	$-8, %rax			//  K = K - ( K % 8 )
-	je	.L2_16
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_12:
-
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL4x2_1(xxx)
-	KERNEL4x2_2(xxx)
-	KERNEL4x2_3(xxx)
-	KERNEL4x2_4(xxx)
-
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL4x2_1(xxx)
-	KERNEL4x2_2(xxx)
-	KERNEL4x2_3(xxx)
-	KERNEL4x2_4(xxx)
-
-	je	.L2_16
-
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL4x2_1(xxx)
-	KERNEL4x2_2(xxx)
-	KERNEL4x2_3(xxx)
-	KERNEL4x2_4(xxx)
-
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL4x2_1(xxx)
-	KERNEL4x2_2(xxx)
-	KERNEL4x2_3(xxx)
-	KERNEL4x2_4(xxx)
-
-	je	.L2_16
-
-	jmp	.L2_12
-	ALIGN_4
-
-.L2_16:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_19
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_17:
-
-	KERNEL4x2_SUB(xxx)
-	jl	.L2_17
-	ALIGN_4
-
-
-.L2_19:
-
-	vbroadcastss	ALPHA_R, %xmm0
-	vbroadcastss	ALPHA_I, %xmm1
-
-	// swap high and low 64 bytes
-        vshufps $0xb1, %xmm9 , %xmm9, %xmm9
-        vshufps $0xb1, %xmm11, %xmm11, %xmm11
-        vshufps $0xb1, %xmm13, %xmm13, %xmm13
-        vshufps $0xb1, %xmm15, %xmm15, %xmm15
-
-#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
-    defined(NR) || defined(NC) || defined(TR) || defined(TC)
-
-        vaddsubps %xmm9, %xmm8 , %xmm8
-        vaddsubps %xmm11,%xmm10, %xmm10
-        vaddsubps %xmm13,%xmm12, %xmm12
-        vaddsubps %xmm15,%xmm14, %xmm14
-
-        vshufps $0xb1, %xmm8 , %xmm8, %xmm9
-        vshufps $0xb1, %xmm10, %xmm10, %xmm11
-        vshufps $0xb1, %xmm12, %xmm12, %xmm13
-        vshufps $0xb1, %xmm14, %xmm14, %xmm15
-
-#else
-        vaddsubps %xmm8,  %xmm9 ,%xmm9
-        vaddsubps %xmm10, %xmm11,%xmm11
-        vaddsubps %xmm12, %xmm13,%xmm13
-        vaddsubps %xmm14, %xmm15,%xmm15
-
-        vmovaps   %xmm9,  %xmm8
-        vmovaps   %xmm11, %xmm10
-        vmovaps   %xmm13, %xmm12
-        vmovaps   %xmm15, %xmm14
-
-	// swap high and low 64 bytes
-        vshufps $0xb1, %xmm9 , %xmm9, %xmm9
-        vshufps $0xb1, %xmm11, %xmm11, %xmm11
-        vshufps $0xb1, %xmm13, %xmm13, %xmm13
-        vshufps $0xb1, %xmm15, %xmm15, %xmm15
-
-#endif
-
-	// multiply with ALPHA_R
-        vmulps  %xmm8 , %xmm0, %xmm8
-        vmulps  %xmm10, %xmm0, %xmm10
-        vmulps  %xmm12, %xmm0, %xmm12
-        vmulps  %xmm14, %xmm0, %xmm14
-
-	// multiply with ALPHA_I
-        vmulps  %xmm9 , %xmm1, %xmm9
-        vmulps  %xmm11, %xmm1, %xmm11
-        vmulps  %xmm13, %xmm1, %xmm13
-        vmulps  %xmm15, %xmm1, %xmm15
-
-	vaddsubps %xmm9, %xmm8 , %xmm8
-        vaddsubps %xmm11,%xmm10, %xmm10
-        vaddsubps %xmm13,%xmm12, %xmm12
-        vaddsubps %xmm15,%xmm14, %xmm14
-
-
-
-#ifndef TRMMKERNEL
-
-	vaddps 	 	(CO1), %xmm8 , %xmm8
-	vaddps  4 * SIZE(CO1), %xmm12, %xmm12
-
-	vaddps 	 	(CO1, LDC), %xmm10, %xmm10
-	vaddps  4 * SIZE(CO1, LDC), %xmm14, %xmm14
-
-#endif
-
-	vmovups	%xmm8 ,  	(CO1)
-	vmovups	%xmm12 , 4 * SIZE(CO1)
-
-	vmovups	%xmm10 ,  	(CO1, LDC)
-	vmovups	%xmm14 , 4 * SIZE(CO1, LDC)
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
-        leaq    (BO, BI, SIZE), BO
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $4, KK
-#endif
-
-	addq	$8 * SIZE, CO1		# coffset += 8
-	decq	I			# i --
-	jg	.L2_11
-	ALIGN_4	
-
-
-
-/**************************************************************************
-* Rest of M 
-***************************************************************************/
-
-.L2_20:
-	testq	$3, M		
-	jz	.L2_60		// to next 2 lines of N
-
-	testq	$2, M		
-	jz	.L2_40
-	ALIGN_4
-
-.L2_21:
-
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-	leaq	BUFFER1, BO		// first buffer to BO
-	addq	$8 * SIZE, BO
-#else
-        movq    KK, %rax
-	leaq	BUFFER1, BO			// first buffer to BO
-	addq	$8 * SIZE, BO
-	movq    %rax, BI                        //  Index for BO
-        leaq    (,BI,4), BI                     //  BI = BI * 4 ; number of values
-        leaq    (BO, BI, SIZE), BO
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $2, %rax        // number of values in AO
-#else
-        addq    $2, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-
-	andq	$-8, %rax			//  K = K - ( K % 8 )
-	je	.L2_26
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_22:
-
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL2x2_1(xxx)
-	KERNEL2x2_2(xxx)
-	KERNEL2x2_3(xxx)
-	KERNEL2x2_4(xxx)
-
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL2x2_1(xxx)
-	KERNEL2x2_2(xxx)
-	KERNEL2x2_3(xxx)
-	KERNEL2x2_4(xxx)
-
-	je	.L2_26
-
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL2x2_1(xxx)
-	KERNEL2x2_2(xxx)
-	KERNEL2x2_3(xxx)
-	KERNEL2x2_4(xxx)
-
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL2x2_1(xxx)
-	KERNEL2x2_2(xxx)
-	KERNEL2x2_3(xxx)
-	KERNEL2x2_4(xxx)
-
-	je	.L2_26
-
-	jmp	.L2_22
-	ALIGN_4
-
-.L2_26:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_29
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_27:
-
-	KERNEL2x2_SUB(xxx)
-	jl	.L2_27
-	ALIGN_4
-
-
-.L2_29:
-
-	vbroadcastss	ALPHA_R, %xmm0
-	vbroadcastss	ALPHA_I, %xmm1
-
-	// swap high and low 64 bytes
-        vshufps $0xb1, %xmm9 , %xmm9, %xmm9
-        vshufps $0xb1, %xmm11, %xmm11, %xmm11
-
-#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
-    defined(NR) || defined(NC) || defined(TR) || defined(TC)
-
-        vaddsubps %xmm9, %xmm8 , %xmm8
-        vaddsubps %xmm11,%xmm10, %xmm10
-
-        vshufps $0xb1, %xmm8 , %xmm8, %xmm9
-        vshufps $0xb1, %xmm10, %xmm10, %xmm11
-
-#else
-        vaddsubps %xmm8,  %xmm9 ,%xmm9
-        vaddsubps %xmm10, %xmm11,%xmm11
-
-        vmovaps   %xmm9,  %xmm8
-        vmovaps   %xmm11, %xmm10
-
-	// swap high and low 64 bytes
-        vshufps $0xb1, %xmm9 , %xmm9, %xmm9
-        vshufps $0xb1, %xmm11, %xmm11, %xmm11
-
-#endif
-
-	// multiply with ALPHA_R
-        vmulps  %xmm8 , %xmm0, %xmm8
-        vmulps  %xmm10, %xmm0, %xmm10
-
-	// multiply with ALPHA_I
-        vmulps  %xmm9 , %xmm1, %xmm9
-        vmulps  %xmm11, %xmm1, %xmm11
-
-	vaddsubps %xmm9, %xmm8 , %xmm8
-        vaddsubps %xmm11,%xmm10, %xmm10
-
-
-
-#ifndef TRMMKERNEL
-
-	vaddps 	 	(CO1), %xmm8 , %xmm8
-
-	vaddps 	 	(CO1, LDC), %xmm10, %xmm10
-
-#endif
-
-	vmovups	%xmm8 ,  	(CO1)
-
-	vmovups	%xmm10 ,  	(CO1, LDC)
-
-
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
-        leaq    (BO, BI, SIZE), BO
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $2, KK
-#endif
-
-	addq	$4 * SIZE, CO1		# coffset += 4
-	ALIGN_4	
-
-
-
-/**************************************************************************/
-.L2_40:
-	testq	$1, M		
-	jz	.L2_60		// to next 2 lines of N
-
-	ALIGN_4
-
-.L2_41:
-
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-	leaq	BUFFER1, BO		// first buffer to BO
-	addq	$8 * SIZE, BO
-#else
-        movq    KK, %rax
-	leaq	BUFFER1, BO			// first buffer to BO
-	addq	$8 * SIZE, BO
-	movq    %rax, BI                        //  Index for BO
-        leaq    (,BI,4), BI                     //  BI = BI * 4 ; number of values
-        leaq    (BO, BI, SIZE), BO
-	salq	$1, %rax			// rax = rax * 2 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $1, %rax        // number of values in AO
-#else
-        addq    $2, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-
-	andq	$-8, %rax			//  K = K - ( K % 8 )
-	je	.L2_46
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
-
-	salq	$1, %rax			// rax = rax * 2 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_42:
-
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL1x2_1(xxx)
-	KERNEL1x2_2(xxx)
-	KERNEL1x2_3(xxx)
-	KERNEL1x2_4(xxx)
-
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL1x2_1(xxx)
-	KERNEL1x2_2(xxx)
-	KERNEL1x2_3(xxx)
-	KERNEL1x2_4(xxx)
-
-	je	.L2_46
-
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL1x2_1(xxx)
-	KERNEL1x2_2(xxx)
-	KERNEL1x2_3(xxx)
-	KERNEL1x2_4(xxx)
-
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL1x2_1(xxx)
-	KERNEL1x2_2(xxx)
-	KERNEL1x2_3(xxx)
-	KERNEL1x2_4(xxx)
-
-	je	.L2_46
-
-	jmp	.L2_42
-	ALIGN_4
-
-.L2_46:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_49
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
-
-	salq	$1, %rax			// rax = rax * 2 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_47:
-
-	KERNEL1x2_SUB(xxx)
-	jl	.L2_47
-	ALIGN_4
-
-
-.L2_49:
-
-	vbroadcastss	ALPHA_R, %xmm0
-	vbroadcastss	ALPHA_I, %xmm1
-
-	// swap high and low 64 bytes
-        vshufps $0xb1, %xmm9 , %xmm9, %xmm9
-        vshufps $0xb1, %xmm11, %xmm11, %xmm11
-
-#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
-    defined(NR) || defined(NC) || defined(TR) || defined(TC)
-
-        vaddsubps %xmm9, %xmm8 , %xmm8
-        vaddsubps %xmm11,%xmm10, %xmm10
-
-        vshufps $0xb1, %xmm8 , %xmm8, %xmm9
-        vshufps $0xb1, %xmm10, %xmm10, %xmm11
-
-#else
-        vaddsubps %xmm8,  %xmm9 ,%xmm9
-        vaddsubps %xmm10, %xmm11,%xmm11
-
-        vmovaps   %xmm9,  %xmm8
-        vmovaps   %xmm11, %xmm10
-
-	// swap high and low 64 bytes
-        vshufps $0xb1, %xmm9 , %xmm9, %xmm9
-        vshufps $0xb1, %xmm11, %xmm11, %xmm11
-
-#endif
-
-	// multiply with ALPHA_R
-        vmulps  %xmm8 , %xmm0, %xmm8
-        vmulps  %xmm10, %xmm0, %xmm10
-
-	// multiply with ALPHA_I
-        vmulps  %xmm9 , %xmm1, %xmm9
-        vmulps  %xmm11, %xmm1, %xmm11
-
-	vaddsubps %xmm9, %xmm8 , %xmm8
-        vaddsubps %xmm11,%xmm10, %xmm10
-
-
-
-#ifndef TRMMKERNEL
-
-	vmovsd		(CO1), %xmm14
-	vaddps 	 	%xmm14, %xmm8 , %xmm8
-
-	vmovsd		(CO1, LDC), %xmm15
-	vaddps 	 	%xmm15, %xmm10, %xmm10
-
-#endif
-
-	vmovsd	%xmm8 ,  	(CO1)
-
-	vmovsd	%xmm10 ,  	(CO1, LDC)
-
-
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
-        leaq    (BO, BI, SIZE), BO
-	salq	$1, %rax			// rax = rax * 2 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $1, KK
-#endif
-
-	addq	$2 * SIZE, CO1		# coffset += 2
-	ALIGN_4	
-
-
-
-	
-.L2_60:
-#if defined(TRMMKERNEL) && !defined(LEFT)
-        addq    $2, KK
-#endif
-
-	decq	J			// j --
-	jg	.L2_01			// next 2 lines of N
-
-
-
-.L1_0:
-
-/************************************************************************************************
-* Loop for Nmod6 % 2 > 0
-*************************************************************************************************/
-
-	movq	Nmod6, J		
-	andq	$1, J			// j % 2
-	je	.L999
-	ALIGN_4
-
-.L1_01:
-	// copy to sub buffer
-	movq	B, BO1
-	leaq    BUFFER1, BO		// first buffer to BO
-	movq	K, %rax
-	ALIGN_4
-
-.L1_02b:
-
-	vmovsd		(BO1), %xmm0
-	vmovsd	%xmm0,       (BO)
-	addq	$2*SIZE,BO1
-	addq	$2*SIZE,BO
-	decq	%rax
-	jnz	.L1_02b
-
-.L1_02c:
-
-	movq	BO1, B			// next offset of B
-
-.L1_10:
-	movq	C, CO1
-	leaq	(C, LDC, 1), C		// c += 1 * ldc
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        movq    OFFSET, %rax
-        movq    %rax, KK
-#endif
-	
-	movq	A, AO		 	// aoffset = a
-	addq	$16 * SIZE, AO
-
-	movq	M,  I
-	sarq	$2, I			// i = (m >> 2)
-	je	.L1_20
-
-	ALIGN_4
-
-.L1_11:
-
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-	leaq	BUFFER1, BO		// first buffer to BO
-	addq	$4 * SIZE, BO
-#else
-        movq    KK, %rax
-	leaq	BUFFER1, BO			// first buffer to BO
-	addq	$4 * SIZE, BO
-	movq    %rax, BI                        //  Index for BO
-        leaq    (,BI,2), BI                     //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $4, %rax        // number of values in AO
-#else
-        addq    $1, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-
-	andq	$-8, %rax			//  K = K - ( K % 8 )
-	je	.L1_16
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_12:
-
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL4x1_1(xxx)
-	KERNEL4x1_2(xxx)
-	KERNEL4x1_3(xxx)
-	KERNEL4x1_4(xxx)
-
-	KERNEL4x1_1(xxx)
-	KERNEL4x1_2(xxx)
-	KERNEL4x1_3(xxx)
-	KERNEL4x1_4(xxx)
-
-	je	.L1_16
-
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL4x1_1(xxx)
-	KERNEL4x1_2(xxx)
-	KERNEL4x1_3(xxx)
-	KERNEL4x1_4(xxx)
-
-	KERNEL4x1_1(xxx)
-	KERNEL4x1_2(xxx)
-	KERNEL4x1_3(xxx)
-	KERNEL4x1_4(xxx)
-
-	je	.L1_16
-
-	jmp	.L1_12
-	ALIGN_4
-
-.L1_16:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_19
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,2), BI                    //  BI = BI * 4 ; number of values
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_17:
-
-	KERNEL4x1_SUB(xxx)
-	jl	.L1_17
-	ALIGN_4
-
-
-.L1_19:
-
-	vbroadcastss	ALPHA_R, %xmm0
-	vbroadcastss	ALPHA_I, %xmm1
-
-	// swap high and low 64 bytes
-        vshufps $0xb1, %xmm9 , %xmm9, %xmm9
-        vshufps $0xb1, %xmm13, %xmm13, %xmm13
-
-#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
-    defined(NR) || defined(NC) || defined(TR) || defined(TC)
-
-        vaddsubps %xmm9, %xmm8 , %xmm8
-        vaddsubps %xmm13,%xmm12, %xmm12
-
-        vshufps $0xb1, %xmm8 , %xmm8, %xmm9
-        vshufps $0xb1, %xmm12, %xmm12, %xmm13
-
-#else
-        vaddsubps %xmm8,  %xmm9 ,%xmm9
-        vaddsubps %xmm12, %xmm13,%xmm13
-
-        vmovaps   %xmm9,  %xmm8
-        vmovaps   %xmm13, %xmm12
-
-	// swap high and low 64 bytes
-        vshufps $0xb1, %xmm9 , %xmm9, %xmm9
-        vshufps $0xb1, %xmm13, %xmm13, %xmm13
-
-#endif
-
-	// multiply with ALPHA_R
-        vmulps  %xmm8 , %xmm0, %xmm8
-        vmulps  %xmm12, %xmm0, %xmm12
-
-	// multiply with ALPHA_I
-        vmulps  %xmm9 , %xmm1, %xmm9
-        vmulps  %xmm13, %xmm1, %xmm13
-
-	vaddsubps %xmm9, %xmm8 , %xmm8
-        vaddsubps %xmm13,%xmm12, %xmm12
-
-
-
-#ifndef TRMMKERNEL
-
-	vaddps 	 	(CO1), %xmm8 , %xmm8
-	vaddps  4 * SIZE(CO1), %xmm12, %xmm12
-
-#endif
-
-	vmovups	%xmm8 ,  	(CO1)
-	vmovups	%xmm12 , 4 * SIZE(CO1)
-
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $4, KK
-#endif
-
-	addq	$8 * SIZE, CO1		# coffset += 8
-	decq	I			# i --
-	jg	.L1_11
-	ALIGN_4	
-
-
-
-/**************************************************************************
-* Rest of M 
-***************************************************************************/
-
-.L1_20:
-	testq	$3, M		
-	jz	.L999
-
-	testq	$2, M		
-	jz	.L1_40
-	ALIGN_4
-
-.L1_21:
-
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-	leaq	BUFFER1, BO		// first buffer to BO
-	addq	$4 * SIZE, BO
-#else
-        movq    KK, %rax
-	leaq	BUFFER1, BO			// first buffer to BO
-	addq	$4 * SIZE, BO
-	movq    %rax, BI                        //  Index for BO
-        leaq    (,BI,2), BI                     //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $2, %rax        // number of values in AO
-#else
-        addq    $1, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-
-	andq	$-8, %rax			//  K = K - ( K % 8 )
-	je	.L1_26
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_22:
-
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL2x1_1(xxx)
-	KERNEL2x1_2(xxx)
-	KERNEL2x1_3(xxx)
-	KERNEL2x1_4(xxx)
-
-	KERNEL2x1_1(xxx)
-	KERNEL2x1_2(xxx)
-	KERNEL2x1_3(xxx)
-	KERNEL2x1_4(xxx)
-
-	je	.L1_26
-
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL2x1_1(xxx)
-	KERNEL2x1_2(xxx)
-	KERNEL2x1_3(xxx)
-	KERNEL2x1_4(xxx)
-
-	KERNEL2x1_1(xxx)
-	KERNEL2x1_2(xxx)
-	KERNEL2x1_3(xxx)
-	KERNEL2x1_4(xxx)
-
-	je	.L1_26
-
-	jmp	.L1_22
-	ALIGN_4
-
-.L1_26:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_29
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,2), BI                    //  BI = BI * 2; number of values
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_27:
-
-	KERNEL2x1_SUB(xxx)
-	jl	.L1_27
-	ALIGN_4
-
-
-.L1_29:
-
-	vbroadcastss	ALPHA_R, %xmm0
-	vbroadcastss	ALPHA_I, %xmm1
-
-	// swap high and low 64 bytes
-        vshufps $0xb1, %xmm9 , %xmm9, %xmm9
-
-#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
-    defined(NR) || defined(NC) || defined(TR) || defined(TC)
-
-        vaddsubps %xmm9, %xmm8 , %xmm8
-
-        vshufps $0xb1, %xmm8 , %xmm8, %xmm9
-
-#else
-        vaddsubps %xmm8,  %xmm9 ,%xmm9
-
-        vmovaps   %xmm9,  %xmm8
-
-	// swap high and low 64 bytes
-        vshufps $0xb1, %xmm9 , %xmm9, %xmm9
-
-#endif
-
-	// multiply with ALPHA_R
-        vmulps  %xmm8 , %xmm0, %xmm8
-
-	// multiply with ALPHA_I
-        vmulps  %xmm9 , %xmm1, %xmm9
-
-	vaddsubps %xmm9, %xmm8 , %xmm8
-
-
-
-#ifndef TRMMKERNEL
-
-	vaddps 	 	(CO1), %xmm8 , %xmm8
-
-#endif
-
-	vmovups	%xmm8 ,  	(CO1)
-
-
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $2, KK
-#endif
-
-	addq	$4 * SIZE, CO1		# coffset += 4
-	ALIGN_4	
-
-
-
-/**************************************************************************/
-.L1_40:
-	testq	$1, M		
-	jz	.L999		// to next 2 lines of N
-
-	ALIGN_4
-
-.L1_41:
-
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-	leaq	BUFFER1, BO		// first buffer to BO
-	addq	$4 * SIZE, BO
-#else
-        movq    KK, %rax
-	leaq	BUFFER1, BO			// first buffer to BO
-	addq	$4 * SIZE, BO
-	movq    %rax, BI                        //  Index for BO
-        leaq    (,BI,2), BI                     //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO
-	salq	$1, %rax			// rax = rax * 2 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $1, %rax        // number of values in AO
-#else
-        addq    $1, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-
-	andq	$-8, %rax			//  K = K - ( K % 8 )
-	je	.L1_46
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values
-
-	salq	$1, %rax			// rax = rax * 2 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_42:
-
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL1x1_1(xxx)
-	KERNEL1x1_2(xxx)
-	KERNEL1x1_3(xxx)
-	KERNEL1x1_4(xxx)
-
-	KERNEL1x1_1(xxx)
-	KERNEL1x1_2(xxx)
-	KERNEL1x1_3(xxx)
-	KERNEL1x1_4(xxx)
-
-	je	.L1_46
-
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL1x1_1(xxx)
-	KERNEL1x1_2(xxx)
-	KERNEL1x1_3(xxx)
-	KERNEL1x1_4(xxx)
-
-	KERNEL1x1_1(xxx)
-	KERNEL1x1_2(xxx)
-	KERNEL1x1_3(xxx)
-	KERNEL1x1_4(xxx)
-
-	je	.L1_46
-
-	jmp	.L1_42
-	ALIGN_4
-
-.L1_46:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_49
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values
-
-	salq	$1, %rax			// rax = rax * 2 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_47:
-
-	KERNEL1x1_SUB(xxx)
-	jl	.L1_47
-	ALIGN_4
-
-
-.L1_49:
-
-	vbroadcastss	ALPHA_R, %xmm0
-	vbroadcastss	ALPHA_I, %xmm1
-
-	// swap high and low 64 bytes
-        vshufps $0xb1, %xmm9 , %xmm9, %xmm9
-
-#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
-    defined(NR) || defined(NC) || defined(TR) || defined(TC)
-
-        vaddsubps %xmm9, %xmm8 , %xmm8
-
-        vshufps $0xb1, %xmm8 , %xmm8, %xmm9
-
-#else
-        vaddsubps %xmm8,  %xmm9 ,%xmm9
-
-        vmovaps   %xmm9,  %xmm8
-
-	// swap high and low 64 bytes
-        vshufps $0xb1, %xmm9 , %xmm9, %xmm9
-
-#endif
-
-	// multiply with ALPHA_R
-        vmulps  %xmm8 , %xmm0, %xmm8
-
-	// multiply with ALPHA_I
-        vmulps  %xmm9 , %xmm1, %xmm9
-
-	vaddsubps %xmm9, %xmm8 , %xmm8
-
-
-
-#ifndef TRMMKERNEL
-
-	vmovsd		(CO1), %xmm14
-	vaddps 	 	%xmm14, %xmm8 , %xmm8
-
-#endif
-
-	vmovsd	%xmm8 ,  	(CO1)
-
-
-
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO
-	salq	$1, %rax			// rax = rax * 2 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $1, KK
-#endif
-
-	addq	$2 * SIZE, CO1		# coffset += 2
-	ALIGN_4	
-
-
-
-
-
-
-
-.L999:
-	vzeroupper
-
-	movq   		SP, %rsp
-	movq	   (%rsp), %rbx
-	movq	  8(%rsp), %rbp
-	movq	 16(%rsp), %r12
-	movq	 24(%rsp), %r13
-	movq	 32(%rsp), %r14
-	movq	 40(%rsp), %r15
-
-#ifdef WINDOWS_ABI
-	movq	 48(%rsp), %rdi
-	movq	 56(%rsp), %rsi
-	vmovups	 64(%rsp), %xmm6
-	vmovups	 80(%rsp), %xmm7
-	vmovups	 96(%rsp), %xmm8
-	vmovups	112(%rsp), %xmm9
-	vmovups	128(%rsp), %xmm10
-	vmovups	144(%rsp), %xmm11
-	vmovups	160(%rsp), %xmm12
-	vmovups	176(%rsp), %xmm13
-	vmovups	192(%rsp), %xmm14
-	vmovups	208(%rsp), %xmm15
-#endif
-
-	addq	$STACKSIZE, %rsp
-	ret
-
-	EPILOGUE
+/*********************************************************************/
+/* Copyright 2009, 2010 The University of Texas at Austin.           */
+/* All rights reserved.                                              */
+/*                                                                   */
+/* Redistribution and use in source and binary forms, with or        */
+/* without modification, are permitted provided that the following   */
+/* conditions are met:                                               */
+/*                                                                   */
+/*   1. Redistributions of source code must retain the above         */
+/*      copyright notice, this list of conditions and the following  */
+/*      disclaimer.                                                  */
+/*                                                                   */
+/*   2. Redistributions in binary form must reproduce the above      */
+/*      copyright notice, this list of conditions and the following  */
+/*      disclaimer in the documentation and/or other materials       */
+/*      provided with the distribution.                              */
+/*                                                                   */
+/*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */
+/*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */
+/*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */
+/*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */
+/*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */
+/*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */
+/*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */
+/*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */
+/*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */
+/*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */
+/*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */
+/*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */
+/*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */
+/*    POSSIBILITY OF SUCH DAMAGE.                                    */
+/*                                                                   */
+/* The views and conclusions contained in the software and           */
+/* documentation are those of the authors and should not be          */
+/* interpreted as representing official policies, either expressed   */
+/* or implied, of The University of Texas at Austin.                 */
+/*********************************************************************/
+
+
+#define ASSEMBLER
+#include "common.h"
+ 
+#define OLD_M	%rdi
+#define OLD_N	%rsi
+#define M	%r13
+#define J	%r14
+#define OLD_K	%rdx
+
+#define A	%rcx
+#define B	%r8
+#define C	%r9
+#define LDC	%r10
+	
+#define I	%r11
+#define AO	%rdi
+#define BO	%rsi
+#define	CO1	%r15
+#define K	%r12
+#define BI	%rbp
+#define	SP	%rbx
+
+#define BO1	%rdi
+#define BO2	%r15
+
+#ifndef WINDOWS_ABI
+
+#define STACKSIZE 96
+
+#else
+
+#define STACKSIZE 320
+
+#define OLD_ALPHA_I     40 + STACKSIZE(%rsp)
+#define OLD_A           48 + STACKSIZE(%rsp)
+#define OLD_B           56 + STACKSIZE(%rsp)
+#define OLD_C           64 + STACKSIZE(%rsp)
+#define OLD_LDC         72 + STACKSIZE(%rsp)
+#define OLD_OFFSET      80 + STACKSIZE(%rsp)
+
+#endif
+
+#define L_BUFFER_SIZE 8192
+
+#define Ndiv6	 24(%rsp)
+#define Nmod6	 32(%rsp)
+#define N	 40(%rsp)
+#define ALPHA_R  48(%rsp)
+#define ALPHA_I  56(%rsp)
+#define OFFSET   64(%rsp)
+#define KK       72(%rsp)
+#define KKK      80(%rsp)
+#define BUFFER1	           128(%rsp)
+#define BUFFER2	LB2_OFFSET+128(%rsp)
+
+#if defined(OS_WINDOWS)
+#if   L_BUFFER_SIZE > 16384
+#define STACK_TOUCH \
+        movl    $0,  4096 * 4(%rsp);\
+        movl    $0,  4096 * 3(%rsp);\
+        movl    $0,  4096 * 2(%rsp);\
+        movl    $0,  4096 * 1(%rsp);
+#elif L_BUFFER_SIZE > 12288
+#define STACK_TOUCH \
+        movl    $0,  4096 * 3(%rsp);\
+        movl    $0,  4096 * 2(%rsp);\
+        movl    $0,  4096 * 1(%rsp);
+#elif L_BUFFER_SIZE > 8192
+#define STACK_TOUCH \
+        movl    $0,  4096 * 2(%rsp);\
+        movl    $0,  4096 * 1(%rsp);
+#elif L_BUFFER_SIZE > 4096
+#define STACK_TOUCH \
+        movl    $0,  4096 * 1(%rsp);
+#else
+#define STACK_TOUCH
+#endif
+#else
+#define STACK_TOUCH
+#endif
+
+
+#if   defined(NN) || defined(NT) || defined(TN) || defined(TT)
+#define VFMADD_R    vfmaddps
+#define VFMADD_I    vfmaddps
+#elif defined(RN) || defined(RT) || defined(CN) || defined(CT)
+#define VFMADD_R    vfnmaddps
+#define VFMADD_I    vfmaddps
+#elif defined(NR) || defined(NC) || defined(TR) || defined(TC)
+#define VFMADD_R    vfmaddps
+#define VFMADD_I    vfnmaddps
+#else
+#define VFMADD_R    vfnmaddps
+#define VFMADD_I    vfnmaddps
+#endif
+
+
+
+#define	A_PR1	384
+#define	B_PR1	192
+
+#define KERNEL4x2_1(xx) \
+        prefetcht0      A_PR1(AO,%rax,SIZE)        ;\
+        vmovups         -16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss         -8 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vmovups         -12 * SIZE(AO, %rax, SIZE), %xmm1 ;\
+        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
+        vbroadcastss         -7 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        VFMADD_I        %xmm13,%xmm5,%xmm1,%xmm13 ;\
+        vbroadcastss         -6 * SIZE(BO, BI, SIZE), %xmm6 ;\
+        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
+        VFMADD_R        %xmm14,%xmm6,%xmm1,%xmm14 ;\
+        vbroadcastss         -5 * SIZE(BO, BI, SIZE), %xmm7 ;\
+        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
+        VFMADD_I        %xmm15,%xmm7,%xmm1,%xmm15 ;\
+
+#define KERNEL4x2_2(xx) \
+        vmovups          -8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss         -4 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vmovups          -4 * SIZE(AO, %rax, SIZE), %xmm1 ;\
+        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
+        vbroadcastss         -3 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        VFMADD_I        %xmm13,%xmm5,%xmm1,%xmm13 ;\
+        vbroadcastss         -2 * SIZE(BO, BI, SIZE), %xmm6 ;\
+        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
+        VFMADD_R        %xmm14,%xmm6,%xmm1,%xmm14 ;\
+        vbroadcastss         -1 * SIZE(BO, BI, SIZE), %xmm7 ;\
+        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
+        VFMADD_I        %xmm15,%xmm7,%xmm1,%xmm15 ;\
+
+#define KERNEL4x2_3(xx) \
+	prefetcht0      A_PR1+64(AO,%rax,SIZE)     ;\
+        vmovups           0 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss          0 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vmovups           4 * SIZE(AO, %rax, SIZE), %xmm1 ;\
+        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
+        vbroadcastss          1 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        VFMADD_I        %xmm13,%xmm5,%xmm1,%xmm13 ;\
+        vbroadcastss          2 * SIZE(BO, BI, SIZE), %xmm6 ;\
+        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
+        VFMADD_R        %xmm14,%xmm6,%xmm1,%xmm14 ;\
+        vbroadcastss          3 * SIZE(BO, BI, SIZE), %xmm7 ;\
+        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
+        VFMADD_I        %xmm15,%xmm7,%xmm1,%xmm15 ;\
+
+#define KERNEL4x2_4(xx) \
+        vmovups           8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss          4 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vmovups          12 * SIZE(AO, %rax, SIZE), %xmm1 ;\
+        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
+        vbroadcastss          5 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        VFMADD_I        %xmm13,%xmm5,%xmm1,%xmm13 ;\
+        vbroadcastss          6 * SIZE(BO, BI, SIZE), %xmm6 ;\
+        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
+        VFMADD_R        %xmm14,%xmm6,%xmm1,%xmm14 ;\
+        vbroadcastss          7 * SIZE(BO, BI, SIZE), %xmm7 ;\
+        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
+        VFMADD_I        %xmm15,%xmm7,%xmm1,%xmm15 ;\
+        addq    $16, BI                            ;\
+        addq    $32, %rax                          ;\
+
+
+#define KERNEL4x2_SUB(xx) \
+        vmovups         -16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss         -8 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vmovups         -12 * SIZE(AO, %rax, SIZE), %xmm1 ;\
+        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
+        vbroadcastss         -7 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        VFMADD_I        %xmm13,%xmm5,%xmm1,%xmm13 ;\
+        vbroadcastss         -6 * SIZE(BO, BI, SIZE), %xmm6 ;\
+        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
+        VFMADD_R        %xmm14,%xmm6,%xmm1,%xmm14 ;\
+        vbroadcastss         -5 * SIZE(BO, BI, SIZE), %xmm7 ;\
+        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
+        VFMADD_I        %xmm15,%xmm7,%xmm1,%xmm15 ;\
+        addq    $4, BI                            ;\
+        addq    $8, %rax                          ;\
+
+/************************************************************************************************/
+
+#define KERNEL2x2_1(xx) \
+        prefetcht0      A_PR1(AO,%rax,SIZE)        ;\
+        vmovups         -16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss         -8 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vbroadcastss         -7 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        vbroadcastss         -6 * SIZE(BO, BI, SIZE), %xmm6 ;\
+        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
+        vbroadcastss         -5 * SIZE(BO, BI, SIZE), %xmm7 ;\
+        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
+
+#define KERNEL2x2_2(xx) \
+        vmovups         -12 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss         -4 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
+        vbroadcastss         -3 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        vbroadcastss         -2 * SIZE(BO, BI, SIZE), %xmm6 ;\
+        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
+        vbroadcastss         -1 * SIZE(BO, BI, SIZE), %xmm7 ;\
+        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
+
+#define KERNEL2x2_3(xx) \
+        vmovups          -8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss          0 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vbroadcastss          1 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        vbroadcastss          2 * SIZE(BO, BI, SIZE), %xmm6 ;\
+        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
+        vbroadcastss          3 * SIZE(BO, BI, SIZE), %xmm7 ;\
+        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
+
+#define KERNEL2x2_4(xx) \
+        vmovups          -4 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss          4 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vbroadcastss          5 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        vbroadcastss          6 * SIZE(BO, BI, SIZE), %xmm6 ;\
+        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
+        vbroadcastss          7 * SIZE(BO, BI, SIZE), %xmm7 ;\
+        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
+        addq    $16, BI                            ;\
+        addq    $16, %rax                          ;\
+
+
+#define KERNEL2x2_SUB(xx) \
+        vmovups         -16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss         -8 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vbroadcastss         -7 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        vbroadcastss         -6 * SIZE(BO, BI, SIZE), %xmm6 ;\
+        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
+        vbroadcastss         -5 * SIZE(BO, BI, SIZE), %xmm7 ;\
+        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
+        addq    $4, BI                            ;\
+        addq    $4, %rax                          ;\
+
+/************************************************************************************************/
+
+#define KERNEL1x2_1(xx) \
+        vmovsd          -16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss         -8 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vbroadcastss         -7 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        vbroadcastss         -6 * SIZE(BO, BI, SIZE), %xmm6 ;\
+        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
+        vbroadcastss         -5 * SIZE(BO, BI, SIZE), %xmm7 ;\
+        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
+
+#define KERNEL1x2_2(xx) \
+        vmovsd          -14 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss         -4 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
+        vbroadcastss         -3 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        vbroadcastss         -2 * SIZE(BO, BI, SIZE), %xmm6 ;\
+        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
+        vbroadcastss         -1 * SIZE(BO, BI, SIZE), %xmm7 ;\
+        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
+
+#define KERNEL1x2_3(xx) \
+        vmovsd          -12 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss          0 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vbroadcastss          1 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        vbroadcastss          2 * SIZE(BO, BI, SIZE), %xmm6 ;\
+        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
+        vbroadcastss          3 * SIZE(BO, BI, SIZE), %xmm7 ;\
+        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
+
+#define KERNEL1x2_4(xx) \
+        vmovsd          -10 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss          4 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vbroadcastss          5 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        vbroadcastss          6 * SIZE(BO, BI, SIZE), %xmm6 ;\
+        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
+        vbroadcastss          7 * SIZE(BO, BI, SIZE), %xmm7 ;\
+        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
+        addq    $16, BI                            ;\
+        addq    $8, %rax                          ;\
+
+
+#define KERNEL1x2_SUB(xx) \
+        vmovsd         -16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss         -8 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vbroadcastss         -7 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        vbroadcastss         -6 * SIZE(BO, BI, SIZE), %xmm6 ;\
+        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
+        vbroadcastss         -5 * SIZE(BO, BI, SIZE), %xmm7 ;\
+        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
+        addq    $4, BI                            ;\
+        addq    $2, %rax                          ;\
+
+
+
+/************************************************************************************************/
+
+#define KERNEL4x1_1(xx) \
+        prefetcht0      A_PR1(AO,%rax,SIZE)        ;\
+        vmovups         -16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss         -4 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vmovups         -12 * SIZE(AO, %rax, SIZE), %xmm1 ;\
+        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
+        vbroadcastss         -3 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        VFMADD_I        %xmm13,%xmm5,%xmm1,%xmm13 ;\
+
+#define KERNEL4x1_2(xx) \
+        vmovups          -8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss         -2 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vmovups          -4 * SIZE(AO, %rax, SIZE), %xmm1 ;\
+        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
+        vbroadcastss         -1 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        VFMADD_I        %xmm13,%xmm5,%xmm1,%xmm13 ;\
+
+#define KERNEL4x1_3(xx) \
+	prefetcht0      A_PR1+64(AO,%rax,SIZE)     ;\
+        vmovups           0 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss          0 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vmovups           4 * SIZE(AO, %rax, SIZE), %xmm1 ;\
+        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
+        vbroadcastss          1 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        VFMADD_I        %xmm13,%xmm5,%xmm1,%xmm13 ;\
+
+#define KERNEL4x1_4(xx) \
+        vmovups           8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss          2 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vmovups          12 * SIZE(AO, %rax, SIZE), %xmm1 ;\
+        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
+        vbroadcastss          3 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        VFMADD_I        %xmm13,%xmm5,%xmm1,%xmm13 ;\
+        addq    $8, BI                            ;\
+        addq    $32, %rax                          ;\
+
+
+#define KERNEL4x1_SUB(xx) \
+        vmovups         -16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss         -4 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vmovups         -12 * SIZE(AO, %rax, SIZE), %xmm1 ;\
+        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
+        vbroadcastss         -3 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        VFMADD_I        %xmm13,%xmm5,%xmm1,%xmm13 ;\
+        addq    $2, BI                            ;\
+        addq    $8, %rax                          ;\
+
+
+/************************************************************************************************/
+
+#define KERNEL2x1_1(xx) \
+        prefetcht0      A_PR1(AO,%rax,SIZE)        ;\
+        vmovups         -16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss         -4 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vbroadcastss         -3 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+
+#define KERNEL2x1_2(xx) \
+        vmovups         -12 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss         -2 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vbroadcastss         -1 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+
+#define KERNEL2x1_3(xx) \
+        vmovups          -8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss          0 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vbroadcastss          1 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+
+#define KERNEL2x1_4(xx) \
+        vmovups          -4 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss          2 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vbroadcastss          3 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        addq    $8, BI                            ;\
+        addq    $16, %rax                          ;\
+
+
+#define KERNEL2x1_SUB(xx) \
+        vmovups         -16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss         -4 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vbroadcastss         -3 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        addq    $2, BI                            ;\
+        addq    $4, %rax                          ;\
+
+
+/************************************************************************************************/
+
+#define KERNEL1x1_1(xx) \
+        vmovsd         -16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss         -4 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vbroadcastss         -3 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+
+#define KERNEL1x1_2(xx) \
+        vmovsd         -14 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss         -2 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vbroadcastss         -1 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+
+#define KERNEL1x1_3(xx) \
+        vmovsd         -12 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss          0 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vbroadcastss          1 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+
+#define KERNEL1x1_4(xx) \
+        vmovsd          -10 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss          2 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vbroadcastss          3 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        addq    $8, BI                            ;\
+        addq    $8, %rax                          ;\
+
+
+#define KERNEL1x1_SUB(xx) \
+        vmovsd         -16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss         -4 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vbroadcastss         -3 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        addq    $2, BI                            ;\
+        addq    $2, %rax                          ;\
+
+
+/************************************************************************************************/
+
+
+
+
+	PROLOGUE
+	PROFCODE
+	
+	subq	$STACKSIZE, %rsp
+	movq	%rbx,   (%rsp)
+	movq	%rbp,  8(%rsp)
+	movq	%r12, 16(%rsp)
+	movq	%r13, 24(%rsp)
+	movq	%r14, 32(%rsp)
+	movq	%r15, 40(%rsp)
+
+	vzeroupper
+
+#ifdef WINDOWS_ABI
+	movq	%rdi,    48(%rsp)
+	movq	%rsi,    56(%rsp)
+	vmovups	%xmm6,   64(%rsp)
+	vmovups	%xmm7,   80(%rsp)
+	vmovups	%xmm8,   96(%rsp)
+	vmovups	%xmm9,  112(%rsp)
+	vmovups	%xmm10, 128(%rsp)
+	vmovups	%xmm11, 144(%rsp)
+	vmovups	%xmm12, 160(%rsp)
+	vmovups	%xmm13, 176(%rsp)
+	vmovups	%xmm14, 192(%rsp)
+	vmovups	%xmm15, 208(%rsp)
+
+	movq	ARG1,      OLD_M
+	movq	ARG2,      OLD_N
+	movq	ARG3,      OLD_K
+	movq	OLD_A,     A
+	movq	OLD_B,     B
+	movq	OLD_C,     C
+	movq	OLD_LDC,   LDC
+#ifdef TRMMKERNEL
+	vmovsd	OLD_OFFSET, %xmm12
+#endif
+	vmovaps	%xmm3, %xmm0
+	vmovsd   OLD_ALPHA_I, %xmm1
+
+#else
+	movq	STACKSIZE +  8(%rsp), LDC
+#ifdef TRMMKERNEL
+	vmovsd	STACKSIZE + 16(%rsp), %xmm12
+#endif
+
+#endif
+
+	movq    %rsp, SP      # save old stack
+        subq    $128 + L_BUFFER_SIZE, %rsp
+        andq    $-4096, %rsp    # align stack
+
+        STACK_TOUCH
+
+	cmpq	$0, OLD_M
+	je	.L999
+
+	cmpq	$0, OLD_N
+	je	.L999
+
+	cmpq	$0, OLD_K
+	je	.L999
+
+	movq	OLD_M, M
+	movq	OLD_N, N
+	movq	OLD_K, K
+
+	vmovss	 %xmm0, ALPHA_R
+	vmovss	 %xmm1, ALPHA_I
+
+	salq	$ZBASE_SHIFT, LDC
+
+	movq    N, %rax
+        xorq    %rdx, %rdx
+        movq    $2,  %rdi
+        divq    %rdi                    //    N / 2
+        movq    %rax, Ndiv6             //    N / 2
+        movq    %rdx, Nmod6             //    N % 2
+
+	
+
+#ifdef TRMMKERNEL
+	vmovsd	%xmm12, OFFSET
+	vmovsd	%xmm12, KK
+#ifndef LEFT
+	negq	KK
+#endif	
+#endif
+
+.L2_0:
+
+	movq	Ndiv6,  J
+	cmpq	$0, J
+	je	.L1_0
+	ALIGN_4
+
+
+
+.L2_01:
+	// copy to sub buffer
+	movq	B, BO1
+	leaq    BUFFER1, BO		// first buffer to BO
+	movq	K, %rax
+	ALIGN_4
+
+.L2_02b:
+
+	vmovups	(BO1), %xmm0
+	vmovups	%xmm0,       (BO)
+	addq	$4*SIZE,BO1
+	addq	$4*SIZE,BO
+	decq	%rax
+	jnz	.L2_02b
+
+.L2_02c:
+
+	movq	BO1, B			// next offset of B
+
+.L2_10:
+	movq	C, CO1
+	leaq	(C, LDC, 2), C		// c += 2 * ldc
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        movq    OFFSET, %rax
+        movq    %rax, KK
+#endif
+	
+	movq	A, AO		 	// aoffset = a
+	addq	$16 * SIZE, AO
+
+	movq	M,  I
+	sarq	$2, I			// i = (m >> 2)
+	je	.L2_20
+
+	ALIGN_4
+
+.L2_11:
+
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+	leaq	BUFFER1, BO		// first buffer to BO
+	addq	$8 * SIZE, BO
+#else
+        movq    KK, %rax
+	leaq	BUFFER1, BO			// first buffer to BO
+	addq	$8 * SIZE, BO
+	movq    %rax, BI                        //  Index for BO
+        leaq    (,BI,4), BI                     //  BI = BI * 4 ; number of values
+        leaq    (BO, BI, SIZE), BO
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $4, %rax        // number of values in AO
+#else
+        addq    $2, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$-8, %rax			//  K = K - ( K % 8 )
+	je	.L2_16
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_12:
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL4x2_1(xxx)
+	KERNEL4x2_2(xxx)
+	KERNEL4x2_3(xxx)
+	KERNEL4x2_4(xxx)
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL4x2_1(xxx)
+	KERNEL4x2_2(xxx)
+	KERNEL4x2_3(xxx)
+	KERNEL4x2_4(xxx)
+
+	je	.L2_16
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL4x2_1(xxx)
+	KERNEL4x2_2(xxx)
+	KERNEL4x2_3(xxx)
+	KERNEL4x2_4(xxx)
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL4x2_1(xxx)
+	KERNEL4x2_2(xxx)
+	KERNEL4x2_3(xxx)
+	KERNEL4x2_4(xxx)
+
+	je	.L2_16
+
+	jmp	.L2_12
+	ALIGN_4
+
+.L2_16:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_19
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_17:
+
+	KERNEL4x2_SUB(xxx)
+	jl	.L2_17
+	ALIGN_4
+
+
+.L2_19:
+
+	vbroadcastss	ALPHA_R, %xmm0
+	vbroadcastss	ALPHA_I, %xmm1
+
+	// swap high and low 64 bytes
+        vshufps $0xb1, %xmm9 , %xmm9, %xmm9
+        vshufps $0xb1, %xmm11, %xmm11, %xmm11
+        vshufps $0xb1, %xmm13, %xmm13, %xmm13
+        vshufps $0xb1, %xmm15, %xmm15, %xmm15
+
+#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
+    defined(NR) || defined(NC) || defined(TR) || defined(TC)
+
+        vaddsubps %xmm9, %xmm8 , %xmm8
+        vaddsubps %xmm11,%xmm10, %xmm10
+        vaddsubps %xmm13,%xmm12, %xmm12
+        vaddsubps %xmm15,%xmm14, %xmm14
+
+        vshufps $0xb1, %xmm8 , %xmm8, %xmm9
+        vshufps $0xb1, %xmm10, %xmm10, %xmm11
+        vshufps $0xb1, %xmm12, %xmm12, %xmm13
+        vshufps $0xb1, %xmm14, %xmm14, %xmm15
+
+#else
+        vaddsubps %xmm8,  %xmm9 ,%xmm9
+        vaddsubps %xmm10, %xmm11,%xmm11
+        vaddsubps %xmm12, %xmm13,%xmm13
+        vaddsubps %xmm14, %xmm15,%xmm15
+
+        vmovaps   %xmm9,  %xmm8
+        vmovaps   %xmm11, %xmm10
+        vmovaps   %xmm13, %xmm12
+        vmovaps   %xmm15, %xmm14
+
+	// swap high and low 64 bytes
+        vshufps $0xb1, %xmm9 , %xmm9, %xmm9
+        vshufps $0xb1, %xmm11, %xmm11, %xmm11
+        vshufps $0xb1, %xmm13, %xmm13, %xmm13
+        vshufps $0xb1, %xmm15, %xmm15, %xmm15
+
+#endif
+
+	// multiply with ALPHA_R
+        vmulps  %xmm8 , %xmm0, %xmm8
+        vmulps  %xmm10, %xmm0, %xmm10
+        vmulps  %xmm12, %xmm0, %xmm12
+        vmulps  %xmm14, %xmm0, %xmm14
+
+	// multiply with ALPHA_I
+        vmulps  %xmm9 , %xmm1, %xmm9
+        vmulps  %xmm11, %xmm1, %xmm11
+        vmulps  %xmm13, %xmm1, %xmm13
+        vmulps  %xmm15, %xmm1, %xmm15
+
+	vaddsubps %xmm9, %xmm8 , %xmm8
+        vaddsubps %xmm11,%xmm10, %xmm10
+        vaddsubps %xmm13,%xmm12, %xmm12
+        vaddsubps %xmm15,%xmm14, %xmm14
+
+
+
+#ifndef TRMMKERNEL
+
+	vaddps 	 	(CO1), %xmm8 , %xmm8
+	vaddps  4 * SIZE(CO1), %xmm12, %xmm12
+
+	vaddps 	 	(CO1, LDC), %xmm10, %xmm10
+	vaddps  4 * SIZE(CO1, LDC), %xmm14, %xmm14
+
+#endif
+
+	vmovups	%xmm8 ,  	(CO1)
+	vmovups	%xmm12 , 4 * SIZE(CO1)
+
+	vmovups	%xmm10 ,  	(CO1, LDC)
+	vmovups	%xmm14 , 4 * SIZE(CO1, LDC)
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
+        leaq    (BO, BI, SIZE), BO
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $4, KK
+#endif
+
+	addq	$8 * SIZE, CO1		# coffset += 8
+	decq	I			# i --
+	jg	.L2_11
+	ALIGN_4	
+
+
+
+/**************************************************************************
+* Rest of M 
+***************************************************************************/
+
+.L2_20:
+	testq	$3, M		
+	jz	.L2_60		// to next 2 lines of N
+
+	testq	$2, M		
+	jz	.L2_40
+	ALIGN_4
+
+.L2_21:
+
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+	leaq	BUFFER1, BO		// first buffer to BO
+	addq	$8 * SIZE, BO
+#else
+        movq    KK, %rax
+	leaq	BUFFER1, BO			// first buffer to BO
+	addq	$8 * SIZE, BO
+	movq    %rax, BI                        //  Index for BO
+        leaq    (,BI,4), BI                     //  BI = BI * 4 ; number of values
+        leaq    (BO, BI, SIZE), BO
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $2, %rax        // number of values in AO
+#else
+        addq    $2, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$-8, %rax			//  K = K - ( K % 8 )
+	je	.L2_26
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_22:
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL2x2_1(xxx)
+	KERNEL2x2_2(xxx)
+	KERNEL2x2_3(xxx)
+	KERNEL2x2_4(xxx)
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL2x2_1(xxx)
+	KERNEL2x2_2(xxx)
+	KERNEL2x2_3(xxx)
+	KERNEL2x2_4(xxx)
+
+	je	.L2_26
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL2x2_1(xxx)
+	KERNEL2x2_2(xxx)
+	KERNEL2x2_3(xxx)
+	KERNEL2x2_4(xxx)
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL2x2_1(xxx)
+	KERNEL2x2_2(xxx)
+	KERNEL2x2_3(xxx)
+	KERNEL2x2_4(xxx)
+
+	je	.L2_26
+
+	jmp	.L2_22
+	ALIGN_4
+
+.L2_26:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_29
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_27:
+
+	KERNEL2x2_SUB(xxx)
+	jl	.L2_27
+	ALIGN_4
+
+
+.L2_29:
+
+	vbroadcastss	ALPHA_R, %xmm0
+	vbroadcastss	ALPHA_I, %xmm1
+
+	// swap high and low 64 bytes
+        vshufps $0xb1, %xmm9 , %xmm9, %xmm9
+        vshufps $0xb1, %xmm11, %xmm11, %xmm11
+
+#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
+    defined(NR) || defined(NC) || defined(TR) || defined(TC)
+
+        vaddsubps %xmm9, %xmm8 , %xmm8
+        vaddsubps %xmm11,%xmm10, %xmm10
+
+        vshufps $0xb1, %xmm8 , %xmm8, %xmm9
+        vshufps $0xb1, %xmm10, %xmm10, %xmm11
+
+#else
+        vaddsubps %xmm8,  %xmm9 ,%xmm9
+        vaddsubps %xmm10, %xmm11,%xmm11
+
+        vmovaps   %xmm9,  %xmm8
+        vmovaps   %xmm11, %xmm10
+
+	// swap high and low 64 bytes
+        vshufps $0xb1, %xmm9 , %xmm9, %xmm9
+        vshufps $0xb1, %xmm11, %xmm11, %xmm11
+
+#endif
+
+	// multiply with ALPHA_R
+        vmulps  %xmm8 , %xmm0, %xmm8
+        vmulps  %xmm10, %xmm0, %xmm10
+
+	// multiply with ALPHA_I
+        vmulps  %xmm9 , %xmm1, %xmm9
+        vmulps  %xmm11, %xmm1, %xmm11
+
+	vaddsubps %xmm9, %xmm8 , %xmm8
+        vaddsubps %xmm11,%xmm10, %xmm10
+
+
+
+#ifndef TRMMKERNEL
+
+	vaddps 	 	(CO1), %xmm8 , %xmm8
+
+	vaddps 	 	(CO1, LDC), %xmm10, %xmm10
+
+#endif
+
+	vmovups	%xmm8 ,  	(CO1)
+
+	vmovups	%xmm10 ,  	(CO1, LDC)
+
+
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
+        leaq    (BO, BI, SIZE), BO
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $2, KK
+#endif
+
+	addq	$4 * SIZE, CO1		# coffset += 4
+	ALIGN_4	
+
+
+
+/**************************************************************************/
+.L2_40:
+	testq	$1, M		
+	jz	.L2_60		// to next 2 lines of N
+
+	ALIGN_4
+
+.L2_41:
+
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+	leaq	BUFFER1, BO		// first buffer to BO
+	addq	$8 * SIZE, BO
+#else
+        movq    KK, %rax
+	leaq	BUFFER1, BO			// first buffer to BO
+	addq	$8 * SIZE, BO
+	movq    %rax, BI                        //  Index for BO
+        leaq    (,BI,4), BI                     //  BI = BI * 4 ; number of values
+        leaq    (BO, BI, SIZE), BO
+	salq	$1, %rax			// rax = rax * 2 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $1, %rax        // number of values in AO
+#else
+        addq    $2, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$-8, %rax			//  K = K - ( K % 8 )
+	je	.L2_46
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
+
+	salq	$1, %rax			// rax = rax * 2 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_42:
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL1x2_1(xxx)
+	KERNEL1x2_2(xxx)
+	KERNEL1x2_3(xxx)
+	KERNEL1x2_4(xxx)
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL1x2_1(xxx)
+	KERNEL1x2_2(xxx)
+	KERNEL1x2_3(xxx)
+	KERNEL1x2_4(xxx)
+
+	je	.L2_46
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL1x2_1(xxx)
+	KERNEL1x2_2(xxx)
+	KERNEL1x2_3(xxx)
+	KERNEL1x2_4(xxx)
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL1x2_1(xxx)
+	KERNEL1x2_2(xxx)
+	KERNEL1x2_3(xxx)
+	KERNEL1x2_4(xxx)
+
+	je	.L2_46
+
+	jmp	.L2_42
+	ALIGN_4
+
+.L2_46:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_49
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
+
+	salq	$1, %rax			// rax = rax * 2 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_47:
+
+	KERNEL1x2_SUB(xxx)
+	jl	.L2_47
+	ALIGN_4
+
+
+.L2_49:
+
+	vbroadcastss	ALPHA_R, %xmm0
+	vbroadcastss	ALPHA_I, %xmm1
+
+	// swap high and low 64 bytes
+        vshufps $0xb1, %xmm9 , %xmm9, %xmm9
+        vshufps $0xb1, %xmm11, %xmm11, %xmm11
+
+#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
+    defined(NR) || defined(NC) || defined(TR) || defined(TC)
+
+        vaddsubps %xmm9, %xmm8 , %xmm8
+        vaddsubps %xmm11,%xmm10, %xmm10
+
+        vshufps $0xb1, %xmm8 , %xmm8, %xmm9
+        vshufps $0xb1, %xmm10, %xmm10, %xmm11
+
+#else
+        vaddsubps %xmm8,  %xmm9 ,%xmm9
+        vaddsubps %xmm10, %xmm11,%xmm11
+
+        vmovaps   %xmm9,  %xmm8
+        vmovaps   %xmm11, %xmm10
+
+	// swap high and low 64 bytes
+        vshufps $0xb1, %xmm9 , %xmm9, %xmm9
+        vshufps $0xb1, %xmm11, %xmm11, %xmm11
+
+#endif
+
+	// multiply with ALPHA_R
+        vmulps  %xmm8 , %xmm0, %xmm8
+        vmulps  %xmm10, %xmm0, %xmm10
+
+	// multiply with ALPHA_I
+        vmulps  %xmm9 , %xmm1, %xmm9
+        vmulps  %xmm11, %xmm1, %xmm11
+
+	vaddsubps %xmm9, %xmm8 , %xmm8
+        vaddsubps %xmm11,%xmm10, %xmm10
+
+
+
+#ifndef TRMMKERNEL
+
+	vmovsd		(CO1), %xmm14
+	vaddps 	 	%xmm14, %xmm8 , %xmm8
+
+	vmovsd		(CO1, LDC), %xmm15
+	vaddps 	 	%xmm15, %xmm10, %xmm10
+
+#endif
+
+	vmovsd	%xmm8 ,  	(CO1)
+
+	vmovsd	%xmm10 ,  	(CO1, LDC)
+
+
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
+        leaq    (BO, BI, SIZE), BO
+	salq	$1, %rax			// rax = rax * 2 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $1, KK
+#endif
+
+	addq	$2 * SIZE, CO1		# coffset += 2
+	ALIGN_4	
+
+
+
+	
+.L2_60:
+#if defined(TRMMKERNEL) && !defined(LEFT)
+        addq    $2, KK
+#endif
+
+	decq	J			// j --
+	jg	.L2_01			// next 2 lines of N
+
+
+
+.L1_0:
+
+/************************************************************************************************
+* Loop for Nmod6 % 2 > 0
+*************************************************************************************************/
+
+	movq	Nmod6, J		
+	andq	$1, J			// j % 2
+	je	.L999
+	ALIGN_4
+
+.L1_01:
+	// copy to sub buffer
+	movq	B, BO1
+	leaq    BUFFER1, BO		// first buffer to BO
+	movq	K, %rax
+	ALIGN_4
+
+.L1_02b:
+
+	vmovsd		(BO1), %xmm0
+	vmovsd	%xmm0,       (BO)
+	addq	$2*SIZE,BO1
+	addq	$2*SIZE,BO
+	decq	%rax
+	jnz	.L1_02b
+
+.L1_02c:
+
+	movq	BO1, B			// next offset of B
+
+.L1_10:
+	movq	C, CO1
+	leaq	(C, LDC, 1), C		// c += 1 * ldc
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        movq    OFFSET, %rax
+        movq    %rax, KK
+#endif
+	
+	movq	A, AO		 	// aoffset = a
+	addq	$16 * SIZE, AO
+
+	movq	M,  I
+	sarq	$2, I			// i = (m >> 2)
+	je	.L1_20
+
+	ALIGN_4
+
+.L1_11:
+
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+	leaq	BUFFER1, BO		// first buffer to BO
+	addq	$4 * SIZE, BO
+#else
+        movq    KK, %rax
+	leaq	BUFFER1, BO			// first buffer to BO
+	addq	$4 * SIZE, BO
+	movq    %rax, BI                        //  Index for BO
+        leaq    (,BI,2), BI                     //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $4, %rax        // number of values in AO
+#else
+        addq    $1, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$-8, %rax			//  K = K - ( K % 8 )
+	je	.L1_16
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_12:
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL4x1_1(xxx)
+	KERNEL4x1_2(xxx)
+	KERNEL4x1_3(xxx)
+	KERNEL4x1_4(xxx)
+
+	KERNEL4x1_1(xxx)
+	KERNEL4x1_2(xxx)
+	KERNEL4x1_3(xxx)
+	KERNEL4x1_4(xxx)
+
+	je	.L1_16
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL4x1_1(xxx)
+	KERNEL4x1_2(xxx)
+	KERNEL4x1_3(xxx)
+	KERNEL4x1_4(xxx)
+
+	KERNEL4x1_1(xxx)
+	KERNEL4x1_2(xxx)
+	KERNEL4x1_3(xxx)
+	KERNEL4x1_4(xxx)
+
+	je	.L1_16
+
+	jmp	.L1_12
+	ALIGN_4
+
+.L1_16:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_19
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,2), BI                    //  BI = BI * 4 ; number of values
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_17:
+
+	KERNEL4x1_SUB(xxx)
+	jl	.L1_17
+	ALIGN_4
+
+
+.L1_19:
+
+	vbroadcastss	ALPHA_R, %xmm0
+	vbroadcastss	ALPHA_I, %xmm1
+
+	// swap high and low 64 bytes
+        vshufps $0xb1, %xmm9 , %xmm9, %xmm9
+        vshufps $0xb1, %xmm13, %xmm13, %xmm13
+
+#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
+    defined(NR) || defined(NC) || defined(TR) || defined(TC)
+
+        vaddsubps %xmm9, %xmm8 , %xmm8
+        vaddsubps %xmm13,%xmm12, %xmm12
+
+        vshufps $0xb1, %xmm8 , %xmm8, %xmm9
+        vshufps $0xb1, %xmm12, %xmm12, %xmm13
+
+#else
+        vaddsubps %xmm8,  %xmm9 ,%xmm9
+        vaddsubps %xmm12, %xmm13,%xmm13
+
+        vmovaps   %xmm9,  %xmm8
+        vmovaps   %xmm13, %xmm12
+
+	// swap high and low 64 bytes
+        vshufps $0xb1, %xmm9 , %xmm9, %xmm9
+        vshufps $0xb1, %xmm13, %xmm13, %xmm13
+
+#endif
+
+	// multiply with ALPHA_R
+        vmulps  %xmm8 , %xmm0, %xmm8
+        vmulps  %xmm12, %xmm0, %xmm12
+
+	// multiply with ALPHA_I
+        vmulps  %xmm9 , %xmm1, %xmm9
+        vmulps  %xmm13, %xmm1, %xmm13
+
+	vaddsubps %xmm9, %xmm8 , %xmm8
+        vaddsubps %xmm13,%xmm12, %xmm12
+
+
+
+#ifndef TRMMKERNEL
+
+	vaddps 	 	(CO1), %xmm8 , %xmm8
+	vaddps  4 * SIZE(CO1), %xmm12, %xmm12
+
+#endif
+
+	vmovups	%xmm8 ,  	(CO1)
+	vmovups	%xmm12 , 4 * SIZE(CO1)
+
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $4, KK
+#endif
+
+	addq	$8 * SIZE, CO1		# coffset += 8
+	decq	I			# i --
+	jg	.L1_11
+	ALIGN_4	
+
+
+
+/**************************************************************************
+* Rest of M 
+***************************************************************************/
+
+.L1_20:
+	testq	$3, M		
+	jz	.L999
+
+	testq	$2, M		
+	jz	.L1_40
+	ALIGN_4
+
+.L1_21:
+
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+	leaq	BUFFER1, BO		// first buffer to BO
+	addq	$4 * SIZE, BO
+#else
+        movq    KK, %rax
+	leaq	BUFFER1, BO			// first buffer to BO
+	addq	$4 * SIZE, BO
+	movq    %rax, BI                        //  Index for BO
+        leaq    (,BI,2), BI                     //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $2, %rax        // number of values in AO
+#else
+        addq    $1, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$-8, %rax			//  K = K - ( K % 8 )
+	je	.L1_26
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_22:
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL2x1_1(xxx)
+	KERNEL2x1_2(xxx)
+	KERNEL2x1_3(xxx)
+	KERNEL2x1_4(xxx)
+
+	KERNEL2x1_1(xxx)
+	KERNEL2x1_2(xxx)
+	KERNEL2x1_3(xxx)
+	KERNEL2x1_4(xxx)
+
+	je	.L1_26
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL2x1_1(xxx)
+	KERNEL2x1_2(xxx)
+	KERNEL2x1_3(xxx)
+	KERNEL2x1_4(xxx)
+
+	KERNEL2x1_1(xxx)
+	KERNEL2x1_2(xxx)
+	KERNEL2x1_3(xxx)
+	KERNEL2x1_4(xxx)
+
+	je	.L1_26
+
+	jmp	.L1_22
+	ALIGN_4
+
+.L1_26:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_29
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,2), BI                    //  BI = BI * 2; number of values
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_27:
+
+	KERNEL2x1_SUB(xxx)
+	jl	.L1_27
+	ALIGN_4
+
+
+.L1_29:
+
+	vbroadcastss	ALPHA_R, %xmm0
+	vbroadcastss	ALPHA_I, %xmm1
+
+	// swap high and low 64 bytes
+        vshufps $0xb1, %xmm9 , %xmm9, %xmm9
+
+#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
+    defined(NR) || defined(NC) || defined(TR) || defined(TC)
+
+        vaddsubps %xmm9, %xmm8 , %xmm8
+
+        vshufps $0xb1, %xmm8 , %xmm8, %xmm9
+
+#else
+        vaddsubps %xmm8,  %xmm9 ,%xmm9
+
+        vmovaps   %xmm9,  %xmm8
+
+	// swap high and low 64 bytes
+        vshufps $0xb1, %xmm9 , %xmm9, %xmm9
+
+#endif
+
+	// multiply with ALPHA_R
+        vmulps  %xmm8 , %xmm0, %xmm8
+
+	// multiply with ALPHA_I
+        vmulps  %xmm9 , %xmm1, %xmm9
+
+	vaddsubps %xmm9, %xmm8 , %xmm8
+
+
+
+#ifndef TRMMKERNEL
+
+	vaddps 	 	(CO1), %xmm8 , %xmm8
+
+#endif
+
+	vmovups	%xmm8 ,  	(CO1)
+
+
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $2, KK
+#endif
+
+	addq	$4 * SIZE, CO1		# coffset += 4
+	ALIGN_4	
+
+
+
+/**************************************************************************/
+.L1_40:
+	testq	$1, M		
+	jz	.L999		// to next 2 lines of N
+
+	ALIGN_4
+
+.L1_41:
+
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+	leaq	BUFFER1, BO		// first buffer to BO
+	addq	$4 * SIZE, BO
+#else
+        movq    KK, %rax
+	leaq	BUFFER1, BO			// first buffer to BO
+	addq	$4 * SIZE, BO
+	movq    %rax, BI                        //  Index for BO
+        leaq    (,BI,2), BI                     //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO
+	salq	$1, %rax			// rax = rax * 2 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $1, %rax        // number of values in AO
+#else
+        addq    $1, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$-8, %rax			//  K = K - ( K % 8 )
+	je	.L1_46
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values
+
+	salq	$1, %rax			// rax = rax * 2 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_42:
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL1x1_1(xxx)
+	KERNEL1x1_2(xxx)
+	KERNEL1x1_3(xxx)
+	KERNEL1x1_4(xxx)
+
+	KERNEL1x1_1(xxx)
+	KERNEL1x1_2(xxx)
+	KERNEL1x1_3(xxx)
+	KERNEL1x1_4(xxx)
+
+	je	.L1_46
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL1x1_1(xxx)
+	KERNEL1x1_2(xxx)
+	KERNEL1x1_3(xxx)
+	KERNEL1x1_4(xxx)
+
+	KERNEL1x1_1(xxx)
+	KERNEL1x1_2(xxx)
+	KERNEL1x1_3(xxx)
+	KERNEL1x1_4(xxx)
+
+	je	.L1_46
+
+	jmp	.L1_42
+	ALIGN_4
+
+.L1_46:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_49
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values
+
+	salq	$1, %rax			// rax = rax * 2 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_47:
+
+	KERNEL1x1_SUB(xxx)
+	jl	.L1_47
+	ALIGN_4
+
+
+.L1_49:
+
+	vbroadcastss	ALPHA_R, %xmm0
+	vbroadcastss	ALPHA_I, %xmm1
+
+	// swap high and low 64 bytes
+        vshufps $0xb1, %xmm9 , %xmm9, %xmm9
+
+#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
+    defined(NR) || defined(NC) || defined(TR) || defined(TC)
+
+        vaddsubps %xmm9, %xmm8 , %xmm8
+
+        vshufps $0xb1, %xmm8 , %xmm8, %xmm9
+
+#else
+        vaddsubps %xmm8,  %xmm9 ,%xmm9
+
+        vmovaps   %xmm9,  %xmm8
+
+	// swap high and low 64 bytes
+        vshufps $0xb1, %xmm9 , %xmm9, %xmm9
+
+#endif
+
+	// multiply with ALPHA_R
+        vmulps  %xmm8 , %xmm0, %xmm8
+
+	// multiply with ALPHA_I
+        vmulps  %xmm9 , %xmm1, %xmm9
+
+	vaddsubps %xmm9, %xmm8 , %xmm8
+
+
+
+#ifndef TRMMKERNEL
+
+	vmovsd		(CO1), %xmm14
+	vaddps 	 	%xmm14, %xmm8 , %xmm8
+
+#endif
+
+	vmovsd	%xmm8 ,  	(CO1)
+
+
+
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO
+	salq	$1, %rax			// rax = rax * 2 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $1, KK
+#endif
+
+	addq	$2 * SIZE, CO1		# coffset += 2
+	ALIGN_4	
+
+
+
+
+
+
+
+.L999:
+	vzeroupper
+
+	movq   		SP, %rsp
+	movq	   (%rsp), %rbx
+	movq	  8(%rsp), %rbp
+	movq	 16(%rsp), %r12
+	movq	 24(%rsp), %r13
+	movq	 32(%rsp), %r14
+	movq	 40(%rsp), %r15
+
+#ifdef WINDOWS_ABI
+	movq	 48(%rsp), %rdi
+	movq	 56(%rsp), %rsi
+	vmovups	 64(%rsp), %xmm6
+	vmovups	 80(%rsp), %xmm7
+	vmovups	 96(%rsp), %xmm8
+	vmovups	112(%rsp), %xmm9
+	vmovups	128(%rsp), %xmm10
+	vmovups	144(%rsp), %xmm11
+	vmovups	160(%rsp), %xmm12
+	vmovups	176(%rsp), %xmm13
+	vmovups	192(%rsp), %xmm14
+	vmovups	208(%rsp), %xmm15
+#endif
+
+	addq	$STACKSIZE, %rsp
+	ret
+
+	EPILOGUE
diff --git a/kernel/x86_64/cgemm_kernel_4x2_piledriver.S b/kernel/x86_64/cgemm_kernel_4x2_piledriver.S
index 72deee12f..bf7f91ee9 100644
--- a/kernel/x86_64/cgemm_kernel_4x2_piledriver.S
+++ b/kernel/x86_64/cgemm_kernel_4x2_piledriver.S
@@ -1,1921 +1,1921 @@
-/***************************************************************************
-Copyright (c) 2013, The OpenBLAS Project
-All rights reserved.
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in
-the documentation and/or other materials provided with the
-distribution.
-3. Neither the name of the OpenBLAS project nor the names of
-its contributors may be used to endorse or promote products
-derived from this software without specific prior written permission.
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
-USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*****************************************************************************/
-/*********************************************************************
-*
-* 2014/06/28 Saar
-*        BLASTEST               : OK
-*        CTEST                  : OK
-*        TEST                   : OK
-*
-*
-* 2013/10/31 Saar
-*
-* Parameter:
-*       UNROLL_M        4
-*       UNROLL_N        2
-*       CGEMM_P         768
-*       CGEMM_Q         168
-*       A_PR1           512
-*       B_PR1           256
-*
-* Performance at m x n on AMD 8320 (ACML-Version: 5.3.1):
-* 
-* 4608x4608    154.0    GFLOPS with 8 threads on 4 modules (ACML: 111.7 ) (BULLDOZER: 153.9 )
-* 4608x4608    148.3    GFLOPS with 4 threads on 4 modules (ACML:  96.0 ) (BULLDOZER: 143.2 )
-* 3456x3456     74.3    GFLOPS with 2 threads on 2 modules (ACML:  47.3 ) (BULLDOZER:  72.3 )
-* 3456x3456     37.3    GFLOPS with 1 threads on 1 modules (ACML:  24.2 ) (BULLDOZER:  36.5 )
-*
-* Performance at m x n on AMD 6380  (ACML-Version: 5.3.1):
-* 
-* 6912x6912    421.5    GFLOPS with 32 threads on 16 modules (ACML: 266.6 ) (BULLDOZER: 422.5 )
-* 6912x6912    407.0    GFLOPS with 16 threads on 16 modules (ACML: 271.5 ) (BULLDOZER: 404.7 )
-* 6912x6912    234.2    GFLOPS with  8 threads on  8 modules (ACML: 164.0 ) (BULLDOZER: 230.5 )
-* 4608x4608    123.1    GFLOPS with  4 threads on  4 modules (ACML:  87.9 ) (BULLDOZER: 120.9 )
-* 3456x3456     62.6    GFLOPS with  2 threads on  2 modules (ACML:  44.5 ) (BULLDOZER:  62.1 )
-* 3456x3456     31.8    GFLOPS with  1 threads on  1 modules (ACML:  22.6 ) (BULLDOZER:  31.4 )
-*
-*********************************************************************/
-
-
-
-#define ASSEMBLER
-#include "common.h"
- 
-#define OLD_M	%rdi
-#define OLD_N	%rsi
-#define M	%r13
-#define J	%r14
-#define OLD_K	%rdx
-
-#define A	%rcx
-#define B	%r8
-#define C	%r9
-#define LDC	%r10
-	
-#define I	%r11
-#define AO	%rdi
-#define BO	%rsi
-#define	CO1	%r15
-#define K	%r12
-#define BI	%rbp
-#define	SP	%rbx
-
-#define BO1	%rdi
-#define BO2	%r15
-
-#ifndef WINDOWS_ABI
-
-#define STACKSIZE 96
-
-#else
-
-#define STACKSIZE 320
-
-#define OLD_ALPHA_I     40 + STACKSIZE(%rsp)
-#define OLD_A           48 + STACKSIZE(%rsp)
-#define OLD_B           56 + STACKSIZE(%rsp)
-#define OLD_C           64 + STACKSIZE(%rsp)
-#define OLD_LDC         72 + STACKSIZE(%rsp)
-#define OLD_OFFSET      80 + STACKSIZE(%rsp)
-
-#endif
-
-#define L_BUFFER_SIZE 256*8*4
-
-#define Ndiv6	 24(%rsp)
-#define Nmod6	 32(%rsp)
-#define N	 40(%rsp)
-#define ALPHA_R  48(%rsp)
-#define ALPHA_I  56(%rsp)
-#define OFFSET   64(%rsp)
-#define KK       72(%rsp)
-#define KKK      80(%rsp)
-#define BUFFER1	           128(%rsp)
-
-#if defined(OS_WINDOWS)
-#if   L_BUFFER_SIZE > 16384
-#define STACK_TOUCH \
-        movl    $0,  4096 * 4(%rsp);\
-        movl    $0,  4096 * 3(%rsp);\
-        movl    $0,  4096 * 2(%rsp);\
-        movl    $0,  4096 * 1(%rsp);
-#elif L_BUFFER_SIZE > 12288
-#define STACK_TOUCH \
-        movl    $0,  4096 * 3(%rsp);\
-        movl    $0,  4096 * 2(%rsp);\
-        movl    $0,  4096 * 1(%rsp);
-#elif L_BUFFER_SIZE > 8192
-#define STACK_TOUCH \
-        movl    $0,  4096 * 2(%rsp);\
-        movl    $0,  4096 * 1(%rsp);
-#elif L_BUFFER_SIZE > 4096
-#define STACK_TOUCH \
-        movl    $0,  4096 * 1(%rsp);
-#else
-#define STACK_TOUCH
-#endif
-#else
-#define STACK_TOUCH
-#endif
-
-
-#if   defined(NN) || defined(NT) || defined(TN) || defined(TT)
-#define VFMADD_R    vfmaddps
-#define VFMADD_I    vfmaddps
-#elif defined(RN) || defined(RT) || defined(CN) || defined(CT)
-#define VFMADD_R    vfnmaddps
-#define VFMADD_I    vfmaddps
-#elif defined(NR) || defined(NC) || defined(TR) || defined(TC)
-#define VFMADD_R    vfmaddps
-#define VFMADD_I    vfnmaddps
-#else
-#define VFMADD_R    vfnmaddps
-#define VFMADD_I    vfnmaddps
-#endif
-
-
-
-#define	A_PR1	512
-#define	B_PR1	256
-
-#define KERNEL4x2_1(xx) \
-        prefetcht0      A_PR1(AO,%rax,SIZE)        ;\
-        vmovups         -16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss         -8 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        vmovups         -12 * SIZE(AO, %rax, SIZE), %xmm1 ;\
-        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
-        vbroadcastss         -7 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-        VFMADD_I        %xmm13,%xmm5,%xmm1,%xmm13 ;\
-        vbroadcastss         -6 * SIZE(BO, BI, SIZE), %xmm6 ;\
-        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
-        VFMADD_R        %xmm14,%xmm6,%xmm1,%xmm14 ;\
-        vbroadcastss         -5 * SIZE(BO, BI, SIZE), %xmm7 ;\
-        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
-        VFMADD_I        %xmm15,%xmm7,%xmm1,%xmm15 ;\
-
-#define KERNEL4x2_2(xx) \
-        vmovups          -8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss         -4 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        vmovups          -4 * SIZE(AO, %rax, SIZE), %xmm1 ;\
-        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
-        vbroadcastss         -3 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-        VFMADD_I        %xmm13,%xmm5,%xmm1,%xmm13 ;\
-        vbroadcastss         -2 * SIZE(BO, BI, SIZE), %xmm6 ;\
-        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
-        VFMADD_R        %xmm14,%xmm6,%xmm1,%xmm14 ;\
-        vbroadcastss         -1 * SIZE(BO, BI, SIZE), %xmm7 ;\
-        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
-        VFMADD_I        %xmm15,%xmm7,%xmm1,%xmm15 ;\
-
-#define KERNEL4x2_3(xx) \
-	prefetcht0      A_PR1+64(AO,%rax,SIZE)     ;\
-        vmovups           0 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss          0 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        vmovups           4 * SIZE(AO, %rax, SIZE), %xmm1 ;\
-        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
-        vbroadcastss          1 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-        VFMADD_I        %xmm13,%xmm5,%xmm1,%xmm13 ;\
-        vbroadcastss          2 * SIZE(BO, BI, SIZE), %xmm6 ;\
-        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
-        VFMADD_R        %xmm14,%xmm6,%xmm1,%xmm14 ;\
-        vbroadcastss          3 * SIZE(BO, BI, SIZE), %xmm7 ;\
-        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
-        VFMADD_I        %xmm15,%xmm7,%xmm1,%xmm15 ;\
-
-#define KERNEL4x2_4(xx) \
-        vmovups           8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss          4 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        vmovups          12 * SIZE(AO, %rax, SIZE), %xmm1 ;\
-        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
-        vbroadcastss          5 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-        VFMADD_I        %xmm13,%xmm5,%xmm1,%xmm13 ;\
-        vbroadcastss          6 * SIZE(BO, BI, SIZE), %xmm6 ;\
-        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
-        VFMADD_R        %xmm14,%xmm6,%xmm1,%xmm14 ;\
-        vbroadcastss          7 * SIZE(BO, BI, SIZE), %xmm7 ;\
-        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
-        VFMADD_I        %xmm15,%xmm7,%xmm1,%xmm15 ;\
-        addq    $16, BI                            ;\
-        addq    $32, %rax                          ;\
-
-
-#define KERNEL4x2_SUB(xx) \
-        vmovups         -16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss         -8 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        vmovups         -12 * SIZE(AO, %rax, SIZE), %xmm1 ;\
-        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
-        vbroadcastss         -7 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-        VFMADD_I        %xmm13,%xmm5,%xmm1,%xmm13 ;\
-        vbroadcastss         -6 * SIZE(BO, BI, SIZE), %xmm6 ;\
-        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
-        VFMADD_R        %xmm14,%xmm6,%xmm1,%xmm14 ;\
-        vbroadcastss         -5 * SIZE(BO, BI, SIZE), %xmm7 ;\
-        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
-        VFMADD_I        %xmm15,%xmm7,%xmm1,%xmm15 ;\
-        addq    $4, BI                            ;\
-        addq    $8, %rax                          ;\
-
-/************************************************************************************************/
-
-#define KERNEL2x2_1(xx) \
-        prefetcht0      A_PR1(AO,%rax,SIZE)        ;\
-        vmovups         -16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss         -8 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        vbroadcastss         -7 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-        vbroadcastss         -6 * SIZE(BO, BI, SIZE), %xmm6 ;\
-        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
-        vbroadcastss         -5 * SIZE(BO, BI, SIZE), %xmm7 ;\
-        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
-
-#define KERNEL2x2_2(xx) \
-        vmovups         -12 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss         -4 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
-        vbroadcastss         -3 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-        vbroadcastss         -2 * SIZE(BO, BI, SIZE), %xmm6 ;\
-        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
-        vbroadcastss         -1 * SIZE(BO, BI, SIZE), %xmm7 ;\
-        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
-
-#define KERNEL2x2_3(xx) \
-        vmovups          -8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss          0 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        vbroadcastss          1 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-        vbroadcastss          2 * SIZE(BO, BI, SIZE), %xmm6 ;\
-        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
-        vbroadcastss          3 * SIZE(BO, BI, SIZE), %xmm7 ;\
-        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
-
-#define KERNEL2x2_4(xx) \
-        vmovups          -4 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss          4 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        vbroadcastss          5 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-        vbroadcastss          6 * SIZE(BO, BI, SIZE), %xmm6 ;\
-        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
-        vbroadcastss          7 * SIZE(BO, BI, SIZE), %xmm7 ;\
-        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
-        addq    $16, BI                            ;\
-        addq    $16, %rax                          ;\
-
-
-#define KERNEL2x2_SUB(xx) \
-        vmovups         -16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss         -8 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        vbroadcastss         -7 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-        vbroadcastss         -6 * SIZE(BO, BI, SIZE), %xmm6 ;\
-        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
-        vbroadcastss         -5 * SIZE(BO, BI, SIZE), %xmm7 ;\
-        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
-        addq    $4, BI                            ;\
-        addq    $4, %rax                          ;\
-
-/************************************************************************************************/
-
-#define KERNEL1x2_1(xx) \
-        vmovsd          -16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss         -8 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        vbroadcastss         -7 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-        vbroadcastss         -6 * SIZE(BO, BI, SIZE), %xmm6 ;\
-        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
-        vbroadcastss         -5 * SIZE(BO, BI, SIZE), %xmm7 ;\
-        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
-
-#define KERNEL1x2_2(xx) \
-        vmovsd          -14 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss         -4 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
-        vbroadcastss         -3 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-        vbroadcastss         -2 * SIZE(BO, BI, SIZE), %xmm6 ;\
-        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
-        vbroadcastss         -1 * SIZE(BO, BI, SIZE), %xmm7 ;\
-        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
-
-#define KERNEL1x2_3(xx) \
-        vmovsd          -12 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss          0 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        vbroadcastss          1 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-        vbroadcastss          2 * SIZE(BO, BI, SIZE), %xmm6 ;\
-        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
-        vbroadcastss          3 * SIZE(BO, BI, SIZE), %xmm7 ;\
-        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
-
-#define KERNEL1x2_4(xx) \
-        vmovsd          -10 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss          4 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        vbroadcastss          5 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-        vbroadcastss          6 * SIZE(BO, BI, SIZE), %xmm6 ;\
-        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
-        vbroadcastss          7 * SIZE(BO, BI, SIZE), %xmm7 ;\
-        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
-        addq    $16, BI                            ;\
-        addq    $8, %rax                          ;\
-
-
-#define KERNEL1x2_SUB(xx) \
-        vmovsd         -16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss         -8 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        vbroadcastss         -7 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-        vbroadcastss         -6 * SIZE(BO, BI, SIZE), %xmm6 ;\
-        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
-        vbroadcastss         -5 * SIZE(BO, BI, SIZE), %xmm7 ;\
-        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
-        addq    $4, BI                            ;\
-        addq    $2, %rax                          ;\
-
-
-
-/************************************************************************************************/
-
-#define KERNEL4x1_1(xx) \
-        prefetcht0      A_PR1(AO,%rax,SIZE)        ;\
-        vmovups         -16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss         -4 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        vmovups         -12 * SIZE(AO, %rax, SIZE), %xmm1 ;\
-        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
-        vbroadcastss         -3 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-        VFMADD_I        %xmm13,%xmm5,%xmm1,%xmm13 ;\
-
-#define KERNEL4x1_2(xx) \
-        vmovups          -8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss         -2 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        vmovups          -4 * SIZE(AO, %rax, SIZE), %xmm1 ;\
-        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
-        vbroadcastss         -1 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-        VFMADD_I        %xmm13,%xmm5,%xmm1,%xmm13 ;\
-
-#define KERNEL4x1_3(xx) \
-	prefetcht0      A_PR1+64(AO,%rax,SIZE)     ;\
-        vmovups           0 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss          0 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        vmovups           4 * SIZE(AO, %rax, SIZE), %xmm1 ;\
-        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
-        vbroadcastss          1 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-        VFMADD_I        %xmm13,%xmm5,%xmm1,%xmm13 ;\
-
-#define KERNEL4x1_4(xx) \
-        vmovups           8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss          2 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        vmovups          12 * SIZE(AO, %rax, SIZE), %xmm1 ;\
-        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
-        vbroadcastss          3 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-        VFMADD_I        %xmm13,%xmm5,%xmm1,%xmm13 ;\
-        addq    $8, BI                            ;\
-        addq    $32, %rax                          ;\
-
-
-#define KERNEL4x1_SUB(xx) \
-        vmovups         -16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss         -4 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        vmovups         -12 * SIZE(AO, %rax, SIZE), %xmm1 ;\
-        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
-        vbroadcastss         -3 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-        VFMADD_I        %xmm13,%xmm5,%xmm1,%xmm13 ;\
-        addq    $2, BI                            ;\
-        addq    $8, %rax                          ;\
-
-
-/************************************************************************************************/
-
-#define KERNEL2x1_1(xx) \
-        prefetcht0      A_PR1(AO,%rax,SIZE)        ;\
-        vmovups         -16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss         -4 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        vbroadcastss         -3 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-
-#define KERNEL2x1_2(xx) \
-        vmovups         -12 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss         -2 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        vbroadcastss         -1 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-
-#define KERNEL2x1_3(xx) \
-        vmovups          -8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss          0 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        vbroadcastss          1 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-
-#define KERNEL2x1_4(xx) \
-        vmovups          -4 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss          2 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        vbroadcastss          3 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-        addq    $8, BI                            ;\
-        addq    $16, %rax                          ;\
-
-
-#define KERNEL2x1_SUB(xx) \
-        vmovups         -16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss         -4 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        vbroadcastss         -3 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-        addq    $2, BI                            ;\
-        addq    $4, %rax                          ;\
-
-
-/************************************************************************************************/
-
-#define KERNEL1x1_1(xx) \
-        vmovsd         -16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss         -4 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        vbroadcastss         -3 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-
-#define KERNEL1x1_2(xx) \
-        vmovsd         -14 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss         -2 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        vbroadcastss         -1 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-
-#define KERNEL1x1_3(xx) \
-        vmovsd         -12 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss          0 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        vbroadcastss          1 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-
-#define KERNEL1x1_4(xx) \
-        vmovsd          -10 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss          2 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        vbroadcastss          3 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-        addq    $8, BI                            ;\
-        addq    $8, %rax                          ;\
-
-
-#define KERNEL1x1_SUB(xx) \
-        vmovsd         -16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-        vbroadcastss         -4 * SIZE(BO, BI, SIZE), %xmm4 ;\
-        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
-        vbroadcastss         -3 * SIZE(BO, BI, SIZE), %xmm5 ;\
-        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
-        addq    $2, BI                            ;\
-        addq    $2, %rax                          ;\
-
-
-/************************************************************************************************/
-
-
-
-
-	PROLOGUE
-	PROFCODE
-	
-	subq	$STACKSIZE, %rsp
-	movq	%rbx,   (%rsp)
-	movq	%rbp,  8(%rsp)
-	movq	%r12, 16(%rsp)
-	movq	%r13, 24(%rsp)
-	movq	%r14, 32(%rsp)
-	movq	%r15, 40(%rsp)
-
-	vzeroupper
-
-#ifdef WINDOWS_ABI
-	movq	%rdi,    48(%rsp)
-	movq	%rsi,    56(%rsp)
-	vmovups	%xmm6,   64(%rsp)
-	vmovups	%xmm7,   80(%rsp)
-	vmovups	%xmm8,   96(%rsp)
-	vmovups	%xmm9,  112(%rsp)
-	vmovups	%xmm10, 128(%rsp)
-	vmovups	%xmm11, 144(%rsp)
-	vmovups	%xmm12, 160(%rsp)
-	vmovups	%xmm13, 176(%rsp)
-	vmovups	%xmm14, 192(%rsp)
-	vmovups	%xmm15, 208(%rsp)
-
-	movq	ARG1,      OLD_M
-	movq	ARG2,      OLD_N
-	movq	ARG3,      OLD_K
-	movq	OLD_A,     A
-	movq	OLD_B,     B
-	movq	OLD_C,     C
-	movq	OLD_LDC,   LDC
-#ifdef TRMMKERNEL
-	movsd	OLD_OFFSET, %xmm12
-#endif
-	vmovaps	%xmm3, %xmm0
-	vmovsd   OLD_ALPHA_I, %xmm1
-
-#else
-	movq	STACKSIZE +  8(%rsp), LDC
-#ifdef TRMMKERNEL
-	movsd	STACKSIZE + 16(%rsp), %xmm12
-#endif
-
-#endif
-
-	movq    %rsp, SP      # save old stack
-        subq    $128 + L_BUFFER_SIZE, %rsp
-        andq    $-4096, %rsp    # align stack
-
-        STACK_TOUCH
-
-	cmpq	$0, OLD_M
-	je	.L999
-
-	cmpq	$0, OLD_N
-	je	.L999
-
-	cmpq	$0, OLD_K
-	je	.L999
-
-	movq	OLD_M, M
-	movq	OLD_N, N
-	movq	OLD_K, K
-
-	vmovss	 %xmm0, ALPHA_R
-	vmovss	 %xmm1, ALPHA_I
-
-	salq	$ZBASE_SHIFT, LDC
-
-	movq    N, %rax
-        xorq    %rdx, %rdx
-        movq    $2,  %rdi
-        divq    %rdi                    //    N / 2
-        movq    %rax, Ndiv6             //    N / 2
-        movq    %rdx, Nmod6             //    N % 2
-
-	
-
-#ifdef TRMMKERNEL
-	vmovsd	%xmm12, OFFSET
-	vmovsd	%xmm12, KK
-#ifndef LEFT
-	negq	KK
-#endif	
-#endif
-
-.L2_0:
-
-	movq	Ndiv6,  J
-	cmpq	$0, J
-	je	.L1_0
-	ALIGN_4
-
-
-
-.L2_01:
-	// copy to sub buffer
-	movq	B, BO1
-	leaq    BUFFER1, BO		// first buffer to BO
-	movq	K, %rax
-	ALIGN_4
-
-.L2_02b:
-
-	vmovups	(BO1), %xmm0
-	vmovups	%xmm0,       (BO)
-	addq	$4*SIZE,BO1
-	addq	$4*SIZE,BO
-	decq	%rax
-	jnz	.L2_02b
-
-.L2_02c:
-
-	movq	BO1, B			// next offset of B
-
-.L2_10:
-	movq	C, CO1
-	leaq	(C, LDC, 2), C		// c += 2 * ldc
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        movq    OFFSET, %rax
-        movq    %rax, KK
-#endif
-	
-	movq	A, AO		 	// aoffset = a
-	addq	$16 * SIZE, AO
-
-	movq	M,  I
-	sarq	$2, I			// i = (m >> 2)
-	je	.L2_20
-
-	ALIGN_4
-
-.L2_11:
-
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-	leaq	BUFFER1, BO		// first buffer to BO
-	addq	$8 * SIZE, BO
-#else
-        movq    KK, %rax
-	leaq	BUFFER1, BO			// first buffer to BO
-	addq	$8 * SIZE, BO
-	movq    %rax, BI                        //  Index for BO
-        leaq    (,BI,4), BI                     //  BI = BI * 4 ; number of values
-        leaq    (BO, BI, SIZE), BO
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $4, %rax        // number of values in AO
-#else
-        addq    $2, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-
-	andq	$-8, %rax			//  K = K - ( K % 8 )
-	je	.L2_16
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_12:
-
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL4x2_1(xxx)
-	KERNEL4x2_2(xxx)
-	KERNEL4x2_3(xxx)
-	KERNEL4x2_4(xxx)
-
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL4x2_1(xxx)
-	KERNEL4x2_2(xxx)
-	KERNEL4x2_3(xxx)
-	KERNEL4x2_4(xxx)
-
-	je	.L2_16
-
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL4x2_1(xxx)
-	KERNEL4x2_2(xxx)
-	KERNEL4x2_3(xxx)
-	KERNEL4x2_4(xxx)
-
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL4x2_1(xxx)
-	KERNEL4x2_2(xxx)
-	KERNEL4x2_3(xxx)
-	KERNEL4x2_4(xxx)
-
-	je	.L2_16
-
-	jmp	.L2_12
-	ALIGN_4
-
-.L2_16:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_19
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_17:
-
-	KERNEL4x2_SUB(xxx)
-	jl	.L2_17
-	ALIGN_4
-
-
-.L2_19:
-
-	vbroadcastss	ALPHA_R, %xmm0
-	vbroadcastss	ALPHA_I, %xmm1
-
-	// swap high and low 64 bytes
-        vshufps $0xb1, %xmm9 , %xmm9, %xmm9
-        vshufps $0xb1, %xmm11, %xmm11, %xmm11
-        vshufps $0xb1, %xmm13, %xmm13, %xmm13
-        vshufps $0xb1, %xmm15, %xmm15, %xmm15
-
-#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
-    defined(NR) || defined(NC) || defined(TR) || defined(TC)
-
-        vaddsubps %xmm9, %xmm8 , %xmm8
-        vaddsubps %xmm11,%xmm10, %xmm10
-        vaddsubps %xmm13,%xmm12, %xmm12
-        vaddsubps %xmm15,%xmm14, %xmm14
-
-        vshufps $0xb1, %xmm8 , %xmm8, %xmm9
-        vshufps $0xb1, %xmm10, %xmm10, %xmm11
-        vshufps $0xb1, %xmm12, %xmm12, %xmm13
-        vshufps $0xb1, %xmm14, %xmm14, %xmm15
-
-#else
-        vaddsubps %xmm8,  %xmm9 ,%xmm9
-        vaddsubps %xmm10, %xmm11,%xmm11
-        vaddsubps %xmm12, %xmm13,%xmm13
-        vaddsubps %xmm14, %xmm15,%xmm15
-
-        vmovaps   %xmm9,  %xmm8
-        vmovaps   %xmm11, %xmm10
-        vmovaps   %xmm13, %xmm12
-        vmovaps   %xmm15, %xmm14
-
-	// swap high and low 64 bytes
-        vshufps $0xb1, %xmm9 , %xmm9, %xmm9
-        vshufps $0xb1, %xmm11, %xmm11, %xmm11
-        vshufps $0xb1, %xmm13, %xmm13, %xmm13
-        vshufps $0xb1, %xmm15, %xmm15, %xmm15
-
-#endif
-
-	// multiply with ALPHA_R
-        vmulps  %xmm8 , %xmm0, %xmm8
-        vmulps  %xmm10, %xmm0, %xmm10
-        vmulps  %xmm12, %xmm0, %xmm12
-        vmulps  %xmm14, %xmm0, %xmm14
-
-	// multiply with ALPHA_I
-        vmulps  %xmm9 , %xmm1, %xmm9
-        vmulps  %xmm11, %xmm1, %xmm11
-        vmulps  %xmm13, %xmm1, %xmm13
-        vmulps  %xmm15, %xmm1, %xmm15
-
-	vaddsubps %xmm9, %xmm8 , %xmm8
-        vaddsubps %xmm11,%xmm10, %xmm10
-        vaddsubps %xmm13,%xmm12, %xmm12
-        vaddsubps %xmm15,%xmm14, %xmm14
-
-
-
-#ifndef TRMMKERNEL
-
-	vaddps 	 	(CO1), %xmm8 , %xmm8
-	vaddps  4 * SIZE(CO1), %xmm12, %xmm12
-
-	vaddps 	 	(CO1, LDC), %xmm10, %xmm10
-	vaddps  4 * SIZE(CO1, LDC), %xmm14, %xmm14
-
-#endif
-
-	vmovups	%xmm8 ,  	(CO1)
-	vmovups	%xmm12 , 4 * SIZE(CO1)
-
-	vmovups	%xmm10 ,  	(CO1, LDC)
-	vmovups	%xmm14 , 4 * SIZE(CO1, LDC)
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
-        leaq    (BO, BI, SIZE), BO
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $4, KK
-#endif
-
-	addq	$8 * SIZE, CO1		# coffset += 8
-	decq	I			# i --
-	jg	.L2_11
-	ALIGN_4	
-
-
-
-/**************************************************************************
-* Rest of M 
-***************************************************************************/
-
-.L2_20:
-	testq	$3, M		
-	jz	.L2_60		// to next 2 lines of N
-
-	testq	$2, M		
-	jz	.L2_40
-	ALIGN_4
-
-.L2_21:
-
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-	leaq	BUFFER1, BO		// first buffer to BO
-	addq	$8 * SIZE, BO
-#else
-        movq    KK, %rax
-	leaq	BUFFER1, BO			// first buffer to BO
-	addq	$8 * SIZE, BO
-	movq    %rax, BI                        //  Index for BO
-        leaq    (,BI,4), BI                     //  BI = BI * 4 ; number of values
-        leaq    (BO, BI, SIZE), BO
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $2, %rax        // number of values in AO
-#else
-        addq    $2, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-
-	andq	$-8, %rax			//  K = K - ( K % 8 )
-	je	.L2_26
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_22:
-
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL2x2_1(xxx)
-	KERNEL2x2_2(xxx)
-	KERNEL2x2_3(xxx)
-	KERNEL2x2_4(xxx)
-
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL2x2_1(xxx)
-	KERNEL2x2_2(xxx)
-	KERNEL2x2_3(xxx)
-	KERNEL2x2_4(xxx)
-
-	je	.L2_26
-
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL2x2_1(xxx)
-	KERNEL2x2_2(xxx)
-	KERNEL2x2_3(xxx)
-	KERNEL2x2_4(xxx)
-
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL2x2_1(xxx)
-	KERNEL2x2_2(xxx)
-	KERNEL2x2_3(xxx)
-	KERNEL2x2_4(xxx)
-
-	je	.L2_26
-
-	jmp	.L2_22
-	ALIGN_4
-
-.L2_26:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_29
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_27:
-
-	KERNEL2x2_SUB(xxx)
-	jl	.L2_27
-	ALIGN_4
-
-
-.L2_29:
-
-	vbroadcastss	ALPHA_R, %xmm0
-	vbroadcastss	ALPHA_I, %xmm1
-
-	// swap high and low 64 bytes
-        vshufps $0xb1, %xmm9 , %xmm9, %xmm9
-        vshufps $0xb1, %xmm11, %xmm11, %xmm11
-
-#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
-    defined(NR) || defined(NC) || defined(TR) || defined(TC)
-
-        vaddsubps %xmm9, %xmm8 , %xmm8
-        vaddsubps %xmm11,%xmm10, %xmm10
-
-        vshufps $0xb1, %xmm8 , %xmm8, %xmm9
-        vshufps $0xb1, %xmm10, %xmm10, %xmm11
-
-#else
-        vaddsubps %xmm8,  %xmm9 ,%xmm9
-        vaddsubps %xmm10, %xmm11,%xmm11
-
-        vmovaps   %xmm9,  %xmm8
-        vmovaps   %xmm11, %xmm10
-
-	// swap high and low 64 bytes
-        vshufps $0xb1, %xmm9 , %xmm9, %xmm9
-        vshufps $0xb1, %xmm11, %xmm11, %xmm11
-
-#endif
-
-	// multiply with ALPHA_R
-        vmulps  %xmm8 , %xmm0, %xmm8
-        vmulps  %xmm10, %xmm0, %xmm10
-
-	// multiply with ALPHA_I
-        vmulps  %xmm9 , %xmm1, %xmm9
-        vmulps  %xmm11, %xmm1, %xmm11
-
-	vaddsubps %xmm9, %xmm8 , %xmm8
-        vaddsubps %xmm11,%xmm10, %xmm10
-
-
-
-#ifndef TRMMKERNEL
-
-	vaddps 	 	(CO1), %xmm8 , %xmm8
-
-	vaddps 	 	(CO1, LDC), %xmm10, %xmm10
-
-#endif
-
-	vmovups	%xmm8 ,  	(CO1)
-
-	vmovups	%xmm10 ,  	(CO1, LDC)
-
-
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
-        leaq    (BO, BI, SIZE), BO
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $2, KK
-#endif
-
-	addq	$4 * SIZE, CO1		# coffset += 4
-	ALIGN_4	
-
-
-
-/**************************************************************************/
-.L2_40:
-	testq	$1, M		
-	jz	.L2_60		// to next 2 lines of N
-
-	ALIGN_4
-
-.L2_41:
-
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-	leaq	BUFFER1, BO		// first buffer to BO
-	addq	$8 * SIZE, BO
-#else
-        movq    KK, %rax
-	leaq	BUFFER1, BO			// first buffer to BO
-	addq	$8 * SIZE, BO
-	movq    %rax, BI                        //  Index for BO
-        leaq    (,BI,4), BI                     //  BI = BI * 4 ; number of values
-        leaq    (BO, BI, SIZE), BO
-	salq	$1, %rax			// rax = rax * 2 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $1, %rax        // number of values in AO
-#else
-        addq    $2, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-
-	andq	$-8, %rax			//  K = K - ( K % 8 )
-	je	.L2_46
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
-
-	salq	$1, %rax			// rax = rax * 2 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_42:
-
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL1x2_1(xxx)
-	KERNEL1x2_2(xxx)
-	KERNEL1x2_3(xxx)
-	KERNEL1x2_4(xxx)
-
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL1x2_1(xxx)
-	KERNEL1x2_2(xxx)
-	KERNEL1x2_3(xxx)
-	KERNEL1x2_4(xxx)
-
-	je	.L2_46
-
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL1x2_1(xxx)
-	KERNEL1x2_2(xxx)
-	KERNEL1x2_3(xxx)
-	KERNEL1x2_4(xxx)
-
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL1x2_1(xxx)
-	KERNEL1x2_2(xxx)
-	KERNEL1x2_3(xxx)
-	KERNEL1x2_4(xxx)
-
-	je	.L2_46
-
-	jmp	.L2_42
-	ALIGN_4
-
-.L2_46:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_49
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
-
-	salq	$1, %rax			// rax = rax * 2 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_47:
-
-	KERNEL1x2_SUB(xxx)
-	jl	.L2_47
-	ALIGN_4
-
-
-.L2_49:
-
-	vbroadcastss	ALPHA_R, %xmm0
-	vbroadcastss	ALPHA_I, %xmm1
-
-	// swap high and low 64 bytes
-        vshufps $0xb1, %xmm9 , %xmm9, %xmm9
-        vshufps $0xb1, %xmm11, %xmm11, %xmm11
-
-#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
-    defined(NR) || defined(NC) || defined(TR) || defined(TC)
-
-        vaddsubps %xmm9, %xmm8 , %xmm8
-        vaddsubps %xmm11,%xmm10, %xmm10
-
-        vshufps $0xb1, %xmm8 , %xmm8, %xmm9
-        vshufps $0xb1, %xmm10, %xmm10, %xmm11
-
-#else
-        vaddsubps %xmm8,  %xmm9 ,%xmm9
-        vaddsubps %xmm10, %xmm11,%xmm11
-
-        vmovaps   %xmm9,  %xmm8
-        vmovaps   %xmm11, %xmm10
-
-	// swap high and low 64 bytes
-        vshufps $0xb1, %xmm9 , %xmm9, %xmm9
-        vshufps $0xb1, %xmm11, %xmm11, %xmm11
-
-#endif
-
-	// multiply with ALPHA_R
-        vmulps  %xmm8 , %xmm0, %xmm8
-        vmulps  %xmm10, %xmm0, %xmm10
-
-	// multiply with ALPHA_I
-        vmulps  %xmm9 , %xmm1, %xmm9
-        vmulps  %xmm11, %xmm1, %xmm11
-
-	vaddsubps %xmm9, %xmm8 , %xmm8
-        vaddsubps %xmm11,%xmm10, %xmm10
-
-
-
-#ifndef TRMMKERNEL
-
-	vmovsd		(CO1), %xmm14
-	vaddps 	 	%xmm14, %xmm8 , %xmm8
-
-	vmovsd		(CO1, LDC), %xmm15
-	vaddps 	 	%xmm15, %xmm10, %xmm10
-
-#endif
-
-	vmovsd	%xmm8 ,  	(CO1)
-
-	vmovsd	%xmm10 ,  	(CO1, LDC)
-
-
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
-        leaq    (BO, BI, SIZE), BO
-	salq	$1, %rax			// rax = rax * 2 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $1, KK
-#endif
-
-	addq	$2 * SIZE, CO1		# coffset += 2
-	ALIGN_4	
-
-
-
-	
-.L2_60:
-#if defined(TRMMKERNEL) && !defined(LEFT)
-        addq    $2, KK
-#endif
-
-	decq	J			// j --
-	jg	.L2_01			// next 2 lines of N
-
-
-
-.L1_0:
-
-/************************************************************************************************
-* Loop for Nmod6 % 2 > 0
-*************************************************************************************************/
-
-	movq	Nmod6, J		
-	andq	$1, J			// j % 2
-	je	.L999
-	ALIGN_4
-
-.L1_01:
-	// copy to sub buffer
-	movq	B, BO1
-	leaq    BUFFER1, BO		// first buffer to BO
-	movq	K, %rax
-	ALIGN_4
-
-.L1_02b:
-
-	vmovsd		(BO1), %xmm0
-	vmovsd	%xmm0,       (BO)
-	addq	$2*SIZE,BO1
-	addq	$2*SIZE,BO
-	decq	%rax
-	jnz	.L1_02b
-
-.L1_02c:
-
-	movq	BO1, B			// next offset of B
-
-.L1_10:
-	movq	C, CO1
-	leaq	(C, LDC, 1), C		// c += 1 * ldc
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        movq    OFFSET, %rax
-        movq    %rax, KK
-#endif
-	
-	movq	A, AO		 	// aoffset = a
-	addq	$16 * SIZE, AO
-
-	movq	M,  I
-	sarq	$2, I			// i = (m >> 2)
-	je	.L1_20
-
-	ALIGN_4
-
-.L1_11:
-
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-	leaq	BUFFER1, BO		// first buffer to BO
-	addq	$4 * SIZE, BO
-#else
-        movq    KK, %rax
-	leaq	BUFFER1, BO			// first buffer to BO
-	addq	$4 * SIZE, BO
-	movq    %rax, BI                        //  Index for BO
-        leaq    (,BI,2), BI                     //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $4, %rax        // number of values in AO
-#else
-        addq    $1, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-
-	andq	$-8, %rax			//  K = K - ( K % 8 )
-	je	.L1_16
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_12:
-
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL4x1_1(xxx)
-	KERNEL4x1_2(xxx)
-	KERNEL4x1_3(xxx)
-	KERNEL4x1_4(xxx)
-
-	KERNEL4x1_1(xxx)
-	KERNEL4x1_2(xxx)
-	KERNEL4x1_3(xxx)
-	KERNEL4x1_4(xxx)
-
-	je	.L1_16
-
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL4x1_1(xxx)
-	KERNEL4x1_2(xxx)
-	KERNEL4x1_3(xxx)
-	KERNEL4x1_4(xxx)
-
-	KERNEL4x1_1(xxx)
-	KERNEL4x1_2(xxx)
-	KERNEL4x1_3(xxx)
-	KERNEL4x1_4(xxx)
-
-	je	.L1_16
-
-	jmp	.L1_12
-	ALIGN_4
-
-.L1_16:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_19
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,2), BI                    //  BI = BI * 4 ; number of values
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_17:
-
-	KERNEL4x1_SUB(xxx)
-	jl	.L1_17
-	ALIGN_4
-
-
-.L1_19:
-
-	vbroadcastss	ALPHA_R, %xmm0
-	vbroadcastss	ALPHA_I, %xmm1
-
-	// swap high and low 64 bytes
-        vshufps $0xb1, %xmm9 , %xmm9, %xmm9
-        vshufps $0xb1, %xmm13, %xmm13, %xmm13
-
-#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
-    defined(NR) || defined(NC) || defined(TR) || defined(TC)
-
-        vaddsubps %xmm9, %xmm8 , %xmm8
-        vaddsubps %xmm13,%xmm12, %xmm12
-
-        vshufps $0xb1, %xmm8 , %xmm8, %xmm9
-        vshufps $0xb1, %xmm12, %xmm12, %xmm13
-
-#else
-        vaddsubps %xmm8,  %xmm9 ,%xmm9
-        vaddsubps %xmm12, %xmm13,%xmm13
-
-        vmovaps   %xmm9,  %xmm8
-        vmovaps   %xmm13, %xmm12
-
-	// swap high and low 64 bytes
-        vshufps $0xb1, %xmm9 , %xmm9, %xmm9
-        vshufps $0xb1, %xmm13, %xmm13, %xmm13
-
-#endif
-
-	// multiply with ALPHA_R
-        vmulps  %xmm8 , %xmm0, %xmm8
-        vmulps  %xmm12, %xmm0, %xmm12
-
-	// multiply with ALPHA_I
-        vmulps  %xmm9 , %xmm1, %xmm9
-        vmulps  %xmm13, %xmm1, %xmm13
-
-	vaddsubps %xmm9, %xmm8 , %xmm8
-        vaddsubps %xmm13,%xmm12, %xmm12
-
-
-
-#ifndef TRMMKERNEL
-
-	vaddps 	 	(CO1), %xmm8 , %xmm8
-	vaddps  4 * SIZE(CO1), %xmm12, %xmm12
-
-#endif
-
-	vmovups	%xmm8 ,  	(CO1)
-	vmovups	%xmm12 , 4 * SIZE(CO1)
-
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $4, KK
-#endif
-
-	addq	$8 * SIZE, CO1		# coffset += 8
-	decq	I			# i --
-	jg	.L1_11
-	ALIGN_4	
-
-
-
-/**************************************************************************
-* Rest of M 
-***************************************************************************/
-
-.L1_20:
-	testq	$3, M		
-	jz	.L999
-
-	testq	$2, M		
-	jz	.L1_40
-	ALIGN_4
-
-.L1_21:
-
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-	leaq	BUFFER1, BO		// first buffer to BO
-	addq	$4 * SIZE, BO
-#else
-        movq    KK, %rax
-	leaq	BUFFER1, BO			// first buffer to BO
-	addq	$4 * SIZE, BO
-	movq    %rax, BI                        //  Index for BO
-        leaq    (,BI,2), BI                     //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $2, %rax        // number of values in AO
-#else
-        addq    $1, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-
-	andq	$-8, %rax			//  K = K - ( K % 8 )
-	je	.L1_26
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_22:
-
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL2x1_1(xxx)
-	KERNEL2x1_2(xxx)
-	KERNEL2x1_3(xxx)
-	KERNEL2x1_4(xxx)
-
-	KERNEL2x1_1(xxx)
-	KERNEL2x1_2(xxx)
-	KERNEL2x1_3(xxx)
-	KERNEL2x1_4(xxx)
-
-	je	.L1_26
-
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL2x1_1(xxx)
-	KERNEL2x1_2(xxx)
-	KERNEL2x1_3(xxx)
-	KERNEL2x1_4(xxx)
-
-	KERNEL2x1_1(xxx)
-	KERNEL2x1_2(xxx)
-	KERNEL2x1_3(xxx)
-	KERNEL2x1_4(xxx)
-
-	je	.L1_26
-
-	jmp	.L1_22
-	ALIGN_4
-
-.L1_26:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_29
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,2), BI                    //  BI = BI * 2; number of values
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_27:
-
-	KERNEL2x1_SUB(xxx)
-	jl	.L1_27
-	ALIGN_4
-
-
-.L1_29:
-
-	vbroadcastss	ALPHA_R, %xmm0
-	vbroadcastss	ALPHA_I, %xmm1
-
-	// swap high and low 64 bytes
-        vshufps $0xb1, %xmm9 , %xmm9, %xmm9
-
-#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
-    defined(NR) || defined(NC) || defined(TR) || defined(TC)
-
-        vaddsubps %xmm9, %xmm8 , %xmm8
-
-        vshufps $0xb1, %xmm8 , %xmm8, %xmm9
-
-#else
-        vaddsubps %xmm8,  %xmm9 ,%xmm9
-
-        vmovaps   %xmm9,  %xmm8
-
-	// swap high and low 64 bytes
-        vshufps $0xb1, %xmm9 , %xmm9, %xmm9
-
-#endif
-
-	// multiply with ALPHA_R
-        vmulps  %xmm8 , %xmm0, %xmm8
-
-	// multiply with ALPHA_I
-        vmulps  %xmm9 , %xmm1, %xmm9
-
-	vaddsubps %xmm9, %xmm8 , %xmm8
-
-
-
-#ifndef TRMMKERNEL
-
-	vaddps 	 	(CO1), %xmm8 , %xmm8
-
-#endif
-
-	vmovups	%xmm8 ,  	(CO1)
-
-
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $2, KK
-#endif
-
-	addq	$4 * SIZE, CO1		# coffset += 4
-	ALIGN_4	
-
-
-
-/**************************************************************************/
-.L1_40:
-	testq	$1, M		
-	jz	.L999		// to next 2 lines of N
-
-	ALIGN_4
-
-.L1_41:
-
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-	leaq	BUFFER1, BO		// first buffer to BO
-	addq	$4 * SIZE, BO
-#else
-        movq    KK, %rax
-	leaq	BUFFER1, BO			// first buffer to BO
-	addq	$4 * SIZE, BO
-	movq    %rax, BI                        //  Index for BO
-        leaq    (,BI,2), BI                     //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO
-	salq	$1, %rax			// rax = rax * 2 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $1, %rax        // number of values in AO
-#else
-        addq    $1, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-
-	andq	$-8, %rax			//  K = K - ( K % 8 )
-	je	.L1_46
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values
-
-	salq	$1, %rax			// rax = rax * 2 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_42:
-
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL1x1_1(xxx)
-	KERNEL1x1_2(xxx)
-	KERNEL1x1_3(xxx)
-	KERNEL1x1_4(xxx)
-
-	KERNEL1x1_1(xxx)
-	KERNEL1x1_2(xxx)
-	KERNEL1x1_3(xxx)
-	KERNEL1x1_4(xxx)
-
-	je	.L1_46
-
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL1x1_1(xxx)
-	KERNEL1x1_2(xxx)
-	KERNEL1x1_3(xxx)
-	KERNEL1x1_4(xxx)
-
-	KERNEL1x1_1(xxx)
-	KERNEL1x1_2(xxx)
-	KERNEL1x1_3(xxx)
-	KERNEL1x1_4(xxx)
-
-	je	.L1_46
-
-	jmp	.L1_42
-	ALIGN_4
-
-.L1_46:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_49
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values
-
-	salq	$1, %rax			// rax = rax * 2 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_47:
-
-	KERNEL1x1_SUB(xxx)
-	jl	.L1_47
-	ALIGN_4
-
-
-.L1_49:
-
-	vbroadcastss	ALPHA_R, %xmm0
-	vbroadcastss	ALPHA_I, %xmm1
-
-	// swap high and low 64 bytes
-        vshufps $0xb1, %xmm9 , %xmm9, %xmm9
-
-#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
-    defined(NR) || defined(NC) || defined(TR) || defined(TC)
-
-        vaddsubps %xmm9, %xmm8 , %xmm8
-
-        vshufps $0xb1, %xmm8 , %xmm8, %xmm9
-
-#else
-        vaddsubps %xmm8,  %xmm9 ,%xmm9
-
-        vmovaps   %xmm9,  %xmm8
-
-	// swap high and low 64 bytes
-        vshufps $0xb1, %xmm9 , %xmm9, %xmm9
-
-#endif
-
-	// multiply with ALPHA_R
-        vmulps  %xmm8 , %xmm0, %xmm8
-
-	// multiply with ALPHA_I
-        vmulps  %xmm9 , %xmm1, %xmm9
-
-	vaddsubps %xmm9, %xmm8 , %xmm8
-
-
-
-#ifndef TRMMKERNEL
-
-	vmovsd		(CO1), %xmm14
-	vaddps 	 	%xmm14, %xmm8 , %xmm8
-
-#endif
-
-	vmovsd	%xmm8 ,  	(CO1)
-
-
-
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO
-	salq	$1, %rax			// rax = rax * 2 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $1, KK
-#endif
-
-	addq	$2 * SIZE, CO1		# coffset += 2
-	ALIGN_4	
-
-
-
-
-
-
-
-.L999:
-	vzeroupper
-
-	movq   		SP, %rsp
-	movq	   (%rsp), %rbx
-	movq	  8(%rsp), %rbp
-	movq	 16(%rsp), %r12
-	movq	 24(%rsp), %r13
-	movq	 32(%rsp), %r14
-	movq	 40(%rsp), %r15
-
-#ifdef WINDOWS_ABI
-	movq	 48(%rsp), %rdi
-	movq	 56(%rsp), %rsi
-	vmovups	 64(%rsp), %xmm6
-	vmovups	 80(%rsp), %xmm7
-	vmovups	 96(%rsp), %xmm8
-	vmovups	112(%rsp), %xmm9
-	vmovups	128(%rsp), %xmm10
-	vmovups	144(%rsp), %xmm11
-	vmovups	160(%rsp), %xmm12
-	vmovups	176(%rsp), %xmm13
-	vmovups	192(%rsp), %xmm14
-	vmovups	208(%rsp), %xmm15
-#endif
-
-	addq	$STACKSIZE, %rsp
-	ret
-
-	EPILOGUE
+/***************************************************************************
+Copyright (c) 2013, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+/*********************************************************************
+*
+* 2014/06/28 Saar
+*        BLASTEST               : OK
+*        CTEST                  : OK
+*        TEST                   : OK
+*
+*
+* 2013/10/31 Saar
+*
+* Parameter:
+*       UNROLL_M        4
+*       UNROLL_N        2
+*       CGEMM_P         768
+*       CGEMM_Q         168
+*       A_PR1           512
+*       B_PR1           256
+*
+* Performance at m x n on AMD 8320 (ACML-Version: 5.3.1):
+* 
+* 4608x4608    154.0    GFLOPS with 8 threads on 4 modules (ACML: 111.7 ) (BULLDOZER: 153.9 )
+* 4608x4608    148.3    GFLOPS with 4 threads on 4 modules (ACML:  96.0 ) (BULLDOZER: 143.2 )
+* 3456x3456     74.3    GFLOPS with 2 threads on 2 modules (ACML:  47.3 ) (BULLDOZER:  72.3 )
+* 3456x3456     37.3    GFLOPS with 1 threads on 1 modules (ACML:  24.2 ) (BULLDOZER:  36.5 )
+*
+* Performance at m x n on AMD 6380  (ACML-Version: 5.3.1):
+* 
+* 6912x6912    421.5    GFLOPS with 32 threads on 16 modules (ACML: 266.6 ) (BULLDOZER: 422.5 )
+* 6912x6912    407.0    GFLOPS with 16 threads on 16 modules (ACML: 271.5 ) (BULLDOZER: 404.7 )
+* 6912x6912    234.2    GFLOPS with  8 threads on  8 modules (ACML: 164.0 ) (BULLDOZER: 230.5 )
+* 4608x4608    123.1    GFLOPS with  4 threads on  4 modules (ACML:  87.9 ) (BULLDOZER: 120.9 )
+* 3456x3456     62.6    GFLOPS with  2 threads on  2 modules (ACML:  44.5 ) (BULLDOZER:  62.1 )
+* 3456x3456     31.8    GFLOPS with  1 threads on  1 modules (ACML:  22.6 ) (BULLDOZER:  31.4 )
+*
+*********************************************************************/
+
+
+
+#define ASSEMBLER
+#include "common.h"
+ 
+#define OLD_M	%rdi
+#define OLD_N	%rsi
+#define M	%r13
+#define J	%r14
+#define OLD_K	%rdx
+
+#define A	%rcx
+#define B	%r8
+#define C	%r9
+#define LDC	%r10
+	
+#define I	%r11
+#define AO	%rdi
+#define BO	%rsi
+#define	CO1	%r15
+#define K	%r12
+#define BI	%rbp
+#define	SP	%rbx
+
+#define BO1	%rdi
+#define BO2	%r15
+
+#ifndef WINDOWS_ABI
+
+#define STACKSIZE 96
+
+#else
+
+#define STACKSIZE 320
+
+#define OLD_ALPHA_I     40 + STACKSIZE(%rsp)
+#define OLD_A           48 + STACKSIZE(%rsp)
+#define OLD_B           56 + STACKSIZE(%rsp)
+#define OLD_C           64 + STACKSIZE(%rsp)
+#define OLD_LDC         72 + STACKSIZE(%rsp)
+#define OLD_OFFSET      80 + STACKSIZE(%rsp)
+
+#endif
+
+#define L_BUFFER_SIZE 256*8*4
+
+#define Ndiv6	 24(%rsp)
+#define Nmod6	 32(%rsp)
+#define N	 40(%rsp)
+#define ALPHA_R  48(%rsp)
+#define ALPHA_I  56(%rsp)
+#define OFFSET   64(%rsp)
+#define KK       72(%rsp)
+#define KKK      80(%rsp)
+#define BUFFER1	           128(%rsp)
+
+#if defined(OS_WINDOWS)
+#if   L_BUFFER_SIZE > 16384
+#define STACK_TOUCH \
+        movl    $0,  4096 * 4(%rsp);\
+        movl    $0,  4096 * 3(%rsp);\
+        movl    $0,  4096 * 2(%rsp);\
+        movl    $0,  4096 * 1(%rsp);
+#elif L_BUFFER_SIZE > 12288
+#define STACK_TOUCH \
+        movl    $0,  4096 * 3(%rsp);\
+        movl    $0,  4096 * 2(%rsp);\
+        movl    $0,  4096 * 1(%rsp);
+#elif L_BUFFER_SIZE > 8192
+#define STACK_TOUCH \
+        movl    $0,  4096 * 2(%rsp);\
+        movl    $0,  4096 * 1(%rsp);
+#elif L_BUFFER_SIZE > 4096
+#define STACK_TOUCH \
+        movl    $0,  4096 * 1(%rsp);
+#else
+#define STACK_TOUCH
+#endif
+#else
+#define STACK_TOUCH
+#endif
+
+
+#if   defined(NN) || defined(NT) || defined(TN) || defined(TT)
+#define VFMADD_R    vfmaddps
+#define VFMADD_I    vfmaddps
+#elif defined(RN) || defined(RT) || defined(CN) || defined(CT)
+#define VFMADD_R    vfnmaddps
+#define VFMADD_I    vfmaddps
+#elif defined(NR) || defined(NC) || defined(TR) || defined(TC)
+#define VFMADD_R    vfmaddps
+#define VFMADD_I    vfnmaddps
+#else
+#define VFMADD_R    vfnmaddps
+#define VFMADD_I    vfnmaddps
+#endif
+
+
+
+#define	A_PR1	512
+#define	B_PR1	256
+
+#define KERNEL4x2_1(xx) \
+        prefetcht0      A_PR1(AO,%rax,SIZE)        ;\
+        vmovups         -16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss         -8 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vmovups         -12 * SIZE(AO, %rax, SIZE), %xmm1 ;\
+        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
+        vbroadcastss         -7 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        VFMADD_I        %xmm13,%xmm5,%xmm1,%xmm13 ;\
+        vbroadcastss         -6 * SIZE(BO, BI, SIZE), %xmm6 ;\
+        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
+        VFMADD_R        %xmm14,%xmm6,%xmm1,%xmm14 ;\
+        vbroadcastss         -5 * SIZE(BO, BI, SIZE), %xmm7 ;\
+        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
+        VFMADD_I        %xmm15,%xmm7,%xmm1,%xmm15 ;\
+
+#define KERNEL4x2_2(xx) \
+        vmovups          -8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss         -4 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vmovups          -4 * SIZE(AO, %rax, SIZE), %xmm1 ;\
+        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
+        vbroadcastss         -3 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        VFMADD_I        %xmm13,%xmm5,%xmm1,%xmm13 ;\
+        vbroadcastss         -2 * SIZE(BO, BI, SIZE), %xmm6 ;\
+        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
+        VFMADD_R        %xmm14,%xmm6,%xmm1,%xmm14 ;\
+        vbroadcastss         -1 * SIZE(BO, BI, SIZE), %xmm7 ;\
+        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
+        VFMADD_I        %xmm15,%xmm7,%xmm1,%xmm15 ;\
+
+#define KERNEL4x2_3(xx) \
+	prefetcht0      A_PR1+64(AO,%rax,SIZE)     ;\
+        vmovups           0 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss          0 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vmovups           4 * SIZE(AO, %rax, SIZE), %xmm1 ;\
+        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
+        vbroadcastss          1 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        VFMADD_I        %xmm13,%xmm5,%xmm1,%xmm13 ;\
+        vbroadcastss          2 * SIZE(BO, BI, SIZE), %xmm6 ;\
+        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
+        VFMADD_R        %xmm14,%xmm6,%xmm1,%xmm14 ;\
+        vbroadcastss          3 * SIZE(BO, BI, SIZE), %xmm7 ;\
+        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
+        VFMADD_I        %xmm15,%xmm7,%xmm1,%xmm15 ;\
+
+#define KERNEL4x2_4(xx) \
+        vmovups           8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss          4 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vmovups          12 * SIZE(AO, %rax, SIZE), %xmm1 ;\
+        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
+        vbroadcastss          5 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        VFMADD_I        %xmm13,%xmm5,%xmm1,%xmm13 ;\
+        vbroadcastss          6 * SIZE(BO, BI, SIZE), %xmm6 ;\
+        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
+        VFMADD_R        %xmm14,%xmm6,%xmm1,%xmm14 ;\
+        vbroadcastss          7 * SIZE(BO, BI, SIZE), %xmm7 ;\
+        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
+        VFMADD_I        %xmm15,%xmm7,%xmm1,%xmm15 ;\
+        addq    $16, BI                            ;\
+        addq    $32, %rax                          ;\
+
+
+#define KERNEL4x2_SUB(xx) \
+        vmovups         -16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss         -8 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vmovups         -12 * SIZE(AO, %rax, SIZE), %xmm1 ;\
+        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
+        vbroadcastss         -7 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        VFMADD_I        %xmm13,%xmm5,%xmm1,%xmm13 ;\
+        vbroadcastss         -6 * SIZE(BO, BI, SIZE), %xmm6 ;\
+        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
+        VFMADD_R        %xmm14,%xmm6,%xmm1,%xmm14 ;\
+        vbroadcastss         -5 * SIZE(BO, BI, SIZE), %xmm7 ;\
+        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
+        VFMADD_I        %xmm15,%xmm7,%xmm1,%xmm15 ;\
+        addq    $4, BI                            ;\
+        addq    $8, %rax                          ;\
+
+/************************************************************************************************/
+
+#define KERNEL2x2_1(xx) \
+        prefetcht0      A_PR1(AO,%rax,SIZE)        ;\
+        vmovups         -16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss         -8 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vbroadcastss         -7 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        vbroadcastss         -6 * SIZE(BO, BI, SIZE), %xmm6 ;\
+        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
+        vbroadcastss         -5 * SIZE(BO, BI, SIZE), %xmm7 ;\
+        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
+
+#define KERNEL2x2_2(xx) \
+        vmovups         -12 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss         -4 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
+        vbroadcastss         -3 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        vbroadcastss         -2 * SIZE(BO, BI, SIZE), %xmm6 ;\
+        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
+        vbroadcastss         -1 * SIZE(BO, BI, SIZE), %xmm7 ;\
+        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
+
+#define KERNEL2x2_3(xx) \
+        vmovups          -8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss          0 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vbroadcastss          1 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        vbroadcastss          2 * SIZE(BO, BI, SIZE), %xmm6 ;\
+        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
+        vbroadcastss          3 * SIZE(BO, BI, SIZE), %xmm7 ;\
+        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
+
+#define KERNEL2x2_4(xx) \
+        vmovups          -4 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss          4 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vbroadcastss          5 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        vbroadcastss          6 * SIZE(BO, BI, SIZE), %xmm6 ;\
+        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
+        vbroadcastss          7 * SIZE(BO, BI, SIZE), %xmm7 ;\
+        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
+        addq    $16, BI                            ;\
+        addq    $16, %rax                          ;\
+
+
+#define KERNEL2x2_SUB(xx) \
+        vmovups         -16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss         -8 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vbroadcastss         -7 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        vbroadcastss         -6 * SIZE(BO, BI, SIZE), %xmm6 ;\
+        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
+        vbroadcastss         -5 * SIZE(BO, BI, SIZE), %xmm7 ;\
+        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
+        addq    $4, BI                            ;\
+        addq    $4, %rax                          ;\
+
+/************************************************************************************************/
+
+#define KERNEL1x2_1(xx) \
+        vmovsd          -16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss         -8 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vbroadcastss         -7 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        vbroadcastss         -6 * SIZE(BO, BI, SIZE), %xmm6 ;\
+        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
+        vbroadcastss         -5 * SIZE(BO, BI, SIZE), %xmm7 ;\
+        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
+
+#define KERNEL1x2_2(xx) \
+        vmovsd          -14 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss         -4 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
+        vbroadcastss         -3 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        vbroadcastss         -2 * SIZE(BO, BI, SIZE), %xmm6 ;\
+        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
+        vbroadcastss         -1 * SIZE(BO, BI, SIZE), %xmm7 ;\
+        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
+
+#define KERNEL1x2_3(xx) \
+        vmovsd          -12 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss          0 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vbroadcastss          1 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        vbroadcastss          2 * SIZE(BO, BI, SIZE), %xmm6 ;\
+        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
+        vbroadcastss          3 * SIZE(BO, BI, SIZE), %xmm7 ;\
+        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
+
+#define KERNEL1x2_4(xx) \
+        vmovsd          -10 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss          4 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vbroadcastss          5 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        vbroadcastss          6 * SIZE(BO, BI, SIZE), %xmm6 ;\
+        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
+        vbroadcastss          7 * SIZE(BO, BI, SIZE), %xmm7 ;\
+        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
+        addq    $16, BI                            ;\
+        addq    $8, %rax                          ;\
+
+
+#define KERNEL1x2_SUB(xx) \
+        vmovsd         -16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss         -8 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vbroadcastss         -7 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        vbroadcastss         -6 * SIZE(BO, BI, SIZE), %xmm6 ;\
+        VFMADD_R        %xmm10,%xmm6,%xmm0,%xmm10 ;\
+        vbroadcastss         -5 * SIZE(BO, BI, SIZE), %xmm7 ;\
+        VFMADD_I        %xmm11,%xmm7,%xmm0,%xmm11 ;\
+        addq    $4, BI                            ;\
+        addq    $2, %rax                          ;\
+
+
+
+/************************************************************************************************/
+
+#define KERNEL4x1_1(xx) \
+        prefetcht0      A_PR1(AO,%rax,SIZE)        ;\
+        vmovups         -16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss         -4 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vmovups         -12 * SIZE(AO, %rax, SIZE), %xmm1 ;\
+        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
+        vbroadcastss         -3 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        VFMADD_I        %xmm13,%xmm5,%xmm1,%xmm13 ;\
+
+#define KERNEL4x1_2(xx) \
+        vmovups          -8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss         -2 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vmovups          -4 * SIZE(AO, %rax, SIZE), %xmm1 ;\
+        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
+        vbroadcastss         -1 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        VFMADD_I        %xmm13,%xmm5,%xmm1,%xmm13 ;\
+
+#define KERNEL4x1_3(xx) \
+	prefetcht0      A_PR1+64(AO,%rax,SIZE)     ;\
+        vmovups           0 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss          0 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vmovups           4 * SIZE(AO, %rax, SIZE), %xmm1 ;\
+        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
+        vbroadcastss          1 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        VFMADD_I        %xmm13,%xmm5,%xmm1,%xmm13 ;\
+
+#define KERNEL4x1_4(xx) \
+        vmovups           8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss          2 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vmovups          12 * SIZE(AO, %rax, SIZE), %xmm1 ;\
+        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
+        vbroadcastss          3 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        VFMADD_I        %xmm13,%xmm5,%xmm1,%xmm13 ;\
+        addq    $8, BI                            ;\
+        addq    $32, %rax                          ;\
+
+
+#define KERNEL4x1_SUB(xx) \
+        vmovups         -16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss         -4 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vmovups         -12 * SIZE(AO, %rax, SIZE), %xmm1 ;\
+        VFMADD_R        %xmm12,%xmm4,%xmm1,%xmm12 ;\
+        vbroadcastss         -3 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        VFMADD_I        %xmm13,%xmm5,%xmm1,%xmm13 ;\
+        addq    $2, BI                            ;\
+        addq    $8, %rax                          ;\
+
+
+/************************************************************************************************/
+
+#define KERNEL2x1_1(xx) \
+        prefetcht0      A_PR1(AO,%rax,SIZE)        ;\
+        vmovups         -16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss         -4 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vbroadcastss         -3 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+
+#define KERNEL2x1_2(xx) \
+        vmovups         -12 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss         -2 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vbroadcastss         -1 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+
+#define KERNEL2x1_3(xx) \
+        vmovups          -8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss          0 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vbroadcastss          1 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+
+#define KERNEL2x1_4(xx) \
+        vmovups          -4 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss          2 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vbroadcastss          3 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        addq    $8, BI                            ;\
+        addq    $16, %rax                          ;\
+
+
+#define KERNEL2x1_SUB(xx) \
+        vmovups         -16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss         -4 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vbroadcastss         -3 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        addq    $2, BI                            ;\
+        addq    $4, %rax                          ;\
+
+
+/************************************************************************************************/
+
+#define KERNEL1x1_1(xx) \
+        vmovsd         -16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss         -4 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vbroadcastss         -3 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+
+#define KERNEL1x1_2(xx) \
+        vmovsd         -14 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss         -2 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vbroadcastss         -1 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+
+#define KERNEL1x1_3(xx) \
+        vmovsd         -12 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss          0 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vbroadcastss          1 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+
+#define KERNEL1x1_4(xx) \
+        vmovsd          -10 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss          2 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vbroadcastss          3 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        addq    $8, BI                            ;\
+        addq    $8, %rax                          ;\
+
+
+#define KERNEL1x1_SUB(xx) \
+        vmovsd         -16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+        vbroadcastss         -4 * SIZE(BO, BI, SIZE), %xmm4 ;\
+        VFMADD_R        %xmm8,%xmm4,%xmm0,%xmm8 ;\
+        vbroadcastss         -3 * SIZE(BO, BI, SIZE), %xmm5 ;\
+        VFMADD_I        %xmm9,%xmm5,%xmm0,%xmm9 ;\
+        addq    $2, BI                            ;\
+        addq    $2, %rax                          ;\
+
+
+/************************************************************************************************/
+
+
+
+
+	PROLOGUE
+	PROFCODE
+	
+	subq	$STACKSIZE, %rsp
+	movq	%rbx,   (%rsp)
+	movq	%rbp,  8(%rsp)
+	movq	%r12, 16(%rsp)
+	movq	%r13, 24(%rsp)
+	movq	%r14, 32(%rsp)
+	movq	%r15, 40(%rsp)
+
+	vzeroupper
+
+#ifdef WINDOWS_ABI
+	movq	%rdi,    48(%rsp)
+	movq	%rsi,    56(%rsp)
+	vmovups	%xmm6,   64(%rsp)
+	vmovups	%xmm7,   80(%rsp)
+	vmovups	%xmm8,   96(%rsp)
+	vmovups	%xmm9,  112(%rsp)
+	vmovups	%xmm10, 128(%rsp)
+	vmovups	%xmm11, 144(%rsp)
+	vmovups	%xmm12, 160(%rsp)
+	vmovups	%xmm13, 176(%rsp)
+	vmovups	%xmm14, 192(%rsp)
+	vmovups	%xmm15, 208(%rsp)
+
+	movq	ARG1,      OLD_M
+	movq	ARG2,      OLD_N
+	movq	ARG3,      OLD_K
+	movq	OLD_A,     A
+	movq	OLD_B,     B
+	movq	OLD_C,     C
+	movq	OLD_LDC,   LDC
+#ifdef TRMMKERNEL
+	movsd	OLD_OFFSET, %xmm12
+#endif
+	vmovaps	%xmm3, %xmm0
+	vmovsd   OLD_ALPHA_I, %xmm1
+
+#else
+	movq	STACKSIZE +  8(%rsp), LDC
+#ifdef TRMMKERNEL
+	movsd	STACKSIZE + 16(%rsp), %xmm12
+#endif
+
+#endif
+
+	movq    %rsp, SP      # save old stack
+        subq    $128 + L_BUFFER_SIZE, %rsp
+        andq    $-4096, %rsp    # align stack
+
+        STACK_TOUCH
+
+	cmpq	$0, OLD_M
+	je	.L999
+
+	cmpq	$0, OLD_N
+	je	.L999
+
+	cmpq	$0, OLD_K
+	je	.L999
+
+	movq	OLD_M, M
+	movq	OLD_N, N
+	movq	OLD_K, K
+
+	vmovss	 %xmm0, ALPHA_R
+	vmovss	 %xmm1, ALPHA_I
+
+	salq	$ZBASE_SHIFT, LDC
+
+	movq    N, %rax
+        xorq    %rdx, %rdx
+        movq    $2,  %rdi
+        divq    %rdi                    //    N / 2
+        movq    %rax, Ndiv6             //    N / 2
+        movq    %rdx, Nmod6             //    N % 2
+
+	
+
+#ifdef TRMMKERNEL
+	vmovsd	%xmm12, OFFSET
+	vmovsd	%xmm12, KK
+#ifndef LEFT
+	negq	KK
+#endif	
+#endif
+
+.L2_0:
+
+	movq	Ndiv6,  J
+	cmpq	$0, J
+	je	.L1_0
+	ALIGN_4
+
+
+
+.L2_01:
+	// copy to sub buffer
+	movq	B, BO1
+	leaq    BUFFER1, BO		// first buffer to BO
+	movq	K, %rax
+	ALIGN_4
+
+.L2_02b:
+
+	vmovups	(BO1), %xmm0
+	vmovups	%xmm0,       (BO)
+	addq	$4*SIZE,BO1
+	addq	$4*SIZE,BO
+	decq	%rax
+	jnz	.L2_02b
+
+.L2_02c:
+
+	movq	BO1, B			// next offset of B
+
+.L2_10:
+	movq	C, CO1
+	leaq	(C, LDC, 2), C		// c += 2 * ldc
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        movq    OFFSET, %rax
+        movq    %rax, KK
+#endif
+	
+	movq	A, AO		 	// aoffset = a
+	addq	$16 * SIZE, AO
+
+	movq	M,  I
+	sarq	$2, I			// i = (m >> 2)
+	je	.L2_20
+
+	ALIGN_4
+
+.L2_11:
+
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+	leaq	BUFFER1, BO		// first buffer to BO
+	addq	$8 * SIZE, BO
+#else
+        movq    KK, %rax
+	leaq	BUFFER1, BO			// first buffer to BO
+	addq	$8 * SIZE, BO
+	movq    %rax, BI                        //  Index for BO
+        leaq    (,BI,4), BI                     //  BI = BI * 4 ; number of values
+        leaq    (BO, BI, SIZE), BO
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $4, %rax        // number of values in AO
+#else
+        addq    $2, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$-8, %rax			//  K = K - ( K % 8 )
+	je	.L2_16
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_12:
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL4x2_1(xxx)
+	KERNEL4x2_2(xxx)
+	KERNEL4x2_3(xxx)
+	KERNEL4x2_4(xxx)
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL4x2_1(xxx)
+	KERNEL4x2_2(xxx)
+	KERNEL4x2_3(xxx)
+	KERNEL4x2_4(xxx)
+
+	je	.L2_16
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL4x2_1(xxx)
+	KERNEL4x2_2(xxx)
+	KERNEL4x2_3(xxx)
+	KERNEL4x2_4(xxx)
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL4x2_1(xxx)
+	KERNEL4x2_2(xxx)
+	KERNEL4x2_3(xxx)
+	KERNEL4x2_4(xxx)
+
+	je	.L2_16
+
+	jmp	.L2_12
+	ALIGN_4
+
+.L2_16:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_19
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_17:
+
+	KERNEL4x2_SUB(xxx)
+	jl	.L2_17
+	ALIGN_4
+
+
+.L2_19:
+
+	vbroadcastss	ALPHA_R, %xmm0
+	vbroadcastss	ALPHA_I, %xmm1
+
+	// swap high and low 64 bytes
+        vshufps $0xb1, %xmm9 , %xmm9, %xmm9
+        vshufps $0xb1, %xmm11, %xmm11, %xmm11
+        vshufps $0xb1, %xmm13, %xmm13, %xmm13
+        vshufps $0xb1, %xmm15, %xmm15, %xmm15
+
+#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
+    defined(NR) || defined(NC) || defined(TR) || defined(TC)
+
+        vaddsubps %xmm9, %xmm8 , %xmm8
+        vaddsubps %xmm11,%xmm10, %xmm10
+        vaddsubps %xmm13,%xmm12, %xmm12
+        vaddsubps %xmm15,%xmm14, %xmm14
+
+        vshufps $0xb1, %xmm8 , %xmm8, %xmm9
+        vshufps $0xb1, %xmm10, %xmm10, %xmm11
+        vshufps $0xb1, %xmm12, %xmm12, %xmm13
+        vshufps $0xb1, %xmm14, %xmm14, %xmm15
+
+#else
+        vaddsubps %xmm8,  %xmm9 ,%xmm9
+        vaddsubps %xmm10, %xmm11,%xmm11
+        vaddsubps %xmm12, %xmm13,%xmm13
+        vaddsubps %xmm14, %xmm15,%xmm15
+
+        vmovaps   %xmm9,  %xmm8
+        vmovaps   %xmm11, %xmm10
+        vmovaps   %xmm13, %xmm12
+        vmovaps   %xmm15, %xmm14
+
+	// swap high and low 64 bytes
+        vshufps $0xb1, %xmm9 , %xmm9, %xmm9
+        vshufps $0xb1, %xmm11, %xmm11, %xmm11
+        vshufps $0xb1, %xmm13, %xmm13, %xmm13
+        vshufps $0xb1, %xmm15, %xmm15, %xmm15
+
+#endif
+
+	// multiply with ALPHA_R
+        vmulps  %xmm8 , %xmm0, %xmm8
+        vmulps  %xmm10, %xmm0, %xmm10
+        vmulps  %xmm12, %xmm0, %xmm12
+        vmulps  %xmm14, %xmm0, %xmm14
+
+	// multiply with ALPHA_I
+        vmulps  %xmm9 , %xmm1, %xmm9
+        vmulps  %xmm11, %xmm1, %xmm11
+        vmulps  %xmm13, %xmm1, %xmm13
+        vmulps  %xmm15, %xmm1, %xmm15
+
+	vaddsubps %xmm9, %xmm8 , %xmm8
+        vaddsubps %xmm11,%xmm10, %xmm10
+        vaddsubps %xmm13,%xmm12, %xmm12
+        vaddsubps %xmm15,%xmm14, %xmm14
+
+
+
+#ifndef TRMMKERNEL
+
+	vaddps 	 	(CO1), %xmm8 , %xmm8
+	vaddps  4 * SIZE(CO1), %xmm12, %xmm12
+
+	vaddps 	 	(CO1, LDC), %xmm10, %xmm10
+	vaddps  4 * SIZE(CO1, LDC), %xmm14, %xmm14
+
+#endif
+
+	vmovups	%xmm8 ,  	(CO1)
+	vmovups	%xmm12 , 4 * SIZE(CO1)
+
+	vmovups	%xmm10 ,  	(CO1, LDC)
+	vmovups	%xmm14 , 4 * SIZE(CO1, LDC)
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
+        leaq    (BO, BI, SIZE), BO
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $4, KK
+#endif
+
+	addq	$8 * SIZE, CO1		# coffset += 8
+	decq	I			# i --
+	jg	.L2_11
+	ALIGN_4	
+
+
+
+/**************************************************************************
+* Rest of M 
+***************************************************************************/
+
+.L2_20:
+	testq	$3, M		
+	jz	.L2_60		// to next 2 lines of N
+
+	testq	$2, M		
+	jz	.L2_40
+	ALIGN_4
+
+.L2_21:
+
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+	leaq	BUFFER1, BO		// first buffer to BO
+	addq	$8 * SIZE, BO
+#else
+        movq    KK, %rax
+	leaq	BUFFER1, BO			// first buffer to BO
+	addq	$8 * SIZE, BO
+	movq    %rax, BI                        //  Index for BO
+        leaq    (,BI,4), BI                     //  BI = BI * 4 ; number of values
+        leaq    (BO, BI, SIZE), BO
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $2, %rax        // number of values in AO
+#else
+        addq    $2, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$-8, %rax			//  K = K - ( K % 8 )
+	je	.L2_26
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_22:
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL2x2_1(xxx)
+	KERNEL2x2_2(xxx)
+	KERNEL2x2_3(xxx)
+	KERNEL2x2_4(xxx)
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL2x2_1(xxx)
+	KERNEL2x2_2(xxx)
+	KERNEL2x2_3(xxx)
+	KERNEL2x2_4(xxx)
+
+	je	.L2_26
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL2x2_1(xxx)
+	KERNEL2x2_2(xxx)
+	KERNEL2x2_3(xxx)
+	KERNEL2x2_4(xxx)
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL2x2_1(xxx)
+	KERNEL2x2_2(xxx)
+	KERNEL2x2_3(xxx)
+	KERNEL2x2_4(xxx)
+
+	je	.L2_26
+
+	jmp	.L2_22
+	ALIGN_4
+
+.L2_26:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_29
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_27:
+
+	KERNEL2x2_SUB(xxx)
+	jl	.L2_27
+	ALIGN_4
+
+
+.L2_29:
+
+	vbroadcastss	ALPHA_R, %xmm0
+	vbroadcastss	ALPHA_I, %xmm1
+
+	// swap high and low 64 bytes
+        vshufps $0xb1, %xmm9 , %xmm9, %xmm9
+        vshufps $0xb1, %xmm11, %xmm11, %xmm11
+
+#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
+    defined(NR) || defined(NC) || defined(TR) || defined(TC)
+
+        vaddsubps %xmm9, %xmm8 , %xmm8
+        vaddsubps %xmm11,%xmm10, %xmm10
+
+        vshufps $0xb1, %xmm8 , %xmm8, %xmm9
+        vshufps $0xb1, %xmm10, %xmm10, %xmm11
+
+#else
+        vaddsubps %xmm8,  %xmm9 ,%xmm9
+        vaddsubps %xmm10, %xmm11,%xmm11
+
+        vmovaps   %xmm9,  %xmm8
+        vmovaps   %xmm11, %xmm10
+
+	// swap high and low 64 bytes
+        vshufps $0xb1, %xmm9 , %xmm9, %xmm9
+        vshufps $0xb1, %xmm11, %xmm11, %xmm11
+
+#endif
+
+	// multiply with ALPHA_R
+        vmulps  %xmm8 , %xmm0, %xmm8
+        vmulps  %xmm10, %xmm0, %xmm10
+
+	// multiply with ALPHA_I
+        vmulps  %xmm9 , %xmm1, %xmm9
+        vmulps  %xmm11, %xmm1, %xmm11
+
+	vaddsubps %xmm9, %xmm8 , %xmm8
+        vaddsubps %xmm11,%xmm10, %xmm10
+
+
+
+#ifndef TRMMKERNEL
+
+	vaddps 	 	(CO1), %xmm8 , %xmm8
+
+	vaddps 	 	(CO1, LDC), %xmm10, %xmm10
+
+#endif
+
+	vmovups	%xmm8 ,  	(CO1)
+
+	vmovups	%xmm10 ,  	(CO1, LDC)
+
+
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
+        leaq    (BO, BI, SIZE), BO
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $2, KK
+#endif
+
+	addq	$4 * SIZE, CO1		# coffset += 4
+	ALIGN_4	
+
+
+
+/**************************************************************************/
+.L2_40:
+	testq	$1, M		
+	jz	.L2_60		// to next 2 lines of N
+
+	ALIGN_4
+
+.L2_41:
+
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+	leaq	BUFFER1, BO		// first buffer to BO
+	addq	$8 * SIZE, BO
+#else
+        movq    KK, %rax
+	leaq	BUFFER1, BO			// first buffer to BO
+	addq	$8 * SIZE, BO
+	movq    %rax, BI                        //  Index for BO
+        leaq    (,BI,4), BI                     //  BI = BI * 4 ; number of values
+        leaq    (BO, BI, SIZE), BO
+	salq	$1, %rax			// rax = rax * 2 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $1, %rax        // number of values in AO
+#else
+        addq    $2, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$-8, %rax			//  K = K - ( K % 8 )
+	je	.L2_46
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
+
+	salq	$1, %rax			// rax = rax * 2 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_42:
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL1x2_1(xxx)
+	KERNEL1x2_2(xxx)
+	KERNEL1x2_3(xxx)
+	KERNEL1x2_4(xxx)
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL1x2_1(xxx)
+	KERNEL1x2_2(xxx)
+	KERNEL1x2_3(xxx)
+	KERNEL1x2_4(xxx)
+
+	je	.L2_46
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL1x2_1(xxx)
+	KERNEL1x2_2(xxx)
+	KERNEL1x2_3(xxx)
+	KERNEL1x2_4(xxx)
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL1x2_1(xxx)
+	KERNEL1x2_2(xxx)
+	KERNEL1x2_3(xxx)
+	KERNEL1x2_4(xxx)
+
+	je	.L2_46
+
+	jmp	.L2_42
+	ALIGN_4
+
+.L2_46:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_49
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
+
+	salq	$1, %rax			// rax = rax * 2 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_47:
+
+	KERNEL1x2_SUB(xxx)
+	jl	.L2_47
+	ALIGN_4
+
+
+.L2_49:
+
+	vbroadcastss	ALPHA_R, %xmm0
+	vbroadcastss	ALPHA_I, %xmm1
+
+	// swap high and low 64 bytes
+        vshufps $0xb1, %xmm9 , %xmm9, %xmm9
+        vshufps $0xb1, %xmm11, %xmm11, %xmm11
+
+#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
+    defined(NR) || defined(NC) || defined(TR) || defined(TC)
+
+        vaddsubps %xmm9, %xmm8 , %xmm8
+        vaddsubps %xmm11,%xmm10, %xmm10
+
+        vshufps $0xb1, %xmm8 , %xmm8, %xmm9
+        vshufps $0xb1, %xmm10, %xmm10, %xmm11
+
+#else
+        vaddsubps %xmm8,  %xmm9 ,%xmm9
+        vaddsubps %xmm10, %xmm11,%xmm11
+
+        vmovaps   %xmm9,  %xmm8
+        vmovaps   %xmm11, %xmm10
+
+	// swap high and low 64 bytes
+        vshufps $0xb1, %xmm9 , %xmm9, %xmm9
+        vshufps $0xb1, %xmm11, %xmm11, %xmm11
+
+#endif
+
+	// multiply with ALPHA_R
+        vmulps  %xmm8 , %xmm0, %xmm8
+        vmulps  %xmm10, %xmm0, %xmm10
+
+	// multiply with ALPHA_I
+        vmulps  %xmm9 , %xmm1, %xmm9
+        vmulps  %xmm11, %xmm1, %xmm11
+
+	vaddsubps %xmm9, %xmm8 , %xmm8
+        vaddsubps %xmm11,%xmm10, %xmm10
+
+
+
+#ifndef TRMMKERNEL
+
+	vmovsd		(CO1), %xmm14
+	vaddps 	 	%xmm14, %xmm8 , %xmm8
+
+	vmovsd		(CO1, LDC), %xmm15
+	vaddps 	 	%xmm15, %xmm10, %xmm10
+
+#endif
+
+	vmovsd	%xmm8 ,  	(CO1)
+
+	vmovsd	%xmm10 ,  	(CO1, LDC)
+
+
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
+        leaq    (BO, BI, SIZE), BO
+	salq	$1, %rax			// rax = rax * 2 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $1, KK
+#endif
+
+	addq	$2 * SIZE, CO1		# coffset += 2
+	ALIGN_4	
+
+
+
+	
+.L2_60:
+#if defined(TRMMKERNEL) && !defined(LEFT)
+        addq    $2, KK
+#endif
+
+	decq	J			// j --
+	jg	.L2_01			// next 2 lines of N
+
+
+
+.L1_0:
+
+/************************************************************************************************
+* Loop for Nmod6 % 2 > 0
+*************************************************************************************************/
+
+	movq	Nmod6, J		
+	andq	$1, J			// j % 2
+	je	.L999
+	ALIGN_4
+
+.L1_01:
+	// copy to sub buffer
+	movq	B, BO1
+	leaq    BUFFER1, BO		// first buffer to BO
+	movq	K, %rax
+	ALIGN_4
+
+.L1_02b:
+
+	vmovsd		(BO1), %xmm0
+	vmovsd	%xmm0,       (BO)
+	addq	$2*SIZE,BO1
+	addq	$2*SIZE,BO
+	decq	%rax
+	jnz	.L1_02b
+
+.L1_02c:
+
+	movq	BO1, B			// next offset of B
+
+.L1_10:
+	movq	C, CO1
+	leaq	(C, LDC, 1), C		// c += 1 * ldc
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        movq    OFFSET, %rax
+        movq    %rax, KK
+#endif
+	
+	movq	A, AO		 	// aoffset = a
+	addq	$16 * SIZE, AO
+
+	movq	M,  I
+	sarq	$2, I			// i = (m >> 2)
+	je	.L1_20
+
+	ALIGN_4
+
+.L1_11:
+
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+	leaq	BUFFER1, BO		// first buffer to BO
+	addq	$4 * SIZE, BO
+#else
+        movq    KK, %rax
+	leaq	BUFFER1, BO			// first buffer to BO
+	addq	$4 * SIZE, BO
+	movq    %rax, BI                        //  Index for BO
+        leaq    (,BI,2), BI                     //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $4, %rax        // number of values in AO
+#else
+        addq    $1, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$-8, %rax			//  K = K - ( K % 8 )
+	je	.L1_16
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_12:
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL4x1_1(xxx)
+	KERNEL4x1_2(xxx)
+	KERNEL4x1_3(xxx)
+	KERNEL4x1_4(xxx)
+
+	KERNEL4x1_1(xxx)
+	KERNEL4x1_2(xxx)
+	KERNEL4x1_3(xxx)
+	KERNEL4x1_4(xxx)
+
+	je	.L1_16
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL4x1_1(xxx)
+	KERNEL4x1_2(xxx)
+	KERNEL4x1_3(xxx)
+	KERNEL4x1_4(xxx)
+
+	KERNEL4x1_1(xxx)
+	KERNEL4x1_2(xxx)
+	KERNEL4x1_3(xxx)
+	KERNEL4x1_4(xxx)
+
+	je	.L1_16
+
+	jmp	.L1_12
+	ALIGN_4
+
+.L1_16:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_19
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,2), BI                    //  BI = BI * 4 ; number of values
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_17:
+
+	KERNEL4x1_SUB(xxx)
+	jl	.L1_17
+	ALIGN_4
+
+
+.L1_19:
+
+	vbroadcastss	ALPHA_R, %xmm0
+	vbroadcastss	ALPHA_I, %xmm1
+
+	// swap high and low 64 bytes
+        vshufps $0xb1, %xmm9 , %xmm9, %xmm9
+        vshufps $0xb1, %xmm13, %xmm13, %xmm13
+
+#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
+    defined(NR) || defined(NC) || defined(TR) || defined(TC)
+
+        vaddsubps %xmm9, %xmm8 , %xmm8
+        vaddsubps %xmm13,%xmm12, %xmm12
+
+        vshufps $0xb1, %xmm8 , %xmm8, %xmm9
+        vshufps $0xb1, %xmm12, %xmm12, %xmm13
+
+#else
+        vaddsubps %xmm8,  %xmm9 ,%xmm9
+        vaddsubps %xmm12, %xmm13,%xmm13
+
+        vmovaps   %xmm9,  %xmm8
+        vmovaps   %xmm13, %xmm12
+
+	// swap high and low 64 bytes
+        vshufps $0xb1, %xmm9 , %xmm9, %xmm9
+        vshufps $0xb1, %xmm13, %xmm13, %xmm13
+
+#endif
+
+	// multiply with ALPHA_R
+        vmulps  %xmm8 , %xmm0, %xmm8
+        vmulps  %xmm12, %xmm0, %xmm12
+
+	// multiply with ALPHA_I
+        vmulps  %xmm9 , %xmm1, %xmm9
+        vmulps  %xmm13, %xmm1, %xmm13
+
+	vaddsubps %xmm9, %xmm8 , %xmm8
+        vaddsubps %xmm13,%xmm12, %xmm12
+
+
+
+#ifndef TRMMKERNEL
+
+	vaddps 	 	(CO1), %xmm8 , %xmm8
+	vaddps  4 * SIZE(CO1), %xmm12, %xmm12
+
+#endif
+
+	vmovups	%xmm8 ,  	(CO1)
+	vmovups	%xmm12 , 4 * SIZE(CO1)
+
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $4, KK
+#endif
+
+	addq	$8 * SIZE, CO1		# coffset += 8
+	decq	I			# i --
+	jg	.L1_11
+	ALIGN_4	
+
+
+
+/**************************************************************************
+* Rest of M 
+***************************************************************************/
+
+.L1_20:
+	testq	$3, M		
+	jz	.L999
+
+	testq	$2, M		
+	jz	.L1_40
+	ALIGN_4
+
+.L1_21:
+
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+	leaq	BUFFER1, BO		// first buffer to BO
+	addq	$4 * SIZE, BO
+#else
+        movq    KK, %rax
+	leaq	BUFFER1, BO			// first buffer to BO
+	addq	$4 * SIZE, BO
+	movq    %rax, BI                        //  Index for BO
+        leaq    (,BI,2), BI                     //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $2, %rax        // number of values in AO
+#else
+        addq    $1, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$-8, %rax			//  K = K - ( K % 8 )
+	je	.L1_26
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_22:
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL2x1_1(xxx)
+	KERNEL2x1_2(xxx)
+	KERNEL2x1_3(xxx)
+	KERNEL2x1_4(xxx)
+
+	KERNEL2x1_1(xxx)
+	KERNEL2x1_2(xxx)
+	KERNEL2x1_3(xxx)
+	KERNEL2x1_4(xxx)
+
+	je	.L1_26
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL2x1_1(xxx)
+	KERNEL2x1_2(xxx)
+	KERNEL2x1_3(xxx)
+	KERNEL2x1_4(xxx)
+
+	KERNEL2x1_1(xxx)
+	KERNEL2x1_2(xxx)
+	KERNEL2x1_3(xxx)
+	KERNEL2x1_4(xxx)
+
+	je	.L1_26
+
+	jmp	.L1_22
+	ALIGN_4
+
+.L1_26:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_29
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,2), BI                    //  BI = BI * 2; number of values
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_27:
+
+	KERNEL2x1_SUB(xxx)
+	jl	.L1_27
+	ALIGN_4
+
+
+.L1_29:
+
+	vbroadcastss	ALPHA_R, %xmm0
+	vbroadcastss	ALPHA_I, %xmm1
+
+	// swap high and low 64 bytes
+        vshufps $0xb1, %xmm9 , %xmm9, %xmm9
+
+#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
+    defined(NR) || defined(NC) || defined(TR) || defined(TC)
+
+        vaddsubps %xmm9, %xmm8 , %xmm8
+
+        vshufps $0xb1, %xmm8 , %xmm8, %xmm9
+
+#else
+        vaddsubps %xmm8,  %xmm9 ,%xmm9
+
+        vmovaps   %xmm9,  %xmm8
+
+	// swap high and low 64 bytes
+        vshufps $0xb1, %xmm9 , %xmm9, %xmm9
+
+#endif
+
+	// multiply with ALPHA_R
+        vmulps  %xmm8 , %xmm0, %xmm8
+
+	// multiply with ALPHA_I
+        vmulps  %xmm9 , %xmm1, %xmm9
+
+	vaddsubps %xmm9, %xmm8 , %xmm8
+
+
+
+#ifndef TRMMKERNEL
+
+	vaddps 	 	(CO1), %xmm8 , %xmm8
+
+#endif
+
+	vmovups	%xmm8 ,  	(CO1)
+
+
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $2, KK
+#endif
+
+	addq	$4 * SIZE, CO1		# coffset += 4
+	ALIGN_4	
+
+
+
+/**************************************************************************/
+.L1_40:
+	testq	$1, M		
+	jz	.L999		// to next 2 lines of N
+
+	ALIGN_4
+
+.L1_41:
+
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+	leaq	BUFFER1, BO		// first buffer to BO
+	addq	$4 * SIZE, BO
+#else
+        movq    KK, %rax
+	leaq	BUFFER1, BO			// first buffer to BO
+	addq	$4 * SIZE, BO
+	movq    %rax, BI                        //  Index for BO
+        leaq    (,BI,2), BI                     //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO
+	salq	$1, %rax			// rax = rax * 2 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $1, %rax        // number of values in AO
+#else
+        addq    $1, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$-8, %rax			//  K = K - ( K % 8 )
+	je	.L1_46
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values
+
+	salq	$1, %rax			// rax = rax * 2 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_42:
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL1x1_1(xxx)
+	KERNEL1x1_2(xxx)
+	KERNEL1x1_3(xxx)
+	KERNEL1x1_4(xxx)
+
+	KERNEL1x1_1(xxx)
+	KERNEL1x1_2(xxx)
+	KERNEL1x1_3(xxx)
+	KERNEL1x1_4(xxx)
+
+	je	.L1_46
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL1x1_1(xxx)
+	KERNEL1x1_2(xxx)
+	KERNEL1x1_3(xxx)
+	KERNEL1x1_4(xxx)
+
+	KERNEL1x1_1(xxx)
+	KERNEL1x1_2(xxx)
+	KERNEL1x1_3(xxx)
+	KERNEL1x1_4(xxx)
+
+	je	.L1_46
+
+	jmp	.L1_42
+	ALIGN_4
+
+.L1_46:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_49
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values
+
+	salq	$1, %rax			// rax = rax * 2 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_47:
+
+	KERNEL1x1_SUB(xxx)
+	jl	.L1_47
+	ALIGN_4
+
+
+.L1_49:
+
+	vbroadcastss	ALPHA_R, %xmm0
+	vbroadcastss	ALPHA_I, %xmm1
+
+	// swap high and low 64 bytes
+        vshufps $0xb1, %xmm9 , %xmm9, %xmm9
+
+#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
+    defined(NR) || defined(NC) || defined(TR) || defined(TC)
+
+        vaddsubps %xmm9, %xmm8 , %xmm8
+
+        vshufps $0xb1, %xmm8 , %xmm8, %xmm9
+
+#else
+        vaddsubps %xmm8,  %xmm9 ,%xmm9
+
+        vmovaps   %xmm9,  %xmm8
+
+	// swap high and low 64 bytes
+        vshufps $0xb1, %xmm9 , %xmm9, %xmm9
+
+#endif
+
+	// multiply with ALPHA_R
+        vmulps  %xmm8 , %xmm0, %xmm8
+
+	// multiply with ALPHA_I
+        vmulps  %xmm9 , %xmm1, %xmm9
+
+	vaddsubps %xmm9, %xmm8 , %xmm8
+
+
+
+#ifndef TRMMKERNEL
+
+	vmovsd		(CO1), %xmm14
+	vaddps 	 	%xmm14, %xmm8 , %xmm8
+
+#endif
+
+	vmovsd	%xmm8 ,  	(CO1)
+
+
+
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO
+	salq	$1, %rax			// rax = rax * 2 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $1, KK
+#endif
+
+	addq	$2 * SIZE, CO1		# coffset += 2
+	ALIGN_4	
+
+
+
+
+
+
+
+.L999:
+	vzeroupper
+
+	movq   		SP, %rsp
+	movq	   (%rsp), %rbx
+	movq	  8(%rsp), %rbp
+	movq	 16(%rsp), %r12
+	movq	 24(%rsp), %r13
+	movq	 32(%rsp), %r14
+	movq	 40(%rsp), %r15
+
+#ifdef WINDOWS_ABI
+	movq	 48(%rsp), %rdi
+	movq	 56(%rsp), %rsi
+	vmovups	 64(%rsp), %xmm6
+	vmovups	 80(%rsp), %xmm7
+	vmovups	 96(%rsp), %xmm8
+	vmovups	112(%rsp), %xmm9
+	vmovups	128(%rsp), %xmm10
+	vmovups	144(%rsp), %xmm11
+	vmovups	160(%rsp), %xmm12
+	vmovups	176(%rsp), %xmm13
+	vmovups	192(%rsp), %xmm14
+	vmovups	208(%rsp), %xmm15
+#endif
+
+	addq	$STACKSIZE, %rsp
+	ret
+
+	EPILOGUE
diff --git a/kernel/x86_64/cgemm_kernel_8x2_sandy.S b/kernel/x86_64/cgemm_kernel_8x2_sandy.S
index c85646d43..988913591 100644
--- a/kernel/x86_64/cgemm_kernel_8x2_sandy.S
+++ b/kernel/x86_64/cgemm_kernel_8x2_sandy.S
@@ -1,2353 +1,2353 @@
-/*********************************************************************************
-Copyright (c) 2013, The OpenBLAS Project
-All rights reserved.
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in
-the documentation and/or other materials provided with the
-distribution.
-3. Neither the name of the OpenBLAS project nor the names of
-its contributors may be used to endorse or promote products
-derived from this software without specific prior written permission.
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
-USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-**********************************************************************************/
-
-/*********************************************************************
-* 2014/07/29 Saar
-*        BLASTEST               : OK
-*        CTEST                  : OK
-*        TEST                   : OK
-*
-* 2013/10/28 Saar
-* Parameter:
-*       CGEMM_DEFAULT_UNROLL_N  2
-*       CGEMM_DEFAULT_UNROLL_M  8
-*       CGEMM_DEFAULT_P         768
-*       CGEMM_DEFAULT_Q         512
-*       A_PR1                   512
-*       B_PR1                   512
-*
-* 2014/07/29 Saar
-* Performance at 6192x6192x6192:
-*       1 thread:       49 GFLOPS       (MKL:   52)
-*       2 threads:      99 GFLOPS       (MKL:  102)
-*       3 threads:     148 GFLOPS       (MKL:  150)
-*       4 threads:     195 GFLOPS       (MKL:  194)
-*       8 threads:     354 GFLOPS       (MKL:  317)
-*
-*
-*********************************************************************/
-
-
-#define ASSEMBLER
-#include "common.h"
- 
-#define OLD_M	%rdi
-#define OLD_N	%rsi
-#define M	%r13
-#define J	%r14
-#define OLD_K	%rdx
-
-#define A	%rcx
-#define B	%r8
-#define C	%r9
-#define LDC	%r10
-	
-#define I	%r11
-#define AO	%rdi
-#define BO	%rsi
-#define	CO1	%r15
-#define K	%r12
-#define BI	%rbp
-#define	SP	%rbx
-
-#define BO1	%rdi
-#define BO2	%r15
-
-#ifndef WINDOWS_ABI
-
-#define STACKSIZE 96
-
-#else
-
-#define STACKSIZE 320
-
-#define OLD_ALPHA_I     40 + STACKSIZE(%rsp)
-#define OLD_A           48 + STACKSIZE(%rsp)
-#define OLD_B           56 + STACKSIZE(%rsp)
-#define OLD_C           64 + STACKSIZE(%rsp)
-#define OLD_LDC         72 + STACKSIZE(%rsp)
-#define OLD_OFFSET      80 + STACKSIZE(%rsp)
-
-#endif
-
-#define L_BUFFER_SIZE 8192
-
-#define Ndiv6	 24(%rsp)
-#define Nmod6	 32(%rsp)
-#define N	 40(%rsp)
-#define ALPHA_R  48(%rsp)
-#define ALPHA_I  56(%rsp)
-#define OFFSET   64(%rsp)
-#define KK       72(%rsp)
-#define KKK      80(%rsp)
-#define BUFFER1	           128(%rsp)
-
-#if defined(OS_WINDOWS)
-#if   L_BUFFER_SIZE > 16384
-#define STACK_TOUCH \
-        movl    $ 0,  4096 * 4(%rsp);\
-        movl    $ 0,  4096 * 3(%rsp);\
-        movl    $ 0,  4096 * 2(%rsp);\
-        movl    $ 0,  4096 * 1(%rsp);
-#elif L_BUFFER_SIZE > 12288
-#define STACK_TOUCH \
-        movl    $ 0,  4096 * 3(%rsp);\
-        movl    $ 0,  4096 * 2(%rsp);\
-        movl    $ 0,  4096 * 1(%rsp);
-#elif L_BUFFER_SIZE > 8192
-#define STACK_TOUCH \
-        movl    $ 0,  4096 * 2(%rsp);\
-        movl    $ 0,  4096 * 1(%rsp);
-#elif L_BUFFER_SIZE > 4096
-#define STACK_TOUCH \
-        movl    $ 0,  4096 * 1(%rsp);
-#else
-#define STACK_TOUCH
-#endif
-#else
-#define STACK_TOUCH
-#endif
-
-
-
-#if   defined(NN) || defined(NT) || defined(TN) || defined(TT)
-
-#define	VFMADDPS_YR( y0,y1,y2 ) \
-                               vmulps y1,y2,%ymm2;\
-                               vaddps y0,%ymm2,y0
-
-#define	VFMADDPS_YI( y0,y1,y2 ) \
-                               vmulps y1,y2,%ymm3;\
-                               vaddps y0,%ymm3,y0
-
-#define	VFMADDPS_R( y0,y1,y2 ) \
-                               vmulps y1,y2,%xmm2;\
-                               vaddps y0,%xmm2,y0
-
-#define	VFMADDPS_I( y0,y1,y2 ) \
-                               vmulps y1,y2,%xmm3;\
-                               vaddps y0,%xmm3,y0
-
-
-#elif defined(RN) || defined(RT) || defined(CN) || defined(CT)
-
-#define	VFMADDPS_YR( y0,y1,y2 ) \
-                               vmulps y1,y2,%ymm2;\
-                               vsubps %ymm2,y0,y0
-
-#define	VFMADDPS_YI( y0,y1,y2 ) \
-                               vmulps y1,y2,%ymm3;\
-                               vaddps y0,%ymm3,y0
-
-#define	VFMADDPS_R( y0,y1,y2 ) \
-                               vmulps y1,y2,%xmm2;\
-                               vsubps %xmm2,y0,y0
-
-#define	VFMADDPS_I( y0,y1,y2 ) \
-                               vmulps y1,y2,%xmm3;\
-                               vaddps y0,%xmm3,y0
-
-
-#elif defined(NR) || defined(NC) || defined(TR) || defined(TC)
-
-#define	VFMADDPS_YR( y0,y1,y2 ) \
-                               vmulps y1,y2,%ymm2;\
-                               vaddps y0,%ymm2,y0
-
-#define	VFMADDPS_YI( y0,y1,y2 ) \
-                               vmulps y1,y2,%ymm3;\
-                               vsubps %ymm3,y0,y0
-
-#define	VFMADDPS_R( y0,y1,y2 ) \
-                               vmulps y1,y2,%xmm2;\
-                               vaddps y0,%xmm2,y0
-
-#define	VFMADDPS_I( y0,y1,y2 ) \
-                               vmulps y1,y2,%xmm3;\
-                               vsubps %xmm3,y0,y0
-
-
-#else
-
-#define	VFMADDPS_YR( y0,y1,y2 ) \
-                               vmulps y1,y2,%ymm2;\
-                               vsubps %ymm2,y0,y0
-
-#define	VFMADDPS_YI( y0,y1,y2 ) \
-                               vmulps y1,y2,%ymm3;\
-                               vsubps %ymm3,y0,y0
-
-#define	VFMADDPS_R( y0,y1,y2 ) \
-                               vmulps y1,y2,%xmm2;\
-                               vsubps %xmm2,y0,y0
-
-#define	VFMADDPS_I( y0,y1,y2 ) \
-                               vmulps y1,y2,%xmm3;\
-                               vsubps %xmm3,y0,y0
-
-
-#endif
-
-
-#define	A_PR1	512
-#define	B_PR1	512
-
-/***************************************************************************************************************************/
-
-.macro KERNEL8x2_1
-
-        vmovups         -16 * SIZE(AO, %rax, SIZE), %ymm0
-        vbroadcastss         -8 * SIZE(BO, BI, SIZE), %ymm4
-        vmovups          -8 * SIZE(AO, %rax, SIZE), %ymm1
-        vbroadcastss         -7 * SIZE(BO, BI, SIZE), %ymm5
-	prefetcht0	A_PR1(AO, %rax, SIZE)
-
-        VFMADDPS_YR(        %ymm8,%ymm4,%ymm0  )
-        vbroadcastss         -6 * SIZE(BO, BI, SIZE), %ymm6
-        VFMADDPS_YI(        %ymm9,%ymm5,%ymm0  )
-        vbroadcastss         -5 * SIZE(BO, BI, SIZE), %ymm7
-        VFMADDPS_YR(        %ymm12,%ymm4,%ymm1 )
-        VFMADDPS_YI(        %ymm13,%ymm5,%ymm1 )
-
-
-        VFMADDPS_YR(        %ymm10,%ymm6,%ymm0 )
-        vbroadcastss         -4 * SIZE(BO, BI, SIZE), %ymm4
-        VFMADDPS_YI(        %ymm11,%ymm7,%ymm0 )
-        vbroadcastss         -3 * SIZE(BO, BI, SIZE), %ymm5
-        VFMADDPS_YR(        %ymm14,%ymm6,%ymm1 )
-        vmovups           0 * SIZE(AO, %rax, SIZE), %ymm0
-        VFMADDPS_YI(        %ymm15,%ymm7,%ymm1 )
-
-        vmovups           8 * SIZE(AO, %rax, SIZE), %ymm1
-	prefetcht0	A_PR1+64(AO, %rax, SIZE)
-
-        VFMADDPS_YR(        %ymm8,%ymm4,%ymm0  )
-        vbroadcastss         -2 * SIZE(BO, BI, SIZE), %ymm6
-        VFMADDPS_YI(        %ymm9,%ymm5,%ymm0  )
-        vbroadcastss         -1 * SIZE(BO, BI, SIZE), %ymm7
-        VFMADDPS_YR(        %ymm12,%ymm4,%ymm1 )
-        VFMADDPS_YI(        %ymm13,%ymm5,%ymm1 )
-
-
-        VFMADDPS_YR(        %ymm10,%ymm6,%ymm0 )
-        vbroadcastss          0 * SIZE(BO, BI, SIZE), %ymm4
-        VFMADDPS_YI(        %ymm11,%ymm7,%ymm0 )
-        vbroadcastss          1 * SIZE(BO, BI, SIZE), %ymm5
-        VFMADDPS_YR(        %ymm14,%ymm6,%ymm1 )
-        vmovups          16 * SIZE(AO, %rax, SIZE), %ymm0
-        VFMADDPS_YI(        %ymm15,%ymm7,%ymm1 )
-
-        vmovups          24 * SIZE(AO, %rax, SIZE), %ymm1
-	prefetcht0	A_PR1+128(AO, %rax, SIZE)
-
-        VFMADDPS_YR(        %ymm8,%ymm4,%ymm0  )
-        vbroadcastss          2 * SIZE(BO, BI, SIZE), %ymm6
-        VFMADDPS_YI(        %ymm9,%ymm5,%ymm0  )
-        vbroadcastss          3 * SIZE(BO, BI, SIZE), %ymm7
-        VFMADDPS_YR(        %ymm12,%ymm4,%ymm1 )
-        VFMADDPS_YI(        %ymm13,%ymm5,%ymm1 )
-
-
-        VFMADDPS_YR(        %ymm10,%ymm6,%ymm0 )
-        vbroadcastss          4 * SIZE(BO, BI, SIZE), %ymm4
-        VFMADDPS_YI(        %ymm11,%ymm7,%ymm0 )
-        vbroadcastss          5 * SIZE(BO, BI, SIZE), %ymm5
-        VFMADDPS_YR(        %ymm14,%ymm6,%ymm1 )
-        vmovups          32 * SIZE(AO, %rax, SIZE), %ymm0
-        VFMADDPS_YI(        %ymm15,%ymm7,%ymm1 )
-
-        vmovups          40 * SIZE(AO, %rax, SIZE), %ymm1
-	prefetcht0	A_PR1+192(AO, %rax, SIZE)
-
-        VFMADDPS_YR(        %ymm8,%ymm4,%ymm0  )
-        vbroadcastss          6 * SIZE(BO, BI, SIZE), %ymm6
-        VFMADDPS_YI(        %ymm9,%ymm5,%ymm0  )
-        vbroadcastss          7 * SIZE(BO, BI, SIZE), %ymm7
-        VFMADDPS_YR(        %ymm12,%ymm4,%ymm1 )
-        VFMADDPS_YI(        %ymm13,%ymm5,%ymm1 )
-
-        VFMADDPS_YR(        %ymm10,%ymm6,%ymm0 )
-        VFMADDPS_YI(        %ymm11,%ymm7,%ymm0 )
-        addq    $ 16, BI                           
-        VFMADDPS_YR(        %ymm14,%ymm6,%ymm1 )
-        VFMADDPS_YI(        %ymm15,%ymm7,%ymm1 )
-
-        addq    $ 64, %rax                         
-.endm
-
-
-.macro KERNEL8x2_SUB
-
-        vmovups         -16 * SIZE(AO, %rax, SIZE), %ymm0
-        vmovups          -8 * SIZE(AO, %rax, SIZE), %ymm1
-        vbroadcastss         -8 * SIZE(BO, BI, SIZE), %ymm4
-        vbroadcastss         -7 * SIZE(BO, BI, SIZE), %ymm5
-
-        VFMADDPS_YR(        %ymm8,%ymm4,%ymm0  )
-        vbroadcastss         -6 * SIZE(BO, BI, SIZE), %ymm6
-        VFMADDPS_YI(        %ymm9,%ymm5,%ymm0  )
-        vbroadcastss         -5 * SIZE(BO, BI, SIZE), %ymm7
-        VFMADDPS_YR(        %ymm12,%ymm4,%ymm1 )
-        VFMADDPS_YI(        %ymm13,%ymm5,%ymm1 )
-
-
-        VFMADDPS_YR(        %ymm10,%ymm6,%ymm0 )
-        VFMADDPS_YI(        %ymm11,%ymm7,%ymm0 )
-        VFMADDPS_YR(        %ymm14,%ymm6,%ymm1 )
-        VFMADDPS_YI(        %ymm15,%ymm7,%ymm1 )
-
-        addq    $ 4 , BI                           
-        addq    $ 16, %rax                         
-.endm
-
-.macro SAVE8x2
-
-	vbroadcastss	ALPHA_R, %ymm0
-	vbroadcastss	ALPHA_I, %ymm1
-
-	// swap high and low 64 bytes
-        vshufps $ 0xb1, %ymm9 , %ymm9, %ymm9
-        vshufps $ 0xb1, %ymm11, %ymm11, %ymm11
-        vshufps $ 0xb1, %ymm13, %ymm13, %ymm13
-        vshufps $ 0xb1, %ymm15, %ymm15, %ymm15
-
-#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
-    defined(NR) || defined(NC) || defined(TR) || defined(TC)
-
-        vaddsubps %ymm9, %ymm8 , %ymm8
-        vaddsubps %ymm11,%ymm10, %ymm10
-        vaddsubps %ymm13,%ymm12, %ymm12
-        vaddsubps %ymm15,%ymm14, %ymm14
-
-        vshufps $ 0xb1, %ymm8 , %ymm8, %ymm9
-        vshufps $ 0xb1, %ymm10, %ymm10, %ymm11
-        vshufps $ 0xb1, %ymm12, %ymm12, %ymm13
-        vshufps $ 0xb1, %ymm14, %ymm14, %ymm15
-
-#else
-        vaddsubps %ymm8,  %ymm9 ,%ymm9
-        vaddsubps %ymm10, %ymm11,%ymm11
-        vaddsubps %ymm12, %ymm13,%ymm13
-        vaddsubps %ymm14, %ymm15,%ymm15
-
-        vmovaps   %ymm9,  %ymm8
-        vmovaps   %ymm11, %ymm10
-        vmovaps   %ymm13, %ymm12
-        vmovaps   %ymm15, %ymm14
-
-	// swap high and low 64 bytes
-        vshufps $ 0xb1, %ymm9 , %ymm9, %ymm9
-        vshufps $ 0xb1, %ymm11, %ymm11, %ymm11
-        vshufps $ 0xb1, %ymm13, %ymm13, %ymm13
-        vshufps $ 0xb1, %ymm15, %ymm15, %ymm15
-
-#endif
-
-	// multiply with ALPHA_R
-        vmulps  %ymm8 , %ymm0, %ymm8
-        vmulps  %ymm10, %ymm0, %ymm10
-        vmulps  %ymm12, %ymm0, %ymm12
-        vmulps  %ymm14, %ymm0, %ymm14
-
-	// multiply with ALPHA_I
-        vmulps  %ymm9 , %ymm1, %ymm9
-        vmulps  %ymm11, %ymm1, %ymm11
-        vmulps  %ymm13, %ymm1, %ymm13
-        vmulps  %ymm15, %ymm1, %ymm15
-
-	vaddsubps %ymm9, %ymm8 , %ymm8
-        vaddsubps %ymm11,%ymm10, %ymm10
-        vaddsubps %ymm13,%ymm12, %ymm12
-        vaddsubps %ymm15,%ymm14, %ymm14
-
-
-
-#ifndef TRMMKERNEL
-
-	vaddps 	 	(CO1), %ymm8 , %ymm8
-	vaddps  8 * SIZE(CO1), %ymm12, %ymm12
-
-	vaddps 	 	(CO1, LDC), %ymm10, %ymm10
-	vaddps  8 * SIZE(CO1, LDC), %ymm14, %ymm14
-
-#endif
-
-	vmovups	%ymm8 ,  	(CO1)
-	vmovups	%ymm12 , 8 * SIZE(CO1)
-
-	vmovups	%ymm10 ,  	(CO1, LDC)
-	vmovups	%ymm14 , 8 * SIZE(CO1, LDC)
-
-	prefetcht0	64(CO1)
-	prefetcht0	64(CO1, LDC)
-
-.endm
-
-/***************************************************************************************************************************/
-
-.macro KERNEL4x2_SUB
-        vmovups         -16 * SIZE(AO, %rax, SIZE), %xmm0
-        vbroadcastss         -8 * SIZE(BO, BI, SIZE), %xmm4
-        VFMADDPS_R(        %xmm8,%xmm4,%xmm0  )
-        vmovups         -12 * SIZE(AO, %rax, SIZE), %xmm1
-        VFMADDPS_R(        %xmm12,%xmm4,%xmm1 )
-        vbroadcastss         -7 * SIZE(BO, BI, SIZE), %xmm5
-        VFMADDPS_I(        %xmm9,%xmm5,%xmm0  )
-        VFMADDPS_I(        %xmm13,%xmm5,%xmm1 )
-        vbroadcastss         -6 * SIZE(BO, BI, SIZE), %xmm6
-        VFMADDPS_R(        %xmm10,%xmm6,%xmm0 )
-        VFMADDPS_R(        %xmm14,%xmm6,%xmm1 )
-        vbroadcastss         -5 * SIZE(BO, BI, SIZE), %xmm7
-        VFMADDPS_I(        %xmm11,%xmm7,%xmm0 )
-        VFMADDPS_I(        %xmm15,%xmm7,%xmm1 )
-        addq    $ 4, BI                           
-        addq    $ 8, %rax                         
-.endm
-
-.macro SAVE4x2
-
-	vbroadcastss	ALPHA_R, %xmm0
-	vbroadcastss	ALPHA_I, %xmm1
-
-	// swap high and low 64 bytes
-        vshufps $ 0xb1, %xmm9 , %xmm9, %xmm9
-        vshufps $ 0xb1, %xmm11, %xmm11, %xmm11
-        vshufps $ 0xb1, %xmm13, %xmm13, %xmm13
-        vshufps $ 0xb1, %xmm15, %xmm15, %xmm15
-
-#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
-    defined(NR) || defined(NC) || defined(TR) || defined(TC)
-
-        vaddsubps %xmm9, %xmm8 , %xmm8
-        vaddsubps %xmm11,%xmm10, %xmm10
-        vaddsubps %xmm13,%xmm12, %xmm12
-        vaddsubps %xmm15,%xmm14, %xmm14
-
-        vshufps $ 0xb1, %xmm8 , %xmm8, %xmm9
-        vshufps $ 0xb1, %xmm10, %xmm10, %xmm11
-        vshufps $ 0xb1, %xmm12, %xmm12, %xmm13
-        vshufps $ 0xb1, %xmm14, %xmm14, %xmm15
-
-#else
-        vaddsubps %xmm8,  %xmm9 ,%xmm9
-        vaddsubps %xmm10, %xmm11,%xmm11
-        vaddsubps %xmm12, %xmm13,%xmm13
-        vaddsubps %xmm14, %xmm15,%xmm15
-
-        vmovaps   %xmm9,  %xmm8
-        vmovaps   %xmm11, %xmm10
-        vmovaps   %xmm13, %xmm12
-        vmovaps   %xmm15, %xmm14
-
-	// swap high and low 64 bytes
-        vshufps $ 0xb1, %xmm9 , %xmm9, %xmm9
-        vshufps $ 0xb1, %xmm11, %xmm11, %xmm11
-        vshufps $ 0xb1, %xmm13, %xmm13, %xmm13
-        vshufps $ 0xb1, %xmm15, %xmm15, %xmm15
-
-#endif
-
-	// multiply with ALPHA_R
-        vmulps  %xmm8 , %xmm0, %xmm8
-        vmulps  %xmm10, %xmm0, %xmm10
-        vmulps  %xmm12, %xmm0, %xmm12
-        vmulps  %xmm14, %xmm0, %xmm14
-
-	// multiply with ALPHA_I
-        vmulps  %xmm9 , %xmm1, %xmm9
-        vmulps  %xmm11, %xmm1, %xmm11
-        vmulps  %xmm13, %xmm1, %xmm13
-        vmulps  %xmm15, %xmm1, %xmm15
-
-	vaddsubps %xmm9, %xmm8 , %xmm8
-        vaddsubps %xmm11,%xmm10, %xmm10
-        vaddsubps %xmm13,%xmm12, %xmm12
-        vaddsubps %xmm15,%xmm14, %xmm14
-
-#ifndef TRMMKERNEL
-
-	vaddps 	 	(CO1), %xmm8 , %xmm8
-	vaddps  4 * SIZE(CO1), %xmm12, %xmm12
-
-	vaddps 	 	(CO1, LDC), %xmm10, %xmm10
-	vaddps  4 * SIZE(CO1, LDC), %xmm14, %xmm14
-
-#endif
-
-	vmovups	%xmm8 ,  	(CO1)
-	vmovups	%xmm12 , 4 * SIZE(CO1)
-
-	vmovups	%xmm10 ,  	(CO1, LDC)
-	vmovups	%xmm14 , 4 * SIZE(CO1, LDC)
-
-.endm
-
-/************************************************************************************************/
-
-.macro KERNEL2x2_SUB
-        vmovups         -16 * SIZE(AO, %rax, SIZE), %xmm0
-        vbroadcastss         -8 * SIZE(BO, BI, SIZE), %xmm4
-        VFMADDPS_R(        %xmm8,%xmm4,%xmm0  )
-        vbroadcastss         -7 * SIZE(BO, BI, SIZE), %xmm5
-        VFMADDPS_I(        %xmm9,%xmm5,%xmm0  )
-        vbroadcastss         -6 * SIZE(BO, BI, SIZE), %xmm6
-        VFMADDPS_R(        %xmm10,%xmm6,%xmm0 )
-        vbroadcastss         -5 * SIZE(BO, BI, SIZE), %xmm7
-        VFMADDPS_I(        %xmm11,%xmm7,%xmm0 )
-        addq    $ 4, BI                           
-        addq    $ 4, %rax                         
-.endm
-
-.macro SAVE2x2
-
-	vbroadcastss	ALPHA_R, %xmm0
-	vbroadcastss	ALPHA_I, %xmm1
-
-	// swap high and low 4 bytes
-        vshufps $ 0xb1, %xmm9 , %xmm9, %xmm9
-        vshufps $ 0xb1, %xmm11, %xmm11, %xmm11
-
-#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
-    defined(NR) || defined(NC) || defined(TR) || defined(TC)
-
-        vaddsubps %xmm9, %xmm8 , %xmm8
-        vaddsubps %xmm11,%xmm10, %xmm10
-
-        vshufps $ 0xb1, %xmm8 , %xmm8, %xmm9
-        vshufps $ 0xb1, %xmm10, %xmm10, %xmm11
-
-#else
-        vaddsubps %xmm8,  %xmm9 ,%xmm9
-        vaddsubps %xmm10, %xmm11,%xmm11
-
-        vmovaps   %xmm9,  %xmm8
-        vmovaps   %xmm11, %xmm10
-
-	// swap high and low 4 bytes
-        vshufps $ 0xb1, %xmm9 , %xmm9, %xmm9
-        vshufps $ 0xb1, %xmm11, %xmm11, %xmm11
-
-#endif
-
-	// multiply with ALPHA_R
-        vmulps  %xmm8 , %xmm0, %xmm8
-        vmulps  %xmm10, %xmm0, %xmm10
-
-	// multiply with ALPHA_I
-        vmulps  %xmm9 , %xmm1, %xmm9
-        vmulps  %xmm11, %xmm1, %xmm11
-
-	vaddsubps %xmm9, %xmm8 , %xmm8
-        vaddsubps %xmm11,%xmm10, %xmm10
-
-#ifndef TRMMKERNEL
-
-	vaddps 	 	(CO1), %xmm8 , %xmm8
-
-	vaddps 	 	(CO1, LDC), %xmm10, %xmm10
-
-#endif
-
-	vmovups	%xmm8 ,  	(CO1)
-
-	vmovups	%xmm10 ,  	(CO1, LDC)
-
-.endm
-
-/************************************************************************************************/
-
-.macro KERNEL1x2_SUB
-        vmovsd         -16 * SIZE(AO, %rax, SIZE), %xmm0
-        vbroadcastss         -8 * SIZE(BO, BI, SIZE), %xmm4
-        VFMADDPS_R(        %xmm8,%xmm4,%xmm0  )
-        vbroadcastss         -7 * SIZE(BO, BI, SIZE), %xmm5
-        VFMADDPS_I(        %xmm9,%xmm5,%xmm0  )
-        vbroadcastss         -6 * SIZE(BO, BI, SIZE), %xmm6
-        VFMADDPS_R(        %xmm10,%xmm6,%xmm0 )
-        vbroadcastss         -5 * SIZE(BO, BI, SIZE), %xmm7
-        VFMADDPS_I(        %xmm11,%xmm7,%xmm0 )
-        addq    $ 4, BI                           
-        addq    $ 2, %rax                         
-.endm
-
-.macro SAVE1x2
-
-	vbroadcastss	ALPHA_R, %xmm0
-	vbroadcastss	ALPHA_I, %xmm1
-
-	// swap high and low 64 bytes
-        vshufps $ 0xb1, %xmm9 , %xmm9, %xmm9
-        vshufps $ 0xb1, %xmm11, %xmm11, %xmm11
-
-#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
-    defined(NR) || defined(NC) || defined(TR) || defined(TC)
-
-        vaddsubps %xmm9, %xmm8 , %xmm8
-        vaddsubps %xmm11,%xmm10, %xmm10
-
-        vshufps $ 0xb1, %xmm8 , %xmm8, %xmm9
-        vshufps $ 0xb1, %xmm10, %xmm10, %xmm11
-
-#else
-        vaddsubps %xmm8,  %xmm9 ,%xmm9
-        vaddsubps %xmm10, %xmm11,%xmm11
-
-        vmovaps   %xmm9,  %xmm8
-        vmovaps   %xmm11, %xmm10
-
-	// swap high and low 64 bytes
-        vshufps $ 0xb1, %xmm9 , %xmm9, %xmm9
-        vshufps $ 0xb1, %xmm11, %xmm11, %xmm11
-
-#endif
-
-	// multiply with ALPHA_R
-        vmulps  %xmm8 , %xmm0, %xmm8
-        vmulps  %xmm10, %xmm0, %xmm10
-
-	// multiply with ALPHA_I
-        vmulps  %xmm9 , %xmm1, %xmm9
-        vmulps  %xmm11, %xmm1, %xmm11
-
-	vaddsubps %xmm9, %xmm8 , %xmm8
-        vaddsubps %xmm11,%xmm10, %xmm10
-
-#ifndef TRMMKERNEL
-
-	vmovsd		(CO1), %xmm14
-	vaddps 	 	%xmm14, %xmm8 , %xmm8
-
-	vmovsd		(CO1, LDC), %xmm15
-	vaddps 	 	%xmm15, %xmm10, %xmm10
-
-#endif
-
-	vmovsd	%xmm8 ,  	(CO1)
-	vmovsd	%xmm10 ,  	(CO1, LDC)
-
-.endm
-
-/************************************************************************************************/
-
-.macro KERNEL8x1_SUB
-        vmovups         -16 * SIZE(AO, %rax, SIZE), %ymm0
-        vmovups          -8 * SIZE(AO, %rax, SIZE), %ymm1
-        vbroadcastss         -4 * SIZE(BO, BI, SIZE), %ymm4
-        VFMADDPS_YR(        %ymm8,%ymm4,%ymm0  )
-        VFMADDPS_YR(        %ymm12,%ymm4,%ymm1 )
-        vbroadcastss         -3 * SIZE(BO, BI, SIZE), %ymm5
-        VFMADDPS_YI(        %ymm9,%ymm5,%ymm0  )
-        VFMADDPS_YI(        %ymm13,%ymm5,%ymm1 )
-        addq    $ 2 , BI                           
-        addq    $ 16, %rax                         
-.endm
-
-.macro SAVE8x1
-
-	vbroadcastss	ALPHA_R, %ymm0
-	vbroadcastss	ALPHA_I, %ymm1
-
-	// swap high and low 64 bytes
-        vshufps $ 0xb1, %ymm9 , %ymm9, %ymm9
-        vshufps $ 0xb1, %ymm13, %ymm13, %ymm13
-
-#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
-    defined(NR) || defined(NC) || defined(TR) || defined(TC)
-
-        vaddsubps %ymm9, %ymm8 , %ymm8
-        vaddsubps %ymm13,%ymm12, %ymm12
-
-        vshufps $ 0xb1, %ymm8 , %ymm8, %ymm9
-        vshufps $ 0xb1, %ymm12, %ymm12, %ymm13
-
-#else
-        vaddsubps %ymm8,  %ymm9 ,%ymm9
-        vaddsubps %ymm12, %ymm13,%ymm13
-
-        vmovaps   %ymm9,  %ymm8
-        vmovaps   %ymm13, %ymm12
-
-	// swap high and low 64 bytes
-        vshufps $ 0xb1, %ymm9 , %ymm9, %ymm9
-        vshufps $ 0xb1, %ymm13, %ymm13, %ymm13
-
-#endif
-
-	// multiply with ALPHA_R
-        vmulps  %ymm8 , %ymm0, %ymm8
-        vmulps  %ymm12, %ymm0, %ymm12
-
-	// multiply with ALPHA_I
-        vmulps  %ymm9 , %ymm1, %ymm9
-        vmulps  %ymm13, %ymm1, %ymm13
-
-	vaddsubps %ymm9, %ymm8 , %ymm8
-        vaddsubps %ymm13,%ymm12, %ymm12
-
-
-
-#ifndef TRMMKERNEL
-
-	vaddps 	 	(CO1), %ymm8 , %ymm8
-	vaddps  8 * SIZE(CO1), %ymm12, %ymm12
-
-#endif
-
-	vmovups	%ymm8 ,  	(CO1)
-	vmovups	%ymm12 , 8 * SIZE(CO1)
-
-.endm
-
-
-/************************************************************************************************/
-
-.macro KERNEL4x1_SUB
-        vmovups         -16 * SIZE(AO, %rax, SIZE), %xmm0
-        vbroadcastss         -4 * SIZE(BO, BI, SIZE), %xmm4
-        VFMADDPS_R(        %xmm8,%xmm4,%xmm0  )
-        vmovups         -12 * SIZE(AO, %rax, SIZE), %xmm1
-        VFMADDPS_R(        %xmm12,%xmm4,%xmm1 )
-        vbroadcastss         -3 * SIZE(BO, BI, SIZE), %xmm5
-        VFMADDPS_I(        %xmm9,%xmm5,%xmm0  )
-        VFMADDPS_I(        %xmm13,%xmm5,%xmm1 )
-        addq    $ 2, BI                           
-        addq    $ 8, %rax                         
-.endm
-
-.macro SAVE4x1
-
-	vbroadcastss	ALPHA_R, %xmm0
-	vbroadcastss	ALPHA_I, %xmm1
-
-	// swap high and low 4 bytes
-        vshufps $ 0xb1, %xmm9 , %xmm9, %xmm9
-        vshufps $ 0xb1, %xmm13, %xmm13, %xmm13
-
-#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
-    defined(NR) || defined(NC) || defined(TR) || defined(TC)
-
-        vaddsubps %xmm9, %xmm8 , %xmm8
-        vaddsubps %xmm13,%xmm12, %xmm12
-
-        vshufps $ 0xb1, %xmm8 , %xmm8, %xmm9
-        vshufps $ 0xb1, %xmm12, %xmm12, %xmm13
-
-#else
-        vaddsubps %xmm8,  %xmm9 ,%xmm9
-        vaddsubps %xmm12, %xmm13,%xmm13
-
-        vmovaps   %xmm9,  %xmm8
-        vmovaps   %xmm13, %xmm12
-
-	// swap high and low 4 bytes
-        vshufps $ 0xb1, %xmm9 , %xmm9, %xmm9
-        vshufps $ 0xb1, %xmm13, %xmm13, %xmm13
-
-#endif
-
-	// multiply with ALPHA_R
-        vmulps  %xmm8 , %xmm0, %xmm8
-        vmulps  %xmm12, %xmm0, %xmm12
-
-	// multiply with ALPHA_I
-        vmulps  %xmm9 , %xmm1, %xmm9
-        vmulps  %xmm13, %xmm1, %xmm13
-
-	vaddsubps %xmm9, %xmm8 , %xmm8
-        vaddsubps %xmm13,%xmm12, %xmm12
-
-#ifndef TRMMKERNEL
-
-	vaddps 	 	(CO1), %xmm8 , %xmm8
-	vaddps  4 * SIZE(CO1), %xmm12, %xmm12
-
-#endif
-
-	vmovups	%xmm8 ,  	(CO1)
-	vmovups	%xmm12 , 4 * SIZE(CO1)
-
-.endm
-
-/************************************************************************************************/
-
-.macro KERNEL2x1_SUB
-        vmovups         -16 * SIZE(AO, %rax, SIZE), %xmm0
-        vbroadcastss         -4 * SIZE(BO, BI, SIZE), %xmm4
-        VFMADDPS_R(        %xmm8,%xmm4,%xmm0  )
-        vbroadcastss         -3 * SIZE(BO, BI, SIZE), %xmm5
-        VFMADDPS_I(        %xmm9,%xmm5,%xmm0  )
-        addq    $ 2, BI                           
-        addq    $ 4, %rax                         
-.endm
-
-.macro SAVE2x1
-
-	vbroadcastss	ALPHA_R, %xmm0
-	vbroadcastss	ALPHA_I, %xmm1
-
-	// swap high and low 64 bytes
-        vshufps $ 0xb1, %xmm9 , %xmm9, %xmm9
-
-#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
-    defined(NR) || defined(NC) || defined(TR) || defined(TC)
-
-        vaddsubps %xmm9, %xmm8 , %xmm8
-
-        vshufps $ 0xb1, %xmm8 , %xmm8, %xmm9
-
-#else
-        vaddsubps %xmm8,  %xmm9 ,%xmm9
-
-        vmovaps   %xmm9,  %xmm8
-
-	// swap high and low 64 bytes
-        vshufps $ 0xb1, %xmm9 , %xmm9, %xmm9
-
-#endif
-
-	// multiply with ALPHA_R
-        vmulps  %xmm8 , %xmm0, %xmm8
-
-	// multiply with ALPHA_I
-        vmulps  %xmm9 , %xmm1, %xmm9
-
-	vaddsubps %xmm9, %xmm8 , %xmm8
-
-#ifndef TRMMKERNEL
-
-	vaddps 	 	(CO1), %xmm8 , %xmm8
-
-#endif
-
-	vmovups	%xmm8 ,  	(CO1)
-
-.endm
-
-/************************************************************************************************/
-
-.macro KERNEL1x1_SUB
-        vmovsd         -16 * SIZE(AO, %rax, SIZE), %xmm0
-        vbroadcastss         -4 * SIZE(BO, BI, SIZE), %xmm4
-        VFMADDPS_R(        %xmm8,%xmm4,%xmm0 )
-        vbroadcastss         -3 * SIZE(BO, BI, SIZE), %xmm5
-        VFMADDPS_I(        %xmm9,%xmm5,%xmm0 )
-        addq    $ 2, BI                           
-        addq    $ 2, %rax                         
-.endm
-
-.macro SAVE1x1
-
-	vbroadcastss	ALPHA_R, %xmm0
-	vbroadcastss	ALPHA_I, %xmm1
-
-	// swap high and low 64 bytes
-        vshufps $ 0xb1, %xmm9 , %xmm9, %xmm9
-
-#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
-    defined(NR) || defined(NC) || defined(TR) || defined(TC)
-
-        vaddsubps %xmm9, %xmm8 , %xmm8
-
-        vshufps $ 0xb1, %xmm8 , %xmm8, %xmm9
-
-#else
-        vaddsubps %xmm8,  %xmm9 ,%xmm9
-
-        vmovaps   %xmm9,  %xmm8
-
-	// swap high and low 64 bytes
-        vshufps $ 0xb1, %xmm9 , %xmm9, %xmm9
-
-#endif
-
-	// multiply with ALPHA_R
-        vmulps  %xmm8 , %xmm0, %xmm8
-
-	// multiply with ALPHA_I
-        vmulps  %xmm9 , %xmm1, %xmm9
-
-	vaddsubps %xmm9, %xmm8 , %xmm8
-
-#ifndef TRMMKERNEL
-
-	vmovsd		(CO1), %xmm14
-	vaddps 	 	%xmm14, %xmm8 , %xmm8
-
-#endif
-
-	vmovsd	%xmm8 ,  	(CO1)
-
-.endm
-
-/************************************************************************************************/
-
-
-
-
-	PROLOGUE
-	PROFCODE
-	
-	subq	$ STACKSIZE, %rsp
-	movq	%rbx,   (%rsp)
-	movq	%rbp,  8(%rsp)
-	movq	%r12, 16(%rsp)
-	movq	%r13, 24(%rsp)
-	movq	%r14, 32(%rsp)
-	movq	%r15, 40(%rsp)
-
-	vzeroupper
-
-#ifdef WINDOWS_ABI
-	movq	%rdi,    48(%rsp)
-	movq	%rsi,    56(%rsp)
-	vmovups	%xmm6,   64(%rsp)
-	vmovups	%xmm7,   80(%rsp)
-	vmovups	%xmm8,   96(%rsp)
-	vmovups	%xmm9,  112(%rsp)
-	vmovups	%xmm10, 128(%rsp)
-	vmovups	%xmm11, 144(%rsp)
-	vmovups	%xmm12, 160(%rsp)
-	vmovups	%xmm13, 176(%rsp)
-	vmovups	%xmm14, 192(%rsp)
-	vmovups	%xmm15, 208(%rsp)
-
-	movq	ARG1,      OLD_M
-	movq	ARG2,      OLD_N
-	movq	ARG3,      OLD_K
-	movq	OLD_A,     A
-	movq	OLD_B,     B
-	movq	OLD_C,     C
-	movq	OLD_LDC,   LDC
-#ifdef TRMMKERNEL
-	movsd	OLD_OFFSET, %xmm12
-#endif
-	vmovaps	%xmm3, %xmm0
-	vmovsd   OLD_ALPHA_I, %xmm1
-
-#else
-	movq	STACKSIZE +  8(%rsp), LDC
-#ifdef TRMMKERNEL
-	movsd	STACKSIZE + 16(%rsp), %xmm12
-#endif
-
-#endif
-
-	movq    %rsp, SP      # save old stack
-        subq    $ 128 + L_BUFFER_SIZE, %rsp
-        andq    $ -4096, %rsp    # align stack
-
-        STACK_TOUCH
-
-	cmpq	$ 0, OLD_M
-	je	.L999
-
-	cmpq	$ 0, OLD_N
-	je	.L999
-
-	cmpq	$ 0, OLD_K
-	je	.L999
-
-	movq	OLD_M, M
-	movq	OLD_N, N
-	movq	OLD_K, K
-
-	vmovss	 %xmm0, ALPHA_R
-	vmovss	 %xmm1, ALPHA_I
-
-	salq	$ ZBASE_SHIFT, LDC
-
-	movq    N, %rax
-        xorq    %rdx, %rdx
-        movq    $ 2,  %rdi
-        divq    %rdi                    //    N / 2
-        movq    %rax, Ndiv6             //    N / 2
-        movq    %rdx, Nmod6             //    N % 2
-
-	
-
-#ifdef TRMMKERNEL
-	vmovsd	%xmm12, OFFSET
-	vmovsd	%xmm12, KK
-#ifndef LEFT
-	negq	KK
-#endif	
-#endif
-
-.L2_0:
-
-	movq	Ndiv6,  J
-	cmpq	$ 0, J
-	je	.L1_0
-	ALIGN_4
-
-
-
-.L2_01:
-	// copy to sub buffer
-	movq	B, BO1
-	leaq    BUFFER1, BO		// first buffer to BO
-	movq	K, %rax
-	ALIGN_4
-
-.L2_02b:
-
-	vmovups	(BO1), %xmm0
-	vmovups	%xmm0,       (BO)
-	addq	$ 4*SIZE,BO1
-	addq	$ 4*SIZE,BO
-	decq	%rax
-	jnz	.L2_02b
-
-.L2_02c:
-
-	movq	BO1, B			// next offset of B
-
-.L2_10:
-	movq	C, CO1
-	leaq	(C, LDC, 2), C		// c += 2 * ldc
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        movq    OFFSET, %rax
-        movq    %rax, KK
-#endif
-	
-	movq	A, AO		 	// aoffset = a
-	addq	$ 16 * SIZE, AO
-
-	movq	M,  I
-	sarq	$ 3, I			// i = (m >> 3)
-	je	.L2_4_10
-
-	ALIGN_4
-/**********************************************************************************************************/
-
-.L2_8_11:
-
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-	leaq	BUFFER1, BO		// first buffer to BO
-	addq	$ 8 * SIZE, BO
-#else
-        movq    KK, %rax
-	leaq	BUFFER1, BO			// first buffer to BO
-	addq	$ 8 * SIZE, BO
-	movq    %rax, BI                        //  Index for BO
-        leaq    (,BI,4), BI                     //  BI = BI * 4 ; number of values
-        leaq    (BO, BI, SIZE), BO
-	salq	$ 4, %rax			// rax = rax *16 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $ 8, %rax        // number of values in AO
-#else
-        addq    $ 2, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-
-	andq	$ -8, %rax			//  K = K - ( K % 8 )
-	je	.L2_8_16
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
-
-	salq	$ 4, %rax			// rax = rax *16 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_8_12:
-
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL8x2_1
-
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL8x2_1
-
-	je	.L2_8_16
-
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL8x2_1
-
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL8x2_1
-
-	je	.L2_8_16
-
-	jmp	.L2_8_12
-	ALIGN_4
-
-.L2_8_16:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$ 7, %rax		# if (k & 1)
-	je .L2_8_19
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
-
-	salq	$ 4, %rax			// rax = rax *16 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_8_17:
-
-	KERNEL8x2_SUB
-
-	jl	.L2_8_17
-	ALIGN_4
-
-
-.L2_8_19:
-
-	SAVE8x2
-
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
-        leaq    (BO, BI, SIZE), BO
-	salq	$ 4, %rax			// rax = rax *16 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $ 8, KK
-#endif
-
-	addq	$ 16 * SIZE, CO1		# coffset += 16
-	decq	I			# i --
-	jg	.L2_8_11
-	ALIGN_4	
-
-
-/**********************************************************************************************************/
-
-
-
-
-.L2_4_10:
-	testq	$ 7, M		
-	jz	.L2_4_60		// to next 2 lines of N
-
-	testq	$ 4, M		
-	jz	.L2_4_20
-	ALIGN_4
-
-
-.L2_4_11:
-
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-	leaq	BUFFER1, BO		// first buffer to BO
-	addq	$ 8 * SIZE, BO
-#else
-        movq    KK, %rax
-	leaq	BUFFER1, BO			// first buffer to BO
-	addq	$ 8 * SIZE, BO
-	movq    %rax, BI                        //  Index for BO
-        leaq    (,BI,4), BI                     //  BI = BI * 4 ; number of values
-        leaq    (BO, BI, SIZE), BO
-	salq	$ 3, %rax			// rax = rax * 8 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $ 4, %rax        // number of values in AO
-#else
-        addq    $ 2, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-
-	andq	$ -8, %rax			//  K = K - ( K % 8 )
-	je	.L2_4_16
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
-
-	salq	$ 3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_4_12:
-
-	prefetcht0	A_PR1(AO,%rax,SIZE)
-	KERNEL4x2_SUB
-	KERNEL4x2_SUB
-	prefetcht0	A_PR1(AO,%rax,SIZE)
-	KERNEL4x2_SUB
-	KERNEL4x2_SUB
-
-	prefetcht0	A_PR1(AO,%rax,SIZE)
-	KERNEL4x2_SUB
-	KERNEL4x2_SUB
-	prefetcht0	A_PR1(AO,%rax,SIZE)
-	KERNEL4x2_SUB
-	KERNEL4x2_SUB
-
-	je	.L2_4_16
-
-	prefetcht0	A_PR1(AO,%rax,SIZE)
-	KERNEL4x2_SUB
-	KERNEL4x2_SUB
-	prefetcht0	A_PR1(AO,%rax,SIZE)
-	KERNEL4x2_SUB
-	KERNEL4x2_SUB
-
-	prefetcht0	A_PR1(AO,%rax,SIZE)
-	KERNEL4x2_SUB
-	KERNEL4x2_SUB
-	prefetcht0	A_PR1(AO,%rax,SIZE)
-	KERNEL4x2_SUB
-	KERNEL4x2_SUB
-
-	je	.L2_4_16
-
-	jmp	.L2_4_12
-	ALIGN_4
-
-.L2_4_16:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$ 7, %rax		# if (k & 1)
-	je .L2_4_19
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
-
-	salq	$ 3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_4_17:
-
-	KERNEL4x2_SUB
-
-	jl	.L2_4_17
-	ALIGN_4
-
-
-.L2_4_19:
-
-	SAVE4x2
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
-        leaq    (BO, BI, SIZE), BO
-	salq	$ 3, %rax			// rax = rax * 8 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $ 4, KK
-#endif
-
-	addq	$ 8 * SIZE, CO1		# coffset += 8
-	ALIGN_4	
-
-
-
-/**************************************************************************
-* Rest of M 
-***************************************************************************/
-
-.L2_4_20:
-
-	testq	$ 2, M		
-	jz	.L2_4_40
-	ALIGN_4
-
-.L2_4_21:
-
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-	leaq	BUFFER1, BO		// first buffer to BO
-	addq	$ 8 * SIZE, BO
-#else
-        movq    KK, %rax
-	leaq	BUFFER1, BO			// first buffer to BO
-	addq	$ 8 * SIZE, BO
-	movq    %rax, BI                        //  Index for BO
-        leaq    (,BI,4), BI                     //  BI = BI * 4 ; number of values
-        leaq    (BO, BI, SIZE), BO
-	salq	$ 2, %rax			// rax = rax * 4 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $ 2, %rax        // number of values in AO
-#else
-        addq    $ 2, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-
-	andq	$ -8, %rax			//  K = K - ( K % 8 )
-	je	.L2_4_26
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
-
-	salq	$ 2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_4_22:
-
-	prefetcht0	A_PR1(AO,%rax,SIZE)
-	KERNEL2x2_SUB
-	KERNEL2x2_SUB
-	KERNEL2x2_SUB
-	KERNEL2x2_SUB
-
-	prefetcht0	A_PR1(AO,%rax,SIZE)
-	KERNEL2x2_SUB
-	KERNEL2x2_SUB
-	KERNEL2x2_SUB
-	KERNEL2x2_SUB
-
-	je	.L2_4_26
-
-	prefetcht0	A_PR1(AO,%rax,SIZE)
-	KERNEL2x2_SUB
-	KERNEL2x2_SUB
-	KERNEL2x2_SUB
-	KERNEL2x2_SUB
-
-	prefetcht0	A_PR1(AO,%rax,SIZE)
-	KERNEL2x2_SUB
-	KERNEL2x2_SUB
-	KERNEL2x2_SUB
-	KERNEL2x2_SUB
-
-	je	.L2_4_26
-
-	jmp	.L2_4_22
-	ALIGN_4
-
-.L2_4_26:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$ 7, %rax		# if (k & 1)
-	je .L2_4_29
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
-
-	salq	$ 2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_4_27:
-
-	KERNEL2x2_SUB
-
-	jl	.L2_4_27
-	ALIGN_4
-
-
-.L2_4_29:
-
-	vbroadcastss	ALPHA_R, %xmm0
-	vbroadcastss	ALPHA_I, %xmm1
-
-	// swap high and low 64 bytes
-        vshufps $ 0xb1, %xmm9 , %xmm9, %xmm9
-        vshufps $ 0xb1, %xmm11, %xmm11, %xmm11
-
-#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
-    defined(NR) || defined(NC) || defined(TR) || defined(TC)
-
-        vaddsubps %xmm9, %xmm8 , %xmm8
-        vaddsubps %xmm11,%xmm10, %xmm10
-
-        vshufps $ 0xb1, %xmm8 , %xmm8, %xmm9
-        vshufps $ 0xb1, %xmm10, %xmm10, %xmm11
-
-#else
-        vaddsubps %xmm8,  %xmm9 ,%xmm9
-        vaddsubps %xmm10, %xmm11,%xmm11
-
-        vmovaps   %xmm9,  %xmm8
-        vmovaps   %xmm11, %xmm10
-
-	// swap high and low 64 bytes
-        vshufps $ 0xb1, %xmm9 , %xmm9, %xmm9
-        vshufps $ 0xb1, %xmm11, %xmm11, %xmm11
-
-#endif
-
-	// multiply with ALPHA_R
-        vmulps  %xmm8 , %xmm0, %xmm8
-        vmulps  %xmm10, %xmm0, %xmm10
-
-	// multiply with ALPHA_I
-        vmulps  %xmm9 , %xmm1, %xmm9
-        vmulps  %xmm11, %xmm1, %xmm11
-
-	vaddsubps %xmm9, %xmm8 , %xmm8
-        vaddsubps %xmm11,%xmm10, %xmm10
-
-
-
-#ifndef TRMMKERNEL
-
-	vaddps 	 	(CO1), %xmm8 , %xmm8
-
-	vaddps 	 	(CO1, LDC), %xmm10, %xmm10
-
-#endif
-
-	vmovups	%xmm8 ,  	(CO1)
-
-	vmovups	%xmm10 ,  	(CO1, LDC)
-
-
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
-        leaq    (BO, BI, SIZE), BO
-	salq	$ 2, %rax			// rax = rax * 4 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $ 2, KK
-#endif
-
-	addq	$ 4 * SIZE, CO1		# coffset += 4
-	decq	I			# i --
-	jg	.L2_4_21
-	ALIGN_4	
-
-
-
-/**************************************************************************/
-.L2_4_40:
-	testq	$ 1, M		
-	jz	.L2_4_60		// to next 2 lines of N
-
-	ALIGN_4
-
-.L2_4_41:
-
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-	leaq	BUFFER1, BO		// first buffer to BO
-	addq	$ 8 * SIZE, BO
-#else
-        movq    KK, %rax
-	leaq	BUFFER1, BO			// first buffer to BO
-	addq	$ 8 * SIZE, BO
-	movq    %rax, BI                        //  Index for BO
-        leaq    (,BI,4), BI                     //  BI = BI * 4 ; number of values
-        leaq    (BO, BI, SIZE), BO
-	salq	$ 1, %rax			// rax = rax * 2 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $ 1, %rax        // number of values in AO
-#else
-        addq    $ 2, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-
-	andq	$ -8, %rax			//  K = K - ( K % 8 )
-	je	.L2_4_46
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
-
-	salq	$ 1, %rax			// rax = rax * 2 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_4_42:
-
-	prefetcht0	A_PR1(AO,%rax,SIZE)
-	KERNEL1x2_SUB
-	KERNEL1x2_SUB
-	KERNEL1x2_SUB
-	KERNEL1x2_SUB
-
-	KERNEL1x2_SUB
-	KERNEL1x2_SUB
-	KERNEL1x2_SUB
-	KERNEL1x2_SUB
-
-	je	.L2_4_46
-
-	prefetcht0	A_PR1(AO,%rax,SIZE)
-	KERNEL1x2_SUB
-	KERNEL1x2_SUB
-	KERNEL1x2_SUB
-	KERNEL1x2_SUB
-
-	KERNEL1x2_SUB
-	KERNEL1x2_SUB
-	KERNEL1x2_SUB
-	KERNEL1x2_SUB
-
-	je	.L2_4_46
-
-	jmp	.L2_4_42
-	ALIGN_4
-
-.L2_4_46:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$ 7, %rax		# if (k & 1)
-	je .L2_4_49
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
-
-	salq	$ 1, %rax			// rax = rax * 2 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_4_47:
-
-	KERNEL1x2_SUB
-
-	jl	.L2_4_47
-	ALIGN_4
-
-
-.L2_4_49:
-
-	SAVE1x2
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
-        leaq    (BO, BI, SIZE), BO
-	salq	$ 1, %rax			// rax = rax * 2 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $ 1, KK
-#endif
-
-	addq	$ 2 * SIZE, CO1		# coffset += 2
-	decq	I			# i --
-	jg	.L2_4_41
-	ALIGN_4	
-
-
-
-	
-.L2_4_60:
-#if defined(TRMMKERNEL) && !defined(LEFT)
-        addq    $ 2, KK
-#endif
-
-	decq	J			// j --
-	jg	.L2_01			// next 2 lines of N
-
-
-
-.L1_0:
-
-/************************************************************************************************
-* Loop for Nmod6 % 2 > 0
-*************************************************************************************************/
-
-	movq	Nmod6, J		
-	andq	$ 1, J			// j % 2
-	je	.L999
-	ALIGN_4
-
-.L1_01:
-	// copy to sub buffer
-	movq	B, BO1
-	leaq    BUFFER1, BO		// first buffer to BO
-	movq	K, %rax
-	ALIGN_4
-
-.L1_02b:
-
-	vmovsd		(BO1), %xmm0
-	vmovsd	%xmm0,       (BO)
-	addq	$ 2*SIZE,BO1
-	addq	$ 2*SIZE,BO
-	decq	%rax
-	jnz	.L1_02b
-
-.L1_02c:
-
-	movq	BO1, B			// next offset of B
-
-.L1_10:
-	movq	C, CO1
-	leaq	(C, LDC, 1), C		// c += 1 * ldc
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        movq    OFFSET, %rax
-        movq    %rax, KK
-#endif
-	
-	movq	A, AO		 	// aoffset = a
-	addq	$ 16 * SIZE, AO
-
-	movq	M,  I
-	sarq	$ 3, I			// i = (m >> 3)
-	je	.L1_4_10
-
-	ALIGN_4
-
-/**************************************************************************************************/
-
-.L1_8_11:
-
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-	leaq	BUFFER1, BO		// first buffer to BO
-	addq	$ 4 * SIZE, BO
-#else
-        movq    KK, %rax
-	leaq	BUFFER1, BO			// first buffer to BO
-	addq	$ 4 * SIZE, BO
-	movq    %rax, BI                        //  Index for BO
-        leaq    (,BI,2), BI                     //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO
-	salq	$ 4, %rax			// rax = rax *16 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $ 8, %rax        // number of values in AO
-#else
-        addq    $ 1, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-
-	andq	$ -8, %rax			//  K = K - ( K % 8 )
-	je	.L1_8_16
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values
-
-	salq	$ 4, %rax			// rax = rax *16 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_8_12:
-
-	prefetcht0	A_PR1(AO,%rax,SIZE)
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL8x1_SUB
-	prefetcht0	A_PR1(AO,%rax,SIZE)
-	KERNEL8x1_SUB
-	prefetcht0	A_PR1(AO,%rax,SIZE)
-	KERNEL8x1_SUB
-	prefetcht0	A_PR1(AO,%rax,SIZE)
-	KERNEL8x1_SUB
-
-	prefetcht0	A_PR1(AO,%rax,SIZE)
-	KERNEL8x1_SUB
-	prefetcht0	A_PR1(AO,%rax,SIZE)
-	KERNEL8x1_SUB
-	prefetcht0	A_PR1(AO,%rax,SIZE)
-	KERNEL8x1_SUB
-	prefetcht0	A_PR1(AO,%rax,SIZE)
-	KERNEL8x1_SUB
-
-	je	.L1_8_16
-
-	prefetcht0	A_PR1(AO,%rax,SIZE)
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL8x1_SUB
-	prefetcht0	A_PR1(AO,%rax,SIZE)
-	KERNEL8x1_SUB
-	prefetcht0	A_PR1(AO,%rax,SIZE)
-	KERNEL8x1_SUB
-	prefetcht0	A_PR1(AO,%rax,SIZE)
-	KERNEL8x1_SUB
-
-	prefetcht0	A_PR1(AO,%rax,SIZE)
-	KERNEL8x1_SUB
-	prefetcht0	A_PR1(AO,%rax,SIZE)
-	KERNEL8x1_SUB
-	prefetcht0	A_PR1(AO,%rax,SIZE)
-	KERNEL8x1_SUB
-	prefetcht0	A_PR1(AO,%rax,SIZE)
-	KERNEL8x1_SUB
-
-	je	.L1_8_16
-
-	jmp	.L1_8_12
-	ALIGN_4
-
-.L1_8_16:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$ 7, %rax		# if (k & 1)
-	je .L1_8_19
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,2), BI                    //  BI = BI * 4 ; number of values
-
-	salq	$ 4, %rax			// rax = rax *16 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_8_17:
-
-	KERNEL8x1_SUB
-
-	jl	.L1_8_17
-	ALIGN_4
-
-
-.L1_8_19:
-
-	SAVE8x1
-
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO
-	salq	$ 4, %rax			// rax = rax *16 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $ 8, KK
-#endif
-
-	addq	$ 16 * SIZE, CO1		# coffset += 16
-	decq	I			# i --
-	jg	.L1_8_11
-	ALIGN_4	
-
-
-
-/**************************************************************************************************/
-.L1_4_10:
-
-	testq	$ 7, M		
-	jz	.L999
-
-	testq	$ 4, M		
-	jz	.L1_4_20
-
-
-.L1_4_11:
-
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-	leaq	BUFFER1, BO		// first buffer to BO
-	addq	$ 4 * SIZE, BO
-#else
-        movq    KK, %rax
-	leaq	BUFFER1, BO			// first buffer to BO
-	addq	$ 4 * SIZE, BO
-	movq    %rax, BI                        //  Index for BO
-        leaq    (,BI,2), BI                     //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO
-	salq	$ 3, %rax			// rax = rax * 8 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $ 4, %rax        // number of values in AO
-#else
-        addq    $ 1, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-
-	andq	$ -8, %rax			//  K = K - ( K % 8 )
-	je	.L1_4_16
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values
-
-	salq	$ 3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_4_12:
-
-	prefetcht0	A_PR1(AO,%rax,SIZE)
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL4x1_SUB
-	KERNEL4x1_SUB
-	prefetcht0	A_PR1(AO,%rax,SIZE)
-	KERNEL4x1_SUB
-	KERNEL4x1_SUB
-
-	prefetcht0	A_PR1(AO,%rax,SIZE)
-	KERNEL4x1_SUB
-	KERNEL4x1_SUB
-	prefetcht0	A_PR1(AO,%rax,SIZE)
-	KERNEL4x1_SUB
-	KERNEL4x1_SUB
-
-	je	.L1_4_16
-
-	prefetcht0	A_PR1(AO,%rax,SIZE)
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL4x1_SUB
-	KERNEL4x1_SUB
-	prefetcht0	A_PR1(AO,%rax,SIZE)
-	KERNEL4x1_SUB
-	KERNEL4x1_SUB
-
-	prefetcht0	A_PR1(AO,%rax,SIZE)
-	KERNEL4x1_SUB
-	KERNEL4x1_SUB
-	prefetcht0	A_PR1(AO,%rax,SIZE)
-	KERNEL4x1_SUB
-	KERNEL4x1_SUB
-
-	je	.L1_4_16
-
-	jmp	.L1_4_12
-	ALIGN_4
-
-.L1_4_16:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$ 7, %rax		# if (k & 1)
-	je .L1_4_19
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,2), BI                    //  BI = BI * 4 ; number of values
-
-	salq	$ 3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_4_17:
-
-	KERNEL4x1_SUB
-
-	jl	.L1_4_17
-	ALIGN_4
-
-
-.L1_4_19:
-
-	SAVE4x1
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO
-	salq	$ 3, %rax			// rax = rax * 8 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $ 4, KK
-#endif
-
-	addq	$ 8 * SIZE, CO1		# coffset += 8
-	ALIGN_4	
-
-
-
-/**************************************************************************
-* Rest of M 
-***************************************************************************/
-
-.L1_4_20:
-
-	testq	$ 2, M		
-	jz	.L1_4_40
-	ALIGN_4
-
-.L1_4_21:
-
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-	leaq	BUFFER1, BO		// first buffer to BO
-	addq	$ 4 * SIZE, BO
-#else
-        movq    KK, %rax
-	leaq	BUFFER1, BO			// first buffer to BO
-	addq	$ 4 * SIZE, BO
-	movq    %rax, BI                        //  Index for BO
-        leaq    (,BI,2), BI                     //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO
-	salq	$ 2, %rax			// rax = rax * 4 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $ 2, %rax        // number of values in AO
-#else
-        addq    $ 1, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-
-	andq	$ -8, %rax			//  K = K - ( K % 8 )
-	je	.L1_4_26
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values
-
-	salq	$ 2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_4_22:
-
-	prefetcht0	A_PR1(AO,%rax,SIZE)
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL2x1_SUB
-	KERNEL2x1_SUB
-	KERNEL2x1_SUB
-	KERNEL2x1_SUB
-
-	prefetcht0	A_PR1(AO,%rax,SIZE)
-	KERNEL2x1_SUB
-	KERNEL2x1_SUB
-	KERNEL2x1_SUB
-	KERNEL2x1_SUB
-
-	je	.L1_4_26
-
-	prefetcht0	A_PR1(AO,%rax,SIZE)
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL2x1_SUB
-	KERNEL2x1_SUB
-	KERNEL2x1_SUB
-	KERNEL2x1_SUB
-
-	prefetcht0	A_PR1(AO,%rax,SIZE)
-	KERNEL2x1_SUB
-	KERNEL2x1_SUB
-	KERNEL2x1_SUB
-	KERNEL2x1_SUB
-
-	je	.L1_4_26
-
-	jmp	.L1_4_22
-	ALIGN_4
-
-.L1_4_26:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$ 7, %rax		# if (k & 1)
-	je .L1_4_29
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,2), BI                    //  BI = BI * 2; number of values
-
-	salq	$ 2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_4_27:
-
-	KERNEL2x1_SUB
-
-	jl	.L1_4_27
-	ALIGN_4
-
-
-.L1_4_29:
-
-	SAVE2x1
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO
-	salq	$ 2, %rax			// rax = rax * 4 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $ 2, KK
-#endif
-
-	addq	$ 4 * SIZE, CO1		# coffset += 4
-	ALIGN_4	
-
-
-
-/**************************************************************************/
-.L1_4_40:
-	testq	$ 1, M		
-	jz	.L999		// to next 2 lines of N
-
-	ALIGN_4
-
-.L1_4_41:
-
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-	leaq	BUFFER1, BO		// first buffer to BO
-	addq	$ 4 * SIZE, BO
-#else
-        movq    KK, %rax
-	leaq	BUFFER1, BO			// first buffer to BO
-	addq	$ 4 * SIZE, BO
-	movq    %rax, BI                        //  Index for BO
-        leaq    (,BI,2), BI                     //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO
-	salq	$ 1, %rax			// rax = rax * 2 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $ 1, %rax        // number of values in AO
-#else
-        addq    $ 1, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-
-	andq	$ -8, %rax			//  K = K - ( K % 8 )
-	je	.L1_4_46
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values
-
-	salq	$ 1, %rax			// rax = rax * 2 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_4_42:
-
-	prefetcht0	A_PR1(AO,%rax,SIZE)
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL1x1_SUB
-	KERNEL1x1_SUB
-	KERNEL1x1_SUB
-	KERNEL1x1_SUB
-
-	KERNEL1x1_SUB
-	KERNEL1x1_SUB
-	KERNEL1x1_SUB
-	KERNEL1x1_SUB
-
-	je	.L1_4_46
-
-	prefetcht0	A_PR1(AO,%rax,SIZE)
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL1x1_SUB
-	KERNEL1x1_SUB
-	KERNEL1x1_SUB
-	KERNEL1x1_SUB
-
-	KERNEL1x1_SUB
-	KERNEL1x1_SUB
-	KERNEL1x1_SUB
-	KERNEL1x1_SUB
-
-	je	.L1_4_46
-
-	jmp	.L1_4_42
-	ALIGN_4
-
-.L1_4_46:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$ 7, %rax		# if (k & 1)
-	je .L1_4_49
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values
-
-	salq	$ 1, %rax			// rax = rax * 2 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_4_47:
-
-	KERNEL1x1_SUB
-
-	jl	.L1_4_47
-	ALIGN_4
-
-
-.L1_4_49:
-
-	SAVE1x1
-
-
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-	movq    %rax, BI                        //  Index for BO
-        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO
-	salq	$ 1, %rax			// rax = rax * 2 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $ 1, KK
-#endif
-
-	addq	$ 2 * SIZE, CO1		# coffset += 2
-	ALIGN_4	
-
-
-.L999:
-	vzeroupper
-
-	movq   		SP, %rsp
-	movq	   (%rsp), %rbx
-	movq	  8(%rsp), %rbp
-	movq	 16(%rsp), %r12
-	movq	 24(%rsp), %r13
-	movq	 32(%rsp), %r14
-	movq	 40(%rsp), %r15
-
-#ifdef WINDOWS_ABI
-	movq	 48(%rsp), %rdi
-	movq	 56(%rsp), %rsi
-	vmovups	 64(%rsp), %xmm6
-	vmovups	 80(%rsp), %xmm7
-	vmovups	 96(%rsp), %xmm8
-	vmovups	112(%rsp), %xmm9
-	vmovups	128(%rsp), %xmm10
-	vmovups	144(%rsp), %xmm11
-	vmovups	160(%rsp), %xmm12
-	vmovups	176(%rsp), %xmm13
-	vmovups	192(%rsp), %xmm14
-	vmovups	208(%rsp), %xmm15
-#endif
-
-	addq	$ STACKSIZE, %rsp
-	ret
-
-	EPILOGUE
+/*********************************************************************************
+Copyright (c) 2013, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+**********************************************************************************/
+
+/*********************************************************************
+* 2014/07/29 Saar
+*        BLASTEST               : OK
+*        CTEST                  : OK
+*        TEST                   : OK
+*
+* 2013/10/28 Saar
+* Parameter:
+*       CGEMM_DEFAULT_UNROLL_N  2
+*       CGEMM_DEFAULT_UNROLL_M  8
+*       CGEMM_DEFAULT_P         768
+*       CGEMM_DEFAULT_Q         512
+*       A_PR1                   512
+*       B_PR1                   512
+*
+* 2014/07/29 Saar
+* Performance at 6192x6192x6192:
+*       1 thread:       49 GFLOPS       (MKL:   52)
+*       2 threads:      99 GFLOPS       (MKL:  102)
+*       3 threads:     148 GFLOPS       (MKL:  150)
+*       4 threads:     195 GFLOPS       (MKL:  194)
+*       8 threads:     354 GFLOPS       (MKL:  317)
+*
+*
+*********************************************************************/
+
+
+#define ASSEMBLER
+#include "common.h"
+ 
+#define OLD_M	%rdi
+#define OLD_N	%rsi
+#define M	%r13
+#define J	%r14
+#define OLD_K	%rdx
+
+#define A	%rcx
+#define B	%r8
+#define C	%r9
+#define LDC	%r10
+	
+#define I	%r11
+#define AO	%rdi
+#define BO	%rsi
+#define	CO1	%r15
+#define K	%r12
+#define BI	%rbp
+#define	SP	%rbx
+
+#define BO1	%rdi
+#define BO2	%r15
+
+#ifndef WINDOWS_ABI
+
+#define STACKSIZE 96
+
+#else
+
+#define STACKSIZE 320
+
+#define OLD_ALPHA_I     40 + STACKSIZE(%rsp)
+#define OLD_A           48 + STACKSIZE(%rsp)
+#define OLD_B           56 + STACKSIZE(%rsp)
+#define OLD_C           64 + STACKSIZE(%rsp)
+#define OLD_LDC         72 + STACKSIZE(%rsp)
+#define OLD_OFFSET      80 + STACKSIZE(%rsp)
+
+#endif
+
+#define L_BUFFER_SIZE 8192
+
+#define Ndiv6	 24(%rsp)
+#define Nmod6	 32(%rsp)
+#define N	 40(%rsp)
+#define ALPHA_R  48(%rsp)
+#define ALPHA_I  56(%rsp)
+#define OFFSET   64(%rsp)
+#define KK       72(%rsp)
+#define KKK      80(%rsp)
+#define BUFFER1	           128(%rsp)
+
+#if defined(OS_WINDOWS)
+#if   L_BUFFER_SIZE > 16384
+#define STACK_TOUCH \
+        movl    $ 0,  4096 * 4(%rsp);\
+        movl    $ 0,  4096 * 3(%rsp);\
+        movl    $ 0,  4096 * 2(%rsp);\
+        movl    $ 0,  4096 * 1(%rsp);
+#elif L_BUFFER_SIZE > 12288
+#define STACK_TOUCH \
+        movl    $ 0,  4096 * 3(%rsp);\
+        movl    $ 0,  4096 * 2(%rsp);\
+        movl    $ 0,  4096 * 1(%rsp);
+#elif L_BUFFER_SIZE > 8192
+#define STACK_TOUCH \
+        movl    $ 0,  4096 * 2(%rsp);\
+        movl    $ 0,  4096 * 1(%rsp);
+#elif L_BUFFER_SIZE > 4096
+#define STACK_TOUCH \
+        movl    $ 0,  4096 * 1(%rsp);
+#else
+#define STACK_TOUCH
+#endif
+#else
+#define STACK_TOUCH
+#endif
+
+
+
+#if   defined(NN) || defined(NT) || defined(TN) || defined(TT)
+
+#define	VFMADDPS_YR( y0,y1,y2 ) \
+                               vmulps y1,y2,%ymm2;\
+                               vaddps y0,%ymm2,y0
+
+#define	VFMADDPS_YI( y0,y1,y2 ) \
+                               vmulps y1,y2,%ymm3;\
+                               vaddps y0,%ymm3,y0
+
+#define	VFMADDPS_R( y0,y1,y2 ) \
+                               vmulps y1,y2,%xmm2;\
+                               vaddps y0,%xmm2,y0
+
+#define	VFMADDPS_I( y0,y1,y2 ) \
+                               vmulps y1,y2,%xmm3;\
+                               vaddps y0,%xmm3,y0
+
+
+#elif defined(RN) || defined(RT) || defined(CN) || defined(CT)
+
+#define	VFMADDPS_YR( y0,y1,y2 ) \
+                               vmulps y1,y2,%ymm2;\
+                               vsubps %ymm2,y0,y0
+
+#define	VFMADDPS_YI( y0,y1,y2 ) \
+                               vmulps y1,y2,%ymm3;\
+                               vaddps y0,%ymm3,y0
+
+#define	VFMADDPS_R( y0,y1,y2 ) \
+                               vmulps y1,y2,%xmm2;\
+                               vsubps %xmm2,y0,y0
+
+#define	VFMADDPS_I( y0,y1,y2 ) \
+                               vmulps y1,y2,%xmm3;\
+                               vaddps y0,%xmm3,y0
+
+
+#elif defined(NR) || defined(NC) || defined(TR) || defined(TC)
+
+#define	VFMADDPS_YR( y0,y1,y2 ) \
+                               vmulps y1,y2,%ymm2;\
+                               vaddps y0,%ymm2,y0
+
+#define	VFMADDPS_YI( y0,y1,y2 ) \
+                               vmulps y1,y2,%ymm3;\
+                               vsubps %ymm3,y0,y0
+
+#define	VFMADDPS_R( y0,y1,y2 ) \
+                               vmulps y1,y2,%xmm2;\
+                               vaddps y0,%xmm2,y0
+
+#define	VFMADDPS_I( y0,y1,y2 ) \
+                               vmulps y1,y2,%xmm3;\
+                               vsubps %xmm3,y0,y0
+
+
+#else
+
+#define	VFMADDPS_YR( y0,y1,y2 ) \
+                               vmulps y1,y2,%ymm2;\
+                               vsubps %ymm2,y0,y0
+
+#define	VFMADDPS_YI( y0,y1,y2 ) \
+                               vmulps y1,y2,%ymm3;\
+                               vsubps %ymm3,y0,y0
+
+#define	VFMADDPS_R( y0,y1,y2 ) \
+                               vmulps y1,y2,%xmm2;\
+                               vsubps %xmm2,y0,y0
+
+#define	VFMADDPS_I( y0,y1,y2 ) \
+                               vmulps y1,y2,%xmm3;\
+                               vsubps %xmm3,y0,y0
+
+
+#endif
+
+
+#define	A_PR1	512
+#define	B_PR1	512
+
+/***************************************************************************************************************************/
+
+.macro KERNEL8x2_1
+
+        vmovups         -16 * SIZE(AO, %rax, SIZE), %ymm0
+        vbroadcastss         -8 * SIZE(BO, BI, SIZE), %ymm4
+        vmovups          -8 * SIZE(AO, %rax, SIZE), %ymm1
+        vbroadcastss         -7 * SIZE(BO, BI, SIZE), %ymm5
+	prefetcht0	A_PR1(AO, %rax, SIZE)
+
+        VFMADDPS_YR(        %ymm8,%ymm4,%ymm0  )
+        vbroadcastss         -6 * SIZE(BO, BI, SIZE), %ymm6
+        VFMADDPS_YI(        %ymm9,%ymm5,%ymm0  )
+        vbroadcastss         -5 * SIZE(BO, BI, SIZE), %ymm7
+        VFMADDPS_YR(        %ymm12,%ymm4,%ymm1 )
+        VFMADDPS_YI(        %ymm13,%ymm5,%ymm1 )
+
+
+        VFMADDPS_YR(        %ymm10,%ymm6,%ymm0 )
+        vbroadcastss         -4 * SIZE(BO, BI, SIZE), %ymm4
+        VFMADDPS_YI(        %ymm11,%ymm7,%ymm0 )
+        vbroadcastss         -3 * SIZE(BO, BI, SIZE), %ymm5
+        VFMADDPS_YR(        %ymm14,%ymm6,%ymm1 )
+        vmovups           0 * SIZE(AO, %rax, SIZE), %ymm0
+        VFMADDPS_YI(        %ymm15,%ymm7,%ymm1 )
+
+        vmovups           8 * SIZE(AO, %rax, SIZE), %ymm1
+	prefetcht0	A_PR1+64(AO, %rax, SIZE)
+
+        VFMADDPS_YR(        %ymm8,%ymm4,%ymm0  )
+        vbroadcastss         -2 * SIZE(BO, BI, SIZE), %ymm6
+        VFMADDPS_YI(        %ymm9,%ymm5,%ymm0  )
+        vbroadcastss         -1 * SIZE(BO, BI, SIZE), %ymm7
+        VFMADDPS_YR(        %ymm12,%ymm4,%ymm1 )
+        VFMADDPS_YI(        %ymm13,%ymm5,%ymm1 )
+
+
+        VFMADDPS_YR(        %ymm10,%ymm6,%ymm0 )
+        vbroadcastss          0 * SIZE(BO, BI, SIZE), %ymm4
+        VFMADDPS_YI(        %ymm11,%ymm7,%ymm0 )
+        vbroadcastss          1 * SIZE(BO, BI, SIZE), %ymm5
+        VFMADDPS_YR(        %ymm14,%ymm6,%ymm1 )
+        vmovups          16 * SIZE(AO, %rax, SIZE), %ymm0
+        VFMADDPS_YI(        %ymm15,%ymm7,%ymm1 )
+
+        vmovups          24 * SIZE(AO, %rax, SIZE), %ymm1
+	prefetcht0	A_PR1+128(AO, %rax, SIZE)
+
+        VFMADDPS_YR(        %ymm8,%ymm4,%ymm0  )
+        vbroadcastss          2 * SIZE(BO, BI, SIZE), %ymm6
+        VFMADDPS_YI(        %ymm9,%ymm5,%ymm0  )
+        vbroadcastss          3 * SIZE(BO, BI, SIZE), %ymm7
+        VFMADDPS_YR(        %ymm12,%ymm4,%ymm1 )
+        VFMADDPS_YI(        %ymm13,%ymm5,%ymm1 )
+
+
+        VFMADDPS_YR(        %ymm10,%ymm6,%ymm0 )
+        vbroadcastss          4 * SIZE(BO, BI, SIZE), %ymm4
+        VFMADDPS_YI(        %ymm11,%ymm7,%ymm0 )
+        vbroadcastss          5 * SIZE(BO, BI, SIZE), %ymm5
+        VFMADDPS_YR(        %ymm14,%ymm6,%ymm1 )
+        vmovups          32 * SIZE(AO, %rax, SIZE), %ymm0
+        VFMADDPS_YI(        %ymm15,%ymm7,%ymm1 )
+
+        vmovups          40 * SIZE(AO, %rax, SIZE), %ymm1
+	prefetcht0	A_PR1+192(AO, %rax, SIZE)
+
+        VFMADDPS_YR(        %ymm8,%ymm4,%ymm0  )
+        vbroadcastss          6 * SIZE(BO, BI, SIZE), %ymm6
+        VFMADDPS_YI(        %ymm9,%ymm5,%ymm0  )
+        vbroadcastss          7 * SIZE(BO, BI, SIZE), %ymm7
+        VFMADDPS_YR(        %ymm12,%ymm4,%ymm1 )
+        VFMADDPS_YI(        %ymm13,%ymm5,%ymm1 )
+
+        VFMADDPS_YR(        %ymm10,%ymm6,%ymm0 )
+        VFMADDPS_YI(        %ymm11,%ymm7,%ymm0 )
+        addq    $ 16, BI                           
+        VFMADDPS_YR(        %ymm14,%ymm6,%ymm1 )
+        VFMADDPS_YI(        %ymm15,%ymm7,%ymm1 )
+
+        addq    $ 64, %rax                         
+.endm
+
+
+.macro KERNEL8x2_SUB
+
+        vmovups         -16 * SIZE(AO, %rax, SIZE), %ymm0
+        vmovups          -8 * SIZE(AO, %rax, SIZE), %ymm1
+        vbroadcastss         -8 * SIZE(BO, BI, SIZE), %ymm4
+        vbroadcastss         -7 * SIZE(BO, BI, SIZE), %ymm5
+
+        VFMADDPS_YR(        %ymm8,%ymm4,%ymm0  )
+        vbroadcastss         -6 * SIZE(BO, BI, SIZE), %ymm6
+        VFMADDPS_YI(        %ymm9,%ymm5,%ymm0  )
+        vbroadcastss         -5 * SIZE(BO, BI, SIZE), %ymm7
+        VFMADDPS_YR(        %ymm12,%ymm4,%ymm1 )
+        VFMADDPS_YI(        %ymm13,%ymm5,%ymm1 )
+
+
+        VFMADDPS_YR(        %ymm10,%ymm6,%ymm0 )
+        VFMADDPS_YI(        %ymm11,%ymm7,%ymm0 )
+        VFMADDPS_YR(        %ymm14,%ymm6,%ymm1 )
+        VFMADDPS_YI(        %ymm15,%ymm7,%ymm1 )
+
+        addq    $ 4 , BI                           
+        addq    $ 16, %rax                         
+.endm
+
+.macro SAVE8x2
+
+	vbroadcastss	ALPHA_R, %ymm0
+	vbroadcastss	ALPHA_I, %ymm1
+
+	// swap high and low 64 bytes
+        vshufps $ 0xb1, %ymm9 , %ymm9, %ymm9
+        vshufps $ 0xb1, %ymm11, %ymm11, %ymm11
+        vshufps $ 0xb1, %ymm13, %ymm13, %ymm13
+        vshufps $ 0xb1, %ymm15, %ymm15, %ymm15
+
+#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
+    defined(NR) || defined(NC) || defined(TR) || defined(TC)
+
+        vaddsubps %ymm9, %ymm8 , %ymm8
+        vaddsubps %ymm11,%ymm10, %ymm10
+        vaddsubps %ymm13,%ymm12, %ymm12
+        vaddsubps %ymm15,%ymm14, %ymm14
+
+        vshufps $ 0xb1, %ymm8 , %ymm8, %ymm9
+        vshufps $ 0xb1, %ymm10, %ymm10, %ymm11
+        vshufps $ 0xb1, %ymm12, %ymm12, %ymm13
+        vshufps $ 0xb1, %ymm14, %ymm14, %ymm15
+
+#else
+        vaddsubps %ymm8,  %ymm9 ,%ymm9
+        vaddsubps %ymm10, %ymm11,%ymm11
+        vaddsubps %ymm12, %ymm13,%ymm13
+        vaddsubps %ymm14, %ymm15,%ymm15
+
+        vmovaps   %ymm9,  %ymm8
+        vmovaps   %ymm11, %ymm10
+        vmovaps   %ymm13, %ymm12
+        vmovaps   %ymm15, %ymm14
+
+	// swap high and low 64 bytes
+        vshufps $ 0xb1, %ymm9 , %ymm9, %ymm9
+        vshufps $ 0xb1, %ymm11, %ymm11, %ymm11
+        vshufps $ 0xb1, %ymm13, %ymm13, %ymm13
+        vshufps $ 0xb1, %ymm15, %ymm15, %ymm15
+
+#endif
+
+	// multiply with ALPHA_R
+        vmulps  %ymm8 , %ymm0, %ymm8
+        vmulps  %ymm10, %ymm0, %ymm10
+        vmulps  %ymm12, %ymm0, %ymm12
+        vmulps  %ymm14, %ymm0, %ymm14
+
+	// multiply with ALPHA_I
+        vmulps  %ymm9 , %ymm1, %ymm9
+        vmulps  %ymm11, %ymm1, %ymm11
+        vmulps  %ymm13, %ymm1, %ymm13
+        vmulps  %ymm15, %ymm1, %ymm15
+
+	vaddsubps %ymm9, %ymm8 , %ymm8
+        vaddsubps %ymm11,%ymm10, %ymm10
+        vaddsubps %ymm13,%ymm12, %ymm12
+        vaddsubps %ymm15,%ymm14, %ymm14
+
+
+
+#ifndef TRMMKERNEL
+
+	vaddps 	 	(CO1), %ymm8 , %ymm8
+	vaddps  8 * SIZE(CO1), %ymm12, %ymm12
+
+	vaddps 	 	(CO1, LDC), %ymm10, %ymm10
+	vaddps  8 * SIZE(CO1, LDC), %ymm14, %ymm14
+
+#endif
+
+	vmovups	%ymm8 ,  	(CO1)
+	vmovups	%ymm12 , 8 * SIZE(CO1)
+
+	vmovups	%ymm10 ,  	(CO1, LDC)
+	vmovups	%ymm14 , 8 * SIZE(CO1, LDC)
+
+	prefetcht0	64(CO1)
+	prefetcht0	64(CO1, LDC)
+
+.endm
+
+/***************************************************************************************************************************/
+
+.macro KERNEL4x2_SUB
+        vmovups         -16 * SIZE(AO, %rax, SIZE), %xmm0
+        vbroadcastss         -8 * SIZE(BO, BI, SIZE), %xmm4
+        VFMADDPS_R(        %xmm8,%xmm4,%xmm0  )
+        vmovups         -12 * SIZE(AO, %rax, SIZE), %xmm1
+        VFMADDPS_R(        %xmm12,%xmm4,%xmm1 )
+        vbroadcastss         -7 * SIZE(BO, BI, SIZE), %xmm5
+        VFMADDPS_I(        %xmm9,%xmm5,%xmm0  )
+        VFMADDPS_I(        %xmm13,%xmm5,%xmm1 )
+        vbroadcastss         -6 * SIZE(BO, BI, SIZE), %xmm6
+        VFMADDPS_R(        %xmm10,%xmm6,%xmm0 )
+        VFMADDPS_R(        %xmm14,%xmm6,%xmm1 )
+        vbroadcastss         -5 * SIZE(BO, BI, SIZE), %xmm7
+        VFMADDPS_I(        %xmm11,%xmm7,%xmm0 )
+        VFMADDPS_I(        %xmm15,%xmm7,%xmm1 )
+        addq    $ 4, BI                           
+        addq    $ 8, %rax                         
+.endm
+
+.macro SAVE4x2
+
+	vbroadcastss	ALPHA_R, %xmm0
+	vbroadcastss	ALPHA_I, %xmm1
+
+	// swap high and low 64 bytes
+        vshufps $ 0xb1, %xmm9 , %xmm9, %xmm9
+        vshufps $ 0xb1, %xmm11, %xmm11, %xmm11
+        vshufps $ 0xb1, %xmm13, %xmm13, %xmm13
+        vshufps $ 0xb1, %xmm15, %xmm15, %xmm15
+
+#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
+    defined(NR) || defined(NC) || defined(TR) || defined(TC)
+
+        vaddsubps %xmm9, %xmm8 , %xmm8
+        vaddsubps %xmm11,%xmm10, %xmm10
+        vaddsubps %xmm13,%xmm12, %xmm12
+        vaddsubps %xmm15,%xmm14, %xmm14
+
+        vshufps $ 0xb1, %xmm8 , %xmm8, %xmm9
+        vshufps $ 0xb1, %xmm10, %xmm10, %xmm11
+        vshufps $ 0xb1, %xmm12, %xmm12, %xmm13
+        vshufps $ 0xb1, %xmm14, %xmm14, %xmm15
+
+#else
+        vaddsubps %xmm8,  %xmm9 ,%xmm9
+        vaddsubps %xmm10, %xmm11,%xmm11
+        vaddsubps %xmm12, %xmm13,%xmm13
+        vaddsubps %xmm14, %xmm15,%xmm15
+
+        vmovaps   %xmm9,  %xmm8
+        vmovaps   %xmm11, %xmm10
+        vmovaps   %xmm13, %xmm12
+        vmovaps   %xmm15, %xmm14
+
+	// swap high and low 64 bytes
+        vshufps $ 0xb1, %xmm9 , %xmm9, %xmm9
+        vshufps $ 0xb1, %xmm11, %xmm11, %xmm11
+        vshufps $ 0xb1, %xmm13, %xmm13, %xmm13
+        vshufps $ 0xb1, %xmm15, %xmm15, %xmm15
+
+#endif
+
+	// multiply with ALPHA_R
+        vmulps  %xmm8 , %xmm0, %xmm8
+        vmulps  %xmm10, %xmm0, %xmm10
+        vmulps  %xmm12, %xmm0, %xmm12
+        vmulps  %xmm14, %xmm0, %xmm14
+
+	// multiply with ALPHA_I
+        vmulps  %xmm9 , %xmm1, %xmm9
+        vmulps  %xmm11, %xmm1, %xmm11
+        vmulps  %xmm13, %xmm1, %xmm13
+        vmulps  %xmm15, %xmm1, %xmm15
+
+	vaddsubps %xmm9, %xmm8 , %xmm8
+        vaddsubps %xmm11,%xmm10, %xmm10
+        vaddsubps %xmm13,%xmm12, %xmm12
+        vaddsubps %xmm15,%xmm14, %xmm14
+
+#ifndef TRMMKERNEL
+
+	vaddps 	 	(CO1), %xmm8 , %xmm8
+	vaddps  4 * SIZE(CO1), %xmm12, %xmm12
+
+	vaddps 	 	(CO1, LDC), %xmm10, %xmm10
+	vaddps  4 * SIZE(CO1, LDC), %xmm14, %xmm14
+
+#endif
+
+	vmovups	%xmm8 ,  	(CO1)
+	vmovups	%xmm12 , 4 * SIZE(CO1)
+
+	vmovups	%xmm10 ,  	(CO1, LDC)
+	vmovups	%xmm14 , 4 * SIZE(CO1, LDC)
+
+.endm
+
+/************************************************************************************************/
+
+.macro KERNEL2x2_SUB
+        vmovups         -16 * SIZE(AO, %rax, SIZE), %xmm0
+        vbroadcastss         -8 * SIZE(BO, BI, SIZE), %xmm4
+        VFMADDPS_R(        %xmm8,%xmm4,%xmm0  )
+        vbroadcastss         -7 * SIZE(BO, BI, SIZE), %xmm5
+        VFMADDPS_I(        %xmm9,%xmm5,%xmm0  )
+        vbroadcastss         -6 * SIZE(BO, BI, SIZE), %xmm6
+        VFMADDPS_R(        %xmm10,%xmm6,%xmm0 )
+        vbroadcastss         -5 * SIZE(BO, BI, SIZE), %xmm7
+        VFMADDPS_I(        %xmm11,%xmm7,%xmm0 )
+        addq    $ 4, BI                           
+        addq    $ 4, %rax                         
+.endm
+
+.macro SAVE2x2
+
+	vbroadcastss	ALPHA_R, %xmm0
+	vbroadcastss	ALPHA_I, %xmm1
+
+	// swap high and low 4 bytes
+        vshufps $ 0xb1, %xmm9 , %xmm9, %xmm9
+        vshufps $ 0xb1, %xmm11, %xmm11, %xmm11
+
+#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
+    defined(NR) || defined(NC) || defined(TR) || defined(TC)
+
+        vaddsubps %xmm9, %xmm8 , %xmm8
+        vaddsubps %xmm11,%xmm10, %xmm10
+
+        vshufps $ 0xb1, %xmm8 , %xmm8, %xmm9
+        vshufps $ 0xb1, %xmm10, %xmm10, %xmm11
+
+#else
+        vaddsubps %xmm8,  %xmm9 ,%xmm9
+        vaddsubps %xmm10, %xmm11,%xmm11
+
+        vmovaps   %xmm9,  %xmm8
+        vmovaps   %xmm11, %xmm10
+
+	// swap high and low 4 bytes
+        vshufps $ 0xb1, %xmm9 , %xmm9, %xmm9
+        vshufps $ 0xb1, %xmm11, %xmm11, %xmm11
+
+#endif
+
+	// multiply with ALPHA_R
+        vmulps  %xmm8 , %xmm0, %xmm8
+        vmulps  %xmm10, %xmm0, %xmm10
+
+	// multiply with ALPHA_I
+        vmulps  %xmm9 , %xmm1, %xmm9
+        vmulps  %xmm11, %xmm1, %xmm11
+
+	vaddsubps %xmm9, %xmm8 , %xmm8
+        vaddsubps %xmm11,%xmm10, %xmm10
+
+#ifndef TRMMKERNEL
+
+	vaddps 	 	(CO1), %xmm8 , %xmm8
+
+	vaddps 	 	(CO1, LDC), %xmm10, %xmm10
+
+#endif
+
+	vmovups	%xmm8 ,  	(CO1)
+
+	vmovups	%xmm10 ,  	(CO1, LDC)
+
+.endm
+
+/************************************************************************************************/
+
+.macro KERNEL1x2_SUB
+        vmovsd         -16 * SIZE(AO, %rax, SIZE), %xmm0
+        vbroadcastss         -8 * SIZE(BO, BI, SIZE), %xmm4
+        VFMADDPS_R(        %xmm8,%xmm4,%xmm0  )
+        vbroadcastss         -7 * SIZE(BO, BI, SIZE), %xmm5
+        VFMADDPS_I(        %xmm9,%xmm5,%xmm0  )
+        vbroadcastss         -6 * SIZE(BO, BI, SIZE), %xmm6
+        VFMADDPS_R(        %xmm10,%xmm6,%xmm0 )
+        vbroadcastss         -5 * SIZE(BO, BI, SIZE), %xmm7
+        VFMADDPS_I(        %xmm11,%xmm7,%xmm0 )
+        addq    $ 4, BI                           
+        addq    $ 2, %rax                         
+.endm
+
+.macro SAVE1x2
+
+	vbroadcastss	ALPHA_R, %xmm0
+	vbroadcastss	ALPHA_I, %xmm1
+
+	// swap high and low 64 bytes
+        vshufps $ 0xb1, %xmm9 , %xmm9, %xmm9
+        vshufps $ 0xb1, %xmm11, %xmm11, %xmm11
+
+#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
+    defined(NR) || defined(NC) || defined(TR) || defined(TC)
+
+        vaddsubps %xmm9, %xmm8 , %xmm8
+        vaddsubps %xmm11,%xmm10, %xmm10
+
+        vshufps $ 0xb1, %xmm8 , %xmm8, %xmm9
+        vshufps $ 0xb1, %xmm10, %xmm10, %xmm11
+
+#else
+        vaddsubps %xmm8,  %xmm9 ,%xmm9
+        vaddsubps %xmm10, %xmm11,%xmm11
+
+        vmovaps   %xmm9,  %xmm8
+        vmovaps   %xmm11, %xmm10
+
+	// swap high and low 64 bytes
+        vshufps $ 0xb1, %xmm9 , %xmm9, %xmm9
+        vshufps $ 0xb1, %xmm11, %xmm11, %xmm11
+
+#endif
+
+	// multiply with ALPHA_R
+        vmulps  %xmm8 , %xmm0, %xmm8
+        vmulps  %xmm10, %xmm0, %xmm10
+
+	// multiply with ALPHA_I
+        vmulps  %xmm9 , %xmm1, %xmm9
+        vmulps  %xmm11, %xmm1, %xmm11
+
+	vaddsubps %xmm9, %xmm8 , %xmm8
+        vaddsubps %xmm11,%xmm10, %xmm10
+
+#ifndef TRMMKERNEL
+
+	vmovsd		(CO1), %xmm14
+	vaddps 	 	%xmm14, %xmm8 , %xmm8
+
+	vmovsd		(CO1, LDC), %xmm15
+	vaddps 	 	%xmm15, %xmm10, %xmm10
+
+#endif
+
+	vmovsd	%xmm8 ,  	(CO1)
+	vmovsd	%xmm10 ,  	(CO1, LDC)
+
+.endm
+
+/************************************************************************************************/
+
+.macro KERNEL8x1_SUB
+        vmovups         -16 * SIZE(AO, %rax, SIZE), %ymm0
+        vmovups          -8 * SIZE(AO, %rax, SIZE), %ymm1
+        vbroadcastss         -4 * SIZE(BO, BI, SIZE), %ymm4
+        VFMADDPS_YR(        %ymm8,%ymm4,%ymm0  )
+        VFMADDPS_YR(        %ymm12,%ymm4,%ymm1 )
+        vbroadcastss         -3 * SIZE(BO, BI, SIZE), %ymm5
+        VFMADDPS_YI(        %ymm9,%ymm5,%ymm0  )
+        VFMADDPS_YI(        %ymm13,%ymm5,%ymm1 )
+        addq    $ 2 , BI                           
+        addq    $ 16, %rax                         
+.endm
+
+.macro SAVE8x1
+
+	vbroadcastss	ALPHA_R, %ymm0
+	vbroadcastss	ALPHA_I, %ymm1
+
+	// swap high and low 64 bytes
+        vshufps $ 0xb1, %ymm9 , %ymm9, %ymm9
+        vshufps $ 0xb1, %ymm13, %ymm13, %ymm13
+
+#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
+    defined(NR) || defined(NC) || defined(TR) || defined(TC)
+
+        vaddsubps %ymm9, %ymm8 , %ymm8
+        vaddsubps %ymm13,%ymm12, %ymm12
+
+        vshufps $ 0xb1, %ymm8 , %ymm8, %ymm9
+        vshufps $ 0xb1, %ymm12, %ymm12, %ymm13
+
+#else
+        vaddsubps %ymm8,  %ymm9 ,%ymm9
+        vaddsubps %ymm12, %ymm13,%ymm13
+
+        vmovaps   %ymm9,  %ymm8
+        vmovaps   %ymm13, %ymm12
+
+	// swap high and low 64 bytes
+        vshufps $ 0xb1, %ymm9 , %ymm9, %ymm9
+        vshufps $ 0xb1, %ymm13, %ymm13, %ymm13
+
+#endif
+
+	// multiply with ALPHA_R
+        vmulps  %ymm8 , %ymm0, %ymm8
+        vmulps  %ymm12, %ymm0, %ymm12
+
+	// multiply with ALPHA_I
+        vmulps  %ymm9 , %ymm1, %ymm9
+        vmulps  %ymm13, %ymm1, %ymm13
+
+	vaddsubps %ymm9, %ymm8 , %ymm8
+        vaddsubps %ymm13,%ymm12, %ymm12
+
+
+
+#ifndef TRMMKERNEL
+
+	vaddps 	 	(CO1), %ymm8 , %ymm8
+	vaddps  8 * SIZE(CO1), %ymm12, %ymm12
+
+#endif
+
+	vmovups	%ymm8 ,  	(CO1)
+	vmovups	%ymm12 , 8 * SIZE(CO1)
+
+.endm
+
+
+/************************************************************************************************/
+
+.macro KERNEL4x1_SUB
+        vmovups         -16 * SIZE(AO, %rax, SIZE), %xmm0
+        vbroadcastss         -4 * SIZE(BO, BI, SIZE), %xmm4
+        VFMADDPS_R(        %xmm8,%xmm4,%xmm0  )
+        vmovups         -12 * SIZE(AO, %rax, SIZE), %xmm1
+        VFMADDPS_R(        %xmm12,%xmm4,%xmm1 )
+        vbroadcastss         -3 * SIZE(BO, BI, SIZE), %xmm5
+        VFMADDPS_I(        %xmm9,%xmm5,%xmm0  )
+        VFMADDPS_I(        %xmm13,%xmm5,%xmm1 )
+        addq    $ 2, BI                           
+        addq    $ 8, %rax                         
+.endm
+
+.macro SAVE4x1
+
+	vbroadcastss	ALPHA_R, %xmm0
+	vbroadcastss	ALPHA_I, %xmm1
+
+	// swap high and low 4 bytes
+        vshufps $ 0xb1, %xmm9 , %xmm9, %xmm9
+        vshufps $ 0xb1, %xmm13, %xmm13, %xmm13
+
+#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
+    defined(NR) || defined(NC) || defined(TR) || defined(TC)
+
+        vaddsubps %xmm9, %xmm8 , %xmm8
+        vaddsubps %xmm13,%xmm12, %xmm12
+
+        vshufps $ 0xb1, %xmm8 , %xmm8, %xmm9
+        vshufps $ 0xb1, %xmm12, %xmm12, %xmm13
+
+#else
+        vaddsubps %xmm8,  %xmm9 ,%xmm9
+        vaddsubps %xmm12, %xmm13,%xmm13
+
+        vmovaps   %xmm9,  %xmm8
+        vmovaps   %xmm13, %xmm12
+
+	// swap high and low 4 bytes
+        vshufps $ 0xb1, %xmm9 , %xmm9, %xmm9
+        vshufps $ 0xb1, %xmm13, %xmm13, %xmm13
+
+#endif
+
+	// multiply with ALPHA_R
+        vmulps  %xmm8 , %xmm0, %xmm8
+        vmulps  %xmm12, %xmm0, %xmm12
+
+	// multiply with ALPHA_I
+        vmulps  %xmm9 , %xmm1, %xmm9
+        vmulps  %xmm13, %xmm1, %xmm13
+
+	vaddsubps %xmm9, %xmm8 , %xmm8
+        vaddsubps %xmm13,%xmm12, %xmm12
+
+#ifndef TRMMKERNEL
+
+	vaddps 	 	(CO1), %xmm8 , %xmm8
+	vaddps  4 * SIZE(CO1), %xmm12, %xmm12
+
+#endif
+
+	vmovups	%xmm8 ,  	(CO1)
+	vmovups	%xmm12 , 4 * SIZE(CO1)
+
+.endm
+
+/************************************************************************************************/
+
+.macro KERNEL2x1_SUB
+        vmovups         -16 * SIZE(AO, %rax, SIZE), %xmm0
+        vbroadcastss         -4 * SIZE(BO, BI, SIZE), %xmm4
+        VFMADDPS_R(        %xmm8,%xmm4,%xmm0  )
+        vbroadcastss         -3 * SIZE(BO, BI, SIZE), %xmm5
+        VFMADDPS_I(        %xmm9,%xmm5,%xmm0  )
+        addq    $ 2, BI                           
+        addq    $ 4, %rax                         
+.endm
+
+.macro SAVE2x1
+
+	vbroadcastss	ALPHA_R, %xmm0
+	vbroadcastss	ALPHA_I, %xmm1
+
+	// swap high and low 64 bytes
+        vshufps $ 0xb1, %xmm9 , %xmm9, %xmm9
+
+#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
+    defined(NR) || defined(NC) || defined(TR) || defined(TC)
+
+        vaddsubps %xmm9, %xmm8 , %xmm8
+
+        vshufps $ 0xb1, %xmm8 , %xmm8, %xmm9
+
+#else
+        vaddsubps %xmm8,  %xmm9 ,%xmm9
+
+        vmovaps   %xmm9,  %xmm8
+
+	// swap high and low 64 bytes
+        vshufps $ 0xb1, %xmm9 , %xmm9, %xmm9
+
+#endif
+
+	// multiply with ALPHA_R
+        vmulps  %xmm8 , %xmm0, %xmm8
+
+	// multiply with ALPHA_I
+        vmulps  %xmm9 , %xmm1, %xmm9
+
+	vaddsubps %xmm9, %xmm8 , %xmm8
+
+#ifndef TRMMKERNEL
+
+	vaddps 	 	(CO1), %xmm8 , %xmm8
+
+#endif
+
+	vmovups	%xmm8 ,  	(CO1)
+
+.endm
+
+/************************************************************************************************/
+
+.macro KERNEL1x1_SUB
+        vmovsd         -16 * SIZE(AO, %rax, SIZE), %xmm0
+        vbroadcastss         -4 * SIZE(BO, BI, SIZE), %xmm4
+        VFMADDPS_R(        %xmm8,%xmm4,%xmm0 )
+        vbroadcastss         -3 * SIZE(BO, BI, SIZE), %xmm5
+        VFMADDPS_I(        %xmm9,%xmm5,%xmm0 )
+        addq    $ 2, BI                           
+        addq    $ 2, %rax                         
+.endm
+
+.macro SAVE1x1
+
+	vbroadcastss	ALPHA_R, %xmm0
+	vbroadcastss	ALPHA_I, %xmm1
+
+	// swap high and low 64 bytes
+        vshufps $ 0xb1, %xmm9 , %xmm9, %xmm9
+
+#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
+    defined(NR) || defined(NC) || defined(TR) || defined(TC)
+
+        vaddsubps %xmm9, %xmm8 , %xmm8
+
+        vshufps $ 0xb1, %xmm8 , %xmm8, %xmm9
+
+#else
+        vaddsubps %xmm8,  %xmm9 ,%xmm9
+
+        vmovaps   %xmm9,  %xmm8
+
+	// swap high and low 64 bytes
+        vshufps $ 0xb1, %xmm9 , %xmm9, %xmm9
+
+#endif
+
+	// multiply with ALPHA_R
+        vmulps  %xmm8 , %xmm0, %xmm8
+
+	// multiply with ALPHA_I
+        vmulps  %xmm9 , %xmm1, %xmm9
+
+	vaddsubps %xmm9, %xmm8 , %xmm8
+
+#ifndef TRMMKERNEL
+
+	vmovsd		(CO1), %xmm14
+	vaddps 	 	%xmm14, %xmm8 , %xmm8
+
+#endif
+
+	vmovsd	%xmm8 ,  	(CO1)
+
+.endm
+
+/************************************************************************************************/
+
+
+
+
+	PROLOGUE
+	PROFCODE
+	
+	subq	$ STACKSIZE, %rsp
+	movq	%rbx,   (%rsp)
+	movq	%rbp,  8(%rsp)
+	movq	%r12, 16(%rsp)
+	movq	%r13, 24(%rsp)
+	movq	%r14, 32(%rsp)
+	movq	%r15, 40(%rsp)
+
+	vzeroupper
+
+#ifdef WINDOWS_ABI
+	movq	%rdi,    48(%rsp)
+	movq	%rsi,    56(%rsp)
+	vmovups	%xmm6,   64(%rsp)
+	vmovups	%xmm7,   80(%rsp)
+	vmovups	%xmm8,   96(%rsp)
+	vmovups	%xmm9,  112(%rsp)
+	vmovups	%xmm10, 128(%rsp)
+	vmovups	%xmm11, 144(%rsp)
+	vmovups	%xmm12, 160(%rsp)
+	vmovups	%xmm13, 176(%rsp)
+	vmovups	%xmm14, 192(%rsp)
+	vmovups	%xmm15, 208(%rsp)
+
+	movq	ARG1,      OLD_M
+	movq	ARG2,      OLD_N
+	movq	ARG3,      OLD_K
+	movq	OLD_A,     A
+	movq	OLD_B,     B
+	movq	OLD_C,     C
+	movq	OLD_LDC,   LDC
+#ifdef TRMMKERNEL
+	movsd	OLD_OFFSET, %xmm12
+#endif
+	vmovaps	%xmm3, %xmm0
+	vmovsd   OLD_ALPHA_I, %xmm1
+
+#else
+	movq	STACKSIZE +  8(%rsp), LDC
+#ifdef TRMMKERNEL
+	movsd	STACKSIZE + 16(%rsp), %xmm12
+#endif
+
+#endif
+
+	movq    %rsp, SP      # save old stack
+        subq    $ 128 + L_BUFFER_SIZE, %rsp
+        andq    $ -4096, %rsp    # align stack
+
+        STACK_TOUCH
+
+	cmpq	$ 0, OLD_M
+	je	.L999
+
+	cmpq	$ 0, OLD_N
+	je	.L999
+
+	cmpq	$ 0, OLD_K
+	je	.L999
+
+	movq	OLD_M, M
+	movq	OLD_N, N
+	movq	OLD_K, K
+
+	vmovss	 %xmm0, ALPHA_R
+	vmovss	 %xmm1, ALPHA_I
+
+	salq	$ ZBASE_SHIFT, LDC
+
+	movq    N, %rax
+        xorq    %rdx, %rdx
+        movq    $ 2,  %rdi
+        divq    %rdi                    //    N / 2
+        movq    %rax, Ndiv6             //    N / 2
+        movq    %rdx, Nmod6             //    N % 2
+
+	
+
+#ifdef TRMMKERNEL
+	vmovsd	%xmm12, OFFSET
+	vmovsd	%xmm12, KK
+#ifndef LEFT
+	negq	KK
+#endif	
+#endif
+
+.L2_0:
+
+	movq	Ndiv6,  J
+	cmpq	$ 0, J
+	je	.L1_0
+	ALIGN_4
+
+
+
+.L2_01:
+	// copy to sub buffer
+	movq	B, BO1
+	leaq    BUFFER1, BO		// first buffer to BO
+	movq	K, %rax
+	ALIGN_4
+
+.L2_02b:
+
+	vmovups	(BO1), %xmm0
+	vmovups	%xmm0,       (BO)
+	addq	$ 4*SIZE,BO1
+	addq	$ 4*SIZE,BO
+	decq	%rax
+	jnz	.L2_02b
+
+.L2_02c:
+
+	movq	BO1, B			// next offset of B
+
+.L2_10:
+	movq	C, CO1
+	leaq	(C, LDC, 2), C		// c += 2 * ldc
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        movq    OFFSET, %rax
+        movq    %rax, KK
+#endif
+	
+	movq	A, AO		 	// aoffset = a
+	addq	$ 16 * SIZE, AO
+
+	movq	M,  I
+	sarq	$ 3, I			// i = (m >> 3)
+	je	.L2_4_10
+
+	ALIGN_4
+/**********************************************************************************************************/
+
+.L2_8_11:
+
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+	leaq	BUFFER1, BO		// first buffer to BO
+	addq	$ 8 * SIZE, BO
+#else
+        movq    KK, %rax
+	leaq	BUFFER1, BO			// first buffer to BO
+	addq	$ 8 * SIZE, BO
+	movq    %rax, BI                        //  Index for BO
+        leaq    (,BI,4), BI                     //  BI = BI * 4 ; number of values
+        leaq    (BO, BI, SIZE), BO
+	salq	$ 4, %rax			// rax = rax *16 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $ 8, %rax        // number of values in AO
+#else
+        addq    $ 2, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$ -8, %rax			//  K = K - ( K % 8 )
+	je	.L2_8_16
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
+
+	salq	$ 4, %rax			// rax = rax *16 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_8_12:
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL8x2_1
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL8x2_1
+
+	je	.L2_8_16
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL8x2_1
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL8x2_1
+
+	je	.L2_8_16
+
+	jmp	.L2_8_12
+	ALIGN_4
+
+.L2_8_16:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$ 7, %rax		# if (k & 1)
+	je .L2_8_19
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
+
+	salq	$ 4, %rax			// rax = rax *16 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_8_17:
+
+	KERNEL8x2_SUB
+
+	jl	.L2_8_17
+	ALIGN_4
+
+
+.L2_8_19:
+
+	SAVE8x2
+
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
+        leaq    (BO, BI, SIZE), BO
+	salq	$ 4, %rax			// rax = rax *16 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $ 8, KK
+#endif
+
+	addq	$ 16 * SIZE, CO1		# coffset += 16
+	decq	I			# i --
+	jg	.L2_8_11
+	ALIGN_4	
+
+
+/**********************************************************************************************************/
+
+
+
+
+.L2_4_10:
+	testq	$ 7, M		
+	jz	.L2_4_60		// to next 2 lines of N
+
+	testq	$ 4, M		
+	jz	.L2_4_20
+	ALIGN_4
+
+
+.L2_4_11:
+
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+	leaq	BUFFER1, BO		// first buffer to BO
+	addq	$ 8 * SIZE, BO
+#else
+        movq    KK, %rax
+	leaq	BUFFER1, BO			// first buffer to BO
+	addq	$ 8 * SIZE, BO
+	movq    %rax, BI                        //  Index for BO
+        leaq    (,BI,4), BI                     //  BI = BI * 4 ; number of values
+        leaq    (BO, BI, SIZE), BO
+	salq	$ 3, %rax			// rax = rax * 8 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $ 4, %rax        // number of values in AO
+#else
+        addq    $ 2, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$ -8, %rax			//  K = K - ( K % 8 )
+	je	.L2_4_16
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
+
+	salq	$ 3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_4_12:
+
+	prefetcht0	A_PR1(AO,%rax,SIZE)
+	KERNEL4x2_SUB
+	KERNEL4x2_SUB
+	prefetcht0	A_PR1(AO,%rax,SIZE)
+	KERNEL4x2_SUB
+	KERNEL4x2_SUB
+
+	prefetcht0	A_PR1(AO,%rax,SIZE)
+	KERNEL4x2_SUB
+	KERNEL4x2_SUB
+	prefetcht0	A_PR1(AO,%rax,SIZE)
+	KERNEL4x2_SUB
+	KERNEL4x2_SUB
+
+	je	.L2_4_16
+
+	prefetcht0	A_PR1(AO,%rax,SIZE)
+	KERNEL4x2_SUB
+	KERNEL4x2_SUB
+	prefetcht0	A_PR1(AO,%rax,SIZE)
+	KERNEL4x2_SUB
+	KERNEL4x2_SUB
+
+	prefetcht0	A_PR1(AO,%rax,SIZE)
+	KERNEL4x2_SUB
+	KERNEL4x2_SUB
+	prefetcht0	A_PR1(AO,%rax,SIZE)
+	KERNEL4x2_SUB
+	KERNEL4x2_SUB
+
+	je	.L2_4_16
+
+	jmp	.L2_4_12
+	ALIGN_4
+
+.L2_4_16:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$ 7, %rax		# if (k & 1)
+	je .L2_4_19
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
+
+	salq	$ 3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_4_17:
+
+	KERNEL4x2_SUB
+
+	jl	.L2_4_17
+	ALIGN_4
+
+
+.L2_4_19:
+
+	SAVE4x2
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
+        leaq    (BO, BI, SIZE), BO
+	salq	$ 3, %rax			// rax = rax * 8 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $ 4, KK
+#endif
+
+	addq	$ 8 * SIZE, CO1		# coffset += 8
+	ALIGN_4	
+
+
+
+/**************************************************************************
+* Rest of M 
+***************************************************************************/
+
+.L2_4_20:
+
+	testq	$ 2, M		
+	jz	.L2_4_40
+	ALIGN_4
+
+.L2_4_21:
+
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+	leaq	BUFFER1, BO		// first buffer to BO
+	addq	$ 8 * SIZE, BO
+#else
+        movq    KK, %rax
+	leaq	BUFFER1, BO			// first buffer to BO
+	addq	$ 8 * SIZE, BO
+	movq    %rax, BI                        //  Index for BO
+        leaq    (,BI,4), BI                     //  BI = BI * 4 ; number of values
+        leaq    (BO, BI, SIZE), BO
+	salq	$ 2, %rax			// rax = rax * 4 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $ 2, %rax        // number of values in AO
+#else
+        addq    $ 2, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$ -8, %rax			//  K = K - ( K % 8 )
+	je	.L2_4_26
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
+
+	salq	$ 2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_4_22:
+
+	prefetcht0	A_PR1(AO,%rax,SIZE)
+	KERNEL2x2_SUB
+	KERNEL2x2_SUB
+	KERNEL2x2_SUB
+	KERNEL2x2_SUB
+
+	prefetcht0	A_PR1(AO,%rax,SIZE)
+	KERNEL2x2_SUB
+	KERNEL2x2_SUB
+	KERNEL2x2_SUB
+	KERNEL2x2_SUB
+
+	je	.L2_4_26
+
+	prefetcht0	A_PR1(AO,%rax,SIZE)
+	KERNEL2x2_SUB
+	KERNEL2x2_SUB
+	KERNEL2x2_SUB
+	KERNEL2x2_SUB
+
+	prefetcht0	A_PR1(AO,%rax,SIZE)
+	KERNEL2x2_SUB
+	KERNEL2x2_SUB
+	KERNEL2x2_SUB
+	KERNEL2x2_SUB
+
+	je	.L2_4_26
+
+	jmp	.L2_4_22
+	ALIGN_4
+
+.L2_4_26:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$ 7, %rax		# if (k & 1)
+	je .L2_4_29
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
+
+	salq	$ 2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_4_27:
+
+	KERNEL2x2_SUB
+
+	jl	.L2_4_27
+	ALIGN_4
+
+
+.L2_4_29:
+
+	vbroadcastss	ALPHA_R, %xmm0
+	vbroadcastss	ALPHA_I, %xmm1
+
+	// swap high and low 64 bytes
+        vshufps $ 0xb1, %xmm9 , %xmm9, %xmm9
+        vshufps $ 0xb1, %xmm11, %xmm11, %xmm11
+
+#if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \
+    defined(NR) || defined(NC) || defined(TR) || defined(TC)
+
+        vaddsubps %xmm9, %xmm8 , %xmm8
+        vaddsubps %xmm11,%xmm10, %xmm10
+
+        vshufps $ 0xb1, %xmm8 , %xmm8, %xmm9
+        vshufps $ 0xb1, %xmm10, %xmm10, %xmm11
+
+#else
+        vaddsubps %xmm8,  %xmm9 ,%xmm9
+        vaddsubps %xmm10, %xmm11,%xmm11
+
+        vmovaps   %xmm9,  %xmm8
+        vmovaps   %xmm11, %xmm10
+
+	// swap high and low 64 bytes
+        vshufps $ 0xb1, %xmm9 , %xmm9, %xmm9
+        vshufps $ 0xb1, %xmm11, %xmm11, %xmm11
+
+#endif
+
+	// multiply with ALPHA_R
+        vmulps  %xmm8 , %xmm0, %xmm8
+        vmulps  %xmm10, %xmm0, %xmm10
+
+	// multiply with ALPHA_I
+        vmulps  %xmm9 , %xmm1, %xmm9
+        vmulps  %xmm11, %xmm1, %xmm11
+
+	vaddsubps %xmm9, %xmm8 , %xmm8
+        vaddsubps %xmm11,%xmm10, %xmm10
+
+
+
+#ifndef TRMMKERNEL
+
+	vaddps 	 	(CO1), %xmm8 , %xmm8
+
+	vaddps 	 	(CO1, LDC), %xmm10, %xmm10
+
+#endif
+
+	vmovups	%xmm8 ,  	(CO1)
+
+	vmovups	%xmm10 ,  	(CO1, LDC)
+
+
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
+        leaq    (BO, BI, SIZE), BO
+	salq	$ 2, %rax			// rax = rax * 4 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $ 2, KK
+#endif
+
+	addq	$ 4 * SIZE, CO1		# coffset += 4
+	decq	I			# i --
+	jg	.L2_4_21
+	ALIGN_4	
+
+
+
+/**************************************************************************/
+.L2_4_40:
+	testq	$ 1, M		
+	jz	.L2_4_60		// to next 2 lines of N
+
+	ALIGN_4
+
+.L2_4_41:
+
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+	leaq	BUFFER1, BO		// first buffer to BO
+	addq	$ 8 * SIZE, BO
+#else
+        movq    KK, %rax
+	leaq	BUFFER1, BO			// first buffer to BO
+	addq	$ 8 * SIZE, BO
+	movq    %rax, BI                        //  Index for BO
+        leaq    (,BI,4), BI                     //  BI = BI * 4 ; number of values
+        leaq    (BO, BI, SIZE), BO
+	salq	$ 1, %rax			// rax = rax * 2 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $ 1, %rax        // number of values in AO
+#else
+        addq    $ 2, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$ -8, %rax			//  K = K - ( K % 8 )
+	je	.L2_4_46
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
+
+	salq	$ 1, %rax			// rax = rax * 2 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_4_42:
+
+	prefetcht0	A_PR1(AO,%rax,SIZE)
+	KERNEL1x2_SUB
+	KERNEL1x2_SUB
+	KERNEL1x2_SUB
+	KERNEL1x2_SUB
+
+	KERNEL1x2_SUB
+	KERNEL1x2_SUB
+	KERNEL1x2_SUB
+	KERNEL1x2_SUB
+
+	je	.L2_4_46
+
+	prefetcht0	A_PR1(AO,%rax,SIZE)
+	KERNEL1x2_SUB
+	KERNEL1x2_SUB
+	KERNEL1x2_SUB
+	KERNEL1x2_SUB
+
+	KERNEL1x2_SUB
+	KERNEL1x2_SUB
+	KERNEL1x2_SUB
+	KERNEL1x2_SUB
+
+	je	.L2_4_46
+
+	jmp	.L2_4_42
+	ALIGN_4
+
+.L2_4_46:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$ 7, %rax		# if (k & 1)
+	je .L2_4_49
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
+
+	salq	$ 1, %rax			// rax = rax * 2 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_4_47:
+
+	KERNEL1x2_SUB
+
+	jl	.L2_4_47
+	ALIGN_4
+
+
+.L2_4_49:
+
+	SAVE1x2
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,4), BI                    //  BI = BI * 4 ; number of values
+        leaq    (BO, BI, SIZE), BO
+	salq	$ 1, %rax			// rax = rax * 2 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $ 1, KK
+#endif
+
+	addq	$ 2 * SIZE, CO1		# coffset += 2
+	decq	I			# i --
+	jg	.L2_4_41
+	ALIGN_4	
+
+
+
+	
+.L2_4_60:
+#if defined(TRMMKERNEL) && !defined(LEFT)
+        addq    $ 2, KK
+#endif
+
+	decq	J			// j --
+	jg	.L2_01			// next 2 lines of N
+
+
+
+.L1_0:
+
+/************************************************************************************************
+* Loop for Nmod6 % 2 > 0
+*************************************************************************************************/
+
+	movq	Nmod6, J		
+	andq	$ 1, J			// j % 2
+	je	.L999
+	ALIGN_4
+
+.L1_01:
+	// copy to sub buffer
+	movq	B, BO1
+	leaq    BUFFER1, BO		// first buffer to BO
+	movq	K, %rax
+	ALIGN_4
+
+.L1_02b:
+
+	vmovsd		(BO1), %xmm0
+	vmovsd	%xmm0,       (BO)
+	addq	$ 2*SIZE,BO1
+	addq	$ 2*SIZE,BO
+	decq	%rax
+	jnz	.L1_02b
+
+.L1_02c:
+
+	movq	BO1, B			// next offset of B
+
+.L1_10:
+	movq	C, CO1
+	leaq	(C, LDC, 1), C		// c += 1 * ldc
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        movq    OFFSET, %rax
+        movq    %rax, KK
+#endif
+	
+	movq	A, AO		 	// aoffset = a
+	addq	$ 16 * SIZE, AO
+
+	movq	M,  I
+	sarq	$ 3, I			// i = (m >> 3)
+	je	.L1_4_10
+
+	ALIGN_4
+
+/**************************************************************************************************/
+
+.L1_8_11:
+
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+	leaq	BUFFER1, BO		// first buffer to BO
+	addq	$ 4 * SIZE, BO
+#else
+        movq    KK, %rax
+	leaq	BUFFER1, BO			// first buffer to BO
+	addq	$ 4 * SIZE, BO
+	movq    %rax, BI                        //  Index for BO
+        leaq    (,BI,2), BI                     //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO
+	salq	$ 4, %rax			// rax = rax *16 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $ 8, %rax        // number of values in AO
+#else
+        addq    $ 1, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$ -8, %rax			//  K = K - ( K % 8 )
+	je	.L1_8_16
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values
+
+	salq	$ 4, %rax			// rax = rax *16 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_8_12:
+
+	prefetcht0	A_PR1(AO,%rax,SIZE)
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL8x1_SUB
+	prefetcht0	A_PR1(AO,%rax,SIZE)
+	KERNEL8x1_SUB
+	prefetcht0	A_PR1(AO,%rax,SIZE)
+	KERNEL8x1_SUB
+	prefetcht0	A_PR1(AO,%rax,SIZE)
+	KERNEL8x1_SUB
+
+	prefetcht0	A_PR1(AO,%rax,SIZE)
+	KERNEL8x1_SUB
+	prefetcht0	A_PR1(AO,%rax,SIZE)
+	KERNEL8x1_SUB
+	prefetcht0	A_PR1(AO,%rax,SIZE)
+	KERNEL8x1_SUB
+	prefetcht0	A_PR1(AO,%rax,SIZE)
+	KERNEL8x1_SUB
+
+	je	.L1_8_16
+
+	prefetcht0	A_PR1(AO,%rax,SIZE)
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL8x1_SUB
+	prefetcht0	A_PR1(AO,%rax,SIZE)
+	KERNEL8x1_SUB
+	prefetcht0	A_PR1(AO,%rax,SIZE)
+	KERNEL8x1_SUB
+	prefetcht0	A_PR1(AO,%rax,SIZE)
+	KERNEL8x1_SUB
+
+	prefetcht0	A_PR1(AO,%rax,SIZE)
+	KERNEL8x1_SUB
+	prefetcht0	A_PR1(AO,%rax,SIZE)
+	KERNEL8x1_SUB
+	prefetcht0	A_PR1(AO,%rax,SIZE)
+	KERNEL8x1_SUB
+	prefetcht0	A_PR1(AO,%rax,SIZE)
+	KERNEL8x1_SUB
+
+	je	.L1_8_16
+
+	jmp	.L1_8_12
+	ALIGN_4
+
+.L1_8_16:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$ 7, %rax		# if (k & 1)
+	je .L1_8_19
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,2), BI                    //  BI = BI * 4 ; number of values
+
+	salq	$ 4, %rax			// rax = rax *16 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_8_17:
+
+	KERNEL8x1_SUB
+
+	jl	.L1_8_17
+	ALIGN_4
+
+
+.L1_8_19:
+
+	SAVE8x1
+
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO
+	salq	$ 4, %rax			// rax = rax *16 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $ 8, KK
+#endif
+
+	addq	$ 16 * SIZE, CO1		# coffset += 16
+	decq	I			# i --
+	jg	.L1_8_11
+	ALIGN_4	
+
+
+
+/**************************************************************************************************/
+.L1_4_10:
+
+	testq	$ 7, M		
+	jz	.L999
+
+	testq	$ 4, M		
+	jz	.L1_4_20
+
+
+.L1_4_11:
+
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+	leaq	BUFFER1, BO		// first buffer to BO
+	addq	$ 4 * SIZE, BO
+#else
+        movq    KK, %rax
+	leaq	BUFFER1, BO			// first buffer to BO
+	addq	$ 4 * SIZE, BO
+	movq    %rax, BI                        //  Index for BO
+        leaq    (,BI,2), BI                     //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO
+	salq	$ 3, %rax			// rax = rax * 8 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $ 4, %rax        // number of values in AO
+#else
+        addq    $ 1, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$ -8, %rax			//  K = K - ( K % 8 )
+	je	.L1_4_16
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values
+
+	salq	$ 3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_4_12:
+
+	prefetcht0	A_PR1(AO,%rax,SIZE)
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL4x1_SUB
+	KERNEL4x1_SUB
+	prefetcht0	A_PR1(AO,%rax,SIZE)
+	KERNEL4x1_SUB
+	KERNEL4x1_SUB
+
+	prefetcht0	A_PR1(AO,%rax,SIZE)
+	KERNEL4x1_SUB
+	KERNEL4x1_SUB
+	prefetcht0	A_PR1(AO,%rax,SIZE)
+	KERNEL4x1_SUB
+	KERNEL4x1_SUB
+
+	je	.L1_4_16
+
+	prefetcht0	A_PR1(AO,%rax,SIZE)
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL4x1_SUB
+	KERNEL4x1_SUB
+	prefetcht0	A_PR1(AO,%rax,SIZE)
+	KERNEL4x1_SUB
+	KERNEL4x1_SUB
+
+	prefetcht0	A_PR1(AO,%rax,SIZE)
+	KERNEL4x1_SUB
+	KERNEL4x1_SUB
+	prefetcht0	A_PR1(AO,%rax,SIZE)
+	KERNEL4x1_SUB
+	KERNEL4x1_SUB
+
+	je	.L1_4_16
+
+	jmp	.L1_4_12
+	ALIGN_4
+
+.L1_4_16:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$ 7, %rax		# if (k & 1)
+	je .L1_4_19
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,2), BI                    //  BI = BI * 4 ; number of values
+
+	salq	$ 3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_4_17:
+
+	KERNEL4x1_SUB
+
+	jl	.L1_4_17
+	ALIGN_4
+
+
+.L1_4_19:
+
+	SAVE4x1
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO
+	salq	$ 3, %rax			// rax = rax * 8 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $ 4, KK
+#endif
+
+	addq	$ 8 * SIZE, CO1		# coffset += 8
+	ALIGN_4	
+
+
+
+/**************************************************************************
+* Rest of M 
+***************************************************************************/
+
+.L1_4_20:
+
+	testq	$ 2, M		
+	jz	.L1_4_40
+	ALIGN_4
+
+.L1_4_21:
+
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+	leaq	BUFFER1, BO		// first buffer to BO
+	addq	$ 4 * SIZE, BO
+#else
+        movq    KK, %rax
+	leaq	BUFFER1, BO			// first buffer to BO
+	addq	$ 4 * SIZE, BO
+	movq    %rax, BI                        //  Index for BO
+        leaq    (,BI,2), BI                     //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO
+	salq	$ 2, %rax			// rax = rax * 4 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $ 2, %rax        // number of values in AO
+#else
+        addq    $ 1, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$ -8, %rax			//  K = K - ( K % 8 )
+	je	.L1_4_26
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values
+
+	salq	$ 2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_4_22:
+
+	prefetcht0	A_PR1(AO,%rax,SIZE)
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL2x1_SUB
+	KERNEL2x1_SUB
+	KERNEL2x1_SUB
+	KERNEL2x1_SUB
+
+	prefetcht0	A_PR1(AO,%rax,SIZE)
+	KERNEL2x1_SUB
+	KERNEL2x1_SUB
+	KERNEL2x1_SUB
+	KERNEL2x1_SUB
+
+	je	.L1_4_26
+
+	prefetcht0	A_PR1(AO,%rax,SIZE)
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL2x1_SUB
+	KERNEL2x1_SUB
+	KERNEL2x1_SUB
+	KERNEL2x1_SUB
+
+	prefetcht0	A_PR1(AO,%rax,SIZE)
+	KERNEL2x1_SUB
+	KERNEL2x1_SUB
+	KERNEL2x1_SUB
+	KERNEL2x1_SUB
+
+	je	.L1_4_26
+
+	jmp	.L1_4_22
+	ALIGN_4
+
+.L1_4_26:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$ 7, %rax		# if (k & 1)
+	je .L1_4_29
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,2), BI                    //  BI = BI * 2; number of values
+
+	salq	$ 2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_4_27:
+
+	KERNEL2x1_SUB
+
+	jl	.L1_4_27
+	ALIGN_4
+
+
+.L1_4_29:
+
+	SAVE2x1
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO
+	salq	$ 2, %rax			// rax = rax * 4 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $ 2, KK
+#endif
+
+	addq	$ 4 * SIZE, CO1		# coffset += 4
+	ALIGN_4	
+
+
+
+/**************************************************************************/
+.L1_4_40:
+	testq	$ 1, M		
+	jz	.L999		// to next 2 lines of N
+
+	ALIGN_4
+
+.L1_4_41:
+
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+	leaq	BUFFER1, BO		// first buffer to BO
+	addq	$ 4 * SIZE, BO
+#else
+        movq    KK, %rax
+	leaq	BUFFER1, BO			// first buffer to BO
+	addq	$ 4 * SIZE, BO
+	movq    %rax, BI                        //  Index for BO
+        leaq    (,BI,2), BI                     //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO
+	salq	$ 1, %rax			// rax = rax * 2 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $ 1, %rax        // number of values in AO
+#else
+        addq    $ 1, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$ -8, %rax			//  K = K - ( K % 8 )
+	je	.L1_4_46
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values
+
+	salq	$ 1, %rax			// rax = rax * 2 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_4_42:
+
+	prefetcht0	A_PR1(AO,%rax,SIZE)
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL1x1_SUB
+	KERNEL1x1_SUB
+	KERNEL1x1_SUB
+	KERNEL1x1_SUB
+
+	KERNEL1x1_SUB
+	KERNEL1x1_SUB
+	KERNEL1x1_SUB
+	KERNEL1x1_SUB
+
+	je	.L1_4_46
+
+	prefetcht0	A_PR1(AO,%rax,SIZE)
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL1x1_SUB
+	KERNEL1x1_SUB
+	KERNEL1x1_SUB
+	KERNEL1x1_SUB
+
+	KERNEL1x1_SUB
+	KERNEL1x1_SUB
+	KERNEL1x1_SUB
+	KERNEL1x1_SUB
+
+	je	.L1_4_46
+
+	jmp	.L1_4_42
+	ALIGN_4
+
+.L1_4_46:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$ 7, %rax		# if (k & 1)
+	je .L1_4_49
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values
+
+	salq	$ 1, %rax			// rax = rax * 2 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_4_47:
+
+	KERNEL1x1_SUB
+
+	jl	.L1_4_47
+	ALIGN_4
+
+
+.L1_4_49:
+
+	SAVE1x1
+
+
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+	movq    %rax, BI                        //  Index for BO
+        leaq    ( ,BI,2), BI                    //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO
+	salq	$ 1, %rax			// rax = rax * 2 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $ 1, KK
+#endif
+
+	addq	$ 2 * SIZE, CO1		# coffset += 2
+	ALIGN_4	
+
+
+.L999:
+	vzeroupper
+
+	movq   		SP, %rsp
+	movq	   (%rsp), %rbx
+	movq	  8(%rsp), %rbp
+	movq	 16(%rsp), %r12
+	movq	 24(%rsp), %r13
+	movq	 32(%rsp), %r14
+	movq	 40(%rsp), %r15
+
+#ifdef WINDOWS_ABI
+	movq	 48(%rsp), %rdi
+	movq	 56(%rsp), %rsi
+	vmovups	 64(%rsp), %xmm6
+	vmovups	 80(%rsp), %xmm7
+	vmovups	 96(%rsp), %xmm8
+	vmovups	112(%rsp), %xmm9
+	vmovups	128(%rsp), %xmm10
+	vmovups	144(%rsp), %xmm11
+	vmovups	160(%rsp), %xmm12
+	vmovups	176(%rsp), %xmm13
+	vmovups	192(%rsp), %xmm14
+	vmovups	208(%rsp), %xmm15
+#endif
+
+	addq	$ STACKSIZE, %rsp
+	ret
+
+	EPILOGUE
diff --git a/kernel/x86_64/dgemm_kernel_16x2_haswell.S b/kernel/x86_64/dgemm_kernel_16x2_haswell.S
index 98b582c0d..899c5f241 100644
--- a/kernel/x86_64/dgemm_kernel_16x2_haswell.S
+++ b/kernel/x86_64/dgemm_kernel_16x2_haswell.S
@@ -1,5215 +1,5215 @@
-/*********************************************************************/
-/* Copyright 2009, 2010 The University of Texas at Austin.           */
-/* All rights reserved.                                              */
-/*                                                                   */
-/* Redistribution and use in source and binary forms, with or        */
-/* without modification, are permitted provided that the following   */
-/* conditions are met:                                               */
-/*                                                                   */
-/*   1. Redistributions of source code must retain the above         */
-/*      copyright notice, this list of conditions and the following  */
-/*      disclaimer.                                                  */
-/*                                                                   */
-/*   2. Redistributions in binary form must reproduce the above      */
-/*      copyright notice, this list of conditions and the following  */
-/*      disclaimer in the documentation and/or other materials       */
-/*      provided with the distribution.                              */
-/*                                                                   */
-/*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */
-/*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */
-/*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */
-/*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */
-/*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */
-/*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */
-/*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */
-/*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */
-/*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */
-/*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */
-/*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */
-/*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */
-/*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */
-/*    POSSIBILITY OF SUCH DAMAGE.                                    */
-/*                                                                   */
-/* The views and conclusions contained in the software and           */
-/* documentation are those of the authors and should not be          */
-/* interpreted as representing official policies, either expressed   */
-/* or implied, of The University of Texas at Austin.                 */
-/*********************************************************************/
-
-/*********************************************************************
-* 2013/10/20 Saar
-*        BLASTEST               : OK
-*        CTEST                  : OK
-*        TEST                   : OK
-
-*
-*
-* 2013/10/20 Saar
-* Parameter:
-*       DGEMM_DEFAULT_UNROLL_N  2
-*       DGEMM_DEFAULT_UNROLL_M  16
-*       DGEMM_DEFAULT_P         192
-*       DGEMM_DEFAULT_Q         128
-*	A_PR1			512
-*
-*
-* Performance without prefetch of B:
-*       1 thread:       45.8 GFLOPS (MKL:  45)
-*       2 threads:      80.0 GFLOPS (MKL:  91)
-*       4 threads:     135.0 GFLOPS (MKL: 135)
-*********************************************************************/
-
-
-#define ASSEMBLER
-#include "common.h"
- 
-#define OLD_M	%rdi
-#define OLD_N	%rsi
-#define M	%r13
-#define J	%r14
-#define OLD_K	%rdx
-
-#define A	%rcx
-#define B	%r8
-#define C	%r9
-#define LDC	%r10
-	
-#define I	%r11
-#define AO	%rdi
-#define BO	%rsi
-#define	CO1	%r15
-#define K	%r12
-#define BI	%rbp
-#define	SP	%rbx
-
-#define BO1	%rdi
-#define BO2	%r15
-
-#ifndef WINDOWS_ABI
-
-#define STACKSIZE 96
-
-#else
-
-#define STACKSIZE 256
-
-#define OLD_A		40 + STACKSIZE(%rsp)
-#define OLD_B		48 + STACKSIZE(%rsp)
-#define OLD_C		56 + STACKSIZE(%rsp)
-#define OLD_LDC		64 + STACKSIZE(%rsp)
-#define OLD_OFFSET	72 + STACKSIZE(%rsp)
-
-#endif
-
-#define L_BUFFER_SIZE 512*8*4
-#define LB2_OFFSET    512*8*2
-
-#define Ndiv6	 24(%rsp)
-#define Nmod6	 32(%rsp)
-#define N	 40(%rsp)
-#define ALPHA	 48(%rsp)
-#define OFFSET	 56(%rsp)
-#define KK	 64(%rsp)
-#define KKK	 72(%rsp)
-#define BUFFER1	           128(%rsp)
-#define BUFFER2	LB2_OFFSET+128(%rsp)
-
-#if defined(OS_WINDOWS)
-#if   L_BUFFER_SIZE > 16384
-#define STACK_TOUCH \
-        movl    $0,  4096 * 4(%rsp);\
-        movl    $0,  4096 * 3(%rsp);\
-        movl    $0,  4096 * 2(%rsp);\
-        movl    $0,  4096 * 1(%rsp);
-#elif L_BUFFER_SIZE > 12288
-#define STACK_TOUCH \
-        movl    $0,  4096 * 3(%rsp);\
-        movl    $0,  4096 * 2(%rsp);\
-        movl    $0,  4096 * 1(%rsp);
-#elif L_BUFFER_SIZE > 8192
-#define STACK_TOUCH \
-        movl    $0,  4096 * 2(%rsp);\
-        movl    $0,  4096 * 1(%rsp);
-#elif L_BUFFER_SIZE > 4096
-#define STACK_TOUCH \
-        movl    $0,  4096 * 1(%rsp);
-#else
-#define STACK_TOUCH
-#endif
-#else
-#define STACK_TOUCH
-#endif
-
-#if defined(BULLDOZER)
-
-.macro VFMADD231PD_ y0,y1,y2
-	vfmaddpd \y0,\y1,\y2,\y0
-.endm
-
-.macro VFMADD231SD_ x0,x1,x2
-	vfmaddsd \x0,\x1,\x2,\x0
-.endm
-
-#else
-
-.macro VFMADD231PD_ y0,y1,y2
-	vfmadd231pd \y2,\y1,\y0
-.endm
-
-.macro VFMADD231SD_ x0,x1,x2
-	vfmadd231sd \x2,\x1,\x0
-.endm
-
-#endif
-
-
-#define	A_PR1	512
-#define	B_PR1	256
-
-/*******************************************************************************************
-* 3 lines of N
-*******************************************************************************************/
-
-.macro KERNEL16x3_SUBN
-	prefetcht0	A_PR1(AO)
-	vbroadcastsd	-12 * SIZE(BO), %ymm1
-	vmovaps 	-16 * SIZE(AO), %ymm0
-	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
-	vbroadcastsd	-11 * SIZE(BO), %ymm2
-	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
-	vbroadcastsd	-10 * SIZE(BO), %ymm3
-	VFMADD231PD_  	%ymm6,%ymm3,%ymm0
-	vmovaps 	-12 * SIZE(AO), %ymm0
-	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
-	prefetcht0	A_PR1+64(AO)
-	VFMADD231PD_  	%ymm8,%ymm2,%ymm0
-	VFMADD231PD_  	%ymm9,%ymm3,%ymm0
-	vmovaps 	 -8 * SIZE(AO), %ymm0
-	VFMADD231PD_  	%ymm10,%ymm1,%ymm0
-	VFMADD231PD_  	%ymm11,%ymm2,%ymm0
-	VFMADD231PD_  	%ymm12,%ymm3,%ymm0
-	vmovaps 	 -4 * SIZE(AO), %ymm0
-	VFMADD231PD_  	%ymm13,%ymm1,%ymm0
-	VFMADD231PD_  	%ymm14,%ymm2,%ymm0
-	VFMADD231PD_  	%ymm15,%ymm3,%ymm0
-	addq	$ 3*SIZE , BO	
-	addq	$ 16*SIZE, AO
-.endm
-
-
-.macro KERNEL8x3_SUBN
-	//prefetcht0	A_PR1(AO)
-	vbroadcastsd	-12 * SIZE(BO), %ymm1
-	vmovaps 	-16 * SIZE(AO), %ymm0
-	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
-	vbroadcastsd	-11 * SIZE(BO), %ymm2
-	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
-	vbroadcastsd	-10 * SIZE(BO), %ymm3
-	VFMADD231PD_  	%ymm6,%ymm3,%ymm0
-	vmovaps 	-12 * SIZE(AO), %ymm0
-	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
-	//prefetcht0	A_PR1+64(AO)
-	VFMADD231PD_  	%ymm8,%ymm2,%ymm0
-	VFMADD231PD_  	%ymm9,%ymm3,%ymm0
-	prefetcht0	B_PR1(BO)
-	addq	$ 3*SIZE , BO	
-	addq	$ 8*SIZE, AO
-.endm
-
-.macro KERNEL4x3_SUBN
-	vbroadcastsd	-12 * SIZE(BO), %ymm1
-	vmovaps 	-16 * SIZE(AO), %ymm0
-	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
-	vbroadcastsd	-11 * SIZE(BO), %ymm2
-	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
-	vbroadcastsd	-10 * SIZE(BO), %ymm3
-	VFMADD231PD_  	%ymm6,%ymm3,%ymm0
-	addq	$ 3*SIZE , BO	
-	addq	$ 4*SIZE, AO
-.endm
-
-.macro KERNEL2x3_SUBN
-	vmovsd	-12 * SIZE(BO), %xmm1
-	vmovsd 	-16 * SIZE(AO), %xmm0
-	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
-	vmovsd	-11 * SIZE(BO), %xmm2
-	VFMADD231SD_  	%xmm5,%xmm2,%xmm0
-	vmovsd	-10 * SIZE(BO), %xmm3
-	VFMADD231SD_  	%xmm6,%xmm3,%xmm0
-	vmovsd 	-15 * SIZE(AO), %xmm0
-	VFMADD231SD_  	%xmm8,%xmm1,%xmm0
-	VFMADD231SD_  	%xmm10,%xmm2,%xmm0
-	VFMADD231SD_  	%xmm12,%xmm3,%xmm0
-	addq	$ 3*SIZE , BO	
-	addq	$ 2*SIZE, AO
-.endm
-
-.macro KERNEL1x3_SUBN
-	vmovsd	-12 * SIZE(BO), %xmm1
-	vmovsd 	-16 * SIZE(AO), %xmm0
-	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
-	vmovsd	-11 * SIZE(BO), %xmm2
-	VFMADD231SD_  	%xmm5,%xmm2,%xmm0
-	vmovsd	-10 * SIZE(BO), %xmm3
-	VFMADD231SD_  	%xmm6,%xmm3,%xmm0
-	addq	$ 3*SIZE , BO	
-	addq	$ 1*SIZE, AO
-.endm
-
-
-
-
-
-
-/******************************************************************************************/
-
-.macro KERNEL16x3_1
-	prefetcht0	A_PR1(AO, %rax, SIZE)
-	vbroadcastsd	 -6 * SIZE(BO, BI, SIZE), %ymm1
-	vmovups 	-32 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
-	vbroadcastsd	 -5 * SIZE(BO, BI, SIZE), %ymm2
-	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
-	vbroadcastsd	 -4 * SIZE(BO, BI, SIZE), %ymm3
-	VFMADD231PD_  	%ymm6,%ymm3,%ymm0
-	vmovups 	-28 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
-	prefetcht0	64+A_PR1(AO, %rax, SIZE)
-	VFMADD231PD_  	%ymm8,%ymm2,%ymm0
-	VFMADD231PD_  	%ymm9,%ymm3,%ymm0
-	vmovups 	-24 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm10,%ymm1,%ymm0
-	VFMADD231PD_  	%ymm11,%ymm2,%ymm0
-	VFMADD231PD_  	%ymm12,%ymm3,%ymm0
-	vmovups 	-20 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm13,%ymm1,%ymm0
-	VFMADD231PD_  	%ymm14,%ymm2,%ymm0
-	vbroadcastsd	 -3 * SIZE(BO, BI, SIZE), %ymm1
-	vbroadcastsd	 -2 * SIZE(BO, BI, SIZE), %ymm2
-	VFMADD231PD_  	%ymm15,%ymm3,%ymm0
-.endm
-
-
-
-
-.macro KERNEL16x3_2
-	prefetcht0	128+A_PR1(AO, %rax, SIZE)
-	vmovups 	-16 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
-	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
-	vbroadcastsd	 -1 * SIZE(BO, BI, SIZE), %ymm3
-	VFMADD231PD_  	%ymm6,%ymm3,%ymm0
-	vmovups 	-12 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
-	VFMADD231PD_  	%ymm8,%ymm2,%ymm0
-	prefetcht0	A_PR1+64(AO,%rax,SIZE)
-	VFMADD231PD_  	%ymm9,%ymm3,%ymm0
-	vmovups 	 -8 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm10,%ymm1,%ymm0
-	prefetcht0	192+A_PR1(AO, %rax, SIZE)
-	VFMADD231PD_  	%ymm11,%ymm2,%ymm0
-	VFMADD231PD_  	%ymm12,%ymm3,%ymm0
-	vmovups 	 -4 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm13,%ymm1,%ymm0
-	VFMADD231PD_  	%ymm14,%ymm2,%ymm0
-	vbroadcastsd	  0 * SIZE(BO, BI, SIZE), %ymm1
-	vbroadcastsd	  1 * SIZE(BO, BI, SIZE), %ymm2
-	VFMADD231PD_  	%ymm15,%ymm3,%ymm0
-.endm
-
-.macro KERNEL16x3_3
-	prefetcht0	256+A_PR1(AO, %rax, SIZE)
-	vmovups 	  0 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
-	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
-	vbroadcastsd	  2 * SIZE(BO, BI, SIZE), %ymm3
-	VFMADD231PD_  	%ymm6,%ymm3,%ymm0
-	vmovups 	  4 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
-	prefetcht0	320+A_PR1(AO, %rax, SIZE)
-	VFMADD231PD_  	%ymm8,%ymm2,%ymm0
-	VFMADD231PD_  	%ymm9,%ymm3,%ymm0
-	vmovups 	  8 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm10,%ymm1,%ymm0
-	VFMADD231PD_  	%ymm11,%ymm2,%ymm0
-	VFMADD231PD_  	%ymm12,%ymm3,%ymm0
-	vmovups 	 12 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm13,%ymm1,%ymm0
-	VFMADD231PD_  	%ymm14,%ymm2,%ymm0
-	vbroadcastsd	  3 * SIZE(BO, BI, SIZE), %ymm1
-	vbroadcastsd	  4 * SIZE(BO, BI, SIZE), %ymm2
-	VFMADD231PD_  	%ymm15,%ymm3,%ymm0
-.endm
-
-.macro KERNEL16x3_4
-	prefetcht0	384+A_PR1(AO, %rax, SIZE)
-	vmovups 	 16 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
-	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
-	vbroadcastsd	  5 * SIZE(BO, BI, SIZE), %ymm3
-	VFMADD231PD_  	%ymm6,%ymm3,%ymm0
-	vmovups 	 20 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
-	prefetcht0	448+A_PR1(AO, %rax, SIZE)
-	VFMADD231PD_  	%ymm8,%ymm2,%ymm0
-	VFMADD231PD_  	%ymm9,%ymm3,%ymm0
-	vmovups 	 24 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm10,%ymm1,%ymm0
-	VFMADD231PD_  	%ymm11,%ymm2,%ymm0
-	addq	$12, BI	
-	VFMADD231PD_  	%ymm12,%ymm3,%ymm0
-	vmovups 	 28 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm13,%ymm1,%ymm0
-	VFMADD231PD_  	%ymm14,%ymm2,%ymm0
-	addq	$64, %rax 
-	VFMADD231PD_  	%ymm15,%ymm3,%ymm0
-.endm
-
-.macro KERNEL16x3_SUB
-	vbroadcastsd	 -6 * SIZE(BO, BI, SIZE), %ymm1
-	vmovups 	-32 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
-	vbroadcastsd	 -5 * SIZE(BO, BI, SIZE), %ymm2
-	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
-	vbroadcastsd	 -4 * SIZE(BO, BI, SIZE), %ymm3
-	VFMADD231PD_  	%ymm6,%ymm3,%ymm0
-	vmovups 	-28 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
-	VFMADD231PD_  	%ymm8,%ymm2,%ymm0
-	VFMADD231PD_  	%ymm9,%ymm3,%ymm0
-	vmovups 	-24 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm10,%ymm1,%ymm0
-	VFMADD231PD_  	%ymm11,%ymm2,%ymm0
-	VFMADD231PD_  	%ymm12,%ymm3,%ymm0
-	vmovups 	-20 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm13,%ymm1,%ymm0
-	VFMADD231PD_  	%ymm14,%ymm2,%ymm0
-	VFMADD231PD_  	%ymm15,%ymm3,%ymm0
-	addq	$3 , BI	
-	addq	$16, %rax 
-.endm
-
-.macro SAVE16x3
-
-	vbroadcastsd	ALPHA, %ymm0
-
-	vmulpd	%ymm0 , %ymm4 , %ymm4
-	vmulpd	%ymm0 , %ymm7 , %ymm7
-	vmulpd	%ymm0 , %ymm10, %ymm10
-	vmulpd	%ymm0 , %ymm13, %ymm13
-
-	vmulpd	%ymm0 , %ymm5 , %ymm5
-	vmulpd	%ymm0 , %ymm8 , %ymm8
-	vmulpd	%ymm0 , %ymm11, %ymm11
-	vmulpd	%ymm0 , %ymm14, %ymm14
-
-	vmulpd	%ymm0 , %ymm6 , %ymm6
-	vmulpd	%ymm0 , %ymm9 , %ymm9
-	vmulpd	%ymm0 , %ymm12, %ymm12
-	vmulpd	%ymm0 , %ymm15, %ymm15
-
-#if !defined(TRMMKERNEL)
-
-	vaddpd 	        (CO1), %ymm4,%ymm4
-	vaddpd  4 * SIZE(CO1), %ymm7,%ymm7
-	vaddpd  8 * SIZE(CO1), %ymm10,%ymm10
-	vaddpd 12 * SIZE(CO1), %ymm13,%ymm13
-
-	vaddpd 	        (CO1, LDC), %ymm5,%ymm5
-	vaddpd  4 * SIZE(CO1, LDC), %ymm8,%ymm8
-	vaddpd  8 * SIZE(CO1, LDC), %ymm11,%ymm11
-	vaddpd 12 * SIZE(CO1, LDC), %ymm14,%ymm14
-
-	vaddpd 	        (CO1, LDC, 2), %ymm6,%ymm6
-	vaddpd  4 * SIZE(CO1, LDC, 2), %ymm9,%ymm9
-	vaddpd  8 * SIZE(CO1, LDC, 2), %ymm12,%ymm12
-	vaddpd 12 * SIZE(CO1, LDC, 2), %ymm15,%ymm15
-
-#endif
-
-	vmovups	%ymm4 ,  	(CO1)
-	vmovups	%ymm7 , 4 * SIZE(CO1)
-	vmovups	%ymm10, 8 * SIZE(CO1)
-	vmovups	%ymm13,12 * SIZE(CO1)
-
-	vmovups	%ymm5 ,  	(CO1, LDC)
-	vmovups	%ymm8 , 4 * SIZE(CO1, LDC)
-	vmovups	%ymm11, 8 * SIZE(CO1, LDC)
-	vmovups	%ymm14,12 * SIZE(CO1, LDC)
-
-	vmovups	%ymm6 ,  	(CO1, LDC, 2)
-	vmovups	%ymm9 , 4 * SIZE(CO1, LDC, 2)
-	vmovups	%ymm12, 8 * SIZE(CO1, LDC, 2)
-	vmovups	%ymm15,12 * SIZE(CO1, LDC, 2)
-
-.endm
-
-
-
-/*******************************************************************************************/
-
-.macro KERNEL8x3_1
-	prefetcht0	A_PR1(AO, %rax, SIZE)
-	vbroadcastsd	 -6 * SIZE(BO, BI, SIZE), %ymm1
-	vmovups 	-32 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
-	vbroadcastsd	 -5 * SIZE(BO, BI, SIZE), %ymm2
-	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
-	vbroadcastsd	 -4 * SIZE(BO, BI, SIZE), %ymm3
-	VFMADD231PD_  	%ymm6,%ymm3,%ymm0
-	vmovups 	-28 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
-	VFMADD231PD_  	%ymm8,%ymm2,%ymm0
-	VFMADD231PD_  	%ymm9,%ymm3,%ymm0
-.endm
-
-.macro KERNEL8x3_2
-	prefetcht0	64+A_PR1(AO, %rax, SIZE)
-	vbroadcastsd	 -3 * SIZE(BO, BI, SIZE), %ymm1
-	vmovups 	-24 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
-	vbroadcastsd	 -2 * SIZE(BO, BI, SIZE), %ymm2
-	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
-	vbroadcastsd	 -1 * SIZE(BO, BI, SIZE), %ymm3
-	VFMADD231PD_  	%ymm6,%ymm3,%ymm0
-	vmovups 	-20 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
-	VFMADD231PD_  	%ymm8,%ymm2,%ymm0
-	VFMADD231PD_  	%ymm9,%ymm3,%ymm0
-.endm
-
-.macro KERNEL8x3_3
-	prefetcht0	128+A_PR1(AO, %rax, SIZE)
-	vbroadcastsd	  0 * SIZE(BO, BI, SIZE), %ymm1
-	vmovups 	-16 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
-	vbroadcastsd	  1 * SIZE(BO, BI, SIZE), %ymm2
-	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
-	vbroadcastsd	  2 * SIZE(BO, BI, SIZE), %ymm3
-	VFMADD231PD_  	%ymm6,%ymm3,%ymm0
-	vmovups 	-12 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
-	VFMADD231PD_  	%ymm8,%ymm2,%ymm0
-	VFMADD231PD_  	%ymm9,%ymm3,%ymm0
-.endm
-
-.macro KERNEL8x3_4
-	prefetcht0	192+A_PR1(AO, %rax, SIZE)
-	vbroadcastsd	  3 * SIZE(BO, BI, SIZE), %ymm1
-	vmovups 	 -8 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
-	vbroadcastsd	  4 * SIZE(BO, BI, SIZE), %ymm2
-	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
-	vbroadcastsd	  5 * SIZE(BO, BI, SIZE), %ymm3
-	VFMADD231PD_  	%ymm6,%ymm3,%ymm0
-	vmovups 	 -4 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
-	VFMADD231PD_  	%ymm8,%ymm2,%ymm0
-	VFMADD231PD_  	%ymm9,%ymm3,%ymm0
-	addq	$12, BI
-	addq	$32, %rax
-.endm
-
-.macro KERNEL8x3_SUB
-	vbroadcastsd	 -6 * SIZE(BO, BI, SIZE), %ymm1
-	vmovups 	-32 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
-	vbroadcastsd	 -5 * SIZE(BO, BI, SIZE), %ymm2
-	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
-	vbroadcastsd	 -4 * SIZE(BO, BI, SIZE), %ymm3
-	VFMADD231PD_  	%ymm6,%ymm3,%ymm0
-	vmovups 	-28 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
-	VFMADD231PD_  	%ymm8,%ymm2,%ymm0
-	VFMADD231PD_  	%ymm9,%ymm3,%ymm0
-	addq	$3 , BI
-	addq	$8 , %rax
-.endm
-
-.macro SAVE8x3
-
-	vbroadcastsd	ALPHA, %ymm0
-
-	vmulpd	%ymm0 , %ymm4 , %ymm4
-	vmulpd	%ymm0 , %ymm7 , %ymm7
-
-	vmulpd	%ymm0 , %ymm5 , %ymm5
-	vmulpd	%ymm0 , %ymm8 , %ymm8
-
-	vmulpd	%ymm0 , %ymm6 , %ymm6
-	vmulpd	%ymm0 , %ymm9 , %ymm9
-
-#if !defined(TRMMKERNEL)
-
-	vaddpd 	        (CO1), %ymm4,%ymm4
-	vaddpd  4 * SIZE(CO1), %ymm7,%ymm7
-
-	vaddpd 	        (CO1, LDC), %ymm5,%ymm5
-	vaddpd  4 * SIZE(CO1, LDC), %ymm8,%ymm8
-
-	vaddpd 	        (CO1, LDC, 2), %ymm6,%ymm6
-	vaddpd  4 * SIZE(CO1, LDC, 2), %ymm9,%ymm9
-
-#endif
-
-	vmovups	%ymm4 ,  	(CO1)
-	vmovups	%ymm7 , 4 * SIZE(CO1)
-
-	vmovups	%ymm5 ,  	(CO1, LDC)
-	vmovups	%ymm8 , 4 * SIZE(CO1, LDC)
-
-	vmovups	%ymm6 ,  	(CO1, LDC, 2)
-	vmovups	%ymm9 , 4 * SIZE(CO1, LDC, 2)
-
-.endm
-
-
-
-/*******************************************************************************************/
-
-.macro KERNEL4x3_1
-	prefetcht0	A_PR1(AO, %rax, SIZE)
-	vbroadcastsd	 -6 * SIZE(BO, BI, SIZE), %ymm1
-	vmovups 	-32 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
-	vbroadcastsd	 -5 * SIZE(BO, BI, SIZE), %ymm2
-	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
-	vbroadcastsd	 -4 * SIZE(BO, BI, SIZE), %ymm3
-	VFMADD231PD_  	%ymm6,%ymm3,%ymm0
-.endm
-
-.macro KERNEL4x3_2
-	vbroadcastsd	 -3 * SIZE(BO, BI, SIZE), %ymm1
-	vmovups 	-28 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
-	vbroadcastsd	 -2 * SIZE(BO, BI, SIZE), %ymm2
-	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
-	vbroadcastsd	 -1 * SIZE(BO, BI, SIZE), %ymm3
-	VFMADD231PD_  	%ymm6,%ymm3,%ymm0
-.endm
-
-.macro KERNEL4x3_3
-	prefetcht0	A_PR1(AO, %rax, SIZE)
-	vbroadcastsd	  0 * SIZE(BO, BI, SIZE), %ymm1
-	vmovups 	-24 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
-	vbroadcastsd	  1 * SIZE(BO, BI, SIZE), %ymm2
-	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
-	vbroadcastsd	  2 * SIZE(BO, BI, SIZE), %ymm3
-	VFMADD231PD_  	%ymm6,%ymm3,%ymm0
-.endm
-
-.macro KERNEL4x3_4
-	vbroadcastsd	  3 * SIZE(BO, BI, SIZE), %ymm1
-	vmovups 	-20 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
-	vbroadcastsd	  4 * SIZE(BO, BI, SIZE), %ymm2
-	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
-	vbroadcastsd	  5 * SIZE(BO, BI, SIZE), %ymm3
-	VFMADD231PD_  	%ymm6,%ymm3,%ymm0
-	addq	$12, BI
-	addq	$16, %rax
-.endm
-
-.macro KERNEL4x3_SUB
-	vbroadcastsd	 -6 * SIZE(BO, BI, SIZE), %ymm1
-	vmovups 	-32 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
-	vbroadcastsd	 -5 * SIZE(BO, BI, SIZE), %ymm2
-	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
-	vbroadcastsd	 -4 * SIZE(BO, BI, SIZE), %ymm3
-	VFMADD231PD_  	%ymm6,%ymm3,%ymm0
-	addq	$3 , BI
-	addq	$4 , %rax
-.endm
-
-.macro SAVE4x3
-
-	vbroadcastsd	ALPHA, %ymm0
-
-	vmulpd	%ymm0 , %ymm4 , %ymm4
-	vmulpd	%ymm0 , %ymm5 , %ymm5
-	vmulpd	%ymm0 , %ymm6 , %ymm6
-
-#if !defined(TRMMKERNEL)
-
-	vaddpd 	        (CO1), %ymm4,%ymm4
-	vaddpd 	        (CO1, LDC), %ymm5,%ymm5
-	vaddpd 	        (CO1, LDC, 2), %ymm6,%ymm6
-
-#endif
-
-	vmovups	%ymm4 ,  	(CO1)
-	vmovups	%ymm5 ,  	(CO1, LDC)
-	vmovups	%ymm6 ,  	(CO1, LDC, 2)
-
-.endm
-
-
-/*******************************************************************************************/
-
-.macro KERNEL2x3_1
-	prefetcht0	A_PR1(AO, %rax, SIZE)
-	vmovsd	 -6 * SIZE(BO, BI, SIZE), %xmm1
-	vmovsd 	-32 * SIZE(AO, %rax, SIZE), %xmm0
-	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
-	vmovsd	 -5 * SIZE(BO, BI, SIZE), %xmm2
-	VFMADD231SD_  	%xmm5,%xmm2,%xmm0
-	vmovsd	 -4 * SIZE(BO, BI, SIZE), %xmm3
-	VFMADD231SD_  	%xmm6,%xmm3,%xmm0
-	vmovsd 	-31 * SIZE(AO, %rax, SIZE), %xmm0
-	VFMADD231SD_  	%xmm8,%xmm1,%xmm0
-	VFMADD231SD_  	%xmm10,%xmm2,%xmm0
-	VFMADD231SD_  	%xmm12,%xmm3,%xmm0
-.endm
-
-.macro KERNEL2x3_2
-	vmovsd	 -3 * SIZE(BO, BI, SIZE), %xmm1
-	vmovsd 	-30 * SIZE(AO, %rax, SIZE), %xmm0
-	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
-	vmovsd	 -2 * SIZE(BO, BI, SIZE), %xmm2
-	VFMADD231SD_  	%xmm5,%xmm2,%xmm0
-	vmovsd	 -1 * SIZE(BO, BI, SIZE), %xmm3
-	VFMADD231SD_  	%xmm6,%xmm3,%xmm0
-	vmovsd 	-29 * SIZE(AO, %rax, SIZE), %xmm0
-	VFMADD231SD_  	%xmm8,%xmm1,%xmm0
-	VFMADD231SD_  	%xmm10,%xmm2,%xmm0
-	VFMADD231SD_  	%xmm12,%xmm3,%xmm0
-.endm
-
-.macro KERNEL2x3_3
-	vmovsd	  0 * SIZE(BO, BI, SIZE), %xmm1
-	vmovsd 	-28 * SIZE(AO, %rax, SIZE), %xmm0
-	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
-	vmovsd	  1 * SIZE(BO, BI, SIZE), %xmm2
-	VFMADD231SD_  	%xmm5,%xmm2,%xmm0
-	vmovsd	  2 * SIZE(BO, BI, SIZE), %xmm3
-	VFMADD231SD_  	%xmm6,%xmm3,%xmm0
-	vmovsd 	-27 * SIZE(AO, %rax, SIZE), %xmm0
-	VFMADD231SD_  	%xmm8,%xmm1,%xmm0
-	VFMADD231SD_  	%xmm10,%xmm2,%xmm0
-	VFMADD231SD_  	%xmm12,%xmm3,%xmm0
-.endm
-
-.macro KERNEL2x3_4
-	vmovsd	  3 * SIZE(BO, BI, SIZE), %xmm1
-	vmovsd 	-26 * SIZE(AO, %rax, SIZE), %xmm0
-	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
-	vmovsd	  4 * SIZE(BO, BI, SIZE), %xmm2
-	VFMADD231SD_  	%xmm5,%xmm2,%xmm0
-	vmovsd	  5 * SIZE(BO, BI, SIZE), %xmm3
-	VFMADD231SD_  	%xmm6,%xmm3,%xmm0
-	vmovsd 	-25 * SIZE(AO, %rax, SIZE), %xmm0
-	VFMADD231SD_  	%xmm8,%xmm1,%xmm0
-	VFMADD231SD_  	%xmm10,%xmm2,%xmm0
-	VFMADD231SD_  	%xmm12,%xmm3,%xmm0
-	addq	$12, BI
-	addq	$8, %rax
-.endm
-
-.macro KERNEL2x3_SUB
-	vmovsd	 -6 * SIZE(BO, BI, SIZE), %xmm1
-	vmovsd 	-32 * SIZE(AO, %rax, SIZE), %xmm0
-	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
-	vmovsd	 -5 * SIZE(BO, BI, SIZE), %xmm2
-	VFMADD231SD_  	%xmm5,%xmm2,%xmm0
-	vmovsd	 -4 * SIZE(BO, BI, SIZE), %xmm3
-	VFMADD231SD_  	%xmm6,%xmm3,%xmm0
-	vmovsd 	-31 * SIZE(AO, %rax, SIZE), %xmm0
-	VFMADD231SD_  	%xmm8,%xmm1,%xmm0
-	VFMADD231SD_  	%xmm10,%xmm2,%xmm0
-	VFMADD231SD_  	%xmm12,%xmm3,%xmm0
-	addq	$3 , BI
-	addq	$2 , %rax
-.endm
-
-.macro SAVE2x3
-
-	vmovsd	ALPHA, %xmm0
-
-	vmulsd	%xmm0 , %xmm4 , %xmm4
-	vmulsd	%xmm0 , %xmm8 , %xmm8
-	vmulsd	%xmm0 , %xmm5 , %xmm5
-	vmulsd	%xmm0 , %xmm10, %xmm10
-	vmulsd	%xmm0 , %xmm6 , %xmm6
-	vmulsd	%xmm0 , %xmm12, %xmm12
-
-#if !defined(TRMMKERNEL)
-
-	vaddsd 	 (CO1), %xmm4,%xmm4
-	vaddsd 1 * SIZE(CO1), %xmm8,%xmm8
-	vaddsd 	 (CO1, LDC), %xmm5,%xmm5
-	vaddsd 1 * SIZE(CO1, LDC), %xmm10,%xmm10
-	vaddsd 	 (CO1, LDC, 2), %xmm6,%xmm6
-	vaddsd 1 * SIZE(CO1, LDC, 2), %xmm12,%xmm12
-
-#endif
-
-	vmovsd	%xmm4 ,  	(CO1)
-	vmovsd	%xmm8 , 1 * SIZE(CO1)
-	vmovsd	%xmm5 ,  	(CO1, LDC)
-	vmovsd	%xmm10, 1 * SIZE(CO1, LDC)
-	vmovsd	%xmm6 ,  	(CO1, LDC, 2)
-	vmovsd	%xmm12, 1 * SIZE(CO1, LDC, 2)
-
-.endm
-
-/*******************************************************************************************/
-
-.macro KERNEL1x3_1
-	vmovsd	 -6 * SIZE(BO, BI, SIZE), %xmm1
-	vmovsd 	-32 * SIZE(AO, %rax, SIZE), %xmm0
-	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
-	vmovsd	 -5 * SIZE(BO, BI, SIZE), %xmm2
-	VFMADD231SD_  	%xmm5,%xmm2,%xmm0
-	vmovsd	 -4 * SIZE(BO, BI, SIZE), %xmm3
-	VFMADD231SD_  	%xmm6,%xmm3,%xmm0
-.endm
-
-.macro KERNEL1x3_2
-	vmovsd	 -3 * SIZE(BO, BI, SIZE), %xmm1
-	vmovsd 	-31 * SIZE(AO, %rax, SIZE), %xmm0
-	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
-	vmovsd	 -2 * SIZE(BO, BI, SIZE), %xmm2
-	VFMADD231SD_  	%xmm5,%xmm2,%xmm0
-	vmovsd	 -1 * SIZE(BO, BI, SIZE), %xmm3
-	VFMADD231SD_  	%xmm6,%xmm3,%xmm0
-.endm
-
-.macro KERNEL1x3_3
-	vmovsd	  0 * SIZE(BO, BI, SIZE), %xmm1
-	vmovsd 	-30 * SIZE(AO, %rax, SIZE), %xmm0
-	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
-	vmovsd	  1 * SIZE(BO, BI, SIZE), %xmm2
-	VFMADD231SD_  	%xmm5,%xmm2,%xmm0
-	vmovsd	  2 * SIZE(BO, BI, SIZE), %xmm3
-	VFMADD231SD_  	%xmm6,%xmm3,%xmm0
-.endm
-
-.macro KERNEL1x3_4
-	vmovsd	  3 * SIZE(BO, BI, SIZE), %xmm1
-	vmovsd 	-29 * SIZE(AO, %rax, SIZE), %xmm0
-	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
-	vmovsd	  4 * SIZE(BO, BI, SIZE), %xmm2
-	VFMADD231SD_  	%xmm5,%xmm2,%xmm0
-	vmovsd	  5 * SIZE(BO, BI, SIZE), %xmm3
-	VFMADD231SD_  	%xmm6,%xmm3,%xmm0
-	addq	$12, BI
-	addq	$4, %rax
-.endm
-
-.macro KERNEL1x3_SUB
-	vmovsd	 -6 * SIZE(BO, BI, SIZE), %xmm1
-	vmovsd 	-32 * SIZE(AO, %rax, SIZE), %xmm0
-	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
-	vmovsd	 -5 * SIZE(BO, BI, SIZE), %xmm2
-	VFMADD231SD_  	%xmm5,%xmm2,%xmm0
-	vmovsd	 -4 * SIZE(BO, BI, SIZE), %xmm3
-	VFMADD231SD_  	%xmm6,%xmm3,%xmm0
-	addq	$3 , BI
-	addq	$1 , %rax
-.endm
-
-.macro SAVE1x3
-
-	vmovsd	ALPHA, %xmm0
-
-	vmulsd	%xmm0 , %xmm4 , %xmm4
-	vmulsd	%xmm0 , %xmm5 , %xmm5
-	vmulsd	%xmm0 , %xmm6 , %xmm6
-
-#if !defined(TRMMKERNEL)
-
-	vaddsd 	 (CO1), %xmm4,%xmm4
-	vaddsd 	 (CO1, LDC), %xmm5,%xmm5
-	vaddsd 	 (CO1, LDC, 2), %xmm6,%xmm6
-
-#endif
-
-	vmovsd	%xmm4 ,  	(CO1)
-	vmovsd	%xmm5 ,  	(CO1, LDC)
-	vmovsd	%xmm6 ,  	(CO1, LDC, 2)
-
-.endm
-
-
-/*******************************************************************************************/
-
-/*******************************************************************************************
-* 2 lines of N
-*******************************************************************************************/
-
-.macro KERNEL16x2_1
-	prefetcht0	A_PR1(AO, %rax, SIZE)
-	vbroadcastsd	 -4 * SIZE(BO, BI, SIZE), %ymm1
-	vmovups 	-32 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
-	vbroadcastsd	 -3 * SIZE(BO, BI, SIZE), %ymm2
-	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
-	vmovups 	-28 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
-	prefetcht0	64+A_PR1(AO, %rax, SIZE)
-	VFMADD231PD_  	%ymm8,%ymm2,%ymm0
-	vmovups 	-24 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm10,%ymm1,%ymm0
-	VFMADD231PD_  	%ymm11,%ymm2,%ymm0
-	vmovups 	-20 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm13,%ymm1,%ymm0
-	VFMADD231PD_  	%ymm14,%ymm2,%ymm0
-.endm
-
-.macro KERNEL16x2_2
-	prefetcht0	128+A_PR1(AO, %rax, SIZE)
-	vbroadcastsd	 -2 * SIZE(BO, BI, SIZE), %ymm1
-	vmovups 	-16 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
-	vbroadcastsd	 -1 * SIZE(BO, BI, SIZE), %ymm2
-	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
-	vmovups 	-12 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
-	prefetcht0	192+A_PR1(AO, %rax, SIZE)
-	VFMADD231PD_  	%ymm8,%ymm2,%ymm0
-	vmovups 	 -8 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm10,%ymm1,%ymm0
-	VFMADD231PD_  	%ymm11,%ymm2,%ymm0
-	vmovups 	 -4 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm13,%ymm1,%ymm0
-	VFMADD231PD_  	%ymm14,%ymm2,%ymm0
-.endm
-
-.macro KERNEL16x2_3
-	prefetcht0	256+A_PR1(AO, %rax, SIZE)
-	vbroadcastsd	  0 * SIZE(BO, BI, SIZE), %ymm1
-	vmovups 	  0 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
-	vbroadcastsd	  1 * SIZE(BO, BI, SIZE), %ymm2
-	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
-	vmovups 	  4 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
-	prefetcht0	320+A_PR1(AO, %rax, SIZE)
-	VFMADD231PD_  	%ymm8,%ymm2,%ymm0
-	vmovups 	  8 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm10,%ymm1,%ymm0
-	VFMADD231PD_  	%ymm11,%ymm2,%ymm0
-	vmovups 	 12 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm13,%ymm1,%ymm0
-	VFMADD231PD_  	%ymm14,%ymm2,%ymm0
-.endm
-
-.macro KERNEL16x2_4
-	prefetcht0	384+A_PR1(AO, %rax, SIZE)
-	vbroadcastsd	  2 * SIZE(BO, BI, SIZE), %ymm1
-	vmovups 	 16 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
-	vbroadcastsd	  3 * SIZE(BO, BI, SIZE), %ymm2
-	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
-	vmovups 	 20 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
-	prefetcht0	448+A_PR1(AO, %rax, SIZE)
-	VFMADD231PD_  	%ymm8,%ymm2,%ymm0
-	vmovups 	 24 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm10,%ymm1,%ymm0
-	VFMADD231PD_  	%ymm11,%ymm2,%ymm0
-	vmovups 	 28 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm13,%ymm1,%ymm0
-	VFMADD231PD_  	%ymm14,%ymm2,%ymm0
-	addq	$8, BI
-	addq	$64, %rax
-.endm
-
-.macro KERNEL16x2_SUB
-	vbroadcastsd	 -4 * SIZE(BO, BI, SIZE), %ymm1
-	vmovups 	-32 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
-	vbroadcastsd	 -3 * SIZE(BO, BI, SIZE), %ymm2
-	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
-	vmovups 	-28 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
-	VFMADD231PD_  	%ymm8,%ymm2,%ymm0
-	vmovups 	-24 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm10,%ymm1,%ymm0
-	VFMADD231PD_  	%ymm11,%ymm2,%ymm0
-	vmovups 	-20 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm13,%ymm1,%ymm0
-	VFMADD231PD_  	%ymm14,%ymm2,%ymm0
-	addq	$2, BI
-	addq	$16, %rax
-.endm
-
-.macro SAVE16x2
-
-	vbroadcastsd	ALPHA, %ymm0
-
-	vmulpd	%ymm0 , %ymm4 , %ymm4
-	vmulpd	%ymm0 , %ymm7 , %ymm7
-	vmulpd	%ymm0 , %ymm10, %ymm10
-	vmulpd	%ymm0 , %ymm13, %ymm13
-
-	vmulpd	%ymm0 , %ymm5 , %ymm5
-	vmulpd	%ymm0 , %ymm8 , %ymm8
-	vmulpd	%ymm0 , %ymm11, %ymm11
-	vmulpd	%ymm0 , %ymm14, %ymm14
-
-#if !defined(TRMMKERNEL)
-
-	vaddpd 	        (CO1), %ymm4,%ymm4
-	vaddpd  4 * SIZE(CO1), %ymm7,%ymm7
-	vaddpd  8 * SIZE(CO1), %ymm10,%ymm10
-	vaddpd 12 * SIZE(CO1), %ymm13,%ymm13
-
-	vaddpd 	        (CO1, LDC), %ymm5,%ymm5
-	vaddpd  4 * SIZE(CO1, LDC), %ymm8,%ymm8
-	vaddpd  8 * SIZE(CO1, LDC), %ymm11,%ymm11
-	vaddpd 12 * SIZE(CO1, LDC), %ymm14,%ymm14
-
-#endif
-
-	vmovups	%ymm4 ,  	(CO1)
-	vmovups	%ymm7 , 4 * SIZE(CO1)
-	vmovups	%ymm10, 8 * SIZE(CO1)
-	vmovups	%ymm13,12 * SIZE(CO1)
-
-	vmovups	%ymm5 ,  	(CO1, LDC)
-	vmovups	%ymm8 , 4 * SIZE(CO1, LDC)
-	vmovups	%ymm11, 8 * SIZE(CO1, LDC)
-	vmovups	%ymm14,12 * SIZE(CO1, LDC)
-
-.endm
-
-
-
-/*******************************************************************************************/
-
-.macro KERNEL8x2_1
-	prefetcht0	A_PR1(AO, %rax, SIZE)
-	vbroadcastsd	 -4 * SIZE(BO, BI, SIZE), %ymm1
-	vmovups 	-32 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
-	vbroadcastsd	 -3 * SIZE(BO, BI, SIZE), %ymm2
-	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
-	vmovups 	-28 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
-	VFMADD231PD_  	%ymm8,%ymm2,%ymm0
-.endm
-
-.macro KERNEL8x2_2
-	prefetcht0	64+A_PR1(AO, %rax, SIZE)
-	vbroadcastsd	 -2 * SIZE(BO, BI, SIZE), %ymm1
-	vmovups 	-24 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
-	vbroadcastsd	 -1 * SIZE(BO, BI, SIZE), %ymm2
-	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
-	vmovups 	-20 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
-	VFMADD231PD_  	%ymm8,%ymm2,%ymm0
-.endm
-
-.macro KERNEL8x2_3
-	prefetcht0	128+A_PR1(AO, %rax, SIZE)
-	vbroadcastsd	  0 * SIZE(BO, BI, SIZE), %ymm1
-	vmovups 	-16 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
-	vbroadcastsd	  1 * SIZE(BO, BI, SIZE), %ymm2
-	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
-	vmovups 	-12 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
-	VFMADD231PD_  	%ymm8,%ymm2,%ymm0
-.endm
-
-.macro KERNEL8x2_4
-	prefetcht0	192+A_PR1(AO, %rax, SIZE)
-	vbroadcastsd	  2 * SIZE(BO, BI, SIZE), %ymm1
-	vmovups 	 -8 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
-	vbroadcastsd	  3 * SIZE(BO, BI, SIZE), %ymm2
-	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
-	vmovups 	 -4 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
-	VFMADD231PD_  	%ymm8,%ymm2,%ymm0
-	addq	$8, BI				 
-	addq	$32, %rax 			 
-.endm
-
-.macro KERNEL8x2_SUB
-	vbroadcastsd	 -4 * SIZE(BO, BI, SIZE), %ymm1
-	vmovups 	-32 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
-	vbroadcastsd	 -3 * SIZE(BO, BI, SIZE), %ymm2
-	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
-	vmovups 	-28 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
-	VFMADD231PD_  	%ymm8,%ymm2,%ymm0
-	addq	$2, BI				 
-	addq	$8 , %rax 			 
-.endm
-
-.macro SAVE8x2
-
-	vbroadcastsd	ALPHA, %ymm0
-
-	vmulpd	%ymm0 , %ymm4 , %ymm4
-	vmulpd	%ymm0 , %ymm7 , %ymm7
-
-	vmulpd	%ymm0 , %ymm5 , %ymm5
-	vmulpd	%ymm0 , %ymm8 , %ymm8
-
-#if !defined(TRMMKERNEL)
-
-	vaddpd 	        (CO1), %ymm4,%ymm4
-	vaddpd  4 * SIZE(CO1), %ymm7,%ymm7
-
-	vaddpd 	        (CO1, LDC), %ymm5,%ymm5
-	vaddpd  4 * SIZE(CO1, LDC), %ymm8,%ymm8
-
-#endif
-
-	vmovups	%ymm4 ,  	(CO1)
-	vmovups	%ymm7 , 4 * SIZE(CO1)
-
-	vmovups	%ymm5 ,  	(CO1, LDC)
-	vmovups	%ymm8 , 4 * SIZE(CO1, LDC)
-
-.endm
-
-
-
-/*******************************************************************************************/
-
-.macro KERNEL4x2_1
-	prefetcht0	A_PR1(AO, %rax, SIZE)
-	vbroadcastsd	 -4 * SIZE(BO, BI, SIZE), %ymm1
-	vmovups 	-32 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
-	vbroadcastsd	 -3 * SIZE(BO, BI, SIZE), %ymm2
-	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
-.endm
-
-.macro KERNEL4x2_2
-	vbroadcastsd	 -2 * SIZE(BO, BI, SIZE), %ymm1
-	vmovups 	-28 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
-	vbroadcastsd	 -1 * SIZE(BO, BI, SIZE), %ymm2
-	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
-.endm
-
-.macro KERNEL4x2_3
-	prefetcht0	64+A_PR1(AO, %rax, SIZE)
-	vbroadcastsd	  0 * SIZE(BO, BI, SIZE), %ymm1
-	vmovups 	-24 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
-	vbroadcastsd	  1 * SIZE(BO, BI, SIZE), %ymm2
-	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
-.endm
-
-.macro KERNEL4x2_4
-	vbroadcastsd	  2 * SIZE(BO, BI, SIZE), %ymm1
-	vmovups 	-20 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
-	vbroadcastsd	  3 * SIZE(BO, BI, SIZE), %ymm2
-	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
-	addq	$8, BI				 
-	addq	$16, %rax 			 
-.endm
-
-.macro KERNEL4x2_SUB
-	vbroadcastsd	 -4 * SIZE(BO, BI, SIZE), %ymm1
-	vmovups 	-32 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
-	vbroadcastsd	 -3 * SIZE(BO, BI, SIZE), %ymm2
-	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
-	addq	$2, BI				 
-	addq	$4 , %rax 			 
-.endm
-
-.macro SAVE4x2
-
-	vbroadcastsd	ALPHA, %ymm0
-
-	vmulpd	%ymm0 , %ymm4 , %ymm4
-	vmulpd	%ymm0 , %ymm5 , %ymm5
-
-#if !defined(TRMMKERNEL)
-
-	vaddpd 	        (CO1), %ymm4,%ymm4
-	vaddpd 	        (CO1, LDC), %ymm5,%ymm5
-
-#endif
-
-	vmovups	%ymm4 ,  	(CO1)
-	vmovups	%ymm5 ,  	(CO1, LDC)
-
-.endm
-
-
-/*******************************************************************************************/
-
-.macro KERNEL2x2_1
-	prefetcht0	A_PR1(AO, %rax, SIZE)
-	vmovsd	 -4 * SIZE(BO, BI, SIZE), %xmm1
-	vmovsd 	-32 * SIZE(AO, %rax, SIZE), %xmm0
-	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
-	vmovsd	 -3 * SIZE(BO, BI, SIZE), %xmm2
-	VFMADD231SD_  	%xmm5,%xmm2,%xmm0
-	vmovsd 	-31 * SIZE(AO, %rax, SIZE), %xmm0
-	VFMADD231SD_  	%xmm8,%xmm1,%xmm0
-	VFMADD231SD_  	%xmm10,%xmm2,%xmm0
-.endm
-
-.macro KERNEL2x2_2
-	vmovsd	 -2 * SIZE(BO, BI, SIZE), %xmm1
-	vmovsd 	-30 * SIZE(AO, %rax, SIZE), %xmm0
-	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
-	vmovsd	 -1 * SIZE(BO, BI, SIZE), %xmm2
-	VFMADD231SD_  	%xmm5,%xmm2,%xmm0
-	vmovsd 	-29 * SIZE(AO, %rax, SIZE), %xmm0
-	VFMADD231SD_  	%xmm8,%xmm1,%xmm0
-	VFMADD231SD_  	%xmm10,%xmm2,%xmm0
-.endm
-
-.macro KERNEL2x2_3
-	vmovsd	  0 * SIZE(BO, BI, SIZE), %xmm1
-	vmovsd 	-28 * SIZE(AO, %rax, SIZE), %xmm0
-	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
-	vmovsd	  1 * SIZE(BO, BI, SIZE), %xmm2
-	VFMADD231SD_  	%xmm5,%xmm2,%xmm0
-	vmovsd 	-27 * SIZE(AO, %rax, SIZE), %xmm0
-	VFMADD231SD_  	%xmm8,%xmm1,%xmm0
-	VFMADD231SD_  	%xmm10,%xmm2,%xmm0
-.endm
-
-.macro KERNEL2x2_4
-	vmovsd	  2 * SIZE(BO, BI, SIZE), %xmm1
-	vmovsd 	-26 * SIZE(AO, %rax, SIZE), %xmm0
-	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
-	vmovsd	  3 * SIZE(BO, BI, SIZE), %xmm2
-	VFMADD231SD_  	%xmm5,%xmm2,%xmm0
-	vmovsd 	-25 * SIZE(AO, %rax, SIZE), %xmm0
-	VFMADD231SD_  	%xmm8,%xmm1,%xmm0
-	VFMADD231SD_  	%xmm10,%xmm2,%xmm0
-	addq	$8, BI				 
-	addq	$8, %rax 			 
-.endm
-
-.macro KERNEL2x2_SUB
-	vmovsd	 -4 * SIZE(BO, BI, SIZE), %xmm1
-	vmovsd 	-32 * SIZE(AO, %rax, SIZE), %xmm0
-	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
-	vmovsd	 -3 * SIZE(BO, BI, SIZE), %xmm2
-	VFMADD231SD_  	%xmm5,%xmm2,%xmm0
-	vmovsd 	-31 * SIZE(AO, %rax, SIZE), %xmm0
-	VFMADD231SD_  	%xmm8,%xmm1,%xmm0
-	VFMADD231SD_  	%xmm10,%xmm2,%xmm0
-	addq	$2, BI				 
-	addq	$2, %rax 			 
-.endm
-
-.macro SAVE2x2
-
-	vmovsd	ALPHA, %xmm0
-
-	vmulsd	%xmm0 , %xmm4 , %xmm4
-	vmulsd	%xmm0 , %xmm8 , %xmm8
-	vmulsd	%xmm0 , %xmm5 , %xmm5
-	vmulsd	%xmm0 , %xmm10, %xmm10
-
-#if !defined(TRMMKERNEL)
-
-	vaddsd 	 (CO1), %xmm4,%xmm4
-	vaddsd 1 * SIZE(CO1), %xmm8,%xmm8
-	vaddsd 	 (CO1, LDC), %xmm5,%xmm5
-	vaddsd 1 * SIZE(CO1, LDC), %xmm10,%xmm10
-
-#endif
-
-	vmovsd	%xmm4 ,  	(CO1)
-	vmovsd	%xmm8 , 1 * SIZE(CO1)
-	vmovsd	%xmm5 ,  	(CO1, LDC)
-	vmovsd	%xmm10, 1 * SIZE(CO1, LDC)
-
-.endm
-
-
-/*******************************************************************************************/
-
-.macro KERNEL1x2_1
-	vmovsd	 -4 * SIZE(BO, BI, SIZE), %xmm1
-	vmovsd 	-32 * SIZE(AO, %rax, SIZE), %xmm0
-	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
-	vmovsd	 -3 * SIZE(BO, BI, SIZE), %xmm2
-	VFMADD231SD_  	%xmm5,%xmm2,%xmm0
-.endm
-
-.macro KERNEL1x2_2
-	vmovsd	 -2 * SIZE(BO, BI, SIZE), %xmm1
-	vmovsd 	-31 * SIZE(AO, %rax, SIZE), %xmm0
-	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
-	vmovsd	 -1 * SIZE(BO, BI, SIZE), %xmm2
-	VFMADD231SD_  	%xmm5,%xmm2,%xmm0
-.endm
-
-.macro KERNEL1x2_3
-	vmovsd	  0 * SIZE(BO, BI, SIZE), %xmm1
-	vmovsd 	-30 * SIZE(AO, %rax, SIZE), %xmm0
-	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
-	vmovsd	  1 * SIZE(BO, BI, SIZE), %xmm2
-	VFMADD231SD_  	%xmm5,%xmm2,%xmm0
-.endm
-
-.macro KERNEL1x2_4
-	vmovsd	  2 * SIZE(BO, BI, SIZE), %xmm1
-	vmovsd 	-29 * SIZE(AO, %rax, SIZE), %xmm0
-	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
-	vmovsd	  3 * SIZE(BO, BI, SIZE), %xmm2
-	VFMADD231SD_  	%xmm5,%xmm2,%xmm0
-	addq	$8, BI				 
-	addq	$4, %rax 			 
-.endm
-
-.macro KERNEL1x2_SUB
-	vmovsd	 -4 * SIZE(BO, BI, SIZE), %xmm1
-	vmovsd 	-32 * SIZE(AO, %rax, SIZE), %xmm0
-	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
-	vmovsd	 -3 * SIZE(BO, BI, SIZE), %xmm2
-	VFMADD231SD_  	%xmm5,%xmm2,%xmm0
-	addq	$2, BI				 
-	addq	$1, %rax 			 
-.endm
-
-.macro SAVE1x2
-
-	vmovsd	ALPHA, %xmm0
-
-	vmulsd	%xmm0 , %xmm4 , %xmm4
-	vmulsd	%xmm0 , %xmm5 , %xmm5
-
-#if !defined(TRMMKERNEL)
-
-	vaddsd 	 (CO1), %xmm4,%xmm4
-	vaddsd 	 (CO1, LDC), %xmm5,%xmm5
-
-#endif
-
-	vmovsd	%xmm4 ,  	(CO1)
-	vmovsd	%xmm5 ,  	(CO1, LDC)
-
-.endm
-
-
-/*******************************************************************************************/
-
-/*******************************************************************************************
-* 1 line of N
-*******************************************************************************************/
-
-.macro KERNEL16x1_1
-	vbroadcastsd	 -2 * SIZE(BO, BI, SIZE), %ymm1
-	vmovups 	-32 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
-	vmovups 	-28 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
-	vmovups 	-24 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm10,%ymm1,%ymm0
-	vmovups 	-20 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm13,%ymm1,%ymm0
-.endm
-
-.macro KERNEL16x1_2
-	vbroadcastsd	 -1 * SIZE(BO, BI, SIZE), %ymm1
-	vmovups 	-16 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
-	vmovups 	-12 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
-	vmovups 	 -8 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm10,%ymm1,%ymm0
-	vmovups 	 -4 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm13,%ymm1,%ymm0
-.endm
-
-.macro KERNEL16x1_3
-	vbroadcastsd	  0 * SIZE(BO, BI, SIZE), %ymm1
-	vmovups 	  0 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
-	vmovups 	  4 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
-	vmovups 	  8 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm10,%ymm1,%ymm0
-	vmovups 	 12 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm13,%ymm1,%ymm0
-.endm
-
-.macro KERNEL16x1_4
-	vbroadcastsd	  1 * SIZE(BO, BI, SIZE), %ymm1
-	vmovups 	 16 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
-	vmovups 	 20 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
-	vmovups 	 24 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm10,%ymm1,%ymm0
-	vmovups 	 28 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm13,%ymm1,%ymm0
-	addq	$4, BI				 
-	addq	$64, %rax 			 
-.endm
-
-.macro KERNEL16x1_SUB
-	vbroadcastsd	 -2 * SIZE(BO, BI, SIZE), %ymm1
-	vmovups 	-32 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
-	vmovups 	-28 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
-	vmovups 	-24 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm10,%ymm1,%ymm0
-	vmovups 	-20 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm13,%ymm1,%ymm0
-	addq	$1, BI				 
-	addq	$16, %rax 			 
-.endm
-
-.macro SAVE16x1
-
-	vbroadcastsd	ALPHA, %ymm0
-
-	vmulpd	%ymm0 , %ymm4 , %ymm4
-	vmulpd	%ymm0 , %ymm7 , %ymm7
-	vmulpd	%ymm0 , %ymm10, %ymm10
-	vmulpd	%ymm0 , %ymm13, %ymm13
-
-#if !defined(TRMMKERNEL)
-
-	vaddpd 	        (CO1), %ymm4,%ymm4
-	vaddpd  4 * SIZE(CO1), %ymm7,%ymm7
-	vaddpd  8 * SIZE(CO1), %ymm10,%ymm10
-	vaddpd 12 * SIZE(CO1), %ymm13,%ymm13
-
-#endif
-
-	vmovups	%ymm4 ,  	(CO1)
-	vmovups	%ymm7 , 4 * SIZE(CO1)
-	vmovups	%ymm10, 8 * SIZE(CO1)
-	vmovups	%ymm13,12 * SIZE(CO1)
-
-.endm
-
-
-
-/*******************************************************************************************/
-
-.macro KERNEL8x1_1
-	vbroadcastsd	 -2 * SIZE(BO, BI, SIZE), %ymm1
-	vmovups 	-32 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
-	vmovups 	-28 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
-.endm
-
-.macro KERNEL8x1_2
-	vbroadcastsd	 -1 * SIZE(BO, BI, SIZE), %ymm1
-	vmovups 	-24 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
-	vmovups 	-20 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
-.endm
-
-.macro KERNEL8x1_3
-	vbroadcastsd	  0 * SIZE(BO, BI, SIZE), %ymm1
-	vmovups 	-16 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
-	vmovups 	-12 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
-.endm
-
-.macro KERNEL8x1_4
-	vbroadcastsd	  1 * SIZE(BO, BI, SIZE), %ymm1
-	vmovups 	 -8 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
-	vmovups 	 -4 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
-	addq	$4, BI				 
-	addq	$32, %rax 			 
-.endm
-
-.macro KERNEL8x1_SUB
-	vbroadcastsd	 -2 * SIZE(BO, BI, SIZE), %ymm1
-	vmovups 	-32 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
-	vmovups 	-28 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
-	addq	$1, BI				 
-	addq	$8 , %rax 			 
-.endm
-
-.macro SAVE8x1
-
-	vbroadcastsd	ALPHA, %ymm0
-
-	vmulpd	%ymm0 , %ymm4 , %ymm4
-	vmulpd	%ymm0 , %ymm7 , %ymm7
-
-#if !defined(TRMMKERNEL)
-
-	vaddpd 	        (CO1), %ymm4,%ymm4
-	vaddpd  4 * SIZE(CO1), %ymm7,%ymm7
-
-#endif
-
-	vmovups	%ymm4 ,  	(CO1)
-	vmovups	%ymm7 , 4 * SIZE(CO1)
-
-.endm
-
-
-
-/*******************************************************************************************/
-
-.macro KERNEL4x1_1
-	vbroadcastsd	 -2 * SIZE(BO, BI, SIZE), %ymm1
-	vmovups 	-32 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
-.endm
-
-.macro KERNEL4x1_2
-	vbroadcastsd	 -1 * SIZE(BO, BI, SIZE), %ymm1
-	vmovups 	-28 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
-.endm
-
-.macro KERNEL4x1_3
-	vbroadcastsd	  0 * SIZE(BO, BI, SIZE), %ymm1
-	vmovups 	-24 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
-.endm
-
-.macro KERNEL4x1_4
-	vbroadcastsd	  1 * SIZE(BO, BI, SIZE), %ymm1
-	vmovups 	-20 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
-	addq	$4, BI				 
-	addq	$16, %rax 			 
-.endm
-
-.macro KERNEL4x1_SUB
-	vbroadcastsd	 -2 * SIZE(BO, BI, SIZE), %ymm1
-	vmovups 	-32 * SIZE(AO, %rax, SIZE), %ymm0
-	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
-	addq	$1, BI				 
-	addq	$4 , %rax 			 
-.endm
-
-.macro SAVE4x1
-
-	vbroadcastsd	ALPHA, %ymm0
-
-	vmulpd	%ymm0 , %ymm4 , %ymm4
-
-#if !defined(TRMMKERNEL)
-
-	vaddpd 	        (CO1), %ymm4,%ymm4
-
-#endif
-
-	vmovups	%ymm4 ,  	(CO1)
-
-.endm
-
-
-/*******************************************************************************************/
-
-.macro KERNEL2x1_1
-	vmovsd	 -2 * SIZE(BO, BI, SIZE), %xmm1
-	vmovsd 	-32 * SIZE(AO, %rax, SIZE), %xmm0
-	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
-	vmovsd 	-31 * SIZE(AO, %rax, SIZE), %xmm0
-	VFMADD231SD_  	%xmm8,%xmm1,%xmm0
-.endm
-
-.macro KERNEL2x1_2
-	vmovsd	 -1 * SIZE(BO, BI, SIZE), %xmm1
-	vmovsd 	-30 * SIZE(AO, %rax, SIZE), %xmm0
-	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
-	vmovsd 	-29 * SIZE(AO, %rax, SIZE), %xmm0
-	VFMADD231SD_  	%xmm8,%xmm1,%xmm0
-.endm
-
-.macro KERNEL2x1_3
-	vmovsd	  0 * SIZE(BO, BI, SIZE), %xmm1
-	vmovsd 	-28 * SIZE(AO, %rax, SIZE), %xmm0
-	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
-	vmovsd 	-27 * SIZE(AO, %rax, SIZE), %xmm0
-	VFMADD231SD_  	%xmm8,%xmm1,%xmm0
-.endm
-
-.macro KERNEL2x1_4
-	vmovsd	  1 * SIZE(BO, BI, SIZE), %xmm1
-	vmovsd 	-26 * SIZE(AO, %rax, SIZE), %xmm0
-	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
-	vmovsd 	-25 * SIZE(AO, %rax, SIZE), %xmm0
-	VFMADD231SD_  	%xmm8,%xmm1,%xmm0
-	addq	$4, BI				 
-	addq	$8, %rax 			 
-.endm
-
-.macro KERNEL2x1_SUB
-	vmovsd	 -2 * SIZE(BO, BI, SIZE), %xmm1
-	vmovsd 	-32 * SIZE(AO, %rax, SIZE), %xmm0
-	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
-	vmovsd 	-31 * SIZE(AO, %rax, SIZE), %xmm0
-	VFMADD231SD_  	%xmm8,%xmm1,%xmm0
-	addq	$1, BI				 
-	addq	$2 , %rax 			 
-.endm
-
-.macro SAVE2x1
-
-	vmovsd	ALPHA, %xmm0
-
-	vmulsd	%xmm0 , %xmm4 , %xmm4
-	vmulsd	%xmm0 , %xmm8 , %xmm8
-
-#if !defined(TRMMKERNEL)
-
-	vaddsd 	 (CO1), %xmm4,%xmm4
-	vaddsd 1 * SIZE(CO1), %xmm8,%xmm8
-
-#endif
-
-	vmovsd	%xmm4 ,  	(CO1)
-	vmovsd	%xmm8 , 1 * SIZE(CO1)
-
-.endm
-
-
-/*******************************************************************************************/
-
-.macro KERNEL1x1_1
-	vmovsd	 -2 * SIZE(BO, BI, SIZE), %xmm1
-	vmovsd 	-32 * SIZE(AO, %rax, SIZE), %xmm0
-	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
-.endm
-
-.macro KERNEL1x1_2
-	vmovsd	 -1 * SIZE(BO, BI, SIZE), %xmm1
-	vmovsd 	-31 * SIZE(AO, %rax, SIZE), %xmm0
-	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
-.endm
-
-.macro KERNEL1x1_3
-	vmovsd	  0 * SIZE(BO, BI, SIZE), %xmm1
-	vmovsd 	-30 * SIZE(AO, %rax, SIZE), %xmm0
-	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
-.endm
-
-.macro KERNEL1x1_4
-	vmovsd	  1 * SIZE(BO, BI, SIZE), %xmm1
-	vmovsd 	-29 * SIZE(AO, %rax, SIZE), %xmm0
-	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
-	addq	$ 4, BI				 
-	addq	$ 4, %rax 			 
-.endm
-
-.macro KERNEL1x1_SUB
-	vmovsd	 -2 * SIZE(BO, BI, SIZE), %xmm1
-	vmovsd 	-32 * SIZE(AO, %rax, SIZE), %xmm0
-	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
-	addq	$ 1, BI				 
-	addq	$ 1 , %rax 			 
-.endm
-
-.macro SAVE1x1
-
-	vmovsd	ALPHA, %xmm0
-
-	vmulsd	%xmm0 , %xmm4 , %xmm4
-
-#if !defined(TRMMKERNEL)
-
-	vaddsd 	 (CO1), %xmm4,%xmm4
-
-#endif
-
-	vmovsd	%xmm4 ,  	(CO1)
-
-.endm
-
-
-/*******************************************************************************************/
-
-#if !defined(TRMMKERNEL)
-
-
-	PROLOGUE
-	PROFCODE
-	
-	subq	$STACKSIZE, %rsp
-	movq	%rbx,   (%rsp)
-	movq	%rbp,  8(%rsp)
-	movq	%r12, 16(%rsp)
-	movq	%r13, 24(%rsp)
-	movq	%r14, 32(%rsp)
-	movq	%r15, 40(%rsp)
-
-	vzeroupper
-
-#ifdef WINDOWS_ABI
-	movq	%rdi,    48(%rsp)
-	movq	%rsi,    56(%rsp)
-	movups	%xmm6,   64(%rsp)
-	movups	%xmm7,   80(%rsp)
-	movups	%xmm8,   96(%rsp)
-	movups	%xmm9,  112(%rsp)
-	movups	%xmm10, 128(%rsp)
-	movups	%xmm11, 144(%rsp)
-	movups	%xmm12, 160(%rsp)
-	movups	%xmm13, 176(%rsp)
-	movups	%xmm14, 192(%rsp)
-	movups	%xmm15, 208(%rsp)
-
-	movq	ARG1,      OLD_M
-	movq	ARG2,      OLD_N
-	movq	ARG3,      OLD_K
-	movq	OLD_A,     A
-	movq	OLD_B,     B
-	movq	OLD_C,     C
-	movq	OLD_LDC,   LDC
-
-	vmovaps	%xmm3, %xmm0
-
-#else
-	movq	STACKSIZE +  8(%rsp), LDC
-
-#endif
-
-	movq    %rsp, SP      # save old stack
-        subq    $128 + L_BUFFER_SIZE, %rsp
-        andq    $-4096, %rsp    # align stack
-
-        STACK_TOUCH
-
-	cmpq	$0, OLD_M
-	je	.L999
-
-	cmpq	$0, OLD_N
-	je	.L999
-
-	cmpq	$0, OLD_K
-	je	.L999
-
-	movq	OLD_M, M
-	movq	OLD_N, N
-	movq	OLD_K, K
-
-	vmovsd	 %xmm0, ALPHA
-
-	salq	$BASE_SHIFT, LDC
-
-	movq    N, %rax
-        xorq    %rdx, %rdx
-        movq    $6,  %rdi
-        divq    %rdi                    //    N / 6
-        movq    %rax, Ndiv6             //    N / 6
-        movq    %rdx, Nmod6             //    N % 6
-
-
-	movq	Ndiv6,  J
-	cmpq	$0, J
-	je	.L2_0
-	ALIGN_4
-
-.L6_01:
-        // copy to sub buffer
-        movq    K, %rax
-        salq    $1,%rax                 // K * 2 ; read 2 values
-        movq    B, BO1
-        leaq    (B,%rax, SIZE), BO2     // next offset to BO2
-        leaq    BUFFER1, BO             // first buffer to BO
-        movq    K, %rax
-	sarq	$3 , %rax		// K / 8
-	jz	.L6_01a_2
-        ALIGN_4
-
-.L6_01a_1:
-
-        prefetcht0 512(BO1)
-        prefetcht0 512(BO2)
-        prefetchw  512(BO)
-
-
-	vmovups	0 * SIZE(BO1), %xmm0
-	vmovups	2 * SIZE(BO1), %xmm2
-	vmovups	4 * SIZE(BO1), %xmm4
-	vmovups	6 * SIZE(BO1), %xmm6
-	vmovsd  0 * SIZE(BO2), %xmm1
-	vmovsd  2 * SIZE(BO2), %xmm3
-	vmovsd  4 * SIZE(BO2), %xmm5
-	vmovsd  6 * SIZE(BO2), %xmm7
-	vmovups	%xmm0, 0*SIZE(BO)
-	vmovsd	%xmm1, 2*SIZE(BO)
-	vmovups	%xmm2, 3*SIZE(BO)
-	vmovsd	%xmm3, 5*SIZE(BO)
-	vmovups	%xmm4, 6*SIZE(BO)
-	vmovsd	%xmm5, 8*SIZE(BO)
-	vmovups	%xmm6, 9*SIZE(BO)
-	vmovsd	%xmm7,11*SIZE(BO)
-	addq	$ 8*SIZE,BO1
-	addq	$ 8*SIZE,BO2
-	addq	$ 12*SIZE,BO
-
-	vmovups	0 * SIZE(BO1), %xmm0
-	vmovups	2 * SIZE(BO1), %xmm2
-	vmovups	4 * SIZE(BO1), %xmm4
-	vmovups	6 * SIZE(BO1), %xmm6
-	vmovsd  0 * SIZE(BO2), %xmm1
-	vmovsd  2 * SIZE(BO2), %xmm3
-	vmovsd  4 * SIZE(BO2), %xmm5
-	vmovsd  6 * SIZE(BO2), %xmm7
-	vmovups	%xmm0, 0*SIZE(BO)
-	vmovsd	%xmm1, 2*SIZE(BO)
-	vmovups	%xmm2, 3*SIZE(BO)
-	vmovsd	%xmm3, 5*SIZE(BO)
-	vmovups	%xmm4, 6*SIZE(BO)
-	vmovsd	%xmm5, 8*SIZE(BO)
-	vmovups	%xmm6, 9*SIZE(BO)
-	vmovsd	%xmm7,11*SIZE(BO)
-	addq	$ 8*SIZE,BO1
-	addq	$ 8*SIZE,BO2
-	addq	$ 12*SIZE,BO
-
-	decq	%rax
-	jnz	.L6_01a_1
-
-
-
-.L6_01a_2:
-
-	movq    K, %rax
-        andq    $7, %rax                // K % 8
-        jz      .L6_02c
-        ALIGN_4
-
-
-.L6_02b:
-
-	vmovups	0 * SIZE(BO1), %xmm0
-	vmovsd  0 * SIZE(BO2), %xmm2
-	vmovups	%xmm0, 0*SIZE(BO)
-	vmovsd	%xmm2, 2*SIZE(BO)
-	addq	$ 2*SIZE,BO1
-	addq	$ 2*SIZE,BO2
-	addq	$ 3*SIZE,BO
-	decq	%rax
-	jnz	.L6_02b
-
-.L6_02c:
-
-	movq	K, %rax
-	salq	$1,%rax			// K * 2
-	leaq	(B,%rax, SIZE), BO1	// next offset to BO1
-	leaq	(BO1,%rax, SIZE), BO2	// next offset to BO2
-	leaq    BUFFER2, BO		// second buffer to BO
-	movq	K, %rax
-	sarq	$3 , %rax		// K / 8
-	jz	.L6_02c_2
-	ALIGN_4
-
-.L6_02c_1:
-
-	prefetcht0 512(BO2)
-        prefetchw  512(BO)
-
-	vmovups	0 * SIZE(BO2), %xmm0
-	vmovups	2 * SIZE(BO2), %xmm2
-	vmovups	4 * SIZE(BO2), %xmm4
-	vmovups	6 * SIZE(BO2), %xmm6
-	vmovsd  1 * SIZE(BO1), %xmm1
-	vmovsd  3 * SIZE(BO1), %xmm3
-	vmovsd  5 * SIZE(BO1), %xmm5
-	vmovsd  7 * SIZE(BO1), %xmm7
-	vmovsd	%xmm1, 0*SIZE(BO)
-	vmovups	%xmm0, 1*SIZE(BO)
-	vmovsd	%xmm3, 3*SIZE(BO)
-	vmovups	%xmm2, 4*SIZE(BO)
-	vmovsd	%xmm5, 6*SIZE(BO)
-	vmovups	%xmm4, 7*SIZE(BO)
-	vmovsd	%xmm7, 9*SIZE(BO)
-	vmovups	%xmm6,10*SIZE(BO)
-	addq	$8*SIZE,BO1
-	addq	$8*SIZE,BO2
-	addq	$12*SIZE,BO
-
-
-	vmovups	0 * SIZE(BO2), %xmm0
-	vmovups	2 * SIZE(BO2), %xmm2
-	vmovups	4 * SIZE(BO2), %xmm4
-	vmovups	6 * SIZE(BO2), %xmm6
-	vmovsd  1 * SIZE(BO1), %xmm1
-	vmovsd  3 * SIZE(BO1), %xmm3
-	vmovsd  5 * SIZE(BO1), %xmm5
-	vmovsd  7 * SIZE(BO1), %xmm7
-	vmovsd	%xmm1, 0*SIZE(BO)
-	vmovups	%xmm0, 1*SIZE(BO)
-	vmovsd	%xmm3, 3*SIZE(BO)
-	vmovups	%xmm2, 4*SIZE(BO)
-	vmovsd	%xmm5, 6*SIZE(BO)
-	vmovups	%xmm4, 7*SIZE(BO)
-	vmovsd	%xmm7, 9*SIZE(BO)
-	vmovups	%xmm6,10*SIZE(BO)
-	addq	$8*SIZE,BO1
-	addq	$8*SIZE,BO2
-	addq	$12*SIZE,BO
-
-	decq	%rax
-	jnz	.L6_02c_1
-
-
-.L6_02c_2:
-
-	movq    K, %rax
-        andq    $7, %rax                // K % 8
-        jz      .L6_03c
-        ALIGN_4
-
-.L6_03b:
-
-	vmovsd	  1*SIZE(BO1), %xmm0
-	vmovups	  0*SIZE(BO2), %xmm1
-	vmovsd	%xmm0, 0*SIZE(BO)
-	vmovups	%xmm1, 1*SIZE(BO)
-	addq	$2*SIZE,BO1
-	addq	$2*SIZE,BO2
-	addq	$3*SIZE,BO
-	decq	%rax
-	jnz	.L6_03b
-
-
-.L6_03c:
-
-	movq	BO2, B			// next offset of B
-
-.L6_10:
-	movq	C, CO1
-	leaq	(C, LDC, 2), C		 
-	leaq	(C, LDC, 1), C		// c += 3 * ldc
-
-	
-	movq	A, AO		 	// aoffset = a
-	addq	$16 * SIZE, AO
-
-	movq	M,  I
-	sarq	$4, I			// i = (m >> 4)
-	je	.L6_20
-
-	ALIGN_4
-
-.L6_11:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $12 * SIZE, BO
-
-	prefetcht0	(CO1)
-	prefetcht0	(CO1,LDC,1)
-	prefetcht0	(CO1,LDC,2)
-	prefetcht0	64(CO1)
-	prefetcht0	64(CO1,LDC,1)
-	prefetcht0	64(CO1,LDC,2)
-
-	vzeroall
-
-        movq    K, %rax
-
-	sarq $1, %rax			//  K / 8
-	je	.L6_16
-
-	ALIGN_5
-
-.L6_12:
-/*
-	prefetcht0	B_PR1(BO)
-	prefetcht0	B_PR1+64(BO)
-	prefetcht0	B_PR1+128(BO)
-*/
-	KERNEL16x3_SUBN
-	KERNEL16x3_SUBN
-/*
-	KERNEL16x3_SUBN
-	KERNEL16x3_SUBN
-
-	KERNEL16x3_SUBN
-	KERNEL16x3_SUBN
-	KERNEL16x3_SUBN
-	KERNEL16x3_SUBN
-*/
-	dec	%rax
-	jne	.L6_12
-
-.L6_16:
-        movq    K, %rax
-
-	andq	$1, %rax		# if (k & 1)
-	je .L6_19
-
-	ALIGN_4
-
-.L6_17:
-
-	KERNEL16x3_SUBN
-
-	dec	%rax
-	jne	.L6_17
-	ALIGN_4
-
-
-.L6_19:
-
-	SAVE16x3
-
-	addq	$16 * SIZE, CO1		# coffset += 16
-	decq	I			# i --
-	jg	.L6_11
-	ALIGN_4	
-
-/**************************************************************************
-* Rest of M 
-***************************************************************************/
-.L6_20:
-	// Test rest of M
-
-	testq	$15, M
-	jz	.L7_10		// to next 3 lines of N
-
-	testq	$8, M		
-	jz	.L6_21pre
-	ALIGN_4
-
-/**************************************************************************/
-
-.L6_20_1:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $12 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	sarq	$3, %rax
-	je	.L6_20_6
-
-	ALIGN_4
-
-.L6_20_2:
-
-	KERNEL8x3_SUBN
-	KERNEL8x3_SUBN
-	KERNEL8x3_SUBN
-	KERNEL8x3_SUBN
-
-	KERNEL8x3_SUBN
-	KERNEL8x3_SUBN
-	KERNEL8x3_SUBN
-	KERNEL8x3_SUBN
-	dec	%rax
-	jne	.L6_20_2
-	ALIGN_4
-
-.L6_20_6:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L6_20_9
-
-
-	ALIGN_4
-
-.L6_20_7:
-
-	KERNEL8x3_SUBN
-
-	dec	%rax
-	jne	.L6_20_7
-	ALIGN_4
-
-
-.L6_20_9:
-
-	SAVE8x3
-
-	addq	$8 * SIZE, CO1		# coffset += 8
-	ALIGN_4
-	
-
-
-/**************************************************************************/
-
-.L6_21pre:
-
-	testq	$4, M		
-	jz	.L6_30
-	ALIGN_4
-
-.L6_21:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $12 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	sarq	$3, %rax
-	je	.L6_26
-
-	ALIGN_4
-
-.L6_22:
-
-	KERNEL4x3_SUBN
-	KERNEL4x3_SUBN
-	KERNEL4x3_SUBN
-	KERNEL4x3_SUBN
-
-	KERNEL4x3_SUBN
-	KERNEL4x3_SUBN
-	KERNEL4x3_SUBN
-	KERNEL4x3_SUBN
-	dec	%rax
-	jne	.L6_22
-	ALIGN_4
-
-.L6_26:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L6_29
-
-	ALIGN_4
-
-.L6_27:
-
-	KERNEL4x3_SUBN
-
-	dec %rax
-	jne	.L6_27
-	ALIGN_4
-
-
-.L6_29:
-
-	SAVE4x3
-
-	addq	$4 * SIZE, CO1		# coffset += 4
-	ALIGN_4
-	
-
-.L6_30:
-	testq	$2, M		
-	jz	.L6_40
-
-	ALIGN_4
-
-.L6_31:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $12 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	sarq	$3, %rax
-	je	.L6_36
-	ALIGN_4
-
-.L6_32:
-
-	KERNEL2x3_SUBN
-	KERNEL2x3_SUBN
-	KERNEL2x3_SUBN
-	KERNEL2x3_SUBN
-
-	KERNEL2x3_SUBN
-	KERNEL2x3_SUBN
-	KERNEL2x3_SUBN
-	KERNEL2x3_SUBN
-	dec %rax
-	jne	.L6_32
-	ALIGN_4
-
-.L6_36:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L6_39
-
-	ALIGN_4
-
-.L6_37:
-
-	KERNEL2x3_SUBN
-
-	dec %rax
-	jne	.L6_37
-	ALIGN_4
-
-
-.L6_39:
-
-	SAVE2x3
-
-	addq	$2 * SIZE, CO1		# coffset += 2
-	ALIGN_4
-
-.L6_40:
-	testq	$1, M		
-	jz	.L7_10		// to next 3 lines of N
-
-	ALIGN_4
-
-.L6_41:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $12 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	sarq	$3,%rax
-	je	.L6_46
-
-	ALIGN_4
-
-.L6_42:
-
-	KERNEL1x3_SUBN
-	KERNEL1x3_SUBN
-	KERNEL1x3_SUBN
-	KERNEL1x3_SUBN
-
-	KERNEL1x3_SUBN
-	KERNEL1x3_SUBN
-	KERNEL1x3_SUBN
-	KERNEL1x3_SUBN
-
-	dec %rax
-	jne	.L6_42
-	ALIGN_4
-
-.L6_46:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L6_49
-
-	ALIGN_4
-
-.L6_47:
-
-	KERNEL1x3_SUBN
-
-	dec	%rax
-	jne	.L6_47
-	ALIGN_4
-
-
-.L6_49:
-
-	SAVE1x3
-
-	addq	$1 * SIZE, CO1		# coffset += 1
-	ALIGN_4
-	
-
-
-
-/***************************************************************************************************************/
-
-.L7_10:
-	movq	C, CO1
-	leaq	(C, LDC, 2), C		 
-	leaq	(C, LDC, 1), C		// c += 3 * ldc
-
-	
-	movq	A, AO		 	// aoffset = a
-	addq	$16 * SIZE, AO
-
-	movq	M,  I
-	sarq	$4, I			// i = (m >> 4)
-	je	.L7_20
-
-	ALIGN_4
-
-.L7_11:
-        leaq    BUFFER2, BO             // second buffer to BO
-        addq    $12 * SIZE, BO
-
-	prefetcht0	(CO1)
-	prefetcht0	(CO1,LDC,1)
-	prefetcht0	(CO1,LDC,2)
-	prefetcht0	64(CO1)
-	prefetcht0	64(CO1,LDC,1)
-	prefetcht0	64(CO1,LDC,2)
-
-	vzeroall
-
-        movq    K, %rax
-
-	sarq $3, %rax			// K / 8
-	je	.L7_16
-	ALIGN_5
-
-.L7_12:
-/*
-	prefetcht0	B_PR1(BO)
-	prefetcht0	B_PR1+64(BO)
-	prefetcht0	B_PR1+128(BO)
-*/
-	KERNEL16x3_SUBN
-	KERNEL16x3_SUBN
-	KERNEL16x3_SUBN
-	KERNEL16x3_SUBN
-
-	KERNEL16x3_SUBN
-	KERNEL16x3_SUBN
-	KERNEL16x3_SUBN
-	KERNEL16x3_SUBN
-	dec %rax
-	jne	.L7_12
-	ALIGN_4
-
-.L7_16:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L7_19
-
-	ALIGN_5
-
-.L7_17:
-
-	KERNEL16x3_SUBN
-
-	dec	%rax
-	jne	.L7_17
-
-
-.L7_19:
-
-	SAVE16x3
-
-	addq	$16 * SIZE, CO1		# coffset += 16
-	decq	I			# i --
-	jg	.L7_11
-	ALIGN_4	
-
-/**************************************************************************
-* Rest of M 
-***************************************************************************/
-.L7_20:
-	// Test rest of M
-
-	testq	$15, M
-	jz	.L7_60		// to next 3 lines of N
-
-	testq	$8, M		
-	jz	.L7_21pre
-	ALIGN_4
-
-/**************************************************************************/
-
-.L7_20_1:
-        leaq    BUFFER2, BO             // first buffer to BO
-        addq    $12 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	sarq	$3, %rax
-	je	.L7_20_6
-
-	ALIGN_4
-
-.L7_20_2:
-
-	KERNEL8x3_SUBN
-	KERNEL8x3_SUBN
-	KERNEL8x3_SUBN
-	KERNEL8x3_SUBN
-
-	KERNEL8x3_SUBN
-	KERNEL8x3_SUBN
-	KERNEL8x3_SUBN
-	KERNEL8x3_SUBN
-
-	dec %rax
-	jne	.L7_20_2
-	ALIGN_4
-
-.L7_20_6:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L7_20_9
-
-	ALIGN_4
-
-.L7_20_7:
-
-	KERNEL8x3_SUBN
-
-	dec %rax
-	jne	.L7_20_7
-	ALIGN_4
-
-.L7_20_9:
-
-	SAVE8x3
-
-	addq	$8 * SIZE, CO1		# coffset += 8
-	ALIGN_4
-	
-
-
-/**************************************************************************/
-
-.L7_21pre:
-
-	testq	$4, M		
-	jz	.L7_30
-	ALIGN_4
-
-.L7_21:
-        leaq    BUFFER2, BO             // second buffer to BO
-        addq    $12 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	sarq	$3, %rax
-	je	.L7_26
-
-	ALIGN_4
-
-.L7_22:
-
-	KERNEL4x3_SUBN
-	KERNEL4x3_SUBN
-	KERNEL4x3_SUBN
-	KERNEL4x3_SUBN
-
-	KERNEL4x3_SUBN
-	KERNEL4x3_SUBN
-	KERNEL4x3_SUBN
-	KERNEL4x3_SUBN
-
-	dec %rax
-	jne	.L7_22
-	ALIGN_4
-
-.L7_26:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L7_29
-
-	ALIGN_4
-
-.L7_27:
-
-	KERNEL4x3_SUBN
-
-	dec %rax
-	jne	.L7_27
-	ALIGN_4
-
-
-.L7_29:
-
-	SAVE4x3
-
-	addq	$4 * SIZE, CO1		# coffset += 4
-	ALIGN_4
-	
-
-.L7_30:
-	testq	$2, M		
-	jz	.L7_40
-
-	ALIGN_4
-
-.L7_31:
-        leaq    BUFFER2, BO             // second buffer to BO
-        addq    $12 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	sarq	$3, %rax
-	je	.L7_36
-
-	ALIGN_4
-
-.L7_32:
-
-	KERNEL2x3_SUBN
-	KERNEL2x3_SUBN
-	KERNEL2x3_SUBN
-	KERNEL2x3_SUBN
-
-	KERNEL2x3_SUBN
-	KERNEL2x3_SUBN
-	KERNEL2x3_SUBN
-	KERNEL2x3_SUBN
-
-	dec %rax
-	jne	.L7_32
-	ALIGN_4
-
-.L7_36:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L7_39
-
-	ALIGN_4
-
-.L7_37:
-
-	KERNEL2x3_SUBN
-
-	dec %rax
-	jne	.L7_37
-	ALIGN_4
-
-
-.L7_39:
-
-	SAVE2x3
-
-	addq	$2 * SIZE, CO1		# coffset += 2
-	ALIGN_4
-
-.L7_40:
-	testq	$1, M		
-	jz	.L7_60		// to next 3 lines of N
-
-	ALIGN_4
-
-.L7_41:
-        leaq    BUFFER2, BO             // second buffer to BO
-        addq    $12 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	sarq	$3, %rax
-	je	.L7_46
-
-	ALIGN_4
-
-.L7_42:
-	KERNEL1x3_SUBN
-	KERNEL1x3_SUBN
-	KERNEL1x3_SUBN
-	KERNEL1x3_SUBN
-
-	KERNEL1x3_SUBN
-	KERNEL1x3_SUBN
-	KERNEL1x3_SUBN
-	KERNEL1x3_SUBN
-
-	dec %rax
-	jne	.L7_42
-	ALIGN_4
-
-.L7_46:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L7_49
-
-	ALIGN_4
-
-.L7_47:
-
-	KERNEL1x3_SUBN
-
-	dec %rax
-	jne	.L7_47
-	ALIGN_4
-
-
-.L7_49:
-
-	SAVE1x3
-
-	addq	$1 * SIZE, CO1		# coffset += 1
-	ALIGN_4
-	
-
-
-.L7_60:
-
-	decq	J			// j --
-	jg	.L6_01
-
-
-.L2_0:
-	cmpq	$0, Nmod6		// N % 6 == 0
-	je	.L999
-
-/************************************************************************************************
-* Loop for Nmod6 / 2 > 0
-*************************************************************************************************/
-
-	movq	Nmod6, J		
-	sarq	$1, J			// j = j / 2
-	je	.L1_0
-	ALIGN_4
-
-.L2_01:
-	// copy to sub buffer
-	movq	B, BO1
-	leaq    BUFFER1, BO		// first buffer to BO
-	movq	K, %rax
-	sarq	$2, %rax		// K / 4
-	jz	.L2_01b
-	ALIGN_4
-
-.L2_01a:
-        prefetcht0 512(BO1)
-        prefetchw  512(BO)
-
-	vmovups	      (BO1), %xmm0
-	vmovups	2*SIZE(BO1), %xmm1
-	vmovups	4*SIZE(BO1), %xmm2
-	vmovups	6*SIZE(BO1), %xmm3
-
-	vmovups	%xmm0,       (BO)
-	vmovups	%xmm1, 2*SIZE(BO)
-	vmovups	%xmm2, 4*SIZE(BO)
-	vmovups	%xmm3, 6*SIZE(BO)
-
-	addq	$8*SIZE,BO1
-	addq	$8*SIZE,BO
-	decq	%rax
-	jnz	.L2_01a
-
-
-.L2_01b:
-
-        movq    K, %rax
-        andq    $3, %rax                // K % 4
-        jz      .L2_02d
-        ALIGN_4
-
-.L2_02c:
-
-	vmovups	(BO1), %xmm0
-	vmovups	%xmm0, (BO)
-	addq	$2*SIZE,BO1
-	addq	$2*SIZE,BO
-	decq	%rax
-	jnz	.L2_02c
-
-.L2_02d:
-
-	movq	BO1, B			// next offset of B
-
-.L2_10:
-	movq	C, CO1
-	leaq	(C, LDC, 2), C		// c += 2 * ldc
-
-	
-	movq	A, AO		 	// aoffset = a
-	addq	$32 * SIZE, AO
-
-	movq	M,  I
-	sarq	$4, I			// i = (m >> 4)
-	je	.L2_20
-
-	ALIGN_4
-
-.L2_11:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax			//  K = K - ( K % 8 )
-	je	.L2_16
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$4, %rax			// rax = rax * 16 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_12:
-
-	prefetcht0      B_PR1(BO,BI,8)
-	KERNEL16x2_1
-	KERNEL16x2_2
-	KERNEL16x2_3
-	KERNEL16x2_4
-
-	prefetcht0      B_PR1(BO,BI,8)
-	KERNEL16x2_1
-	KERNEL16x2_2
-	KERNEL16x2_3
-	KERNEL16x2_4
-
-	je	.L2_16
-
-	prefetcht0      B_PR1(BO,BI,8)
-	KERNEL16x2_1
-	KERNEL16x2_2
-	KERNEL16x2_3
-	KERNEL16x2_4
-
-	prefetcht0      B_PR1(BO,BI,8)
-	KERNEL16x2_1
-	KERNEL16x2_2
-	KERNEL16x2_3
-	KERNEL16x2_4
-
-	je	.L2_16
-
-	jmp	.L2_12
-	ALIGN_4
-
-.L2_16:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_19
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$4, %rax			// rax = rax * 16 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_17:
-
-	KERNEL16x2_SUB
-
-	jl	.L2_17
-	ALIGN_4
-
-
-.L2_19:
-
-	SAVE16x2
-
-	addq	$16 * SIZE, CO1		# coffset += 16
-	decq	I			# i --
-	jg	.L2_11
-	ALIGN_4	
-
-/**************************************************************************
-* Rest of M 
-***************************************************************************/
-.L2_20:
-	// Test rest of M
-
-	testq	$15, M
-	jz	.L2_60		// to next 3 lines of N
-
-	testq	$8, M		
-	jz	.L2_21pre
-	ALIGN_4
-
-/**************************************************************************/
-
-.L2_20_1:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L2_20_6
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_20_2:
-
-	prefetcht0      B_PR1(BO,BI,8)
-	KERNEL8x2_1
-	KERNEL8x2_2
-	KERNEL8x2_3
-	KERNEL8x2_4
-
-	prefetcht0      B_PR1(BO,BI,8)
-	KERNEL8x2_1
-	KERNEL8x2_2
-	KERNEL8x2_3
-	KERNEL8x2_4
-
-	je	.L2_20_6
-
-	prefetcht0      B_PR1(BO,BI,8)
-	KERNEL8x2_1
-	KERNEL8x2_2
-	KERNEL8x2_3
-	KERNEL8x2_4
-
-	prefetcht0      B_PR1(BO,BI,8)
-	KERNEL8x2_1
-	KERNEL8x2_2
-	KERNEL8x2_3
-	KERNEL8x2_4
-
-	je	.L2_20_6
-
-	jmp	.L2_20_2
-	ALIGN_4
-
-.L2_20_6:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_20_9
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_20_7:
-
-	KERNEL8x2_SUB
-
-	jl	.L2_20_7
-	ALIGN_4
-
-
-.L2_20_9:
-
-	SAVE8x2
-
-	addq	$8 * SIZE, CO1		# coffset += 8
-	ALIGN_4
-	
-
-
-/**************************************************************************/
-
-.L2_21pre:
-
-	testq	$4, M		
-	jz	.L2_30
-	ALIGN_4
-
-.L2_21:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L2_26
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 1 ; number of values
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_22:
-
-	prefetcht0      B_PR1(BO,BI,8)
-	KERNEL4x2_1
-	KERNEL4x2_2
-	KERNEL4x2_3
-	KERNEL4x2_4
-
-	prefetcht0      B_PR1(BO,BI,8)
-	KERNEL4x2_1
-	KERNEL4x2_2
-	KERNEL4x2_3
-	KERNEL4x2_4
-
-	je	.L2_26
-
-	prefetcht0      B_PR1(BO,BI,8)
-	KERNEL4x2_1
-	KERNEL4x2_2
-	KERNEL4x2_3
-	KERNEL4x2_4
-
-	prefetcht0      B_PR1(BO,BI,8)
-	KERNEL4x2_1
-	KERNEL4x2_2
-	KERNEL4x2_3
-	KERNEL4x2_4
-
-	je	.L2_26
-
-	jmp	.L2_22
-	ALIGN_4
-
-.L2_26:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_29
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_27:
-
-	KERNEL4x2_SUB
-
-	jl	.L2_27
-	ALIGN_4
-
-
-.L2_29:
-
-	SAVE4x2
-
-	addq	$4 * SIZE, CO1		# coffset += 4
-	ALIGN_4
-	
-
-.L2_30:
-	testq	$2, M		
-	jz	.L2_40
-
-	ALIGN_4
-
-.L2_31:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L2_36
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_32:
-
-	KERNEL2x2_1
-	KERNEL2x2_2
-	KERNEL2x2_3
-	KERNEL2x2_4
-
-	KERNEL2x2_1
-	KERNEL2x2_2
-	KERNEL2x2_3
-	KERNEL2x2_4
-
-	je	.L2_36
-
-	KERNEL2x2_1
-	KERNEL2x2_2
-	KERNEL2x2_3
-	KERNEL2x2_4
-
-	KERNEL2x2_1
-	KERNEL2x2_2
-	KERNEL2x2_3
-	KERNEL2x2_4
-
-	je	.L2_36
-
-	jmp	.L2_32
-	ALIGN_4
-
-.L2_36:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_39
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-	
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_37:
-
-	KERNEL2x2_SUB
-
-	jl	.L2_37
-	ALIGN_4
-
-
-.L2_39:
-
-	SAVE2x2
-
-	addq	$2 * SIZE, CO1		# coffset += 2
-	ALIGN_4
-
-.L2_40:
-	testq	$1, M		
-	jz	.L2_60		// to next 2 lines of N
-
-	ALIGN_4
-
-.L2_41:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L2_46
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_42:
-
-	KERNEL1x2_1
-	KERNEL1x2_2
-	KERNEL1x2_3
-	KERNEL1x2_4
-
-	KERNEL1x2_1
-	KERNEL1x2_2
-	KERNEL1x2_3
-	KERNEL1x2_4
-
-	je	.L2_46
-
-	KERNEL1x2_1
-	KERNEL1x2_2
-	KERNEL1x2_3
-	KERNEL1x2_4
-
-	KERNEL1x2_1
-	KERNEL1x2_2
-	KERNEL1x2_3
-	KERNEL1x2_4
-
-	je	.L2_46
-
-	jmp	.L2_42
-	ALIGN_4
-
-.L2_46:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_49
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_47:
-
-	KERNEL1x2_SUB
-
-	jl	.L2_47
-	ALIGN_4
-
-
-.L2_49:
-
-	SAVE1x2
-
-	addq	$1 * SIZE, CO1		# coffset += 1
-	ALIGN_4
-	
-.L2_60:
-
-	decq	J			// j --
-	jg	.L2_01			// next 2 lines of N
-
-
-
-.L1_0:
-
-/************************************************************************************************
-* Loop for Nmod6 % 2 > 0
-*************************************************************************************************/
-
-	movq	Nmod6, J		
-	andq	$1, J			// j % 2
-	je	.L999
-	ALIGN_4
-
-.L1_01:
-	// copy to sub buffer
-	movq	B, BO1
-	leaq    BUFFER1, BO		// first buffer to BO
-	movq	K, %rax
-	ALIGN_4
-
-.L1_02b:
-
-	vmovsd	(BO1), %xmm0
-	vmovsd	%xmm0,       (BO)
-	addq	$1*SIZE,BO1
-	addq	$1*SIZE,BO
-	decq	%rax
-	jnz	.L1_02b
-
-.L1_02c:
-
-	movq	BO1, B			// next offset of B
-
-.L1_10:
-	movq	C, CO1
-	leaq	(C, LDC, 1), C		// c += 1 * ldc
-
-	
-	movq	A, AO		 	// aoffset = a
-	addq	$32 * SIZE, AO
-
-	movq	M,  I
-	sarq	$4, I			// i = (m >> 4)
-	je	.L1_20
-
-	ALIGN_4
-
-.L1_11:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax			//  K = K - ( K % 8 )
-	je	.L1_16
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$4, %rax			// rax = rax * 16 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_12:
-
-	prefetcht0      B_PR1(BO,BI,8)
-	KERNEL16x1_1
-	KERNEL16x1_2
-	KERNEL16x1_3
-	KERNEL16x1_4
-
-	KERNEL16x1_1
-	KERNEL16x1_2
-	KERNEL16x1_3
-	KERNEL16x1_4
-
-	je	.L1_16
-
-	prefetcht0      B_PR1(BO,BI,8)
-	KERNEL16x1_1
-	KERNEL16x1_2
-	KERNEL16x1_3
-	KERNEL16x1_4
-
-	KERNEL16x1_1
-	KERNEL16x1_2
-	KERNEL16x1_3
-	KERNEL16x1_4
-
-	je	.L1_16
-
-	jmp	.L1_12
-	ALIGN_4
-
-.L1_16:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_19
-
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$4, %rax			// rax = rax * 16 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_17:
-
-	KERNEL16x1_SUB
-
-	jl	.L1_17
-	ALIGN_4
-
-
-.L1_19:
-
-	SAVE16x1
-
-	addq	$16 * SIZE, CO1		# coffset += 16
-	decq	I			# i --
-	jg	.L1_11
-	ALIGN_4	
-
-/**************************************************************************
-* Rest of M 
-***************************************************************************/
-.L1_20:
-	// Test rest of M
-
-	testq	$15, M
-	jz	.L999
-
-	testq	$8, M		
-	jz	.L1_21pre
-	ALIGN_4
-
-/**************************************************************************/
-
-.L1_20_1:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L1_20_6
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_20_2:
-
-	prefetcht0      B_PR1(BO,BI,8)
-	KERNEL8x1_1
-	KERNEL8x1_2
-	KERNEL8x1_3
-	KERNEL8x1_4
-
-	KERNEL8x1_1
-	KERNEL8x1_2
-	KERNEL8x1_3
-	KERNEL8x1_4
-
-	je	.L1_20_6
-
-	prefetcht0      B_PR1(BO,BI,8)
-	KERNEL8x1_1
-	KERNEL8x1_2
-	KERNEL8x1_3
-	KERNEL8x1_4
-
-	KERNEL8x1_1
-	KERNEL8x1_2
-	KERNEL8x1_3
-	KERNEL8x1_4
-
-	je	.L1_20_6
-
-	jmp	.L1_20_2
-	ALIGN_4
-
-.L1_20_6:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_20_9
-
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_20_7:
-
-	KERNEL8x1_SUB
-
-	jl	.L1_20_7
-	ALIGN_4
-
-
-.L1_20_9:
-
-	SAVE8x1
-
-	addq	$8 * SIZE, CO1		# coffset += 8
-	ALIGN_4
-	
-
-
-/**************************************************************************/
-
-.L1_21pre:
-
-	testq	$4, M		
-	jz	.L1_30
-	ALIGN_4
-
-.L1_21:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L1_26
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_22:
-
-	prefetcht0      B_PR1(BO,BI,8)
-	KERNEL4x1_1
-	KERNEL4x1_2
-	KERNEL4x1_3
-	KERNEL4x1_4
-
-	KERNEL4x1_1
-	KERNEL4x1_2
-	KERNEL4x1_3
-	KERNEL4x1_4
-
-	je	.L1_26
-
-	prefetcht0      B_PR1(BO,BI,8)
-	KERNEL4x1_1
-	KERNEL4x1_2
-	KERNEL4x1_3
-	KERNEL4x1_4
-
-	KERNEL4x1_1
-	KERNEL4x1_2
-	KERNEL4x1_3
-	KERNEL4x1_4
-
-	je	.L1_26
-
-	jmp	.L1_22
-	ALIGN_4
-
-.L1_26:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_29
-
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_27:
-
-	KERNEL4x1_SUB
-
-	jl	.L1_27
-	ALIGN_4
-
-
-.L1_29:
-
-	SAVE4x1
-
-	addq	$4 * SIZE, CO1		# coffset += 4
-	ALIGN_4
-	
-
-.L1_30:
-	testq	$2, M		
-	jz	.L1_40
-
-	ALIGN_4
-
-.L1_31:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L1_36
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_32:
-
-	KERNEL2x1_1
-	KERNEL2x1_2
-	KERNEL2x1_3
-	KERNEL2x1_4
-
-	KERNEL2x1_1
-	KERNEL2x1_2
-	KERNEL2x1_3
-	KERNEL2x1_4
-
-	je	.L1_36
-
-	KERNEL2x1_1
-	KERNEL2x1_2
-	KERNEL2x1_3
-	KERNEL2x1_4
-
-	KERNEL2x1_1
-	KERNEL2x1_2
-	KERNEL2x1_3
-	KERNEL2x1_4
-
-	je	.L1_36
-
-	jmp	.L1_32
-	ALIGN_4
-
-.L1_36:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_39
-
-	movq    %rax, BI                        //  Index for BO
-	
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_37:
-
-	KERNEL2x1_SUB
-
-	jl	.L1_37
-	ALIGN_4
-
-
-.L1_39:
-
-	SAVE2x1
-
-	addq	$2 * SIZE, CO1		# coffset += 2
-	ALIGN_4
-
-.L1_40:
-	testq	$1, M		
-	jz	.L999
-
-	ALIGN_4
-
-.L1_41:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L1_46
-	movq    %rax, BI                        //  Index for BO
-
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_42:
-
-	KERNEL1x1_1
-	KERNEL1x1_2
-	KERNEL1x1_3
-	KERNEL1x1_4
-
-	KERNEL1x1_1
-	KERNEL1x1_2
-	KERNEL1x1_3
-	KERNEL1x1_4
-
-	je	.L1_46
-
-	KERNEL1x1_1
-	KERNEL1x1_2
-	KERNEL1x1_3
-	KERNEL1x1_4
-
-	KERNEL1x1_1
-	KERNEL1x1_2
-	KERNEL1x1_3
-	KERNEL1x1_4
-
-	je	.L1_46
-
-	jmp	.L1_42
-	ALIGN_4
-
-.L1_46:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_49
-
-	movq    %rax, BI                        //  Index for BO
-
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_47:
-
-	KERNEL1x1_SUB
-
-	jl	.L1_47
-	ALIGN_4
-
-
-.L1_49:
-
-	SAVE1x1
-
-	addq	$1 * SIZE, CO1		# coffset += 1
-	ALIGN_4
-	
-
-.L999:
-	movq   		SP, %rsp
-	movq	   (%rsp), %rbx
-	movq	  8(%rsp), %rbp
-	movq	 16(%rsp), %r12
-	movq	 24(%rsp), %r13
-	movq	 32(%rsp), %r14
-	movq	 40(%rsp), %r15
-
-#ifdef WINDOWS_ABI
-	movq	 48(%rsp), %rdi
-	movq	 56(%rsp), %rsi
-	movups	 64(%rsp), %xmm6
-	movups	 80(%rsp), %xmm7
-	movups	 96(%rsp), %xmm8
-	movups	112(%rsp), %xmm9
-	movups	128(%rsp), %xmm10
-	movups	144(%rsp), %xmm11
-	movups	160(%rsp), %xmm12
-	movups	176(%rsp), %xmm13
-	movups	192(%rsp), %xmm14
-	movups	208(%rsp), %xmm15
-#endif
-
-	addq	$STACKSIZE, %rsp
-	ret
-
-	EPILOGUE
-
-
-#else
-/*************************************************************************************
-* TRMM Kernel
-*************************************************************************************/
-
-
-	PROLOGUE
-	PROFCODE
-	
-	subq	$STACKSIZE, %rsp
-	movq	%rbx,   (%rsp)
-	movq	%rbp,  8(%rsp)
-	movq	%r12, 16(%rsp)
-	movq	%r13, 24(%rsp)
-	movq	%r14, 32(%rsp)
-	movq	%r15, 40(%rsp)
-
-	vzeroupper
-
-#ifdef WINDOWS_ABI
-	movq	%rdi,    48(%rsp)
-	movq	%rsi,    56(%rsp)
-	movups	%xmm6,   64(%rsp)
-	movups	%xmm7,   80(%rsp)
-	movups	%xmm8,   96(%rsp)
-	movups	%xmm9,  112(%rsp)
-	movups	%xmm10, 128(%rsp)
-	movups	%xmm11, 144(%rsp)
-	movups	%xmm12, 160(%rsp)
-	movups	%xmm13, 176(%rsp)
-	movups	%xmm14, 192(%rsp)
-	movups	%xmm15, 208(%rsp)
-
-	movq	ARG1,      OLD_M
-	movq	ARG2,      OLD_N
-	movq	ARG3,      OLD_K
-	movq	OLD_A,     A
-	movq	OLD_B,     B
-	movq	OLD_C,     C
-	movq	OLD_LDC,   LDC
-#ifdef TRMMKERNEL
-	movsd	OLD_OFFSET, %xmm12
-#endif
-	vmovaps	%xmm3, %xmm0
-
-#else
-	movq	STACKSIZE +  8(%rsp), LDC
-#ifdef TRMMKERNEL
-	movsd	STACKSIZE + 16(%rsp), %xmm12
-#endif
-
-#endif
-
-	movq    %rsp, SP      # save old stack
-        subq    $128 + L_BUFFER_SIZE, %rsp
-        andq    $-4096, %rsp    # align stack
-
-        STACK_TOUCH
-
-	cmpq	$0, OLD_M
-	je	.L999
-
-	cmpq	$0, OLD_N
-	je	.L999
-
-	cmpq	$0, OLD_K
-	je	.L999
-
-	movq	OLD_M, M
-	movq	OLD_N, N
-	movq	OLD_K, K
-
-	vmovsd	 %xmm0, ALPHA
-
-	salq	$BASE_SHIFT, LDC
-
-	movq    N, %rax
-        xorq    %rdx, %rdx
-        movq    $2,  %rdi
-        divq    %rdi                    //    N / 6
-        movq    %rax, Ndiv6             //    N / 6
-        movq    %rdx, Nmod6             //    N % 6
-
-	
-
-#ifdef TRMMKERNEL
-	vmovsd	%xmm12, OFFSET
-	vmovsd	%xmm12, KK
-#ifndef LEFT
-	negq	KK
-#endif	
-#endif
-
-	movq	Ndiv6,  J
-	cmpq	$0, J
-	je	.L1_0
-	ALIGN_4
-
-.L2_01:
-	// copy to sub buffer
-	movq	B, BO1
-	leaq    BUFFER1, BO		// first buffer to BO
-	movq	K, %rax
-	sarq	$2, %rax		// K / 4
-	jz	.L2_01b
-	ALIGN_4
-
-.L2_01a:
-        prefetcht0 512(BO1)
-        prefetchw  512(BO)
-
-	vmovups	      (BO1), %xmm0
-	vmovups	2*SIZE(BO1), %xmm1
-	vmovups	4*SIZE(BO1), %xmm2
-	vmovups	6*SIZE(BO1), %xmm3
-
-	vmovups	%xmm0,       (BO)
-	vmovups	%xmm1, 2*SIZE(BO)
-	vmovups	%xmm2, 4*SIZE(BO)
-	vmovups	%xmm3, 6*SIZE(BO)
-
-	addq	$8*SIZE,BO1
-	addq	$8*SIZE,BO
-	decq	%rax
-	jnz	.L2_01a
-
-
-.L2_01b:
-
-        movq    K, %rax
-        andq    $3, %rax                // K % 4
-        jz      .L2_02d
-        ALIGN_4
-
-.L2_02c:
-
-	vmovups	(BO1), %xmm0
-	vmovups	%xmm0, (BO)
-	addq	$2*SIZE,BO1
-	addq	$2*SIZE,BO
-	decq	%rax
-	jnz	.L2_02c
-
-.L2_02d:
-
-	movq	BO1, B			// next offset of B
-
-.L2_10:
-	movq	C, CO1
-	leaq	(C, LDC, 2), C		// c += 2 * ldc
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        movq    OFFSET, %rax
-        movq    %rax, KK
-#endif
-	
-	movq	A, AO		 	// aoffset = a
-	addq	$32 * SIZE, AO
-
-	movq	M,  I
-	sarq	$4, I			// i = (m >> 4)
-	je	.L2_20
-
-	ALIGN_4
-
-.L2_11:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO
-        salq    $4, %rax                        // rax = rax * 16 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $16, %rax	// number of values in AO
-#else
-        addq    $2, %rax	// number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-	andq	$-8, %rax			//  K = K - ( K % 8 )
-	je	.L2_16
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$4, %rax			// rax = rax * 16 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_12:
-
-	prefetcht0      B_PR1(BO,BI,8)
-	KERNEL16x2_1
-	KERNEL16x2_2
-	KERNEL16x2_3
-	KERNEL16x2_4
-
-	prefetcht0      B_PR1(BO,BI,8)
-	KERNEL16x2_1
-	KERNEL16x2_2
-	KERNEL16x2_3
-	KERNEL16x2_4
-
-	je	.L2_16
-
-	prefetcht0      B_PR1(BO,BI,8)
-	KERNEL16x2_1
-	KERNEL16x2_2
-	KERNEL16x2_3
-	KERNEL16x2_4
-
-	prefetcht0      B_PR1(BO,BI,8)
-	KERNEL16x2_1
-	KERNEL16x2_2
-	KERNEL16x2_3
-	KERNEL16x2_4
-
-	je	.L2_16
-
-	jmp	.L2_12
-	ALIGN_4
-
-.L2_16:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_19
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$4, %rax			// rax = rax * 16 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_17:
-
-	KERNEL16x2_SUB
-
-	jl	.L2_17
-	ALIGN_4
-
-
-.L2_19:
-
-	SAVE16x2
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO
-        salq    $4, %rax                        // rax = rax * 16 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $16, KK				
-#endif
-
-	addq	$16 * SIZE, CO1		# coffset += 16
-	decq	I			# i --
-	jg	.L2_11
-	ALIGN_4	
-
-/**************************************************************************
-* Rest of M 
-***************************************************************************/
-.L2_20:
-	// Test rest of M
-
-	testq	$15, M
-	jz	.L2_60		// to next 3 lines of N
-
-	testq	$8, M		
-	jz	.L2_21pre
-	ALIGN_4
-
-/**************************************************************************/
-
-.L2_20_1:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO
-        salq    $3, %rax                        // rax = rax * 8 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $8, %rax        // number of values in A
-#else
-        addq    $2, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-
-	andq	$-8, %rax
-	je	.L2_20_6
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_20_2:
-
-	prefetcht0      B_PR1(BO,BI,8)
-	KERNEL8x2_1
-	KERNEL8x2_2
-	KERNEL8x2_3
-	KERNEL8x2_4
-
-	prefetcht0      B_PR1(BO,BI,8)
-	KERNEL8x2_1
-	KERNEL8x2_2
-	KERNEL8x2_3
-	KERNEL8x2_4
-
-	je	.L2_20_6
-
-	prefetcht0      B_PR1(BO,BI,8)
-	KERNEL8x2_1
-	KERNEL8x2_2
-	KERNEL8x2_3
-	KERNEL8x2_4
-
-	prefetcht0      B_PR1(BO,BI,8)
-	KERNEL8x2_1
-	KERNEL8x2_2
-	KERNEL8x2_3
-	KERNEL8x2_4
-
-	je	.L2_20_6
-
-	jmp	.L2_20_2
-	ALIGN_4
-
-.L2_20_6:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_20_9
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_20_7:
-
-	KERNEL8x2_SUB
-
-	jl	.L2_20_7
-	ALIGN_4
-
-
-.L2_20_9:
-
-	SAVE8x2
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax 
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO         
-        salq    $3, %rax                        // rax = rax * 8 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif  
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $8, KK
-#endif
-
-	addq	$8 * SIZE, CO1		# coffset += 8
-	ALIGN_4
-	
-
-
-/**************************************************************************/
-
-.L2_21pre:
-
-	testq	$4, M		
-	jz	.L2_30
-	ALIGN_4
-
-.L2_21:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO
-        salq    $2, %rax                        // rax = rax * 4 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $4, %rax        // number of values in A
-#else
-        addq    $2, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-
-	andq	$-8, %rax
-	je	.L2_26
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 1 ; number of values
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_22:
-
-	prefetcht0      B_PR1(BO,BI,8)
-	KERNEL4x2_1
-	KERNEL4x2_2
-	KERNEL4x2_3
-	KERNEL4x2_4
-
-	prefetcht0      B_PR1(BO,BI,8)
-	KERNEL4x2_1
-	KERNEL4x2_2
-	KERNEL4x2_3
-	KERNEL4x2_4
-
-	je	.L2_26
-
-	prefetcht0      B_PR1(BO,BI,8)
-	KERNEL4x2_1
-	KERNEL4x2_2
-	KERNEL4x2_3
-	KERNEL4x2_4
-
-	prefetcht0      B_PR1(BO,BI,8)
-	KERNEL4x2_1
-	KERNEL4x2_2
-	KERNEL4x2_3
-	KERNEL4x2_4
-
-	je	.L2_26
-
-	jmp	.L2_22
-	ALIGN_4
-
-.L2_26:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_29
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_27:
-
-	KERNEL4x2_SUB
-
-	jl	.L2_27
-	ALIGN_4
-
-
-.L2_29:
-
-	SAVE4x2
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax 
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO         
-        salq    $2, %rax                        // rax = rax * 4 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif  
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $4, KK
-#endif
-
-	addq	$4 * SIZE, CO1		# coffset += 4
-	ALIGN_4
-	
-
-.L2_30:
-	testq	$2, M		
-	jz	.L2_40
-
-	ALIGN_4
-
-.L2_31:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO
-        salq    $1, %rax                        // rax = rax * 2 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $2, %rax        // number of values in AO
-#else
-        addq    $2, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-
-	andq	$-8, %rax
-	je	.L2_36
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_32:
-
-	KERNEL2x2_1
-	KERNEL2x2_2
-	KERNEL2x2_3
-	KERNEL2x2_4
-
-	KERNEL2x2_1
-	KERNEL2x2_2
-	KERNEL2x2_3
-	KERNEL2x2_4
-
-	je	.L2_36
-
-	KERNEL2x2_1
-	KERNEL2x2_2
-	KERNEL2x2_3
-	KERNEL2x2_4
-
-	KERNEL2x2_1
-	KERNEL2x2_2
-	KERNEL2x2_3
-	KERNEL2x2_4
-
-	je	.L2_36
-
-	jmp	.L2_32
-	ALIGN_4
-
-.L2_36:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_39
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-	
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_37:
-
-	KERNEL2x2_SUB
-
-	jl	.L2_37
-	ALIGN_4
-
-
-.L2_39:
-
-	SAVE2x2
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax 
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO         
-        salq    $1, %rax                        // rax = rax * 2 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif  
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $2, KK
-#endif
-
-	addq	$2 * SIZE, CO1		# coffset += 2
-	ALIGN_4
-
-.L2_40:
-	testq	$1, M		
-	jz	.L2_60		// to next 2 lines of N
-
-	ALIGN_4
-
-.L2_41:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $1, %rax        // number of values in AO
-#else
-        addq    $2, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-	andq	$-8, %rax
-	je	.L2_46
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_42:
-
-	KERNEL1x2_1
-	KERNEL1x2_2
-	KERNEL1x2_3
-	KERNEL1x2_4
-
-	KERNEL1x2_1
-	KERNEL1x2_2
-	KERNEL1x2_3
-	KERNEL1x2_4
-
-	je	.L2_46
-
-	KERNEL1x2_1
-	KERNEL1x2_2
-	KERNEL1x2_3
-	KERNEL1x2_4
-
-	KERNEL1x2_1
-	KERNEL1x2_2
-	KERNEL1x2_3
-	KERNEL1x2_4
-
-	je	.L2_46
-
-	jmp	.L2_42
-	ALIGN_4
-
-.L2_46:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_49
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_47:
-
-	KERNEL1x2_SUB
-
-	jl	.L2_47
-	ALIGN_4
-
-
-.L2_49:
-
-	SAVE1x2
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax 
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO         
-        leaq    (AO, %rax, SIZE), AO
-#endif  
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $1, KK
-#endif
-
-	addq	$1 * SIZE, CO1		# coffset += 1
-	ALIGN_4
-	
-
-
-
-	
-.L2_60:
-#if defined(TRMMKERNEL) && !defined(LEFT)
-        addq    $2, KK
-#endif
-
-	decq	J			// j --
-	jg	.L2_01			// next 2 lines of N
-
-
-
-.L1_0:
-
-/************************************************************************************************
-* Loop for Nmod6 % 2 > 0
-*************************************************************************************************/
-
-	movq	Nmod6, J		
-	andq	$1, J			// j % 2
-	je	.L999
-	ALIGN_4
-
-.L1_01:
-	// copy to sub buffer
-	movq	B, BO1
-	leaq    BUFFER1, BO		// first buffer to BO
-	movq	K, %rax
-	ALIGN_4
-
-.L1_02b:
-
-	vmovsd	(BO1), %xmm0
-	vmovsd	%xmm0,       (BO)
-	addq	$1*SIZE,BO1
-	addq	$1*SIZE,BO
-	decq	%rax
-	jnz	.L1_02b
-
-.L1_02c:
-
-	movq	BO1, B			// next offset of B
-
-.L1_10:
-	movq	C, CO1
-	leaq	(C, LDC, 1), C		// c += 1 * ldc
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        movq    OFFSET, %rax
-        movq    %rax, KK
-#endif
-	
-	movq	A, AO		 	// aoffset = a
-	addq	$32 * SIZE, AO
-
-	movq	M,  I
-	sarq	$4, I			// i = (m >> 4)
-	je	.L1_20
-
-	ALIGN_4
-
-.L1_11:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BO, BI, SIZE), BO
-        salq    $4, %rax                        // rax = rax * 16 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $16, %rax	// number of values in AO
-#else
-        addq    $1, %rax	// number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-	andq	$-8, %rax			//  K = K - ( K % 8 )
-	je	.L1_16
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$4, %rax			// rax = rax * 16 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_12:
-
-	prefetcht0      B_PR1(BO,BI,8)
-	KERNEL16x1_1
-	KERNEL16x1_2
-	KERNEL16x1_3
-	KERNEL16x1_4
-
-	KERNEL16x1_1
-	KERNEL16x1_2
-	KERNEL16x1_3
-	KERNEL16x1_4
-
-	je	.L1_16
-
-	prefetcht0      B_PR1(BO,BI,8)
-	KERNEL16x1_1
-	KERNEL16x1_2
-	KERNEL16x1_3
-	KERNEL16x1_4
-
-	KERNEL16x1_1
-	KERNEL16x1_2
-	KERNEL16x1_3
-	KERNEL16x1_4
-
-	je	.L1_16
-
-	jmp	.L1_12
-	ALIGN_4
-
-.L1_16:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_19
-
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$4, %rax			// rax = rax * 16 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_17:
-
-	KERNEL16x1_SUB
-
-	jl	.L1_17
-	ALIGN_4
-
-
-.L1_19:
-
-	SAVE16x1
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BO, BI, SIZE), BO
-        salq    $4, %rax                        // rax = rax * 16 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $16, KK				
-#endif
-
-	addq	$16 * SIZE, CO1		# coffset += 16
-	decq	I			# i --
-	jg	.L1_11
-	ALIGN_4	
-
-/**************************************************************************
-* Rest of M 
-***************************************************************************/
-.L1_20:
-	// Test rest of M
-
-	testq	$15, M
-	jz	.L999
-
-	testq	$8, M		
-	jz	.L1_21pre
-	ALIGN_4
-
-/**************************************************************************/
-
-.L1_20_1:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BO, BI, SIZE), BO
-        salq    $3, %rax                        // rax = rax * 8 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $8, %rax        // number of values in A
-#else
-        addq    $1, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-
-	andq	$-8, %rax
-	je	.L1_20_6
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_20_2:
-
-	prefetcht0      B_PR1(BO,BI,8)
-	KERNEL8x1_1
-	KERNEL8x1_2
-	KERNEL8x1_3
-	KERNEL8x1_4
-
-	KERNEL8x1_1
-	KERNEL8x1_2
-	KERNEL8x1_3
-	KERNEL8x1_4
-
-	je	.L1_20_6
-
-	prefetcht0      B_PR1(BO,BI,8)
-	KERNEL8x1_1
-	KERNEL8x1_2
-	KERNEL8x1_3
-	KERNEL8x1_4
-
-	KERNEL8x1_1
-	KERNEL8x1_2
-	KERNEL8x1_3
-	KERNEL8x1_4
-
-	je	.L1_20_6
-
-	jmp	.L1_20_2
-	ALIGN_4
-
-.L1_20_6:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_20_9
-
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_20_7:
-
-	KERNEL8x1_SUB
-
-	jl	.L1_20_7
-	ALIGN_4
-
-
-.L1_20_9:
-
-	SAVE8x1
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax 
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BO, BI, SIZE), BO         
-        salq    $3, %rax                        // rax = rax * 8 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif  
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $8, KK
-#endif
-
-	addq	$8 * SIZE, CO1		# coffset += 8
-	ALIGN_4
-	
-
-
-/**************************************************************************/
-
-.L1_21pre:
-
-	testq	$4, M		
-	jz	.L1_30
-	ALIGN_4
-
-.L1_21:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BO, BI, SIZE), BO
-        salq    $2, %rax                        // rax = rax * 4 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $4, %rax        // number of values in A
-#else
-        addq    $1, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-
-	andq	$-8, %rax
-	je	.L1_26
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_22:
-
-	prefetcht0      B_PR1(BO,BI,8)
-	KERNEL4x1_1
-	KERNEL4x1_2
-	KERNEL4x1_3
-	KERNEL4x1_4
-
-	KERNEL4x1_1
-	KERNEL4x1_2
-	KERNEL4x1_3
-	KERNEL4x1_4
-
-	je	.L1_26
-
-	prefetcht0      B_PR1(BO,BI,8)
-	KERNEL4x1_1
-	KERNEL4x1_2
-	KERNEL4x1_3
-	KERNEL4x1_4
-
-	KERNEL4x1_1
-	KERNEL4x1_2
-	KERNEL4x1_3
-	KERNEL4x1_4
-
-	je	.L1_26
-
-	jmp	.L1_22
-	ALIGN_4
-
-.L1_26:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_29
-
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_27:
-
-	KERNEL4x1_SUB
-
-	jl	.L1_27
-	ALIGN_4
-
-
-.L1_29:
-
-	SAVE4x1
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax 
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BO, BI, SIZE), BO         
-        salq    $2, %rax                        // rax = rax * 4 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif  
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $4, KK
-#endif
-
-	addq	$4 * SIZE, CO1		# coffset += 4
-	ALIGN_4
-	
-
-.L1_30:
-	testq	$2, M		
-	jz	.L1_40
-
-	ALIGN_4
-
-.L1_31:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BO, BI, SIZE), BO
-        salq    $1, %rax                        // rax = rax * 2 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $2, %rax        // number of values in AO
-#else
-        addq    $1, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-
-	andq	$-8, %rax
-	je	.L1_36
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_32:
-
-	KERNEL2x1_1
-	KERNEL2x1_2
-	KERNEL2x1_3
-	KERNEL2x1_4
-
-	KERNEL2x1_1
-	KERNEL2x1_2
-	KERNEL2x1_3
-	KERNEL2x1_4
-
-	je	.L1_36
-
-	KERNEL2x1_1
-	KERNEL2x1_2
-	KERNEL2x1_3
-	KERNEL2x1_4
-
-	KERNEL2x1_1
-	KERNEL2x1_2
-	KERNEL2x1_3
-	KERNEL2x1_4
-
-	je	.L1_36
-
-	jmp	.L1_32
-	ALIGN_4
-
-.L1_36:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_39
-
-	movq    %rax, BI                        //  Index for BO
-	
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_37:
-
-	KERNEL2x1_SUB
-
-	jl	.L1_37
-	ALIGN_4
-
-
-.L1_39:
-
-	SAVE2x1
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax 
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BO, BI, SIZE), BO         
-        salq    $1, %rax                        // rax = rax * 2 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif  
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $2, KK
-#endif
-
-	addq	$2 * SIZE, CO1		# coffset += 2
-	ALIGN_4
-
-.L1_40:
-	testq	$1, M		
-	jz	.L999
-
-	ALIGN_4
-
-.L1_41:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BO, BI, SIZE), BO
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $1, %rax        // number of values in AO
-#else
-        addq    $1, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-	andq	$-8, %rax
-	je	.L1_46
-	movq    %rax, BI                        //  Index for BO
-
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_42:
-
-	KERNEL1x1_1
-	KERNEL1x1_2
-	KERNEL1x1_3
-	KERNEL1x1_4
-
-	KERNEL1x1_1
-	KERNEL1x1_2
-	KERNEL1x1_3
-	KERNEL1x1_4
-
-	je	.L1_46
-
-	KERNEL1x1_1
-	KERNEL1x1_2
-	KERNEL1x1_3
-	KERNEL1x1_4
-
-	KERNEL1x1_1
-	KERNEL1x1_2
-	KERNEL1x1_3
-	KERNEL1x1_4
-
-	je	.L1_46
-
-	jmp	.L1_42
-	ALIGN_4
-
-.L1_46:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_49
-
-	movq    %rax, BI                        //  Index for BO
-
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_47:
-
-	KERNEL1x1_SUB
-
-	jl	.L1_47
-	ALIGN_4
-
-
-.L1_49:
-
-	SAVE1x1
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax 
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BO, BI, SIZE), BO         
-        leaq    (AO, %rax, SIZE), AO
-#endif  
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $1, KK
-#endif
-
-	addq	$1 * SIZE, CO1		# coffset += 1
-	ALIGN_4
-	
-
-.L999:
-	movq   		SP, %rsp
-	movq	   (%rsp), %rbx
-	movq	  8(%rsp), %rbp
-	movq	 16(%rsp), %r12
-	movq	 24(%rsp), %r13
-	movq	 32(%rsp), %r14
-	movq	 40(%rsp), %r15
-
-#ifdef WINDOWS_ABI
-	movq	 48(%rsp), %rdi
-	movq	 56(%rsp), %rsi
-	movups	 64(%rsp), %xmm6
-	movups	 80(%rsp), %xmm7
-	movups	 96(%rsp), %xmm8
-	movups	112(%rsp), %xmm9
-	movups	128(%rsp), %xmm10
-	movups	144(%rsp), %xmm11
-	movups	160(%rsp), %xmm12
-	movups	176(%rsp), %xmm13
-	movups	192(%rsp), %xmm14
-	movups	208(%rsp), %xmm15
-#endif
-
-	addq	$STACKSIZE, %rsp
-	ret
-
-	EPILOGUE
-
-
-
-
-
-#endif
+/*********************************************************************/
+/* Copyright 2009, 2010 The University of Texas at Austin.           */
+/* All rights reserved.                                              */
+/*                                                                   */
+/* Redistribution and use in source and binary forms, with or        */
+/* without modification, are permitted provided that the following   */
+/* conditions are met:                                               */
+/*                                                                   */
+/*   1. Redistributions of source code must retain the above         */
+/*      copyright notice, this list of conditions and the following  */
+/*      disclaimer.                                                  */
+/*                                                                   */
+/*   2. Redistributions in binary form must reproduce the above      */
+/*      copyright notice, this list of conditions and the following  */
+/*      disclaimer in the documentation and/or other materials       */
+/*      provided with the distribution.                              */
+/*                                                                   */
+/*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */
+/*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */
+/*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */
+/*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */
+/*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */
+/*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */
+/*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */
+/*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */
+/*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */
+/*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */
+/*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */
+/*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */
+/*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */
+/*    POSSIBILITY OF SUCH DAMAGE.                                    */
+/*                                                                   */
+/* The views and conclusions contained in the software and           */
+/* documentation are those of the authors and should not be          */
+/* interpreted as representing official policies, either expressed   */
+/* or implied, of The University of Texas at Austin.                 */
+/*********************************************************************/
+
+/*********************************************************************
+* 2013/10/20 Saar
+*        BLASTEST               : OK
+*        CTEST                  : OK
+*        TEST                   : OK
+
+*
+*
+* 2013/10/20 Saar
+* Parameter:
+*       DGEMM_DEFAULT_UNROLL_N  2
+*       DGEMM_DEFAULT_UNROLL_M  16
+*       DGEMM_DEFAULT_P         192
+*       DGEMM_DEFAULT_Q         128
+*	A_PR1			512
+*
+*
+* Performance without prefetch of B:
+*       1 thread:       45.8 GFLOPS (MKL:  45)
+*       2 threads:      80.0 GFLOPS (MKL:  91)
+*       4 threads:     135.0 GFLOPS (MKL: 135)
+*********************************************************************/
+
+
+#define ASSEMBLER
+#include "common.h"
+ 
+#define OLD_M	%rdi
+#define OLD_N	%rsi
+#define M	%r13
+#define J	%r14
+#define OLD_K	%rdx
+
+#define A	%rcx
+#define B	%r8
+#define C	%r9
+#define LDC	%r10
+	
+#define I	%r11
+#define AO	%rdi
+#define BO	%rsi
+#define	CO1	%r15
+#define K	%r12
+#define BI	%rbp
+#define	SP	%rbx
+
+#define BO1	%rdi
+#define BO2	%r15
+
+#ifndef WINDOWS_ABI
+
+#define STACKSIZE 96
+
+#else
+
+#define STACKSIZE 256
+
+#define OLD_A		40 + STACKSIZE(%rsp)
+#define OLD_B		48 + STACKSIZE(%rsp)
+#define OLD_C		56 + STACKSIZE(%rsp)
+#define OLD_LDC		64 + STACKSIZE(%rsp)
+#define OLD_OFFSET	72 + STACKSIZE(%rsp)
+
+#endif
+
+#define L_BUFFER_SIZE 512*8*4
+#define LB2_OFFSET    512*8*2
+
+#define Ndiv6	 24(%rsp)
+#define Nmod6	 32(%rsp)
+#define N	 40(%rsp)
+#define ALPHA	 48(%rsp)
+#define OFFSET	 56(%rsp)
+#define KK	 64(%rsp)
+#define KKK	 72(%rsp)
+#define BUFFER1	           128(%rsp)
+#define BUFFER2	LB2_OFFSET+128(%rsp)
+
+#if defined(OS_WINDOWS)
+#if   L_BUFFER_SIZE > 16384
+#define STACK_TOUCH \
+        movl    $0,  4096 * 4(%rsp);\
+        movl    $0,  4096 * 3(%rsp);\
+        movl    $0,  4096 * 2(%rsp);\
+        movl    $0,  4096 * 1(%rsp);
+#elif L_BUFFER_SIZE > 12288
+#define STACK_TOUCH \
+        movl    $0,  4096 * 3(%rsp);\
+        movl    $0,  4096 * 2(%rsp);\
+        movl    $0,  4096 * 1(%rsp);
+#elif L_BUFFER_SIZE > 8192
+#define STACK_TOUCH \
+        movl    $0,  4096 * 2(%rsp);\
+        movl    $0,  4096 * 1(%rsp);
+#elif L_BUFFER_SIZE > 4096
+#define STACK_TOUCH \
+        movl    $0,  4096 * 1(%rsp);
+#else
+#define STACK_TOUCH
+#endif
+#else
+#define STACK_TOUCH
+#endif
+
+#if defined(BULLDOZER)
+
+.macro VFMADD231PD_ y0,y1,y2
+	vfmaddpd \y0,\y1,\y2,\y0
+.endm
+
+.macro VFMADD231SD_ x0,x1,x2
+	vfmaddsd \x0,\x1,\x2,\x0
+.endm
+
+#else
+
+.macro VFMADD231PD_ y0,y1,y2
+	vfmadd231pd \y2,\y1,\y0
+.endm
+
+.macro VFMADD231SD_ x0,x1,x2
+	vfmadd231sd \x2,\x1,\x0
+.endm
+
+#endif
+
+
+#define	A_PR1	512
+#define	B_PR1	256
+
+/*******************************************************************************************
+* 3 lines of N
+*******************************************************************************************/
+
+.macro KERNEL16x3_SUBN
+	prefetcht0	A_PR1(AO)
+	vbroadcastsd	-12 * SIZE(BO), %ymm1
+	vmovaps 	-16 * SIZE(AO), %ymm0
+	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
+	vbroadcastsd	-11 * SIZE(BO), %ymm2
+	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
+	vbroadcastsd	-10 * SIZE(BO), %ymm3
+	VFMADD231PD_  	%ymm6,%ymm3,%ymm0
+	vmovaps 	-12 * SIZE(AO), %ymm0
+	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
+	prefetcht0	A_PR1+64(AO)
+	VFMADD231PD_  	%ymm8,%ymm2,%ymm0
+	VFMADD231PD_  	%ymm9,%ymm3,%ymm0
+	vmovaps 	 -8 * SIZE(AO), %ymm0
+	VFMADD231PD_  	%ymm10,%ymm1,%ymm0
+	VFMADD231PD_  	%ymm11,%ymm2,%ymm0
+	VFMADD231PD_  	%ymm12,%ymm3,%ymm0
+	vmovaps 	 -4 * SIZE(AO), %ymm0
+	VFMADD231PD_  	%ymm13,%ymm1,%ymm0
+	VFMADD231PD_  	%ymm14,%ymm2,%ymm0
+	VFMADD231PD_  	%ymm15,%ymm3,%ymm0
+	addq	$ 3*SIZE , BO	
+	addq	$ 16*SIZE, AO
+.endm
+
+
+.macro KERNEL8x3_SUBN
+	//prefetcht0	A_PR1(AO)
+	vbroadcastsd	-12 * SIZE(BO), %ymm1
+	vmovaps 	-16 * SIZE(AO), %ymm0
+	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
+	vbroadcastsd	-11 * SIZE(BO), %ymm2
+	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
+	vbroadcastsd	-10 * SIZE(BO), %ymm3
+	VFMADD231PD_  	%ymm6,%ymm3,%ymm0
+	vmovaps 	-12 * SIZE(AO), %ymm0
+	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
+	//prefetcht0	A_PR1+64(AO)
+	VFMADD231PD_  	%ymm8,%ymm2,%ymm0
+	VFMADD231PD_  	%ymm9,%ymm3,%ymm0
+	prefetcht0	B_PR1(BO)
+	addq	$ 3*SIZE , BO	
+	addq	$ 8*SIZE, AO
+.endm
+
+.macro KERNEL4x3_SUBN
+	vbroadcastsd	-12 * SIZE(BO), %ymm1
+	vmovaps 	-16 * SIZE(AO), %ymm0
+	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
+	vbroadcastsd	-11 * SIZE(BO), %ymm2
+	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
+	vbroadcastsd	-10 * SIZE(BO), %ymm3
+	VFMADD231PD_  	%ymm6,%ymm3,%ymm0
+	addq	$ 3*SIZE , BO	
+	addq	$ 4*SIZE, AO
+.endm
+
+.macro KERNEL2x3_SUBN
+	vmovsd	-12 * SIZE(BO), %xmm1
+	vmovsd 	-16 * SIZE(AO), %xmm0
+	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
+	vmovsd	-11 * SIZE(BO), %xmm2
+	VFMADD231SD_  	%xmm5,%xmm2,%xmm0
+	vmovsd	-10 * SIZE(BO), %xmm3
+	VFMADD231SD_  	%xmm6,%xmm3,%xmm0
+	vmovsd 	-15 * SIZE(AO), %xmm0
+	VFMADD231SD_  	%xmm8,%xmm1,%xmm0
+	VFMADD231SD_  	%xmm10,%xmm2,%xmm0
+	VFMADD231SD_  	%xmm12,%xmm3,%xmm0
+	addq	$ 3*SIZE , BO	
+	addq	$ 2*SIZE, AO
+.endm
+
+.macro KERNEL1x3_SUBN
+	vmovsd	-12 * SIZE(BO), %xmm1
+	vmovsd 	-16 * SIZE(AO), %xmm0
+	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
+	vmovsd	-11 * SIZE(BO), %xmm2
+	VFMADD231SD_  	%xmm5,%xmm2,%xmm0
+	vmovsd	-10 * SIZE(BO), %xmm3
+	VFMADD231SD_  	%xmm6,%xmm3,%xmm0
+	addq	$ 3*SIZE , BO	
+	addq	$ 1*SIZE, AO
+.endm
+
+
+
+
+
+
+/******************************************************************************************/
+
+.macro KERNEL16x3_1
+	prefetcht0	A_PR1(AO, %rax, SIZE)
+	vbroadcastsd	 -6 * SIZE(BO, BI, SIZE), %ymm1
+	vmovups 	-32 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
+	vbroadcastsd	 -5 * SIZE(BO, BI, SIZE), %ymm2
+	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
+	vbroadcastsd	 -4 * SIZE(BO, BI, SIZE), %ymm3
+	VFMADD231PD_  	%ymm6,%ymm3,%ymm0
+	vmovups 	-28 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
+	prefetcht0	64+A_PR1(AO, %rax, SIZE)
+	VFMADD231PD_  	%ymm8,%ymm2,%ymm0
+	VFMADD231PD_  	%ymm9,%ymm3,%ymm0
+	vmovups 	-24 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm10,%ymm1,%ymm0
+	VFMADD231PD_  	%ymm11,%ymm2,%ymm0
+	VFMADD231PD_  	%ymm12,%ymm3,%ymm0
+	vmovups 	-20 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm13,%ymm1,%ymm0
+	VFMADD231PD_  	%ymm14,%ymm2,%ymm0
+	vbroadcastsd	 -3 * SIZE(BO, BI, SIZE), %ymm1
+	vbroadcastsd	 -2 * SIZE(BO, BI, SIZE), %ymm2
+	VFMADD231PD_  	%ymm15,%ymm3,%ymm0
+.endm
+
+
+
+
+.macro KERNEL16x3_2
+	prefetcht0	128+A_PR1(AO, %rax, SIZE)
+	vmovups 	-16 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
+	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
+	vbroadcastsd	 -1 * SIZE(BO, BI, SIZE), %ymm3
+	VFMADD231PD_  	%ymm6,%ymm3,%ymm0
+	vmovups 	-12 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
+	VFMADD231PD_  	%ymm8,%ymm2,%ymm0
+	prefetcht0	A_PR1+64(AO,%rax,SIZE)
+	VFMADD231PD_  	%ymm9,%ymm3,%ymm0
+	vmovups 	 -8 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm10,%ymm1,%ymm0
+	prefetcht0	192+A_PR1(AO, %rax, SIZE)
+	VFMADD231PD_  	%ymm11,%ymm2,%ymm0
+	VFMADD231PD_  	%ymm12,%ymm3,%ymm0
+	vmovups 	 -4 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm13,%ymm1,%ymm0
+	VFMADD231PD_  	%ymm14,%ymm2,%ymm0
+	vbroadcastsd	  0 * SIZE(BO, BI, SIZE), %ymm1
+	vbroadcastsd	  1 * SIZE(BO, BI, SIZE), %ymm2
+	VFMADD231PD_  	%ymm15,%ymm3,%ymm0
+.endm
+
+.macro KERNEL16x3_3
+	prefetcht0	256+A_PR1(AO, %rax, SIZE)
+	vmovups 	  0 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
+	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
+	vbroadcastsd	  2 * SIZE(BO, BI, SIZE), %ymm3
+	VFMADD231PD_  	%ymm6,%ymm3,%ymm0
+	vmovups 	  4 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
+	prefetcht0	320+A_PR1(AO, %rax, SIZE)
+	VFMADD231PD_  	%ymm8,%ymm2,%ymm0
+	VFMADD231PD_  	%ymm9,%ymm3,%ymm0
+	vmovups 	  8 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm10,%ymm1,%ymm0
+	VFMADD231PD_  	%ymm11,%ymm2,%ymm0
+	VFMADD231PD_  	%ymm12,%ymm3,%ymm0
+	vmovups 	 12 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm13,%ymm1,%ymm0
+	VFMADD231PD_  	%ymm14,%ymm2,%ymm0
+	vbroadcastsd	  3 * SIZE(BO, BI, SIZE), %ymm1
+	vbroadcastsd	  4 * SIZE(BO, BI, SIZE), %ymm2
+	VFMADD231PD_  	%ymm15,%ymm3,%ymm0
+.endm
+
+.macro KERNEL16x3_4
+	prefetcht0	384+A_PR1(AO, %rax, SIZE)
+	vmovups 	 16 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
+	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
+	vbroadcastsd	  5 * SIZE(BO, BI, SIZE), %ymm3
+	VFMADD231PD_  	%ymm6,%ymm3,%ymm0
+	vmovups 	 20 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
+	prefetcht0	448+A_PR1(AO, %rax, SIZE)
+	VFMADD231PD_  	%ymm8,%ymm2,%ymm0
+	VFMADD231PD_  	%ymm9,%ymm3,%ymm0
+	vmovups 	 24 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm10,%ymm1,%ymm0
+	VFMADD231PD_  	%ymm11,%ymm2,%ymm0
+	addq	$12, BI	
+	VFMADD231PD_  	%ymm12,%ymm3,%ymm0
+	vmovups 	 28 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm13,%ymm1,%ymm0
+	VFMADD231PD_  	%ymm14,%ymm2,%ymm0
+	addq	$64, %rax 
+	VFMADD231PD_  	%ymm15,%ymm3,%ymm0
+.endm
+
+.macro KERNEL16x3_SUB
+	vbroadcastsd	 -6 * SIZE(BO, BI, SIZE), %ymm1
+	vmovups 	-32 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
+	vbroadcastsd	 -5 * SIZE(BO, BI, SIZE), %ymm2
+	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
+	vbroadcastsd	 -4 * SIZE(BO, BI, SIZE), %ymm3
+	VFMADD231PD_  	%ymm6,%ymm3,%ymm0
+	vmovups 	-28 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
+	VFMADD231PD_  	%ymm8,%ymm2,%ymm0
+	VFMADD231PD_  	%ymm9,%ymm3,%ymm0
+	vmovups 	-24 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm10,%ymm1,%ymm0
+	VFMADD231PD_  	%ymm11,%ymm2,%ymm0
+	VFMADD231PD_  	%ymm12,%ymm3,%ymm0
+	vmovups 	-20 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm13,%ymm1,%ymm0
+	VFMADD231PD_  	%ymm14,%ymm2,%ymm0
+	VFMADD231PD_  	%ymm15,%ymm3,%ymm0
+	addq	$3 , BI	
+	addq	$16, %rax 
+.endm
+
+.macro SAVE16x3
+
+	vbroadcastsd	ALPHA, %ymm0
+
+	vmulpd	%ymm0 , %ymm4 , %ymm4
+	vmulpd	%ymm0 , %ymm7 , %ymm7
+	vmulpd	%ymm0 , %ymm10, %ymm10
+	vmulpd	%ymm0 , %ymm13, %ymm13
+
+	vmulpd	%ymm0 , %ymm5 , %ymm5
+	vmulpd	%ymm0 , %ymm8 , %ymm8
+	vmulpd	%ymm0 , %ymm11, %ymm11
+	vmulpd	%ymm0 , %ymm14, %ymm14
+
+	vmulpd	%ymm0 , %ymm6 , %ymm6
+	vmulpd	%ymm0 , %ymm9 , %ymm9
+	vmulpd	%ymm0 , %ymm12, %ymm12
+	vmulpd	%ymm0 , %ymm15, %ymm15
+
+#if !defined(TRMMKERNEL)
+
+	vaddpd 	        (CO1), %ymm4,%ymm4
+	vaddpd  4 * SIZE(CO1), %ymm7,%ymm7
+	vaddpd  8 * SIZE(CO1), %ymm10,%ymm10
+	vaddpd 12 * SIZE(CO1), %ymm13,%ymm13
+
+	vaddpd 	        (CO1, LDC), %ymm5,%ymm5
+	vaddpd  4 * SIZE(CO1, LDC), %ymm8,%ymm8
+	vaddpd  8 * SIZE(CO1, LDC), %ymm11,%ymm11
+	vaddpd 12 * SIZE(CO1, LDC), %ymm14,%ymm14
+
+	vaddpd 	        (CO1, LDC, 2), %ymm6,%ymm6
+	vaddpd  4 * SIZE(CO1, LDC, 2), %ymm9,%ymm9
+	vaddpd  8 * SIZE(CO1, LDC, 2), %ymm12,%ymm12
+	vaddpd 12 * SIZE(CO1, LDC, 2), %ymm15,%ymm15
+
+#endif
+
+	vmovups	%ymm4 ,  	(CO1)
+	vmovups	%ymm7 , 4 * SIZE(CO1)
+	vmovups	%ymm10, 8 * SIZE(CO1)
+	vmovups	%ymm13,12 * SIZE(CO1)
+
+	vmovups	%ymm5 ,  	(CO1, LDC)
+	vmovups	%ymm8 , 4 * SIZE(CO1, LDC)
+	vmovups	%ymm11, 8 * SIZE(CO1, LDC)
+	vmovups	%ymm14,12 * SIZE(CO1, LDC)
+
+	vmovups	%ymm6 ,  	(CO1, LDC, 2)
+	vmovups	%ymm9 , 4 * SIZE(CO1, LDC, 2)
+	vmovups	%ymm12, 8 * SIZE(CO1, LDC, 2)
+	vmovups	%ymm15,12 * SIZE(CO1, LDC, 2)
+
+.endm
+
+
+
+/*******************************************************************************************/
+
+.macro KERNEL8x3_1
+	prefetcht0	A_PR1(AO, %rax, SIZE)
+	vbroadcastsd	 -6 * SIZE(BO, BI, SIZE), %ymm1
+	vmovups 	-32 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
+	vbroadcastsd	 -5 * SIZE(BO, BI, SIZE), %ymm2
+	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
+	vbroadcastsd	 -4 * SIZE(BO, BI, SIZE), %ymm3
+	VFMADD231PD_  	%ymm6,%ymm3,%ymm0
+	vmovups 	-28 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
+	VFMADD231PD_  	%ymm8,%ymm2,%ymm0
+	VFMADD231PD_  	%ymm9,%ymm3,%ymm0
+.endm
+
+.macro KERNEL8x3_2
+	prefetcht0	64+A_PR1(AO, %rax, SIZE)
+	vbroadcastsd	 -3 * SIZE(BO, BI, SIZE), %ymm1
+	vmovups 	-24 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
+	vbroadcastsd	 -2 * SIZE(BO, BI, SIZE), %ymm2
+	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
+	vbroadcastsd	 -1 * SIZE(BO, BI, SIZE), %ymm3
+	VFMADD231PD_  	%ymm6,%ymm3,%ymm0
+	vmovups 	-20 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
+	VFMADD231PD_  	%ymm8,%ymm2,%ymm0
+	VFMADD231PD_  	%ymm9,%ymm3,%ymm0
+.endm
+
+.macro KERNEL8x3_3
+	prefetcht0	128+A_PR1(AO, %rax, SIZE)
+	vbroadcastsd	  0 * SIZE(BO, BI, SIZE), %ymm1
+	vmovups 	-16 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
+	vbroadcastsd	  1 * SIZE(BO, BI, SIZE), %ymm2
+	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
+	vbroadcastsd	  2 * SIZE(BO, BI, SIZE), %ymm3
+	VFMADD231PD_  	%ymm6,%ymm3,%ymm0
+	vmovups 	-12 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
+	VFMADD231PD_  	%ymm8,%ymm2,%ymm0
+	VFMADD231PD_  	%ymm9,%ymm3,%ymm0
+.endm
+
+.macro KERNEL8x3_4
+	prefetcht0	192+A_PR1(AO, %rax, SIZE)
+	vbroadcastsd	  3 * SIZE(BO, BI, SIZE), %ymm1
+	vmovups 	 -8 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
+	vbroadcastsd	  4 * SIZE(BO, BI, SIZE), %ymm2
+	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
+	vbroadcastsd	  5 * SIZE(BO, BI, SIZE), %ymm3
+	VFMADD231PD_  	%ymm6,%ymm3,%ymm0
+	vmovups 	 -4 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
+	VFMADD231PD_  	%ymm8,%ymm2,%ymm0
+	VFMADD231PD_  	%ymm9,%ymm3,%ymm0
+	addq	$12, BI
+	addq	$32, %rax
+.endm
+
+.macro KERNEL8x3_SUB
+	vbroadcastsd	 -6 * SIZE(BO, BI, SIZE), %ymm1
+	vmovups 	-32 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
+	vbroadcastsd	 -5 * SIZE(BO, BI, SIZE), %ymm2
+	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
+	vbroadcastsd	 -4 * SIZE(BO, BI, SIZE), %ymm3
+	VFMADD231PD_  	%ymm6,%ymm3,%ymm0
+	vmovups 	-28 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
+	VFMADD231PD_  	%ymm8,%ymm2,%ymm0
+	VFMADD231PD_  	%ymm9,%ymm3,%ymm0
+	addq	$3 , BI
+	addq	$8 , %rax
+.endm
+
+.macro SAVE8x3
+
+	vbroadcastsd	ALPHA, %ymm0
+
+	vmulpd	%ymm0 , %ymm4 , %ymm4
+	vmulpd	%ymm0 , %ymm7 , %ymm7
+
+	vmulpd	%ymm0 , %ymm5 , %ymm5
+	vmulpd	%ymm0 , %ymm8 , %ymm8
+
+	vmulpd	%ymm0 , %ymm6 , %ymm6
+	vmulpd	%ymm0 , %ymm9 , %ymm9
+
+#if !defined(TRMMKERNEL)
+
+	vaddpd 	        (CO1), %ymm4,%ymm4
+	vaddpd  4 * SIZE(CO1), %ymm7,%ymm7
+
+	vaddpd 	        (CO1, LDC), %ymm5,%ymm5
+	vaddpd  4 * SIZE(CO1, LDC), %ymm8,%ymm8
+
+	vaddpd 	        (CO1, LDC, 2), %ymm6,%ymm6
+	vaddpd  4 * SIZE(CO1, LDC, 2), %ymm9,%ymm9
+
+#endif
+
+	vmovups	%ymm4 ,  	(CO1)
+	vmovups	%ymm7 , 4 * SIZE(CO1)
+
+	vmovups	%ymm5 ,  	(CO1, LDC)
+	vmovups	%ymm8 , 4 * SIZE(CO1, LDC)
+
+	vmovups	%ymm6 ,  	(CO1, LDC, 2)
+	vmovups	%ymm9 , 4 * SIZE(CO1, LDC, 2)
+
+.endm
+
+
+
+/*******************************************************************************************/
+
+.macro KERNEL4x3_1
+	prefetcht0	A_PR1(AO, %rax, SIZE)
+	vbroadcastsd	 -6 * SIZE(BO, BI, SIZE), %ymm1
+	vmovups 	-32 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
+	vbroadcastsd	 -5 * SIZE(BO, BI, SIZE), %ymm2
+	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
+	vbroadcastsd	 -4 * SIZE(BO, BI, SIZE), %ymm3
+	VFMADD231PD_  	%ymm6,%ymm3,%ymm0
+.endm
+
+.macro KERNEL4x3_2
+	vbroadcastsd	 -3 * SIZE(BO, BI, SIZE), %ymm1
+	vmovups 	-28 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
+	vbroadcastsd	 -2 * SIZE(BO, BI, SIZE), %ymm2
+	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
+	vbroadcastsd	 -1 * SIZE(BO, BI, SIZE), %ymm3
+	VFMADD231PD_  	%ymm6,%ymm3,%ymm0
+.endm
+
+.macro KERNEL4x3_3
+	prefetcht0	A_PR1(AO, %rax, SIZE)
+	vbroadcastsd	  0 * SIZE(BO, BI, SIZE), %ymm1
+	vmovups 	-24 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
+	vbroadcastsd	  1 * SIZE(BO, BI, SIZE), %ymm2
+	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
+	vbroadcastsd	  2 * SIZE(BO, BI, SIZE), %ymm3
+	VFMADD231PD_  	%ymm6,%ymm3,%ymm0
+.endm
+
+.macro KERNEL4x3_4
+	vbroadcastsd	  3 * SIZE(BO, BI, SIZE), %ymm1
+	vmovups 	-20 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
+	vbroadcastsd	  4 * SIZE(BO, BI, SIZE), %ymm2
+	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
+	vbroadcastsd	  5 * SIZE(BO, BI, SIZE), %ymm3
+	VFMADD231PD_  	%ymm6,%ymm3,%ymm0
+	addq	$12, BI
+	addq	$16, %rax
+.endm
+
+.macro KERNEL4x3_SUB
+	vbroadcastsd	 -6 * SIZE(BO, BI, SIZE), %ymm1
+	vmovups 	-32 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
+	vbroadcastsd	 -5 * SIZE(BO, BI, SIZE), %ymm2
+	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
+	vbroadcastsd	 -4 * SIZE(BO, BI, SIZE), %ymm3
+	VFMADD231PD_  	%ymm6,%ymm3,%ymm0
+	addq	$3 , BI
+	addq	$4 , %rax
+.endm
+
+.macro SAVE4x3
+
+	vbroadcastsd	ALPHA, %ymm0
+
+	vmulpd	%ymm0 , %ymm4 , %ymm4
+	vmulpd	%ymm0 , %ymm5 , %ymm5
+	vmulpd	%ymm0 , %ymm6 , %ymm6
+
+#if !defined(TRMMKERNEL)
+
+	vaddpd 	        (CO1), %ymm4,%ymm4
+	vaddpd 	        (CO1, LDC), %ymm5,%ymm5
+	vaddpd 	        (CO1, LDC, 2), %ymm6,%ymm6
+
+#endif
+
+	vmovups	%ymm4 ,  	(CO1)
+	vmovups	%ymm5 ,  	(CO1, LDC)
+	vmovups	%ymm6 ,  	(CO1, LDC, 2)
+
+.endm
+
+
+/*******************************************************************************************/
+
+.macro KERNEL2x3_1
+	prefetcht0	A_PR1(AO, %rax, SIZE)
+	vmovsd	 -6 * SIZE(BO, BI, SIZE), %xmm1
+	vmovsd 	-32 * SIZE(AO, %rax, SIZE), %xmm0
+	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
+	vmovsd	 -5 * SIZE(BO, BI, SIZE), %xmm2
+	VFMADD231SD_  	%xmm5,%xmm2,%xmm0
+	vmovsd	 -4 * SIZE(BO, BI, SIZE), %xmm3
+	VFMADD231SD_  	%xmm6,%xmm3,%xmm0
+	vmovsd 	-31 * SIZE(AO, %rax, SIZE), %xmm0
+	VFMADD231SD_  	%xmm8,%xmm1,%xmm0
+	VFMADD231SD_  	%xmm10,%xmm2,%xmm0
+	VFMADD231SD_  	%xmm12,%xmm3,%xmm0
+.endm
+
+.macro KERNEL2x3_2
+	vmovsd	 -3 * SIZE(BO, BI, SIZE), %xmm1
+	vmovsd 	-30 * SIZE(AO, %rax, SIZE), %xmm0
+	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
+	vmovsd	 -2 * SIZE(BO, BI, SIZE), %xmm2
+	VFMADD231SD_  	%xmm5,%xmm2,%xmm0
+	vmovsd	 -1 * SIZE(BO, BI, SIZE), %xmm3
+	VFMADD231SD_  	%xmm6,%xmm3,%xmm0
+	vmovsd 	-29 * SIZE(AO, %rax, SIZE), %xmm0
+	VFMADD231SD_  	%xmm8,%xmm1,%xmm0
+	VFMADD231SD_  	%xmm10,%xmm2,%xmm0
+	VFMADD231SD_  	%xmm12,%xmm3,%xmm0
+.endm
+
+.macro KERNEL2x3_3
+	vmovsd	  0 * SIZE(BO, BI, SIZE), %xmm1
+	vmovsd 	-28 * SIZE(AO, %rax, SIZE), %xmm0
+	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
+	vmovsd	  1 * SIZE(BO, BI, SIZE), %xmm2
+	VFMADD231SD_  	%xmm5,%xmm2,%xmm0
+	vmovsd	  2 * SIZE(BO, BI, SIZE), %xmm3
+	VFMADD231SD_  	%xmm6,%xmm3,%xmm0
+	vmovsd 	-27 * SIZE(AO, %rax, SIZE), %xmm0
+	VFMADD231SD_  	%xmm8,%xmm1,%xmm0
+	VFMADD231SD_  	%xmm10,%xmm2,%xmm0
+	VFMADD231SD_  	%xmm12,%xmm3,%xmm0
+.endm
+
+.macro KERNEL2x3_4
+	vmovsd	  3 * SIZE(BO, BI, SIZE), %xmm1
+	vmovsd 	-26 * SIZE(AO, %rax, SIZE), %xmm0
+	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
+	vmovsd	  4 * SIZE(BO, BI, SIZE), %xmm2
+	VFMADD231SD_  	%xmm5,%xmm2,%xmm0
+	vmovsd	  5 * SIZE(BO, BI, SIZE), %xmm3
+	VFMADD231SD_  	%xmm6,%xmm3,%xmm0
+	vmovsd 	-25 * SIZE(AO, %rax, SIZE), %xmm0
+	VFMADD231SD_  	%xmm8,%xmm1,%xmm0
+	VFMADD231SD_  	%xmm10,%xmm2,%xmm0
+	VFMADD231SD_  	%xmm12,%xmm3,%xmm0
+	addq	$12, BI
+	addq	$8, %rax
+.endm
+
+.macro KERNEL2x3_SUB
+	vmovsd	 -6 * SIZE(BO, BI, SIZE), %xmm1
+	vmovsd 	-32 * SIZE(AO, %rax, SIZE), %xmm0
+	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
+	vmovsd	 -5 * SIZE(BO, BI, SIZE), %xmm2
+	VFMADD231SD_  	%xmm5,%xmm2,%xmm0
+	vmovsd	 -4 * SIZE(BO, BI, SIZE), %xmm3
+	VFMADD231SD_  	%xmm6,%xmm3,%xmm0
+	vmovsd 	-31 * SIZE(AO, %rax, SIZE), %xmm0
+	VFMADD231SD_  	%xmm8,%xmm1,%xmm0
+	VFMADD231SD_  	%xmm10,%xmm2,%xmm0
+	VFMADD231SD_  	%xmm12,%xmm3,%xmm0
+	addq	$3 , BI
+	addq	$2 , %rax
+.endm
+
+.macro SAVE2x3
+
+	vmovsd	ALPHA, %xmm0
+
+	vmulsd	%xmm0 , %xmm4 , %xmm4
+	vmulsd	%xmm0 , %xmm8 , %xmm8
+	vmulsd	%xmm0 , %xmm5 , %xmm5
+	vmulsd	%xmm0 , %xmm10, %xmm10
+	vmulsd	%xmm0 , %xmm6 , %xmm6
+	vmulsd	%xmm0 , %xmm12, %xmm12
+
+#if !defined(TRMMKERNEL)
+
+	vaddsd 	 (CO1), %xmm4,%xmm4
+	vaddsd 1 * SIZE(CO1), %xmm8,%xmm8
+	vaddsd 	 (CO1, LDC), %xmm5,%xmm5
+	vaddsd 1 * SIZE(CO1, LDC), %xmm10,%xmm10
+	vaddsd 	 (CO1, LDC, 2), %xmm6,%xmm6
+	vaddsd 1 * SIZE(CO1, LDC, 2), %xmm12,%xmm12
+
+#endif
+
+	vmovsd	%xmm4 ,  	(CO1)
+	vmovsd	%xmm8 , 1 * SIZE(CO1)
+	vmovsd	%xmm5 ,  	(CO1, LDC)
+	vmovsd	%xmm10, 1 * SIZE(CO1, LDC)
+	vmovsd	%xmm6 ,  	(CO1, LDC, 2)
+	vmovsd	%xmm12, 1 * SIZE(CO1, LDC, 2)
+
+.endm
+
+/*******************************************************************************************/
+
+.macro KERNEL1x3_1
+	vmovsd	 -6 * SIZE(BO, BI, SIZE), %xmm1
+	vmovsd 	-32 * SIZE(AO, %rax, SIZE), %xmm0
+	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
+	vmovsd	 -5 * SIZE(BO, BI, SIZE), %xmm2
+	VFMADD231SD_  	%xmm5,%xmm2,%xmm0
+	vmovsd	 -4 * SIZE(BO, BI, SIZE), %xmm3
+	VFMADD231SD_  	%xmm6,%xmm3,%xmm0
+.endm
+
+.macro KERNEL1x3_2
+	vmovsd	 -3 * SIZE(BO, BI, SIZE), %xmm1
+	vmovsd 	-31 * SIZE(AO, %rax, SIZE), %xmm0
+	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
+	vmovsd	 -2 * SIZE(BO, BI, SIZE), %xmm2
+	VFMADD231SD_  	%xmm5,%xmm2,%xmm0
+	vmovsd	 -1 * SIZE(BO, BI, SIZE), %xmm3
+	VFMADD231SD_  	%xmm6,%xmm3,%xmm0
+.endm
+
+.macro KERNEL1x3_3
+	vmovsd	  0 * SIZE(BO, BI, SIZE), %xmm1
+	vmovsd 	-30 * SIZE(AO, %rax, SIZE), %xmm0
+	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
+	vmovsd	  1 * SIZE(BO, BI, SIZE), %xmm2
+	VFMADD231SD_  	%xmm5,%xmm2,%xmm0
+	vmovsd	  2 * SIZE(BO, BI, SIZE), %xmm3
+	VFMADD231SD_  	%xmm6,%xmm3,%xmm0
+.endm
+
+.macro KERNEL1x3_4
+	vmovsd	  3 * SIZE(BO, BI, SIZE), %xmm1
+	vmovsd 	-29 * SIZE(AO, %rax, SIZE), %xmm0
+	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
+	vmovsd	  4 * SIZE(BO, BI, SIZE), %xmm2
+	VFMADD231SD_  	%xmm5,%xmm2,%xmm0
+	vmovsd	  5 * SIZE(BO, BI, SIZE), %xmm3
+	VFMADD231SD_  	%xmm6,%xmm3,%xmm0
+	addq	$12, BI
+	addq	$4, %rax
+.endm
+
+.macro KERNEL1x3_SUB
+	vmovsd	 -6 * SIZE(BO, BI, SIZE), %xmm1
+	vmovsd 	-32 * SIZE(AO, %rax, SIZE), %xmm0
+	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
+	vmovsd	 -5 * SIZE(BO, BI, SIZE), %xmm2
+	VFMADD231SD_  	%xmm5,%xmm2,%xmm0
+	vmovsd	 -4 * SIZE(BO, BI, SIZE), %xmm3
+	VFMADD231SD_  	%xmm6,%xmm3,%xmm0
+	addq	$3 , BI
+	addq	$1 , %rax
+.endm
+
+.macro SAVE1x3
+
+	vmovsd	ALPHA, %xmm0
+
+	vmulsd	%xmm0 , %xmm4 , %xmm4
+	vmulsd	%xmm0 , %xmm5 , %xmm5
+	vmulsd	%xmm0 , %xmm6 , %xmm6
+
+#if !defined(TRMMKERNEL)
+
+	vaddsd 	 (CO1), %xmm4,%xmm4
+	vaddsd 	 (CO1, LDC), %xmm5,%xmm5
+	vaddsd 	 (CO1, LDC, 2), %xmm6,%xmm6
+
+#endif
+
+	vmovsd	%xmm4 ,  	(CO1)
+	vmovsd	%xmm5 ,  	(CO1, LDC)
+	vmovsd	%xmm6 ,  	(CO1, LDC, 2)
+
+.endm
+
+
+/*******************************************************************************************/
+
+/*******************************************************************************************
+* 2 lines of N
+*******************************************************************************************/
+
+.macro KERNEL16x2_1
+	prefetcht0	A_PR1(AO, %rax, SIZE)
+	vbroadcastsd	 -4 * SIZE(BO, BI, SIZE), %ymm1
+	vmovups 	-32 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
+	vbroadcastsd	 -3 * SIZE(BO, BI, SIZE), %ymm2
+	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
+	vmovups 	-28 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
+	prefetcht0	64+A_PR1(AO, %rax, SIZE)
+	VFMADD231PD_  	%ymm8,%ymm2,%ymm0
+	vmovups 	-24 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm10,%ymm1,%ymm0
+	VFMADD231PD_  	%ymm11,%ymm2,%ymm0
+	vmovups 	-20 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm13,%ymm1,%ymm0
+	VFMADD231PD_  	%ymm14,%ymm2,%ymm0
+.endm
+
+.macro KERNEL16x2_2
+	prefetcht0	128+A_PR1(AO, %rax, SIZE)
+	vbroadcastsd	 -2 * SIZE(BO, BI, SIZE), %ymm1
+	vmovups 	-16 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
+	vbroadcastsd	 -1 * SIZE(BO, BI, SIZE), %ymm2
+	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
+	vmovups 	-12 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
+	prefetcht0	192+A_PR1(AO, %rax, SIZE)
+	VFMADD231PD_  	%ymm8,%ymm2,%ymm0
+	vmovups 	 -8 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm10,%ymm1,%ymm0
+	VFMADD231PD_  	%ymm11,%ymm2,%ymm0
+	vmovups 	 -4 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm13,%ymm1,%ymm0
+	VFMADD231PD_  	%ymm14,%ymm2,%ymm0
+.endm
+
+.macro KERNEL16x2_3
+	prefetcht0	256+A_PR1(AO, %rax, SIZE)
+	vbroadcastsd	  0 * SIZE(BO, BI, SIZE), %ymm1
+	vmovups 	  0 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
+	vbroadcastsd	  1 * SIZE(BO, BI, SIZE), %ymm2
+	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
+	vmovups 	  4 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
+	prefetcht0	320+A_PR1(AO, %rax, SIZE)
+	VFMADD231PD_  	%ymm8,%ymm2,%ymm0
+	vmovups 	  8 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm10,%ymm1,%ymm0
+	VFMADD231PD_  	%ymm11,%ymm2,%ymm0
+	vmovups 	 12 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm13,%ymm1,%ymm0
+	VFMADD231PD_  	%ymm14,%ymm2,%ymm0
+.endm
+
+.macro KERNEL16x2_4
+	prefetcht0	384+A_PR1(AO, %rax, SIZE)
+	vbroadcastsd	  2 * SIZE(BO, BI, SIZE), %ymm1
+	vmovups 	 16 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
+	vbroadcastsd	  3 * SIZE(BO, BI, SIZE), %ymm2
+	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
+	vmovups 	 20 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
+	prefetcht0	448+A_PR1(AO, %rax, SIZE)
+	VFMADD231PD_  	%ymm8,%ymm2,%ymm0
+	vmovups 	 24 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm10,%ymm1,%ymm0
+	VFMADD231PD_  	%ymm11,%ymm2,%ymm0
+	vmovups 	 28 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm13,%ymm1,%ymm0
+	VFMADD231PD_  	%ymm14,%ymm2,%ymm0
+	addq	$8, BI
+	addq	$64, %rax
+.endm
+
+.macro KERNEL16x2_SUB
+	vbroadcastsd	 -4 * SIZE(BO, BI, SIZE), %ymm1
+	vmovups 	-32 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
+	vbroadcastsd	 -3 * SIZE(BO, BI, SIZE), %ymm2
+	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
+	vmovups 	-28 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
+	VFMADD231PD_  	%ymm8,%ymm2,%ymm0
+	vmovups 	-24 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm10,%ymm1,%ymm0
+	VFMADD231PD_  	%ymm11,%ymm2,%ymm0
+	vmovups 	-20 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm13,%ymm1,%ymm0
+	VFMADD231PD_  	%ymm14,%ymm2,%ymm0
+	addq	$2, BI
+	addq	$16, %rax
+.endm
+
+.macro SAVE16x2
+
+	vbroadcastsd	ALPHA, %ymm0
+
+	vmulpd	%ymm0 , %ymm4 , %ymm4
+	vmulpd	%ymm0 , %ymm7 , %ymm7
+	vmulpd	%ymm0 , %ymm10, %ymm10
+	vmulpd	%ymm0 , %ymm13, %ymm13
+
+	vmulpd	%ymm0 , %ymm5 , %ymm5
+	vmulpd	%ymm0 , %ymm8 , %ymm8
+	vmulpd	%ymm0 , %ymm11, %ymm11
+	vmulpd	%ymm0 , %ymm14, %ymm14
+
+#if !defined(TRMMKERNEL)
+
+	vaddpd 	        (CO1), %ymm4,%ymm4
+	vaddpd  4 * SIZE(CO1), %ymm7,%ymm7
+	vaddpd  8 * SIZE(CO1), %ymm10,%ymm10
+	vaddpd 12 * SIZE(CO1), %ymm13,%ymm13
+
+	vaddpd 	        (CO1, LDC), %ymm5,%ymm5
+	vaddpd  4 * SIZE(CO1, LDC), %ymm8,%ymm8
+	vaddpd  8 * SIZE(CO1, LDC), %ymm11,%ymm11
+	vaddpd 12 * SIZE(CO1, LDC), %ymm14,%ymm14
+
+#endif
+
+	vmovups	%ymm4 ,  	(CO1)
+	vmovups	%ymm7 , 4 * SIZE(CO1)
+	vmovups	%ymm10, 8 * SIZE(CO1)
+	vmovups	%ymm13,12 * SIZE(CO1)
+
+	vmovups	%ymm5 ,  	(CO1, LDC)
+	vmovups	%ymm8 , 4 * SIZE(CO1, LDC)
+	vmovups	%ymm11, 8 * SIZE(CO1, LDC)
+	vmovups	%ymm14,12 * SIZE(CO1, LDC)
+
+.endm
+
+
+
+/*******************************************************************************************/
+
+.macro KERNEL8x2_1
+	prefetcht0	A_PR1(AO, %rax, SIZE)
+	vbroadcastsd	 -4 * SIZE(BO, BI, SIZE), %ymm1
+	vmovups 	-32 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
+	vbroadcastsd	 -3 * SIZE(BO, BI, SIZE), %ymm2
+	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
+	vmovups 	-28 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
+	VFMADD231PD_  	%ymm8,%ymm2,%ymm0
+.endm
+
+.macro KERNEL8x2_2
+	prefetcht0	64+A_PR1(AO, %rax, SIZE)
+	vbroadcastsd	 -2 * SIZE(BO, BI, SIZE), %ymm1
+	vmovups 	-24 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
+	vbroadcastsd	 -1 * SIZE(BO, BI, SIZE), %ymm2
+	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
+	vmovups 	-20 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
+	VFMADD231PD_  	%ymm8,%ymm2,%ymm0
+.endm
+
+.macro KERNEL8x2_3
+	prefetcht0	128+A_PR1(AO, %rax, SIZE)
+	vbroadcastsd	  0 * SIZE(BO, BI, SIZE), %ymm1
+	vmovups 	-16 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
+	vbroadcastsd	  1 * SIZE(BO, BI, SIZE), %ymm2
+	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
+	vmovups 	-12 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
+	VFMADD231PD_  	%ymm8,%ymm2,%ymm0
+.endm
+
+.macro KERNEL8x2_4
+	prefetcht0	192+A_PR1(AO, %rax, SIZE)
+	vbroadcastsd	  2 * SIZE(BO, BI, SIZE), %ymm1
+	vmovups 	 -8 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
+	vbroadcastsd	  3 * SIZE(BO, BI, SIZE), %ymm2
+	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
+	vmovups 	 -4 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
+	VFMADD231PD_  	%ymm8,%ymm2,%ymm0
+	addq	$8, BI				 
+	addq	$32, %rax 			 
+.endm
+
+.macro KERNEL8x2_SUB
+	vbroadcastsd	 -4 * SIZE(BO, BI, SIZE), %ymm1
+	vmovups 	-32 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
+	vbroadcastsd	 -3 * SIZE(BO, BI, SIZE), %ymm2
+	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
+	vmovups 	-28 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
+	VFMADD231PD_  	%ymm8,%ymm2,%ymm0
+	addq	$2, BI				 
+	addq	$8 , %rax 			 
+.endm
+
+.macro SAVE8x2
+
+	vbroadcastsd	ALPHA, %ymm0
+
+	vmulpd	%ymm0 , %ymm4 , %ymm4
+	vmulpd	%ymm0 , %ymm7 , %ymm7
+
+	vmulpd	%ymm0 , %ymm5 , %ymm5
+	vmulpd	%ymm0 , %ymm8 , %ymm8
+
+#if !defined(TRMMKERNEL)
+
+	vaddpd 	        (CO1), %ymm4,%ymm4
+	vaddpd  4 * SIZE(CO1), %ymm7,%ymm7
+
+	vaddpd 	        (CO1, LDC), %ymm5,%ymm5
+	vaddpd  4 * SIZE(CO1, LDC), %ymm8,%ymm8
+
+#endif
+
+	vmovups	%ymm4 ,  	(CO1)
+	vmovups	%ymm7 , 4 * SIZE(CO1)
+
+	vmovups	%ymm5 ,  	(CO1, LDC)
+	vmovups	%ymm8 , 4 * SIZE(CO1, LDC)
+
+.endm
+
+
+
+/*******************************************************************************************/
+
+.macro KERNEL4x2_1
+	prefetcht0	A_PR1(AO, %rax, SIZE)
+	vbroadcastsd	 -4 * SIZE(BO, BI, SIZE), %ymm1
+	vmovups 	-32 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
+	vbroadcastsd	 -3 * SIZE(BO, BI, SIZE), %ymm2
+	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
+.endm
+
+.macro KERNEL4x2_2
+	vbroadcastsd	 -2 * SIZE(BO, BI, SIZE), %ymm1
+	vmovups 	-28 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
+	vbroadcastsd	 -1 * SIZE(BO, BI, SIZE), %ymm2
+	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
+.endm
+
+.macro KERNEL4x2_3
+	prefetcht0	64+A_PR1(AO, %rax, SIZE)
+	vbroadcastsd	  0 * SIZE(BO, BI, SIZE), %ymm1
+	vmovups 	-24 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
+	vbroadcastsd	  1 * SIZE(BO, BI, SIZE), %ymm2
+	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
+.endm
+
+.macro KERNEL4x2_4
+	vbroadcastsd	  2 * SIZE(BO, BI, SIZE), %ymm1
+	vmovups 	-20 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
+	vbroadcastsd	  3 * SIZE(BO, BI, SIZE), %ymm2
+	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
+	addq	$8, BI				 
+	addq	$16, %rax 			 
+.endm
+
+.macro KERNEL4x2_SUB
+	vbroadcastsd	 -4 * SIZE(BO, BI, SIZE), %ymm1
+	vmovups 	-32 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
+	vbroadcastsd	 -3 * SIZE(BO, BI, SIZE), %ymm2
+	VFMADD231PD_  	%ymm5,%ymm2,%ymm0
+	addq	$2, BI				 
+	addq	$4 , %rax 			 
+.endm
+
+.macro SAVE4x2
+
+	vbroadcastsd	ALPHA, %ymm0
+
+	vmulpd	%ymm0 , %ymm4 , %ymm4
+	vmulpd	%ymm0 , %ymm5 , %ymm5
+
+#if !defined(TRMMKERNEL)
+
+	vaddpd 	        (CO1), %ymm4,%ymm4
+	vaddpd 	        (CO1, LDC), %ymm5,%ymm5
+
+#endif
+
+	vmovups	%ymm4 ,  	(CO1)
+	vmovups	%ymm5 ,  	(CO1, LDC)
+
+.endm
+
+
+/*******************************************************************************************/
+
+.macro KERNEL2x2_1
+	prefetcht0	A_PR1(AO, %rax, SIZE)
+	vmovsd	 -4 * SIZE(BO, BI, SIZE), %xmm1
+	vmovsd 	-32 * SIZE(AO, %rax, SIZE), %xmm0
+	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
+	vmovsd	 -3 * SIZE(BO, BI, SIZE), %xmm2
+	VFMADD231SD_  	%xmm5,%xmm2,%xmm0
+	vmovsd 	-31 * SIZE(AO, %rax, SIZE), %xmm0
+	VFMADD231SD_  	%xmm8,%xmm1,%xmm0
+	VFMADD231SD_  	%xmm10,%xmm2,%xmm0
+.endm
+
+.macro KERNEL2x2_2
+	vmovsd	 -2 * SIZE(BO, BI, SIZE), %xmm1
+	vmovsd 	-30 * SIZE(AO, %rax, SIZE), %xmm0
+	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
+	vmovsd	 -1 * SIZE(BO, BI, SIZE), %xmm2
+	VFMADD231SD_  	%xmm5,%xmm2,%xmm0
+	vmovsd 	-29 * SIZE(AO, %rax, SIZE), %xmm0
+	VFMADD231SD_  	%xmm8,%xmm1,%xmm0
+	VFMADD231SD_  	%xmm10,%xmm2,%xmm0
+.endm
+
+.macro KERNEL2x2_3
+	vmovsd	  0 * SIZE(BO, BI, SIZE), %xmm1
+	vmovsd 	-28 * SIZE(AO, %rax, SIZE), %xmm0
+	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
+	vmovsd	  1 * SIZE(BO, BI, SIZE), %xmm2
+	VFMADD231SD_  	%xmm5,%xmm2,%xmm0
+	vmovsd 	-27 * SIZE(AO, %rax, SIZE), %xmm0
+	VFMADD231SD_  	%xmm8,%xmm1,%xmm0
+	VFMADD231SD_  	%xmm10,%xmm2,%xmm0
+.endm
+
+.macro KERNEL2x2_4
+	vmovsd	  2 * SIZE(BO, BI, SIZE), %xmm1
+	vmovsd 	-26 * SIZE(AO, %rax, SIZE), %xmm0
+	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
+	vmovsd	  3 * SIZE(BO, BI, SIZE), %xmm2
+	VFMADD231SD_  	%xmm5,%xmm2,%xmm0
+	vmovsd 	-25 * SIZE(AO, %rax, SIZE), %xmm0
+	VFMADD231SD_  	%xmm8,%xmm1,%xmm0
+	VFMADD231SD_  	%xmm10,%xmm2,%xmm0
+	addq	$8, BI				 
+	addq	$8, %rax 			 
+.endm
+
+.macro KERNEL2x2_SUB
+	vmovsd	 -4 * SIZE(BO, BI, SIZE), %xmm1
+	vmovsd 	-32 * SIZE(AO, %rax, SIZE), %xmm0
+	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
+	vmovsd	 -3 * SIZE(BO, BI, SIZE), %xmm2
+	VFMADD231SD_  	%xmm5,%xmm2,%xmm0
+	vmovsd 	-31 * SIZE(AO, %rax, SIZE), %xmm0
+	VFMADD231SD_  	%xmm8,%xmm1,%xmm0
+	VFMADD231SD_  	%xmm10,%xmm2,%xmm0
+	addq	$2, BI				 
+	addq	$2, %rax 			 
+.endm
+
+.macro SAVE2x2
+
+	vmovsd	ALPHA, %xmm0
+
+	vmulsd	%xmm0 , %xmm4 , %xmm4
+	vmulsd	%xmm0 , %xmm8 , %xmm8
+	vmulsd	%xmm0 , %xmm5 , %xmm5
+	vmulsd	%xmm0 , %xmm10, %xmm10
+
+#if !defined(TRMMKERNEL)
+
+	vaddsd 	 (CO1), %xmm4,%xmm4
+	vaddsd 1 * SIZE(CO1), %xmm8,%xmm8
+	vaddsd 	 (CO1, LDC), %xmm5,%xmm5
+	vaddsd 1 * SIZE(CO1, LDC), %xmm10,%xmm10
+
+#endif
+
+	vmovsd	%xmm4 ,  	(CO1)
+	vmovsd	%xmm8 , 1 * SIZE(CO1)
+	vmovsd	%xmm5 ,  	(CO1, LDC)
+	vmovsd	%xmm10, 1 * SIZE(CO1, LDC)
+
+.endm
+
+
+/*******************************************************************************************/
+
+.macro KERNEL1x2_1
+	vmovsd	 -4 * SIZE(BO, BI, SIZE), %xmm1
+	vmovsd 	-32 * SIZE(AO, %rax, SIZE), %xmm0
+	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
+	vmovsd	 -3 * SIZE(BO, BI, SIZE), %xmm2
+	VFMADD231SD_  	%xmm5,%xmm2,%xmm0
+.endm
+
+.macro KERNEL1x2_2
+	vmovsd	 -2 * SIZE(BO, BI, SIZE), %xmm1
+	vmovsd 	-31 * SIZE(AO, %rax, SIZE), %xmm0
+	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
+	vmovsd	 -1 * SIZE(BO, BI, SIZE), %xmm2
+	VFMADD231SD_  	%xmm5,%xmm2,%xmm0
+.endm
+
+.macro KERNEL1x2_3
+	vmovsd	  0 * SIZE(BO, BI, SIZE), %xmm1
+	vmovsd 	-30 * SIZE(AO, %rax, SIZE), %xmm0
+	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
+	vmovsd	  1 * SIZE(BO, BI, SIZE), %xmm2
+	VFMADD231SD_  	%xmm5,%xmm2,%xmm0
+.endm
+
+.macro KERNEL1x2_4
+	vmovsd	  2 * SIZE(BO, BI, SIZE), %xmm1
+	vmovsd 	-29 * SIZE(AO, %rax, SIZE), %xmm0
+	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
+	vmovsd	  3 * SIZE(BO, BI, SIZE), %xmm2
+	VFMADD231SD_  	%xmm5,%xmm2,%xmm0
+	addq	$8, BI				 
+	addq	$4, %rax 			 
+.endm
+
+.macro KERNEL1x2_SUB
+	vmovsd	 -4 * SIZE(BO, BI, SIZE), %xmm1
+	vmovsd 	-32 * SIZE(AO, %rax, SIZE), %xmm0
+	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
+	vmovsd	 -3 * SIZE(BO, BI, SIZE), %xmm2
+	VFMADD231SD_  	%xmm5,%xmm2,%xmm0
+	addq	$2, BI				 
+	addq	$1, %rax 			 
+.endm
+
+.macro SAVE1x2
+
+	vmovsd	ALPHA, %xmm0
+
+	vmulsd	%xmm0 , %xmm4 , %xmm4
+	vmulsd	%xmm0 , %xmm5 , %xmm5
+
+#if !defined(TRMMKERNEL)
+
+	vaddsd 	 (CO1), %xmm4,%xmm4
+	vaddsd 	 (CO1, LDC), %xmm5,%xmm5
+
+#endif
+
+	vmovsd	%xmm4 ,  	(CO1)
+	vmovsd	%xmm5 ,  	(CO1, LDC)
+
+.endm
+
+
+/*******************************************************************************************/
+
+/*******************************************************************************************
+* 1 line of N
+*******************************************************************************************/
+
+.macro KERNEL16x1_1
+	vbroadcastsd	 -2 * SIZE(BO, BI, SIZE), %ymm1
+	vmovups 	-32 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
+	vmovups 	-28 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
+	vmovups 	-24 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm10,%ymm1,%ymm0
+	vmovups 	-20 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm13,%ymm1,%ymm0
+.endm
+
+.macro KERNEL16x1_2
+	vbroadcastsd	 -1 * SIZE(BO, BI, SIZE), %ymm1
+	vmovups 	-16 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
+	vmovups 	-12 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
+	vmovups 	 -8 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm10,%ymm1,%ymm0
+	vmovups 	 -4 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm13,%ymm1,%ymm0
+.endm
+
+.macro KERNEL16x1_3
+	vbroadcastsd	  0 * SIZE(BO, BI, SIZE), %ymm1
+	vmovups 	  0 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
+	vmovups 	  4 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
+	vmovups 	  8 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm10,%ymm1,%ymm0
+	vmovups 	 12 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm13,%ymm1,%ymm0
+.endm
+
+.macro KERNEL16x1_4
+	vbroadcastsd	  1 * SIZE(BO, BI, SIZE), %ymm1
+	vmovups 	 16 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
+	vmovups 	 20 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
+	vmovups 	 24 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm10,%ymm1,%ymm0
+	vmovups 	 28 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm13,%ymm1,%ymm0
+	addq	$4, BI				 
+	addq	$64, %rax 			 
+.endm
+
+.macro KERNEL16x1_SUB
+	vbroadcastsd	 -2 * SIZE(BO, BI, SIZE), %ymm1
+	vmovups 	-32 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
+	vmovups 	-28 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
+	vmovups 	-24 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm10,%ymm1,%ymm0
+	vmovups 	-20 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm13,%ymm1,%ymm0
+	addq	$1, BI				 
+	addq	$16, %rax 			 
+.endm
+
+.macro SAVE16x1
+
+	vbroadcastsd	ALPHA, %ymm0
+
+	vmulpd	%ymm0 , %ymm4 , %ymm4
+	vmulpd	%ymm0 , %ymm7 , %ymm7
+	vmulpd	%ymm0 , %ymm10, %ymm10
+	vmulpd	%ymm0 , %ymm13, %ymm13
+
+#if !defined(TRMMKERNEL)
+
+	vaddpd 	        (CO1), %ymm4,%ymm4
+	vaddpd  4 * SIZE(CO1), %ymm7,%ymm7
+	vaddpd  8 * SIZE(CO1), %ymm10,%ymm10
+	vaddpd 12 * SIZE(CO1), %ymm13,%ymm13
+
+#endif
+
+	vmovups	%ymm4 ,  	(CO1)
+	vmovups	%ymm7 , 4 * SIZE(CO1)
+	vmovups	%ymm10, 8 * SIZE(CO1)
+	vmovups	%ymm13,12 * SIZE(CO1)
+
+.endm
+
+
+
+/*******************************************************************************************/
+
+.macro KERNEL8x1_1
+	vbroadcastsd	 -2 * SIZE(BO, BI, SIZE), %ymm1
+	vmovups 	-32 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
+	vmovups 	-28 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
+.endm
+
+.macro KERNEL8x1_2
+	vbroadcastsd	 -1 * SIZE(BO, BI, SIZE), %ymm1
+	vmovups 	-24 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
+	vmovups 	-20 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
+.endm
+
+.macro KERNEL8x1_3
+	vbroadcastsd	  0 * SIZE(BO, BI, SIZE), %ymm1
+	vmovups 	-16 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
+	vmovups 	-12 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
+.endm
+
+.macro KERNEL8x1_4
+	vbroadcastsd	  1 * SIZE(BO, BI, SIZE), %ymm1
+	vmovups 	 -8 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
+	vmovups 	 -4 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
+	addq	$4, BI				 
+	addq	$32, %rax 			 
+.endm
+
+.macro KERNEL8x1_SUB
+	vbroadcastsd	 -2 * SIZE(BO, BI, SIZE), %ymm1
+	vmovups 	-32 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
+	vmovups 	-28 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm7,%ymm1,%ymm0
+	addq	$1, BI				 
+	addq	$8 , %rax 			 
+.endm
+
+.macro SAVE8x1
+
+	vbroadcastsd	ALPHA, %ymm0
+
+	vmulpd	%ymm0 , %ymm4 , %ymm4
+	vmulpd	%ymm0 , %ymm7 , %ymm7
+
+#if !defined(TRMMKERNEL)
+
+	vaddpd 	        (CO1), %ymm4,%ymm4
+	vaddpd  4 * SIZE(CO1), %ymm7,%ymm7
+
+#endif
+
+	vmovups	%ymm4 ,  	(CO1)
+	vmovups	%ymm7 , 4 * SIZE(CO1)
+
+.endm
+
+
+
+/*******************************************************************************************/
+
+.macro KERNEL4x1_1
+	vbroadcastsd	 -2 * SIZE(BO, BI, SIZE), %ymm1
+	vmovups 	-32 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
+.endm
+
+.macro KERNEL4x1_2
+	vbroadcastsd	 -1 * SIZE(BO, BI, SIZE), %ymm1
+	vmovups 	-28 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
+.endm
+
+.macro KERNEL4x1_3
+	vbroadcastsd	  0 * SIZE(BO, BI, SIZE), %ymm1
+	vmovups 	-24 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
+.endm
+
+.macro KERNEL4x1_4
+	vbroadcastsd	  1 * SIZE(BO, BI, SIZE), %ymm1
+	vmovups 	-20 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
+	addq	$4, BI				 
+	addq	$16, %rax 			 
+.endm
+
+.macro KERNEL4x1_SUB
+	vbroadcastsd	 -2 * SIZE(BO, BI, SIZE), %ymm1
+	vmovups 	-32 * SIZE(AO, %rax, SIZE), %ymm0
+	VFMADD231PD_  	%ymm4,%ymm1,%ymm0
+	addq	$1, BI				 
+	addq	$4 , %rax 			 
+.endm
+
+.macro SAVE4x1
+
+	vbroadcastsd	ALPHA, %ymm0
+
+	vmulpd	%ymm0 , %ymm4 , %ymm4
+
+#if !defined(TRMMKERNEL)
+
+	vaddpd 	        (CO1), %ymm4,%ymm4
+
+#endif
+
+	vmovups	%ymm4 ,  	(CO1)
+
+.endm
+
+
+/*******************************************************************************************/
+
+.macro KERNEL2x1_1
+	vmovsd	 -2 * SIZE(BO, BI, SIZE), %xmm1
+	vmovsd 	-32 * SIZE(AO, %rax, SIZE), %xmm0
+	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
+	vmovsd 	-31 * SIZE(AO, %rax, SIZE), %xmm0
+	VFMADD231SD_  	%xmm8,%xmm1,%xmm0
+.endm
+
+.macro KERNEL2x1_2
+	vmovsd	 -1 * SIZE(BO, BI, SIZE), %xmm1
+	vmovsd 	-30 * SIZE(AO, %rax, SIZE), %xmm0
+	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
+	vmovsd 	-29 * SIZE(AO, %rax, SIZE), %xmm0
+	VFMADD231SD_  	%xmm8,%xmm1,%xmm0
+.endm
+
+.macro KERNEL2x1_3
+	vmovsd	  0 * SIZE(BO, BI, SIZE), %xmm1
+	vmovsd 	-28 * SIZE(AO, %rax, SIZE), %xmm0
+	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
+	vmovsd 	-27 * SIZE(AO, %rax, SIZE), %xmm0
+	VFMADD231SD_  	%xmm8,%xmm1,%xmm0
+.endm
+
+.macro KERNEL2x1_4
+	vmovsd	  1 * SIZE(BO, BI, SIZE), %xmm1
+	vmovsd 	-26 * SIZE(AO, %rax, SIZE), %xmm0
+	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
+	vmovsd 	-25 * SIZE(AO, %rax, SIZE), %xmm0
+	VFMADD231SD_  	%xmm8,%xmm1,%xmm0
+	addq	$4, BI				 
+	addq	$8, %rax 			 
+.endm
+
+.macro KERNEL2x1_SUB
+	vmovsd	 -2 * SIZE(BO, BI, SIZE), %xmm1
+	vmovsd 	-32 * SIZE(AO, %rax, SIZE), %xmm0
+	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
+	vmovsd 	-31 * SIZE(AO, %rax, SIZE), %xmm0
+	VFMADD231SD_  	%xmm8,%xmm1,%xmm0
+	addq	$1, BI				 
+	addq	$2 , %rax 			 
+.endm
+
+.macro SAVE2x1
+
+	vmovsd	ALPHA, %xmm0
+
+	vmulsd	%xmm0 , %xmm4 , %xmm4
+	vmulsd	%xmm0 , %xmm8 , %xmm8
+
+#if !defined(TRMMKERNEL)
+
+	vaddsd 	 (CO1), %xmm4,%xmm4
+	vaddsd 1 * SIZE(CO1), %xmm8,%xmm8
+
+#endif
+
+	vmovsd	%xmm4 ,  	(CO1)
+	vmovsd	%xmm8 , 1 * SIZE(CO1)
+
+.endm
+
+
+/*******************************************************************************************/
+
+.macro KERNEL1x1_1
+	vmovsd	 -2 * SIZE(BO, BI, SIZE), %xmm1
+	vmovsd 	-32 * SIZE(AO, %rax, SIZE), %xmm0
+	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
+.endm
+
+.macro KERNEL1x1_2
+	vmovsd	 -1 * SIZE(BO, BI, SIZE), %xmm1
+	vmovsd 	-31 * SIZE(AO, %rax, SIZE), %xmm0
+	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
+.endm
+
+.macro KERNEL1x1_3
+	vmovsd	  0 * SIZE(BO, BI, SIZE), %xmm1
+	vmovsd 	-30 * SIZE(AO, %rax, SIZE), %xmm0
+	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
+.endm
+
+.macro KERNEL1x1_4
+	vmovsd	  1 * SIZE(BO, BI, SIZE), %xmm1
+	vmovsd 	-29 * SIZE(AO, %rax, SIZE), %xmm0
+	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
+	addq	$ 4, BI				 
+	addq	$ 4, %rax 			 
+.endm
+
+.macro KERNEL1x1_SUB
+	vmovsd	 -2 * SIZE(BO, BI, SIZE), %xmm1
+	vmovsd 	-32 * SIZE(AO, %rax, SIZE), %xmm0
+	VFMADD231SD_  	%xmm4,%xmm1,%xmm0
+	addq	$ 1, BI				 
+	addq	$ 1 , %rax 			 
+.endm
+
+.macro SAVE1x1
+
+	vmovsd	ALPHA, %xmm0
+
+	vmulsd	%xmm0 , %xmm4 , %xmm4
+
+#if !defined(TRMMKERNEL)
+
+	vaddsd 	 (CO1), %xmm4,%xmm4
+
+#endif
+
+	vmovsd	%xmm4 ,  	(CO1)
+
+.endm
+
+
+/*******************************************************************************************/
+
+#if !defined(TRMMKERNEL)
+
+
+	PROLOGUE
+	PROFCODE
+	
+	subq	$STACKSIZE, %rsp
+	movq	%rbx,   (%rsp)
+	movq	%rbp,  8(%rsp)
+	movq	%r12, 16(%rsp)
+	movq	%r13, 24(%rsp)
+	movq	%r14, 32(%rsp)
+	movq	%r15, 40(%rsp)
+
+	vzeroupper
+
+#ifdef WINDOWS_ABI
+	movq	%rdi,    48(%rsp)
+	movq	%rsi,    56(%rsp)
+	movups	%xmm6,   64(%rsp)
+	movups	%xmm7,   80(%rsp)
+	movups	%xmm8,   96(%rsp)
+	movups	%xmm9,  112(%rsp)
+	movups	%xmm10, 128(%rsp)
+	movups	%xmm11, 144(%rsp)
+	movups	%xmm12, 160(%rsp)
+	movups	%xmm13, 176(%rsp)
+	movups	%xmm14, 192(%rsp)
+	movups	%xmm15, 208(%rsp)
+
+	movq	ARG1,      OLD_M
+	movq	ARG2,      OLD_N
+	movq	ARG3,      OLD_K
+	movq	OLD_A,     A
+	movq	OLD_B,     B
+	movq	OLD_C,     C
+	movq	OLD_LDC,   LDC
+
+	vmovaps	%xmm3, %xmm0
+
+#else
+	movq	STACKSIZE +  8(%rsp), LDC
+
+#endif
+
+	movq    %rsp, SP      # save old stack
+        subq    $128 + L_BUFFER_SIZE, %rsp
+        andq    $-4096, %rsp    # align stack
+
+        STACK_TOUCH
+
+	cmpq	$0, OLD_M
+	je	.L999
+
+	cmpq	$0, OLD_N
+	je	.L999
+
+	cmpq	$0, OLD_K
+	je	.L999
+
+	movq	OLD_M, M
+	movq	OLD_N, N
+	movq	OLD_K, K
+
+	vmovsd	 %xmm0, ALPHA
+
+	salq	$BASE_SHIFT, LDC
+
+	movq    N, %rax
+        xorq    %rdx, %rdx
+        movq    $6,  %rdi
+        divq    %rdi                    //    N / 6
+        movq    %rax, Ndiv6             //    N / 6
+        movq    %rdx, Nmod6             //    N % 6
+
+
+	movq	Ndiv6,  J
+	cmpq	$0, J
+	je	.L2_0
+	ALIGN_4
+
+.L6_01:
+        // copy to sub buffer
+        movq    K, %rax
+        salq    $1,%rax                 // K * 2 ; read 2 values
+        movq    B, BO1
+        leaq    (B,%rax, SIZE), BO2     // next offset to BO2
+        leaq    BUFFER1, BO             // first buffer to BO
+        movq    K, %rax
+	sarq	$3 , %rax		// K / 8
+	jz	.L6_01a_2
+        ALIGN_4
+
+.L6_01a_1:
+
+        prefetcht0 512(BO1)
+        prefetcht0 512(BO2)
+        prefetchw  512(BO)
+
+
+	vmovups	0 * SIZE(BO1), %xmm0
+	vmovups	2 * SIZE(BO1), %xmm2
+	vmovups	4 * SIZE(BO1), %xmm4
+	vmovups	6 * SIZE(BO1), %xmm6
+	vmovsd  0 * SIZE(BO2), %xmm1
+	vmovsd  2 * SIZE(BO2), %xmm3
+	vmovsd  4 * SIZE(BO2), %xmm5
+	vmovsd  6 * SIZE(BO2), %xmm7
+	vmovups	%xmm0, 0*SIZE(BO)
+	vmovsd	%xmm1, 2*SIZE(BO)
+	vmovups	%xmm2, 3*SIZE(BO)
+	vmovsd	%xmm3, 5*SIZE(BO)
+	vmovups	%xmm4, 6*SIZE(BO)
+	vmovsd	%xmm5, 8*SIZE(BO)
+	vmovups	%xmm6, 9*SIZE(BO)
+	vmovsd	%xmm7,11*SIZE(BO)
+	addq	$ 8*SIZE,BO1
+	addq	$ 8*SIZE,BO2
+	addq	$ 12*SIZE,BO
+
+	vmovups	0 * SIZE(BO1), %xmm0
+	vmovups	2 * SIZE(BO1), %xmm2
+	vmovups	4 * SIZE(BO1), %xmm4
+	vmovups	6 * SIZE(BO1), %xmm6
+	vmovsd  0 * SIZE(BO2), %xmm1
+	vmovsd  2 * SIZE(BO2), %xmm3
+	vmovsd  4 * SIZE(BO2), %xmm5
+	vmovsd  6 * SIZE(BO2), %xmm7
+	vmovups	%xmm0, 0*SIZE(BO)
+	vmovsd	%xmm1, 2*SIZE(BO)
+	vmovups	%xmm2, 3*SIZE(BO)
+	vmovsd	%xmm3, 5*SIZE(BO)
+	vmovups	%xmm4, 6*SIZE(BO)
+	vmovsd	%xmm5, 8*SIZE(BO)
+	vmovups	%xmm6, 9*SIZE(BO)
+	vmovsd	%xmm7,11*SIZE(BO)
+	addq	$ 8*SIZE,BO1
+	addq	$ 8*SIZE,BO2
+	addq	$ 12*SIZE,BO
+
+	decq	%rax
+	jnz	.L6_01a_1
+
+
+
+.L6_01a_2:
+
+	movq    K, %rax
+        andq    $7, %rax                // K % 8
+        jz      .L6_02c
+        ALIGN_4
+
+
+.L6_02b:
+
+	vmovups	0 * SIZE(BO1), %xmm0
+	vmovsd  0 * SIZE(BO2), %xmm2
+	vmovups	%xmm0, 0*SIZE(BO)
+	vmovsd	%xmm2, 2*SIZE(BO)
+	addq	$ 2*SIZE,BO1
+	addq	$ 2*SIZE,BO2
+	addq	$ 3*SIZE,BO
+	decq	%rax
+	jnz	.L6_02b
+
+.L6_02c:
+
+	movq	K, %rax
+	salq	$1,%rax			// K * 2
+	leaq	(B,%rax, SIZE), BO1	// next offset to BO1
+	leaq	(BO1,%rax, SIZE), BO2	// next offset to BO2
+	leaq    BUFFER2, BO		// second buffer to BO
+	movq	K, %rax
+	sarq	$3 , %rax		// K / 8
+	jz	.L6_02c_2
+	ALIGN_4
+
+.L6_02c_1:
+
+	prefetcht0 512(BO2)
+        prefetchw  512(BO)
+
+	vmovups	0 * SIZE(BO2), %xmm0
+	vmovups	2 * SIZE(BO2), %xmm2
+	vmovups	4 * SIZE(BO2), %xmm4
+	vmovups	6 * SIZE(BO2), %xmm6
+	vmovsd  1 * SIZE(BO1), %xmm1
+	vmovsd  3 * SIZE(BO1), %xmm3
+	vmovsd  5 * SIZE(BO1), %xmm5
+	vmovsd  7 * SIZE(BO1), %xmm7
+	vmovsd	%xmm1, 0*SIZE(BO)
+	vmovups	%xmm0, 1*SIZE(BO)
+	vmovsd	%xmm3, 3*SIZE(BO)
+	vmovups	%xmm2, 4*SIZE(BO)
+	vmovsd	%xmm5, 6*SIZE(BO)
+	vmovups	%xmm4, 7*SIZE(BO)
+	vmovsd	%xmm7, 9*SIZE(BO)
+	vmovups	%xmm6,10*SIZE(BO)
+	addq	$8*SIZE,BO1
+	addq	$8*SIZE,BO2
+	addq	$12*SIZE,BO
+
+
+	vmovups	0 * SIZE(BO2), %xmm0
+	vmovups	2 * SIZE(BO2), %xmm2
+	vmovups	4 * SIZE(BO2), %xmm4
+	vmovups	6 * SIZE(BO2), %xmm6
+	vmovsd  1 * SIZE(BO1), %xmm1
+	vmovsd  3 * SIZE(BO1), %xmm3
+	vmovsd  5 * SIZE(BO1), %xmm5
+	vmovsd  7 * SIZE(BO1), %xmm7
+	vmovsd	%xmm1, 0*SIZE(BO)
+	vmovups	%xmm0, 1*SIZE(BO)
+	vmovsd	%xmm3, 3*SIZE(BO)
+	vmovups	%xmm2, 4*SIZE(BO)
+	vmovsd	%xmm5, 6*SIZE(BO)
+	vmovups	%xmm4, 7*SIZE(BO)
+	vmovsd	%xmm7, 9*SIZE(BO)
+	vmovups	%xmm6,10*SIZE(BO)
+	addq	$8*SIZE,BO1
+	addq	$8*SIZE,BO2
+	addq	$12*SIZE,BO
+
+	decq	%rax
+	jnz	.L6_02c_1
+
+
+.L6_02c_2:
+
+	movq    K, %rax
+        andq    $7, %rax                // K % 8
+        jz      .L6_03c
+        ALIGN_4
+
+.L6_03b:
+
+	vmovsd	  1*SIZE(BO1), %xmm0
+	vmovups	  0*SIZE(BO2), %xmm1
+	vmovsd	%xmm0, 0*SIZE(BO)
+	vmovups	%xmm1, 1*SIZE(BO)
+	addq	$2*SIZE,BO1
+	addq	$2*SIZE,BO2
+	addq	$3*SIZE,BO
+	decq	%rax
+	jnz	.L6_03b
+
+
+.L6_03c:
+
+	movq	BO2, B			// next offset of B
+
+.L6_10:
+	movq	C, CO1
+	leaq	(C, LDC, 2), C		 
+	leaq	(C, LDC, 1), C		// c += 3 * ldc
+
+	
+	movq	A, AO		 	// aoffset = a
+	addq	$16 * SIZE, AO
+
+	movq	M,  I
+	sarq	$4, I			// i = (m >> 4)
+	je	.L6_20
+
+	ALIGN_4
+
+.L6_11:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $12 * SIZE, BO
+
+	prefetcht0	(CO1)
+	prefetcht0	(CO1,LDC,1)
+	prefetcht0	(CO1,LDC,2)
+	prefetcht0	64(CO1)
+	prefetcht0	64(CO1,LDC,1)
+	prefetcht0	64(CO1,LDC,2)
+
+	vzeroall
+
+        movq    K, %rax
+
+	sarq $1, %rax			//  K / 8
+	je	.L6_16
+
+	ALIGN_5
+
+.L6_12:
+/*
+	prefetcht0	B_PR1(BO)
+	prefetcht0	B_PR1+64(BO)
+	prefetcht0	B_PR1+128(BO)
+*/
+	KERNEL16x3_SUBN
+	KERNEL16x3_SUBN
+/*
+	KERNEL16x3_SUBN
+	KERNEL16x3_SUBN
+
+	KERNEL16x3_SUBN
+	KERNEL16x3_SUBN
+	KERNEL16x3_SUBN
+	KERNEL16x3_SUBN
+*/
+	dec	%rax
+	jne	.L6_12
+
+.L6_16:
+        movq    K, %rax
+
+	andq	$1, %rax		# if (k & 1)
+	je .L6_19
+
+	ALIGN_4
+
+.L6_17:
+
+	KERNEL16x3_SUBN
+
+	dec	%rax
+	jne	.L6_17
+	ALIGN_4
+
+
+.L6_19:
+
+	SAVE16x3
+
+	addq	$16 * SIZE, CO1		# coffset += 16
+	decq	I			# i --
+	jg	.L6_11
+	ALIGN_4	
+
+/**************************************************************************
+* Rest of M 
+***************************************************************************/
+.L6_20:
+	// Test rest of M
+
+	testq	$15, M
+	jz	.L7_10		// to next 3 lines of N
+
+	testq	$8, M		
+	jz	.L6_21pre
+	ALIGN_4
+
+/**************************************************************************/
+
+.L6_20_1:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $12 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	sarq	$3, %rax
+	je	.L6_20_6
+
+	ALIGN_4
+
+.L6_20_2:
+
+	KERNEL8x3_SUBN
+	KERNEL8x3_SUBN
+	KERNEL8x3_SUBN
+	KERNEL8x3_SUBN
+
+	KERNEL8x3_SUBN
+	KERNEL8x3_SUBN
+	KERNEL8x3_SUBN
+	KERNEL8x3_SUBN
+	dec	%rax
+	jne	.L6_20_2
+	ALIGN_4
+
+.L6_20_6:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L6_20_9
+
+
+	ALIGN_4
+
+.L6_20_7:
+
+	KERNEL8x3_SUBN
+
+	dec	%rax
+	jne	.L6_20_7
+	ALIGN_4
+
+
+.L6_20_9:
+
+	SAVE8x3
+
+	addq	$8 * SIZE, CO1		# coffset += 8
+	ALIGN_4
+	
+
+
+/**************************************************************************/
+
+.L6_21pre:
+
+	testq	$4, M		
+	jz	.L6_30
+	ALIGN_4
+
+.L6_21:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $12 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	sarq	$3, %rax
+	je	.L6_26
+
+	ALIGN_4
+
+.L6_22:
+
+	KERNEL4x3_SUBN
+	KERNEL4x3_SUBN
+	KERNEL4x3_SUBN
+	KERNEL4x3_SUBN
+
+	KERNEL4x3_SUBN
+	KERNEL4x3_SUBN
+	KERNEL4x3_SUBN
+	KERNEL4x3_SUBN
+	dec	%rax
+	jne	.L6_22
+	ALIGN_4
+
+.L6_26:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L6_29
+
+	ALIGN_4
+
+.L6_27:
+
+	KERNEL4x3_SUBN
+
+	dec %rax
+	jne	.L6_27
+	ALIGN_4
+
+
+.L6_29:
+
+	SAVE4x3
+
+	addq	$4 * SIZE, CO1		# coffset += 4
+	ALIGN_4
+	
+
+.L6_30:
+	testq	$2, M		
+	jz	.L6_40
+
+	ALIGN_4
+
+.L6_31:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $12 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	sarq	$3, %rax
+	je	.L6_36
+	ALIGN_4
+
+.L6_32:
+
+	KERNEL2x3_SUBN
+	KERNEL2x3_SUBN
+	KERNEL2x3_SUBN
+	KERNEL2x3_SUBN
+
+	KERNEL2x3_SUBN
+	KERNEL2x3_SUBN
+	KERNEL2x3_SUBN
+	KERNEL2x3_SUBN
+	dec %rax
+	jne	.L6_32
+	ALIGN_4
+
+.L6_36:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L6_39
+
+	ALIGN_4
+
+.L6_37:
+
+	KERNEL2x3_SUBN
+
+	dec %rax
+	jne	.L6_37
+	ALIGN_4
+
+
+.L6_39:
+
+	SAVE2x3
+
+	addq	$2 * SIZE, CO1		# coffset += 2
+	ALIGN_4
+
+.L6_40:
+	testq	$1, M		
+	jz	.L7_10		// to next 3 lines of N
+
+	ALIGN_4
+
+.L6_41:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $12 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	sarq	$3,%rax
+	je	.L6_46
+
+	ALIGN_4
+
+.L6_42:
+
+	KERNEL1x3_SUBN
+	KERNEL1x3_SUBN
+	KERNEL1x3_SUBN
+	KERNEL1x3_SUBN
+
+	KERNEL1x3_SUBN
+	KERNEL1x3_SUBN
+	KERNEL1x3_SUBN
+	KERNEL1x3_SUBN
+
+	dec %rax
+	jne	.L6_42
+	ALIGN_4
+
+.L6_46:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L6_49
+
+	ALIGN_4
+
+.L6_47:
+
+	KERNEL1x3_SUBN
+
+	dec	%rax
+	jne	.L6_47
+	ALIGN_4
+
+
+.L6_49:
+
+	SAVE1x3
+
+	addq	$1 * SIZE, CO1		# coffset += 1
+	ALIGN_4
+	
+
+
+
+/***************************************************************************************************************/
+
+.L7_10:
+	movq	C, CO1
+	leaq	(C, LDC, 2), C		 
+	leaq	(C, LDC, 1), C		// c += 3 * ldc
+
+	
+	movq	A, AO		 	// aoffset = a
+	addq	$16 * SIZE, AO
+
+	movq	M,  I
+	sarq	$4, I			// i = (m >> 4)
+	je	.L7_20
+
+	ALIGN_4
+
+.L7_11:
+        leaq    BUFFER2, BO             // second buffer to BO
+        addq    $12 * SIZE, BO
+
+	prefetcht0	(CO1)
+	prefetcht0	(CO1,LDC,1)
+	prefetcht0	(CO1,LDC,2)
+	prefetcht0	64(CO1)
+	prefetcht0	64(CO1,LDC,1)
+	prefetcht0	64(CO1,LDC,2)
+
+	vzeroall
+
+        movq    K, %rax
+
+	sarq $3, %rax			// K / 8
+	je	.L7_16
+	ALIGN_5
+
+.L7_12:
+/*
+	prefetcht0	B_PR1(BO)
+	prefetcht0	B_PR1+64(BO)
+	prefetcht0	B_PR1+128(BO)
+*/
+	KERNEL16x3_SUBN
+	KERNEL16x3_SUBN
+	KERNEL16x3_SUBN
+	KERNEL16x3_SUBN
+
+	KERNEL16x3_SUBN
+	KERNEL16x3_SUBN
+	KERNEL16x3_SUBN
+	KERNEL16x3_SUBN
+	dec %rax
+	jne	.L7_12
+	ALIGN_4
+
+.L7_16:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L7_19
+
+	ALIGN_5
+
+.L7_17:
+
+	KERNEL16x3_SUBN
+
+	dec	%rax
+	jne	.L7_17
+
+
+.L7_19:
+
+	SAVE16x3
+
+	addq	$16 * SIZE, CO1		# coffset += 16
+	decq	I			# i --
+	jg	.L7_11
+	ALIGN_4	
+
+/**************************************************************************
+* Rest of M 
+***************************************************************************/
+.L7_20:
+	// Test rest of M
+
+	testq	$15, M
+	jz	.L7_60		// to next 3 lines of N
+
+	testq	$8, M		
+	jz	.L7_21pre
+	ALIGN_4
+
+/**************************************************************************/
+
+.L7_20_1:
+        leaq    BUFFER2, BO             // first buffer to BO
+        addq    $12 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	sarq	$3, %rax
+	je	.L7_20_6
+
+	ALIGN_4
+
+.L7_20_2:
+
+	KERNEL8x3_SUBN
+	KERNEL8x3_SUBN
+	KERNEL8x3_SUBN
+	KERNEL8x3_SUBN
+
+	KERNEL8x3_SUBN
+	KERNEL8x3_SUBN
+	KERNEL8x3_SUBN
+	KERNEL8x3_SUBN
+
+	dec %rax
+	jne	.L7_20_2
+	ALIGN_4
+
+.L7_20_6:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L7_20_9
+
+	ALIGN_4
+
+.L7_20_7:
+
+	KERNEL8x3_SUBN
+
+	dec %rax
+	jne	.L7_20_7
+	ALIGN_4
+
+.L7_20_9:
+
+	SAVE8x3
+
+	addq	$8 * SIZE, CO1		# coffset += 8
+	ALIGN_4
+	
+
+
+/**************************************************************************/
+
+.L7_21pre:
+
+	testq	$4, M		
+	jz	.L7_30
+	ALIGN_4
+
+.L7_21:
+        leaq    BUFFER2, BO             // second buffer to BO
+        addq    $12 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	sarq	$3, %rax
+	je	.L7_26
+
+	ALIGN_4
+
+.L7_22:
+
+	KERNEL4x3_SUBN
+	KERNEL4x3_SUBN
+	KERNEL4x3_SUBN
+	KERNEL4x3_SUBN
+
+	KERNEL4x3_SUBN
+	KERNEL4x3_SUBN
+	KERNEL4x3_SUBN
+	KERNEL4x3_SUBN
+
+	dec %rax
+	jne	.L7_22
+	ALIGN_4
+
+.L7_26:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L7_29
+
+	ALIGN_4
+
+.L7_27:
+
+	KERNEL4x3_SUBN
+
+	dec %rax
+	jne	.L7_27
+	ALIGN_4
+
+
+.L7_29:
+
+	SAVE4x3
+
+	addq	$4 * SIZE, CO1		# coffset += 4
+	ALIGN_4
+	
+
+.L7_30:
+	testq	$2, M		
+	jz	.L7_40
+
+	ALIGN_4
+
+.L7_31:
+        leaq    BUFFER2, BO             // second buffer to BO
+        addq    $12 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	sarq	$3, %rax
+	je	.L7_36
+
+	ALIGN_4
+
+.L7_32:
+
+	KERNEL2x3_SUBN
+	KERNEL2x3_SUBN
+	KERNEL2x3_SUBN
+	KERNEL2x3_SUBN
+
+	KERNEL2x3_SUBN
+	KERNEL2x3_SUBN
+	KERNEL2x3_SUBN
+	KERNEL2x3_SUBN
+
+	dec %rax
+	jne	.L7_32
+	ALIGN_4
+
+.L7_36:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L7_39
+
+	ALIGN_4
+
+.L7_37:
+
+	KERNEL2x3_SUBN
+
+	dec %rax
+	jne	.L7_37
+	ALIGN_4
+
+
+.L7_39:
+
+	SAVE2x3
+
+	addq	$2 * SIZE, CO1		# coffset += 2
+	ALIGN_4
+
+.L7_40:
+	testq	$1, M		
+	jz	.L7_60		// to next 3 lines of N
+
+	ALIGN_4
+
+.L7_41:
+        leaq    BUFFER2, BO             // second buffer to BO
+        addq    $12 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	sarq	$3, %rax
+	je	.L7_46
+
+	ALIGN_4
+
+.L7_42:
+	KERNEL1x3_SUBN
+	KERNEL1x3_SUBN
+	KERNEL1x3_SUBN
+	KERNEL1x3_SUBN
+
+	KERNEL1x3_SUBN
+	KERNEL1x3_SUBN
+	KERNEL1x3_SUBN
+	KERNEL1x3_SUBN
+
+	dec %rax
+	jne	.L7_42
+	ALIGN_4
+
+.L7_46:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L7_49
+
+	ALIGN_4
+
+.L7_47:
+
+	KERNEL1x3_SUBN
+
+	dec %rax
+	jne	.L7_47
+	ALIGN_4
+
+
+.L7_49:
+
+	SAVE1x3
+
+	addq	$1 * SIZE, CO1		# coffset += 1
+	ALIGN_4
+	
+
+
+.L7_60:
+
+	decq	J			// j --
+	jg	.L6_01
+
+
+.L2_0:
+	cmpq	$0, Nmod6		// N % 6 == 0
+	je	.L999
+
+/************************************************************************************************
+* Loop for Nmod6 / 2 > 0
+*************************************************************************************************/
+
+	movq	Nmod6, J		
+	sarq	$1, J			// j = j / 2
+	je	.L1_0
+	ALIGN_4
+
+.L2_01:
+	// copy to sub buffer
+	movq	B, BO1
+	leaq    BUFFER1, BO		// first buffer to BO
+	movq	K, %rax
+	sarq	$2, %rax		// K / 4
+	jz	.L2_01b
+	ALIGN_4
+
+.L2_01a:
+        prefetcht0 512(BO1)
+        prefetchw  512(BO)
+
+	vmovups	      (BO1), %xmm0
+	vmovups	2*SIZE(BO1), %xmm1
+	vmovups	4*SIZE(BO1), %xmm2
+	vmovups	6*SIZE(BO1), %xmm3
+
+	vmovups	%xmm0,       (BO)
+	vmovups	%xmm1, 2*SIZE(BO)
+	vmovups	%xmm2, 4*SIZE(BO)
+	vmovups	%xmm3, 6*SIZE(BO)
+
+	addq	$8*SIZE,BO1
+	addq	$8*SIZE,BO
+	decq	%rax
+	jnz	.L2_01a
+
+
+.L2_01b:
+
+        movq    K, %rax
+        andq    $3, %rax                // K % 4
+        jz      .L2_02d
+        ALIGN_4
+
+.L2_02c:
+
+	vmovups	(BO1), %xmm0
+	vmovups	%xmm0, (BO)
+	addq	$2*SIZE,BO1
+	addq	$2*SIZE,BO
+	decq	%rax
+	jnz	.L2_02c
+
+.L2_02d:
+
+	movq	BO1, B			// next offset of B
+
+.L2_10:
+	movq	C, CO1
+	leaq	(C, LDC, 2), C		// c += 2 * ldc
+
+	
+	movq	A, AO		 	// aoffset = a
+	addq	$32 * SIZE, AO
+
+	movq	M,  I
+	sarq	$4, I			// i = (m >> 4)
+	je	.L2_20
+
+	ALIGN_4
+
+.L2_11:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax			//  K = K - ( K % 8 )
+	je	.L2_16
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$4, %rax			// rax = rax * 16 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_12:
+
+	prefetcht0      B_PR1(BO,BI,8)
+	KERNEL16x2_1
+	KERNEL16x2_2
+	KERNEL16x2_3
+	KERNEL16x2_4
+
+	prefetcht0      B_PR1(BO,BI,8)
+	KERNEL16x2_1
+	KERNEL16x2_2
+	KERNEL16x2_3
+	KERNEL16x2_4
+
+	je	.L2_16
+
+	prefetcht0      B_PR1(BO,BI,8)
+	KERNEL16x2_1
+	KERNEL16x2_2
+	KERNEL16x2_3
+	KERNEL16x2_4
+
+	prefetcht0      B_PR1(BO,BI,8)
+	KERNEL16x2_1
+	KERNEL16x2_2
+	KERNEL16x2_3
+	KERNEL16x2_4
+
+	je	.L2_16
+
+	jmp	.L2_12
+	ALIGN_4
+
+.L2_16:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_19
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$4, %rax			// rax = rax * 16 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_17:
+
+	KERNEL16x2_SUB
+
+	jl	.L2_17
+	ALIGN_4
+
+
+.L2_19:
+
+	SAVE16x2
+
+	addq	$16 * SIZE, CO1		# coffset += 16
+	decq	I			# i --
+	jg	.L2_11
+	ALIGN_4	
+
+/**************************************************************************
+* Rest of M 
+***************************************************************************/
+.L2_20:
+	// Test rest of M
+
+	testq	$15, M
+	jz	.L2_60		// to next 3 lines of N
+
+	testq	$8, M		
+	jz	.L2_21pre
+	ALIGN_4
+
+/**************************************************************************/
+
+.L2_20_1:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L2_20_6
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_20_2:
+
+	prefetcht0      B_PR1(BO,BI,8)
+	KERNEL8x2_1
+	KERNEL8x2_2
+	KERNEL8x2_3
+	KERNEL8x2_4
+
+	prefetcht0      B_PR1(BO,BI,8)
+	KERNEL8x2_1
+	KERNEL8x2_2
+	KERNEL8x2_3
+	KERNEL8x2_4
+
+	je	.L2_20_6
+
+	prefetcht0      B_PR1(BO,BI,8)
+	KERNEL8x2_1
+	KERNEL8x2_2
+	KERNEL8x2_3
+	KERNEL8x2_4
+
+	prefetcht0      B_PR1(BO,BI,8)
+	KERNEL8x2_1
+	KERNEL8x2_2
+	KERNEL8x2_3
+	KERNEL8x2_4
+
+	je	.L2_20_6
+
+	jmp	.L2_20_2
+	ALIGN_4
+
+.L2_20_6:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_20_9
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_20_7:
+
+	KERNEL8x2_SUB
+
+	jl	.L2_20_7
+	ALIGN_4
+
+
+.L2_20_9:
+
+	SAVE8x2
+
+	addq	$8 * SIZE, CO1		# coffset += 8
+	ALIGN_4
+	
+
+
+/**************************************************************************/
+
+.L2_21pre:
+
+	testq	$4, M		
+	jz	.L2_30
+	ALIGN_4
+
+.L2_21:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L2_26
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 1 ; number of values
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_22:
+
+	prefetcht0      B_PR1(BO,BI,8)
+	KERNEL4x2_1
+	KERNEL4x2_2
+	KERNEL4x2_3
+	KERNEL4x2_4
+
+	prefetcht0      B_PR1(BO,BI,8)
+	KERNEL4x2_1
+	KERNEL4x2_2
+	KERNEL4x2_3
+	KERNEL4x2_4
+
+	je	.L2_26
+
+	prefetcht0      B_PR1(BO,BI,8)
+	KERNEL4x2_1
+	KERNEL4x2_2
+	KERNEL4x2_3
+	KERNEL4x2_4
+
+	prefetcht0      B_PR1(BO,BI,8)
+	KERNEL4x2_1
+	KERNEL4x2_2
+	KERNEL4x2_3
+	KERNEL4x2_4
+
+	je	.L2_26
+
+	jmp	.L2_22
+	ALIGN_4
+
+.L2_26:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_29
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_27:
+
+	KERNEL4x2_SUB
+
+	jl	.L2_27
+	ALIGN_4
+
+
+.L2_29:
+
+	SAVE4x2
+
+	addq	$4 * SIZE, CO1		# coffset += 4
+	ALIGN_4
+	
+
+.L2_30:
+	testq	$2, M		
+	jz	.L2_40
+
+	ALIGN_4
+
+.L2_31:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L2_36
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_32:
+
+	KERNEL2x2_1
+	KERNEL2x2_2
+	KERNEL2x2_3
+	KERNEL2x2_4
+
+	KERNEL2x2_1
+	KERNEL2x2_2
+	KERNEL2x2_3
+	KERNEL2x2_4
+
+	je	.L2_36
+
+	KERNEL2x2_1
+	KERNEL2x2_2
+	KERNEL2x2_3
+	KERNEL2x2_4
+
+	KERNEL2x2_1
+	KERNEL2x2_2
+	KERNEL2x2_3
+	KERNEL2x2_4
+
+	je	.L2_36
+
+	jmp	.L2_32
+	ALIGN_4
+
+.L2_36:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_39
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+	
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_37:
+
+	KERNEL2x2_SUB
+
+	jl	.L2_37
+	ALIGN_4
+
+
+.L2_39:
+
+	SAVE2x2
+
+	addq	$2 * SIZE, CO1		# coffset += 2
+	ALIGN_4
+
+.L2_40:
+	testq	$1, M		
+	jz	.L2_60		// to next 2 lines of N
+
+	ALIGN_4
+
+.L2_41:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L2_46
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_42:
+
+	KERNEL1x2_1
+	KERNEL1x2_2
+	KERNEL1x2_3
+	KERNEL1x2_4
+
+	KERNEL1x2_1
+	KERNEL1x2_2
+	KERNEL1x2_3
+	KERNEL1x2_4
+
+	je	.L2_46
+
+	KERNEL1x2_1
+	KERNEL1x2_2
+	KERNEL1x2_3
+	KERNEL1x2_4
+
+	KERNEL1x2_1
+	KERNEL1x2_2
+	KERNEL1x2_3
+	KERNEL1x2_4
+
+	je	.L2_46
+
+	jmp	.L2_42
+	ALIGN_4
+
+.L2_46:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_49
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_47:
+
+	KERNEL1x2_SUB
+
+	jl	.L2_47
+	ALIGN_4
+
+
+.L2_49:
+
+	SAVE1x2
+
+	addq	$1 * SIZE, CO1		# coffset += 1
+	ALIGN_4
+	
+.L2_60:
+
+	decq	J			// j --
+	jg	.L2_01			// next 2 lines of N
+
+
+
+.L1_0:
+
+/************************************************************************************************
+* Loop for Nmod6 % 2 > 0
+*************************************************************************************************/
+
+	movq	Nmod6, J		
+	andq	$1, J			// j % 2
+	je	.L999
+	ALIGN_4
+
+.L1_01:
+	// copy to sub buffer
+	movq	B, BO1
+	leaq    BUFFER1, BO		// first buffer to BO
+	movq	K, %rax
+	ALIGN_4
+
+.L1_02b:
+
+	vmovsd	(BO1), %xmm0
+	vmovsd	%xmm0,       (BO)
+	addq	$1*SIZE,BO1
+	addq	$1*SIZE,BO
+	decq	%rax
+	jnz	.L1_02b
+
+.L1_02c:
+
+	movq	BO1, B			// next offset of B
+
+.L1_10:
+	movq	C, CO1
+	leaq	(C, LDC, 1), C		// c += 1 * ldc
+
+	
+	movq	A, AO		 	// aoffset = a
+	addq	$32 * SIZE, AO
+
+	movq	M,  I
+	sarq	$4, I			// i = (m >> 4)
+	je	.L1_20
+
+	ALIGN_4
+
+.L1_11:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax			//  K = K - ( K % 8 )
+	je	.L1_16
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$4, %rax			// rax = rax * 16 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_12:
+
+	prefetcht0      B_PR1(BO,BI,8)
+	KERNEL16x1_1
+	KERNEL16x1_2
+	KERNEL16x1_3
+	KERNEL16x1_4
+
+	KERNEL16x1_1
+	KERNEL16x1_2
+	KERNEL16x1_3
+	KERNEL16x1_4
+
+	je	.L1_16
+
+	prefetcht0      B_PR1(BO,BI,8)
+	KERNEL16x1_1
+	KERNEL16x1_2
+	KERNEL16x1_3
+	KERNEL16x1_4
+
+	KERNEL16x1_1
+	KERNEL16x1_2
+	KERNEL16x1_3
+	KERNEL16x1_4
+
+	je	.L1_16
+
+	jmp	.L1_12
+	ALIGN_4
+
+.L1_16:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_19
+
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$4, %rax			// rax = rax * 16 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_17:
+
+	KERNEL16x1_SUB
+
+	jl	.L1_17
+	ALIGN_4
+
+
+.L1_19:
+
+	SAVE16x1
+
+	addq	$16 * SIZE, CO1		# coffset += 16
+	decq	I			# i --
+	jg	.L1_11
+	ALIGN_4	
+
+/**************************************************************************
+* Rest of M 
+***************************************************************************/
+.L1_20:
+	// Test rest of M
+
+	testq	$15, M
+	jz	.L999
+
+	testq	$8, M		
+	jz	.L1_21pre
+	ALIGN_4
+
+/**************************************************************************/
+
+.L1_20_1:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L1_20_6
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_20_2:
+
+	prefetcht0      B_PR1(BO,BI,8)
+	KERNEL8x1_1
+	KERNEL8x1_2
+	KERNEL8x1_3
+	KERNEL8x1_4
+
+	KERNEL8x1_1
+	KERNEL8x1_2
+	KERNEL8x1_3
+	KERNEL8x1_4
+
+	je	.L1_20_6
+
+	prefetcht0      B_PR1(BO,BI,8)
+	KERNEL8x1_1
+	KERNEL8x1_2
+	KERNEL8x1_3
+	KERNEL8x1_4
+
+	KERNEL8x1_1
+	KERNEL8x1_2
+	KERNEL8x1_3
+	KERNEL8x1_4
+
+	je	.L1_20_6
+
+	jmp	.L1_20_2
+	ALIGN_4
+
+.L1_20_6:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_20_9
+
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_20_7:
+
+	KERNEL8x1_SUB
+
+	jl	.L1_20_7
+	ALIGN_4
+
+
+.L1_20_9:
+
+	SAVE8x1
+
+	addq	$8 * SIZE, CO1		# coffset += 8
+	ALIGN_4
+	
+
+
+/**************************************************************************/
+
+.L1_21pre:
+
+	testq	$4, M		
+	jz	.L1_30
+	ALIGN_4
+
+.L1_21:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L1_26
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_22:
+
+	prefetcht0      B_PR1(BO,BI,8)
+	KERNEL4x1_1
+	KERNEL4x1_2
+	KERNEL4x1_3
+	KERNEL4x1_4
+
+	KERNEL4x1_1
+	KERNEL4x1_2
+	KERNEL4x1_3
+	KERNEL4x1_4
+
+	je	.L1_26
+
+	prefetcht0      B_PR1(BO,BI,8)
+	KERNEL4x1_1
+	KERNEL4x1_2
+	KERNEL4x1_3
+	KERNEL4x1_4
+
+	KERNEL4x1_1
+	KERNEL4x1_2
+	KERNEL4x1_3
+	KERNEL4x1_4
+
+	je	.L1_26
+
+	jmp	.L1_22
+	ALIGN_4
+
+.L1_26:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_29
+
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_27:
+
+	KERNEL4x1_SUB
+
+	jl	.L1_27
+	ALIGN_4
+
+
+.L1_29:
+
+	SAVE4x1
+
+	addq	$4 * SIZE, CO1		# coffset += 4
+	ALIGN_4
+	
+
+.L1_30:
+	testq	$2, M		
+	jz	.L1_40
+
+	ALIGN_4
+
+.L1_31:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L1_36
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_32:
+
+	KERNEL2x1_1
+	KERNEL2x1_2
+	KERNEL2x1_3
+	KERNEL2x1_4
+
+	KERNEL2x1_1
+	KERNEL2x1_2
+	KERNEL2x1_3
+	KERNEL2x1_4
+
+	je	.L1_36
+
+	KERNEL2x1_1
+	KERNEL2x1_2
+	KERNEL2x1_3
+	KERNEL2x1_4
+
+	KERNEL2x1_1
+	KERNEL2x1_2
+	KERNEL2x1_3
+	KERNEL2x1_4
+
+	je	.L1_36
+
+	jmp	.L1_32
+	ALIGN_4
+
+.L1_36:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_39
+
+	movq    %rax, BI                        //  Index for BO
+	
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_37:
+
+	KERNEL2x1_SUB
+
+	jl	.L1_37
+	ALIGN_4
+
+
+.L1_39:
+
+	SAVE2x1
+
+	addq	$2 * SIZE, CO1		# coffset += 2
+	ALIGN_4
+
+.L1_40:
+	testq	$1, M		
+	jz	.L999
+
+	ALIGN_4
+
+.L1_41:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L1_46
+	movq    %rax, BI                        //  Index for BO
+
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_42:
+
+	KERNEL1x1_1
+	KERNEL1x1_2
+	KERNEL1x1_3
+	KERNEL1x1_4
+
+	KERNEL1x1_1
+	KERNEL1x1_2
+	KERNEL1x1_3
+	KERNEL1x1_4
+
+	je	.L1_46
+
+	KERNEL1x1_1
+	KERNEL1x1_2
+	KERNEL1x1_3
+	KERNEL1x1_4
+
+	KERNEL1x1_1
+	KERNEL1x1_2
+	KERNEL1x1_3
+	KERNEL1x1_4
+
+	je	.L1_46
+
+	jmp	.L1_42
+	ALIGN_4
+
+.L1_46:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_49
+
+	movq    %rax, BI                        //  Index for BO
+
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_47:
+
+	KERNEL1x1_SUB
+
+	jl	.L1_47
+	ALIGN_4
+
+
+.L1_49:
+
+	SAVE1x1
+
+	addq	$1 * SIZE, CO1		# coffset += 1
+	ALIGN_4
+	
+
+.L999:
+	movq   		SP, %rsp
+	movq	   (%rsp), %rbx
+	movq	  8(%rsp), %rbp
+	movq	 16(%rsp), %r12
+	movq	 24(%rsp), %r13
+	movq	 32(%rsp), %r14
+	movq	 40(%rsp), %r15
+
+#ifdef WINDOWS_ABI
+	movq	 48(%rsp), %rdi
+	movq	 56(%rsp), %rsi
+	movups	 64(%rsp), %xmm6
+	movups	 80(%rsp), %xmm7
+	movups	 96(%rsp), %xmm8
+	movups	112(%rsp), %xmm9
+	movups	128(%rsp), %xmm10
+	movups	144(%rsp), %xmm11
+	movups	160(%rsp), %xmm12
+	movups	176(%rsp), %xmm13
+	movups	192(%rsp), %xmm14
+	movups	208(%rsp), %xmm15
+#endif
+
+	addq	$STACKSIZE, %rsp
+	ret
+
+	EPILOGUE
+
+
+#else
+/*************************************************************************************
+* TRMM Kernel
+*************************************************************************************/
+
+
+	PROLOGUE
+	PROFCODE
+	
+	subq	$STACKSIZE, %rsp
+	movq	%rbx,   (%rsp)
+	movq	%rbp,  8(%rsp)
+	movq	%r12, 16(%rsp)
+	movq	%r13, 24(%rsp)
+	movq	%r14, 32(%rsp)
+	movq	%r15, 40(%rsp)
+
+	vzeroupper
+
+#ifdef WINDOWS_ABI
+	movq	%rdi,    48(%rsp)
+	movq	%rsi,    56(%rsp)
+	movups	%xmm6,   64(%rsp)
+	movups	%xmm7,   80(%rsp)
+	movups	%xmm8,   96(%rsp)
+	movups	%xmm9,  112(%rsp)
+	movups	%xmm10, 128(%rsp)
+	movups	%xmm11, 144(%rsp)
+	movups	%xmm12, 160(%rsp)
+	movups	%xmm13, 176(%rsp)
+	movups	%xmm14, 192(%rsp)
+	movups	%xmm15, 208(%rsp)
+
+	movq	ARG1,      OLD_M
+	movq	ARG2,      OLD_N
+	movq	ARG3,      OLD_K
+	movq	OLD_A,     A
+	movq	OLD_B,     B
+	movq	OLD_C,     C
+	movq	OLD_LDC,   LDC
+#ifdef TRMMKERNEL
+	movsd	OLD_OFFSET, %xmm12
+#endif
+	vmovaps	%xmm3, %xmm0
+
+#else
+	movq	STACKSIZE +  8(%rsp), LDC
+#ifdef TRMMKERNEL
+	movsd	STACKSIZE + 16(%rsp), %xmm12
+#endif
+
+#endif
+
+	movq    %rsp, SP      # save old stack
+        subq    $128 + L_BUFFER_SIZE, %rsp
+        andq    $-4096, %rsp    # align stack
+
+        STACK_TOUCH
+
+	cmpq	$0, OLD_M
+	je	.L999
+
+	cmpq	$0, OLD_N
+	je	.L999
+
+	cmpq	$0, OLD_K
+	je	.L999
+
+	movq	OLD_M, M
+	movq	OLD_N, N
+	movq	OLD_K, K
+
+	vmovsd	 %xmm0, ALPHA
+
+	salq	$BASE_SHIFT, LDC
+
+	movq    N, %rax
+        xorq    %rdx, %rdx
+        movq    $2,  %rdi
+        divq    %rdi                    //    N / 6
+        movq    %rax, Ndiv6             //    N / 6
+        movq    %rdx, Nmod6             //    N % 6
+
+	
+
+#ifdef TRMMKERNEL
+	vmovsd	%xmm12, OFFSET
+	vmovsd	%xmm12, KK
+#ifndef LEFT
+	negq	KK
+#endif	
+#endif
+
+	movq	Ndiv6,  J
+	cmpq	$0, J
+	je	.L1_0
+	ALIGN_4
+
+.L2_01:
+	// copy to sub buffer
+	movq	B, BO1
+	leaq    BUFFER1, BO		// first buffer to BO
+	movq	K, %rax
+	sarq	$2, %rax		// K / 4
+	jz	.L2_01b
+	ALIGN_4
+
+.L2_01a:
+        prefetcht0 512(BO1)
+        prefetchw  512(BO)
+
+	vmovups	      (BO1), %xmm0
+	vmovups	2*SIZE(BO1), %xmm1
+	vmovups	4*SIZE(BO1), %xmm2
+	vmovups	6*SIZE(BO1), %xmm3
+
+	vmovups	%xmm0,       (BO)
+	vmovups	%xmm1, 2*SIZE(BO)
+	vmovups	%xmm2, 4*SIZE(BO)
+	vmovups	%xmm3, 6*SIZE(BO)
+
+	addq	$8*SIZE,BO1
+	addq	$8*SIZE,BO
+	decq	%rax
+	jnz	.L2_01a
+
+
+.L2_01b:
+
+        movq    K, %rax
+        andq    $3, %rax                // K % 4
+        jz      .L2_02d
+        ALIGN_4
+
+.L2_02c:
+
+	vmovups	(BO1), %xmm0
+	vmovups	%xmm0, (BO)
+	addq	$2*SIZE,BO1
+	addq	$2*SIZE,BO
+	decq	%rax
+	jnz	.L2_02c
+
+.L2_02d:
+
+	movq	BO1, B			// next offset of B
+
+.L2_10:
+	movq	C, CO1
+	leaq	(C, LDC, 2), C		// c += 2 * ldc
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        movq    OFFSET, %rax
+        movq    %rax, KK
+#endif
+	
+	movq	A, AO		 	// aoffset = a
+	addq	$32 * SIZE, AO
+
+	movq	M,  I
+	sarq	$4, I			// i = (m >> 4)
+	je	.L2_20
+
+	ALIGN_4
+
+.L2_11:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO
+        salq    $4, %rax                        // rax = rax * 16 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $16, %rax	// number of values in AO
+#else
+        addq    $2, %rax	// number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+	andq	$-8, %rax			//  K = K - ( K % 8 )
+	je	.L2_16
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$4, %rax			// rax = rax * 16 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_12:
+
+	prefetcht0      B_PR1(BO,BI,8)
+	KERNEL16x2_1
+	KERNEL16x2_2
+	KERNEL16x2_3
+	KERNEL16x2_4
+
+	prefetcht0      B_PR1(BO,BI,8)
+	KERNEL16x2_1
+	KERNEL16x2_2
+	KERNEL16x2_3
+	KERNEL16x2_4
+
+	je	.L2_16
+
+	prefetcht0      B_PR1(BO,BI,8)
+	KERNEL16x2_1
+	KERNEL16x2_2
+	KERNEL16x2_3
+	KERNEL16x2_4
+
+	prefetcht0      B_PR1(BO,BI,8)
+	KERNEL16x2_1
+	KERNEL16x2_2
+	KERNEL16x2_3
+	KERNEL16x2_4
+
+	je	.L2_16
+
+	jmp	.L2_12
+	ALIGN_4
+
+.L2_16:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_19
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$4, %rax			// rax = rax * 16 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_17:
+
+	KERNEL16x2_SUB
+
+	jl	.L2_17
+	ALIGN_4
+
+
+.L2_19:
+
+	SAVE16x2
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO
+        salq    $4, %rax                        // rax = rax * 16 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $16, KK				
+#endif
+
+	addq	$16 * SIZE, CO1		# coffset += 16
+	decq	I			# i --
+	jg	.L2_11
+	ALIGN_4	
+
+/**************************************************************************
+* Rest of M 
+***************************************************************************/
+.L2_20:
+	// Test rest of M
+
+	testq	$15, M
+	jz	.L2_60		// to next 3 lines of N
+
+	testq	$8, M		
+	jz	.L2_21pre
+	ALIGN_4
+
+/**************************************************************************/
+
+.L2_20_1:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO
+        salq    $3, %rax                        // rax = rax * 8 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $8, %rax        // number of values in A
+#else
+        addq    $2, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$-8, %rax
+	je	.L2_20_6
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_20_2:
+
+	prefetcht0      B_PR1(BO,BI,8)
+	KERNEL8x2_1
+	KERNEL8x2_2
+	KERNEL8x2_3
+	KERNEL8x2_4
+
+	prefetcht0      B_PR1(BO,BI,8)
+	KERNEL8x2_1
+	KERNEL8x2_2
+	KERNEL8x2_3
+	KERNEL8x2_4
+
+	je	.L2_20_6
+
+	prefetcht0      B_PR1(BO,BI,8)
+	KERNEL8x2_1
+	KERNEL8x2_2
+	KERNEL8x2_3
+	KERNEL8x2_4
+
+	prefetcht0      B_PR1(BO,BI,8)
+	KERNEL8x2_1
+	KERNEL8x2_2
+	KERNEL8x2_3
+	KERNEL8x2_4
+
+	je	.L2_20_6
+
+	jmp	.L2_20_2
+	ALIGN_4
+
+.L2_20_6:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_20_9
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_20_7:
+
+	KERNEL8x2_SUB
+
+	jl	.L2_20_7
+	ALIGN_4
+
+
+.L2_20_9:
+
+	SAVE8x2
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax 
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO         
+        salq    $3, %rax                        // rax = rax * 8 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif  
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $8, KK
+#endif
+
+	addq	$8 * SIZE, CO1		# coffset += 8
+	ALIGN_4
+	
+
+
+/**************************************************************************/
+
+.L2_21pre:
+
+	testq	$4, M		
+	jz	.L2_30
+	ALIGN_4
+
+.L2_21:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO
+        salq    $2, %rax                        // rax = rax * 4 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $4, %rax        // number of values in A
+#else
+        addq    $2, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$-8, %rax
+	je	.L2_26
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 1 ; number of values
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_22:
+
+	prefetcht0      B_PR1(BO,BI,8)
+	KERNEL4x2_1
+	KERNEL4x2_2
+	KERNEL4x2_3
+	KERNEL4x2_4
+
+	prefetcht0      B_PR1(BO,BI,8)
+	KERNEL4x2_1
+	KERNEL4x2_2
+	KERNEL4x2_3
+	KERNEL4x2_4
+
+	je	.L2_26
+
+	prefetcht0      B_PR1(BO,BI,8)
+	KERNEL4x2_1
+	KERNEL4x2_2
+	KERNEL4x2_3
+	KERNEL4x2_4
+
+	prefetcht0      B_PR1(BO,BI,8)
+	KERNEL4x2_1
+	KERNEL4x2_2
+	KERNEL4x2_3
+	KERNEL4x2_4
+
+	je	.L2_26
+
+	jmp	.L2_22
+	ALIGN_4
+
+.L2_26:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_29
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_27:
+
+	KERNEL4x2_SUB
+
+	jl	.L2_27
+	ALIGN_4
+
+
+.L2_29:
+
+	SAVE4x2
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax 
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO         
+        salq    $2, %rax                        // rax = rax * 4 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif  
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $4, KK
+#endif
+
+	addq	$4 * SIZE, CO1		# coffset += 4
+	ALIGN_4
+	
+
+.L2_30:
+	testq	$2, M		
+	jz	.L2_40
+
+	ALIGN_4
+
+.L2_31:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO
+        salq    $1, %rax                        // rax = rax * 2 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $2, %rax        // number of values in AO
+#else
+        addq    $2, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$-8, %rax
+	je	.L2_36
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_32:
+
+	KERNEL2x2_1
+	KERNEL2x2_2
+	KERNEL2x2_3
+	KERNEL2x2_4
+
+	KERNEL2x2_1
+	KERNEL2x2_2
+	KERNEL2x2_3
+	KERNEL2x2_4
+
+	je	.L2_36
+
+	KERNEL2x2_1
+	KERNEL2x2_2
+	KERNEL2x2_3
+	KERNEL2x2_4
+
+	KERNEL2x2_1
+	KERNEL2x2_2
+	KERNEL2x2_3
+	KERNEL2x2_4
+
+	je	.L2_36
+
+	jmp	.L2_32
+	ALIGN_4
+
+.L2_36:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_39
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+	
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_37:
+
+	KERNEL2x2_SUB
+
+	jl	.L2_37
+	ALIGN_4
+
+
+.L2_39:
+
+	SAVE2x2
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax 
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO         
+        salq    $1, %rax                        // rax = rax * 2 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif  
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $2, KK
+#endif
+
+	addq	$2 * SIZE, CO1		# coffset += 2
+	ALIGN_4
+
+.L2_40:
+	testq	$1, M		
+	jz	.L2_60		// to next 2 lines of N
+
+	ALIGN_4
+
+.L2_41:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $1, %rax        // number of values in AO
+#else
+        addq    $2, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+	andq	$-8, %rax
+	je	.L2_46
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_42:
+
+	KERNEL1x2_1
+	KERNEL1x2_2
+	KERNEL1x2_3
+	KERNEL1x2_4
+
+	KERNEL1x2_1
+	KERNEL1x2_2
+	KERNEL1x2_3
+	KERNEL1x2_4
+
+	je	.L2_46
+
+	KERNEL1x2_1
+	KERNEL1x2_2
+	KERNEL1x2_3
+	KERNEL1x2_4
+
+	KERNEL1x2_1
+	KERNEL1x2_2
+	KERNEL1x2_3
+	KERNEL1x2_4
+
+	je	.L2_46
+
+	jmp	.L2_42
+	ALIGN_4
+
+.L2_46:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_49
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_47:
+
+	KERNEL1x2_SUB
+
+	jl	.L2_47
+	ALIGN_4
+
+
+.L2_49:
+
+	SAVE1x2
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax 
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO         
+        leaq    (AO, %rax, SIZE), AO
+#endif  
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $1, KK
+#endif
+
+	addq	$1 * SIZE, CO1		# coffset += 1
+	ALIGN_4
+	
+
+
+
+	
+.L2_60:
+#if defined(TRMMKERNEL) && !defined(LEFT)
+        addq    $2, KK
+#endif
+
+	decq	J			// j --
+	jg	.L2_01			// next 2 lines of N
+
+
+
+.L1_0:
+
+/************************************************************************************************
+* Loop for Nmod6 % 2 > 0
+*************************************************************************************************/
+
+	movq	Nmod6, J		
+	andq	$1, J			// j % 2
+	je	.L999
+	ALIGN_4
+
+.L1_01:
+	// copy to sub buffer
+	movq	B, BO1
+	leaq    BUFFER1, BO		// first buffer to BO
+	movq	K, %rax
+	ALIGN_4
+
+.L1_02b:
+
+	vmovsd	(BO1), %xmm0
+	vmovsd	%xmm0,       (BO)
+	addq	$1*SIZE,BO1
+	addq	$1*SIZE,BO
+	decq	%rax
+	jnz	.L1_02b
+
+.L1_02c:
+
+	movq	BO1, B			// next offset of B
+
+.L1_10:
+	movq	C, CO1
+	leaq	(C, LDC, 1), C		// c += 1 * ldc
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        movq    OFFSET, %rax
+        movq    %rax, KK
+#endif
+	
+	movq	A, AO		 	// aoffset = a
+	addq	$32 * SIZE, AO
+
+	movq	M,  I
+	sarq	$4, I			// i = (m >> 4)
+	je	.L1_20
+
+	ALIGN_4
+
+.L1_11:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BO, BI, SIZE), BO
+        salq    $4, %rax                        // rax = rax * 16 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $16, %rax	// number of values in AO
+#else
+        addq    $1, %rax	// number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+	andq	$-8, %rax			//  K = K - ( K % 8 )
+	je	.L1_16
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$4, %rax			// rax = rax * 16 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_12:
+
+	prefetcht0      B_PR1(BO,BI,8)
+	KERNEL16x1_1
+	KERNEL16x1_2
+	KERNEL16x1_3
+	KERNEL16x1_4
+
+	KERNEL16x1_1
+	KERNEL16x1_2
+	KERNEL16x1_3
+	KERNEL16x1_4
+
+	je	.L1_16
+
+	prefetcht0      B_PR1(BO,BI,8)
+	KERNEL16x1_1
+	KERNEL16x1_2
+	KERNEL16x1_3
+	KERNEL16x1_4
+
+	KERNEL16x1_1
+	KERNEL16x1_2
+	KERNEL16x1_3
+	KERNEL16x1_4
+
+	je	.L1_16
+
+	jmp	.L1_12
+	ALIGN_4
+
+.L1_16:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_19
+
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$4, %rax			// rax = rax * 16 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_17:
+
+	KERNEL16x1_SUB
+
+	jl	.L1_17
+	ALIGN_4
+
+
+.L1_19:
+
+	SAVE16x1
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BO, BI, SIZE), BO
+        salq    $4, %rax                        // rax = rax * 16 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $16, KK				
+#endif
+
+	addq	$16 * SIZE, CO1		# coffset += 16
+	decq	I			# i --
+	jg	.L1_11
+	ALIGN_4	
+
+/**************************************************************************
+* Rest of M 
+***************************************************************************/
+.L1_20:
+	// Test rest of M
+
+	testq	$15, M
+	jz	.L999
+
+	testq	$8, M		
+	jz	.L1_21pre
+	ALIGN_4
+
+/**************************************************************************/
+
+.L1_20_1:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BO, BI, SIZE), BO
+        salq    $3, %rax                        // rax = rax * 8 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $8, %rax        // number of values in A
+#else
+        addq    $1, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$-8, %rax
+	je	.L1_20_6
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_20_2:
+
+	prefetcht0      B_PR1(BO,BI,8)
+	KERNEL8x1_1
+	KERNEL8x1_2
+	KERNEL8x1_3
+	KERNEL8x1_4
+
+	KERNEL8x1_1
+	KERNEL8x1_2
+	KERNEL8x1_3
+	KERNEL8x1_4
+
+	je	.L1_20_6
+
+	prefetcht0      B_PR1(BO,BI,8)
+	KERNEL8x1_1
+	KERNEL8x1_2
+	KERNEL8x1_3
+	KERNEL8x1_4
+
+	KERNEL8x1_1
+	KERNEL8x1_2
+	KERNEL8x1_3
+	KERNEL8x1_4
+
+	je	.L1_20_6
+
+	jmp	.L1_20_2
+	ALIGN_4
+
+.L1_20_6:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_20_9
+
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_20_7:
+
+	KERNEL8x1_SUB
+
+	jl	.L1_20_7
+	ALIGN_4
+
+
+.L1_20_9:
+
+	SAVE8x1
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax 
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BO, BI, SIZE), BO         
+        salq    $3, %rax                        // rax = rax * 8 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif  
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $8, KK
+#endif
+
+	addq	$8 * SIZE, CO1		# coffset += 8
+	ALIGN_4
+	
+
+
+/**************************************************************************/
+
+.L1_21pre:
+
+	testq	$4, M		
+	jz	.L1_30
+	ALIGN_4
+
+.L1_21:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BO, BI, SIZE), BO
+        salq    $2, %rax                        // rax = rax * 4 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $4, %rax        // number of values in A
+#else
+        addq    $1, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$-8, %rax
+	je	.L1_26
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_22:
+
+	prefetcht0      B_PR1(BO,BI,8)
+	KERNEL4x1_1
+	KERNEL4x1_2
+	KERNEL4x1_3
+	KERNEL4x1_4
+
+	KERNEL4x1_1
+	KERNEL4x1_2
+	KERNEL4x1_3
+	KERNEL4x1_4
+
+	je	.L1_26
+
+	prefetcht0      B_PR1(BO,BI,8)
+	KERNEL4x1_1
+	KERNEL4x1_2
+	KERNEL4x1_3
+	KERNEL4x1_4
+
+	KERNEL4x1_1
+	KERNEL4x1_2
+	KERNEL4x1_3
+	KERNEL4x1_4
+
+	je	.L1_26
+
+	jmp	.L1_22
+	ALIGN_4
+
+.L1_26:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_29
+
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_27:
+
+	KERNEL4x1_SUB
+
+	jl	.L1_27
+	ALIGN_4
+
+
+.L1_29:
+
+	SAVE4x1
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax 
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BO, BI, SIZE), BO         
+        salq    $2, %rax                        // rax = rax * 4 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif  
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $4, KK
+#endif
+
+	addq	$4 * SIZE, CO1		# coffset += 4
+	ALIGN_4
+	
+
+.L1_30:
+	testq	$2, M		
+	jz	.L1_40
+
+	ALIGN_4
+
+.L1_31:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BO, BI, SIZE), BO
+        salq    $1, %rax                        // rax = rax * 2 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $2, %rax        // number of values in AO
+#else
+        addq    $1, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$-8, %rax
+	je	.L1_36
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_32:
+
+	KERNEL2x1_1
+	KERNEL2x1_2
+	KERNEL2x1_3
+	KERNEL2x1_4
+
+	KERNEL2x1_1
+	KERNEL2x1_2
+	KERNEL2x1_3
+	KERNEL2x1_4
+
+	je	.L1_36
+
+	KERNEL2x1_1
+	KERNEL2x1_2
+	KERNEL2x1_3
+	KERNEL2x1_4
+
+	KERNEL2x1_1
+	KERNEL2x1_2
+	KERNEL2x1_3
+	KERNEL2x1_4
+
+	je	.L1_36
+
+	jmp	.L1_32
+	ALIGN_4
+
+.L1_36:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_39
+
+	movq    %rax, BI                        //  Index for BO
+	
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_37:
+
+	KERNEL2x1_SUB
+
+	jl	.L1_37
+	ALIGN_4
+
+
+.L1_39:
+
+	SAVE2x1
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax 
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BO, BI, SIZE), BO         
+        salq    $1, %rax                        // rax = rax * 2 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif  
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $2, KK
+#endif
+
+	addq	$2 * SIZE, CO1		# coffset += 2
+	ALIGN_4
+
+.L1_40:
+	testq	$1, M		
+	jz	.L999
+
+	ALIGN_4
+
+.L1_41:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BO, BI, SIZE), BO
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $1, %rax        // number of values in AO
+#else
+        addq    $1, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+	andq	$-8, %rax
+	je	.L1_46
+	movq    %rax, BI                        //  Index for BO
+
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_42:
+
+	KERNEL1x1_1
+	KERNEL1x1_2
+	KERNEL1x1_3
+	KERNEL1x1_4
+
+	KERNEL1x1_1
+	KERNEL1x1_2
+	KERNEL1x1_3
+	KERNEL1x1_4
+
+	je	.L1_46
+
+	KERNEL1x1_1
+	KERNEL1x1_2
+	KERNEL1x1_3
+	KERNEL1x1_4
+
+	KERNEL1x1_1
+	KERNEL1x1_2
+	KERNEL1x1_3
+	KERNEL1x1_4
+
+	je	.L1_46
+
+	jmp	.L1_42
+	ALIGN_4
+
+.L1_46:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_49
+
+	movq    %rax, BI                        //  Index for BO
+
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_47:
+
+	KERNEL1x1_SUB
+
+	jl	.L1_47
+	ALIGN_4
+
+
+.L1_49:
+
+	SAVE1x1
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax 
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BO, BI, SIZE), BO         
+        leaq    (AO, %rax, SIZE), AO
+#endif  
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $1, KK
+#endif
+
+	addq	$1 * SIZE, CO1		# coffset += 1
+	ALIGN_4
+	
+
+.L999:
+	movq   		SP, %rsp
+	movq	   (%rsp), %rbx
+	movq	  8(%rsp), %rbp
+	movq	 16(%rsp), %r12
+	movq	 24(%rsp), %r13
+	movq	 32(%rsp), %r14
+	movq	 40(%rsp), %r15
+
+#ifdef WINDOWS_ABI
+	movq	 48(%rsp), %rdi
+	movq	 56(%rsp), %rsi
+	movups	 64(%rsp), %xmm6
+	movups	 80(%rsp), %xmm7
+	movups	 96(%rsp), %xmm8
+	movups	112(%rsp), %xmm9
+	movups	128(%rsp), %xmm10
+	movups	144(%rsp), %xmm11
+	movups	160(%rsp), %xmm12
+	movups	176(%rsp), %xmm13
+	movups	192(%rsp), %xmm14
+	movups	208(%rsp), %xmm15
+#endif
+
+	addq	$STACKSIZE, %rsp
+	ret
+
+	EPILOGUE
+
+
+
+
+
+#endif
diff --git a/kernel/x86_64/dgemm_kernel_4x4_haswell.S b/kernel/x86_64/dgemm_kernel_4x4_haswell.S
index 0a2ca7ae3..29501df8e 100644
--- a/kernel/x86_64/dgemm_kernel_4x4_haswell.S
+++ b/kernel/x86_64/dgemm_kernel_4x4_haswell.S
@@ -1,3494 +1,3494 @@
-/*********************************************************************************
-Copyright (c) 2013, The OpenBLAS Project
-All rights reserved.
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in
-the documentation and/or other materials provided with the
-distribution.
-3. Neither the name of the OpenBLAS project nor the names of
-its contributors may be used to endorse or promote products
-derived from this software without specific prior written permission.
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
-USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-**********************************************************************************/
-
-
-/*********************************************************************
-* 2013/10/28 Saar
-*        BLASTEST               : OK
-*        CTEST                  : OK
-*        TEST                   : OK
-
-*
-*
-* 2013/10/27 Saar
-* Parameter:
-*       DGEMM_DEFAULT_UNROLL_N  4
-*       DGEMM_DEFAULT_UNROLL_M  4
-*       DGEMM_DEFAULT_P         512
-*       DGEMM_DEFAULT_Q         256
-*	A_PR1			512
-*	B_PR1			512
-*
-*
-* Performance at 9216x9216x9216:
-*       1 thread:       53.3 GFLOPS (MKL:  54)
-*       2 threads:     100.0 GFLOPS (MKL:  97)
-*       3 threads:     147.0 GFLOPS (MKL: 133)
-*       4 threads:     184.0 GFLOPS (MKL: 170)
-*********************************************************************/
-
-
-#define ASSEMBLER
-#include "common.h"
- 
-#define OLD_M	%rdi
-#define OLD_N	%rsi
-#define M	%r13
-#define J	%r14
-#define OLD_K	%rdx
-
-#define A	%rcx
-#define B	%r8
-#define C	%r9
-#define LDC	%r10
-	
-#define I	%r11
-#define AO	%rdi
-#define BO	%rsi
-#define	CO1	%r15
-#define K	%r12
-#define	SP	%rbx
-
-#define BO1	%rdi
-#define BO2	%r15
-#define BO3	%rbp
-
-#ifndef WINDOWS_ABI
-
-#define STACKSIZE 96
-#define L_BUFFER_SIZE 256*8*12+4096
-
-#else
-
-#define STACKSIZE 256
-#define L_BUFFER_SIZE 128*8*12+512
-
-#define OLD_A		40 + STACKSIZE(%rsp)
-#define OLD_B		48 + STACKSIZE(%rsp)
-#define OLD_C		56 + STACKSIZE(%rsp)
-#define OLD_LDC		64 + STACKSIZE(%rsp)
-#define OLD_OFFSET	72 + STACKSIZE(%rsp)
-
-#endif
-
-
-#define Ndiv12	 24(%rsp)
-#define Nmod12	 32(%rsp)
-#define N	 40(%rsp)
-#define ALPHA	 48(%rsp)
-#define OFFSET	 56(%rsp)
-#define KK	 64(%rsp)
-#define KKK	 72(%rsp)
-#define BUFFER1	           128(%rsp)
-
-#if defined(OS_WINDOWS)
-#if   L_BUFFER_SIZE > 16384
-#define STACK_TOUCH \
-        movl    $ 0,  4096 * 4(%rsp);\
-        movl    $ 0,  4096 * 3(%rsp);\
-        movl    $ 0,  4096 * 2(%rsp);\
-        movl    $ 0,  4096 * 1(%rsp);
-#elif L_BUFFER_SIZE > 12288
-#define STACK_TOUCH \
-        movl    $ 0,  4096 * 3(%rsp);\
-        movl    $ 0,  4096 * 2(%rsp);\
-        movl    $ 0,  4096 * 1(%rsp);
-#elif L_BUFFER_SIZE > 8192
-#define STACK_TOUCH \
-        movl    $ 0,  4096 * 2(%rsp);\
-        movl    $ 0,  4096 * 1(%rsp);
-#elif L_BUFFER_SIZE > 4096
-#define STACK_TOUCH \
-        movl    $ 0,  4096 * 1(%rsp);
-#else
-#define STACK_TOUCH
-#endif
-#else
-#define STACK_TOUCH
-#endif
-
-#define	A_PR1	512
-#define	B_PR1	512
-
-/*******************************************************************************************
-* Macro definitions
-*******************************************************************************************/
-
-.macro INIT4x12
-
-	vxorpd		%ymm4 , %ymm4 , %ymm4
-	vxorpd		%ymm5 , %ymm5 , %ymm5
-	vxorpd		%ymm6 , %ymm6 , %ymm6
-	vxorpd		%ymm7 , %ymm7 , %ymm7
-	vxorpd		%ymm8 , %ymm8 , %ymm8
-	vxorpd		%ymm9 , %ymm9 , %ymm9
-	vxorpd		%ymm10, %ymm10, %ymm10
-	vxorpd		%ymm11, %ymm11, %ymm11
-	vxorpd		%ymm12, %ymm12, %ymm12
-	vxorpd		%ymm13, %ymm13, %ymm13
-	vxorpd		%ymm14, %ymm14, %ymm14
-	vxorpd		%ymm15, %ymm15, %ymm15
-
-.endm
-
-.macro KERNEL4x12_I
-	prefetcht0	A_PR1(AO)
-	vmovups		-12 * SIZE(BO), %ymm1
-	prefetcht0	B_PR1(BO)
-	vmovups 	-16 * SIZE(AO), %ymm0
-	prefetcht0	B_PR1+64(BO)
-	vmovups		 -8 * SIZE(BO), %ymm2
-	prefetcht0	B_PR1+128(BO)
-	vmovups		 -4 * SIZE(BO), %ymm3
-	vmulpd  	%ymm0 ,%ymm1  , %ymm4
-	prefetcht0	B_PR1+192(BO)
-	vmulpd  	%ymm0 ,%ymm2  , %ymm8
-	vmulpd  	%ymm0 ,%ymm3  , %ymm12
-	prefetcht0	B_PR1+256(BO)
-	vpermpd		$ 0xb1, %ymm0  , %ymm0
-	vmulpd  	%ymm0 ,%ymm1  , %ymm5
-	vmulpd  	%ymm0 ,%ymm2  , %ymm9
-	vmulpd  	%ymm0 ,%ymm3  , %ymm13
-	vpermpd		$ 0x1b, %ymm0  , %ymm0
-	vmulpd  	%ymm0 ,%ymm1  , %ymm6
-	vmulpd  	%ymm0 ,%ymm2  , %ymm10
-
-	addq		$ 12*SIZE, BO
-	vmulpd  	%ymm0 ,%ymm3  , %ymm14
-	vpermpd		$ 0xb1, %ymm0  , %ymm0
-	vmulpd  	%ymm0 ,%ymm1  , %ymm7
-	vmovups		-12 * SIZE(BO), %ymm1
-	vmulpd  	%ymm0 ,%ymm2  , %ymm11
-	vmovups		 -8 * SIZE(BO), %ymm2
-	vmulpd  	%ymm0 ,%ymm3  , %ymm15
-	vmovups		 -4 * SIZE(BO), %ymm3
-
-.endm
-
-.macro KERNEL4x12_M1
-	prefetcht0	A_PR1(AO)
-	vmovups 	-16 * SIZE(AO), %ymm0
-	prefetcht0	B_PR1(BO)
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4
-	prefetcht0	B_PR1+64(BO)
-	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm8
-	prefetcht0	B_PR1+128(BO)
-	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm12
-	vpermpd		$ 0xb1, %ymm0  , %ymm0
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5
-	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm9
-	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm13
-	vpermpd		$ 0x1b, %ymm0  , %ymm0
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6
-	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm10
-
-	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm14
-	vpermpd		$ 0xb1, %ymm0  , %ymm0
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7
-	vmovups		-12 * SIZE(BO), %ymm1
-	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm11
-	vmovups		 -8 * SIZE(BO), %ymm2
-	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm15
-	vmovups		 -4 * SIZE(BO), %ymm3
-
-.endm
-
-.macro KERNEL4x12_M2
-	vmovups 	-12 * SIZE(AO), %ymm0
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4
-	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm8
-	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm12
-	vpermpd		$ 0xb1, %ymm0  , %ymm0
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5
-	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm9
-	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm13
-	vpermpd		$ 0x1b, %ymm0  , %ymm0
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6
-	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm10
-
-	addq		$ 8*SIZE, AO
-	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm14
-	vpermpd		$ 0xb1, %ymm0  , %ymm0
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7
-	vmovups		  0 * SIZE(BO), %ymm1
-	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm11
-	vmovups		  4 * SIZE(BO), %ymm2
-	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm15
-	vmovups		  8 * SIZE(BO), %ymm3
-	addq		$ 24*SIZE, BO
-.endm
-
-
-.macro KERNEL4x12_E
-	vmovups 	-12 * SIZE(AO), %ymm0
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4
-	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm8
-	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm12
-	vpermpd		$ 0xb1, %ymm0  , %ymm0
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5
-	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm9
-	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm13
-	vpermpd		$ 0x1b, %ymm0  , %ymm0
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6
-	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm10
-
-	addq		$ 8*SIZE, AO
-	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm14
-	vpermpd		$ 0xb1, %ymm0  , %ymm0
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7
-	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm11
-	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm15
-	addq		$ 12*SIZE, BO
-.endm
-
-.macro KERNEL4x12_SUB
-	vmovups		-12 * SIZE(BO), %ymm1
-	vmovups 	-16 * SIZE(AO), %ymm0
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4
-	vmovups		 -8 * SIZE(BO), %ymm2
-	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm8
-	vmovups		 -4 * SIZE(BO), %ymm3
-	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm12
-	vpermpd		$ 0xb1, %ymm0  , %ymm0
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5
-	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm9
-	addq		$ 12*SIZE, BO
-	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm13
-	vpermpd		$ 0x1b, %ymm0  , %ymm0
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6
-	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm10
-	addq		$ 4*SIZE, AO
-	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm14
-	vpermpd		$ 0xb1, %ymm0  , %ymm0
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7
-	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm11
-	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm15
-
-.endm
-
-
-.macro SAVE4x12
-
-	vbroadcastsd	ALPHA, %ymm0
-
-	vmulpd	%ymm0 , %ymm4 , %ymm4
-	vmulpd	%ymm0 , %ymm5 , %ymm5
-	vmulpd	%ymm0 , %ymm6 , %ymm6
-	vmulpd	%ymm0 , %ymm7 , %ymm7
-
-	vmulpd	%ymm0 , %ymm8 , %ymm8
-	vmulpd	%ymm0 , %ymm9 , %ymm9
-	vmulpd	%ymm0 , %ymm10, %ymm10
-	vmulpd	%ymm0 , %ymm11, %ymm11
-
-	vmulpd	%ymm0 , %ymm12, %ymm12
-	vmulpd	%ymm0 , %ymm13, %ymm13
-	vmulpd	%ymm0 , %ymm14, %ymm14
-	vmulpd	%ymm0 , %ymm15, %ymm15
-
-	vpermpd $ 0xb1 , %ymm5, %ymm5
-	vpermpd $ 0xb1 , %ymm7, %ymm7
-
-	vblendpd $ 0x0a, %ymm5, %ymm4, %ymm0
-	vblendpd $ 0x05, %ymm5, %ymm4, %ymm1
-	vblendpd $ 0x0a, %ymm7, %ymm6, %ymm2
-	vblendpd $ 0x05, %ymm7, %ymm6, %ymm3
-
-	vpermpd $ 0x1b , %ymm2, %ymm2
-	vpermpd $ 0x1b , %ymm3, %ymm3
-	vpermpd $ 0xb1 , %ymm2, %ymm2
-	vpermpd $ 0xb1 , %ymm3, %ymm3
-
-	vblendpd $ 0x03, %ymm0, %ymm2 , %ymm4
-	vblendpd $ 0x03, %ymm1, %ymm3 , %ymm5
-	vblendpd $ 0x03, %ymm2, %ymm0 , %ymm6
-	vblendpd $ 0x03, %ymm3, %ymm1 , %ymm7
-
-        leaq    (CO1, LDC, 2), %rax     
-	
-
-#if !defined(TRMMKERNEL)
-
-	vaddpd 	                (CO1), %ymm4, %ymm4
-	vaddpd 	           (CO1, LDC), %ymm5, %ymm5
-	vaddpd 	               (%rax), %ymm6, %ymm6
-	vaddpd 	          (%rax, LDC), %ymm7, %ymm7
-
-#endif
-
-	vmovups	%ymm4 ,  	(CO1)
-	vmovups	%ymm5 ,  	(CO1, LDC)
-	vmovups	%ymm6 ,  	(%rax)
-	vmovups	%ymm7 ,  	(%rax, LDC)
-
-	prefetcht0	32(CO1)
-	prefetcht0	32(CO1,LDC)
-	prefetcht0	32(%rax)
-	prefetcht0	32(%rax,LDC)
-
-	vpermpd $ 0xb1 , %ymm9 , %ymm9
-	vpermpd $ 0xb1 , %ymm11, %ymm11
-
-	vblendpd $ 0x0a, %ymm9 , %ymm8 , %ymm0
-	vblendpd $ 0x05, %ymm9 , %ymm8 , %ymm1
-	vblendpd $ 0x0a, %ymm11, %ymm10, %ymm2
-	vblendpd $ 0x05, %ymm11, %ymm10, %ymm3
-
-	vpermpd $ 0x1b , %ymm2, %ymm2
-	vpermpd $ 0x1b , %ymm3, %ymm3
-	vpermpd $ 0xb1 , %ymm2, %ymm2
-	vpermpd $ 0xb1 , %ymm3, %ymm3
-
-	vblendpd $ 0x03, %ymm0, %ymm2 , %ymm4
-	vblendpd $ 0x03, %ymm1, %ymm3 , %ymm5
-	vblendpd $ 0x03, %ymm2, %ymm0 , %ymm6
-	vblendpd $ 0x03, %ymm3, %ymm1 , %ymm7
-
-
-	leaq	(%rax, LDC, 2), %rax
-	leaq	(%rax, LDC, 2), %rbp
-
-#if !defined(TRMMKERNEL)
-
-	vaddpd 	                (%rax), %ymm4, %ymm4
-	vaddpd 	           (%rax, LDC), %ymm5, %ymm5
-	vaddpd 	                (%rbp), %ymm6, %ymm6
-	vaddpd 	           (%rbp, LDC), %ymm7, %ymm7
-
-#endif
-
-	vmovups	%ymm4 ,  	(%rax)
-	vmovups	%ymm5 ,  	(%rax, LDC)
-	vmovups	%ymm6 ,  	(%rbp)
-	vmovups	%ymm7 ,  	(%rbp, LDC)
-
-	prefetcht0	32(%rax)
-	prefetcht0	32(%rax,LDC)
-	prefetcht0	32(%rbp)
-	prefetcht0	32(%rbp,LDC)
-
-	vpermpd $ 0xb1 , %ymm13, %ymm13
-	vpermpd $ 0xb1 , %ymm15, %ymm15
-
-	vblendpd $ 0x0a, %ymm13, %ymm12, %ymm0
-	vblendpd $ 0x05, %ymm13, %ymm12, %ymm1
-	vblendpd $ 0x0a, %ymm15, %ymm14, %ymm2
-	vblendpd $ 0x05, %ymm15, %ymm14, %ymm3
-
-	vpermpd $ 0x1b , %ymm2, %ymm2
-	vpermpd $ 0x1b , %ymm3, %ymm3
-	vpermpd $ 0xb1 , %ymm2, %ymm2
-	vpermpd $ 0xb1 , %ymm3, %ymm3
-
-	vblendpd $ 0x03, %ymm0, %ymm2 , %ymm4
-	vblendpd $ 0x03, %ymm1, %ymm3 , %ymm5
-	vblendpd $ 0x03, %ymm2, %ymm0 , %ymm6
-	vblendpd $ 0x03, %ymm3, %ymm1 , %ymm7
-
-
-	leaq	(%rax, LDC, 4), %rax
-	leaq	(%rbp, LDC, 4), %rbp
-
-#if !defined(TRMMKERNEL)
-
-	vaddpd 	                (%rax), %ymm4, %ymm4
-	vaddpd 	           (%rax, LDC), %ymm5, %ymm5
-	vaddpd 	                (%rbp), %ymm6, %ymm6
-	vaddpd 	           (%rbp, LDC), %ymm7, %ymm7
-
-#endif
-
-	vmovups	%ymm4 ,  	(%rax)
-	vmovups	%ymm5 ,  	(%rax, LDC)
-	vmovups	%ymm6 ,  	(%rbp)
-	vmovups	%ymm7 ,  	(%rbp, LDC)
-
-	prefetcht0	32(%rax)
-	prefetcht0	32(%rax,LDC)
-	prefetcht0	32(%rbp)
-	prefetcht0	32(%rbp,LDC)
-
-	addq	$ 4*SIZE, CO1
-.endm
-
-/******************************************************************************************/
-
-.macro INIT2x12
-
-	vxorpd		%xmm4 , %xmm4 , %xmm4
-	vxorpd		%xmm5 , %xmm5 , %xmm5
-	vxorpd		%xmm6 , %xmm6 , %xmm6
-	vxorpd		%xmm7 , %xmm7 , %xmm7
-	vxorpd		%xmm8 , %xmm8 , %xmm8
-	vxorpd		%xmm9 , %xmm9 , %xmm9
-	vxorpd		%xmm10, %xmm10, %xmm10
-	vxorpd		%xmm11, %xmm11, %xmm11
-	vxorpd		%xmm12, %xmm12, %xmm12
-	vxorpd		%xmm13, %xmm13, %xmm13
-	vxorpd		%xmm14, %xmm14, %xmm14
-	vxorpd		%xmm15, %xmm15, %xmm15
-
-.endm
-
-.macro KERNEL2x12_SUB
-	vmovups 	-16 * SIZE(AO), %xmm0
-	vmovddup	-12 * SIZE(BO), %xmm1
-	vmovddup	-11 * SIZE(BO), %xmm2
-	vmovddup	-10 * SIZE(BO), %xmm3
-	vfmadd231pd  	%xmm0 ,%xmm1  , %xmm4
-	vmovddup	 -9 * SIZE(BO), %xmm1
-	vfmadd231pd  	%xmm0 ,%xmm2  , %xmm5
-	vmovddup	 -8 * SIZE(BO), %xmm2
-	vfmadd231pd  	%xmm0 ,%xmm3  , %xmm6
-	vmovddup	 -7 * SIZE(BO), %xmm3
-	vfmadd231pd  	%xmm0 ,%xmm1  , %xmm7
-	vmovddup	 -6 * SIZE(BO), %xmm1
-	vfmadd231pd  	%xmm0 ,%xmm2  , %xmm8
-	vmovddup	 -5 * SIZE(BO), %xmm2
-	vfmadd231pd  	%xmm0 ,%xmm3  , %xmm9
-	vmovddup	 -4 * SIZE(BO), %xmm3
-	vfmadd231pd  	%xmm0 ,%xmm1  , %xmm10
-	vmovddup	 -3 * SIZE(BO), %xmm1
-	vfmadd231pd  	%xmm0 ,%xmm2  , %xmm11
-	vmovddup	 -2 * SIZE(BO), %xmm2
-	vfmadd231pd  	%xmm0 ,%xmm3  , %xmm12
-	vmovddup	 -1 * SIZE(BO), %xmm3
-	vfmadd231pd  	%xmm0 ,%xmm1  , %xmm13
-	addq		$ 12*SIZE, BO
-	vfmadd231pd  	%xmm0 ,%xmm2  , %xmm14
-	addq		$ 2*SIZE, AO
-	vfmadd231pd  	%xmm0 ,%xmm3  , %xmm15
-
-.endm
-
-.macro SAVE2x12
-
-	vmovddup	ALPHA, %xmm0
-
-	vmulpd	%xmm0 , %xmm4 , %xmm4
-	vmulpd	%xmm0 , %xmm5 , %xmm5
-	vmulpd	%xmm0 , %xmm6 , %xmm6
-	vmulpd	%xmm0 , %xmm7 , %xmm7
-
-	vmulpd	%xmm0 , %xmm8 , %xmm8
-	vmulpd	%xmm0 , %xmm9 , %xmm9
-	vmulpd	%xmm0 , %xmm10, %xmm10
-	vmulpd	%xmm0 , %xmm11, %xmm11
-
-	vmulpd	%xmm0 , %xmm12, %xmm12
-	vmulpd	%xmm0 , %xmm13, %xmm13
-	vmulpd	%xmm0 , %xmm14, %xmm14
-	vmulpd	%xmm0 , %xmm15, %xmm15
-
-
-        leaq    (CO1, LDC, 2), %rax     
-	
-
-#if !defined(TRMMKERNEL)
-
-	vaddpd 	                (CO1), %xmm4, %xmm4
-	vaddpd 	           (CO1, LDC), %xmm5, %xmm5
-	vaddpd 	               (%rax), %xmm6, %xmm6
-	vaddpd 	          (%rax, LDC), %xmm7, %xmm7
-
-#endif
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm5 ,  	(CO1, LDC)
-	vmovups	%xmm6 ,  	(%rax)
-	vmovups	%xmm7 ,  	(%rax, LDC)
-
-
-	leaq	(%rax, LDC, 2), %rax
-	leaq	(%rax, LDC, 2), %rbp
-
-#if !defined(TRMMKERNEL)
-
-	vaddpd 	                (%rax), %xmm8 , %xmm4
-	vaddpd 	           (%rax, LDC), %xmm9 , %xmm5
-	vaddpd 	                (%rbp), %xmm10, %xmm6
-	vaddpd 	           (%rbp, LDC), %xmm11, %xmm7
-
-#endif
-
-	vmovups	%xmm4 ,  	(%rax)
-	vmovups	%xmm5 ,  	(%rax, LDC)
-	vmovups	%xmm6 ,  	(%rbp)
-	vmovups	%xmm7 ,  	(%rbp, LDC)
-
-
-	leaq	(%rax, LDC, 4), %rax
-	leaq	(%rbp, LDC, 4), %rbp
-
-#if !defined(TRMMKERNEL)
-
-	vaddpd 	                (%rax), %xmm12, %xmm4
-	vaddpd 	           (%rax, LDC), %xmm13, %xmm5
-	vaddpd 	                (%rbp), %xmm14, %xmm6
-	vaddpd 	           (%rbp, LDC), %xmm15, %xmm7
-
-#endif
-
-	vmovups	%xmm4 ,  	(%rax)
-	vmovups	%xmm5 ,  	(%rax, LDC)
-	vmovups	%xmm6 ,  	(%rbp)
-	vmovups	%xmm7 ,  	(%rbp, LDC)
-
-	addq	$ 2*SIZE, CO1
-.endm
-
-
-/******************************************************************************************/
-
-.macro INIT1x12
-
-	vxorpd		%xmm4 , %xmm4 , %xmm4
-	vxorpd		%xmm5 , %xmm5 , %xmm5
-	vxorpd		%xmm6 , %xmm6 , %xmm6
-	vxorpd		%xmm7 , %xmm7 , %xmm7
-	vxorpd		%xmm8 , %xmm8 , %xmm8
-	vxorpd		%xmm9 , %xmm9 , %xmm9
-	vxorpd		%xmm10, %xmm10, %xmm10
-	vxorpd		%xmm11, %xmm11, %xmm11
-	vxorpd		%xmm12, %xmm12, %xmm12
-	vxorpd		%xmm13, %xmm13, %xmm13
-	vxorpd		%xmm14, %xmm14, %xmm14
-	vxorpd		%xmm15, %xmm15, %xmm15
-
-.endm
-
-.macro KERNEL1x12_SUB
-	vmovsd 	-16 * SIZE(AO), %xmm0
-	vmovsd	-12 * SIZE(BO), %xmm1
-	vmovsd	-11 * SIZE(BO), %xmm2
-	vmovsd	-10 * SIZE(BO), %xmm3
-	vfmadd231sd  	%xmm0 ,%xmm1  , %xmm4
-	vmovsd	 -9 * SIZE(BO), %xmm1
-	vfmadd231sd  	%xmm0 ,%xmm2  , %xmm5
-	vmovsd	 -8 * SIZE(BO), %xmm2
-	vfmadd231sd  	%xmm0 ,%xmm3  , %xmm6
-	vmovsd	 -7 * SIZE(BO), %xmm3
-	vfmadd231sd  	%xmm0 ,%xmm1  , %xmm7
-	vmovsd	 -6 * SIZE(BO), %xmm1
-	vfmadd231sd  	%xmm0 ,%xmm2  , %xmm8
-	vmovsd	 -5 * SIZE(BO), %xmm2
-	vfmadd231sd  	%xmm0 ,%xmm3  , %xmm9
-	vmovsd	 -4 * SIZE(BO), %xmm3
-	vfmadd231sd  	%xmm0 ,%xmm1  , %xmm10
-	vmovsd	 -3 * SIZE(BO), %xmm1
-	vfmadd231sd  	%xmm0 ,%xmm2  , %xmm11
-	vmovsd	 -2 * SIZE(BO), %xmm2
-	vfmadd231sd  	%xmm0 ,%xmm3  , %xmm12
-	vmovsd	 -1 * SIZE(BO), %xmm3
-	vfmadd231sd  	%xmm0 ,%xmm1  , %xmm13
-	addq		$ 12*SIZE, BO
-	vfmadd231sd  	%xmm0 ,%xmm2  , %xmm14
-	addq		$ 1*SIZE, AO
-	vfmadd231sd  	%xmm0 ,%xmm3  , %xmm15
-
-.endm
-
-.macro SAVE1x12
-
-	vmovsd	ALPHA, %xmm0
-
-	vmulsd	%xmm0 , %xmm4 , %xmm4
-	vmulsd	%xmm0 , %xmm5 , %xmm5
-	vmulsd	%xmm0 , %xmm6 , %xmm6
-	vmulsd	%xmm0 , %xmm7 , %xmm7
-
-	vmulsd	%xmm0 , %xmm8 , %xmm8
-	vmulsd	%xmm0 , %xmm9 , %xmm9
-	vmulsd	%xmm0 , %xmm10, %xmm10
-	vmulsd	%xmm0 , %xmm11, %xmm11
-
-	vmulsd	%xmm0 , %xmm12, %xmm12
-	vmulsd	%xmm0 , %xmm13, %xmm13
-	vmulsd	%xmm0 , %xmm14, %xmm14
-	vmulsd	%xmm0 , %xmm15, %xmm15
-
-
-        leaq    (CO1, LDC, 2), %rax     
-	
-
-#if !defined(TRMMKERNEL)
-
-	vaddsd 	                (CO1), %xmm4, %xmm4
-	vaddsd 	           (CO1, LDC), %xmm5, %xmm5
-	vaddsd 	               (%rax), %xmm6, %xmm6
-	vaddsd 	          (%rax, LDC), %xmm7, %xmm7
-
-#endif
-
-	vmovsd	%xmm4 ,  	(CO1)
-	vmovsd	%xmm5 ,  	(CO1, LDC)
-	vmovsd	%xmm6 ,  	(%rax)
-	vmovsd	%xmm7 ,  	(%rax, LDC)
-
-
-	leaq	(%rax, LDC, 2), %rax
-	leaq	(%rax, LDC, 2), %rbp
-
-#if !defined(TRMMKERNEL)
-
-	vaddsd 	                (%rax), %xmm8 , %xmm4
-	vaddsd 	           (%rax, LDC), %xmm9 , %xmm5
-	vaddsd 	                (%rbp), %xmm10, %xmm6
-	vaddsd 	           (%rbp, LDC), %xmm11, %xmm7
-
-#endif
-
-	vmovsd	%xmm4 ,  	(%rax)
-	vmovsd	%xmm5 ,  	(%rax, LDC)
-	vmovsd	%xmm6 ,  	(%rbp)
-	vmovsd	%xmm7 ,  	(%rbp, LDC)
-
-
-	leaq	(%rax, LDC, 4), %rax
-	leaq	(%rbp, LDC, 4), %rbp
-
-#if !defined(TRMMKERNEL)
-
-	vaddsd 	                (%rax), %xmm12, %xmm4
-	vaddsd 	           (%rax, LDC), %xmm13, %xmm5
-	vaddsd 	                (%rbp), %xmm14, %xmm6
-	vaddsd 	           (%rbp, LDC), %xmm15, %xmm7
-
-#endif
-
-	vmovsd	%xmm4 ,  	(%rax)
-	vmovsd	%xmm5 ,  	(%rax, LDC)
-	vmovsd	%xmm6 ,  	(%rbp)
-	vmovsd	%xmm7 ,  	(%rbp, LDC)
-
-	addq	$ 1*SIZE, CO1
-.endm
-
-
-
-
-/******************************************************************************************/
-/******************************************************************************************/
-
-.macro INIT4x4
-
-	vxorpd		%ymm4 , %ymm4 , %ymm4
-	vxorpd		%ymm5 , %ymm5 , %ymm5
-	vxorpd		%ymm6 , %ymm6 , %ymm6
-	vxorpd		%ymm7 , %ymm7 , %ymm7
-
-.endm
-
-.macro KERNEL4x4_I
-	prefetcht0	A_PR1(AO)
-	vmovups		-12 * SIZE(BO), %ymm1
-	vmovups 	-16 * SIZE(AO), %ymm0
-	vmulpd  	%ymm0 ,%ymm1  , %ymm4
-	vpermpd		$ 0xb1, %ymm0  , %ymm0
-	vmulpd  	%ymm0 ,%ymm1  , %ymm5
-	vpermpd		$ 0x1b, %ymm0  , %ymm0
-	vmulpd  	%ymm0 ,%ymm1  , %ymm6
-
-	addq		$ 4*SIZE, BO
-	vpermpd		$ 0xb1, %ymm0  , %ymm0
-	vmulpd  	%ymm0 ,%ymm1  , %ymm7
-	vmovups		-12 * SIZE(BO), %ymm1
-
-.endm
-
-.macro KERNEL4x4_M1
-	prefetcht0	A_PR1(AO)
-	vmovups 	-16 * SIZE(AO), %ymm0
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4
-	vpermpd		$ 0xb1, %ymm0  , %ymm0
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5
-	vpermpd		$ 0x1b, %ymm0  , %ymm0
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6
-
-	vpermpd		$ 0xb1, %ymm0  , %ymm0
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7
-	vmovups		-12 * SIZE(BO), %ymm1
-
-.endm
-
-.macro KERNEL4x4_M2
-	vmovups 	-12 * SIZE(AO), %ymm0
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4
-	vpermpd		$ 0xb1, %ymm0  , %ymm0
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5
-	vpermpd		$ 0x1b, %ymm0  , %ymm0
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6
-
-	addq		$ 8*SIZE, AO
-	vpermpd		$ 0xb1, %ymm0  , %ymm0
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7
-	vmovups		 -8 * SIZE(BO), %ymm1
-	addq		$ 8*SIZE, BO
-.endm
-
-
-.macro KERNEL4x4_E
-	vmovups 	-12 * SIZE(AO), %ymm0
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4
-	vpermpd		$ 0xb1, %ymm0  , %ymm0
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5
-	vpermpd		$ 0x1b, %ymm0  , %ymm0
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6
-
-	addq		$ 8*SIZE, AO
-	vpermpd		$ 0xb1, %ymm0  , %ymm0
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7
-	addq		$ 4*SIZE, BO
-.endm
-
-.macro KERNEL4x4_SUB
-	vmovups		-12 * SIZE(BO), %ymm1
-	vmovups 	-16 * SIZE(AO), %ymm0
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4
-	vpermpd		$ 0xb1, %ymm0  , %ymm0
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5
-	addq		$ 4*SIZE, BO
-	vpermpd		$ 0x1b, %ymm0  , %ymm0
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6
-	addq		$ 4*SIZE, AO
-	vpermpd		$ 0xb1, %ymm0  , %ymm0
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7
-
-.endm
-
-.macro SAVE4x4
-
-	vbroadcastsd	ALPHA, %ymm0
-
-	vmulpd	%ymm0 , %ymm4 , %ymm4
-	vmulpd	%ymm0 , %ymm7 , %ymm7
-	vmulpd	%ymm0 , %ymm5 , %ymm5
-	vmulpd	%ymm0 , %ymm6 , %ymm6
-
-	vpermpd $ 0xb1 , %ymm5, %ymm5
-	vpermpd $ 0xb1 , %ymm7, %ymm7
-
-	vblendpd $ 0x0a, %ymm5, %ymm4, %ymm0
-	vblendpd $ 0x05, %ymm5, %ymm4, %ymm1
-	vblendpd $ 0x0a, %ymm7, %ymm6, %ymm2
-	vblendpd $ 0x05, %ymm7, %ymm6, %ymm3
-
-	vpermpd $ 0x1b , %ymm2, %ymm2
-	vpermpd $ 0x1b , %ymm3, %ymm3
-	vpermpd $ 0xb1 , %ymm2, %ymm2
-	vpermpd $ 0xb1 , %ymm3, %ymm3
-
-	vblendpd $ 0x03, %ymm0, %ymm2 , %ymm4
-	vblendpd $ 0x03, %ymm1, %ymm3 , %ymm5
-	vblendpd $ 0x03, %ymm2, %ymm0 , %ymm6
-	vblendpd $ 0x03, %ymm3, %ymm1 , %ymm7
-
-        leaq    (CO1, LDC, 2), %rax     
-	
-
-#if !defined(TRMMKERNEL)
-
-	vaddpd 	                (CO1), %ymm4, %ymm4
-	vaddpd 	           (CO1, LDC), %ymm5, %ymm5
-	vaddpd 	               (%rax), %ymm6, %ymm6
-	vaddpd 	          (%rax, LDC), %ymm7, %ymm7
-
-#endif
-
-	vmovups	%ymm4 ,  	(CO1)
-	vmovups	%ymm5 ,  	(CO1, LDC)
-	vmovups	%ymm6 ,  	(%rax)
-	vmovups	%ymm7 ,  	(%rax, LDC)
-
-	addq	$ 4*SIZE, CO1
-.endm
-
-/******************************************************************************************/
-/******************************************************************************************/
-
-.macro INIT2x4
-
-	vxorpd		%xmm4 , %xmm4 , %xmm4
-	vxorpd		%xmm5 , %xmm5 , %xmm5
-	vxorpd		%xmm6 , %xmm6 , %xmm6
-	vxorpd		%xmm7 , %xmm7 , %xmm7
-
-.endm
-
-
-.macro KERNEL2x4_SUB
-	vmovddup	-12 * SIZE(BO), %xmm1
-	vmovups 	-16 * SIZE(AO), %xmm0
-	vmovddup	-11 * SIZE(BO), %xmm2
-	vfmadd231pd  	%xmm0 ,%xmm1  , %xmm4
-	vmovddup	-10 * SIZE(BO), %xmm3
-	vfmadd231pd  	%xmm0 ,%xmm2  , %xmm5
-	vmovddup	 -9 * SIZE(BO), %xmm8
-	vfmadd231pd  	%xmm0 ,%xmm3  , %xmm6
-	addq		$ 4*SIZE, BO
-	vfmadd231pd  	%xmm0 ,%xmm8  , %xmm7
-	addq		$ 2*SIZE, AO
-
-.endm
-
-
-.macro SAVE2x4
-
-	vmovddup	ALPHA, %xmm0
-
-	vmulpd	%xmm0 , %xmm4 , %xmm4
-	vmulpd	%xmm0 , %xmm5 , %xmm5
-	vmulpd	%xmm0 , %xmm6 , %xmm6
-	vmulpd	%xmm0 , %xmm7 , %xmm7
-
-        leaq    (CO1, LDC, 2), %rax     
-
-#if !defined(TRMMKERNEL)
-
-	vaddpd 	                (CO1), %xmm4, %xmm4
-	vaddpd 	           (CO1, LDC), %xmm5, %xmm5
-	vaddpd 	               (%rax), %xmm6, %xmm6
-	vaddpd 	          (%rax, LDC), %xmm7, %xmm7
-
-#endif
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm5 ,  	(CO1, LDC)
-	vmovups	%xmm6 ,  	(%rax)
-	vmovups	%xmm7 ,  	(%rax, LDC)
-
-	addq	$ 2*SIZE, CO1
-.endm
-
-/******************************************************************************************/
-/******************************************************************************************/
-
-.macro INIT1x4
-
-	vxorpd		%xmm4 , %xmm4 , %xmm4
-	vxorpd		%xmm5 , %xmm5 , %xmm5
-	vxorpd		%xmm6 , %xmm6 , %xmm6
-	vxorpd		%xmm7 , %xmm7 , %xmm7
-
-.endm
-
-
-.macro KERNEL1x4_SUB
-	vmovsd	-12 * SIZE(BO), %xmm1
-	vmovsd 	-16 * SIZE(AO), %xmm0
-	vmovsd	-11 * SIZE(BO), %xmm2
-	vfmadd231sd  	%xmm0 ,%xmm1  , %xmm4
-	vmovsd	-10 * SIZE(BO), %xmm3
-	vfmadd231sd  	%xmm0 ,%xmm2  , %xmm5
-	vmovsd	 -9 * SIZE(BO), %xmm8
-	vfmadd231sd  	%xmm0 ,%xmm3  , %xmm6
-	addq		$ 4*SIZE, BO
-	vfmadd231sd  	%xmm0 ,%xmm8  , %xmm7
-	addq		$ 1*SIZE, AO
-
-.endm
-
-
-.macro SAVE1x4
-
-	vmovsd	ALPHA, %xmm0
-
-	vmulsd	%xmm0 , %xmm4 , %xmm4
-	vmulsd	%xmm0 , %xmm5 , %xmm5
-	vmulsd	%xmm0 , %xmm6 , %xmm6
-	vmulsd	%xmm0 , %xmm7 , %xmm7
-
-        leaq    (CO1, LDC, 2), %rax     
-
-#if !defined(TRMMKERNEL)
-
-	vaddsd 	                (CO1), %xmm4, %xmm4
-	vaddsd 	           (CO1, LDC), %xmm5, %xmm5
-	vaddsd 	               (%rax), %xmm6, %xmm6
-	vaddsd 	          (%rax, LDC), %xmm7, %xmm7
-
-#endif
-
-	vmovsd	%xmm4 ,  	(CO1)
-	vmovsd	%xmm5 ,  	(CO1, LDC)
-	vmovsd	%xmm6 ,  	(%rax)
-	vmovsd	%xmm7 ,  	(%rax, LDC)
-
-	addq	$ 1*SIZE, CO1
-.endm
-
-
-/******************************************************************************************/
-/******************************************************************************************/
-
-.macro INIT4x2
-
-	vxorpd		%xmm4 , %xmm4 , %xmm4
-	vxorpd		%xmm5 , %xmm5 , %xmm5
-	vxorpd		%xmm6 , %xmm6 , %xmm6
-	vxorpd		%xmm7 , %xmm7 , %xmm7
-
-.endm
-
-
-.macro KERNEL4x2_SUB
-	vmovddup	-12 * SIZE(BO), %xmm2
-	vmovups 	-16 * SIZE(AO), %xmm0
-	vmovups 	-14 * SIZE(AO), %xmm1
-	vmovddup	-11 * SIZE(BO), %xmm3
-	vfmadd231pd  	%xmm0 ,%xmm2  , %xmm4
-	vfmadd231pd  	%xmm1 ,%xmm2  , %xmm5
-	vfmadd231pd  	%xmm0 ,%xmm3  , %xmm6
-	vfmadd231pd  	%xmm1 ,%xmm3  , %xmm7
-	addq		$ 2*SIZE, BO
-	addq		$ 4*SIZE, AO
-
-.endm
-
-
-.macro SAVE4x2
-
-	vmovddup	ALPHA, %xmm0
-
-	vmulpd	%xmm0 , %xmm4 , %xmm4
-	vmulpd	%xmm0 , %xmm5 , %xmm5
-	vmulpd	%xmm0 , %xmm6 , %xmm6
-	vmulpd	%xmm0 , %xmm7 , %xmm7
-
-
-#if !defined(TRMMKERNEL)
-
-	vaddpd 	                (CO1)     , %xmm4, %xmm4
-	vaddpd 	        2 * SIZE(CO1)     , %xmm5, %xmm5
-	vaddpd 	                (CO1, LDC), %xmm6, %xmm6
-	vaddpd 	        2 * SIZE(CO1, LDC), %xmm7, %xmm7
-
-#endif
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm5 , 2 * SIZE(CO1)
-	vmovups	%xmm6 ,  	(CO1, LDC)
-	vmovups	%xmm7 , 2 * SIZE(CO1, LDC)
-
-	addq	$ 4*SIZE, CO1
-.endm
-
-
-/******************************************************************************************/
-/******************************************************************************************/
-
-.macro INIT2x2
-
-	vxorpd		%xmm4 , %xmm4 , %xmm4
-	vxorpd		%xmm6 , %xmm6 , %xmm6
-
-.endm
-
-
-.macro KERNEL2x2_SUB
-	vmovddup	-12 * SIZE(BO), %xmm2
-	vmovups 	-16 * SIZE(AO), %xmm0
-	vmovddup	-11 * SIZE(BO), %xmm3
-	vfmadd231pd  	%xmm0 ,%xmm2  , %xmm4
-	vfmadd231pd  	%xmm0 ,%xmm3  , %xmm6
-	addq		$ 2*SIZE, BO
-	addq		$ 2*SIZE, AO
-
-.endm
-
-
-.macro SAVE2x2
-
-	vmovddup	ALPHA, %xmm0
-
-	vmulpd	%xmm0 , %xmm4 , %xmm4
-	vmulpd	%xmm0 , %xmm6 , %xmm6
-
-
-#if !defined(TRMMKERNEL)
-
-	vaddpd 	                (CO1)     , %xmm4, %xmm4
-	vaddpd 	                (CO1, LDC), %xmm6, %xmm6
-
-#endif
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm6 ,  	(CO1, LDC)
-
-	addq	$ 2*SIZE, CO1
-.endm
-
-/******************************************************************************************/
-/******************************************************************************************/
-
-.macro INIT1x2
-
-	vxorpd		%xmm4 , %xmm4 , %xmm4
-	vxorpd		%xmm5 , %xmm5 , %xmm5
-
-.endm
-
-
-.macro KERNEL1x2_SUB
-	vmovsd	-12 * SIZE(BO), %xmm1
-	vmovsd 	-16 * SIZE(AO), %xmm0
-	vmovsd	-11 * SIZE(BO), %xmm2
-	vfmadd231sd  	%xmm0 ,%xmm1  , %xmm4
-	vfmadd231sd  	%xmm0 ,%xmm2  , %xmm5
-	addq		$ 2*SIZE, BO
-	addq		$ 1*SIZE, AO
-
-.endm
-
-
-.macro SAVE1x2
-
-	vmovsd	ALPHA, %xmm0
-
-	vmulsd	%xmm0 , %xmm4 , %xmm4
-	vmulsd	%xmm0 , %xmm5 , %xmm5
-
-
-#if !defined(TRMMKERNEL)
-
-	vaddsd 	                (CO1), %xmm4, %xmm4
-	vaddsd 	           (CO1, LDC), %xmm5, %xmm5
-
-#endif
-
-	vmovsd	%xmm4 ,  	(CO1)
-	vmovsd	%xmm5 ,  	(CO1, LDC)
-
-	addq	$ 1*SIZE, CO1
-.endm
-
-
-/******************************************************************************************/
-/******************************************************************************************/
-
-.macro INIT4x1
-
-	vxorpd		%ymm4 , %ymm4 , %ymm4
-	vxorpd		%ymm5 , %ymm5 , %ymm5
-	vxorpd		%ymm6 , %ymm6 , %ymm6
-	vxorpd		%ymm7 , %ymm7 , %ymm7
-
-.endm
-
-
-.macro KERNEL4x1
-
-	vbroadcastsd	-12 * SIZE(BO), %ymm0
-	vbroadcastsd	-11 * SIZE(BO), %ymm1
-	vbroadcastsd	-10 * SIZE(BO), %ymm2
-	vbroadcastsd	-9  * SIZE(BO), %ymm3
-
-	vfmadd231pd  	-16 * SIZE(AO) ,%ymm0  , %ymm4
-	vfmadd231pd  	-12 * SIZE(AO) ,%ymm1  , %ymm5
-
-	vbroadcastsd	-8  * SIZE(BO), %ymm0
-	vbroadcastsd	-7  * SIZE(BO), %ymm1
-
-	vfmadd231pd  	-8  * SIZE(AO) ,%ymm2  , %ymm6
-	vfmadd231pd  	-4  * SIZE(AO) ,%ymm3  , %ymm7
-
-	vbroadcastsd	-6  * SIZE(BO), %ymm2
-	vbroadcastsd	-5  * SIZE(BO), %ymm3
-
-	vfmadd231pd  	 0  * SIZE(AO) ,%ymm0  , %ymm4
-	vfmadd231pd  	 4  * SIZE(AO) ,%ymm1  , %ymm5
-	vfmadd231pd  	 8  * SIZE(AO) ,%ymm2  , %ymm6
-	vfmadd231pd  	 12 * SIZE(AO) ,%ymm3  , %ymm7
-
-	addq		$ 8 *SIZE, BO
-	addq		$ 32*SIZE, AO
-
-.endm
-
-
-.macro KERNEL4x1_SUB
-	vbroadcastsd	-12 * SIZE(BO), %ymm2
-	vmovups 	-16 * SIZE(AO), %ymm0
-	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm4
-	addq		$ 1*SIZE, BO
-	addq		$ 4*SIZE, AO
-
-.endm
-
-
-.macro SAVE4x1
-
-	vbroadcastsd	ALPHA, %ymm0
-
-	vaddpd	%ymm4,%ymm5, %ymm4 
-	vaddpd	%ymm6,%ymm7, %ymm6 
-	vaddpd	%ymm4,%ymm6, %ymm4 
-
-	vmulpd	%ymm0 , %ymm4 , %ymm4
-
-
-#if !defined(TRMMKERNEL)
-
-	vaddpd 	                (CO1)     , %ymm4, %ymm4
-
-#endif
-
-	vmovups	%ymm4 ,  	(CO1)
-
-	addq	$ 4*SIZE, CO1
-.endm
-
-
-/******************************************************************************************/
-/******************************************************************************************/
-
-.macro INIT2x1
-
-	vxorpd		%xmm4 , %xmm4 , %xmm4
-
-.endm
-
-
-.macro KERNEL2x1_SUB
-	vmovddup	-12 * SIZE(BO), %xmm2
-	vmovups 	-16 * SIZE(AO), %xmm0
-	vfmadd231pd  	%xmm0 ,%xmm2  , %xmm4
-	addq		$ 1*SIZE, BO
-	addq		$ 2*SIZE, AO
-
-.endm
-
-
-.macro SAVE2x1
-
-	vmovddup	ALPHA, %xmm0
-
-	vmulpd	%xmm0 , %xmm4 , %xmm4
-
-
-#if !defined(TRMMKERNEL)
-
-	vaddpd 	                (CO1)     , %xmm4, %xmm4
-
-#endif
-
-	vmovups	%xmm4 ,  	(CO1)
-
-	addq	$ 2*SIZE, CO1
-.endm
-
-
-/******************************************************************************************/
-/******************************************************************************************/
-
-.macro INIT1x1
-
-	vxorpd		%xmm4 , %xmm4 , %xmm4
-
-.endm
-
-
-.macro KERNEL1x1_SUB
-	vmovsd	-12 * SIZE(BO), %xmm1
-	vmovsd 	-16 * SIZE(AO), %xmm0
-	vfmadd231sd  	%xmm0 ,%xmm1  , %xmm4
-	addq		$ 1*SIZE, BO
-	addq		$ 1*SIZE, AO
-
-.endm
-
-
-.macro SAVE1x1
-
-	vmovsd	ALPHA, %xmm0
-
-	vmulsd	%xmm0 , %xmm4 , %xmm4
-
-
-#if !defined(TRMMKERNEL)
-
-	vaddsd 	                (CO1), %xmm4, %xmm4
-
-#endif
-
-	vmovsd	%xmm4 ,  	(CO1)
-
-	addq	$ 1*SIZE, CO1
-.endm
-
-
-/*******************************************************************************************/
-
-#if !defined(TRMMKERNEL)
-
-
-	PROLOGUE
-	PROFCODE
-	
-	subq	$STACKSIZE, %rsp
-	movq	%rbx,   (%rsp)
-	movq	%rbp,  8(%rsp)
-	movq	%r12, 16(%rsp)
-	movq	%r13, 24(%rsp)
-	movq	%r14, 32(%rsp)
-	movq	%r15, 40(%rsp)
-
-	vzeroupper
-
-#ifdef WINDOWS_ABI
-	movq	%rdi,    48(%rsp)
-	movq	%rsi,    56(%rsp)
-	vmovups	%xmm6,   64(%rsp)
-	vmovups	%xmm7,   80(%rsp)
-	vmovups	%xmm8,   96(%rsp)
-	vmovups	%xmm9,  112(%rsp)
-	vmovups	%xmm10, 128(%rsp)
-	vmovups	%xmm11, 144(%rsp)
-	vmovups	%xmm12, 160(%rsp)
-	vmovups	%xmm13, 176(%rsp)
-	vmovups	%xmm14, 192(%rsp)
-	vmovups	%xmm15, 208(%rsp)
-
-	movq	ARG1,      OLD_M
-	movq	ARG2,      OLD_N
-	movq	ARG3,      OLD_K
-	movq	OLD_A,     A
-	movq	OLD_B,     B
-	movq	OLD_C,     C
-	movq	OLD_LDC,   LDC
-
-	vmovups	%xmm3, %xmm0
-
-#else
-	movq	STACKSIZE +  8(%rsp), LDC
-
-#endif
-
-	movq    %rsp, SP      # save old stack
-        subq    $128 + L_BUFFER_SIZE, %rsp
-        andq    $-4096, %rsp    # align stack
-
-        STACK_TOUCH
-
-	cmpq	$ 0, OLD_M
-	je	.L999
-
-	cmpq	$ 0, OLD_N
-	je	.L999
-
-	cmpq	$ 0, OLD_K
-	je	.L999
-
-	movq	OLD_M, M
-	movq	OLD_N, N
-	movq	OLD_K, K
-
-	vmovsd	 %xmm0, ALPHA
-
-	salq	$BASE_SHIFT, LDC
-
-	movq    N, %rax
-        xorq    %rdx, %rdx
-        movq    $12,  %rdi
-        divq    %rdi                    //    N / 12
-        movq    %rax, Ndiv12             //    N / 12
-        movq    %rdx, Nmod12             //    N % 12
-
-
-	movq	Ndiv12,  J
-	cmpq	$ 0, J
-	je	.L4_0
-	ALIGN_4
-
-.L12_01:
-        // copy to sub buffer
-        movq    K, %rax
-        salq    $2,%rax                 // K * 4 ; read 2 values
-        movq    B, BO1
-        leaq    (B,%rax, SIZE), BO2     // next offset to BO2
-        leaq    (BO2,%rax, SIZE), BO3     // next offset to BO2
-
-
-        leaq    BUFFER1, BO             // first buffer to BO
-        movq    K, %rax
-	sarq	$1 , %rax		// K / 2
-	jz	.L12_01a_2
-        ALIGN_4
-
-.L12_01a_1:
-
-        prefetcht0 512(BO1)
-        prefetcht0 512(BO2)
-        prefetcht0 512(BO3)
-        prefetchw  512(BO)
-
-
-	vmovups	0 * SIZE(BO1), %ymm1
-	vmovups	4 * SIZE(BO1), %ymm5
-	vmovups	0 * SIZE(BO2), %ymm2
-	vmovups	4 * SIZE(BO2), %ymm6
-	vmovups	0 * SIZE(BO3), %ymm3
-	vmovups	4 * SIZE(BO3), %ymm7
-
-	vmovups	%ymm1,  0 * SIZE(BO)
-	vmovups	%ymm2,  4 * SIZE(BO)
-	vmovups	%ymm3,  8 * SIZE(BO)
-
-	vmovups	%ymm5, 12 * SIZE(BO)
-	vmovups	%ymm6, 16 * SIZE(BO)
-	vmovups	%ymm7, 20 * SIZE(BO)
-
-	addq	$ 8 * SIZE ,BO1
-	addq	$ 8 * SIZE ,BO2
-	addq	$ 8 * SIZE ,BO3
-	addq    $ 24 *SIZE ,BO
-
-	decq	%rax
-	jnz	.L12_01a_1
-
-
-
-.L12_01a_2:
-
-	movq    K, %rax
-        andq    $1, %rax                // K % 2
-        jz      .L12_03c
-        ALIGN_4
-
-
-.L12_02b:
-
-	vmovups	0 * SIZE(BO1), %ymm1
-	vmovups	0 * SIZE(BO2), %ymm2
-	vmovups	0 * SIZE(BO3), %ymm3
-	vmovups	%ymm1, 0 * SIZE(BO)
-	vmovups	%ymm2, 4 * SIZE(BO)
-	vmovups	%ymm3, 8 * SIZE(BO)
-	addq	$ 4*SIZE,BO1
-	addq	$ 4*SIZE,BO2
-	addq	$ 4*SIZE,BO3
-	addq	$ 12*SIZE,BO
-	decq	%rax
-	jnz	.L12_02b
-
-.L12_03c:
-
-	movq	BO3, B			// next offset of B
-
-.L12_10:
-	movq	C, CO1
-	leaq	(C, LDC, 8), C		 
-	leaq	(C, LDC, 4), C		// c += 12 * ldc
-
-	
-	movq	A, AO		 	// aoffset = a
-	addq	$16 * SIZE, AO
-
-	movq	M,  I
-	sarq	$2, I			// i = m / 4
-	je	.L12_20
-
-	ALIGN_4
-
-.L12_11:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $12 * SIZE, BO
-
-        movq    K, %rax
-
-	sarq $3, %rax			//  K / 8
-	cmpq $2, %rax
-
-	jl	.L12_13
-
-
-	KERNEL4x12_I
-	KERNEL4x12_M2
-	KERNEL4x12_M1
-	KERNEL4x12_M2
-
-	KERNEL4x12_M1
-	KERNEL4x12_M2
-	KERNEL4x12_M1
-	KERNEL4x12_M2
-
-	subq $2, %rax
-	je	.L12_12a
-
-	ALIGN_5
-.L12_12:
-
-	KERNEL4x12_M1
-	KERNEL4x12_M2
-	KERNEL4x12_M1
-	KERNEL4x12_M2
-
-	KERNEL4x12_M1
-	KERNEL4x12_M2
-	KERNEL4x12_M1
-	KERNEL4x12_M2
-
-	dec	%rax
-	jne	.L12_12
-
-.L12_12a:
-
-	KERNEL4x12_M1
-	KERNEL4x12_M2
-	KERNEL4x12_M1
-	KERNEL4x12_M2
-
-	KERNEL4x12_M1
-	KERNEL4x12_M2
-	KERNEL4x12_M1
-	KERNEL4x12_E
-
-	jmp .L12_16
-
-
-.L12_13:
-
-	test $1, %rax
-	jz .L12_14
-
-	KERNEL4x12_I
-	KERNEL4x12_M2
-	KERNEL4x12_M1
-	KERNEL4x12_M2
-
-	KERNEL4x12_M1
-	KERNEL4x12_M2
-	KERNEL4x12_M1
-	KERNEL4x12_E
-
-	jmp .L12_16
-
-
-.L12_14:
-
-	INIT4x12
-
-
-.L12_16:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L12_19
-
-	ALIGN_4
-
-.L12_17:
-
-	KERNEL4x12_SUB
-
-	dec	%rax
-	jne	.L12_17
-	ALIGN_4
-
-
-.L12_19:
-
-	SAVE4x12
-
-	decq	I			# i --
-	jne	.L12_11
-	ALIGN_4	
-
-/**************************************************************************
-* Rest of M 
-***************************************************************************/
-.L12_20:
-	// Test rest of M
-
-	testq	$3, M
-	jz	.L12_100			// to next 16 lines of N
-
-
-.L12_30:
-	testq	$2, M		
-	jz	.L12_40
-
-	ALIGN_4
-
-.L12_31:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $12 * SIZE, BO
-
-	INIT2x12
-
-        movq    K, %rax
-
-	sarq	$3, %rax
-	je	.L12_36
-	ALIGN_4
-
-.L12_32:
-
-	KERNEL2x12_SUB
-	KERNEL2x12_SUB
-	KERNEL2x12_SUB
-	KERNEL2x12_SUB
-
-	KERNEL2x12_SUB
-	KERNEL2x12_SUB
-	KERNEL2x12_SUB
-	KERNEL2x12_SUB
-
-	dec %rax
-	jne	.L12_32
-	ALIGN_4
-
-.L12_36:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L12_39
-
-	ALIGN_4
-
-.L12_37:
-
-	KERNEL2x12_SUB
-
-	dec %rax
-	jne	.L12_37
-	ALIGN_4
-
-
-.L12_39:
-
-	SAVE2x12
-
-	ALIGN_4
-
-.L12_40:
-	testq	$1, M		
-	jz	.L12_100		// to next 3 lines of N
-
-	ALIGN_4
-
-.L12_41:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $12 * SIZE, BO
-
-	INIT1x12
-
-        movq    K, %rax
-
-	sarq	$3,%rax
-	je	.L12_46
-
-	ALIGN_4
-
-.L12_42:
-
-	KERNEL1x12_SUB
-	KERNEL1x12_SUB
-	KERNEL1x12_SUB
-	KERNEL1x12_SUB
-
-	KERNEL1x12_SUB
-	KERNEL1x12_SUB
-	KERNEL1x12_SUB
-	KERNEL1x12_SUB
-
-
-	dec %rax
-	jne	.L12_42
-	ALIGN_4
-
-.L12_46:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L12_49
-
-	ALIGN_4
-
-.L12_47:
-
-	KERNEL1x12_SUB
-
-	dec	%rax
-	jne	.L12_47
-	ALIGN_4
-
-
-.L12_49:
-
-	SAVE1x12
-
-	ALIGN_4
-	
-.L12_100:
-
-	decq	J			// j --
-	jg	.L12_01
-
-
-.L4_0:
-
-	cmpq	$ 0, Nmod12		// N % 12 == 0
-	je	.L999
-
-	movq	Nmod12, J		
-	sarq	$2, J			// j = j / 4
-	je	.L2_0
-
-.L4_10:
-	movq	C, CO1
-	leaq	(C, LDC, 4), C		// c += 4 * ldc
-
-	
-	movq	A, AO		 	// aoffset = a
-	addq	$16 * SIZE, AO
-
-	movq	M,  I
-	sarq	$2, I			// i = m / 4
-	je	.L4_20
-
-	ALIGN_4
-
-.L4_11:
-        movq    B, BO        
-        addq    $12 * SIZE, BO
-
-        movq    K, %rax
-
-	sarq	$3, %rax			//  K / 8
-	cmpq    $2, %rax
-	jl	.L4_13
-
-
-	KERNEL4x4_I
-	KERNEL4x4_M2
-	KERNEL4x4_M1
-	KERNEL4x4_M2
-
-	KERNEL4x4_M1
-	KERNEL4x4_M2
-	KERNEL4x4_M1
-	KERNEL4x4_M2
-
-	subq $2, %rax
-	je	.L4_12a
-
-	ALIGN_5
-
-.L4_12:
-
-	KERNEL4x4_M1
-	KERNEL4x4_M2
-	KERNEL4x4_M1
-	KERNEL4x4_M2
-
-	KERNEL4x4_M1
-	KERNEL4x4_M2
-	KERNEL4x4_M1
-	KERNEL4x4_M2
-
-	dec	%rax
-	jne	.L4_12
-
-.L4_12a:
-
-	KERNEL4x4_M1
-	KERNEL4x4_M2
-	KERNEL4x4_M1
-	KERNEL4x4_M2
-
-	KERNEL4x4_M1
-	KERNEL4x4_M2
-	KERNEL4x4_M1
-	KERNEL4x4_E
-
-	jmp .L4_16
-
-
-.L4_13:
-
-	test $1, %rax
-	jz .L4_14
-
-	KERNEL4x4_I
-	KERNEL4x4_M2
-	KERNEL4x4_M1
-	KERNEL4x4_M2
-
-	KERNEL4x4_M1
-	KERNEL4x4_M2
-	KERNEL4x4_M1
-	KERNEL4x4_E
-
-	jmp .L4_16
-
-
-.L4_14:
-
-	INIT4x4
-
-
-.L4_16:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L4_19
-
-	ALIGN_4
-
-.L4_17:
-
-	KERNEL4x4_SUB
-
-	dec	%rax
-	jne	.L4_17
-	ALIGN_4
-
-
-.L4_19:
-
-	SAVE4x4
-
-	decq	I			# i --
-	jg	.L4_11
-	ALIGN_4	
-
-/**************************************************************************
-* Rest of M 
-***************************************************************************/
-.L4_20:
-	// Test rest of M
-
-	testq	$3, M
-	jz	.L4_100			// to next 16 lines of N
-
-
-.L4_30:
-	testq	$2, M		
-	jz	.L4_40
-
-	ALIGN_4
-
-.L4_31:
-        movq    B, BO             // first buffer to BO
-        addq    $12 * SIZE, BO
-
-	INIT2x4
-
-        movq    K, %rax
-
-	sarq	$3, %rax
-	je	.L4_36
-	ALIGN_4
-
-.L4_32:
-
-	KERNEL2x4_SUB
-	KERNEL2x4_SUB
-	KERNEL2x4_SUB
-	KERNEL2x4_SUB
-
-	KERNEL2x4_SUB
-	KERNEL2x4_SUB
-	KERNEL2x4_SUB
-	KERNEL2x4_SUB
-
-	dec %rax
-	jne	.L4_32
-	ALIGN_4
-
-.L4_36:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L4_39
-
-	ALIGN_4
-
-.L4_37:
-
-	KERNEL2x4_SUB
-
-	dec %rax
-	jne	.L4_37
-
-
-.L4_39:
-
-	SAVE2x4
-
-.L4_40:
-	testq	$1, M		
-	jz	.L4_100		// to next 3 lines of N
-
-	ALIGN_4
-
-.L4_41:
-        movq    B, BO             // first buffer to BO
-        addq    $12 * SIZE, BO
-
-	INIT1x4
-
-        movq    K, %rax
-
-	sarq	$3,%rax
-	je	.L4_46
-
-	ALIGN_4
-
-.L4_42:
-
-	KERNEL1x4_SUB
-	KERNEL1x4_SUB
-	KERNEL1x4_SUB
-	KERNEL1x4_SUB
-
-	KERNEL1x4_SUB
-	KERNEL1x4_SUB
-	KERNEL1x4_SUB
-	KERNEL1x4_SUB
-
-	dec %rax
-	jne	.L4_42
-	ALIGN_4
-
-.L4_46:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L4_49
-
-	ALIGN_4
-
-.L4_47:
-
-	KERNEL1x4_SUB
-
-	dec	%rax
-	jne	.L4_47
-	ALIGN_4
-
-
-.L4_49:
-
-	SAVE1x4
-
-	ALIGN_4
-	
-.L4_100:
-
-	movq	K, %rax
-	salq	$2, %rax		// * 4
-	leaq	(B , %rax, SIZE), B
-	decq	J			// j --
-	jg	.L4_10
-
-
-
-
-/***************************************************************************************************************/
-
-.L2_0:
-
-	movq	Nmod12, J		
-	testq	$2, J
-	je	.L1_0
-
-.L2_10:
-	movq	C, CO1
-	leaq	(C, LDC, 2), C		// c += 2 * ldc
-
-	
-	movq	A, AO		 	// aoffset = a
-	addq	$16 * SIZE, AO
-
-	movq	M,  I
-	sarq	$2, I			// i = m / 4
-	je	.L2_20
-
-	ALIGN_4
-
-.L2_11:
-        movq    B, BO        
-        addq    $12 * SIZE, BO
-
-	INIT4x2
-
-        movq    K, %rax
-	sarq $3, %rax			//  K / 8
-
-	je	.L2_16
-
-	ALIGN_5
-
-.L2_12:
-
-	KERNEL4x2_SUB
-	KERNEL4x2_SUB
-	KERNEL4x2_SUB
-	KERNEL4x2_SUB
-
-	KERNEL4x2_SUB
-	KERNEL4x2_SUB
-	KERNEL4x2_SUB
-	KERNEL4x2_SUB
-
-	dec	%rax
-	jne	.L2_12
-
-
-.L2_16:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_19
-
-	ALIGN_4
-
-.L2_17:
-
-	KERNEL4x2_SUB
-
-	dec	%rax
-	jne	.L2_17
-	ALIGN_4
-
-
-.L2_19:
-
-	SAVE4x2
-
-	decq	I			# i --
-	jg	.L2_11
-	ALIGN_4	
-
-/**************************************************************************
-* Rest of M 
-***************************************************************************/
-.L2_20:
-	// Test rest of M
-
-	testq	$3, M
-	jz	.L2_100			// to next 16 lines of N
-
-
-.L2_30:
-	testq	$2, M		
-	jz	.L2_40
-
-	ALIGN_4
-
-.L2_31:
-        movq    B, BO             // first buffer to BO
-        addq    $12 * SIZE, BO
-
-	INIT2x2
-
-        movq    K, %rax
-
-	sarq	$3, %rax
-	je	.L2_36
-	ALIGN_4
-
-.L2_32:
-
-	KERNEL2x2_SUB
-	KERNEL2x2_SUB
-	KERNEL2x2_SUB
-	KERNEL2x2_SUB
-
-	KERNEL2x2_SUB
-	KERNEL2x2_SUB
-	KERNEL2x2_SUB
-	KERNEL2x2_SUB
-
-	dec %rax
-	jne	.L2_32
-
-.L2_36:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_39
-
-	ALIGN_4
-
-.L2_37:
-
-	KERNEL2x2_SUB
-
-	dec %rax
-	jne	.L2_37
-
-
-.L2_39:
-
-	SAVE2x2
-
-.L2_40:
-	testq	$1, M		
-	jz	.L2_100		// to next 3 lines of N
-
-.L2_41:
-        movq    B, BO             // first buffer to BO
-        addq    $12 * SIZE, BO
-
-	INIT1x2
-
-        movq    K, %rax
-
-	sarq	$3,%rax
-	je	.L2_46
-
-	ALIGN_4
-
-.L2_42:
-
-	KERNEL1x2_SUB
-	KERNEL1x2_SUB
-	KERNEL1x2_SUB
-	KERNEL1x2_SUB
-
-	KERNEL1x2_SUB
-	KERNEL1x2_SUB
-	KERNEL1x2_SUB
-	KERNEL1x2_SUB
-
-	dec %rax
-	jne	.L2_42
-
-.L2_46:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_49
-
-	ALIGN_4
-
-.L2_47:
-
-	KERNEL1x2_SUB
-
-	dec	%rax
-	jne	.L2_47
-
-.L2_49:
-
-	SAVE1x2
-
-.L2_100:
-
-	movq	K, %rax
-	salq	$1, %rax		// * 2
-	leaq	(B , %rax, SIZE), B
-
-/***************************************************************************************************************/
-
-.L1_0:
-
-	movq	Nmod12, J		
-	testq	$1, J
-	je	.L999
-
-.L1_10:
-	movq	C, CO1
-	leaq	(C, LDC, 1), C		// c += 1 * ldc
-
-	
-	movq	A, AO		 	// aoffset = a
-	addq	$16 * SIZE, AO
-
-	movq	M,  I
-	sarq	$2, I			// i = m / 4
-	je	.L1_20
-
-	ALIGN_4
-
-.L1_11:
-        movq    B, BO        
-        addq    $12 * SIZE, BO
-
-	INIT4x1
-
-        movq    K, %rax
-
-	sarq	$3, %rax			//  K / 8
-	je	.L1_16
-
-	ALIGN_5
-
-.L1_12:
-
-	KERNEL4x1
-
-	dec	%rax
-	jne	.L1_12
-
-
-.L1_16:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_19
-
-	ALIGN_4
-
-.L1_17:
-
-	KERNEL4x1_SUB
-
-	dec	%rax
-	jne	.L1_17
-	ALIGN_4
-
-
-.L1_19:
-
-	SAVE4x1
-
-	decq	I			# i --
-	jg	.L1_11
-
-/**************************************************************************
-* Rest of M 
-***************************************************************************/
-.L1_20:
-	// Test rest of M
-
-	testq	$3, M
-	jz	.L1_100	
-
-
-.L1_30:
-	testq	$2, M		
-	jz	.L1_40
-
-	ALIGN_4
-
-.L1_31:
-        movq    B, BO             // first buffer to BO
-        addq    $12 * SIZE, BO
-
-	INIT2x1
-
-        movq    K, %rax
-
-	sarq	$3, %rax
-	je	.L1_36
-	ALIGN_4
-
-.L1_32:
-
-	KERNEL2x1_SUB
-	KERNEL2x1_SUB
-	KERNEL2x1_SUB
-	KERNEL2x1_SUB
-
-	KERNEL2x1_SUB
-	KERNEL2x1_SUB
-	KERNEL2x1_SUB
-	KERNEL2x1_SUB
-
-
-	dec %rax
-	jne	.L1_32
-
-.L1_36:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_39
-
-	ALIGN_4
-
-.L1_37:
-
-	KERNEL2x1_SUB
-
-	dec %rax
-	jne	.L1_37
-
-.L1_39:
-
-	SAVE2x1
-
-.L1_40:
-	testq	$1, M		
-	jz	.L1_100		// to next 3 lines of N
-
-
-.L1_41:
-        movq    B, BO             // first buffer to BO
-        addq    $12 * SIZE, BO
-
-	INIT1x1
-
-        movq    K, %rax
-
-	sarq	$3,%rax
-	je	.L1_46
-
-	ALIGN_4
-
-.L1_42:
-
-	KERNEL1x1_SUB
-	KERNEL1x1_SUB
-	KERNEL1x1_SUB
-	KERNEL1x1_SUB
-
-	KERNEL1x1_SUB
-	KERNEL1x1_SUB
-	KERNEL1x1_SUB
-	KERNEL1x1_SUB
-
-	dec %rax
-	jne	.L1_42
-
-.L1_46:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_49
-
-	ALIGN_4
-
-.L1_47:
-
-	KERNEL1x1_SUB
-
-	dec	%rax
-	jne	.L1_47
-
-
-.L1_49:
-
-	SAVE1x1
-
-.L1_100:
-
-
-
-
-.L999:
-	vzeroupper
-
-	movq   		SP, %rsp
-	movq	   (%rsp), %rbx
-	movq	  8(%rsp), %rbp
-	movq	 16(%rsp), %r12
-	movq	 24(%rsp), %r13
-	movq	 32(%rsp), %r14
-	movq	 40(%rsp), %r15
-
-#ifdef WINDOWS_ABI
-	movq	 48(%rsp), %rdi
-	movq	 56(%rsp), %rsi
-	vmovups	 64(%rsp), %xmm6
-	vmovups	 80(%rsp), %xmm7
-	vmovups	 96(%rsp), %xmm8
-	vmovups	112(%rsp), %xmm9
-	vmovups	128(%rsp), %xmm10
-	vmovups	144(%rsp), %xmm11
-	vmovups	160(%rsp), %xmm12
-	vmovups	176(%rsp), %xmm13
-	vmovups	192(%rsp), %xmm14
-	vmovups	208(%rsp), %xmm15
-#endif
-
-	addq	$STACKSIZE, %rsp
-	ret
-
-	EPILOGUE
-
-
-#else
-/*************************************************************************************
-* TRMM Kernel
-*************************************************************************************/
-
-
-	PROLOGUE
-	PROFCODE
-	
-	subq	$STACKSIZE, %rsp
-	movq	%rbx,   (%rsp)
-	movq	%rbp,  8(%rsp)
-	movq	%r12, 16(%rsp)
-	movq	%r13, 24(%rsp)
-	movq	%r14, 32(%rsp)
-	movq	%r15, 40(%rsp)
-
-	vzeroupper
-
-#ifdef WINDOWS_ABI
-	movq	%rdi,    48(%rsp)
-	movq	%rsi,    56(%rsp)
-	vmovups	%xmm6,   64(%rsp)
-	vmovups	%xmm7,   80(%rsp)
-	vmovups	%xmm8,   96(%rsp)
-	vmovups	%xmm9,  112(%rsp)
-	vmovups	%xmm10, 128(%rsp)
-	vmovups	%xmm11, 144(%rsp)
-	vmovups	%xmm12, 160(%rsp)
-	vmovups	%xmm13, 176(%rsp)
-	vmovups	%xmm14, 192(%rsp)
-	vmovups	%xmm15, 208(%rsp)
-
-	movq	ARG1,      OLD_M
-	movq	ARG2,      OLD_N
-	movq	ARG3,      OLD_K
-	movq	OLD_A,     A
-	movq	OLD_B,     B
-	movq	OLD_C,     C
-	movq	OLD_LDC,   LDC
-#ifdef TRMMKERNEL
-	vmovsd	OLD_OFFSET, %xmm12
-#endif
-	vmovups	%xmm3, %xmm0
-
-#else
-	movq	STACKSIZE +  8(%rsp), LDC
-#ifdef TRMMKERNEL
-	vmovsd	STACKSIZE + 16(%rsp), %xmm12
-#endif
-
-#endif
-
-	movq    %rsp, SP      # save old stack
-        subq    $128 + L_BUFFER_SIZE, %rsp
-        andq    $-4096, %rsp    # align stack
-
-        STACK_TOUCH
-
-	cmpq	$ 0, OLD_M
-	je	.L999
-
-	cmpq	$ 0, OLD_N
-	je	.L999
-
-	cmpq	$ 0, OLD_K
-	je	.L999
-
-	movq	OLD_M, M
-	movq	OLD_N, N
-	movq	OLD_K, K
-
-	vmovsd	 %xmm0, ALPHA
-
-	salq	$BASE_SHIFT, LDC
-
-	movq    N, %rax
-        xorq    %rdx, %rdx
-        movq    $4,  %rdi
-        divq    %rdi                     //    N / 4
-        movq    %rax, Ndiv12             //    N / 4
-        movq    %rdx, Nmod12             //    N % 4
-
-#ifdef TRMMKERNEL
-        vmovsd  %xmm12, OFFSET
-        vmovsd  %xmm12, KK
-#ifndef LEFT
-        negq    KK
-#endif  
-#endif
-
-
-
-	movq	Ndiv12,  J
-	cmpq	$ 0, J
-	je	.L2_0
-	ALIGN_4
-
-.L4_10:
-	movq	C, CO1
-	leaq	(C, LDC, 4), C		// c += 4 * ldc
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        movq    OFFSET, %rax
-        movq    %rax, KK
-#endif
-
-	
-	movq	A, AO		 	// aoffset = a
-	addq	$16 * SIZE, AO
-
-	movq	M,  I
-	sarq	$2, I			// i = m / 4
-	je	.L4_20
-
-	ALIGN_4
-
-.L4_11:
-
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    B, BO        
-        addq    $12 * SIZE, BO
-#else
-        movq    B, BO        
-        addq    $12 * SIZE, BO
-        movq    KK, %rax
-	salq	$3, %rax		// rax * SIZE
-	leaq	(BO,%rax,4), BO		// add number of values in B
-	leaq	(AO,%rax,4), AO		// add number of values in A
-#endif
-
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $4, %rax        // number of values in AO
-#else
-        addq    $4, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-	sarq	$3, %rax			//  K / 8
-	cmpq    $2, %rax
-	jl	.L4_13
-
-
-	KERNEL4x4_I
-	KERNEL4x4_M2
-	KERNEL4x4_M1
-	KERNEL4x4_M2
-
-	KERNEL4x4_M1
-	KERNEL4x4_M2
-	KERNEL4x4_M1
-	KERNEL4x4_M2
-
-	subq $2, %rax
-	je	.L4_12a
-
-	ALIGN_5
-
-.L4_12:
-
-	KERNEL4x4_M1
-	KERNEL4x4_M2
-	KERNEL4x4_M1
-	KERNEL4x4_M2
-
-	KERNEL4x4_M1
-	KERNEL4x4_M2
-	KERNEL4x4_M1
-	KERNEL4x4_M2
-
-	dec	%rax
-	jne	.L4_12
-
-.L4_12a:
-
-	KERNEL4x4_M1
-	KERNEL4x4_M2
-	KERNEL4x4_M1
-	KERNEL4x4_M2
-
-	KERNEL4x4_M1
-	KERNEL4x4_M2
-	KERNEL4x4_M1
-	KERNEL4x4_E
-
-	jmp .L4_16
-
-
-.L4_13:
-
-	test $1, %rax
-	jz .L4_14
-
-	KERNEL4x4_I
-	KERNEL4x4_M2
-	KERNEL4x4_M1
-	KERNEL4x4_M2
-
-	KERNEL4x4_M1
-	KERNEL4x4_M2
-	KERNEL4x4_M1
-	KERNEL4x4_E
-
-	jmp .L4_16
-
-
-.L4_14:
-
-	INIT4x4
-
-
-.L4_16:
-        movq    KKK, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L4_19
-
-	ALIGN_4
-
-.L4_17:
-
-	KERNEL4x4_SUB
-
-	dec	%rax
-	jne	.L4_17
-	ALIGN_4
-
-
-.L4_19:
-
-	SAVE4x4
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-	salq	$3, %rax			// rax + SIZE
-        leaq    (BO, %rax, 4), BO		// number of values in B
-        leaq    (AO, %rax, 4), AO		// number of values in A
-#endif
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $4, KK				// number of values in A
-#endif
-
-	decq	I			# i --
-	jg	.L4_11
-	ALIGN_4	
-
-/**************************************************************************
-* Rest of M 
-***************************************************************************/
-.L4_20:
-	// Test rest of M
-
-	testq	$3, M
-	jz	.L4_100			// to next 16 lines of N
-
-
-.L4_30:
-	testq	$2, M		
-	jz	.L4_40
-
-	ALIGN_4
-
-.L4_31:
-
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    B, BO        
-        addq    $12 * SIZE, BO
-#else
-        movq    B, BO        
-        addq    $12 * SIZE, BO
-        movq    KK, %rax
-	salq	$3, %rax		// rax * SIZE
-	leaq	(BO,%rax,4), BO		// add number of values in B
-	leaq	(AO,%rax,2), AO		// add number of values in A
-#endif
-
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $2, %rax        // number of values in AO
-#else
-        addq    $4, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-	INIT2x4
-
-	sarq	$3, %rax
-	je	.L4_36
-	ALIGN_4
-
-.L4_32:
-
-	KERNEL2x4_SUB
-	KERNEL2x4_SUB
-	KERNEL2x4_SUB
-	KERNEL2x4_SUB
-
-	KERNEL2x4_SUB
-	KERNEL2x4_SUB
-	KERNEL2x4_SUB
-	KERNEL2x4_SUB
-
-	dec %rax
-	jne	.L4_32
-	ALIGN_4
-
-.L4_36:
-        movq    KKK, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L4_39
-
-	ALIGN_4
-
-.L4_37:
-
-	KERNEL2x4_SUB
-
-	dec %rax
-	jne	.L4_37
-
-
-.L4_39:
-
-	SAVE2x4
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-	salq	$3, %rax			// rax + SIZE
-        leaq    (BO, %rax, 4), BO		// number of values in B
-        leaq    (AO, %rax, 2), AO		// number of values in A
-#endif
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $2, KK				// number of values in A
-#endif
-
-
-.L4_40:
-	testq	$1, M		
-	jz	.L4_100		// to next 3 lines of N
-
-	ALIGN_4
-
-.L4_41:
-
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    B, BO        
-        addq    $12 * SIZE, BO
-#else
-        movq    B, BO        
-        addq    $12 * SIZE, BO
-        movq    KK, %rax
-	salq	$3, %rax		// rax * SIZE
-	leaq	(BO,%rax,4), BO		// add number of values in B
-	leaq	(AO,%rax,1), AO		// add number of values in A
-#endif
-
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $1, %rax        // number of values in AO
-#else
-        addq    $4, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-	INIT1x4
-
-	sarq	$3,%rax
-	je	.L4_46
-
-	ALIGN_4
-
-.L4_42:
-
-	KERNEL1x4_SUB
-	KERNEL1x4_SUB
-	KERNEL1x4_SUB
-	KERNEL1x4_SUB
-
-	KERNEL1x4_SUB
-	KERNEL1x4_SUB
-	KERNEL1x4_SUB
-	KERNEL1x4_SUB
-
-	dec %rax
-	jne	.L4_42
-	ALIGN_4
-
-.L4_46:
-        movq    KKK, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L4_49
-
-	ALIGN_4
-
-.L4_47:
-
-	KERNEL1x4_SUB
-
-	dec	%rax
-	jne	.L4_47
-	ALIGN_4
-
-
-.L4_49:
-
-	SAVE1x4
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-	salq	$3, %rax			// rax + SIZE
-        leaq    (BO, %rax, 4), BO		// number of values in B
-        leaq    (AO, %rax, 1), AO		// number of values in A
-#endif
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $1, KK				// number of values in A
-#endif
-
-.L4_100:
-
-#if defined(TRMMKERNEL) && !defined(LEFT)
-        addq    $4, KK				// number of values in B
-#endif
-
-
-	movq	K, %rax
-	salq	$2, %rax		// * 4
-	leaq	(B , %rax, SIZE), B
-	decq	J			// j --
-	jg	.L4_10
-
-
-
-
-/***************************************************************************************************************/
-
-.L2_0:
-
-	movq	Nmod12, J		
-	testq	$2, J
-	je	.L1_0
-
-.L2_10:
-	movq	C, CO1
-	leaq	(C, LDC, 2), C		// c += 2 * ldc
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        movq    OFFSET, %rax
-        movq    %rax, KK
-#endif
-
-
-	
-	movq	A, AO		 	// aoffset = a
-	addq	$16 * SIZE, AO
-
-	movq	M,  I
-	sarq	$2, I			// i = m / 4
-	je	.L2_20
-
-	ALIGN_4
-
-.L2_11:
-
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    B, BO        
-        addq    $12 * SIZE, BO
-#else
-        movq    B, BO        
-        addq    $12 * SIZE, BO
-        movq    KK, %rax
-	salq	$3, %rax		// rax * SIZE
-	leaq	(BO,%rax,2), BO		// add number of values in B
-	leaq	(AO,%rax,4), AO		// add number of values in A
-#endif
-
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $4, %rax        // number of values in AO
-#else
-        addq    $2, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-	INIT4x2
-
-	sarq $3, %rax			//  K / 8
-
-	je	.L2_16
-
-	ALIGN_5
-
-.L2_12:
-
-	KERNEL4x2_SUB
-	KERNEL4x2_SUB
-	KERNEL4x2_SUB
-	KERNEL4x2_SUB
-
-	KERNEL4x2_SUB
-	KERNEL4x2_SUB
-	KERNEL4x2_SUB
-	KERNEL4x2_SUB
-
-	dec	%rax
-	jne	.L2_12
-
-
-.L2_16:
-        movq    KKK, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_19
-
-	ALIGN_4
-
-.L2_17:
-
-	KERNEL4x2_SUB
-
-	dec	%rax
-	jne	.L2_17
-	ALIGN_4
-
-
-.L2_19:
-
-	SAVE4x2
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-	salq	$3, %rax			// rax + SIZE
-        leaq    (BO, %rax, 2), BO		// number of values in B
-        leaq    (AO, %rax, 4), AO		// number of values in A
-#endif
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $4, KK				// number of values in A
-#endif
-
-
-	decq	I			# i --
-	jg	.L2_11
-	ALIGN_4	
-
-/**************************************************************************
-* Rest of M 
-***************************************************************************/
-.L2_20:
-	// Test rest of M
-
-	testq	$3, M
-	jz	.L2_100			// to next 16 lines of N
-
-
-.L2_30:
-	testq	$2, M		
-	jz	.L2_40
-
-	ALIGN_4
-
-.L2_31:
-
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    B, BO        
-        addq    $12 * SIZE, BO
-#else
-        movq    B, BO        
-        addq    $12 * SIZE, BO
-        movq    KK, %rax
-	salq	$3, %rax		// rax * SIZE
-	leaq	(BO,%rax,2), BO		// add number of values in B
-	leaq	(AO,%rax,2), AO		// add number of values in A
-#endif
-
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $2, %rax        // number of values in AO
-#else
-        addq    $2, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-	INIT2x2
-
-	sarq	$3, %rax
-	je	.L2_36
-	ALIGN_4
-
-.L2_32:
-
-	KERNEL2x2_SUB
-	KERNEL2x2_SUB
-	KERNEL2x2_SUB
-	KERNEL2x2_SUB
-
-	KERNEL2x2_SUB
-	KERNEL2x2_SUB
-	KERNEL2x2_SUB
-	KERNEL2x2_SUB
-
-	dec %rax
-	jne	.L2_32
-
-.L2_36:
-        movq    KKK, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_39
-
-	ALIGN_4
-
-.L2_37:
-
-	KERNEL2x2_SUB
-
-	dec %rax
-	jne	.L2_37
-
-
-.L2_39:
-
-	SAVE2x2
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-	salq	$3, %rax			// rax + SIZE
-        leaq    (BO, %rax, 2), BO		// number of values in B
-        leaq    (AO, %rax, 2), AO		// number of values in A
-#endif
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $2, KK				// number of values in A
-#endif
-
-
-.L2_40:
-	testq	$1, M		
-	jz	.L2_100		// to next 3 lines of N
-
-.L2_41:
-
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    B, BO        
-        addq    $12 * SIZE, BO
-#else
-        movq    B, BO        
-        addq    $12 * SIZE, BO
-        movq    KK, %rax
-	salq	$3, %rax		// rax * SIZE
-	leaq	(BO,%rax,2), BO		// add number of values in B
-	leaq	(AO,%rax,1), AO		// add number of values in A
-#endif
-
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $1, %rax        // number of values in AO
-#else
-        addq    $2, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-	INIT1x2
-
-	sarq	$3,%rax
-	je	.L2_46
-
-	ALIGN_4
-
-.L2_42:
-
-	KERNEL1x2_SUB
-	KERNEL1x2_SUB
-	KERNEL1x2_SUB
-	KERNEL1x2_SUB
-
-	KERNEL1x2_SUB
-	KERNEL1x2_SUB
-	KERNEL1x2_SUB
-	KERNEL1x2_SUB
-
-	dec %rax
-	jne	.L2_42
-
-.L2_46:
-        movq    KKK, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_49
-
-	ALIGN_4
-
-.L2_47:
-
-	KERNEL1x2_SUB
-
-	dec	%rax
-	jne	.L2_47
-
-.L2_49:
-
-	SAVE1x2
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-	salq	$3, %rax			// rax * SIZE
-        leaq    (BO, %rax, 2), BO		// number of values in B
-        leaq    (AO, %rax, 1), AO		// number of values in A
-#endif
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $1, KK				// number of values in A
-#endif
-
-
-.L2_100:
-
-
-#if defined(TRMMKERNEL) && !defined(LEFT)
-        addq    $2, KK				// number of values in B
-#endif
-
-	movq	K, %rax
-	salq	$1, %rax		// * 2
-	leaq	(B , %rax, SIZE), B
-
-/***************************************************************************************************************/
-
-.L1_0:
-
-	movq	Nmod12, J		
-	testq	$1, J
-	je	.L999
-
-.L1_10:
-	movq	C, CO1
-	leaq	(C, LDC, 1), C		// c += 1 * ldc
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        movq    OFFSET, %rax
-        movq    %rax, KK
-#endif
-	
-	movq	A, AO		 	// aoffset = a
-	addq	$16 * SIZE, AO
-
-	movq	M,  I
-	sarq	$2, I			// i = m / 4
-	je	.L1_20
-
-	ALIGN_4
-
-.L1_11:
-
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    B, BO        
-        addq    $12 * SIZE, BO
-#else
-        movq    B, BO        
-        addq    $12 * SIZE, BO
-        movq    KK, %rax
-	salq	$3, %rax		// rax * SIZE
-	leaq	(BO,%rax,1), BO		// add number of values in B
-	leaq	(AO,%rax,4), AO		// add number of values in A
-#endif
-
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $4, %rax        // number of values in AO
-#else
-        addq    $1, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-	INIT4x1
-
-	sarq	$3, %rax			//  K / 8
-	je	.L1_16
-
-	ALIGN_5
-
-.L1_12:
-
-	KERNEL4x1
-
-	dec	%rax
-	jne	.L1_12
-
-
-.L1_16:
-        movq    KKK, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_19
-
-	ALIGN_4
-
-.L1_17:
-
-	KERNEL4x1_SUB
-
-	dec	%rax
-	jne	.L1_17
-	ALIGN_4
-
-
-.L1_19:
-
-	SAVE4x1
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-	salq	$3, %rax			// rax * SIZE
-        leaq    (BO, %rax, 1), BO		// number of values in B
-        leaq    (AO, %rax, 4), AO		// number of values in A
-#endif
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $4, KK				// number of values in A
-#endif
-
-
-	decq	I			# i --
-	jg	.L1_11
-
-/**************************************************************************
-* Rest of M 
-***************************************************************************/
-.L1_20:
-	// Test rest of M
-
-	testq	$3, M
-	jz	.L1_100	
-
-
-.L1_30:
-	testq	$2, M		
-	jz	.L1_40
-
-	ALIGN_4
-
-.L1_31:
-
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    B, BO        
-        addq    $12 * SIZE, BO
-#else
-        movq    B, BO        
-        addq    $12 * SIZE, BO
-        movq    KK, %rax
-	salq	$3, %rax		// rax * SIZE
-	leaq	(BO,%rax,1), BO		// add number of values in B
-	leaq	(AO,%rax,2), AO		// add number of values in A
-#endif
-
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $2, %rax        // number of values in AO
-#else
-        addq    $1, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-	INIT2x1
-
-	sarq	$3, %rax
-	je	.L1_36
-	ALIGN_4
-
-.L1_32:
-
-	KERNEL2x1_SUB
-	KERNEL2x1_SUB
-	KERNEL2x1_SUB
-	KERNEL2x1_SUB
-
-	KERNEL2x1_SUB
-	KERNEL2x1_SUB
-	KERNEL2x1_SUB
-	KERNEL2x1_SUB
-
-
-	dec %rax
-	jne	.L1_32
-
-.L1_36:
-        movq    KKK, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_39
-
-	ALIGN_4
-
-.L1_37:
-
-	KERNEL2x1_SUB
-
-	dec %rax
-	jne	.L1_37
-
-.L1_39:
-
-	SAVE2x1
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-	salq	$3, %rax			// rax * SIZE
-        leaq    (BO, %rax, 1), BO		// number of values in B
-        leaq    (AO, %rax, 2), AO		// number of values in A
-#endif
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $2, KK				// number of values in A
-#endif
-
-
-.L1_40:
-	testq	$1, M		
-	jz	.L1_100		// to next 3 lines of N
-
-
-.L1_41:
-
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    B, BO        
-        addq    $12 * SIZE, BO
-#else
-        movq    B, BO        
-        addq    $12 * SIZE, BO
-        movq    KK, %rax
-	salq	$3, %rax		// rax * SIZE
-	leaq	(BO,%rax,1), BO		// add number of values in B
-	leaq	(AO,%rax,1), AO		// add number of values in A
-#endif
-
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $1, %rax        // number of values in AO
-#else
-        addq    $1, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-	INIT1x1
-
-	sarq	$3,%rax
-	je	.L1_46
-
-	ALIGN_4
-
-.L1_42:
-
-	KERNEL1x1_SUB
-	KERNEL1x1_SUB
-	KERNEL1x1_SUB
-	KERNEL1x1_SUB
-
-	KERNEL1x1_SUB
-	KERNEL1x1_SUB
-	KERNEL1x1_SUB
-	KERNEL1x1_SUB
-
-	dec %rax
-	jne	.L1_42
-
-.L1_46:
-        movq    KKK, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_49
-
-	ALIGN_4
-
-.L1_47:
-
-	KERNEL1x1_SUB
-
-	dec	%rax
-	jne	.L1_47
-
-
-.L1_49:
-
-	SAVE1x1
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-	salq	$3, %rax			// rax * SIZE
-        leaq    (BO, %rax, 1), BO		// number of values in B
-        leaq    (AO, %rax, 1), AO		// number of values in A
-#endif
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $1, KK				// number of values in A
-#endif
-
-
-
-.L1_100:
-
-
-#if defined(TRMMKERNEL) && !defined(LEFT)
-        addq    $1, KK				// number of values in B
-#endif
-
-
-
-.L999:
-
-	vzeroupper
-
-	movq   		SP, %rsp
-	movq	   (%rsp), %rbx
-	movq	  8(%rsp), %rbp
-	movq	 16(%rsp), %r12
-	movq	 24(%rsp), %r13
-	movq	 32(%rsp), %r14
-	movq	 40(%rsp), %r15
-
-#ifdef WINDOWS_ABI
-	movq	 48(%rsp), %rdi
-	movq	 56(%rsp), %rsi
-	vmovups	 64(%rsp), %xmm6
-	vmovups	 80(%rsp), %xmm7
-	vmovups	 96(%rsp), %xmm8
-	vmovups	112(%rsp), %xmm9
-	vmovups	128(%rsp), %xmm10
-	vmovups	144(%rsp), %xmm11
-	vmovups	160(%rsp), %xmm12
-	vmovups	176(%rsp), %xmm13
-	vmovups	192(%rsp), %xmm14
-	vmovups	208(%rsp), %xmm15
-#endif
-
-	addq	$STACKSIZE, %rsp
-	ret
-
-	EPILOGUE
-
-
-
-
-
-#endif
+/*********************************************************************************
+Copyright (c) 2013, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+**********************************************************************************/
+
+
+/*********************************************************************
+* 2013/10/28 Saar
+*        BLASTEST               : OK
+*        CTEST                  : OK
+*        TEST                   : OK
+
+*
+*
+* 2013/10/27 Saar
+* Parameter:
+*       DGEMM_DEFAULT_UNROLL_N  4
+*       DGEMM_DEFAULT_UNROLL_M  4
+*       DGEMM_DEFAULT_P         512
+*       DGEMM_DEFAULT_Q         256
+*	A_PR1			512
+*	B_PR1			512
+*
+*
+* Performance at 9216x9216x9216:
+*       1 thread:       53.3 GFLOPS (MKL:  54)
+*       2 threads:     100.0 GFLOPS (MKL:  97)
+*       3 threads:     147.0 GFLOPS (MKL: 133)
+*       4 threads:     184.0 GFLOPS (MKL: 170)
+*********************************************************************/
+
+
+#define ASSEMBLER
+#include "common.h"
+ 
+#define OLD_M	%rdi
+#define OLD_N	%rsi
+#define M	%r13
+#define J	%r14
+#define OLD_K	%rdx
+
+#define A	%rcx
+#define B	%r8
+#define C	%r9
+#define LDC	%r10
+	
+#define I	%r11
+#define AO	%rdi
+#define BO	%rsi
+#define	CO1	%r15
+#define K	%r12
+#define	SP	%rbx
+
+#define BO1	%rdi
+#define BO2	%r15
+#define BO3	%rbp
+
+#ifndef WINDOWS_ABI
+
+#define STACKSIZE 96
+#define L_BUFFER_SIZE 256*8*12+4096
+
+#else
+
+#define STACKSIZE 256
+#define L_BUFFER_SIZE 128*8*12+512
+
+#define OLD_A		40 + STACKSIZE(%rsp)
+#define OLD_B		48 + STACKSIZE(%rsp)
+#define OLD_C		56 + STACKSIZE(%rsp)
+#define OLD_LDC		64 + STACKSIZE(%rsp)
+#define OLD_OFFSET	72 + STACKSIZE(%rsp)
+
+#endif
+
+
+#define Ndiv12	 24(%rsp)
+#define Nmod12	 32(%rsp)
+#define N	 40(%rsp)
+#define ALPHA	 48(%rsp)
+#define OFFSET	 56(%rsp)
+#define KK	 64(%rsp)
+#define KKK	 72(%rsp)
+#define BUFFER1	           128(%rsp)
+
+#if defined(OS_WINDOWS)
+#if   L_BUFFER_SIZE > 16384
+#define STACK_TOUCH \
+        movl    $ 0,  4096 * 4(%rsp);\
+        movl    $ 0,  4096 * 3(%rsp);\
+        movl    $ 0,  4096 * 2(%rsp);\
+        movl    $ 0,  4096 * 1(%rsp);
+#elif L_BUFFER_SIZE > 12288
+#define STACK_TOUCH \
+        movl    $ 0,  4096 * 3(%rsp);\
+        movl    $ 0,  4096 * 2(%rsp);\
+        movl    $ 0,  4096 * 1(%rsp);
+#elif L_BUFFER_SIZE > 8192
+#define STACK_TOUCH \
+        movl    $ 0,  4096 * 2(%rsp);\
+        movl    $ 0,  4096 * 1(%rsp);
+#elif L_BUFFER_SIZE > 4096
+#define STACK_TOUCH \
+        movl    $ 0,  4096 * 1(%rsp);
+#else
+#define STACK_TOUCH
+#endif
+#else
+#define STACK_TOUCH
+#endif
+
+#define	A_PR1	512
+#define	B_PR1	512
+
+/*******************************************************************************************
+* Macro definitions
+*******************************************************************************************/
+
+.macro INIT4x12
+
+	vxorpd		%ymm4 , %ymm4 , %ymm4
+	vxorpd		%ymm5 , %ymm5 , %ymm5
+	vxorpd		%ymm6 , %ymm6 , %ymm6
+	vxorpd		%ymm7 , %ymm7 , %ymm7
+	vxorpd		%ymm8 , %ymm8 , %ymm8
+	vxorpd		%ymm9 , %ymm9 , %ymm9
+	vxorpd		%ymm10, %ymm10, %ymm10
+	vxorpd		%ymm11, %ymm11, %ymm11
+	vxorpd		%ymm12, %ymm12, %ymm12
+	vxorpd		%ymm13, %ymm13, %ymm13
+	vxorpd		%ymm14, %ymm14, %ymm14
+	vxorpd		%ymm15, %ymm15, %ymm15
+
+.endm
+
+.macro KERNEL4x12_I
+	prefetcht0	A_PR1(AO)
+	vmovups		-12 * SIZE(BO), %ymm1
+	prefetcht0	B_PR1(BO)
+	vmovups 	-16 * SIZE(AO), %ymm0
+	prefetcht0	B_PR1+64(BO)
+	vmovups		 -8 * SIZE(BO), %ymm2
+	prefetcht0	B_PR1+128(BO)
+	vmovups		 -4 * SIZE(BO), %ymm3
+	vmulpd  	%ymm0 ,%ymm1  , %ymm4
+	prefetcht0	B_PR1+192(BO)
+	vmulpd  	%ymm0 ,%ymm2  , %ymm8
+	vmulpd  	%ymm0 ,%ymm3  , %ymm12
+	prefetcht0	B_PR1+256(BO)
+	vpermpd		$ 0xb1, %ymm0  , %ymm0
+	vmulpd  	%ymm0 ,%ymm1  , %ymm5
+	vmulpd  	%ymm0 ,%ymm2  , %ymm9
+	vmulpd  	%ymm0 ,%ymm3  , %ymm13
+	vpermpd		$ 0x1b, %ymm0  , %ymm0
+	vmulpd  	%ymm0 ,%ymm1  , %ymm6
+	vmulpd  	%ymm0 ,%ymm2  , %ymm10
+
+	addq		$ 12*SIZE, BO
+	vmulpd  	%ymm0 ,%ymm3  , %ymm14
+	vpermpd		$ 0xb1, %ymm0  , %ymm0
+	vmulpd  	%ymm0 ,%ymm1  , %ymm7
+	vmovups		-12 * SIZE(BO), %ymm1
+	vmulpd  	%ymm0 ,%ymm2  , %ymm11
+	vmovups		 -8 * SIZE(BO), %ymm2
+	vmulpd  	%ymm0 ,%ymm3  , %ymm15
+	vmovups		 -4 * SIZE(BO), %ymm3
+
+.endm
+
+.macro KERNEL4x12_M1
+	prefetcht0	A_PR1(AO)
+	vmovups 	-16 * SIZE(AO), %ymm0
+	prefetcht0	B_PR1(BO)
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4
+	prefetcht0	B_PR1+64(BO)
+	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm8
+	prefetcht0	B_PR1+128(BO)
+	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm12
+	vpermpd		$ 0xb1, %ymm0  , %ymm0
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5
+	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm9
+	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm13
+	vpermpd		$ 0x1b, %ymm0  , %ymm0
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6
+	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm10
+
+	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm14
+	vpermpd		$ 0xb1, %ymm0  , %ymm0
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7
+	vmovups		-12 * SIZE(BO), %ymm1
+	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm11
+	vmovups		 -8 * SIZE(BO), %ymm2
+	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm15
+	vmovups		 -4 * SIZE(BO), %ymm3
+
+.endm
+
+.macro KERNEL4x12_M2
+	vmovups 	-12 * SIZE(AO), %ymm0
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4
+	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm8
+	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm12
+	vpermpd		$ 0xb1, %ymm0  , %ymm0
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5
+	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm9
+	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm13
+	vpermpd		$ 0x1b, %ymm0  , %ymm0
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6
+	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm10
+
+	addq		$ 8*SIZE, AO
+	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm14
+	vpermpd		$ 0xb1, %ymm0  , %ymm0
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7
+	vmovups		  0 * SIZE(BO), %ymm1
+	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm11
+	vmovups		  4 * SIZE(BO), %ymm2
+	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm15
+	vmovups		  8 * SIZE(BO), %ymm3
+	addq		$ 24*SIZE, BO
+.endm
+
+
+.macro KERNEL4x12_E
+	vmovups 	-12 * SIZE(AO), %ymm0
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4
+	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm8
+	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm12
+	vpermpd		$ 0xb1, %ymm0  , %ymm0
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5
+	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm9
+	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm13
+	vpermpd		$ 0x1b, %ymm0  , %ymm0
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6
+	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm10
+
+	addq		$ 8*SIZE, AO
+	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm14
+	vpermpd		$ 0xb1, %ymm0  , %ymm0
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7
+	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm11
+	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm15
+	addq		$ 12*SIZE, BO
+.endm
+
+.macro KERNEL4x12_SUB
+	vmovups		-12 * SIZE(BO), %ymm1
+	vmovups 	-16 * SIZE(AO), %ymm0
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4
+	vmovups		 -8 * SIZE(BO), %ymm2
+	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm8
+	vmovups		 -4 * SIZE(BO), %ymm3
+	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm12
+	vpermpd		$ 0xb1, %ymm0  , %ymm0
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5
+	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm9
+	addq		$ 12*SIZE, BO
+	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm13
+	vpermpd		$ 0x1b, %ymm0  , %ymm0
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6
+	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm10
+	addq		$ 4*SIZE, AO
+	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm14
+	vpermpd		$ 0xb1, %ymm0  , %ymm0
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7
+	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm11
+	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm15
+
+.endm
+
+
+.macro SAVE4x12
+
+	vbroadcastsd	ALPHA, %ymm0
+
+	vmulpd	%ymm0 , %ymm4 , %ymm4
+	vmulpd	%ymm0 , %ymm5 , %ymm5
+	vmulpd	%ymm0 , %ymm6 , %ymm6
+	vmulpd	%ymm0 , %ymm7 , %ymm7
+
+	vmulpd	%ymm0 , %ymm8 , %ymm8
+	vmulpd	%ymm0 , %ymm9 , %ymm9
+	vmulpd	%ymm0 , %ymm10, %ymm10
+	vmulpd	%ymm0 , %ymm11, %ymm11
+
+	vmulpd	%ymm0 , %ymm12, %ymm12
+	vmulpd	%ymm0 , %ymm13, %ymm13
+	vmulpd	%ymm0 , %ymm14, %ymm14
+	vmulpd	%ymm0 , %ymm15, %ymm15
+
+	vpermpd $ 0xb1 , %ymm5, %ymm5
+	vpermpd $ 0xb1 , %ymm7, %ymm7
+
+	vblendpd $ 0x0a, %ymm5, %ymm4, %ymm0
+	vblendpd $ 0x05, %ymm5, %ymm4, %ymm1
+	vblendpd $ 0x0a, %ymm7, %ymm6, %ymm2
+	vblendpd $ 0x05, %ymm7, %ymm6, %ymm3
+
+	vpermpd $ 0x1b , %ymm2, %ymm2
+	vpermpd $ 0x1b , %ymm3, %ymm3
+	vpermpd $ 0xb1 , %ymm2, %ymm2
+	vpermpd $ 0xb1 , %ymm3, %ymm3
+
+	vblendpd $ 0x03, %ymm0, %ymm2 , %ymm4
+	vblendpd $ 0x03, %ymm1, %ymm3 , %ymm5
+	vblendpd $ 0x03, %ymm2, %ymm0 , %ymm6
+	vblendpd $ 0x03, %ymm3, %ymm1 , %ymm7
+
+        leaq    (CO1, LDC, 2), %rax     
+	
+
+#if !defined(TRMMKERNEL)
+
+	vaddpd 	                (CO1), %ymm4, %ymm4
+	vaddpd 	           (CO1, LDC), %ymm5, %ymm5
+	vaddpd 	               (%rax), %ymm6, %ymm6
+	vaddpd 	          (%rax, LDC), %ymm7, %ymm7
+
+#endif
+
+	vmovups	%ymm4 ,  	(CO1)
+	vmovups	%ymm5 ,  	(CO1, LDC)
+	vmovups	%ymm6 ,  	(%rax)
+	vmovups	%ymm7 ,  	(%rax, LDC)
+
+	prefetcht0	32(CO1)
+	prefetcht0	32(CO1,LDC)
+	prefetcht0	32(%rax)
+	prefetcht0	32(%rax,LDC)
+
+	vpermpd $ 0xb1 , %ymm9 , %ymm9
+	vpermpd $ 0xb1 , %ymm11, %ymm11
+
+	vblendpd $ 0x0a, %ymm9 , %ymm8 , %ymm0
+	vblendpd $ 0x05, %ymm9 , %ymm8 , %ymm1
+	vblendpd $ 0x0a, %ymm11, %ymm10, %ymm2
+	vblendpd $ 0x05, %ymm11, %ymm10, %ymm3
+
+	vpermpd $ 0x1b , %ymm2, %ymm2
+	vpermpd $ 0x1b , %ymm3, %ymm3
+	vpermpd $ 0xb1 , %ymm2, %ymm2
+	vpermpd $ 0xb1 , %ymm3, %ymm3
+
+	vblendpd $ 0x03, %ymm0, %ymm2 , %ymm4
+	vblendpd $ 0x03, %ymm1, %ymm3 , %ymm5
+	vblendpd $ 0x03, %ymm2, %ymm0 , %ymm6
+	vblendpd $ 0x03, %ymm3, %ymm1 , %ymm7
+
+
+	leaq	(%rax, LDC, 2), %rax
+	leaq	(%rax, LDC, 2), %rbp
+
+#if !defined(TRMMKERNEL)
+
+	vaddpd 	                (%rax), %ymm4, %ymm4
+	vaddpd 	           (%rax, LDC), %ymm5, %ymm5
+	vaddpd 	                (%rbp), %ymm6, %ymm6
+	vaddpd 	           (%rbp, LDC), %ymm7, %ymm7
+
+#endif
+
+	vmovups	%ymm4 ,  	(%rax)
+	vmovups	%ymm5 ,  	(%rax, LDC)
+	vmovups	%ymm6 ,  	(%rbp)
+	vmovups	%ymm7 ,  	(%rbp, LDC)
+
+	prefetcht0	32(%rax)
+	prefetcht0	32(%rax,LDC)
+	prefetcht0	32(%rbp)
+	prefetcht0	32(%rbp,LDC)
+
+	vpermpd $ 0xb1 , %ymm13, %ymm13
+	vpermpd $ 0xb1 , %ymm15, %ymm15
+
+	vblendpd $ 0x0a, %ymm13, %ymm12, %ymm0
+	vblendpd $ 0x05, %ymm13, %ymm12, %ymm1
+	vblendpd $ 0x0a, %ymm15, %ymm14, %ymm2
+	vblendpd $ 0x05, %ymm15, %ymm14, %ymm3
+
+	vpermpd $ 0x1b , %ymm2, %ymm2
+	vpermpd $ 0x1b , %ymm3, %ymm3
+	vpermpd $ 0xb1 , %ymm2, %ymm2
+	vpermpd $ 0xb1 , %ymm3, %ymm3
+
+	vblendpd $ 0x03, %ymm0, %ymm2 , %ymm4
+	vblendpd $ 0x03, %ymm1, %ymm3 , %ymm5
+	vblendpd $ 0x03, %ymm2, %ymm0 , %ymm6
+	vblendpd $ 0x03, %ymm3, %ymm1 , %ymm7
+
+
+	leaq	(%rax, LDC, 4), %rax
+	leaq	(%rbp, LDC, 4), %rbp
+
+#if !defined(TRMMKERNEL)
+
+	vaddpd 	                (%rax), %ymm4, %ymm4
+	vaddpd 	           (%rax, LDC), %ymm5, %ymm5
+	vaddpd 	                (%rbp), %ymm6, %ymm6
+	vaddpd 	           (%rbp, LDC), %ymm7, %ymm7
+
+#endif
+
+	vmovups	%ymm4 ,  	(%rax)
+	vmovups	%ymm5 ,  	(%rax, LDC)
+	vmovups	%ymm6 ,  	(%rbp)
+	vmovups	%ymm7 ,  	(%rbp, LDC)
+
+	prefetcht0	32(%rax)
+	prefetcht0	32(%rax,LDC)
+	prefetcht0	32(%rbp)
+	prefetcht0	32(%rbp,LDC)
+
+	addq	$ 4*SIZE, CO1
+.endm
+
+/******************************************************************************************/
+
+.macro INIT2x12
+
+	vxorpd		%xmm4 , %xmm4 , %xmm4
+	vxorpd		%xmm5 , %xmm5 , %xmm5
+	vxorpd		%xmm6 , %xmm6 , %xmm6
+	vxorpd		%xmm7 , %xmm7 , %xmm7
+	vxorpd		%xmm8 , %xmm8 , %xmm8
+	vxorpd		%xmm9 , %xmm9 , %xmm9
+	vxorpd		%xmm10, %xmm10, %xmm10
+	vxorpd		%xmm11, %xmm11, %xmm11
+	vxorpd		%xmm12, %xmm12, %xmm12
+	vxorpd		%xmm13, %xmm13, %xmm13
+	vxorpd		%xmm14, %xmm14, %xmm14
+	vxorpd		%xmm15, %xmm15, %xmm15
+
+.endm
+
+.macro KERNEL2x12_SUB
+	vmovups 	-16 * SIZE(AO), %xmm0
+	vmovddup	-12 * SIZE(BO), %xmm1
+	vmovddup	-11 * SIZE(BO), %xmm2
+	vmovddup	-10 * SIZE(BO), %xmm3
+	vfmadd231pd  	%xmm0 ,%xmm1  , %xmm4
+	vmovddup	 -9 * SIZE(BO), %xmm1
+	vfmadd231pd  	%xmm0 ,%xmm2  , %xmm5
+	vmovddup	 -8 * SIZE(BO), %xmm2
+	vfmadd231pd  	%xmm0 ,%xmm3  , %xmm6
+	vmovddup	 -7 * SIZE(BO), %xmm3
+	vfmadd231pd  	%xmm0 ,%xmm1  , %xmm7
+	vmovddup	 -6 * SIZE(BO), %xmm1
+	vfmadd231pd  	%xmm0 ,%xmm2  , %xmm8
+	vmovddup	 -5 * SIZE(BO), %xmm2
+	vfmadd231pd  	%xmm0 ,%xmm3  , %xmm9
+	vmovddup	 -4 * SIZE(BO), %xmm3
+	vfmadd231pd  	%xmm0 ,%xmm1  , %xmm10
+	vmovddup	 -3 * SIZE(BO), %xmm1
+	vfmadd231pd  	%xmm0 ,%xmm2  , %xmm11
+	vmovddup	 -2 * SIZE(BO), %xmm2
+	vfmadd231pd  	%xmm0 ,%xmm3  , %xmm12
+	vmovddup	 -1 * SIZE(BO), %xmm3
+	vfmadd231pd  	%xmm0 ,%xmm1  , %xmm13
+	addq		$ 12*SIZE, BO
+	vfmadd231pd  	%xmm0 ,%xmm2  , %xmm14
+	addq		$ 2*SIZE, AO
+	vfmadd231pd  	%xmm0 ,%xmm3  , %xmm15
+
+.endm
+
+.macro SAVE2x12
+
+	vmovddup	ALPHA, %xmm0
+
+	vmulpd	%xmm0 , %xmm4 , %xmm4
+	vmulpd	%xmm0 , %xmm5 , %xmm5
+	vmulpd	%xmm0 , %xmm6 , %xmm6
+	vmulpd	%xmm0 , %xmm7 , %xmm7
+
+	vmulpd	%xmm0 , %xmm8 , %xmm8
+	vmulpd	%xmm0 , %xmm9 , %xmm9
+	vmulpd	%xmm0 , %xmm10, %xmm10
+	vmulpd	%xmm0 , %xmm11, %xmm11
+
+	vmulpd	%xmm0 , %xmm12, %xmm12
+	vmulpd	%xmm0 , %xmm13, %xmm13
+	vmulpd	%xmm0 , %xmm14, %xmm14
+	vmulpd	%xmm0 , %xmm15, %xmm15
+
+
+        leaq    (CO1, LDC, 2), %rax     
+	
+
+#if !defined(TRMMKERNEL)
+
+	vaddpd 	                (CO1), %xmm4, %xmm4
+	vaddpd 	           (CO1, LDC), %xmm5, %xmm5
+	vaddpd 	               (%rax), %xmm6, %xmm6
+	vaddpd 	          (%rax, LDC), %xmm7, %xmm7
+
+#endif
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm5 ,  	(CO1, LDC)
+	vmovups	%xmm6 ,  	(%rax)
+	vmovups	%xmm7 ,  	(%rax, LDC)
+
+
+	leaq	(%rax, LDC, 2), %rax
+	leaq	(%rax, LDC, 2), %rbp
+
+#if !defined(TRMMKERNEL)
+
+	vaddpd 	                (%rax), %xmm8 , %xmm4
+	vaddpd 	           (%rax, LDC), %xmm9 , %xmm5
+	vaddpd 	                (%rbp), %xmm10, %xmm6
+	vaddpd 	           (%rbp, LDC), %xmm11, %xmm7
+
+#endif
+
+	vmovups	%xmm4 ,  	(%rax)
+	vmovups	%xmm5 ,  	(%rax, LDC)
+	vmovups	%xmm6 ,  	(%rbp)
+	vmovups	%xmm7 ,  	(%rbp, LDC)
+
+
+	leaq	(%rax, LDC, 4), %rax
+	leaq	(%rbp, LDC, 4), %rbp
+
+#if !defined(TRMMKERNEL)
+
+	vaddpd 	                (%rax), %xmm12, %xmm4
+	vaddpd 	           (%rax, LDC), %xmm13, %xmm5
+	vaddpd 	                (%rbp), %xmm14, %xmm6
+	vaddpd 	           (%rbp, LDC), %xmm15, %xmm7
+
+#endif
+
+	vmovups	%xmm4 ,  	(%rax)
+	vmovups	%xmm5 ,  	(%rax, LDC)
+	vmovups	%xmm6 ,  	(%rbp)
+	vmovups	%xmm7 ,  	(%rbp, LDC)
+
+	addq	$ 2*SIZE, CO1
+.endm
+
+
+/******************************************************************************************/
+
+.macro INIT1x12
+
+	vxorpd		%xmm4 , %xmm4 , %xmm4
+	vxorpd		%xmm5 , %xmm5 , %xmm5
+	vxorpd		%xmm6 , %xmm6 , %xmm6
+	vxorpd		%xmm7 , %xmm7 , %xmm7
+	vxorpd		%xmm8 , %xmm8 , %xmm8
+	vxorpd		%xmm9 , %xmm9 , %xmm9
+	vxorpd		%xmm10, %xmm10, %xmm10
+	vxorpd		%xmm11, %xmm11, %xmm11
+	vxorpd		%xmm12, %xmm12, %xmm12
+	vxorpd		%xmm13, %xmm13, %xmm13
+	vxorpd		%xmm14, %xmm14, %xmm14
+	vxorpd		%xmm15, %xmm15, %xmm15
+
+.endm
+
+.macro KERNEL1x12_SUB
+	vmovsd 	-16 * SIZE(AO), %xmm0
+	vmovsd	-12 * SIZE(BO), %xmm1
+	vmovsd	-11 * SIZE(BO), %xmm2
+	vmovsd	-10 * SIZE(BO), %xmm3
+	vfmadd231sd  	%xmm0 ,%xmm1  , %xmm4
+	vmovsd	 -9 * SIZE(BO), %xmm1
+	vfmadd231sd  	%xmm0 ,%xmm2  , %xmm5
+	vmovsd	 -8 * SIZE(BO), %xmm2
+	vfmadd231sd  	%xmm0 ,%xmm3  , %xmm6
+	vmovsd	 -7 * SIZE(BO), %xmm3
+	vfmadd231sd  	%xmm0 ,%xmm1  , %xmm7
+	vmovsd	 -6 * SIZE(BO), %xmm1
+	vfmadd231sd  	%xmm0 ,%xmm2  , %xmm8
+	vmovsd	 -5 * SIZE(BO), %xmm2
+	vfmadd231sd  	%xmm0 ,%xmm3  , %xmm9
+	vmovsd	 -4 * SIZE(BO), %xmm3
+	vfmadd231sd  	%xmm0 ,%xmm1  , %xmm10
+	vmovsd	 -3 * SIZE(BO), %xmm1
+	vfmadd231sd  	%xmm0 ,%xmm2  , %xmm11
+	vmovsd	 -2 * SIZE(BO), %xmm2
+	vfmadd231sd  	%xmm0 ,%xmm3  , %xmm12
+	vmovsd	 -1 * SIZE(BO), %xmm3
+	vfmadd231sd  	%xmm0 ,%xmm1  , %xmm13
+	addq		$ 12*SIZE, BO
+	vfmadd231sd  	%xmm0 ,%xmm2  , %xmm14
+	addq		$ 1*SIZE, AO
+	vfmadd231sd  	%xmm0 ,%xmm3  , %xmm15
+
+.endm
+
+.macro SAVE1x12
+
+	vmovsd	ALPHA, %xmm0
+
+	vmulsd	%xmm0 , %xmm4 , %xmm4
+	vmulsd	%xmm0 , %xmm5 , %xmm5
+	vmulsd	%xmm0 , %xmm6 , %xmm6
+	vmulsd	%xmm0 , %xmm7 , %xmm7
+
+	vmulsd	%xmm0 , %xmm8 , %xmm8
+	vmulsd	%xmm0 , %xmm9 , %xmm9
+	vmulsd	%xmm0 , %xmm10, %xmm10
+	vmulsd	%xmm0 , %xmm11, %xmm11
+
+	vmulsd	%xmm0 , %xmm12, %xmm12
+	vmulsd	%xmm0 , %xmm13, %xmm13
+	vmulsd	%xmm0 , %xmm14, %xmm14
+	vmulsd	%xmm0 , %xmm15, %xmm15
+
+
+        leaq    (CO1, LDC, 2), %rax     
+	
+
+#if !defined(TRMMKERNEL)
+
+	vaddsd 	                (CO1), %xmm4, %xmm4
+	vaddsd 	           (CO1, LDC), %xmm5, %xmm5
+	vaddsd 	               (%rax), %xmm6, %xmm6
+	vaddsd 	          (%rax, LDC), %xmm7, %xmm7
+
+#endif
+
+	vmovsd	%xmm4 ,  	(CO1)
+	vmovsd	%xmm5 ,  	(CO1, LDC)
+	vmovsd	%xmm6 ,  	(%rax)
+	vmovsd	%xmm7 ,  	(%rax, LDC)
+
+
+	leaq	(%rax, LDC, 2), %rax
+	leaq	(%rax, LDC, 2), %rbp
+
+#if !defined(TRMMKERNEL)
+
+	vaddsd 	                (%rax), %xmm8 , %xmm4
+	vaddsd 	           (%rax, LDC), %xmm9 , %xmm5
+	vaddsd 	                (%rbp), %xmm10, %xmm6
+	vaddsd 	           (%rbp, LDC), %xmm11, %xmm7
+
+#endif
+
+	vmovsd	%xmm4 ,  	(%rax)
+	vmovsd	%xmm5 ,  	(%rax, LDC)
+	vmovsd	%xmm6 ,  	(%rbp)
+	vmovsd	%xmm7 ,  	(%rbp, LDC)
+
+
+	leaq	(%rax, LDC, 4), %rax
+	leaq	(%rbp, LDC, 4), %rbp
+
+#if !defined(TRMMKERNEL)
+
+	vaddsd 	                (%rax), %xmm12, %xmm4
+	vaddsd 	           (%rax, LDC), %xmm13, %xmm5
+	vaddsd 	                (%rbp), %xmm14, %xmm6
+	vaddsd 	           (%rbp, LDC), %xmm15, %xmm7
+
+#endif
+
+	vmovsd	%xmm4 ,  	(%rax)
+	vmovsd	%xmm5 ,  	(%rax, LDC)
+	vmovsd	%xmm6 ,  	(%rbp)
+	vmovsd	%xmm7 ,  	(%rbp, LDC)
+
+	addq	$ 1*SIZE, CO1
+.endm
+
+
+
+
+/******************************************************************************************/
+/******************************************************************************************/
+
+.macro INIT4x4
+
+	vxorpd		%ymm4 , %ymm4 , %ymm4
+	vxorpd		%ymm5 , %ymm5 , %ymm5
+	vxorpd		%ymm6 , %ymm6 , %ymm6
+	vxorpd		%ymm7 , %ymm7 , %ymm7
+
+.endm
+
+.macro KERNEL4x4_I
+	prefetcht0	A_PR1(AO)
+	vmovups		-12 * SIZE(BO), %ymm1
+	vmovups 	-16 * SIZE(AO), %ymm0
+	vmulpd  	%ymm0 ,%ymm1  , %ymm4
+	vpermpd		$ 0xb1, %ymm0  , %ymm0
+	vmulpd  	%ymm0 ,%ymm1  , %ymm5
+	vpermpd		$ 0x1b, %ymm0  , %ymm0
+	vmulpd  	%ymm0 ,%ymm1  , %ymm6
+
+	addq		$ 4*SIZE, BO
+	vpermpd		$ 0xb1, %ymm0  , %ymm0
+	vmulpd  	%ymm0 ,%ymm1  , %ymm7
+	vmovups		-12 * SIZE(BO), %ymm1
+
+.endm
+
+.macro KERNEL4x4_M1
+	prefetcht0	A_PR1(AO)
+	vmovups 	-16 * SIZE(AO), %ymm0
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4
+	vpermpd		$ 0xb1, %ymm0  , %ymm0
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5
+	vpermpd		$ 0x1b, %ymm0  , %ymm0
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6
+
+	vpermpd		$ 0xb1, %ymm0  , %ymm0
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7
+	vmovups		-12 * SIZE(BO), %ymm1
+
+.endm
+
+.macro KERNEL4x4_M2
+	vmovups 	-12 * SIZE(AO), %ymm0
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4
+	vpermpd		$ 0xb1, %ymm0  , %ymm0
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5
+	vpermpd		$ 0x1b, %ymm0  , %ymm0
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6
+
+	addq		$ 8*SIZE, AO
+	vpermpd		$ 0xb1, %ymm0  , %ymm0
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7
+	vmovups		 -8 * SIZE(BO), %ymm1
+	addq		$ 8*SIZE, BO
+.endm
+
+
+.macro KERNEL4x4_E
+	vmovups 	-12 * SIZE(AO), %ymm0
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4
+	vpermpd		$ 0xb1, %ymm0  , %ymm0
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5
+	vpermpd		$ 0x1b, %ymm0  , %ymm0
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6
+
+	addq		$ 8*SIZE, AO
+	vpermpd		$ 0xb1, %ymm0  , %ymm0
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7
+	addq		$ 4*SIZE, BO
+.endm
+
+.macro KERNEL4x4_SUB
+	vmovups		-12 * SIZE(BO), %ymm1
+	vmovups 	-16 * SIZE(AO), %ymm0
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4
+	vpermpd		$ 0xb1, %ymm0  , %ymm0
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5
+	addq		$ 4*SIZE, BO
+	vpermpd		$ 0x1b, %ymm0  , %ymm0
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6
+	addq		$ 4*SIZE, AO
+	vpermpd		$ 0xb1, %ymm0  , %ymm0
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7
+
+.endm
+
+.macro SAVE4x4
+
+	vbroadcastsd	ALPHA, %ymm0
+
+	vmulpd	%ymm0 , %ymm4 , %ymm4
+	vmulpd	%ymm0 , %ymm7 , %ymm7
+	vmulpd	%ymm0 , %ymm5 , %ymm5
+	vmulpd	%ymm0 , %ymm6 , %ymm6
+
+	vpermpd $ 0xb1 , %ymm5, %ymm5
+	vpermpd $ 0xb1 , %ymm7, %ymm7
+
+	vblendpd $ 0x0a, %ymm5, %ymm4, %ymm0
+	vblendpd $ 0x05, %ymm5, %ymm4, %ymm1
+	vblendpd $ 0x0a, %ymm7, %ymm6, %ymm2
+	vblendpd $ 0x05, %ymm7, %ymm6, %ymm3
+
+	vpermpd $ 0x1b , %ymm2, %ymm2
+	vpermpd $ 0x1b , %ymm3, %ymm3
+	vpermpd $ 0xb1 , %ymm2, %ymm2
+	vpermpd $ 0xb1 , %ymm3, %ymm3
+
+	vblendpd $ 0x03, %ymm0, %ymm2 , %ymm4
+	vblendpd $ 0x03, %ymm1, %ymm3 , %ymm5
+	vblendpd $ 0x03, %ymm2, %ymm0 , %ymm6
+	vblendpd $ 0x03, %ymm3, %ymm1 , %ymm7
+
+        leaq    (CO1, LDC, 2), %rax     
+	
+
+#if !defined(TRMMKERNEL)
+
+	vaddpd 	                (CO1), %ymm4, %ymm4
+	vaddpd 	           (CO1, LDC), %ymm5, %ymm5
+	vaddpd 	               (%rax), %ymm6, %ymm6
+	vaddpd 	          (%rax, LDC), %ymm7, %ymm7
+
+#endif
+
+	vmovups	%ymm4 ,  	(CO1)
+	vmovups	%ymm5 ,  	(CO1, LDC)
+	vmovups	%ymm6 ,  	(%rax)
+	vmovups	%ymm7 ,  	(%rax, LDC)
+
+	addq	$ 4*SIZE, CO1
+.endm
+
+/******************************************************************************************/
+/******************************************************************************************/
+
+.macro INIT2x4
+
+	vxorpd		%xmm4 , %xmm4 , %xmm4
+	vxorpd		%xmm5 , %xmm5 , %xmm5
+	vxorpd		%xmm6 , %xmm6 , %xmm6
+	vxorpd		%xmm7 , %xmm7 , %xmm7
+
+.endm
+
+
+.macro KERNEL2x4_SUB
+	vmovddup	-12 * SIZE(BO), %xmm1
+	vmovups 	-16 * SIZE(AO), %xmm0
+	vmovddup	-11 * SIZE(BO), %xmm2
+	vfmadd231pd  	%xmm0 ,%xmm1  , %xmm4
+	vmovddup	-10 * SIZE(BO), %xmm3
+	vfmadd231pd  	%xmm0 ,%xmm2  , %xmm5
+	vmovddup	 -9 * SIZE(BO), %xmm8
+	vfmadd231pd  	%xmm0 ,%xmm3  , %xmm6
+	addq		$ 4*SIZE, BO
+	vfmadd231pd  	%xmm0 ,%xmm8  , %xmm7
+	addq		$ 2*SIZE, AO
+
+.endm
+
+
+.macro SAVE2x4
+
+	vmovddup	ALPHA, %xmm0
+
+	vmulpd	%xmm0 , %xmm4 , %xmm4
+	vmulpd	%xmm0 , %xmm5 , %xmm5
+	vmulpd	%xmm0 , %xmm6 , %xmm6
+	vmulpd	%xmm0 , %xmm7 , %xmm7
+
+        leaq    (CO1, LDC, 2), %rax     
+
+#if !defined(TRMMKERNEL)
+
+	vaddpd 	                (CO1), %xmm4, %xmm4
+	vaddpd 	           (CO1, LDC), %xmm5, %xmm5
+	vaddpd 	               (%rax), %xmm6, %xmm6
+	vaddpd 	          (%rax, LDC), %xmm7, %xmm7
+
+#endif
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm5 ,  	(CO1, LDC)
+	vmovups	%xmm6 ,  	(%rax)
+	vmovups	%xmm7 ,  	(%rax, LDC)
+
+	addq	$ 2*SIZE, CO1
+.endm
+
+/******************************************************************************************/
+/******************************************************************************************/
+
+.macro INIT1x4
+
+	vxorpd		%xmm4 , %xmm4 , %xmm4
+	vxorpd		%xmm5 , %xmm5 , %xmm5
+	vxorpd		%xmm6 , %xmm6 , %xmm6
+	vxorpd		%xmm7 , %xmm7 , %xmm7
+
+.endm
+
+
+.macro KERNEL1x4_SUB
+	vmovsd	-12 * SIZE(BO), %xmm1
+	vmovsd 	-16 * SIZE(AO), %xmm0
+	vmovsd	-11 * SIZE(BO), %xmm2
+	vfmadd231sd  	%xmm0 ,%xmm1  , %xmm4
+	vmovsd	-10 * SIZE(BO), %xmm3
+	vfmadd231sd  	%xmm0 ,%xmm2  , %xmm5
+	vmovsd	 -9 * SIZE(BO), %xmm8
+	vfmadd231sd  	%xmm0 ,%xmm3  , %xmm6
+	addq		$ 4*SIZE, BO
+	vfmadd231sd  	%xmm0 ,%xmm8  , %xmm7
+	addq		$ 1*SIZE, AO
+
+.endm
+
+
+.macro SAVE1x4
+
+	vmovsd	ALPHA, %xmm0
+
+	vmulsd	%xmm0 , %xmm4 , %xmm4
+	vmulsd	%xmm0 , %xmm5 , %xmm5
+	vmulsd	%xmm0 , %xmm6 , %xmm6
+	vmulsd	%xmm0 , %xmm7 , %xmm7
+
+        leaq    (CO1, LDC, 2), %rax     
+
+#if !defined(TRMMKERNEL)
+
+	vaddsd 	                (CO1), %xmm4, %xmm4
+	vaddsd 	           (CO1, LDC), %xmm5, %xmm5
+	vaddsd 	               (%rax), %xmm6, %xmm6
+	vaddsd 	          (%rax, LDC), %xmm7, %xmm7
+
+#endif
+
+	vmovsd	%xmm4 ,  	(CO1)
+	vmovsd	%xmm5 ,  	(CO1, LDC)
+	vmovsd	%xmm6 ,  	(%rax)
+	vmovsd	%xmm7 ,  	(%rax, LDC)
+
+	addq	$ 1*SIZE, CO1
+.endm
+
+
+/******************************************************************************************/
+/******************************************************************************************/
+
+.macro INIT4x2
+
+	vxorpd		%xmm4 , %xmm4 , %xmm4
+	vxorpd		%xmm5 , %xmm5 , %xmm5
+	vxorpd		%xmm6 , %xmm6 , %xmm6
+	vxorpd		%xmm7 , %xmm7 , %xmm7
+
+.endm
+
+
+.macro KERNEL4x2_SUB
+	vmovddup	-12 * SIZE(BO), %xmm2
+	vmovups 	-16 * SIZE(AO), %xmm0
+	vmovups 	-14 * SIZE(AO), %xmm1
+	vmovddup	-11 * SIZE(BO), %xmm3
+	vfmadd231pd  	%xmm0 ,%xmm2  , %xmm4
+	vfmadd231pd  	%xmm1 ,%xmm2  , %xmm5
+	vfmadd231pd  	%xmm0 ,%xmm3  , %xmm6
+	vfmadd231pd  	%xmm1 ,%xmm3  , %xmm7
+	addq		$ 2*SIZE, BO
+	addq		$ 4*SIZE, AO
+
+.endm
+
+
+.macro SAVE4x2
+
+	vmovddup	ALPHA, %xmm0
+
+	vmulpd	%xmm0 , %xmm4 , %xmm4
+	vmulpd	%xmm0 , %xmm5 , %xmm5
+	vmulpd	%xmm0 , %xmm6 , %xmm6
+	vmulpd	%xmm0 , %xmm7 , %xmm7
+
+
+#if !defined(TRMMKERNEL)
+
+	vaddpd 	                (CO1)     , %xmm4, %xmm4
+	vaddpd 	        2 * SIZE(CO1)     , %xmm5, %xmm5
+	vaddpd 	                (CO1, LDC), %xmm6, %xmm6
+	vaddpd 	        2 * SIZE(CO1, LDC), %xmm7, %xmm7
+
+#endif
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm5 , 2 * SIZE(CO1)
+	vmovups	%xmm6 ,  	(CO1, LDC)
+	vmovups	%xmm7 , 2 * SIZE(CO1, LDC)
+
+	addq	$ 4*SIZE, CO1
+.endm
+
+
+/******************************************************************************************/
+/******************************************************************************************/
+
+.macro INIT2x2
+
+	vxorpd		%xmm4 , %xmm4 , %xmm4
+	vxorpd		%xmm6 , %xmm6 , %xmm6
+
+.endm
+
+
+.macro KERNEL2x2_SUB
+	vmovddup	-12 * SIZE(BO), %xmm2
+	vmovups 	-16 * SIZE(AO), %xmm0
+	vmovddup	-11 * SIZE(BO), %xmm3
+	vfmadd231pd  	%xmm0 ,%xmm2  , %xmm4
+	vfmadd231pd  	%xmm0 ,%xmm3  , %xmm6
+	addq		$ 2*SIZE, BO
+	addq		$ 2*SIZE, AO
+
+.endm
+
+
+.macro SAVE2x2
+
+	vmovddup	ALPHA, %xmm0
+
+	vmulpd	%xmm0 , %xmm4 , %xmm4
+	vmulpd	%xmm0 , %xmm6 , %xmm6
+
+
+#if !defined(TRMMKERNEL)
+
+	vaddpd 	                (CO1)     , %xmm4, %xmm4
+	vaddpd 	                (CO1, LDC), %xmm6, %xmm6
+
+#endif
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm6 ,  	(CO1, LDC)
+
+	addq	$ 2*SIZE, CO1
+.endm
+
+/******************************************************************************************/
+/******************************************************************************************/
+
+.macro INIT1x2
+
+	vxorpd		%xmm4 , %xmm4 , %xmm4
+	vxorpd		%xmm5 , %xmm5 , %xmm5
+
+.endm
+
+
+.macro KERNEL1x2_SUB
+	vmovsd	-12 * SIZE(BO), %xmm1
+	vmovsd 	-16 * SIZE(AO), %xmm0
+	vmovsd	-11 * SIZE(BO), %xmm2
+	vfmadd231sd  	%xmm0 ,%xmm1  , %xmm4
+	vfmadd231sd  	%xmm0 ,%xmm2  , %xmm5
+	addq		$ 2*SIZE, BO
+	addq		$ 1*SIZE, AO
+
+.endm
+
+
+.macro SAVE1x2
+
+	vmovsd	ALPHA, %xmm0
+
+	vmulsd	%xmm0 , %xmm4 , %xmm4
+	vmulsd	%xmm0 , %xmm5 , %xmm5
+
+
+#if !defined(TRMMKERNEL)
+
+	vaddsd 	                (CO1), %xmm4, %xmm4
+	vaddsd 	           (CO1, LDC), %xmm5, %xmm5
+
+#endif
+
+	vmovsd	%xmm4 ,  	(CO1)
+	vmovsd	%xmm5 ,  	(CO1, LDC)
+
+	addq	$ 1*SIZE, CO1
+.endm
+
+
+/******************************************************************************************/
+/******************************************************************************************/
+
+.macro INIT4x1
+
+	vxorpd		%ymm4 , %ymm4 , %ymm4
+	vxorpd		%ymm5 , %ymm5 , %ymm5
+	vxorpd		%ymm6 , %ymm6 , %ymm6
+	vxorpd		%ymm7 , %ymm7 , %ymm7
+
+.endm
+
+
+.macro KERNEL4x1
+
+	vbroadcastsd	-12 * SIZE(BO), %ymm0
+	vbroadcastsd	-11 * SIZE(BO), %ymm1
+	vbroadcastsd	-10 * SIZE(BO), %ymm2
+	vbroadcastsd	-9  * SIZE(BO), %ymm3
+
+	vfmadd231pd  	-16 * SIZE(AO) ,%ymm0  , %ymm4
+	vfmadd231pd  	-12 * SIZE(AO) ,%ymm1  , %ymm5
+
+	vbroadcastsd	-8  * SIZE(BO), %ymm0
+	vbroadcastsd	-7  * SIZE(BO), %ymm1
+
+	vfmadd231pd  	-8  * SIZE(AO) ,%ymm2  , %ymm6
+	vfmadd231pd  	-4  * SIZE(AO) ,%ymm3  , %ymm7
+
+	vbroadcastsd	-6  * SIZE(BO), %ymm2
+	vbroadcastsd	-5  * SIZE(BO), %ymm3
+
+	vfmadd231pd  	 0  * SIZE(AO) ,%ymm0  , %ymm4
+	vfmadd231pd  	 4  * SIZE(AO) ,%ymm1  , %ymm5
+	vfmadd231pd  	 8  * SIZE(AO) ,%ymm2  , %ymm6
+	vfmadd231pd  	 12 * SIZE(AO) ,%ymm3  , %ymm7
+
+	addq		$ 8 *SIZE, BO
+	addq		$ 32*SIZE, AO
+
+.endm
+
+
+.macro KERNEL4x1_SUB
+	vbroadcastsd	-12 * SIZE(BO), %ymm2
+	vmovups 	-16 * SIZE(AO), %ymm0
+	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm4
+	addq		$ 1*SIZE, BO
+	addq		$ 4*SIZE, AO
+
+.endm
+
+
+.macro SAVE4x1
+
+	vbroadcastsd	ALPHA, %ymm0
+
+	vaddpd	%ymm4,%ymm5, %ymm4 
+	vaddpd	%ymm6,%ymm7, %ymm6 
+	vaddpd	%ymm4,%ymm6, %ymm4 
+
+	vmulpd	%ymm0 , %ymm4 , %ymm4
+
+
+#if !defined(TRMMKERNEL)
+
+	vaddpd 	                (CO1)     , %ymm4, %ymm4
+
+#endif
+
+	vmovups	%ymm4 ,  	(CO1)
+
+	addq	$ 4*SIZE, CO1
+.endm
+
+
+/******************************************************************************************/
+/******************************************************************************************/
+
+.macro INIT2x1
+
+	vxorpd		%xmm4 , %xmm4 , %xmm4
+
+.endm
+
+
+.macro KERNEL2x1_SUB
+	vmovddup	-12 * SIZE(BO), %xmm2
+	vmovups 	-16 * SIZE(AO), %xmm0
+	vfmadd231pd  	%xmm0 ,%xmm2  , %xmm4
+	addq		$ 1*SIZE, BO
+	addq		$ 2*SIZE, AO
+
+.endm
+
+
+.macro SAVE2x1
+
+	vmovddup	ALPHA, %xmm0
+
+	vmulpd	%xmm0 , %xmm4 , %xmm4
+
+
+#if !defined(TRMMKERNEL)
+
+	vaddpd 	                (CO1)     , %xmm4, %xmm4
+
+#endif
+
+	vmovups	%xmm4 ,  	(CO1)
+
+	addq	$ 2*SIZE, CO1
+.endm
+
+
+/******************************************************************************************/
+/******************************************************************************************/
+
+.macro INIT1x1
+
+	vxorpd		%xmm4 , %xmm4 , %xmm4
+
+.endm
+
+
+.macro KERNEL1x1_SUB
+	vmovsd	-12 * SIZE(BO), %xmm1
+	vmovsd 	-16 * SIZE(AO), %xmm0
+	vfmadd231sd  	%xmm0 ,%xmm1  , %xmm4
+	addq		$ 1*SIZE, BO
+	addq		$ 1*SIZE, AO
+
+.endm
+
+
+.macro SAVE1x1
+
+	vmovsd	ALPHA, %xmm0
+
+	vmulsd	%xmm0 , %xmm4 , %xmm4
+
+
+#if !defined(TRMMKERNEL)
+
+	vaddsd 	                (CO1), %xmm4, %xmm4
+
+#endif
+
+	vmovsd	%xmm4 ,  	(CO1)
+
+	addq	$ 1*SIZE, CO1
+.endm
+
+
+/*******************************************************************************************/
+
+#if !defined(TRMMKERNEL)
+
+
+	PROLOGUE
+	PROFCODE
+	
+	subq	$STACKSIZE, %rsp
+	movq	%rbx,   (%rsp)
+	movq	%rbp,  8(%rsp)
+	movq	%r12, 16(%rsp)
+	movq	%r13, 24(%rsp)
+	movq	%r14, 32(%rsp)
+	movq	%r15, 40(%rsp)
+
+	vzeroupper
+
+#ifdef WINDOWS_ABI
+	movq	%rdi,    48(%rsp)
+	movq	%rsi,    56(%rsp)
+	vmovups	%xmm6,   64(%rsp)
+	vmovups	%xmm7,   80(%rsp)
+	vmovups	%xmm8,   96(%rsp)
+	vmovups	%xmm9,  112(%rsp)
+	vmovups	%xmm10, 128(%rsp)
+	vmovups	%xmm11, 144(%rsp)
+	vmovups	%xmm12, 160(%rsp)
+	vmovups	%xmm13, 176(%rsp)
+	vmovups	%xmm14, 192(%rsp)
+	vmovups	%xmm15, 208(%rsp)
+
+	movq	ARG1,      OLD_M
+	movq	ARG2,      OLD_N
+	movq	ARG3,      OLD_K
+	movq	OLD_A,     A
+	movq	OLD_B,     B
+	movq	OLD_C,     C
+	movq	OLD_LDC,   LDC
+
+	vmovups	%xmm3, %xmm0
+
+#else
+	movq	STACKSIZE +  8(%rsp), LDC
+
+#endif
+
+	movq    %rsp, SP      # save old stack
+        subq    $128 + L_BUFFER_SIZE, %rsp
+        andq    $-4096, %rsp    # align stack
+
+        STACK_TOUCH
+
+	cmpq	$ 0, OLD_M
+	je	.L999
+
+	cmpq	$ 0, OLD_N
+	je	.L999
+
+	cmpq	$ 0, OLD_K
+	je	.L999
+
+	movq	OLD_M, M
+	movq	OLD_N, N
+	movq	OLD_K, K
+
+	vmovsd	 %xmm0, ALPHA
+
+	salq	$BASE_SHIFT, LDC
+
+	movq    N, %rax
+        xorq    %rdx, %rdx
+        movq    $12,  %rdi
+        divq    %rdi                    //    N / 12
+        movq    %rax, Ndiv12             //    N / 12
+        movq    %rdx, Nmod12             //    N % 12
+
+
+	movq	Ndiv12,  J
+	cmpq	$ 0, J
+	je	.L4_0
+	ALIGN_4
+
+.L12_01:
+        // copy to sub buffer
+        movq    K, %rax
+        salq    $2,%rax                 // K * 4 ; read 2 values
+        movq    B, BO1
+        leaq    (B,%rax, SIZE), BO2     // next offset to BO2
+        leaq    (BO2,%rax, SIZE), BO3     // next offset to BO2
+
+
+        leaq    BUFFER1, BO             // first buffer to BO
+        movq    K, %rax
+	sarq	$1 , %rax		// K / 2
+	jz	.L12_01a_2
+        ALIGN_4
+
+.L12_01a_1:
+
+        prefetcht0 512(BO1)
+        prefetcht0 512(BO2)
+        prefetcht0 512(BO3)
+        prefetchw  512(BO)
+
+
+	vmovups	0 * SIZE(BO1), %ymm1
+	vmovups	4 * SIZE(BO1), %ymm5
+	vmovups	0 * SIZE(BO2), %ymm2
+	vmovups	4 * SIZE(BO2), %ymm6
+	vmovups	0 * SIZE(BO3), %ymm3
+	vmovups	4 * SIZE(BO3), %ymm7
+
+	vmovups	%ymm1,  0 * SIZE(BO)
+	vmovups	%ymm2,  4 * SIZE(BO)
+	vmovups	%ymm3,  8 * SIZE(BO)
+
+	vmovups	%ymm5, 12 * SIZE(BO)
+	vmovups	%ymm6, 16 * SIZE(BO)
+	vmovups	%ymm7, 20 * SIZE(BO)
+
+	addq	$ 8 * SIZE ,BO1
+	addq	$ 8 * SIZE ,BO2
+	addq	$ 8 * SIZE ,BO3
+	addq    $ 24 *SIZE ,BO
+
+	decq	%rax
+	jnz	.L12_01a_1
+
+
+
+.L12_01a_2:
+
+	movq    K, %rax
+        andq    $1, %rax                // K % 2
+        jz      .L12_03c
+        ALIGN_4
+
+
+.L12_02b:
+
+	vmovups	0 * SIZE(BO1), %ymm1
+	vmovups	0 * SIZE(BO2), %ymm2
+	vmovups	0 * SIZE(BO3), %ymm3
+	vmovups	%ymm1, 0 * SIZE(BO)
+	vmovups	%ymm2, 4 * SIZE(BO)
+	vmovups	%ymm3, 8 * SIZE(BO)
+	addq	$ 4*SIZE,BO1
+	addq	$ 4*SIZE,BO2
+	addq	$ 4*SIZE,BO3
+	addq	$ 12*SIZE,BO
+	decq	%rax
+	jnz	.L12_02b
+
+.L12_03c:
+
+	movq	BO3, B			// next offset of B
+
+.L12_10:
+	movq	C, CO1
+	leaq	(C, LDC, 8), C		 
+	leaq	(C, LDC, 4), C		// c += 12 * ldc
+
+	
+	movq	A, AO		 	// aoffset = a
+	addq	$16 * SIZE, AO
+
+	movq	M,  I
+	sarq	$2, I			// i = m / 4
+	je	.L12_20
+
+	ALIGN_4
+
+.L12_11:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $12 * SIZE, BO
+
+        movq    K, %rax
+
+	sarq $3, %rax			//  K / 8
+	cmpq $2, %rax
+
+	jl	.L12_13
+
+
+	KERNEL4x12_I
+	KERNEL4x12_M2
+	KERNEL4x12_M1
+	KERNEL4x12_M2
+
+	KERNEL4x12_M1
+	KERNEL4x12_M2
+	KERNEL4x12_M1
+	KERNEL4x12_M2
+
+	subq $2, %rax
+	je	.L12_12a
+
+	ALIGN_5
+.L12_12:
+
+	KERNEL4x12_M1
+	KERNEL4x12_M2
+	KERNEL4x12_M1
+	KERNEL4x12_M2
+
+	KERNEL4x12_M1
+	KERNEL4x12_M2
+	KERNEL4x12_M1
+	KERNEL4x12_M2
+
+	dec	%rax
+	jne	.L12_12
+
+.L12_12a:
+
+	KERNEL4x12_M1
+	KERNEL4x12_M2
+	KERNEL4x12_M1
+	KERNEL4x12_M2
+
+	KERNEL4x12_M1
+	KERNEL4x12_M2
+	KERNEL4x12_M1
+	KERNEL4x12_E
+
+	jmp .L12_16
+
+
+.L12_13:
+
+	test $1, %rax
+	jz .L12_14
+
+	KERNEL4x12_I
+	KERNEL4x12_M2
+	KERNEL4x12_M1
+	KERNEL4x12_M2
+
+	KERNEL4x12_M1
+	KERNEL4x12_M2
+	KERNEL4x12_M1
+	KERNEL4x12_E
+
+	jmp .L12_16
+
+
+.L12_14:
+
+	INIT4x12
+
+
+.L12_16:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L12_19
+
+	ALIGN_4
+
+.L12_17:
+
+	KERNEL4x12_SUB
+
+	dec	%rax
+	jne	.L12_17
+	ALIGN_4
+
+
+.L12_19:
+
+	SAVE4x12
+
+	decq	I			# i --
+	jne	.L12_11
+	ALIGN_4	
+
+/**************************************************************************
+* Rest of M 
+***************************************************************************/
+.L12_20:
+	// Test rest of M
+
+	testq	$3, M
+	jz	.L12_100			// to next 16 lines of N
+
+
+.L12_30:
+	testq	$2, M		
+	jz	.L12_40
+
+	ALIGN_4
+
+.L12_31:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $12 * SIZE, BO
+
+	INIT2x12
+
+        movq    K, %rax
+
+	sarq	$3, %rax
+	je	.L12_36
+	ALIGN_4
+
+.L12_32:
+
+	KERNEL2x12_SUB
+	KERNEL2x12_SUB
+	KERNEL2x12_SUB
+	KERNEL2x12_SUB
+
+	KERNEL2x12_SUB
+	KERNEL2x12_SUB
+	KERNEL2x12_SUB
+	KERNEL2x12_SUB
+
+	dec %rax
+	jne	.L12_32
+	ALIGN_4
+
+.L12_36:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L12_39
+
+	ALIGN_4
+
+.L12_37:
+
+	KERNEL2x12_SUB
+
+	dec %rax
+	jne	.L12_37
+	ALIGN_4
+
+
+.L12_39:
+
+	SAVE2x12
+
+	ALIGN_4
+
+.L12_40:
+	testq	$1, M		
+	jz	.L12_100		// to next 3 lines of N
+
+	ALIGN_4
+
+.L12_41:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $12 * SIZE, BO
+
+	INIT1x12
+
+        movq    K, %rax
+
+	sarq	$3,%rax
+	je	.L12_46
+
+	ALIGN_4
+
+.L12_42:
+
+	KERNEL1x12_SUB
+	KERNEL1x12_SUB
+	KERNEL1x12_SUB
+	KERNEL1x12_SUB
+
+	KERNEL1x12_SUB
+	KERNEL1x12_SUB
+	KERNEL1x12_SUB
+	KERNEL1x12_SUB
+
+
+	dec %rax
+	jne	.L12_42
+	ALIGN_4
+
+.L12_46:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L12_49
+
+	ALIGN_4
+
+.L12_47:
+
+	KERNEL1x12_SUB
+
+	dec	%rax
+	jne	.L12_47
+	ALIGN_4
+
+
+.L12_49:
+
+	SAVE1x12
+
+	ALIGN_4
+	
+.L12_100:
+
+	decq	J			// j --
+	jg	.L12_01
+
+
+.L4_0:
+
+	cmpq	$ 0, Nmod12		// N % 12 == 0
+	je	.L999
+
+	movq	Nmod12, J		
+	sarq	$2, J			// j = j / 4
+	je	.L2_0
+
+.L4_10:
+	movq	C, CO1
+	leaq	(C, LDC, 4), C		// c += 4 * ldc
+
+	
+	movq	A, AO		 	// aoffset = a
+	addq	$16 * SIZE, AO
+
+	movq	M,  I
+	sarq	$2, I			// i = m / 4
+	je	.L4_20
+
+	ALIGN_4
+
+.L4_11:
+        movq    B, BO        
+        addq    $12 * SIZE, BO
+
+        movq    K, %rax
+
+	sarq	$3, %rax			//  K / 8
+	cmpq    $2, %rax
+	jl	.L4_13
+
+
+	KERNEL4x4_I
+	KERNEL4x4_M2
+	KERNEL4x4_M1
+	KERNEL4x4_M2
+
+	KERNEL4x4_M1
+	KERNEL4x4_M2
+	KERNEL4x4_M1
+	KERNEL4x4_M2
+
+	subq $2, %rax
+	je	.L4_12a
+
+	ALIGN_5
+
+.L4_12:
+
+	KERNEL4x4_M1
+	KERNEL4x4_M2
+	KERNEL4x4_M1
+	KERNEL4x4_M2
+
+	KERNEL4x4_M1
+	KERNEL4x4_M2
+	KERNEL4x4_M1
+	KERNEL4x4_M2
+
+	dec	%rax
+	jne	.L4_12
+
+.L4_12a:
+
+	KERNEL4x4_M1
+	KERNEL4x4_M2
+	KERNEL4x4_M1
+	KERNEL4x4_M2
+
+	KERNEL4x4_M1
+	KERNEL4x4_M2
+	KERNEL4x4_M1
+	KERNEL4x4_E
+
+	jmp .L4_16
+
+
+.L4_13:
+
+	test $1, %rax
+	jz .L4_14
+
+	KERNEL4x4_I
+	KERNEL4x4_M2
+	KERNEL4x4_M1
+	KERNEL4x4_M2
+
+	KERNEL4x4_M1
+	KERNEL4x4_M2
+	KERNEL4x4_M1
+	KERNEL4x4_E
+
+	jmp .L4_16
+
+
+.L4_14:
+
+	INIT4x4
+
+
+.L4_16:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L4_19
+
+	ALIGN_4
+
+.L4_17:
+
+	KERNEL4x4_SUB
+
+	dec	%rax
+	jne	.L4_17
+	ALIGN_4
+
+
+.L4_19:
+
+	SAVE4x4
+
+	decq	I			# i --
+	jg	.L4_11
+	ALIGN_4	
+
+/**************************************************************************
+* Rest of M 
+***************************************************************************/
+.L4_20:
+	// Test rest of M
+
+	testq	$3, M
+	jz	.L4_100			// to next 16 lines of N
+
+
+.L4_30:
+	testq	$2, M		
+	jz	.L4_40
+
+	ALIGN_4
+
+.L4_31:
+        movq    B, BO             // first buffer to BO
+        addq    $12 * SIZE, BO
+
+	INIT2x4
+
+        movq    K, %rax
+
+	sarq	$3, %rax
+	je	.L4_36
+	ALIGN_4
+
+.L4_32:
+
+	KERNEL2x4_SUB
+	KERNEL2x4_SUB
+	KERNEL2x4_SUB
+	KERNEL2x4_SUB
+
+	KERNEL2x4_SUB
+	KERNEL2x4_SUB
+	KERNEL2x4_SUB
+	KERNEL2x4_SUB
+
+	dec %rax
+	jne	.L4_32
+	ALIGN_4
+
+.L4_36:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L4_39
+
+	ALIGN_4
+
+.L4_37:
+
+	KERNEL2x4_SUB
+
+	dec %rax
+	jne	.L4_37
+
+
+.L4_39:
+
+	SAVE2x4
+
+.L4_40:
+	testq	$1, M		
+	jz	.L4_100		// to next 3 lines of N
+
+	ALIGN_4
+
+.L4_41:
+        movq    B, BO             // first buffer to BO
+        addq    $12 * SIZE, BO
+
+	INIT1x4
+
+        movq    K, %rax
+
+	sarq	$3,%rax
+	je	.L4_46
+
+	ALIGN_4
+
+.L4_42:
+
+	KERNEL1x4_SUB
+	KERNEL1x4_SUB
+	KERNEL1x4_SUB
+	KERNEL1x4_SUB
+
+	KERNEL1x4_SUB
+	KERNEL1x4_SUB
+	KERNEL1x4_SUB
+	KERNEL1x4_SUB
+
+	dec %rax
+	jne	.L4_42
+	ALIGN_4
+
+.L4_46:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L4_49
+
+	ALIGN_4
+
+.L4_47:
+
+	KERNEL1x4_SUB
+
+	dec	%rax
+	jne	.L4_47
+	ALIGN_4
+
+
+.L4_49:
+
+	SAVE1x4
+
+	ALIGN_4
+	
+.L4_100:
+
+	movq	K, %rax
+	salq	$2, %rax		// * 4
+	leaq	(B , %rax, SIZE), B
+	decq	J			// j --
+	jg	.L4_10
+
+
+
+
+/***************************************************************************************************************/
+
+.L2_0:
+
+	movq	Nmod12, J		
+	testq	$2, J
+	je	.L1_0
+
+.L2_10:
+	movq	C, CO1
+	leaq	(C, LDC, 2), C		// c += 2 * ldc
+
+	
+	movq	A, AO		 	// aoffset = a
+	addq	$16 * SIZE, AO
+
+	movq	M,  I
+	sarq	$2, I			// i = m / 4
+	je	.L2_20
+
+	ALIGN_4
+
+.L2_11:
+        movq    B, BO        
+        addq    $12 * SIZE, BO
+
+	INIT4x2
+
+        movq    K, %rax
+	sarq $3, %rax			//  K / 8
+
+	je	.L2_16
+
+	ALIGN_5
+
+.L2_12:
+
+	KERNEL4x2_SUB
+	KERNEL4x2_SUB
+	KERNEL4x2_SUB
+	KERNEL4x2_SUB
+
+	KERNEL4x2_SUB
+	KERNEL4x2_SUB
+	KERNEL4x2_SUB
+	KERNEL4x2_SUB
+
+	dec	%rax
+	jne	.L2_12
+
+
+.L2_16:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_19
+
+	ALIGN_4
+
+.L2_17:
+
+	KERNEL4x2_SUB
+
+	dec	%rax
+	jne	.L2_17
+	ALIGN_4
+
+
+.L2_19:
+
+	SAVE4x2
+
+	decq	I			# i --
+	jg	.L2_11
+	ALIGN_4	
+
+/**************************************************************************
+* Rest of M 
+***************************************************************************/
+.L2_20:
+	// Test rest of M
+
+	testq	$3, M
+	jz	.L2_100			// to next 16 lines of N
+
+
+.L2_30:
+	testq	$2, M		
+	jz	.L2_40
+
+	ALIGN_4
+
+.L2_31:
+        movq    B, BO             // first buffer to BO
+        addq    $12 * SIZE, BO
+
+	INIT2x2
+
+        movq    K, %rax
+
+	sarq	$3, %rax
+	je	.L2_36
+	ALIGN_4
+
+.L2_32:
+
+	KERNEL2x2_SUB
+	KERNEL2x2_SUB
+	KERNEL2x2_SUB
+	KERNEL2x2_SUB
+
+	KERNEL2x2_SUB
+	KERNEL2x2_SUB
+	KERNEL2x2_SUB
+	KERNEL2x2_SUB
+
+	dec %rax
+	jne	.L2_32
+
+.L2_36:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_39
+
+	ALIGN_4
+
+.L2_37:
+
+	KERNEL2x2_SUB
+
+	dec %rax
+	jne	.L2_37
+
+
+.L2_39:
+
+	SAVE2x2
+
+.L2_40:
+	testq	$1, M		
+	jz	.L2_100		// to next 3 lines of N
+
+.L2_41:
+        movq    B, BO             // first buffer to BO
+        addq    $12 * SIZE, BO
+
+	INIT1x2
+
+        movq    K, %rax
+
+	sarq	$3,%rax
+	je	.L2_46
+
+	ALIGN_4
+
+.L2_42:
+
+	KERNEL1x2_SUB
+	KERNEL1x2_SUB
+	KERNEL1x2_SUB
+	KERNEL1x2_SUB
+
+	KERNEL1x2_SUB
+	KERNEL1x2_SUB
+	KERNEL1x2_SUB
+	KERNEL1x2_SUB
+
+	dec %rax
+	jne	.L2_42
+
+.L2_46:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_49
+
+	ALIGN_4
+
+.L2_47:
+
+	KERNEL1x2_SUB
+
+	dec	%rax
+	jne	.L2_47
+
+.L2_49:
+
+	SAVE1x2
+
+.L2_100:
+
+	movq	K, %rax
+	salq	$1, %rax		// * 2
+	leaq	(B , %rax, SIZE), B
+
+/***************************************************************************************************************/
+
+.L1_0:
+
+	movq	Nmod12, J		
+	testq	$1, J
+	je	.L999
+
+.L1_10:
+	movq	C, CO1
+	leaq	(C, LDC, 1), C		// c += 1 * ldc
+
+	
+	movq	A, AO		 	// aoffset = a
+	addq	$16 * SIZE, AO
+
+	movq	M,  I
+	sarq	$2, I			// i = m / 4
+	je	.L1_20
+
+	ALIGN_4
+
+.L1_11:
+        movq    B, BO        
+        addq    $12 * SIZE, BO
+
+	INIT4x1
+
+        movq    K, %rax
+
+	sarq	$3, %rax			//  K / 8
+	je	.L1_16
+
+	ALIGN_5
+
+.L1_12:
+
+	KERNEL4x1
+
+	dec	%rax
+	jne	.L1_12
+
+
+.L1_16:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_19
+
+	ALIGN_4
+
+.L1_17:
+
+	KERNEL4x1_SUB
+
+	dec	%rax
+	jne	.L1_17
+	ALIGN_4
+
+
+.L1_19:
+
+	SAVE4x1
+
+	decq	I			# i --
+	jg	.L1_11
+
+/**************************************************************************
+* Rest of M 
+***************************************************************************/
+.L1_20:
+	// Test rest of M
+
+	testq	$3, M
+	jz	.L1_100	
+
+
+.L1_30:
+	testq	$2, M		
+	jz	.L1_40
+
+	ALIGN_4
+
+.L1_31:
+        movq    B, BO             // first buffer to BO
+        addq    $12 * SIZE, BO
+
+	INIT2x1
+
+        movq    K, %rax
+
+	sarq	$3, %rax
+	je	.L1_36
+	ALIGN_4
+
+.L1_32:
+
+	KERNEL2x1_SUB
+	KERNEL2x1_SUB
+	KERNEL2x1_SUB
+	KERNEL2x1_SUB
+
+	KERNEL2x1_SUB
+	KERNEL2x1_SUB
+	KERNEL2x1_SUB
+	KERNEL2x1_SUB
+
+
+	dec %rax
+	jne	.L1_32
+
+.L1_36:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_39
+
+	ALIGN_4
+
+.L1_37:
+
+	KERNEL2x1_SUB
+
+	dec %rax
+	jne	.L1_37
+
+.L1_39:
+
+	SAVE2x1
+
+.L1_40:
+	testq	$1, M		
+	jz	.L1_100		// to next 3 lines of N
+
+
+.L1_41:
+        movq    B, BO             // first buffer to BO
+        addq    $12 * SIZE, BO
+
+	INIT1x1
+
+        movq    K, %rax
+
+	sarq	$3,%rax
+	je	.L1_46
+
+	ALIGN_4
+
+.L1_42:
+
+	KERNEL1x1_SUB
+	KERNEL1x1_SUB
+	KERNEL1x1_SUB
+	KERNEL1x1_SUB
+
+	KERNEL1x1_SUB
+	KERNEL1x1_SUB
+	KERNEL1x1_SUB
+	KERNEL1x1_SUB
+
+	dec %rax
+	jne	.L1_42
+
+.L1_46:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_49
+
+	ALIGN_4
+
+.L1_47:
+
+	KERNEL1x1_SUB
+
+	dec	%rax
+	jne	.L1_47
+
+
+.L1_49:
+
+	SAVE1x1
+
+.L1_100:
+
+
+
+
+.L999:
+	vzeroupper
+
+	movq   		SP, %rsp
+	movq	   (%rsp), %rbx
+	movq	  8(%rsp), %rbp
+	movq	 16(%rsp), %r12
+	movq	 24(%rsp), %r13
+	movq	 32(%rsp), %r14
+	movq	 40(%rsp), %r15
+
+#ifdef WINDOWS_ABI
+	movq	 48(%rsp), %rdi
+	movq	 56(%rsp), %rsi
+	vmovups	 64(%rsp), %xmm6
+	vmovups	 80(%rsp), %xmm7
+	vmovups	 96(%rsp), %xmm8
+	vmovups	112(%rsp), %xmm9
+	vmovups	128(%rsp), %xmm10
+	vmovups	144(%rsp), %xmm11
+	vmovups	160(%rsp), %xmm12
+	vmovups	176(%rsp), %xmm13
+	vmovups	192(%rsp), %xmm14
+	vmovups	208(%rsp), %xmm15
+#endif
+
+	addq	$STACKSIZE, %rsp
+	ret
+
+	EPILOGUE
+
+
+#else
+/*************************************************************************************
+* TRMM Kernel
+*************************************************************************************/
+
+
+	PROLOGUE
+	PROFCODE
+	
+	subq	$STACKSIZE, %rsp
+	movq	%rbx,   (%rsp)
+	movq	%rbp,  8(%rsp)
+	movq	%r12, 16(%rsp)
+	movq	%r13, 24(%rsp)
+	movq	%r14, 32(%rsp)
+	movq	%r15, 40(%rsp)
+
+	vzeroupper
+
+#ifdef WINDOWS_ABI
+	movq	%rdi,    48(%rsp)
+	movq	%rsi,    56(%rsp)
+	vmovups	%xmm6,   64(%rsp)
+	vmovups	%xmm7,   80(%rsp)
+	vmovups	%xmm8,   96(%rsp)
+	vmovups	%xmm9,  112(%rsp)
+	vmovups	%xmm10, 128(%rsp)
+	vmovups	%xmm11, 144(%rsp)
+	vmovups	%xmm12, 160(%rsp)
+	vmovups	%xmm13, 176(%rsp)
+	vmovups	%xmm14, 192(%rsp)
+	vmovups	%xmm15, 208(%rsp)
+
+	movq	ARG1,      OLD_M
+	movq	ARG2,      OLD_N
+	movq	ARG3,      OLD_K
+	movq	OLD_A,     A
+	movq	OLD_B,     B
+	movq	OLD_C,     C
+	movq	OLD_LDC,   LDC
+#ifdef TRMMKERNEL
+	vmovsd	OLD_OFFSET, %xmm12
+#endif
+	vmovups	%xmm3, %xmm0
+
+#else
+	movq	STACKSIZE +  8(%rsp), LDC
+#ifdef TRMMKERNEL
+	vmovsd	STACKSIZE + 16(%rsp), %xmm12
+#endif
+
+#endif
+
+	movq    %rsp, SP      # save old stack
+        subq    $128 + L_BUFFER_SIZE, %rsp
+        andq    $-4096, %rsp    # align stack
+
+        STACK_TOUCH
+
+	cmpq	$ 0, OLD_M
+	je	.L999
+
+	cmpq	$ 0, OLD_N
+	je	.L999
+
+	cmpq	$ 0, OLD_K
+	je	.L999
+
+	movq	OLD_M, M
+	movq	OLD_N, N
+	movq	OLD_K, K
+
+	vmovsd	 %xmm0, ALPHA
+
+	salq	$BASE_SHIFT, LDC
+
+	movq    N, %rax
+        xorq    %rdx, %rdx
+        movq    $4,  %rdi
+        divq    %rdi                     //    N / 4
+        movq    %rax, Ndiv12             //    N / 4
+        movq    %rdx, Nmod12             //    N % 4
+
+#ifdef TRMMKERNEL
+        vmovsd  %xmm12, OFFSET
+        vmovsd  %xmm12, KK
+#ifndef LEFT
+        negq    KK
+#endif  
+#endif
+
+
+
+	movq	Ndiv12,  J
+	cmpq	$ 0, J
+	je	.L2_0
+	ALIGN_4
+
+.L4_10:
+	movq	C, CO1
+	leaq	(C, LDC, 4), C		// c += 4 * ldc
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        movq    OFFSET, %rax
+        movq    %rax, KK
+#endif
+
+	
+	movq	A, AO		 	// aoffset = a
+	addq	$16 * SIZE, AO
+
+	movq	M,  I
+	sarq	$2, I			// i = m / 4
+	je	.L4_20
+
+	ALIGN_4
+
+.L4_11:
+
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    B, BO        
+        addq    $12 * SIZE, BO
+#else
+        movq    B, BO        
+        addq    $12 * SIZE, BO
+        movq    KK, %rax
+	salq	$3, %rax		// rax * SIZE
+	leaq	(BO,%rax,4), BO		// add number of values in B
+	leaq	(AO,%rax,4), AO		// add number of values in A
+#endif
+
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $4, %rax        // number of values in AO
+#else
+        addq    $4, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+	sarq	$3, %rax			//  K / 8
+	cmpq    $2, %rax
+	jl	.L4_13
+
+
+	KERNEL4x4_I
+	KERNEL4x4_M2
+	KERNEL4x4_M1
+	KERNEL4x4_M2
+
+	KERNEL4x4_M1
+	KERNEL4x4_M2
+	KERNEL4x4_M1
+	KERNEL4x4_M2
+
+	subq $2, %rax
+	je	.L4_12a
+
+	ALIGN_5
+
+.L4_12:
+
+	KERNEL4x4_M1
+	KERNEL4x4_M2
+	KERNEL4x4_M1
+	KERNEL4x4_M2
+
+	KERNEL4x4_M1
+	KERNEL4x4_M2
+	KERNEL4x4_M1
+	KERNEL4x4_M2
+
+	dec	%rax
+	jne	.L4_12
+
+.L4_12a:
+
+	KERNEL4x4_M1
+	KERNEL4x4_M2
+	KERNEL4x4_M1
+	KERNEL4x4_M2
+
+	KERNEL4x4_M1
+	KERNEL4x4_M2
+	KERNEL4x4_M1
+	KERNEL4x4_E
+
+	jmp .L4_16
+
+
+.L4_13:
+
+	test $1, %rax
+	jz .L4_14
+
+	KERNEL4x4_I
+	KERNEL4x4_M2
+	KERNEL4x4_M1
+	KERNEL4x4_M2
+
+	KERNEL4x4_M1
+	KERNEL4x4_M2
+	KERNEL4x4_M1
+	KERNEL4x4_E
+
+	jmp .L4_16
+
+
+.L4_14:
+
+	INIT4x4
+
+
+.L4_16:
+        movq    KKK, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L4_19
+
+	ALIGN_4
+
+.L4_17:
+
+	KERNEL4x4_SUB
+
+	dec	%rax
+	jne	.L4_17
+	ALIGN_4
+
+
+.L4_19:
+
+	SAVE4x4
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+	salq	$3, %rax			// rax + SIZE
+        leaq    (BO, %rax, 4), BO		// number of values in B
+        leaq    (AO, %rax, 4), AO		// number of values in A
+#endif
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $4, KK				// number of values in A
+#endif
+
+	decq	I			# i --
+	jg	.L4_11
+	ALIGN_4	
+
+/**************************************************************************
+* Rest of M 
+***************************************************************************/
+.L4_20:
+	// Test rest of M
+
+	testq	$3, M
+	jz	.L4_100			// to next 16 lines of N
+
+
+.L4_30:
+	testq	$2, M		
+	jz	.L4_40
+
+	ALIGN_4
+
+.L4_31:
+
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    B, BO        
+        addq    $12 * SIZE, BO
+#else
+        movq    B, BO        
+        addq    $12 * SIZE, BO
+        movq    KK, %rax
+	salq	$3, %rax		// rax * SIZE
+	leaq	(BO,%rax,4), BO		// add number of values in B
+	leaq	(AO,%rax,2), AO		// add number of values in A
+#endif
+
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $2, %rax        // number of values in AO
+#else
+        addq    $4, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+	INIT2x4
+
+	sarq	$3, %rax
+	je	.L4_36
+	ALIGN_4
+
+.L4_32:
+
+	KERNEL2x4_SUB
+	KERNEL2x4_SUB
+	KERNEL2x4_SUB
+	KERNEL2x4_SUB
+
+	KERNEL2x4_SUB
+	KERNEL2x4_SUB
+	KERNEL2x4_SUB
+	KERNEL2x4_SUB
+
+	dec %rax
+	jne	.L4_32
+	ALIGN_4
+
+.L4_36:
+        movq    KKK, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L4_39
+
+	ALIGN_4
+
+.L4_37:
+
+	KERNEL2x4_SUB
+
+	dec %rax
+	jne	.L4_37
+
+
+.L4_39:
+
+	SAVE2x4
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+	salq	$3, %rax			// rax + SIZE
+        leaq    (BO, %rax, 4), BO		// number of values in B
+        leaq    (AO, %rax, 2), AO		// number of values in A
+#endif
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $2, KK				// number of values in A
+#endif
+
+
+.L4_40:
+	testq	$1, M		
+	jz	.L4_100		// to next 3 lines of N
+
+	ALIGN_4
+
+.L4_41:
+
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    B, BO        
+        addq    $12 * SIZE, BO
+#else
+        movq    B, BO        
+        addq    $12 * SIZE, BO
+        movq    KK, %rax
+	salq	$3, %rax		// rax * SIZE
+	leaq	(BO,%rax,4), BO		// add number of values in B
+	leaq	(AO,%rax,1), AO		// add number of values in A
+#endif
+
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $1, %rax        // number of values in AO
+#else
+        addq    $4, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+	INIT1x4
+
+	sarq	$3,%rax
+	je	.L4_46
+
+	ALIGN_4
+
+.L4_42:
+
+	KERNEL1x4_SUB
+	KERNEL1x4_SUB
+	KERNEL1x4_SUB
+	KERNEL1x4_SUB
+
+	KERNEL1x4_SUB
+	KERNEL1x4_SUB
+	KERNEL1x4_SUB
+	KERNEL1x4_SUB
+
+	dec %rax
+	jne	.L4_42
+	ALIGN_4
+
+.L4_46:
+        movq    KKK, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L4_49
+
+	ALIGN_4
+
+.L4_47:
+
+	KERNEL1x4_SUB
+
+	dec	%rax
+	jne	.L4_47
+	ALIGN_4
+
+
+.L4_49:
+
+	SAVE1x4
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+	salq	$3, %rax			// rax + SIZE
+        leaq    (BO, %rax, 4), BO		// number of values in B
+        leaq    (AO, %rax, 1), AO		// number of values in A
+#endif
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $1, KK				// number of values in A
+#endif
+
+.L4_100:
+
+#if defined(TRMMKERNEL) && !defined(LEFT)
+        addq    $4, KK				// number of values in B
+#endif
+
+
+	movq	K, %rax
+	salq	$2, %rax		// * 4
+	leaq	(B , %rax, SIZE), B
+	decq	J			// j --
+	jg	.L4_10
+
+
+
+
+/***************************************************************************************************************/
+
+.L2_0:
+
+	movq	Nmod12, J		
+	testq	$2, J
+	je	.L1_0
+
+.L2_10:
+	movq	C, CO1
+	leaq	(C, LDC, 2), C		// c += 2 * ldc
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        movq    OFFSET, %rax
+        movq    %rax, KK
+#endif
+
+
+	
+	movq	A, AO		 	// aoffset = a
+	addq	$16 * SIZE, AO
+
+	movq	M,  I
+	sarq	$2, I			// i = m / 4
+	je	.L2_20
+
+	ALIGN_4
+
+.L2_11:
+
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    B, BO        
+        addq    $12 * SIZE, BO
+#else
+        movq    B, BO        
+        addq    $12 * SIZE, BO
+        movq    KK, %rax
+	salq	$3, %rax		// rax * SIZE
+	leaq	(BO,%rax,2), BO		// add number of values in B
+	leaq	(AO,%rax,4), AO		// add number of values in A
+#endif
+
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $4, %rax        // number of values in AO
+#else
+        addq    $2, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+	INIT4x2
+
+	sarq $3, %rax			//  K / 8
+
+	je	.L2_16
+
+	ALIGN_5
+
+.L2_12:
+
+	KERNEL4x2_SUB
+	KERNEL4x2_SUB
+	KERNEL4x2_SUB
+	KERNEL4x2_SUB
+
+	KERNEL4x2_SUB
+	KERNEL4x2_SUB
+	KERNEL4x2_SUB
+	KERNEL4x2_SUB
+
+	dec	%rax
+	jne	.L2_12
+
+
+.L2_16:
+        movq    KKK, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_19
+
+	ALIGN_4
+
+.L2_17:
+
+	KERNEL4x2_SUB
+
+	dec	%rax
+	jne	.L2_17
+	ALIGN_4
+
+
+.L2_19:
+
+	SAVE4x2
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+	salq	$3, %rax			// rax + SIZE
+        leaq    (BO, %rax, 2), BO		// number of values in B
+        leaq    (AO, %rax, 4), AO		// number of values in A
+#endif
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $4, KK				// number of values in A
+#endif
+
+
+	decq	I			# i --
+	jg	.L2_11
+	ALIGN_4	
+
+/**************************************************************************
+* Rest of M 
+***************************************************************************/
+.L2_20:
+	// Test rest of M
+
+	testq	$3, M
+	jz	.L2_100			// to next 16 lines of N
+
+
+.L2_30:
+	testq	$2, M		
+	jz	.L2_40
+
+	ALIGN_4
+
+.L2_31:
+
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    B, BO        
+        addq    $12 * SIZE, BO
+#else
+        movq    B, BO        
+        addq    $12 * SIZE, BO
+        movq    KK, %rax
+	salq	$3, %rax		// rax * SIZE
+	leaq	(BO,%rax,2), BO		// add number of values in B
+	leaq	(AO,%rax,2), AO		// add number of values in A
+#endif
+
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $2, %rax        // number of values in AO
+#else
+        addq    $2, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+	INIT2x2
+
+	sarq	$3, %rax
+	je	.L2_36
+	ALIGN_4
+
+.L2_32:
+
+	KERNEL2x2_SUB
+	KERNEL2x2_SUB
+	KERNEL2x2_SUB
+	KERNEL2x2_SUB
+
+	KERNEL2x2_SUB
+	KERNEL2x2_SUB
+	KERNEL2x2_SUB
+	KERNEL2x2_SUB
+
+	dec %rax
+	jne	.L2_32
+
+.L2_36:
+        movq    KKK, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_39
+
+	ALIGN_4
+
+.L2_37:
+
+	KERNEL2x2_SUB
+
+	dec %rax
+	jne	.L2_37
+
+
+.L2_39:
+
+	SAVE2x2
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+	salq	$3, %rax			// rax + SIZE
+        leaq    (BO, %rax, 2), BO		// number of values in B
+        leaq    (AO, %rax, 2), AO		// number of values in A
+#endif
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $2, KK				// number of values in A
+#endif
+
+
+.L2_40:
+	testq	$1, M		
+	jz	.L2_100		// to next 3 lines of N
+
+.L2_41:
+
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    B, BO        
+        addq    $12 * SIZE, BO
+#else
+        movq    B, BO        
+        addq    $12 * SIZE, BO
+        movq    KK, %rax
+	salq	$3, %rax		// rax * SIZE
+	leaq	(BO,%rax,2), BO		// add number of values in B
+	leaq	(AO,%rax,1), AO		// add number of values in A
+#endif
+
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $1, %rax        // number of values in AO
+#else
+        addq    $2, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+	INIT1x2
+
+	sarq	$3,%rax
+	je	.L2_46
+
+	ALIGN_4
+
+.L2_42:
+
+	KERNEL1x2_SUB
+	KERNEL1x2_SUB
+	KERNEL1x2_SUB
+	KERNEL1x2_SUB
+
+	KERNEL1x2_SUB
+	KERNEL1x2_SUB
+	KERNEL1x2_SUB
+	KERNEL1x2_SUB
+
+	dec %rax
+	jne	.L2_42
+
+.L2_46:
+        movq    KKK, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_49
+
+	ALIGN_4
+
+.L2_47:
+
+	KERNEL1x2_SUB
+
+	dec	%rax
+	jne	.L2_47
+
+.L2_49:
+
+	SAVE1x2
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+	salq	$3, %rax			// rax * SIZE
+        leaq    (BO, %rax, 2), BO		// number of values in B
+        leaq    (AO, %rax, 1), AO		// number of values in A
+#endif
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $1, KK				// number of values in A
+#endif
+
+
+.L2_100:
+
+
+#if defined(TRMMKERNEL) && !defined(LEFT)
+        addq    $2, KK				// number of values in B
+#endif
+
+	movq	K, %rax
+	salq	$1, %rax		// * 2
+	leaq	(B , %rax, SIZE), B
+
+/***************************************************************************************************************/
+
+.L1_0:
+
+	movq	Nmod12, J		
+	testq	$1, J
+	je	.L999
+
+.L1_10:
+	movq	C, CO1
+	leaq	(C, LDC, 1), C		// c += 1 * ldc
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        movq    OFFSET, %rax
+        movq    %rax, KK
+#endif
+	
+	movq	A, AO		 	// aoffset = a
+	addq	$16 * SIZE, AO
+
+	movq	M,  I
+	sarq	$2, I			// i = m / 4
+	je	.L1_20
+
+	ALIGN_4
+
+.L1_11:
+
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    B, BO        
+        addq    $12 * SIZE, BO
+#else
+        movq    B, BO        
+        addq    $12 * SIZE, BO
+        movq    KK, %rax
+	salq	$3, %rax		// rax * SIZE
+	leaq	(BO,%rax,1), BO		// add number of values in B
+	leaq	(AO,%rax,4), AO		// add number of values in A
+#endif
+
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $4, %rax        // number of values in AO
+#else
+        addq    $1, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+	INIT4x1
+
+	sarq	$3, %rax			//  K / 8
+	je	.L1_16
+
+	ALIGN_5
+
+.L1_12:
+
+	KERNEL4x1
+
+	dec	%rax
+	jne	.L1_12
+
+
+.L1_16:
+        movq    KKK, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_19
+
+	ALIGN_4
+
+.L1_17:
+
+	KERNEL4x1_SUB
+
+	dec	%rax
+	jne	.L1_17
+	ALIGN_4
+
+
+.L1_19:
+
+	SAVE4x1
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+	salq	$3, %rax			// rax * SIZE
+        leaq    (BO, %rax, 1), BO		// number of values in B
+        leaq    (AO, %rax, 4), AO		// number of values in A
+#endif
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $4, KK				// number of values in A
+#endif
+
+
+	decq	I			# i --
+	jg	.L1_11
+
+/**************************************************************************
+* Rest of M 
+***************************************************************************/
+.L1_20:
+	// Test rest of M
+
+	testq	$3, M
+	jz	.L1_100	
+
+
+.L1_30:
+	testq	$2, M		
+	jz	.L1_40
+
+	ALIGN_4
+
+.L1_31:
+
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    B, BO        
+        addq    $12 * SIZE, BO
+#else
+        movq    B, BO        
+        addq    $12 * SIZE, BO
+        movq    KK, %rax
+	salq	$3, %rax		// rax * SIZE
+	leaq	(BO,%rax,1), BO		// add number of values in B
+	leaq	(AO,%rax,2), AO		// add number of values in A
+#endif
+
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $2, %rax        // number of values in AO
+#else
+        addq    $1, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+	INIT2x1
+
+	sarq	$3, %rax
+	je	.L1_36
+	ALIGN_4
+
+.L1_32:
+
+	KERNEL2x1_SUB
+	KERNEL2x1_SUB
+	KERNEL2x1_SUB
+	KERNEL2x1_SUB
+
+	KERNEL2x1_SUB
+	KERNEL2x1_SUB
+	KERNEL2x1_SUB
+	KERNEL2x1_SUB
+
+
+	dec %rax
+	jne	.L1_32
+
+.L1_36:
+        movq    KKK, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_39
+
+	ALIGN_4
+
+.L1_37:
+
+	KERNEL2x1_SUB
+
+	dec %rax
+	jne	.L1_37
+
+.L1_39:
+
+	SAVE2x1
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+	salq	$3, %rax			// rax * SIZE
+        leaq    (BO, %rax, 1), BO		// number of values in B
+        leaq    (AO, %rax, 2), AO		// number of values in A
+#endif
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $2, KK				// number of values in A
+#endif
+
+
+.L1_40:
+	testq	$1, M		
+	jz	.L1_100		// to next 3 lines of N
+
+
+.L1_41:
+
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    B, BO        
+        addq    $12 * SIZE, BO
+#else
+        movq    B, BO        
+        addq    $12 * SIZE, BO
+        movq    KK, %rax
+	salq	$3, %rax		// rax * SIZE
+	leaq	(BO,%rax,1), BO		// add number of values in B
+	leaq	(AO,%rax,1), AO		// add number of values in A
+#endif
+
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $1, %rax        // number of values in AO
+#else
+        addq    $1, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+	INIT1x1
+
+	sarq	$3,%rax
+	je	.L1_46
+
+	ALIGN_4
+
+.L1_42:
+
+	KERNEL1x1_SUB
+	KERNEL1x1_SUB
+	KERNEL1x1_SUB
+	KERNEL1x1_SUB
+
+	KERNEL1x1_SUB
+	KERNEL1x1_SUB
+	KERNEL1x1_SUB
+	KERNEL1x1_SUB
+
+	dec %rax
+	jne	.L1_42
+
+.L1_46:
+        movq    KKK, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_49
+
+	ALIGN_4
+
+.L1_47:
+
+	KERNEL1x1_SUB
+
+	dec	%rax
+	jne	.L1_47
+
+
+.L1_49:
+
+	SAVE1x1
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+	salq	$3, %rax			// rax * SIZE
+        leaq    (BO, %rax, 1), BO		// number of values in B
+        leaq    (AO, %rax, 1), AO		// number of values in A
+#endif
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $1, KK				// number of values in A
+#endif
+
+
+
+.L1_100:
+
+
+#if defined(TRMMKERNEL) && !defined(LEFT)
+        addq    $1, KK				// number of values in B
+#endif
+
+
+
+.L999:
+
+	vzeroupper
+
+	movq   		SP, %rsp
+	movq	   (%rsp), %rbx
+	movq	  8(%rsp), %rbp
+	movq	 16(%rsp), %r12
+	movq	 24(%rsp), %r13
+	movq	 32(%rsp), %r14
+	movq	 40(%rsp), %r15
+
+#ifdef WINDOWS_ABI
+	movq	 48(%rsp), %rdi
+	movq	 56(%rsp), %rsi
+	vmovups	 64(%rsp), %xmm6
+	vmovups	 80(%rsp), %xmm7
+	vmovups	 96(%rsp), %xmm8
+	vmovups	112(%rsp), %xmm9
+	vmovups	128(%rsp), %xmm10
+	vmovups	144(%rsp), %xmm11
+	vmovups	160(%rsp), %xmm12
+	vmovups	176(%rsp), %xmm13
+	vmovups	192(%rsp), %xmm14
+	vmovups	208(%rsp), %xmm15
+#endif
+
+	addq	$STACKSIZE, %rsp
+	ret
+
+	EPILOGUE
+
+
+
+
+
+#endif
diff --git a/kernel/x86_64/dgemm_kernel_4x8_haswell.S b/kernel/x86_64/dgemm_kernel_4x8_haswell.S
index 19e32ef2c..adaa28bbc 100644
--- a/kernel/x86_64/dgemm_kernel_4x8_haswell.S
+++ b/kernel/x86_64/dgemm_kernel_4x8_haswell.S
@@ -1,5153 +1,5153 @@
-/*********************************************************************************
-Copyright (c) 2015, The OpenBLAS Project
-All rights reserved.
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in
-the documentation and/or other materials provided with the
-distribution.
-3. Neither the name of the OpenBLAS project nor the names of
-its contributors may be used to endorse or promote products
-derived from this software without specific prior written permission.
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
-USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-**********************************************************************************/
-
-
-#define ASSEMBLER
-#include "common.h"
- 
-#define OLD_M	%rdi
-#define OLD_N	%rsi
-#define M	%r13
-#define J	%r14
-#define OLD_K	%rdx
-
-#define A	%rcx
-#define B	%r8
-#define C	%r9
-#define LDC	%r10
-	
-#define I	%r11
-#define AO	%rdi
-#define BO	%rsi
-#define	CO1	%r15
-#define K	%r12
-#define	SP	%rbx
-
-#define BO1	%rdi
-#define BO2	%r15
-#define BO3	%rbp
-
-#ifndef WINDOWS_ABI
-
-#define STACKSIZE 96
-#define L_BUFFER_SIZE 256*8*12+4096
-
-#else
-
-#define STACKSIZE 256
-#define L_BUFFER_SIZE 128*8*12+512
-
-#define OLD_A		40 + STACKSIZE(%rsp)
-#define OLD_B		48 + STACKSIZE(%rsp)
-#define OLD_C		56 + STACKSIZE(%rsp)
-#define OLD_LDC		64 + STACKSIZE(%rsp)
-#define OLD_OFFSET	72 + STACKSIZE(%rsp)
-
-#endif
-
-
-#define Ndiv12	 24(%rsp)
-#define Nmod12	 32(%rsp)
-#define N	 40(%rsp)
-#define ALPHA	 48(%rsp)
-#define OFFSET	 56(%rsp)
-#define KK	 64(%rsp)
-#define KKK	 72(%rsp)
-#define BUFFER1	           128(%rsp)
-
-#if defined(OS_WINDOWS)
-#if   L_BUFFER_SIZE > 16384
-#define STACK_TOUCH \
-        movl    $ 0,  4096 * 4(%rsp);\
-        movl    $ 0,  4096 * 3(%rsp);\
-        movl    $ 0,  4096 * 2(%rsp);\
-        movl    $ 0,  4096 * 1(%rsp);
-#elif L_BUFFER_SIZE > 12288
-#define STACK_TOUCH \
-        movl    $ 0,  4096 * 3(%rsp);\
-        movl    $ 0,  4096 * 2(%rsp);\
-        movl    $ 0,  4096 * 1(%rsp);
-#elif L_BUFFER_SIZE > 8192
-#define STACK_TOUCH \
-        movl    $ 0,  4096 * 2(%rsp);\
-        movl    $ 0,  4096 * 1(%rsp);
-#elif L_BUFFER_SIZE > 4096
-#define STACK_TOUCH \
-        movl    $ 0,  4096 * 1(%rsp);
-#else
-#define STACK_TOUCH
-#endif
-#else
-#define STACK_TOUCH
-#endif
-
-#define	A_PR1	512
-#define	B_PR1	160
-#define BROADCASTKERNEL
-
-/*******************************************************************************************
-* Macro definitions
-*******************************************************************************************/
-
-.macro INIT4x12
-
-	vxorpd		%ymm4 , %ymm4 , %ymm4
-	vxorpd		%ymm5 , %ymm5 , %ymm5
-	vxorpd		%ymm6 , %ymm6 , %ymm6
-	vxorpd		%ymm7 , %ymm7 , %ymm7
-	vxorpd		%ymm8 , %ymm8 , %ymm8
-	vxorpd		%ymm9 , %ymm9 , %ymm9
-	vxorpd		%ymm10, %ymm10, %ymm10
-	vxorpd		%ymm11, %ymm11, %ymm11
-	vxorpd		%ymm12, %ymm12, %ymm12
-	vxorpd		%ymm13, %ymm13, %ymm13
-	vxorpd		%ymm14, %ymm14, %ymm14
-	vxorpd		%ymm15, %ymm15, %ymm15
-
-.endm
-
-.macro KERNEL4x12_I
-	prefetcht0	A_PR1(AO)
-	vmovups		-12 * SIZE(BO), %ymm1
-	prefetcht0	B_PR1(BO)
-# if defined BROADCASTKERNEL
-        vbroadcastsd    -16 * SIZE(AO), %ymm0
-# else
-	vmovups 	-16 * SIZE(AO), %ymm0
-# endif
-	prefetcht0	B_PR1+64(BO)
-	vmovups		 -8 * SIZE(BO), %ymm2
-	prefetcht0	B_PR1+128(BO)
-	vmovups		 -4 * SIZE(BO), %ymm3
-	vmulpd  	%ymm0 ,%ymm1  , %ymm4
-	prefetcht0	B_PR1+192(BO)
-	vmulpd  	%ymm0 ,%ymm2  , %ymm8
-	vmulpd  	%ymm0 ,%ymm3  , %ymm12
-	prefetcht0	B_PR1+256(BO)
-# if defined BROADCASTKERNEL
-        vbroadcastsd    -15 * SIZE(AO), %ymm0
-# else
-	vpermilpd	$ 0x05, %ymm0  , %ymm0
-# endif
-	vmulpd  	%ymm0 ,%ymm1  , %ymm5
-	vmulpd  	%ymm0 ,%ymm2  , %ymm9
-	vmulpd  	%ymm0 ,%ymm3  , %ymm13
-# if defined BROADCASTKERNEL
-        vbroadcastsd    -14 * SIZE(AO), %ymm0
-# else
-	vpermpd		$ 0x1b, %ymm0  , %ymm0
-# endif
-	vmulpd  	%ymm0 ,%ymm1  , %ymm6
-	vmulpd  	%ymm0 ,%ymm2  , %ymm10
-
-	addq		$ 12*SIZE, BO
-	vmulpd  	%ymm0 ,%ymm3  , %ymm14
-# if defined BROADCASTKERNEL
-        vbroadcastsd    -13 * SIZE(AO), %ymm0
-# else
-	vpermilpd	$ 0x05, %ymm0  , %ymm0
-# endif
-	vmulpd  	%ymm0 ,%ymm1  , %ymm7
-	vmovups		-12 * SIZE(BO), %ymm1
-	vmulpd  	%ymm0 ,%ymm2  , %ymm11
-	vmovups		 -8 * SIZE(BO), %ymm2
-	vmulpd  	%ymm0 ,%ymm3  , %ymm15
-	vmovups		 -4 * SIZE(BO), %ymm3
-
-.endm
-
-.macro KERNEL4x12_M1
-	prefetcht0	A_PR1(AO)
-# if defined BROADCASTKERNEL
-        vbroadcastsd    -16 * SIZE(AO), %ymm0
-# else
-	vmovups 	-16 * SIZE(AO), %ymm0
-# endif
-	prefetcht0	B_PR1(BO)
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4
-	prefetcht0	B_PR1+64(BO)
-	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm8
-	prefetcht0	B_PR1+128(BO)
-	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm12
-# if defined BROADCASTKERNEL
-        vbroadcastsd    -15 * SIZE(AO), %ymm0
-# else
-	vpermilpd	$ 0x05, %ymm0  , %ymm0
-# endif
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5
-	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm9
-	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm13
-# if defined BROADCASTKERNEL
-        vbroadcastsd    -14 * SIZE(AO), %ymm0
-# else
-	vpermpd		$ 0x1b, %ymm0  , %ymm0
-# endif
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6
-	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm10
-	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm14
-# if defined BROADCASTKERNEL
-        vbroadcastsd    -13 * SIZE(AO), %ymm0
-# else
-	vpermilpd	$ 0x05, %ymm0  , %ymm0
-# endif
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7
-	vmovups		-12 * SIZE(BO), %ymm1
-	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm11
-	vmovups		 -8 * SIZE(BO), %ymm2
-	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm15
-	vmovups		 -4 * SIZE(BO), %ymm3
-
-.endm
-
-.macro KERNEL4x12_M2
-# if defined BROADCASTKERNEL
-        vbroadcastsd    -12 * SIZE(AO), %ymm0
-# else
-	vmovups 	-12 * SIZE(AO), %ymm0
-# endif
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4
-	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm8
-	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm12
-# if defined BROADCASTKERNEL
-        vbroadcastsd    -11 * SIZE(AO), %ymm0
-# else
-	vpermilpd	$ 0x05, %ymm0  , %ymm0
-# endif
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5
-	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm9
-	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm13
-# if defined BROADCASTKERNEL
-        vbroadcastsd    -10 * SIZE(AO), %ymm0
-# else
-	vpermpd		$ 0x1b, %ymm0  , %ymm0
-# endif
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6
-	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm10
-
-	addq		$ 8*SIZE, AO
-	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm14
-# if defined BROADCASTKERNEL
-        vbroadcastsd    -17 * SIZE(AO), %ymm0
-# else
-	vpermilpd	$ 0x05, %ymm0  , %ymm0
-# endif
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7
-	vmovups		  0 * SIZE(BO), %ymm1
-	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm11
-	vmovups		  4 * SIZE(BO), %ymm2
-	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm15
-	vmovups		  8 * SIZE(BO), %ymm3
-	addq		$ 24*SIZE, BO
-.endm
-
-
-.macro KERNEL4x12_E
-# if defined BROADCASTKERNEL
-        vbroadcastsd    -12 * SIZE(AO), %ymm0
-# else
-	vmovups 	-12 * SIZE(AO), %ymm0
-# endif
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4
-	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm8
-	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm12
-# if defined BROADCASTKERNEL
-        vbroadcastsd    -11 * SIZE(AO), %ymm0
-# else
-	vpermilpd	$ 0x05, %ymm0  , %ymm0
-# endif
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5
-	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm9
-	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm13
-# if defined BROADCASTKERNEL
-        vbroadcastsd    -10 * SIZE(AO), %ymm0
-# else
-	vpermpd		$ 0x1b, %ymm0  , %ymm0
-# endif
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6
-	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm10
-
-	addq		$ 8*SIZE, AO
-	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm14
-# if defined BROADCASTKERNEL
-        vbroadcastsd    -17 * SIZE(AO), %ymm0
-# else
-	vpermilpd	$ 0x05, %ymm0  , %ymm0
-# endif
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7
-	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm11
-	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm15
-	addq		$ 12*SIZE, BO
-.endm
-
-.macro KERNEL4x12_SUB
-	vmovups		-12 * SIZE(BO), %ymm1
-# if defined BROADCASTKERNEL
-        vbroadcastsd    -16 * SIZE(AO), %ymm0
-# else
-	vmovups 	-16 * SIZE(AO), %ymm0
-# endif
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4
-	vmovups		 -8 * SIZE(BO), %ymm2
-	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm8
-	vmovups		 -4 * SIZE(BO), %ymm3
-	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm12
-# if defined BROADCASTKERNEL
-        vbroadcastsd    -15 * SIZE(AO), %ymm0
-# else
-	vpermilpd	$ 0x05, %ymm0  , %ymm0
-# endif
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5
-	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm9
-	addq		$ 12*SIZE, BO
-	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm13
-# if defined BROADCASTKERNEL
-        vbroadcastsd    -14 * SIZE(AO), %ymm0
-# else
-	vpermpd		$ 0x1b, %ymm0  , %ymm0
-# endif
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6
-	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm10
-	addq		$ 4*SIZE, AO
-	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm14
-# if defined BROADCASTKERNEL
-        vbroadcastsd    -17 * SIZE(AO), %ymm0
-# else
-	vpermilpd	$ 0x05, %ymm0  , %ymm0
-# endif
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7
-	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm11
-	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm15
-
-.endm
-
-
-.macro SAVE4x12
-
-        prefetcht0      BUFFER1
-	vbroadcastsd	ALPHA, %ymm0
-
-	vmulpd	%ymm0 , %ymm4 , %ymm4
-	vmulpd	%ymm0 , %ymm5 , %ymm5
-	vmulpd	%ymm0 , %ymm6 , %ymm6
-	vmulpd	%ymm0 , %ymm7 , %ymm7
-        prefetcht0      64 + BUFFER1
-	vmulpd	%ymm0 , %ymm8 , %ymm8
-	vmulpd	%ymm0 , %ymm9 , %ymm9
-	vmulpd	%ymm0 , %ymm10, %ymm10
-	vmulpd	%ymm0 , %ymm11, %ymm11
-#if B_PR1 > 32
-        prefetcht0      128 + BUFFER1
-#endif
-	vmulpd	%ymm0 , %ymm12, %ymm12
-	vmulpd	%ymm0 , %ymm13, %ymm13
-	vmulpd	%ymm0 , %ymm14, %ymm14
-	vmulpd	%ymm0 , %ymm15, %ymm15
-#if B_PR1 > 96
-        prefetcht0      192 + BUFFER1
-#endif
-
-#if defined BROADCASTKERNEL
-        vperm2f128 $ 0x20 , %ymm6, %ymm4 , %ymm0
-        vperm2f128 $ 0x20 , %ymm7, %ymm5 , %ymm1
-        vperm2f128 $ 0x31 , %ymm6, %ymm4 , %ymm2
-        vperm2f128 $ 0x31 , %ymm7, %ymm5 , %ymm3
-#else
-	vpermilpd $ 0x05 , %ymm5, %ymm5
-	vpermilpd $ 0x05 , %ymm7, %ymm7
-#endif
-
-#if B_PR1 > 160
-        prefetcht0      256 + BUFFER1
-#endif
-
-#if defined BROADCASTKERNEL
-        vunpcklpd %ymm1, %ymm0, %ymm4
-        vunpckhpd %ymm1, %ymm0, %ymm5
-        vunpcklpd %ymm3, %ymm2, %ymm6
-        vunpckhpd %ymm3, %ymm2, %ymm7
-#else
-	vblendpd $ 0x0a, %ymm5, %ymm4, %ymm0
-	vblendpd $ 0x05, %ymm5, %ymm4, %ymm1
-	vblendpd $ 0x0a, %ymm7, %ymm6, %ymm2
-	vblendpd $ 0x05, %ymm7, %ymm6, %ymm3
-#endif
-
-#if B_PR1 > 224
-        prefetcht0      320 + BUFFER1
-#endif
-
-#ifndef BROADCASTKERNEL
-	vperm2f128 $ 0x01 , %ymm2, %ymm2 , %ymm2
-	vperm2f128 $ 0x01 , %ymm3, %ymm3 , %ymm3
-#endif
-
-#if B_PR1 > 288
-        prefetcht0      384 + BUFFER1
-#endif
-
-#ifndef BROADCASTKERNEL
-	vblendpd $ 0x03, %ymm0, %ymm2 , %ymm4
-	vblendpd $ 0x03, %ymm1, %ymm3 , %ymm5
-	vblendpd $ 0x03, %ymm2, %ymm0 , %ymm6
-	vblendpd $ 0x03, %ymm3, %ymm1 , %ymm7
-#endif
-
-#if B_PR1 > 352
-        prefetcht0      448 + BUFFER1
-#endif
-        leaq    (CO1, LDC, 2), %rax     
-	
-#if B_PR1 > 416
-        prefetcht0      512 + BUFFER1
-#endif
-
-#if !defined(TRMMKERNEL)
-
-	vaddpd 	                (CO1), %ymm4, %ymm4
-	vaddpd 	           (CO1, LDC), %ymm5, %ymm5
-	vaddpd 	               (%rax), %ymm6, %ymm6
-	vaddpd 	          (%rax, LDC), %ymm7, %ymm7
-
-#endif
-
-	vmovups	%ymm4 ,  	(CO1)
-	vmovups	%ymm5 ,  	(CO1, LDC)
-	vmovups	%ymm6 ,  	(%rax)
-	vmovups	%ymm7 ,  	(%rax, LDC)
-
-	prefetcht1	56(CO1)
-	prefetcht1	56(CO1,LDC)
-	prefetcht1	56(%rax)
-	prefetcht1	56(%rax,LDC)
-
-#if defined BROADCASTKERNEL
-        vperm2f128 $ 0x20 , %ymm10, %ymm8 , %ymm0
-        vperm2f128 $ 0x20 , %ymm11, %ymm9 , %ymm1
-        vperm2f128 $ 0x31 , %ymm10, %ymm8 , %ymm2
-        vperm2f128 $ 0x31 , %ymm11, %ymm9 , %ymm3
-        vunpcklpd %ymm1, %ymm0, %ymm4
-        vunpckhpd %ymm1, %ymm0, %ymm5
-        vunpcklpd %ymm3, %ymm2, %ymm6
-        vunpckhpd %ymm3, %ymm2, %ymm7
-#else
-	vpermilpd $ 0x05 , %ymm9, %ymm9
-	vpermilpd $ 0x05 , %ymm11, %ymm11
-
-	vblendpd $ 0x0a, %ymm9, %ymm8, %ymm0
-	vblendpd $ 0x05, %ymm9, %ymm8, %ymm1
-	vblendpd $ 0x0a, %ymm11, %ymm10, %ymm2
-	vblendpd $ 0x05, %ymm11, %ymm10, %ymm3
-
-	vperm2f128 $ 0x01 , %ymm2, %ymm2 , %ymm2
-	vperm2f128 $ 0x01 , %ymm3, %ymm3 , %ymm3
-
-	vblendpd $ 0x03, %ymm0, %ymm2 , %ymm4
-	vblendpd $ 0x03, %ymm1, %ymm3 , %ymm5
-	vblendpd $ 0x03, %ymm2, %ymm0 , %ymm6
-	vblendpd $ 0x03, %ymm3, %ymm1 , %ymm7
-#endif
-
-	leaq	(%rax, LDC, 2), %rax
-	leaq	(%rax, LDC, 2), %rbp
-
-#if !defined(TRMMKERNEL)
-
-	vaddpd 	                (%rax), %ymm4, %ymm4
-	vaddpd 	           (%rax, LDC), %ymm5, %ymm5
-	vaddpd 	                (%rbp), %ymm6, %ymm6
-	vaddpd 	           (%rbp, LDC), %ymm7, %ymm7
-
-#endif
-
-	vmovups	%ymm4 ,  	(%rax)
-	vmovups	%ymm5 ,  	(%rax, LDC)
-	vmovups	%ymm6 ,  	(%rbp)
-	vmovups	%ymm7 ,  	(%rbp, LDC)
-
-	prefetcht1	56(%rax)
-	prefetcht1	56(%rax,LDC)
-	prefetcht1	56(%rbp)
-	prefetcht1	56(%rbp,LDC)
-
-#if defined BROADCASTKERNEL
-        vperm2f128 $ 0x20 , %ymm14, %ymm12 , %ymm0
-        vperm2f128 $ 0x20 , %ymm15, %ymm13 , %ymm1
-        vperm2f128 $ 0x31 , %ymm14, %ymm12 , %ymm2
-        vperm2f128 $ 0x31 , %ymm15, %ymm13 , %ymm3
-        vunpcklpd %ymm1, %ymm0, %ymm4
-        vunpckhpd %ymm1, %ymm0, %ymm5
-        vunpcklpd %ymm3, %ymm2, %ymm6
-        vunpckhpd %ymm3, %ymm2, %ymm7
-#else
-	vpermilpd $ 0x05 , %ymm13, %ymm13
-	vpermilpd $ 0x05 , %ymm15, %ymm15
-
-	vblendpd $ 0x0a, %ymm13, %ymm12, %ymm0
-	vblendpd $ 0x05, %ymm13, %ymm12, %ymm1
-	vblendpd $ 0x0a, %ymm15, %ymm14, %ymm2
-	vblendpd $ 0x05, %ymm15, %ymm14, %ymm3
-
-	vperm2f128 $ 0x01 , %ymm2, %ymm2 , %ymm2
-	vperm2f128 $ 0x01 , %ymm3, %ymm3 , %ymm3
-
-	vblendpd $ 0x03, %ymm0, %ymm2 , %ymm4
-	vblendpd $ 0x03, %ymm1, %ymm3 , %ymm5
-	vblendpd $ 0x03, %ymm2, %ymm0 , %ymm6
-	vblendpd $ 0x03, %ymm3, %ymm1 , %ymm7
-#endif
-
-	leaq	(%rax, LDC, 4), %rax
-	leaq	(%rbp, LDC, 4), %rbp
-
-#if !defined(TRMMKERNEL)
-
-	vaddpd 	                (%rax), %ymm4, %ymm4
-	vaddpd 	           (%rax, LDC), %ymm5, %ymm5
-	vaddpd 	                (%rbp), %ymm6, %ymm6
-	vaddpd 	           (%rbp, LDC), %ymm7, %ymm7
-
-#endif
-
-	vmovups	%ymm4 ,  	(%rax)
-	vmovups	%ymm5 ,  	(%rax, LDC)
-	vmovups	%ymm6 ,  	(%rbp)
-	vmovups	%ymm7 ,  	(%rbp, LDC)
-
-	prefetcht1	56(%rax)
-	prefetcht1	56(%rax,LDC)
-	prefetcht1	56(%rbp)
-	prefetcht1	56(%rbp,LDC)
-
-	addq	$ 4*SIZE, CO1
-.endm
-
-/******************************************************************************************/
-
-.macro INIT2x12
-
-	vxorpd		%xmm4 , %xmm4 , %xmm4
-	vxorpd		%xmm5 , %xmm5 , %xmm5
-	vxorpd		%xmm6 , %xmm6 , %xmm6
-	vxorpd		%xmm7 , %xmm7 , %xmm7
-	vxorpd		%xmm8 , %xmm8 , %xmm8
-	vxorpd		%xmm9 , %xmm9 , %xmm9
-	vxorpd		%xmm10, %xmm10, %xmm10
-	vxorpd		%xmm11, %xmm11, %xmm11
-	vxorpd		%xmm12, %xmm12, %xmm12
-	vxorpd		%xmm13, %xmm13, %xmm13
-	vxorpd		%xmm14, %xmm14, %xmm14
-	vxorpd		%xmm15, %xmm15, %xmm15
-
-.endm
-
-.macro KERNEL2x12_SUB
-	vmovups 	-16 * SIZE(AO), %xmm0
-	vmovddup	-12 * SIZE(BO), %xmm1
-	vmovddup	-11 * SIZE(BO), %xmm2
-	vmovddup	-10 * SIZE(BO), %xmm3
-	vfmadd231pd  	%xmm0 ,%xmm1  , %xmm4
-	vmovddup	 -9 * SIZE(BO), %xmm1
-	vfmadd231pd  	%xmm0 ,%xmm2  , %xmm5
-	vmovddup	 -8 * SIZE(BO), %xmm2
-	vfmadd231pd  	%xmm0 ,%xmm3  , %xmm6
-	vmovddup	 -7 * SIZE(BO), %xmm3
-	vfmadd231pd  	%xmm0 ,%xmm1  , %xmm7
-	vmovddup	 -6 * SIZE(BO), %xmm1
-	vfmadd231pd  	%xmm0 ,%xmm2  , %xmm8
-	vmovddup	 -5 * SIZE(BO), %xmm2
-	vfmadd231pd  	%xmm0 ,%xmm3  , %xmm9
-	vmovddup	 -4 * SIZE(BO), %xmm3
-	vfmadd231pd  	%xmm0 ,%xmm1  , %xmm10
-	vmovddup	 -3 * SIZE(BO), %xmm1
-	vfmadd231pd  	%xmm0 ,%xmm2  , %xmm11
-	vmovddup	 -2 * SIZE(BO), %xmm2
-	vfmadd231pd  	%xmm0 ,%xmm3  , %xmm12
-	vmovddup	 -1 * SIZE(BO), %xmm3
-	vfmadd231pd  	%xmm0 ,%xmm1  , %xmm13
-	addq		$ 12*SIZE, BO
-	vfmadd231pd  	%xmm0 ,%xmm2  , %xmm14
-	addq		$ 2*SIZE, AO
-	vfmadd231pd  	%xmm0 ,%xmm3  , %xmm15
-
-.endm
-
-.macro SAVE2x12
-
-	vmovddup	ALPHA, %xmm0
-
-	vmulpd	%xmm0 , %xmm4 , %xmm4
-	vmulpd	%xmm0 , %xmm5 , %xmm5
-	vmulpd	%xmm0 , %xmm6 , %xmm6
-	vmulpd	%xmm0 , %xmm7 , %xmm7
-
-	vmulpd	%xmm0 , %xmm8 , %xmm8
-	vmulpd	%xmm0 , %xmm9 , %xmm9
-	vmulpd	%xmm0 , %xmm10, %xmm10
-	vmulpd	%xmm0 , %xmm11, %xmm11
-
-	vmulpd	%xmm0 , %xmm12, %xmm12
-	vmulpd	%xmm0 , %xmm13, %xmm13
-	vmulpd	%xmm0 , %xmm14, %xmm14
-	vmulpd	%xmm0 , %xmm15, %xmm15
-
-
-        leaq    (CO1, LDC, 2), %rax     
-	
-
-#if !defined(TRMMKERNEL)
-
-	vaddpd 	                (CO1), %xmm4, %xmm4
-	vaddpd 	           (CO1, LDC), %xmm5, %xmm5
-	vaddpd 	               (%rax), %xmm6, %xmm6
-	vaddpd 	          (%rax, LDC), %xmm7, %xmm7
-
-#endif
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm5 ,  	(CO1, LDC)
-	vmovups	%xmm6 ,  	(%rax)
-	vmovups	%xmm7 ,  	(%rax, LDC)
-
-
-	leaq	(%rax, LDC, 2), %rax
-	leaq	(%rax, LDC, 2), %rbp
-
-#if !defined(TRMMKERNEL)
-
-	vaddpd 	                (%rax), %xmm8 , %xmm4
-	vaddpd 	           (%rax, LDC), %xmm9 , %xmm5
-	vaddpd 	                (%rbp), %xmm10, %xmm6
-	vaddpd 	           (%rbp, LDC), %xmm11, %xmm7
-
-#endif
-
-	vmovups	%xmm4 ,  	(%rax)
-	vmovups	%xmm5 ,  	(%rax, LDC)
-	vmovups	%xmm6 ,  	(%rbp)
-	vmovups	%xmm7 ,  	(%rbp, LDC)
-
-
-	leaq	(%rax, LDC, 4), %rax
-	leaq	(%rbp, LDC, 4), %rbp
-
-#if !defined(TRMMKERNEL)
-
-	vaddpd 	                (%rax), %xmm12, %xmm4
-	vaddpd 	           (%rax, LDC), %xmm13, %xmm5
-	vaddpd 	                (%rbp), %xmm14, %xmm6
-	vaddpd 	           (%rbp, LDC), %xmm15, %xmm7
-
-#endif
-
-	vmovups	%xmm4 ,  	(%rax)
-	vmovups	%xmm5 ,  	(%rax, LDC)
-	vmovups	%xmm6 ,  	(%rbp)
-	vmovups	%xmm7 ,  	(%rbp, LDC)
-
-	addq	$ 2*SIZE, CO1
-.endm
-
-
-/******************************************************************************************/
-
-.macro INIT1x12
-
-	vxorpd		%xmm4 , %xmm4 , %xmm4
-	vxorpd		%xmm5 , %xmm5 , %xmm5
-	vxorpd		%xmm6 , %xmm6 , %xmm6
-	vxorpd		%xmm7 , %xmm7 , %xmm7
-	vxorpd		%xmm8 , %xmm8 , %xmm8
-	vxorpd		%xmm9 , %xmm9 , %xmm9
-	vxorpd		%xmm10, %xmm10, %xmm10
-	vxorpd		%xmm11, %xmm11, %xmm11
-	vxorpd		%xmm12, %xmm12, %xmm12
-	vxorpd		%xmm13, %xmm13, %xmm13
-	vxorpd		%xmm14, %xmm14, %xmm14
-	vxorpd		%xmm15, %xmm15, %xmm15
-
-.endm
-
-.macro KERNEL1x12_SUB
-	vmovsd 	-16 * SIZE(AO), %xmm0
-	vmovsd	-12 * SIZE(BO), %xmm1
-	vmovsd	-11 * SIZE(BO), %xmm2
-	vmovsd	-10 * SIZE(BO), %xmm3
-	vfmadd231sd  	%xmm0 ,%xmm1  , %xmm4
-	vmovsd	 -9 * SIZE(BO), %xmm1
-	vfmadd231sd  	%xmm0 ,%xmm2  , %xmm5
-	vmovsd	 -8 * SIZE(BO), %xmm2
-	vfmadd231sd  	%xmm0 ,%xmm3  , %xmm6
-	vmovsd	 -7 * SIZE(BO), %xmm3
-	vfmadd231sd  	%xmm0 ,%xmm1  , %xmm7
-	vmovsd	 -6 * SIZE(BO), %xmm1
-	vfmadd231sd  	%xmm0 ,%xmm2  , %xmm8
-	vmovsd	 -5 * SIZE(BO), %xmm2
-	vfmadd231sd  	%xmm0 ,%xmm3  , %xmm9
-	vmovsd	 -4 * SIZE(BO), %xmm3
-	vfmadd231sd  	%xmm0 ,%xmm1  , %xmm10
-	vmovsd	 -3 * SIZE(BO), %xmm1
-	vfmadd231sd  	%xmm0 ,%xmm2  , %xmm11
-	vmovsd	 -2 * SIZE(BO), %xmm2
-	vfmadd231sd  	%xmm0 ,%xmm3  , %xmm12
-	vmovsd	 -1 * SIZE(BO), %xmm3
-	vfmadd231sd  	%xmm0 ,%xmm1  , %xmm13
-	addq		$ 12*SIZE, BO
-	vfmadd231sd  	%xmm0 ,%xmm2  , %xmm14
-	addq		$ 1*SIZE, AO
-	vfmadd231sd  	%xmm0 ,%xmm3  , %xmm15
-
-.endm
-
-.macro SAVE1x12
-
-	vmovsd	ALPHA, %xmm0
-
-	vmulsd	%xmm0 , %xmm4 , %xmm4
-	vmulsd	%xmm0 , %xmm5 , %xmm5
-	vmulsd	%xmm0 , %xmm6 , %xmm6
-	vmulsd	%xmm0 , %xmm7 , %xmm7
-
-	vmulsd	%xmm0 , %xmm8 , %xmm8
-	vmulsd	%xmm0 , %xmm9 , %xmm9
-	vmulsd	%xmm0 , %xmm10, %xmm10
-	vmulsd	%xmm0 , %xmm11, %xmm11
-
-	vmulsd	%xmm0 , %xmm12, %xmm12
-	vmulsd	%xmm0 , %xmm13, %xmm13
-	vmulsd	%xmm0 , %xmm14, %xmm14
-	vmulsd	%xmm0 , %xmm15, %xmm15
-
-
-        leaq    (CO1, LDC, 2), %rax     
-	
-
-#if !defined(TRMMKERNEL)
-
-	vaddsd 	                (CO1), %xmm4, %xmm4
-	vaddsd 	           (CO1, LDC), %xmm5, %xmm5
-	vaddsd 	               (%rax), %xmm6, %xmm6
-	vaddsd 	          (%rax, LDC), %xmm7, %xmm7
-
-#endif
-
-	vmovsd	%xmm4 ,  	(CO1)
-	vmovsd	%xmm5 ,  	(CO1, LDC)
-	vmovsd	%xmm6 ,  	(%rax)
-	vmovsd	%xmm7 ,  	(%rax, LDC)
-
-
-	leaq	(%rax, LDC, 2), %rax
-	leaq	(%rax, LDC, 2), %rbp
-
-#if !defined(TRMMKERNEL)
-
-	vaddsd 	                (%rax), %xmm8 , %xmm4
-	vaddsd 	           (%rax, LDC), %xmm9 , %xmm5
-	vaddsd 	                (%rbp), %xmm10, %xmm6
-	vaddsd 	           (%rbp, LDC), %xmm11, %xmm7
-
-#endif
-
-	vmovsd	%xmm4 ,  	(%rax)
-	vmovsd	%xmm5 ,  	(%rax, LDC)
-	vmovsd	%xmm6 ,  	(%rbp)
-	vmovsd	%xmm7 ,  	(%rbp, LDC)
-
-
-	leaq	(%rax, LDC, 4), %rax
-	leaq	(%rbp, LDC, 4), %rbp
-
-#if !defined(TRMMKERNEL)
-
-	vaddsd 	                (%rax), %xmm12, %xmm4
-	vaddsd 	           (%rax, LDC), %xmm13, %xmm5
-	vaddsd 	                (%rbp), %xmm14, %xmm6
-	vaddsd 	           (%rbp, LDC), %xmm15, %xmm7
-
-#endif
-
-	vmovsd	%xmm4 ,  	(%rax)
-	vmovsd	%xmm5 ,  	(%rax, LDC)
-	vmovsd	%xmm6 ,  	(%rbp)
-	vmovsd	%xmm7 ,  	(%rbp, LDC)
-
-	addq	$ 1*SIZE, CO1
-.endm
-
-
-
-
-/******************************************************************************************/
-
-
-.macro INIT4x8
-
-	vxorpd		%ymm4 , %ymm4 , %ymm4
-	vxorpd		%ymm5 , %ymm5 , %ymm5
-	vxorpd		%ymm6 , %ymm6 , %ymm6
-	vxorpd		%ymm7 , %ymm7 , %ymm7
-	vxorpd		%ymm8 , %ymm8 , %ymm8
-	vxorpd		%ymm9 , %ymm9 , %ymm9
-	vxorpd		%ymm10, %ymm10, %ymm10
-	vxorpd		%ymm11, %ymm11, %ymm11
-
-.endm
-
-.macro KERNEL4x8_I
-	vmovups		-12 * SIZE(BO), %ymm1
-#if defined BROADCASTKERNEL
-        vbroadcastsd    -16 * SIZE(AO), %ymm0
-#else
-	vmovups 	-16 * SIZE(AO), %ymm0
-#endif
-	vmovups		 -8 * SIZE(BO), %ymm2
-	vmulpd  	%ymm0 ,%ymm1  , %ymm4
-	vmulpd  	%ymm0 ,%ymm2  , %ymm8
-#if defined BROADCASTKERNEL
-        vbroadcastsd    -15 * SIZE(AO), %ymm0
-#else
-	vpermilpd	$ 0x05, %ymm0  , %ymm0
-#endif
-	vmulpd  	%ymm0 ,%ymm1  , %ymm5
-	vmulpd  	%ymm0 ,%ymm2  , %ymm9
-#if defined BROADCASTKERNEL
-        vbroadcastsd    -14 * SIZE(AO), %ymm0
-#else
-	vpermpd		$ 0x1b, %ymm0  , %ymm0
-#endif
-	vmulpd  	%ymm0 ,%ymm1  , %ymm6
-	vmulpd  	%ymm0 ,%ymm2  , %ymm10
-
-	addq		$  8*SIZE, BO
-#if defined BROADCASTKERNEL
-        vbroadcastsd    -13 * SIZE(AO), %ymm0
-#else
-	vpermilpd	$ 0x05, %ymm0  , %ymm0
-#endif
-	vmulpd  	%ymm0 ,%ymm1  , %ymm7
-	vmovups		-12 * SIZE(BO), %ymm1
-	vmulpd  	%ymm0 ,%ymm2  , %ymm11
-	vmovups		 -8 * SIZE(BO), %ymm2
-
-.endm
-
-.macro KERNEL4x8_M1
-	prefetcht0	A_PR1(AO)
-#if defined BROADCASTKERNEL
-        vbroadcastsd    -16 * SIZE(AO), %ymm0
-#else
-	vmovups 	-16 * SIZE(AO), %ymm0
-#endif
-	prefetcht0	B_PR1(BO)
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4
-	prefetcht0	B_PR1+64(BO)
-	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm8
-#if defined BROADCASTKERNEL
-        vbroadcastsd    -15 * SIZE(AO), %ymm0
-#else
-	vpermilpd	$ 0x05, %ymm0  , %ymm0
-#endif
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5
-	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm9
-#if defined BROADCASTKERNEL
-        vbroadcastsd    -14 * SIZE(AO), %ymm0
-#else
-	vpermpd		$ 0x1b, %ymm0  , %ymm0
-#endif
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6
-	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm10
-#if defined BROADCASTKERNEL
-        vbroadcastsd    -13 * SIZE(AO), %ymm0
-#else
-	vpermilpd	$ 0x05, %ymm0  , %ymm0
-#endif
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7
-	vmovups		-12 * SIZE(BO), %ymm1
-	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm11
-	vmovups		 -8 * SIZE(BO), %ymm2
-
-.endm
-
-.macro KERNEL4x8_M2
-#if defined BROADCASTKERNEL
-        vbroadcastsd    -12 * SIZE(AO), %ymm0
-#else
-	vmovups 	-12 * SIZE(AO), %ymm0
-#endif
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4
-	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm8
-#if defined BROADCASTKERNEL
-        vbroadcastsd    -11 * SIZE(AO), %ymm0
-#else
-	vpermilpd	$ 0x05, %ymm0  , %ymm0
-#endif
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5
-	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm9
-#if defined BROADCASTKERNEL
-        vbroadcastsd    -10 * SIZE(AO), %ymm0
-#else
-	vpermpd		$ 0x1b, %ymm0  , %ymm0
-#endif
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6
-	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm10
-
-	addq		$ 8*SIZE, AO
-#if defined BROADCASTKERNEL
-        vbroadcastsd    -17 * SIZE(AO), %ymm0
-#else
-	vpermilpd	$ 0x05, %ymm0  , %ymm0
-#endif
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7
-	vmovups		 -4 * SIZE(BO), %ymm1
-	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm11
-	vmovups		  0 * SIZE(BO), %ymm2
-	addq		$ 16*SIZE, BO
-.endm
-
-
-.macro KERNEL4x8_E
-#if defined BROADCASTKERNEL
-        vbroadcastsd    -12 * SIZE(AO), %ymm0
-#else
-	vmovups 	-12 * SIZE(AO), %ymm0
-#endif
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4
-	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm8
-#if defined BROADCASTKERNEL
-        vbroadcastsd    -11 * SIZE(AO), %ymm0
-#else
-	vpermilpd	$ 0x05, %ymm0  , %ymm0
-#endif
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5
-	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm9
-#if defined BROADCASTKERNEL
-        vbroadcastsd    -10 * SIZE(AO), %ymm0
-#else
-	vpermpd		$ 0x1b, %ymm0  , %ymm0
-#endif
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6
-	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm10
-
-	addq		$ 8*SIZE, AO
-#if defined BROADCASTKERNEL
-        vbroadcastsd    -17 * SIZE(AO), %ymm0
-#else
-	vpermilpd	$ 0x05, %ymm0  , %ymm0
-#endif
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7
-	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm11
-	addq		$  8*SIZE, BO
-.endm
-
-.macro KERNEL4x8_SUB
-	vmovups		-12 * SIZE(BO), %ymm1
-#if defined BROADCASTKERNEL
-        vbroadcastsd    -16 * SIZE(AO), %ymm0
-#else
-	vmovups 	-16 * SIZE(AO), %ymm0
-#endif
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4
-	vmovups		 -8 * SIZE(BO), %ymm2
-	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm8
-#if defined BROADCASTKERNEL
-        vbroadcastsd    -15 * SIZE(AO), %ymm0
-#else
-	vpermilpd	$ 0x05, %ymm0  , %ymm0
-#endif
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5
-	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm9
-	addq		$  8*SIZE, BO
-#if defined BROADCASTKERNEL
-        vbroadcastsd    -14 * SIZE(AO), %ymm0
-#else
-	vpermpd		$ 0x1b, %ymm0  , %ymm0
-#endif
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6
-	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm10
-	addq		$ 4*SIZE, AO
-#if defined BROADCASTKERNEL
-        vbroadcastsd    -17 * SIZE(AO), %ymm0
-#else
-	vpermilpd	$ 0x05, %ymm0  , %ymm0
-#endif
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7
-	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm11
-
-.endm
-
-
-.macro SAVE4x8
-
-	vbroadcastsd	ALPHA, %ymm0
-
-	vmulpd	%ymm0 , %ymm4 , %ymm4
-	vmulpd	%ymm0 , %ymm5 , %ymm5
-	vmulpd	%ymm0 , %ymm6 , %ymm6
-	vmulpd	%ymm0 , %ymm7 , %ymm7
-
-	vmulpd	%ymm0 , %ymm8 , %ymm8
-	vmulpd	%ymm0 , %ymm9 , %ymm9
-	vmulpd	%ymm0 , %ymm10, %ymm10
-	vmulpd	%ymm0 , %ymm11, %ymm11
-
-#if defined BROADCASTKERNEL
-        vperm2f128 $ 0x20 , %ymm6, %ymm4 , %ymm0
-        vperm2f128 $ 0x20 , %ymm7, %ymm5 , %ymm1
-        vperm2f128 $ 0x31 , %ymm6, %ymm4 , %ymm2
-        vperm2f128 $ 0x31 , %ymm7, %ymm5 , %ymm3
-        vunpcklpd %ymm1, %ymm0, %ymm4
-        vunpckhpd %ymm1, %ymm0, %ymm5
-        vunpcklpd %ymm3, %ymm2, %ymm6
-        vunpckhpd %ymm3, %ymm2, %ymm7
-#else
-	vpermilpd $ 0x05 , %ymm5, %ymm5
-	vpermilpd $ 0x05 , %ymm7, %ymm7
-
-	vblendpd $ 0x0a, %ymm5, %ymm4, %ymm0
-	vblendpd $ 0x05, %ymm5, %ymm4, %ymm1
-	vblendpd $ 0x0a, %ymm7, %ymm6, %ymm2
-	vblendpd $ 0x05, %ymm7, %ymm6, %ymm3
-
-	vperm2f128 $ 0x01 , %ymm2, %ymm2 , %ymm2
-	vperm2f128 $ 0x01 , %ymm3, %ymm3 , %ymm3
-
-	vblendpd $ 0x03, %ymm0, %ymm2 , %ymm4
-	vblendpd $ 0x03, %ymm1, %ymm3 , %ymm5
-	vblendpd $ 0x03, %ymm2, %ymm0 , %ymm6
-	vblendpd $ 0x03, %ymm3, %ymm1 , %ymm7
-#endif
-
-        leaq    (CO1, LDC, 2), %rax     
-	
-
-#if !defined(TRMMKERNEL)
-
-	vaddpd 	                (CO1), %ymm4, %ymm4
-	vaddpd 	           (CO1, LDC), %ymm5, %ymm5
-	vaddpd 	               (%rax), %ymm6, %ymm6
-	vaddpd 	          (%rax, LDC), %ymm7, %ymm7
-
-#endif
-
-	vmovups	%ymm4 ,  	(CO1)
-	vmovups	%ymm5 ,  	(CO1, LDC)
-	vmovups	%ymm6 ,  	(%rax)
-	vmovups	%ymm7 ,  	(%rax, LDC)
-
-	prefetcht0	56(CO1)
-	prefetcht0	56(CO1,LDC)
-	prefetcht0	56(%rax)
-	prefetcht0	56(%rax,LDC)
-
-#if defined BROADCASTKERNEL
-        vperm2f128 $ 0x20 , %ymm10, %ymm8 , %ymm0
-        vperm2f128 $ 0x20 , %ymm11, %ymm9 , %ymm1
-        vperm2f128 $ 0x31 , %ymm10, %ymm8 , %ymm2
-        vperm2f128 $ 0x31 , %ymm11, %ymm9 , %ymm3
-        vunpcklpd %ymm1, %ymm0, %ymm4
-        vunpckhpd %ymm1, %ymm0, %ymm5
-        vunpcklpd %ymm3, %ymm2, %ymm6
-        vunpckhpd %ymm3, %ymm2, %ymm7
-#else
-	vpermilpd $ 0x05 , %ymm9 , %ymm9
-	vpermilpd $ 0x05 , %ymm11, %ymm11
-
-	vblendpd $ 0x0a, %ymm9 , %ymm8 , %ymm0
-	vblendpd $ 0x05, %ymm9 , %ymm8 , %ymm1
-	vblendpd $ 0x0a, %ymm11, %ymm10, %ymm2
-	vblendpd $ 0x05, %ymm11, %ymm10, %ymm3
-
-	vperm2f128 $ 0x01 , %ymm2, %ymm2 , %ymm2
-	vperm2f128 $ 0x01 , %ymm3, %ymm3 , %ymm3
-
-	vblendpd $ 0x03, %ymm0, %ymm2 , %ymm4
-	vblendpd $ 0x03, %ymm1, %ymm3 , %ymm5
-	vblendpd $ 0x03, %ymm2, %ymm0 , %ymm6
-	vblendpd $ 0x03, %ymm3, %ymm1 , %ymm7
-#endif
-
-	leaq	(%rax, LDC, 2), %rax
-	leaq	(%rax, LDC, 2), %rbp
-
-#if !defined(TRMMKERNEL)
-
-	vaddpd 	                (%rax), %ymm4, %ymm4
-	vaddpd 	           (%rax, LDC), %ymm5, %ymm5
-	vaddpd 	                (%rbp), %ymm6, %ymm6
-	vaddpd 	           (%rbp, LDC), %ymm7, %ymm7
-
-#endif
-
-	vmovups	%ymm4 ,  	(%rax)
-	vmovups	%ymm5 ,  	(%rax, LDC)
-	vmovups	%ymm6 ,  	(%rbp)
-	vmovups	%ymm7 ,  	(%rbp, LDC)
-
-	prefetcht0	56(%rax)
-	prefetcht0	56(%rax,LDC)
-	prefetcht0	56(%rbp)
-	prefetcht0	56(%rbp,LDC)
-
-	addq	$ 4*SIZE, CO1
-.endm
-
-/******************************************************************************************/
-
-.macro INIT2x8
-
-	vxorpd		%xmm4 , %xmm4 , %xmm4
-	vxorpd		%xmm5 , %xmm5 , %xmm5
-	vxorpd		%xmm6 , %xmm6 , %xmm6
-	vxorpd		%xmm7 , %xmm7 , %xmm7
-	vxorpd		%xmm8 , %xmm8 , %xmm8
-	vxorpd		%xmm9 , %xmm9 , %xmm9
-	vxorpd		%xmm10, %xmm10, %xmm10
-	vxorpd		%xmm11, %xmm11, %xmm11
-
-.endm
-
-.macro KERNEL2x8_SUB
-	vmovups 	-16 * SIZE(AO), %xmm0
-	vmovddup	-12 * SIZE(BO), %xmm1
-	vmovddup	-11 * SIZE(BO), %xmm2
-	vmovddup	-10 * SIZE(BO), %xmm3
-	vfmadd231pd  	%xmm0 ,%xmm1  , %xmm4
-	vmovddup	 -9 * SIZE(BO), %xmm1
-	vfmadd231pd  	%xmm0 ,%xmm2  , %xmm5
-	vmovddup	 -8 * SIZE(BO), %xmm2
-	vfmadd231pd  	%xmm0 ,%xmm3  , %xmm6
-	vmovddup	 -7 * SIZE(BO), %xmm3
-	vfmadd231pd  	%xmm0 ,%xmm1  , %xmm7
-	vmovddup	 -6 * SIZE(BO), %xmm1
-	vfmadd231pd  	%xmm0 ,%xmm2  , %xmm8
-	vmovddup	 -5 * SIZE(BO), %xmm2
-	vfmadd231pd  	%xmm0 ,%xmm3  , %xmm9
-	vfmadd231pd  	%xmm0 ,%xmm1  , %xmm10
-	vfmadd231pd  	%xmm0 ,%xmm2  , %xmm11
-	addq		$  8*SIZE, BO
-	addq		$ 2*SIZE, AO
-
-.endm
-
-.macro SAVE2x8
-
-	vmovddup	ALPHA, %xmm0
-
-	vmulpd	%xmm0 , %xmm4 , %xmm4
-	vmulpd	%xmm0 , %xmm5 , %xmm5
-	vmulpd	%xmm0 , %xmm6 , %xmm6
-	vmulpd	%xmm0 , %xmm7 , %xmm7
-
-	vmulpd	%xmm0 , %xmm8 , %xmm8
-	vmulpd	%xmm0 , %xmm9 , %xmm9
-	vmulpd	%xmm0 , %xmm10, %xmm10
-	vmulpd	%xmm0 , %xmm11, %xmm11
-
-        leaq    (CO1, LDC, 2), %rax     
-	
-
-#if !defined(TRMMKERNEL)
-
-	vaddpd 	                (CO1), %xmm4, %xmm4
-	vaddpd 	           (CO1, LDC), %xmm5, %xmm5
-	vaddpd 	               (%rax), %xmm6, %xmm6
-	vaddpd 	          (%rax, LDC), %xmm7, %xmm7
-
-#endif
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm5 ,  	(CO1, LDC)
-	vmovups	%xmm6 ,  	(%rax)
-	vmovups	%xmm7 ,  	(%rax, LDC)
-
-
-	leaq	(%rax, LDC, 2), %rax
-	leaq	(%rax, LDC, 2), %rbp
-
-#if !defined(TRMMKERNEL)
-
-	vaddpd 	                (%rax), %xmm8 , %xmm4
-	vaddpd 	           (%rax, LDC), %xmm9 , %xmm5
-	vaddpd 	                (%rbp), %xmm10, %xmm6
-	vaddpd 	           (%rbp, LDC), %xmm11, %xmm7
-
-#endif
-
-	vmovups	%xmm4 ,  	(%rax)
-	vmovups	%xmm5 ,  	(%rax, LDC)
-	vmovups	%xmm6 ,  	(%rbp)
-	vmovups	%xmm7 ,  	(%rbp, LDC)
-
-	addq	$ 2*SIZE, CO1
-.endm
-
-
-/******************************************************************************************/
-
-.macro INIT1x8
-
-	vxorpd		%xmm4 , %xmm4 , %xmm4
-	vxorpd		%xmm5 , %xmm5 , %xmm5
-	vxorpd		%xmm6 , %xmm6 , %xmm6
-	vxorpd		%xmm7 , %xmm7 , %xmm7
-	vxorpd		%xmm8 , %xmm8 , %xmm8
-	vxorpd		%xmm9 , %xmm9 , %xmm9
-	vxorpd		%xmm10, %xmm10, %xmm10
-	vxorpd		%xmm11, %xmm11, %xmm11
-
-.endm
-
-.macro KERNEL1x8_SUB
-	vmovsd 	-16 * SIZE(AO), %xmm0
-	vmovsd	-12 * SIZE(BO), %xmm1
-	vmovsd	-11 * SIZE(BO), %xmm2
-	vmovsd	-10 * SIZE(BO), %xmm3
-	vfmadd231sd  	%xmm0 ,%xmm1  , %xmm4
-	vmovsd	 -9 * SIZE(BO), %xmm1
-	vfmadd231sd  	%xmm0 ,%xmm2  , %xmm5
-	vmovsd	 -8 * SIZE(BO), %xmm2
-	vfmadd231sd  	%xmm0 ,%xmm3  , %xmm6
-	vmovsd	 -7 * SIZE(BO), %xmm3
-	vfmadd231sd  	%xmm0 ,%xmm1  , %xmm7
-	vmovsd	 -6 * SIZE(BO), %xmm1
-	vfmadd231sd  	%xmm0 ,%xmm2  , %xmm8
-	vmovsd	 -5 * SIZE(BO), %xmm2
-	vfmadd231sd  	%xmm0 ,%xmm3  , %xmm9
-	vfmadd231sd  	%xmm0 ,%xmm1  , %xmm10
-	vfmadd231sd  	%xmm0 ,%xmm2  , %xmm11
-	addq		$  8*SIZE, BO
-	addq		$ 1*SIZE, AO
-
-.endm
-
-.macro SAVE1x8
-
-	vmovsd	ALPHA, %xmm0
-
-	vmulsd	%xmm0 , %xmm4 , %xmm4
-	vmulsd	%xmm0 , %xmm5 , %xmm5
-	vmulsd	%xmm0 , %xmm6 , %xmm6
-	vmulsd	%xmm0 , %xmm7 , %xmm7
-
-	vmulsd	%xmm0 , %xmm8 , %xmm8
-	vmulsd	%xmm0 , %xmm9 , %xmm9
-	vmulsd	%xmm0 , %xmm10, %xmm10
-	vmulsd	%xmm0 , %xmm11, %xmm11
-
-        leaq    (CO1, LDC, 2), %rax     
-	
-
-#if !defined(TRMMKERNEL)
-
-	vaddsd 	                (CO1), %xmm4, %xmm4
-	vaddsd 	           (CO1, LDC), %xmm5, %xmm5
-	vaddsd 	               (%rax), %xmm6, %xmm6
-	vaddsd 	          (%rax, LDC), %xmm7, %xmm7
-
-#endif
-
-	vmovsd	%xmm4 ,  	(CO1)
-	vmovsd	%xmm5 ,  	(CO1, LDC)
-	vmovsd	%xmm6 ,  	(%rax)
-	vmovsd	%xmm7 ,  	(%rax, LDC)
-
-
-	leaq	(%rax, LDC, 2), %rax
-	leaq	(%rax, LDC, 2), %rbp
-
-#if !defined(TRMMKERNEL)
-
-	vaddsd 	                (%rax), %xmm8 , %xmm4
-	vaddsd 	           (%rax, LDC), %xmm9 , %xmm5
-	vaddsd 	                (%rbp), %xmm10, %xmm6
-	vaddsd 	           (%rbp, LDC), %xmm11, %xmm7
-
-#endif
-
-	vmovsd	%xmm4 ,  	(%rax)
-	vmovsd	%xmm5 ,  	(%rax, LDC)
-	vmovsd	%xmm6 ,  	(%rbp)
-	vmovsd	%xmm7 ,  	(%rbp, LDC)
-
-	addq	$ 1*SIZE, CO1
-.endm
-
-
-
-
-
-/******************************************************************************************/
-
-.macro INIT4x4
-
-	vxorpd		%ymm4 , %ymm4 , %ymm4
-	vxorpd		%ymm5 , %ymm5 , %ymm5
-	vxorpd		%ymm6 , %ymm6 , %ymm6
-	vxorpd		%ymm7 , %ymm7 , %ymm7
-
-.endm
-
-.macro KERNEL4x4_I
-	prefetcht0	A_PR1(AO)
-	vmovups		-12 * SIZE(BO), %ymm1
-#if defined BROADCASTKERNEL
-        vbroadcastsd    -16 * SIZE(AO), %ymm0
-#else
-	vmovups 	-16 * SIZE(AO), %ymm0
-#endif
-	vmulpd  	%ymm0 ,%ymm1  , %ymm4
-#if defined BROADCASTKERNEL
-        vbroadcastsd    -15 * SIZE(AO), %ymm0
-#else
-	vpermilpd	$ 0x05, %ymm0  , %ymm0
-#endif
-	vmulpd  	%ymm0 ,%ymm1  , %ymm5
-#if defined BROADCASTKERNEL
-        vbroadcastsd    -14 * SIZE(AO), %ymm0
-#else
-	vpermpd		$ 0x1b, %ymm0  , %ymm0
-#endif
-	vmulpd  	%ymm0 ,%ymm1  , %ymm6
-
-	addq		$ 4*SIZE, BO
-#if defined BROADCASTKERNEL
-        vbroadcastsd    -13 * SIZE(AO), %ymm0
-#else
-	vpermilpd	$ 0x05, %ymm0  , %ymm0
-#endif
-	vmulpd  	%ymm0 ,%ymm1  , %ymm7
-	vmovups		-12 * SIZE(BO), %ymm1
-
-.endm
-
-.macro KERNEL4x4_M1
-	prefetcht0	A_PR1(AO)
-#if defined BROADCASTKERNEL
-        vbroadcastsd    -16 * SIZE(AO), %ymm0
-#else
-	vmovups 	-16 * SIZE(AO), %ymm0
-#endif
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4
-#if defined BROADCASTKERNEL
-        vbroadcastsd    -15 * SIZE(AO), %ymm0
-#else
-	vpermilpd	$ 0x05, %ymm0  , %ymm0
-#endif
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5
-#if defined BROADCASTKERNEL
-        vbroadcastsd    -14 * SIZE(AO), %ymm0
-#else
-	vpermpd		$ 0x1b, %ymm0  , %ymm0
-#endif
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6
-#if defined BROADCASTKERNEL
-        vbroadcastsd    -13 * SIZE(AO), %ymm0
-#else
-	vpermilpd	$ 0x05, %ymm0  , %ymm0
-#endif
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7
-	vmovups		-12 * SIZE(BO), %ymm1
-
-.endm
-
-.macro KERNEL4x4_M2
-#if defined BROADCASTKERNEL
-        vbroadcastsd    -12 * SIZE(AO), %ymm0
-#else
-	vmovups 	-12 * SIZE(AO), %ymm0
-#endif
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4
-#if defined BROADCASTKERNEL
-        vbroadcastsd    -11 * SIZE(AO), %ymm0
-#else
-	vpermilpd	$ 0x05, %ymm0  , %ymm0
-#endif
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5
-#if defined BROADCASTKERNEL
-        vbroadcastsd    -10 * SIZE(AO), %ymm0
-#else
-	vpermpd		$ 0x1b, %ymm0  , %ymm0
-#endif
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6
-
-	addq		$ 8*SIZE, AO
-#if defined BROADCASTKERNEL
-        vbroadcastsd    -17 * SIZE(AO), %ymm0
-#else
-	vpermilpd	$ 0x05, %ymm0  , %ymm0
-#endif
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7
-	vmovups		 -8 * SIZE(BO), %ymm1
-	addq		$ 8*SIZE, BO
-.endm
-
-
-.macro KERNEL4x4_E
-#if defined BROADCASTKERNEL
-        vbroadcastsd    -12 * SIZE(AO), %ymm0
-#else
-	vmovups 	-12 * SIZE(AO), %ymm0
-#endif
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4
-#if defined BROADCASTKERNEL
-        vbroadcastsd    -11 * SIZE(AO), %ymm0
-#else
-	vpermilpd	$ 0x05, %ymm0  , %ymm0
-#endif
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5
-#if defined BROADCASTKERNEL
-        vbroadcastsd    -10 * SIZE(AO), %ymm0
-#else
-	vpermpd		$ 0x1b, %ymm0  , %ymm0
-#endif
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6
-
-	addq		$ 8*SIZE, AO
-#if defined BROADCASTKERNEL
-        vbroadcastsd    -17 * SIZE(AO), %ymm0
-#else
-	vpermilpd	$ 0x05, %ymm0  , %ymm0
-#endif
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7
-	addq		$ 4*SIZE, BO
-.endm
-
-.macro KERNEL4x4_SUB
-	vmovups		-12 * SIZE(BO), %ymm1
-#if defined BROADCASTKERNEL
-        vbroadcastsd    -16 * SIZE(AO), %ymm0
-#else
-	vmovups 	-16 * SIZE(AO), %ymm0
-#endif
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4
-#if defined BROADCASTKERNEL
-        vbroadcastsd    -15 * SIZE(AO), %ymm0
-#else
-	vpermilpd	$ 0x05, %ymm0  , %ymm0
-#endif
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5
-	addq		$ 4*SIZE, BO
-#if defined BROADCASTKERNEL
-        vbroadcastsd    -14 * SIZE(AO), %ymm0
-#else
-	vpermpd		$ 0x1b, %ymm0  , %ymm0
-#endif
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6
-	addq		$ 4*SIZE, AO
-#if defined BROADCASTKERNEL
-        vbroadcastsd    -17 * SIZE(AO), %ymm0
-#else
-	vpermilpd	$ 0x05, %ymm0  , %ymm0
-#endif
-	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7
-
-.endm
-
-.macro SAVE4x4
-
-	vbroadcastsd	ALPHA, %ymm0
-
-	vmulpd	%ymm0 , %ymm4 , %ymm4
-	vmulpd	%ymm0 , %ymm7 , %ymm7
-	vmulpd	%ymm0 , %ymm5 , %ymm5
-	vmulpd	%ymm0 , %ymm6 , %ymm6
-
-#if defined BROADCASTKERNEL
-        vperm2f128 $ 0x20 , %ymm6, %ymm4 , %ymm0
-        vperm2f128 $ 0x20 , %ymm7, %ymm5 , %ymm1
-        vperm2f128 $ 0x31 , %ymm6, %ymm4 , %ymm2
-        vperm2f128 $ 0x31 , %ymm7, %ymm5 , %ymm3
-        vunpcklpd %ymm1, %ymm0, %ymm4
-        vunpckhpd %ymm1, %ymm0, %ymm5
-        vunpcklpd %ymm3, %ymm2, %ymm6
-        vunpckhpd %ymm3, %ymm2, %ymm7
-#else
-	vpermilpd $ 0x05 , %ymm5, %ymm5
-	vpermilpd $ 0x05 , %ymm7, %ymm7
-
-	vblendpd $ 0x0a, %ymm5, %ymm4, %ymm0
-	vblendpd $ 0x05, %ymm5, %ymm4, %ymm1
-	vblendpd $ 0x0a, %ymm7, %ymm6, %ymm2
-	vblendpd $ 0x05, %ymm7, %ymm6, %ymm3
-
-	vperm2f128 $ 0x01 , %ymm2, %ymm2 , %ymm2
-	vperm2f128 $ 0x01 , %ymm3, %ymm3 , %ymm3
-
-	vblendpd $ 0x03, %ymm0, %ymm2 , %ymm4
-	vblendpd $ 0x03, %ymm1, %ymm3 , %ymm5
-	vblendpd $ 0x03, %ymm2, %ymm0 , %ymm6
-	vblendpd $ 0x03, %ymm3, %ymm1 , %ymm7
-#endif
-
-        leaq    (CO1, LDC, 2), %rax     
-	
-
-#if !defined(TRMMKERNEL)
-
-	vaddpd 	                (CO1), %ymm4, %ymm4
-	vaddpd 	           (CO1, LDC), %ymm5, %ymm5
-	vaddpd 	               (%rax), %ymm6, %ymm6
-	vaddpd 	          (%rax, LDC), %ymm7, %ymm7
-
-#endif
-
-	vmovups	%ymm4 ,  	(CO1)
-	vmovups	%ymm5 ,  	(CO1, LDC)
-	vmovups	%ymm6 ,  	(%rax)
-	vmovups	%ymm7 ,  	(%rax, LDC)
-
-	addq	$ 4*SIZE, CO1
-.endm
-
-/******************************************************************************************/
-/******************************************************************************************/
-
-.macro INIT2x4
-
-	vxorpd		%xmm4 , %xmm4 , %xmm4
-	vxorpd		%xmm5 , %xmm5 , %xmm5
-	vxorpd		%xmm6 , %xmm6 , %xmm6
-	vxorpd		%xmm7 , %xmm7 , %xmm7
-
-.endm
-
-
-.macro KERNEL2x4_SUB
-	vmovddup	-12 * SIZE(BO), %xmm1
-	vmovups 	-16 * SIZE(AO), %xmm0
-	vmovddup	-11 * SIZE(BO), %xmm2
-	vfmadd231pd  	%xmm0 ,%xmm1  , %xmm4
-	vmovddup	-10 * SIZE(BO), %xmm3
-	vfmadd231pd  	%xmm0 ,%xmm2  , %xmm5
-	vmovddup	 -9 * SIZE(BO), %xmm8
-	vfmadd231pd  	%xmm0 ,%xmm3  , %xmm6
-	addq		$ 4*SIZE, BO
-	vfmadd231pd  	%xmm0 ,%xmm8  , %xmm7
-	addq		$ 2*SIZE, AO
-
-.endm
-
-
-.macro SAVE2x4
-
-	vmovddup	ALPHA, %xmm0
-
-	vmulpd	%xmm0 , %xmm4 , %xmm4
-	vmulpd	%xmm0 , %xmm5 , %xmm5
-	vmulpd	%xmm0 , %xmm6 , %xmm6
-	vmulpd	%xmm0 , %xmm7 , %xmm7
-
-        leaq    (CO1, LDC, 2), %rax     
-
-#if !defined(TRMMKERNEL)
-
-	vaddpd 	                (CO1), %xmm4, %xmm4
-	vaddpd 	           (CO1, LDC), %xmm5, %xmm5
-	vaddpd 	               (%rax), %xmm6, %xmm6
-	vaddpd 	          (%rax, LDC), %xmm7, %xmm7
-
-#endif
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm5 ,  	(CO1, LDC)
-	vmovups	%xmm6 ,  	(%rax)
-	vmovups	%xmm7 ,  	(%rax, LDC)
-
-	addq	$ 2*SIZE, CO1
-.endm
-
-/******************************************************************************************/
-/******************************************************************************************/
-
-.macro INIT1x4
-
-	vxorpd		%xmm4 , %xmm4 , %xmm4
-	vxorpd		%xmm5 , %xmm5 , %xmm5
-	vxorpd		%xmm6 , %xmm6 , %xmm6
-	vxorpd		%xmm7 , %xmm7 , %xmm7
-
-.endm
-
-
-.macro KERNEL1x4_SUB
-	vmovsd	-12 * SIZE(BO), %xmm1
-	vmovsd 	-16 * SIZE(AO), %xmm0
-	vmovsd	-11 * SIZE(BO), %xmm2
-	vfmadd231sd  	%xmm0 ,%xmm1  , %xmm4
-	vmovsd	-10 * SIZE(BO), %xmm3
-	vfmadd231sd  	%xmm0 ,%xmm2  , %xmm5
-	vmovsd	 -9 * SIZE(BO), %xmm8
-	vfmadd231sd  	%xmm0 ,%xmm3  , %xmm6
-	addq		$ 4*SIZE, BO
-	vfmadd231sd  	%xmm0 ,%xmm8  , %xmm7
-	addq		$ 1*SIZE, AO
-
-.endm
-
-
-.macro SAVE1x4
-
-	vmovsd	ALPHA, %xmm0
-
-	vmulsd	%xmm0 , %xmm4 , %xmm4
-	vmulsd	%xmm0 , %xmm5 , %xmm5
-	vmulsd	%xmm0 , %xmm6 , %xmm6
-	vmulsd	%xmm0 , %xmm7 , %xmm7
-
-        leaq    (CO1, LDC, 2), %rax     
-
-#if !defined(TRMMKERNEL)
-
-	vaddsd 	                (CO1), %xmm4, %xmm4
-	vaddsd 	           (CO1, LDC), %xmm5, %xmm5
-	vaddsd 	               (%rax), %xmm6, %xmm6
-	vaddsd 	          (%rax, LDC), %xmm7, %xmm7
-
-#endif
-
-	vmovsd	%xmm4 ,  	(CO1)
-	vmovsd	%xmm5 ,  	(CO1, LDC)
-	vmovsd	%xmm6 ,  	(%rax)
-	vmovsd	%xmm7 ,  	(%rax, LDC)
-
-	addq	$ 1*SIZE, CO1
-.endm
-
-
-/******************************************************************************************/
-/******************************************************************************************/
-
-.macro INIT4x2
-
-	vxorpd		%xmm4 , %xmm4 , %xmm4
-	vxorpd		%xmm5 , %xmm5 , %xmm5
-	vxorpd		%xmm6 , %xmm6 , %xmm6
-	vxorpd		%xmm7 , %xmm7 , %xmm7
-
-.endm
-
-
-.macro KERNEL4x2_SUB
-	vmovddup	-12 * SIZE(BO), %xmm2
-	vmovups 	-16 * SIZE(AO), %xmm0
-	vmovups 	-14 * SIZE(AO), %xmm1
-	vmovddup	-11 * SIZE(BO), %xmm3
-	vfmadd231pd  	%xmm0 ,%xmm2  , %xmm4
-	vfmadd231pd  	%xmm1 ,%xmm2  , %xmm5
-	vfmadd231pd  	%xmm0 ,%xmm3  , %xmm6
-	vfmadd231pd  	%xmm1 ,%xmm3  , %xmm7
-	addq		$ 2*SIZE, BO
-	addq		$ 4*SIZE, AO
-
-.endm
-
-
-.macro SAVE4x2
-
-	vmovddup	ALPHA, %xmm0
-
-	vmulpd	%xmm0 , %xmm4 , %xmm4
-	vmulpd	%xmm0 , %xmm5 , %xmm5
-	vmulpd	%xmm0 , %xmm6 , %xmm6
-	vmulpd	%xmm0 , %xmm7 , %xmm7
-
-
-#if !defined(TRMMKERNEL)
-
-	vaddpd 	                (CO1)     , %xmm4, %xmm4
-	vaddpd 	        2 * SIZE(CO1)     , %xmm5, %xmm5
-	vaddpd 	                (CO1, LDC), %xmm6, %xmm6
-	vaddpd 	        2 * SIZE(CO1, LDC), %xmm7, %xmm7
-
-#endif
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm5 , 2 * SIZE(CO1)
-	vmovups	%xmm6 ,  	(CO1, LDC)
-	vmovups	%xmm7 , 2 * SIZE(CO1, LDC)
-
-	addq	$ 4*SIZE, CO1
-.endm
-
-
-/******************************************************************************************/
-/******************************************************************************************/
-
-.macro INIT2x2
-
-	vxorpd		%xmm4 , %xmm4 , %xmm4
-	vxorpd		%xmm6 , %xmm6 , %xmm6
-
-.endm
-
-
-.macro KERNEL2x2_SUB
-	vmovddup	-12 * SIZE(BO), %xmm2
-	vmovups 	-16 * SIZE(AO), %xmm0
-	vmovddup	-11 * SIZE(BO), %xmm3
-	vfmadd231pd  	%xmm0 ,%xmm2  , %xmm4
-	vfmadd231pd  	%xmm0 ,%xmm3  , %xmm6
-	addq		$ 2*SIZE, BO
-	addq		$ 2*SIZE, AO
-
-.endm
-
-
-.macro SAVE2x2
-
-	vmovddup	ALPHA, %xmm0
-
-	vmulpd	%xmm0 , %xmm4 , %xmm4
-	vmulpd	%xmm0 , %xmm6 , %xmm6
-
-
-#if !defined(TRMMKERNEL)
-
-	vaddpd 	                (CO1)     , %xmm4, %xmm4
-	vaddpd 	                (CO1, LDC), %xmm6, %xmm6
-
-#endif
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm6 ,  	(CO1, LDC)
-
-	addq	$ 2*SIZE, CO1
-.endm
-
-/******************************************************************************************/
-/******************************************************************************************/
-
-.macro INIT1x2
-
-	vxorpd		%xmm4 , %xmm4 , %xmm4
-	vxorpd		%xmm5 , %xmm5 , %xmm5
-
-.endm
-
-
-.macro KERNEL1x2_SUB
-	vmovsd	-12 * SIZE(BO), %xmm1
-	vmovsd 	-16 * SIZE(AO), %xmm0
-	vmovsd	-11 * SIZE(BO), %xmm2
-	vfmadd231sd  	%xmm0 ,%xmm1  , %xmm4
-	vfmadd231sd  	%xmm0 ,%xmm2  , %xmm5
-	addq		$ 2*SIZE, BO
-	addq		$ 1*SIZE, AO
-
-.endm
-
-
-.macro SAVE1x2
-
-	vmovsd	ALPHA, %xmm0
-
-	vmulsd	%xmm0 , %xmm4 , %xmm4
-	vmulsd	%xmm0 , %xmm5 , %xmm5
-
-
-#if !defined(TRMMKERNEL)
-
-	vaddsd 	                (CO1), %xmm4, %xmm4
-	vaddsd 	           (CO1, LDC), %xmm5, %xmm5
-
-#endif
-
-	vmovsd	%xmm4 ,  	(CO1)
-	vmovsd	%xmm5 ,  	(CO1, LDC)
-
-	addq	$ 1*SIZE, CO1
-.endm
-
-
-/******************************************************************************************/
-/******************************************************************************************/
-
-.macro INIT4x1
-
-	vxorpd		%ymm4 , %ymm4 , %ymm4
-	vxorpd		%ymm5 , %ymm5 , %ymm5
-	vxorpd		%ymm6 , %ymm6 , %ymm6
-	vxorpd		%ymm7 , %ymm7 , %ymm7
-
-.endm
-
-
-.macro KERNEL4x1
-
-	vbroadcastsd	-12 * SIZE(BO), %ymm0
-	vbroadcastsd	-11 * SIZE(BO), %ymm1
-	vbroadcastsd	-10 * SIZE(BO), %ymm2
-	vbroadcastsd	-9  * SIZE(BO), %ymm3
-
-	vfmadd231pd  	-16 * SIZE(AO) ,%ymm0  , %ymm4
-	vfmadd231pd  	-12 * SIZE(AO) ,%ymm1  , %ymm5
-
-	vbroadcastsd	-8  * SIZE(BO), %ymm0
-	vbroadcastsd	-7  * SIZE(BO), %ymm1
-
-	vfmadd231pd  	-8  * SIZE(AO) ,%ymm2  , %ymm6
-	vfmadd231pd  	-4  * SIZE(AO) ,%ymm3  , %ymm7
-
-	vbroadcastsd	-6  * SIZE(BO), %ymm2
-	vbroadcastsd	-5  * SIZE(BO), %ymm3
-
-	vfmadd231pd  	 0  * SIZE(AO) ,%ymm0  , %ymm4
-	vfmadd231pd  	 4  * SIZE(AO) ,%ymm1  , %ymm5
-	vfmadd231pd  	 8  * SIZE(AO) ,%ymm2  , %ymm6
-	vfmadd231pd  	 12 * SIZE(AO) ,%ymm3  , %ymm7
-
-	addq		$ 8 *SIZE, BO
-	addq		$ 32*SIZE, AO
-
-.endm
-
-
-.macro KERNEL4x1_SUB
-	vbroadcastsd	-12 * SIZE(BO), %ymm2
-	vmovups 	-16 * SIZE(AO), %ymm0
-	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm4
-	addq		$ 1*SIZE, BO
-	addq		$ 4*SIZE, AO
-
-.endm
-
-
-.macro SAVE4x1
-
-	vbroadcastsd	ALPHA, %ymm0
-
-	vaddpd	%ymm4,%ymm5, %ymm4 
-	vaddpd	%ymm6,%ymm7, %ymm6 
-	vaddpd	%ymm4,%ymm6, %ymm4 
-
-	vmulpd	%ymm0 , %ymm4 , %ymm4
-
-
-#if !defined(TRMMKERNEL)
-
-	vaddpd 	                (CO1)     , %ymm4, %ymm4
-
-#endif
-
-	vmovups	%ymm4 ,  	(CO1)
-
-	addq	$ 4*SIZE, CO1
-.endm
-
-
-/******************************************************************************************/
-/******************************************************************************************/
-
-.macro INIT2x1
-
-	vxorpd		%xmm4 , %xmm4 , %xmm4
-
-.endm
-
-
-.macro KERNEL2x1_SUB
-	vmovddup	-12 * SIZE(BO), %xmm2
-	vmovups 	-16 * SIZE(AO), %xmm0
-	vfmadd231pd  	%xmm0 ,%xmm2  , %xmm4
-	addq		$ 1*SIZE, BO
-	addq		$ 2*SIZE, AO
-
-.endm
-
-
-.macro SAVE2x1
-
-	vmovddup	ALPHA, %xmm0
-
-	vmulpd	%xmm0 , %xmm4 , %xmm4
-
-
-#if !defined(TRMMKERNEL)
-
-	vaddpd 	                (CO1)     , %xmm4, %xmm4
-
-#endif
-
-	vmovups	%xmm4 ,  	(CO1)
-
-	addq	$ 2*SIZE, CO1
-.endm
-
-
-/******************************************************************************************/
-/******************************************************************************************/
-
-.macro INIT1x1
-
-	vxorpd		%xmm4 , %xmm4 , %xmm4
-
-.endm
-
-
-.macro KERNEL1x1_SUB
-	vmovsd	-12 * SIZE(BO), %xmm1
-	vmovsd 	-16 * SIZE(AO), %xmm0
-	vfmadd231sd  	%xmm0 ,%xmm1  , %xmm4
-	addq		$ 1*SIZE, BO
-	addq		$ 1*SIZE, AO
-
-.endm
-
-
-.macro SAVE1x1
-
-	vmovsd	ALPHA, %xmm0
-
-	vmulsd	%xmm0 , %xmm4 , %xmm4
-
-
-#if !defined(TRMMKERNEL)
-
-	vaddsd 	                (CO1), %xmm4, %xmm4
-
-#endif
-
-	vmovsd	%xmm4 ,  	(CO1)
-
-	addq	$ 1*SIZE, CO1
-.endm
-
-
-.macro PREFETCHT0_C
-        prefetcht0 (CO1)
-        prefetcht0 24(CO1)
-        prefetcht0 (CO1,LDC,4)
-        prefetcht0 24(CO1,LDC,4)
-        prefetcht0 (CO1,LDC,8)
-        prefetcht0 24(CO1,LDC,8)
-.endm
-/*******************************************************************************************/
-
-#if !defined(TRMMKERNEL)
-
-
-	PROLOGUE
-	PROFCODE
-	
-	subq	$STACKSIZE, %rsp
-	movq	%rbx,   (%rsp)
-	movq	%rbp,  8(%rsp)
-	movq	%r12, 16(%rsp)
-	movq	%r13, 24(%rsp)
-	movq	%r14, 32(%rsp)
-	movq	%r15, 40(%rsp)
-
-	vzeroupper
-
-#ifdef WINDOWS_ABI
-	movq	%rdi,    48(%rsp)
-	movq	%rsi,    56(%rsp)
-	vmovups	%xmm6,   64(%rsp)
-	vmovups	%xmm7,   80(%rsp)
-	vmovups	%xmm8,   96(%rsp)
-	vmovups	%xmm9,  112(%rsp)
-	vmovups	%xmm10, 128(%rsp)
-	vmovups	%xmm11, 144(%rsp)
-	vmovups	%xmm12, 160(%rsp)
-	vmovups	%xmm13, 176(%rsp)
-	vmovups	%xmm14, 192(%rsp)
-	vmovups	%xmm15, 208(%rsp)
-
-	movq	ARG1,      OLD_M
-	movq	ARG2,      OLD_N
-	movq	ARG3,      OLD_K
-	movq	OLD_A,     A
-	movq	OLD_B,     B
-	movq	OLD_C,     C
-	movq	OLD_LDC,   LDC
-
-	vmovups	%xmm3, %xmm0
-
-#else
-	movq	STACKSIZE +  8(%rsp), LDC
-
-#endif
-
-	movq    %rsp, SP      # save old stack
-        subq    $128 + L_BUFFER_SIZE, %rsp
-        andq    $-4096, %rsp    # align stack
-
-        STACK_TOUCH
-
-	cmpq	$ 0, OLD_M
-	je	.L999
-
-	cmpq	$ 0, OLD_N
-	je	.L999
-
-	cmpq	$ 0, OLD_K
-	je	.L999
-
-	movq	OLD_M, M
-	movq	OLD_N, N
-	movq	OLD_K, K
-
-	vmovsd	 %xmm0, ALPHA
-
-	salq	$BASE_SHIFT, LDC
-
-	movq    N, %rax
-        xorq    %rdx, %rdx
-        movq    $24,  %rdi
-        divq    %rdi                     //    N / 24
-        movq    %rax, Ndiv12             //    N / 24
-        movq    %rdx, Nmod12             //    N % 24
-
-
-	movq	Ndiv12,  J
-	cmpq	$ 0, J
-	je	.L8_0
-	ALIGN_4
-
-.L12_01:
-        // copy to sub buffer
-        movq    K, %rax
-        salq    $3,%rax                 // K * 8 ; read 8 values from BO1
-        movq    B, BO1
-        leaq    (B,%rax, SIZE), BO2     // next offset to BO2
-	movq	BO2 , B			
-
-        leaq    BUFFER1, BO             // first buffer to BO
-        movq    K, %rax
-
-        ALIGN_4
-
-.L12_02b:
-
-	vmovups	0 * SIZE(BO1), %ymm1
-	vmovups	4 * SIZE(BO1), %ymm2
-	vmovups	0 * SIZE(BO2), %ymm3
-	vmovups	%ymm1, 0 * SIZE(BO)
-	vmovups	%ymm2, 4 * SIZE(BO)
-	vmovups	%ymm3, 8 * SIZE(BO)
-	addq	$ 8*SIZE,BO1
-	addq	$ 8*SIZE,BO2
-	addq	$ 12*SIZE,BO
-	decq	%rax
-	jnz	.L12_02b
-
-.L12_03c:
-
-
-.L12_10:
-	movq	C, CO1
-	leaq	(C, LDC, 8), C		 
-	leaq	(C, LDC, 4), C		// c += 12 * ldc
-	
-	movq	A, AO		 	// aoffset = a
-	addq	$16 * SIZE, AO
-
-	movq	M,  I
-	sarq	$2, I			// i = m / 4
-	je	.L12_20
-
-	ALIGN_4
-
-.L12_11:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $12 * SIZE, BO
-
-        movq    K, %rax
-
-	sarq $3, %rax			//  K / 8
-	cmpq $2, %rax
-
-	jl	.L12_13
-
-
-	KERNEL4x12_I
-	KERNEL4x12_M2
-	KERNEL4x12_M1
-	KERNEL4x12_M2
-
-	KERNEL4x12_M1
-	KERNEL4x12_M2
-	KERNEL4x12_M1
-	KERNEL4x12_M2
-
-	subq $2, %rax
-	je	.L12_12a
-
-	ALIGN_5
-.L12_12:
-
-	KERNEL4x12_M1
-	KERNEL4x12_M2
-	KERNEL4x12_M1
-	KERNEL4x12_M2
-
-	KERNEL4x12_M1
-	KERNEL4x12_M2
-	KERNEL4x12_M1
-	KERNEL4x12_M2
-
-	dec	%rax
-	jne	.L12_12
-	
-.L12_12a:
-        prefetcht0 ALPHA
-        PREFETCHT0_C
-        addq  LDC,CO1
-	KERNEL4x12_M1
-        PREFETCHT0_C
-        leaq  (CO1,LDC,2),CO1
-	KERNEL4x12_M2
-        PREFETCHT0_C
-        subq  LDC,CO1
-	KERNEL4x12_M1
-        PREFETCHT0_C
-        subq  LDC,CO1
-        subq  LDC,CO1
-	KERNEL4x12_M2
-
-	KERNEL4x12_M1
-	KERNEL4x12_M2
-	KERNEL4x12_M1
-	KERNEL4x12_E
-
-	jmp .L12_16
-
-
-.L12_13:
-
-	test $1, %rax
-	jz .L12_14
-
-	KERNEL4x12_I
-	KERNEL4x12_M2
-	KERNEL4x12_M1
-	KERNEL4x12_M2
-
-	KERNEL4x12_M1
-	KERNEL4x12_M2
-	KERNEL4x12_M1
-	KERNEL4x12_E
-
-	jmp .L12_16
-
-
-.L12_14:
-
-	INIT4x12
-
-
-.L12_16:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L12_19
-
-	ALIGN_4
-
-.L12_17:
-
-	KERNEL4x12_SUB
-
-	dec	%rax
-	jne	.L12_17
-	ALIGN_4
-
-
-.L12_19:
-
-	SAVE4x12
-
-        /* here for the prefetch of next b source block */
-	/* the increment should be proportional to GEMM_Q/GEMM_P */
-
-        salq    $3, K
-#ifdef WINDOWS_ABI /* GEMM_P == GEMM_Q * 4 */
-        prefetcht2 32(B)
-        prefetcht2 32(B, K, 8)
-        addq    $64, B /* increment */
-#else /* GEMM_P == GEMM_Q * 2 under linux x86_64 */
-        prefetcht2 32(B)
-        prefetcht2 32(B, K, 8)
-        prefetcht2 96(B)
-        prefetcht2 96(B, K, 8)
-        addq    $128, B /* increment */
-#endif
-        sarq    $3, K
-
-	decq	I			# i --
-	jne	.L12_11
-	ALIGN_4	
-
-/**************************************************************************
-* Rest of M 
-***************************************************************************/
-
-        /* recover the original value of pointer B after prefetch */
-        movq    M, I
-        sarq    $2, I
-#ifdef WINDOWS_ABI /* GEMM_P == GEMM_Q * 4 */
-        salq    $6, I
-#else /* GEMM_P == GEMM_Q * 2 under linux x86_64 */
-        salq    $7, I
-#endif
-        subq    I, B
-
-.L12_20:
-	// Test rest of M
-
-	testq	$3, M
-	jz	.L12_100			// to next 16 lines of N
-
-
-.L12_30:
-	testq	$2, M		
-	jz	.L12_40
-
-	ALIGN_4
-
-.L12_31:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $12 * SIZE, BO
-
-	INIT2x12
-
-        movq    K, %rax
-
-	sarq	$3, %rax
-	je	.L12_36
-	ALIGN_4
-
-.L12_32:
-
-	KERNEL2x12_SUB
-	KERNEL2x12_SUB
-	KERNEL2x12_SUB
-	KERNEL2x12_SUB
-
-	KERNEL2x12_SUB
-	KERNEL2x12_SUB
-	KERNEL2x12_SUB
-	KERNEL2x12_SUB
-
-	dec %rax
-	jne	.L12_32
-	ALIGN_4
-
-.L12_36:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L12_39
-
-	ALIGN_4
-
-.L12_37:
-
-	KERNEL2x12_SUB
-
-	dec %rax
-	jne	.L12_37
-	ALIGN_4
-
-
-.L12_39:
-
-	SAVE2x12
-
-	ALIGN_4
-
-.L12_40:
-	testq	$1, M		
-	jz	.L12_100		// to next 3 lines of N
-
-	ALIGN_4
-
-.L12_41:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $12 * SIZE, BO
-
-	INIT1x12
-
-        movq    K, %rax
-
-	sarq	$3,%rax
-	je	.L12_46
-
-	ALIGN_4
-
-.L12_42:
-
-	KERNEL1x12_SUB
-	KERNEL1x12_SUB
-	KERNEL1x12_SUB
-	KERNEL1x12_SUB
-
-	KERNEL1x12_SUB
-	KERNEL1x12_SUB
-	KERNEL1x12_SUB
-	KERNEL1x12_SUB
-
-
-	dec %rax
-	jne	.L12_42
-	ALIGN_4
-
-.L12_46:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L12_49
-
-	ALIGN_4
-
-.L12_47:
-
-	KERNEL1x12_SUB
-
-	dec	%rax
-	jne	.L12_47
-	ALIGN_4
-
-
-.L12_49:
-
-	SAVE1x12
-
-	ALIGN_4
-	
-.L12_100:
-
-
-
-/**************************************************************************************************/
-
-.L13_01:
-        // copy to sub buffer
-        movq    K, %rax
-        salq    $3,%rax                 // K * 8 ; read 8 values
-        movq    B, BO2
-        leaq    (B,%rax, SIZE), BO3     // next offset to BO2
-        leaq    (BO3,%rax, SIZE), B     // next offset to B
-
-
-        leaq    BUFFER1, BO             // first buffer to BO
-        movq    K, %rax
-
-        ALIGN_4
-
-
-.L13_02b:
-
-	vmovups	4 * SIZE(BO2), %ymm1
-	vmovups	0 * SIZE(BO3), %ymm2
-	vmovups	4 * SIZE(BO3), %ymm3
-	vmovups	%ymm1, 0 * SIZE(BO)
-	vmovups	%ymm2, 4 * SIZE(BO)
-	vmovups	%ymm3, 8 * SIZE(BO)
-	addq	$ 8*SIZE,BO2
-	addq	$ 8*SIZE,BO3
-	addq	$ 12*SIZE,BO
-	decq	%rax
-	jnz	.L13_02b
-
-
-
-.L13_10:
-	movq	C, CO1
-	leaq	(C, LDC, 8), C		 
-	leaq	(C, LDC, 4), C		// c += 12 * ldc
-
-	
-	movq	A, AO		 	// aoffset = a
-	addq	$16 * SIZE, AO
-
-	movq	M,  I
-	sarq	$2, I			// i = m / 4
-	je	.L13_20
-
-	ALIGN_4
-
-.L13_11:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $12 * SIZE, BO
-
-        movq    K, %rax
-
-	sarq $3, %rax			//  K / 8
-	cmpq $2, %rax
-
-	jl	.L13_13
-
-
-	KERNEL4x12_I
-	KERNEL4x12_M2
-	KERNEL4x12_M1
-	KERNEL4x12_M2
-
-	KERNEL4x12_M1
-	KERNEL4x12_M2
-	KERNEL4x12_M1
-	KERNEL4x12_M2
-
-	subq $2, %rax
-	je	.L13_12a
-
-	ALIGN_5
-.L13_12:
-
-	KERNEL4x12_M1
-	KERNEL4x12_M2
-	KERNEL4x12_M1
-	KERNEL4x12_M2
-
-	KERNEL4x12_M1
-	KERNEL4x12_M2
-	KERNEL4x12_M1
-	KERNEL4x12_M2
-
-	dec	%rax
-	jne	.L13_12
-
-.L13_12a:
-        prefetcht0 ALPHA
-        PREFETCHT0_C
-        addq  LDC,CO1
-	KERNEL4x12_M1
-        PREFETCHT0_C
-        leaq  (CO1,LDC,2),CO1
-	KERNEL4x12_M2
-        PREFETCHT0_C
-        subq  LDC,CO1
-	KERNEL4x12_M1
-        PREFETCHT0_C
-        subq  LDC,CO1
-        subq  LDC,CO1
-	KERNEL4x12_M2
-
-	KERNEL4x12_M1
-	KERNEL4x12_M2
-	KERNEL4x12_M1
-	KERNEL4x12_E
-
-	jmp .L13_16
-
-.L13_13:
-
-	test $1, %rax
-	jz .L13_14
-
-	KERNEL4x12_I
-	KERNEL4x12_M2
-	KERNEL4x12_M1
-	KERNEL4x12_M2
-
-	KERNEL4x12_M1
-	KERNEL4x12_M2
-	KERNEL4x12_M1
-	KERNEL4x12_E
-
-	jmp .L13_16
-
-
-.L13_14:
-
-	INIT4x12
-
-
-.L13_16:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L13_19
-
-	ALIGN_4
-
-.L13_17:
-
-	KERNEL4x12_SUB
-
-	dec	%rax
-	jne	.L13_17
-	ALIGN_4
-
-
-.L13_19:
-
-	SAVE4x12
-
-        /* here for the prefetch of next b source block */
-	/* the increment should be proportional to GEMM_Q/GEMM_P */
-
-        salq    $3, K
-#ifdef WINDOWS_ABI /* GEMM_P == GEMM_Q * 4 */
-        prefetcht2 (B)
-        prefetcht2 (B, K, 8)
-        addq    $64, B /* increment */
-#else /* GEMM_P == GEMM_Q * 2 under linux x86_64 */
-        prefetcht2 (B)
-        prefetcht2 (B, K, 8)
-        prefetcht2 64(B)
-        prefetcht2 64(B, K, 8)
-        addq    $128, B /* increment */
-#endif
-        sarq    $3, K
-
-	decq	I			# i --
-	jne	.L13_11
-	ALIGN_4	
-
-/**************************************************************************
-* Rest of M 
-***************************************************************************/
-        /* recover the original value of pointer B */
-        movq    M, I
-        sarq    $2, I
-#ifdef WINDOWS_ABI /* GEMM_P == GEMM_Q * 4 */
-        salq    $6, I
-#else /* GEMM_P == GEMM_Q * 2 under linux x86_64 */
-        salq    $7, I
-#endif
-        subq    I, B
-
-.L13_20:
-	// Test rest of M
-
-	testq	$3, M
-	jz	.L13_100			// to next 16 lines of N
-
-
-.L13_30:
-	testq	$2, M		
-	jz	.L13_40
-
-	ALIGN_4
-
-.L13_31:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $12 * SIZE, BO
-
-	INIT2x12
-
-        movq    K, %rax
-
-	sarq	$3, %rax
-	je	.L13_36
-	ALIGN_4
-
-.L13_32:
-
-	KERNEL2x12_SUB
-	KERNEL2x12_SUB
-	KERNEL2x12_SUB
-	KERNEL2x12_SUB
-
-	KERNEL2x12_SUB
-	KERNEL2x12_SUB
-	KERNEL2x12_SUB
-	KERNEL2x12_SUB
-
-	dec %rax
-	jne	.L13_32
-	ALIGN_4
-
-.L13_36:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L13_39
-
-	ALIGN_4
-
-.L13_37:
-
-	KERNEL2x12_SUB
-
-	dec %rax
-	jne	.L13_37
-	ALIGN_4
-
-
-.L13_39:
-
-	SAVE2x12
-
-	ALIGN_4
-
-.L13_40:
-	testq	$1, M		
-	jz	.L13_100		// to next 3 lines of N
-
-	ALIGN_4
-
-.L13_41:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $12 * SIZE, BO
-
-	INIT1x12
-
-        movq    K, %rax
-
-	sarq	$3,%rax
-	je	.L13_46
-
-	ALIGN_4
-
-.L13_42:
-
-	KERNEL1x12_SUB
-	KERNEL1x12_SUB
-	KERNEL1x12_SUB
-	KERNEL1x12_SUB
-
-	KERNEL1x12_SUB
-	KERNEL1x12_SUB
-	KERNEL1x12_SUB
-	KERNEL1x12_SUB
-
-
-	dec %rax
-	jne	.L13_42
-	ALIGN_4
-
-.L13_46:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L13_49
-
-	ALIGN_4
-
-.L13_47:
-
-	KERNEL1x12_SUB
-
-	dec	%rax
-	jne	.L13_47
-	ALIGN_4
-
-
-.L13_49:
-
-	SAVE1x12
-
-	ALIGN_4
-	
-.L13_100:
-
-	decq	J			// j --
-	jg	.L12_01
-
-
-
-
-/**************************************************************************************************/
-
-.L8_0:
-
-	cmpq	$ 0, Nmod12		// N % 12 == 0
-	je	.L999
-
-	movq	Nmod12, J		
-	sarq	$3, J			// j = j / 8
-	je	.L4_0
-
-.L8_10:
-	movq	C, CO1
-	leaq	(C, LDC, 8), C		// c += 4 * ldc
-
-	
-	movq	A, AO		 	// aoffset = a
-	addq	$16 * SIZE, AO
-
-	movq	M,  I
-	sarq	$2, I			// i = m / 4
-	je	.L8_20
-
-	ALIGN_4
-
-.L8_11:
-        movq    B, BO        
-        addq    $12 * SIZE, BO
-
-        movq    K, %rax
-
-	sarq	$3, %rax			//  K / 8
-	cmpq    $2, %rax
-	jl	.L8_13
-
-
-	KERNEL4x8_I
-	KERNEL4x8_M2
-	KERNEL4x8_M1
-	KERNEL4x8_M2
-
-	KERNEL4x8_M1
-	KERNEL4x8_M2
-	KERNEL4x8_M1
-	KERNEL4x8_M2
-
-	subq $2, %rax
-	je	.L8_12a
-
-	ALIGN_5
-
-.L8_12:
-
-	KERNEL4x8_M1
-	KERNEL4x8_M2
-	KERNEL4x8_M1
-	KERNEL4x8_M2
-
-	KERNEL4x8_M1
-	KERNEL4x8_M2
-	KERNEL4x8_M1
-	KERNEL4x8_M2
-
-	dec	%rax
-	jne	.L8_12
-
-.L8_12a:
-
-	KERNEL4x8_M1
-	KERNEL4x8_M2
-	KERNEL4x8_M1
-	KERNEL4x8_M2
-
-	KERNEL4x8_M1
-	KERNEL4x8_M2
-	KERNEL4x8_M1
-	KERNEL4x8_E
-
-	jmp .L8_16
-
-
-.L8_13:
-
-	test $1, %rax
-	jz .L8_14
-
-	KERNEL4x8_I
-	KERNEL4x8_M2
-	KERNEL4x8_M1
-	KERNEL4x8_M2
-
-	KERNEL4x8_M1
-	KERNEL4x8_M2
-	KERNEL4x8_M1
-	KERNEL4x8_E
-
-	jmp .L8_16
-
-
-.L8_14:
-
-	INIT4x8
-
-
-.L8_16:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L8_19
-
-	ALIGN_4
-
-.L8_17:
-
-	KERNEL4x8_SUB
-
-	dec	%rax
-	jne	.L8_17
-	ALIGN_4
-
-
-.L8_19:
-
-	SAVE4x8
-
-	decq	I			# i --
-	jg	.L8_11
-	ALIGN_4	
-
-/**************************************************************************
-* Rest of M 
-***************************************************************************/
-.L8_20:
-	// Test rest of M
-
-	testq	$3, M
-	jz	.L8_100			// to next 16 lines of N
-
-
-.L8_30:
-	testq	$2, M		
-	jz	.L8_40
-
-	ALIGN_4
-
-.L8_31:
-        movq    B, BO             // first buffer to BO
-        addq    $12 * SIZE, BO
-
-	INIT2x8
-
-        movq    K, %rax
-
-	sarq	$3, %rax
-	je	.L8_36
-	ALIGN_4
-
-.L8_32:
-
-	KERNEL2x8_SUB
-	KERNEL2x8_SUB
-	KERNEL2x8_SUB
-	KERNEL2x8_SUB
-
-	KERNEL2x8_SUB
-	KERNEL2x8_SUB
-	KERNEL2x8_SUB
-	KERNEL2x8_SUB
-
-	dec %rax
-	jne	.L8_32
-	ALIGN_4
-
-.L8_36:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L8_39
-
-	ALIGN_4
-
-.L8_37:
-
-	KERNEL2x8_SUB
-
-	dec %rax
-	jne	.L8_37
-
-
-.L8_39:
-
-	SAVE2x8
-
-.L8_40:
-	testq	$1, M		
-	jz	.L8_100		// to next 3 lines of N
-
-	ALIGN_4
-
-.L8_41:
-        movq    B, BO             // first buffer to BO
-        addq    $12 * SIZE, BO
-
-	INIT1x8
-
-        movq    K, %rax
-
-	sarq	$3,%rax
-	je	.L8_46
-
-	ALIGN_4
-
-.L8_42:
-
-	KERNEL1x8_SUB
-	KERNEL1x8_SUB
-	KERNEL1x8_SUB
-	KERNEL1x8_SUB
-
-	KERNEL1x8_SUB
-	KERNEL1x8_SUB
-	KERNEL1x8_SUB
-	KERNEL1x8_SUB
-
-	dec %rax
-	jne	.L8_42
-	ALIGN_4
-
-.L8_46:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L8_49
-
-	ALIGN_4
-
-.L8_47:
-
-	KERNEL1x8_SUB
-
-	dec	%rax
-	jne	.L8_47
-	ALIGN_4
-
-
-.L8_49:
-
-	SAVE1x8
-
-	ALIGN_4
-	
-.L8_100:
-
-	movq	K, %rax
-	salq	$3, %rax		// * 8
-	leaq	(B , %rax, SIZE), B
-	decq	J			// j --
-	jg	.L8_10
-
-
-
-/**************************************************************************************************/
-
-.L4_0:
-
-	cmpq	$ 0, Nmod12		// N % 12 == 0
-	je	.L999
-
-	movq	Nmod12, J		
-	testq   $4, J			// j = j / 4
-	je	.L2_0
-
-.L4_10:
-	movq	C, CO1
-	leaq	(C, LDC, 4), C		// c += 4 * ldc
-
-	
-	movq	A, AO		 	// aoffset = a
-	addq	$16 * SIZE, AO
-
-	movq	M,  I
-	sarq	$2, I			// i = m / 4
-	je	.L4_20
-
-	ALIGN_4
-
-.L4_11:
-        movq    B, BO        
-        addq    $12 * SIZE, BO
-
-        movq    K, %rax
-
-	sarq	$3, %rax			//  K / 8
-	cmpq    $2, %rax
-	jl	.L4_13
-
-
-	KERNEL4x4_I
-	KERNEL4x4_M2
-	KERNEL4x4_M1
-	KERNEL4x4_M2
-
-	KERNEL4x4_M1
-	KERNEL4x4_M2
-	KERNEL4x4_M1
-	KERNEL4x4_M2
-
-	subq $2, %rax
-	je	.L4_12a
-
-	ALIGN_5
-
-.L4_12:
-
-	KERNEL4x4_M1
-	KERNEL4x4_M2
-	KERNEL4x4_M1
-	KERNEL4x4_M2
-
-	KERNEL4x4_M1
-	KERNEL4x4_M2
-	KERNEL4x4_M1
-	KERNEL4x4_M2
-
-	dec	%rax
-	jne	.L4_12
-
-.L4_12a:
-
-	KERNEL4x4_M1
-	KERNEL4x4_M2
-	KERNEL4x4_M1
-	KERNEL4x4_M2
-
-	KERNEL4x4_M1
-	KERNEL4x4_M2
-	KERNEL4x4_M1
-	KERNEL4x4_E
-
-	jmp .L4_16
-
-
-.L4_13:
-
-	test $1, %rax
-	jz .L4_14
-
-	KERNEL4x4_I
-	KERNEL4x4_M2
-	KERNEL4x4_M1
-	KERNEL4x4_M2
-
-	KERNEL4x4_M1
-	KERNEL4x4_M2
-	KERNEL4x4_M1
-	KERNEL4x4_E
-
-	jmp .L4_16
-
-
-.L4_14:
-
-	INIT4x4
-
-
-.L4_16:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L4_19
-
-	ALIGN_4
-
-.L4_17:
-
-	KERNEL4x4_SUB
-
-	dec	%rax
-	jne	.L4_17
-	ALIGN_4
-
-
-.L4_19:
-
-	SAVE4x4
-
-	decq	I			# i --
-	jg	.L4_11
-
-	ALIGN_4	
-
-/**************************************************************************
-* Rest of M 
-***************************************************************************/
-.L4_20:
-	// Test rest of M
-
-	testq	$3, M
-	jz	.L4_100			// to next 16 lines of N
-
-
-.L4_30:
-	testq	$2, M		
-	jz	.L4_40
-
-	ALIGN_4
-
-.L4_31:
-        movq    B, BO             // first buffer to BO
-        addq    $12 * SIZE, BO
-
-	INIT2x4
-
-        movq    K, %rax
-
-	sarq	$3, %rax
-	je	.L4_36
-	ALIGN_4
-
-.L4_32:
-
-	KERNEL2x4_SUB
-	KERNEL2x4_SUB
-	KERNEL2x4_SUB
-	KERNEL2x4_SUB
-
-	KERNEL2x4_SUB
-	KERNEL2x4_SUB
-	KERNEL2x4_SUB
-	KERNEL2x4_SUB
-
-	dec %rax
-	jne	.L4_32
-	ALIGN_4
-
-.L4_36:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L4_39
-
-	ALIGN_4
-
-.L4_37:
-
-	KERNEL2x4_SUB
-
-	dec %rax
-	jne	.L4_37
-
-
-.L4_39:
-
-	SAVE2x4
-
-.L4_40:
-	testq	$1, M		
-	jz	.L4_100		// to next 3 lines of N
-
-	ALIGN_4
-
-.L4_41:
-        movq    B, BO             // first buffer to BO
-        addq    $12 * SIZE, BO
-
-	INIT1x4
-
-        movq    K, %rax
-
-	sarq	$3,%rax
-	je	.L4_46
-
-	ALIGN_4
-
-.L4_42:
-
-	KERNEL1x4_SUB
-	KERNEL1x4_SUB
-	KERNEL1x4_SUB
-	KERNEL1x4_SUB
-
-	KERNEL1x4_SUB
-	KERNEL1x4_SUB
-	KERNEL1x4_SUB
-	KERNEL1x4_SUB
-
-	dec %rax
-	jne	.L4_42
-	ALIGN_4
-
-.L4_46:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L4_49
-
-	ALIGN_4
-
-.L4_47:
-
-	KERNEL1x4_SUB
-
-	dec	%rax
-	jne	.L4_47
-	ALIGN_4
-
-
-.L4_49:
-
-	SAVE1x4
-
-	ALIGN_4
-	
-.L4_100:
-
-	movq	K, %rax
-	salq	$2, %rax		// * 4
-	leaq	(B , %rax, SIZE), B
-
-
-
-
-/***************************************************************************************************************/
-
-.L2_0:
-
-	movq	Nmod12, J		
-	testq	$2, J
-	je	.L1_0
-
-.L2_10:
-	movq	C, CO1
-	leaq	(C, LDC, 2), C		// c += 2 * ldc
-
-	
-	movq	A, AO		 	// aoffset = a
-	addq	$16 * SIZE, AO
-
-	movq	M,  I
-	sarq	$2, I			// i = m / 4
-	je	.L2_20
-
-	ALIGN_4
-
-.L2_11:
-        movq    B, BO        
-        addq    $12 * SIZE, BO
-
-	INIT4x2
-
-        movq    K, %rax
-	sarq $3, %rax			//  K / 8
-
-	je	.L2_16
-
-	ALIGN_5
-
-.L2_12:
-
-	KERNEL4x2_SUB
-	KERNEL4x2_SUB
-	KERNEL4x2_SUB
-	KERNEL4x2_SUB
-
-	KERNEL4x2_SUB
-	KERNEL4x2_SUB
-	KERNEL4x2_SUB
-	KERNEL4x2_SUB
-
-	dec	%rax
-	jne	.L2_12
-
-
-.L2_16:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_19
-
-	ALIGN_4
-
-.L2_17:
-
-	KERNEL4x2_SUB
-
-	dec	%rax
-	jne	.L2_17
-	ALIGN_4
-
-
-.L2_19:
-
-	SAVE4x2
-
-	decq	I			# i --
-	jg	.L2_11
-	ALIGN_4	
-
-/**************************************************************************
-* Rest of M 
-***************************************************************************/
-.L2_20:
-	// Test rest of M
-
-	testq	$3, M
-	jz	.L2_100			// to next 16 lines of N
-
-
-.L2_30:
-	testq	$2, M		
-	jz	.L2_40
-
-	ALIGN_4
-
-.L2_31:
-        movq    B, BO             // first buffer to BO
-        addq    $12 * SIZE, BO
-
-	INIT2x2
-
-        movq    K, %rax
-
-	sarq	$3, %rax
-	je	.L2_36
-	ALIGN_4
-
-.L2_32:
-
-	KERNEL2x2_SUB
-	KERNEL2x2_SUB
-	KERNEL2x2_SUB
-	KERNEL2x2_SUB
-
-	KERNEL2x2_SUB
-	KERNEL2x2_SUB
-	KERNEL2x2_SUB
-	KERNEL2x2_SUB
-
-	dec %rax
-	jne	.L2_32
-
-.L2_36:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_39
-
-	ALIGN_4
-
-.L2_37:
-
-	KERNEL2x2_SUB
-
-	dec %rax
-	jne	.L2_37
-
-
-.L2_39:
-
-	SAVE2x2
-
-.L2_40:
-	testq	$1, M		
-	jz	.L2_100		// to next 3 lines of N
-
-.L2_41:
-        movq    B, BO             // first buffer to BO
-        addq    $12 * SIZE, BO
-
-	INIT1x2
-
-        movq    K, %rax
-
-	sarq	$3,%rax
-	je	.L2_46
-
-	ALIGN_4
-
-.L2_42:
-
-	KERNEL1x2_SUB
-	KERNEL1x2_SUB
-	KERNEL1x2_SUB
-	KERNEL1x2_SUB
-
-	KERNEL1x2_SUB
-	KERNEL1x2_SUB
-	KERNEL1x2_SUB
-	KERNEL1x2_SUB
-
-	dec %rax
-	jne	.L2_42
-
-.L2_46:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_49
-
-	ALIGN_4
-
-.L2_47:
-
-	KERNEL1x2_SUB
-
-	dec	%rax
-	jne	.L2_47
-
-.L2_49:
-
-	SAVE1x2
-
-.L2_100:
-
-	movq	K, %rax
-	salq	$1, %rax		// * 2
-	leaq	(B , %rax, SIZE), B
-
-/***************************************************************************************************************/
-
-.L1_0:
-
-	movq	Nmod12, J		
-	testq	$1, J
-	je	.L999
-
-.L1_10:
-	movq	C, CO1
-	leaq	(C, LDC, 1), C		// c += 1 * ldc
-
-	
-	movq	A, AO		 	// aoffset = a
-	addq	$16 * SIZE, AO
-
-	movq	M,  I
-	sarq	$2, I			// i = m / 4
-	je	.L1_20
-
-	ALIGN_4
-
-.L1_11:
-        movq    B, BO        
-        addq    $12 * SIZE, BO
-
-	INIT4x1
-
-        movq    K, %rax
-
-	sarq	$3, %rax			//  K / 8
-	je	.L1_16
-
-	ALIGN_5
-
-.L1_12:
-
-	KERNEL4x1
-
-	dec	%rax
-	jne	.L1_12
-
-
-.L1_16:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_19
-
-	ALIGN_4
-
-.L1_17:
-
-	KERNEL4x1_SUB
-
-	dec	%rax
-	jne	.L1_17
-	ALIGN_4
-
-
-.L1_19:
-
-	SAVE4x1
-
-	decq	I			# i --
-	jg	.L1_11
-
-/**************************************************************************
-* Rest of M 
-***************************************************************************/
-.L1_20:
-	// Test rest of M
-
-	testq	$3, M
-	jz	.L1_100	
-
-
-.L1_30:
-	testq	$2, M		
-	jz	.L1_40
-
-	ALIGN_4
-
-.L1_31:
-        movq    B, BO             // first buffer to BO
-        addq    $12 * SIZE, BO
-
-	INIT2x1
-
-        movq    K, %rax
-
-	sarq	$3, %rax
-	je	.L1_36
-	ALIGN_4
-
-.L1_32:
-
-	KERNEL2x1_SUB
-	KERNEL2x1_SUB
-	KERNEL2x1_SUB
-	KERNEL2x1_SUB
-
-	KERNEL2x1_SUB
-	KERNEL2x1_SUB
-	KERNEL2x1_SUB
-	KERNEL2x1_SUB
-
-
-	dec %rax
-	jne	.L1_32
-
-.L1_36:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_39
-
-	ALIGN_4
-
-.L1_37:
-
-	KERNEL2x1_SUB
-
-	dec %rax
-	jne	.L1_37
-
-.L1_39:
-
-	SAVE2x1
-
-.L1_40:
-	testq	$1, M		
-	jz	.L1_100		// to next 3 lines of N
-
-
-.L1_41:
-        movq    B, BO             // first buffer to BO
-        addq    $12 * SIZE, BO
-
-	INIT1x1
-
-        movq    K, %rax
-
-	sarq	$3,%rax
-	je	.L1_46
-
-	ALIGN_4
-
-.L1_42:
-
-	KERNEL1x1_SUB
-	KERNEL1x1_SUB
-	KERNEL1x1_SUB
-	KERNEL1x1_SUB
-
-	KERNEL1x1_SUB
-	KERNEL1x1_SUB
-	KERNEL1x1_SUB
-	KERNEL1x1_SUB
-
-	dec %rax
-	jne	.L1_42
-
-.L1_46:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_49
-
-	ALIGN_4
-
-.L1_47:
-
-	KERNEL1x1_SUB
-
-	dec	%rax
-	jne	.L1_47
-
-
-.L1_49:
-
-	SAVE1x1
-
-.L1_100:
-
-
-
-
-.L999:
-	vzeroupper
-
-	movq   		SP, %rsp
-	movq	   (%rsp), %rbx
-	movq	  8(%rsp), %rbp
-	movq	 16(%rsp), %r12
-	movq	 24(%rsp), %r13
-	movq	 32(%rsp), %r14
-	movq	 40(%rsp), %r15
-
-#ifdef WINDOWS_ABI
-	movq	 48(%rsp), %rdi
-	movq	 56(%rsp), %rsi
-	vmovups	 64(%rsp), %xmm6
-	vmovups	 80(%rsp), %xmm7
-	vmovups	 96(%rsp), %xmm8
-	vmovups	112(%rsp), %xmm9
-	vmovups	128(%rsp), %xmm10
-	vmovups	144(%rsp), %xmm11
-	vmovups	160(%rsp), %xmm12
-	vmovups	176(%rsp), %xmm13
-	vmovups	192(%rsp), %xmm14
-	vmovups	208(%rsp), %xmm15
-#endif
-
-	addq	$STACKSIZE, %rsp
-	ret
-
-	EPILOGUE
-
-
-#else
-/*************************************************************************************
-* TRMM Kernel
-*************************************************************************************/
-
-
-	PROLOGUE
-	PROFCODE
-	
-	subq	$STACKSIZE, %rsp
-	movq	%rbx,   (%rsp)
-	movq	%rbp,  8(%rsp)
-	movq	%r12, 16(%rsp)
-	movq	%r13, 24(%rsp)
-	movq	%r14, 32(%rsp)
-	movq	%r15, 40(%rsp)
-
-	vzeroupper
-
-#ifdef WINDOWS_ABI
-	movq	%rdi,    48(%rsp)
-	movq	%rsi,    56(%rsp)
-	vmovups	%xmm6,   64(%rsp)
-	vmovups	%xmm7,   80(%rsp)
-	vmovups	%xmm8,   96(%rsp)
-	vmovups	%xmm9,  112(%rsp)
-	vmovups	%xmm10, 128(%rsp)
-	vmovups	%xmm11, 144(%rsp)
-	vmovups	%xmm12, 160(%rsp)
-	vmovups	%xmm13, 176(%rsp)
-	vmovups	%xmm14, 192(%rsp)
-	vmovups	%xmm15, 208(%rsp)
-
-	movq	ARG1,      OLD_M
-	movq	ARG2,      OLD_N
-	movq	ARG3,      OLD_K
-	movq	OLD_A,     A
-	movq	OLD_B,     B
-	movq	OLD_C,     C
-	movq	OLD_LDC,   LDC
-#ifdef TRMMKERNEL
-	vmovsd	OLD_OFFSET, %xmm12
-#endif
-	vmovups	%xmm3, %xmm0
-
-#else
-	movq	STACKSIZE +  8(%rsp), LDC
-#ifdef TRMMKERNEL
-	vmovsd	STACKSIZE + 16(%rsp), %xmm12
-#endif
-
-#endif
-
-	movq    %rsp, SP      # save old stack
-        subq    $128 + L_BUFFER_SIZE, %rsp
-        andq    $-4096, %rsp    # align stack
-
-        STACK_TOUCH
-
-	cmpq	$ 0, OLD_M
-	je	.L999
-
-	cmpq	$ 0, OLD_N
-	je	.L999
-
-	cmpq	$ 0, OLD_K
-	je	.L999
-
-	movq	OLD_M, M
-	movq	OLD_N, N
-	movq	OLD_K, K
-
-	vmovsd	 %xmm0, ALPHA
-
-	salq	$BASE_SHIFT, LDC
-
-	movq    N, %rax
-        xorq    %rdx, %rdx
-        movq    $8,  %rdi
-        divq    %rdi                     //    N / 8
-        movq    %rax, Ndiv12             //    N / 8
-        movq    %rdx, Nmod12             //    N % 8
-
-#ifdef TRMMKERNEL
-        vmovsd  %xmm12, OFFSET
-        vmovsd  %xmm12, KK
-#ifndef LEFT
-        negq    KK
-#endif  
-#endif
-
-/*************************************************************************************************/
-.L8_0:
-	movq	Ndiv12,  J
-	cmpq	$ 0, J
-	je	.L4_0
-	ALIGN_4
-
-.L8_10:
-	movq	C, CO1
-	leaq	(C, LDC, 8), C		// c += 8 * ldc
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        movq    OFFSET, %rax
-        movq    %rax, KK
-#endif
-
-	
-	movq	A, AO		 	// aoffset = a
-	addq	$16 * SIZE, AO
-
-	movq	M,  I
-	sarq	$2, I			// i = m / 4
-	je	.L8_20
-
-	ALIGN_4
-
-.L8_11:
-
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    B, BO        
-        addq    $12 * SIZE, BO
-#else
-        movq    B, BO        
-        addq    $12 * SIZE, BO
-        movq    KK, %rax
-	salq	$3, %rax		// rax * SIZE
-	leaq	(BO,%rax,8), BO		// add number of values in B
-	leaq	(AO,%rax,4), AO		// add number of values in A
-#endif
-
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $4, %rax        // number of values in AO
-#else
-        addq    $8, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-	sarq	$3, %rax			//  K / 8
-	cmpq    $2, %rax
-	jl	.L8_13
-
-
-	KERNEL4x8_I
-	KERNEL4x8_M2
-	KERNEL4x8_M1
-	KERNEL4x8_M2
-
-	KERNEL4x8_M1
-	KERNEL4x8_M2
-	KERNEL4x8_M1
-	KERNEL4x8_M2
-
-	subq $2, %rax
-	je	.L8_12a
-
-	ALIGN_5
-
-.L8_12:
-
-	KERNEL4x8_M1
-	KERNEL4x8_M2
-	KERNEL4x8_M1
-	KERNEL4x8_M2
-
-	KERNEL4x8_M1
-	KERNEL4x8_M2
-	KERNEL4x8_M1
-	KERNEL4x8_M2
-
-	dec	%rax
-	jne	.L8_12
-
-.L8_12a:
-
-	KERNEL4x8_M1
-	KERNEL4x8_M2
-	KERNEL4x8_M1
-	KERNEL4x8_M2
-
-	KERNEL4x8_M1
-	KERNEL4x8_M2
-	KERNEL4x8_M1
-	KERNEL4x8_E
-
-	jmp .L8_16
-
-
-.L8_13:
-
-	test $1, %rax
-	jz .L8_14
-
-	KERNEL4x8_I
-	KERNEL4x8_M2
-	KERNEL4x8_M1
-	KERNEL4x8_M2
-
-	KERNEL4x8_M1
-	KERNEL4x8_M2
-	KERNEL4x8_M1
-	KERNEL4x8_E
-
-	jmp .L8_16
-
-
-.L8_14:
-
-	INIT4x8
-
-
-.L8_16:
-        movq    KKK, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L8_19
-
-	ALIGN_4
-
-.L8_17:
-
-	KERNEL4x8_SUB
-
-	dec	%rax
-	jne	.L8_17
-	ALIGN_4
-
-
-.L8_19:
-
-	SAVE4x8
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-	salq	$3, %rax			// rax + SIZE
-        leaq    (BO, %rax, 8), BO		// number of values in B
-        leaq    (AO, %rax, 4), AO		// number of values in A
-#endif
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $4, KK				// number of values in A
-#endif
-
-	decq	I			# i --
-	jg	.L8_11
-	ALIGN_4	
-
-/**************************************************************************
-* Rest of M 
-***************************************************************************/
-.L8_20:
-	// Test rest of M
-
-	testq	$3, M
-	jz	.L8_100			// to next 16 lines of N
-
-
-.L8_30:
-	testq	$2, M		
-	jz	.L8_40
-
-	ALIGN_4
-
-.L8_31:
-
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    B, BO        
-        addq    $12 * SIZE, BO
-#else
-        movq    B, BO        
-        addq    $12 * SIZE, BO
-        movq    KK, %rax
-	salq	$3, %rax		// rax * SIZE
-	leaq	(BO,%rax,8), BO		// add number of values in B
-	leaq	(AO,%rax,2), AO		// add number of values in A
-#endif
-
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $2, %rax        // number of values in AO
-#else
-        addq    $8, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-	INIT2x8
-
-	sarq	$3, %rax
-	je	.L8_36
-	ALIGN_4
-
-.L8_32:
-
-	KERNEL2x8_SUB
-	KERNEL2x8_SUB
-	KERNEL2x8_SUB
-	KERNEL2x8_SUB
-
-	KERNEL2x8_SUB
-	KERNEL2x8_SUB
-	KERNEL2x8_SUB
-	KERNEL2x8_SUB
-
-	dec %rax
-	jne	.L8_32
-	ALIGN_4
-
-.L8_36:
-        movq    KKK, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L8_39
-
-	ALIGN_4
-
-.L8_37:
-
-	KERNEL2x8_SUB
-
-	dec %rax
-	jne	.L8_37
-
-
-.L8_39:
-
-	SAVE2x8
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-	salq	$3, %rax			// rax + SIZE
-        leaq    (BO, %rax, 8), BO		// number of values in B
-        leaq    (AO, %rax, 2), AO		// number of values in A
-#endif
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $2, KK				// number of values in A
-#endif
-
-
-.L8_40:
-	testq	$1, M		
-	jz	.L8_100		// to next 3 lines of N
-
-	ALIGN_4
-
-.L8_41:
-
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    B, BO        
-        addq    $12 * SIZE, BO
-#else
-        movq    B, BO        
-        addq    $12 * SIZE, BO
-        movq    KK, %rax
-	salq	$3, %rax		// rax * SIZE
-	leaq	(BO,%rax,8), BO		// add number of values in B
-	leaq	(AO,%rax,1), AO		// add number of values in A
-#endif
-
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $1, %rax        // number of values in AO
-#else
-        addq    $8, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-	INIT1x8
-
-	sarq	$3,%rax
-	je	.L8_46
-
-	ALIGN_4
-
-.L8_42:
-
-	KERNEL1x8_SUB
-	KERNEL1x8_SUB
-	KERNEL1x8_SUB
-	KERNEL1x8_SUB
-
-	KERNEL1x8_SUB
-	KERNEL1x8_SUB
-	KERNEL1x8_SUB
-	KERNEL1x8_SUB
-
-	dec %rax
-	jne	.L8_42
-	ALIGN_4
-
-.L8_46:
-        movq    KKK, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L8_49
-
-	ALIGN_4
-
-.L8_47:
-
-	KERNEL1x8_SUB
-
-	dec	%rax
-	jne	.L8_47
-	ALIGN_4
-
-
-.L8_49:
-
-	SAVE1x8
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-	salq	$3, %rax			// rax + SIZE
-        leaq    (BO, %rax, 8), BO		// number of values in B
-        leaq    (AO, %rax, 1), AO		// number of values in A
-#endif
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $1, KK				// number of values in A
-#endif
-
-.L8_100:
-
-#if defined(TRMMKERNEL) && !defined(LEFT)
-        addq    $8, KK				// number of values in B
-#endif
-
-
-	decq	J			// j --
-	jg	.L8_10
-
-
-
-
-
-/*************************************************************************************************/
-.L4_0:
-	movq	Nmod12, J		
-	testq	$4, J
-	je	.L2_0
-	ALIGN_4
-
-.L4_10:
-	movq	C, CO1
-	leaq	(C, LDC, 4), C		// c += 4 * ldc
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        movq    OFFSET, %rax
-        movq    %rax, KK
-#endif
-
-	
-	movq	A, AO		 	// aoffset = a
-	addq	$16 * SIZE, AO
-
-	movq	M,  I
-	sarq	$2, I			// i = m / 4
-	je	.L4_20
-
-	ALIGN_4
-
-.L4_11:
-
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    B, BO        
-        addq    $12 * SIZE, BO
-#else
-        movq    B, BO        
-        addq    $12 * SIZE, BO
-        movq    KK, %rax
-	salq	$3, %rax		// rax * SIZE
-	leaq	(BO,%rax,4), BO		// add number of values in B
-	leaq	(AO,%rax,4), AO		// add number of values in A
-#endif
-
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $4, %rax        // number of values in AO
-#else
-        addq    $4, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-	sarq	$3, %rax			//  K / 8
-	cmpq    $2, %rax
-	jl	.L4_13
-
-
-	KERNEL4x4_I
-	KERNEL4x4_M2
-	KERNEL4x4_M1
-	KERNEL4x4_M2
-
-	KERNEL4x4_M1
-	KERNEL4x4_M2
-	KERNEL4x4_M1
-	KERNEL4x4_M2
-
-	subq $2, %rax
-	je	.L4_12a
-
-	ALIGN_5
-
-.L4_12:
-
-	KERNEL4x4_M1
-	KERNEL4x4_M2
-	KERNEL4x4_M1
-	KERNEL4x4_M2
-
-	KERNEL4x4_M1
-	KERNEL4x4_M2
-	KERNEL4x4_M1
-	KERNEL4x4_M2
-
-	dec	%rax
-	jne	.L4_12
-
-.L4_12a:
-
-	KERNEL4x4_M1
-	KERNEL4x4_M2
-	KERNEL4x4_M1
-	KERNEL4x4_M2
-
-	KERNEL4x4_M1
-	KERNEL4x4_M2
-	KERNEL4x4_M1
-	KERNEL4x4_E
-
-	jmp .L4_16
-
-
-.L4_13:
-
-	test $1, %rax
-	jz .L4_14
-
-	KERNEL4x4_I
-	KERNEL4x4_M2
-	KERNEL4x4_M1
-	KERNEL4x4_M2
-
-	KERNEL4x4_M1
-	KERNEL4x4_M2
-	KERNEL4x4_M1
-	KERNEL4x4_E
-
-	jmp .L4_16
-
-
-.L4_14:
-
-	INIT4x4
-
-
-.L4_16:
-        movq    KKK, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L4_19
-
-	ALIGN_4
-
-.L4_17:
-
-	KERNEL4x4_SUB
-
-	dec	%rax
-	jne	.L4_17
-	ALIGN_4
-
-
-.L4_19:
-
-	SAVE4x4
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-	salq	$3, %rax			// rax + SIZE
-        leaq    (BO, %rax, 4), BO		// number of values in B
-        leaq    (AO, %rax, 4), AO		// number of values in A
-#endif
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $4, KK				// number of values in A
-#endif
-
-	decq	I			# i --
-	jg	.L4_11
-	ALIGN_4	
-
-/**************************************************************************
-* Rest of M 
-***************************************************************************/
-.L4_20:
-	// Test rest of M
-
-	testq	$3, M
-	jz	.L4_100			// to next 16 lines of N
-
-
-.L4_30:
-	testq	$2, M		
-	jz	.L4_40
-
-	ALIGN_4
-
-.L4_31:
-
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    B, BO        
-        addq    $12 * SIZE, BO
-#else
-        movq    B, BO        
-        addq    $12 * SIZE, BO
-        movq    KK, %rax
-	salq	$3, %rax		// rax * SIZE
-	leaq	(BO,%rax,4), BO		// add number of values in B
-	leaq	(AO,%rax,2), AO		// add number of values in A
-#endif
-
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $2, %rax        // number of values in AO
-#else
-        addq    $4, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-	INIT2x4
-
-	sarq	$3, %rax
-	je	.L4_36
-	ALIGN_4
-
-.L4_32:
-
-	KERNEL2x4_SUB
-	KERNEL2x4_SUB
-	KERNEL2x4_SUB
-	KERNEL2x4_SUB
-
-	KERNEL2x4_SUB
-	KERNEL2x4_SUB
-	KERNEL2x4_SUB
-	KERNEL2x4_SUB
-
-	dec %rax
-	jne	.L4_32
-	ALIGN_4
-
-.L4_36:
-        movq    KKK, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L4_39
-
-	ALIGN_4
-
-.L4_37:
-
-	KERNEL2x4_SUB
-
-	dec %rax
-	jne	.L4_37
-
-
-.L4_39:
-
-	SAVE2x4
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-	salq	$3, %rax			// rax + SIZE
-        leaq    (BO, %rax, 4), BO		// number of values in B
-        leaq    (AO, %rax, 2), AO		// number of values in A
-#endif
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $2, KK				// number of values in A
-#endif
-
-
-.L4_40:
-	testq	$1, M		
-	jz	.L4_100		// to next 3 lines of N
-
-	ALIGN_4
-
-.L4_41:
-
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    B, BO        
-        addq    $12 * SIZE, BO
-#else
-        movq    B, BO        
-        addq    $12 * SIZE, BO
-        movq    KK, %rax
-	salq	$3, %rax		// rax * SIZE
-	leaq	(BO,%rax,4), BO		// add number of values in B
-	leaq	(AO,%rax,1), AO		// add number of values in A
-#endif
-
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $1, %rax        // number of values in AO
-#else
-        addq    $4, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-	INIT1x4
-
-	sarq	$3,%rax
-	je	.L4_46
-
-	ALIGN_4
-
-.L4_42:
-
-	KERNEL1x4_SUB
-	KERNEL1x4_SUB
-	KERNEL1x4_SUB
-	KERNEL1x4_SUB
-
-	KERNEL1x4_SUB
-	KERNEL1x4_SUB
-	KERNEL1x4_SUB
-	KERNEL1x4_SUB
-
-	dec %rax
-	jne	.L4_42
-	ALIGN_4
-
-.L4_46:
-        movq    KKK, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L4_49
-
-	ALIGN_4
-
-.L4_47:
-
-	KERNEL1x4_SUB
-
-	dec	%rax
-	jne	.L4_47
-	ALIGN_4
-
-
-.L4_49:
-
-	SAVE1x4
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-	salq	$3, %rax			// rax + SIZE
-        leaq    (BO, %rax, 4), BO		// number of values in B
-        leaq    (AO, %rax, 1), AO		// number of values in A
-#endif
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $1, KK				// number of values in A
-#endif
-
-.L4_100:
-
-#if defined(TRMMKERNEL) && !defined(LEFT)
-        addq    $4, KK				// number of values in B
-#endif
-
-
-	movq	K, %rax
-	salq	$2, %rax		// * 4
-	leaq	(B , %rax, SIZE), B
-
-
-
-
-/***************************************************************************************************************/
-
-.L2_0:
-
-	movq	Nmod12, J		
-	testq	$2, J
-	je	.L1_0
-
-.L2_10:
-	movq	C, CO1
-	leaq	(C, LDC, 2), C		// c += 2 * ldc
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        movq    OFFSET, %rax
-        movq    %rax, KK
-#endif
-
-
-	
-	movq	A, AO		 	// aoffset = a
-	addq	$16 * SIZE, AO
-
-	movq	M,  I
-	sarq	$2, I			// i = m / 4
-	je	.L2_20
-
-	ALIGN_4
-
-.L2_11:
-
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    B, BO        
-        addq    $12 * SIZE, BO
-#else
-        movq    B, BO        
-        addq    $12 * SIZE, BO
-        movq    KK, %rax
-	salq	$3, %rax		// rax * SIZE
-	leaq	(BO,%rax,2), BO		// add number of values in B
-	leaq	(AO,%rax,4), AO		// add number of values in A
-#endif
-
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $4, %rax        // number of values in AO
-#else
-        addq    $2, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-	INIT4x2
-
-	sarq $3, %rax			//  K / 8
-
-	je	.L2_16
-
-	ALIGN_5
-
-.L2_12:
-
-	KERNEL4x2_SUB
-	KERNEL4x2_SUB
-	KERNEL4x2_SUB
-	KERNEL4x2_SUB
-
-	KERNEL4x2_SUB
-	KERNEL4x2_SUB
-	KERNEL4x2_SUB
-	KERNEL4x2_SUB
-
-	dec	%rax
-	jne	.L2_12
-
-
-.L2_16:
-        movq    KKK, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_19
-
-	ALIGN_4
-
-.L2_17:
-
-	KERNEL4x2_SUB
-
-	dec	%rax
-	jne	.L2_17
-	ALIGN_4
-
-
-.L2_19:
-
-	SAVE4x2
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-	salq	$3, %rax			// rax + SIZE
-        leaq    (BO, %rax, 2), BO		// number of values in B
-        leaq    (AO, %rax, 4), AO		// number of values in A
-#endif
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $4, KK				// number of values in A
-#endif
-
-
-	decq	I			# i --
-	jg	.L2_11
-	ALIGN_4	
-
-/**************************************************************************
-* Rest of M 
-***************************************************************************/
-.L2_20:
-	// Test rest of M
-
-	testq	$3, M
-	jz	.L2_100			// to next 16 lines of N
-
-
-.L2_30:
-	testq	$2, M		
-	jz	.L2_40
-
-	ALIGN_4
-
-.L2_31:
-
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    B, BO        
-        addq    $12 * SIZE, BO
-#else
-        movq    B, BO        
-        addq    $12 * SIZE, BO
-        movq    KK, %rax
-	salq	$3, %rax		// rax * SIZE
-	leaq	(BO,%rax,2), BO		// add number of values in B
-	leaq	(AO,%rax,2), AO		// add number of values in A
-#endif
-
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $2, %rax        // number of values in AO
-#else
-        addq    $2, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-	INIT2x2
-
-	sarq	$3, %rax
-	je	.L2_36
-	ALIGN_4
-
-.L2_32:
-
-	KERNEL2x2_SUB
-	KERNEL2x2_SUB
-	KERNEL2x2_SUB
-	KERNEL2x2_SUB
-
-	KERNEL2x2_SUB
-	KERNEL2x2_SUB
-	KERNEL2x2_SUB
-	KERNEL2x2_SUB
-
-	dec %rax
-	jne	.L2_32
-
-.L2_36:
-        movq    KKK, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_39
-
-	ALIGN_4
-
-.L2_37:
-
-	KERNEL2x2_SUB
-
-	dec %rax
-	jne	.L2_37
-
-
-.L2_39:
-
-	SAVE2x2
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-	salq	$3, %rax			// rax + SIZE
-        leaq    (BO, %rax, 2), BO		// number of values in B
-        leaq    (AO, %rax, 2), AO		// number of values in A
-#endif
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $2, KK				// number of values in A
-#endif
-
-
-.L2_40:
-	testq	$1, M		
-	jz	.L2_100		// to next 3 lines of N
-
-.L2_41:
-
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    B, BO        
-        addq    $12 * SIZE, BO
-#else
-        movq    B, BO        
-        addq    $12 * SIZE, BO
-        movq    KK, %rax
-	salq	$3, %rax		// rax * SIZE
-	leaq	(BO,%rax,2), BO		// add number of values in B
-	leaq	(AO,%rax,1), AO		// add number of values in A
-#endif
-
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $1, %rax        // number of values in AO
-#else
-        addq    $2, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-	INIT1x2
-
-	sarq	$3,%rax
-	je	.L2_46
-
-	ALIGN_4
-
-.L2_42:
-
-	KERNEL1x2_SUB
-	KERNEL1x2_SUB
-	KERNEL1x2_SUB
-	KERNEL1x2_SUB
-
-	KERNEL1x2_SUB
-	KERNEL1x2_SUB
-	KERNEL1x2_SUB
-	KERNEL1x2_SUB
-
-	dec %rax
-	jne	.L2_42
-
-.L2_46:
-        movq    KKK, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_49
-
-	ALIGN_4
-
-.L2_47:
-
-	KERNEL1x2_SUB
-
-	dec	%rax
-	jne	.L2_47
-
-.L2_49:
-
-	SAVE1x2
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-	salq	$3, %rax			// rax * SIZE
-        leaq    (BO, %rax, 2), BO		// number of values in B
-        leaq    (AO, %rax, 1), AO		// number of values in A
-#endif
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $1, KK				// number of values in A
-#endif
-
-
-.L2_100:
-
-
-#if defined(TRMMKERNEL) && !defined(LEFT)
-        addq    $2, KK				// number of values in B
-#endif
-
-	movq	K, %rax
-	salq	$1, %rax		// * 2
-	leaq	(B , %rax, SIZE), B
-
-/***************************************************************************************************************/
-
-.L1_0:
-
-	movq	Nmod12, J		
-	testq	$1, J
-	je	.L999
-
-.L1_10:
-	movq	C, CO1
-	leaq	(C, LDC, 1), C		// c += 1 * ldc
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        movq    OFFSET, %rax
-        movq    %rax, KK
-#endif
-	
-	movq	A, AO		 	// aoffset = a
-	addq	$16 * SIZE, AO
-
-	movq	M,  I
-	sarq	$2, I			// i = m / 4
-	je	.L1_20
-
-	ALIGN_4
-
-.L1_11:
-
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    B, BO        
-        addq    $12 * SIZE, BO
-#else
-        movq    B, BO        
-        addq    $12 * SIZE, BO
-        movq    KK, %rax
-	salq	$3, %rax		// rax * SIZE
-	leaq	(BO,%rax,1), BO		// add number of values in B
-	leaq	(AO,%rax,4), AO		// add number of values in A
-#endif
-
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $4, %rax        // number of values in AO
-#else
-        addq    $1, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-	INIT4x1
-
-	sarq	$3, %rax			//  K / 8
-	je	.L1_16
-
-	ALIGN_5
-
-.L1_12:
-
-	KERNEL4x1
-
-	dec	%rax
-	jne	.L1_12
-
-
-.L1_16:
-        movq    KKK, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_19
-
-	ALIGN_4
-
-.L1_17:
-
-	KERNEL4x1_SUB
-
-	dec	%rax
-	jne	.L1_17
-	ALIGN_4
-
-
-.L1_19:
-
-	SAVE4x1
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-	salq	$3, %rax			// rax * SIZE
-        leaq    (BO, %rax, 1), BO		// number of values in B
-        leaq    (AO, %rax, 4), AO		// number of values in A
-#endif
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $4, KK				// number of values in A
-#endif
-
-
-	decq	I			# i --
-	jg	.L1_11
-
-/**************************************************************************
-* Rest of M 
-***************************************************************************/
-.L1_20:
-	// Test rest of M
-
-	testq	$3, M
-	jz	.L1_100	
-
-
-.L1_30:
-	testq	$2, M		
-	jz	.L1_40
-
-	ALIGN_4
-
-.L1_31:
-
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    B, BO        
-        addq    $12 * SIZE, BO
-#else
-        movq    B, BO        
-        addq    $12 * SIZE, BO
-        movq    KK, %rax
-	salq	$3, %rax		// rax * SIZE
-	leaq	(BO,%rax,1), BO		// add number of values in B
-	leaq	(AO,%rax,2), AO		// add number of values in A
-#endif
-
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $2, %rax        // number of values in AO
-#else
-        addq    $1, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-	INIT2x1
-
-	sarq	$3, %rax
-	je	.L1_36
-	ALIGN_4
-
-.L1_32:
-
-	KERNEL2x1_SUB
-	KERNEL2x1_SUB
-	KERNEL2x1_SUB
-	KERNEL2x1_SUB
-
-	KERNEL2x1_SUB
-	KERNEL2x1_SUB
-	KERNEL2x1_SUB
-	KERNEL2x1_SUB
-
-
-	dec %rax
-	jne	.L1_32
-
-.L1_36:
-        movq    KKK, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_39
-
-	ALIGN_4
-
-.L1_37:
-
-	KERNEL2x1_SUB
-
-	dec %rax
-	jne	.L1_37
-
-.L1_39:
-
-	SAVE2x1
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-	salq	$3, %rax			// rax * SIZE
-        leaq    (BO, %rax, 1), BO		// number of values in B
-        leaq    (AO, %rax, 2), AO		// number of values in A
-#endif
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $2, KK				// number of values in A
-#endif
-
-
-.L1_40:
-	testq	$1, M		
-	jz	.L1_100		// to next 3 lines of N
-
-
-.L1_41:
-
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    B, BO        
-        addq    $12 * SIZE, BO
-#else
-        movq    B, BO        
-        addq    $12 * SIZE, BO
-        movq    KK, %rax
-	salq	$3, %rax		// rax * SIZE
-	leaq	(BO,%rax,1), BO		// add number of values in B
-	leaq	(AO,%rax,1), AO		// add number of values in A
-#endif
-
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $1, %rax        // number of values in AO
-#else
-        addq    $1, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-	INIT1x1
-
-	sarq	$3,%rax
-	je	.L1_46
-
-	ALIGN_4
-
-.L1_42:
-
-	KERNEL1x1_SUB
-	KERNEL1x1_SUB
-	KERNEL1x1_SUB
-	KERNEL1x1_SUB
-
-	KERNEL1x1_SUB
-	KERNEL1x1_SUB
-	KERNEL1x1_SUB
-	KERNEL1x1_SUB
-
-	dec %rax
-	jne	.L1_42
-
-.L1_46:
-        movq    KKK, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_49
-
-	ALIGN_4
-
-.L1_47:
-
-	KERNEL1x1_SUB
-
-	dec	%rax
-	jne	.L1_47
-
-
-.L1_49:
-
-	SAVE1x1
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-	salq	$3, %rax			// rax * SIZE
-        leaq    (BO, %rax, 1), BO		// number of values in B
-        leaq    (AO, %rax, 1), AO		// number of values in A
-#endif
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $1, KK				// number of values in A
-#endif
-
-
-
-.L1_100:
-
-
-#if defined(TRMMKERNEL) && !defined(LEFT)
-        addq    $1, KK				// number of values in B
-#endif
-
-
-
-.L999:
-
-	vzeroupper
-
-	movq   		SP, %rsp
-	movq	   (%rsp), %rbx
-	movq	  8(%rsp), %rbp
-	movq	 16(%rsp), %r12
-	movq	 24(%rsp), %r13
-	movq	 32(%rsp), %r14
-	movq	 40(%rsp), %r15
-
-#ifdef WINDOWS_ABI
-	movq	 48(%rsp), %rdi
-	movq	 56(%rsp), %rsi
-	vmovups	 64(%rsp), %xmm6
-	vmovups	 80(%rsp), %xmm7
-	vmovups	 96(%rsp), %xmm8
-	vmovups	112(%rsp), %xmm9
-	vmovups	128(%rsp), %xmm10
-	vmovups	144(%rsp), %xmm11
-	vmovups	160(%rsp), %xmm12
-	vmovups	176(%rsp), %xmm13
-	vmovups	192(%rsp), %xmm14
-	vmovups	208(%rsp), %xmm15
-#endif
-
-	addq	$STACKSIZE, %rsp
-	ret
-
-	EPILOGUE
-
-
-
-
-
-#endif
+/*********************************************************************************
+Copyright (c) 2015, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+**********************************************************************************/
+
+
+#define ASSEMBLER
+#include "common.h"
+ 
+#define OLD_M	%rdi
+#define OLD_N	%rsi
+#define M	%r13
+#define J	%r14
+#define OLD_K	%rdx
+
+#define A	%rcx
+#define B	%r8
+#define C	%r9
+#define LDC	%r10
+	
+#define I	%r11
+#define AO	%rdi
+#define BO	%rsi
+#define	CO1	%r15
+#define K	%r12
+#define	SP	%rbx
+
+#define BO1	%rdi
+#define BO2	%r15
+#define BO3	%rbp
+
+#ifndef WINDOWS_ABI
+
+#define STACKSIZE 96
+#define L_BUFFER_SIZE 256*8*12+4096
+
+#else
+
+#define STACKSIZE 256
+#define L_BUFFER_SIZE 128*8*12+512
+
+#define OLD_A		40 + STACKSIZE(%rsp)
+#define OLD_B		48 + STACKSIZE(%rsp)
+#define OLD_C		56 + STACKSIZE(%rsp)
+#define OLD_LDC		64 + STACKSIZE(%rsp)
+#define OLD_OFFSET	72 + STACKSIZE(%rsp)
+
+#endif
+
+
+#define Ndiv12	 24(%rsp)
+#define Nmod12	 32(%rsp)
+#define N	 40(%rsp)
+#define ALPHA	 48(%rsp)
+#define OFFSET	 56(%rsp)
+#define KK	 64(%rsp)
+#define KKK	 72(%rsp)
+#define BUFFER1	           128(%rsp)
+
+#if defined(OS_WINDOWS)
+#if   L_BUFFER_SIZE > 16384
+#define STACK_TOUCH \
+        movl    $ 0,  4096 * 4(%rsp);\
+        movl    $ 0,  4096 * 3(%rsp);\
+        movl    $ 0,  4096 * 2(%rsp);\
+        movl    $ 0,  4096 * 1(%rsp);
+#elif L_BUFFER_SIZE > 12288
+#define STACK_TOUCH \
+        movl    $ 0,  4096 * 3(%rsp);\
+        movl    $ 0,  4096 * 2(%rsp);\
+        movl    $ 0,  4096 * 1(%rsp);
+#elif L_BUFFER_SIZE > 8192
+#define STACK_TOUCH \
+        movl    $ 0,  4096 * 2(%rsp);\
+        movl    $ 0,  4096 * 1(%rsp);
+#elif L_BUFFER_SIZE > 4096
+#define STACK_TOUCH \
+        movl    $ 0,  4096 * 1(%rsp);
+#else
+#define STACK_TOUCH
+#endif
+#else
+#define STACK_TOUCH
+#endif
+
+#define	A_PR1	512
+#define	B_PR1	160
+#define BROADCASTKERNEL
+
+/*******************************************************************************************
+* Macro definitions
+*******************************************************************************************/
+
+.macro INIT4x12
+
+	vxorpd		%ymm4 , %ymm4 , %ymm4
+	vxorpd		%ymm5 , %ymm5 , %ymm5
+	vxorpd		%ymm6 , %ymm6 , %ymm6
+	vxorpd		%ymm7 , %ymm7 , %ymm7
+	vxorpd		%ymm8 , %ymm8 , %ymm8
+	vxorpd		%ymm9 , %ymm9 , %ymm9
+	vxorpd		%ymm10, %ymm10, %ymm10
+	vxorpd		%ymm11, %ymm11, %ymm11
+	vxorpd		%ymm12, %ymm12, %ymm12
+	vxorpd		%ymm13, %ymm13, %ymm13
+	vxorpd		%ymm14, %ymm14, %ymm14
+	vxorpd		%ymm15, %ymm15, %ymm15
+
+.endm
+
+.macro KERNEL4x12_I
+	prefetcht0	A_PR1(AO)
+	vmovups		-12 * SIZE(BO), %ymm1
+	prefetcht0	B_PR1(BO)
+# if defined BROADCASTKERNEL
+        vbroadcastsd    -16 * SIZE(AO), %ymm0
+# else
+	vmovups 	-16 * SIZE(AO), %ymm0
+# endif
+	prefetcht0	B_PR1+64(BO)
+	vmovups		 -8 * SIZE(BO), %ymm2
+	prefetcht0	B_PR1+128(BO)
+	vmovups		 -4 * SIZE(BO), %ymm3
+	vmulpd  	%ymm0 ,%ymm1  , %ymm4
+	prefetcht0	B_PR1+192(BO)
+	vmulpd  	%ymm0 ,%ymm2  , %ymm8
+	vmulpd  	%ymm0 ,%ymm3  , %ymm12
+	prefetcht0	B_PR1+256(BO)
+# if defined BROADCASTKERNEL
+        vbroadcastsd    -15 * SIZE(AO), %ymm0
+# else
+	vpermilpd	$ 0x05, %ymm0  , %ymm0
+# endif
+	vmulpd  	%ymm0 ,%ymm1  , %ymm5
+	vmulpd  	%ymm0 ,%ymm2  , %ymm9
+	vmulpd  	%ymm0 ,%ymm3  , %ymm13
+# if defined BROADCASTKERNEL
+        vbroadcastsd    -14 * SIZE(AO), %ymm0
+# else
+	vpermpd		$ 0x1b, %ymm0  , %ymm0
+# endif
+	vmulpd  	%ymm0 ,%ymm1  , %ymm6
+	vmulpd  	%ymm0 ,%ymm2  , %ymm10
+
+	addq		$ 12*SIZE, BO
+	vmulpd  	%ymm0 ,%ymm3  , %ymm14
+# if defined BROADCASTKERNEL
+        vbroadcastsd    -13 * SIZE(AO), %ymm0
+# else
+	vpermilpd	$ 0x05, %ymm0  , %ymm0
+# endif
+	vmulpd  	%ymm0 ,%ymm1  , %ymm7
+	vmovups		-12 * SIZE(BO), %ymm1
+	vmulpd  	%ymm0 ,%ymm2  , %ymm11
+	vmovups		 -8 * SIZE(BO), %ymm2
+	vmulpd  	%ymm0 ,%ymm3  , %ymm15
+	vmovups		 -4 * SIZE(BO), %ymm3
+
+.endm
+
+.macro KERNEL4x12_M1
+	prefetcht0	A_PR1(AO)
+# if defined BROADCASTKERNEL
+        vbroadcastsd    -16 * SIZE(AO), %ymm0
+# else
+	vmovups 	-16 * SIZE(AO), %ymm0
+# endif
+	prefetcht0	B_PR1(BO)
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4
+	prefetcht0	B_PR1+64(BO)
+	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm8
+	prefetcht0	B_PR1+128(BO)
+	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm12
+# if defined BROADCASTKERNEL
+        vbroadcastsd    -15 * SIZE(AO), %ymm0
+# else
+	vpermilpd	$ 0x05, %ymm0  , %ymm0
+# endif
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5
+	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm9
+	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm13
+# if defined BROADCASTKERNEL
+        vbroadcastsd    -14 * SIZE(AO), %ymm0
+# else
+	vpermpd		$ 0x1b, %ymm0  , %ymm0
+# endif
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6
+	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm10
+	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm14
+# if defined BROADCASTKERNEL
+        vbroadcastsd    -13 * SIZE(AO), %ymm0
+# else
+	vpermilpd	$ 0x05, %ymm0  , %ymm0
+# endif
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7
+	vmovups		-12 * SIZE(BO), %ymm1
+	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm11
+	vmovups		 -8 * SIZE(BO), %ymm2
+	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm15
+	vmovups		 -4 * SIZE(BO), %ymm3
+
+.endm
+
+.macro KERNEL4x12_M2
+# if defined BROADCASTKERNEL
+        vbroadcastsd    -12 * SIZE(AO), %ymm0
+# else
+	vmovups 	-12 * SIZE(AO), %ymm0
+# endif
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4
+	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm8
+	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm12
+# if defined BROADCASTKERNEL
+        vbroadcastsd    -11 * SIZE(AO), %ymm0
+# else
+	vpermilpd	$ 0x05, %ymm0  , %ymm0
+# endif
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5
+	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm9
+	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm13
+# if defined BROADCASTKERNEL
+        vbroadcastsd    -10 * SIZE(AO), %ymm0
+# else
+	vpermpd		$ 0x1b, %ymm0  , %ymm0
+# endif
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6
+	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm10
+
+	addq		$ 8*SIZE, AO
+	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm14
+# if defined BROADCASTKERNEL
+        vbroadcastsd    -17 * SIZE(AO), %ymm0
+# else
+	vpermilpd	$ 0x05, %ymm0  , %ymm0
+# endif
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7
+	vmovups		  0 * SIZE(BO), %ymm1
+	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm11
+	vmovups		  4 * SIZE(BO), %ymm2
+	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm15
+	vmovups		  8 * SIZE(BO), %ymm3
+	addq		$ 24*SIZE, BO
+.endm
+
+
+.macro KERNEL4x12_E
+# if defined BROADCASTKERNEL
+        vbroadcastsd    -12 * SIZE(AO), %ymm0
+# else
+	vmovups 	-12 * SIZE(AO), %ymm0
+# endif
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4
+	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm8
+	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm12
+# if defined BROADCASTKERNEL
+        vbroadcastsd    -11 * SIZE(AO), %ymm0
+# else
+	vpermilpd	$ 0x05, %ymm0  , %ymm0
+# endif
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5
+	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm9
+	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm13
+# if defined BROADCASTKERNEL
+        vbroadcastsd    -10 * SIZE(AO), %ymm0
+# else
+	vpermpd		$ 0x1b, %ymm0  , %ymm0
+# endif
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6
+	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm10
+
+	addq		$ 8*SIZE, AO
+	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm14
+# if defined BROADCASTKERNEL
+        vbroadcastsd    -17 * SIZE(AO), %ymm0
+# else
+	vpermilpd	$ 0x05, %ymm0  , %ymm0
+# endif
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7
+	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm11
+	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm15
+	addq		$ 12*SIZE, BO
+.endm
+
+.macro KERNEL4x12_SUB
+	vmovups		-12 * SIZE(BO), %ymm1
+# if defined BROADCASTKERNEL
+        vbroadcastsd    -16 * SIZE(AO), %ymm0
+# else
+	vmovups 	-16 * SIZE(AO), %ymm0
+# endif
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4
+	vmovups		 -8 * SIZE(BO), %ymm2
+	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm8
+	vmovups		 -4 * SIZE(BO), %ymm3
+	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm12
+# if defined BROADCASTKERNEL
+        vbroadcastsd    -15 * SIZE(AO), %ymm0
+# else
+	vpermilpd	$ 0x05, %ymm0  , %ymm0
+# endif
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5
+	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm9
+	addq		$ 12*SIZE, BO
+	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm13
+# if defined BROADCASTKERNEL
+        vbroadcastsd    -14 * SIZE(AO), %ymm0
+# else
+	vpermpd		$ 0x1b, %ymm0  , %ymm0
+# endif
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6
+	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm10
+	addq		$ 4*SIZE, AO
+	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm14
+# if defined BROADCASTKERNEL
+        vbroadcastsd    -17 * SIZE(AO), %ymm0
+# else
+	vpermilpd	$ 0x05, %ymm0  , %ymm0
+# endif
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7
+	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm11
+	vfmadd231pd  	%ymm0 ,%ymm3  , %ymm15
+
+.endm
+
+
+.macro SAVE4x12
+
+        prefetcht0      BUFFER1
+	vbroadcastsd	ALPHA, %ymm0
+
+	vmulpd	%ymm0 , %ymm4 , %ymm4
+	vmulpd	%ymm0 , %ymm5 , %ymm5
+	vmulpd	%ymm0 , %ymm6 , %ymm6
+	vmulpd	%ymm0 , %ymm7 , %ymm7
+        prefetcht0      64 + BUFFER1
+	vmulpd	%ymm0 , %ymm8 , %ymm8
+	vmulpd	%ymm0 , %ymm9 , %ymm9
+	vmulpd	%ymm0 , %ymm10, %ymm10
+	vmulpd	%ymm0 , %ymm11, %ymm11
+#if B_PR1 > 32
+        prefetcht0      128 + BUFFER1
+#endif
+	vmulpd	%ymm0 , %ymm12, %ymm12
+	vmulpd	%ymm0 , %ymm13, %ymm13
+	vmulpd	%ymm0 , %ymm14, %ymm14
+	vmulpd	%ymm0 , %ymm15, %ymm15
+#if B_PR1 > 96
+        prefetcht0      192 + BUFFER1
+#endif
+
+#if defined BROADCASTKERNEL
+        vperm2f128 $ 0x20 , %ymm6, %ymm4 , %ymm0
+        vperm2f128 $ 0x20 , %ymm7, %ymm5 , %ymm1
+        vperm2f128 $ 0x31 , %ymm6, %ymm4 , %ymm2
+        vperm2f128 $ 0x31 , %ymm7, %ymm5 , %ymm3
+#else
+	vpermilpd $ 0x05 , %ymm5, %ymm5
+	vpermilpd $ 0x05 , %ymm7, %ymm7
+#endif
+
+#if B_PR1 > 160
+        prefetcht0      256 + BUFFER1
+#endif
+
+#if defined BROADCASTKERNEL
+        vunpcklpd %ymm1, %ymm0, %ymm4
+        vunpckhpd %ymm1, %ymm0, %ymm5
+        vunpcklpd %ymm3, %ymm2, %ymm6
+        vunpckhpd %ymm3, %ymm2, %ymm7
+#else
+	vblendpd $ 0x0a, %ymm5, %ymm4, %ymm0
+	vblendpd $ 0x05, %ymm5, %ymm4, %ymm1
+	vblendpd $ 0x0a, %ymm7, %ymm6, %ymm2
+	vblendpd $ 0x05, %ymm7, %ymm6, %ymm3
+#endif
+
+#if B_PR1 > 224
+        prefetcht0      320 + BUFFER1
+#endif
+
+#ifndef BROADCASTKERNEL
+	vperm2f128 $ 0x01 , %ymm2, %ymm2 , %ymm2
+	vperm2f128 $ 0x01 , %ymm3, %ymm3 , %ymm3
+#endif
+
+#if B_PR1 > 288
+        prefetcht0      384 + BUFFER1
+#endif
+
+#ifndef BROADCASTKERNEL
+	vblendpd $ 0x03, %ymm0, %ymm2 , %ymm4
+	vblendpd $ 0x03, %ymm1, %ymm3 , %ymm5
+	vblendpd $ 0x03, %ymm2, %ymm0 , %ymm6
+	vblendpd $ 0x03, %ymm3, %ymm1 , %ymm7
+#endif
+
+#if B_PR1 > 352
+        prefetcht0      448 + BUFFER1
+#endif
+        leaq    (CO1, LDC, 2), %rax     
+	
+#if B_PR1 > 416
+        prefetcht0      512 + BUFFER1
+#endif
+
+#if !defined(TRMMKERNEL)
+
+	vaddpd 	                (CO1), %ymm4, %ymm4
+	vaddpd 	           (CO1, LDC), %ymm5, %ymm5
+	vaddpd 	               (%rax), %ymm6, %ymm6
+	vaddpd 	          (%rax, LDC), %ymm7, %ymm7
+
+#endif
+
+	vmovups	%ymm4 ,  	(CO1)
+	vmovups	%ymm5 ,  	(CO1, LDC)
+	vmovups	%ymm6 ,  	(%rax)
+	vmovups	%ymm7 ,  	(%rax, LDC)
+
+	prefetcht1	56(CO1)
+	prefetcht1	56(CO1,LDC)
+	prefetcht1	56(%rax)
+	prefetcht1	56(%rax,LDC)
+
+#if defined BROADCASTKERNEL
+        vperm2f128 $ 0x20 , %ymm10, %ymm8 , %ymm0
+        vperm2f128 $ 0x20 , %ymm11, %ymm9 , %ymm1
+        vperm2f128 $ 0x31 , %ymm10, %ymm8 , %ymm2
+        vperm2f128 $ 0x31 , %ymm11, %ymm9 , %ymm3
+        vunpcklpd %ymm1, %ymm0, %ymm4
+        vunpckhpd %ymm1, %ymm0, %ymm5
+        vunpcklpd %ymm3, %ymm2, %ymm6
+        vunpckhpd %ymm3, %ymm2, %ymm7
+#else
+	vpermilpd $ 0x05 , %ymm9, %ymm9
+	vpermilpd $ 0x05 , %ymm11, %ymm11
+
+	vblendpd $ 0x0a, %ymm9, %ymm8, %ymm0
+	vblendpd $ 0x05, %ymm9, %ymm8, %ymm1
+	vblendpd $ 0x0a, %ymm11, %ymm10, %ymm2
+	vblendpd $ 0x05, %ymm11, %ymm10, %ymm3
+
+	vperm2f128 $ 0x01 , %ymm2, %ymm2 , %ymm2
+	vperm2f128 $ 0x01 , %ymm3, %ymm3 , %ymm3
+
+	vblendpd $ 0x03, %ymm0, %ymm2 , %ymm4
+	vblendpd $ 0x03, %ymm1, %ymm3 , %ymm5
+	vblendpd $ 0x03, %ymm2, %ymm0 , %ymm6
+	vblendpd $ 0x03, %ymm3, %ymm1 , %ymm7
+#endif
+
+	leaq	(%rax, LDC, 2), %rax
+	leaq	(%rax, LDC, 2), %rbp
+
+#if !defined(TRMMKERNEL)
+
+	vaddpd 	                (%rax), %ymm4, %ymm4
+	vaddpd 	           (%rax, LDC), %ymm5, %ymm5
+	vaddpd 	                (%rbp), %ymm6, %ymm6
+	vaddpd 	           (%rbp, LDC), %ymm7, %ymm7
+
+#endif
+
+	vmovups	%ymm4 ,  	(%rax)
+	vmovups	%ymm5 ,  	(%rax, LDC)
+	vmovups	%ymm6 ,  	(%rbp)
+	vmovups	%ymm7 ,  	(%rbp, LDC)
+
+	prefetcht1	56(%rax)
+	prefetcht1	56(%rax,LDC)
+	prefetcht1	56(%rbp)
+	prefetcht1	56(%rbp,LDC)
+
+#if defined BROADCASTKERNEL
+        vperm2f128 $ 0x20 , %ymm14, %ymm12 , %ymm0
+        vperm2f128 $ 0x20 , %ymm15, %ymm13 , %ymm1
+        vperm2f128 $ 0x31 , %ymm14, %ymm12 , %ymm2
+        vperm2f128 $ 0x31 , %ymm15, %ymm13 , %ymm3
+        vunpcklpd %ymm1, %ymm0, %ymm4
+        vunpckhpd %ymm1, %ymm0, %ymm5
+        vunpcklpd %ymm3, %ymm2, %ymm6
+        vunpckhpd %ymm3, %ymm2, %ymm7
+#else
+	vpermilpd $ 0x05 , %ymm13, %ymm13
+	vpermilpd $ 0x05 , %ymm15, %ymm15
+
+	vblendpd $ 0x0a, %ymm13, %ymm12, %ymm0
+	vblendpd $ 0x05, %ymm13, %ymm12, %ymm1
+	vblendpd $ 0x0a, %ymm15, %ymm14, %ymm2
+	vblendpd $ 0x05, %ymm15, %ymm14, %ymm3
+
+	vperm2f128 $ 0x01 , %ymm2, %ymm2 , %ymm2
+	vperm2f128 $ 0x01 , %ymm3, %ymm3 , %ymm3
+
+	vblendpd $ 0x03, %ymm0, %ymm2 , %ymm4
+	vblendpd $ 0x03, %ymm1, %ymm3 , %ymm5
+	vblendpd $ 0x03, %ymm2, %ymm0 , %ymm6
+	vblendpd $ 0x03, %ymm3, %ymm1 , %ymm7
+#endif
+
+	leaq	(%rax, LDC, 4), %rax
+	leaq	(%rbp, LDC, 4), %rbp
+
+#if !defined(TRMMKERNEL)
+
+	vaddpd 	                (%rax), %ymm4, %ymm4
+	vaddpd 	           (%rax, LDC), %ymm5, %ymm5
+	vaddpd 	                (%rbp), %ymm6, %ymm6
+	vaddpd 	           (%rbp, LDC), %ymm7, %ymm7
+
+#endif
+
+	vmovups	%ymm4 ,  	(%rax)
+	vmovups	%ymm5 ,  	(%rax, LDC)
+	vmovups	%ymm6 ,  	(%rbp)
+	vmovups	%ymm7 ,  	(%rbp, LDC)
+
+	prefetcht1	56(%rax)
+	prefetcht1	56(%rax,LDC)
+	prefetcht1	56(%rbp)
+	prefetcht1	56(%rbp,LDC)
+
+	addq	$ 4*SIZE, CO1
+.endm
+
+/******************************************************************************************/
+
+.macro INIT2x12
+
+	vxorpd		%xmm4 , %xmm4 , %xmm4
+	vxorpd		%xmm5 , %xmm5 , %xmm5
+	vxorpd		%xmm6 , %xmm6 , %xmm6
+	vxorpd		%xmm7 , %xmm7 , %xmm7
+	vxorpd		%xmm8 , %xmm8 , %xmm8
+	vxorpd		%xmm9 , %xmm9 , %xmm9
+	vxorpd		%xmm10, %xmm10, %xmm10
+	vxorpd		%xmm11, %xmm11, %xmm11
+	vxorpd		%xmm12, %xmm12, %xmm12
+	vxorpd		%xmm13, %xmm13, %xmm13
+	vxorpd		%xmm14, %xmm14, %xmm14
+	vxorpd		%xmm15, %xmm15, %xmm15
+
+.endm
+
+.macro KERNEL2x12_SUB
+	vmovups 	-16 * SIZE(AO), %xmm0
+	vmovddup	-12 * SIZE(BO), %xmm1
+	vmovddup	-11 * SIZE(BO), %xmm2
+	vmovddup	-10 * SIZE(BO), %xmm3
+	vfmadd231pd  	%xmm0 ,%xmm1  , %xmm4
+	vmovddup	 -9 * SIZE(BO), %xmm1
+	vfmadd231pd  	%xmm0 ,%xmm2  , %xmm5
+	vmovddup	 -8 * SIZE(BO), %xmm2
+	vfmadd231pd  	%xmm0 ,%xmm3  , %xmm6
+	vmovddup	 -7 * SIZE(BO), %xmm3
+	vfmadd231pd  	%xmm0 ,%xmm1  , %xmm7
+	vmovddup	 -6 * SIZE(BO), %xmm1
+	vfmadd231pd  	%xmm0 ,%xmm2  , %xmm8
+	vmovddup	 -5 * SIZE(BO), %xmm2
+	vfmadd231pd  	%xmm0 ,%xmm3  , %xmm9
+	vmovddup	 -4 * SIZE(BO), %xmm3
+	vfmadd231pd  	%xmm0 ,%xmm1  , %xmm10
+	vmovddup	 -3 * SIZE(BO), %xmm1
+	vfmadd231pd  	%xmm0 ,%xmm2  , %xmm11
+	vmovddup	 -2 * SIZE(BO), %xmm2
+	vfmadd231pd  	%xmm0 ,%xmm3  , %xmm12
+	vmovddup	 -1 * SIZE(BO), %xmm3
+	vfmadd231pd  	%xmm0 ,%xmm1  , %xmm13
+	addq		$ 12*SIZE, BO
+	vfmadd231pd  	%xmm0 ,%xmm2  , %xmm14
+	addq		$ 2*SIZE, AO
+	vfmadd231pd  	%xmm0 ,%xmm3  , %xmm15
+
+.endm
+
+.macro SAVE2x12
+
+	vmovddup	ALPHA, %xmm0
+
+	vmulpd	%xmm0 , %xmm4 , %xmm4
+	vmulpd	%xmm0 , %xmm5 , %xmm5
+	vmulpd	%xmm0 , %xmm6 , %xmm6
+	vmulpd	%xmm0 , %xmm7 , %xmm7
+
+	vmulpd	%xmm0 , %xmm8 , %xmm8
+	vmulpd	%xmm0 , %xmm9 , %xmm9
+	vmulpd	%xmm0 , %xmm10, %xmm10
+	vmulpd	%xmm0 , %xmm11, %xmm11
+
+	vmulpd	%xmm0 , %xmm12, %xmm12
+	vmulpd	%xmm0 , %xmm13, %xmm13
+	vmulpd	%xmm0 , %xmm14, %xmm14
+	vmulpd	%xmm0 , %xmm15, %xmm15
+
+
+        leaq    (CO1, LDC, 2), %rax     
+	
+
+#if !defined(TRMMKERNEL)
+
+	vaddpd 	                (CO1), %xmm4, %xmm4
+	vaddpd 	           (CO1, LDC), %xmm5, %xmm5
+	vaddpd 	               (%rax), %xmm6, %xmm6
+	vaddpd 	          (%rax, LDC), %xmm7, %xmm7
+
+#endif
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm5 ,  	(CO1, LDC)
+	vmovups	%xmm6 ,  	(%rax)
+	vmovups	%xmm7 ,  	(%rax, LDC)
+
+
+	leaq	(%rax, LDC, 2), %rax
+	leaq	(%rax, LDC, 2), %rbp
+
+#if !defined(TRMMKERNEL)
+
+	vaddpd 	                (%rax), %xmm8 , %xmm4
+	vaddpd 	           (%rax, LDC), %xmm9 , %xmm5
+	vaddpd 	                (%rbp), %xmm10, %xmm6
+	vaddpd 	           (%rbp, LDC), %xmm11, %xmm7
+
+#endif
+
+	vmovups	%xmm4 ,  	(%rax)
+	vmovups	%xmm5 ,  	(%rax, LDC)
+	vmovups	%xmm6 ,  	(%rbp)
+	vmovups	%xmm7 ,  	(%rbp, LDC)
+
+
+	leaq	(%rax, LDC, 4), %rax
+	leaq	(%rbp, LDC, 4), %rbp
+
+#if !defined(TRMMKERNEL)
+
+	vaddpd 	                (%rax), %xmm12, %xmm4
+	vaddpd 	           (%rax, LDC), %xmm13, %xmm5
+	vaddpd 	                (%rbp), %xmm14, %xmm6
+	vaddpd 	           (%rbp, LDC), %xmm15, %xmm7
+
+#endif
+
+	vmovups	%xmm4 ,  	(%rax)
+	vmovups	%xmm5 ,  	(%rax, LDC)
+	vmovups	%xmm6 ,  	(%rbp)
+	vmovups	%xmm7 ,  	(%rbp, LDC)
+
+	addq	$ 2*SIZE, CO1
+.endm
+
+
+/******************************************************************************************/
+
+.macro INIT1x12
+
+	vxorpd		%xmm4 , %xmm4 , %xmm4
+	vxorpd		%xmm5 , %xmm5 , %xmm5
+	vxorpd		%xmm6 , %xmm6 , %xmm6
+	vxorpd		%xmm7 , %xmm7 , %xmm7
+	vxorpd		%xmm8 , %xmm8 , %xmm8
+	vxorpd		%xmm9 , %xmm9 , %xmm9
+	vxorpd		%xmm10, %xmm10, %xmm10
+	vxorpd		%xmm11, %xmm11, %xmm11
+	vxorpd		%xmm12, %xmm12, %xmm12
+	vxorpd		%xmm13, %xmm13, %xmm13
+	vxorpd		%xmm14, %xmm14, %xmm14
+	vxorpd		%xmm15, %xmm15, %xmm15
+
+.endm
+
+.macro KERNEL1x12_SUB
+	vmovsd 	-16 * SIZE(AO), %xmm0
+	vmovsd	-12 * SIZE(BO), %xmm1
+	vmovsd	-11 * SIZE(BO), %xmm2
+	vmovsd	-10 * SIZE(BO), %xmm3
+	vfmadd231sd  	%xmm0 ,%xmm1  , %xmm4
+	vmovsd	 -9 * SIZE(BO), %xmm1
+	vfmadd231sd  	%xmm0 ,%xmm2  , %xmm5
+	vmovsd	 -8 * SIZE(BO), %xmm2
+	vfmadd231sd  	%xmm0 ,%xmm3  , %xmm6
+	vmovsd	 -7 * SIZE(BO), %xmm3
+	vfmadd231sd  	%xmm0 ,%xmm1  , %xmm7
+	vmovsd	 -6 * SIZE(BO), %xmm1
+	vfmadd231sd  	%xmm0 ,%xmm2  , %xmm8
+	vmovsd	 -5 * SIZE(BO), %xmm2
+	vfmadd231sd  	%xmm0 ,%xmm3  , %xmm9
+	vmovsd	 -4 * SIZE(BO), %xmm3
+	vfmadd231sd  	%xmm0 ,%xmm1  , %xmm10
+	vmovsd	 -3 * SIZE(BO), %xmm1
+	vfmadd231sd  	%xmm0 ,%xmm2  , %xmm11
+	vmovsd	 -2 * SIZE(BO), %xmm2
+	vfmadd231sd  	%xmm0 ,%xmm3  , %xmm12
+	vmovsd	 -1 * SIZE(BO), %xmm3
+	vfmadd231sd  	%xmm0 ,%xmm1  , %xmm13
+	addq		$ 12*SIZE, BO
+	vfmadd231sd  	%xmm0 ,%xmm2  , %xmm14
+	addq		$ 1*SIZE, AO
+	vfmadd231sd  	%xmm0 ,%xmm3  , %xmm15
+
+.endm
+
+.macro SAVE1x12
+
+	vmovsd	ALPHA, %xmm0
+
+	vmulsd	%xmm0 , %xmm4 , %xmm4
+	vmulsd	%xmm0 , %xmm5 , %xmm5
+	vmulsd	%xmm0 , %xmm6 , %xmm6
+	vmulsd	%xmm0 , %xmm7 , %xmm7
+
+	vmulsd	%xmm0 , %xmm8 , %xmm8
+	vmulsd	%xmm0 , %xmm9 , %xmm9
+	vmulsd	%xmm0 , %xmm10, %xmm10
+	vmulsd	%xmm0 , %xmm11, %xmm11
+
+	vmulsd	%xmm0 , %xmm12, %xmm12
+	vmulsd	%xmm0 , %xmm13, %xmm13
+	vmulsd	%xmm0 , %xmm14, %xmm14
+	vmulsd	%xmm0 , %xmm15, %xmm15
+
+
+        leaq    (CO1, LDC, 2), %rax     
+	
+
+#if !defined(TRMMKERNEL)
+
+	vaddsd 	                (CO1), %xmm4, %xmm4
+	vaddsd 	           (CO1, LDC), %xmm5, %xmm5
+	vaddsd 	               (%rax), %xmm6, %xmm6
+	vaddsd 	          (%rax, LDC), %xmm7, %xmm7
+
+#endif
+
+	vmovsd	%xmm4 ,  	(CO1)
+	vmovsd	%xmm5 ,  	(CO1, LDC)
+	vmovsd	%xmm6 ,  	(%rax)
+	vmovsd	%xmm7 ,  	(%rax, LDC)
+
+
+	leaq	(%rax, LDC, 2), %rax
+	leaq	(%rax, LDC, 2), %rbp
+
+#if !defined(TRMMKERNEL)
+
+	vaddsd 	                (%rax), %xmm8 , %xmm4
+	vaddsd 	           (%rax, LDC), %xmm9 , %xmm5
+	vaddsd 	                (%rbp), %xmm10, %xmm6
+	vaddsd 	           (%rbp, LDC), %xmm11, %xmm7
+
+#endif
+
+	vmovsd	%xmm4 ,  	(%rax)
+	vmovsd	%xmm5 ,  	(%rax, LDC)
+	vmovsd	%xmm6 ,  	(%rbp)
+	vmovsd	%xmm7 ,  	(%rbp, LDC)
+
+
+	leaq	(%rax, LDC, 4), %rax
+	leaq	(%rbp, LDC, 4), %rbp
+
+#if !defined(TRMMKERNEL)
+
+	vaddsd 	                (%rax), %xmm12, %xmm4
+	vaddsd 	           (%rax, LDC), %xmm13, %xmm5
+	vaddsd 	                (%rbp), %xmm14, %xmm6
+	vaddsd 	           (%rbp, LDC), %xmm15, %xmm7
+
+#endif
+
+	vmovsd	%xmm4 ,  	(%rax)
+	vmovsd	%xmm5 ,  	(%rax, LDC)
+	vmovsd	%xmm6 ,  	(%rbp)
+	vmovsd	%xmm7 ,  	(%rbp, LDC)
+
+	addq	$ 1*SIZE, CO1
+.endm
+
+
+
+
+/******************************************************************************************/
+
+
+.macro INIT4x8
+
+	vxorpd		%ymm4 , %ymm4 , %ymm4
+	vxorpd		%ymm5 , %ymm5 , %ymm5
+	vxorpd		%ymm6 , %ymm6 , %ymm6
+	vxorpd		%ymm7 , %ymm7 , %ymm7
+	vxorpd		%ymm8 , %ymm8 , %ymm8
+	vxorpd		%ymm9 , %ymm9 , %ymm9
+	vxorpd		%ymm10, %ymm10, %ymm10
+	vxorpd		%ymm11, %ymm11, %ymm11
+
+.endm
+
+.macro KERNEL4x8_I
+	vmovups		-12 * SIZE(BO), %ymm1
+#if defined BROADCASTKERNEL
+        vbroadcastsd    -16 * SIZE(AO), %ymm0
+#else
+	vmovups 	-16 * SIZE(AO), %ymm0
+#endif
+	vmovups		 -8 * SIZE(BO), %ymm2
+	vmulpd  	%ymm0 ,%ymm1  , %ymm4
+	vmulpd  	%ymm0 ,%ymm2  , %ymm8
+#if defined BROADCASTKERNEL
+        vbroadcastsd    -15 * SIZE(AO), %ymm0
+#else
+	vpermilpd	$ 0x05, %ymm0  , %ymm0
+#endif
+	vmulpd  	%ymm0 ,%ymm1  , %ymm5
+	vmulpd  	%ymm0 ,%ymm2  , %ymm9
+#if defined BROADCASTKERNEL
+        vbroadcastsd    -14 * SIZE(AO), %ymm0
+#else
+	vpermpd		$ 0x1b, %ymm0  , %ymm0
+#endif
+	vmulpd  	%ymm0 ,%ymm1  , %ymm6
+	vmulpd  	%ymm0 ,%ymm2  , %ymm10
+
+	addq		$  8*SIZE, BO
+#if defined BROADCASTKERNEL
+        vbroadcastsd    -13 * SIZE(AO), %ymm0
+#else
+	vpermilpd	$ 0x05, %ymm0  , %ymm0
+#endif
+	vmulpd  	%ymm0 ,%ymm1  , %ymm7
+	vmovups		-12 * SIZE(BO), %ymm1
+	vmulpd  	%ymm0 ,%ymm2  , %ymm11
+	vmovups		 -8 * SIZE(BO), %ymm2
+
+.endm
+
+.macro KERNEL4x8_M1
+	prefetcht0	A_PR1(AO)
+#if defined BROADCASTKERNEL
+        vbroadcastsd    -16 * SIZE(AO), %ymm0
+#else
+	vmovups 	-16 * SIZE(AO), %ymm0
+#endif
+	prefetcht0	B_PR1(BO)
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4
+	prefetcht0	B_PR1+64(BO)
+	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm8
+#if defined BROADCASTKERNEL
+        vbroadcastsd    -15 * SIZE(AO), %ymm0
+#else
+	vpermilpd	$ 0x05, %ymm0  , %ymm0
+#endif
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5
+	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm9
+#if defined BROADCASTKERNEL
+        vbroadcastsd    -14 * SIZE(AO), %ymm0
+#else
+	vpermpd		$ 0x1b, %ymm0  , %ymm0
+#endif
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6
+	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm10
+#if defined BROADCASTKERNEL
+        vbroadcastsd    -13 * SIZE(AO), %ymm0
+#else
+	vpermilpd	$ 0x05, %ymm0  , %ymm0
+#endif
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7
+	vmovups		-12 * SIZE(BO), %ymm1
+	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm11
+	vmovups		 -8 * SIZE(BO), %ymm2
+
+.endm
+
+.macro KERNEL4x8_M2
+#if defined BROADCASTKERNEL
+        vbroadcastsd    -12 * SIZE(AO), %ymm0
+#else
+	vmovups 	-12 * SIZE(AO), %ymm0
+#endif
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4
+	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm8
+#if defined BROADCASTKERNEL
+        vbroadcastsd    -11 * SIZE(AO), %ymm0
+#else
+	vpermilpd	$ 0x05, %ymm0  , %ymm0
+#endif
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5
+	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm9
+#if defined BROADCASTKERNEL
+        vbroadcastsd    -10 * SIZE(AO), %ymm0
+#else
+	vpermpd		$ 0x1b, %ymm0  , %ymm0
+#endif
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6
+	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm10
+
+	addq		$ 8*SIZE, AO
+#if defined BROADCASTKERNEL
+        vbroadcastsd    -17 * SIZE(AO), %ymm0
+#else
+	vpermilpd	$ 0x05, %ymm0  , %ymm0
+#endif
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7
+	vmovups		 -4 * SIZE(BO), %ymm1
+	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm11
+	vmovups		  0 * SIZE(BO), %ymm2
+	addq		$ 16*SIZE, BO
+.endm
+
+
+.macro KERNEL4x8_E
+#if defined BROADCASTKERNEL
+        vbroadcastsd    -12 * SIZE(AO), %ymm0
+#else
+	vmovups 	-12 * SIZE(AO), %ymm0
+#endif
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4
+	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm8
+#if defined BROADCASTKERNEL
+        vbroadcastsd    -11 * SIZE(AO), %ymm0
+#else
+	vpermilpd	$ 0x05, %ymm0  , %ymm0
+#endif
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5
+	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm9
+#if defined BROADCASTKERNEL
+        vbroadcastsd    -10 * SIZE(AO), %ymm0
+#else
+	vpermpd		$ 0x1b, %ymm0  , %ymm0
+#endif
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6
+	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm10
+
+	addq		$ 8*SIZE, AO
+#if defined BROADCASTKERNEL
+        vbroadcastsd    -17 * SIZE(AO), %ymm0
+#else
+	vpermilpd	$ 0x05, %ymm0  , %ymm0
+#endif
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7
+	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm11
+	addq		$  8*SIZE, BO
+.endm
+
+.macro KERNEL4x8_SUB
+	vmovups		-12 * SIZE(BO), %ymm1
+#if defined BROADCASTKERNEL
+        vbroadcastsd    -16 * SIZE(AO), %ymm0
+#else
+	vmovups 	-16 * SIZE(AO), %ymm0
+#endif
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4
+	vmovups		 -8 * SIZE(BO), %ymm2
+	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm8
+#if defined BROADCASTKERNEL
+        vbroadcastsd    -15 * SIZE(AO), %ymm0
+#else
+	vpermilpd	$ 0x05, %ymm0  , %ymm0
+#endif
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5
+	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm9
+	addq		$  8*SIZE, BO
+#if defined BROADCASTKERNEL
+        vbroadcastsd    -14 * SIZE(AO), %ymm0
+#else
+	vpermpd		$ 0x1b, %ymm0  , %ymm0
+#endif
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6
+	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm10
+	addq		$ 4*SIZE, AO
+#if defined BROADCASTKERNEL
+        vbroadcastsd    -17 * SIZE(AO), %ymm0
+#else
+	vpermilpd	$ 0x05, %ymm0  , %ymm0
+#endif
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7
+	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm11
+
+.endm
+
+
+.macro SAVE4x8
+
+	vbroadcastsd	ALPHA, %ymm0
+
+	vmulpd	%ymm0 , %ymm4 , %ymm4
+	vmulpd	%ymm0 , %ymm5 , %ymm5
+	vmulpd	%ymm0 , %ymm6 , %ymm6
+	vmulpd	%ymm0 , %ymm7 , %ymm7
+
+	vmulpd	%ymm0 , %ymm8 , %ymm8
+	vmulpd	%ymm0 , %ymm9 , %ymm9
+	vmulpd	%ymm0 , %ymm10, %ymm10
+	vmulpd	%ymm0 , %ymm11, %ymm11
+
+#if defined BROADCASTKERNEL
+        vperm2f128 $ 0x20 , %ymm6, %ymm4 , %ymm0
+        vperm2f128 $ 0x20 , %ymm7, %ymm5 , %ymm1
+        vperm2f128 $ 0x31 , %ymm6, %ymm4 , %ymm2
+        vperm2f128 $ 0x31 , %ymm7, %ymm5 , %ymm3
+        vunpcklpd %ymm1, %ymm0, %ymm4
+        vunpckhpd %ymm1, %ymm0, %ymm5
+        vunpcklpd %ymm3, %ymm2, %ymm6
+        vunpckhpd %ymm3, %ymm2, %ymm7
+#else
+	vpermilpd $ 0x05 , %ymm5, %ymm5
+	vpermilpd $ 0x05 , %ymm7, %ymm7
+
+	vblendpd $ 0x0a, %ymm5, %ymm4, %ymm0
+	vblendpd $ 0x05, %ymm5, %ymm4, %ymm1
+	vblendpd $ 0x0a, %ymm7, %ymm6, %ymm2
+	vblendpd $ 0x05, %ymm7, %ymm6, %ymm3
+
+	vperm2f128 $ 0x01 , %ymm2, %ymm2 , %ymm2
+	vperm2f128 $ 0x01 , %ymm3, %ymm3 , %ymm3
+
+	vblendpd $ 0x03, %ymm0, %ymm2 , %ymm4
+	vblendpd $ 0x03, %ymm1, %ymm3 , %ymm5
+	vblendpd $ 0x03, %ymm2, %ymm0 , %ymm6
+	vblendpd $ 0x03, %ymm3, %ymm1 , %ymm7
+#endif
+
+        leaq    (CO1, LDC, 2), %rax     
+	
+
+#if !defined(TRMMKERNEL)
+
+	vaddpd 	                (CO1), %ymm4, %ymm4
+	vaddpd 	           (CO1, LDC), %ymm5, %ymm5
+	vaddpd 	               (%rax), %ymm6, %ymm6
+	vaddpd 	          (%rax, LDC), %ymm7, %ymm7
+
+#endif
+
+	vmovups	%ymm4 ,  	(CO1)
+	vmovups	%ymm5 ,  	(CO1, LDC)
+	vmovups	%ymm6 ,  	(%rax)
+	vmovups	%ymm7 ,  	(%rax, LDC)
+
+	prefetcht0	56(CO1)
+	prefetcht0	56(CO1,LDC)
+	prefetcht0	56(%rax)
+	prefetcht0	56(%rax,LDC)
+
+#if defined BROADCASTKERNEL
+        vperm2f128 $ 0x20 , %ymm10, %ymm8 , %ymm0
+        vperm2f128 $ 0x20 , %ymm11, %ymm9 , %ymm1
+        vperm2f128 $ 0x31 , %ymm10, %ymm8 , %ymm2
+        vperm2f128 $ 0x31 , %ymm11, %ymm9 , %ymm3
+        vunpcklpd %ymm1, %ymm0, %ymm4
+        vunpckhpd %ymm1, %ymm0, %ymm5
+        vunpcklpd %ymm3, %ymm2, %ymm6
+        vunpckhpd %ymm3, %ymm2, %ymm7
+#else
+	vpermilpd $ 0x05 , %ymm9 , %ymm9
+	vpermilpd $ 0x05 , %ymm11, %ymm11
+
+	vblendpd $ 0x0a, %ymm9 , %ymm8 , %ymm0
+	vblendpd $ 0x05, %ymm9 , %ymm8 , %ymm1
+	vblendpd $ 0x0a, %ymm11, %ymm10, %ymm2
+	vblendpd $ 0x05, %ymm11, %ymm10, %ymm3
+
+	vperm2f128 $ 0x01 , %ymm2, %ymm2 , %ymm2
+	vperm2f128 $ 0x01 , %ymm3, %ymm3 , %ymm3
+
+	vblendpd $ 0x03, %ymm0, %ymm2 , %ymm4
+	vblendpd $ 0x03, %ymm1, %ymm3 , %ymm5
+	vblendpd $ 0x03, %ymm2, %ymm0 , %ymm6
+	vblendpd $ 0x03, %ymm3, %ymm1 , %ymm7
+#endif
+
+	leaq	(%rax, LDC, 2), %rax
+	leaq	(%rax, LDC, 2), %rbp
+
+#if !defined(TRMMKERNEL)
+
+	vaddpd 	                (%rax), %ymm4, %ymm4
+	vaddpd 	           (%rax, LDC), %ymm5, %ymm5
+	vaddpd 	                (%rbp), %ymm6, %ymm6
+	vaddpd 	           (%rbp, LDC), %ymm7, %ymm7
+
+#endif
+
+	vmovups	%ymm4 ,  	(%rax)
+	vmovups	%ymm5 ,  	(%rax, LDC)
+	vmovups	%ymm6 ,  	(%rbp)
+	vmovups	%ymm7 ,  	(%rbp, LDC)
+
+	prefetcht0	56(%rax)
+	prefetcht0	56(%rax,LDC)
+	prefetcht0	56(%rbp)
+	prefetcht0	56(%rbp,LDC)
+
+	addq	$ 4*SIZE, CO1
+.endm
+
+/******************************************************************************************/
+
+.macro INIT2x8
+
+	vxorpd		%xmm4 , %xmm4 , %xmm4
+	vxorpd		%xmm5 , %xmm5 , %xmm5
+	vxorpd		%xmm6 , %xmm6 , %xmm6
+	vxorpd		%xmm7 , %xmm7 , %xmm7
+	vxorpd		%xmm8 , %xmm8 , %xmm8
+	vxorpd		%xmm9 , %xmm9 , %xmm9
+	vxorpd		%xmm10, %xmm10, %xmm10
+	vxorpd		%xmm11, %xmm11, %xmm11
+
+.endm
+
+.macro KERNEL2x8_SUB
+	vmovups 	-16 * SIZE(AO), %xmm0
+	vmovddup	-12 * SIZE(BO), %xmm1
+	vmovddup	-11 * SIZE(BO), %xmm2
+	vmovddup	-10 * SIZE(BO), %xmm3
+	vfmadd231pd  	%xmm0 ,%xmm1  , %xmm4
+	vmovddup	 -9 * SIZE(BO), %xmm1
+	vfmadd231pd  	%xmm0 ,%xmm2  , %xmm5
+	vmovddup	 -8 * SIZE(BO), %xmm2
+	vfmadd231pd  	%xmm0 ,%xmm3  , %xmm6
+	vmovddup	 -7 * SIZE(BO), %xmm3
+	vfmadd231pd  	%xmm0 ,%xmm1  , %xmm7
+	vmovddup	 -6 * SIZE(BO), %xmm1
+	vfmadd231pd  	%xmm0 ,%xmm2  , %xmm8
+	vmovddup	 -5 * SIZE(BO), %xmm2
+	vfmadd231pd  	%xmm0 ,%xmm3  , %xmm9
+	vfmadd231pd  	%xmm0 ,%xmm1  , %xmm10
+	vfmadd231pd  	%xmm0 ,%xmm2  , %xmm11
+	addq		$  8*SIZE, BO
+	addq		$ 2*SIZE, AO
+
+.endm
+
+.macro SAVE2x8
+
+	vmovddup	ALPHA, %xmm0
+
+	vmulpd	%xmm0 , %xmm4 , %xmm4
+	vmulpd	%xmm0 , %xmm5 , %xmm5
+	vmulpd	%xmm0 , %xmm6 , %xmm6
+	vmulpd	%xmm0 , %xmm7 , %xmm7
+
+	vmulpd	%xmm0 , %xmm8 , %xmm8
+	vmulpd	%xmm0 , %xmm9 , %xmm9
+	vmulpd	%xmm0 , %xmm10, %xmm10
+	vmulpd	%xmm0 , %xmm11, %xmm11
+
+        leaq    (CO1, LDC, 2), %rax     
+	
+
+#if !defined(TRMMKERNEL)
+
+	vaddpd 	                (CO1), %xmm4, %xmm4
+	vaddpd 	           (CO1, LDC), %xmm5, %xmm5
+	vaddpd 	               (%rax), %xmm6, %xmm6
+	vaddpd 	          (%rax, LDC), %xmm7, %xmm7
+
+#endif
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm5 ,  	(CO1, LDC)
+	vmovups	%xmm6 ,  	(%rax)
+	vmovups	%xmm7 ,  	(%rax, LDC)
+
+
+	leaq	(%rax, LDC, 2), %rax
+	leaq	(%rax, LDC, 2), %rbp
+
+#if !defined(TRMMKERNEL)
+
+	vaddpd 	                (%rax), %xmm8 , %xmm4
+	vaddpd 	           (%rax, LDC), %xmm9 , %xmm5
+	vaddpd 	                (%rbp), %xmm10, %xmm6
+	vaddpd 	           (%rbp, LDC), %xmm11, %xmm7
+
+#endif
+
+	vmovups	%xmm4 ,  	(%rax)
+	vmovups	%xmm5 ,  	(%rax, LDC)
+	vmovups	%xmm6 ,  	(%rbp)
+	vmovups	%xmm7 ,  	(%rbp, LDC)
+
+	addq	$ 2*SIZE, CO1
+.endm
+
+
+/******************************************************************************************/
+
+.macro INIT1x8
+
+	vxorpd		%xmm4 , %xmm4 , %xmm4
+	vxorpd		%xmm5 , %xmm5 , %xmm5
+	vxorpd		%xmm6 , %xmm6 , %xmm6
+	vxorpd		%xmm7 , %xmm7 , %xmm7
+	vxorpd		%xmm8 , %xmm8 , %xmm8
+	vxorpd		%xmm9 , %xmm9 , %xmm9
+	vxorpd		%xmm10, %xmm10, %xmm10
+	vxorpd		%xmm11, %xmm11, %xmm11
+
+.endm
+
+.macro KERNEL1x8_SUB
+	vmovsd 	-16 * SIZE(AO), %xmm0
+	vmovsd	-12 * SIZE(BO), %xmm1
+	vmovsd	-11 * SIZE(BO), %xmm2
+	vmovsd	-10 * SIZE(BO), %xmm3
+	vfmadd231sd  	%xmm0 ,%xmm1  , %xmm4
+	vmovsd	 -9 * SIZE(BO), %xmm1
+	vfmadd231sd  	%xmm0 ,%xmm2  , %xmm5
+	vmovsd	 -8 * SIZE(BO), %xmm2
+	vfmadd231sd  	%xmm0 ,%xmm3  , %xmm6
+	vmovsd	 -7 * SIZE(BO), %xmm3
+	vfmadd231sd  	%xmm0 ,%xmm1  , %xmm7
+	vmovsd	 -6 * SIZE(BO), %xmm1
+	vfmadd231sd  	%xmm0 ,%xmm2  , %xmm8
+	vmovsd	 -5 * SIZE(BO), %xmm2
+	vfmadd231sd  	%xmm0 ,%xmm3  , %xmm9
+	vfmadd231sd  	%xmm0 ,%xmm1  , %xmm10
+	vfmadd231sd  	%xmm0 ,%xmm2  , %xmm11
+	addq		$  8*SIZE, BO
+	addq		$ 1*SIZE, AO
+
+.endm
+
+.macro SAVE1x8
+
+	vmovsd	ALPHA, %xmm0
+
+	vmulsd	%xmm0 , %xmm4 , %xmm4
+	vmulsd	%xmm0 , %xmm5 , %xmm5
+	vmulsd	%xmm0 , %xmm6 , %xmm6
+	vmulsd	%xmm0 , %xmm7 , %xmm7
+
+	vmulsd	%xmm0 , %xmm8 , %xmm8
+	vmulsd	%xmm0 , %xmm9 , %xmm9
+	vmulsd	%xmm0 , %xmm10, %xmm10
+	vmulsd	%xmm0 , %xmm11, %xmm11
+
+        leaq    (CO1, LDC, 2), %rax     
+	
+
+#if !defined(TRMMKERNEL)
+
+	vaddsd 	                (CO1), %xmm4, %xmm4
+	vaddsd 	           (CO1, LDC), %xmm5, %xmm5
+	vaddsd 	               (%rax), %xmm6, %xmm6
+	vaddsd 	          (%rax, LDC), %xmm7, %xmm7
+
+#endif
+
+	vmovsd	%xmm4 ,  	(CO1)
+	vmovsd	%xmm5 ,  	(CO1, LDC)
+	vmovsd	%xmm6 ,  	(%rax)
+	vmovsd	%xmm7 ,  	(%rax, LDC)
+
+
+	leaq	(%rax, LDC, 2), %rax
+	leaq	(%rax, LDC, 2), %rbp
+
+#if !defined(TRMMKERNEL)
+
+	vaddsd 	                (%rax), %xmm8 , %xmm4
+	vaddsd 	           (%rax, LDC), %xmm9 , %xmm5
+	vaddsd 	                (%rbp), %xmm10, %xmm6
+	vaddsd 	           (%rbp, LDC), %xmm11, %xmm7
+
+#endif
+
+	vmovsd	%xmm4 ,  	(%rax)
+	vmovsd	%xmm5 ,  	(%rax, LDC)
+	vmovsd	%xmm6 ,  	(%rbp)
+	vmovsd	%xmm7 ,  	(%rbp, LDC)
+
+	addq	$ 1*SIZE, CO1
+.endm
+
+
+
+
+
+/******************************************************************************************/
+
+.macro INIT4x4
+
+	vxorpd		%ymm4 , %ymm4 , %ymm4
+	vxorpd		%ymm5 , %ymm5 , %ymm5
+	vxorpd		%ymm6 , %ymm6 , %ymm6
+	vxorpd		%ymm7 , %ymm7 , %ymm7
+
+.endm
+
+.macro KERNEL4x4_I
+	prefetcht0	A_PR1(AO)
+	vmovups		-12 * SIZE(BO), %ymm1
+#if defined BROADCASTKERNEL
+        vbroadcastsd    -16 * SIZE(AO), %ymm0
+#else
+	vmovups 	-16 * SIZE(AO), %ymm0
+#endif
+	vmulpd  	%ymm0 ,%ymm1  , %ymm4
+#if defined BROADCASTKERNEL
+        vbroadcastsd    -15 * SIZE(AO), %ymm0
+#else
+	vpermilpd	$ 0x05, %ymm0  , %ymm0
+#endif
+	vmulpd  	%ymm0 ,%ymm1  , %ymm5
+#if defined BROADCASTKERNEL
+        vbroadcastsd    -14 * SIZE(AO), %ymm0
+#else
+	vpermpd		$ 0x1b, %ymm0  , %ymm0
+#endif
+	vmulpd  	%ymm0 ,%ymm1  , %ymm6
+
+	addq		$ 4*SIZE, BO
+#if defined BROADCASTKERNEL
+        vbroadcastsd    -13 * SIZE(AO), %ymm0
+#else
+	vpermilpd	$ 0x05, %ymm0  , %ymm0
+#endif
+	vmulpd  	%ymm0 ,%ymm1  , %ymm7
+	vmovups		-12 * SIZE(BO), %ymm1
+
+.endm
+
+.macro KERNEL4x4_M1
+	prefetcht0	A_PR1(AO)
+#if defined BROADCASTKERNEL
+        vbroadcastsd    -16 * SIZE(AO), %ymm0
+#else
+	vmovups 	-16 * SIZE(AO), %ymm0
+#endif
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4
+#if defined BROADCASTKERNEL
+        vbroadcastsd    -15 * SIZE(AO), %ymm0
+#else
+	vpermilpd	$ 0x05, %ymm0  , %ymm0
+#endif
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5
+#if defined BROADCASTKERNEL
+        vbroadcastsd    -14 * SIZE(AO), %ymm0
+#else
+	vpermpd		$ 0x1b, %ymm0  , %ymm0
+#endif
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6
+#if defined BROADCASTKERNEL
+        vbroadcastsd    -13 * SIZE(AO), %ymm0
+#else
+	vpermilpd	$ 0x05, %ymm0  , %ymm0
+#endif
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7
+	vmovups		-12 * SIZE(BO), %ymm1
+
+.endm
+
+.macro KERNEL4x4_M2
+#if defined BROADCASTKERNEL
+        vbroadcastsd    -12 * SIZE(AO), %ymm0
+#else
+	vmovups 	-12 * SIZE(AO), %ymm0
+#endif
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4
+#if defined BROADCASTKERNEL
+        vbroadcastsd    -11 * SIZE(AO), %ymm0
+#else
+	vpermilpd	$ 0x05, %ymm0  , %ymm0
+#endif
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5
+#if defined BROADCASTKERNEL
+        vbroadcastsd    -10 * SIZE(AO), %ymm0
+#else
+	vpermpd		$ 0x1b, %ymm0  , %ymm0
+#endif
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6
+
+	addq		$ 8*SIZE, AO
+#if defined BROADCASTKERNEL
+        vbroadcastsd    -17 * SIZE(AO), %ymm0
+#else
+	vpermilpd	$ 0x05, %ymm0  , %ymm0
+#endif
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7
+	vmovups		 -8 * SIZE(BO), %ymm1
+	addq		$ 8*SIZE, BO
+.endm
+
+
+.macro KERNEL4x4_E
+#if defined BROADCASTKERNEL
+        vbroadcastsd    -12 * SIZE(AO), %ymm0
+#else
+	vmovups 	-12 * SIZE(AO), %ymm0
+#endif
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4
+#if defined BROADCASTKERNEL
+        vbroadcastsd    -11 * SIZE(AO), %ymm0
+#else
+	vpermilpd	$ 0x05, %ymm0  , %ymm0
+#endif
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5
+#if defined BROADCASTKERNEL
+        vbroadcastsd    -10 * SIZE(AO), %ymm0
+#else
+	vpermpd		$ 0x1b, %ymm0  , %ymm0
+#endif
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6
+
+	addq		$ 8*SIZE, AO
+#if defined BROADCASTKERNEL
+        vbroadcastsd    -17 * SIZE(AO), %ymm0
+#else
+	vpermilpd	$ 0x05, %ymm0  , %ymm0
+#endif
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7
+	addq		$ 4*SIZE, BO
+.endm
+
+.macro KERNEL4x4_SUB
+	vmovups		-12 * SIZE(BO), %ymm1
+#if defined BROADCASTKERNEL
+        vbroadcastsd    -16 * SIZE(AO), %ymm0
+#else
+	vmovups 	-16 * SIZE(AO), %ymm0
+#endif
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm4
+#if defined BROADCASTKERNEL
+        vbroadcastsd    -15 * SIZE(AO), %ymm0
+#else
+	vpermilpd	$ 0x05, %ymm0  , %ymm0
+#endif
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm5
+	addq		$ 4*SIZE, BO
+#if defined BROADCASTKERNEL
+        vbroadcastsd    -14 * SIZE(AO), %ymm0
+#else
+	vpermpd		$ 0x1b, %ymm0  , %ymm0
+#endif
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm6
+	addq		$ 4*SIZE, AO
+#if defined BROADCASTKERNEL
+        vbroadcastsd    -17 * SIZE(AO), %ymm0
+#else
+	vpermilpd	$ 0x05, %ymm0  , %ymm0
+#endif
+	vfmadd231pd  	%ymm0 ,%ymm1  , %ymm7
+
+.endm
+
+.macro SAVE4x4
+
+	vbroadcastsd	ALPHA, %ymm0
+
+	vmulpd	%ymm0 , %ymm4 , %ymm4
+	vmulpd	%ymm0 , %ymm7 , %ymm7
+	vmulpd	%ymm0 , %ymm5 , %ymm5
+	vmulpd	%ymm0 , %ymm6 , %ymm6
+
+#if defined BROADCASTKERNEL
+        vperm2f128 $ 0x20 , %ymm6, %ymm4 , %ymm0
+        vperm2f128 $ 0x20 , %ymm7, %ymm5 , %ymm1
+        vperm2f128 $ 0x31 , %ymm6, %ymm4 , %ymm2
+        vperm2f128 $ 0x31 , %ymm7, %ymm5 , %ymm3
+        vunpcklpd %ymm1, %ymm0, %ymm4
+        vunpckhpd %ymm1, %ymm0, %ymm5
+        vunpcklpd %ymm3, %ymm2, %ymm6
+        vunpckhpd %ymm3, %ymm2, %ymm7
+#else
+	vpermilpd $ 0x05 , %ymm5, %ymm5
+	vpermilpd $ 0x05 , %ymm7, %ymm7
+
+	vblendpd $ 0x0a, %ymm5, %ymm4, %ymm0
+	vblendpd $ 0x05, %ymm5, %ymm4, %ymm1
+	vblendpd $ 0x0a, %ymm7, %ymm6, %ymm2
+	vblendpd $ 0x05, %ymm7, %ymm6, %ymm3
+
+	vperm2f128 $ 0x01 , %ymm2, %ymm2 , %ymm2
+	vperm2f128 $ 0x01 , %ymm3, %ymm3 , %ymm3
+
+	vblendpd $ 0x03, %ymm0, %ymm2 , %ymm4
+	vblendpd $ 0x03, %ymm1, %ymm3 , %ymm5
+	vblendpd $ 0x03, %ymm2, %ymm0 , %ymm6
+	vblendpd $ 0x03, %ymm3, %ymm1 , %ymm7
+#endif
+
+        leaq    (CO1, LDC, 2), %rax     
+	
+
+#if !defined(TRMMKERNEL)
+
+	vaddpd 	                (CO1), %ymm4, %ymm4
+	vaddpd 	           (CO1, LDC), %ymm5, %ymm5
+	vaddpd 	               (%rax), %ymm6, %ymm6
+	vaddpd 	          (%rax, LDC), %ymm7, %ymm7
+
+#endif
+
+	vmovups	%ymm4 ,  	(CO1)
+	vmovups	%ymm5 ,  	(CO1, LDC)
+	vmovups	%ymm6 ,  	(%rax)
+	vmovups	%ymm7 ,  	(%rax, LDC)
+
+	addq	$ 4*SIZE, CO1
+.endm
+
+/******************************************************************************************/
+/******************************************************************************************/
+
+.macro INIT2x4
+
+	vxorpd		%xmm4 , %xmm4 , %xmm4
+	vxorpd		%xmm5 , %xmm5 , %xmm5
+	vxorpd		%xmm6 , %xmm6 , %xmm6
+	vxorpd		%xmm7 , %xmm7 , %xmm7
+
+.endm
+
+
+.macro KERNEL2x4_SUB
+	vmovddup	-12 * SIZE(BO), %xmm1
+	vmovups 	-16 * SIZE(AO), %xmm0
+	vmovddup	-11 * SIZE(BO), %xmm2
+	vfmadd231pd  	%xmm0 ,%xmm1  , %xmm4
+	vmovddup	-10 * SIZE(BO), %xmm3
+	vfmadd231pd  	%xmm0 ,%xmm2  , %xmm5
+	vmovddup	 -9 * SIZE(BO), %xmm8
+	vfmadd231pd  	%xmm0 ,%xmm3  , %xmm6
+	addq		$ 4*SIZE, BO
+	vfmadd231pd  	%xmm0 ,%xmm8  , %xmm7
+	addq		$ 2*SIZE, AO
+
+.endm
+
+
+.macro SAVE2x4
+
+	vmovddup	ALPHA, %xmm0
+
+	vmulpd	%xmm0 , %xmm4 , %xmm4
+	vmulpd	%xmm0 , %xmm5 , %xmm5
+	vmulpd	%xmm0 , %xmm6 , %xmm6
+	vmulpd	%xmm0 , %xmm7 , %xmm7
+
+        leaq    (CO1, LDC, 2), %rax     
+
+#if !defined(TRMMKERNEL)
+
+	vaddpd 	                (CO1), %xmm4, %xmm4
+	vaddpd 	           (CO1, LDC), %xmm5, %xmm5
+	vaddpd 	               (%rax), %xmm6, %xmm6
+	vaddpd 	          (%rax, LDC), %xmm7, %xmm7
+
+#endif
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm5 ,  	(CO1, LDC)
+	vmovups	%xmm6 ,  	(%rax)
+	vmovups	%xmm7 ,  	(%rax, LDC)
+
+	addq	$ 2*SIZE, CO1
+.endm
+
+/******************************************************************************************/
+/******************************************************************************************/
+
+.macro INIT1x4
+
+	vxorpd		%xmm4 , %xmm4 , %xmm4
+	vxorpd		%xmm5 , %xmm5 , %xmm5
+	vxorpd		%xmm6 , %xmm6 , %xmm6
+	vxorpd		%xmm7 , %xmm7 , %xmm7
+
+.endm
+
+
+.macro KERNEL1x4_SUB
+	vmovsd	-12 * SIZE(BO), %xmm1
+	vmovsd 	-16 * SIZE(AO), %xmm0
+	vmovsd	-11 * SIZE(BO), %xmm2
+	vfmadd231sd  	%xmm0 ,%xmm1  , %xmm4
+	vmovsd	-10 * SIZE(BO), %xmm3
+	vfmadd231sd  	%xmm0 ,%xmm2  , %xmm5
+	vmovsd	 -9 * SIZE(BO), %xmm8
+	vfmadd231sd  	%xmm0 ,%xmm3  , %xmm6
+	addq		$ 4*SIZE, BO
+	vfmadd231sd  	%xmm0 ,%xmm8  , %xmm7
+	addq		$ 1*SIZE, AO
+
+.endm
+
+
+.macro SAVE1x4
+
+	vmovsd	ALPHA, %xmm0
+
+	vmulsd	%xmm0 , %xmm4 , %xmm4
+	vmulsd	%xmm0 , %xmm5 , %xmm5
+	vmulsd	%xmm0 , %xmm6 , %xmm6
+	vmulsd	%xmm0 , %xmm7 , %xmm7
+
+        leaq    (CO1, LDC, 2), %rax     
+
+#if !defined(TRMMKERNEL)
+
+	vaddsd 	                (CO1), %xmm4, %xmm4
+	vaddsd 	           (CO1, LDC), %xmm5, %xmm5
+	vaddsd 	               (%rax), %xmm6, %xmm6
+	vaddsd 	          (%rax, LDC), %xmm7, %xmm7
+
+#endif
+
+	vmovsd	%xmm4 ,  	(CO1)
+	vmovsd	%xmm5 ,  	(CO1, LDC)
+	vmovsd	%xmm6 ,  	(%rax)
+	vmovsd	%xmm7 ,  	(%rax, LDC)
+
+	addq	$ 1*SIZE, CO1
+.endm
+
+
+/******************************************************************************************/
+/******************************************************************************************/
+
+.macro INIT4x2
+
+	vxorpd		%xmm4 , %xmm4 , %xmm4
+	vxorpd		%xmm5 , %xmm5 , %xmm5
+	vxorpd		%xmm6 , %xmm6 , %xmm6
+	vxorpd		%xmm7 , %xmm7 , %xmm7
+
+.endm
+
+
+.macro KERNEL4x2_SUB
+	vmovddup	-12 * SIZE(BO), %xmm2
+	vmovups 	-16 * SIZE(AO), %xmm0
+	vmovups 	-14 * SIZE(AO), %xmm1
+	vmovddup	-11 * SIZE(BO), %xmm3
+	vfmadd231pd  	%xmm0 ,%xmm2  , %xmm4
+	vfmadd231pd  	%xmm1 ,%xmm2  , %xmm5
+	vfmadd231pd  	%xmm0 ,%xmm3  , %xmm6
+	vfmadd231pd  	%xmm1 ,%xmm3  , %xmm7
+	addq		$ 2*SIZE, BO
+	addq		$ 4*SIZE, AO
+
+.endm
+
+
+.macro SAVE4x2
+
+	vmovddup	ALPHA, %xmm0
+
+	vmulpd	%xmm0 , %xmm4 , %xmm4
+	vmulpd	%xmm0 , %xmm5 , %xmm5
+	vmulpd	%xmm0 , %xmm6 , %xmm6
+	vmulpd	%xmm0 , %xmm7 , %xmm7
+
+
+#if !defined(TRMMKERNEL)
+
+	vaddpd 	                (CO1)     , %xmm4, %xmm4
+	vaddpd 	        2 * SIZE(CO1)     , %xmm5, %xmm5
+	vaddpd 	                (CO1, LDC), %xmm6, %xmm6
+	vaddpd 	        2 * SIZE(CO1, LDC), %xmm7, %xmm7
+
+#endif
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm5 , 2 * SIZE(CO1)
+	vmovups	%xmm6 ,  	(CO1, LDC)
+	vmovups	%xmm7 , 2 * SIZE(CO1, LDC)
+
+	addq	$ 4*SIZE, CO1
+.endm
+
+
+/******************************************************************************************/
+/******************************************************************************************/
+
+.macro INIT2x2
+
+	vxorpd		%xmm4 , %xmm4 , %xmm4
+	vxorpd		%xmm6 , %xmm6 , %xmm6
+
+.endm
+
+
+.macro KERNEL2x2_SUB
+	vmovddup	-12 * SIZE(BO), %xmm2
+	vmovups 	-16 * SIZE(AO), %xmm0
+	vmovddup	-11 * SIZE(BO), %xmm3
+	vfmadd231pd  	%xmm0 ,%xmm2  , %xmm4
+	vfmadd231pd  	%xmm0 ,%xmm3  , %xmm6
+	addq		$ 2*SIZE, BO
+	addq		$ 2*SIZE, AO
+
+.endm
+
+
+.macro SAVE2x2
+
+	vmovddup	ALPHA, %xmm0
+
+	vmulpd	%xmm0 , %xmm4 , %xmm4
+	vmulpd	%xmm0 , %xmm6 , %xmm6
+
+
+#if !defined(TRMMKERNEL)
+
+	vaddpd 	                (CO1)     , %xmm4, %xmm4
+	vaddpd 	                (CO1, LDC), %xmm6, %xmm6
+
+#endif
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm6 ,  	(CO1, LDC)
+
+	addq	$ 2*SIZE, CO1
+.endm
+
+/******************************************************************************************/
+/******************************************************************************************/
+
+.macro INIT1x2
+
+	vxorpd		%xmm4 , %xmm4 , %xmm4
+	vxorpd		%xmm5 , %xmm5 , %xmm5
+
+.endm
+
+
+.macro KERNEL1x2_SUB
+	vmovsd	-12 * SIZE(BO), %xmm1
+	vmovsd 	-16 * SIZE(AO), %xmm0
+	vmovsd	-11 * SIZE(BO), %xmm2
+	vfmadd231sd  	%xmm0 ,%xmm1  , %xmm4
+	vfmadd231sd  	%xmm0 ,%xmm2  , %xmm5
+	addq		$ 2*SIZE, BO
+	addq		$ 1*SIZE, AO
+
+.endm
+
+
+.macro SAVE1x2
+
+	vmovsd	ALPHA, %xmm0
+
+	vmulsd	%xmm0 , %xmm4 , %xmm4
+	vmulsd	%xmm0 , %xmm5 , %xmm5
+
+
+#if !defined(TRMMKERNEL)
+
+	vaddsd 	                (CO1), %xmm4, %xmm4
+	vaddsd 	           (CO1, LDC), %xmm5, %xmm5
+
+#endif
+
+	vmovsd	%xmm4 ,  	(CO1)
+	vmovsd	%xmm5 ,  	(CO1, LDC)
+
+	addq	$ 1*SIZE, CO1
+.endm
+
+
+/******************************************************************************************/
+/******************************************************************************************/
+
+.macro INIT4x1
+
+	vxorpd		%ymm4 , %ymm4 , %ymm4
+	vxorpd		%ymm5 , %ymm5 , %ymm5
+	vxorpd		%ymm6 , %ymm6 , %ymm6
+	vxorpd		%ymm7 , %ymm7 , %ymm7
+
+.endm
+
+
+.macro KERNEL4x1
+
+	vbroadcastsd	-12 * SIZE(BO), %ymm0
+	vbroadcastsd	-11 * SIZE(BO), %ymm1
+	vbroadcastsd	-10 * SIZE(BO), %ymm2
+	vbroadcastsd	-9  * SIZE(BO), %ymm3
+
+	vfmadd231pd  	-16 * SIZE(AO) ,%ymm0  , %ymm4
+	vfmadd231pd  	-12 * SIZE(AO) ,%ymm1  , %ymm5
+
+	vbroadcastsd	-8  * SIZE(BO), %ymm0
+	vbroadcastsd	-7  * SIZE(BO), %ymm1
+
+	vfmadd231pd  	-8  * SIZE(AO) ,%ymm2  , %ymm6
+	vfmadd231pd  	-4  * SIZE(AO) ,%ymm3  , %ymm7
+
+	vbroadcastsd	-6  * SIZE(BO), %ymm2
+	vbroadcastsd	-5  * SIZE(BO), %ymm3
+
+	vfmadd231pd  	 0  * SIZE(AO) ,%ymm0  , %ymm4
+	vfmadd231pd  	 4  * SIZE(AO) ,%ymm1  , %ymm5
+	vfmadd231pd  	 8  * SIZE(AO) ,%ymm2  , %ymm6
+	vfmadd231pd  	 12 * SIZE(AO) ,%ymm3  , %ymm7
+
+	addq		$ 8 *SIZE, BO
+	addq		$ 32*SIZE, AO
+
+.endm
+
+
+.macro KERNEL4x1_SUB
+	vbroadcastsd	-12 * SIZE(BO), %ymm2
+	vmovups 	-16 * SIZE(AO), %ymm0
+	vfmadd231pd  	%ymm0 ,%ymm2  , %ymm4
+	addq		$ 1*SIZE, BO
+	addq		$ 4*SIZE, AO
+
+.endm
+
+
+.macro SAVE4x1
+
+	vbroadcastsd	ALPHA, %ymm0
+
+	vaddpd	%ymm4,%ymm5, %ymm4 
+	vaddpd	%ymm6,%ymm7, %ymm6 
+	vaddpd	%ymm4,%ymm6, %ymm4 
+
+	vmulpd	%ymm0 , %ymm4 , %ymm4
+
+
+#if !defined(TRMMKERNEL)
+
+	vaddpd 	                (CO1)     , %ymm4, %ymm4
+
+#endif
+
+	vmovups	%ymm4 ,  	(CO1)
+
+	addq	$ 4*SIZE, CO1
+.endm
+
+
+/******************************************************************************************/
+/******************************************************************************************/
+
+.macro INIT2x1
+
+	vxorpd		%xmm4 , %xmm4 , %xmm4
+
+.endm
+
+
+.macro KERNEL2x1_SUB
+	vmovddup	-12 * SIZE(BO), %xmm2
+	vmovups 	-16 * SIZE(AO), %xmm0
+	vfmadd231pd  	%xmm0 ,%xmm2  , %xmm4
+	addq		$ 1*SIZE, BO
+	addq		$ 2*SIZE, AO
+
+.endm
+
+
+.macro SAVE2x1
+
+	vmovddup	ALPHA, %xmm0
+
+	vmulpd	%xmm0 , %xmm4 , %xmm4
+
+
+#if !defined(TRMMKERNEL)
+
+	vaddpd 	                (CO1)     , %xmm4, %xmm4
+
+#endif
+
+	vmovups	%xmm4 ,  	(CO1)
+
+	addq	$ 2*SIZE, CO1
+.endm
+
+
+/******************************************************************************************/
+/******************************************************************************************/
+
+.macro INIT1x1
+
+	vxorpd		%xmm4 , %xmm4 , %xmm4
+
+.endm
+
+
+.macro KERNEL1x1_SUB
+	vmovsd	-12 * SIZE(BO), %xmm1
+	vmovsd 	-16 * SIZE(AO), %xmm0
+	vfmadd231sd  	%xmm0 ,%xmm1  , %xmm4
+	addq		$ 1*SIZE, BO
+	addq		$ 1*SIZE, AO
+
+.endm
+
+
+.macro SAVE1x1
+
+	vmovsd	ALPHA, %xmm0
+
+	vmulsd	%xmm0 , %xmm4 , %xmm4
+
+
+#if !defined(TRMMKERNEL)
+
+	vaddsd 	                (CO1), %xmm4, %xmm4
+
+#endif
+
+	vmovsd	%xmm4 ,  	(CO1)
+
+	addq	$ 1*SIZE, CO1
+.endm
+
+
+.macro PREFETCHT0_C
+        prefetcht0 (CO1)
+        prefetcht0 24(CO1)
+        prefetcht0 (CO1,LDC,4)
+        prefetcht0 24(CO1,LDC,4)
+        prefetcht0 (CO1,LDC,8)
+        prefetcht0 24(CO1,LDC,8)
+.endm
+/*******************************************************************************************/
+
+#if !defined(TRMMKERNEL)
+
+
+	PROLOGUE
+	PROFCODE
+	
+	subq	$STACKSIZE, %rsp
+	movq	%rbx,   (%rsp)
+	movq	%rbp,  8(%rsp)
+	movq	%r12, 16(%rsp)
+	movq	%r13, 24(%rsp)
+	movq	%r14, 32(%rsp)
+	movq	%r15, 40(%rsp)
+
+	vzeroupper
+
+#ifdef WINDOWS_ABI
+	movq	%rdi,    48(%rsp)
+	movq	%rsi,    56(%rsp)
+	vmovups	%xmm6,   64(%rsp)
+	vmovups	%xmm7,   80(%rsp)
+	vmovups	%xmm8,   96(%rsp)
+	vmovups	%xmm9,  112(%rsp)
+	vmovups	%xmm10, 128(%rsp)
+	vmovups	%xmm11, 144(%rsp)
+	vmovups	%xmm12, 160(%rsp)
+	vmovups	%xmm13, 176(%rsp)
+	vmovups	%xmm14, 192(%rsp)
+	vmovups	%xmm15, 208(%rsp)
+
+	movq	ARG1,      OLD_M
+	movq	ARG2,      OLD_N
+	movq	ARG3,      OLD_K
+	movq	OLD_A,     A
+	movq	OLD_B,     B
+	movq	OLD_C,     C
+	movq	OLD_LDC,   LDC
+
+	vmovups	%xmm3, %xmm0
+
+#else
+	movq	STACKSIZE +  8(%rsp), LDC
+
+#endif
+
+	movq    %rsp, SP      # save old stack
+        subq    $128 + L_BUFFER_SIZE, %rsp
+        andq    $-4096, %rsp    # align stack
+
+        STACK_TOUCH
+
+	cmpq	$ 0, OLD_M
+	je	.L999
+
+	cmpq	$ 0, OLD_N
+	je	.L999
+
+	cmpq	$ 0, OLD_K
+	je	.L999
+
+	movq	OLD_M, M
+	movq	OLD_N, N
+	movq	OLD_K, K
+
+	vmovsd	 %xmm0, ALPHA
+
+	salq	$BASE_SHIFT, LDC
+
+	movq    N, %rax
+        xorq    %rdx, %rdx
+        movq    $24,  %rdi
+        divq    %rdi                     //    N / 24
+        movq    %rax, Ndiv12             //    N / 24
+        movq    %rdx, Nmod12             //    N % 24
+
+
+	movq	Ndiv12,  J
+	cmpq	$ 0, J
+	je	.L8_0
+	ALIGN_4
+
+.L12_01:
+        // copy to sub buffer
+        movq    K, %rax
+        salq    $3,%rax                 // K * 8 ; read 8 values from BO1
+        movq    B, BO1
+        leaq    (B,%rax, SIZE), BO2     // next offset to BO2
+	movq	BO2 , B			
+
+        leaq    BUFFER1, BO             // first buffer to BO
+        movq    K, %rax
+
+        ALIGN_4
+
+.L12_02b:
+
+	vmovups	0 * SIZE(BO1), %ymm1
+	vmovups	4 * SIZE(BO1), %ymm2
+	vmovups	0 * SIZE(BO2), %ymm3
+	vmovups	%ymm1, 0 * SIZE(BO)
+	vmovups	%ymm2, 4 * SIZE(BO)
+	vmovups	%ymm3, 8 * SIZE(BO)
+	addq	$ 8*SIZE,BO1
+	addq	$ 8*SIZE,BO2
+	addq	$ 12*SIZE,BO
+	decq	%rax
+	jnz	.L12_02b
+
+.L12_03c:
+
+
+.L12_10:
+	movq	C, CO1
+	leaq	(C, LDC, 8), C		 
+	leaq	(C, LDC, 4), C		// c += 12 * ldc
+	
+	movq	A, AO		 	// aoffset = a
+	addq	$16 * SIZE, AO
+
+	movq	M,  I
+	sarq	$2, I			// i = m / 4
+	je	.L12_20
+
+	ALIGN_4
+
+.L12_11:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $12 * SIZE, BO
+
+        movq    K, %rax
+
+	sarq $3, %rax			//  K / 8
+	cmpq $2, %rax
+
+	jl	.L12_13
+
+
+	KERNEL4x12_I
+	KERNEL4x12_M2
+	KERNEL4x12_M1
+	KERNEL4x12_M2
+
+	KERNEL4x12_M1
+	KERNEL4x12_M2
+	KERNEL4x12_M1
+	KERNEL4x12_M2
+
+	subq $2, %rax
+	je	.L12_12a
+
+	ALIGN_5
+.L12_12:
+
+	KERNEL4x12_M1
+	KERNEL4x12_M2
+	KERNEL4x12_M1
+	KERNEL4x12_M2
+
+	KERNEL4x12_M1
+	KERNEL4x12_M2
+	KERNEL4x12_M1
+	KERNEL4x12_M2
+
+	dec	%rax
+	jne	.L12_12
+	
+.L12_12a:
+        prefetcht0 ALPHA
+        PREFETCHT0_C
+        addq  LDC,CO1
+	KERNEL4x12_M1
+        PREFETCHT0_C
+        leaq  (CO1,LDC,2),CO1
+	KERNEL4x12_M2
+        PREFETCHT0_C
+        subq  LDC,CO1
+	KERNEL4x12_M1
+        PREFETCHT0_C
+        subq  LDC,CO1
+        subq  LDC,CO1
+	KERNEL4x12_M2
+
+	KERNEL4x12_M1
+	KERNEL4x12_M2
+	KERNEL4x12_M1
+	KERNEL4x12_E
+
+	jmp .L12_16
+
+
+.L12_13:
+
+	test $1, %rax
+	jz .L12_14
+
+	KERNEL4x12_I
+	KERNEL4x12_M2
+	KERNEL4x12_M1
+	KERNEL4x12_M2
+
+	KERNEL4x12_M1
+	KERNEL4x12_M2
+	KERNEL4x12_M1
+	KERNEL4x12_E
+
+	jmp .L12_16
+
+
+.L12_14:
+
+	INIT4x12
+
+
+.L12_16:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L12_19
+
+	ALIGN_4
+
+.L12_17:
+
+	KERNEL4x12_SUB
+
+	dec	%rax
+	jne	.L12_17
+	ALIGN_4
+
+
+.L12_19:
+
+	SAVE4x12
+
+        /* here for the prefetch of next b source block */
+	/* the increment should be proportional to GEMM_Q/GEMM_P */
+
+        salq    $3, K
+#ifdef WINDOWS_ABI /* GEMM_P == GEMM_Q * 4 */
+        prefetcht2 32(B)
+        prefetcht2 32(B, K, 8)
+        addq    $64, B /* increment */
+#else /* GEMM_P == GEMM_Q * 2 under linux x86_64 */
+        prefetcht2 32(B)
+        prefetcht2 32(B, K, 8)
+        prefetcht2 96(B)
+        prefetcht2 96(B, K, 8)
+        addq    $128, B /* increment */
+#endif
+        sarq    $3, K
+
+	decq	I			# i --
+	jne	.L12_11
+	ALIGN_4	
+
+/**************************************************************************
+* Rest of M 
+***************************************************************************/
+
+        /* recover the original value of pointer B after prefetch */
+        movq    M, I
+        sarq    $2, I
+#ifdef WINDOWS_ABI /* GEMM_P == GEMM_Q * 4 */
+        salq    $6, I
+#else /* GEMM_P == GEMM_Q * 2 under linux x86_64 */
+        salq    $7, I
+#endif
+        subq    I, B
+
+.L12_20:
+	// Test rest of M
+
+	testq	$3, M
+	jz	.L12_100			// to next 16 lines of N
+
+
+.L12_30:
+	testq	$2, M		
+	jz	.L12_40
+
+	ALIGN_4
+
+.L12_31:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $12 * SIZE, BO
+
+	INIT2x12
+
+        movq    K, %rax
+
+	sarq	$3, %rax
+	je	.L12_36
+	ALIGN_4
+
+.L12_32:
+
+	KERNEL2x12_SUB
+	KERNEL2x12_SUB
+	KERNEL2x12_SUB
+	KERNEL2x12_SUB
+
+	KERNEL2x12_SUB
+	KERNEL2x12_SUB
+	KERNEL2x12_SUB
+	KERNEL2x12_SUB
+
+	dec %rax
+	jne	.L12_32
+	ALIGN_4
+
+.L12_36:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L12_39
+
+	ALIGN_4
+
+.L12_37:
+
+	KERNEL2x12_SUB
+
+	dec %rax
+	jne	.L12_37
+	ALIGN_4
+
+
+.L12_39:
+
+	SAVE2x12
+
+	ALIGN_4
+
+.L12_40:
+	testq	$1, M		
+	jz	.L12_100		// to next 3 lines of N
+
+	ALIGN_4
+
+.L12_41:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $12 * SIZE, BO
+
+	INIT1x12
+
+        movq    K, %rax
+
+	sarq	$3,%rax
+	je	.L12_46
+
+	ALIGN_4
+
+.L12_42:
+
+	KERNEL1x12_SUB
+	KERNEL1x12_SUB
+	KERNEL1x12_SUB
+	KERNEL1x12_SUB
+
+	KERNEL1x12_SUB
+	KERNEL1x12_SUB
+	KERNEL1x12_SUB
+	KERNEL1x12_SUB
+
+
+	dec %rax
+	jne	.L12_42
+	ALIGN_4
+
+.L12_46:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L12_49
+
+	ALIGN_4
+
+.L12_47:
+
+	KERNEL1x12_SUB
+
+	dec	%rax
+	jne	.L12_47
+	ALIGN_4
+
+
+.L12_49:
+
+	SAVE1x12
+
+	ALIGN_4
+	
+.L12_100:
+
+
+
+/**************************************************************************************************/
+
+.L13_01:
+        // copy to sub buffer
+        movq    K, %rax
+        salq    $3,%rax                 // K * 8 ; read 8 values
+        movq    B, BO2
+        leaq    (B,%rax, SIZE), BO3     // next offset to BO2
+        leaq    (BO3,%rax, SIZE), B     // next offset to B
+
+
+        leaq    BUFFER1, BO             // first buffer to BO
+        movq    K, %rax
+
+        ALIGN_4
+
+
+.L13_02b:
+
+	vmovups	4 * SIZE(BO2), %ymm1
+	vmovups	0 * SIZE(BO3), %ymm2
+	vmovups	4 * SIZE(BO3), %ymm3
+	vmovups	%ymm1, 0 * SIZE(BO)
+	vmovups	%ymm2, 4 * SIZE(BO)
+	vmovups	%ymm3, 8 * SIZE(BO)
+	addq	$ 8*SIZE,BO2
+	addq	$ 8*SIZE,BO3
+	addq	$ 12*SIZE,BO
+	decq	%rax
+	jnz	.L13_02b
+
+
+
+.L13_10:
+	movq	C, CO1
+	leaq	(C, LDC, 8), C		 
+	leaq	(C, LDC, 4), C		// c += 12 * ldc
+
+	
+	movq	A, AO		 	// aoffset = a
+	addq	$16 * SIZE, AO
+
+	movq	M,  I
+	sarq	$2, I			// i = m / 4
+	je	.L13_20
+
+	ALIGN_4
+
+.L13_11:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $12 * SIZE, BO
+
+        movq    K, %rax
+
+	sarq $3, %rax			//  K / 8
+	cmpq $2, %rax
+
+	jl	.L13_13
+
+
+	KERNEL4x12_I
+	KERNEL4x12_M2
+	KERNEL4x12_M1
+	KERNEL4x12_M2
+
+	KERNEL4x12_M1
+	KERNEL4x12_M2
+	KERNEL4x12_M1
+	KERNEL4x12_M2
+
+	subq $2, %rax
+	je	.L13_12a
+
+	ALIGN_5
+.L13_12:
+
+	KERNEL4x12_M1
+	KERNEL4x12_M2
+	KERNEL4x12_M1
+	KERNEL4x12_M2
+
+	KERNEL4x12_M1
+	KERNEL4x12_M2
+	KERNEL4x12_M1
+	KERNEL4x12_M2
+
+	dec	%rax
+	jne	.L13_12
+
+.L13_12a:
+        prefetcht0 ALPHA
+        PREFETCHT0_C
+        addq  LDC,CO1
+	KERNEL4x12_M1
+        PREFETCHT0_C
+        leaq  (CO1,LDC,2),CO1
+	KERNEL4x12_M2
+        PREFETCHT0_C
+        subq  LDC,CO1
+	KERNEL4x12_M1
+        PREFETCHT0_C
+        subq  LDC,CO1
+        subq  LDC,CO1
+	KERNEL4x12_M2
+
+	KERNEL4x12_M1
+	KERNEL4x12_M2
+	KERNEL4x12_M1
+	KERNEL4x12_E
+
+	jmp .L13_16
+
+.L13_13:
+
+	test $1, %rax
+	jz .L13_14
+
+	KERNEL4x12_I
+	KERNEL4x12_M2
+	KERNEL4x12_M1
+	KERNEL4x12_M2
+
+	KERNEL4x12_M1
+	KERNEL4x12_M2
+	KERNEL4x12_M1
+	KERNEL4x12_E
+
+	jmp .L13_16
+
+
+.L13_14:
+
+	INIT4x12
+
+
+.L13_16:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L13_19
+
+	ALIGN_4
+
+.L13_17:
+
+	KERNEL4x12_SUB
+
+	dec	%rax
+	jne	.L13_17
+	ALIGN_4
+
+
+.L13_19:
+
+	SAVE4x12
+
+        /* here for the prefetch of next b source block */
+	/* the increment should be proportional to GEMM_Q/GEMM_P */
+
+        salq    $3, K
+#ifdef WINDOWS_ABI /* GEMM_P == GEMM_Q * 4 */
+        prefetcht2 (B)
+        prefetcht2 (B, K, 8)
+        addq    $64, B /* increment */
+#else /* GEMM_P == GEMM_Q * 2 under linux x86_64 */
+        prefetcht2 (B)
+        prefetcht2 (B, K, 8)
+        prefetcht2 64(B)
+        prefetcht2 64(B, K, 8)
+        addq    $128, B /* increment */
+#endif
+        sarq    $3, K
+
+	decq	I			# i --
+	jne	.L13_11
+	ALIGN_4	
+
+/**************************************************************************
+* Rest of M 
+***************************************************************************/
+        /* recover the original value of pointer B */
+        movq    M, I
+        sarq    $2, I
+#ifdef WINDOWS_ABI /* GEMM_P == GEMM_Q * 4 */
+        salq    $6, I
+#else /* GEMM_P == GEMM_Q * 2 under linux x86_64 */
+        salq    $7, I
+#endif
+        subq    I, B
+
+.L13_20:
+	// Test rest of M
+
+	testq	$3, M
+	jz	.L13_100			// to next 16 lines of N
+
+
+.L13_30:
+	testq	$2, M		
+	jz	.L13_40
+
+	ALIGN_4
+
+.L13_31:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $12 * SIZE, BO
+
+	INIT2x12
+
+        movq    K, %rax
+
+	sarq	$3, %rax
+	je	.L13_36
+	ALIGN_4
+
+.L13_32:
+
+	KERNEL2x12_SUB
+	KERNEL2x12_SUB
+	KERNEL2x12_SUB
+	KERNEL2x12_SUB
+
+	KERNEL2x12_SUB
+	KERNEL2x12_SUB
+	KERNEL2x12_SUB
+	KERNEL2x12_SUB
+
+	dec %rax
+	jne	.L13_32
+	ALIGN_4
+
+.L13_36:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L13_39
+
+	ALIGN_4
+
+.L13_37:
+
+	KERNEL2x12_SUB
+
+	dec %rax
+	jne	.L13_37
+	ALIGN_4
+
+
+.L13_39:
+
+	SAVE2x12
+
+	ALIGN_4
+
+.L13_40:
+	testq	$1, M		
+	jz	.L13_100		// to next 3 lines of N
+
+	ALIGN_4
+
+.L13_41:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $12 * SIZE, BO
+
+	INIT1x12
+
+        movq    K, %rax
+
+	sarq	$3,%rax
+	je	.L13_46
+
+	ALIGN_4
+
+.L13_42:
+
+	KERNEL1x12_SUB
+	KERNEL1x12_SUB
+	KERNEL1x12_SUB
+	KERNEL1x12_SUB
+
+	KERNEL1x12_SUB
+	KERNEL1x12_SUB
+	KERNEL1x12_SUB
+	KERNEL1x12_SUB
+
+
+	dec %rax
+	jne	.L13_42
+	ALIGN_4
+
+.L13_46:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L13_49
+
+	ALIGN_4
+
+.L13_47:
+
+	KERNEL1x12_SUB
+
+	dec	%rax
+	jne	.L13_47
+	ALIGN_4
+
+
+.L13_49:
+
+	SAVE1x12
+
+	ALIGN_4
+	
+.L13_100:
+
+	decq	J			// j --
+	jg	.L12_01
+
+
+
+
+/**************************************************************************************************/
+
+.L8_0:
+
+	cmpq	$ 0, Nmod12		// N % 12 == 0
+	je	.L999
+
+	movq	Nmod12, J		
+	sarq	$3, J			// j = j / 8
+	je	.L4_0
+
+.L8_10:
+	movq	C, CO1
+	leaq	(C, LDC, 8), C		// c += 4 * ldc
+
+	
+	movq	A, AO		 	// aoffset = a
+	addq	$16 * SIZE, AO
+
+	movq	M,  I
+	sarq	$2, I			// i = m / 4
+	je	.L8_20
+
+	ALIGN_4
+
+.L8_11:
+        movq    B, BO        
+        addq    $12 * SIZE, BO
+
+        movq    K, %rax
+
+	sarq	$3, %rax			//  K / 8
+	cmpq    $2, %rax
+	jl	.L8_13
+
+
+	KERNEL4x8_I
+	KERNEL4x8_M2
+	KERNEL4x8_M1
+	KERNEL4x8_M2
+
+	KERNEL4x8_M1
+	KERNEL4x8_M2
+	KERNEL4x8_M1
+	KERNEL4x8_M2
+
+	subq $2, %rax
+	je	.L8_12a
+
+	ALIGN_5
+
+.L8_12:
+
+	KERNEL4x8_M1
+	KERNEL4x8_M2
+	KERNEL4x8_M1
+	KERNEL4x8_M2
+
+	KERNEL4x8_M1
+	KERNEL4x8_M2
+	KERNEL4x8_M1
+	KERNEL4x8_M2
+
+	dec	%rax
+	jne	.L8_12
+
+.L8_12a:
+
+	KERNEL4x8_M1
+	KERNEL4x8_M2
+	KERNEL4x8_M1
+	KERNEL4x8_M2
+
+	KERNEL4x8_M1
+	KERNEL4x8_M2
+	KERNEL4x8_M1
+	KERNEL4x8_E
+
+	jmp .L8_16
+
+
+.L8_13:
+
+	test $1, %rax
+	jz .L8_14
+
+	KERNEL4x8_I
+	KERNEL4x8_M2
+	KERNEL4x8_M1
+	KERNEL4x8_M2
+
+	KERNEL4x8_M1
+	KERNEL4x8_M2
+	KERNEL4x8_M1
+	KERNEL4x8_E
+
+	jmp .L8_16
+
+
+.L8_14:
+
+	INIT4x8
+
+
+.L8_16:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L8_19
+
+	ALIGN_4
+
+.L8_17:
+
+	KERNEL4x8_SUB
+
+	dec	%rax
+	jne	.L8_17
+	ALIGN_4
+
+
+.L8_19:
+
+	SAVE4x8
+
+	decq	I			# i --
+	jg	.L8_11
+	ALIGN_4	
+
+/**************************************************************************
+* Rest of M 
+***************************************************************************/
+.L8_20:
+	// Test rest of M
+
+	testq	$3, M
+	jz	.L8_100			// to next 16 lines of N
+
+
+.L8_30:
+	testq	$2, M		
+	jz	.L8_40
+
+	ALIGN_4
+
+.L8_31:
+        movq    B, BO             // first buffer to BO
+        addq    $12 * SIZE, BO
+
+	INIT2x8
+
+        movq    K, %rax
+
+	sarq	$3, %rax
+	je	.L8_36
+	ALIGN_4
+
+.L8_32:
+
+	KERNEL2x8_SUB
+	KERNEL2x8_SUB
+	KERNEL2x8_SUB
+	KERNEL2x8_SUB
+
+	KERNEL2x8_SUB
+	KERNEL2x8_SUB
+	KERNEL2x8_SUB
+	KERNEL2x8_SUB
+
+	dec %rax
+	jne	.L8_32
+	ALIGN_4
+
+.L8_36:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L8_39
+
+	ALIGN_4
+
+.L8_37:
+
+	KERNEL2x8_SUB
+
+	dec %rax
+	jne	.L8_37
+
+
+.L8_39:
+
+	SAVE2x8
+
+.L8_40:
+	testq	$1, M		
+	jz	.L8_100		// to next 3 lines of N
+
+	ALIGN_4
+
+.L8_41:
+        movq    B, BO             // first buffer to BO
+        addq    $12 * SIZE, BO
+
+	INIT1x8
+
+        movq    K, %rax
+
+	sarq	$3,%rax
+	je	.L8_46
+
+	ALIGN_4
+
+.L8_42:
+
+	KERNEL1x8_SUB
+	KERNEL1x8_SUB
+	KERNEL1x8_SUB
+	KERNEL1x8_SUB
+
+	KERNEL1x8_SUB
+	KERNEL1x8_SUB
+	KERNEL1x8_SUB
+	KERNEL1x8_SUB
+
+	dec %rax
+	jne	.L8_42
+	ALIGN_4
+
+.L8_46:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L8_49
+
+	ALIGN_4
+
+.L8_47:
+
+	KERNEL1x8_SUB
+
+	dec	%rax
+	jne	.L8_47
+	ALIGN_4
+
+
+.L8_49:
+
+	SAVE1x8
+
+	ALIGN_4
+	
+.L8_100:
+
+	movq	K, %rax
+	salq	$3, %rax		// * 8
+	leaq	(B , %rax, SIZE), B
+	decq	J			// j --
+	jg	.L8_10
+
+
+
+/**************************************************************************************************/
+
+.L4_0:
+
+	cmpq	$ 0, Nmod12		// N % 12 == 0
+	je	.L999
+
+	movq	Nmod12, J		
+	testq   $4, J			// j = j / 4
+	je	.L2_0
+
+.L4_10:
+	movq	C, CO1
+	leaq	(C, LDC, 4), C		// c += 4 * ldc
+
+	
+	movq	A, AO		 	// aoffset = a
+	addq	$16 * SIZE, AO
+
+	movq	M,  I
+	sarq	$2, I			// i = m / 4
+	je	.L4_20
+
+	ALIGN_4
+
+.L4_11:
+        movq    B, BO        
+        addq    $12 * SIZE, BO
+
+        movq    K, %rax
+
+	sarq	$3, %rax			//  K / 8
+	cmpq    $2, %rax
+	jl	.L4_13
+
+
+	KERNEL4x4_I
+	KERNEL4x4_M2
+	KERNEL4x4_M1
+	KERNEL4x4_M2
+
+	KERNEL4x4_M1
+	KERNEL4x4_M2
+	KERNEL4x4_M1
+	KERNEL4x4_M2
+
+	subq $2, %rax
+	je	.L4_12a
+
+	ALIGN_5
+
+.L4_12:
+
+	KERNEL4x4_M1
+	KERNEL4x4_M2
+	KERNEL4x4_M1
+	KERNEL4x4_M2
+
+	KERNEL4x4_M1
+	KERNEL4x4_M2
+	KERNEL4x4_M1
+	KERNEL4x4_M2
+
+	dec	%rax
+	jne	.L4_12
+
+.L4_12a:
+
+	KERNEL4x4_M1
+	KERNEL4x4_M2
+	KERNEL4x4_M1
+	KERNEL4x4_M2
+
+	KERNEL4x4_M1
+	KERNEL4x4_M2
+	KERNEL4x4_M1
+	KERNEL4x4_E
+
+	jmp .L4_16
+
+
+.L4_13:
+
+	test $1, %rax
+	jz .L4_14
+
+	KERNEL4x4_I
+	KERNEL4x4_M2
+	KERNEL4x4_M1
+	KERNEL4x4_M2
+
+	KERNEL4x4_M1
+	KERNEL4x4_M2
+	KERNEL4x4_M1
+	KERNEL4x4_E
+
+	jmp .L4_16
+
+
+.L4_14:
+
+	INIT4x4
+
+
+.L4_16:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L4_19
+
+	ALIGN_4
+
+.L4_17:
+
+	KERNEL4x4_SUB
+
+	dec	%rax
+	jne	.L4_17
+	ALIGN_4
+
+
+.L4_19:
+
+	SAVE4x4
+
+	decq	I			# i --
+	jg	.L4_11
+
+	ALIGN_4	
+
+/**************************************************************************
+* Rest of M 
+***************************************************************************/
+.L4_20:
+	// Test rest of M
+
+	testq	$3, M
+	jz	.L4_100			// to next 16 lines of N
+
+
+.L4_30:
+	testq	$2, M		
+	jz	.L4_40
+
+	ALIGN_4
+
+.L4_31:
+        movq    B, BO             // first buffer to BO
+        addq    $12 * SIZE, BO
+
+	INIT2x4
+
+        movq    K, %rax
+
+	sarq	$3, %rax
+	je	.L4_36
+	ALIGN_4
+
+.L4_32:
+
+	KERNEL2x4_SUB
+	KERNEL2x4_SUB
+	KERNEL2x4_SUB
+	KERNEL2x4_SUB
+
+	KERNEL2x4_SUB
+	KERNEL2x4_SUB
+	KERNEL2x4_SUB
+	KERNEL2x4_SUB
+
+	dec %rax
+	jne	.L4_32
+	ALIGN_4
+
+.L4_36:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L4_39
+
+	ALIGN_4
+
+.L4_37:
+
+	KERNEL2x4_SUB
+
+	dec %rax
+	jne	.L4_37
+
+
+.L4_39:
+
+	SAVE2x4
+
+.L4_40:
+	testq	$1, M		
+	jz	.L4_100		// to next 3 lines of N
+
+	ALIGN_4
+
+.L4_41:
+        movq    B, BO             // first buffer to BO
+        addq    $12 * SIZE, BO
+
+	INIT1x4
+
+        movq    K, %rax
+
+	sarq	$3,%rax
+	je	.L4_46
+
+	ALIGN_4
+
+.L4_42:
+
+	KERNEL1x4_SUB
+	KERNEL1x4_SUB
+	KERNEL1x4_SUB
+	KERNEL1x4_SUB
+
+	KERNEL1x4_SUB
+	KERNEL1x4_SUB
+	KERNEL1x4_SUB
+	KERNEL1x4_SUB
+
+	dec %rax
+	jne	.L4_42
+	ALIGN_4
+
+.L4_46:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L4_49
+
+	ALIGN_4
+
+.L4_47:
+
+	KERNEL1x4_SUB
+
+	dec	%rax
+	jne	.L4_47
+	ALIGN_4
+
+
+.L4_49:
+
+	SAVE1x4
+
+	ALIGN_4
+	
+.L4_100:
+
+	movq	K, %rax
+	salq	$2, %rax		// * 4
+	leaq	(B , %rax, SIZE), B
+
+
+
+
+/***************************************************************************************************************/
+
+.L2_0:
+
+	movq	Nmod12, J		
+	testq	$2, J
+	je	.L1_0
+
+.L2_10:
+	movq	C, CO1
+	leaq	(C, LDC, 2), C		// c += 2 * ldc
+
+	
+	movq	A, AO		 	// aoffset = a
+	addq	$16 * SIZE, AO
+
+	movq	M,  I
+	sarq	$2, I			// i = m / 4
+	je	.L2_20
+
+	ALIGN_4
+
+.L2_11:
+        movq    B, BO        
+        addq    $12 * SIZE, BO
+
+	INIT4x2
+
+        movq    K, %rax
+	sarq $3, %rax			//  K / 8
+
+	je	.L2_16
+
+	ALIGN_5
+
+.L2_12:
+
+	KERNEL4x2_SUB
+	KERNEL4x2_SUB
+	KERNEL4x2_SUB
+	KERNEL4x2_SUB
+
+	KERNEL4x2_SUB
+	KERNEL4x2_SUB
+	KERNEL4x2_SUB
+	KERNEL4x2_SUB
+
+	dec	%rax
+	jne	.L2_12
+
+
+.L2_16:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_19
+
+	ALIGN_4
+
+.L2_17:
+
+	KERNEL4x2_SUB
+
+	dec	%rax
+	jne	.L2_17
+	ALIGN_4
+
+
+.L2_19:
+
+	SAVE4x2
+
+	decq	I			# i --
+	jg	.L2_11
+	ALIGN_4	
+
+/**************************************************************************
+* Rest of M 
+***************************************************************************/
+.L2_20:
+	// Test rest of M
+
+	testq	$3, M
+	jz	.L2_100			// to next 16 lines of N
+
+
+.L2_30:
+	testq	$2, M		
+	jz	.L2_40
+
+	ALIGN_4
+
+.L2_31:
+        movq    B, BO             // first buffer to BO
+        addq    $12 * SIZE, BO
+
+	INIT2x2
+
+        movq    K, %rax
+
+	sarq	$3, %rax
+	je	.L2_36
+	ALIGN_4
+
+.L2_32:
+
+	KERNEL2x2_SUB
+	KERNEL2x2_SUB
+	KERNEL2x2_SUB
+	KERNEL2x2_SUB
+
+	KERNEL2x2_SUB
+	KERNEL2x2_SUB
+	KERNEL2x2_SUB
+	KERNEL2x2_SUB
+
+	dec %rax
+	jne	.L2_32
+
+.L2_36:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_39
+
+	ALIGN_4
+
+.L2_37:
+
+	KERNEL2x2_SUB
+
+	dec %rax
+	jne	.L2_37
+
+
+.L2_39:
+
+	SAVE2x2
+
+.L2_40:
+	testq	$1, M		
+	jz	.L2_100		// to next 3 lines of N
+
+.L2_41:
+        movq    B, BO             // first buffer to BO
+        addq    $12 * SIZE, BO
+
+	INIT1x2
+
+        movq    K, %rax
+
+	sarq	$3,%rax
+	je	.L2_46
+
+	ALIGN_4
+
+.L2_42:
+
+	KERNEL1x2_SUB
+	KERNEL1x2_SUB
+	KERNEL1x2_SUB
+	KERNEL1x2_SUB
+
+	KERNEL1x2_SUB
+	KERNEL1x2_SUB
+	KERNEL1x2_SUB
+	KERNEL1x2_SUB
+
+	dec %rax
+	jne	.L2_42
+
+.L2_46:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_49
+
+	ALIGN_4
+
+.L2_47:
+
+	KERNEL1x2_SUB
+
+	dec	%rax
+	jne	.L2_47
+
+.L2_49:
+
+	SAVE1x2
+
+.L2_100:
+
+	movq	K, %rax
+	salq	$1, %rax		// * 2
+	leaq	(B , %rax, SIZE), B
+
+/***************************************************************************************************************/
+
+.L1_0:
+
+	movq	Nmod12, J		
+	testq	$1, J
+	je	.L999
+
+.L1_10:
+	movq	C, CO1
+	leaq	(C, LDC, 1), C		// c += 1 * ldc
+
+	
+	movq	A, AO		 	// aoffset = a
+	addq	$16 * SIZE, AO
+
+	movq	M,  I
+	sarq	$2, I			// i = m / 4
+	je	.L1_20
+
+	ALIGN_4
+
+.L1_11:
+        movq    B, BO        
+        addq    $12 * SIZE, BO
+
+	INIT4x1
+
+        movq    K, %rax
+
+	sarq	$3, %rax			//  K / 8
+	je	.L1_16
+
+	ALIGN_5
+
+.L1_12:
+
+	KERNEL4x1
+
+	dec	%rax
+	jne	.L1_12
+
+
+.L1_16:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_19
+
+	ALIGN_4
+
+.L1_17:
+
+	KERNEL4x1_SUB
+
+	dec	%rax
+	jne	.L1_17
+	ALIGN_4
+
+
+.L1_19:
+
+	SAVE4x1
+
+	decq	I			# i --
+	jg	.L1_11
+
+/**************************************************************************
+* Rest of M 
+***************************************************************************/
+.L1_20:
+	// Test rest of M
+
+	testq	$3, M
+	jz	.L1_100	
+
+
+.L1_30:
+	testq	$2, M		
+	jz	.L1_40
+
+	ALIGN_4
+
+.L1_31:
+        movq    B, BO             // first buffer to BO
+        addq    $12 * SIZE, BO
+
+	INIT2x1
+
+        movq    K, %rax
+
+	sarq	$3, %rax
+	je	.L1_36
+	ALIGN_4
+
+.L1_32:
+
+	KERNEL2x1_SUB
+	KERNEL2x1_SUB
+	KERNEL2x1_SUB
+	KERNEL2x1_SUB
+
+	KERNEL2x1_SUB
+	KERNEL2x1_SUB
+	KERNEL2x1_SUB
+	KERNEL2x1_SUB
+
+
+	dec %rax
+	jne	.L1_32
+
+.L1_36:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_39
+
+	ALIGN_4
+
+.L1_37:
+
+	KERNEL2x1_SUB
+
+	dec %rax
+	jne	.L1_37
+
+.L1_39:
+
+	SAVE2x1
+
+.L1_40:
+	testq	$1, M		
+	jz	.L1_100		// to next 3 lines of N
+
+
+.L1_41:
+        movq    B, BO             // first buffer to BO
+        addq    $12 * SIZE, BO
+
+	INIT1x1
+
+        movq    K, %rax
+
+	sarq	$3,%rax
+	je	.L1_46
+
+	ALIGN_4
+
+.L1_42:
+
+	KERNEL1x1_SUB
+	KERNEL1x1_SUB
+	KERNEL1x1_SUB
+	KERNEL1x1_SUB
+
+	KERNEL1x1_SUB
+	KERNEL1x1_SUB
+	KERNEL1x1_SUB
+	KERNEL1x1_SUB
+
+	dec %rax
+	jne	.L1_42
+
+.L1_46:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_49
+
+	ALIGN_4
+
+.L1_47:
+
+	KERNEL1x1_SUB
+
+	dec	%rax
+	jne	.L1_47
+
+
+.L1_49:
+
+	SAVE1x1
+
+.L1_100:
+
+
+
+
+.L999:
+	vzeroupper
+
+	movq   		SP, %rsp
+	movq	   (%rsp), %rbx
+	movq	  8(%rsp), %rbp
+	movq	 16(%rsp), %r12
+	movq	 24(%rsp), %r13
+	movq	 32(%rsp), %r14
+	movq	 40(%rsp), %r15
+
+#ifdef WINDOWS_ABI
+	movq	 48(%rsp), %rdi
+	movq	 56(%rsp), %rsi
+	vmovups	 64(%rsp), %xmm6
+	vmovups	 80(%rsp), %xmm7
+	vmovups	 96(%rsp), %xmm8
+	vmovups	112(%rsp), %xmm9
+	vmovups	128(%rsp), %xmm10
+	vmovups	144(%rsp), %xmm11
+	vmovups	160(%rsp), %xmm12
+	vmovups	176(%rsp), %xmm13
+	vmovups	192(%rsp), %xmm14
+	vmovups	208(%rsp), %xmm15
+#endif
+
+	addq	$STACKSIZE, %rsp
+	ret
+
+	EPILOGUE
+
+
+#else
+/*************************************************************************************
+* TRMM Kernel
+*************************************************************************************/
+
+
+	PROLOGUE
+	PROFCODE
+	
+	subq	$STACKSIZE, %rsp
+	movq	%rbx,   (%rsp)
+	movq	%rbp,  8(%rsp)
+	movq	%r12, 16(%rsp)
+	movq	%r13, 24(%rsp)
+	movq	%r14, 32(%rsp)
+	movq	%r15, 40(%rsp)
+
+	vzeroupper
+
+#ifdef WINDOWS_ABI
+	movq	%rdi,    48(%rsp)
+	movq	%rsi,    56(%rsp)
+	vmovups	%xmm6,   64(%rsp)
+	vmovups	%xmm7,   80(%rsp)
+	vmovups	%xmm8,   96(%rsp)
+	vmovups	%xmm9,  112(%rsp)
+	vmovups	%xmm10, 128(%rsp)
+	vmovups	%xmm11, 144(%rsp)
+	vmovups	%xmm12, 160(%rsp)
+	vmovups	%xmm13, 176(%rsp)
+	vmovups	%xmm14, 192(%rsp)
+	vmovups	%xmm15, 208(%rsp)
+
+	movq	ARG1,      OLD_M
+	movq	ARG2,      OLD_N
+	movq	ARG3,      OLD_K
+	movq	OLD_A,     A
+	movq	OLD_B,     B
+	movq	OLD_C,     C
+	movq	OLD_LDC,   LDC
+#ifdef TRMMKERNEL
+	vmovsd	OLD_OFFSET, %xmm12
+#endif
+	vmovups	%xmm3, %xmm0
+
+#else
+	movq	STACKSIZE +  8(%rsp), LDC
+#ifdef TRMMKERNEL
+	vmovsd	STACKSIZE + 16(%rsp), %xmm12
+#endif
+
+#endif
+
+	movq    %rsp, SP      # save old stack
+        subq    $128 + L_BUFFER_SIZE, %rsp
+        andq    $-4096, %rsp    # align stack
+
+        STACK_TOUCH
+
+	cmpq	$ 0, OLD_M
+	je	.L999
+
+	cmpq	$ 0, OLD_N
+	je	.L999
+
+	cmpq	$ 0, OLD_K
+	je	.L999
+
+	movq	OLD_M, M
+	movq	OLD_N, N
+	movq	OLD_K, K
+
+	vmovsd	 %xmm0, ALPHA
+
+	salq	$BASE_SHIFT, LDC
+
+	movq    N, %rax
+        xorq    %rdx, %rdx
+        movq    $8,  %rdi
+        divq    %rdi                     //    N / 8
+        movq    %rax, Ndiv12             //    N / 8
+        movq    %rdx, Nmod12             //    N % 8
+
+#ifdef TRMMKERNEL
+        vmovsd  %xmm12, OFFSET
+        vmovsd  %xmm12, KK
+#ifndef LEFT
+        negq    KK
+#endif  
+#endif
+
+/*************************************************************************************************/
+.L8_0:
+	movq	Ndiv12,  J
+	cmpq	$ 0, J
+	je	.L4_0
+	ALIGN_4
+
+.L8_10:
+	movq	C, CO1
+	leaq	(C, LDC, 8), C		// c += 8 * ldc
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        movq    OFFSET, %rax
+        movq    %rax, KK
+#endif
+
+	
+	movq	A, AO		 	// aoffset = a
+	addq	$16 * SIZE, AO
+
+	movq	M,  I
+	sarq	$2, I			// i = m / 4
+	je	.L8_20
+
+	ALIGN_4
+
+.L8_11:
+
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    B, BO        
+        addq    $12 * SIZE, BO
+#else
+        movq    B, BO        
+        addq    $12 * SIZE, BO
+        movq    KK, %rax
+	salq	$3, %rax		// rax * SIZE
+	leaq	(BO,%rax,8), BO		// add number of values in B
+	leaq	(AO,%rax,4), AO		// add number of values in A
+#endif
+
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $4, %rax        // number of values in AO
+#else
+        addq    $8, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+	sarq	$3, %rax			//  K / 8
+	cmpq    $2, %rax
+	jl	.L8_13
+
+
+	KERNEL4x8_I
+	KERNEL4x8_M2
+	KERNEL4x8_M1
+	KERNEL4x8_M2
+
+	KERNEL4x8_M1
+	KERNEL4x8_M2
+	KERNEL4x8_M1
+	KERNEL4x8_M2
+
+	subq $2, %rax
+	je	.L8_12a
+
+	ALIGN_5
+
+.L8_12:
+
+	KERNEL4x8_M1
+	KERNEL4x8_M2
+	KERNEL4x8_M1
+	KERNEL4x8_M2
+
+	KERNEL4x8_M1
+	KERNEL4x8_M2
+	KERNEL4x8_M1
+	KERNEL4x8_M2
+
+	dec	%rax
+	jne	.L8_12
+
+.L8_12a:
+
+	KERNEL4x8_M1
+	KERNEL4x8_M2
+	KERNEL4x8_M1
+	KERNEL4x8_M2
+
+	KERNEL4x8_M1
+	KERNEL4x8_M2
+	KERNEL4x8_M1
+	KERNEL4x8_E
+
+	jmp .L8_16
+
+
+.L8_13:
+
+	test $1, %rax
+	jz .L8_14
+
+	KERNEL4x8_I
+	KERNEL4x8_M2
+	KERNEL4x8_M1
+	KERNEL4x8_M2
+
+	KERNEL4x8_M1
+	KERNEL4x8_M2
+	KERNEL4x8_M1
+	KERNEL4x8_E
+
+	jmp .L8_16
+
+
+.L8_14:
+
+	INIT4x8
+
+
+.L8_16:
+        movq    KKK, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L8_19
+
+	ALIGN_4
+
+.L8_17:
+
+	KERNEL4x8_SUB
+
+	dec	%rax
+	jne	.L8_17
+	ALIGN_4
+
+
+.L8_19:
+
+	SAVE4x8
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+	salq	$3, %rax			// rax + SIZE
+        leaq    (BO, %rax, 8), BO		// number of values in B
+        leaq    (AO, %rax, 4), AO		// number of values in A
+#endif
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $4, KK				// number of values in A
+#endif
+
+	decq	I			# i --
+	jg	.L8_11
+	ALIGN_4	
+
+/**************************************************************************
+* Rest of M 
+***************************************************************************/
+.L8_20:
+	// Test rest of M
+
+	testq	$3, M
+	jz	.L8_100			// to next 16 lines of N
+
+
+.L8_30:
+	testq	$2, M		
+	jz	.L8_40
+
+	ALIGN_4
+
+.L8_31:
+
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    B, BO        
+        addq    $12 * SIZE, BO
+#else
+        movq    B, BO        
+        addq    $12 * SIZE, BO
+        movq    KK, %rax
+	salq	$3, %rax		// rax * SIZE
+	leaq	(BO,%rax,8), BO		// add number of values in B
+	leaq	(AO,%rax,2), AO		// add number of values in A
+#endif
+
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $2, %rax        // number of values in AO
+#else
+        addq    $8, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+	INIT2x8
+
+	sarq	$3, %rax
+	je	.L8_36
+	ALIGN_4
+
+.L8_32:
+
+	KERNEL2x8_SUB
+	KERNEL2x8_SUB
+	KERNEL2x8_SUB
+	KERNEL2x8_SUB
+
+	KERNEL2x8_SUB
+	KERNEL2x8_SUB
+	KERNEL2x8_SUB
+	KERNEL2x8_SUB
+
+	dec %rax
+	jne	.L8_32
+	ALIGN_4
+
+.L8_36:
+        movq    KKK, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L8_39
+
+	ALIGN_4
+
+.L8_37:
+
+	KERNEL2x8_SUB
+
+	dec %rax
+	jne	.L8_37
+
+
+.L8_39:
+
+	SAVE2x8
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+	salq	$3, %rax			// rax + SIZE
+        leaq    (BO, %rax, 8), BO		// number of values in B
+        leaq    (AO, %rax, 2), AO		// number of values in A
+#endif
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $2, KK				// number of values in A
+#endif
+
+
+.L8_40:
+	testq	$1, M		
+	jz	.L8_100		// to next 3 lines of N
+
+	ALIGN_4
+
+.L8_41:
+
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    B, BO        
+        addq    $12 * SIZE, BO
+#else
+        movq    B, BO        
+        addq    $12 * SIZE, BO
+        movq    KK, %rax
+	salq	$3, %rax		// rax * SIZE
+	leaq	(BO,%rax,8), BO		// add number of values in B
+	leaq	(AO,%rax,1), AO		// add number of values in A
+#endif
+
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $1, %rax        // number of values in AO
+#else
+        addq    $8, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+	INIT1x8
+
+	sarq	$3,%rax
+	je	.L8_46
+
+	ALIGN_4
+
+.L8_42:
+
+	KERNEL1x8_SUB
+	KERNEL1x8_SUB
+	KERNEL1x8_SUB
+	KERNEL1x8_SUB
+
+	KERNEL1x8_SUB
+	KERNEL1x8_SUB
+	KERNEL1x8_SUB
+	KERNEL1x8_SUB
+
+	dec %rax
+	jne	.L8_42
+	ALIGN_4
+
+.L8_46:
+        movq    KKK, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L8_49
+
+	ALIGN_4
+
+.L8_47:
+
+	KERNEL1x8_SUB
+
+	dec	%rax
+	jne	.L8_47
+	ALIGN_4
+
+
+.L8_49:
+
+	SAVE1x8
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+	salq	$3, %rax			// rax + SIZE
+        leaq    (BO, %rax, 8), BO		// number of values in B
+        leaq    (AO, %rax, 1), AO		// number of values in A
+#endif
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $1, KK				// number of values in A
+#endif
+
+.L8_100:
+
+#if defined(TRMMKERNEL) && !defined(LEFT)
+        addq    $8, KK				// number of values in B
+#endif
+
+
+	decq	J			// j --
+	jg	.L8_10
+
+
+
+
+
+/*************************************************************************************************/
+.L4_0:
+	movq	Nmod12, J		
+	testq	$4, J
+	je	.L2_0
+	ALIGN_4
+
+.L4_10:
+	movq	C, CO1
+	leaq	(C, LDC, 4), C		// c += 4 * ldc
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        movq    OFFSET, %rax
+        movq    %rax, KK
+#endif
+
+	
+	movq	A, AO		 	// aoffset = a
+	addq	$16 * SIZE, AO
+
+	movq	M,  I
+	sarq	$2, I			// i = m / 4
+	je	.L4_20
+
+	ALIGN_4
+
+.L4_11:
+
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    B, BO        
+        addq    $12 * SIZE, BO
+#else
+        movq    B, BO        
+        addq    $12 * SIZE, BO
+        movq    KK, %rax
+	salq	$3, %rax		// rax * SIZE
+	leaq	(BO,%rax,4), BO		// add number of values in B
+	leaq	(AO,%rax,4), AO		// add number of values in A
+#endif
+
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $4, %rax        // number of values in AO
+#else
+        addq    $4, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+	sarq	$3, %rax			//  K / 8
+	cmpq    $2, %rax
+	jl	.L4_13
+
+
+	KERNEL4x4_I
+	KERNEL4x4_M2
+	KERNEL4x4_M1
+	KERNEL4x4_M2
+
+	KERNEL4x4_M1
+	KERNEL4x4_M2
+	KERNEL4x4_M1
+	KERNEL4x4_M2
+
+	subq $2, %rax
+	je	.L4_12a
+
+	ALIGN_5
+
+.L4_12:
+
+	KERNEL4x4_M1
+	KERNEL4x4_M2
+	KERNEL4x4_M1
+	KERNEL4x4_M2
+
+	KERNEL4x4_M1
+	KERNEL4x4_M2
+	KERNEL4x4_M1
+	KERNEL4x4_M2
+
+	dec	%rax
+	jne	.L4_12
+
+.L4_12a:
+
+	KERNEL4x4_M1
+	KERNEL4x4_M2
+	KERNEL4x4_M1
+	KERNEL4x4_M2
+
+	KERNEL4x4_M1
+	KERNEL4x4_M2
+	KERNEL4x4_M1
+	KERNEL4x4_E
+
+	jmp .L4_16
+
+
+.L4_13:
+
+	test $1, %rax
+	jz .L4_14
+
+	KERNEL4x4_I
+	KERNEL4x4_M2
+	KERNEL4x4_M1
+	KERNEL4x4_M2
+
+	KERNEL4x4_M1
+	KERNEL4x4_M2
+	KERNEL4x4_M1
+	KERNEL4x4_E
+
+	jmp .L4_16
+
+
+.L4_14:
+
+	INIT4x4
+
+
+.L4_16:
+        movq    KKK, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L4_19
+
+	ALIGN_4
+
+.L4_17:
+
+	KERNEL4x4_SUB
+
+	dec	%rax
+	jne	.L4_17
+	ALIGN_4
+
+
+.L4_19:
+
+	SAVE4x4
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+	salq	$3, %rax			// rax + SIZE
+        leaq    (BO, %rax, 4), BO		// number of values in B
+        leaq    (AO, %rax, 4), AO		// number of values in A
+#endif
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $4, KK				// number of values in A
+#endif
+
+	decq	I			# i --
+	jg	.L4_11
+	ALIGN_4	
+
+/**************************************************************************
+* Rest of M 
+***************************************************************************/
+.L4_20:
+	// Test rest of M
+
+	testq	$3, M
+	jz	.L4_100			// to next 16 lines of N
+
+
+.L4_30:
+	testq	$2, M		
+	jz	.L4_40
+
+	ALIGN_4
+
+.L4_31:
+
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    B, BO        
+        addq    $12 * SIZE, BO
+#else
+        movq    B, BO        
+        addq    $12 * SIZE, BO
+        movq    KK, %rax
+	salq	$3, %rax		// rax * SIZE
+	leaq	(BO,%rax,4), BO		// add number of values in B
+	leaq	(AO,%rax,2), AO		// add number of values in A
+#endif
+
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $2, %rax        // number of values in AO
+#else
+        addq    $4, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+	INIT2x4
+
+	sarq	$3, %rax
+	je	.L4_36
+	ALIGN_4
+
+.L4_32:
+
+	KERNEL2x4_SUB
+	KERNEL2x4_SUB
+	KERNEL2x4_SUB
+	KERNEL2x4_SUB
+
+	KERNEL2x4_SUB
+	KERNEL2x4_SUB
+	KERNEL2x4_SUB
+	KERNEL2x4_SUB
+
+	dec %rax
+	jne	.L4_32
+	ALIGN_4
+
+.L4_36:
+        movq    KKK, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L4_39
+
+	ALIGN_4
+
+.L4_37:
+
+	KERNEL2x4_SUB
+
+	dec %rax
+	jne	.L4_37
+
+
+.L4_39:
+
+	SAVE2x4
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+	salq	$3, %rax			// rax + SIZE
+        leaq    (BO, %rax, 4), BO		// number of values in B
+        leaq    (AO, %rax, 2), AO		// number of values in A
+#endif
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $2, KK				// number of values in A
+#endif
+
+
+.L4_40:
+	testq	$1, M		
+	jz	.L4_100		// to next 3 lines of N
+
+	ALIGN_4
+
+.L4_41:
+
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    B, BO        
+        addq    $12 * SIZE, BO
+#else
+        movq    B, BO        
+        addq    $12 * SIZE, BO
+        movq    KK, %rax
+	salq	$3, %rax		// rax * SIZE
+	leaq	(BO,%rax,4), BO		// add number of values in B
+	leaq	(AO,%rax,1), AO		// add number of values in A
+#endif
+
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $1, %rax        // number of values in AO
+#else
+        addq    $4, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+	INIT1x4
+
+	sarq	$3,%rax
+	je	.L4_46
+
+	ALIGN_4
+
+.L4_42:
+
+	KERNEL1x4_SUB
+	KERNEL1x4_SUB
+	KERNEL1x4_SUB
+	KERNEL1x4_SUB
+
+	KERNEL1x4_SUB
+	KERNEL1x4_SUB
+	KERNEL1x4_SUB
+	KERNEL1x4_SUB
+
+	dec %rax
+	jne	.L4_42
+	ALIGN_4
+
+.L4_46:
+        movq    KKK, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L4_49
+
+	ALIGN_4
+
+.L4_47:
+
+	KERNEL1x4_SUB
+
+	dec	%rax
+	jne	.L4_47
+	ALIGN_4
+
+
+.L4_49:
+
+	SAVE1x4
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+	salq	$3, %rax			// rax + SIZE
+        leaq    (BO, %rax, 4), BO		// number of values in B
+        leaq    (AO, %rax, 1), AO		// number of values in A
+#endif
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $1, KK				// number of values in A
+#endif
+
+.L4_100:
+
+#if defined(TRMMKERNEL) && !defined(LEFT)
+        addq    $4, KK				// number of values in B
+#endif
+
+
+	movq	K, %rax
+	salq	$2, %rax		// * 4
+	leaq	(B , %rax, SIZE), B
+
+
+
+
+/***************************************************************************************************************/
+
+.L2_0:
+
+	movq	Nmod12, J		
+	testq	$2, J
+	je	.L1_0
+
+.L2_10:
+	movq	C, CO1
+	leaq	(C, LDC, 2), C		// c += 2 * ldc
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        movq    OFFSET, %rax
+        movq    %rax, KK
+#endif
+
+
+	
+	movq	A, AO		 	// aoffset = a
+	addq	$16 * SIZE, AO
+
+	movq	M,  I
+	sarq	$2, I			// i = m / 4
+	je	.L2_20
+
+	ALIGN_4
+
+.L2_11:
+
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    B, BO        
+        addq    $12 * SIZE, BO
+#else
+        movq    B, BO        
+        addq    $12 * SIZE, BO
+        movq    KK, %rax
+	salq	$3, %rax		// rax * SIZE
+	leaq	(BO,%rax,2), BO		// add number of values in B
+	leaq	(AO,%rax,4), AO		// add number of values in A
+#endif
+
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $4, %rax        // number of values in AO
+#else
+        addq    $2, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+	INIT4x2
+
+	sarq $3, %rax			//  K / 8
+
+	je	.L2_16
+
+	ALIGN_5
+
+.L2_12:
+
+	KERNEL4x2_SUB
+	KERNEL4x2_SUB
+	KERNEL4x2_SUB
+	KERNEL4x2_SUB
+
+	KERNEL4x2_SUB
+	KERNEL4x2_SUB
+	KERNEL4x2_SUB
+	KERNEL4x2_SUB
+
+	dec	%rax
+	jne	.L2_12
+
+
+.L2_16:
+        movq    KKK, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_19
+
+	ALIGN_4
+
+.L2_17:
+
+	KERNEL4x2_SUB
+
+	dec	%rax
+	jne	.L2_17
+	ALIGN_4
+
+
+.L2_19:
+
+	SAVE4x2
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+	salq	$3, %rax			// rax + SIZE
+        leaq    (BO, %rax, 2), BO		// number of values in B
+        leaq    (AO, %rax, 4), AO		// number of values in A
+#endif
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $4, KK				// number of values in A
+#endif
+
+
+	decq	I			# i --
+	jg	.L2_11
+	ALIGN_4	
+
+/**************************************************************************
+* Rest of M 
+***************************************************************************/
+.L2_20:
+	// Test rest of M
+
+	testq	$3, M
+	jz	.L2_100			// to next 16 lines of N
+
+
+.L2_30:
+	testq	$2, M		
+	jz	.L2_40
+
+	ALIGN_4
+
+.L2_31:
+
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    B, BO        
+        addq    $12 * SIZE, BO
+#else
+        movq    B, BO        
+        addq    $12 * SIZE, BO
+        movq    KK, %rax
+	salq	$3, %rax		// rax * SIZE
+	leaq	(BO,%rax,2), BO		// add number of values in B
+	leaq	(AO,%rax,2), AO		// add number of values in A
+#endif
+
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $2, %rax        // number of values in AO
+#else
+        addq    $2, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+	INIT2x2
+
+	sarq	$3, %rax
+	je	.L2_36
+	ALIGN_4
+
+.L2_32:
+
+	KERNEL2x2_SUB
+	KERNEL2x2_SUB
+	KERNEL2x2_SUB
+	KERNEL2x2_SUB
+
+	KERNEL2x2_SUB
+	KERNEL2x2_SUB
+	KERNEL2x2_SUB
+	KERNEL2x2_SUB
+
+	dec %rax
+	jne	.L2_32
+
+.L2_36:
+        movq    KKK, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_39
+
+	ALIGN_4
+
+.L2_37:
+
+	KERNEL2x2_SUB
+
+	dec %rax
+	jne	.L2_37
+
+
+.L2_39:
+
+	SAVE2x2
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+	salq	$3, %rax			// rax + SIZE
+        leaq    (BO, %rax, 2), BO		// number of values in B
+        leaq    (AO, %rax, 2), AO		// number of values in A
+#endif
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $2, KK				// number of values in A
+#endif
+
+
+.L2_40:
+	testq	$1, M		
+	jz	.L2_100		// to next 3 lines of N
+
+.L2_41:
+
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    B, BO        
+        addq    $12 * SIZE, BO
+#else
+        movq    B, BO        
+        addq    $12 * SIZE, BO
+        movq    KK, %rax
+	salq	$3, %rax		// rax * SIZE
+	leaq	(BO,%rax,2), BO		// add number of values in B
+	leaq	(AO,%rax,1), AO		// add number of values in A
+#endif
+
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $1, %rax        // number of values in AO
+#else
+        addq    $2, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+	INIT1x2
+
+	sarq	$3,%rax
+	je	.L2_46
+
+	ALIGN_4
+
+.L2_42:
+
+	KERNEL1x2_SUB
+	KERNEL1x2_SUB
+	KERNEL1x2_SUB
+	KERNEL1x2_SUB
+
+	KERNEL1x2_SUB
+	KERNEL1x2_SUB
+	KERNEL1x2_SUB
+	KERNEL1x2_SUB
+
+	dec %rax
+	jne	.L2_42
+
+.L2_46:
+        movq    KKK, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_49
+
+	ALIGN_4
+
+.L2_47:
+
+	KERNEL1x2_SUB
+
+	dec	%rax
+	jne	.L2_47
+
+.L2_49:
+
+	SAVE1x2
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+	salq	$3, %rax			// rax * SIZE
+        leaq    (BO, %rax, 2), BO		// number of values in B
+        leaq    (AO, %rax, 1), AO		// number of values in A
+#endif
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $1, KK				// number of values in A
+#endif
+
+
+.L2_100:
+
+
+#if defined(TRMMKERNEL) && !defined(LEFT)
+        addq    $2, KK				// number of values in B
+#endif
+
+	movq	K, %rax
+	salq	$1, %rax		// * 2
+	leaq	(B , %rax, SIZE), B
+
+/***************************************************************************************************************/
+
+.L1_0:
+
+	movq	Nmod12, J		
+	testq	$1, J
+	je	.L999
+
+.L1_10:
+	movq	C, CO1
+	leaq	(C, LDC, 1), C		// c += 1 * ldc
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        movq    OFFSET, %rax
+        movq    %rax, KK
+#endif
+	
+	movq	A, AO		 	// aoffset = a
+	addq	$16 * SIZE, AO
+
+	movq	M,  I
+	sarq	$2, I			// i = m / 4
+	je	.L1_20
+
+	ALIGN_4
+
+.L1_11:
+
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    B, BO        
+        addq    $12 * SIZE, BO
+#else
+        movq    B, BO        
+        addq    $12 * SIZE, BO
+        movq    KK, %rax
+	salq	$3, %rax		// rax * SIZE
+	leaq	(BO,%rax,1), BO		// add number of values in B
+	leaq	(AO,%rax,4), AO		// add number of values in A
+#endif
+
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $4, %rax        // number of values in AO
+#else
+        addq    $1, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+	INIT4x1
+
+	sarq	$3, %rax			//  K / 8
+	je	.L1_16
+
+	ALIGN_5
+
+.L1_12:
+
+	KERNEL4x1
+
+	dec	%rax
+	jne	.L1_12
+
+
+.L1_16:
+        movq    KKK, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_19
+
+	ALIGN_4
+
+.L1_17:
+
+	KERNEL4x1_SUB
+
+	dec	%rax
+	jne	.L1_17
+	ALIGN_4
+
+
+.L1_19:
+
+	SAVE4x1
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+	salq	$3, %rax			// rax * SIZE
+        leaq    (BO, %rax, 1), BO		// number of values in B
+        leaq    (AO, %rax, 4), AO		// number of values in A
+#endif
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $4, KK				// number of values in A
+#endif
+
+
+	decq	I			# i --
+	jg	.L1_11
+
+/**************************************************************************
+* Rest of M 
+***************************************************************************/
+.L1_20:
+	// Test rest of M
+
+	testq	$3, M
+	jz	.L1_100	
+
+
+.L1_30:
+	testq	$2, M		
+	jz	.L1_40
+
+	ALIGN_4
+
+.L1_31:
+
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    B, BO        
+        addq    $12 * SIZE, BO
+#else
+        movq    B, BO        
+        addq    $12 * SIZE, BO
+        movq    KK, %rax
+	salq	$3, %rax		// rax * SIZE
+	leaq	(BO,%rax,1), BO		// add number of values in B
+	leaq	(AO,%rax,2), AO		// add number of values in A
+#endif
+
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $2, %rax        // number of values in AO
+#else
+        addq    $1, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+	INIT2x1
+
+	sarq	$3, %rax
+	je	.L1_36
+	ALIGN_4
+
+.L1_32:
+
+	KERNEL2x1_SUB
+	KERNEL2x1_SUB
+	KERNEL2x1_SUB
+	KERNEL2x1_SUB
+
+	KERNEL2x1_SUB
+	KERNEL2x1_SUB
+	KERNEL2x1_SUB
+	KERNEL2x1_SUB
+
+
+	dec %rax
+	jne	.L1_32
+
+.L1_36:
+        movq    KKK, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_39
+
+	ALIGN_4
+
+.L1_37:
+
+	KERNEL2x1_SUB
+
+	dec %rax
+	jne	.L1_37
+
+.L1_39:
+
+	SAVE2x1
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+	salq	$3, %rax			// rax * SIZE
+        leaq    (BO, %rax, 1), BO		// number of values in B
+        leaq    (AO, %rax, 2), AO		// number of values in A
+#endif
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $2, KK				// number of values in A
+#endif
+
+
+.L1_40:
+	testq	$1, M		
+	jz	.L1_100		// to next 3 lines of N
+
+
+.L1_41:
+
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    B, BO        
+        addq    $12 * SIZE, BO
+#else
+        movq    B, BO        
+        addq    $12 * SIZE, BO
+        movq    KK, %rax
+	salq	$3, %rax		// rax * SIZE
+	leaq	(BO,%rax,1), BO		// add number of values in B
+	leaq	(AO,%rax,1), AO		// add number of values in A
+#endif
+
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $1, %rax        // number of values in AO
+#else
+        addq    $1, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+	INIT1x1
+
+	sarq	$3,%rax
+	je	.L1_46
+
+	ALIGN_4
+
+.L1_42:
+
+	KERNEL1x1_SUB
+	KERNEL1x1_SUB
+	KERNEL1x1_SUB
+	KERNEL1x1_SUB
+
+	KERNEL1x1_SUB
+	KERNEL1x1_SUB
+	KERNEL1x1_SUB
+	KERNEL1x1_SUB
+
+	dec %rax
+	jne	.L1_42
+
+.L1_46:
+        movq    KKK, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_49
+
+	ALIGN_4
+
+.L1_47:
+
+	KERNEL1x1_SUB
+
+	dec	%rax
+	jne	.L1_47
+
+
+.L1_49:
+
+	SAVE1x1
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+	salq	$3, %rax			// rax * SIZE
+        leaq    (BO, %rax, 1), BO		// number of values in B
+        leaq    (AO, %rax, 1), AO		// number of values in A
+#endif
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $1, KK				// number of values in A
+#endif
+
+
+
+.L1_100:
+
+
+#if defined(TRMMKERNEL) && !defined(LEFT)
+        addq    $1, KK				// number of values in B
+#endif
+
+
+
+.L999:
+
+	vzeroupper
+
+	movq   		SP, %rsp
+	movq	   (%rsp), %rbx
+	movq	  8(%rsp), %rbp
+	movq	 16(%rsp), %r12
+	movq	 24(%rsp), %r13
+	movq	 32(%rsp), %r14
+	movq	 40(%rsp), %r15
+
+#ifdef WINDOWS_ABI
+	movq	 48(%rsp), %rdi
+	movq	 56(%rsp), %rsi
+	vmovups	 64(%rsp), %xmm6
+	vmovups	 80(%rsp), %xmm7
+	vmovups	 96(%rsp), %xmm8
+	vmovups	112(%rsp), %xmm9
+	vmovups	128(%rsp), %xmm10
+	vmovups	144(%rsp), %xmm11
+	vmovups	160(%rsp), %xmm12
+	vmovups	176(%rsp), %xmm13
+	vmovups	192(%rsp), %xmm14
+	vmovups	208(%rsp), %xmm15
+#endif
+
+	addq	$STACKSIZE, %rsp
+	ret
+
+	EPILOGUE
+
+
+
+
+
+#endif
diff --git a/kernel/x86_64/dgemm_kernel_4x8_skylakex_2.c b/kernel/x86_64/dgemm_kernel_4x8_skylakex_2.c
index 90a4c2b1d..a5daffb94 100644
--- a/kernel/x86_64/dgemm_kernel_4x8_skylakex_2.c
+++ b/kernel/x86_64/dgemm_kernel_4x8_skylakex_2.c
@@ -1,670 +1,670 @@
-#include "common.h"
-#include <stdint.h>
-#include <immintrin.h>
-
-//register usage: zmm3 for alpha, zmm0-zmm2 and zmm4-zmm7 for temporary use, zmm8-zmm31 for accumulators.
-
-/* row-major c_block */
-#define INNER_KERNEL_k1m1n8 \
-    "prefetcht0 384(%1);"\
-    "vmovupd (%1),%%zmm5; addq $64,%1;"\
-    "vbroadcastsd   (%0),%%zmm4;vfmadd231pd %%zmm5,%%zmm4,%%zmm8;"
-
-#define INNER_KERNEL_k1m2n8 \
-    INNER_KERNEL_k1m1n8\
-    "vbroadcastsd  8(%0),%%zmm4;vfmadd231pd %%zmm5,%%zmm4,%%zmm9;"
-
-#define INNER_KERNEL_k1m1n16 \
-    "prefetcht0 128(%1); prefetcht0 128(%1,%%r12,2);"\
-    "vmovupd (%1),%%zmm5; vmovupd (%1,%%r12,2),%%zmm6; addq $64,%1;"\
-    "vbroadcastsd   (%0),%%zmm4;vfmadd231pd %%zmm5,%%zmm4,%%zmm8; vfmadd231pd %%zmm6,%%zmm4,%%zmm9;"
-
-#define INNER_KERNEL_k1m2n16 \
-    INNER_KERNEL_k1m1n16\
-    "vbroadcastsd  8(%0),%%zmm4;vfmadd231pd %%zmm5,%%zmm4,%%zmm10;vfmadd231pd %%zmm6,%%zmm4,%%zmm11;"
-
-#define INNER_KERNEL_k1m1n24 \
-    "prefetcht0 128(%1); prefetcht0 128(%1,%%r12,2); prefetcht0 128(%1,%%r12,4);"\
-    "vmovupd (%1),%%zmm5; vmovupd (%1,%%r12,2),%%zmm6; vmovupd (%1,%%r12,4),%%zmm7; addq $64,%1;"\
-    "vbroadcastsd   (%0),%%zmm4;vfmadd231pd %%zmm5,%%zmm4,%%zmm8; vfmadd231pd %%zmm6,%%zmm4,%%zmm9; vfmadd231pd %%zmm7,%%zmm4,%%zmm10;"
-
-#define INNER_KERNEL_k1m2n24 \
-    INNER_KERNEL_k1m1n24\
-    "vbroadcastsd  8(%0),%%zmm4;vfmadd231pd %%zmm5,%%zmm4,%%zmm11;vfmadd231pd %%zmm6,%%zmm4,%%zmm12;vfmadd231pd %%zmm7,%%zmm4,%%zmm13;"
-
-/* row-major z-partition c_block */
-#define INNER_KERNEL_k1m4n8 \
-    "vbroadcastf32x4 (%0),%%zmm4; vbroadcastf32x4 16(%0),%%zmm5; addq $32,%0;"\
-    "vmovddup (%1),%%zmm6; vfmadd231pd %%zmm4,%%zmm6,%%zmm8; vfmadd231pd %%zmm5,%%zmm6,%%zmm10;"\
-    "vmovddup 8(%1),%%zmm7; vfmadd231pd %%zmm4,%%zmm7,%%zmm9; vfmadd231pd %%zmm5,%%zmm7,%%zmm11;"
-
-#define INNER_KERNEL_k1m4n16 \
-    INNER_KERNEL_k1m4n8\
-    "vmovddup (%1,%%r12,2),%%zmm6; vfmadd231pd %%zmm4,%%zmm6,%%zmm12; vfmadd231pd %%zmm5,%%zmm6,%%zmm14;"\
-    "vmovddup 8(%1,%%r12,2),%%zmm7; vfmadd231pd %%zmm4,%%zmm7,%%zmm13; vfmadd231pd %%zmm5,%%zmm7,%%zmm15;"
-
-#define INNER_KERNEL_k1m4n24 \
-    INNER_KERNEL_k1m4n16\
-    "vmovddup (%1,%%r12,4),%%zmm6; vfmadd231pd %%zmm4,%%zmm6,%%zmm16; vfmadd231pd %%zmm5,%%zmm6,%%zmm18;"\
-    "vmovddup 8(%1,%%r12,4),%%zmm7; vfmadd231pd %%zmm4,%%zmm7,%%zmm17; vfmadd231pd %%zmm5,%%zmm7,%%zmm19;"
-
-#define INNER_KERNEL_k1m8n8 \
-    "vbroadcastf32x4 (%0),%%zmm4; vbroadcastf32x4 16(%0),%%zmm5;"\
-    "vbroadcastf32x4 (%0,%%r12,1),%%zmm6; vbroadcastf32x4 16(%0,%%r12,1),%%zmm7; addq $32,%0;"\
-    "prefetcht0 128(%1);"\
-    "vmovddup (%1),%%zmm2; vfmadd231pd %%zmm4,%%zmm2,%%zmm8; vfmadd231pd %%zmm5,%%zmm2,%%zmm10;"\
-    "vfmadd231pd %%zmm6,%%zmm2,%%zmm12; vfmadd231pd %%zmm7,%%zmm2,%%zmm14;"\
-    "vmovddup 8(%1),%%zmm1; vfmadd231pd %%zmm4,%%zmm1,%%zmm9; vfmadd231pd %%zmm5,%%zmm1,%%zmm11;"\
-    "vfmadd231pd %%zmm6,%%zmm1,%%zmm13; vfmadd231pd %%zmm7,%%zmm1,%%zmm15;"
-
-#define INNER_KERNEL_k1m8n16 \
-    INNER_KERNEL_k1m8n8\
-    "prefetcht0 128(%1,%%r12,2);"\
-    "vmovddup (%1,%%r12,2),%%zmm2; vfmadd231pd %%zmm4,%%zmm2,%%zmm16; vfmadd231pd %%zmm5,%%zmm2,%%zmm18;"\
-    "vfmadd231pd %%zmm6,%%zmm2,%%zmm20; vfmadd231pd %%zmm7,%%zmm2,%%zmm22;"\
-    "vmovddup 8(%1,%%r12,2),%%zmm1; vfmadd231pd %%zmm4,%%zmm1,%%zmm17; vfmadd231pd %%zmm5,%%zmm1,%%zmm19;"\
-    "vfmadd231pd %%zmm6,%%zmm1,%%zmm21; vfmadd231pd %%zmm7,%%zmm1,%%zmm23;"
-
-#define INNER_KERNEL_k1m8n24 \
-    INNER_KERNEL_k1m8n16\
-    "prefetcht0 128(%1,%%r12,4);"\
-    "vmovddup (%1,%%r12,4),%%zmm2; vfmadd231pd %%zmm4,%%zmm2,%%zmm24; vfmadd231pd %%zmm5,%%zmm2,%%zmm26;"\
-    "vfmadd231pd %%zmm6,%%zmm2,%%zmm28; vfmadd231pd %%zmm7,%%zmm2,%%zmm30;"\
-    "vmovddup 8(%1,%%r12,4),%%zmm1; vfmadd231pd %%zmm4,%%zmm1,%%zmm25; vfmadd231pd %%zmm5,%%zmm1,%%zmm27;"\
-    "vfmadd231pd %%zmm6,%%zmm1,%%zmm29; vfmadd231pd %%zmm7,%%zmm1,%%zmm31;"
-
-/* micro kernels */
-#define INNER_KERNELm1(nn) \
-    "cmpq $1,%2;jb "#nn"3f;"\
-    #nn"4:\n\t"\
-    INNER_KERNEL_k1m1n##nn "addq $8,%0;"\
-    "decq %2;cmpq $1,%2;jnb "#nn"4b;"\
-    #nn"3:\n\t"
-
-#define INNER_KERNELm2(nn) \
-    "cmpq $1,%2;jb "#nn"0f;"\
-    #nn"1:\n\t"\
-    INNER_KERNEL_k1m2n##nn "addq $16,%0;"\
-    "decq %2;cmpq $1,%2;jnb "#nn"1b;"\
-    #nn"0:\n\t"
-
-#define INNER_KERNELm4(nn) \
-    "cmpq $1,%2;jb "#nn"00f;"\
-    #nn"01:\n\t"\
-    INNER_KERNEL_k1m4n##nn "addq $64,%1;"\
-    "decq %2;cmpq $1,%2;jnb "#nn"01b;"\
-    #nn"00:\n\t"
-
-/* %10 for prefetch of C elements before storage; %4 = ldc(in bytes),%11 for prefetch of next B block */
-#define INNER_KERNELm8(nn) \
-    "movq %3,%10;cmpq $18,%2;jb "#nn"001f;"\
-    #nn"008:\n\t"\
-    INNER_KERNEL_k1m8n##nn "addq $64,%1;"\
-    INNER_KERNEL_k1m8n##nn "addq $64,%1;"\
-    INNER_KERNEL_k1m8n##nn "addq $64,%1;"\
-    "prefetcht1 (%10); prefetcht1 63(%10); addq %4,%10;"\
-    INNER_KERNEL_k1m8n##nn "addq $64,%1;"\
-    INNER_KERNEL_k1m8n##nn "addq $64,%1;"\
-    INNER_KERNEL_k1m8n##nn "addq $64,%1;"\
-    "prefetcht1 (%11); addq $32,%11;"\
-    "subq $6,%2;cmpq $18,%2;jnb "#nn"008b;"\
-    "movq %3,%10;"\
-    #nn"001:\n\t"\
-    "cmpq $1,%2;jb "#nn"000f;"\
-    "prefetcht0 (%10); prefetcht0 63(%10); prefetcht0 (%10,%4,1); prefetcht0 63(%10,%4,1); leaq (%10,%4,2),%10;"\
-    INNER_KERNEL_k1m8n##nn "addq $64,%1;"\
-    "decq %2;jmp "#nn"001b;"\
-    ""#nn"000:\n\t"
-
-#define INNER_INIT_m1n8 \
-    "vpxorq %%zmm8, %%zmm8, %%zmm8;"
-
-#define INNER_INIT_m2n8 \
-    "vpxorq %%zmm8, %%zmm8, %%zmm8; vpxorq %%zmm9, %%zmm9, %%zmm9;"
-
-#define INNER_INIT_m4n8 \
-    "vpxorq %%zmm8, %%zmm8, %%zmm8; vpxorq %%zmm9, %%zmm9, %%zmm9; vpxorq %%zmm10,%%zmm10,%%zmm10;vpxorq %%zmm11,%%zmm11,%%zmm11;"
-
-#define INNER_INIT_m8n8 \
-    INNER_INIT_m4n8\
-    "vpxorq %%zmm12,%%zmm12,%%zmm12;vpxorq %%zmm13,%%zmm13,%%zmm13;vpxorq %%zmm14,%%zmm14,%%zmm14;vpxorq %%zmm15,%%zmm15,%%zmm15;"
-
-#define INNER_INIT_m1n16 INNER_INIT_m2n8
-
-#define INNER_INIT_m2n16 INNER_INIT_m4n8
-
-#define INNER_INIT_m4n16 INNER_INIT_m8n8
-
-#define INNER_INIT_m8n16 \
-    INNER_INIT_m8n8\
-    "vpxorq %%zmm16,%%zmm16,%%zmm16;vpxorq %%zmm17,%%zmm17,%%zmm17;vpxorq %%zmm18,%%zmm18,%%zmm18;vpxorq %%zmm19,%%zmm19,%%zmm19;"\
-    "vpxorq %%zmm20,%%zmm20,%%zmm20;vpxorq %%zmm21,%%zmm21,%%zmm21;vpxorq %%zmm22,%%zmm22,%%zmm22;vpxorq %%zmm23,%%zmm23,%%zmm23;"
-
-#define INNER_INIT_m1n24 \
-    "vpxorq %%zmm8, %%zmm8, %%zmm8; vpxorq %%zmm9, %%zmm9, %%zmm9; vpxorq %%zmm10,%%zmm10,%%zmm10;"
-
-#define INNER_INIT_m2n24 \
-    INNER_INIT_m1n24\
-    "vpxorq %%zmm11,%%zmm11,%%zmm11; vpxorq %%zmm12,%%zmm12,%%zmm12; vpxorq %%zmm13,%%zmm13,%%zmm13;"
-
-#define INNER_INIT_m4n24 \
-    INNER_INIT_m4n16\
-    "vpxorq %%zmm16,%%zmm16,%%zmm16;vpxorq %%zmm17,%%zmm17,%%zmm17;vpxorq %%zmm18,%%zmm18,%%zmm18;vpxorq %%zmm19,%%zmm19,%%zmm19;"
-
-#define INNER_INIT_m8n24 \
-    INNER_INIT_m8n16\
-    "vpxorq %%zmm24,%%zmm24,%%zmm24;vpxorq %%zmm25,%%zmm25,%%zmm25;vpxorq %%zmm26,%%zmm26,%%zmm26;vpxorq %%zmm27,%%zmm27,%%zmm27;"\
-    "vpxorq %%zmm28,%%zmm28,%%zmm28;vpxorq %%zmm29,%%zmm29,%%zmm29;vpxorq %%zmm30,%%zmm30,%%zmm30;vpxorq %%zmm31,%%zmm31,%%zmm31;"
-
-#define INNER_SETINDEX \
-    "vpinsrq $0,%4,%%xmm4,%%xmm4; vbroadcastsd %%xmm4,%%zmm4;"\
-    "kxnorw %%k1,%%k1,%%k1; kshiftlw $1,%%k1,%%k1; vpxorq %%zmm6,%%zmm6,%%zmm6; vmovapd %%zmm4,%%zmm6%{%%k1%};"\
-    "kshiftlw $1,%%k1,%%k1; vpaddq %%zmm4,%%zmm6,%%zmm6%{%%k1%};"\
-    "kshiftlw $1,%%k1,%%k1; vpaddq %%zmm4,%%zmm6,%%zmm6%{%%k1%};"\
-    "kshiftlw $1,%%k1,%%k1; vpaddq %%zmm4,%%zmm6,%%zmm6%{%%k1%};"\
-    "kshiftlw $1,%%k1,%%k1; vpaddq %%zmm4,%%zmm6,%%zmm6%{%%k1%};"\
-    "kshiftlw $1,%%k1,%%k1; vpaddq %%zmm4,%%zmm6,%%zmm6%{%%k1%};"\
-    "kshiftlw $1,%%k1,%%k1; vpaddq %%zmm4,%%zmm6,%%zmm6%{%%k1%};"
-
-#define INNER_STORE_m1n8(c1,disp) \
-    "kxnorw %%k1,%%k1,%%k1;"\
-    "vgatherqpd "#disp"(%10,%%zmm6,1), %%zmm7 %{%%k1%};"\
-    "vfmadd132pd %%zmm3,%%zmm7,"#c1";"\
-    "kxnorw %%k1,%%k1,%%k1;"\
-    "vscatterqpd "#c1", "#disp"(%10,%%zmm6,1) %{%%k1%};"
-
-#define INNER_SAVE_m1n8 \
-    "movq %3,%10;"\
-    INNER_SETINDEX\
-    INNER_STORE_m1n8(%%zmm8,0)
-
-#define INNER_SAVE_m1n16 \
-    INNER_SAVE_m1n8\
-    "leaq (%10,%4,8),%10;"\
-    INNER_STORE_m1n8(%%zmm9,0)
-
-#define INNER_SAVE_m1n24 \
-    INNER_SAVE_m1n16\
-    "leaq (%10,%4,8),%10;"\
-    INNER_STORE_m1n8(%%zmm10,0)
-
-#define INNER_SAVE_m2n8 \
-    "movq %3,%10;"\
-    INNER_SETINDEX\
-    INNER_STORE_m1n8(%%zmm8,0)\
-    INNER_STORE_m1n8(%%zmm9,8)
-
-#define INNER_SAVE_m2n16 \
-    "movq %3,%10;"\
-    INNER_SETINDEX\
-    INNER_STORE_m1n8(%%zmm8,0)\
-    INNER_STORE_m1n8(%%zmm10,8)\
-    "leaq (%10,%4,8),%10;"\
-    INNER_STORE_m1n8(%%zmm9,0)\
-    INNER_STORE_m1n8(%%zmm11,8)
-
-#define INNER_SAVE_m2n24 \
-    "movq %3,%10;"\
-    INNER_SETINDEX\
-    INNER_STORE_m1n8(%%zmm8,0)\
-    INNER_STORE_m1n8(%%zmm11,8)\
-    "leaq (%10,%4,8),%10;"\
-    INNER_STORE_m1n8(%%zmm9,0)\
-    INNER_STORE_m1n8(%%zmm12,8)\
-    "leaq (%10,%4,8),%10;"\
-    INNER_STORE_m1n8(%%zmm10,0)\
-    INNER_STORE_m1n8(%%zmm13,8)
-
-#define INNER_TRANS_4x8(c1,c2,c3,c4) \
-    "vblendmpd "#c3","#c1",%%zmm4%{%6%}; vblendmpd "#c4","#c2",%%zmm6%{%6%};"\
-    "vshuff64x2 $177,%%zmm4,%%zmm4,%%zmm4; vshuff64x2 $177,%%zmm6,%%zmm6,%%zmm6;"\
-    "vblendmpd "#c1",%%zmm4,"#c1"%{%6%}; vblendmpd "#c2",%%zmm6,"#c2"%{%6%};"\
-    "vblendmpd %%zmm4,"#c3","#c3"%{%6%}; vblendmpd %%zmm6,"#c4","#c4"%{%6%};"\
-
-#define INNER_TRANS_f128_4x4(c1,c2,c3,c4) \
-    "vshuff64x2 $68,"#c3","#c1",%%zmm4; vshuff64x2 $17,"#c4","#c2",%%zmm5;"\
-    "vshuff64x2 $238,"#c3","#c1",%%zmm6; vshuff64x2 $187,"#c4","#c2",%%zmm7;"\
-    "vblendmpd %%zmm5,%%zmm4,"#c2"%{%6%}; vshuff64x2 $177,"#c2","#c2","#c2"; vblendmpd %%zmm4,%%zmm5,"#c1"%{%6%};"\
-    "vblendmpd %%zmm7,%%zmm6,"#c4"%{%6%}; vshuff64x2 $177,"#c4","#c4","#c4"; vblendmpd %%zmm6,%%zmm7,"#c3"%{%6%};"
-
-#define INNER_TRANS_8x8(c1,c2,c3,c4,c5,c6,c7,c8) \
-    INNER_TRANS_f128_4x4(c1,c3,c5,c7) INNER_TRANS_f128_4x4(c2,c4,c6,c8)
-
-//%7 for k01(input) only when m=4
-#define INNER_STORE_4x8(c1,c2,c3,c4) \
-    "vmovupd (%10),%%zmm4%{%5%};vmovupd -32(%10,%4,4),%%zmm4%{%7%};vfmadd132pd %%zmm3,%%zmm4,"#c1";"\
-    "vmovupd "#c1",(%10)%{%5%}; vmovupd "#c1",-32(%10,%4,4)%{%7%}; leaq (%10,%4,1),%10;"\
-    "vmovupd (%10),%%zmm5%{%5%};vmovupd -32(%10,%4,4),%%zmm5%{%7%};vfmadd132pd %%zmm3,%%zmm5,"#c2";"\
-    "vmovupd "#c2",(%10)%{%5%}; vmovupd "#c2",-32(%10,%4,4)%{%7%}; leaq (%10,%4,1),%10;"\
-    "vmovupd (%10),%%zmm6%{%5%};vmovupd -32(%10,%4,4),%%zmm6%{%7%};vfmadd132pd %%zmm3,%%zmm6,"#c3";"\
-    "vmovupd "#c3",(%10)%{%5%}; vmovupd "#c3",-32(%10,%4,4)%{%7%}; leaq (%10,%4,1),%10;"\
-    "vmovupd (%10),%%zmm7%{%5%};vmovupd -32(%10,%4,4),%%zmm7%{%7%};vfmadd132pd %%zmm3,%%zmm7,"#c4";"\
-    "vmovupd "#c4",(%10)%{%5%}; vmovupd "#c4",-32(%10,%4,4)%{%7%}; leaq (%10,%4,1),%10;"\
-    "leaq (%10,%4,4),%10;"
-
-#define INNER_STORE_8x8(c1,c2,c3,c4,c5,c6,c7,c8) \
-    "vfmadd213pd (%10),%%zmm3,"#c1"; vmovupd "#c1",(%10); vfmadd213pd (%10,%4,1),%%zmm3,"#c2"; vmovupd "#c2",(%10,%4,1); leaq (%10,%4,2),%10;"\
-    "vfmadd213pd (%10),%%zmm3,"#c3"; vmovupd "#c3",(%10); vfmadd213pd (%10,%4,1),%%zmm3,"#c4"; vmovupd "#c4",(%10,%4,1); leaq (%10,%4,2),%10;"\
-    "vfmadd213pd (%10),%%zmm3,"#c5"; vmovupd "#c5",(%10); vfmadd213pd (%10,%4,1),%%zmm3,"#c6"; vmovupd "#c6",(%10,%4,1); leaq (%10,%4,2),%10;"\
-    "vfmadd213pd (%10),%%zmm3,"#c7"; vmovupd "#c7",(%10); vfmadd213pd (%10,%4,1),%%zmm3,"#c8"; vmovupd "#c8",(%10,%4,1); leaq (%10,%4,2),%10;"
-
-#define INNER_SAVE_m4n8 \
-    "movq %3,%10;"\
-    INNER_TRANS_4x8(%%zmm8,%%zmm9,%%zmm10,%%zmm11)\
-    INNER_STORE_4x8(%%zmm8,%%zmm9,%%zmm10,%%zmm11)
-
-#define INNER_SAVE_m4n16 \
-    INNER_SAVE_m4n8\
-    INNER_TRANS_4x8(%%zmm12,%%zmm13,%%zmm14,%%zmm15)\
-    INNER_STORE_4x8(%%zmm12,%%zmm13,%%zmm14,%%zmm15)
-
-#define INNER_SAVE_m4n24 \
-    INNER_SAVE_m4n16\
-    INNER_TRANS_4x8(%%zmm16,%%zmm17,%%zmm18,%%zmm19)\
-    INNER_STORE_4x8(%%zmm16,%%zmm17,%%zmm18,%%zmm19)
-
-#define INNER_SAVE_m8n8 \
-    "movq %3,%10;"\
-    INNER_TRANS_8x8(%%zmm8,%%zmm9,%%zmm10,%%zmm11,%%zmm12,%%zmm13,%%zmm14,%%zmm15)\
-    INNER_STORE_8x8(%%zmm8,%%zmm9,%%zmm10,%%zmm11,%%zmm12,%%zmm13,%%zmm14,%%zmm15)
-
-#define INNER_SAVE_m8n16 \
-    INNER_SAVE_m8n8\
-    INNER_TRANS_8x8(%%zmm16,%%zmm17,%%zmm18,%%zmm19,%%zmm20,%%zmm21,%%zmm22,%%zmm23)\
-    INNER_STORE_8x8(%%zmm16,%%zmm17,%%zmm18,%%zmm19,%%zmm20,%%zmm21,%%zmm22,%%zmm23)
-
-#define INNER_SAVE_m8n24 \
-    INNER_SAVE_m8n16\
-    INNER_TRANS_8x8(%%zmm24,%%zmm25,%%zmm26,%%zmm27,%%zmm28,%%zmm29,%%zmm30,%%zmm31)\
-    INNER_STORE_8x8(%%zmm24,%%zmm25,%%zmm26,%%zmm27,%%zmm28,%%zmm29,%%zmm30,%%zmm31)
-
-#define COMPUTE_n8 {\
-    b_pref = packed_b_pointer + 8 * K;\
-    __asm__ __volatile__(\
-    "vbroadcastsd (%9),%%zmm3;"\
-    "movq %8,%%r14;movq %2,%%r13;movq %2,%%r12;shlq $5,%%r12;"\
-    "cmpq $8,%8; jb 42222f;"\
-    "42221:\n\t"\
-    INNER_INIT_m8n8\
-    INNER_KERNELm8(8)\
-    INNER_SAVE_m8n8\
-    "movq %%r13,%2; subq %%r12,%1; subq %%r12,%1; addq %%r12,%0;"\
-    "addq $64,%3;"\
-    "subq $8,%8; cmpq $8,%8; jnb 42221b;"\
-    "42222:\n\t"\
-    "cmpq $4,%8; jb 42223f;"\
-    INNER_INIT_m4n8\
-    INNER_KERNELm4(8)\
-    INNER_SAVE_m4n8\
-    "movq %%r13,%2; subq %%r12,%1; subq %%r12,%1;"\
-    "addq $32,%3;"\
-    "subq $4,%8;"\
-    "42223:\n\t"\
-    "cmpq $2,%8; jb 42224f;"\
-    INNER_INIT_m2n8\
-    INNER_KERNELm2(8)\
-    INNER_SAVE_m2n8\
-    "movq %%r13,%2; subq %%r12,%1; subq %%r12,%1;"\
-    "addq $16,%3;"\
-    "subq $2,%8;"\
-    "42224:\n\t"\
-    "cmpq $1,%8; jb 42225f;"\
-    INNER_INIT_m1n8\
-    INNER_KERNELm1(8)\
-    INNER_SAVE_m1n8\
-    "movq %%r13,%2; subq %%r12,%1; subq %%r12,%1;"\
-    "addq $8,%3;"\
-    "42225:\n\t"\
-    "movq %%r14,%8;shlq $3,%8;subq %8,%3;shrq $3,%8;"\
-    "shlq $3,%4;addq %4,%3;shrq $3,%4;"\
-    :"+r"(a_block_pointer),"+r"(packed_b_pointer),"+r"(K),"+r"(c_pointer),"+r"(ldc_in_bytes),"+Yk"(k02),"+Yk"(k03),"+Yk"(k01),\
-    "+r"(M),"+r"(alpha),"+r"(c_store),"+r"(b_pref)\
-    ::"zmm0","zmm1","zmm2","zmm3","zmm4","zmm5","zmm6","zmm7","zmm8","zmm9","zmm10","zmm11","zmm12","zmm13","zmm14","zmm15","cc","memory","k1","r12","r13","r14");\
-    a_block_pointer -= M * K;\
-}
-#define COMPUTE_n16 {\
-    b_pref = packed_b_pointer + 16 * K;\
-    __asm__ __volatile__(\
-    "vbroadcastsd (%9),%%zmm3;"\
-    "movq %8,%%r14;movq %2,%%r13;movq %2,%%r12;shlq $5,%%r12;"\
-    "cmpq $8,%8; jb 32222f;"\
-    "32221:\n\t"\
-    INNER_INIT_m8n16\
-    INNER_KERNELm8(16)\
-    INNER_SAVE_m8n16\
-    "movq %%r13,%2; subq %%r12,%1; subq %%r12,%1; addq %%r12,%0;"\
-    "addq $64,%3;"\
-    "subq $8,%8; cmpq $8,%8; jnb 32221b;"\
-    "32222:\n\t"\
-    "cmpq $4,%8; jb 32223f;"\
-    INNER_INIT_m4n16\
-    INNER_KERNELm4(16)\
-    INNER_SAVE_m4n16\
-    "movq %%r13,%2; subq %%r12,%1; subq %%r12,%1;"\
-    "addq $32,%3;"\
-    "subq $4,%8;"\
-    "32223:\n\t"\
-    "cmpq $2,%8; jb 32224f;"\
-    INNER_INIT_m2n16\
-    INNER_KERNELm2(16)\
-    INNER_SAVE_m2n16\
-    "movq %%r13,%2; subq %%r12,%1; subq %%r12,%1;"\
-    "addq $16,%3;"\
-    "subq $2,%8;"\
-    "32224:\n\t"\
-    "cmpq $1,%8; jb 32225f;"\
-    INNER_INIT_m1n16\
-    INNER_KERNELm1(16)\
-    INNER_SAVE_m1n16\
-    "movq %%r13,%2; subq %%r12,%1; subq %%r12,%1;"\
-    "addq $8,%3;"\
-    "32225:\n\t"\
-    "movq %%r14,%8;shlq $3,%8;subq %8,%3;shrq $3,%8;"\
-    "shlq $4,%4;addq %4,%3;shrq $4,%4;"\
-    "leaq (%1,%%r12,4),%1;"\
-    :"+r"(a_block_pointer),"+r"(packed_b_pointer),"+r"(K),"+r"(c_pointer),"+r"(ldc_in_bytes),"+Yk"(k02),"+Yk"(k03),"+Yk"(k01),\
-    "+r"(M),"+r"(alpha),"+r"(c_store),"+r"(b_pref)\
-    ::"zmm0","zmm1","zmm2","zmm3","zmm4","zmm5","zmm6","zmm7","zmm8","zmm9","zmm10","zmm11","zmm12","zmm13","zmm14","zmm15","zmm16","zmm17",\
-    "zmm18","zmm19","zmm20","zmm21","zmm22","zmm23","cc","memory","k1","r12","r13","r14");\
-    a_block_pointer -= M * K;\
-}
-#define COMPUTE_n24 {\
-    b_pref = packed_b_pointer + 24 * K;\
-    __asm__ __volatile__(\
-    "vbroadcastsd (%9),%%zmm3;"\
-    "movq %8,%%r14;movq %2,%%r13;movq %2,%%r12;shlq $5,%%r12;"\
-    "cmpq $8,%8; jb 22222f;"\
-    "22221:\n\t"\
-    INNER_INIT_m8n24\
-    INNER_KERNELm8(24)\
-    INNER_SAVE_m8n24\
-    "movq %%r13,%2; subq %%r12,%1; subq %%r12,%1; addq %%r12,%0;"\
-    "addq $64,%3;"\
-    "subq $8,%8; cmpq $8,%8; jnb 22221b;"\
-    "22222:\n\t"\
-    "cmpq $4,%8; jb 22223f;"\
-    INNER_INIT_m4n24\
-    INNER_KERNELm4(24)\
-    INNER_SAVE_m4n24\
-    "movq %%r13,%2; subq %%r12,%1; subq %%r12,%1;"\
-    "addq $32,%3;"\
-    "subq $4,%8;"\
-    "22223:\n\t"\
-    "cmpq $2,%8; jb 22224f;"\
-    INNER_INIT_m2n24\
-    INNER_KERNELm2(24)\
-    INNER_SAVE_m2n24\
-    "movq %%r13,%2; subq %%r12,%1; subq %%r12,%1;"\
-    "addq $16,%3;"\
-    "subq $2,%8;"\
-    "22224:\n\t"\
-    "cmpq $1,%8; jb 22225f;"\
-    INNER_INIT_m1n24\
-    INNER_KERNELm1(24)\
-    INNER_SAVE_m1n24\
-    "movq %%r13,%2; subq %%r12,%1; subq %%r12,%1;"\
-    "addq $8,%3;"\
-    "22225:\n\t"\
-    "movq %%r14,%8;shlq $3,%8;subq %8,%3;shrq $3,%8;"\
-    "shlq $3,%4;addq %4,%3;shlq $1,%4;addq %4,%3;shrq $4,%4;"\
-    "leaq (%1,%%r12,4),%1; leaq (%1,%%r12,2),%1;"\
-    :"+r"(a_block_pointer),"+r"(packed_b_pointer),"+r"(K),"+r"(c_pointer),"+r"(ldc_in_bytes),"+Yk"(k02),"+Yk"(k03),"+Yk"(k01),\
-    "+r"(M),"+r"(alpha),"+r"(c_store),"+r"(b_pref)::\
-    "zmm0","zmm1","zmm2","zmm3","zmm4","zmm5","zmm6","zmm7","zmm8","zmm9","zmm10","zmm11","zmm12","zmm13","zmm14","zmm15","zmm16","zmm17","zmm18",\
-    "zmm19","zmm20","zmm21","zmm22","zmm23","zmm24","zmm25","zmm26","zmm27","zmm28","zmm29","zmm30","zmm31","cc","memory","k1","r12","r13","r14");\
-    a_block_pointer -= M * K;\
-}
-static void KERNEL_MAIN(double *packed_a, double *packed_b, BLASLONG m, BLASLONG ndiv8, BLASLONG k, BLASLONG LDC, double *c,double *alpha){//icopy=4,ocopy=8
-//perform C += A<pack> B<pack>
-    if(k==0 || m==0 || ndiv8==0) return;
-    int64_t ldc_in_bytes = (int64_t)LDC * sizeof(double);
-    int64_t K = (int64_t)k; int64_t M = (int64_t)m;
-    double *a_block_pointer,*b_pref;
-    double *c_pointer = c,*c_store = c;
-    __mmask16 k01 = 0x00f0,k02 = 0x000f,k03 = 0x0033;
-    BLASLONG ndiv8_count;
-    double *packed_b_pointer = packed_b;
-    a_block_pointer = packed_a;
-    for(ndiv8_count=ndiv8;ndiv8_count>2;ndiv8_count-=3){
-      COMPUTE_n24
-    }
-    for(;ndiv8_count>1;ndiv8_count-=2){
-      COMPUTE_n16
-    }
-    if(ndiv8_count>0){
-      COMPUTE_n8
-    }
-}
-
-/* __m256d accumulators: yc1-yc4; temporary variables: ya1,yb1-yb2 */
-/* __m128d accumulators: xc1-xc2; temporary variables: xa1,xb1-xb2 */
-/*  double accumulator:  sc1;     temporary variables: sa1,sb1 */
-/* column-major c_block */
-#define KERNEL_m4n4k1 {\
-    ya1 = _mm256_loadu_pd(a_block_pointer);a_block_pointer+=4;\
-    yb1 = _mm256_broadcast_sd(b_block_pointer);   yc1 = _mm256_fmadd_pd(ya1,yb1,yc1);\
-    yb2 = _mm256_broadcast_sd(b_block_pointer+1); yc2 = _mm256_fmadd_pd(ya1,yb2,yc2);\
-    yb1 = _mm256_broadcast_sd(b_block_pointer+2); yc3 = _mm256_fmadd_pd(ya1,yb1,yc3);\
-    yb2 = _mm256_broadcast_sd(b_block_pointer+3); yc4 = _mm256_fmadd_pd(ya1,yb2,yc4);\
-    b_block_pointer+=4;\
-}
-#define KERNEL_m4n2k1 {\
-    ya1 = _mm256_loadu_pd(a_block_pointer);a_block_pointer+=4;\
-    yb1 = _mm256_broadcast_sd(b_block_pointer);   yc1 = _mm256_fmadd_pd(ya1,yb1,yc1);\
-    yb2 = _mm256_broadcast_sd(b_block_pointer+1); yc2 = _mm256_fmadd_pd(ya1,yb2,yc2);\
-    b_block_pointer+=2;\
-}
-#define KERNEL_m4n1k1 {\
-    ya1 = _mm256_loadu_pd(a_block_pointer);a_block_pointer+=4;\
-    yb1 = _mm256_broadcast_sd(b_block_pointer);   yc1 = _mm256_fmadd_pd(ya1,yb1,yc1);\
-    b_block_pointer++;\
-}
-#define INIT_m4n1 yc1=_mm256_setzero_pd();
-#define INIT_m4n2 yc2=INIT_m4n1
-#define INIT_m4n4 yc4=yc3=INIT_m4n2
-#define SAVE_m4n1 {\
-    yb1 = _mm256_broadcast_sd(alpha);\
-    ya1 = _mm256_loadu_pd(c_pointer);\
-    yc1 = _mm256_fmadd_pd(yc1,yb1,ya1);\
-    _mm256_storeu_pd(c_pointer,yc1);\
-    c_pointer += 4;\
-}
-#define SAVE_m4n2 {\
-    ya1 = _mm256_broadcast_sd(alpha);\
-    yb1 = _mm256_loadu_pd(c_pointer); yb2 = _mm256_loadu_pd(c_pointer+LDC);\
-    yc1 = _mm256_fmadd_pd(yc1,ya1,yb1); yc2 = _mm256_fmadd_pd(yc2,ya1,yb2);\
-    _mm256_storeu_pd(c_pointer,yc1); _mm256_storeu_pd(c_pointer+LDC,yc2);\
-    c_pointer += 4;\
-}
-#define SAVE_m4n4 {\
-    ya1 = _mm256_broadcast_sd(alpha);\
-    yb1 = _mm256_loadu_pd(c_pointer); yb2 = _mm256_loadu_pd(c_pointer+LDC);\
-    yc1 = _mm256_fmadd_pd(yc1,ya1,yb1); yc2 = _mm256_fmadd_pd(yc2,ya1,yb2);\
-    _mm256_storeu_pd(c_pointer,yc1); _mm256_storeu_pd(c_pointer+LDC,yc2);\
-    c_pointer += LDC*2;\
-    yb1 = _mm256_loadu_pd(c_pointer); yb2 = _mm256_loadu_pd(c_pointer+LDC);\
-    yc3 = _mm256_fmadd_pd(yc3,ya1,yb1); yc4 = _mm256_fmadd_pd(yc4,ya1,yb2);\
-    _mm256_storeu_pd(c_pointer,yc3); _mm256_storeu_pd(c_pointer+LDC,yc4);\
-    c_pointer += 4-LDC*2;\
-}
-#define KERNEL_m2n2k1 {\
-    xa1 = _mm_loadu_pd(a_block_pointer); a_block_pointer+=2;\
-    xb1 = _mm_loaddup_pd(b_block_pointer);   xc1 = _mm_fmadd_pd(xa1,xb1,xc1);\
-    xb2 = _mm_loaddup_pd(b_block_pointer+1); xc2 = _mm_fmadd_pd(xa1,xb2,xc2);\
-    b_block_pointer += 2;\
-}
-#define KERNEL_m2n1k1 {\
-    xa1 = _mm_loadu_pd(a_block_pointer); a_block_pointer+=2;\
-    xb1 = _mm_loaddup_pd(b_block_pointer);   xc1 = _mm_fmadd_pd(xa1,xb1,xc1);\
-    b_block_pointer ++;\
-}
-#define INIT_m2n1 xc1=_mm_setzero_pd();
-#define INIT_m2n2 xc2=INIT_m2n1
-#define SAVE_m2n1 {\
-    xb1 = _mm_loaddup_pd(alpha);\
-    xa1 = _mm_loadu_pd(c_pointer);\
-    xc1 = _mm_fmadd_pd(xc1,xb1,xa1);\
-    _mm_storeu_pd(c_pointer,xc1);\
-    c_pointer += 2;\
-}
-#define SAVE_m2n2 {\
-    xa1 = _mm_loaddup_pd(alpha);\
-    xb1 = _mm_loadu_pd(c_pointer); xb2 = _mm_loadu_pd(c_pointer+LDC);\
-    xc1 = _mm_fmadd_pd(xc1,xa1,xb1); xc2 = _mm_fmadd_pd(xc2,xa1,xb2);\
-    _mm_storeu_pd(c_pointer,xc1); _mm_storeu_pd(c_pointer+LDC,xc2);\
-    c_pointer += 2;\
-}
-#define KERNEL_m1n1k1 {\
-    sa1 = *a_block_pointer; a_block_pointer++;\
-    sb1 = *b_block_pointer; sc1 += sa1 * sb1;\
-    b_block_pointer ++;\
-}
-#define INIT_m1n1 sc1=0.0;
-#define SAVE_m1n1 {\
-    *c_pointer += sc1 * (*alpha);\
-    c_pointer++;\
-}
-/* row-major c_block */
-#define KERNEL_m2n4k1 {\
-    yb1 = _mm256_loadu_pd(b_block_pointer);b_block_pointer+=4;\
-    ya1 = _mm256_broadcast_sd(a_block_pointer);  yc1 = _mm256_fmadd_pd(ya1,yb1,yc1);\
-    ya1 = _mm256_broadcast_sd(a_block_pointer+1);yc2 = _mm256_fmadd_pd(ya1,yb1,yc2);\
-    a_block_pointer += 2;\
-}
-#define KERNEL_m1n4k1 {\
-    yb1 = _mm256_loadu_pd(b_block_pointer);b_block_pointer+=4;\
-    ya1 = _mm256_broadcast_sd(a_block_pointer);  yc1 = _mm256_fmadd_pd(ya1,yb1,yc1);\
-    a_block_pointer ++;\
-}
-#define KERNEL_m1n2k1 {\
-    xb1 = _mm_loadu_pd(b_block_pointer);b_block_pointer+=2;\
-    xa1 = _mm_loaddup_pd(a_block_pointer); xc1 = _mm_fmadd_pd(xa1,xb1,xc1);\
-    a_block_pointer ++;\
-}
-#define INIT_m1n2 INIT_m2n1
-#define INIT_m1n4 INIT_m4n1
-#define INIT_m2n4 INIT_m4n2
-#define SAVE_m2n4 {\
-    ya1 = _mm256_broadcast_sd(alpha);\
-    yc1 = _mm256_mul_pd(yc1,ya1);\
-    yc2 = _mm256_mul_pd(yc2,ya1);\
-    yb1 = _mm256_unpacklo_pd(yc1,yc2);\
-    yb2 = _mm256_unpackhi_pd(yc1,yc2);\
-    xb1 = _mm_add_pd(_mm_loadu_pd(c_pointer),_mm256_extractf128_pd(yb1,0));\
-    xb2 = _mm_add_pd(_mm_loadu_pd(c_pointer+LDC),_mm256_extractf128_pd(yb2,0));\
-    _mm_storeu_pd(c_pointer,xb1);\
-    _mm_storeu_pd(c_pointer+LDC,xb2);\
-    xb1 = _mm_add_pd(_mm_loadu_pd(c_pointer+2*LDC),_mm256_extractf128_pd(yb1,1));\
-    xb2 = _mm_add_pd(_mm_loadu_pd(c_pointer+3*LDC),_mm256_extractf128_pd(yb2,1));\
-    _mm_storeu_pd(c_pointer+2*LDC,xb1);\
-    _mm_storeu_pd(c_pointer+3*LDC,xb2);\
-    c_pointer += 2;\
-}
-#define SAVE_m1n2 {\
-    xb1 = _mm_loaddup_pd(alpha);\
-    xc1 = _mm_mul_pd(xc1,xb1);\
-    *c_pointer += _mm_cvtsd_f64(xc1);\
-    xa1 = _mm_unpackhi_pd(xc1,xc1);\
-    c_pointer[LDC]+= _mm_cvtsd_f64(xa1);\
-    c_pointer ++;\
-}
-#define SAVE_m1n4 {\
-    ya1 = _mm256_broadcast_sd(alpha);\
-    yc1 = _mm256_mul_pd(yc1,ya1);\
-    xb1 = _mm256_extractf128_pd(yc1,0);\
-    *c_pointer += _mm_cvtsd_f64(xb1);\
-    xb2 = _mm_unpackhi_pd(xb1,xb1);\
-    c_pointer[LDC] += _mm_cvtsd_f64(xb2);\
-    xb1 = _mm256_extractf128_pd(yc1,1);\
-    c_pointer[LDC*2] += _mm_cvtsd_f64(xb1);\
-    xb2 = _mm_unpackhi_pd(xb1,xb1);\
-    c_pointer[LDC*3] += _mm_cvtsd_f64(xb2);\
-    c_pointer ++;\
-}
-static void KERNEL_EDGE(double *packed_a, double *packed_b, BLASLONG m, BLASLONG edge_n, BLASLONG k, BLASLONG LDC, double *c,double *alpha){//icopy=8,ocopy=8
-//perform C += A<pack> B<pack> , edge_n<8 must be satisfied.
-    if(k==0 || m==0 || edge_n==0 || (*alpha)==0.0) return;
-    double *a_block_pointer,*b_block_pointer,*b_base_pointer;
-    double *c_pointer = c;
-    __m256d yc1,yc2,yc3,yc4,ya1,yb1,yb2;
-    __m128d xc1,xc2,xa1,xb1,xb2;
-    double sc1,sa1,sb1;
-    BLASLONG m_count,n_count,k_count;
-    b_base_pointer = packed_b;
-//now start calculation of the edge part
-    for(n_count=edge_n;n_count>3;n_count-=4){
-      a_block_pointer = packed_a;
-      for(m_count=m;m_count>3;m_count-=4){
-        b_block_pointer = b_base_pointer;
-        INIT_m4n4
-        for(k_count=0;k_count<k;k_count++) KERNEL_m4n4k1
-        SAVE_m4n4
-      }
-      for(;m_count>1;m_count-=2){
-        b_block_pointer = b_base_pointer;
-        INIT_m2n4
-        for(k_count=0;k_count<k;k_count++) KERNEL_m2n4k1
-        SAVE_m2n4
-      }
-      if(m_count>0){
-        b_block_pointer = b_base_pointer;
-        INIT_m1n4
-        for(k_count=0;k_count<k;k_count++) KERNEL_m1n4k1
-        SAVE_m1n4
-      }
-      b_base_pointer += 4*k;
-      c_pointer += 4 * LDC - m;
-    }
-    for(;n_count>1;n_count-=2){
-      a_block_pointer = packed_a;
-      for(m_count=m;m_count>3;m_count-=4){
-        b_block_pointer = b_base_pointer;
-        INIT_m4n2
-        for(k_count=0;k_count<k;k_count++) KERNEL_m4n2k1
-        SAVE_m4n2
-      }
-      for(;m_count>1;m_count-=2){
-        b_block_pointer = b_base_pointer;
-        INIT_m2n2
-        for(k_count=0;k_count<k;k_count++) KERNEL_m2n2k1
-        SAVE_m2n2
-      }
-      if(m_count>0){
-        b_block_pointer = b_base_pointer;
-        INIT_m1n2
-        for(k_count=0;k_count<k;k_count++) KERNEL_m1n2k1
-        SAVE_m1n2
-      }
-      b_base_pointer += 2*k;
-      c_pointer += 2 * LDC - m;
-    }
-    if(n_count>0){
-      a_block_pointer = packed_a;
-      for(m_count=m;m_count>3;m_count-=4){
-        b_block_pointer = b_base_pointer;
-        INIT_m4n1
-        for(k_count=0;k_count<k;k_count++) KERNEL_m4n1k1
-        SAVE_m4n1
-      }
-      for(;m_count>1;m_count-=2){
-        b_block_pointer = b_base_pointer;
-        INIT_m2n1
-        for(k_count=0;k_count<k;k_count++) KERNEL_m2n1k1
-        SAVE_m2n1
-      }
-      if(m_count>0){
-        b_block_pointer = b_base_pointer;
-        INIT_m1n1
-        for(k_count=0;k_count<k;k_count++) KERNEL_m1n1k1
-        SAVE_m1n1
-      }
-    }
-}
-int __attribute__ ((noinline)) CNAME(BLASLONG m, BLASLONG n, BLASLONG k, double alpha, double * __restrict__ A, double * __restrict__ B, double * __restrict__ C, BLASLONG ldc){
-    if(m==0 || n==0 || k==0 || alpha == 0.0) return 0;
-    BLASLONG ndiv8 = n/8;double ALPHA = alpha;
-    double *packed_a = A;
-    if(ndiv8>0) KERNEL_MAIN(packed_a,B,m,ndiv8,k,ldc,C,&ALPHA);
-    if(n>ndiv8*8) KERNEL_EDGE(packed_a,B+(int64_t)k*(int64_t)ndiv8*8,m,n-ndiv8*8,k,ldc,C+(int64_t)ldc*(int64_t)ndiv8*8,&ALPHA);
-    return 0;
-}
+#include "common.h"
+#include <stdint.h>
+#include <immintrin.h>
+
+//register usage: zmm3 for alpha, zmm0-zmm2 and zmm4-zmm7 for temporary use, zmm8-zmm31 for accumulators.
+
+/* row-major c_block */
+#define INNER_KERNEL_k1m1n8 \
+    "prefetcht0 384(%1);"\
+    "vmovupd (%1),%%zmm5; addq $64,%1;"\
+    "vbroadcastsd   (%0),%%zmm4;vfmadd231pd %%zmm5,%%zmm4,%%zmm8;"
+
+#define INNER_KERNEL_k1m2n8 \
+    INNER_KERNEL_k1m1n8\
+    "vbroadcastsd  8(%0),%%zmm4;vfmadd231pd %%zmm5,%%zmm4,%%zmm9;"
+
+#define INNER_KERNEL_k1m1n16 \
+    "prefetcht0 128(%1); prefetcht0 128(%1,%%r12,2);"\
+    "vmovupd (%1),%%zmm5; vmovupd (%1,%%r12,2),%%zmm6; addq $64,%1;"\
+    "vbroadcastsd   (%0),%%zmm4;vfmadd231pd %%zmm5,%%zmm4,%%zmm8; vfmadd231pd %%zmm6,%%zmm4,%%zmm9;"
+
+#define INNER_KERNEL_k1m2n16 \
+    INNER_KERNEL_k1m1n16\
+    "vbroadcastsd  8(%0),%%zmm4;vfmadd231pd %%zmm5,%%zmm4,%%zmm10;vfmadd231pd %%zmm6,%%zmm4,%%zmm11;"
+
+#define INNER_KERNEL_k1m1n24 \
+    "prefetcht0 128(%1); prefetcht0 128(%1,%%r12,2); prefetcht0 128(%1,%%r12,4);"\
+    "vmovupd (%1),%%zmm5; vmovupd (%1,%%r12,2),%%zmm6; vmovupd (%1,%%r12,4),%%zmm7; addq $64,%1;"\
+    "vbroadcastsd   (%0),%%zmm4;vfmadd231pd %%zmm5,%%zmm4,%%zmm8; vfmadd231pd %%zmm6,%%zmm4,%%zmm9; vfmadd231pd %%zmm7,%%zmm4,%%zmm10;"
+
+#define INNER_KERNEL_k1m2n24 \
+    INNER_KERNEL_k1m1n24\
+    "vbroadcastsd  8(%0),%%zmm4;vfmadd231pd %%zmm5,%%zmm4,%%zmm11;vfmadd231pd %%zmm6,%%zmm4,%%zmm12;vfmadd231pd %%zmm7,%%zmm4,%%zmm13;"
+
+/* row-major z-partition c_block */
+#define INNER_KERNEL_k1m4n8 \
+    "vbroadcastf32x4 (%0),%%zmm4; vbroadcastf32x4 16(%0),%%zmm5; addq $32,%0;"\
+    "vmovddup (%1),%%zmm6; vfmadd231pd %%zmm4,%%zmm6,%%zmm8; vfmadd231pd %%zmm5,%%zmm6,%%zmm10;"\
+    "vmovddup 8(%1),%%zmm7; vfmadd231pd %%zmm4,%%zmm7,%%zmm9; vfmadd231pd %%zmm5,%%zmm7,%%zmm11;"
+
+#define INNER_KERNEL_k1m4n16 \
+    INNER_KERNEL_k1m4n8\
+    "vmovddup (%1,%%r12,2),%%zmm6; vfmadd231pd %%zmm4,%%zmm6,%%zmm12; vfmadd231pd %%zmm5,%%zmm6,%%zmm14;"\
+    "vmovddup 8(%1,%%r12,2),%%zmm7; vfmadd231pd %%zmm4,%%zmm7,%%zmm13; vfmadd231pd %%zmm5,%%zmm7,%%zmm15;"
+
+#define INNER_KERNEL_k1m4n24 \
+    INNER_KERNEL_k1m4n16\
+    "vmovddup (%1,%%r12,4),%%zmm6; vfmadd231pd %%zmm4,%%zmm6,%%zmm16; vfmadd231pd %%zmm5,%%zmm6,%%zmm18;"\
+    "vmovddup 8(%1,%%r12,4),%%zmm7; vfmadd231pd %%zmm4,%%zmm7,%%zmm17; vfmadd231pd %%zmm5,%%zmm7,%%zmm19;"
+
+#define INNER_KERNEL_k1m8n8 \
+    "vbroadcastf32x4 (%0),%%zmm4; vbroadcastf32x4 16(%0),%%zmm5;"\
+    "vbroadcastf32x4 (%0,%%r12,1),%%zmm6; vbroadcastf32x4 16(%0,%%r12,1),%%zmm7; addq $32,%0;"\
+    "prefetcht0 128(%1);"\
+    "vmovddup (%1),%%zmm2; vfmadd231pd %%zmm4,%%zmm2,%%zmm8; vfmadd231pd %%zmm5,%%zmm2,%%zmm10;"\
+    "vfmadd231pd %%zmm6,%%zmm2,%%zmm12; vfmadd231pd %%zmm7,%%zmm2,%%zmm14;"\
+    "vmovddup 8(%1),%%zmm1; vfmadd231pd %%zmm4,%%zmm1,%%zmm9; vfmadd231pd %%zmm5,%%zmm1,%%zmm11;"\
+    "vfmadd231pd %%zmm6,%%zmm1,%%zmm13; vfmadd231pd %%zmm7,%%zmm1,%%zmm15;"
+
+#define INNER_KERNEL_k1m8n16 \
+    INNER_KERNEL_k1m8n8\
+    "prefetcht0 128(%1,%%r12,2);"\
+    "vmovddup (%1,%%r12,2),%%zmm2; vfmadd231pd %%zmm4,%%zmm2,%%zmm16; vfmadd231pd %%zmm5,%%zmm2,%%zmm18;"\
+    "vfmadd231pd %%zmm6,%%zmm2,%%zmm20; vfmadd231pd %%zmm7,%%zmm2,%%zmm22;"\
+    "vmovddup 8(%1,%%r12,2),%%zmm1; vfmadd231pd %%zmm4,%%zmm1,%%zmm17; vfmadd231pd %%zmm5,%%zmm1,%%zmm19;"\
+    "vfmadd231pd %%zmm6,%%zmm1,%%zmm21; vfmadd231pd %%zmm7,%%zmm1,%%zmm23;"
+
+#define INNER_KERNEL_k1m8n24 \
+    INNER_KERNEL_k1m8n16\
+    "prefetcht0 128(%1,%%r12,4);"\
+    "vmovddup (%1,%%r12,4),%%zmm2; vfmadd231pd %%zmm4,%%zmm2,%%zmm24; vfmadd231pd %%zmm5,%%zmm2,%%zmm26;"\
+    "vfmadd231pd %%zmm6,%%zmm2,%%zmm28; vfmadd231pd %%zmm7,%%zmm2,%%zmm30;"\
+    "vmovddup 8(%1,%%r12,4),%%zmm1; vfmadd231pd %%zmm4,%%zmm1,%%zmm25; vfmadd231pd %%zmm5,%%zmm1,%%zmm27;"\
+    "vfmadd231pd %%zmm6,%%zmm1,%%zmm29; vfmadd231pd %%zmm7,%%zmm1,%%zmm31;"
+
+/* micro kernels */
+#define INNER_KERNELm1(nn) \
+    "cmpq $1,%2;jb "#nn"3f;"\
+    #nn"4:\n\t"\
+    INNER_KERNEL_k1m1n##nn "addq $8,%0;"\
+    "decq %2;cmpq $1,%2;jnb "#nn"4b;"\
+    #nn"3:\n\t"
+
+#define INNER_KERNELm2(nn) \
+    "cmpq $1,%2;jb "#nn"0f;"\
+    #nn"1:\n\t"\
+    INNER_KERNEL_k1m2n##nn "addq $16,%0;"\
+    "decq %2;cmpq $1,%2;jnb "#nn"1b;"\
+    #nn"0:\n\t"
+
+#define INNER_KERNELm4(nn) \
+    "cmpq $1,%2;jb "#nn"00f;"\
+    #nn"01:\n\t"\
+    INNER_KERNEL_k1m4n##nn "addq $64,%1;"\
+    "decq %2;cmpq $1,%2;jnb "#nn"01b;"\
+    #nn"00:\n\t"
+
+/* %10 for prefetch of C elements before storage; %4 = ldc(in bytes),%11 for prefetch of next B block */
+#define INNER_KERNELm8(nn) \
+    "movq %3,%10;cmpq $18,%2;jb "#nn"001f;"\
+    #nn"008:\n\t"\
+    INNER_KERNEL_k1m8n##nn "addq $64,%1;"\
+    INNER_KERNEL_k1m8n##nn "addq $64,%1;"\
+    INNER_KERNEL_k1m8n##nn "addq $64,%1;"\
+    "prefetcht1 (%10); prefetcht1 63(%10); addq %4,%10;"\
+    INNER_KERNEL_k1m8n##nn "addq $64,%1;"\
+    INNER_KERNEL_k1m8n##nn "addq $64,%1;"\
+    INNER_KERNEL_k1m8n##nn "addq $64,%1;"\
+    "prefetcht1 (%11); addq $32,%11;"\
+    "subq $6,%2;cmpq $18,%2;jnb "#nn"008b;"\
+    "movq %3,%10;"\
+    #nn"001:\n\t"\
+    "cmpq $1,%2;jb "#nn"000f;"\
+    "prefetcht0 (%10); prefetcht0 63(%10); prefetcht0 (%10,%4,1); prefetcht0 63(%10,%4,1); leaq (%10,%4,2),%10;"\
+    INNER_KERNEL_k1m8n##nn "addq $64,%1;"\
+    "decq %2;jmp "#nn"001b;"\
+    ""#nn"000:\n\t"
+
+#define INNER_INIT_m1n8 \
+    "vpxorq %%zmm8, %%zmm8, %%zmm8;"
+
+#define INNER_INIT_m2n8 \
+    "vpxorq %%zmm8, %%zmm8, %%zmm8; vpxorq %%zmm9, %%zmm9, %%zmm9;"
+
+#define INNER_INIT_m4n8 \
+    "vpxorq %%zmm8, %%zmm8, %%zmm8; vpxorq %%zmm9, %%zmm9, %%zmm9; vpxorq %%zmm10,%%zmm10,%%zmm10;vpxorq %%zmm11,%%zmm11,%%zmm11;"
+
+#define INNER_INIT_m8n8 \
+    INNER_INIT_m4n8\
+    "vpxorq %%zmm12,%%zmm12,%%zmm12;vpxorq %%zmm13,%%zmm13,%%zmm13;vpxorq %%zmm14,%%zmm14,%%zmm14;vpxorq %%zmm15,%%zmm15,%%zmm15;"
+
+#define INNER_INIT_m1n16 INNER_INIT_m2n8
+
+#define INNER_INIT_m2n16 INNER_INIT_m4n8
+
+#define INNER_INIT_m4n16 INNER_INIT_m8n8
+
+#define INNER_INIT_m8n16 \
+    INNER_INIT_m8n8\
+    "vpxorq %%zmm16,%%zmm16,%%zmm16;vpxorq %%zmm17,%%zmm17,%%zmm17;vpxorq %%zmm18,%%zmm18,%%zmm18;vpxorq %%zmm19,%%zmm19,%%zmm19;"\
+    "vpxorq %%zmm20,%%zmm20,%%zmm20;vpxorq %%zmm21,%%zmm21,%%zmm21;vpxorq %%zmm22,%%zmm22,%%zmm22;vpxorq %%zmm23,%%zmm23,%%zmm23;"
+
+#define INNER_INIT_m1n24 \
+    "vpxorq %%zmm8, %%zmm8, %%zmm8; vpxorq %%zmm9, %%zmm9, %%zmm9; vpxorq %%zmm10,%%zmm10,%%zmm10;"
+
+#define INNER_INIT_m2n24 \
+    INNER_INIT_m1n24\
+    "vpxorq %%zmm11,%%zmm11,%%zmm11; vpxorq %%zmm12,%%zmm12,%%zmm12; vpxorq %%zmm13,%%zmm13,%%zmm13;"
+
+#define INNER_INIT_m4n24 \
+    INNER_INIT_m4n16\
+    "vpxorq %%zmm16,%%zmm16,%%zmm16;vpxorq %%zmm17,%%zmm17,%%zmm17;vpxorq %%zmm18,%%zmm18,%%zmm18;vpxorq %%zmm19,%%zmm19,%%zmm19;"
+
+#define INNER_INIT_m8n24 \
+    INNER_INIT_m8n16\
+    "vpxorq %%zmm24,%%zmm24,%%zmm24;vpxorq %%zmm25,%%zmm25,%%zmm25;vpxorq %%zmm26,%%zmm26,%%zmm26;vpxorq %%zmm27,%%zmm27,%%zmm27;"\
+    "vpxorq %%zmm28,%%zmm28,%%zmm28;vpxorq %%zmm29,%%zmm29,%%zmm29;vpxorq %%zmm30,%%zmm30,%%zmm30;vpxorq %%zmm31,%%zmm31,%%zmm31;"
+
+#define INNER_SETINDEX \
+    "vpinsrq $0,%4,%%xmm4,%%xmm4; vbroadcastsd %%xmm4,%%zmm4;"\
+    "kxnorw %%k1,%%k1,%%k1; kshiftlw $1,%%k1,%%k1; vpxorq %%zmm6,%%zmm6,%%zmm6; vmovapd %%zmm4,%%zmm6%{%%k1%};"\
+    "kshiftlw $1,%%k1,%%k1; vpaddq %%zmm4,%%zmm6,%%zmm6%{%%k1%};"\
+    "kshiftlw $1,%%k1,%%k1; vpaddq %%zmm4,%%zmm6,%%zmm6%{%%k1%};"\
+    "kshiftlw $1,%%k1,%%k1; vpaddq %%zmm4,%%zmm6,%%zmm6%{%%k1%};"\
+    "kshiftlw $1,%%k1,%%k1; vpaddq %%zmm4,%%zmm6,%%zmm6%{%%k1%};"\
+    "kshiftlw $1,%%k1,%%k1; vpaddq %%zmm4,%%zmm6,%%zmm6%{%%k1%};"\
+    "kshiftlw $1,%%k1,%%k1; vpaddq %%zmm4,%%zmm6,%%zmm6%{%%k1%};"
+
+#define INNER_STORE_m1n8(c1,disp) \
+    "kxnorw %%k1,%%k1,%%k1;"\
+    "vgatherqpd "#disp"(%10,%%zmm6,1), %%zmm7 %{%%k1%};"\
+    "vfmadd132pd %%zmm3,%%zmm7,"#c1";"\
+    "kxnorw %%k1,%%k1,%%k1;"\
+    "vscatterqpd "#c1", "#disp"(%10,%%zmm6,1) %{%%k1%};"
+
+#define INNER_SAVE_m1n8 \
+    "movq %3,%10;"\
+    INNER_SETINDEX\
+    INNER_STORE_m1n8(%%zmm8,0)
+
+#define INNER_SAVE_m1n16 \
+    INNER_SAVE_m1n8\
+    "leaq (%10,%4,8),%10;"\
+    INNER_STORE_m1n8(%%zmm9,0)
+
+#define INNER_SAVE_m1n24 \
+    INNER_SAVE_m1n16\
+    "leaq (%10,%4,8),%10;"\
+    INNER_STORE_m1n8(%%zmm10,0)
+
+#define INNER_SAVE_m2n8 \
+    "movq %3,%10;"\
+    INNER_SETINDEX\
+    INNER_STORE_m1n8(%%zmm8,0)\
+    INNER_STORE_m1n8(%%zmm9,8)
+
+#define INNER_SAVE_m2n16 \
+    "movq %3,%10;"\
+    INNER_SETINDEX\
+    INNER_STORE_m1n8(%%zmm8,0)\
+    INNER_STORE_m1n8(%%zmm10,8)\
+    "leaq (%10,%4,8),%10;"\
+    INNER_STORE_m1n8(%%zmm9,0)\
+    INNER_STORE_m1n8(%%zmm11,8)
+
+#define INNER_SAVE_m2n24 \
+    "movq %3,%10;"\
+    INNER_SETINDEX\
+    INNER_STORE_m1n8(%%zmm8,0)\
+    INNER_STORE_m1n8(%%zmm11,8)\
+    "leaq (%10,%4,8),%10;"\
+    INNER_STORE_m1n8(%%zmm9,0)\
+    INNER_STORE_m1n8(%%zmm12,8)\
+    "leaq (%10,%4,8),%10;"\
+    INNER_STORE_m1n8(%%zmm10,0)\
+    INNER_STORE_m1n8(%%zmm13,8)
+
+#define INNER_TRANS_4x8(c1,c2,c3,c4) \
+    "vblendmpd "#c3","#c1",%%zmm4%{%6%}; vblendmpd "#c4","#c2",%%zmm6%{%6%};"\
+    "vshuff64x2 $177,%%zmm4,%%zmm4,%%zmm4; vshuff64x2 $177,%%zmm6,%%zmm6,%%zmm6;"\
+    "vblendmpd "#c1",%%zmm4,"#c1"%{%6%}; vblendmpd "#c2",%%zmm6,"#c2"%{%6%};"\
+    "vblendmpd %%zmm4,"#c3","#c3"%{%6%}; vblendmpd %%zmm6,"#c4","#c4"%{%6%};"\
+
+#define INNER_TRANS_f128_4x4(c1,c2,c3,c4) \
+    "vshuff64x2 $68,"#c3","#c1",%%zmm4; vshuff64x2 $17,"#c4","#c2",%%zmm5;"\
+    "vshuff64x2 $238,"#c3","#c1",%%zmm6; vshuff64x2 $187,"#c4","#c2",%%zmm7;"\
+    "vblendmpd %%zmm5,%%zmm4,"#c2"%{%6%}; vshuff64x2 $177,"#c2","#c2","#c2"; vblendmpd %%zmm4,%%zmm5,"#c1"%{%6%};"\
+    "vblendmpd %%zmm7,%%zmm6,"#c4"%{%6%}; vshuff64x2 $177,"#c4","#c4","#c4"; vblendmpd %%zmm6,%%zmm7,"#c3"%{%6%};"
+
+#define INNER_TRANS_8x8(c1,c2,c3,c4,c5,c6,c7,c8) \
+    INNER_TRANS_f128_4x4(c1,c3,c5,c7) INNER_TRANS_f128_4x4(c2,c4,c6,c8)
+
+//%7 for k01(input) only when m=4
+#define INNER_STORE_4x8(c1,c2,c3,c4) \
+    "vmovupd (%10),%%zmm4%{%5%};vmovupd -32(%10,%4,4),%%zmm4%{%7%};vfmadd132pd %%zmm3,%%zmm4,"#c1";"\
+    "vmovupd "#c1",(%10)%{%5%}; vmovupd "#c1",-32(%10,%4,4)%{%7%}; leaq (%10,%4,1),%10;"\
+    "vmovupd (%10),%%zmm5%{%5%};vmovupd -32(%10,%4,4),%%zmm5%{%7%};vfmadd132pd %%zmm3,%%zmm5,"#c2";"\
+    "vmovupd "#c2",(%10)%{%5%}; vmovupd "#c2",-32(%10,%4,4)%{%7%}; leaq (%10,%4,1),%10;"\
+    "vmovupd (%10),%%zmm6%{%5%};vmovupd -32(%10,%4,4),%%zmm6%{%7%};vfmadd132pd %%zmm3,%%zmm6,"#c3";"\
+    "vmovupd "#c3",(%10)%{%5%}; vmovupd "#c3",-32(%10,%4,4)%{%7%}; leaq (%10,%4,1),%10;"\
+    "vmovupd (%10),%%zmm7%{%5%};vmovupd -32(%10,%4,4),%%zmm7%{%7%};vfmadd132pd %%zmm3,%%zmm7,"#c4";"\
+    "vmovupd "#c4",(%10)%{%5%}; vmovupd "#c4",-32(%10,%4,4)%{%7%}; leaq (%10,%4,1),%10;"\
+    "leaq (%10,%4,4),%10;"
+
+#define INNER_STORE_8x8(c1,c2,c3,c4,c5,c6,c7,c8) \
+    "vfmadd213pd (%10),%%zmm3,"#c1"; vmovupd "#c1",(%10); vfmadd213pd (%10,%4,1),%%zmm3,"#c2"; vmovupd "#c2",(%10,%4,1); leaq (%10,%4,2),%10;"\
+    "vfmadd213pd (%10),%%zmm3,"#c3"; vmovupd "#c3",(%10); vfmadd213pd (%10,%4,1),%%zmm3,"#c4"; vmovupd "#c4",(%10,%4,1); leaq (%10,%4,2),%10;"\
+    "vfmadd213pd (%10),%%zmm3,"#c5"; vmovupd "#c5",(%10); vfmadd213pd (%10,%4,1),%%zmm3,"#c6"; vmovupd "#c6",(%10,%4,1); leaq (%10,%4,2),%10;"\
+    "vfmadd213pd (%10),%%zmm3,"#c7"; vmovupd "#c7",(%10); vfmadd213pd (%10,%4,1),%%zmm3,"#c8"; vmovupd "#c8",(%10,%4,1); leaq (%10,%4,2),%10;"
+
+#define INNER_SAVE_m4n8 \
+    "movq %3,%10;"\
+    INNER_TRANS_4x8(%%zmm8,%%zmm9,%%zmm10,%%zmm11)\
+    INNER_STORE_4x8(%%zmm8,%%zmm9,%%zmm10,%%zmm11)
+
+#define INNER_SAVE_m4n16 \
+    INNER_SAVE_m4n8\
+    INNER_TRANS_4x8(%%zmm12,%%zmm13,%%zmm14,%%zmm15)\
+    INNER_STORE_4x8(%%zmm12,%%zmm13,%%zmm14,%%zmm15)
+
+#define INNER_SAVE_m4n24 \
+    INNER_SAVE_m4n16\
+    INNER_TRANS_4x8(%%zmm16,%%zmm17,%%zmm18,%%zmm19)\
+    INNER_STORE_4x8(%%zmm16,%%zmm17,%%zmm18,%%zmm19)
+
+#define INNER_SAVE_m8n8 \
+    "movq %3,%10;"\
+    INNER_TRANS_8x8(%%zmm8,%%zmm9,%%zmm10,%%zmm11,%%zmm12,%%zmm13,%%zmm14,%%zmm15)\
+    INNER_STORE_8x8(%%zmm8,%%zmm9,%%zmm10,%%zmm11,%%zmm12,%%zmm13,%%zmm14,%%zmm15)
+
+#define INNER_SAVE_m8n16 \
+    INNER_SAVE_m8n8\
+    INNER_TRANS_8x8(%%zmm16,%%zmm17,%%zmm18,%%zmm19,%%zmm20,%%zmm21,%%zmm22,%%zmm23)\
+    INNER_STORE_8x8(%%zmm16,%%zmm17,%%zmm18,%%zmm19,%%zmm20,%%zmm21,%%zmm22,%%zmm23)
+
+#define INNER_SAVE_m8n24 \
+    INNER_SAVE_m8n16\
+    INNER_TRANS_8x8(%%zmm24,%%zmm25,%%zmm26,%%zmm27,%%zmm28,%%zmm29,%%zmm30,%%zmm31)\
+    INNER_STORE_8x8(%%zmm24,%%zmm25,%%zmm26,%%zmm27,%%zmm28,%%zmm29,%%zmm30,%%zmm31)
+
+#define COMPUTE_n8 {\
+    b_pref = packed_b_pointer + 8 * K;\
+    __asm__ __volatile__(\
+    "vbroadcastsd (%9),%%zmm3;"\
+    "movq %8,%%r14;movq %2,%%r13;movq %2,%%r12;shlq $5,%%r12;"\
+    "cmpq $8,%8; jb 42222f;"\
+    "42221:\n\t"\
+    INNER_INIT_m8n8\
+    INNER_KERNELm8(8)\
+    INNER_SAVE_m8n8\
+    "movq %%r13,%2; subq %%r12,%1; subq %%r12,%1; addq %%r12,%0;"\
+    "addq $64,%3;"\
+    "subq $8,%8; cmpq $8,%8; jnb 42221b;"\
+    "42222:\n\t"\
+    "cmpq $4,%8; jb 42223f;"\
+    INNER_INIT_m4n8\
+    INNER_KERNELm4(8)\
+    INNER_SAVE_m4n8\
+    "movq %%r13,%2; subq %%r12,%1; subq %%r12,%1;"\
+    "addq $32,%3;"\
+    "subq $4,%8;"\
+    "42223:\n\t"\
+    "cmpq $2,%8; jb 42224f;"\
+    INNER_INIT_m2n8\
+    INNER_KERNELm2(8)\
+    INNER_SAVE_m2n8\
+    "movq %%r13,%2; subq %%r12,%1; subq %%r12,%1;"\
+    "addq $16,%3;"\
+    "subq $2,%8;"\
+    "42224:\n\t"\
+    "cmpq $1,%8; jb 42225f;"\
+    INNER_INIT_m1n8\
+    INNER_KERNELm1(8)\
+    INNER_SAVE_m1n8\
+    "movq %%r13,%2; subq %%r12,%1; subq %%r12,%1;"\
+    "addq $8,%3;"\
+    "42225:\n\t"\
+    "movq %%r14,%8;shlq $3,%8;subq %8,%3;shrq $3,%8;"\
+    "shlq $3,%4;addq %4,%3;shrq $3,%4;"\
+    :"+r"(a_block_pointer),"+r"(packed_b_pointer),"+r"(K),"+r"(c_pointer),"+r"(ldc_in_bytes),"+Yk"(k02),"+Yk"(k03),"+Yk"(k01),\
+    "+r"(M),"+r"(alpha),"+r"(c_store),"+r"(b_pref)\
+    ::"zmm0","zmm1","zmm2","zmm3","zmm4","zmm5","zmm6","zmm7","zmm8","zmm9","zmm10","zmm11","zmm12","zmm13","zmm14","zmm15","cc","memory","k1","r12","r13","r14");\
+    a_block_pointer -= M * K;\
+}
+#define COMPUTE_n16 {\
+    b_pref = packed_b_pointer + 16 * K;\
+    __asm__ __volatile__(\
+    "vbroadcastsd (%9),%%zmm3;"\
+    "movq %8,%%r14;movq %2,%%r13;movq %2,%%r12;shlq $5,%%r12;"\
+    "cmpq $8,%8; jb 32222f;"\
+    "32221:\n\t"\
+    INNER_INIT_m8n16\
+    INNER_KERNELm8(16)\
+    INNER_SAVE_m8n16\
+    "movq %%r13,%2; subq %%r12,%1; subq %%r12,%1; addq %%r12,%0;"\
+    "addq $64,%3;"\
+    "subq $8,%8; cmpq $8,%8; jnb 32221b;"\
+    "32222:\n\t"\
+    "cmpq $4,%8; jb 32223f;"\
+    INNER_INIT_m4n16\
+    INNER_KERNELm4(16)\
+    INNER_SAVE_m4n16\
+    "movq %%r13,%2; subq %%r12,%1; subq %%r12,%1;"\
+    "addq $32,%3;"\
+    "subq $4,%8;"\
+    "32223:\n\t"\
+    "cmpq $2,%8; jb 32224f;"\
+    INNER_INIT_m2n16\
+    INNER_KERNELm2(16)\
+    INNER_SAVE_m2n16\
+    "movq %%r13,%2; subq %%r12,%1; subq %%r12,%1;"\
+    "addq $16,%3;"\
+    "subq $2,%8;"\
+    "32224:\n\t"\
+    "cmpq $1,%8; jb 32225f;"\
+    INNER_INIT_m1n16\
+    INNER_KERNELm1(16)\
+    INNER_SAVE_m1n16\
+    "movq %%r13,%2; subq %%r12,%1; subq %%r12,%1;"\
+    "addq $8,%3;"\
+    "32225:\n\t"\
+    "movq %%r14,%8;shlq $3,%8;subq %8,%3;shrq $3,%8;"\
+    "shlq $4,%4;addq %4,%3;shrq $4,%4;"\
+    "leaq (%1,%%r12,4),%1;"\
+    :"+r"(a_block_pointer),"+r"(packed_b_pointer),"+r"(K),"+r"(c_pointer),"+r"(ldc_in_bytes),"+Yk"(k02),"+Yk"(k03),"+Yk"(k01),\
+    "+r"(M),"+r"(alpha),"+r"(c_store),"+r"(b_pref)\
+    ::"zmm0","zmm1","zmm2","zmm3","zmm4","zmm5","zmm6","zmm7","zmm8","zmm9","zmm10","zmm11","zmm12","zmm13","zmm14","zmm15","zmm16","zmm17",\
+    "zmm18","zmm19","zmm20","zmm21","zmm22","zmm23","cc","memory","k1","r12","r13","r14");\
+    a_block_pointer -= M * K;\
+}
+#define COMPUTE_n24 {\
+    b_pref = packed_b_pointer + 24 * K;\
+    __asm__ __volatile__(\
+    "vbroadcastsd (%9),%%zmm3;"\
+    "movq %8,%%r14;movq %2,%%r13;movq %2,%%r12;shlq $5,%%r12;"\
+    "cmpq $8,%8; jb 22222f;"\
+    "22221:\n\t"\
+    INNER_INIT_m8n24\
+    INNER_KERNELm8(24)\
+    INNER_SAVE_m8n24\
+    "movq %%r13,%2; subq %%r12,%1; subq %%r12,%1; addq %%r12,%0;"\
+    "addq $64,%3;"\
+    "subq $8,%8; cmpq $8,%8; jnb 22221b;"\
+    "22222:\n\t"\
+    "cmpq $4,%8; jb 22223f;"\
+    INNER_INIT_m4n24\
+    INNER_KERNELm4(24)\
+    INNER_SAVE_m4n24\
+    "movq %%r13,%2; subq %%r12,%1; subq %%r12,%1;"\
+    "addq $32,%3;"\
+    "subq $4,%8;"\
+    "22223:\n\t"\
+    "cmpq $2,%8; jb 22224f;"\
+    INNER_INIT_m2n24\
+    INNER_KERNELm2(24)\
+    INNER_SAVE_m2n24\
+    "movq %%r13,%2; subq %%r12,%1; subq %%r12,%1;"\
+    "addq $16,%3;"\
+    "subq $2,%8;"\
+    "22224:\n\t"\
+    "cmpq $1,%8; jb 22225f;"\
+    INNER_INIT_m1n24\
+    INNER_KERNELm1(24)\
+    INNER_SAVE_m1n24\
+    "movq %%r13,%2; subq %%r12,%1; subq %%r12,%1;"\
+    "addq $8,%3;"\
+    "22225:\n\t"\
+    "movq %%r14,%8;shlq $3,%8;subq %8,%3;shrq $3,%8;"\
+    "shlq $3,%4;addq %4,%3;shlq $1,%4;addq %4,%3;shrq $4,%4;"\
+    "leaq (%1,%%r12,4),%1; leaq (%1,%%r12,2),%1;"\
+    :"+r"(a_block_pointer),"+r"(packed_b_pointer),"+r"(K),"+r"(c_pointer),"+r"(ldc_in_bytes),"+Yk"(k02),"+Yk"(k03),"+Yk"(k01),\
+    "+r"(M),"+r"(alpha),"+r"(c_store),"+r"(b_pref)::\
+    "zmm0","zmm1","zmm2","zmm3","zmm4","zmm5","zmm6","zmm7","zmm8","zmm9","zmm10","zmm11","zmm12","zmm13","zmm14","zmm15","zmm16","zmm17","zmm18",\
+    "zmm19","zmm20","zmm21","zmm22","zmm23","zmm24","zmm25","zmm26","zmm27","zmm28","zmm29","zmm30","zmm31","cc","memory","k1","r12","r13","r14");\
+    a_block_pointer -= M * K;\
+}
+static void KERNEL_MAIN(double *packed_a, double *packed_b, BLASLONG m, BLASLONG ndiv8, BLASLONG k, BLASLONG LDC, double *c,double *alpha){//icopy=4,ocopy=8
+//perform C += A<pack> B<pack>
+    if(k==0 || m==0 || ndiv8==0) return;
+    int64_t ldc_in_bytes = (int64_t)LDC * sizeof(double);
+    int64_t K = (int64_t)k; int64_t M = (int64_t)m;
+    double *a_block_pointer,*b_pref;
+    double *c_pointer = c,*c_store = c;
+    __mmask16 k01 = 0x00f0,k02 = 0x000f,k03 = 0x0033;
+    BLASLONG ndiv8_count;
+    double *packed_b_pointer = packed_b;
+    a_block_pointer = packed_a;
+    for(ndiv8_count=ndiv8;ndiv8_count>2;ndiv8_count-=3){
+      COMPUTE_n24
+    }
+    for(;ndiv8_count>1;ndiv8_count-=2){
+      COMPUTE_n16
+    }
+    if(ndiv8_count>0){
+      COMPUTE_n8
+    }
+}
+
+/* __m256d accumulators: yc1-yc4; temporary variables: ya1,yb1-yb2 */
+/* __m128d accumulators: xc1-xc2; temporary variables: xa1,xb1-xb2 */
+/*  double accumulator:  sc1;     temporary variables: sa1,sb1 */
+/* column-major c_block */
+#define KERNEL_m4n4k1 {\
+    ya1 = _mm256_loadu_pd(a_block_pointer);a_block_pointer+=4;\
+    yb1 = _mm256_broadcast_sd(b_block_pointer);   yc1 = _mm256_fmadd_pd(ya1,yb1,yc1);\
+    yb2 = _mm256_broadcast_sd(b_block_pointer+1); yc2 = _mm256_fmadd_pd(ya1,yb2,yc2);\
+    yb1 = _mm256_broadcast_sd(b_block_pointer+2); yc3 = _mm256_fmadd_pd(ya1,yb1,yc3);\
+    yb2 = _mm256_broadcast_sd(b_block_pointer+3); yc4 = _mm256_fmadd_pd(ya1,yb2,yc4);\
+    b_block_pointer+=4;\
+}
+#define KERNEL_m4n2k1 {\
+    ya1 = _mm256_loadu_pd(a_block_pointer);a_block_pointer+=4;\
+    yb1 = _mm256_broadcast_sd(b_block_pointer);   yc1 = _mm256_fmadd_pd(ya1,yb1,yc1);\
+    yb2 = _mm256_broadcast_sd(b_block_pointer+1); yc2 = _mm256_fmadd_pd(ya1,yb2,yc2);\
+    b_block_pointer+=2;\
+}
+#define KERNEL_m4n1k1 {\
+    ya1 = _mm256_loadu_pd(a_block_pointer);a_block_pointer+=4;\
+    yb1 = _mm256_broadcast_sd(b_block_pointer);   yc1 = _mm256_fmadd_pd(ya1,yb1,yc1);\
+    b_block_pointer++;\
+}
+#define INIT_m4n1 yc1=_mm256_setzero_pd();
+#define INIT_m4n2 yc2=INIT_m4n1
+#define INIT_m4n4 yc4=yc3=INIT_m4n2
+#define SAVE_m4n1 {\
+    yb1 = _mm256_broadcast_sd(alpha);\
+    ya1 = _mm256_loadu_pd(c_pointer);\
+    yc1 = _mm256_fmadd_pd(yc1,yb1,ya1);\
+    _mm256_storeu_pd(c_pointer,yc1);\
+    c_pointer += 4;\
+}
+#define SAVE_m4n2 {\
+    ya1 = _mm256_broadcast_sd(alpha);\
+    yb1 = _mm256_loadu_pd(c_pointer); yb2 = _mm256_loadu_pd(c_pointer+LDC);\
+    yc1 = _mm256_fmadd_pd(yc1,ya1,yb1); yc2 = _mm256_fmadd_pd(yc2,ya1,yb2);\
+    _mm256_storeu_pd(c_pointer,yc1); _mm256_storeu_pd(c_pointer+LDC,yc2);\
+    c_pointer += 4;\
+}
+#define SAVE_m4n4 {\
+    ya1 = _mm256_broadcast_sd(alpha);\
+    yb1 = _mm256_loadu_pd(c_pointer); yb2 = _mm256_loadu_pd(c_pointer+LDC);\
+    yc1 = _mm256_fmadd_pd(yc1,ya1,yb1); yc2 = _mm256_fmadd_pd(yc2,ya1,yb2);\
+    _mm256_storeu_pd(c_pointer,yc1); _mm256_storeu_pd(c_pointer+LDC,yc2);\
+    c_pointer += LDC*2;\
+    yb1 = _mm256_loadu_pd(c_pointer); yb2 = _mm256_loadu_pd(c_pointer+LDC);\
+    yc3 = _mm256_fmadd_pd(yc3,ya1,yb1); yc4 = _mm256_fmadd_pd(yc4,ya1,yb2);\
+    _mm256_storeu_pd(c_pointer,yc3); _mm256_storeu_pd(c_pointer+LDC,yc4);\
+    c_pointer += 4-LDC*2;\
+}
+#define KERNEL_m2n2k1 {\
+    xa1 = _mm_loadu_pd(a_block_pointer); a_block_pointer+=2;\
+    xb1 = _mm_loaddup_pd(b_block_pointer);   xc1 = _mm_fmadd_pd(xa1,xb1,xc1);\
+    xb2 = _mm_loaddup_pd(b_block_pointer+1); xc2 = _mm_fmadd_pd(xa1,xb2,xc2);\
+    b_block_pointer += 2;\
+}
+#define KERNEL_m2n1k1 {\
+    xa1 = _mm_loadu_pd(a_block_pointer); a_block_pointer+=2;\
+    xb1 = _mm_loaddup_pd(b_block_pointer);   xc1 = _mm_fmadd_pd(xa1,xb1,xc1);\
+    b_block_pointer ++;\
+}
+#define INIT_m2n1 xc1=_mm_setzero_pd();
+#define INIT_m2n2 xc2=INIT_m2n1
+#define SAVE_m2n1 {\
+    xb1 = _mm_loaddup_pd(alpha);\
+    xa1 = _mm_loadu_pd(c_pointer);\
+    xc1 = _mm_fmadd_pd(xc1,xb1,xa1);\
+    _mm_storeu_pd(c_pointer,xc1);\
+    c_pointer += 2;\
+}
+#define SAVE_m2n2 {\
+    xa1 = _mm_loaddup_pd(alpha);\
+    xb1 = _mm_loadu_pd(c_pointer); xb2 = _mm_loadu_pd(c_pointer+LDC);\
+    xc1 = _mm_fmadd_pd(xc1,xa1,xb1); xc2 = _mm_fmadd_pd(xc2,xa1,xb2);\
+    _mm_storeu_pd(c_pointer,xc1); _mm_storeu_pd(c_pointer+LDC,xc2);\
+    c_pointer += 2;\
+}
+#define KERNEL_m1n1k1 {\
+    sa1 = *a_block_pointer; a_block_pointer++;\
+    sb1 = *b_block_pointer; sc1 += sa1 * sb1;\
+    b_block_pointer ++;\
+}
+#define INIT_m1n1 sc1=0.0;
+#define SAVE_m1n1 {\
+    *c_pointer += sc1 * (*alpha);\
+    c_pointer++;\
+}
+/* row-major c_block */
+#define KERNEL_m2n4k1 {\
+    yb1 = _mm256_loadu_pd(b_block_pointer);b_block_pointer+=4;\
+    ya1 = _mm256_broadcast_sd(a_block_pointer);  yc1 = _mm256_fmadd_pd(ya1,yb1,yc1);\
+    ya1 = _mm256_broadcast_sd(a_block_pointer+1);yc2 = _mm256_fmadd_pd(ya1,yb1,yc2);\
+    a_block_pointer += 2;\
+}
+#define KERNEL_m1n4k1 {\
+    yb1 = _mm256_loadu_pd(b_block_pointer);b_block_pointer+=4;\
+    ya1 = _mm256_broadcast_sd(a_block_pointer);  yc1 = _mm256_fmadd_pd(ya1,yb1,yc1);\
+    a_block_pointer ++;\
+}
+#define KERNEL_m1n2k1 {\
+    xb1 = _mm_loadu_pd(b_block_pointer);b_block_pointer+=2;\
+    xa1 = _mm_loaddup_pd(a_block_pointer); xc1 = _mm_fmadd_pd(xa1,xb1,xc1);\
+    a_block_pointer ++;\
+}
+#define INIT_m1n2 INIT_m2n1
+#define INIT_m1n4 INIT_m4n1
+#define INIT_m2n4 INIT_m4n2
+#define SAVE_m2n4 {\
+    ya1 = _mm256_broadcast_sd(alpha);\
+    yc1 = _mm256_mul_pd(yc1,ya1);\
+    yc2 = _mm256_mul_pd(yc2,ya1);\
+    yb1 = _mm256_unpacklo_pd(yc1,yc2);\
+    yb2 = _mm256_unpackhi_pd(yc1,yc2);\
+    xb1 = _mm_add_pd(_mm_loadu_pd(c_pointer),_mm256_extractf128_pd(yb1,0));\
+    xb2 = _mm_add_pd(_mm_loadu_pd(c_pointer+LDC),_mm256_extractf128_pd(yb2,0));\
+    _mm_storeu_pd(c_pointer,xb1);\
+    _mm_storeu_pd(c_pointer+LDC,xb2);\
+    xb1 = _mm_add_pd(_mm_loadu_pd(c_pointer+2*LDC),_mm256_extractf128_pd(yb1,1));\
+    xb2 = _mm_add_pd(_mm_loadu_pd(c_pointer+3*LDC),_mm256_extractf128_pd(yb2,1));\
+    _mm_storeu_pd(c_pointer+2*LDC,xb1);\
+    _mm_storeu_pd(c_pointer+3*LDC,xb2);\
+    c_pointer += 2;\
+}
+#define SAVE_m1n2 {\
+    xb1 = _mm_loaddup_pd(alpha);\
+    xc1 = _mm_mul_pd(xc1,xb1);\
+    *c_pointer += _mm_cvtsd_f64(xc1);\
+    xa1 = _mm_unpackhi_pd(xc1,xc1);\
+    c_pointer[LDC]+= _mm_cvtsd_f64(xa1);\
+    c_pointer ++;\
+}
+#define SAVE_m1n4 {\
+    ya1 = _mm256_broadcast_sd(alpha);\
+    yc1 = _mm256_mul_pd(yc1,ya1);\
+    xb1 = _mm256_extractf128_pd(yc1,0);\
+    *c_pointer += _mm_cvtsd_f64(xb1);\
+    xb2 = _mm_unpackhi_pd(xb1,xb1);\
+    c_pointer[LDC] += _mm_cvtsd_f64(xb2);\
+    xb1 = _mm256_extractf128_pd(yc1,1);\
+    c_pointer[LDC*2] += _mm_cvtsd_f64(xb1);\
+    xb2 = _mm_unpackhi_pd(xb1,xb1);\
+    c_pointer[LDC*3] += _mm_cvtsd_f64(xb2);\
+    c_pointer ++;\
+}
+static void KERNEL_EDGE(double *packed_a, double *packed_b, BLASLONG m, BLASLONG edge_n, BLASLONG k, BLASLONG LDC, double *c,double *alpha){//icopy=8,ocopy=8
+//perform C += A<pack> B<pack> , edge_n<8 must be satisfied.
+    if(k==0 || m==0 || edge_n==0 || (*alpha)==0.0) return;
+    double *a_block_pointer,*b_block_pointer,*b_base_pointer;
+    double *c_pointer = c;
+    __m256d yc1,yc2,yc3,yc4,ya1,yb1,yb2;
+    __m128d xc1,xc2,xa1,xb1,xb2;
+    double sc1,sa1,sb1;
+    BLASLONG m_count,n_count,k_count;
+    b_base_pointer = packed_b;
+//now start calculation of the edge part
+    for(n_count=edge_n;n_count>3;n_count-=4){
+      a_block_pointer = packed_a;
+      for(m_count=m;m_count>3;m_count-=4){
+        b_block_pointer = b_base_pointer;
+        INIT_m4n4
+        for(k_count=0;k_count<k;k_count++) KERNEL_m4n4k1
+        SAVE_m4n4
+      }
+      for(;m_count>1;m_count-=2){
+        b_block_pointer = b_base_pointer;
+        INIT_m2n4
+        for(k_count=0;k_count<k;k_count++) KERNEL_m2n4k1
+        SAVE_m2n4
+      }
+      if(m_count>0){
+        b_block_pointer = b_base_pointer;
+        INIT_m1n4
+        for(k_count=0;k_count<k;k_count++) KERNEL_m1n4k1
+        SAVE_m1n4
+      }
+      b_base_pointer += 4*k;
+      c_pointer += 4 * LDC - m;
+    }
+    for(;n_count>1;n_count-=2){
+      a_block_pointer = packed_a;
+      for(m_count=m;m_count>3;m_count-=4){
+        b_block_pointer = b_base_pointer;
+        INIT_m4n2
+        for(k_count=0;k_count<k;k_count++) KERNEL_m4n2k1
+        SAVE_m4n2
+      }
+      for(;m_count>1;m_count-=2){
+        b_block_pointer = b_base_pointer;
+        INIT_m2n2
+        for(k_count=0;k_count<k;k_count++) KERNEL_m2n2k1
+        SAVE_m2n2
+      }
+      if(m_count>0){
+        b_block_pointer = b_base_pointer;
+        INIT_m1n2
+        for(k_count=0;k_count<k;k_count++) KERNEL_m1n2k1
+        SAVE_m1n2
+      }
+      b_base_pointer += 2*k;
+      c_pointer += 2 * LDC - m;
+    }
+    if(n_count>0){
+      a_block_pointer = packed_a;
+      for(m_count=m;m_count>3;m_count-=4){
+        b_block_pointer = b_base_pointer;
+        INIT_m4n1
+        for(k_count=0;k_count<k;k_count++) KERNEL_m4n1k1
+        SAVE_m4n1
+      }
+      for(;m_count>1;m_count-=2){
+        b_block_pointer = b_base_pointer;
+        INIT_m2n1
+        for(k_count=0;k_count<k;k_count++) KERNEL_m2n1k1
+        SAVE_m2n1
+      }
+      if(m_count>0){
+        b_block_pointer = b_base_pointer;
+        INIT_m1n1
+        for(k_count=0;k_count<k;k_count++) KERNEL_m1n1k1
+        SAVE_m1n1
+      }
+    }
+}
+int __attribute__ ((noinline)) CNAME(BLASLONG m, BLASLONG n, BLASLONG k, double alpha, double * __restrict__ A, double * __restrict__ B, double * __restrict__ C, BLASLONG ldc){
+    if(m==0 || n==0 || k==0 || alpha == 0.0) return 0;
+    BLASLONG ndiv8 = n/8;double ALPHA = alpha;
+    double *packed_a = A;
+    if(ndiv8>0) KERNEL_MAIN(packed_a,B,m,ndiv8,k,ldc,C,&ALPHA);
+    if(n>ndiv8*8) KERNEL_EDGE(packed_a,B+(int64_t)k*(int64_t)ndiv8*8,m,n-ndiv8*8,k,ldc,C+(int64_t)ldc*(int64_t)ndiv8*8,&ALPHA);
+    return 0;
+}
diff --git a/kernel/x86_64/dgemm_kernel_8x2_bulldozer.S b/kernel/x86_64/dgemm_kernel_8x2_bulldozer.S
index 40c5892c6..c353a5913 100644
--- a/kernel/x86_64/dgemm_kernel_8x2_bulldozer.S
+++ b/kernel/x86_64/dgemm_kernel_8x2_bulldozer.S
@@ -1,4413 +1,4413 @@
-/*********************************************************************/
-/* Copyright 2009, 2010 The University of Texas at Austin.           */
-/* All rights reserved.                                              */
-/*                                                                   */
-/* Redistribution and use in source and binary forms, with or        */
-/* without modification, are permitted provided that the following   */
-/* conditions are met:                                               */
-/*                                                                   */
-/*   1. Redistributions of source code must retain the above         */
-/*      copyright notice, this list of conditions and the following  */
-/*      disclaimer.                                                  */
-/*                                                                   */
-/*   2. Redistributions in binary form must reproduce the above      */
-/*      copyright notice, this list of conditions and the following  */
-/*      disclaimer in the documentation and/or other materials       */
-/*      provided with the distribution.                              */
-/*                                                                   */
-/*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */
-/*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */
-/*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */
-/*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */
-/*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */
-/*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */
-/*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */
-/*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */
-/*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */
-/*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */
-/*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */
-/*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */
-/*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */
-/*    POSSIBILITY OF SUCH DAMAGE.                                    */
-/*                                                                   */
-/* The views and conclusions contained in the software and           */
-/* documentation are those of the authors and should not be          */
-/* interpreted as representing official policies, either expressed   */
-/* or implied, of The University of Texas at Austin.                 */
-/*********************************************************************/
-
-/*********************************************************************
-* 2013/06/02 Saar
-*
-* Parameter:
-* 	UNROLL_M	8
-*	UNROLL_N	2
-*	DGEMM_P		360
-*	DGEMM_Q		160
-*
-* Performance at m x n without prefetch of BO:
-* 
-* 5760x5760	93.4	GFLOPS with 8 threads on 4 modules (ACML: 90.8 GFLOPS)
-* 5760x5760	84.2	GFLOPS with 4 threads on 4 modules (ACML: 82.4 GFLOPS)
-* 3840x3840	50.3	GFLOPS with 2 threads on 2 modules (ACML: 49.5 GFLOPS)
-*
-* 5760x5760	56.4	GFLOPS with 4 threads on 2 modules (ACML: 58.5 GFLOPS)
-* 3840x3840	29.0	GFLOPS with 2 threads on 1 modules (ACML: 30.2 GFLOPS)
-* 3840x3840	26.1	GFLOPS with 1 threads on 1 modules (ACML: 25.9 GFLOPS)
-*
-*********************************************************************/
-
-/*********************************************************************
-* 2013/06/03 Saar
-*
-* Parameter:
-* 	UNROLL_M	8
-*	UNROLL_N	2
-*	DGEMM_P		336
-*	DGEMM_Q		168
-*	NO_WARMUP	1
-*	NO_AFFINITY	1
-*	GEMM_MULTITHREAD_THRESHOLD 4
-*
-* Performance at m x n with prefetch of BO:
-* 
-* 8064x3840	93.7	GFLOPS with 8 threads on 4 modules (ACML: 93.6 GFLOPS)
-* 6048x2880	85.1	GFLOPS with 4 threads on 4 modules (ACML: 84.2 GFLOPS)
-* 6048x2880	52.0	GFLOPS with 2 threads on 2 modules (ACML: 50.0 GFLOPS)
-*
-* 6048x2880	56.3	GFLOPS with 4 threads on 2 modules (ACML: 57.6 GFLOPS)
-* 4032x1920	29.5	GFLOPS with 2 threads on 1 modules (ACML: 30.5 GFLOPS)
-* 4032x1920	26.9	GFLOPS with 1 threads on 1 modules (ACML: 26.1 GFLOPS)
-*
-*********************************************************************/
-
-/*********************************************************************
-* 2013/06/04 Saar
-*
-* Parameter:
-* 	UNROLL_M	8
-*	UNROLL_N	2
-*	DGEMM_P		384
-*	DGEMM_Q		168
-*	NO_WARMUP	1
-*	NO_AFFINITY	1
-*	GEMM_MULTITHREAD_THRESHOLD 4
-*
-* Performance at m x n with prefetch of BO:
-* 
-* 6144x5376	94.6	GFLOPS with 8 threads on 4 modules (ACML: 90.5 GFLOPS)
-* 6144x5376	86.0	GFLOPS with 4 threads on 4 modules (ACML: 81.5 GFLOPS)
-* 4608x4032	52.0	GFLOPS with 2 threads on 2 modules (ACML: 47.5 GFLOPS)
-*
-* 6144x5376	57.3	GFLOPS with 4 threads on 2 modules (ACML: 56.5 GFLOPS)
-* 4608x4032	29.6	GFLOPS with 2 threads on 1 modules (ACML: 30.2 GFLOPS)
-* 4608x4032	26.9	GFLOPS with 1 threads on 1 modules (ACML: 25.6 GFLOPS)
-*
-*********************************************************************/
-
-
-
-#define ASSEMBLER
-#include "common.h"
- 
-#define OLD_M	%rdi
-#define OLD_N	%rsi
-#define M	%r13
-#define J	%r14
-#define OLD_K	%rdx
-
-#define A	%rcx
-#define B	%r8
-#define C	%r9
-#define LDC	%r10
-	
-#define I	%r11
-#define AO	%rdi
-#define BO	%rsi
-#define	CO1	%r15
-#define K	%r12
-#define BI	%rbp
-#define	SP	%rbx
-
-#define BO1	%rdi
-#define BO2	%r15
-
-#ifndef WINDOWS_ABI
-
-#define STACKSIZE 96
-
-#else
-
-#define STACKSIZE 256
-
-#define OLD_A		40 + STACKSIZE(%rsp)
-#define OLD_B		48 + STACKSIZE(%rsp)
-#define OLD_C		56 + STACKSIZE(%rsp)
-#define OLD_LDC		64 + STACKSIZE(%rsp)
-#define OLD_OFFSET	72 + STACKSIZE(%rsp)
-
-#endif
-
-#define L_BUFFER_SIZE 8192
-#define LB2_OFFSET    4096
-
-#define Ndiv6	 24(%rsp)
-#define Nmod6	 32(%rsp)
-#define N	 40(%rsp)
-#define ALPHA	 48(%rsp)
-#define OFFSET	 56(%rsp)
-#define KK	 64(%rsp)
-#define KKK	 72(%rsp)
-#define BUFFER1	           128(%rsp)
-#define BUFFER2	LB2_OFFSET+128(%rsp)
-
-#if defined(OS_WINDOWS)
-#if   L_BUFFER_SIZE > 16384
-#define STACK_TOUCH \
-        movl    $0,  4096 * 4(%rsp);\
-        movl    $0,  4096 * 3(%rsp);\
-        movl    $0,  4096 * 2(%rsp);\
-        movl    $0,  4096 * 1(%rsp);
-#elif L_BUFFER_SIZE > 12288
-#define STACK_TOUCH \
-        movl    $0,  4096 * 3(%rsp);\
-        movl    $0,  4096 * 2(%rsp);\
-        movl    $0,  4096 * 1(%rsp);
-#elif L_BUFFER_SIZE > 8192
-#define STACK_TOUCH \
-        movl    $0,  4096 * 2(%rsp);\
-        movl    $0,  4096 * 1(%rsp);
-#elif L_BUFFER_SIZE > 4096
-#define STACK_TOUCH \
-        movl    $0,  4096 * 1(%rsp);
-#else
-#define STACK_TOUCH
-#endif
-#else
-#define STACK_TOUCH
-#endif
-
-
-
-#define	A_PR1	384
-#define	B_PR1	192
-
-#define KERNEL8x3_1(xx) \
-	prefetcht0	A_PR1(AO,%rax,8)	;\
-	vmovddup	 -6 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovddup	 -5 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovddup	 -4 * SIZE(BO, BI, 8), %xmm3 ;\
-	vfmaddpd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-	vmovups 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	vfmaddpd  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
-	vmovups 	-12 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
-	vfmaddpd  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
-	vfmaddpd  	%xmm12,%xmm3,%xmm0,%xmm12 ;\
-	vmovups 	-10 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
-	vfmaddpd  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
-	vfmaddpd  	%xmm15,%xmm3,%xmm0,%xmm15 ;\
-
-#define KERNEL8x3_2(xx) \
-	prefetcht0	A_PR1+64(AO,%rax,8)	;\
-	vmovddup	 -3 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	 -8 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovddup	 -2 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovddup	 -1 * SIZE(BO, BI, 8), %xmm3 ;\
-	vfmaddpd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-	vmovups 	 -6 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	vfmaddpd  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
-	vmovups 	 -4 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
-	vfmaddpd  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
-	vfmaddpd  	%xmm12,%xmm3,%xmm0,%xmm12 ;\
-	vmovups 	 -2 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
-	vfmaddpd  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
-	vfmaddpd  	%xmm15,%xmm3,%xmm0,%xmm15 ;\
-
-#define KERNEL8x3_3(xx) \
-	prefetcht0	A_PR1+128(AO,%rax,8)	;\
-	vmovddup	  0 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	  0 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovddup	  1 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovddup	  2 * SIZE(BO, BI, 8), %xmm3 ;\
-	vfmaddpd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-	vmovups 	  2 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	vfmaddpd  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
-	vmovups 	  4 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
-	vfmaddpd  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
-	vfmaddpd  	%xmm12,%xmm3,%xmm0,%xmm12 ;\
-	vmovups 	  6 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
-	vfmaddpd  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
-	vfmaddpd  	%xmm15,%xmm3,%xmm0,%xmm15 ;\
-
-#define KERNEL8x3_4(xx) \
-	prefetcht0	A_PR1+192(AO,%rax,8)	;\
-	vmovddup	  3 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	  8 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovddup	  4 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovddup	  5 * SIZE(BO, BI, 8), %xmm3 ;\
-	vfmaddpd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-	vmovups 	 10 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	vfmaddpd  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
-	vmovups 	 12 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
-	vfmaddpd  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
-	vfmaddpd  	%xmm12,%xmm3,%xmm0,%xmm12 ;\
-	vmovups 	 14 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
-	vfmaddpd  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
-	vfmaddpd  	%xmm15,%xmm3,%xmm0,%xmm15 ;\
-	addq	$12, BI				  ;\
-	addq	$32, %rax 			  ;\
-
-#define KERNEL8x3_SUB(xx) \
-	vmovddup	 -6 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovddup	 -5 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovddup	 -4 * SIZE(BO, BI, 8), %xmm3 ;\
-	vfmaddpd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-	vmovups 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	vfmaddpd  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
-	vmovups 	-12 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
-	vfmaddpd  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
-	vfmaddpd  	%xmm12,%xmm3,%xmm0,%xmm12 ;\
-	vmovups 	-10 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
-	vfmaddpd  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
-	vfmaddpd  	%xmm15,%xmm3,%xmm0,%xmm15 ;\
-
-
-/*******************************************************************************************/
-
-#define KERNEL4x3_1(xx) \
-	vmovddup	 -6 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovddup	 -5 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovddup	 -4 * SIZE(BO, BI, 8), %xmm3 ;\
-	vfmaddpd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-	vmovups 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	vfmaddpd  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
-
-#define KERNEL4x3_2(xx) \
-	vmovddup	 -3 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	-12 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovddup	 -2 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovddup	 -1 * SIZE(BO, BI, 8), %xmm3 ;\
-	vfmaddpd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-	vmovups 	-10 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	vfmaddpd  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
-
-#define KERNEL4x3_3(xx) \
-	vmovddup	  0 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	 -8 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovddup	  1 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovddup	  2 * SIZE(BO, BI, 8), %xmm3 ;\
-	vfmaddpd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-	vmovups 	 -6 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	vfmaddpd  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
-
-#define KERNEL4x3_4(xx) \
-	vmovddup	  3 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	 -4 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovddup	  4 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovddup	  5 * SIZE(BO, BI, 8), %xmm3 ;\
-	vfmaddpd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-	vmovups 	 -2 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	vfmaddpd  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
-	addq	$12, BI				  ;\
-	addq	$16, %rax 			  ;\
-
-#define KERNEL4x3_SUB(xx) \
-	vmovddup	 -6 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovddup	 -5 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovddup	 -4 * SIZE(BO, BI, 8), %xmm3 ;\
-	vfmaddpd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-	vmovups 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	vfmaddpd  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
-
-
-
-
-
-/*******************************************************************************************/
-
-#define KERNEL2x3_1(xx) \
-	vmovddup	 -6 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovddup	 -5 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovddup	 -4 * SIZE(BO, BI, 8), %xmm3 ;\
-	vfmaddpd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-
-#define KERNEL2x3_2(xx) \
-	vmovddup	 -3 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovddup	 -2 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovddup	 -1 * SIZE(BO, BI, 8), %xmm3 ;\
-	vfmaddpd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-
-#define KERNEL2x3_3(xx) \
-	vmovddup	  0 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	-12 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovddup	  1 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovddup	  2 * SIZE(BO, BI, 8), %xmm3 ;\
-	vfmaddpd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-
-#define KERNEL2x3_4(xx) \
-	vmovddup	  3 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	-10 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovddup	  4 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovddup	  5 * SIZE(BO, BI, 8), %xmm3 ;\
-	vfmaddpd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-	addq	$12, BI				  ;\
-	addq	$8, %rax 			  ;\
-
-#define KERNEL2x3_SUB(xx) \
-	vmovddup	 -6 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovddup	 -5 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovddup	 -4 * SIZE(BO, BI, 8), %xmm3 ;\
-	vfmaddpd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-
-/*******************************************************************************************/
-
-#define KERNEL1x3_1(xx) \
-	vmovsd	 -6 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovsd 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovsd	 -5 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddsd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovsd	 -4 * SIZE(BO, BI, 8), %xmm3 ;\
-	vfmaddsd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-
-#define KERNEL1x3_2(xx) \
-	vmovsd	 -3 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovsd 	-15 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovsd	 -2 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddsd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovsd	 -1 * SIZE(BO, BI, 8), %xmm3 ;\
-	vfmaddsd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-
-#define KERNEL1x3_3(xx) \
-	vmovsd	  0 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovsd 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovsd	  1 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddsd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovsd	  2 * SIZE(BO, BI, 8), %xmm3 ;\
-	vfmaddsd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-
-#define KERNEL1x3_4(xx) \
-	vmovsd	  3 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovsd 	-13 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovsd	  4 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddsd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovsd	  5 * SIZE(BO, BI, 8), %xmm3 ;\
-	vfmaddsd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-	addq	$12, BI				  ;\
-	addq	$4, %rax 			  ;\
-
-#define KERNEL1x3_SUB(xx) \
-	vmovsd	 -6 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovsd 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovsd	 -5 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddsd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovsd	 -4 * SIZE(BO, BI, 8), %xmm3 ;\
-	vfmaddsd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-
-
-
-/*******************************************************************************************
-* 2 lines of N
-*******************************************************************************************/
-
-#define KERNEL8x2_1(xx) \
-	prefetcht0	A_PR1(AO,%rax,8)	;\
-	vmovddup	 -4 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovddup	 -3 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovups 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	vmovups 	-12 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
-	vfmaddpd  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
-	vmovups 	-10 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
-	vfmaddpd  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
-
-#define KERNEL8x2_2(xx) \
-	prefetcht0	A_PR1+64(AO,%rax,8)	;\
-	vmovddup	 -2 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	 -8 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovddup	 -1 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovups 	 -6 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	vmovups 	 -4 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
-	vfmaddpd  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
-	vmovups 	 -2 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
-	vfmaddpd  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
-
-#define KERNEL8x2_3(xx) \
-	prefetcht0	A_PR1+128(AO,%rax,8)	;\
-	vmovddup	  0 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	  0 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovddup	  1 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovups 	  2 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	vmovups 	  4 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
-	vfmaddpd  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
-	vmovups 	  6 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
-	vfmaddpd  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
-
-#define KERNEL8x2_4(xx) \
-	prefetcht0	A_PR1+192(AO,%rax,8)	;\
-	vmovddup	  2 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	  8 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovddup	  3 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovups 	 10 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	vmovups 	 12 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
-	vfmaddpd  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
-	vmovups 	 14 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
-	vfmaddpd  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
-	addq	$8, BI				  ;\
-	addq	$32, %rax 			  ;\
-
-#define KERNEL8x2_SUB(xx) \
-	vmovddup	 -4 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovddup	 -3 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovups 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	vmovups 	-12 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
-	vfmaddpd  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
-	vmovups 	-10 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
-	vfmaddpd  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
-
-
-/*******************************************************************************************/
-
-#define KERNEL4x2_1(xx) \
-	vmovddup	 -4 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovddup	 -3 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovups 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-
-#define KERNEL4x2_2(xx) \
-	vmovddup	 -2 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	-12 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovddup	 -1 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovups 	-10 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-
-#define KERNEL4x2_3(xx) \
-	vmovddup	  0 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	 -8 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovddup	  1 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovups 	 -6 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-
-#define KERNEL4x2_4(xx) \
-	vmovddup	  2 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	 -4 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovddup	  3 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovups 	 -2 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	addq	$8, BI				  ;\
-	addq	$16, %rax 			  ;\
-
-#define KERNEL4x2_SUB(xx) \
-	vmovddup	 -4 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovddup	 -3 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovups 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-
-
-/*******************************************************************************************/
-
-#define KERNEL2x2_1(xx) \
-	vmovddup	 -4 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovddup	 -3 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-
-#define KERNEL2x2_2(xx) \
-	vmovddup	 -2 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovddup	 -1 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-
-#define KERNEL2x2_3(xx) \
-	vmovddup	  0 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	-12 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovddup	  1 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-
-#define KERNEL2x2_4(xx) \
-	vmovddup	  2 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	-10 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovddup	  3 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	addq	$8, BI				  ;\
-	addq	$8, %rax 			  ;\
-
-#define KERNEL2x2_SUB(xx) \
-	vmovddup	 -4 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovddup	 -3 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-
-/*******************************************************************************************/
-
-#define KERNEL1x2_1(xx) \
-	vmovsd	 -4 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovsd 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovsd	 -3 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddsd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-
-#define KERNEL1x2_2(xx) \
-	vmovsd	 -2 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovsd 	-15 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovsd	 -1 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddsd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-
-#define KERNEL1x2_3(xx) \
-	vmovsd	  0 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovsd 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovsd	  1 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddsd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-
-#define KERNEL1x2_4(xx) \
-	vmovsd	  2 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovsd 	-13 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovsd	  3 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddsd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	addq	$8, BI				  ;\
-	addq	$4, %rax 			  ;\
-
-#define KERNEL1x2_SUB(xx) \
-	vmovsd	 -4 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovsd 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovsd	 -3 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddsd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-
-
-
-/*******************************************************************************************
-* 1 line of N
-*******************************************************************************************/
-
-#define KERNEL8x1_1(xx) \
-	prefetcht0	A_PR1(AO,%rax,8)	;\
-	vmovddup	 -2 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovups 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vmovups 	-12 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
-	vmovups 	-10 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
-
-#define KERNEL8x1_2(xx) \
-	prefetcht0	A_PR1+64(AO,%rax,8)	;\
-	vmovddup	 -1 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	 -8 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovups 	 -6 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vmovups 	 -4 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
-	vmovups 	 -2 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
-
-#define KERNEL8x1_3(xx) \
-	prefetcht0	A_PR1+128(AO,%rax,8)	;\
-	vmovddup	  0 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	  0 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovups 	  2 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vmovups 	  4 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
-	vmovups 	  6 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
-
-#define KERNEL8x1_4(xx) \
-	prefetcht0	A_PR1+192(AO,%rax,8)	;\
-	vmovddup	  1 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	  8 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovups 	 10 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vmovups 	 12 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
-	vmovups 	 14 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
-	addq	$4, BI				  ;\
-	addq	$32, %rax 			  ;\
-
-#define KERNEL8x1_SUB(xx) \
-	vmovddup	 -2 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovups 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vmovups 	-12 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
-	vmovups 	-10 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
-
-
-/*******************************************************************************************/
-
-#define KERNEL4x1_1(xx) \
-	vmovddup	 -2 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovups 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-
-#define KERNEL4x1_2(xx) \
-	vmovddup	 -1 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	-12 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovups 	-10 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-
-#define KERNEL4x1_3(xx) \
-	vmovddup	  0 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	 -8 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovups 	 -6 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-
-#define KERNEL4x1_4(xx) \
-	vmovddup	  1 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	 -4 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovups 	 -2 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	addq	$4, BI				  ;\
-	addq	$16, %rax 			  ;\
-
-#define KERNEL4x1_SUB(xx) \
-	vmovddup	 -2 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovups 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-
-
-/*******************************************************************************************/
-
-#define KERNEL2x1_1(xx) \
-	vmovddup	 -2 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-
-#define KERNEL2x1_2(xx) \
-	vmovddup	 -1 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-
-#define KERNEL2x1_3(xx) \
-	vmovddup	  0 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	-12 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-
-#define KERNEL2x1_4(xx) \
-	vmovddup	  1 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	-10 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	addq	$4, BI				  ;\
-	addq	$8, %rax 			  ;\
-
-#define KERNEL2x1_SUB(xx) \
-	vmovddup	 -2 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-
-/*******************************************************************************************/
-
-#define KERNEL1x1_1(xx) \
-	vmovsd	 -2 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovsd 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-
-#define KERNEL1x1_2(xx) \
-	vmovsd	 -1 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovsd 	-15 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-
-#define KERNEL1x1_3(xx) \
-	vmovsd	  0 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovsd 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-
-#define KERNEL1x1_4(xx) \
-	vmovsd	  1 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovsd 	-13 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	addq	$4, BI				  ;\
-	addq	$4, %rax 			  ;\
-
-#define KERNEL1x1_SUB(xx) \
-	vmovsd	 -2 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovsd 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-
-
-/*******************************************************************************************/
-
-#if !defined(TRMMKERNEL)
-
-
-	PROLOGUE
-	PROFCODE
-	
-	subq	$STACKSIZE, %rsp
-	movq	%rbx,   (%rsp)
-	movq	%rbp,  8(%rsp)
-	movq	%r12, 16(%rsp)
-	movq	%r13, 24(%rsp)
-	movq	%r14, 32(%rsp)
-	movq	%r15, 40(%rsp)
-
-	vzeroupper
-
-#ifdef WINDOWS_ABI
-	movq	%rdi,    48(%rsp)
-	movq	%rsi,    56(%rsp)
-	movups	%xmm6,   64(%rsp)
-	movups	%xmm7,   80(%rsp)
-	movups	%xmm8,   96(%rsp)
-	movups	%xmm9,  112(%rsp)
-	movups	%xmm10, 128(%rsp)
-	movups	%xmm11, 144(%rsp)
-	movups	%xmm12, 160(%rsp)
-	movups	%xmm13, 176(%rsp)
-	movups	%xmm14, 192(%rsp)
-	movups	%xmm15, 208(%rsp)
-
-	movq	ARG1,      OLD_M
-	movq	ARG2,      OLD_N
-	movq	ARG3,      OLD_K
-	movq	OLD_A,     A
-	movq	OLD_B,     B
-	movq	OLD_C,     C
-	movq	OLD_LDC,   LDC
-
-	vmovaps	%xmm3, %xmm0
-
-#else
-	movq	STACKSIZE +  8(%rsp), LDC
-
-#endif
-
-	movq    %rsp, SP      # save old stack
-        subq    $128 + L_BUFFER_SIZE, %rsp
-        andq    $-4096, %rsp    # align stack
-
-        STACK_TOUCH
-
-	cmpq	$0, OLD_M
-	je	.L999
-
-	cmpq	$0, OLD_N
-	je	.L999
-
-	cmpq	$0, OLD_K
-	je	.L999
-
-	movq	OLD_M, M
-	movq	OLD_N, N
-	movq	OLD_K, K
-
-	vmovsd	 %xmm0, ALPHA
-
-	salq	$BASE_SHIFT, LDC
-
-	movq    N, %rax
-        xorq    %rdx, %rdx
-        movq    $6,  %rdi
-        divq    %rdi                    //    N / 6
-        movq    %rax, Ndiv6             //    N / 6
-        movq    %rdx, Nmod6             //    N % 6
-
-	
-
-
-	movq	Ndiv6,  J
-	cmpq	$0, J
-	je	.L2_0
-	ALIGN_4
-
-.L6_01:
-        // copy to sub buffer
-        movq    K, %rax
-        salq    $1,%rax                 // K * 2
-        movq    B, BO1
-        leaq    (B,%rax,8), BO2         // next offset to BO2
-        leaq    BUFFER1, BO             // first buffer to BO
-        movq    K, %rax
-        sarq    $2, %rax                // K / 4
-        jz      .L6_02a
-        ALIGN_4
-
-.L6_02:
-	prefetcht0 512(BO1)
-	prefetcht0 512(BO2)
-	prefetchw  512(BO)
-	vmovups	      (BO1), %xmm0
-	vmovups	2*SIZE(BO1), %xmm2
-	vmovups	4*SIZE(BO1), %xmm4
-	vmovups	6*SIZE(BO1), %xmm6
-	vmovsd        (BO2), %xmm1
-	vmovsd  2*SIZE(BO2), %xmm3
-	vmovsd  4*SIZE(BO2), %xmm5
-	vmovsd  6*SIZE(BO2), %xmm7
-	vmovups	%xmm0,       (BO)
-	vmovsd	%xmm1, 2*SIZE(BO)
-	vmovups	%xmm2, 3*SIZE(BO)
-	vmovsd	%xmm3, 5*SIZE(BO)
-	vmovups	%xmm4, 6*SIZE(BO)
-	vmovsd	%xmm5, 8*SIZE(BO)
-	vmovups	%xmm6, 9*SIZE(BO)
-	vmovsd	%xmm7,11*SIZE(BO)
-	addq	$8*SIZE,BO1
-	addq	$8*SIZE,BO2
-	addq	$12*SIZE,BO
-	decq	%rax
-	jnz	.L6_02
-
-.L6_02a:
-
-	movq	K, %rax
-	andq	$3, %rax		// K % 4
-	jz	.L6_02c
-	ALIGN_4
-
-.L6_02b:
-
-	vmovups	(BO1), %xmm0
-	vmovsd  (BO2), %xmm1
-	vmovups	%xmm0,       (BO)
-	vmovsd	%xmm1, 2*SIZE(BO)
-	addq	$2*SIZE,BO1
-	addq	$2*SIZE,BO2
-	addq	$3*SIZE,BO
-	decq	%rax
-	jnz	.L6_02b
-
-.L6_02c:
-
-	movq	K, %rax
-	salq	$1,%rax			// K * 2
-	leaq	(B,%rax,8), BO1		// next offset to BO1
-	leaq	(BO1,%rax,8), BO2	// next offset to BO1
-	leaq    BUFFER2, BO		// second buffer to BO
-	movq	K, %rax
-	sarq	$2, %rax		// k / 4
-	jz	.L6_03a
-	ALIGN_4
-
-
-.L6_03:
-
-	prefetcht0 512(BO2)
-	prefetchw  512(BO)
-	vmovups	      (BO2), %xmm0
-	vmovups	2*SIZE(BO2), %xmm2
-	vmovups	4*SIZE(BO2), %xmm4
-	vmovups	6*SIZE(BO2), %xmm6
-	vmovsd  1*SIZE(BO1), %xmm1
-	vmovsd  3*SIZE(BO1), %xmm3
-	vmovsd  5*SIZE(BO1), %xmm5
-	vmovsd  7*SIZE(BO1), %xmm7
-	vmovsd	%xmm1, 0*SIZE(BO)
-	vmovups	%xmm0, 1*SIZE(BO)
-	vmovsd	%xmm3, 3*SIZE(BO)
-	vmovups	%xmm2, 4*SIZE(BO)
-	vmovsd	%xmm5, 6*SIZE(BO)
-	vmovups	%xmm4, 7*SIZE(BO)
-	vmovsd	%xmm7, 9*SIZE(BO)
-	vmovups	%xmm6,10*SIZE(BO)
-	addq	$8*SIZE,BO1
-	addq	$8*SIZE,BO2
-	addq	$12*SIZE,BO
-	decq	%rax
-	jnz	.L6_03
-
-.L6_03a:
-
-        movq    K, %rax
-        andq    $3, %rax                // K % 4
-        jz      .L6_03c
-        ALIGN_4
-
-
-.L6_03b:
-
-	vmovsd	  1*SIZE(BO1), %xmm0
-	vmovups  	(BO2), %xmm1
-	vmovsd	%xmm0,       (BO)
-	vmovups %xmm1, 1*SIZE(BO)
-	addq	$2*SIZE,BO1
-	addq	$2*SIZE,BO2
-	addq	$3*SIZE,BO
-	decq	%rax
-	jnz	.L6_03b
-
-
-.L6_03c:
-
-	movq	BO2, B			// next offset of B
-
-.L6_10:
-	movq	C, CO1
-	leaq	(C, LDC, 2), C		 
-	leaq	(C, LDC, 1), C		// c += 3 * ldc
-
-	
-	movq	A, AO		 	// aoffset = a
-	addq	$16 * SIZE, AO
-
-	movq	M,  I
-	sarq	$3, I			// i = (m >> 3)
-	je	.L6_20
-
-	ALIGN_4
-
-.L6_11:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $6 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax			//  K = K - ( K % 8 )
-	je	.L6_16
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L6_12:
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL8x3_1(xxx)
-	KERNEL8x3_2(xxx)
-	KERNEL8x3_3(xxx)
-	prefetcht0	B_PR1+64(BO,BI,8)
-	KERNEL8x3_4(xxx)
-
-	KERNEL8x3_1(xxx)
-	KERNEL8x3_2(xxx)
-	prefetcht0	B_PR1+32(BO,BI,8)
-	KERNEL8x3_3(xxx)
-	KERNEL8x3_4(xxx)
-
-	je	.L6_16
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL8x3_1(xxx)
-	KERNEL8x3_2(xxx)
-	KERNEL8x3_3(xxx)
-	prefetcht0	B_PR1+64(BO,BI,8)
-	KERNEL8x3_4(xxx)
-
-	KERNEL8x3_1(xxx)
-	KERNEL8x3_2(xxx)
-	prefetcht0	B_PR1+32(BO,BI,8)
-	KERNEL8x3_3(xxx)
-	KERNEL8x3_4(xxx)
-
-	je	.L6_16
-
-	jmp	.L6_12
-	ALIGN_4
-
-.L6_16:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L6_19
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L6_17:
-
-	KERNEL8x3_SUB(xxx)
-	addq	$3, BI
-	addq	$8, %rax
-	jl	.L6_17
-	ALIGN_4
-
-
-.L6_19:
-
-	vmovddup	ALPHA, %xmm0
-
-
-	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7
-	vfmaddpd 4 * SIZE(CO1),%xmm0, %xmm10,%xmm10
-	vfmaddpd 6 * SIZE(CO1),%xmm0, %xmm13,%xmm13
-
-	vfmaddpd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-	vfmaddpd 2 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
-	vfmaddpd 4 * SIZE(CO1, LDC),%xmm0, %xmm11,%xmm11
-	vfmaddpd 6 * SIZE(CO1, LDC),%xmm0, %xmm14,%xmm14
-
-	vfmaddpd 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
-	vfmaddpd 2 * SIZE(CO1, LDC, 2),%xmm0, %xmm9,%xmm9
-	vfmaddpd 4 * SIZE(CO1, LDC, 2),%xmm0, %xmm12,%xmm12
-	vfmaddpd 6 * SIZE(CO1, LDC, 2),%xmm0, %xmm15,%xmm15
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm7 , 2 * SIZE(CO1)
-	vmovups	%xmm10, 4 * SIZE(CO1)
-	vmovups	%xmm13, 6 * SIZE(CO1)
-
-	vmovups	%xmm5 ,  	(CO1, LDC)
-	vmovups	%xmm8 , 2 * SIZE(CO1, LDC)
-	vmovups	%xmm11, 4 * SIZE(CO1, LDC)
-	vmovups	%xmm14, 6 * SIZE(CO1, LDC)
-
-	vmovups	%xmm6 ,  	(CO1, LDC, 2)
-	vmovups	%xmm9 , 2 * SIZE(CO1, LDC, 2)
-	vmovups	%xmm12, 4 * SIZE(CO1, LDC, 2)
-	vmovups	%xmm15, 6 * SIZE(CO1, LDC, 2)
-
-
-	addq	$8 * SIZE, CO1		# coffset += 8
-	decq	I			# i --
-	jg	.L6_11
-	ALIGN_4	
-
-/**************************************************************************
-* Rest of M 
-***************************************************************************/
-.L6_20:
-	// Test rest of M
-
-	testq	$7, M
-	jz	.L7_10		// to next 3 lines of N
-
-	testq	$4, M		
-	jz	.L6_30
-
-	ALIGN_4
-
-.L6_21:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $6 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L6_26
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L6_22:
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL4x3_1(xxx)
-	KERNEL4x3_2(xxx)
-	KERNEL4x3_3(xxx)
-	prefetcht0	B_PR1+64(BO,BI,8)
-	KERNEL4x3_4(xxx)
-
-	KERNEL4x3_1(xxx)
-	KERNEL4x3_2(xxx)
-	prefetcht0	B_PR1+32(BO,BI,8)
-	KERNEL4x3_3(xxx)
-	KERNEL4x3_4(xxx)
-
-	je	.L6_26
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL4x3_1(xxx)
-	KERNEL4x3_2(xxx)
-	KERNEL4x3_3(xxx)
-	prefetcht0	B_PR1+64(BO,BI,8)
-	KERNEL4x3_4(xxx)
-
-	KERNEL4x3_1(xxx)
-	KERNEL4x3_2(xxx)
-	prefetcht0	B_PR1+32(BO,BI,8)
-	KERNEL4x3_3(xxx)
-	KERNEL4x3_4(xxx)
-
-	je	.L6_26
-
-	jmp	.L6_22
-	ALIGN_4
-
-.L6_26:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L6_29
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L6_27:
-
-	KERNEL4x3_SUB(xxx)
-	addq	$3, BI
-	addq	$4, %rax
-	jl	.L6_27
-	ALIGN_4
-
-
-.L6_29:
-
-	vmovddup	ALPHA, %xmm0
-
-
-	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7
-
-	vfmaddpd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-	vfmaddpd 2 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
-
-	vfmaddpd 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
-	vfmaddpd 2 * SIZE(CO1, LDC, 2),%xmm0, %xmm9,%xmm9
-
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm7 , 2 * SIZE(CO1)
-
-	vmovups	%xmm5 ,  	(CO1, LDC)
-	vmovups	%xmm8 , 2 * SIZE(CO1, LDC)
-
-	vmovups	%xmm6 ,  	(CO1, LDC, 2)
-	vmovups	%xmm9 , 2 * SIZE(CO1, LDC, 2)
-
-
-	addq	$4 * SIZE, CO1		# coffset += 4
-	ALIGN_4
-	
-
-.L6_30:
-	testq	$2, M		
-	jz	.L6_40
-
-	ALIGN_4
-
-.L6_31:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $6 * SIZE, BO
-
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L6_36
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L6_32:
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL2x3_1(xxx)
-	KERNEL2x3_2(xxx)
-	KERNEL2x3_3(xxx)
-	prefetcht0	B_PR1+64(BO,BI,8)
-	KERNEL2x3_4(xxx)
-
-	KERNEL2x3_1(xxx)
-	KERNEL2x3_2(xxx)
-	prefetcht0	B_PR1+32(BO,BI,8)
-	KERNEL2x3_3(xxx)
-	KERNEL2x3_4(xxx)
-
-	je	.L6_36
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL2x3_1(xxx)
-	KERNEL2x3_2(xxx)
-	KERNEL2x3_3(xxx)
-	prefetcht0	B_PR1+64(BO,BI,8)
-	KERNEL2x3_4(xxx)
-
-	KERNEL2x3_1(xxx)
-	KERNEL2x3_2(xxx)
-	prefetcht0	B_PR1+32(BO,BI,8)
-	KERNEL2x3_3(xxx)
-	KERNEL2x3_4(xxx)
-
-	je	.L6_36
-
-	jmp	.L6_32
-	ALIGN_4
-
-.L6_36:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L6_39
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-	
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L6_37:
-
-	KERNEL2x3_SUB(xxx)
-	addq	$3, BI
-	addq	$2, %rax
-	jl	.L6_37
-	ALIGN_4
-
-
-.L6_39:
-
-	vmovddup	ALPHA, %xmm0
-
-
-	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddpd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-	vfmaddpd 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
-
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm5 ,  	(CO1, LDC)
-	vmovups	%xmm6 ,  	(CO1, LDC, 2)
-
-
-	addq	$2 * SIZE, CO1		# coffset += 2
-	ALIGN_4
-
-.L6_40:
-	testq	$1, M		
-	jz	.L7_10		// to next 3 lines of N
-
-	ALIGN_4
-
-.L6_41:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $6 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L6_46
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L6_42:
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL1x3_1(xxx)
-	KERNEL1x3_2(xxx)
-	KERNEL1x3_3(xxx)
-	prefetcht0	B_PR1+64(BO,BI,8)
-	KERNEL1x3_4(xxx)
-
-	KERNEL1x3_1(xxx)
-	KERNEL1x3_2(xxx)
-	prefetcht0	B_PR1+32(BO,BI,8)
-	KERNEL1x3_3(xxx)
-	KERNEL1x3_4(xxx)
-
-	je	.L6_46
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL1x3_1(xxx)
-	KERNEL1x3_2(xxx)
-	KERNEL1x3_3(xxx)
-	prefetcht0	B_PR1+64(BO,BI,8)
-	KERNEL1x3_4(xxx)
-
-	KERNEL1x3_1(xxx)
-	KERNEL1x3_2(xxx)
-	prefetcht0	B_PR1+32(BO,BI,8)
-	KERNEL1x3_3(xxx)
-	KERNEL1x3_4(xxx)
-
-	je	.L6_46
-
-	jmp	.L6_42
-	ALIGN_4
-
-.L6_46:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L6_49
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L6_47:
-
-	KERNEL1x3_SUB(xxx)
-	addq	$3, BI
-	addq	$1, %rax
-	jl	.L6_47
-	ALIGN_4
-
-
-.L6_49:
-
-	vmovddup	ALPHA, %xmm0
-
-
-	vfmaddsd 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddsd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-	vfmaddsd 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
-
-	vmovsd	%xmm4 ,  	(CO1)
-	vmovsd	%xmm5 ,  	(CO1, LDC)
-	vmovsd	%xmm6 ,  	(CO1, LDC, 2)
-
-
-	addq	$1 * SIZE, CO1		# coffset += 1
-	ALIGN_4
-	
-
-
-
-/***************************************************************************************************************/
-
-.L7_10:
-	movq	C, CO1
-	leaq	(C, LDC, 2), C		 
-	leaq	(C, LDC, 1), C		// c += 3 * ldc
-
-
-	movq	A, AO		 	// aoffset = a
-	addq	$16 * SIZE, AO
-
-	movq	M,  I
-	sarq	$3, I			// i = (m >> 3)
-	je	.L7_20
-	ALIGN_4
-
-.L7_11:
-        leaq    BUFFER2, BO             // second buffer to BO
-        addq    $6 * SIZE, BO
-
-
-	vzeroall
-
-        movq    K, %rax
-
-
-	andq	$-8, %rax
-	je	.L7_16
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-
-	ALIGN_4
-
-.L7_12:
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL8x3_1(xxx)
-	KERNEL8x3_2(xxx)
-	KERNEL8x3_3(xxx)
-	prefetcht0	B_PR1+64(BO,BI,8)
-	KERNEL8x3_4(xxx)
-
-	KERNEL8x3_1(xxx)
-	KERNEL8x3_2(xxx)
-	prefetcht0	B_PR1+32(BO,BI,8)
-	KERNEL8x3_3(xxx)
-	KERNEL8x3_4(xxx)
-
-	je	.L7_16
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL8x3_1(xxx)
-	KERNEL8x3_2(xxx)
-	KERNEL8x3_3(xxx)
-	prefetcht0	B_PR1+64(BO,BI,8)
-	KERNEL8x3_4(xxx)
-
-	KERNEL8x3_1(xxx)
-	KERNEL8x3_2(xxx)
-	prefetcht0	B_PR1+32(BO,BI,8)
-	KERNEL8x3_3(xxx)
-	KERNEL8x3_4(xxx)
-
-	je	.L7_16
-
-	jmp	.L7_12
-	ALIGN_4
-
-.L7_16:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L7_19
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L7_17:
-
-	KERNEL8x3_SUB(xxx)
-	addq	$3, BI
-	addq	$8, %rax
-	jl	.L7_17
-	ALIGN_4
-
-
-.L7_19:
-
-	vmovddup	ALPHA, %xmm0
-
-	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7
-	vfmaddpd 4 * SIZE(CO1),%xmm0, %xmm10,%xmm10
-	vfmaddpd 6 * SIZE(CO1),%xmm0, %xmm13,%xmm13
-
-	vfmaddpd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-	vfmaddpd 2 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
-	vfmaddpd 4 * SIZE(CO1, LDC),%xmm0, %xmm11,%xmm11
-	vfmaddpd 6 * SIZE(CO1, LDC),%xmm0, %xmm14,%xmm14
-
-	vfmaddpd 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
-	vfmaddpd 2 * SIZE(CO1, LDC, 2),%xmm0, %xmm9,%xmm9
-	vfmaddpd 4 * SIZE(CO1, LDC, 2),%xmm0, %xmm12,%xmm12
-	vfmaddpd 6 * SIZE(CO1, LDC, 2),%xmm0, %xmm15,%xmm15
-
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm7 , 2 * SIZE(CO1)
-	vmovups	%xmm10, 4 * SIZE(CO1)
-	vmovups	%xmm13, 6 * SIZE(CO1)
-
-	vmovups	%xmm5 ,  	(CO1, LDC)
-	vmovups	%xmm8 , 2 * SIZE(CO1, LDC)
-	vmovups	%xmm11, 4 * SIZE(CO1, LDC)
-	vmovups	%xmm14, 6 * SIZE(CO1, LDC)
-
-	vmovups	%xmm6 ,  	(CO1, LDC, 2)
-	vmovups	%xmm9 , 2 * SIZE(CO1, LDC, 2)
-	vmovups	%xmm12, 4 * SIZE(CO1, LDC, 2)
-	vmovups	%xmm15, 6 * SIZE(CO1, LDC, 2)
-
-
-
-
-	addq	$8 * SIZE, CO1		# coffset += 8
-	decq	I			# i --
-	jg	.L7_11
-	ALIGN_4	
-
-.L7_20:
-	// Test rest of M
-
-	testq	$7, M
-	jz	.L7_60		// to next 6 lines of N
-
-	testq	$4, M		
-	jz	.L7_30
-
-	ALIGN_4
-
-.L7_21:
-        leaq    BUFFER2, BO             // second buffer to BO
-        addq    $6 * SIZE, BO
-
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L7_26
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L7_22:
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL4x3_1(xxx)
-	KERNEL4x3_2(xxx)
-	KERNEL4x3_3(xxx)
-	prefetcht0	B_PR1+64(BO,BI,8)
-	KERNEL4x3_4(xxx)
-
-	KERNEL4x3_1(xxx)
-	KERNEL4x3_2(xxx)
-	prefetcht0	B_PR1+32(BO,BI,8)
-	KERNEL4x3_3(xxx)
-	KERNEL4x3_4(xxx)
-
-	je	.L7_26
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL4x3_1(xxx)
-	KERNEL4x3_2(xxx)
-	KERNEL4x3_3(xxx)
-	prefetcht0	B_PR1+64(BO,BI,8)
-	KERNEL4x3_4(xxx)
-
-	KERNEL4x3_1(xxx)
-	KERNEL4x3_2(xxx)
-	prefetcht0	B_PR1+32(BO,BI,8)
-	KERNEL4x3_3(xxx)
-	KERNEL4x3_4(xxx)
-
-	je	.L7_26
-
-	jmp	.L7_22
-	ALIGN_4
-
-.L7_26:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L7_29
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L7_27:
-
-	KERNEL4x3_SUB(xxx)
-	addq	$3, BI
-	addq	$4, %rax
-	jl	.L7_27
-	ALIGN_4
-
-
-.L7_29:
-
-	vmovddup	ALPHA, %xmm0
-
-
-	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7
-
-	vfmaddpd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-	vfmaddpd 2 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
-
-	vfmaddpd 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
-	vfmaddpd 2 * SIZE(CO1, LDC, 2),%xmm0, %xmm9,%xmm9
-
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm7 , 2 * SIZE(CO1)
-
-	vmovups	%xmm5 ,  	(CO1, LDC)
-	vmovups	%xmm8 , 2 * SIZE(CO1, LDC)
-
-	vmovups	%xmm6 ,  	(CO1, LDC, 2)
-	vmovups	%xmm9 , 2 * SIZE(CO1, LDC, 2)
-
-
-	addq	$4 * SIZE, CO1		# coffset += 4
-	ALIGN_4
-	
-
-.L7_30:
-	testq	$2, M		
-	jz	.L7_40
-
-	ALIGN_4
-
-.L7_31:
-        leaq    BUFFER2, BO             // second buffer to BO
-        addq    $6 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L7_36
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L7_32:
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL2x3_1(xxx)
-	KERNEL2x3_2(xxx)
-	KERNEL2x3_3(xxx)
-	prefetcht0	B_PR1+64(BO,BI,8)
-	KERNEL2x3_4(xxx)
-
-	KERNEL2x3_1(xxx)
-	KERNEL2x3_2(xxx)
-	prefetcht0	B_PR1+32(BO,BI,8)
-	KERNEL2x3_3(xxx)
-	KERNEL2x3_4(xxx)
-
-	je	.L7_36
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL2x3_1(xxx)
-	KERNEL2x3_2(xxx)
-	KERNEL2x3_3(xxx)
-	prefetcht0	B_PR1+64(BO,BI,8)
-	KERNEL2x3_4(xxx)
-
-	KERNEL2x3_1(xxx)
-	KERNEL2x3_2(xxx)
-	prefetcht0	B_PR1+32(BO,BI,8)
-	KERNEL2x3_3(xxx)
-	KERNEL2x3_4(xxx)
-
-	je	.L7_36
-
-	jmp	.L7_32
-	ALIGN_4
-
-.L7_36:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L7_39
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L7_37:
-
-	KERNEL2x3_SUB(xxx)
-	addq	$3, BI
-	addq	$2, %rax
-	jl	.L7_37
-	ALIGN_4
-
-
-.L7_39:
-
-	vmovddup	ALPHA, %xmm0
-
-
-	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddpd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-	vfmaddpd 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
-
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm5 ,  	(CO1, LDC)
-	vmovups	%xmm6 ,  	(CO1, LDC, 2)
-
-
-	addq	$2 * SIZE, CO1		# coffset += 2
-	ALIGN_4
-	
-
-
-
-
-.L7_40:
-	testq	$1, M		
-	jz	.L7_60		// to next 6 lines of N
-
-	ALIGN_4
-
-.L7_41:
-        leaq    BUFFER2, BO             // second buffer to BO
-        addq    $6 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-
-	andq	$-8, %rax
-	je	.L7_46
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L7_42:
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL1x3_1(xxx)
-	KERNEL1x3_2(xxx)
-	KERNEL1x3_3(xxx)
-	prefetcht0	B_PR1+64(BO,BI,8)
-	KERNEL1x3_4(xxx)
-
-	KERNEL1x3_1(xxx)
-	KERNEL1x3_2(xxx)
-	prefetcht0	B_PR1+32(BO,BI,8)
-	KERNEL1x3_3(xxx)
-	KERNEL1x3_4(xxx)
-
-	je	.L7_46
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL1x3_1(xxx)
-	KERNEL1x3_2(xxx)
-	KERNEL1x3_3(xxx)
-	prefetcht0	B_PR1+64(BO,BI,8)
-	KERNEL1x3_4(xxx)
-
-	KERNEL1x3_1(xxx)
-	KERNEL1x3_2(xxx)
-	prefetcht0	B_PR1+32(BO,BI,8)
-	KERNEL1x3_3(xxx)
-	KERNEL1x3_4(xxx)
-
-	je	.L7_46
-
-	jmp	.L7_42
-	ALIGN_4
-
-.L7_46:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L7_49
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L7_47:
-
-	KERNEL1x3_SUB(xxx)
-	addq	$3, BI
-	addq	$1, %rax
-	jl	.L7_47
-	ALIGN_4
-
-
-.L7_49:
-
-	vmovddup	ALPHA, %xmm0
-
-
-	vfmaddsd 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddsd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-	vfmaddsd 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
-
-
-	vmovsd	%xmm4 ,  	(CO1)
-	vmovsd	%xmm5 ,  	(CO1, LDC)
-	vmovsd	%xmm6 ,  	(CO1, LDC, 2)
-
-
-	addq	$1 * SIZE, CO1		# coffset += 1
-
-.L7_60:
-
-	decq	J			// j --
-	jg	.L6_01
-
-
-.L2_0:
-	cmpq	$0, Nmod6		// N % 6 == 0
-	je	.L999
-
-/************************************************************************************************
-* Loop for Nmod6 / 2 > 0
-*************************************************************************************************/
-
-	movq	Nmod6, J		
-	sarq	$1, J			// j = j / 2
-	je	.L1_0
-	ALIGN_4
-
-.L2_01:
-	// copy to sub buffer
-	movq	B, BO1
-	leaq    BUFFER1, BO		// first buffer to BO
-	movq	K, %rax
-	ALIGN_4
-
-.L2_02b:
-
-	vmovups	(BO1), %xmm0
-	vmovups	%xmm0,       (BO)
-	addq	$2*SIZE,BO1
-	addq	$2*SIZE,BO
-	decq	%rax
-	jnz	.L2_02b
-
-.L2_02c:
-
-	movq	BO1, B			// next offset of B
-
-.L2_10:
-	movq	C, CO1
-	leaq	(C, LDC, 2), C		// c += 2 * ldc
-
-	
-	movq	A, AO		 	// aoffset = a
-	addq	$16 * SIZE, AO
-
-	movq	M,  I
-	sarq	$3, I			// i = (m >> 3)
-	je	.L2_20
-
-	ALIGN_4
-
-.L2_11:
-
-	leaq	BUFFER1, BO		// first buffer to BO
-	addq	$4 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax			//  K = K - ( K % 8 )
-	je	.L2_16
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_12:
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL8x2_1(xxx)
-	KERNEL8x2_2(xxx)
-	KERNEL8x2_3(xxx)
-	KERNEL8x2_4(xxx)
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL8x2_1(xxx)
-	KERNEL8x2_2(xxx)
-	KERNEL8x2_3(xxx)
-	KERNEL8x2_4(xxx)
-
-	je	.L2_16
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL8x2_1(xxx)
-	KERNEL8x2_2(xxx)
-	KERNEL8x2_3(xxx)
-	KERNEL8x2_4(xxx)
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL8x2_1(xxx)
-	KERNEL8x2_2(xxx)
-	KERNEL8x2_3(xxx)
-	KERNEL8x2_4(xxx)
-
-	je	.L2_16
-
-	jmp	.L2_12
-	ALIGN_4
-
-.L2_16:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_19
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_17:
-
-	KERNEL8x2_SUB(xxx)
-	addq	$2, BI
-	addq	$8, %rax
-	jl	.L2_17
-	ALIGN_4
-
-
-.L2_19:
-
-	vmovddup	ALPHA, %xmm0
-
-	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7
-	vfmaddpd 4 * SIZE(CO1),%xmm0, %xmm10,%xmm10
-	vfmaddpd 6 * SIZE(CO1),%xmm0, %xmm13,%xmm13
-
-	vfmaddpd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-	vfmaddpd 2 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
-	vfmaddpd 4 * SIZE(CO1, LDC),%xmm0, %xmm11,%xmm11
-	vfmaddpd 6 * SIZE(CO1, LDC),%xmm0, %xmm14,%xmm14
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm7 , 2 * SIZE(CO1)
-	vmovups	%xmm10, 4 * SIZE(CO1)
-	vmovups	%xmm13, 6 * SIZE(CO1)
-
-	vmovups	%xmm5 ,  	(CO1, LDC)
-	vmovups	%xmm8 , 2 * SIZE(CO1, LDC)
-	vmovups	%xmm11, 4 * SIZE(CO1, LDC)
-	vmovups	%xmm14, 6 * SIZE(CO1, LDC)
-
-
-	addq	$8 * SIZE, CO1		# coffset += 8
-	decq	I			# i --
-	jg	.L2_11
-	ALIGN_4	
-
-/**************************************************************************
-* Rest of M 
-***************************************************************************/
-.L2_20:
-	// Test rest of M
-
-	testq	$7, M
-	jz	.L2_60		// to next 2 lines of N
-
-	testq	$4, M		
-	jz	.L2_30
-
-	ALIGN_4
-
-.L2_21:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L2_26
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_22:
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL4x2_1(xxx)
-	KERNEL4x2_2(xxx)
-	KERNEL4x2_3(xxx)
-	KERNEL4x2_4(xxx)
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL4x2_1(xxx)
-	KERNEL4x2_2(xxx)
-	KERNEL4x2_3(xxx)
-	KERNEL4x2_4(xxx)
-
-	je	.L2_26
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL4x2_1(xxx)
-	KERNEL4x2_2(xxx)
-	KERNEL4x2_3(xxx)
-	KERNEL4x2_4(xxx)
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL4x2_1(xxx)
-	KERNEL4x2_2(xxx)
-	KERNEL4x2_3(xxx)
-	KERNEL4x2_4(xxx)
-
-	je	.L2_26
-
-	jmp	.L2_22
-	ALIGN_4
-
-.L2_26:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_29
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_27:
-
-	KERNEL4x2_SUB(xxx)
-	addq	$2, BI
-	addq	$4, %rax
-	jl	.L2_27
-	ALIGN_4
-
-
-.L2_29:
-
-	vmovddup	ALPHA, %xmm0
-
-	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7
-
-	vfmaddpd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-	vfmaddpd 2 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm7 , 2 * SIZE(CO1)
-
-	vmovups	%xmm5 ,  	(CO1, LDC)
-	vmovups	%xmm8 , 2 * SIZE(CO1, LDC)
-
-	addq	$4 * SIZE, CO1		# coffset += 4
-	ALIGN_4
-	
-
-.L2_30:
-	testq	$2, M		
-	jz	.L2_40
-
-	ALIGN_4
-
-.L2_31:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L2_36
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_32:
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL2x2_1(xxx)
-	KERNEL2x2_2(xxx)
-	KERNEL2x2_3(xxx)
-	KERNEL2x2_4(xxx)
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL2x2_1(xxx)
-	KERNEL2x2_2(xxx)
-	KERNEL2x2_3(xxx)
-	KERNEL2x2_4(xxx)
-
-	je	.L2_36
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL2x2_1(xxx)
-	KERNEL2x2_2(xxx)
-	KERNEL2x2_3(xxx)
-	KERNEL2x2_4(xxx)
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL2x2_1(xxx)
-	KERNEL2x2_2(xxx)
-	KERNEL2x2_3(xxx)
-	KERNEL2x2_4(xxx)
-
-	je	.L2_36
-
-	jmp	.L2_32
-	ALIGN_4
-
-.L2_36:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_39
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-	
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_37:
-
-	KERNEL2x2_SUB(xxx)
-	addq	$2, BI
-	addq	$2, %rax
-	jl	.L2_37
-	ALIGN_4
-
-
-.L2_39:
-
-	vmovddup	ALPHA, %xmm0
-
-	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddpd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm5 ,  	(CO1, LDC)
-
-	addq	$2 * SIZE, CO1		# coffset += 2
-	ALIGN_4
-	
-
-.L2_40:
-	testq	$1, M		
-	jz	.L2_60		// to next 2 lines of N
-
-	ALIGN_4
-
-.L2_41:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L2_46
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_42:
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL1x2_1(xxx)
-	KERNEL1x2_2(xxx)
-	KERNEL1x2_3(xxx)
-	KERNEL1x2_4(xxx)
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL1x2_1(xxx)
-	KERNEL1x2_2(xxx)
-	KERNEL1x2_3(xxx)
-	KERNEL1x2_4(xxx)
-
-	je	.L2_46
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL1x2_1(xxx)
-	KERNEL1x2_2(xxx)
-	KERNEL1x2_3(xxx)
-	KERNEL1x2_4(xxx)
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL1x2_1(xxx)
-	KERNEL1x2_2(xxx)
-	KERNEL1x2_3(xxx)
-	KERNEL1x2_4(xxx)
-
-	je	.L2_46
-
-	jmp	.L2_42
-	ALIGN_4
-
-.L2_46:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_49
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_47:
-
-	KERNEL1x2_SUB(xxx)
-	addq	$2, BI
-	addq	$1, %rax
-	jl	.L2_47
-	ALIGN_4
-
-
-.L2_49:
-
-	vmovddup	ALPHA, %xmm0
-
-	vfmaddsd 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddsd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-
-	vmovsd	%xmm4 ,  	(CO1)
-	vmovsd	%xmm5 ,  	(CO1, LDC)
-
-
-	addq	$1 * SIZE, CO1		# coffset += 1
-	ALIGN_4
-	
-
-	
-.L2_60:
-
-	decq	J			// j --
-	jg	.L2_01			// next 2 lines of N
-
-
-
-.L1_0:
-
-/************************************************************************************************
-* Loop for Nmod6 % 2 > 0
-*************************************************************************************************/
-
-	movq	Nmod6, J		
-	andq	$1, J			// j % 2
-	je	.L999
-	ALIGN_4
-
-.L1_01:
-	// copy to sub buffer
-	movq	B, BO1
-	leaq    BUFFER1, BO		// first buffer to BO
-	movq	K, %rax
-	ALIGN_4
-
-.L1_02b:
-
-	vmovsd	(BO1), %xmm0
-	vmovsd	%xmm0,       (BO)
-	addq	$1*SIZE,BO1
-	addq	$1*SIZE,BO
-	decq	%rax
-	jnz	.L1_02b
-
-.L1_02c:
-
-	movq	BO1, B			// next offset of B
-
-.L1_10:
-	movq	C, CO1
-	leaq	(C, LDC, 1), C		// c += 1 * ldc
-
-	movq	A, AO		 	// aoffset = a
-	addq	$16 * SIZE, AO
-
-	movq	M,  I
-	sarq	$3, I			// i = (m >> 3)
-	je	.L1_20
-
-	ALIGN_4
-
-.L1_11:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax			//  K = K - ( K % 8 )
-	je	.L1_16
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_12:
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL8x1_1(xxx)
-	KERNEL8x1_2(xxx)
-	KERNEL8x1_3(xxx)
-	KERNEL8x1_4(xxx)
-
-	KERNEL8x1_1(xxx)
-	KERNEL8x1_2(xxx)
-	KERNEL8x1_3(xxx)
-	KERNEL8x1_4(xxx)
-
-	je	.L1_16
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL8x1_1(xxx)
-	KERNEL8x1_2(xxx)
-	KERNEL8x1_3(xxx)
-	KERNEL8x1_4(xxx)
-
-	KERNEL8x1_1(xxx)
-	KERNEL8x1_2(xxx)
-	KERNEL8x1_3(xxx)
-	KERNEL8x1_4(xxx)
-
-	je	.L1_16
-
-	jmp	.L1_12
-	ALIGN_4
-
-.L1_16:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_19
-
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_17:
-
-	KERNEL8x1_SUB(xxx)
-	addq	$1, BI
-	addq	$8, %rax
-	jl	.L1_17
-	ALIGN_4
-
-
-.L1_19:
-
-	vmovddup	ALPHA, %xmm0
-
-	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7
-	vfmaddpd 4 * SIZE(CO1),%xmm0, %xmm10,%xmm10
-	vfmaddpd 6 * SIZE(CO1),%xmm0, %xmm13,%xmm13
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm7 , 2 * SIZE(CO1)
-	vmovups	%xmm10, 4 * SIZE(CO1)
-	vmovups	%xmm13, 6 * SIZE(CO1)
-
-	addq	$8 * SIZE, CO1		# coffset += 8
-	decq	I			# i --
-	jg	.L1_11
-	ALIGN_4	
-
-/**************************************************************************
-* Rest of M 
-***************************************************************************/
-.L1_20:
-	// Test rest of M
-
-	testq	$7, M
-	jz	.L999
-
-	testq	$4, M		
-	jz	.L1_30
-
-	ALIGN_4
-
-.L1_21:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L1_26
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_22:
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL4x1_1(xxx)
-	KERNEL4x1_2(xxx)
-	KERNEL4x1_3(xxx)
-	KERNEL4x1_4(xxx)
-
-	KERNEL4x1_1(xxx)
-	KERNEL4x1_2(xxx)
-	KERNEL4x1_3(xxx)
-	KERNEL4x1_4(xxx)
-
-	je	.L1_26
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL4x1_1(xxx)
-	KERNEL4x1_2(xxx)
-	KERNEL4x1_3(xxx)
-	KERNEL4x1_4(xxx)
-
-	KERNEL4x1_1(xxx)
-	KERNEL4x1_2(xxx)
-	KERNEL4x1_3(xxx)
-	KERNEL4x1_4(xxx)
-
-	je	.L1_26
-
-	jmp	.L1_22
-	ALIGN_4
-
-.L1_26:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_29
-
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_27:
-
-	KERNEL4x1_SUB(xxx)
-	addq	$1, BI
-	addq	$4, %rax
-	jl	.L1_27
-	ALIGN_4
-
-
-.L1_29:
-
-	vmovddup	ALPHA, %xmm0
-
-	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm7 , 2 * SIZE(CO1)
-
-	addq	$4 * SIZE, CO1		# coffset += 4
-	ALIGN_4
-	
-
-.L1_30:
-	testq	$2, M		
-	jz	.L1_40
-
-	ALIGN_4
-
-.L1_31:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L1_36
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_32:
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL2x1_1(xxx)
-	KERNEL2x1_2(xxx)
-	KERNEL2x1_3(xxx)
-	KERNEL2x1_4(xxx)
-
-	KERNEL2x1_1(xxx)
-	KERNEL2x1_2(xxx)
-	KERNEL2x1_3(xxx)
-	KERNEL2x1_4(xxx)
-
-	je	.L1_36
-
-	KERNEL2x1_1(xxx)
-	KERNEL2x1_2(xxx)
-	KERNEL2x1_3(xxx)
-	KERNEL2x1_4(xxx)
-
-	KERNEL2x1_1(xxx)
-	KERNEL2x1_2(xxx)
-	KERNEL2x1_3(xxx)
-	KERNEL2x1_4(xxx)
-
-	je	.L1_36
-
-	jmp	.L1_32
-	ALIGN_4
-
-.L1_36:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_39
-
-	movq    %rax, BI                        //  Index for BO
-	
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_37:
-
-	KERNEL2x1_SUB(xxx)
-	addq	$1, BI
-	addq	$2, %rax
-	jl	.L1_37
-	ALIGN_4
-
-
-.L1_39:
-
-	vmovddup	ALPHA, %xmm0
-
-	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
-
-	vmovups	%xmm4 ,  	(CO1)
-
-	addq	$2 * SIZE, CO1		# coffset += 2
-	ALIGN_4
-	
-
-.L1_40:
-	testq	$1, M		
-	jz	.L999
-
-	ALIGN_4
-
-.L1_41:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L1_46
-	movq    %rax, BI                        //  Index for BO
-
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_42:
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL1x1_1(xxx)
-	KERNEL1x1_2(xxx)
-	KERNEL1x1_3(xxx)
-	KERNEL1x1_4(xxx)
-
-	KERNEL1x1_1(xxx)
-	KERNEL1x1_2(xxx)
-	KERNEL1x1_3(xxx)
-	KERNEL1x1_4(xxx)
-
-	je	.L1_46
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL1x1_1(xxx)
-	KERNEL1x1_2(xxx)
-	KERNEL1x1_3(xxx)
-	KERNEL1x1_4(xxx)
-
-	KERNEL1x1_1(xxx)
-	KERNEL1x1_2(xxx)
-	KERNEL1x1_3(xxx)
-	KERNEL1x1_4(xxx)
-
-	je	.L1_46
-
-	jmp	.L1_42
-	ALIGN_4
-
-.L1_46:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_49
-
-	movq    %rax, BI                        //  Index for BO
-
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_47:
-
-	KERNEL1x1_SUB(xxx)
-	addq	$1, BI
-	addq	$1, %rax
-	jl	.L1_47
-	ALIGN_4
-
-
-.L1_49:
-
-	vmovddup	ALPHA, %xmm0
-
-	vfmaddsd 	 (CO1),%xmm0, %xmm4,%xmm4
-
-	vmovsd	%xmm4 ,  	(CO1)
-
-	addq	$1 * SIZE, CO1		# coffset += 1
-	ALIGN_4
-	
-
-.L999:
-	movq   		SP, %rsp
-	movq	   (%rsp), %rbx
-	movq	  8(%rsp), %rbp
-	movq	 16(%rsp), %r12
-	movq	 24(%rsp), %r13
-	movq	 32(%rsp), %r14
-	movq	 40(%rsp), %r15
-
-#ifdef WINDOWS_ABI
-	movq	 48(%rsp), %rdi
-	movq	 56(%rsp), %rsi
-	movups	 64(%rsp), %xmm6
-	movups	 80(%rsp), %xmm7
-	movups	 96(%rsp), %xmm8
-	movups	112(%rsp), %xmm9
-	movups	128(%rsp), %xmm10
-	movups	144(%rsp), %xmm11
-	movups	160(%rsp), %xmm12
-	movups	176(%rsp), %xmm13
-	movups	192(%rsp), %xmm14
-	movups	208(%rsp), %xmm15
-#endif
-
-	addq	$STACKSIZE, %rsp
-	ret
-
-	EPILOGUE
-
-
-#else
-/*************************************************************************************
-* TRMM Kernel
-*************************************************************************************/
-	PROLOGUE
-	PROFCODE
-	
-	subq	$STACKSIZE, %rsp
-	movq	%rbx,   (%rsp)
-	movq	%rbp,  8(%rsp)
-	movq	%r12, 16(%rsp)
-	movq	%r13, 24(%rsp)
-	movq	%r14, 32(%rsp)
-	movq	%r15, 40(%rsp)
-
-	vzeroupper
-
-#ifdef WINDOWS_ABI
-	movq	%rdi,    48(%rsp)
-	movq	%rsi,    56(%rsp)
-	movups	%xmm6,   64(%rsp)
-	movups	%xmm7,   80(%rsp)
-	movups	%xmm8,   96(%rsp)
-	movups	%xmm9,  112(%rsp)
-	movups	%xmm10, 128(%rsp)
-	movups	%xmm11, 144(%rsp)
-	movups	%xmm12, 160(%rsp)
-	movups	%xmm13, 176(%rsp)
-	movups	%xmm14, 192(%rsp)
-	movups	%xmm15, 208(%rsp)
-
-	movq	ARG1,      OLD_M
-	movq	ARG2,      OLD_N
-	movq	ARG3,      OLD_K
-	movq	OLD_A,     A
-	movq	OLD_B,     B
-	movq	OLD_C,     C
-	movq	OLD_LDC,   LDC
-#ifdef TRMMKERNEL
-	movsd	OLD_OFFSET, %xmm12
-#endif
-	vmovaps	%xmm3, %xmm0
-
-#else
-	movq	STACKSIZE +  8(%rsp), LDC
-#ifdef TRMMKERNEL
-	movsd	STACKSIZE + 16(%rsp), %xmm12
-#endif
-
-#endif
-
-	movq    %rsp, SP      # save old stack
-        subq    $128 + L_BUFFER_SIZE, %rsp
-        andq    $-4096, %rsp    # align stack
-
-        STACK_TOUCH
-
-	cmpq	$0, OLD_M
-	je	.L999
-
-	cmpq	$0, OLD_N
-	je	.L999
-
-	cmpq	$0, OLD_K
-	je	.L999
-
-	movq	OLD_M, M
-	movq	OLD_N, N
-	movq	OLD_K, K
-
-	vmovsd	 %xmm0, ALPHA
-
-	salq	$BASE_SHIFT, LDC
-
-	movq    N, %rax
-        xorq    %rdx, %rdx
-        movq    $2,  %rdi
-        divq    %rdi                    //    N / 2
-        movq    %rax, Ndiv6             //    N / 2
-        movq    %rdx, Nmod6             //    N % 2
-
-	
-
-#ifdef TRMMKERNEL
-	vmovsd	%xmm12, OFFSET
-	vmovsd	%xmm12, KK
-#ifndef LEFT
-	negq	KK
-#endif	
-#endif
-
-	movq	Ndiv6,  J
-	cmpq	$0, J
-	je	.L1_0
-	ALIGN_4
-
-.L2_0:
-
-.L2_01:
-	// copy to sub buffer
-	movq	B, BO1
-	leaq    BUFFER1, BO		// first buffer to BO
-	movq	K, %rax
-	ALIGN_4
-
-.L2_02b:
-
-	vmovups	(BO1), %xmm0
-	vmovups	%xmm0,       (BO)
-	addq	$2*SIZE,BO1
-	addq	$2*SIZE,BO
-	decq	%rax
-	jnz	.L2_02b
-
-.L2_02c:
-
-	movq	BO1, B			// next offset of B
-
-.L2_10:
-	movq	C, CO1
-	leaq	(C, LDC, 2), C		// c += 2 * ldc
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        movq    OFFSET, %rax
-        movq    %rax, KK
-#endif
-	
-	movq	A, AO		 	// aoffset = a
-	addq	$16 * SIZE, AO
-
-	movq	M,  I
-	sarq	$3, I			// i = (m >> 3)
-	je	.L2_20
-
-	ALIGN_4
-
-.L2_11:
-
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-	leaq	BUFFER1, BO		// first buffer to BO
-	addq	$4 * SIZE, BO
-#else
-        movq    KK, %rax
-	leaq	BUFFER1, BO		// first buffer to BO
-	addq	$4 * SIZE, BO
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, 8), BO
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-        leaq    (AO, %rax, 8), AO
-#endif
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $8, %rax        // number of values in AO
-#else
-        addq    $2, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-
-	andq	$-8, %rax			//  K = K - ( K % 8 )
-	je	.L2_16
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_12:
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL8x2_1(xxx)
-	KERNEL8x2_2(xxx)
-	KERNEL8x2_3(xxx)
-	KERNEL8x2_4(xxx)
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL8x2_1(xxx)
-	KERNEL8x2_2(xxx)
-	KERNEL8x2_3(xxx)
-	KERNEL8x2_4(xxx)
-
-	je	.L2_16
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL8x2_1(xxx)
-	KERNEL8x2_2(xxx)
-	KERNEL8x2_3(xxx)
-	KERNEL8x2_4(xxx)
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL8x2_1(xxx)
-	KERNEL8x2_2(xxx)
-	KERNEL8x2_3(xxx)
-	KERNEL8x2_4(xxx)
-
-	je	.L2_16
-
-	jmp	.L2_12
-	ALIGN_4
-
-.L2_16:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_19
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_17:
-
-	KERNEL8x2_SUB(xxx)
-	addq	$2, BI
-	addq	$8, %rax
-	jl	.L2_17
-	ALIGN_4
-
-
-.L2_19:
-
-	vmovddup	ALPHA, %xmm0
-
-#ifndef TRMMKERNEL
-
-	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7
-	vfmaddpd 4 * SIZE(CO1),%xmm0, %xmm10,%xmm10
-	vfmaddpd 6 * SIZE(CO1),%xmm0, %xmm13,%xmm13
-
-	vfmaddpd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-	vfmaddpd 2 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
-	vfmaddpd 4 * SIZE(CO1, LDC),%xmm0, %xmm11,%xmm11
-	vfmaddpd 6 * SIZE(CO1, LDC),%xmm0, %xmm14,%xmm14
-
-
-#else
-	vmulpd	%xmm0, %xmm4,%xmm4
-	vmulpd	%xmm0, %xmm7,%xmm7
-	vmulpd	%xmm0, %xmm10,%xmm10
-	vmulpd	%xmm0, %xmm13,%xmm13
-
-	vmulpd	%xmm0, %xmm5,%xmm5
-	vmulpd	%xmm0, %xmm8,%xmm8
-	vmulpd	%xmm0, %xmm11,%xmm11
-	vmulpd	%xmm0, %xmm14,%xmm14
-
-#endif
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm7 , 2 * SIZE(CO1)
-	vmovups	%xmm10, 4 * SIZE(CO1)
-	vmovups	%xmm13, 6 * SIZE(CO1)
-
-	vmovups	%xmm5 ,  	(CO1, LDC)
-	vmovups	%xmm8 , 2 * SIZE(CO1, LDC)
-	vmovups	%xmm11, 4 * SIZE(CO1, LDC)
-	vmovups	%xmm14, 6 * SIZE(CO1, LDC)
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, 8), BO
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-        leaq    (AO, %rax, 8), AO
-#endif
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $8, KK
-#endif
-
-	addq	$8 * SIZE, CO1		# coffset += 8
-	decq	I			# i --
-	jg	.L2_11
-	ALIGN_4	
-
-/**************************************************************************
-* Rest of M 
-***************************************************************************/
-.L2_20:
-	// Test rest of M
-
-	testq	$7, M
-	jz	.L2_60		// to next 2 lines of N
-
-	testq	$4, M		
-	jz	.L2_30
-
-	ALIGN_4
-
-.L2_21:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, 8), BO
-        salq    $2, %rax                        // rax = rax * 4 ; number of values
-        leaq    (AO, %rax, 8), AO
-#endif
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $4, %rax        // number of values in AO
-#else
-        addq    $2, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-
-	andq	$-8, %rax
-	je	.L2_26
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_22:
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL4x2_1(xxx)
-	KERNEL4x2_2(xxx)
-	KERNEL4x2_3(xxx)
-	KERNEL4x2_4(xxx)
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL4x2_1(xxx)
-	KERNEL4x2_2(xxx)
-	KERNEL4x2_3(xxx)
-	KERNEL4x2_4(xxx)
-
-	je	.L2_26
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL4x2_1(xxx)
-	KERNEL4x2_2(xxx)
-	KERNEL4x2_3(xxx)
-	KERNEL4x2_4(xxx)
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL4x2_1(xxx)
-	KERNEL4x2_2(xxx)
-	KERNEL4x2_3(xxx)
-	KERNEL4x2_4(xxx)
-
-	je	.L2_26
-
-	jmp	.L2_22
-	ALIGN_4
-
-.L2_26:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_29
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_27:
-
-	KERNEL4x2_SUB(xxx)
-	addq	$2, BI
-	addq	$4, %rax
-	jl	.L2_27
-	ALIGN_4
-
-
-.L2_29:
-
-	vmovddup	ALPHA, %xmm0
-
-#ifndef TRMMKERNEL
-
-	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7
-
-	vfmaddpd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-	vfmaddpd 2 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
-
-
-#else
-	vmulpd	%xmm0, %xmm4,%xmm4
-	vmulpd	%xmm0, %xmm7,%xmm7
-
-	vmulpd	%xmm0, %xmm5,%xmm5
-	vmulpd	%xmm0, %xmm8,%xmm8
-
-#endif
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm7 , 2 * SIZE(CO1)
-
-	vmovups	%xmm5 ,  	(CO1, LDC)
-	vmovups	%xmm8 , 2 * SIZE(CO1, LDC)
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, 8), BO
-        salq    $2, %rax                        // rax = rax * 4 ; number of values
-        leaq    (AO, %rax, 8), AO
-#endif
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $4, KK
-#endif
-
-	addq	$4 * SIZE, CO1		# coffset += 4
-	ALIGN_4
-	
-
-.L2_30:
-	testq	$2, M		
-	jz	.L2_40
-
-	ALIGN_4
-
-.L2_31:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, 8), BO
-        salq    $1, %rax                        // rax = rax * 2 ; number of values
-        leaq    (AO, %rax, 8), AO
-#endif
-
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $2, %rax        // number of values in AO
-#else
-        addq    $2, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-
-	andq	$-8, %rax
-	je	.L2_36
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_32:
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL2x2_1(xxx)
-	KERNEL2x2_2(xxx)
-	KERNEL2x2_3(xxx)
-	KERNEL2x2_4(xxx)
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL2x2_1(xxx)
-	KERNEL2x2_2(xxx)
-	KERNEL2x2_3(xxx)
-	KERNEL2x2_4(xxx)
-
-	je	.L2_36
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL2x2_1(xxx)
-	KERNEL2x2_2(xxx)
-	KERNEL2x2_3(xxx)
-	KERNEL2x2_4(xxx)
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL2x2_1(xxx)
-	KERNEL2x2_2(xxx)
-	KERNEL2x2_3(xxx)
-	KERNEL2x2_4(xxx)
-
-	je	.L2_36
-
-	jmp	.L2_32
-	ALIGN_4
-
-.L2_36:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_39
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-	
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_37:
-
-	KERNEL2x2_SUB(xxx)
-	addq	$2, BI
-	addq	$2, %rax
-	jl	.L2_37
-	ALIGN_4
-
-
-.L2_39:
-
-	vmovddup	ALPHA, %xmm0
-
-#ifndef TRMMKERNEL
-
-	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddpd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-
-#else
-	vmulpd	%xmm0, %xmm4,%xmm4
-	vmulpd	%xmm0, %xmm5,%xmm5
-
-#endif
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm5 ,  	(CO1, LDC)
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax 
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, 8), BO
-        salq    $1, %rax                        // rax = rax * 2 ; number of values
-        leaq    (AO, %rax, 8), AO
-#endif
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $2, KK
-#endif
-
-	addq	$2 * SIZE, CO1		# coffset += 2
-	ALIGN_4
-	
-
-.L2_40:
-	testq	$1, M		
-	jz	.L2_60		// to next 2 lines of N
-
-	ALIGN_4
-
-.L2_41:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, 8), BO
-        leaq    (AO, %rax, 8), AO
-#endif
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $1, %rax        // number of values in AO
-#else
-        addq    $2, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-	andq	$-8, %rax
-	je	.L2_46
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_42:
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL1x2_1(xxx)
-	KERNEL1x2_2(xxx)
-	KERNEL1x2_3(xxx)
-	KERNEL1x2_4(xxx)
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL1x2_1(xxx)
-	KERNEL1x2_2(xxx)
-	KERNEL1x2_3(xxx)
-	KERNEL1x2_4(xxx)
-
-	je	.L2_46
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL1x2_1(xxx)
-	KERNEL1x2_2(xxx)
-	KERNEL1x2_3(xxx)
-	KERNEL1x2_4(xxx)
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL1x2_1(xxx)
-	KERNEL1x2_2(xxx)
-	KERNEL1x2_3(xxx)
-	KERNEL1x2_4(xxx)
-
-	je	.L2_46
-
-	jmp	.L2_42
-	ALIGN_4
-
-.L2_46:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_49
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_47:
-
-	KERNEL1x2_SUB(xxx)
-	addq	$2, BI
-	addq	$1, %rax
-	jl	.L2_47
-	ALIGN_4
-
-
-.L2_49:
-
-	vmovddup	ALPHA, %xmm0
-
-#ifndef TRMMKERNEL
-
-	vfmaddsd 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddsd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-
-#else
-	vmulsd	%xmm0, %xmm4,%xmm4
-	vmulsd	%xmm0, %xmm5,%xmm5
-
-#endif
-
-	vmovsd	%xmm4 ,  	(CO1)
-	vmovsd	%xmm5 ,  	(CO1, LDC)
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, 8), BO
-        leaq    (AO, %rax, 8), AO
-#endif
-
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $1, KK
-#endif
-
-	addq	$1 * SIZE, CO1		# coffset += 1
-	ALIGN_4
-	
-
-	
-.L2_60:
-#if defined(TRMMKERNEL) && !defined(LEFT)
-        addq    $2, KK
-#endif
-
-	decq	J			// j --
-	jg	.L2_01			// next 2 lines of N
-
-
-
-.L1_0:
-
-/************************************************************************************************
-* Loop for Nmod6 % 2 > 0
-*************************************************************************************************/
-
-	movq	Nmod6, J		
-	andq	$1, J			// j % 2
-	je	.L999
-	ALIGN_4
-
-.L1_01:
-	// copy to sub buffer
-	movq	B, BO1
-	leaq    BUFFER1, BO		// first buffer to BO
-	movq	K, %rax
-	ALIGN_4
-
-.L1_02b:
-
-	vmovsd	(BO1), %xmm0
-	vmovsd	%xmm0,       (BO)
-	addq	$1*SIZE,BO1
-	addq	$1*SIZE,BO
-	decq	%rax
-	jnz	.L1_02b
-
-.L1_02c:
-
-	movq	BO1, B			// next offset of B
-
-.L1_10:
-	movq	C, CO1
-	leaq	(C, LDC, 1), C		// c += 1 * ldc
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        movq    OFFSET, %rax
-        movq    %rax, KK
-#endif
-
-	movq	A, AO		 	// aoffset = a
-	addq	$16 * SIZE, AO
-
-	movq	M,  I
-	sarq	$3, I			// i = (m >> 3)
-	je	.L1_20
-
-	ALIGN_4
-
-.L1_11:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BO, BI, 8), BO
-        salq    $3, %rax                        // rax = rax * 8 ; number of values
-        leaq    (AO, %rax, 8), AO
-#endif
-
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $8, %rax        // number of values in AO
-#else
-        addq    $1, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-	andq	$-8, %rax			//  K = K - ( K % 8 )
-	je	.L1_16
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_12:
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL8x1_1(xxx)
-	KERNEL8x1_2(xxx)
-	KERNEL8x1_3(xxx)
-	KERNEL8x1_4(xxx)
-
-	KERNEL8x1_1(xxx)
-	KERNEL8x1_2(xxx)
-	KERNEL8x1_3(xxx)
-	KERNEL8x1_4(xxx)
-
-	je	.L1_16
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL8x1_1(xxx)
-	KERNEL8x1_2(xxx)
-	KERNEL8x1_3(xxx)
-	KERNEL8x1_4(xxx)
-
-	KERNEL8x1_1(xxx)
-	KERNEL8x1_2(xxx)
-	KERNEL8x1_3(xxx)
-	KERNEL8x1_4(xxx)
-
-	je	.L1_16
-
-	jmp	.L1_12
-	ALIGN_4
-
-.L1_16:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_19
-
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_17:
-
-	KERNEL8x1_SUB(xxx)
-	addq	$1, BI
-	addq	$8, %rax
-	jl	.L1_17
-	ALIGN_4
-
-
-.L1_19:
-
-	vmovddup	ALPHA, %xmm0
-
-#ifndef TRMMKERNEL
-
-	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7
-	vfmaddpd 4 * SIZE(CO1),%xmm0, %xmm10,%xmm10
-	vfmaddpd 6 * SIZE(CO1),%xmm0, %xmm13,%xmm13
-
-#else
-	vmulpd	%xmm0, %xmm4,%xmm4
-	vmulpd	%xmm0, %xmm7,%xmm7
-	vmulpd	%xmm0, %xmm10,%xmm10
-	vmulpd	%xmm0, %xmm13,%xmm13
-
-#endif
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm7 , 2 * SIZE(CO1)
-	vmovups	%xmm10, 4 * SIZE(CO1)
-	vmovups	%xmm13, 6 * SIZE(CO1)
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BO, BI, 8), BO
-        salq    $3, %rax                        // rax = rax * 8 ; number of values
-        leaq    (AO, %rax, 8), AO
-#endif
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $8, KK
-#endif
-	addq	$8 * SIZE, CO1		# coffset += 8
-	decq	I			# i --
-	jg	.L1_11
-	ALIGN_4	
-
-/**************************************************************************
-* Rest of M 
-***************************************************************************/
-.L1_20:
-	// Test rest of M
-
-	testq	$7, M
-	jz	.L999
-
-	testq	$4, M		
-	jz	.L1_30
-
-	ALIGN_4
-
-.L1_21:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BO, BI, 8), BO
-        salq    $2, %rax                        // rax = rax * 4 ; number of values
-        leaq    (AO, %rax, 8), AO
-#endif
-
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $4, %rax        // number of values in AO
-#else
-        addq    $1, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-
-	andq	$-8, %rax
-	je	.L1_26
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_22:
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL4x1_1(xxx)
-	KERNEL4x1_2(xxx)
-	KERNEL4x1_3(xxx)
-	KERNEL4x1_4(xxx)
-
-	KERNEL4x1_1(xxx)
-	KERNEL4x1_2(xxx)
-	KERNEL4x1_3(xxx)
-	KERNEL4x1_4(xxx)
-
-	je	.L1_26
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL4x1_1(xxx)
-	KERNEL4x1_2(xxx)
-	KERNEL4x1_3(xxx)
-	KERNEL4x1_4(xxx)
-
-	KERNEL4x1_1(xxx)
-	KERNEL4x1_2(xxx)
-	KERNEL4x1_3(xxx)
-	KERNEL4x1_4(xxx)
-
-	je	.L1_26
-
-	jmp	.L1_22
-	ALIGN_4
-
-.L1_26:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_29
-
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_27:
-
-	KERNEL4x1_SUB(xxx)
-	addq	$1, BI
-	addq	$4, %rax
-	jl	.L1_27
-	ALIGN_4
-
-
-.L1_29:
-
-	vmovddup	ALPHA, %xmm0
-
-#ifndef TRMMKERNEL
-
-	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7
-
-#else
-	vmulpd	%xmm0, %xmm4,%xmm4
-	vmulpd	%xmm0, %xmm7,%xmm7
-
-#endif
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm7 , 2 * SIZE(CO1)
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BO, BI, 8), BO
-        salq    $2, %rax                        // rax = rax * 4 ; number of values
-        leaq    (AO, %rax, 8), AO
-#endif
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $4, KK
-#endif
-
-	addq	$4 * SIZE, CO1		# coffset += 4
-	ALIGN_4
-	
-
-.L1_30:
-	testq	$2, M		
-	jz	.L1_40
-
-	ALIGN_4
-
-.L1_31:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BO, BI, 8), BO
-        salq    $1, %rax                        // rax = rax * 2 ; number of values
-        leaq    (AO, %rax, 8), AO
-#endif
-
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $2, %rax        // number of values in AO
-#else
-        addq    $1, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-
-	andq	$-8, %rax
-	je	.L1_36
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_32:
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL2x1_1(xxx)
-	KERNEL2x1_2(xxx)
-	KERNEL2x1_3(xxx)
-	KERNEL2x1_4(xxx)
-
-	KERNEL2x1_1(xxx)
-	KERNEL2x1_2(xxx)
-	KERNEL2x1_3(xxx)
-	KERNEL2x1_4(xxx)
-
-	je	.L1_36
-
-	KERNEL2x1_1(xxx)
-	KERNEL2x1_2(xxx)
-	KERNEL2x1_3(xxx)
-	KERNEL2x1_4(xxx)
-
-	KERNEL2x1_1(xxx)
-	KERNEL2x1_2(xxx)
-	KERNEL2x1_3(xxx)
-	KERNEL2x1_4(xxx)
-
-	je	.L1_36
-
-	jmp	.L1_32
-	ALIGN_4
-
-.L1_36:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_39
-
-	movq    %rax, BI                        //  Index for BO
-	
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_37:
-
-	KERNEL2x1_SUB(xxx)
-	addq	$1, BI
-	addq	$2, %rax
-	jl	.L1_37
-	ALIGN_4
-
-
-.L1_39:
-
-	vmovddup	ALPHA, %xmm0
-
-#ifndef TRMMKERNEL
-
-	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
-
-#else
-	vmulpd	%xmm0, %xmm4,%xmm4
-
-#endif
-
-	vmovups	%xmm4 ,  	(CO1)
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BO, BI, 8), BO
-        salq    $1, %rax                        // rax = rax * 2 ; number of values
-        leaq    (AO, %rax, 8), AO
-#endif
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $2, KK
-#endif
-	addq	$2 * SIZE, CO1		# coffset += 2
-	ALIGN_4
-	
-
-.L1_40:
-	testq	$1, M		
-	jz	.L999
-
-	ALIGN_4
-
-.L1_41:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BO, BI, 8), BO
-        leaq    (AO, %rax, 8), AO
-#endif
-
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $1, %rax        // number of values in AO
-#else
-        addq    $1, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-	andq	$-8, %rax
-	je	.L1_46
-	movq    %rax, BI                        //  Index for BO
-
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_42:
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL1x1_1(xxx)
-	KERNEL1x1_2(xxx)
-	KERNEL1x1_3(xxx)
-	KERNEL1x1_4(xxx)
-
-	KERNEL1x1_1(xxx)
-	KERNEL1x1_2(xxx)
-	KERNEL1x1_3(xxx)
-	KERNEL1x1_4(xxx)
-
-	je	.L1_46
-
-	prefetcht0	B_PR1(BO,BI,8)
-	KERNEL1x1_1(xxx)
-	KERNEL1x1_2(xxx)
-	KERNEL1x1_3(xxx)
-	KERNEL1x1_4(xxx)
-
-	KERNEL1x1_1(xxx)
-	KERNEL1x1_2(xxx)
-	KERNEL1x1_3(xxx)
-	KERNEL1x1_4(xxx)
-
-	je	.L1_46
-
-	jmp	.L1_42
-	ALIGN_4
-
-.L1_46:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_49
-
-	movq    %rax, BI                        //  Index for BO
-
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_47:
-
-	KERNEL1x1_SUB(xxx)
-	addq	$1, BI
-	addq	$1, %rax
-	jl	.L1_47
-	ALIGN_4
-
-
-.L1_49:
-
-	vmovddup	ALPHA, %xmm0
-
-#ifndef TRMMKERNEL
-
-	vfmaddsd 	 (CO1),%xmm0, %xmm4,%xmm4
-
-#else
-	vmulsd	%xmm0, %xmm4,%xmm4
-
-#endif
-
-	vmovsd	%xmm4 ,  	(CO1)
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BO, BI, 8), BO
-        leaq    (AO, %rax, 8), AO
-#endif
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $1, KK
-#endif
-
-	addq	$1 * SIZE, CO1		# coffset += 1
-	ALIGN_4
-	
-
-.L999:
-	movq   		SP, %rsp
-	movq	   (%rsp), %rbx
-	movq	  8(%rsp), %rbp
-	movq	 16(%rsp), %r12
-	movq	 24(%rsp), %r13
-	movq	 32(%rsp), %r14
-	movq	 40(%rsp), %r15
-
-#ifdef WINDOWS_ABI
-	movq	 48(%rsp), %rdi
-	movq	 56(%rsp), %rsi
-	movups	 64(%rsp), %xmm6
-	movups	 80(%rsp), %xmm7
-	movups	 96(%rsp), %xmm8
-	movups	112(%rsp), %xmm9
-	movups	128(%rsp), %xmm10
-	movups	144(%rsp), %xmm11
-	movups	160(%rsp), %xmm12
-	movups	176(%rsp), %xmm13
-	movups	192(%rsp), %xmm14
-	movups	208(%rsp), %xmm15
-#endif
-
-	addq	$STACKSIZE, %rsp
-	ret
-
-	EPILOGUE
-
-
-
-#endif
+/*********************************************************************/
+/* Copyright 2009, 2010 The University of Texas at Austin.           */
+/* All rights reserved.                                              */
+/*                                                                   */
+/* Redistribution and use in source and binary forms, with or        */
+/* without modification, are permitted provided that the following   */
+/* conditions are met:                                               */
+/*                                                                   */
+/*   1. Redistributions of source code must retain the above         */
+/*      copyright notice, this list of conditions and the following  */
+/*      disclaimer.                                                  */
+/*                                                                   */
+/*   2. Redistributions in binary form must reproduce the above      */
+/*      copyright notice, this list of conditions and the following  */
+/*      disclaimer in the documentation and/or other materials       */
+/*      provided with the distribution.                              */
+/*                                                                   */
+/*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */
+/*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */
+/*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */
+/*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */
+/*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */
+/*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */
+/*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */
+/*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */
+/*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */
+/*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */
+/*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */
+/*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */
+/*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */
+/*    POSSIBILITY OF SUCH DAMAGE.                                    */
+/*                                                                   */
+/* The views and conclusions contained in the software and           */
+/* documentation are those of the authors and should not be          */
+/* interpreted as representing official policies, either expressed   */
+/* or implied, of The University of Texas at Austin.                 */
+/*********************************************************************/
+
+/*********************************************************************
+* 2013/06/02 Saar
+*
+* Parameter:
+* 	UNROLL_M	8
+*	UNROLL_N	2
+*	DGEMM_P		360
+*	DGEMM_Q		160
+*
+* Performance at m x n without prefetch of BO:
+* 
+* 5760x5760	93.4	GFLOPS with 8 threads on 4 modules (ACML: 90.8 GFLOPS)
+* 5760x5760	84.2	GFLOPS with 4 threads on 4 modules (ACML: 82.4 GFLOPS)
+* 3840x3840	50.3	GFLOPS with 2 threads on 2 modules (ACML: 49.5 GFLOPS)
+*
+* 5760x5760	56.4	GFLOPS with 4 threads on 2 modules (ACML: 58.5 GFLOPS)
+* 3840x3840	29.0	GFLOPS with 2 threads on 1 modules (ACML: 30.2 GFLOPS)
+* 3840x3840	26.1	GFLOPS with 1 threads on 1 modules (ACML: 25.9 GFLOPS)
+*
+*********************************************************************/
+
+/*********************************************************************
+* 2013/06/03 Saar
+*
+* Parameter:
+* 	UNROLL_M	8
+*	UNROLL_N	2
+*	DGEMM_P		336
+*	DGEMM_Q		168
+*	NO_WARMUP	1
+*	NO_AFFINITY	1
+*	GEMM_MULTITHREAD_THRESHOLD 4
+*
+* Performance at m x n with prefetch of BO:
+* 
+* 8064x3840	93.7	GFLOPS with 8 threads on 4 modules (ACML: 93.6 GFLOPS)
+* 6048x2880	85.1	GFLOPS with 4 threads on 4 modules (ACML: 84.2 GFLOPS)
+* 6048x2880	52.0	GFLOPS with 2 threads on 2 modules (ACML: 50.0 GFLOPS)
+*
+* 6048x2880	56.3	GFLOPS with 4 threads on 2 modules (ACML: 57.6 GFLOPS)
+* 4032x1920	29.5	GFLOPS with 2 threads on 1 modules (ACML: 30.5 GFLOPS)
+* 4032x1920	26.9	GFLOPS with 1 threads on 1 modules (ACML: 26.1 GFLOPS)
+*
+*********************************************************************/
+
+/*********************************************************************
+* 2013/06/04 Saar
+*
+* Parameter:
+* 	UNROLL_M	8
+*	UNROLL_N	2
+*	DGEMM_P		384
+*	DGEMM_Q		168
+*	NO_WARMUP	1
+*	NO_AFFINITY	1
+*	GEMM_MULTITHREAD_THRESHOLD 4
+*
+* Performance at m x n with prefetch of BO:
+* 
+* 6144x5376	94.6	GFLOPS with 8 threads on 4 modules (ACML: 90.5 GFLOPS)
+* 6144x5376	86.0	GFLOPS with 4 threads on 4 modules (ACML: 81.5 GFLOPS)
+* 4608x4032	52.0	GFLOPS with 2 threads on 2 modules (ACML: 47.5 GFLOPS)
+*
+* 6144x5376	57.3	GFLOPS with 4 threads on 2 modules (ACML: 56.5 GFLOPS)
+* 4608x4032	29.6	GFLOPS with 2 threads on 1 modules (ACML: 30.2 GFLOPS)
+* 4608x4032	26.9	GFLOPS with 1 threads on 1 modules (ACML: 25.6 GFLOPS)
+*
+*********************************************************************/
+
+
+
+#define ASSEMBLER
+#include "common.h"
+ 
+#define OLD_M	%rdi
+#define OLD_N	%rsi
+#define M	%r13
+#define J	%r14
+#define OLD_K	%rdx
+
+#define A	%rcx
+#define B	%r8
+#define C	%r9
+#define LDC	%r10
+	
+#define I	%r11
+#define AO	%rdi
+#define BO	%rsi
+#define	CO1	%r15
+#define K	%r12
+#define BI	%rbp
+#define	SP	%rbx
+
+#define BO1	%rdi
+#define BO2	%r15
+
+#ifndef WINDOWS_ABI
+
+#define STACKSIZE 96
+
+#else
+
+#define STACKSIZE 256
+
+#define OLD_A		40 + STACKSIZE(%rsp)
+#define OLD_B		48 + STACKSIZE(%rsp)
+#define OLD_C		56 + STACKSIZE(%rsp)
+#define OLD_LDC		64 + STACKSIZE(%rsp)
+#define OLD_OFFSET	72 + STACKSIZE(%rsp)
+
+#endif
+
+#define L_BUFFER_SIZE 8192
+#define LB2_OFFSET    4096
+
+#define Ndiv6	 24(%rsp)
+#define Nmod6	 32(%rsp)
+#define N	 40(%rsp)
+#define ALPHA	 48(%rsp)
+#define OFFSET	 56(%rsp)
+#define KK	 64(%rsp)
+#define KKK	 72(%rsp)
+#define BUFFER1	           128(%rsp)
+#define BUFFER2	LB2_OFFSET+128(%rsp)
+
+#if defined(OS_WINDOWS)
+#if   L_BUFFER_SIZE > 16384
+#define STACK_TOUCH \
+        movl    $0,  4096 * 4(%rsp);\
+        movl    $0,  4096 * 3(%rsp);\
+        movl    $0,  4096 * 2(%rsp);\
+        movl    $0,  4096 * 1(%rsp);
+#elif L_BUFFER_SIZE > 12288
+#define STACK_TOUCH \
+        movl    $0,  4096 * 3(%rsp);\
+        movl    $0,  4096 * 2(%rsp);\
+        movl    $0,  4096 * 1(%rsp);
+#elif L_BUFFER_SIZE > 8192
+#define STACK_TOUCH \
+        movl    $0,  4096 * 2(%rsp);\
+        movl    $0,  4096 * 1(%rsp);
+#elif L_BUFFER_SIZE > 4096
+#define STACK_TOUCH \
+        movl    $0,  4096 * 1(%rsp);
+#else
+#define STACK_TOUCH
+#endif
+#else
+#define STACK_TOUCH
+#endif
+
+
+
+#define	A_PR1	384
+#define	B_PR1	192
+
+#define KERNEL8x3_1(xx) \
+	prefetcht0	A_PR1(AO,%rax,8)	;\
+	vmovddup	 -6 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovddup	 -5 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovddup	 -4 * SIZE(BO, BI, 8), %xmm3 ;\
+	vfmaddpd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+	vmovups 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	vfmaddpd  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
+	vmovups 	-12 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
+	vfmaddpd  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
+	vfmaddpd  	%xmm12,%xmm3,%xmm0,%xmm12 ;\
+	vmovups 	-10 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
+	vfmaddpd  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
+	vfmaddpd  	%xmm15,%xmm3,%xmm0,%xmm15 ;\
+
+#define KERNEL8x3_2(xx) \
+	prefetcht0	A_PR1+64(AO,%rax,8)	;\
+	vmovddup	 -3 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	 -8 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovddup	 -2 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovddup	 -1 * SIZE(BO, BI, 8), %xmm3 ;\
+	vfmaddpd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+	vmovups 	 -6 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	vfmaddpd  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
+	vmovups 	 -4 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
+	vfmaddpd  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
+	vfmaddpd  	%xmm12,%xmm3,%xmm0,%xmm12 ;\
+	vmovups 	 -2 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
+	vfmaddpd  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
+	vfmaddpd  	%xmm15,%xmm3,%xmm0,%xmm15 ;\
+
+#define KERNEL8x3_3(xx) \
+	prefetcht0	A_PR1+128(AO,%rax,8)	;\
+	vmovddup	  0 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	  0 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovddup	  1 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovddup	  2 * SIZE(BO, BI, 8), %xmm3 ;\
+	vfmaddpd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+	vmovups 	  2 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	vfmaddpd  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
+	vmovups 	  4 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
+	vfmaddpd  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
+	vfmaddpd  	%xmm12,%xmm3,%xmm0,%xmm12 ;\
+	vmovups 	  6 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
+	vfmaddpd  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
+	vfmaddpd  	%xmm15,%xmm3,%xmm0,%xmm15 ;\
+
+#define KERNEL8x3_4(xx) \
+	prefetcht0	A_PR1+192(AO,%rax,8)	;\
+	vmovddup	  3 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	  8 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovddup	  4 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovddup	  5 * SIZE(BO, BI, 8), %xmm3 ;\
+	vfmaddpd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+	vmovups 	 10 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	vfmaddpd  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
+	vmovups 	 12 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
+	vfmaddpd  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
+	vfmaddpd  	%xmm12,%xmm3,%xmm0,%xmm12 ;\
+	vmovups 	 14 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
+	vfmaddpd  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
+	vfmaddpd  	%xmm15,%xmm3,%xmm0,%xmm15 ;\
+	addq	$12, BI				  ;\
+	addq	$32, %rax 			  ;\
+
+#define KERNEL8x3_SUB(xx) \
+	vmovddup	 -6 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovddup	 -5 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovddup	 -4 * SIZE(BO, BI, 8), %xmm3 ;\
+	vfmaddpd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+	vmovups 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	vfmaddpd  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
+	vmovups 	-12 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
+	vfmaddpd  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
+	vfmaddpd  	%xmm12,%xmm3,%xmm0,%xmm12 ;\
+	vmovups 	-10 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
+	vfmaddpd  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
+	vfmaddpd  	%xmm15,%xmm3,%xmm0,%xmm15 ;\
+
+
+/*******************************************************************************************/
+
+#define KERNEL4x3_1(xx) \
+	vmovddup	 -6 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovddup	 -5 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovddup	 -4 * SIZE(BO, BI, 8), %xmm3 ;\
+	vfmaddpd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+	vmovups 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	vfmaddpd  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
+
+#define KERNEL4x3_2(xx) \
+	vmovddup	 -3 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	-12 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovddup	 -2 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovddup	 -1 * SIZE(BO, BI, 8), %xmm3 ;\
+	vfmaddpd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+	vmovups 	-10 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	vfmaddpd  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
+
+#define KERNEL4x3_3(xx) \
+	vmovddup	  0 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	 -8 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovddup	  1 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovddup	  2 * SIZE(BO, BI, 8), %xmm3 ;\
+	vfmaddpd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+	vmovups 	 -6 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	vfmaddpd  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
+
+#define KERNEL4x3_4(xx) \
+	vmovddup	  3 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	 -4 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovddup	  4 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovddup	  5 * SIZE(BO, BI, 8), %xmm3 ;\
+	vfmaddpd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+	vmovups 	 -2 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	vfmaddpd  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
+	addq	$12, BI				  ;\
+	addq	$16, %rax 			  ;\
+
+#define KERNEL4x3_SUB(xx) \
+	vmovddup	 -6 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovddup	 -5 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovddup	 -4 * SIZE(BO, BI, 8), %xmm3 ;\
+	vfmaddpd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+	vmovups 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	vfmaddpd  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
+
+
+
+
+
+/*******************************************************************************************/
+
+#define KERNEL2x3_1(xx) \
+	vmovddup	 -6 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovddup	 -5 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovddup	 -4 * SIZE(BO, BI, 8), %xmm3 ;\
+	vfmaddpd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+
+#define KERNEL2x3_2(xx) \
+	vmovddup	 -3 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovddup	 -2 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovddup	 -1 * SIZE(BO, BI, 8), %xmm3 ;\
+	vfmaddpd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+
+#define KERNEL2x3_3(xx) \
+	vmovddup	  0 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	-12 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovddup	  1 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovddup	  2 * SIZE(BO, BI, 8), %xmm3 ;\
+	vfmaddpd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+
+#define KERNEL2x3_4(xx) \
+	vmovddup	  3 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	-10 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovddup	  4 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovddup	  5 * SIZE(BO, BI, 8), %xmm3 ;\
+	vfmaddpd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+	addq	$12, BI				  ;\
+	addq	$8, %rax 			  ;\
+
+#define KERNEL2x3_SUB(xx) \
+	vmovddup	 -6 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovddup	 -5 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovddup	 -4 * SIZE(BO, BI, 8), %xmm3 ;\
+	vfmaddpd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+
+/*******************************************************************************************/
+
+#define KERNEL1x3_1(xx) \
+	vmovsd	 -6 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovsd 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovsd	 -5 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddsd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovsd	 -4 * SIZE(BO, BI, 8), %xmm3 ;\
+	vfmaddsd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+
+#define KERNEL1x3_2(xx) \
+	vmovsd	 -3 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovsd 	-15 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovsd	 -2 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddsd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovsd	 -1 * SIZE(BO, BI, 8), %xmm3 ;\
+	vfmaddsd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+
+#define KERNEL1x3_3(xx) \
+	vmovsd	  0 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovsd 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovsd	  1 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddsd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovsd	  2 * SIZE(BO, BI, 8), %xmm3 ;\
+	vfmaddsd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+
+#define KERNEL1x3_4(xx) \
+	vmovsd	  3 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovsd 	-13 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovsd	  4 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddsd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovsd	  5 * SIZE(BO, BI, 8), %xmm3 ;\
+	vfmaddsd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+	addq	$12, BI				  ;\
+	addq	$4, %rax 			  ;\
+
+#define KERNEL1x3_SUB(xx) \
+	vmovsd	 -6 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovsd 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovsd	 -5 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddsd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovsd	 -4 * SIZE(BO, BI, 8), %xmm3 ;\
+	vfmaddsd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+
+
+
+/*******************************************************************************************
+* 2 lines of N
+*******************************************************************************************/
+
+#define KERNEL8x2_1(xx) \
+	prefetcht0	A_PR1(AO,%rax,8)	;\
+	vmovddup	 -4 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovddup	 -3 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovups 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	vmovups 	-12 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
+	vfmaddpd  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
+	vmovups 	-10 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
+	vfmaddpd  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
+
+#define KERNEL8x2_2(xx) \
+	prefetcht0	A_PR1+64(AO,%rax,8)	;\
+	vmovddup	 -2 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	 -8 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovddup	 -1 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovups 	 -6 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	vmovups 	 -4 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
+	vfmaddpd  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
+	vmovups 	 -2 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
+	vfmaddpd  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
+
+#define KERNEL8x2_3(xx) \
+	prefetcht0	A_PR1+128(AO,%rax,8)	;\
+	vmovddup	  0 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	  0 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovddup	  1 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovups 	  2 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	vmovups 	  4 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
+	vfmaddpd  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
+	vmovups 	  6 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
+	vfmaddpd  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
+
+#define KERNEL8x2_4(xx) \
+	prefetcht0	A_PR1+192(AO,%rax,8)	;\
+	vmovddup	  2 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	  8 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovddup	  3 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovups 	 10 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	vmovups 	 12 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
+	vfmaddpd  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
+	vmovups 	 14 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
+	vfmaddpd  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
+	addq	$8, BI				  ;\
+	addq	$32, %rax 			  ;\
+
+#define KERNEL8x2_SUB(xx) \
+	vmovddup	 -4 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovddup	 -3 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovups 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	vmovups 	-12 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
+	vfmaddpd  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
+	vmovups 	-10 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
+	vfmaddpd  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
+
+
+/*******************************************************************************************/
+
+#define KERNEL4x2_1(xx) \
+	vmovddup	 -4 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovddup	 -3 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovups 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+
+#define KERNEL4x2_2(xx) \
+	vmovddup	 -2 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	-12 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovddup	 -1 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovups 	-10 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+
+#define KERNEL4x2_3(xx) \
+	vmovddup	  0 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	 -8 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovddup	  1 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovups 	 -6 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+
+#define KERNEL4x2_4(xx) \
+	vmovddup	  2 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	 -4 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovddup	  3 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovups 	 -2 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	addq	$8, BI				  ;\
+	addq	$16, %rax 			  ;\
+
+#define KERNEL4x2_SUB(xx) \
+	vmovddup	 -4 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovddup	 -3 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovups 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+
+
+/*******************************************************************************************/
+
+#define KERNEL2x2_1(xx) \
+	vmovddup	 -4 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovddup	 -3 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+
+#define KERNEL2x2_2(xx) \
+	vmovddup	 -2 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovddup	 -1 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+
+#define KERNEL2x2_3(xx) \
+	vmovddup	  0 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	-12 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovddup	  1 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+
+#define KERNEL2x2_4(xx) \
+	vmovddup	  2 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	-10 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovddup	  3 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	addq	$8, BI				  ;\
+	addq	$8, %rax 			  ;\
+
+#define KERNEL2x2_SUB(xx) \
+	vmovddup	 -4 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovddup	 -3 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+
+/*******************************************************************************************/
+
+#define KERNEL1x2_1(xx) \
+	vmovsd	 -4 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovsd 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovsd	 -3 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddsd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+
+#define KERNEL1x2_2(xx) \
+	vmovsd	 -2 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovsd 	-15 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovsd	 -1 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddsd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+
+#define KERNEL1x2_3(xx) \
+	vmovsd	  0 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovsd 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovsd	  1 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddsd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+
+#define KERNEL1x2_4(xx) \
+	vmovsd	  2 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovsd 	-13 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovsd	  3 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddsd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	addq	$8, BI				  ;\
+	addq	$4, %rax 			  ;\
+
+#define KERNEL1x2_SUB(xx) \
+	vmovsd	 -4 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovsd 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovsd	 -3 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddsd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+
+
+
+/*******************************************************************************************
+* 1 line of N
+*******************************************************************************************/
+
+#define KERNEL8x1_1(xx) \
+	prefetcht0	A_PR1(AO,%rax,8)	;\
+	vmovddup	 -2 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovups 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vmovups 	-12 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
+	vmovups 	-10 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
+
+#define KERNEL8x1_2(xx) \
+	prefetcht0	A_PR1+64(AO,%rax,8)	;\
+	vmovddup	 -1 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	 -8 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovups 	 -6 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vmovups 	 -4 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
+	vmovups 	 -2 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
+
+#define KERNEL8x1_3(xx) \
+	prefetcht0	A_PR1+128(AO,%rax,8)	;\
+	vmovddup	  0 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	  0 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovups 	  2 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vmovups 	  4 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
+	vmovups 	  6 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
+
+#define KERNEL8x1_4(xx) \
+	prefetcht0	A_PR1+192(AO,%rax,8)	;\
+	vmovddup	  1 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	  8 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovups 	 10 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vmovups 	 12 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
+	vmovups 	 14 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
+	addq	$4, BI				  ;\
+	addq	$32, %rax 			  ;\
+
+#define KERNEL8x1_SUB(xx) \
+	vmovddup	 -2 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovups 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vmovups 	-12 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
+	vmovups 	-10 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
+
+
+/*******************************************************************************************/
+
+#define KERNEL4x1_1(xx) \
+	vmovddup	 -2 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovups 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+
+#define KERNEL4x1_2(xx) \
+	vmovddup	 -1 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	-12 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovups 	-10 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+
+#define KERNEL4x1_3(xx) \
+	vmovddup	  0 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	 -8 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovups 	 -6 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+
+#define KERNEL4x1_4(xx) \
+	vmovddup	  1 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	 -4 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovups 	 -2 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	addq	$4, BI				  ;\
+	addq	$16, %rax 			  ;\
+
+#define KERNEL4x1_SUB(xx) \
+	vmovddup	 -2 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovups 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+
+
+/*******************************************************************************************/
+
+#define KERNEL2x1_1(xx) \
+	vmovddup	 -2 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+
+#define KERNEL2x1_2(xx) \
+	vmovddup	 -1 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+
+#define KERNEL2x1_3(xx) \
+	vmovddup	  0 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	-12 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+
+#define KERNEL2x1_4(xx) \
+	vmovddup	  1 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	-10 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	addq	$4, BI				  ;\
+	addq	$8, %rax 			  ;\
+
+#define KERNEL2x1_SUB(xx) \
+	vmovddup	 -2 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+
+/*******************************************************************************************/
+
+#define KERNEL1x1_1(xx) \
+	vmovsd	 -2 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovsd 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+
+#define KERNEL1x1_2(xx) \
+	vmovsd	 -1 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovsd 	-15 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+
+#define KERNEL1x1_3(xx) \
+	vmovsd	  0 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovsd 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+
+#define KERNEL1x1_4(xx) \
+	vmovsd	  1 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovsd 	-13 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	addq	$4, BI				  ;\
+	addq	$4, %rax 			  ;\
+
+#define KERNEL1x1_SUB(xx) \
+	vmovsd	 -2 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovsd 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+
+
+/*******************************************************************************************/
+
+#if !defined(TRMMKERNEL)
+
+
+	PROLOGUE
+	PROFCODE
+	
+	subq	$STACKSIZE, %rsp
+	movq	%rbx,   (%rsp)
+	movq	%rbp,  8(%rsp)
+	movq	%r12, 16(%rsp)
+	movq	%r13, 24(%rsp)
+	movq	%r14, 32(%rsp)
+	movq	%r15, 40(%rsp)
+
+	vzeroupper
+
+#ifdef WINDOWS_ABI
+	movq	%rdi,    48(%rsp)
+	movq	%rsi,    56(%rsp)
+	movups	%xmm6,   64(%rsp)
+	movups	%xmm7,   80(%rsp)
+	movups	%xmm8,   96(%rsp)
+	movups	%xmm9,  112(%rsp)
+	movups	%xmm10, 128(%rsp)
+	movups	%xmm11, 144(%rsp)
+	movups	%xmm12, 160(%rsp)
+	movups	%xmm13, 176(%rsp)
+	movups	%xmm14, 192(%rsp)
+	movups	%xmm15, 208(%rsp)
+
+	movq	ARG1,      OLD_M
+	movq	ARG2,      OLD_N
+	movq	ARG3,      OLD_K
+	movq	OLD_A,     A
+	movq	OLD_B,     B
+	movq	OLD_C,     C
+	movq	OLD_LDC,   LDC
+
+	vmovaps	%xmm3, %xmm0
+
+#else
+	movq	STACKSIZE +  8(%rsp), LDC
+
+#endif
+
+	movq    %rsp, SP      # save old stack
+        subq    $128 + L_BUFFER_SIZE, %rsp
+        andq    $-4096, %rsp    # align stack
+
+        STACK_TOUCH
+
+	cmpq	$0, OLD_M
+	je	.L999
+
+	cmpq	$0, OLD_N
+	je	.L999
+
+	cmpq	$0, OLD_K
+	je	.L999
+
+	movq	OLD_M, M
+	movq	OLD_N, N
+	movq	OLD_K, K
+
+	vmovsd	 %xmm0, ALPHA
+
+	salq	$BASE_SHIFT, LDC
+
+	movq    N, %rax
+        xorq    %rdx, %rdx
+        movq    $6,  %rdi
+        divq    %rdi                    //    N / 6
+        movq    %rax, Ndiv6             //    N / 6
+        movq    %rdx, Nmod6             //    N % 6
+
+	
+
+
+	movq	Ndiv6,  J
+	cmpq	$0, J
+	je	.L2_0
+	ALIGN_4
+
+.L6_01:
+        // copy to sub buffer
+        movq    K, %rax
+        salq    $1,%rax                 // K * 2
+        movq    B, BO1
+        leaq    (B,%rax,8), BO2         // next offset to BO2
+        leaq    BUFFER1, BO             // first buffer to BO
+        movq    K, %rax
+        sarq    $2, %rax                // K / 4
+        jz      .L6_02a
+        ALIGN_4
+
+.L6_02:
+	prefetcht0 512(BO1)
+	prefetcht0 512(BO2)
+	prefetchw  512(BO)
+	vmovups	      (BO1), %xmm0
+	vmovups	2*SIZE(BO1), %xmm2
+	vmovups	4*SIZE(BO1), %xmm4
+	vmovups	6*SIZE(BO1), %xmm6
+	vmovsd        (BO2), %xmm1
+	vmovsd  2*SIZE(BO2), %xmm3
+	vmovsd  4*SIZE(BO2), %xmm5
+	vmovsd  6*SIZE(BO2), %xmm7
+	vmovups	%xmm0,       (BO)
+	vmovsd	%xmm1, 2*SIZE(BO)
+	vmovups	%xmm2, 3*SIZE(BO)
+	vmovsd	%xmm3, 5*SIZE(BO)
+	vmovups	%xmm4, 6*SIZE(BO)
+	vmovsd	%xmm5, 8*SIZE(BO)
+	vmovups	%xmm6, 9*SIZE(BO)
+	vmovsd	%xmm7,11*SIZE(BO)
+	addq	$8*SIZE,BO1
+	addq	$8*SIZE,BO2
+	addq	$12*SIZE,BO
+	decq	%rax
+	jnz	.L6_02
+
+.L6_02a:
+
+	movq	K, %rax
+	andq	$3, %rax		// K % 4
+	jz	.L6_02c
+	ALIGN_4
+
+.L6_02b:
+
+	vmovups	(BO1), %xmm0
+	vmovsd  (BO2), %xmm1
+	vmovups	%xmm0,       (BO)
+	vmovsd	%xmm1, 2*SIZE(BO)
+	addq	$2*SIZE,BO1
+	addq	$2*SIZE,BO2
+	addq	$3*SIZE,BO
+	decq	%rax
+	jnz	.L6_02b
+
+.L6_02c:
+
+	movq	K, %rax
+	salq	$1,%rax			// K * 2
+	leaq	(B,%rax,8), BO1		// next offset to BO1
+	leaq	(BO1,%rax,8), BO2	// next offset to BO1
+	leaq    BUFFER2, BO		// second buffer to BO
+	movq	K, %rax
+	sarq	$2, %rax		// k / 4
+	jz	.L6_03a
+	ALIGN_4
+
+
+.L6_03:
+
+	prefetcht0 512(BO2)
+	prefetchw  512(BO)
+	vmovups	      (BO2), %xmm0
+	vmovups	2*SIZE(BO2), %xmm2
+	vmovups	4*SIZE(BO2), %xmm4
+	vmovups	6*SIZE(BO2), %xmm6
+	vmovsd  1*SIZE(BO1), %xmm1
+	vmovsd  3*SIZE(BO1), %xmm3
+	vmovsd  5*SIZE(BO1), %xmm5
+	vmovsd  7*SIZE(BO1), %xmm7
+	vmovsd	%xmm1, 0*SIZE(BO)
+	vmovups	%xmm0, 1*SIZE(BO)
+	vmovsd	%xmm3, 3*SIZE(BO)
+	vmovups	%xmm2, 4*SIZE(BO)
+	vmovsd	%xmm5, 6*SIZE(BO)
+	vmovups	%xmm4, 7*SIZE(BO)
+	vmovsd	%xmm7, 9*SIZE(BO)
+	vmovups	%xmm6,10*SIZE(BO)
+	addq	$8*SIZE,BO1
+	addq	$8*SIZE,BO2
+	addq	$12*SIZE,BO
+	decq	%rax
+	jnz	.L6_03
+
+.L6_03a:
+
+        movq    K, %rax
+        andq    $3, %rax                // K % 4
+        jz      .L6_03c
+        ALIGN_4
+
+
+.L6_03b:
+
+	vmovsd	  1*SIZE(BO1), %xmm0
+	vmovups  	(BO2), %xmm1
+	vmovsd	%xmm0,       (BO)
+	vmovups %xmm1, 1*SIZE(BO)
+	addq	$2*SIZE,BO1
+	addq	$2*SIZE,BO2
+	addq	$3*SIZE,BO
+	decq	%rax
+	jnz	.L6_03b
+
+
+.L6_03c:
+
+	movq	BO2, B			// next offset of B
+
+.L6_10:
+	movq	C, CO1
+	leaq	(C, LDC, 2), C		 
+	leaq	(C, LDC, 1), C		// c += 3 * ldc
+
+	
+	movq	A, AO		 	// aoffset = a
+	addq	$16 * SIZE, AO
+
+	movq	M,  I
+	sarq	$3, I			// i = (m >> 3)
+	je	.L6_20
+
+	ALIGN_4
+
+.L6_11:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $6 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax			//  K = K - ( K % 8 )
+	je	.L6_16
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L6_12:
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL8x3_1(xxx)
+	KERNEL8x3_2(xxx)
+	KERNEL8x3_3(xxx)
+	prefetcht0	B_PR1+64(BO,BI,8)
+	KERNEL8x3_4(xxx)
+
+	KERNEL8x3_1(xxx)
+	KERNEL8x3_2(xxx)
+	prefetcht0	B_PR1+32(BO,BI,8)
+	KERNEL8x3_3(xxx)
+	KERNEL8x3_4(xxx)
+
+	je	.L6_16
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL8x3_1(xxx)
+	KERNEL8x3_2(xxx)
+	KERNEL8x3_3(xxx)
+	prefetcht0	B_PR1+64(BO,BI,8)
+	KERNEL8x3_4(xxx)
+
+	KERNEL8x3_1(xxx)
+	KERNEL8x3_2(xxx)
+	prefetcht0	B_PR1+32(BO,BI,8)
+	KERNEL8x3_3(xxx)
+	KERNEL8x3_4(xxx)
+
+	je	.L6_16
+
+	jmp	.L6_12
+	ALIGN_4
+
+.L6_16:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L6_19
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L6_17:
+
+	KERNEL8x3_SUB(xxx)
+	addq	$3, BI
+	addq	$8, %rax
+	jl	.L6_17
+	ALIGN_4
+
+
+.L6_19:
+
+	vmovddup	ALPHA, %xmm0
+
+
+	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7
+	vfmaddpd 4 * SIZE(CO1),%xmm0, %xmm10,%xmm10
+	vfmaddpd 6 * SIZE(CO1),%xmm0, %xmm13,%xmm13
+
+	vfmaddpd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+	vfmaddpd 2 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
+	vfmaddpd 4 * SIZE(CO1, LDC),%xmm0, %xmm11,%xmm11
+	vfmaddpd 6 * SIZE(CO1, LDC),%xmm0, %xmm14,%xmm14
+
+	vfmaddpd 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
+	vfmaddpd 2 * SIZE(CO1, LDC, 2),%xmm0, %xmm9,%xmm9
+	vfmaddpd 4 * SIZE(CO1, LDC, 2),%xmm0, %xmm12,%xmm12
+	vfmaddpd 6 * SIZE(CO1, LDC, 2),%xmm0, %xmm15,%xmm15
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm7 , 2 * SIZE(CO1)
+	vmovups	%xmm10, 4 * SIZE(CO1)
+	vmovups	%xmm13, 6 * SIZE(CO1)
+
+	vmovups	%xmm5 ,  	(CO1, LDC)
+	vmovups	%xmm8 , 2 * SIZE(CO1, LDC)
+	vmovups	%xmm11, 4 * SIZE(CO1, LDC)
+	vmovups	%xmm14, 6 * SIZE(CO1, LDC)
+
+	vmovups	%xmm6 ,  	(CO1, LDC, 2)
+	vmovups	%xmm9 , 2 * SIZE(CO1, LDC, 2)
+	vmovups	%xmm12, 4 * SIZE(CO1, LDC, 2)
+	vmovups	%xmm15, 6 * SIZE(CO1, LDC, 2)
+
+
+	addq	$8 * SIZE, CO1		# coffset += 8
+	decq	I			# i --
+	jg	.L6_11
+	ALIGN_4	
+
+/**************************************************************************
+* Rest of M 
+***************************************************************************/
+.L6_20:
+	// Test rest of M
+
+	testq	$7, M
+	jz	.L7_10		// to next 3 lines of N
+
+	testq	$4, M		
+	jz	.L6_30
+
+	ALIGN_4
+
+.L6_21:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $6 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L6_26
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L6_22:
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL4x3_1(xxx)
+	KERNEL4x3_2(xxx)
+	KERNEL4x3_3(xxx)
+	prefetcht0	B_PR1+64(BO,BI,8)
+	KERNEL4x3_4(xxx)
+
+	KERNEL4x3_1(xxx)
+	KERNEL4x3_2(xxx)
+	prefetcht0	B_PR1+32(BO,BI,8)
+	KERNEL4x3_3(xxx)
+	KERNEL4x3_4(xxx)
+
+	je	.L6_26
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL4x3_1(xxx)
+	KERNEL4x3_2(xxx)
+	KERNEL4x3_3(xxx)
+	prefetcht0	B_PR1+64(BO,BI,8)
+	KERNEL4x3_4(xxx)
+
+	KERNEL4x3_1(xxx)
+	KERNEL4x3_2(xxx)
+	prefetcht0	B_PR1+32(BO,BI,8)
+	KERNEL4x3_3(xxx)
+	KERNEL4x3_4(xxx)
+
+	je	.L6_26
+
+	jmp	.L6_22
+	ALIGN_4
+
+.L6_26:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L6_29
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L6_27:
+
+	KERNEL4x3_SUB(xxx)
+	addq	$3, BI
+	addq	$4, %rax
+	jl	.L6_27
+	ALIGN_4
+
+
+.L6_29:
+
+	vmovddup	ALPHA, %xmm0
+
+
+	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7
+
+	vfmaddpd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+	vfmaddpd 2 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
+
+	vfmaddpd 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
+	vfmaddpd 2 * SIZE(CO1, LDC, 2),%xmm0, %xmm9,%xmm9
+
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm7 , 2 * SIZE(CO1)
+
+	vmovups	%xmm5 ,  	(CO1, LDC)
+	vmovups	%xmm8 , 2 * SIZE(CO1, LDC)
+
+	vmovups	%xmm6 ,  	(CO1, LDC, 2)
+	vmovups	%xmm9 , 2 * SIZE(CO1, LDC, 2)
+
+
+	addq	$4 * SIZE, CO1		# coffset += 4
+	ALIGN_4
+	
+
+.L6_30:
+	testq	$2, M		
+	jz	.L6_40
+
+	ALIGN_4
+
+.L6_31:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $6 * SIZE, BO
+
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L6_36
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L6_32:
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL2x3_1(xxx)
+	KERNEL2x3_2(xxx)
+	KERNEL2x3_3(xxx)
+	prefetcht0	B_PR1+64(BO,BI,8)
+	KERNEL2x3_4(xxx)
+
+	KERNEL2x3_1(xxx)
+	KERNEL2x3_2(xxx)
+	prefetcht0	B_PR1+32(BO,BI,8)
+	KERNEL2x3_3(xxx)
+	KERNEL2x3_4(xxx)
+
+	je	.L6_36
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL2x3_1(xxx)
+	KERNEL2x3_2(xxx)
+	KERNEL2x3_3(xxx)
+	prefetcht0	B_PR1+64(BO,BI,8)
+	KERNEL2x3_4(xxx)
+
+	KERNEL2x3_1(xxx)
+	KERNEL2x3_2(xxx)
+	prefetcht0	B_PR1+32(BO,BI,8)
+	KERNEL2x3_3(xxx)
+	KERNEL2x3_4(xxx)
+
+	je	.L6_36
+
+	jmp	.L6_32
+	ALIGN_4
+
+.L6_36:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L6_39
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+	
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L6_37:
+
+	KERNEL2x3_SUB(xxx)
+	addq	$3, BI
+	addq	$2, %rax
+	jl	.L6_37
+	ALIGN_4
+
+
+.L6_39:
+
+	vmovddup	ALPHA, %xmm0
+
+
+	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddpd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+	vfmaddpd 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
+
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm5 ,  	(CO1, LDC)
+	vmovups	%xmm6 ,  	(CO1, LDC, 2)
+
+
+	addq	$2 * SIZE, CO1		# coffset += 2
+	ALIGN_4
+
+.L6_40:
+	testq	$1, M		
+	jz	.L7_10		// to next 3 lines of N
+
+	ALIGN_4
+
+.L6_41:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $6 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L6_46
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L6_42:
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL1x3_1(xxx)
+	KERNEL1x3_2(xxx)
+	KERNEL1x3_3(xxx)
+	prefetcht0	B_PR1+64(BO,BI,8)
+	KERNEL1x3_4(xxx)
+
+	KERNEL1x3_1(xxx)
+	KERNEL1x3_2(xxx)
+	prefetcht0	B_PR1+32(BO,BI,8)
+	KERNEL1x3_3(xxx)
+	KERNEL1x3_4(xxx)
+
+	je	.L6_46
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL1x3_1(xxx)
+	KERNEL1x3_2(xxx)
+	KERNEL1x3_3(xxx)
+	prefetcht0	B_PR1+64(BO,BI,8)
+	KERNEL1x3_4(xxx)
+
+	KERNEL1x3_1(xxx)
+	KERNEL1x3_2(xxx)
+	prefetcht0	B_PR1+32(BO,BI,8)
+	KERNEL1x3_3(xxx)
+	KERNEL1x3_4(xxx)
+
+	je	.L6_46
+
+	jmp	.L6_42
+	ALIGN_4
+
+.L6_46:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L6_49
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L6_47:
+
+	KERNEL1x3_SUB(xxx)
+	addq	$3, BI
+	addq	$1, %rax
+	jl	.L6_47
+	ALIGN_4
+
+
+.L6_49:
+
+	vmovddup	ALPHA, %xmm0
+
+
+	vfmaddsd 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddsd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+	vfmaddsd 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
+
+	vmovsd	%xmm4 ,  	(CO1)
+	vmovsd	%xmm5 ,  	(CO1, LDC)
+	vmovsd	%xmm6 ,  	(CO1, LDC, 2)
+
+
+	addq	$1 * SIZE, CO1		# coffset += 1
+	ALIGN_4
+	
+
+
+
+/***************************************************************************************************************/
+
+.L7_10:
+	movq	C, CO1
+	leaq	(C, LDC, 2), C		 
+	leaq	(C, LDC, 1), C		// c += 3 * ldc
+
+
+	movq	A, AO		 	// aoffset = a
+	addq	$16 * SIZE, AO
+
+	movq	M,  I
+	sarq	$3, I			// i = (m >> 3)
+	je	.L7_20
+	ALIGN_4
+
+.L7_11:
+        leaq    BUFFER2, BO             // second buffer to BO
+        addq    $6 * SIZE, BO
+
+
+	vzeroall
+
+        movq    K, %rax
+
+
+	andq	$-8, %rax
+	je	.L7_16
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+
+	ALIGN_4
+
+.L7_12:
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL8x3_1(xxx)
+	KERNEL8x3_2(xxx)
+	KERNEL8x3_3(xxx)
+	prefetcht0	B_PR1+64(BO,BI,8)
+	KERNEL8x3_4(xxx)
+
+	KERNEL8x3_1(xxx)
+	KERNEL8x3_2(xxx)
+	prefetcht0	B_PR1+32(BO,BI,8)
+	KERNEL8x3_3(xxx)
+	KERNEL8x3_4(xxx)
+
+	je	.L7_16
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL8x3_1(xxx)
+	KERNEL8x3_2(xxx)
+	KERNEL8x3_3(xxx)
+	prefetcht0	B_PR1+64(BO,BI,8)
+	KERNEL8x3_4(xxx)
+
+	KERNEL8x3_1(xxx)
+	KERNEL8x3_2(xxx)
+	prefetcht0	B_PR1+32(BO,BI,8)
+	KERNEL8x3_3(xxx)
+	KERNEL8x3_4(xxx)
+
+	je	.L7_16
+
+	jmp	.L7_12
+	ALIGN_4
+
+.L7_16:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L7_19
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L7_17:
+
+	KERNEL8x3_SUB(xxx)
+	addq	$3, BI
+	addq	$8, %rax
+	jl	.L7_17
+	ALIGN_4
+
+
+.L7_19:
+
+	vmovddup	ALPHA, %xmm0
+
+	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7
+	vfmaddpd 4 * SIZE(CO1),%xmm0, %xmm10,%xmm10
+	vfmaddpd 6 * SIZE(CO1),%xmm0, %xmm13,%xmm13
+
+	vfmaddpd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+	vfmaddpd 2 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
+	vfmaddpd 4 * SIZE(CO1, LDC),%xmm0, %xmm11,%xmm11
+	vfmaddpd 6 * SIZE(CO1, LDC),%xmm0, %xmm14,%xmm14
+
+	vfmaddpd 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
+	vfmaddpd 2 * SIZE(CO1, LDC, 2),%xmm0, %xmm9,%xmm9
+	vfmaddpd 4 * SIZE(CO1, LDC, 2),%xmm0, %xmm12,%xmm12
+	vfmaddpd 6 * SIZE(CO1, LDC, 2),%xmm0, %xmm15,%xmm15
+
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm7 , 2 * SIZE(CO1)
+	vmovups	%xmm10, 4 * SIZE(CO1)
+	vmovups	%xmm13, 6 * SIZE(CO1)
+
+	vmovups	%xmm5 ,  	(CO1, LDC)
+	vmovups	%xmm8 , 2 * SIZE(CO1, LDC)
+	vmovups	%xmm11, 4 * SIZE(CO1, LDC)
+	vmovups	%xmm14, 6 * SIZE(CO1, LDC)
+
+	vmovups	%xmm6 ,  	(CO1, LDC, 2)
+	vmovups	%xmm9 , 2 * SIZE(CO1, LDC, 2)
+	vmovups	%xmm12, 4 * SIZE(CO1, LDC, 2)
+	vmovups	%xmm15, 6 * SIZE(CO1, LDC, 2)
+
+
+
+
+	addq	$8 * SIZE, CO1		# coffset += 8
+	decq	I			# i --
+	jg	.L7_11
+	ALIGN_4	
+
+.L7_20:
+	// Test rest of M
+
+	testq	$7, M
+	jz	.L7_60		// to next 6 lines of N
+
+	testq	$4, M		
+	jz	.L7_30
+
+	ALIGN_4
+
+.L7_21:
+        leaq    BUFFER2, BO             // second buffer to BO
+        addq    $6 * SIZE, BO
+
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L7_26
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L7_22:
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL4x3_1(xxx)
+	KERNEL4x3_2(xxx)
+	KERNEL4x3_3(xxx)
+	prefetcht0	B_PR1+64(BO,BI,8)
+	KERNEL4x3_4(xxx)
+
+	KERNEL4x3_1(xxx)
+	KERNEL4x3_2(xxx)
+	prefetcht0	B_PR1+32(BO,BI,8)
+	KERNEL4x3_3(xxx)
+	KERNEL4x3_4(xxx)
+
+	je	.L7_26
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL4x3_1(xxx)
+	KERNEL4x3_2(xxx)
+	KERNEL4x3_3(xxx)
+	prefetcht0	B_PR1+64(BO,BI,8)
+	KERNEL4x3_4(xxx)
+
+	KERNEL4x3_1(xxx)
+	KERNEL4x3_2(xxx)
+	prefetcht0	B_PR1+32(BO,BI,8)
+	KERNEL4x3_3(xxx)
+	KERNEL4x3_4(xxx)
+
+	je	.L7_26
+
+	jmp	.L7_22
+	ALIGN_4
+
+.L7_26:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L7_29
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L7_27:
+
+	KERNEL4x3_SUB(xxx)
+	addq	$3, BI
+	addq	$4, %rax
+	jl	.L7_27
+	ALIGN_4
+
+
+.L7_29:
+
+	vmovddup	ALPHA, %xmm0
+
+
+	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7
+
+	vfmaddpd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+	vfmaddpd 2 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
+
+	vfmaddpd 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
+	vfmaddpd 2 * SIZE(CO1, LDC, 2),%xmm0, %xmm9,%xmm9
+
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm7 , 2 * SIZE(CO1)
+
+	vmovups	%xmm5 ,  	(CO1, LDC)
+	vmovups	%xmm8 , 2 * SIZE(CO1, LDC)
+
+	vmovups	%xmm6 ,  	(CO1, LDC, 2)
+	vmovups	%xmm9 , 2 * SIZE(CO1, LDC, 2)
+
+
+	addq	$4 * SIZE, CO1		# coffset += 4
+	ALIGN_4
+	
+
+.L7_30:
+	testq	$2, M		
+	jz	.L7_40
+
+	ALIGN_4
+
+.L7_31:
+        leaq    BUFFER2, BO             // second buffer to BO
+        addq    $6 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L7_36
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L7_32:
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL2x3_1(xxx)
+	KERNEL2x3_2(xxx)
+	KERNEL2x3_3(xxx)
+	prefetcht0	B_PR1+64(BO,BI,8)
+	KERNEL2x3_4(xxx)
+
+	KERNEL2x3_1(xxx)
+	KERNEL2x3_2(xxx)
+	prefetcht0	B_PR1+32(BO,BI,8)
+	KERNEL2x3_3(xxx)
+	KERNEL2x3_4(xxx)
+
+	je	.L7_36
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL2x3_1(xxx)
+	KERNEL2x3_2(xxx)
+	KERNEL2x3_3(xxx)
+	prefetcht0	B_PR1+64(BO,BI,8)
+	KERNEL2x3_4(xxx)
+
+	KERNEL2x3_1(xxx)
+	KERNEL2x3_2(xxx)
+	prefetcht0	B_PR1+32(BO,BI,8)
+	KERNEL2x3_3(xxx)
+	KERNEL2x3_4(xxx)
+
+	je	.L7_36
+
+	jmp	.L7_32
+	ALIGN_4
+
+.L7_36:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L7_39
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L7_37:
+
+	KERNEL2x3_SUB(xxx)
+	addq	$3, BI
+	addq	$2, %rax
+	jl	.L7_37
+	ALIGN_4
+
+
+.L7_39:
+
+	vmovddup	ALPHA, %xmm0
+
+
+	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddpd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+	vfmaddpd 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
+
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm5 ,  	(CO1, LDC)
+	vmovups	%xmm6 ,  	(CO1, LDC, 2)
+
+
+	addq	$2 * SIZE, CO1		# coffset += 2
+	ALIGN_4
+	
+
+
+
+
+.L7_40:
+	testq	$1, M		
+	jz	.L7_60		// to next 6 lines of N
+
+	ALIGN_4
+
+.L7_41:
+        leaq    BUFFER2, BO             // second buffer to BO
+        addq    $6 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+
+	andq	$-8, %rax
+	je	.L7_46
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L7_42:
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL1x3_1(xxx)
+	KERNEL1x3_2(xxx)
+	KERNEL1x3_3(xxx)
+	prefetcht0	B_PR1+64(BO,BI,8)
+	KERNEL1x3_4(xxx)
+
+	KERNEL1x3_1(xxx)
+	KERNEL1x3_2(xxx)
+	prefetcht0	B_PR1+32(BO,BI,8)
+	KERNEL1x3_3(xxx)
+	KERNEL1x3_4(xxx)
+
+	je	.L7_46
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL1x3_1(xxx)
+	KERNEL1x3_2(xxx)
+	KERNEL1x3_3(xxx)
+	prefetcht0	B_PR1+64(BO,BI,8)
+	KERNEL1x3_4(xxx)
+
+	KERNEL1x3_1(xxx)
+	KERNEL1x3_2(xxx)
+	prefetcht0	B_PR1+32(BO,BI,8)
+	KERNEL1x3_3(xxx)
+	KERNEL1x3_4(xxx)
+
+	je	.L7_46
+
+	jmp	.L7_42
+	ALIGN_4
+
+.L7_46:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L7_49
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L7_47:
+
+	KERNEL1x3_SUB(xxx)
+	addq	$3, BI
+	addq	$1, %rax
+	jl	.L7_47
+	ALIGN_4
+
+
+.L7_49:
+
+	vmovddup	ALPHA, %xmm0
+
+
+	vfmaddsd 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddsd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+	vfmaddsd 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
+
+
+	vmovsd	%xmm4 ,  	(CO1)
+	vmovsd	%xmm5 ,  	(CO1, LDC)
+	vmovsd	%xmm6 ,  	(CO1, LDC, 2)
+
+
+	addq	$1 * SIZE, CO1		# coffset += 1
+
+.L7_60:
+
+	decq	J			// j --
+	jg	.L6_01
+
+
+.L2_0:
+	cmpq	$0, Nmod6		// N % 6 == 0
+	je	.L999
+
+/************************************************************************************************
+* Loop for Nmod6 / 2 > 0
+*************************************************************************************************/
+
+	movq	Nmod6, J		
+	sarq	$1, J			// j = j / 2
+	je	.L1_0
+	ALIGN_4
+
+.L2_01:
+	// copy to sub buffer
+	movq	B, BO1
+	leaq    BUFFER1, BO		// first buffer to BO
+	movq	K, %rax
+	ALIGN_4
+
+.L2_02b:
+
+	vmovups	(BO1), %xmm0
+	vmovups	%xmm0,       (BO)
+	addq	$2*SIZE,BO1
+	addq	$2*SIZE,BO
+	decq	%rax
+	jnz	.L2_02b
+
+.L2_02c:
+
+	movq	BO1, B			// next offset of B
+
+.L2_10:
+	movq	C, CO1
+	leaq	(C, LDC, 2), C		// c += 2 * ldc
+
+	
+	movq	A, AO		 	// aoffset = a
+	addq	$16 * SIZE, AO
+
+	movq	M,  I
+	sarq	$3, I			// i = (m >> 3)
+	je	.L2_20
+
+	ALIGN_4
+
+.L2_11:
+
+	leaq	BUFFER1, BO		// first buffer to BO
+	addq	$4 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax			//  K = K - ( K % 8 )
+	je	.L2_16
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_12:
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL8x2_1(xxx)
+	KERNEL8x2_2(xxx)
+	KERNEL8x2_3(xxx)
+	KERNEL8x2_4(xxx)
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL8x2_1(xxx)
+	KERNEL8x2_2(xxx)
+	KERNEL8x2_3(xxx)
+	KERNEL8x2_4(xxx)
+
+	je	.L2_16
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL8x2_1(xxx)
+	KERNEL8x2_2(xxx)
+	KERNEL8x2_3(xxx)
+	KERNEL8x2_4(xxx)
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL8x2_1(xxx)
+	KERNEL8x2_2(xxx)
+	KERNEL8x2_3(xxx)
+	KERNEL8x2_4(xxx)
+
+	je	.L2_16
+
+	jmp	.L2_12
+	ALIGN_4
+
+.L2_16:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_19
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_17:
+
+	KERNEL8x2_SUB(xxx)
+	addq	$2, BI
+	addq	$8, %rax
+	jl	.L2_17
+	ALIGN_4
+
+
+.L2_19:
+
+	vmovddup	ALPHA, %xmm0
+
+	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7
+	vfmaddpd 4 * SIZE(CO1),%xmm0, %xmm10,%xmm10
+	vfmaddpd 6 * SIZE(CO1),%xmm0, %xmm13,%xmm13
+
+	vfmaddpd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+	vfmaddpd 2 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
+	vfmaddpd 4 * SIZE(CO1, LDC),%xmm0, %xmm11,%xmm11
+	vfmaddpd 6 * SIZE(CO1, LDC),%xmm0, %xmm14,%xmm14
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm7 , 2 * SIZE(CO1)
+	vmovups	%xmm10, 4 * SIZE(CO1)
+	vmovups	%xmm13, 6 * SIZE(CO1)
+
+	vmovups	%xmm5 ,  	(CO1, LDC)
+	vmovups	%xmm8 , 2 * SIZE(CO1, LDC)
+	vmovups	%xmm11, 4 * SIZE(CO1, LDC)
+	vmovups	%xmm14, 6 * SIZE(CO1, LDC)
+
+
+	addq	$8 * SIZE, CO1		# coffset += 8
+	decq	I			# i --
+	jg	.L2_11
+	ALIGN_4	
+
+/**************************************************************************
+* Rest of M 
+***************************************************************************/
+.L2_20:
+	// Test rest of M
+
+	testq	$7, M
+	jz	.L2_60		// to next 2 lines of N
+
+	testq	$4, M		
+	jz	.L2_30
+
+	ALIGN_4
+
+.L2_21:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L2_26
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_22:
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL4x2_1(xxx)
+	KERNEL4x2_2(xxx)
+	KERNEL4x2_3(xxx)
+	KERNEL4x2_4(xxx)
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL4x2_1(xxx)
+	KERNEL4x2_2(xxx)
+	KERNEL4x2_3(xxx)
+	KERNEL4x2_4(xxx)
+
+	je	.L2_26
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL4x2_1(xxx)
+	KERNEL4x2_2(xxx)
+	KERNEL4x2_3(xxx)
+	KERNEL4x2_4(xxx)
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL4x2_1(xxx)
+	KERNEL4x2_2(xxx)
+	KERNEL4x2_3(xxx)
+	KERNEL4x2_4(xxx)
+
+	je	.L2_26
+
+	jmp	.L2_22
+	ALIGN_4
+
+.L2_26:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_29
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_27:
+
+	KERNEL4x2_SUB(xxx)
+	addq	$2, BI
+	addq	$4, %rax
+	jl	.L2_27
+	ALIGN_4
+
+
+.L2_29:
+
+	vmovddup	ALPHA, %xmm0
+
+	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7
+
+	vfmaddpd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+	vfmaddpd 2 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm7 , 2 * SIZE(CO1)
+
+	vmovups	%xmm5 ,  	(CO1, LDC)
+	vmovups	%xmm8 , 2 * SIZE(CO1, LDC)
+
+	addq	$4 * SIZE, CO1		# coffset += 4
+	ALIGN_4
+	
+
+.L2_30:
+	testq	$2, M		
+	jz	.L2_40
+
+	ALIGN_4
+
+.L2_31:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L2_36
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_32:
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL2x2_1(xxx)
+	KERNEL2x2_2(xxx)
+	KERNEL2x2_3(xxx)
+	KERNEL2x2_4(xxx)
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL2x2_1(xxx)
+	KERNEL2x2_2(xxx)
+	KERNEL2x2_3(xxx)
+	KERNEL2x2_4(xxx)
+
+	je	.L2_36
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL2x2_1(xxx)
+	KERNEL2x2_2(xxx)
+	KERNEL2x2_3(xxx)
+	KERNEL2x2_4(xxx)
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL2x2_1(xxx)
+	KERNEL2x2_2(xxx)
+	KERNEL2x2_3(xxx)
+	KERNEL2x2_4(xxx)
+
+	je	.L2_36
+
+	jmp	.L2_32
+	ALIGN_4
+
+.L2_36:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_39
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+	
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_37:
+
+	KERNEL2x2_SUB(xxx)
+	addq	$2, BI
+	addq	$2, %rax
+	jl	.L2_37
+	ALIGN_4
+
+
+.L2_39:
+
+	vmovddup	ALPHA, %xmm0
+
+	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddpd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm5 ,  	(CO1, LDC)
+
+	addq	$2 * SIZE, CO1		# coffset += 2
+	ALIGN_4
+	
+
+.L2_40:
+	testq	$1, M		
+	jz	.L2_60		// to next 2 lines of N
+
+	ALIGN_4
+
+.L2_41:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L2_46
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_42:
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL1x2_1(xxx)
+	KERNEL1x2_2(xxx)
+	KERNEL1x2_3(xxx)
+	KERNEL1x2_4(xxx)
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL1x2_1(xxx)
+	KERNEL1x2_2(xxx)
+	KERNEL1x2_3(xxx)
+	KERNEL1x2_4(xxx)
+
+	je	.L2_46
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL1x2_1(xxx)
+	KERNEL1x2_2(xxx)
+	KERNEL1x2_3(xxx)
+	KERNEL1x2_4(xxx)
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL1x2_1(xxx)
+	KERNEL1x2_2(xxx)
+	KERNEL1x2_3(xxx)
+	KERNEL1x2_4(xxx)
+
+	je	.L2_46
+
+	jmp	.L2_42
+	ALIGN_4
+
+.L2_46:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_49
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_47:
+
+	KERNEL1x2_SUB(xxx)
+	addq	$2, BI
+	addq	$1, %rax
+	jl	.L2_47
+	ALIGN_4
+
+
+.L2_49:
+
+	vmovddup	ALPHA, %xmm0
+
+	vfmaddsd 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddsd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+
+	vmovsd	%xmm4 ,  	(CO1)
+	vmovsd	%xmm5 ,  	(CO1, LDC)
+
+
+	addq	$1 * SIZE, CO1		# coffset += 1
+	ALIGN_4
+	
+
+	
+.L2_60:
+
+	decq	J			// j --
+	jg	.L2_01			// next 2 lines of N
+
+
+
+.L1_0:
+
+/************************************************************************************************
+* Loop for Nmod6 % 2 > 0
+*************************************************************************************************/
+
+	movq	Nmod6, J		
+	andq	$1, J			// j % 2
+	je	.L999
+	ALIGN_4
+
+.L1_01:
+	// copy to sub buffer
+	movq	B, BO1
+	leaq    BUFFER1, BO		// first buffer to BO
+	movq	K, %rax
+	ALIGN_4
+
+.L1_02b:
+
+	vmovsd	(BO1), %xmm0
+	vmovsd	%xmm0,       (BO)
+	addq	$1*SIZE,BO1
+	addq	$1*SIZE,BO
+	decq	%rax
+	jnz	.L1_02b
+
+.L1_02c:
+
+	movq	BO1, B			// next offset of B
+
+.L1_10:
+	movq	C, CO1
+	leaq	(C, LDC, 1), C		// c += 1 * ldc
+
+	movq	A, AO		 	// aoffset = a
+	addq	$16 * SIZE, AO
+
+	movq	M,  I
+	sarq	$3, I			// i = (m >> 3)
+	je	.L1_20
+
+	ALIGN_4
+
+.L1_11:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax			//  K = K - ( K % 8 )
+	je	.L1_16
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_12:
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL8x1_1(xxx)
+	KERNEL8x1_2(xxx)
+	KERNEL8x1_3(xxx)
+	KERNEL8x1_4(xxx)
+
+	KERNEL8x1_1(xxx)
+	KERNEL8x1_2(xxx)
+	KERNEL8x1_3(xxx)
+	KERNEL8x1_4(xxx)
+
+	je	.L1_16
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL8x1_1(xxx)
+	KERNEL8x1_2(xxx)
+	KERNEL8x1_3(xxx)
+	KERNEL8x1_4(xxx)
+
+	KERNEL8x1_1(xxx)
+	KERNEL8x1_2(xxx)
+	KERNEL8x1_3(xxx)
+	KERNEL8x1_4(xxx)
+
+	je	.L1_16
+
+	jmp	.L1_12
+	ALIGN_4
+
+.L1_16:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_19
+
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_17:
+
+	KERNEL8x1_SUB(xxx)
+	addq	$1, BI
+	addq	$8, %rax
+	jl	.L1_17
+	ALIGN_4
+
+
+.L1_19:
+
+	vmovddup	ALPHA, %xmm0
+
+	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7
+	vfmaddpd 4 * SIZE(CO1),%xmm0, %xmm10,%xmm10
+	vfmaddpd 6 * SIZE(CO1),%xmm0, %xmm13,%xmm13
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm7 , 2 * SIZE(CO1)
+	vmovups	%xmm10, 4 * SIZE(CO1)
+	vmovups	%xmm13, 6 * SIZE(CO1)
+
+	addq	$8 * SIZE, CO1		# coffset += 8
+	decq	I			# i --
+	jg	.L1_11
+	ALIGN_4	
+
+/**************************************************************************
+* Rest of M 
+***************************************************************************/
+.L1_20:
+	// Test rest of M
+
+	testq	$7, M
+	jz	.L999
+
+	testq	$4, M		
+	jz	.L1_30
+
+	ALIGN_4
+
+.L1_21:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L1_26
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_22:
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL4x1_1(xxx)
+	KERNEL4x1_2(xxx)
+	KERNEL4x1_3(xxx)
+	KERNEL4x1_4(xxx)
+
+	KERNEL4x1_1(xxx)
+	KERNEL4x1_2(xxx)
+	KERNEL4x1_3(xxx)
+	KERNEL4x1_4(xxx)
+
+	je	.L1_26
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL4x1_1(xxx)
+	KERNEL4x1_2(xxx)
+	KERNEL4x1_3(xxx)
+	KERNEL4x1_4(xxx)
+
+	KERNEL4x1_1(xxx)
+	KERNEL4x1_2(xxx)
+	KERNEL4x1_3(xxx)
+	KERNEL4x1_4(xxx)
+
+	je	.L1_26
+
+	jmp	.L1_22
+	ALIGN_4
+
+.L1_26:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_29
+
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_27:
+
+	KERNEL4x1_SUB(xxx)
+	addq	$1, BI
+	addq	$4, %rax
+	jl	.L1_27
+	ALIGN_4
+
+
+.L1_29:
+
+	vmovddup	ALPHA, %xmm0
+
+	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm7 , 2 * SIZE(CO1)
+
+	addq	$4 * SIZE, CO1		# coffset += 4
+	ALIGN_4
+	
+
+.L1_30:
+	testq	$2, M		
+	jz	.L1_40
+
+	ALIGN_4
+
+.L1_31:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L1_36
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_32:
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL2x1_1(xxx)
+	KERNEL2x1_2(xxx)
+	KERNEL2x1_3(xxx)
+	KERNEL2x1_4(xxx)
+
+	KERNEL2x1_1(xxx)
+	KERNEL2x1_2(xxx)
+	KERNEL2x1_3(xxx)
+	KERNEL2x1_4(xxx)
+
+	je	.L1_36
+
+	KERNEL2x1_1(xxx)
+	KERNEL2x1_2(xxx)
+	KERNEL2x1_3(xxx)
+	KERNEL2x1_4(xxx)
+
+	KERNEL2x1_1(xxx)
+	KERNEL2x1_2(xxx)
+	KERNEL2x1_3(xxx)
+	KERNEL2x1_4(xxx)
+
+	je	.L1_36
+
+	jmp	.L1_32
+	ALIGN_4
+
+.L1_36:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_39
+
+	movq    %rax, BI                        //  Index for BO
+	
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_37:
+
+	KERNEL2x1_SUB(xxx)
+	addq	$1, BI
+	addq	$2, %rax
+	jl	.L1_37
+	ALIGN_4
+
+
+.L1_39:
+
+	vmovddup	ALPHA, %xmm0
+
+	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
+
+	vmovups	%xmm4 ,  	(CO1)
+
+	addq	$2 * SIZE, CO1		# coffset += 2
+	ALIGN_4
+	
+
+.L1_40:
+	testq	$1, M		
+	jz	.L999
+
+	ALIGN_4
+
+.L1_41:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L1_46
+	movq    %rax, BI                        //  Index for BO
+
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_42:
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL1x1_1(xxx)
+	KERNEL1x1_2(xxx)
+	KERNEL1x1_3(xxx)
+	KERNEL1x1_4(xxx)
+
+	KERNEL1x1_1(xxx)
+	KERNEL1x1_2(xxx)
+	KERNEL1x1_3(xxx)
+	KERNEL1x1_4(xxx)
+
+	je	.L1_46
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL1x1_1(xxx)
+	KERNEL1x1_2(xxx)
+	KERNEL1x1_3(xxx)
+	KERNEL1x1_4(xxx)
+
+	KERNEL1x1_1(xxx)
+	KERNEL1x1_2(xxx)
+	KERNEL1x1_3(xxx)
+	KERNEL1x1_4(xxx)
+
+	je	.L1_46
+
+	jmp	.L1_42
+	ALIGN_4
+
+.L1_46:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_49
+
+	movq    %rax, BI                        //  Index for BO
+
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_47:
+
+	KERNEL1x1_SUB(xxx)
+	addq	$1, BI
+	addq	$1, %rax
+	jl	.L1_47
+	ALIGN_4
+
+
+.L1_49:
+
+	vmovddup	ALPHA, %xmm0
+
+	vfmaddsd 	 (CO1),%xmm0, %xmm4,%xmm4
+
+	vmovsd	%xmm4 ,  	(CO1)
+
+	addq	$1 * SIZE, CO1		# coffset += 1
+	ALIGN_4
+	
+
+.L999:
+	movq   		SP, %rsp
+	movq	   (%rsp), %rbx
+	movq	  8(%rsp), %rbp
+	movq	 16(%rsp), %r12
+	movq	 24(%rsp), %r13
+	movq	 32(%rsp), %r14
+	movq	 40(%rsp), %r15
+
+#ifdef WINDOWS_ABI
+	movq	 48(%rsp), %rdi
+	movq	 56(%rsp), %rsi
+	movups	 64(%rsp), %xmm6
+	movups	 80(%rsp), %xmm7
+	movups	 96(%rsp), %xmm8
+	movups	112(%rsp), %xmm9
+	movups	128(%rsp), %xmm10
+	movups	144(%rsp), %xmm11
+	movups	160(%rsp), %xmm12
+	movups	176(%rsp), %xmm13
+	movups	192(%rsp), %xmm14
+	movups	208(%rsp), %xmm15
+#endif
+
+	addq	$STACKSIZE, %rsp
+	ret
+
+	EPILOGUE
+
+
+#else
+/*************************************************************************************
+* TRMM Kernel
+*************************************************************************************/
+	PROLOGUE
+	PROFCODE
+	
+	subq	$STACKSIZE, %rsp
+	movq	%rbx,   (%rsp)
+	movq	%rbp,  8(%rsp)
+	movq	%r12, 16(%rsp)
+	movq	%r13, 24(%rsp)
+	movq	%r14, 32(%rsp)
+	movq	%r15, 40(%rsp)
+
+	vzeroupper
+
+#ifdef WINDOWS_ABI
+	movq	%rdi,    48(%rsp)
+	movq	%rsi,    56(%rsp)
+	movups	%xmm6,   64(%rsp)
+	movups	%xmm7,   80(%rsp)
+	movups	%xmm8,   96(%rsp)
+	movups	%xmm9,  112(%rsp)
+	movups	%xmm10, 128(%rsp)
+	movups	%xmm11, 144(%rsp)
+	movups	%xmm12, 160(%rsp)
+	movups	%xmm13, 176(%rsp)
+	movups	%xmm14, 192(%rsp)
+	movups	%xmm15, 208(%rsp)
+
+	movq	ARG1,      OLD_M
+	movq	ARG2,      OLD_N
+	movq	ARG3,      OLD_K
+	movq	OLD_A,     A
+	movq	OLD_B,     B
+	movq	OLD_C,     C
+	movq	OLD_LDC,   LDC
+#ifdef TRMMKERNEL
+	movsd	OLD_OFFSET, %xmm12
+#endif
+	vmovaps	%xmm3, %xmm0
+
+#else
+	movq	STACKSIZE +  8(%rsp), LDC
+#ifdef TRMMKERNEL
+	movsd	STACKSIZE + 16(%rsp), %xmm12
+#endif
+
+#endif
+
+	movq    %rsp, SP      # save old stack
+        subq    $128 + L_BUFFER_SIZE, %rsp
+        andq    $-4096, %rsp    # align stack
+
+        STACK_TOUCH
+
+	cmpq	$0, OLD_M
+	je	.L999
+
+	cmpq	$0, OLD_N
+	je	.L999
+
+	cmpq	$0, OLD_K
+	je	.L999
+
+	movq	OLD_M, M
+	movq	OLD_N, N
+	movq	OLD_K, K
+
+	vmovsd	 %xmm0, ALPHA
+
+	salq	$BASE_SHIFT, LDC
+
+	movq    N, %rax
+        xorq    %rdx, %rdx
+        movq    $2,  %rdi
+        divq    %rdi                    //    N / 2
+        movq    %rax, Ndiv6             //    N / 2
+        movq    %rdx, Nmod6             //    N % 2
+
+	
+
+#ifdef TRMMKERNEL
+	vmovsd	%xmm12, OFFSET
+	vmovsd	%xmm12, KK
+#ifndef LEFT
+	negq	KK
+#endif	
+#endif
+
+	movq	Ndiv6,  J
+	cmpq	$0, J
+	je	.L1_0
+	ALIGN_4
+
+.L2_0:
+
+.L2_01:
+	// copy to sub buffer
+	movq	B, BO1
+	leaq    BUFFER1, BO		// first buffer to BO
+	movq	K, %rax
+	ALIGN_4
+
+.L2_02b:
+
+	vmovups	(BO1), %xmm0
+	vmovups	%xmm0,       (BO)
+	addq	$2*SIZE,BO1
+	addq	$2*SIZE,BO
+	decq	%rax
+	jnz	.L2_02b
+
+.L2_02c:
+
+	movq	BO1, B			// next offset of B
+
+.L2_10:
+	movq	C, CO1
+	leaq	(C, LDC, 2), C		// c += 2 * ldc
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        movq    OFFSET, %rax
+        movq    %rax, KK
+#endif
+	
+	movq	A, AO		 	// aoffset = a
+	addq	$16 * SIZE, AO
+
+	movq	M,  I
+	sarq	$3, I			// i = (m >> 3)
+	je	.L2_20
+
+	ALIGN_4
+
+.L2_11:
+
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+	leaq	BUFFER1, BO		// first buffer to BO
+	addq	$4 * SIZE, BO
+#else
+        movq    KK, %rax
+	leaq	BUFFER1, BO		// first buffer to BO
+	addq	$4 * SIZE, BO
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, 8), BO
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+        leaq    (AO, %rax, 8), AO
+#endif
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $8, %rax        // number of values in AO
+#else
+        addq    $2, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$-8, %rax			//  K = K - ( K % 8 )
+	je	.L2_16
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_12:
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL8x2_1(xxx)
+	KERNEL8x2_2(xxx)
+	KERNEL8x2_3(xxx)
+	KERNEL8x2_4(xxx)
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL8x2_1(xxx)
+	KERNEL8x2_2(xxx)
+	KERNEL8x2_3(xxx)
+	KERNEL8x2_4(xxx)
+
+	je	.L2_16
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL8x2_1(xxx)
+	KERNEL8x2_2(xxx)
+	KERNEL8x2_3(xxx)
+	KERNEL8x2_4(xxx)
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL8x2_1(xxx)
+	KERNEL8x2_2(xxx)
+	KERNEL8x2_3(xxx)
+	KERNEL8x2_4(xxx)
+
+	je	.L2_16
+
+	jmp	.L2_12
+	ALIGN_4
+
+.L2_16:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_19
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_17:
+
+	KERNEL8x2_SUB(xxx)
+	addq	$2, BI
+	addq	$8, %rax
+	jl	.L2_17
+	ALIGN_4
+
+
+.L2_19:
+
+	vmovddup	ALPHA, %xmm0
+
+#ifndef TRMMKERNEL
+
+	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7
+	vfmaddpd 4 * SIZE(CO1),%xmm0, %xmm10,%xmm10
+	vfmaddpd 6 * SIZE(CO1),%xmm0, %xmm13,%xmm13
+
+	vfmaddpd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+	vfmaddpd 2 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
+	vfmaddpd 4 * SIZE(CO1, LDC),%xmm0, %xmm11,%xmm11
+	vfmaddpd 6 * SIZE(CO1, LDC),%xmm0, %xmm14,%xmm14
+
+
+#else
+	vmulpd	%xmm0, %xmm4,%xmm4
+	vmulpd	%xmm0, %xmm7,%xmm7
+	vmulpd	%xmm0, %xmm10,%xmm10
+	vmulpd	%xmm0, %xmm13,%xmm13
+
+	vmulpd	%xmm0, %xmm5,%xmm5
+	vmulpd	%xmm0, %xmm8,%xmm8
+	vmulpd	%xmm0, %xmm11,%xmm11
+	vmulpd	%xmm0, %xmm14,%xmm14
+
+#endif
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm7 , 2 * SIZE(CO1)
+	vmovups	%xmm10, 4 * SIZE(CO1)
+	vmovups	%xmm13, 6 * SIZE(CO1)
+
+	vmovups	%xmm5 ,  	(CO1, LDC)
+	vmovups	%xmm8 , 2 * SIZE(CO1, LDC)
+	vmovups	%xmm11, 4 * SIZE(CO1, LDC)
+	vmovups	%xmm14, 6 * SIZE(CO1, LDC)
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, 8), BO
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+        leaq    (AO, %rax, 8), AO
+#endif
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $8, KK
+#endif
+
+	addq	$8 * SIZE, CO1		# coffset += 8
+	decq	I			# i --
+	jg	.L2_11
+	ALIGN_4	
+
+/**************************************************************************
+* Rest of M 
+***************************************************************************/
+.L2_20:
+	// Test rest of M
+
+	testq	$7, M
+	jz	.L2_60		// to next 2 lines of N
+
+	testq	$4, M		
+	jz	.L2_30
+
+	ALIGN_4
+
+.L2_21:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, 8), BO
+        salq    $2, %rax                        // rax = rax * 4 ; number of values
+        leaq    (AO, %rax, 8), AO
+#endif
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $4, %rax        // number of values in AO
+#else
+        addq    $2, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$-8, %rax
+	je	.L2_26
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_22:
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL4x2_1(xxx)
+	KERNEL4x2_2(xxx)
+	KERNEL4x2_3(xxx)
+	KERNEL4x2_4(xxx)
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL4x2_1(xxx)
+	KERNEL4x2_2(xxx)
+	KERNEL4x2_3(xxx)
+	KERNEL4x2_4(xxx)
+
+	je	.L2_26
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL4x2_1(xxx)
+	KERNEL4x2_2(xxx)
+	KERNEL4x2_3(xxx)
+	KERNEL4x2_4(xxx)
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL4x2_1(xxx)
+	KERNEL4x2_2(xxx)
+	KERNEL4x2_3(xxx)
+	KERNEL4x2_4(xxx)
+
+	je	.L2_26
+
+	jmp	.L2_22
+	ALIGN_4
+
+.L2_26:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_29
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_27:
+
+	KERNEL4x2_SUB(xxx)
+	addq	$2, BI
+	addq	$4, %rax
+	jl	.L2_27
+	ALIGN_4
+
+
+.L2_29:
+
+	vmovddup	ALPHA, %xmm0
+
+#ifndef TRMMKERNEL
+
+	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7
+
+	vfmaddpd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+	vfmaddpd 2 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
+
+
+#else
+	vmulpd	%xmm0, %xmm4,%xmm4
+	vmulpd	%xmm0, %xmm7,%xmm7
+
+	vmulpd	%xmm0, %xmm5,%xmm5
+	vmulpd	%xmm0, %xmm8,%xmm8
+
+#endif
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm7 , 2 * SIZE(CO1)
+
+	vmovups	%xmm5 ,  	(CO1, LDC)
+	vmovups	%xmm8 , 2 * SIZE(CO1, LDC)
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, 8), BO
+        salq    $2, %rax                        // rax = rax * 4 ; number of values
+        leaq    (AO, %rax, 8), AO
+#endif
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $4, KK
+#endif
+
+	addq	$4 * SIZE, CO1		# coffset += 4
+	ALIGN_4
+	
+
+.L2_30:
+	testq	$2, M		
+	jz	.L2_40
+
+	ALIGN_4
+
+.L2_31:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, 8), BO
+        salq    $1, %rax                        // rax = rax * 2 ; number of values
+        leaq    (AO, %rax, 8), AO
+#endif
+
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $2, %rax        // number of values in AO
+#else
+        addq    $2, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$-8, %rax
+	je	.L2_36
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_32:
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL2x2_1(xxx)
+	KERNEL2x2_2(xxx)
+	KERNEL2x2_3(xxx)
+	KERNEL2x2_4(xxx)
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL2x2_1(xxx)
+	KERNEL2x2_2(xxx)
+	KERNEL2x2_3(xxx)
+	KERNEL2x2_4(xxx)
+
+	je	.L2_36
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL2x2_1(xxx)
+	KERNEL2x2_2(xxx)
+	KERNEL2x2_3(xxx)
+	KERNEL2x2_4(xxx)
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL2x2_1(xxx)
+	KERNEL2x2_2(xxx)
+	KERNEL2x2_3(xxx)
+	KERNEL2x2_4(xxx)
+
+	je	.L2_36
+
+	jmp	.L2_32
+	ALIGN_4
+
+.L2_36:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_39
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+	
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_37:
+
+	KERNEL2x2_SUB(xxx)
+	addq	$2, BI
+	addq	$2, %rax
+	jl	.L2_37
+	ALIGN_4
+
+
+.L2_39:
+
+	vmovddup	ALPHA, %xmm0
+
+#ifndef TRMMKERNEL
+
+	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddpd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+
+#else
+	vmulpd	%xmm0, %xmm4,%xmm4
+	vmulpd	%xmm0, %xmm5,%xmm5
+
+#endif
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm5 ,  	(CO1, LDC)
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax 
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, 8), BO
+        salq    $1, %rax                        // rax = rax * 2 ; number of values
+        leaq    (AO, %rax, 8), AO
+#endif
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $2, KK
+#endif
+
+	addq	$2 * SIZE, CO1		# coffset += 2
+	ALIGN_4
+	
+
+.L2_40:
+	testq	$1, M		
+	jz	.L2_60		// to next 2 lines of N
+
+	ALIGN_4
+
+.L2_41:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, 8), BO
+        leaq    (AO, %rax, 8), AO
+#endif
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $1, %rax        // number of values in AO
+#else
+        addq    $2, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+	andq	$-8, %rax
+	je	.L2_46
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_42:
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL1x2_1(xxx)
+	KERNEL1x2_2(xxx)
+	KERNEL1x2_3(xxx)
+	KERNEL1x2_4(xxx)
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL1x2_1(xxx)
+	KERNEL1x2_2(xxx)
+	KERNEL1x2_3(xxx)
+	KERNEL1x2_4(xxx)
+
+	je	.L2_46
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL1x2_1(xxx)
+	KERNEL1x2_2(xxx)
+	KERNEL1x2_3(xxx)
+	KERNEL1x2_4(xxx)
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL1x2_1(xxx)
+	KERNEL1x2_2(xxx)
+	KERNEL1x2_3(xxx)
+	KERNEL1x2_4(xxx)
+
+	je	.L2_46
+
+	jmp	.L2_42
+	ALIGN_4
+
+.L2_46:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_49
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_47:
+
+	KERNEL1x2_SUB(xxx)
+	addq	$2, BI
+	addq	$1, %rax
+	jl	.L2_47
+	ALIGN_4
+
+
+.L2_49:
+
+	vmovddup	ALPHA, %xmm0
+
+#ifndef TRMMKERNEL
+
+	vfmaddsd 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddsd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+
+#else
+	vmulsd	%xmm0, %xmm4,%xmm4
+	vmulsd	%xmm0, %xmm5,%xmm5
+
+#endif
+
+	vmovsd	%xmm4 ,  	(CO1)
+	vmovsd	%xmm5 ,  	(CO1, LDC)
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, 8), BO
+        leaq    (AO, %rax, 8), AO
+#endif
+
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $1, KK
+#endif
+
+	addq	$1 * SIZE, CO1		# coffset += 1
+	ALIGN_4
+	
+
+	
+.L2_60:
+#if defined(TRMMKERNEL) && !defined(LEFT)
+        addq    $2, KK
+#endif
+
+	decq	J			// j --
+	jg	.L2_01			// next 2 lines of N
+
+
+
+.L1_0:
+
+/************************************************************************************************
+* Loop for Nmod6 % 2 > 0
+*************************************************************************************************/
+
+	movq	Nmod6, J		
+	andq	$1, J			// j % 2
+	je	.L999
+	ALIGN_4
+
+.L1_01:
+	// copy to sub buffer
+	movq	B, BO1
+	leaq    BUFFER1, BO		// first buffer to BO
+	movq	K, %rax
+	ALIGN_4
+
+.L1_02b:
+
+	vmovsd	(BO1), %xmm0
+	vmovsd	%xmm0,       (BO)
+	addq	$1*SIZE,BO1
+	addq	$1*SIZE,BO
+	decq	%rax
+	jnz	.L1_02b
+
+.L1_02c:
+
+	movq	BO1, B			// next offset of B
+
+.L1_10:
+	movq	C, CO1
+	leaq	(C, LDC, 1), C		// c += 1 * ldc
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        movq    OFFSET, %rax
+        movq    %rax, KK
+#endif
+
+	movq	A, AO		 	// aoffset = a
+	addq	$16 * SIZE, AO
+
+	movq	M,  I
+	sarq	$3, I			// i = (m >> 3)
+	je	.L1_20
+
+	ALIGN_4
+
+.L1_11:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BO, BI, 8), BO
+        salq    $3, %rax                        // rax = rax * 8 ; number of values
+        leaq    (AO, %rax, 8), AO
+#endif
+
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $8, %rax        // number of values in AO
+#else
+        addq    $1, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+	andq	$-8, %rax			//  K = K - ( K % 8 )
+	je	.L1_16
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_12:
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL8x1_1(xxx)
+	KERNEL8x1_2(xxx)
+	KERNEL8x1_3(xxx)
+	KERNEL8x1_4(xxx)
+
+	KERNEL8x1_1(xxx)
+	KERNEL8x1_2(xxx)
+	KERNEL8x1_3(xxx)
+	KERNEL8x1_4(xxx)
+
+	je	.L1_16
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL8x1_1(xxx)
+	KERNEL8x1_2(xxx)
+	KERNEL8x1_3(xxx)
+	KERNEL8x1_4(xxx)
+
+	KERNEL8x1_1(xxx)
+	KERNEL8x1_2(xxx)
+	KERNEL8x1_3(xxx)
+	KERNEL8x1_4(xxx)
+
+	je	.L1_16
+
+	jmp	.L1_12
+	ALIGN_4
+
+.L1_16:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_19
+
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_17:
+
+	KERNEL8x1_SUB(xxx)
+	addq	$1, BI
+	addq	$8, %rax
+	jl	.L1_17
+	ALIGN_4
+
+
+.L1_19:
+
+	vmovddup	ALPHA, %xmm0
+
+#ifndef TRMMKERNEL
+
+	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7
+	vfmaddpd 4 * SIZE(CO1),%xmm0, %xmm10,%xmm10
+	vfmaddpd 6 * SIZE(CO1),%xmm0, %xmm13,%xmm13
+
+#else
+	vmulpd	%xmm0, %xmm4,%xmm4
+	vmulpd	%xmm0, %xmm7,%xmm7
+	vmulpd	%xmm0, %xmm10,%xmm10
+	vmulpd	%xmm0, %xmm13,%xmm13
+
+#endif
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm7 , 2 * SIZE(CO1)
+	vmovups	%xmm10, 4 * SIZE(CO1)
+	vmovups	%xmm13, 6 * SIZE(CO1)
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BO, BI, 8), BO
+        salq    $3, %rax                        // rax = rax * 8 ; number of values
+        leaq    (AO, %rax, 8), AO
+#endif
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $8, KK
+#endif
+	addq	$8 * SIZE, CO1		# coffset += 8
+	decq	I			# i --
+	jg	.L1_11
+	ALIGN_4	
+
+/**************************************************************************
+* Rest of M 
+***************************************************************************/
+.L1_20:
+	// Test rest of M
+
+	testq	$7, M
+	jz	.L999
+
+	testq	$4, M		
+	jz	.L1_30
+
+	ALIGN_4
+
+.L1_21:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BO, BI, 8), BO
+        salq    $2, %rax                        // rax = rax * 4 ; number of values
+        leaq    (AO, %rax, 8), AO
+#endif
+
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $4, %rax        // number of values in AO
+#else
+        addq    $1, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$-8, %rax
+	je	.L1_26
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_22:
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL4x1_1(xxx)
+	KERNEL4x1_2(xxx)
+	KERNEL4x1_3(xxx)
+	KERNEL4x1_4(xxx)
+
+	KERNEL4x1_1(xxx)
+	KERNEL4x1_2(xxx)
+	KERNEL4x1_3(xxx)
+	KERNEL4x1_4(xxx)
+
+	je	.L1_26
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL4x1_1(xxx)
+	KERNEL4x1_2(xxx)
+	KERNEL4x1_3(xxx)
+	KERNEL4x1_4(xxx)
+
+	KERNEL4x1_1(xxx)
+	KERNEL4x1_2(xxx)
+	KERNEL4x1_3(xxx)
+	KERNEL4x1_4(xxx)
+
+	je	.L1_26
+
+	jmp	.L1_22
+	ALIGN_4
+
+.L1_26:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_29
+
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_27:
+
+	KERNEL4x1_SUB(xxx)
+	addq	$1, BI
+	addq	$4, %rax
+	jl	.L1_27
+	ALIGN_4
+
+
+.L1_29:
+
+	vmovddup	ALPHA, %xmm0
+
+#ifndef TRMMKERNEL
+
+	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7
+
+#else
+	vmulpd	%xmm0, %xmm4,%xmm4
+	vmulpd	%xmm0, %xmm7,%xmm7
+
+#endif
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm7 , 2 * SIZE(CO1)
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BO, BI, 8), BO
+        salq    $2, %rax                        // rax = rax * 4 ; number of values
+        leaq    (AO, %rax, 8), AO
+#endif
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $4, KK
+#endif
+
+	addq	$4 * SIZE, CO1		# coffset += 4
+	ALIGN_4
+	
+
+.L1_30:
+	testq	$2, M		
+	jz	.L1_40
+
+	ALIGN_4
+
+.L1_31:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BO, BI, 8), BO
+        salq    $1, %rax                        // rax = rax * 2 ; number of values
+        leaq    (AO, %rax, 8), AO
+#endif
+
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $2, %rax        // number of values in AO
+#else
+        addq    $1, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$-8, %rax
+	je	.L1_36
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_32:
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL2x1_1(xxx)
+	KERNEL2x1_2(xxx)
+	KERNEL2x1_3(xxx)
+	KERNEL2x1_4(xxx)
+
+	KERNEL2x1_1(xxx)
+	KERNEL2x1_2(xxx)
+	KERNEL2x1_3(xxx)
+	KERNEL2x1_4(xxx)
+
+	je	.L1_36
+
+	KERNEL2x1_1(xxx)
+	KERNEL2x1_2(xxx)
+	KERNEL2x1_3(xxx)
+	KERNEL2x1_4(xxx)
+
+	KERNEL2x1_1(xxx)
+	KERNEL2x1_2(xxx)
+	KERNEL2x1_3(xxx)
+	KERNEL2x1_4(xxx)
+
+	je	.L1_36
+
+	jmp	.L1_32
+	ALIGN_4
+
+.L1_36:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_39
+
+	movq    %rax, BI                        //  Index for BO
+	
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_37:
+
+	KERNEL2x1_SUB(xxx)
+	addq	$1, BI
+	addq	$2, %rax
+	jl	.L1_37
+	ALIGN_4
+
+
+.L1_39:
+
+	vmovddup	ALPHA, %xmm0
+
+#ifndef TRMMKERNEL
+
+	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
+
+#else
+	vmulpd	%xmm0, %xmm4,%xmm4
+
+#endif
+
+	vmovups	%xmm4 ,  	(CO1)
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BO, BI, 8), BO
+        salq    $1, %rax                        // rax = rax * 2 ; number of values
+        leaq    (AO, %rax, 8), AO
+#endif
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $2, KK
+#endif
+	addq	$2 * SIZE, CO1		# coffset += 2
+	ALIGN_4
+	
+
+.L1_40:
+	testq	$1, M		
+	jz	.L999
+
+	ALIGN_4
+
+.L1_41:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BO, BI, 8), BO
+        leaq    (AO, %rax, 8), AO
+#endif
+
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $1, %rax        // number of values in AO
+#else
+        addq    $1, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+	andq	$-8, %rax
+	je	.L1_46
+	movq    %rax, BI                        //  Index for BO
+
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_42:
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL1x1_1(xxx)
+	KERNEL1x1_2(xxx)
+	KERNEL1x1_3(xxx)
+	KERNEL1x1_4(xxx)
+
+	KERNEL1x1_1(xxx)
+	KERNEL1x1_2(xxx)
+	KERNEL1x1_3(xxx)
+	KERNEL1x1_4(xxx)
+
+	je	.L1_46
+
+	prefetcht0	B_PR1(BO,BI,8)
+	KERNEL1x1_1(xxx)
+	KERNEL1x1_2(xxx)
+	KERNEL1x1_3(xxx)
+	KERNEL1x1_4(xxx)
+
+	KERNEL1x1_1(xxx)
+	KERNEL1x1_2(xxx)
+	KERNEL1x1_3(xxx)
+	KERNEL1x1_4(xxx)
+
+	je	.L1_46
+
+	jmp	.L1_42
+	ALIGN_4
+
+.L1_46:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_49
+
+	movq    %rax, BI                        //  Index for BO
+
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_47:
+
+	KERNEL1x1_SUB(xxx)
+	addq	$1, BI
+	addq	$1, %rax
+	jl	.L1_47
+	ALIGN_4
+
+
+.L1_49:
+
+	vmovddup	ALPHA, %xmm0
+
+#ifndef TRMMKERNEL
+
+	vfmaddsd 	 (CO1),%xmm0, %xmm4,%xmm4
+
+#else
+	vmulsd	%xmm0, %xmm4,%xmm4
+
+#endif
+
+	vmovsd	%xmm4 ,  	(CO1)
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BO, BI, 8), BO
+        leaq    (AO, %rax, 8), AO
+#endif
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $1, KK
+#endif
+
+	addq	$1 * SIZE, CO1		# coffset += 1
+	ALIGN_4
+	
+
+.L999:
+	movq   		SP, %rsp
+	movq	   (%rsp), %rbx
+	movq	  8(%rsp), %rbp
+	movq	 16(%rsp), %r12
+	movq	 24(%rsp), %r13
+	movq	 32(%rsp), %r14
+	movq	 40(%rsp), %r15
+
+#ifdef WINDOWS_ABI
+	movq	 48(%rsp), %rdi
+	movq	 56(%rsp), %rsi
+	movups	 64(%rsp), %xmm6
+	movups	 80(%rsp), %xmm7
+	movups	 96(%rsp), %xmm8
+	movups	112(%rsp), %xmm9
+	movups	128(%rsp), %xmm10
+	movups	144(%rsp), %xmm11
+	movups	160(%rsp), %xmm12
+	movups	176(%rsp), %xmm13
+	movups	192(%rsp), %xmm14
+	movups	208(%rsp), %xmm15
+#endif
+
+	addq	$STACKSIZE, %rsp
+	ret
+
+	EPILOGUE
+
+
+
+#endif
diff --git a/kernel/x86_64/dgemm_kernel_8x2_piledriver.S b/kernel/x86_64/dgemm_kernel_8x2_piledriver.S
index adc00cca3..48eb1bcbe 100644
--- a/kernel/x86_64/dgemm_kernel_8x2_piledriver.S
+++ b/kernel/x86_64/dgemm_kernel_8x2_piledriver.S
@@ -1,4523 +1,4523 @@
-/***************************************************************************
-Copyright (c) 2013, The OpenBLAS Project
-All rights reserved.
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in
-the documentation and/or other materials provided with the
-distribution.
-3. Neither the name of the OpenBLAS project nor the names of
-its contributors may be used to endorse or promote products
-derived from this software without specific prior written permission.
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
-USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*****************************************************************************/
-
-
-/*********************************************************************
-*
-* 2013/11/13 Saar
-*        BLASTEST               : OK
-*        CTEST                  : OK
-*        TEST                   : OK
-*
-*
-* 2013/10/31 Saar
-*
-* Parameter:
-* 	UNROLL_M	8
-*	UNROLL_N	2
-*	DGEMM_P		768
-*	DGEMM_Q		168
-*	DGEMM_R		12288
-*	A_PR1		512
-*	B_PR1		256
-*
-* Performance at m x n on AMD 8320 (ACML-Version: 5.3.1):
-* 
-* 4608x4608	83.9	GFLOPS with 8 threads on 4 modules (ACML: 78.4 GFLOPS)
-* 4608x4608	80.9	GFLOPS with 4 threads on 4 modules (ACML: 78.4 GFLOPS)
-* 4608x4608	41.3    GFLOPS with 2 threads on 2 modules (ACML: 40.9 GFLOPS)
-* 4608x4608	20.7	GFLOPS with 1 threads on 1 modules (ACML: 20.8 GFLOPS)
-*
-* Performance at m x n on AMD 6380  (ACML-Version: 5.3.1):
-* 
-* 13824x13824  234.5	GFLOPS with 32 threads on 16 modules (ACML:  88.5 GFLOPS) !strange thermal behavior
-* 13824x13824  241.9	GFLOPS with 16 threads on 16 modules (ACML: 191.5 GFLOPS) !strange thermal behavior
-* 9216x9216    137.6	GFLOPS with  8 threads on  8 modules (ACML: 106.5 GFLOPS)
-* 4608x4608	75.7	GFLOPS with  4 threads on  4 modules (ACML:  56.3 GFLOPS)
-* 4608x4608	38.6	GFLOPS with  2 threads on  2 modules (ACML:  34.1 GFLOPS)
-* 4608x4608	19.6	GFLOPS with  1 threads on  1 modules (ACML:  18.3 GFLOPS)
-*
-*********************************************************************/
-
-
-#define ASSEMBLER
-#include "common.h"
- 
-#define OLD_M	%rdi
-#define OLD_N	%rsi
-#define M	%r13
-#define J	%r14
-#define OLD_K	%rdx
-
-#define A	%rcx
-#define B	%r8
-#define C	%r9
-#define LDC	%r10
-	
-#define I	%r11
-#define AO	%rdi
-#define BO	%rsi
-#define	CO1	%r15
-#define K	%r12
-#define BI	%rbp
-#define	SP	%rbx
-
-#define BO1	%rdi
-#define BO2	%r15
-
-#ifndef WINDOWS_ABI
-
-#define STACKSIZE 96
-
-#else
-
-#define STACKSIZE 256
-
-#define OLD_A		40 + STACKSIZE(%rsp)
-#define OLD_B		48 + STACKSIZE(%rsp)
-#define OLD_C		56 + STACKSIZE(%rsp)
-#define OLD_LDC		64 + STACKSIZE(%rsp)
-#define OLD_OFFSET	72 + STACKSIZE(%rsp)
-
-#endif
-
-#define L_BUFFER_SIZE 8192
-#define LB2_OFFSET    4096
-
-#define Ndiv6	 24(%rsp)
-#define Nmod6	 32(%rsp)
-#define N	 40(%rsp)
-#define ALPHA	 48(%rsp)
-#define OFFSET	 56(%rsp)
-#define KK	 64(%rsp)
-#define KKK	 72(%rsp)
-#define BUFFER1	           128(%rsp)
-#define BUFFER2	LB2_OFFSET+128(%rsp)
-
-#if defined(OS_WINDOWS)
-#if   L_BUFFER_SIZE > 16384
-#define STACK_TOUCH \
-        movl    $0,  4096 * 4(%rsp);\
-        movl    $0,  4096 * 3(%rsp);\
-        movl    $0,  4096 * 2(%rsp);\
-        movl    $0,  4096 * 1(%rsp);
-#elif L_BUFFER_SIZE > 12288
-#define STACK_TOUCH \
-        movl    $0,  4096 * 3(%rsp);\
-        movl    $0,  4096 * 2(%rsp);\
-        movl    $0,  4096 * 1(%rsp);
-#elif L_BUFFER_SIZE > 8192
-#define STACK_TOUCH \
-        movl    $0,  4096 * 2(%rsp);\
-        movl    $0,  4096 * 1(%rsp);
-#elif L_BUFFER_SIZE > 4096
-#define STACK_TOUCH \
-        movl    $0,  4096 * 1(%rsp);
-#else
-#define STACK_TOUCH
-#endif
-#else
-#define STACK_TOUCH
-#endif
-
-#if defined(BULLDOZER)
-
-#define VFMADD231PD_( y1,y2,y0 ) vfmaddpd y0,y1,y2,y0
-
-#define VFMADD231SD_( x1,x2,x0 ) vfmaddsd x0,x1,x2,x0
-
-#else
-
-#define VFMADD231PD_( y1,y2,y0 ) vfmadd231pd y2,y1,y0
-
-#define VFMADD231SD_( x1,x2,x0 ) vfmadd231sd x2,x1,x0
-
-#endif
-
-
-
-
-#define	A_PR1	512
-#define	B_PR1	256
-#define	C_PR1	64
-
-.macro INIT8x3
-	vxorpd	%xmm4 , %xmm4 , %xmm4
-	vxorpd	%xmm5 , %xmm5 , %xmm5
-	vxorpd	%xmm6 , %xmm6 , %xmm6
-	vxorpd	%xmm7 , %xmm7 , %xmm7
-	vxorpd	%xmm8 , %xmm8 , %xmm8
-	vxorpd	%xmm9 , %xmm9 , %xmm9
-	vxorpd	%xmm10, %xmm10, %xmm10
-	vxorpd	%xmm11, %xmm11, %xmm11
-	vxorpd	%xmm12, %xmm12, %xmm12
-	vxorpd	%xmm13, %xmm13, %xmm13
-	vxorpd	%xmm14, %xmm14, %xmm14
-	vxorpd	%xmm15, %xmm15, %xmm15
-.endm
-
-.macro KERNEL8x3_INIT 
-	vmovddup	-12 * SIZE(BO), %xmm1
-	vmovups 	-16 * SIZE(AO), %xmm0
-	prefetcht0	A_PR1(AO)
-	vmulpd  	%xmm1,%xmm0,%xmm4
-	vmovddup	-11 * SIZE(BO), %xmm2
-	vmulpd  	%xmm2,%xmm0,%xmm5
-	vmovddup	-10 * SIZE(BO), %xmm3
-	vmulpd  	%xmm3,%xmm0,%xmm6
-	vmovups 	-14 * SIZE(AO), %xmm0
-	vmulpd  	%xmm1,%xmm0,%xmm7
-	vmulpd  	%xmm2,%xmm0,%xmm8
-	vmulpd  	%xmm3,%xmm0,%xmm9
-	vmovups 	-12 * SIZE(AO), %xmm0
-	vmulpd  	%xmm1,%xmm0,%xmm10
-	vmulpd  	%xmm2,%xmm0,%xmm11
-	addq		$ 3 * SIZE, BO
-	vmulpd  	%xmm3,%xmm0,%xmm12
-	vmovups 	-10 * SIZE(AO), %xmm0
-	vmulpd  	%xmm1,%xmm0,%xmm13
-	vmovddup	-12 * SIZE(BO), %xmm1
-	vmulpd  	%xmm2,%xmm0,%xmm14
-	vmovddup	-11 * SIZE(BO), %xmm2
-	vmulpd  	%xmm3,%xmm0,%xmm15
-.endm
-
-
-.macro KERNEL8x3_M1 
-	vmovups 	-16 * SIZE(AO), %xmm0
-	prefetcht0	A_PR1(AO)
-	VFMADD231PD_(  	%xmm1,%xmm0,%xmm4 )
-	VFMADD231PD_(  	%xmm2,%xmm0,%xmm5 )
-	VFMADD231PD_(  	%xmm3,%xmm0,%xmm6 )
-	vmovups 	-14 * SIZE(AO), %xmm0
-	VFMADD231PD_(  	%xmm1,%xmm0,%xmm7 )
-	VFMADD231PD_(  	%xmm2,%xmm0,%xmm8 )
-	VFMADD231PD_(  	%xmm3,%xmm0,%xmm9 )
-	vmovups 	-12 * SIZE(AO), %xmm0
-	VFMADD231PD_(  	%xmm1,%xmm0,%xmm10 )
-	VFMADD231PD_(  	%xmm2,%xmm0,%xmm11 )
-	VFMADD231PD_(  	%xmm3,%xmm0,%xmm12 )
-	vmovups 	-10 * SIZE(AO), %xmm0
-	VFMADD231PD_(  	%xmm1,%xmm0,%xmm13 )
-	vmovddup	-12 * SIZE(BO), %xmm1
-	VFMADD231PD_(  	%xmm2,%xmm0,%xmm14 )
-	vmovddup	-11 * SIZE(BO), %xmm2
-	VFMADD231PD_(  	%xmm3,%xmm0,%xmm15 )
-.endm
-
-.macro KERNEL8x3_M2 
-	vmovups 	 -8 * SIZE(AO), %xmm0
-	prefetcht0	A_PR1+64(AO)
-	vmovddup	-10 * SIZE(BO), %xmm3
-	VFMADD231PD_(  	%xmm1,%xmm0,%xmm4 )
-	VFMADD231PD_(  	%xmm2,%xmm0,%xmm5 )
-	VFMADD231PD_(  	%xmm3,%xmm0,%xmm6 )
-	vmovups 	 -6 * SIZE(AO), %xmm0
-	VFMADD231PD_(  	%xmm1,%xmm0,%xmm7 )
-	VFMADD231PD_(  	%xmm2,%xmm0,%xmm8 )
-	VFMADD231PD_(  	%xmm3,%xmm0,%xmm9 )
-	vmovups 	 -4 * SIZE(AO), %xmm0
-	VFMADD231PD_(  	%xmm1,%xmm0,%xmm10 )
-	VFMADD231PD_(  	%xmm2,%xmm0,%xmm11 )
-	VFMADD231PD_(  	%xmm3,%xmm0,%xmm12 )
-	vmovups 	 -2 * SIZE(AO), %xmm0
-	VFMADD231PD_(  	%xmm1,%xmm0,%xmm13 )
-	vmovddup	 -9 * SIZE(BO), %xmm1
-	VFMADD231PD_(  	%xmm2,%xmm0,%xmm14 )
-	vmovddup	 -8 * SIZE(BO), %xmm2
-	VFMADD231PD_(  	%xmm3,%xmm0,%xmm15 )
-.endm
-
-
-.macro KERNEL8x3_M3 
-	vmovups 	  0 * SIZE(AO), %xmm0
-	prefetcht0	A_PR1+128(AO)
-	vmovddup	 -7 * SIZE(BO), %xmm3
-	VFMADD231PD_(  	%xmm1,%xmm0,%xmm4 )
-	VFMADD231PD_(  	%xmm2,%xmm0,%xmm5 )
-	VFMADD231PD_(  	%xmm3,%xmm0,%xmm6 )
-	vmovups 	  2 * SIZE(AO), %xmm0
-	VFMADD231PD_(  	%xmm1,%xmm0,%xmm7 )
-	VFMADD231PD_(  	%xmm2,%xmm0,%xmm8 )
-	VFMADD231PD_(  	%xmm3,%xmm0,%xmm9 )
-	vmovups 	  4 * SIZE(AO), %xmm0
-	VFMADD231PD_(  	%xmm1,%xmm0,%xmm10 )
-	VFMADD231PD_(  	%xmm2,%xmm0,%xmm11 )
-	VFMADD231PD_(  	%xmm3,%xmm0,%xmm12 )
-	vmovups 	  6 * SIZE(AO), %xmm0
-	VFMADD231PD_(  	%xmm1,%xmm0,%xmm13 )
-	vmovddup	 -6 * SIZE(BO), %xmm1
-	VFMADD231PD_(  	%xmm2,%xmm0,%xmm14 )
-	vmovddup	 -5 * SIZE(BO), %xmm2
-	VFMADD231PD_(  	%xmm3,%xmm0,%xmm15 )
-.endm
-
-.macro KERNEL8x3_M4 
-	vmovups 	  8 * SIZE(AO), %xmm0
-	prefetcht0	A_PR1+192(AO)
-	vmovddup	 -4 * SIZE(BO), %xmm3
-	VFMADD231PD_(  	%xmm1,%xmm0,%xmm4 )
-	VFMADD231PD_(  	%xmm2,%xmm0,%xmm5 )
-	VFMADD231PD_(  	%xmm3,%xmm0,%xmm6 )
-	vmovups 	 10 * SIZE(AO), %xmm0
-	VFMADD231PD_(  	%xmm1,%xmm0,%xmm7 )
-	VFMADD231PD_(  	%xmm2,%xmm0,%xmm8 )
-	VFMADD231PD_(  	%xmm3,%xmm0,%xmm9 )
-	vmovups 	 12 * SIZE(AO), %xmm0
-	VFMADD231PD_(  	%xmm1,%xmm0,%xmm10 )
-	VFMADD231PD_(  	%xmm2,%xmm0,%xmm11 )
-	VFMADD231PD_(  	%xmm3,%xmm0,%xmm12 )
-	vmovups 	 14 * SIZE(AO), %xmm0
-	VFMADD231PD_(  	%xmm1,%xmm0,%xmm13 )
-	vmovddup	 -3 * SIZE(BO), %xmm1
-	addq		$ 32 * SIZE, AO
-	VFMADD231PD_(  	%xmm2,%xmm0,%xmm14 )
-	vmovddup	 -2 * SIZE(BO), %xmm2
-	VFMADD231PD_(  	%xmm3,%xmm0,%xmm15 )
-.endm
-
-.macro KERNEL8x3_M5 
-	vmovups 	-16 * SIZE(AO), %xmm0
-	prefetcht0	A_PR1(AO)
-	vmovddup	 -1 * SIZE(BO), %xmm3
-	VFMADD231PD_(  	%xmm1,%xmm0,%xmm4 )
-	VFMADD231PD_(  	%xmm2,%xmm0,%xmm5 )
-	VFMADD231PD_(  	%xmm3,%xmm0,%xmm6 )
-	vmovups 	-14 * SIZE(AO), %xmm0
-	VFMADD231PD_(  	%xmm1,%xmm0,%xmm7 )
-	VFMADD231PD_(  	%xmm2,%xmm0,%xmm8 )
-	VFMADD231PD_(  	%xmm3,%xmm0,%xmm9 )
-	vmovups 	-12 * SIZE(AO), %xmm0
-	VFMADD231PD_(  	%xmm1,%xmm0,%xmm10 )
-	VFMADD231PD_(  	%xmm2,%xmm0,%xmm11 )
-	VFMADD231PD_(  	%xmm3,%xmm0,%xmm12 )
-	vmovups 	-10 * SIZE(AO), %xmm0
-	VFMADD231PD_(  	%xmm1,%xmm0,%xmm13 )
-	vmovddup	  0 * SIZE(BO), %xmm1
-	VFMADD231PD_(  	%xmm2,%xmm0,%xmm14 )
-	vmovddup	  1 * SIZE(BO), %xmm2
-	VFMADD231PD_(  	%xmm3,%xmm0,%xmm15 )
-.endm
-
-.macro KERNEL8x3_M6 
-	vmovups 	 -8 * SIZE(AO), %xmm0
-	prefetcht0	A_PR1+64(AO)
-	vmovddup	  2 * SIZE(BO), %xmm3
-	VFMADD231PD_(  	%xmm1,%xmm0,%xmm4 )
-	VFMADD231PD_(  	%xmm2,%xmm0,%xmm5 )
-	VFMADD231PD_(  	%xmm3,%xmm0,%xmm6 )
-	vmovups 	 -6 * SIZE(AO), %xmm0
-	VFMADD231PD_(  	%xmm1,%xmm0,%xmm7 )
-	VFMADD231PD_(  	%xmm2,%xmm0,%xmm8 )
-	VFMADD231PD_(  	%xmm3,%xmm0,%xmm9 )
-	vmovups 	 -4 * SIZE(AO), %xmm0
-	VFMADD231PD_(  	%xmm1,%xmm0,%xmm10 )
-	VFMADD231PD_(  	%xmm2,%xmm0,%xmm11 )
-	VFMADD231PD_(  	%xmm3,%xmm0,%xmm12 )
-	vmovups 	 -2 * SIZE(AO), %xmm0
-	VFMADD231PD_(  	%xmm1,%xmm0,%xmm13 )
-	vmovddup	  3 * SIZE(BO), %xmm1
-	VFMADD231PD_(  	%xmm2,%xmm0,%xmm14 )
-	vmovddup	  4 * SIZE(BO), %xmm2
-	VFMADD231PD_(  	%xmm3,%xmm0,%xmm15 )
-.endm
-
-
-.macro KERNEL8x3_M7 
-	vmovups 	  0 * SIZE(AO), %xmm0
-	prefetcht0	A_PR1+128(AO)
-	vmovddup	  5 * SIZE(BO), %xmm3
-	VFMADD231PD_(  	%xmm1,%xmm0,%xmm4 )
-	VFMADD231PD_(  	%xmm2,%xmm0,%xmm5 )
-	VFMADD231PD_(  	%xmm3,%xmm0,%xmm6 )
-	vmovups 	  2 * SIZE(AO), %xmm0
-	VFMADD231PD_(  	%xmm1,%xmm0,%xmm7 )
-	VFMADD231PD_(  	%xmm2,%xmm0,%xmm8 )
-	VFMADD231PD_(  	%xmm3,%xmm0,%xmm9 )
-	vmovups 	  4 * SIZE(AO), %xmm0
-	VFMADD231PD_(  	%xmm1,%xmm0,%xmm10 )
-	VFMADD231PD_(  	%xmm2,%xmm0,%xmm11 )
-	VFMADD231PD_(  	%xmm3,%xmm0,%xmm12 )
-	vmovups 	  6 * SIZE(AO), %xmm0
-	VFMADD231PD_(  	%xmm1,%xmm0,%xmm13 )
-	vmovddup	  6 * SIZE(BO), %xmm1
-	VFMADD231PD_(  	%xmm2,%xmm0,%xmm14 )
-	vmovddup	  7 * SIZE(BO), %xmm2
-	VFMADD231PD_(  	%xmm3,%xmm0,%xmm15 )
-.endm
-
-.macro KERNEL8x3_M8 
-	vmovups 	  8 * SIZE(AO), %xmm0
-	prefetcht0	A_PR1+192(AO)
-	vmovddup	  8 * SIZE(BO), %xmm3
-	VFMADD231PD_(  	%xmm1,%xmm0,%xmm4 )
-	VFMADD231PD_(  	%xmm2,%xmm0,%xmm5 )
-	VFMADD231PD_(  	%xmm3,%xmm0,%xmm6 )
-	vmovups 	 10 * SIZE(AO), %xmm0
-	VFMADD231PD_(  	%xmm1,%xmm0,%xmm7 )
-	VFMADD231PD_(  	%xmm2,%xmm0,%xmm8 )
-	VFMADD231PD_(  	%xmm3,%xmm0,%xmm9 )
-	vmovups 	 12 * SIZE(AO), %xmm0
-	VFMADD231PD_(  	%xmm1,%xmm0,%xmm10 )
-	VFMADD231PD_(  	%xmm2,%xmm0,%xmm11 )
-	VFMADD231PD_(  	%xmm3,%xmm0,%xmm12 )
-	vmovups 	 14 * SIZE(AO), %xmm0
-	VFMADD231PD_(  	%xmm1,%xmm0,%xmm13 )
-	vmovddup	  9 * SIZE(BO), %xmm1
-	VFMADD231PD_(  	%xmm2,%xmm0,%xmm14 )
-	vmovddup	 10 * SIZE(BO), %xmm2
-	VFMADD231PD_(  	%xmm3,%xmm0,%xmm15 )
-	vmovddup	 11 * SIZE(BO), %xmm3
-	addq		$ 32 * SIZE, AO
-	addq		$ 24 * SIZE, BO
-.endm
-
-
-.macro KERNEL8x3_E 
-	vmovups 	  8 * SIZE(AO), %xmm0
-	prefetcht0	A_PR1+192(AO)
-	vmovddup	  8 * SIZE(BO), %xmm3
-	VFMADD231PD_(  	%xmm1,%xmm0,%xmm4 )
-	VFMADD231PD_(  	%xmm2,%xmm0,%xmm5 )
-	VFMADD231PD_(  	%xmm3,%xmm0,%xmm6 )
-	vmovups 	 10 * SIZE(AO), %xmm0
-	VFMADD231PD_(  	%xmm1,%xmm0,%xmm7 )
-	VFMADD231PD_(  	%xmm2,%xmm0,%xmm8 )
-	VFMADD231PD_(  	%xmm3,%xmm0,%xmm9 )
-	vmovups 	 12 * SIZE(AO), %xmm0
-	VFMADD231PD_(  	%xmm1,%xmm0,%xmm10 )
-	VFMADD231PD_(  	%xmm2,%xmm0,%xmm11 )
-	VFMADD231PD_(  	%xmm3,%xmm0,%xmm12 )
-	vmovups 	 14 * SIZE(AO), %xmm0
-	VFMADD231PD_(  	%xmm1,%xmm0,%xmm13 )
-	addq		$ 32 * SIZE, AO
-	VFMADD231PD_(  	%xmm2,%xmm0,%xmm14 )
-	addq		$ 21 * SIZE, BO
-	VFMADD231PD_(  	%xmm3,%xmm0,%xmm15 )
-.endm
-
-.macro KERNEL8x3_SUBN 
-	vmovddup	-12 * SIZE(BO), %xmm1
-	vmovups 	-16 * SIZE(AO), %xmm0
-	VFMADD231PD_(  	%xmm1,%xmm0,%xmm4 )
-	vmovddup	-11 * SIZE(BO), %xmm2
-	VFMADD231PD_(  	%xmm2,%xmm0,%xmm5 )
-	vmovddup	-10 * SIZE(BO), %xmm3
-	VFMADD231PD_(  	%xmm3,%xmm0,%xmm6 )
-	vmovups 	-14 * SIZE(AO), %xmm0
-	VFMADD231PD_(  	%xmm1,%xmm0,%xmm7 )
-	VFMADD231PD_(  	%xmm2,%xmm0,%xmm8 )
-	VFMADD231PD_(  	%xmm3,%xmm0,%xmm9 )
-	vmovups 	-12 * SIZE(AO), %xmm0
-	VFMADD231PD_(  	%xmm1,%xmm0,%xmm10 )
-	VFMADD231PD_(  	%xmm2,%xmm0,%xmm11 )
-	VFMADD231PD_(  	%xmm3,%xmm0,%xmm12 )
-	vmovups 	-10 * SIZE(AO), %xmm0
-	VFMADD231PD_(  	%xmm1,%xmm0,%xmm13 )
-	addq		$ 3 * SIZE, BO
-	VFMADD231PD_(  	%xmm2,%xmm0,%xmm14 )
-	addq		$ 8 * SIZE, AO
-	VFMADD231PD_(  	%xmm3,%xmm0,%xmm15 )
-.endm
-
-.macro	SAVE8x3
-	vmovddup	ALPHA, %xmm0
-
-
-	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddpd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-	vfmaddpd 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
-
-	vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7
-	vfmaddpd 4 * SIZE(CO1),%xmm0, %xmm10,%xmm10
-	vfmaddpd 6 * SIZE(CO1),%xmm0, %xmm13,%xmm13
-
-	vfmaddpd 2 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
-	vfmaddpd 4 * SIZE(CO1, LDC),%xmm0, %xmm11,%xmm11
-	vfmaddpd 6 * SIZE(CO1, LDC),%xmm0, %xmm14,%xmm14
-
-	vfmaddpd 2 * SIZE(CO1, LDC, 2),%xmm0, %xmm9,%xmm9
-	vfmaddpd 4 * SIZE(CO1, LDC, 2),%xmm0, %xmm12,%xmm12
-	vfmaddpd 6 * SIZE(CO1, LDC, 2),%xmm0, %xmm15,%xmm15
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm7 , 2 * SIZE(CO1)
-	vmovups	%xmm10, 4 * SIZE(CO1)
-	vmovups	%xmm13, 6 * SIZE(CO1)
-
-	vmovups	%xmm5 ,  	(CO1, LDC)
-	vmovups	%xmm8 , 2 * SIZE(CO1, LDC)
-	vmovups	%xmm11, 4 * SIZE(CO1, LDC)
-	vmovups	%xmm14, 6 * SIZE(CO1, LDC)
-
-	vmovups	%xmm6 ,  	(CO1, LDC, 2)
-	vmovups	%xmm9 , 2 * SIZE(CO1, LDC, 2)
-	vmovups	%xmm12, 4 * SIZE(CO1, LDC, 2)
-	vmovups	%xmm15, 6 * SIZE(CO1, LDC, 2)
-
-	prefetcht0	 C_PR1(CO1)
-	prefetcht0	 C_PR1(CO1,LDC)
-	prefetcht0	 C_PR1(CO1,LDC,2)
-
-	addq	$ 8 * SIZE, CO1		# coffset += 8
-.endm
-
-
-/*******************************************************************************************/
-
-#define KERNEL4x3_1(xx) \
-	vmovddup	 -6 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovddup	 -5 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovddup	 -4 * SIZE(BO, BI, 8), %xmm3 ;\
-	vfmaddpd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-	vmovups 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	vfmaddpd  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
-
-#define KERNEL4x3_2(xx) \
-	vmovddup	 -3 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	-12 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovddup	 -2 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovddup	 -1 * SIZE(BO, BI, 8), %xmm3 ;\
-	vfmaddpd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-	vmovups 	-10 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	vfmaddpd  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
-
-#define KERNEL4x3_3(xx) \
-	vmovddup	  0 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	 -8 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovddup	  1 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovddup	  2 * SIZE(BO, BI, 8), %xmm3 ;\
-	vfmaddpd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-	vmovups 	 -6 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	vfmaddpd  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
-
-#define KERNEL4x3_4(xx) \
-	vmovddup	  3 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	 -4 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovddup	  4 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovddup	  5 * SIZE(BO, BI, 8), %xmm3 ;\
-	vfmaddpd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-	vmovups 	 -2 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	vfmaddpd  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
-	addq	$12, BI				  ;\
-	addq	$16, %rax 			  ;\
-
-#define KERNEL4x3_SUB(xx) \
-	vmovddup	 -6 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovddup	 -5 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovddup	 -4 * SIZE(BO, BI, 8), %xmm3 ;\
-	vfmaddpd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-	vmovups 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	vfmaddpd  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
-
-
-
-
-
-/*******************************************************************************************/
-
-#define KERNEL2x3_1(xx) \
-	vmovddup	 -6 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovddup	 -5 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovddup	 -4 * SIZE(BO, BI, 8), %xmm3 ;\
-	vfmaddpd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-
-#define KERNEL2x3_2(xx) \
-	vmovddup	 -3 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovddup	 -2 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovddup	 -1 * SIZE(BO, BI, 8), %xmm3 ;\
-	vfmaddpd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-
-#define KERNEL2x3_3(xx) \
-	vmovddup	  0 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	-12 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovddup	  1 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovddup	  2 * SIZE(BO, BI, 8), %xmm3 ;\
-	vfmaddpd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-
-#define KERNEL2x3_4(xx) \
-	vmovddup	  3 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	-10 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovddup	  4 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovddup	  5 * SIZE(BO, BI, 8), %xmm3 ;\
-	vfmaddpd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-	addq	$12, BI				  ;\
-	addq	$8, %rax 			  ;\
-
-#define KERNEL2x3_SUB(xx) \
-	vmovddup	 -6 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovddup	 -5 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovddup	 -4 * SIZE(BO, BI, 8), %xmm3 ;\
-	vfmaddpd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-
-/*******************************************************************************************/
-
-#define KERNEL1x3_1(xx) \
-	vmovsd	 -6 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovsd 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovsd	 -5 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddsd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovsd	 -4 * SIZE(BO, BI, 8), %xmm3 ;\
-	vfmaddsd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-
-#define KERNEL1x3_2(xx) \
-	vmovsd	 -3 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovsd 	-15 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovsd	 -2 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddsd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovsd	 -1 * SIZE(BO, BI, 8), %xmm3 ;\
-	vfmaddsd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-
-#define KERNEL1x3_3(xx) \
-	vmovsd	  0 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovsd 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovsd	  1 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddsd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovsd	  2 * SIZE(BO, BI, 8), %xmm3 ;\
-	vfmaddsd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-
-#define KERNEL1x3_4(xx) \
-	vmovsd	  3 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovsd 	-13 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovsd	  4 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddsd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovsd	  5 * SIZE(BO, BI, 8), %xmm3 ;\
-	vfmaddsd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-	addq	$12, BI				  ;\
-	addq	$4, %rax 			  ;\
-
-#define KERNEL1x3_SUB(xx) \
-	vmovsd	 -6 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovsd 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovsd	 -5 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddsd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovsd	 -4 * SIZE(BO, BI, 8), %xmm3 ;\
-	vfmaddsd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-
-
-
-/*******************************************************************************************
-* 2 lines of N
-*******************************************************************************************/
-
-#define KERNEL8x2_1(xx) \
-	prefetcht0	A_PR1(AO,%rax,8)	;\
-	vmovddup	 -4 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovddup	 -3 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovups 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	vmovups 	-12 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
-	vfmaddpd  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
-	vmovups 	-10 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
-	vfmaddpd  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
-
-#define KERNEL8x2_2(xx) \
-	prefetcht0	A_PR1+64(AO,%rax,8)	;\
-	vmovddup	 -2 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	 -8 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovddup	 -1 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovups 	 -6 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	vmovups 	 -4 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
-	vfmaddpd  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
-	vmovups 	 -2 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
-	vfmaddpd  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
-
-#define KERNEL8x2_3(xx) \
-	prefetcht0	A_PR1+128(AO,%rax,8)	;\
-	vmovddup	  0 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	  0 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovddup	  1 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovups 	  2 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	vmovups 	  4 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
-	vfmaddpd  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
-	vmovups 	  6 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
-	vfmaddpd  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
-
-#define KERNEL8x2_4(xx) \
-	prefetcht0	A_PR1+192(AO,%rax,8)	;\
-	vmovddup	  2 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	  8 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovddup	  3 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovups 	 10 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	vmovups 	 12 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
-	vfmaddpd  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
-	vmovups 	 14 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
-	vfmaddpd  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
-	addq	$8, BI				  ;\
-	addq	$32, %rax 			  ;\
-
-#define KERNEL8x2_SUB(xx) \
-	vmovddup	 -4 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovddup	 -3 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovups 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	vmovups 	-12 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
-	vfmaddpd  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
-	vmovups 	-10 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
-	vfmaddpd  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
-
-
-/*******************************************************************************************/
-
-#define KERNEL4x2_1(xx) \
-	vmovddup	 -4 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovddup	 -3 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovups 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-
-#define KERNEL4x2_2(xx) \
-	vmovddup	 -2 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	-12 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovddup	 -1 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovups 	-10 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-
-#define KERNEL4x2_3(xx) \
-	vmovddup	  0 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	 -8 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovddup	  1 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovups 	 -6 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-
-#define KERNEL4x2_4(xx) \
-	vmovddup	  2 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	 -4 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovddup	  3 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovups 	 -2 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	addq	$8, BI				  ;\
-	addq	$16, %rax 			  ;\
-
-#define KERNEL4x2_SUB(xx) \
-	vmovddup	 -4 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovddup	 -3 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovups 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-
-
-/*******************************************************************************************/
-
-#define KERNEL2x2_1(xx) \
-	vmovddup	 -4 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovddup	 -3 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-
-#define KERNEL2x2_2(xx) \
-	vmovddup	 -2 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovddup	 -1 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-
-#define KERNEL2x2_3(xx) \
-	vmovddup	  0 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	-12 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovddup	  1 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-
-#define KERNEL2x2_4(xx) \
-	vmovddup	  2 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	-10 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovddup	  3 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	addq	$8, BI				  ;\
-	addq	$8, %rax 			  ;\
-
-#define KERNEL2x2_SUB(xx) \
-	vmovddup	 -4 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovddup	 -3 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-
-/*******************************************************************************************/
-
-#define KERNEL1x2_1(xx) \
-	vmovsd	 -4 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovsd 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovsd	 -3 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddsd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-
-#define KERNEL1x2_2(xx) \
-	vmovsd	 -2 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovsd 	-15 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovsd	 -1 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddsd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-
-#define KERNEL1x2_3(xx) \
-	vmovsd	  0 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovsd 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovsd	  1 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddsd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-
-#define KERNEL1x2_4(xx) \
-	vmovsd	  2 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovsd 	-13 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovsd	  3 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddsd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	addq	$8, BI				  ;\
-	addq	$4, %rax 			  ;\
-
-#define KERNEL1x2_SUB(xx) \
-	vmovsd	 -4 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovsd 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovsd	 -3 * SIZE(BO, BI, 8), %xmm2 ;\
-	vfmaddsd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-
-
-
-/*******************************************************************************************
-* 1 line of N
-*******************************************************************************************/
-
-#define KERNEL8x1_1(xx) \
-	prefetcht0	A_PR1(AO,%rax,8)	;\
-	vmovddup	 -2 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovups 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vmovups 	-12 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
-	vmovups 	-10 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
-
-#define KERNEL8x1_2(xx) \
-	prefetcht0	A_PR1+64(AO,%rax,8)	;\
-	vmovddup	 -1 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	 -8 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovups 	 -6 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vmovups 	 -4 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
-	vmovups 	 -2 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
-
-#define KERNEL8x1_3(xx) \
-	prefetcht0	A_PR1+128(AO,%rax,8)	;\
-	vmovddup	  0 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	  0 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovups 	  2 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vmovups 	  4 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
-	vmovups 	  6 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
-
-#define KERNEL8x1_4(xx) \
-	prefetcht0	A_PR1+192(AO,%rax,8)	;\
-	vmovddup	  1 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	  8 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovups 	 10 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vmovups 	 12 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
-	vmovups 	 14 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
-	addq	$4, BI				  ;\
-	addq	$32, %rax 			  ;\
-
-#define KERNEL8x1_SUB(xx) \
-	vmovddup	 -2 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovups 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vmovups 	-12 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
-	vmovups 	-10 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
-
-
-/*******************************************************************************************/
-
-#define KERNEL4x1_1(xx) \
-	vmovddup	 -2 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovups 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-
-#define KERNEL4x1_2(xx) \
-	vmovddup	 -1 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	-12 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovups 	-10 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-
-#define KERNEL4x1_3(xx) \
-	vmovddup	  0 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	 -8 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovups 	 -6 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-
-#define KERNEL4x1_4(xx) \
-	vmovddup	  1 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	 -4 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovups 	 -2 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	addq	$4, BI				  ;\
-	addq	$16, %rax 			  ;\
-
-#define KERNEL4x1_SUB(xx) \
-	vmovddup	 -2 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovups 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-
-
-/*******************************************************************************************/
-
-#define KERNEL2x1_1(xx) \
-	vmovddup	 -2 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-
-#define KERNEL2x1_2(xx) \
-	vmovddup	 -1 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-
-#define KERNEL2x1_3(xx) \
-	vmovddup	  0 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	-12 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-
-#define KERNEL2x1_4(xx) \
-	vmovddup	  1 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	-10 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	addq	$4, BI				  ;\
-	addq	$8, %rax 			  ;\
-
-#define KERNEL2x1_SUB(xx) \
-	vmovddup	 -2 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-
-/*******************************************************************************************/
-
-#define KERNEL1x1_1(xx) \
-	vmovsd	 -2 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovsd 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-
-#define KERNEL1x1_2(xx) \
-	vmovsd	 -1 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovsd 	-15 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-
-#define KERNEL1x1_3(xx) \
-	vmovsd	  0 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovsd 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-
-#define KERNEL1x1_4(xx) \
-	vmovsd	  1 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovsd 	-13 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	addq	$4, BI				  ;\
-	addq	$4, %rax 			  ;\
-
-#define KERNEL1x1_SUB(xx) \
-	vmovsd	 -2 * SIZE(BO, BI, 8), %xmm1 ;\
-	vmovsd 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
-	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-
-
-/*******************************************************************************************/
-
-#if !defined(TRMMKERNEL)
-
-
-	PROLOGUE
-	PROFCODE
-	
-	subq	$STACKSIZE, %rsp
-	movq	%rbx,   (%rsp)
-	movq	%rbp,  8(%rsp)
-	movq	%r12, 16(%rsp)
-	movq	%r13, 24(%rsp)
-	movq	%r14, 32(%rsp)
-	movq	%r15, 40(%rsp)
-
-	vzeroupper
-
-#ifdef WINDOWS_ABI
-	movq	%rdi,    48(%rsp)
-	movq	%rsi,    56(%rsp)
-	movups	%xmm6,   64(%rsp)
-	movups	%xmm7,   80(%rsp)
-	movups	%xmm8,   96(%rsp)
-	movups	%xmm9,  112(%rsp)
-	movups	%xmm10, 128(%rsp)
-	movups	%xmm11, 144(%rsp)
-	movups	%xmm12, 160(%rsp)
-	movups	%xmm13, 176(%rsp)
-	movups	%xmm14, 192(%rsp)
-	movups	%xmm15, 208(%rsp)
-
-	movq	ARG1,      OLD_M
-	movq	ARG2,      OLD_N
-	movq	ARG3,      OLD_K
-	movq	OLD_A,     A
-	movq	OLD_B,     B
-	movq	OLD_C,     C
-	movq	OLD_LDC,   LDC
-
-	vmovaps	%xmm3, %xmm0
-
-#else
-	movq	STACKSIZE +  8(%rsp), LDC
-
-#endif
-
-	movq    %rsp, SP      # save old stack
-        subq    $128 + L_BUFFER_SIZE, %rsp
-        andq    $-4096, %rsp    # align stack
-
-        STACK_TOUCH
-
-	cmpq	$0, OLD_M
-	je	.L999
-
-	cmpq	$0, OLD_N
-	je	.L999
-
-	cmpq	$0, OLD_K
-	je	.L999
-
-	movq	OLD_M, M
-	movq	OLD_N, N
-	movq	OLD_K, K
-
-	vmovsd	 %xmm0, ALPHA
-
-	salq	$BASE_SHIFT, LDC
-
-	movq    N, %rax
-        xorq    %rdx, %rdx
-        movq    $6,  %rdi
-        divq    %rdi                    //    N / 6
-        movq    %rax, Ndiv6             //    N / 6
-        movq    %rdx, Nmod6             //    N % 6
-
-	movq	Ndiv6,  J
-	cmpq	$0, J
-	je	.L2_0
-	ALIGN_4
-
-.L6_01:
-        // copy to sub buffer
-        movq    K, %rax
-        salq    $1,%rax                 // K * 2
-        movq    B, BO1
-        leaq    (B,%rax,8), BO2         // next offset to BO2
-        leaq    BUFFER1, BO             // first buffer to BO
-        movq    K, %rax
-        sarq    $2, %rax                // K / 4
-        jz      .L6_02a
-        ALIGN_4
-
-.L6_02:
-	prefetcht0 B_PR1(BO1)
-	prefetcht0 B_PR1(BO2)
-	prefetchw  B_PR1(BO)
-	vmovups	      (BO1), %xmm0
-	vmovups	2*SIZE(BO1), %xmm2
-	vmovups	4*SIZE(BO1), %xmm4
-	vmovups	6*SIZE(BO1), %xmm6
-	vmovsd        (BO2), %xmm1
-	vmovsd  2*SIZE(BO2), %xmm3
-	vmovsd  4*SIZE(BO2), %xmm5
-	vmovsd  6*SIZE(BO2), %xmm7
-	vmovups	%xmm0,       (BO)
-	vmovsd	%xmm1, 2*SIZE(BO)
-	vmovups	%xmm2, 3*SIZE(BO)
-	vmovsd	%xmm3, 5*SIZE(BO)
-	vmovups	%xmm4, 6*SIZE(BO)
-	vmovsd	%xmm5, 8*SIZE(BO)
-	vmovups	%xmm6, 9*SIZE(BO)
-	vmovsd	%xmm7,11*SIZE(BO)
-	addq	$ 8*SIZE,BO1
-	addq	$ 8*SIZE,BO2
-	addq	$ 12*SIZE,BO
-	decq	%rax
-	jnz	.L6_02
-
-.L6_02a:
-
-	movq	K, %rax
-	andq	$3, %rax		// K % 4
-	jz	.L6_02c
-	ALIGN_4
-
-.L6_02b:
-
-	vmovups	(BO1), %xmm0
-	vmovsd  (BO2), %xmm1
-	vmovups	%xmm0,       (BO)
-	vmovsd	%xmm1, 2*SIZE(BO)
-	addq	$ 2*SIZE,BO1
-	addq	$ 2*SIZE,BO2
-	addq	$ 3*SIZE,BO
-	decq	%rax
-	jnz	.L6_02b
-
-.L6_02c:
-
-	movq	K, %rax
-	salq	$1,%rax			// K * 2
-	leaq	(B,%rax,8), BO1		// next offset to BO1
-	leaq	(BO1,%rax,8), BO2	// next offset to BO1
-	leaq    BUFFER2, BO		// second buffer to BO
-	movq	K, %rax
-	sarq	$2, %rax		// k / 4
-	jz	.L6_03a
-	ALIGN_4
-
-
-.L6_03:
-
-	prefetcht0 B_PR1(BO2)
-	prefetchw  B_PR1(BO)
-	vmovups	      (BO2), %xmm0
-	vmovups	2*SIZE(BO2), %xmm2
-	vmovups	4*SIZE(BO2), %xmm4
-	vmovups	6*SIZE(BO2), %xmm6
-	vmovsd  1*SIZE(BO1), %xmm1
-	vmovsd  3*SIZE(BO1), %xmm3
-	vmovsd  5*SIZE(BO1), %xmm5
-	vmovsd  7*SIZE(BO1), %xmm7
-	vmovsd	%xmm1, 0*SIZE(BO)
-	vmovups	%xmm0, 1*SIZE(BO)
-	vmovsd	%xmm3, 3*SIZE(BO)
-	vmovups	%xmm2, 4*SIZE(BO)
-	vmovsd	%xmm5, 6*SIZE(BO)
-	vmovups	%xmm4, 7*SIZE(BO)
-	vmovsd	%xmm7, 9*SIZE(BO)
-	vmovups	%xmm6,10*SIZE(BO)
-	addq	$ 8*SIZE,BO1
-	addq	$ 8*SIZE,BO2
-	addq	$ 12*SIZE,BO
-	decq	%rax
-	jnz	.L6_03
-
-.L6_03a:
-
-        movq    K, %rax
-        andq    $3, %rax                // K % 4
-        jz      .L6_03c
-        ALIGN_4
-
-
-.L6_03b:
-
-	vmovsd	  1*SIZE(BO1), %xmm0
-	vmovups  	(BO2), %xmm1
-	vmovsd	%xmm0,       (BO)
-	vmovups %xmm1, 1*SIZE(BO)
-	addq	$ 2*SIZE,BO1
-	addq	$ 2*SIZE,BO2
-	addq	$ 3*SIZE,BO
-	decq	%rax
-	jnz	.L6_03b
-
-
-.L6_03c:
-
-	movq	BO2, B			// next offset of B
-
-.L6_10:
-	movq	C, CO1
-	leaq	(C, LDC, 2), C		 
-	leaq	(C, LDC, 1), C		// c += 3 * ldc
-
-	
-	movq	A, AO		 	// aoffset = a
-	addq	$16 * SIZE, AO
-
-	movq	M,  I
-	sarq	$3, I			// i = (m >> 3)
-	je	.L6_20
-
-	ALIGN_4
-
-.L6_11:
-
-        leaq    BUFFER1, BO             // first buffer to BO
-	addq	$12 * SIZE, BO
-        movq    K, %rax
-	sarq	$3, %rax			//  K / 8
-	cmpq	$3, %rax
-	jl	.L6_13
-
-	prefetcht0 B_PR1(BO)
-	prefetcht0 B_PR1+64(BO)
-	prefetcht0 B_PR1+128(BO)
-	KERNEL8x3_INIT
-	KERNEL8x3_M2
-	KERNEL8x3_M3
-	KERNEL8x3_M4
-	KERNEL8x3_M5
-	KERNEL8x3_M6
-	KERNEL8x3_M7
-	KERNEL8x3_M8
-
-	subq	$2, %rax
-
-	ALIGN_5
-
-.L6_12:
-
-	prefetcht0 B_PR1-24(BO)
-	prefetcht0 B_PR1+40(BO)
-	KERNEL8x3_M1
-	KERNEL8x3_M2
-	KERNEL8x3_M3
-	KERNEL8x3_M4
-	KERNEL8x3_M5
-	prefetcht0 B_PR1+104(BO)
-	KERNEL8x3_M6
-	KERNEL8x3_M7
-	KERNEL8x3_M8
-
-	dec	%rax
-	jne	.L6_12
-
-.L6_12_E:
-
-	prefetcht0 B_PR1(BO)
-	prefetcht0 B_PR1+64(BO)
-	KERNEL8x3_M1
-	KERNEL8x3_M2
-	KERNEL8x3_M3
-	KERNEL8x3_M4
-	KERNEL8x3_M5
-	KERNEL8x3_M6
-	KERNEL8x3_M7
-	KERNEL8x3_E
-
-	jmp	.L6_16
-
-.L6_13:
-
-	test $2, %rax
-	jz	.L6_14
-
-	KERNEL8x3_INIT
-	KERNEL8x3_M2
-	KERNEL8x3_M3
-	KERNEL8x3_M4
-	KERNEL8x3_M5
-	KERNEL8x3_M6
-	KERNEL8x3_M7
-	KERNEL8x3_M8
-
-	KERNEL8x3_M1
-	KERNEL8x3_M2
-	KERNEL8x3_M3
-	KERNEL8x3_M4
-	KERNEL8x3_M5
-	KERNEL8x3_M6
-	KERNEL8x3_M7
-	KERNEL8x3_E
-
-	jmp	.L6_16
-
-
-.L6_14:
-
-	test $1, %rax
-	jz	.L6_15
-	
-	KERNEL8x3_INIT
-	KERNEL8x3_M2
-	KERNEL8x3_M3
-	KERNEL8x3_M4
-	KERNEL8x3_M5
-	KERNEL8x3_M6
-	KERNEL8x3_M7
-	KERNEL8x3_E
-
-
-	jmp	.L6_16
-
-.L6_15:
-
-	INIT8x3
-
-.L6_16:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L6_19
-
-	ALIGN_4
-
-.L6_17:
-
-	KERNEL8x3_SUBN
-	dec	%rax
-	jne	.L6_17
-	ALIGN_4
-
-
-.L6_19:
-
-	SAVE8x3
-
-	decq	I			# i --
-	jg	.L6_11
-
-/**************************************************************************
-* Rest of M 
-***************************************************************************/
-.L6_20:
-	// Test rest of M
-
-	testq	$7, M
-	jz	.L7_10		// to next 3 lines of N
-
-	testq	$4, M		
-	jz	.L6_30
-
-	ALIGN_4
-
-.L6_21:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $6 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L6_26
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L6_22:
-
-	KERNEL4x3_1(xxx)
-	KERNEL4x3_2(xxx)
-	KERNEL4x3_3(xxx)
-	KERNEL4x3_4(xxx)
-
-	KERNEL4x3_1(xxx)
-	KERNEL4x3_2(xxx)
-	KERNEL4x3_3(xxx)
-	KERNEL4x3_4(xxx)
-
-	je	.L6_26
-
-	KERNEL4x3_1(xxx)
-	KERNEL4x3_2(xxx)
-	KERNEL4x3_3(xxx)
-	KERNEL4x3_4(xxx)
-
-	KERNEL4x3_1(xxx)
-	KERNEL4x3_2(xxx)
-	KERNEL4x3_3(xxx)
-	KERNEL4x3_4(xxx)
-
-	je	.L6_26
-
-	jmp	.L6_22
-	ALIGN_4
-
-.L6_26:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L6_29
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L6_27:
-
-	KERNEL4x3_SUB(xxx)
-	addq	$3, BI
-	addq	$4, %rax
-	jl	.L6_27
-	ALIGN_4
-
-
-.L6_29:
-
-	vmovddup	ALPHA, %xmm0
-
-
-	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7
-
-	vfmaddpd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-	vfmaddpd 2 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
-
-	vfmaddpd 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
-	vfmaddpd 2 * SIZE(CO1, LDC, 2),%xmm0, %xmm9,%xmm9
-
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm7 , 2 * SIZE(CO1)
-
-	vmovups	%xmm5 ,  	(CO1, LDC)
-	vmovups	%xmm8 , 2 * SIZE(CO1, LDC)
-
-	vmovups	%xmm6 ,  	(CO1, LDC, 2)
-	vmovups	%xmm9 , 2 * SIZE(CO1, LDC, 2)
-
-
-	addq	$4 * SIZE, CO1		# coffset += 4
-	ALIGN_4
-	
-
-.L6_30:
-	testq	$2, M		
-	jz	.L6_40
-
-	ALIGN_4
-
-.L6_31:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $6 * SIZE, BO
-
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L6_36
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L6_32:
-
-	KERNEL2x3_1(xxx)
-	KERNEL2x3_2(xxx)
-	KERNEL2x3_3(xxx)
-	KERNEL2x3_4(xxx)
-
-	KERNEL2x3_1(xxx)
-	KERNEL2x3_2(xxx)
-	KERNEL2x3_3(xxx)
-	KERNEL2x3_4(xxx)
-
-	je	.L6_36
-
-	KERNEL2x3_1(xxx)
-	KERNEL2x3_2(xxx)
-	KERNEL2x3_3(xxx)
-	KERNEL2x3_4(xxx)
-
-	KERNEL2x3_1(xxx)
-	KERNEL2x3_2(xxx)
-	KERNEL2x3_3(xxx)
-	KERNEL2x3_4(xxx)
-
-	je	.L6_36
-
-	jmp	.L6_32
-	ALIGN_4
-
-.L6_36:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L6_39
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-	
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L6_37:
-
-	KERNEL2x3_SUB(xxx)
-	addq	$3, BI
-	addq	$2, %rax
-	jl	.L6_37
-	ALIGN_4
-
-
-.L6_39:
-
-	vmovddup	ALPHA, %xmm0
-
-
-	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddpd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-	vfmaddpd 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
-
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm5 ,  	(CO1, LDC)
-	vmovups	%xmm6 ,  	(CO1, LDC, 2)
-
-
-	addq	$2 * SIZE, CO1		# coffset += 2
-	ALIGN_4
-
-.L6_40:
-	testq	$1, M		
-	jz	.L7_10		// to next 3 lines of N
-
-	ALIGN_4
-
-.L6_41:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $6 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L6_46
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L6_42:
-
-	KERNEL1x3_1(xxx)
-	KERNEL1x3_2(xxx)
-	KERNEL1x3_3(xxx)
-	KERNEL1x3_4(xxx)
-
-	KERNEL1x3_1(xxx)
-	KERNEL1x3_2(xxx)
-	KERNEL1x3_3(xxx)
-	KERNEL1x3_4(xxx)
-
-	je	.L6_46
-
-	KERNEL1x3_1(xxx)
-	KERNEL1x3_2(xxx)
-	KERNEL1x3_3(xxx)
-	KERNEL1x3_4(xxx)
-
-	KERNEL1x3_1(xxx)
-	KERNEL1x3_2(xxx)
-	KERNEL1x3_3(xxx)
-	KERNEL1x3_4(xxx)
-
-	je	.L6_46
-
-	jmp	.L6_42
-	ALIGN_4
-
-.L6_46:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L6_49
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L6_47:
-
-	KERNEL1x3_SUB(xxx)
-	addq	$3, BI
-	addq	$1, %rax
-	jl	.L6_47
-	ALIGN_4
-
-
-.L6_49:
-
-	vmovddup	ALPHA, %xmm0
-
-
-	vfmaddsd 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddsd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-	vfmaddsd 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
-
-	vmovsd	%xmm4 ,  	(CO1)
-	vmovsd	%xmm5 ,  	(CO1, LDC)
-	vmovsd	%xmm6 ,  	(CO1, LDC, 2)
-
-
-	addq	$1 * SIZE, CO1		# coffset += 1
-	ALIGN_4
-	
-
-
-
-/***************************************************************************************************************/
-
-.L7_10:
-	movq	C, CO1
-	leaq	(C, LDC, 2), C		 
-	leaq	(C, LDC, 1), C		// c += 3 * ldc
-
-
-	movq	A, AO		 	// aoffset = a
-	addq	$16 * SIZE, AO
-
-	movq	M,  I
-	sarq	$3, I			// i = (m >> 3)
-	je	.L7_20
-	ALIGN_4
-
-.L7_11:
-
-        leaq    BUFFER2, BO             // first buffer to BO
-	addq	$12 * SIZE, BO
-        movq    K, %rax
-	sarq	$3, %rax			//  K / 8
-	cmpq	$3, %rax
-	jl	.L7_13
-
-	prefetcht0 B_PR1(BO)
-	prefetcht0 B_PR1+64(BO)
-	prefetcht0 B_PR1+128(BO)
-	KERNEL8x3_INIT
-	KERNEL8x3_M2
-	KERNEL8x3_M3
-	KERNEL8x3_M4
-	KERNEL8x3_M5
-	KERNEL8x3_M6
-	KERNEL8x3_M7
-	KERNEL8x3_M8
-
-	subq	$2, %rax
-
-	ALIGN_5
-
-.L7_12:
-
-	prefetcht0 B_PR1-24(BO)
-	prefetcht0 B_PR1+40(BO)
-	KERNEL8x3_M1
-	KERNEL8x3_M2
-	KERNEL8x3_M3
-	KERNEL8x3_M4
-	prefetcht0 B_PR1+104(BO)
-	KERNEL8x3_M5
-	KERNEL8x3_M6
-	KERNEL8x3_M7
-	KERNEL8x3_M8
-
-	dec	%rax
-	jne	.L7_12
-
-.L7_12_E:
-
-	prefetcht0 B_PR1(BO)
-	prefetcht0 B_PR1+64(BO)
-	KERNEL8x3_M1
-	KERNEL8x3_M2
-	KERNEL8x3_M3
-	KERNEL8x3_M4
-	KERNEL8x3_M5
-	KERNEL8x3_M6
-	KERNEL8x3_M7
-	KERNEL8x3_E
-
-	jmp	.L7_16
-
-
-
-.L7_13:
-
-	test $2, %rax
-	jz	.L7_14
-
-	KERNEL8x3_INIT
-	KERNEL8x3_M2
-	KERNEL8x3_M3
-	KERNEL8x3_M4
-	KERNEL8x3_M5
-	KERNEL8x3_M6
-	KERNEL8x3_M7
-	KERNEL8x3_M8
-
-	KERNEL8x3_M1
-	KERNEL8x3_M2
-	KERNEL8x3_M3
-	KERNEL8x3_M4
-	KERNEL8x3_M5
-	KERNEL8x3_M6
-	KERNEL8x3_M7
-	KERNEL8x3_E
-
-	jmp	.L7_16
-
-
-.L7_14:
-
-	test $1, %rax
-	jz	.L7_15
-	
-	KERNEL8x3_INIT
-	KERNEL8x3_M2
-	KERNEL8x3_M3
-	KERNEL8x3_M4
-	KERNEL8x3_M5
-	KERNEL8x3_M6
-	KERNEL8x3_M7
-	KERNEL8x3_E
-
-	jmp	.L7_16
-
-
-
-.L7_15:
-
-	INIT8x3
-
-.L7_16:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L7_19
-
-
-	ALIGN_4
-
-.L7_17:
-
-	KERNEL8x3_SUBN
-	dec	%rax
-	jne	.L7_17
-	ALIGN_4
-
-
-.L7_19:
-
-	SAVE8x3
-
-	decq	I			# i --
-	jg	.L7_11
-	ALIGN_4	
-
-.L7_20:
-	// Test rest of M
-
-	testq	$7, M
-	jz	.L7_60		// to next 6 lines of N
-
-	testq	$4, M		
-	jz	.L7_30
-
-	ALIGN_4
-
-.L7_21:
-        leaq    BUFFER2, BO             // second buffer to BO
-        addq    $6 * SIZE, BO
-
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L7_26
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L7_22:
-
-	KERNEL4x3_1(xxx)
-	KERNEL4x3_2(xxx)
-	KERNEL4x3_3(xxx)
-	KERNEL4x3_4(xxx)
-
-	KERNEL4x3_1(xxx)
-	KERNEL4x3_2(xxx)
-	KERNEL4x3_3(xxx)
-	KERNEL4x3_4(xxx)
-
-	je	.L7_26
-
-	KERNEL4x3_1(xxx)
-	KERNEL4x3_2(xxx)
-	KERNEL4x3_3(xxx)
-	KERNEL4x3_4(xxx)
-
-	KERNEL4x3_1(xxx)
-	KERNEL4x3_2(xxx)
-	KERNEL4x3_3(xxx)
-	KERNEL4x3_4(xxx)
-
-	je	.L7_26
-
-	jmp	.L7_22
-	ALIGN_4
-
-.L7_26:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L7_29
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L7_27:
-
-	KERNEL4x3_SUB(xxx)
-	addq	$3, BI
-	addq	$4, %rax
-	jl	.L7_27
-	ALIGN_4
-
-
-.L7_29:
-
-	vmovddup	ALPHA, %xmm0
-
-
-	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7
-
-	vfmaddpd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-	vfmaddpd 2 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
-
-	vfmaddpd 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
-	vfmaddpd 2 * SIZE(CO1, LDC, 2),%xmm0, %xmm9,%xmm9
-
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm7 , 2 * SIZE(CO1)
-
-	vmovups	%xmm5 ,  	(CO1, LDC)
-	vmovups	%xmm8 , 2 * SIZE(CO1, LDC)
-
-	vmovups	%xmm6 ,  	(CO1, LDC, 2)
-	vmovups	%xmm9 , 2 * SIZE(CO1, LDC, 2)
-
-
-	addq	$4 * SIZE, CO1		# coffset += 4
-	ALIGN_4
-	
-
-.L7_30:
-	testq	$2, M		
-	jz	.L7_40
-
-	ALIGN_4
-
-.L7_31:
-        leaq    BUFFER2, BO             // second buffer to BO
-        addq    $6 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L7_36
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L7_32:
-
-	KERNEL2x3_1(xxx)
-	KERNEL2x3_2(xxx)
-	KERNEL2x3_3(xxx)
-	KERNEL2x3_4(xxx)
-
-	KERNEL2x3_1(xxx)
-	KERNEL2x3_2(xxx)
-	KERNEL2x3_3(xxx)
-	KERNEL2x3_4(xxx)
-
-	je	.L7_36
-
-	KERNEL2x3_1(xxx)
-	KERNEL2x3_2(xxx)
-	KERNEL2x3_3(xxx)
-	KERNEL2x3_4(xxx)
-
-	KERNEL2x3_1(xxx)
-	KERNEL2x3_2(xxx)
-	KERNEL2x3_3(xxx)
-	KERNEL2x3_4(xxx)
-
-	je	.L7_36
-
-	jmp	.L7_32
-	ALIGN_4
-
-.L7_36:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L7_39
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L7_37:
-
-	KERNEL2x3_SUB(xxx)
-	addq	$3, BI
-	addq	$2, %rax
-	jl	.L7_37
-	ALIGN_4
-
-
-.L7_39:
-
-	vmovddup	ALPHA, %xmm0
-
-
-	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddpd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-	vfmaddpd 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
-
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm5 ,  	(CO1, LDC)
-	vmovups	%xmm6 ,  	(CO1, LDC, 2)
-
-
-	addq	$2 * SIZE, CO1		# coffset += 2
-	ALIGN_4
-	
-
-
-
-
-.L7_40:
-	testq	$1, M		
-	jz	.L7_60		// to next 6 lines of N
-
-	ALIGN_4
-
-.L7_41:
-        leaq    BUFFER2, BO             // second buffer to BO
-        addq    $6 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-
-	andq	$-8, %rax
-	je	.L7_46
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L7_42:
-
-	KERNEL1x3_1(xxx)
-	KERNEL1x3_2(xxx)
-	KERNEL1x3_3(xxx)
-	KERNEL1x3_4(xxx)
-
-	KERNEL1x3_1(xxx)
-	KERNEL1x3_2(xxx)
-	KERNEL1x3_3(xxx)
-	KERNEL1x3_4(xxx)
-
-	je	.L7_46
-
-	KERNEL1x3_1(xxx)
-	KERNEL1x3_2(xxx)
-	KERNEL1x3_3(xxx)
-	KERNEL1x3_4(xxx)
-
-	KERNEL1x3_1(xxx)
-	KERNEL1x3_2(xxx)
-	KERNEL1x3_3(xxx)
-	KERNEL1x3_4(xxx)
-
-	je	.L7_46
-
-	jmp	.L7_42
-	ALIGN_4
-
-.L7_46:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L7_49
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L7_47:
-
-	KERNEL1x3_SUB(xxx)
-	addq	$3, BI
-	addq	$1, %rax
-	jl	.L7_47
-	ALIGN_4
-
-
-.L7_49:
-
-	vmovddup	ALPHA, %xmm0
-
-
-	vfmaddsd 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddsd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-	vfmaddsd 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
-
-
-	vmovsd	%xmm4 ,  	(CO1)
-	vmovsd	%xmm5 ,  	(CO1, LDC)
-	vmovsd	%xmm6 ,  	(CO1, LDC, 2)
-
-
-	addq	$1 * SIZE, CO1		# coffset += 1
-
-.L7_60:
-
-	decq	J			// j --
-	jg	.L6_01
-
-
-.L2_0:
-	cmpq	$0, Nmod6		// N % 6 == 0
-	je	.L999
-
-/************************************************************************************************
-* Loop for Nmod6 / 2 > 0
-*************************************************************************************************/
-
-	movq	Nmod6, J		
-	sarq	$1, J			// j = j / 2
-	je	.L1_0
-	ALIGN_4
-
-.L2_01:
-	// copy to sub buffer
-	movq	B, BO1
-	leaq    BUFFER1, BO		// first buffer to BO
-	movq	K, %rax
-	ALIGN_4
-
-.L2_02b:
-
-	vmovups	(BO1), %xmm0
-	vmovups	%xmm0,       (BO)
-	addq	$2*SIZE,BO1
-	addq	$2*SIZE,BO
-	decq	%rax
-	jnz	.L2_02b
-
-.L2_02c:
-
-	movq	BO1, B			// next offset of B
-
-.L2_10:
-	movq	C, CO1
-	leaq	(C, LDC, 2), C		// c += 2 * ldc
-
-	
-	movq	A, AO		 	// aoffset = a
-	addq	$16 * SIZE, AO
-
-	movq	M,  I
-	sarq	$3, I			// i = (m >> 3)
-	je	.L2_20
-
-	ALIGN_4
-
-.L2_11:
-
-	leaq	BUFFER1, BO		// first buffer to BO
-	addq	$4 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax			//  K = K - ( K % 8 )
-	je	.L2_16
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_12:
-
-	KERNEL8x2_1(xxx)
-	KERNEL8x2_2(xxx)
-	KERNEL8x2_3(xxx)
-	KERNEL8x2_4(xxx)
-
-	KERNEL8x2_1(xxx)
-	KERNEL8x2_2(xxx)
-	KERNEL8x2_3(xxx)
-	KERNEL8x2_4(xxx)
-
-	je	.L2_16
-
-	KERNEL8x2_1(xxx)
-	KERNEL8x2_2(xxx)
-	KERNEL8x2_3(xxx)
-	KERNEL8x2_4(xxx)
-
-	KERNEL8x2_1(xxx)
-	KERNEL8x2_2(xxx)
-	KERNEL8x2_3(xxx)
-	KERNEL8x2_4(xxx)
-
-	je	.L2_16
-
-	jmp	.L2_12
-	ALIGN_4
-
-.L2_16:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_19
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_17:
-
-	KERNEL8x2_SUB(xxx)
-	addq	$2, BI
-	addq	$8, %rax
-	jl	.L2_17
-	ALIGN_4
-
-
-.L2_19:
-
-	vmovddup	ALPHA, %xmm0
-
-	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7
-	vfmaddpd 4 * SIZE(CO1),%xmm0, %xmm10,%xmm10
-	vfmaddpd 6 * SIZE(CO1),%xmm0, %xmm13,%xmm13
-
-	vfmaddpd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-	vfmaddpd 2 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
-	vfmaddpd 4 * SIZE(CO1, LDC),%xmm0, %xmm11,%xmm11
-	vfmaddpd 6 * SIZE(CO1, LDC),%xmm0, %xmm14,%xmm14
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm7 , 2 * SIZE(CO1)
-	vmovups	%xmm10, 4 * SIZE(CO1)
-	vmovups	%xmm13, 6 * SIZE(CO1)
-
-	vmovups	%xmm5 ,  	(CO1, LDC)
-	vmovups	%xmm8 , 2 * SIZE(CO1, LDC)
-	vmovups	%xmm11, 4 * SIZE(CO1, LDC)
-	vmovups	%xmm14, 6 * SIZE(CO1, LDC)
-
-
-	addq	$8 * SIZE, CO1		# coffset += 8
-	decq	I			# i --
-	jg	.L2_11
-	ALIGN_4	
-
-/**************************************************************************
-* Rest of M 
-***************************************************************************/
-.L2_20:
-	// Test rest of M
-
-	testq	$7, M
-	jz	.L2_60		// to next 2 lines of N
-
-	testq	$4, M		
-	jz	.L2_30
-
-	ALIGN_4
-
-.L2_21:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L2_26
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_22:
-
-	KERNEL4x2_1(xxx)
-	KERNEL4x2_2(xxx)
-	KERNEL4x2_3(xxx)
-	KERNEL4x2_4(xxx)
-
-	KERNEL4x2_1(xxx)
-	KERNEL4x2_2(xxx)
-	KERNEL4x2_3(xxx)
-	KERNEL4x2_4(xxx)
-
-	je	.L2_26
-
-	KERNEL4x2_1(xxx)
-	KERNEL4x2_2(xxx)
-	KERNEL4x2_3(xxx)
-	KERNEL4x2_4(xxx)
-
-	KERNEL4x2_1(xxx)
-	KERNEL4x2_2(xxx)
-	KERNEL4x2_3(xxx)
-	KERNEL4x2_4(xxx)
-
-	je	.L2_26
-
-	jmp	.L2_22
-	ALIGN_4
-
-.L2_26:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_29
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_27:
-
-	KERNEL4x2_SUB(xxx)
-	addq	$2, BI
-	addq	$4, %rax
-	jl	.L2_27
-	ALIGN_4
-
-
-.L2_29:
-
-	vmovddup	ALPHA, %xmm0
-
-	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7
-
-	vfmaddpd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-	vfmaddpd 2 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm7 , 2 * SIZE(CO1)
-
-	vmovups	%xmm5 ,  	(CO1, LDC)
-	vmovups	%xmm8 , 2 * SIZE(CO1, LDC)
-
-	addq	$4 * SIZE, CO1		# coffset += 4
-	ALIGN_4
-	
-
-.L2_30:
-	testq	$2, M		
-	jz	.L2_40
-
-	ALIGN_4
-
-.L2_31:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L2_36
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_32:
-
-	KERNEL2x2_1(xxx)
-	KERNEL2x2_2(xxx)
-	KERNEL2x2_3(xxx)
-	KERNEL2x2_4(xxx)
-
-	KERNEL2x2_1(xxx)
-	KERNEL2x2_2(xxx)
-	KERNEL2x2_3(xxx)
-	KERNEL2x2_4(xxx)
-
-	je	.L2_36
-
-	KERNEL2x2_1(xxx)
-	KERNEL2x2_2(xxx)
-	KERNEL2x2_3(xxx)
-	KERNEL2x2_4(xxx)
-
-	KERNEL2x2_1(xxx)
-	KERNEL2x2_2(xxx)
-	KERNEL2x2_3(xxx)
-	KERNEL2x2_4(xxx)
-
-	je	.L2_36
-
-	jmp	.L2_32
-	ALIGN_4
-
-.L2_36:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_39
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-	
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_37:
-
-	KERNEL2x2_SUB(xxx)
-	addq	$2, BI
-	addq	$2, %rax
-	jl	.L2_37
-	ALIGN_4
-
-
-.L2_39:
-
-	vmovddup	ALPHA, %xmm0
-
-	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddpd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm5 ,  	(CO1, LDC)
-
-	addq	$2 * SIZE, CO1		# coffset += 2
-	ALIGN_4
-	
-
-.L2_40:
-	testq	$1, M		
-	jz	.L2_60		// to next 2 lines of N
-
-	ALIGN_4
-
-.L2_41:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L2_46
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_42:
-
-	KERNEL1x2_1(xxx)
-	KERNEL1x2_2(xxx)
-	KERNEL1x2_3(xxx)
-	KERNEL1x2_4(xxx)
-
-	KERNEL1x2_1(xxx)
-	KERNEL1x2_2(xxx)
-	KERNEL1x2_3(xxx)
-	KERNEL1x2_4(xxx)
-
-	je	.L2_46
-
-	KERNEL1x2_1(xxx)
-	KERNEL1x2_2(xxx)
-	KERNEL1x2_3(xxx)
-	KERNEL1x2_4(xxx)
-
-	KERNEL1x2_1(xxx)
-	KERNEL1x2_2(xxx)
-	KERNEL1x2_3(xxx)
-	KERNEL1x2_4(xxx)
-
-	je	.L2_46
-
-	jmp	.L2_42
-	ALIGN_4
-
-.L2_46:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_49
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_47:
-
-	KERNEL1x2_SUB(xxx)
-	addq	$2, BI
-	addq	$1, %rax
-	jl	.L2_47
-	ALIGN_4
-
-
-.L2_49:
-
-	vmovddup	ALPHA, %xmm0
-
-	vfmaddsd 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddsd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-
-	vmovsd	%xmm4 ,  	(CO1)
-	vmovsd	%xmm5 ,  	(CO1, LDC)
-
-
-	addq	$1 * SIZE, CO1		# coffset += 1
-	ALIGN_4
-	
-
-	
-.L2_60:
-
-	decq	J			// j --
-	jg	.L2_01			// next 2 lines of N
-
-
-
-.L1_0:
-
-/************************************************************************************************
-* Loop for Nmod6 % 2 > 0
-*************************************************************************************************/
-
-	movq	Nmod6, J		
-	andq	$1, J			// j % 2
-	je	.L999
-	ALIGN_4
-
-.L1_01:
-	// copy to sub buffer
-	movq	B, BO1
-	leaq    BUFFER1, BO		// first buffer to BO
-	movq	K, %rax
-	ALIGN_4
-
-.L1_02b:
-
-	vmovsd	(BO1), %xmm0
-	vmovsd	%xmm0,       (BO)
-	addq	$1*SIZE,BO1
-	addq	$1*SIZE,BO
-	decq	%rax
-	jnz	.L1_02b
-
-.L1_02c:
-
-	movq	BO1, B			// next offset of B
-
-.L1_10:
-	movq	C, CO1
-	leaq	(C, LDC, 1), C		// c += 1 * ldc
-
-	movq	A, AO		 	// aoffset = a
-	addq	$16 * SIZE, AO
-
-	movq	M,  I
-	sarq	$3, I			// i = (m >> 3)
-	je	.L1_20
-
-	ALIGN_4
-
-.L1_11:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax			//  K = K - ( K % 8 )
-	je	.L1_16
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_12:
-
-	KERNEL8x1_1(xxx)
-	KERNEL8x1_2(xxx)
-	KERNEL8x1_3(xxx)
-	KERNEL8x1_4(xxx)
-
-	KERNEL8x1_1(xxx)
-	KERNEL8x1_2(xxx)
-	KERNEL8x1_3(xxx)
-	KERNEL8x1_4(xxx)
-
-	je	.L1_16
-
-	KERNEL8x1_1(xxx)
-	KERNEL8x1_2(xxx)
-	KERNEL8x1_3(xxx)
-	KERNEL8x1_4(xxx)
-
-	KERNEL8x1_1(xxx)
-	KERNEL8x1_2(xxx)
-	KERNEL8x1_3(xxx)
-	KERNEL8x1_4(xxx)
-
-	je	.L1_16
-
-	jmp	.L1_12
-	ALIGN_4
-
-.L1_16:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_19
-
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_17:
-
-	KERNEL8x1_SUB(xxx)
-	addq	$1, BI
-	addq	$8, %rax
-	jl	.L1_17
-	ALIGN_4
-
-
-.L1_19:
-
-	vmovddup	ALPHA, %xmm0
-
-	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7
-	vfmaddpd 4 * SIZE(CO1),%xmm0, %xmm10,%xmm10
-	vfmaddpd 6 * SIZE(CO1),%xmm0, %xmm13,%xmm13
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm7 , 2 * SIZE(CO1)
-	vmovups	%xmm10, 4 * SIZE(CO1)
-	vmovups	%xmm13, 6 * SIZE(CO1)
-
-	addq	$8 * SIZE, CO1		# coffset += 8
-	decq	I			# i --
-	jg	.L1_11
-	ALIGN_4	
-
-/**************************************************************************
-* Rest of M 
-***************************************************************************/
-.L1_20:
-	// Test rest of M
-
-	testq	$7, M
-	jz	.L999
-
-	testq	$4, M		
-	jz	.L1_30
-
-	ALIGN_4
-
-.L1_21:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L1_26
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_22:
-
-	KERNEL4x1_1(xxx)
-	KERNEL4x1_2(xxx)
-	KERNEL4x1_3(xxx)
-	KERNEL4x1_4(xxx)
-
-	KERNEL4x1_1(xxx)
-	KERNEL4x1_2(xxx)
-	KERNEL4x1_3(xxx)
-	KERNEL4x1_4(xxx)
-
-	je	.L1_26
-
-	KERNEL4x1_1(xxx)
-	KERNEL4x1_2(xxx)
-	KERNEL4x1_3(xxx)
-	KERNEL4x1_4(xxx)
-
-	KERNEL4x1_1(xxx)
-	KERNEL4x1_2(xxx)
-	KERNEL4x1_3(xxx)
-	KERNEL4x1_4(xxx)
-
-	je	.L1_26
-
-	jmp	.L1_22
-	ALIGN_4
-
-.L1_26:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_29
-
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_27:
-
-	KERNEL4x1_SUB(xxx)
-	addq	$1, BI
-	addq	$4, %rax
-	jl	.L1_27
-	ALIGN_4
-
-
-.L1_29:
-
-	vmovddup	ALPHA, %xmm0
-
-	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm7 , 2 * SIZE(CO1)
-
-	addq	$4 * SIZE, CO1		# coffset += 4
-	ALIGN_4
-	
-
-.L1_30:
-	testq	$2, M		
-	jz	.L1_40
-
-	ALIGN_4
-
-.L1_31:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L1_36
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_32:
-
-	KERNEL2x1_1(xxx)
-	KERNEL2x1_2(xxx)
-	KERNEL2x1_3(xxx)
-	KERNEL2x1_4(xxx)
-
-	KERNEL2x1_1(xxx)
-	KERNEL2x1_2(xxx)
-	KERNEL2x1_3(xxx)
-	KERNEL2x1_4(xxx)
-
-	je	.L1_36
-
-	KERNEL2x1_1(xxx)
-	KERNEL2x1_2(xxx)
-	KERNEL2x1_3(xxx)
-	KERNEL2x1_4(xxx)
-
-	KERNEL2x1_1(xxx)
-	KERNEL2x1_2(xxx)
-	KERNEL2x1_3(xxx)
-	KERNEL2x1_4(xxx)
-
-	je	.L1_36
-
-	jmp	.L1_32
-	ALIGN_4
-
-.L1_36:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_39
-
-	movq    %rax, BI                        //  Index for BO
-	
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_37:
-
-	KERNEL2x1_SUB(xxx)
-	addq	$1, BI
-	addq	$2, %rax
-	jl	.L1_37
-	ALIGN_4
-
-
-.L1_39:
-
-	vmovddup	ALPHA, %xmm0
-
-	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
-
-	vmovups	%xmm4 ,  	(CO1)
-
-	addq	$2 * SIZE, CO1		# coffset += 2
-	ALIGN_4
-	
-
-.L1_40:
-	testq	$1, M		
-	jz	.L999
-
-	ALIGN_4
-
-.L1_41:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L1_46
-	movq    %rax, BI                        //  Index for BO
-
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_42:
-
-	KERNEL1x1_1(xxx)
-	KERNEL1x1_2(xxx)
-	KERNEL1x1_3(xxx)
-	KERNEL1x1_4(xxx)
-
-	KERNEL1x1_1(xxx)
-	KERNEL1x1_2(xxx)
-	KERNEL1x1_3(xxx)
-	KERNEL1x1_4(xxx)
-
-	je	.L1_46
-
-	KERNEL1x1_1(xxx)
-	KERNEL1x1_2(xxx)
-	KERNEL1x1_3(xxx)
-	KERNEL1x1_4(xxx)
-
-	KERNEL1x1_1(xxx)
-	KERNEL1x1_2(xxx)
-	KERNEL1x1_3(xxx)
-	KERNEL1x1_4(xxx)
-
-	je	.L1_46
-
-	jmp	.L1_42
-	ALIGN_4
-
-.L1_46:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_49
-
-	movq    %rax, BI                        //  Index for BO
-
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_47:
-
-	KERNEL1x1_SUB(xxx)
-	addq	$1, BI
-	addq	$1, %rax
-	jl	.L1_47
-	ALIGN_4
-
-
-.L1_49:
-
-	vmovddup	ALPHA, %xmm0
-
-	vfmaddsd 	 (CO1),%xmm0, %xmm4,%xmm4
-
-	vmovsd	%xmm4 ,  	(CO1)
-
-	addq	$1 * SIZE, CO1		# coffset += 1
-	ALIGN_4
-	
-
-.L999:
-	movq   		SP, %rsp
-	movq	   (%rsp), %rbx
-	movq	  8(%rsp), %rbp
-	movq	 16(%rsp), %r12
-	movq	 24(%rsp), %r13
-	movq	 32(%rsp), %r14
-	movq	 40(%rsp), %r15
-
-#ifdef WINDOWS_ABI
-	movq	 48(%rsp), %rdi
-	movq	 56(%rsp), %rsi
-	movups	 64(%rsp), %xmm6
-	movups	 80(%rsp), %xmm7
-	movups	 96(%rsp), %xmm8
-	movups	112(%rsp), %xmm9
-	movups	128(%rsp), %xmm10
-	movups	144(%rsp), %xmm11
-	movups	160(%rsp), %xmm12
-	movups	176(%rsp), %xmm13
-	movups	192(%rsp), %xmm14
-	movups	208(%rsp), %xmm15
-#endif
-
-	addq	$STACKSIZE, %rsp
-	ret
-
-	EPILOGUE
-
-
-#else
-/*************************************************************************************
-* TRMM Kernel
-*************************************************************************************/
-	PROLOGUE
-	PROFCODE
-	
-	subq	$STACKSIZE, %rsp
-	movq	%rbx,   (%rsp)
-	movq	%rbp,  8(%rsp)
-	movq	%r12, 16(%rsp)
-	movq	%r13, 24(%rsp)
-	movq	%r14, 32(%rsp)
-	movq	%r15, 40(%rsp)
-
-	vzeroupper
-
-#ifdef WINDOWS_ABI
-	movq	%rdi,    48(%rsp)
-	movq	%rsi,    56(%rsp)
-	movups	%xmm6,   64(%rsp)
-	movups	%xmm7,   80(%rsp)
-	movups	%xmm8,   96(%rsp)
-	movups	%xmm9,  112(%rsp)
-	movups	%xmm10, 128(%rsp)
-	movups	%xmm11, 144(%rsp)
-	movups	%xmm12, 160(%rsp)
-	movups	%xmm13, 176(%rsp)
-	movups	%xmm14, 192(%rsp)
-	movups	%xmm15, 208(%rsp)
-
-	movq	ARG1,      OLD_M
-	movq	ARG2,      OLD_N
-	movq	ARG3,      OLD_K
-	movq	OLD_A,     A
-	movq	OLD_B,     B
-	movq	OLD_C,     C
-	movq	OLD_LDC,   LDC
-#ifdef TRMMKERNEL
-	movsd	OLD_OFFSET, %xmm12
-#endif
-	vmovaps	%xmm3, %xmm0
-
-#else
-	movq	STACKSIZE +  8(%rsp), LDC
-#ifdef TRMMKERNEL
-	movsd	STACKSIZE + 16(%rsp), %xmm12
-#endif
-
-#endif
-
-	movq    %rsp, SP      # save old stack
-        subq    $128 + L_BUFFER_SIZE, %rsp
-        andq    $-4096, %rsp    # align stack
-
-        STACK_TOUCH
-
-	cmpq	$0, OLD_M
-	je	.L999
-
-	cmpq	$0, OLD_N
-	je	.L999
-
-	cmpq	$0, OLD_K
-	je	.L999
-
-	movq	OLD_M, M
-	movq	OLD_N, N
-	movq	OLD_K, K
-
-	vmovsd	 %xmm0, ALPHA
-
-	salq	$BASE_SHIFT, LDC
-
-	movq    N, %rax
-        xorq    %rdx, %rdx
-        movq    $2,  %rdi
-        divq    %rdi                    //    N / 2
-        movq    %rax, Ndiv6             //    N / 2
-        movq    %rdx, Nmod6             //    N % 2
-
-	
-
-#ifdef TRMMKERNEL
-	vmovsd	%xmm12, OFFSET
-	vmovsd	%xmm12, KK
-#ifndef LEFT
-	negq	KK
-#endif	
-#endif
-
-	movq	Ndiv6,  J
-	cmpq	$0, J
-	je	.L1_0
-	ALIGN_4
-
-.L2_0:
-
-.L2_01:
-	// copy to sub buffer
-	movq	B, BO1
-	leaq    BUFFER1, BO		// first buffer to BO
-	movq	K, %rax
-	ALIGN_4
-
-.L2_02b:
-
-	vmovups	(BO1), %xmm0
-	vmovups	%xmm0,       (BO)
-	addq	$2*SIZE,BO1
-	addq	$2*SIZE,BO
-	decq	%rax
-	jnz	.L2_02b
-
-.L2_02c:
-
-	movq	BO1, B			// next offset of B
-
-.L2_10:
-	movq	C, CO1
-	leaq	(C, LDC, 2), C		// c += 2 * ldc
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        movq    OFFSET, %rax
-        movq    %rax, KK
-#endif
-	
-	movq	A, AO		 	// aoffset = a
-	addq	$16 * SIZE, AO
-
-	movq	M,  I
-	sarq	$3, I			// i = (m >> 3)
-	je	.L2_20
-
-	ALIGN_4
-
-.L2_11:
-
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-	leaq	BUFFER1, BO		// first buffer to BO
-	addq	$4 * SIZE, BO
-#else
-        movq    KK, %rax
-	leaq	BUFFER1, BO		// first buffer to BO
-	addq	$4 * SIZE, BO
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, 8), BO
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-        leaq    (AO, %rax, 8), AO
-#endif
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $8, %rax        // number of values in AO
-#else
-        addq    $2, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-
-	andq	$-8, %rax			//  K = K - ( K % 8 )
-	je	.L2_16
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_12:
-
-	KERNEL8x2_1(xxx)
-	KERNEL8x2_2(xxx)
-	KERNEL8x2_3(xxx)
-	KERNEL8x2_4(xxx)
-
-	KERNEL8x2_1(xxx)
-	KERNEL8x2_2(xxx)
-	KERNEL8x2_3(xxx)
-	KERNEL8x2_4(xxx)
-
-	je	.L2_16
-
-	KERNEL8x2_1(xxx)
-	KERNEL8x2_2(xxx)
-	KERNEL8x2_3(xxx)
-	KERNEL8x2_4(xxx)
-
-	KERNEL8x2_1(xxx)
-	KERNEL8x2_2(xxx)
-	KERNEL8x2_3(xxx)
-	KERNEL8x2_4(xxx)
-
-	je	.L2_16
-
-	jmp	.L2_12
-	ALIGN_4
-
-.L2_16:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_19
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_17:
-
-	KERNEL8x2_SUB(xxx)
-	addq	$2, BI
-	addq	$8, %rax
-	jl	.L2_17
-	ALIGN_4
-
-
-.L2_19:
-
-	vmovddup	ALPHA, %xmm0
-
-#ifndef TRMMKERNEL
-
-	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7
-	vfmaddpd 4 * SIZE(CO1),%xmm0, %xmm10,%xmm10
-	vfmaddpd 6 * SIZE(CO1),%xmm0, %xmm13,%xmm13
-
-	vfmaddpd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-	vfmaddpd 2 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
-	vfmaddpd 4 * SIZE(CO1, LDC),%xmm0, %xmm11,%xmm11
-	vfmaddpd 6 * SIZE(CO1, LDC),%xmm0, %xmm14,%xmm14
-
-
-#else
-	vmulpd	%xmm0, %xmm4,%xmm4
-	vmulpd	%xmm0, %xmm7,%xmm7
-	vmulpd	%xmm0, %xmm10,%xmm10
-	vmulpd	%xmm0, %xmm13,%xmm13
-
-	vmulpd	%xmm0, %xmm5,%xmm5
-	vmulpd	%xmm0, %xmm8,%xmm8
-	vmulpd	%xmm0, %xmm11,%xmm11
-	vmulpd	%xmm0, %xmm14,%xmm14
-
-#endif
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm7 , 2 * SIZE(CO1)
-	vmovups	%xmm10, 4 * SIZE(CO1)
-	vmovups	%xmm13, 6 * SIZE(CO1)
-
-	vmovups	%xmm5 ,  	(CO1, LDC)
-	vmovups	%xmm8 , 2 * SIZE(CO1, LDC)
-	vmovups	%xmm11, 4 * SIZE(CO1, LDC)
-	vmovups	%xmm14, 6 * SIZE(CO1, LDC)
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, 8), BO
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-        leaq    (AO, %rax, 8), AO
-#endif
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $8, KK
-#endif
-
-	addq	$8 * SIZE, CO1		# coffset += 8
-	decq	I			# i --
-	jg	.L2_11
-	ALIGN_4	
-
-/**************************************************************************
-* Rest of M 
-***************************************************************************/
-.L2_20:
-	// Test rest of M
-
-	testq	$7, M
-	jz	.L2_60		// to next 2 lines of N
-
-	testq	$4, M		
-	jz	.L2_30
-
-	ALIGN_4
-
-.L2_21:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, 8), BO
-        salq    $2, %rax                        // rax = rax * 4 ; number of values
-        leaq    (AO, %rax, 8), AO
-#endif
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $4, %rax        // number of values in AO
-#else
-        addq    $2, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-
-	andq	$-8, %rax
-	je	.L2_26
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_22:
-
-	KERNEL4x2_1(xxx)
-	KERNEL4x2_2(xxx)
-	KERNEL4x2_3(xxx)
-	KERNEL4x2_4(xxx)
-
-	KERNEL4x2_1(xxx)
-	KERNEL4x2_2(xxx)
-	KERNEL4x2_3(xxx)
-	KERNEL4x2_4(xxx)
-
-	je	.L2_26
-
-	KERNEL4x2_1(xxx)
-	KERNEL4x2_2(xxx)
-	KERNEL4x2_3(xxx)
-	KERNEL4x2_4(xxx)
-
-	KERNEL4x2_1(xxx)
-	KERNEL4x2_2(xxx)
-	KERNEL4x2_3(xxx)
-	KERNEL4x2_4(xxx)
-
-	je	.L2_26
-
-	jmp	.L2_22
-	ALIGN_4
-
-.L2_26:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_29
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_27:
-
-	KERNEL4x2_SUB(xxx)
-	addq	$2, BI
-	addq	$4, %rax
-	jl	.L2_27
-	ALIGN_4
-
-
-.L2_29:
-
-	vmovddup	ALPHA, %xmm0
-
-#ifndef TRMMKERNEL
-
-	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7
-
-	vfmaddpd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-	vfmaddpd 2 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
-
-
-#else
-	vmulpd	%xmm0, %xmm4,%xmm4
-	vmulpd	%xmm0, %xmm7,%xmm7
-
-	vmulpd	%xmm0, %xmm5,%xmm5
-	vmulpd	%xmm0, %xmm8,%xmm8
-
-#endif
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm7 , 2 * SIZE(CO1)
-
-	vmovups	%xmm5 ,  	(CO1, LDC)
-	vmovups	%xmm8 , 2 * SIZE(CO1, LDC)
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, 8), BO
-        salq    $2, %rax                        // rax = rax * 4 ; number of values
-        leaq    (AO, %rax, 8), AO
-#endif
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $4, KK
-#endif
-
-	addq	$4 * SIZE, CO1		# coffset += 4
-	ALIGN_4
-	
-
-.L2_30:
-	testq	$2, M		
-	jz	.L2_40
-
-	ALIGN_4
-
-.L2_31:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, 8), BO
-        salq    $1, %rax                        // rax = rax * 2 ; number of values
-        leaq    (AO, %rax, 8), AO
-#endif
-
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $2, %rax        // number of values in AO
-#else
-        addq    $2, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-
-	andq	$-8, %rax
-	je	.L2_36
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_32:
-
-	KERNEL2x2_1(xxx)
-	KERNEL2x2_2(xxx)
-	KERNEL2x2_3(xxx)
-	KERNEL2x2_4(xxx)
-
-	KERNEL2x2_1(xxx)
-	KERNEL2x2_2(xxx)
-	KERNEL2x2_3(xxx)
-	KERNEL2x2_4(xxx)
-
-	je	.L2_36
-
-	KERNEL2x2_1(xxx)
-	KERNEL2x2_2(xxx)
-	KERNEL2x2_3(xxx)
-	KERNEL2x2_4(xxx)
-
-	KERNEL2x2_1(xxx)
-	KERNEL2x2_2(xxx)
-	KERNEL2x2_3(xxx)
-	KERNEL2x2_4(xxx)
-
-	je	.L2_36
-
-	jmp	.L2_32
-	ALIGN_4
-
-.L2_36:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_39
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-	
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_37:
-
-	KERNEL2x2_SUB(xxx)
-	addq	$2, BI
-	addq	$2, %rax
-	jl	.L2_37
-	ALIGN_4
-
-
-.L2_39:
-
-	vmovddup	ALPHA, %xmm0
-
-#ifndef TRMMKERNEL
-
-	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddpd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-
-#else
-	vmulpd	%xmm0, %xmm4,%xmm4
-	vmulpd	%xmm0, %xmm5,%xmm5
-
-#endif
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm5 ,  	(CO1, LDC)
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax 
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, 8), BO
-        salq    $1, %rax                        // rax = rax * 2 ; number of values
-        leaq    (AO, %rax, 8), AO
-#endif
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $2, KK
-#endif
-
-	addq	$2 * SIZE, CO1		# coffset += 2
-	ALIGN_4
-	
-
-.L2_40:
-	testq	$1, M		
-	jz	.L2_60		// to next 2 lines of N
-
-	ALIGN_4
-
-.L2_41:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, 8), BO
-        leaq    (AO, %rax, 8), AO
-#endif
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $1, %rax        // number of values in AO
-#else
-        addq    $2, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-	andq	$-8, %rax
-	je	.L2_46
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_42:
-
-	KERNEL1x2_1(xxx)
-	KERNEL1x2_2(xxx)
-	KERNEL1x2_3(xxx)
-	KERNEL1x2_4(xxx)
-
-	KERNEL1x2_1(xxx)
-	KERNEL1x2_2(xxx)
-	KERNEL1x2_3(xxx)
-	KERNEL1x2_4(xxx)
-
-	je	.L2_46
-
-	KERNEL1x2_1(xxx)
-	KERNEL1x2_2(xxx)
-	KERNEL1x2_3(xxx)
-	KERNEL1x2_4(xxx)
-
-	KERNEL1x2_1(xxx)
-	KERNEL1x2_2(xxx)
-	KERNEL1x2_3(xxx)
-	KERNEL1x2_4(xxx)
-
-	je	.L2_46
-
-	jmp	.L2_42
-	ALIGN_4
-
-.L2_46:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_49
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_47:
-
-	KERNEL1x2_SUB(xxx)
-	addq	$2, BI
-	addq	$1, %rax
-	jl	.L2_47
-	ALIGN_4
-
-
-.L2_49:
-
-	vmovddup	ALPHA, %xmm0
-
-#ifndef TRMMKERNEL
-
-	vfmaddsd 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddsd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-
-#else
-	vmulsd	%xmm0, %xmm4,%xmm4
-	vmulsd	%xmm0, %xmm5,%xmm5
-
-#endif
-
-	vmovsd	%xmm4 ,  	(CO1)
-	vmovsd	%xmm5 ,  	(CO1, LDC)
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, 8), BO
-        leaq    (AO, %rax, 8), AO
-#endif
-
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $1, KK
-#endif
-
-	addq	$1 * SIZE, CO1		# coffset += 1
-	ALIGN_4
-	
-
-	
-.L2_60:
-#if defined(TRMMKERNEL) && !defined(LEFT)
-        addq    $2, KK
-#endif
-
-	decq	J			// j --
-	jg	.L2_01			// next 2 lines of N
-
-
-
-.L1_0:
-
-/************************************************************************************************
-* Loop for Nmod6 % 2 > 0
-*************************************************************************************************/
-
-	movq	Nmod6, J		
-	andq	$1, J			// j % 2
-	je	.L999
-	ALIGN_4
-
-.L1_01:
-	// copy to sub buffer
-	movq	B, BO1
-	leaq    BUFFER1, BO		// first buffer to BO
-	movq	K, %rax
-	ALIGN_4
-
-.L1_02b:
-
-	vmovsd	(BO1), %xmm0
-	vmovsd	%xmm0,       (BO)
-	addq	$1*SIZE,BO1
-	addq	$1*SIZE,BO
-	decq	%rax
-	jnz	.L1_02b
-
-.L1_02c:
-
-	movq	BO1, B			// next offset of B
-
-.L1_10:
-	movq	C, CO1
-	leaq	(C, LDC, 1), C		// c += 1 * ldc
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        movq    OFFSET, %rax
-        movq    %rax, KK
-#endif
-
-	movq	A, AO		 	// aoffset = a
-	addq	$16 * SIZE, AO
-
-	movq	M,  I
-	sarq	$3, I			// i = (m >> 3)
-	je	.L1_20
-
-	ALIGN_4
-
-.L1_11:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BO, BI, 8), BO
-        salq    $3, %rax                        // rax = rax * 8 ; number of values
-        leaq    (AO, %rax, 8), AO
-#endif
-
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $8, %rax        // number of values in AO
-#else
-        addq    $1, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-	andq	$-8, %rax			//  K = K - ( K % 8 )
-	je	.L1_16
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_12:
-
-	KERNEL8x1_1(xxx)
-	KERNEL8x1_2(xxx)
-	KERNEL8x1_3(xxx)
-	KERNEL8x1_4(xxx)
-
-	KERNEL8x1_1(xxx)
-	KERNEL8x1_2(xxx)
-	KERNEL8x1_3(xxx)
-	KERNEL8x1_4(xxx)
-
-	je	.L1_16
-
-	KERNEL8x1_1(xxx)
-	KERNEL8x1_2(xxx)
-	KERNEL8x1_3(xxx)
-	KERNEL8x1_4(xxx)
-
-	KERNEL8x1_1(xxx)
-	KERNEL8x1_2(xxx)
-	KERNEL8x1_3(xxx)
-	KERNEL8x1_4(xxx)
-
-	je	.L1_16
-
-	jmp	.L1_12
-	ALIGN_4
-
-.L1_16:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_19
-
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_17:
-
-	KERNEL8x1_SUB(xxx)
-	addq	$1, BI
-	addq	$8, %rax
-	jl	.L1_17
-	ALIGN_4
-
-
-.L1_19:
-
-	vmovddup	ALPHA, %xmm0
-
-#ifndef TRMMKERNEL
-
-	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7
-	vfmaddpd 4 * SIZE(CO1),%xmm0, %xmm10,%xmm10
-	vfmaddpd 6 * SIZE(CO1),%xmm0, %xmm13,%xmm13
-
-#else
-	vmulpd	%xmm0, %xmm4,%xmm4
-	vmulpd	%xmm0, %xmm7,%xmm7
-	vmulpd	%xmm0, %xmm10,%xmm10
-	vmulpd	%xmm0, %xmm13,%xmm13
-
-#endif
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm7 , 2 * SIZE(CO1)
-	vmovups	%xmm10, 4 * SIZE(CO1)
-	vmovups	%xmm13, 6 * SIZE(CO1)
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BO, BI, 8), BO
-        salq    $3, %rax                        // rax = rax * 8 ; number of values
-        leaq    (AO, %rax, 8), AO
-#endif
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $8, KK
-#endif
-	addq	$8 * SIZE, CO1		# coffset += 8
-	decq	I			# i --
-	jg	.L1_11
-	ALIGN_4	
-
-/**************************************************************************
-* Rest of M 
-***************************************************************************/
-.L1_20:
-	// Test rest of M
-
-	testq	$7, M
-	jz	.L999
-
-	testq	$4, M		
-	jz	.L1_30
-
-	ALIGN_4
-
-.L1_21:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BO, BI, 8), BO
-        salq    $2, %rax                        // rax = rax * 4 ; number of values
-        leaq    (AO, %rax, 8), AO
-#endif
-
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $4, %rax        // number of values in AO
-#else
-        addq    $1, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-
-	andq	$-8, %rax
-	je	.L1_26
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_22:
-
-	KERNEL4x1_1(xxx)
-	KERNEL4x1_2(xxx)
-	KERNEL4x1_3(xxx)
-	KERNEL4x1_4(xxx)
-
-	KERNEL4x1_1(xxx)
-	KERNEL4x1_2(xxx)
-	KERNEL4x1_3(xxx)
-	KERNEL4x1_4(xxx)
-
-	je	.L1_26
-
-	KERNEL4x1_1(xxx)
-	KERNEL4x1_2(xxx)
-	KERNEL4x1_3(xxx)
-	KERNEL4x1_4(xxx)
-
-	KERNEL4x1_1(xxx)
-	KERNEL4x1_2(xxx)
-	KERNEL4x1_3(xxx)
-	KERNEL4x1_4(xxx)
-
-	je	.L1_26
-
-	jmp	.L1_22
-	ALIGN_4
-
-.L1_26:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_29
-
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_27:
-
-	KERNEL4x1_SUB(xxx)
-	addq	$1, BI
-	addq	$4, %rax
-	jl	.L1_27
-	ALIGN_4
-
-
-.L1_29:
-
-	vmovddup	ALPHA, %xmm0
-
-#ifndef TRMMKERNEL
-
-	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7
-
-#else
-	vmulpd	%xmm0, %xmm4,%xmm4
-	vmulpd	%xmm0, %xmm7,%xmm7
-
-#endif
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm7 , 2 * SIZE(CO1)
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BO, BI, 8), BO
-        salq    $2, %rax                        // rax = rax * 4 ; number of values
-        leaq    (AO, %rax, 8), AO
-#endif
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $4, KK
-#endif
-
-	addq	$4 * SIZE, CO1		# coffset += 4
-	ALIGN_4
-	
-
-.L1_30:
-	testq	$2, M		
-	jz	.L1_40
-
-	ALIGN_4
-
-.L1_31:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BO, BI, 8), BO
-        salq    $1, %rax                        // rax = rax * 2 ; number of values
-        leaq    (AO, %rax, 8), AO
-#endif
-
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $2, %rax        // number of values in AO
-#else
-        addq    $1, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-
-	andq	$-8, %rax
-	je	.L1_36
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_32:
-
-	KERNEL2x1_1(xxx)
-	KERNEL2x1_2(xxx)
-	KERNEL2x1_3(xxx)
-	KERNEL2x1_4(xxx)
-
-	KERNEL2x1_1(xxx)
-	KERNEL2x1_2(xxx)
-	KERNEL2x1_3(xxx)
-	KERNEL2x1_4(xxx)
-
-	je	.L1_36
-
-	KERNEL2x1_1(xxx)
-	KERNEL2x1_2(xxx)
-	KERNEL2x1_3(xxx)
-	KERNEL2x1_4(xxx)
-
-	KERNEL2x1_1(xxx)
-	KERNEL2x1_2(xxx)
-	KERNEL2x1_3(xxx)
-	KERNEL2x1_4(xxx)
-
-	je	.L1_36
-
-	jmp	.L1_32
-	ALIGN_4
-
-.L1_36:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_39
-
-	movq    %rax, BI                        //  Index for BO
-	
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_37:
-
-	KERNEL2x1_SUB(xxx)
-	addq	$1, BI
-	addq	$2, %rax
-	jl	.L1_37
-	ALIGN_4
-
-
-.L1_39:
-
-	vmovddup	ALPHA, %xmm0
-
-#ifndef TRMMKERNEL
-
-	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
-
-#else
-	vmulpd	%xmm0, %xmm4,%xmm4
-
-#endif
-
-	vmovups	%xmm4 ,  	(CO1)
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BO, BI, 8), BO
-        salq    $1, %rax                        // rax = rax * 2 ; number of values
-        leaq    (AO, %rax, 8), AO
-#endif
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $2, KK
-#endif
-	addq	$2 * SIZE, CO1		# coffset += 2
-	ALIGN_4
-	
-
-.L1_40:
-	testq	$1, M		
-	jz	.L999
-
-	ALIGN_4
-
-.L1_41:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BO, BI, 8), BO
-        leaq    (AO, %rax, 8), AO
-#endif
-
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $1, %rax        // number of values in AO
-#else
-        addq    $1, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-	andq	$-8, %rax
-	je	.L1_46
-	movq    %rax, BI                        //  Index for BO
-
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_42:
-
-	KERNEL1x1_1(xxx)
-	KERNEL1x1_2(xxx)
-	KERNEL1x1_3(xxx)
-	KERNEL1x1_4(xxx)
-
-	KERNEL1x1_1(xxx)
-	KERNEL1x1_2(xxx)
-	KERNEL1x1_3(xxx)
-	KERNEL1x1_4(xxx)
-
-	je	.L1_46
-
-	KERNEL1x1_1(xxx)
-	KERNEL1x1_2(xxx)
-	KERNEL1x1_3(xxx)
-	KERNEL1x1_4(xxx)
-
-	KERNEL1x1_1(xxx)
-	KERNEL1x1_2(xxx)
-	KERNEL1x1_3(xxx)
-	KERNEL1x1_4(xxx)
-
-	je	.L1_46
-
-	jmp	.L1_42
-	ALIGN_4
-
-.L1_46:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_49
-
-	movq    %rax, BI                        //  Index for BO
-
-	leaq	(AO, %rax, 8), AO
-	leaq	(BO, BI, 8), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_47:
-
-	KERNEL1x1_SUB(xxx)
-	addq	$1, BI
-	addq	$1, %rax
-	jl	.L1_47
-	ALIGN_4
-
-
-.L1_49:
-
-	vmovddup	ALPHA, %xmm0
-
-#ifndef TRMMKERNEL
-
-	vfmaddsd 	 (CO1),%xmm0, %xmm4,%xmm4
-
-#else
-	vmulsd	%xmm0, %xmm4,%xmm4
-
-#endif
-
-	vmovsd	%xmm4 ,  	(CO1)
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BO, BI, 8), BO
-        leaq    (AO, %rax, 8), AO
-#endif
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $1, KK
-#endif
-
-	addq	$1 * SIZE, CO1		# coffset += 1
-	ALIGN_4
-	
-
-.L999:
-	movq   		SP, %rsp
-	movq	   (%rsp), %rbx
-	movq	  8(%rsp), %rbp
-	movq	 16(%rsp), %r12
-	movq	 24(%rsp), %r13
-	movq	 32(%rsp), %r14
-	movq	 40(%rsp), %r15
-
-#ifdef WINDOWS_ABI
-	movq	 48(%rsp), %rdi
-	movq	 56(%rsp), %rsi
-	movups	 64(%rsp), %xmm6
-	movups	 80(%rsp), %xmm7
-	movups	 96(%rsp), %xmm8
-	movups	112(%rsp), %xmm9
-	movups	128(%rsp), %xmm10
-	movups	144(%rsp), %xmm11
-	movups	160(%rsp), %xmm12
-	movups	176(%rsp), %xmm13
-	movups	192(%rsp), %xmm14
-	movups	208(%rsp), %xmm15
-#endif
-
-	addq	$STACKSIZE, %rsp
-	ret
-
-	EPILOGUE
-
-
-
-#endif
+/***************************************************************************
+Copyright (c) 2013, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+
+/*********************************************************************
+*
+* 2013/11/13 Saar
+*        BLASTEST               : OK
+*        CTEST                  : OK
+*        TEST                   : OK
+*
+*
+* 2013/10/31 Saar
+*
+* Parameter:
+* 	UNROLL_M	8
+*	UNROLL_N	2
+*	DGEMM_P		768
+*	DGEMM_Q		168
+*	DGEMM_R		12288
+*	A_PR1		512
+*	B_PR1		256
+*
+* Performance at m x n on AMD 8320 (ACML-Version: 5.3.1):
+* 
+* 4608x4608	83.9	GFLOPS with 8 threads on 4 modules (ACML: 78.4 GFLOPS)
+* 4608x4608	80.9	GFLOPS with 4 threads on 4 modules (ACML: 78.4 GFLOPS)
+* 4608x4608	41.3    GFLOPS with 2 threads on 2 modules (ACML: 40.9 GFLOPS)
+* 4608x4608	20.7	GFLOPS with 1 threads on 1 modules (ACML: 20.8 GFLOPS)
+*
+* Performance at m x n on AMD 6380  (ACML-Version: 5.3.1):
+* 
+* 13824x13824  234.5	GFLOPS with 32 threads on 16 modules (ACML:  88.5 GFLOPS) !strange thermal behavior
+* 13824x13824  241.9	GFLOPS with 16 threads on 16 modules (ACML: 191.5 GFLOPS) !strange thermal behavior
+* 9216x9216    137.6	GFLOPS with  8 threads on  8 modules (ACML: 106.5 GFLOPS)
+* 4608x4608	75.7	GFLOPS with  4 threads on  4 modules (ACML:  56.3 GFLOPS)
+* 4608x4608	38.6	GFLOPS with  2 threads on  2 modules (ACML:  34.1 GFLOPS)
+* 4608x4608	19.6	GFLOPS with  1 threads on  1 modules (ACML:  18.3 GFLOPS)
+*
+*********************************************************************/
+
+
+#define ASSEMBLER
+#include "common.h"
+ 
+#define OLD_M	%rdi
+#define OLD_N	%rsi
+#define M	%r13
+#define J	%r14
+#define OLD_K	%rdx
+
+#define A	%rcx
+#define B	%r8
+#define C	%r9
+#define LDC	%r10
+	
+#define I	%r11
+#define AO	%rdi
+#define BO	%rsi
+#define	CO1	%r15
+#define K	%r12
+#define BI	%rbp
+#define	SP	%rbx
+
+#define BO1	%rdi
+#define BO2	%r15
+
+#ifndef WINDOWS_ABI
+
+#define STACKSIZE 96
+
+#else
+
+#define STACKSIZE 256
+
+#define OLD_A		40 + STACKSIZE(%rsp)
+#define OLD_B		48 + STACKSIZE(%rsp)
+#define OLD_C		56 + STACKSIZE(%rsp)
+#define OLD_LDC		64 + STACKSIZE(%rsp)
+#define OLD_OFFSET	72 + STACKSIZE(%rsp)
+
+#endif
+
+#define L_BUFFER_SIZE 8192
+#define LB2_OFFSET    4096
+
+#define Ndiv6	 24(%rsp)
+#define Nmod6	 32(%rsp)
+#define N	 40(%rsp)
+#define ALPHA	 48(%rsp)
+#define OFFSET	 56(%rsp)
+#define KK	 64(%rsp)
+#define KKK	 72(%rsp)
+#define BUFFER1	           128(%rsp)
+#define BUFFER2	LB2_OFFSET+128(%rsp)
+
+#if defined(OS_WINDOWS)
+#if   L_BUFFER_SIZE > 16384
+#define STACK_TOUCH \
+        movl    $0,  4096 * 4(%rsp);\
+        movl    $0,  4096 * 3(%rsp);\
+        movl    $0,  4096 * 2(%rsp);\
+        movl    $0,  4096 * 1(%rsp);
+#elif L_BUFFER_SIZE > 12288
+#define STACK_TOUCH \
+        movl    $0,  4096 * 3(%rsp);\
+        movl    $0,  4096 * 2(%rsp);\
+        movl    $0,  4096 * 1(%rsp);
+#elif L_BUFFER_SIZE > 8192
+#define STACK_TOUCH \
+        movl    $0,  4096 * 2(%rsp);\
+        movl    $0,  4096 * 1(%rsp);
+#elif L_BUFFER_SIZE > 4096
+#define STACK_TOUCH \
+        movl    $0,  4096 * 1(%rsp);
+#else
+#define STACK_TOUCH
+#endif
+#else
+#define STACK_TOUCH
+#endif
+
+#if defined(BULLDOZER)
+
+#define VFMADD231PD_( y1,y2,y0 ) vfmaddpd y0,y1,y2,y0
+
+#define VFMADD231SD_( x1,x2,x0 ) vfmaddsd x0,x1,x2,x0
+
+#else
+
+#define VFMADD231PD_( y1,y2,y0 ) vfmadd231pd y2,y1,y0
+
+#define VFMADD231SD_( x1,x2,x0 ) vfmadd231sd x2,x1,x0
+
+#endif
+
+
+
+
+#define	A_PR1	512
+#define	B_PR1	256
+#define	C_PR1	64
+
+.macro INIT8x3
+	vxorpd	%xmm4 , %xmm4 , %xmm4
+	vxorpd	%xmm5 , %xmm5 , %xmm5
+	vxorpd	%xmm6 , %xmm6 , %xmm6
+	vxorpd	%xmm7 , %xmm7 , %xmm7
+	vxorpd	%xmm8 , %xmm8 , %xmm8
+	vxorpd	%xmm9 , %xmm9 , %xmm9
+	vxorpd	%xmm10, %xmm10, %xmm10
+	vxorpd	%xmm11, %xmm11, %xmm11
+	vxorpd	%xmm12, %xmm12, %xmm12
+	vxorpd	%xmm13, %xmm13, %xmm13
+	vxorpd	%xmm14, %xmm14, %xmm14
+	vxorpd	%xmm15, %xmm15, %xmm15
+.endm
+
+.macro KERNEL8x3_INIT 
+	vmovddup	-12 * SIZE(BO), %xmm1
+	vmovups 	-16 * SIZE(AO), %xmm0
+	prefetcht0	A_PR1(AO)
+	vmulpd  	%xmm1,%xmm0,%xmm4
+	vmovddup	-11 * SIZE(BO), %xmm2
+	vmulpd  	%xmm2,%xmm0,%xmm5
+	vmovddup	-10 * SIZE(BO), %xmm3
+	vmulpd  	%xmm3,%xmm0,%xmm6
+	vmovups 	-14 * SIZE(AO), %xmm0
+	vmulpd  	%xmm1,%xmm0,%xmm7
+	vmulpd  	%xmm2,%xmm0,%xmm8
+	vmulpd  	%xmm3,%xmm0,%xmm9
+	vmovups 	-12 * SIZE(AO), %xmm0
+	vmulpd  	%xmm1,%xmm0,%xmm10
+	vmulpd  	%xmm2,%xmm0,%xmm11
+	addq		$ 3 * SIZE, BO
+	vmulpd  	%xmm3,%xmm0,%xmm12
+	vmovups 	-10 * SIZE(AO), %xmm0
+	vmulpd  	%xmm1,%xmm0,%xmm13
+	vmovddup	-12 * SIZE(BO), %xmm1
+	vmulpd  	%xmm2,%xmm0,%xmm14
+	vmovddup	-11 * SIZE(BO), %xmm2
+	vmulpd  	%xmm3,%xmm0,%xmm15
+.endm
+
+
+.macro KERNEL8x3_M1 
+	vmovups 	-16 * SIZE(AO), %xmm0
+	prefetcht0	A_PR1(AO)
+	VFMADD231PD_(  	%xmm1,%xmm0,%xmm4 )
+	VFMADD231PD_(  	%xmm2,%xmm0,%xmm5 )
+	VFMADD231PD_(  	%xmm3,%xmm0,%xmm6 )
+	vmovups 	-14 * SIZE(AO), %xmm0
+	VFMADD231PD_(  	%xmm1,%xmm0,%xmm7 )
+	VFMADD231PD_(  	%xmm2,%xmm0,%xmm8 )
+	VFMADD231PD_(  	%xmm3,%xmm0,%xmm9 )
+	vmovups 	-12 * SIZE(AO), %xmm0
+	VFMADD231PD_(  	%xmm1,%xmm0,%xmm10 )
+	VFMADD231PD_(  	%xmm2,%xmm0,%xmm11 )
+	VFMADD231PD_(  	%xmm3,%xmm0,%xmm12 )
+	vmovups 	-10 * SIZE(AO), %xmm0
+	VFMADD231PD_(  	%xmm1,%xmm0,%xmm13 )
+	vmovddup	-12 * SIZE(BO), %xmm1
+	VFMADD231PD_(  	%xmm2,%xmm0,%xmm14 )
+	vmovddup	-11 * SIZE(BO), %xmm2
+	VFMADD231PD_(  	%xmm3,%xmm0,%xmm15 )
+.endm
+
+.macro KERNEL8x3_M2 
+	vmovups 	 -8 * SIZE(AO), %xmm0
+	prefetcht0	A_PR1+64(AO)
+	vmovddup	-10 * SIZE(BO), %xmm3
+	VFMADD231PD_(  	%xmm1,%xmm0,%xmm4 )
+	VFMADD231PD_(  	%xmm2,%xmm0,%xmm5 )
+	VFMADD231PD_(  	%xmm3,%xmm0,%xmm6 )
+	vmovups 	 -6 * SIZE(AO), %xmm0
+	VFMADD231PD_(  	%xmm1,%xmm0,%xmm7 )
+	VFMADD231PD_(  	%xmm2,%xmm0,%xmm8 )
+	VFMADD231PD_(  	%xmm3,%xmm0,%xmm9 )
+	vmovups 	 -4 * SIZE(AO), %xmm0
+	VFMADD231PD_(  	%xmm1,%xmm0,%xmm10 )
+	VFMADD231PD_(  	%xmm2,%xmm0,%xmm11 )
+	VFMADD231PD_(  	%xmm3,%xmm0,%xmm12 )
+	vmovups 	 -2 * SIZE(AO), %xmm0
+	VFMADD231PD_(  	%xmm1,%xmm0,%xmm13 )
+	vmovddup	 -9 * SIZE(BO), %xmm1
+	VFMADD231PD_(  	%xmm2,%xmm0,%xmm14 )
+	vmovddup	 -8 * SIZE(BO), %xmm2
+	VFMADD231PD_(  	%xmm3,%xmm0,%xmm15 )
+.endm
+
+
+.macro KERNEL8x3_M3 
+	vmovups 	  0 * SIZE(AO), %xmm0
+	prefetcht0	A_PR1+128(AO)
+	vmovddup	 -7 * SIZE(BO), %xmm3
+	VFMADD231PD_(  	%xmm1,%xmm0,%xmm4 )
+	VFMADD231PD_(  	%xmm2,%xmm0,%xmm5 )
+	VFMADD231PD_(  	%xmm3,%xmm0,%xmm6 )
+	vmovups 	  2 * SIZE(AO), %xmm0
+	VFMADD231PD_(  	%xmm1,%xmm0,%xmm7 )
+	VFMADD231PD_(  	%xmm2,%xmm0,%xmm8 )
+	VFMADD231PD_(  	%xmm3,%xmm0,%xmm9 )
+	vmovups 	  4 * SIZE(AO), %xmm0
+	VFMADD231PD_(  	%xmm1,%xmm0,%xmm10 )
+	VFMADD231PD_(  	%xmm2,%xmm0,%xmm11 )
+	VFMADD231PD_(  	%xmm3,%xmm0,%xmm12 )
+	vmovups 	  6 * SIZE(AO), %xmm0
+	VFMADD231PD_(  	%xmm1,%xmm0,%xmm13 )
+	vmovddup	 -6 * SIZE(BO), %xmm1
+	VFMADD231PD_(  	%xmm2,%xmm0,%xmm14 )
+	vmovddup	 -5 * SIZE(BO), %xmm2
+	VFMADD231PD_(  	%xmm3,%xmm0,%xmm15 )
+.endm
+
+.macro KERNEL8x3_M4 
+	vmovups 	  8 * SIZE(AO), %xmm0
+	prefetcht0	A_PR1+192(AO)
+	vmovddup	 -4 * SIZE(BO), %xmm3
+	VFMADD231PD_(  	%xmm1,%xmm0,%xmm4 )
+	VFMADD231PD_(  	%xmm2,%xmm0,%xmm5 )
+	VFMADD231PD_(  	%xmm3,%xmm0,%xmm6 )
+	vmovups 	 10 * SIZE(AO), %xmm0
+	VFMADD231PD_(  	%xmm1,%xmm0,%xmm7 )
+	VFMADD231PD_(  	%xmm2,%xmm0,%xmm8 )
+	VFMADD231PD_(  	%xmm3,%xmm0,%xmm9 )
+	vmovups 	 12 * SIZE(AO), %xmm0
+	VFMADD231PD_(  	%xmm1,%xmm0,%xmm10 )
+	VFMADD231PD_(  	%xmm2,%xmm0,%xmm11 )
+	VFMADD231PD_(  	%xmm3,%xmm0,%xmm12 )
+	vmovups 	 14 * SIZE(AO), %xmm0
+	VFMADD231PD_(  	%xmm1,%xmm0,%xmm13 )
+	vmovddup	 -3 * SIZE(BO), %xmm1
+	addq		$ 32 * SIZE, AO
+	VFMADD231PD_(  	%xmm2,%xmm0,%xmm14 )
+	vmovddup	 -2 * SIZE(BO), %xmm2
+	VFMADD231PD_(  	%xmm3,%xmm0,%xmm15 )
+.endm
+
+.macro KERNEL8x3_M5 
+	vmovups 	-16 * SIZE(AO), %xmm0
+	prefetcht0	A_PR1(AO)
+	vmovddup	 -1 * SIZE(BO), %xmm3
+	VFMADD231PD_(  	%xmm1,%xmm0,%xmm4 )
+	VFMADD231PD_(  	%xmm2,%xmm0,%xmm5 )
+	VFMADD231PD_(  	%xmm3,%xmm0,%xmm6 )
+	vmovups 	-14 * SIZE(AO), %xmm0
+	VFMADD231PD_(  	%xmm1,%xmm0,%xmm7 )
+	VFMADD231PD_(  	%xmm2,%xmm0,%xmm8 )
+	VFMADD231PD_(  	%xmm3,%xmm0,%xmm9 )
+	vmovups 	-12 * SIZE(AO), %xmm0
+	VFMADD231PD_(  	%xmm1,%xmm0,%xmm10 )
+	VFMADD231PD_(  	%xmm2,%xmm0,%xmm11 )
+	VFMADD231PD_(  	%xmm3,%xmm0,%xmm12 )
+	vmovups 	-10 * SIZE(AO), %xmm0
+	VFMADD231PD_(  	%xmm1,%xmm0,%xmm13 )
+	vmovddup	  0 * SIZE(BO), %xmm1
+	VFMADD231PD_(  	%xmm2,%xmm0,%xmm14 )
+	vmovddup	  1 * SIZE(BO), %xmm2
+	VFMADD231PD_(  	%xmm3,%xmm0,%xmm15 )
+.endm
+
+.macro KERNEL8x3_M6 
+	vmovups 	 -8 * SIZE(AO), %xmm0
+	prefetcht0	A_PR1+64(AO)
+	vmovddup	  2 * SIZE(BO), %xmm3
+	VFMADD231PD_(  	%xmm1,%xmm0,%xmm4 )
+	VFMADD231PD_(  	%xmm2,%xmm0,%xmm5 )
+	VFMADD231PD_(  	%xmm3,%xmm0,%xmm6 )
+	vmovups 	 -6 * SIZE(AO), %xmm0
+	VFMADD231PD_(  	%xmm1,%xmm0,%xmm7 )
+	VFMADD231PD_(  	%xmm2,%xmm0,%xmm8 )
+	VFMADD231PD_(  	%xmm3,%xmm0,%xmm9 )
+	vmovups 	 -4 * SIZE(AO), %xmm0
+	VFMADD231PD_(  	%xmm1,%xmm0,%xmm10 )
+	VFMADD231PD_(  	%xmm2,%xmm0,%xmm11 )
+	VFMADD231PD_(  	%xmm3,%xmm0,%xmm12 )
+	vmovups 	 -2 * SIZE(AO), %xmm0
+	VFMADD231PD_(  	%xmm1,%xmm0,%xmm13 )
+	vmovddup	  3 * SIZE(BO), %xmm1
+	VFMADD231PD_(  	%xmm2,%xmm0,%xmm14 )
+	vmovddup	  4 * SIZE(BO), %xmm2
+	VFMADD231PD_(  	%xmm3,%xmm0,%xmm15 )
+.endm
+
+
+.macro KERNEL8x3_M7 
+	vmovups 	  0 * SIZE(AO), %xmm0
+	prefetcht0	A_PR1+128(AO)
+	vmovddup	  5 * SIZE(BO), %xmm3
+	VFMADD231PD_(  	%xmm1,%xmm0,%xmm4 )
+	VFMADD231PD_(  	%xmm2,%xmm0,%xmm5 )
+	VFMADD231PD_(  	%xmm3,%xmm0,%xmm6 )
+	vmovups 	  2 * SIZE(AO), %xmm0
+	VFMADD231PD_(  	%xmm1,%xmm0,%xmm7 )
+	VFMADD231PD_(  	%xmm2,%xmm0,%xmm8 )
+	VFMADD231PD_(  	%xmm3,%xmm0,%xmm9 )
+	vmovups 	  4 * SIZE(AO), %xmm0
+	VFMADD231PD_(  	%xmm1,%xmm0,%xmm10 )
+	VFMADD231PD_(  	%xmm2,%xmm0,%xmm11 )
+	VFMADD231PD_(  	%xmm3,%xmm0,%xmm12 )
+	vmovups 	  6 * SIZE(AO), %xmm0
+	VFMADD231PD_(  	%xmm1,%xmm0,%xmm13 )
+	vmovddup	  6 * SIZE(BO), %xmm1
+	VFMADD231PD_(  	%xmm2,%xmm0,%xmm14 )
+	vmovddup	  7 * SIZE(BO), %xmm2
+	VFMADD231PD_(  	%xmm3,%xmm0,%xmm15 )
+.endm
+
+.macro KERNEL8x3_M8 
+	vmovups 	  8 * SIZE(AO), %xmm0
+	prefetcht0	A_PR1+192(AO)
+	vmovddup	  8 * SIZE(BO), %xmm3
+	VFMADD231PD_(  	%xmm1,%xmm0,%xmm4 )
+	VFMADD231PD_(  	%xmm2,%xmm0,%xmm5 )
+	VFMADD231PD_(  	%xmm3,%xmm0,%xmm6 )
+	vmovups 	 10 * SIZE(AO), %xmm0
+	VFMADD231PD_(  	%xmm1,%xmm0,%xmm7 )
+	VFMADD231PD_(  	%xmm2,%xmm0,%xmm8 )
+	VFMADD231PD_(  	%xmm3,%xmm0,%xmm9 )
+	vmovups 	 12 * SIZE(AO), %xmm0
+	VFMADD231PD_(  	%xmm1,%xmm0,%xmm10 )
+	VFMADD231PD_(  	%xmm2,%xmm0,%xmm11 )
+	VFMADD231PD_(  	%xmm3,%xmm0,%xmm12 )
+	vmovups 	 14 * SIZE(AO), %xmm0
+	VFMADD231PD_(  	%xmm1,%xmm0,%xmm13 )
+	vmovddup	  9 * SIZE(BO), %xmm1
+	VFMADD231PD_(  	%xmm2,%xmm0,%xmm14 )
+	vmovddup	 10 * SIZE(BO), %xmm2
+	VFMADD231PD_(  	%xmm3,%xmm0,%xmm15 )
+	vmovddup	 11 * SIZE(BO), %xmm3
+	addq		$ 32 * SIZE, AO
+	addq		$ 24 * SIZE, BO
+.endm
+
+
+.macro KERNEL8x3_E 
+	vmovups 	  8 * SIZE(AO), %xmm0
+	prefetcht0	A_PR1+192(AO)
+	vmovddup	  8 * SIZE(BO), %xmm3
+	VFMADD231PD_(  	%xmm1,%xmm0,%xmm4 )
+	VFMADD231PD_(  	%xmm2,%xmm0,%xmm5 )
+	VFMADD231PD_(  	%xmm3,%xmm0,%xmm6 )
+	vmovups 	 10 * SIZE(AO), %xmm0
+	VFMADD231PD_(  	%xmm1,%xmm0,%xmm7 )
+	VFMADD231PD_(  	%xmm2,%xmm0,%xmm8 )
+	VFMADD231PD_(  	%xmm3,%xmm0,%xmm9 )
+	vmovups 	 12 * SIZE(AO), %xmm0
+	VFMADD231PD_(  	%xmm1,%xmm0,%xmm10 )
+	VFMADD231PD_(  	%xmm2,%xmm0,%xmm11 )
+	VFMADD231PD_(  	%xmm3,%xmm0,%xmm12 )
+	vmovups 	 14 * SIZE(AO), %xmm0
+	VFMADD231PD_(  	%xmm1,%xmm0,%xmm13 )
+	addq		$ 32 * SIZE, AO
+	VFMADD231PD_(  	%xmm2,%xmm0,%xmm14 )
+	addq		$ 21 * SIZE, BO
+	VFMADD231PD_(  	%xmm3,%xmm0,%xmm15 )
+.endm
+
+.macro KERNEL8x3_SUBN 
+	vmovddup	-12 * SIZE(BO), %xmm1
+	vmovups 	-16 * SIZE(AO), %xmm0
+	VFMADD231PD_(  	%xmm1,%xmm0,%xmm4 )
+	vmovddup	-11 * SIZE(BO), %xmm2
+	VFMADD231PD_(  	%xmm2,%xmm0,%xmm5 )
+	vmovddup	-10 * SIZE(BO), %xmm3
+	VFMADD231PD_(  	%xmm3,%xmm0,%xmm6 )
+	vmovups 	-14 * SIZE(AO), %xmm0
+	VFMADD231PD_(  	%xmm1,%xmm0,%xmm7 )
+	VFMADD231PD_(  	%xmm2,%xmm0,%xmm8 )
+	VFMADD231PD_(  	%xmm3,%xmm0,%xmm9 )
+	vmovups 	-12 * SIZE(AO), %xmm0
+	VFMADD231PD_(  	%xmm1,%xmm0,%xmm10 )
+	VFMADD231PD_(  	%xmm2,%xmm0,%xmm11 )
+	VFMADD231PD_(  	%xmm3,%xmm0,%xmm12 )
+	vmovups 	-10 * SIZE(AO), %xmm0
+	VFMADD231PD_(  	%xmm1,%xmm0,%xmm13 )
+	addq		$ 3 * SIZE, BO
+	VFMADD231PD_(  	%xmm2,%xmm0,%xmm14 )
+	addq		$ 8 * SIZE, AO
+	VFMADD231PD_(  	%xmm3,%xmm0,%xmm15 )
+.endm
+
+.macro	SAVE8x3
+	vmovddup	ALPHA, %xmm0
+
+
+	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddpd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+	vfmaddpd 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
+
+	vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7
+	vfmaddpd 4 * SIZE(CO1),%xmm0, %xmm10,%xmm10
+	vfmaddpd 6 * SIZE(CO1),%xmm0, %xmm13,%xmm13
+
+	vfmaddpd 2 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
+	vfmaddpd 4 * SIZE(CO1, LDC),%xmm0, %xmm11,%xmm11
+	vfmaddpd 6 * SIZE(CO1, LDC),%xmm0, %xmm14,%xmm14
+
+	vfmaddpd 2 * SIZE(CO1, LDC, 2),%xmm0, %xmm9,%xmm9
+	vfmaddpd 4 * SIZE(CO1, LDC, 2),%xmm0, %xmm12,%xmm12
+	vfmaddpd 6 * SIZE(CO1, LDC, 2),%xmm0, %xmm15,%xmm15
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm7 , 2 * SIZE(CO1)
+	vmovups	%xmm10, 4 * SIZE(CO1)
+	vmovups	%xmm13, 6 * SIZE(CO1)
+
+	vmovups	%xmm5 ,  	(CO1, LDC)
+	vmovups	%xmm8 , 2 * SIZE(CO1, LDC)
+	vmovups	%xmm11, 4 * SIZE(CO1, LDC)
+	vmovups	%xmm14, 6 * SIZE(CO1, LDC)
+
+	vmovups	%xmm6 ,  	(CO1, LDC, 2)
+	vmovups	%xmm9 , 2 * SIZE(CO1, LDC, 2)
+	vmovups	%xmm12, 4 * SIZE(CO1, LDC, 2)
+	vmovups	%xmm15, 6 * SIZE(CO1, LDC, 2)
+
+	prefetcht0	 C_PR1(CO1)
+	prefetcht0	 C_PR1(CO1,LDC)
+	prefetcht0	 C_PR1(CO1,LDC,2)
+
+	addq	$ 8 * SIZE, CO1		# coffset += 8
+.endm
+
+
+/*******************************************************************************************/
+
+#define KERNEL4x3_1(xx) \
+	vmovddup	 -6 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovddup	 -5 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovddup	 -4 * SIZE(BO, BI, 8), %xmm3 ;\
+	vfmaddpd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+	vmovups 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	vfmaddpd  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
+
+#define KERNEL4x3_2(xx) \
+	vmovddup	 -3 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	-12 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovddup	 -2 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovddup	 -1 * SIZE(BO, BI, 8), %xmm3 ;\
+	vfmaddpd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+	vmovups 	-10 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	vfmaddpd  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
+
+#define KERNEL4x3_3(xx) \
+	vmovddup	  0 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	 -8 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovddup	  1 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovddup	  2 * SIZE(BO, BI, 8), %xmm3 ;\
+	vfmaddpd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+	vmovups 	 -6 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	vfmaddpd  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
+
+#define KERNEL4x3_4(xx) \
+	vmovddup	  3 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	 -4 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovddup	  4 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovddup	  5 * SIZE(BO, BI, 8), %xmm3 ;\
+	vfmaddpd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+	vmovups 	 -2 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	vfmaddpd  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
+	addq	$12, BI				  ;\
+	addq	$16, %rax 			  ;\
+
+#define KERNEL4x3_SUB(xx) \
+	vmovddup	 -6 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovddup	 -5 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovddup	 -4 * SIZE(BO, BI, 8), %xmm3 ;\
+	vfmaddpd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+	vmovups 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	vfmaddpd  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
+
+
+
+
+
+/*******************************************************************************************/
+
+#define KERNEL2x3_1(xx) \
+	vmovddup	 -6 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovddup	 -5 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovddup	 -4 * SIZE(BO, BI, 8), %xmm3 ;\
+	vfmaddpd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+
+#define KERNEL2x3_2(xx) \
+	vmovddup	 -3 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovddup	 -2 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovddup	 -1 * SIZE(BO, BI, 8), %xmm3 ;\
+	vfmaddpd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+
+#define KERNEL2x3_3(xx) \
+	vmovddup	  0 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	-12 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovddup	  1 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovddup	  2 * SIZE(BO, BI, 8), %xmm3 ;\
+	vfmaddpd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+
+#define KERNEL2x3_4(xx) \
+	vmovddup	  3 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	-10 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovddup	  4 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovddup	  5 * SIZE(BO, BI, 8), %xmm3 ;\
+	vfmaddpd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+	addq	$12, BI				  ;\
+	addq	$8, %rax 			  ;\
+
+#define KERNEL2x3_SUB(xx) \
+	vmovddup	 -6 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovddup	 -5 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovddup	 -4 * SIZE(BO, BI, 8), %xmm3 ;\
+	vfmaddpd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+
+/*******************************************************************************************/
+
+#define KERNEL1x3_1(xx) \
+	vmovsd	 -6 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovsd 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovsd	 -5 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddsd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovsd	 -4 * SIZE(BO, BI, 8), %xmm3 ;\
+	vfmaddsd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+
+#define KERNEL1x3_2(xx) \
+	vmovsd	 -3 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovsd 	-15 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovsd	 -2 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddsd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovsd	 -1 * SIZE(BO, BI, 8), %xmm3 ;\
+	vfmaddsd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+
+#define KERNEL1x3_3(xx) \
+	vmovsd	  0 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovsd 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovsd	  1 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddsd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovsd	  2 * SIZE(BO, BI, 8), %xmm3 ;\
+	vfmaddsd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+
+#define KERNEL1x3_4(xx) \
+	vmovsd	  3 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovsd 	-13 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovsd	  4 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddsd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovsd	  5 * SIZE(BO, BI, 8), %xmm3 ;\
+	vfmaddsd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+	addq	$12, BI				  ;\
+	addq	$4, %rax 			  ;\
+
+#define KERNEL1x3_SUB(xx) \
+	vmovsd	 -6 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovsd 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovsd	 -5 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddsd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovsd	 -4 * SIZE(BO, BI, 8), %xmm3 ;\
+	vfmaddsd  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+
+
+
+/*******************************************************************************************
+* 2 lines of N
+*******************************************************************************************/
+
+#define KERNEL8x2_1(xx) \
+	prefetcht0	A_PR1(AO,%rax,8)	;\
+	vmovddup	 -4 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovddup	 -3 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovups 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	vmovups 	-12 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
+	vfmaddpd  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
+	vmovups 	-10 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
+	vfmaddpd  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
+
+#define KERNEL8x2_2(xx) \
+	prefetcht0	A_PR1+64(AO,%rax,8)	;\
+	vmovddup	 -2 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	 -8 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovddup	 -1 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovups 	 -6 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	vmovups 	 -4 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
+	vfmaddpd  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
+	vmovups 	 -2 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
+	vfmaddpd  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
+
+#define KERNEL8x2_3(xx) \
+	prefetcht0	A_PR1+128(AO,%rax,8)	;\
+	vmovddup	  0 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	  0 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovddup	  1 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovups 	  2 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	vmovups 	  4 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
+	vfmaddpd  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
+	vmovups 	  6 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
+	vfmaddpd  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
+
+#define KERNEL8x2_4(xx) \
+	prefetcht0	A_PR1+192(AO,%rax,8)	;\
+	vmovddup	  2 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	  8 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovddup	  3 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovups 	 10 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	vmovups 	 12 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
+	vfmaddpd  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
+	vmovups 	 14 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
+	vfmaddpd  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
+	addq	$8, BI				  ;\
+	addq	$32, %rax 			  ;\
+
+#define KERNEL8x2_SUB(xx) \
+	vmovddup	 -4 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovddup	 -3 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovups 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	vmovups 	-12 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
+	vfmaddpd  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
+	vmovups 	-10 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
+	vfmaddpd  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
+
+
+/*******************************************************************************************/
+
+#define KERNEL4x2_1(xx) \
+	vmovddup	 -4 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovddup	 -3 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovups 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+
+#define KERNEL4x2_2(xx) \
+	vmovddup	 -2 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	-12 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovddup	 -1 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovups 	-10 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+
+#define KERNEL4x2_3(xx) \
+	vmovddup	  0 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	 -8 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovddup	  1 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovups 	 -6 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+
+#define KERNEL4x2_4(xx) \
+	vmovddup	  2 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	 -4 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovddup	  3 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovups 	 -2 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	addq	$8, BI				  ;\
+	addq	$16, %rax 			  ;\
+
+#define KERNEL4x2_SUB(xx) \
+	vmovddup	 -4 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovddup	 -3 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovups 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddpd  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+
+
+/*******************************************************************************************/
+
+#define KERNEL2x2_1(xx) \
+	vmovddup	 -4 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovddup	 -3 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+
+#define KERNEL2x2_2(xx) \
+	vmovddup	 -2 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovddup	 -1 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+
+#define KERNEL2x2_3(xx) \
+	vmovddup	  0 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	-12 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovddup	  1 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+
+#define KERNEL2x2_4(xx) \
+	vmovddup	  2 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	-10 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovddup	  3 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	addq	$8, BI				  ;\
+	addq	$8, %rax 			  ;\
+
+#define KERNEL2x2_SUB(xx) \
+	vmovddup	 -4 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovddup	 -3 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddpd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+
+/*******************************************************************************************/
+
+#define KERNEL1x2_1(xx) \
+	vmovsd	 -4 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovsd 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovsd	 -3 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddsd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+
+#define KERNEL1x2_2(xx) \
+	vmovsd	 -2 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovsd 	-15 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovsd	 -1 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddsd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+
+#define KERNEL1x2_3(xx) \
+	vmovsd	  0 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovsd 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovsd	  1 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddsd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+
+#define KERNEL1x2_4(xx) \
+	vmovsd	  2 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovsd 	-13 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovsd	  3 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddsd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	addq	$8, BI				  ;\
+	addq	$4, %rax 			  ;\
+
+#define KERNEL1x2_SUB(xx) \
+	vmovsd	 -4 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovsd 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovsd	 -3 * SIZE(BO, BI, 8), %xmm2 ;\
+	vfmaddsd  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+
+
+
+/*******************************************************************************************
+* 1 line of N
+*******************************************************************************************/
+
+#define KERNEL8x1_1(xx) \
+	prefetcht0	A_PR1(AO,%rax,8)	;\
+	vmovddup	 -2 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovups 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vmovups 	-12 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
+	vmovups 	-10 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
+
+#define KERNEL8x1_2(xx) \
+	prefetcht0	A_PR1+64(AO,%rax,8)	;\
+	vmovddup	 -1 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	 -8 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovups 	 -6 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vmovups 	 -4 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
+	vmovups 	 -2 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
+
+#define KERNEL8x1_3(xx) \
+	prefetcht0	A_PR1+128(AO,%rax,8)	;\
+	vmovddup	  0 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	  0 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovups 	  2 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vmovups 	  4 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
+	vmovups 	  6 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
+
+#define KERNEL8x1_4(xx) \
+	prefetcht0	A_PR1+192(AO,%rax,8)	;\
+	vmovddup	  1 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	  8 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovups 	 10 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vmovups 	 12 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
+	vmovups 	 14 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
+	addq	$4, BI				  ;\
+	addq	$32, %rax 			  ;\
+
+#define KERNEL8x1_SUB(xx) \
+	vmovddup	 -2 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovups 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vmovups 	-12 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
+	vmovups 	-10 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
+
+
+/*******************************************************************************************/
+
+#define KERNEL4x1_1(xx) \
+	vmovddup	 -2 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovups 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+
+#define KERNEL4x1_2(xx) \
+	vmovddup	 -1 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	-12 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovups 	-10 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+
+#define KERNEL4x1_3(xx) \
+	vmovddup	  0 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	 -8 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovups 	 -6 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+
+#define KERNEL4x1_4(xx) \
+	vmovddup	  1 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	 -4 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovups 	 -2 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	addq	$4, BI				  ;\
+	addq	$16, %rax 			  ;\
+
+#define KERNEL4x1_SUB(xx) \
+	vmovddup	 -2 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovups 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+
+
+/*******************************************************************************************/
+
+#define KERNEL2x1_1(xx) \
+	vmovddup	 -2 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+
+#define KERNEL2x1_2(xx) \
+	vmovddup	 -1 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+
+#define KERNEL2x1_3(xx) \
+	vmovddup	  0 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	-12 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+
+#define KERNEL2x1_4(xx) \
+	vmovddup	  1 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	-10 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	addq	$4, BI				  ;\
+	addq	$8, %rax 			  ;\
+
+#define KERNEL2x1_SUB(xx) \
+	vmovddup	 -2 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovups 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddpd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+
+/*******************************************************************************************/
+
+#define KERNEL1x1_1(xx) \
+	vmovsd	 -2 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovsd 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+
+#define KERNEL1x1_2(xx) \
+	vmovsd	 -1 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovsd 	-15 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+
+#define KERNEL1x1_3(xx) \
+	vmovsd	  0 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovsd 	-14 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+
+#define KERNEL1x1_4(xx) \
+	vmovsd	  1 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovsd 	-13 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	addq	$4, BI				  ;\
+	addq	$4, %rax 			  ;\
+
+#define KERNEL1x1_SUB(xx) \
+	vmovsd	 -2 * SIZE(BO, BI, 8), %xmm1 ;\
+	vmovsd 	-16 * SIZE(AO, %rax, 8), %xmm0 ;\
+	vfmaddsd  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+
+
+/*******************************************************************************************/
+
+#if !defined(TRMMKERNEL)
+
+
+	PROLOGUE
+	PROFCODE
+	
+	subq	$STACKSIZE, %rsp
+	movq	%rbx,   (%rsp)
+	movq	%rbp,  8(%rsp)
+	movq	%r12, 16(%rsp)
+	movq	%r13, 24(%rsp)
+	movq	%r14, 32(%rsp)
+	movq	%r15, 40(%rsp)
+
+	vzeroupper
+
+#ifdef WINDOWS_ABI
+	movq	%rdi,    48(%rsp)
+	movq	%rsi,    56(%rsp)
+	movups	%xmm6,   64(%rsp)
+	movups	%xmm7,   80(%rsp)
+	movups	%xmm8,   96(%rsp)
+	movups	%xmm9,  112(%rsp)
+	movups	%xmm10, 128(%rsp)
+	movups	%xmm11, 144(%rsp)
+	movups	%xmm12, 160(%rsp)
+	movups	%xmm13, 176(%rsp)
+	movups	%xmm14, 192(%rsp)
+	movups	%xmm15, 208(%rsp)
+
+	movq	ARG1,      OLD_M
+	movq	ARG2,      OLD_N
+	movq	ARG3,      OLD_K
+	movq	OLD_A,     A
+	movq	OLD_B,     B
+	movq	OLD_C,     C
+	movq	OLD_LDC,   LDC
+
+	vmovaps	%xmm3, %xmm0
+
+#else
+	movq	STACKSIZE +  8(%rsp), LDC
+
+#endif
+
+	movq    %rsp, SP      # save old stack
+        subq    $128 + L_BUFFER_SIZE, %rsp
+        andq    $-4096, %rsp    # align stack
+
+        STACK_TOUCH
+
+	cmpq	$0, OLD_M
+	je	.L999
+
+	cmpq	$0, OLD_N
+	je	.L999
+
+	cmpq	$0, OLD_K
+	je	.L999
+
+	movq	OLD_M, M
+	movq	OLD_N, N
+	movq	OLD_K, K
+
+	vmovsd	 %xmm0, ALPHA
+
+	salq	$BASE_SHIFT, LDC
+
+	movq    N, %rax
+        xorq    %rdx, %rdx
+        movq    $6,  %rdi
+        divq    %rdi                    //    N / 6
+        movq    %rax, Ndiv6             //    N / 6
+        movq    %rdx, Nmod6             //    N % 6
+
+	movq	Ndiv6,  J
+	cmpq	$0, J
+	je	.L2_0
+	ALIGN_4
+
+.L6_01:
+        // copy to sub buffer
+        movq    K, %rax
+        salq    $1,%rax                 // K * 2
+        movq    B, BO1
+        leaq    (B,%rax,8), BO2         // next offset to BO2
+        leaq    BUFFER1, BO             // first buffer to BO
+        movq    K, %rax
+        sarq    $2, %rax                // K / 4
+        jz      .L6_02a
+        ALIGN_4
+
+.L6_02:
+	prefetcht0 B_PR1(BO1)
+	prefetcht0 B_PR1(BO2)
+	prefetchw  B_PR1(BO)
+	vmovups	      (BO1), %xmm0
+	vmovups	2*SIZE(BO1), %xmm2
+	vmovups	4*SIZE(BO1), %xmm4
+	vmovups	6*SIZE(BO1), %xmm6
+	vmovsd        (BO2), %xmm1
+	vmovsd  2*SIZE(BO2), %xmm3
+	vmovsd  4*SIZE(BO2), %xmm5
+	vmovsd  6*SIZE(BO2), %xmm7
+	vmovups	%xmm0,       (BO)
+	vmovsd	%xmm1, 2*SIZE(BO)
+	vmovups	%xmm2, 3*SIZE(BO)
+	vmovsd	%xmm3, 5*SIZE(BO)
+	vmovups	%xmm4, 6*SIZE(BO)
+	vmovsd	%xmm5, 8*SIZE(BO)
+	vmovups	%xmm6, 9*SIZE(BO)
+	vmovsd	%xmm7,11*SIZE(BO)
+	addq	$ 8*SIZE,BO1
+	addq	$ 8*SIZE,BO2
+	addq	$ 12*SIZE,BO
+	decq	%rax
+	jnz	.L6_02
+
+.L6_02a:
+
+	movq	K, %rax
+	andq	$3, %rax		// K % 4
+	jz	.L6_02c
+	ALIGN_4
+
+.L6_02b:
+
+	vmovups	(BO1), %xmm0
+	vmovsd  (BO2), %xmm1
+	vmovups	%xmm0,       (BO)
+	vmovsd	%xmm1, 2*SIZE(BO)
+	addq	$ 2*SIZE,BO1
+	addq	$ 2*SIZE,BO2
+	addq	$ 3*SIZE,BO
+	decq	%rax
+	jnz	.L6_02b
+
+.L6_02c:
+
+	movq	K, %rax
+	salq	$1,%rax			// K * 2
+	leaq	(B,%rax,8), BO1		// next offset to BO1
+	leaq	(BO1,%rax,8), BO2	// next offset to BO1
+	leaq    BUFFER2, BO		// second buffer to BO
+	movq	K, %rax
+	sarq	$2, %rax		// k / 4
+	jz	.L6_03a
+	ALIGN_4
+
+
+.L6_03:
+
+	prefetcht0 B_PR1(BO2)
+	prefetchw  B_PR1(BO)
+	vmovups	      (BO2), %xmm0
+	vmovups	2*SIZE(BO2), %xmm2
+	vmovups	4*SIZE(BO2), %xmm4
+	vmovups	6*SIZE(BO2), %xmm6
+	vmovsd  1*SIZE(BO1), %xmm1
+	vmovsd  3*SIZE(BO1), %xmm3
+	vmovsd  5*SIZE(BO1), %xmm5
+	vmovsd  7*SIZE(BO1), %xmm7
+	vmovsd	%xmm1, 0*SIZE(BO)
+	vmovups	%xmm0, 1*SIZE(BO)
+	vmovsd	%xmm3, 3*SIZE(BO)
+	vmovups	%xmm2, 4*SIZE(BO)
+	vmovsd	%xmm5, 6*SIZE(BO)
+	vmovups	%xmm4, 7*SIZE(BO)
+	vmovsd	%xmm7, 9*SIZE(BO)
+	vmovups	%xmm6,10*SIZE(BO)
+	addq	$ 8*SIZE,BO1
+	addq	$ 8*SIZE,BO2
+	addq	$ 12*SIZE,BO
+	decq	%rax
+	jnz	.L6_03
+
+.L6_03a:
+
+        movq    K, %rax
+        andq    $3, %rax                // K % 4
+        jz      .L6_03c
+        ALIGN_4
+
+
+.L6_03b:
+
+	vmovsd	  1*SIZE(BO1), %xmm0
+	vmovups  	(BO2), %xmm1
+	vmovsd	%xmm0,       (BO)
+	vmovups %xmm1, 1*SIZE(BO)
+	addq	$ 2*SIZE,BO1
+	addq	$ 2*SIZE,BO2
+	addq	$ 3*SIZE,BO
+	decq	%rax
+	jnz	.L6_03b
+
+
+.L6_03c:
+
+	movq	BO2, B			// next offset of B
+
+.L6_10:
+	movq	C, CO1
+	leaq	(C, LDC, 2), C		 
+	leaq	(C, LDC, 1), C		// c += 3 * ldc
+
+	
+	movq	A, AO		 	// aoffset = a
+	addq	$16 * SIZE, AO
+
+	movq	M,  I
+	sarq	$3, I			// i = (m >> 3)
+	je	.L6_20
+
+	ALIGN_4
+
+.L6_11:
+
+        leaq    BUFFER1, BO             // first buffer to BO
+	addq	$12 * SIZE, BO
+        movq    K, %rax
+	sarq	$3, %rax			//  K / 8
+	cmpq	$3, %rax
+	jl	.L6_13
+
+	prefetcht0 B_PR1(BO)
+	prefetcht0 B_PR1+64(BO)
+	prefetcht0 B_PR1+128(BO)
+	KERNEL8x3_INIT
+	KERNEL8x3_M2
+	KERNEL8x3_M3
+	KERNEL8x3_M4
+	KERNEL8x3_M5
+	KERNEL8x3_M6
+	KERNEL8x3_M7
+	KERNEL8x3_M8
+
+	subq	$2, %rax
+
+	ALIGN_5
+
+.L6_12:
+
+	prefetcht0 B_PR1-24(BO)
+	prefetcht0 B_PR1+40(BO)
+	KERNEL8x3_M1
+	KERNEL8x3_M2
+	KERNEL8x3_M3
+	KERNEL8x3_M4
+	KERNEL8x3_M5
+	prefetcht0 B_PR1+104(BO)
+	KERNEL8x3_M6
+	KERNEL8x3_M7
+	KERNEL8x3_M8
+
+	dec	%rax
+	jne	.L6_12
+
+.L6_12_E:
+
+	prefetcht0 B_PR1(BO)
+	prefetcht0 B_PR1+64(BO)
+	KERNEL8x3_M1
+	KERNEL8x3_M2
+	KERNEL8x3_M3
+	KERNEL8x3_M4
+	KERNEL8x3_M5
+	KERNEL8x3_M6
+	KERNEL8x3_M7
+	KERNEL8x3_E
+
+	jmp	.L6_16
+
+.L6_13:
+
+	test $2, %rax
+	jz	.L6_14
+
+	KERNEL8x3_INIT
+	KERNEL8x3_M2
+	KERNEL8x3_M3
+	KERNEL8x3_M4
+	KERNEL8x3_M5
+	KERNEL8x3_M6
+	KERNEL8x3_M7
+	KERNEL8x3_M8
+
+	KERNEL8x3_M1
+	KERNEL8x3_M2
+	KERNEL8x3_M3
+	KERNEL8x3_M4
+	KERNEL8x3_M5
+	KERNEL8x3_M6
+	KERNEL8x3_M7
+	KERNEL8x3_E
+
+	jmp	.L6_16
+
+
+.L6_14:
+
+	test $1, %rax
+	jz	.L6_15
+	
+	KERNEL8x3_INIT
+	KERNEL8x3_M2
+	KERNEL8x3_M3
+	KERNEL8x3_M4
+	KERNEL8x3_M5
+	KERNEL8x3_M6
+	KERNEL8x3_M7
+	KERNEL8x3_E
+
+
+	jmp	.L6_16
+
+.L6_15:
+
+	INIT8x3
+
+.L6_16:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L6_19
+
+	ALIGN_4
+
+.L6_17:
+
+	KERNEL8x3_SUBN
+	dec	%rax
+	jne	.L6_17
+	ALIGN_4
+
+
+.L6_19:
+
+	SAVE8x3
+
+	decq	I			# i --
+	jg	.L6_11
+
+/**************************************************************************
+* Rest of M 
+***************************************************************************/
+.L6_20:
+	// Test rest of M
+
+	testq	$7, M
+	jz	.L7_10		// to next 3 lines of N
+
+	testq	$4, M		
+	jz	.L6_30
+
+	ALIGN_4
+
+.L6_21:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $6 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L6_26
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L6_22:
+
+	KERNEL4x3_1(xxx)
+	KERNEL4x3_2(xxx)
+	KERNEL4x3_3(xxx)
+	KERNEL4x3_4(xxx)
+
+	KERNEL4x3_1(xxx)
+	KERNEL4x3_2(xxx)
+	KERNEL4x3_3(xxx)
+	KERNEL4x3_4(xxx)
+
+	je	.L6_26
+
+	KERNEL4x3_1(xxx)
+	KERNEL4x3_2(xxx)
+	KERNEL4x3_3(xxx)
+	KERNEL4x3_4(xxx)
+
+	KERNEL4x3_1(xxx)
+	KERNEL4x3_2(xxx)
+	KERNEL4x3_3(xxx)
+	KERNEL4x3_4(xxx)
+
+	je	.L6_26
+
+	jmp	.L6_22
+	ALIGN_4
+
+.L6_26:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L6_29
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L6_27:
+
+	KERNEL4x3_SUB(xxx)
+	addq	$3, BI
+	addq	$4, %rax
+	jl	.L6_27
+	ALIGN_4
+
+
+.L6_29:
+
+	vmovddup	ALPHA, %xmm0
+
+
+	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7
+
+	vfmaddpd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+	vfmaddpd 2 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
+
+	vfmaddpd 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
+	vfmaddpd 2 * SIZE(CO1, LDC, 2),%xmm0, %xmm9,%xmm9
+
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm7 , 2 * SIZE(CO1)
+
+	vmovups	%xmm5 ,  	(CO1, LDC)
+	vmovups	%xmm8 , 2 * SIZE(CO1, LDC)
+
+	vmovups	%xmm6 ,  	(CO1, LDC, 2)
+	vmovups	%xmm9 , 2 * SIZE(CO1, LDC, 2)
+
+
+	addq	$4 * SIZE, CO1		# coffset += 4
+	ALIGN_4
+	
+
+.L6_30:
+	testq	$2, M		
+	jz	.L6_40
+
+	ALIGN_4
+
+.L6_31:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $6 * SIZE, BO
+
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L6_36
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L6_32:
+
+	KERNEL2x3_1(xxx)
+	KERNEL2x3_2(xxx)
+	KERNEL2x3_3(xxx)
+	KERNEL2x3_4(xxx)
+
+	KERNEL2x3_1(xxx)
+	KERNEL2x3_2(xxx)
+	KERNEL2x3_3(xxx)
+	KERNEL2x3_4(xxx)
+
+	je	.L6_36
+
+	KERNEL2x3_1(xxx)
+	KERNEL2x3_2(xxx)
+	KERNEL2x3_3(xxx)
+	KERNEL2x3_4(xxx)
+
+	KERNEL2x3_1(xxx)
+	KERNEL2x3_2(xxx)
+	KERNEL2x3_3(xxx)
+	KERNEL2x3_4(xxx)
+
+	je	.L6_36
+
+	jmp	.L6_32
+	ALIGN_4
+
+.L6_36:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L6_39
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+	
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L6_37:
+
+	KERNEL2x3_SUB(xxx)
+	addq	$3, BI
+	addq	$2, %rax
+	jl	.L6_37
+	ALIGN_4
+
+
+.L6_39:
+
+	vmovddup	ALPHA, %xmm0
+
+
+	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddpd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+	vfmaddpd 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
+
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm5 ,  	(CO1, LDC)
+	vmovups	%xmm6 ,  	(CO1, LDC, 2)
+
+
+	addq	$2 * SIZE, CO1		# coffset += 2
+	ALIGN_4
+
+.L6_40:
+	testq	$1, M		
+	jz	.L7_10		// to next 3 lines of N
+
+	ALIGN_4
+
+.L6_41:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $6 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L6_46
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L6_42:
+
+	KERNEL1x3_1(xxx)
+	KERNEL1x3_2(xxx)
+	KERNEL1x3_3(xxx)
+	KERNEL1x3_4(xxx)
+
+	KERNEL1x3_1(xxx)
+	KERNEL1x3_2(xxx)
+	KERNEL1x3_3(xxx)
+	KERNEL1x3_4(xxx)
+
+	je	.L6_46
+
+	KERNEL1x3_1(xxx)
+	KERNEL1x3_2(xxx)
+	KERNEL1x3_3(xxx)
+	KERNEL1x3_4(xxx)
+
+	KERNEL1x3_1(xxx)
+	KERNEL1x3_2(xxx)
+	KERNEL1x3_3(xxx)
+	KERNEL1x3_4(xxx)
+
+	je	.L6_46
+
+	jmp	.L6_42
+	ALIGN_4
+
+.L6_46:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L6_49
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L6_47:
+
+	KERNEL1x3_SUB(xxx)
+	addq	$3, BI
+	addq	$1, %rax
+	jl	.L6_47
+	ALIGN_4
+
+
+.L6_49:
+
+	vmovddup	ALPHA, %xmm0
+
+
+	vfmaddsd 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddsd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+	vfmaddsd 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
+
+	vmovsd	%xmm4 ,  	(CO1)
+	vmovsd	%xmm5 ,  	(CO1, LDC)
+	vmovsd	%xmm6 ,  	(CO1, LDC, 2)
+
+
+	addq	$1 * SIZE, CO1		# coffset += 1
+	ALIGN_4
+	
+
+
+
+/***************************************************************************************************************/
+
+.L7_10:
+	movq	C, CO1
+	leaq	(C, LDC, 2), C		 
+	leaq	(C, LDC, 1), C		// c += 3 * ldc
+
+
+	movq	A, AO		 	// aoffset = a
+	addq	$16 * SIZE, AO
+
+	movq	M,  I
+	sarq	$3, I			// i = (m >> 3)
+	je	.L7_20
+	ALIGN_4
+
+.L7_11:
+
+        leaq    BUFFER2, BO             // first buffer to BO
+	addq	$12 * SIZE, BO
+        movq    K, %rax
+	sarq	$3, %rax			//  K / 8
+	cmpq	$3, %rax
+	jl	.L7_13
+
+	prefetcht0 B_PR1(BO)
+	prefetcht0 B_PR1+64(BO)
+	prefetcht0 B_PR1+128(BO)
+	KERNEL8x3_INIT
+	KERNEL8x3_M2
+	KERNEL8x3_M3
+	KERNEL8x3_M4
+	KERNEL8x3_M5
+	KERNEL8x3_M6
+	KERNEL8x3_M7
+	KERNEL8x3_M8
+
+	subq	$2, %rax
+
+	ALIGN_5
+
+.L7_12:
+
+	prefetcht0 B_PR1-24(BO)
+	prefetcht0 B_PR1+40(BO)
+	KERNEL8x3_M1
+	KERNEL8x3_M2
+	KERNEL8x3_M3
+	KERNEL8x3_M4
+	prefetcht0 B_PR1+104(BO)
+	KERNEL8x3_M5
+	KERNEL8x3_M6
+	KERNEL8x3_M7
+	KERNEL8x3_M8
+
+	dec	%rax
+	jne	.L7_12
+
+.L7_12_E:
+
+	prefetcht0 B_PR1(BO)
+	prefetcht0 B_PR1+64(BO)
+	KERNEL8x3_M1
+	KERNEL8x3_M2
+	KERNEL8x3_M3
+	KERNEL8x3_M4
+	KERNEL8x3_M5
+	KERNEL8x3_M6
+	KERNEL8x3_M7
+	KERNEL8x3_E
+
+	jmp	.L7_16
+
+
+
+.L7_13:
+
+	test $2, %rax
+	jz	.L7_14
+
+	KERNEL8x3_INIT
+	KERNEL8x3_M2
+	KERNEL8x3_M3
+	KERNEL8x3_M4
+	KERNEL8x3_M5
+	KERNEL8x3_M6
+	KERNEL8x3_M7
+	KERNEL8x3_M8
+
+	KERNEL8x3_M1
+	KERNEL8x3_M2
+	KERNEL8x3_M3
+	KERNEL8x3_M4
+	KERNEL8x3_M5
+	KERNEL8x3_M6
+	KERNEL8x3_M7
+	KERNEL8x3_E
+
+	jmp	.L7_16
+
+
+.L7_14:
+
+	test $1, %rax
+	jz	.L7_15
+	
+	KERNEL8x3_INIT
+	KERNEL8x3_M2
+	KERNEL8x3_M3
+	KERNEL8x3_M4
+	KERNEL8x3_M5
+	KERNEL8x3_M6
+	KERNEL8x3_M7
+	KERNEL8x3_E
+
+	jmp	.L7_16
+
+
+
+.L7_15:
+
+	INIT8x3
+
+.L7_16:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L7_19
+
+
+	ALIGN_4
+
+.L7_17:
+
+	KERNEL8x3_SUBN
+	dec	%rax
+	jne	.L7_17
+	ALIGN_4
+
+
+.L7_19:
+
+	SAVE8x3
+
+	decq	I			# i --
+	jg	.L7_11
+	ALIGN_4	
+
+.L7_20:
+	// Test rest of M
+
+	testq	$7, M
+	jz	.L7_60		// to next 6 lines of N
+
+	testq	$4, M		
+	jz	.L7_30
+
+	ALIGN_4
+
+.L7_21:
+        leaq    BUFFER2, BO             // second buffer to BO
+        addq    $6 * SIZE, BO
+
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L7_26
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L7_22:
+
+	KERNEL4x3_1(xxx)
+	KERNEL4x3_2(xxx)
+	KERNEL4x3_3(xxx)
+	KERNEL4x3_4(xxx)
+
+	KERNEL4x3_1(xxx)
+	KERNEL4x3_2(xxx)
+	KERNEL4x3_3(xxx)
+	KERNEL4x3_4(xxx)
+
+	je	.L7_26
+
+	KERNEL4x3_1(xxx)
+	KERNEL4x3_2(xxx)
+	KERNEL4x3_3(xxx)
+	KERNEL4x3_4(xxx)
+
+	KERNEL4x3_1(xxx)
+	KERNEL4x3_2(xxx)
+	KERNEL4x3_3(xxx)
+	KERNEL4x3_4(xxx)
+
+	je	.L7_26
+
+	jmp	.L7_22
+	ALIGN_4
+
+.L7_26:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L7_29
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L7_27:
+
+	KERNEL4x3_SUB(xxx)
+	addq	$3, BI
+	addq	$4, %rax
+	jl	.L7_27
+	ALIGN_4
+
+
+.L7_29:
+
+	vmovddup	ALPHA, %xmm0
+
+
+	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7
+
+	vfmaddpd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+	vfmaddpd 2 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
+
+	vfmaddpd 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
+	vfmaddpd 2 * SIZE(CO1, LDC, 2),%xmm0, %xmm9,%xmm9
+
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm7 , 2 * SIZE(CO1)
+
+	vmovups	%xmm5 ,  	(CO1, LDC)
+	vmovups	%xmm8 , 2 * SIZE(CO1, LDC)
+
+	vmovups	%xmm6 ,  	(CO1, LDC, 2)
+	vmovups	%xmm9 , 2 * SIZE(CO1, LDC, 2)
+
+
+	addq	$4 * SIZE, CO1		# coffset += 4
+	ALIGN_4
+	
+
+.L7_30:
+	testq	$2, M		
+	jz	.L7_40
+
+	ALIGN_4
+
+.L7_31:
+        leaq    BUFFER2, BO             // second buffer to BO
+        addq    $6 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L7_36
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L7_32:
+
+	KERNEL2x3_1(xxx)
+	KERNEL2x3_2(xxx)
+	KERNEL2x3_3(xxx)
+	KERNEL2x3_4(xxx)
+
+	KERNEL2x3_1(xxx)
+	KERNEL2x3_2(xxx)
+	KERNEL2x3_3(xxx)
+	KERNEL2x3_4(xxx)
+
+	je	.L7_36
+
+	KERNEL2x3_1(xxx)
+	KERNEL2x3_2(xxx)
+	KERNEL2x3_3(xxx)
+	KERNEL2x3_4(xxx)
+
+	KERNEL2x3_1(xxx)
+	KERNEL2x3_2(xxx)
+	KERNEL2x3_3(xxx)
+	KERNEL2x3_4(xxx)
+
+	je	.L7_36
+
+	jmp	.L7_32
+	ALIGN_4
+
+.L7_36:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L7_39
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L7_37:
+
+	KERNEL2x3_SUB(xxx)
+	addq	$3, BI
+	addq	$2, %rax
+	jl	.L7_37
+	ALIGN_4
+
+
+.L7_39:
+
+	vmovddup	ALPHA, %xmm0
+
+
+	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddpd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+	vfmaddpd 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
+
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm5 ,  	(CO1, LDC)
+	vmovups	%xmm6 ,  	(CO1, LDC, 2)
+
+
+	addq	$2 * SIZE, CO1		# coffset += 2
+	ALIGN_4
+	
+
+
+
+
+.L7_40:
+	testq	$1, M		
+	jz	.L7_60		// to next 6 lines of N
+
+	ALIGN_4
+
+.L7_41:
+        leaq    BUFFER2, BO             // second buffer to BO
+        addq    $6 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+
+	andq	$-8, %rax
+	je	.L7_46
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L7_42:
+
+	KERNEL1x3_1(xxx)
+	KERNEL1x3_2(xxx)
+	KERNEL1x3_3(xxx)
+	KERNEL1x3_4(xxx)
+
+	KERNEL1x3_1(xxx)
+	KERNEL1x3_2(xxx)
+	KERNEL1x3_3(xxx)
+	KERNEL1x3_4(xxx)
+
+	je	.L7_46
+
+	KERNEL1x3_1(xxx)
+	KERNEL1x3_2(xxx)
+	KERNEL1x3_3(xxx)
+	KERNEL1x3_4(xxx)
+
+	KERNEL1x3_1(xxx)
+	KERNEL1x3_2(xxx)
+	KERNEL1x3_3(xxx)
+	KERNEL1x3_4(xxx)
+
+	je	.L7_46
+
+	jmp	.L7_42
+	ALIGN_4
+
+.L7_46:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L7_49
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L7_47:
+
+	KERNEL1x3_SUB(xxx)
+	addq	$3, BI
+	addq	$1, %rax
+	jl	.L7_47
+	ALIGN_4
+
+
+.L7_49:
+
+	vmovddup	ALPHA, %xmm0
+
+
+	vfmaddsd 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddsd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+	vfmaddsd 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
+
+
+	vmovsd	%xmm4 ,  	(CO1)
+	vmovsd	%xmm5 ,  	(CO1, LDC)
+	vmovsd	%xmm6 ,  	(CO1, LDC, 2)
+
+
+	addq	$1 * SIZE, CO1		# coffset += 1
+
+.L7_60:
+
+	decq	J			// j --
+	jg	.L6_01
+
+
+.L2_0:
+	cmpq	$0, Nmod6		// N % 6 == 0
+	je	.L999
+
+/************************************************************************************************
+* Loop for Nmod6 / 2 > 0
+*************************************************************************************************/
+
+	movq	Nmod6, J		
+	sarq	$1, J			// j = j / 2
+	je	.L1_0
+	ALIGN_4
+
+.L2_01:
+	// copy to sub buffer
+	movq	B, BO1
+	leaq    BUFFER1, BO		// first buffer to BO
+	movq	K, %rax
+	ALIGN_4
+
+.L2_02b:
+
+	vmovups	(BO1), %xmm0
+	vmovups	%xmm0,       (BO)
+	addq	$2*SIZE,BO1
+	addq	$2*SIZE,BO
+	decq	%rax
+	jnz	.L2_02b
+
+.L2_02c:
+
+	movq	BO1, B			// next offset of B
+
+.L2_10:
+	movq	C, CO1
+	leaq	(C, LDC, 2), C		// c += 2 * ldc
+
+	
+	movq	A, AO		 	// aoffset = a
+	addq	$16 * SIZE, AO
+
+	movq	M,  I
+	sarq	$3, I			// i = (m >> 3)
+	je	.L2_20
+
+	ALIGN_4
+
+.L2_11:
+
+	leaq	BUFFER1, BO		// first buffer to BO
+	addq	$4 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax			//  K = K - ( K % 8 )
+	je	.L2_16
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_12:
+
+	KERNEL8x2_1(xxx)
+	KERNEL8x2_2(xxx)
+	KERNEL8x2_3(xxx)
+	KERNEL8x2_4(xxx)
+
+	KERNEL8x2_1(xxx)
+	KERNEL8x2_2(xxx)
+	KERNEL8x2_3(xxx)
+	KERNEL8x2_4(xxx)
+
+	je	.L2_16
+
+	KERNEL8x2_1(xxx)
+	KERNEL8x2_2(xxx)
+	KERNEL8x2_3(xxx)
+	KERNEL8x2_4(xxx)
+
+	KERNEL8x2_1(xxx)
+	KERNEL8x2_2(xxx)
+	KERNEL8x2_3(xxx)
+	KERNEL8x2_4(xxx)
+
+	je	.L2_16
+
+	jmp	.L2_12
+	ALIGN_4
+
+.L2_16:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_19
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_17:
+
+	KERNEL8x2_SUB(xxx)
+	addq	$2, BI
+	addq	$8, %rax
+	jl	.L2_17
+	ALIGN_4
+
+
+.L2_19:
+
+	vmovddup	ALPHA, %xmm0
+
+	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7
+	vfmaddpd 4 * SIZE(CO1),%xmm0, %xmm10,%xmm10
+	vfmaddpd 6 * SIZE(CO1),%xmm0, %xmm13,%xmm13
+
+	vfmaddpd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+	vfmaddpd 2 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
+	vfmaddpd 4 * SIZE(CO1, LDC),%xmm0, %xmm11,%xmm11
+	vfmaddpd 6 * SIZE(CO1, LDC),%xmm0, %xmm14,%xmm14
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm7 , 2 * SIZE(CO1)
+	vmovups	%xmm10, 4 * SIZE(CO1)
+	vmovups	%xmm13, 6 * SIZE(CO1)
+
+	vmovups	%xmm5 ,  	(CO1, LDC)
+	vmovups	%xmm8 , 2 * SIZE(CO1, LDC)
+	vmovups	%xmm11, 4 * SIZE(CO1, LDC)
+	vmovups	%xmm14, 6 * SIZE(CO1, LDC)
+
+
+	addq	$8 * SIZE, CO1		# coffset += 8
+	decq	I			# i --
+	jg	.L2_11
+	ALIGN_4	
+
+/**************************************************************************
+* Rest of M 
+***************************************************************************/
+.L2_20:
+	// Test rest of M
+
+	testq	$7, M
+	jz	.L2_60		// to next 2 lines of N
+
+	testq	$4, M		
+	jz	.L2_30
+
+	ALIGN_4
+
+.L2_21:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L2_26
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_22:
+
+	KERNEL4x2_1(xxx)
+	KERNEL4x2_2(xxx)
+	KERNEL4x2_3(xxx)
+	KERNEL4x2_4(xxx)
+
+	KERNEL4x2_1(xxx)
+	KERNEL4x2_2(xxx)
+	KERNEL4x2_3(xxx)
+	KERNEL4x2_4(xxx)
+
+	je	.L2_26
+
+	KERNEL4x2_1(xxx)
+	KERNEL4x2_2(xxx)
+	KERNEL4x2_3(xxx)
+	KERNEL4x2_4(xxx)
+
+	KERNEL4x2_1(xxx)
+	KERNEL4x2_2(xxx)
+	KERNEL4x2_3(xxx)
+	KERNEL4x2_4(xxx)
+
+	je	.L2_26
+
+	jmp	.L2_22
+	ALIGN_4
+
+.L2_26:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_29
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_27:
+
+	KERNEL4x2_SUB(xxx)
+	addq	$2, BI
+	addq	$4, %rax
+	jl	.L2_27
+	ALIGN_4
+
+
+.L2_29:
+
+	vmovddup	ALPHA, %xmm0
+
+	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7
+
+	vfmaddpd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+	vfmaddpd 2 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm7 , 2 * SIZE(CO1)
+
+	vmovups	%xmm5 ,  	(CO1, LDC)
+	vmovups	%xmm8 , 2 * SIZE(CO1, LDC)
+
+	addq	$4 * SIZE, CO1		# coffset += 4
+	ALIGN_4
+	
+
+.L2_30:
+	testq	$2, M		
+	jz	.L2_40
+
+	ALIGN_4
+
+.L2_31:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L2_36
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_32:
+
+	KERNEL2x2_1(xxx)
+	KERNEL2x2_2(xxx)
+	KERNEL2x2_3(xxx)
+	KERNEL2x2_4(xxx)
+
+	KERNEL2x2_1(xxx)
+	KERNEL2x2_2(xxx)
+	KERNEL2x2_3(xxx)
+	KERNEL2x2_4(xxx)
+
+	je	.L2_36
+
+	KERNEL2x2_1(xxx)
+	KERNEL2x2_2(xxx)
+	KERNEL2x2_3(xxx)
+	KERNEL2x2_4(xxx)
+
+	KERNEL2x2_1(xxx)
+	KERNEL2x2_2(xxx)
+	KERNEL2x2_3(xxx)
+	KERNEL2x2_4(xxx)
+
+	je	.L2_36
+
+	jmp	.L2_32
+	ALIGN_4
+
+.L2_36:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_39
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+	
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_37:
+
+	KERNEL2x2_SUB(xxx)
+	addq	$2, BI
+	addq	$2, %rax
+	jl	.L2_37
+	ALIGN_4
+
+
+.L2_39:
+
+	vmovddup	ALPHA, %xmm0
+
+	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddpd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm5 ,  	(CO1, LDC)
+
+	addq	$2 * SIZE, CO1		# coffset += 2
+	ALIGN_4
+	
+
+.L2_40:
+	testq	$1, M		
+	jz	.L2_60		// to next 2 lines of N
+
+	ALIGN_4
+
+.L2_41:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L2_46
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_42:
+
+	KERNEL1x2_1(xxx)
+	KERNEL1x2_2(xxx)
+	KERNEL1x2_3(xxx)
+	KERNEL1x2_4(xxx)
+
+	KERNEL1x2_1(xxx)
+	KERNEL1x2_2(xxx)
+	KERNEL1x2_3(xxx)
+	KERNEL1x2_4(xxx)
+
+	je	.L2_46
+
+	KERNEL1x2_1(xxx)
+	KERNEL1x2_2(xxx)
+	KERNEL1x2_3(xxx)
+	KERNEL1x2_4(xxx)
+
+	KERNEL1x2_1(xxx)
+	KERNEL1x2_2(xxx)
+	KERNEL1x2_3(xxx)
+	KERNEL1x2_4(xxx)
+
+	je	.L2_46
+
+	jmp	.L2_42
+	ALIGN_4
+
+.L2_46:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_49
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_47:
+
+	KERNEL1x2_SUB(xxx)
+	addq	$2, BI
+	addq	$1, %rax
+	jl	.L2_47
+	ALIGN_4
+
+
+.L2_49:
+
+	vmovddup	ALPHA, %xmm0
+
+	vfmaddsd 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddsd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+
+	vmovsd	%xmm4 ,  	(CO1)
+	vmovsd	%xmm5 ,  	(CO1, LDC)
+
+
+	addq	$1 * SIZE, CO1		# coffset += 1
+	ALIGN_4
+	
+
+	
+.L2_60:
+
+	decq	J			// j --
+	jg	.L2_01			// next 2 lines of N
+
+
+
+.L1_0:
+
+/************************************************************************************************
+* Loop for Nmod6 % 2 > 0
+*************************************************************************************************/
+
+	movq	Nmod6, J		
+	andq	$1, J			// j % 2
+	je	.L999
+	ALIGN_4
+
+.L1_01:
+	// copy to sub buffer
+	movq	B, BO1
+	leaq    BUFFER1, BO		// first buffer to BO
+	movq	K, %rax
+	ALIGN_4
+
+.L1_02b:
+
+	vmovsd	(BO1), %xmm0
+	vmovsd	%xmm0,       (BO)
+	addq	$1*SIZE,BO1
+	addq	$1*SIZE,BO
+	decq	%rax
+	jnz	.L1_02b
+
+.L1_02c:
+
+	movq	BO1, B			// next offset of B
+
+.L1_10:
+	movq	C, CO1
+	leaq	(C, LDC, 1), C		// c += 1 * ldc
+
+	movq	A, AO		 	// aoffset = a
+	addq	$16 * SIZE, AO
+
+	movq	M,  I
+	sarq	$3, I			// i = (m >> 3)
+	je	.L1_20
+
+	ALIGN_4
+
+.L1_11:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax			//  K = K - ( K % 8 )
+	je	.L1_16
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_12:
+
+	KERNEL8x1_1(xxx)
+	KERNEL8x1_2(xxx)
+	KERNEL8x1_3(xxx)
+	KERNEL8x1_4(xxx)
+
+	KERNEL8x1_1(xxx)
+	KERNEL8x1_2(xxx)
+	KERNEL8x1_3(xxx)
+	KERNEL8x1_4(xxx)
+
+	je	.L1_16
+
+	KERNEL8x1_1(xxx)
+	KERNEL8x1_2(xxx)
+	KERNEL8x1_3(xxx)
+	KERNEL8x1_4(xxx)
+
+	KERNEL8x1_1(xxx)
+	KERNEL8x1_2(xxx)
+	KERNEL8x1_3(xxx)
+	KERNEL8x1_4(xxx)
+
+	je	.L1_16
+
+	jmp	.L1_12
+	ALIGN_4
+
+.L1_16:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_19
+
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_17:
+
+	KERNEL8x1_SUB(xxx)
+	addq	$1, BI
+	addq	$8, %rax
+	jl	.L1_17
+	ALIGN_4
+
+
+.L1_19:
+
+	vmovddup	ALPHA, %xmm0
+
+	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7
+	vfmaddpd 4 * SIZE(CO1),%xmm0, %xmm10,%xmm10
+	vfmaddpd 6 * SIZE(CO1),%xmm0, %xmm13,%xmm13
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm7 , 2 * SIZE(CO1)
+	vmovups	%xmm10, 4 * SIZE(CO1)
+	vmovups	%xmm13, 6 * SIZE(CO1)
+
+	addq	$8 * SIZE, CO1		# coffset += 8
+	decq	I			# i --
+	jg	.L1_11
+	ALIGN_4	
+
+/**************************************************************************
+* Rest of M 
+***************************************************************************/
+.L1_20:
+	// Test rest of M
+
+	testq	$7, M
+	jz	.L999
+
+	testq	$4, M		
+	jz	.L1_30
+
+	ALIGN_4
+
+.L1_21:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L1_26
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_22:
+
+	KERNEL4x1_1(xxx)
+	KERNEL4x1_2(xxx)
+	KERNEL4x1_3(xxx)
+	KERNEL4x1_4(xxx)
+
+	KERNEL4x1_1(xxx)
+	KERNEL4x1_2(xxx)
+	KERNEL4x1_3(xxx)
+	KERNEL4x1_4(xxx)
+
+	je	.L1_26
+
+	KERNEL4x1_1(xxx)
+	KERNEL4x1_2(xxx)
+	KERNEL4x1_3(xxx)
+	KERNEL4x1_4(xxx)
+
+	KERNEL4x1_1(xxx)
+	KERNEL4x1_2(xxx)
+	KERNEL4x1_3(xxx)
+	KERNEL4x1_4(xxx)
+
+	je	.L1_26
+
+	jmp	.L1_22
+	ALIGN_4
+
+.L1_26:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_29
+
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_27:
+
+	KERNEL4x1_SUB(xxx)
+	addq	$1, BI
+	addq	$4, %rax
+	jl	.L1_27
+	ALIGN_4
+
+
+.L1_29:
+
+	vmovddup	ALPHA, %xmm0
+
+	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm7 , 2 * SIZE(CO1)
+
+	addq	$4 * SIZE, CO1		# coffset += 4
+	ALIGN_4
+	
+
+.L1_30:
+	testq	$2, M		
+	jz	.L1_40
+
+	ALIGN_4
+
+.L1_31:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L1_36
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_32:
+
+	KERNEL2x1_1(xxx)
+	KERNEL2x1_2(xxx)
+	KERNEL2x1_3(xxx)
+	KERNEL2x1_4(xxx)
+
+	KERNEL2x1_1(xxx)
+	KERNEL2x1_2(xxx)
+	KERNEL2x1_3(xxx)
+	KERNEL2x1_4(xxx)
+
+	je	.L1_36
+
+	KERNEL2x1_1(xxx)
+	KERNEL2x1_2(xxx)
+	KERNEL2x1_3(xxx)
+	KERNEL2x1_4(xxx)
+
+	KERNEL2x1_1(xxx)
+	KERNEL2x1_2(xxx)
+	KERNEL2x1_3(xxx)
+	KERNEL2x1_4(xxx)
+
+	je	.L1_36
+
+	jmp	.L1_32
+	ALIGN_4
+
+.L1_36:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_39
+
+	movq    %rax, BI                        //  Index for BO
+	
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_37:
+
+	KERNEL2x1_SUB(xxx)
+	addq	$1, BI
+	addq	$2, %rax
+	jl	.L1_37
+	ALIGN_4
+
+
+.L1_39:
+
+	vmovddup	ALPHA, %xmm0
+
+	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
+
+	vmovups	%xmm4 ,  	(CO1)
+
+	addq	$2 * SIZE, CO1		# coffset += 2
+	ALIGN_4
+	
+
+.L1_40:
+	testq	$1, M		
+	jz	.L999
+
+	ALIGN_4
+
+.L1_41:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L1_46
+	movq    %rax, BI                        //  Index for BO
+
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_42:
+
+	KERNEL1x1_1(xxx)
+	KERNEL1x1_2(xxx)
+	KERNEL1x1_3(xxx)
+	KERNEL1x1_4(xxx)
+
+	KERNEL1x1_1(xxx)
+	KERNEL1x1_2(xxx)
+	KERNEL1x1_3(xxx)
+	KERNEL1x1_4(xxx)
+
+	je	.L1_46
+
+	KERNEL1x1_1(xxx)
+	KERNEL1x1_2(xxx)
+	KERNEL1x1_3(xxx)
+	KERNEL1x1_4(xxx)
+
+	KERNEL1x1_1(xxx)
+	KERNEL1x1_2(xxx)
+	KERNEL1x1_3(xxx)
+	KERNEL1x1_4(xxx)
+
+	je	.L1_46
+
+	jmp	.L1_42
+	ALIGN_4
+
+.L1_46:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_49
+
+	movq    %rax, BI                        //  Index for BO
+
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_47:
+
+	KERNEL1x1_SUB(xxx)
+	addq	$1, BI
+	addq	$1, %rax
+	jl	.L1_47
+	ALIGN_4
+
+
+.L1_49:
+
+	vmovddup	ALPHA, %xmm0
+
+	vfmaddsd 	 (CO1),%xmm0, %xmm4,%xmm4
+
+	vmovsd	%xmm4 ,  	(CO1)
+
+	addq	$1 * SIZE, CO1		# coffset += 1
+	ALIGN_4
+	
+
+.L999:
+	movq   		SP, %rsp
+	movq	   (%rsp), %rbx
+	movq	  8(%rsp), %rbp
+	movq	 16(%rsp), %r12
+	movq	 24(%rsp), %r13
+	movq	 32(%rsp), %r14
+	movq	 40(%rsp), %r15
+
+#ifdef WINDOWS_ABI
+	movq	 48(%rsp), %rdi
+	movq	 56(%rsp), %rsi
+	movups	 64(%rsp), %xmm6
+	movups	 80(%rsp), %xmm7
+	movups	 96(%rsp), %xmm8
+	movups	112(%rsp), %xmm9
+	movups	128(%rsp), %xmm10
+	movups	144(%rsp), %xmm11
+	movups	160(%rsp), %xmm12
+	movups	176(%rsp), %xmm13
+	movups	192(%rsp), %xmm14
+	movups	208(%rsp), %xmm15
+#endif
+
+	addq	$STACKSIZE, %rsp
+	ret
+
+	EPILOGUE
+
+
+#else
+/*************************************************************************************
+* TRMM Kernel
+*************************************************************************************/
+	PROLOGUE
+	PROFCODE
+	
+	subq	$STACKSIZE, %rsp
+	movq	%rbx,   (%rsp)
+	movq	%rbp,  8(%rsp)
+	movq	%r12, 16(%rsp)
+	movq	%r13, 24(%rsp)
+	movq	%r14, 32(%rsp)
+	movq	%r15, 40(%rsp)
+
+	vzeroupper
+
+#ifdef WINDOWS_ABI
+	movq	%rdi,    48(%rsp)
+	movq	%rsi,    56(%rsp)
+	movups	%xmm6,   64(%rsp)
+	movups	%xmm7,   80(%rsp)
+	movups	%xmm8,   96(%rsp)
+	movups	%xmm9,  112(%rsp)
+	movups	%xmm10, 128(%rsp)
+	movups	%xmm11, 144(%rsp)
+	movups	%xmm12, 160(%rsp)
+	movups	%xmm13, 176(%rsp)
+	movups	%xmm14, 192(%rsp)
+	movups	%xmm15, 208(%rsp)
+
+	movq	ARG1,      OLD_M
+	movq	ARG2,      OLD_N
+	movq	ARG3,      OLD_K
+	movq	OLD_A,     A
+	movq	OLD_B,     B
+	movq	OLD_C,     C
+	movq	OLD_LDC,   LDC
+#ifdef TRMMKERNEL
+	movsd	OLD_OFFSET, %xmm12
+#endif
+	vmovaps	%xmm3, %xmm0
+
+#else
+	movq	STACKSIZE +  8(%rsp), LDC
+#ifdef TRMMKERNEL
+	movsd	STACKSIZE + 16(%rsp), %xmm12
+#endif
+
+#endif
+
+	movq    %rsp, SP      # save old stack
+        subq    $128 + L_BUFFER_SIZE, %rsp
+        andq    $-4096, %rsp    # align stack
+
+        STACK_TOUCH
+
+	cmpq	$0, OLD_M
+	je	.L999
+
+	cmpq	$0, OLD_N
+	je	.L999
+
+	cmpq	$0, OLD_K
+	je	.L999
+
+	movq	OLD_M, M
+	movq	OLD_N, N
+	movq	OLD_K, K
+
+	vmovsd	 %xmm0, ALPHA
+
+	salq	$BASE_SHIFT, LDC
+
+	movq    N, %rax
+        xorq    %rdx, %rdx
+        movq    $2,  %rdi
+        divq    %rdi                    //    N / 2
+        movq    %rax, Ndiv6             //    N / 2
+        movq    %rdx, Nmod6             //    N % 2
+
+	
+
+#ifdef TRMMKERNEL
+	vmovsd	%xmm12, OFFSET
+	vmovsd	%xmm12, KK
+#ifndef LEFT
+	negq	KK
+#endif	
+#endif
+
+	movq	Ndiv6,  J
+	cmpq	$0, J
+	je	.L1_0
+	ALIGN_4
+
+.L2_0:
+
+.L2_01:
+	// copy to sub buffer
+	movq	B, BO1
+	leaq    BUFFER1, BO		// first buffer to BO
+	movq	K, %rax
+	ALIGN_4
+
+.L2_02b:
+
+	vmovups	(BO1), %xmm0
+	vmovups	%xmm0,       (BO)
+	addq	$2*SIZE,BO1
+	addq	$2*SIZE,BO
+	decq	%rax
+	jnz	.L2_02b
+
+.L2_02c:
+
+	movq	BO1, B			// next offset of B
+
+.L2_10:
+	movq	C, CO1
+	leaq	(C, LDC, 2), C		// c += 2 * ldc
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        movq    OFFSET, %rax
+        movq    %rax, KK
+#endif
+	
+	movq	A, AO		 	// aoffset = a
+	addq	$16 * SIZE, AO
+
+	movq	M,  I
+	sarq	$3, I			// i = (m >> 3)
+	je	.L2_20
+
+	ALIGN_4
+
+.L2_11:
+
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+	leaq	BUFFER1, BO		// first buffer to BO
+	addq	$4 * SIZE, BO
+#else
+        movq    KK, %rax
+	leaq	BUFFER1, BO		// first buffer to BO
+	addq	$4 * SIZE, BO
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, 8), BO
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+        leaq    (AO, %rax, 8), AO
+#endif
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $8, %rax        // number of values in AO
+#else
+        addq    $2, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$-8, %rax			//  K = K - ( K % 8 )
+	je	.L2_16
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_12:
+
+	KERNEL8x2_1(xxx)
+	KERNEL8x2_2(xxx)
+	KERNEL8x2_3(xxx)
+	KERNEL8x2_4(xxx)
+
+	KERNEL8x2_1(xxx)
+	KERNEL8x2_2(xxx)
+	KERNEL8x2_3(xxx)
+	KERNEL8x2_4(xxx)
+
+	je	.L2_16
+
+	KERNEL8x2_1(xxx)
+	KERNEL8x2_2(xxx)
+	KERNEL8x2_3(xxx)
+	KERNEL8x2_4(xxx)
+
+	KERNEL8x2_1(xxx)
+	KERNEL8x2_2(xxx)
+	KERNEL8x2_3(xxx)
+	KERNEL8x2_4(xxx)
+
+	je	.L2_16
+
+	jmp	.L2_12
+	ALIGN_4
+
+.L2_16:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_19
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_17:
+
+	KERNEL8x2_SUB(xxx)
+	addq	$2, BI
+	addq	$8, %rax
+	jl	.L2_17
+	ALIGN_4
+
+
+.L2_19:
+
+	vmovddup	ALPHA, %xmm0
+
+#ifndef TRMMKERNEL
+
+	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7
+	vfmaddpd 4 * SIZE(CO1),%xmm0, %xmm10,%xmm10
+	vfmaddpd 6 * SIZE(CO1),%xmm0, %xmm13,%xmm13
+
+	vfmaddpd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+	vfmaddpd 2 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
+	vfmaddpd 4 * SIZE(CO1, LDC),%xmm0, %xmm11,%xmm11
+	vfmaddpd 6 * SIZE(CO1, LDC),%xmm0, %xmm14,%xmm14
+
+
+#else
+	vmulpd	%xmm0, %xmm4,%xmm4
+	vmulpd	%xmm0, %xmm7,%xmm7
+	vmulpd	%xmm0, %xmm10,%xmm10
+	vmulpd	%xmm0, %xmm13,%xmm13
+
+	vmulpd	%xmm0, %xmm5,%xmm5
+	vmulpd	%xmm0, %xmm8,%xmm8
+	vmulpd	%xmm0, %xmm11,%xmm11
+	vmulpd	%xmm0, %xmm14,%xmm14
+
+#endif
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm7 , 2 * SIZE(CO1)
+	vmovups	%xmm10, 4 * SIZE(CO1)
+	vmovups	%xmm13, 6 * SIZE(CO1)
+
+	vmovups	%xmm5 ,  	(CO1, LDC)
+	vmovups	%xmm8 , 2 * SIZE(CO1, LDC)
+	vmovups	%xmm11, 4 * SIZE(CO1, LDC)
+	vmovups	%xmm14, 6 * SIZE(CO1, LDC)
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, 8), BO
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+        leaq    (AO, %rax, 8), AO
+#endif
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $8, KK
+#endif
+
+	addq	$8 * SIZE, CO1		# coffset += 8
+	decq	I			# i --
+	jg	.L2_11
+	ALIGN_4	
+
+/**************************************************************************
+* Rest of M 
+***************************************************************************/
+.L2_20:
+	// Test rest of M
+
+	testq	$7, M
+	jz	.L2_60		// to next 2 lines of N
+
+	testq	$4, M		
+	jz	.L2_30
+
+	ALIGN_4
+
+.L2_21:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, 8), BO
+        salq    $2, %rax                        // rax = rax * 4 ; number of values
+        leaq    (AO, %rax, 8), AO
+#endif
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $4, %rax        // number of values in AO
+#else
+        addq    $2, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$-8, %rax
+	je	.L2_26
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_22:
+
+	KERNEL4x2_1(xxx)
+	KERNEL4x2_2(xxx)
+	KERNEL4x2_3(xxx)
+	KERNEL4x2_4(xxx)
+
+	KERNEL4x2_1(xxx)
+	KERNEL4x2_2(xxx)
+	KERNEL4x2_3(xxx)
+	KERNEL4x2_4(xxx)
+
+	je	.L2_26
+
+	KERNEL4x2_1(xxx)
+	KERNEL4x2_2(xxx)
+	KERNEL4x2_3(xxx)
+	KERNEL4x2_4(xxx)
+
+	KERNEL4x2_1(xxx)
+	KERNEL4x2_2(xxx)
+	KERNEL4x2_3(xxx)
+	KERNEL4x2_4(xxx)
+
+	je	.L2_26
+
+	jmp	.L2_22
+	ALIGN_4
+
+.L2_26:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_29
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_27:
+
+	KERNEL4x2_SUB(xxx)
+	addq	$2, BI
+	addq	$4, %rax
+	jl	.L2_27
+	ALIGN_4
+
+
+.L2_29:
+
+	vmovddup	ALPHA, %xmm0
+
+#ifndef TRMMKERNEL
+
+	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7
+
+	vfmaddpd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+	vfmaddpd 2 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
+
+
+#else
+	vmulpd	%xmm0, %xmm4,%xmm4
+	vmulpd	%xmm0, %xmm7,%xmm7
+
+	vmulpd	%xmm0, %xmm5,%xmm5
+	vmulpd	%xmm0, %xmm8,%xmm8
+
+#endif
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm7 , 2 * SIZE(CO1)
+
+	vmovups	%xmm5 ,  	(CO1, LDC)
+	vmovups	%xmm8 , 2 * SIZE(CO1, LDC)
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, 8), BO
+        salq    $2, %rax                        // rax = rax * 4 ; number of values
+        leaq    (AO, %rax, 8), AO
+#endif
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $4, KK
+#endif
+
+	addq	$4 * SIZE, CO1		# coffset += 4
+	ALIGN_4
+	
+
+.L2_30:
+	testq	$2, M		
+	jz	.L2_40
+
+	ALIGN_4
+
+.L2_31:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, 8), BO
+        salq    $1, %rax                        // rax = rax * 2 ; number of values
+        leaq    (AO, %rax, 8), AO
+#endif
+
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $2, %rax        // number of values in AO
+#else
+        addq    $2, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$-8, %rax
+	je	.L2_36
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_32:
+
+	KERNEL2x2_1(xxx)
+	KERNEL2x2_2(xxx)
+	KERNEL2x2_3(xxx)
+	KERNEL2x2_4(xxx)
+
+	KERNEL2x2_1(xxx)
+	KERNEL2x2_2(xxx)
+	KERNEL2x2_3(xxx)
+	KERNEL2x2_4(xxx)
+
+	je	.L2_36
+
+	KERNEL2x2_1(xxx)
+	KERNEL2x2_2(xxx)
+	KERNEL2x2_3(xxx)
+	KERNEL2x2_4(xxx)
+
+	KERNEL2x2_1(xxx)
+	KERNEL2x2_2(xxx)
+	KERNEL2x2_3(xxx)
+	KERNEL2x2_4(xxx)
+
+	je	.L2_36
+
+	jmp	.L2_32
+	ALIGN_4
+
+.L2_36:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_39
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+	
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_37:
+
+	KERNEL2x2_SUB(xxx)
+	addq	$2, BI
+	addq	$2, %rax
+	jl	.L2_37
+	ALIGN_4
+
+
+.L2_39:
+
+	vmovddup	ALPHA, %xmm0
+
+#ifndef TRMMKERNEL
+
+	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddpd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+
+#else
+	vmulpd	%xmm0, %xmm4,%xmm4
+	vmulpd	%xmm0, %xmm5,%xmm5
+
+#endif
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm5 ,  	(CO1, LDC)
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax 
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, 8), BO
+        salq    $1, %rax                        // rax = rax * 2 ; number of values
+        leaq    (AO, %rax, 8), AO
+#endif
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $2, KK
+#endif
+
+	addq	$2 * SIZE, CO1		# coffset += 2
+	ALIGN_4
+	
+
+.L2_40:
+	testq	$1, M		
+	jz	.L2_60		// to next 2 lines of N
+
+	ALIGN_4
+
+.L2_41:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, 8), BO
+        leaq    (AO, %rax, 8), AO
+#endif
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $1, %rax        // number of values in AO
+#else
+        addq    $2, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+	andq	$-8, %rax
+	je	.L2_46
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_42:
+
+	KERNEL1x2_1(xxx)
+	KERNEL1x2_2(xxx)
+	KERNEL1x2_3(xxx)
+	KERNEL1x2_4(xxx)
+
+	KERNEL1x2_1(xxx)
+	KERNEL1x2_2(xxx)
+	KERNEL1x2_3(xxx)
+	KERNEL1x2_4(xxx)
+
+	je	.L2_46
+
+	KERNEL1x2_1(xxx)
+	KERNEL1x2_2(xxx)
+	KERNEL1x2_3(xxx)
+	KERNEL1x2_4(xxx)
+
+	KERNEL1x2_1(xxx)
+	KERNEL1x2_2(xxx)
+	KERNEL1x2_3(xxx)
+	KERNEL1x2_4(xxx)
+
+	je	.L2_46
+
+	jmp	.L2_42
+	ALIGN_4
+
+.L2_46:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_49
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_47:
+
+	KERNEL1x2_SUB(xxx)
+	addq	$2, BI
+	addq	$1, %rax
+	jl	.L2_47
+	ALIGN_4
+
+
+.L2_49:
+
+	vmovddup	ALPHA, %xmm0
+
+#ifndef TRMMKERNEL
+
+	vfmaddsd 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddsd 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+
+#else
+	vmulsd	%xmm0, %xmm4,%xmm4
+	vmulsd	%xmm0, %xmm5,%xmm5
+
+#endif
+
+	vmovsd	%xmm4 ,  	(CO1)
+	vmovsd	%xmm5 ,  	(CO1, LDC)
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, 8), BO
+        leaq    (AO, %rax, 8), AO
+#endif
+
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $1, KK
+#endif
+
+	addq	$1 * SIZE, CO1		# coffset += 1
+	ALIGN_4
+	
+
+	
+.L2_60:
+#if defined(TRMMKERNEL) && !defined(LEFT)
+        addq    $2, KK
+#endif
+
+	decq	J			// j --
+	jg	.L2_01			// next 2 lines of N
+
+
+
+.L1_0:
+
+/************************************************************************************************
+* Loop for Nmod6 % 2 > 0
+*************************************************************************************************/
+
+	movq	Nmod6, J		
+	andq	$1, J			// j % 2
+	je	.L999
+	ALIGN_4
+
+.L1_01:
+	// copy to sub buffer
+	movq	B, BO1
+	leaq    BUFFER1, BO		// first buffer to BO
+	movq	K, %rax
+	ALIGN_4
+
+.L1_02b:
+
+	vmovsd	(BO1), %xmm0
+	vmovsd	%xmm0,       (BO)
+	addq	$1*SIZE,BO1
+	addq	$1*SIZE,BO
+	decq	%rax
+	jnz	.L1_02b
+
+.L1_02c:
+
+	movq	BO1, B			// next offset of B
+
+.L1_10:
+	movq	C, CO1
+	leaq	(C, LDC, 1), C		// c += 1 * ldc
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        movq    OFFSET, %rax
+        movq    %rax, KK
+#endif
+
+	movq	A, AO		 	// aoffset = a
+	addq	$16 * SIZE, AO
+
+	movq	M,  I
+	sarq	$3, I			// i = (m >> 3)
+	je	.L1_20
+
+	ALIGN_4
+
+.L1_11:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BO, BI, 8), BO
+        salq    $3, %rax                        // rax = rax * 8 ; number of values
+        leaq    (AO, %rax, 8), AO
+#endif
+
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $8, %rax        // number of values in AO
+#else
+        addq    $1, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+	andq	$-8, %rax			//  K = K - ( K % 8 )
+	je	.L1_16
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_12:
+
+	KERNEL8x1_1(xxx)
+	KERNEL8x1_2(xxx)
+	KERNEL8x1_3(xxx)
+	KERNEL8x1_4(xxx)
+
+	KERNEL8x1_1(xxx)
+	KERNEL8x1_2(xxx)
+	KERNEL8x1_3(xxx)
+	KERNEL8x1_4(xxx)
+
+	je	.L1_16
+
+	KERNEL8x1_1(xxx)
+	KERNEL8x1_2(xxx)
+	KERNEL8x1_3(xxx)
+	KERNEL8x1_4(xxx)
+
+	KERNEL8x1_1(xxx)
+	KERNEL8x1_2(xxx)
+	KERNEL8x1_3(xxx)
+	KERNEL8x1_4(xxx)
+
+	je	.L1_16
+
+	jmp	.L1_12
+	ALIGN_4
+
+.L1_16:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_19
+
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_17:
+
+	KERNEL8x1_SUB(xxx)
+	addq	$1, BI
+	addq	$8, %rax
+	jl	.L1_17
+	ALIGN_4
+
+
+.L1_19:
+
+	vmovddup	ALPHA, %xmm0
+
+#ifndef TRMMKERNEL
+
+	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7
+	vfmaddpd 4 * SIZE(CO1),%xmm0, %xmm10,%xmm10
+	vfmaddpd 6 * SIZE(CO1),%xmm0, %xmm13,%xmm13
+
+#else
+	vmulpd	%xmm0, %xmm4,%xmm4
+	vmulpd	%xmm0, %xmm7,%xmm7
+	vmulpd	%xmm0, %xmm10,%xmm10
+	vmulpd	%xmm0, %xmm13,%xmm13
+
+#endif
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm7 , 2 * SIZE(CO1)
+	vmovups	%xmm10, 4 * SIZE(CO1)
+	vmovups	%xmm13, 6 * SIZE(CO1)
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BO, BI, 8), BO
+        salq    $3, %rax                        // rax = rax * 8 ; number of values
+        leaq    (AO, %rax, 8), AO
+#endif
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $8, KK
+#endif
+	addq	$8 * SIZE, CO1		# coffset += 8
+	decq	I			# i --
+	jg	.L1_11
+	ALIGN_4	
+
+/**************************************************************************
+* Rest of M 
+***************************************************************************/
+.L1_20:
+	// Test rest of M
+
+	testq	$7, M
+	jz	.L999
+
+	testq	$4, M		
+	jz	.L1_30
+
+	ALIGN_4
+
+.L1_21:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BO, BI, 8), BO
+        salq    $2, %rax                        // rax = rax * 4 ; number of values
+        leaq    (AO, %rax, 8), AO
+#endif
+
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $4, %rax        // number of values in AO
+#else
+        addq    $1, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$-8, %rax
+	je	.L1_26
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_22:
+
+	KERNEL4x1_1(xxx)
+	KERNEL4x1_2(xxx)
+	KERNEL4x1_3(xxx)
+	KERNEL4x1_4(xxx)
+
+	KERNEL4x1_1(xxx)
+	KERNEL4x1_2(xxx)
+	KERNEL4x1_3(xxx)
+	KERNEL4x1_4(xxx)
+
+	je	.L1_26
+
+	KERNEL4x1_1(xxx)
+	KERNEL4x1_2(xxx)
+	KERNEL4x1_3(xxx)
+	KERNEL4x1_4(xxx)
+
+	KERNEL4x1_1(xxx)
+	KERNEL4x1_2(xxx)
+	KERNEL4x1_3(xxx)
+	KERNEL4x1_4(xxx)
+
+	je	.L1_26
+
+	jmp	.L1_22
+	ALIGN_4
+
+.L1_26:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_29
+
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_27:
+
+	KERNEL4x1_SUB(xxx)
+	addq	$1, BI
+	addq	$4, %rax
+	jl	.L1_27
+	ALIGN_4
+
+
+.L1_29:
+
+	vmovddup	ALPHA, %xmm0
+
+#ifndef TRMMKERNEL
+
+	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddpd 2 * SIZE(CO1),%xmm0, %xmm7,%xmm7
+
+#else
+	vmulpd	%xmm0, %xmm4,%xmm4
+	vmulpd	%xmm0, %xmm7,%xmm7
+
+#endif
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm7 , 2 * SIZE(CO1)
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BO, BI, 8), BO
+        salq    $2, %rax                        // rax = rax * 4 ; number of values
+        leaq    (AO, %rax, 8), AO
+#endif
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $4, KK
+#endif
+
+	addq	$4 * SIZE, CO1		# coffset += 4
+	ALIGN_4
+	
+
+.L1_30:
+	testq	$2, M		
+	jz	.L1_40
+
+	ALIGN_4
+
+.L1_31:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BO, BI, 8), BO
+        salq    $1, %rax                        // rax = rax * 2 ; number of values
+        leaq    (AO, %rax, 8), AO
+#endif
+
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $2, %rax        // number of values in AO
+#else
+        addq    $1, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$-8, %rax
+	je	.L1_36
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_32:
+
+	KERNEL2x1_1(xxx)
+	KERNEL2x1_2(xxx)
+	KERNEL2x1_3(xxx)
+	KERNEL2x1_4(xxx)
+
+	KERNEL2x1_1(xxx)
+	KERNEL2x1_2(xxx)
+	KERNEL2x1_3(xxx)
+	KERNEL2x1_4(xxx)
+
+	je	.L1_36
+
+	KERNEL2x1_1(xxx)
+	KERNEL2x1_2(xxx)
+	KERNEL2x1_3(xxx)
+	KERNEL2x1_4(xxx)
+
+	KERNEL2x1_1(xxx)
+	KERNEL2x1_2(xxx)
+	KERNEL2x1_3(xxx)
+	KERNEL2x1_4(xxx)
+
+	je	.L1_36
+
+	jmp	.L1_32
+	ALIGN_4
+
+.L1_36:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_39
+
+	movq    %rax, BI                        //  Index for BO
+	
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_37:
+
+	KERNEL2x1_SUB(xxx)
+	addq	$1, BI
+	addq	$2, %rax
+	jl	.L1_37
+	ALIGN_4
+
+
+.L1_39:
+
+	vmovddup	ALPHA, %xmm0
+
+#ifndef TRMMKERNEL
+
+	vfmaddpd 	 (CO1),%xmm0, %xmm4,%xmm4
+
+#else
+	vmulpd	%xmm0, %xmm4,%xmm4
+
+#endif
+
+	vmovups	%xmm4 ,  	(CO1)
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BO, BI, 8), BO
+        salq    $1, %rax                        // rax = rax * 2 ; number of values
+        leaq    (AO, %rax, 8), AO
+#endif
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $2, KK
+#endif
+	addq	$2 * SIZE, CO1		# coffset += 2
+	ALIGN_4
+	
+
+.L1_40:
+	testq	$1, M		
+	jz	.L999
+
+	ALIGN_4
+
+.L1_41:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BO, BI, 8), BO
+        leaq    (AO, %rax, 8), AO
+#endif
+
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $1, %rax        // number of values in AO
+#else
+        addq    $1, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+	andq	$-8, %rax
+	je	.L1_46
+	movq    %rax, BI                        //  Index for BO
+
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_42:
+
+	KERNEL1x1_1(xxx)
+	KERNEL1x1_2(xxx)
+	KERNEL1x1_3(xxx)
+	KERNEL1x1_4(xxx)
+
+	KERNEL1x1_1(xxx)
+	KERNEL1x1_2(xxx)
+	KERNEL1x1_3(xxx)
+	KERNEL1x1_4(xxx)
+
+	je	.L1_46
+
+	KERNEL1x1_1(xxx)
+	KERNEL1x1_2(xxx)
+	KERNEL1x1_3(xxx)
+	KERNEL1x1_4(xxx)
+
+	KERNEL1x1_1(xxx)
+	KERNEL1x1_2(xxx)
+	KERNEL1x1_3(xxx)
+	KERNEL1x1_4(xxx)
+
+	je	.L1_46
+
+	jmp	.L1_42
+	ALIGN_4
+
+.L1_46:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_49
+
+	movq    %rax, BI                        //  Index for BO
+
+	leaq	(AO, %rax, 8), AO
+	leaq	(BO, BI, 8), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_47:
+
+	KERNEL1x1_SUB(xxx)
+	addq	$1, BI
+	addq	$1, %rax
+	jl	.L1_47
+	ALIGN_4
+
+
+.L1_49:
+
+	vmovddup	ALPHA, %xmm0
+
+#ifndef TRMMKERNEL
+
+	vfmaddsd 	 (CO1),%xmm0, %xmm4,%xmm4
+
+#else
+	vmulsd	%xmm0, %xmm4,%xmm4
+
+#endif
+
+	vmovsd	%xmm4 ,  	(CO1)
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BO, BI, 8), BO
+        leaq    (AO, %rax, 8), AO
+#endif
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $1, KK
+#endif
+
+	addq	$1 * SIZE, CO1		# coffset += 1
+	ALIGN_4
+	
+
+.L999:
+	movq   		SP, %rsp
+	movq	   (%rsp), %rbx
+	movq	  8(%rsp), %rbp
+	movq	 16(%rsp), %r12
+	movq	 24(%rsp), %r13
+	movq	 32(%rsp), %r14
+	movq	 40(%rsp), %r15
+
+#ifdef WINDOWS_ABI
+	movq	 48(%rsp), %rdi
+	movq	 56(%rsp), %rsi
+	movups	 64(%rsp), %xmm6
+	movups	 80(%rsp), %xmm7
+	movups	 96(%rsp), %xmm8
+	movups	112(%rsp), %xmm9
+	movups	128(%rsp), %xmm10
+	movups	144(%rsp), %xmm11
+	movups	160(%rsp), %xmm12
+	movups	176(%rsp), %xmm13
+	movups	192(%rsp), %xmm14
+	movups	208(%rsp), %xmm15
+#endif
+
+	addq	$STACKSIZE, %rsp
+	ret
+
+	EPILOGUE
+
+
+
+#endif
diff --git a/kernel/x86_64/sgemm_kernel_16x2_bulldozer.S b/kernel/x86_64/sgemm_kernel_16x2_bulldozer.S
index 9cc27184d..b31a934f2 100644
--- a/kernel/x86_64/sgemm_kernel_16x2_bulldozer.S
+++ b/kernel/x86_64/sgemm_kernel_16x2_bulldozer.S
@@ -1,5231 +1,5231 @@
-/*********************************************************************/
-/* Copyright 2009, 2010 The University of Texas at Austin.           */
-/* All rights reserved.                                              */
-/*                                                                   */
-/* Redistribution and use in source and binary forms, with or        */
-/* without modification, are permitted provided that the following   */
-/* conditions are met:                                               */
-/*                                                                   */
-/*   1. Redistributions of source code must retain the above         */
-/*      copyright notice, this list of conditions and the following  */
-/*      disclaimer.                                                  */
-/*                                                                   */
-/*   2. Redistributions in binary form must reproduce the above      */
-/*      copyright notice, this list of conditions and the following  */
-/*      disclaimer in the documentation and/or other materials       */
-/*      provided with the distribution.                              */
-/*                                                                   */
-/*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */
-/*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */
-/*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */
-/*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */
-/*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */
-/*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */
-/*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */
-/*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */
-/*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */
-/*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */
-/*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */
-/*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */
-/*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */
-/*    POSSIBILITY OF SUCH DAMAGE.                                    */
-/*                                                                   */
-/* The views and conclusions contained in the software and           */
-/* documentation are those of the authors and should not be          */
-/* interpreted as representing official policies, either expressed   */
-/* or implied, of The University of Texas at Austin.                 */
-/*********************************************************************/
-
-
-#define ASSEMBLER
-#include "common.h"
- 
-#define OLD_M	%rdi
-#define OLD_N	%rsi
-#define M	%r13
-#define J	%r14
-#define OLD_K	%rdx
-
-#define A	%rcx
-#define B	%r8
-#define C	%r9
-#define LDC	%r10
-	
-#define I	%r11
-#define AO	%rdi
-#define BO	%rsi
-#define	CO1	%r15
-#define K	%r12
-#define BI	%rbp
-#define	SP	%rbx
-
-#define BO1	%rdi
-#define BO2	%r15
-
-#ifndef WINDOWS_ABI
-
-#define STACKSIZE 96
-
-#else
-
-#define STACKSIZE 256
-
-#define OLD_A		40 + STACKSIZE(%rsp)
-#define OLD_B		48 + STACKSIZE(%rsp)
-#define OLD_C		56 + STACKSIZE(%rsp)
-#define OLD_LDC		64 + STACKSIZE(%rsp)
-#define OLD_OFFSET	72 + STACKSIZE(%rsp)
-
-#endif
-
-#define L_BUFFER_SIZE 8192
-#define LB2_OFFSET    4096
-
-#define Ndiv6	 24(%rsp)
-#define Nmod6	 32(%rsp)
-#define N	 40(%rsp)
-#define ALPHA	 48(%rsp)
-#define OFFSET	 56(%rsp)
-#define KK	 64(%rsp)
-#define KKK	 72(%rsp)
-#define BUFFER1	           128(%rsp)
-#define BUFFER2	LB2_OFFSET+128(%rsp)
-
-#if defined(OS_WINDOWS)
-#if   L_BUFFER_SIZE > 16384
-#define STACK_TOUCH \
-        movl    $0,  4096 * 4(%rsp);\
-        movl    $0,  4096 * 3(%rsp);\
-        movl    $0,  4096 * 2(%rsp);\
-        movl    $0,  4096 * 1(%rsp);
-#elif L_BUFFER_SIZE > 12288
-#define STACK_TOUCH \
-        movl    $0,  4096 * 3(%rsp);\
-        movl    $0,  4096 * 2(%rsp);\
-        movl    $0,  4096 * 1(%rsp);
-#elif L_BUFFER_SIZE > 8192
-#define STACK_TOUCH \
-        movl    $0,  4096 * 2(%rsp);\
-        movl    $0,  4096 * 1(%rsp);
-#elif L_BUFFER_SIZE > 4096
-#define STACK_TOUCH \
-        movl    $0,  4096 * 1(%rsp);
-#else
-#define STACK_TOUCH
-#endif
-#else
-#define STACK_TOUCH
-#endif
-
-
-
-#define	A_PR1	384
-#define	B_PR1	192
-
-/*******************************************************************************************
-* 3 lines of N
-*******************************************************************************************/
-
-#define KERNEL16x3_1(xx) \
-	vbroadcastss	 -6 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vbroadcastss	 -5 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm3 ;\
-	nop						;\
-	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	prefetcht0	A_PR1(AO,%rax,SIZE)	;\
-	vfmaddps  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
-	vmovups 	-24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
-	vfmaddps  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
-	vfmaddps  	%xmm12,%xmm3,%xmm0,%xmm12 ;\
-	vmovups 	-20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
-	vfmaddps  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
-	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm15,%xmm3,%xmm0,%xmm15 ;\
-
-#define KERNEL16x3_2(xx) \
-	vmovups 	-16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vbroadcastss	 -1 * SIZE(BO, BI, SIZE), %xmm3 ;\
-	nop						;\
-	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-	vmovups 	-12 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	prefetcht0	A_PR1+64(AO,%rax,SIZE)	;\
-	vfmaddps  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
-	vmovups 	 -8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
-	vfmaddps  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
-	vfmaddps  	%xmm12,%xmm3,%xmm0,%xmm12 ;\
-	vmovups 	 -4 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
-	vfmaddps  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
-	vbroadcastss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vbroadcastss	  1 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm15,%xmm3,%xmm0,%xmm15 ;\
-
-#define KERNEL16x3_3(xx) \
-	vmovups 	  0 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vbroadcastss	  2 * SIZE(BO, BI, SIZE), %xmm3 ;\
-	nop						;\
-	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-	vmovups 	  4 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	prefetcht0	A_PR1+128(AO,%rax,SIZE)	;\
-	vfmaddps  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
-	vmovups 	  8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
-	vfmaddps  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
-	vfmaddps  	%xmm12,%xmm3,%xmm0,%xmm12 ;\
-	vmovups 	 12 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
-	vfmaddps  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
-	vbroadcastss	  3 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vbroadcastss	  4 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm15,%xmm3,%xmm0,%xmm15 ;\
-
-#define KERNEL16x3_4(xx) \
-	vmovups 	 16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vbroadcastss	  5 * SIZE(BO, BI, SIZE), %xmm3 ;\
-	nop						;\
-	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-	vmovups 	 20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	prefetcht0	A_PR1+192(AO,%rax,SIZE)	;\
-	vfmaddps  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
-	vmovups 	 24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
-	vfmaddps  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
-	addq	$12, BI				  ;\
-	vfmaddps  	%xmm12,%xmm3,%xmm0,%xmm12 ;\
-	vmovups 	 28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
-	vfmaddps  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
-	addq	$64, %rax 			  ;\
-	vfmaddps  	%xmm15,%xmm3,%xmm0,%xmm15 ;\
-
-#define KERNEL16x3_SUB(xx) \
-	vbroadcastss	 -6 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vbroadcastss	 -5 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm3 ;\
-	nop						;\
-	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	vfmaddps  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
-	vmovups 	-24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
-	vfmaddps  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
-	vfmaddps  	%xmm12,%xmm3,%xmm0,%xmm12 ;\
-	vmovups 	-20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
-	vfmaddps  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
-	vfmaddps  	%xmm15,%xmm3,%xmm0,%xmm15 ;\
-
-
-/*******************************************************************************************/
-
-#define KERNEL8x3_1(xx) \
-	prefetcht0	A_PR1(AO,%rax,SIZE)	;\
-	vbroadcastss	 -6 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vbroadcastss	 -5 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm3 ;\
-	nop						;\
-	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	vfmaddps  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
-
-#define KERNEL8x3_2(xx) \
-	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vbroadcastss	 -1 * SIZE(BO, BI, SIZE), %xmm3 ;\
-	nop						;\
-	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-	vmovups 	-20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	vfmaddps  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
-
-#define KERNEL8x3_3(xx) \
-	prefetcht0	A_PR1+64(AO,%rax,SIZE)	;\
-	vbroadcastss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vbroadcastss	  1 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vbroadcastss	  2 * SIZE(BO, BI, SIZE), %xmm3 ;\
-	nop						;\
-	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-	vmovups 	-12 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	vfmaddps  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
-
-#define KERNEL8x3_4(xx) \
-	vbroadcastss	  3 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	 -8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vbroadcastss	  4 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vbroadcastss	  5 * SIZE(BO, BI, SIZE), %xmm3 ;\
-	nop						;\
-	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-	vmovups 	 -4 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	vfmaddps  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
-	addq	$12, BI				  ;\
-	addq	$32, %rax 			  ;\
-
-#define KERNEL8x3_SUB(xx) \
-	vbroadcastss	 -6 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vbroadcastss	 -5 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm3 ;\
-	nop						;\
-	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	vfmaddps  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
-
-
-/*******************************************************************************************/
-
-#define KERNEL4x3_1(xx) \
-	prefetcht0	A_PR1(AO,%rax,SIZE)	;\
-	vbroadcastss	 -6 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vbroadcastss	 -5 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm3 ;\
-	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-
-#define KERNEL4x3_2(xx) \
-	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vbroadcastss	 -1 * SIZE(BO, BI, SIZE), %xmm3 ;\
-	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-
-#define KERNEL4x3_3(xx) \
-	vbroadcastss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vbroadcastss	  1 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vbroadcastss	  2 * SIZE(BO, BI, SIZE), %xmm3 ;\
-	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-
-#define KERNEL4x3_4(xx) \
-	vbroadcastss	  3 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vbroadcastss	  4 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vbroadcastss	  5 * SIZE(BO, BI, SIZE), %xmm3 ;\
-	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-	addq	$12, BI				  ;\
-	addq	$16, %rax 			  ;\
-
-#define KERNEL4x3_SUB(xx) \
-	vbroadcastss	 -6 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vbroadcastss	 -5 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm3 ;\
-	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-
-/*******************************************************************************************/
-
-#define KERNEL2x3_1(xx) \
-	vmovss	 -6 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovss	 -5 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovss	 -4 * SIZE(BO, BI, SIZE), %xmm3 ;\
-	vfmaddss  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-	vmovss 	-31 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
-	vfmaddss  	%xmm10,%xmm2,%xmm0,%xmm10 ;\
-	vfmaddss  	%xmm12,%xmm3,%xmm0,%xmm12 ;\
-
-#define KERNEL2x3_2(xx) \
-	vmovss	 -3 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-30 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovss	 -2 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovss	 -1 * SIZE(BO, BI, SIZE), %xmm3 ;\
-	vfmaddss  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-	vmovss 	-29 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
-	vfmaddss  	%xmm10,%xmm2,%xmm0,%xmm10 ;\
-	vfmaddss  	%xmm12,%xmm3,%xmm0,%xmm12 ;\
-
-#define KERNEL2x3_3(xx) \
-	vmovss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovss	  1 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovss	  2 * SIZE(BO, BI, SIZE), %xmm3 ;\
-	vfmaddss  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-	vmovss 	-27 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
-	vfmaddss  	%xmm10,%xmm2,%xmm0,%xmm10 ;\
-	vfmaddss  	%xmm12,%xmm3,%xmm0,%xmm12 ;\
-
-#define KERNEL2x3_4(xx) \
-	vmovss	  3 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-26 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovss	  4 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovss	  5 * SIZE(BO, BI, SIZE), %xmm3 ;\
-	vfmaddss  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-	vmovss 	-25 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
-	vfmaddss  	%xmm10,%xmm2,%xmm0,%xmm10 ;\
-	vfmaddss  	%xmm12,%xmm3,%xmm0,%xmm12 ;\
-	addq	$12, BI				  ;\
-	addq	$8, %rax 			  ;\
-
-#define KERNEL2x3_SUB(xx) \
-	vmovss	 -6 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovss	 -5 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovss	 -4 * SIZE(BO, BI, SIZE), %xmm3 ;\
-	vfmaddss  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-	vmovss 	-31 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
-	vfmaddss  	%xmm10,%xmm2,%xmm0,%xmm10 ;\
-	vfmaddss  	%xmm12,%xmm3,%xmm0,%xmm12 ;\
-
-/*******************************************************************************************/
-
-#define KERNEL1x3_1(xx) \
-	vmovss	 -6 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovss	 -5 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovss	 -4 * SIZE(BO, BI, SIZE), %xmm3 ;\
-	vfmaddss  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-
-#define KERNEL1x3_2(xx) \
-	vmovss	 -3 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-31 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovss	 -2 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovss	 -1 * SIZE(BO, BI, SIZE), %xmm3 ;\
-	vfmaddss  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-
-#define KERNEL1x3_3(xx) \
-	vmovss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-30 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovss	  1 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovss	  2 * SIZE(BO, BI, SIZE), %xmm3 ;\
-	vfmaddss  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-
-#define KERNEL1x3_4(xx) \
-	vmovss	  3 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-29 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovss	  4 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovss	  5 * SIZE(BO, BI, SIZE), %xmm3 ;\
-	vfmaddss  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-	addq	$12, BI				  ;\
-	addq	$4, %rax 			  ;\
-
-#define KERNEL1x3_SUB(xx) \
-	vmovss	 -6 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovss	 -5 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovss	 -4 * SIZE(BO, BI, SIZE), %xmm3 ;\
-	vfmaddss  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-
-/*******************************************************************************************/
-
-/*******************************************************************************************
-* 2 lines of N
-*******************************************************************************************/
-
-#define KERNEL16x2_1(xx) \
-	prefetcht0	A_PR1(AO,%rax,SIZE)	;\
-	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	vmovups 	-24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
-	vfmaddps  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
-	vmovups 	-20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
-	vfmaddps  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
-
-#define KERNEL16x2_2(xx) \
-	prefetcht0	A_PR1+64(AO,%rax,SIZE)	;\
-	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vbroadcastss	 -1 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovups 	-12 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	vmovups 	 -8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
-	vfmaddps  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
-	vmovups 	 -4 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
-	vfmaddps  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
-
-#define KERNEL16x2_3(xx) \
-	prefetcht0	A_PR1+128(AO,%rax,SIZE)	;\
-	vbroadcastss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	  0 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vbroadcastss	  1 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovups 	  4 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	vmovups 	  8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
-	vfmaddps  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
-	vmovups 	 12 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
-	vfmaddps  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
-
-#define KERNEL16x2_4(xx) \
-	prefetcht0	A_PR1+192(AO,%rax,SIZE)	;\
-	vbroadcastss	  2 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	 16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vbroadcastss	  3 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovups 	 20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	vmovups 	 24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
-	vfmaddps  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
-	vmovups 	 28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
-	vfmaddps  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
-	addq	$8, BI				  ;\
-	addq	$64, %rax 			  ;\
-
-#define KERNEL16x2_SUB(xx) \
-	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	vmovups 	-24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
-	vfmaddps  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
-	vmovups 	-20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
-	vfmaddps  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
-
-
-/*******************************************************************************************/
-
-#define KERNEL8x2_1(xx) \
-	prefetcht0	A_PR1(AO,%rax,SIZE)	;\
-	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-
-#define KERNEL8x2_2(xx) \
-	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vbroadcastss	 -1 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovups 	-20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-
-#define KERNEL8x2_3(xx) \
-	prefetcht0	A_PR1+64(AO,%rax,SIZE)	;\
-	vbroadcastss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vbroadcastss	  1 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovups 	-12 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-
-#define KERNEL8x2_4(xx) \
-	vbroadcastss	  2 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	 -8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vbroadcastss	  3 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovups 	 -4 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	addq	$8, BI				  ;\
-	addq	$32, %rax 			  ;\
-
-#define KERNEL8x2_SUB(xx) \
-	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-
-
-/*******************************************************************************************/
-
-#define KERNEL4x2_1(xx) \
-	prefetcht0	A_PR1(AO,%rax,SIZE)	;\
-	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-
-#define KERNEL4x2_2(xx) \
-	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vbroadcastss	 -1 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-
-#define KERNEL4x2_3(xx) \
-	vbroadcastss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vbroadcastss	  1 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-
-#define KERNEL4x2_4(xx) \
-	vbroadcastss	  2 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vbroadcastss	  3 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	addq	$8, BI				  ;\
-	addq	$16, %rax 			  ;\
-
-#define KERNEL4x2_SUB(xx) \
-	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-
-/*******************************************************************************************/
-
-#define KERNEL2x2_1(xx) \
-	vmovss	 -4 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovss	 -3 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovss 	-31 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
-	vfmaddss  	%xmm10,%xmm2,%xmm0,%xmm10 ;\
-
-#define KERNEL2x2_2(xx) \
-	vmovss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-30 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovss	 -1 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovss 	-29 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
-	vfmaddss  	%xmm10,%xmm2,%xmm0,%xmm10 ;\
-
-#define KERNEL2x2_3(xx) \
-	vmovss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovss	  1 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovss 	-27 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
-	vfmaddss  	%xmm10,%xmm2,%xmm0,%xmm10 ;\
-
-#define KERNEL2x2_4(xx) \
-	vmovss	  2 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-26 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovss	  3 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovss 	-25 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
-	vfmaddss  	%xmm10,%xmm2,%xmm0,%xmm10 ;\
-	addq	$8, BI				  ;\
-	addq	$8, %rax 			  ;\
-
-#define KERNEL2x2_SUB(xx) \
-	vmovss	 -4 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovss	 -3 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovss 	-31 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
-	vfmaddss  	%xmm10,%xmm2,%xmm0,%xmm10 ;\
-
-/*******************************************************************************************/
-
-#define KERNEL1x2_1(xx) \
-	vmovss	 -4 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovss	 -3 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-
-#define KERNEL1x2_2(xx) \
-	vmovss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-31 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovss	 -1 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-
-#define KERNEL1x2_3(xx) \
-	vmovss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-30 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovss	  1 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-
-#define KERNEL1x2_4(xx) \
-	vmovss	  2 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-29 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovss	  3 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	addq	$8, BI				  ;\
-	addq	$4, %rax 			  ;\
-
-#define KERNEL1x2_SUB(xx) \
-	vmovss	 -4 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovss	 -3 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-
-/*******************************************************************************************/
-
-/*******************************************************************************************
-* 1 line of N
-*******************************************************************************************/
-
-#define KERNEL16x1_1(xx) \
-	prefetcht0	A_PR1(AO,%rax,SIZE)	;\
-	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vmovups 	-24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
-	vmovups 	-20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
-
-#define KERNEL16x1_2(xx) \
-	prefetcht0	A_PR1+64(AO,%rax,SIZE)	;\
-	vbroadcastss	 -1 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovups 	-12 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vmovups 	 -8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
-	vmovups 	 -4 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
-
-#define KERNEL16x1_3(xx) \
-	prefetcht0	A_PR1+128(AO,%rax,SIZE)	;\
-	vbroadcastss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	  0 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovups 	  4 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vmovups 	  8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
-	vmovups 	 12 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
-
-#define KERNEL16x1_4(xx) \
-	prefetcht0	A_PR1+192(AO,%rax,SIZE)	;\
-	vbroadcastss	  1 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	 16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovups 	 20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vmovups 	 24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
-	vmovups 	 28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
-	addq	$4, BI				  ;\
-	addq	$64, %rax 			  ;\
-
-#define KERNEL16x1_SUB(xx) \
-	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vmovups 	-24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
-	vmovups 	-20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
-
-
-/*******************************************************************************************/
-
-#define KERNEL8x1_1(xx) \
-	prefetcht0	A_PR1(AO,%rax,SIZE)	;\
-	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-
-#define KERNEL8x1_2(xx) \
-	vbroadcastss	 -1 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovups 	-20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-
-#define KERNEL8x1_3(xx) \
-	prefetcht0	A_PR1+64(AO,%rax,SIZE)	;\
-	vbroadcastss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovups 	-12 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-
-#define KERNEL8x1_4(xx) \
-	vbroadcastss	  1 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	 -8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovups 	 -4 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	addq	$4, BI				  ;\
-	addq	$32, %rax 			  ;\
-
-#define KERNEL8x1_SUB(xx) \
-	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-
-
-/*******************************************************************************************/
-
-#define KERNEL4x1_1(xx) \
-	prefetcht0	A_PR1(AO,%rax,SIZE)	;\
-	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-
-#define KERNEL4x1_2(xx) \
-	vbroadcastss	 -1 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-
-#define KERNEL4x1_3(xx) \
-	vbroadcastss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-
-#define KERNEL4x1_4(xx) \
-	vbroadcastss	  1 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	addq	$4, BI				  ;\
-	addq	$16, %rax 			  ;\
-
-#define KERNEL4x1_SUB(xx) \
-	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-
-/*******************************************************************************************/
-
-#define KERNEL2x1_1(xx) \
-	vmovss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovss 	-31 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
-
-#define KERNEL2x1_2(xx) \
-	vmovss	 -1 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-30 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovss 	-29 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
-
-#define KERNEL2x1_3(xx) \
-	vmovss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovss 	-27 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
-
-#define KERNEL2x1_4(xx) \
-	vmovss	  1 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-26 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovss 	-25 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
-	addq	$4, BI				  ;\
-	addq	$8, %rax 			  ;\
-
-#define KERNEL2x1_SUB(xx) \
-	vmovss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovss 	-31 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
-
-/*******************************************************************************************/
-
-#define KERNEL1x1_1(xx) \
-	vmovss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-
-#define KERNEL1x1_2(xx) \
-	vmovss	 -1 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-31 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-
-#define KERNEL1x1_3(xx) \
-	vmovss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-30 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-
-#define KERNEL1x1_4(xx) \
-	vmovss	  1 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-29 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	addq	$4, BI				  ;\
-	addq	$4, %rax 			  ;\
-
-#define KERNEL1x1_SUB(xx) \
-	vmovss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-
-/*******************************************************************************************/
-
-#if !defined(TRMMKERNEL)
-
-
-	PROLOGUE
-	PROFCODE
-	
-	subq	$STACKSIZE, %rsp
-	movq	%rbx,   (%rsp)
-	movq	%rbp,  8(%rsp)
-	movq	%r12, 16(%rsp)
-	movq	%r13, 24(%rsp)
-	movq	%r14, 32(%rsp)
-	movq	%r15, 40(%rsp)
-
-	vzeroupper
-
-#ifdef WINDOWS_ABI
-	movq	%rdi,    48(%rsp)
-	movq	%rsi,    56(%rsp)
-	movups	%xmm6,   64(%rsp)
-	movups	%xmm7,   80(%rsp)
-	movups	%xmm8,   96(%rsp)
-	movups	%xmm9,  112(%rsp)
-	movups	%xmm10, 128(%rsp)
-	movups	%xmm11, 144(%rsp)
-	movups	%xmm12, 160(%rsp)
-	movups	%xmm13, 176(%rsp)
-	movups	%xmm14, 192(%rsp)
-	movups	%xmm15, 208(%rsp)
-
-	movq	ARG1,      OLD_M
-	movq	ARG2,      OLD_N
-	movq	ARG3,      OLD_K
-	movq	OLD_A,     A
-	movq	OLD_B,     B
-	movq	OLD_C,     C
-	movq	OLD_LDC,   LDC
-
-	vmovaps	%xmm3, %xmm0
-
-#else
-	movq	STACKSIZE +  8(%rsp), LDC
-
-#endif
-
-	movq    %rsp, SP      # save old stack
-        subq    $128 + L_BUFFER_SIZE, %rsp
-        andq    $-4096, %rsp    # align stack
-
-        STACK_TOUCH
-
-	cmpq	$0, OLD_M
-	je	.L999
-
-	cmpq	$0, OLD_N
-	je	.L999
-
-	cmpq	$0, OLD_K
-	je	.L999
-
-	movq	OLD_M, M
-	movq	OLD_N, N
-	movq	OLD_K, K
-
-	vmovsd	 %xmm0, ALPHA
-
-	salq	$BASE_SHIFT, LDC
-
-	movq    N, %rax
-        xorq    %rdx, %rdx
-        movq    $6,  %rdi
-        divq    %rdi                    //    N / 6
-        movq    %rax, Ndiv6             //    N / 6
-        movq    %rdx, Nmod6             //    N % 6
-
-
-	movq	Ndiv6,  J
-	cmpq	$0, J
-	je	.L2_0
-	ALIGN_4
-
-.L6_01:
-        // copy to sub buffer
-        movq    K, %rax
-        salq    $1,%rax                 // K * 2 ; read 2 values
-        movq    B, BO1
-        leaq    (B,%rax, SIZE), BO2     // next offset to BO2
-        leaq    BUFFER1, BO             // first buffer to BO
-        movq    K, %rax
-	sarq	$3 , %rax		// K / 8
-	jz	.L6_01a_2
-        ALIGN_4
-
-.L6_01a_1:
-
-        prefetcht0 512(BO1)
-        prefetcht0 512(BO2)
-        prefetchw  512(BO)
-
-	vmovsd	0 * SIZE(BO1), %xmm0
-	vmovsd	2 * SIZE(BO1), %xmm2
-	vmovsd	4 * SIZE(BO1), %xmm4
-	vmovsd	6 * SIZE(BO1), %xmm6
-	vmovss  0 * SIZE(BO2), %xmm1
-	vmovss  2 * SIZE(BO2), %xmm3
-	vmovss  4 * SIZE(BO2), %xmm5
-	vmovss  6 * SIZE(BO2), %xmm7
-	vmovsd	%xmm0, 0*SIZE(BO)
-	vmovss	%xmm1, 2*SIZE(BO)
-	vmovsd	%xmm2, 3*SIZE(BO)
-	vmovss	%xmm3, 5*SIZE(BO)
-	vmovsd	%xmm4, 6*SIZE(BO)
-	vmovss	%xmm5, 8*SIZE(BO)
-	vmovsd	%xmm6, 9*SIZE(BO)
-	vmovss	%xmm7,11*SIZE(BO)
-	addq	$8*SIZE,BO1
-	addq	$8*SIZE,BO2
-	addq	$12*SIZE,BO
-
-	vmovsd	0 * SIZE(BO1), %xmm0
-	vmovsd	2 * SIZE(BO1), %xmm2
-	vmovsd	4 * SIZE(BO1), %xmm4
-	vmovsd	6 * SIZE(BO1), %xmm6
-	vmovss  0 * SIZE(BO2), %xmm1
-	vmovss  2 * SIZE(BO2), %xmm3
-	vmovss  4 * SIZE(BO2), %xmm5
-	vmovss  6 * SIZE(BO2), %xmm7
-	vmovsd	%xmm0, 0*SIZE(BO)
-	vmovss	%xmm1, 2*SIZE(BO)
-	vmovsd	%xmm2, 3*SIZE(BO)
-	vmovss	%xmm3, 5*SIZE(BO)
-	vmovsd	%xmm4, 6*SIZE(BO)
-	vmovss	%xmm5, 8*SIZE(BO)
-	vmovsd	%xmm6, 9*SIZE(BO)
-	vmovss	%xmm7,11*SIZE(BO)
-	addq	$8*SIZE,BO1
-	addq	$8*SIZE,BO2
-	addq	$12*SIZE,BO
-
-	decq	%rax
-	jnz	.L6_01a_1
-
-
-
-.L6_01a_2:
-
-	movq    K, %rax
-        andq    $7, %rax                // K % 8
-        jz      .L6_02c
-        ALIGN_4
-
-
-.L6_02b:
-
-	vmovsd	0 * SIZE(BO1), %xmm0
-	vmovss  0 * SIZE(BO2), %xmm2
-	vmovsd	%xmm0, 0*SIZE(BO)
-	vmovss	%xmm2, 2*SIZE(BO)
-	addq	$2*SIZE,BO1
-	addq	$2*SIZE,BO2
-	addq	$3*SIZE,BO
-	decq	%rax
-	jnz	.L6_02b
-
-.L6_02c:
-
-	movq	K, %rax
-	salq	$1,%rax			// K * 2
-	leaq	(B,%rax, SIZE), BO1	// next offset to BO1
-	leaq	(BO1,%rax, SIZE), BO2	// next offset to BO2
-	leaq    BUFFER2, BO		// second buffer to BO
-	movq	K, %rax
-	sarq	$3 , %rax		// K / 8
-	jz	.L6_02c_2
-	ALIGN_4
-
-.L6_02c_1:
-
-	prefetcht0 512(BO2)
-        prefetchw  512(BO)
-
-	vmovsd	0 * SIZE(BO2), %xmm0
-	vmovsd	2 * SIZE(BO2), %xmm2
-	vmovsd	4 * SIZE(BO2), %xmm4
-	vmovsd	6 * SIZE(BO2), %xmm6
-	vmovss  1 * SIZE(BO1), %xmm1
-	vmovss  3 * SIZE(BO1), %xmm3
-	vmovss  5 * SIZE(BO1), %xmm5
-	vmovss  7 * SIZE(BO1), %xmm7
-	vmovss	%xmm1, 0*SIZE(BO)
-	vmovsd	%xmm0, 1*SIZE(BO)
-	vmovss	%xmm3, 3*SIZE(BO)
-	vmovsd	%xmm2, 4*SIZE(BO)
-	vmovss	%xmm5, 6*SIZE(BO)
-	vmovsd	%xmm4, 7*SIZE(BO)
-	vmovss	%xmm7, 9*SIZE(BO)
-	vmovsd	%xmm6,10*SIZE(BO)
-	addq	$8*SIZE,BO1
-	addq	$8*SIZE,BO2
-	addq	$12*SIZE,BO
-
-
-	vmovsd	0 * SIZE(BO2), %xmm0
-	vmovsd	2 * SIZE(BO2), %xmm2
-	vmovsd	4 * SIZE(BO2), %xmm4
-	vmovsd	6 * SIZE(BO2), %xmm6
-	vmovss  1 * SIZE(BO1), %xmm1
-	vmovss  3 * SIZE(BO1), %xmm3
-	vmovss  5 * SIZE(BO1), %xmm5
-	vmovss  7 * SIZE(BO1), %xmm7
-	vmovss	%xmm1, 0*SIZE(BO)
-	vmovsd	%xmm0, 1*SIZE(BO)
-	vmovss	%xmm3, 3*SIZE(BO)
-	vmovsd	%xmm2, 4*SIZE(BO)
-	vmovss	%xmm5, 6*SIZE(BO)
-	vmovsd	%xmm4, 7*SIZE(BO)
-	vmovss	%xmm7, 9*SIZE(BO)
-	vmovsd	%xmm6,10*SIZE(BO)
-	addq	$8*SIZE,BO1
-	addq	$8*SIZE,BO2
-	addq	$12*SIZE,BO
-
-	decq	%rax
-	jnz	.L6_02c_1
-
-
-.L6_02c_2:
-
-	movq    K, %rax
-        andq    $7, %rax                // K % 8
-        jz      .L6_03c
-        ALIGN_4
-
-.L6_03b:
-
-	vmovss	  1*SIZE(BO1), %xmm0
-	vmovsd	  0*SIZE(BO2), %xmm1
-	vmovss	%xmm0, 0*SIZE(BO)
-	vmovsd	%xmm1, 1*SIZE(BO)
-	addq	$2*SIZE,BO1
-	addq	$2*SIZE,BO2
-	addq	$3*SIZE,BO
-	decq	%rax
-	jnz	.L6_03b
-
-
-.L6_03c:
-
-	movq	BO2, B			// next offset of B
-
-.L6_10:
-	movq	C, CO1
-	leaq	(C, LDC, 2), C		 
-	leaq	(C, LDC, 1), C		// c += 3 * ldc
-
-	
-	movq	A, AO		 	// aoffset = a
-	addq	$32 * SIZE, AO
-
-	movq	M,  I
-	sarq	$4, I			// i = (m >> 4)
-	je	.L6_20
-
-	ALIGN_4
-
-.L6_11:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $6 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax			//  K = K - ( K % 8 )
-	je	.L6_16
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	salq	$4, %rax			// rax = rax * 16 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L6_12:
-
-	prefetcht0	B_PR1(BO,BI, SIZE)
-	KERNEL16x3_1(xxx)
-	KERNEL16x3_2(xxx)
-	KERNEL16x3_3(xxx)
-	KERNEL16x3_4(xxx)
-
-	KERNEL16x3_1(xxx)
-	prefetcht0	B_PR1+16(BO,BI, SIZE)
-	KERNEL16x3_2(xxx)
-	KERNEL16x3_3(xxx)
-	KERNEL16x3_4(xxx)
-
-	je	.L6_16
-
-	KERNEL16x3_1(xxx)
-	KERNEL16x3_2(xxx)
-	prefetcht0	B_PR1+32(BO,BI, SIZE)
-	KERNEL16x3_3(xxx)
-	KERNEL16x3_4(xxx)
-
-	KERNEL16x3_1(xxx)
-	KERNEL16x3_2(xxx)
-	KERNEL16x3_3(xxx)
-	KERNEL16x3_4(xxx)
-
-	je	.L6_16
-
-	jmp	.L6_12
-	ALIGN_4
-
-.L6_16:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L6_19
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	salq	$4, %rax			// rax = rax * 16 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L6_17:
-
-	KERNEL16x3_SUB(xxx)
-	addq	$3, BI
-	addq	$16, %rax
-	jl	.L6_17
-	ALIGN_4
-
-
-.L6_19:
-
-	vbroadcastss	ALPHA, %xmm0
-
-	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddps 4 * SIZE(CO1),%xmm0, %xmm7,%xmm7
-	vfmaddps 8 * SIZE(CO1),%xmm0, %xmm10,%xmm10
-	vfmaddps 12 * SIZE(CO1),%xmm0, %xmm13,%xmm13
-
-	vfmaddps 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-	vfmaddps 4 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
-	vfmaddps 8 * SIZE(CO1, LDC),%xmm0, %xmm11,%xmm11
-	vfmaddps 12 * SIZE(CO1, LDC),%xmm0, %xmm14,%xmm14
-
-	vfmaddps 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
-	vfmaddps 4 * SIZE(CO1, LDC, 2),%xmm0, %xmm9,%xmm9
-	vfmaddps 8 * SIZE(CO1, LDC, 2),%xmm0, %xmm12,%xmm12
-	vfmaddps 12 * SIZE(CO1, LDC, 2),%xmm0, %xmm15,%xmm15
-
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm7 , 4 * SIZE(CO1)
-	vmovups	%xmm10, 8 * SIZE(CO1)
-	vmovups	%xmm13,12 * SIZE(CO1)
-
-	vmovups	%xmm5 ,  	(CO1, LDC)
-	vmovups	%xmm8 , 4 * SIZE(CO1, LDC)
-	vmovups	%xmm11, 8 * SIZE(CO1, LDC)
-	vmovups	%xmm14,12 * SIZE(CO1, LDC)
-
-	vmovups	%xmm6 ,  	(CO1, LDC, 2)
-	vmovups	%xmm9 , 4 * SIZE(CO1, LDC, 2)
-	vmovups	%xmm12, 8 * SIZE(CO1, LDC, 2)
-	vmovups	%xmm15,12 * SIZE(CO1, LDC, 2)
-
-
-	addq	$16 * SIZE, CO1		# coffset += 16
-	decq	I			# i --
-	jg	.L6_11
-	ALIGN_4	
-
-/**************************************************************************
-* Rest of M 
-***************************************************************************/
-.L6_20:
-	// Test rest of M
-
-	testq	$15, M
-	jz	.L7_10		// to next 3 lines of N
-
-	testq	$8, M		
-	jz	.L6_21pre
-	ALIGN_4
-
-/**************************************************************************/
-
-.L6_20_1:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $6 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L6_20_6
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L6_20_2:
-
-	prefetcht0	B_PR1(BO,BI, SIZE)
-	KERNEL8x3_1(xxx)
-	KERNEL8x3_2(xxx)
-	KERNEL8x3_3(xxx)
-	KERNEL8x3_4(xxx)
-
-	KERNEL8x3_1(xxx)
-	prefetcht0	B_PR1+16(BO,BI, SIZE)
-	KERNEL8x3_2(xxx)
-	KERNEL8x3_3(xxx)
-	KERNEL8x3_4(xxx)
-
-	je	.L6_20_6
-
-	KERNEL8x3_1(xxx)
-	KERNEL8x3_2(xxx)
-	prefetcht0	B_PR1+32(BO,BI, SIZE)
-	KERNEL8x3_3(xxx)
-	KERNEL8x3_4(xxx)
-
-	KERNEL8x3_1(xxx)
-	KERNEL8x3_2(xxx)
-	KERNEL8x3_3(xxx)
-	KERNEL8x3_4(xxx)
-
-	je	.L6_20_6
-
-	jmp	.L6_20_2
-	ALIGN_4
-
-.L6_20_6:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L6_20_9
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L6_20_7:
-
-	KERNEL8x3_SUB(xxx)
-	addq	$3, BI
-	addq	$8, %rax
-	jl	.L6_20_7
-	ALIGN_4
-
-
-.L6_20_9:
-
-	vbroadcastss	ALPHA, %xmm0
-
-	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddps 4 * SIZE(CO1),%xmm0, %xmm7,%xmm7
-
-	vfmaddps 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-	vfmaddps 4 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
-
-	vfmaddps 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
-	vfmaddps 4 * SIZE(CO1, LDC, 2),%xmm0, %xmm9,%xmm9
-
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm7 , 4 * SIZE(CO1)
-
-	vmovups	%xmm5 ,  	(CO1, LDC)
-	vmovups	%xmm8 , 4 * SIZE(CO1, LDC)
-
-	vmovups	%xmm6 ,  	(CO1, LDC, 2)
-	vmovups	%xmm9 , 4 * SIZE(CO1, LDC, 2)
-
-
-	addq	$8 * SIZE, CO1		# coffset += 8
-	ALIGN_4
-	
-
-
-/**************************************************************************/
-
-.L6_21pre:
-
-	testq	$4, M		
-	jz	.L6_30
-	ALIGN_4
-
-.L6_21:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $6 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L6_26
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L6_22:
-
-	prefetcht0	B_PR1(BO,BI, SIZE)
-	KERNEL4x3_1(xxx)
-	KERNEL4x3_2(xxx)
-	KERNEL4x3_3(xxx)
-	KERNEL4x3_4(xxx)
-
-	KERNEL4x3_1(xxx)
-	prefetcht0	B_PR1+16(BO,BI, SIZE)
-	KERNEL4x3_2(xxx)
-	KERNEL4x3_3(xxx)
-	KERNEL4x3_4(xxx)
-
-	je	.L6_26
-
-	KERNEL4x3_1(xxx)
-	KERNEL4x3_2(xxx)
-	prefetcht0	B_PR1+32(BO,BI, SIZE)
-	KERNEL4x3_3(xxx)
-	KERNEL4x3_4(xxx)
-
-	KERNEL4x3_1(xxx)
-	KERNEL4x3_2(xxx)
-	KERNEL4x3_3(xxx)
-	KERNEL4x3_4(xxx)
-
-	je	.L6_26
-
-	jmp	.L6_22
-	ALIGN_4
-
-.L6_26:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L6_29
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L6_27:
-
-	KERNEL4x3_SUB(xxx)
-	addq	$3, BI
-	addq	$4, %rax
-	jl	.L6_27
-	ALIGN_4
-
-
-.L6_29:
-
-	vbroadcastss	ALPHA, %xmm0
-
-	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddps 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-	vfmaddps 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
-
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm5 ,  	(CO1, LDC)
-	vmovups	%xmm6 ,  	(CO1, LDC, 2)
-
-	addq	$4 * SIZE, CO1		# coffset += 4
-	ALIGN_4
-	
-
-.L6_30:
-	testq	$2, M		
-	jz	.L6_40
-
-	ALIGN_4
-
-.L6_31:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $6 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L6_36
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L6_32:
-
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL2x3_1(xxx)
-	KERNEL2x3_2(xxx)
-	KERNEL2x3_3(xxx)
-	KERNEL2x3_4(xxx)
-
-	KERNEL2x3_1(xxx)
-	prefetcht0	B_PR1+16(BO,BI,SIZE)
-	KERNEL2x3_2(xxx)
-	KERNEL2x3_3(xxx)
-	KERNEL2x3_4(xxx)
-
-	je	.L6_36
-
-	KERNEL2x3_1(xxx)
-	KERNEL2x3_2(xxx)
-	prefetcht0	B_PR1+32(BO,BI,SIZE)
-	KERNEL2x3_3(xxx)
-	KERNEL2x3_4(xxx)
-
-	KERNEL2x3_1(xxx)
-	KERNEL2x3_2(xxx)
-	KERNEL2x3_3(xxx)
-	KERNEL2x3_4(xxx)
-
-	je	.L6_36
-
-	jmp	.L6_32
-	ALIGN_4
-
-.L6_36:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L6_39
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-	
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L6_37:
-
-	KERNEL2x3_SUB(xxx)
-	addq	$3, BI
-	addq	$2, %rax
-	jl	.L6_37
-	ALIGN_4
-
-
-.L6_39:
-
-	vmovss	ALPHA, %xmm0
-
-	vfmaddss 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddss 1 * SIZE(CO1),%xmm0, %xmm8,%xmm8
-	vfmaddss 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-	vfmaddss 1 * SIZE(CO1, LDC),%xmm0, %xmm10,%xmm10
-	vfmaddss 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
-	vfmaddss 1 * SIZE(CO1, LDC, 2),%xmm0, %xmm12,%xmm12
-
-
-	vmovss	%xmm4 ,  	(CO1)
-	vmovss	%xmm8 , 1 * SIZE(CO1)
-	vmovss	%xmm5 ,  	(CO1, LDC)
-	vmovss	%xmm10, 1 * SIZE(CO1, LDC)
-	vmovss	%xmm6 ,  	(CO1, LDC, 2)
-	vmovss	%xmm12, 1 * SIZE(CO1, LDC, 2)
-
-	addq	$2 * SIZE, CO1		# coffset += 2
-	ALIGN_4
-
-.L6_40:
-	testq	$1, M		
-	jz	.L7_10		// to next 3 lines of N
-
-	ALIGN_4
-
-.L6_41:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $6 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L6_46
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L6_42:
-
-	KERNEL1x3_1(xxx)
-	KERNEL1x3_2(xxx)
-	KERNEL1x3_3(xxx)
-	KERNEL1x3_4(xxx)
-
-	KERNEL1x3_1(xxx)
-	KERNEL1x3_2(xxx)
-	KERNEL1x3_3(xxx)
-	KERNEL1x3_4(xxx)
-
-	je	.L6_46
-
-	KERNEL1x3_1(xxx)
-	KERNEL1x3_2(xxx)
-	KERNEL1x3_3(xxx)
-	KERNEL1x3_4(xxx)
-
-	KERNEL1x3_1(xxx)
-	KERNEL1x3_2(xxx)
-	KERNEL1x3_3(xxx)
-	KERNEL1x3_4(xxx)
-
-	je	.L6_46
-
-	jmp	.L6_42
-	ALIGN_4
-
-.L6_46:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L6_49
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L6_47:
-
-	KERNEL1x3_SUB(xxx)
-	addq	$3, BI
-	addq	$1, %rax
-	jl	.L6_47
-	ALIGN_4
-
-
-.L6_49:
-
-	vmovss	ALPHA, %xmm0
-
-	vfmaddss 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddss 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-	vfmaddss 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
-
-
-	vmovss	%xmm4 ,  	(CO1)
-	vmovss	%xmm5 ,  	(CO1, LDC)
-	vmovss	%xmm6 ,  	(CO1, LDC, 2)
-
-	addq	$1 * SIZE, CO1		# coffset += 1
-	ALIGN_4
-	
-
-
-
-/***************************************************************************************************************/
-
-.L7_10:
-	movq	C, CO1
-	leaq	(C, LDC, 2), C		 
-	leaq	(C, LDC, 1), C		// c += 3 * ldc
-
-	
-	movq	A, AO		 	// aoffset = a
-	addq	$32 * SIZE, AO
-
-	movq	M,  I
-	sarq	$4, I			// i = (m >> 4)
-	je	.L7_20
-
-	ALIGN_4
-
-.L7_11:
-        leaq    BUFFER2, BO             // second buffer to BO
-        addq    $6 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax			//  K = K - ( K % 8 )
-	je	.L7_16
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	salq	$4, %rax			// rax = rax * 16 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L7_12:
-
-	prefetcht0	B_PR1(BO,BI, SIZE)
-	KERNEL16x3_1(xxx)
-	KERNEL16x3_2(xxx)
-	KERNEL16x3_3(xxx)
-	KERNEL16x3_4(xxx)
-
-	KERNEL16x3_1(xxx)
-	prefetcht0	B_PR1+16(BO,BI, SIZE)
-	KERNEL16x3_2(xxx)
-	KERNEL16x3_3(xxx)
-	KERNEL16x3_4(xxx)
-
-	je	.L7_16
-
-	KERNEL16x3_1(xxx)
-	KERNEL16x3_2(xxx)
-	prefetcht0	B_PR1+32(BO,BI, SIZE)
-	KERNEL16x3_3(xxx)
-	KERNEL16x3_4(xxx)
-
-	KERNEL16x3_1(xxx)
-	KERNEL16x3_2(xxx)
-	KERNEL16x3_3(xxx)
-	KERNEL16x3_4(xxx)
-
-	je	.L7_16
-
-	jmp	.L7_12
-	ALIGN_4
-
-.L7_16:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L7_19
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	salq	$4, %rax			// rax = rax * 16 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L7_17:
-
-	KERNEL16x3_SUB(xxx)
-	addq	$3, BI
-	addq	$16, %rax
-	jl	.L7_17
-	ALIGN_4
-
-
-.L7_19:
-
-	vbroadcastss	ALPHA, %xmm0
-
-	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddps 4 * SIZE(CO1),%xmm0, %xmm7,%xmm7
-	vfmaddps 8 * SIZE(CO1),%xmm0, %xmm10,%xmm10
-	vfmaddps 12 * SIZE(CO1),%xmm0, %xmm13,%xmm13
-
-	vfmaddps 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-	vfmaddps 4 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
-	vfmaddps 8 * SIZE(CO1, LDC),%xmm0, %xmm11,%xmm11
-	vfmaddps 12 * SIZE(CO1, LDC),%xmm0, %xmm14,%xmm14
-
-	vfmaddps 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
-	vfmaddps 4 * SIZE(CO1, LDC, 2),%xmm0, %xmm9,%xmm9
-	vfmaddps 8 * SIZE(CO1, LDC, 2),%xmm0, %xmm12,%xmm12
-	vfmaddps 12 * SIZE(CO1, LDC, 2),%xmm0, %xmm15,%xmm15
-
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm7 , 4 * SIZE(CO1)
-	vmovups	%xmm10, 8 * SIZE(CO1)
-	vmovups	%xmm13,12 * SIZE(CO1)
-
-	vmovups	%xmm5 ,  	(CO1, LDC)
-	vmovups	%xmm8 , 4 * SIZE(CO1, LDC)
-	vmovups	%xmm11, 8 * SIZE(CO1, LDC)
-	vmovups	%xmm14,12 * SIZE(CO1, LDC)
-
-	vmovups	%xmm6 ,  	(CO1, LDC, 2)
-	vmovups	%xmm9 , 4 * SIZE(CO1, LDC, 2)
-	vmovups	%xmm12, 8 * SIZE(CO1, LDC, 2)
-	vmovups	%xmm15,12 * SIZE(CO1, LDC, 2)
-
-
-	addq	$16 * SIZE, CO1		# coffset += 16
-	decq	I			# i --
-	jg	.L7_11
-	ALIGN_4	
-
-/**************************************************************************
-* Rest of M 
-***************************************************************************/
-.L7_20:
-	// Test rest of M
-
-	testq	$15, M
-	jz	.L7_60		// to next 3 lines of N
-
-	testq	$8, M		
-	jz	.L7_21pre
-	ALIGN_4
-
-/**************************************************************************/
-
-.L7_20_1:
-        leaq    BUFFER2, BO             // first buffer to BO
-        addq    $6 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L7_20_6
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L7_20_2:
-
-	prefetcht0	B_PR1(BO,BI, SIZE)
-	KERNEL8x3_1(xxx)
-	KERNEL8x3_2(xxx)
-	KERNEL8x3_3(xxx)
-	KERNEL8x3_4(xxx)
-
-	KERNEL8x3_1(xxx)
-	prefetcht0	B_PR1+16(BO,BI, SIZE)
-	KERNEL8x3_2(xxx)
-	KERNEL8x3_3(xxx)
-	KERNEL8x3_4(xxx)
-
-	je	.L7_20_6
-
-	KERNEL8x3_1(xxx)
-	KERNEL8x3_2(xxx)
-	prefetcht0	B_PR1+32(BO,BI, SIZE)
-	KERNEL8x3_3(xxx)
-	KERNEL8x3_4(xxx)
-
-	KERNEL8x3_1(xxx)
-	KERNEL8x3_2(xxx)
-	KERNEL8x3_3(xxx)
-	KERNEL8x3_4(xxx)
-
-	je	.L7_20_6
-
-	jmp	.L7_20_2
-	ALIGN_4
-
-.L7_20_6:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L7_20_9
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L7_20_7:
-
-	KERNEL8x3_SUB(xxx)
-	addq	$3, BI
-	addq	$8, %rax
-	jl	.L7_20_7
-	ALIGN_4
-
-.L7_20_9:
-
-
-	vbroadcastss	ALPHA, %xmm0
-
-	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddps 4 * SIZE(CO1),%xmm0, %xmm7,%xmm7
-
-	vfmaddps 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-	vfmaddps 4 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
-
-	vfmaddps 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
-	vfmaddps 4 * SIZE(CO1, LDC, 2),%xmm0, %xmm9,%xmm9
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm7 , 4 * SIZE(CO1)
-
-	vmovups	%xmm5 ,  	(CO1, LDC)
-	vmovups	%xmm8 , 4 * SIZE(CO1, LDC)
-
-	vmovups	%xmm6 ,  	(CO1, LDC, 2)
-	vmovups	%xmm9 , 4 * SIZE(CO1, LDC, 2)
-
-
-	addq	$8 * SIZE, CO1		# coffset += 8
-	ALIGN_4
-	
-
-
-/**************************************************************************/
-
-.L7_21pre:
-
-	testq	$4, M		
-	jz	.L7_30
-	ALIGN_4
-
-.L7_21:
-        leaq    BUFFER2, BO             // second buffer to BO
-        addq    $6 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L7_26
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L7_22:
-
-	prefetcht0	B_PR1(BO,BI, SIZE)
-	KERNEL4x3_1(xxx)
-	KERNEL4x3_2(xxx)
-	KERNEL4x3_3(xxx)
-	KERNEL4x3_4(xxx)
-
-	KERNEL4x3_1(xxx)
-	prefetcht0	B_PR1+16(BO,BI, SIZE)
-	KERNEL4x3_2(xxx)
-	KERNEL4x3_3(xxx)
-	KERNEL4x3_4(xxx)
-
-	je	.L7_26
-
-	KERNEL4x3_1(xxx)
-	KERNEL4x3_2(xxx)
-	prefetcht0	B_PR1+32(BO,BI, SIZE)
-	KERNEL4x3_3(xxx)
-	KERNEL4x3_4(xxx)
-
-	KERNEL4x3_1(xxx)
-	KERNEL4x3_2(xxx)
-	KERNEL4x3_3(xxx)
-	KERNEL4x3_4(xxx)
-
-	je	.L7_26
-
-	jmp	.L7_22
-	ALIGN_4
-
-.L7_26:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L7_29
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L7_27:
-
-	KERNEL4x3_SUB(xxx)
-	addq	$3, BI
-	addq	$4, %rax
-	jl	.L7_27
-	ALIGN_4
-
-
-.L7_29:
-
-	vbroadcastss	ALPHA, %xmm0
-
-	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddps 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-	vfmaddps 	 (CO1, LDC, 2),%xmm0, %xmm6 ,%xmm6
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm5 ,  	(CO1, LDC)
-	vmovups	%xmm6 ,  	(CO1, LDC, 2)
-
-	addq	$4 * SIZE, CO1		# coffset += 4
-	ALIGN_4
-	
-
-.L7_30:
-	testq	$2, M		
-	jz	.L7_40
-
-	ALIGN_4
-
-.L7_31:
-        leaq    BUFFER2, BO             // second buffer to BO
-        addq    $6 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L7_36
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L7_32:
-
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL2x3_1(xxx)
-	KERNEL2x3_2(xxx)
-	KERNEL2x3_3(xxx)
-	KERNEL2x3_4(xxx)
-
-	KERNEL2x3_1(xxx)
-	prefetcht0	B_PR1+16(BO,BI,SIZE)
-	KERNEL2x3_2(xxx)
-	KERNEL2x3_3(xxx)
-	KERNEL2x3_4(xxx)
-
-	je	.L7_36
-
-	KERNEL2x3_1(xxx)
-	KERNEL2x3_2(xxx)
-	prefetcht0	B_PR1+32(BO,BI,SIZE)
-	KERNEL2x3_3(xxx)
-	KERNEL2x3_4(xxx)
-
-	KERNEL2x3_1(xxx)
-	KERNEL2x3_2(xxx)
-	KERNEL2x3_3(xxx)
-	KERNEL2x3_4(xxx)
-
-	je	.L7_36
-
-	jmp	.L7_32
-	ALIGN_4
-
-.L7_36:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L7_39
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-	
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L7_37:
-
-	KERNEL2x3_SUB(xxx)
-	addq	$3, BI
-	addq	$2, %rax
-	jl	.L7_37
-	ALIGN_4
-
-
-.L7_39:
-
-	vmovss	ALPHA, %xmm0
-
-	vfmaddss 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddss 1 * SIZE(CO1),%xmm0, %xmm8,%xmm8
-	vfmaddss 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-	vfmaddss 1 * SIZE(CO1, LDC),%xmm0, %xmm10,%xmm10
-	vfmaddss 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
-	vfmaddss 1 * SIZE(CO1, LDC, 2),%xmm0, %xmm12,%xmm12
-
-	vmovss	%xmm4 ,  	(CO1)
-	vmovss	%xmm8 , 1 * SIZE(CO1)
-	vmovss	%xmm5 ,  	(CO1, LDC)
-	vmovss	%xmm10, 1 * SIZE(CO1, LDC)
-	vmovss	%xmm6 ,  	(CO1, LDC, 2)
-	vmovss	%xmm12, 1 * SIZE(CO1, LDC, 2)
-
-	addq	$2 * SIZE, CO1		# coffset += 2
-	ALIGN_4
-
-.L7_40:
-	testq	$1, M		
-	jz	.L7_60		// to next 3 lines of N
-
-	ALIGN_4
-
-.L7_41:
-        leaq    BUFFER2, BO             // second buffer to BO
-        addq    $6 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L7_46
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L7_42:
-
-	KERNEL1x3_1(xxx)
-	KERNEL1x3_2(xxx)
-	KERNEL1x3_3(xxx)
-	KERNEL1x3_4(xxx)
-
-	KERNEL1x3_1(xxx)
-	KERNEL1x3_2(xxx)
-	KERNEL1x3_3(xxx)
-	KERNEL1x3_4(xxx)
-
-	je	.L7_46
-
-	KERNEL1x3_1(xxx)
-	KERNEL1x3_2(xxx)
-	KERNEL1x3_3(xxx)
-	KERNEL1x3_4(xxx)
-
-	KERNEL1x3_1(xxx)
-	KERNEL1x3_2(xxx)
-	KERNEL1x3_3(xxx)
-	KERNEL1x3_4(xxx)
-
-	je	.L7_46
-
-	jmp	.L7_42
-	ALIGN_4
-
-.L7_46:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L7_49
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L7_47:
-
-	KERNEL1x3_SUB(xxx)
-	addq	$3, BI
-	addq	$1, %rax
-	jl	.L7_47
-	ALIGN_4
-
-
-.L7_49:
-
-	vmovss	ALPHA, %xmm0
-
-	vfmaddss 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddss 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-	vfmaddss 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
-
-	vmovss	%xmm4 ,  	(CO1)
-	vmovss	%xmm5 ,  	(CO1, LDC)
-	vmovss	%xmm6 ,  	(CO1, LDC, 2)
-
-	addq	$1 * SIZE, CO1		# coffset += 1
-	ALIGN_4
-	
-
-
-.L7_60:
-
-	decq	J			// j --
-	jg	.L6_01
-
-
-.L2_0:
-	cmpq	$0, Nmod6		// N % 6 == 0
-	je	.L999
-
-/************************************************************************************************
-* Loop for Nmod6 / 2 > 0
-*************************************************************************************************/
-
-	movq	Nmod6, J		
-	sarq	$1, J			// j = j / 2
-	je	.L1_0
-	ALIGN_4
-
-.L2_01:
-	// copy to sub buffer
-	movq	B, BO1
-	leaq    BUFFER1, BO		// first buffer to BO
-	movq	K, %rax
-	ALIGN_4
-
-.L2_02b:
-
-	vmovsd	(BO1), %xmm0
-	vmovsd	%xmm0,       (BO)
-	addq	$2*SIZE,BO1
-	addq	$2*SIZE,BO
-	decq	%rax
-	jnz	.L2_02b
-
-.L2_02c:
-
-	movq	BO1, B			// next offset of B
-
-.L2_10:
-	movq	C, CO1
-	leaq	(C, LDC, 2), C		// c += 2 * ldc
-
-	
-	movq	A, AO		 	// aoffset = a
-	addq	$32 * SIZE, AO
-
-	movq	M,  I
-	sarq	$4, I			// i = (m >> 4)
-	je	.L2_20
-
-	ALIGN_4
-
-.L2_11:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax			//  K = K - ( K % 8 )
-	je	.L2_16
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$4, %rax			// rax = rax * 16 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_12:
-
-	prefetcht0	B_PR1(BO,BI, SIZE)
-	KERNEL16x2_1(xxx)
-	KERNEL16x2_2(xxx)
-	KERNEL16x2_3(xxx)
-	KERNEL16x2_4(xxx)
-
-	KERNEL16x2_1(xxx)
-	KERNEL16x2_2(xxx)
-	KERNEL16x2_3(xxx)
-	KERNEL16x2_4(xxx)
-
-	je	.L2_16
-
-	prefetcht0	B_PR1(BO,BI, SIZE)
-	KERNEL16x2_1(xxx)
-	KERNEL16x2_2(xxx)
-	KERNEL16x2_3(xxx)
-	KERNEL16x2_4(xxx)
-
-	KERNEL16x2_1(xxx)
-	KERNEL16x2_2(xxx)
-	KERNEL16x2_3(xxx)
-	KERNEL16x2_4(xxx)
-
-	je	.L2_16
-
-	jmp	.L2_12
-	ALIGN_4
-
-.L2_16:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_19
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$4, %rax			// rax = rax * 16 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_17:
-
-	KERNEL16x2_SUB(xxx)
-	addq	$2, BI
-	addq	$16, %rax
-	jl	.L2_17
-	ALIGN_4
-
-
-.L2_19:
-
-	vbroadcastss	ALPHA, %xmm0
-
-	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddps 4 * SIZE(CO1),%xmm0, %xmm7,%xmm7
-	vfmaddps 8 * SIZE(CO1),%xmm0, %xmm10,%xmm10
-	vfmaddps 12 * SIZE(CO1),%xmm0, %xmm13,%xmm13
-
-	vfmaddps 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-	vfmaddps 4 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
-	vfmaddps 8 * SIZE(CO1, LDC),%xmm0, %xmm11,%xmm11
-	vfmaddps 12 * SIZE(CO1, LDC),%xmm0, %xmm14,%xmm14
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm7 , 4 * SIZE(CO1)
-	vmovups	%xmm10, 8 * SIZE(CO1)
-	vmovups	%xmm13,12 * SIZE(CO1)
-
-	vmovups	%xmm5 ,  	(CO1, LDC)
-	vmovups	%xmm8 , 4 * SIZE(CO1, LDC)
-	vmovups	%xmm11, 8 * SIZE(CO1, LDC)
-	vmovups	%xmm14,12 * SIZE(CO1, LDC)
-
-	addq	$16 * SIZE, CO1		# coffset += 16
-	decq	I			# i --
-	jg	.L2_11
-	ALIGN_4	
-
-/**************************************************************************
-* Rest of M 
-***************************************************************************/
-.L2_20:
-	// Test rest of M
-
-	testq	$15, M
-	jz	.L2_60		// to next 3 lines of N
-
-	testq	$8, M		
-	jz	.L2_21pre
-	ALIGN_4
-
-/**************************************************************************/
-
-.L2_20_1:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L2_20_6
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_20_2:
-
-	prefetcht0	B_PR1(BO,BI, SIZE)
-	KERNEL8x2_1(xxx)
-	KERNEL8x2_2(xxx)
-	KERNEL8x2_3(xxx)
-	KERNEL8x2_4(xxx)
-
-	KERNEL8x2_1(xxx)
-	KERNEL8x2_2(xxx)
-	KERNEL8x2_3(xxx)
-	KERNEL8x2_4(xxx)
-
-	je	.L2_20_6
-
-	prefetcht0	B_PR1(BO,BI, SIZE)
-	KERNEL8x2_1(xxx)
-	KERNEL8x2_2(xxx)
-	KERNEL8x2_3(xxx)
-	KERNEL8x2_4(xxx)
-
-	KERNEL8x2_1(xxx)
-	KERNEL8x2_2(xxx)
-	KERNEL8x2_3(xxx)
-	KERNEL8x2_4(xxx)
-
-	je	.L2_20_6
-
-	jmp	.L2_20_2
-	ALIGN_4
-
-.L2_20_6:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_20_9
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_20_7:
-
-	KERNEL8x2_SUB(xxx)
-	addq	$2, BI
-	addq	$8, %rax
-	jl	.L2_20_7
-	ALIGN_4
-
-
-.L2_20_9:
-
-	vbroadcastss	ALPHA, %xmm0
-
-	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddps 4 * SIZE(CO1),%xmm0, %xmm7,%xmm7
-
-	vfmaddps 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-	vfmaddps 4 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm7 , 4 * SIZE(CO1)
-
-	vmovups	%xmm5 ,  	(CO1, LDC)
-	vmovups	%xmm8 , 4 * SIZE(CO1, LDC)
-
-
-	addq	$8 * SIZE, CO1		# coffset += 8
-	ALIGN_4
-	
-
-
-/**************************************************************************/
-
-.L2_21pre:
-
-	testq	$4, M		
-	jz	.L2_30
-	ALIGN_4
-
-.L2_21:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L2_26
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 1 ; number of values
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_22:
-
-	prefetcht0	B_PR1(BO,BI, SIZE)
-	KERNEL4x2_1(xxx)
-	KERNEL4x2_2(xxx)
-	KERNEL4x2_3(xxx)
-	KERNEL4x2_4(xxx)
-
-	KERNEL4x2_1(xxx)
-	KERNEL4x2_2(xxx)
-	KERNEL4x2_3(xxx)
-	KERNEL4x2_4(xxx)
-
-	je	.L2_26
-
-	prefetcht0	B_PR1(BO,BI, SIZE)
-	KERNEL4x2_1(xxx)
-	KERNEL4x2_2(xxx)
-	KERNEL4x2_3(xxx)
-	KERNEL4x2_4(xxx)
-
-	KERNEL4x2_1(xxx)
-	KERNEL4x2_2(xxx)
-	KERNEL4x2_3(xxx)
-	KERNEL4x2_4(xxx)
-
-	je	.L2_26
-
-	jmp	.L2_22
-	ALIGN_4
-
-.L2_26:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_29
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_27:
-
-	KERNEL4x2_SUB(xxx)
-	addq	$2, BI
-	addq	$4, %rax
-	jl	.L2_27
-	ALIGN_4
-
-
-.L2_29:
-
-	vbroadcastss	ALPHA, %xmm0
-
-	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddps 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm5 ,  	(CO1, LDC)
-
-
-	addq	$4 * SIZE, CO1		# coffset += 4
-	ALIGN_4
-	
-
-.L2_30:
-	testq	$2, M		
-	jz	.L2_40
-
-	ALIGN_4
-
-.L2_31:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L2_36
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_32:
-
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL2x2_1(xxx)
-	KERNEL2x2_2(xxx)
-	KERNEL2x2_3(xxx)
-	KERNEL2x2_4(xxx)
-
-	KERNEL2x2_1(xxx)
-	KERNEL2x2_2(xxx)
-	KERNEL2x2_3(xxx)
-	KERNEL2x2_4(xxx)
-
-	je	.L2_36
-
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL2x2_1(xxx)
-	KERNEL2x2_2(xxx)
-	KERNEL2x2_3(xxx)
-	KERNEL2x2_4(xxx)
-
-	KERNEL2x2_1(xxx)
-	KERNEL2x2_2(xxx)
-	KERNEL2x2_3(xxx)
-	KERNEL2x2_4(xxx)
-
-	je	.L2_36
-
-	jmp	.L2_32
-	ALIGN_4
-
-.L2_36:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_39
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-	
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_37:
-
-	KERNEL2x2_SUB(xxx)
-	addq	$2, BI
-	addq	$2, %rax
-	jl	.L2_37
-	ALIGN_4
-
-
-.L2_39:
-
-	vmovss	ALPHA, %xmm0
-
-	vfmaddss 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddss 1 * SIZE(CO1),%xmm0, %xmm8,%xmm8
-	vfmaddss 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-	vfmaddss 1 * SIZE(CO1, LDC),%xmm0, %xmm10,%xmm10
-
-	vmovss	%xmm4 ,  	(CO1)
-	vmovss	%xmm8 , 1 * SIZE(CO1)
-	vmovss	%xmm5 ,  	(CO1, LDC)
-	vmovss	%xmm10, 1 * SIZE(CO1, LDC)
-
-	addq	$2 * SIZE, CO1		# coffset += 2
-	ALIGN_4
-
-.L2_40:
-	testq	$1, M		
-	jz	.L2_60		// to next 2 lines of N
-
-	ALIGN_4
-
-.L2_41:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L2_46
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_42:
-
-	KERNEL1x2_1(xxx)
-	KERNEL1x2_2(xxx)
-	KERNEL1x2_3(xxx)
-	KERNEL1x2_4(xxx)
-
-	KERNEL1x2_1(xxx)
-	KERNEL1x2_2(xxx)
-	KERNEL1x2_3(xxx)
-	KERNEL1x2_4(xxx)
-
-	je	.L2_46
-
-	KERNEL1x2_1(xxx)
-	KERNEL1x2_2(xxx)
-	KERNEL1x2_3(xxx)
-	KERNEL1x2_4(xxx)
-
-	KERNEL1x2_1(xxx)
-	KERNEL1x2_2(xxx)
-	KERNEL1x2_3(xxx)
-	KERNEL1x2_4(xxx)
-
-	je	.L2_46
-
-	jmp	.L2_42
-	ALIGN_4
-
-.L2_46:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_49
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_47:
-
-	KERNEL1x2_SUB(xxx)
-	addq	$2, BI
-	addq	$1, %rax
-	jl	.L2_47
-	ALIGN_4
-
-
-.L2_49:
-
-	vmovss	ALPHA, %xmm0
-
-	vfmaddss 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddss 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-
-	vmovss	%xmm4 ,  	(CO1)
-	vmovss	%xmm5 ,  	(CO1, LDC)
-
-	addq	$1 * SIZE, CO1		# coffset += 1
-	ALIGN_4
-	
-.L2_60:
-
-	decq	J			// j --
-	jg	.L2_01			// next 2 lines of N
-
-
-
-.L1_0:
-
-/************************************************************************************************
-* Loop for Nmod6 % 2 > 0
-*************************************************************************************************/
-
-	movq	Nmod6, J		
-	andq	$1, J			// j % 2
-	je	.L999
-	ALIGN_4
-
-.L1_01:
-	// copy to sub buffer
-	movq	B, BO1
-	leaq    BUFFER1, BO		// first buffer to BO
-	movq	K, %rax
-	ALIGN_4
-
-.L1_02b:
-
-	vmovss	(BO1), %xmm0
-	vmovss	%xmm0,       (BO)
-	addq	$1*SIZE,BO1
-	addq	$1*SIZE,BO
-	decq	%rax
-	jnz	.L1_02b
-
-.L1_02c:
-
-	movq	BO1, B			// next offset of B
-
-.L1_10:
-	movq	C, CO1
-	leaq	(C, LDC, 1), C		// c += 1 * ldc
-
-	
-	movq	A, AO		 	// aoffset = a
-	addq	$32 * SIZE, AO
-
-	movq	M,  I
-	sarq	$4, I			// i = (m >> 4)
-	je	.L1_20
-
-	ALIGN_4
-
-.L1_11:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax			//  K = K - ( K % 8 )
-	je	.L1_16
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$4, %rax			// rax = rax * 16 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_12:
-
-	prefetcht0	B_PR1(BO,BI, SIZE)
-	KERNEL16x1_1(xxx)
-	KERNEL16x1_2(xxx)
-	KERNEL16x1_3(xxx)
-	KERNEL16x1_4(xxx)
-
-	KERNEL16x1_1(xxx)
-	KERNEL16x1_2(xxx)
-	KERNEL16x1_3(xxx)
-	KERNEL16x1_4(xxx)
-
-	je	.L1_16
-
-	KERNEL16x1_1(xxx)
-	KERNEL16x1_2(xxx)
-	KERNEL16x1_3(xxx)
-	KERNEL16x1_4(xxx)
-
-	KERNEL16x1_1(xxx)
-	KERNEL16x1_2(xxx)
-	KERNEL16x1_3(xxx)
-	KERNEL16x1_4(xxx)
-
-	je	.L1_16
-
-	jmp	.L1_12
-	ALIGN_4
-
-.L1_16:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_19
-
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$4, %rax			// rax = rax * 16 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_17:
-
-	KERNEL16x1_SUB(xxx)
-	addq	$1, BI
-	addq	$16, %rax
-	jl	.L1_17
-	ALIGN_4
-
-
-.L1_19:
-
-	vbroadcastss	ALPHA, %xmm0
-
-	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddps 4 * SIZE(CO1),%xmm0, %xmm7,%xmm7
-	vfmaddps 8 * SIZE(CO1),%xmm0, %xmm10,%xmm10
-	vfmaddps 12 * SIZE(CO1),%xmm0, %xmm13,%xmm13
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm7 , 4 * SIZE(CO1)
-	vmovups	%xmm10, 8 * SIZE(CO1)
-	vmovups	%xmm13,12 * SIZE(CO1)
-
-	addq	$16 * SIZE, CO1		# coffset += 16
-	decq	I			# i --
-	jg	.L1_11
-	ALIGN_4	
-
-/**************************************************************************
-* Rest of M 
-***************************************************************************/
-.L1_20:
-	// Test rest of M
-
-	testq	$15, M
-	jz	.L999
-
-	testq	$8, M		
-	jz	.L1_21pre
-	ALIGN_4
-
-/**************************************************************************/
-
-.L1_20_1:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L1_20_6
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_20_2:
-
-	prefetcht0	B_PR1(BO,BI, SIZE)
-	KERNEL8x1_1(xxx)
-	KERNEL8x1_2(xxx)
-	KERNEL8x1_3(xxx)
-	KERNEL8x1_4(xxx)
-
-	KERNEL8x1_1(xxx)
-	KERNEL8x1_2(xxx)
-	KERNEL8x1_3(xxx)
-	KERNEL8x1_4(xxx)
-
-	je	.L1_20_6
-
-	KERNEL8x1_1(xxx)
-	KERNEL8x1_2(xxx)
-	KERNEL8x1_3(xxx)
-	KERNEL8x1_4(xxx)
-
-	KERNEL8x1_1(xxx)
-	KERNEL8x1_2(xxx)
-	KERNEL8x1_3(xxx)
-	KERNEL8x1_4(xxx)
-
-	je	.L1_20_6
-
-	jmp	.L1_20_2
-	ALIGN_4
-
-.L1_20_6:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_20_9
-
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_20_7:
-
-	KERNEL8x1_SUB(xxx)
-	addq	$1, BI
-	addq	$8, %rax
-	jl	.L1_20_7
-	ALIGN_4
-
-
-.L1_20_9:
-
-	vbroadcastss	ALPHA, %xmm0
-
-	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddps 4 * SIZE(CO1),%xmm0, %xmm7,%xmm7
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm7 , 4 * SIZE(CO1)
-
-	addq	$8 * SIZE, CO1		# coffset += 8
-	ALIGN_4
-	
-
-
-/**************************************************************************/
-
-.L1_21pre:
-
-	testq	$4, M		
-	jz	.L1_30
-	ALIGN_4
-
-.L1_21:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L1_26
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_22:
-
-	prefetcht0	B_PR1(BO,BI, SIZE)
-	KERNEL4x1_1(xxx)
-	KERNEL4x1_2(xxx)
-	KERNEL4x1_3(xxx)
-	KERNEL4x1_4(xxx)
-
-	KERNEL4x1_1(xxx)
-	KERNEL4x1_2(xxx)
-	KERNEL4x1_3(xxx)
-	KERNEL4x1_4(xxx)
-
-	je	.L1_26
-
-	KERNEL4x1_1(xxx)
-	KERNEL4x1_2(xxx)
-	KERNEL4x1_3(xxx)
-	KERNEL4x1_4(xxx)
-
-	KERNEL4x1_1(xxx)
-	KERNEL4x1_2(xxx)
-	KERNEL4x1_3(xxx)
-	KERNEL4x1_4(xxx)
-
-	je	.L1_26
-
-	jmp	.L1_22
-	ALIGN_4
-
-.L1_26:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_29
-
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_27:
-
-	KERNEL4x1_SUB(xxx)
-	addq	$1, BI
-	addq	$4, %rax
-	jl	.L1_27
-	ALIGN_4
-
-
-.L1_29:
-
-	vbroadcastss	ALPHA, %xmm0
-
-	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
-
-	vmovups	%xmm4 ,  	(CO1)
-
-	addq	$4 * SIZE, CO1		# coffset += 4
-	ALIGN_4
-	
-
-.L1_30:
-	testq	$2, M		
-	jz	.L1_40
-
-	ALIGN_4
-
-.L1_31:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L1_36
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_32:
-
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL2x1_1(xxx)
-	KERNEL2x1_2(xxx)
-	KERNEL2x1_3(xxx)
-	KERNEL2x1_4(xxx)
-
-	KERNEL2x1_1(xxx)
-	KERNEL2x1_2(xxx)
-	KERNEL2x1_3(xxx)
-	KERNEL2x1_4(xxx)
-
-	je	.L1_36
-
-	KERNEL2x1_1(xxx)
-	KERNEL2x1_2(xxx)
-	KERNEL2x1_3(xxx)
-	KERNEL2x1_4(xxx)
-
-	KERNEL2x1_1(xxx)
-	KERNEL2x1_2(xxx)
-	KERNEL2x1_3(xxx)
-	KERNEL2x1_4(xxx)
-
-	je	.L1_36
-
-	jmp	.L1_32
-	ALIGN_4
-
-.L1_36:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_39
-
-	movq    %rax, BI                        //  Index for BO
-	
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_37:
-
-	KERNEL2x1_SUB(xxx)
-	addq	$1, BI
-	addq	$2, %rax
-	jl	.L1_37
-	ALIGN_4
-
-
-.L1_39:
-
-	vmovss	ALPHA, %xmm0
-
-	vfmaddss 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddss 1 * SIZE(CO1),%xmm0, %xmm8,%xmm8
-
-	vmovss	%xmm4 ,  	(CO1)
-	vmovss	%xmm8 , 1 * SIZE(CO1)
-
-	addq	$2 * SIZE, CO1		# coffset += 2
-	ALIGN_4
-
-.L1_40:
-	testq	$1, M		
-	jz	.L999
-
-	ALIGN_4
-
-.L1_41:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L1_46
-	movq    %rax, BI                        //  Index for BO
-
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_42:
-
-	KERNEL1x1_1(xxx)
-	KERNEL1x1_2(xxx)
-	KERNEL1x1_3(xxx)
-	KERNEL1x1_4(xxx)
-
-	KERNEL1x1_1(xxx)
-	KERNEL1x1_2(xxx)
-	KERNEL1x1_3(xxx)
-	KERNEL1x1_4(xxx)
-
-	je	.L1_46
-
-	KERNEL1x1_1(xxx)
-	KERNEL1x1_2(xxx)
-	KERNEL1x1_3(xxx)
-	KERNEL1x1_4(xxx)
-
-	KERNEL1x1_1(xxx)
-	KERNEL1x1_2(xxx)
-	KERNEL1x1_3(xxx)
-	KERNEL1x1_4(xxx)
-
-	je	.L1_46
-
-	jmp	.L1_42
-	ALIGN_4
-
-.L1_46:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_49
-
-	movq    %rax, BI                        //  Index for BO
-
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_47:
-
-	KERNEL1x1_SUB(xxx)
-	addq	$1, BI
-	addq	$1, %rax
-	jl	.L1_47
-	ALIGN_4
-
-
-.L1_49:
-
-	vmovss	ALPHA, %xmm0
-
-	vfmaddss 	 (CO1),%xmm0, %xmm4,%xmm4
-
-	vmovss	%xmm4 ,  	(CO1)
-
-	addq	$1 * SIZE, CO1		# coffset += 1
-	ALIGN_4
-	
-
-.L999:
-	movq   		SP, %rsp
-	movq	   (%rsp), %rbx
-	movq	  8(%rsp), %rbp
-	movq	 16(%rsp), %r12
-	movq	 24(%rsp), %r13
-	movq	 32(%rsp), %r14
-	movq	 40(%rsp), %r15
-
-#ifdef WINDOWS_ABI
-	movq	 48(%rsp), %rdi
-	movq	 56(%rsp), %rsi
-	movups	 64(%rsp), %xmm6
-	movups	 80(%rsp), %xmm7
-	movups	 96(%rsp), %xmm8
-	movups	112(%rsp), %xmm9
-	movups	128(%rsp), %xmm10
-	movups	144(%rsp), %xmm11
-	movups	160(%rsp), %xmm12
-	movups	176(%rsp), %xmm13
-	movups	192(%rsp), %xmm14
-	movups	208(%rsp), %xmm15
-#endif
-
-	addq	$STACKSIZE, %rsp
-	ret
-
-	EPILOGUE
-
-
-#else
-/*************************************************************************************
-* TRMM Kernel
-*************************************************************************************/
-
-
-	PROLOGUE
-	PROFCODE
-	
-	subq	$STACKSIZE, %rsp
-	movq	%rbx,   (%rsp)
-	movq	%rbp,  8(%rsp)
-	movq	%r12, 16(%rsp)
-	movq	%r13, 24(%rsp)
-	movq	%r14, 32(%rsp)
-	movq	%r15, 40(%rsp)
-
-	vzeroupper
-
-#ifdef WINDOWS_ABI
-	movq	%rdi,    48(%rsp)
-	movq	%rsi,    56(%rsp)
-	movups	%xmm6,   64(%rsp)
-	movups	%xmm7,   80(%rsp)
-	movups	%xmm8,   96(%rsp)
-	movups	%xmm9,  112(%rsp)
-	movups	%xmm10, 128(%rsp)
-	movups	%xmm11, 144(%rsp)
-	movups	%xmm12, 160(%rsp)
-	movups	%xmm13, 176(%rsp)
-	movups	%xmm14, 192(%rsp)
-	movups	%xmm15, 208(%rsp)
-
-	movq	ARG1,      OLD_M
-	movq	ARG2,      OLD_N
-	movq	ARG3,      OLD_K
-	movq	OLD_A,     A
-	movq	OLD_B,     B
-	movq	OLD_C,     C
-	movq	OLD_LDC,   LDC
-#ifdef TRMMKERNEL
-	movsd	OLD_OFFSET, %xmm12
-#endif
-	vmovaps	%xmm3, %xmm0
-
-#else
-	movq	STACKSIZE +  8(%rsp), LDC
-#ifdef TRMMKERNEL
-	movsd	STACKSIZE + 16(%rsp), %xmm12
-#endif
-
-#endif
-
-	movq    %rsp, SP      # save old stack
-        subq    $128 + L_BUFFER_SIZE, %rsp
-        andq    $-4096, %rsp    # align stack
-
-        STACK_TOUCH
-
-	cmpq	$0, OLD_M
-	je	.L999
-
-	cmpq	$0, OLD_N
-	je	.L999
-
-	cmpq	$0, OLD_K
-	je	.L999
-
-	movq	OLD_M, M
-	movq	OLD_N, N
-	movq	OLD_K, K
-
-	vmovsd	 %xmm0, ALPHA
-
-	salq	$BASE_SHIFT, LDC
-
-	movq    N, %rax
-        xorq    %rdx, %rdx
-        movq    $2,  %rdi
-        divq    %rdi                    //    N / 6
-        movq    %rax, Ndiv6             //    N / 6
-        movq    %rdx, Nmod6             //    N % 6
-
-	
-
-#ifdef TRMMKERNEL
-	vmovsd	%xmm12, OFFSET
-	vmovsd	%xmm12, KK
-#ifndef LEFT
-	negq	KK
-#endif	
-#endif
-
-	movq	Ndiv6,  J
-	cmpq	$0, J
-	je	.L1_0
-	ALIGN_4
-
-.L2_01:
-	// copy to sub buffer
-	movq	B, BO1
-	leaq    BUFFER1, BO		// first buffer to BO
-	movq	K, %rax
-	ALIGN_4
-
-.L2_02b:
-
-	vmovsd	(BO1), %xmm0
-	vmovsd	%xmm0,       (BO)
-	addq	$2*SIZE,BO1
-	addq	$2*SIZE,BO
-	decq	%rax
-	jnz	.L2_02b
-
-.L2_02c:
-
-	movq	BO1, B			// next offset of B
-
-.L2_10:
-	movq	C, CO1
-	leaq	(C, LDC, 2), C		// c += 2 * ldc
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        movq    OFFSET, %rax
-        movq    %rax, KK
-#endif
-	
-	movq	A, AO		 	// aoffset = a
-	addq	$32 * SIZE, AO
-
-	movq	M,  I
-	sarq	$4, I			// i = (m >> 4)
-	je	.L2_20
-
-	ALIGN_4
-
-.L2_11:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO
-        salq    $4, %rax                        // rax = rax * 16 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $16, %rax	// number of values in AO
-#else
-        addq    $2, %rax	// number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-	andq	$-8, %rax			//  K = K - ( K % 8 )
-	je	.L2_16
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$4, %rax			// rax = rax * 16 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_12:
-
-	prefetcht0	B_PR1(BO,BI, SIZE)
-	KERNEL16x2_1(xxx)
-	KERNEL16x2_2(xxx)
-	KERNEL16x2_3(xxx)
-	KERNEL16x2_4(xxx)
-
-	KERNEL16x2_1(xxx)
-	KERNEL16x2_2(xxx)
-	KERNEL16x2_3(xxx)
-	KERNEL16x2_4(xxx)
-
-	je	.L2_16
-
-	prefetcht0	B_PR1(BO,BI, SIZE)
-	KERNEL16x2_1(xxx)
-	KERNEL16x2_2(xxx)
-	KERNEL16x2_3(xxx)
-	KERNEL16x2_4(xxx)
-
-	KERNEL16x2_1(xxx)
-	KERNEL16x2_2(xxx)
-	KERNEL16x2_3(xxx)
-	KERNEL16x2_4(xxx)
-
-	je	.L2_16
-
-	jmp	.L2_12
-	ALIGN_4
-
-.L2_16:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_19
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$4, %rax			// rax = rax * 16 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_17:
-
-	KERNEL16x2_SUB(xxx)
-	addq	$2, BI
-	addq	$16, %rax
-	jl	.L2_17
-	ALIGN_4
-
-
-.L2_19:
-
-	vbroadcastss	ALPHA, %xmm0
-
-#ifndef TRMMKERNEL
-
-	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddps 4 * SIZE(CO1),%xmm0, %xmm7,%xmm7
-	vfmaddps 8 * SIZE(CO1),%xmm0, %xmm10,%xmm10
-	vfmaddps 12 * SIZE(CO1),%xmm0, %xmm13,%xmm13
-
-	vfmaddps 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-	vfmaddps 4 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
-	vfmaddps 8 * SIZE(CO1, LDC),%xmm0, %xmm11,%xmm11
-	vfmaddps 12 * SIZE(CO1, LDC),%xmm0, %xmm14,%xmm14
-
-#else
-	vmulps	%xmm0, %xmm4,%xmm4
-	vmulps	%xmm0, %xmm7,%xmm7
-	vmulps	%xmm0, %xmm10,%xmm10
-	vmulps	%xmm0, %xmm13,%xmm13
-
-	vmulps	%xmm0, %xmm5,%xmm5
-	vmulps	%xmm0, %xmm8,%xmm8
-	vmulps	%xmm0, %xmm11,%xmm11
-	vmulps	%xmm0, %xmm14,%xmm14
-
-#endif
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm7 , 4 * SIZE(CO1)
-	vmovups	%xmm10, 8 * SIZE(CO1)
-	vmovups	%xmm13,12 * SIZE(CO1)
-
-	vmovups	%xmm5 ,  	(CO1, LDC)
-	vmovups	%xmm8 , 4 * SIZE(CO1, LDC)
-	vmovups	%xmm11, 8 * SIZE(CO1, LDC)
-	vmovups	%xmm14,12 * SIZE(CO1, LDC)
-
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO
-        salq    $4, %rax                        // rax = rax * 16 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $16, KK				
-#endif
-
-	addq	$16 * SIZE, CO1		# coffset += 16
-	decq	I			# i --
-	jg	.L2_11
-	ALIGN_4	
-
-/**************************************************************************
-* Rest of M 
-***************************************************************************/
-.L2_20:
-	// Test rest of M
-
-	testq	$15, M
-	jz	.L2_60		// to next 3 lines of N
-
-	testq	$8, M		
-	jz	.L2_21pre
-	ALIGN_4
-
-/**************************************************************************/
-
-.L2_20_1:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO
-        salq    $3, %rax                        // rax = rax * 8 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $8, %rax        // number of values in A
-#else
-        addq    $2, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-
-	andq	$-8, %rax
-	je	.L2_20_6
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_20_2:
-
-	prefetcht0	B_PR1(BO,BI, SIZE)
-	KERNEL8x2_1(xxx)
-	KERNEL8x2_2(xxx)
-	KERNEL8x2_3(xxx)
-	KERNEL8x2_4(xxx)
-
-	KERNEL8x2_1(xxx)
-	KERNEL8x2_2(xxx)
-	KERNEL8x2_3(xxx)
-	KERNEL8x2_4(xxx)
-
-	je	.L2_20_6
-
-	prefetcht0	B_PR1(BO,BI, SIZE)
-	KERNEL8x2_1(xxx)
-	KERNEL8x2_2(xxx)
-	KERNEL8x2_3(xxx)
-	KERNEL8x2_4(xxx)
-
-	KERNEL8x2_1(xxx)
-	KERNEL8x2_2(xxx)
-	KERNEL8x2_3(xxx)
-	KERNEL8x2_4(xxx)
-
-	je	.L2_20_6
-
-	jmp	.L2_20_2
-	ALIGN_4
-
-.L2_20_6:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_20_9
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_20_7:
-
-	KERNEL8x2_SUB(xxx)
-	addq	$2, BI
-	addq	$8, %rax
-	jl	.L2_20_7
-	ALIGN_4
-
-
-.L2_20_9:
-
-	vbroadcastss	ALPHA, %xmm0
-
-#ifndef TRMMKERNEL
-
-	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddps 4 * SIZE(CO1),%xmm0, %xmm7,%xmm7
-
-	vfmaddps 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-	vfmaddps 4 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
-
-#else
-	vmulps	%xmm0, %xmm4,%xmm4
-	vmulps	%xmm0, %xmm7,%xmm7
-
-	vmulps	%xmm0, %xmm5,%xmm5
-	vmulps	%xmm0, %xmm8,%xmm8
-
-#endif
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm7 , 4 * SIZE(CO1)
-
-	vmovups	%xmm5 ,  	(CO1, LDC)
-	vmovups	%xmm8 , 4 * SIZE(CO1, LDC)
-
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax 
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO         
-        salq    $3, %rax                        // rax = rax * 8 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif  
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $8, KK
-#endif
-
-	addq	$8 * SIZE, CO1		# coffset += 8
-	ALIGN_4
-	
-
-
-/**************************************************************************/
-
-.L2_21pre:
-
-	testq	$4, M		
-	jz	.L2_30
-	ALIGN_4
-
-.L2_21:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO
-        salq    $2, %rax                        // rax = rax * 4 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $4, %rax        // number of values in A
-#else
-        addq    $2, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-
-	andq	$-8, %rax
-	je	.L2_26
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 1 ; number of values
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_22:
-
-	prefetcht0	B_PR1(BO,BI, SIZE)
-	KERNEL4x2_1(xxx)
-	KERNEL4x2_2(xxx)
-	KERNEL4x2_3(xxx)
-	KERNEL4x2_4(xxx)
-
-	KERNEL4x2_1(xxx)
-	KERNEL4x2_2(xxx)
-	KERNEL4x2_3(xxx)
-	KERNEL4x2_4(xxx)
-
-	je	.L2_26
-
-	prefetcht0	B_PR1(BO,BI, SIZE)
-	KERNEL4x2_1(xxx)
-	KERNEL4x2_2(xxx)
-	KERNEL4x2_3(xxx)
-	KERNEL4x2_4(xxx)
-
-	KERNEL4x2_1(xxx)
-	KERNEL4x2_2(xxx)
-	KERNEL4x2_3(xxx)
-	KERNEL4x2_4(xxx)
-
-	je	.L2_26
-
-	jmp	.L2_22
-	ALIGN_4
-
-.L2_26:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_29
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_27:
-
-	KERNEL4x2_SUB(xxx)
-	addq	$2, BI
-	addq	$4, %rax
-	jl	.L2_27
-	ALIGN_4
-
-
-.L2_29:
-
-	vbroadcastss	ALPHA, %xmm0
-
-#ifndef TRMMKERNEL
-
-	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddps 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-
-#else
-	vmulps	%xmm0, %xmm4,%xmm4
-	vmulps	%xmm0, %xmm5,%xmm5
-
-#endif
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm5 ,  	(CO1, LDC)
-
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax 
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO         
-        salq    $2, %rax                        // rax = rax * 4 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif  
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $4, KK
-#endif
-
-	addq	$4 * SIZE, CO1		# coffset += 4
-	ALIGN_4
-	
-
-.L2_30:
-	testq	$2, M		
-	jz	.L2_40
-
-	ALIGN_4
-
-.L2_31:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO
-        salq    $1, %rax                        // rax = rax * 2 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $2, %rax        // number of values in AO
-#else
-        addq    $2, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-
-	andq	$-8, %rax
-	je	.L2_36
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_32:
-
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL2x2_1(xxx)
-	KERNEL2x2_2(xxx)
-	KERNEL2x2_3(xxx)
-	KERNEL2x2_4(xxx)
-
-	KERNEL2x2_1(xxx)
-	KERNEL2x2_2(xxx)
-	KERNEL2x2_3(xxx)
-	KERNEL2x2_4(xxx)
-
-	je	.L2_36
-
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL2x2_1(xxx)
-	KERNEL2x2_2(xxx)
-	KERNEL2x2_3(xxx)
-	KERNEL2x2_4(xxx)
-
-	KERNEL2x2_1(xxx)
-	KERNEL2x2_2(xxx)
-	KERNEL2x2_3(xxx)
-	KERNEL2x2_4(xxx)
-
-	je	.L2_36
-
-	jmp	.L2_32
-	ALIGN_4
-
-.L2_36:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_39
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-	
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_37:
-
-	KERNEL2x2_SUB(xxx)
-	addq	$2, BI
-	addq	$2, %rax
-	jl	.L2_37
-	ALIGN_4
-
-
-.L2_39:
-
-	vmovss	ALPHA, %xmm0
-
-#ifndef TRMMKERNEL
-
-	vfmaddss 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddss 1 * SIZE(CO1),%xmm0, %xmm8,%xmm8
-	vfmaddss 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-	vfmaddss 1 * SIZE(CO1, LDC),%xmm0, %xmm10,%xmm10
-
-#else
-	vmulss	%xmm0, %xmm4,%xmm4
-	vmulss	%xmm0, %xmm8,%xmm8
-	vmulss	%xmm0, %xmm5,%xmm5
-	vmulss	%xmm0, %xmm10,%xmm10
-
-#endif
-
-	vmovss	%xmm4 ,  	(CO1)
-	vmovss	%xmm8 , 1 * SIZE(CO1)
-	vmovss	%xmm5 ,  	(CO1, LDC)
-	vmovss	%xmm10, 1 * SIZE(CO1, LDC)
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax 
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO         
-        salq    $1, %rax                        // rax = rax * 2 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif  
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $2, KK
-#endif
-
-	addq	$2 * SIZE, CO1		# coffset += 2
-	ALIGN_4
-
-.L2_40:
-	testq	$1, M		
-	jz	.L2_60		// to next 2 lines of N
-
-	ALIGN_4
-
-.L2_41:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $1, %rax        // number of values in AO
-#else
-        addq    $2, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-	andq	$-8, %rax
-	je	.L2_46
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_42:
-
-	KERNEL1x2_1(xxx)
-	KERNEL1x2_2(xxx)
-	KERNEL1x2_3(xxx)
-	KERNEL1x2_4(xxx)
-
-	KERNEL1x2_1(xxx)
-	KERNEL1x2_2(xxx)
-	KERNEL1x2_3(xxx)
-	KERNEL1x2_4(xxx)
-
-	je	.L2_46
-
-	KERNEL1x2_1(xxx)
-	KERNEL1x2_2(xxx)
-	KERNEL1x2_3(xxx)
-	KERNEL1x2_4(xxx)
-
-	KERNEL1x2_1(xxx)
-	KERNEL1x2_2(xxx)
-	KERNEL1x2_3(xxx)
-	KERNEL1x2_4(xxx)
-
-	je	.L2_46
-
-	jmp	.L2_42
-	ALIGN_4
-
-.L2_46:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_49
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_47:
-
-	KERNEL1x2_SUB(xxx)
-	addq	$2, BI
-	addq	$1, %rax
-	jl	.L2_47
-	ALIGN_4
-
-
-.L2_49:
-
-	vmovss	ALPHA, %xmm0
-
-#ifndef TRMMKERNEL
-
-	vfmaddss 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddss 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-
-#else
-	vmulss	%xmm0, %xmm4,%xmm4
-	vmulss	%xmm0, %xmm5,%xmm5
-
-#endif
-
-	vmovss	%xmm4 ,  	(CO1)
-	vmovss	%xmm5 ,  	(CO1, LDC)
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax 
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO         
-        leaq    (AO, %rax, SIZE), AO
-#endif  
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $1, KK
-#endif
-
-	addq	$1 * SIZE, CO1		# coffset += 1
-	ALIGN_4
-	
-
-
-
-	
-.L2_60:
-#if defined(TRMMKERNEL) && !defined(LEFT)
-        addq    $2, KK
-#endif
-
-	decq	J			// j --
-	jg	.L2_01			// next 2 lines of N
-
-
-
-.L1_0:
-
-/************************************************************************************************
-* Loop for Nmod6 % 2 > 0
-*************************************************************************************************/
-
-	movq	Nmod6, J		
-	andq	$1, J			// j % 2
-	je	.L999
-	ALIGN_4
-
-.L1_01:
-	// copy to sub buffer
-	movq	B, BO1
-	leaq    BUFFER1, BO		// first buffer to BO
-	movq	K, %rax
-	ALIGN_4
-
-.L1_02b:
-
-	vmovss	(BO1), %xmm0
-	vmovss	%xmm0,       (BO)
-	addq	$1*SIZE,BO1
-	addq	$1*SIZE,BO
-	decq	%rax
-	jnz	.L1_02b
-
-.L1_02c:
-
-	movq	BO1, B			// next offset of B
-
-.L1_10:
-	movq	C, CO1
-	leaq	(C, LDC, 1), C		// c += 1 * ldc
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        movq    OFFSET, %rax
-        movq    %rax, KK
-#endif
-	
-	movq	A, AO		 	// aoffset = a
-	addq	$32 * SIZE, AO
-
-	movq	M,  I
-	sarq	$4, I			// i = (m >> 4)
-	je	.L1_20
-
-	ALIGN_4
-
-.L1_11:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BO, BI, SIZE), BO
-        salq    $4, %rax                        // rax = rax * 16 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $16, %rax	// number of values in AO
-#else
-        addq    $1, %rax	// number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-	andq	$-8, %rax			//  K = K - ( K % 8 )
-	je	.L1_16
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$4, %rax			// rax = rax * 16 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_12:
-
-	prefetcht0	B_PR1(BO,BI, SIZE)
-	KERNEL16x1_1(xxx)
-	KERNEL16x1_2(xxx)
-	KERNEL16x1_3(xxx)
-	KERNEL16x1_4(xxx)
-
-	KERNEL16x1_1(xxx)
-	KERNEL16x1_2(xxx)
-	KERNEL16x1_3(xxx)
-	KERNEL16x1_4(xxx)
-
-	je	.L1_16
-
-	KERNEL16x1_1(xxx)
-	KERNEL16x1_2(xxx)
-	KERNEL16x1_3(xxx)
-	KERNEL16x1_4(xxx)
-
-	KERNEL16x1_1(xxx)
-	KERNEL16x1_2(xxx)
-	KERNEL16x1_3(xxx)
-	KERNEL16x1_4(xxx)
-
-	je	.L1_16
-
-	jmp	.L1_12
-	ALIGN_4
-
-.L1_16:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_19
-
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$4, %rax			// rax = rax * 16 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_17:
-
-	KERNEL16x1_SUB(xxx)
-	addq	$1, BI
-	addq	$16, %rax
-	jl	.L1_17
-	ALIGN_4
-
-
-.L1_19:
-
-	vbroadcastss	ALPHA, %xmm0
-
-#ifndef TRMMKERNEL
-
-	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddps 4 * SIZE(CO1),%xmm0, %xmm7,%xmm7
-	vfmaddps 8 * SIZE(CO1),%xmm0, %xmm10,%xmm10
-	vfmaddps 12 * SIZE(CO1),%xmm0, %xmm13,%xmm13
-
-#else
-	vmulps	%xmm0, %xmm4,%xmm4
-	vmulps	%xmm0, %xmm7,%xmm7
-	vmulps	%xmm0, %xmm10,%xmm10
-	vmulps	%xmm0, %xmm13,%xmm13
-
-#endif
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm7 , 4 * SIZE(CO1)
-	vmovups	%xmm10, 8 * SIZE(CO1)
-	vmovups	%xmm13,12 * SIZE(CO1)
-
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BO, BI, SIZE), BO
-        salq    $4, %rax                        // rax = rax * 16 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $16, KK				
-#endif
-
-	addq	$16 * SIZE, CO1		# coffset += 16
-	decq	I			# i --
-	jg	.L1_11
-	ALIGN_4	
-
-/**************************************************************************
-* Rest of M 
-***************************************************************************/
-.L1_20:
-	// Test rest of M
-
-	testq	$15, M
-	jz	.L999
-
-	testq	$8, M		
-	jz	.L1_21pre
-	ALIGN_4
-
-/**************************************************************************/
-
-.L1_20_1:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BO, BI, SIZE), BO
-        salq    $3, %rax                        // rax = rax * 8 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $8, %rax        // number of values in A
-#else
-        addq    $1, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-
-	andq	$-8, %rax
-	je	.L1_20_6
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_20_2:
-
-	prefetcht0	B_PR1(BO,BI, SIZE)
-	KERNEL8x1_1(xxx)
-	KERNEL8x1_2(xxx)
-	KERNEL8x1_3(xxx)
-	KERNEL8x1_4(xxx)
-
-	KERNEL8x1_1(xxx)
-	KERNEL8x1_2(xxx)
-	KERNEL8x1_3(xxx)
-	KERNEL8x1_4(xxx)
-
-	je	.L1_20_6
-
-	KERNEL8x1_1(xxx)
-	KERNEL8x1_2(xxx)
-	KERNEL8x1_3(xxx)
-	KERNEL8x1_4(xxx)
-
-	KERNEL8x1_1(xxx)
-	KERNEL8x1_2(xxx)
-	KERNEL8x1_3(xxx)
-	KERNEL8x1_4(xxx)
-
-	je	.L1_20_6
-
-	jmp	.L1_20_2
-	ALIGN_4
-
-.L1_20_6:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_20_9
-
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_20_7:
-
-	KERNEL8x1_SUB(xxx)
-	addq	$1, BI
-	addq	$8, %rax
-	jl	.L1_20_7
-	ALIGN_4
-
-
-.L1_20_9:
-
-	vbroadcastss	ALPHA, %xmm0
-
-#ifndef TRMMKERNEL
-
-	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddps 4 * SIZE(CO1),%xmm0, %xmm7,%xmm7
-
-#else
-	vmulps	%xmm0, %xmm4,%xmm4
-	vmulps	%xmm0, %xmm7,%xmm7
-
-#endif
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm7 , 4 * SIZE(CO1)
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax 
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BO, BI, SIZE), BO         
-        salq    $3, %rax                        // rax = rax * 8 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif  
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $8, KK
-#endif
-
-	addq	$8 * SIZE, CO1		# coffset += 8
-	ALIGN_4
-	
-
-
-/**************************************************************************/
-
-.L1_21pre:
-
-	testq	$4, M		
-	jz	.L1_30
-	ALIGN_4
-
-.L1_21:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BO, BI, SIZE), BO
-        salq    $2, %rax                        // rax = rax * 4 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $4, %rax        // number of values in A
-#else
-        addq    $1, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-
-	andq	$-8, %rax
-	je	.L1_26
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_22:
-
-	prefetcht0	B_PR1(BO,BI, SIZE)
-	KERNEL4x1_1(xxx)
-	KERNEL4x1_2(xxx)
-	KERNEL4x1_3(xxx)
-	KERNEL4x1_4(xxx)
-
-	KERNEL4x1_1(xxx)
-	KERNEL4x1_2(xxx)
-	KERNEL4x1_3(xxx)
-	KERNEL4x1_4(xxx)
-
-	je	.L1_26
-
-	KERNEL4x1_1(xxx)
-	KERNEL4x1_2(xxx)
-	KERNEL4x1_3(xxx)
-	KERNEL4x1_4(xxx)
-
-	KERNEL4x1_1(xxx)
-	KERNEL4x1_2(xxx)
-	KERNEL4x1_3(xxx)
-	KERNEL4x1_4(xxx)
-
-	je	.L1_26
-
-	jmp	.L1_22
-	ALIGN_4
-
-.L1_26:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_29
-
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_27:
-
-	KERNEL4x1_SUB(xxx)
-	addq	$1, BI
-	addq	$4, %rax
-	jl	.L1_27
-	ALIGN_4
-
-
-.L1_29:
-
-	vbroadcastss	ALPHA, %xmm0
-
-#ifndef TRMMKERNEL
-
-	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
-
-#else
-	vmulps	%xmm0, %xmm4,%xmm4
-
-#endif
-
-	vmovups	%xmm4 ,  	(CO1)
-
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax 
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BO, BI, SIZE), BO         
-        salq    $2, %rax                        // rax = rax * 4 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif  
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $4, KK
-#endif
-
-	addq	$4 * SIZE, CO1		# coffset += 4
-	ALIGN_4
-	
-
-.L1_30:
-	testq	$2, M		
-	jz	.L1_40
-
-	ALIGN_4
-
-.L1_31:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BO, BI, SIZE), BO
-        salq    $1, %rax                        // rax = rax * 2 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $2, %rax        // number of values in AO
-#else
-        addq    $1, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-
-	andq	$-8, %rax
-	je	.L1_36
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_32:
-
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL2x1_1(xxx)
-	KERNEL2x1_2(xxx)
-	KERNEL2x1_3(xxx)
-	KERNEL2x1_4(xxx)
-
-	KERNEL2x1_1(xxx)
-	KERNEL2x1_2(xxx)
-	KERNEL2x1_3(xxx)
-	KERNEL2x1_4(xxx)
-
-	je	.L1_36
-
-	KERNEL2x1_1(xxx)
-	KERNEL2x1_2(xxx)
-	KERNEL2x1_3(xxx)
-	KERNEL2x1_4(xxx)
-
-	KERNEL2x1_1(xxx)
-	KERNEL2x1_2(xxx)
-	KERNEL2x1_3(xxx)
-	KERNEL2x1_4(xxx)
-
-	je	.L1_36
-
-	jmp	.L1_32
-	ALIGN_4
-
-.L1_36:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_39
-
-	movq    %rax, BI                        //  Index for BO
-	
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_37:
-
-	KERNEL2x1_SUB(xxx)
-	addq	$1, BI
-	addq	$2, %rax
-	jl	.L1_37
-	ALIGN_4
-
-
-.L1_39:
-
-	vmovss	ALPHA, %xmm0
-
-#ifndef TRMMKERNEL
-
-	vfmaddss 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddss 1 * SIZE(CO1),%xmm0, %xmm8,%xmm8
-
-#else
-	vmulss	%xmm0, %xmm4,%xmm4
-	vmulss	%xmm0, %xmm8,%xmm8
-
-#endif
-
-	vmovss	%xmm4 ,  	(CO1)
-	vmovss	%xmm8 , 1 * SIZE(CO1)
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax 
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BO, BI, SIZE), BO         
-        salq    $1, %rax                        // rax = rax * 2 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif  
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $2, KK
-#endif
-
-	addq	$2 * SIZE, CO1		# coffset += 2
-	ALIGN_4
-
-.L1_40:
-	testq	$1, M		
-	jz	.L999
-
-	ALIGN_4
-
-.L1_41:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BO, BI, SIZE), BO
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $1, %rax        // number of values in AO
-#else
-        addq    $1, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-	andq	$-8, %rax
-	je	.L1_46
-	movq    %rax, BI                        //  Index for BO
-
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_42:
-
-	KERNEL1x1_1(xxx)
-	KERNEL1x1_2(xxx)
-	KERNEL1x1_3(xxx)
-	KERNEL1x1_4(xxx)
-
-	KERNEL1x1_1(xxx)
-	KERNEL1x1_2(xxx)
-	KERNEL1x1_3(xxx)
-	KERNEL1x1_4(xxx)
-
-	je	.L1_46
-
-	KERNEL1x1_1(xxx)
-	KERNEL1x1_2(xxx)
-	KERNEL1x1_3(xxx)
-	KERNEL1x1_4(xxx)
-
-	KERNEL1x1_1(xxx)
-	KERNEL1x1_2(xxx)
-	KERNEL1x1_3(xxx)
-	KERNEL1x1_4(xxx)
-
-	je	.L1_46
-
-	jmp	.L1_42
-	ALIGN_4
-
-.L1_46:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_49
-
-	movq    %rax, BI                        //  Index for BO
-
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_47:
-
-	KERNEL1x1_SUB(xxx)
-	addq	$1, BI
-	addq	$1, %rax
-	jl	.L1_47
-	ALIGN_4
-
-
-.L1_49:
-
-	vmovss	ALPHA, %xmm0
-
-#ifndef TRMMKERNEL
-
-	vfmaddss 	 (CO1),%xmm0, %xmm4,%xmm4
-
-#else
-	vmulss	%xmm0, %xmm4,%xmm4
-
-#endif
-
-	vmovss	%xmm4 ,  	(CO1)
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax 
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BO, BI, SIZE), BO         
-        leaq    (AO, %rax, SIZE), AO
-#endif  
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $1, KK
-#endif
-
-	addq	$1 * SIZE, CO1		# coffset += 1
-	ALIGN_4
-	
-
-.L999:
-	movq   		SP, %rsp
-	movq	   (%rsp), %rbx
-	movq	  8(%rsp), %rbp
-	movq	 16(%rsp), %r12
-	movq	 24(%rsp), %r13
-	movq	 32(%rsp), %r14
-	movq	 40(%rsp), %r15
-
-#ifdef WINDOWS_ABI
-	movq	 48(%rsp), %rdi
-	movq	 56(%rsp), %rsi
-	movups	 64(%rsp), %xmm6
-	movups	 80(%rsp), %xmm7
-	movups	 96(%rsp), %xmm8
-	movups	112(%rsp), %xmm9
-	movups	128(%rsp), %xmm10
-	movups	144(%rsp), %xmm11
-	movups	160(%rsp), %xmm12
-	movups	176(%rsp), %xmm13
-	movups	192(%rsp), %xmm14
-	movups	208(%rsp), %xmm15
-#endif
-
-	addq	$STACKSIZE, %rsp
-	ret
-
-	EPILOGUE
-
-
-
-
-
-#endif
+/*********************************************************************/
+/* Copyright 2009, 2010 The University of Texas at Austin.           */
+/* All rights reserved.                                              */
+/*                                                                   */
+/* Redistribution and use in source and binary forms, with or        */
+/* without modification, are permitted provided that the following   */
+/* conditions are met:                                               */
+/*                                                                   */
+/*   1. Redistributions of source code must retain the above         */
+/*      copyright notice, this list of conditions and the following  */
+/*      disclaimer.                                                  */
+/*                                                                   */
+/*   2. Redistributions in binary form must reproduce the above      */
+/*      copyright notice, this list of conditions and the following  */
+/*      disclaimer in the documentation and/or other materials       */
+/*      provided with the distribution.                              */
+/*                                                                   */
+/*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */
+/*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */
+/*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */
+/*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */
+/*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */
+/*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */
+/*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */
+/*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */
+/*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */
+/*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */
+/*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */
+/*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */
+/*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */
+/*    POSSIBILITY OF SUCH DAMAGE.                                    */
+/*                                                                   */
+/* The views and conclusions contained in the software and           */
+/* documentation are those of the authors and should not be          */
+/* interpreted as representing official policies, either expressed   */
+/* or implied, of The University of Texas at Austin.                 */
+/*********************************************************************/
+
+
+#define ASSEMBLER
+#include "common.h"
+ 
+#define OLD_M	%rdi
+#define OLD_N	%rsi
+#define M	%r13
+#define J	%r14
+#define OLD_K	%rdx
+
+#define A	%rcx
+#define B	%r8
+#define C	%r9
+#define LDC	%r10
+	
+#define I	%r11
+#define AO	%rdi
+#define BO	%rsi
+#define	CO1	%r15
+#define K	%r12
+#define BI	%rbp
+#define	SP	%rbx
+
+#define BO1	%rdi
+#define BO2	%r15
+
+#ifndef WINDOWS_ABI
+
+#define STACKSIZE 96
+
+#else
+
+#define STACKSIZE 256
+
+#define OLD_A		40 + STACKSIZE(%rsp)
+#define OLD_B		48 + STACKSIZE(%rsp)
+#define OLD_C		56 + STACKSIZE(%rsp)
+#define OLD_LDC		64 + STACKSIZE(%rsp)
+#define OLD_OFFSET	72 + STACKSIZE(%rsp)
+
+#endif
+
+#define L_BUFFER_SIZE 8192
+#define LB2_OFFSET    4096
+
+#define Ndiv6	 24(%rsp)
+#define Nmod6	 32(%rsp)
+#define N	 40(%rsp)
+#define ALPHA	 48(%rsp)
+#define OFFSET	 56(%rsp)
+#define KK	 64(%rsp)
+#define KKK	 72(%rsp)
+#define BUFFER1	           128(%rsp)
+#define BUFFER2	LB2_OFFSET+128(%rsp)
+
+#if defined(OS_WINDOWS)
+#if   L_BUFFER_SIZE > 16384
+#define STACK_TOUCH \
+        movl    $0,  4096 * 4(%rsp);\
+        movl    $0,  4096 * 3(%rsp);\
+        movl    $0,  4096 * 2(%rsp);\
+        movl    $0,  4096 * 1(%rsp);
+#elif L_BUFFER_SIZE > 12288
+#define STACK_TOUCH \
+        movl    $0,  4096 * 3(%rsp);\
+        movl    $0,  4096 * 2(%rsp);\
+        movl    $0,  4096 * 1(%rsp);
+#elif L_BUFFER_SIZE > 8192
+#define STACK_TOUCH \
+        movl    $0,  4096 * 2(%rsp);\
+        movl    $0,  4096 * 1(%rsp);
+#elif L_BUFFER_SIZE > 4096
+#define STACK_TOUCH \
+        movl    $0,  4096 * 1(%rsp);
+#else
+#define STACK_TOUCH
+#endif
+#else
+#define STACK_TOUCH
+#endif
+
+
+
+#define	A_PR1	384
+#define	B_PR1	192
+
+/*******************************************************************************************
+* 3 lines of N
+*******************************************************************************************/
+
+#define KERNEL16x3_1(xx) \
+	vbroadcastss	 -6 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vbroadcastss	 -5 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm3 ;\
+	nop						;\
+	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	prefetcht0	A_PR1(AO,%rax,SIZE)	;\
+	vfmaddps  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
+	vmovups 	-24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
+	vfmaddps  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
+	vfmaddps  	%xmm12,%xmm3,%xmm0,%xmm12 ;\
+	vmovups 	-20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
+	vfmaddps  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
+	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm15,%xmm3,%xmm0,%xmm15 ;\
+
+#define KERNEL16x3_2(xx) \
+	vmovups 	-16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vbroadcastss	 -1 * SIZE(BO, BI, SIZE), %xmm3 ;\
+	nop						;\
+	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+	vmovups 	-12 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	prefetcht0	A_PR1+64(AO,%rax,SIZE)	;\
+	vfmaddps  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
+	vmovups 	 -8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
+	vfmaddps  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
+	vfmaddps  	%xmm12,%xmm3,%xmm0,%xmm12 ;\
+	vmovups 	 -4 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
+	vfmaddps  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
+	vbroadcastss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vbroadcastss	  1 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm15,%xmm3,%xmm0,%xmm15 ;\
+
+#define KERNEL16x3_3(xx) \
+	vmovups 	  0 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vbroadcastss	  2 * SIZE(BO, BI, SIZE), %xmm3 ;\
+	nop						;\
+	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+	vmovups 	  4 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	prefetcht0	A_PR1+128(AO,%rax,SIZE)	;\
+	vfmaddps  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
+	vmovups 	  8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
+	vfmaddps  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
+	vfmaddps  	%xmm12,%xmm3,%xmm0,%xmm12 ;\
+	vmovups 	 12 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
+	vfmaddps  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
+	vbroadcastss	  3 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vbroadcastss	  4 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm15,%xmm3,%xmm0,%xmm15 ;\
+
+#define KERNEL16x3_4(xx) \
+	vmovups 	 16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vbroadcastss	  5 * SIZE(BO, BI, SIZE), %xmm3 ;\
+	nop						;\
+	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+	vmovups 	 20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	prefetcht0	A_PR1+192(AO,%rax,SIZE)	;\
+	vfmaddps  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
+	vmovups 	 24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
+	vfmaddps  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
+	addq	$12, BI				  ;\
+	vfmaddps  	%xmm12,%xmm3,%xmm0,%xmm12 ;\
+	vmovups 	 28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
+	vfmaddps  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
+	addq	$64, %rax 			  ;\
+	vfmaddps  	%xmm15,%xmm3,%xmm0,%xmm15 ;\
+
+#define KERNEL16x3_SUB(xx) \
+	vbroadcastss	 -6 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vbroadcastss	 -5 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm3 ;\
+	nop						;\
+	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	vfmaddps  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
+	vmovups 	-24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
+	vfmaddps  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
+	vfmaddps  	%xmm12,%xmm3,%xmm0,%xmm12 ;\
+	vmovups 	-20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
+	vfmaddps  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
+	vfmaddps  	%xmm15,%xmm3,%xmm0,%xmm15 ;\
+
+
+/*******************************************************************************************/
+
+#define KERNEL8x3_1(xx) \
+	prefetcht0	A_PR1(AO,%rax,SIZE)	;\
+	vbroadcastss	 -6 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vbroadcastss	 -5 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm3 ;\
+	nop						;\
+	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	vfmaddps  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
+
+#define KERNEL8x3_2(xx) \
+	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vbroadcastss	 -1 * SIZE(BO, BI, SIZE), %xmm3 ;\
+	nop						;\
+	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+	vmovups 	-20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	vfmaddps  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
+
+#define KERNEL8x3_3(xx) \
+	prefetcht0	A_PR1+64(AO,%rax,SIZE)	;\
+	vbroadcastss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vbroadcastss	  1 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vbroadcastss	  2 * SIZE(BO, BI, SIZE), %xmm3 ;\
+	nop						;\
+	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+	vmovups 	-12 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	vfmaddps  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
+
+#define KERNEL8x3_4(xx) \
+	vbroadcastss	  3 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	 -8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vbroadcastss	  4 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vbroadcastss	  5 * SIZE(BO, BI, SIZE), %xmm3 ;\
+	nop						;\
+	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+	vmovups 	 -4 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	vfmaddps  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
+	addq	$12, BI				  ;\
+	addq	$32, %rax 			  ;\
+
+#define KERNEL8x3_SUB(xx) \
+	vbroadcastss	 -6 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vbroadcastss	 -5 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm3 ;\
+	nop						;\
+	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	vfmaddps  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
+
+
+/*******************************************************************************************/
+
+#define KERNEL4x3_1(xx) \
+	prefetcht0	A_PR1(AO,%rax,SIZE)	;\
+	vbroadcastss	 -6 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vbroadcastss	 -5 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm3 ;\
+	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+
+#define KERNEL4x3_2(xx) \
+	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vbroadcastss	 -1 * SIZE(BO, BI, SIZE), %xmm3 ;\
+	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+
+#define KERNEL4x3_3(xx) \
+	vbroadcastss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vbroadcastss	  1 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vbroadcastss	  2 * SIZE(BO, BI, SIZE), %xmm3 ;\
+	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+
+#define KERNEL4x3_4(xx) \
+	vbroadcastss	  3 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vbroadcastss	  4 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vbroadcastss	  5 * SIZE(BO, BI, SIZE), %xmm3 ;\
+	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+	addq	$12, BI				  ;\
+	addq	$16, %rax 			  ;\
+
+#define KERNEL4x3_SUB(xx) \
+	vbroadcastss	 -6 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vbroadcastss	 -5 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm3 ;\
+	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+
+/*******************************************************************************************/
+
+#define KERNEL2x3_1(xx) \
+	vmovss	 -6 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovss	 -5 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovss	 -4 * SIZE(BO, BI, SIZE), %xmm3 ;\
+	vfmaddss  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+	vmovss 	-31 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
+	vfmaddss  	%xmm10,%xmm2,%xmm0,%xmm10 ;\
+	vfmaddss  	%xmm12,%xmm3,%xmm0,%xmm12 ;\
+
+#define KERNEL2x3_2(xx) \
+	vmovss	 -3 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-30 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovss	 -2 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovss	 -1 * SIZE(BO, BI, SIZE), %xmm3 ;\
+	vfmaddss  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+	vmovss 	-29 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
+	vfmaddss  	%xmm10,%xmm2,%xmm0,%xmm10 ;\
+	vfmaddss  	%xmm12,%xmm3,%xmm0,%xmm12 ;\
+
+#define KERNEL2x3_3(xx) \
+	vmovss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovss	  1 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovss	  2 * SIZE(BO, BI, SIZE), %xmm3 ;\
+	vfmaddss  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+	vmovss 	-27 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
+	vfmaddss  	%xmm10,%xmm2,%xmm0,%xmm10 ;\
+	vfmaddss  	%xmm12,%xmm3,%xmm0,%xmm12 ;\
+
+#define KERNEL2x3_4(xx) \
+	vmovss	  3 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-26 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovss	  4 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovss	  5 * SIZE(BO, BI, SIZE), %xmm3 ;\
+	vfmaddss  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+	vmovss 	-25 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
+	vfmaddss  	%xmm10,%xmm2,%xmm0,%xmm10 ;\
+	vfmaddss  	%xmm12,%xmm3,%xmm0,%xmm12 ;\
+	addq	$12, BI				  ;\
+	addq	$8, %rax 			  ;\
+
+#define KERNEL2x3_SUB(xx) \
+	vmovss	 -6 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovss	 -5 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovss	 -4 * SIZE(BO, BI, SIZE), %xmm3 ;\
+	vfmaddss  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+	vmovss 	-31 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
+	vfmaddss  	%xmm10,%xmm2,%xmm0,%xmm10 ;\
+	vfmaddss  	%xmm12,%xmm3,%xmm0,%xmm12 ;\
+
+/*******************************************************************************************/
+
+#define KERNEL1x3_1(xx) \
+	vmovss	 -6 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovss	 -5 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovss	 -4 * SIZE(BO, BI, SIZE), %xmm3 ;\
+	vfmaddss  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+
+#define KERNEL1x3_2(xx) \
+	vmovss	 -3 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-31 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovss	 -2 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovss	 -1 * SIZE(BO, BI, SIZE), %xmm3 ;\
+	vfmaddss  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+
+#define KERNEL1x3_3(xx) \
+	vmovss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-30 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovss	  1 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovss	  2 * SIZE(BO, BI, SIZE), %xmm3 ;\
+	vfmaddss  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+
+#define KERNEL1x3_4(xx) \
+	vmovss	  3 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-29 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovss	  4 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovss	  5 * SIZE(BO, BI, SIZE), %xmm3 ;\
+	vfmaddss  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+	addq	$12, BI				  ;\
+	addq	$4, %rax 			  ;\
+
+#define KERNEL1x3_SUB(xx) \
+	vmovss	 -6 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovss	 -5 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovss	 -4 * SIZE(BO, BI, SIZE), %xmm3 ;\
+	vfmaddss  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+
+/*******************************************************************************************/
+
+/*******************************************************************************************
+* 2 lines of N
+*******************************************************************************************/
+
+#define KERNEL16x2_1(xx) \
+	prefetcht0	A_PR1(AO,%rax,SIZE)	;\
+	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	vmovups 	-24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
+	vfmaddps  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
+	vmovups 	-20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
+	vfmaddps  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
+
+#define KERNEL16x2_2(xx) \
+	prefetcht0	A_PR1+64(AO,%rax,SIZE)	;\
+	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vbroadcastss	 -1 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovups 	-12 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	vmovups 	 -8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
+	vfmaddps  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
+	vmovups 	 -4 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
+	vfmaddps  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
+
+#define KERNEL16x2_3(xx) \
+	prefetcht0	A_PR1+128(AO,%rax,SIZE)	;\
+	vbroadcastss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	  0 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vbroadcastss	  1 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovups 	  4 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	vmovups 	  8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
+	vfmaddps  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
+	vmovups 	 12 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
+	vfmaddps  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
+
+#define KERNEL16x2_4(xx) \
+	prefetcht0	A_PR1+192(AO,%rax,SIZE)	;\
+	vbroadcastss	  2 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	 16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vbroadcastss	  3 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovups 	 20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	vmovups 	 24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
+	vfmaddps  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
+	vmovups 	 28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
+	vfmaddps  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
+	addq	$8, BI				  ;\
+	addq	$64, %rax 			  ;\
+
+#define KERNEL16x2_SUB(xx) \
+	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	vmovups 	-24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
+	vfmaddps  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
+	vmovups 	-20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
+	vfmaddps  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
+
+
+/*******************************************************************************************/
+
+#define KERNEL8x2_1(xx) \
+	prefetcht0	A_PR1(AO,%rax,SIZE)	;\
+	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+
+#define KERNEL8x2_2(xx) \
+	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vbroadcastss	 -1 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovups 	-20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+
+#define KERNEL8x2_3(xx) \
+	prefetcht0	A_PR1+64(AO,%rax,SIZE)	;\
+	vbroadcastss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vbroadcastss	  1 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovups 	-12 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+
+#define KERNEL8x2_4(xx) \
+	vbroadcastss	  2 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	 -8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vbroadcastss	  3 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovups 	 -4 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	addq	$8, BI				  ;\
+	addq	$32, %rax 			  ;\
+
+#define KERNEL8x2_SUB(xx) \
+	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+
+
+/*******************************************************************************************/
+
+#define KERNEL4x2_1(xx) \
+	prefetcht0	A_PR1(AO,%rax,SIZE)	;\
+	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+
+#define KERNEL4x2_2(xx) \
+	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vbroadcastss	 -1 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+
+#define KERNEL4x2_3(xx) \
+	vbroadcastss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vbroadcastss	  1 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+
+#define KERNEL4x2_4(xx) \
+	vbroadcastss	  2 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vbroadcastss	  3 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	addq	$8, BI				  ;\
+	addq	$16, %rax 			  ;\
+
+#define KERNEL4x2_SUB(xx) \
+	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+
+/*******************************************************************************************/
+
+#define KERNEL2x2_1(xx) \
+	vmovss	 -4 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovss	 -3 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovss 	-31 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
+	vfmaddss  	%xmm10,%xmm2,%xmm0,%xmm10 ;\
+
+#define KERNEL2x2_2(xx) \
+	vmovss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-30 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovss	 -1 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovss 	-29 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
+	vfmaddss  	%xmm10,%xmm2,%xmm0,%xmm10 ;\
+
+#define KERNEL2x2_3(xx) \
+	vmovss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovss	  1 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovss 	-27 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
+	vfmaddss  	%xmm10,%xmm2,%xmm0,%xmm10 ;\
+
+#define KERNEL2x2_4(xx) \
+	vmovss	  2 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-26 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovss	  3 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovss 	-25 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
+	vfmaddss  	%xmm10,%xmm2,%xmm0,%xmm10 ;\
+	addq	$8, BI				  ;\
+	addq	$8, %rax 			  ;\
+
+#define KERNEL2x2_SUB(xx) \
+	vmovss	 -4 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovss	 -3 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovss 	-31 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
+	vfmaddss  	%xmm10,%xmm2,%xmm0,%xmm10 ;\
+
+/*******************************************************************************************/
+
+#define KERNEL1x2_1(xx) \
+	vmovss	 -4 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovss	 -3 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+
+#define KERNEL1x2_2(xx) \
+	vmovss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-31 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovss	 -1 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+
+#define KERNEL1x2_3(xx) \
+	vmovss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-30 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovss	  1 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+
+#define KERNEL1x2_4(xx) \
+	vmovss	  2 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-29 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovss	  3 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	addq	$8, BI				  ;\
+	addq	$4, %rax 			  ;\
+
+#define KERNEL1x2_SUB(xx) \
+	vmovss	 -4 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovss	 -3 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+
+/*******************************************************************************************/
+
+/*******************************************************************************************
+* 1 line of N
+*******************************************************************************************/
+
+#define KERNEL16x1_1(xx) \
+	prefetcht0	A_PR1(AO,%rax,SIZE)	;\
+	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vmovups 	-24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
+	vmovups 	-20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
+
+#define KERNEL16x1_2(xx) \
+	prefetcht0	A_PR1+64(AO,%rax,SIZE)	;\
+	vbroadcastss	 -1 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovups 	-12 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vmovups 	 -8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
+	vmovups 	 -4 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
+
+#define KERNEL16x1_3(xx) \
+	prefetcht0	A_PR1+128(AO,%rax,SIZE)	;\
+	vbroadcastss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	  0 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovups 	  4 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vmovups 	  8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
+	vmovups 	 12 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
+
+#define KERNEL16x1_4(xx) \
+	prefetcht0	A_PR1+192(AO,%rax,SIZE)	;\
+	vbroadcastss	  1 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	 16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovups 	 20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vmovups 	 24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
+	vmovups 	 28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
+	addq	$4, BI				  ;\
+	addq	$64, %rax 			  ;\
+
+#define KERNEL16x1_SUB(xx) \
+	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vmovups 	-24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
+	vmovups 	-20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
+
+
+/*******************************************************************************************/
+
+#define KERNEL8x1_1(xx) \
+	prefetcht0	A_PR1(AO,%rax,SIZE)	;\
+	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+
+#define KERNEL8x1_2(xx) \
+	vbroadcastss	 -1 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovups 	-20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+
+#define KERNEL8x1_3(xx) \
+	prefetcht0	A_PR1+64(AO,%rax,SIZE)	;\
+	vbroadcastss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovups 	-12 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+
+#define KERNEL8x1_4(xx) \
+	vbroadcastss	  1 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	 -8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovups 	 -4 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	addq	$4, BI				  ;\
+	addq	$32, %rax 			  ;\
+
+#define KERNEL8x1_SUB(xx) \
+	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+
+
+/*******************************************************************************************/
+
+#define KERNEL4x1_1(xx) \
+	prefetcht0	A_PR1(AO,%rax,SIZE)	;\
+	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+
+#define KERNEL4x1_2(xx) \
+	vbroadcastss	 -1 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+
+#define KERNEL4x1_3(xx) \
+	vbroadcastss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+
+#define KERNEL4x1_4(xx) \
+	vbroadcastss	  1 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	addq	$4, BI				  ;\
+	addq	$16, %rax 			  ;\
+
+#define KERNEL4x1_SUB(xx) \
+	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+
+/*******************************************************************************************/
+
+#define KERNEL2x1_1(xx) \
+	vmovss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovss 	-31 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
+
+#define KERNEL2x1_2(xx) \
+	vmovss	 -1 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-30 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovss 	-29 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
+
+#define KERNEL2x1_3(xx) \
+	vmovss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovss 	-27 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
+
+#define KERNEL2x1_4(xx) \
+	vmovss	  1 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-26 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovss 	-25 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
+	addq	$4, BI				  ;\
+	addq	$8, %rax 			  ;\
+
+#define KERNEL2x1_SUB(xx) \
+	vmovss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovss 	-31 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
+
+/*******************************************************************************************/
+
+#define KERNEL1x1_1(xx) \
+	vmovss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+
+#define KERNEL1x1_2(xx) \
+	vmovss	 -1 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-31 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+
+#define KERNEL1x1_3(xx) \
+	vmovss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-30 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+
+#define KERNEL1x1_4(xx) \
+	vmovss	  1 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-29 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	addq	$4, BI				  ;\
+	addq	$4, %rax 			  ;\
+
+#define KERNEL1x1_SUB(xx) \
+	vmovss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+
+/*******************************************************************************************/
+
+#if !defined(TRMMKERNEL)
+
+
+	PROLOGUE
+	PROFCODE
+	
+	subq	$STACKSIZE, %rsp
+	movq	%rbx,   (%rsp)
+	movq	%rbp,  8(%rsp)
+	movq	%r12, 16(%rsp)
+	movq	%r13, 24(%rsp)
+	movq	%r14, 32(%rsp)
+	movq	%r15, 40(%rsp)
+
+	vzeroupper
+
+#ifdef WINDOWS_ABI
+	movq	%rdi,    48(%rsp)
+	movq	%rsi,    56(%rsp)
+	movups	%xmm6,   64(%rsp)
+	movups	%xmm7,   80(%rsp)
+	movups	%xmm8,   96(%rsp)
+	movups	%xmm9,  112(%rsp)
+	movups	%xmm10, 128(%rsp)
+	movups	%xmm11, 144(%rsp)
+	movups	%xmm12, 160(%rsp)
+	movups	%xmm13, 176(%rsp)
+	movups	%xmm14, 192(%rsp)
+	movups	%xmm15, 208(%rsp)
+
+	movq	ARG1,      OLD_M
+	movq	ARG2,      OLD_N
+	movq	ARG3,      OLD_K
+	movq	OLD_A,     A
+	movq	OLD_B,     B
+	movq	OLD_C,     C
+	movq	OLD_LDC,   LDC
+
+	vmovaps	%xmm3, %xmm0
+
+#else
+	movq	STACKSIZE +  8(%rsp), LDC
+
+#endif
+
+	movq    %rsp, SP      # save old stack
+        subq    $128 + L_BUFFER_SIZE, %rsp
+        andq    $-4096, %rsp    # align stack
+
+        STACK_TOUCH
+
+	cmpq	$0, OLD_M
+	je	.L999
+
+	cmpq	$0, OLD_N
+	je	.L999
+
+	cmpq	$0, OLD_K
+	je	.L999
+
+	movq	OLD_M, M
+	movq	OLD_N, N
+	movq	OLD_K, K
+
+	vmovsd	 %xmm0, ALPHA
+
+	salq	$BASE_SHIFT, LDC
+
+	movq    N, %rax
+        xorq    %rdx, %rdx
+        movq    $6,  %rdi
+        divq    %rdi                    //    N / 6
+        movq    %rax, Ndiv6             //    N / 6
+        movq    %rdx, Nmod6             //    N % 6
+
+
+	movq	Ndiv6,  J
+	cmpq	$0, J
+	je	.L2_0
+	ALIGN_4
+
+.L6_01:
+        // copy to sub buffer
+        movq    K, %rax
+        salq    $1,%rax                 // K * 2 ; read 2 values
+        movq    B, BO1
+        leaq    (B,%rax, SIZE), BO2     // next offset to BO2
+        leaq    BUFFER1, BO             // first buffer to BO
+        movq    K, %rax
+	sarq	$3 , %rax		// K / 8
+	jz	.L6_01a_2
+        ALIGN_4
+
+.L6_01a_1:
+
+        prefetcht0 512(BO1)
+        prefetcht0 512(BO2)
+        prefetchw  512(BO)
+
+	vmovsd	0 * SIZE(BO1), %xmm0
+	vmovsd	2 * SIZE(BO1), %xmm2
+	vmovsd	4 * SIZE(BO1), %xmm4
+	vmovsd	6 * SIZE(BO1), %xmm6
+	vmovss  0 * SIZE(BO2), %xmm1
+	vmovss  2 * SIZE(BO2), %xmm3
+	vmovss  4 * SIZE(BO2), %xmm5
+	vmovss  6 * SIZE(BO2), %xmm7
+	vmovsd	%xmm0, 0*SIZE(BO)
+	vmovss	%xmm1, 2*SIZE(BO)
+	vmovsd	%xmm2, 3*SIZE(BO)
+	vmovss	%xmm3, 5*SIZE(BO)
+	vmovsd	%xmm4, 6*SIZE(BO)
+	vmovss	%xmm5, 8*SIZE(BO)
+	vmovsd	%xmm6, 9*SIZE(BO)
+	vmovss	%xmm7,11*SIZE(BO)
+	addq	$8*SIZE,BO1
+	addq	$8*SIZE,BO2
+	addq	$12*SIZE,BO
+
+	vmovsd	0 * SIZE(BO1), %xmm0
+	vmovsd	2 * SIZE(BO1), %xmm2
+	vmovsd	4 * SIZE(BO1), %xmm4
+	vmovsd	6 * SIZE(BO1), %xmm6
+	vmovss  0 * SIZE(BO2), %xmm1
+	vmovss  2 * SIZE(BO2), %xmm3
+	vmovss  4 * SIZE(BO2), %xmm5
+	vmovss  6 * SIZE(BO2), %xmm7
+	vmovsd	%xmm0, 0*SIZE(BO)
+	vmovss	%xmm1, 2*SIZE(BO)
+	vmovsd	%xmm2, 3*SIZE(BO)
+	vmovss	%xmm3, 5*SIZE(BO)
+	vmovsd	%xmm4, 6*SIZE(BO)
+	vmovss	%xmm5, 8*SIZE(BO)
+	vmovsd	%xmm6, 9*SIZE(BO)
+	vmovss	%xmm7,11*SIZE(BO)
+	addq	$8*SIZE,BO1
+	addq	$8*SIZE,BO2
+	addq	$12*SIZE,BO
+
+	decq	%rax
+	jnz	.L6_01a_1
+
+
+
+.L6_01a_2:
+
+	movq    K, %rax
+        andq    $7, %rax                // K % 8
+        jz      .L6_02c
+        ALIGN_4
+
+
+.L6_02b:
+
+	vmovsd	0 * SIZE(BO1), %xmm0
+	vmovss  0 * SIZE(BO2), %xmm2
+	vmovsd	%xmm0, 0*SIZE(BO)
+	vmovss	%xmm2, 2*SIZE(BO)
+	addq	$2*SIZE,BO1
+	addq	$2*SIZE,BO2
+	addq	$3*SIZE,BO
+	decq	%rax
+	jnz	.L6_02b
+
+.L6_02c:
+
+	movq	K, %rax
+	salq	$1,%rax			// K * 2
+	leaq	(B,%rax, SIZE), BO1	// next offset to BO1
+	leaq	(BO1,%rax, SIZE), BO2	// next offset to BO2
+	leaq    BUFFER2, BO		// second buffer to BO
+	movq	K, %rax
+	sarq	$3 , %rax		// K / 8
+	jz	.L6_02c_2
+	ALIGN_4
+
+.L6_02c_1:
+
+	prefetcht0 512(BO2)
+        prefetchw  512(BO)
+
+	vmovsd	0 * SIZE(BO2), %xmm0
+	vmovsd	2 * SIZE(BO2), %xmm2
+	vmovsd	4 * SIZE(BO2), %xmm4
+	vmovsd	6 * SIZE(BO2), %xmm6
+	vmovss  1 * SIZE(BO1), %xmm1
+	vmovss  3 * SIZE(BO1), %xmm3
+	vmovss  5 * SIZE(BO1), %xmm5
+	vmovss  7 * SIZE(BO1), %xmm7
+	vmovss	%xmm1, 0*SIZE(BO)
+	vmovsd	%xmm0, 1*SIZE(BO)
+	vmovss	%xmm3, 3*SIZE(BO)
+	vmovsd	%xmm2, 4*SIZE(BO)
+	vmovss	%xmm5, 6*SIZE(BO)
+	vmovsd	%xmm4, 7*SIZE(BO)
+	vmovss	%xmm7, 9*SIZE(BO)
+	vmovsd	%xmm6,10*SIZE(BO)
+	addq	$8*SIZE,BO1
+	addq	$8*SIZE,BO2
+	addq	$12*SIZE,BO
+
+
+	vmovsd	0 * SIZE(BO2), %xmm0
+	vmovsd	2 * SIZE(BO2), %xmm2
+	vmovsd	4 * SIZE(BO2), %xmm4
+	vmovsd	6 * SIZE(BO2), %xmm6
+	vmovss  1 * SIZE(BO1), %xmm1
+	vmovss  3 * SIZE(BO1), %xmm3
+	vmovss  5 * SIZE(BO1), %xmm5
+	vmovss  7 * SIZE(BO1), %xmm7
+	vmovss	%xmm1, 0*SIZE(BO)
+	vmovsd	%xmm0, 1*SIZE(BO)
+	vmovss	%xmm3, 3*SIZE(BO)
+	vmovsd	%xmm2, 4*SIZE(BO)
+	vmovss	%xmm5, 6*SIZE(BO)
+	vmovsd	%xmm4, 7*SIZE(BO)
+	vmovss	%xmm7, 9*SIZE(BO)
+	vmovsd	%xmm6,10*SIZE(BO)
+	addq	$8*SIZE,BO1
+	addq	$8*SIZE,BO2
+	addq	$12*SIZE,BO
+
+	decq	%rax
+	jnz	.L6_02c_1
+
+
+.L6_02c_2:
+
+	movq    K, %rax
+        andq    $7, %rax                // K % 8
+        jz      .L6_03c
+        ALIGN_4
+
+.L6_03b:
+
+	vmovss	  1*SIZE(BO1), %xmm0
+	vmovsd	  0*SIZE(BO2), %xmm1
+	vmovss	%xmm0, 0*SIZE(BO)
+	vmovsd	%xmm1, 1*SIZE(BO)
+	addq	$2*SIZE,BO1
+	addq	$2*SIZE,BO2
+	addq	$3*SIZE,BO
+	decq	%rax
+	jnz	.L6_03b
+
+
+.L6_03c:
+
+	movq	BO2, B			// next offset of B
+
+.L6_10:
+	movq	C, CO1
+	leaq	(C, LDC, 2), C		 
+	leaq	(C, LDC, 1), C		// c += 3 * ldc
+
+	
+	movq	A, AO		 	// aoffset = a
+	addq	$32 * SIZE, AO
+
+	movq	M,  I
+	sarq	$4, I			// i = (m >> 4)
+	je	.L6_20
+
+	ALIGN_4
+
+.L6_11:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $6 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax			//  K = K - ( K % 8 )
+	je	.L6_16
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	salq	$4, %rax			// rax = rax * 16 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L6_12:
+
+	prefetcht0	B_PR1(BO,BI, SIZE)
+	KERNEL16x3_1(xxx)
+	KERNEL16x3_2(xxx)
+	KERNEL16x3_3(xxx)
+	KERNEL16x3_4(xxx)
+
+	KERNEL16x3_1(xxx)
+	prefetcht0	B_PR1+16(BO,BI, SIZE)
+	KERNEL16x3_2(xxx)
+	KERNEL16x3_3(xxx)
+	KERNEL16x3_4(xxx)
+
+	je	.L6_16
+
+	KERNEL16x3_1(xxx)
+	KERNEL16x3_2(xxx)
+	prefetcht0	B_PR1+32(BO,BI, SIZE)
+	KERNEL16x3_3(xxx)
+	KERNEL16x3_4(xxx)
+
+	KERNEL16x3_1(xxx)
+	KERNEL16x3_2(xxx)
+	KERNEL16x3_3(xxx)
+	KERNEL16x3_4(xxx)
+
+	je	.L6_16
+
+	jmp	.L6_12
+	ALIGN_4
+
+.L6_16:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L6_19
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	salq	$4, %rax			// rax = rax * 16 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L6_17:
+
+	KERNEL16x3_SUB(xxx)
+	addq	$3, BI
+	addq	$16, %rax
+	jl	.L6_17
+	ALIGN_4
+
+
+.L6_19:
+
+	vbroadcastss	ALPHA, %xmm0
+
+	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddps 4 * SIZE(CO1),%xmm0, %xmm7,%xmm7
+	vfmaddps 8 * SIZE(CO1),%xmm0, %xmm10,%xmm10
+	vfmaddps 12 * SIZE(CO1),%xmm0, %xmm13,%xmm13
+
+	vfmaddps 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+	vfmaddps 4 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
+	vfmaddps 8 * SIZE(CO1, LDC),%xmm0, %xmm11,%xmm11
+	vfmaddps 12 * SIZE(CO1, LDC),%xmm0, %xmm14,%xmm14
+
+	vfmaddps 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
+	vfmaddps 4 * SIZE(CO1, LDC, 2),%xmm0, %xmm9,%xmm9
+	vfmaddps 8 * SIZE(CO1, LDC, 2),%xmm0, %xmm12,%xmm12
+	vfmaddps 12 * SIZE(CO1, LDC, 2),%xmm0, %xmm15,%xmm15
+
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm7 , 4 * SIZE(CO1)
+	vmovups	%xmm10, 8 * SIZE(CO1)
+	vmovups	%xmm13,12 * SIZE(CO1)
+
+	vmovups	%xmm5 ,  	(CO1, LDC)
+	vmovups	%xmm8 , 4 * SIZE(CO1, LDC)
+	vmovups	%xmm11, 8 * SIZE(CO1, LDC)
+	vmovups	%xmm14,12 * SIZE(CO1, LDC)
+
+	vmovups	%xmm6 ,  	(CO1, LDC, 2)
+	vmovups	%xmm9 , 4 * SIZE(CO1, LDC, 2)
+	vmovups	%xmm12, 8 * SIZE(CO1, LDC, 2)
+	vmovups	%xmm15,12 * SIZE(CO1, LDC, 2)
+
+
+	addq	$16 * SIZE, CO1		# coffset += 16
+	decq	I			# i --
+	jg	.L6_11
+	ALIGN_4	
+
+/**************************************************************************
+* Rest of M 
+***************************************************************************/
+.L6_20:
+	// Test rest of M
+
+	testq	$15, M
+	jz	.L7_10		// to next 3 lines of N
+
+	testq	$8, M		
+	jz	.L6_21pre
+	ALIGN_4
+
+/**************************************************************************/
+
+.L6_20_1:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $6 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L6_20_6
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L6_20_2:
+
+	prefetcht0	B_PR1(BO,BI, SIZE)
+	KERNEL8x3_1(xxx)
+	KERNEL8x3_2(xxx)
+	KERNEL8x3_3(xxx)
+	KERNEL8x3_4(xxx)
+
+	KERNEL8x3_1(xxx)
+	prefetcht0	B_PR1+16(BO,BI, SIZE)
+	KERNEL8x3_2(xxx)
+	KERNEL8x3_3(xxx)
+	KERNEL8x3_4(xxx)
+
+	je	.L6_20_6
+
+	KERNEL8x3_1(xxx)
+	KERNEL8x3_2(xxx)
+	prefetcht0	B_PR1+32(BO,BI, SIZE)
+	KERNEL8x3_3(xxx)
+	KERNEL8x3_4(xxx)
+
+	KERNEL8x3_1(xxx)
+	KERNEL8x3_2(xxx)
+	KERNEL8x3_3(xxx)
+	KERNEL8x3_4(xxx)
+
+	je	.L6_20_6
+
+	jmp	.L6_20_2
+	ALIGN_4
+
+.L6_20_6:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L6_20_9
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L6_20_7:
+
+	KERNEL8x3_SUB(xxx)
+	addq	$3, BI
+	addq	$8, %rax
+	jl	.L6_20_7
+	ALIGN_4
+
+
+.L6_20_9:
+
+	vbroadcastss	ALPHA, %xmm0
+
+	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddps 4 * SIZE(CO1),%xmm0, %xmm7,%xmm7
+
+	vfmaddps 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+	vfmaddps 4 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
+
+	vfmaddps 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
+	vfmaddps 4 * SIZE(CO1, LDC, 2),%xmm0, %xmm9,%xmm9
+
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm7 , 4 * SIZE(CO1)
+
+	vmovups	%xmm5 ,  	(CO1, LDC)
+	vmovups	%xmm8 , 4 * SIZE(CO1, LDC)
+
+	vmovups	%xmm6 ,  	(CO1, LDC, 2)
+	vmovups	%xmm9 , 4 * SIZE(CO1, LDC, 2)
+
+
+	addq	$8 * SIZE, CO1		# coffset += 8
+	ALIGN_4
+	
+
+
+/**************************************************************************/
+
+.L6_21pre:
+
+	testq	$4, M		
+	jz	.L6_30
+	ALIGN_4
+
+.L6_21:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $6 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L6_26
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L6_22:
+
+	prefetcht0	B_PR1(BO,BI, SIZE)
+	KERNEL4x3_1(xxx)
+	KERNEL4x3_2(xxx)
+	KERNEL4x3_3(xxx)
+	KERNEL4x3_4(xxx)
+
+	KERNEL4x3_1(xxx)
+	prefetcht0	B_PR1+16(BO,BI, SIZE)
+	KERNEL4x3_2(xxx)
+	KERNEL4x3_3(xxx)
+	KERNEL4x3_4(xxx)
+
+	je	.L6_26
+
+	KERNEL4x3_1(xxx)
+	KERNEL4x3_2(xxx)
+	prefetcht0	B_PR1+32(BO,BI, SIZE)
+	KERNEL4x3_3(xxx)
+	KERNEL4x3_4(xxx)
+
+	KERNEL4x3_1(xxx)
+	KERNEL4x3_2(xxx)
+	KERNEL4x3_3(xxx)
+	KERNEL4x3_4(xxx)
+
+	je	.L6_26
+
+	jmp	.L6_22
+	ALIGN_4
+
+.L6_26:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L6_29
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L6_27:
+
+	KERNEL4x3_SUB(xxx)
+	addq	$3, BI
+	addq	$4, %rax
+	jl	.L6_27
+	ALIGN_4
+
+
+.L6_29:
+
+	vbroadcastss	ALPHA, %xmm0
+
+	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddps 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+	vfmaddps 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
+
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm5 ,  	(CO1, LDC)
+	vmovups	%xmm6 ,  	(CO1, LDC, 2)
+
+	addq	$4 * SIZE, CO1		# coffset += 4
+	ALIGN_4
+	
+
+.L6_30:
+	testq	$2, M		
+	jz	.L6_40
+
+	ALIGN_4
+
+.L6_31:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $6 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L6_36
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L6_32:
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL2x3_1(xxx)
+	KERNEL2x3_2(xxx)
+	KERNEL2x3_3(xxx)
+	KERNEL2x3_4(xxx)
+
+	KERNEL2x3_1(xxx)
+	prefetcht0	B_PR1+16(BO,BI,SIZE)
+	KERNEL2x3_2(xxx)
+	KERNEL2x3_3(xxx)
+	KERNEL2x3_4(xxx)
+
+	je	.L6_36
+
+	KERNEL2x3_1(xxx)
+	KERNEL2x3_2(xxx)
+	prefetcht0	B_PR1+32(BO,BI,SIZE)
+	KERNEL2x3_3(xxx)
+	KERNEL2x3_4(xxx)
+
+	KERNEL2x3_1(xxx)
+	KERNEL2x3_2(xxx)
+	KERNEL2x3_3(xxx)
+	KERNEL2x3_4(xxx)
+
+	je	.L6_36
+
+	jmp	.L6_32
+	ALIGN_4
+
+.L6_36:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L6_39
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+	
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L6_37:
+
+	KERNEL2x3_SUB(xxx)
+	addq	$3, BI
+	addq	$2, %rax
+	jl	.L6_37
+	ALIGN_4
+
+
+.L6_39:
+
+	vmovss	ALPHA, %xmm0
+
+	vfmaddss 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddss 1 * SIZE(CO1),%xmm0, %xmm8,%xmm8
+	vfmaddss 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+	vfmaddss 1 * SIZE(CO1, LDC),%xmm0, %xmm10,%xmm10
+	vfmaddss 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
+	vfmaddss 1 * SIZE(CO1, LDC, 2),%xmm0, %xmm12,%xmm12
+
+
+	vmovss	%xmm4 ,  	(CO1)
+	vmovss	%xmm8 , 1 * SIZE(CO1)
+	vmovss	%xmm5 ,  	(CO1, LDC)
+	vmovss	%xmm10, 1 * SIZE(CO1, LDC)
+	vmovss	%xmm6 ,  	(CO1, LDC, 2)
+	vmovss	%xmm12, 1 * SIZE(CO1, LDC, 2)
+
+	addq	$2 * SIZE, CO1		# coffset += 2
+	ALIGN_4
+
+.L6_40:
+	testq	$1, M		
+	jz	.L7_10		// to next 3 lines of N
+
+	ALIGN_4
+
+.L6_41:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $6 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L6_46
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L6_42:
+
+	KERNEL1x3_1(xxx)
+	KERNEL1x3_2(xxx)
+	KERNEL1x3_3(xxx)
+	KERNEL1x3_4(xxx)
+
+	KERNEL1x3_1(xxx)
+	KERNEL1x3_2(xxx)
+	KERNEL1x3_3(xxx)
+	KERNEL1x3_4(xxx)
+
+	je	.L6_46
+
+	KERNEL1x3_1(xxx)
+	KERNEL1x3_2(xxx)
+	KERNEL1x3_3(xxx)
+	KERNEL1x3_4(xxx)
+
+	KERNEL1x3_1(xxx)
+	KERNEL1x3_2(xxx)
+	KERNEL1x3_3(xxx)
+	KERNEL1x3_4(xxx)
+
+	je	.L6_46
+
+	jmp	.L6_42
+	ALIGN_4
+
+.L6_46:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L6_49
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L6_47:
+
+	KERNEL1x3_SUB(xxx)
+	addq	$3, BI
+	addq	$1, %rax
+	jl	.L6_47
+	ALIGN_4
+
+
+.L6_49:
+
+	vmovss	ALPHA, %xmm0
+
+	vfmaddss 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddss 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+	vfmaddss 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
+
+
+	vmovss	%xmm4 ,  	(CO1)
+	vmovss	%xmm5 ,  	(CO1, LDC)
+	vmovss	%xmm6 ,  	(CO1, LDC, 2)
+
+	addq	$1 * SIZE, CO1		# coffset += 1
+	ALIGN_4
+	
+
+
+
+/***************************************************************************************************************/
+
+.L7_10:
+	movq	C, CO1
+	leaq	(C, LDC, 2), C		 
+	leaq	(C, LDC, 1), C		// c += 3 * ldc
+
+	
+	movq	A, AO		 	// aoffset = a
+	addq	$32 * SIZE, AO
+
+	movq	M,  I
+	sarq	$4, I			// i = (m >> 4)
+	je	.L7_20
+
+	ALIGN_4
+
+.L7_11:
+        leaq    BUFFER2, BO             // second buffer to BO
+        addq    $6 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax			//  K = K - ( K % 8 )
+	je	.L7_16
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	salq	$4, %rax			// rax = rax * 16 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L7_12:
+
+	prefetcht0	B_PR1(BO,BI, SIZE)
+	KERNEL16x3_1(xxx)
+	KERNEL16x3_2(xxx)
+	KERNEL16x3_3(xxx)
+	KERNEL16x3_4(xxx)
+
+	KERNEL16x3_1(xxx)
+	prefetcht0	B_PR1+16(BO,BI, SIZE)
+	KERNEL16x3_2(xxx)
+	KERNEL16x3_3(xxx)
+	KERNEL16x3_4(xxx)
+
+	je	.L7_16
+
+	KERNEL16x3_1(xxx)
+	KERNEL16x3_2(xxx)
+	prefetcht0	B_PR1+32(BO,BI, SIZE)
+	KERNEL16x3_3(xxx)
+	KERNEL16x3_4(xxx)
+
+	KERNEL16x3_1(xxx)
+	KERNEL16x3_2(xxx)
+	KERNEL16x3_3(xxx)
+	KERNEL16x3_4(xxx)
+
+	je	.L7_16
+
+	jmp	.L7_12
+	ALIGN_4
+
+.L7_16:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L7_19
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	salq	$4, %rax			// rax = rax * 16 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L7_17:
+
+	KERNEL16x3_SUB(xxx)
+	addq	$3, BI
+	addq	$16, %rax
+	jl	.L7_17
+	ALIGN_4
+
+
+.L7_19:
+
+	vbroadcastss	ALPHA, %xmm0
+
+	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddps 4 * SIZE(CO1),%xmm0, %xmm7,%xmm7
+	vfmaddps 8 * SIZE(CO1),%xmm0, %xmm10,%xmm10
+	vfmaddps 12 * SIZE(CO1),%xmm0, %xmm13,%xmm13
+
+	vfmaddps 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+	vfmaddps 4 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
+	vfmaddps 8 * SIZE(CO1, LDC),%xmm0, %xmm11,%xmm11
+	vfmaddps 12 * SIZE(CO1, LDC),%xmm0, %xmm14,%xmm14
+
+	vfmaddps 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
+	vfmaddps 4 * SIZE(CO1, LDC, 2),%xmm0, %xmm9,%xmm9
+	vfmaddps 8 * SIZE(CO1, LDC, 2),%xmm0, %xmm12,%xmm12
+	vfmaddps 12 * SIZE(CO1, LDC, 2),%xmm0, %xmm15,%xmm15
+
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm7 , 4 * SIZE(CO1)
+	vmovups	%xmm10, 8 * SIZE(CO1)
+	vmovups	%xmm13,12 * SIZE(CO1)
+
+	vmovups	%xmm5 ,  	(CO1, LDC)
+	vmovups	%xmm8 , 4 * SIZE(CO1, LDC)
+	vmovups	%xmm11, 8 * SIZE(CO1, LDC)
+	vmovups	%xmm14,12 * SIZE(CO1, LDC)
+
+	vmovups	%xmm6 ,  	(CO1, LDC, 2)
+	vmovups	%xmm9 , 4 * SIZE(CO1, LDC, 2)
+	vmovups	%xmm12, 8 * SIZE(CO1, LDC, 2)
+	vmovups	%xmm15,12 * SIZE(CO1, LDC, 2)
+
+
+	addq	$16 * SIZE, CO1		# coffset += 16
+	decq	I			# i --
+	jg	.L7_11
+	ALIGN_4	
+
+/**************************************************************************
+* Rest of M 
+***************************************************************************/
+.L7_20:
+	// Test rest of M
+
+	testq	$15, M
+	jz	.L7_60		// to next 3 lines of N
+
+	testq	$8, M		
+	jz	.L7_21pre
+	ALIGN_4
+
+/**************************************************************************/
+
+.L7_20_1:
+        leaq    BUFFER2, BO             // first buffer to BO
+        addq    $6 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L7_20_6
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L7_20_2:
+
+	prefetcht0	B_PR1(BO,BI, SIZE)
+	KERNEL8x3_1(xxx)
+	KERNEL8x3_2(xxx)
+	KERNEL8x3_3(xxx)
+	KERNEL8x3_4(xxx)
+
+	KERNEL8x3_1(xxx)
+	prefetcht0	B_PR1+16(BO,BI, SIZE)
+	KERNEL8x3_2(xxx)
+	KERNEL8x3_3(xxx)
+	KERNEL8x3_4(xxx)
+
+	je	.L7_20_6
+
+	KERNEL8x3_1(xxx)
+	KERNEL8x3_2(xxx)
+	prefetcht0	B_PR1+32(BO,BI, SIZE)
+	KERNEL8x3_3(xxx)
+	KERNEL8x3_4(xxx)
+
+	KERNEL8x3_1(xxx)
+	KERNEL8x3_2(xxx)
+	KERNEL8x3_3(xxx)
+	KERNEL8x3_4(xxx)
+
+	je	.L7_20_6
+
+	jmp	.L7_20_2
+	ALIGN_4
+
+.L7_20_6:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L7_20_9
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L7_20_7:
+
+	KERNEL8x3_SUB(xxx)
+	addq	$3, BI
+	addq	$8, %rax
+	jl	.L7_20_7
+	ALIGN_4
+
+.L7_20_9:
+
+
+	vbroadcastss	ALPHA, %xmm0
+
+	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddps 4 * SIZE(CO1),%xmm0, %xmm7,%xmm7
+
+	vfmaddps 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+	vfmaddps 4 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
+
+	vfmaddps 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
+	vfmaddps 4 * SIZE(CO1, LDC, 2),%xmm0, %xmm9,%xmm9
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm7 , 4 * SIZE(CO1)
+
+	vmovups	%xmm5 ,  	(CO1, LDC)
+	vmovups	%xmm8 , 4 * SIZE(CO1, LDC)
+
+	vmovups	%xmm6 ,  	(CO1, LDC, 2)
+	vmovups	%xmm9 , 4 * SIZE(CO1, LDC, 2)
+
+
+	addq	$8 * SIZE, CO1		# coffset += 8
+	ALIGN_4
+	
+
+
+/**************************************************************************/
+
+.L7_21pre:
+
+	testq	$4, M		
+	jz	.L7_30
+	ALIGN_4
+
+.L7_21:
+        leaq    BUFFER2, BO             // second buffer to BO
+        addq    $6 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L7_26
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L7_22:
+
+	prefetcht0	B_PR1(BO,BI, SIZE)
+	KERNEL4x3_1(xxx)
+	KERNEL4x3_2(xxx)
+	KERNEL4x3_3(xxx)
+	KERNEL4x3_4(xxx)
+
+	KERNEL4x3_1(xxx)
+	prefetcht0	B_PR1+16(BO,BI, SIZE)
+	KERNEL4x3_2(xxx)
+	KERNEL4x3_3(xxx)
+	KERNEL4x3_4(xxx)
+
+	je	.L7_26
+
+	KERNEL4x3_1(xxx)
+	KERNEL4x3_2(xxx)
+	prefetcht0	B_PR1+32(BO,BI, SIZE)
+	KERNEL4x3_3(xxx)
+	KERNEL4x3_4(xxx)
+
+	KERNEL4x3_1(xxx)
+	KERNEL4x3_2(xxx)
+	KERNEL4x3_3(xxx)
+	KERNEL4x3_4(xxx)
+
+	je	.L7_26
+
+	jmp	.L7_22
+	ALIGN_4
+
+.L7_26:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L7_29
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L7_27:
+
+	KERNEL4x3_SUB(xxx)
+	addq	$3, BI
+	addq	$4, %rax
+	jl	.L7_27
+	ALIGN_4
+
+
+.L7_29:
+
+	vbroadcastss	ALPHA, %xmm0
+
+	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddps 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+	vfmaddps 	 (CO1, LDC, 2),%xmm0, %xmm6 ,%xmm6
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm5 ,  	(CO1, LDC)
+	vmovups	%xmm6 ,  	(CO1, LDC, 2)
+
+	addq	$4 * SIZE, CO1		# coffset += 4
+	ALIGN_4
+	
+
+.L7_30:
+	testq	$2, M		
+	jz	.L7_40
+
+	ALIGN_4
+
+.L7_31:
+        leaq    BUFFER2, BO             // second buffer to BO
+        addq    $6 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L7_36
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L7_32:
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL2x3_1(xxx)
+	KERNEL2x3_2(xxx)
+	KERNEL2x3_3(xxx)
+	KERNEL2x3_4(xxx)
+
+	KERNEL2x3_1(xxx)
+	prefetcht0	B_PR1+16(BO,BI,SIZE)
+	KERNEL2x3_2(xxx)
+	KERNEL2x3_3(xxx)
+	KERNEL2x3_4(xxx)
+
+	je	.L7_36
+
+	KERNEL2x3_1(xxx)
+	KERNEL2x3_2(xxx)
+	prefetcht0	B_PR1+32(BO,BI,SIZE)
+	KERNEL2x3_3(xxx)
+	KERNEL2x3_4(xxx)
+
+	KERNEL2x3_1(xxx)
+	KERNEL2x3_2(xxx)
+	KERNEL2x3_3(xxx)
+	KERNEL2x3_4(xxx)
+
+	je	.L7_36
+
+	jmp	.L7_32
+	ALIGN_4
+
+.L7_36:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L7_39
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+	
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L7_37:
+
+	KERNEL2x3_SUB(xxx)
+	addq	$3, BI
+	addq	$2, %rax
+	jl	.L7_37
+	ALIGN_4
+
+
+.L7_39:
+
+	vmovss	ALPHA, %xmm0
+
+	vfmaddss 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddss 1 * SIZE(CO1),%xmm0, %xmm8,%xmm8
+	vfmaddss 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+	vfmaddss 1 * SIZE(CO1, LDC),%xmm0, %xmm10,%xmm10
+	vfmaddss 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
+	vfmaddss 1 * SIZE(CO1, LDC, 2),%xmm0, %xmm12,%xmm12
+
+	vmovss	%xmm4 ,  	(CO1)
+	vmovss	%xmm8 , 1 * SIZE(CO1)
+	vmovss	%xmm5 ,  	(CO1, LDC)
+	vmovss	%xmm10, 1 * SIZE(CO1, LDC)
+	vmovss	%xmm6 ,  	(CO1, LDC, 2)
+	vmovss	%xmm12, 1 * SIZE(CO1, LDC, 2)
+
+	addq	$2 * SIZE, CO1		# coffset += 2
+	ALIGN_4
+
+.L7_40:
+	testq	$1, M		
+	jz	.L7_60		// to next 3 lines of N
+
+	ALIGN_4
+
+.L7_41:
+        leaq    BUFFER2, BO             // second buffer to BO
+        addq    $6 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L7_46
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L7_42:
+
+	KERNEL1x3_1(xxx)
+	KERNEL1x3_2(xxx)
+	KERNEL1x3_3(xxx)
+	KERNEL1x3_4(xxx)
+
+	KERNEL1x3_1(xxx)
+	KERNEL1x3_2(xxx)
+	KERNEL1x3_3(xxx)
+	KERNEL1x3_4(xxx)
+
+	je	.L7_46
+
+	KERNEL1x3_1(xxx)
+	KERNEL1x3_2(xxx)
+	KERNEL1x3_3(xxx)
+	KERNEL1x3_4(xxx)
+
+	KERNEL1x3_1(xxx)
+	KERNEL1x3_2(xxx)
+	KERNEL1x3_3(xxx)
+	KERNEL1x3_4(xxx)
+
+	je	.L7_46
+
+	jmp	.L7_42
+	ALIGN_4
+
+.L7_46:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L7_49
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L7_47:
+
+	KERNEL1x3_SUB(xxx)
+	addq	$3, BI
+	addq	$1, %rax
+	jl	.L7_47
+	ALIGN_4
+
+
+.L7_49:
+
+	vmovss	ALPHA, %xmm0
+
+	vfmaddss 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddss 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+	vfmaddss 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
+
+	vmovss	%xmm4 ,  	(CO1)
+	vmovss	%xmm5 ,  	(CO1, LDC)
+	vmovss	%xmm6 ,  	(CO1, LDC, 2)
+
+	addq	$1 * SIZE, CO1		# coffset += 1
+	ALIGN_4
+	
+
+
+.L7_60:
+
+	decq	J			// j --
+	jg	.L6_01
+
+
+.L2_0:
+	cmpq	$0, Nmod6		// N % 6 == 0
+	je	.L999
+
+/************************************************************************************************
+* Loop for Nmod6 / 2 > 0
+*************************************************************************************************/
+
+	movq	Nmod6, J		
+	sarq	$1, J			// j = j / 2
+	je	.L1_0
+	ALIGN_4
+
+.L2_01:
+	// copy to sub buffer
+	movq	B, BO1
+	leaq    BUFFER1, BO		// first buffer to BO
+	movq	K, %rax
+	ALIGN_4
+
+.L2_02b:
+
+	vmovsd	(BO1), %xmm0
+	vmovsd	%xmm0,       (BO)
+	addq	$2*SIZE,BO1
+	addq	$2*SIZE,BO
+	decq	%rax
+	jnz	.L2_02b
+
+.L2_02c:
+
+	movq	BO1, B			// next offset of B
+
+.L2_10:
+	movq	C, CO1
+	leaq	(C, LDC, 2), C		// c += 2 * ldc
+
+	
+	movq	A, AO		 	// aoffset = a
+	addq	$32 * SIZE, AO
+
+	movq	M,  I
+	sarq	$4, I			// i = (m >> 4)
+	je	.L2_20
+
+	ALIGN_4
+
+.L2_11:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax			//  K = K - ( K % 8 )
+	je	.L2_16
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$4, %rax			// rax = rax * 16 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_12:
+
+	prefetcht0	B_PR1(BO,BI, SIZE)
+	KERNEL16x2_1(xxx)
+	KERNEL16x2_2(xxx)
+	KERNEL16x2_3(xxx)
+	KERNEL16x2_4(xxx)
+
+	KERNEL16x2_1(xxx)
+	KERNEL16x2_2(xxx)
+	KERNEL16x2_3(xxx)
+	KERNEL16x2_4(xxx)
+
+	je	.L2_16
+
+	prefetcht0	B_PR1(BO,BI, SIZE)
+	KERNEL16x2_1(xxx)
+	KERNEL16x2_2(xxx)
+	KERNEL16x2_3(xxx)
+	KERNEL16x2_4(xxx)
+
+	KERNEL16x2_1(xxx)
+	KERNEL16x2_2(xxx)
+	KERNEL16x2_3(xxx)
+	KERNEL16x2_4(xxx)
+
+	je	.L2_16
+
+	jmp	.L2_12
+	ALIGN_4
+
+.L2_16:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_19
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$4, %rax			// rax = rax * 16 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_17:
+
+	KERNEL16x2_SUB(xxx)
+	addq	$2, BI
+	addq	$16, %rax
+	jl	.L2_17
+	ALIGN_4
+
+
+.L2_19:
+
+	vbroadcastss	ALPHA, %xmm0
+
+	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddps 4 * SIZE(CO1),%xmm0, %xmm7,%xmm7
+	vfmaddps 8 * SIZE(CO1),%xmm0, %xmm10,%xmm10
+	vfmaddps 12 * SIZE(CO1),%xmm0, %xmm13,%xmm13
+
+	vfmaddps 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+	vfmaddps 4 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
+	vfmaddps 8 * SIZE(CO1, LDC),%xmm0, %xmm11,%xmm11
+	vfmaddps 12 * SIZE(CO1, LDC),%xmm0, %xmm14,%xmm14
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm7 , 4 * SIZE(CO1)
+	vmovups	%xmm10, 8 * SIZE(CO1)
+	vmovups	%xmm13,12 * SIZE(CO1)
+
+	vmovups	%xmm5 ,  	(CO1, LDC)
+	vmovups	%xmm8 , 4 * SIZE(CO1, LDC)
+	vmovups	%xmm11, 8 * SIZE(CO1, LDC)
+	vmovups	%xmm14,12 * SIZE(CO1, LDC)
+
+	addq	$16 * SIZE, CO1		# coffset += 16
+	decq	I			# i --
+	jg	.L2_11
+	ALIGN_4	
+
+/**************************************************************************
+* Rest of M 
+***************************************************************************/
+.L2_20:
+	// Test rest of M
+
+	testq	$15, M
+	jz	.L2_60		// to next 3 lines of N
+
+	testq	$8, M		
+	jz	.L2_21pre
+	ALIGN_4
+
+/**************************************************************************/
+
+.L2_20_1:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L2_20_6
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_20_2:
+
+	prefetcht0	B_PR1(BO,BI, SIZE)
+	KERNEL8x2_1(xxx)
+	KERNEL8x2_2(xxx)
+	KERNEL8x2_3(xxx)
+	KERNEL8x2_4(xxx)
+
+	KERNEL8x2_1(xxx)
+	KERNEL8x2_2(xxx)
+	KERNEL8x2_3(xxx)
+	KERNEL8x2_4(xxx)
+
+	je	.L2_20_6
+
+	prefetcht0	B_PR1(BO,BI, SIZE)
+	KERNEL8x2_1(xxx)
+	KERNEL8x2_2(xxx)
+	KERNEL8x2_3(xxx)
+	KERNEL8x2_4(xxx)
+
+	KERNEL8x2_1(xxx)
+	KERNEL8x2_2(xxx)
+	KERNEL8x2_3(xxx)
+	KERNEL8x2_4(xxx)
+
+	je	.L2_20_6
+
+	jmp	.L2_20_2
+	ALIGN_4
+
+.L2_20_6:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_20_9
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_20_7:
+
+	KERNEL8x2_SUB(xxx)
+	addq	$2, BI
+	addq	$8, %rax
+	jl	.L2_20_7
+	ALIGN_4
+
+
+.L2_20_9:
+
+	vbroadcastss	ALPHA, %xmm0
+
+	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddps 4 * SIZE(CO1),%xmm0, %xmm7,%xmm7
+
+	vfmaddps 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+	vfmaddps 4 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm7 , 4 * SIZE(CO1)
+
+	vmovups	%xmm5 ,  	(CO1, LDC)
+	vmovups	%xmm8 , 4 * SIZE(CO1, LDC)
+
+
+	addq	$8 * SIZE, CO1		# coffset += 8
+	ALIGN_4
+	
+
+
+/**************************************************************************/
+
+.L2_21pre:
+
+	testq	$4, M		
+	jz	.L2_30
+	ALIGN_4
+
+.L2_21:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L2_26
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 1 ; number of values
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_22:
+
+	prefetcht0	B_PR1(BO,BI, SIZE)
+	KERNEL4x2_1(xxx)
+	KERNEL4x2_2(xxx)
+	KERNEL4x2_3(xxx)
+	KERNEL4x2_4(xxx)
+
+	KERNEL4x2_1(xxx)
+	KERNEL4x2_2(xxx)
+	KERNEL4x2_3(xxx)
+	KERNEL4x2_4(xxx)
+
+	je	.L2_26
+
+	prefetcht0	B_PR1(BO,BI, SIZE)
+	KERNEL4x2_1(xxx)
+	KERNEL4x2_2(xxx)
+	KERNEL4x2_3(xxx)
+	KERNEL4x2_4(xxx)
+
+	KERNEL4x2_1(xxx)
+	KERNEL4x2_2(xxx)
+	KERNEL4x2_3(xxx)
+	KERNEL4x2_4(xxx)
+
+	je	.L2_26
+
+	jmp	.L2_22
+	ALIGN_4
+
+.L2_26:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_29
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_27:
+
+	KERNEL4x2_SUB(xxx)
+	addq	$2, BI
+	addq	$4, %rax
+	jl	.L2_27
+	ALIGN_4
+
+
+.L2_29:
+
+	vbroadcastss	ALPHA, %xmm0
+
+	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddps 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm5 ,  	(CO1, LDC)
+
+
+	addq	$4 * SIZE, CO1		# coffset += 4
+	ALIGN_4
+	
+
+.L2_30:
+	testq	$2, M		
+	jz	.L2_40
+
+	ALIGN_4
+
+.L2_31:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L2_36
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_32:
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL2x2_1(xxx)
+	KERNEL2x2_2(xxx)
+	KERNEL2x2_3(xxx)
+	KERNEL2x2_4(xxx)
+
+	KERNEL2x2_1(xxx)
+	KERNEL2x2_2(xxx)
+	KERNEL2x2_3(xxx)
+	KERNEL2x2_4(xxx)
+
+	je	.L2_36
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL2x2_1(xxx)
+	KERNEL2x2_2(xxx)
+	KERNEL2x2_3(xxx)
+	KERNEL2x2_4(xxx)
+
+	KERNEL2x2_1(xxx)
+	KERNEL2x2_2(xxx)
+	KERNEL2x2_3(xxx)
+	KERNEL2x2_4(xxx)
+
+	je	.L2_36
+
+	jmp	.L2_32
+	ALIGN_4
+
+.L2_36:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_39
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+	
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_37:
+
+	KERNEL2x2_SUB(xxx)
+	addq	$2, BI
+	addq	$2, %rax
+	jl	.L2_37
+	ALIGN_4
+
+
+.L2_39:
+
+	vmovss	ALPHA, %xmm0
+
+	vfmaddss 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddss 1 * SIZE(CO1),%xmm0, %xmm8,%xmm8
+	vfmaddss 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+	vfmaddss 1 * SIZE(CO1, LDC),%xmm0, %xmm10,%xmm10
+
+	vmovss	%xmm4 ,  	(CO1)
+	vmovss	%xmm8 , 1 * SIZE(CO1)
+	vmovss	%xmm5 ,  	(CO1, LDC)
+	vmovss	%xmm10, 1 * SIZE(CO1, LDC)
+
+	addq	$2 * SIZE, CO1		# coffset += 2
+	ALIGN_4
+
+.L2_40:
+	testq	$1, M		
+	jz	.L2_60		// to next 2 lines of N
+
+	ALIGN_4
+
+.L2_41:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L2_46
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_42:
+
+	KERNEL1x2_1(xxx)
+	KERNEL1x2_2(xxx)
+	KERNEL1x2_3(xxx)
+	KERNEL1x2_4(xxx)
+
+	KERNEL1x2_1(xxx)
+	KERNEL1x2_2(xxx)
+	KERNEL1x2_3(xxx)
+	KERNEL1x2_4(xxx)
+
+	je	.L2_46
+
+	KERNEL1x2_1(xxx)
+	KERNEL1x2_2(xxx)
+	KERNEL1x2_3(xxx)
+	KERNEL1x2_4(xxx)
+
+	KERNEL1x2_1(xxx)
+	KERNEL1x2_2(xxx)
+	KERNEL1x2_3(xxx)
+	KERNEL1x2_4(xxx)
+
+	je	.L2_46
+
+	jmp	.L2_42
+	ALIGN_4
+
+.L2_46:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_49
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_47:
+
+	KERNEL1x2_SUB(xxx)
+	addq	$2, BI
+	addq	$1, %rax
+	jl	.L2_47
+	ALIGN_4
+
+
+.L2_49:
+
+	vmovss	ALPHA, %xmm0
+
+	vfmaddss 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddss 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+
+	vmovss	%xmm4 ,  	(CO1)
+	vmovss	%xmm5 ,  	(CO1, LDC)
+
+	addq	$1 * SIZE, CO1		# coffset += 1
+	ALIGN_4
+	
+.L2_60:
+
+	decq	J			// j --
+	jg	.L2_01			// next 2 lines of N
+
+
+
+.L1_0:
+
+/************************************************************************************************
+* Loop for Nmod6 % 2 > 0
+*************************************************************************************************/
+
+	movq	Nmod6, J		
+	andq	$1, J			// j % 2
+	je	.L999
+	ALIGN_4
+
+.L1_01:
+	// copy to sub buffer
+	movq	B, BO1
+	leaq    BUFFER1, BO		// first buffer to BO
+	movq	K, %rax
+	ALIGN_4
+
+.L1_02b:
+
+	vmovss	(BO1), %xmm0
+	vmovss	%xmm0,       (BO)
+	addq	$1*SIZE,BO1
+	addq	$1*SIZE,BO
+	decq	%rax
+	jnz	.L1_02b
+
+.L1_02c:
+
+	movq	BO1, B			// next offset of B
+
+.L1_10:
+	movq	C, CO1
+	leaq	(C, LDC, 1), C		// c += 1 * ldc
+
+	
+	movq	A, AO		 	// aoffset = a
+	addq	$32 * SIZE, AO
+
+	movq	M,  I
+	sarq	$4, I			// i = (m >> 4)
+	je	.L1_20
+
+	ALIGN_4
+
+.L1_11:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax			//  K = K - ( K % 8 )
+	je	.L1_16
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$4, %rax			// rax = rax * 16 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_12:
+
+	prefetcht0	B_PR1(BO,BI, SIZE)
+	KERNEL16x1_1(xxx)
+	KERNEL16x1_2(xxx)
+	KERNEL16x1_3(xxx)
+	KERNEL16x1_4(xxx)
+
+	KERNEL16x1_1(xxx)
+	KERNEL16x1_2(xxx)
+	KERNEL16x1_3(xxx)
+	KERNEL16x1_4(xxx)
+
+	je	.L1_16
+
+	KERNEL16x1_1(xxx)
+	KERNEL16x1_2(xxx)
+	KERNEL16x1_3(xxx)
+	KERNEL16x1_4(xxx)
+
+	KERNEL16x1_1(xxx)
+	KERNEL16x1_2(xxx)
+	KERNEL16x1_3(xxx)
+	KERNEL16x1_4(xxx)
+
+	je	.L1_16
+
+	jmp	.L1_12
+	ALIGN_4
+
+.L1_16:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_19
+
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$4, %rax			// rax = rax * 16 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_17:
+
+	KERNEL16x1_SUB(xxx)
+	addq	$1, BI
+	addq	$16, %rax
+	jl	.L1_17
+	ALIGN_4
+
+
+.L1_19:
+
+	vbroadcastss	ALPHA, %xmm0
+
+	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddps 4 * SIZE(CO1),%xmm0, %xmm7,%xmm7
+	vfmaddps 8 * SIZE(CO1),%xmm0, %xmm10,%xmm10
+	vfmaddps 12 * SIZE(CO1),%xmm0, %xmm13,%xmm13
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm7 , 4 * SIZE(CO1)
+	vmovups	%xmm10, 8 * SIZE(CO1)
+	vmovups	%xmm13,12 * SIZE(CO1)
+
+	addq	$16 * SIZE, CO1		# coffset += 16
+	decq	I			# i --
+	jg	.L1_11
+	ALIGN_4	
+
+/**************************************************************************
+* Rest of M 
+***************************************************************************/
+.L1_20:
+	// Test rest of M
+
+	testq	$15, M
+	jz	.L999
+
+	testq	$8, M		
+	jz	.L1_21pre
+	ALIGN_4
+
+/**************************************************************************/
+
+.L1_20_1:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L1_20_6
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_20_2:
+
+	prefetcht0	B_PR1(BO,BI, SIZE)
+	KERNEL8x1_1(xxx)
+	KERNEL8x1_2(xxx)
+	KERNEL8x1_3(xxx)
+	KERNEL8x1_4(xxx)
+
+	KERNEL8x1_1(xxx)
+	KERNEL8x1_2(xxx)
+	KERNEL8x1_3(xxx)
+	KERNEL8x1_4(xxx)
+
+	je	.L1_20_6
+
+	KERNEL8x1_1(xxx)
+	KERNEL8x1_2(xxx)
+	KERNEL8x1_3(xxx)
+	KERNEL8x1_4(xxx)
+
+	KERNEL8x1_1(xxx)
+	KERNEL8x1_2(xxx)
+	KERNEL8x1_3(xxx)
+	KERNEL8x1_4(xxx)
+
+	je	.L1_20_6
+
+	jmp	.L1_20_2
+	ALIGN_4
+
+.L1_20_6:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_20_9
+
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_20_7:
+
+	KERNEL8x1_SUB(xxx)
+	addq	$1, BI
+	addq	$8, %rax
+	jl	.L1_20_7
+	ALIGN_4
+
+
+.L1_20_9:
+
+	vbroadcastss	ALPHA, %xmm0
+
+	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddps 4 * SIZE(CO1),%xmm0, %xmm7,%xmm7
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm7 , 4 * SIZE(CO1)
+
+	addq	$8 * SIZE, CO1		# coffset += 8
+	ALIGN_4
+	
+
+
+/**************************************************************************/
+
+.L1_21pre:
+
+	testq	$4, M		
+	jz	.L1_30
+	ALIGN_4
+
+.L1_21:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L1_26
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_22:
+
+	prefetcht0	B_PR1(BO,BI, SIZE)
+	KERNEL4x1_1(xxx)
+	KERNEL4x1_2(xxx)
+	KERNEL4x1_3(xxx)
+	KERNEL4x1_4(xxx)
+
+	KERNEL4x1_1(xxx)
+	KERNEL4x1_2(xxx)
+	KERNEL4x1_3(xxx)
+	KERNEL4x1_4(xxx)
+
+	je	.L1_26
+
+	KERNEL4x1_1(xxx)
+	KERNEL4x1_2(xxx)
+	KERNEL4x1_3(xxx)
+	KERNEL4x1_4(xxx)
+
+	KERNEL4x1_1(xxx)
+	KERNEL4x1_2(xxx)
+	KERNEL4x1_3(xxx)
+	KERNEL4x1_4(xxx)
+
+	je	.L1_26
+
+	jmp	.L1_22
+	ALIGN_4
+
+.L1_26:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_29
+
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_27:
+
+	KERNEL4x1_SUB(xxx)
+	addq	$1, BI
+	addq	$4, %rax
+	jl	.L1_27
+	ALIGN_4
+
+
+.L1_29:
+
+	vbroadcastss	ALPHA, %xmm0
+
+	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
+
+	vmovups	%xmm4 ,  	(CO1)
+
+	addq	$4 * SIZE, CO1		# coffset += 4
+	ALIGN_4
+	
+
+.L1_30:
+	testq	$2, M		
+	jz	.L1_40
+
+	ALIGN_4
+
+.L1_31:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L1_36
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_32:
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL2x1_1(xxx)
+	KERNEL2x1_2(xxx)
+	KERNEL2x1_3(xxx)
+	KERNEL2x1_4(xxx)
+
+	KERNEL2x1_1(xxx)
+	KERNEL2x1_2(xxx)
+	KERNEL2x1_3(xxx)
+	KERNEL2x1_4(xxx)
+
+	je	.L1_36
+
+	KERNEL2x1_1(xxx)
+	KERNEL2x1_2(xxx)
+	KERNEL2x1_3(xxx)
+	KERNEL2x1_4(xxx)
+
+	KERNEL2x1_1(xxx)
+	KERNEL2x1_2(xxx)
+	KERNEL2x1_3(xxx)
+	KERNEL2x1_4(xxx)
+
+	je	.L1_36
+
+	jmp	.L1_32
+	ALIGN_4
+
+.L1_36:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_39
+
+	movq    %rax, BI                        //  Index for BO
+	
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_37:
+
+	KERNEL2x1_SUB(xxx)
+	addq	$1, BI
+	addq	$2, %rax
+	jl	.L1_37
+	ALIGN_4
+
+
+.L1_39:
+
+	vmovss	ALPHA, %xmm0
+
+	vfmaddss 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddss 1 * SIZE(CO1),%xmm0, %xmm8,%xmm8
+
+	vmovss	%xmm4 ,  	(CO1)
+	vmovss	%xmm8 , 1 * SIZE(CO1)
+
+	addq	$2 * SIZE, CO1		# coffset += 2
+	ALIGN_4
+
+.L1_40:
+	testq	$1, M		
+	jz	.L999
+
+	ALIGN_4
+
+.L1_41:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L1_46
+	movq    %rax, BI                        //  Index for BO
+
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_42:
+
+	KERNEL1x1_1(xxx)
+	KERNEL1x1_2(xxx)
+	KERNEL1x1_3(xxx)
+	KERNEL1x1_4(xxx)
+
+	KERNEL1x1_1(xxx)
+	KERNEL1x1_2(xxx)
+	KERNEL1x1_3(xxx)
+	KERNEL1x1_4(xxx)
+
+	je	.L1_46
+
+	KERNEL1x1_1(xxx)
+	KERNEL1x1_2(xxx)
+	KERNEL1x1_3(xxx)
+	KERNEL1x1_4(xxx)
+
+	KERNEL1x1_1(xxx)
+	KERNEL1x1_2(xxx)
+	KERNEL1x1_3(xxx)
+	KERNEL1x1_4(xxx)
+
+	je	.L1_46
+
+	jmp	.L1_42
+	ALIGN_4
+
+.L1_46:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_49
+
+	movq    %rax, BI                        //  Index for BO
+
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_47:
+
+	KERNEL1x1_SUB(xxx)
+	addq	$1, BI
+	addq	$1, %rax
+	jl	.L1_47
+	ALIGN_4
+
+
+.L1_49:
+
+	vmovss	ALPHA, %xmm0
+
+	vfmaddss 	 (CO1),%xmm0, %xmm4,%xmm4
+
+	vmovss	%xmm4 ,  	(CO1)
+
+	addq	$1 * SIZE, CO1		# coffset += 1
+	ALIGN_4
+	
+
+.L999:
+	movq   		SP, %rsp
+	movq	   (%rsp), %rbx
+	movq	  8(%rsp), %rbp
+	movq	 16(%rsp), %r12
+	movq	 24(%rsp), %r13
+	movq	 32(%rsp), %r14
+	movq	 40(%rsp), %r15
+
+#ifdef WINDOWS_ABI
+	movq	 48(%rsp), %rdi
+	movq	 56(%rsp), %rsi
+	movups	 64(%rsp), %xmm6
+	movups	 80(%rsp), %xmm7
+	movups	 96(%rsp), %xmm8
+	movups	112(%rsp), %xmm9
+	movups	128(%rsp), %xmm10
+	movups	144(%rsp), %xmm11
+	movups	160(%rsp), %xmm12
+	movups	176(%rsp), %xmm13
+	movups	192(%rsp), %xmm14
+	movups	208(%rsp), %xmm15
+#endif
+
+	addq	$STACKSIZE, %rsp
+	ret
+
+	EPILOGUE
+
+
+#else
+/*************************************************************************************
+* TRMM Kernel
+*************************************************************************************/
+
+
+	PROLOGUE
+	PROFCODE
+	
+	subq	$STACKSIZE, %rsp
+	movq	%rbx,   (%rsp)
+	movq	%rbp,  8(%rsp)
+	movq	%r12, 16(%rsp)
+	movq	%r13, 24(%rsp)
+	movq	%r14, 32(%rsp)
+	movq	%r15, 40(%rsp)
+
+	vzeroupper
+
+#ifdef WINDOWS_ABI
+	movq	%rdi,    48(%rsp)
+	movq	%rsi,    56(%rsp)
+	movups	%xmm6,   64(%rsp)
+	movups	%xmm7,   80(%rsp)
+	movups	%xmm8,   96(%rsp)
+	movups	%xmm9,  112(%rsp)
+	movups	%xmm10, 128(%rsp)
+	movups	%xmm11, 144(%rsp)
+	movups	%xmm12, 160(%rsp)
+	movups	%xmm13, 176(%rsp)
+	movups	%xmm14, 192(%rsp)
+	movups	%xmm15, 208(%rsp)
+
+	movq	ARG1,      OLD_M
+	movq	ARG2,      OLD_N
+	movq	ARG3,      OLD_K
+	movq	OLD_A,     A
+	movq	OLD_B,     B
+	movq	OLD_C,     C
+	movq	OLD_LDC,   LDC
+#ifdef TRMMKERNEL
+	movsd	OLD_OFFSET, %xmm12
+#endif
+	vmovaps	%xmm3, %xmm0
+
+#else
+	movq	STACKSIZE +  8(%rsp), LDC
+#ifdef TRMMKERNEL
+	movsd	STACKSIZE + 16(%rsp), %xmm12
+#endif
+
+#endif
+
+	movq    %rsp, SP      # save old stack
+        subq    $128 + L_BUFFER_SIZE, %rsp
+        andq    $-4096, %rsp    # align stack
+
+        STACK_TOUCH
+
+	cmpq	$0, OLD_M
+	je	.L999
+
+	cmpq	$0, OLD_N
+	je	.L999
+
+	cmpq	$0, OLD_K
+	je	.L999
+
+	movq	OLD_M, M
+	movq	OLD_N, N
+	movq	OLD_K, K
+
+	vmovsd	 %xmm0, ALPHA
+
+	salq	$BASE_SHIFT, LDC
+
+	movq    N, %rax
+        xorq    %rdx, %rdx
+        movq    $2,  %rdi
+        divq    %rdi                    //    N / 6
+        movq    %rax, Ndiv6             //    N / 6
+        movq    %rdx, Nmod6             //    N % 6
+
+	
+
+#ifdef TRMMKERNEL
+	vmovsd	%xmm12, OFFSET
+	vmovsd	%xmm12, KK
+#ifndef LEFT
+	negq	KK
+#endif	
+#endif
+
+	movq	Ndiv6,  J
+	cmpq	$0, J
+	je	.L1_0
+	ALIGN_4
+
+.L2_01:
+	// copy to sub buffer
+	movq	B, BO1
+	leaq    BUFFER1, BO		// first buffer to BO
+	movq	K, %rax
+	ALIGN_4
+
+.L2_02b:
+
+	vmovsd	(BO1), %xmm0
+	vmovsd	%xmm0,       (BO)
+	addq	$2*SIZE,BO1
+	addq	$2*SIZE,BO
+	decq	%rax
+	jnz	.L2_02b
+
+.L2_02c:
+
+	movq	BO1, B			// next offset of B
+
+.L2_10:
+	movq	C, CO1
+	leaq	(C, LDC, 2), C		// c += 2 * ldc
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        movq    OFFSET, %rax
+        movq    %rax, KK
+#endif
+	
+	movq	A, AO		 	// aoffset = a
+	addq	$32 * SIZE, AO
+
+	movq	M,  I
+	sarq	$4, I			// i = (m >> 4)
+	je	.L2_20
+
+	ALIGN_4
+
+.L2_11:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO
+        salq    $4, %rax                        // rax = rax * 16 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $16, %rax	// number of values in AO
+#else
+        addq    $2, %rax	// number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+	andq	$-8, %rax			//  K = K - ( K % 8 )
+	je	.L2_16
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$4, %rax			// rax = rax * 16 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_12:
+
+	prefetcht0	B_PR1(BO,BI, SIZE)
+	KERNEL16x2_1(xxx)
+	KERNEL16x2_2(xxx)
+	KERNEL16x2_3(xxx)
+	KERNEL16x2_4(xxx)
+
+	KERNEL16x2_1(xxx)
+	KERNEL16x2_2(xxx)
+	KERNEL16x2_3(xxx)
+	KERNEL16x2_4(xxx)
+
+	je	.L2_16
+
+	prefetcht0	B_PR1(BO,BI, SIZE)
+	KERNEL16x2_1(xxx)
+	KERNEL16x2_2(xxx)
+	KERNEL16x2_3(xxx)
+	KERNEL16x2_4(xxx)
+
+	KERNEL16x2_1(xxx)
+	KERNEL16x2_2(xxx)
+	KERNEL16x2_3(xxx)
+	KERNEL16x2_4(xxx)
+
+	je	.L2_16
+
+	jmp	.L2_12
+	ALIGN_4
+
+.L2_16:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_19
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$4, %rax			// rax = rax * 16 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_17:
+
+	KERNEL16x2_SUB(xxx)
+	addq	$2, BI
+	addq	$16, %rax
+	jl	.L2_17
+	ALIGN_4
+
+
+.L2_19:
+
+	vbroadcastss	ALPHA, %xmm0
+
+#ifndef TRMMKERNEL
+
+	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddps 4 * SIZE(CO1),%xmm0, %xmm7,%xmm7
+	vfmaddps 8 * SIZE(CO1),%xmm0, %xmm10,%xmm10
+	vfmaddps 12 * SIZE(CO1),%xmm0, %xmm13,%xmm13
+
+	vfmaddps 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+	vfmaddps 4 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
+	vfmaddps 8 * SIZE(CO1, LDC),%xmm0, %xmm11,%xmm11
+	vfmaddps 12 * SIZE(CO1, LDC),%xmm0, %xmm14,%xmm14
+
+#else
+	vmulps	%xmm0, %xmm4,%xmm4
+	vmulps	%xmm0, %xmm7,%xmm7
+	vmulps	%xmm0, %xmm10,%xmm10
+	vmulps	%xmm0, %xmm13,%xmm13
+
+	vmulps	%xmm0, %xmm5,%xmm5
+	vmulps	%xmm0, %xmm8,%xmm8
+	vmulps	%xmm0, %xmm11,%xmm11
+	vmulps	%xmm0, %xmm14,%xmm14
+
+#endif
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm7 , 4 * SIZE(CO1)
+	vmovups	%xmm10, 8 * SIZE(CO1)
+	vmovups	%xmm13,12 * SIZE(CO1)
+
+	vmovups	%xmm5 ,  	(CO1, LDC)
+	vmovups	%xmm8 , 4 * SIZE(CO1, LDC)
+	vmovups	%xmm11, 8 * SIZE(CO1, LDC)
+	vmovups	%xmm14,12 * SIZE(CO1, LDC)
+
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO
+        salq    $4, %rax                        // rax = rax * 16 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $16, KK				
+#endif
+
+	addq	$16 * SIZE, CO1		# coffset += 16
+	decq	I			# i --
+	jg	.L2_11
+	ALIGN_4	
+
+/**************************************************************************
+* Rest of M 
+***************************************************************************/
+.L2_20:
+	// Test rest of M
+
+	testq	$15, M
+	jz	.L2_60		// to next 3 lines of N
+
+	testq	$8, M		
+	jz	.L2_21pre
+	ALIGN_4
+
+/**************************************************************************/
+
+.L2_20_1:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO
+        salq    $3, %rax                        // rax = rax * 8 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $8, %rax        // number of values in A
+#else
+        addq    $2, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$-8, %rax
+	je	.L2_20_6
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_20_2:
+
+	prefetcht0	B_PR1(BO,BI, SIZE)
+	KERNEL8x2_1(xxx)
+	KERNEL8x2_2(xxx)
+	KERNEL8x2_3(xxx)
+	KERNEL8x2_4(xxx)
+
+	KERNEL8x2_1(xxx)
+	KERNEL8x2_2(xxx)
+	KERNEL8x2_3(xxx)
+	KERNEL8x2_4(xxx)
+
+	je	.L2_20_6
+
+	prefetcht0	B_PR1(BO,BI, SIZE)
+	KERNEL8x2_1(xxx)
+	KERNEL8x2_2(xxx)
+	KERNEL8x2_3(xxx)
+	KERNEL8x2_4(xxx)
+
+	KERNEL8x2_1(xxx)
+	KERNEL8x2_2(xxx)
+	KERNEL8x2_3(xxx)
+	KERNEL8x2_4(xxx)
+
+	je	.L2_20_6
+
+	jmp	.L2_20_2
+	ALIGN_4
+
+.L2_20_6:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_20_9
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_20_7:
+
+	KERNEL8x2_SUB(xxx)
+	addq	$2, BI
+	addq	$8, %rax
+	jl	.L2_20_7
+	ALIGN_4
+
+
+.L2_20_9:
+
+	vbroadcastss	ALPHA, %xmm0
+
+#ifndef TRMMKERNEL
+
+	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddps 4 * SIZE(CO1),%xmm0, %xmm7,%xmm7
+
+	vfmaddps 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+	vfmaddps 4 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
+
+#else
+	vmulps	%xmm0, %xmm4,%xmm4
+	vmulps	%xmm0, %xmm7,%xmm7
+
+	vmulps	%xmm0, %xmm5,%xmm5
+	vmulps	%xmm0, %xmm8,%xmm8
+
+#endif
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm7 , 4 * SIZE(CO1)
+
+	vmovups	%xmm5 ,  	(CO1, LDC)
+	vmovups	%xmm8 , 4 * SIZE(CO1, LDC)
+
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax 
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO         
+        salq    $3, %rax                        // rax = rax * 8 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif  
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $8, KK
+#endif
+
+	addq	$8 * SIZE, CO1		# coffset += 8
+	ALIGN_4
+	
+
+
+/**************************************************************************/
+
+.L2_21pre:
+
+	testq	$4, M		
+	jz	.L2_30
+	ALIGN_4
+
+.L2_21:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO
+        salq    $2, %rax                        // rax = rax * 4 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $4, %rax        // number of values in A
+#else
+        addq    $2, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$-8, %rax
+	je	.L2_26
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 1 ; number of values
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_22:
+
+	prefetcht0	B_PR1(BO,BI, SIZE)
+	KERNEL4x2_1(xxx)
+	KERNEL4x2_2(xxx)
+	KERNEL4x2_3(xxx)
+	KERNEL4x2_4(xxx)
+
+	KERNEL4x2_1(xxx)
+	KERNEL4x2_2(xxx)
+	KERNEL4x2_3(xxx)
+	KERNEL4x2_4(xxx)
+
+	je	.L2_26
+
+	prefetcht0	B_PR1(BO,BI, SIZE)
+	KERNEL4x2_1(xxx)
+	KERNEL4x2_2(xxx)
+	KERNEL4x2_3(xxx)
+	KERNEL4x2_4(xxx)
+
+	KERNEL4x2_1(xxx)
+	KERNEL4x2_2(xxx)
+	KERNEL4x2_3(xxx)
+	KERNEL4x2_4(xxx)
+
+	je	.L2_26
+
+	jmp	.L2_22
+	ALIGN_4
+
+.L2_26:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_29
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_27:
+
+	KERNEL4x2_SUB(xxx)
+	addq	$2, BI
+	addq	$4, %rax
+	jl	.L2_27
+	ALIGN_4
+
+
+.L2_29:
+
+	vbroadcastss	ALPHA, %xmm0
+
+#ifndef TRMMKERNEL
+
+	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddps 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+
+#else
+	vmulps	%xmm0, %xmm4,%xmm4
+	vmulps	%xmm0, %xmm5,%xmm5
+
+#endif
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm5 ,  	(CO1, LDC)
+
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax 
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO         
+        salq    $2, %rax                        // rax = rax * 4 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif  
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $4, KK
+#endif
+
+	addq	$4 * SIZE, CO1		# coffset += 4
+	ALIGN_4
+	
+
+.L2_30:
+	testq	$2, M		
+	jz	.L2_40
+
+	ALIGN_4
+
+.L2_31:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO
+        salq    $1, %rax                        // rax = rax * 2 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $2, %rax        // number of values in AO
+#else
+        addq    $2, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$-8, %rax
+	je	.L2_36
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_32:
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL2x2_1(xxx)
+	KERNEL2x2_2(xxx)
+	KERNEL2x2_3(xxx)
+	KERNEL2x2_4(xxx)
+
+	KERNEL2x2_1(xxx)
+	KERNEL2x2_2(xxx)
+	KERNEL2x2_3(xxx)
+	KERNEL2x2_4(xxx)
+
+	je	.L2_36
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL2x2_1(xxx)
+	KERNEL2x2_2(xxx)
+	KERNEL2x2_3(xxx)
+	KERNEL2x2_4(xxx)
+
+	KERNEL2x2_1(xxx)
+	KERNEL2x2_2(xxx)
+	KERNEL2x2_3(xxx)
+	KERNEL2x2_4(xxx)
+
+	je	.L2_36
+
+	jmp	.L2_32
+	ALIGN_4
+
+.L2_36:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_39
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+	
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_37:
+
+	KERNEL2x2_SUB(xxx)
+	addq	$2, BI
+	addq	$2, %rax
+	jl	.L2_37
+	ALIGN_4
+
+
+.L2_39:
+
+	vmovss	ALPHA, %xmm0
+
+#ifndef TRMMKERNEL
+
+	vfmaddss 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddss 1 * SIZE(CO1),%xmm0, %xmm8,%xmm8
+	vfmaddss 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+	vfmaddss 1 * SIZE(CO1, LDC),%xmm0, %xmm10,%xmm10
+
+#else
+	vmulss	%xmm0, %xmm4,%xmm4
+	vmulss	%xmm0, %xmm8,%xmm8
+	vmulss	%xmm0, %xmm5,%xmm5
+	vmulss	%xmm0, %xmm10,%xmm10
+
+#endif
+
+	vmovss	%xmm4 ,  	(CO1)
+	vmovss	%xmm8 , 1 * SIZE(CO1)
+	vmovss	%xmm5 ,  	(CO1, LDC)
+	vmovss	%xmm10, 1 * SIZE(CO1, LDC)
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax 
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO         
+        salq    $1, %rax                        // rax = rax * 2 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif  
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $2, KK
+#endif
+
+	addq	$2 * SIZE, CO1		# coffset += 2
+	ALIGN_4
+
+.L2_40:
+	testq	$1, M		
+	jz	.L2_60		// to next 2 lines of N
+
+	ALIGN_4
+
+.L2_41:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $1, %rax        // number of values in AO
+#else
+        addq    $2, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+	andq	$-8, %rax
+	je	.L2_46
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_42:
+
+	KERNEL1x2_1(xxx)
+	KERNEL1x2_2(xxx)
+	KERNEL1x2_3(xxx)
+	KERNEL1x2_4(xxx)
+
+	KERNEL1x2_1(xxx)
+	KERNEL1x2_2(xxx)
+	KERNEL1x2_3(xxx)
+	KERNEL1x2_4(xxx)
+
+	je	.L2_46
+
+	KERNEL1x2_1(xxx)
+	KERNEL1x2_2(xxx)
+	KERNEL1x2_3(xxx)
+	KERNEL1x2_4(xxx)
+
+	KERNEL1x2_1(xxx)
+	KERNEL1x2_2(xxx)
+	KERNEL1x2_3(xxx)
+	KERNEL1x2_4(xxx)
+
+	je	.L2_46
+
+	jmp	.L2_42
+	ALIGN_4
+
+.L2_46:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_49
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_47:
+
+	KERNEL1x2_SUB(xxx)
+	addq	$2, BI
+	addq	$1, %rax
+	jl	.L2_47
+	ALIGN_4
+
+
+.L2_49:
+
+	vmovss	ALPHA, %xmm0
+
+#ifndef TRMMKERNEL
+
+	vfmaddss 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddss 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+
+#else
+	vmulss	%xmm0, %xmm4,%xmm4
+	vmulss	%xmm0, %xmm5,%xmm5
+
+#endif
+
+	vmovss	%xmm4 ,  	(CO1)
+	vmovss	%xmm5 ,  	(CO1, LDC)
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax 
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO         
+        leaq    (AO, %rax, SIZE), AO
+#endif  
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $1, KK
+#endif
+
+	addq	$1 * SIZE, CO1		# coffset += 1
+	ALIGN_4
+	
+
+
+
+	
+.L2_60:
+#if defined(TRMMKERNEL) && !defined(LEFT)
+        addq    $2, KK
+#endif
+
+	decq	J			// j --
+	jg	.L2_01			// next 2 lines of N
+
+
+
+.L1_0:
+
+/************************************************************************************************
+* Loop for Nmod6 % 2 > 0
+*************************************************************************************************/
+
+	movq	Nmod6, J		
+	andq	$1, J			// j % 2
+	je	.L999
+	ALIGN_4
+
+.L1_01:
+	// copy to sub buffer
+	movq	B, BO1
+	leaq    BUFFER1, BO		// first buffer to BO
+	movq	K, %rax
+	ALIGN_4
+
+.L1_02b:
+
+	vmovss	(BO1), %xmm0
+	vmovss	%xmm0,       (BO)
+	addq	$1*SIZE,BO1
+	addq	$1*SIZE,BO
+	decq	%rax
+	jnz	.L1_02b
+
+.L1_02c:
+
+	movq	BO1, B			// next offset of B
+
+.L1_10:
+	movq	C, CO1
+	leaq	(C, LDC, 1), C		// c += 1 * ldc
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        movq    OFFSET, %rax
+        movq    %rax, KK
+#endif
+	
+	movq	A, AO		 	// aoffset = a
+	addq	$32 * SIZE, AO
+
+	movq	M,  I
+	sarq	$4, I			// i = (m >> 4)
+	je	.L1_20
+
+	ALIGN_4
+
+.L1_11:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BO, BI, SIZE), BO
+        salq    $4, %rax                        // rax = rax * 16 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $16, %rax	// number of values in AO
+#else
+        addq    $1, %rax	// number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+	andq	$-8, %rax			//  K = K - ( K % 8 )
+	je	.L1_16
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$4, %rax			// rax = rax * 16 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_12:
+
+	prefetcht0	B_PR1(BO,BI, SIZE)
+	KERNEL16x1_1(xxx)
+	KERNEL16x1_2(xxx)
+	KERNEL16x1_3(xxx)
+	KERNEL16x1_4(xxx)
+
+	KERNEL16x1_1(xxx)
+	KERNEL16x1_2(xxx)
+	KERNEL16x1_3(xxx)
+	KERNEL16x1_4(xxx)
+
+	je	.L1_16
+
+	KERNEL16x1_1(xxx)
+	KERNEL16x1_2(xxx)
+	KERNEL16x1_3(xxx)
+	KERNEL16x1_4(xxx)
+
+	KERNEL16x1_1(xxx)
+	KERNEL16x1_2(xxx)
+	KERNEL16x1_3(xxx)
+	KERNEL16x1_4(xxx)
+
+	je	.L1_16
+
+	jmp	.L1_12
+	ALIGN_4
+
+.L1_16:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_19
+
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$4, %rax			// rax = rax * 16 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_17:
+
+	KERNEL16x1_SUB(xxx)
+	addq	$1, BI
+	addq	$16, %rax
+	jl	.L1_17
+	ALIGN_4
+
+
+.L1_19:
+
+	vbroadcastss	ALPHA, %xmm0
+
+#ifndef TRMMKERNEL
+
+	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddps 4 * SIZE(CO1),%xmm0, %xmm7,%xmm7
+	vfmaddps 8 * SIZE(CO1),%xmm0, %xmm10,%xmm10
+	vfmaddps 12 * SIZE(CO1),%xmm0, %xmm13,%xmm13
+
+#else
+	vmulps	%xmm0, %xmm4,%xmm4
+	vmulps	%xmm0, %xmm7,%xmm7
+	vmulps	%xmm0, %xmm10,%xmm10
+	vmulps	%xmm0, %xmm13,%xmm13
+
+#endif
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm7 , 4 * SIZE(CO1)
+	vmovups	%xmm10, 8 * SIZE(CO1)
+	vmovups	%xmm13,12 * SIZE(CO1)
+
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BO, BI, SIZE), BO
+        salq    $4, %rax                        // rax = rax * 16 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $16, KK				
+#endif
+
+	addq	$16 * SIZE, CO1		# coffset += 16
+	decq	I			# i --
+	jg	.L1_11
+	ALIGN_4	
+
+/**************************************************************************
+* Rest of M 
+***************************************************************************/
+.L1_20:
+	// Test rest of M
+
+	testq	$15, M
+	jz	.L999
+
+	testq	$8, M		
+	jz	.L1_21pre
+	ALIGN_4
+
+/**************************************************************************/
+
+.L1_20_1:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BO, BI, SIZE), BO
+        salq    $3, %rax                        // rax = rax * 8 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $8, %rax        // number of values in A
+#else
+        addq    $1, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$-8, %rax
+	je	.L1_20_6
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_20_2:
+
+	prefetcht0	B_PR1(BO,BI, SIZE)
+	KERNEL8x1_1(xxx)
+	KERNEL8x1_2(xxx)
+	KERNEL8x1_3(xxx)
+	KERNEL8x1_4(xxx)
+
+	KERNEL8x1_1(xxx)
+	KERNEL8x1_2(xxx)
+	KERNEL8x1_3(xxx)
+	KERNEL8x1_4(xxx)
+
+	je	.L1_20_6
+
+	KERNEL8x1_1(xxx)
+	KERNEL8x1_2(xxx)
+	KERNEL8x1_3(xxx)
+	KERNEL8x1_4(xxx)
+
+	KERNEL8x1_1(xxx)
+	KERNEL8x1_2(xxx)
+	KERNEL8x1_3(xxx)
+	KERNEL8x1_4(xxx)
+
+	je	.L1_20_6
+
+	jmp	.L1_20_2
+	ALIGN_4
+
+.L1_20_6:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_20_9
+
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_20_7:
+
+	KERNEL8x1_SUB(xxx)
+	addq	$1, BI
+	addq	$8, %rax
+	jl	.L1_20_7
+	ALIGN_4
+
+
+.L1_20_9:
+
+	vbroadcastss	ALPHA, %xmm0
+
+#ifndef TRMMKERNEL
+
+	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddps 4 * SIZE(CO1),%xmm0, %xmm7,%xmm7
+
+#else
+	vmulps	%xmm0, %xmm4,%xmm4
+	vmulps	%xmm0, %xmm7,%xmm7
+
+#endif
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm7 , 4 * SIZE(CO1)
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax 
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BO, BI, SIZE), BO         
+        salq    $3, %rax                        // rax = rax * 8 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif  
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $8, KK
+#endif
+
+	addq	$8 * SIZE, CO1		# coffset += 8
+	ALIGN_4
+	
+
+
+/**************************************************************************/
+
+.L1_21pre:
+
+	testq	$4, M		
+	jz	.L1_30
+	ALIGN_4
+
+.L1_21:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BO, BI, SIZE), BO
+        salq    $2, %rax                        // rax = rax * 4 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $4, %rax        // number of values in A
+#else
+        addq    $1, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$-8, %rax
+	je	.L1_26
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_22:
+
+	prefetcht0	B_PR1(BO,BI, SIZE)
+	KERNEL4x1_1(xxx)
+	KERNEL4x1_2(xxx)
+	KERNEL4x1_3(xxx)
+	KERNEL4x1_4(xxx)
+
+	KERNEL4x1_1(xxx)
+	KERNEL4x1_2(xxx)
+	KERNEL4x1_3(xxx)
+	KERNEL4x1_4(xxx)
+
+	je	.L1_26
+
+	KERNEL4x1_1(xxx)
+	KERNEL4x1_2(xxx)
+	KERNEL4x1_3(xxx)
+	KERNEL4x1_4(xxx)
+
+	KERNEL4x1_1(xxx)
+	KERNEL4x1_2(xxx)
+	KERNEL4x1_3(xxx)
+	KERNEL4x1_4(xxx)
+
+	je	.L1_26
+
+	jmp	.L1_22
+	ALIGN_4
+
+.L1_26:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_29
+
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_27:
+
+	KERNEL4x1_SUB(xxx)
+	addq	$1, BI
+	addq	$4, %rax
+	jl	.L1_27
+	ALIGN_4
+
+
+.L1_29:
+
+	vbroadcastss	ALPHA, %xmm0
+
+#ifndef TRMMKERNEL
+
+	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
+
+#else
+	vmulps	%xmm0, %xmm4,%xmm4
+
+#endif
+
+	vmovups	%xmm4 ,  	(CO1)
+
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax 
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BO, BI, SIZE), BO         
+        salq    $2, %rax                        // rax = rax * 4 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif  
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $4, KK
+#endif
+
+	addq	$4 * SIZE, CO1		# coffset += 4
+	ALIGN_4
+	
+
+.L1_30:
+	testq	$2, M		
+	jz	.L1_40
+
+	ALIGN_4
+
+.L1_31:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BO, BI, SIZE), BO
+        salq    $1, %rax                        // rax = rax * 2 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $2, %rax        // number of values in AO
+#else
+        addq    $1, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$-8, %rax
+	je	.L1_36
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_32:
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL2x1_1(xxx)
+	KERNEL2x1_2(xxx)
+	KERNEL2x1_3(xxx)
+	KERNEL2x1_4(xxx)
+
+	KERNEL2x1_1(xxx)
+	KERNEL2x1_2(xxx)
+	KERNEL2x1_3(xxx)
+	KERNEL2x1_4(xxx)
+
+	je	.L1_36
+
+	KERNEL2x1_1(xxx)
+	KERNEL2x1_2(xxx)
+	KERNEL2x1_3(xxx)
+	KERNEL2x1_4(xxx)
+
+	KERNEL2x1_1(xxx)
+	KERNEL2x1_2(xxx)
+	KERNEL2x1_3(xxx)
+	KERNEL2x1_4(xxx)
+
+	je	.L1_36
+
+	jmp	.L1_32
+	ALIGN_4
+
+.L1_36:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_39
+
+	movq    %rax, BI                        //  Index for BO
+	
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_37:
+
+	KERNEL2x1_SUB(xxx)
+	addq	$1, BI
+	addq	$2, %rax
+	jl	.L1_37
+	ALIGN_4
+
+
+.L1_39:
+
+	vmovss	ALPHA, %xmm0
+
+#ifndef TRMMKERNEL
+
+	vfmaddss 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddss 1 * SIZE(CO1),%xmm0, %xmm8,%xmm8
+
+#else
+	vmulss	%xmm0, %xmm4,%xmm4
+	vmulss	%xmm0, %xmm8,%xmm8
+
+#endif
+
+	vmovss	%xmm4 ,  	(CO1)
+	vmovss	%xmm8 , 1 * SIZE(CO1)
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax 
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BO, BI, SIZE), BO         
+        salq    $1, %rax                        // rax = rax * 2 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif  
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $2, KK
+#endif
+
+	addq	$2 * SIZE, CO1		# coffset += 2
+	ALIGN_4
+
+.L1_40:
+	testq	$1, M		
+	jz	.L999
+
+	ALIGN_4
+
+.L1_41:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BO, BI, SIZE), BO
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $1, %rax        // number of values in AO
+#else
+        addq    $1, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+	andq	$-8, %rax
+	je	.L1_46
+	movq    %rax, BI                        //  Index for BO
+
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_42:
+
+	KERNEL1x1_1(xxx)
+	KERNEL1x1_2(xxx)
+	KERNEL1x1_3(xxx)
+	KERNEL1x1_4(xxx)
+
+	KERNEL1x1_1(xxx)
+	KERNEL1x1_2(xxx)
+	KERNEL1x1_3(xxx)
+	KERNEL1x1_4(xxx)
+
+	je	.L1_46
+
+	KERNEL1x1_1(xxx)
+	KERNEL1x1_2(xxx)
+	KERNEL1x1_3(xxx)
+	KERNEL1x1_4(xxx)
+
+	KERNEL1x1_1(xxx)
+	KERNEL1x1_2(xxx)
+	KERNEL1x1_3(xxx)
+	KERNEL1x1_4(xxx)
+
+	je	.L1_46
+
+	jmp	.L1_42
+	ALIGN_4
+
+.L1_46:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_49
+
+	movq    %rax, BI                        //  Index for BO
+
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_47:
+
+	KERNEL1x1_SUB(xxx)
+	addq	$1, BI
+	addq	$1, %rax
+	jl	.L1_47
+	ALIGN_4
+
+
+.L1_49:
+
+	vmovss	ALPHA, %xmm0
+
+#ifndef TRMMKERNEL
+
+	vfmaddss 	 (CO1),%xmm0, %xmm4,%xmm4
+
+#else
+	vmulss	%xmm0, %xmm4,%xmm4
+
+#endif
+
+	vmovss	%xmm4 ,  	(CO1)
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax 
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BO, BI, SIZE), BO         
+        leaq    (AO, %rax, SIZE), AO
+#endif  
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $1, KK
+#endif
+
+	addq	$1 * SIZE, CO1		# coffset += 1
+	ALIGN_4
+	
+
+.L999:
+	movq   		SP, %rsp
+	movq	   (%rsp), %rbx
+	movq	  8(%rsp), %rbp
+	movq	 16(%rsp), %r12
+	movq	 24(%rsp), %r13
+	movq	 32(%rsp), %r14
+	movq	 40(%rsp), %r15
+
+#ifdef WINDOWS_ABI
+	movq	 48(%rsp), %rdi
+	movq	 56(%rsp), %rsi
+	movups	 64(%rsp), %xmm6
+	movups	 80(%rsp), %xmm7
+	movups	 96(%rsp), %xmm8
+	movups	112(%rsp), %xmm9
+	movups	128(%rsp), %xmm10
+	movups	144(%rsp), %xmm11
+	movups	160(%rsp), %xmm12
+	movups	176(%rsp), %xmm13
+	movups	192(%rsp), %xmm14
+	movups	208(%rsp), %xmm15
+#endif
+
+	addq	$STACKSIZE, %rsp
+	ret
+
+	EPILOGUE
+
+
+
+
+
+#endif
diff --git a/kernel/x86_64/sgemm_kernel_16x2_piledriver.S b/kernel/x86_64/sgemm_kernel_16x2_piledriver.S
index 7c42f1e12..35b01de07 100644
--- a/kernel/x86_64/sgemm_kernel_16x2_piledriver.S
+++ b/kernel/x86_64/sgemm_kernel_16x2_piledriver.S
@@ -1,5258 +1,5258 @@
-/***************************************************************************
-Copyright (c) 2013, The OpenBLAS Project
-All rights reserved.
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in
-the documentation and/or other materials provided with the
-distribution.
-3. Neither the name of the OpenBLAS project nor the names of
-its contributors may be used to endorse or promote products
-derived from this software without specific prior written permission.
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
-USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*****************************************************************************/
-
-/*********************************************************************
-*
-* 2013/10/18 Saar
-*        BLASTEST               : OK
-*        CTEST                  : OK
-*        TEST                   : OK
-*
-*
-* 2013/10/29 Saar
-*
-* Parameter:
-*       UNROLL_M        16
-*       UNROLL_N        2
-*       SGEMM_P         768
-*       SGEMM_Q         192
-*       SGEMM_R         12288
-*       A_PR1           384
-*       B_PR1           192
-*
-* Performance at m x n on AMD 8320 (ACML-Version: 5.3.1):
-* 
-* 6144x6144    168.2    GFLOPS with 8 threads on 4 modules (ACML: 158.0 ) (BULLDOZER: 167.4 )
-* 6144x6144    162.7    GFLOPS with 4 threads on 4 modules (ACML: 157.6 ) (BULLDOZER: 159.0 )
-* 6144x6144     82.0    GFLOPS with 2 threads on 2 modules (ACML:  81.4 ) (BULLDOZER:  80.3 )
-* 6144x6144     41.3    GFLOPS with 1 threads on 1 modules (ACML:  41.1 ) (BULLDOZER:  40.4 )
-*
-* Performance at m x n on AMD 6380  (ACML-Version: 5.3.1):
-* 
-* 12288x12288  469.5    GFLOPS with 32 threads on 16 modules (ACML: 375.3 ) (BULLDOZER: 445.5 )
-* 12288x12288  442.9    GFLOPS with 16 threads on 16 modules (ACML: 378.5 ) (BULLDOZER: 416.3 )
-* 12288x12288  265.1    GFLOPS with  8 threads on  8 modules (ACML: 218.5 ) (BULLDOZER: 261.5 )
-* 6144x6144    139.7    GFLOPS with  4 threads on  4 modules (ACML: 116.0 ) (BULLDOZER: 137.7 )
-* 6144x6144     70.9    GFLOPS with  2 threads on  2 modules (ACML:  67.4 ) (BULLDOZER:  69.5 )
-* 6144x6144     35.6    GFLOPS with  1 threads on  1 modules (ACML:  36.1 ) (BULLDOZER:  35.1 )
-*
-*********************************************************************/
-
-
-
-#define ASSEMBLER
-#include "common.h"
- 
-#define OLD_M	%rdi
-#define OLD_N	%rsi
-#define M	%r13
-#define J	%r14
-#define OLD_K	%rdx
-
-#define A	%rcx
-#define B	%r8
-#define C	%r9
-#define LDC	%r10
-	
-#define I	%r11
-#define AO	%rdi
-#define BO	%rsi
-#define	CO1	%r15
-#define K	%r12
-#define BI	%rbp
-#define	SP	%rbx
-
-#define BO1	%rdi
-#define BO2	%r15
-
-#ifndef WINDOWS_ABI
-
-#define STACKSIZE 96
-
-#else
-
-#define STACKSIZE 256
-
-#define OLD_A		40 + STACKSIZE(%rsp)
-#define OLD_B		48 + STACKSIZE(%rsp)
-#define OLD_C		56 + STACKSIZE(%rsp)
-#define OLD_LDC		64 + STACKSIZE(%rsp)
-#define OLD_OFFSET	72 + STACKSIZE(%rsp)
-
-#endif
-
-#define L_BUFFER_SIZE 8192
-#define LB2_OFFSET    4096
-
-#define Ndiv6	 24(%rsp)
-#define Nmod6	 32(%rsp)
-#define N	 40(%rsp)
-#define ALPHA	 48(%rsp)
-#define OFFSET	 56(%rsp)
-#define KK	 64(%rsp)
-#define KKK	 72(%rsp)
-#define BUFFER1	           128(%rsp)
-#define BUFFER2	LB2_OFFSET+128(%rsp)
-
-#if defined(OS_WINDOWS)
-#if   L_BUFFER_SIZE > 16384
-#define STACK_TOUCH \
-        movl    $0,  4096 * 4(%rsp);\
-        movl    $0,  4096 * 3(%rsp);\
-        movl    $0,  4096 * 2(%rsp);\
-        movl    $0,  4096 * 1(%rsp);
-#elif L_BUFFER_SIZE > 12288
-#define STACK_TOUCH \
-        movl    $0,  4096 * 3(%rsp);\
-        movl    $0,  4096 * 2(%rsp);\
-        movl    $0,  4096 * 1(%rsp);
-#elif L_BUFFER_SIZE > 8192
-#define STACK_TOUCH \
-        movl    $0,  4096 * 2(%rsp);\
-        movl    $0,  4096 * 1(%rsp);
-#elif L_BUFFER_SIZE > 4096
-#define STACK_TOUCH \
-        movl    $0,  4096 * 1(%rsp);
-#else
-#define STACK_TOUCH
-#endif
-#else
-#define STACK_TOUCH
-#endif
-
-
-
-#define	A_PR1	384
-#define	B_PR1	192
-
-/*******************************************************************************************
-* 3 lines of N
-*******************************************************************************************/
-
-#define KERNEL16x3_1(xx) \
-	vbroadcastss	 -6 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vbroadcastss	 -5 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm3 ;\
-	nop						;\
-	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	prefetcht0	A_PR1(AO,%rax,SIZE)	;\
-	vfmaddps  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
-	vmovups 	-24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
-	vfmaddps  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
-	vfmaddps  	%xmm12,%xmm3,%xmm0,%xmm12 ;\
-	vmovups 	-20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
-	vfmaddps  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
-	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm15,%xmm3,%xmm0,%xmm15 ;\
-
-#define KERNEL16x3_2(xx) \
-	vmovups 	-16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vbroadcastss	 -1 * SIZE(BO, BI, SIZE), %xmm3 ;\
-	nop						;\
-	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-	vmovups 	-12 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	prefetcht0	A_PR1+64(AO,%rax,SIZE)	;\
-	vfmaddps  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
-	vmovups 	 -8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
-	vfmaddps  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
-	vfmaddps  	%xmm12,%xmm3,%xmm0,%xmm12 ;\
-	vmovups 	 -4 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
-	vfmaddps  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
-	vbroadcastss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vbroadcastss	  1 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm15,%xmm3,%xmm0,%xmm15 ;\
-
-#define KERNEL16x3_3(xx) \
-	vmovups 	  0 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vbroadcastss	  2 * SIZE(BO, BI, SIZE), %xmm3 ;\
-	nop						;\
-	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-	vmovups 	  4 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	prefetcht0	A_PR1+128(AO,%rax,SIZE)	;\
-	vfmaddps  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
-	vmovups 	  8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
-	vfmaddps  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
-	vfmaddps  	%xmm12,%xmm3,%xmm0,%xmm12 ;\
-	vmovups 	 12 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
-	vfmaddps  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
-	vbroadcastss	  3 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vbroadcastss	  4 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm15,%xmm3,%xmm0,%xmm15 ;\
-
-#define KERNEL16x3_4(xx) \
-	vmovups 	 16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vbroadcastss	  5 * SIZE(BO, BI, SIZE), %xmm3 ;\
-	nop						;\
-	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-	vmovups 	 20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	prefetcht0	A_PR1+192(AO,%rax,SIZE)	;\
-	vfmaddps  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
-	vmovups 	 24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
-	vfmaddps  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
-	addq	$12, BI				  ;\
-	vfmaddps  	%xmm12,%xmm3,%xmm0,%xmm12 ;\
-	vmovups 	 28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
-	vfmaddps  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
-	addq	$64, %rax 			  ;\
-	vfmaddps  	%xmm15,%xmm3,%xmm0,%xmm15 ;\
-
-#define KERNEL16x3_SUB(xx) \
-	vbroadcastss	 -6 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vbroadcastss	 -5 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm3 ;\
-	nop						;\
-	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	vfmaddps  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
-	vmovups 	-24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
-	vfmaddps  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
-	vfmaddps  	%xmm12,%xmm3,%xmm0,%xmm12 ;\
-	vmovups 	-20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
-	vfmaddps  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
-	vfmaddps  	%xmm15,%xmm3,%xmm0,%xmm15 ;\
-
-
-/*******************************************************************************************/
-
-#define KERNEL8x3_1(xx) \
-	prefetcht0	A_PR1(AO,%rax,SIZE)	;\
-	vbroadcastss	 -6 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vbroadcastss	 -5 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm3 ;\
-	nop						;\
-	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	vfmaddps  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
-
-#define KERNEL8x3_2(xx) \
-	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vbroadcastss	 -1 * SIZE(BO, BI, SIZE), %xmm3 ;\
-	nop						;\
-	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-	vmovups 	-20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	vfmaddps  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
-
-#define KERNEL8x3_3(xx) \
-	prefetcht0	A_PR1+64(AO,%rax,SIZE)	;\
-	vbroadcastss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vbroadcastss	  1 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vbroadcastss	  2 * SIZE(BO, BI, SIZE), %xmm3 ;\
-	nop						;\
-	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-	vmovups 	-12 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	vfmaddps  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
-
-#define KERNEL8x3_4(xx) \
-	vbroadcastss	  3 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	 -8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vbroadcastss	  4 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vbroadcastss	  5 * SIZE(BO, BI, SIZE), %xmm3 ;\
-	nop						;\
-	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-	vmovups 	 -4 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	vfmaddps  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
-	addq	$12, BI				  ;\
-	addq	$32, %rax 			  ;\
-
-#define KERNEL8x3_SUB(xx) \
-	vbroadcastss	 -6 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vbroadcastss	 -5 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm3 ;\
-	nop						;\
-	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	vfmaddps  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
-
-
-/*******************************************************************************************/
-
-#define KERNEL4x3_1(xx) \
-	prefetcht0	A_PR1(AO,%rax,SIZE)	;\
-	vbroadcastss	 -6 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vbroadcastss	 -5 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm3 ;\
-	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-
-#define KERNEL4x3_2(xx) \
-	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vbroadcastss	 -1 * SIZE(BO, BI, SIZE), %xmm3 ;\
-	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-
-#define KERNEL4x3_3(xx) \
-	vbroadcastss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vbroadcastss	  1 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vbroadcastss	  2 * SIZE(BO, BI, SIZE), %xmm3 ;\
-	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-
-#define KERNEL4x3_4(xx) \
-	vbroadcastss	  3 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vbroadcastss	  4 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vbroadcastss	  5 * SIZE(BO, BI, SIZE), %xmm3 ;\
-	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-	addq	$12, BI				  ;\
-	addq	$16, %rax 			  ;\
-
-#define KERNEL4x3_SUB(xx) \
-	vbroadcastss	 -6 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vbroadcastss	 -5 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm3 ;\
-	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-
-/*******************************************************************************************/
-
-#define KERNEL2x3_1(xx) \
-	vmovss	 -6 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovss	 -5 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovss	 -4 * SIZE(BO, BI, SIZE), %xmm3 ;\
-	vfmaddss  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-	vmovss 	-31 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
-	vfmaddss  	%xmm10,%xmm2,%xmm0,%xmm10 ;\
-	vfmaddss  	%xmm12,%xmm3,%xmm0,%xmm12 ;\
-
-#define KERNEL2x3_2(xx) \
-	vmovss	 -3 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-30 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovss	 -2 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovss	 -1 * SIZE(BO, BI, SIZE), %xmm3 ;\
-	vfmaddss  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-	vmovss 	-29 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
-	vfmaddss  	%xmm10,%xmm2,%xmm0,%xmm10 ;\
-	vfmaddss  	%xmm12,%xmm3,%xmm0,%xmm12 ;\
-
-#define KERNEL2x3_3(xx) \
-	vmovss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovss	  1 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovss	  2 * SIZE(BO, BI, SIZE), %xmm3 ;\
-	vfmaddss  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-	vmovss 	-27 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
-	vfmaddss  	%xmm10,%xmm2,%xmm0,%xmm10 ;\
-	vfmaddss  	%xmm12,%xmm3,%xmm0,%xmm12 ;\
-
-#define KERNEL2x3_4(xx) \
-	vmovss	  3 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-26 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovss	  4 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovss	  5 * SIZE(BO, BI, SIZE), %xmm3 ;\
-	vfmaddss  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-	vmovss 	-25 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
-	vfmaddss  	%xmm10,%xmm2,%xmm0,%xmm10 ;\
-	vfmaddss  	%xmm12,%xmm3,%xmm0,%xmm12 ;\
-	addq	$12, BI				  ;\
-	addq	$8, %rax 			  ;\
-
-#define KERNEL2x3_SUB(xx) \
-	vmovss	 -6 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovss	 -5 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovss	 -4 * SIZE(BO, BI, SIZE), %xmm3 ;\
-	vfmaddss  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-	vmovss 	-31 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
-	vfmaddss  	%xmm10,%xmm2,%xmm0,%xmm10 ;\
-	vfmaddss  	%xmm12,%xmm3,%xmm0,%xmm12 ;\
-
-/*******************************************************************************************/
-
-#define KERNEL1x3_1(xx) \
-	vmovss	 -6 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovss	 -5 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovss	 -4 * SIZE(BO, BI, SIZE), %xmm3 ;\
-	vfmaddss  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-
-#define KERNEL1x3_2(xx) \
-	vmovss	 -3 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-31 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovss	 -2 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovss	 -1 * SIZE(BO, BI, SIZE), %xmm3 ;\
-	vfmaddss  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-
-#define KERNEL1x3_3(xx) \
-	vmovss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-30 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovss	  1 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovss	  2 * SIZE(BO, BI, SIZE), %xmm3 ;\
-	vfmaddss  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-
-#define KERNEL1x3_4(xx) \
-	vmovss	  3 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-29 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovss	  4 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovss	  5 * SIZE(BO, BI, SIZE), %xmm3 ;\
-	vfmaddss  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-	addq	$12, BI				  ;\
-	addq	$4, %rax 			  ;\
-
-#define KERNEL1x3_SUB(xx) \
-	vmovss	 -6 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovss	 -5 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovss	 -4 * SIZE(BO, BI, SIZE), %xmm3 ;\
-	vfmaddss  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
-
-/*******************************************************************************************/
-
-/*******************************************************************************************
-* 2 lines of N
-*******************************************************************************************/
-
-#define KERNEL16x2_1(xx) \
-	prefetcht0	A_PR1(AO,%rax,SIZE)	;\
-	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	vmovups 	-24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
-	vfmaddps  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
-	vmovups 	-20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
-	vfmaddps  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
-
-#define KERNEL16x2_2(xx) \
-	prefetcht0	A_PR1+64(AO,%rax,SIZE)	;\
-	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vbroadcastss	 -1 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovups 	-12 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	vmovups 	 -8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
-	vfmaddps  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
-	vmovups 	 -4 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
-	vfmaddps  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
-
-#define KERNEL16x2_3(xx) \
-	prefetcht0	A_PR1+128(AO,%rax,SIZE)	;\
-	vbroadcastss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	  0 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vbroadcastss	  1 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovups 	  4 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	vmovups 	  8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
-	vfmaddps  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
-	vmovups 	 12 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
-	vfmaddps  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
-
-#define KERNEL16x2_4(xx) \
-	prefetcht0	A_PR1+192(AO,%rax,SIZE)	;\
-	vbroadcastss	  2 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	 16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vbroadcastss	  3 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovups 	 20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	vmovups 	 24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
-	vfmaddps  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
-	vmovups 	 28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
-	vfmaddps  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
-	addq	$8, BI				  ;\
-	addq	$64, %rax 			  ;\
-
-#define KERNEL16x2_SUB(xx) \
-	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	vmovups 	-24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
-	vfmaddps  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
-	vmovups 	-20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
-	vfmaddps  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
-
-
-/*******************************************************************************************/
-
-#define KERNEL8x2_1(xx) \
-	prefetcht0	A_PR1(AO,%rax,SIZE)	;\
-	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-
-#define KERNEL8x2_2(xx) \
-	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vbroadcastss	 -1 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovups 	-20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-
-#define KERNEL8x2_3(xx) \
-	prefetcht0	A_PR1+64(AO,%rax,SIZE)	;\
-	vbroadcastss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vbroadcastss	  1 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovups 	-12 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-
-#define KERNEL8x2_4(xx) \
-	vbroadcastss	  2 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	 -8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vbroadcastss	  3 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovups 	 -4 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-	addq	$8, BI				  ;\
-	addq	$32, %rax 			  ;\
-
-#define KERNEL8x2_SUB(xx) \
-	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
-
-
-/*******************************************************************************************/
-
-#define KERNEL4x2_1(xx) \
-	prefetcht0	A_PR1(AO,%rax,SIZE)	;\
-	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-
-#define KERNEL4x2_2(xx) \
-	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vbroadcastss	 -1 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-
-#define KERNEL4x2_3(xx) \
-	vbroadcastss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vbroadcastss	  1 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-
-#define KERNEL4x2_4(xx) \
-	vbroadcastss	  2 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vbroadcastss	  3 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	addq	$8, BI				  ;\
-	addq	$16, %rax 			  ;\
-
-#define KERNEL4x2_SUB(xx) \
-	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-
-/*******************************************************************************************/
-
-#define KERNEL2x2_1(xx) \
-	vmovss	 -4 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovss	 -3 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovss 	-31 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
-	vfmaddss  	%xmm10,%xmm2,%xmm0,%xmm10 ;\
-
-#define KERNEL2x2_2(xx) \
-	vmovss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-30 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovss	 -1 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovss 	-29 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
-	vfmaddss  	%xmm10,%xmm2,%xmm0,%xmm10 ;\
-
-#define KERNEL2x2_3(xx) \
-	vmovss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovss	  1 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovss 	-27 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
-	vfmaddss  	%xmm10,%xmm2,%xmm0,%xmm10 ;\
-
-#define KERNEL2x2_4(xx) \
-	vmovss	  2 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-26 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovss	  3 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovss 	-25 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
-	vfmaddss  	%xmm10,%xmm2,%xmm0,%xmm10 ;\
-	addq	$8, BI				  ;\
-	addq	$8, %rax 			  ;\
-
-#define KERNEL2x2_SUB(xx) \
-	vmovss	 -4 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovss	 -3 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	vmovss 	-31 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
-	vfmaddss  	%xmm10,%xmm2,%xmm0,%xmm10 ;\
-
-/*******************************************************************************************/
-
-#define KERNEL1x2_1(xx) \
-	vmovss	 -4 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovss	 -3 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-
-#define KERNEL1x2_2(xx) \
-	vmovss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-31 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovss	 -1 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-
-#define KERNEL1x2_3(xx) \
-	vmovss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-30 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovss	  1 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-
-#define KERNEL1x2_4(xx) \
-	vmovss	  2 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-29 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovss	  3 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-	addq	$8, BI				  ;\
-	addq	$4, %rax 			  ;\
-
-#define KERNEL1x2_SUB(xx) \
-	vmovss	 -4 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovss	 -3 * SIZE(BO, BI, SIZE), %xmm2 ;\
-	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
-
-/*******************************************************************************************/
-
-/*******************************************************************************************
-* 1 line of N
-*******************************************************************************************/
-
-#define KERNEL16x1_1(xx) \
-	prefetcht0	A_PR1(AO,%rax,SIZE)	;\
-	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vmovups 	-24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
-	vmovups 	-20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
-
-#define KERNEL16x1_2(xx) \
-	prefetcht0	A_PR1+64(AO,%rax,SIZE)	;\
-	vbroadcastss	 -1 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovups 	-12 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vmovups 	 -8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
-	vmovups 	 -4 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
-
-#define KERNEL16x1_3(xx) \
-	prefetcht0	A_PR1+128(AO,%rax,SIZE)	;\
-	vbroadcastss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	  0 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovups 	  4 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vmovups 	  8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
-	vmovups 	 12 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
-
-#define KERNEL16x1_4(xx) \
-	prefetcht0	A_PR1+192(AO,%rax,SIZE)	;\
-	vbroadcastss	  1 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	 16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovups 	 20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vmovups 	 24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
-	vmovups 	 28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
-	addq	$4, BI				  ;\
-	addq	$64, %rax 			  ;\
-
-#define KERNEL16x1_SUB(xx) \
-	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	vmovups 	-24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
-	vmovups 	-20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
-
-
-/*******************************************************************************************/
-
-#define KERNEL8x1_1(xx) \
-	prefetcht0	A_PR1(AO,%rax,SIZE)	;\
-	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-
-#define KERNEL8x1_2(xx) \
-	vbroadcastss	 -1 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovups 	-20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-
-#define KERNEL8x1_3(xx) \
-	prefetcht0	A_PR1+64(AO,%rax,SIZE)	;\
-	vbroadcastss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovups 	-12 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-
-#define KERNEL8x1_4(xx) \
-	vbroadcastss	  1 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	 -8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovups 	 -4 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-	addq	$4, BI				  ;\
-	addq	$32, %rax 			  ;\
-
-#define KERNEL8x1_SUB(xx) \
-	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
-
-
-/*******************************************************************************************/
-
-#define KERNEL4x1_1(xx) \
-	prefetcht0	A_PR1(AO,%rax,SIZE)	;\
-	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-
-#define KERNEL4x1_2(xx) \
-	vbroadcastss	 -1 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-
-#define KERNEL4x1_3(xx) \
-	vbroadcastss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-
-#define KERNEL4x1_4(xx) \
-	vbroadcastss	  1 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	addq	$4, BI				  ;\
-	addq	$16, %rax 			  ;\
-
-#define KERNEL4x1_SUB(xx) \
-	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-
-/*******************************************************************************************/
-
-#define KERNEL2x1_1(xx) \
-	vmovss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovss 	-31 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
-
-#define KERNEL2x1_2(xx) \
-	vmovss	 -1 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-30 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovss 	-29 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
-
-#define KERNEL2x1_3(xx) \
-	vmovss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovss 	-27 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
-
-#define KERNEL2x1_4(xx) \
-	vmovss	  1 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-26 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovss 	-25 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
-	addq	$4, BI				  ;\
-	addq	$8, %rax 			  ;\
-
-#define KERNEL2x1_SUB(xx) \
-	vmovss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	vmovss 	-31 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
-
-/*******************************************************************************************/
-
-#define KERNEL1x1_1(xx) \
-	vmovss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-
-#define KERNEL1x1_2(xx) \
-	vmovss	 -1 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-31 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-
-#define KERNEL1x1_3(xx) \
-	vmovss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-30 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-
-#define KERNEL1x1_4(xx) \
-	vmovss	  1 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-29 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-	addq	$4, BI				  ;\
-	addq	$4, %rax 			  ;\
-
-#define KERNEL1x1_SUB(xx) \
-	vmovss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
-	vmovss 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
-	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
-
-/*******************************************************************************************/
-
-#if !defined(TRMMKERNEL)
-
-
-	PROLOGUE
-	PROFCODE
-	
-	subq	$STACKSIZE, %rsp
-	movq	%rbx,   (%rsp)
-	movq	%rbp,  8(%rsp)
-	movq	%r12, 16(%rsp)
-	movq	%r13, 24(%rsp)
-	movq	%r14, 32(%rsp)
-	movq	%r15, 40(%rsp)
-
-	vzeroupper
-
-#ifdef WINDOWS_ABI
-	movq	%rdi,    48(%rsp)
-	movq	%rsi,    56(%rsp)
-	movups	%xmm6,   64(%rsp)
-	movups	%xmm7,   80(%rsp)
-	movups	%xmm8,   96(%rsp)
-	movups	%xmm9,  112(%rsp)
-	movups	%xmm10, 128(%rsp)
-	movups	%xmm11, 144(%rsp)
-	movups	%xmm12, 160(%rsp)
-	movups	%xmm13, 176(%rsp)
-	movups	%xmm14, 192(%rsp)
-	movups	%xmm15, 208(%rsp)
-
-	movq	ARG1,      OLD_M
-	movq	ARG2,      OLD_N
-	movq	ARG3,      OLD_K
-	movq	OLD_A,     A
-	movq	OLD_B,     B
-	movq	OLD_C,     C
-	movq	OLD_LDC,   LDC
-
-	vmovaps	%xmm3, %xmm0
-
-#else
-	movq	STACKSIZE +  8(%rsp), LDC
-
-#endif
-
-	movq    %rsp, SP      # save old stack
-        subq    $128 + L_BUFFER_SIZE, %rsp
-        andq    $-4096, %rsp    # align stack
-
-        STACK_TOUCH
-
-	cmpq	$0, OLD_M
-	je	.L999
-
-	cmpq	$0, OLD_N
-	je	.L999
-
-	cmpq	$0, OLD_K
-	je	.L999
-
-	movq	OLD_M, M
-	movq	OLD_N, N
-	movq	OLD_K, K
-
-	vmovsd	 %xmm0, ALPHA
-
-	salq	$BASE_SHIFT, LDC
-
-	movq    N, %rax
-        xorq    %rdx, %rdx
-        movq    $6,  %rdi
-        divq    %rdi                    //    N / 6
-        movq    %rax, Ndiv6             //    N / 6
-        movq    %rdx, Nmod6             //    N % 6
-
-
-	movq	Ndiv6,  J
-	cmpq	$0, J
-	je	.L2_0
-	ALIGN_4
-
-.L6_01:
-        // copy to sub buffer
-        movq    K, %rax
-        salq    $1,%rax                 // K * 2 ; read 2 values
-        movq    B, BO1
-        leaq    (B,%rax, SIZE), BO2     // next offset to BO2
-        leaq    BUFFER1, BO             // first buffer to BO
-        movq    K, %rax
-	sarq	$3 , %rax		// K / 8
-	jz	.L6_01a_2
-        ALIGN_4
-
-.L6_01a_1:
-
-        prefetcht0 512(BO1)
-        prefetcht0 512(BO2)
-        prefetchw  512(BO)
-
-	vmovsd	0 * SIZE(BO1), %xmm0
-	vmovsd	2 * SIZE(BO1), %xmm2
-	vmovsd	4 * SIZE(BO1), %xmm4
-	vmovsd	6 * SIZE(BO1), %xmm6
-	vmovss  0 * SIZE(BO2), %xmm1
-	vmovss  2 * SIZE(BO2), %xmm3
-	vmovss  4 * SIZE(BO2), %xmm5
-	vmovss  6 * SIZE(BO2), %xmm7
-	vmovsd	%xmm0, 0*SIZE(BO)
-	vmovss	%xmm1, 2*SIZE(BO)
-	vmovsd	%xmm2, 3*SIZE(BO)
-	vmovss	%xmm3, 5*SIZE(BO)
-	vmovsd	%xmm4, 6*SIZE(BO)
-	vmovss	%xmm5, 8*SIZE(BO)
-	vmovsd	%xmm6, 9*SIZE(BO)
-	vmovss	%xmm7,11*SIZE(BO)
-	addq	$8*SIZE,BO1
-	addq	$8*SIZE,BO2
-	addq	$12*SIZE,BO
-
-	vmovsd	0 * SIZE(BO1), %xmm0
-	vmovsd	2 * SIZE(BO1), %xmm2
-	vmovsd	4 * SIZE(BO1), %xmm4
-	vmovsd	6 * SIZE(BO1), %xmm6
-	vmovss  0 * SIZE(BO2), %xmm1
-	vmovss  2 * SIZE(BO2), %xmm3
-	vmovss  4 * SIZE(BO2), %xmm5
-	vmovss  6 * SIZE(BO2), %xmm7
-	vmovsd	%xmm0, 0*SIZE(BO)
-	vmovss	%xmm1, 2*SIZE(BO)
-	vmovsd	%xmm2, 3*SIZE(BO)
-	vmovss	%xmm3, 5*SIZE(BO)
-	vmovsd	%xmm4, 6*SIZE(BO)
-	vmovss	%xmm5, 8*SIZE(BO)
-	vmovsd	%xmm6, 9*SIZE(BO)
-	vmovss	%xmm7,11*SIZE(BO)
-	addq	$8*SIZE,BO1
-	addq	$8*SIZE,BO2
-	addq	$12*SIZE,BO
-
-	decq	%rax
-	jnz	.L6_01a_1
-
-
-
-.L6_01a_2:
-
-	movq    K, %rax
-        andq    $7, %rax                // K % 8
-        jz      .L6_02c
-        ALIGN_4
-
-
-.L6_02b:
-
-	vmovsd	0 * SIZE(BO1), %xmm0
-	vmovss  0 * SIZE(BO2), %xmm2
-	vmovsd	%xmm0, 0*SIZE(BO)
-	vmovss	%xmm2, 2*SIZE(BO)
-	addq	$2*SIZE,BO1
-	addq	$2*SIZE,BO2
-	addq	$3*SIZE,BO
-	decq	%rax
-	jnz	.L6_02b
-
-.L6_02c:
-
-	movq	K, %rax
-	salq	$1,%rax			// K * 2
-	leaq	(B,%rax, SIZE), BO1	// next offset to BO1
-	leaq	(BO1,%rax, SIZE), BO2	// next offset to BO2
-	leaq    BUFFER2, BO		// second buffer to BO
-	movq	K, %rax
-	sarq	$3 , %rax		// K / 8
-	jz	.L6_02c_2
-	ALIGN_4
-
-.L6_02c_1:
-
-	prefetcht0 512(BO2)
-        prefetchw  512(BO)
-
-	vmovsd	0 * SIZE(BO2), %xmm0
-	vmovsd	2 * SIZE(BO2), %xmm2
-	vmovsd	4 * SIZE(BO2), %xmm4
-	vmovsd	6 * SIZE(BO2), %xmm6
-	vmovss  1 * SIZE(BO1), %xmm1
-	vmovss  3 * SIZE(BO1), %xmm3
-	vmovss  5 * SIZE(BO1), %xmm5
-	vmovss  7 * SIZE(BO1), %xmm7
-	vmovss	%xmm1, 0*SIZE(BO)
-	vmovsd	%xmm0, 1*SIZE(BO)
-	vmovss	%xmm3, 3*SIZE(BO)
-	vmovsd	%xmm2, 4*SIZE(BO)
-	vmovss	%xmm5, 6*SIZE(BO)
-	vmovsd	%xmm4, 7*SIZE(BO)
-	vmovss	%xmm7, 9*SIZE(BO)
-	vmovsd	%xmm6,10*SIZE(BO)
-	addq	$8*SIZE,BO1
-	addq	$8*SIZE,BO2
-	addq	$12*SIZE,BO
-
-
-	vmovsd	0 * SIZE(BO2), %xmm0
-	vmovsd	2 * SIZE(BO2), %xmm2
-	vmovsd	4 * SIZE(BO2), %xmm4
-	vmovsd	6 * SIZE(BO2), %xmm6
-	vmovss  1 * SIZE(BO1), %xmm1
-	vmovss  3 * SIZE(BO1), %xmm3
-	vmovss  5 * SIZE(BO1), %xmm5
-	vmovss  7 * SIZE(BO1), %xmm7
-	vmovss	%xmm1, 0*SIZE(BO)
-	vmovsd	%xmm0, 1*SIZE(BO)
-	vmovss	%xmm3, 3*SIZE(BO)
-	vmovsd	%xmm2, 4*SIZE(BO)
-	vmovss	%xmm5, 6*SIZE(BO)
-	vmovsd	%xmm4, 7*SIZE(BO)
-	vmovss	%xmm7, 9*SIZE(BO)
-	vmovsd	%xmm6,10*SIZE(BO)
-	addq	$8*SIZE,BO1
-	addq	$8*SIZE,BO2
-	addq	$12*SIZE,BO
-
-	decq	%rax
-	jnz	.L6_02c_1
-
-
-.L6_02c_2:
-
-	movq    K, %rax
-        andq    $7, %rax                // K % 8
-        jz      .L6_03c
-        ALIGN_4
-
-.L6_03b:
-
-	vmovss	  1*SIZE(BO1), %xmm0
-	vmovsd	  0*SIZE(BO2), %xmm1
-	vmovss	%xmm0, 0*SIZE(BO)
-	vmovsd	%xmm1, 1*SIZE(BO)
-	addq	$2*SIZE,BO1
-	addq	$2*SIZE,BO2
-	addq	$3*SIZE,BO
-	decq	%rax
-	jnz	.L6_03b
-
-
-.L6_03c:
-
-	movq	BO2, B			// next offset of B
-
-.L6_10:
-	movq	C, CO1
-	leaq	(C, LDC, 2), C		 
-	leaq	(C, LDC, 1), C		// c += 3 * ldc
-
-	
-	movq	A, AO		 	// aoffset = a
-	addq	$32 * SIZE, AO
-
-	movq	M,  I
-	sarq	$4, I			// i = (m >> 4)
-	je	.L6_20
-
-	ALIGN_4
-
-.L6_11:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $6 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax			//  K = K - ( K % 8 )
-	je	.L6_16
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	salq	$4, %rax			// rax = rax * 16 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L6_12:
-
-	prefetcht0	B_PR1(BO,BI, SIZE)
-	KERNEL16x3_1(xxx)
-	KERNEL16x3_2(xxx)
-	KERNEL16x3_3(xxx)
-	KERNEL16x3_4(xxx)
-
-	KERNEL16x3_1(xxx)
-	prefetcht0	B_PR1+16(BO,BI, SIZE)
-	KERNEL16x3_2(xxx)
-	KERNEL16x3_3(xxx)
-	KERNEL16x3_4(xxx)
-
-	je	.L6_16
-
-	KERNEL16x3_1(xxx)
-	KERNEL16x3_2(xxx)
-	prefetcht0	B_PR1+32(BO,BI, SIZE)
-	KERNEL16x3_3(xxx)
-	KERNEL16x3_4(xxx)
-
-	KERNEL16x3_1(xxx)
-	KERNEL16x3_2(xxx)
-	KERNEL16x3_3(xxx)
-	KERNEL16x3_4(xxx)
-
-	je	.L6_16
-
-	jmp	.L6_12
-	ALIGN_4
-
-.L6_16:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L6_19
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	salq	$4, %rax			// rax = rax * 16 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L6_17:
-
-	KERNEL16x3_SUB(xxx)
-	addq	$3, BI
-	addq	$16, %rax
-	jl	.L6_17
-	ALIGN_4
-
-
-.L6_19:
-
-	vbroadcastss	ALPHA, %xmm0
-
-	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddps 4 * SIZE(CO1),%xmm0, %xmm7,%xmm7
-	vfmaddps 8 * SIZE(CO1),%xmm0, %xmm10,%xmm10
-	vfmaddps 12 * SIZE(CO1),%xmm0, %xmm13,%xmm13
-
-	vfmaddps 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-	vfmaddps 4 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
-	vfmaddps 8 * SIZE(CO1, LDC),%xmm0, %xmm11,%xmm11
-	vfmaddps 12 * SIZE(CO1, LDC),%xmm0, %xmm14,%xmm14
-
-	vfmaddps 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
-	vfmaddps 4 * SIZE(CO1, LDC, 2),%xmm0, %xmm9,%xmm9
-	vfmaddps 8 * SIZE(CO1, LDC, 2),%xmm0, %xmm12,%xmm12
-	vfmaddps 12 * SIZE(CO1, LDC, 2),%xmm0, %xmm15,%xmm15
-
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm7 , 4 * SIZE(CO1)
-	vmovups	%xmm10, 8 * SIZE(CO1)
-	vmovups	%xmm13,12 * SIZE(CO1)
-
-	vmovups	%xmm5 ,  	(CO1, LDC)
-	vmovups	%xmm8 , 4 * SIZE(CO1, LDC)
-	vmovups	%xmm11, 8 * SIZE(CO1, LDC)
-	vmovups	%xmm14,12 * SIZE(CO1, LDC)
-
-	vmovups	%xmm6 ,  	(CO1, LDC, 2)
-	vmovups	%xmm9 , 4 * SIZE(CO1, LDC, 2)
-	vmovups	%xmm12, 8 * SIZE(CO1, LDC, 2)
-	vmovups	%xmm15,12 * SIZE(CO1, LDC, 2)
-
-
-	addq	$16 * SIZE, CO1		# coffset += 16
-	decq	I			# i --
-	jg	.L6_11
-	ALIGN_4	
-
-/**************************************************************************
-* Rest of M 
-***************************************************************************/
-.L6_20:
-	// Test rest of M
-
-	testq	$15, M
-	jz	.L7_10		// to next 3 lines of N
-
-	testq	$8, M		
-	jz	.L6_21pre
-	ALIGN_4
-
-/**************************************************************************/
-
-.L6_20_1:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $6 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L6_20_6
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L6_20_2:
-
-	prefetcht0	B_PR1(BO,BI, SIZE)
-	KERNEL8x3_1(xxx)
-	KERNEL8x3_2(xxx)
-	KERNEL8x3_3(xxx)
-	KERNEL8x3_4(xxx)
-
-	KERNEL8x3_1(xxx)
-	prefetcht0	B_PR1+16(BO,BI, SIZE)
-	KERNEL8x3_2(xxx)
-	KERNEL8x3_3(xxx)
-	KERNEL8x3_4(xxx)
-
-	je	.L6_20_6
-
-	KERNEL8x3_1(xxx)
-	KERNEL8x3_2(xxx)
-	prefetcht0	B_PR1+32(BO,BI, SIZE)
-	KERNEL8x3_3(xxx)
-	KERNEL8x3_4(xxx)
-
-	KERNEL8x3_1(xxx)
-	KERNEL8x3_2(xxx)
-	KERNEL8x3_3(xxx)
-	KERNEL8x3_4(xxx)
-
-	je	.L6_20_6
-
-	jmp	.L6_20_2
-	ALIGN_4
-
-.L6_20_6:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L6_20_9
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L6_20_7:
-
-	KERNEL8x3_SUB(xxx)
-	addq	$3, BI
-	addq	$8, %rax
-	jl	.L6_20_7
-	ALIGN_4
-
-
-.L6_20_9:
-
-	vbroadcastss	ALPHA, %xmm0
-
-	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddps 4 * SIZE(CO1),%xmm0, %xmm7,%xmm7
-
-	vfmaddps 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-	vfmaddps 4 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
-
-	vfmaddps 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
-	vfmaddps 4 * SIZE(CO1, LDC, 2),%xmm0, %xmm9,%xmm9
-
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm7 , 4 * SIZE(CO1)
-
-	vmovups	%xmm5 ,  	(CO1, LDC)
-	vmovups	%xmm8 , 4 * SIZE(CO1, LDC)
-
-	vmovups	%xmm6 ,  	(CO1, LDC, 2)
-	vmovups	%xmm9 , 4 * SIZE(CO1, LDC, 2)
-
-
-	addq	$8 * SIZE, CO1		# coffset += 8
-	ALIGN_4
-	
-
-
-/**************************************************************************/
-
-.L6_21pre:
-
-	testq	$4, M		
-	jz	.L6_30
-	ALIGN_4
-
-.L6_21:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $6 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L6_26
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L6_22:
-
-	prefetcht0	B_PR1(BO,BI, SIZE)
-	KERNEL4x3_1(xxx)
-	KERNEL4x3_2(xxx)
-	KERNEL4x3_3(xxx)
-	KERNEL4x3_4(xxx)
-
-	KERNEL4x3_1(xxx)
-	prefetcht0	B_PR1+16(BO,BI, SIZE)
-	KERNEL4x3_2(xxx)
-	KERNEL4x3_3(xxx)
-	KERNEL4x3_4(xxx)
-
-	je	.L6_26
-
-	KERNEL4x3_1(xxx)
-	KERNEL4x3_2(xxx)
-	prefetcht0	B_PR1+32(BO,BI, SIZE)
-	KERNEL4x3_3(xxx)
-	KERNEL4x3_4(xxx)
-
-	KERNEL4x3_1(xxx)
-	KERNEL4x3_2(xxx)
-	KERNEL4x3_3(xxx)
-	KERNEL4x3_4(xxx)
-
-	je	.L6_26
-
-	jmp	.L6_22
-	ALIGN_4
-
-.L6_26:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L6_29
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L6_27:
-
-	KERNEL4x3_SUB(xxx)
-	addq	$3, BI
-	addq	$4, %rax
-	jl	.L6_27
-	ALIGN_4
-
-
-.L6_29:
-
-	vbroadcastss	ALPHA, %xmm0
-
-	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddps 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-	vfmaddps 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
-
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm5 ,  	(CO1, LDC)
-	vmovups	%xmm6 ,  	(CO1, LDC, 2)
-
-	addq	$4 * SIZE, CO1		# coffset += 4
-	ALIGN_4
-	
-
-.L6_30:
-	testq	$2, M		
-	jz	.L6_40
-
-	ALIGN_4
-
-.L6_31:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $6 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L6_36
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L6_32:
-
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL2x3_1(xxx)
-	KERNEL2x3_2(xxx)
-	KERNEL2x3_3(xxx)
-	KERNEL2x3_4(xxx)
-
-	KERNEL2x3_1(xxx)
-	prefetcht0	B_PR1+16(BO,BI,SIZE)
-	KERNEL2x3_2(xxx)
-	KERNEL2x3_3(xxx)
-	KERNEL2x3_4(xxx)
-
-	je	.L6_36
-
-	KERNEL2x3_1(xxx)
-	KERNEL2x3_2(xxx)
-	prefetcht0	B_PR1+32(BO,BI,SIZE)
-	KERNEL2x3_3(xxx)
-	KERNEL2x3_4(xxx)
-
-	KERNEL2x3_1(xxx)
-	KERNEL2x3_2(xxx)
-	KERNEL2x3_3(xxx)
-	KERNEL2x3_4(xxx)
-
-	je	.L6_36
-
-	jmp	.L6_32
-	ALIGN_4
-
-.L6_36:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L6_39
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-	
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L6_37:
-
-	KERNEL2x3_SUB(xxx)
-	addq	$3, BI
-	addq	$2, %rax
-	jl	.L6_37
-	ALIGN_4
-
-
-.L6_39:
-
-	vmovss	ALPHA, %xmm0
-
-	vfmaddss 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddss 1 * SIZE(CO1),%xmm0, %xmm8,%xmm8
-	vfmaddss 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-	vfmaddss 1 * SIZE(CO1, LDC),%xmm0, %xmm10,%xmm10
-	vfmaddss 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
-	vfmaddss 1 * SIZE(CO1, LDC, 2),%xmm0, %xmm12,%xmm12
-
-
-	vmovss	%xmm4 ,  	(CO1)
-	vmovss	%xmm8 , 1 * SIZE(CO1)
-	vmovss	%xmm5 ,  	(CO1, LDC)
-	vmovss	%xmm10, 1 * SIZE(CO1, LDC)
-	vmovss	%xmm6 ,  	(CO1, LDC, 2)
-	vmovss	%xmm12, 1 * SIZE(CO1, LDC, 2)
-
-	addq	$2 * SIZE, CO1		# coffset += 2
-	ALIGN_4
-
-.L6_40:
-	testq	$1, M		
-	jz	.L7_10		// to next 3 lines of N
-
-	ALIGN_4
-
-.L6_41:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $6 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L6_46
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L6_42:
-
-	KERNEL1x3_1(xxx)
-	KERNEL1x3_2(xxx)
-	KERNEL1x3_3(xxx)
-	KERNEL1x3_4(xxx)
-
-	KERNEL1x3_1(xxx)
-	KERNEL1x3_2(xxx)
-	KERNEL1x3_3(xxx)
-	KERNEL1x3_4(xxx)
-
-	je	.L6_46
-
-	KERNEL1x3_1(xxx)
-	KERNEL1x3_2(xxx)
-	KERNEL1x3_3(xxx)
-	KERNEL1x3_4(xxx)
-
-	KERNEL1x3_1(xxx)
-	KERNEL1x3_2(xxx)
-	KERNEL1x3_3(xxx)
-	KERNEL1x3_4(xxx)
-
-	je	.L6_46
-
-	jmp	.L6_42
-	ALIGN_4
-
-.L6_46:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L6_49
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L6_47:
-
-	KERNEL1x3_SUB(xxx)
-	addq	$3, BI
-	addq	$1, %rax
-	jl	.L6_47
-	ALIGN_4
-
-
-.L6_49:
-
-	vmovss	ALPHA, %xmm0
-
-	vfmaddss 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddss 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-	vfmaddss 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
-
-
-	vmovss	%xmm4 ,  	(CO1)
-	vmovss	%xmm5 ,  	(CO1, LDC)
-	vmovss	%xmm6 ,  	(CO1, LDC, 2)
-
-	addq	$1 * SIZE, CO1		# coffset += 1
-	ALIGN_4
-	
-
-
-
-/***************************************************************************************************************/
-
-.L7_10:
-	movq	C, CO1
-	leaq	(C, LDC, 2), C		 
-	leaq	(C, LDC, 1), C		// c += 3 * ldc
-
-	
-	movq	A, AO		 	// aoffset = a
-	addq	$32 * SIZE, AO
-
-	movq	M,  I
-	sarq	$4, I			// i = (m >> 4)
-	je	.L7_20
-
-	ALIGN_4
-
-.L7_11:
-        leaq    BUFFER2, BO             // second buffer to BO
-        addq    $6 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax			//  K = K - ( K % 8 )
-	je	.L7_16
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	salq	$4, %rax			// rax = rax * 16 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L7_12:
-
-	prefetcht0	B_PR1(BO,BI, SIZE)
-	KERNEL16x3_1(xxx)
-	KERNEL16x3_2(xxx)
-	KERNEL16x3_3(xxx)
-	KERNEL16x3_4(xxx)
-
-	KERNEL16x3_1(xxx)
-	prefetcht0	B_PR1+16(BO,BI, SIZE)
-	KERNEL16x3_2(xxx)
-	KERNEL16x3_3(xxx)
-	KERNEL16x3_4(xxx)
-
-	je	.L7_16
-
-	KERNEL16x3_1(xxx)
-	KERNEL16x3_2(xxx)
-	prefetcht0	B_PR1+32(BO,BI, SIZE)
-	KERNEL16x3_3(xxx)
-	KERNEL16x3_4(xxx)
-
-	KERNEL16x3_1(xxx)
-	KERNEL16x3_2(xxx)
-	KERNEL16x3_3(xxx)
-	KERNEL16x3_4(xxx)
-
-	je	.L7_16
-
-	jmp	.L7_12
-	ALIGN_4
-
-.L7_16:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L7_19
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	salq	$4, %rax			// rax = rax * 16 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L7_17:
-
-	KERNEL16x3_SUB(xxx)
-	addq	$3, BI
-	addq	$16, %rax
-	jl	.L7_17
-	ALIGN_4
-
-
-.L7_19:
-
-	vbroadcastss	ALPHA, %xmm0
-
-	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddps 4 * SIZE(CO1),%xmm0, %xmm7,%xmm7
-	vfmaddps 8 * SIZE(CO1),%xmm0, %xmm10,%xmm10
-	vfmaddps 12 * SIZE(CO1),%xmm0, %xmm13,%xmm13
-
-	vfmaddps 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-	vfmaddps 4 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
-	vfmaddps 8 * SIZE(CO1, LDC),%xmm0, %xmm11,%xmm11
-	vfmaddps 12 * SIZE(CO1, LDC),%xmm0, %xmm14,%xmm14
-
-	vfmaddps 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
-	vfmaddps 4 * SIZE(CO1, LDC, 2),%xmm0, %xmm9,%xmm9
-	vfmaddps 8 * SIZE(CO1, LDC, 2),%xmm0, %xmm12,%xmm12
-	vfmaddps 12 * SIZE(CO1, LDC, 2),%xmm0, %xmm15,%xmm15
-
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm7 , 4 * SIZE(CO1)
-	vmovups	%xmm10, 8 * SIZE(CO1)
-	vmovups	%xmm13,12 * SIZE(CO1)
-
-	vmovups	%xmm5 ,  	(CO1, LDC)
-	vmovups	%xmm8 , 4 * SIZE(CO1, LDC)
-	vmovups	%xmm11, 8 * SIZE(CO1, LDC)
-	vmovups	%xmm14,12 * SIZE(CO1, LDC)
-
-	vmovups	%xmm6 ,  	(CO1, LDC, 2)
-	vmovups	%xmm9 , 4 * SIZE(CO1, LDC, 2)
-	vmovups	%xmm12, 8 * SIZE(CO1, LDC, 2)
-	vmovups	%xmm15,12 * SIZE(CO1, LDC, 2)
-
-
-	addq	$16 * SIZE, CO1		# coffset += 16
-	decq	I			# i --
-	jg	.L7_11
-	ALIGN_4	
-
-/**************************************************************************
-* Rest of M 
-***************************************************************************/
-.L7_20:
-	// Test rest of M
-
-	testq	$15, M
-	jz	.L7_60		// to next 3 lines of N
-
-	testq	$8, M		
-	jz	.L7_21pre
-	ALIGN_4
-
-/**************************************************************************/
-
-.L7_20_1:
-        leaq    BUFFER2, BO             // first buffer to BO
-        addq    $6 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L7_20_6
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L7_20_2:
-
-	prefetcht0	B_PR1(BO,BI, SIZE)
-	KERNEL8x3_1(xxx)
-	KERNEL8x3_2(xxx)
-	KERNEL8x3_3(xxx)
-	KERNEL8x3_4(xxx)
-
-	KERNEL8x3_1(xxx)
-	prefetcht0	B_PR1+16(BO,BI, SIZE)
-	KERNEL8x3_2(xxx)
-	KERNEL8x3_3(xxx)
-	KERNEL8x3_4(xxx)
-
-	je	.L7_20_6
-
-	KERNEL8x3_1(xxx)
-	KERNEL8x3_2(xxx)
-	prefetcht0	B_PR1+32(BO,BI, SIZE)
-	KERNEL8x3_3(xxx)
-	KERNEL8x3_4(xxx)
-
-	KERNEL8x3_1(xxx)
-	KERNEL8x3_2(xxx)
-	KERNEL8x3_3(xxx)
-	KERNEL8x3_4(xxx)
-
-	je	.L7_20_6
-
-	jmp	.L7_20_2
-	ALIGN_4
-
-.L7_20_6:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L7_20_9
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L7_20_7:
-
-	KERNEL8x3_SUB(xxx)
-	addq	$3, BI
-	addq	$8, %rax
-	jl	.L7_20_7
-	ALIGN_4
-
-.L7_20_9:
-
-
-	vbroadcastss	ALPHA, %xmm0
-
-	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddps 4 * SIZE(CO1),%xmm0, %xmm7,%xmm7
-
-	vfmaddps 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-	vfmaddps 4 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
-
-	vfmaddps 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
-	vfmaddps 4 * SIZE(CO1, LDC, 2),%xmm0, %xmm9,%xmm9
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm7 , 4 * SIZE(CO1)
-
-	vmovups	%xmm5 ,  	(CO1, LDC)
-	vmovups	%xmm8 , 4 * SIZE(CO1, LDC)
-
-	vmovups	%xmm6 ,  	(CO1, LDC, 2)
-	vmovups	%xmm9 , 4 * SIZE(CO1, LDC, 2)
-
-
-	addq	$8 * SIZE, CO1		# coffset += 8
-	ALIGN_4
-	
-
-
-/**************************************************************************/
-
-.L7_21pre:
-
-	testq	$4, M		
-	jz	.L7_30
-	ALIGN_4
-
-.L7_21:
-        leaq    BUFFER2, BO             // second buffer to BO
-        addq    $6 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L7_26
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L7_22:
-
-	prefetcht0	B_PR1(BO,BI, SIZE)
-	KERNEL4x3_1(xxx)
-	KERNEL4x3_2(xxx)
-	KERNEL4x3_3(xxx)
-	KERNEL4x3_4(xxx)
-
-	KERNEL4x3_1(xxx)
-	prefetcht0	B_PR1+16(BO,BI, SIZE)
-	KERNEL4x3_2(xxx)
-	KERNEL4x3_3(xxx)
-	KERNEL4x3_4(xxx)
-
-	je	.L7_26
-
-	KERNEL4x3_1(xxx)
-	KERNEL4x3_2(xxx)
-	prefetcht0	B_PR1+32(BO,BI, SIZE)
-	KERNEL4x3_3(xxx)
-	KERNEL4x3_4(xxx)
-
-	KERNEL4x3_1(xxx)
-	KERNEL4x3_2(xxx)
-	KERNEL4x3_3(xxx)
-	KERNEL4x3_4(xxx)
-
-	je	.L7_26
-
-	jmp	.L7_22
-	ALIGN_4
-
-.L7_26:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L7_29
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L7_27:
-
-	KERNEL4x3_SUB(xxx)
-	addq	$3, BI
-	addq	$4, %rax
-	jl	.L7_27
-	ALIGN_4
-
-
-.L7_29:
-
-	vbroadcastss	ALPHA, %xmm0
-
-	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddps 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-	vfmaddps 	 (CO1, LDC, 2),%xmm0, %xmm6 ,%xmm6
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm5 ,  	(CO1, LDC)
-	vmovups	%xmm6 ,  	(CO1, LDC, 2)
-
-	addq	$4 * SIZE, CO1		# coffset += 4
-	ALIGN_4
-	
-
-.L7_30:
-	testq	$2, M		
-	jz	.L7_40
-
-	ALIGN_4
-
-.L7_31:
-        leaq    BUFFER2, BO             // second buffer to BO
-        addq    $6 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L7_36
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L7_32:
-
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL2x3_1(xxx)
-	KERNEL2x3_2(xxx)
-	KERNEL2x3_3(xxx)
-	KERNEL2x3_4(xxx)
-
-	KERNEL2x3_1(xxx)
-	prefetcht0	B_PR1+16(BO,BI,SIZE)
-	KERNEL2x3_2(xxx)
-	KERNEL2x3_3(xxx)
-	KERNEL2x3_4(xxx)
-
-	je	.L7_36
-
-	KERNEL2x3_1(xxx)
-	KERNEL2x3_2(xxx)
-	prefetcht0	B_PR1+32(BO,BI,SIZE)
-	KERNEL2x3_3(xxx)
-	KERNEL2x3_4(xxx)
-
-	KERNEL2x3_1(xxx)
-	KERNEL2x3_2(xxx)
-	KERNEL2x3_3(xxx)
-	KERNEL2x3_4(xxx)
-
-	je	.L7_36
-
-	jmp	.L7_32
-	ALIGN_4
-
-.L7_36:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L7_39
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-	
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L7_37:
-
-	KERNEL2x3_SUB(xxx)
-	addq	$3, BI
-	addq	$2, %rax
-	jl	.L7_37
-	ALIGN_4
-
-
-.L7_39:
-
-	vmovss	ALPHA, %xmm0
-
-	vfmaddss 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddss 1 * SIZE(CO1),%xmm0, %xmm8,%xmm8
-	vfmaddss 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-	vfmaddss 1 * SIZE(CO1, LDC),%xmm0, %xmm10,%xmm10
-	vfmaddss 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
-	vfmaddss 1 * SIZE(CO1, LDC, 2),%xmm0, %xmm12,%xmm12
-
-	vmovss	%xmm4 ,  	(CO1)
-	vmovss	%xmm8 , 1 * SIZE(CO1)
-	vmovss	%xmm5 ,  	(CO1, LDC)
-	vmovss	%xmm10, 1 * SIZE(CO1, LDC)
-	vmovss	%xmm6 ,  	(CO1, LDC, 2)
-	vmovss	%xmm12, 1 * SIZE(CO1, LDC, 2)
-
-	addq	$2 * SIZE, CO1		# coffset += 2
-	ALIGN_4
-
-.L7_40:
-	testq	$1, M		
-	jz	.L7_60		// to next 3 lines of N
-
-	ALIGN_4
-
-.L7_41:
-        leaq    BUFFER2, BO             // second buffer to BO
-        addq    $6 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L7_46
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L7_42:
-
-	KERNEL1x3_1(xxx)
-	KERNEL1x3_2(xxx)
-	KERNEL1x3_3(xxx)
-	KERNEL1x3_4(xxx)
-
-	KERNEL1x3_1(xxx)
-	KERNEL1x3_2(xxx)
-	KERNEL1x3_3(xxx)
-	KERNEL1x3_4(xxx)
-
-	je	.L7_46
-
-	KERNEL1x3_1(xxx)
-	KERNEL1x3_2(xxx)
-	KERNEL1x3_3(xxx)
-	KERNEL1x3_4(xxx)
-
-	KERNEL1x3_1(xxx)
-	KERNEL1x3_2(xxx)
-	KERNEL1x3_3(xxx)
-	KERNEL1x3_4(xxx)
-
-	je	.L7_46
-
-	jmp	.L7_42
-	ALIGN_4
-
-.L7_46:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L7_49
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
-
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L7_47:
-
-	KERNEL1x3_SUB(xxx)
-	addq	$3, BI
-	addq	$1, %rax
-	jl	.L7_47
-	ALIGN_4
-
-
-.L7_49:
-
-	vmovss	ALPHA, %xmm0
-
-	vfmaddss 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddss 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-	vfmaddss 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
-
-	vmovss	%xmm4 ,  	(CO1)
-	vmovss	%xmm5 ,  	(CO1, LDC)
-	vmovss	%xmm6 ,  	(CO1, LDC, 2)
-
-	addq	$1 * SIZE, CO1		# coffset += 1
-	ALIGN_4
-	
-
-
-.L7_60:
-
-	decq	J			// j --
-	jg	.L6_01
-
-
-.L2_0:
-	cmpq	$0, Nmod6		// N % 6 == 0
-	je	.L999
-
-/************************************************************************************************
-* Loop for Nmod6 / 2 > 0
-*************************************************************************************************/
-
-	movq	Nmod6, J		
-	sarq	$1, J			// j = j / 2
-	je	.L1_0
-	ALIGN_4
-
-.L2_01:
-	// copy to sub buffer
-	movq	B, BO1
-	leaq    BUFFER1, BO		// first buffer to BO
-	movq	K, %rax
-	ALIGN_4
-
-.L2_02b:
-
-	vmovsd	(BO1), %xmm0
-	vmovsd	%xmm0,       (BO)
-	addq	$2*SIZE,BO1
-	addq	$2*SIZE,BO
-	decq	%rax
-	jnz	.L2_02b
-
-.L2_02c:
-
-	movq	BO1, B			// next offset of B
-
-.L2_10:
-	movq	C, CO1
-	leaq	(C, LDC, 2), C		// c += 2 * ldc
-
-	
-	movq	A, AO		 	// aoffset = a
-	addq	$32 * SIZE, AO
-
-	movq	M,  I
-	sarq	$4, I			// i = (m >> 4)
-	je	.L2_20
-
-	ALIGN_4
-
-.L2_11:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax			//  K = K - ( K % 8 )
-	je	.L2_16
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$4, %rax			// rax = rax * 16 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_12:
-
-	prefetcht0	B_PR1(BO,BI, SIZE)
-	KERNEL16x2_1(xxx)
-	KERNEL16x2_2(xxx)
-	KERNEL16x2_3(xxx)
-	KERNEL16x2_4(xxx)
-
-	KERNEL16x2_1(xxx)
-	KERNEL16x2_2(xxx)
-	KERNEL16x2_3(xxx)
-	KERNEL16x2_4(xxx)
-
-	je	.L2_16
-
-	prefetcht0	B_PR1(BO,BI, SIZE)
-	KERNEL16x2_1(xxx)
-	KERNEL16x2_2(xxx)
-	KERNEL16x2_3(xxx)
-	KERNEL16x2_4(xxx)
-
-	KERNEL16x2_1(xxx)
-	KERNEL16x2_2(xxx)
-	KERNEL16x2_3(xxx)
-	KERNEL16x2_4(xxx)
-
-	je	.L2_16
-
-	jmp	.L2_12
-	ALIGN_4
-
-.L2_16:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_19
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$4, %rax			// rax = rax * 16 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_17:
-
-	KERNEL16x2_SUB(xxx)
-	addq	$2, BI
-	addq	$16, %rax
-	jl	.L2_17
-	ALIGN_4
-
-
-.L2_19:
-
-	vbroadcastss	ALPHA, %xmm0
-
-	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddps 4 * SIZE(CO1),%xmm0, %xmm7,%xmm7
-	vfmaddps 8 * SIZE(CO1),%xmm0, %xmm10,%xmm10
-	vfmaddps 12 * SIZE(CO1),%xmm0, %xmm13,%xmm13
-
-	vfmaddps 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-	vfmaddps 4 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
-	vfmaddps 8 * SIZE(CO1, LDC),%xmm0, %xmm11,%xmm11
-	vfmaddps 12 * SIZE(CO1, LDC),%xmm0, %xmm14,%xmm14
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm7 , 4 * SIZE(CO1)
-	vmovups	%xmm10, 8 * SIZE(CO1)
-	vmovups	%xmm13,12 * SIZE(CO1)
-
-	vmovups	%xmm5 ,  	(CO1, LDC)
-	vmovups	%xmm8 , 4 * SIZE(CO1, LDC)
-	vmovups	%xmm11, 8 * SIZE(CO1, LDC)
-	vmovups	%xmm14,12 * SIZE(CO1, LDC)
-
-	addq	$16 * SIZE, CO1		# coffset += 16
-	decq	I			# i --
-	jg	.L2_11
-	ALIGN_4	
-
-/**************************************************************************
-* Rest of M 
-***************************************************************************/
-.L2_20:
-	// Test rest of M
-
-	testq	$15, M
-	jz	.L2_60		// to next 3 lines of N
-
-	testq	$8, M		
-	jz	.L2_21pre
-	ALIGN_4
-
-/**************************************************************************/
-
-.L2_20_1:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L2_20_6
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_20_2:
-
-	prefetcht0	B_PR1(BO,BI, SIZE)
-	KERNEL8x2_1(xxx)
-	KERNEL8x2_2(xxx)
-	KERNEL8x2_3(xxx)
-	KERNEL8x2_4(xxx)
-
-	KERNEL8x2_1(xxx)
-	KERNEL8x2_2(xxx)
-	KERNEL8x2_3(xxx)
-	KERNEL8x2_4(xxx)
-
-	je	.L2_20_6
-
-	prefetcht0	B_PR1(BO,BI, SIZE)
-	KERNEL8x2_1(xxx)
-	KERNEL8x2_2(xxx)
-	KERNEL8x2_3(xxx)
-	KERNEL8x2_4(xxx)
-
-	KERNEL8x2_1(xxx)
-	KERNEL8x2_2(xxx)
-	KERNEL8x2_3(xxx)
-	KERNEL8x2_4(xxx)
-
-	je	.L2_20_6
-
-	jmp	.L2_20_2
-	ALIGN_4
-
-.L2_20_6:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_20_9
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_20_7:
-
-	KERNEL8x2_SUB(xxx)
-	addq	$2, BI
-	addq	$8, %rax
-	jl	.L2_20_7
-	ALIGN_4
-
-
-.L2_20_9:
-
-	vbroadcastss	ALPHA, %xmm0
-
-	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddps 4 * SIZE(CO1),%xmm0, %xmm7,%xmm7
-
-	vfmaddps 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-	vfmaddps 4 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm7 , 4 * SIZE(CO1)
-
-	vmovups	%xmm5 ,  	(CO1, LDC)
-	vmovups	%xmm8 , 4 * SIZE(CO1, LDC)
-
-
-	addq	$8 * SIZE, CO1		# coffset += 8
-	ALIGN_4
-	
-
-
-/**************************************************************************/
-
-.L2_21pre:
-
-	testq	$4, M		
-	jz	.L2_30
-	ALIGN_4
-
-.L2_21:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L2_26
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 1 ; number of values
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_22:
-
-	prefetcht0	B_PR1(BO,BI, SIZE)
-	KERNEL4x2_1(xxx)
-	KERNEL4x2_2(xxx)
-	KERNEL4x2_3(xxx)
-	KERNEL4x2_4(xxx)
-
-	KERNEL4x2_1(xxx)
-	KERNEL4x2_2(xxx)
-	KERNEL4x2_3(xxx)
-	KERNEL4x2_4(xxx)
-
-	je	.L2_26
-
-	prefetcht0	B_PR1(BO,BI, SIZE)
-	KERNEL4x2_1(xxx)
-	KERNEL4x2_2(xxx)
-	KERNEL4x2_3(xxx)
-	KERNEL4x2_4(xxx)
-
-	KERNEL4x2_1(xxx)
-	KERNEL4x2_2(xxx)
-	KERNEL4x2_3(xxx)
-	KERNEL4x2_4(xxx)
-
-	je	.L2_26
-
-	jmp	.L2_22
-	ALIGN_4
-
-.L2_26:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_29
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_27:
-
-	KERNEL4x2_SUB(xxx)
-	addq	$2, BI
-	addq	$4, %rax
-	jl	.L2_27
-	ALIGN_4
-
-
-.L2_29:
-
-	vbroadcastss	ALPHA, %xmm0
-
-	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddps 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm5 ,  	(CO1, LDC)
-
-
-	addq	$4 * SIZE, CO1		# coffset += 4
-	ALIGN_4
-	
-
-.L2_30:
-	testq	$2, M		
-	jz	.L2_40
-
-	ALIGN_4
-
-.L2_31:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L2_36
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_32:
-
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL2x2_1(xxx)
-	KERNEL2x2_2(xxx)
-	KERNEL2x2_3(xxx)
-	KERNEL2x2_4(xxx)
-
-	KERNEL2x2_1(xxx)
-	KERNEL2x2_2(xxx)
-	KERNEL2x2_3(xxx)
-	KERNEL2x2_4(xxx)
-
-	je	.L2_36
-
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL2x2_1(xxx)
-	KERNEL2x2_2(xxx)
-	KERNEL2x2_3(xxx)
-	KERNEL2x2_4(xxx)
-
-	KERNEL2x2_1(xxx)
-	KERNEL2x2_2(xxx)
-	KERNEL2x2_3(xxx)
-	KERNEL2x2_4(xxx)
-
-	je	.L2_36
-
-	jmp	.L2_32
-	ALIGN_4
-
-.L2_36:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_39
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-	
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_37:
-
-	KERNEL2x2_SUB(xxx)
-	addq	$2, BI
-	addq	$2, %rax
-	jl	.L2_37
-	ALIGN_4
-
-
-.L2_39:
-
-	vmovss	ALPHA, %xmm0
-
-	vfmaddss 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddss 1 * SIZE(CO1),%xmm0, %xmm8,%xmm8
-	vfmaddss 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-	vfmaddss 1 * SIZE(CO1, LDC),%xmm0, %xmm10,%xmm10
-
-	vmovss	%xmm4 ,  	(CO1)
-	vmovss	%xmm8 , 1 * SIZE(CO1)
-	vmovss	%xmm5 ,  	(CO1, LDC)
-	vmovss	%xmm10, 1 * SIZE(CO1, LDC)
-
-	addq	$2 * SIZE, CO1		# coffset += 2
-	ALIGN_4
-
-.L2_40:
-	testq	$1, M		
-	jz	.L2_60		// to next 2 lines of N
-
-	ALIGN_4
-
-.L2_41:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L2_46
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_42:
-
-	KERNEL1x2_1(xxx)
-	KERNEL1x2_2(xxx)
-	KERNEL1x2_3(xxx)
-	KERNEL1x2_4(xxx)
-
-	KERNEL1x2_1(xxx)
-	KERNEL1x2_2(xxx)
-	KERNEL1x2_3(xxx)
-	KERNEL1x2_4(xxx)
-
-	je	.L2_46
-
-	KERNEL1x2_1(xxx)
-	KERNEL1x2_2(xxx)
-	KERNEL1x2_3(xxx)
-	KERNEL1x2_4(xxx)
-
-	KERNEL1x2_1(xxx)
-	KERNEL1x2_2(xxx)
-	KERNEL1x2_3(xxx)
-	KERNEL1x2_4(xxx)
-
-	je	.L2_46
-
-	jmp	.L2_42
-	ALIGN_4
-
-.L2_46:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_49
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_47:
-
-	KERNEL1x2_SUB(xxx)
-	addq	$2, BI
-	addq	$1, %rax
-	jl	.L2_47
-	ALIGN_4
-
-
-.L2_49:
-
-	vmovss	ALPHA, %xmm0
-
-	vfmaddss 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddss 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-
-	vmovss	%xmm4 ,  	(CO1)
-	vmovss	%xmm5 ,  	(CO1, LDC)
-
-	addq	$1 * SIZE, CO1		# coffset += 1
-	ALIGN_4
-	
-.L2_60:
-
-	decq	J			// j --
-	jg	.L2_01			// next 2 lines of N
-
-
-
-.L1_0:
-
-/************************************************************************************************
-* Loop for Nmod6 % 2 > 0
-*************************************************************************************************/
-
-	movq	Nmod6, J		
-	andq	$1, J			// j % 2
-	je	.L999
-	ALIGN_4
-
-.L1_01:
-	// copy to sub buffer
-	movq	B, BO1
-	leaq    BUFFER1, BO		// first buffer to BO
-	movq	K, %rax
-	ALIGN_4
-
-.L1_02b:
-
-	vmovss	(BO1), %xmm0
-	vmovss	%xmm0,       (BO)
-	addq	$1*SIZE,BO1
-	addq	$1*SIZE,BO
-	decq	%rax
-	jnz	.L1_02b
-
-.L1_02c:
-
-	movq	BO1, B			// next offset of B
-
-.L1_10:
-	movq	C, CO1
-	leaq	(C, LDC, 1), C		// c += 1 * ldc
-
-	
-	movq	A, AO		 	// aoffset = a
-	addq	$32 * SIZE, AO
-
-	movq	M,  I
-	sarq	$4, I			// i = (m >> 4)
-	je	.L1_20
-
-	ALIGN_4
-
-.L1_11:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax			//  K = K - ( K % 8 )
-	je	.L1_16
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$4, %rax			// rax = rax * 16 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_12:
-
-	prefetcht0	B_PR1(BO,BI, SIZE)
-	KERNEL16x1_1(xxx)
-	KERNEL16x1_2(xxx)
-	KERNEL16x1_3(xxx)
-	KERNEL16x1_4(xxx)
-
-	KERNEL16x1_1(xxx)
-	KERNEL16x1_2(xxx)
-	KERNEL16x1_3(xxx)
-	KERNEL16x1_4(xxx)
-
-	je	.L1_16
-
-	KERNEL16x1_1(xxx)
-	KERNEL16x1_2(xxx)
-	KERNEL16x1_3(xxx)
-	KERNEL16x1_4(xxx)
-
-	KERNEL16x1_1(xxx)
-	KERNEL16x1_2(xxx)
-	KERNEL16x1_3(xxx)
-	KERNEL16x1_4(xxx)
-
-	je	.L1_16
-
-	jmp	.L1_12
-	ALIGN_4
-
-.L1_16:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_19
-
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$4, %rax			// rax = rax * 16 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_17:
-
-	KERNEL16x1_SUB(xxx)
-	addq	$1, BI
-	addq	$16, %rax
-	jl	.L1_17
-	ALIGN_4
-
-
-.L1_19:
-
-	vbroadcastss	ALPHA, %xmm0
-
-	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddps 4 * SIZE(CO1),%xmm0, %xmm7,%xmm7
-	vfmaddps 8 * SIZE(CO1),%xmm0, %xmm10,%xmm10
-	vfmaddps 12 * SIZE(CO1),%xmm0, %xmm13,%xmm13
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm7 , 4 * SIZE(CO1)
-	vmovups	%xmm10, 8 * SIZE(CO1)
-	vmovups	%xmm13,12 * SIZE(CO1)
-
-	addq	$16 * SIZE, CO1		# coffset += 16
-	decq	I			# i --
-	jg	.L1_11
-	ALIGN_4	
-
-/**************************************************************************
-* Rest of M 
-***************************************************************************/
-.L1_20:
-	// Test rest of M
-
-	testq	$15, M
-	jz	.L999
-
-	testq	$8, M		
-	jz	.L1_21pre
-	ALIGN_4
-
-/**************************************************************************/
-
-.L1_20_1:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L1_20_6
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_20_2:
-
-	prefetcht0	B_PR1(BO,BI, SIZE)
-	KERNEL8x1_1(xxx)
-	KERNEL8x1_2(xxx)
-	KERNEL8x1_3(xxx)
-	KERNEL8x1_4(xxx)
-
-	KERNEL8x1_1(xxx)
-	KERNEL8x1_2(xxx)
-	KERNEL8x1_3(xxx)
-	KERNEL8x1_4(xxx)
-
-	je	.L1_20_6
-
-	KERNEL8x1_1(xxx)
-	KERNEL8x1_2(xxx)
-	KERNEL8x1_3(xxx)
-	KERNEL8x1_4(xxx)
-
-	KERNEL8x1_1(xxx)
-	KERNEL8x1_2(xxx)
-	KERNEL8x1_3(xxx)
-	KERNEL8x1_4(xxx)
-
-	je	.L1_20_6
-
-	jmp	.L1_20_2
-	ALIGN_4
-
-.L1_20_6:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_20_9
-
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_20_7:
-
-	KERNEL8x1_SUB(xxx)
-	addq	$1, BI
-	addq	$8, %rax
-	jl	.L1_20_7
-	ALIGN_4
-
-
-.L1_20_9:
-
-	vbroadcastss	ALPHA, %xmm0
-
-	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddps 4 * SIZE(CO1),%xmm0, %xmm7,%xmm7
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm7 , 4 * SIZE(CO1)
-
-	addq	$8 * SIZE, CO1		# coffset += 8
-	ALIGN_4
-	
-
-
-/**************************************************************************/
-
-.L1_21pre:
-
-	testq	$4, M		
-	jz	.L1_30
-	ALIGN_4
-
-.L1_21:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L1_26
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_22:
-
-	prefetcht0	B_PR1(BO,BI, SIZE)
-	KERNEL4x1_1(xxx)
-	KERNEL4x1_2(xxx)
-	KERNEL4x1_3(xxx)
-	KERNEL4x1_4(xxx)
-
-	KERNEL4x1_1(xxx)
-	KERNEL4x1_2(xxx)
-	KERNEL4x1_3(xxx)
-	KERNEL4x1_4(xxx)
-
-	je	.L1_26
-
-	KERNEL4x1_1(xxx)
-	KERNEL4x1_2(xxx)
-	KERNEL4x1_3(xxx)
-	KERNEL4x1_4(xxx)
-
-	KERNEL4x1_1(xxx)
-	KERNEL4x1_2(xxx)
-	KERNEL4x1_3(xxx)
-	KERNEL4x1_4(xxx)
-
-	je	.L1_26
-
-	jmp	.L1_22
-	ALIGN_4
-
-.L1_26:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_29
-
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_27:
-
-	KERNEL4x1_SUB(xxx)
-	addq	$1, BI
-	addq	$4, %rax
-	jl	.L1_27
-	ALIGN_4
-
-
-.L1_29:
-
-	vbroadcastss	ALPHA, %xmm0
-
-	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
-
-	vmovups	%xmm4 ,  	(CO1)
-
-	addq	$4 * SIZE, CO1		# coffset += 4
-	ALIGN_4
-	
-
-.L1_30:
-	testq	$2, M		
-	jz	.L1_40
-
-	ALIGN_4
-
-.L1_31:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L1_36
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_32:
-
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL2x1_1(xxx)
-	KERNEL2x1_2(xxx)
-	KERNEL2x1_3(xxx)
-	KERNEL2x1_4(xxx)
-
-	KERNEL2x1_1(xxx)
-	KERNEL2x1_2(xxx)
-	KERNEL2x1_3(xxx)
-	KERNEL2x1_4(xxx)
-
-	je	.L1_36
-
-	KERNEL2x1_1(xxx)
-	KERNEL2x1_2(xxx)
-	KERNEL2x1_3(xxx)
-	KERNEL2x1_4(xxx)
-
-	KERNEL2x1_1(xxx)
-	KERNEL2x1_2(xxx)
-	KERNEL2x1_3(xxx)
-	KERNEL2x1_4(xxx)
-
-	je	.L1_36
-
-	jmp	.L1_32
-	ALIGN_4
-
-.L1_36:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_39
-
-	movq    %rax, BI                        //  Index for BO
-	
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_37:
-
-	KERNEL2x1_SUB(xxx)
-	addq	$1, BI
-	addq	$2, %rax
-	jl	.L1_37
-	ALIGN_4
-
-
-.L1_39:
-
-	vmovss	ALPHA, %xmm0
-
-	vfmaddss 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddss 1 * SIZE(CO1),%xmm0, %xmm8,%xmm8
-
-	vmovss	%xmm4 ,  	(CO1)
-	vmovss	%xmm8 , 1 * SIZE(CO1)
-
-	addq	$2 * SIZE, CO1		# coffset += 2
-	ALIGN_4
-
-.L1_40:
-	testq	$1, M		
-	jz	.L999
-
-	ALIGN_4
-
-.L1_41:
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-
-	vzeroall
-
-        movq    K, %rax
-
-	andq	$-8, %rax
-	je	.L1_46
-	movq    %rax, BI                        //  Index for BO
-
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_42:
-
-	KERNEL1x1_1(xxx)
-	KERNEL1x1_2(xxx)
-	KERNEL1x1_3(xxx)
-	KERNEL1x1_4(xxx)
-
-	KERNEL1x1_1(xxx)
-	KERNEL1x1_2(xxx)
-	KERNEL1x1_3(xxx)
-	KERNEL1x1_4(xxx)
-
-	je	.L1_46
-
-	KERNEL1x1_1(xxx)
-	KERNEL1x1_2(xxx)
-	KERNEL1x1_3(xxx)
-	KERNEL1x1_4(xxx)
-
-	KERNEL1x1_1(xxx)
-	KERNEL1x1_2(xxx)
-	KERNEL1x1_3(xxx)
-	KERNEL1x1_4(xxx)
-
-	je	.L1_46
-
-	jmp	.L1_42
-	ALIGN_4
-
-.L1_46:
-        movq    K, %rax
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_49
-
-	movq    %rax, BI                        //  Index for BO
-
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_47:
-
-	KERNEL1x1_SUB(xxx)
-	addq	$1, BI
-	addq	$1, %rax
-	jl	.L1_47
-	ALIGN_4
-
-
-.L1_49:
-
-	vmovss	ALPHA, %xmm0
-
-	vfmaddss 	 (CO1),%xmm0, %xmm4,%xmm4
-
-	vmovss	%xmm4 ,  	(CO1)
-
-	addq	$1 * SIZE, CO1		# coffset += 1
-	ALIGN_4
-	
-
-.L999:
-	movq   		SP, %rsp
-	movq	   (%rsp), %rbx
-	movq	  8(%rsp), %rbp
-	movq	 16(%rsp), %r12
-	movq	 24(%rsp), %r13
-	movq	 32(%rsp), %r14
-	movq	 40(%rsp), %r15
-
-#ifdef WINDOWS_ABI
-	movq	 48(%rsp), %rdi
-	movq	 56(%rsp), %rsi
-	movups	 64(%rsp), %xmm6
-	movups	 80(%rsp), %xmm7
-	movups	 96(%rsp), %xmm8
-	movups	112(%rsp), %xmm9
-	movups	128(%rsp), %xmm10
-	movups	144(%rsp), %xmm11
-	movups	160(%rsp), %xmm12
-	movups	176(%rsp), %xmm13
-	movups	192(%rsp), %xmm14
-	movups	208(%rsp), %xmm15
-#endif
-
-	addq	$STACKSIZE, %rsp
-	ret
-
-	EPILOGUE
-
-
-#else
-/*************************************************************************************
-* TRMM Kernel
-*************************************************************************************/
-
-
-	PROLOGUE
-	PROFCODE
-	
-	subq	$STACKSIZE, %rsp
-	movq	%rbx,   (%rsp)
-	movq	%rbp,  8(%rsp)
-	movq	%r12, 16(%rsp)
-	movq	%r13, 24(%rsp)
-	movq	%r14, 32(%rsp)
-	movq	%r15, 40(%rsp)
-
-	vzeroupper
-
-#ifdef WINDOWS_ABI
-	movq	%rdi,    48(%rsp)
-	movq	%rsi,    56(%rsp)
-	movups	%xmm6,   64(%rsp)
-	movups	%xmm7,   80(%rsp)
-	movups	%xmm8,   96(%rsp)
-	movups	%xmm9,  112(%rsp)
-	movups	%xmm10, 128(%rsp)
-	movups	%xmm11, 144(%rsp)
-	movups	%xmm12, 160(%rsp)
-	movups	%xmm13, 176(%rsp)
-	movups	%xmm14, 192(%rsp)
-	movups	%xmm15, 208(%rsp)
-
-	movq	ARG1,      OLD_M
-	movq	ARG2,      OLD_N
-	movq	ARG3,      OLD_K
-	movq	OLD_A,     A
-	movq	OLD_B,     B
-	movq	OLD_C,     C
-	movq	OLD_LDC,   LDC
-#ifdef TRMMKERNEL
-	movsd	OLD_OFFSET, %xmm12
-#endif
-	vmovaps	%xmm3, %xmm0
-
-#else
-	movq	STACKSIZE +  8(%rsp), LDC
-#ifdef TRMMKERNEL
-	movsd	STACKSIZE + 16(%rsp), %xmm12
-#endif
-
-#endif
-
-	movq    %rsp, SP      # save old stack
-        subq    $128 + L_BUFFER_SIZE, %rsp
-        andq    $-4096, %rsp    # align stack
-
-        STACK_TOUCH
-
-	cmpq	$0, OLD_M
-	je	.L999
-
-	cmpq	$0, OLD_N
-	je	.L999
-
-	cmpq	$0, OLD_K
-	je	.L999
-
-	movq	OLD_M, M
-	movq	OLD_N, N
-	movq	OLD_K, K
-
-	vmovsd	 %xmm0, ALPHA
-
-	salq	$BASE_SHIFT, LDC
-
-	movq    N, %rax
-        xorq    %rdx, %rdx
-        movq    $2,  %rdi
-        divq    %rdi                    //    N / 6
-        movq    %rax, Ndiv6             //    N / 6
-        movq    %rdx, Nmod6             //    N % 6
-
-	
-
-#ifdef TRMMKERNEL
-	vmovsd	%xmm12, OFFSET
-	vmovsd	%xmm12, KK
-#ifndef LEFT
-	negq	KK
-#endif	
-#endif
-
-	movq	Ndiv6,  J
-	cmpq	$0, J
-	je	.L1_0
-	ALIGN_4
-
-.L2_01:
-	// copy to sub buffer
-	movq	B, BO1
-	leaq    BUFFER1, BO		// first buffer to BO
-	movq	K, %rax
-	ALIGN_4
-
-.L2_02b:
-
-	vmovsd	(BO1), %xmm0
-	vmovsd	%xmm0,       (BO)
-	addq	$2*SIZE,BO1
-	addq	$2*SIZE,BO
-	decq	%rax
-	jnz	.L2_02b
-
-.L2_02c:
-
-	movq	BO1, B			// next offset of B
-
-.L2_10:
-	movq	C, CO1
-	leaq	(C, LDC, 2), C		// c += 2 * ldc
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        movq    OFFSET, %rax
-        movq    %rax, KK
-#endif
-	
-	movq	A, AO		 	// aoffset = a
-	addq	$32 * SIZE, AO
-
-	movq	M,  I
-	sarq	$4, I			// i = (m >> 4)
-	je	.L2_20
-
-	ALIGN_4
-
-.L2_11:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO
-        salq    $4, %rax                        // rax = rax * 16 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $16, %rax	// number of values in AO
-#else
-        addq    $2, %rax	// number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-	andq	$-8, %rax			//  K = K - ( K % 8 )
-	je	.L2_16
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$4, %rax			// rax = rax * 16 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_12:
-
-	prefetcht0	B_PR1(BO,BI, SIZE)
-	KERNEL16x2_1(xxx)
-	KERNEL16x2_2(xxx)
-	KERNEL16x2_3(xxx)
-	KERNEL16x2_4(xxx)
-
-	KERNEL16x2_1(xxx)
-	KERNEL16x2_2(xxx)
-	KERNEL16x2_3(xxx)
-	KERNEL16x2_4(xxx)
-
-	je	.L2_16
-
-	prefetcht0	B_PR1(BO,BI, SIZE)
-	KERNEL16x2_1(xxx)
-	KERNEL16x2_2(xxx)
-	KERNEL16x2_3(xxx)
-	KERNEL16x2_4(xxx)
-
-	KERNEL16x2_1(xxx)
-	KERNEL16x2_2(xxx)
-	KERNEL16x2_3(xxx)
-	KERNEL16x2_4(xxx)
-
-	je	.L2_16
-
-	jmp	.L2_12
-	ALIGN_4
-
-.L2_16:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_19
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$4, %rax			// rax = rax * 16 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_17:
-
-	KERNEL16x2_SUB(xxx)
-	addq	$2, BI
-	addq	$16, %rax
-	jl	.L2_17
-	ALIGN_4
-
-
-.L2_19:
-
-	vbroadcastss	ALPHA, %xmm0
-
-#ifndef TRMMKERNEL
-
-	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddps 4 * SIZE(CO1),%xmm0, %xmm7,%xmm7
-	vfmaddps 8 * SIZE(CO1),%xmm0, %xmm10,%xmm10
-	vfmaddps 12 * SIZE(CO1),%xmm0, %xmm13,%xmm13
-
-	vfmaddps 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-	vfmaddps 4 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
-	vfmaddps 8 * SIZE(CO1, LDC),%xmm0, %xmm11,%xmm11
-	vfmaddps 12 * SIZE(CO1, LDC),%xmm0, %xmm14,%xmm14
-
-#else
-	vmulps	%xmm0, %xmm4,%xmm4
-	vmulps	%xmm0, %xmm7,%xmm7
-	vmulps	%xmm0, %xmm10,%xmm10
-	vmulps	%xmm0, %xmm13,%xmm13
-
-	vmulps	%xmm0, %xmm5,%xmm5
-	vmulps	%xmm0, %xmm8,%xmm8
-	vmulps	%xmm0, %xmm11,%xmm11
-	vmulps	%xmm0, %xmm14,%xmm14
-
-#endif
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm7 , 4 * SIZE(CO1)
-	vmovups	%xmm10, 8 * SIZE(CO1)
-	vmovups	%xmm13,12 * SIZE(CO1)
-
-	vmovups	%xmm5 ,  	(CO1, LDC)
-	vmovups	%xmm8 , 4 * SIZE(CO1, LDC)
-	vmovups	%xmm11, 8 * SIZE(CO1, LDC)
-	vmovups	%xmm14,12 * SIZE(CO1, LDC)
-
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO
-        salq    $4, %rax                        // rax = rax * 16 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $16, KK				
-#endif
-
-	addq	$16 * SIZE, CO1		# coffset += 16
-	decq	I			# i --
-	jg	.L2_11
-	ALIGN_4	
-
-/**************************************************************************
-* Rest of M 
-***************************************************************************/
-.L2_20:
-	// Test rest of M
-
-	testq	$15, M
-	jz	.L2_60		// to next 3 lines of N
-
-	testq	$8, M		
-	jz	.L2_21pre
-	ALIGN_4
-
-/**************************************************************************/
-
-.L2_20_1:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO
-        salq    $3, %rax                        // rax = rax * 8 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $8, %rax        // number of values in A
-#else
-        addq    $2, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-
-	andq	$-8, %rax
-	je	.L2_20_6
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_20_2:
-
-	prefetcht0	B_PR1(BO,BI, SIZE)
-	KERNEL8x2_1(xxx)
-	KERNEL8x2_2(xxx)
-	KERNEL8x2_3(xxx)
-	KERNEL8x2_4(xxx)
-
-	KERNEL8x2_1(xxx)
-	KERNEL8x2_2(xxx)
-	KERNEL8x2_3(xxx)
-	KERNEL8x2_4(xxx)
-
-	je	.L2_20_6
-
-	prefetcht0	B_PR1(BO,BI, SIZE)
-	KERNEL8x2_1(xxx)
-	KERNEL8x2_2(xxx)
-	KERNEL8x2_3(xxx)
-	KERNEL8x2_4(xxx)
-
-	KERNEL8x2_1(xxx)
-	KERNEL8x2_2(xxx)
-	KERNEL8x2_3(xxx)
-	KERNEL8x2_4(xxx)
-
-	je	.L2_20_6
-
-	jmp	.L2_20_2
-	ALIGN_4
-
-.L2_20_6:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_20_9
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_20_7:
-
-	KERNEL8x2_SUB(xxx)
-	addq	$2, BI
-	addq	$8, %rax
-	jl	.L2_20_7
-	ALIGN_4
-
-
-.L2_20_9:
-
-	vbroadcastss	ALPHA, %xmm0
-
-#ifndef TRMMKERNEL
-
-	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddps 4 * SIZE(CO1),%xmm0, %xmm7,%xmm7
-
-	vfmaddps 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-	vfmaddps 4 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
-
-#else
-	vmulps	%xmm0, %xmm4,%xmm4
-	vmulps	%xmm0, %xmm7,%xmm7
-
-	vmulps	%xmm0, %xmm5,%xmm5
-	vmulps	%xmm0, %xmm8,%xmm8
-
-#endif
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm7 , 4 * SIZE(CO1)
-
-	vmovups	%xmm5 ,  	(CO1, LDC)
-	vmovups	%xmm8 , 4 * SIZE(CO1, LDC)
-
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax 
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO         
-        salq    $3, %rax                        // rax = rax * 8 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif  
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $8, KK
-#endif
-
-	addq	$8 * SIZE, CO1		# coffset += 8
-	ALIGN_4
-	
-
-
-/**************************************************************************/
-
-.L2_21pre:
-
-	testq	$4, M		
-	jz	.L2_30
-	ALIGN_4
-
-.L2_21:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO
-        salq    $2, %rax                        // rax = rax * 4 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $4, %rax        // number of values in A
-#else
-        addq    $2, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-
-	andq	$-8, %rax
-	je	.L2_26
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 1 ; number of values
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_22:
-
-	prefetcht0	B_PR1(BO,BI, SIZE)
-	KERNEL4x2_1(xxx)
-	KERNEL4x2_2(xxx)
-	KERNEL4x2_3(xxx)
-	KERNEL4x2_4(xxx)
-
-	KERNEL4x2_1(xxx)
-	KERNEL4x2_2(xxx)
-	KERNEL4x2_3(xxx)
-	KERNEL4x2_4(xxx)
-
-	je	.L2_26
-
-	prefetcht0	B_PR1(BO,BI, SIZE)
-	KERNEL4x2_1(xxx)
-	KERNEL4x2_2(xxx)
-	KERNEL4x2_3(xxx)
-	KERNEL4x2_4(xxx)
-
-	KERNEL4x2_1(xxx)
-	KERNEL4x2_2(xxx)
-	KERNEL4x2_3(xxx)
-	KERNEL4x2_4(xxx)
-
-	je	.L2_26
-
-	jmp	.L2_22
-	ALIGN_4
-
-.L2_26:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_29
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_27:
-
-	KERNEL4x2_SUB(xxx)
-	addq	$2, BI
-	addq	$4, %rax
-	jl	.L2_27
-	ALIGN_4
-
-
-.L2_29:
-
-	vbroadcastss	ALPHA, %xmm0
-
-#ifndef TRMMKERNEL
-
-	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddps 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-
-#else
-	vmulps	%xmm0, %xmm4,%xmm4
-	vmulps	%xmm0, %xmm5,%xmm5
-
-#endif
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm5 ,  	(CO1, LDC)
-
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax 
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO         
-        salq    $2, %rax                        // rax = rax * 4 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif  
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $4, KK
-#endif
-
-	addq	$4 * SIZE, CO1		# coffset += 4
-	ALIGN_4
-	
-
-.L2_30:
-	testq	$2, M		
-	jz	.L2_40
-
-	ALIGN_4
-
-.L2_31:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO
-        salq    $1, %rax                        // rax = rax * 2 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $2, %rax        // number of values in AO
-#else
-        addq    $2, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-
-	andq	$-8, %rax
-	je	.L2_36
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_32:
-
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL2x2_1(xxx)
-	KERNEL2x2_2(xxx)
-	KERNEL2x2_3(xxx)
-	KERNEL2x2_4(xxx)
-
-	KERNEL2x2_1(xxx)
-	KERNEL2x2_2(xxx)
-	KERNEL2x2_3(xxx)
-	KERNEL2x2_4(xxx)
-
-	je	.L2_36
-
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL2x2_1(xxx)
-	KERNEL2x2_2(xxx)
-	KERNEL2x2_3(xxx)
-	KERNEL2x2_4(xxx)
-
-	KERNEL2x2_1(xxx)
-	KERNEL2x2_2(xxx)
-	KERNEL2x2_3(xxx)
-	KERNEL2x2_4(xxx)
-
-	je	.L2_36
-
-	jmp	.L2_32
-	ALIGN_4
-
-.L2_36:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_39
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-	
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_37:
-
-	KERNEL2x2_SUB(xxx)
-	addq	$2, BI
-	addq	$2, %rax
-	jl	.L2_37
-	ALIGN_4
-
-
-.L2_39:
-
-	vmovss	ALPHA, %xmm0
-
-#ifndef TRMMKERNEL
-
-	vfmaddss 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddss 1 * SIZE(CO1),%xmm0, %xmm8,%xmm8
-	vfmaddss 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-	vfmaddss 1 * SIZE(CO1, LDC),%xmm0, %xmm10,%xmm10
-
-#else
-	vmulss	%xmm0, %xmm4,%xmm4
-	vmulss	%xmm0, %xmm8,%xmm8
-	vmulss	%xmm0, %xmm5,%xmm5
-	vmulss	%xmm0, %xmm10,%xmm10
-
-#endif
-
-	vmovss	%xmm4 ,  	(CO1)
-	vmovss	%xmm8 , 1 * SIZE(CO1)
-	vmovss	%xmm5 ,  	(CO1, LDC)
-	vmovss	%xmm10, 1 * SIZE(CO1, LDC)
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax 
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO         
-        salq    $1, %rax                        // rax = rax * 2 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif  
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $2, KK
-#endif
-
-	addq	$2 * SIZE, CO1		# coffset += 2
-	ALIGN_4
-
-.L2_40:
-	testq	$1, M		
-	jz	.L2_60		// to next 2 lines of N
-
-	ALIGN_4
-
-.L2_41:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $1, %rax        // number of values in AO
-#else
-        addq    $2, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-	andq	$-8, %rax
-	je	.L2_46
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_42:
-
-	KERNEL1x2_1(xxx)
-	KERNEL1x2_2(xxx)
-	KERNEL1x2_3(xxx)
-	KERNEL1x2_4(xxx)
-
-	KERNEL1x2_1(xxx)
-	KERNEL1x2_2(xxx)
-	KERNEL1x2_3(xxx)
-	KERNEL1x2_4(xxx)
-
-	je	.L2_46
-
-	KERNEL1x2_1(xxx)
-	KERNEL1x2_2(xxx)
-	KERNEL1x2_3(xxx)
-	KERNEL1x2_4(xxx)
-
-	KERNEL1x2_1(xxx)
-	KERNEL1x2_2(xxx)
-	KERNEL1x2_3(xxx)
-	KERNEL1x2_4(xxx)
-
-	je	.L2_46
-
-	jmp	.L2_42
-	ALIGN_4
-
-.L2_46:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_49
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_47:
-
-	KERNEL1x2_SUB(xxx)
-	addq	$2, BI
-	addq	$1, %rax
-	jl	.L2_47
-	ALIGN_4
-
-
-.L2_49:
-
-	vmovss	ALPHA, %xmm0
-
-#ifndef TRMMKERNEL
-
-	vfmaddss 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddss 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
-
-#else
-	vmulss	%xmm0, %xmm4,%xmm4
-	vmulss	%xmm0, %xmm5,%xmm5
-
-#endif
-
-	vmovss	%xmm4 ,  	(CO1)
-	vmovss	%xmm5 ,  	(CO1, LDC)
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax 
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO         
-        leaq    (AO, %rax, SIZE), AO
-#endif  
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $1, KK
-#endif
-
-	addq	$1 * SIZE, CO1		# coffset += 1
-	ALIGN_4
-	
-
-
-
-	
-.L2_60:
-#if defined(TRMMKERNEL) && !defined(LEFT)
-        addq    $2, KK
-#endif
-
-	decq	J			// j --
-	jg	.L2_01			// next 2 lines of N
-
-
-
-.L1_0:
-
-/************************************************************************************************
-* Loop for Nmod6 % 2 > 0
-*************************************************************************************************/
-
-	movq	Nmod6, J		
-	andq	$1, J			// j % 2
-	je	.L999
-	ALIGN_4
-
-.L1_01:
-	// copy to sub buffer
-	movq	B, BO1
-	leaq    BUFFER1, BO		// first buffer to BO
-	movq	K, %rax
-	ALIGN_4
-
-.L1_02b:
-
-	vmovss	(BO1), %xmm0
-	vmovss	%xmm0,       (BO)
-	addq	$1*SIZE,BO1
-	addq	$1*SIZE,BO
-	decq	%rax
-	jnz	.L1_02b
-
-.L1_02c:
-
-	movq	BO1, B			// next offset of B
-
-.L1_10:
-	movq	C, CO1
-	leaq	(C, LDC, 1), C		// c += 1 * ldc
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        movq    OFFSET, %rax
-        movq    %rax, KK
-#endif
-	
-	movq	A, AO		 	// aoffset = a
-	addq	$32 * SIZE, AO
-
-	movq	M,  I
-	sarq	$4, I			// i = (m >> 4)
-	je	.L1_20
-
-	ALIGN_4
-
-.L1_11:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BO, BI, SIZE), BO
-        salq    $4, %rax                        // rax = rax * 16 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $16, %rax	// number of values in AO
-#else
-        addq    $1, %rax	// number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-	andq	$-8, %rax			//  K = K - ( K % 8 )
-	je	.L1_16
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$4, %rax			// rax = rax * 16 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_12:
-
-	prefetcht0	B_PR1(BO,BI, SIZE)
-	KERNEL16x1_1(xxx)
-	KERNEL16x1_2(xxx)
-	KERNEL16x1_3(xxx)
-	KERNEL16x1_4(xxx)
-
-	KERNEL16x1_1(xxx)
-	KERNEL16x1_2(xxx)
-	KERNEL16x1_3(xxx)
-	KERNEL16x1_4(xxx)
-
-	je	.L1_16
-
-	KERNEL16x1_1(xxx)
-	KERNEL16x1_2(xxx)
-	KERNEL16x1_3(xxx)
-	KERNEL16x1_4(xxx)
-
-	KERNEL16x1_1(xxx)
-	KERNEL16x1_2(xxx)
-	KERNEL16x1_3(xxx)
-	KERNEL16x1_4(xxx)
-
-	je	.L1_16
-
-	jmp	.L1_12
-	ALIGN_4
-
-.L1_16:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_19
-
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$4, %rax			// rax = rax * 16 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_17:
-
-	KERNEL16x1_SUB(xxx)
-	addq	$1, BI
-	addq	$16, %rax
-	jl	.L1_17
-	ALIGN_4
-
-
-.L1_19:
-
-	vbroadcastss	ALPHA, %xmm0
-
-#ifndef TRMMKERNEL
-
-	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddps 4 * SIZE(CO1),%xmm0, %xmm7,%xmm7
-	vfmaddps 8 * SIZE(CO1),%xmm0, %xmm10,%xmm10
-	vfmaddps 12 * SIZE(CO1),%xmm0, %xmm13,%xmm13
-
-#else
-	vmulps	%xmm0, %xmm4,%xmm4
-	vmulps	%xmm0, %xmm7,%xmm7
-	vmulps	%xmm0, %xmm10,%xmm10
-	vmulps	%xmm0, %xmm13,%xmm13
-
-#endif
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm7 , 4 * SIZE(CO1)
-	vmovups	%xmm10, 8 * SIZE(CO1)
-	vmovups	%xmm13,12 * SIZE(CO1)
-
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BO, BI, SIZE), BO
-        salq    $4, %rax                        // rax = rax * 16 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $16, KK				
-#endif
-
-	addq	$16 * SIZE, CO1		# coffset += 16
-	decq	I			# i --
-	jg	.L1_11
-	ALIGN_4	
-
-/**************************************************************************
-* Rest of M 
-***************************************************************************/
-.L1_20:
-	// Test rest of M
-
-	testq	$15, M
-	jz	.L999
-
-	testq	$8, M		
-	jz	.L1_21pre
-	ALIGN_4
-
-/**************************************************************************/
-
-.L1_20_1:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BO, BI, SIZE), BO
-        salq    $3, %rax                        // rax = rax * 8 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $8, %rax        // number of values in A
-#else
-        addq    $1, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-
-	andq	$-8, %rax
-	je	.L1_20_6
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_20_2:
-
-	prefetcht0	B_PR1(BO,BI, SIZE)
-	KERNEL8x1_1(xxx)
-	KERNEL8x1_2(xxx)
-	KERNEL8x1_3(xxx)
-	KERNEL8x1_4(xxx)
-
-	KERNEL8x1_1(xxx)
-	KERNEL8x1_2(xxx)
-	KERNEL8x1_3(xxx)
-	KERNEL8x1_4(xxx)
-
-	je	.L1_20_6
-
-	KERNEL8x1_1(xxx)
-	KERNEL8x1_2(xxx)
-	KERNEL8x1_3(xxx)
-	KERNEL8x1_4(xxx)
-
-	KERNEL8x1_1(xxx)
-	KERNEL8x1_2(xxx)
-	KERNEL8x1_3(xxx)
-	KERNEL8x1_4(xxx)
-
-	je	.L1_20_6
-
-	jmp	.L1_20_2
-	ALIGN_4
-
-.L1_20_6:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_20_9
-
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_20_7:
-
-	KERNEL8x1_SUB(xxx)
-	addq	$1, BI
-	addq	$8, %rax
-	jl	.L1_20_7
-	ALIGN_4
-
-
-.L1_20_9:
-
-	vbroadcastss	ALPHA, %xmm0
-
-#ifndef TRMMKERNEL
-
-	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddps 4 * SIZE(CO1),%xmm0, %xmm7,%xmm7
-
-#else
-	vmulps	%xmm0, %xmm4,%xmm4
-	vmulps	%xmm0, %xmm7,%xmm7
-
-#endif
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm7 , 4 * SIZE(CO1)
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax 
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BO, BI, SIZE), BO         
-        salq    $3, %rax                        // rax = rax * 8 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif  
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $8, KK
-#endif
-
-	addq	$8 * SIZE, CO1		# coffset += 8
-	ALIGN_4
-	
-
-
-/**************************************************************************/
-
-.L1_21pre:
-
-	testq	$4, M		
-	jz	.L1_30
-	ALIGN_4
-
-.L1_21:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BO, BI, SIZE), BO
-        salq    $2, %rax                        // rax = rax * 4 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $4, %rax        // number of values in A
-#else
-        addq    $1, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-
-	andq	$-8, %rax
-	je	.L1_26
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_22:
-
-	prefetcht0	B_PR1(BO,BI, SIZE)
-	KERNEL4x1_1(xxx)
-	KERNEL4x1_2(xxx)
-	KERNEL4x1_3(xxx)
-	KERNEL4x1_4(xxx)
-
-	KERNEL4x1_1(xxx)
-	KERNEL4x1_2(xxx)
-	KERNEL4x1_3(xxx)
-	KERNEL4x1_4(xxx)
-
-	je	.L1_26
-
-	KERNEL4x1_1(xxx)
-	KERNEL4x1_2(xxx)
-	KERNEL4x1_3(xxx)
-	KERNEL4x1_4(xxx)
-
-	KERNEL4x1_1(xxx)
-	KERNEL4x1_2(xxx)
-	KERNEL4x1_3(xxx)
-	KERNEL4x1_4(xxx)
-
-	je	.L1_26
-
-	jmp	.L1_22
-	ALIGN_4
-
-.L1_26:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_29
-
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_27:
-
-	KERNEL4x1_SUB(xxx)
-	addq	$1, BI
-	addq	$4, %rax
-	jl	.L1_27
-	ALIGN_4
-
-
-.L1_29:
-
-	vbroadcastss	ALPHA, %xmm0
-
-#ifndef TRMMKERNEL
-
-	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
-
-#else
-	vmulps	%xmm0, %xmm4,%xmm4
-
-#endif
-
-	vmovups	%xmm4 ,  	(CO1)
-
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax 
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BO, BI, SIZE), BO         
-        salq    $2, %rax                        // rax = rax * 4 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif  
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $4, KK
-#endif
-
-	addq	$4 * SIZE, CO1		# coffset += 4
-	ALIGN_4
-	
-
-.L1_30:
-	testq	$2, M		
-	jz	.L1_40
-
-	ALIGN_4
-
-.L1_31:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BO, BI, SIZE), BO
-        salq    $1, %rax                        // rax = rax * 2 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $2, %rax        // number of values in AO
-#else
-        addq    $1, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-
-	andq	$-8, %rax
-	je	.L1_36
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_32:
-
-	prefetcht0	B_PR1(BO,BI,SIZE)
-	KERNEL2x1_1(xxx)
-	KERNEL2x1_2(xxx)
-	KERNEL2x1_3(xxx)
-	KERNEL2x1_4(xxx)
-
-	KERNEL2x1_1(xxx)
-	KERNEL2x1_2(xxx)
-	KERNEL2x1_3(xxx)
-	KERNEL2x1_4(xxx)
-
-	je	.L1_36
-
-	KERNEL2x1_1(xxx)
-	KERNEL2x1_2(xxx)
-	KERNEL2x1_3(xxx)
-	KERNEL2x1_4(xxx)
-
-	KERNEL2x1_1(xxx)
-	KERNEL2x1_2(xxx)
-	KERNEL2x1_3(xxx)
-	KERNEL2x1_4(xxx)
-
-	je	.L1_36
-
-	jmp	.L1_32
-	ALIGN_4
-
-.L1_36:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_39
-
-	movq    %rax, BI                        //  Index for BO
-	
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_37:
-
-	KERNEL2x1_SUB(xxx)
-	addq	$1, BI
-	addq	$2, %rax
-	jl	.L1_37
-	ALIGN_4
-
-
-.L1_39:
-
-	vmovss	ALPHA, %xmm0
-
-#ifndef TRMMKERNEL
-
-	vfmaddss 	 (CO1),%xmm0, %xmm4,%xmm4
-	vfmaddss 1 * SIZE(CO1),%xmm0, %xmm8,%xmm8
-
-#else
-	vmulss	%xmm0, %xmm4,%xmm4
-	vmulss	%xmm0, %xmm8,%xmm8
-
-#endif
-
-	vmovss	%xmm4 ,  	(CO1)
-	vmovss	%xmm8 , 1 * SIZE(CO1)
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax 
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BO, BI, SIZE), BO         
-        salq    $1, %rax                        // rax = rax * 2 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif  
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $2, KK
-#endif
-
-	addq	$2 * SIZE, CO1		# coffset += 2
-	ALIGN_4
-
-.L1_40:
-	testq	$1, M		
-	jz	.L999
-
-	ALIGN_4
-
-.L1_41:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $2 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BO, BI, SIZE), BO
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $1, %rax        // number of values in AO
-#else
-        addq    $1, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-	andq	$-8, %rax
-	je	.L1_46
-	movq    %rax, BI                        //  Index for BO
-
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_42:
-
-	KERNEL1x1_1(xxx)
-	KERNEL1x1_2(xxx)
-	KERNEL1x1_3(xxx)
-	KERNEL1x1_4(xxx)
-
-	KERNEL1x1_1(xxx)
-	KERNEL1x1_2(xxx)
-	KERNEL1x1_3(xxx)
-	KERNEL1x1_4(xxx)
-
-	je	.L1_46
-
-	KERNEL1x1_1(xxx)
-	KERNEL1x1_2(xxx)
-	KERNEL1x1_3(xxx)
-	KERNEL1x1_4(xxx)
-
-	KERNEL1x1_1(xxx)
-	KERNEL1x1_2(xxx)
-	KERNEL1x1_3(xxx)
-	KERNEL1x1_4(xxx)
-
-	je	.L1_46
-
-	jmp	.L1_42
-	ALIGN_4
-
-.L1_46:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_49
-
-	movq    %rax, BI                        //  Index for BO
-
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_47:
-
-	KERNEL1x1_SUB(xxx)
-	addq	$1, BI
-	addq	$1, %rax
-	jl	.L1_47
-	ALIGN_4
-
-
-.L1_49:
-
-	vmovss	ALPHA, %xmm0
-
-#ifndef TRMMKERNEL
-
-	vfmaddss 	 (CO1),%xmm0, %xmm4,%xmm4
-
-#else
-	vmulss	%xmm0, %xmm4,%xmm4
-
-#endif
-
-	vmovss	%xmm4 ,  	(CO1)
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax 
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BO, BI, SIZE), BO         
-        leaq    (AO, %rax, SIZE), AO
-#endif  
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $1, KK
-#endif
-
-	addq	$1 * SIZE, CO1		# coffset += 1
-	ALIGN_4
-	
-
-.L999:
-	movq   		SP, %rsp
-	movq	   (%rsp), %rbx
-	movq	  8(%rsp), %rbp
-	movq	 16(%rsp), %r12
-	movq	 24(%rsp), %r13
-	movq	 32(%rsp), %r14
-	movq	 40(%rsp), %r15
-
-#ifdef WINDOWS_ABI
-	movq	 48(%rsp), %rdi
-	movq	 56(%rsp), %rsi
-	movups	 64(%rsp), %xmm6
-	movups	 80(%rsp), %xmm7
-	movups	 96(%rsp), %xmm8
-	movups	112(%rsp), %xmm9
-	movups	128(%rsp), %xmm10
-	movups	144(%rsp), %xmm11
-	movups	160(%rsp), %xmm12
-	movups	176(%rsp), %xmm13
-	movups	192(%rsp), %xmm14
-	movups	208(%rsp), %xmm15
-#endif
-
-	addq	$STACKSIZE, %rsp
-	ret
-
-	EPILOGUE
-
-
-
-
-
-#endif
+/***************************************************************************
+Copyright (c) 2013, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+/*********************************************************************
+*
+* 2013/10/18 Saar
+*        BLASTEST               : OK
+*        CTEST                  : OK
+*        TEST                   : OK
+*
+*
+* 2013/10/29 Saar
+*
+* Parameter:
+*       UNROLL_M        16
+*       UNROLL_N        2
+*       SGEMM_P         768
+*       SGEMM_Q         192
+*       SGEMM_R         12288
+*       A_PR1           384
+*       B_PR1           192
+*
+* Performance at m x n on AMD 8320 (ACML-Version: 5.3.1):
+* 
+* 6144x6144    168.2    GFLOPS with 8 threads on 4 modules (ACML: 158.0 ) (BULLDOZER: 167.4 )
+* 6144x6144    162.7    GFLOPS with 4 threads on 4 modules (ACML: 157.6 ) (BULLDOZER: 159.0 )
+* 6144x6144     82.0    GFLOPS with 2 threads on 2 modules (ACML:  81.4 ) (BULLDOZER:  80.3 )
+* 6144x6144     41.3    GFLOPS with 1 threads on 1 modules (ACML:  41.1 ) (BULLDOZER:  40.4 )
+*
+* Performance at m x n on AMD 6380  (ACML-Version: 5.3.1):
+* 
+* 12288x12288  469.5    GFLOPS with 32 threads on 16 modules (ACML: 375.3 ) (BULLDOZER: 445.5 )
+* 12288x12288  442.9    GFLOPS with 16 threads on 16 modules (ACML: 378.5 ) (BULLDOZER: 416.3 )
+* 12288x12288  265.1    GFLOPS with  8 threads on  8 modules (ACML: 218.5 ) (BULLDOZER: 261.5 )
+* 6144x6144    139.7    GFLOPS with  4 threads on  4 modules (ACML: 116.0 ) (BULLDOZER: 137.7 )
+* 6144x6144     70.9    GFLOPS with  2 threads on  2 modules (ACML:  67.4 ) (BULLDOZER:  69.5 )
+* 6144x6144     35.6    GFLOPS with  1 threads on  1 modules (ACML:  36.1 ) (BULLDOZER:  35.1 )
+*
+*********************************************************************/
+
+
+
+#define ASSEMBLER
+#include "common.h"
+ 
+#define OLD_M	%rdi
+#define OLD_N	%rsi
+#define M	%r13
+#define J	%r14
+#define OLD_K	%rdx
+
+#define A	%rcx
+#define B	%r8
+#define C	%r9
+#define LDC	%r10
+	
+#define I	%r11
+#define AO	%rdi
+#define BO	%rsi
+#define	CO1	%r15
+#define K	%r12
+#define BI	%rbp
+#define	SP	%rbx
+
+#define BO1	%rdi
+#define BO2	%r15
+
+#ifndef WINDOWS_ABI
+
+#define STACKSIZE 96
+
+#else
+
+#define STACKSIZE 256
+
+#define OLD_A		40 + STACKSIZE(%rsp)
+#define OLD_B		48 + STACKSIZE(%rsp)
+#define OLD_C		56 + STACKSIZE(%rsp)
+#define OLD_LDC		64 + STACKSIZE(%rsp)
+#define OLD_OFFSET	72 + STACKSIZE(%rsp)
+
+#endif
+
+#define L_BUFFER_SIZE 8192
+#define LB2_OFFSET    4096
+
+#define Ndiv6	 24(%rsp)
+#define Nmod6	 32(%rsp)
+#define N	 40(%rsp)
+#define ALPHA	 48(%rsp)
+#define OFFSET	 56(%rsp)
+#define KK	 64(%rsp)
+#define KKK	 72(%rsp)
+#define BUFFER1	           128(%rsp)
+#define BUFFER2	LB2_OFFSET+128(%rsp)
+
+#if defined(OS_WINDOWS)
+#if   L_BUFFER_SIZE > 16384
+#define STACK_TOUCH \
+        movl    $0,  4096 * 4(%rsp);\
+        movl    $0,  4096 * 3(%rsp);\
+        movl    $0,  4096 * 2(%rsp);\
+        movl    $0,  4096 * 1(%rsp);
+#elif L_BUFFER_SIZE > 12288
+#define STACK_TOUCH \
+        movl    $0,  4096 * 3(%rsp);\
+        movl    $0,  4096 * 2(%rsp);\
+        movl    $0,  4096 * 1(%rsp);
+#elif L_BUFFER_SIZE > 8192
+#define STACK_TOUCH \
+        movl    $0,  4096 * 2(%rsp);\
+        movl    $0,  4096 * 1(%rsp);
+#elif L_BUFFER_SIZE > 4096
+#define STACK_TOUCH \
+        movl    $0,  4096 * 1(%rsp);
+#else
+#define STACK_TOUCH
+#endif
+#else
+#define STACK_TOUCH
+#endif
+
+
+
+#define	A_PR1	384
+#define	B_PR1	192
+
+/*******************************************************************************************
+* 3 lines of N
+*******************************************************************************************/
+
+#define KERNEL16x3_1(xx) \
+	vbroadcastss	 -6 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vbroadcastss	 -5 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm3 ;\
+	nop						;\
+	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	prefetcht0	A_PR1(AO,%rax,SIZE)	;\
+	vfmaddps  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
+	vmovups 	-24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
+	vfmaddps  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
+	vfmaddps  	%xmm12,%xmm3,%xmm0,%xmm12 ;\
+	vmovups 	-20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
+	vfmaddps  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
+	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm15,%xmm3,%xmm0,%xmm15 ;\
+
+#define KERNEL16x3_2(xx) \
+	vmovups 	-16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vbroadcastss	 -1 * SIZE(BO, BI, SIZE), %xmm3 ;\
+	nop						;\
+	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+	vmovups 	-12 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	prefetcht0	A_PR1+64(AO,%rax,SIZE)	;\
+	vfmaddps  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
+	vmovups 	 -8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
+	vfmaddps  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
+	vfmaddps  	%xmm12,%xmm3,%xmm0,%xmm12 ;\
+	vmovups 	 -4 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
+	vfmaddps  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
+	vbroadcastss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vbroadcastss	  1 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm15,%xmm3,%xmm0,%xmm15 ;\
+
+#define KERNEL16x3_3(xx) \
+	vmovups 	  0 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vbroadcastss	  2 * SIZE(BO, BI, SIZE), %xmm3 ;\
+	nop						;\
+	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+	vmovups 	  4 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	prefetcht0	A_PR1+128(AO,%rax,SIZE)	;\
+	vfmaddps  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
+	vmovups 	  8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
+	vfmaddps  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
+	vfmaddps  	%xmm12,%xmm3,%xmm0,%xmm12 ;\
+	vmovups 	 12 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
+	vfmaddps  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
+	vbroadcastss	  3 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vbroadcastss	  4 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm15,%xmm3,%xmm0,%xmm15 ;\
+
+#define KERNEL16x3_4(xx) \
+	vmovups 	 16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vbroadcastss	  5 * SIZE(BO, BI, SIZE), %xmm3 ;\
+	nop						;\
+	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+	vmovups 	 20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	prefetcht0	A_PR1+192(AO,%rax,SIZE)	;\
+	vfmaddps  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
+	vmovups 	 24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
+	vfmaddps  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
+	addq	$12, BI				  ;\
+	vfmaddps  	%xmm12,%xmm3,%xmm0,%xmm12 ;\
+	vmovups 	 28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
+	vfmaddps  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
+	addq	$64, %rax 			  ;\
+	vfmaddps  	%xmm15,%xmm3,%xmm0,%xmm15 ;\
+
+#define KERNEL16x3_SUB(xx) \
+	vbroadcastss	 -6 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vbroadcastss	 -5 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm3 ;\
+	nop						;\
+	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	vfmaddps  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
+	vmovups 	-24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
+	vfmaddps  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
+	vfmaddps  	%xmm12,%xmm3,%xmm0,%xmm12 ;\
+	vmovups 	-20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
+	vfmaddps  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
+	vfmaddps  	%xmm15,%xmm3,%xmm0,%xmm15 ;\
+
+
+/*******************************************************************************************/
+
+#define KERNEL8x3_1(xx) \
+	prefetcht0	A_PR1(AO,%rax,SIZE)	;\
+	vbroadcastss	 -6 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vbroadcastss	 -5 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm3 ;\
+	nop						;\
+	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	vfmaddps  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
+
+#define KERNEL8x3_2(xx) \
+	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vbroadcastss	 -1 * SIZE(BO, BI, SIZE), %xmm3 ;\
+	nop						;\
+	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+	vmovups 	-20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	vfmaddps  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
+
+#define KERNEL8x3_3(xx) \
+	prefetcht0	A_PR1+64(AO,%rax,SIZE)	;\
+	vbroadcastss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vbroadcastss	  1 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vbroadcastss	  2 * SIZE(BO, BI, SIZE), %xmm3 ;\
+	nop						;\
+	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+	vmovups 	-12 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	vfmaddps  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
+
+#define KERNEL8x3_4(xx) \
+	vbroadcastss	  3 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	 -8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vbroadcastss	  4 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vbroadcastss	  5 * SIZE(BO, BI, SIZE), %xmm3 ;\
+	nop						;\
+	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+	vmovups 	 -4 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	vfmaddps  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
+	addq	$12, BI				  ;\
+	addq	$32, %rax 			  ;\
+
+#define KERNEL8x3_SUB(xx) \
+	vbroadcastss	 -6 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vbroadcastss	 -5 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm3 ;\
+	nop						;\
+	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	vfmaddps  	%xmm9,%xmm3,%xmm0,%xmm9 ;\
+
+
+/*******************************************************************************************/
+
+#define KERNEL4x3_1(xx) \
+	prefetcht0	A_PR1(AO,%rax,SIZE)	;\
+	vbroadcastss	 -6 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vbroadcastss	 -5 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm3 ;\
+	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+
+#define KERNEL4x3_2(xx) \
+	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vbroadcastss	 -1 * SIZE(BO, BI, SIZE), %xmm3 ;\
+	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+
+#define KERNEL4x3_3(xx) \
+	vbroadcastss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vbroadcastss	  1 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vbroadcastss	  2 * SIZE(BO, BI, SIZE), %xmm3 ;\
+	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+
+#define KERNEL4x3_4(xx) \
+	vbroadcastss	  3 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vbroadcastss	  4 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vbroadcastss	  5 * SIZE(BO, BI, SIZE), %xmm3 ;\
+	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+	addq	$12, BI				  ;\
+	addq	$16, %rax 			  ;\
+
+#define KERNEL4x3_SUB(xx) \
+	vbroadcastss	 -6 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vbroadcastss	 -5 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm3 ;\
+	vfmaddps  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+
+/*******************************************************************************************/
+
+#define KERNEL2x3_1(xx) \
+	vmovss	 -6 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovss	 -5 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovss	 -4 * SIZE(BO, BI, SIZE), %xmm3 ;\
+	vfmaddss  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+	vmovss 	-31 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
+	vfmaddss  	%xmm10,%xmm2,%xmm0,%xmm10 ;\
+	vfmaddss  	%xmm12,%xmm3,%xmm0,%xmm12 ;\
+
+#define KERNEL2x3_2(xx) \
+	vmovss	 -3 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-30 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovss	 -2 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovss	 -1 * SIZE(BO, BI, SIZE), %xmm3 ;\
+	vfmaddss  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+	vmovss 	-29 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
+	vfmaddss  	%xmm10,%xmm2,%xmm0,%xmm10 ;\
+	vfmaddss  	%xmm12,%xmm3,%xmm0,%xmm12 ;\
+
+#define KERNEL2x3_3(xx) \
+	vmovss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovss	  1 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovss	  2 * SIZE(BO, BI, SIZE), %xmm3 ;\
+	vfmaddss  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+	vmovss 	-27 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
+	vfmaddss  	%xmm10,%xmm2,%xmm0,%xmm10 ;\
+	vfmaddss  	%xmm12,%xmm3,%xmm0,%xmm12 ;\
+
+#define KERNEL2x3_4(xx) \
+	vmovss	  3 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-26 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovss	  4 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovss	  5 * SIZE(BO, BI, SIZE), %xmm3 ;\
+	vfmaddss  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+	vmovss 	-25 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
+	vfmaddss  	%xmm10,%xmm2,%xmm0,%xmm10 ;\
+	vfmaddss  	%xmm12,%xmm3,%xmm0,%xmm12 ;\
+	addq	$12, BI				  ;\
+	addq	$8, %rax 			  ;\
+
+#define KERNEL2x3_SUB(xx) \
+	vmovss	 -6 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovss	 -5 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovss	 -4 * SIZE(BO, BI, SIZE), %xmm3 ;\
+	vfmaddss  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+	vmovss 	-31 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
+	vfmaddss  	%xmm10,%xmm2,%xmm0,%xmm10 ;\
+	vfmaddss  	%xmm12,%xmm3,%xmm0,%xmm12 ;\
+
+/*******************************************************************************************/
+
+#define KERNEL1x3_1(xx) \
+	vmovss	 -6 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovss	 -5 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovss	 -4 * SIZE(BO, BI, SIZE), %xmm3 ;\
+	vfmaddss  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+
+#define KERNEL1x3_2(xx) \
+	vmovss	 -3 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-31 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovss	 -2 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovss	 -1 * SIZE(BO, BI, SIZE), %xmm3 ;\
+	vfmaddss  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+
+#define KERNEL1x3_3(xx) \
+	vmovss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-30 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovss	  1 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovss	  2 * SIZE(BO, BI, SIZE), %xmm3 ;\
+	vfmaddss  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+
+#define KERNEL1x3_4(xx) \
+	vmovss	  3 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-29 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovss	  4 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovss	  5 * SIZE(BO, BI, SIZE), %xmm3 ;\
+	vfmaddss  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+	addq	$12, BI				  ;\
+	addq	$4, %rax 			  ;\
+
+#define KERNEL1x3_SUB(xx) \
+	vmovss	 -6 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovss	 -5 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovss	 -4 * SIZE(BO, BI, SIZE), %xmm3 ;\
+	vfmaddss  	%xmm6,%xmm3,%xmm0,%xmm6 ;\
+
+/*******************************************************************************************/
+
+/*******************************************************************************************
+* 2 lines of N
+*******************************************************************************************/
+
+#define KERNEL16x2_1(xx) \
+	prefetcht0	A_PR1(AO,%rax,SIZE)	;\
+	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	vmovups 	-24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
+	vfmaddps  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
+	vmovups 	-20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
+	vfmaddps  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
+
+#define KERNEL16x2_2(xx) \
+	prefetcht0	A_PR1+64(AO,%rax,SIZE)	;\
+	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vbroadcastss	 -1 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovups 	-12 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	vmovups 	 -8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
+	vfmaddps  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
+	vmovups 	 -4 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
+	vfmaddps  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
+
+#define KERNEL16x2_3(xx) \
+	prefetcht0	A_PR1+128(AO,%rax,SIZE)	;\
+	vbroadcastss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	  0 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vbroadcastss	  1 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovups 	  4 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	vmovups 	  8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
+	vfmaddps  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
+	vmovups 	 12 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
+	vfmaddps  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
+
+#define KERNEL16x2_4(xx) \
+	prefetcht0	A_PR1+192(AO,%rax,SIZE)	;\
+	vbroadcastss	  2 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	 16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vbroadcastss	  3 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovups 	 20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	vmovups 	 24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
+	vfmaddps  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
+	vmovups 	 28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
+	vfmaddps  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
+	addq	$8, BI				  ;\
+	addq	$64, %rax 			  ;\
+
+#define KERNEL16x2_SUB(xx) \
+	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	vmovups 	-24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
+	vfmaddps  	%xmm11,%xmm2,%xmm0,%xmm11 ;\
+	vmovups 	-20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
+	vfmaddps  	%xmm14,%xmm2,%xmm0,%xmm14 ;\
+
+
+/*******************************************************************************************/
+
+#define KERNEL8x2_1(xx) \
+	prefetcht0	A_PR1(AO,%rax,SIZE)	;\
+	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+
+#define KERNEL8x2_2(xx) \
+	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vbroadcastss	 -1 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovups 	-20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+
+#define KERNEL8x2_3(xx) \
+	prefetcht0	A_PR1+64(AO,%rax,SIZE)	;\
+	vbroadcastss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vbroadcastss	  1 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovups 	-12 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+
+#define KERNEL8x2_4(xx) \
+	vbroadcastss	  2 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	 -8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vbroadcastss	  3 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovups 	 -4 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+	addq	$8, BI				  ;\
+	addq	$32, %rax 			  ;\
+
+#define KERNEL8x2_SUB(xx) \
+	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vfmaddps  	%xmm8,%xmm2,%xmm0,%xmm8 ;\
+
+
+/*******************************************************************************************/
+
+#define KERNEL4x2_1(xx) \
+	prefetcht0	A_PR1(AO,%rax,SIZE)	;\
+	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+
+#define KERNEL4x2_2(xx) \
+	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vbroadcastss	 -1 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+
+#define KERNEL4x2_3(xx) \
+	vbroadcastss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vbroadcastss	  1 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+
+#define KERNEL4x2_4(xx) \
+	vbroadcastss	  2 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vbroadcastss	  3 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	addq	$8, BI				  ;\
+	addq	$16, %rax 			  ;\
+
+#define KERNEL4x2_SUB(xx) \
+	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddps  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+
+/*******************************************************************************************/
+
+#define KERNEL2x2_1(xx) \
+	vmovss	 -4 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovss	 -3 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovss 	-31 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
+	vfmaddss  	%xmm10,%xmm2,%xmm0,%xmm10 ;\
+
+#define KERNEL2x2_2(xx) \
+	vmovss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-30 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovss	 -1 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovss 	-29 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
+	vfmaddss  	%xmm10,%xmm2,%xmm0,%xmm10 ;\
+
+#define KERNEL2x2_3(xx) \
+	vmovss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovss	  1 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovss 	-27 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
+	vfmaddss  	%xmm10,%xmm2,%xmm0,%xmm10 ;\
+
+#define KERNEL2x2_4(xx) \
+	vmovss	  2 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-26 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovss	  3 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovss 	-25 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
+	vfmaddss  	%xmm10,%xmm2,%xmm0,%xmm10 ;\
+	addq	$8, BI				  ;\
+	addq	$8, %rax 			  ;\
+
+#define KERNEL2x2_SUB(xx) \
+	vmovss	 -4 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovss	 -3 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	vmovss 	-31 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
+	vfmaddss  	%xmm10,%xmm2,%xmm0,%xmm10 ;\
+
+/*******************************************************************************************/
+
+#define KERNEL1x2_1(xx) \
+	vmovss	 -4 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovss	 -3 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+
+#define KERNEL1x2_2(xx) \
+	vmovss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-31 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovss	 -1 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+
+#define KERNEL1x2_3(xx) \
+	vmovss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-30 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovss	  1 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+
+#define KERNEL1x2_4(xx) \
+	vmovss	  2 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-29 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovss	  3 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+	addq	$8, BI				  ;\
+	addq	$4, %rax 			  ;\
+
+#define KERNEL1x2_SUB(xx) \
+	vmovss	 -4 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovss	 -3 * SIZE(BO, BI, SIZE), %xmm2 ;\
+	vfmaddss  	%xmm5,%xmm2,%xmm0,%xmm5 ;\
+
+/*******************************************************************************************/
+
+/*******************************************************************************************
+* 1 line of N
+*******************************************************************************************/
+
+#define KERNEL16x1_1(xx) \
+	prefetcht0	A_PR1(AO,%rax,SIZE)	;\
+	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vmovups 	-24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
+	vmovups 	-20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
+
+#define KERNEL16x1_2(xx) \
+	prefetcht0	A_PR1+64(AO,%rax,SIZE)	;\
+	vbroadcastss	 -1 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovups 	-12 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vmovups 	 -8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
+	vmovups 	 -4 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
+
+#define KERNEL16x1_3(xx) \
+	prefetcht0	A_PR1+128(AO,%rax,SIZE)	;\
+	vbroadcastss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	  0 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovups 	  4 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vmovups 	  8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
+	vmovups 	 12 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
+
+#define KERNEL16x1_4(xx) \
+	prefetcht0	A_PR1+192(AO,%rax,SIZE)	;\
+	vbroadcastss	  1 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	 16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovups 	 20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vmovups 	 24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
+	vmovups 	 28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
+	addq	$4, BI				  ;\
+	addq	$64, %rax 			  ;\
+
+#define KERNEL16x1_SUB(xx) \
+	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	vmovups 	-24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm10,%xmm1,%xmm0,%xmm10 ;\
+	vmovups 	-20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm13,%xmm1,%xmm0,%xmm13 ;\
+
+
+/*******************************************************************************************/
+
+#define KERNEL8x1_1(xx) \
+	prefetcht0	A_PR1(AO,%rax,SIZE)	;\
+	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+
+#define KERNEL8x1_2(xx) \
+	vbroadcastss	 -1 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovups 	-20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+
+#define KERNEL8x1_3(xx) \
+	prefetcht0	A_PR1+64(AO,%rax,SIZE)	;\
+	vbroadcastss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-16 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovups 	-12 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+
+#define KERNEL8x1_4(xx) \
+	vbroadcastss	  1 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	 -8 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovups 	 -4 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+	addq	$4, BI				  ;\
+	addq	$32, %rax 			  ;\
+
+#define KERNEL8x1_SUB(xx) \
+	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm7,%xmm1,%xmm0,%xmm7 ;\
+
+
+/*******************************************************************************************/
+
+#define KERNEL4x1_1(xx) \
+	prefetcht0	A_PR1(AO,%rax,SIZE)	;\
+	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+
+#define KERNEL4x1_2(xx) \
+	vbroadcastss	 -1 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+
+#define KERNEL4x1_3(xx) \
+	vbroadcastss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-24 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+
+#define KERNEL4x1_4(xx) \
+	vbroadcastss	  1 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-20 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	addq	$4, BI				  ;\
+	addq	$16, %rax 			  ;\
+
+#define KERNEL4x1_SUB(xx) \
+	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovups 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddps  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+
+/*******************************************************************************************/
+
+#define KERNEL2x1_1(xx) \
+	vmovss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovss 	-31 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
+
+#define KERNEL2x1_2(xx) \
+	vmovss	 -1 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-30 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovss 	-29 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
+
+#define KERNEL2x1_3(xx) \
+	vmovss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-28 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovss 	-27 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
+
+#define KERNEL2x1_4(xx) \
+	vmovss	  1 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-26 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovss 	-25 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
+	addq	$4, BI				  ;\
+	addq	$8, %rax 			  ;\
+
+#define KERNEL2x1_SUB(xx) \
+	vmovss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	vmovss 	-31 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm8,%xmm1,%xmm0,%xmm8 ;\
+
+/*******************************************************************************************/
+
+#define KERNEL1x1_1(xx) \
+	vmovss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+
+#define KERNEL1x1_2(xx) \
+	vmovss	 -1 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-31 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+
+#define KERNEL1x1_3(xx) \
+	vmovss	  0 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-30 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+
+#define KERNEL1x1_4(xx) \
+	vmovss	  1 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-29 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+	addq	$4, BI				  ;\
+	addq	$4, %rax 			  ;\
+
+#define KERNEL1x1_SUB(xx) \
+	vmovss	 -2 * SIZE(BO, BI, SIZE), %xmm1 ;\
+	vmovss 	-32 * SIZE(AO, %rax, SIZE), %xmm0 ;\
+	vfmaddss  	%xmm4,%xmm1,%xmm0,%xmm4 ;\
+
+/*******************************************************************************************/
+
+#if !defined(TRMMKERNEL)
+
+
+	PROLOGUE
+	PROFCODE
+	
+	subq	$STACKSIZE, %rsp
+	movq	%rbx,   (%rsp)
+	movq	%rbp,  8(%rsp)
+	movq	%r12, 16(%rsp)
+	movq	%r13, 24(%rsp)
+	movq	%r14, 32(%rsp)
+	movq	%r15, 40(%rsp)
+
+	vzeroupper
+
+#ifdef WINDOWS_ABI
+	movq	%rdi,    48(%rsp)
+	movq	%rsi,    56(%rsp)
+	movups	%xmm6,   64(%rsp)
+	movups	%xmm7,   80(%rsp)
+	movups	%xmm8,   96(%rsp)
+	movups	%xmm9,  112(%rsp)
+	movups	%xmm10, 128(%rsp)
+	movups	%xmm11, 144(%rsp)
+	movups	%xmm12, 160(%rsp)
+	movups	%xmm13, 176(%rsp)
+	movups	%xmm14, 192(%rsp)
+	movups	%xmm15, 208(%rsp)
+
+	movq	ARG1,      OLD_M
+	movq	ARG2,      OLD_N
+	movq	ARG3,      OLD_K
+	movq	OLD_A,     A
+	movq	OLD_B,     B
+	movq	OLD_C,     C
+	movq	OLD_LDC,   LDC
+
+	vmovaps	%xmm3, %xmm0
+
+#else
+	movq	STACKSIZE +  8(%rsp), LDC
+
+#endif
+
+	movq    %rsp, SP      # save old stack
+        subq    $128 + L_BUFFER_SIZE, %rsp
+        andq    $-4096, %rsp    # align stack
+
+        STACK_TOUCH
+
+	cmpq	$0, OLD_M
+	je	.L999
+
+	cmpq	$0, OLD_N
+	je	.L999
+
+	cmpq	$0, OLD_K
+	je	.L999
+
+	movq	OLD_M, M
+	movq	OLD_N, N
+	movq	OLD_K, K
+
+	vmovsd	 %xmm0, ALPHA
+
+	salq	$BASE_SHIFT, LDC
+
+	movq    N, %rax
+        xorq    %rdx, %rdx
+        movq    $6,  %rdi
+        divq    %rdi                    //    N / 6
+        movq    %rax, Ndiv6             //    N / 6
+        movq    %rdx, Nmod6             //    N % 6
+
+
+	movq	Ndiv6,  J
+	cmpq	$0, J
+	je	.L2_0
+	ALIGN_4
+
+.L6_01:
+        // copy to sub buffer
+        movq    K, %rax
+        salq    $1,%rax                 // K * 2 ; read 2 values
+        movq    B, BO1
+        leaq    (B,%rax, SIZE), BO2     // next offset to BO2
+        leaq    BUFFER1, BO             // first buffer to BO
+        movq    K, %rax
+	sarq	$3 , %rax		// K / 8
+	jz	.L6_01a_2
+        ALIGN_4
+
+.L6_01a_1:
+
+        prefetcht0 512(BO1)
+        prefetcht0 512(BO2)
+        prefetchw  512(BO)
+
+	vmovsd	0 * SIZE(BO1), %xmm0
+	vmovsd	2 * SIZE(BO1), %xmm2
+	vmovsd	4 * SIZE(BO1), %xmm4
+	vmovsd	6 * SIZE(BO1), %xmm6
+	vmovss  0 * SIZE(BO2), %xmm1
+	vmovss  2 * SIZE(BO2), %xmm3
+	vmovss  4 * SIZE(BO2), %xmm5
+	vmovss  6 * SIZE(BO2), %xmm7
+	vmovsd	%xmm0, 0*SIZE(BO)
+	vmovss	%xmm1, 2*SIZE(BO)
+	vmovsd	%xmm2, 3*SIZE(BO)
+	vmovss	%xmm3, 5*SIZE(BO)
+	vmovsd	%xmm4, 6*SIZE(BO)
+	vmovss	%xmm5, 8*SIZE(BO)
+	vmovsd	%xmm6, 9*SIZE(BO)
+	vmovss	%xmm7,11*SIZE(BO)
+	addq	$8*SIZE,BO1
+	addq	$8*SIZE,BO2
+	addq	$12*SIZE,BO
+
+	vmovsd	0 * SIZE(BO1), %xmm0
+	vmovsd	2 * SIZE(BO1), %xmm2
+	vmovsd	4 * SIZE(BO1), %xmm4
+	vmovsd	6 * SIZE(BO1), %xmm6
+	vmovss  0 * SIZE(BO2), %xmm1
+	vmovss  2 * SIZE(BO2), %xmm3
+	vmovss  4 * SIZE(BO2), %xmm5
+	vmovss  6 * SIZE(BO2), %xmm7
+	vmovsd	%xmm0, 0*SIZE(BO)
+	vmovss	%xmm1, 2*SIZE(BO)
+	vmovsd	%xmm2, 3*SIZE(BO)
+	vmovss	%xmm3, 5*SIZE(BO)
+	vmovsd	%xmm4, 6*SIZE(BO)
+	vmovss	%xmm5, 8*SIZE(BO)
+	vmovsd	%xmm6, 9*SIZE(BO)
+	vmovss	%xmm7,11*SIZE(BO)
+	addq	$8*SIZE,BO1
+	addq	$8*SIZE,BO2
+	addq	$12*SIZE,BO
+
+	decq	%rax
+	jnz	.L6_01a_1
+
+
+
+.L6_01a_2:
+
+	movq    K, %rax
+        andq    $7, %rax                // K % 8
+        jz      .L6_02c
+        ALIGN_4
+
+
+.L6_02b:
+
+	vmovsd	0 * SIZE(BO1), %xmm0
+	vmovss  0 * SIZE(BO2), %xmm2
+	vmovsd	%xmm0, 0*SIZE(BO)
+	vmovss	%xmm2, 2*SIZE(BO)
+	addq	$2*SIZE,BO1
+	addq	$2*SIZE,BO2
+	addq	$3*SIZE,BO
+	decq	%rax
+	jnz	.L6_02b
+
+.L6_02c:
+
+	movq	K, %rax
+	salq	$1,%rax			// K * 2
+	leaq	(B,%rax, SIZE), BO1	// next offset to BO1
+	leaq	(BO1,%rax, SIZE), BO2	// next offset to BO2
+	leaq    BUFFER2, BO		// second buffer to BO
+	movq	K, %rax
+	sarq	$3 , %rax		// K / 8
+	jz	.L6_02c_2
+	ALIGN_4
+
+.L6_02c_1:
+
+	prefetcht0 512(BO2)
+        prefetchw  512(BO)
+
+	vmovsd	0 * SIZE(BO2), %xmm0
+	vmovsd	2 * SIZE(BO2), %xmm2
+	vmovsd	4 * SIZE(BO2), %xmm4
+	vmovsd	6 * SIZE(BO2), %xmm6
+	vmovss  1 * SIZE(BO1), %xmm1
+	vmovss  3 * SIZE(BO1), %xmm3
+	vmovss  5 * SIZE(BO1), %xmm5
+	vmovss  7 * SIZE(BO1), %xmm7
+	vmovss	%xmm1, 0*SIZE(BO)
+	vmovsd	%xmm0, 1*SIZE(BO)
+	vmovss	%xmm3, 3*SIZE(BO)
+	vmovsd	%xmm2, 4*SIZE(BO)
+	vmovss	%xmm5, 6*SIZE(BO)
+	vmovsd	%xmm4, 7*SIZE(BO)
+	vmovss	%xmm7, 9*SIZE(BO)
+	vmovsd	%xmm6,10*SIZE(BO)
+	addq	$8*SIZE,BO1
+	addq	$8*SIZE,BO2
+	addq	$12*SIZE,BO
+
+
+	vmovsd	0 * SIZE(BO2), %xmm0
+	vmovsd	2 * SIZE(BO2), %xmm2
+	vmovsd	4 * SIZE(BO2), %xmm4
+	vmovsd	6 * SIZE(BO2), %xmm6
+	vmovss  1 * SIZE(BO1), %xmm1
+	vmovss  3 * SIZE(BO1), %xmm3
+	vmovss  5 * SIZE(BO1), %xmm5
+	vmovss  7 * SIZE(BO1), %xmm7
+	vmovss	%xmm1, 0*SIZE(BO)
+	vmovsd	%xmm0, 1*SIZE(BO)
+	vmovss	%xmm3, 3*SIZE(BO)
+	vmovsd	%xmm2, 4*SIZE(BO)
+	vmovss	%xmm5, 6*SIZE(BO)
+	vmovsd	%xmm4, 7*SIZE(BO)
+	vmovss	%xmm7, 9*SIZE(BO)
+	vmovsd	%xmm6,10*SIZE(BO)
+	addq	$8*SIZE,BO1
+	addq	$8*SIZE,BO2
+	addq	$12*SIZE,BO
+
+	decq	%rax
+	jnz	.L6_02c_1
+
+
+.L6_02c_2:
+
+	movq    K, %rax
+        andq    $7, %rax                // K % 8
+        jz      .L6_03c
+        ALIGN_4
+
+.L6_03b:
+
+	vmovss	  1*SIZE(BO1), %xmm0
+	vmovsd	  0*SIZE(BO2), %xmm1
+	vmovss	%xmm0, 0*SIZE(BO)
+	vmovsd	%xmm1, 1*SIZE(BO)
+	addq	$2*SIZE,BO1
+	addq	$2*SIZE,BO2
+	addq	$3*SIZE,BO
+	decq	%rax
+	jnz	.L6_03b
+
+
+.L6_03c:
+
+	movq	BO2, B			// next offset of B
+
+.L6_10:
+	movq	C, CO1
+	leaq	(C, LDC, 2), C		 
+	leaq	(C, LDC, 1), C		// c += 3 * ldc
+
+	
+	movq	A, AO		 	// aoffset = a
+	addq	$32 * SIZE, AO
+
+	movq	M,  I
+	sarq	$4, I			// i = (m >> 4)
+	je	.L6_20
+
+	ALIGN_4
+
+.L6_11:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $6 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax			//  K = K - ( K % 8 )
+	je	.L6_16
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	salq	$4, %rax			// rax = rax * 16 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L6_12:
+
+	prefetcht0	B_PR1(BO,BI, SIZE)
+	KERNEL16x3_1(xxx)
+	KERNEL16x3_2(xxx)
+	KERNEL16x3_3(xxx)
+	KERNEL16x3_4(xxx)
+
+	KERNEL16x3_1(xxx)
+	prefetcht0	B_PR1+16(BO,BI, SIZE)
+	KERNEL16x3_2(xxx)
+	KERNEL16x3_3(xxx)
+	KERNEL16x3_4(xxx)
+
+	je	.L6_16
+
+	KERNEL16x3_1(xxx)
+	KERNEL16x3_2(xxx)
+	prefetcht0	B_PR1+32(BO,BI, SIZE)
+	KERNEL16x3_3(xxx)
+	KERNEL16x3_4(xxx)
+
+	KERNEL16x3_1(xxx)
+	KERNEL16x3_2(xxx)
+	KERNEL16x3_3(xxx)
+	KERNEL16x3_4(xxx)
+
+	je	.L6_16
+
+	jmp	.L6_12
+	ALIGN_4
+
+.L6_16:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L6_19
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	salq	$4, %rax			// rax = rax * 16 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L6_17:
+
+	KERNEL16x3_SUB(xxx)
+	addq	$3, BI
+	addq	$16, %rax
+	jl	.L6_17
+	ALIGN_4
+
+
+.L6_19:
+
+	vbroadcastss	ALPHA, %xmm0
+
+	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddps 4 * SIZE(CO1),%xmm0, %xmm7,%xmm7
+	vfmaddps 8 * SIZE(CO1),%xmm0, %xmm10,%xmm10
+	vfmaddps 12 * SIZE(CO1),%xmm0, %xmm13,%xmm13
+
+	vfmaddps 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+	vfmaddps 4 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
+	vfmaddps 8 * SIZE(CO1, LDC),%xmm0, %xmm11,%xmm11
+	vfmaddps 12 * SIZE(CO1, LDC),%xmm0, %xmm14,%xmm14
+
+	vfmaddps 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
+	vfmaddps 4 * SIZE(CO1, LDC, 2),%xmm0, %xmm9,%xmm9
+	vfmaddps 8 * SIZE(CO1, LDC, 2),%xmm0, %xmm12,%xmm12
+	vfmaddps 12 * SIZE(CO1, LDC, 2),%xmm0, %xmm15,%xmm15
+
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm7 , 4 * SIZE(CO1)
+	vmovups	%xmm10, 8 * SIZE(CO1)
+	vmovups	%xmm13,12 * SIZE(CO1)
+
+	vmovups	%xmm5 ,  	(CO1, LDC)
+	vmovups	%xmm8 , 4 * SIZE(CO1, LDC)
+	vmovups	%xmm11, 8 * SIZE(CO1, LDC)
+	vmovups	%xmm14,12 * SIZE(CO1, LDC)
+
+	vmovups	%xmm6 ,  	(CO1, LDC, 2)
+	vmovups	%xmm9 , 4 * SIZE(CO1, LDC, 2)
+	vmovups	%xmm12, 8 * SIZE(CO1, LDC, 2)
+	vmovups	%xmm15,12 * SIZE(CO1, LDC, 2)
+
+
+	addq	$16 * SIZE, CO1		# coffset += 16
+	decq	I			# i --
+	jg	.L6_11
+	ALIGN_4	
+
+/**************************************************************************
+* Rest of M 
+***************************************************************************/
+.L6_20:
+	// Test rest of M
+
+	testq	$15, M
+	jz	.L7_10		// to next 3 lines of N
+
+	testq	$8, M		
+	jz	.L6_21pre
+	ALIGN_4
+
+/**************************************************************************/
+
+.L6_20_1:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $6 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L6_20_6
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L6_20_2:
+
+	prefetcht0	B_PR1(BO,BI, SIZE)
+	KERNEL8x3_1(xxx)
+	KERNEL8x3_2(xxx)
+	KERNEL8x3_3(xxx)
+	KERNEL8x3_4(xxx)
+
+	KERNEL8x3_1(xxx)
+	prefetcht0	B_PR1+16(BO,BI, SIZE)
+	KERNEL8x3_2(xxx)
+	KERNEL8x3_3(xxx)
+	KERNEL8x3_4(xxx)
+
+	je	.L6_20_6
+
+	KERNEL8x3_1(xxx)
+	KERNEL8x3_2(xxx)
+	prefetcht0	B_PR1+32(BO,BI, SIZE)
+	KERNEL8x3_3(xxx)
+	KERNEL8x3_4(xxx)
+
+	KERNEL8x3_1(xxx)
+	KERNEL8x3_2(xxx)
+	KERNEL8x3_3(xxx)
+	KERNEL8x3_4(xxx)
+
+	je	.L6_20_6
+
+	jmp	.L6_20_2
+	ALIGN_4
+
+.L6_20_6:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L6_20_9
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L6_20_7:
+
+	KERNEL8x3_SUB(xxx)
+	addq	$3, BI
+	addq	$8, %rax
+	jl	.L6_20_7
+	ALIGN_4
+
+
+.L6_20_9:
+
+	vbroadcastss	ALPHA, %xmm0
+
+	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddps 4 * SIZE(CO1),%xmm0, %xmm7,%xmm7
+
+	vfmaddps 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+	vfmaddps 4 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
+
+	vfmaddps 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
+	vfmaddps 4 * SIZE(CO1, LDC, 2),%xmm0, %xmm9,%xmm9
+
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm7 , 4 * SIZE(CO1)
+
+	vmovups	%xmm5 ,  	(CO1, LDC)
+	vmovups	%xmm8 , 4 * SIZE(CO1, LDC)
+
+	vmovups	%xmm6 ,  	(CO1, LDC, 2)
+	vmovups	%xmm9 , 4 * SIZE(CO1, LDC, 2)
+
+
+	addq	$8 * SIZE, CO1		# coffset += 8
+	ALIGN_4
+	
+
+
+/**************************************************************************/
+
+.L6_21pre:
+
+	testq	$4, M		
+	jz	.L6_30
+	ALIGN_4
+
+.L6_21:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $6 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L6_26
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L6_22:
+
+	prefetcht0	B_PR1(BO,BI, SIZE)
+	KERNEL4x3_1(xxx)
+	KERNEL4x3_2(xxx)
+	KERNEL4x3_3(xxx)
+	KERNEL4x3_4(xxx)
+
+	KERNEL4x3_1(xxx)
+	prefetcht0	B_PR1+16(BO,BI, SIZE)
+	KERNEL4x3_2(xxx)
+	KERNEL4x3_3(xxx)
+	KERNEL4x3_4(xxx)
+
+	je	.L6_26
+
+	KERNEL4x3_1(xxx)
+	KERNEL4x3_2(xxx)
+	prefetcht0	B_PR1+32(BO,BI, SIZE)
+	KERNEL4x3_3(xxx)
+	KERNEL4x3_4(xxx)
+
+	KERNEL4x3_1(xxx)
+	KERNEL4x3_2(xxx)
+	KERNEL4x3_3(xxx)
+	KERNEL4x3_4(xxx)
+
+	je	.L6_26
+
+	jmp	.L6_22
+	ALIGN_4
+
+.L6_26:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L6_29
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L6_27:
+
+	KERNEL4x3_SUB(xxx)
+	addq	$3, BI
+	addq	$4, %rax
+	jl	.L6_27
+	ALIGN_4
+
+
+.L6_29:
+
+	vbroadcastss	ALPHA, %xmm0
+
+	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddps 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+	vfmaddps 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
+
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm5 ,  	(CO1, LDC)
+	vmovups	%xmm6 ,  	(CO1, LDC, 2)
+
+	addq	$4 * SIZE, CO1		# coffset += 4
+	ALIGN_4
+	
+
+.L6_30:
+	testq	$2, M		
+	jz	.L6_40
+
+	ALIGN_4
+
+.L6_31:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $6 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L6_36
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L6_32:
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL2x3_1(xxx)
+	KERNEL2x3_2(xxx)
+	KERNEL2x3_3(xxx)
+	KERNEL2x3_4(xxx)
+
+	KERNEL2x3_1(xxx)
+	prefetcht0	B_PR1+16(BO,BI,SIZE)
+	KERNEL2x3_2(xxx)
+	KERNEL2x3_3(xxx)
+	KERNEL2x3_4(xxx)
+
+	je	.L6_36
+
+	KERNEL2x3_1(xxx)
+	KERNEL2x3_2(xxx)
+	prefetcht0	B_PR1+32(BO,BI,SIZE)
+	KERNEL2x3_3(xxx)
+	KERNEL2x3_4(xxx)
+
+	KERNEL2x3_1(xxx)
+	KERNEL2x3_2(xxx)
+	KERNEL2x3_3(xxx)
+	KERNEL2x3_4(xxx)
+
+	je	.L6_36
+
+	jmp	.L6_32
+	ALIGN_4
+
+.L6_36:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L6_39
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+	
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L6_37:
+
+	KERNEL2x3_SUB(xxx)
+	addq	$3, BI
+	addq	$2, %rax
+	jl	.L6_37
+	ALIGN_4
+
+
+.L6_39:
+
+	vmovss	ALPHA, %xmm0
+
+	vfmaddss 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddss 1 * SIZE(CO1),%xmm0, %xmm8,%xmm8
+	vfmaddss 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+	vfmaddss 1 * SIZE(CO1, LDC),%xmm0, %xmm10,%xmm10
+	vfmaddss 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
+	vfmaddss 1 * SIZE(CO1, LDC, 2),%xmm0, %xmm12,%xmm12
+
+
+	vmovss	%xmm4 ,  	(CO1)
+	vmovss	%xmm8 , 1 * SIZE(CO1)
+	vmovss	%xmm5 ,  	(CO1, LDC)
+	vmovss	%xmm10, 1 * SIZE(CO1, LDC)
+	vmovss	%xmm6 ,  	(CO1, LDC, 2)
+	vmovss	%xmm12, 1 * SIZE(CO1, LDC, 2)
+
+	addq	$2 * SIZE, CO1		# coffset += 2
+	ALIGN_4
+
+.L6_40:
+	testq	$1, M		
+	jz	.L7_10		// to next 3 lines of N
+
+	ALIGN_4
+
+.L6_41:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $6 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L6_46
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L6_42:
+
+	KERNEL1x3_1(xxx)
+	KERNEL1x3_2(xxx)
+	KERNEL1x3_3(xxx)
+	KERNEL1x3_4(xxx)
+
+	KERNEL1x3_1(xxx)
+	KERNEL1x3_2(xxx)
+	KERNEL1x3_3(xxx)
+	KERNEL1x3_4(xxx)
+
+	je	.L6_46
+
+	KERNEL1x3_1(xxx)
+	KERNEL1x3_2(xxx)
+	KERNEL1x3_3(xxx)
+	KERNEL1x3_4(xxx)
+
+	KERNEL1x3_1(xxx)
+	KERNEL1x3_2(xxx)
+	KERNEL1x3_3(xxx)
+	KERNEL1x3_4(xxx)
+
+	je	.L6_46
+
+	jmp	.L6_42
+	ALIGN_4
+
+.L6_46:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L6_49
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L6_47:
+
+	KERNEL1x3_SUB(xxx)
+	addq	$3, BI
+	addq	$1, %rax
+	jl	.L6_47
+	ALIGN_4
+
+
+.L6_49:
+
+	vmovss	ALPHA, %xmm0
+
+	vfmaddss 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddss 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+	vfmaddss 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
+
+
+	vmovss	%xmm4 ,  	(CO1)
+	vmovss	%xmm5 ,  	(CO1, LDC)
+	vmovss	%xmm6 ,  	(CO1, LDC, 2)
+
+	addq	$1 * SIZE, CO1		# coffset += 1
+	ALIGN_4
+	
+
+
+
+/***************************************************************************************************************/
+
+.L7_10:
+	movq	C, CO1
+	leaq	(C, LDC, 2), C		 
+	leaq	(C, LDC, 1), C		// c += 3 * ldc
+
+	
+	movq	A, AO		 	// aoffset = a
+	addq	$32 * SIZE, AO
+
+	movq	M,  I
+	sarq	$4, I			// i = (m >> 4)
+	je	.L7_20
+
+	ALIGN_4
+
+.L7_11:
+        leaq    BUFFER2, BO             // second buffer to BO
+        addq    $6 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax			//  K = K - ( K % 8 )
+	je	.L7_16
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	salq	$4, %rax			// rax = rax * 16 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L7_12:
+
+	prefetcht0	B_PR1(BO,BI, SIZE)
+	KERNEL16x3_1(xxx)
+	KERNEL16x3_2(xxx)
+	KERNEL16x3_3(xxx)
+	KERNEL16x3_4(xxx)
+
+	KERNEL16x3_1(xxx)
+	prefetcht0	B_PR1+16(BO,BI, SIZE)
+	KERNEL16x3_2(xxx)
+	KERNEL16x3_3(xxx)
+	KERNEL16x3_4(xxx)
+
+	je	.L7_16
+
+	KERNEL16x3_1(xxx)
+	KERNEL16x3_2(xxx)
+	prefetcht0	B_PR1+32(BO,BI, SIZE)
+	KERNEL16x3_3(xxx)
+	KERNEL16x3_4(xxx)
+
+	KERNEL16x3_1(xxx)
+	KERNEL16x3_2(xxx)
+	KERNEL16x3_3(xxx)
+	KERNEL16x3_4(xxx)
+
+	je	.L7_16
+
+	jmp	.L7_12
+	ALIGN_4
+
+.L7_16:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L7_19
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	salq	$4, %rax			// rax = rax * 16 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L7_17:
+
+	KERNEL16x3_SUB(xxx)
+	addq	$3, BI
+	addq	$16, %rax
+	jl	.L7_17
+	ALIGN_4
+
+
+.L7_19:
+
+	vbroadcastss	ALPHA, %xmm0
+
+	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddps 4 * SIZE(CO1),%xmm0, %xmm7,%xmm7
+	vfmaddps 8 * SIZE(CO1),%xmm0, %xmm10,%xmm10
+	vfmaddps 12 * SIZE(CO1),%xmm0, %xmm13,%xmm13
+
+	vfmaddps 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+	vfmaddps 4 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
+	vfmaddps 8 * SIZE(CO1, LDC),%xmm0, %xmm11,%xmm11
+	vfmaddps 12 * SIZE(CO1, LDC),%xmm0, %xmm14,%xmm14
+
+	vfmaddps 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
+	vfmaddps 4 * SIZE(CO1, LDC, 2),%xmm0, %xmm9,%xmm9
+	vfmaddps 8 * SIZE(CO1, LDC, 2),%xmm0, %xmm12,%xmm12
+	vfmaddps 12 * SIZE(CO1, LDC, 2),%xmm0, %xmm15,%xmm15
+
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm7 , 4 * SIZE(CO1)
+	vmovups	%xmm10, 8 * SIZE(CO1)
+	vmovups	%xmm13,12 * SIZE(CO1)
+
+	vmovups	%xmm5 ,  	(CO1, LDC)
+	vmovups	%xmm8 , 4 * SIZE(CO1, LDC)
+	vmovups	%xmm11, 8 * SIZE(CO1, LDC)
+	vmovups	%xmm14,12 * SIZE(CO1, LDC)
+
+	vmovups	%xmm6 ,  	(CO1, LDC, 2)
+	vmovups	%xmm9 , 4 * SIZE(CO1, LDC, 2)
+	vmovups	%xmm12, 8 * SIZE(CO1, LDC, 2)
+	vmovups	%xmm15,12 * SIZE(CO1, LDC, 2)
+
+
+	addq	$16 * SIZE, CO1		# coffset += 16
+	decq	I			# i --
+	jg	.L7_11
+	ALIGN_4	
+
+/**************************************************************************
+* Rest of M 
+***************************************************************************/
+.L7_20:
+	// Test rest of M
+
+	testq	$15, M
+	jz	.L7_60		// to next 3 lines of N
+
+	testq	$8, M		
+	jz	.L7_21pre
+	ALIGN_4
+
+/**************************************************************************/
+
+.L7_20_1:
+        leaq    BUFFER2, BO             // first buffer to BO
+        addq    $6 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L7_20_6
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L7_20_2:
+
+	prefetcht0	B_PR1(BO,BI, SIZE)
+	KERNEL8x3_1(xxx)
+	KERNEL8x3_2(xxx)
+	KERNEL8x3_3(xxx)
+	KERNEL8x3_4(xxx)
+
+	KERNEL8x3_1(xxx)
+	prefetcht0	B_PR1+16(BO,BI, SIZE)
+	KERNEL8x3_2(xxx)
+	KERNEL8x3_3(xxx)
+	KERNEL8x3_4(xxx)
+
+	je	.L7_20_6
+
+	KERNEL8x3_1(xxx)
+	KERNEL8x3_2(xxx)
+	prefetcht0	B_PR1+32(BO,BI, SIZE)
+	KERNEL8x3_3(xxx)
+	KERNEL8x3_4(xxx)
+
+	KERNEL8x3_1(xxx)
+	KERNEL8x3_2(xxx)
+	KERNEL8x3_3(xxx)
+	KERNEL8x3_4(xxx)
+
+	je	.L7_20_6
+
+	jmp	.L7_20_2
+	ALIGN_4
+
+.L7_20_6:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L7_20_9
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L7_20_7:
+
+	KERNEL8x3_SUB(xxx)
+	addq	$3, BI
+	addq	$8, %rax
+	jl	.L7_20_7
+	ALIGN_4
+
+.L7_20_9:
+
+
+	vbroadcastss	ALPHA, %xmm0
+
+	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddps 4 * SIZE(CO1),%xmm0, %xmm7,%xmm7
+
+	vfmaddps 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+	vfmaddps 4 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
+
+	vfmaddps 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
+	vfmaddps 4 * SIZE(CO1, LDC, 2),%xmm0, %xmm9,%xmm9
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm7 , 4 * SIZE(CO1)
+
+	vmovups	%xmm5 ,  	(CO1, LDC)
+	vmovups	%xmm8 , 4 * SIZE(CO1, LDC)
+
+	vmovups	%xmm6 ,  	(CO1, LDC, 2)
+	vmovups	%xmm9 , 4 * SIZE(CO1, LDC, 2)
+
+
+	addq	$8 * SIZE, CO1		# coffset += 8
+	ALIGN_4
+	
+
+
+/**************************************************************************/
+
+.L7_21pre:
+
+	testq	$4, M		
+	jz	.L7_30
+	ALIGN_4
+
+.L7_21:
+        leaq    BUFFER2, BO             // second buffer to BO
+        addq    $6 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L7_26
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L7_22:
+
+	prefetcht0	B_PR1(BO,BI, SIZE)
+	KERNEL4x3_1(xxx)
+	KERNEL4x3_2(xxx)
+	KERNEL4x3_3(xxx)
+	KERNEL4x3_4(xxx)
+
+	KERNEL4x3_1(xxx)
+	prefetcht0	B_PR1+16(BO,BI, SIZE)
+	KERNEL4x3_2(xxx)
+	KERNEL4x3_3(xxx)
+	KERNEL4x3_4(xxx)
+
+	je	.L7_26
+
+	KERNEL4x3_1(xxx)
+	KERNEL4x3_2(xxx)
+	prefetcht0	B_PR1+32(BO,BI, SIZE)
+	KERNEL4x3_3(xxx)
+	KERNEL4x3_4(xxx)
+
+	KERNEL4x3_1(xxx)
+	KERNEL4x3_2(xxx)
+	KERNEL4x3_3(xxx)
+	KERNEL4x3_4(xxx)
+
+	je	.L7_26
+
+	jmp	.L7_22
+	ALIGN_4
+
+.L7_26:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L7_29
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L7_27:
+
+	KERNEL4x3_SUB(xxx)
+	addq	$3, BI
+	addq	$4, %rax
+	jl	.L7_27
+	ALIGN_4
+
+
+.L7_29:
+
+	vbroadcastss	ALPHA, %xmm0
+
+	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddps 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+	vfmaddps 	 (CO1, LDC, 2),%xmm0, %xmm6 ,%xmm6
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm5 ,  	(CO1, LDC)
+	vmovups	%xmm6 ,  	(CO1, LDC, 2)
+
+	addq	$4 * SIZE, CO1		# coffset += 4
+	ALIGN_4
+	
+
+.L7_30:
+	testq	$2, M		
+	jz	.L7_40
+
+	ALIGN_4
+
+.L7_31:
+        leaq    BUFFER2, BO             // second buffer to BO
+        addq    $6 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L7_36
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L7_32:
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL2x3_1(xxx)
+	KERNEL2x3_2(xxx)
+	KERNEL2x3_3(xxx)
+	KERNEL2x3_4(xxx)
+
+	KERNEL2x3_1(xxx)
+	prefetcht0	B_PR1+16(BO,BI,SIZE)
+	KERNEL2x3_2(xxx)
+	KERNEL2x3_3(xxx)
+	KERNEL2x3_4(xxx)
+
+	je	.L7_36
+
+	KERNEL2x3_1(xxx)
+	KERNEL2x3_2(xxx)
+	prefetcht0	B_PR1+32(BO,BI,SIZE)
+	KERNEL2x3_3(xxx)
+	KERNEL2x3_4(xxx)
+
+	KERNEL2x3_1(xxx)
+	KERNEL2x3_2(xxx)
+	KERNEL2x3_3(xxx)
+	KERNEL2x3_4(xxx)
+
+	je	.L7_36
+
+	jmp	.L7_32
+	ALIGN_4
+
+.L7_36:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L7_39
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+	
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L7_37:
+
+	KERNEL2x3_SUB(xxx)
+	addq	$3, BI
+	addq	$2, %rax
+	jl	.L7_37
+	ALIGN_4
+
+
+.L7_39:
+
+	vmovss	ALPHA, %xmm0
+
+	vfmaddss 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddss 1 * SIZE(CO1),%xmm0, %xmm8,%xmm8
+	vfmaddss 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+	vfmaddss 1 * SIZE(CO1, LDC),%xmm0, %xmm10,%xmm10
+	vfmaddss 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
+	vfmaddss 1 * SIZE(CO1, LDC, 2),%xmm0, %xmm12,%xmm12
+
+	vmovss	%xmm4 ,  	(CO1)
+	vmovss	%xmm8 , 1 * SIZE(CO1)
+	vmovss	%xmm5 ,  	(CO1, LDC)
+	vmovss	%xmm10, 1 * SIZE(CO1, LDC)
+	vmovss	%xmm6 ,  	(CO1, LDC, 2)
+	vmovss	%xmm12, 1 * SIZE(CO1, LDC, 2)
+
+	addq	$2 * SIZE, CO1		# coffset += 2
+	ALIGN_4
+
+.L7_40:
+	testq	$1, M		
+	jz	.L7_60		// to next 3 lines of N
+
+	ALIGN_4
+
+.L7_41:
+        leaq    BUFFER2, BO             // second buffer to BO
+        addq    $6 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L7_46
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L7_42:
+
+	KERNEL1x3_1(xxx)
+	KERNEL1x3_2(xxx)
+	KERNEL1x3_3(xxx)
+	KERNEL1x3_4(xxx)
+
+	KERNEL1x3_1(xxx)
+	KERNEL1x3_2(xxx)
+	KERNEL1x3_3(xxx)
+	KERNEL1x3_4(xxx)
+
+	je	.L7_46
+
+	KERNEL1x3_1(xxx)
+	KERNEL1x3_2(xxx)
+	KERNEL1x3_3(xxx)
+	KERNEL1x3_4(xxx)
+
+	KERNEL1x3_1(xxx)
+	KERNEL1x3_2(xxx)
+	KERNEL1x3_3(xxx)
+	KERNEL1x3_4(xxx)
+
+	je	.L7_46
+
+	jmp	.L7_42
+	ALIGN_4
+
+.L7_46:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L7_49
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,2), BI                   //  BI = BI * 3 ; number of values
+
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L7_47:
+
+	KERNEL1x3_SUB(xxx)
+	addq	$3, BI
+	addq	$1, %rax
+	jl	.L7_47
+	ALIGN_4
+
+
+.L7_49:
+
+	vmovss	ALPHA, %xmm0
+
+	vfmaddss 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddss 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+	vfmaddss 	 (CO1, LDC, 2),%xmm0, %xmm6,%xmm6
+
+	vmovss	%xmm4 ,  	(CO1)
+	vmovss	%xmm5 ,  	(CO1, LDC)
+	vmovss	%xmm6 ,  	(CO1, LDC, 2)
+
+	addq	$1 * SIZE, CO1		# coffset += 1
+	ALIGN_4
+	
+
+
+.L7_60:
+
+	decq	J			// j --
+	jg	.L6_01
+
+
+.L2_0:
+	cmpq	$0, Nmod6		// N % 6 == 0
+	je	.L999
+
+/************************************************************************************************
+* Loop for Nmod6 / 2 > 0
+*************************************************************************************************/
+
+	movq	Nmod6, J		
+	sarq	$1, J			// j = j / 2
+	je	.L1_0
+	ALIGN_4
+
+.L2_01:
+	// copy to sub buffer
+	movq	B, BO1
+	leaq    BUFFER1, BO		// first buffer to BO
+	movq	K, %rax
+	ALIGN_4
+
+.L2_02b:
+
+	vmovsd	(BO1), %xmm0
+	vmovsd	%xmm0,       (BO)
+	addq	$2*SIZE,BO1
+	addq	$2*SIZE,BO
+	decq	%rax
+	jnz	.L2_02b
+
+.L2_02c:
+
+	movq	BO1, B			// next offset of B
+
+.L2_10:
+	movq	C, CO1
+	leaq	(C, LDC, 2), C		// c += 2 * ldc
+
+	
+	movq	A, AO		 	// aoffset = a
+	addq	$32 * SIZE, AO
+
+	movq	M,  I
+	sarq	$4, I			// i = (m >> 4)
+	je	.L2_20
+
+	ALIGN_4
+
+.L2_11:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax			//  K = K - ( K % 8 )
+	je	.L2_16
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$4, %rax			// rax = rax * 16 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_12:
+
+	prefetcht0	B_PR1(BO,BI, SIZE)
+	KERNEL16x2_1(xxx)
+	KERNEL16x2_2(xxx)
+	KERNEL16x2_3(xxx)
+	KERNEL16x2_4(xxx)
+
+	KERNEL16x2_1(xxx)
+	KERNEL16x2_2(xxx)
+	KERNEL16x2_3(xxx)
+	KERNEL16x2_4(xxx)
+
+	je	.L2_16
+
+	prefetcht0	B_PR1(BO,BI, SIZE)
+	KERNEL16x2_1(xxx)
+	KERNEL16x2_2(xxx)
+	KERNEL16x2_3(xxx)
+	KERNEL16x2_4(xxx)
+
+	KERNEL16x2_1(xxx)
+	KERNEL16x2_2(xxx)
+	KERNEL16x2_3(xxx)
+	KERNEL16x2_4(xxx)
+
+	je	.L2_16
+
+	jmp	.L2_12
+	ALIGN_4
+
+.L2_16:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_19
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$4, %rax			// rax = rax * 16 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_17:
+
+	KERNEL16x2_SUB(xxx)
+	addq	$2, BI
+	addq	$16, %rax
+	jl	.L2_17
+	ALIGN_4
+
+
+.L2_19:
+
+	vbroadcastss	ALPHA, %xmm0
+
+	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddps 4 * SIZE(CO1),%xmm0, %xmm7,%xmm7
+	vfmaddps 8 * SIZE(CO1),%xmm0, %xmm10,%xmm10
+	vfmaddps 12 * SIZE(CO1),%xmm0, %xmm13,%xmm13
+
+	vfmaddps 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+	vfmaddps 4 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
+	vfmaddps 8 * SIZE(CO1, LDC),%xmm0, %xmm11,%xmm11
+	vfmaddps 12 * SIZE(CO1, LDC),%xmm0, %xmm14,%xmm14
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm7 , 4 * SIZE(CO1)
+	vmovups	%xmm10, 8 * SIZE(CO1)
+	vmovups	%xmm13,12 * SIZE(CO1)
+
+	vmovups	%xmm5 ,  	(CO1, LDC)
+	vmovups	%xmm8 , 4 * SIZE(CO1, LDC)
+	vmovups	%xmm11, 8 * SIZE(CO1, LDC)
+	vmovups	%xmm14,12 * SIZE(CO1, LDC)
+
+	addq	$16 * SIZE, CO1		# coffset += 16
+	decq	I			# i --
+	jg	.L2_11
+	ALIGN_4	
+
+/**************************************************************************
+* Rest of M 
+***************************************************************************/
+.L2_20:
+	// Test rest of M
+
+	testq	$15, M
+	jz	.L2_60		// to next 3 lines of N
+
+	testq	$8, M		
+	jz	.L2_21pre
+	ALIGN_4
+
+/**************************************************************************/
+
+.L2_20_1:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L2_20_6
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_20_2:
+
+	prefetcht0	B_PR1(BO,BI, SIZE)
+	KERNEL8x2_1(xxx)
+	KERNEL8x2_2(xxx)
+	KERNEL8x2_3(xxx)
+	KERNEL8x2_4(xxx)
+
+	KERNEL8x2_1(xxx)
+	KERNEL8x2_2(xxx)
+	KERNEL8x2_3(xxx)
+	KERNEL8x2_4(xxx)
+
+	je	.L2_20_6
+
+	prefetcht0	B_PR1(BO,BI, SIZE)
+	KERNEL8x2_1(xxx)
+	KERNEL8x2_2(xxx)
+	KERNEL8x2_3(xxx)
+	KERNEL8x2_4(xxx)
+
+	KERNEL8x2_1(xxx)
+	KERNEL8x2_2(xxx)
+	KERNEL8x2_3(xxx)
+	KERNEL8x2_4(xxx)
+
+	je	.L2_20_6
+
+	jmp	.L2_20_2
+	ALIGN_4
+
+.L2_20_6:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_20_9
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_20_7:
+
+	KERNEL8x2_SUB(xxx)
+	addq	$2, BI
+	addq	$8, %rax
+	jl	.L2_20_7
+	ALIGN_4
+
+
+.L2_20_9:
+
+	vbroadcastss	ALPHA, %xmm0
+
+	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddps 4 * SIZE(CO1),%xmm0, %xmm7,%xmm7
+
+	vfmaddps 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+	vfmaddps 4 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm7 , 4 * SIZE(CO1)
+
+	vmovups	%xmm5 ,  	(CO1, LDC)
+	vmovups	%xmm8 , 4 * SIZE(CO1, LDC)
+
+
+	addq	$8 * SIZE, CO1		# coffset += 8
+	ALIGN_4
+	
+
+
+/**************************************************************************/
+
+.L2_21pre:
+
+	testq	$4, M		
+	jz	.L2_30
+	ALIGN_4
+
+.L2_21:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L2_26
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 1 ; number of values
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_22:
+
+	prefetcht0	B_PR1(BO,BI, SIZE)
+	KERNEL4x2_1(xxx)
+	KERNEL4x2_2(xxx)
+	KERNEL4x2_3(xxx)
+	KERNEL4x2_4(xxx)
+
+	KERNEL4x2_1(xxx)
+	KERNEL4x2_2(xxx)
+	KERNEL4x2_3(xxx)
+	KERNEL4x2_4(xxx)
+
+	je	.L2_26
+
+	prefetcht0	B_PR1(BO,BI, SIZE)
+	KERNEL4x2_1(xxx)
+	KERNEL4x2_2(xxx)
+	KERNEL4x2_3(xxx)
+	KERNEL4x2_4(xxx)
+
+	KERNEL4x2_1(xxx)
+	KERNEL4x2_2(xxx)
+	KERNEL4x2_3(xxx)
+	KERNEL4x2_4(xxx)
+
+	je	.L2_26
+
+	jmp	.L2_22
+	ALIGN_4
+
+.L2_26:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_29
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_27:
+
+	KERNEL4x2_SUB(xxx)
+	addq	$2, BI
+	addq	$4, %rax
+	jl	.L2_27
+	ALIGN_4
+
+
+.L2_29:
+
+	vbroadcastss	ALPHA, %xmm0
+
+	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddps 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm5 ,  	(CO1, LDC)
+
+
+	addq	$4 * SIZE, CO1		# coffset += 4
+	ALIGN_4
+	
+
+.L2_30:
+	testq	$2, M		
+	jz	.L2_40
+
+	ALIGN_4
+
+.L2_31:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L2_36
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_32:
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL2x2_1(xxx)
+	KERNEL2x2_2(xxx)
+	KERNEL2x2_3(xxx)
+	KERNEL2x2_4(xxx)
+
+	KERNEL2x2_1(xxx)
+	KERNEL2x2_2(xxx)
+	KERNEL2x2_3(xxx)
+	KERNEL2x2_4(xxx)
+
+	je	.L2_36
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL2x2_1(xxx)
+	KERNEL2x2_2(xxx)
+	KERNEL2x2_3(xxx)
+	KERNEL2x2_4(xxx)
+
+	KERNEL2x2_1(xxx)
+	KERNEL2x2_2(xxx)
+	KERNEL2x2_3(xxx)
+	KERNEL2x2_4(xxx)
+
+	je	.L2_36
+
+	jmp	.L2_32
+	ALIGN_4
+
+.L2_36:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_39
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+	
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_37:
+
+	KERNEL2x2_SUB(xxx)
+	addq	$2, BI
+	addq	$2, %rax
+	jl	.L2_37
+	ALIGN_4
+
+
+.L2_39:
+
+	vmovss	ALPHA, %xmm0
+
+	vfmaddss 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddss 1 * SIZE(CO1),%xmm0, %xmm8,%xmm8
+	vfmaddss 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+	vfmaddss 1 * SIZE(CO1, LDC),%xmm0, %xmm10,%xmm10
+
+	vmovss	%xmm4 ,  	(CO1)
+	vmovss	%xmm8 , 1 * SIZE(CO1)
+	vmovss	%xmm5 ,  	(CO1, LDC)
+	vmovss	%xmm10, 1 * SIZE(CO1, LDC)
+
+	addq	$2 * SIZE, CO1		# coffset += 2
+	ALIGN_4
+
+.L2_40:
+	testq	$1, M		
+	jz	.L2_60		// to next 2 lines of N
+
+	ALIGN_4
+
+.L2_41:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L2_46
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_42:
+
+	KERNEL1x2_1(xxx)
+	KERNEL1x2_2(xxx)
+	KERNEL1x2_3(xxx)
+	KERNEL1x2_4(xxx)
+
+	KERNEL1x2_1(xxx)
+	KERNEL1x2_2(xxx)
+	KERNEL1x2_3(xxx)
+	KERNEL1x2_4(xxx)
+
+	je	.L2_46
+
+	KERNEL1x2_1(xxx)
+	KERNEL1x2_2(xxx)
+	KERNEL1x2_3(xxx)
+	KERNEL1x2_4(xxx)
+
+	KERNEL1x2_1(xxx)
+	KERNEL1x2_2(xxx)
+	KERNEL1x2_3(xxx)
+	KERNEL1x2_4(xxx)
+
+	je	.L2_46
+
+	jmp	.L2_42
+	ALIGN_4
+
+.L2_46:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_49
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_47:
+
+	KERNEL1x2_SUB(xxx)
+	addq	$2, BI
+	addq	$1, %rax
+	jl	.L2_47
+	ALIGN_4
+
+
+.L2_49:
+
+	vmovss	ALPHA, %xmm0
+
+	vfmaddss 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddss 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+
+	vmovss	%xmm4 ,  	(CO1)
+	vmovss	%xmm5 ,  	(CO1, LDC)
+
+	addq	$1 * SIZE, CO1		# coffset += 1
+	ALIGN_4
+	
+.L2_60:
+
+	decq	J			// j --
+	jg	.L2_01			// next 2 lines of N
+
+
+
+.L1_0:
+
+/************************************************************************************************
+* Loop for Nmod6 % 2 > 0
+*************************************************************************************************/
+
+	movq	Nmod6, J		
+	andq	$1, J			// j % 2
+	je	.L999
+	ALIGN_4
+
+.L1_01:
+	// copy to sub buffer
+	movq	B, BO1
+	leaq    BUFFER1, BO		// first buffer to BO
+	movq	K, %rax
+	ALIGN_4
+
+.L1_02b:
+
+	vmovss	(BO1), %xmm0
+	vmovss	%xmm0,       (BO)
+	addq	$1*SIZE,BO1
+	addq	$1*SIZE,BO
+	decq	%rax
+	jnz	.L1_02b
+
+.L1_02c:
+
+	movq	BO1, B			// next offset of B
+
+.L1_10:
+	movq	C, CO1
+	leaq	(C, LDC, 1), C		// c += 1 * ldc
+
+	
+	movq	A, AO		 	// aoffset = a
+	addq	$32 * SIZE, AO
+
+	movq	M,  I
+	sarq	$4, I			// i = (m >> 4)
+	je	.L1_20
+
+	ALIGN_4
+
+.L1_11:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax			//  K = K - ( K % 8 )
+	je	.L1_16
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$4, %rax			// rax = rax * 16 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_12:
+
+	prefetcht0	B_PR1(BO,BI, SIZE)
+	KERNEL16x1_1(xxx)
+	KERNEL16x1_2(xxx)
+	KERNEL16x1_3(xxx)
+	KERNEL16x1_4(xxx)
+
+	KERNEL16x1_1(xxx)
+	KERNEL16x1_2(xxx)
+	KERNEL16x1_3(xxx)
+	KERNEL16x1_4(xxx)
+
+	je	.L1_16
+
+	KERNEL16x1_1(xxx)
+	KERNEL16x1_2(xxx)
+	KERNEL16x1_3(xxx)
+	KERNEL16x1_4(xxx)
+
+	KERNEL16x1_1(xxx)
+	KERNEL16x1_2(xxx)
+	KERNEL16x1_3(xxx)
+	KERNEL16x1_4(xxx)
+
+	je	.L1_16
+
+	jmp	.L1_12
+	ALIGN_4
+
+.L1_16:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_19
+
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$4, %rax			// rax = rax * 16 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_17:
+
+	KERNEL16x1_SUB(xxx)
+	addq	$1, BI
+	addq	$16, %rax
+	jl	.L1_17
+	ALIGN_4
+
+
+.L1_19:
+
+	vbroadcastss	ALPHA, %xmm0
+
+	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddps 4 * SIZE(CO1),%xmm0, %xmm7,%xmm7
+	vfmaddps 8 * SIZE(CO1),%xmm0, %xmm10,%xmm10
+	vfmaddps 12 * SIZE(CO1),%xmm0, %xmm13,%xmm13
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm7 , 4 * SIZE(CO1)
+	vmovups	%xmm10, 8 * SIZE(CO1)
+	vmovups	%xmm13,12 * SIZE(CO1)
+
+	addq	$16 * SIZE, CO1		# coffset += 16
+	decq	I			# i --
+	jg	.L1_11
+	ALIGN_4	
+
+/**************************************************************************
+* Rest of M 
+***************************************************************************/
+.L1_20:
+	// Test rest of M
+
+	testq	$15, M
+	jz	.L999
+
+	testq	$8, M		
+	jz	.L1_21pre
+	ALIGN_4
+
+/**************************************************************************/
+
+.L1_20_1:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L1_20_6
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_20_2:
+
+	prefetcht0	B_PR1(BO,BI, SIZE)
+	KERNEL8x1_1(xxx)
+	KERNEL8x1_2(xxx)
+	KERNEL8x1_3(xxx)
+	KERNEL8x1_4(xxx)
+
+	KERNEL8x1_1(xxx)
+	KERNEL8x1_2(xxx)
+	KERNEL8x1_3(xxx)
+	KERNEL8x1_4(xxx)
+
+	je	.L1_20_6
+
+	KERNEL8x1_1(xxx)
+	KERNEL8x1_2(xxx)
+	KERNEL8x1_3(xxx)
+	KERNEL8x1_4(xxx)
+
+	KERNEL8x1_1(xxx)
+	KERNEL8x1_2(xxx)
+	KERNEL8x1_3(xxx)
+	KERNEL8x1_4(xxx)
+
+	je	.L1_20_6
+
+	jmp	.L1_20_2
+	ALIGN_4
+
+.L1_20_6:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_20_9
+
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_20_7:
+
+	KERNEL8x1_SUB(xxx)
+	addq	$1, BI
+	addq	$8, %rax
+	jl	.L1_20_7
+	ALIGN_4
+
+
+.L1_20_9:
+
+	vbroadcastss	ALPHA, %xmm0
+
+	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddps 4 * SIZE(CO1),%xmm0, %xmm7,%xmm7
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm7 , 4 * SIZE(CO1)
+
+	addq	$8 * SIZE, CO1		# coffset += 8
+	ALIGN_4
+	
+
+
+/**************************************************************************/
+
+.L1_21pre:
+
+	testq	$4, M		
+	jz	.L1_30
+	ALIGN_4
+
+.L1_21:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L1_26
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_22:
+
+	prefetcht0	B_PR1(BO,BI, SIZE)
+	KERNEL4x1_1(xxx)
+	KERNEL4x1_2(xxx)
+	KERNEL4x1_3(xxx)
+	KERNEL4x1_4(xxx)
+
+	KERNEL4x1_1(xxx)
+	KERNEL4x1_2(xxx)
+	KERNEL4x1_3(xxx)
+	KERNEL4x1_4(xxx)
+
+	je	.L1_26
+
+	KERNEL4x1_1(xxx)
+	KERNEL4x1_2(xxx)
+	KERNEL4x1_3(xxx)
+	KERNEL4x1_4(xxx)
+
+	KERNEL4x1_1(xxx)
+	KERNEL4x1_2(xxx)
+	KERNEL4x1_3(xxx)
+	KERNEL4x1_4(xxx)
+
+	je	.L1_26
+
+	jmp	.L1_22
+	ALIGN_4
+
+.L1_26:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_29
+
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_27:
+
+	KERNEL4x1_SUB(xxx)
+	addq	$1, BI
+	addq	$4, %rax
+	jl	.L1_27
+	ALIGN_4
+
+
+.L1_29:
+
+	vbroadcastss	ALPHA, %xmm0
+
+	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
+
+	vmovups	%xmm4 ,  	(CO1)
+
+	addq	$4 * SIZE, CO1		# coffset += 4
+	ALIGN_4
+	
+
+.L1_30:
+	testq	$2, M		
+	jz	.L1_40
+
+	ALIGN_4
+
+.L1_31:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L1_36
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_32:
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL2x1_1(xxx)
+	KERNEL2x1_2(xxx)
+	KERNEL2x1_3(xxx)
+	KERNEL2x1_4(xxx)
+
+	KERNEL2x1_1(xxx)
+	KERNEL2x1_2(xxx)
+	KERNEL2x1_3(xxx)
+	KERNEL2x1_4(xxx)
+
+	je	.L1_36
+
+	KERNEL2x1_1(xxx)
+	KERNEL2x1_2(xxx)
+	KERNEL2x1_3(xxx)
+	KERNEL2x1_4(xxx)
+
+	KERNEL2x1_1(xxx)
+	KERNEL2x1_2(xxx)
+	KERNEL2x1_3(xxx)
+	KERNEL2x1_4(xxx)
+
+	je	.L1_36
+
+	jmp	.L1_32
+	ALIGN_4
+
+.L1_36:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_39
+
+	movq    %rax, BI                        //  Index for BO
+	
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_37:
+
+	KERNEL2x1_SUB(xxx)
+	addq	$1, BI
+	addq	$2, %rax
+	jl	.L1_37
+	ALIGN_4
+
+
+.L1_39:
+
+	vmovss	ALPHA, %xmm0
+
+	vfmaddss 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddss 1 * SIZE(CO1),%xmm0, %xmm8,%xmm8
+
+	vmovss	%xmm4 ,  	(CO1)
+	vmovss	%xmm8 , 1 * SIZE(CO1)
+
+	addq	$2 * SIZE, CO1		# coffset += 2
+	ALIGN_4
+
+.L1_40:
+	testq	$1, M		
+	jz	.L999
+
+	ALIGN_4
+
+.L1_41:
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+
+	vzeroall
+
+        movq    K, %rax
+
+	andq	$-8, %rax
+	je	.L1_46
+	movq    %rax, BI                        //  Index for BO
+
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_42:
+
+	KERNEL1x1_1(xxx)
+	KERNEL1x1_2(xxx)
+	KERNEL1x1_3(xxx)
+	KERNEL1x1_4(xxx)
+
+	KERNEL1x1_1(xxx)
+	KERNEL1x1_2(xxx)
+	KERNEL1x1_3(xxx)
+	KERNEL1x1_4(xxx)
+
+	je	.L1_46
+
+	KERNEL1x1_1(xxx)
+	KERNEL1x1_2(xxx)
+	KERNEL1x1_3(xxx)
+	KERNEL1x1_4(xxx)
+
+	KERNEL1x1_1(xxx)
+	KERNEL1x1_2(xxx)
+	KERNEL1x1_3(xxx)
+	KERNEL1x1_4(xxx)
+
+	je	.L1_46
+
+	jmp	.L1_42
+	ALIGN_4
+
+.L1_46:
+        movq    K, %rax
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_49
+
+	movq    %rax, BI                        //  Index for BO
+
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_47:
+
+	KERNEL1x1_SUB(xxx)
+	addq	$1, BI
+	addq	$1, %rax
+	jl	.L1_47
+	ALIGN_4
+
+
+.L1_49:
+
+	vmovss	ALPHA, %xmm0
+
+	vfmaddss 	 (CO1),%xmm0, %xmm4,%xmm4
+
+	vmovss	%xmm4 ,  	(CO1)
+
+	addq	$1 * SIZE, CO1		# coffset += 1
+	ALIGN_4
+	
+
+.L999:
+	movq   		SP, %rsp
+	movq	   (%rsp), %rbx
+	movq	  8(%rsp), %rbp
+	movq	 16(%rsp), %r12
+	movq	 24(%rsp), %r13
+	movq	 32(%rsp), %r14
+	movq	 40(%rsp), %r15
+
+#ifdef WINDOWS_ABI
+	movq	 48(%rsp), %rdi
+	movq	 56(%rsp), %rsi
+	movups	 64(%rsp), %xmm6
+	movups	 80(%rsp), %xmm7
+	movups	 96(%rsp), %xmm8
+	movups	112(%rsp), %xmm9
+	movups	128(%rsp), %xmm10
+	movups	144(%rsp), %xmm11
+	movups	160(%rsp), %xmm12
+	movups	176(%rsp), %xmm13
+	movups	192(%rsp), %xmm14
+	movups	208(%rsp), %xmm15
+#endif
+
+	addq	$STACKSIZE, %rsp
+	ret
+
+	EPILOGUE
+
+
+#else
+/*************************************************************************************
+* TRMM Kernel
+*************************************************************************************/
+
+
+	PROLOGUE
+	PROFCODE
+	
+	subq	$STACKSIZE, %rsp
+	movq	%rbx,   (%rsp)
+	movq	%rbp,  8(%rsp)
+	movq	%r12, 16(%rsp)
+	movq	%r13, 24(%rsp)
+	movq	%r14, 32(%rsp)
+	movq	%r15, 40(%rsp)
+
+	vzeroupper
+
+#ifdef WINDOWS_ABI
+	movq	%rdi,    48(%rsp)
+	movq	%rsi,    56(%rsp)
+	movups	%xmm6,   64(%rsp)
+	movups	%xmm7,   80(%rsp)
+	movups	%xmm8,   96(%rsp)
+	movups	%xmm9,  112(%rsp)
+	movups	%xmm10, 128(%rsp)
+	movups	%xmm11, 144(%rsp)
+	movups	%xmm12, 160(%rsp)
+	movups	%xmm13, 176(%rsp)
+	movups	%xmm14, 192(%rsp)
+	movups	%xmm15, 208(%rsp)
+
+	movq	ARG1,      OLD_M
+	movq	ARG2,      OLD_N
+	movq	ARG3,      OLD_K
+	movq	OLD_A,     A
+	movq	OLD_B,     B
+	movq	OLD_C,     C
+	movq	OLD_LDC,   LDC
+#ifdef TRMMKERNEL
+	movsd	OLD_OFFSET, %xmm12
+#endif
+	vmovaps	%xmm3, %xmm0
+
+#else
+	movq	STACKSIZE +  8(%rsp), LDC
+#ifdef TRMMKERNEL
+	movsd	STACKSIZE + 16(%rsp), %xmm12
+#endif
+
+#endif
+
+	movq    %rsp, SP      # save old stack
+        subq    $128 + L_BUFFER_SIZE, %rsp
+        andq    $-4096, %rsp    # align stack
+
+        STACK_TOUCH
+
+	cmpq	$0, OLD_M
+	je	.L999
+
+	cmpq	$0, OLD_N
+	je	.L999
+
+	cmpq	$0, OLD_K
+	je	.L999
+
+	movq	OLD_M, M
+	movq	OLD_N, N
+	movq	OLD_K, K
+
+	vmovsd	 %xmm0, ALPHA
+
+	salq	$BASE_SHIFT, LDC
+
+	movq    N, %rax
+        xorq    %rdx, %rdx
+        movq    $2,  %rdi
+        divq    %rdi                    //    N / 6
+        movq    %rax, Ndiv6             //    N / 6
+        movq    %rdx, Nmod6             //    N % 6
+
+	
+
+#ifdef TRMMKERNEL
+	vmovsd	%xmm12, OFFSET
+	vmovsd	%xmm12, KK
+#ifndef LEFT
+	negq	KK
+#endif	
+#endif
+
+	movq	Ndiv6,  J
+	cmpq	$0, J
+	je	.L1_0
+	ALIGN_4
+
+.L2_01:
+	// copy to sub buffer
+	movq	B, BO1
+	leaq    BUFFER1, BO		// first buffer to BO
+	movq	K, %rax
+	ALIGN_4
+
+.L2_02b:
+
+	vmovsd	(BO1), %xmm0
+	vmovsd	%xmm0,       (BO)
+	addq	$2*SIZE,BO1
+	addq	$2*SIZE,BO
+	decq	%rax
+	jnz	.L2_02b
+
+.L2_02c:
+
+	movq	BO1, B			// next offset of B
+
+.L2_10:
+	movq	C, CO1
+	leaq	(C, LDC, 2), C		// c += 2 * ldc
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        movq    OFFSET, %rax
+        movq    %rax, KK
+#endif
+	
+	movq	A, AO		 	// aoffset = a
+	addq	$32 * SIZE, AO
+
+	movq	M,  I
+	sarq	$4, I			// i = (m >> 4)
+	je	.L2_20
+
+	ALIGN_4
+
+.L2_11:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO
+        salq    $4, %rax                        // rax = rax * 16 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $16, %rax	// number of values in AO
+#else
+        addq    $2, %rax	// number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+	andq	$-8, %rax			//  K = K - ( K % 8 )
+	je	.L2_16
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$4, %rax			// rax = rax * 16 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_12:
+
+	prefetcht0	B_PR1(BO,BI, SIZE)
+	KERNEL16x2_1(xxx)
+	KERNEL16x2_2(xxx)
+	KERNEL16x2_3(xxx)
+	KERNEL16x2_4(xxx)
+
+	KERNEL16x2_1(xxx)
+	KERNEL16x2_2(xxx)
+	KERNEL16x2_3(xxx)
+	KERNEL16x2_4(xxx)
+
+	je	.L2_16
+
+	prefetcht0	B_PR1(BO,BI, SIZE)
+	KERNEL16x2_1(xxx)
+	KERNEL16x2_2(xxx)
+	KERNEL16x2_3(xxx)
+	KERNEL16x2_4(xxx)
+
+	KERNEL16x2_1(xxx)
+	KERNEL16x2_2(xxx)
+	KERNEL16x2_3(xxx)
+	KERNEL16x2_4(xxx)
+
+	je	.L2_16
+
+	jmp	.L2_12
+	ALIGN_4
+
+.L2_16:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_19
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$4, %rax			// rax = rax * 16 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_17:
+
+	KERNEL16x2_SUB(xxx)
+	addq	$2, BI
+	addq	$16, %rax
+	jl	.L2_17
+	ALIGN_4
+
+
+.L2_19:
+
+	vbroadcastss	ALPHA, %xmm0
+
+#ifndef TRMMKERNEL
+
+	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddps 4 * SIZE(CO1),%xmm0, %xmm7,%xmm7
+	vfmaddps 8 * SIZE(CO1),%xmm0, %xmm10,%xmm10
+	vfmaddps 12 * SIZE(CO1),%xmm0, %xmm13,%xmm13
+
+	vfmaddps 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+	vfmaddps 4 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
+	vfmaddps 8 * SIZE(CO1, LDC),%xmm0, %xmm11,%xmm11
+	vfmaddps 12 * SIZE(CO1, LDC),%xmm0, %xmm14,%xmm14
+
+#else
+	vmulps	%xmm0, %xmm4,%xmm4
+	vmulps	%xmm0, %xmm7,%xmm7
+	vmulps	%xmm0, %xmm10,%xmm10
+	vmulps	%xmm0, %xmm13,%xmm13
+
+	vmulps	%xmm0, %xmm5,%xmm5
+	vmulps	%xmm0, %xmm8,%xmm8
+	vmulps	%xmm0, %xmm11,%xmm11
+	vmulps	%xmm0, %xmm14,%xmm14
+
+#endif
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm7 , 4 * SIZE(CO1)
+	vmovups	%xmm10, 8 * SIZE(CO1)
+	vmovups	%xmm13,12 * SIZE(CO1)
+
+	vmovups	%xmm5 ,  	(CO1, LDC)
+	vmovups	%xmm8 , 4 * SIZE(CO1, LDC)
+	vmovups	%xmm11, 8 * SIZE(CO1, LDC)
+	vmovups	%xmm14,12 * SIZE(CO1, LDC)
+
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO
+        salq    $4, %rax                        // rax = rax * 16 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $16, KK				
+#endif
+
+	addq	$16 * SIZE, CO1		# coffset += 16
+	decq	I			# i --
+	jg	.L2_11
+	ALIGN_4	
+
+/**************************************************************************
+* Rest of M 
+***************************************************************************/
+.L2_20:
+	// Test rest of M
+
+	testq	$15, M
+	jz	.L2_60		// to next 3 lines of N
+
+	testq	$8, M		
+	jz	.L2_21pre
+	ALIGN_4
+
+/**************************************************************************/
+
+.L2_20_1:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO
+        salq    $3, %rax                        // rax = rax * 8 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $8, %rax        // number of values in A
+#else
+        addq    $2, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$-8, %rax
+	je	.L2_20_6
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_20_2:
+
+	prefetcht0	B_PR1(BO,BI, SIZE)
+	KERNEL8x2_1(xxx)
+	KERNEL8x2_2(xxx)
+	KERNEL8x2_3(xxx)
+	KERNEL8x2_4(xxx)
+
+	KERNEL8x2_1(xxx)
+	KERNEL8x2_2(xxx)
+	KERNEL8x2_3(xxx)
+	KERNEL8x2_4(xxx)
+
+	je	.L2_20_6
+
+	prefetcht0	B_PR1(BO,BI, SIZE)
+	KERNEL8x2_1(xxx)
+	KERNEL8x2_2(xxx)
+	KERNEL8x2_3(xxx)
+	KERNEL8x2_4(xxx)
+
+	KERNEL8x2_1(xxx)
+	KERNEL8x2_2(xxx)
+	KERNEL8x2_3(xxx)
+	KERNEL8x2_4(xxx)
+
+	je	.L2_20_6
+
+	jmp	.L2_20_2
+	ALIGN_4
+
+.L2_20_6:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_20_9
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_20_7:
+
+	KERNEL8x2_SUB(xxx)
+	addq	$2, BI
+	addq	$8, %rax
+	jl	.L2_20_7
+	ALIGN_4
+
+
+.L2_20_9:
+
+	vbroadcastss	ALPHA, %xmm0
+
+#ifndef TRMMKERNEL
+
+	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddps 4 * SIZE(CO1),%xmm0, %xmm7,%xmm7
+
+	vfmaddps 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+	vfmaddps 4 * SIZE(CO1, LDC),%xmm0, %xmm8,%xmm8
+
+#else
+	vmulps	%xmm0, %xmm4,%xmm4
+	vmulps	%xmm0, %xmm7,%xmm7
+
+	vmulps	%xmm0, %xmm5,%xmm5
+	vmulps	%xmm0, %xmm8,%xmm8
+
+#endif
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm7 , 4 * SIZE(CO1)
+
+	vmovups	%xmm5 ,  	(CO1, LDC)
+	vmovups	%xmm8 , 4 * SIZE(CO1, LDC)
+
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax 
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO         
+        salq    $3, %rax                        // rax = rax * 8 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif  
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $8, KK
+#endif
+
+	addq	$8 * SIZE, CO1		# coffset += 8
+	ALIGN_4
+	
+
+
+/**************************************************************************/
+
+.L2_21pre:
+
+	testq	$4, M		
+	jz	.L2_30
+	ALIGN_4
+
+.L2_21:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO
+        salq    $2, %rax                        // rax = rax * 4 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $4, %rax        // number of values in A
+#else
+        addq    $2, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$-8, %rax
+	je	.L2_26
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 1 ; number of values
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_22:
+
+	prefetcht0	B_PR1(BO,BI, SIZE)
+	KERNEL4x2_1(xxx)
+	KERNEL4x2_2(xxx)
+	KERNEL4x2_3(xxx)
+	KERNEL4x2_4(xxx)
+
+	KERNEL4x2_1(xxx)
+	KERNEL4x2_2(xxx)
+	KERNEL4x2_3(xxx)
+	KERNEL4x2_4(xxx)
+
+	je	.L2_26
+
+	prefetcht0	B_PR1(BO,BI, SIZE)
+	KERNEL4x2_1(xxx)
+	KERNEL4x2_2(xxx)
+	KERNEL4x2_3(xxx)
+	KERNEL4x2_4(xxx)
+
+	KERNEL4x2_1(xxx)
+	KERNEL4x2_2(xxx)
+	KERNEL4x2_3(xxx)
+	KERNEL4x2_4(xxx)
+
+	je	.L2_26
+
+	jmp	.L2_22
+	ALIGN_4
+
+.L2_26:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_29
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_27:
+
+	KERNEL4x2_SUB(xxx)
+	addq	$2, BI
+	addq	$4, %rax
+	jl	.L2_27
+	ALIGN_4
+
+
+.L2_29:
+
+	vbroadcastss	ALPHA, %xmm0
+
+#ifndef TRMMKERNEL
+
+	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddps 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+
+#else
+	vmulps	%xmm0, %xmm4,%xmm4
+	vmulps	%xmm0, %xmm5,%xmm5
+
+#endif
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm5 ,  	(CO1, LDC)
+
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax 
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO         
+        salq    $2, %rax                        // rax = rax * 4 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif  
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $4, KK
+#endif
+
+	addq	$4 * SIZE, CO1		# coffset += 4
+	ALIGN_4
+	
+
+.L2_30:
+	testq	$2, M		
+	jz	.L2_40
+
+	ALIGN_4
+
+.L2_31:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO
+        salq    $1, %rax                        // rax = rax * 2 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $2, %rax        // number of values in AO
+#else
+        addq    $2, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$-8, %rax
+	je	.L2_36
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_32:
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL2x2_1(xxx)
+	KERNEL2x2_2(xxx)
+	KERNEL2x2_3(xxx)
+	KERNEL2x2_4(xxx)
+
+	KERNEL2x2_1(xxx)
+	KERNEL2x2_2(xxx)
+	KERNEL2x2_3(xxx)
+	KERNEL2x2_4(xxx)
+
+	je	.L2_36
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL2x2_1(xxx)
+	KERNEL2x2_2(xxx)
+	KERNEL2x2_3(xxx)
+	KERNEL2x2_4(xxx)
+
+	KERNEL2x2_1(xxx)
+	KERNEL2x2_2(xxx)
+	KERNEL2x2_3(xxx)
+	KERNEL2x2_4(xxx)
+
+	je	.L2_36
+
+	jmp	.L2_32
+	ALIGN_4
+
+.L2_36:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_39
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+	
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_37:
+
+	KERNEL2x2_SUB(xxx)
+	addq	$2, BI
+	addq	$2, %rax
+	jl	.L2_37
+	ALIGN_4
+
+
+.L2_39:
+
+	vmovss	ALPHA, %xmm0
+
+#ifndef TRMMKERNEL
+
+	vfmaddss 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddss 1 * SIZE(CO1),%xmm0, %xmm8,%xmm8
+	vfmaddss 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+	vfmaddss 1 * SIZE(CO1, LDC),%xmm0, %xmm10,%xmm10
+
+#else
+	vmulss	%xmm0, %xmm4,%xmm4
+	vmulss	%xmm0, %xmm8,%xmm8
+	vmulss	%xmm0, %xmm5,%xmm5
+	vmulss	%xmm0, %xmm10,%xmm10
+
+#endif
+
+	vmovss	%xmm4 ,  	(CO1)
+	vmovss	%xmm8 , 1 * SIZE(CO1)
+	vmovss	%xmm5 ,  	(CO1, LDC)
+	vmovss	%xmm10, 1 * SIZE(CO1, LDC)
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax 
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO         
+        salq    $1, %rax                        // rax = rax * 2 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif  
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $2, KK
+#endif
+
+	addq	$2 * SIZE, CO1		# coffset += 2
+	ALIGN_4
+
+.L2_40:
+	testq	$1, M		
+	jz	.L2_60		// to next 2 lines of N
+
+	ALIGN_4
+
+.L2_41:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $1, %rax        // number of values in AO
+#else
+        addq    $2, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+	andq	$-8, %rax
+	je	.L2_46
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_42:
+
+	KERNEL1x2_1(xxx)
+	KERNEL1x2_2(xxx)
+	KERNEL1x2_3(xxx)
+	KERNEL1x2_4(xxx)
+
+	KERNEL1x2_1(xxx)
+	KERNEL1x2_2(xxx)
+	KERNEL1x2_3(xxx)
+	KERNEL1x2_4(xxx)
+
+	je	.L2_46
+
+	KERNEL1x2_1(xxx)
+	KERNEL1x2_2(xxx)
+	KERNEL1x2_3(xxx)
+	KERNEL1x2_4(xxx)
+
+	KERNEL1x2_1(xxx)
+	KERNEL1x2_2(xxx)
+	KERNEL1x2_3(xxx)
+	KERNEL1x2_4(xxx)
+
+	je	.L2_46
+
+	jmp	.L2_42
+	ALIGN_4
+
+.L2_46:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_49
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_47:
+
+	KERNEL1x2_SUB(xxx)
+	addq	$2, BI
+	addq	$1, %rax
+	jl	.L2_47
+	ALIGN_4
+
+
+.L2_49:
+
+	vmovss	ALPHA, %xmm0
+
+#ifndef TRMMKERNEL
+
+	vfmaddss 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddss 	 (CO1, LDC),%xmm0, %xmm5,%xmm5
+
+#else
+	vmulss	%xmm0, %xmm4,%xmm4
+	vmulss	%xmm0, %xmm5,%xmm5
+
+#endif
+
+	vmovss	%xmm4 ,  	(CO1)
+	vmovss	%xmm5 ,  	(CO1, LDC)
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax 
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO         
+        leaq    (AO, %rax, SIZE), AO
+#endif  
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $1, KK
+#endif
+
+	addq	$1 * SIZE, CO1		# coffset += 1
+	ALIGN_4
+	
+
+
+
+	
+.L2_60:
+#if defined(TRMMKERNEL) && !defined(LEFT)
+        addq    $2, KK
+#endif
+
+	decq	J			// j --
+	jg	.L2_01			// next 2 lines of N
+
+
+
+.L1_0:
+
+/************************************************************************************************
+* Loop for Nmod6 % 2 > 0
+*************************************************************************************************/
+
+	movq	Nmod6, J		
+	andq	$1, J			// j % 2
+	je	.L999
+	ALIGN_4
+
+.L1_01:
+	// copy to sub buffer
+	movq	B, BO1
+	leaq    BUFFER1, BO		// first buffer to BO
+	movq	K, %rax
+	ALIGN_4
+
+.L1_02b:
+
+	vmovss	(BO1), %xmm0
+	vmovss	%xmm0,       (BO)
+	addq	$1*SIZE,BO1
+	addq	$1*SIZE,BO
+	decq	%rax
+	jnz	.L1_02b
+
+.L1_02c:
+
+	movq	BO1, B			// next offset of B
+
+.L1_10:
+	movq	C, CO1
+	leaq	(C, LDC, 1), C		// c += 1 * ldc
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        movq    OFFSET, %rax
+        movq    %rax, KK
+#endif
+	
+	movq	A, AO		 	// aoffset = a
+	addq	$32 * SIZE, AO
+
+	movq	M,  I
+	sarq	$4, I			// i = (m >> 4)
+	je	.L1_20
+
+	ALIGN_4
+
+.L1_11:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BO, BI, SIZE), BO
+        salq    $4, %rax                        // rax = rax * 16 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $16, %rax	// number of values in AO
+#else
+        addq    $1, %rax	// number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+	andq	$-8, %rax			//  K = K - ( K % 8 )
+	je	.L1_16
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$4, %rax			// rax = rax * 16 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_12:
+
+	prefetcht0	B_PR1(BO,BI, SIZE)
+	KERNEL16x1_1(xxx)
+	KERNEL16x1_2(xxx)
+	KERNEL16x1_3(xxx)
+	KERNEL16x1_4(xxx)
+
+	KERNEL16x1_1(xxx)
+	KERNEL16x1_2(xxx)
+	KERNEL16x1_3(xxx)
+	KERNEL16x1_4(xxx)
+
+	je	.L1_16
+
+	KERNEL16x1_1(xxx)
+	KERNEL16x1_2(xxx)
+	KERNEL16x1_3(xxx)
+	KERNEL16x1_4(xxx)
+
+	KERNEL16x1_1(xxx)
+	KERNEL16x1_2(xxx)
+	KERNEL16x1_3(xxx)
+	KERNEL16x1_4(xxx)
+
+	je	.L1_16
+
+	jmp	.L1_12
+	ALIGN_4
+
+.L1_16:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_19
+
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$4, %rax			// rax = rax * 16 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_17:
+
+	KERNEL16x1_SUB(xxx)
+	addq	$1, BI
+	addq	$16, %rax
+	jl	.L1_17
+	ALIGN_4
+
+
+.L1_19:
+
+	vbroadcastss	ALPHA, %xmm0
+
+#ifndef TRMMKERNEL
+
+	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddps 4 * SIZE(CO1),%xmm0, %xmm7,%xmm7
+	vfmaddps 8 * SIZE(CO1),%xmm0, %xmm10,%xmm10
+	vfmaddps 12 * SIZE(CO1),%xmm0, %xmm13,%xmm13
+
+#else
+	vmulps	%xmm0, %xmm4,%xmm4
+	vmulps	%xmm0, %xmm7,%xmm7
+	vmulps	%xmm0, %xmm10,%xmm10
+	vmulps	%xmm0, %xmm13,%xmm13
+
+#endif
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm7 , 4 * SIZE(CO1)
+	vmovups	%xmm10, 8 * SIZE(CO1)
+	vmovups	%xmm13,12 * SIZE(CO1)
+
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BO, BI, SIZE), BO
+        salq    $4, %rax                        // rax = rax * 16 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $16, KK				
+#endif
+
+	addq	$16 * SIZE, CO1		# coffset += 16
+	decq	I			# i --
+	jg	.L1_11
+	ALIGN_4	
+
+/**************************************************************************
+* Rest of M 
+***************************************************************************/
+.L1_20:
+	// Test rest of M
+
+	testq	$15, M
+	jz	.L999
+
+	testq	$8, M		
+	jz	.L1_21pre
+	ALIGN_4
+
+/**************************************************************************/
+
+.L1_20_1:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BO, BI, SIZE), BO
+        salq    $3, %rax                        // rax = rax * 8 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $8, %rax        // number of values in A
+#else
+        addq    $1, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$-8, %rax
+	je	.L1_20_6
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_20_2:
+
+	prefetcht0	B_PR1(BO,BI, SIZE)
+	KERNEL8x1_1(xxx)
+	KERNEL8x1_2(xxx)
+	KERNEL8x1_3(xxx)
+	KERNEL8x1_4(xxx)
+
+	KERNEL8x1_1(xxx)
+	KERNEL8x1_2(xxx)
+	KERNEL8x1_3(xxx)
+	KERNEL8x1_4(xxx)
+
+	je	.L1_20_6
+
+	KERNEL8x1_1(xxx)
+	KERNEL8x1_2(xxx)
+	KERNEL8x1_3(xxx)
+	KERNEL8x1_4(xxx)
+
+	KERNEL8x1_1(xxx)
+	KERNEL8x1_2(xxx)
+	KERNEL8x1_3(xxx)
+	KERNEL8x1_4(xxx)
+
+	je	.L1_20_6
+
+	jmp	.L1_20_2
+	ALIGN_4
+
+.L1_20_6:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_20_9
+
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_20_7:
+
+	KERNEL8x1_SUB(xxx)
+	addq	$1, BI
+	addq	$8, %rax
+	jl	.L1_20_7
+	ALIGN_4
+
+
+.L1_20_9:
+
+	vbroadcastss	ALPHA, %xmm0
+
+#ifndef TRMMKERNEL
+
+	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddps 4 * SIZE(CO1),%xmm0, %xmm7,%xmm7
+
+#else
+	vmulps	%xmm0, %xmm4,%xmm4
+	vmulps	%xmm0, %xmm7,%xmm7
+
+#endif
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm7 , 4 * SIZE(CO1)
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax 
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BO, BI, SIZE), BO         
+        salq    $3, %rax                        // rax = rax * 8 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif  
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $8, KK
+#endif
+
+	addq	$8 * SIZE, CO1		# coffset += 8
+	ALIGN_4
+	
+
+
+/**************************************************************************/
+
+.L1_21pre:
+
+	testq	$4, M		
+	jz	.L1_30
+	ALIGN_4
+
+.L1_21:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BO, BI, SIZE), BO
+        salq    $2, %rax                        // rax = rax * 4 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $4, %rax        // number of values in A
+#else
+        addq    $1, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$-8, %rax
+	je	.L1_26
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_22:
+
+	prefetcht0	B_PR1(BO,BI, SIZE)
+	KERNEL4x1_1(xxx)
+	KERNEL4x1_2(xxx)
+	KERNEL4x1_3(xxx)
+	KERNEL4x1_4(xxx)
+
+	KERNEL4x1_1(xxx)
+	KERNEL4x1_2(xxx)
+	KERNEL4x1_3(xxx)
+	KERNEL4x1_4(xxx)
+
+	je	.L1_26
+
+	KERNEL4x1_1(xxx)
+	KERNEL4x1_2(xxx)
+	KERNEL4x1_3(xxx)
+	KERNEL4x1_4(xxx)
+
+	KERNEL4x1_1(xxx)
+	KERNEL4x1_2(xxx)
+	KERNEL4x1_3(xxx)
+	KERNEL4x1_4(xxx)
+
+	je	.L1_26
+
+	jmp	.L1_22
+	ALIGN_4
+
+.L1_26:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_29
+
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_27:
+
+	KERNEL4x1_SUB(xxx)
+	addq	$1, BI
+	addq	$4, %rax
+	jl	.L1_27
+	ALIGN_4
+
+
+.L1_29:
+
+	vbroadcastss	ALPHA, %xmm0
+
+#ifndef TRMMKERNEL
+
+	vfmaddps 	 (CO1),%xmm0, %xmm4,%xmm4
+
+#else
+	vmulps	%xmm0, %xmm4,%xmm4
+
+#endif
+
+	vmovups	%xmm4 ,  	(CO1)
+
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax 
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BO, BI, SIZE), BO         
+        salq    $2, %rax                        // rax = rax * 4 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif  
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $4, KK
+#endif
+
+	addq	$4 * SIZE, CO1		# coffset += 4
+	ALIGN_4
+	
+
+.L1_30:
+	testq	$2, M		
+	jz	.L1_40
+
+	ALIGN_4
+
+.L1_31:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BO, BI, SIZE), BO
+        salq    $1, %rax                        // rax = rax * 2 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $2, %rax        // number of values in AO
+#else
+        addq    $1, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$-8, %rax
+	je	.L1_36
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_32:
+
+	prefetcht0	B_PR1(BO,BI,SIZE)
+	KERNEL2x1_1(xxx)
+	KERNEL2x1_2(xxx)
+	KERNEL2x1_3(xxx)
+	KERNEL2x1_4(xxx)
+
+	KERNEL2x1_1(xxx)
+	KERNEL2x1_2(xxx)
+	KERNEL2x1_3(xxx)
+	KERNEL2x1_4(xxx)
+
+	je	.L1_36
+
+	KERNEL2x1_1(xxx)
+	KERNEL2x1_2(xxx)
+	KERNEL2x1_3(xxx)
+	KERNEL2x1_4(xxx)
+
+	KERNEL2x1_1(xxx)
+	KERNEL2x1_2(xxx)
+	KERNEL2x1_3(xxx)
+	KERNEL2x1_4(xxx)
+
+	je	.L1_36
+
+	jmp	.L1_32
+	ALIGN_4
+
+.L1_36:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_39
+
+	movq    %rax, BI                        //  Index for BO
+	
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_37:
+
+	KERNEL2x1_SUB(xxx)
+	addq	$1, BI
+	addq	$2, %rax
+	jl	.L1_37
+	ALIGN_4
+
+
+.L1_39:
+
+	vmovss	ALPHA, %xmm0
+
+#ifndef TRMMKERNEL
+
+	vfmaddss 	 (CO1),%xmm0, %xmm4,%xmm4
+	vfmaddss 1 * SIZE(CO1),%xmm0, %xmm8,%xmm8
+
+#else
+	vmulss	%xmm0, %xmm4,%xmm4
+	vmulss	%xmm0, %xmm8,%xmm8
+
+#endif
+
+	vmovss	%xmm4 ,  	(CO1)
+	vmovss	%xmm8 , 1 * SIZE(CO1)
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax 
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BO, BI, SIZE), BO         
+        salq    $1, %rax                        // rax = rax * 2 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif  
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $2, KK
+#endif
+
+	addq	$2 * SIZE, CO1		# coffset += 2
+	ALIGN_4
+
+.L1_40:
+	testq	$1, M		
+	jz	.L999
+
+	ALIGN_4
+
+.L1_41:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $2 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BO, BI, SIZE), BO
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $1, %rax        // number of values in AO
+#else
+        addq    $1, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+	andq	$-8, %rax
+	je	.L1_46
+	movq    %rax, BI                        //  Index for BO
+
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_42:
+
+	KERNEL1x1_1(xxx)
+	KERNEL1x1_2(xxx)
+	KERNEL1x1_3(xxx)
+	KERNEL1x1_4(xxx)
+
+	KERNEL1x1_1(xxx)
+	KERNEL1x1_2(xxx)
+	KERNEL1x1_3(xxx)
+	KERNEL1x1_4(xxx)
+
+	je	.L1_46
+
+	KERNEL1x1_1(xxx)
+	KERNEL1x1_2(xxx)
+	KERNEL1x1_3(xxx)
+	KERNEL1x1_4(xxx)
+
+	KERNEL1x1_1(xxx)
+	KERNEL1x1_2(xxx)
+	KERNEL1x1_3(xxx)
+	KERNEL1x1_4(xxx)
+
+	je	.L1_46
+
+	jmp	.L1_42
+	ALIGN_4
+
+.L1_46:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_49
+
+	movq    %rax, BI                        //  Index for BO
+
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_47:
+
+	KERNEL1x1_SUB(xxx)
+	addq	$1, BI
+	addq	$1, %rax
+	jl	.L1_47
+	ALIGN_4
+
+
+.L1_49:
+
+	vmovss	ALPHA, %xmm0
+
+#ifndef TRMMKERNEL
+
+	vfmaddss 	 (CO1),%xmm0, %xmm4,%xmm4
+
+#else
+	vmulss	%xmm0, %xmm4,%xmm4
+
+#endif
+
+	vmovss	%xmm4 ,  	(CO1)
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax 
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BO, BI, SIZE), BO         
+        leaq    (AO, %rax, SIZE), AO
+#endif  
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $1, KK
+#endif
+
+	addq	$1 * SIZE, CO1		# coffset += 1
+	ALIGN_4
+	
+
+.L999:
+	movq   		SP, %rsp
+	movq	   (%rsp), %rbx
+	movq	  8(%rsp), %rbp
+	movq	 16(%rsp), %r12
+	movq	 24(%rsp), %r13
+	movq	 32(%rsp), %r14
+	movq	 40(%rsp), %r15
+
+#ifdef WINDOWS_ABI
+	movq	 48(%rsp), %rdi
+	movq	 56(%rsp), %rsi
+	movups	 64(%rsp), %xmm6
+	movups	 80(%rsp), %xmm7
+	movups	 96(%rsp), %xmm8
+	movups	112(%rsp), %xmm9
+	movups	128(%rsp), %xmm10
+	movups	144(%rsp), %xmm11
+	movups	160(%rsp), %xmm12
+	movups	176(%rsp), %xmm13
+	movups	192(%rsp), %xmm14
+	movups	208(%rsp), %xmm15
+#endif
+
+	addq	$STACKSIZE, %rsp
+	ret
+
+	EPILOGUE
+
+
+
+
+
+#endif
diff --git a/kernel/x86_64/sgemm_kernel_16x4_sandy.S b/kernel/x86_64/sgemm_kernel_16x4_sandy.S
index ea15cd87e..2ee4b1554 100644
--- a/kernel/x86_64/sgemm_kernel_16x4_sandy.S
+++ b/kernel/x86_64/sgemm_kernel_16x4_sandy.S
@@ -1,3167 +1,3167 @@
-/*********************************************************************************
-Copyright (c) 2013, The OpenBLAS Project
-All rights reserved.
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in
-the documentation and/or other materials provided with the
-distribution.
-3. Neither the name of the OpenBLAS project nor the names of
-its contributors may be used to endorse or promote products
-derived from this software without specific prior written permission.
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
-USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-**********************************************************************************/
-
-#define ASSEMBLER
-#include "common.h"
- 
-#define OLD_M	%rdi
-#define OLD_N	%rsi
-#define M	%r13
-#define J	%r14
-#define OLD_K	%rdx
-
-#define A	%rcx
-#define B	%r8
-#define C	%r9
-#define LDC	%r10
-	
-#define I	%r11
-#define AO	%rdi
-#define BO	%rsi
-#define	CO1	%r15
-#define K	%r12
-#define BI	%rbp
-#define	SP	%rbx
-
-#define BO1	%rdi
-#define	CO2	%rdx
-
-#ifndef WINDOWS_ABI
-
-#define STACKSIZE 96
-
-#else
-
-#define STACKSIZE 256
-
-#define OLD_A		40 + STACKSIZE(%rsp)
-#define OLD_B		48 + STACKSIZE(%rsp)
-#define OLD_C		56 + STACKSIZE(%rsp)
-#define OLD_LDC		64 + STACKSIZE(%rsp)
-#define OLD_OFFSET	72 + STACKSIZE(%rsp)
-
-#endif
-
-#define L_BUFFER_SIZE 8192
-
-#define Ndiv6	 24(%rsp)
-#define Nmod6	 32(%rsp)
-#define N	 40(%rsp)
-#define ALPHA	 48(%rsp)
-#define OFFSET	 56(%rsp)
-#define KK	 64(%rsp)
-#define KKK	 72(%rsp)
-#define BUFFER1	           128(%rsp)
-
-#if defined(OS_WINDOWS)
-#if   L_BUFFER_SIZE > 16384
-#define STACK_TOUCH \
-        movl    $0,  4096 * 4(%rsp);\
-        movl    $0,  4096 * 3(%rsp);\
-        movl    $0,  4096 * 2(%rsp);\
-        movl    $0,  4096 * 1(%rsp);
-#elif L_BUFFER_SIZE > 12288
-#define STACK_TOUCH \
-        movl    $0,  4096 * 3(%rsp);\
-        movl    $0,  4096 * 2(%rsp);\
-        movl    $0,  4096 * 1(%rsp);
-#elif L_BUFFER_SIZE > 8192
-#define STACK_TOUCH \
-        movl    $0,  4096 * 2(%rsp);\
-        movl    $0,  4096 * 1(%rsp);
-#elif L_BUFFER_SIZE > 4096
-#define STACK_TOUCH \
-        movl    $0,  4096 * 1(%rsp);
-#else
-#define STACK_TOUCH
-#endif
-#else
-#define STACK_TOUCH
-#endif
-
-#define	A_PR1	512
-#define	B_PR1	512
-
-/*******************************************************************************************
-* 4 lines of N
-*******************************************************************************************/
-
-.macro KERNEL16x4_SUB
-	vmovups 	-16 * SIZE(AO, %rax, SIZE), %ymm0
-	vmovups 	 -8 * SIZE(AO, %rax, SIZE), %ymm1
-	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %ymm2
-	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %ymm3
-	vmulps		%ymm2 , %ymm0 , %ymm12
-	vmulps		%ymm2 , %ymm1 , %ymm13
-	vmulps		%ymm3 , %ymm0 , %ymm14
-	vmulps		%ymm3 , %ymm1 , %ymm15
-	vaddps 	        %ymm12, %ymm4 , %ymm4
-	vaddps 	        %ymm13, %ymm5 , %ymm5
-	vaddps 	        %ymm14, %ymm6 , %ymm6
-	vaddps 	        %ymm15, %ymm7 , %ymm7
-	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %ymm2
-	vbroadcastss	 -1 * SIZE(BO, BI, SIZE), %ymm3
-	vmulps		%ymm2 , %ymm0 , %ymm12
-	vmulps		%ymm2 , %ymm1 , %ymm13
-	vmulps		%ymm3 , %ymm0 , %ymm14
-	vmulps		%ymm3 , %ymm1 , %ymm15
-	vaddps 	        %ymm12, %ymm8 , %ymm8
-	vaddps 	        %ymm13, %ymm9 , %ymm9
-	vaddps 	        %ymm14, %ymm10, %ymm10
-	vaddps 	        %ymm15, %ymm11, %ymm11
-	addq	$ 4 , BI	
-	addq	$ 16, %rax 
-.endm
-
-.macro SAVE16x4
-
-	vbroadcastss	ALPHA, %ymm0
-
-	vmulps	%ymm0 , %ymm4 , %ymm4
-	vmulps	%ymm0 , %ymm5 , %ymm5
-	vmulps	%ymm0 , %ymm6 , %ymm6
-	vmulps	%ymm0 , %ymm7 , %ymm7
-	vmulps	%ymm0 , %ymm8 , %ymm8
-	vmulps	%ymm0 , %ymm9 , %ymm9
-	vmulps	%ymm0 , %ymm10, %ymm10
-	vmulps	%ymm0 , %ymm11, %ymm11
-
-
-#if !defined(TRMMKERNEL)
-
-	vaddps 	        (CO1), %ymm4,%ymm4
-	vaddps  8 * SIZE(CO1), %ymm5,%ymm5
-
-	vaddps 	        (CO1, LDC), %ymm6,%ymm6
-	vaddps  8 * SIZE(CO1, LDC), %ymm7,%ymm7
-
-	vaddps 	        (CO2), %ymm8,%ymm8
-	vaddps  8 * SIZE(CO2), %ymm9,%ymm9
-
-	vaddps 	        (CO2, LDC), %ymm10,%ymm10
-	vaddps  8 * SIZE(CO2, LDC), %ymm11,%ymm11
-
-#endif
-
-	vmovups	%ymm4 ,  	(CO1)
-	vmovups	%ymm5 , 8 * SIZE(CO1)
-
-	vmovups	%ymm6 ,  	(CO1, LDC)
-	vmovups	%ymm7 , 8 * SIZE(CO1, LDC)
-
-	vmovups	%ymm8 ,  	(CO2)
-	vmovups	%ymm9 , 8 * SIZE(CO2)
-
-	vmovups	%ymm10,  	(CO2, LDC)
-	vmovups	%ymm11, 8 * SIZE(CO2, LDC)
-
-	prefetcht0	64(CO1)
-	prefetcht0	64(CO1, LDC)
-	prefetcht0	64(CO2)
-	prefetcht0	64(CO2, LDC)
-
-.endm
-
-
-
-/*******************************************************************************************/
-
-.macro KERNEL8x4_SUB
-	vmovups 	-16 * SIZE(AO, %rax, SIZE), %ymm0
-	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %ymm2
-	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %ymm3
-	vmulps		%ymm2 , %ymm0 , %ymm12
-	vmulps		%ymm3 , %ymm0 , %ymm14
-	vaddps 	        %ymm12, %ymm4 , %ymm4
-	vaddps 	        %ymm14, %ymm6 , %ymm6
-	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %ymm2
-	vbroadcastss	 -1 * SIZE(BO, BI, SIZE), %ymm3
-	vmulps		%ymm2 , %ymm0 , %ymm12
-	vmulps		%ymm3 , %ymm0 , %ymm14
-	vaddps 	        %ymm12, %ymm8 , %ymm8
-	vaddps 	        %ymm14, %ymm10, %ymm10
-	addq	$ 4 , BI	
-	addq	$ 8 , %rax 
-.endm
-
-.macro SAVE8x4
-
-	vbroadcastss	ALPHA, %ymm0
-
-	vmulps	%ymm0 , %ymm4 , %ymm4
-	vmulps	%ymm0 , %ymm6 , %ymm6
-	vmulps	%ymm0 , %ymm8 , %ymm8
-	vmulps	%ymm0 , %ymm10, %ymm10
-
-
-#if !defined(TRMMKERNEL)
-
-	vaddps 	        (CO1), %ymm4,%ymm4
-	vaddps 	        (CO1, LDC), %ymm6,%ymm6
-	vaddps 	        (CO2), %ymm8,%ymm8
-	vaddps 	        (CO2, LDC), %ymm10,%ymm10
-
-#endif
-
-	vmovups	%ymm4 ,  	(CO1)
-	vmovups	%ymm6 ,  	(CO1, LDC)
-	vmovups	%ymm8 ,  	(CO2)
-	vmovups	%ymm10,  	(CO2, LDC)
-
-.endm
-
-
-
-/*******************************************************************************************/
-
-.macro KERNEL4x4_SUB
-	vmovups 	-16 * SIZE(AO, %rax, SIZE), %xmm0
-	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm2
-	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %xmm3
-	vmulps		%xmm2 , %xmm0 , %xmm12
-	vmulps		%xmm3 , %xmm0 , %xmm14
-	vaddps 	        %xmm12, %xmm4 , %xmm4
-	vaddps 	        %xmm14, %xmm6 , %xmm6
-	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %xmm2
-	vbroadcastss	 -1 * SIZE(BO, BI, SIZE), %xmm3
-	vmulps		%xmm2 , %xmm0 , %xmm12
-	vmulps		%xmm3 , %xmm0 , %xmm14
-	vaddps 	        %xmm12, %xmm8 , %xmm8
-	vaddps 	        %xmm14, %xmm10, %xmm10
-	addq	$ 4 , BI	
-	addq	$ 4 , %rax 
-.endm
-
-.macro SAVE4x4
-
-	vbroadcastss	ALPHA, %xmm0
-
-	vmulps	%xmm0 , %xmm4 , %xmm4
-	vmulps	%xmm0 , %xmm6 , %xmm6
-	vmulps	%xmm0 , %xmm8 , %xmm8
-	vmulps	%xmm0 , %xmm10, %xmm10
-
-
-#if !defined(TRMMKERNEL)
-
-	vaddps 	        (CO1), %xmm4,%xmm4
-	vaddps 	        (CO1, LDC), %xmm6,%xmm6
-	vaddps 	        (CO2), %xmm8,%xmm8
-	vaddps 	        (CO2, LDC), %xmm10,%xmm10
-
-#endif
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm6 ,  	(CO1, LDC)
-	vmovups	%xmm8 ,  	(CO2)
-	vmovups	%xmm10,  	(CO2, LDC)
-
-.endm
-
-
-/*******************************************************************************************/
-
-.macro KERNEL2x4_SUB
-	vmovss 	-16 * SIZE(AO, %rax, SIZE), %xmm0
-	vmovss 	-15 * SIZE(AO, %rax, SIZE), %xmm1
-	vmovss	 -4 * SIZE(BO, BI, SIZE), %xmm2
-	vmovss	 -3 * SIZE(BO, BI, SIZE), %xmm3
-	vmulss		%xmm2 , %xmm0 , %xmm12
-	vmulss		%xmm2 , %xmm1 , %xmm13
-	vmulss		%xmm3 , %xmm0 , %xmm14
-	vmulss		%xmm3 , %xmm1 , %xmm15
-	vaddss 	        %xmm12, %xmm4 , %xmm4
-	vaddss 	        %xmm13, %xmm5 , %xmm5
-	vaddss 	        %xmm14, %xmm6 , %xmm6
-	vaddss 	        %xmm15, %xmm7 , %xmm7
-	vmovss	 -2 * SIZE(BO, BI, SIZE), %xmm2
-	vmovss	 -1 * SIZE(BO, BI, SIZE), %xmm3
-	vmulss		%xmm2 , %xmm0 , %xmm12
-	vmulss		%xmm2 , %xmm1 , %xmm13
-	vmulss		%xmm3 , %xmm0 , %xmm14
-	vmulss		%xmm3 , %xmm1 , %xmm15
-	vaddss 	        %xmm12, %xmm8 , %xmm8
-	vaddss 	        %xmm13, %xmm9 , %xmm9
-	vaddss 	        %xmm14, %xmm10, %xmm10
-	vaddss 	        %xmm15, %xmm11, %xmm11
-	addq	$ 4 , BI	
-	addq	$ 2, %rax 
-.endm
-
-.macro SAVE2x4
-
-	vmovss	ALPHA, %xmm0
-
-	vmulss	%xmm0 , %xmm4 , %xmm4
-	vmulss	%xmm0 , %xmm5 , %xmm5
-	vmulss	%xmm0 , %xmm6 , %xmm6
-	vmulss	%xmm0 , %xmm7 , %xmm7
-	vmulss	%xmm0 , %xmm8 , %xmm8
-	vmulss	%xmm0 , %xmm9 , %xmm9
-	vmulss	%xmm0 , %xmm10, %xmm10
-	vmulss	%xmm0 , %xmm11, %xmm11
-
-
-#if !defined(TRMMKERNEL)
-
-	vaddss 	        (CO1), %xmm4,%xmm4
-	vaddss  1 * SIZE(CO1), %xmm5,%xmm5
-
-	vaddss 	        (CO1, LDC), %xmm6,%xmm6
-	vaddss  1 * SIZE(CO1, LDC), %xmm7,%xmm7
-
-	vaddss 	        (CO2), %xmm8,%xmm8
-	vaddss  1 * SIZE(CO2), %xmm9,%xmm9
-
-	vaddss 	        (CO2, LDC), %xmm10,%xmm10
-	vaddss  1 * SIZE(CO2, LDC), %xmm11,%xmm11
-
-#endif
-
-	vmovss	%xmm4 ,  	(CO1)
-	vmovss	%xmm5 , 1 * SIZE(CO1)
-
-	vmovss	%xmm6 ,  	(CO1, LDC)
-	vmovss	%xmm7 , 1 * SIZE(CO1, LDC)
-
-	vmovss	%xmm8 ,  	(CO2)
-	vmovss	%xmm9 , 1 * SIZE(CO2)
-
-	vmovss	%xmm10,  	(CO2, LDC)
-	vmovss	%xmm11, 1 * SIZE(CO2, LDC)
-
-.endm
-
-
-/*******************************************************************************************/
-
-.macro KERNEL1x4_SUB
-	vmovss 	-16 * SIZE(AO, %rax, SIZE), %xmm0
-	vmovss	 -4 * SIZE(BO, BI, SIZE), %xmm2
-	vmovss	 -3 * SIZE(BO, BI, SIZE), %xmm3
-	vmulss		%xmm2 , %xmm0 , %xmm12
-	vmulss		%xmm3 , %xmm0 , %xmm14
-	vaddss 	        %xmm12, %xmm4 , %xmm4
-	vaddss 	        %xmm14, %xmm6 , %xmm6
-	vmovss	 -2 * SIZE(BO, BI, SIZE), %xmm2
-	vmovss	 -1 * SIZE(BO, BI, SIZE), %xmm3
-	vmulss		%xmm2 , %xmm0 , %xmm12
-	vmulss		%xmm3 , %xmm0 , %xmm14
-	vaddss 	        %xmm12, %xmm8 , %xmm8
-	vaddss 	        %xmm14, %xmm10, %xmm10
-	addq	$ 4 , BI	
-	addq	$ 1, %rax 
-.endm
-
-.macro SAVE1x4
-
-	vmovss	ALPHA, %xmm0
-
-	vmulss	%xmm0 , %xmm4 , %xmm4
-	vmulss	%xmm0 , %xmm6 , %xmm6
-	vmulss	%xmm0 , %xmm8 , %xmm8
-	vmulss	%xmm0 , %xmm10, %xmm10
-
-
-#if !defined(TRMMKERNEL)
-
-	vaddss 	        (CO1), %xmm4,%xmm4
-	vaddss 	        (CO1, LDC), %xmm6,%xmm6
-	vaddss 	        (CO2), %xmm8,%xmm8
-	vaddss 	        (CO2, LDC), %xmm10,%xmm10
-
-#endif
-
-	vmovss	%xmm4 ,  	(CO1)
-	vmovss	%xmm6 ,  	(CO1, LDC)
-	vmovss	%xmm8 ,  	(CO2)
-	vmovss	%xmm10,  	(CO2, LDC)
-
-.endm
-
-
-/*******************************************************************************************/
-
-/*******************************************************************************************
-* 2 lines of N
-*******************************************************************************************/
-
-.macro KERNEL16x2_SUB
-	vmovups 	-16 * SIZE(AO, %rax, SIZE), %ymm0
-	vmovups 	 -8 * SIZE(AO, %rax, SIZE), %ymm1
-	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %ymm2
-	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %ymm3
-	vmulps		%ymm2 , %ymm0 , %ymm12
-	vmulps		%ymm2 , %ymm1 , %ymm13
-	vmulps		%ymm3 , %ymm0 , %ymm14
-	vmulps		%ymm3 , %ymm1 , %ymm15
-	vaddps 	        %ymm12, %ymm4 , %ymm4
-	vaddps 	        %ymm13, %ymm5 , %ymm5
-	vaddps 	        %ymm14, %ymm6 , %ymm6
-	vaddps 	        %ymm15, %ymm7 , %ymm7
-	addq	$ 2 , BI	
-	addq	$ 16, %rax 
-.endm
-
-.macro SAVE16x2
-
-	vbroadcastss	ALPHA, %ymm0
-
-	vmulps	%ymm0 , %ymm4 , %ymm4
-	vmulps	%ymm0 , %ymm5 , %ymm5
-	vmulps	%ymm0 , %ymm6 , %ymm6
-	vmulps	%ymm0 , %ymm7 , %ymm7
-
-
-#if !defined(TRMMKERNEL)
-
-	vaddps 	        (CO1), %ymm4,%ymm4
-	vaddps  8 * SIZE(CO1), %ymm5,%ymm5
-
-	vaddps 	        (CO1, LDC), %ymm6,%ymm6
-	vaddps  8 * SIZE(CO1, LDC), %ymm7,%ymm7
-
-#endif
-
-	vmovups	%ymm4 ,  	(CO1)
-	vmovups	%ymm5 , 8 * SIZE(CO1)
-
-	vmovups	%ymm6 ,  	(CO1, LDC)
-	vmovups	%ymm7 , 8 * SIZE(CO1, LDC)
-
-.endm
-
-
-
-/*******************************************************************************************/
-
-.macro KERNEL8x2_SUB
-	vmovups 	-16 * SIZE(AO, %rax, SIZE), %ymm0
-	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %ymm2
-	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %ymm3
-	vmulps		%ymm2 , %ymm0 , %ymm12
-	vmulps		%ymm3 , %ymm0 , %ymm14
-	vaddps 	        %ymm12, %ymm4 , %ymm4
-	vaddps 	        %ymm14, %ymm6 , %ymm6
-	addq	$ 2 , BI	
-	addq	$ 8 , %rax 
-.endm
-
-.macro SAVE8x2
-
-	vbroadcastss	ALPHA, %ymm0
-
-	vmulps	%ymm0 , %ymm4 , %ymm4
-	vmulps	%ymm0 , %ymm6 , %ymm6
-
-#if !defined(TRMMKERNEL)
-
-	vaddps 	        (CO1), %ymm4,%ymm4
-	vaddps 	        (CO1, LDC), %ymm6,%ymm6
-
-#endif
-
-	vmovups	%ymm4 ,  	(CO1)
-	vmovups	%ymm6 ,  	(CO1, LDC)
-
-.endm
-
-
-
-/*******************************************************************************************/
-
-.macro KERNEL4x2_SUB
-	vmovups 	-16 * SIZE(AO, %rax, SIZE), %xmm0
-	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm2
-	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %xmm3
-	vmulps		%xmm2 , %xmm0 , %xmm12
-	vmulps		%xmm3 , %xmm0 , %xmm14
-	vaddps 	        %xmm12, %xmm4 , %xmm4
-	vaddps 	        %xmm14, %xmm6 , %xmm6
-	addq	$ 2 , BI	
-	addq	$ 4 , %rax 
-.endm
-
-.macro SAVE4x2
-
-	vbroadcastss	ALPHA, %xmm0
-
-	vmulps	%xmm0 , %xmm4 , %xmm4
-	vmulps	%xmm0 , %xmm6 , %xmm6
-
-#if !defined(TRMMKERNEL)
-
-	vaddps 	        (CO1), %xmm4,%xmm4
-	vaddps 	        (CO1, LDC), %xmm6,%xmm6
-
-#endif
-
-	vmovups	%xmm4 ,  	(CO1)
-	vmovups	%xmm6 ,  	(CO1, LDC)
-
-.endm
-
-
-/*******************************************************************************************/
-
-.macro KERNEL2x2_SUB
-	vmovss 	-16 * SIZE(AO, %rax, SIZE), %xmm0
-	vmovss 	-15 * SIZE(AO, %rax, SIZE), %xmm1
-	vmovss	 -4 * SIZE(BO, BI, SIZE), %xmm2
-	vmovss	 -3 * SIZE(BO, BI, SIZE), %xmm3
-	vmulss		%xmm2 , %xmm0 , %xmm12
-	vmulss		%xmm2 , %xmm1 , %xmm13
-	vmulss		%xmm3 , %xmm0 , %xmm14
-	vmulss		%xmm3 , %xmm1 , %xmm15
-	vaddss 	        %xmm12, %xmm4 , %xmm4
-	vaddss 	        %xmm13, %xmm5 , %xmm5
-	vaddss 	        %xmm14, %xmm6 , %xmm6
-	vaddss 	        %xmm15, %xmm7 , %xmm7
-	addq	$ 2 , BI	
-	addq	$ 2, %rax 
-.endm
-
-.macro SAVE2x2
-
-	vmovss	ALPHA, %xmm0
-
-	vmulss	%xmm0 , %xmm4 , %xmm4
-	vmulss	%xmm0 , %xmm5 , %xmm5
-	vmulss	%xmm0 , %xmm6 , %xmm6
-	vmulss	%xmm0 , %xmm7 , %xmm7
-
-
-#if !defined(TRMMKERNEL)
-
-	vaddss 	        (CO1), %xmm4,%xmm4
-	vaddss  1 * SIZE(CO1), %xmm5,%xmm5
-
-	vaddss 	        (CO1, LDC), %xmm6,%xmm6
-	vaddss  1 * SIZE(CO1, LDC), %xmm7,%xmm7
-
-#endif
-
-	vmovss	%xmm4 ,  	(CO1)
-	vmovss	%xmm5 , 1 * SIZE(CO1)
-
-	vmovss	%xmm6 ,  	(CO1, LDC)
-	vmovss	%xmm7 , 1 * SIZE(CO1, LDC)
-
-.endm
-
-
-/*******************************************************************************************/
-
-.macro KERNEL1x2_SUB
-	vmovss 	-16 * SIZE(AO, %rax, SIZE), %xmm0
-	vmovss	 -4 * SIZE(BO, BI, SIZE), %xmm2
-	vmovss	 -3 * SIZE(BO, BI, SIZE), %xmm3
-	vmulss		%xmm2 , %xmm0 , %xmm12
-	vmulss		%xmm3 , %xmm0 , %xmm14
-	vaddss 	        %xmm12, %xmm4 , %xmm4
-	vaddss 	        %xmm14, %xmm6 , %xmm6
-	addq	$ 2 , BI	
-	addq	$ 1, %rax 
-.endm
-
-.macro SAVE1x2
-
-	vmovss	ALPHA, %xmm0
-
-	vmulss	%xmm0 , %xmm4 , %xmm4
-	vmulss	%xmm0 , %xmm6 , %xmm6
-
-#if !defined(TRMMKERNEL)
-
-	vaddss 	        (CO1), %xmm4,%xmm4
-	vaddss 	        (CO1, LDC), %xmm6,%xmm6
-
-#endif
-
-	vmovss	%xmm4 ,  	(CO1)
-	vmovss	%xmm6 ,  	(CO1, LDC)
-
-.endm
-
-
-/*******************************************************************************************/
-
-/*******************************************************************************************
-* 1 line of N
-*******************************************************************************************/
-
-.macro KERNEL16x1_SUB
-	vmovups 	-16 * SIZE(AO, %rax, SIZE), %ymm0
-	vmovups 	 -8 * SIZE(AO, %rax, SIZE), %ymm1
-	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %ymm2
-	vmulps		%ymm2 , %ymm0 , %ymm12
-	vmulps		%ymm2 , %ymm1 , %ymm13
-	vaddps 	        %ymm12, %ymm4 , %ymm4
-	vaddps 	        %ymm13, %ymm5 , %ymm5
-	addq	$ 1 , BI	
-	addq	$ 16, %rax 
-.endm
-
-.macro SAVE16x1
-
-	vbroadcastss	ALPHA, %ymm0
-
-	vmulps	%ymm0 , %ymm4 , %ymm4
-	vmulps	%ymm0 , %ymm5 , %ymm5
-
-#if !defined(TRMMKERNEL)
-
-	vaddps 	        (CO1), %ymm4,%ymm4
-	vaddps  8 * SIZE(CO1), %ymm5,%ymm5
-
-#endif
-
-	vmovups	%ymm4 ,  	(CO1)
-	vmovups	%ymm5 , 8 * SIZE(CO1)
-
-.endm
-
-
-/*******************************************************************************************/
-
-.macro KERNEL8x1_SUB
-	vmovups 	-16 * SIZE(AO, %rax, SIZE), %ymm0
-	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %ymm2
-	vmulps		%ymm2 , %ymm0 , %ymm12
-	vaddps 	        %ymm12, %ymm4 , %ymm4
-	addq	$ 1 , BI	
-	addq	$ 8 , %rax 
-.endm
-
-.macro SAVE8x1
-
-	vbroadcastss	ALPHA, %ymm0
-
-	vmulps	%ymm0 , %ymm4 , %ymm4
-
-#if !defined(TRMMKERNEL)
-
-	vaddps 	        (CO1), %ymm4,%ymm4
-
-#endif
-
-	vmovups	%ymm4 ,  	(CO1)
-
-.endm
-
-
-
-/*******************************************************************************************/
-
-.macro KERNEL4x1_SUB
-	vmovups 	-16 * SIZE(AO, %rax, SIZE), %xmm0
-	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm2
-	vmulps		%xmm2 , %xmm0 , %xmm12
-	vaddps 	        %xmm12, %xmm4 , %xmm4
-	addq	$ 1 , BI	
-	addq	$ 4 , %rax 
-.endm
-
-.macro SAVE4x1
-
-	vbroadcastss	ALPHA, %xmm0
-
-	vmulps	%xmm0 , %xmm4 , %xmm4
-
-#if !defined(TRMMKERNEL)
-
-	vaddps 	        (CO1), %xmm4,%xmm4
-
-#endif
-
-	vmovups	%xmm4 ,  	(CO1)
-
-.endm
-
-
-/*******************************************************************************************/
-
-.macro KERNEL2x1_SUB
-	vmovss 	-16 * SIZE(AO, %rax, SIZE), %xmm0
-	vmovss 	-15 * SIZE(AO, %rax, SIZE), %xmm1
-	vmovss	 -4 * SIZE(BO, BI, SIZE), %xmm2
-	vmulss		%xmm2 , %xmm0 , %xmm12
-	vmulss		%xmm2 , %xmm1 , %xmm13
-	vaddss 	        %xmm12, %xmm4 , %xmm4
-	vaddss 	        %xmm13, %xmm5 , %xmm5
-	addq	$ 1 , BI	
-	addq	$ 2 , %rax 
-.endm
-
-.macro SAVE2x1
-
-	vmovss	ALPHA, %xmm0
-
-	vmulss	%xmm0 , %xmm4 , %xmm4
-	vmulss	%xmm0 , %xmm5 , %xmm5
-
-#if !defined(TRMMKERNEL)
-
-	vaddss 	        (CO1), %xmm4,%xmm4
-	vaddss  1 * SIZE(CO1), %xmm5,%xmm5
-
-#endif
-
-	vmovss	%xmm4 ,  	(CO1)
-	vmovss	%xmm5 , 1 * SIZE(CO1)
-
-.endm
-
-
-/*******************************************************************************************/
-
-.macro KERNEL1x1_SUB
-	vmovss 	-16 * SIZE(AO, %rax, SIZE), %xmm0
-	vmovss	 -4 * SIZE(BO, BI, SIZE), %xmm2
-	vmulss		%xmm2 , %xmm0 , %xmm12
-	vaddss 	        %xmm12, %xmm4 , %xmm4
-	addq	$ 1 , BI	
-	addq	$ 1 , %rax 
-.endm
-
-.macro SAVE1x1
-
-	vmovss	ALPHA, %xmm0
-
-	vmulss	%xmm0 , %xmm4 , %xmm4
-
-#if !defined(TRMMKERNEL)
-
-	vaddss 	        (CO1), %xmm4,%xmm4
-
-#endif
-
-	vmovss	%xmm4 ,  	(CO1)
-
-.endm
-
-
-/*******************************************************************************************/
-
-/*************************************************************************************
-* TRMM Kernel
-*************************************************************************************/
-
-
-	PROLOGUE
-	PROFCODE
-	
-	subq	$STACKSIZE, %rsp
-	movq	%rbx,   (%rsp)
-	movq	%rbp,  8(%rsp)
-	movq	%r12, 16(%rsp)
-	movq	%r13, 24(%rsp)
-	movq	%r14, 32(%rsp)
-	movq	%r15, 40(%rsp)
-
-	vzeroupper
-
-#ifdef WINDOWS_ABI
-	movq	%rdi,    48(%rsp)
-	movq	%rsi,    56(%rsp)
-	movups	%xmm6,   64(%rsp)
-	movups	%xmm7,   80(%rsp)
-	movups	%xmm8,   96(%rsp)
-	movups	%xmm9,  112(%rsp)
-	movups	%xmm10, 128(%rsp)
-	movups	%xmm11, 144(%rsp)
-	movups	%xmm12, 160(%rsp)
-	movups	%xmm13, 176(%rsp)
-	movups	%xmm14, 192(%rsp)
-	movups	%xmm15, 208(%rsp)
-
-	movq	ARG1,      OLD_M
-	movq	ARG2,      OLD_N
-	movq	ARG3,      OLD_K
-	movq	OLD_A,     A
-	movq	OLD_B,     B
-	movq	OLD_C,     C
-	movq	OLD_LDC,   LDC
-#ifdef TRMMKERNEL
-	vmovsd	OLD_OFFSET, %xmm12
-#endif
-	vmovaps	%xmm3, %xmm0
-
-#else
-	movq	STACKSIZE +  8(%rsp), LDC
-#ifdef TRMMKERNEL
-	movsd	STACKSIZE + 16(%rsp), %xmm12
-#endif
-
-#endif
-
-	movq    %rsp, SP      # save old stack
-        subq    $128 + L_BUFFER_SIZE, %rsp
-        andq    $-4096, %rsp    # align stack
-
-        STACK_TOUCH
-
-	cmpq	$0, OLD_M
-	je	.L999
-
-	cmpq	$0, OLD_N
-	je	.L999
-
-	cmpq	$0, OLD_K
-	je	.L999
-
-	movq	OLD_M, M
-	movq	OLD_N, N
-	movq	OLD_K, K
-
-	vmovss	 %xmm0, ALPHA
-
-	salq	$BASE_SHIFT, LDC
-
-	movq    N, %rax
-        xorq    %rdx, %rdx
-        movq    $4,  %rdi
-        divq    %rdi                    //    N / 4
-        movq    %rax, Ndiv6             //    N / 4
-        movq    %rdx, Nmod6             //    N % 4
-
-	
-
-#ifdef TRMMKERNEL
-	vmovsd	%xmm12, OFFSET
-	vmovsd	%xmm12, KK
-#ifndef LEFT
-	negq	KK
-#endif	
-#endif
-
-	movq	Ndiv6,  J
-	cmpq	$0, J
-	je	.L2_0
-	ALIGN_4
-
-/*******************************************************************************************/
-
-.L4_01:
-	// copy to sub buffer
-	movq	B, BO1
-	leaq    BUFFER1, BO		// first buffer to BO
-	movq	K, %rax
-	sarq	$2, %rax		// K / 4
-	jz	.L4_01b
-	ALIGN_4
-
-
-.L4_01a:
-        prefetcht0 512(BO1)
-        prefetchw  512(BO)
-
-	vmovups	       (BO1), %xmm0
-	vmovups	 4*SIZE(BO1), %xmm1
-	vmovups	 8*SIZE(BO1), %xmm2
-	vmovups	12*SIZE(BO1), %xmm3
-
-	vmovups	%xmm0,       (BO)
-	vmovups	%xmm1, 4*SIZE(BO)
-	vmovups	%xmm2, 8*SIZE(BO)
-	vmovups	%xmm3,12*SIZE(BO)
-
-	addq	$ 16*SIZE,BO1
-	addq	$ 16*SIZE,BO
-	decq	%rax
-	jnz	.L4_01a
-
-
-.L4_01b:
-
-        movq    K, %rax
-        andq    $3, %rax                // K % 4
-        jz      .L4_02d
-        ALIGN_4
-
-.L4_02c:
-
-	vmovups	(BO1), %xmm0
-	vmovups	%xmm0, (BO)
-	addq	$ 4*SIZE,BO1
-	addq	$ 4*SIZE,BO
-	decq	%rax
-	jnz	.L4_02c
-
-.L4_02d:
-
-	movq	BO1, B			// next offset of B
-
-.L4_10:
-	movq	 C, CO1
-	leaq	(C, LDC, 2), CO2	
-	leaq	(C, LDC, 4), C		// c += 4 * ldc
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        movq    OFFSET, %rax
-        movq    %rax, KK
-#endif
-	
-	movq	A, AO		 	// aoffset = a
-	addq	$ 16 * SIZE, AO
-
-	movq	M,  I
-	sarq	$4, I			// i = (m >> 4)
-	je	.L4_20
-
-	ALIGN_4
-
-.L4_11:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             	// first buffer to BO
-        addq    $4 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-	leaq	(,BI, 4), BI			// BI = BI * 4 ; number of values
-        leaq    (BO, BI, SIZE), BO
-        salq    $4, %rax                        // rax = rax * 16 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $16, %rax	// number of values in AO
-#else
-        addq    $4, %rax	// number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-	andq	$-8, %rax			//  K = K - ( K % 8 )
-	je	.L4_16
-	movq    %rax, BI                        //  Index for BO
-        leaq    (,BI,4) , BI                   	//  BI = BI * 4 ; number of values
-
-	salq	$4, %rax			// rax = rax * 16 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L4_12:
-
-	prefetcht0	A_PR1(AO, %rax, SIZE)
-	prefetcht0	B_PR1(BO, BI  , SIZE)
-	KERNEL16x4_SUB
-	prefetcht0	A_PR1(AO, %rax, SIZE)
-	KERNEL16x4_SUB
-	prefetcht0	A_PR1(AO, %rax, SIZE)
-	KERNEL16x4_SUB
-	prefetcht0	A_PR1(AO, %rax, SIZE)
-	KERNEL16x4_SUB
-
-	prefetcht0	A_PR1(AO, %rax, SIZE)
-	prefetcht0	B_PR1(BO, BI  , SIZE)
-	KERNEL16x4_SUB
-	prefetcht0	A_PR1(AO, %rax, SIZE)
-	KERNEL16x4_SUB
-	prefetcht0	A_PR1(AO, %rax, SIZE)
-	KERNEL16x4_SUB
-	prefetcht0	A_PR1(AO, %rax, SIZE)
-	KERNEL16x4_SUB
-
-	je	.L4_16
-
-	prefetcht0	A_PR1(AO, %rax, SIZE)
-	prefetcht0	B_PR1(BO, BI  , SIZE)
-	KERNEL16x4_SUB
-	prefetcht0	A_PR1(AO, %rax, SIZE)
-	KERNEL16x4_SUB
-	prefetcht0	A_PR1(AO, %rax, SIZE)
-	KERNEL16x4_SUB
-	prefetcht0	A_PR1(AO, %rax, SIZE)
-	KERNEL16x4_SUB
-
-	prefetcht0	A_PR1(AO, %rax, SIZE)
-	prefetcht0	B_PR1(BO, BI  , SIZE)
-	KERNEL16x4_SUB
-	prefetcht0	A_PR1(AO, %rax, SIZE)
-	KERNEL16x4_SUB
-	prefetcht0	A_PR1(AO, %rax, SIZE)
-	KERNEL16x4_SUB
-	prefetcht0	A_PR1(AO, %rax, SIZE)
-	KERNEL16x4_SUB
-
-	je	.L4_16
-
-	jmp	.L4_12
-	ALIGN_4
-
-.L4_16:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L4_19
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (,BI,4), BI                   	//  BI = BI * 4 ; number of values
-
-	salq	$4, %rax			// rax = rax * 16 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L4_17:
-
-	KERNEL16x4_SUB
-
-	jl	.L4_17
-	ALIGN_4
-
-
-.L4_19:
-
-	SAVE16x4
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-	leaq	(,BI, 4), BI			// BI = BI * 4 ; number of values
-        leaq    (BO, BI, SIZE), BO
-        salq    $4, %rax                        // rax = rax * 16 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $16, KK				
-#endif
-
-	addq	$16 * SIZE, CO1		# coffset += 16
-	addq	$16 * SIZE, CO2		# coffset += 16
-	decq	I			# i --
-	jg	.L4_11
-	ALIGN_4	
-
-/**************************************************************************
-* Rest of M 
-***************************************************************************/
-.L4_20:
-	// Test rest of M
-
-	testq	$15, M
-	jz	.L4_60		// to next 3 lines of N
-
-	testq	$8, M		
-	jz	.L4_21pre
-	ALIGN_4
-
-/**************************************************************************/
-
-.L4_20_1:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-	leaq	(,BI, 4), BI			// BI = BI * 4 ; number of values
-        leaq    (BO, BI, SIZE), BO
-        salq    $3, %rax                        // rax = rax * 8 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $8, %rax        // number of values in A
-#else
-        addq    $4, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-
-	andq	$-8, %rax
-	je	.L4_20_6
-	movq    %rax, BI                        //  Index for BO
-        leaq    (,BI,4), BI                   	//  BI = BI * 4 ; number of values
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L4_20_2:
-
-	KERNEL8x4_SUB
-	KERNEL8x4_SUB
-	KERNEL8x4_SUB
-	KERNEL8x4_SUB
-
-	KERNEL8x4_SUB
-	KERNEL8x4_SUB
-	KERNEL8x4_SUB
-	KERNEL8x4_SUB
-
-	je	.L4_20_6
-
-	KERNEL8x4_SUB
-	KERNEL8x4_SUB
-	KERNEL8x4_SUB
-	KERNEL8x4_SUB
-
-	KERNEL8x4_SUB
-	KERNEL8x4_SUB
-	KERNEL8x4_SUB
-	KERNEL8x4_SUB
-
-	je	.L4_20_6
-
-	jmp	.L4_20_2
-	ALIGN_4
-
-.L4_20_6:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L4_20_9
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (,BI,4), BI                   	//  BI = BI * 4 ; number of values
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L4_20_7:
-
-	KERNEL8x4_SUB
-
-	jl	.L4_20_7
-	ALIGN_4
-
-
-.L4_20_9:
-
-	SAVE8x4
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax 
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-	leaq	(,BI, 4), BI			// BI = BI * 4 ; number of values
-        leaq    (BO, BI, SIZE), BO         
-        salq    $3, %rax                        // rax = rax * 8 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif  
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $8, KK
-#endif
-
-	addq	$8 * SIZE, CO1		# coffset += 8
-	addq	$8 * SIZE, CO2		# coffset += 8
-	ALIGN_4
-	
-
-
-/**************************************************************************/
-
-.L4_21pre:
-
-	testq	$4, M		
-	jz	.L4_30
-	ALIGN_4
-
-.L4_21:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-	leaq	(,BI, 4), BI			// BI = BI * 4 ; number of values
-        leaq    (BO, BI, SIZE), BO
-        salq    $2, %rax                        // rax = rax * 4 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $4, %rax        // number of values in A
-#else
-        addq    $4, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-
-	andq	$-8, %rax
-	je	.L4_26
-	movq    %rax, BI                        //  Index for BO
-        leaq    (,BI,4), BI                   	//  BI = BI * 4 ; number of values
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L4_22:
-
-	KERNEL4x4_SUB
-	KERNEL4x4_SUB
-	KERNEL4x4_SUB
-	KERNEL4x4_SUB
-
-	KERNEL4x4_SUB
-	KERNEL4x4_SUB
-	KERNEL4x4_SUB
-	KERNEL4x4_SUB
-
-	je	.L4_26
-
-	KERNEL4x4_SUB
-	KERNEL4x4_SUB
-	KERNEL4x4_SUB
-	KERNEL4x4_SUB
-
-	KERNEL4x4_SUB
-	KERNEL4x4_SUB
-	KERNEL4x4_SUB
-	KERNEL4x4_SUB
-
-	je	.L4_26
-
-	jmp	.L4_22
-	ALIGN_4
-
-.L4_26:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L4_29
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (,BI,4), BI                   	//  BI = BI * 4 ; number of values
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L4_27:
-
-	KERNEL4x4_SUB
-
-	jl	.L4_27
-	ALIGN_4
-
-
-.L4_29:
-
-	SAVE4x4
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax 
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-	leaq	(,BI, 4), BI			// BI = BI * 4 ; number of values
-        leaq    (BO, BI, SIZE), BO         
-        salq    $2, %rax                        // rax = rax * 4 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif  
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $4, KK
-#endif
-
-	addq	$4 * SIZE, CO1		# coffset += 4
-	addq	$4 * SIZE, CO2		# coffset += 4
-	ALIGN_4
-	
-
-.L4_30:
-	testq	$2, M		
-	jz	.L4_40
-
-	ALIGN_4
-
-.L4_31:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-	leaq	(,BI, 4), BI			// BI = BI * 4 ; number of values
-        leaq    (BO, BI, SIZE), BO
-        salq    $1, %rax                        // rax = rax * 2 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $2, %rax        // number of values in AO
-#else
-        addq    $4, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-
-	andq	$-8, %rax
-	je	.L4_36
-	movq    %rax, BI                        //  Index for BO
-        leaq    (,BI,4), BI                   	//  BI = BI * 4 ; number of values
-
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L4_32:
-
-	KERNEL2x4_SUB
-	KERNEL2x4_SUB
-	KERNEL2x4_SUB
-	KERNEL2x4_SUB
-
-	KERNEL2x4_SUB
-	KERNEL2x4_SUB
-	KERNEL2x4_SUB
-	KERNEL2x4_SUB
-
-	je	.L4_36
-
-	KERNEL2x4_SUB
-	KERNEL2x4_SUB
-	KERNEL2x4_SUB
-	KERNEL2x4_SUB
-
-	KERNEL2x4_SUB
-	KERNEL2x4_SUB
-	KERNEL2x4_SUB
-	KERNEL2x4_SUB
-
-	je	.L4_36
-
-	jmp	.L4_32
-	ALIGN_4
-
-.L4_36:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L4_39
-
-	movq    %rax, BI                        //  Index for BO
-	leaq	(,BI, 4), BI			// BI = BI * 4 ; number of values
-	
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L4_37:
-
-	KERNEL2x4_SUB
-
-	jl	.L4_37
-	ALIGN_4
-
-
-.L4_39:
-
-	SAVE2x4
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax 
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-	leaq	(,BI, 4), BI			// BI = BI * 4 ; number of values
-        leaq    (BO, BI, SIZE), BO         
-        salq    $1, %rax                        // rax = rax * 2 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif  
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $2, KK
-#endif
-
-	addq	$2 * SIZE, CO1		# coffset += 2
-	addq	$2 * SIZE, CO2		# coffset += 2
-	ALIGN_4
-
-.L4_40:
-	testq	$1, M		
-	jz	.L4_60		// to next 4 lines of N
-
-	ALIGN_4
-
-.L4_41:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-	leaq	(,BI, 4), BI			// BI = BI * 4 ; number of values
-        leaq    (BO, BI, SIZE), BO
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $1, %rax        // number of values in AO
-#else
-        addq    $4, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-	andq	$-8, %rax
-	je	.L4_46
-	movq    %rax, BI                        //  Index for BO
-        leaq    (,BI,4), BI                   	//  BI = BI * 4 ; number of values
-
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L4_42:
-
-	KERNEL1x4_SUB
-	KERNEL1x4_SUB
-	KERNEL1x4_SUB
-	KERNEL1x4_SUB
-
-	KERNEL1x4_SUB
-	KERNEL1x4_SUB
-	KERNEL1x4_SUB
-	KERNEL1x4_SUB
-
-	je	.L4_46
-
-	KERNEL1x4_SUB
-	KERNEL1x4_SUB
-	KERNEL1x4_SUB
-	KERNEL1x4_SUB
-
-	KERNEL1x4_SUB
-	KERNEL1x4_SUB
-	KERNEL1x4_SUB
-	KERNEL1x4_SUB
-
-	je	.L4_46
-
-	jmp	.L4_42
-	ALIGN_4
-
-.L4_46:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L4_49
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (,BI,4), BI                   	//  BI = BI * 4 ; number of values
-
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L4_47:
-
-	KERNEL1x4_SUB
-
-	jl	.L4_47
-	ALIGN_4
-
-
-.L4_49:
-
-	SAVE1x4
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax 
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-	leaq	(,BI, 4), BI			// BI = BI * 4 ; number of values
-        leaq    (BO, BI, SIZE), BO         
-        leaq    (AO, %rax, SIZE), AO
-#endif  
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $1, KK
-#endif
-
-	addq	$1 * SIZE, CO1		# coffset += 1
-	addq	$1 * SIZE, CO2		# coffset += 1
-	ALIGN_4
-	
-
-
-
-	
-.L4_60:
-#if defined(TRMMKERNEL) && !defined(LEFT)
-        addq    $4, KK
-#endif
-
-	decq	J			// j --
-	jg	.L4_01			// next 4 lines of N
-
-
-
-/*******************************************************************************************/
-.L2_0:
-
-	movq	Nmod6, J		
-	andq	$3, J			// j % 4
-	je	.L999
-
-	movq	Nmod6, J		
-	andq	$2, J			// j % 4
-	je	.L1_0
-
-.L2_01:
-
-	// copy to sub buffer
-	movq	B, BO1
-	leaq    BUFFER1, BO		// first buffer to BO
-	movq	K, %rax
-	sarq	$2, %rax		// K / 4
-	jz	.L2_01b
-	ALIGN_4
-
-.L2_01a:
-
-	vmovsd	      (BO1), %xmm0
-	vmovsd	2*SIZE(BO1), %xmm1
-	vmovsd	4*SIZE(BO1), %xmm2
-	vmovsd	6*SIZE(BO1), %xmm3
-
-	vmovsd	%xmm0,       (BO)
-	vmovsd	%xmm1, 2*SIZE(BO)
-	vmovsd	%xmm2, 4*SIZE(BO)
-	vmovsd	%xmm3, 6*SIZE(BO)
-
-	addq	$8*SIZE,BO1
-	addq	$8*SIZE,BO
-	decq	%rax
-	jnz	.L2_01a
-
-
-.L2_01b:
-
-        movq    K, %rax
-        andq    $3, %rax                // K % 4
-        jz      .L2_02d
-        ALIGN_4
-
-.L2_02c:
-
-	vmovsd 	(BO1), %xmm0
-	vmovsd 	%xmm0, (BO)
-	addq	$2*SIZE,BO1
-	addq	$2*SIZE,BO
-	decq	%rax
-	jnz	.L2_02c
-
-.L2_02d:
-
-	movq	BO1, B			// next offset of B
-
-.L2_10:
-	movq	C, CO1
-	leaq	(C, LDC, 2), C		// c += 2 * ldc
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        movq    OFFSET, %rax
-        movq    %rax, KK
-#endif
-	
-	movq	A, AO		 	// aoffset = a
-	addq	$16 * SIZE, AO
-
-	movq	M,  I
-	sarq	$4, I			// i = (m >> 4)
-	je	.L2_20
-
-	ALIGN_4
-
-.L2_11:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO
-        salq    $4, %rax                        // rax = rax * 16 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $16, %rax	// number of values in AO
-#else
-        addq    $2, %rax	// number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-	andq	$-8, %rax			//  K = K - ( K % 8 )
-	je	.L2_16
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$4, %rax			// rax = rax * 16 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_12:
-
-	KERNEL16x2_SUB
-	KERNEL16x2_SUB
-	KERNEL16x2_SUB
-	KERNEL16x2_SUB
-
-	KERNEL16x2_SUB
-	KERNEL16x2_SUB
-	KERNEL16x2_SUB
-	KERNEL16x2_SUB
-
-	je	.L2_16
-
-	KERNEL16x2_SUB
-	KERNEL16x2_SUB
-	KERNEL16x2_SUB
-	KERNEL16x2_SUB
-
-	KERNEL16x2_SUB
-	KERNEL16x2_SUB
-	KERNEL16x2_SUB
-	KERNEL16x2_SUB
-
-	je	.L2_16
-
-	jmp	.L2_12
-	ALIGN_4
-
-.L2_16:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_19
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$4, %rax			// rax = rax * 16 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_17:
-
-	KERNEL16x2_SUB
-
-	jl	.L2_17
-	ALIGN_4
-
-
-.L2_19:
-
-	SAVE16x2
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO
-        salq    $4, %rax                        // rax = rax * 16 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $16, KK				
-#endif
-
-	addq	$16 * SIZE, CO1		# coffset += 16
-	decq	I			# i --
-	jg	.L2_11
-	ALIGN_4	
-
-/**************************************************************************
-* Rest of M 
-***************************************************************************/
-.L2_20:
-	// Test rest of M
-
-	testq	$15, M
-	jz	.L2_60		// to next 2 lines of N
-
-	testq	$8, M		
-	jz	.L2_21pre
-	ALIGN_4
-
-/**************************************************************************/
-
-.L2_20_1:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO
-        salq    $3, %rax                        // rax = rax * 8 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $8, %rax        // number of values in A
-#else
-        addq    $2, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-
-	andq	$-8, %rax
-	je	.L2_20_6
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_20_2:
-
-
-	KERNEL8x2_SUB
-	KERNEL8x2_SUB
-	KERNEL8x2_SUB
-	KERNEL8x2_SUB
-
-	KERNEL8x2_SUB
-	KERNEL8x2_SUB
-	KERNEL8x2_SUB
-	KERNEL8x2_SUB
-
-	je	.L2_20_6
-
-	KERNEL8x2_SUB
-	KERNEL8x2_SUB
-	KERNEL8x2_SUB
-	KERNEL8x2_SUB
-
-	KERNEL8x2_SUB
-	KERNEL8x2_SUB
-	KERNEL8x2_SUB
-	KERNEL8x2_SUB
-
-	je	.L2_20_6
-
-	jmp	.L2_20_2
-	ALIGN_4
-
-.L2_20_6:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_20_9
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_20_7:
-
-	KERNEL8x2_SUB
-
-	jl	.L2_20_7
-	ALIGN_4
-
-
-.L2_20_9:
-
-	SAVE8x2
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax 
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO         
-        salq    $3, %rax                        // rax = rax * 8 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif  
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $8, KK
-#endif
-
-	addq	$8 * SIZE, CO1		# coffset += 8
-	ALIGN_4
-	
-
-
-/**************************************************************************/
-
-.L2_21pre:
-
-	testq	$4, M		
-	jz	.L2_30
-	ALIGN_4
-
-.L2_21:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO
-        salq    $2, %rax                        // rax = rax * 4 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $4, %rax        // number of values in A
-#else
-        addq    $2, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-
-	andq	$-8, %rax
-	je	.L2_26
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 1 ; number of values
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_22:
-
-
-	KERNEL4x2_SUB
-	KERNEL4x2_SUB
-	KERNEL4x2_SUB
-	KERNEL4x2_SUB
-
-	KERNEL4x2_SUB
-	KERNEL4x2_SUB
-	KERNEL4x2_SUB
-	KERNEL4x2_SUB
-
-	je	.L2_26
-
-	KERNEL4x2_SUB
-	KERNEL4x2_SUB
-	KERNEL4x2_SUB
-	KERNEL4x2_SUB
-
-	KERNEL4x2_SUB
-	KERNEL4x2_SUB
-	KERNEL4x2_SUB
-	KERNEL4x2_SUB
-
-	je	.L2_26
-
-	jmp	.L2_22
-	ALIGN_4
-
-.L2_26:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_29
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_27:
-
-	KERNEL4x2_SUB
-
-	jl	.L2_27
-	ALIGN_4
-
-
-.L2_29:
-
-	SAVE4x2
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax 
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO         
-        salq    $2, %rax                        // rax = rax * 4 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif  
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $4, KK
-#endif
-
-	addq	$4 * SIZE, CO1		# coffset += 4
-	ALIGN_4
-	
-
-.L2_30:
-	testq	$2, M		
-	jz	.L2_40
-
-	ALIGN_4
-
-.L2_31:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO
-        salq    $1, %rax                        // rax = rax * 2 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $2, %rax        // number of values in AO
-#else
-        addq    $2, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-
-	andq	$-8, %rax
-	je	.L2_36
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_32:
-
-	KERNEL2x2_SUB
-	KERNEL2x2_SUB
-	KERNEL2x2_SUB
-	KERNEL2x2_SUB
-
-	KERNEL2x2_SUB
-	KERNEL2x2_SUB
-	KERNEL2x2_SUB
-	KERNEL2x2_SUB
-
-	je	.L2_36
-
-	KERNEL2x2_SUB
-	KERNEL2x2_SUB
-	KERNEL2x2_SUB
-	KERNEL2x2_SUB
-
-	KERNEL2x2_SUB
-	KERNEL2x2_SUB
-	KERNEL2x2_SUB
-	KERNEL2x2_SUB
-
-	je	.L2_36
-
-	jmp	.L2_32
-	ALIGN_4
-
-.L2_36:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_39
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-	
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_37:
-
-	KERNEL2x2_SUB
-
-	jl	.L2_37
-	ALIGN_4
-
-
-.L2_39:
-
-	SAVE2x2
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax 
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO         
-        salq    $1, %rax                        // rax = rax * 2 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif  
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $2, KK
-#endif
-
-	addq	$2 * SIZE, CO1		# coffset += 2
-	ALIGN_4
-
-.L2_40:
-	testq	$1, M		
-	jz	.L2_60		// to next 2 lines of N
-
-	ALIGN_4
-
-.L2_41:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $1, %rax        // number of values in AO
-#else
-        addq    $2, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-	andq	$-8, %rax
-	je	.L2_46
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_42:
-
-	KERNEL1x2_SUB
-	KERNEL1x2_SUB
-	KERNEL1x2_SUB
-	KERNEL1x2_SUB
-
-	KERNEL1x2_SUB
-	KERNEL1x2_SUB
-	KERNEL1x2_SUB
-	KERNEL1x2_SUB
-
-	je	.L2_46
-
-	KERNEL1x2_SUB
-	KERNEL1x2_SUB
-	KERNEL1x2_SUB
-	KERNEL1x2_SUB
-
-	KERNEL1x2_SUB
-	KERNEL1x2_SUB
-	KERNEL1x2_SUB
-	KERNEL1x2_SUB
-
-	je	.L2_46
-
-	jmp	.L2_42
-	ALIGN_4
-
-.L2_46:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L2_49
-
-	movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L2_47:
-
-	KERNEL1x2_SUB
-
-	jl	.L2_47
-	ALIGN_4
-
-
-.L2_49:
-
-	SAVE1x2
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax 
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
-        leaq    (BO, BI, SIZE), BO         
-        leaq    (AO, %rax, SIZE), AO
-#endif  
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $1, KK
-#endif
-
-	addq	$1 * SIZE, CO1		# coffset += 1
-	ALIGN_4
-	
-
-
-
-	
-.L2_60:
-#if defined(TRMMKERNEL) && !defined(LEFT)
-        addq    $2, KK
-#endif
-
-
-
-
-.L1_0:
-
-/************************************************************************************************
-* Loop for Nmod6 % 2 > 0
-*************************************************************************************************/
-
-	movq	Nmod6, J		
-	andq	$1, J			// j % 2
-	je	.L999
-	ALIGN_4
-
-.L1_01:
-	// copy to sub buffer
-	movq	B, BO1
-	leaq    BUFFER1, BO		// first buffer to BO
-	movq	K, %rax
-	ALIGN_4
-
-.L1_02b:
-
-	vmovss	(BO1), %xmm0
-	vmovss	%xmm0,       (BO)
-	addq	$1*SIZE,BO1
-	addq	$1*SIZE,BO
-	decq	%rax
-	jnz	.L1_02b
-
-.L1_02c:
-
-	movq	BO1, B			// next offset of B
-
-.L1_10:
-	movq	C, CO1
-	leaq	(C, LDC, 1), C		// c += 1 * ldc
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        movq    OFFSET, %rax
-        movq    %rax, KK
-#endif
-	
-	movq	A, AO		 	// aoffset = a
-	addq	$16 * SIZE, AO
-
-	movq	M,  I
-	sarq	$4, I			// i = (m >> 4)
-	je	.L1_20
-
-	ALIGN_4
-
-.L1_11:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BO, BI, SIZE), BO
-        salq    $4, %rax                        // rax = rax * 16 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $16, %rax	// number of values in AO
-#else
-        addq    $1, %rax	// number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-	andq	$-8, %rax			//  K = K - ( K % 8 )
-	je	.L1_16
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$4, %rax			// rax = rax * 16 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_12:
-
-	KERNEL16x1_SUB
-	KERNEL16x1_SUB
-	KERNEL16x1_SUB
-	KERNEL16x1_SUB
-
-	KERNEL16x1_SUB
-	KERNEL16x1_SUB
-	KERNEL16x1_SUB
-	KERNEL16x1_SUB
-
-	je	.L1_16
-
-	KERNEL16x1_SUB
-	KERNEL16x1_SUB
-	KERNEL16x1_SUB
-	KERNEL16x1_SUB
-
-	KERNEL16x1_SUB
-	KERNEL16x1_SUB
-	KERNEL16x1_SUB
-	KERNEL16x1_SUB
-
-	je	.L1_16
-
-	jmp	.L1_12
-	ALIGN_4
-
-.L1_16:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_19
-
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$4, %rax			// rax = rax * 16 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_17:
-
-	KERNEL16x1_SUB
-
-	jl	.L1_17
-	ALIGN_4
-
-
-.L1_19:
-
-	SAVE16x1
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BO, BI, SIZE), BO
-        salq    $4, %rax                        // rax = rax * 16 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $16, KK				
-#endif
-
-	addq	$16 * SIZE, CO1		# coffset += 16
-	decq	I			# i --
-	jg	.L1_11
-	ALIGN_4	
-
-/**************************************************************************
-* Rest of M 
-***************************************************************************/
-.L1_20:
-	// Test rest of M
-
-	testq	$15, M
-	jz	.L999
-
-	testq	$8, M		
-	jz	.L1_21pre
-	ALIGN_4
-
-/**************************************************************************/
-
-.L1_20_1:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BO, BI, SIZE), BO
-        salq    $3, %rax                        // rax = rax * 8 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $8, %rax        // number of values in A
-#else
-        addq    $1, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-
-	andq	$-8, %rax
-	je	.L1_20_6
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_20_2:
-
-	KERNEL8x1_SUB
-	KERNEL8x1_SUB
-	KERNEL8x1_SUB
-	KERNEL8x1_SUB
-
-	KERNEL8x1_SUB
-	KERNEL8x1_SUB
-	KERNEL8x1_SUB
-	KERNEL8x1_SUB
-
-	je	.L1_20_6
-
-	KERNEL8x1_SUB
-	KERNEL8x1_SUB
-	KERNEL8x1_SUB
-	KERNEL8x1_SUB
-
-	KERNEL8x1_SUB
-	KERNEL8x1_SUB
-	KERNEL8x1_SUB
-	KERNEL8x1_SUB
-
-	je	.L1_20_6
-
-	jmp	.L1_20_2
-	ALIGN_4
-
-.L1_20_6:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_20_9
-
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$3, %rax			// rax = rax * 8 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_20_7:
-
-	KERNEL8x1_SUB
-
-	jl	.L1_20_7
-	ALIGN_4
-
-
-.L1_20_9:
-
-	SAVE8x1
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax 
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BO, BI, SIZE), BO         
-        salq    $3, %rax                        // rax = rax * 8 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif  
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $8, KK
-#endif
-
-	addq	$8 * SIZE, CO1		# coffset += 8
-	ALIGN_4
-	
-
-
-/**************************************************************************/
-
-.L1_21pre:
-
-	testq	$4, M		
-	jz	.L1_30
-	ALIGN_4
-
-.L1_21:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BO, BI, SIZE), BO
-        salq    $2, %rax                        // rax = rax * 4 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $4, %rax        // number of values in A
-#else
-        addq    $1, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-
-	andq	$-8, %rax
-	je	.L1_26
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_22:
-
-	KERNEL4x1_SUB
-	KERNEL4x1_SUB
-	KERNEL4x1_SUB
-	KERNEL4x1_SUB
-
-	KERNEL4x1_SUB
-	KERNEL4x1_SUB
-	KERNEL4x1_SUB
-	KERNEL4x1_SUB
-
-	je	.L1_26
-
-	KERNEL4x1_SUB
-	KERNEL4x1_SUB
-	KERNEL4x1_SUB
-	KERNEL4x1_SUB
-
-	KERNEL4x1_SUB
-	KERNEL4x1_SUB
-	KERNEL4x1_SUB
-	KERNEL4x1_SUB
-
-	je	.L1_26
-
-	jmp	.L1_22
-	ALIGN_4
-
-.L1_26:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_29
-
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$2, %rax			// rax = rax * 4 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_27:
-
-	KERNEL4x1_SUB
-
-	jl	.L1_27
-	ALIGN_4
-
-
-.L1_29:
-
-	SAVE4x1
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax 
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BO, BI, SIZE), BO         
-        salq    $2, %rax                        // rax = rax * 4 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif  
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $4, KK
-#endif
-
-	addq	$4 * SIZE, CO1		# coffset += 4
-	ALIGN_4
-	
-
-.L1_30:
-	testq	$2, M		
-	jz	.L1_40
-
-	ALIGN_4
-
-.L1_31:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BO, BI, SIZE), BO
-        salq    $1, %rax                        // rax = rax * 2 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $2, %rax        // number of values in AO
-#else
-        addq    $1, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-
-	andq	$-8, %rax
-	je	.L1_36
-	movq    %rax, BI                        //  Index for BO
-
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_32:
-
-	KERNEL2x1_SUB
-	KERNEL2x1_SUB
-	KERNEL2x1_SUB
-	KERNEL2x1_SUB
-
-	KERNEL2x1_SUB
-	KERNEL2x1_SUB
-	KERNEL2x1_SUB
-	KERNEL2x1_SUB
-
-	je	.L1_36
-
-	KERNEL2x1_SUB
-	KERNEL2x1_SUB
-	KERNEL2x1_SUB
-	KERNEL2x1_SUB
-
-	KERNEL2x1_SUB
-	KERNEL2x1_SUB
-	KERNEL2x1_SUB
-	KERNEL2x1_SUB
-
-	je	.L1_36
-
-	jmp	.L1_32
-	ALIGN_4
-
-.L1_36:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_39
-
-	movq    %rax, BI                        //  Index for BO
-	
-	salq	$1, %rax			// rax = rax *2 ; number of values
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_37:
-
-	KERNEL2x1_SUB
-
-	jl	.L1_37
-	ALIGN_4
-
-
-.L1_39:
-
-	SAVE2x1
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax 
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BO, BI, SIZE), BO         
-        salq    $1, %rax                        // rax = rax * 2 ; number of values
-        leaq    (AO, %rax, SIZE), AO
-#endif  
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $2, KK
-#endif
-
-	addq	$2 * SIZE, CO1		# coffset += 2
-	ALIGN_4
-
-.L1_40:
-	testq	$1, M		
-	jz	.L999
-
-	ALIGN_4
-
-.L1_41:
-#if !defined(TRMMKERNEL) || \
-        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-#else
-        movq    KK, %rax
-        leaq    BUFFER1, BO             // first buffer to BO
-        addq    $4 * SIZE, BO
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BO, BI, SIZE), BO
-        leaq    (AO, %rax, SIZE), AO
-#endif
-
-
-	vzeroall
-
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
-        movq    K, %rax
-        subq    KK, %rax
-        movq    %rax, KKK
-#else
-        movq    KK, %rax
-#ifdef LEFT
-        addq    $1, %rax        // number of values in AO
-#else
-        addq    $1, %rax        // number of values in BO
-#endif
-        movq    %rax, KKK
-#endif
-
-	andq	$-8, %rax
-	je	.L1_46
-	movq    %rax, BI                        //  Index for BO
-
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_42:
-
-	KERNEL1x1_SUB
-	KERNEL1x1_SUB
-	KERNEL1x1_SUB
-	KERNEL1x1_SUB
-
-	KERNEL1x1_SUB
-	KERNEL1x1_SUB
-	KERNEL1x1_SUB
-	KERNEL1x1_SUB
-
-	je	.L1_46
-
-	KERNEL1x1_SUB
-	KERNEL1x1_SUB
-	KERNEL1x1_SUB
-	KERNEL1x1_SUB
-
-	KERNEL1x1_SUB
-	KERNEL1x1_SUB
-	KERNEL1x1_SUB
-	KERNEL1x1_SUB
-
-	je	.L1_46
-
-	jmp	.L1_42
-	ALIGN_4
-
-.L1_46:
-#ifndef TRMMKERNEL
-        movq    K, %rax
-#else
-        movq    KKK, %rax
-#endif
-
-	andq	$7, %rax		# if (k & 1)
-	je .L1_49
-
-	movq    %rax, BI                        //  Index for BO
-
-	leaq	(AO, %rax, SIZE), AO
-	leaq	(BO, BI, SIZE), BO
-	negq	BI
-	negq	%rax
-	ALIGN_4
-
-.L1_47:
-
-	KERNEL1x1_SUB
-
-	jl	.L1_47
-	ALIGN_4
-
-
-.L1_49:
-
-	SAVE1x1
-
-#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
-    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
-        movq    K, %rax 
-        subq    KKK, %rax
-        movq    %rax, BI                        //  Index for BO
-        leaq    (BO, BI, SIZE), BO         
-        leaq    (AO, %rax, SIZE), AO
-#endif  
-
-
-#if defined(TRMMKERNEL) && defined(LEFT)
-        addq    $1, KK
-#endif
-
-	addq	$1 * SIZE, CO1		# coffset += 1
-	ALIGN_4
-	
-
-.L999:
-	movq   		SP, %rsp
-	movq	   (%rsp), %rbx
-	movq	  8(%rsp), %rbp
-	movq	 16(%rsp), %r12
-	movq	 24(%rsp), %r13
-	movq	 32(%rsp), %r14
-	movq	 40(%rsp), %r15
-
-#ifdef WINDOWS_ABI
-	movq	 48(%rsp), %rdi
-	movq	 56(%rsp), %rsi
-	movups	 64(%rsp), %xmm6
-	movups	 80(%rsp), %xmm7
-	movups	 96(%rsp), %xmm8
-	movups	112(%rsp), %xmm9
-	movups	128(%rsp), %xmm10
-	movups	144(%rsp), %xmm11
-	movups	160(%rsp), %xmm12
-	movups	176(%rsp), %xmm13
-	movups	192(%rsp), %xmm14
-	movups	208(%rsp), %xmm15
-#endif
-
-	addq	$STACKSIZE, %rsp
-	ret
-
-	EPILOGUE
-
-
-
-
-
+/*********************************************************************************
+Copyright (c) 2013, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+**********************************************************************************/
+
+#define ASSEMBLER
+#include "common.h"
+ 
+#define OLD_M	%rdi
+#define OLD_N	%rsi
+#define M	%r13
+#define J	%r14
+#define OLD_K	%rdx
+
+#define A	%rcx
+#define B	%r8
+#define C	%r9
+#define LDC	%r10
+	
+#define I	%r11
+#define AO	%rdi
+#define BO	%rsi
+#define	CO1	%r15
+#define K	%r12
+#define BI	%rbp
+#define	SP	%rbx
+
+#define BO1	%rdi
+#define	CO2	%rdx
+
+#ifndef WINDOWS_ABI
+
+#define STACKSIZE 96
+
+#else
+
+#define STACKSIZE 256
+
+#define OLD_A		40 + STACKSIZE(%rsp)
+#define OLD_B		48 + STACKSIZE(%rsp)
+#define OLD_C		56 + STACKSIZE(%rsp)
+#define OLD_LDC		64 + STACKSIZE(%rsp)
+#define OLD_OFFSET	72 + STACKSIZE(%rsp)
+
+#endif
+
+#define L_BUFFER_SIZE 8192
+
+#define Ndiv6	 24(%rsp)
+#define Nmod6	 32(%rsp)
+#define N	 40(%rsp)
+#define ALPHA	 48(%rsp)
+#define OFFSET	 56(%rsp)
+#define KK	 64(%rsp)
+#define KKK	 72(%rsp)
+#define BUFFER1	           128(%rsp)
+
+#if defined(OS_WINDOWS)
+#if   L_BUFFER_SIZE > 16384
+#define STACK_TOUCH \
+        movl    $0,  4096 * 4(%rsp);\
+        movl    $0,  4096 * 3(%rsp);\
+        movl    $0,  4096 * 2(%rsp);\
+        movl    $0,  4096 * 1(%rsp);
+#elif L_BUFFER_SIZE > 12288
+#define STACK_TOUCH \
+        movl    $0,  4096 * 3(%rsp);\
+        movl    $0,  4096 * 2(%rsp);\
+        movl    $0,  4096 * 1(%rsp);
+#elif L_BUFFER_SIZE > 8192
+#define STACK_TOUCH \
+        movl    $0,  4096 * 2(%rsp);\
+        movl    $0,  4096 * 1(%rsp);
+#elif L_BUFFER_SIZE > 4096
+#define STACK_TOUCH \
+        movl    $0,  4096 * 1(%rsp);
+#else
+#define STACK_TOUCH
+#endif
+#else
+#define STACK_TOUCH
+#endif
+
+#define	A_PR1	512
+#define	B_PR1	512
+
+/*******************************************************************************************
+* 4 lines of N
+*******************************************************************************************/
+
+.macro KERNEL16x4_SUB
+	vmovups 	-16 * SIZE(AO, %rax, SIZE), %ymm0
+	vmovups 	 -8 * SIZE(AO, %rax, SIZE), %ymm1
+	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %ymm2
+	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %ymm3
+	vmulps		%ymm2 , %ymm0 , %ymm12
+	vmulps		%ymm2 , %ymm1 , %ymm13
+	vmulps		%ymm3 , %ymm0 , %ymm14
+	vmulps		%ymm3 , %ymm1 , %ymm15
+	vaddps 	        %ymm12, %ymm4 , %ymm4
+	vaddps 	        %ymm13, %ymm5 , %ymm5
+	vaddps 	        %ymm14, %ymm6 , %ymm6
+	vaddps 	        %ymm15, %ymm7 , %ymm7
+	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %ymm2
+	vbroadcastss	 -1 * SIZE(BO, BI, SIZE), %ymm3
+	vmulps		%ymm2 , %ymm0 , %ymm12
+	vmulps		%ymm2 , %ymm1 , %ymm13
+	vmulps		%ymm3 , %ymm0 , %ymm14
+	vmulps		%ymm3 , %ymm1 , %ymm15
+	vaddps 	        %ymm12, %ymm8 , %ymm8
+	vaddps 	        %ymm13, %ymm9 , %ymm9
+	vaddps 	        %ymm14, %ymm10, %ymm10
+	vaddps 	        %ymm15, %ymm11, %ymm11
+	addq	$ 4 , BI	
+	addq	$ 16, %rax 
+.endm
+
+.macro SAVE16x4
+
+	vbroadcastss	ALPHA, %ymm0
+
+	vmulps	%ymm0 , %ymm4 , %ymm4
+	vmulps	%ymm0 , %ymm5 , %ymm5
+	vmulps	%ymm0 , %ymm6 , %ymm6
+	vmulps	%ymm0 , %ymm7 , %ymm7
+	vmulps	%ymm0 , %ymm8 , %ymm8
+	vmulps	%ymm0 , %ymm9 , %ymm9
+	vmulps	%ymm0 , %ymm10, %ymm10
+	vmulps	%ymm0 , %ymm11, %ymm11
+
+
+#if !defined(TRMMKERNEL)
+
+	vaddps 	        (CO1), %ymm4,%ymm4
+	vaddps  8 * SIZE(CO1), %ymm5,%ymm5
+
+	vaddps 	        (CO1, LDC), %ymm6,%ymm6
+	vaddps  8 * SIZE(CO1, LDC), %ymm7,%ymm7
+
+	vaddps 	        (CO2), %ymm8,%ymm8
+	vaddps  8 * SIZE(CO2), %ymm9,%ymm9
+
+	vaddps 	        (CO2, LDC), %ymm10,%ymm10
+	vaddps  8 * SIZE(CO2, LDC), %ymm11,%ymm11
+
+#endif
+
+	vmovups	%ymm4 ,  	(CO1)
+	vmovups	%ymm5 , 8 * SIZE(CO1)
+
+	vmovups	%ymm6 ,  	(CO1, LDC)
+	vmovups	%ymm7 , 8 * SIZE(CO1, LDC)
+
+	vmovups	%ymm8 ,  	(CO2)
+	vmovups	%ymm9 , 8 * SIZE(CO2)
+
+	vmovups	%ymm10,  	(CO2, LDC)
+	vmovups	%ymm11, 8 * SIZE(CO2, LDC)
+
+	prefetcht0	64(CO1)
+	prefetcht0	64(CO1, LDC)
+	prefetcht0	64(CO2)
+	prefetcht0	64(CO2, LDC)
+
+.endm
+
+
+
+/*******************************************************************************************/
+
+.macro KERNEL8x4_SUB
+	vmovups 	-16 * SIZE(AO, %rax, SIZE), %ymm0
+	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %ymm2
+	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %ymm3
+	vmulps		%ymm2 , %ymm0 , %ymm12
+	vmulps		%ymm3 , %ymm0 , %ymm14
+	vaddps 	        %ymm12, %ymm4 , %ymm4
+	vaddps 	        %ymm14, %ymm6 , %ymm6
+	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %ymm2
+	vbroadcastss	 -1 * SIZE(BO, BI, SIZE), %ymm3
+	vmulps		%ymm2 , %ymm0 , %ymm12
+	vmulps		%ymm3 , %ymm0 , %ymm14
+	vaddps 	        %ymm12, %ymm8 , %ymm8
+	vaddps 	        %ymm14, %ymm10, %ymm10
+	addq	$ 4 , BI	
+	addq	$ 8 , %rax 
+.endm
+
+.macro SAVE8x4
+
+	vbroadcastss	ALPHA, %ymm0
+
+	vmulps	%ymm0 , %ymm4 , %ymm4
+	vmulps	%ymm0 , %ymm6 , %ymm6
+	vmulps	%ymm0 , %ymm8 , %ymm8
+	vmulps	%ymm0 , %ymm10, %ymm10
+
+
+#if !defined(TRMMKERNEL)
+
+	vaddps 	        (CO1), %ymm4,%ymm4
+	vaddps 	        (CO1, LDC), %ymm6,%ymm6
+	vaddps 	        (CO2), %ymm8,%ymm8
+	vaddps 	        (CO2, LDC), %ymm10,%ymm10
+
+#endif
+
+	vmovups	%ymm4 ,  	(CO1)
+	vmovups	%ymm6 ,  	(CO1, LDC)
+	vmovups	%ymm8 ,  	(CO2)
+	vmovups	%ymm10,  	(CO2, LDC)
+
+.endm
+
+
+
+/*******************************************************************************************/
+
+.macro KERNEL4x4_SUB
+	vmovups 	-16 * SIZE(AO, %rax, SIZE), %xmm0
+	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm2
+	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %xmm3
+	vmulps		%xmm2 , %xmm0 , %xmm12
+	vmulps		%xmm3 , %xmm0 , %xmm14
+	vaddps 	        %xmm12, %xmm4 , %xmm4
+	vaddps 	        %xmm14, %xmm6 , %xmm6
+	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %xmm2
+	vbroadcastss	 -1 * SIZE(BO, BI, SIZE), %xmm3
+	vmulps		%xmm2 , %xmm0 , %xmm12
+	vmulps		%xmm3 , %xmm0 , %xmm14
+	vaddps 	        %xmm12, %xmm8 , %xmm8
+	vaddps 	        %xmm14, %xmm10, %xmm10
+	addq	$ 4 , BI	
+	addq	$ 4 , %rax 
+.endm
+
+.macro SAVE4x4
+
+	vbroadcastss	ALPHA, %xmm0
+
+	vmulps	%xmm0 , %xmm4 , %xmm4
+	vmulps	%xmm0 , %xmm6 , %xmm6
+	vmulps	%xmm0 , %xmm8 , %xmm8
+	vmulps	%xmm0 , %xmm10, %xmm10
+
+
+#if !defined(TRMMKERNEL)
+
+	vaddps 	        (CO1), %xmm4,%xmm4
+	vaddps 	        (CO1, LDC), %xmm6,%xmm6
+	vaddps 	        (CO2), %xmm8,%xmm8
+	vaddps 	        (CO2, LDC), %xmm10,%xmm10
+
+#endif
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm6 ,  	(CO1, LDC)
+	vmovups	%xmm8 ,  	(CO2)
+	vmovups	%xmm10,  	(CO2, LDC)
+
+.endm
+
+
+/*******************************************************************************************/
+
+.macro KERNEL2x4_SUB
+	vmovss 	-16 * SIZE(AO, %rax, SIZE), %xmm0
+	vmovss 	-15 * SIZE(AO, %rax, SIZE), %xmm1
+	vmovss	 -4 * SIZE(BO, BI, SIZE), %xmm2
+	vmovss	 -3 * SIZE(BO, BI, SIZE), %xmm3
+	vmulss		%xmm2 , %xmm0 , %xmm12
+	vmulss		%xmm2 , %xmm1 , %xmm13
+	vmulss		%xmm3 , %xmm0 , %xmm14
+	vmulss		%xmm3 , %xmm1 , %xmm15
+	vaddss 	        %xmm12, %xmm4 , %xmm4
+	vaddss 	        %xmm13, %xmm5 , %xmm5
+	vaddss 	        %xmm14, %xmm6 , %xmm6
+	vaddss 	        %xmm15, %xmm7 , %xmm7
+	vmovss	 -2 * SIZE(BO, BI, SIZE), %xmm2
+	vmovss	 -1 * SIZE(BO, BI, SIZE), %xmm3
+	vmulss		%xmm2 , %xmm0 , %xmm12
+	vmulss		%xmm2 , %xmm1 , %xmm13
+	vmulss		%xmm3 , %xmm0 , %xmm14
+	vmulss		%xmm3 , %xmm1 , %xmm15
+	vaddss 	        %xmm12, %xmm8 , %xmm8
+	vaddss 	        %xmm13, %xmm9 , %xmm9
+	vaddss 	        %xmm14, %xmm10, %xmm10
+	vaddss 	        %xmm15, %xmm11, %xmm11
+	addq	$ 4 , BI	
+	addq	$ 2, %rax 
+.endm
+
+.macro SAVE2x4
+
+	vmovss	ALPHA, %xmm0
+
+	vmulss	%xmm0 , %xmm4 , %xmm4
+	vmulss	%xmm0 , %xmm5 , %xmm5
+	vmulss	%xmm0 , %xmm6 , %xmm6
+	vmulss	%xmm0 , %xmm7 , %xmm7
+	vmulss	%xmm0 , %xmm8 , %xmm8
+	vmulss	%xmm0 , %xmm9 , %xmm9
+	vmulss	%xmm0 , %xmm10, %xmm10
+	vmulss	%xmm0 , %xmm11, %xmm11
+
+
+#if !defined(TRMMKERNEL)
+
+	vaddss 	        (CO1), %xmm4,%xmm4
+	vaddss  1 * SIZE(CO1), %xmm5,%xmm5
+
+	vaddss 	        (CO1, LDC), %xmm6,%xmm6
+	vaddss  1 * SIZE(CO1, LDC), %xmm7,%xmm7
+
+	vaddss 	        (CO2), %xmm8,%xmm8
+	vaddss  1 * SIZE(CO2), %xmm9,%xmm9
+
+	vaddss 	        (CO2, LDC), %xmm10,%xmm10
+	vaddss  1 * SIZE(CO2, LDC), %xmm11,%xmm11
+
+#endif
+
+	vmovss	%xmm4 ,  	(CO1)
+	vmovss	%xmm5 , 1 * SIZE(CO1)
+
+	vmovss	%xmm6 ,  	(CO1, LDC)
+	vmovss	%xmm7 , 1 * SIZE(CO1, LDC)
+
+	vmovss	%xmm8 ,  	(CO2)
+	vmovss	%xmm9 , 1 * SIZE(CO2)
+
+	vmovss	%xmm10,  	(CO2, LDC)
+	vmovss	%xmm11, 1 * SIZE(CO2, LDC)
+
+.endm
+
+
+/*******************************************************************************************/
+
+.macro KERNEL1x4_SUB
+	vmovss 	-16 * SIZE(AO, %rax, SIZE), %xmm0
+	vmovss	 -4 * SIZE(BO, BI, SIZE), %xmm2
+	vmovss	 -3 * SIZE(BO, BI, SIZE), %xmm3
+	vmulss		%xmm2 , %xmm0 , %xmm12
+	vmulss		%xmm3 , %xmm0 , %xmm14
+	vaddss 	        %xmm12, %xmm4 , %xmm4
+	vaddss 	        %xmm14, %xmm6 , %xmm6
+	vmovss	 -2 * SIZE(BO, BI, SIZE), %xmm2
+	vmovss	 -1 * SIZE(BO, BI, SIZE), %xmm3
+	vmulss		%xmm2 , %xmm0 , %xmm12
+	vmulss		%xmm3 , %xmm0 , %xmm14
+	vaddss 	        %xmm12, %xmm8 , %xmm8
+	vaddss 	        %xmm14, %xmm10, %xmm10
+	addq	$ 4 , BI	
+	addq	$ 1, %rax 
+.endm
+
+.macro SAVE1x4
+
+	vmovss	ALPHA, %xmm0
+
+	vmulss	%xmm0 , %xmm4 , %xmm4
+	vmulss	%xmm0 , %xmm6 , %xmm6
+	vmulss	%xmm0 , %xmm8 , %xmm8
+	vmulss	%xmm0 , %xmm10, %xmm10
+
+
+#if !defined(TRMMKERNEL)
+
+	vaddss 	        (CO1), %xmm4,%xmm4
+	vaddss 	        (CO1, LDC), %xmm6,%xmm6
+	vaddss 	        (CO2), %xmm8,%xmm8
+	vaddss 	        (CO2, LDC), %xmm10,%xmm10
+
+#endif
+
+	vmovss	%xmm4 ,  	(CO1)
+	vmovss	%xmm6 ,  	(CO1, LDC)
+	vmovss	%xmm8 ,  	(CO2)
+	vmovss	%xmm10,  	(CO2, LDC)
+
+.endm
+
+
+/*******************************************************************************************/
+
+/*******************************************************************************************
+* 2 lines of N
+*******************************************************************************************/
+
+.macro KERNEL16x2_SUB
+	vmovups 	-16 * SIZE(AO, %rax, SIZE), %ymm0
+	vmovups 	 -8 * SIZE(AO, %rax, SIZE), %ymm1
+	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %ymm2
+	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %ymm3
+	vmulps		%ymm2 , %ymm0 , %ymm12
+	vmulps		%ymm2 , %ymm1 , %ymm13
+	vmulps		%ymm3 , %ymm0 , %ymm14
+	vmulps		%ymm3 , %ymm1 , %ymm15
+	vaddps 	        %ymm12, %ymm4 , %ymm4
+	vaddps 	        %ymm13, %ymm5 , %ymm5
+	vaddps 	        %ymm14, %ymm6 , %ymm6
+	vaddps 	        %ymm15, %ymm7 , %ymm7
+	addq	$ 2 , BI	
+	addq	$ 16, %rax 
+.endm
+
+.macro SAVE16x2
+
+	vbroadcastss	ALPHA, %ymm0
+
+	vmulps	%ymm0 , %ymm4 , %ymm4
+	vmulps	%ymm0 , %ymm5 , %ymm5
+	vmulps	%ymm0 , %ymm6 , %ymm6
+	vmulps	%ymm0 , %ymm7 , %ymm7
+
+
+#if !defined(TRMMKERNEL)
+
+	vaddps 	        (CO1), %ymm4,%ymm4
+	vaddps  8 * SIZE(CO1), %ymm5,%ymm5
+
+	vaddps 	        (CO1, LDC), %ymm6,%ymm6
+	vaddps  8 * SIZE(CO1, LDC), %ymm7,%ymm7
+
+#endif
+
+	vmovups	%ymm4 ,  	(CO1)
+	vmovups	%ymm5 , 8 * SIZE(CO1)
+
+	vmovups	%ymm6 ,  	(CO1, LDC)
+	vmovups	%ymm7 , 8 * SIZE(CO1, LDC)
+
+.endm
+
+
+
+/*******************************************************************************************/
+
+.macro KERNEL8x2_SUB
+	vmovups 	-16 * SIZE(AO, %rax, SIZE), %ymm0
+	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %ymm2
+	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %ymm3
+	vmulps		%ymm2 , %ymm0 , %ymm12
+	vmulps		%ymm3 , %ymm0 , %ymm14
+	vaddps 	        %ymm12, %ymm4 , %ymm4
+	vaddps 	        %ymm14, %ymm6 , %ymm6
+	addq	$ 2 , BI	
+	addq	$ 8 , %rax 
+.endm
+
+.macro SAVE8x2
+
+	vbroadcastss	ALPHA, %ymm0
+
+	vmulps	%ymm0 , %ymm4 , %ymm4
+	vmulps	%ymm0 , %ymm6 , %ymm6
+
+#if !defined(TRMMKERNEL)
+
+	vaddps 	        (CO1), %ymm4,%ymm4
+	vaddps 	        (CO1, LDC), %ymm6,%ymm6
+
+#endif
+
+	vmovups	%ymm4 ,  	(CO1)
+	vmovups	%ymm6 ,  	(CO1, LDC)
+
+.endm
+
+
+
+/*******************************************************************************************/
+
+.macro KERNEL4x2_SUB
+	vmovups 	-16 * SIZE(AO, %rax, SIZE), %xmm0
+	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm2
+	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %xmm3
+	vmulps		%xmm2 , %xmm0 , %xmm12
+	vmulps		%xmm3 , %xmm0 , %xmm14
+	vaddps 	        %xmm12, %xmm4 , %xmm4
+	vaddps 	        %xmm14, %xmm6 , %xmm6
+	addq	$ 2 , BI	
+	addq	$ 4 , %rax 
+.endm
+
+.macro SAVE4x2
+
+	vbroadcastss	ALPHA, %xmm0
+
+	vmulps	%xmm0 , %xmm4 , %xmm4
+	vmulps	%xmm0 , %xmm6 , %xmm6
+
+#if !defined(TRMMKERNEL)
+
+	vaddps 	        (CO1), %xmm4,%xmm4
+	vaddps 	        (CO1, LDC), %xmm6,%xmm6
+
+#endif
+
+	vmovups	%xmm4 ,  	(CO1)
+	vmovups	%xmm6 ,  	(CO1, LDC)
+
+.endm
+
+
+/*******************************************************************************************/
+
+.macro KERNEL2x2_SUB
+	vmovss 	-16 * SIZE(AO, %rax, SIZE), %xmm0
+	vmovss 	-15 * SIZE(AO, %rax, SIZE), %xmm1
+	vmovss	 -4 * SIZE(BO, BI, SIZE), %xmm2
+	vmovss	 -3 * SIZE(BO, BI, SIZE), %xmm3
+	vmulss		%xmm2 , %xmm0 , %xmm12
+	vmulss		%xmm2 , %xmm1 , %xmm13
+	vmulss		%xmm3 , %xmm0 , %xmm14
+	vmulss		%xmm3 , %xmm1 , %xmm15
+	vaddss 	        %xmm12, %xmm4 , %xmm4
+	vaddss 	        %xmm13, %xmm5 , %xmm5
+	vaddss 	        %xmm14, %xmm6 , %xmm6
+	vaddss 	        %xmm15, %xmm7 , %xmm7
+	addq	$ 2 , BI	
+	addq	$ 2, %rax 
+.endm
+
+.macro SAVE2x2
+
+	vmovss	ALPHA, %xmm0
+
+	vmulss	%xmm0 , %xmm4 , %xmm4
+	vmulss	%xmm0 , %xmm5 , %xmm5
+	vmulss	%xmm0 , %xmm6 , %xmm6
+	vmulss	%xmm0 , %xmm7 , %xmm7
+
+
+#if !defined(TRMMKERNEL)
+
+	vaddss 	        (CO1), %xmm4,%xmm4
+	vaddss  1 * SIZE(CO1), %xmm5,%xmm5
+
+	vaddss 	        (CO1, LDC), %xmm6,%xmm6
+	vaddss  1 * SIZE(CO1, LDC), %xmm7,%xmm7
+
+#endif
+
+	vmovss	%xmm4 ,  	(CO1)
+	vmovss	%xmm5 , 1 * SIZE(CO1)
+
+	vmovss	%xmm6 ,  	(CO1, LDC)
+	vmovss	%xmm7 , 1 * SIZE(CO1, LDC)
+
+.endm
+
+
+/*******************************************************************************************/
+
+.macro KERNEL1x2_SUB
+	vmovss 	-16 * SIZE(AO, %rax, SIZE), %xmm0
+	vmovss	 -4 * SIZE(BO, BI, SIZE), %xmm2
+	vmovss	 -3 * SIZE(BO, BI, SIZE), %xmm3
+	vmulss		%xmm2 , %xmm0 , %xmm12
+	vmulss		%xmm3 , %xmm0 , %xmm14
+	vaddss 	        %xmm12, %xmm4 , %xmm4
+	vaddss 	        %xmm14, %xmm6 , %xmm6
+	addq	$ 2 , BI	
+	addq	$ 1, %rax 
+.endm
+
+.macro SAVE1x2
+
+	vmovss	ALPHA, %xmm0
+
+	vmulss	%xmm0 , %xmm4 , %xmm4
+	vmulss	%xmm0 , %xmm6 , %xmm6
+
+#if !defined(TRMMKERNEL)
+
+	vaddss 	        (CO1), %xmm4,%xmm4
+	vaddss 	        (CO1, LDC), %xmm6,%xmm6
+
+#endif
+
+	vmovss	%xmm4 ,  	(CO1)
+	vmovss	%xmm6 ,  	(CO1, LDC)
+
+.endm
+
+
+/*******************************************************************************************/
+
+/*******************************************************************************************
+* 1 line of N
+*******************************************************************************************/
+
+.macro KERNEL16x1_SUB
+	vmovups 	-16 * SIZE(AO, %rax, SIZE), %ymm0
+	vmovups 	 -8 * SIZE(AO, %rax, SIZE), %ymm1
+	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %ymm2
+	vmulps		%ymm2 , %ymm0 , %ymm12
+	vmulps		%ymm2 , %ymm1 , %ymm13
+	vaddps 	        %ymm12, %ymm4 , %ymm4
+	vaddps 	        %ymm13, %ymm5 , %ymm5
+	addq	$ 1 , BI	
+	addq	$ 16, %rax 
+.endm
+
+.macro SAVE16x1
+
+	vbroadcastss	ALPHA, %ymm0
+
+	vmulps	%ymm0 , %ymm4 , %ymm4
+	vmulps	%ymm0 , %ymm5 , %ymm5
+
+#if !defined(TRMMKERNEL)
+
+	vaddps 	        (CO1), %ymm4,%ymm4
+	vaddps  8 * SIZE(CO1), %ymm5,%ymm5
+
+#endif
+
+	vmovups	%ymm4 ,  	(CO1)
+	vmovups	%ymm5 , 8 * SIZE(CO1)
+
+.endm
+
+
+/*******************************************************************************************/
+
+.macro KERNEL8x1_SUB
+	vmovups 	-16 * SIZE(AO, %rax, SIZE), %ymm0
+	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %ymm2
+	vmulps		%ymm2 , %ymm0 , %ymm12
+	vaddps 	        %ymm12, %ymm4 , %ymm4
+	addq	$ 1 , BI	
+	addq	$ 8 , %rax 
+.endm
+
+.macro SAVE8x1
+
+	vbroadcastss	ALPHA, %ymm0
+
+	vmulps	%ymm0 , %ymm4 , %ymm4
+
+#if !defined(TRMMKERNEL)
+
+	vaddps 	        (CO1), %ymm4,%ymm4
+
+#endif
+
+	vmovups	%ymm4 ,  	(CO1)
+
+.endm
+
+
+
+/*******************************************************************************************/
+
+.macro KERNEL4x1_SUB
+	vmovups 	-16 * SIZE(AO, %rax, SIZE), %xmm0
+	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm2
+	vmulps		%xmm2 , %xmm0 , %xmm12
+	vaddps 	        %xmm12, %xmm4 , %xmm4
+	addq	$ 1 , BI	
+	addq	$ 4 , %rax 
+.endm
+
+.macro SAVE4x1
+
+	vbroadcastss	ALPHA, %xmm0
+
+	vmulps	%xmm0 , %xmm4 , %xmm4
+
+#if !defined(TRMMKERNEL)
+
+	vaddps 	        (CO1), %xmm4,%xmm4
+
+#endif
+
+	vmovups	%xmm4 ,  	(CO1)
+
+.endm
+
+
+/*******************************************************************************************/
+
+.macro KERNEL2x1_SUB
+	vmovss 	-16 * SIZE(AO, %rax, SIZE), %xmm0
+	vmovss 	-15 * SIZE(AO, %rax, SIZE), %xmm1
+	vmovss	 -4 * SIZE(BO, BI, SIZE), %xmm2
+	vmulss		%xmm2 , %xmm0 , %xmm12
+	vmulss		%xmm2 , %xmm1 , %xmm13
+	vaddss 	        %xmm12, %xmm4 , %xmm4
+	vaddss 	        %xmm13, %xmm5 , %xmm5
+	addq	$ 1 , BI	
+	addq	$ 2 , %rax 
+.endm
+
+.macro SAVE2x1
+
+	vmovss	ALPHA, %xmm0
+
+	vmulss	%xmm0 , %xmm4 , %xmm4
+	vmulss	%xmm0 , %xmm5 , %xmm5
+
+#if !defined(TRMMKERNEL)
+
+	vaddss 	        (CO1), %xmm4,%xmm4
+	vaddss  1 * SIZE(CO1), %xmm5,%xmm5
+
+#endif
+
+	vmovss	%xmm4 ,  	(CO1)
+	vmovss	%xmm5 , 1 * SIZE(CO1)
+
+.endm
+
+
+/*******************************************************************************************/
+
+.macro KERNEL1x1_SUB
+	vmovss 	-16 * SIZE(AO, %rax, SIZE), %xmm0
+	vmovss	 -4 * SIZE(BO, BI, SIZE), %xmm2
+	vmulss		%xmm2 , %xmm0 , %xmm12
+	vaddss 	        %xmm12, %xmm4 , %xmm4
+	addq	$ 1 , BI	
+	addq	$ 1 , %rax 
+.endm
+
+.macro SAVE1x1
+
+	vmovss	ALPHA, %xmm0
+
+	vmulss	%xmm0 , %xmm4 , %xmm4
+
+#if !defined(TRMMKERNEL)
+
+	vaddss 	        (CO1), %xmm4,%xmm4
+
+#endif
+
+	vmovss	%xmm4 ,  	(CO1)
+
+.endm
+
+
+/*******************************************************************************************/
+
+/*************************************************************************************
+* TRMM Kernel
+*************************************************************************************/
+
+
+	PROLOGUE
+	PROFCODE
+	
+	subq	$STACKSIZE, %rsp
+	movq	%rbx,   (%rsp)
+	movq	%rbp,  8(%rsp)
+	movq	%r12, 16(%rsp)
+	movq	%r13, 24(%rsp)
+	movq	%r14, 32(%rsp)
+	movq	%r15, 40(%rsp)
+
+	vzeroupper
+
+#ifdef WINDOWS_ABI
+	movq	%rdi,    48(%rsp)
+	movq	%rsi,    56(%rsp)
+	movups	%xmm6,   64(%rsp)
+	movups	%xmm7,   80(%rsp)
+	movups	%xmm8,   96(%rsp)
+	movups	%xmm9,  112(%rsp)
+	movups	%xmm10, 128(%rsp)
+	movups	%xmm11, 144(%rsp)
+	movups	%xmm12, 160(%rsp)
+	movups	%xmm13, 176(%rsp)
+	movups	%xmm14, 192(%rsp)
+	movups	%xmm15, 208(%rsp)
+
+	movq	ARG1,      OLD_M
+	movq	ARG2,      OLD_N
+	movq	ARG3,      OLD_K
+	movq	OLD_A,     A
+	movq	OLD_B,     B
+	movq	OLD_C,     C
+	movq	OLD_LDC,   LDC
+#ifdef TRMMKERNEL
+	vmovsd	OLD_OFFSET, %xmm12
+#endif
+	vmovaps	%xmm3, %xmm0
+
+#else
+	movq	STACKSIZE +  8(%rsp), LDC
+#ifdef TRMMKERNEL
+	movsd	STACKSIZE + 16(%rsp), %xmm12
+#endif
+
+#endif
+
+	movq    %rsp, SP      # save old stack
+        subq    $128 + L_BUFFER_SIZE, %rsp
+        andq    $-4096, %rsp    # align stack
+
+        STACK_TOUCH
+
+	cmpq	$0, OLD_M
+	je	.L999
+
+	cmpq	$0, OLD_N
+	je	.L999
+
+	cmpq	$0, OLD_K
+	je	.L999
+
+	movq	OLD_M, M
+	movq	OLD_N, N
+	movq	OLD_K, K
+
+	vmovss	 %xmm0, ALPHA
+
+	salq	$BASE_SHIFT, LDC
+
+	movq    N, %rax
+        xorq    %rdx, %rdx
+        movq    $4,  %rdi
+        divq    %rdi                    //    N / 4
+        movq    %rax, Ndiv6             //    N / 4
+        movq    %rdx, Nmod6             //    N % 4
+
+	
+
+#ifdef TRMMKERNEL
+	vmovsd	%xmm12, OFFSET
+	vmovsd	%xmm12, KK
+#ifndef LEFT
+	negq	KK
+#endif	
+#endif
+
+	movq	Ndiv6,  J
+	cmpq	$0, J
+	je	.L2_0
+	ALIGN_4
+
+/*******************************************************************************************/
+
+.L4_01:
+	// copy to sub buffer
+	movq	B, BO1
+	leaq    BUFFER1, BO		// first buffer to BO
+	movq	K, %rax
+	sarq	$2, %rax		// K / 4
+	jz	.L4_01b
+	ALIGN_4
+
+
+.L4_01a:
+        prefetcht0 512(BO1)
+        prefetchw  512(BO)
+
+	vmovups	       (BO1), %xmm0
+	vmovups	 4*SIZE(BO1), %xmm1
+	vmovups	 8*SIZE(BO1), %xmm2
+	vmovups	12*SIZE(BO1), %xmm3
+
+	vmovups	%xmm0,       (BO)
+	vmovups	%xmm1, 4*SIZE(BO)
+	vmovups	%xmm2, 8*SIZE(BO)
+	vmovups	%xmm3,12*SIZE(BO)
+
+	addq	$ 16*SIZE,BO1
+	addq	$ 16*SIZE,BO
+	decq	%rax
+	jnz	.L4_01a
+
+
+.L4_01b:
+
+        movq    K, %rax
+        andq    $3, %rax                // K % 4
+        jz      .L4_02d
+        ALIGN_4
+
+.L4_02c:
+
+	vmovups	(BO1), %xmm0
+	vmovups	%xmm0, (BO)
+	addq	$ 4*SIZE,BO1
+	addq	$ 4*SIZE,BO
+	decq	%rax
+	jnz	.L4_02c
+
+.L4_02d:
+
+	movq	BO1, B			// next offset of B
+
+.L4_10:
+	movq	 C, CO1
+	leaq	(C, LDC, 2), CO2	
+	leaq	(C, LDC, 4), C		// c += 4 * ldc
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        movq    OFFSET, %rax
+        movq    %rax, KK
+#endif
+	
+	movq	A, AO		 	// aoffset = a
+	addq	$ 16 * SIZE, AO
+
+	movq	M,  I
+	sarq	$4, I			// i = (m >> 4)
+	je	.L4_20
+
+	ALIGN_4
+
+.L4_11:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             	// first buffer to BO
+        addq    $4 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+	leaq	(,BI, 4), BI			// BI = BI * 4 ; number of values
+        leaq    (BO, BI, SIZE), BO
+        salq    $4, %rax                        // rax = rax * 16 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $16, %rax	// number of values in AO
+#else
+        addq    $4, %rax	// number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+	andq	$-8, %rax			//  K = K - ( K % 8 )
+	je	.L4_16
+	movq    %rax, BI                        //  Index for BO
+        leaq    (,BI,4) , BI                   	//  BI = BI * 4 ; number of values
+
+	salq	$4, %rax			// rax = rax * 16 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L4_12:
+
+	prefetcht0	A_PR1(AO, %rax, SIZE)
+	prefetcht0	B_PR1(BO, BI  , SIZE)
+	KERNEL16x4_SUB
+	prefetcht0	A_PR1(AO, %rax, SIZE)
+	KERNEL16x4_SUB
+	prefetcht0	A_PR1(AO, %rax, SIZE)
+	KERNEL16x4_SUB
+	prefetcht0	A_PR1(AO, %rax, SIZE)
+	KERNEL16x4_SUB
+
+	prefetcht0	A_PR1(AO, %rax, SIZE)
+	prefetcht0	B_PR1(BO, BI  , SIZE)
+	KERNEL16x4_SUB
+	prefetcht0	A_PR1(AO, %rax, SIZE)
+	KERNEL16x4_SUB
+	prefetcht0	A_PR1(AO, %rax, SIZE)
+	KERNEL16x4_SUB
+	prefetcht0	A_PR1(AO, %rax, SIZE)
+	KERNEL16x4_SUB
+
+	je	.L4_16
+
+	prefetcht0	A_PR1(AO, %rax, SIZE)
+	prefetcht0	B_PR1(BO, BI  , SIZE)
+	KERNEL16x4_SUB
+	prefetcht0	A_PR1(AO, %rax, SIZE)
+	KERNEL16x4_SUB
+	prefetcht0	A_PR1(AO, %rax, SIZE)
+	KERNEL16x4_SUB
+	prefetcht0	A_PR1(AO, %rax, SIZE)
+	KERNEL16x4_SUB
+
+	prefetcht0	A_PR1(AO, %rax, SIZE)
+	prefetcht0	B_PR1(BO, BI  , SIZE)
+	KERNEL16x4_SUB
+	prefetcht0	A_PR1(AO, %rax, SIZE)
+	KERNEL16x4_SUB
+	prefetcht0	A_PR1(AO, %rax, SIZE)
+	KERNEL16x4_SUB
+	prefetcht0	A_PR1(AO, %rax, SIZE)
+	KERNEL16x4_SUB
+
+	je	.L4_16
+
+	jmp	.L4_12
+	ALIGN_4
+
+.L4_16:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L4_19
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (,BI,4), BI                   	//  BI = BI * 4 ; number of values
+
+	salq	$4, %rax			// rax = rax * 16 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L4_17:
+
+	KERNEL16x4_SUB
+
+	jl	.L4_17
+	ALIGN_4
+
+
+.L4_19:
+
+	SAVE16x4
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+	leaq	(,BI, 4), BI			// BI = BI * 4 ; number of values
+        leaq    (BO, BI, SIZE), BO
+        salq    $4, %rax                        // rax = rax * 16 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $16, KK				
+#endif
+
+	addq	$16 * SIZE, CO1		# coffset += 16
+	addq	$16 * SIZE, CO2		# coffset += 16
+	decq	I			# i --
+	jg	.L4_11
+	ALIGN_4	
+
+/**************************************************************************
+* Rest of M 
+***************************************************************************/
+.L4_20:
+	// Test rest of M
+
+	testq	$15, M
+	jz	.L4_60		// to next 3 lines of N
+
+	testq	$8, M		
+	jz	.L4_21pre
+	ALIGN_4
+
+/**************************************************************************/
+
+.L4_20_1:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+	leaq	(,BI, 4), BI			// BI = BI * 4 ; number of values
+        leaq    (BO, BI, SIZE), BO
+        salq    $3, %rax                        // rax = rax * 8 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $8, %rax        // number of values in A
+#else
+        addq    $4, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$-8, %rax
+	je	.L4_20_6
+	movq    %rax, BI                        //  Index for BO
+        leaq    (,BI,4), BI                   	//  BI = BI * 4 ; number of values
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L4_20_2:
+
+	KERNEL8x4_SUB
+	KERNEL8x4_SUB
+	KERNEL8x4_SUB
+	KERNEL8x4_SUB
+
+	KERNEL8x4_SUB
+	KERNEL8x4_SUB
+	KERNEL8x4_SUB
+	KERNEL8x4_SUB
+
+	je	.L4_20_6
+
+	KERNEL8x4_SUB
+	KERNEL8x4_SUB
+	KERNEL8x4_SUB
+	KERNEL8x4_SUB
+
+	KERNEL8x4_SUB
+	KERNEL8x4_SUB
+	KERNEL8x4_SUB
+	KERNEL8x4_SUB
+
+	je	.L4_20_6
+
+	jmp	.L4_20_2
+	ALIGN_4
+
+.L4_20_6:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L4_20_9
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (,BI,4), BI                   	//  BI = BI * 4 ; number of values
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L4_20_7:
+
+	KERNEL8x4_SUB
+
+	jl	.L4_20_7
+	ALIGN_4
+
+
+.L4_20_9:
+
+	SAVE8x4
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax 
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+	leaq	(,BI, 4), BI			// BI = BI * 4 ; number of values
+        leaq    (BO, BI, SIZE), BO         
+        salq    $3, %rax                        // rax = rax * 8 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif  
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $8, KK
+#endif
+
+	addq	$8 * SIZE, CO1		# coffset += 8
+	addq	$8 * SIZE, CO2		# coffset += 8
+	ALIGN_4
+	
+
+
+/**************************************************************************/
+
+.L4_21pre:
+
+	testq	$4, M		
+	jz	.L4_30
+	ALIGN_4
+
+.L4_21:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+	leaq	(,BI, 4), BI			// BI = BI * 4 ; number of values
+        leaq    (BO, BI, SIZE), BO
+        salq    $2, %rax                        // rax = rax * 4 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $4, %rax        // number of values in A
+#else
+        addq    $4, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$-8, %rax
+	je	.L4_26
+	movq    %rax, BI                        //  Index for BO
+        leaq    (,BI,4), BI                   	//  BI = BI * 4 ; number of values
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L4_22:
+
+	KERNEL4x4_SUB
+	KERNEL4x4_SUB
+	KERNEL4x4_SUB
+	KERNEL4x4_SUB
+
+	KERNEL4x4_SUB
+	KERNEL4x4_SUB
+	KERNEL4x4_SUB
+	KERNEL4x4_SUB
+
+	je	.L4_26
+
+	KERNEL4x4_SUB
+	KERNEL4x4_SUB
+	KERNEL4x4_SUB
+	KERNEL4x4_SUB
+
+	KERNEL4x4_SUB
+	KERNEL4x4_SUB
+	KERNEL4x4_SUB
+	KERNEL4x4_SUB
+
+	je	.L4_26
+
+	jmp	.L4_22
+	ALIGN_4
+
+.L4_26:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L4_29
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (,BI,4), BI                   	//  BI = BI * 4 ; number of values
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L4_27:
+
+	KERNEL4x4_SUB
+
+	jl	.L4_27
+	ALIGN_4
+
+
+.L4_29:
+
+	SAVE4x4
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax 
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+	leaq	(,BI, 4), BI			// BI = BI * 4 ; number of values
+        leaq    (BO, BI, SIZE), BO         
+        salq    $2, %rax                        // rax = rax * 4 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif  
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $4, KK
+#endif
+
+	addq	$4 * SIZE, CO1		# coffset += 4
+	addq	$4 * SIZE, CO2		# coffset += 4
+	ALIGN_4
+	
+
+.L4_30:
+	testq	$2, M		
+	jz	.L4_40
+
+	ALIGN_4
+
+.L4_31:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+	leaq	(,BI, 4), BI			// BI = BI * 4 ; number of values
+        leaq    (BO, BI, SIZE), BO
+        salq    $1, %rax                        // rax = rax * 2 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $2, %rax        // number of values in AO
+#else
+        addq    $4, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$-8, %rax
+	je	.L4_36
+	movq    %rax, BI                        //  Index for BO
+        leaq    (,BI,4), BI                   	//  BI = BI * 4 ; number of values
+
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L4_32:
+
+	KERNEL2x4_SUB
+	KERNEL2x4_SUB
+	KERNEL2x4_SUB
+	KERNEL2x4_SUB
+
+	KERNEL2x4_SUB
+	KERNEL2x4_SUB
+	KERNEL2x4_SUB
+	KERNEL2x4_SUB
+
+	je	.L4_36
+
+	KERNEL2x4_SUB
+	KERNEL2x4_SUB
+	KERNEL2x4_SUB
+	KERNEL2x4_SUB
+
+	KERNEL2x4_SUB
+	KERNEL2x4_SUB
+	KERNEL2x4_SUB
+	KERNEL2x4_SUB
+
+	je	.L4_36
+
+	jmp	.L4_32
+	ALIGN_4
+
+.L4_36:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L4_39
+
+	movq    %rax, BI                        //  Index for BO
+	leaq	(,BI, 4), BI			// BI = BI * 4 ; number of values
+	
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L4_37:
+
+	KERNEL2x4_SUB
+
+	jl	.L4_37
+	ALIGN_4
+
+
+.L4_39:
+
+	SAVE2x4
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax 
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+	leaq	(,BI, 4), BI			// BI = BI * 4 ; number of values
+        leaq    (BO, BI, SIZE), BO         
+        salq    $1, %rax                        // rax = rax * 2 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif  
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $2, KK
+#endif
+
+	addq	$2 * SIZE, CO1		# coffset += 2
+	addq	$2 * SIZE, CO2		# coffset += 2
+	ALIGN_4
+
+.L4_40:
+	testq	$1, M		
+	jz	.L4_60		// to next 4 lines of N
+
+	ALIGN_4
+
+.L4_41:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+	leaq	(,BI, 4), BI			// BI = BI * 4 ; number of values
+        leaq    (BO, BI, SIZE), BO
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $1, %rax        // number of values in AO
+#else
+        addq    $4, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+	andq	$-8, %rax
+	je	.L4_46
+	movq    %rax, BI                        //  Index for BO
+        leaq    (,BI,4), BI                   	//  BI = BI * 4 ; number of values
+
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L4_42:
+
+	KERNEL1x4_SUB
+	KERNEL1x4_SUB
+	KERNEL1x4_SUB
+	KERNEL1x4_SUB
+
+	KERNEL1x4_SUB
+	KERNEL1x4_SUB
+	KERNEL1x4_SUB
+	KERNEL1x4_SUB
+
+	je	.L4_46
+
+	KERNEL1x4_SUB
+	KERNEL1x4_SUB
+	KERNEL1x4_SUB
+	KERNEL1x4_SUB
+
+	KERNEL1x4_SUB
+	KERNEL1x4_SUB
+	KERNEL1x4_SUB
+	KERNEL1x4_SUB
+
+	je	.L4_46
+
+	jmp	.L4_42
+	ALIGN_4
+
+.L4_46:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L4_49
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (,BI,4), BI                   	//  BI = BI * 4 ; number of values
+
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L4_47:
+
+	KERNEL1x4_SUB
+
+	jl	.L4_47
+	ALIGN_4
+
+
+.L4_49:
+
+	SAVE1x4
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax 
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+	leaq	(,BI, 4), BI			// BI = BI * 4 ; number of values
+        leaq    (BO, BI, SIZE), BO         
+        leaq    (AO, %rax, SIZE), AO
+#endif  
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $1, KK
+#endif
+
+	addq	$1 * SIZE, CO1		# coffset += 1
+	addq	$1 * SIZE, CO2		# coffset += 1
+	ALIGN_4
+	
+
+
+
+	
+.L4_60:
+#if defined(TRMMKERNEL) && !defined(LEFT)
+        addq    $4, KK
+#endif
+
+	decq	J			// j --
+	jg	.L4_01			// next 4 lines of N
+
+
+
+/*******************************************************************************************/
+.L2_0:
+
+	movq	Nmod6, J		
+	andq	$3, J			// j % 4
+	je	.L999
+
+	movq	Nmod6, J		
+	andq	$2, J			// j % 4
+	je	.L1_0
+
+.L2_01:
+
+	// copy to sub buffer
+	movq	B, BO1
+	leaq    BUFFER1, BO		// first buffer to BO
+	movq	K, %rax
+	sarq	$2, %rax		// K / 4
+	jz	.L2_01b
+	ALIGN_4
+
+.L2_01a:
+
+	vmovsd	      (BO1), %xmm0
+	vmovsd	2*SIZE(BO1), %xmm1
+	vmovsd	4*SIZE(BO1), %xmm2
+	vmovsd	6*SIZE(BO1), %xmm3
+
+	vmovsd	%xmm0,       (BO)
+	vmovsd	%xmm1, 2*SIZE(BO)
+	vmovsd	%xmm2, 4*SIZE(BO)
+	vmovsd	%xmm3, 6*SIZE(BO)
+
+	addq	$8*SIZE,BO1
+	addq	$8*SIZE,BO
+	decq	%rax
+	jnz	.L2_01a
+
+
+.L2_01b:
+
+        movq    K, %rax
+        andq    $3, %rax                // K % 4
+        jz      .L2_02d
+        ALIGN_4
+
+.L2_02c:
+
+	vmovsd 	(BO1), %xmm0
+	vmovsd 	%xmm0, (BO)
+	addq	$2*SIZE,BO1
+	addq	$2*SIZE,BO
+	decq	%rax
+	jnz	.L2_02c
+
+.L2_02d:
+
+	movq	BO1, B			// next offset of B
+
+.L2_10:
+	movq	C, CO1
+	leaq	(C, LDC, 2), C		// c += 2 * ldc
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        movq    OFFSET, %rax
+        movq    %rax, KK
+#endif
+	
+	movq	A, AO		 	// aoffset = a
+	addq	$16 * SIZE, AO
+
+	movq	M,  I
+	sarq	$4, I			// i = (m >> 4)
+	je	.L2_20
+
+	ALIGN_4
+
+.L2_11:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO
+        salq    $4, %rax                        // rax = rax * 16 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $16, %rax	// number of values in AO
+#else
+        addq    $2, %rax	// number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+	andq	$-8, %rax			//  K = K - ( K % 8 )
+	je	.L2_16
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$4, %rax			// rax = rax * 16 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_12:
+
+	KERNEL16x2_SUB
+	KERNEL16x2_SUB
+	KERNEL16x2_SUB
+	KERNEL16x2_SUB
+
+	KERNEL16x2_SUB
+	KERNEL16x2_SUB
+	KERNEL16x2_SUB
+	KERNEL16x2_SUB
+
+	je	.L2_16
+
+	KERNEL16x2_SUB
+	KERNEL16x2_SUB
+	KERNEL16x2_SUB
+	KERNEL16x2_SUB
+
+	KERNEL16x2_SUB
+	KERNEL16x2_SUB
+	KERNEL16x2_SUB
+	KERNEL16x2_SUB
+
+	je	.L2_16
+
+	jmp	.L2_12
+	ALIGN_4
+
+.L2_16:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_19
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$4, %rax			// rax = rax * 16 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_17:
+
+	KERNEL16x2_SUB
+
+	jl	.L2_17
+	ALIGN_4
+
+
+.L2_19:
+
+	SAVE16x2
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO
+        salq    $4, %rax                        // rax = rax * 16 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $16, KK				
+#endif
+
+	addq	$16 * SIZE, CO1		# coffset += 16
+	decq	I			# i --
+	jg	.L2_11
+	ALIGN_4	
+
+/**************************************************************************
+* Rest of M 
+***************************************************************************/
+.L2_20:
+	// Test rest of M
+
+	testq	$15, M
+	jz	.L2_60		// to next 2 lines of N
+
+	testq	$8, M		
+	jz	.L2_21pre
+	ALIGN_4
+
+/**************************************************************************/
+
+.L2_20_1:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO
+        salq    $3, %rax                        // rax = rax * 8 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $8, %rax        // number of values in A
+#else
+        addq    $2, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$-8, %rax
+	je	.L2_20_6
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_20_2:
+
+
+	KERNEL8x2_SUB
+	KERNEL8x2_SUB
+	KERNEL8x2_SUB
+	KERNEL8x2_SUB
+
+	KERNEL8x2_SUB
+	KERNEL8x2_SUB
+	KERNEL8x2_SUB
+	KERNEL8x2_SUB
+
+	je	.L2_20_6
+
+	KERNEL8x2_SUB
+	KERNEL8x2_SUB
+	KERNEL8x2_SUB
+	KERNEL8x2_SUB
+
+	KERNEL8x2_SUB
+	KERNEL8x2_SUB
+	KERNEL8x2_SUB
+	KERNEL8x2_SUB
+
+	je	.L2_20_6
+
+	jmp	.L2_20_2
+	ALIGN_4
+
+.L2_20_6:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_20_9
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_20_7:
+
+	KERNEL8x2_SUB
+
+	jl	.L2_20_7
+	ALIGN_4
+
+
+.L2_20_9:
+
+	SAVE8x2
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax 
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO         
+        salq    $3, %rax                        // rax = rax * 8 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif  
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $8, KK
+#endif
+
+	addq	$8 * SIZE, CO1		# coffset += 8
+	ALIGN_4
+	
+
+
+/**************************************************************************/
+
+.L2_21pre:
+
+	testq	$4, M		
+	jz	.L2_30
+	ALIGN_4
+
+.L2_21:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO
+        salq    $2, %rax                        // rax = rax * 4 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $4, %rax        // number of values in A
+#else
+        addq    $2, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$-8, %rax
+	je	.L2_26
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 1 ; number of values
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_22:
+
+
+	KERNEL4x2_SUB
+	KERNEL4x2_SUB
+	KERNEL4x2_SUB
+	KERNEL4x2_SUB
+
+	KERNEL4x2_SUB
+	KERNEL4x2_SUB
+	KERNEL4x2_SUB
+	KERNEL4x2_SUB
+
+	je	.L2_26
+
+	KERNEL4x2_SUB
+	KERNEL4x2_SUB
+	KERNEL4x2_SUB
+	KERNEL4x2_SUB
+
+	KERNEL4x2_SUB
+	KERNEL4x2_SUB
+	KERNEL4x2_SUB
+	KERNEL4x2_SUB
+
+	je	.L2_26
+
+	jmp	.L2_22
+	ALIGN_4
+
+.L2_26:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_29
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_27:
+
+	KERNEL4x2_SUB
+
+	jl	.L2_27
+	ALIGN_4
+
+
+.L2_29:
+
+	SAVE4x2
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax 
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO         
+        salq    $2, %rax                        // rax = rax * 4 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif  
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $4, KK
+#endif
+
+	addq	$4 * SIZE, CO1		# coffset += 4
+	ALIGN_4
+	
+
+.L2_30:
+	testq	$2, M		
+	jz	.L2_40
+
+	ALIGN_4
+
+.L2_31:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO
+        salq    $1, %rax                        // rax = rax * 2 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $2, %rax        // number of values in AO
+#else
+        addq    $2, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$-8, %rax
+	je	.L2_36
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_32:
+
+	KERNEL2x2_SUB
+	KERNEL2x2_SUB
+	KERNEL2x2_SUB
+	KERNEL2x2_SUB
+
+	KERNEL2x2_SUB
+	KERNEL2x2_SUB
+	KERNEL2x2_SUB
+	KERNEL2x2_SUB
+
+	je	.L2_36
+
+	KERNEL2x2_SUB
+	KERNEL2x2_SUB
+	KERNEL2x2_SUB
+	KERNEL2x2_SUB
+
+	KERNEL2x2_SUB
+	KERNEL2x2_SUB
+	KERNEL2x2_SUB
+	KERNEL2x2_SUB
+
+	je	.L2_36
+
+	jmp	.L2_32
+	ALIGN_4
+
+.L2_36:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_39
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+	
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_37:
+
+	KERNEL2x2_SUB
+
+	jl	.L2_37
+	ALIGN_4
+
+
+.L2_39:
+
+	SAVE2x2
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax 
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO         
+        salq    $1, %rax                        // rax = rax * 2 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif  
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $2, KK
+#endif
+
+	addq	$2 * SIZE, CO1		# coffset += 2
+	ALIGN_4
+
+.L2_40:
+	testq	$1, M		
+	jz	.L2_60		// to next 2 lines of N
+
+	ALIGN_4
+
+.L2_41:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $1, %rax        // number of values in AO
+#else
+        addq    $2, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+	andq	$-8, %rax
+	je	.L2_46
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_42:
+
+	KERNEL1x2_SUB
+	KERNEL1x2_SUB
+	KERNEL1x2_SUB
+	KERNEL1x2_SUB
+
+	KERNEL1x2_SUB
+	KERNEL1x2_SUB
+	KERNEL1x2_SUB
+	KERNEL1x2_SUB
+
+	je	.L2_46
+
+	KERNEL1x2_SUB
+	KERNEL1x2_SUB
+	KERNEL1x2_SUB
+	KERNEL1x2_SUB
+
+	KERNEL1x2_SUB
+	KERNEL1x2_SUB
+	KERNEL1x2_SUB
+	KERNEL1x2_SUB
+
+	je	.L2_46
+
+	jmp	.L2_42
+	ALIGN_4
+
+.L2_46:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L2_49
+
+	movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L2_47:
+
+	KERNEL1x2_SUB
+
+	jl	.L2_47
+	ALIGN_4
+
+
+.L2_49:
+
+	SAVE1x2
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax 
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values
+        leaq    (BO, BI, SIZE), BO         
+        leaq    (AO, %rax, SIZE), AO
+#endif  
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $1, KK
+#endif
+
+	addq	$1 * SIZE, CO1		# coffset += 1
+	ALIGN_4
+	
+
+
+
+	
+.L2_60:
+#if defined(TRMMKERNEL) && !defined(LEFT)
+        addq    $2, KK
+#endif
+
+
+
+
+.L1_0:
+
+/************************************************************************************************
+* Loop for Nmod6 % 2 > 0
+*************************************************************************************************/
+
+	movq	Nmod6, J		
+	andq	$1, J			// j % 2
+	je	.L999
+	ALIGN_4
+
+.L1_01:
+	// copy to sub buffer
+	movq	B, BO1
+	leaq    BUFFER1, BO		// first buffer to BO
+	movq	K, %rax
+	ALIGN_4
+
+.L1_02b:
+
+	vmovss	(BO1), %xmm0
+	vmovss	%xmm0,       (BO)
+	addq	$1*SIZE,BO1
+	addq	$1*SIZE,BO
+	decq	%rax
+	jnz	.L1_02b
+
+.L1_02c:
+
+	movq	BO1, B			// next offset of B
+
+.L1_10:
+	movq	C, CO1
+	leaq	(C, LDC, 1), C		// c += 1 * ldc
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        movq    OFFSET, %rax
+        movq    %rax, KK
+#endif
+	
+	movq	A, AO		 	// aoffset = a
+	addq	$16 * SIZE, AO
+
+	movq	M,  I
+	sarq	$4, I			// i = (m >> 4)
+	je	.L1_20
+
+	ALIGN_4
+
+.L1_11:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BO, BI, SIZE), BO
+        salq    $4, %rax                        // rax = rax * 16 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $16, %rax	// number of values in AO
+#else
+        addq    $1, %rax	// number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+	andq	$-8, %rax			//  K = K - ( K % 8 )
+	je	.L1_16
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$4, %rax			// rax = rax * 16 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_12:
+
+	KERNEL16x1_SUB
+	KERNEL16x1_SUB
+	KERNEL16x1_SUB
+	KERNEL16x1_SUB
+
+	KERNEL16x1_SUB
+	KERNEL16x1_SUB
+	KERNEL16x1_SUB
+	KERNEL16x1_SUB
+
+	je	.L1_16
+
+	KERNEL16x1_SUB
+	KERNEL16x1_SUB
+	KERNEL16x1_SUB
+	KERNEL16x1_SUB
+
+	KERNEL16x1_SUB
+	KERNEL16x1_SUB
+	KERNEL16x1_SUB
+	KERNEL16x1_SUB
+
+	je	.L1_16
+
+	jmp	.L1_12
+	ALIGN_4
+
+.L1_16:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_19
+
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$4, %rax			// rax = rax * 16 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_17:
+
+	KERNEL16x1_SUB
+
+	jl	.L1_17
+	ALIGN_4
+
+
+.L1_19:
+
+	SAVE16x1
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BO, BI, SIZE), BO
+        salq    $4, %rax                        // rax = rax * 16 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $16, KK				
+#endif
+
+	addq	$16 * SIZE, CO1		# coffset += 16
+	decq	I			# i --
+	jg	.L1_11
+	ALIGN_4	
+
+/**************************************************************************
+* Rest of M 
+***************************************************************************/
+.L1_20:
+	// Test rest of M
+
+	testq	$15, M
+	jz	.L999
+
+	testq	$8, M		
+	jz	.L1_21pre
+	ALIGN_4
+
+/**************************************************************************/
+
+.L1_20_1:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BO, BI, SIZE), BO
+        salq    $3, %rax                        // rax = rax * 8 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $8, %rax        // number of values in A
+#else
+        addq    $1, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$-8, %rax
+	je	.L1_20_6
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_20_2:
+
+	KERNEL8x1_SUB
+	KERNEL8x1_SUB
+	KERNEL8x1_SUB
+	KERNEL8x1_SUB
+
+	KERNEL8x1_SUB
+	KERNEL8x1_SUB
+	KERNEL8x1_SUB
+	KERNEL8x1_SUB
+
+	je	.L1_20_6
+
+	KERNEL8x1_SUB
+	KERNEL8x1_SUB
+	KERNEL8x1_SUB
+	KERNEL8x1_SUB
+
+	KERNEL8x1_SUB
+	KERNEL8x1_SUB
+	KERNEL8x1_SUB
+	KERNEL8x1_SUB
+
+	je	.L1_20_6
+
+	jmp	.L1_20_2
+	ALIGN_4
+
+.L1_20_6:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_20_9
+
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$3, %rax			// rax = rax * 8 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_20_7:
+
+	KERNEL8x1_SUB
+
+	jl	.L1_20_7
+	ALIGN_4
+
+
+.L1_20_9:
+
+	SAVE8x1
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax 
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BO, BI, SIZE), BO         
+        salq    $3, %rax                        // rax = rax * 8 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif  
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $8, KK
+#endif
+
+	addq	$8 * SIZE, CO1		# coffset += 8
+	ALIGN_4
+	
+
+
+/**************************************************************************/
+
+.L1_21pre:
+
+	testq	$4, M		
+	jz	.L1_30
+	ALIGN_4
+
+.L1_21:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BO, BI, SIZE), BO
+        salq    $2, %rax                        // rax = rax * 4 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $4, %rax        // number of values in A
+#else
+        addq    $1, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$-8, %rax
+	je	.L1_26
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_22:
+
+	KERNEL4x1_SUB
+	KERNEL4x1_SUB
+	KERNEL4x1_SUB
+	KERNEL4x1_SUB
+
+	KERNEL4x1_SUB
+	KERNEL4x1_SUB
+	KERNEL4x1_SUB
+	KERNEL4x1_SUB
+
+	je	.L1_26
+
+	KERNEL4x1_SUB
+	KERNEL4x1_SUB
+	KERNEL4x1_SUB
+	KERNEL4x1_SUB
+
+	KERNEL4x1_SUB
+	KERNEL4x1_SUB
+	KERNEL4x1_SUB
+	KERNEL4x1_SUB
+
+	je	.L1_26
+
+	jmp	.L1_22
+	ALIGN_4
+
+.L1_26:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_29
+
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$2, %rax			// rax = rax * 4 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_27:
+
+	KERNEL4x1_SUB
+
+	jl	.L1_27
+	ALIGN_4
+
+
+.L1_29:
+
+	SAVE4x1
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax 
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BO, BI, SIZE), BO         
+        salq    $2, %rax                        // rax = rax * 4 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif  
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $4, KK
+#endif
+
+	addq	$4 * SIZE, CO1		# coffset += 4
+	ALIGN_4
+	
+
+.L1_30:
+	testq	$2, M		
+	jz	.L1_40
+
+	ALIGN_4
+
+.L1_31:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BO, BI, SIZE), BO
+        salq    $1, %rax                        // rax = rax * 2 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $2, %rax        // number of values in AO
+#else
+        addq    $1, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+
+	andq	$-8, %rax
+	je	.L1_36
+	movq    %rax, BI                        //  Index for BO
+
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_32:
+
+	KERNEL2x1_SUB
+	KERNEL2x1_SUB
+	KERNEL2x1_SUB
+	KERNEL2x1_SUB
+
+	KERNEL2x1_SUB
+	KERNEL2x1_SUB
+	KERNEL2x1_SUB
+	KERNEL2x1_SUB
+
+	je	.L1_36
+
+	KERNEL2x1_SUB
+	KERNEL2x1_SUB
+	KERNEL2x1_SUB
+	KERNEL2x1_SUB
+
+	KERNEL2x1_SUB
+	KERNEL2x1_SUB
+	KERNEL2x1_SUB
+	KERNEL2x1_SUB
+
+	je	.L1_36
+
+	jmp	.L1_32
+	ALIGN_4
+
+.L1_36:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_39
+
+	movq    %rax, BI                        //  Index for BO
+	
+	salq	$1, %rax			// rax = rax *2 ; number of values
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_37:
+
+	KERNEL2x1_SUB
+
+	jl	.L1_37
+	ALIGN_4
+
+
+.L1_39:
+
+	SAVE2x1
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax 
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BO, BI, SIZE), BO         
+        salq    $1, %rax                        // rax = rax * 2 ; number of values
+        leaq    (AO, %rax, SIZE), AO
+#endif  
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $2, KK
+#endif
+
+	addq	$2 * SIZE, CO1		# coffset += 2
+	ALIGN_4
+
+.L1_40:
+	testq	$1, M		
+	jz	.L999
+
+	ALIGN_4
+
+.L1_41:
+#if !defined(TRMMKERNEL) || \
+        (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+        (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+#else
+        movq    KK, %rax
+        leaq    BUFFER1, BO             // first buffer to BO
+        addq    $4 * SIZE, BO
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BO, BI, SIZE), BO
+        leaq    (AO, %rax, SIZE), AO
+#endif
+
+
+	vzeroall
+
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
+        movq    K, %rax
+        subq    KK, %rax
+        movq    %rax, KKK
+#else
+        movq    KK, %rax
+#ifdef LEFT
+        addq    $1, %rax        // number of values in AO
+#else
+        addq    $1, %rax        // number of values in BO
+#endif
+        movq    %rax, KKK
+#endif
+
+	andq	$-8, %rax
+	je	.L1_46
+	movq    %rax, BI                        //  Index for BO
+
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_42:
+
+	KERNEL1x1_SUB
+	KERNEL1x1_SUB
+	KERNEL1x1_SUB
+	KERNEL1x1_SUB
+
+	KERNEL1x1_SUB
+	KERNEL1x1_SUB
+	KERNEL1x1_SUB
+	KERNEL1x1_SUB
+
+	je	.L1_46
+
+	KERNEL1x1_SUB
+	KERNEL1x1_SUB
+	KERNEL1x1_SUB
+	KERNEL1x1_SUB
+
+	KERNEL1x1_SUB
+	KERNEL1x1_SUB
+	KERNEL1x1_SUB
+	KERNEL1x1_SUB
+
+	je	.L1_46
+
+	jmp	.L1_42
+	ALIGN_4
+
+.L1_46:
+#ifndef TRMMKERNEL
+        movq    K, %rax
+#else
+        movq    KKK, %rax
+#endif
+
+	andq	$7, %rax		# if (k & 1)
+	je .L1_49
+
+	movq    %rax, BI                        //  Index for BO
+
+	leaq	(AO, %rax, SIZE), AO
+	leaq	(BO, BI, SIZE), BO
+	negq	BI
+	negq	%rax
+	ALIGN_4
+
+.L1_47:
+
+	KERNEL1x1_SUB
+
+	jl	.L1_47
+	ALIGN_4
+
+
+.L1_49:
+
+	SAVE1x1
+
+#if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \
+    (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))
+        movq    K, %rax 
+        subq    KKK, %rax
+        movq    %rax, BI                        //  Index for BO
+        leaq    (BO, BI, SIZE), BO         
+        leaq    (AO, %rax, SIZE), AO
+#endif  
+
+
+#if defined(TRMMKERNEL) && defined(LEFT)
+        addq    $1, KK
+#endif
+
+	addq	$1 * SIZE, CO1		# coffset += 1
+	ALIGN_4
+	
+
+.L999:
+	movq   		SP, %rsp
+	movq	   (%rsp), %rbx
+	movq	  8(%rsp), %rbp
+	movq	 16(%rsp), %r12
+	movq	 24(%rsp), %r13
+	movq	 32(%rsp), %r14
+	movq	 40(%rsp), %r15
+
+#ifdef WINDOWS_ABI
+	movq	 48(%rsp), %rdi
+	movq	 56(%rsp), %rsi
+	movups	 64(%rsp), %xmm6
+	movups	 80(%rsp), %xmm7
+	movups	 96(%rsp), %xmm8
+	movups	112(%rsp), %xmm9
+	movups	128(%rsp), %xmm10
+	movups	144(%rsp), %xmm11
+	movups	160(%rsp), %xmm12
+	movups	176(%rsp), %xmm13
+	movups	192(%rsp), %xmm14
+	movups	208(%rsp), %xmm15
+#endif
+
+	addq	$STACKSIZE, %rsp
+	ret
+
+	EPILOGUE
+
+
+
+
+
diff --git a/kernel/x86_64/strsm_kernel_8x4_haswell_RN.c b/kernel/x86_64/strsm_kernel_8x4_haswell_RN.c
index 4e2cd4fe6..dbfcd55d7 100644
--- a/kernel/x86_64/strsm_kernel_8x4_haswell_RN.c
+++ b/kernel/x86_64/strsm_kernel_8x4_haswell_RN.c
@@ -1,279 +1,279 @@
-#include "common.h"
-#include <stdint.h>
-#include "strsm_kernel_8x4_haswell_R_common.h"
-
-#define SOLVE_RN_m8n4 \
-  "movq %2,%3;" GEMM_SUM_REORDER_8x4(4,5,6,7,63) "movq %2,%3; addq $32,%2;"\
-  SOLVE_leri_m8n2(0,4,5,%1) SUBTRACT_m8n2(8,6,7,%1)\
-  SOLVE_ri_m8n2(16,4,5,%1) SUBTRACT_m8n2(24,6,7,%1)\
-  SAVE_SOLUTION_m8n2(4,5,0)\
-  SOLVE_leri_m8n2(40,6,7,%1)\
-  SOLVE_ri_m8n2(56,6,7,%1)\
-  SAVE_SOLUTION_m8n2(6,7,64)
-
-#define SOLVE_RN_m8n8 \
-  "movq %2,%3;" GEMM_SUM_REORDER_8x4(4,5,6,7,63) GEMM_SUM_REORDER_8x4(8,9,10,11,63) "movq %2,%3; addq $32,%2;"\
-  SOLVE_leri_m8n2(0,4,5,%1) SUBTRACT_m8n2(8,6,7,%1) SUBTRACT_m8n2(0,8,9,%1,%%r12,4) SUBTRACT_m8n2(8,10,11,%1,%%r12,4)\
-  SOLVE_ri_m8n2(16,4,5,%1) SUBTRACT_m8n2(24,6,7,%1) SUBTRACT_m8n2(16,8,9,%1,%%r12,4) SUBTRACT_m8n2(24,10,11,%1,%%r12,4)\
-  SAVE_SOLUTION_m8n2(4,5,0)\
-  SOLVE_leri_m8n2(40,6,7,%1) SUBTRACT_m8n2(32,8,9,%1,%%r12,4) SUBTRACT_m8n2(40,10,11,%1,%%r12,4)\
-  SOLVE_ri_m8n2(56,6,7,%1) SUBTRACT_m8n2(48,8,9,%1,%%r12,4) SUBTRACT_m8n2(56,10,11,%1,%%r12,4)\
-  SAVE_SOLUTION_m8n2(6,7,64)\
-  SOLVE_leri_m8n2(64,8,9,%1,%%r12,4) SUBTRACT_m8n2(72,10,11,%1,%%r12,4)\
-  SOLVE_ri_m8n2(80,8,9,%1,%%r12,4) SUBTRACT_m8n2(88,10,11,%1,%%r12,4)\
-  SAVE_SOLUTION_m8n2(8,9,128)\
-  SOLVE_leri_m8n2(104,10,11,%1,%%r12,4)\
-  SOLVE_ri_m8n2(120,10,11,%1,%%r12,4)\
-  SAVE_SOLUTION_m8n2(10,11,192)
-
-#define SOLVE_RN_m8n12 \
-  "movq %2,%3;" GEMM_SUM_REORDER_8x4(4,5,6,7,63) GEMM_SUM_REORDER_8x4(8,9,10,11,63) GEMM_SUM_REORDER_8x4(12,13,14,15,63) "movq %2,%3; addq $32,%2;"\
-  SOLVE_leri_m8n2(0,4,5,%1) SUBTRACT_m8n2(8,6,7,%1) SUBTRACT_m8n2(0,8,9,%1,%%r12,4) SUBTRACT_m8n2(8,10,11,%1,%%r12,4) SUBTRACT_m8n2(0,12,13,%1,%%r12,8) SUBTRACT_m8n2(8,14,15,%1,%%r12,8)\
-  SOLVE_ri_m8n2(16,4,5,%1) SUBTRACT_m8n2(24,6,7,%1) SUBTRACT_m8n2(16,8,9,%1,%%r12,4) SUBTRACT_m8n2(24,10,11,%1,%%r12,4) SUBTRACT_m8n2(16,12,13,%1,%%r12,8) SUBTRACT_m8n2(24,14,15,%1,%%r12,8)\
-  SAVE_SOLUTION_m8n2(4,5,0)\
-  SOLVE_leri_m8n2(40,6,7,%1) SUBTRACT_m8n2(32,8,9,%1,%%r12,4) SUBTRACT_m8n2(40,10,11,%1,%%r12,4) SUBTRACT_m8n2(32,12,13,%1,%%r12,8) SUBTRACT_m8n2(40,14,15,%1,%%r12,8)\
-  SOLVE_ri_m8n2(56,6,7,%1) SUBTRACT_m8n2(48,8,9,%1,%%r12,4) SUBTRACT_m8n2(56,10,11,%1,%%r12,4) SUBTRACT_m8n2(48,12,13,%1,%%r12,8) SUBTRACT_m8n2(56,14,15,%1,%%r12,8)\
-  SAVE_SOLUTION_m8n2(6,7,64)\
-  SOLVE_leri_m8n2(64,8,9,%1,%%r12,4) SUBTRACT_m8n2(72,10,11,%1,%%r12,4) SUBTRACT_m8n2(64,12,13,%1,%%r12,8) SUBTRACT_m8n2(72,14,15,%1,%%r12,8)\
-  SOLVE_ri_m8n2(80,8,9,%1,%%r12,4) SUBTRACT_m8n2(88,10,11,%1,%%r12,4) SUBTRACT_m8n2(80,12,13,%1,%%r12,8) SUBTRACT_m8n2(88,14,15,%1,%%r12,8)\
-  SAVE_SOLUTION_m8n2(8,9,128)\
-  SOLVE_leri_m8n2(104,10,11,%1,%%r12,4) SUBTRACT_m8n2(96,12,13,%1,%%r12,8) SUBTRACT_m8n2(104,14,15,%1,%%r12,8)\
-  SOLVE_ri_m8n2(120,10,11,%1,%%r12,4) SUBTRACT_m8n2(112,12,13,%1,%%r12,8) SUBTRACT_m8n2(120,14,15,%1,%%r12,8)\
-  SAVE_SOLUTION_m8n2(10,11,192)\
-  SOLVE_leri_m8n2(128,12,13,%1,%%r12,8) SUBTRACT_m8n2(136,14,15,%1,%%r12,8)\
-  SOLVE_ri_m8n2(144,12,13,%1,%%r12,8) SUBTRACT_m8n2(152,14,15,%1,%%r12,8)\
-  SAVE_SOLUTION_m8n2(12,13,256)\
-  SOLVE_leri_m8n2(168,14,15,%1,%%r12,8)\
-  SOLVE_ri_m8n2(184,14,15,%1,%%r12,8)\
-  SAVE_SOLUTION_m8n2(14,15,320)
-
-#define SOLVE_RN_m4n4 \
-  "movq %2,%3;" GEMM_SUM_REORDER_4x4(4,5,6,7,4,5) "movq %2,%3; addq $16,%2;"\
-  SOLVE_leri_m4n2(0,4,%1) SUBTRACT_m4n2(8,5,%1)\
-  SOLVE_ri_m4n2(16,4,%1) SUBTRACT_m4n2(24,5,%1)\
-  SAVE_SOLUTION_m4n2(4,0)\
-  SOLVE_leri_m4n2(40,5,%1)\
-  SOLVE_ri_m4n2(56,5,%1)\
-  SAVE_SOLUTION_m4n2(5,32)
-
-#define SOLVE_RN_m4n8 \
-  "movq %2,%3;" GEMM_SUM_REORDER_4x4(4,5,6,7,4,5) GEMM_SUM_REORDER_4x4(8,9,10,11,6,7) "movq %2,%3; addq $16,%2;"\
-  SOLVE_leri_m4n2(0,4,%1) SUBTRACT_m4n2(8,5,%1) SUBTRACT_m4n2(0,6,%1,%%r12,4) SUBTRACT_m4n2(8,7,%1,%%r12,4)\
-  SOLVE_ri_m4n2(16,4,%1) SUBTRACT_m4n2(24,5,%1) SUBTRACT_m4n2(16,6,%1,%%r12,4) SUBTRACT_m4n2(24,7,%1,%%r12,4)\
-  SAVE_SOLUTION_m4n2(4,0)\
-  SOLVE_leri_m4n2(40,5,%1) SUBTRACT_m4n2(32,6,%1,%%r12,4) SUBTRACT_m4n2(40,7,%1,%%r12,4)\
-  SOLVE_ri_m4n2(56,5,%1) SUBTRACT_m4n2(48,6,%1,%%r12,4) SUBTRACT_m4n2(56,7,%1,%%r12,4)\
-  SAVE_SOLUTION_m4n2(5,32)\
-  SOLVE_leri_m4n2(64,6,%1,%%r12,4) SUBTRACT_m4n2(72,7,%1,%%r12,4)\
-  SOLVE_ri_m4n2(80,6,%1,%%r12,4) SUBTRACT_m4n2(88,7,%1,%%r12,4)\
-  SAVE_SOLUTION_m4n2(6,64)\
-  SOLVE_leri_m4n2(104,7,%1,%%r12,4)\
-  SOLVE_ri_m4n2(120,7,%1,%%r12,4)\
-  SAVE_SOLUTION_m4n2(7,96)
-
-#define SOLVE_RN_m4n12 \
-  "movq %2,%3;" GEMM_SUM_REORDER_4x4(4,5,6,7,4,5) GEMM_SUM_REORDER_4x4(8,9,10,11,6,7) GEMM_SUM_REORDER_4x4(12,13,14,15,8,9) "movq %2,%3; addq $16,%2;"\
-  SOLVE_leri_m4n2(0,4,%1) SUBTRACT_m4n2(8,5,%1) SUBTRACT_m4n2(0,6,%1,%%r12,4) SUBTRACT_m4n2(8,7,%1,%%r12,4) SUBTRACT_m4n2(0,8,%1,%%r12,8) SUBTRACT_m4n2(8,9,%1,%%r12,8)\
-  SOLVE_ri_m4n2(16,4,%1) SUBTRACT_m4n2(24,5,%1) SUBTRACT_m4n2(16,6,%1,%%r12,4) SUBTRACT_m4n2(24,7,%1,%%r12,4) SUBTRACT_m4n2(16,8,%1,%%r12,8) SUBTRACT_m4n2(24,9,%1,%%r12,8)\
-  SAVE_SOLUTION_m4n2(4,0)\
-  SOLVE_leri_m4n2(40,5,%1) SUBTRACT_m4n2(32,6,%1,%%r12,4) SUBTRACT_m4n2(40,7,%1,%%r12,4) SUBTRACT_m4n2(32,8,%1,%%r12,8) SUBTRACT_m4n2(40,9,%1,%%r12,8)\
-  SOLVE_ri_m4n2(56,5,%1) SUBTRACT_m4n2(48,6,%1,%%r12,4) SUBTRACT_m4n2(56,7,%1,%%r12,4) SUBTRACT_m4n2(48,8,%1,%%r12,8) SUBTRACT_m4n2(56,9,%1,%%r12,8)\
-  SAVE_SOLUTION_m4n2(5,32)\
-  SOLVE_leri_m4n2(64,6,%1,%%r12,4) SUBTRACT_m4n2(72,7,%1,%%r12,4) SUBTRACT_m4n2(64,8,%1,%%r12,8) SUBTRACT_m4n2(72,9,%1,%%r12,8)\
-  SOLVE_ri_m4n2(80,6,%1,%%r12,4) SUBTRACT_m4n2(88,7,%1,%%r12,4) SUBTRACT_m4n2(80,8,%1,%%r12,8) SUBTRACT_m4n2(88,9,%1,%%r12,8)\
-  SAVE_SOLUTION_m4n2(6,64)\
-  SOLVE_leri_m4n2(104,7,%1,%%r12,4) SUBTRACT_m4n2(96,8,%1,%%r12,8) SUBTRACT_m4n2(104,9,%1,%%r12,8)\
-  SOLVE_ri_m4n2(120,7,%1,%%r12,4) SUBTRACT_m4n2(112,8,%1,%%r12,8) SUBTRACT_m4n2(120,9,%1,%%r12,8)\
-  SAVE_SOLUTION_m4n2(7,96)\
-  SOLVE_leri_m4n2(128,8,%1,%%r12,8) SUBTRACT_m4n2(136,9,%1,%%r12,8)\
-  SOLVE_ri_m4n2(144,8,%1,%%r12,8) SUBTRACT_m4n2(152,9,%1,%%r12,8)\
-  SAVE_SOLUTION_m4n2(8,128)\
-  SOLVE_leri_m4n2(168,9,%1,%%r12,8)\
-  SOLVE_ri_m4n2(184,9,%1,%%r12,8)\
-  SAVE_SOLUTION_m4n2(9,160)
-
-#define SOLVE_RN_m2n4 \
-  "movq %2,%3;" GEMM_SUM_REORDER_2x4(4,5) "movq %2,%3; addq $8,%2;"\
-  SOLVE_col1_ltor_m2n4(0,4,5,%1)\
-  SOLVE_col2_ltor_m2n4(16,4,5,%1)\
-  SOLVE_col3_ltor_m2n4(32,4,5,%1)\
-  SOLVE_col4_ltor_m2n4(48,4,5,%1)\
-  SAVE_SOLUTION_m2n4(4,5,0)
-
-#define SOLVE_RN_m2n8 \
-  "movq %2,%3;" GEMM_SUM_REORDER_2x4(4,5) GEMM_SUM_REORDER_2x4(6,7) "movq %2,%3; addq $8,%2;"\
-  SOLVE_col1_ltor_m2n4(0,4,5,%1) SUBTRACT_m2n4(0,6,7,%1,%%r12,4)\
-  SOLVE_col2_ltor_m2n4(16,4,5,%1) SUBTRACT_m2n4(16,6,7,%1,%%r12,4)\
-  SOLVE_col3_ltor_m2n4(32,4,5,%1) SUBTRACT_m2n4(32,6,7,%1,%%r12,4)\
-  SOLVE_col4_ltor_m2n4(48,4,5,%1) SUBTRACT_m2n4(48,6,7,%1,%%r12,4)\
-  SAVE_SOLUTION_m2n4(4,5,0)\
-  SOLVE_col1_ltor_m2n4(64,6,7,%1,%%r12,4)\
-  SOLVE_col2_ltor_m2n4(80,6,7,%1,%%r12,4)\
-  SOLVE_col3_ltor_m2n4(96,6,7,%1,%%r12,4)\
-  SOLVE_col4_ltor_m2n4(112,6,7,%1,%%r12,4)\
-  SAVE_SOLUTION_m2n4(6,7,32)
-
-#define SOLVE_RN_m2n12 \
-  "movq %2,%3;" GEMM_SUM_REORDER_2x4(4,5) GEMM_SUM_REORDER_2x4(6,7) GEMM_SUM_REORDER_2x4(8,9) "movq %2,%3; addq $8,%2;"\
-  SOLVE_col1_ltor_m2n4(0,4,5,%1) SUBTRACT_m2n4(0,6,7,%1,%%r12,4) SUBTRACT_m2n4(0,8,9,%1,%%r12,8)\
-  SOLVE_col2_ltor_m2n4(16,4,5,%1) SUBTRACT_m2n4(16,6,7,%1,%%r12,4) SUBTRACT_m2n4(16,8,9,%1,%%r12,8)\
-  SOLVE_col3_ltor_m2n4(32,4,5,%1) SUBTRACT_m2n4(32,6,7,%1,%%r12,4) SUBTRACT_m2n4(32,8,9,%1,%%r12,8)\
-  SOLVE_col4_ltor_m2n4(48,4,5,%1) SUBTRACT_m2n4(48,6,7,%1,%%r12,4) SUBTRACT_m2n4(48,8,9,%1,%%r12,8)\
-  SAVE_SOLUTION_m2n4(4,5,0)\
-  SOLVE_col1_ltor_m2n4(64,6,7,%1,%%r12,4) SUBTRACT_m2n4(64,8,9,%1,%%r12,8)\
-  SOLVE_col2_ltor_m2n4(80,6,7,%1,%%r12,4) SUBTRACT_m2n4(80,8,9,%1,%%r12,8)\
-  SOLVE_col3_ltor_m2n4(96,6,7,%1,%%r12,4) SUBTRACT_m2n4(96,8,9,%1,%%r12,8)\
-  SOLVE_col4_ltor_m2n4(112,6,7,%1,%%r12,4) SUBTRACT_m2n4(112,8,9,%1,%%r12,8)\
-  SAVE_SOLUTION_m2n4(6,7,32)\
-  SOLVE_col1_ltor_m2n4(128,8,9,%1,%%r12,8)\
-  SOLVE_col2_ltor_m2n4(144,8,9,%1,%%r12,8)\
-  SOLVE_col3_ltor_m2n4(160,8,9,%1,%%r12,8)\
-  SOLVE_col4_ltor_m2n4(176,8,9,%1,%%r12,8)\
-  SAVE_SOLUTION_m2n4(8,9,64)
-
-#define SOLVE_RN_m1n4 \
-  "movq %2,%3;" GEMM_SUM_REORDER_1x4(4) "movq %2,%3; addq $4,%2;"\
-  SOLVE_col1_ltor_m1n4(0,4,%1)\
-  SOLVE_col2_ltor_m1n4(16,4,%1)\
-  SOLVE_col3_ltor_m1n4(32,4,%1)\
-  SOLVE_col4_ltor_m1n4(48,4,%1)\
-  SAVE_SOLUTION_m1n4(4,0)
-
-#define SOLVE_RN_m1n8 \
-  "movq %2,%3;" GEMM_SUM_REORDER_1x4(4) GEMM_SUM_REORDER_1x4(5) "movq %2,%3; addq $4,%2;"\
-  SOLVE_col1_ltor_m1n4(0,4,%1) SUBTRACT_m1n4(0,5,%1,%%r12,4)\
-  SOLVE_col2_ltor_m1n4(16,4,%1) SUBTRACT_m1n4(16,5,%1,%%r12,4)\
-  SOLVE_col3_ltor_m1n4(32,4,%1) SUBTRACT_m1n4(32,5,%1,%%r12,4)\
-  SOLVE_col4_ltor_m1n4(48,4,%1) SUBTRACT_m1n4(48,5,%1,%%r12,4)\
-  SAVE_SOLUTION_m1n4(4,0)\
-  SOLVE_col1_ltor_m1n4(64,5,%1,%%r12,4)\
-  SOLVE_col2_ltor_m1n4(80,5,%1,%%r12,4)\
-  SOLVE_col3_ltor_m1n4(96,5,%1,%%r12,4)\
-  SOLVE_col4_ltor_m1n4(112,5,%1,%%r12,4)\
-  SAVE_SOLUTION_m1n4(5,16)
-
-#define SOLVE_RN_m1n12 \
-  "movq %2,%3;" GEMM_SUM_REORDER_1x4(4) GEMM_SUM_REORDER_1x4(5) GEMM_SUM_REORDER_1x4(6) "movq %2,%3; addq $4,%2;"\
-  SOLVE_col1_ltor_m1n4(0,4,%1) SUBTRACT_m1n4(0,5,%1,%%r12,4) SUBTRACT_m1n4(0,6,%1,%%r12,8)\
-  SOLVE_col2_ltor_m1n4(16,4,%1) SUBTRACT_m1n4(16,5,%1,%%r12,4) SUBTRACT_m1n4(16,6,%1,%%r12,8)\
-  SOLVE_col3_ltor_m1n4(32,4,%1) SUBTRACT_m1n4(32,5,%1,%%r12,4) SUBTRACT_m1n4(32,6,%1,%%r12,8)\
-  SOLVE_col4_ltor_m1n4(48,4,%1) SUBTRACT_m1n4(48,5,%1,%%r12,4) SUBTRACT_m1n4(48,6,%1,%%r12,8)\
-  SAVE_SOLUTION_m1n4(4,0)\
-  SOLVE_col1_ltor_m1n4(64,5,%1,%%r12,4) SUBTRACT_m1n4(64,6,%1,%%r12,8)\
-  SOLVE_col2_ltor_m1n4(80,5,%1,%%r12,4) SUBTRACT_m1n4(80,6,%1,%%r12,8)\
-  SOLVE_col3_ltor_m1n4(96,5,%1,%%r12,4) SUBTRACT_m1n4(96,6,%1,%%r12,8)\
-  SOLVE_col4_ltor_m1n4(112,5,%1,%%r12,4) SUBTRACT_m1n4(112,6,%1,%%r12,8)\
-  SAVE_SOLUTION_m1n4(5,16)\
-  SOLVE_col1_ltor_m1n4(128,6,%1,%%r12,8)\
-  SOLVE_col2_ltor_m1n4(144,6,%1,%%r12,8)\
-  SOLVE_col3_ltor_m1n4(160,6,%1,%%r12,8)\
-  SOLVE_col4_ltor_m1n4(176,6,%1,%%r12,8)\
-  SAVE_SOLUTION_m1n4(6,32)
-
-#define GEMM_RN_SIMPLE(mdim,ndim) \
-  "movq %%r15,%0; leaq (%%r15,%%r12,"#mdim"),%%r15; movq %%r13,%5; movq %%r14,%1;" INIT_m##mdim##n##ndim\
-  "testq %5,%5; jz 1"#mdim""#ndim"2f;"\
-  "1"#mdim""#ndim"1:\n\t"\
-  GEMM_KERNEL_k1m##mdim##n##ndim "addq $16,%1; addq $"#mdim"*4,%0; decq %5; jnz 1"#mdim""#ndim"1b;"\
-  "1"#mdim""#ndim"2:\n\t"
-#define GEMM_RN_m8n4 GEMM_RN_SIMPLE(8,4)
-#define GEMM_RN_m8n8 GEMM_RN_SIMPLE(8,8)
-#define GEMM_RN_m8n12 \
-  "movq %%r15,%0; leaq (%%r15,%%r12,8),%%r15; movq %%r13,%5; movq %%r14,%1;" INIT_m8n12\
-  "cmpq $8,%5; jb 18122f;"\
-  "18121:\n\t"\
-  GEMM_KERNEL_k1m8n12 "prefetcht0 384(%0); addq $32,%0; addq $16,%1;"\
-  GEMM_KERNEL_k1m8n12 "addq $32,%0; addq $16,%1;"\
-  GEMM_KERNEL_k1m8n12 "prefetcht0 384(%0); addq $32,%0; addq $16,%1;"\
-  GEMM_KERNEL_k1m8n12 "addq $32,%0; addq $16,%1;"\
-  GEMM_KERNEL_k1m8n12 "prefetcht0 384(%0); addq $32,%0; addq $16,%1;"\
-  GEMM_KERNEL_k1m8n12 "addq $32,%0; addq $16,%1;"\
-  GEMM_KERNEL_k1m8n12 "prefetcht0 384(%0); addq $32,%0; addq $16,%1;"\
-  GEMM_KERNEL_k1m8n12 "addq $32,%0; addq $16,%1;"\
-  "subq $8,%5; cmpq $8,%5; jnb 18121b;"\
-  "18122:\n\t"\
-  "testq %5,%5; jz 18124f;"\
-  "18123:\n\t"\
-  GEMM_KERNEL_k1m8n12 "addq $32,%0; addq $16,%1; decq %5; jnz 18123b;"\
-  "18124:\n\t"
-#define GEMM_RN_m4n4 GEMM_RN_SIMPLE(4,4)
-#define GEMM_RN_m4n8 GEMM_RN_SIMPLE(4,8)
-#define GEMM_RN_m4n12 GEMM_RN_SIMPLE(4,12)
-#define GEMM_RN_m2n4 GEMM_RN_SIMPLE(2,4)
-#define GEMM_RN_m2n8 GEMM_RN_SIMPLE(2,8)
-#define GEMM_RN_m2n12 GEMM_RN_SIMPLE(2,12)
-#define GEMM_RN_m1n4 GEMM_RN_SIMPLE(1,4)
-#define GEMM_RN_m1n8 GEMM_RN_SIMPLE(1,8)
-#define GEMM_RN_m1n12 GEMM_RN_SIMPLE(1,12)
-
-#define COMPUTE(ndim) {\
-  __asm__ __volatile__(\
-    "movq %0,%%r15; movq %1,%%r14; movq %7,%%r13; movq %6,%%r12; salq $2,%%r12; movq %10,%%r11;"\
-    "cmpq $8,%%r11; jb "#ndim"772f;"\
-    #ndim"771:\n\t"\
-    GEMM_RN_m8n##ndim SOLVE_RN_m8n##ndim "subq $8,%%r11; cmpq $8,%%r11; jnb "#ndim"771b;"\
-    #ndim"772:\n\t"\
-    "testq $4,%%r11; jz "#ndim"773f;"\
-    GEMM_RN_m4n##ndim SOLVE_RN_m4n##ndim "subq $4,%%r11;"\
-    #ndim"773:\n\t"\
-    "testq $2,%%r11; jz "#ndim"774f;"\
-    GEMM_RN_m2n##ndim SOLVE_RN_m2n##ndim "subq $2,%%r11;"\
-    #ndim"774:\n\t"\
-    "testq $1,%%r11; jz "#ndim"775f;"\
-    GEMM_RN_m1n##ndim SOLVE_RN_m1n##ndim "subq $1,%%r11;"\
-    #ndim"775:\n\t"\
-    "movq %%r15,%0; movq %%r14,%1; vzeroupper;"\
-  :"+r"(a_ptr),"+r"(b_ptr),"+r"(c_ptr),"+r"(c_tmp),"+r"(ldc_bytes),"+r"(k_cnt):"m"(K),"m"(OFF),"m"(one[0]),"m"(zero[0]),"m"(M)\
-  :"r11","r12","r13","r14","r15","cc","memory",\
-  "xmm0","xmm1","xmm2","xmm3","xmm4","xmm5","xmm6","xmm7","xmm8","xmm9","xmm10","xmm11","xmm12","xmm13","xmm14","xmm15");\
-  a_ptr -= M * K; b_ptr += ndim * K; c_ptr += ldc * ndim - M; OFF += ndim;\
-}
-
-static void solve_RN(BLASLONG m, BLASLONG n, FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc) {
-  FLOAT a0, b0;
-  int i, j, k;
-  for (i=0; i<n; i++) {
-    b0 = b[i*n+i];
-    for (j=0; j<m; j++) {
-      a0 = c[i*ldc+j] * b0;
-      a[i*m+j] = c[i*ldc+j] = a0;
-      for (k=i+1; k<n; k++) c[k*ldc+j] -= a0 * b[i*n+k];
-    }
-  }
-}
-static void COMPUTE_EDGE_1_nchunk(BLASLONG m, BLASLONG n, FLOAT *sa, FLOAT *sb, FLOAT *C, BLASLONG ldc, BLASLONG k, BLASLONG offset) {
-  BLASLONG m_count = m, kk = offset; FLOAT *a_ptr = sa, *c_ptr = C;
-  for(;m_count>7;m_count-=8){
-    if(kk>0) GEMM_KERNEL_N(8,n,kk,-1.0,a_ptr,sb,c_ptr,ldc);
-    solve_RN(8,n,a_ptr+kk*8,sb+kk*n,c_ptr,ldc);
-    a_ptr += k * 8; c_ptr += 8;
-  }
-  for(;m_count>3;m_count-=4){
-    if(kk>0) GEMM_KERNEL_N(4,n,kk,-1.0,a_ptr,sb,c_ptr,ldc);
-    solve_RN(4,n,a_ptr+kk*4,sb+kk*n,c_ptr,ldc);
-    a_ptr += k * 4; c_ptr += 4;
-  }
-  for(;m_count>1;m_count-=2){
-    if(kk>0) GEMM_KERNEL_N(2,n,kk,-1.0,a_ptr,sb,c_ptr,ldc);
-    solve_RN(2,n,a_ptr+kk*2,sb+kk*n,c_ptr,ldc);
-    a_ptr += k * 2; c_ptr += 2;
-  }
-  if(m_count>0){
-    if(kk>0) GEMM_KERNEL_N(1,n,kk,-1.0,a_ptr,sb,c_ptr,ldc);
-    solve_RN(1,n,a_ptr+kk*1,sb+kk*n,c_ptr,ldc);
-    a_ptr += k * 1; c_ptr += 1;
-  }
-}
-int CNAME(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT dummy1, FLOAT *sa, FLOAT *sb, FLOAT *C, BLASLONG ldc, BLASLONG offset){
-  float *a_ptr = sa, *b_ptr = sb, *c_ptr = C, *c_tmp = C;
-  float one[8] = {1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0};
-  float zero[8] = {0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0};
-  uint64_t ldc_bytes = (uint64_t)ldc * sizeof(float), K = (uint64_t)k, M = (uint64_t)m, OFF = (uint64_t)-offset, k_cnt = 0;
-  BLASLONG n_count = n;
-  for(;n_count>11;n_count-=12) COMPUTE(12)
-  for(;n_count>7;n_count-=8) COMPUTE(8)
-  for(;n_count>3;n_count-=4) COMPUTE(4)
-  for(;n_count>1;n_count-=2) { COMPUTE_EDGE_1_nchunk(m,2,a_ptr,b_ptr,c_ptr,ldc,k,OFF); b_ptr += 2*k; c_ptr += ldc*2; OFF+=2;}
-  if(n_count>0) COMPUTE_EDGE_1_nchunk(m,1,a_ptr,b_ptr,c_ptr,ldc,k,OFF);
-  return 0;
-}
+#include "common.h"
+#include <stdint.h>
+#include "strsm_kernel_8x4_haswell_R_common.h"
+
+#define SOLVE_RN_m8n4 \
+  "movq %2,%3;" GEMM_SUM_REORDER_8x4(4,5,6,7,63) "movq %2,%3; addq $32,%2;"\
+  SOLVE_leri_m8n2(0,4,5,%1) SUBTRACT_m8n2(8,6,7,%1)\
+  SOLVE_ri_m8n2(16,4,5,%1) SUBTRACT_m8n2(24,6,7,%1)\
+  SAVE_SOLUTION_m8n2(4,5,0)\
+  SOLVE_leri_m8n2(40,6,7,%1)\
+  SOLVE_ri_m8n2(56,6,7,%1)\
+  SAVE_SOLUTION_m8n2(6,7,64)
+
+#define SOLVE_RN_m8n8 \
+  "movq %2,%3;" GEMM_SUM_REORDER_8x4(4,5,6,7,63) GEMM_SUM_REORDER_8x4(8,9,10,11,63) "movq %2,%3; addq $32,%2;"\
+  SOLVE_leri_m8n2(0,4,5,%1) SUBTRACT_m8n2(8,6,7,%1) SUBTRACT_m8n2(0,8,9,%1,%%r12,4) SUBTRACT_m8n2(8,10,11,%1,%%r12,4)\
+  SOLVE_ri_m8n2(16,4,5,%1) SUBTRACT_m8n2(24,6,7,%1) SUBTRACT_m8n2(16,8,9,%1,%%r12,4) SUBTRACT_m8n2(24,10,11,%1,%%r12,4)\
+  SAVE_SOLUTION_m8n2(4,5,0)\
+  SOLVE_leri_m8n2(40,6,7,%1) SUBTRACT_m8n2(32,8,9,%1,%%r12,4) SUBTRACT_m8n2(40,10,11,%1,%%r12,4)\
+  SOLVE_ri_m8n2(56,6,7,%1) SUBTRACT_m8n2(48,8,9,%1,%%r12,4) SUBTRACT_m8n2(56,10,11,%1,%%r12,4)\
+  SAVE_SOLUTION_m8n2(6,7,64)\
+  SOLVE_leri_m8n2(64,8,9,%1,%%r12,4) SUBTRACT_m8n2(72,10,11,%1,%%r12,4)\
+  SOLVE_ri_m8n2(80,8,9,%1,%%r12,4) SUBTRACT_m8n2(88,10,11,%1,%%r12,4)\
+  SAVE_SOLUTION_m8n2(8,9,128)\
+  SOLVE_leri_m8n2(104,10,11,%1,%%r12,4)\
+  SOLVE_ri_m8n2(120,10,11,%1,%%r12,4)\
+  SAVE_SOLUTION_m8n2(10,11,192)
+
+#define SOLVE_RN_m8n12 \
+  "movq %2,%3;" GEMM_SUM_REORDER_8x4(4,5,6,7,63) GEMM_SUM_REORDER_8x4(8,9,10,11,63) GEMM_SUM_REORDER_8x4(12,13,14,15,63) "movq %2,%3; addq $32,%2;"\
+  SOLVE_leri_m8n2(0,4,5,%1) SUBTRACT_m8n2(8,6,7,%1) SUBTRACT_m8n2(0,8,9,%1,%%r12,4) SUBTRACT_m8n2(8,10,11,%1,%%r12,4) SUBTRACT_m8n2(0,12,13,%1,%%r12,8) SUBTRACT_m8n2(8,14,15,%1,%%r12,8)\
+  SOLVE_ri_m8n2(16,4,5,%1) SUBTRACT_m8n2(24,6,7,%1) SUBTRACT_m8n2(16,8,9,%1,%%r12,4) SUBTRACT_m8n2(24,10,11,%1,%%r12,4) SUBTRACT_m8n2(16,12,13,%1,%%r12,8) SUBTRACT_m8n2(24,14,15,%1,%%r12,8)\
+  SAVE_SOLUTION_m8n2(4,5,0)\
+  SOLVE_leri_m8n2(40,6,7,%1) SUBTRACT_m8n2(32,8,9,%1,%%r12,4) SUBTRACT_m8n2(40,10,11,%1,%%r12,4) SUBTRACT_m8n2(32,12,13,%1,%%r12,8) SUBTRACT_m8n2(40,14,15,%1,%%r12,8)\
+  SOLVE_ri_m8n2(56,6,7,%1) SUBTRACT_m8n2(48,8,9,%1,%%r12,4) SUBTRACT_m8n2(56,10,11,%1,%%r12,4) SUBTRACT_m8n2(48,12,13,%1,%%r12,8) SUBTRACT_m8n2(56,14,15,%1,%%r12,8)\
+  SAVE_SOLUTION_m8n2(6,7,64)\
+  SOLVE_leri_m8n2(64,8,9,%1,%%r12,4) SUBTRACT_m8n2(72,10,11,%1,%%r12,4) SUBTRACT_m8n2(64,12,13,%1,%%r12,8) SUBTRACT_m8n2(72,14,15,%1,%%r12,8)\
+  SOLVE_ri_m8n2(80,8,9,%1,%%r12,4) SUBTRACT_m8n2(88,10,11,%1,%%r12,4) SUBTRACT_m8n2(80,12,13,%1,%%r12,8) SUBTRACT_m8n2(88,14,15,%1,%%r12,8)\
+  SAVE_SOLUTION_m8n2(8,9,128)\
+  SOLVE_leri_m8n2(104,10,11,%1,%%r12,4) SUBTRACT_m8n2(96,12,13,%1,%%r12,8) SUBTRACT_m8n2(104,14,15,%1,%%r12,8)\
+  SOLVE_ri_m8n2(120,10,11,%1,%%r12,4) SUBTRACT_m8n2(112,12,13,%1,%%r12,8) SUBTRACT_m8n2(120,14,15,%1,%%r12,8)\
+  SAVE_SOLUTION_m8n2(10,11,192)\
+  SOLVE_leri_m8n2(128,12,13,%1,%%r12,8) SUBTRACT_m8n2(136,14,15,%1,%%r12,8)\
+  SOLVE_ri_m8n2(144,12,13,%1,%%r12,8) SUBTRACT_m8n2(152,14,15,%1,%%r12,8)\
+  SAVE_SOLUTION_m8n2(12,13,256)\
+  SOLVE_leri_m8n2(168,14,15,%1,%%r12,8)\
+  SOLVE_ri_m8n2(184,14,15,%1,%%r12,8)\
+  SAVE_SOLUTION_m8n2(14,15,320)
+
+#define SOLVE_RN_m4n4 \
+  "movq %2,%3;" GEMM_SUM_REORDER_4x4(4,5,6,7,4,5) "movq %2,%3; addq $16,%2;"\
+  SOLVE_leri_m4n2(0,4,%1) SUBTRACT_m4n2(8,5,%1)\
+  SOLVE_ri_m4n2(16,4,%1) SUBTRACT_m4n2(24,5,%1)\
+  SAVE_SOLUTION_m4n2(4,0)\
+  SOLVE_leri_m4n2(40,5,%1)\
+  SOLVE_ri_m4n2(56,5,%1)\
+  SAVE_SOLUTION_m4n2(5,32)
+
+#define SOLVE_RN_m4n8 \
+  "movq %2,%3;" GEMM_SUM_REORDER_4x4(4,5,6,7,4,5) GEMM_SUM_REORDER_4x4(8,9,10,11,6,7) "movq %2,%3; addq $16,%2;"\
+  SOLVE_leri_m4n2(0,4,%1) SUBTRACT_m4n2(8,5,%1) SUBTRACT_m4n2(0,6,%1,%%r12,4) SUBTRACT_m4n2(8,7,%1,%%r12,4)\
+  SOLVE_ri_m4n2(16,4,%1) SUBTRACT_m4n2(24,5,%1) SUBTRACT_m4n2(16,6,%1,%%r12,4) SUBTRACT_m4n2(24,7,%1,%%r12,4)\
+  SAVE_SOLUTION_m4n2(4,0)\
+  SOLVE_leri_m4n2(40,5,%1) SUBTRACT_m4n2(32,6,%1,%%r12,4) SUBTRACT_m4n2(40,7,%1,%%r12,4)\
+  SOLVE_ri_m4n2(56,5,%1) SUBTRACT_m4n2(48,6,%1,%%r12,4) SUBTRACT_m4n2(56,7,%1,%%r12,4)\
+  SAVE_SOLUTION_m4n2(5,32)\
+  SOLVE_leri_m4n2(64,6,%1,%%r12,4) SUBTRACT_m4n2(72,7,%1,%%r12,4)\
+  SOLVE_ri_m4n2(80,6,%1,%%r12,4) SUBTRACT_m4n2(88,7,%1,%%r12,4)\
+  SAVE_SOLUTION_m4n2(6,64)\
+  SOLVE_leri_m4n2(104,7,%1,%%r12,4)\
+  SOLVE_ri_m4n2(120,7,%1,%%r12,4)\
+  SAVE_SOLUTION_m4n2(7,96)
+
+#define SOLVE_RN_m4n12 \
+  "movq %2,%3;" GEMM_SUM_REORDER_4x4(4,5,6,7,4,5) GEMM_SUM_REORDER_4x4(8,9,10,11,6,7) GEMM_SUM_REORDER_4x4(12,13,14,15,8,9) "movq %2,%3; addq $16,%2;"\
+  SOLVE_leri_m4n2(0,4,%1) SUBTRACT_m4n2(8,5,%1) SUBTRACT_m4n2(0,6,%1,%%r12,4) SUBTRACT_m4n2(8,7,%1,%%r12,4) SUBTRACT_m4n2(0,8,%1,%%r12,8) SUBTRACT_m4n2(8,9,%1,%%r12,8)\
+  SOLVE_ri_m4n2(16,4,%1) SUBTRACT_m4n2(24,5,%1) SUBTRACT_m4n2(16,6,%1,%%r12,4) SUBTRACT_m4n2(24,7,%1,%%r12,4) SUBTRACT_m4n2(16,8,%1,%%r12,8) SUBTRACT_m4n2(24,9,%1,%%r12,8)\
+  SAVE_SOLUTION_m4n2(4,0)\
+  SOLVE_leri_m4n2(40,5,%1) SUBTRACT_m4n2(32,6,%1,%%r12,4) SUBTRACT_m4n2(40,7,%1,%%r12,4) SUBTRACT_m4n2(32,8,%1,%%r12,8) SUBTRACT_m4n2(40,9,%1,%%r12,8)\
+  SOLVE_ri_m4n2(56,5,%1) SUBTRACT_m4n2(48,6,%1,%%r12,4) SUBTRACT_m4n2(56,7,%1,%%r12,4) SUBTRACT_m4n2(48,8,%1,%%r12,8) SUBTRACT_m4n2(56,9,%1,%%r12,8)\
+  SAVE_SOLUTION_m4n2(5,32)\
+  SOLVE_leri_m4n2(64,6,%1,%%r12,4) SUBTRACT_m4n2(72,7,%1,%%r12,4) SUBTRACT_m4n2(64,8,%1,%%r12,8) SUBTRACT_m4n2(72,9,%1,%%r12,8)\
+  SOLVE_ri_m4n2(80,6,%1,%%r12,4) SUBTRACT_m4n2(88,7,%1,%%r12,4) SUBTRACT_m4n2(80,8,%1,%%r12,8) SUBTRACT_m4n2(88,9,%1,%%r12,8)\
+  SAVE_SOLUTION_m4n2(6,64)\
+  SOLVE_leri_m4n2(104,7,%1,%%r12,4) SUBTRACT_m4n2(96,8,%1,%%r12,8) SUBTRACT_m4n2(104,9,%1,%%r12,8)\
+  SOLVE_ri_m4n2(120,7,%1,%%r12,4) SUBTRACT_m4n2(112,8,%1,%%r12,8) SUBTRACT_m4n2(120,9,%1,%%r12,8)\
+  SAVE_SOLUTION_m4n2(7,96)\
+  SOLVE_leri_m4n2(128,8,%1,%%r12,8) SUBTRACT_m4n2(136,9,%1,%%r12,8)\
+  SOLVE_ri_m4n2(144,8,%1,%%r12,8) SUBTRACT_m4n2(152,9,%1,%%r12,8)\
+  SAVE_SOLUTION_m4n2(8,128)\
+  SOLVE_leri_m4n2(168,9,%1,%%r12,8)\
+  SOLVE_ri_m4n2(184,9,%1,%%r12,8)\
+  SAVE_SOLUTION_m4n2(9,160)
+
+#define SOLVE_RN_m2n4 \
+  "movq %2,%3;" GEMM_SUM_REORDER_2x4(4,5) "movq %2,%3; addq $8,%2;"\
+  SOLVE_col1_ltor_m2n4(0,4,5,%1)\
+  SOLVE_col2_ltor_m2n4(16,4,5,%1)\
+  SOLVE_col3_ltor_m2n4(32,4,5,%1)\
+  SOLVE_col4_ltor_m2n4(48,4,5,%1)\
+  SAVE_SOLUTION_m2n4(4,5,0)
+
+#define SOLVE_RN_m2n8 \
+  "movq %2,%3;" GEMM_SUM_REORDER_2x4(4,5) GEMM_SUM_REORDER_2x4(6,7) "movq %2,%3; addq $8,%2;"\
+  SOLVE_col1_ltor_m2n4(0,4,5,%1) SUBTRACT_m2n4(0,6,7,%1,%%r12,4)\
+  SOLVE_col2_ltor_m2n4(16,4,5,%1) SUBTRACT_m2n4(16,6,7,%1,%%r12,4)\
+  SOLVE_col3_ltor_m2n4(32,4,5,%1) SUBTRACT_m2n4(32,6,7,%1,%%r12,4)\
+  SOLVE_col4_ltor_m2n4(48,4,5,%1) SUBTRACT_m2n4(48,6,7,%1,%%r12,4)\
+  SAVE_SOLUTION_m2n4(4,5,0)\
+  SOLVE_col1_ltor_m2n4(64,6,7,%1,%%r12,4)\
+  SOLVE_col2_ltor_m2n4(80,6,7,%1,%%r12,4)\
+  SOLVE_col3_ltor_m2n4(96,6,7,%1,%%r12,4)\
+  SOLVE_col4_ltor_m2n4(112,6,7,%1,%%r12,4)\
+  SAVE_SOLUTION_m2n4(6,7,32)
+
+#define SOLVE_RN_m2n12 \
+  "movq %2,%3;" GEMM_SUM_REORDER_2x4(4,5) GEMM_SUM_REORDER_2x4(6,7) GEMM_SUM_REORDER_2x4(8,9) "movq %2,%3; addq $8,%2;"\
+  SOLVE_col1_ltor_m2n4(0,4,5,%1) SUBTRACT_m2n4(0,6,7,%1,%%r12,4) SUBTRACT_m2n4(0,8,9,%1,%%r12,8)\
+  SOLVE_col2_ltor_m2n4(16,4,5,%1) SUBTRACT_m2n4(16,6,7,%1,%%r12,4) SUBTRACT_m2n4(16,8,9,%1,%%r12,8)\
+  SOLVE_col3_ltor_m2n4(32,4,5,%1) SUBTRACT_m2n4(32,6,7,%1,%%r12,4) SUBTRACT_m2n4(32,8,9,%1,%%r12,8)\
+  SOLVE_col4_ltor_m2n4(48,4,5,%1) SUBTRACT_m2n4(48,6,7,%1,%%r12,4) SUBTRACT_m2n4(48,8,9,%1,%%r12,8)\
+  SAVE_SOLUTION_m2n4(4,5,0)\
+  SOLVE_col1_ltor_m2n4(64,6,7,%1,%%r12,4) SUBTRACT_m2n4(64,8,9,%1,%%r12,8)\
+  SOLVE_col2_ltor_m2n4(80,6,7,%1,%%r12,4) SUBTRACT_m2n4(80,8,9,%1,%%r12,8)\
+  SOLVE_col3_ltor_m2n4(96,6,7,%1,%%r12,4) SUBTRACT_m2n4(96,8,9,%1,%%r12,8)\
+  SOLVE_col4_ltor_m2n4(112,6,7,%1,%%r12,4) SUBTRACT_m2n4(112,8,9,%1,%%r12,8)\
+  SAVE_SOLUTION_m2n4(6,7,32)\
+  SOLVE_col1_ltor_m2n4(128,8,9,%1,%%r12,8)\
+  SOLVE_col2_ltor_m2n4(144,8,9,%1,%%r12,8)\
+  SOLVE_col3_ltor_m2n4(160,8,9,%1,%%r12,8)\
+  SOLVE_col4_ltor_m2n4(176,8,9,%1,%%r12,8)\
+  SAVE_SOLUTION_m2n4(8,9,64)
+
+#define SOLVE_RN_m1n4 \
+  "movq %2,%3;" GEMM_SUM_REORDER_1x4(4) "movq %2,%3; addq $4,%2;"\
+  SOLVE_col1_ltor_m1n4(0,4,%1)\
+  SOLVE_col2_ltor_m1n4(16,4,%1)\
+  SOLVE_col3_ltor_m1n4(32,4,%1)\
+  SOLVE_col4_ltor_m1n4(48,4,%1)\
+  SAVE_SOLUTION_m1n4(4,0)
+
+#define SOLVE_RN_m1n8 \
+  "movq %2,%3;" GEMM_SUM_REORDER_1x4(4) GEMM_SUM_REORDER_1x4(5) "movq %2,%3; addq $4,%2;"\
+  SOLVE_col1_ltor_m1n4(0,4,%1) SUBTRACT_m1n4(0,5,%1,%%r12,4)\
+  SOLVE_col2_ltor_m1n4(16,4,%1) SUBTRACT_m1n4(16,5,%1,%%r12,4)\
+  SOLVE_col3_ltor_m1n4(32,4,%1) SUBTRACT_m1n4(32,5,%1,%%r12,4)\
+  SOLVE_col4_ltor_m1n4(48,4,%1) SUBTRACT_m1n4(48,5,%1,%%r12,4)\
+  SAVE_SOLUTION_m1n4(4,0)\
+  SOLVE_col1_ltor_m1n4(64,5,%1,%%r12,4)\
+  SOLVE_col2_ltor_m1n4(80,5,%1,%%r12,4)\
+  SOLVE_col3_ltor_m1n4(96,5,%1,%%r12,4)\
+  SOLVE_col4_ltor_m1n4(112,5,%1,%%r12,4)\
+  SAVE_SOLUTION_m1n4(5,16)
+
+#define SOLVE_RN_m1n12 \
+  "movq %2,%3;" GEMM_SUM_REORDER_1x4(4) GEMM_SUM_REORDER_1x4(5) GEMM_SUM_REORDER_1x4(6) "movq %2,%3; addq $4,%2;"\
+  SOLVE_col1_ltor_m1n4(0,4,%1) SUBTRACT_m1n4(0,5,%1,%%r12,4) SUBTRACT_m1n4(0,6,%1,%%r12,8)\
+  SOLVE_col2_ltor_m1n4(16,4,%1) SUBTRACT_m1n4(16,5,%1,%%r12,4) SUBTRACT_m1n4(16,6,%1,%%r12,8)\
+  SOLVE_col3_ltor_m1n4(32,4,%1) SUBTRACT_m1n4(32,5,%1,%%r12,4) SUBTRACT_m1n4(32,6,%1,%%r12,8)\
+  SOLVE_col4_ltor_m1n4(48,4,%1) SUBTRACT_m1n4(48,5,%1,%%r12,4) SUBTRACT_m1n4(48,6,%1,%%r12,8)\
+  SAVE_SOLUTION_m1n4(4,0)\
+  SOLVE_col1_ltor_m1n4(64,5,%1,%%r12,4) SUBTRACT_m1n4(64,6,%1,%%r12,8)\
+  SOLVE_col2_ltor_m1n4(80,5,%1,%%r12,4) SUBTRACT_m1n4(80,6,%1,%%r12,8)\
+  SOLVE_col3_ltor_m1n4(96,5,%1,%%r12,4) SUBTRACT_m1n4(96,6,%1,%%r12,8)\
+  SOLVE_col4_ltor_m1n4(112,5,%1,%%r12,4) SUBTRACT_m1n4(112,6,%1,%%r12,8)\
+  SAVE_SOLUTION_m1n4(5,16)\
+  SOLVE_col1_ltor_m1n4(128,6,%1,%%r12,8)\
+  SOLVE_col2_ltor_m1n4(144,6,%1,%%r12,8)\
+  SOLVE_col3_ltor_m1n4(160,6,%1,%%r12,8)\
+  SOLVE_col4_ltor_m1n4(176,6,%1,%%r12,8)\
+  SAVE_SOLUTION_m1n4(6,32)
+
+#define GEMM_RN_SIMPLE(mdim,ndim) \
+  "movq %%r15,%0; leaq (%%r15,%%r12,"#mdim"),%%r15; movq %%r13,%5; movq %%r14,%1;" INIT_m##mdim##n##ndim\
+  "testq %5,%5; jz 1"#mdim""#ndim"2f;"\
+  "1"#mdim""#ndim"1:\n\t"\
+  GEMM_KERNEL_k1m##mdim##n##ndim "addq $16,%1; addq $"#mdim"*4,%0; decq %5; jnz 1"#mdim""#ndim"1b;"\
+  "1"#mdim""#ndim"2:\n\t"
+#define GEMM_RN_m8n4 GEMM_RN_SIMPLE(8,4)
+#define GEMM_RN_m8n8 GEMM_RN_SIMPLE(8,8)
+#define GEMM_RN_m8n12 \
+  "movq %%r15,%0; leaq (%%r15,%%r12,8),%%r15; movq %%r13,%5; movq %%r14,%1;" INIT_m8n12\
+  "cmpq $8,%5; jb 18122f;"\
+  "18121:\n\t"\
+  GEMM_KERNEL_k1m8n12 "prefetcht0 384(%0); addq $32,%0; addq $16,%1;"\
+  GEMM_KERNEL_k1m8n12 "addq $32,%0; addq $16,%1;"\
+  GEMM_KERNEL_k1m8n12 "prefetcht0 384(%0); addq $32,%0; addq $16,%1;"\
+  GEMM_KERNEL_k1m8n12 "addq $32,%0; addq $16,%1;"\
+  GEMM_KERNEL_k1m8n12 "prefetcht0 384(%0); addq $32,%0; addq $16,%1;"\
+  GEMM_KERNEL_k1m8n12 "addq $32,%0; addq $16,%1;"\
+  GEMM_KERNEL_k1m8n12 "prefetcht0 384(%0); addq $32,%0; addq $16,%1;"\
+  GEMM_KERNEL_k1m8n12 "addq $32,%0; addq $16,%1;"\
+  "subq $8,%5; cmpq $8,%5; jnb 18121b;"\
+  "18122:\n\t"\
+  "testq %5,%5; jz 18124f;"\
+  "18123:\n\t"\
+  GEMM_KERNEL_k1m8n12 "addq $32,%0; addq $16,%1; decq %5; jnz 18123b;"\
+  "18124:\n\t"
+#define GEMM_RN_m4n4 GEMM_RN_SIMPLE(4,4)
+#define GEMM_RN_m4n8 GEMM_RN_SIMPLE(4,8)
+#define GEMM_RN_m4n12 GEMM_RN_SIMPLE(4,12)
+#define GEMM_RN_m2n4 GEMM_RN_SIMPLE(2,4)
+#define GEMM_RN_m2n8 GEMM_RN_SIMPLE(2,8)
+#define GEMM_RN_m2n12 GEMM_RN_SIMPLE(2,12)
+#define GEMM_RN_m1n4 GEMM_RN_SIMPLE(1,4)
+#define GEMM_RN_m1n8 GEMM_RN_SIMPLE(1,8)
+#define GEMM_RN_m1n12 GEMM_RN_SIMPLE(1,12)
+
+#define COMPUTE(ndim) {\
+  __asm__ __volatile__(\
+    "movq %0,%%r15; movq %1,%%r14; movq %7,%%r13; movq %6,%%r12; salq $2,%%r12; movq %10,%%r11;"\
+    "cmpq $8,%%r11; jb "#ndim"772f;"\
+    #ndim"771:\n\t"\
+    GEMM_RN_m8n##ndim SOLVE_RN_m8n##ndim "subq $8,%%r11; cmpq $8,%%r11; jnb "#ndim"771b;"\
+    #ndim"772:\n\t"\
+    "testq $4,%%r11; jz "#ndim"773f;"\
+    GEMM_RN_m4n##ndim SOLVE_RN_m4n##ndim "subq $4,%%r11;"\
+    #ndim"773:\n\t"\
+    "testq $2,%%r11; jz "#ndim"774f;"\
+    GEMM_RN_m2n##ndim SOLVE_RN_m2n##ndim "subq $2,%%r11;"\
+    #ndim"774:\n\t"\
+    "testq $1,%%r11; jz "#ndim"775f;"\
+    GEMM_RN_m1n##ndim SOLVE_RN_m1n##ndim "subq $1,%%r11;"\
+    #ndim"775:\n\t"\
+    "movq %%r15,%0; movq %%r14,%1; vzeroupper;"\
+  :"+r"(a_ptr),"+r"(b_ptr),"+r"(c_ptr),"+r"(c_tmp),"+r"(ldc_bytes),"+r"(k_cnt):"m"(K),"m"(OFF),"m"(one[0]),"m"(zero[0]),"m"(M)\
+  :"r11","r12","r13","r14","r15","cc","memory",\
+  "xmm0","xmm1","xmm2","xmm3","xmm4","xmm5","xmm6","xmm7","xmm8","xmm9","xmm10","xmm11","xmm12","xmm13","xmm14","xmm15");\
+  a_ptr -= M * K; b_ptr += ndim * K; c_ptr += ldc * ndim - M; OFF += ndim;\
+}
+
+static void solve_RN(BLASLONG m, BLASLONG n, FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc) {
+  FLOAT a0, b0;
+  int i, j, k;
+  for (i=0; i<n; i++) {
+    b0 = b[i*n+i];
+    for (j=0; j<m; j++) {
+      a0 = c[i*ldc+j] * b0;
+      a[i*m+j] = c[i*ldc+j] = a0;
+      for (k=i+1; k<n; k++) c[k*ldc+j] -= a0 * b[i*n+k];
+    }
+  }
+}
+static void COMPUTE_EDGE_1_nchunk(BLASLONG m, BLASLONG n, FLOAT *sa, FLOAT *sb, FLOAT *C, BLASLONG ldc, BLASLONG k, BLASLONG offset) {
+  BLASLONG m_count = m, kk = offset; FLOAT *a_ptr = sa, *c_ptr = C;
+  for(;m_count>7;m_count-=8){
+    if(kk>0) GEMM_KERNEL_N(8,n,kk,-1.0,a_ptr,sb,c_ptr,ldc);
+    solve_RN(8,n,a_ptr+kk*8,sb+kk*n,c_ptr,ldc);
+    a_ptr += k * 8; c_ptr += 8;
+  }
+  for(;m_count>3;m_count-=4){
+    if(kk>0) GEMM_KERNEL_N(4,n,kk,-1.0,a_ptr,sb,c_ptr,ldc);
+    solve_RN(4,n,a_ptr+kk*4,sb+kk*n,c_ptr,ldc);
+    a_ptr += k * 4; c_ptr += 4;
+  }
+  for(;m_count>1;m_count-=2){
+    if(kk>0) GEMM_KERNEL_N(2,n,kk,-1.0,a_ptr,sb,c_ptr,ldc);
+    solve_RN(2,n,a_ptr+kk*2,sb+kk*n,c_ptr,ldc);
+    a_ptr += k * 2; c_ptr += 2;
+  }
+  if(m_count>0){
+    if(kk>0) GEMM_KERNEL_N(1,n,kk,-1.0,a_ptr,sb,c_ptr,ldc);
+    solve_RN(1,n,a_ptr+kk*1,sb+kk*n,c_ptr,ldc);
+    a_ptr += k * 1; c_ptr += 1;
+  }
+}
+int CNAME(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT dummy1, FLOAT *sa, FLOAT *sb, FLOAT *C, BLASLONG ldc, BLASLONG offset){
+  float *a_ptr = sa, *b_ptr = sb, *c_ptr = C, *c_tmp = C;
+  float one[8] = {1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0};
+  float zero[8] = {0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0};
+  uint64_t ldc_bytes = (uint64_t)ldc * sizeof(float), K = (uint64_t)k, M = (uint64_t)m, OFF = (uint64_t)-offset, k_cnt = 0;
+  BLASLONG n_count = n;
+  for(;n_count>11;n_count-=12) COMPUTE(12)
+  for(;n_count>7;n_count-=8) COMPUTE(8)
+  for(;n_count>3;n_count-=4) COMPUTE(4)
+  for(;n_count>1;n_count-=2) { COMPUTE_EDGE_1_nchunk(m,2,a_ptr,b_ptr,c_ptr,ldc,k,OFF); b_ptr += 2*k; c_ptr += ldc*2; OFF+=2;}
+  if(n_count>0) COMPUTE_EDGE_1_nchunk(m,1,a_ptr,b_ptr,c_ptr,ldc,k,OFF);
+  return 0;
+}
diff --git a/kernel/x86_64/strsm_kernel_8x4_haswell_RT.c b/kernel/x86_64/strsm_kernel_8x4_haswell_RT.c
index ffcbfbbf0..9de3354de 100644
--- a/kernel/x86_64/strsm_kernel_8x4_haswell_RT.c
+++ b/kernel/x86_64/strsm_kernel_8x4_haswell_RT.c
@@ -1,281 +1,281 @@
-#include "common.h"
-#include <stdint.h>
-#include "strsm_kernel_8x4_haswell_R_common.h"
-
-#define SOLVE_RT_m8n4 \
-  "movq %2,%3;" GEMM_SUM_REORDER_8x4(4,5,6,7,63) "negq %4; leaq (%3,%4,2),%3; negq %4; addq $32,%2;"\
-  SOLVE_rile_m8n2(-8,6,7,%1) SUBTRACT_m8n2(-16,4,5,%1)\
-  SOLVE_le_m8n2(-24,6,7,%1) SUBTRACT_m8n2(-32,4,5,%1)\
-  SAVE_SOLUTION_m8n2(6,7,-64) "negq %4; leaq (%3,%4,4),%3; negq %4;"\
-  SOLVE_rile_m8n2(-48,4,5,%1)\
-  SOLVE_le_m8n2(-64,4,5,%1)\
-  SAVE_SOLUTION_m8n2(4,5,-128)
-
-#define SOLVE_RT_m8n8 \
-  "movq %2,%3;" GEMM_SUM_REORDER_8x4(4,5,6,7,63) GEMM_SUM_REORDER_8x4(8,9,10,11,63) "negq %4; leaq (%3,%4,2),%3; negq %4; addq $32,%2;"\
-  SOLVE_rile_m8n2(-8,10,11,%1,%%r12,4) SUBTRACT_m8n2(-16,8,9,%1,%%r12,4) SUBTRACT_m8n2(-8,6,7,%1) SUBTRACT_m8n2(-16,4,5,%1)\
-  SOLVE_le_m8n2(-24,10,11,%1,%%r12,4) SUBTRACT_m8n2(-32,8,9,%1,%%r12,4) SUBTRACT_m8n2(-24,6,7,%1) SUBTRACT_m8n2(-32,4,5,%1)\
-  SAVE_SOLUTION_m8n2(10,11,-64) "negq %4; leaq (%3,%4,4),%3; negq %4;"\
-  SOLVE_rile_m8n2(-48,8,9,%1,%%r12,4) SUBTRACT_m8n2(-40,6,7,%1) SUBTRACT_m8n2(-48,4,5,%1)\
-  SOLVE_le_m8n2(-64,8,9,%1,%%r12,4) SUBTRACT_m8n2(-56,6,7,%1) SUBTRACT_m8n2(-64,4,5,%1)\
-  SAVE_SOLUTION_m8n2(8,9,-128) "negq %4; leaq (%3,%4,4),%3; negq %4;"\
-  SOLVE_rile_m8n2(-72,6,7,%1) SUBTRACT_m8n2(-80,4,5,%1)\
-  SOLVE_le_m8n2(-88,6,7,%1) SUBTRACT_m8n2(-96,4,5,%1)\
-  SAVE_SOLUTION_m8n2(6,7,-192) "negq %4; leaq (%3,%4,4),%3; negq %4;"\
-  SOLVE_rile_m8n2(-112,4,5,%1)\
-  SOLVE_le_m8n2(-128,4,5,%1)\
-  SAVE_SOLUTION_m8n2(4,5,-256)
-
-#define SOLVE_RT_m8n12 \
-  "movq %2,%3;" GEMM_SUM_REORDER_8x4(4,5,6,7,63) GEMM_SUM_REORDER_8x4(8,9,10,11,63) GEMM_SUM_REORDER_8x4(12,13,14,15,63) "negq %4; leaq (%3,%4,2),%3; negq %4; addq $32,%2;"\
-  SOLVE_rile_m8n2(-8,14,15,%1,%%r12,8) SUBTRACT_m8n2(-16,12,13,%1,%%r12,8) SUBTRACT_m8n2(-8,10,11,%1,%%r12,4) SUBTRACT_m8n2(-16,8,9,%1,%%r12,4) SUBTRACT_m8n2(-8,6,7,%1) SUBTRACT_m8n2(-16,4,5,%1)\
-  SOLVE_le_m8n2(-24,14,15,%1,%%r12,8) SUBTRACT_m8n2(-32,12,13,%1,%%r12,8) SUBTRACT_m8n2(-24,10,11,%1,%%r12,4) SUBTRACT_m8n2(-32,8,9,%1,%%r12,4) SUBTRACT_m8n2(-24,6,7,%1) SUBTRACT_m8n2(-32,4,5,%1)\
-  SAVE_SOLUTION_m8n2(14,15,-64) "negq %4; leaq (%3,%4,4),%3; negq %4;"\
-  SOLVE_rile_m8n2(-48,12,13,%1,%%r12,8) SUBTRACT_m8n2(-40,10,11,%1,%%r12,4) SUBTRACT_m8n2(-48,8,9,%1,%%r12,4) SUBTRACT_m8n2(-40,6,7,%1) SUBTRACT_m8n2(-48,4,5,%1)\
-  SOLVE_le_m8n2(-64,12,13,%1,%%r12,8) SUBTRACT_m8n2(-56,10,11,%1,%%r12,4) SUBTRACT_m8n2(-64,8,9,%1,%%r12,4) SUBTRACT_m8n2(-56,6,7,%1) SUBTRACT_m8n2(-64,4,5,%1)\
-  SAVE_SOLUTION_m8n2(12,13,-128) "negq %4; leaq (%3,%4,4),%3; negq %4;"\
-  SOLVE_rile_m8n2(-72,10,11,%1,%%r12,4) SUBTRACT_m8n2(-80,8,9,%1,%%r12,4) SUBTRACT_m8n2(-72,6,7,%1) SUBTRACT_m8n2(-80,4,5,%1)\
-  SOLVE_le_m8n2(-88,10,11,%1,%%r12,4) SUBTRACT_m8n2(-96,8,9,%1,%%r12,4) SUBTRACT_m8n2(-88,6,7,%1) SUBTRACT_m8n2(-96,4,5,%1)\
-  SAVE_SOLUTION_m8n2(10,11,-192) "negq %4; leaq (%3,%4,4),%3; negq %4;"\
-  SOLVE_rile_m8n2(-112,8,9,%1,%%r12,4) SUBTRACT_m8n2(-104,6,7,%1) SUBTRACT_m8n2(-112,4,5,%1)\
-  SOLVE_le_m8n2(-128,8,9,%1,%%r12,4) SUBTRACT_m8n2(-120,6,7,%1) SUBTRACT_m8n2(-128,4,5,%1)\
-  SAVE_SOLUTION_m8n2(8,9,-256) "negq %4; leaq (%3,%4,4),%3; negq %4;"\
-  SOLVE_rile_m8n2(-136,6,7,%1) SUBTRACT_m8n2(-144,4,5,%1)\
-  SOLVE_le_m8n2(-152,6,7,%1) SUBTRACT_m8n2(-160,4,5,%1)\
-  SAVE_SOLUTION_m8n2(6,7,-320) "negq %4; leaq (%3,%4,4),%3; negq %4;"\
-  SOLVE_rile_m8n2(-176,4,5,%1)\
-  SOLVE_le_m8n2(-192,4,5,%1)\
-  SAVE_SOLUTION_m8n2(4,5,-384)
-
-#define SOLVE_RT_m4n4 \
-  "movq %2,%3;" GEMM_SUM_REORDER_4x4(4,5,6,7,4,5) "negq %4; leaq (%3,%4,2),%3; negq %4; addq $16,%2;"\
-  SOLVE_rile_m4n2(-8,5,%1) SUBTRACT_m4n2(-16,4,%1)\
-  SOLVE_le_m4n2(-24,5,%1) SUBTRACT_m4n2(-32,4,%1)\
-  SAVE_SOLUTION_m4n2(5,-32) "negq %4; leaq (%3,%4,4),%3; negq %4;"\
-  SOLVE_rile_m4n2(-48,4,%1)\
-  SOLVE_le_m4n2(-64,4,%1)\
-  SAVE_SOLUTION_m4n2(4,-64)
-
-#define SOLVE_RT_m4n8 \
-  "movq %2,%3;" GEMM_SUM_REORDER_4x4(4,5,6,7,4,5) GEMM_SUM_REORDER_4x4(8,9,10,11,6,7) "negq %4; leaq (%3,%4,2),%3; negq %4; addq $16,%2;"\
-  SOLVE_rile_m4n2(-8,7,%1,%%r12,4) SUBTRACT_m4n2(-16,6,%1,%%r12,4) SUBTRACT_m4n2(-8,5,%1) SUBTRACT_m4n2(-16,4,%1)\
-  SOLVE_le_m4n2(-24,7,%1,%%r12,4) SUBTRACT_m4n2(-32,6,%1,%%r12,4) SUBTRACT_m4n2(-24,5,%1) SUBTRACT_m4n2(-32,4,%1)\
-  SAVE_SOLUTION_m4n2(7,-32) "negq %4; leaq (%3,%4,4),%3; negq %4;"\
-  SOLVE_rile_m4n2(-48,6,%1,%%r12,4) SUBTRACT_m4n2(-40,5,%1) SUBTRACT_m4n2(-48,4,%1)\
-  SOLVE_le_m4n2(-64,6,%1,%%r12,4) SUBTRACT_m4n2(-56,5,%1) SUBTRACT_m4n2(-64,4,%1)\
-  SAVE_SOLUTION_m4n2(6,-64) "negq %4; leaq (%3,%4,4),%3; negq %4;"\
-  SOLVE_rile_m4n2(-72,5,%1) SUBTRACT_m4n2(-80,4,%1)\
-  SOLVE_le_m4n2(-88,5,%1) SUBTRACT_m4n2(-96,4,%1)\
-  SAVE_SOLUTION_m4n2(5,-96) "negq %4; leaq (%3,%4,4),%3; negq %4;"\
-  SOLVE_rile_m4n2(-112,4,%1)\
-  SOLVE_le_m4n2(-128,4,%1)\
-  SAVE_SOLUTION_m4n2(4,-128)
-
-#define SOLVE_RT_m4n12 \
-  "movq %2,%3;" GEMM_SUM_REORDER_4x4(4,5,6,7,4,5) GEMM_SUM_REORDER_4x4(8,9,10,11,6,7) GEMM_SUM_REORDER_4x4(12,13,14,15,8,9) "negq %4; leaq (%3,%4,2),%3; negq %4; addq $16,%2;"\
-  SOLVE_rile_m4n2(-8,9,%1,%%r12,8) SUBTRACT_m4n2(-16,8,%1,%%r12,8) SUBTRACT_m4n2(-8,7,%1,%%r12,4) SUBTRACT_m4n2(-16,6,%1,%%r12,4) SUBTRACT_m4n2(-8,5,%1) SUBTRACT_m4n2(-16,4,%1)\
-  SOLVE_le_m4n2(-24,9,%1,%%r12,8) SUBTRACT_m4n2(-32,8,%1,%%r12,8) SUBTRACT_m4n2(-24,7,%1,%%r12,4) SUBTRACT_m4n2(-32,6,%1,%%r12,4) SUBTRACT_m4n2(-24,5,%1) SUBTRACT_m4n2(-32,4,%1)\
-  SAVE_SOLUTION_m4n2(9,-32) "negq %4; leaq (%3,%4,4),%3; negq %4;"\
-  SOLVE_rile_m4n2(-48,8,%1,%%r12,8) SUBTRACT_m4n2(-40,7,%1,%%r12,4) SUBTRACT_m4n2(-48,6,%1,%%r12,4) SUBTRACT_m4n2(-40,5,%1) SUBTRACT_m4n2(-48,4,%1)\
-  SOLVE_le_m4n2(-64,8,%1,%%r12,8) SUBTRACT_m4n2(-56,7,%1,%%r12,4) SUBTRACT_m4n2(-64,6,%1,%%r12,4) SUBTRACT_m4n2(-56,5,%1) SUBTRACT_m4n2(-64,4,%1)\
-  SAVE_SOLUTION_m4n2(8,-64) "negq %4; leaq (%3,%4,4),%3; negq %4;"\
-  SOLVE_rile_m4n2(-72,7,%1,%%r12,4) SUBTRACT_m4n2(-80,6,%1,%%r12,4) SUBTRACT_m4n2(-72,5,%1) SUBTRACT_m4n2(-80,4,%1)\
-  SOLVE_le_m4n2(-88,7,%1,%%r12,4) SUBTRACT_m4n2(-96,6,%1,%%r12,4) SUBTRACT_m4n2(-88,5,%1) SUBTRACT_m4n2(-96,4,%1)\
-  SAVE_SOLUTION_m4n2(7,-96) "negq %4; leaq (%3,%4,4),%3; negq %4;"\
-  SOLVE_rile_m4n2(-112,6,%1,%%r12,4) SUBTRACT_m4n2(-104,5,%1) SUBTRACT_m4n2(-112,4,%1)\
-  SOLVE_le_m4n2(-128,6,%1,%%r12,4) SUBTRACT_m4n2(-120,5,%1) SUBTRACT_m4n2(-128,4,%1)\
-  SAVE_SOLUTION_m4n2(6,-128) "negq %4; leaq (%3,%4,4),%3; negq %4;"\
-  SOLVE_rile_m4n2(-136,5,%1) SUBTRACT_m4n2(-144,4,%1)\
-  SOLVE_le_m4n2(-152,5,%1) SUBTRACT_m4n2(-160,4,%1)\
-  SAVE_SOLUTION_m4n2(5,-160) "negq %4; leaq (%3,%4,4),%3; negq %4;"\
-  SOLVE_rile_m4n2(-176,4,%1)\
-  SOLVE_le_m4n2(-192,4,%1)\
-  SAVE_SOLUTION_m4n2(4,-192)
-
-#define SOLVE_RT_m2n4 \
-  "movq %2,%3;" GEMM_SUM_REORDER_2x4(4,5) "negq %4; leaq (%3,%4,4),%3; negq %4; addq $8,%2;"\
-  SOLVE_col4_rtol_m2n4(-16,4,5,%1)\
-  SOLVE_col3_rtol_m2n4(-32,4,5,%1)\
-  SOLVE_col2_rtol_m2n4(-48,4,5,%1)\
-  SOLVE_col1_rtol_m2n4(-64,4,5,%1)\
-  SAVE_SOLUTION_m2n4(4,5,-32)
-
-#define SOLVE_RT_m2n8 \
-  "movq %2,%3;" GEMM_SUM_REORDER_2x4(4,5) GEMM_SUM_REORDER_2x4(6,7) "negq %4; leaq (%3,%4,4),%3; negq %4; addq $8,%2;"\
-  SOLVE_col4_rtol_m2n4(-16,6,7,%1,%%r12,4) SUBTRACT_m2n4(-16,4,5,%1)\
-  SOLVE_col3_rtol_m2n4(-32,6,7,%1,%%r12,4) SUBTRACT_m2n4(-32,4,5,%1)\
-  SOLVE_col2_rtol_m2n4(-48,6,7,%1,%%r12,4) SUBTRACT_m2n4(-48,4,5,%1)\
-  SOLVE_col1_rtol_m2n4(-64,6,7,%1,%%r12,4) SUBTRACT_m2n4(-64,4,5,%1)\
-  SAVE_SOLUTION_m2n4(6,7,-32) "negq %4; leaq (%3,%4,8),%3; negq %4;"\
-  SOLVE_col4_rtol_m2n4(-80,4,5,%1)\
-  SOLVE_col3_rtol_m2n4(-96,4,5,%1)\
-  SOLVE_col2_rtol_m2n4(-112,4,5,%1)\
-  SOLVE_col1_rtol_m2n4(-128,4,5,%1)\
-  SAVE_SOLUTION_m2n4(4,5,-64)
-
-#define SOLVE_RT_m2n12 \
-  "movq %2,%3;" GEMM_SUM_REORDER_2x4(4,5) GEMM_SUM_REORDER_2x4(6,7) GEMM_SUM_REORDER_2x4(8,9) "negq %4; leaq (%3,%4,4),%3; negq %4; addq $8,%2;"\
-  SOLVE_col4_rtol_m2n4(-16,8,9,%1,%%r12,8) SUBTRACT_m2n4(-16,6,7,%1,%%r12,4) SUBTRACT_m2n4(-16,4,5,%1)\
-  SOLVE_col3_rtol_m2n4(-32,8,9,%1,%%r12,8) SUBTRACT_m2n4(-32,6,7,%1,%%r12,4) SUBTRACT_m2n4(-32,4,5,%1)\
-  SOLVE_col2_rtol_m2n4(-48,8,9,%1,%%r12,8) SUBTRACT_m2n4(-48,6,7,%1,%%r12,4) SUBTRACT_m2n4(-48,4,5,%1)\
-  SOLVE_col1_rtol_m2n4(-64,8,9,%1,%%r12,8) SUBTRACT_m2n4(-64,6,7,%1,%%r12,4) SUBTRACT_m2n4(-64,4,5,%1)\
-  SAVE_SOLUTION_m2n4(8,9,-32) "negq %4; leaq (%3,%4,8),%3; negq %4;"\
-  SOLVE_col4_rtol_m2n4(-80,6,7,%1,%%r12,4) SUBTRACT_m2n4(-80,4,5,%1)\
-  SOLVE_col3_rtol_m2n4(-96,6,7,%1,%%r12,4) SUBTRACT_m2n4(-96,4,5,%1)\
-  SOLVE_col2_rtol_m2n4(-112,6,7,%1,%%r12,4) SUBTRACT_m2n4(-112,4,5,%1)\
-  SOLVE_col1_rtol_m2n4(-128,6,7,%1,%%r12,4) SUBTRACT_m2n4(-128,4,5,%1)\
-  SAVE_SOLUTION_m2n4(6,7,-64) "negq %4; leaq (%3,%4,8),%3; negq %4;"\
-  SOLVE_col4_rtol_m2n4(-144,4,5,%1)\
-  SOLVE_col3_rtol_m2n4(-160,4,5,%1)\
-  SOLVE_col2_rtol_m2n4(-176,4,5,%1)\
-  SOLVE_col1_rtol_m2n4(-192,4,5,%1)\
-  SAVE_SOLUTION_m2n4(4,5,-96)
-
-#define SOLVE_RT_m1n4 \
-  "movq %2,%3;" GEMM_SUM_REORDER_1x4(4) "negq %4; leaq (%3,%4,4),%3; negq %4; addq $4,%2;"\
-  SOLVE_col4_rtol_m1n4(-16,4,%1)\
-  SOLVE_col3_rtol_m1n4(-32,4,%1)\
-  SOLVE_col2_rtol_m1n4(-48,4,%1)\
-  SOLVE_col1_rtol_m1n4(-64,4,%1)\
-  SAVE_SOLUTION_m1n4(4,-16)
-
-#define SOLVE_RT_m1n8 \
-  "movq %2,%3;" GEMM_SUM_REORDER_1x4(4) GEMM_SUM_REORDER_1x4(5) "negq %4; leaq (%3,%4,4),%3; negq %4; addq $4,%2;"\
-  SOLVE_col4_rtol_m1n4(-16,5,%1,%%r12,4) SUBTRACT_m1n4(-16,4,%1)\
-  SOLVE_col3_rtol_m1n4(-32,5,%1,%%r12,4) SUBTRACT_m1n4(-32,4,%1)\
-  SOLVE_col2_rtol_m1n4(-48,5,%1,%%r12,4) SUBTRACT_m1n4(-48,4,%1)\
-  SOLVE_col1_rtol_m1n4(-64,5,%1,%%r12,4) SUBTRACT_m1n4(-64,4,%1)\
-  SAVE_SOLUTION_m1n4(5,-16) "negq %4; leaq (%3,%4,8),%3; negq %4;"\
-  SOLVE_col4_rtol_m1n4(-80,4,%1)\
-  SOLVE_col3_rtol_m1n4(-96,4,%1)\
-  SOLVE_col2_rtol_m1n4(-112,4,%1)\
-  SOLVE_col1_rtol_m1n4(-128,4,%1)\
-  SAVE_SOLUTION_m1n4(4,-32)
-
-#define SOLVE_RT_m1n12 \
-  "movq %2,%3;" GEMM_SUM_REORDER_1x4(4) GEMM_SUM_REORDER_1x4(5) GEMM_SUM_REORDER_1x4(6) "negq %4; leaq (%3,%4,4),%3; negq %4; addq $4,%2;"\
-  SOLVE_col4_rtol_m1n4(-16,6,%1,%%r12,8) SUBTRACT_m1n4(-16,5,%1,%%r12,4) SUBTRACT_m1n4(-16,4,%1)\
-  SOLVE_col3_rtol_m1n4(-32,6,%1,%%r12,8) SUBTRACT_m1n4(-32,5,%1,%%r12,4) SUBTRACT_m1n4(-32,4,%1)\
-  SOLVE_col2_rtol_m1n4(-48,6,%1,%%r12,8) SUBTRACT_m1n4(-48,5,%1,%%r12,4) SUBTRACT_m1n4(-48,4,%1)\
-  SOLVE_col1_rtol_m1n4(-64,6,%1,%%r12,8) SUBTRACT_m1n4(-64,5,%1,%%r12,4) SUBTRACT_m1n4(-64,4,%1)\
-  SAVE_SOLUTION_m1n4(6,-16) "negq %4; leaq (%3,%4,8),%3; negq %4;"\
-  SOLVE_col4_rtol_m1n4(-80,5,%1,%%r12,4) SUBTRACT_m1n4(-80,4,%1)\
-  SOLVE_col3_rtol_m1n4(-96,5,%1,%%r12,4) SUBTRACT_m1n4(-96,4,%1)\
-  SOLVE_col2_rtol_m1n4(-112,5,%1,%%r12,4) SUBTRACT_m1n4(-112,4,%1)\
-  SOLVE_col1_rtol_m1n4(-128,5,%1,%%r12,4) SUBTRACT_m1n4(-128,4,%1)\
-  SAVE_SOLUTION_m1n4(5,-32) "negq %4; leaq (%3,%4,8),%3; negq %4;"\
-  SOLVE_col4_rtol_m1n4(-144,4,%1)\
-  SOLVE_col3_rtol_m1n4(-160,4,%1)\
-  SOLVE_col2_rtol_m1n4(-176,4,%1)\
-  SOLVE_col1_rtol_m1n4(-192,4,%1)\
-  SAVE_SOLUTION_m1n4(4,-48)
-
-/* r14 = b_tail, r15 = a_tail, r13 = k-kk */
-#define GEMM_RT_SIMPLE(mdim,ndim) \
-  "leaq (%%r15,%%r12,"#mdim"),%%r15; movq %%r15,%0; movq %%r13,%5; movq %%r14,%1;" INIT_m##mdim##n##ndim\
-  "testq %5,%5; jz 1"#mdim""#ndim"2f;"\
-  "1"#mdim""#ndim"1:\n\t"\
-  "subq $16,%1; subq $"#mdim"*4,%0;" GEMM_KERNEL_k1m##mdim##n##ndim "decq %5; jnz 1"#mdim""#ndim"1b;"\
-  "1"#mdim""#ndim"2:\n\t"
-#define GEMM_RT_m8n4 GEMM_RT_SIMPLE(8,4)
-#define GEMM_RT_m8n8 GEMM_RT_SIMPLE(8,8)
-#define GEMM_RT_m8n12 \
-  "leaq (%%r15,%%r12,8),%%r15; movq %%r15,%0; movq %%r13,%5; movq %%r14,%1;" INIT_m8n12\
-  "cmpq $8,%5; jb 18122f;"\
-  "18121:\n\t"\
-  "prefetcht0 -384(%0); subq $32,%0; subq $16,%1;" GEMM_KERNEL_k1m8n12\
-                       "subq $32,%0; subq $16,%1;" GEMM_KERNEL_k1m8n12\
-  "prefetcht0 -384(%0); subq $32,%0; subq $16,%1;" GEMM_KERNEL_k1m8n12\
-                       "subq $32,%0; subq $16,%1;" GEMM_KERNEL_k1m8n12\
-  "prefetcht0 -384(%0); subq $32,%0; subq $16,%1;" GEMM_KERNEL_k1m8n12\
-                       "subq $32,%0; subq $16,%1;" GEMM_KERNEL_k1m8n12\
-  "prefetcht0 -384(%0); subq $32,%0; subq $16,%1;" GEMM_KERNEL_k1m8n12\
-                       "subq $32,%0; subq $16,%1;" GEMM_KERNEL_k1m8n12\
-  "subq $8,%5; cmpq $8,%5; jnb 18121b;"\
-  "18122:\n\t"\
-  "testq %5,%5; jz 18124f;"\
-  "18123:\n\t"\
-  "subq $32,%0; subq $16,%1;" GEMM_KERNEL_k1m8n12 "decq %5; jnz 18123b;"\
-  "18124:\n\t"
-#define GEMM_RT_m4n4 GEMM_RT_SIMPLE(4,4)
-#define GEMM_RT_m4n8 GEMM_RT_SIMPLE(4,8)
-#define GEMM_RT_m4n12 GEMM_RT_SIMPLE(4,12)
-#define GEMM_RT_m2n4 GEMM_RT_SIMPLE(2,4)
-#define GEMM_RT_m2n8 GEMM_RT_SIMPLE(2,8)
-#define GEMM_RT_m2n12 GEMM_RT_SIMPLE(2,12)
-#define GEMM_RT_m1n4 GEMM_RT_SIMPLE(1,4)
-#define GEMM_RT_m1n8 GEMM_RT_SIMPLE(1,8)
-#define GEMM_RT_m1n12 GEMM_RT_SIMPLE(1,12)
-
-#define COMPUTE(ndim) {\
-  b_ptr -= (ndim-4)*K; c_ptr -= ndim * ldc;\
-  __asm__ __volatile__(\
-    "movq %0,%%r15; movq %6,%%r13; subq %7,%%r13; movq %6,%%r12; salq $2,%%r12; movq %1,%%r14; movq %10,%%r11;"\
-    "cmpq $8,%%r11; jb "#ndim"772f;"\
-    #ndim"771:\n\t"\
-    GEMM_RT_m8n##ndim SOLVE_RT_m8n##ndim "subq $8,%%r11; cmpq $8,%%r11; jnb "#ndim"771b;"\
-    #ndim"772:\n\t"\
-    "testq $4,%%r11; jz "#ndim"773f;"\
-    GEMM_RT_m4n##ndim SOLVE_RT_m4n##ndim "subq $4,%%r11;"\
-    #ndim"773:\n\t"\
-    "testq $2,%%r11; jz "#ndim"774f;"\
-    GEMM_RT_m2n##ndim SOLVE_RT_m2n##ndim "subq $2,%%r11;"\
-    #ndim"774:\n\t"\
-    "testq $1,%%r11; jz "#ndim"775f;"\
-    GEMM_RT_m1n##ndim SOLVE_RT_m1n##ndim "subq $1,%%r11;"\
-    #ndim"775:\n\t"\
-    "movq %%r15,%0; movq %%r14,%1; vzeroupper;"\
-  :"+r"(a_ptr),"+r"(b_ptr),"+r"(c_ptr),"+r"(c_tmp),"+r"(ldc_bytes),"+r"(k_cnt):"m"(K),"m"(OFF),"m"(one[0]),"m"(zero[0]),"m"(M)\
-  :"r11","r12","r13","r14","r15","cc","memory",\
-  "xmm0","xmm1","xmm2","xmm3","xmm4","xmm5","xmm6","xmm7","xmm8","xmm9","xmm10","xmm11","xmm12","xmm13","xmm14","xmm15");\
-  a_ptr -= M * K; b_ptr -= 4 * K; c_ptr -= M; OFF -= ndim;\
-}
-
-static void solve_RT(BLASLONG m, BLASLONG n, FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc){
-  FLOAT a0, b0;
-  int i, j, k;
-  for (i=n-1;i>=0;i--) {
-    b0 = b[i*n+i];
-    for (j=0;j<m;j++) {
-      a0 = c[i*ldc+j] * b0;
-      a[i*m+j] = c[i*ldc+j] = a0;
-      for (k=0;k<i;k++) c[k*ldc+j] -= a0 * b[i*n+k];
-    }
-  }
-}
-static void COMPUTE_EDGE_1_nchunk(BLASLONG m, BLASLONG n, FLOAT *sa, FLOAT *sb, FLOAT *C, BLASLONG ldc, BLASLONG k, BLASLONG offset) {
-  BLASLONG m_count = m, kk = offset; FLOAT *a_ptr = sa, *c_ptr = C;
-  for(;m_count>7;m_count-=8){
-    if(k-kk>0) GEMM_KERNEL_N(8,n,k-kk,-1.0,a_ptr+kk*8,sb+kk*n,c_ptr,ldc);
-    solve_RT(8,n,a_ptr+(kk-n)*8,sb+(kk-n)*n,c_ptr,ldc);
-    a_ptr += k * 8; c_ptr += 8;
-  }
-  for(;m_count>3;m_count-=4){
-    if(k-kk>0) GEMM_KERNEL_N(4,n,k-kk,-1.0,a_ptr+kk*4,sb+kk*n,c_ptr,ldc);
-    solve_RT(4,n,a_ptr+(kk-n)*4,sb+(kk-n)*n,c_ptr,ldc);
-    a_ptr += k * 4; c_ptr += 4;
-  }
-  for(;m_count>1;m_count-=2){
-    if(k-kk>0) GEMM_KERNEL_N(2,n,k-kk,-1.0,a_ptr+kk*2,sb+kk*n,c_ptr,ldc);
-    solve_RT(2,n,a_ptr+(kk-n)*2,sb+(kk-n)*n,c_ptr,ldc);
-    a_ptr += k * 2; c_ptr += 2;
-  }
-  if(m_count>0){
-    if(k-kk>0) GEMM_KERNEL_N(1,n,k-kk,-1.0,a_ptr+kk*1,sb+kk*n,c_ptr,ldc);
-    solve_RT(1,n,a_ptr+(kk-n)*1,sb+(kk-n)*n,c_ptr,ldc);
-    a_ptr += k * 1; c_ptr += 1;
-  }
-}
-int CNAME(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT dummy1, FLOAT *sa, FLOAT *sb, FLOAT *C, BLASLONG ldc, BLASLONG offset){
-  float *a_ptr = sa, *b_ptr = sb+n*k, *c_ptr = C+n*ldc, *c_tmp = C;
-  float one[8] = {1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0};
-  float zero[8] = {0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0};
-  uint64_t ldc_bytes = (uint64_t)ldc * sizeof(float), K = (uint64_t)k, M = (uint64_t)m, OFF = (uint64_t)(n-offset), k_cnt = 0;
-  BLASLONG n_count = n;
-  if(n&1){b_ptr-=k; c_ptr-=ldc; COMPUTE_EDGE_1_nchunk(m,1,a_ptr,b_ptr,c_ptr,ldc,k,OFF); OFF--; n_count--;}
-  if(n&2){b_ptr-=k*2; c_ptr-=ldc*2; COMPUTE_EDGE_1_nchunk(m,2,a_ptr,b_ptr,c_ptr,ldc,k,OFF); OFF-=2; n_count-=2;}
-  for(;n_count>11;n_count-=12) COMPUTE(12)
-  for(;n_count>7;n_count-=8) COMPUTE(8)
-  for(;n_count>3;n_count-=4) COMPUTE(4)
-  return 0;
-}
+#include "common.h"
+#include <stdint.h>
+#include "strsm_kernel_8x4_haswell_R_common.h"
+
+#define SOLVE_RT_m8n4 \
+  "movq %2,%3;" GEMM_SUM_REORDER_8x4(4,5,6,7,63) "negq %4; leaq (%3,%4,2),%3; negq %4; addq $32,%2;"\
+  SOLVE_rile_m8n2(-8,6,7,%1) SUBTRACT_m8n2(-16,4,5,%1)\
+  SOLVE_le_m8n2(-24,6,7,%1) SUBTRACT_m8n2(-32,4,5,%1)\
+  SAVE_SOLUTION_m8n2(6,7,-64) "negq %4; leaq (%3,%4,4),%3; negq %4;"\
+  SOLVE_rile_m8n2(-48,4,5,%1)\
+  SOLVE_le_m8n2(-64,4,5,%1)\
+  SAVE_SOLUTION_m8n2(4,5,-128)
+
+#define SOLVE_RT_m8n8 \
+  "movq %2,%3;" GEMM_SUM_REORDER_8x4(4,5,6,7,63) GEMM_SUM_REORDER_8x4(8,9,10,11,63) "negq %4; leaq (%3,%4,2),%3; negq %4; addq $32,%2;"\
+  SOLVE_rile_m8n2(-8,10,11,%1,%%r12,4) SUBTRACT_m8n2(-16,8,9,%1,%%r12,4) SUBTRACT_m8n2(-8,6,7,%1) SUBTRACT_m8n2(-16,4,5,%1)\
+  SOLVE_le_m8n2(-24,10,11,%1,%%r12,4) SUBTRACT_m8n2(-32,8,9,%1,%%r12,4) SUBTRACT_m8n2(-24,6,7,%1) SUBTRACT_m8n2(-32,4,5,%1)\
+  SAVE_SOLUTION_m8n2(10,11,-64) "negq %4; leaq (%3,%4,4),%3; negq %4;"\
+  SOLVE_rile_m8n2(-48,8,9,%1,%%r12,4) SUBTRACT_m8n2(-40,6,7,%1) SUBTRACT_m8n2(-48,4,5,%1)\
+  SOLVE_le_m8n2(-64,8,9,%1,%%r12,4) SUBTRACT_m8n2(-56,6,7,%1) SUBTRACT_m8n2(-64,4,5,%1)\
+  SAVE_SOLUTION_m8n2(8,9,-128) "negq %4; leaq (%3,%4,4),%3; negq %4;"\
+  SOLVE_rile_m8n2(-72,6,7,%1) SUBTRACT_m8n2(-80,4,5,%1)\
+  SOLVE_le_m8n2(-88,6,7,%1) SUBTRACT_m8n2(-96,4,5,%1)\
+  SAVE_SOLUTION_m8n2(6,7,-192) "negq %4; leaq (%3,%4,4),%3; negq %4;"\
+  SOLVE_rile_m8n2(-112,4,5,%1)\
+  SOLVE_le_m8n2(-128,4,5,%1)\
+  SAVE_SOLUTION_m8n2(4,5,-256)
+
+#define SOLVE_RT_m8n12 \
+  "movq %2,%3;" GEMM_SUM_REORDER_8x4(4,5,6,7,63) GEMM_SUM_REORDER_8x4(8,9,10,11,63) GEMM_SUM_REORDER_8x4(12,13,14,15,63) "negq %4; leaq (%3,%4,2),%3; negq %4; addq $32,%2;"\
+  SOLVE_rile_m8n2(-8,14,15,%1,%%r12,8) SUBTRACT_m8n2(-16,12,13,%1,%%r12,8) SUBTRACT_m8n2(-8,10,11,%1,%%r12,4) SUBTRACT_m8n2(-16,8,9,%1,%%r12,4) SUBTRACT_m8n2(-8,6,7,%1) SUBTRACT_m8n2(-16,4,5,%1)\
+  SOLVE_le_m8n2(-24,14,15,%1,%%r12,8) SUBTRACT_m8n2(-32,12,13,%1,%%r12,8) SUBTRACT_m8n2(-24,10,11,%1,%%r12,4) SUBTRACT_m8n2(-32,8,9,%1,%%r12,4) SUBTRACT_m8n2(-24,6,7,%1) SUBTRACT_m8n2(-32,4,5,%1)\
+  SAVE_SOLUTION_m8n2(14,15,-64) "negq %4; leaq (%3,%4,4),%3; negq %4;"\
+  SOLVE_rile_m8n2(-48,12,13,%1,%%r12,8) SUBTRACT_m8n2(-40,10,11,%1,%%r12,4) SUBTRACT_m8n2(-48,8,9,%1,%%r12,4) SUBTRACT_m8n2(-40,6,7,%1) SUBTRACT_m8n2(-48,4,5,%1)\
+  SOLVE_le_m8n2(-64,12,13,%1,%%r12,8) SUBTRACT_m8n2(-56,10,11,%1,%%r12,4) SUBTRACT_m8n2(-64,8,9,%1,%%r12,4) SUBTRACT_m8n2(-56,6,7,%1) SUBTRACT_m8n2(-64,4,5,%1)\
+  SAVE_SOLUTION_m8n2(12,13,-128) "negq %4; leaq (%3,%4,4),%3; negq %4;"\
+  SOLVE_rile_m8n2(-72,10,11,%1,%%r12,4) SUBTRACT_m8n2(-80,8,9,%1,%%r12,4) SUBTRACT_m8n2(-72,6,7,%1) SUBTRACT_m8n2(-80,4,5,%1)\
+  SOLVE_le_m8n2(-88,10,11,%1,%%r12,4) SUBTRACT_m8n2(-96,8,9,%1,%%r12,4) SUBTRACT_m8n2(-88,6,7,%1) SUBTRACT_m8n2(-96,4,5,%1)\
+  SAVE_SOLUTION_m8n2(10,11,-192) "negq %4; leaq (%3,%4,4),%3; negq %4;"\
+  SOLVE_rile_m8n2(-112,8,9,%1,%%r12,4) SUBTRACT_m8n2(-104,6,7,%1) SUBTRACT_m8n2(-112,4,5,%1)\
+  SOLVE_le_m8n2(-128,8,9,%1,%%r12,4) SUBTRACT_m8n2(-120,6,7,%1) SUBTRACT_m8n2(-128,4,5,%1)\
+  SAVE_SOLUTION_m8n2(8,9,-256) "negq %4; leaq (%3,%4,4),%3; negq %4;"\
+  SOLVE_rile_m8n2(-136,6,7,%1) SUBTRACT_m8n2(-144,4,5,%1)\
+  SOLVE_le_m8n2(-152,6,7,%1) SUBTRACT_m8n2(-160,4,5,%1)\
+  SAVE_SOLUTION_m8n2(6,7,-320) "negq %4; leaq (%3,%4,4),%3; negq %4;"\
+  SOLVE_rile_m8n2(-176,4,5,%1)\
+  SOLVE_le_m8n2(-192,4,5,%1)\
+  SAVE_SOLUTION_m8n2(4,5,-384)
+
+#define SOLVE_RT_m4n4 \
+  "movq %2,%3;" GEMM_SUM_REORDER_4x4(4,5,6,7,4,5) "negq %4; leaq (%3,%4,2),%3; negq %4; addq $16,%2;"\
+  SOLVE_rile_m4n2(-8,5,%1) SUBTRACT_m4n2(-16,4,%1)\
+  SOLVE_le_m4n2(-24,5,%1) SUBTRACT_m4n2(-32,4,%1)\
+  SAVE_SOLUTION_m4n2(5,-32) "negq %4; leaq (%3,%4,4),%3; negq %4;"\
+  SOLVE_rile_m4n2(-48,4,%1)\
+  SOLVE_le_m4n2(-64,4,%1)\
+  SAVE_SOLUTION_m4n2(4,-64)
+
+#define SOLVE_RT_m4n8 \
+  "movq %2,%3;" GEMM_SUM_REORDER_4x4(4,5,6,7,4,5) GEMM_SUM_REORDER_4x4(8,9,10,11,6,7) "negq %4; leaq (%3,%4,2),%3; negq %4; addq $16,%2;"\
+  SOLVE_rile_m4n2(-8,7,%1,%%r12,4) SUBTRACT_m4n2(-16,6,%1,%%r12,4) SUBTRACT_m4n2(-8,5,%1) SUBTRACT_m4n2(-16,4,%1)\
+  SOLVE_le_m4n2(-24,7,%1,%%r12,4) SUBTRACT_m4n2(-32,6,%1,%%r12,4) SUBTRACT_m4n2(-24,5,%1) SUBTRACT_m4n2(-32,4,%1)\
+  SAVE_SOLUTION_m4n2(7,-32) "negq %4; leaq (%3,%4,4),%3; negq %4;"\
+  SOLVE_rile_m4n2(-48,6,%1,%%r12,4) SUBTRACT_m4n2(-40,5,%1) SUBTRACT_m4n2(-48,4,%1)\
+  SOLVE_le_m4n2(-64,6,%1,%%r12,4) SUBTRACT_m4n2(-56,5,%1) SUBTRACT_m4n2(-64,4,%1)\
+  SAVE_SOLUTION_m4n2(6,-64) "negq %4; leaq (%3,%4,4),%3; negq %4;"\
+  SOLVE_rile_m4n2(-72,5,%1) SUBTRACT_m4n2(-80,4,%1)\
+  SOLVE_le_m4n2(-88,5,%1) SUBTRACT_m4n2(-96,4,%1)\
+  SAVE_SOLUTION_m4n2(5,-96) "negq %4; leaq (%3,%4,4),%3; negq %4;"\
+  SOLVE_rile_m4n2(-112,4,%1)\
+  SOLVE_le_m4n2(-128,4,%1)\
+  SAVE_SOLUTION_m4n2(4,-128)
+
+#define SOLVE_RT_m4n12 \
+  "movq %2,%3;" GEMM_SUM_REORDER_4x4(4,5,6,7,4,5) GEMM_SUM_REORDER_4x4(8,9,10,11,6,7) GEMM_SUM_REORDER_4x4(12,13,14,15,8,9) "negq %4; leaq (%3,%4,2),%3; negq %4; addq $16,%2;"\
+  SOLVE_rile_m4n2(-8,9,%1,%%r12,8) SUBTRACT_m4n2(-16,8,%1,%%r12,8) SUBTRACT_m4n2(-8,7,%1,%%r12,4) SUBTRACT_m4n2(-16,6,%1,%%r12,4) SUBTRACT_m4n2(-8,5,%1) SUBTRACT_m4n2(-16,4,%1)\
+  SOLVE_le_m4n2(-24,9,%1,%%r12,8) SUBTRACT_m4n2(-32,8,%1,%%r12,8) SUBTRACT_m4n2(-24,7,%1,%%r12,4) SUBTRACT_m4n2(-32,6,%1,%%r12,4) SUBTRACT_m4n2(-24,5,%1) SUBTRACT_m4n2(-32,4,%1)\
+  SAVE_SOLUTION_m4n2(9,-32) "negq %4; leaq (%3,%4,4),%3; negq %4;"\
+  SOLVE_rile_m4n2(-48,8,%1,%%r12,8) SUBTRACT_m4n2(-40,7,%1,%%r12,4) SUBTRACT_m4n2(-48,6,%1,%%r12,4) SUBTRACT_m4n2(-40,5,%1) SUBTRACT_m4n2(-48,4,%1)\
+  SOLVE_le_m4n2(-64,8,%1,%%r12,8) SUBTRACT_m4n2(-56,7,%1,%%r12,4) SUBTRACT_m4n2(-64,6,%1,%%r12,4) SUBTRACT_m4n2(-56,5,%1) SUBTRACT_m4n2(-64,4,%1)\
+  SAVE_SOLUTION_m4n2(8,-64) "negq %4; leaq (%3,%4,4),%3; negq %4;"\
+  SOLVE_rile_m4n2(-72,7,%1,%%r12,4) SUBTRACT_m4n2(-80,6,%1,%%r12,4) SUBTRACT_m4n2(-72,5,%1) SUBTRACT_m4n2(-80,4,%1)\
+  SOLVE_le_m4n2(-88,7,%1,%%r12,4) SUBTRACT_m4n2(-96,6,%1,%%r12,4) SUBTRACT_m4n2(-88,5,%1) SUBTRACT_m4n2(-96,4,%1)\
+  SAVE_SOLUTION_m4n2(7,-96) "negq %4; leaq (%3,%4,4),%3; negq %4;"\
+  SOLVE_rile_m4n2(-112,6,%1,%%r12,4) SUBTRACT_m4n2(-104,5,%1) SUBTRACT_m4n2(-112,4,%1)\
+  SOLVE_le_m4n2(-128,6,%1,%%r12,4) SUBTRACT_m4n2(-120,5,%1) SUBTRACT_m4n2(-128,4,%1)\
+  SAVE_SOLUTION_m4n2(6,-128) "negq %4; leaq (%3,%4,4),%3; negq %4;"\
+  SOLVE_rile_m4n2(-136,5,%1) SUBTRACT_m4n2(-144,4,%1)\
+  SOLVE_le_m4n2(-152,5,%1) SUBTRACT_m4n2(-160,4,%1)\
+  SAVE_SOLUTION_m4n2(5,-160) "negq %4; leaq (%3,%4,4),%3; negq %4;"\
+  SOLVE_rile_m4n2(-176,4,%1)\
+  SOLVE_le_m4n2(-192,4,%1)\
+  SAVE_SOLUTION_m4n2(4,-192)
+
+#define SOLVE_RT_m2n4 \
+  "movq %2,%3;" GEMM_SUM_REORDER_2x4(4,5) "negq %4; leaq (%3,%4,4),%3; negq %4; addq $8,%2;"\
+  SOLVE_col4_rtol_m2n4(-16,4,5,%1)\
+  SOLVE_col3_rtol_m2n4(-32,4,5,%1)\
+  SOLVE_col2_rtol_m2n4(-48,4,5,%1)\
+  SOLVE_col1_rtol_m2n4(-64,4,5,%1)\
+  SAVE_SOLUTION_m2n4(4,5,-32)
+
+#define SOLVE_RT_m2n8 \
+  "movq %2,%3;" GEMM_SUM_REORDER_2x4(4,5) GEMM_SUM_REORDER_2x4(6,7) "negq %4; leaq (%3,%4,4),%3; negq %4; addq $8,%2;"\
+  SOLVE_col4_rtol_m2n4(-16,6,7,%1,%%r12,4) SUBTRACT_m2n4(-16,4,5,%1)\
+  SOLVE_col3_rtol_m2n4(-32,6,7,%1,%%r12,4) SUBTRACT_m2n4(-32,4,5,%1)\
+  SOLVE_col2_rtol_m2n4(-48,6,7,%1,%%r12,4) SUBTRACT_m2n4(-48,4,5,%1)\
+  SOLVE_col1_rtol_m2n4(-64,6,7,%1,%%r12,4) SUBTRACT_m2n4(-64,4,5,%1)\
+  SAVE_SOLUTION_m2n4(6,7,-32) "negq %4; leaq (%3,%4,8),%3; negq %4;"\
+  SOLVE_col4_rtol_m2n4(-80,4,5,%1)\
+  SOLVE_col3_rtol_m2n4(-96,4,5,%1)\
+  SOLVE_col2_rtol_m2n4(-112,4,5,%1)\
+  SOLVE_col1_rtol_m2n4(-128,4,5,%1)\
+  SAVE_SOLUTION_m2n4(4,5,-64)
+
+#define SOLVE_RT_m2n12 \
+  "movq %2,%3;" GEMM_SUM_REORDER_2x4(4,5) GEMM_SUM_REORDER_2x4(6,7) GEMM_SUM_REORDER_2x4(8,9) "negq %4; leaq (%3,%4,4),%3; negq %4; addq $8,%2;"\
+  SOLVE_col4_rtol_m2n4(-16,8,9,%1,%%r12,8) SUBTRACT_m2n4(-16,6,7,%1,%%r12,4) SUBTRACT_m2n4(-16,4,5,%1)\
+  SOLVE_col3_rtol_m2n4(-32,8,9,%1,%%r12,8) SUBTRACT_m2n4(-32,6,7,%1,%%r12,4) SUBTRACT_m2n4(-32,4,5,%1)\
+  SOLVE_col2_rtol_m2n4(-48,8,9,%1,%%r12,8) SUBTRACT_m2n4(-48,6,7,%1,%%r12,4) SUBTRACT_m2n4(-48,4,5,%1)\
+  SOLVE_col1_rtol_m2n4(-64,8,9,%1,%%r12,8) SUBTRACT_m2n4(-64,6,7,%1,%%r12,4) SUBTRACT_m2n4(-64,4,5,%1)\
+  SAVE_SOLUTION_m2n4(8,9,-32) "negq %4; leaq (%3,%4,8),%3; negq %4;"\
+  SOLVE_col4_rtol_m2n4(-80,6,7,%1,%%r12,4) SUBTRACT_m2n4(-80,4,5,%1)\
+  SOLVE_col3_rtol_m2n4(-96,6,7,%1,%%r12,4) SUBTRACT_m2n4(-96,4,5,%1)\
+  SOLVE_col2_rtol_m2n4(-112,6,7,%1,%%r12,4) SUBTRACT_m2n4(-112,4,5,%1)\
+  SOLVE_col1_rtol_m2n4(-128,6,7,%1,%%r12,4) SUBTRACT_m2n4(-128,4,5,%1)\
+  SAVE_SOLUTION_m2n4(6,7,-64) "negq %4; leaq (%3,%4,8),%3; negq %4;"\
+  SOLVE_col4_rtol_m2n4(-144,4,5,%1)\
+  SOLVE_col3_rtol_m2n4(-160,4,5,%1)\
+  SOLVE_col2_rtol_m2n4(-176,4,5,%1)\
+  SOLVE_col1_rtol_m2n4(-192,4,5,%1)\
+  SAVE_SOLUTION_m2n4(4,5,-96)
+
+#define SOLVE_RT_m1n4 \
+  "movq %2,%3;" GEMM_SUM_REORDER_1x4(4) "negq %4; leaq (%3,%4,4),%3; negq %4; addq $4,%2;"\
+  SOLVE_col4_rtol_m1n4(-16,4,%1)\
+  SOLVE_col3_rtol_m1n4(-32,4,%1)\
+  SOLVE_col2_rtol_m1n4(-48,4,%1)\
+  SOLVE_col1_rtol_m1n4(-64,4,%1)\
+  SAVE_SOLUTION_m1n4(4,-16)
+
+#define SOLVE_RT_m1n8 \
+  "movq %2,%3;" GEMM_SUM_REORDER_1x4(4) GEMM_SUM_REORDER_1x4(5) "negq %4; leaq (%3,%4,4),%3; negq %4; addq $4,%2;"\
+  SOLVE_col4_rtol_m1n4(-16,5,%1,%%r12,4) SUBTRACT_m1n4(-16,4,%1)\
+  SOLVE_col3_rtol_m1n4(-32,5,%1,%%r12,4) SUBTRACT_m1n4(-32,4,%1)\
+  SOLVE_col2_rtol_m1n4(-48,5,%1,%%r12,4) SUBTRACT_m1n4(-48,4,%1)\
+  SOLVE_col1_rtol_m1n4(-64,5,%1,%%r12,4) SUBTRACT_m1n4(-64,4,%1)\
+  SAVE_SOLUTION_m1n4(5,-16) "negq %4; leaq (%3,%4,8),%3; negq %4;"\
+  SOLVE_col4_rtol_m1n4(-80,4,%1)\
+  SOLVE_col3_rtol_m1n4(-96,4,%1)\
+  SOLVE_col2_rtol_m1n4(-112,4,%1)\
+  SOLVE_col1_rtol_m1n4(-128,4,%1)\
+  SAVE_SOLUTION_m1n4(4,-32)
+
+#define SOLVE_RT_m1n12 \
+  "movq %2,%3;" GEMM_SUM_REORDER_1x4(4) GEMM_SUM_REORDER_1x4(5) GEMM_SUM_REORDER_1x4(6) "negq %4; leaq (%3,%4,4),%3; negq %4; addq $4,%2;"\
+  SOLVE_col4_rtol_m1n4(-16,6,%1,%%r12,8) SUBTRACT_m1n4(-16,5,%1,%%r12,4) SUBTRACT_m1n4(-16,4,%1)\
+  SOLVE_col3_rtol_m1n4(-32,6,%1,%%r12,8) SUBTRACT_m1n4(-32,5,%1,%%r12,4) SUBTRACT_m1n4(-32,4,%1)\
+  SOLVE_col2_rtol_m1n4(-48,6,%1,%%r12,8) SUBTRACT_m1n4(-48,5,%1,%%r12,4) SUBTRACT_m1n4(-48,4,%1)\
+  SOLVE_col1_rtol_m1n4(-64,6,%1,%%r12,8) SUBTRACT_m1n4(-64,5,%1,%%r12,4) SUBTRACT_m1n4(-64,4,%1)\
+  SAVE_SOLUTION_m1n4(6,-16) "negq %4; leaq (%3,%4,8),%3; negq %4;"\
+  SOLVE_col4_rtol_m1n4(-80,5,%1,%%r12,4) SUBTRACT_m1n4(-80,4,%1)\
+  SOLVE_col3_rtol_m1n4(-96,5,%1,%%r12,4) SUBTRACT_m1n4(-96,4,%1)\
+  SOLVE_col2_rtol_m1n4(-112,5,%1,%%r12,4) SUBTRACT_m1n4(-112,4,%1)\
+  SOLVE_col1_rtol_m1n4(-128,5,%1,%%r12,4) SUBTRACT_m1n4(-128,4,%1)\
+  SAVE_SOLUTION_m1n4(5,-32) "negq %4; leaq (%3,%4,8),%3; negq %4;"\
+  SOLVE_col4_rtol_m1n4(-144,4,%1)\
+  SOLVE_col3_rtol_m1n4(-160,4,%1)\
+  SOLVE_col2_rtol_m1n4(-176,4,%1)\
+  SOLVE_col1_rtol_m1n4(-192,4,%1)\
+  SAVE_SOLUTION_m1n4(4,-48)
+
+/* r14 = b_tail, r15 = a_tail, r13 = k-kk */
+#define GEMM_RT_SIMPLE(mdim,ndim) \
+  "leaq (%%r15,%%r12,"#mdim"),%%r15; movq %%r15,%0; movq %%r13,%5; movq %%r14,%1;" INIT_m##mdim##n##ndim\
+  "testq %5,%5; jz 1"#mdim""#ndim"2f;"\
+  "1"#mdim""#ndim"1:\n\t"\
+  "subq $16,%1; subq $"#mdim"*4,%0;" GEMM_KERNEL_k1m##mdim##n##ndim "decq %5; jnz 1"#mdim""#ndim"1b;"\
+  "1"#mdim""#ndim"2:\n\t"
+#define GEMM_RT_m8n4 GEMM_RT_SIMPLE(8,4)
+#define GEMM_RT_m8n8 GEMM_RT_SIMPLE(8,8)
+#define GEMM_RT_m8n12 \
+  "leaq (%%r15,%%r12,8),%%r15; movq %%r15,%0; movq %%r13,%5; movq %%r14,%1;" INIT_m8n12\
+  "cmpq $8,%5; jb 18122f;"\
+  "18121:\n\t"\
+  "prefetcht0 -384(%0); subq $32,%0; subq $16,%1;" GEMM_KERNEL_k1m8n12\
+                       "subq $32,%0; subq $16,%1;" GEMM_KERNEL_k1m8n12\
+  "prefetcht0 -384(%0); subq $32,%0; subq $16,%1;" GEMM_KERNEL_k1m8n12\
+                       "subq $32,%0; subq $16,%1;" GEMM_KERNEL_k1m8n12\
+  "prefetcht0 -384(%0); subq $32,%0; subq $16,%1;" GEMM_KERNEL_k1m8n12\
+                       "subq $32,%0; subq $16,%1;" GEMM_KERNEL_k1m8n12\
+  "prefetcht0 -384(%0); subq $32,%0; subq $16,%1;" GEMM_KERNEL_k1m8n12\
+                       "subq $32,%0; subq $16,%1;" GEMM_KERNEL_k1m8n12\
+  "subq $8,%5; cmpq $8,%5; jnb 18121b;"\
+  "18122:\n\t"\
+  "testq %5,%5; jz 18124f;"\
+  "18123:\n\t"\
+  "subq $32,%0; subq $16,%1;" GEMM_KERNEL_k1m8n12 "decq %5; jnz 18123b;"\
+  "18124:\n\t"
+#define GEMM_RT_m4n4 GEMM_RT_SIMPLE(4,4)
+#define GEMM_RT_m4n8 GEMM_RT_SIMPLE(4,8)
+#define GEMM_RT_m4n12 GEMM_RT_SIMPLE(4,12)
+#define GEMM_RT_m2n4 GEMM_RT_SIMPLE(2,4)
+#define GEMM_RT_m2n8 GEMM_RT_SIMPLE(2,8)
+#define GEMM_RT_m2n12 GEMM_RT_SIMPLE(2,12)
+#define GEMM_RT_m1n4 GEMM_RT_SIMPLE(1,4)
+#define GEMM_RT_m1n8 GEMM_RT_SIMPLE(1,8)
+#define GEMM_RT_m1n12 GEMM_RT_SIMPLE(1,12)
+
+#define COMPUTE(ndim) {\
+  b_ptr -= (ndim-4)*K; c_ptr -= ndim * ldc;\
+  __asm__ __volatile__(\
+    "movq %0,%%r15; movq %6,%%r13; subq %7,%%r13; movq %6,%%r12; salq $2,%%r12; movq %1,%%r14; movq %10,%%r11;"\
+    "cmpq $8,%%r11; jb "#ndim"772f;"\
+    #ndim"771:\n\t"\
+    GEMM_RT_m8n##ndim SOLVE_RT_m8n##ndim "subq $8,%%r11; cmpq $8,%%r11; jnb "#ndim"771b;"\
+    #ndim"772:\n\t"\
+    "testq $4,%%r11; jz "#ndim"773f;"\
+    GEMM_RT_m4n##ndim SOLVE_RT_m4n##ndim "subq $4,%%r11;"\
+    #ndim"773:\n\t"\
+    "testq $2,%%r11; jz "#ndim"774f;"\
+    GEMM_RT_m2n##ndim SOLVE_RT_m2n##ndim "subq $2,%%r11;"\
+    #ndim"774:\n\t"\
+    "testq $1,%%r11; jz "#ndim"775f;"\
+    GEMM_RT_m1n##ndim SOLVE_RT_m1n##ndim "subq $1,%%r11;"\
+    #ndim"775:\n\t"\
+    "movq %%r15,%0; movq %%r14,%1; vzeroupper;"\
+  :"+r"(a_ptr),"+r"(b_ptr),"+r"(c_ptr),"+r"(c_tmp),"+r"(ldc_bytes),"+r"(k_cnt):"m"(K),"m"(OFF),"m"(one[0]),"m"(zero[0]),"m"(M)\
+  :"r11","r12","r13","r14","r15","cc","memory",\
+  "xmm0","xmm1","xmm2","xmm3","xmm4","xmm5","xmm6","xmm7","xmm8","xmm9","xmm10","xmm11","xmm12","xmm13","xmm14","xmm15");\
+  a_ptr -= M * K; b_ptr -= 4 * K; c_ptr -= M; OFF -= ndim;\
+}
+
+static void solve_RT(BLASLONG m, BLASLONG n, FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc){
+  FLOAT a0, b0;
+  int i, j, k;
+  for (i=n-1;i>=0;i--) {
+    b0 = b[i*n+i];
+    for (j=0;j<m;j++) {
+      a0 = c[i*ldc+j] * b0;
+      a[i*m+j] = c[i*ldc+j] = a0;
+      for (k=0;k<i;k++) c[k*ldc+j] -= a0 * b[i*n+k];
+    }
+  }
+}
+static void COMPUTE_EDGE_1_nchunk(BLASLONG m, BLASLONG n, FLOAT *sa, FLOAT *sb, FLOAT *C, BLASLONG ldc, BLASLONG k, BLASLONG offset) {
+  BLASLONG m_count = m, kk = offset; FLOAT *a_ptr = sa, *c_ptr = C;
+  for(;m_count>7;m_count-=8){
+    if(k-kk>0) GEMM_KERNEL_N(8,n,k-kk,-1.0,a_ptr+kk*8,sb+kk*n,c_ptr,ldc);
+    solve_RT(8,n,a_ptr+(kk-n)*8,sb+(kk-n)*n,c_ptr,ldc);
+    a_ptr += k * 8; c_ptr += 8;
+  }
+  for(;m_count>3;m_count-=4){
+    if(k-kk>0) GEMM_KERNEL_N(4,n,k-kk,-1.0,a_ptr+kk*4,sb+kk*n,c_ptr,ldc);
+    solve_RT(4,n,a_ptr+(kk-n)*4,sb+(kk-n)*n,c_ptr,ldc);
+    a_ptr += k * 4; c_ptr += 4;
+  }
+  for(;m_count>1;m_count-=2){
+    if(k-kk>0) GEMM_KERNEL_N(2,n,k-kk,-1.0,a_ptr+kk*2,sb+kk*n,c_ptr,ldc);
+    solve_RT(2,n,a_ptr+(kk-n)*2,sb+(kk-n)*n,c_ptr,ldc);
+    a_ptr += k * 2; c_ptr += 2;
+  }
+  if(m_count>0){
+    if(k-kk>0) GEMM_KERNEL_N(1,n,k-kk,-1.0,a_ptr+kk*1,sb+kk*n,c_ptr,ldc);
+    solve_RT(1,n,a_ptr+(kk-n)*1,sb+(kk-n)*n,c_ptr,ldc);
+    a_ptr += k * 1; c_ptr += 1;
+  }
+}
+int CNAME(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT dummy1, FLOAT *sa, FLOAT *sb, FLOAT *C, BLASLONG ldc, BLASLONG offset){
+  float *a_ptr = sa, *b_ptr = sb+n*k, *c_ptr = C+n*ldc, *c_tmp = C;
+  float one[8] = {1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0};
+  float zero[8] = {0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0};
+  uint64_t ldc_bytes = (uint64_t)ldc * sizeof(float), K = (uint64_t)k, M = (uint64_t)m, OFF = (uint64_t)(n-offset), k_cnt = 0;
+  BLASLONG n_count = n;
+  if(n&1){b_ptr-=k; c_ptr-=ldc; COMPUTE_EDGE_1_nchunk(m,1,a_ptr,b_ptr,c_ptr,ldc,k,OFF); OFF--; n_count--;}
+  if(n&2){b_ptr-=k*2; c_ptr-=ldc*2; COMPUTE_EDGE_1_nchunk(m,2,a_ptr,b_ptr,c_ptr,ldc,k,OFF); OFF-=2; n_count-=2;}
+  for(;n_count>11;n_count-=12) COMPUTE(12)
+  for(;n_count>7;n_count-=8) COMPUTE(8)
+  for(;n_count>3;n_count-=4) COMPUTE(4)
+  return 0;
+}