Merge branch 'develop'

CMakeLists.txt

@@ -6,7 +6,7 @@ cmake_minimum_required(VERSION 2.8.4)
 project(OpenBLAS)
 set(OpenBLAS_MAJOR_VERSION 0)
 set(OpenBLAS_MINOR_VERSION 2)
-set(OpenBLAS_PATCH_VERSION 17)
+set(OpenBLAS_PATCH_VERSION 18)
 set(OpenBLAS_VERSION "${OpenBLAS_MAJOR_VERSION}.${OpenBLAS_MINOR_VERSION}.${OpenBLAS_PATCH_VERSION}")
 
 enable_language(ASM)

CONTRIBUTORS.md

@@ -147,5 +147,6 @@ In chronological order:
   * [2016-03-14] Additional functional Assembly Kernels for Cortex-A57
   * [2016-03-14] Optimize Dgemm 4x4 for Cortex-A57
 
-* [Your name or handle] <[email or website]>
-  * [Date] [Brief summary of your changes]
+* theoractice <https://github.com/theoractice/>
+  * [2016-03-20] Fix compiler error in VisualStudio with CMake
+  * [2016-03-22] Fix access violation on Windows while static linking

Changelog.txt

@@ -1,4 +1,22 @@
 OpenBLAS ChangeLog
+====================================================================
+Version 0.2.18
+12-Apr-2016
+common:
+	* If you set MAKE_NB_JOBS flag less or equal than zero,
+	  make will be without -j.
+
+x86/x86_64:
+	* Support building Visual Studio static library. (#813, Thanks, theoractice)
+	* Fix bugs to pass buidbot CI tests (http://build.openblas.net)
+
+ARM:
+	* Provide DGEMM 8x4 kernel for Cortex-A57 (Thanks, Ashwin Sekhar T K)
+
+POWER:
+	* Optimize S and C BLAS3 on Power8
+	* Optimize BLAS2/1 on Power8
+
 ====================================================================
 Version 0.2.17
 20-Mar-2016

Makefile.rule

@@ -3,7 +3,7 @@
 #
 
 # This library's version
-VERSION = 0.2.17
+VERSION = 0.2.18
 
 # If you set the suffix, the library name will be libopenblas_$(LIBNAMESUFFIX).a
 # and libopenblas_$(LIBNAMESUFFIX).so. Meanwhile, the soname in shared library
@@ -112,7 +112,10 @@ NO_AFFINITY = 1
 # NO_PARALLEL_MAKE = 1
 
 # Force number of make jobs. The default is the number of logical CPU of the host.
-# This is particularly useful when using distcc
+# This is particularly useful when using distcc.
+# A negative value will disable adding a -j flag to make, allowing to use a parent
+# make -j value. This is useful to call OpenBLAS make from an other project
+# makefile
 # MAKE_NB_JOBS = 2
 
 # If you would like to know minute performance report of GotoBLAS.

appveyor.yml

@@ -1,4 +1,4 @@
-version: 0.2.15.{build}
+version: 0.2.18.{build}
 
 #environment:
 

benchmark/Makefile

@@ -33,6 +33,10 @@ LIBMKL = -L$(MKL) -lmkl_intel_lp64 -lmkl_gnu_thread -lmkl_core -lgomp -lpthread
 # Apple vecLib
 LIBVECLIB = -framework Accelerate
 
+ESSL=/opt/ibm/lib
+#LIBESSL = -lesslsmp $(ESSL)/libxlomp_ser.so.1 $(ESSL)/libxlf90_r.so.1 $(ESSL)/libxlfmath.so.1 $(ESSL)/libxlsmp.so.1 /opt/ibm/xlC/13.1.3/lib/libxl.a
+LIBESSL = -lesslsmp  $(ESSL)/libxlf90_r.so.1 $(ESSL)/libxlfmath.so.1 $(ESSL)/libxlsmp.so.1 /opt/ibm/xlC/13.1.3/lib/libxl.a
+
 ifeq ($(OSNAME), WINNT)
 
 goto :: slinpack.goto dlinpack.goto clinpack.goto zlinpack.goto \
@@ -44,6 +48,7 @@ goto :: slinpack.goto dlinpack.goto clinpack.goto zlinpack.goto \
        ssyr2k.goto dsyr2k.goto csyr2k.goto zsyr2k.goto \
        sger.goto dger.goto cger.goto zger.goto \
        sdot.goto ddot.goto \
+       srot.goto drot.goto \
        saxpy.goto daxpy.goto caxpy.goto zaxpy.goto \
        scopy.goto dcopy.goto ccopy.goto zcopy.goto \
        sswap.goto dswap.goto cswap.goto zswap.goto \
@@ -151,6 +156,7 @@ goto :: slinpack.goto dlinpack.goto clinpack.goto zlinpack.goto \
        ssyr2k.goto dsyr2k.goto csyr2k.goto zsyr2k.goto \
        sger.goto dger.goto cger.goto zger.goto \
        sdot.goto ddot.goto cdot.goto zdot.goto \
+       srot.goto drot.goto \
        saxpy.goto daxpy.goto caxpy.goto zaxpy.goto \
        scopy.goto dcopy.goto ccopy.goto zcopy.goto \
        sswap.goto dswap.goto cswap.goto zswap.goto \
@@ -253,7 +259,9 @@ mkl :: slinpack.mkl dlinpack.mkl clinpack.mkl zlinpack.mkl \
 
 endif
 
-
+essl :: sgemm.essl strmm.essl dgemm.essl dtrmm.essl  \
+	cgemm.essl ctrmm.essl zgemm.essl ztrmm.essl  \
+	slinpack.essl clinpack.essl dlinpack.essl zlinpack.essl
 
 veclib :: slinpack.veclib dlinpack.veclib clinpack.veclib zlinpack.veclib \
        scholesky.veclib dcholesky.veclib ccholesky.veclib zcholesky.veclib \
@@ -306,6 +314,9 @@ slinpack.mkl : slinpack.$(SUFFIX)
 slinpack.veclib : slinpack.$(SUFFIX)
 	-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBVECLIB) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
 
+slinpack.essl : slinpack.$(SUFFIX)
+	-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBESSL) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
+
 ##################################### Dlinpack ####################################################
 dlinpack.goto : dlinpack.$(SUFFIX) ../$(LIBNAME)
 	$(CC) $(CFLAGS) -o $(@F) $^ $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB) -lm
@@ -322,6 +333,9 @@ dlinpack.mkl : dlinpack.$(SUFFIX)
 dlinpack.veclib : dlinpack.$(SUFFIX)
 	-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBVECLIB) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
 
+dlinpack.essl : dlinpack.$(SUFFIX)
+	-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBESSL) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
+
 ##################################### Clinpack ####################################################
 
 clinpack.goto : clinpack.$(SUFFIX) ../$(LIBNAME)
@@ -339,6 +353,9 @@ clinpack.mkl : clinpack.$(SUFFIX)
 clinpack.veclib : clinpack.$(SUFFIX)
 	-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBVECLIB) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
 
+clinpack.essl : clinpack.$(SUFFIX)
+	-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBESSL) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
+
 ##################################### Zlinpack ####################################################
 
 zlinpack.goto : zlinpack.$(SUFFIX) ../$(LIBNAME)
@@ -356,6 +373,9 @@ zlinpack.mkl : zlinpack.$(SUFFIX)
 zlinpack.veclib : zlinpack.$(SUFFIX)
 	-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBVECLIB) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
 
+zlinpack.essl : zlinpack.$(SUFFIX)
+	-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBESSL) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
+
 ##################################### Scholesky ###################################################
 
 scholesky.goto : scholesky.$(SUFFIX) ../$(LIBNAME)
@@ -441,6 +461,9 @@ sgemm.mkl : sgemm.$(SUFFIX)
 sgemm.veclib : sgemm.$(SUFFIX)
 	-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBVECLIB) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
 
+sgemm.essl : sgemm.$(SUFFIX)
+	-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBESSL) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
+
 ##################################### Dgemm ####################################################
 dgemm.goto : dgemm.$(SUFFIX) ../$(LIBNAME)
 	$(CC) $(CFLAGS) -o $(@F) $^ $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB) -lm
@@ -457,6 +480,9 @@ dgemm.mkl : dgemm.$(SUFFIX)
 dgemm.veclib : dgemm.$(SUFFIX)
 	-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBVECLIB) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
 
+dgemm.essl : dgemm.$(SUFFIX)
+	-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBESSL) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
+
 ##################################### Cgemm ####################################################
 
 cgemm.goto : cgemm.$(SUFFIX) ../$(LIBNAME)
@@ -474,6 +500,9 @@ cgemm.mkl : cgemm.$(SUFFIX)
 cgemm.veclib : cgemm.$(SUFFIX)
 	-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBVECLIB) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
 
+cgemm.essl : cgemm.$(SUFFIX)
+	-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBESSL) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
+
 ##################################### Zgemm ####################################################
 
 zgemm.goto : zgemm.$(SUFFIX) ../$(LIBNAME)
@@ -491,6 +520,9 @@ zgemm.mkl : zgemm.$(SUFFIX)
 zgemm.veclib : zgemm.$(SUFFIX)
 	-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBVECLIB) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
 
+zgemm.essl : zgemm.$(SUFFIX)
+	-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBESSL) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
+
 ##################################### Ssymm ####################################################
 ssymm.goto : ssymm.$(SUFFIX) ../$(LIBNAME)
 	$(CC) $(CFLAGS) -o $(@F) $^ $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB) -lm
@@ -573,6 +605,9 @@ strmm.mkl : strmm.$(SUFFIX)
 strmm.veclib : strmm.$(SUFFIX)
 	-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBVECLIB) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
 
+strmm.essl : strmm.$(SUFFIX)
+	-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBESSL) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
+
 ##################################### Dtrmm ####################################################
 dtrmm.goto : dtrmm.$(SUFFIX) ../$(LIBNAME)
 	$(CC) $(CFLAGS) -o $(@F) $^ $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB) -lm
@@ -589,6 +624,9 @@ dtrmm.mkl : dtrmm.$(SUFFIX)
 dtrmm.veclib : dtrmm.$(SUFFIX)
 	-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBVECLIB) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
 
+dtrmm.essl : dtrmm.$(SUFFIX)
+	-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBESSL) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
+
 ##################################### Ctrmm ####################################################
 
 ctrmm.goto : ctrmm.$(SUFFIX) ../$(LIBNAME)
@@ -606,6 +644,9 @@ ctrmm.mkl : ctrmm.$(SUFFIX)
 ctrmm.veclib : ctrmm.$(SUFFIX)
 	-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBVECLIB) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
 
+ctrmm.essl : ctrmm.$(SUFFIX)
+	-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBESSL) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
+
 ##################################### Ztrmm ####################################################
 
 ztrmm.goto : ztrmm.$(SUFFIX) ../$(LIBNAME)
@@ -623,6 +664,9 @@ ztrmm.mkl : ztrmm.$(SUFFIX)
 ztrmm.veclib : ztrmm.$(SUFFIX)
 	-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBVECLIB) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
 
+ztrmm.essl : ztrmm.$(SUFFIX)
+	-$(CC) $(CFLAGS) -o $(@F) $^ $(LIBESSL) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
+
 ##################################### Strsm ####################################################
 strsm.goto : strsm.$(SUFFIX) ../$(LIBNAME)
 	$(CC) $(CFLAGS) -o $(@F) $^ $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB) -lm
@@ -1413,6 +1457,39 @@ zdot.mkl : zdot-intel.$(SUFFIX)
 zdot.veclib : zdot-intel.$(SUFFIX)
 	$(CC) $(CFLAGS) -o $(@F) $^ $(LIBVECLIB) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
 
+##################################### Srot ####################################################
+srot.goto : srot.$(SUFFIX) ../$(LIBNAME)
+	$(CC) $(CFLAGS) -o $(@F) $^ $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB) -lm
+
+srot.acml : srot.$(SUFFIX)
+	$(CC) $(CFLAGS) -o $(@F) $^ $(LIBACML) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
+
+srot.atlas : srot.$(SUFFIX)
+	$(CC) $(CFLAGS) -o $(@F) $^ $(LIBATLAS) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
+
+srot.mkl : srot.$(SUFFIX)
+	$(CC) $(CFLAGS) -o $(@F) $^ $(LIBMKL) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
+
+srot.veclib : srot.$(SUFFIX)
+	$(CC) $(CFLAGS) -o $(@F) $^ $(LIBVECLIB) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
+
+##################################### Drot ####################################################
+drot.goto : drot.$(SUFFIX) ../$(LIBNAME)
+	$(CC) $(CFLAGS) -o $(@F) $^ $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB) -lm
+
+drot.acml : drot.$(SUFFIX)
+	$(CC) $(CFLAGS) -o $(@F) $^ $(LIBACML) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
+
+drot.atlas : drot.$(SUFFIX)
+	$(CC) $(CFLAGS) -o $(@F) $^ $(LIBATLAS) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
+
+drot.mkl : drot.$(SUFFIX)
+	$(CC) $(CFLAGS) -o $(@F) $^ $(LIBMKL) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
+
+drot.veclib : drot.$(SUFFIX)
+	$(CC) $(CFLAGS) -o $(@F) $^ $(LIBVECLIB) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB)
+
+
 ##################################### Saxpy ####################################################
 saxpy.goto : saxpy.$(SUFFIX) ../$(LIBNAME)
 	$(CC) $(CFLAGS) -o $(@F) $^ $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB) -lm
@@ -2124,6 +2201,13 @@ cgesv.$(SUFFIX) : gesv.c
 zgesv.$(SUFFIX) : gesv.c
 	$(CC) $(CFLAGS) -c -DCOMPLEX -DDOUBLE -o $(@F) $^
 
+srot.$(SUFFIX) : rot.c
+	$(CC) $(CFLAGS) -c -UCOMPLEX -UDOUBLE -o $(@F) $^
+
+drot.$(SUFFIX) : rot.c
+	$(CC) $(CFLAGS) -c -UCOMPLEX -DDOUBLE -o $(@F) $^
+
+
 
 
 
@@ -2137,7 +2221,7 @@ smallscaling: smallscaling.c ../$(LIBNAME)
 	$(CC) $(CFLAGS) -o $(@F) $^ $(EXTRALIB) -fopenmp -lm
 
 clean ::
-	@rm -f *.goto *.mkl *.acml *.atlas *.veclib
+	@rm -f *.goto *.mkl *.acml *.atlas *.veclib *.essl
 
 include $(TOPDIR)/Makefile.tail
 

benchmark/rot.c

@@ -0,0 +1,197 @@
+/***************************************************************************
+Copyright (c) 2014, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+#include <stdio.h>
+#include <stdlib.h>
+#ifdef __CYGWIN32__
+#include <sys/time.h>
+#endif
+#include "common.h"
+
+
+#undef DOT
+
+
+#ifdef DOUBLE
+#define ROT   BLASFUNC(drot)
+#else
+#define ROT   BLASFUNC(srot)
+#endif
+
+
+#if defined(__WIN32__) || defined(__WIN64__)
+
+#ifndef DELTA_EPOCH_IN_MICROSECS
+#define DELTA_EPOCH_IN_MICROSECS 11644473600000000ULL
+#endif
+
+int gettimeofday(struct timeval *tv, void *tz){
+
+  FILETIME ft;
+  unsigned __int64 tmpres = 0;
+  static int tzflag;
+
+  if (NULL != tv)
+    {
+      GetSystemTimeAsFileTime(&ft);
+
+      tmpres |= ft.dwHighDateTime;
+      tmpres <<= 32;
+      tmpres |= ft.dwLowDateTime;
+
+      /*converting file time to unix epoch*/
+      tmpres /= 10;  /*convert into microseconds*/
+      tmpres -= DELTA_EPOCH_IN_MICROSECS;
+      tv->tv_sec = (long)(tmpres / 1000000UL);
+      tv->tv_usec = (long)(tmpres % 1000000UL);
+    }
+
+  return 0;
+}
+
+#endif
+
+#if !defined(__WIN32__) && !defined(__WIN64__) && !defined(__CYGWIN32__) && 0
+
+static void *huge_malloc(BLASLONG size){
+  int shmid;
+  void *address;
+
+#ifndef SHM_HUGETLB
+#define SHM_HUGETLB 04000
+#endif
+
+  if ((shmid =shmget(IPC_PRIVATE,
+		     (size + HUGE_PAGESIZE) & ~(HUGE_PAGESIZE - 1),
+		     SHM_HUGETLB | IPC_CREAT |0600)) < 0) {
+    printf( "Memory allocation failed(shmget).\n");
+    exit(1);
+  }
+
+  address = shmat(shmid, NULL, SHM_RND);
+
+  if ((BLASLONG)address == -1){
+    printf( "Memory allocation failed(shmat).\n");
+    exit(1);
+  }
+
+  shmctl(shmid, IPC_RMID, 0);
+
+  return address;
+}
+
+#define malloc huge_malloc
+
+#endif
+
+int main(int argc, char *argv[]){
+
+  FLOAT *x, *y;
+  // FLOAT result;
+  blasint m, i;
+  blasint inc_x=1,inc_y=1;
+  FLOAT c[1] = { 2.0 };
+  FLOAT s[1] = { 2.0 };
+  int loops = 1;
+  int l;
+  char *p;
+
+  int from =   1;
+  int to   = 200;
+  int step =   1;
+
+  struct timeval start, stop;
+  double time1,timeg;
+
+  argc--;argv++;
+
+  if (argc > 0) { from     = atol(*argv);		argc--; argv++;}
+  if (argc > 0) { to       = MAX(atol(*argv), from);	argc--; argv++;}
+  if (argc > 0) { step     = atol(*argv);		argc--; argv++;}
+
+  if ((p = getenv("OPENBLAS_LOOPS")))  loops = atoi(p);
+  if ((p = getenv("OPENBLAS_INCX")))   inc_x = atoi(p);
+  if ((p = getenv("OPENBLAS_INCY")))   inc_y = atoi(p);
+
+  fprintf(stderr, "From : %3d  To : %3d Step = %3d Inc_x = %d Inc_y = %d Loops = %d\n", from, to, step,inc_x,inc_y,loops);
+
+  if (( x = (FLOAT *)malloc(sizeof(FLOAT) * to * abs(inc_x) * COMPSIZE)) == NULL){
+    fprintf(stderr,"Out of Memory!!\n");exit(1);
+  }
+
+  if (( y = (FLOAT *)malloc(sizeof(FLOAT) * to * abs(inc_y) * COMPSIZE)) == NULL){
+    fprintf(stderr,"Out of Memory!!\n");exit(1);
+  }
+
+#ifdef linux
+  srandom(getpid());
+#endif
+
+  fprintf(stderr, "   SIZE       Flops\n");
+
+  for(m = from; m <= to; m += step)
+  {
+
+   timeg=0;
+
+   fprintf(stderr, " %6d : ", (int)m);
+
+
+   for (l=0; l<loops; l++)
+   {
+
+   	for(i = 0; i < m * COMPSIZE * abs(inc_x); i++){
+			x[i] = ((FLOAT) rand() / (FLOAT) RAND_MAX) - 0.5;
+   	}
+
+   	for(i = 0; i < m * COMPSIZE * abs(inc_y); i++){
+			y[i] = ((FLOAT) rand() / (FLOAT) RAND_MAX) - 0.5;
+   	}
+    	gettimeofday( &start, (struct timezone *)0);
+
+    	ROT (&m, x, &inc_x, y, &inc_y, c, s);
+
+    	gettimeofday( &stop, (struct timezone *)0);
+
+    	time1 = (double)(stop.tv_sec - start.tv_sec) + (double)((stop.tv_usec - start.tv_usec)) * 1.e-6;
+
+	timeg += time1;
+
+    }
+
+    timeg /= loops;
+
+    fprintf(stderr,
+	    " %10.2f MFlops\n",
+	    COMPSIZE * COMPSIZE * 6. * (double)m / timeg * 1.e-6);
+
+  }
+
+  return 0;
+}
+
+// void main(int argc, char *argv[]) __attribute__((weak, alias("MAIN__")));

common_power.h

@@ -798,7 +798,7 @@ Lmcount$lazy_ptr:
 #elif defined(PPC440FP2)
 #define BUFFER_SIZE     ( 16 << 20)
 #elif defined(POWER8)
-#define BUFFER_SIZE     ( 64 << 20)
+#define BUFFER_SIZE     ( 32 << 20)
 #else
 #define BUFFER_SIZE     ( 16 << 20)
 #endif

common_x86.h

@@ -62,7 +62,7 @@ static void __inline blas_lock(volatile BLASULONG *address){
 
 #if defined(_MSC_VER) && !defined(__clang__)
 	// use intrinsic instead of inline assembly
-	ret = _InterlockedExchange(address, 1);
+	ret = _InterlockedExchange((volatile LONG *)address, 1);
 	// inline assembly
 	/*__asm {
 		mov eax, address

driver/others/memory.c

@@ -1452,6 +1452,31 @@ BOOL APIENTRY DllMain(HMODULE hModule, DWORD  ul_reason_for_call, LPVOID lpReser
   }
   return TRUE;
 }
+
+/*
+  This is to allow static linking.
+  Code adapted from Google performance tools:
+  https://gperftools.googlecode.com/git-history/perftools-1.0/src/windows/port.cc
+  Reference:
+  https://sourceware.org/ml/pthreads-win32/2008/msg00028.html
+  http://ci.boost.org/svn-trac/browser/trunk/libs/thread/src/win32/tss_pe.cpp
+*/
+static int on_process_term(void)
+{
+	gotoblas_quit();
+	return 0;
+}
+#ifdef _WIN64
+#pragma comment(linker, "/INCLUDE:_tls_used")
+#else
+#pragma comment(linker, "/INCLUDE:__tls_used")
+#endif
+#pragma data_seg(push, old_seg)
+#pragma data_seg(".CRT$XLB")
+static void (APIENTRY *dll_callback)(HINSTANCE h, DWORD ul_reason_for_call, PVOID pv) = DllMain;
+#pragma data_seg(".CRT$XTU")
+static int(*p_process_term)(void) = on_process_term;
+#pragma data_seg(pop, old_seg)
 #endif
 
 #if (defined(C_PGI) || (!defined(C_SUN) && defined(F_INTERFACE_SUN))) && (defined(ARCH_X86) || defined(ARCH_X86_64))

getarch.c

@@ -1013,7 +1013,12 @@ int main(int argc, char *argv[]){
 #endif
 
 #ifdef MAKE_NB_JOBS
+  #if MAKE_NB_JOBS > 0
     printf("MAKE += -j %d\n", MAKE_NB_JOBS);
+  #else
+    // Let make use parent -j argument or -j1 if there
+    // is no make parent
+  #endif
 #elif NO_PARALLEL_MAKE==1
     printf("MAKE += -j 1\n");
 #else

getarch_2nd.c

@@ -64,10 +64,13 @@ int main(int argc, char **argv) {
 
 
   if ((argc >= 2) && (*argv[1] == '1')) {
+
+#if defined(ARCH_X86) || defined(ARCH_X86_64)
     printf("#define SLOCAL_BUFFER_SIZE\t%ld\n", (SGEMM_DEFAULT_Q * SGEMM_DEFAULT_UNROLL_N * 4 * 1 *  sizeof(float)));
     printf("#define DLOCAL_BUFFER_SIZE\t%ld\n", (DGEMM_DEFAULT_Q * DGEMM_DEFAULT_UNROLL_N * 2 * 1 *  sizeof(double)));
     printf("#define CLOCAL_BUFFER_SIZE\t%ld\n", (CGEMM_DEFAULT_Q * CGEMM_DEFAULT_UNROLL_N * 4 * 2 *  sizeof(float)));
     printf("#define ZLOCAL_BUFFER_SIZE\t%ld\n", (ZGEMM_DEFAULT_Q * ZGEMM_DEFAULT_UNROLL_N * 2 * 2 *  sizeof(double)));
+#endif
 
 #ifdef USE64BITINT
 	printf("#define USE64BITINT\n");

kernel/arm64/cgemm_kernel_4x4.S

@@ -179,93 +179,93 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld2	{v2.4s, v3.4s}, [ppA]
 	add	ppA, ppA, #32
 
-	fmul	v16.4s, v0.4s, v8.4s[0]
-	OP_ii	v16.4s, v1.4s, v9.4s[0]
+	fmul	v16.4s, v0.4s, v8.s[0]
+	OP_ii	v16.4s, v1.4s, v9.s[0]
 #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
     defined(RR) || defined(RC) || defined(CR) || defined(CC)
 	eor	v17.16b, v17.16b, v17.16b
-	fmls	v17.4s, v0.4s, v9.4s[0]
+	fmls	v17.4s, v0.4s, v9.s[0]
 #else
-	fmul	v17.4s, v0.4s, v9.4s[0]
+	fmul	v17.4s, v0.4s, v9.s[0]
 #endif
-	OP_ir	v17.4s, v1.4s, v8.4s[0]
+	OP_ir	v17.4s, v1.4s, v8.s[0]
 
-	fmul	v20.4s, v0.4s, v8.4s[1]
-	OP_ii	v20.4s, v1.4s, v9.4s[1]
+	fmul	v20.4s, v0.4s, v8.s[1]
+	OP_ii	v20.4s, v1.4s, v9.s[1]
 #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
     defined(RR) || defined(RC) || defined(CR) || defined(CC)
 	eor	v21.16b, v21.16b, v21.16b
-	fmls	v21.4s, v0.4s, v9.4s[1]
+	fmls	v21.4s, v0.4s, v9.s[1]
 #else
-	fmul	v21.4s, v0.4s, v9.4s[1]
+	fmul	v21.4s, v0.4s, v9.s[1]
 #endif
-	OP_ir	v21.4s, v1.4s, v8.4s[1]
+	OP_ir	v21.4s, v1.4s, v8.s[1]
 
-	fmul	v24.4s, v0.4s, v8.4s[2]
-	OP_ii	v24.4s, v1.4s, v9.4s[2]
+	fmul	v24.4s, v0.4s, v8.s[2]
+	OP_ii	v24.4s, v1.4s, v9.s[2]
 #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
     defined(RR) || defined(RC) || defined(CR) || defined(CC)
 	eor	v25.16b, v25.16b, v25.16b
-	fmls	v25.4s, v0.4s, v9.4s[2]
+	fmls	v25.4s, v0.4s, v9.s[2]
 #else
-	fmul	v25.4s, v0.4s, v9.4s[2]
+	fmul	v25.4s, v0.4s, v9.s[2]
 #endif
-	OP_ir	v25.4s, v1.4s, v8.4s[2]
+	OP_ir	v25.4s, v1.4s, v8.s[2]
 
-	fmul	v28.4s, v0.4s, v8.4s[3]
-	OP_ii	v28.4s, v1.4s, v9.4s[3]
+	fmul	v28.4s, v0.4s, v8.s[3]
+	OP_ii	v28.4s, v1.4s, v9.s[3]
 #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
     defined(RR) || defined(RC) || defined(CR) || defined(CC)
 	eor	v29.16b, v29.16b, v29.16b
-	fmls	v29.4s, v0.4s, v9.4s[3]
+	fmls	v29.4s, v0.4s, v9.s[3]
 #else
-	fmul	v29.4s, v0.4s, v9.4s[3]
+	fmul	v29.4s, v0.4s, v9.s[3]
 #endif
-	OP_ir	v29.4s, v1.4s, v8.4s[3]
+	OP_ir	v29.4s, v1.4s, v8.s[3]
 
-	fmul	v18.4s, v2.4s, v8.4s[0]
-	OP_ii	v18.4s, v3.4s, v9.4s[0]
+	fmul	v18.4s, v2.4s, v8.s[0]
+	OP_ii	v18.4s, v3.4s, v9.s[0]
 #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
     defined(RR) || defined(RC) || defined(CR) || defined(CC)
 	eor	v19.16b, v19.16b, v19.16b
-	fmls	v19.4s, v2.4s, v9.4s[0]
+	fmls	v19.4s, v2.4s, v9.s[0]
 #else
-	fmul	v19.4s, v2.4s, v9.4s[0]
+	fmul	v19.4s, v2.4s, v9.s[0]
 #endif
-	OP_ir	v19.4s, v3.4s, v8.4s[0]
+	OP_ir	v19.4s, v3.4s, v8.s[0]
 
-	fmul	v22.4s, v2.4s, v8.4s[1]
-	OP_ii	v22.4s, v3.4s, v9.4s[1]
+	fmul	v22.4s, v2.4s, v8.s[1]
+	OP_ii	v22.4s, v3.4s, v9.s[1]
 #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
     defined(RR) || defined(RC) || defined(CR) || defined(CC)
 	eor	v23.16b, v23.16b, v23.16b
-	fmls	v23.4s, v2.4s, v9.4s[1]
+	fmls	v23.4s, v2.4s, v9.s[1]
 #else
-	fmul	v23.4s, v2.4s, v9.4s[1]
+	fmul	v23.4s, v2.4s, v9.s[1]
 #endif
-	OP_ir	v23.4s, v3.4s, v8.4s[1]
+	OP_ir	v23.4s, v3.4s, v8.s[1]
 
-	fmul	v26.4s, v2.4s, v8.4s[2]
-	OP_ii	v26.4s, v3.4s, v9.4s[2]
+	fmul	v26.4s, v2.4s, v8.s[2]
+	OP_ii	v26.4s, v3.4s, v9.s[2]
 #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
     defined(RR) || defined(RC) || defined(CR) || defined(CC)
 	eor	v27.16b, v27.16b, v27.16b
-	fmls	v27.4s, v2.4s, v9.4s[2]
+	fmls	v27.4s, v2.4s, v9.s[2]
 #else
-	fmul	v27.4s, v2.4s, v9.4s[2]
+	fmul	v27.4s, v2.4s, v9.s[2]
 #endif
-	OP_ir	v27.4s, v3.4s, v8.4s[2]
+	OP_ir	v27.4s, v3.4s, v8.s[2]
 
-	fmul	v30.4s, v2.4s, v8.4s[3]
-	OP_ii	v30.4s, v3.4s, v9.4s[3]
+	fmul	v30.4s, v2.4s, v8.s[3]
+	OP_ii	v30.4s, v3.4s, v9.s[3]
 #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
     defined(RR) || defined(RC) || defined(CR) || defined(CC)
 	eor	v31.16b, v31.16b, v31.16b
-	fmls	v31.4s, v2.4s, v9.4s[3]
+	fmls	v31.4s, v2.4s, v9.s[3]
 #else
-	fmul	v31.4s, v2.4s, v9.4s[3]
+	fmul	v31.4s, v2.4s, v9.s[3]
 #endif
-	OP_ir	v31.4s, v3.4s, v8.4s[3]
+	OP_ir	v31.4s, v3.4s, v8.s[3]
 
 	ld2	{v12.4s, v13.4s}, [pB]
 	add	pB, pB, #32
@@ -276,159 +276,159 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL8x4_M1
-	OP_rr	v16.4s, v0.4s, v8.4s[0]
-	OP_ii	v16.4s, v1.4s, v9.4s[0]
-	OP_ri	v17.4s, v0.4s, v9.4s[0]
-	OP_ir	v17.4s, v1.4s, v8.4s[0]
+	OP_rr	v16.4s, v0.4s, v8.s[0]
+	OP_ii	v16.4s, v1.4s, v9.s[0]
+	OP_ri	v17.4s, v0.4s, v9.s[0]
+	OP_ir	v17.4s, v1.4s, v8.s[0]
 
 	ld2	{v12.4s, v13.4s}, [pB]		// for next round
 	add	pB, pB, #32
 
-	OP_rr	v20.4s, v0.4s, v8.4s[1]
-	OP_ii	v20.4s, v1.4s, v9.4s[1]
-	OP_ri	v21.4s, v0.4s, v9.4s[1]
-	OP_ir	v21.4s, v1.4s, v8.4s[1]
+	OP_rr	v20.4s, v0.4s, v8.s[1]
+	OP_ii	v20.4s, v1.4s, v9.s[1]
+	OP_ri	v21.4s, v0.4s, v9.s[1]
+	OP_ir	v21.4s, v1.4s, v8.s[1]
 
 	prfm	PLDL1KEEP, [pB, #512]
 
-	OP_rr	v24.4s, v0.4s, v8.4s[2]
-	OP_ii	v24.4s, v1.4s, v9.4s[2]
-	OP_ri	v25.4s, v0.4s, v9.4s[2]
-	OP_ir	v25.4s, v1.4s, v8.4s[2]
+	OP_rr	v24.4s, v0.4s, v8.s[2]
+	OP_ii	v24.4s, v1.4s, v9.s[2]
+	OP_ri	v25.4s, v0.4s, v9.s[2]
+	OP_ir	v25.4s, v1.4s, v8.s[2]
 
 	ld2	{v4.4s, v5.4s} , [pA]		// for next round
 	add	pA, pA, #32
 
-	OP_rr	v28.4s, v0.4s, v8.4s[3]
-	OP_ii	v28.4s, v1.4s, v9.4s[3]
-	OP_ri	v29.4s, v0.4s, v9.4s[3]
-	OP_ir	v29.4s, v1.4s, v8.4s[3]
+	OP_rr	v28.4s, v0.4s, v8.s[3]
+	OP_ii	v28.4s, v1.4s, v9.s[3]
+	OP_ri	v29.4s, v0.4s, v9.s[3]
+	OP_ir	v29.4s, v1.4s, v8.s[3]
 
 	prfm	PLDL1KEEP, [pA, #512]
 
-	OP_rr	v18.4s, v2.4s, v8.4s[0]
-	OP_ii	v18.4s, v3.4s, v9.4s[0]
-	OP_ri	v19.4s, v2.4s, v9.4s[0]
-	OP_ir	v19.4s, v3.4s, v8.4s[0]
+	OP_rr	v18.4s, v2.4s, v8.s[0]
+	OP_ii	v18.4s, v3.4s, v9.s[0]
+	OP_ri	v19.4s, v2.4s, v9.s[0]
+	OP_ir	v19.4s, v3.4s, v8.s[0]
 
 	ld2	{v6.4s, v7.4s} , [ppA]		// for next round
 	add	ppA, ppA, #32
 
-	OP_rr	v22.4s, v2.4s, v8.4s[1]
-	OP_ii	v22.4s, v3.4s, v9.4s[1]
-	OP_ri	v23.4s, v2.4s, v9.4s[1]
-	OP_ir	v23.4s, v3.4s, v8.4s[1]
+	OP_rr	v22.4s, v2.4s, v8.s[1]
+	OP_ii	v22.4s, v3.4s, v9.s[1]
+	OP_ri	v23.4s, v2.4s, v9.s[1]
+	OP_ir	v23.4s, v3.4s, v8.s[1]
 
 	prfm	PLDL1KEEP, [ppA, #512]
 
-	OP_rr	v26.4s, v2.4s, v8.4s[2]
-	OP_ii	v26.4s, v3.4s, v9.4s[2]
-	OP_ri	v27.4s, v2.4s, v9.4s[2]
-	OP_ir	v27.4s, v3.4s, v8.4s[2]
+	OP_rr	v26.4s, v2.4s, v8.s[2]
+	OP_ii	v26.4s, v3.4s, v9.s[2]
+	OP_ri	v27.4s, v2.4s, v9.s[2]
+	OP_ir	v27.4s, v3.4s, v8.s[2]
 
-	OP_rr	v30.4s, v2.4s, v8.4s[3]
-	OP_ii	v30.4s, v3.4s, v9.4s[3]
-	OP_ri	v31.4s, v2.4s, v9.4s[3]
-	OP_ir	v31.4s, v3.4s, v8.4s[3]
+	OP_rr	v30.4s, v2.4s, v8.s[3]
+	OP_ii	v30.4s, v3.4s, v9.s[3]
+	OP_ri	v31.4s, v2.4s, v9.s[3]
+	OP_ir	v31.4s, v3.4s, v8.s[3]
 .endm
 
 .macro KERNEL8x4_M2
-	OP_rr	v16.4s, v4.4s, v12.4s[0]
-	OP_ii	v16.4s, v5.4s, v13.4s[0]
-	OP_ri	v17.4s, v4.4s, v13.4s[0]
-	OP_ir	v17.4s, v5.4s, v12.4s[0]
+	OP_rr	v16.4s, v4.4s, v12.s[0]
+	OP_ii	v16.4s, v5.4s, v13.s[0]
+	OP_ri	v17.4s, v4.4s, v13.s[0]
+	OP_ir	v17.4s, v5.4s, v12.s[0]
 
 	ld2	{v8.4s, v9.4s}, [pB]		// for next round
 	add	pB, pB, #32
 
-	OP_rr	v20.4s, v4.4s, v12.4s[1]
-	OP_ii	v20.4s, v5.4s, v13.4s[1]
-	OP_ri	v21.4s, v4.4s, v13.4s[1]
-	OP_ir	v21.4s, v5.4s, v12.4s[1]
+	OP_rr	v20.4s, v4.4s, v12.s[1]
+	OP_ii	v20.4s, v5.4s, v13.s[1]
+	OP_ri	v21.4s, v4.4s, v13.s[1]
+	OP_ir	v21.4s, v5.4s, v12.s[1]
 
 	prfm	PLDL1KEEP, [pA, #512]
 
-	OP_rr	v24.4s, v4.4s, v12.4s[2]
-	OP_ii	v24.4s, v5.4s, v13.4s[2]
-	OP_ri	v25.4s, v4.4s, v13.4s[2]
-	OP_ir	v25.4s, v5.4s, v12.4s[2]
+	OP_rr	v24.4s, v4.4s, v12.s[2]
+	OP_ii	v24.4s, v5.4s, v13.s[2]
+	OP_ri	v25.4s, v4.4s, v13.s[2]
+	OP_ir	v25.4s, v5.4s, v12.s[2]
 
 	ld2	{v0.4s, v1.4s}, [pA]		// for next round
 	add	pA, pA, #32
 
-	OP_rr	v28.4s, v4.4s, v12.4s[3]
-	OP_ii	v28.4s, v5.4s, v13.4s[3]
-	OP_ri	v29.4s, v4.4s, v13.4s[3]
-	OP_ir	v29.4s, v5.4s, v12.4s[3]
+	OP_rr	v28.4s, v4.4s, v12.s[3]
+	OP_ii	v28.4s, v5.4s, v13.s[3]
+	OP_ri	v29.4s, v4.4s, v13.s[3]
+	OP_ir	v29.4s, v5.4s, v12.s[3]
 
 	prfm	PLDL1KEEP, [ppA, #512]
 
-	OP_rr	v18.4s, v6.4s, v12.4s[0]
-	OP_ii	v18.4s, v7.4s, v13.4s[0]
-	OP_ri	v19.4s, v6.4s, v13.4s[0]
-	OP_ir	v19.4s, v7.4s, v12.4s[0]
+	OP_rr	v18.4s, v6.4s, v12.s[0]
+	OP_ii	v18.4s, v7.4s, v13.s[0]
+	OP_ri	v19.4s, v6.4s, v13.s[0]
+	OP_ir	v19.4s, v7.4s, v12.s[0]
 
 	ld2	{v2.4s, v3.4s}, [ppA]		// for next round
 	add	ppA, ppA, #32
 
-	OP_rr	v22.4s, v6.4s, v12.4s[1]
-	OP_ii	v22.4s, v7.4s, v13.4s[1]
-	OP_ri	v23.4s, v6.4s, v13.4s[1]
-	OP_ir	v23.4s, v7.4s, v12.4s[1]
+	OP_rr	v22.4s, v6.4s, v12.s[1]
+	OP_ii	v22.4s, v7.4s, v13.s[1]
+	OP_ri	v23.4s, v6.4s, v13.s[1]
+	OP_ir	v23.4s, v7.4s, v12.s[1]
 
 	prfm	PLDL1KEEP, [pB, #512]
 
-	OP_rr	v26.4s, v6.4s, v12.4s[2]
-	OP_ii	v26.4s, v7.4s, v13.4s[2]
-	OP_ri	v27.4s, v6.4s, v13.4s[2]
-	OP_ir	v27.4s, v7.4s, v12.4s[2]
+	OP_rr	v26.4s, v6.4s, v12.s[2]
+	OP_ii	v26.4s, v7.4s, v13.s[2]
+	OP_ri	v27.4s, v6.4s, v13.s[2]
+	OP_ir	v27.4s, v7.4s, v12.s[2]
 
-	OP_rr	v30.4s, v6.4s, v12.4s[3]
-	OP_ii	v30.4s, v7.4s, v13.4s[3]
-	OP_ri	v31.4s, v6.4s, v13.4s[3]
-	OP_ir	v31.4s, v7.4s, v12.4s[3]
+	OP_rr	v30.4s, v6.4s, v12.s[3]
+	OP_ii	v30.4s, v7.4s, v13.s[3]
+	OP_ri	v31.4s, v6.4s, v13.s[3]
+	OP_ir	v31.4s, v7.4s, v12.s[3]
 .endm
 
 .macro KERNEL8x4_E
-	OP_rr	v16.4s, v4.4s, v12.4s[0]
-	OP_ii	v16.4s, v5.4s, v13.4s[0]
-	OP_ri	v17.4s, v4.4s, v13.4s[0]
-	OP_ir	v17.4s, v5.4s, v12.4s[0]
-
-	OP_rr	v20.4s, v4.4s, v12.4s[1]
-	OP_ii	v20.4s, v5.4s, v13.4s[1]
-	OP_ri	v21.4s, v4.4s, v13.4s[1]
-	OP_ir	v21.4s, v5.4s, v12.4s[1]
-
-	OP_rr	v24.4s, v4.4s, v12.4s[2]
-	OP_ii	v24.4s, v5.4s, v13.4s[2]
-	OP_ri	v25.4s, v4.4s, v13.4s[2]
-	OP_ir	v25.4s, v5.4s, v12.4s[2]
-
-	OP_rr	v28.4s, v4.4s, v12.4s[3]
-	OP_ii	v28.4s, v5.4s, v13.4s[3]
-	OP_ri	v29.4s, v4.4s, v13.4s[3]
-	OP_ir	v29.4s, v5.4s, v12.4s[3]
-
-	OP_rr	v18.4s, v6.4s, v12.4s[0]
-	OP_ii	v18.4s, v7.4s, v13.4s[0]
-	OP_ri	v19.4s, v6.4s, v13.4s[0]
-	OP_ir	v19.4s, v7.4s, v12.4s[0]
-
-	OP_rr	v22.4s, v6.4s, v12.4s[1]
-	OP_ii	v22.4s, v7.4s, v13.4s[1]
-	OP_ri	v23.4s, v6.4s, v13.4s[1]
-	OP_ir	v23.4s, v7.4s, v12.4s[1]
-
-	OP_rr	v26.4s, v6.4s, v12.4s[2]
-	OP_ii	v26.4s, v7.4s, v13.4s[2]
-	OP_ri	v27.4s, v6.4s, v13.4s[2]
-	OP_ir	v27.4s, v7.4s, v12.4s[2]
-
-	OP_rr	v30.4s, v6.4s, v12.4s[3]
-	OP_ii	v30.4s, v7.4s, v13.4s[3]
-	OP_ri	v31.4s, v6.4s, v13.4s[3]
-	OP_ir	v31.4s, v7.4s, v12.4s[3]
+	OP_rr	v16.4s, v4.4s, v12.s[0]
+	OP_ii	v16.4s, v5.4s, v13.s[0]
+	OP_ri	v17.4s, v4.4s, v13.s[0]
+	OP_ir	v17.4s, v5.4s, v12.s[0]
+
+	OP_rr	v20.4s, v4.4s, v12.s[1]
+	OP_ii	v20.4s, v5.4s, v13.s[1]
+	OP_ri	v21.4s, v4.4s, v13.s[1]
+	OP_ir	v21.4s, v5.4s, v12.s[1]
+
+	OP_rr	v24.4s, v4.4s, v12.s[2]
+	OP_ii	v24.4s, v5.4s, v13.s[2]
+	OP_ri	v25.4s, v4.4s, v13.s[2]
+	OP_ir	v25.4s, v5.4s, v12.s[2]
+
+	OP_rr	v28.4s, v4.4s, v12.s[3]
+	OP_ii	v28.4s, v5.4s, v13.s[3]
+	OP_ri	v29.4s, v4.4s, v13.s[3]
+	OP_ir	v29.4s, v5.4s, v12.s[3]
+
+	OP_rr	v18.4s, v6.4s, v12.s[0]
+	OP_ii	v18.4s, v7.4s, v13.s[0]
+	OP_ri	v19.4s, v6.4s, v13.s[0]
+	OP_ir	v19.4s, v7.4s, v12.s[0]
+
+	OP_rr	v22.4s, v6.4s, v12.s[1]
+	OP_ii	v22.4s, v7.4s, v13.s[1]
+	OP_ri	v23.4s, v6.4s, v13.s[1]
+	OP_ir	v23.4s, v7.4s, v12.s[1]
+
+	OP_rr	v26.4s, v6.4s, v12.s[2]
+	OP_ii	v26.4s, v7.4s, v13.s[2]
+	OP_ri	v27.4s, v6.4s, v13.s[2]
+	OP_ir	v27.4s, v7.4s, v12.s[2]
+
+	OP_rr	v30.4s, v6.4s, v12.s[3]
+	OP_ii	v30.4s, v7.4s, v13.s[3]
+	OP_ri	v31.4s, v6.4s, v13.s[3]
+	OP_ir	v31.4s, v7.4s, v12.s[3]
 .endm
 
 .macro KERNEL8x4_SUB
@@ -437,48 +437,48 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld2	{v0.4s, v1.4s}, [pA]
 	add	pA, pA, #32
 
-	OP_rr	v16.4s, v0.4s, v8.4s[0]
-	OP_ii	v16.4s, v1.4s, v9.4s[0]
-	OP_ri	v17.4s, v0.4s, v9.4s[0]
-	OP_ir	v17.4s, v1.4s, v8.4s[0]
+	OP_rr	v16.4s, v0.4s, v8.s[0]
+	OP_ii	v16.4s, v1.4s, v9.s[0]
+	OP_ri	v17.4s, v0.4s, v9.s[0]
+	OP_ir	v17.4s, v1.4s, v8.s[0]
 
-	OP_rr	v20.4s, v0.4s, v8.4s[1]
-	OP_ii	v20.4s, v1.4s, v9.4s[1]
-	OP_ri	v21.4s, v0.4s, v9.4s[1]
-	OP_ir	v21.4s, v1.4s, v8.4s[1]
+	OP_rr	v20.4s, v0.4s, v8.s[1]
+	OP_ii	v20.4s, v1.4s, v9.s[1]
+	OP_ri	v21.4s, v0.4s, v9.s[1]
+	OP_ir	v21.4s, v1.4s, v8.s[1]
 
 	ld2	{v2.4s, v3.4s}, [ppA]
 	add	ppA, ppA, #32
 
-	OP_rr	v24.4s, v0.4s, v8.4s[2]
-	OP_ii	v24.4s, v1.4s, v9.4s[2]
-	OP_ri	v25.4s, v0.4s, v9.4s[2]
-	OP_ir	v25.4s, v1.4s, v8.4s[2]
-
-	OP_rr	v28.4s, v0.4s, v8.4s[3]
-	OP_ii	v28.4s, v1.4s, v9.4s[3]
-	OP_ri	v29.4s, v0.4s, v9.4s[3]
-	OP_ir	v29.4s, v1.4s, v8.4s[3]
-
-	OP_rr	v18.4s, v2.4s, v8.4s[0]
-	OP_ii	v18.4s, v3.4s, v9.4s[0]
-	OP_ri	v19.4s, v2.4s, v9.4s[0]
-	OP_ir	v19.4s, v3.4s, v8.4s[0]
-
-	OP_rr	v22.4s, v2.4s, v8.4s[1]
-	OP_ii	v22.4s, v3.4s, v9.4s[1]
-	OP_ri	v23.4s, v2.4s, v9.4s[1]
-	OP_ir	v23.4s, v3.4s, v8.4s[1]
-
-	OP_rr	v26.4s, v2.4s, v8.4s[2]
-	OP_ii	v26.4s, v3.4s, v9.4s[2]
-	OP_ri	v27.4s, v2.4s, v9.4s[2]
-	OP_ir	v27.4s, v3.4s, v8.4s[2]
-
-	OP_rr	v30.4s, v2.4s, v8.4s[3]
-	OP_ii	v30.4s, v3.4s, v9.4s[3]
-	OP_ri	v31.4s, v2.4s, v9.4s[3]
-	OP_ir	v31.4s, v3.4s, v8.4s[3]
+	OP_rr	v24.4s, v0.4s, v8.s[2]
+	OP_ii	v24.4s, v1.4s, v9.s[2]
+	OP_ri	v25.4s, v0.4s, v9.s[2]
+	OP_ir	v25.4s, v1.4s, v8.s[2]
+
+	OP_rr	v28.4s, v0.4s, v8.s[3]
+	OP_ii	v28.4s, v1.4s, v9.s[3]
+	OP_ri	v29.4s, v0.4s, v9.s[3]
+	OP_ir	v29.4s, v1.4s, v8.s[3]
+
+	OP_rr	v18.4s, v2.4s, v8.s[0]
+	OP_ii	v18.4s, v3.4s, v9.s[0]
+	OP_ri	v19.4s, v2.4s, v9.s[0]
+	OP_ir	v19.4s, v3.4s, v8.s[0]
+
+	OP_rr	v22.4s, v2.4s, v8.s[1]
+	OP_ii	v22.4s, v3.4s, v9.s[1]
+	OP_ri	v23.4s, v2.4s, v9.s[1]
+	OP_ir	v23.4s, v3.4s, v8.s[1]
+
+	OP_rr	v26.4s, v2.4s, v8.s[2]
+	OP_ii	v26.4s, v3.4s, v9.s[2]
+	OP_ri	v27.4s, v2.4s, v9.s[2]
+	OP_ir	v27.4s, v3.4s, v8.s[2]
+
+	OP_rr	v30.4s, v2.4s, v8.s[3]
+	OP_ii	v30.4s, v3.4s, v9.s[3]
+	OP_ri	v31.4s, v2.4s, v9.s[3]
+	OP_ir	v31.4s, v3.4s, v8.s[3]
 .endm
 
 .macro SAVE8x4
@@ -578,25 +578,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld2	{v0.4s, v1.4s}, [pA]
 	add	pA, pA, #32
 
-	OP_rr	v16.4s, v0.4s, v8.4s[0]
-	OP_ii	v16.4s, v1.4s, v9.4s[0]
-	OP_ri	v17.4s, v0.4s, v9.4s[0]
-	OP_ir	v17.4s, v1.4s, v8.4s[0]
-
-	OP_rr	v20.4s, v0.4s, v8.4s[1]
-	OP_ii	v20.4s, v1.4s, v9.4s[1]
-	OP_ri	v21.4s, v0.4s, v9.4s[1]
-	OP_ir	v21.4s, v1.4s, v8.4s[1]
-
-	OP_rr	v24.4s, v0.4s, v8.4s[2]
-	OP_ii	v24.4s, v1.4s, v9.4s[2]
-	OP_ri	v25.4s, v0.4s, v9.4s[2]
-	OP_ir	v25.4s, v1.4s, v8.4s[2]
-
-	OP_rr	v28.4s, v0.4s, v8.4s[3]
-	OP_ii	v28.4s, v1.4s, v9.4s[3]
-	OP_ri	v29.4s, v0.4s, v9.4s[3]
-	OP_ir	v29.4s, v1.4s, v8.4s[3]
+	OP_rr	v16.4s, v0.4s, v8.s[0]
+	OP_ii	v16.4s, v1.4s, v9.s[0]
+	OP_ri	v17.4s, v0.4s, v9.s[0]
+	OP_ir	v17.4s, v1.4s, v8.s[0]
+
+	OP_rr	v20.4s, v0.4s, v8.s[1]
+	OP_ii	v20.4s, v1.4s, v9.s[1]
+	OP_ri	v21.4s, v0.4s, v9.s[1]
+	OP_ir	v21.4s, v1.4s, v8.s[1]
+
+	OP_rr	v24.4s, v0.4s, v8.s[2]
+	OP_ii	v24.4s, v1.4s, v9.s[2]
+	OP_ri	v25.4s, v0.4s, v9.s[2]
+	OP_ir	v25.4s, v1.4s, v8.s[2]
+
+	OP_rr	v28.4s, v0.4s, v8.s[3]
+	OP_ii	v28.4s, v1.4s, v9.s[3]
+	OP_ri	v29.4s, v0.4s, v9.s[3]
+	OP_ir	v29.4s, v1.4s, v8.s[3]
 .endm
 
 .macro SAVE4x4
@@ -658,25 +658,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld2	{v0.2s, v1.2s}, [pA]
 	add	pA, pA, #16
 
-	OP_rr	v16.2s, v0.2s, v8.4s[0]
-	OP_ii	v16.2s, v1.2s, v9.4s[0]
-	OP_ri	v17.2s, v0.2s, v9.4s[0]
-	OP_ir	v17.2s, v1.2s, v8.4s[0]
-
-	OP_rr	v20.2s, v0.2s, v8.4s[1]
-	OP_ii	v20.2s, v1.2s, v9.4s[1]
-	OP_ri	v21.2s, v0.2s, v9.4s[1]
-	OP_ir	v21.2s, v1.2s, v8.4s[1]
-
-	OP_rr	v24.2s, v0.2s, v8.4s[2]
-	OP_ii	v24.2s, v1.2s, v9.4s[2]
-	OP_ri	v25.2s, v0.2s, v9.4s[2]
-	OP_ir	v25.2s, v1.2s, v8.4s[2]
-
-	OP_rr	v28.2s, v0.2s, v8.4s[3]
-	OP_ii	v28.2s, v1.2s, v9.4s[3]
-	OP_ri	v29.2s, v0.2s, v9.4s[3]
-	OP_ir	v29.2s, v1.2s, v8.4s[3]
+	OP_rr	v16.2s, v0.2s, v8.s[0]
+	OP_ii	v16.2s, v1.2s, v9.s[0]
+	OP_ri	v17.2s, v0.2s, v9.s[0]
+	OP_ir	v17.2s, v1.2s, v8.s[0]
+
+	OP_rr	v20.2s, v0.2s, v8.s[1]
+	OP_ii	v20.2s, v1.2s, v9.s[1]
+	OP_ri	v21.2s, v0.2s, v9.s[1]
+	OP_ir	v21.2s, v1.2s, v8.s[1]
+
+	OP_rr	v24.2s, v0.2s, v8.s[2]
+	OP_ii	v24.2s, v1.2s, v9.s[2]
+	OP_ri	v25.2s, v0.2s, v9.s[2]
+	OP_ir	v25.2s, v1.2s, v8.s[2]
+
+	OP_rr	v28.2s, v0.2s, v8.s[3]
+	OP_ii	v28.2s, v1.2s, v9.s[3]
+	OP_ri	v29.2s, v0.2s, v9.s[3]
+	OP_ir	v29.2s, v1.2s, v8.s[3]
 .endm
 
 .macro SAVE2x4
@@ -738,25 +738,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld2	{v0.s, v1.s}[0], [pA]
 	add	pA, pA, #8
 
-	OP_rr	s16, s0, v8.4s[0]
-	OP_ii	s16, s1, v9.4s[0]
-	OP_ri	s17, s0, v9.4s[0]
-	OP_ir	s17, s1, v8.4s[0]
-
-	OP_rr	s20, s0, v8.4s[1]
-	OP_ii	s20, s1, v9.4s[1]
-	OP_ri	s21, s0, v9.4s[1]
-	OP_ir	s21, s1, v8.4s[1]
-
-	OP_rr	s24, s0, v8.4s[2]
-	OP_ii	s24, s1, v9.4s[2]
-	OP_ri	s25, s0, v9.4s[2]
-	OP_ir	s25, s1, v8.4s[2]
-
-	OP_rr	s28, s0, v8.4s[3]
-	OP_ii	s28, s1, v9.4s[3]
-	OP_ri	s29, s0, v9.4s[3]
-	OP_ir	s29, s1, v8.4s[3]
+	OP_rr	s16, s0, v8.s[0]
+	OP_ii	s16, s1, v9.s[0]
+	OP_ri	s17, s0, v9.s[0]
+	OP_ir	s17, s1, v8.s[0]
+
+	OP_rr	s20, s0, v8.s[1]
+	OP_ii	s20, s1, v9.s[1]
+	OP_ri	s21, s0, v9.s[1]
+	OP_ir	s21, s1, v8.s[1]
+
+	OP_rr	s24, s0, v8.s[2]
+	OP_ii	s24, s1, v9.s[2]
+	OP_ri	s25, s0, v9.s[2]
+	OP_ir	s25, s1, v8.s[2]
+
+	OP_rr	s28, s0, v8.s[3]
+	OP_ii	s28, s1, v9.s[3]
+	OP_ri	s29, s0, v9.s[3]
+	OP_ir	s29, s1, v8.s[3]
 .endm
 
 .macro SAVE1x4
@@ -814,15 +814,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld2	{v0.4s, v1.4s}, [pA]
 	add	pA, pA, #32
 
-	OP_rr	v16.4s, v0.4s, v8.2s[0]
-	OP_ii	v16.4s, v1.4s, v9.2s[0]
-	OP_ri	v17.4s, v0.4s, v9.2s[0]
-	OP_ir	v17.4s, v1.4s, v8.2s[0]
+	OP_rr	v16.4s, v0.4s, v8.s[0]
+	OP_ii	v16.4s, v1.4s, v9.s[0]
+	OP_ri	v17.4s, v0.4s, v9.s[0]
+	OP_ir	v17.4s, v1.4s, v8.s[0]
 
-	OP_rr	v20.4s, v0.4s, v8.2s[1]
-	OP_ii	v20.4s, v1.4s, v9.2s[1]
-	OP_ri	v21.4s, v0.4s, v9.2s[1]
-	OP_ir	v21.4s, v1.4s, v8.2s[1]
+	OP_rr	v20.4s, v0.4s, v8.s[1]
+	OP_ii	v20.4s, v1.4s, v9.s[1]
+	OP_ri	v21.4s, v0.4s, v9.s[1]
+	OP_ir	v21.4s, v1.4s, v8.s[1]
 .endm
 
 .macro SAVE4x2
@@ -862,15 +862,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld2	{v0.2s, v1.2s}, [pA]
 	add	pA, pA, #16
 
-	OP_rr	v16.2s, v0.2s, v8.2s[0]
-	OP_ii	v16.2s, v1.2s, v9.2s[0]
-	OP_ri	v17.2s, v0.2s, v9.2s[0]
-	OP_ir	v17.2s, v1.2s, v8.2s[0]
+	OP_rr	v16.2s, v0.2s, v8.s[0]
+	OP_ii	v16.2s, v1.2s, v9.s[0]
+	OP_ri	v17.2s, v0.2s, v9.s[0]
+	OP_ir	v17.2s, v1.2s, v8.s[0]
 
-	OP_rr	v20.2s, v0.2s, v8.2s[1]
-	OP_ii	v20.2s, v1.2s, v9.2s[1]
-	OP_ri	v21.2s, v0.2s, v9.2s[1]
-	OP_ir	v21.2s, v1.2s, v8.2s[1]
+	OP_rr	v20.2s, v0.2s, v8.s[1]
+	OP_ii	v20.2s, v1.2s, v9.s[1]
+	OP_ri	v21.2s, v0.2s, v9.s[1]
+	OP_ir	v21.2s, v1.2s, v8.s[1]
 .endm
 
 .macro SAVE2x2
@@ -910,15 +910,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld2	{v0.s, v1.s}[0], [pA]
 	add	pA, pA, #8
 
-	OP_rr	s16, s0, v8.2s[0]
-	OP_ii	s16, s1, v9.2s[0]
-	OP_ri	s17, s0, v9.2s[0]
-	OP_ir	s17, s1, v8.2s[0]
+	OP_rr	s16, s0, v8.s[0]
+	OP_ii	s16, s1, v9.s[0]
+	OP_ri	s17, s0, v9.s[0]
+	OP_ir	s17, s1, v8.s[0]
 
-	OP_rr	s20, s0, v8.2s[1]
-	OP_ii	s20, s1, v9.2s[1]
-	OP_ri	s21, s0, v9.2s[1]
-	OP_ir	s21, s1, v8.2s[1]
+	OP_rr	s20, s0, v8.s[1]
+	OP_ii	s20, s1, v9.s[1]
+	OP_ri	s21, s0, v9.s[1]
+	OP_ir	s21, s1, v8.s[1]
 .endm
 
 .macro SAVE1x2

kernel/arm64/cgemm_kernel_8x4.S

@@ -178,93 +178,93 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld2	{v2.4s, v3.4s}, [pA]
 	add	pA, pA, #32
 
-	fmul	v16.4s, v0.4s, v8.4s[0]
-	OP_ii	v16.4s, v1.4s, v9.4s[0]
+	fmul	v16.4s, v0.4s, v8.s[0]
+	OP_ii	v16.4s, v1.4s, v9.s[0]
 #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
     defined(RR) || defined(RC) || defined(CR) || defined(CC)
 	eor	v17.16b, v17.16b, v17.16b
-	fmls	v17.4s, v0.4s, v9.4s[0]
+	fmls	v17.4s, v0.4s, v9.s[0]
 #else
-	fmul	v17.4s, v0.4s, v9.4s[0]
+	fmul	v17.4s, v0.4s, v9.s[0]
 #endif
-	OP_ir	v17.4s, v1.4s, v8.4s[0]
+	OP_ir	v17.4s, v1.4s, v8.s[0]
 
-	fmul	v18.4s, v2.4s, v8.4s[0]
-	OP_ii	v18.4s, v3.4s, v9.4s[0]
+	fmul	v18.4s, v2.4s, v8.s[0]
+	OP_ii	v18.4s, v3.4s, v9.s[0]
 #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
     defined(RR) || defined(RC) || defined(CR) || defined(CC)
 	eor	v19.16b, v19.16b, v19.16b
-	fmls	v19.4s, v2.4s, v9.4s[0]
+	fmls	v19.4s, v2.4s, v9.s[0]
 #else
-	fmul	v19.4s, v2.4s, v9.4s[0]
+	fmul	v19.4s, v2.4s, v9.s[0]
 #endif
-	OP_ir	v19.4s, v3.4s, v8.4s[0]
+	OP_ir	v19.4s, v3.4s, v8.s[0]
 
-	fmul	v20.4s, v0.4s, v8.4s[1]
-	OP_ii	v20.4s, v1.4s, v9.4s[1]
+	fmul	v20.4s, v0.4s, v8.s[1]
+	OP_ii	v20.4s, v1.4s, v9.s[1]
 #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
     defined(RR) || defined(RC) || defined(CR) || defined(CC)
 	eor	v21.16b, v21.16b, v21.16b
-	fmls	v21.4s, v0.4s, v9.4s[1]
+	fmls	v21.4s, v0.4s, v9.s[1]
 #else
-	fmul	v21.4s, v0.4s, v9.4s[1]
+	fmul	v21.4s, v0.4s, v9.s[1]
 #endif
-	OP_ir	v21.4s, v1.4s, v8.4s[1]
+	OP_ir	v21.4s, v1.4s, v8.s[1]
 
-	fmul	v22.4s, v2.4s, v8.4s[1]
-	OP_ii	v22.4s, v3.4s, v9.4s[1]
+	fmul	v22.4s, v2.4s, v8.s[1]
+	OP_ii	v22.4s, v3.4s, v9.s[1]
 #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
     defined(RR) || defined(RC) || defined(CR) || defined(CC)
 	eor	v23.16b, v23.16b, v23.16b
-	fmls	v23.4s, v2.4s, v9.4s[1]
+	fmls	v23.4s, v2.4s, v9.s[1]
 #else
-	fmul	v23.4s, v2.4s, v9.4s[1]
+	fmul	v23.4s, v2.4s, v9.s[1]
 #endif
-	OP_ir	v23.4s, v3.4s, v8.4s[1]
+	OP_ir	v23.4s, v3.4s, v8.s[1]
 
-	fmul	v24.4s, v0.4s, v8.4s[2]
-	OP_ii	v24.4s, v1.4s, v9.4s[2]
+	fmul	v24.4s, v0.4s, v8.s[2]
+	OP_ii	v24.4s, v1.4s, v9.s[2]
 #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
     defined(RR) || defined(RC) || defined(CR) || defined(CC)
 	eor	v25.16b, v25.16b, v25.16b
-	fmls	v25.4s, v0.4s, v9.4s[2]
+	fmls	v25.4s, v0.4s, v9.s[2]
 #else
-	fmul	v25.4s, v0.4s, v9.4s[2]
+	fmul	v25.4s, v0.4s, v9.s[2]
 #endif
-	OP_ir	v25.4s, v1.4s, v8.4s[2]
+	OP_ir	v25.4s, v1.4s, v8.s[2]
 
-	fmul	v26.4s, v2.4s, v8.4s[2]
-	OP_ii	v26.4s, v3.4s, v9.4s[2]
+	fmul	v26.4s, v2.4s, v8.s[2]
+	OP_ii	v26.4s, v3.4s, v9.s[2]
 #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
     defined(RR) || defined(RC) || defined(CR) || defined(CC)
 	eor	v27.16b, v27.16b, v27.16b
-	fmls	v27.4s, v2.4s, v9.4s[2]
+	fmls	v27.4s, v2.4s, v9.s[2]
 #else
-	fmul	v27.4s, v2.4s, v9.4s[2]
+	fmul	v27.4s, v2.4s, v9.s[2]
 #endif
-	OP_ir	v27.4s, v3.4s, v8.4s[2]
+	OP_ir	v27.4s, v3.4s, v8.s[2]
 
-	fmul	v28.4s, v0.4s, v8.4s[3]
-	OP_ii	v28.4s, v1.4s, v9.4s[3]
+	fmul	v28.4s, v0.4s, v8.s[3]
+	OP_ii	v28.4s, v1.4s, v9.s[3]
 #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
     defined(RR) || defined(RC) || defined(CR) || defined(CC)
 	eor	v29.16b, v29.16b, v29.16b
-	fmls	v29.4s, v0.4s, v9.4s[3]
+	fmls	v29.4s, v0.4s, v9.s[3]
 #else
-	fmul	v29.4s, v0.4s, v9.4s[3]
+	fmul	v29.4s, v0.4s, v9.s[3]
 #endif
-	OP_ir	v29.4s, v1.4s, v8.4s[3]
+	OP_ir	v29.4s, v1.4s, v8.s[3]
 
-	fmul	v30.4s, v2.4s, v8.4s[3]
-	OP_ii	v30.4s, v3.4s, v9.4s[3]
+	fmul	v30.4s, v2.4s, v8.s[3]
+	OP_ii	v30.4s, v3.4s, v9.s[3]
 #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
     defined(RR) || defined(RC) || defined(CR) || defined(CC)
 	eor	v31.16b, v31.16b, v31.16b
-	fmls	v31.4s, v2.4s, v9.4s[3]
+	fmls	v31.4s, v2.4s, v9.s[3]
 #else
-	fmul	v31.4s, v2.4s, v9.4s[3]
+	fmul	v31.4s, v2.4s, v9.s[3]
 #endif
-	OP_ir	v31.4s, v3.4s, v8.4s[3]
+	OP_ir	v31.4s, v3.4s, v8.s[3]
 
 	ld2	{v12.4s, v13.4s}, [pB]
 	add	pB, pB, #32
@@ -275,45 +275,45 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL8x4_M1
-	OP_rr	v16.4s, v0.4s, v8.4s[0]
-	OP_ii	v16.4s, v1.4s, v9.4s[0]
-	OP_ri	v17.4s, v0.4s, v9.4s[0]
-	OP_ir	v17.4s, v1.4s, v8.4s[0]
-
-	OP_rr	v18.4s, v2.4s, v8.4s[0]
-	OP_ii	v18.4s, v3.4s, v9.4s[0]
-	OP_ri	v19.4s, v2.4s, v9.4s[0]
-	OP_ir	v19.4s, v3.4s, v8.4s[0]
-
-	OP_rr	v20.4s, v0.4s, v8.4s[1]
-	OP_ii	v20.4s, v1.4s, v9.4s[1]
-	OP_ri	v21.4s, v0.4s, v9.4s[1]
-	OP_ir	v21.4s, v1.4s, v8.4s[1]
-
-	OP_rr	v22.4s, v2.4s, v8.4s[1]
-	OP_ii	v22.4s, v3.4s, v9.4s[1]
-	OP_ri	v23.4s, v2.4s, v9.4s[1]
-	OP_ir	v23.4s, v3.4s, v8.4s[1]
-
-	OP_rr	v24.4s, v0.4s, v8.4s[2]
-	OP_ii	v24.4s, v1.4s, v9.4s[2]
-	OP_ri	v25.4s, v0.4s, v9.4s[2]
-	OP_ir	v25.4s, v1.4s, v8.4s[2]
-
-	OP_rr	v26.4s, v2.4s, v8.4s[2]
-	OP_ii	v26.4s, v3.4s, v9.4s[2]
-	OP_ri	v27.4s, v2.4s, v9.4s[2]
-	OP_ir	v27.4s, v3.4s, v8.4s[2]
-
-	OP_rr	v28.4s, v0.4s, v8.4s[3]
-	OP_ii	v28.4s, v1.4s, v9.4s[3]
-	OP_ri	v29.4s, v0.4s, v9.4s[3]
-	OP_ir	v29.4s, v1.4s, v8.4s[3]
-
-	OP_rr	v30.4s, v2.4s, v8.4s[3]
-	OP_ii	v30.4s, v3.4s, v9.4s[3]
-	OP_ri	v31.4s, v2.4s, v9.4s[3]
-	OP_ir	v31.4s, v3.4s, v8.4s[3]
+	OP_rr	v16.4s, v0.4s, v8.s[0]
+	OP_ii	v16.4s, v1.4s, v9.s[0]
+	OP_ri	v17.4s, v0.4s, v9.s[0]
+	OP_ir	v17.4s, v1.4s, v8.s[0]
+
+	OP_rr	v18.4s, v2.4s, v8.s[0]
+	OP_ii	v18.4s, v3.4s, v9.s[0]
+	OP_ri	v19.4s, v2.4s, v9.s[0]
+	OP_ir	v19.4s, v3.4s, v8.s[0]
+
+	OP_rr	v20.4s, v0.4s, v8.s[1]
+	OP_ii	v20.4s, v1.4s, v9.s[1]
+	OP_ri	v21.4s, v0.4s, v9.s[1]
+	OP_ir	v21.4s, v1.4s, v8.s[1]
+
+	OP_rr	v22.4s, v2.4s, v8.s[1]
+	OP_ii	v22.4s, v3.4s, v9.s[1]
+	OP_ri	v23.4s, v2.4s, v9.s[1]
+	OP_ir	v23.4s, v3.4s, v8.s[1]
+
+	OP_rr	v24.4s, v0.4s, v8.s[2]
+	OP_ii	v24.4s, v1.4s, v9.s[2]
+	OP_ri	v25.4s, v0.4s, v9.s[2]
+	OP_ir	v25.4s, v1.4s, v8.s[2]
+
+	OP_rr	v26.4s, v2.4s, v8.s[2]
+	OP_ii	v26.4s, v3.4s, v9.s[2]
+	OP_ri	v27.4s, v2.4s, v9.s[2]
+	OP_ir	v27.4s, v3.4s, v8.s[2]
+
+	OP_rr	v28.4s, v0.4s, v8.s[3]
+	OP_ii	v28.4s, v1.4s, v9.s[3]
+	OP_ri	v29.4s, v0.4s, v9.s[3]
+	OP_ir	v29.4s, v1.4s, v8.s[3]
+
+	OP_rr	v30.4s, v2.4s, v8.s[3]
+	OP_ii	v30.4s, v3.4s, v9.s[3]
+	OP_ri	v31.4s, v2.4s, v9.s[3]
+	OP_ir	v31.4s, v3.4s, v8.s[3]
 
 	ld2	{v12.4s, v13.4s}, [pB]		// For next round
 	add	pB, pB, #32
@@ -324,45 +324,45 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL8x4_M2
-	OP_rr	v16.4s, v4.4s, v12.4s[0]
-	OP_ii	v16.4s, v5.4s, v13.4s[0]
-	OP_ri	v17.4s, v4.4s, v13.4s[0]
-	OP_ir	v17.4s, v5.4s, v12.4s[0]
-
-	OP_rr	v18.4s, v6.4s, v12.4s[0]
-	OP_ii	v18.4s, v7.4s, v13.4s[0]
-	OP_ri	v19.4s, v6.4s, v13.4s[0]
-	OP_ir	v19.4s, v7.4s, v12.4s[0]
-
-	OP_rr	v20.4s, v4.4s, v12.4s[1]
-	OP_ii	v20.4s, v5.4s, v13.4s[1]
-	OP_ri	v21.4s, v4.4s, v13.4s[1]
-	OP_ir	v21.4s, v5.4s, v12.4s[1]
-
-	OP_rr	v22.4s, v6.4s, v12.4s[1]
-	OP_ii	v22.4s, v7.4s, v13.4s[1]
-	OP_ri	v23.4s, v6.4s, v13.4s[1]
-	OP_ir	v23.4s, v7.4s, v12.4s[1]
-
-	OP_rr	v24.4s, v4.4s, v12.4s[2]
-	OP_ii	v24.4s, v5.4s, v13.4s[2]
-	OP_ri	v25.4s, v4.4s, v13.4s[2]
-	OP_ir	v25.4s, v5.4s, v12.4s[2]
-
-	OP_rr	v26.4s, v6.4s, v12.4s[2]
-	OP_ii	v26.4s, v7.4s, v13.4s[2]
-	OP_ri	v27.4s, v6.4s, v13.4s[2]
-	OP_ir	v27.4s, v7.4s, v12.4s[2]
-
-	OP_rr	v28.4s, v4.4s, v12.4s[3]
-	OP_ii	v28.4s, v5.4s, v13.4s[3]
-	OP_ri	v29.4s, v4.4s, v13.4s[3]
-	OP_ir	v29.4s, v5.4s, v12.4s[3]
-
-	OP_rr	v30.4s, v6.4s, v12.4s[3]
-	OP_ii	v30.4s, v7.4s, v13.4s[3]
-	OP_ri	v31.4s, v6.4s, v13.4s[3]
-	OP_ir	v31.4s, v7.4s, v12.4s[3]
+	OP_rr	v16.4s, v4.4s, v12.s[0]
+	OP_ii	v16.4s, v5.4s, v13.s[0]
+	OP_ri	v17.4s, v4.4s, v13.s[0]
+	OP_ir	v17.4s, v5.4s, v12.s[0]
+
+	OP_rr	v18.4s, v6.4s, v12.s[0]
+	OP_ii	v18.4s, v7.4s, v13.s[0]
+	OP_ri	v19.4s, v6.4s, v13.s[0]
+	OP_ir	v19.4s, v7.4s, v12.s[0]
+
+	OP_rr	v20.4s, v4.4s, v12.s[1]
+	OP_ii	v20.4s, v5.4s, v13.s[1]
+	OP_ri	v21.4s, v4.4s, v13.s[1]
+	OP_ir	v21.4s, v5.4s, v12.s[1]
+
+	OP_rr	v22.4s, v6.4s, v12.s[1]
+	OP_ii	v22.4s, v7.4s, v13.s[1]
+	OP_ri	v23.4s, v6.4s, v13.s[1]
+	OP_ir	v23.4s, v7.4s, v12.s[1]
+
+	OP_rr	v24.4s, v4.4s, v12.s[2]
+	OP_ii	v24.4s, v5.4s, v13.s[2]
+	OP_ri	v25.4s, v4.4s, v13.s[2]
+	OP_ir	v25.4s, v5.4s, v12.s[2]
+
+	OP_rr	v26.4s, v6.4s, v12.s[2]
+	OP_ii	v26.4s, v7.4s, v13.s[2]
+	OP_ri	v27.4s, v6.4s, v13.s[2]
+	OP_ir	v27.4s, v7.4s, v12.s[2]
+
+	OP_rr	v28.4s, v4.4s, v12.s[3]
+	OP_ii	v28.4s, v5.4s, v13.s[3]
+	OP_ri	v29.4s, v4.4s, v13.s[3]
+	OP_ir	v29.4s, v5.4s, v12.s[3]
+
+	OP_rr	v30.4s, v6.4s, v12.s[3]
+	OP_ii	v30.4s, v7.4s, v13.s[3]
+	OP_ri	v31.4s, v6.4s, v13.s[3]
+	OP_ir	v31.4s, v7.4s, v12.s[3]
 
 	ld2	{v8.4s, v9.4s}, [pB]
 	add	pB, pB, #32
@@ -373,45 +373,45 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL8x4_E
-	OP_rr	v16.4s, v4.4s, v12.4s[0]
-	OP_ii	v16.4s, v5.4s, v13.4s[0]
-	OP_ri	v17.4s, v4.4s, v13.4s[0]
-	OP_ir	v17.4s, v5.4s, v12.4s[0]
-
-	OP_rr	v18.4s, v6.4s, v12.4s[0]
-	OP_ii	v18.4s, v7.4s, v13.4s[0]
-	OP_ri	v19.4s, v6.4s, v13.4s[0]
-	OP_ir	v19.4s, v7.4s, v12.4s[0]
-
-	OP_rr	v20.4s, v4.4s, v12.4s[1]
-	OP_ii	v20.4s, v5.4s, v13.4s[1]
-	OP_ri	v21.4s, v4.4s, v13.4s[1]
-	OP_ir	v21.4s, v5.4s, v12.4s[1]
-
-	OP_rr	v22.4s, v6.4s, v12.4s[1]
-	OP_ii	v22.4s, v7.4s, v13.4s[1]
-	OP_ri	v23.4s, v6.4s, v13.4s[1]
-	OP_ir	v23.4s, v7.4s, v12.4s[1]
-
-	OP_rr	v24.4s, v4.4s, v12.4s[2]
-	OP_ii	v24.4s, v5.4s, v13.4s[2]
-	OP_ri	v25.4s, v4.4s, v13.4s[2]
-	OP_ir	v25.4s, v5.4s, v12.4s[2]
-
-	OP_rr	v26.4s, v6.4s, v12.4s[2]
-	OP_ii	v26.4s, v7.4s, v13.4s[2]
-	OP_ri	v27.4s, v6.4s, v13.4s[2]
-	OP_ir	v27.4s, v7.4s, v12.4s[2]
-
-	OP_rr	v28.4s, v4.4s, v12.4s[3]
-	OP_ii	v28.4s, v5.4s, v13.4s[3]
-	OP_ri	v29.4s, v4.4s, v13.4s[3]
-	OP_ir	v29.4s, v5.4s, v12.4s[3]
-
-	OP_rr	v30.4s, v6.4s, v12.4s[3]
-	OP_ii	v30.4s, v7.4s, v13.4s[3]
-	OP_ri	v31.4s, v6.4s, v13.4s[3]
-	OP_ir	v31.4s, v7.4s, v12.4s[3]
+	OP_rr	v16.4s, v4.4s, v12.s[0]
+	OP_ii	v16.4s, v5.4s, v13.s[0]
+	OP_ri	v17.4s, v4.4s, v13.s[0]
+	OP_ir	v17.4s, v5.4s, v12.s[0]
+
+	OP_rr	v18.4s, v6.4s, v12.s[0]
+	OP_ii	v18.4s, v7.4s, v13.s[0]
+	OP_ri	v19.4s, v6.4s, v13.s[0]
+	OP_ir	v19.4s, v7.4s, v12.s[0]
+
+	OP_rr	v20.4s, v4.4s, v12.s[1]
+	OP_ii	v20.4s, v5.4s, v13.s[1]
+	OP_ri	v21.4s, v4.4s, v13.s[1]
+	OP_ir	v21.4s, v5.4s, v12.s[1]
+
+	OP_rr	v22.4s, v6.4s, v12.s[1]
+	OP_ii	v22.4s, v7.4s, v13.s[1]
+	OP_ri	v23.4s, v6.4s, v13.s[1]
+	OP_ir	v23.4s, v7.4s, v12.s[1]
+
+	OP_rr	v24.4s, v4.4s, v12.s[2]
+	OP_ii	v24.4s, v5.4s, v13.s[2]
+	OP_ri	v25.4s, v4.4s, v13.s[2]
+	OP_ir	v25.4s, v5.4s, v12.s[2]
+
+	OP_rr	v26.4s, v6.4s, v12.s[2]
+	OP_ii	v26.4s, v7.4s, v13.s[2]
+	OP_ri	v27.4s, v6.4s, v13.s[2]
+	OP_ir	v27.4s, v7.4s, v12.s[2]
+
+	OP_rr	v28.4s, v4.4s, v12.s[3]
+	OP_ii	v28.4s, v5.4s, v13.s[3]
+	OP_ri	v29.4s, v4.4s, v13.s[3]
+	OP_ir	v29.4s, v5.4s, v12.s[3]
+
+	OP_rr	v30.4s, v6.4s, v12.s[3]
+	OP_ii	v30.4s, v7.4s, v13.s[3]
+	OP_ri	v31.4s, v6.4s, v13.s[3]
+	OP_ir	v31.4s, v7.4s, v12.s[3]
 
 .endm
 
@@ -423,45 +423,45 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld2	{v2.4s, v3.4s}, [pA]
 	add	pA, pA, #32
 
-	OP_rr	v16.4s, v0.4s, v8.4s[0]
-	OP_ii	v16.4s, v1.4s, v9.4s[0]
-	OP_ri	v17.4s, v0.4s, v9.4s[0]
-	OP_ir	v17.4s, v1.4s, v8.4s[0]
-
-	OP_rr	v18.4s, v2.4s, v8.4s[0]
-	OP_ii	v18.4s, v3.4s, v9.4s[0]
-	OP_ri	v19.4s, v2.4s, v9.4s[0]
-	OP_ir	v19.4s, v3.4s, v8.4s[0]
-
-	OP_rr	v20.4s, v0.4s, v8.4s[1]
-	OP_ii	v20.4s, v1.4s, v9.4s[1]
-	OP_ri	v21.4s, v0.4s, v9.4s[1]
-	OP_ir	v21.4s, v1.4s, v8.4s[1]
-
-	OP_rr	v22.4s, v2.4s, v8.4s[1]
-	OP_ii	v22.4s, v3.4s, v9.4s[1]
-	OP_ri	v23.4s, v2.4s, v9.4s[1]
-	OP_ir	v23.4s, v3.4s, v8.4s[1]
-
-	OP_rr	v24.4s, v0.4s, v8.4s[2]
-	OP_ii	v24.4s, v1.4s, v9.4s[2]
-	OP_ri	v25.4s, v0.4s, v9.4s[2]
-	OP_ir	v25.4s, v1.4s, v8.4s[2]
-
-	OP_rr	v26.4s, v2.4s, v8.4s[2]
-	OP_ii	v26.4s, v3.4s, v9.4s[2]
-	OP_ri	v27.4s, v2.4s, v9.4s[2]
-	OP_ir	v27.4s, v3.4s, v8.4s[2]
-
-	OP_rr	v28.4s, v0.4s, v8.4s[3]
-	OP_ii	v28.4s, v1.4s, v9.4s[3]
-	OP_ri	v29.4s, v0.4s, v9.4s[3]
-	OP_ir	v29.4s, v1.4s, v8.4s[3]
-
-	OP_rr	v30.4s, v2.4s, v8.4s[3]
-	OP_ii	v30.4s, v3.4s, v9.4s[3]
-	OP_ri	v31.4s, v2.4s, v9.4s[3]
-	OP_ir	v31.4s, v3.4s, v8.4s[3]
+	OP_rr	v16.4s, v0.4s, v8.s[0]
+	OP_ii	v16.4s, v1.4s, v9.s[0]
+	OP_ri	v17.4s, v0.4s, v9.s[0]
+	OP_ir	v17.4s, v1.4s, v8.s[0]
+
+	OP_rr	v18.4s, v2.4s, v8.s[0]
+	OP_ii	v18.4s, v3.4s, v9.s[0]
+	OP_ri	v19.4s, v2.4s, v9.s[0]
+	OP_ir	v19.4s, v3.4s, v8.s[0]
+
+	OP_rr	v20.4s, v0.4s, v8.s[1]
+	OP_ii	v20.4s, v1.4s, v9.s[1]
+	OP_ri	v21.4s, v0.4s, v9.s[1]
+	OP_ir	v21.4s, v1.4s, v8.s[1]
+
+	OP_rr	v22.4s, v2.4s, v8.s[1]
+	OP_ii	v22.4s, v3.4s, v9.s[1]
+	OP_ri	v23.4s, v2.4s, v9.s[1]
+	OP_ir	v23.4s, v3.4s, v8.s[1]
+
+	OP_rr	v24.4s, v0.4s, v8.s[2]
+	OP_ii	v24.4s, v1.4s, v9.s[2]
+	OP_ri	v25.4s, v0.4s, v9.s[2]
+	OP_ir	v25.4s, v1.4s, v8.s[2]
+
+	OP_rr	v26.4s, v2.4s, v8.s[2]
+	OP_ii	v26.4s, v3.4s, v9.s[2]
+	OP_ri	v27.4s, v2.4s, v9.s[2]
+	OP_ir	v27.4s, v3.4s, v8.s[2]
+
+	OP_rr	v28.4s, v0.4s, v8.s[3]
+	OP_ii	v28.4s, v1.4s, v9.s[3]
+	OP_ri	v29.4s, v0.4s, v9.s[3]
+	OP_ir	v29.4s, v1.4s, v8.s[3]
+
+	OP_rr	v30.4s, v2.4s, v8.s[3]
+	OP_ii	v30.4s, v3.4s, v9.s[3]
+	OP_ri	v31.4s, v2.4s, v9.s[3]
+	OP_ir	v31.4s, v3.4s, v8.s[3]
 
 .endm
 
@@ -560,49 +560,49 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld2	{v0.4s, v1.4s}, [pA]
 	add	pA, pA, #32
 
-	fmul	v16.4s, v0.4s, v8.4s[0]
-	OP_ii	v16.4s, v1.4s, v9.4s[0]
+	fmul	v16.4s, v0.4s, v8.s[0]
+	OP_ii	v16.4s, v1.4s, v9.s[0]
 #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
     defined(RR) || defined(RC) || defined(CR) || defined(CC)
 	eor	v17.16b, v17.16b, v17.16b
-	fmls	v17.4s, v0.4s, v9.4s[0]
+	fmls	v17.4s, v0.4s, v9.s[0]
 #else
-	fmul	v17.4s, v0.4s, v9.4s[0]
+	fmul	v17.4s, v0.4s, v9.s[0]
 #endif
-	OP_ir	v17.4s, v1.4s, v8.4s[0]
+	OP_ir	v17.4s, v1.4s, v8.s[0]
 
-	fmul	v20.4s, v0.4s, v8.4s[1]
-	OP_ii	v20.4s, v1.4s, v9.4s[1]
+	fmul	v20.4s, v0.4s, v8.s[1]
+	OP_ii	v20.4s, v1.4s, v9.s[1]
 #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
     defined(RR) || defined(RC) || defined(CR) || defined(CC)
 	eor	v21.16b, v21.16b, v21.16b
-	fmls	v21.4s, v0.4s, v9.4s[1]
+	fmls	v21.4s, v0.4s, v9.s[1]
 #else
-	fmul	v21.4s, v0.4s, v9.4s[1]
+	fmul	v21.4s, v0.4s, v9.s[1]
 #endif
-	OP_ir	v21.4s, v1.4s, v8.4s[1]
+	OP_ir	v21.4s, v1.4s, v8.s[1]
 
-	fmul	v24.4s, v0.4s, v8.4s[2]
-	OP_ii	v24.4s, v1.4s, v9.4s[2]
+	fmul	v24.4s, v0.4s, v8.s[2]
+	OP_ii	v24.4s, v1.4s, v9.s[2]
 #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
     defined(RR) || defined(RC) || defined(CR) || defined(CC)
 	eor	v25.16b, v25.16b, v25.16b
-	fmls	v25.4s, v0.4s, v9.4s[2]
+	fmls	v25.4s, v0.4s, v9.s[2]
 #else
-	fmul	v25.4s, v0.4s, v9.4s[2]
+	fmul	v25.4s, v0.4s, v9.s[2]
 #endif
-	OP_ir	v25.4s, v1.4s, v8.4s[2]
+	OP_ir	v25.4s, v1.4s, v8.s[2]
 
-	fmul	v28.4s, v0.4s, v8.4s[3]
-	OP_ii	v28.4s, v1.4s, v9.4s[3]
+	fmul	v28.4s, v0.4s, v8.s[3]
+	OP_ii	v28.4s, v1.4s, v9.s[3]
 #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
     defined(RR) || defined(RC) || defined(CR) || defined(CC)
 	eor	v29.16b, v29.16b, v29.16b
-	fmls	v29.4s, v0.4s, v9.4s[3]
+	fmls	v29.4s, v0.4s, v9.s[3]
 #else
-	fmul	v29.4s, v0.4s, v9.4s[3]
+	fmul	v29.4s, v0.4s, v9.s[3]
 #endif
-	OP_ir	v29.4s, v1.4s, v8.4s[3]
+	OP_ir	v29.4s, v1.4s, v8.s[3]
 
 	ld2	{v12.4s, v13.4s}, [pB]
 	add	pB, pB, #32
@@ -611,85 +611,85 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL4x4_M1
-	OP_rr	v16.4s, v0.4s, v8.4s[0]
-	OP_ii	v16.4s, v1.4s, v9.4s[0]
-	OP_ri	v17.4s, v0.4s, v9.4s[0]
-	OP_ir	v17.4s, v1.4s, v8.4s[0]
+	OP_rr	v16.4s, v0.4s, v8.s[0]
+	OP_ii	v16.4s, v1.4s, v9.s[0]
+	OP_ri	v17.4s, v0.4s, v9.s[0]
+	OP_ir	v17.4s, v1.4s, v8.s[0]
 
 	ld2	{v12.4s, v13.4s}, [pB]		// For next round
 	add	pB, pB, #32
 
-	OP_rr	v20.4s, v0.4s, v8.4s[1]
-	OP_ii	v20.4s, v1.4s, v9.4s[1]
-	OP_ri	v21.4s, v0.4s, v9.4s[1]
-	OP_ir	v21.4s, v1.4s, v8.4s[1]
+	OP_rr	v20.4s, v0.4s, v8.s[1]
+	OP_ii	v20.4s, v1.4s, v9.s[1]
+	OP_ri	v21.4s, v0.4s, v9.s[1]
+	OP_ir	v21.4s, v1.4s, v8.s[1]
 
 	ld2	{v4.4s, v5.4s}, [pA]		// For next round
 	add	pA, pA, #32
 
-	OP_rr	v24.4s, v0.4s, v8.4s[2]
-	OP_ii	v24.4s, v1.4s, v9.4s[2]
-	OP_ri	v25.4s, v0.4s, v9.4s[2]
-	OP_ir	v25.4s, v1.4s, v8.4s[2]
+	OP_rr	v24.4s, v0.4s, v8.s[2]
+	OP_ii	v24.4s, v1.4s, v9.s[2]
+	OP_ri	v25.4s, v0.4s, v9.s[2]
+	OP_ir	v25.4s, v1.4s, v8.s[2]
 
 	prfm	PLDL1KEEP, [pA, #512]
 
-	OP_rr	v28.4s, v0.4s, v8.4s[3]
-	OP_ii	v28.4s, v1.4s, v9.4s[3]
-	OP_ri	v29.4s, v0.4s, v9.4s[3]
-	OP_ir	v29.4s, v1.4s, v8.4s[3]
+	OP_rr	v28.4s, v0.4s, v8.s[3]
+	OP_ii	v28.4s, v1.4s, v9.s[3]
+	OP_ri	v29.4s, v0.4s, v9.s[3]
+	OP_ir	v29.4s, v1.4s, v8.s[3]
 .endm
 
 .macro KERNEL4x4_M2
-	OP_rr	v16.4s, v4.4s, v12.4s[0]
-	OP_ii	v16.4s, v5.4s, v13.4s[0]
-	OP_ri	v17.4s, v4.4s, v13.4s[0]
-	OP_ir	v17.4s, v5.4s, v12.4s[0]
+	OP_rr	v16.4s, v4.4s, v12.s[0]
+	OP_ii	v16.4s, v5.4s, v13.s[0]
+	OP_ri	v17.4s, v4.4s, v13.s[0]
+	OP_ir	v17.4s, v5.4s, v12.s[0]
 
 	ld2	{v8.4s, v9.4s}, [pB]		// For next round
 	add	pB, pB, #32
 
-	OP_rr	v20.4s, v4.4s, v12.4s[1]
-	OP_ii	v20.4s, v5.4s, v13.4s[1]
-	OP_ri	v21.4s, v4.4s, v13.4s[1]
-	OP_ir	v21.4s, v5.4s, v12.4s[1]
+	OP_rr	v20.4s, v4.4s, v12.s[1]
+	OP_ii	v20.4s, v5.4s, v13.s[1]
+	OP_ri	v21.4s, v4.4s, v13.s[1]
+	OP_ir	v21.4s, v5.4s, v12.s[1]
 
 	ld2	{v0.4s, v1.4s}, [pA]		// For next round
 	add	pA, pA, #32
 
-	OP_rr	v24.4s, v4.4s, v12.4s[2]
-	OP_ii	v24.4s, v5.4s, v13.4s[2]
-	OP_ri	v25.4s, v4.4s, v13.4s[2]
-	OP_ir	v25.4s, v5.4s, v12.4s[2]
+	OP_rr	v24.4s, v4.4s, v12.s[2]
+	OP_ii	v24.4s, v5.4s, v13.s[2]
+	OP_ri	v25.4s, v4.4s, v13.s[2]
+	OP_ir	v25.4s, v5.4s, v12.s[2]
 
 	prfm	PLDL1KEEP, [pB, #512]
 
-	OP_rr	v28.4s, v4.4s, v12.4s[3]
-	OP_ii	v28.4s, v5.4s, v13.4s[3]
-	OP_ri	v29.4s, v4.4s, v13.4s[3]
-	OP_ir	v29.4s, v5.4s, v12.4s[3]
+	OP_rr	v28.4s, v4.4s, v12.s[3]
+	OP_ii	v28.4s, v5.4s, v13.s[3]
+	OP_ri	v29.4s, v4.4s, v13.s[3]
+	OP_ir	v29.4s, v5.4s, v12.s[3]
 .endm
 
 .macro KERNEL4x4_E
-	OP_rr	v16.4s, v4.4s, v12.4s[0]
-	OP_ii	v16.4s, v5.4s, v13.4s[0]
-	OP_ri	v17.4s, v4.4s, v13.4s[0]
-	OP_ir	v17.4s, v5.4s, v12.4s[0]
-
-	OP_rr	v20.4s, v4.4s, v12.4s[1]
-	OP_ii	v20.4s, v5.4s, v13.4s[1]
-	OP_ri	v21.4s, v4.4s, v13.4s[1]
-	OP_ir	v21.4s, v5.4s, v12.4s[1]
-
-	OP_rr	v24.4s, v4.4s, v12.4s[2]
-	OP_ii	v24.4s, v5.4s, v13.4s[2]
-	OP_ri	v25.4s, v4.4s, v13.4s[2]
-	OP_ir	v25.4s, v5.4s, v12.4s[2]
-
-	OP_rr	v28.4s, v4.4s, v12.4s[3]
-	OP_ii	v28.4s, v5.4s, v13.4s[3]
-	OP_ri	v29.4s, v4.4s, v13.4s[3]
-	OP_ir	v29.4s, v5.4s, v12.4s[3]
+	OP_rr	v16.4s, v4.4s, v12.s[0]
+	OP_ii	v16.4s, v5.4s, v13.s[0]
+	OP_ri	v17.4s, v4.4s, v13.s[0]
+	OP_ir	v17.4s, v5.4s, v12.s[0]
+
+	OP_rr	v20.4s, v4.4s, v12.s[1]
+	OP_ii	v20.4s, v5.4s, v13.s[1]
+	OP_ri	v21.4s, v4.4s, v13.s[1]
+	OP_ir	v21.4s, v5.4s, v12.s[1]
+
+	OP_rr	v24.4s, v4.4s, v12.s[2]
+	OP_ii	v24.4s, v5.4s, v13.s[2]
+	OP_ri	v25.4s, v4.4s, v13.s[2]
+	OP_ir	v25.4s, v5.4s, v12.s[2]
+
+	OP_rr	v28.4s, v4.4s, v12.s[3]
+	OP_ii	v28.4s, v5.4s, v13.s[3]
+	OP_ri	v29.4s, v4.4s, v13.s[3]
+	OP_ir	v29.4s, v5.4s, v12.s[3]
 .endm
 
 .macro KERNEL4x4_SUB
@@ -698,25 +698,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld2	{v0.4s, v1.4s}, [pA]
 	add	pA, pA, #32
 
-	OP_rr	v16.4s, v0.4s, v8.4s[0]
-	OP_ii	v16.4s, v1.4s, v9.4s[0]
-	OP_ri	v17.4s, v0.4s, v9.4s[0]
-	OP_ir	v17.4s, v1.4s, v8.4s[0]
-
-	OP_rr	v20.4s, v0.4s, v8.4s[1]
-	OP_ii	v20.4s, v1.4s, v9.4s[1]
-	OP_ri	v21.4s, v0.4s, v9.4s[1]
-	OP_ir	v21.4s, v1.4s, v8.4s[1]
-
-	OP_rr	v24.4s, v0.4s, v8.4s[2]
-	OP_ii	v24.4s, v1.4s, v9.4s[2]
-	OP_ri	v25.4s, v0.4s, v9.4s[2]
-	OP_ir	v25.4s, v1.4s, v8.4s[2]
-
-	OP_rr	v28.4s, v0.4s, v8.4s[3]
-	OP_ii	v28.4s, v1.4s, v9.4s[3]
-	OP_ri	v29.4s, v0.4s, v9.4s[3]
-	OP_ir	v29.4s, v1.4s, v8.4s[3]
+	OP_rr	v16.4s, v0.4s, v8.s[0]
+	OP_ii	v16.4s, v1.4s, v9.s[0]
+	OP_ri	v17.4s, v0.4s, v9.s[0]
+	OP_ir	v17.4s, v1.4s, v8.s[0]
+
+	OP_rr	v20.4s, v0.4s, v8.s[1]
+	OP_ii	v20.4s, v1.4s, v9.s[1]
+	OP_ri	v21.4s, v0.4s, v9.s[1]
+	OP_ir	v21.4s, v1.4s, v8.s[1]
+
+	OP_rr	v24.4s, v0.4s, v8.s[2]
+	OP_ii	v24.4s, v1.4s, v9.s[2]
+	OP_ri	v25.4s, v0.4s, v9.s[2]
+	OP_ir	v25.4s, v1.4s, v8.s[2]
+
+	OP_rr	v28.4s, v0.4s, v8.s[3]
+	OP_ii	v28.4s, v1.4s, v9.s[3]
+	OP_ri	v29.4s, v0.4s, v9.s[3]
+	OP_ir	v29.4s, v1.4s, v8.s[3]
 .endm
 
 .macro SAVE4x4
@@ -778,25 +778,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld2	{v0.2s, v1.2s}, [pA]
 	add	pA, pA, #16
 
-	OP_rr	v16.2s, v0.2s, v8.4s[0]
-	OP_ii	v16.2s, v1.2s, v9.4s[0]
-	OP_ri	v17.2s, v0.2s, v9.4s[0]
-	OP_ir	v17.2s, v1.2s, v8.4s[0]
-
-	OP_rr	v20.2s, v0.2s, v8.4s[1]
-	OP_ii	v20.2s, v1.2s, v9.4s[1]
-	OP_ri	v21.2s, v0.2s, v9.4s[1]
-	OP_ir	v21.2s, v1.2s, v8.4s[1]
-
-	OP_rr	v24.2s, v0.2s, v8.4s[2]
-	OP_ii	v24.2s, v1.2s, v9.4s[2]
-	OP_ri	v25.2s, v0.2s, v9.4s[2]
-	OP_ir	v25.2s, v1.2s, v8.4s[2]
-
-	OP_rr	v28.2s, v0.2s, v8.4s[3]
-	OP_ii	v28.2s, v1.2s, v9.4s[3]
-	OP_ri	v29.2s, v0.2s, v9.4s[3]
-	OP_ir	v29.2s, v1.2s, v8.4s[3]
+	OP_rr	v16.2s, v0.2s, v8.s[0]
+	OP_ii	v16.2s, v1.2s, v9.s[0]
+	OP_ri	v17.2s, v0.2s, v9.s[0]
+	OP_ir	v17.2s, v1.2s, v8.s[0]
+
+	OP_rr	v20.2s, v0.2s, v8.s[1]
+	OP_ii	v20.2s, v1.2s, v9.s[1]
+	OP_ri	v21.2s, v0.2s, v9.s[1]
+	OP_ir	v21.2s, v1.2s, v8.s[1]
+
+	OP_rr	v24.2s, v0.2s, v8.s[2]
+	OP_ii	v24.2s, v1.2s, v9.s[2]
+	OP_ri	v25.2s, v0.2s, v9.s[2]
+	OP_ir	v25.2s, v1.2s, v8.s[2]
+
+	OP_rr	v28.2s, v0.2s, v8.s[3]
+	OP_ii	v28.2s, v1.2s, v9.s[3]
+	OP_ri	v29.2s, v0.2s, v9.s[3]
+	OP_ir	v29.2s, v1.2s, v8.s[3]
 .endm
 
 .macro SAVE2x4
@@ -858,25 +858,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld2	{v0.s, v1.s}[0], [pA]
 	add	pA, pA, #8
 
-	OP_rr	s16, s0, v8.4s[0]
-	OP_ii	s16, s1, v9.4s[0]
-	OP_ri	s17, s0, v9.4s[0]
-	OP_ir	s17, s1, v8.4s[0]
-
-	OP_rr	s20, s0, v8.4s[1]
-	OP_ii	s20, s1, v9.4s[1]
-	OP_ri	s21, s0, v9.4s[1]
-	OP_ir	s21, s1, v8.4s[1]
-
-	OP_rr	s24, s0, v8.4s[2]
-	OP_ii	s24, s1, v9.4s[2]
-	OP_ri	s25, s0, v9.4s[2]
-	OP_ir	s25, s1, v8.4s[2]
-
-	OP_rr	s28, s0, v8.4s[3]
-	OP_ii	s28, s1, v9.4s[3]
-	OP_ri	s29, s0, v9.4s[3]
-	OP_ir	s29, s1, v8.4s[3]
+	OP_rr	s16, s0, v8.s[0]
+	OP_ii	s16, s1, v9.s[0]
+	OP_ri	s17, s0, v9.s[0]
+	OP_ir	s17, s1, v8.s[0]
+
+	OP_rr	s20, s0, v8.s[1]
+	OP_ii	s20, s1, v9.s[1]
+	OP_ri	s21, s0, v9.s[1]
+	OP_ir	s21, s1, v8.s[1]
+
+	OP_rr	s24, s0, v8.s[2]
+	OP_ii	s24, s1, v9.s[2]
+	OP_ri	s25, s0, v9.s[2]
+	OP_ir	s25, s1, v8.s[2]
+
+	OP_rr	s28, s0, v8.s[3]
+	OP_ii	s28, s1, v9.s[3]
+	OP_ri	s29, s0, v9.s[3]
+	OP_ir	s29, s1, v8.s[3]
 .endm
 
 .macro SAVE1x4
@@ -940,25 +940,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld2	{v2.4s, v3.4s}, [pA]
 	add	pA, pA, #32
 
-	OP_rr	v16.4s, v0.4s, v8.2s[0]
-	OP_ii	v16.4s, v1.4s, v9.2s[0]
-	OP_ri	v17.4s, v0.4s, v9.2s[0]
-	OP_ir	v17.4s, v1.4s, v8.2s[0]
-
-	OP_rr	v18.4s, v2.4s, v8.2s[0]
-	OP_ii	v18.4s, v3.4s, v9.2s[0]
-	OP_ri	v19.4s, v2.4s, v9.2s[0]
-	OP_ir	v19.4s, v3.4s, v8.2s[0]
-
-	OP_rr	v20.4s, v0.4s, v8.2s[1]
-	OP_ii	v20.4s, v1.4s, v9.2s[1]
-	OP_ri	v21.4s, v0.4s, v9.2s[1]
-	OP_ir	v21.4s, v1.4s, v8.2s[1]
-
-	OP_rr	v22.4s, v2.4s, v8.2s[1]
-	OP_ii	v22.4s, v3.4s, v9.2s[1]
-	OP_ri	v23.4s, v2.4s, v9.2s[1]
-	OP_ir	v23.4s, v3.4s, v8.2s[1]
+	OP_rr	v16.4s, v0.4s, v8.s[0]
+	OP_ii	v16.4s, v1.4s, v9.s[0]
+	OP_ri	v17.4s, v0.4s, v9.s[0]
+	OP_ir	v17.4s, v1.4s, v8.s[0]
+
+	OP_rr	v18.4s, v2.4s, v8.s[0]
+	OP_ii	v18.4s, v3.4s, v9.s[0]
+	OP_ri	v19.4s, v2.4s, v9.s[0]
+	OP_ir	v19.4s, v3.4s, v8.s[0]
+
+	OP_rr	v20.4s, v0.4s, v8.s[1]
+	OP_ii	v20.4s, v1.4s, v9.s[1]
+	OP_ri	v21.4s, v0.4s, v9.s[1]
+	OP_ir	v21.4s, v1.4s, v8.s[1]
+
+	OP_rr	v22.4s, v2.4s, v8.s[1]
+	OP_ii	v22.4s, v3.4s, v9.s[1]
+	OP_ri	v23.4s, v2.4s, v9.s[1]
+	OP_ir	v23.4s, v3.4s, v8.s[1]
 .endm
 
 .macro SAVE8x2
@@ -1016,15 +1016,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld2	{v0.4s, v1.4s}, [pA]
 	add	pA, pA, #32
 
-	OP_rr	v16.4s, v0.4s, v8.2s[0]
-	OP_ii	v16.4s, v1.4s, v9.2s[0]
-	OP_ri	v17.4s, v0.4s, v9.2s[0]
-	OP_ir	v17.4s, v1.4s, v8.2s[0]
+	OP_rr	v16.4s, v0.4s, v8.s[0]
+	OP_ii	v16.4s, v1.4s, v9.s[0]
+	OP_ri	v17.4s, v0.4s, v9.s[0]
+	OP_ir	v17.4s, v1.4s, v8.s[0]
 
-	OP_rr	v20.4s, v0.4s, v8.2s[1]
-	OP_ii	v20.4s, v1.4s, v9.2s[1]
-	OP_ri	v21.4s, v0.4s, v9.2s[1]
-	OP_ir	v21.4s, v1.4s, v8.2s[1]
+	OP_rr	v20.4s, v0.4s, v8.s[1]
+	OP_ii	v20.4s, v1.4s, v9.s[1]
+	OP_ri	v21.4s, v0.4s, v9.s[1]
+	OP_ir	v21.4s, v1.4s, v8.s[1]
 .endm
 
 .macro SAVE4x2
@@ -1064,15 +1064,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld2	{v0.2s, v1.2s}, [pA]
 	add	pA, pA, #16
 
-	OP_rr	v16.2s, v0.2s, v8.2s[0]
-	OP_ii	v16.2s, v1.2s, v9.2s[0]
-	OP_ri	v17.2s, v0.2s, v9.2s[0]
-	OP_ir	v17.2s, v1.2s, v8.2s[0]
+	OP_rr	v16.2s, v0.2s, v8.s[0]
+	OP_ii	v16.2s, v1.2s, v9.s[0]
+	OP_ri	v17.2s, v0.2s, v9.s[0]
+	OP_ir	v17.2s, v1.2s, v8.s[0]
 
-	OP_rr	v20.2s, v0.2s, v8.2s[1]
-	OP_ii	v20.2s, v1.2s, v9.2s[1]
-	OP_ri	v21.2s, v0.2s, v9.2s[1]
-	OP_ir	v21.2s, v1.2s, v8.2s[1]
+	OP_rr	v20.2s, v0.2s, v8.s[1]
+	OP_ii	v20.2s, v1.2s, v9.s[1]
+	OP_ri	v21.2s, v0.2s, v9.s[1]
+	OP_ir	v21.2s, v1.2s, v8.s[1]
 .endm
 
 .macro SAVE2x2
@@ -1112,15 +1112,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld2	{v0.s, v1.s}[0], [pA]
 	add	pA, pA, #8
 
-	OP_rr	s16, s0, v8.2s[0]
-	OP_ii	s16, s1, v9.2s[0]
-	OP_ri	s17, s0, v9.2s[0]
-	OP_ir	s17, s1, v8.2s[0]
+	OP_rr	s16, s0, v8.s[0]
+	OP_ii	s16, s1, v9.s[0]
+	OP_ri	s17, s0, v9.s[0]
+	OP_ir	s17, s1, v8.s[0]
 
-	OP_rr	s20, s0, v8.2s[1]
-	OP_ii	s20, s1, v9.2s[1]
-	OP_ri	s21, s0, v9.2s[1]
-	OP_ir	s21, s1, v8.2s[1]
+	OP_rr	s20, s0, v8.s[1]
+	OP_ii	s20, s1, v9.s[1]
+	OP_ri	s21, s0, v9.s[1]
+	OP_ir	s21, s1, v8.s[1]
 .endm
 
 .macro SAVE1x2
@@ -1162,15 +1162,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld2	{v2.4s, v3.4s}, [pA]
 	add	pA, pA, #32
 
-	OP_rr	v16.4s, v0.4s, v8.4s[0]
-	OP_ii	v16.4s, v1.4s, v8.4s[1]
-	OP_ri	v17.4s, v0.4s, v8.4s[1]
-	OP_ir	v17.4s, v1.4s, v8.4s[0]
+	OP_rr	v16.4s, v0.4s, v8.s[0]
+	OP_ii	v16.4s, v1.4s, v8.s[1]
+	OP_ri	v17.4s, v0.4s, v8.s[1]
+	OP_ir	v17.4s, v1.4s, v8.s[0]
 
-	OP_rr	v18.4s, v2.4s, v8.4s[0]
-	OP_ii	v18.4s, v3.4s, v8.4s[1]
-	OP_ri	v19.4s, v2.4s, v8.4s[1]
-	OP_ir	v19.4s, v3.4s, v8.4s[0]
+	OP_rr	v18.4s, v2.4s, v8.s[0]
+	OP_ii	v18.4s, v3.4s, v8.s[1]
+	OP_ri	v19.4s, v2.4s, v8.s[1]
+	OP_ir	v19.4s, v3.4s, v8.s[0]
 .endm
 
 .macro SAVE8x1

kernel/arm64/ctrmm_kernel_4x4.S

@@ -170,49 +170,49 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld2	{v0.4s, v1.4s}, [pA]
 	add	pA, pA, #32
 
-	fmul	v16.4s, v0.4s, v8.4s[0]
-	OP_ii	v16.4s, v1.4s, v9.4s[0]
+	fmul	v16.4s, v0.4s, v8.s[0]
+	OP_ii	v16.4s, v1.4s, v9.s[0]
 #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
     defined(RR) || defined(RC) || defined(CR) || defined(CC)
 	eor	v17.16b, v17.16b, v17.16b
-	fmls	v17.4s, v0.4s, v9.4s[0]
+	fmls	v17.4s, v0.4s, v9.s[0]
 #else
-	fmul	v17.4s, v0.4s, v9.4s[0]
+	fmul	v17.4s, v0.4s, v9.s[0]
 #endif
-	OP_ir	v17.4s, v1.4s, v8.4s[0]
+	OP_ir	v17.4s, v1.4s, v8.s[0]
 
-	fmul	v20.4s, v0.4s, v8.4s[1]
-	OP_ii	v20.4s, v1.4s, v9.4s[1]
+	fmul	v20.4s, v0.4s, v8.s[1]
+	OP_ii	v20.4s, v1.4s, v9.s[1]
 #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
     defined(RR) || defined(RC) || defined(CR) || defined(CC)
 	eor	v21.16b, v21.16b, v21.16b
-	fmls	v21.4s, v0.4s, v9.4s[1]
+	fmls	v21.4s, v0.4s, v9.s[1]
 #else
-	fmul	v21.4s, v0.4s, v9.4s[1]
+	fmul	v21.4s, v0.4s, v9.s[1]
 #endif
-	OP_ir	v21.4s, v1.4s, v8.4s[1]
+	OP_ir	v21.4s, v1.4s, v8.s[1]
 
-	fmul	v24.4s, v0.4s, v8.4s[2]
-	OP_ii	v24.4s, v1.4s, v9.4s[2]
+	fmul	v24.4s, v0.4s, v8.s[2]
+	OP_ii	v24.4s, v1.4s, v9.s[2]
 #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
     defined(RR) || defined(RC) || defined(CR) || defined(CC)
 	eor	v25.16b, v25.16b, v25.16b
-	fmls	v25.4s, v0.4s, v9.4s[2]
+	fmls	v25.4s, v0.4s, v9.s[2]
 #else
-	fmul	v25.4s, v0.4s, v9.4s[2]
+	fmul	v25.4s, v0.4s, v9.s[2]
 #endif
-	OP_ir	v25.4s, v1.4s, v8.4s[2]
+	OP_ir	v25.4s, v1.4s, v8.s[2]
 
-	fmul	v28.4s, v0.4s, v8.4s[3]
-	OP_ii	v28.4s, v1.4s, v9.4s[3]
+	fmul	v28.4s, v0.4s, v8.s[3]
+	OP_ii	v28.4s, v1.4s, v9.s[3]
 #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
     defined(RR) || defined(RC) || defined(CR) || defined(CC)
 	eor	v29.16b, v29.16b, v29.16b
-	fmls	v29.4s, v0.4s, v9.4s[3]
+	fmls	v29.4s, v0.4s, v9.s[3]
 #else
-	fmul	v29.4s, v0.4s, v9.4s[3]
+	fmul	v29.4s, v0.4s, v9.s[3]
 #endif
-	OP_ir	v29.4s, v1.4s, v8.4s[3]
+	OP_ir	v29.4s, v1.4s, v8.s[3]
 
 	ld2	{v12.4s, v13.4s}, [pB]
 	add	pB, pB, #32
@@ -221,85 +221,85 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL4x4_M1
-	OP_rr	v16.4s, v0.4s, v8.4s[0]
-	OP_ii	v16.4s, v1.4s, v9.4s[0]
-	OP_ri	v17.4s, v0.4s, v9.4s[0]
-	OP_ir	v17.4s, v1.4s, v8.4s[0]
+	OP_rr	v16.4s, v0.4s, v8.s[0]
+	OP_ii	v16.4s, v1.4s, v9.s[0]
+	OP_ri	v17.4s, v0.4s, v9.s[0]
+	OP_ir	v17.4s, v1.4s, v8.s[0]
 
 	ld2	{v12.4s, v13.4s}, [pB]		// For next round
 	add	pB, pB, #32
 
-	OP_rr	v20.4s, v0.4s, v8.4s[1]
-	OP_ii	v20.4s, v1.4s, v9.4s[1]
-	OP_ri	v21.4s, v0.4s, v9.4s[1]
-	OP_ir	v21.4s, v1.4s, v8.4s[1]
+	OP_rr	v20.4s, v0.4s, v8.s[1]
+	OP_ii	v20.4s, v1.4s, v9.s[1]
+	OP_ri	v21.4s, v0.4s, v9.s[1]
+	OP_ir	v21.4s, v1.4s, v8.s[1]
 
 	ld2	{v4.4s, v5.4s}, [pA]		// For next round
 	add	pA, pA, #32
 
-	OP_rr	v24.4s, v0.4s, v8.4s[2]
-	OP_ii	v24.4s, v1.4s, v9.4s[2]
-	OP_ri	v25.4s, v0.4s, v9.4s[2]
-	OP_ir	v25.4s, v1.4s, v8.4s[2]
+	OP_rr	v24.4s, v0.4s, v8.s[2]
+	OP_ii	v24.4s, v1.4s, v9.s[2]
+	OP_ri	v25.4s, v0.4s, v9.s[2]
+	OP_ir	v25.4s, v1.4s, v8.s[2]
 
 	prfm	PLDL1KEEP, [pA, #512]
 
-	OP_rr	v28.4s, v0.4s, v8.4s[3]
-	OP_ii	v28.4s, v1.4s, v9.4s[3]
-	OP_ri	v29.4s, v0.4s, v9.4s[3]
-	OP_ir	v29.4s, v1.4s, v8.4s[3]
+	OP_rr	v28.4s, v0.4s, v8.s[3]
+	OP_ii	v28.4s, v1.4s, v9.s[3]
+	OP_ri	v29.4s, v0.4s, v9.s[3]
+	OP_ir	v29.4s, v1.4s, v8.s[3]
 .endm
 
 .macro KERNEL4x4_M2
-	OP_rr	v16.4s, v4.4s, v12.4s[0]
-	OP_ii	v16.4s, v5.4s, v13.4s[0]
-	OP_ri	v17.4s, v4.4s, v13.4s[0]
-	OP_ir	v17.4s, v5.4s, v12.4s[0]
+	OP_rr	v16.4s, v4.4s, v12.s[0]
+	OP_ii	v16.4s, v5.4s, v13.s[0]
+	OP_ri	v17.4s, v4.4s, v13.s[0]
+	OP_ir	v17.4s, v5.4s, v12.s[0]
 
 	ld2	{v8.4s, v9.4s}, [pB]		// For next round
 	add	pB, pB, #32
 
-	OP_rr	v20.4s, v4.4s, v12.4s[1]
-	OP_ii	v20.4s, v5.4s, v13.4s[1]
-	OP_ri	v21.4s, v4.4s, v13.4s[1]
-	OP_ir	v21.4s, v5.4s, v12.4s[1]
+	OP_rr	v20.4s, v4.4s, v12.s[1]
+	OP_ii	v20.4s, v5.4s, v13.s[1]
+	OP_ri	v21.4s, v4.4s, v13.s[1]
+	OP_ir	v21.4s, v5.4s, v12.s[1]
 
 	ld2	{v0.4s, v1.4s}, [pA]		// For next round
 	add	pA, pA, #32
 
-	OP_rr	v24.4s, v4.4s, v12.4s[2]
-	OP_ii	v24.4s, v5.4s, v13.4s[2]
-	OP_ri	v25.4s, v4.4s, v13.4s[2]
-	OP_ir	v25.4s, v5.4s, v12.4s[2]
+	OP_rr	v24.4s, v4.4s, v12.s[2]
+	OP_ii	v24.4s, v5.4s, v13.s[2]
+	OP_ri	v25.4s, v4.4s, v13.s[2]
+	OP_ir	v25.4s, v5.4s, v12.s[2]
 
 	prfm	PLDL1KEEP, [pB, #512]
 
-	OP_rr	v28.4s, v4.4s, v12.4s[3]
-	OP_ii	v28.4s, v5.4s, v13.4s[3]
-	OP_ri	v29.4s, v4.4s, v13.4s[3]
-	OP_ir	v29.4s, v5.4s, v12.4s[3]
+	OP_rr	v28.4s, v4.4s, v12.s[3]
+	OP_ii	v28.4s, v5.4s, v13.s[3]
+	OP_ri	v29.4s, v4.4s, v13.s[3]
+	OP_ir	v29.4s, v5.4s, v12.s[3]
 .endm
 
 .macro KERNEL4x4_E
-	OP_rr	v16.4s, v4.4s, v12.4s[0]
-	OP_ii	v16.4s, v5.4s, v13.4s[0]
-	OP_ri	v17.4s, v4.4s, v13.4s[0]
-	OP_ir	v17.4s, v5.4s, v12.4s[0]
-
-	OP_rr	v20.4s, v4.4s, v12.4s[1]
-	OP_ii	v20.4s, v5.4s, v13.4s[1]
-	OP_ri	v21.4s, v4.4s, v13.4s[1]
-	OP_ir	v21.4s, v5.4s, v12.4s[1]
-
-	OP_rr	v24.4s, v4.4s, v12.4s[2]
-	OP_ii	v24.4s, v5.4s, v13.4s[2]
-	OP_ri	v25.4s, v4.4s, v13.4s[2]
-	OP_ir	v25.4s, v5.4s, v12.4s[2]
-
-	OP_rr	v28.4s, v4.4s, v12.4s[3]
-	OP_ii	v28.4s, v5.4s, v13.4s[3]
-	OP_ri	v29.4s, v4.4s, v13.4s[3]
-	OP_ir	v29.4s, v5.4s, v12.4s[3]
+	OP_rr	v16.4s, v4.4s, v12.s[0]
+	OP_ii	v16.4s, v5.4s, v13.s[0]
+	OP_ri	v17.4s, v4.4s, v13.s[0]
+	OP_ir	v17.4s, v5.4s, v12.s[0]
+
+	OP_rr	v20.4s, v4.4s, v12.s[1]
+	OP_ii	v20.4s, v5.4s, v13.s[1]
+	OP_ri	v21.4s, v4.4s, v13.s[1]
+	OP_ir	v21.4s, v5.4s, v12.s[1]
+
+	OP_rr	v24.4s, v4.4s, v12.s[2]
+	OP_ii	v24.4s, v5.4s, v13.s[2]
+	OP_ri	v25.4s, v4.4s, v13.s[2]
+	OP_ir	v25.4s, v5.4s, v12.s[2]
+
+	OP_rr	v28.4s, v4.4s, v12.s[3]
+	OP_ii	v28.4s, v5.4s, v13.s[3]
+	OP_ri	v29.4s, v4.4s, v13.s[3]
+	OP_ir	v29.4s, v5.4s, v12.s[3]
 .endm
 
 .macro KERNEL4x4_SUB
@@ -308,25 +308,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld2	{v0.4s, v1.4s}, [pA]
 	add	pA, pA, #32
 
-	OP_rr	v16.4s, v0.4s, v8.4s[0]
-	OP_ii	v16.4s, v1.4s, v9.4s[0]
-	OP_ri	v17.4s, v0.4s, v9.4s[0]
-	OP_ir	v17.4s, v1.4s, v8.4s[0]
-
-	OP_rr	v20.4s, v0.4s, v8.4s[1]
-	OP_ii	v20.4s, v1.4s, v9.4s[1]
-	OP_ri	v21.4s, v0.4s, v9.4s[1]
-	OP_ir	v21.4s, v1.4s, v8.4s[1]
-
-	OP_rr	v24.4s, v0.4s, v8.4s[2]
-	OP_ii	v24.4s, v1.4s, v9.4s[2]
-	OP_ri	v25.4s, v0.4s, v9.4s[2]
-	OP_ir	v25.4s, v1.4s, v8.4s[2]
-
-	OP_rr	v28.4s, v0.4s, v8.4s[3]
-	OP_ii	v28.4s, v1.4s, v9.4s[3]
-	OP_ri	v29.4s, v0.4s, v9.4s[3]
-	OP_ir	v29.4s, v1.4s, v8.4s[3]
+	OP_rr	v16.4s, v0.4s, v8.s[0]
+	OP_ii	v16.4s, v1.4s, v9.s[0]
+	OP_ri	v17.4s, v0.4s, v9.s[0]
+	OP_ir	v17.4s, v1.4s, v8.s[0]
+
+	OP_rr	v20.4s, v0.4s, v8.s[1]
+	OP_ii	v20.4s, v1.4s, v9.s[1]
+	OP_ri	v21.4s, v0.4s, v9.s[1]
+	OP_ir	v21.4s, v1.4s, v8.s[1]
+
+	OP_rr	v24.4s, v0.4s, v8.s[2]
+	OP_ii	v24.4s, v1.4s, v9.s[2]
+	OP_ri	v25.4s, v0.4s, v9.s[2]
+	OP_ir	v25.4s, v1.4s, v8.s[2]
+
+	OP_rr	v28.4s, v0.4s, v8.s[3]
+	OP_ii	v28.4s, v1.4s, v9.s[3]
+	OP_ri	v29.4s, v0.4s, v9.s[3]
+	OP_ir	v29.4s, v1.4s, v8.s[3]
 .endm
 
 .macro SAVE4x4
@@ -384,25 +384,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld2	{v0.2s, v1.2s}, [pA]
 	add	pA, pA, #16
 
-	OP_rr	v16.2s, v0.2s, v8.4s[0]
-	OP_ii	v16.2s, v1.2s, v9.4s[0]
-	OP_ri	v17.2s, v0.2s, v9.4s[0]
-	OP_ir	v17.2s, v1.2s, v8.4s[0]
-
-	OP_rr	v20.2s, v0.2s, v8.4s[1]
-	OP_ii	v20.2s, v1.2s, v9.4s[1]
-	OP_ri	v21.2s, v0.2s, v9.4s[1]
-	OP_ir	v21.2s, v1.2s, v8.4s[1]
-
-	OP_rr	v24.2s, v0.2s, v8.4s[2]
-	OP_ii	v24.2s, v1.2s, v9.4s[2]
-	OP_ri	v25.2s, v0.2s, v9.4s[2]
-	OP_ir	v25.2s, v1.2s, v8.4s[2]
-
-	OP_rr	v28.2s, v0.2s, v8.4s[3]
-	OP_ii	v28.2s, v1.2s, v9.4s[3]
-	OP_ri	v29.2s, v0.2s, v9.4s[3]
-	OP_ir	v29.2s, v1.2s, v8.4s[3]
+	OP_rr	v16.2s, v0.2s, v8.s[0]
+	OP_ii	v16.2s, v1.2s, v9.s[0]
+	OP_ri	v17.2s, v0.2s, v9.s[0]
+	OP_ir	v17.2s, v1.2s, v8.s[0]
+
+	OP_rr	v20.2s, v0.2s, v8.s[1]
+	OP_ii	v20.2s, v1.2s, v9.s[1]
+	OP_ri	v21.2s, v0.2s, v9.s[1]
+	OP_ir	v21.2s, v1.2s, v8.s[1]
+
+	OP_rr	v24.2s, v0.2s, v8.s[2]
+	OP_ii	v24.2s, v1.2s, v9.s[2]
+	OP_ri	v25.2s, v0.2s, v9.s[2]
+	OP_ir	v25.2s, v1.2s, v8.s[2]
+
+	OP_rr	v28.2s, v0.2s, v8.s[3]
+	OP_ii	v28.2s, v1.2s, v9.s[3]
+	OP_ri	v29.2s, v0.2s, v9.s[3]
+	OP_ir	v29.2s, v1.2s, v8.s[3]
 .endm
 
 .macro SAVE2x4
@@ -460,25 +460,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld2	{v0.s, v1.s}[0], [pA]
 	add	pA, pA, #8
 
-	OP_rr	s16, s0, v8.4s[0]
-	OP_ii	s16, s1, v9.4s[0]
-	OP_ri	s17, s0, v9.4s[0]
-	OP_ir	s17, s1, v8.4s[0]
-
-	OP_rr	s20, s0, v8.4s[1]
-	OP_ii	s20, s1, v9.4s[1]
-	OP_ri	s21, s0, v9.4s[1]
-	OP_ir	s21, s1, v8.4s[1]
-
-	OP_rr	s24, s0, v8.4s[2]
-	OP_ii	s24, s1, v9.4s[2]
-	OP_ri	s25, s0, v9.4s[2]
-	OP_ir	s25, s1, v8.4s[2]
-
-	OP_rr	s28, s0, v8.4s[3]
-	OP_ii	s28, s1, v9.4s[3]
-	OP_ri	s29, s0, v9.4s[3]
-	OP_ir	s29, s1, v8.4s[3]
+	OP_rr	s16, s0, v8.s[0]
+	OP_ii	s16, s1, v9.s[0]
+	OP_ri	s17, s0, v9.s[0]
+	OP_ir	s17, s1, v8.s[0]
+
+	OP_rr	s20, s0, v8.s[1]
+	OP_ii	s20, s1, v9.s[1]
+	OP_ri	s21, s0, v9.s[1]
+	OP_ir	s21, s1, v8.s[1]
+
+	OP_rr	s24, s0, v8.s[2]
+	OP_ii	s24, s1, v9.s[2]
+	OP_ri	s25, s0, v9.s[2]
+	OP_ir	s25, s1, v8.s[2]
+
+	OP_rr	s28, s0, v8.s[3]
+	OP_ii	s28, s1, v9.s[3]
+	OP_ri	s29, s0, v9.s[3]
+	OP_ir	s29, s1, v8.s[3]
 .endm
 
 .macro SAVE1x4
@@ -532,15 +532,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld2	{v0.4s, v1.4s}, [pA]
 	add	pA, pA, #32
 
-	OP_rr	v16.4s, v0.4s, v8.2s[0]
-	OP_ii	v16.4s, v1.4s, v9.2s[0]
-	OP_ri	v17.4s, v0.4s, v9.2s[0]
-	OP_ir	v17.4s, v1.4s, v8.2s[0]
+	OP_rr	v16.4s, v0.4s, v8.s[0]
+	OP_ii	v16.4s, v1.4s, v9.s[0]
+	OP_ri	v17.4s, v0.4s, v9.s[0]
+	OP_ir	v17.4s, v1.4s, v8.s[0]
 
-	OP_rr	v20.4s, v0.4s, v8.2s[1]
-	OP_ii	v20.4s, v1.4s, v9.2s[1]
-	OP_ri	v21.4s, v0.4s, v9.2s[1]
-	OP_ir	v21.4s, v1.4s, v8.2s[1]
+	OP_rr	v20.4s, v0.4s, v8.s[1]
+	OP_ii	v20.4s, v1.4s, v9.s[1]
+	OP_ri	v21.4s, v0.4s, v9.s[1]
+	OP_ir	v21.4s, v1.4s, v8.s[1]
 .endm
 
 .macro SAVE4x2
@@ -578,15 +578,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld2	{v0.2s, v1.2s}, [pA]
 	add	pA, pA, #16
 
-	OP_rr	v16.2s, v0.2s, v8.2s[0]
-	OP_ii	v16.2s, v1.2s, v9.2s[0]
-	OP_ri	v17.2s, v0.2s, v9.2s[0]
-	OP_ir	v17.2s, v1.2s, v8.2s[0]
+	OP_rr	v16.2s, v0.2s, v8.s[0]
+	OP_ii	v16.2s, v1.2s, v9.s[0]
+	OP_ri	v17.2s, v0.2s, v9.s[0]
+	OP_ir	v17.2s, v1.2s, v8.s[0]
 
-	OP_rr	v20.2s, v0.2s, v8.2s[1]
-	OP_ii	v20.2s, v1.2s, v9.2s[1]
-	OP_ri	v21.2s, v0.2s, v9.2s[1]
-	OP_ir	v21.2s, v1.2s, v8.2s[1]
+	OP_rr	v20.2s, v0.2s, v8.s[1]
+	OP_ii	v20.2s, v1.2s, v9.s[1]
+	OP_ri	v21.2s, v0.2s, v9.s[1]
+	OP_ir	v21.2s, v1.2s, v8.s[1]
 .endm
 
 .macro SAVE2x2
@@ -624,15 +624,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld2	{v0.s, v1.s}[0], [pA]
 	add	pA, pA, #8
 
-	OP_rr	s16, s0, v8.2s[0]
-	OP_ii	s16, s1, v9.2s[0]
-	OP_ri	s17, s0, v9.2s[0]
-	OP_ir	s17, s1, v8.2s[0]
+	OP_rr	s16, s0, v8.s[0]
+	OP_ii	s16, s1, v9.s[0]
+	OP_ri	s17, s0, v9.s[0]
+	OP_ir	s17, s1, v8.s[0]
 
-	OP_rr	s20, s0, v8.2s[1]
-	OP_ii	s20, s1, v9.2s[1]
-	OP_ri	s21, s0, v9.2s[1]
-	OP_ir	s21, s1, v8.2s[1]
+	OP_rr	s20, s0, v8.s[1]
+	OP_ii	s20, s1, v9.s[1]
+	OP_ri	s21, s0, v9.s[1]
+	OP_ir	s21, s1, v8.s[1]
 .endm
 
 .macro SAVE1x2

kernel/arm64/ctrmm_kernel_8x4.S

@@ -180,93 +180,93 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld2	{v2.4s, v3.4s}, [pA]
 	add	pA, pA, #32
 
-	fmul	v16.4s, v0.4s, v8.4s[0]
-	OP_ii	v16.4s, v1.4s, v9.4s[0]
+	fmul	v16.4s, v0.4s, v8.s[0]
+	OP_ii	v16.4s, v1.4s, v9.s[0]
 #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
     defined(RR) || defined(RC) || defined(CR) || defined(CC)
 	eor	v17.16b, v17.16b, v17.16b
-	fmls	v17.4s, v0.4s, v9.4s[0]
+	fmls	v17.4s, v0.4s, v9.s[0]
 #else
-	fmul	v17.4s, v0.4s, v9.4s[0]
+	fmul	v17.4s, v0.4s, v9.s[0]
 #endif
-	OP_ir	v17.4s, v1.4s, v8.4s[0]
+	OP_ir	v17.4s, v1.4s, v8.s[0]
 
-	fmul	v18.4s, v2.4s, v8.4s[0]
-	OP_ii	v18.4s, v3.4s, v9.4s[0]
+	fmul	v18.4s, v2.4s, v8.s[0]
+	OP_ii	v18.4s, v3.4s, v9.s[0]
 #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
     defined(RR) || defined(RC) || defined(CR) || defined(CC)
 	eor	v19.16b, v19.16b, v19.16b
-	fmls	v19.4s, v2.4s, v9.4s[0]
+	fmls	v19.4s, v2.4s, v9.s[0]
 #else
-	fmul	v19.4s, v2.4s, v9.4s[0]
+	fmul	v19.4s, v2.4s, v9.s[0]
 #endif
-	OP_ir	v19.4s, v3.4s, v8.4s[0]
+	OP_ir	v19.4s, v3.4s, v8.s[0]
 
-	fmul	v20.4s, v0.4s, v8.4s[1]
-	OP_ii	v20.4s, v1.4s, v9.4s[1]
+	fmul	v20.4s, v0.4s, v8.s[1]
+	OP_ii	v20.4s, v1.4s, v9.s[1]
 #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
     defined(RR) || defined(RC) || defined(CR) || defined(CC)
 	eor	v21.16b, v21.16b, v21.16b
-	fmls	v21.4s, v0.4s, v9.4s[1]
+	fmls	v21.4s, v0.4s, v9.s[1]
 #else
-	fmul	v21.4s, v0.4s, v9.4s[1]
+	fmul	v21.4s, v0.4s, v9.s[1]
 #endif
-	OP_ir	v21.4s, v1.4s, v8.4s[1]
+	OP_ir	v21.4s, v1.4s, v8.s[1]
 
-	fmul	v22.4s, v2.4s, v8.4s[1]
-	OP_ii	v22.4s, v3.4s, v9.4s[1]
+	fmul	v22.4s, v2.4s, v8.s[1]
+	OP_ii	v22.4s, v3.4s, v9.s[1]
 #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
     defined(RR) || defined(RC) || defined(CR) || defined(CC)
 	eor	v23.16b, v23.16b, v23.16b
-	fmls	v23.4s, v2.4s, v9.4s[1]
+	fmls	v23.4s, v2.4s, v9.s[1]
 #else
-	fmul	v23.4s, v2.4s, v9.4s[1]
+	fmul	v23.4s, v2.4s, v9.s[1]
 #endif
-	OP_ir	v23.4s, v3.4s, v8.4s[1]
+	OP_ir	v23.4s, v3.4s, v8.s[1]
 
-	fmul	v24.4s, v0.4s, v8.4s[2]
-	OP_ii	v24.4s, v1.4s, v9.4s[2]
+	fmul	v24.4s, v0.4s, v8.s[2]
+	OP_ii	v24.4s, v1.4s, v9.s[2]
 #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
     defined(RR) || defined(RC) || defined(CR) || defined(CC)
 	eor	v25.16b, v25.16b, v25.16b
-	fmls	v25.4s, v0.4s, v9.4s[2]
+	fmls	v25.4s, v0.4s, v9.s[2]
 #else
-	fmul	v25.4s, v0.4s, v9.4s[2]
+	fmul	v25.4s, v0.4s, v9.s[2]
 #endif
-	OP_ir	v25.4s, v1.4s, v8.4s[2]
+	OP_ir	v25.4s, v1.4s, v8.s[2]
 
-	fmul	v26.4s, v2.4s, v8.4s[2]
-	OP_ii	v26.4s, v3.4s, v9.4s[2]
+	fmul	v26.4s, v2.4s, v8.s[2]
+	OP_ii	v26.4s, v3.4s, v9.s[2]
 #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
     defined(RR) || defined(RC) || defined(CR) || defined(CC)
 	eor	v27.16b, v27.16b, v27.16b
-	fmls	v27.4s, v2.4s, v9.4s[2]
+	fmls	v27.4s, v2.4s, v9.s[2]
 #else
-	fmul	v27.4s, v2.4s, v9.4s[2]
+	fmul	v27.4s, v2.4s, v9.s[2]
 #endif
-	OP_ir	v27.4s, v3.4s, v8.4s[2]
+	OP_ir	v27.4s, v3.4s, v8.s[2]
 
-	fmul	v28.4s, v0.4s, v8.4s[3]
-	OP_ii	v28.4s, v1.4s, v9.4s[3]
+	fmul	v28.4s, v0.4s, v8.s[3]
+	OP_ii	v28.4s, v1.4s, v9.s[3]
 #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
     defined(RR) || defined(RC) || defined(CR) || defined(CC)
 	eor	v29.16b, v29.16b, v29.16b
-	fmls	v29.4s, v0.4s, v9.4s[3]
+	fmls	v29.4s, v0.4s, v9.s[3]
 #else
-	fmul	v29.4s, v0.4s, v9.4s[3]
+	fmul	v29.4s, v0.4s, v9.s[3]
 #endif
-	OP_ir	v29.4s, v1.4s, v8.4s[3]
+	OP_ir	v29.4s, v1.4s, v8.s[3]
 
-	fmul	v30.4s, v2.4s, v8.4s[3]
-	OP_ii	v30.4s, v3.4s, v9.4s[3]
+	fmul	v30.4s, v2.4s, v8.s[3]
+	OP_ii	v30.4s, v3.4s, v9.s[3]
 #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
     defined(RR) || defined(RC) || defined(CR) || defined(CC)
 	eor	v31.16b, v31.16b, v31.16b
-	fmls	v31.4s, v2.4s, v9.4s[3]
+	fmls	v31.4s, v2.4s, v9.s[3]
 #else
-	fmul	v31.4s, v2.4s, v9.4s[3]
+	fmul	v31.4s, v2.4s, v9.s[3]
 #endif
-	OP_ir	v31.4s, v3.4s, v8.4s[3]
+	OP_ir	v31.4s, v3.4s, v8.s[3]
 
 	ld2	{v12.4s, v13.4s}, [pB]
 	add	pB, pB, #32
@@ -277,45 +277,45 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL8x4_M1
-	OP_rr	v16.4s, v0.4s, v8.4s[0]
-	OP_ii	v16.4s, v1.4s, v9.4s[0]
-	OP_ri	v17.4s, v0.4s, v9.4s[0]
-	OP_ir	v17.4s, v1.4s, v8.4s[0]
-
-	OP_rr	v18.4s, v2.4s, v8.4s[0]
-	OP_ii	v18.4s, v3.4s, v9.4s[0]
-	OP_ri	v19.4s, v2.4s, v9.4s[0]
-	OP_ir	v19.4s, v3.4s, v8.4s[0]
-
-	OP_rr	v20.4s, v0.4s, v8.4s[1]
-	OP_ii	v20.4s, v1.4s, v9.4s[1]
-	OP_ri	v21.4s, v0.4s, v9.4s[1]
-	OP_ir	v21.4s, v1.4s, v8.4s[1]
-
-	OP_rr	v22.4s, v2.4s, v8.4s[1]
-	OP_ii	v22.4s, v3.4s, v9.4s[1]
-	OP_ri	v23.4s, v2.4s, v9.4s[1]
-	OP_ir	v23.4s, v3.4s, v8.4s[1]
-
-	OP_rr	v24.4s, v0.4s, v8.4s[2]
-	OP_ii	v24.4s, v1.4s, v9.4s[2]
-	OP_ri	v25.4s, v0.4s, v9.4s[2]
-	OP_ir	v25.4s, v1.4s, v8.4s[2]
-
-	OP_rr	v26.4s, v2.4s, v8.4s[2]
-	OP_ii	v26.4s, v3.4s, v9.4s[2]
-	OP_ri	v27.4s, v2.4s, v9.4s[2]
-	OP_ir	v27.4s, v3.4s, v8.4s[2]
-
-	OP_rr	v28.4s, v0.4s, v8.4s[3]
-	OP_ii	v28.4s, v1.4s, v9.4s[3]
-	OP_ri	v29.4s, v0.4s, v9.4s[3]
-	OP_ir	v29.4s, v1.4s, v8.4s[3]
-
-	OP_rr	v30.4s, v2.4s, v8.4s[3]
-	OP_ii	v30.4s, v3.4s, v9.4s[3]
-	OP_ri	v31.4s, v2.4s, v9.4s[3]
-	OP_ir	v31.4s, v3.4s, v8.4s[3]
+	OP_rr	v16.4s, v0.4s, v8.s[0]
+	OP_ii	v16.4s, v1.4s, v9.s[0]
+	OP_ri	v17.4s, v0.4s, v9.s[0]
+	OP_ir	v17.4s, v1.4s, v8.s[0]
+
+	OP_rr	v18.4s, v2.4s, v8.s[0]
+	OP_ii	v18.4s, v3.4s, v9.s[0]
+	OP_ri	v19.4s, v2.4s, v9.s[0]
+	OP_ir	v19.4s, v3.4s, v8.s[0]
+
+	OP_rr	v20.4s, v0.4s, v8.s[1]
+	OP_ii	v20.4s, v1.4s, v9.s[1]
+	OP_ri	v21.4s, v0.4s, v9.s[1]
+	OP_ir	v21.4s, v1.4s, v8.s[1]
+
+	OP_rr	v22.4s, v2.4s, v8.s[1]
+	OP_ii	v22.4s, v3.4s, v9.s[1]
+	OP_ri	v23.4s, v2.4s, v9.s[1]
+	OP_ir	v23.4s, v3.4s, v8.s[1]
+
+	OP_rr	v24.4s, v0.4s, v8.s[2]
+	OP_ii	v24.4s, v1.4s, v9.s[2]
+	OP_ri	v25.4s, v0.4s, v9.s[2]
+	OP_ir	v25.4s, v1.4s, v8.s[2]
+
+	OP_rr	v26.4s, v2.4s, v8.s[2]
+	OP_ii	v26.4s, v3.4s, v9.s[2]
+	OP_ri	v27.4s, v2.4s, v9.s[2]
+	OP_ir	v27.4s, v3.4s, v8.s[2]
+
+	OP_rr	v28.4s, v0.4s, v8.s[3]
+	OP_ii	v28.4s, v1.4s, v9.s[3]
+	OP_ri	v29.4s, v0.4s, v9.s[3]
+	OP_ir	v29.4s, v1.4s, v8.s[3]
+
+	OP_rr	v30.4s, v2.4s, v8.s[3]
+	OP_ii	v30.4s, v3.4s, v9.s[3]
+	OP_ri	v31.4s, v2.4s, v9.s[3]
+	OP_ir	v31.4s, v3.4s, v8.s[3]
 
 	ld2	{v12.4s, v13.4s}, [pB]		// For next round
 	add	pB, pB, #32
@@ -326,45 +326,45 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL8x4_M2
-	OP_rr	v16.4s, v4.4s, v12.4s[0]
-	OP_ii	v16.4s, v5.4s, v13.4s[0]
-	OP_ri	v17.4s, v4.4s, v13.4s[0]
-	OP_ir	v17.4s, v5.4s, v12.4s[0]
-
-	OP_rr	v18.4s, v6.4s, v12.4s[0]
-	OP_ii	v18.4s, v7.4s, v13.4s[0]
-	OP_ri	v19.4s, v6.4s, v13.4s[0]
-	OP_ir	v19.4s, v7.4s, v12.4s[0]
-
-	OP_rr	v20.4s, v4.4s, v12.4s[1]
-	OP_ii	v20.4s, v5.4s, v13.4s[1]
-	OP_ri	v21.4s, v4.4s, v13.4s[1]
-	OP_ir	v21.4s, v5.4s, v12.4s[1]
-
-	OP_rr	v22.4s, v6.4s, v12.4s[1]
-	OP_ii	v22.4s, v7.4s, v13.4s[1]
-	OP_ri	v23.4s, v6.4s, v13.4s[1]
-	OP_ir	v23.4s, v7.4s, v12.4s[1]
-
-	OP_rr	v24.4s, v4.4s, v12.4s[2]
-	OP_ii	v24.4s, v5.4s, v13.4s[2]
-	OP_ri	v25.4s, v4.4s, v13.4s[2]
-	OP_ir	v25.4s, v5.4s, v12.4s[2]
-
-	OP_rr	v26.4s, v6.4s, v12.4s[2]
-	OP_ii	v26.4s, v7.4s, v13.4s[2]
-	OP_ri	v27.4s, v6.4s, v13.4s[2]
-	OP_ir	v27.4s, v7.4s, v12.4s[2]
-
-	OP_rr	v28.4s, v4.4s, v12.4s[3]
-	OP_ii	v28.4s, v5.4s, v13.4s[3]
-	OP_ri	v29.4s, v4.4s, v13.4s[3]
-	OP_ir	v29.4s, v5.4s, v12.4s[3]
-
-	OP_rr	v30.4s, v6.4s, v12.4s[3]
-	OP_ii	v30.4s, v7.4s, v13.4s[3]
-	OP_ri	v31.4s, v6.4s, v13.4s[3]
-	OP_ir	v31.4s, v7.4s, v12.4s[3]
+	OP_rr	v16.4s, v4.4s, v12.s[0]
+	OP_ii	v16.4s, v5.4s, v13.s[0]
+	OP_ri	v17.4s, v4.4s, v13.s[0]
+	OP_ir	v17.4s, v5.4s, v12.s[0]
+
+	OP_rr	v18.4s, v6.4s, v12.s[0]
+	OP_ii	v18.4s, v7.4s, v13.s[0]
+	OP_ri	v19.4s, v6.4s, v13.s[0]
+	OP_ir	v19.4s, v7.4s, v12.s[0]
+
+	OP_rr	v20.4s, v4.4s, v12.s[1]
+	OP_ii	v20.4s, v5.4s, v13.s[1]
+	OP_ri	v21.4s, v4.4s, v13.s[1]
+	OP_ir	v21.4s, v5.4s, v12.s[1]
+
+	OP_rr	v22.4s, v6.4s, v12.s[1]
+	OP_ii	v22.4s, v7.4s, v13.s[1]
+	OP_ri	v23.4s, v6.4s, v13.s[1]
+	OP_ir	v23.4s, v7.4s, v12.s[1]
+
+	OP_rr	v24.4s, v4.4s, v12.s[2]
+	OP_ii	v24.4s, v5.4s, v13.s[2]
+	OP_ri	v25.4s, v4.4s, v13.s[2]
+	OP_ir	v25.4s, v5.4s, v12.s[2]
+
+	OP_rr	v26.4s, v6.4s, v12.s[2]
+	OP_ii	v26.4s, v7.4s, v13.s[2]
+	OP_ri	v27.4s, v6.4s, v13.s[2]
+	OP_ir	v27.4s, v7.4s, v12.s[2]
+
+	OP_rr	v28.4s, v4.4s, v12.s[3]
+	OP_ii	v28.4s, v5.4s, v13.s[3]
+	OP_ri	v29.4s, v4.4s, v13.s[3]
+	OP_ir	v29.4s, v5.4s, v12.s[3]
+
+	OP_rr	v30.4s, v6.4s, v12.s[3]
+	OP_ii	v30.4s, v7.4s, v13.s[3]
+	OP_ri	v31.4s, v6.4s, v13.s[3]
+	OP_ir	v31.4s, v7.4s, v12.s[3]
 
 	ld2	{v8.4s, v9.4s}, [pB]
 	add	pB, pB, #32
@@ -375,45 +375,45 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL8x4_E
-	OP_rr	v16.4s, v4.4s, v12.4s[0]
-	OP_ii	v16.4s, v5.4s, v13.4s[0]
-	OP_ri	v17.4s, v4.4s, v13.4s[0]
-	OP_ir	v17.4s, v5.4s, v12.4s[0]
-
-	OP_rr	v18.4s, v6.4s, v12.4s[0]
-	OP_ii	v18.4s, v7.4s, v13.4s[0]
-	OP_ri	v19.4s, v6.4s, v13.4s[0]
-	OP_ir	v19.4s, v7.4s, v12.4s[0]
-
-	OP_rr	v20.4s, v4.4s, v12.4s[1]
-	OP_ii	v20.4s, v5.4s, v13.4s[1]
-	OP_ri	v21.4s, v4.4s, v13.4s[1]
-	OP_ir	v21.4s, v5.4s, v12.4s[1]
-
-	OP_rr	v22.4s, v6.4s, v12.4s[1]
-	OP_ii	v22.4s, v7.4s, v13.4s[1]
-	OP_ri	v23.4s, v6.4s, v13.4s[1]
-	OP_ir	v23.4s, v7.4s, v12.4s[1]
-
-	OP_rr	v24.4s, v4.4s, v12.4s[2]
-	OP_ii	v24.4s, v5.4s, v13.4s[2]
-	OP_ri	v25.4s, v4.4s, v13.4s[2]
-	OP_ir	v25.4s, v5.4s, v12.4s[2]
-
-	OP_rr	v26.4s, v6.4s, v12.4s[2]
-	OP_ii	v26.4s, v7.4s, v13.4s[2]
-	OP_ri	v27.4s, v6.4s, v13.4s[2]
-	OP_ir	v27.4s, v7.4s, v12.4s[2]
-
-	OP_rr	v28.4s, v4.4s, v12.4s[3]
-	OP_ii	v28.4s, v5.4s, v13.4s[3]
-	OP_ri	v29.4s, v4.4s, v13.4s[3]
-	OP_ir	v29.4s, v5.4s, v12.4s[3]
-
-	OP_rr	v30.4s, v6.4s, v12.4s[3]
-	OP_ii	v30.4s, v7.4s, v13.4s[3]
-	OP_ri	v31.4s, v6.4s, v13.4s[3]
-	OP_ir	v31.4s, v7.4s, v12.4s[3]
+	OP_rr	v16.4s, v4.4s, v12.s[0]
+	OP_ii	v16.4s, v5.4s, v13.s[0]
+	OP_ri	v17.4s, v4.4s, v13.s[0]
+	OP_ir	v17.4s, v5.4s, v12.s[0]
+
+	OP_rr	v18.4s, v6.4s, v12.s[0]
+	OP_ii	v18.4s, v7.4s, v13.s[0]
+	OP_ri	v19.4s, v6.4s, v13.s[0]
+	OP_ir	v19.4s, v7.4s, v12.s[0]
+
+	OP_rr	v20.4s, v4.4s, v12.s[1]
+	OP_ii	v20.4s, v5.4s, v13.s[1]
+	OP_ri	v21.4s, v4.4s, v13.s[1]
+	OP_ir	v21.4s, v5.4s, v12.s[1]
+
+	OP_rr	v22.4s, v6.4s, v12.s[1]
+	OP_ii	v22.4s, v7.4s, v13.s[1]
+	OP_ri	v23.4s, v6.4s, v13.s[1]
+	OP_ir	v23.4s, v7.4s, v12.s[1]
+
+	OP_rr	v24.4s, v4.4s, v12.s[2]
+	OP_ii	v24.4s, v5.4s, v13.s[2]
+	OP_ri	v25.4s, v4.4s, v13.s[2]
+	OP_ir	v25.4s, v5.4s, v12.s[2]
+
+	OP_rr	v26.4s, v6.4s, v12.s[2]
+	OP_ii	v26.4s, v7.4s, v13.s[2]
+	OP_ri	v27.4s, v6.4s, v13.s[2]
+	OP_ir	v27.4s, v7.4s, v12.s[2]
+
+	OP_rr	v28.4s, v4.4s, v12.s[3]
+	OP_ii	v28.4s, v5.4s, v13.s[3]
+	OP_ri	v29.4s, v4.4s, v13.s[3]
+	OP_ir	v29.4s, v5.4s, v12.s[3]
+
+	OP_rr	v30.4s, v6.4s, v12.s[3]
+	OP_ii	v30.4s, v7.4s, v13.s[3]
+	OP_ri	v31.4s, v6.4s, v13.s[3]
+	OP_ir	v31.4s, v7.4s, v12.s[3]
 
 .endm
 
@@ -425,45 +425,45 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld2	{v2.4s, v3.4s}, [pA]
 	add	pA, pA, #32
 
-	OP_rr	v16.4s, v0.4s, v8.4s[0]
-	OP_ii	v16.4s, v1.4s, v9.4s[0]
-	OP_ri	v17.4s, v0.4s, v9.4s[0]
-	OP_ir	v17.4s, v1.4s, v8.4s[0]
-
-	OP_rr	v18.4s, v2.4s, v8.4s[0]
-	OP_ii	v18.4s, v3.4s, v9.4s[0]
-	OP_ri	v19.4s, v2.4s, v9.4s[0]
-	OP_ir	v19.4s, v3.4s, v8.4s[0]
-
-	OP_rr	v20.4s, v0.4s, v8.4s[1]
-	OP_ii	v20.4s, v1.4s, v9.4s[1]
-	OP_ri	v21.4s, v0.4s, v9.4s[1]
-	OP_ir	v21.4s, v1.4s, v8.4s[1]
-
-	OP_rr	v22.4s, v2.4s, v8.4s[1]
-	OP_ii	v22.4s, v3.4s, v9.4s[1]
-	OP_ri	v23.4s, v2.4s, v9.4s[1]
-	OP_ir	v23.4s, v3.4s, v8.4s[1]
-
-	OP_rr	v24.4s, v0.4s, v8.4s[2]
-	OP_ii	v24.4s, v1.4s, v9.4s[2]
-	OP_ri	v25.4s, v0.4s, v9.4s[2]
-	OP_ir	v25.4s, v1.4s, v8.4s[2]
-
-	OP_rr	v26.4s, v2.4s, v8.4s[2]
-	OP_ii	v26.4s, v3.4s, v9.4s[2]
-	OP_ri	v27.4s, v2.4s, v9.4s[2]
-	OP_ir	v27.4s, v3.4s, v8.4s[2]
-
-	OP_rr	v28.4s, v0.4s, v8.4s[3]
-	OP_ii	v28.4s, v1.4s, v9.4s[3]
-	OP_ri	v29.4s, v0.4s, v9.4s[3]
-	OP_ir	v29.4s, v1.4s, v8.4s[3]
-
-	OP_rr	v30.4s, v2.4s, v8.4s[3]
-	OP_ii	v30.4s, v3.4s, v9.4s[3]
-	OP_ri	v31.4s, v2.4s, v9.4s[3]
-	OP_ir	v31.4s, v3.4s, v8.4s[3]
+	OP_rr	v16.4s, v0.4s, v8.s[0]
+	OP_ii	v16.4s, v1.4s, v9.s[0]
+	OP_ri	v17.4s, v0.4s, v9.s[0]
+	OP_ir	v17.4s, v1.4s, v8.s[0]
+
+	OP_rr	v18.4s, v2.4s, v8.s[0]
+	OP_ii	v18.4s, v3.4s, v9.s[0]
+	OP_ri	v19.4s, v2.4s, v9.s[0]
+	OP_ir	v19.4s, v3.4s, v8.s[0]
+
+	OP_rr	v20.4s, v0.4s, v8.s[1]
+	OP_ii	v20.4s, v1.4s, v9.s[1]
+	OP_ri	v21.4s, v0.4s, v9.s[1]
+	OP_ir	v21.4s, v1.4s, v8.s[1]
+
+	OP_rr	v22.4s, v2.4s, v8.s[1]
+	OP_ii	v22.4s, v3.4s, v9.s[1]
+	OP_ri	v23.4s, v2.4s, v9.s[1]
+	OP_ir	v23.4s, v3.4s, v8.s[1]
+
+	OP_rr	v24.4s, v0.4s, v8.s[2]
+	OP_ii	v24.4s, v1.4s, v9.s[2]
+	OP_ri	v25.4s, v0.4s, v9.s[2]
+	OP_ir	v25.4s, v1.4s, v8.s[2]
+
+	OP_rr	v26.4s, v2.4s, v8.s[2]
+	OP_ii	v26.4s, v3.4s, v9.s[2]
+	OP_ri	v27.4s, v2.4s, v9.s[2]
+	OP_ir	v27.4s, v3.4s, v8.s[2]
+
+	OP_rr	v28.4s, v0.4s, v8.s[3]
+	OP_ii	v28.4s, v1.4s, v9.s[3]
+	OP_ri	v29.4s, v0.4s, v9.s[3]
+	OP_ir	v29.4s, v1.4s, v8.s[3]
+
+	OP_rr	v30.4s, v2.4s, v8.s[3]
+	OP_ii	v30.4s, v3.4s, v9.s[3]
+	OP_ri	v31.4s, v2.4s, v9.s[3]
+	OP_ir	v31.4s, v3.4s, v8.s[3]
 
 .endm
 
@@ -562,49 +562,49 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld2	{v0.4s, v1.4s}, [pA]
 	add	pA, pA, #32
 
-	fmul	v16.4s, v0.4s, v8.4s[0]
-	OP_ii	v16.4s, v1.4s, v9.4s[0]
+	fmul	v16.4s, v0.4s, v8.s[0]
+	OP_ii	v16.4s, v1.4s, v9.s[0]
 #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
     defined(RR) || defined(RC) || defined(CR) || defined(CC)
 	eor	v17.16b, v17.16b, v17.16b
-	fmls	v17.4s, v0.4s, v9.4s[0]
+	fmls	v17.4s, v0.4s, v9.s[0]
 #else
-	fmul	v17.4s, v0.4s, v9.4s[0]
+	fmul	v17.4s, v0.4s, v9.s[0]
 #endif
-	OP_ir	v17.4s, v1.4s, v8.4s[0]
+	OP_ir	v17.4s, v1.4s, v8.s[0]
 
-	fmul	v20.4s, v0.4s, v8.4s[1]
-	OP_ii	v20.4s, v1.4s, v9.4s[1]
+	fmul	v20.4s, v0.4s, v8.s[1]
+	OP_ii	v20.4s, v1.4s, v9.s[1]
 #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
     defined(RR) || defined(RC) || defined(CR) || defined(CC)
 	eor	v21.16b, v21.16b, v21.16b
-	fmls	v21.4s, v0.4s, v9.4s[1]
+	fmls	v21.4s, v0.4s, v9.s[1]
 #else
-	fmul	v21.4s, v0.4s, v9.4s[1]
+	fmul	v21.4s, v0.4s, v9.s[1]
 #endif
-	OP_ir	v21.4s, v1.4s, v8.4s[1]
+	OP_ir	v21.4s, v1.4s, v8.s[1]
 
-	fmul	v24.4s, v0.4s, v8.4s[2]
-	OP_ii	v24.4s, v1.4s, v9.4s[2]
+	fmul	v24.4s, v0.4s, v8.s[2]
+	OP_ii	v24.4s, v1.4s, v9.s[2]
 #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
     defined(RR) || defined(RC) || defined(CR) || defined(CC)
 	eor	v25.16b, v25.16b, v25.16b
-	fmls	v25.4s, v0.4s, v9.4s[2]
+	fmls	v25.4s, v0.4s, v9.s[2]
 #else
-	fmul	v25.4s, v0.4s, v9.4s[2]
+	fmul	v25.4s, v0.4s, v9.s[2]
 #endif
-	OP_ir	v25.4s, v1.4s, v8.4s[2]
+	OP_ir	v25.4s, v1.4s, v8.s[2]
 
-	fmul	v28.4s, v0.4s, v8.4s[3]
-	OP_ii	v28.4s, v1.4s, v9.4s[3]
+	fmul	v28.4s, v0.4s, v8.s[3]
+	OP_ii	v28.4s, v1.4s, v9.s[3]
 #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
     defined(RR) || defined(RC) || defined(CR) || defined(CC)
 	eor	v29.16b, v29.16b, v29.16b
-	fmls	v29.4s, v0.4s, v9.4s[3]
+	fmls	v29.4s, v0.4s, v9.s[3]
 #else
-	fmul	v29.4s, v0.4s, v9.4s[3]
+	fmul	v29.4s, v0.4s, v9.s[3]
 #endif
-	OP_ir	v29.4s, v1.4s, v8.4s[3]
+	OP_ir	v29.4s, v1.4s, v8.s[3]
 
 	ld2	{v12.4s, v13.4s}, [pB]
 	add	pB, pB, #32
@@ -613,85 +613,85 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL4x4_M1
-	OP_rr	v16.4s, v0.4s, v8.4s[0]
-	OP_ii	v16.4s, v1.4s, v9.4s[0]
-	OP_ri	v17.4s, v0.4s, v9.4s[0]
-	OP_ir	v17.4s, v1.4s, v8.4s[0]
+	OP_rr	v16.4s, v0.4s, v8.s[0]
+	OP_ii	v16.4s, v1.4s, v9.s[0]
+	OP_ri	v17.4s, v0.4s, v9.s[0]
+	OP_ir	v17.4s, v1.4s, v8.s[0]
 
 	ld2	{v12.4s, v13.4s}, [pB]		// For next round
 	add	pB, pB, #32
 
-	OP_rr	v20.4s, v0.4s, v8.4s[1]
-	OP_ii	v20.4s, v1.4s, v9.4s[1]
-	OP_ri	v21.4s, v0.4s, v9.4s[1]
-	OP_ir	v21.4s, v1.4s, v8.4s[1]
+	OP_rr	v20.4s, v0.4s, v8.s[1]
+	OP_ii	v20.4s, v1.4s, v9.s[1]
+	OP_ri	v21.4s, v0.4s, v9.s[1]
+	OP_ir	v21.4s, v1.4s, v8.s[1]
 
 	ld2	{v4.4s, v5.4s}, [pA]		// For next round
 	add	pA, pA, #32
 
-	OP_rr	v24.4s, v0.4s, v8.4s[2]
-	OP_ii	v24.4s, v1.4s, v9.4s[2]
-	OP_ri	v25.4s, v0.4s, v9.4s[2]
-	OP_ir	v25.4s, v1.4s, v8.4s[2]
+	OP_rr	v24.4s, v0.4s, v8.s[2]
+	OP_ii	v24.4s, v1.4s, v9.s[2]
+	OP_ri	v25.4s, v0.4s, v9.s[2]
+	OP_ir	v25.4s, v1.4s, v8.s[2]
 
 	prfm	PLDL1KEEP, [pA, #512]
 
-	OP_rr	v28.4s, v0.4s, v8.4s[3]
-	OP_ii	v28.4s, v1.4s, v9.4s[3]
-	OP_ri	v29.4s, v0.4s, v9.4s[3]
-	OP_ir	v29.4s, v1.4s, v8.4s[3]
+	OP_rr	v28.4s, v0.4s, v8.s[3]
+	OP_ii	v28.4s, v1.4s, v9.s[3]
+	OP_ri	v29.4s, v0.4s, v9.s[3]
+	OP_ir	v29.4s, v1.4s, v8.s[3]
 .endm
 
 .macro KERNEL4x4_M2
-	OP_rr	v16.4s, v4.4s, v12.4s[0]
-	OP_ii	v16.4s, v5.4s, v13.4s[0]
-	OP_ri	v17.4s, v4.4s, v13.4s[0]
-	OP_ir	v17.4s, v5.4s, v12.4s[0]
+	OP_rr	v16.4s, v4.4s, v12.s[0]
+	OP_ii	v16.4s, v5.4s, v13.s[0]
+	OP_ri	v17.4s, v4.4s, v13.s[0]
+	OP_ir	v17.4s, v5.4s, v12.s[0]
 
 	ld2	{v8.4s, v9.4s}, [pB]		// For next round
 	add	pB, pB, #32
 
-	OP_rr	v20.4s, v4.4s, v12.4s[1]
-	OP_ii	v20.4s, v5.4s, v13.4s[1]
-	OP_ri	v21.4s, v4.4s, v13.4s[1]
-	OP_ir	v21.4s, v5.4s, v12.4s[1]
+	OP_rr	v20.4s, v4.4s, v12.s[1]
+	OP_ii	v20.4s, v5.4s, v13.s[1]
+	OP_ri	v21.4s, v4.4s, v13.s[1]
+	OP_ir	v21.4s, v5.4s, v12.s[1]
 
 	ld2	{v0.4s, v1.4s}, [pA]		// For next round
 	add	pA, pA, #32
 
-	OP_rr	v24.4s, v4.4s, v12.4s[2]
-	OP_ii	v24.4s, v5.4s, v13.4s[2]
-	OP_ri	v25.4s, v4.4s, v13.4s[2]
-	OP_ir	v25.4s, v5.4s, v12.4s[2]
+	OP_rr	v24.4s, v4.4s, v12.s[2]
+	OP_ii	v24.4s, v5.4s, v13.s[2]
+	OP_ri	v25.4s, v4.4s, v13.s[2]
+	OP_ir	v25.4s, v5.4s, v12.s[2]
 
 	prfm	PLDL1KEEP, [pB, #512]
 
-	OP_rr	v28.4s, v4.4s, v12.4s[3]
-	OP_ii	v28.4s, v5.4s, v13.4s[3]
-	OP_ri	v29.4s, v4.4s, v13.4s[3]
-	OP_ir	v29.4s, v5.4s, v12.4s[3]
+	OP_rr	v28.4s, v4.4s, v12.s[3]
+	OP_ii	v28.4s, v5.4s, v13.s[3]
+	OP_ri	v29.4s, v4.4s, v13.s[3]
+	OP_ir	v29.4s, v5.4s, v12.s[3]
 .endm
 
 .macro KERNEL4x4_E
-	OP_rr	v16.4s, v4.4s, v12.4s[0]
-	OP_ii	v16.4s, v5.4s, v13.4s[0]
-	OP_ri	v17.4s, v4.4s, v13.4s[0]
-	OP_ir	v17.4s, v5.4s, v12.4s[0]
-
-	OP_rr	v20.4s, v4.4s, v12.4s[1]
-	OP_ii	v20.4s, v5.4s, v13.4s[1]
-	OP_ri	v21.4s, v4.4s, v13.4s[1]
-	OP_ir	v21.4s, v5.4s, v12.4s[1]
-
-	OP_rr	v24.4s, v4.4s, v12.4s[2]
-	OP_ii	v24.4s, v5.4s, v13.4s[2]
-	OP_ri	v25.4s, v4.4s, v13.4s[2]
-	OP_ir	v25.4s, v5.4s, v12.4s[2]
-
-	OP_rr	v28.4s, v4.4s, v12.4s[3]
-	OP_ii	v28.4s, v5.4s, v13.4s[3]
-	OP_ri	v29.4s, v4.4s, v13.4s[3]
-	OP_ir	v29.4s, v5.4s, v12.4s[3]
+	OP_rr	v16.4s, v4.4s, v12.s[0]
+	OP_ii	v16.4s, v5.4s, v13.s[0]
+	OP_ri	v17.4s, v4.4s, v13.s[0]
+	OP_ir	v17.4s, v5.4s, v12.s[0]
+
+	OP_rr	v20.4s, v4.4s, v12.s[1]
+	OP_ii	v20.4s, v5.4s, v13.s[1]
+	OP_ri	v21.4s, v4.4s, v13.s[1]
+	OP_ir	v21.4s, v5.4s, v12.s[1]
+
+	OP_rr	v24.4s, v4.4s, v12.s[2]
+	OP_ii	v24.4s, v5.4s, v13.s[2]
+	OP_ri	v25.4s, v4.4s, v13.s[2]
+	OP_ir	v25.4s, v5.4s, v12.s[2]
+
+	OP_rr	v28.4s, v4.4s, v12.s[3]
+	OP_ii	v28.4s, v5.4s, v13.s[3]
+	OP_ri	v29.4s, v4.4s, v13.s[3]
+	OP_ir	v29.4s, v5.4s, v12.s[3]
 .endm
 
 .macro KERNEL4x4_SUB
@@ -700,25 +700,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld2	{v0.4s, v1.4s}, [pA]
 	add	pA, pA, #32
 
-	OP_rr	v16.4s, v0.4s, v8.4s[0]
-	OP_ii	v16.4s, v1.4s, v9.4s[0]
-	OP_ri	v17.4s, v0.4s, v9.4s[0]
-	OP_ir	v17.4s, v1.4s, v8.4s[0]
-
-	OP_rr	v20.4s, v0.4s, v8.4s[1]
-	OP_ii	v20.4s, v1.4s, v9.4s[1]
-	OP_ri	v21.4s, v0.4s, v9.4s[1]
-	OP_ir	v21.4s, v1.4s, v8.4s[1]
-
-	OP_rr	v24.4s, v0.4s, v8.4s[2]
-	OP_ii	v24.4s, v1.4s, v9.4s[2]
-	OP_ri	v25.4s, v0.4s, v9.4s[2]
-	OP_ir	v25.4s, v1.4s, v8.4s[2]
-
-	OP_rr	v28.4s, v0.4s, v8.4s[3]
-	OP_ii	v28.4s, v1.4s, v9.4s[3]
-	OP_ri	v29.4s, v0.4s, v9.4s[3]
-	OP_ir	v29.4s, v1.4s, v8.4s[3]
+	OP_rr	v16.4s, v0.4s, v8.s[0]
+	OP_ii	v16.4s, v1.4s, v9.s[0]
+	OP_ri	v17.4s, v0.4s, v9.s[0]
+	OP_ir	v17.4s, v1.4s, v8.s[0]
+
+	OP_rr	v20.4s, v0.4s, v8.s[1]
+	OP_ii	v20.4s, v1.4s, v9.s[1]
+	OP_ri	v21.4s, v0.4s, v9.s[1]
+	OP_ir	v21.4s, v1.4s, v8.s[1]
+
+	OP_rr	v24.4s, v0.4s, v8.s[2]
+	OP_ii	v24.4s, v1.4s, v9.s[2]
+	OP_ri	v25.4s, v0.4s, v9.s[2]
+	OP_ir	v25.4s, v1.4s, v8.s[2]
+
+	OP_rr	v28.4s, v0.4s, v8.s[3]
+	OP_ii	v28.4s, v1.4s, v9.s[3]
+	OP_ri	v29.4s, v0.4s, v9.s[3]
+	OP_ir	v29.4s, v1.4s, v8.s[3]
 .endm
 
 .macro SAVE4x4
@@ -780,25 +780,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld2	{v0.2s, v1.2s}, [pA]
 	add	pA, pA, #16
 
-	OP_rr	v16.2s, v0.2s, v8.4s[0]
-	OP_ii	v16.2s, v1.2s, v9.4s[0]
-	OP_ri	v17.2s, v0.2s, v9.4s[0]
-	OP_ir	v17.2s, v1.2s, v8.4s[0]
-
-	OP_rr	v20.2s, v0.2s, v8.4s[1]
-	OP_ii	v20.2s, v1.2s, v9.4s[1]
-	OP_ri	v21.2s, v0.2s, v9.4s[1]
-	OP_ir	v21.2s, v1.2s, v8.4s[1]
-
-	OP_rr	v24.2s, v0.2s, v8.4s[2]
-	OP_ii	v24.2s, v1.2s, v9.4s[2]
-	OP_ri	v25.2s, v0.2s, v9.4s[2]
-	OP_ir	v25.2s, v1.2s, v8.4s[2]
-
-	OP_rr	v28.2s, v0.2s, v8.4s[3]
-	OP_ii	v28.2s, v1.2s, v9.4s[3]
-	OP_ri	v29.2s, v0.2s, v9.4s[3]
-	OP_ir	v29.2s, v1.2s, v8.4s[3]
+	OP_rr	v16.2s, v0.2s, v8.s[0]
+	OP_ii	v16.2s, v1.2s, v9.s[0]
+	OP_ri	v17.2s, v0.2s, v9.s[0]
+	OP_ir	v17.2s, v1.2s, v8.s[0]
+
+	OP_rr	v20.2s, v0.2s, v8.s[1]
+	OP_ii	v20.2s, v1.2s, v9.s[1]
+	OP_ri	v21.2s, v0.2s, v9.s[1]
+	OP_ir	v21.2s, v1.2s, v8.s[1]
+
+	OP_rr	v24.2s, v0.2s, v8.s[2]
+	OP_ii	v24.2s, v1.2s, v9.s[2]
+	OP_ri	v25.2s, v0.2s, v9.s[2]
+	OP_ir	v25.2s, v1.2s, v8.s[2]
+
+	OP_rr	v28.2s, v0.2s, v8.s[3]
+	OP_ii	v28.2s, v1.2s, v9.s[3]
+	OP_ri	v29.2s, v0.2s, v9.s[3]
+	OP_ir	v29.2s, v1.2s, v8.s[3]
 .endm
 
 .macro SAVE2x4
@@ -860,25 +860,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld2	{v0.s, v1.s}[0], [pA]
 	add	pA, pA, #8
 
-	OP_rr	s16, s0, v8.4s[0]
-	OP_ii	s16, s1, v9.4s[0]
-	OP_ri	s17, s0, v9.4s[0]
-	OP_ir	s17, s1, v8.4s[0]
-
-	OP_rr	s20, s0, v8.4s[1]
-	OP_ii	s20, s1, v9.4s[1]
-	OP_ri	s21, s0, v9.4s[1]
-	OP_ir	s21, s1, v8.4s[1]
-
-	OP_rr	s24, s0, v8.4s[2]
-	OP_ii	s24, s1, v9.4s[2]
-	OP_ri	s25, s0, v9.4s[2]
-	OP_ir	s25, s1, v8.4s[2]
-
-	OP_rr	s28, s0, v8.4s[3]
-	OP_ii	s28, s1, v9.4s[3]
-	OP_ri	s29, s0, v9.4s[3]
-	OP_ir	s29, s1, v8.4s[3]
+	OP_rr	s16, s0, v8.s[0]
+	OP_ii	s16, s1, v9.s[0]
+	OP_ri	s17, s0, v9.s[0]
+	OP_ir	s17, s1, v8.s[0]
+
+	OP_rr	s20, s0, v8.s[1]
+	OP_ii	s20, s1, v9.s[1]
+	OP_ri	s21, s0, v9.s[1]
+	OP_ir	s21, s1, v8.s[1]
+
+	OP_rr	s24, s0, v8.s[2]
+	OP_ii	s24, s1, v9.s[2]
+	OP_ri	s25, s0, v9.s[2]
+	OP_ir	s25, s1, v8.s[2]
+
+	OP_rr	s28, s0, v8.s[3]
+	OP_ii	s28, s1, v9.s[3]
+	OP_ri	s29, s0, v9.s[3]
+	OP_ir	s29, s1, v8.s[3]
 .endm
 
 .macro SAVE1x4
@@ -942,25 +942,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld2	{v2.4s, v3.4s}, [pA]
 	add	pA, pA, #32
 
-	OP_rr	v16.4s, v0.4s, v8.2s[0]
-	OP_ii	v16.4s, v1.4s, v9.2s[0]
-	OP_ri	v17.4s, v0.4s, v9.2s[0]
-	OP_ir	v17.4s, v1.4s, v8.2s[0]
-
-	OP_rr	v18.4s, v2.4s, v8.2s[0]
-	OP_ii	v18.4s, v3.4s, v9.2s[0]
-	OP_ri	v19.4s, v2.4s, v9.2s[0]
-	OP_ir	v19.4s, v3.4s, v8.2s[0]
-
-	OP_rr	v20.4s, v0.4s, v8.2s[1]
-	OP_ii	v20.4s, v1.4s, v9.2s[1]
-	OP_ri	v21.4s, v0.4s, v9.2s[1]
-	OP_ir	v21.4s, v1.4s, v8.2s[1]
-
-	OP_rr	v22.4s, v2.4s, v8.2s[1]
-	OP_ii	v22.4s, v3.4s, v9.2s[1]
-	OP_ri	v23.4s, v2.4s, v9.2s[1]
-	OP_ir	v23.4s, v3.4s, v8.2s[1]
+	OP_rr	v16.4s, v0.4s, v8.s[0]
+	OP_ii	v16.4s, v1.4s, v9.s[0]
+	OP_ri	v17.4s, v0.4s, v9.s[0]
+	OP_ir	v17.4s, v1.4s, v8.s[0]
+
+	OP_rr	v18.4s, v2.4s, v8.s[0]
+	OP_ii	v18.4s, v3.4s, v9.s[0]
+	OP_ri	v19.4s, v2.4s, v9.s[0]
+	OP_ir	v19.4s, v3.4s, v8.s[0]
+
+	OP_rr	v20.4s, v0.4s, v8.s[1]
+	OP_ii	v20.4s, v1.4s, v9.s[1]
+	OP_ri	v21.4s, v0.4s, v9.s[1]
+	OP_ir	v21.4s, v1.4s, v8.s[1]
+
+	OP_rr	v22.4s, v2.4s, v8.s[1]
+	OP_ii	v22.4s, v3.4s, v9.s[1]
+	OP_ri	v23.4s, v2.4s, v9.s[1]
+	OP_ir	v23.4s, v3.4s, v8.s[1]
 .endm
 
 .macro SAVE8x2
@@ -1018,15 +1018,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld2	{v0.4s, v1.4s}, [pA]
 	add	pA, pA, #32
 
-	OP_rr	v16.4s, v0.4s, v8.2s[0]
-	OP_ii	v16.4s, v1.4s, v9.2s[0]
-	OP_ri	v17.4s, v0.4s, v9.2s[0]
-	OP_ir	v17.4s, v1.4s, v8.2s[0]
+	OP_rr	v16.4s, v0.4s, v8.s[0]
+	OP_ii	v16.4s, v1.4s, v9.s[0]
+	OP_ri	v17.4s, v0.4s, v9.s[0]
+	OP_ir	v17.4s, v1.4s, v8.s[0]
 
-	OP_rr	v20.4s, v0.4s, v8.2s[1]
-	OP_ii	v20.4s, v1.4s, v9.2s[1]
-	OP_ri	v21.4s, v0.4s, v9.2s[1]
-	OP_ir	v21.4s, v1.4s, v8.2s[1]
+	OP_rr	v20.4s, v0.4s, v8.s[1]
+	OP_ii	v20.4s, v1.4s, v9.s[1]
+	OP_ri	v21.4s, v0.4s, v9.s[1]
+	OP_ir	v21.4s, v1.4s, v8.s[1]
 .endm
 
 .macro SAVE4x2
@@ -1066,15 +1066,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld2	{v0.2s, v1.2s}, [pA]
 	add	pA, pA, #16
 
-	OP_rr	v16.2s, v0.2s, v8.2s[0]
-	OP_ii	v16.2s, v1.2s, v9.2s[0]
-	OP_ri	v17.2s, v0.2s, v9.2s[0]
-	OP_ir	v17.2s, v1.2s, v8.2s[0]
+	OP_rr	v16.2s, v0.2s, v8.s[0]
+	OP_ii	v16.2s, v1.2s, v9.s[0]
+	OP_ri	v17.2s, v0.2s, v9.s[0]
+	OP_ir	v17.2s, v1.2s, v8.s[0]
 
-	OP_rr	v20.2s, v0.2s, v8.2s[1]
-	OP_ii	v20.2s, v1.2s, v9.2s[1]
-	OP_ri	v21.2s, v0.2s, v9.2s[1]
-	OP_ir	v21.2s, v1.2s, v8.2s[1]
+	OP_rr	v20.2s, v0.2s, v8.s[1]
+	OP_ii	v20.2s, v1.2s, v9.s[1]
+	OP_ri	v21.2s, v0.2s, v9.s[1]
+	OP_ir	v21.2s, v1.2s, v8.s[1]
 .endm
 
 .macro SAVE2x2
@@ -1114,15 +1114,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld2	{v0.s, v1.s}[0], [pA]
 	add	pA, pA, #8
 
-	OP_rr	s16, s0, v8.2s[0]
-	OP_ii	s16, s1, v9.2s[0]
-	OP_ri	s17, s0, v9.2s[0]
-	OP_ir	s17, s1, v8.2s[0]
+	OP_rr	s16, s0, v8.s[0]
+	OP_ii	s16, s1, v9.s[0]
+	OP_ri	s17, s0, v9.s[0]
+	OP_ir	s17, s1, v8.s[0]
 
-	OP_rr	s20, s0, v8.2s[1]
-	OP_ii	s20, s1, v9.2s[1]
-	OP_ri	s21, s0, v9.2s[1]
-	OP_ir	s21, s1, v8.2s[1]
+	OP_rr	s20, s0, v8.s[1]
+	OP_ii	s20, s1, v9.s[1]
+	OP_ri	s21, s0, v9.s[1]
+	OP_ir	s21, s1, v8.s[1]
 .endm
 
 .macro SAVE1x2
@@ -1164,15 +1164,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld2	{v2.4s, v3.4s}, [pA]
 	add	pA, pA, #32
 
-	OP_rr	v16.4s, v0.4s, v8.4s[0]
-	OP_ii	v16.4s, v1.4s, v8.4s[1]
-	OP_ri	v17.4s, v0.4s, v8.4s[1]
-	OP_ir	v17.4s, v1.4s, v8.4s[0]
+	OP_rr	v16.4s, v0.4s, v8.s[0]
+	OP_ii	v16.4s, v1.4s, v8.s[1]
+	OP_ri	v17.4s, v0.4s, v8.s[1]
+	OP_ir	v17.4s, v1.4s, v8.s[0]
 
-	OP_rr	v18.4s, v2.4s, v8.4s[0]
-	OP_ii	v18.4s, v3.4s, v8.4s[1]
-	OP_ri	v19.4s, v2.4s, v8.4s[1]
-	OP_ir	v19.4s, v3.4s, v8.4s[0]
+	OP_rr	v18.4s, v2.4s, v8.s[0]
+	OP_ii	v18.4s, v3.4s, v8.s[1]
+	OP_ri	v19.4s, v2.4s, v8.s[1]
+	OP_ir	v19.4s, v3.4s, v8.s[0]
 .endm
 
 .macro SAVE8x1

kernel/arm64/dgemm_kernel_4x4.S

@@ -161,150 +161,150 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ldp	q0, q1, [pA]
 	add	pA, pA, #32
 
-	fmul	v16.2d, v0.2d, v8.2d[0]
-	fmul	v29.2d, v1.2d, v11.2d[0]
+	fmul	v16.2d, v0.2d, v8.d[0]
+	fmul	v29.2d, v1.2d, v11.d[0]
 
 	ldp	q2, q3, [ppA]
 	add	ppA, ppA, #32
 
-	fmul	v20.2d, v0.2d, v9.2d[0]
-	fmul	v25.2d, v1.2d, v10.2d[0]
+	fmul	v20.2d, v0.2d, v9.d[0]
+	fmul	v25.2d, v1.2d, v10.d[0]
 
 	prfm	PLDL1KEEP, [pA, #A_PRE_SIZE]
 
-	fmul	v18.2d, v2.2d, v8.2d[0]
-	fmul	v31.2d, v3.2d, v11.2d[0]
+	fmul	v18.2d, v2.2d, v8.d[0]
+	fmul	v31.2d, v3.2d, v11.d[0]
 
 	prfm	PLDL1KEEP, [ppA, #A_PRE_SIZE]
 
-	fmul	v22.2d, v2.2d, v9.2d[0]
-	fmul	v27.2d, v3.2d, v10.2d[0]
+	fmul	v22.2d, v2.2d, v9.d[0]
+	fmul	v27.2d, v3.2d, v10.d[0]
 
 	ldp	d12, d13, [pB]
 	add	pB, pB, #16
 
-	fmul	v24.2d, v0.2d, v10.2d[0]
-	fmul	v21.2d, v1.2d, v9.2d[0]
+	fmul	v24.2d, v0.2d, v10.d[0]
+	fmul	v21.2d, v1.2d, v9.d[0]
 
 	ldp	q4, q5, [pA]		// for next round
 	add	pA, pA, #32
 
-	fmul	v26.2d, v2.2d, v10.2d[0]
-	fmul	v23.2d, v3.2d, v9.2d[0]
+	fmul	v26.2d, v2.2d, v10.d[0]
+	fmul	v23.2d, v3.2d, v9.d[0]
 
 	ldp	q6, q7, [ppA]		// for next round
 	add	ppA, ppA, #32
 
-	fmul	v28.2d, v0.2d, v11.2d[0]
-	fmul	v17.2d, v1.2d, v8.2d[0]
+	fmul	v28.2d, v0.2d, v11.d[0]
+	fmul	v17.2d, v1.2d, v8.d[0]
 
 	ldp	d14, d15, [pB]
 	add	pB, pB, #16
 
-	fmul	v30.2d, v2.2d, v11.2d[0]
-	fmul	v19.2d, v3.2d, v8.2d[0]
+	fmul	v30.2d, v2.2d, v11.d[0]
+	fmul	v19.2d, v3.2d, v8.d[0]
 .endm
 
 .macro KERNEL8x4_M2
-	fmla	v16.2d, v4.2d, v12.2d[0]
-	fmla	v29.2d, v5.2d, v15.2d[0]
+	fmla	v16.2d, v4.2d, v12.d[0]
+	fmla	v29.2d, v5.2d, v15.d[0]
 
 	ldp	d8, d9, [pB]
 	add	pB, pB, #16
 
-	fmla	v18.2d, v6.2d, v12.2d[0]
-	fmla	v31.2d, v7.2d, v15.2d[0]
+	fmla	v18.2d, v6.2d, v12.d[0]
+	fmla	v31.2d, v7.2d, v15.d[0]
 
 	ldp	d10, d11, [pB]
 	add	pB, pB, #16
 
-	fmla	v20.2d, v4.2d, v13.2d[0]
-	fmla	v25.2d, v5.2d, v14.2d[0]
+	fmla	v20.2d, v4.2d, v13.d[0]
+	fmla	v25.2d, v5.2d, v14.d[0]
 
 	prfm	PLDL1KEEP, [pB, #B_PRE_SIZE]
 
-	fmla	v22.2d, v6.2d, v13.2d[0]
-	fmla	v27.2d, v7.2d, v14.2d[0]
-	fmla	v24.2d, v4.2d, v14.2d[0]
-	fmla	v21.2d, v5.2d, v13.2d[0]
+	fmla	v22.2d, v6.2d, v13.d[0]
+	fmla	v27.2d, v7.2d, v14.d[0]
+	fmla	v24.2d, v4.2d, v14.d[0]
+	fmla	v21.2d, v5.2d, v13.d[0]
 
 	ldp	q0, q1, [pA]
 	add	pA, pA, #32
 
-	fmla	v26.2d, v6.2d, v14.2d[0]
-	fmla	v23.2d, v7.2d, v13.2d[0]
-	fmla	v28.2d, v4.2d, v15.2d[0]
-	fmla	v17.2d, v5.2d, v12.2d[0]
+	fmla	v26.2d, v6.2d, v14.d[0]
+	fmla	v23.2d, v7.2d, v13.d[0]
+	fmla	v28.2d, v4.2d, v15.d[0]
+	fmla	v17.2d, v5.2d, v12.d[0]
 
 	ldp	q2, q3, [ppA]
 	add	ppA, ppA, #32
 
-	fmla	v30.2d, v6.2d, v15.2d[0]
-	fmla	v19.2d, v7.2d, v12.2d[0]
+	fmla	v30.2d, v6.2d, v15.d[0]
+	fmla	v19.2d, v7.2d, v12.d[0]
 .endm
 
 .macro KERNEL8x4_M1
-	fmla	v16.2d, v0.2d, v8.2d[0]
-	fmla	v29.2d, v1.2d, v11.2d[0]
+	fmla	v16.2d, v0.2d, v8.d[0]
+	fmla	v29.2d, v1.2d, v11.d[0]
 
 	ldp	d12, d13, [pB]
 	add	pB, pB, #16
 
-	fmla	v18.2d, v2.2d, v8.2d[0]
-	fmla	v31.2d, v3.2d, v11.2d[0]
+	fmla	v18.2d, v2.2d, v8.d[0]
+	fmla	v31.2d, v3.2d, v11.d[0]
 
 	ldp	d14, d15, [pB]
 	add	pB, pB, #16
 
-	fmla	v20.2d, v0.2d, v9.2d[0]
-	fmla	v25.2d, v1.2d, v10.2d[0]
+	fmla	v20.2d, v0.2d, v9.d[0]
+	fmla	v25.2d, v1.2d, v10.d[0]
 
 	prfm	PLDL1KEEP, [pA, #A_PRE_SIZE]
 
-	fmla	v22.2d, v2.2d, v9.2d[0]
-	fmla	v27.2d, v3.2d, v10.2d[0]
+	fmla	v22.2d, v2.2d, v9.d[0]
+	fmla	v27.2d, v3.2d, v10.d[0]
 
 	prfm	PLDL1KEEP, [ppA, #A_PRE_SIZE]
 
-	fmla	v24.2d, v0.2d, v10.2d[0]
-	fmla	v21.2d, v1.2d, v9.2d[0]
+	fmla	v24.2d, v0.2d, v10.d[0]
+	fmla	v21.2d, v1.2d, v9.d[0]
 
 	ldp	q4, q5, [pA]
 	add	pA, pA, #32
 
-	fmla	v26.2d, v2.2d, v10.2d[0]
-	fmla	v23.2d, v3.2d, v9.2d[0]
+	fmla	v26.2d, v2.2d, v10.d[0]
+	fmla	v23.2d, v3.2d, v9.d[0]
 
-	fmla	v28.2d, v0.2d, v11.2d[0]
-	fmla	v17.2d, v1.2d, v8.2d[0]
+	fmla	v28.2d, v0.2d, v11.d[0]
+	fmla	v17.2d, v1.2d, v8.d[0]
 
 	ldp	q6, q7, [ppA]
 	add	ppA, ppA, #32
 
-	fmla	v30.2d, v2.2d, v11.2d[0]
-	fmla	v19.2d, v3.2d, v8.2d[0]
+	fmla	v30.2d, v2.2d, v11.d[0]
+	fmla	v19.2d, v3.2d, v8.d[0]
 .endm
 
 .macro KERNEL8x4_E
-	fmla	v16.2d, v4.2d, v12.2d[0]
-	fmla	v25.2d, v5.2d, v14.2d[0]
-	fmla	v18.2d, v6.2d, v12.2d[0]
-	fmla	v27.2d, v7.2d, v14.2d[0]
-
-	fmla	v20.2d, v4.2d, v13.2d[0]
-	fmla	v29.2d, v5.2d, v15.2d[0]
-	fmla	v22.2d, v6.2d, v13.2d[0]
-	fmla	v31.2d, v7.2d, v15.2d[0]
-
-	fmla	v24.2d, v4.2d, v14.2d[0]
-	fmla	v17.2d, v5.2d, v12.2d[0]
-	fmla	v26.2d, v6.2d, v14.2d[0]
-	fmla	v19.2d, v7.2d, v12.2d[0]
-
-	fmla	v28.2d, v4.2d, v15.2d[0]
-	fmla	v21.2d, v5.2d, v13.2d[0]
-	fmla	v30.2d, v6.2d, v15.2d[0]
-	fmla	v23.2d, v7.2d, v13.2d[0]
+	fmla	v16.2d, v4.2d, v12.d[0]
+	fmla	v25.2d, v5.2d, v14.d[0]
+	fmla	v18.2d, v6.2d, v12.d[0]
+	fmla	v27.2d, v7.2d, v14.d[0]
+
+	fmla	v20.2d, v4.2d, v13.d[0]
+	fmla	v29.2d, v5.2d, v15.d[0]
+	fmla	v22.2d, v6.2d, v13.d[0]
+	fmla	v31.2d, v7.2d, v15.d[0]
+
+	fmla	v24.2d, v4.2d, v14.d[0]
+	fmla	v17.2d, v5.2d, v12.d[0]
+	fmla	v26.2d, v6.2d, v14.d[0]
+	fmla	v19.2d, v7.2d, v12.d[0]
+
+	fmla	v28.2d, v4.2d, v15.d[0]
+	fmla	v21.2d, v5.2d, v13.d[0]
+	fmla	v30.2d, v6.2d, v15.d[0]
+	fmla	v23.2d, v7.2d, v13.d[0]
 .endm
 
 .macro KERNEL8x4_SUB
@@ -315,28 +315,28 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ldp	q0, q1, [pA]
 	add	pA, pA, #32
 
-	fmla	v16.2d, v0.2d, v8.2d[0]
-	fmla	v29.2d, v1.2d, v11.2d[0]
-	fmla	v20.2d, v0.2d, v9.2d[0]
-	fmla	v25.2d, v1.2d, v10.2d[0]
+	fmla	v16.2d, v0.2d, v8.d[0]
+	fmla	v29.2d, v1.2d, v11.d[0]
+	fmla	v20.2d, v0.2d, v9.d[0]
+	fmla	v25.2d, v1.2d, v10.d[0]
 
 	ldp	q2, q3, [ppA]
 	add	ppA, ppA, #32
 
-	fmla	v24.2d, v0.2d, v10.2d[0]
-	fmla	v21.2d, v1.2d, v9.2d[0]
-	fmla	v28.2d, v0.2d, v11.2d[0]
-	fmla	v17.2d, v1.2d, v8.2d[0]
+	fmla	v24.2d, v0.2d, v10.d[0]
+	fmla	v21.2d, v1.2d, v9.d[0]
+	fmla	v28.2d, v0.2d, v11.d[0]
+	fmla	v17.2d, v1.2d, v8.d[0]
 
-	fmla	v18.2d, v2.2d, v8.2d[0]
-	fmla	v31.2d, v3.2d, v11.2d[0]
-	fmla	v22.2d, v2.2d, v9.2d[0]
-	fmla	v27.2d, v3.2d, v10.2d[0]
+	fmla	v18.2d, v2.2d, v8.d[0]
+	fmla	v31.2d, v3.2d, v11.d[0]
+	fmla	v22.2d, v2.2d, v9.d[0]
+	fmla	v27.2d, v3.2d, v10.d[0]
 
-	fmla	v26.2d, v2.2d, v10.2d[0]
-	fmla	v23.2d, v3.2d, v9.2d[0]
-	fmla	v30.2d, v2.2d, v11.2d[0]
-	fmla	v19.2d, v3.2d, v8.2d[0]
+	fmla	v26.2d, v2.2d, v10.d[0]
+	fmla	v23.2d, v3.2d, v9.d[0]
+	fmla	v30.2d, v2.2d, v11.d[0]
+	fmla	v19.2d, v3.2d, v8.d[0]
 .endm
 
 .macro SAVE8x4
@@ -422,17 +422,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2d, v1.2d}, [pA]
 	add	pA, pA, #32
 
-	fmla	v16.2d, v0.2d, v8.2d[0]
-	fmla	v29.2d, v1.2d, v9.2d[1]
+	fmla	v16.2d, v0.2d, v8.d[0]
+	fmla	v29.2d, v1.2d, v9.d[1]
 
-	fmla	v20.2d, v0.2d, v8.2d[1]
-	fmla	v25.2d, v1.2d, v9.2d[0]
+	fmla	v20.2d, v0.2d, v8.d[1]
+	fmla	v25.2d, v1.2d, v9.d[0]
 
-	fmla	v24.2d, v0.2d, v9.2d[0]
-	fmla	v21.2d, v1.2d, v8.2d[1]
+	fmla	v24.2d, v0.2d, v9.d[0]
+	fmla	v21.2d, v1.2d, v8.d[1]
 
-	fmla	v28.2d, v0.2d, v9.2d[1]
-	fmla	v17.2d, v1.2d, v8.2d[0]
+	fmla	v28.2d, v0.2d, v9.d[1]
+	fmla	v17.2d, v1.2d, v8.d[0]
 .endm
 
 .macro SAVE4x4
@@ -482,10 +482,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2d}, [pA]
 	add	pA, pA, #16
 
-	fmla	v16.2d, v0.2d, v8.2d[0]
-	fmla	v20.2d, v0.2d, v8.2d[1]
-	fmla	v24.2d, v0.2d, v9.2d[0]
-	fmla	v28.2d, v0.2d, v9.2d[1]
+	fmla	v16.2d, v0.2d, v8.d[0]
+	fmla	v20.2d, v0.2d, v8.d[1]
+	fmla	v24.2d, v0.2d, v9.d[0]
+	fmla	v28.2d, v0.2d, v9.d[1]
 .endm
 
 .macro SAVE2x4
@@ -572,10 +572,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2d, v1.2d}, [pA]
 	add	pA, pA, #32
 
-	fmla	v16.2d, v0.2d, v8.2d[0]
-	fmla	v17.2d, v1.2d, v8.2d[0]
-	fmla	v20.2d, v0.2d, v8.2d[1]
-	fmla	v21.2d, v1.2d, v8.2d[1]
+	fmla	v16.2d, v0.2d, v8.d[0]
+	fmla	v17.2d, v1.2d, v8.d[0]
+	fmla	v20.2d, v0.2d, v8.d[1]
+	fmla	v21.2d, v1.2d, v8.d[1]
 .endm
 
 .macro SAVE4x2
@@ -610,8 +610,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2d}, [pA]
 	add	pA, pA, #16
 
-	fmla	v16.2d, v0.2d, v8.2d[0]
-	fmla	v20.2d, v0.2d, v8.2d[1]
+	fmla	v16.2d, v0.2d, v8.d[0]
+	fmla	v20.2d, v0.2d, v8.d[1]
 .endm
 
 .macro SAVE2x2
@@ -643,7 +643,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ldr	d0 , [pA]
 	add	pA, pA, #8
 
-	fmla	v16.2d, v8.2d, v0.2d[0]
+	fmla	v16.2d, v8.2d, v0.d[0]
 .endm
 
 .macro SAVE1x2
@@ -674,8 +674,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2d, v1.2d}, [pA]
 	add	pA , pA, #32
 
-	fmla	v16.2d, v0.2d, v8.2d[0]
-	fmla	v17.2d, v1.2d, v8.2d[0]
+	fmla	v16.2d, v0.2d, v8.d[0]
+	fmla	v17.2d, v1.2d, v8.d[0]
 .endm
 
 .macro SAVE4x1
@@ -705,7 +705,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2d}, [pA]
 	add	pA , pA, #16
 
-	fmla	v16.2d, v0.2d, v8.2d[0]
+	fmla	v16.2d, v0.2d, v8.d[0]
 .endm
 
 .macro SAVE2x1

kernel/arm64/dgemm_kernel_4x8.S

@@ -154,25 +154,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v10.2d, v11.2d}, [pB]
 	add	pB, pB, #32
 
-	fmul	v16.2d, v0.2d, v8.2d[0]
-	fmul	v17.2d, v1.2d, v8.2d[0]
-	fmul	v18.2d, v0.2d, v8.2d[1]
-	fmul	v19.2d, v1.2d, v8.2d[1]
-
-	fmul	v20.2d, v0.2d, v9.2d[0]
-	fmul	v21.2d, v1.2d, v9.2d[0]
-	fmul	v22.2d, v0.2d, v9.2d[1]
-	fmul	v23.2d, v1.2d, v9.2d[1]
-
-	fmul	v24.2d, v0.2d, v10.2d[0]
-	fmul	v25.2d, v1.2d, v10.2d[0]
-	fmul	v26.2d, v0.2d, v10.2d[1]
-	fmul	v27.2d, v1.2d, v10.2d[1]
-
-	fmul	v28.2d, v0.2d, v11.2d[0]
-	fmul	v29.2d, v1.2d, v11.2d[0]
-	fmul	v30.2d, v0.2d, v11.2d[1]
-	fmul	v31.2d, v1.2d, v11.2d[1]
+	fmul	v16.2d, v0.2d, v8.d[0]
+	fmul	v17.2d, v1.2d, v8.d[0]
+	fmul	v18.2d, v0.2d, v8.d[1]
+	fmul	v19.2d, v1.2d, v8.d[1]
+
+	fmul	v20.2d, v0.2d, v9.d[0]
+	fmul	v21.2d, v1.2d, v9.d[0]
+	fmul	v22.2d, v0.2d, v9.d[1]
+	fmul	v23.2d, v1.2d, v9.d[1]
+
+	fmul	v24.2d, v0.2d, v10.d[0]
+	fmul	v25.2d, v1.2d, v10.d[0]
+	fmul	v26.2d, v0.2d, v10.d[1]
+	fmul	v27.2d, v1.2d, v10.d[1]
+
+	fmul	v28.2d, v0.2d, v11.d[0]
+	fmul	v29.2d, v1.2d, v11.d[0]
+	fmul	v30.2d, v0.2d, v11.d[1]
+	fmul	v31.2d, v1.2d, v11.d[1]
 
 	ld1	{v12.2d, v13.2d}, [pB]
 	add	pB, pB, #32
@@ -183,25 +183,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL4x8_M1
-	fmla	v16.2d, v0.2d, v8.2d[0]
-	fmla	v17.2d, v1.2d, v8.2d[0]
-	fmla	v18.2d, v0.2d, v8.2d[1]
-	fmla	v19.2d, v1.2d, v8.2d[1]
-
-	fmla	v20.2d, v0.2d, v9.2d[0]
-	fmla	v21.2d, v1.2d, v9.2d[0]
-	fmla	v22.2d, v0.2d, v9.2d[1]
-	fmla	v23.2d, v1.2d, v9.2d[1]
-
-	fmla	v24.2d, v0.2d, v10.2d[0]
-	fmla	v25.2d, v1.2d, v10.2d[0]
-	fmla	v26.2d, v0.2d, v10.2d[1]
-	fmla	v27.2d, v1.2d, v10.2d[1]
-
-	fmla	v28.2d, v0.2d, v11.2d[0]
-	fmla	v29.2d, v1.2d, v11.2d[0]
-	fmla	v30.2d, v0.2d, v11.2d[1]
-	fmla	v31.2d, v1.2d, v11.2d[1]
+	fmla	v16.2d, v0.2d, v8.d[0]
+	fmla	v17.2d, v1.2d, v8.d[0]
+	fmla	v18.2d, v0.2d, v8.d[1]
+	fmla	v19.2d, v1.2d, v8.d[1]
+
+	fmla	v20.2d, v0.2d, v9.d[0]
+	fmla	v21.2d, v1.2d, v9.d[0]
+	fmla	v22.2d, v0.2d, v9.d[1]
+	fmla	v23.2d, v1.2d, v9.d[1]
+
+	fmla	v24.2d, v0.2d, v10.d[0]
+	fmla	v25.2d, v1.2d, v10.d[0]
+	fmla	v26.2d, v0.2d, v10.d[1]
+	fmla	v27.2d, v1.2d, v10.d[1]
+
+	fmla	v28.2d, v0.2d, v11.d[0]
+	fmla	v29.2d, v1.2d, v11.d[0]
+	fmla	v30.2d, v0.2d, v11.d[1]
+	fmla	v31.2d, v1.2d, v11.d[1]
 
 	ld1	{v12.2d, v13.2d}, [pB]		// For next round
 	add	pB, pB, #32
@@ -214,25 +214,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL4x8_M2
-	fmla	v16.2d, v4.2d, v12.2d[0]
-	fmla	v17.2d, v5.2d, v12.2d[0]
-	fmla	v18.2d, v4.2d, v12.2d[1]
-	fmla	v19.2d, v5.2d, v12.2d[1]
-
-	fmla	v20.2d, v4.2d, v13.2d[0]
-	fmla	v21.2d, v5.2d, v13.2d[0]
-	fmla	v22.2d, v4.2d, v13.2d[1]
-	fmla	v23.2d, v5.2d, v13.2d[1]
-
-	fmla	v24.2d, v4.2d, v14.2d[0]
-	fmla	v25.2d, v5.2d, v14.2d[0]
-	fmla	v26.2d, v4.2d, v14.2d[1]
-	fmla	v27.2d, v5.2d, v14.2d[1]
-
-	fmla	v28.2d, v4.2d, v15.2d[0]
-	fmla	v29.2d, v5.2d, v15.2d[0]
-	fmla	v30.2d, v4.2d, v15.2d[1]
-	fmla	v31.2d, v5.2d, v15.2d[1]
+	fmla	v16.2d, v4.2d, v12.d[0]
+	fmla	v17.2d, v5.2d, v12.d[0]
+	fmla	v18.2d, v4.2d, v12.d[1]
+	fmla	v19.2d, v5.2d, v12.d[1]
+
+	fmla	v20.2d, v4.2d, v13.d[0]
+	fmla	v21.2d, v5.2d, v13.d[0]
+	fmla	v22.2d, v4.2d, v13.d[1]
+	fmla	v23.2d, v5.2d, v13.d[1]
+
+	fmla	v24.2d, v4.2d, v14.d[0]
+	fmla	v25.2d, v5.2d, v14.d[0]
+	fmla	v26.2d, v4.2d, v14.d[1]
+	fmla	v27.2d, v5.2d, v14.d[1]
+
+	fmla	v28.2d, v4.2d, v15.d[0]
+	fmla	v29.2d, v5.2d, v15.d[0]
+	fmla	v30.2d, v4.2d, v15.d[1]
+	fmla	v31.2d, v5.2d, v15.d[1]
 
 	ld1	{v8.2d, v9.2d}, [pB]		// For next round
 	add	pB, pB, #32
@@ -245,25 +245,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL4x8_E
-	fmla	v16.2d, v4.2d, v12.2d[0]
-	fmla	v17.2d, v5.2d, v12.2d[0]
-	fmla	v18.2d, v4.2d, v12.2d[1]
-	fmla	v19.2d, v5.2d, v12.2d[1]
-
-	fmla	v20.2d, v4.2d, v13.2d[0]
-	fmla	v21.2d, v5.2d, v13.2d[0]
-	fmla	v22.2d, v4.2d, v13.2d[1]
-	fmla	v23.2d, v5.2d, v13.2d[1]
-
-	fmla	v24.2d, v4.2d, v14.2d[0]
-	fmla	v25.2d, v5.2d, v14.2d[0]
-	fmla	v26.2d, v4.2d, v14.2d[1]
-	fmla	v27.2d, v5.2d, v14.2d[1]
-
-	fmla	v28.2d, v4.2d, v15.2d[0]
-	fmla	v29.2d, v5.2d, v15.2d[0]
-	fmla	v30.2d, v4.2d, v15.2d[1]
-	fmla	v31.2d, v5.2d, v15.2d[1]
+	fmla	v16.2d, v4.2d, v12.d[0]
+	fmla	v17.2d, v5.2d, v12.d[0]
+	fmla	v18.2d, v4.2d, v12.d[1]
+	fmla	v19.2d, v5.2d, v12.d[1]
+
+	fmla	v20.2d, v4.2d, v13.d[0]
+	fmla	v21.2d, v5.2d, v13.d[0]
+	fmla	v22.2d, v4.2d, v13.d[1]
+	fmla	v23.2d, v5.2d, v13.d[1]
+
+	fmla	v24.2d, v4.2d, v14.d[0]
+	fmla	v25.2d, v5.2d, v14.d[0]
+	fmla	v26.2d, v4.2d, v14.d[1]
+	fmla	v27.2d, v5.2d, v14.d[1]
+
+	fmla	v28.2d, v4.2d, v15.d[0]
+	fmla	v29.2d, v5.2d, v15.d[0]
+	fmla	v30.2d, v4.2d, v15.d[1]
+	fmla	v31.2d, v5.2d, v15.d[1]
 .endm
 
 .macro KERNEL4x8_SUB
@@ -274,25 +274,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v10.2d, v11.2d}, [pB]
 	add	pB, pB, #32
 
-	fmla	v16.2d, v0.2d, v8.2d[0]
-	fmla	v17.2d, v1.2d, v8.2d[0]
-	fmla	v18.2d, v0.2d, v8.2d[1]
-	fmla	v19.2d, v1.2d, v8.2d[1]
-
-	fmla	v20.2d, v0.2d, v9.2d[0]
-	fmla	v21.2d, v1.2d, v9.2d[0]
-	fmla	v22.2d, v0.2d, v9.2d[1]
-	fmla	v23.2d, v1.2d, v9.2d[1]
-
-	fmla	v24.2d, v0.2d, v10.2d[0]
-	fmla	v25.2d, v1.2d, v10.2d[0]
-	fmla	v26.2d, v0.2d, v10.2d[1]
-	fmla	v27.2d, v1.2d, v10.2d[1]
-
-	fmla	v28.2d, v0.2d, v11.2d[0]
-	fmla	v29.2d, v1.2d, v11.2d[0]
-	fmla	v30.2d, v0.2d, v11.2d[1]
-	fmla	v31.2d, v1.2d, v11.2d[1]
+	fmla	v16.2d, v0.2d, v8.d[0]
+	fmla	v17.2d, v1.2d, v8.d[0]
+	fmla	v18.2d, v0.2d, v8.d[1]
+	fmla	v19.2d, v1.2d, v8.d[1]
+
+	fmla	v20.2d, v0.2d, v9.d[0]
+	fmla	v21.2d, v1.2d, v9.d[0]
+	fmla	v22.2d, v0.2d, v9.d[1]
+	fmla	v23.2d, v1.2d, v9.d[1]
+
+	fmla	v24.2d, v0.2d, v10.d[0]
+	fmla	v25.2d, v1.2d, v10.d[0]
+	fmla	v26.2d, v0.2d, v10.d[1]
+	fmla	v27.2d, v1.2d, v10.d[1]
+
+	fmla	v28.2d, v0.2d, v11.d[0]
+	fmla	v29.2d, v1.2d, v11.d[0]
+	fmla	v30.2d, v0.2d, v11.d[1]
+	fmla	v31.2d, v1.2d, v11.d[1]
 .endm
 
 .macro SAVE4x8
@@ -374,17 +374,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v10.2d, v11.2d}, [pB]
 	add	pB, pB, #32
 
-	fmla	v16.2d, v0.2d, v8.2d[0]
-	fmla	v18.2d, v0.2d, v8.2d[1]
+	fmla	v16.2d, v0.2d, v8.d[0]
+	fmla	v18.2d, v0.2d, v8.d[1]
 
-	fmla	v20.2d, v0.2d, v9.2d[0]
-	fmla	v22.2d, v0.2d, v9.2d[1]
+	fmla	v20.2d, v0.2d, v9.d[0]
+	fmla	v22.2d, v0.2d, v9.d[1]
 
-	fmla	v24.2d, v0.2d, v10.2d[0]
-	fmla	v26.2d, v0.2d, v10.2d[1]
+	fmla	v24.2d, v0.2d, v10.d[0]
+	fmla	v26.2d, v0.2d, v10.d[1]
 
-	fmla	v28.2d, v0.2d, v11.2d[0]
-	fmla	v30.2d, v0.2d, v11.2d[1]
+	fmla	v28.2d, v0.2d, v11.d[0]
+	fmla	v30.2d, v0.2d, v11.d[1]
 .endm
 
 .macro SAVE2x8
@@ -520,17 +520,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2d, v1.2d}, [pA]
 	add	pA, pA, #32
 
-	fmul	v16.2d, v0.2d, v8.2d[0]
-	fmul	v29.2d, v1.2d, v9.2d[1]
+	fmul	v16.2d, v0.2d, v8.d[0]
+	fmul	v29.2d, v1.2d, v9.d[1]
 
-	fmul	v20.2d, v0.2d, v8.2d[1]
-	fmul	v25.2d, v1.2d, v9.2d[0]
+	fmul	v20.2d, v0.2d, v8.d[1]
+	fmul	v25.2d, v1.2d, v9.d[0]
 
-	fmul	v24.2d, v0.2d, v9.2d[0]
-	fmul	v21.2d, v1.2d, v8.2d[1]
+	fmul	v24.2d, v0.2d, v9.d[0]
+	fmul	v21.2d, v1.2d, v8.d[1]
 
-	fmul	v28.2d, v0.2d, v9.2d[1]
-	fmul	v17.2d, v1.2d, v8.2d[0]
+	fmul	v28.2d, v0.2d, v9.d[1]
+	fmul	v17.2d, v1.2d, v8.d[0]
 
 	ld1	{v12.2d, v13.2d}, [pB]
 	add	pB, pB, #32
@@ -539,61 +539,61 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL4x4_M1
-	fmla	v16.2d, v0.2d, v8.2d[0]
-	fmla	v29.2d, v1.2d, v9.2d[1]
+	fmla	v16.2d, v0.2d, v8.d[0]
+	fmla	v29.2d, v1.2d, v9.d[1]
 
 	ld1	{v12.2d, v13.2d}, [pB]		// For next round
 	add	pB, pB, #32
 
-	fmla	v20.2d, v0.2d, v8.2d[1]
-	fmla	v25.2d, v1.2d, v9.2d[0]
+	fmla	v20.2d, v0.2d, v8.d[1]
+	fmla	v25.2d, v1.2d, v9.d[0]
 
 	ld1	{v4.2d, v5.2d}, [pA]		// For next round
 	add	pA, pA, #32
 
-	fmla	v24.2d, v0.2d, v9.2d[0]
-	fmla	v21.2d, v1.2d, v8.2d[1]
+	fmla	v24.2d, v0.2d, v9.d[0]
+	fmla	v21.2d, v1.2d, v8.d[1]
 
 	prfm	PLDL1KEEP, [pA, #512]
 
-	fmla	v28.2d, v0.2d, v9.2d[1]
-	fmla	v17.2d, v1.2d, v8.2d[0]
+	fmla	v28.2d, v0.2d, v9.d[1]
+	fmla	v17.2d, v1.2d, v8.d[0]
 .endm
 
 .macro KERNEL4x4_M2
-	fmla	v16.2d, v4.2d, v12.2d[0]
-	fmla	v29.2d, v5.2d, v13.2d[1]
+	fmla	v16.2d, v4.2d, v12.d[0]
+	fmla	v29.2d, v5.2d, v13.d[1]
 
 	ld1	{v8.2d, v9.2d}, [pB]		// For next round
 	add	pB, pB, #32
 
-	fmla	v20.2d, v4.2d, v12.2d[1]
-	fmla	v25.2d, v5.2d, v13.2d[0]
+	fmla	v20.2d, v4.2d, v12.d[1]
+	fmla	v25.2d, v5.2d, v13.d[0]
 
 	ld1	{v0.2d, v1.2d}, [pA]		// For next round
 	add	pA, pA, #32
 
-	fmla	v24.2d, v4.2d, v13.2d[0]
-	fmla	v21.2d, v5.2d, v12.2d[1]
+	fmla	v24.2d, v4.2d, v13.d[0]
+	fmla	v21.2d, v5.2d, v12.d[1]
 
 	prfm	PLDL1KEEP, [pB, #512]
 
-	fmla	v28.2d, v4.2d, v13.2d[1]
-	fmla	v17.2d, v5.2d, v12.2d[0]
+	fmla	v28.2d, v4.2d, v13.d[1]
+	fmla	v17.2d, v5.2d, v12.d[0]
 .endm
 
 .macro KERNEL4x4_E
-	fmla	v16.2d, v4.2d, v12.2d[0]
-	fmla	v29.2d, v5.2d, v13.2d[1]
+	fmla	v16.2d, v4.2d, v12.d[0]
+	fmla	v29.2d, v5.2d, v13.d[1]
 
-	fmla	v20.2d, v4.2d, v12.2d[1]
-	fmla	v25.2d, v5.2d, v13.2d[0]
+	fmla	v20.2d, v4.2d, v12.d[1]
+	fmla	v25.2d, v5.2d, v13.d[0]
 
-	fmla	v24.2d, v4.2d, v13.2d[0]
-	fmla	v21.2d, v5.2d, v12.2d[1]
+	fmla	v24.2d, v4.2d, v13.d[0]
+	fmla	v21.2d, v5.2d, v12.d[1]
 
-	fmla	v28.2d, v4.2d, v13.2d[1]
-	fmla	v17.2d, v5.2d, v12.2d[0]
+	fmla	v28.2d, v4.2d, v13.d[1]
+	fmla	v17.2d, v5.2d, v12.d[0]
 .endm
 
 .macro KERNEL4x4_SUB
@@ -602,17 +602,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2d, v1.2d}, [pA]
 	add	pA, pA, #32
 
-	fmla	v16.2d, v0.2d, v8.2d[0]
-	fmla	v29.2d, v1.2d, v9.2d[1]
+	fmla	v16.2d, v0.2d, v8.d[0]
+	fmla	v29.2d, v1.2d, v9.d[1]
 
-	fmla	v20.2d, v0.2d, v8.2d[1]
-	fmla	v25.2d, v1.2d, v9.2d[0]
+	fmla	v20.2d, v0.2d, v8.d[1]
+	fmla	v25.2d, v1.2d, v9.d[0]
 
-	fmla	v24.2d, v0.2d, v9.2d[0]
-	fmla	v21.2d, v1.2d, v8.2d[1]
+	fmla	v24.2d, v0.2d, v9.d[0]
+	fmla	v21.2d, v1.2d, v8.d[1]
 
-	fmla	v28.2d, v0.2d, v9.2d[1]
-	fmla	v17.2d, v1.2d, v8.2d[0]
+	fmla	v28.2d, v0.2d, v9.d[1]
+	fmla	v17.2d, v1.2d, v8.d[0]
 .endm
 
 .macro SAVE4x4
@@ -660,10 +660,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2d}, [pA]
 	add	pA, pA, #16
 
-	fmla	v16.2d, v0.2d, v8.2d[0]
-	fmla	v20.2d, v0.2d, v8.2d[1]
-	fmla	v24.2d, v0.2d, v9.2d[0]
-	fmla	v28.2d, v0.2d, v9.2d[1]
+	fmla	v16.2d, v0.2d, v8.d[0]
+	fmla	v20.2d, v0.2d, v8.d[1]
+	fmla	v24.2d, v0.2d, v9.d[0]
+	fmla	v28.2d, v0.2d, v9.d[1]
 .endm
 
 .macro SAVE2x4
@@ -746,10 +746,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2d, v1.2d}, [pA]
 	add	pA, pA, #32
 
-	fmla	v16.2d, v0.2d, v8.2d[0]
-	fmla	v17.2d, v1.2d, v8.2d[0]
-	fmla	v20.2d, v0.2d, v8.2d[1]
-	fmla	v21.2d, v1.2d, v8.2d[1]
+	fmla	v16.2d, v0.2d, v8.d[0]
+	fmla	v17.2d, v1.2d, v8.d[0]
+	fmla	v20.2d, v0.2d, v8.d[1]
+	fmla	v21.2d, v1.2d, v8.d[1]
 .endm
 
 .macro SAVE4x2
@@ -782,8 +782,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2d}, [pA]
 	add	pA, pA, #16
 
-	fmla	v16.2d, v0.2d, v8.2d[0]
-	fmla	v20.2d, v0.2d, v8.2d[1]
+	fmla	v16.2d, v0.2d, v8.d[0]
+	fmla	v20.2d, v0.2d, v8.d[1]
 .endm
 
 .macro SAVE2x2
@@ -813,7 +813,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ldr	d0 , [pA]
 	add	pA, pA, #8
 
-	fmla	v16.2d, v8.2d, v0.2d[0]
+	fmla	v16.2d, v8.2d, v0.d[0]
 .endm
 
 .macro SAVE1x2
@@ -842,8 +842,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2d, v1.2d}, [pA]
 	add	pA , pA, #32
 
-	fmla	v16.2d, v0.2d, v8.2d[0]
-	fmla	v17.2d, v1.2d, v8.2d[0]
+	fmla	v16.2d, v0.2d, v8.d[0]
+	fmla	v17.2d, v1.2d, v8.d[0]
 .endm
 
 .macro SAVE4x1
@@ -871,7 +871,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2d}, [pA]
 	add	pA , pA, #16
 
-	fmla	v16.2d, v0.2d, v8.2d[0]
+	fmla	v16.2d, v0.2d, v8.d[0]
 .endm
 
 .macro SAVE2x1

kernel/arm64/dgemm_kernel_8x4.S

@@ -52,12 +52,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #define alpha0		d10
 #define alphaV0		v10.d[0]
-#define alpha1		d11
-#define alphaV1		v11.d[0]
-#define alpha2		d14
-#define alphaV2		v14.d[0]
-#define alpha3		d15
-#define alphaV3		v15.d[0]
+
+#define A_PRE_SIZE	2560
+#define B_PRE_SIZE	448
+#define C_PRE_SIZE	128
 
 // 00 origM
 // 01 origN
@@ -74,8 +72,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 // 12 pCRow0
 // 13 pCRow1
 // 14 pCRow2
-// 15 pA
-// 16
+// 15 pCRow3
+// 16 pA
 // 17
 // 18 must save
 // 19 must save
@@ -100,14 +98,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 //v05 pA1_2, pA1_3
 //v06 pA1_4, pA1_5
 //v07 pA1_6, pA1_7
-//v08 must save pB0_0, pB0_1
-//v09 must save pB0_2, pB0_3
-//v10 must save ALPHA0
-//v11 must save ALPHA1
-//v12 must save pB1_0, pB1_1
-//v13 must save pB1_2, pB1_3
-//v14 must save ALPHA2
-//v15 must save ALPHA3
+//v08 must save pB0_0
+//v09 must save pB0_1
+//v10 must save pB0_2 --> ALPHA0
+//v11 must save pB0_3
+//v12 must save pB1_0
+//v13 must save pB1_1
+//v14 must save pB1_2
+//v15 must save pB1_3
 //v16 must save C00, C01
 //v17 must save C02, C03
 //v18 C04, C05
@@ -149,244 +147,257 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL8x4_I
-	ld1	{v0.2d, v1.2d}, [pA]
-	add	pA, pA, #32
-	ld1	{v2.2d, v3.2d}, [pA]
-	add	pA, pA, #32
-	ldp	d8, d9, [pB]
-	add	pB, pB, #16
-	ldp	d10, d11, [pB]
-	add	pB, pB, #16
+	ldp	q0, q1, [pA], #32
 
-	fmul	v16.2d, v0.2d, v8.2d[0]
-	fmul	v17.2d, v1.2d, v8.2d[0]
+	ldp	d8, d9, [pB], #16
 
-	fmul	v18.2d, v2.2d, v8.2d[0]
-	fmul	v19.2d, v3.2d, v8.2d[0]
+	fmul	v16.2d, v0.2d, v8.d[0]
+	fmul	v20.2d, v0.2d, v9.d[0]
 
-	fmul	v20.2d, v0.2d, v9.2d[0]
-	fmul	v21.2d, v1.2d, v9.2d[0]
+	ldp	d10, d11, [pB], #16
 
-	fmul	v22.2d, v2.2d, v9.2d[0]
-	fmul	v23.2d, v3.2d, v9.2d[0]
+	fmul	v17.2d, v1.2d, v8.d[0]
+	fmul	v21.2d, v1.2d, v9.d[0]
 
-	fmul	v24.2d, v0.2d, v10.2d[0]
-	fmul	v25.2d, v1.2d, v10.2d[0]
+	ldp	q2, q3, [pA], #32
 
-	fmul	v26.2d, v2.2d, v10.2d[0]
-	fmul	v27.2d, v3.2d, v10.2d[0]
+	fmul	v24.2d, v0.2d, v10.d[0]
+	fmul	v28.2d, v0.2d, v11.d[0]
 
-	fmul	v28.2d, v0.2d, v11.2d[0]
-	fmul	v29.2d, v1.2d, v11.2d[0]
+	ldp	q4, q5, [pA], #32
 
-	fmul	v30.2d, v2.2d, v11.2d[0]
-	fmul	v31.2d, v3.2d, v11.2d[0]
+	fmul	v25.2d, v1.2d, v10.d[0]
+	fmul	v29.2d, v1.2d, v11.d[0]
 
-	ld1	{v4.2d, v5.2d}, [pA]
-	add	pA, pA, #32
-	ld1	{v6.2d, v7.2d}, [pA]
-	add	pA, pA, #32
-	ldp	d12, d13, [pB]
-	add	pB, pB, #16
-	ldp	d14, d15, [pB]
-	add	pB, pB, #16
+	ldp	d12, d13, [pB], #16
+
+	fmul	v18.2d, v2.2d, v8.d[0]
+	fmul	v22.2d, v2.2d, v9.d[0]
+
+	ldp	d14, d15, [pB], #16
+
+	fmul	v26.2d, v2.2d, v10.d[0]
+	fmul	v30.2d, v2.2d, v11.d[0]
+
+	ldp	q6, q7, [pA], #32
+
+	fmul	v19.2d, v3.2d, v8.d[0]
+	fmul	v27.2d, v3.2d, v10.d[0]
+
+	prfm	PLDL1KEEP, [pA, #A_PRE_SIZE]
+
+	fmul	v31.2d, v3.2d, v11.d[0]
+	fmul	v23.2d, v3.2d, v9.d[0]
+
+	prfm	PLDL1KEEP, [pA, #A_PRE_SIZE+64]
 .endm
 
 .macro KERNEL8x4_M1
-	fmla	v16.2d, v0.2d, v8.2d[0]
-	fmla	v21.2d, v1.2d, v9.2d[0]
-	fmla	v26.2d, v2.2d, v10.2d[0]
-	fmla	v31.2d, v3.2d, v11.2d[0]
+	fmla	v16.2d, v0.2d, v8.d[0]
+	fmla	v20.2d, v0.2d, v9.d[0]
 
-	ld1	{v4.2d}, [pA], #16
+	ldp	q4, q5, [pA], #32
 
-	fmla	v20.2d, v0.2d, v9.2d[0]
-	fmla	v17.2d, v1.2d, v8.2d[0]
+	fmla	v24.2d, v0.2d, v10.d[0]
+	fmla	v28.2d, v0.2d, v11.d[0]
 
-	ld1	{v5.2d}, [pA], #16
+	ldp	d12, d13, [pB], #16
 
-	fmla	v30.2d, v2.2d, v11.2d[0]
-	fmla	v27.2d, v3.2d, v10.2d[0]
+	fmla	v17.2d, v1.2d, v8.d[0]
+	fmla	v25.2d, v1.2d, v10.d[0]
 
-	ldp	d12, d13, [pB]
-	add	pB, pB, #16
+	prfm	PLDL1KEEP, [pA, #A_PRE_SIZE+64]
 
-	fmla	v28.2d, v0.2d, v11.2d[0]
-	fmla	v25.2d, v1.2d, v10.2d[0]
+	fmla	v21.2d, v1.2d, v9.d[0]
+	fmla	v29.2d, v1.2d, v11.d[0]
 
-	ldp	d14, d15, [pB]
-	add	pB, pB, #16
+	ldp	d14, d15, [pB], #16
 
-	fmla	v18.2d, v2.2d, v8.2d[0]
-	fmla	v23.2d, v3.2d, v9.2d[0]
+	fmla	v18.2d, v2.2d, v8.d[0]
+	fmla	v22.2d, v2.2d, v9.d[0]
 
-	ld1	{v6.2d}, [pA], #16
+	prfm	PLDL1KEEP, [pA, #A_PRE_SIZE]
 
-	fmla	v24.2d, v0.2d, v10.2d[0]
-	fmla	v29.2d, v1.2d, v11.2d[0]
+	fmla	v26.2d, v2.2d, v10.d[0]
+	fmla	v30.2d, v2.2d, v11.d[0]
+	fmla	v19.2d, v3.2d, v8.d[0]
+	fmla	v23.2d, v3.2d, v9.d[0]
 
-	ld1	{v7.2d}, [pA], #16
+	ldp	q6, q7, [pA], #32
 
-	fmla	v22.2d, v2.2d, v9.2d[0]
-	fmla	v19.2d, v3.2d, v8.2d[0]
-
-	prfm	PLDL1KEEP, [pA, #224]
-	prfm	PLDL1KEEP, [pA, #224+64]
+	fmla	v27.2d, v3.2d, v10.d[0]
+	fmla	v31.2d, v3.2d, v11.d[0]
 .endm
 
 .macro KERNEL8x4_M2
-	fmla	v16.2d, v4.2d, v12.2d[0]
-	fmla	v21.2d, v5.2d, v13.2d[0]
-	fmla	v26.2d, v6.2d, v14.2d[0]
-	fmla	v31.2d, v7.2d, v15.2d[0]
-
-	ld1	{v0.2d}, [pA], #16
+	fmla	v16.2d, v4.2d, v12.d[0]
+	fmla	v20.2d, v4.2d, v13.d[0]
+	fmla	v24.2d, v4.2d, v14.d[0]
+	fmla	v28.2d, v4.2d, v15.d[0]
 
-	fmla	v20.2d, v4.2d, v13.2d[0]
-	fmla	v17.2d, v5.2d, v12.2d[0]
+	ldp	q0, q1, [pA], #32
 
-	ld1	{v1.2d}, [pA], #16
+	fmla	v17.2d, v5.2d, v12.d[0]
+	fmla	v25.2d, v5.2d, v14.d[0]
 
-	fmla	v30.2d, v6.2d, v15.2d[0]
-	fmla	v27.2d, v7.2d, v14.2d[0]
+	ldp	d8, d9, [pB], #16
 
-	ldp	d8, d9, [pB]
-	add	pB, pB, #16
+	fmla	v21.2d, v5.2d, v13.d[0]
+	fmla	v29.2d, v5.2d, v15.d[0]
 
-	fmla	v28.2d, v4.2d, v15.2d[0]
-	fmla	v25.2d, v5.2d, v14.2d[0]
-
-	ldp	d10, d11, [pB]
-	add	pB, pB, #16
+	ldp	d10, d11, [pB], #16
 
-	fmla	v22.2d, v6.2d, v13.2d[0]
-	fmla	v19.2d, v7.2d, v12.2d[0]
+	fmla	v18.2d, v6.2d, v12.d[0]
+	fmla	v22.2d, v6.2d, v13.d[0]
 
-	ld1	{v2.2d}, [pA], #16
+	prfm	PLDL1KEEP, [pB, #B_PRE_SIZE]
 
-	fmla	v24.2d, v4.2d, v14.2d[0]
-	fmla	v29.2d, v5.2d, v15.2d[0]
+	fmla	v26.2d, v6.2d, v14.d[0]
+	fmla	v30.2d, v6.2d, v15.d[0]
 
-	ld1	{v3.2d}, [pA], #16
+	fmla	v19.2d, v7.2d, v12.d[0]
+	fmla	v23.2d, v7.2d, v13.d[0]
 
-	fmla	v18.2d, v6.2d, v12.2d[0]
-	fmla	v23.2d, v7.2d, v13.2d[0]
+	ldp	q2, q3, [pA], #32
 
-	prfm	PLDL1KEEP, [pB, #640]
+	fmla	v27.2d, v7.2d, v14.d[0]
+	fmla	v31.2d, v7.2d, v15.d[0]
 .endm
 
 .macro KERNEL8x4_E
-	fmla	v16.2d, v4.2d, v12.2d[0]
-	fmla	v17.2d, v5.2d, v12.2d[0]
-	fmla	v18.2d, v6.2d, v12.2d[0]
-	fmla	v19.2d, v7.2d, v12.2d[0]
-	fmla	v20.2d, v4.2d, v13.2d[0]
-	fmla	v21.2d, v5.2d, v13.2d[0]
-	fmla	v22.2d, v6.2d, v13.2d[0]
-	fmla	v23.2d, v7.2d, v13.2d[0]
-	fmla	v24.2d, v4.2d, v14.2d[0]
-	fmla	v25.2d, v5.2d, v14.2d[0]
-	fmla	v26.2d, v6.2d, v14.2d[0]
-	fmla	v27.2d, v7.2d, v14.2d[0]
-	fmla	v28.2d, v4.2d, v15.2d[0]
-	fmla	v29.2d, v5.2d, v15.2d[0]
-	fmla	v30.2d, v6.2d, v15.2d[0]
-	fmla	v31.2d, v7.2d, v15.2d[0]
+	fmla	v16.2d, v4.2d, v12.d[0]
+	fmla	v20.2d, v4.2d, v13.d[0]
+	fmla	v24.2d, v4.2d, v14.d[0]
+	fmla	v28.2d, v4.2d, v15.d[0]
+
+	fmla	v17.2d, v5.2d, v12.d[0]
+	fmla	v25.2d, v5.2d, v14.d[0]
+	fmla	v21.2d, v5.2d, v13.d[0]
+	fmla	v29.2d, v5.2d, v15.d[0]
+
+	prfm	PLDL1KEEP, [pB, #B_PRE_SIZE]
+
+	fmla	v18.2d, v6.2d, v12.d[0]
+	fmla	v22.2d, v6.2d, v13.d[0]
+	fmla	v26.2d, v6.2d, v14.d[0]
+	fmla	v30.2d, v6.2d, v15.d[0]
+
+	fmla	v19.2d, v7.2d, v12.d[0]
+	fmla	v23.2d, v7.2d, v13.d[0]
+	fmla	v27.2d, v7.2d, v14.d[0]
+	fmla	v31.2d, v7.2d, v15.d[0]
 .endm
 
 .macro KERNEL8x4_SUB
-	ld1	{v0.2d, v1.2d}, [pA]
-	add	pA, pA, #32
-	ld1	{v2.2d, v3.2d}, [pA]
-	add	pA, pA, #32
-	ldp	d8, d9, [pB]
-	add	pB, pB, #16
-	ldp	d10, d11, [pB]
-	add	pB, pB, #16
+	ldp	q0, q1, [pA], #32
+
+	ldp	d8, d9, [pB], #16
+
+	fmla	v16.2d, v0.2d, v8.d[0]
+	fmla	v20.2d, v0.2d, v9.d[0]
+
+	ldp	d10, d11, [pB], #16
+
+	fmla	v17.2d, v1.2d, v8.d[0]
+	fmla	v21.2d, v1.2d, v9.d[0]
+
+	ldp	q2, q3, [pA], #32
+
+	fmla	v24.2d, v0.2d, v10.d[0]
+	fmla	v28.2d, v0.2d, v11.d[0]
+
+	fmla	v25.2d, v1.2d, v10.d[0]
+	fmla	v29.2d, v1.2d, v11.d[0]
+
+	prfm	PLDL1KEEP, [pA, #A_PRE_SIZE]
+
+	fmla	v18.2d, v2.2d, v8.d[0]
+	fmla	v22.2d, v2.2d, v9.d[0]
 
-	fmla	v16.2d, v0.2d, v8.2d[0]
-	fmla	v17.2d, v1.2d, v8.2d[0]
-	fmla	v18.2d, v2.2d, v8.2d[0]
-	fmla	v19.2d, v3.2d, v8.2d[0]
-
-	fmla	v20.2d, v0.2d, v9.2d[0]
-	fmla	v21.2d, v1.2d, v9.2d[0]
-	fmla	v22.2d, v2.2d, v9.2d[0]
-	fmla	v23.2d, v3.2d, v9.2d[0]
-
-	fmla	v24.2d, v0.2d, v10.2d[0]
-	fmla	v25.2d, v1.2d, v10.2d[0]
-	fmla	v26.2d, v2.2d, v10.2d[0]
-	fmla	v27.2d, v3.2d, v10.2d[0]
-
-	fmla	v28.2d, v0.2d, v11.2d[0]
-	fmla	v29.2d, v1.2d, v11.2d[0]
-	fmla	v30.2d, v2.2d, v11.2d[0]
-	fmla	v31.2d, v3.2d, v11.2d[0]
+	prfm	PLDL1KEEP, [pA, #A_PRE_SIZE+64]
+
+	fmla	v26.2d, v2.2d, v10.d[0]
+	fmla	v30.2d, v2.2d, v11.d[0]
+
+	prfm	PLDL1KEEP, [pB, #B_PRE_SIZE]
+
+	fmla	v19.2d, v3.2d, v8.d[0]
+	fmla	v27.2d, v3.2d, v10.d[0]
+
+	fmla	v31.2d, v3.2d, v11.d[0]
+	fmla	v23.2d, v3.2d, v9.d[0]
 .endm
 
 .macro SAVE8x4
 	fmov	alpha0, alpha
 
-	ld1	{v0.2d, v1.2d}, [pCRow0]
+	prfm	PLDL2KEEP, [pCRow0, #C_PRE_SIZE]
+
+	ldp	q0, q1, [pCRow0]
 	fmla	v0.2d, v16.2d, alphaV0
 	fmla	v1.2d, v17.2d, alphaV0
-	st1 	{v0.2d, v1.2d}, [pCRow0]
+	stp 	q0, q1, [pCRow0]
 
 	add	pCRow0, pCRow0, #32
+	prfm	PLDL2KEEP, [pCRow0, #C_PRE_SIZE]
 
-	ld1	{v2.2d, v3.2d}, [pCRow0]
+	ldp	q2, q3, [pCRow0]
 	fmla	v2.2d, v18.2d, alphaV0
 	fmla	v3.2d, v19.2d, alphaV0
-	st1 	{v2.2d, v3.2d}, [pCRow0]
+	stp 	q2, q3, [pCRow0]
 
 	add	pCRow0, pCRow0, #32
 
-	ld1	{v4.2d, v5.2d}, [pCRow1]
+	prfm	PLDL2KEEP, [pCRow1, #C_PRE_SIZE]
+
+	ldp	q4, q5, [pCRow1]
 	fmla	v4.2d, v20.2d, alphaV0
 	fmla	v5.2d, v21.2d, alphaV0
-	st1 	{v4.2d, v5.2d}, [pCRow1]
+	stp 	q4, q5, [pCRow1]
 
 	add	pCRow1, pCRow1, #32
+	prfm	PLDL2KEEP, [pCRow1, #C_PRE_SIZE]
 
-	ld1	{v6.2d, v7.2d}, [pCRow1]
+	ldp	q6, q7, [pCRow1]
 	fmla	v6.2d, v22.2d, alphaV0
 	fmla	v7.2d, v23.2d, alphaV0
-	st1 	{v6.2d, v7.2d}, [pCRow1]
+	stp 	q6, q7, [pCRow1]
 
 	add	pCRow1, pCRow1, #32
 
-	ld1	{v0.2d, v1.2d}, [pCRow2]
+	prfm	PLDL2KEEP, [pCRow2, #C_PRE_SIZE]
+
+	ldp	q0, q1, [pCRow2]
 	fmla	v0.2d, v24.2d, alphaV0
 	fmla	v1.2d, v25.2d, alphaV0
-	st1 	{v0.2d, v1.2d}, [pCRow2]
+	stp 	q0, q1, [pCRow2]
 
 	add	pCRow2, pCRow2, #32
-	ld1	{v2.2d, v3.2d}, [pCRow2]
+	prfm	PLDL2KEEP, [pCRow2, #C_PRE_SIZE]
+
+	ldp	q2, q3, [pCRow2]
 	fmla	v2.2d, v26.2d, alphaV0
 	fmla	v3.2d, v27.2d, alphaV0
-	st1 	{v2.2d, v3.2d}, [pCRow2]
+	stp 	q2, q3, [pCRow2]
 
 	add	pCRow2, pCRow2, #32
 
-	ld1	{v4.2d, v5.2d}, [pCRow3]
+	prfm	PLDL2KEEP, [pCRow3, #C_PRE_SIZE]
+
+	ldp	q4, q5, [pCRow3]
 	fmla	v4.2d, v28.2d, alphaV0
 	fmla	v5.2d, v29.2d, alphaV0
-	st1 	{v4.2d, v5.2d}, [pCRow3]
+	stp 	q4, q5, [pCRow3]
 
 	add	pCRow3, pCRow3, #32
+	prfm	PLDL2KEEP, [pCRow3, #C_PRE_SIZE]
 
-	ld1	{v6.2d, v7.2d}, [pCRow3]
+	ldp	q6, q7, [pCRow3]
 	fmla	v6.2d, v30.2d, alphaV0
 	fmla	v7.2d, v31.2d, alphaV0
-	st1 	{v6.2d, v7.2d}, [pCRow3]
+	stp 	q6, q7, [pCRow3]
 
 	add	pCRow3, pCRow3, #32
-
-	prfm	PLDL2KEEP, [pCRow0, #128]
-	prfm	PLDL2KEEP, [pCRow1, #128]
-	prfm	PLDL2KEEP, [pCRow2, #128]
-	prfm	PLDL2KEEP, [pCRow3, #128]
 .endm
 
 /******************************************************************************/
@@ -408,44 +419,45 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2d, v1.2d}, [pA]
 	add	pA, pA, #32
 
-	fmla	v16.2d, v0.2d, v8.2d[0]
-	fmla	v29.2d, v1.2d, v9.2d[1]
+	fmla	v16.2d, v0.2d, v8.d[0]
+	fmla	v29.2d, v1.2d, v9.d[1]
 
-	fmla	v20.2d, v0.2d, v8.2d[1]
-	fmla	v25.2d, v1.2d, v9.2d[0]
+	fmla	v20.2d, v0.2d, v8.d[1]
+	fmla	v25.2d, v1.2d, v9.d[0]
 
-	fmla	v24.2d, v0.2d, v9.2d[0]
-	fmla	v21.2d, v1.2d, v8.2d[1]
+	fmla	v24.2d, v0.2d, v9.d[0]
+	fmla	v21.2d, v1.2d, v8.d[1]
 
-	fmla	v28.2d, v0.2d, v9.2d[1]
-	fmla	v17.2d, v1.2d, v8.2d[0]
+	fmla	v28.2d, v0.2d, v9.d[1]
+	fmla	v17.2d, v1.2d, v8.d[0]
 .endm
 
 .macro SAVE4x4
+	fmov	alpha0, alpha
 	ld1	{v8.2d, v9.2d}, [pCRow0]
 	fmla	v8.2d, v16.2d, alphaV0
-	fmla	v9.2d, v17.2d, alphaV1
+	fmla	v9.2d, v17.2d, alphaV0
 	st1 	{v8.2d, v9.2d}, [pCRow0]
 
 	add	pCRow1, pCRow0, LDC
 
 	ld1	{v12.2d, v13.2d}, [pCRow1]
-	fmla	v12.2d, v20.2d, alphaV2
-	fmla	v13.2d, v21.2d, alphaV3
+	fmla	v12.2d, v20.2d, alphaV0
+	fmla	v13.2d, v21.2d, alphaV0
 	st1 	{v12.2d, v13.2d}, [pCRow1]
 
 	add	pCRow2, pCRow1, LDC
 
 	ld1	{v8.2d, v9.2d}, [pCRow2]
 	fmla	v8.2d, v24.2d, alphaV0
-	fmla	v9.2d, v25.2d, alphaV1
+	fmla	v9.2d, v25.2d, alphaV0
 	st1 	{v8.2d, v9.2d}, [pCRow2]
 
 	add	pCRow1, pCRow2, LDC
 
 	ld1	{v12.2d, v13.2d}, [pCRow1]
-	fmla	v12.2d, v28.2d, alphaV2
-	fmla	v13.2d, v29.2d, alphaV3
+	fmla	v12.2d, v28.2d, alphaV0
+	fmla	v13.2d, v29.2d, alphaV0
 	st1 	{v12.2d, v13.2d}, [pCRow1]
 
 	add	pCRow0, pCRow0, #32
@@ -467,13 +479,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2d}, [pA]
 	add	pA, pA, #16
 
-	fmla	v16.2d, v0.2d, v8.2d[0]
-	fmla	v20.2d, v0.2d, v8.2d[1]
-	fmla	v24.2d, v0.2d, v9.2d[0]
-	fmla	v28.2d, v0.2d, v9.2d[1]
+	fmla	v16.2d, v0.2d, v8.d[0]
+	fmla	v20.2d, v0.2d, v8.d[1]
+	fmla	v24.2d, v0.2d, v9.d[0]
+	fmla	v28.2d, v0.2d, v9.d[1]
 .endm
 
 .macro SAVE2x4
+	fmov	alpha0, alpha
 	ld1	{v8.2d}, [pCRow0]
 	fmla	v8.2d, v16.2d, alphaV0
 	st1	{v8.2d}, [pCRow0]
@@ -481,19 +494,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	add	pCRow1, pCRow0, LDC
 
 	ld1	{v12.2d}, [pCRow1]
-	fmla	v12.2d, v20.2d, alphaV1
+	fmla	v12.2d, v20.2d, alphaV0
 	st1	{v12.2d}, [pCRow1]
 
 	add	pCRow2, pCRow1, LDC
 
 	ld1	{v8.2d}, [pCRow2]
-	fmla	v8.2d, v24.2d, alphaV2
+	fmla	v8.2d, v24.2d, alphaV0
 	st1	{v8.2d}, [pCRow2]
 
 	add	pCRow1, pCRow2, LDC
 
 	ld1	{v12.2d}, [pCRow1]
-	fmla	v12.2d, v28.2d, alphaV3
+	fmla	v12.2d, v28.2d, alphaV0
 	st1	{v12.2d}, [pCRow1]
 
 	add	pCRow0, pCRow0, #16
@@ -518,6 +531,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro SAVE1x4
+	fmov	alpha0, alpha
 	add	pCRow1, pCRow0, LDC
 
 	ld1	{v8.d}[0], [pCRow0]
@@ -531,7 +545,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 	ld1	{v12.d}[0], [pCRow2]
 	ld1	{v12.d}[1], [pCRow1]
-	fmla	v12.2d, v20.2d, alphaV1
+	fmla	v12.2d, v20.2d, alphaV0
 	st1	{v12.d}[0], [pCRow2]
 	st1	{v12.d}[1], [pCRow1]
 
@@ -559,32 +573,33 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v2.2d, v3.2d}, [pA]
 	add	pA, pA, #32
 
-	fmla	v16.2d, v0.2d, v8.2d[0]
-	fmla	v17.2d, v1.2d, v8.2d[0]
-	fmla	v18.2d, v2.2d, v8.2d[0]
-	fmla	v19.2d, v3.2d, v8.2d[0]
+	fmla	v16.2d, v0.2d, v8.d[0]
+	fmla	v17.2d, v1.2d, v8.d[0]
+	fmla	v18.2d, v2.2d, v8.d[0]
+	fmla	v19.2d, v3.2d, v8.d[0]
 
-	fmla	v20.2d, v0.2d, v8.2d[1]
-	fmla	v21.2d, v1.2d, v8.2d[1]
-	fmla	v22.2d, v2.2d, v8.2d[1]
-	fmla	v23.2d, v3.2d, v8.2d[1]
+	fmla	v20.2d, v0.2d, v8.d[1]
+	fmla	v21.2d, v1.2d, v8.d[1]
+	fmla	v22.2d, v2.2d, v8.d[1]
+	fmla	v23.2d, v3.2d, v8.d[1]
 .endm
 
 .macro SAVE8x2
+	fmov	alpha0, alpha
 	add	pCRow1, pCRow0, LDC
 
 	ld1	{v0.2d, v1.2d, v2.2d, v3.2d}, [pCRow0]
 	fmla	v0.2d, v16.2d, alphaV0
-	fmla	v1.2d, v17.2d, alphaV1
-	fmla	v2.2d, v18.2d, alphaV2
-	fmla	v3.2d, v19.2d, alphaV3
+	fmla	v1.2d, v17.2d, alphaV0
+	fmla	v2.2d, v18.2d, alphaV0
+	fmla	v3.2d, v19.2d, alphaV0
 	st1 	{v0.2d, v1.2d, v2.2d, v3.2d}, [pCRow0]
 
 	ld1	{v4.2d, v5.2d, v6.2d, v7.2d}, [pCRow1]
 	fmla	v4.2d, v20.2d, alphaV0
-	fmla	v5.2d, v21.2d, alphaV1
-	fmla	v6.2d, v22.2d, alphaV2
-	fmla	v7.2d, v23.2d, alphaV3
+	fmla	v5.2d, v21.2d, alphaV0
+	fmla	v6.2d, v22.2d, alphaV0
+	fmla	v7.2d, v23.2d, alphaV0
 	st1 	{v4.2d, v5.2d, v6.2d, v7.2d}, [pCRow1]
 
 	add	pCRow0, pCRow0, #64
@@ -605,23 +620,24 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2d, v1.2d}, [pA]
 	add	pA, pA, #32
 
-	fmla	v16.2d, v0.2d, v8.2d[0]
-	fmla	v17.2d, v1.2d, v8.2d[0]
-	fmla	v20.2d, v0.2d, v8.2d[1]
-	fmla	v21.2d, v1.2d, v8.2d[1]
+	fmla	v16.2d, v0.2d, v8.d[0]
+	fmla	v17.2d, v1.2d, v8.d[0]
+	fmla	v20.2d, v0.2d, v8.d[1]
+	fmla	v21.2d, v1.2d, v8.d[1]
 .endm
 
 .macro SAVE4x2
+	fmov	alpha0, alpha
 	ld1	{v8.2d, v9.2d}, [pCRow0]
 	fmla	v8.2d, v16.2d, alphaV0
-	fmla	v9.2d, v17.2d, alphaV1
+	fmla	v9.2d, v17.2d, alphaV0
 	st1	{v8.2d, v9.2d}, [pCRow0]
 
 	add	pCRow1, pCRow0, LDC
 
 	ld1	{v12.2d, v13.2d}, [pCRow1]
-	fmla	v12.2d, v20.2d, alphaV2
-	fmla	v13.2d, v21.2d, alphaV3
+	fmla	v12.2d, v20.2d, alphaV0
+	fmla	v13.2d, v21.2d, alphaV0
 	st1	{v12.2d, v13.2d}, [pCRow1]
 
 	add	pCRow0, pCRow0, #32
@@ -641,11 +657,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2d}, [pA]
 	add	pA, pA, #16
 
-	fmla	v16.2d, v0.2d, v8.2d[0]
-	fmla	v20.2d, v0.2d, v8.2d[1]
+	fmla	v16.2d, v0.2d, v8.d[0]
+	fmla	v20.2d, v0.2d, v8.d[1]
 .endm
 
 .macro SAVE2x2
+	fmov	alpha0, alpha
 	ld1	{v8.2d}, [pCRow0]
 	fmla	v8.2d, v16.2d, alphaV0
 	st1	{v8.2d}, [pCRow0]
@@ -653,7 +670,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	add	pCRow1 , pCRow0, LDC
 
 	ld1	{v12.2d}, [pCRow1]
-	fmla	v12.2d, v20.2d, alphaV1
+	fmla	v12.2d, v20.2d, alphaV0
 	st1	{v12.2d}, [pCRow1]
 
 	add	pCRow0, pCRow0, #16
@@ -672,10 +689,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ldr	d0 , [pA]
 	add	pA, pA, #8
 
-	fmla	v16.2d, v8.2d, v0.2d[0]
+	fmla	v16.2d, v8.2d, v0.d[0]
 .endm
 
 .macro SAVE1x2
+	fmov	alpha0, alpha
 	add	pCRow1 , pCRow0, LDC
 
 	ld1	{v8.d}[0], [pCRow0]
@@ -706,18 +724,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v2.2d, v3.2d}, [pA]
 	add	pA, pA, #32
 
-	fmla	v16.2d, v0.2d, v8.2d[0]
-	fmla	v17.2d, v1.2d, v8.2d[0]
-	fmla	v18.2d, v2.2d, v8.2d[0]
-	fmla	v19.2d, v3.2d, v8.2d[0]
+	fmla	v16.2d, v0.2d, v8.d[0]
+	fmla	v17.2d, v1.2d, v8.d[0]
+	fmla	v18.2d, v2.2d, v8.d[0]
+	fmla	v19.2d, v3.2d, v8.d[0]
 .endm
 
 .macro SAVE8x1
+	fmov	alpha0, alpha
 	ld1	{v0.2d, v1.2d, v2.2d, v3.2d}, [pCRow0]
 	fmla	v0.2d, v16.2d, alphaV0
-	fmla	v1.2d, v17.2d, alphaV1
-	fmla	v2.2d, v18.2d, alphaV2
-	fmla	v3.2d, v19.2d, alphaV3
+	fmla	v1.2d, v17.2d, alphaV0
+	fmla	v2.2d, v18.2d, alphaV0
+	fmla	v3.2d, v19.2d, alphaV0
 	st1 	{v0.2d, v1.2d, v2.2d, v3.2d}, [pCRow0]
 
 	add	pCRow0, pCRow0, #64
@@ -738,14 +757,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2d, v1.2d}, [pA]
 	add	pA , pA, #32
 
-	fmla	v16.2d, v0.2d, v8.2d[0]
-	fmla	v17.2d, v1.2d, v8.2d[0]
+	fmla	v16.2d, v0.2d, v8.d[0]
+	fmla	v17.2d, v1.2d, v8.d[0]
 .endm
 
 .macro SAVE4x1
+	fmov	alpha0, alpha
 	ld1	{v8.2d, v9.2d}, [pCRow0]
 	fmla	v8.2d, v16.2d, alphaV0
-	fmla	v9.2d, v17.2d, alphaV1
+	fmla	v9.2d, v17.2d, alphaV0
 	st1	{v8.2d, v9.2d}, [pCRow0]
 
 	add	pCRow0, pCRow0, #32
@@ -765,10 +785,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2d}, [pA]
 	add	pA , pA, #16
 
-	fmla	v16.2d, v0.2d, v8.2d[0]
+	fmla	v16.2d, v0.2d, v8.d[0]
 .endm
 
 .macro SAVE2x1
+	fmov	alpha0, alpha
 	ld1	{v8.2d}, [pCRow0]
 	fmla	v8.2d, v16.2d, alphaV0
 	st1	{v8.2d}, [pCRow0]
@@ -793,6 +814,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro SAVE1x1
+	fmov	alpha0, alpha
 	ldr	d8, [pCRow0]
 	fmadd	d8, d16, alpha0, d8
 	str 	d8, [pCRow0]
@@ -820,6 +842,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	stp	x26, x27, [sp, #(9 * 16)]
 	str	x28, [sp, #(10 * 16)]
 
+	prfm	PLDL1KEEP, [origPB]
+	prfm	PLDL1KEEP, [origPA]
+
 	fmov	alpha, d0
 
 	lsl	LDC, LDC, #3			// ldc = ldc * 8
@@ -838,6 +863,7 @@ dgemm_kernel_L4_BEGIN:
 	add	pCRow1, pCRow0, LDC
 	add	pCRow2, pCRow1, LDC
 	add	pCRow3, pCRow2, LDC
+
 	add	pC, pCRow3, LDC
 
 	mov	pA, origPA			// pA = start of A array
@@ -849,6 +875,7 @@ dgemm_kernel_L4_M8_BEGIN:
 	cmp 	counterI, #0
 	ble	dgemm_kernel_L4_M4_BEGIN
 
+	.align 5
 dgemm_kernel_L4_M8_20:
 
 	mov	pB, origPB
@@ -868,8 +895,8 @@ dgemm_kernel_L4_M8_20:
 
 	subs	counterL, counterL, #2		// subtract 2
 	ble	dgemm_kernel_L4_M8_22a
-	.align 5
 
+	.align 5
 dgemm_kernel_L4_M8_22:
 
 	KERNEL8x4_M1
@@ -884,7 +911,7 @@ dgemm_kernel_L4_M8_22:
 	subs	counterL, counterL, #1
 	bgt	dgemm_kernel_L4_M8_22
 
-
+	.align 5
 dgemm_kernel_L4_M8_22a:
 
 	KERNEL8x4_M1
@@ -898,6 +925,7 @@ dgemm_kernel_L4_M8_22a:
 
 	b	 dgemm_kernel_L4_M8_44
 
+	.align 5
 dgemm_kernel_L4_M8_32:
 
 	tst	counterL, #1
@@ -923,6 +951,7 @@ dgemm_kernel_L4_M8_44:
 	ands	counterL , origK, #7
 	ble	dgemm_kernel_L4_M8_100
 
+	.align 5
 dgemm_kernel_L4_M8_46:
 
 	KERNEL8x4_SUB
@@ -931,6 +960,9 @@ dgemm_kernel_L4_M8_46:
 	bne	dgemm_kernel_L4_M8_46
 
 dgemm_kernel_L4_M8_100:
+	prfm	PLDL1KEEP, [pA]
+	prfm	PLDL1KEEP, [pA, #64]
+	prfm	PLDL1KEEP, [origPB]
 
 	SAVE8x4
 

kernel/arm64/dtrmm_kernel_4x4.S

@@ -147,17 +147,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2d, v1.2d}, [pA]
 	add	pA, pA, #32
 
-	fmul	v16.2d, v0.2d, v8.2d[0]
-	fmul	v29.2d, v1.2d, v9.2d[1]
+	fmul	v16.2d, v0.2d, v8.d[0]
+	fmul	v29.2d, v1.2d, v9.d[1]
 
-	fmul	v20.2d, v0.2d, v8.2d[1]
-	fmul	v25.2d, v1.2d, v9.2d[0]
+	fmul	v20.2d, v0.2d, v8.d[1]
+	fmul	v25.2d, v1.2d, v9.d[0]
 
-	fmul	v24.2d, v0.2d, v9.2d[0]
-	fmul	v21.2d, v1.2d, v8.2d[1]
+	fmul	v24.2d, v0.2d, v9.d[0]
+	fmul	v21.2d, v1.2d, v8.d[1]
 
-	fmul	v28.2d, v0.2d, v9.2d[1]
-	fmul	v17.2d, v1.2d, v8.2d[0]
+	fmul	v28.2d, v0.2d, v9.d[1]
+	fmul	v17.2d, v1.2d, v8.d[0]
 
 	ld1	{v12.2d, v13.2d}, [pB]
 	add	pB, pB, #32
@@ -166,61 +166,61 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL4x4_M1
-	fmla	v16.2d, v0.2d, v8.2d[0]
-	fmla	v29.2d, v1.2d, v9.2d[1]
+	fmla	v16.2d, v0.2d, v8.d[0]
+	fmla	v29.2d, v1.2d, v9.d[1]
 
 	ld1	{v12.2d, v13.2d}, [pB]		// For next round
 	add	pB, pB, #32
 
-	fmla	v20.2d, v0.2d, v8.2d[1]
-	fmla	v25.2d, v1.2d, v9.2d[0]
+	fmla	v20.2d, v0.2d, v8.d[1]
+	fmla	v25.2d, v1.2d, v9.d[0]
 
 	ld1	{v4.2d, v5.2d}, [pA]		// For next round
 	add	pA, pA, #32
 
-	fmla	v24.2d, v0.2d, v9.2d[0]
-	fmla	v21.2d, v1.2d, v8.2d[1]
+	fmla	v24.2d, v0.2d, v9.d[0]
+	fmla	v21.2d, v1.2d, v8.d[1]
 
 	prfm	PLDL1KEEP, [pA, #512]
 
-	fmla	v28.2d, v0.2d, v9.2d[1]
-	fmla	v17.2d, v1.2d, v8.2d[0]
+	fmla	v28.2d, v0.2d, v9.d[1]
+	fmla	v17.2d, v1.2d, v8.d[0]
 .endm
 
 .macro KERNEL4x4_M2
-	fmla	v16.2d, v4.2d, v12.2d[0]
-	fmla	v29.2d, v5.2d, v13.2d[1]
+	fmla	v16.2d, v4.2d, v12.d[0]
+	fmla	v29.2d, v5.2d, v13.d[1]
 
 	ld1	{v8.2d, v9.2d}, [pB]		// For next round
 	add	pB, pB, #32
 
-	fmla	v20.2d, v4.2d, v12.2d[1]
-	fmla	v25.2d, v5.2d, v13.2d[0]
+	fmla	v20.2d, v4.2d, v12.d[1]
+	fmla	v25.2d, v5.2d, v13.d[0]
 
 	ld1	{v0.2d, v1.2d}, [pA]		// For next round
 	add	pA, pA, #32
 
-	fmla	v24.2d, v4.2d, v13.2d[0]
-	fmla	v21.2d, v5.2d, v12.2d[1]
+	fmla	v24.2d, v4.2d, v13.d[0]
+	fmla	v21.2d, v5.2d, v12.d[1]
 
 	prfm	PLDL1KEEP, [pB, #512]
 
-	fmla	v28.2d, v4.2d, v13.2d[1]
-	fmla	v17.2d, v5.2d, v12.2d[0]
+	fmla	v28.2d, v4.2d, v13.d[1]
+	fmla	v17.2d, v5.2d, v12.d[0]
 .endm
 
 .macro KERNEL4x4_E
-	fmla	v16.2d, v4.2d, v12.2d[0]
-	fmla	v29.2d, v5.2d, v13.2d[1]
+	fmla	v16.2d, v4.2d, v12.d[0]
+	fmla	v29.2d, v5.2d, v13.d[1]
 
-	fmla	v20.2d, v4.2d, v12.2d[1]
-	fmla	v25.2d, v5.2d, v13.2d[0]
+	fmla	v20.2d, v4.2d, v12.d[1]
+	fmla	v25.2d, v5.2d, v13.d[0]
 
-	fmla	v24.2d, v4.2d, v13.2d[0]
-	fmla	v21.2d, v5.2d, v12.2d[1]
+	fmla	v24.2d, v4.2d, v13.d[0]
+	fmla	v21.2d, v5.2d, v12.d[1]
 
-	fmla	v28.2d, v4.2d, v13.2d[1]
-	fmla	v17.2d, v5.2d, v12.2d[0]
+	fmla	v28.2d, v4.2d, v13.d[1]
+	fmla	v17.2d, v5.2d, v12.d[0]
 .endm
 
 .macro KERNEL4x4_SUB
@@ -229,17 +229,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2d, v1.2d}, [pA]
 	add	pA, pA, #32
 
-	fmla	v16.2d, v0.2d, v8.2d[0]
-	fmla	v29.2d, v1.2d, v9.2d[1]
+	fmla	v16.2d, v0.2d, v8.d[0]
+	fmla	v29.2d, v1.2d, v9.d[1]
 
-	fmla	v20.2d, v0.2d, v8.2d[1]
-	fmla	v25.2d, v1.2d, v9.2d[0]
+	fmla	v20.2d, v0.2d, v8.d[1]
+	fmla	v25.2d, v1.2d, v9.d[0]
 
-	fmla	v24.2d, v0.2d, v9.2d[0]
-	fmla	v21.2d, v1.2d, v8.2d[1]
+	fmla	v24.2d, v0.2d, v9.d[0]
+	fmla	v21.2d, v1.2d, v8.d[1]
 
-	fmla	v28.2d, v0.2d, v9.2d[1]
-	fmla	v17.2d, v1.2d, v8.2d[0]
+	fmla	v28.2d, v0.2d, v9.d[1]
+	fmla	v17.2d, v1.2d, v8.d[0]
 .endm
 
 .macro SAVE4x4
@@ -283,10 +283,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2d}, [pA]
 	add	pA, pA, #16
 
-	fmla	v16.2d, v0.2d, v8.2d[0]
-	fmla	v20.2d, v0.2d, v8.2d[1]
-	fmla	v24.2d, v0.2d, v9.2d[0]
-	fmla	v28.2d, v0.2d, v9.2d[1]
+	fmla	v16.2d, v0.2d, v8.d[0]
+	fmla	v20.2d, v0.2d, v8.d[1]
+	fmla	v24.2d, v0.2d, v9.d[0]
+	fmla	v28.2d, v0.2d, v9.d[1]
 .endm
 
 .macro SAVE2x4
@@ -361,10 +361,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2d, v1.2d}, [pA]
 	add	pA, pA, #32
 
-	fmla	v16.2d, v0.2d, v8.2d[0]
-	fmla	v17.2d, v1.2d, v8.2d[0]
-	fmla	v20.2d, v0.2d, v8.2d[1]
-	fmla	v21.2d, v1.2d, v8.2d[1]
+	fmla	v16.2d, v0.2d, v8.d[0]
+	fmla	v17.2d, v1.2d, v8.d[0]
+	fmla	v20.2d, v0.2d, v8.d[1]
+	fmla	v21.2d, v1.2d, v8.d[1]
 .endm
 
 .macro SAVE4x2
@@ -395,8 +395,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2d}, [pA]
 	add	pA, pA, #16
 
-	fmla	v16.2d, v0.2d, v8.2d[0]
-	fmla	v20.2d, v0.2d, v8.2d[1]
+	fmla	v16.2d, v0.2d, v8.d[0]
+	fmla	v20.2d, v0.2d, v8.d[1]
 .endm
 
 .macro SAVE2x2
@@ -424,7 +424,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ldr	d0 , [pA]
 	add	pA, pA, #8
 
-	fmla	v16.2d, v8.2d, v0.2d[0]
+	fmla	v16.2d, v8.2d, v0.d[0]
 .endm
 
 .macro SAVE1x2
@@ -451,8 +451,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2d, v1.2d}, [pA]
 	add	pA , pA, #32
 
-	fmla	v16.2d, v0.2d, v8.2d[0]
-	fmla	v17.2d, v1.2d, v8.2d[0]
+	fmla	v16.2d, v0.2d, v8.d[0]
+	fmla	v17.2d, v1.2d, v8.d[0]
 .endm
 
 .macro SAVE4x1
@@ -479,7 +479,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2d}, [pA]
 	add	pA , pA, #16
 
-	fmla	v16.2d, v0.2d, v8.2d[0]
+	fmla	v16.2d, v0.2d, v8.d[0]
 .endm
 
 .macro SAVE2x1

kernel/arm64/dtrmm_kernel_4x8.S

@@ -157,25 +157,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v10.2d, v11.2d}, [pB]
 	add	pB, pB, #32
 
-	fmul	v16.2d, v0.2d, v8.2d[0]
-	fmul	v17.2d, v1.2d, v8.2d[0]
-	fmul	v18.2d, v0.2d, v8.2d[1]
-	fmul	v19.2d, v1.2d, v8.2d[1]
-
-	fmul	v20.2d, v0.2d, v9.2d[0]
-	fmul	v21.2d, v1.2d, v9.2d[0]
-	fmul	v22.2d, v0.2d, v9.2d[1]
-	fmul	v23.2d, v1.2d, v9.2d[1]
-
-	fmul	v24.2d, v0.2d, v10.2d[0]
-	fmul	v25.2d, v1.2d, v10.2d[0]
-	fmul	v26.2d, v0.2d, v10.2d[1]
-	fmul	v27.2d, v1.2d, v10.2d[1]
-
-	fmul	v28.2d, v0.2d, v11.2d[0]
-	fmul	v29.2d, v1.2d, v11.2d[0]
-	fmul	v30.2d, v0.2d, v11.2d[1]
-	fmul	v31.2d, v1.2d, v11.2d[1]
+	fmul	v16.2d, v0.2d, v8.d[0]
+	fmul	v17.2d, v1.2d, v8.d[0]
+	fmul	v18.2d, v0.2d, v8.d[1]
+	fmul	v19.2d, v1.2d, v8.d[1]
+
+	fmul	v20.2d, v0.2d, v9.d[0]
+	fmul	v21.2d, v1.2d, v9.d[0]
+	fmul	v22.2d, v0.2d, v9.d[1]
+	fmul	v23.2d, v1.2d, v9.d[1]
+
+	fmul	v24.2d, v0.2d, v10.d[0]
+	fmul	v25.2d, v1.2d, v10.d[0]
+	fmul	v26.2d, v0.2d, v10.d[1]
+	fmul	v27.2d, v1.2d, v10.d[1]
+
+	fmul	v28.2d, v0.2d, v11.d[0]
+	fmul	v29.2d, v1.2d, v11.d[0]
+	fmul	v30.2d, v0.2d, v11.d[1]
+	fmul	v31.2d, v1.2d, v11.d[1]
 
 	ld1	{v12.2d, v13.2d}, [pB]
 	add	pB, pB, #32
@@ -186,25 +186,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL4x8_M1
-	fmla	v16.2d, v0.2d, v8.2d[0]
-	fmla	v17.2d, v1.2d, v8.2d[0]
-	fmla	v18.2d, v0.2d, v8.2d[1]
-	fmla	v19.2d, v1.2d, v8.2d[1]
-
-	fmla	v20.2d, v0.2d, v9.2d[0]
-	fmla	v21.2d, v1.2d, v9.2d[0]
-	fmla	v22.2d, v0.2d, v9.2d[1]
-	fmla	v23.2d, v1.2d, v9.2d[1]
-
-	fmla	v24.2d, v0.2d, v10.2d[0]
-	fmla	v25.2d, v1.2d, v10.2d[0]
-	fmla	v26.2d, v0.2d, v10.2d[1]
-	fmla	v27.2d, v1.2d, v10.2d[1]
-
-	fmla	v28.2d, v0.2d, v11.2d[0]
-	fmla	v29.2d, v1.2d, v11.2d[0]
-	fmla	v30.2d, v0.2d, v11.2d[1]
-	fmla	v31.2d, v1.2d, v11.2d[1]
+	fmla	v16.2d, v0.2d, v8.d[0]
+	fmla	v17.2d, v1.2d, v8.d[0]
+	fmla	v18.2d, v0.2d, v8.d[1]
+	fmla	v19.2d, v1.2d, v8.d[1]
+
+	fmla	v20.2d, v0.2d, v9.d[0]
+	fmla	v21.2d, v1.2d, v9.d[0]
+	fmla	v22.2d, v0.2d, v9.d[1]
+	fmla	v23.2d, v1.2d, v9.d[1]
+
+	fmla	v24.2d, v0.2d, v10.d[0]
+	fmla	v25.2d, v1.2d, v10.d[0]
+	fmla	v26.2d, v0.2d, v10.d[1]
+	fmla	v27.2d, v1.2d, v10.d[1]
+
+	fmla	v28.2d, v0.2d, v11.d[0]
+	fmla	v29.2d, v1.2d, v11.d[0]
+	fmla	v30.2d, v0.2d, v11.d[1]
+	fmla	v31.2d, v1.2d, v11.d[1]
 
 	ld1	{v12.2d, v13.2d}, [pB]		// For next round
 	add	pB, pB, #32
@@ -217,25 +217,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL4x8_M2
-	fmla	v16.2d, v4.2d, v12.2d[0]
-	fmla	v17.2d, v5.2d, v12.2d[0]
-	fmla	v18.2d, v4.2d, v12.2d[1]
-	fmla	v19.2d, v5.2d, v12.2d[1]
-
-	fmla	v20.2d, v4.2d, v13.2d[0]
-	fmla	v21.2d, v5.2d, v13.2d[0]
-	fmla	v22.2d, v4.2d, v13.2d[1]
-	fmla	v23.2d, v5.2d, v13.2d[1]
-
-	fmla	v24.2d, v4.2d, v14.2d[0]
-	fmla	v25.2d, v5.2d, v14.2d[0]
-	fmla	v26.2d, v4.2d, v14.2d[1]
-	fmla	v27.2d, v5.2d, v14.2d[1]
-
-	fmla	v28.2d, v4.2d, v15.2d[0]
-	fmla	v29.2d, v5.2d, v15.2d[0]
-	fmla	v30.2d, v4.2d, v15.2d[1]
-	fmla	v31.2d, v5.2d, v15.2d[1]
+	fmla	v16.2d, v4.2d, v12.d[0]
+	fmla	v17.2d, v5.2d, v12.d[0]
+	fmla	v18.2d, v4.2d, v12.d[1]
+	fmla	v19.2d, v5.2d, v12.d[1]
+
+	fmla	v20.2d, v4.2d, v13.d[0]
+	fmla	v21.2d, v5.2d, v13.d[0]
+	fmla	v22.2d, v4.2d, v13.d[1]
+	fmla	v23.2d, v5.2d, v13.d[1]
+
+	fmla	v24.2d, v4.2d, v14.d[0]
+	fmla	v25.2d, v5.2d, v14.d[0]
+	fmla	v26.2d, v4.2d, v14.d[1]
+	fmla	v27.2d, v5.2d, v14.d[1]
+
+	fmla	v28.2d, v4.2d, v15.d[0]
+	fmla	v29.2d, v5.2d, v15.d[0]
+	fmla	v30.2d, v4.2d, v15.d[1]
+	fmla	v31.2d, v5.2d, v15.d[1]
 
 	ld1	{v8.2d, v9.2d}, [pB]		// For next round
 	add	pB, pB, #32
@@ -248,25 +248,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL4x8_E
-	fmla	v16.2d, v4.2d, v12.2d[0]
-	fmla	v17.2d, v5.2d, v12.2d[0]
-	fmla	v18.2d, v4.2d, v12.2d[1]
-	fmla	v19.2d, v5.2d, v12.2d[1]
-
-	fmla	v20.2d, v4.2d, v13.2d[0]
-	fmla	v21.2d, v5.2d, v13.2d[0]
-	fmla	v22.2d, v4.2d, v13.2d[1]
-	fmla	v23.2d, v5.2d, v13.2d[1]
-
-	fmla	v24.2d, v4.2d, v14.2d[0]
-	fmla	v25.2d, v5.2d, v14.2d[0]
-	fmla	v26.2d, v4.2d, v14.2d[1]
-	fmla	v27.2d, v5.2d, v14.2d[1]
-
-	fmla	v28.2d, v4.2d, v15.2d[0]
-	fmla	v29.2d, v5.2d, v15.2d[0]
-	fmla	v30.2d, v4.2d, v15.2d[1]
-	fmla	v31.2d, v5.2d, v15.2d[1]
+	fmla	v16.2d, v4.2d, v12.d[0]
+	fmla	v17.2d, v5.2d, v12.d[0]
+	fmla	v18.2d, v4.2d, v12.d[1]
+	fmla	v19.2d, v5.2d, v12.d[1]
+
+	fmla	v20.2d, v4.2d, v13.d[0]
+	fmla	v21.2d, v5.2d, v13.d[0]
+	fmla	v22.2d, v4.2d, v13.d[1]
+	fmla	v23.2d, v5.2d, v13.d[1]
+
+	fmla	v24.2d, v4.2d, v14.d[0]
+	fmla	v25.2d, v5.2d, v14.d[0]
+	fmla	v26.2d, v4.2d, v14.d[1]
+	fmla	v27.2d, v5.2d, v14.d[1]
+
+	fmla	v28.2d, v4.2d, v15.d[0]
+	fmla	v29.2d, v5.2d, v15.d[0]
+	fmla	v30.2d, v4.2d, v15.d[1]
+	fmla	v31.2d, v5.2d, v15.d[1]
 .endm
 
 .macro KERNEL4x8_SUB
@@ -277,25 +277,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v10.2d, v11.2d}, [pB]
 	add	pB, pB, #32
 
-	fmla	v16.2d, v0.2d, v8.2d[0]
-	fmla	v17.2d, v1.2d, v8.2d[0]
-	fmla	v18.2d, v0.2d, v8.2d[1]
-	fmla	v19.2d, v1.2d, v8.2d[1]
-
-	fmla	v20.2d, v0.2d, v9.2d[0]
-	fmla	v21.2d, v1.2d, v9.2d[0]
-	fmla	v22.2d, v0.2d, v9.2d[1]
-	fmla	v23.2d, v1.2d, v9.2d[1]
-
-	fmla	v24.2d, v0.2d, v10.2d[0]
-	fmla	v25.2d, v1.2d, v10.2d[0]
-	fmla	v26.2d, v0.2d, v10.2d[1]
-	fmla	v27.2d, v1.2d, v10.2d[1]
-
-	fmla	v28.2d, v0.2d, v11.2d[0]
-	fmla	v29.2d, v1.2d, v11.2d[0]
-	fmla	v30.2d, v0.2d, v11.2d[1]
-	fmla	v31.2d, v1.2d, v11.2d[1]
+	fmla	v16.2d, v0.2d, v8.d[0]
+	fmla	v17.2d, v1.2d, v8.d[0]
+	fmla	v18.2d, v0.2d, v8.d[1]
+	fmla	v19.2d, v1.2d, v8.d[1]
+
+	fmla	v20.2d, v0.2d, v9.d[0]
+	fmla	v21.2d, v1.2d, v9.d[0]
+	fmla	v22.2d, v0.2d, v9.d[1]
+	fmla	v23.2d, v1.2d, v9.d[1]
+
+	fmla	v24.2d, v0.2d, v10.d[0]
+	fmla	v25.2d, v1.2d, v10.d[0]
+	fmla	v26.2d, v0.2d, v10.d[1]
+	fmla	v27.2d, v1.2d, v10.d[1]
+
+	fmla	v28.2d, v0.2d, v11.d[0]
+	fmla	v29.2d, v1.2d, v11.d[0]
+	fmla	v30.2d, v0.2d, v11.d[1]
+	fmla	v31.2d, v1.2d, v11.d[1]
 .endm
 
 .macro SAVE4x8
@@ -369,17 +369,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v10.2d, v11.2d}, [pB]
 	add	pB, pB, #32
 
-	fmla	v16.2d, v0.2d, v8.2d[0]
-	fmla	v18.2d, v0.2d, v8.2d[1]
+	fmla	v16.2d, v0.2d, v8.d[0]
+	fmla	v18.2d, v0.2d, v8.d[1]
 
-	fmla	v20.2d, v0.2d, v9.2d[0]
-	fmla	v22.2d, v0.2d, v9.2d[1]
+	fmla	v20.2d, v0.2d, v9.d[0]
+	fmla	v22.2d, v0.2d, v9.d[1]
 
-	fmla	v24.2d, v0.2d, v10.2d[0]
-	fmla	v26.2d, v0.2d, v10.2d[1]
+	fmla	v24.2d, v0.2d, v10.d[0]
+	fmla	v26.2d, v0.2d, v10.d[1]
 
-	fmla	v28.2d, v0.2d, v11.2d[0]
-	fmla	v30.2d, v0.2d, v11.2d[1]
+	fmla	v28.2d, v0.2d, v11.d[0]
+	fmla	v30.2d, v0.2d, v11.d[1]
 .endm
 
 .macro SAVE2x8
@@ -499,17 +499,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2d, v1.2d}, [pA]
 	add	pA, pA, #32
 
-	fmul	v16.2d, v0.2d, v8.2d[0]
-	fmul	v29.2d, v1.2d, v9.2d[1]
+	fmul	v16.2d, v0.2d, v8.d[0]
+	fmul	v29.2d, v1.2d, v9.d[1]
 
-	fmul	v20.2d, v0.2d, v8.2d[1]
-	fmul	v25.2d, v1.2d, v9.2d[0]
+	fmul	v20.2d, v0.2d, v8.d[1]
+	fmul	v25.2d, v1.2d, v9.d[0]
 
-	fmul	v24.2d, v0.2d, v9.2d[0]
-	fmul	v21.2d, v1.2d, v8.2d[1]
+	fmul	v24.2d, v0.2d, v9.d[0]
+	fmul	v21.2d, v1.2d, v8.d[1]
 
-	fmul	v28.2d, v0.2d, v9.2d[1]
-	fmul	v17.2d, v1.2d, v8.2d[0]
+	fmul	v28.2d, v0.2d, v9.d[1]
+	fmul	v17.2d, v1.2d, v8.d[0]
 
 	ld1	{v12.2d, v13.2d}, [pB]
 	add	pB, pB, #32
@@ -518,61 +518,61 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL4x4_M1
-	fmla	v16.2d, v0.2d, v8.2d[0]
-	fmla	v29.2d, v1.2d, v9.2d[1]
+	fmla	v16.2d, v0.2d, v8.d[0]
+	fmla	v29.2d, v1.2d, v9.d[1]
 
 	ld1	{v12.2d, v13.2d}, [pB]		// For next round
 	add	pB, pB, #32
 
-	fmla	v20.2d, v0.2d, v8.2d[1]
-	fmla	v25.2d, v1.2d, v9.2d[0]
+	fmla	v20.2d, v0.2d, v8.d[1]
+	fmla	v25.2d, v1.2d, v9.d[0]
 
 	ld1	{v4.2d, v5.2d}, [pA]		// For next round
 	add	pA, pA, #32
 
-	fmla	v24.2d, v0.2d, v9.2d[0]
-	fmla	v21.2d, v1.2d, v8.2d[1]
+	fmla	v24.2d, v0.2d, v9.d[0]
+	fmla	v21.2d, v1.2d, v8.d[1]
 
 	prfm	PLDL1KEEP, [pA, #512]
 
-	fmla	v28.2d, v0.2d, v9.2d[1]
-	fmla	v17.2d, v1.2d, v8.2d[0]
+	fmla	v28.2d, v0.2d, v9.d[1]
+	fmla	v17.2d, v1.2d, v8.d[0]
 .endm
 
 .macro KERNEL4x4_M2
-	fmla	v16.2d, v4.2d, v12.2d[0]
-	fmla	v29.2d, v5.2d, v13.2d[1]
+	fmla	v16.2d, v4.2d, v12.d[0]
+	fmla	v29.2d, v5.2d, v13.d[1]
 
 	ld1	{v8.2d, v9.2d}, [pB]		// For next round
 	add	pB, pB, #32
 
-	fmla	v20.2d, v4.2d, v12.2d[1]
-	fmla	v25.2d, v5.2d, v13.2d[0]
+	fmla	v20.2d, v4.2d, v12.d[1]
+	fmla	v25.2d, v5.2d, v13.d[0]
 
 	ld1	{v0.2d, v1.2d}, [pA]		// For next round
 	add	pA, pA, #32
 
-	fmla	v24.2d, v4.2d, v13.2d[0]
-	fmla	v21.2d, v5.2d, v12.2d[1]
+	fmla	v24.2d, v4.2d, v13.d[0]
+	fmla	v21.2d, v5.2d, v12.d[1]
 
 	prfm	PLDL1KEEP, [pB, #512]
 
-	fmla	v28.2d, v4.2d, v13.2d[1]
-	fmla	v17.2d, v5.2d, v12.2d[0]
+	fmla	v28.2d, v4.2d, v13.d[1]
+	fmla	v17.2d, v5.2d, v12.d[0]
 .endm
 
 .macro KERNEL4x4_E
-	fmla	v16.2d, v4.2d, v12.2d[0]
-	fmla	v29.2d, v5.2d, v13.2d[1]
+	fmla	v16.2d, v4.2d, v12.d[0]
+	fmla	v29.2d, v5.2d, v13.d[1]
 
-	fmla	v20.2d, v4.2d, v12.2d[1]
-	fmla	v25.2d, v5.2d, v13.2d[0]
+	fmla	v20.2d, v4.2d, v12.d[1]
+	fmla	v25.2d, v5.2d, v13.d[0]
 
-	fmla	v24.2d, v4.2d, v13.2d[0]
-	fmla	v21.2d, v5.2d, v12.2d[1]
+	fmla	v24.2d, v4.2d, v13.d[0]
+	fmla	v21.2d, v5.2d, v12.d[1]
 
-	fmla	v28.2d, v4.2d, v13.2d[1]
-	fmla	v17.2d, v5.2d, v12.2d[0]
+	fmla	v28.2d, v4.2d, v13.d[1]
+	fmla	v17.2d, v5.2d, v12.d[0]
 .endm
 
 .macro KERNEL4x4_SUB
@@ -581,17 +581,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2d, v1.2d}, [pA]
 	add	pA, pA, #32
 
-	fmla	v16.2d, v0.2d, v8.2d[0]
-	fmla	v29.2d, v1.2d, v9.2d[1]
+	fmla	v16.2d, v0.2d, v8.d[0]
+	fmla	v29.2d, v1.2d, v9.d[1]
 
-	fmla	v20.2d, v0.2d, v8.2d[1]
-	fmla	v25.2d, v1.2d, v9.2d[0]
+	fmla	v20.2d, v0.2d, v8.d[1]
+	fmla	v25.2d, v1.2d, v9.d[0]
 
-	fmla	v24.2d, v0.2d, v9.2d[0]
-	fmla	v21.2d, v1.2d, v8.2d[1]
+	fmla	v24.2d, v0.2d, v9.d[0]
+	fmla	v21.2d, v1.2d, v8.d[1]
 
-	fmla	v28.2d, v0.2d, v9.2d[1]
-	fmla	v17.2d, v1.2d, v8.2d[0]
+	fmla	v28.2d, v0.2d, v9.d[1]
+	fmla	v17.2d, v1.2d, v8.d[0]
 .endm
 
 .macro SAVE4x4
@@ -635,10 +635,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2d}, [pA]
 	add	pA, pA, #16
 
-	fmla	v16.2d, v0.2d, v8.2d[0]
-	fmla	v20.2d, v0.2d, v8.2d[1]
-	fmla	v24.2d, v0.2d, v9.2d[0]
-	fmla	v28.2d, v0.2d, v9.2d[1]
+	fmla	v16.2d, v0.2d, v8.d[0]
+	fmla	v20.2d, v0.2d, v8.d[1]
+	fmla	v24.2d, v0.2d, v9.d[0]
+	fmla	v28.2d, v0.2d, v9.d[1]
 .endm
 
 .macro SAVE2x4
@@ -713,10 +713,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2d, v1.2d}, [pA]
 	add	pA, pA, #32
 
-	fmla	v16.2d, v0.2d, v8.2d[0]
-	fmla	v17.2d, v1.2d, v8.2d[0]
-	fmla	v20.2d, v0.2d, v8.2d[1]
-	fmla	v21.2d, v1.2d, v8.2d[1]
+	fmla	v16.2d, v0.2d, v8.d[0]
+	fmla	v17.2d, v1.2d, v8.d[0]
+	fmla	v20.2d, v0.2d, v8.d[1]
+	fmla	v21.2d, v1.2d, v8.d[1]
 .endm
 
 .macro SAVE4x2
@@ -747,8 +747,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2d}, [pA]
 	add	pA, pA, #16
 
-	fmla	v16.2d, v0.2d, v8.2d[0]
-	fmla	v20.2d, v0.2d, v8.2d[1]
+	fmla	v16.2d, v0.2d, v8.d[0]
+	fmla	v20.2d, v0.2d, v8.d[1]
 .endm
 
 .macro SAVE2x2
@@ -776,7 +776,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ldr	d0 , [pA]
 	add	pA, pA, #8
 
-	fmla	v16.2d, v8.2d, v0.2d[0]
+	fmla	v16.2d, v8.2d, v0.d[0]
 .endm
 
 .macro SAVE1x2
@@ -803,8 +803,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2d, v1.2d}, [pA]
 	add	pA , pA, #32
 
-	fmla	v16.2d, v0.2d, v8.2d[0]
-	fmla	v17.2d, v1.2d, v8.2d[0]
+	fmla	v16.2d, v0.2d, v8.d[0]
+	fmla	v17.2d, v1.2d, v8.d[0]
 .endm
 
 .macro SAVE4x1
@@ -831,7 +831,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2d}, [pA]
 	add	pA , pA, #16
 
-	fmla	v16.2d, v0.2d, v8.2d[0]
+	fmla	v16.2d, v0.2d, v8.d[0]
 .endm
 
 .macro SAVE2x1

kernel/arm64/dtrmm_kernel_8x4.S

@@ -157,25 +157,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v2.2d, v3.2d}, [pA]
 	add	pA, pA, #32
 
-	fmul	v16.2d, v0.2d, v8.2d[0]
-	fmul	v17.2d, v1.2d, v8.2d[0]
-	fmul	v18.2d, v2.2d, v8.2d[0]
-	fmul	v19.2d, v3.2d, v8.2d[0]
-
-	fmul	v20.2d, v0.2d, v8.2d[1]
-	fmul	v21.2d, v1.2d, v8.2d[1]
-	fmul	v22.2d, v2.2d, v8.2d[1]
-	fmul	v23.2d, v3.2d, v8.2d[1]
-
-	fmul	v24.2d, v0.2d, v9.2d[0]
-	fmul	v25.2d, v1.2d, v9.2d[0]
-	fmul	v26.2d, v2.2d, v9.2d[0]
-	fmul	v27.2d, v3.2d, v9.2d[0]
-
-	fmul	v28.2d, v0.2d, v9.2d[1]
-	fmul	v29.2d, v1.2d, v9.2d[1]
-	fmul	v30.2d, v2.2d, v9.2d[1]
-	fmul	v31.2d, v3.2d, v9.2d[1]
+	fmul	v16.2d, v0.2d, v8.d[0]
+	fmul	v17.2d, v1.2d, v8.d[0]
+	fmul	v18.2d, v2.2d, v8.d[0]
+	fmul	v19.2d, v3.2d, v8.d[0]
+
+	fmul	v20.2d, v0.2d, v8.d[1]
+	fmul	v21.2d, v1.2d, v8.d[1]
+	fmul	v22.2d, v2.2d, v8.d[1]
+	fmul	v23.2d, v3.2d, v8.d[1]
+
+	fmul	v24.2d, v0.2d, v9.d[0]
+	fmul	v25.2d, v1.2d, v9.d[0]
+	fmul	v26.2d, v2.2d, v9.d[0]
+	fmul	v27.2d, v3.2d, v9.d[0]
+
+	fmul	v28.2d, v0.2d, v9.d[1]
+	fmul	v29.2d, v1.2d, v9.d[1]
+	fmul	v30.2d, v2.2d, v9.d[1]
+	fmul	v31.2d, v3.2d, v9.d[1]
 
 	ld1	{v4.2d, v5.2d}, [pA]
 	add	pA, pA, #32
@@ -186,25 +186,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL8x4_M1
-	fmla	v16.2d, v0.2d, v8.2d[0]
-	fmla	v17.2d, v1.2d, v8.2d[0]
-	fmla	v18.2d, v2.2d, v8.2d[0]
-	fmla	v19.2d, v3.2d, v8.2d[0]
-
-	fmla	v20.2d, v0.2d, v8.2d[1]
-	fmla	v21.2d, v1.2d, v8.2d[1]
-	fmla	v22.2d, v2.2d, v8.2d[1]
-	fmla	v23.2d, v3.2d, v8.2d[1]
-
-	fmla	v24.2d, v0.2d, v9.2d[0]
-	fmla	v25.2d, v1.2d, v9.2d[0]
-	fmla	v26.2d, v2.2d, v9.2d[0]
-	fmla	v27.2d, v3.2d, v9.2d[0]
-
-	fmla	v28.2d, v0.2d, v9.2d[1]
-	fmla	v29.2d, v1.2d, v9.2d[1]
-	fmla	v30.2d, v2.2d, v9.2d[1]
-	fmla	v31.2d, v3.2d, v9.2d[1]
+	fmla	v16.2d, v0.2d, v8.d[0]
+	fmla	v17.2d, v1.2d, v8.d[0]
+	fmla	v18.2d, v2.2d, v8.d[0]
+	fmla	v19.2d, v3.2d, v8.d[0]
+
+	fmla	v20.2d, v0.2d, v8.d[1]
+	fmla	v21.2d, v1.2d, v8.d[1]
+	fmla	v22.2d, v2.2d, v8.d[1]
+	fmla	v23.2d, v3.2d, v8.d[1]
+
+	fmla	v24.2d, v0.2d, v9.d[0]
+	fmla	v25.2d, v1.2d, v9.d[0]
+	fmla	v26.2d, v2.2d, v9.d[0]
+	fmla	v27.2d, v3.2d, v9.d[0]
+
+	fmla	v28.2d, v0.2d, v9.d[1]
+	fmla	v29.2d, v1.2d, v9.d[1]
+	fmla	v30.2d, v2.2d, v9.d[1]
+	fmla	v31.2d, v3.2d, v9.d[1]
 
 	ld1	{v4.2d, v5.2d}, [pA]
 	add	pA, pA, #32
@@ -217,25 +217,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL8x4_M2
-	fmla	v16.2d, v4.2d, v12.2d[0]
-	fmla	v17.2d, v5.2d, v12.2d[0]
-	fmla	v18.2d, v6.2d, v12.2d[0]
-	fmla	v19.2d, v7.2d, v12.2d[0]
-
-	fmla	v20.2d, v4.2d, v12.2d[1]
-	fmla	v21.2d, v5.2d, v12.2d[1]
-	fmla	v22.2d, v6.2d, v12.2d[1]
-	fmla	v23.2d, v7.2d, v12.2d[1]
-
-	fmla	v24.2d, v4.2d, v13.2d[0]
-	fmla	v25.2d, v5.2d, v13.2d[0]
-	fmla	v26.2d, v6.2d, v13.2d[0]
-	fmla	v27.2d, v7.2d, v13.2d[0]
-
-	fmla	v28.2d, v4.2d, v13.2d[1]
-	fmla	v29.2d, v5.2d, v13.2d[1]
-	fmla	v30.2d, v6.2d, v13.2d[1]
-	fmla	v31.2d, v7.2d, v13.2d[1]
+	fmla	v16.2d, v4.2d, v12.d[0]
+	fmla	v17.2d, v5.2d, v12.d[0]
+	fmla	v18.2d, v6.2d, v12.d[0]
+	fmla	v19.2d, v7.2d, v12.d[0]
+
+	fmla	v20.2d, v4.2d, v12.d[1]
+	fmla	v21.2d, v5.2d, v12.d[1]
+	fmla	v22.2d, v6.2d, v12.d[1]
+	fmla	v23.2d, v7.2d, v12.d[1]
+
+	fmla	v24.2d, v4.2d, v13.d[0]
+	fmla	v25.2d, v5.2d, v13.d[0]
+	fmla	v26.2d, v6.2d, v13.d[0]
+	fmla	v27.2d, v7.2d, v13.d[0]
+
+	fmla	v28.2d, v4.2d, v13.d[1]
+	fmla	v29.2d, v5.2d, v13.d[1]
+	fmla	v30.2d, v6.2d, v13.d[1]
+	fmla	v31.2d, v7.2d, v13.d[1]
 
 	ld1	{v0.2d, v1.2d}, [pA]
 	add	pA, pA, #32
@@ -248,25 +248,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL8x4_E
-	fmla	v16.2d, v4.2d, v12.2d[0]
-	fmla	v17.2d, v5.2d, v12.2d[0]
-	fmla	v18.2d, v6.2d, v12.2d[0]
-	fmla	v19.2d, v7.2d, v12.2d[0]
-
-	fmla	v20.2d, v4.2d, v12.2d[1]
-	fmla	v21.2d, v5.2d, v12.2d[1]
-	fmla	v22.2d, v6.2d, v12.2d[1]
-	fmla	v23.2d, v7.2d, v12.2d[1]
-
-	fmla	v24.2d, v4.2d, v13.2d[0]
-	fmla	v25.2d, v5.2d, v13.2d[0]
-	fmla	v26.2d, v6.2d, v13.2d[0]
-	fmla	v27.2d, v7.2d, v13.2d[0]
-
-	fmla	v28.2d, v4.2d, v13.2d[1]
-	fmla	v29.2d, v5.2d, v13.2d[1]
-	fmla	v30.2d, v6.2d, v13.2d[1]
-	fmla	v31.2d, v7.2d, v13.2d[1]
+	fmla	v16.2d, v4.2d, v12.d[0]
+	fmla	v17.2d, v5.2d, v12.d[0]
+	fmla	v18.2d, v6.2d, v12.d[0]
+	fmla	v19.2d, v7.2d, v12.d[0]
+
+	fmla	v20.2d, v4.2d, v12.d[1]
+	fmla	v21.2d, v5.2d, v12.d[1]
+	fmla	v22.2d, v6.2d, v12.d[1]
+	fmla	v23.2d, v7.2d, v12.d[1]
+
+	fmla	v24.2d, v4.2d, v13.d[0]
+	fmla	v25.2d, v5.2d, v13.d[0]
+	fmla	v26.2d, v6.2d, v13.d[0]
+	fmla	v27.2d, v7.2d, v13.d[0]
+
+	fmla	v28.2d, v4.2d, v13.d[1]
+	fmla	v29.2d, v5.2d, v13.d[1]
+	fmla	v30.2d, v6.2d, v13.d[1]
+	fmla	v31.2d, v7.2d, v13.d[1]
 .endm
 
 .macro KERNEL8x4_SUB
@@ -277,25 +277,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v2.2d, v3.2d}, [pA]
 	add	pA, pA, #32
 
-	fmla	v16.2d, v0.2d, v8.2d[0]
-	fmla	v17.2d, v1.2d, v8.2d[0]
-	fmla	v18.2d, v2.2d, v8.2d[0]
-	fmla	v19.2d, v3.2d, v8.2d[0]
-
-	fmla	v20.2d, v0.2d, v8.2d[1]
-	fmla	v21.2d, v1.2d, v8.2d[1]
-	fmla	v22.2d, v2.2d, v8.2d[1]
-	fmla	v23.2d, v3.2d, v8.2d[1]
-
-	fmla	v24.2d, v0.2d, v9.2d[0]
-	fmla	v25.2d, v1.2d, v9.2d[0]
-	fmla	v26.2d, v2.2d, v9.2d[0]
-	fmla	v27.2d, v3.2d, v9.2d[0]
-
-	fmla	v28.2d, v0.2d, v9.2d[1]
-	fmla	v29.2d, v1.2d, v9.2d[1]
-	fmla	v30.2d, v2.2d, v9.2d[1]
-	fmla	v31.2d, v3.2d, v9.2d[1]
+	fmla	v16.2d, v0.2d, v8.d[0]
+	fmla	v17.2d, v1.2d, v8.d[0]
+	fmla	v18.2d, v2.2d, v8.d[0]
+	fmla	v19.2d, v3.2d, v8.d[0]
+
+	fmla	v20.2d, v0.2d, v8.d[1]
+	fmla	v21.2d, v1.2d, v8.d[1]
+	fmla	v22.2d, v2.2d, v8.d[1]
+	fmla	v23.2d, v3.2d, v8.d[1]
+
+	fmla	v24.2d, v0.2d, v9.d[0]
+	fmla	v25.2d, v1.2d, v9.d[0]
+	fmla	v26.2d, v2.2d, v9.d[0]
+	fmla	v27.2d, v3.2d, v9.d[0]
+
+	fmla	v28.2d, v0.2d, v9.d[1]
+	fmla	v29.2d, v1.2d, v9.d[1]
+	fmla	v30.2d, v2.2d, v9.d[1]
+	fmla	v31.2d, v3.2d, v9.d[1]
 .endm
 
 .macro SAVE8x4
@@ -351,17 +351,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2d, v1.2d}, [pA]
 	add	pA, pA, #32
 
-	fmla	v16.2d, v0.2d, v8.2d[0]
-	fmla	v29.2d, v1.2d, v9.2d[1]
+	fmla	v16.2d, v0.2d, v8.d[0]
+	fmla	v29.2d, v1.2d, v9.d[1]
 
-	fmla	v20.2d, v0.2d, v8.2d[1]
-	fmla	v25.2d, v1.2d, v9.2d[0]
+	fmla	v20.2d, v0.2d, v8.d[1]
+	fmla	v25.2d, v1.2d, v9.d[0]
 
-	fmla	v24.2d, v0.2d, v9.2d[0]
-	fmla	v21.2d, v1.2d, v8.2d[1]
+	fmla	v24.2d, v0.2d, v9.d[0]
+	fmla	v21.2d, v1.2d, v8.d[1]
 
-	fmla	v28.2d, v0.2d, v9.2d[1]
-	fmla	v17.2d, v1.2d, v8.2d[0]
+	fmla	v28.2d, v0.2d, v9.d[1]
+	fmla	v17.2d, v1.2d, v8.d[0]
 .endm
 
 .macro SAVE4x4
@@ -406,10 +406,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2d}, [pA]
 	add	pA, pA, #16
 
-	fmla	v16.2d, v0.2d, v8.2d[0]
-	fmla	v20.2d, v0.2d, v8.2d[1]
-	fmla	v24.2d, v0.2d, v9.2d[0]
-	fmla	v28.2d, v0.2d, v9.2d[1]
+	fmla	v16.2d, v0.2d, v8.d[0]
+	fmla	v20.2d, v0.2d, v8.d[1]
+	fmla	v24.2d, v0.2d, v9.d[0]
+	fmla	v28.2d, v0.2d, v9.d[1]
 .endm
 
 .macro SAVE2x4
@@ -490,15 +490,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v2.2d, v3.2d}, [pA]
 	add	pA, pA, #32
 
-	fmla	v16.2d, v0.2d, v8.2d[0]
-	fmla	v17.2d, v1.2d, v8.2d[0]
-	fmla	v18.2d, v2.2d, v8.2d[0]
-	fmla	v19.2d, v3.2d, v8.2d[0]
+	fmla	v16.2d, v0.2d, v8.d[0]
+	fmla	v17.2d, v1.2d, v8.d[0]
+	fmla	v18.2d, v2.2d, v8.d[0]
+	fmla	v19.2d, v3.2d, v8.d[0]
 
-	fmla	v20.2d, v0.2d, v8.2d[1]
-	fmla	v21.2d, v1.2d, v8.2d[1]
-	fmla	v22.2d, v2.2d, v8.2d[1]
-	fmla	v23.2d, v3.2d, v8.2d[1]
+	fmla	v20.2d, v0.2d, v8.d[1]
+	fmla	v21.2d, v1.2d, v8.d[1]
+	fmla	v22.2d, v2.2d, v8.d[1]
+	fmla	v23.2d, v3.2d, v8.d[1]
 .endm
 
 .macro SAVE8x2
@@ -534,10 +534,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2d, v1.2d}, [pA]
 	add	pA, pA, #32
 
-	fmla	v16.2d, v0.2d, v8.2d[0]
-	fmla	v17.2d, v1.2d, v8.2d[0]
-	fmla	v20.2d, v0.2d, v8.2d[1]
-	fmla	v21.2d, v1.2d, v8.2d[1]
+	fmla	v16.2d, v0.2d, v8.d[0]
+	fmla	v17.2d, v1.2d, v8.d[0]
+	fmla	v20.2d, v0.2d, v8.d[1]
+	fmla	v21.2d, v1.2d, v8.d[1]
 .endm
 
 .macro SAVE4x2
@@ -568,8 +568,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2d}, [pA]
 	add	pA, pA, #16
 
-	fmla	v16.2d, v0.2d, v8.2d[0]
-	fmla	v20.2d, v0.2d, v8.2d[1]
+	fmla	v16.2d, v0.2d, v8.d[0]
+	fmla	v20.2d, v0.2d, v8.d[1]
 .endm
 
 .macro SAVE2x2
@@ -597,7 +597,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ldr	d0 , [pA]
 	add	pA, pA, #8
 
-	fmla	v16.2d, v8.2d, v0.2d[0]
+	fmla	v16.2d, v8.2d, v0.d[0]
 .endm
 
 .macro SAVE1x2
@@ -629,10 +629,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v2.2d, v3.2d}, [pA]
 	add	pA, pA, #32
 
-	fmla	v16.2d, v0.2d, v8.2d[0]
-	fmla	v17.2d, v1.2d, v8.2d[0]
-	fmla	v18.2d, v2.2d, v8.2d[0]
-	fmla	v19.2d, v3.2d, v8.2d[0]
+	fmla	v16.2d, v0.2d, v8.d[0]
+	fmla	v17.2d, v1.2d, v8.d[0]
+	fmla	v18.2d, v2.2d, v8.d[0]
+	fmla	v19.2d, v3.2d, v8.d[0]
 .endm
 
 .macro SAVE8x1
@@ -660,8 +660,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2d, v1.2d}, [pA]
 	add	pA , pA, #32
 
-	fmla	v16.2d, v0.2d, v8.2d[0]
-	fmla	v17.2d, v1.2d, v8.2d[0]
+	fmla	v16.2d, v0.2d, v8.d[0]
+	fmla	v17.2d, v1.2d, v8.d[0]
 .endm
 
 .macro SAVE4x1
@@ -686,7 +686,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2d}, [pA]
 	add	pA , pA, #16
 
-	fmla	v16.2d, v0.2d, v8.2d[0]
+	fmla	v16.2d, v0.2d, v8.d[0]
 .endm
 
 .macro SAVE2x1

kernel/arm64/sgemm_kernel_16x4.S

@@ -158,25 +158,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v3.4s}, [pA]
 	add	pA, pA, #16
 
-	fmul	v16.4s, v0.4s, v8.2s[0]
-	fmul	v17.4s, v1.4s, v8.2s[0]
-	fmul	v18.4s, v2.4s, v8.2s[0]
-	fmul	v19.4s, v3.4s, v8.2s[0]
-
-	fmul	v20.4s, v0.4s, v8.2s[1]
-	fmul	v21.4s, v1.4s, v8.2s[1]
-	fmul	v22.4s, v2.4s, v8.2s[1]
-	fmul	v23.4s, v3.4s, v8.2s[1]
-
-	fmul	v24.4s, v0.4s, v9.2s[0]
-	fmul	v25.4s, v1.4s, v9.2s[0]
-	fmul	v26.4s, v2.4s, v9.2s[0]
-	fmul	v27.4s, v3.4s, v9.2s[0]
-
-	fmul	v28.4s, v0.4s, v9.2s[1]
-	fmul	v29.4s, v1.4s, v9.2s[1]
-	fmul	v30.4s, v2.4s, v9.2s[1]
-	fmul	v31.4s, v3.4s, v9.2s[1]
+	fmul	v16.4s, v0.4s, v8.s[0]
+	fmul	v17.4s, v1.4s, v8.s[0]
+	fmul	v18.4s, v2.4s, v8.s[0]
+	fmul	v19.4s, v3.4s, v8.s[0]
+
+	fmul	v20.4s, v0.4s, v8.s[1]
+	fmul	v21.4s, v1.4s, v8.s[1]
+	fmul	v22.4s, v2.4s, v8.s[1]
+	fmul	v23.4s, v3.4s, v8.s[1]
+
+	fmul	v24.4s, v0.4s, v9.s[0]
+	fmul	v25.4s, v1.4s, v9.s[0]
+	fmul	v26.4s, v2.4s, v9.s[0]
+	fmul	v27.4s, v3.4s, v9.s[0]
+
+	fmul	v28.4s, v0.4s, v9.s[1]
+	fmul	v29.4s, v1.4s, v9.s[1]
+	fmul	v30.4s, v2.4s, v9.s[1]
+	fmul	v31.4s, v3.4s, v9.s[1]
 
 	ld1	{v12.2s, v13.2s}, [pB]
 	add	pB, pB, #16
@@ -191,25 +191,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL16x4_M1
-	fmla	v16.4s, v0.4s, v8.2s[0]
-	fmla	v17.4s, v1.4s, v8.2s[0]
-	fmla	v18.4s, v2.4s, v8.2s[0]
-	fmla	v19.4s, v3.4s, v8.2s[0]
-
-	fmla	v20.4s, v0.4s, v8.2s[1]
-	fmla	v21.4s, v1.4s, v8.2s[1]
-	fmla	v22.4s, v2.4s, v8.2s[1]
-	fmla	v23.4s, v3.4s, v8.2s[1]
-
-	fmla	v24.4s, v0.4s, v9.2s[0]
-	fmla	v25.4s, v1.4s, v9.2s[0]
-	fmla	v26.4s, v2.4s, v9.2s[0]
-	fmla	v27.4s, v3.4s, v9.2s[0]
-
-	fmla	v28.4s, v0.4s, v9.2s[1]
-	fmla	v29.4s, v1.4s, v9.2s[1]
-	fmla	v30.4s, v2.4s, v9.2s[1]
-	fmla	v31.4s, v3.4s, v9.2s[1]
+	fmla	v16.4s, v0.4s, v8.s[0]
+	fmla	v17.4s, v1.4s, v8.s[0]
+	fmla	v18.4s, v2.4s, v8.s[0]
+	fmla	v19.4s, v3.4s, v8.s[0]
+
+	fmla	v20.4s, v0.4s, v8.s[1]
+	fmla	v21.4s, v1.4s, v8.s[1]
+	fmla	v22.4s, v2.4s, v8.s[1]
+	fmla	v23.4s, v3.4s, v8.s[1]
+
+	fmla	v24.4s, v0.4s, v9.s[0]
+	fmla	v25.4s, v1.4s, v9.s[0]
+	fmla	v26.4s, v2.4s, v9.s[0]
+	fmla	v27.4s, v3.4s, v9.s[0]
+
+	fmla	v28.4s, v0.4s, v9.s[1]
+	fmla	v29.4s, v1.4s, v9.s[1]
+	fmla	v30.4s, v2.4s, v9.s[1]
+	fmla	v31.4s, v3.4s, v9.s[1]
 
 	ld1	{v12.2s, v13.2s}, [pB]
 	add	pB, pB, #16
@@ -224,25 +224,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL16x4_M2
-	fmla	v16.4s, v4.4s, v12.2s[0]
-	fmla	v17.4s, v5.4s, v12.2s[0]
-	fmla	v18.4s, v6.4s, v12.2s[0]
-	fmla	v19.4s, v7.4s, v12.2s[0]
-
-	fmla	v20.4s, v4.4s, v12.2s[1]
-	fmla	v21.4s, v5.4s, v12.2s[1]
-	fmla	v22.4s, v6.4s, v12.2s[1]
-	fmla	v23.4s, v7.4s, v12.2s[1]
-
-	fmla	v24.4s, v4.4s, v13.2s[0]
-	fmla	v25.4s, v5.4s, v13.2s[0]
-	fmla	v26.4s, v6.4s, v13.2s[0]
-	fmla	v27.4s, v7.4s, v13.2s[0]
-
-	fmla	v28.4s, v4.4s, v13.2s[1]
-	fmla	v29.4s, v5.4s, v13.2s[1]
-	fmla	v30.4s, v6.4s, v13.2s[1]
-	fmla	v31.4s, v7.4s, v13.2s[1]
+	fmla	v16.4s, v4.4s, v12.s[0]
+	fmla	v17.4s, v5.4s, v12.s[0]
+	fmla	v18.4s, v6.4s, v12.s[0]
+	fmla	v19.4s, v7.4s, v12.s[0]
+
+	fmla	v20.4s, v4.4s, v12.s[1]
+	fmla	v21.4s, v5.4s, v12.s[1]
+	fmla	v22.4s, v6.4s, v12.s[1]
+	fmla	v23.4s, v7.4s, v12.s[1]
+
+	fmla	v24.4s, v4.4s, v13.s[0]
+	fmla	v25.4s, v5.4s, v13.s[0]
+	fmla	v26.4s, v6.4s, v13.s[0]
+	fmla	v27.4s, v7.4s, v13.s[0]
+
+	fmla	v28.4s, v4.4s, v13.s[1]
+	fmla	v29.4s, v5.4s, v13.s[1]
+	fmla	v30.4s, v6.4s, v13.s[1]
+	fmla	v31.4s, v7.4s, v13.s[1]
 
 	ld1	{v8.2s, v9.2s}, [pB]
 	add	pB, pB, #16
@@ -257,25 +257,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL16x4_E
-	fmla	v16.4s, v4.4s, v12.2s[0]
-	fmla	v17.4s, v5.4s, v12.2s[0]
-	fmla	v18.4s, v6.4s, v12.2s[0]
-	fmla	v19.4s, v7.4s, v12.2s[0]
-
-	fmla	v20.4s, v4.4s, v12.2s[1]
-	fmla	v21.4s, v5.4s, v12.2s[1]
-	fmla	v22.4s, v6.4s, v12.2s[1]
-	fmla	v23.4s, v7.4s, v12.2s[1]
-
-	fmla	v24.4s, v4.4s, v13.2s[0]
-	fmla	v25.4s, v5.4s, v13.2s[0]
-	fmla	v26.4s, v6.4s, v13.2s[0]
-	fmla	v27.4s, v7.4s, v13.2s[0]
-
-	fmla	v28.4s, v4.4s, v13.2s[1]
-	fmla	v29.4s, v5.4s, v13.2s[1]
-	fmla	v30.4s, v6.4s, v13.2s[1]
-	fmla	v31.4s, v7.4s, v13.2s[1]
+	fmla	v16.4s, v4.4s, v12.s[0]
+	fmla	v17.4s, v5.4s, v12.s[0]
+	fmla	v18.4s, v6.4s, v12.s[0]
+	fmla	v19.4s, v7.4s, v12.s[0]
+
+	fmla	v20.4s, v4.4s, v12.s[1]
+	fmla	v21.4s, v5.4s, v12.s[1]
+	fmla	v22.4s, v6.4s, v12.s[1]
+	fmla	v23.4s, v7.4s, v12.s[1]
+
+	fmla	v24.4s, v4.4s, v13.s[0]
+	fmla	v25.4s, v5.4s, v13.s[0]
+	fmla	v26.4s, v6.4s, v13.s[0]
+	fmla	v27.4s, v7.4s, v13.s[0]
+
+	fmla	v28.4s, v4.4s, v13.s[1]
+	fmla	v29.4s, v5.4s, v13.s[1]
+	fmla	v30.4s, v6.4s, v13.s[1]
+	fmla	v31.4s, v7.4s, v13.s[1]
 .endm
 
 .macro KERNEL16x4_SUB
@@ -290,25 +290,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v3.4s}, [pA]
 	add	pA, pA, #16
 
-	fmla	v16.4s, v0.4s, v8.2s[0]
-	fmla	v17.4s, v1.4s, v8.2s[0]
-	fmla	v18.4s, v2.4s, v8.2s[0]
-	fmla	v19.4s, v3.4s, v8.2s[0]
-
-	fmla	v20.4s, v0.4s, v8.2s[1]
-	fmla	v21.4s, v1.4s, v8.2s[1]
-	fmla	v22.4s, v2.4s, v8.2s[1]
-	fmla	v23.4s, v3.4s, v8.2s[1]
-
-	fmla	v24.4s, v0.4s, v9.2s[0]
-	fmla	v25.4s, v1.4s, v9.2s[0]
-	fmla	v26.4s, v2.4s, v9.2s[0]
-	fmla	v27.4s, v3.4s, v9.2s[0]
-
-	fmla	v28.4s, v0.4s, v9.2s[1]
-	fmla	v29.4s, v1.4s, v9.2s[1]
-	fmla	v30.4s, v2.4s, v9.2s[1]
-	fmla	v31.4s, v3.4s, v9.2s[1]
+	fmla	v16.4s, v0.4s, v8.s[0]
+	fmla	v17.4s, v1.4s, v8.s[0]
+	fmla	v18.4s, v2.4s, v8.s[0]
+	fmla	v19.4s, v3.4s, v8.s[0]
+
+	fmla	v20.4s, v0.4s, v8.s[1]
+	fmla	v21.4s, v1.4s, v8.s[1]
+	fmla	v22.4s, v2.4s, v8.s[1]
+	fmla	v23.4s, v3.4s, v8.s[1]
+
+	fmla	v24.4s, v0.4s, v9.s[0]
+	fmla	v25.4s, v1.4s, v9.s[0]
+	fmla	v26.4s, v2.4s, v9.s[0]
+	fmla	v27.4s, v3.4s, v9.s[0]
+
+	fmla	v28.4s, v0.4s, v9.s[1]
+	fmla	v29.4s, v1.4s, v9.s[1]
+	fmla	v30.4s, v2.4s, v9.s[1]
+	fmla	v31.4s, v3.4s, v9.s[1]
 .endm
 
 .macro SAVE16x4
@@ -370,14 +370,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v1.4s}, [pA]
 	add	pA, pA, #16
 
-	fmul	v16.4s, v0.4s, v8.2s[0]
-	fmul	v17.4s, v1.4s, v8.2s[0]
-	fmul	v20.4s, v0.4s, v8.2s[1]
-	fmul	v21.4s, v1.4s, v8.2s[1]
-	fmul	v24.4s, v0.4s, v9.2s[0]
-	fmul	v25.4s, v1.4s, v9.2s[0]
-	fmul	v28.4s, v0.4s, v9.2s[1]
-	fmul	v29.4s, v1.4s, v9.2s[1]
+	fmul	v16.4s, v0.4s, v8.s[0]
+	fmul	v17.4s, v1.4s, v8.s[0]
+	fmul	v20.4s, v0.4s, v8.s[1]
+	fmul	v21.4s, v1.4s, v8.s[1]
+	fmul	v24.4s, v0.4s, v9.s[0]
+	fmul	v25.4s, v1.4s, v9.s[0]
+	fmul	v28.4s, v0.4s, v9.s[1]
+	fmul	v29.4s, v1.4s, v9.s[1]
 
 	ld1	{v12.2s, v13.2s}, [pB]
 	add	pB, pB, #16
@@ -388,14 +388,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL8x4_M1
-	fmla	v16.4s, v0.4s, v8.2s[0]
-	fmla	v17.4s, v1.4s, v8.2s[0]
-	fmla	v20.4s, v0.4s, v8.2s[1]
-	fmla	v21.4s, v1.4s, v8.2s[1]
-	fmla	v24.4s, v0.4s, v9.2s[0]
-	fmla	v25.4s, v1.4s, v9.2s[0]
-	fmla	v28.4s, v0.4s, v9.2s[1]
-	fmla	v29.4s, v1.4s, v9.2s[1]
+	fmla	v16.4s, v0.4s, v8.s[0]
+	fmla	v17.4s, v1.4s, v8.s[0]
+	fmla	v20.4s, v0.4s, v8.s[1]
+	fmla	v21.4s, v1.4s, v8.s[1]
+	fmla	v24.4s, v0.4s, v9.s[0]
+	fmla	v25.4s, v1.4s, v9.s[0]
+	fmla	v28.4s, v0.4s, v9.s[1]
+	fmla	v29.4s, v1.4s, v9.s[1]
 
 	ld1	{v12.2s, v13.2s}, [pB]
 	add	pB, pB, #16
@@ -406,14 +406,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL8x4_M2
-	fmla	v16.4s, v4.4s, v12.2s[0]
-	fmla	v17.4s, v5.4s, v12.2s[0]
-	fmla	v20.4s, v4.4s, v12.2s[1]
-	fmla	v21.4s, v5.4s, v12.2s[1]
-	fmla	v24.4s, v4.4s, v13.2s[0]
-	fmla	v25.4s, v5.4s, v13.2s[0]
-	fmla	v28.4s, v4.4s, v13.2s[1]
-	fmla	v29.4s, v5.4s, v13.2s[1]
+	fmla	v16.4s, v4.4s, v12.s[0]
+	fmla	v17.4s, v5.4s, v12.s[0]
+	fmla	v20.4s, v4.4s, v12.s[1]
+	fmla	v21.4s, v5.4s, v12.s[1]
+	fmla	v24.4s, v4.4s, v13.s[0]
+	fmla	v25.4s, v5.4s, v13.s[0]
+	fmla	v28.4s, v4.4s, v13.s[1]
+	fmla	v29.4s, v5.4s, v13.s[1]
 
 	ld1	{v8.2s, v9.2s}, [pB]
 	add	pB, pB, #16
@@ -424,14 +424,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL8x4_E
-	fmla	v16.4s, v4.4s, v12.2s[0]
-	fmla	v17.4s, v5.4s, v12.2s[0]
-	fmla	v20.4s, v4.4s, v12.2s[1]
-	fmla	v21.4s, v5.4s, v12.2s[1]
-	fmla	v24.4s, v4.4s, v13.2s[0]
-	fmla	v25.4s, v5.4s, v13.2s[0]
-	fmla	v28.4s, v4.4s, v13.2s[1]
-	fmla	v29.4s, v5.4s, v13.2s[1]
+	fmla	v16.4s, v4.4s, v12.s[0]
+	fmla	v17.4s, v5.4s, v12.s[0]
+	fmla	v20.4s, v4.4s, v12.s[1]
+	fmla	v21.4s, v5.4s, v12.s[1]
+	fmla	v24.4s, v4.4s, v13.s[0]
+	fmla	v25.4s, v5.4s, v13.s[0]
+	fmla	v28.4s, v4.4s, v13.s[1]
+	fmla	v29.4s, v5.4s, v13.s[1]
 .endm
 
 .macro KERNEL8x4_SUB
@@ -442,14 +442,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v1.4s}, [pA]
 	add	pA, pA, #16
 
-	fmla	v16.4s, v0.4s, v8.2s[0]
-	fmla	v17.4s, v1.4s, v8.2s[0]
-	fmla	v20.4s, v0.4s, v8.2s[1]
-	fmla	v21.4s, v1.4s, v8.2s[1]
-	fmla	v24.4s, v0.4s, v9.2s[0]
-	fmla	v25.4s, v1.4s, v9.2s[0]
-	fmla	v28.4s, v0.4s, v9.2s[1]
-	fmla	v29.4s, v1.4s, v9.2s[1]
+	fmla	v16.4s, v0.4s, v8.s[0]
+	fmla	v17.4s, v1.4s, v8.s[0]
+	fmla	v20.4s, v0.4s, v8.s[1]
+	fmla	v21.4s, v1.4s, v8.s[1]
+	fmla	v24.4s, v0.4s, v9.s[0]
+	fmla	v25.4s, v1.4s, v9.s[0]
+	fmla	v28.4s, v0.4s, v9.s[1]
+	fmla	v29.4s, v1.4s, v9.s[1]
 .endm
 
 .macro SAVE8x4
@@ -501,17 +501,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2s, v1.2s}, [pA]
 	add	pA, pA, #16
 
-	fmul	v16.2s, v0.2s, v8.2s[0]
-	fmul	v29.2s, v1.2s, v9.2s[1]
+	fmul	v16.2s, v0.2s, v8.s[0]
+	fmul	v29.2s, v1.2s, v9.s[1]
 
-	fmul	v20.2s, v0.2s, v8.2s[1]
-	fmul	v25.2s, v1.2s, v9.2s[0]
+	fmul	v20.2s, v0.2s, v8.s[1]
+	fmul	v25.2s, v1.2s, v9.s[0]
 
-	fmul	v24.2s, v0.2s, v9.2s[0]
-	fmul	v21.2s, v1.2s, v8.2s[1]
+	fmul	v24.2s, v0.2s, v9.s[0]
+	fmul	v21.2s, v1.2s, v8.s[1]
 
-	fmul	v28.2s, v0.2s, v9.2s[1]
-	fmul	v17.2s, v1.2s, v8.2s[0]
+	fmul	v28.2s, v0.2s, v9.s[1]
+	fmul	v17.2s, v1.2s, v8.s[0]
 
 	ld1	{v12.2s, v13.2s}, [pB]
 	add	pB, pB, #16
@@ -520,61 +520,61 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL4x4_M1
-	fmla	v16.2s, v0.2s, v8.2s[0]
-	fmla	v29.2s, v1.2s, v9.2s[1]
+	fmla	v16.2s, v0.2s, v8.s[0]
+	fmla	v29.2s, v1.2s, v9.s[1]
 
 	ld1	{v12.2s, v13.2s}, [pB]		// For next round
 	add	pB, pB, #16
 
-	fmla	v20.2s, v0.2s, v8.2s[1]
-	fmla	v25.2s, v1.2s, v9.2s[0]
+	fmla	v20.2s, v0.2s, v8.s[1]
+	fmla	v25.2s, v1.2s, v9.s[0]
 
 	ld1	{v4.2s, v5.2s}, [pA]		// For next round
 	add	pA, pA, #16
 
-	fmla	v24.2s, v0.2s, v9.2s[0]
-	fmla	v21.2s, v1.2s, v8.2s[1]
+	fmla	v24.2s, v0.2s, v9.s[0]
+	fmla	v21.2s, v1.2s, v8.s[1]
 
 	prfm	PLDL1KEEP, [pB, #512]
 
-	fmla	v28.2s, v0.2s, v9.2s[1]
-	fmla	v17.2s, v1.2s, v8.2s[0]
+	fmla	v28.2s, v0.2s, v9.s[1]
+	fmla	v17.2s, v1.2s, v8.s[0]
 .endm
 
 .macro KERNEL4x4_M2
-	fmla	v16.2s, v4.2s, v12.2s[0]
-	fmla	v29.2s, v5.2s, v13.2s[1]
+	fmla	v16.2s, v4.2s, v12.s[0]
+	fmla	v29.2s, v5.2s, v13.s[1]
 
 	ld1	{v8.2s, v9.2s}, [pB]		// For next round
 	add	pB, pB, #16
 
-	fmla	v20.2s, v4.2s, v12.2s[1]
-	fmla	v25.2s, v5.2s, v13.2s[0]
+	fmla	v20.2s, v4.2s, v12.s[1]
+	fmla	v25.2s, v5.2s, v13.s[0]
 
 	ld1	{v0.2s, v1.2s}, [pA]		// For next round
 	add	pA, pA, #16
 
-	fmla	v24.2s, v4.2s, v13.2s[0]
-	fmla	v21.2s, v5.2s, v12.2s[1]
+	fmla	v24.2s, v4.2s, v13.s[0]
+	fmla	v21.2s, v5.2s, v12.s[1]
 
 	prfm	PLDL1KEEP, [pA, #512]
 
-	fmla	v28.2s, v4.2s, v13.2s[1]
-	fmla	v17.2s, v5.2s, v12.2s[0]
+	fmla	v28.2s, v4.2s, v13.s[1]
+	fmla	v17.2s, v5.2s, v12.s[0]
 .endm
 
 .macro KERNEL4x4_E
-	fmla	v16.2s, v4.2s, v12.2s[0]
-	fmla	v29.2s, v5.2s, v13.2s[1]
+	fmla	v16.2s, v4.2s, v12.s[0]
+	fmla	v29.2s, v5.2s, v13.s[1]
 
-	fmla	v20.2s, v4.2s, v12.2s[1]
-	fmla	v25.2s, v5.2s, v13.2s[0]
+	fmla	v20.2s, v4.2s, v12.s[1]
+	fmla	v25.2s, v5.2s, v13.s[0]
 
-	fmla	v24.2s, v4.2s, v13.2s[0]
-	fmla	v21.2s, v5.2s, v12.2s[1]
+	fmla	v24.2s, v4.2s, v13.s[0]
+	fmla	v21.2s, v5.2s, v12.s[1]
 
-	fmla	v28.2s, v4.2s, v13.2s[1]
-	fmla	v17.2s, v5.2s, v12.2s[0]
+	fmla	v28.2s, v4.2s, v13.s[1]
+	fmla	v17.2s, v5.2s, v12.s[0]
 .endm
 
 .macro KERNEL4x4_SUB
@@ -583,17 +583,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2s, v1.2s}, [pA]
 	add	pA, pA, #16
 
-	fmla	v16.2s, v0.2s, v8.2s[0]
-	fmla	v29.2s, v1.2s, v9.2s[1]
+	fmla	v16.2s, v0.2s, v8.s[0]
+	fmla	v29.2s, v1.2s, v9.s[1]
 
-	fmla	v20.2s, v0.2s, v8.2s[1]
-	fmla	v25.2s, v1.2s, v9.2s[0]
+	fmla	v20.2s, v0.2s, v8.s[1]
+	fmla	v25.2s, v1.2s, v9.s[0]
 
-	fmla	v24.2s, v0.2s, v9.2s[0]
-	fmla	v21.2s, v1.2s, v8.2s[1]
+	fmla	v24.2s, v0.2s, v9.s[0]
+	fmla	v21.2s, v1.2s, v8.s[1]
 
-	fmla	v28.2s, v0.2s, v9.2s[1]
-	fmla	v17.2s, v1.2s, v8.2s[0]
+	fmla	v28.2s, v0.2s, v9.s[1]
+	fmla	v17.2s, v1.2s, v8.s[0]
 .endm
 
 .macro SAVE4x4
@@ -638,10 +638,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2s}, [pA]
 	add	pA, pA, #8
 
-	fmla	v16.2s, v0.2s, v8.2s[0]
-	fmla	v20.2s, v0.2s, v8.2s[1]
-	fmla	v24.2s, v0.2s, v9.2s[0]
-	fmla	v28.2s, v0.2s, v9.2s[1]
+	fmla	v16.2s, v0.2s, v8.s[0]
+	fmla	v20.2s, v0.2s, v8.s[1]
+	fmla	v24.2s, v0.2s, v9.s[0]
+	fmla	v28.2s, v0.2s, v9.s[1]
 .endm
 
 .macro SAVE2x4
@@ -729,15 +729,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v3.4s}, [pA]
 	add	pA, pA, #16
 
-	fmla	v16.4s, v0.4s, v8.2s[0]
-	fmla	v17.4s, v1.4s, v8.2s[0]
-	fmla	v18.4s, v2.4s, v8.2s[0]
-	fmla	v19.4s, v3.4s, v8.2s[0]
+	fmla	v16.4s, v0.4s, v8.s[0]
+	fmla	v17.4s, v1.4s, v8.s[0]
+	fmla	v18.4s, v2.4s, v8.s[0]
+	fmla	v19.4s, v3.4s, v8.s[0]
 
-	fmla	v20.4s, v0.4s, v8.2s[1]
-	fmla	v21.4s, v1.4s, v8.2s[1]
-	fmla	v22.4s, v2.4s, v8.2s[1]
-	fmla	v23.4s, v3.4s, v8.2s[1]
+	fmla	v20.4s, v0.4s, v8.s[1]
+	fmla	v21.4s, v1.4s, v8.s[1]
+	fmla	v22.4s, v2.4s, v8.s[1]
+	fmla	v23.4s, v3.4s, v8.s[1]
 .endm
 
 .macro SAVE16x2
@@ -777,11 +777,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v1.4s}, [pA]
 	add	pA, pA, #16
 
-	fmla	v16.4s, v0.4s, v8.2s[0]
-	fmla	v17.4s, v1.4s, v8.2s[0]
+	fmla	v16.4s, v0.4s, v8.s[0]
+	fmla	v17.4s, v1.4s, v8.s[0]
 
-	fmla	v20.4s, v0.4s, v8.2s[1]
-	fmla	v21.4s, v1.4s, v8.2s[1]
+	fmla	v20.4s, v0.4s, v8.s[1]
+	fmla	v21.4s, v1.4s, v8.s[1]
 .endm
 
 .macro SAVE8x2
@@ -817,10 +817,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2s, v1.2s}, [pA]
 	add	pA, pA, #16
 
-	fmla	v16.2s, v0.2s, v8.2s[0]
-	fmla	v17.2s, v1.2s, v8.2s[0]
-	fmla	v20.2s, v0.2s, v8.2s[1]
-	fmla	v21.2s, v1.2s, v8.2s[1]
+	fmla	v16.2s, v0.2s, v8.s[0]
+	fmla	v17.2s, v1.2s, v8.s[0]
+	fmla	v20.2s, v0.2s, v8.s[1]
+	fmla	v21.2s, v1.2s, v8.s[1]
 .endm
 
 .macro SAVE4x2
@@ -852,8 +852,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2s}, [pA]
 	add	pA, pA, #8
 
-	fmla	v16.2s, v0.2s, v8.2s[0]
-	fmla	v20.2s, v0.2s, v8.2s[1]
+	fmla	v16.2s, v0.2s, v8.s[0]
+	fmla	v20.2s, v0.2s, v8.s[1]
 .endm
 
 .macro SAVE2x2
@@ -882,7 +882,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ldr	s0 , [pA]
 	add	pA, pA, #4
 
-	fmla	v16.2s, v8.2s, v0.2s[0]
+	fmla	v16.2s, v8.2s, v0.s[0]
 .endm
 
 .macro SAVE1x2
@@ -918,10 +918,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v3.4s}, [pA]
 	add	pA, pA, #16
 
-	fmla	v16.4s, v0.4s, v8.2s[0]
-	fmla	v17.4s, v1.4s, v8.2s[0]
-	fmla	v18.4s, v2.4s, v8.2s[0]
-	fmla	v19.4s, v3.4s, v8.2s[0]
+	fmla	v16.4s, v0.4s, v8.s[0]
+	fmla	v17.4s, v1.4s, v8.s[0]
+	fmla	v18.4s, v2.4s, v8.s[0]
+	fmla	v19.4s, v3.4s, v8.s[0]
 .endm
 
 .macro SAVE16x1
@@ -951,8 +951,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v1.4s}, [pA]
 	add	pA, pA, #16
 
-	fmla	v16.4s, v0.4s, v8.2s[0]
-	fmla	v17.4s, v1.4s, v8.2s[0]
+	fmla	v16.4s, v0.4s, v8.s[0]
+	fmla	v17.4s, v1.4s, v8.s[0]
 .endm
 
 .macro SAVE8x1
@@ -978,8 +978,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2s, v1.2s}, [pA]
 	add	pA , pA, #16
 
-	fmla	v16.2s, v0.2s, v8.2s[0]
-	fmla	v17.2s, v1.2s, v8.2s[0]
+	fmla	v16.2s, v0.2s, v8.s[0]
+	fmla	v17.2s, v1.2s, v8.s[0]
 .endm
 
 .macro SAVE4x1
@@ -1004,7 +1004,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2s}, [pA]
 	add	pA , pA, #8
 
-	fmla	v16.2s, v0.2s, v8.2s[0]
+	fmla	v16.2s, v0.2s, v8.s[0]
 .endm
 
 .macro SAVE2x1

kernel/arm64/sgemm_kernel_4x4.S

@@ -192,164 +192,164 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.4s}, [pA_0]
 	add	pA_0, pA_0, #16
 
-	fmul	v16.4s, v0.4s, v8.4s[0]
-	fmul	v20.4s, v0.4s, v8.4s[1]
+	fmul	v16.4s, v0.4s, v8.s[0]
+	fmul	v20.4s, v0.4s, v8.s[1]
 
 	ld1	{v2.4s}, [pA_1]
 	add	pA_1, pA_1, #16
 
-	fmul	v24.4s, v0.4s, v8.4s[2]
-	fmul	v28.4s, v0.4s, v8.4s[3]
+	fmul	v24.4s, v0.4s, v8.s[2]
+	fmul	v28.4s, v0.4s, v8.s[3]
 
 	ld1	{v4.4s}, [pA_2]
 	add	pA_2, pA_2, #16
 
-	fmul	v17.4s, v2.4s, v8.4s[0]
-	fmul	v21.4s, v2.4s, v8.4s[1]
+	fmul	v17.4s, v2.4s, v8.s[0]
+	fmul	v21.4s, v2.4s, v8.s[1]
 
 	ld1	{v6.4s}, [pA_3]
 	add	pA_3, pA_3, #16
 
-	fmul	v25.4s, v2.4s, v8.4s[2]
-	fmul	v29.4s, v2.4s, v8.4s[3]
+	fmul	v25.4s, v2.4s, v8.s[2]
+	fmul	v29.4s, v2.4s, v8.s[3]
 
 	ld1	{v12.4s}, [pB]		// for next round
 	add	pB, pB, #16
 
-	fmul	v18.4s, v4.4s, v8.4s[0]
-	fmul	v19.4s, v6.4s, v8.4s[0]
+	fmul	v18.4s, v4.4s, v8.s[0]
+	fmul	v19.4s, v6.4s, v8.s[0]
 
 	ld1	{v1.4s}, [pA_0]		// for next round
 	add	pA_0, pA_0, #16
 
-	fmul	v22.4s, v4.4s, v8.4s[1]
-	fmul	v23.4s, v6.4s, v8.4s[1]
+	fmul	v22.4s, v4.4s, v8.s[1]
+	fmul	v23.4s, v6.4s, v8.s[1]
 
 	ld1	{v3.4s}, [pA_1]		// for next round
 	add	pA_1, pA_1, #16
 
-	fmul	v26.4s, v4.4s, v8.4s[2]
-	fmul	v27.4s, v6.4s, v8.4s[2]
+	fmul	v26.4s, v4.4s, v8.s[2]
+	fmul	v27.4s, v6.4s, v8.s[2]
 
 	ld1	{v5.4s}, [pA_2]		// for next round
 	add	pA_2, pA_2, #16
 
-	fmul	v30.4s, v4.4s, v8.4s[3]
-	fmul	v31.4s, v6.4s, v8.4s[3]
+	fmul	v30.4s, v4.4s, v8.s[3]
+	fmul	v31.4s, v6.4s, v8.s[3]
 
 	ld1	{v7.4s}, [pA_3]		// for next round
 	add	pA_3, pA_3, #16
 .endm
 
 .macro KERNEL16x4_M2
-	fmla	v16.4s, v1.4s, v12.4s[0]
-	fmla	v17.4s, v3.4s, v12.4s[0]
+	fmla	v16.4s, v1.4s, v12.s[0]
+	fmla	v17.4s, v3.4s, v12.s[0]
 
 	ld1	{v8.4s}, [pB]		// for next round
 	add	pB, pB, #16
 
-	fmla	v18.4s, v5.4s, v12.4s[0]
-	fmla	v19.4s, v7.4s, v12.4s[0]
+	fmla	v18.4s, v5.4s, v12.s[0]
+	fmla	v19.4s, v7.4s, v12.s[0]
 
 	ld1	{v0.4s}, [pA_0]		// for next round
 	add	pA_0, pA_0, #16
 
-	fmla	v20.4s, v1.4s, v12.4s[1]
-	fmla	v21.4s, v3.4s, v12.4s[1]
+	fmla	v20.4s, v1.4s, v12.s[1]
+	fmla	v21.4s, v3.4s, v12.s[1]
 
 	ld1	{v2.4s}, [pA_1]		// for next round
 	add	pA_1, pA_1, #16
 
-	fmla	v22.4s, v5.4s, v12.4s[1]
-	fmla	v23.4s, v7.4s, v12.4s[1]
+	fmla	v22.4s, v5.4s, v12.s[1]
+	fmla	v23.4s, v7.4s, v12.s[1]
 
 	ld1	{v4.4s}, [pA_2]		// for next round
 	add	pA_2, pA_2, #16
 
-	fmla	v24.4s, v1.4s, v12.4s[2]
-	fmla	v25.4s, v3.4s, v12.4s[2]
+	fmla	v24.4s, v1.4s, v12.s[2]
+	fmla	v25.4s, v3.4s, v12.s[2]
 
 	ld1	{v6.4s}, [pA_3]		// for next round
 	add	pA_3, pA_3, #16
 
-	fmla	v26.4s, v5.4s, v12.4s[2]
-	fmla	v27.4s, v7.4s, v12.4s[2]
+	fmla	v26.4s, v5.4s, v12.s[2]
+	fmla	v27.4s, v7.4s, v12.s[2]
 
 	prfm	PLDL1KEEP, [pA_2, #512]
 
-	fmla	v28.4s, v1.4s, v12.4s[3]
-	fmla	v29.4s, v3.4s, v12.4s[3]
+	fmla	v28.4s, v1.4s, v12.s[3]
+	fmla	v29.4s, v3.4s, v12.s[3]
 
 	prfm	PLDL1KEEP, [pA_3, #512]
 
-	fmla	v30.4s, v5.4s, v12.4s[3]
-	fmla	v31.4s, v7.4s, v12.4s[3]
+	fmla	v30.4s, v5.4s, v12.s[3]
+	fmla	v31.4s, v7.4s, v12.s[3]
 
 	prfm	PLDL1KEEP, [pB, #512]
 .endm
 
 .macro KERNEL16x4_M1
-	fmla	v16.4s, v0.4s, v8.4s[0]
-	fmla	v17.4s, v2.4s, v8.4s[0]
+	fmla	v16.4s, v0.4s, v8.s[0]
+	fmla	v17.4s, v2.4s, v8.s[0]
 
 	ld1	{v12.4s}, [pB]		// for next round
 	add	pB, pB, #16
 
-	fmla	v18.4s, v4.4s, v8.4s[0]
-	fmla	v19.4s, v6.4s, v8.4s[0]
+	fmla	v18.4s, v4.4s, v8.s[0]
+	fmla	v19.4s, v6.4s, v8.s[0]
 
 	ld1	{v1.4s}, [pA_0]		// for next round
 	add	pA_0, pA_0, #16
 
-	fmla	v20.4s, v0.4s, v8.4s[1]
-	fmla	v21.4s, v2.4s, v8.4s[1]
+	fmla	v20.4s, v0.4s, v8.s[1]
+	fmla	v21.4s, v2.4s, v8.s[1]
 
 	ld1	{v3.4s}, [pA_1]		// for next round
 	add	pA_1, pA_1, #16
 
-	fmla	v22.4s, v4.4s, v8.4s[1]
-	fmla	v23.4s, v6.4s, v8.4s[1]
+	fmla	v22.4s, v4.4s, v8.s[1]
+	fmla	v23.4s, v6.4s, v8.s[1]
 
 	ld1	{v5.4s}, [pA_2]		// for next round
 	add	pA_2, pA_2, #16
 
-	fmla	v24.4s, v0.4s, v8.4s[2]
-	fmla	v25.4s, v2.4s, v8.4s[2]
+	fmla	v24.4s, v0.4s, v8.s[2]
+	fmla	v25.4s, v2.4s, v8.s[2]
 
 	ld1	{v7.4s}, [pA_3]		// for next round
 	add	pA_3, pA_3, #16
 
-	fmla	v26.4s, v4.4s, v8.4s[2]
-	fmla	v27.4s, v6.4s, v8.4s[2]
+	fmla	v26.4s, v4.4s, v8.s[2]
+	fmla	v27.4s, v6.4s, v8.s[2]
 
 	prfm	PLDL1KEEP, [pA_0, #512]
 
-	fmla	v28.4s, v0.4s, v8.4s[3]
-	fmla	v29.4s, v2.4s, v8.4s[3]
+	fmla	v28.4s, v0.4s, v8.s[3]
+	fmla	v29.4s, v2.4s, v8.s[3]
 
 	prfm	PLDL1KEEP, [pA_1, #512]
 
-	fmla	v30.4s, v4.4s, v8.4s[3]
-	fmla	v31.4s, v6.4s, v8.4s[3]
+	fmla	v30.4s, v4.4s, v8.s[3]
+	fmla	v31.4s, v6.4s, v8.s[3]
 .endm
 
 .macro KERNEL16x4_E
-	fmla	v16.4s, v1.4s, v12.4s[0]
-	fmla	v17.4s, v3.4s, v12.4s[0]
-	fmla	v18.4s, v5.4s, v12.4s[0]
-	fmla	v19.4s, v7.4s, v12.4s[0]
-	fmla	v20.4s, v1.4s, v12.4s[1]
-	fmla	v21.4s, v3.4s, v12.4s[1]
-	fmla	v22.4s, v5.4s, v12.4s[1]
-	fmla	v23.4s, v7.4s, v12.4s[1]
-	fmla	v24.4s, v1.4s, v12.4s[2]
-	fmla	v25.4s, v3.4s, v12.4s[2]
-	fmla	v26.4s, v5.4s, v12.4s[2]
-	fmla	v27.4s, v7.4s, v12.4s[2]
-	fmla	v28.4s, v1.4s, v12.4s[3]
-	fmla	v29.4s, v3.4s, v12.4s[3]
-	fmla	v30.4s, v5.4s, v12.4s[3]
-	fmla	v31.4s, v7.4s, v12.4s[3]
+	fmla	v16.4s, v1.4s, v12.s[0]
+	fmla	v17.4s, v3.4s, v12.s[0]
+	fmla	v18.4s, v5.4s, v12.s[0]
+	fmla	v19.4s, v7.4s, v12.s[0]
+	fmla	v20.4s, v1.4s, v12.s[1]
+	fmla	v21.4s, v3.4s, v12.s[1]
+	fmla	v22.4s, v5.4s, v12.s[1]
+	fmla	v23.4s, v7.4s, v12.s[1]
+	fmla	v24.4s, v1.4s, v12.s[2]
+	fmla	v25.4s, v3.4s, v12.s[2]
+	fmla	v26.4s, v5.4s, v12.s[2]
+	fmla	v27.4s, v7.4s, v12.s[2]
+	fmla	v28.4s, v1.4s, v12.s[3]
+	fmla	v29.4s, v3.4s, v12.s[3]
+	fmla	v30.4s, v5.4s, v12.s[3]
+	fmla	v31.4s, v7.4s, v12.s[3]
 .endm
 
 .macro KERNEL16x4_SUB
@@ -359,34 +359,34 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.4s}, [pA_0]
 	add	pA_0, pA_0, #16
 
-	fmla	v16.4s, v0.4s, v8.4s[0]
-	fmla	v20.4s, v0.4s, v8.4s[1]
-	fmla	v24.4s, v0.4s, v8.4s[2]
-	fmla	v28.4s, v0.4s, v8.4s[3]
+	fmla	v16.4s, v0.4s, v8.s[0]
+	fmla	v20.4s, v0.4s, v8.s[1]
+	fmla	v24.4s, v0.4s, v8.s[2]
+	fmla	v28.4s, v0.4s, v8.s[3]
 
 	ld1	{v2.4s}, [pA_1]
 	add	pA_1, pA_1, #16
 
-	fmla	v17.4s, v2.4s, v8.4s[0]
-	fmla	v21.4s, v2.4s, v8.4s[1]
-	fmla	v25.4s, v2.4s, v8.4s[2]
-	fmla	v29.4s, v2.4s, v8.4s[3]
+	fmla	v17.4s, v2.4s, v8.s[0]
+	fmla	v21.4s, v2.4s, v8.s[1]
+	fmla	v25.4s, v2.4s, v8.s[2]
+	fmla	v29.4s, v2.4s, v8.s[3]
 
 	ld1	{v4.4s}, [pA_2]
 	add	pA_2, pA_2, #16
 
-	fmla	v18.4s, v4.4s, v8.4s[0]
-	fmla	v22.4s, v4.4s, v8.4s[1]
-	fmla	v26.4s, v4.4s, v8.4s[2]
-	fmla	v30.4s, v4.4s, v8.4s[3]
+	fmla	v18.4s, v4.4s, v8.s[0]
+	fmla	v22.4s, v4.4s, v8.s[1]
+	fmla	v26.4s, v4.4s, v8.s[2]
+	fmla	v30.4s, v4.4s, v8.s[3]
 
 	ld1	{v6.4s}, [pA_3]
 	add	pA_3, pA_3, #16
 
-	fmla	v19.4s, v6.4s, v8.4s[0]
-	fmla	v23.4s, v6.4s, v8.4s[1]
-	fmla	v27.4s, v6.4s, v8.4s[2]
-	fmla	v31.4s, v6.4s, v8.4s[3]
+	fmla	v19.4s, v6.4s, v8.s[0]
+	fmla	v23.4s, v6.4s, v8.s[1]
+	fmla	v27.4s, v6.4s, v8.s[2]
+	fmla	v31.4s, v6.4s, v8.s[3]
 .endm
 
 .macro SAVE16x4
@@ -456,28 +456,28 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2s, v1.2s}, [pA_0]
 	add	pA_0, pA_0, #16
 
-	fmla	v16.2s, v0.2s, v8.2s[0]
-	fmla	v29.2s, v1.2s, v9.2s[1]
-	fmla	v20.2s, v0.2s, v8.2s[1]
-	fmla	v25.2s, v1.2s, v9.2s[0]
+	fmla	v16.2s, v0.2s, v8.s[0]
+	fmla	v29.2s, v1.2s, v9.s[1]
+	fmla	v20.2s, v0.2s, v8.s[1]
+	fmla	v25.2s, v1.2s, v9.s[0]
 
 	ld1	{v2.2s, v3.2s}, [pA_1]
 	add	pA_1, pA_1, #16
 
-	fmla	v24.2s, v0.2s, v9.2s[0]
-	fmla	v21.2s, v1.2s, v8.2s[1]
-	fmla	v28.2s, v0.2s, v9.2s[1]
-	fmla	v17.2s, v1.2s, v8.2s[0]
+	fmla	v24.2s, v0.2s, v9.s[0]
+	fmla	v21.2s, v1.2s, v8.s[1]
+	fmla	v28.2s, v0.2s, v9.s[1]
+	fmla	v17.2s, v1.2s, v8.s[0]
 
-	fmla	v18.2s, v2.2s, v8.2s[0]
-	fmla	v31.2s, v3.2s, v9.2s[1]
-	fmla	v22.2s, v2.2s, v8.2s[1]
-	fmla	v27.2s, v3.2s, v9.2s[0]
+	fmla	v18.2s, v2.2s, v8.s[0]
+	fmla	v31.2s, v3.2s, v9.s[1]
+	fmla	v22.2s, v2.2s, v8.s[1]
+	fmla	v27.2s, v3.2s, v9.s[0]
 
-	fmla	v26.2s, v2.2s, v9.2s[0]
-	fmla	v23.2s, v3.2s, v8.2s[1]
-	fmla	v30.2s, v2.2s, v9.2s[1]
-	fmla	v19.2s, v3.2s, v8.2s[0]
+	fmla	v26.2s, v2.2s, v9.s[0]
+	fmla	v23.2s, v3.2s, v8.s[1]
+	fmla	v30.2s, v2.2s, v9.s[1]
+	fmla	v19.2s, v3.2s, v8.s[0]
 .endm
 
 .macro SAVE8x4
@@ -556,17 +556,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2s, v1.2s}, [pA_0]
 	add	pA_0, pA_0, #16
 
-	fmla	v16.2s, v0.2s, v8.2s[0]
-	fmla	v29.2s, v1.2s, v9.2s[1]
+	fmla	v16.2s, v0.2s, v8.s[0]
+	fmla	v29.2s, v1.2s, v9.s[1]
 
-	fmla	v20.2s, v0.2s, v8.2s[1]
-	fmla	v25.2s, v1.2s, v9.2s[0]
+	fmla	v20.2s, v0.2s, v8.s[1]
+	fmla	v25.2s, v1.2s, v9.s[0]
 
-	fmla	v24.2s, v0.2s, v9.2s[0]
-	fmla	v21.2s, v1.2s, v8.2s[1]
+	fmla	v24.2s, v0.2s, v9.s[0]
+	fmla	v21.2s, v1.2s, v8.s[1]
 
-	fmla	v28.2s, v0.2s, v9.2s[1]
-	fmla	v17.2s, v1.2s, v8.2s[0]
+	fmla	v28.2s, v0.2s, v9.s[1]
+	fmla	v17.2s, v1.2s, v8.s[0]
 .endm
 
 .macro SAVE4x4
@@ -614,10 +614,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2s}, [pA_0]
 	add	pA_0, pA_0, #8
 
-	fmla	v16.2s, v0.2s, v8.2s[0]
-	fmla	v20.2s, v0.2s, v8.2s[1]
-	fmla	v24.2s, v0.2s, v9.2s[0]
-	fmla	v28.2s, v0.2s, v9.2s[1]
+	fmla	v16.2s, v0.2s, v8.s[0]
+	fmla	v20.2s, v0.2s, v8.s[1]
+	fmla	v24.2s, v0.2s, v9.s[0]
+	fmla	v28.2s, v0.2s, v9.s[1]
 .endm
 
 .macro SAVE2x4
@@ -700,10 +700,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2s, v1.2s}, [pA_0]
 	add	pA_0, pA_0, #16
 
-	fmla	v16.2s, v0.2s, v8.2s[0]
-	fmla	v17.2s, v1.2s, v8.2s[0]
-	fmla	v20.2s, v0.2s, v8.2s[1]
-	fmla	v21.2s, v1.2s, v8.2s[1]
+	fmla	v16.2s, v0.2s, v8.s[0]
+	fmla	v17.2s, v1.2s, v8.s[0]
+	fmla	v20.2s, v0.2s, v8.s[1]
+	fmla	v21.2s, v1.2s, v8.s[1]
 .endm
 
 .macro SAVE4x2
@@ -736,8 +736,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2s}, [pA_0]
 	add	pA_0, pA_0, #8
 
-	fmla	v16.2s, v0.2s, v8.2s[0]
-	fmla	v20.2s, v0.2s, v8.2s[1]
+	fmla	v16.2s, v0.2s, v8.s[0]
+	fmla	v20.2s, v0.2s, v8.s[1]
 .endm
 
 .macro SAVE2x2
@@ -767,7 +767,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ldr	s0 , [pA_0]
 	add	pA_0, pA_0, #4
 
-	fmla	v16.2s, v8.2s, v0.2s[0]
+	fmla	v16.2s, v8.2s, v0.s[0]
 .endm
 
 .macro SAVE1x2
@@ -796,8 +796,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2s, v1.2s}, [pA_0]
 	add	pA_0 , pA_0, #16
 
-	fmla	v16.2s, v0.2s, v8.2s[0]
-	fmla	v17.2s, v1.2s, v8.2s[0]
+	fmla	v16.2s, v0.2s, v8.s[0]
+	fmla	v17.2s, v1.2s, v8.s[0]
 .endm
 
 .macro SAVE4x1
@@ -825,7 +825,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2s}, [pA_0]
 	add	pA_0 , pA_0, #8
 
-	fmla	v16.2s, v0.2s, v8.2s[0]
+	fmla	v16.2s, v0.2s, v8.s[0]
 .endm
 
 .macro SAVE2x1

kernel/arm64/sgemm_kernel_8x8.S

@@ -157,22 +157,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v1.4s}, [pA]
 	add	pA, pA, #16
 
-	fmul	v16.4s, v0.4s, v4.4s[0]
-	fmul	v17.4s, v1.4s, v4.4s[0]
-	fmul	v18.4s, v0.4s, v4.4s[1]
-	fmul	v19.4s, v1.4s, v4.4s[1]
-	fmul	v20.4s, v0.4s, v4.4s[2]
-	fmul	v21.4s, v1.4s, v4.4s[2]
-	fmul	v22.4s, v0.4s, v4.4s[3]
-	fmul	v23.4s, v1.4s, v4.4s[3]
-	fmul	v24.4s, v0.4s, v5.4s[0]
-	fmul	v25.4s, v1.4s, v5.4s[0]
-	fmul	v26.4s, v0.4s, v5.4s[1]
-	fmul	v27.4s, v1.4s, v5.4s[1]
-	fmul	v28.4s, v0.4s, v5.4s[2]
-	fmul	v29.4s, v1.4s, v5.4s[2]
-	fmul	v30.4s, v0.4s, v5.4s[3]
-	fmul	v31.4s, v1.4s, v5.4s[3]
+	fmul	v16.4s, v0.4s, v4.s[0]
+	fmul	v17.4s, v1.4s, v4.s[0]
+	fmul	v18.4s, v0.4s, v4.s[1]
+	fmul	v19.4s, v1.4s, v4.s[1]
+	fmul	v20.4s, v0.4s, v4.s[2]
+	fmul	v21.4s, v1.4s, v4.s[2]
+	fmul	v22.4s, v0.4s, v4.s[3]
+	fmul	v23.4s, v1.4s, v4.s[3]
+	fmul	v24.4s, v0.4s, v5.s[0]
+	fmul	v25.4s, v1.4s, v5.s[0]
+	fmul	v26.4s, v0.4s, v5.s[1]
+	fmul	v27.4s, v1.4s, v5.s[1]
+	fmul	v28.4s, v0.4s, v5.s[2]
+	fmul	v29.4s, v1.4s, v5.s[2]
+	fmul	v30.4s, v0.4s, v5.s[3]
+	fmul	v31.4s, v1.4s, v5.s[3]
 
 	ld1	{v6.4s}, [pB]
 	add	pB, pB, #16
@@ -185,22 +185,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL8x8_M1
-	fmla	v16.4s, v0.4s, v4.4s[0]
-	fmla	v17.4s, v1.4s, v4.4s[0]
-	fmla	v18.4s, v0.4s, v4.4s[1]
-	fmla	v19.4s, v1.4s, v4.4s[1]
-	fmla	v20.4s, v0.4s, v4.4s[2]
-	fmla	v21.4s, v1.4s, v4.4s[2]
-	fmla	v22.4s, v0.4s, v4.4s[3]
-	fmla	v23.4s, v1.4s, v4.4s[3]
-	fmla	v24.4s, v0.4s, v5.4s[0]
-	fmla	v25.4s, v1.4s, v5.4s[0]
-	fmla	v26.4s, v0.4s, v5.4s[1]
-	fmla	v27.4s, v1.4s, v5.4s[1]
-	fmla	v28.4s, v0.4s, v5.4s[2]
-	fmla	v29.4s, v1.4s, v5.4s[2]
-	fmla	v30.4s, v0.4s, v5.4s[3]
-	fmla	v31.4s, v1.4s, v5.4s[3]
+	fmla	v16.4s, v0.4s, v4.s[0]
+	fmla	v17.4s, v1.4s, v4.s[0]
+	fmla	v18.4s, v0.4s, v4.s[1]
+	fmla	v19.4s, v1.4s, v4.s[1]
+	fmla	v20.4s, v0.4s, v4.s[2]
+	fmla	v21.4s, v1.4s, v4.s[2]
+	fmla	v22.4s, v0.4s, v4.s[3]
+	fmla	v23.4s, v1.4s, v4.s[3]
+	fmla	v24.4s, v0.4s, v5.s[0]
+	fmla	v25.4s, v1.4s, v5.s[0]
+	fmla	v26.4s, v0.4s, v5.s[1]
+	fmla	v27.4s, v1.4s, v5.s[1]
+	fmla	v28.4s, v0.4s, v5.s[2]
+	fmla	v29.4s, v1.4s, v5.s[2]
+	fmla	v30.4s, v0.4s, v5.s[3]
+	fmla	v31.4s, v1.4s, v5.s[3]
 
 	ld1	{v6.4s}, [pB]
 	add	pB, pB, #16
@@ -213,22 +213,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL8x8_M2
-	fmla	v16.4s, v2.4s, v6.4s[0]
-	fmla	v17.4s, v3.4s, v6.4s[0]
-	fmla	v18.4s, v2.4s, v6.4s[1]
-	fmla	v19.4s, v3.4s, v6.4s[1]
-	fmla	v20.4s, v2.4s, v6.4s[2]
-	fmla	v21.4s, v3.4s, v6.4s[2]
-	fmla	v22.4s, v2.4s, v6.4s[3]
-	fmla	v23.4s, v3.4s, v6.4s[3]
-	fmla	v24.4s, v2.4s, v7.4s[0]
-	fmla	v25.4s, v3.4s, v7.4s[0]
-	fmla	v26.4s, v2.4s, v7.4s[1]
-	fmla	v27.4s, v3.4s, v7.4s[1]
-	fmla	v28.4s, v2.4s, v7.4s[2]
-	fmla	v29.4s, v3.4s, v7.4s[2]
-	fmla	v30.4s, v2.4s, v7.4s[3]
-	fmla	v31.4s, v3.4s, v7.4s[3]
+	fmla	v16.4s, v2.4s, v6.s[0]
+	fmla	v17.4s, v3.4s, v6.s[0]
+	fmla	v18.4s, v2.4s, v6.s[1]
+	fmla	v19.4s, v3.4s, v6.s[1]
+	fmla	v20.4s, v2.4s, v6.s[2]
+	fmla	v21.4s, v3.4s, v6.s[2]
+	fmla	v22.4s, v2.4s, v6.s[3]
+	fmla	v23.4s, v3.4s, v6.s[3]
+	fmla	v24.4s, v2.4s, v7.s[0]
+	fmla	v25.4s, v3.4s, v7.s[0]
+	fmla	v26.4s, v2.4s, v7.s[1]
+	fmla	v27.4s, v3.4s, v7.s[1]
+	fmla	v28.4s, v2.4s, v7.s[2]
+	fmla	v29.4s, v3.4s, v7.s[2]
+	fmla	v30.4s, v2.4s, v7.s[3]
+	fmla	v31.4s, v3.4s, v7.s[3]
 
 	ld1	{v4.4s}, [pB]
 	add	pB, pB, #16
@@ -241,22 +241,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL8x8_E
-	fmla	v16.4s, v2.4s, v6.4s[0]
-	fmla	v17.4s, v3.4s, v6.4s[0]
-	fmla	v18.4s, v2.4s, v6.4s[1]
-	fmla	v19.4s, v3.4s, v6.4s[1]
-	fmla	v20.4s, v2.4s, v6.4s[2]
-	fmla	v21.4s, v3.4s, v6.4s[2]
-	fmla	v22.4s, v2.4s, v6.4s[3]
-	fmla	v23.4s, v3.4s, v6.4s[3]
-	fmla	v24.4s, v2.4s, v7.4s[0]
-	fmla	v25.4s, v3.4s, v7.4s[0]
-	fmla	v26.4s, v2.4s, v7.4s[1]
-	fmla	v27.4s, v3.4s, v7.4s[1]
-	fmla	v28.4s, v2.4s, v7.4s[2]
-	fmla	v29.4s, v3.4s, v7.4s[2]
-	fmla	v30.4s, v2.4s, v7.4s[3]
-	fmla	v31.4s, v3.4s, v7.4s[3]
+	fmla	v16.4s, v2.4s, v6.s[0]
+	fmla	v17.4s, v3.4s, v6.s[0]
+	fmla	v18.4s, v2.4s, v6.s[1]
+	fmla	v19.4s, v3.4s, v6.s[1]
+	fmla	v20.4s, v2.4s, v6.s[2]
+	fmla	v21.4s, v3.4s, v6.s[2]
+	fmla	v22.4s, v2.4s, v6.s[3]
+	fmla	v23.4s, v3.4s, v6.s[3]
+	fmla	v24.4s, v2.4s, v7.s[0]
+	fmla	v25.4s, v3.4s, v7.s[0]
+	fmla	v26.4s, v2.4s, v7.s[1]
+	fmla	v27.4s, v3.4s, v7.s[1]
+	fmla	v28.4s, v2.4s, v7.s[2]
+	fmla	v29.4s, v3.4s, v7.s[2]
+	fmla	v30.4s, v2.4s, v7.s[3]
+	fmla	v31.4s, v3.4s, v7.s[3]
 .endm
 
 .macro KERNEL8x8_SUB
@@ -269,22 +269,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v1.4s}, [pA]
 	add	pA, pA, #16
 
-	fmla	v16.4s, v0.4s, v4.4s[0]
-	fmla	v17.4s, v1.4s, v4.4s[0]
-	fmla	v18.4s, v0.4s, v4.4s[1]
-	fmla	v19.4s, v1.4s, v4.4s[1]
-	fmla	v20.4s, v0.4s, v4.4s[2]
-	fmla	v21.4s, v1.4s, v4.4s[2]
-	fmla	v22.4s, v0.4s, v4.4s[3]
-	fmla	v23.4s, v1.4s, v4.4s[3]
-	fmla	v24.4s, v0.4s, v5.4s[0]
-	fmla	v25.4s, v1.4s, v5.4s[0]
-	fmla	v26.4s, v0.4s, v5.4s[1]
-	fmla	v27.4s, v1.4s, v5.4s[1]
-	fmla	v28.4s, v0.4s, v5.4s[2]
-	fmla	v29.4s, v1.4s, v5.4s[2]
-	fmla	v30.4s, v0.4s, v5.4s[3]
-	fmla	v31.4s, v1.4s, v5.4s[3]
+	fmla	v16.4s, v0.4s, v4.s[0]
+	fmla	v17.4s, v1.4s, v4.s[0]
+	fmla	v18.4s, v0.4s, v4.s[1]
+	fmla	v19.4s, v1.4s, v4.s[1]
+	fmla	v20.4s, v0.4s, v4.s[2]
+	fmla	v21.4s, v1.4s, v4.s[2]
+	fmla	v22.4s, v0.4s, v4.s[3]
+	fmla	v23.4s, v1.4s, v4.s[3]
+	fmla	v24.4s, v0.4s, v5.s[0]
+	fmla	v25.4s, v1.4s, v5.s[0]
+	fmla	v26.4s, v0.4s, v5.s[1]
+	fmla	v27.4s, v1.4s, v5.s[1]
+	fmla	v28.4s, v0.4s, v5.s[2]
+	fmla	v29.4s, v1.4s, v5.s[2]
+	fmla	v30.4s, v0.4s, v5.s[3]
+	fmla	v31.4s, v1.4s, v5.s[3]
 .endm
 
 .macro SAVE8x8
@@ -367,14 +367,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.4s}, [pA]
 	add	pA, pA, #16
 
-	fmul	v16.4s, v0.4s, v4.4s[0]
-	fmul	v18.4s, v0.4s, v4.4s[1]
-	fmul	v20.4s, v0.4s, v4.4s[2]
-	fmul	v22.4s, v0.4s, v4.4s[3]
-	fmul	v24.4s, v0.4s, v5.4s[0]
-	fmul	v26.4s, v0.4s, v5.4s[1]
-	fmul	v28.4s, v0.4s, v5.4s[2]
-	fmul	v30.4s, v0.4s, v5.4s[3]
+	fmul	v16.4s, v0.4s, v4.s[0]
+	fmul	v18.4s, v0.4s, v4.s[1]
+	fmul	v20.4s, v0.4s, v4.s[2]
+	fmul	v22.4s, v0.4s, v4.s[3]
+	fmul	v24.4s, v0.4s, v5.s[0]
+	fmul	v26.4s, v0.4s, v5.s[1]
+	fmul	v28.4s, v0.4s, v5.s[2]
+	fmul	v30.4s, v0.4s, v5.s[3]
 
 	ld1	{v6.4s}, [pB]
 	add	pB, pB, #16
@@ -385,14 +385,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL4x8_M1
-	fmla	v16.4s, v0.4s, v4.4s[0]
-	fmla	v18.4s, v0.4s, v4.4s[1]
-	fmla	v20.4s, v0.4s, v4.4s[2]
-	fmla	v22.4s, v0.4s, v4.4s[3]
-	fmla	v24.4s, v0.4s, v5.4s[0]
-	fmla	v26.4s, v0.4s, v5.4s[1]
-	fmla	v28.4s, v0.4s, v5.4s[2]
-	fmla	v30.4s, v0.4s, v5.4s[3]
+	fmla	v16.4s, v0.4s, v4.s[0]
+	fmla	v18.4s, v0.4s, v4.s[1]
+	fmla	v20.4s, v0.4s, v4.s[2]
+	fmla	v22.4s, v0.4s, v4.s[3]
+	fmla	v24.4s, v0.4s, v5.s[0]
+	fmla	v26.4s, v0.4s, v5.s[1]
+	fmla	v28.4s, v0.4s, v5.s[2]
+	fmla	v30.4s, v0.4s, v5.s[3]
 
 	ld1	{v6.4s}, [pB]
 	add	pB, pB, #16
@@ -403,14 +403,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL4x8_M2
-	fmla	v16.4s, v2.4s, v6.4s[0]
-	fmla	v18.4s, v2.4s, v6.4s[1]
-	fmla	v20.4s, v2.4s, v6.4s[2]
-	fmla	v22.4s, v2.4s, v6.4s[3]
-	fmla	v24.4s, v2.4s, v7.4s[0]
-	fmla	v26.4s, v2.4s, v7.4s[1]
-	fmla	v28.4s, v2.4s, v7.4s[2]
-	fmla	v30.4s, v2.4s, v7.4s[3]
+	fmla	v16.4s, v2.4s, v6.s[0]
+	fmla	v18.4s, v2.4s, v6.s[1]
+	fmla	v20.4s, v2.4s, v6.s[2]
+	fmla	v22.4s, v2.4s, v6.s[3]
+	fmla	v24.4s, v2.4s, v7.s[0]
+	fmla	v26.4s, v2.4s, v7.s[1]
+	fmla	v28.4s, v2.4s, v7.s[2]
+	fmla	v30.4s, v2.4s, v7.s[3]
 
 	ld1	{v4.4s}, [pB]
 	add	pB, pB, #16
@@ -421,14 +421,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL4x8_E
-	fmla	v16.4s, v2.4s, v6.4s[0]
-	fmla	v18.4s, v2.4s, v6.4s[1]
-	fmla	v20.4s, v2.4s, v6.4s[2]
-	fmla	v22.4s, v2.4s, v6.4s[3]
-	fmla	v24.4s, v2.4s, v7.4s[0]
-	fmla	v26.4s, v2.4s, v7.4s[1]
-	fmla	v28.4s, v2.4s, v7.4s[2]
-	fmla	v30.4s, v2.4s, v7.4s[3]
+	fmla	v16.4s, v2.4s, v6.s[0]
+	fmla	v18.4s, v2.4s, v6.s[1]
+	fmla	v20.4s, v2.4s, v6.s[2]
+	fmla	v22.4s, v2.4s, v6.s[3]
+	fmla	v24.4s, v2.4s, v7.s[0]
+	fmla	v26.4s, v2.4s, v7.s[1]
+	fmla	v28.4s, v2.4s, v7.s[2]
+	fmla	v30.4s, v2.4s, v7.s[3]
 .endm
 
 .macro KERNEL4x8_SUB
@@ -439,14 +439,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.4s}, [pA]
 	add	pA, pA, #16
 
-	fmla	v16.4s, v0.4s, v4.4s[0]
-	fmla	v18.4s, v0.4s, v4.4s[1]
-	fmla	v20.4s, v0.4s, v4.4s[2]
-	fmla	v22.4s, v0.4s, v4.4s[3]
-	fmla	v24.4s, v0.4s, v5.4s[0]
-	fmla	v26.4s, v0.4s, v5.4s[1]
-	fmla	v28.4s, v0.4s, v5.4s[2]
-	fmla	v30.4s, v0.4s, v5.4s[3]
+	fmla	v16.4s, v0.4s, v4.s[0]
+	fmla	v18.4s, v0.4s, v4.s[1]
+	fmla	v20.4s, v0.4s, v4.s[2]
+	fmla	v22.4s, v0.4s, v4.s[3]
+	fmla	v24.4s, v0.4s, v5.s[0]
+	fmla	v26.4s, v0.4s, v5.s[1]
+	fmla	v28.4s, v0.4s, v5.s[2]
+	fmla	v30.4s, v0.4s, v5.s[3]
 .endm
 
 .macro SAVE4x8
@@ -520,14 +520,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2s}, [pA]
 	add	pA, pA, #8
 
-	fmla	v16.2s, v0.2s, v4.4s[0]
-	fmla	v18.2s, v0.2s, v4.4s[1]
-	fmla	v20.2s, v0.2s, v4.4s[2]
-	fmla	v22.2s, v0.2s, v4.4s[3]
-	fmla	v24.2s, v0.2s, v5.4s[0]
-	fmla	v26.2s, v0.2s, v5.4s[1]
-	fmla	v28.2s, v0.2s, v5.4s[2]
-	fmla	v30.2s, v0.2s, v5.4s[3]
+	fmla	v16.2s, v0.2s, v4.s[0]
+	fmla	v18.2s, v0.2s, v4.s[1]
+	fmla	v20.2s, v0.2s, v4.s[2]
+	fmla	v22.2s, v0.2s, v4.s[3]
+	fmla	v24.2s, v0.2s, v5.s[0]
+	fmla	v26.2s, v0.2s, v5.s[1]
+	fmla	v28.2s, v0.2s, v5.s[2]
+	fmla	v30.2s, v0.2s, v5.s[3]
 .endm
 
 .macro SAVE2x8
@@ -601,14 +601,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ldr	s0, [pA]
 	add	pA, pA, #4
 
-	fmla	s16, s0, v4.4s[0]
-	fmla	s18, s0, v4.4s[1]
-	fmla	s20, s0, v4.4s[2]
-	fmla	s22, s0, v4.4s[3]
-	fmla	s24, s0, v5.4s[0]
-	fmla	s26, s0, v5.4s[1]
-	fmla	s28, s0, v5.4s[2]
-	fmla	s30, s0, v5.4s[3]
+	fmla	s16, s0, v4.s[0]
+	fmla	s18, s0, v4.s[1]
+	fmla	s20, s0, v4.s[2]
+	fmla	s22, s0, v4.s[3]
+	fmla	s24, s0, v5.s[0]
+	fmla	s26, s0, v5.s[1]
+	fmla	s28, s0, v5.s[2]
+	fmla	s30, s0, v5.s[3]
 .endm
 
 .macro SAVE1x8
@@ -682,14 +682,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v1.4s}, [pA]
 	add	pA, pA, #16
 
-	fmul	v16.4s, v0.4s, v8.2s[0]
-	fmul	v17.4s, v1.4s, v8.2s[0]
-	fmul	v20.4s, v0.4s, v8.2s[1]
-	fmul	v21.4s, v1.4s, v8.2s[1]
-	fmul	v24.4s, v0.4s, v9.2s[0]
-	fmul	v25.4s, v1.4s, v9.2s[0]
-	fmul	v28.4s, v0.4s, v9.2s[1]
-	fmul	v29.4s, v1.4s, v9.2s[1]
+	fmul	v16.4s, v0.4s, v8.s[0]
+	fmul	v17.4s, v1.4s, v8.s[0]
+	fmul	v20.4s, v0.4s, v8.s[1]
+	fmul	v21.4s, v1.4s, v8.s[1]
+	fmul	v24.4s, v0.4s, v9.s[0]
+	fmul	v25.4s, v1.4s, v9.s[0]
+	fmul	v28.4s, v0.4s, v9.s[1]
+	fmul	v29.4s, v1.4s, v9.s[1]
 
 	ld1	{v12.2s, v13.2s}, [pB]
 	add	pB, pB, #16
@@ -700,14 +700,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL8x4_M1
-	fmla	v16.4s, v0.4s, v8.2s[0]
-	fmla	v17.4s, v1.4s, v8.2s[0]
-	fmla	v20.4s, v0.4s, v8.2s[1]
-	fmla	v21.4s, v1.4s, v8.2s[1]
-	fmla	v24.4s, v0.4s, v9.2s[0]
-	fmla	v25.4s, v1.4s, v9.2s[0]
-	fmla	v28.4s, v0.4s, v9.2s[1]
-	fmla	v29.4s, v1.4s, v9.2s[1]
+	fmla	v16.4s, v0.4s, v8.s[0]
+	fmla	v17.4s, v1.4s, v8.s[0]
+	fmla	v20.4s, v0.4s, v8.s[1]
+	fmla	v21.4s, v1.4s, v8.s[1]
+	fmla	v24.4s, v0.4s, v9.s[0]
+	fmla	v25.4s, v1.4s, v9.s[0]
+	fmla	v28.4s, v0.4s, v9.s[1]
+	fmla	v29.4s, v1.4s, v9.s[1]
 
 	ld1	{v12.2s, v13.2s}, [pB]
 	add	pB, pB, #16
@@ -718,14 +718,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL8x4_M2
-	fmla	v16.4s, v4.4s, v12.2s[0]
-	fmla	v17.4s, v5.4s, v12.2s[0]
-	fmla	v20.4s, v4.4s, v12.2s[1]
-	fmla	v21.4s, v5.4s, v12.2s[1]
-	fmla	v24.4s, v4.4s, v13.2s[0]
-	fmla	v25.4s, v5.4s, v13.2s[0]
-	fmla	v28.4s, v4.4s, v13.2s[1]
-	fmla	v29.4s, v5.4s, v13.2s[1]
+	fmla	v16.4s, v4.4s, v12.s[0]
+	fmla	v17.4s, v5.4s, v12.s[0]
+	fmla	v20.4s, v4.4s, v12.s[1]
+	fmla	v21.4s, v5.4s, v12.s[1]
+	fmla	v24.4s, v4.4s, v13.s[0]
+	fmla	v25.4s, v5.4s, v13.s[0]
+	fmla	v28.4s, v4.4s, v13.s[1]
+	fmla	v29.4s, v5.4s, v13.s[1]
 
 	ld1	{v8.2s, v9.2s}, [pB]
 	add	pB, pB, #16
@@ -736,14 +736,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL8x4_E
-	fmla	v16.4s, v4.4s, v12.2s[0]
-	fmla	v17.4s, v5.4s, v12.2s[0]
-	fmla	v20.4s, v4.4s, v12.2s[1]
-	fmla	v21.4s, v5.4s, v12.2s[1]
-	fmla	v24.4s, v4.4s, v13.2s[0]
-	fmla	v25.4s, v5.4s, v13.2s[0]
-	fmla	v28.4s, v4.4s, v13.2s[1]
-	fmla	v29.4s, v5.4s, v13.2s[1]
+	fmla	v16.4s, v4.4s, v12.s[0]
+	fmla	v17.4s, v5.4s, v12.s[0]
+	fmla	v20.4s, v4.4s, v12.s[1]
+	fmla	v21.4s, v5.4s, v12.s[1]
+	fmla	v24.4s, v4.4s, v13.s[0]
+	fmla	v25.4s, v5.4s, v13.s[0]
+	fmla	v28.4s, v4.4s, v13.s[1]
+	fmla	v29.4s, v5.4s, v13.s[1]
 .endm
 
 .macro KERNEL8x4_SUB
@@ -754,14 +754,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v1.4s}, [pA]
 	add	pA, pA, #16
 
-	fmla	v16.4s, v0.4s, v8.2s[0]
-	fmla	v17.4s, v1.4s, v8.2s[0]
-	fmla	v20.4s, v0.4s, v8.2s[1]
-	fmla	v21.4s, v1.4s, v8.2s[1]
-	fmla	v24.4s, v0.4s, v9.2s[0]
-	fmla	v25.4s, v1.4s, v9.2s[0]
-	fmla	v28.4s, v0.4s, v9.2s[1]
-	fmla	v29.4s, v1.4s, v9.2s[1]
+	fmla	v16.4s, v0.4s, v8.s[0]
+	fmla	v17.4s, v1.4s, v8.s[0]
+	fmla	v20.4s, v0.4s, v8.s[1]
+	fmla	v21.4s, v1.4s, v8.s[1]
+	fmla	v24.4s, v0.4s, v9.s[0]
+	fmla	v25.4s, v1.4s, v9.s[0]
+	fmla	v28.4s, v0.4s, v9.s[1]
+	fmla	v29.4s, v1.4s, v9.s[1]
 .endm
 
 .macro SAVE8x4
@@ -814,17 +814,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2s, v1.2s}, [pA]
 	add	pA, pA, #16
 
-	fmul	v16.2s, v0.2s, v8.2s[0]
-	fmul	v29.2s, v1.2s, v9.2s[1]
+	fmul	v16.2s, v0.2s, v8.s[0]
+	fmul	v29.2s, v1.2s, v9.s[1]
 
-	fmul	v20.2s, v0.2s, v8.2s[1]
-	fmul	v25.2s, v1.2s, v9.2s[0]
+	fmul	v20.2s, v0.2s, v8.s[1]
+	fmul	v25.2s, v1.2s, v9.s[0]
 
-	fmul	v24.2s, v0.2s, v9.2s[0]
-	fmul	v21.2s, v1.2s, v8.2s[1]
+	fmul	v24.2s, v0.2s, v9.s[0]
+	fmul	v21.2s, v1.2s, v8.s[1]
 
-	fmul	v28.2s, v0.2s, v9.2s[1]
-	fmul	v17.2s, v1.2s, v8.2s[0]
+	fmul	v28.2s, v0.2s, v9.s[1]
+	fmul	v17.2s, v1.2s, v8.s[0]
 
 	ld1	{v12.2s, v13.2s}, [pB]
 	add	pB, pB, #16
@@ -833,61 +833,61 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL4x4_M1
-	fmla	v16.2s, v0.2s, v8.2s[0]
-	fmla	v29.2s, v1.2s, v9.2s[1]
+	fmla	v16.2s, v0.2s, v8.s[0]
+	fmla	v29.2s, v1.2s, v9.s[1]
 
 	ld1	{v12.2s, v13.2s}, [pB]		// For next round
 	add	pB, pB, #16
 
-	fmla	v20.2s, v0.2s, v8.2s[1]
-	fmla	v25.2s, v1.2s, v9.2s[0]
+	fmla	v20.2s, v0.2s, v8.s[1]
+	fmla	v25.2s, v1.2s, v9.s[0]
 
 	ld1	{v4.2s, v5.2s}, [pA]		// For next round
 	add	pA, pA, #16
 
-	fmla	v24.2s, v0.2s, v9.2s[0]
-	fmla	v21.2s, v1.2s, v8.2s[1]
+	fmla	v24.2s, v0.2s, v9.s[0]
+	fmla	v21.2s, v1.2s, v8.s[1]
 
 	prfm	PLDL1KEEP, [pB, #512]
 
-	fmla	v28.2s, v0.2s, v9.2s[1]
-	fmla	v17.2s, v1.2s, v8.2s[0]
+	fmla	v28.2s, v0.2s, v9.s[1]
+	fmla	v17.2s, v1.2s, v8.s[0]
 .endm
 
 .macro KERNEL4x4_M2
-	fmla	v16.2s, v4.2s, v12.2s[0]
-	fmla	v29.2s, v5.2s, v13.2s[1]
+	fmla	v16.2s, v4.2s, v12.s[0]
+	fmla	v29.2s, v5.2s, v13.s[1]
 
 	ld1	{v8.2s, v9.2s}, [pB]		// For next round
 	add	pB, pB, #16
 
-	fmla	v20.2s, v4.2s, v12.2s[1]
-	fmla	v25.2s, v5.2s, v13.2s[0]
+	fmla	v20.2s, v4.2s, v12.s[1]
+	fmla	v25.2s, v5.2s, v13.s[0]
 
 	ld1	{v0.2s, v1.2s}, [pA]		// For next round
 	add	pA, pA, #16
 
-	fmla	v24.2s, v4.2s, v13.2s[0]
-	fmla	v21.2s, v5.2s, v12.2s[1]
+	fmla	v24.2s, v4.2s, v13.s[0]
+	fmla	v21.2s, v5.2s, v12.s[1]
 
 	prfm	PLDL1KEEP, [pA, #512]
 
-	fmla	v28.2s, v4.2s, v13.2s[1]
-	fmla	v17.2s, v5.2s, v12.2s[0]
+	fmla	v28.2s, v4.2s, v13.s[1]
+	fmla	v17.2s, v5.2s, v12.s[0]
 .endm
 
 .macro KERNEL4x4_E
-	fmla	v16.2s, v4.2s, v12.2s[0]
-	fmla	v29.2s, v5.2s, v13.2s[1]
+	fmla	v16.2s, v4.2s, v12.s[0]
+	fmla	v29.2s, v5.2s, v13.s[1]
 
-	fmla	v20.2s, v4.2s, v12.2s[1]
-	fmla	v25.2s, v5.2s, v13.2s[0]
+	fmla	v20.2s, v4.2s, v12.s[1]
+	fmla	v25.2s, v5.2s, v13.s[0]
 
-	fmla	v24.2s, v4.2s, v13.2s[0]
-	fmla	v21.2s, v5.2s, v12.2s[1]
+	fmla	v24.2s, v4.2s, v13.s[0]
+	fmla	v21.2s, v5.2s, v12.s[1]
 
-	fmla	v28.2s, v4.2s, v13.2s[1]
-	fmla	v17.2s, v5.2s, v12.2s[0]
+	fmla	v28.2s, v4.2s, v13.s[1]
+	fmla	v17.2s, v5.2s, v12.s[0]
 .endm
 
 .macro KERNEL4x4_SUB
@@ -896,17 +896,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2s, v1.2s}, [pA]
 	add	pA, pA, #16
 
-	fmla	v16.2s, v0.2s, v8.2s[0]
-	fmla	v29.2s, v1.2s, v9.2s[1]
+	fmla	v16.2s, v0.2s, v8.s[0]
+	fmla	v29.2s, v1.2s, v9.s[1]
 
-	fmla	v20.2s, v0.2s, v8.2s[1]
-	fmla	v25.2s, v1.2s, v9.2s[0]
+	fmla	v20.2s, v0.2s, v8.s[1]
+	fmla	v25.2s, v1.2s, v9.s[0]
 
-	fmla	v24.2s, v0.2s, v9.2s[0]
-	fmla	v21.2s, v1.2s, v8.2s[1]
+	fmla	v24.2s, v0.2s, v9.s[0]
+	fmla	v21.2s, v1.2s, v8.s[1]
 
-	fmla	v28.2s, v0.2s, v9.2s[1]
-	fmla	v17.2s, v1.2s, v8.2s[0]
+	fmla	v28.2s, v0.2s, v9.s[1]
+	fmla	v17.2s, v1.2s, v8.s[0]
 .endm
 
 .macro SAVE4x4
@@ -951,10 +951,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2s}, [pA]
 	add	pA, pA, #8
 
-	fmla	v16.2s, v0.2s, v8.2s[0]
-	fmla	v20.2s, v0.2s, v8.2s[1]
-	fmla	v24.2s, v0.2s, v9.2s[0]
-	fmla	v28.2s, v0.2s, v9.2s[1]
+	fmla	v16.2s, v0.2s, v8.s[0]
+	fmla	v20.2s, v0.2s, v8.s[1]
+	fmla	v24.2s, v0.2s, v9.s[0]
+	fmla	v28.2s, v0.2s, v9.s[1]
 .endm
 
 .macro SAVE2x4
@@ -1034,11 +1034,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v1.4s}, [pA]
 	add	pA, pA, #16
 
-	fmla	v16.4s, v0.4s, v8.2s[0]
-	fmla	v17.4s, v1.4s, v8.2s[0]
+	fmla	v16.4s, v0.4s, v8.s[0]
+	fmla	v17.4s, v1.4s, v8.s[0]
 
-	fmla	v20.4s, v0.4s, v8.2s[1]
-	fmla	v21.4s, v1.4s, v8.2s[1]
+	fmla	v20.4s, v0.4s, v8.s[1]
+	fmla	v21.4s, v1.4s, v8.s[1]
 .endm
 
 .macro SAVE8x2
@@ -1074,10 +1074,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2s, v1.2s}, [pA]
 	add	pA, pA, #16
 
-	fmla	v16.2s, v0.2s, v8.2s[0]
-	fmla	v17.2s, v1.2s, v8.2s[0]
-	fmla	v20.2s, v0.2s, v8.2s[1]
-	fmla	v21.2s, v1.2s, v8.2s[1]
+	fmla	v16.2s, v0.2s, v8.s[0]
+	fmla	v17.2s, v1.2s, v8.s[0]
+	fmla	v20.2s, v0.2s, v8.s[1]
+	fmla	v21.2s, v1.2s, v8.s[1]
 .endm
 
 .macro SAVE4x2
@@ -1109,8 +1109,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2s}, [pA]
 	add	pA, pA, #8
 
-	fmla	v16.2s, v0.2s, v8.2s[0]
-	fmla	v20.2s, v0.2s, v8.2s[1]
+	fmla	v16.2s, v0.2s, v8.s[0]
+	fmla	v20.2s, v0.2s, v8.s[1]
 .endm
 
 .macro SAVE2x2
@@ -1139,7 +1139,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ldr	s0 , [pA]
 	add	pA, pA, #4
 
-	fmla	v16.2s, v8.2s, v0.2s[0]
+	fmla	v16.2s, v8.2s, v0.s[0]
 .endm
 
 .macro SAVE1x2
@@ -1169,8 +1169,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v1.4s}, [pA]
 	add	pA, pA, #16
 
-	fmla	v16.4s, v0.4s, v8.2s[0]
-	fmla	v17.4s, v1.4s, v8.2s[0]
+	fmla	v16.4s, v0.4s, v8.s[0]
+	fmla	v17.4s, v1.4s, v8.s[0]
 .endm
 
 .macro SAVE8x1
@@ -1196,8 +1196,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2s, v1.2s}, [pA]
 	add	pA , pA, #16
 
-	fmla	v16.2s, v0.2s, v8.2s[0]
-	fmla	v17.2s, v1.2s, v8.2s[0]
+	fmla	v16.2s, v0.2s, v8.s[0]
+	fmla	v17.2s, v1.2s, v8.s[0]
 .endm
 
 .macro SAVE4x1
@@ -1222,7 +1222,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2s}, [pA]
 	add	pA , pA, #8
 
-	fmla	v16.2s, v0.2s, v8.2s[0]
+	fmla	v16.2s, v0.2s, v8.s[0]
 .endm
 
 .macro SAVE2x1

kernel/arm64/strmm_kernel_16x4.S

@@ -161,25 +161,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v3.4s}, [pA]
 	add	pA, pA, #16
 
-	fmul	v16.4s, v0.4s, v8.2s[0]
-	fmul	v17.4s, v1.4s, v8.2s[0]
-	fmul	v18.4s, v2.4s, v8.2s[0]
-	fmul	v19.4s, v3.4s, v8.2s[0]
-
-	fmul	v20.4s, v0.4s, v8.2s[1]
-	fmul	v21.4s, v1.4s, v8.2s[1]
-	fmul	v22.4s, v2.4s, v8.2s[1]
-	fmul	v23.4s, v3.4s, v8.2s[1]
-
-	fmul	v24.4s, v0.4s, v9.2s[0]
-	fmul	v25.4s, v1.4s, v9.2s[0]
-	fmul	v26.4s, v2.4s, v9.2s[0]
-	fmul	v27.4s, v3.4s, v9.2s[0]
-
-	fmul	v28.4s, v0.4s, v9.2s[1]
-	fmul	v29.4s, v1.4s, v9.2s[1]
-	fmul	v30.4s, v2.4s, v9.2s[1]
-	fmul	v31.4s, v3.4s, v9.2s[1]
+	fmul	v16.4s, v0.4s, v8.s[0]
+	fmul	v17.4s, v1.4s, v8.s[0]
+	fmul	v18.4s, v2.4s, v8.s[0]
+	fmul	v19.4s, v3.4s, v8.s[0]
+
+	fmul	v20.4s, v0.4s, v8.s[1]
+	fmul	v21.4s, v1.4s, v8.s[1]
+	fmul	v22.4s, v2.4s, v8.s[1]
+	fmul	v23.4s, v3.4s, v8.s[1]
+
+	fmul	v24.4s, v0.4s, v9.s[0]
+	fmul	v25.4s, v1.4s, v9.s[0]
+	fmul	v26.4s, v2.4s, v9.s[0]
+	fmul	v27.4s, v3.4s, v9.s[0]
+
+	fmul	v28.4s, v0.4s, v9.s[1]
+	fmul	v29.4s, v1.4s, v9.s[1]
+	fmul	v30.4s, v2.4s, v9.s[1]
+	fmul	v31.4s, v3.4s, v9.s[1]
 
 	ld1	{v12.2s, v13.2s}, [pB]
 	add	pB, pB, #16
@@ -194,25 +194,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL16x4_M1
-	fmla	v16.4s, v0.4s, v8.2s[0]
-	fmla	v17.4s, v1.4s, v8.2s[0]
-	fmla	v18.4s, v2.4s, v8.2s[0]
-	fmla	v19.4s, v3.4s, v8.2s[0]
-
-	fmla	v20.4s, v0.4s, v8.2s[1]
-	fmla	v21.4s, v1.4s, v8.2s[1]
-	fmla	v22.4s, v2.4s, v8.2s[1]
-	fmla	v23.4s, v3.4s, v8.2s[1]
-
-	fmla	v24.4s, v0.4s, v9.2s[0]
-	fmla	v25.4s, v1.4s, v9.2s[0]
-	fmla	v26.4s, v2.4s, v9.2s[0]
-	fmla	v27.4s, v3.4s, v9.2s[0]
-
-	fmla	v28.4s, v0.4s, v9.2s[1]
-	fmla	v29.4s, v1.4s, v9.2s[1]
-	fmla	v30.4s, v2.4s, v9.2s[1]
-	fmla	v31.4s, v3.4s, v9.2s[1]
+	fmla	v16.4s, v0.4s, v8.s[0]
+	fmla	v17.4s, v1.4s, v8.s[0]
+	fmla	v18.4s, v2.4s, v8.s[0]
+	fmla	v19.4s, v3.4s, v8.s[0]
+
+	fmla	v20.4s, v0.4s, v8.s[1]
+	fmla	v21.4s, v1.4s, v8.s[1]
+	fmla	v22.4s, v2.4s, v8.s[1]
+	fmla	v23.4s, v3.4s, v8.s[1]
+
+	fmla	v24.4s, v0.4s, v9.s[0]
+	fmla	v25.4s, v1.4s, v9.s[0]
+	fmla	v26.4s, v2.4s, v9.s[0]
+	fmla	v27.4s, v3.4s, v9.s[0]
+
+	fmla	v28.4s, v0.4s, v9.s[1]
+	fmla	v29.4s, v1.4s, v9.s[1]
+	fmla	v30.4s, v2.4s, v9.s[1]
+	fmla	v31.4s, v3.4s, v9.s[1]
 
 	ld1	{v12.2s, v13.2s}, [pB]
 	add	pB, pB, #16
@@ -227,25 +227,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL16x4_M2
-	fmla	v16.4s, v4.4s, v12.2s[0]
-	fmla	v17.4s, v5.4s, v12.2s[0]
-	fmla	v18.4s, v6.4s, v12.2s[0]
-	fmla	v19.4s, v7.4s, v12.2s[0]
-
-	fmla	v20.4s, v4.4s, v12.2s[1]
-	fmla	v21.4s, v5.4s, v12.2s[1]
-	fmla	v22.4s, v6.4s, v12.2s[1]
-	fmla	v23.4s, v7.4s, v12.2s[1]
-
-	fmla	v24.4s, v4.4s, v13.2s[0]
-	fmla	v25.4s, v5.4s, v13.2s[0]
-	fmla	v26.4s, v6.4s, v13.2s[0]
-	fmla	v27.4s, v7.4s, v13.2s[0]
-
-	fmla	v28.4s, v4.4s, v13.2s[1]
-	fmla	v29.4s, v5.4s, v13.2s[1]
-	fmla	v30.4s, v6.4s, v13.2s[1]
-	fmla	v31.4s, v7.4s, v13.2s[1]
+	fmla	v16.4s, v4.4s, v12.s[0]
+	fmla	v17.4s, v5.4s, v12.s[0]
+	fmla	v18.4s, v6.4s, v12.s[0]
+	fmla	v19.4s, v7.4s, v12.s[0]
+
+	fmla	v20.4s, v4.4s, v12.s[1]
+	fmla	v21.4s, v5.4s, v12.s[1]
+	fmla	v22.4s, v6.4s, v12.s[1]
+	fmla	v23.4s, v7.4s, v12.s[1]
+
+	fmla	v24.4s, v4.4s, v13.s[0]
+	fmla	v25.4s, v5.4s, v13.s[0]
+	fmla	v26.4s, v6.4s, v13.s[0]
+	fmla	v27.4s, v7.4s, v13.s[0]
+
+	fmla	v28.4s, v4.4s, v13.s[1]
+	fmla	v29.4s, v5.4s, v13.s[1]
+	fmla	v30.4s, v6.4s, v13.s[1]
+	fmla	v31.4s, v7.4s, v13.s[1]
 
 	ld1	{v8.2s, v9.2s}, [pB]
 	add	pB, pB, #16
@@ -260,25 +260,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL16x4_E
-	fmla	v16.4s, v4.4s, v12.2s[0]
-	fmla	v17.4s, v5.4s, v12.2s[0]
-	fmla	v18.4s, v6.4s, v12.2s[0]
-	fmla	v19.4s, v7.4s, v12.2s[0]
-
-	fmla	v20.4s, v4.4s, v12.2s[1]
-	fmla	v21.4s, v5.4s, v12.2s[1]
-	fmla	v22.4s, v6.4s, v12.2s[1]
-	fmla	v23.4s, v7.4s, v12.2s[1]
-
-	fmla	v24.4s, v4.4s, v13.2s[0]
-	fmla	v25.4s, v5.4s, v13.2s[0]
-	fmla	v26.4s, v6.4s, v13.2s[0]
-	fmla	v27.4s, v7.4s, v13.2s[0]
-
-	fmla	v28.4s, v4.4s, v13.2s[1]
-	fmla	v29.4s, v5.4s, v13.2s[1]
-	fmla	v30.4s, v6.4s, v13.2s[1]
-	fmla	v31.4s, v7.4s, v13.2s[1]
+	fmla	v16.4s, v4.4s, v12.s[0]
+	fmla	v17.4s, v5.4s, v12.s[0]
+	fmla	v18.4s, v6.4s, v12.s[0]
+	fmla	v19.4s, v7.4s, v12.s[0]
+
+	fmla	v20.4s, v4.4s, v12.s[1]
+	fmla	v21.4s, v5.4s, v12.s[1]
+	fmla	v22.4s, v6.4s, v12.s[1]
+	fmla	v23.4s, v7.4s, v12.s[1]
+
+	fmla	v24.4s, v4.4s, v13.s[0]
+	fmla	v25.4s, v5.4s, v13.s[0]
+	fmla	v26.4s, v6.4s, v13.s[0]
+	fmla	v27.4s, v7.4s, v13.s[0]
+
+	fmla	v28.4s, v4.4s, v13.s[1]
+	fmla	v29.4s, v5.4s, v13.s[1]
+	fmla	v30.4s, v6.4s, v13.s[1]
+	fmla	v31.4s, v7.4s, v13.s[1]
 .endm
 
 .macro KERNEL16x4_SUB
@@ -293,25 +293,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v3.4s}, [pA]
 	add	pA, pA, #16
 
-	fmla	v16.4s, v0.4s, v8.2s[0]
-	fmla	v17.4s, v1.4s, v8.2s[0]
-	fmla	v18.4s, v2.4s, v8.2s[0]
-	fmla	v19.4s, v3.4s, v8.2s[0]
-
-	fmla	v20.4s, v0.4s, v8.2s[1]
-	fmla	v21.4s, v1.4s, v8.2s[1]
-	fmla	v22.4s, v2.4s, v8.2s[1]
-	fmla	v23.4s, v3.4s, v8.2s[1]
-
-	fmla	v24.4s, v0.4s, v9.2s[0]
-	fmla	v25.4s, v1.4s, v9.2s[0]
-	fmla	v26.4s, v2.4s, v9.2s[0]
-	fmla	v27.4s, v3.4s, v9.2s[0]
-
-	fmla	v28.4s, v0.4s, v9.2s[1]
-	fmla	v29.4s, v1.4s, v9.2s[1]
-	fmla	v30.4s, v2.4s, v9.2s[1]
-	fmla	v31.4s, v3.4s, v9.2s[1]
+	fmla	v16.4s, v0.4s, v8.s[0]
+	fmla	v17.4s, v1.4s, v8.s[0]
+	fmla	v18.4s, v2.4s, v8.s[0]
+	fmla	v19.4s, v3.4s, v8.s[0]
+
+	fmla	v20.4s, v0.4s, v8.s[1]
+	fmla	v21.4s, v1.4s, v8.s[1]
+	fmla	v22.4s, v2.4s, v8.s[1]
+	fmla	v23.4s, v3.4s, v8.s[1]
+
+	fmla	v24.4s, v0.4s, v9.s[0]
+	fmla	v25.4s, v1.4s, v9.s[0]
+	fmla	v26.4s, v2.4s, v9.s[0]
+	fmla	v27.4s, v3.4s, v9.s[0]
+
+	fmla	v28.4s, v0.4s, v9.s[1]
+	fmla	v29.4s, v1.4s, v9.s[1]
+	fmla	v30.4s, v2.4s, v9.s[1]
+	fmla	v31.4s, v3.4s, v9.s[1]
 .endm
 
 .macro SAVE16x4
@@ -369,14 +369,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v1.4s}, [pA]
 	add	pA, pA, #16
 
-	fmul	v16.4s, v0.4s, v8.2s[0]
-	fmul	v17.4s, v1.4s, v8.2s[0]
-	fmul	v20.4s, v0.4s, v8.2s[1]
-	fmul	v21.4s, v1.4s, v8.2s[1]
-	fmul	v24.4s, v0.4s, v9.2s[0]
-	fmul	v25.4s, v1.4s, v9.2s[0]
-	fmul	v28.4s, v0.4s, v9.2s[1]
-	fmul	v29.4s, v1.4s, v9.2s[1]
+	fmul	v16.4s, v0.4s, v8.s[0]
+	fmul	v17.4s, v1.4s, v8.s[0]
+	fmul	v20.4s, v0.4s, v8.s[1]
+	fmul	v21.4s, v1.4s, v8.s[1]
+	fmul	v24.4s, v0.4s, v9.s[0]
+	fmul	v25.4s, v1.4s, v9.s[0]
+	fmul	v28.4s, v0.4s, v9.s[1]
+	fmul	v29.4s, v1.4s, v9.s[1]
 
 	ld1	{v12.2s, v13.2s}, [pB]
 	add	pB, pB, #16
@@ -387,14 +387,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL8x4_M1
-	fmla	v16.4s, v0.4s, v8.2s[0]
-	fmla	v17.4s, v1.4s, v8.2s[0]
-	fmla	v20.4s, v0.4s, v8.2s[1]
-	fmla	v21.4s, v1.4s, v8.2s[1]
-	fmla	v24.4s, v0.4s, v9.2s[0]
-	fmla	v25.4s, v1.4s, v9.2s[0]
-	fmla	v28.4s, v0.4s, v9.2s[1]
-	fmla	v29.4s, v1.4s, v9.2s[1]
+	fmla	v16.4s, v0.4s, v8.s[0]
+	fmla	v17.4s, v1.4s, v8.s[0]
+	fmla	v20.4s, v0.4s, v8.s[1]
+	fmla	v21.4s, v1.4s, v8.s[1]
+	fmla	v24.4s, v0.4s, v9.s[0]
+	fmla	v25.4s, v1.4s, v9.s[0]
+	fmla	v28.4s, v0.4s, v9.s[1]
+	fmla	v29.4s, v1.4s, v9.s[1]
 
 	ld1	{v12.2s, v13.2s}, [pB]
 	add	pB, pB, #16
@@ -405,14 +405,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL8x4_M2
-	fmla	v16.4s, v4.4s, v12.2s[0]
-	fmla	v17.4s, v5.4s, v12.2s[0]
-	fmla	v20.4s, v4.4s, v12.2s[1]
-	fmla	v21.4s, v5.4s, v12.2s[1]
-	fmla	v24.4s, v4.4s, v13.2s[0]
-	fmla	v25.4s, v5.4s, v13.2s[0]
-	fmla	v28.4s, v4.4s, v13.2s[1]
-	fmla	v29.4s, v5.4s, v13.2s[1]
+	fmla	v16.4s, v4.4s, v12.s[0]
+	fmla	v17.4s, v5.4s, v12.s[0]
+	fmla	v20.4s, v4.4s, v12.s[1]
+	fmla	v21.4s, v5.4s, v12.s[1]
+	fmla	v24.4s, v4.4s, v13.s[0]
+	fmla	v25.4s, v5.4s, v13.s[0]
+	fmla	v28.4s, v4.4s, v13.s[1]
+	fmla	v29.4s, v5.4s, v13.s[1]
 
 	ld1	{v8.2s, v9.2s}, [pB]
 	add	pB, pB, #16
@@ -423,14 +423,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL8x4_E
-	fmla	v16.4s, v4.4s, v12.2s[0]
-	fmla	v17.4s, v5.4s, v12.2s[0]
-	fmla	v20.4s, v4.4s, v12.2s[1]
-	fmla	v21.4s, v5.4s, v12.2s[1]
-	fmla	v24.4s, v4.4s, v13.2s[0]
-	fmla	v25.4s, v5.4s, v13.2s[0]
-	fmla	v28.4s, v4.4s, v13.2s[1]
-	fmla	v29.4s, v5.4s, v13.2s[1]
+	fmla	v16.4s, v4.4s, v12.s[0]
+	fmla	v17.4s, v5.4s, v12.s[0]
+	fmla	v20.4s, v4.4s, v12.s[1]
+	fmla	v21.4s, v5.4s, v12.s[1]
+	fmla	v24.4s, v4.4s, v13.s[0]
+	fmla	v25.4s, v5.4s, v13.s[0]
+	fmla	v28.4s, v4.4s, v13.s[1]
+	fmla	v29.4s, v5.4s, v13.s[1]
 .endm
 
 .macro KERNEL8x4_SUB
@@ -441,14 +441,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v1.4s}, [pA]
 	add	pA, pA, #16
 
-	fmla	v16.4s, v0.4s, v8.2s[0]
-	fmla	v17.4s, v1.4s, v8.2s[0]
-	fmla	v20.4s, v0.4s, v8.2s[1]
-	fmla	v21.4s, v1.4s, v8.2s[1]
-	fmla	v24.4s, v0.4s, v9.2s[0]
-	fmla	v25.4s, v1.4s, v9.2s[0]
-	fmla	v28.4s, v0.4s, v9.2s[1]
-	fmla	v29.4s, v1.4s, v9.2s[1]
+	fmla	v16.4s, v0.4s, v8.s[0]
+	fmla	v17.4s, v1.4s, v8.s[0]
+	fmla	v20.4s, v0.4s, v8.s[1]
+	fmla	v21.4s, v1.4s, v8.s[1]
+	fmla	v24.4s, v0.4s, v9.s[0]
+	fmla	v25.4s, v1.4s, v9.s[0]
+	fmla	v28.4s, v0.4s, v9.s[1]
+	fmla	v29.4s, v1.4s, v9.s[1]
 .endm
 
 .macro SAVE8x4
@@ -496,17 +496,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2s, v1.2s}, [pA]
 	add	pA, pA, #16
 
-	fmul	v16.2s, v0.2s, v8.2s[0]
-	fmul	v29.2s, v1.2s, v9.2s[1]
+	fmul	v16.2s, v0.2s, v8.s[0]
+	fmul	v29.2s, v1.2s, v9.s[1]
 
-	fmul	v20.2s, v0.2s, v8.2s[1]
-	fmul	v25.2s, v1.2s, v9.2s[0]
+	fmul	v20.2s, v0.2s, v8.s[1]
+	fmul	v25.2s, v1.2s, v9.s[0]
 
-	fmul	v24.2s, v0.2s, v9.2s[0]
-	fmul	v21.2s, v1.2s, v8.2s[1]
+	fmul	v24.2s, v0.2s, v9.s[0]
+	fmul	v21.2s, v1.2s, v8.s[1]
 
-	fmul	v28.2s, v0.2s, v9.2s[1]
-	fmul	v17.2s, v1.2s, v8.2s[0]
+	fmul	v28.2s, v0.2s, v9.s[1]
+	fmul	v17.2s, v1.2s, v8.s[0]
 
 	ld1	{v12.2s, v13.2s}, [pB]
 	add	pB, pB, #16
@@ -515,61 +515,61 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL4x4_M1
-	fmla	v16.2s, v0.2s, v8.2s[0]
-	fmla	v29.2s, v1.2s, v9.2s[1]
+	fmla	v16.2s, v0.2s, v8.s[0]
+	fmla	v29.2s, v1.2s, v9.s[1]
 
 	ld1	{v12.2s, v13.2s}, [pB]		// For next round
 	add	pB, pB, #16
 
-	fmla	v20.2s, v0.2s, v8.2s[1]
-	fmla	v25.2s, v1.2s, v9.2s[0]
+	fmla	v20.2s, v0.2s, v8.s[1]
+	fmla	v25.2s, v1.2s, v9.s[0]
 
 	ld1	{v4.2s, v5.2s}, [pA]		// For next round
 	add	pA, pA, #16
 
-	fmla	v24.2s, v0.2s, v9.2s[0]
-	fmla	v21.2s, v1.2s, v8.2s[1]
+	fmla	v24.2s, v0.2s, v9.s[0]
+	fmla	v21.2s, v1.2s, v8.s[1]
 
 	prfm	PLDL1KEEP, [pB, #512]
 
-	fmla	v28.2s, v0.2s, v9.2s[1]
-	fmla	v17.2s, v1.2s, v8.2s[0]
+	fmla	v28.2s, v0.2s, v9.s[1]
+	fmla	v17.2s, v1.2s, v8.s[0]
 .endm
 
 .macro KERNEL4x4_M2
-	fmla	v16.2s, v4.2s, v12.2s[0]
-	fmla	v29.2s, v5.2s, v13.2s[1]
+	fmla	v16.2s, v4.2s, v12.s[0]
+	fmla	v29.2s, v5.2s, v13.s[1]
 
 	ld1	{v8.2s, v9.2s}, [pB]		// For next round
 	add	pB, pB, #16
 
-	fmla	v20.2s, v4.2s, v12.2s[1]
-	fmla	v25.2s, v5.2s, v13.2s[0]
+	fmla	v20.2s, v4.2s, v12.s[1]
+	fmla	v25.2s, v5.2s, v13.s[0]
 
 	ld1	{v0.2s, v1.2s}, [pA]		// For next round
 	add	pA, pA, #16
 
-	fmla	v24.2s, v4.2s, v13.2s[0]
-	fmla	v21.2s, v5.2s, v12.2s[1]
+	fmla	v24.2s, v4.2s, v13.s[0]
+	fmla	v21.2s, v5.2s, v12.s[1]
 
 	prfm	PLDL1KEEP, [pA, #512]
 
-	fmla	v28.2s, v4.2s, v13.2s[1]
-	fmla	v17.2s, v5.2s, v12.2s[0]
+	fmla	v28.2s, v4.2s, v13.s[1]
+	fmla	v17.2s, v5.2s, v12.s[0]
 .endm
 
 .macro KERNEL4x4_E
-	fmla	v16.2s, v4.2s, v12.2s[0]
-	fmla	v29.2s, v5.2s, v13.2s[1]
+	fmla	v16.2s, v4.2s, v12.s[0]
+	fmla	v29.2s, v5.2s, v13.s[1]
 
-	fmla	v20.2s, v4.2s, v12.2s[1]
-	fmla	v25.2s, v5.2s, v13.2s[0]
+	fmla	v20.2s, v4.2s, v12.s[1]
+	fmla	v25.2s, v5.2s, v13.s[0]
 
-	fmla	v24.2s, v4.2s, v13.2s[0]
-	fmla	v21.2s, v5.2s, v12.2s[1]
+	fmla	v24.2s, v4.2s, v13.s[0]
+	fmla	v21.2s, v5.2s, v12.s[1]
 
-	fmla	v28.2s, v4.2s, v13.2s[1]
-	fmla	v17.2s, v5.2s, v12.2s[0]
+	fmla	v28.2s, v4.2s, v13.s[1]
+	fmla	v17.2s, v5.2s, v12.s[0]
 .endm
 
 .macro KERNEL4x4_SUB
@@ -578,17 +578,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2s, v1.2s}, [pA]
 	add	pA, pA, #16
 
-	fmla	v16.2s, v0.2s, v8.2s[0]
-	fmla	v29.2s, v1.2s, v9.2s[1]
+	fmla	v16.2s, v0.2s, v8.s[0]
+	fmla	v29.2s, v1.2s, v9.s[1]
 
-	fmla	v20.2s, v0.2s, v8.2s[1]
-	fmla	v25.2s, v1.2s, v9.2s[0]
+	fmla	v20.2s, v0.2s, v8.s[1]
+	fmla	v25.2s, v1.2s, v9.s[0]
 
-	fmla	v24.2s, v0.2s, v9.2s[0]
-	fmla	v21.2s, v1.2s, v8.2s[1]
+	fmla	v24.2s, v0.2s, v9.s[0]
+	fmla	v21.2s, v1.2s, v8.s[1]
 
-	fmla	v28.2s, v0.2s, v9.2s[1]
-	fmla	v17.2s, v1.2s, v8.2s[0]
+	fmla	v28.2s, v0.2s, v9.s[1]
+	fmla	v17.2s, v1.2s, v8.s[0]
 .endm
 
 .macro SAVE4x4
@@ -633,10 +633,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2s}, [pA]
 	add	pA, pA, #8
 
-	fmla	v16.2s, v0.2s, v8.2s[0]
-	fmla	v20.2s, v0.2s, v8.2s[1]
-	fmla	v24.2s, v0.2s, v9.2s[0]
-	fmla	v28.2s, v0.2s, v9.2s[1]
+	fmla	v16.2s, v0.2s, v8.s[0]
+	fmla	v20.2s, v0.2s, v8.s[1]
+	fmla	v24.2s, v0.2s, v9.s[0]
+	fmla	v28.2s, v0.2s, v9.s[1]
 .endm
 
 .macro SAVE2x4
@@ -718,15 +718,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v3.4s}, [pA]
 	add	pA, pA, #16
 
-	fmla	v16.4s, v0.4s, v8.2s[0]
-	fmla	v17.4s, v1.4s, v8.2s[0]
-	fmla	v18.4s, v2.4s, v8.2s[0]
-	fmla	v19.4s, v3.4s, v8.2s[0]
+	fmla	v16.4s, v0.4s, v8.s[0]
+	fmla	v17.4s, v1.4s, v8.s[0]
+	fmla	v18.4s, v2.4s, v8.s[0]
+	fmla	v19.4s, v3.4s, v8.s[0]
 
-	fmla	v20.4s, v0.4s, v8.2s[1]
-	fmla	v21.4s, v1.4s, v8.2s[1]
-	fmla	v22.4s, v2.4s, v8.2s[1]
-	fmla	v23.4s, v3.4s, v8.2s[1]
+	fmla	v20.4s, v0.4s, v8.s[1]
+	fmla	v21.4s, v1.4s, v8.s[1]
+	fmla	v22.4s, v2.4s, v8.s[1]
+	fmla	v23.4s, v3.4s, v8.s[1]
 .endm
 
 .macro SAVE16x2
@@ -764,11 +764,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v1.4s}, [pA]
 	add	pA, pA, #16
 
-	fmla	v16.4s, v0.4s, v8.2s[0]
-	fmla	v17.4s, v1.4s, v8.2s[0]
+	fmla	v16.4s, v0.4s, v8.s[0]
+	fmla	v17.4s, v1.4s, v8.s[0]
 
-	fmla	v20.4s, v0.4s, v8.2s[1]
-	fmla	v21.4s, v1.4s, v8.2s[1]
+	fmla	v20.4s, v0.4s, v8.s[1]
+	fmla	v21.4s, v1.4s, v8.s[1]
 .endm
 
 .macro SAVE8x2
@@ -802,10 +802,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2s, v1.2s}, [pA]
 	add	pA, pA, #16
 
-	fmla	v16.2s, v0.2s, v8.2s[0]
-	fmla	v17.2s, v1.2s, v8.2s[0]
-	fmla	v20.2s, v0.2s, v8.2s[1]
-	fmla	v21.2s, v1.2s, v8.2s[1]
+	fmla	v16.2s, v0.2s, v8.s[0]
+	fmla	v17.2s, v1.2s, v8.s[0]
+	fmla	v20.2s, v0.2s, v8.s[1]
+	fmla	v21.2s, v1.2s, v8.s[1]
 .endm
 
 .macro SAVE4x2
@@ -837,8 +837,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2s}, [pA]
 	add	pA, pA, #8
 
-	fmla	v16.2s, v0.2s, v8.2s[0]
-	fmla	v20.2s, v0.2s, v8.2s[1]
+	fmla	v16.2s, v0.2s, v8.s[0]
+	fmla	v20.2s, v0.2s, v8.s[1]
 .endm
 
 .macro SAVE2x2
@@ -866,7 +866,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ldr	s0 , [pA]
 	add	pA, pA, #4
 
-	fmla	v16.2s, v8.2s, v0.2s[0]
+	fmla	v16.2s, v8.2s, v0.s[0]
 .endm
 
 .macro SAVE1x2
@@ -901,10 +901,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v3.4s}, [pA]
 	add	pA, pA, #16
 
-	fmla	v16.4s, v0.4s, v8.2s[0]
-	fmla	v17.4s, v1.4s, v8.2s[0]
-	fmla	v18.4s, v2.4s, v8.2s[0]
-	fmla	v19.4s, v3.4s, v8.2s[0]
+	fmla	v16.4s, v0.4s, v8.s[0]
+	fmla	v17.4s, v1.4s, v8.s[0]
+	fmla	v18.4s, v2.4s, v8.s[0]
+	fmla	v19.4s, v3.4s, v8.s[0]
 .endm
 
 .macro SAVE16x1
@@ -934,8 +934,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v1.4s}, [pA]
 	add	pA, pA, #16
 
-	fmla	v16.4s, v0.4s, v8.2s[0]
-	fmla	v17.4s, v1.4s, v8.2s[0]
+	fmla	v16.4s, v0.4s, v8.s[0]
+	fmla	v17.4s, v1.4s, v8.s[0]
 .endm
 
 .macro SAVE8x1
@@ -961,8 +961,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2s, v1.2s}, [pA]
 	add	pA , pA, #16
 
-	fmla	v16.2s, v0.2s, v8.2s[0]
-	fmla	v17.2s, v1.2s, v8.2s[0]
+	fmla	v16.2s, v0.2s, v8.s[0]
+	fmla	v17.2s, v1.2s, v8.s[0]
 .endm
 
 .macro SAVE4x1
@@ -987,7 +987,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2s}, [pA]
 	add	pA , pA, #8
 
-	fmla	v16.2s, v0.2s, v8.2s[0]
+	fmla	v16.2s, v0.2s, v8.s[0]
 .endm
 
 .macro SAVE2x1

kernel/arm64/strmm_kernel_4x4.S

@@ -147,17 +147,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2s, v1.2s}, [pA]
 	add	pA, pA, #16
 
-	fmul	v16.2s, v0.2s, v8.2s[0]
-	fmul	v29.2s, v1.2s, v9.2s[1]
+	fmul	v16.2s, v0.2s, v8.s[0]
+	fmul	v29.2s, v1.2s, v9.s[1]
 
-	fmul	v20.2s, v0.2s, v8.2s[1]
-	fmul	v25.2s, v1.2s, v9.2s[0]
+	fmul	v20.2s, v0.2s, v8.s[1]
+	fmul	v25.2s, v1.2s, v9.s[0]
 
-	fmul	v24.2s, v0.2s, v9.2s[0]
-	fmul	v21.2s, v1.2s, v8.2s[1]
+	fmul	v24.2s, v0.2s, v9.s[0]
+	fmul	v21.2s, v1.2s, v8.s[1]
 
-	fmul	v28.2s, v0.2s, v9.2s[1]
-	fmul	v17.2s, v1.2s, v8.2s[0]
+	fmul	v28.2s, v0.2s, v9.s[1]
+	fmul	v17.2s, v1.2s, v8.s[0]
 
 	ld1	{v12.2s, v13.2s}, [pB]
 	add	pB, pB, #16
@@ -166,61 +166,61 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL4x4_M1
-	fmla	v16.2s, v0.2s, v8.2s[0]
-	fmla	v29.2s, v1.2s, v9.2s[1]
+	fmla	v16.2s, v0.2s, v8.s[0]
+	fmla	v29.2s, v1.2s, v9.s[1]
 
 	ld1	{v12.2s, v13.2s}, [pB]		// For next round
 	add	pB, pB, #16
 
-	fmla	v20.2s, v0.2s, v8.2s[1]
-	fmla	v25.2s, v1.2s, v9.2s[0]
+	fmla	v20.2s, v0.2s, v8.s[1]
+	fmla	v25.2s, v1.2s, v9.s[0]
 
 	ld1	{v4.2s, v5.2s}, [pA]		// For next round
 	add	pA, pA, #16
 
-	fmla	v24.2s, v0.2s, v9.2s[0]
-	fmla	v21.2s, v1.2s, v8.2s[1]
+	fmla	v24.2s, v0.2s, v9.s[0]
+	fmla	v21.2s, v1.2s, v8.s[1]
 
 	prfm	PLDL1KEEP, [pB, #512]
 
-	fmla	v28.2s, v0.2s, v9.2s[1]
-	fmla	v17.2s, v1.2s, v8.2s[0]
+	fmla	v28.2s, v0.2s, v9.s[1]
+	fmla	v17.2s, v1.2s, v8.s[0]
 .endm
 
 .macro KERNEL4x4_M2
-	fmla	v16.2s, v4.2s, v12.2s[0]
-	fmla	v29.2s, v5.2s, v13.2s[1]
+	fmla	v16.2s, v4.2s, v12.s[0]
+	fmla	v29.2s, v5.2s, v13.s[1]
 
 	ld1	{v8.2s, v9.2s}, [pB]		// For next round
 	add	pB, pB, #16
 
-	fmla	v20.2s, v4.2s, v12.2s[1]
-	fmla	v25.2s, v5.2s, v13.2s[0]
+	fmla	v20.2s, v4.2s, v12.s[1]
+	fmla	v25.2s, v5.2s, v13.s[0]
 
 	ld1	{v0.2s, v1.2s}, [pA]		// For next round
 	add	pA, pA, #16
 
-	fmla	v24.2s, v4.2s, v13.2s[0]
-	fmla	v21.2s, v5.2s, v12.2s[1]
+	fmla	v24.2s, v4.2s, v13.s[0]
+	fmla	v21.2s, v5.2s, v12.s[1]
 
 	prfm	PLDL1KEEP, [pA, #512]
 
-	fmla	v28.2s, v4.2s, v13.2s[1]
-	fmla	v17.2s, v5.2s, v12.2s[0]
+	fmla	v28.2s, v4.2s, v13.s[1]
+	fmla	v17.2s, v5.2s, v12.s[0]
 .endm
 
 .macro KERNEL4x4_E
-	fmla	v16.2s, v4.2s, v12.2s[0]
-	fmla	v29.2s, v5.2s, v13.2s[1]
+	fmla	v16.2s, v4.2s, v12.s[0]
+	fmla	v29.2s, v5.2s, v13.s[1]
 
-	fmla	v20.2s, v4.2s, v12.2s[1]
-	fmla	v25.2s, v5.2s, v13.2s[0]
+	fmla	v20.2s, v4.2s, v12.s[1]
+	fmla	v25.2s, v5.2s, v13.s[0]
 
-	fmla	v24.2s, v4.2s, v13.2s[0]
-	fmla	v21.2s, v5.2s, v12.2s[1]
+	fmla	v24.2s, v4.2s, v13.s[0]
+	fmla	v21.2s, v5.2s, v12.s[1]
 
-	fmla	v28.2s, v4.2s, v13.2s[1]
-	fmla	v17.2s, v5.2s, v12.2s[0]
+	fmla	v28.2s, v4.2s, v13.s[1]
+	fmla	v17.2s, v5.2s, v12.s[0]
 .endm
 
 .macro KERNEL4x4_SUB
@@ -229,17 +229,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2s, v1.2s}, [pA]
 	add	pA, pA, #16
 
-	fmla	v16.2s, v0.2s, v8.2s[0]
-	fmla	v29.2s, v1.2s, v9.2s[1]
+	fmla	v16.2s, v0.2s, v8.s[0]
+	fmla	v29.2s, v1.2s, v9.s[1]
 
-	fmla	v20.2s, v0.2s, v8.2s[1]
-	fmla	v25.2s, v1.2s, v9.2s[0]
+	fmla	v20.2s, v0.2s, v8.s[1]
+	fmla	v25.2s, v1.2s, v9.s[0]
 
-	fmla	v24.2s, v0.2s, v9.2s[0]
-	fmla	v21.2s, v1.2s, v8.2s[1]
+	fmla	v24.2s, v0.2s, v9.s[0]
+	fmla	v21.2s, v1.2s, v8.s[1]
 
-	fmla	v28.2s, v0.2s, v9.2s[1]
-	fmla	v17.2s, v1.2s, v8.2s[0]
+	fmla	v28.2s, v0.2s, v9.s[1]
+	fmla	v17.2s, v1.2s, v8.s[0]
 .endm
 
 .macro SAVE4x4
@@ -280,10 +280,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2s}, [pA]
 	add	pA, pA, #8
 
-	fmla	v16.2s, v0.2s, v8.2s[0]
-	fmla	v20.2s, v0.2s, v8.2s[1]
-	fmla	v24.2s, v0.2s, v9.2s[0]
-	fmla	v28.2s, v0.2s, v9.2s[1]
+	fmla	v16.2s, v0.2s, v8.s[0]
+	fmla	v20.2s, v0.2s, v8.s[1]
+	fmla	v24.2s, v0.2s, v9.s[0]
+	fmla	v28.2s, v0.2s, v9.s[1]
 .endm
 
 .macro SAVE2x4
@@ -353,10 +353,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2s, v1.2s}, [pA]
 	add	pA, pA, #16
 
-	fmla	v16.2s, v0.2s, v8.2s[0]
-	fmla	v17.2s, v1.2s, v8.2s[0]
-	fmla	v20.2s, v0.2s, v8.2s[1]
-	fmla	v21.2s, v1.2s, v8.2s[1]
+	fmla	v16.2s, v0.2s, v8.s[0]
+	fmla	v17.2s, v1.2s, v8.s[0]
+	fmla	v20.2s, v0.2s, v8.s[1]
+	fmla	v21.2s, v1.2s, v8.s[1]
 .endm
 
 .macro SAVE4x2
@@ -386,8 +386,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2s}, [pA]
 	add	pA, pA, #8
 
-	fmla	v16.2s, v0.2s, v8.2s[0]
-	fmla	v20.2s, v0.2s, v8.2s[1]
+	fmla	v16.2s, v0.2s, v8.s[0]
+	fmla	v20.2s, v0.2s, v8.s[1]
 .endm
 
 .macro SAVE2x2
@@ -414,7 +414,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ldr	s0 , [pA]
 	add	pA, pA, #4
 
-	fmla	v16.2s, v8.2s, v0.2s[0]
+	fmla	v16.2s, v8.2s, v0.s[0]
 .endm
 
 .macro SAVE1x2
@@ -440,8 +440,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2s, v1.2s}, [pA]
 	add	pA , pA, #16
 
-	fmla	v16.2s, v0.2s, v8.2s[0]
-	fmla	v17.2s, v1.2s, v8.2s[0]
+	fmla	v16.2s, v0.2s, v8.s[0]
+	fmla	v17.2s, v1.2s, v8.s[0]
 .endm
 
 .macro SAVE4x1
@@ -468,7 +468,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2s}, [pA]
 	add	pA , pA, #8
 
-	fmla	v16.2s, v0.2s, v8.2s[0]
+	fmla	v16.2s, v0.2s, v8.s[0]
 .endm
 
 .macro SAVE2x1

kernel/arm64/strmm_kernel_8x8.S

@@ -159,22 +159,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v1.4s}, [pA]
 	add	pA, pA, #16
 
-	fmul	v16.4s, v0.4s, v4.4s[0]
-	fmul	v17.4s, v1.4s, v4.4s[0]
-	fmul	v18.4s, v0.4s, v4.4s[1]
-	fmul	v19.4s, v1.4s, v4.4s[1]
-	fmul	v20.4s, v0.4s, v4.4s[2]
-	fmul	v21.4s, v1.4s, v4.4s[2]
-	fmul	v22.4s, v0.4s, v4.4s[3]
-	fmul	v23.4s, v1.4s, v4.4s[3]
-	fmul	v24.4s, v0.4s, v5.4s[0]
-	fmul	v25.4s, v1.4s, v5.4s[0]
-	fmul	v26.4s, v0.4s, v5.4s[1]
-	fmul	v27.4s, v1.4s, v5.4s[1]
-	fmul	v28.4s, v0.4s, v5.4s[2]
-	fmul	v29.4s, v1.4s, v5.4s[2]
-	fmul	v30.4s, v0.4s, v5.4s[3]
-	fmul	v31.4s, v1.4s, v5.4s[3]
+	fmul	v16.4s, v0.4s, v4.s[0]
+	fmul	v17.4s, v1.4s, v4.s[0]
+	fmul	v18.4s, v0.4s, v4.s[1]
+	fmul	v19.4s, v1.4s, v4.s[1]
+	fmul	v20.4s, v0.4s, v4.s[2]
+	fmul	v21.4s, v1.4s, v4.s[2]
+	fmul	v22.4s, v0.4s, v4.s[3]
+	fmul	v23.4s, v1.4s, v4.s[3]
+	fmul	v24.4s, v0.4s, v5.s[0]
+	fmul	v25.4s, v1.4s, v5.s[0]
+	fmul	v26.4s, v0.4s, v5.s[1]
+	fmul	v27.4s, v1.4s, v5.s[1]
+	fmul	v28.4s, v0.4s, v5.s[2]
+	fmul	v29.4s, v1.4s, v5.s[2]
+	fmul	v30.4s, v0.4s, v5.s[3]
+	fmul	v31.4s, v1.4s, v5.s[3]
 
 	ld1	{v6.4s}, [pB]
 	add	pB, pB, #16
@@ -187,22 +187,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL8x8_M1
-	fmla	v16.4s, v0.4s, v4.4s[0]
-	fmla	v17.4s, v1.4s, v4.4s[0]
-	fmla	v18.4s, v0.4s, v4.4s[1]
-	fmla	v19.4s, v1.4s, v4.4s[1]
-	fmla	v20.4s, v0.4s, v4.4s[2]
-	fmla	v21.4s, v1.4s, v4.4s[2]
-	fmla	v22.4s, v0.4s, v4.4s[3]
-	fmla	v23.4s, v1.4s, v4.4s[3]
-	fmla	v24.4s, v0.4s, v5.4s[0]
-	fmla	v25.4s, v1.4s, v5.4s[0]
-	fmla	v26.4s, v0.4s, v5.4s[1]
-	fmla	v27.4s, v1.4s, v5.4s[1]
-	fmla	v28.4s, v0.4s, v5.4s[2]
-	fmla	v29.4s, v1.4s, v5.4s[2]
-	fmla	v30.4s, v0.4s, v5.4s[3]
-	fmla	v31.4s, v1.4s, v5.4s[3]
+	fmla	v16.4s, v0.4s, v4.s[0]
+	fmla	v17.4s, v1.4s, v4.s[0]
+	fmla	v18.4s, v0.4s, v4.s[1]
+	fmla	v19.4s, v1.4s, v4.s[1]
+	fmla	v20.4s, v0.4s, v4.s[2]
+	fmla	v21.4s, v1.4s, v4.s[2]
+	fmla	v22.4s, v0.4s, v4.s[3]
+	fmla	v23.4s, v1.4s, v4.s[3]
+	fmla	v24.4s, v0.4s, v5.s[0]
+	fmla	v25.4s, v1.4s, v5.s[0]
+	fmla	v26.4s, v0.4s, v5.s[1]
+	fmla	v27.4s, v1.4s, v5.s[1]
+	fmla	v28.4s, v0.4s, v5.s[2]
+	fmla	v29.4s, v1.4s, v5.s[2]
+	fmla	v30.4s, v0.4s, v5.s[3]
+	fmla	v31.4s, v1.4s, v5.s[3]
 
 	ld1	{v6.4s}, [pB]
 	add	pB, pB, #16
@@ -215,22 +215,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL8x8_M2
-	fmla	v16.4s, v2.4s, v6.4s[0]
-	fmla	v17.4s, v3.4s, v6.4s[0]
-	fmla	v18.4s, v2.4s, v6.4s[1]
-	fmla	v19.4s, v3.4s, v6.4s[1]
-	fmla	v20.4s, v2.4s, v6.4s[2]
-	fmla	v21.4s, v3.4s, v6.4s[2]
-	fmla	v22.4s, v2.4s, v6.4s[3]
-	fmla	v23.4s, v3.4s, v6.4s[3]
-	fmla	v24.4s, v2.4s, v7.4s[0]
-	fmla	v25.4s, v3.4s, v7.4s[0]
-	fmla	v26.4s, v2.4s, v7.4s[1]
-	fmla	v27.4s, v3.4s, v7.4s[1]
-	fmla	v28.4s, v2.4s, v7.4s[2]
-	fmla	v29.4s, v3.4s, v7.4s[2]
-	fmla	v30.4s, v2.4s, v7.4s[3]
-	fmla	v31.4s, v3.4s, v7.4s[3]
+	fmla	v16.4s, v2.4s, v6.s[0]
+	fmla	v17.4s, v3.4s, v6.s[0]
+	fmla	v18.4s, v2.4s, v6.s[1]
+	fmla	v19.4s, v3.4s, v6.s[1]
+	fmla	v20.4s, v2.4s, v6.s[2]
+	fmla	v21.4s, v3.4s, v6.s[2]
+	fmla	v22.4s, v2.4s, v6.s[3]
+	fmla	v23.4s, v3.4s, v6.s[3]
+	fmla	v24.4s, v2.4s, v7.s[0]
+	fmla	v25.4s, v3.4s, v7.s[0]
+	fmla	v26.4s, v2.4s, v7.s[1]
+	fmla	v27.4s, v3.4s, v7.s[1]
+	fmla	v28.4s, v2.4s, v7.s[2]
+	fmla	v29.4s, v3.4s, v7.s[2]
+	fmla	v30.4s, v2.4s, v7.s[3]
+	fmla	v31.4s, v3.4s, v7.s[3]
 
 	ld1	{v4.4s}, [pB]
 	add	pB, pB, #16
@@ -243,22 +243,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL8x8_E
-	fmla	v16.4s, v2.4s, v6.4s[0]
-	fmla	v17.4s, v3.4s, v6.4s[0]
-	fmla	v18.4s, v2.4s, v6.4s[1]
-	fmla	v19.4s, v3.4s, v6.4s[1]
-	fmla	v20.4s, v2.4s, v6.4s[2]
-	fmla	v21.4s, v3.4s, v6.4s[2]
-	fmla	v22.4s, v2.4s, v6.4s[3]
-	fmla	v23.4s, v3.4s, v6.4s[3]
-	fmla	v24.4s, v2.4s, v7.4s[0]
-	fmla	v25.4s, v3.4s, v7.4s[0]
-	fmla	v26.4s, v2.4s, v7.4s[1]
-	fmla	v27.4s, v3.4s, v7.4s[1]
-	fmla	v28.4s, v2.4s, v7.4s[2]
-	fmla	v29.4s, v3.4s, v7.4s[2]
-	fmla	v30.4s, v2.4s, v7.4s[3]
-	fmla	v31.4s, v3.4s, v7.4s[3]
+	fmla	v16.4s, v2.4s, v6.s[0]
+	fmla	v17.4s, v3.4s, v6.s[0]
+	fmla	v18.4s, v2.4s, v6.s[1]
+	fmla	v19.4s, v3.4s, v6.s[1]
+	fmla	v20.4s, v2.4s, v6.s[2]
+	fmla	v21.4s, v3.4s, v6.s[2]
+	fmla	v22.4s, v2.4s, v6.s[3]
+	fmla	v23.4s, v3.4s, v6.s[3]
+	fmla	v24.4s, v2.4s, v7.s[0]
+	fmla	v25.4s, v3.4s, v7.s[0]
+	fmla	v26.4s, v2.4s, v7.s[1]
+	fmla	v27.4s, v3.4s, v7.s[1]
+	fmla	v28.4s, v2.4s, v7.s[2]
+	fmla	v29.4s, v3.4s, v7.s[2]
+	fmla	v30.4s, v2.4s, v7.s[3]
+	fmla	v31.4s, v3.4s, v7.s[3]
 .endm
 
 .macro KERNEL8x8_SUB
@@ -271,22 +271,22 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v1.4s}, [pA]
 	add	pA, pA, #16
 
-	fmla	v16.4s, v0.4s, v4.4s[0]
-	fmla	v17.4s, v1.4s, v4.4s[0]
-	fmla	v18.4s, v0.4s, v4.4s[1]
-	fmla	v19.4s, v1.4s, v4.4s[1]
-	fmla	v20.4s, v0.4s, v4.4s[2]
-	fmla	v21.4s, v1.4s, v4.4s[2]
-	fmla	v22.4s, v0.4s, v4.4s[3]
-	fmla	v23.4s, v1.4s, v4.4s[3]
-	fmla	v24.4s, v0.4s, v5.4s[0]
-	fmla	v25.4s, v1.4s, v5.4s[0]
-	fmla	v26.4s, v0.4s, v5.4s[1]
-	fmla	v27.4s, v1.4s, v5.4s[1]
-	fmla	v28.4s, v0.4s, v5.4s[2]
-	fmla	v29.4s, v1.4s, v5.4s[2]
-	fmla	v30.4s, v0.4s, v5.4s[3]
-	fmla	v31.4s, v1.4s, v5.4s[3]
+	fmla	v16.4s, v0.4s, v4.s[0]
+	fmla	v17.4s, v1.4s, v4.s[0]
+	fmla	v18.4s, v0.4s, v4.s[1]
+	fmla	v19.4s, v1.4s, v4.s[1]
+	fmla	v20.4s, v0.4s, v4.s[2]
+	fmla	v21.4s, v1.4s, v4.s[2]
+	fmla	v22.4s, v0.4s, v4.s[3]
+	fmla	v23.4s, v1.4s, v4.s[3]
+	fmla	v24.4s, v0.4s, v5.s[0]
+	fmla	v25.4s, v1.4s, v5.s[0]
+	fmla	v26.4s, v0.4s, v5.s[1]
+	fmla	v27.4s, v1.4s, v5.s[1]
+	fmla	v28.4s, v0.4s, v5.s[2]
+	fmla	v29.4s, v1.4s, v5.s[2]
+	fmla	v30.4s, v0.4s, v5.s[3]
+	fmla	v31.4s, v1.4s, v5.s[3]
 .endm
 
 .macro SAVE8x8
@@ -361,14 +361,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.4s}, [pA]
 	add	pA, pA, #16
 
-	fmul	v16.4s, v0.4s, v4.4s[0]
-	fmul	v18.4s, v0.4s, v4.4s[1]
-	fmul	v20.4s, v0.4s, v4.4s[2]
-	fmul	v22.4s, v0.4s, v4.4s[3]
-	fmul	v24.4s, v0.4s, v5.4s[0]
-	fmul	v26.4s, v0.4s, v5.4s[1]
-	fmul	v28.4s, v0.4s, v5.4s[2]
-	fmul	v30.4s, v0.4s, v5.4s[3]
+	fmul	v16.4s, v0.4s, v4.s[0]
+	fmul	v18.4s, v0.4s, v4.s[1]
+	fmul	v20.4s, v0.4s, v4.s[2]
+	fmul	v22.4s, v0.4s, v4.s[3]
+	fmul	v24.4s, v0.4s, v5.s[0]
+	fmul	v26.4s, v0.4s, v5.s[1]
+	fmul	v28.4s, v0.4s, v5.s[2]
+	fmul	v30.4s, v0.4s, v5.s[3]
 
 	ld1	{v6.4s}, [pB]
 	add	pB, pB, #16
@@ -379,14 +379,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL4x8_M1
-	fmla	v16.4s, v0.4s, v4.4s[0]
-	fmla	v18.4s, v0.4s, v4.4s[1]
-	fmla	v20.4s, v0.4s, v4.4s[2]
-	fmla	v22.4s, v0.4s, v4.4s[3]
-	fmla	v24.4s, v0.4s, v5.4s[0]
-	fmla	v26.4s, v0.4s, v5.4s[1]
-	fmla	v28.4s, v0.4s, v5.4s[2]
-	fmla	v30.4s, v0.4s, v5.4s[3]
+	fmla	v16.4s, v0.4s, v4.s[0]
+	fmla	v18.4s, v0.4s, v4.s[1]
+	fmla	v20.4s, v0.4s, v4.s[2]
+	fmla	v22.4s, v0.4s, v4.s[3]
+	fmla	v24.4s, v0.4s, v5.s[0]
+	fmla	v26.4s, v0.4s, v5.s[1]
+	fmla	v28.4s, v0.4s, v5.s[2]
+	fmla	v30.4s, v0.4s, v5.s[3]
 
 	ld1	{v6.4s}, [pB]
 	add	pB, pB, #16
@@ -397,14 +397,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL4x8_M2
-	fmla	v16.4s, v2.4s, v6.4s[0]
-	fmla	v18.4s, v2.4s, v6.4s[1]
-	fmla	v20.4s, v2.4s, v6.4s[2]
-	fmla	v22.4s, v2.4s, v6.4s[3]
-	fmla	v24.4s, v2.4s, v7.4s[0]
-	fmla	v26.4s, v2.4s, v7.4s[1]
-	fmla	v28.4s, v2.4s, v7.4s[2]
-	fmla	v30.4s, v2.4s, v7.4s[3]
+	fmla	v16.4s, v2.4s, v6.s[0]
+	fmla	v18.4s, v2.4s, v6.s[1]
+	fmla	v20.4s, v2.4s, v6.s[2]
+	fmla	v22.4s, v2.4s, v6.s[3]
+	fmla	v24.4s, v2.4s, v7.s[0]
+	fmla	v26.4s, v2.4s, v7.s[1]
+	fmla	v28.4s, v2.4s, v7.s[2]
+	fmla	v30.4s, v2.4s, v7.s[3]
 
 	ld1	{v4.4s}, [pB]
 	add	pB, pB, #16
@@ -415,14 +415,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL4x8_E
-	fmla	v16.4s, v2.4s, v6.4s[0]
-	fmla	v18.4s, v2.4s, v6.4s[1]
-	fmla	v20.4s, v2.4s, v6.4s[2]
-	fmla	v22.4s, v2.4s, v6.4s[3]
-	fmla	v24.4s, v2.4s, v7.4s[0]
-	fmla	v26.4s, v2.4s, v7.4s[1]
-	fmla	v28.4s, v2.4s, v7.4s[2]
-	fmla	v30.4s, v2.4s, v7.4s[3]
+	fmla	v16.4s, v2.4s, v6.s[0]
+	fmla	v18.4s, v2.4s, v6.s[1]
+	fmla	v20.4s, v2.4s, v6.s[2]
+	fmla	v22.4s, v2.4s, v6.s[3]
+	fmla	v24.4s, v2.4s, v7.s[0]
+	fmla	v26.4s, v2.4s, v7.s[1]
+	fmla	v28.4s, v2.4s, v7.s[2]
+	fmla	v30.4s, v2.4s, v7.s[3]
 .endm
 
 .macro KERNEL4x8_SUB
@@ -433,14 +433,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.4s}, [pA]
 	add	pA, pA, #16
 
-	fmla	v16.4s, v0.4s, v4.4s[0]
-	fmla	v18.4s, v0.4s, v4.4s[1]
-	fmla	v20.4s, v0.4s, v4.4s[2]
-	fmla	v22.4s, v0.4s, v4.4s[3]
-	fmla	v24.4s, v0.4s, v5.4s[0]
-	fmla	v26.4s, v0.4s, v5.4s[1]
-	fmla	v28.4s, v0.4s, v5.4s[2]
-	fmla	v30.4s, v0.4s, v5.4s[3]
+	fmla	v16.4s, v0.4s, v4.s[0]
+	fmla	v18.4s, v0.4s, v4.s[1]
+	fmla	v20.4s, v0.4s, v4.s[2]
+	fmla	v22.4s, v0.4s, v4.s[3]
+	fmla	v24.4s, v0.4s, v5.s[0]
+	fmla	v26.4s, v0.4s, v5.s[1]
+	fmla	v28.4s, v0.4s, v5.s[2]
+	fmla	v30.4s, v0.4s, v5.s[3]
 .endm
 
 .macro SAVE4x8
@@ -514,14 +514,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2s}, [pA]
 	add	pA, pA, #8
 
-	fmla	v16.2s, v0.2s, v4.4s[0]
-	fmla	v18.2s, v0.2s, v4.4s[1]
-	fmla	v20.2s, v0.2s, v4.4s[2]
-	fmla	v22.2s, v0.2s, v4.4s[3]
-	fmla	v24.2s, v0.2s, v5.4s[0]
-	fmla	v26.2s, v0.2s, v5.4s[1]
-	fmla	v28.2s, v0.2s, v5.4s[2]
-	fmla	v30.2s, v0.2s, v5.4s[3]
+	fmla	v16.2s, v0.2s, v4.s[0]
+	fmla	v18.2s, v0.2s, v4.s[1]
+	fmla	v20.2s, v0.2s, v4.s[2]
+	fmla	v22.2s, v0.2s, v4.s[3]
+	fmla	v24.2s, v0.2s, v5.s[0]
+	fmla	v26.2s, v0.2s, v5.s[1]
+	fmla	v28.2s, v0.2s, v5.s[2]
+	fmla	v30.2s, v0.2s, v5.s[3]
 .endm
 
 .macro SAVE2x8
@@ -595,14 +595,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ldr	s0, [pA]
 	add	pA, pA, #4
 
-	fmla	s16, s0, v4.4s[0]
-	fmla	s18, s0, v4.4s[1]
-	fmla	s20, s0, v4.4s[2]
-	fmla	s22, s0, v4.4s[3]
-	fmla	s24, s0, v5.4s[0]
-	fmla	s26, s0, v5.4s[1]
-	fmla	s28, s0, v5.4s[2]
-	fmla	s30, s0, v5.4s[3]
+	fmla	s16, s0, v4.s[0]
+	fmla	s18, s0, v4.s[1]
+	fmla	s20, s0, v4.s[2]
+	fmla	s22, s0, v4.s[3]
+	fmla	s24, s0, v5.s[0]
+	fmla	s26, s0, v5.s[1]
+	fmla	s28, s0, v5.s[2]
+	fmla	s30, s0, v5.s[3]
 .endm
 
 .macro SAVE1x8
@@ -676,14 +676,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v1.4s}, [pA]
 	add	pA, pA, #16
 
-	fmul	v16.4s, v0.4s, v8.2s[0]
-	fmul	v17.4s, v1.4s, v8.2s[0]
-	fmul	v20.4s, v0.4s, v8.2s[1]
-	fmul	v21.4s, v1.4s, v8.2s[1]
-	fmul	v24.4s, v0.4s, v9.2s[0]
-	fmul	v25.4s, v1.4s, v9.2s[0]
-	fmul	v28.4s, v0.4s, v9.2s[1]
-	fmul	v29.4s, v1.4s, v9.2s[1]
+	fmul	v16.4s, v0.4s, v8.s[0]
+	fmul	v17.4s, v1.4s, v8.s[0]
+	fmul	v20.4s, v0.4s, v8.s[1]
+	fmul	v21.4s, v1.4s, v8.s[1]
+	fmul	v24.4s, v0.4s, v9.s[0]
+	fmul	v25.4s, v1.4s, v9.s[0]
+	fmul	v28.4s, v0.4s, v9.s[1]
+	fmul	v29.4s, v1.4s, v9.s[1]
 
 	ld1	{v12.2s, v13.2s}, [pB]
 	add	pB, pB, #16
@@ -694,14 +694,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL8x4_M1
-	fmla	v16.4s, v0.4s, v8.2s[0]
-	fmla	v17.4s, v1.4s, v8.2s[0]
-	fmla	v20.4s, v0.4s, v8.2s[1]
-	fmla	v21.4s, v1.4s, v8.2s[1]
-	fmla	v24.4s, v0.4s, v9.2s[0]
-	fmla	v25.4s, v1.4s, v9.2s[0]
-	fmla	v28.4s, v0.4s, v9.2s[1]
-	fmla	v29.4s, v1.4s, v9.2s[1]
+	fmla	v16.4s, v0.4s, v8.s[0]
+	fmla	v17.4s, v1.4s, v8.s[0]
+	fmla	v20.4s, v0.4s, v8.s[1]
+	fmla	v21.4s, v1.4s, v8.s[1]
+	fmla	v24.4s, v0.4s, v9.s[0]
+	fmla	v25.4s, v1.4s, v9.s[0]
+	fmla	v28.4s, v0.4s, v9.s[1]
+	fmla	v29.4s, v1.4s, v9.s[1]
 
 	ld1	{v12.2s, v13.2s}, [pB]
 	add	pB, pB, #16
@@ -712,14 +712,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL8x4_M2
-	fmla	v16.4s, v4.4s, v12.2s[0]
-	fmla	v17.4s, v5.4s, v12.2s[0]
-	fmla	v20.4s, v4.4s, v12.2s[1]
-	fmla	v21.4s, v5.4s, v12.2s[1]
-	fmla	v24.4s, v4.4s, v13.2s[0]
-	fmla	v25.4s, v5.4s, v13.2s[0]
-	fmla	v28.4s, v4.4s, v13.2s[1]
-	fmla	v29.4s, v5.4s, v13.2s[1]
+	fmla	v16.4s, v4.4s, v12.s[0]
+	fmla	v17.4s, v5.4s, v12.s[0]
+	fmla	v20.4s, v4.4s, v12.s[1]
+	fmla	v21.4s, v5.4s, v12.s[1]
+	fmla	v24.4s, v4.4s, v13.s[0]
+	fmla	v25.4s, v5.4s, v13.s[0]
+	fmla	v28.4s, v4.4s, v13.s[1]
+	fmla	v29.4s, v5.4s, v13.s[1]
 
 	ld1	{v8.2s, v9.2s}, [pB]
 	add	pB, pB, #16
@@ -730,14 +730,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL8x4_E
-	fmla	v16.4s, v4.4s, v12.2s[0]
-	fmla	v17.4s, v5.4s, v12.2s[0]
-	fmla	v20.4s, v4.4s, v12.2s[1]
-	fmla	v21.4s, v5.4s, v12.2s[1]
-	fmla	v24.4s, v4.4s, v13.2s[0]
-	fmla	v25.4s, v5.4s, v13.2s[0]
-	fmla	v28.4s, v4.4s, v13.2s[1]
-	fmla	v29.4s, v5.4s, v13.2s[1]
+	fmla	v16.4s, v4.4s, v12.s[0]
+	fmla	v17.4s, v5.4s, v12.s[0]
+	fmla	v20.4s, v4.4s, v12.s[1]
+	fmla	v21.4s, v5.4s, v12.s[1]
+	fmla	v24.4s, v4.4s, v13.s[0]
+	fmla	v25.4s, v5.4s, v13.s[0]
+	fmla	v28.4s, v4.4s, v13.s[1]
+	fmla	v29.4s, v5.4s, v13.s[1]
 .endm
 
 .macro KERNEL8x4_SUB
@@ -748,14 +748,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v1.4s}, [pA]
 	add	pA, pA, #16
 
-	fmla	v16.4s, v0.4s, v8.2s[0]
-	fmla	v17.4s, v1.4s, v8.2s[0]
-	fmla	v20.4s, v0.4s, v8.2s[1]
-	fmla	v21.4s, v1.4s, v8.2s[1]
-	fmla	v24.4s, v0.4s, v9.2s[0]
-	fmla	v25.4s, v1.4s, v9.2s[0]
-	fmla	v28.4s, v0.4s, v9.2s[1]
-	fmla	v29.4s, v1.4s, v9.2s[1]
+	fmla	v16.4s, v0.4s, v8.s[0]
+	fmla	v17.4s, v1.4s, v8.s[0]
+	fmla	v20.4s, v0.4s, v8.s[1]
+	fmla	v21.4s, v1.4s, v8.s[1]
+	fmla	v24.4s, v0.4s, v9.s[0]
+	fmla	v25.4s, v1.4s, v9.s[0]
+	fmla	v28.4s, v0.4s, v9.s[1]
+	fmla	v29.4s, v1.4s, v9.s[1]
 .endm
 
 .macro SAVE8x4
@@ -808,17 +808,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2s, v1.2s}, [pA]
 	add	pA, pA, #16
 
-	fmul	v16.2s, v0.2s, v8.2s[0]
-	fmul	v29.2s, v1.2s, v9.2s[1]
+	fmul	v16.2s, v0.2s, v8.s[0]
+	fmul	v29.2s, v1.2s, v9.s[1]
 
-	fmul	v20.2s, v0.2s, v8.2s[1]
-	fmul	v25.2s, v1.2s, v9.2s[0]
+	fmul	v20.2s, v0.2s, v8.s[1]
+	fmul	v25.2s, v1.2s, v9.s[0]
 
-	fmul	v24.2s, v0.2s, v9.2s[0]
-	fmul	v21.2s, v1.2s, v8.2s[1]
+	fmul	v24.2s, v0.2s, v9.s[0]
+	fmul	v21.2s, v1.2s, v8.s[1]
 
-	fmul	v28.2s, v0.2s, v9.2s[1]
-	fmul	v17.2s, v1.2s, v8.2s[0]
+	fmul	v28.2s, v0.2s, v9.s[1]
+	fmul	v17.2s, v1.2s, v8.s[0]
 
 	ld1	{v12.2s, v13.2s}, [pB]
 	add	pB, pB, #16
@@ -827,61 +827,61 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL4x4_M1
-	fmla	v16.2s, v0.2s, v8.2s[0]
-	fmla	v29.2s, v1.2s, v9.2s[1]
+	fmla	v16.2s, v0.2s, v8.s[0]
+	fmla	v29.2s, v1.2s, v9.s[1]
 
 	ld1	{v12.2s, v13.2s}, [pB]		// For next round
 	add	pB, pB, #16
 
-	fmla	v20.2s, v0.2s, v8.2s[1]
-	fmla	v25.2s, v1.2s, v9.2s[0]
+	fmla	v20.2s, v0.2s, v8.s[1]
+	fmla	v25.2s, v1.2s, v9.s[0]
 
 	ld1	{v4.2s, v5.2s}, [pA]		// For next round
 	add	pA, pA, #16
 
-	fmla	v24.2s, v0.2s, v9.2s[0]
-	fmla	v21.2s, v1.2s, v8.2s[1]
+	fmla	v24.2s, v0.2s, v9.s[0]
+	fmla	v21.2s, v1.2s, v8.s[1]
 
 	prfm	PLDL1KEEP, [pB, #512]
 
-	fmla	v28.2s, v0.2s, v9.2s[1]
-	fmla	v17.2s, v1.2s, v8.2s[0]
+	fmla	v28.2s, v0.2s, v9.s[1]
+	fmla	v17.2s, v1.2s, v8.s[0]
 .endm
 
 .macro KERNEL4x4_M2
-	fmla	v16.2s, v4.2s, v12.2s[0]
-	fmla	v29.2s, v5.2s, v13.2s[1]
+	fmla	v16.2s, v4.2s, v12.s[0]
+	fmla	v29.2s, v5.2s, v13.s[1]
 
 	ld1	{v8.2s, v9.2s}, [pB]		// For next round
 	add	pB, pB, #16
 
-	fmla	v20.2s, v4.2s, v12.2s[1]
-	fmla	v25.2s, v5.2s, v13.2s[0]
+	fmla	v20.2s, v4.2s, v12.s[1]
+	fmla	v25.2s, v5.2s, v13.s[0]
 
 	ld1	{v0.2s, v1.2s}, [pA]		// For next round
 	add	pA, pA, #16
 
-	fmla	v24.2s, v4.2s, v13.2s[0]
-	fmla	v21.2s, v5.2s, v12.2s[1]
+	fmla	v24.2s, v4.2s, v13.s[0]
+	fmla	v21.2s, v5.2s, v12.s[1]
 
 	prfm	PLDL1KEEP, [pA, #512]
 
-	fmla	v28.2s, v4.2s, v13.2s[1]
-	fmla	v17.2s, v5.2s, v12.2s[0]
+	fmla	v28.2s, v4.2s, v13.s[1]
+	fmla	v17.2s, v5.2s, v12.s[0]
 .endm
 
 .macro KERNEL4x4_E
-	fmla	v16.2s, v4.2s, v12.2s[0]
-	fmla	v29.2s, v5.2s, v13.2s[1]
+	fmla	v16.2s, v4.2s, v12.s[0]
+	fmla	v29.2s, v5.2s, v13.s[1]
 
-	fmla	v20.2s, v4.2s, v12.2s[1]
-	fmla	v25.2s, v5.2s, v13.2s[0]
+	fmla	v20.2s, v4.2s, v12.s[1]
+	fmla	v25.2s, v5.2s, v13.s[0]
 
-	fmla	v24.2s, v4.2s, v13.2s[0]
-	fmla	v21.2s, v5.2s, v12.2s[1]
+	fmla	v24.2s, v4.2s, v13.s[0]
+	fmla	v21.2s, v5.2s, v12.s[1]
 
-	fmla	v28.2s, v4.2s, v13.2s[1]
-	fmla	v17.2s, v5.2s, v12.2s[0]
+	fmla	v28.2s, v4.2s, v13.s[1]
+	fmla	v17.2s, v5.2s, v12.s[0]
 .endm
 
 .macro KERNEL4x4_SUB
@@ -890,17 +890,17 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2s, v1.2s}, [pA]
 	add	pA, pA, #16
 
-	fmla	v16.2s, v0.2s, v8.2s[0]
-	fmla	v29.2s, v1.2s, v9.2s[1]
+	fmla	v16.2s, v0.2s, v8.s[0]
+	fmla	v29.2s, v1.2s, v9.s[1]
 
-	fmla	v20.2s, v0.2s, v8.2s[1]
-	fmla	v25.2s, v1.2s, v9.2s[0]
+	fmla	v20.2s, v0.2s, v8.s[1]
+	fmla	v25.2s, v1.2s, v9.s[0]
 
-	fmla	v24.2s, v0.2s, v9.2s[0]
-	fmla	v21.2s, v1.2s, v8.2s[1]
+	fmla	v24.2s, v0.2s, v9.s[0]
+	fmla	v21.2s, v1.2s, v8.s[1]
 
-	fmla	v28.2s, v0.2s, v9.2s[1]
-	fmla	v17.2s, v1.2s, v8.2s[0]
+	fmla	v28.2s, v0.2s, v9.s[1]
+	fmla	v17.2s, v1.2s, v8.s[0]
 .endm
 
 .macro SAVE4x4
@@ -945,10 +945,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2s}, [pA]
 	add	pA, pA, #8
 
-	fmla	v16.2s, v0.2s, v8.2s[0]
-	fmla	v20.2s, v0.2s, v8.2s[1]
-	fmla	v24.2s, v0.2s, v9.2s[0]
-	fmla	v28.2s, v0.2s, v9.2s[1]
+	fmla	v16.2s, v0.2s, v8.s[0]
+	fmla	v20.2s, v0.2s, v8.s[1]
+	fmla	v24.2s, v0.2s, v9.s[0]
+	fmla	v28.2s, v0.2s, v9.s[1]
 .endm
 
 .macro SAVE2x4
@@ -1028,11 +1028,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v1.4s}, [pA]
 	add	pA, pA, #16
 
-	fmla	v16.4s, v0.4s, v8.2s[0]
-	fmla	v17.4s, v1.4s, v8.2s[0]
+	fmla	v16.4s, v0.4s, v8.s[0]
+	fmla	v17.4s, v1.4s, v8.s[0]
 
-	fmla	v20.4s, v0.4s, v8.2s[1]
-	fmla	v21.4s, v1.4s, v8.2s[1]
+	fmla	v20.4s, v0.4s, v8.s[1]
+	fmla	v21.4s, v1.4s, v8.s[1]
 .endm
 
 .macro SAVE8x2
@@ -1068,10 +1068,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2s, v1.2s}, [pA]
 	add	pA, pA, #16
 
-	fmla	v16.2s, v0.2s, v8.2s[0]
-	fmla	v17.2s, v1.2s, v8.2s[0]
-	fmla	v20.2s, v0.2s, v8.2s[1]
-	fmla	v21.2s, v1.2s, v8.2s[1]
+	fmla	v16.2s, v0.2s, v8.s[0]
+	fmla	v17.2s, v1.2s, v8.s[0]
+	fmla	v20.2s, v0.2s, v8.s[1]
+	fmla	v21.2s, v1.2s, v8.s[1]
 .endm
 
 .macro SAVE4x2
@@ -1103,8 +1103,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2s}, [pA]
 	add	pA, pA, #8
 
-	fmla	v16.2s, v0.2s, v8.2s[0]
-	fmla	v20.2s, v0.2s, v8.2s[1]
+	fmla	v16.2s, v0.2s, v8.s[0]
+	fmla	v20.2s, v0.2s, v8.s[1]
 .endm
 
 .macro SAVE2x2
@@ -1133,7 +1133,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ldr	s0 , [pA]
 	add	pA, pA, #4
 
-	fmla	v16.2s, v8.2s, v0.2s[0]
+	fmla	v16.2s, v8.2s, v0.s[0]
 .endm
 
 .macro SAVE1x2
@@ -1163,8 +1163,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v1.4s}, [pA]
 	add	pA, pA, #16
 
-	fmla	v16.4s, v0.4s, v8.2s[0]
-	fmla	v17.4s, v1.4s, v8.2s[0]
+	fmla	v16.4s, v0.4s, v8.s[0]
+	fmla	v17.4s, v1.4s, v8.s[0]
 .endm
 
 .macro SAVE8x1
@@ -1190,8 +1190,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2s, v1.2s}, [pA]
 	add	pA , pA, #16
 
-	fmla	v16.2s, v0.2s, v8.2s[0]
-	fmla	v17.2s, v1.2s, v8.2s[0]
+	fmla	v16.2s, v0.2s, v8.s[0]
+	fmla	v17.2s, v1.2s, v8.s[0]
 .endm
 
 .macro SAVE4x1
@@ -1216,7 +1216,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld1	{v0.2s}, [pA]
 	add	pA , pA, #8
 
-	fmla	v16.2s, v0.2s, v8.2s[0]
+	fmla	v16.2s, v0.2s, v8.s[0]
 .endm
 
 .macro SAVE2x1

kernel/arm64/zgemm_kernel_4x4.S

@@ -182,93 +182,93 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld2	{v2.2d, v3.2d}, [pA]
 	add	pA, pA, #32
 
-	fmul	v16.2d, v0.2d, v8.2d[0]
-	OP_ii	v16.2d, v1.2d, v9.2d[0]
+	fmul	v16.2d, v0.2d, v8.d[0]
+	OP_ii	v16.2d, v1.2d, v9.d[0]
 #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
     defined(RR) || defined(RC) || defined(CR) || defined(CC)
 	eor	v17.16b, v17.16b, v17.16b
-	fmls	v17.2d, v0.2d, v9.2d[0]
+	fmls	v17.2d, v0.2d, v9.d[0]
 #else
-	fmul	v17.2d, v0.2d, v9.2d[0]
+	fmul	v17.2d, v0.2d, v9.d[0]
 #endif
-	OP_ir	v17.2d, v1.2d, v8.2d[0]
+	OP_ir	v17.2d, v1.2d, v8.d[0]
 
-	fmul	v18.2d, v2.2d, v8.2d[0]
-	OP_ii	v18.2d, v3.2d, v9.2d[0]
+	fmul	v18.2d, v2.2d, v8.d[0]
+	OP_ii	v18.2d, v3.2d, v9.d[0]
 #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
     defined(RR) || defined(RC) || defined(CR) || defined(CC)
 	eor	v19.16b, v19.16b, v19.16b
-	fmls	v19.2d, v2.2d, v9.2d[0]
+	fmls	v19.2d, v2.2d, v9.d[0]
 #else
-	fmul	v19.2d, v2.2d, v9.2d[0]
+	fmul	v19.2d, v2.2d, v9.d[0]
 #endif
-	OP_ir	v19.2d, v3.2d, v8.2d[0]
+	OP_ir	v19.2d, v3.2d, v8.d[0]
 
-	fmul	v20.2d, v0.2d, v8.2d[1]
-	OP_ii	v20.2d, v1.2d, v9.2d[1]
+	fmul	v20.2d, v0.2d, v8.d[1]
+	OP_ii	v20.2d, v1.2d, v9.d[1]
 #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
     defined(RR) || defined(RC) || defined(CR) || defined(CC)
 	eor	v21.16b, v21.16b, v21.16b
-	fmls	v21.2d, v0.2d, v9.2d[1]
+	fmls	v21.2d, v0.2d, v9.d[1]
 #else
-	fmul	v21.2d, v0.2d, v9.2d[1]
+	fmul	v21.2d, v0.2d, v9.d[1]
 #endif
-	OP_ir	v21.2d, v1.2d, v8.2d[1]
+	OP_ir	v21.2d, v1.2d, v8.d[1]
 
-	fmul	v22.2d, v2.2d, v8.2d[1]
-	OP_ii	v22.2d, v3.2d, v9.2d[1]
+	fmul	v22.2d, v2.2d, v8.d[1]
+	OP_ii	v22.2d, v3.2d, v9.d[1]
 #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
     defined(RR) || defined(RC) || defined(CR) || defined(CC)
 	eor	v23.16b, v23.16b, v23.16b
-	fmls	v23.2d, v2.2d, v9.2d[1]
+	fmls	v23.2d, v2.2d, v9.d[1]
 #else
-	fmul	v23.2d, v2.2d, v9.2d[1]
+	fmul	v23.2d, v2.2d, v9.d[1]
 #endif
-	OP_ir	v23.2d, v3.2d, v8.2d[1]
+	OP_ir	v23.2d, v3.2d, v8.d[1]
 
-	fmul	v24.2d, v0.2d, v10.2d[0]
-	OP_ii	v24.2d, v1.2d, v11.2d[0]
+	fmul	v24.2d, v0.2d, v10.d[0]
+	OP_ii	v24.2d, v1.2d, v11.d[0]
 #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
     defined(RR) || defined(RC) || defined(CR) || defined(CC)
 	eor	v25.16b, v25.16b, v25.16b
-	fmls	v25.2d, v0.2d, v11.2d[0]
+	fmls	v25.2d, v0.2d, v11.d[0]
 #else
-	fmul	v25.2d, v0.2d, v11.2d[0]
+	fmul	v25.2d, v0.2d, v11.d[0]
 #endif
-	OP_ir	v25.2d, v1.2d, v10.2d[0]
+	OP_ir	v25.2d, v1.2d, v10.d[0]
 
-	fmul	v26.2d, v2.2d, v10.2d[0]
-	OP_ii	v26.2d, v3.2d, v11.2d[0]
+	fmul	v26.2d, v2.2d, v10.d[0]
+	OP_ii	v26.2d, v3.2d, v11.d[0]
 #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
     defined(RR) || defined(RC) || defined(CR) || defined(CC)
 	eor	v27.16b, v27.16b, v27.16b
-	fmls	v27.2d, v2.2d, v11.2d[0]
+	fmls	v27.2d, v2.2d, v11.d[0]
 #else
-	fmul	v27.2d, v2.2d, v11.2d[0]
+	fmul	v27.2d, v2.2d, v11.d[0]
 #endif
-	OP_ir	v27.2d, v3.2d, v10.2d[0]
+	OP_ir	v27.2d, v3.2d, v10.d[0]
 
-	fmul	v28.2d, v0.2d, v10.2d[1]
-	OP_ii	v28.2d, v1.2d, v11.2d[1]
+	fmul	v28.2d, v0.2d, v10.d[1]
+	OP_ii	v28.2d, v1.2d, v11.d[1]
 #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
     defined(RR) || defined(RC) || defined(CR) || defined(CC)
 	eor	v29.16b, v29.16b, v29.16b
-	fmls	v29.2d, v0.2d, v11.2d[1]
+	fmls	v29.2d, v0.2d, v11.d[1]
 #else
-	fmul	v29.2d, v0.2d, v11.2d[1]
+	fmul	v29.2d, v0.2d, v11.d[1]
 #endif
-	OP_ir	v29.2d, v1.2d, v10.2d[1]
+	OP_ir	v29.2d, v1.2d, v10.d[1]
 
-	fmul	v30.2d, v2.2d, v10.2d[1]
-	OP_ii	v30.2d, v3.2d, v11.2d[1]
+	fmul	v30.2d, v2.2d, v10.d[1]
+	OP_ii	v30.2d, v3.2d, v11.d[1]
 #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
     defined(RR) || defined(RC) || defined(CR) || defined(CC)
 	eor	v31.16b, v31.16b, v31.16b
-	fmls	v31.2d, v2.2d, v11.2d[1]
+	fmls	v31.2d, v2.2d, v11.d[1]
 #else
-	fmul	v31.2d, v2.2d, v11.2d[1]
+	fmul	v31.2d, v2.2d, v11.d[1]
 #endif
-	OP_ir	v31.2d, v3.2d, v10.2d[1]
+	OP_ir	v31.2d, v3.2d, v10.d[1]
 
 	ld2	{v12.2d, v13.2d}, [pB]
 	add	pB, pB, #32
@@ -281,161 +281,161 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL4x4_M1
-	OP_rr	v16.2d, v0.2d, v8.2d[0]
-	OP_ii	v16.2d, v1.2d, v9.2d[0]
-	OP_ri	v17.2d, v0.2d, v9.2d[0]
-	OP_ir	v17.2d, v1.2d, v8.2d[0]
+	OP_rr	v16.2d, v0.2d, v8.d[0]
+	OP_ii	v16.2d, v1.2d, v9.d[0]
+	OP_ri	v17.2d, v0.2d, v9.d[0]
+	OP_ir	v17.2d, v1.2d, v8.d[0]
 
 	ld2	{v12.2d, v13.2d}, [pB]		// For next round
 	add	pB, pB, #32
 
-	OP_rr	v18.2d, v2.2d, v8.2d[0]
-	OP_ii	v18.2d, v3.2d, v9.2d[0]
-	OP_ri	v19.2d, v2.2d, v9.2d[0]
-	OP_ir	v19.2d, v3.2d, v8.2d[0]
+	OP_rr	v18.2d, v2.2d, v8.d[0]
+	OP_ii	v18.2d, v3.2d, v9.d[0]
+	OP_ri	v19.2d, v2.2d, v9.d[0]
+	OP_ir	v19.2d, v3.2d, v8.d[0]
 
 	ld2	{v14.2d, v15.2d}, [pB]		// For next round
 	add	pB, pB, #32
 
-	OP_rr	v20.2d, v0.2d, v8.2d[1]
-	OP_ii	v20.2d, v1.2d, v9.2d[1]
-	OP_ri	v21.2d, v0.2d, v9.2d[1]
-	OP_ir	v21.2d, v1.2d, v8.2d[1]
+	OP_rr	v20.2d, v0.2d, v8.d[1]
+	OP_ii	v20.2d, v1.2d, v9.d[1]
+	OP_ri	v21.2d, v0.2d, v9.d[1]
+	OP_ir	v21.2d, v1.2d, v8.d[1]
 
 	ld2	{v4.2d, v5.2d} , [pA]		// For next round
 	add	pA, pA, #32
 
-	OP_rr	v22.2d, v2.2d, v8.2d[1]
-	OP_ii	v22.2d, v3.2d, v9.2d[1]
-	OP_ri	v23.2d, v2.2d, v9.2d[1]
-	OP_ir	v23.2d, v3.2d, v8.2d[1]
+	OP_rr	v22.2d, v2.2d, v8.d[1]
+	OP_ii	v22.2d, v3.2d, v9.d[1]
+	OP_ri	v23.2d, v2.2d, v9.d[1]
+	OP_ir	v23.2d, v3.2d, v8.d[1]
 
 	ld2	{v6.2d, v7.2d} , [pA]		// For next round
 	add	pA, pA, #32
 
-	OP_rr	v24.2d, v0.2d, v10.2d[0]
-	OP_ii	v24.2d, v1.2d, v11.2d[0]
-	OP_ri	v25.2d, v0.2d, v11.2d[0]
-	OP_ir	v25.2d, v1.2d, v10.2d[0]
+	OP_rr	v24.2d, v0.2d, v10.d[0]
+	OP_ii	v24.2d, v1.2d, v11.d[0]
+	OP_ri	v25.2d, v0.2d, v11.d[0]
+	OP_ir	v25.2d, v1.2d, v10.d[0]
 
 	prfm	PLDL1KEEP, [pA, #512]
 
-	OP_rr	v26.2d, v2.2d, v10.2d[0]
-	OP_ii	v26.2d, v3.2d, v11.2d[0]
-	OP_ri	v27.2d, v2.2d, v11.2d[0]
-	OP_ir	v27.2d, v3.2d, v10.2d[0]
+	OP_rr	v26.2d, v2.2d, v10.d[0]
+	OP_ii	v26.2d, v3.2d, v11.d[0]
+	OP_ri	v27.2d, v2.2d, v11.d[0]
+	OP_ir	v27.2d, v3.2d, v10.d[0]
 
 	prfm	PLDL1KEEP, [pB, #512]
 
-	OP_rr	v28.2d, v0.2d, v10.2d[1]
-	OP_ii	v28.2d, v1.2d, v11.2d[1]
-	OP_ri	v29.2d, v0.2d, v11.2d[1]
-	OP_ir	v29.2d, v1.2d, v10.2d[1]
+	OP_rr	v28.2d, v0.2d, v10.d[1]
+	OP_ii	v28.2d, v1.2d, v11.d[1]
+	OP_ri	v29.2d, v0.2d, v11.d[1]
+	OP_ir	v29.2d, v1.2d, v10.d[1]
 
-	OP_rr	v30.2d, v2.2d, v10.2d[1]
-	OP_ii	v30.2d, v3.2d, v11.2d[1]
-	OP_ri	v31.2d, v2.2d, v11.2d[1]
-	OP_ir	v31.2d, v3.2d, v10.2d[1]
+	OP_rr	v30.2d, v2.2d, v10.d[1]
+	OP_ii	v30.2d, v3.2d, v11.d[1]
+	OP_ri	v31.2d, v2.2d, v11.d[1]
+	OP_ir	v31.2d, v3.2d, v10.d[1]
 .endm
 
 .macro KERNEL4x4_M2
-	OP_rr	v16.2d, v4.2d, v12.2d[0]
-	OP_ii	v16.2d, v5.2d, v13.2d[0]
-	OP_ri	v17.2d, v4.2d, v13.2d[0]
-	OP_ir	v17.2d, v5.2d, v12.2d[0]
+	OP_rr	v16.2d, v4.2d, v12.d[0]
+	OP_ii	v16.2d, v5.2d, v13.d[0]
+	OP_ri	v17.2d, v4.2d, v13.d[0]
+	OP_ir	v17.2d, v5.2d, v12.d[0]
 
 	ld2	{v8.2d, v9.2d}, [pB]		// For next round
 	add	pB, pB, #32
 
-	OP_rr	v18.2d, v6.2d, v12.2d[0]
-	OP_ii	v18.2d, v7.2d, v13.2d[0]
-	OP_ri	v19.2d, v6.2d, v13.2d[0]
-	OP_ir	v19.2d, v7.2d, v12.2d[0]
+	OP_rr	v18.2d, v6.2d, v12.d[0]
+	OP_ii	v18.2d, v7.2d, v13.d[0]
+	OP_ri	v19.2d, v6.2d, v13.d[0]
+	OP_ir	v19.2d, v7.2d, v12.d[0]
 
 	ld2	{v10.2d, v11.2d}, [pB]		// For next round
 	add	pB, pB, #32
 
-	OP_rr	v20.2d, v4.2d, v12.2d[1]
-	OP_ii	v20.2d, v5.2d, v13.2d[1]
-	OP_ri	v21.2d, v4.2d, v13.2d[1]
-	OP_ir	v21.2d, v5.2d, v12.2d[1]
+	OP_rr	v20.2d, v4.2d, v12.d[1]
+	OP_ii	v20.2d, v5.2d, v13.d[1]
+	OP_ri	v21.2d, v4.2d, v13.d[1]
+	OP_ir	v21.2d, v5.2d, v12.d[1]
 
 	ld2	{v0.2d, v1.2d}, [pA]		// For next round
 	add	pA, pA, #32
 
-	OP_rr	v22.2d, v6.2d, v12.2d[1]
-	OP_ii	v22.2d, v7.2d, v13.2d[1]
-	OP_ri	v23.2d, v6.2d, v13.2d[1]
-	OP_ir	v23.2d, v7.2d, v12.2d[1]
+	OP_rr	v22.2d, v6.2d, v12.d[1]
+	OP_ii	v22.2d, v7.2d, v13.d[1]
+	OP_ri	v23.2d, v6.2d, v13.d[1]
+	OP_ir	v23.2d, v7.2d, v12.d[1]
 
 	ld2	{v2.2d, v3.2d}, [pA]		// For next round
 	add	pA, pA, #32
 
-	OP_rr	v24.2d, v4.2d, v14.2d[0]
-	OP_ii	v24.2d, v5.2d, v15.2d[0]
-	OP_ri	v25.2d, v4.2d, v15.2d[0]
-	OP_ir	v25.2d, v5.2d, v14.2d[0]
+	OP_rr	v24.2d, v4.2d, v14.d[0]
+	OP_ii	v24.2d, v5.2d, v15.d[0]
+	OP_ri	v25.2d, v4.2d, v15.d[0]
+	OP_ir	v25.2d, v5.2d, v14.d[0]
 
 	prfm	PLDL1KEEP, [pA, #512]
 
-	OP_rr	v26.2d, v6.2d, v14.2d[0]
-	OP_ii	v26.2d, v7.2d, v15.2d[0]
-	OP_ri	v27.2d, v6.2d, v15.2d[0]
-	OP_ir	v27.2d, v7.2d, v14.2d[0]
+	OP_rr	v26.2d, v6.2d, v14.d[0]
+	OP_ii	v26.2d, v7.2d, v15.d[0]
+	OP_ri	v27.2d, v6.2d, v15.d[0]
+	OP_ir	v27.2d, v7.2d, v14.d[0]
 
 	prfm	PLDL1KEEP, [pB, #512]
 
-	OP_rr	v28.2d, v4.2d, v14.2d[1]
-	OP_ii	v28.2d, v5.2d, v15.2d[1]
-	OP_ri	v29.2d, v4.2d, v15.2d[1]
-	OP_ir	v29.2d, v5.2d, v14.2d[1]
+	OP_rr	v28.2d, v4.2d, v14.d[1]
+	OP_ii	v28.2d, v5.2d, v15.d[1]
+	OP_ri	v29.2d, v4.2d, v15.d[1]
+	OP_ir	v29.2d, v5.2d, v14.d[1]
 
-	OP_rr	v30.2d, v6.2d, v14.2d[1]
-	OP_ii	v30.2d, v7.2d, v15.2d[1]
-	OP_ri	v31.2d, v6.2d, v15.2d[1]
-	OP_ir	v31.2d, v7.2d, v14.2d[1]
+	OP_rr	v30.2d, v6.2d, v14.d[1]
+	OP_ii	v30.2d, v7.2d, v15.d[1]
+	OP_ri	v31.2d, v6.2d, v15.d[1]
+	OP_ir	v31.2d, v7.2d, v14.d[1]
 .endm
 
 .macro KERNEL4x4_E
-	OP_rr	v16.2d, v4.2d, v12.2d[0]
-	OP_ii	v16.2d, v5.2d, v13.2d[0]
-	OP_ri	v17.2d, v4.2d, v13.2d[0]
-	OP_ir	v17.2d, v5.2d, v12.2d[0]
-
-	OP_rr	v18.2d, v6.2d, v12.2d[0]
-	OP_ii	v18.2d, v7.2d, v13.2d[0]
-	OP_ri	v19.2d, v6.2d, v13.2d[0]
-	OP_ir	v19.2d, v7.2d, v12.2d[0]
-
-	OP_rr	v20.2d, v4.2d, v12.2d[1]
-	OP_ii	v20.2d, v5.2d, v13.2d[1]
-	OP_ri	v21.2d, v4.2d, v13.2d[1]
-	OP_ir	v21.2d, v5.2d, v12.2d[1]
-
-	OP_rr	v22.2d, v6.2d, v12.2d[1]
-	OP_ii	v22.2d, v7.2d, v13.2d[1]
-	OP_ri	v23.2d, v6.2d, v13.2d[1]
-	OP_ir	v23.2d, v7.2d, v12.2d[1]
-
-	OP_rr	v24.2d, v4.2d, v14.2d[0]
-	OP_ii	v24.2d, v5.2d, v15.2d[0]
-	OP_ri	v25.2d, v4.2d, v15.2d[0]
-	OP_ir	v25.2d, v5.2d, v14.2d[0]
-
-	OP_rr	v26.2d, v6.2d, v14.2d[0]
-	OP_ii	v26.2d, v7.2d, v15.2d[0]
-	OP_ri	v27.2d, v6.2d, v15.2d[0]
-	OP_ir	v27.2d, v7.2d, v14.2d[0]
-
-	OP_rr	v28.2d, v4.2d, v14.2d[1]
-	OP_ii	v28.2d, v5.2d, v15.2d[1]
-	OP_ri	v29.2d, v4.2d, v15.2d[1]
-	OP_ir	v29.2d, v5.2d, v14.2d[1]
-
-	OP_rr	v30.2d, v6.2d, v14.2d[1]
-	OP_ii	v30.2d, v7.2d, v15.2d[1]
-	OP_ri	v31.2d, v6.2d, v15.2d[1]
-	OP_ir	v31.2d, v7.2d, v14.2d[1]
+	OP_rr	v16.2d, v4.2d, v12.d[0]
+	OP_ii	v16.2d, v5.2d, v13.d[0]
+	OP_ri	v17.2d, v4.2d, v13.d[0]
+	OP_ir	v17.2d, v5.2d, v12.d[0]
+
+	OP_rr	v18.2d, v6.2d, v12.d[0]
+	OP_ii	v18.2d, v7.2d, v13.d[0]
+	OP_ri	v19.2d, v6.2d, v13.d[0]
+	OP_ir	v19.2d, v7.2d, v12.d[0]
+
+	OP_rr	v20.2d, v4.2d, v12.d[1]
+	OP_ii	v20.2d, v5.2d, v13.d[1]
+	OP_ri	v21.2d, v4.2d, v13.d[1]
+	OP_ir	v21.2d, v5.2d, v12.d[1]
+
+	OP_rr	v22.2d, v6.2d, v12.d[1]
+	OP_ii	v22.2d, v7.2d, v13.d[1]
+	OP_ri	v23.2d, v6.2d, v13.d[1]
+	OP_ir	v23.2d, v7.2d, v12.d[1]
+
+	OP_rr	v24.2d, v4.2d, v14.d[0]
+	OP_ii	v24.2d, v5.2d, v15.d[0]
+	OP_ri	v25.2d, v4.2d, v15.d[0]
+	OP_ir	v25.2d, v5.2d, v14.d[0]
+
+	OP_rr	v26.2d, v6.2d, v14.d[0]
+	OP_ii	v26.2d, v7.2d, v15.d[0]
+	OP_ri	v27.2d, v6.2d, v15.d[0]
+	OP_ir	v27.2d, v7.2d, v14.d[0]
+
+	OP_rr	v28.2d, v4.2d, v14.d[1]
+	OP_ii	v28.2d, v5.2d, v15.d[1]
+	OP_ri	v29.2d, v4.2d, v15.d[1]
+	OP_ir	v29.2d, v5.2d, v14.d[1]
+
+	OP_rr	v30.2d, v6.2d, v14.d[1]
+	OP_ii	v30.2d, v7.2d, v15.d[1]
+	OP_ri	v31.2d, v6.2d, v15.d[1]
+	OP_ir	v31.2d, v7.2d, v14.d[1]
 .endm
 
 .macro KERNEL4x4_SUB
@@ -448,45 +448,45 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld2	{v2.2d, v3.2d}, [pA]
 	add	pA, pA, #32
 
-	OP_rr	v16.2d, v0.2d, v8.2d[0]
-	OP_ii	v16.2d, v1.2d, v9.2d[0]
-	OP_ri	v17.2d, v0.2d, v9.2d[0]
-	OP_ir	v17.2d, v1.2d, v8.2d[0]
-
-	OP_rr	v18.2d, v2.2d, v8.2d[0]
-	OP_ii	v18.2d, v3.2d, v9.2d[0]
-	OP_ri	v19.2d, v2.2d, v9.2d[0]
-	OP_ir	v19.2d, v3.2d, v8.2d[0]
-
-	OP_rr	v20.2d, v0.2d, v8.2d[1]
-	OP_ii	v20.2d, v1.2d, v9.2d[1]
-	OP_ri	v21.2d, v0.2d, v9.2d[1]
-	OP_ir	v21.2d, v1.2d, v8.2d[1]
-
-	OP_rr	v22.2d, v2.2d, v8.2d[1]
-	OP_ii	v22.2d, v3.2d, v9.2d[1]
-	OP_ri	v23.2d, v2.2d, v9.2d[1]
-	OP_ir	v23.2d, v3.2d, v8.2d[1]
-
-	OP_rr	v24.2d, v0.2d, v10.2d[0]
-	OP_ii	v24.2d, v1.2d, v11.2d[0]
-	OP_ri	v25.2d, v0.2d, v11.2d[0]
-	OP_ir	v25.2d, v1.2d, v10.2d[0]
-
-	OP_rr	v26.2d, v2.2d, v10.2d[0]
-	OP_ii	v26.2d, v3.2d, v11.2d[0]
-	OP_ri	v27.2d, v2.2d, v11.2d[0]
-	OP_ir	v27.2d, v3.2d, v10.2d[0]
-
-	OP_rr	v28.2d, v0.2d, v10.2d[1]
-	OP_ii	v28.2d, v1.2d, v11.2d[1]
-	OP_ri	v29.2d, v0.2d, v11.2d[1]
-	OP_ir	v29.2d, v1.2d, v10.2d[1]
-
-	OP_rr	v30.2d, v2.2d, v10.2d[1]
-	OP_ii	v30.2d, v3.2d, v11.2d[1]
-	OP_ri	v31.2d, v2.2d, v11.2d[1]
-	OP_ir	v31.2d, v3.2d, v10.2d[1]
+	OP_rr	v16.2d, v0.2d, v8.d[0]
+	OP_ii	v16.2d, v1.2d, v9.d[0]
+	OP_ri	v17.2d, v0.2d, v9.d[0]
+	OP_ir	v17.2d, v1.2d, v8.d[0]
+
+	OP_rr	v18.2d, v2.2d, v8.d[0]
+	OP_ii	v18.2d, v3.2d, v9.d[0]
+	OP_ri	v19.2d, v2.2d, v9.d[0]
+	OP_ir	v19.2d, v3.2d, v8.d[0]
+
+	OP_rr	v20.2d, v0.2d, v8.d[1]
+	OP_ii	v20.2d, v1.2d, v9.d[1]
+	OP_ri	v21.2d, v0.2d, v9.d[1]
+	OP_ir	v21.2d, v1.2d, v8.d[1]
+
+	OP_rr	v22.2d, v2.2d, v8.d[1]
+	OP_ii	v22.2d, v3.2d, v9.d[1]
+	OP_ri	v23.2d, v2.2d, v9.d[1]
+	OP_ir	v23.2d, v3.2d, v8.d[1]
+
+	OP_rr	v24.2d, v0.2d, v10.d[0]
+	OP_ii	v24.2d, v1.2d, v11.d[0]
+	OP_ri	v25.2d, v0.2d, v11.d[0]
+	OP_ir	v25.2d, v1.2d, v10.d[0]
+
+	OP_rr	v26.2d, v2.2d, v10.d[0]
+	OP_ii	v26.2d, v3.2d, v11.d[0]
+	OP_ri	v27.2d, v2.2d, v11.d[0]
+	OP_ir	v27.2d, v3.2d, v10.d[0]
+
+	OP_rr	v28.2d, v0.2d, v10.d[1]
+	OP_ii	v28.2d, v1.2d, v11.d[1]
+	OP_ri	v29.2d, v0.2d, v11.d[1]
+	OP_ir	v29.2d, v1.2d, v10.d[1]
+
+	OP_rr	v30.2d, v2.2d, v10.d[1]
+	OP_ii	v30.2d, v3.2d, v11.d[1]
+	OP_ri	v31.2d, v2.2d, v11.d[1]
+	OP_ir	v31.2d, v3.2d, v10.d[1]
 .endm
 
 .macro SAVE4x4
@@ -582,25 +582,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld2	{v0.2d, v1.2d}, [pA]
 	add	pA, pA, #32
 
-	OP_rr	v16.2d, v0.2d, v8.2d[0]
-	OP_ii	v16.2d, v1.2d, v9.2d[0]
-	OP_ri	v17.2d, v0.2d, v9.2d[0]
-	OP_ir	v17.2d, v1.2d, v8.2d[0]
-
-	OP_rr	v20.2d, v0.2d, v8.2d[1]
-	OP_ii	v20.2d, v1.2d, v9.2d[1]
-	OP_ri	v21.2d, v0.2d, v9.2d[1]
-	OP_ir	v21.2d, v1.2d, v8.2d[1]
-
-	OP_rr	v24.2d, v0.2d, v10.2d[0]
-	OP_ii	v24.2d, v1.2d, v11.2d[0]
-	OP_ri	v25.2d, v0.2d, v11.2d[0]
-	OP_ir	v25.2d, v1.2d, v10.2d[0]
-
-	OP_rr	v28.2d, v0.2d, v10.2d[1]
-	OP_ii	v28.2d, v1.2d, v11.2d[1]
-	OP_ri	v29.2d, v0.2d, v11.2d[1]
-	OP_ir	v29.2d, v1.2d, v10.2d[1]
+	OP_rr	v16.2d, v0.2d, v8.d[0]
+	OP_ii	v16.2d, v1.2d, v9.d[0]
+	OP_ri	v17.2d, v0.2d, v9.d[0]
+	OP_ir	v17.2d, v1.2d, v8.d[0]
+
+	OP_rr	v20.2d, v0.2d, v8.d[1]
+	OP_ii	v20.2d, v1.2d, v9.d[1]
+	OP_ri	v21.2d, v0.2d, v9.d[1]
+	OP_ir	v21.2d, v1.2d, v8.d[1]
+
+	OP_rr	v24.2d, v0.2d, v10.d[0]
+	OP_ii	v24.2d, v1.2d, v11.d[0]
+	OP_ri	v25.2d, v0.2d, v11.d[0]
+	OP_ir	v25.2d, v1.2d, v10.d[0]
+
+	OP_rr	v28.2d, v0.2d, v10.d[1]
+	OP_ii	v28.2d, v1.2d, v11.d[1]
+	OP_ri	v29.2d, v0.2d, v11.d[1]
+	OP_ir	v29.2d, v1.2d, v10.d[1]
 .endm
 
 .macro SAVE2x4
@@ -669,25 +669,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld2	{v0.d, v1.d}[0], [pA]
 	add	pA, pA, #16
 
-	OP_rr	d16, d0, v8.2d[0]
-	OP_ii	d16, d1, v9.2d[0]
-	OP_ri	d17, d0, v9.2d[0]
-	OP_ir	d17, d1, v8.2d[0]
-
-	OP_rr	d20, d0, v8.2d[1]
-	OP_ii	d20, d1, v9.2d[1]
-	OP_ri	d21, d0, v9.2d[1]
-	OP_ir	d21, d1, v8.2d[1]
-
-	OP_rr	d24, d0, v10.2d[0]
-	OP_ii	d24, d1, v11.2d[0]
-	OP_ri	d25, d0, v11.2d[0]
-	OP_ir	d25, d1, v10.2d[0]
-
-	OP_rr	d28, d0, v10.2d[1]
-	OP_ii	d28, d1, v11.2d[1]
-	OP_ri	d29, d0, v11.2d[1]
-	OP_ir	d29, d1, v10.2d[1]
+	OP_rr	d16, d0, v8.d[0]
+	OP_ii	d16, d1, v9.d[0]
+	OP_ri	d17, d0, v9.d[0]
+	OP_ir	d17, d1, v8.d[0]
+
+	OP_rr	d20, d0, v8.d[1]
+	OP_ii	d20, d1, v9.d[1]
+	OP_ri	d21, d0, v9.d[1]
+	OP_ir	d21, d1, v8.d[1]
+
+	OP_rr	d24, d0, v10.d[0]
+	OP_ii	d24, d1, v11.d[0]
+	OP_ri	d25, d0, v11.d[0]
+	OP_ir	d25, d1, v10.d[0]
+
+	OP_rr	d28, d0, v10.d[1]
+	OP_ii	d28, d1, v11.d[1]
+	OP_ri	d29, d0, v11.d[1]
+	OP_ir	d29, d1, v10.d[1]
 .endm
 
 .macro SAVE1x4
@@ -756,25 +756,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld2	{v2.2d, v3.2d}, [pA]
 	add	pA, pA, #32
 
-	OP_rr	v16.2d, v0.2d, v8.2d[0]
-	OP_ii	v16.2d, v1.2d, v9.2d[0]
-	OP_ri	v17.2d, v0.2d, v9.2d[0]
-	OP_ir	v17.2d, v1.2d, v8.2d[0]
-
-	OP_rr	v18.2d, v2.2d, v8.2d[0]
-	OP_ii	v18.2d, v3.2d, v9.2d[0]
-	OP_ri	v19.2d, v2.2d, v9.2d[0]
-	OP_ir	v19.2d, v3.2d, v8.2d[0]
-
-	OP_rr	v20.2d, v0.2d, v8.2d[1]
-	OP_ii	v20.2d, v1.2d, v9.2d[1]
-	OP_ri	v21.2d, v0.2d, v9.2d[1]
-	OP_ir	v21.2d, v1.2d, v8.2d[1]
-
-	OP_rr	v22.2d, v2.2d, v8.2d[1]
-	OP_ii	v22.2d, v3.2d, v9.2d[1]
-	OP_ri	v23.2d, v2.2d, v9.2d[1]
-	OP_ir	v23.2d, v3.2d, v8.2d[1]
+	OP_rr	v16.2d, v0.2d, v8.d[0]
+	OP_ii	v16.2d, v1.2d, v9.d[0]
+	OP_ri	v17.2d, v0.2d, v9.d[0]
+	OP_ir	v17.2d, v1.2d, v8.d[0]
+
+	OP_rr	v18.2d, v2.2d, v8.d[0]
+	OP_ii	v18.2d, v3.2d, v9.d[0]
+	OP_ri	v19.2d, v2.2d, v9.d[0]
+	OP_ir	v19.2d, v3.2d, v8.d[0]
+
+	OP_rr	v20.2d, v0.2d, v8.d[1]
+	OP_ii	v20.2d, v1.2d, v9.d[1]
+	OP_ri	v21.2d, v0.2d, v9.d[1]
+	OP_ir	v21.2d, v1.2d, v8.d[1]
+
+	OP_rr	v22.2d, v2.2d, v8.d[1]
+	OP_ii	v22.2d, v3.2d, v9.d[1]
+	OP_ri	v23.2d, v2.2d, v9.d[1]
+	OP_ir	v23.2d, v3.2d, v8.d[1]
 .endm
 
 .macro SAVE4x2
@@ -833,15 +833,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld2	{v0.2d, v1.2d}, [pA]
 	add	pA, pA, #32
 
-	OP_rr	v16.2d, v0.2d, v8.2d[0]
-	OP_ii	v16.2d, v1.2d, v9.2d[0]
-	OP_ri	v17.2d, v0.2d, v9.2d[0]
-	OP_ir	v17.2d, v1.2d, v8.2d[0]
+	OP_rr	v16.2d, v0.2d, v8.d[0]
+	OP_ii	v16.2d, v1.2d, v9.d[0]
+	OP_ri	v17.2d, v0.2d, v9.d[0]
+	OP_ir	v17.2d, v1.2d, v8.d[0]
 
-	OP_rr	v20.2d, v0.2d, v8.2d[1]
-	OP_ii	v20.2d, v1.2d, v9.2d[1]
-	OP_ri	v21.2d, v0.2d, v9.2d[1]
-	OP_ir	v21.2d, v1.2d, v8.2d[1]
+	OP_rr	v20.2d, v0.2d, v8.d[1]
+	OP_ii	v20.2d, v1.2d, v9.d[1]
+	OP_ri	v21.2d, v0.2d, v9.d[1]
+	OP_ir	v21.2d, v1.2d, v8.d[1]
 .endm
 
 .macro SAVE2x2
@@ -886,15 +886,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld2	{v0.d, v1.d}[0], [pA]
 	add	pA, pA, #16
 
-	OP_rr	d16, d0, v8.2d[0]
-	OP_ii	d16, d1, v9.2d[0]
-	OP_ri	d17, d0, v9.2d[0]
-	OP_ir	d17, d1, v8.2d[0]
+	OP_rr	d16, d0, v8.d[0]
+	OP_ii	d16, d1, v9.d[0]
+	OP_ri	d17, d0, v9.d[0]
+	OP_ir	d17, d1, v8.d[0]
 
-	OP_rr	d20, d0, v8.2d[1]
-	OP_ii	d20, d1, v9.2d[1]
-	OP_ri	d21, d0, v9.2d[1]
-	OP_ir	d21, d1, v8.2d[1]
+	OP_rr	d20, d0, v8.d[1]
+	OP_ii	d20, d1, v9.d[1]
+	OP_ri	d21, d0, v9.d[1]
+	OP_ir	d21, d1, v8.d[1]
 .endm
 
 .macro SAVE1x2

kernel/arm64/ztrmm_kernel_4x4.S

@@ -185,93 +185,93 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld2	{v2.2d, v3.2d}, [pA]
 	add	pA, pA, #32
 
-	fmul	v16.2d, v0.2d, v8.2d[0]
-	OP_ii	v16.2d, v1.2d, v9.2d[0]
+	fmul	v16.2d, v0.2d, v8.d[0]
+	OP_ii	v16.2d, v1.2d, v9.d[0]
 #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
     defined(RR) || defined(RC) || defined(CR) || defined(CC)
 	eor	v17.16b, v17.16b, v17.16b
-	fmls	v17.2d, v0.2d, v9.2d[0]
+	fmls	v17.2d, v0.2d, v9.d[0]
 #else
-	fmul	v17.2d, v0.2d, v9.2d[0]
+	fmul	v17.2d, v0.2d, v9.d[0]
 #endif
-	OP_ir	v17.2d, v1.2d, v8.2d[0]
+	OP_ir	v17.2d, v1.2d, v8.d[0]
 
-	fmul	v18.2d, v2.2d, v8.2d[0]
-	OP_ii	v18.2d, v3.2d, v9.2d[0]
+	fmul	v18.2d, v2.2d, v8.d[0]
+	OP_ii	v18.2d, v3.2d, v9.d[0]
 #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
     defined(RR) || defined(RC) || defined(CR) || defined(CC)
 	eor	v19.16b, v19.16b, v19.16b
-	fmls	v19.2d, v2.2d, v9.2d[0]
+	fmls	v19.2d, v2.2d, v9.d[0]
 #else
-	fmul	v19.2d, v2.2d, v9.2d[0]
+	fmul	v19.2d, v2.2d, v9.d[0]
 #endif
-	OP_ir	v19.2d, v3.2d, v8.2d[0]
+	OP_ir	v19.2d, v3.2d, v8.d[0]
 
-	fmul	v20.2d, v0.2d, v8.2d[1]
-	OP_ii	v20.2d, v1.2d, v9.2d[1]
+	fmul	v20.2d, v0.2d, v8.d[1]
+	OP_ii	v20.2d, v1.2d, v9.d[1]
 #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
     defined(RR) || defined(RC) || defined(CR) || defined(CC)
 	eor	v21.16b, v21.16b, v21.16b
-	fmls	v21.2d, v0.2d, v9.2d[1]
+	fmls	v21.2d, v0.2d, v9.d[1]
 #else
-	fmul	v21.2d, v0.2d, v9.2d[1]
+	fmul	v21.2d, v0.2d, v9.d[1]
 #endif
-	OP_ir	v21.2d, v1.2d, v8.2d[1]
+	OP_ir	v21.2d, v1.2d, v8.d[1]
 
-	fmul	v22.2d, v2.2d, v8.2d[1]
-	OP_ii	v22.2d, v3.2d, v9.2d[1]
+	fmul	v22.2d, v2.2d, v8.d[1]
+	OP_ii	v22.2d, v3.2d, v9.d[1]
 #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
     defined(RR) || defined(RC) || defined(CR) || defined(CC)
 	eor	v23.16b, v23.16b, v23.16b
-	fmls	v23.2d, v2.2d, v9.2d[1]
+	fmls	v23.2d, v2.2d, v9.d[1]
 #else
-	fmul	v23.2d, v2.2d, v9.2d[1]
+	fmul	v23.2d, v2.2d, v9.d[1]
 #endif
-	OP_ir	v23.2d, v3.2d, v8.2d[1]
+	OP_ir	v23.2d, v3.2d, v8.d[1]
 
-	fmul	v24.2d, v0.2d, v10.2d[0]
-	OP_ii	v24.2d, v1.2d, v11.2d[0]
+	fmul	v24.2d, v0.2d, v10.d[0]
+	OP_ii	v24.2d, v1.2d, v11.d[0]
 #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
     defined(RR) || defined(RC) || defined(CR) || defined(CC)
 	eor	v25.16b, v25.16b, v25.16b
-	fmls	v25.2d, v0.2d, v11.2d[0]
+	fmls	v25.2d, v0.2d, v11.d[0]
 #else
-	fmul	v25.2d, v0.2d, v11.2d[0]
+	fmul	v25.2d, v0.2d, v11.d[0]
 #endif
-	OP_ir	v25.2d, v1.2d, v10.2d[0]
+	OP_ir	v25.2d, v1.2d, v10.d[0]
 
-	fmul	v26.2d, v2.2d, v10.2d[0]
-	OP_ii	v26.2d, v3.2d, v11.2d[0]
+	fmul	v26.2d, v2.2d, v10.d[0]
+	OP_ii	v26.2d, v3.2d, v11.d[0]
 #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
     defined(RR) || defined(RC) || defined(CR) || defined(CC)
 	eor	v27.16b, v27.16b, v27.16b
-	fmls	v27.2d, v2.2d, v11.2d[0]
+	fmls	v27.2d, v2.2d, v11.d[0]
 #else
-	fmul	v27.2d, v2.2d, v11.2d[0]
+	fmul	v27.2d, v2.2d, v11.d[0]
 #endif
-	OP_ir	v27.2d, v3.2d, v10.2d[0]
+	OP_ir	v27.2d, v3.2d, v10.d[0]
 
-	fmul	v28.2d, v0.2d, v10.2d[1]
-	OP_ii	v28.2d, v1.2d, v11.2d[1]
+	fmul	v28.2d, v0.2d, v10.d[1]
+	OP_ii	v28.2d, v1.2d, v11.d[1]
 #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
     defined(RR) || defined(RC) || defined(CR) || defined(CC)
 	eor	v29.16b, v29.16b, v29.16b
-	fmls	v29.2d, v0.2d, v11.2d[1]
+	fmls	v29.2d, v0.2d, v11.d[1]
 #else
-	fmul	v29.2d, v0.2d, v11.2d[1]
+	fmul	v29.2d, v0.2d, v11.d[1]
 #endif
-	OP_ir	v29.2d, v1.2d, v10.2d[1]
+	OP_ir	v29.2d, v1.2d, v10.d[1]
 
-	fmul	v30.2d, v2.2d, v10.2d[1]
-	OP_ii	v30.2d, v3.2d, v11.2d[1]
+	fmul	v30.2d, v2.2d, v10.d[1]
+	OP_ii	v30.2d, v3.2d, v11.d[1]
 #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \
     defined(RR) || defined(RC) || defined(CR) || defined(CC)
 	eor	v31.16b, v31.16b, v31.16b
-	fmls	v31.2d, v2.2d, v11.2d[1]
+	fmls	v31.2d, v2.2d, v11.d[1]
 #else
-	fmul	v31.2d, v2.2d, v11.2d[1]
+	fmul	v31.2d, v2.2d, v11.d[1]
 #endif
-	OP_ir	v31.2d, v3.2d, v10.2d[1]
+	OP_ir	v31.2d, v3.2d, v10.d[1]
 
 	ld2	{v12.2d, v13.2d}, [pB]
 	add	pB, pB, #32
@@ -284,161 +284,161 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .endm
 
 .macro KERNEL4x4_M1
-	OP_rr	v16.2d, v0.2d, v8.2d[0]
-	OP_ii	v16.2d, v1.2d, v9.2d[0]
-	OP_ri	v17.2d, v0.2d, v9.2d[0]
-	OP_ir	v17.2d, v1.2d, v8.2d[0]
+	OP_rr	v16.2d, v0.2d, v8.d[0]
+	OP_ii	v16.2d, v1.2d, v9.d[0]
+	OP_ri	v17.2d, v0.2d, v9.d[0]
+	OP_ir	v17.2d, v1.2d, v8.d[0]
 
 	ld2	{v12.2d, v13.2d}, [pB]		// For next round
 	add	pB, pB, #32
 
-	OP_rr	v18.2d, v2.2d, v8.2d[0]
-	OP_ii	v18.2d, v3.2d, v9.2d[0]
-	OP_ri	v19.2d, v2.2d, v9.2d[0]
-	OP_ir	v19.2d, v3.2d, v8.2d[0]
+	OP_rr	v18.2d, v2.2d, v8.d[0]
+	OP_ii	v18.2d, v3.2d, v9.d[0]
+	OP_ri	v19.2d, v2.2d, v9.d[0]
+	OP_ir	v19.2d, v3.2d, v8.d[0]
 
 	ld2	{v14.2d, v15.2d}, [pB]		// For next round
 	add	pB, pB, #32
 
-	OP_rr	v20.2d, v0.2d, v8.2d[1]
-	OP_ii	v20.2d, v1.2d, v9.2d[1]
-	OP_ri	v21.2d, v0.2d, v9.2d[1]
-	OP_ir	v21.2d, v1.2d, v8.2d[1]
+	OP_rr	v20.2d, v0.2d, v8.d[1]
+	OP_ii	v20.2d, v1.2d, v9.d[1]
+	OP_ri	v21.2d, v0.2d, v9.d[1]
+	OP_ir	v21.2d, v1.2d, v8.d[1]
 
 	ld2	{v4.2d, v5.2d} , [pA]		// For next round
 	add	pA, pA, #32
 
-	OP_rr	v22.2d, v2.2d, v8.2d[1]
-	OP_ii	v22.2d, v3.2d, v9.2d[1]
-	OP_ri	v23.2d, v2.2d, v9.2d[1]
-	OP_ir	v23.2d, v3.2d, v8.2d[1]
+	OP_rr	v22.2d, v2.2d, v8.d[1]
+	OP_ii	v22.2d, v3.2d, v9.d[1]
+	OP_ri	v23.2d, v2.2d, v9.d[1]
+	OP_ir	v23.2d, v3.2d, v8.d[1]
 
 	ld2	{v6.2d, v7.2d} , [pA]		// For next round
 	add	pA, pA, #32
 
-	OP_rr	v24.2d, v0.2d, v10.2d[0]
-	OP_ii	v24.2d, v1.2d, v11.2d[0]
-	OP_ri	v25.2d, v0.2d, v11.2d[0]
-	OP_ir	v25.2d, v1.2d, v10.2d[0]
+	OP_rr	v24.2d, v0.2d, v10.d[0]
+	OP_ii	v24.2d, v1.2d, v11.d[0]
+	OP_ri	v25.2d, v0.2d, v11.d[0]
+	OP_ir	v25.2d, v1.2d, v10.d[0]
 
 	prfm	PLDL1KEEP, [pA, #512]
 
-	OP_rr	v26.2d, v2.2d, v10.2d[0]
-	OP_ii	v26.2d, v3.2d, v11.2d[0]
-	OP_ri	v27.2d, v2.2d, v11.2d[0]
-	OP_ir	v27.2d, v3.2d, v10.2d[0]
+	OP_rr	v26.2d, v2.2d, v10.d[0]
+	OP_ii	v26.2d, v3.2d, v11.d[0]
+	OP_ri	v27.2d, v2.2d, v11.d[0]
+	OP_ir	v27.2d, v3.2d, v10.d[0]
 
 	prfm	PLDL1KEEP, [pB, #512]
 
-	OP_rr	v28.2d, v0.2d, v10.2d[1]
-	OP_ii	v28.2d, v1.2d, v11.2d[1]
-	OP_ri	v29.2d, v0.2d, v11.2d[1]
-	OP_ir	v29.2d, v1.2d, v10.2d[1]
+	OP_rr	v28.2d, v0.2d, v10.d[1]
+	OP_ii	v28.2d, v1.2d, v11.d[1]
+	OP_ri	v29.2d, v0.2d, v11.d[1]
+	OP_ir	v29.2d, v1.2d, v10.d[1]
 
-	OP_rr	v30.2d, v2.2d, v10.2d[1]
-	OP_ii	v30.2d, v3.2d, v11.2d[1]
-	OP_ri	v31.2d, v2.2d, v11.2d[1]
-	OP_ir	v31.2d, v3.2d, v10.2d[1]
+	OP_rr	v30.2d, v2.2d, v10.d[1]
+	OP_ii	v30.2d, v3.2d, v11.d[1]
+	OP_ri	v31.2d, v2.2d, v11.d[1]
+	OP_ir	v31.2d, v3.2d, v10.d[1]
 .endm
 
 .macro KERNEL4x4_M2
-	OP_rr	v16.2d, v4.2d, v12.2d[0]
-	OP_ii	v16.2d, v5.2d, v13.2d[0]
-	OP_ri	v17.2d, v4.2d, v13.2d[0]
-	OP_ir	v17.2d, v5.2d, v12.2d[0]
+	OP_rr	v16.2d, v4.2d, v12.d[0]
+	OP_ii	v16.2d, v5.2d, v13.d[0]
+	OP_ri	v17.2d, v4.2d, v13.d[0]
+	OP_ir	v17.2d, v5.2d, v12.d[0]
 
 	ld2	{v8.2d, v9.2d}, [pB]		// For next round
 	add	pB, pB, #32
 
-	OP_rr	v18.2d, v6.2d, v12.2d[0]
-	OP_ii	v18.2d, v7.2d, v13.2d[0]
-	OP_ri	v19.2d, v6.2d, v13.2d[0]
-	OP_ir	v19.2d, v7.2d, v12.2d[0]
+	OP_rr	v18.2d, v6.2d, v12.d[0]
+	OP_ii	v18.2d, v7.2d, v13.d[0]
+	OP_ri	v19.2d, v6.2d, v13.d[0]
+	OP_ir	v19.2d, v7.2d, v12.d[0]
 
 	ld2	{v10.2d, v11.2d}, [pB]		// For next round
 	add	pB, pB, #32
 
-	OP_rr	v20.2d, v4.2d, v12.2d[1]
-	OP_ii	v20.2d, v5.2d, v13.2d[1]
-	OP_ri	v21.2d, v4.2d, v13.2d[1]
-	OP_ir	v21.2d, v5.2d, v12.2d[1]
+	OP_rr	v20.2d, v4.2d, v12.d[1]
+	OP_ii	v20.2d, v5.2d, v13.d[1]
+	OP_ri	v21.2d, v4.2d, v13.d[1]
+	OP_ir	v21.2d, v5.2d, v12.d[1]
 
 	ld2	{v0.2d, v1.2d}, [pA]		// For next round
 	add	pA, pA, #32
 
-	OP_rr	v22.2d, v6.2d, v12.2d[1]
-	OP_ii	v22.2d, v7.2d, v13.2d[1]
-	OP_ri	v23.2d, v6.2d, v13.2d[1]
-	OP_ir	v23.2d, v7.2d, v12.2d[1]
+	OP_rr	v22.2d, v6.2d, v12.d[1]
+	OP_ii	v22.2d, v7.2d, v13.d[1]
+	OP_ri	v23.2d, v6.2d, v13.d[1]
+	OP_ir	v23.2d, v7.2d, v12.d[1]
 
 	ld2	{v2.2d, v3.2d}, [pA]		// For next round
 	add	pA, pA, #32
 
-	OP_rr	v24.2d, v4.2d, v14.2d[0]
-	OP_ii	v24.2d, v5.2d, v15.2d[0]
-	OP_ri	v25.2d, v4.2d, v15.2d[0]
-	OP_ir	v25.2d, v5.2d, v14.2d[0]
+	OP_rr	v24.2d, v4.2d, v14.d[0]
+	OP_ii	v24.2d, v5.2d, v15.d[0]
+	OP_ri	v25.2d, v4.2d, v15.d[0]
+	OP_ir	v25.2d, v5.2d, v14.d[0]
 
 	prfm	PLDL1KEEP, [pA, #512]
 
-	OP_rr	v26.2d, v6.2d, v14.2d[0]
-	OP_ii	v26.2d, v7.2d, v15.2d[0]
-	OP_ri	v27.2d, v6.2d, v15.2d[0]
-	OP_ir	v27.2d, v7.2d, v14.2d[0]
+	OP_rr	v26.2d, v6.2d, v14.d[0]
+	OP_ii	v26.2d, v7.2d, v15.d[0]
+	OP_ri	v27.2d, v6.2d, v15.d[0]
+	OP_ir	v27.2d, v7.2d, v14.d[0]
 
 	prfm	PLDL1KEEP, [pB, #512]
 
-	OP_rr	v28.2d, v4.2d, v14.2d[1]
-	OP_ii	v28.2d, v5.2d, v15.2d[1]
-	OP_ri	v29.2d, v4.2d, v15.2d[1]
-	OP_ir	v29.2d, v5.2d, v14.2d[1]
+	OP_rr	v28.2d, v4.2d, v14.d[1]
+	OP_ii	v28.2d, v5.2d, v15.d[1]
+	OP_ri	v29.2d, v4.2d, v15.d[1]
+	OP_ir	v29.2d, v5.2d, v14.d[1]
 
-	OP_rr	v30.2d, v6.2d, v14.2d[1]
-	OP_ii	v30.2d, v7.2d, v15.2d[1]
-	OP_ri	v31.2d, v6.2d, v15.2d[1]
-	OP_ir	v31.2d, v7.2d, v14.2d[1]
+	OP_rr	v30.2d, v6.2d, v14.d[1]
+	OP_ii	v30.2d, v7.2d, v15.d[1]
+	OP_ri	v31.2d, v6.2d, v15.d[1]
+	OP_ir	v31.2d, v7.2d, v14.d[1]
 .endm
 
 .macro KERNEL4x4_E
-	OP_rr	v16.2d, v4.2d, v12.2d[0]
-	OP_ii	v16.2d, v5.2d, v13.2d[0]
-	OP_ri	v17.2d, v4.2d, v13.2d[0]
-	OP_ir	v17.2d, v5.2d, v12.2d[0]
-
-	OP_rr	v18.2d, v6.2d, v12.2d[0]
-	OP_ii	v18.2d, v7.2d, v13.2d[0]
-	OP_ri	v19.2d, v6.2d, v13.2d[0]
-	OP_ir	v19.2d, v7.2d, v12.2d[0]
-
-	OP_rr	v20.2d, v4.2d, v12.2d[1]
-	OP_ii	v20.2d, v5.2d, v13.2d[1]
-	OP_ri	v21.2d, v4.2d, v13.2d[1]
-	OP_ir	v21.2d, v5.2d, v12.2d[1]
-
-	OP_rr	v22.2d, v6.2d, v12.2d[1]
-	OP_ii	v22.2d, v7.2d, v13.2d[1]
-	OP_ri	v23.2d, v6.2d, v13.2d[1]
-	OP_ir	v23.2d, v7.2d, v12.2d[1]
-
-	OP_rr	v24.2d, v4.2d, v14.2d[0]
-	OP_ii	v24.2d, v5.2d, v15.2d[0]
-	OP_ri	v25.2d, v4.2d, v15.2d[0]
-	OP_ir	v25.2d, v5.2d, v14.2d[0]
-
-	OP_rr	v26.2d, v6.2d, v14.2d[0]
-	OP_ii	v26.2d, v7.2d, v15.2d[0]
-	OP_ri	v27.2d, v6.2d, v15.2d[0]
-	OP_ir	v27.2d, v7.2d, v14.2d[0]
-
-	OP_rr	v28.2d, v4.2d, v14.2d[1]
-	OP_ii	v28.2d, v5.2d, v15.2d[1]
-	OP_ri	v29.2d, v4.2d, v15.2d[1]
-	OP_ir	v29.2d, v5.2d, v14.2d[1]
-
-	OP_rr	v30.2d, v6.2d, v14.2d[1]
-	OP_ii	v30.2d, v7.2d, v15.2d[1]
-	OP_ri	v31.2d, v6.2d, v15.2d[1]
-	OP_ir	v31.2d, v7.2d, v14.2d[1]
+	OP_rr	v16.2d, v4.2d, v12.d[0]
+	OP_ii	v16.2d, v5.2d, v13.d[0]
+	OP_ri	v17.2d, v4.2d, v13.d[0]
+	OP_ir	v17.2d, v5.2d, v12.d[0]
+
+	OP_rr	v18.2d, v6.2d, v12.d[0]
+	OP_ii	v18.2d, v7.2d, v13.d[0]
+	OP_ri	v19.2d, v6.2d, v13.d[0]
+	OP_ir	v19.2d, v7.2d, v12.d[0]
+
+	OP_rr	v20.2d, v4.2d, v12.d[1]
+	OP_ii	v20.2d, v5.2d, v13.d[1]
+	OP_ri	v21.2d, v4.2d, v13.d[1]
+	OP_ir	v21.2d, v5.2d, v12.d[1]
+
+	OP_rr	v22.2d, v6.2d, v12.d[1]
+	OP_ii	v22.2d, v7.2d, v13.d[1]
+	OP_ri	v23.2d, v6.2d, v13.d[1]
+	OP_ir	v23.2d, v7.2d, v12.d[1]
+
+	OP_rr	v24.2d, v4.2d, v14.d[0]
+	OP_ii	v24.2d, v5.2d, v15.d[0]
+	OP_ri	v25.2d, v4.2d, v15.d[0]
+	OP_ir	v25.2d, v5.2d, v14.d[0]
+
+	OP_rr	v26.2d, v6.2d, v14.d[0]
+	OP_ii	v26.2d, v7.2d, v15.d[0]
+	OP_ri	v27.2d, v6.2d, v15.d[0]
+	OP_ir	v27.2d, v7.2d, v14.d[0]
+
+	OP_rr	v28.2d, v4.2d, v14.d[1]
+	OP_ii	v28.2d, v5.2d, v15.d[1]
+	OP_ri	v29.2d, v4.2d, v15.d[1]
+	OP_ir	v29.2d, v5.2d, v14.d[1]
+
+	OP_rr	v30.2d, v6.2d, v14.d[1]
+	OP_ii	v30.2d, v7.2d, v15.d[1]
+	OP_ri	v31.2d, v6.2d, v15.d[1]
+	OP_ir	v31.2d, v7.2d, v14.d[1]
 .endm
 
 .macro KERNEL4x4_SUB
@@ -451,45 +451,45 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld2	{v2.2d, v3.2d}, [pA]
 	add	pA, pA, #32
 
-	OP_rr	v16.2d, v0.2d, v8.2d[0]
-	OP_ii	v16.2d, v1.2d, v9.2d[0]
-	OP_ri	v17.2d, v0.2d, v9.2d[0]
-	OP_ir	v17.2d, v1.2d, v8.2d[0]
-
-	OP_rr	v18.2d, v2.2d, v8.2d[0]
-	OP_ii	v18.2d, v3.2d, v9.2d[0]
-	OP_ri	v19.2d, v2.2d, v9.2d[0]
-	OP_ir	v19.2d, v3.2d, v8.2d[0]
-
-	OP_rr	v20.2d, v0.2d, v8.2d[1]
-	OP_ii	v20.2d, v1.2d, v9.2d[1]
-	OP_ri	v21.2d, v0.2d, v9.2d[1]
-	OP_ir	v21.2d, v1.2d, v8.2d[1]
-
-	OP_rr	v22.2d, v2.2d, v8.2d[1]
-	OP_ii	v22.2d, v3.2d, v9.2d[1]
-	OP_ri	v23.2d, v2.2d, v9.2d[1]
-	OP_ir	v23.2d, v3.2d, v8.2d[1]
-
-	OP_rr	v24.2d, v0.2d, v10.2d[0]
-	OP_ii	v24.2d, v1.2d, v11.2d[0]
-	OP_ri	v25.2d, v0.2d, v11.2d[0]
-	OP_ir	v25.2d, v1.2d, v10.2d[0]
-
-	OP_rr	v26.2d, v2.2d, v10.2d[0]
-	OP_ii	v26.2d, v3.2d, v11.2d[0]
-	OP_ri	v27.2d, v2.2d, v11.2d[0]
-	OP_ir	v27.2d, v3.2d, v10.2d[0]
-
-	OP_rr	v28.2d, v0.2d, v10.2d[1]
-	OP_ii	v28.2d, v1.2d, v11.2d[1]
-	OP_ri	v29.2d, v0.2d, v11.2d[1]
-	OP_ir	v29.2d, v1.2d, v10.2d[1]
-
-	OP_rr	v30.2d, v2.2d, v10.2d[1]
-	OP_ii	v30.2d, v3.2d, v11.2d[1]
-	OP_ri	v31.2d, v2.2d, v11.2d[1]
-	OP_ir	v31.2d, v3.2d, v10.2d[1]
+	OP_rr	v16.2d, v0.2d, v8.d[0]
+	OP_ii	v16.2d, v1.2d, v9.d[0]
+	OP_ri	v17.2d, v0.2d, v9.d[0]
+	OP_ir	v17.2d, v1.2d, v8.d[0]
+
+	OP_rr	v18.2d, v2.2d, v8.d[0]
+	OP_ii	v18.2d, v3.2d, v9.d[0]
+	OP_ri	v19.2d, v2.2d, v9.d[0]
+	OP_ir	v19.2d, v3.2d, v8.d[0]
+
+	OP_rr	v20.2d, v0.2d, v8.d[1]
+	OP_ii	v20.2d, v1.2d, v9.d[1]
+	OP_ri	v21.2d, v0.2d, v9.d[1]
+	OP_ir	v21.2d, v1.2d, v8.d[1]
+
+	OP_rr	v22.2d, v2.2d, v8.d[1]
+	OP_ii	v22.2d, v3.2d, v9.d[1]
+	OP_ri	v23.2d, v2.2d, v9.d[1]
+	OP_ir	v23.2d, v3.2d, v8.d[1]
+
+	OP_rr	v24.2d, v0.2d, v10.d[0]
+	OP_ii	v24.2d, v1.2d, v11.d[0]
+	OP_ri	v25.2d, v0.2d, v11.d[0]
+	OP_ir	v25.2d, v1.2d, v10.d[0]
+
+	OP_rr	v26.2d, v2.2d, v10.d[0]
+	OP_ii	v26.2d, v3.2d, v11.d[0]
+	OP_ri	v27.2d, v2.2d, v11.d[0]
+	OP_ir	v27.2d, v3.2d, v10.d[0]
+
+	OP_rr	v28.2d, v0.2d, v10.d[1]
+	OP_ii	v28.2d, v1.2d, v11.d[1]
+	OP_ri	v29.2d, v0.2d, v11.d[1]
+	OP_ir	v29.2d, v1.2d, v10.d[1]
+
+	OP_rr	v30.2d, v2.2d, v10.d[1]
+	OP_ii	v30.2d, v3.2d, v11.d[1]
+	OP_ri	v31.2d, v2.2d, v11.d[1]
+	OP_ir	v31.2d, v3.2d, v10.d[1]
 .endm
 
 .macro SAVE4x4
@@ -577,25 +577,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld2	{v0.2d, v1.2d}, [pA]
 	add	pA, pA, #32
 
-	OP_rr	v16.2d, v0.2d, v8.2d[0]
-	OP_ii	v16.2d, v1.2d, v9.2d[0]
-	OP_ri	v17.2d, v0.2d, v9.2d[0]
-	OP_ir	v17.2d, v1.2d, v8.2d[0]
-
-	OP_rr	v20.2d, v0.2d, v8.2d[1]
-	OP_ii	v20.2d, v1.2d, v9.2d[1]
-	OP_ri	v21.2d, v0.2d, v9.2d[1]
-	OP_ir	v21.2d, v1.2d, v8.2d[1]
-
-	OP_rr	v24.2d, v0.2d, v10.2d[0]
-	OP_ii	v24.2d, v1.2d, v11.2d[0]
-	OP_ri	v25.2d, v0.2d, v11.2d[0]
-	OP_ir	v25.2d, v1.2d, v10.2d[0]
-
-	OP_rr	v28.2d, v0.2d, v10.2d[1]
-	OP_ii	v28.2d, v1.2d, v11.2d[1]
-	OP_ri	v29.2d, v0.2d, v11.2d[1]
-	OP_ir	v29.2d, v1.2d, v10.2d[1]
+	OP_rr	v16.2d, v0.2d, v8.d[0]
+	OP_ii	v16.2d, v1.2d, v9.d[0]
+	OP_ri	v17.2d, v0.2d, v9.d[0]
+	OP_ir	v17.2d, v1.2d, v8.d[0]
+
+	OP_rr	v20.2d, v0.2d, v8.d[1]
+	OP_ii	v20.2d, v1.2d, v9.d[1]
+	OP_ri	v21.2d, v0.2d, v9.d[1]
+	OP_ir	v21.2d, v1.2d, v8.d[1]
+
+	OP_rr	v24.2d, v0.2d, v10.d[0]
+	OP_ii	v24.2d, v1.2d, v11.d[0]
+	OP_ri	v25.2d, v0.2d, v11.d[0]
+	OP_ir	v25.2d, v1.2d, v10.d[0]
+
+	OP_rr	v28.2d, v0.2d, v10.d[1]
+	OP_ii	v28.2d, v1.2d, v11.d[1]
+	OP_ri	v29.2d, v0.2d, v11.d[1]
+	OP_ir	v29.2d, v1.2d, v10.d[1]
 .endm
 
 .macro SAVE2x4
@@ -660,25 +660,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld2	{v0.d, v1.d}[0], [pA]
 	add	pA, pA, #16
 
-	OP_rr	d16, d0, v8.2d[0]
-	OP_ii	d16, d1, v9.2d[0]
-	OP_ri	d17, d0, v9.2d[0]
-	OP_ir	d17, d1, v8.2d[0]
-
-	OP_rr	d20, d0, v8.2d[1]
-	OP_ii	d20, d1, v9.2d[1]
-	OP_ri	d21, d0, v9.2d[1]
-	OP_ir	d21, d1, v8.2d[1]
-
-	OP_rr	d24, d0, v10.2d[0]
-	OP_ii	d24, d1, v11.2d[0]
-	OP_ri	d25, d0, v11.2d[0]
-	OP_ir	d25, d1, v10.2d[0]
-
-	OP_rr	d28, d0, v10.2d[1]
-	OP_ii	d28, d1, v11.2d[1]
-	OP_ri	d29, d0, v11.2d[1]
-	OP_ir	d29, d1, v10.2d[1]
+	OP_rr	d16, d0, v8.d[0]
+	OP_ii	d16, d1, v9.d[0]
+	OP_ri	d17, d0, v9.d[0]
+	OP_ir	d17, d1, v8.d[0]
+
+	OP_rr	d20, d0, v8.d[1]
+	OP_ii	d20, d1, v9.d[1]
+	OP_ri	d21, d0, v9.d[1]
+	OP_ir	d21, d1, v8.d[1]
+
+	OP_rr	d24, d0, v10.d[0]
+	OP_ii	d24, d1, v11.d[0]
+	OP_ri	d25, d0, v11.d[0]
+	OP_ir	d25, d1, v10.d[0]
+
+	OP_rr	d28, d0, v10.d[1]
+	OP_ii	d28, d1, v11.d[1]
+	OP_ri	d29, d0, v11.d[1]
+	OP_ir	d29, d1, v10.d[1]
 .endm
 
 .macro SAVE1x4
@@ -743,25 +743,25 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld2	{v2.2d, v3.2d}, [pA]
 	add	pA, pA, #32
 
-	OP_rr	v16.2d, v0.2d, v8.2d[0]
-	OP_ii	v16.2d, v1.2d, v9.2d[0]
-	OP_ri	v17.2d, v0.2d, v9.2d[0]
-	OP_ir	v17.2d, v1.2d, v8.2d[0]
-
-	OP_rr	v18.2d, v2.2d, v8.2d[0]
-	OP_ii	v18.2d, v3.2d, v9.2d[0]
-	OP_ri	v19.2d, v2.2d, v9.2d[0]
-	OP_ir	v19.2d, v3.2d, v8.2d[0]
-
-	OP_rr	v20.2d, v0.2d, v8.2d[1]
-	OP_ii	v20.2d, v1.2d, v9.2d[1]
-	OP_ri	v21.2d, v0.2d, v9.2d[1]
-	OP_ir	v21.2d, v1.2d, v8.2d[1]
-
-	OP_rr	v22.2d, v2.2d, v8.2d[1]
-	OP_ii	v22.2d, v3.2d, v9.2d[1]
-	OP_ri	v23.2d, v2.2d, v9.2d[1]
-	OP_ir	v23.2d, v3.2d, v8.2d[1]
+	OP_rr	v16.2d, v0.2d, v8.d[0]
+	OP_ii	v16.2d, v1.2d, v9.d[0]
+	OP_ri	v17.2d, v0.2d, v9.d[0]
+	OP_ir	v17.2d, v1.2d, v8.d[0]
+
+	OP_rr	v18.2d, v2.2d, v8.d[0]
+	OP_ii	v18.2d, v3.2d, v9.d[0]
+	OP_ri	v19.2d, v2.2d, v9.d[0]
+	OP_ir	v19.2d, v3.2d, v8.d[0]
+
+	OP_rr	v20.2d, v0.2d, v8.d[1]
+	OP_ii	v20.2d, v1.2d, v9.d[1]
+	OP_ri	v21.2d, v0.2d, v9.d[1]
+	OP_ir	v21.2d, v1.2d, v8.d[1]
+
+	OP_rr	v22.2d, v2.2d, v8.d[1]
+	OP_ii	v22.2d, v3.2d, v9.d[1]
+	OP_ri	v23.2d, v2.2d, v9.d[1]
+	OP_ir	v23.2d, v3.2d, v8.d[1]
 .endm
 
 .macro SAVE4x2
@@ -816,15 +816,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld2	{v0.2d, v1.2d}, [pA]
 	add	pA, pA, #32
 
-	OP_rr	v16.2d, v0.2d, v8.2d[0]
-	OP_ii	v16.2d, v1.2d, v9.2d[0]
-	OP_ri	v17.2d, v0.2d, v9.2d[0]
-	OP_ir	v17.2d, v1.2d, v8.2d[0]
+	OP_rr	v16.2d, v0.2d, v8.d[0]
+	OP_ii	v16.2d, v1.2d, v9.d[0]
+	OP_ri	v17.2d, v0.2d, v9.d[0]
+	OP_ir	v17.2d, v1.2d, v8.d[0]
 
-	OP_rr	v20.2d, v0.2d, v8.2d[1]
-	OP_ii	v20.2d, v1.2d, v9.2d[1]
-	OP_ri	v21.2d, v0.2d, v9.2d[1]
-	OP_ir	v21.2d, v1.2d, v8.2d[1]
+	OP_rr	v20.2d, v0.2d, v8.d[1]
+	OP_ii	v20.2d, v1.2d, v9.d[1]
+	OP_ri	v21.2d, v0.2d, v9.d[1]
+	OP_ir	v21.2d, v1.2d, v8.d[1]
 .endm
 
 .macro SAVE2x2
@@ -867,15 +867,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld2	{v0.d, v1.d}[0], [pA]
 	add	pA, pA, #16
 
-	OP_rr	d16, d0, v8.2d[0]
-	OP_ii	d16, d1, v9.2d[0]
-	OP_ri	d17, d0, v9.2d[0]
-	OP_ir	d17, d1, v8.2d[0]
+	OP_rr	d16, d0, v8.d[0]
+	OP_ii	d16, d1, v9.d[0]
+	OP_ri	d17, d0, v9.d[0]
+	OP_ir	d17, d1, v8.d[0]
 
-	OP_rr	d20, d0, v8.2d[1]
-	OP_ii	d20, d1, v9.2d[1]
-	OP_ri	d21, d0, v9.2d[1]
-	OP_ir	d21, d1, v8.2d[1]
+	OP_rr	d20, d0, v8.d[1]
+	OP_ii	d20, d1, v9.d[1]
+	OP_ri	d21, d0, v9.d[1]
+	OP_ir	d21, d1, v8.d[1]
 .endm
 
 .macro SAVE1x2

kernel/power/KERNEL.POWER8

@@ -3,14 +3,18 @@
 #CGEMM_BETA = ../generic/zgemm_beta.c
 #ZGEMM_BETA = ../generic/zgemm_beta.c
 
-STRMMKERNEL	= gemm_kernel_power6.S
+STRMMKERNEL	= strmm_kernel_16x8_power8.S
 DTRMMKERNEL	= dtrmm_kernel_16x4_power8.S
-CTRMMKERNEL	= ../generic/ztrmmkernel_2x2.c
+CTRMMKERNEL	= ctrmm_kernel_8x4_power8.S
 ZTRMMKERNEL	= ztrmm_kernel_8x2_power8.S
 
-SGEMMKERNEL    =  gemm_kernel_power6.S
-SGEMMONCOPY    =  ../generic/gemm_ncopy_4.c
-SGEMMOTCOPY    =  ../generic/gemm_tcopy_4.c
+SGEMMKERNEL    =  sgemm_kernel_16x8_power8.S
+SGEMMINCOPY    = ../generic/gemm_ncopy_16.c
+SGEMMITCOPY    = ../generic/gemm_tcopy_16.c
+SGEMMONCOPY    =  ../generic/gemm_ncopy_8.c
+SGEMMOTCOPY    =  ../generic/gemm_tcopy_8.c
+SGEMMINCOPYOBJ =  sgemm_incopy.o
+SGEMMITCOPYOBJ =  sgemm_itcopy.o
 SGEMMONCOPYOBJ =  sgemm_oncopy.o
 SGEMMOTCOPYOBJ =  sgemm_otcopy.o
 
@@ -24,11 +28,15 @@ DGEMMITCOPYOBJ = dgemm_itcopy.o
 DGEMMONCOPYOBJ = dgemm_oncopy.o
 DGEMMOTCOPYOBJ = dgemm_otcopy.o
 
-CGEMMKERNEL    = ../generic/zgemmkernel_2x2.c
-CGEMMONCOPY    = ../generic/zgemm_ncopy_2.c
-CGEMMOTCOPY    = ../generic/zgemm_tcopy_2.c
+CGEMMKERNEL    = cgemm_kernel_8x4_power8.S
+CGEMMINCOPY    = ../generic/zgemm_ncopy_8.c
+CGEMMITCOPY    = ../generic/zgemm_tcopy_8.c
+CGEMMONCOPY    = ../generic/zgemm_ncopy_4.c
+CGEMMOTCOPY    = ../generic/zgemm_tcopy_4.c
 CGEMMONCOPYOBJ =  cgemm_oncopy.o
 CGEMMOTCOPYOBJ =  cgemm_otcopy.o
+CGEMMINCOPYOBJ =  cgemm_incopy.o
+CGEMMITCOPYOBJ =  cgemm_itcopy.o
 
 ZGEMMKERNEL    = zgemm_kernel_8x2_power8.S
 ZGEMMONCOPY    = ../generic/zgemm_ncopy_2.c
@@ -97,56 +105,56 @@ ZTRSMKERNEL_RT	= ../generic/trsm_kernel_RT.c
 #ISMINKERNEL  = ../arm/imin.c
 #IDMINKERNEL  = ../arm/imin.c
 #
-#SASUMKERNEL  = ../arm/asum.c
-#DASUMKERNEL  = ../arm/asum.c
-#CASUMKERNEL  = ../arm/zasum.c
-#ZASUMKERNEL  = ../arm/zasum.c
+SASUMKERNEL  = sasum.c
+DASUMKERNEL  = dasum.c
+CASUMKERNEL  = casum.c
+ZASUMKERNEL  = zasum.c
 #
 #SAXPYKERNEL  = ../arm/axpy.c
-#DAXPYKERNEL  = ../arm/axpy.c
+DAXPYKERNEL  = daxpy.c
 #CAXPYKERNEL  = ../arm/zaxpy.c
-#ZAXPYKERNEL  = ../arm/zaxpy.c
+ZAXPYKERNEL  = zaxpy.c
 #
-#SCOPYKERNEL  = ../arm/copy.c
-#DCOPYKERNEL  = ../arm/copy.c
-#CCOPYKERNEL  = ../arm/zcopy.c
-#ZCOPYKERNEL  = ../arm/zcopy.c
+SCOPYKERNEL  = scopy.c
+DCOPYKERNEL  = dcopy.c
+CCOPYKERNEL  = ccopy.c
+ZCOPYKERNEL  = zcopy.c
 #
-#SDOTKERNEL   = ../arm/dot.c
-#DDOTKERNEL   = ../arm/dot.c
+SDOTKERNEL   =  sdot.c
+DDOTKERNEL   =  ddot.c
 #CDOTKERNEL   = ../arm/zdot.c
-#ZDOTKERNEL   = ../arm/zdot.c
+ZDOTKERNEL   =  zdot.c
 #
 #SNRM2KERNEL  = ../arm/nrm2.c
 #DNRM2KERNEL  = ../arm/nrm2.c
 #CNRM2KERNEL  = ../arm/znrm2.c
 #ZNRM2KERNEL  = ../arm/znrm2.c
 #
-#SROTKERNEL   = ../arm/rot.c
-#DROTKERNEL   = ../arm/rot.c
+SROTKERNEL   = srot.c
+DROTKERNEL   = drot.c
 #CROTKERNEL   = ../arm/zrot.c
 #ZROTKERNEL   = ../arm/zrot.c
 #
-#SSCALKERNEL  = ../arm/scal.c
-#DSCALKERNEL  = ../arm/scal.c
+SSCALKERNEL  = sscal.c
+DSCALKERNEL  = dscal.c
 #CSCALKERNEL  = ../arm/zscal.c
-#ZSCALKERNEL  = ../arm/zscal.c
+ZSCALKERNEL  = zscal.c
 #
-#SSWAPKERNEL  = ../arm/swap.c
-#DSWAPKERNEL  = ../arm/swap.c
-#CSWAPKERNEL  = ../arm/zswap.c
-#ZSWAPKERNEL  = ../arm/zswap.c
+SSWAPKERNEL  = sswap.c
+DSWAPKERNEL  = dswap.c
+CSWAPKERNEL  = cswap.c
+ZSWAPKERNEL  = zswap.c
 #
 
 #SGEMVNKERNEL = ../arm/gemv_n.c
-#DGEMVNKERNEL = ../arm/gemv_n.c
+DGEMVNKERNEL = dgemv_n.c
 #CGEMVNKERNEL = ../arm/zgemv_n.c
 #ZGEMVNKERNEL = ../arm/zgemv_n.c
 #
 #SGEMVTKERNEL = ../arm/gemv_t.c
 #DGEMVTKERNEL = ../arm/gemv_t.c
 #CGEMVTKERNEL = ../arm/zgemv_t.c
-#ZGEMVTKERNEL = ../arm/zgemv_t.c
+#ZGEMVTKERNEL = zgemv_t_4.c
 
 
 #SSYMV_U_KERNEL =  ../generic/symv_k.c

kernel/power/casum.c

@@ -0,0 +1,151 @@
+/***************************************************************************
+Copyright (c) 2013-2016, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+/**************************************************************************************
+* 2016/03/28 Werner Saar (wernsaar@googlemail.com)
+* 	 BLASTEST 		: OK
+* 	 CTEST			: OK
+* 	 TEST			: OK
+*	 LAPACK-TEST		: OK
+**************************************************************************************/
+
+#include "common.h"
+#include <math.h>
+
+#if defined(DOUBLE)
+
+#define ABS fabs
+
+#else
+
+#define ABS fabsf
+
+#endif
+
+#if defined(POWER8)
+#include "casum_microk_power8.c"
+#endif
+
+
+#ifndef HAVE_KERNEL_16
+
+static void casum_kernel_16(BLASLONG n, FLOAT *x1, FLOAT *svec)
+{
+
+	BLASLONG i=0;
+	FLOAT *x = x1;
+	FLOAT temp0, temp1, temp2, temp3;
+	FLOAT temp4, temp5, temp6, temp7;
+	FLOAT sum0 = 0.0;
+	FLOAT sum1 = 0.0;
+	FLOAT sum2 = 0.0;
+	FLOAT sum3 = 0.0;
+
+	while ( i< n )
+	{
+
+		temp0 = ABS(x[0]);
+		temp1 = ABS(x[1]);
+		temp2 = ABS(x[2]);
+		temp3 = ABS(x[3]);
+		temp4 = ABS(x[4]);
+		temp5 = ABS(x[5]);
+		temp6 = ABS(x[6]);
+		temp7 = ABS(x[7]);
+
+		sum0 += temp0;
+		sum1 += temp1;
+		sum2 += temp2;
+		sum3 += temp3;
+
+		sum0 += temp4;
+		sum1 += temp5;
+		sum2 += temp6;
+		sum3 += temp7;
+
+		x+=8;
+		i+=4;
+
+	}
+
+	svec[0] = sum0+sum1+sum2+sum3;
+	svec[1] = 0.0;
+	svec[2] = 0.0;
+	svec[3] = 0.0;
+
+}
+
+#endif
+
+FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x)
+{
+	BLASLONG i=0;
+	BLASLONG ip=0;
+	FLOAT sumf = 0.0;
+	FLOAT svec[4] __attribute__ ((aligned (16)));;
+	BLASLONG n1;
+	BLASLONG inc_x2;
+
+	if (n <= 0 || inc_x <= 0) return(sumf);
+
+	if ( inc_x == 1 )
+	{
+
+		n1 = n & -16;
+		if ( n1 > 0 )
+		{
+
+			casum_kernel_16(n1, x, svec);
+			sumf = svec[0] + svec[1]+svec[2]+svec[3];
+			i=n1;
+			ip = 2 * n1;
+		}
+
+		while(i < n)
+		{
+			sumf += ABS(x[ip]) + ABS(x[ip+1]);
+			ip += 2;
+			i++;
+		}
+
+	}
+	else
+	{
+		inc_x2 = 2 * inc_x;
+
+		while(i < n)
+		{
+			sumf += ABS(x[ip]) + ABS(x[ip+1]);
+			ip += inc_x2;
+			i++;
+		}
+
+	}
+	return(sumf);
+}
+
+

kernel/power/casum_microk_power8.c

@@ -0,0 +1,177 @@
+/***************************************************************************
+Copyright (c) 2013-2016, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+/**************************************************************************************
+* 2016/03/28 Werner Saar (wernsaar@googlemail.com)
+* 	 BLASTEST 		: OK
+* 	 CTEST			: OK
+* 	 TEST			: OK
+*	 LAPACK-TEST		: OK
+**************************************************************************************/
+
+#define HAVE_KERNEL_16 1
+static void casum_kernel_16( BLASLONG n, FLOAT *x, FLOAT *svec) __attribute__ ((noinline));
+
+static void casum_kernel_16( BLASLONG n, FLOAT *x, FLOAT *svec)
+{
+
+
+	BLASLONG i = n;
+	BLASLONG o16 = 16;
+	BLASLONG o32 = 32;
+	BLASLONG o48 = 48;
+	BLASLONG o64 = 64;
+	BLASLONG o80 = 80;
+	BLASLONG o96 = 96;
+	BLASLONG o112 = 112;
+	FLOAT *x1=x;
+	BLASLONG pre = 384;
+
+	__asm__  __volatile__
+	(
+
+	"dcbt		%2 , %4				    \n\t"
+
+	"xxlxor		32,32,32			    \n\t"
+	"xxlxor		33,33,33			    \n\t"
+	"xxlxor		34,34,34			    \n\t"
+	"xxlxor		35,35,35			    \n\t"
+	"xxlxor		36,36,36			    \n\t"
+	"xxlxor		37,37,37			    \n\t"
+	"xxlxor		38,38,38			    \n\t"
+	"xxlxor		39,39,39			    \n\t"
+
+	"lxvw4x		40, 0, %2			    \n\t"
+	"lxvw4x		41, %5, %2			    \n\t"
+	"lxvw4x		42, %6, %2			    \n\t"
+	"lxvw4x		43, %7, %2			    \n\t"
+	"lxvw4x		44, %8, %2			    \n\t"
+	"lxvw4x		45, %9, %2			    \n\t"
+	"lxvw4x		46, %10, %2			    \n\t"
+	"lxvw4x		47, %11, %2			    \n\t"
+
+	"addi		%2, %2, 128			    \n\t"
+
+	"addic.		%0 , %0	, -16  	 	             \n\t"
+	"ble		2f		             	     \n\t"
+
+	".align 5				            \n\t"
+	"1:				                    \n\t"
+
+	"dcbt		%2 , %4				    \n\t"
+
+	"xvabssp	48, 40				    \n\t"
+	"xvabssp	49, 41				    \n\t"
+	"xvabssp	50, 42				    \n\t"
+	"xvabssp	51, 43				    \n\t"
+
+	"lxvw4x		40, 0, %2			    \n\t"
+	"lxvw4x		41, %5, %2			    \n\t"
+
+	"xvabssp	52, 44				    \n\t"
+	"xvabssp	53, 45				    \n\t"
+
+	"lxvw4x		42, %6, %2			    \n\t"
+	"lxvw4x		43, %7, %2			    \n\t"
+
+	"xvabssp	54, 46				    \n\t"
+	"xvabssp	55, 47				    \n\t"
+
+	"lxvw4x		44, %8, %2			    \n\t"
+	"lxvw4x		45, %9, %2			    \n\t"
+
+	"xvaddsp	32, 32, 48		    \n\t"
+	"xvaddsp	33, 33, 49		    \n\t"
+
+	"lxvw4x		46, %10, %2			    \n\t"
+	"lxvw4x		47, %11, %2			    \n\t"
+
+	"xvaddsp	34, 34, 50		    \n\t"
+	"xvaddsp	35, 35, 51		    \n\t"
+	"addi		%2, %2, 128			    \n\t"
+	"xvaddsp	36, 36, 52		    \n\t"
+	"xvaddsp	37, 37, 53		    \n\t"
+	"addic.		%0 , %0	, -16  	 	             \n\t"
+	"xvaddsp	38, 38, 54		    \n\t"
+	"xvaddsp	39, 39, 55		    \n\t"
+
+	"bgt		1b		             	     \n\t"
+
+	"2:						     \n\t"
+
+
+	"xvabssp	48, 40				    \n\t"
+	"xvabssp	49, 41				    \n\t"
+	"xvabssp	50, 42				    \n\t"
+	"xvabssp	51, 43				    \n\t"
+	"xvabssp	52, 44				    \n\t"
+	"xvabssp	53, 45				    \n\t"
+	"xvabssp	54, 46				    \n\t"
+	"xvabssp	55, 47				    \n\t"
+
+	"xvaddsp	32, 32, 48		    \n\t"
+	"xvaddsp	33, 33, 49		    \n\t"
+	"xvaddsp	34, 34, 50		    \n\t"
+	"xvaddsp	35, 35, 51		    \n\t"
+	"xvaddsp	36, 36, 52		    \n\t"
+	"xvaddsp	37, 37, 53		    \n\t"
+	"xvaddsp	38, 38, 54		    \n\t"
+	"xvaddsp	39, 39, 55		    \n\t"
+
+	"xvaddsp	32, 32, 33		     \n\t"
+	"xvaddsp	34, 34, 35		     \n\t"
+	"xvaddsp	36, 36, 37		     \n\t"
+	"xvaddsp	38, 38, 39		     \n\t"
+
+	"xvaddsp	32, 32, 34		     \n\t"
+	"xvaddsp	36, 36, 38		     \n\t"
+
+	"xvaddsp	32, 32, 36		     \n\t"
+
+
+	"stxvw4x	32, 0, %3		     \n\t"
+
+	:
+        : 
+          "r" (i),	// 0	
+	  "r" (n),  	// 1
+          "r" (x1),     // 2
+          "r" (svec),   // 3
+          "r" (pre),    // 4
+	  "r" (o16),	// 5
+	  "r" (o32),	// 6
+	  "r" (o48),    // 7
+          "r" (o64),    // 8
+          "r" (o80),    // 9
+          "r" (o96),    // 10
+          "r" (o112)   // 11
+	: "cr0", "%0", "%2",  "memory"
+	);
+
+} 
+
+

kernel/power/ccopy.c

@@ -0,0 +1,140 @@
+/***************************************************************************
+Copyright (c) 2013-2016, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+/**************************************************************************************
+* 2016/03/25 Werner Saar (wernsaar@googlemail.com)
+* 	 BLASTEST 		: OK
+* 	 CTEST			: OK
+* 	 TEST			: OK
+*	 LAPACK-TEST		: OK
+**************************************************************************************/
+
+#include "common.h"
+
+#if defined(POWER8)
+#include "ccopy_microk_power8.c"
+#endif
+
+#ifndef HAVE_KERNEL_32
+
+static void ccopy_kernel_32(BLASLONG n, FLOAT *x, FLOAT *y)
+{
+
+	BLASLONG i=0;
+	FLOAT f0, f1, f2, f3, f4, f5, f6, f7;
+	FLOAT *x1=x;
+	FLOAT *y1=y;
+
+	while ( i<n )
+	{
+
+		f0 = x1[0];
+		f1 = x1[1];
+		f2 = x1[2];
+		f3 = x1[3];
+		f4 = x1[4];
+		f5 = x1[5];
+		f6 = x1[6];
+		f7 = x1[7];
+
+		y1[0] = f0;
+		y1[1] = f1;
+		y1[2] = f2;
+		y1[3] = f3;
+		y1[4] = f4;
+		y1[5] = f5;
+		y1[6] = f6;
+		y1[7] = f7;
+
+		x1 += 8;
+		y1 += 8;
+
+		i+=4;
+	}
+	return;
+
+}
+
+
+#endif
+
+
+
+int CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y)
+{
+	BLASLONG i=0;
+	BLASLONG ix=0,iy=0;
+
+	if ( n <= 0     )  return(0);
+
+	if ( (inc_x == 1) && (inc_y == 1 ))
+	{
+
+		BLASLONG n1 = n & -32;
+		if ( n1 > 0 )
+		{
+			ccopy_kernel_32(n1, x, y);
+			i=n1;
+			ix=n1*2;
+			iy=n1*2;
+		}
+
+		while(i < n)
+		{
+			y[iy] = x[iy] ;
+			y[iy+1] = x[ix+1] ;
+			ix+=2;
+			iy+=2;
+			i++ ;
+
+		}
+
+
+	}
+	else
+	{
+
+		BLASLONG inc_x2 = 2 * inc_x;
+		BLASLONG inc_y2 = 2 * inc_y;
+
+		while(i < n)
+		{
+			y[iy] = x[ix] ;
+			y[iy+1] = x[ix+1] ;
+			ix += inc_x2 ;
+			iy += inc_y2 ;
+			i++ ;
+
+		}
+
+	}
+	return(0);
+	
+
+}
+
+

kernel/power/ccopy_microk_power8.c

@@ -0,0 +1,174 @@
+/***************************************************************************
+Copyright (c) 2013-2016, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+/**************************************************************************************
+* 2016/03/25 Werner Saar (wernsaar@googlemail.com)
+* 	 BLASTEST 		: OK
+* 	 CTEST			: OK
+* 	 TEST			: OK
+*	 LAPACK-TEST		: OK
+**************************************************************************************/
+
+#define HAVE_KERNEL_32 1
+
+static void ccopy_kernel_32( BLASLONG n, FLOAT *x, FLOAT *y) __attribute__ ((noinline));
+
+static void ccopy_kernel_32( BLASLONG n, FLOAT *x, FLOAT *y)
+{
+
+
+	BLASLONG i = n;
+	BLASLONG o16 = 16;
+	BLASLONG o32 = 32;
+	BLASLONG o48 = 48;
+	BLASLONG o64 = 64;
+	BLASLONG o80 = 80;
+	BLASLONG o96 = 96;
+	BLASLONG o112 = 112;
+	FLOAT *x1=x;
+	FLOAT *y1=y;
+	BLASLONG pre = 384;
+	BLASLONG alpha=0;
+
+	__asm__  __volatile__
+	(
+
+	"lxvw4x		40, 0, %2			    \n\t"
+	"lxvw4x		41, %5, %2			    \n\t"
+	"lxvw4x		42, %6, %2			    \n\t"
+	"lxvw4x		43, %7, %2			    \n\t"
+	"lxvw4x		44, %8, %2			    \n\t"
+	"lxvw4x		45, %9, %2			    \n\t"
+	"lxvw4x		46, %10, %2			    \n\t"
+	"lxvw4x		47, %11, %2			    \n\t"
+
+	"addi		%2, %2, 128			    \n\t"
+
+	"lxvw4x		50, 0, %2			    \n\t"
+	"lxvw4x		51, %5, %2			    \n\t"
+	"lxvw4x		52, %6, %2			    \n\t"
+	"lxvw4x		53, %7, %2			    \n\t"
+	"lxvw4x		54, %8, %2			    \n\t"
+	"lxvw4x		55, %9, %2			    \n\t"
+	"lxvw4x		56, %10, %2			    \n\t"
+	"lxvw4x		57, %11, %2			    \n\t"
+
+	"addi		%2, %2, 128			    \n\t"
+
+	"addic.		%0 , %0	, -32  	 	             \n\t"
+	"ble		2f		             	     \n\t"
+
+	".align 5				            \n\t"
+	"1:				                    \n\t"
+
+	"stxvw4x		40, 0, %1			    \n\t"
+	"stxvw4x		41, %5, %1			    \n\t"
+	"lxvw4x		40, 0, %2			    \n\t"
+	"lxvw4x		41, %5, %2			    \n\t"
+	"stxvw4x		42, %6, %1			    \n\t"
+	"stxvw4x		43, %7, %1			    \n\t"
+	"lxvw4x		42, %6, %2			    \n\t"
+	"lxvw4x		43, %7, %2			    \n\t"
+	"stxvw4x		44, %8, %1			    \n\t"
+	"stxvw4x		45, %9, %1			    \n\t"
+	"lxvw4x		44, %8, %2			    \n\t"
+	"lxvw4x		45, %9, %2			    \n\t"
+	"stxvw4x		46, %10, %1			    \n\t"
+	"stxvw4x		47, %11, %1			    \n\t"
+	"lxvw4x		46, %10, %2			    \n\t"
+	"lxvw4x		47, %11, %2			    \n\t"
+
+
+	"addi		%1, %1, 128			    \n\t"
+	"addi		%2, %2, 128			    \n\t"
+
+	"stxvw4x		50, 0, %1			    \n\t"
+	"stxvw4x		51, %5, %1			    \n\t"
+	"lxvw4x		50, 0, %2			    \n\t"
+	"lxvw4x		51, %5, %2			    \n\t"
+	"stxvw4x		52, %6, %1			    \n\t"
+	"stxvw4x		53, %7, %1			    \n\t"
+	"lxvw4x		52, %6, %2			    \n\t"
+	"lxvw4x		53, %7, %2			    \n\t"
+	"stxvw4x		54, %8, %1			    \n\t"
+	"stxvw4x		55, %9, %1			    \n\t"
+	"lxvw4x		54, %8, %2			    \n\t"
+	"lxvw4x		55, %9, %2			    \n\t"
+	"stxvw4x		56, %10, %1			    \n\t"
+	"stxvw4x		57, %11, %1			    \n\t"
+	"lxvw4x		56, %10, %2			    \n\t"
+	"lxvw4x		57, %11, %2			    \n\t"
+
+	"addi		%1, %1, 128			    \n\t"
+	"addi		%2, %2, 128			    \n\t"
+
+	"addic.		%0 , %0	, -32  	 	             \n\t"
+	"bgt		1b		             	     \n\t"
+
+	"2:						     \n\t"
+
+	"stxvw4x		40, 0, %1			    \n\t"
+	"stxvw4x		41, %5, %1			    \n\t"
+	"stxvw4x		42, %6, %1			    \n\t"
+	"stxvw4x		43, %7, %1			    \n\t"
+	"stxvw4x		44, %8, %1			    \n\t"
+	"stxvw4x		45, %9, %1			    \n\t"
+	"stxvw4x		46, %10, %1			    \n\t"
+	"stxvw4x		47, %11, %1			    \n\t"
+
+	"addi		%1, %1, 128			    \n\t"
+
+	"stxvw4x		50, 0, %1			    \n\t"
+	"stxvw4x		51, %5, %1			    \n\t"
+	"stxvw4x		52, %6, %1			    \n\t"
+	"stxvw4x		53, %7, %1			    \n\t"
+	"stxvw4x		54, %8, %1			    \n\t"
+	"stxvw4x		55, %9, %1			    \n\t"
+	"stxvw4x		56, %10, %1			    \n\t"
+	"stxvw4x		57, %11, %1			    \n\t"
+
+
+	:
+        : 
+          "r" (i),	// 0	
+	  "r" (y1),  	// 1
+          "r" (x1),     // 2
+          "r" (alpha),  // 3
+          "r" (pre),    // 4
+	  "r" (o16),	// 5
+	  "r" (o32),	// 6
+	  "r" (o48),    // 7
+          "r" (o64),    // 8
+          "r" (o80),    // 9
+          "r" (o96),    // 10
+          "r" (o112)    // 11
+	: "cr0", "%0", "%2" , "%1", "memory"
+	);
+
+} 
+
+

kernel/power/cgemm_kernel_8x4_power8.S

@@ -0,0 +1,407 @@
+/***************************************************************************
+Copyright (c) 2013-2016, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+/**************************************************************************************
+* 2016/04/04 Werner Saar (wernsaar@googlemail.com)
+* 	 BLASTEST 		: OK
+* 	 CTEST			: OK
+* 	 TEST			: OK
+* 	 LAPACK-TEST		: OK
+**************************************************************************************/
+
+/*********************************************************************/
+/* Copyright 2009, 2010 The University of Texas at Austin.           */
+/* All rights reserved.                                              */
+/*                                                                   */
+/* Redistribution and use in source and binary forms, with or        */
+/* without modification, are permitted provided that the following   */
+/* conditions are met:                                               */
+/*                                                                   */
+/*   1. Redistributions of source code must retain the above         */
+/*      copyright notice, this list of conditions and the following  */
+/*      disclaimer.                                                  */
+/*                                                                   */
+/*   2. Redistributions in binary form must reproduce the above      */
+/*      copyright notice, this list of conditions and the following  */
+/*      disclaimer in the documentation and/or other materials       */
+/*      provided with the distribution.                              */
+/*                                                                   */
+/*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */
+/*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */
+/*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */
+/*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */
+/*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */
+/*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */
+/*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */
+/*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */
+/*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */
+/*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */
+/*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */
+/*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */
+/*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */
+/*    POSSIBILITY OF SUCH DAMAGE.                                    */
+/*                                                                   */
+/* The views and conclusions contained in the software and           */
+/* documentation are those of the authors and should not be          */
+/* interpreted as representing official policies, either expressed   */
+/* or implied, of The University of Texas at Austin.                 */
+/*********************************************************************/
+
+#define ASSEMBLER
+#include "common.h"
+#include "def_vsx.h"
+
+#ifndef __64BIT__
+#define LOAD	lwz
+#else
+#define LOAD	ld
+#endif
+
+#ifdef __64BIT__
+#define STACKSIZE 32000
+#define ALPHA_R_SP 296(SP)
+#define ALPHA_I_SP 304(SP)
+#define FZERO	312(SP)
+#else
+#define STACKSIZE 256
+#define ALPHA_R_SP 224(SP)
+#define ALPHA_I_SP 232(SP)
+#define FZERO	240(SP)
+#endif
+
+#define	M	r3
+#define	N	r4
+#define	K	r5
+
+#ifdef linux
+#ifndef __64BIT__
+#define A	r6
+#define	B	r7
+#define	C	r8
+#define	LDC	r9
+#define OFFSET	r10
+#else
+#define A	r8
+#define	B	r9
+#define	C	r10
+#define	LDC	r6
+#define OFFSET	r7
+#endif
+#endif
+
+#if defined(_AIX) || defined(__APPLE__)
+#if !defined(__64BIT__) && defined(DOUBLE)
+#define A	r10
+#define	B	r6
+#define	C	r7
+#define	LDC	r8
+#define OFFSET	r9
+#else
+#define A	r8
+#define	B	r9
+#define	C	r10
+#define	LDC	r6
+#define OFFSET	r7
+#endif
+#endif
+
+#define o0	0
+
+#define alpha_dr vs28
+#define alpha_di vs29
+#define alpha_sr vs30
+#define alpha_si vs31
+
+#define FRAMEPOINTER r12
+
+#define BBUFFER	r14
+#define L	r15
+#define o12	r16
+#define o4	r17
+#define T2	r19
+#define BBO	r20
+#define	o8	r21
+#define	I	r22
+#define J	r23
+#define AO	r24
+#define	BO	r25
+#define	CO	r26
+#define o16	r27
+#define	o32	r28
+#define o48	r29
+
+#define PRE	r30
+#define T1  	r31
+
+#ifndef NEEDPARAM
+
+	PROLOGUE
+	PROFCODE
+
+	mr      FRAMEPOINTER, SP
+	addi	SP, SP, -STACKSIZE
+	addi	SP, SP, -STACKSIZE
+	addi	SP, SP, -STACKSIZE
+	addi	SP, SP, -STACKSIZE
+	li	r0, 0
+
+	stfd	f14,    0(SP)
+	stfd	f15,    8(SP)
+	stfd	f16,   16(SP)
+	stfd	f17,   24(SP)
+
+	stfd	f18,   32(SP)
+	stfd	f19,   40(SP)
+	stfd	f20,   48(SP)
+	stfd	f21,   56(SP)
+
+	stfd	f22,   64(SP)
+	stfd	f23,   72(SP)
+	stfd	f24,   80(SP)
+	stfd	f25,   88(SP)
+
+	stfd	f26,   96(SP)
+	stfd	f27,  104(SP)
+	stfd	f28,  112(SP)
+	stfd	f29,  120(SP)
+
+	stfd	f30,  128(SP)
+	stfd	f31,  136(SP)
+
+#ifdef __64BIT__
+	std	r31,  144(SP)
+	std	r30,  152(SP)
+	std	r29,  160(SP)
+	std	r28,  168(SP)
+	std	r27,  176(SP)
+	std	r26,  184(SP)
+	std	r25,  192(SP)
+	std	r24,  200(SP)
+	std	r23,  208(SP)
+	std	r22,  216(SP)
+	std	r21,  224(SP)
+	std	r20,  232(SP)
+	std	r19,  240(SP)
+	std	r18,  248(SP)
+	std	r17,  256(SP)
+	std	r16,  264(SP)
+	std	r15,  272(SP)
+	std	r14,  280(SP)
+#else
+	stw	r31,  144(SP)
+	stw	r30,  148(SP)
+	stw	r29,  152(SP)
+	stw	r28,  156(SP)
+	stw	r27,  160(SP)
+	stw	r26,  164(SP)
+	stw	r25,  168(SP)
+	stw	r24,  172(SP)
+	stw	r23,  176(SP)
+	stw	r22,  180(SP)
+	stw	r21,  184(SP)
+	stw	r20,  188(SP)
+	stw	r19,  192(SP)
+	stw	r18,  196(SP)
+	stw	r17,  200(SP)
+	stw	r16,  204(SP)
+	stw	r15,  208(SP)
+	stw	r14,  212(SP)
+#endif
+
+	stfs	f1,  ALPHA_R_SP
+	stfs	f2,  ALPHA_I_SP
+	// stw	r0,  FZERO
+
+#ifdef linux
+#ifdef __64BIT__
+	ld	LDC, FRAMESLOT(0) + 0(FRAMEPOINTER)
+#endif
+#endif
+
+#if defined(_AIX) || defined(__APPLE__)
+#ifdef __64BIT__
+	ld	LDC, FRAMESLOT(0) + 0(FRAMEPOINTER)
+#else
+#ifdef DOUBLE
+	lwz	B,   FRAMESLOT(0) + 0(FRAMEPOINTER)
+	lwz	C,   FRAMESLOT(1) + 0(FRAMEPOINTER)
+	lwz	LDC, FRAMESLOT(2) + 0(FRAMEPOINTER)
+#else
+	lwz	LDC, FRAMESLOT(0) + 0(FRAMEPOINTER)
+#endif
+#endif
+#endif
+
+#ifdef TRMMKERNEL
+#if defined(linux) && defined(__64BIT__)
+	ld	OFFSET,  FRAMESLOT(1) + 0(FRAMEPOINTER)
+#endif
+
+#if defined(_AIX) || defined(__APPLE__)
+#ifdef __64BIT__
+	ld	OFFSET,  FRAMESLOT(1) + 0(FRAMEPOINTER)
+#else
+#ifdef DOUBLE
+	lwz	OFFSET,  FRAMESLOT(3) + 0(FRAMEPOINTER)
+#else
+	lwz	OFFSET,  FRAMESLOT(1) + 0(FRAMEPOINTER)
+#endif
+#endif
+#endif
+#if defined(TRMMKERNEL) && !defined(LEFT)
+	neg	KK, OFFSET
+#endif
+#endif
+
+#include "cgemm_macros_8x4_power8.S"
+
+	cmpwi	cr0, M, 0
+	ble	L999_H1
+	cmpwi	cr0, N, 0
+	ble	L999_H1
+	cmpwi	cr0, K, 0
+	ble	L999_H1
+
+	slwi	LDC, LDC, ZBASE_SHIFT
+	li	PRE, 384 
+	li	o4  , 4
+	li	o8  , 8
+	li	o12 , 12
+	li	o16 , 16
+	li	o32 , 32
+	li	o48 , 48
+	
+	addi    BBUFFER, SP, 512+4096
+        li      T1, -4096
+        and     BBUFFER, BBUFFER, T1
+
+
+#ifdef __64BIT__
+	addi	T1 , SP, 296
+#else
+	addi	T1 , SP, 224
+#endif
+
+	stxsspx vs1,  0, T1
+        lxsspx  alpha_dr, 0, T1
+	stxsspx vs2,  o8  , T1
+        lxsspx  alpha_di, o8, T1
+        addi    T1, SP, 360
+        li      T2, 0
+
+        stw             T2, 0(T1)
+        stw             T2, 4(T1)
+        stw             T2, 8(T1)
+        stxsspx         alpha_dr, o12, T1
+        lxvw4x          alpha_sr, o0 , T1
+        addi            T1, T1, 16
+
+        stw             T2, 0(T1)
+        stw             T2, 4(T1)
+        stw             T2, 8(T1)
+        stxsspx         alpha_di, o12, T1
+        lxvw4x          alpha_si, o0 , T1
+
+	.align 5
+
+#include "cgemm_logic_8x4_power8.S"
+
+L999:
+	addi	r3, 0, 0
+
+	lfd	f14,    0(SP)
+	lfd	f15,    8(SP)
+	lfd	f16,   16(SP)
+	lfd	f17,   24(SP)
+
+	lfd	f18,   32(SP)
+	lfd	f19,   40(SP)
+	lfd	f20,   48(SP)
+	lfd	f21,   56(SP)
+
+	lfd	f22,   64(SP)
+	lfd	f23,   72(SP)
+	lfd	f24,   80(SP)
+	lfd	f25,   88(SP)
+
+	lfd	f26,   96(SP)
+	lfd	f27,  104(SP)
+	lfd	f28,  112(SP)
+	lfd	f29,  120(SP)
+
+	lfd	f30,  128(SP)
+	lfd	f31,  136(SP)
+
+#ifdef __64BIT__
+	ld	r31,  144(SP)
+	ld	r30,  152(SP)
+	ld	r29,  160(SP)
+	ld	r28,  168(SP)
+	ld	r27,  176(SP)
+	ld	r26,  184(SP)
+	ld	r25,  192(SP)
+	ld	r24,  200(SP)
+	ld	r23,  208(SP)
+	ld	r22,  216(SP)
+	ld	r21,  224(SP)
+	ld	r20,  232(SP)
+	ld	r19,  240(SP)
+	ld	r18,  248(SP)
+	ld	r17,  256(SP)
+	ld	r16,  264(SP)
+	ld	r15,  272(SP)
+	ld	r14,  280(SP)
+#else
+	lwz	r31,  144(SP)
+	lwz	r30,  148(SP)
+	lwz	r29,  152(SP)
+	lwz	r28,  156(SP)
+	lwz	r27,  160(SP)
+	lwz	r26,  164(SP)
+	lwz	r25,  168(SP)
+	lwz	r24,  172(SP)
+	lwz	r23,  176(SP)
+	lwz	r22,  180(SP)
+	lwz	r21,  184(SP)
+	lwz	r20,  188(SP)
+	lwz	r19,  192(SP)
+	lwz	r18,  196(SP)
+	lwz	r17,  200(SP)
+	lwz	r16,  204(SP)
+	lwz	r15,  208(SP)
+	lwz	r14,  212(SP)
+#endif
+
+	addi	SP, SP, STACKSIZE
+	addi	SP, SP, STACKSIZE
+	addi	SP, SP, STACKSIZE
+	addi	SP, SP, STACKSIZE
+
+	blr
+
+	EPILOGUE
+#endif

kernel/power/cswap.c

@@ -0,0 +1,175 @@
+/***************************************************************************
+Copyright (c) 2013-2016, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+/**************************************************************************************
+* 2016/03/27 Werner Saar (wernsaar@googlemail.com)
+* 	 BLASTEST 		: OK
+* 	 CTEST			: OK
+* 	 TEST			: OK
+*	 LAPACK-TEST		: OK
+**************************************************************************************/
+
+#include "common.h"
+
+
+#if defined(POWER8)
+#include "cswap_microk_power8.c"
+#endif
+
+
+#ifndef HAVE_KERNEL_32
+
+static void cswap_kernel_32(BLASLONG n, FLOAT *x, FLOAT *y)
+{
+
+	BLASLONG i=0;
+	FLOAT f0, f1, f2, f3, f4, f5, f6, f7;
+	FLOAT g0, g1, g2, g3, g4, g5, g6, g7;
+	FLOAT *x1=x;
+	FLOAT *y1=y;
+
+	while ( i<n )
+	{
+
+		f0 = x1[0];
+		f1 = x1[1];
+		f2 = x1[2];
+		f3 = x1[3];
+		f4 = x1[4];
+		f5 = x1[5];
+		f6 = x1[6];
+		f7 = x1[7];
+
+		g0 = y1[0];
+		g1 = y1[1];
+		g2 = y1[2];
+		g3 = y1[3];
+		g4 = y1[4];
+		g5 = y1[5];
+		g6 = y1[6];
+		g7 = y1[7];
+
+		y1[0] = f0;
+		y1[1] = f1;
+		y1[2] = f2;
+		y1[3] = f3;
+		y1[4] = f4;
+		y1[5] = f5;
+		y1[6] = f6;
+		y1[7] = f7;
+
+		x1[0] = g0;
+		x1[1] = g1;
+		x1[2] = g2;
+		x1[3] = g3;
+		x1[4] = g4;
+		x1[5] = g5;
+		x1[6] = g6;
+		x1[7] = g7;
+
+		x1 += 8;
+		y1 += 8;
+
+		i+=4;
+	}
+	return;
+
+}
+
+
+#endif
+
+
+int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT dummy3, FLOAT dummy4, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y, FLOAT *dummy, BLASLONG dummy2)
+{
+	BLASLONG i=0;
+	BLASLONG ix=0,iy=0;
+	FLOAT temp[2];
+	BLASLONG inc_x2, inc_y2;
+
+	if ( n <= 0     )  return(0);
+
+	if ( (inc_x == 1) && (inc_y == 1 ))
+	{
+
+		BLASLONG n1 = n & -32;
+		if ( n1 > 0 )
+		{
+			cswap_kernel_32(n1, x, y);
+			i=n1;
+			ix = 2* n1;
+			iy = 2* n1;
+		}
+
+		while(i < n)
+		{
+
+	                temp[0]  = x[ix]   ;
+        	        temp[1]  = x[ix+1] ;
+                	x[ix]    = y[iy]   ;
+                	x[ix+1]  = y[iy+1] ;
+                	y[iy]    = temp[0] ;
+                	y[iy+1]  = temp[1] ;
+
+                	ix += 2 ;
+                	iy += 2 ;
+                	i++ ;
+
+
+		}
+
+
+	}
+	else
+	{
+
+	        inc_x2 = 2 * inc_x;
+	        inc_y2 = 2 * inc_y;
+
+		while(i < n)
+		{
+
+	                temp[0]  = x[ix]   ;
+        	        temp[1]  = x[ix+1] ;
+                	x[ix]    = y[iy]   ;
+                	x[ix+1]  = y[iy+1] ;
+                	y[iy]    = temp[0] ;
+                	y[iy+1]  = temp[1] ;
+
+                	ix += inc_x2 ;
+                	iy += inc_y2 ;
+                	i++ ;
+
+		}
+
+	}
+	return(0);
+	
+
+}
+
+

kernel/power/cswap_microk_power8.c

@@ -0,0 +1,180 @@
+/***************************************************************************
+Copyright (c) 2013-2016, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+/**************************************************************************************
+* 2016/03/27 Werner Saar (wernsaar@googlemail.com)
+* 	 BLASTEST 		: OK
+* 	 CTEST			: OK
+* 	 TEST			: OK
+*	 LAPACK-TEST		: OK
+**************************************************************************************/
+
+#define HAVE_KERNEL_32 1
+
+static void cswap_kernel_32( BLASLONG n, FLOAT *x, FLOAT *y) __attribute__ ((noinline));
+
+static void cswap_kernel_32( BLASLONG n, FLOAT *x, FLOAT *y)
+{
+
+
+	BLASLONG i = n;
+	BLASLONG o16 = 16;
+	BLASLONG o32 = 32;
+	BLASLONG o48 = 48;
+	BLASLONG o64 = 64;
+	BLASLONG o80 = 80;
+	BLASLONG o96 = 96;
+	BLASLONG o112 = 112;
+	FLOAT *x1=x;
+	FLOAT *y1=y;
+	FLOAT *x2=x+1;
+	FLOAT *y2=y+1;
+	BLASLONG pre = 384;
+	BLASLONG alpha=0;
+
+	__asm__  __volatile__
+	(
+
+	"addi		%3, %3, -4			    \n\t"	
+	"addi		%4, %4, -4			    \n\t"	
+
+	".align 5				            \n\t"
+	"1:				                    \n\t"
+
+	"lxvw4x		32, 0, %2			    \n\t"
+	"lxvw4x		33, %5, %2			    \n\t"
+	"lxvw4x		34, %6, %2			    \n\t"
+	"lxvw4x		35, %7, %2			    \n\t"
+	"lxvw4x		36, %8, %2			    \n\t"
+	"lxvw4x		37, %9, %2			    \n\t"
+	"lxvw4x		38, %10, %2			    \n\t"
+	"lxvw4x		39, %11, %2			    \n\t"
+
+	"addi		%2, %2, 128			    \n\t"
+
+	"lxvw4x		40, 0, %2			    \n\t"
+	"lxvw4x		41, %5, %2			    \n\t"
+	"lxvw4x		42, %6, %2			    \n\t"
+	"lxvw4x		43, %7, %2			    \n\t"
+	"lxvw4x		44, %8, %2			    \n\t"
+	"lxvw4x		45, %9, %2			    \n\t"
+	"lxvw4x		46, %10, %2			    \n\t"
+	"lxvw4x		47, %11, %2			    \n\t"
+
+	"addi		%2, %2, 128			    \n\t"
+
+	"lxvw4x		48, 0, %1			    \n\t"
+	"lxvw4x		49, %5, %1			    \n\t"
+	"lxvw4x		50, %6, %1			    \n\t"
+	"lxvw4x		51, %7, %1			    \n\t"
+	"lxvw4x		52, %8, %1			    \n\t"
+	"lxvw4x		53, %9, %1			    \n\t"
+	"lxvw4x		54, %10, %1			    \n\t"
+	"lxvw4x		55, %11, %1			    \n\t"
+
+	"addi		%1, %1, 128			    \n\t"
+
+	"lxvw4x		56, 0, %1			    \n\t"
+	"lxvw4x		57, %5, %1			    \n\t"
+	"lxvw4x		58, %6, %1			    \n\t"
+	"lxvw4x		59, %7, %1			    \n\t"
+	"lxvw4x		60, %8, %1			    \n\t"
+	"lxvw4x		61, %9, %1			    \n\t"
+	"lxvw4x		62, %10, %1			    \n\t"
+	"lxvw4x		63, %11, %1			    \n\t"
+
+	"addi		%1, %1, 128			    \n\t"
+
+	"stxvw4x		32, 0, %3			    \n\t"
+	"stxvw4x		33, %5, %3			    \n\t"
+	"stxvw4x		34, %6, %3			    \n\t"
+	"stxvw4x		35, %7, %3			    \n\t"
+	"stxvw4x		36, %8, %3			    \n\t"
+	"stxvw4x		37, %9, %3			    \n\t"
+	"stxvw4x		38, %10, %3			    \n\t"
+	"stxvw4x		39, %11, %3			    \n\t"
+
+	"addi		%3, %3, 128			    \n\t"
+
+	"stxvw4x		40, 0, %3			    \n\t"
+	"stxvw4x		41, %5, %3			    \n\t"
+	"stxvw4x		42, %6, %3			    \n\t"
+	"stxvw4x		43, %7, %3			    \n\t"
+	"stxvw4x		44, %8, %3			    \n\t"
+	"stxvw4x		45, %9, %3			    \n\t"
+	"stxvw4x		46, %10, %3			    \n\t"
+	"stxvw4x		47, %11, %3			    \n\t"
+
+	"addi		%3, %3, 128			    \n\t"
+
+	"stxvw4x		48, 0, %4			    \n\t"
+	"stxvw4x		49, %5, %4			    \n\t"
+	"stxvw4x		50, %6, %4			    \n\t"
+	"stxvw4x		51, %7, %4			    \n\t"
+	"stxvw4x		52, %8, %4			    \n\t"
+	"stxvw4x		53, %9, %4			    \n\t"
+	"stxvw4x		54, %10, %4			    \n\t"
+	"stxvw4x		55, %11, %4			    \n\t"
+
+	"addi		%4, %4, 128			    \n\t"
+
+	"stxvw4x		56, 0, %4			    \n\t"
+	"stxvw4x		57, %5, %4			    \n\t"
+	"stxvw4x		58, %6, %4			    \n\t"
+	"stxvw4x		59, %7, %4			    \n\t"
+	"stxvw4x		60, %8, %4			    \n\t"
+	"stxvw4x		61, %9, %4			    \n\t"
+	"stxvw4x		62, %10, %4			    \n\t"
+	"stxvw4x		63, %11, %4			    \n\t"
+
+	"addi		%4, %4, 128			    \n\t"
+
+	"addic.		%0 , %0	, -32  	 	             \n\t"
+	"bgt		1b		             	     \n\t"
+
+	"2:						     \n\t"
+
+	:
+        : 
+          "r" (i),	// 0	
+	  "r" (y1),  	// 1
+          "r" (x1),     // 2
+          "r" (y2),     // 3
+          "r" (x2),     // 4
+	  "r" (o16),	// 5
+	  "r" (o32),	// 6
+	  "r" (o48),    // 7
+          "r" (o64),    // 8
+          "r" (o80),    // 9
+          "r" (o96),    // 10
+          "r" (o112)    // 11
+	: "cr0", "%0", "%2" , "%1", "%3", "%4", "memory"
+	);
+
+} 
+
+

kernel/power/ctrmm_kernel_8x4_power8.S

@@ -0,0 +1,399 @@
+/***************************************************************************
+Copyright (c) 2013-2016, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+/**************************************************************************************
+* 2016/04/04 Werner Saar (wernsaar@googlemail.com)
+* 	 BLASTEST 		: OK
+* 	 CTEST			: OK
+* 	 TEST			: OK
+* 	 LAPACK-TEST		: OK
+**************************************************************************************/
+
+/*********************************************************************/
+/* Copyright 2009, 2010 The University of Texas at Austin.           */
+/* All rights reserved.                                              */
+/*                                                                   */
+/* Redistribution and use in source and binary forms, with or        */
+/* without modification, are permitted provided that the following   */
+/* conditions are met:                                               */
+/*                                                                   */
+/*   1. Redistributions of source code must retain the above         */
+/*      copyright notice, this list of conditions and the following  */
+/*      disclaimer.                                                  */
+/*                                                                   */
+/*   2. Redistributions in binary form must reproduce the above      */
+/*      copyright notice, this list of conditions and the following  */
+/*      disclaimer in the documentation and/or other materials       */
+/*      provided with the distribution.                              */
+/*                                                                   */
+/*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */
+/*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */
+/*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */
+/*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */
+/*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */
+/*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */
+/*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */
+/*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */
+/*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */
+/*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */
+/*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */
+/*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */
+/*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */
+/*    POSSIBILITY OF SUCH DAMAGE.                                    */
+/*                                                                   */
+/* The views and conclusions contained in the software and           */
+/* documentation are those of the authors and should not be          */
+/* interpreted as representing official policies, either expressed   */
+/* or implied, of The University of Texas at Austin.                 */
+/*********************************************************************/
+
+#define ASSEMBLER
+#include "common.h"
+#include "def_vsx.h"
+
+#ifndef __64BIT__
+#define LOAD	lwz
+#else
+#define LOAD	ld
+#endif
+
+#ifdef __64BIT__
+#define STACKSIZE 400
+#define ALPHA_R_SP 304(SP)
+#define ALPHA_I_SP 312(SP)
+#else
+#define STACKSIZE 256
+#define ALPHA_R_SP 224(SP)
+#define ALPHA_I_SP 232(SP)
+#define FZERO	240(SP)
+#endif
+
+#define	M	r3
+#define	N	r4
+#define	K	r5
+
+#ifdef linux
+#ifndef __64BIT__
+#define A	r6
+#define	B	r7
+#define	C	r8
+#define	LDC	r9
+#define OFFSET	r10
+#else
+#define A	r8
+#define	B	r9
+#define	C	r10
+#define	LDC	r6
+#define OFFSET	r7
+#endif
+#endif
+
+#if defined(_AIX) || defined(__APPLE__)
+#if !defined(__64BIT__) && defined(DOUBLE)
+#define A	r10
+#define	B	r6
+#define	C	r7
+#define	LDC	r8
+#define OFFSET	r9
+#else
+#define A	r8
+#define	B	r9
+#define	C	r10
+#define	LDC	r6
+#define OFFSET	r7
+#endif
+#endif
+
+#define o0	0
+
+#define alpha_dr vs28
+#define alpha_di vs29
+#define alpha_sr vs30
+#define alpha_si vs31
+
+#define o12	r12
+#define KKK	r13
+#define K1	r14
+#define L	r15
+#define o16	r16
+#define NOTUSED	r17
+#define T2	r19
+#define KK	r20
+#define	o8	r21
+#define	I	r22
+#define J	r23
+#define AO	r24
+#define	BO	r25
+#define	CO	r26
+#define o4	r27
+#define	o32	r28
+#define o48	r29
+
+#define PRE	r30
+#define T1  	r31
+
+#ifndef NEEDPARAM
+
+	PROLOGUE
+	PROFCODE
+
+	addi	SP, SP, -STACKSIZE
+	li	r0, 0
+
+	stfd	f14,    0(SP)
+	stfd	f15,    8(SP)
+	stfd	f16,   16(SP)
+	stfd	f17,   24(SP)
+
+	stfd	f18,   32(SP)
+	stfd	f19,   40(SP)
+	stfd	f20,   48(SP)
+	stfd	f21,   56(SP)
+
+	stfd	f22,   64(SP)
+	stfd	f23,   72(SP)
+	stfd	f24,   80(SP)
+	stfd	f25,   88(SP)
+
+	stfd	f26,   96(SP)
+	stfd	f27,  104(SP)
+	stfd	f28,  112(SP)
+	stfd	f29,  120(SP)
+
+	stfd	f30,  128(SP)
+	stfd	f31,  136(SP)
+
+#ifdef __64BIT__
+	std	r31,  144(SP)
+	std	r30,  152(SP)
+	std	r29,  160(SP)
+	std	r28,  168(SP)
+	std	r27,  176(SP)
+	std	r26,  184(SP)
+	std	r25,  192(SP)
+	std	r24,  200(SP)
+	std	r23,  208(SP)
+	std	r22,  216(SP)
+	std	r21,  224(SP)
+	std	r20,  232(SP)
+	std	r19,  240(SP)
+	std	r18,  248(SP)
+	std	r17,  256(SP)
+	std	r16,  264(SP)
+	std	r15,  272(SP)
+	std	r14,  280(SP)
+	std	r13,  288(SP)
+	std	r12,  296(SP)
+#else
+	stw	r31,  144(SP)
+	stw	r30,  148(SP)
+	stw	r29,  152(SP)
+	stw	r28,  156(SP)
+	stw	r27,  160(SP)
+	stw	r26,  164(SP)
+	stw	r25,  168(SP)
+	stw	r24,  172(SP)
+	stw	r23,  176(SP)
+	stw	r22,  180(SP)
+	stw	r21,  184(SP)
+	stw	r20,  188(SP)
+	stw	r19,  192(SP)
+	stw	r18,  196(SP)
+	stw	r17,  200(SP)
+	stw	r16,  204(SP)
+	stw	r15,  208(SP)
+	stw	r14,  212(SP)
+	stw	r13,  216(SP)
+#endif
+
+	stfs	f1,  ALPHA_R_SP
+	stfs	f2,  ALPHA_I_SP
+	// stw	r0,  FZERO
+
+#ifdef linux
+#ifdef __64BIT__
+	ld	LDC, FRAMESLOT(0) + STACKSIZE(SP)
+#endif
+#endif
+
+#if defined(_AIX) || defined(__APPLE__)
+#ifdef __64BIT__
+	ld	LDC, FRAMESLOT(0) + STACKSIZE(SP)
+#else
+#ifdef DOUBLE
+	lwz	B,   FRAMESLOT(0) + STACKSIZE(SP)
+	lwz	C,   FRAMESLOT(1) + STACKSIZE(SP)
+	lwz	LDC, FRAMESLOT(2) + STACKSIZE(SP)
+#else
+	lwz	LDC, FRAMESLOT(0) + STACKSIZE(SP)
+#endif
+#endif
+#endif
+
+#ifdef TRMMKERNEL
+#if defined(linux) && defined(__64BIT__)
+	ld	OFFSET,  FRAMESLOT(1) + STACKSIZE(SP)
+#endif
+
+#if defined(_AIX) || defined(__APPLE__)
+#ifdef __64BIT__
+	ld	OFFSET,  FRAMESLOT(1) + STACKSIZE(SP)
+#else
+#ifdef DOUBLE
+	lwz	OFFSET,  FRAMESLOT(3) + STACKSIZE(SP)
+#else
+	lwz	OFFSET,  FRAMESLOT(1) + STACKSIZE(SP)
+#endif
+#endif
+#endif
+#if defined(TRMMKERNEL) && !defined(LEFT)
+	neg	KK, OFFSET
+#endif
+#endif
+
+#include "ctrmm_macros_8x4_power8.S"
+
+	cmpwi	cr0, M, 0
+	ble	L999_H1
+	cmpwi	cr0, N, 0
+	ble	L999_H1
+	cmpwi	cr0, K, 0
+	ble	L999_H1
+
+        slwi    LDC, LDC, ZBASE_SHIFT
+        li      PRE, 384
+        li      o4  , 4
+        li      o8  , 8
+        li      o12 , 12
+        li      o16 , 16
+        li      o32 , 32
+        li      o48 , 48
+
+
+#ifdef __64BIT__
+	addi	T1, SP, 304
+#else
+	addi	T1, SP, 224
+#endif
+
+        lxsspx  alpha_dr, 0, T1
+        lxsspx  alpha_di, o8, T1
+        addi    T1, SP, 360
+        li      T2, 0
+
+        stw             T2, 0(T1)
+        stw             T2, 4(T1)
+        stw             T2, 8(T1)
+        stxsspx         alpha_dr, o12, T1
+        lxvw4x          alpha_sr, o0 , T1
+        addi            T1, T1, 16
+
+        stw             T2, 0(T1)
+        stw             T2, 4(T1)
+        stw             T2, 8(T1)
+        stxsspx         alpha_di, o12, T1
+        lxvw4x          alpha_si, o0 , T1
+
+	.align 5
+
+#include "ctrmm_logic_8x4_power8.S"
+
+L999:
+	addi	r3, 0, 0
+
+	lfd	f14,    0(SP)
+	lfd	f15,    8(SP)
+	lfd	f16,   16(SP)
+	lfd	f17,   24(SP)
+
+	lfd	f18,   32(SP)
+	lfd	f19,   40(SP)
+	lfd	f20,   48(SP)
+	lfd	f21,   56(SP)
+
+	lfd	f22,   64(SP)
+	lfd	f23,   72(SP)
+	lfd	f24,   80(SP)
+	lfd	f25,   88(SP)
+
+	lfd	f26,   96(SP)
+	lfd	f27,  104(SP)
+	lfd	f28,  112(SP)
+	lfd	f29,  120(SP)
+
+	lfd	f30,  128(SP)
+	lfd	f31,  136(SP)
+
+#ifdef __64BIT__
+	ld	r31,  144(SP)
+	ld	r30,  152(SP)
+	ld	r29,  160(SP)
+	ld	r28,  168(SP)
+	ld	r27,  176(SP)
+	ld	r26,  184(SP)
+	ld	r25,  192(SP)
+	ld	r24,  200(SP)
+	ld	r23,  208(SP)
+	ld	r22,  216(SP)
+	ld	r21,  224(SP)
+	ld	r20,  232(SP)
+	ld	r19,  240(SP)
+	ld	r18,  248(SP)
+	ld	r17,  256(SP)
+	ld	r16,  264(SP)
+	ld	r15,  272(SP)
+	ld	r14,  280(SP)
+	ld	r13,  288(SP)
+	ld	r12,  296(SP)
+#else
+	lwz	r31,  144(SP)
+	lwz	r30,  148(SP)
+	lwz	r29,  152(SP)
+	lwz	r28,  156(SP)
+	lwz	r27,  160(SP)
+	lwz	r26,  164(SP)
+	lwz	r25,  168(SP)
+	lwz	r24,  172(SP)
+	lwz	r23,  176(SP)
+	lwz	r22,  180(SP)
+	lwz	r21,  184(SP)
+	lwz	r20,  188(SP)
+	lwz	r19,  192(SP)
+	lwz	r18,  196(SP)
+	lwz	r17,  200(SP)
+	lwz	r16,  204(SP)
+	lwz	r15,  208(SP)
+	lwz	r14,  212(SP)
+	lwz	r13,  216(SP)
+#endif
+
+	addi	SP, SP, STACKSIZE
+
+	blr
+
+	EPILOGUE
+#endif

kernel/power/dasum.c

@@ -0,0 +1,144 @@
+/***************************************************************************
+Copyright (c) 2013-2016, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+/**************************************************************************************
+* 2016/03/28 Werner Saar (wernsaar@googlemail.com)
+* 	 BLASTEST 		: OK
+* 	 CTEST			: OK
+* 	 TEST			: OK
+*	 LAPACK-TEST		: OK
+**************************************************************************************/
+
+#include "common.h"
+#include <math.h>
+
+#if defined(DOUBLE)
+
+#define ABS fabs
+
+#else
+
+#define ABS fabsf
+
+#endif
+
+#if defined(POWER8)
+#include "dasum_microk_power8.c"
+#endif
+
+
+#ifndef HAVE_KERNEL_16
+
+static void dasum_kernel_16(BLASLONG n, FLOAT *x1, FLOAT *svec)
+{
+
+	BLASLONG i=0;
+	FLOAT *x = x1;
+	FLOAT temp0, temp1, temp2, temp3;
+	FLOAT temp4, temp5, temp6, temp7;
+	FLOAT sum0 = 0.0;
+	FLOAT sum1 = 0.0;
+	FLOAT sum2 = 0.0;
+	FLOAT sum3 = 0.0;
+
+	while ( i< n )
+	{
+
+		temp0 = ABS(x[0]);
+		temp1 = ABS(x[1]);
+		temp2 = ABS(x[2]);
+		temp3 = ABS(x[3]);
+		temp4 = ABS(x[4]);
+		temp5 = ABS(x[5]);
+		temp6 = ABS(x[6]);
+		temp7 = ABS(x[7]);
+
+		sum0 += temp0;
+		sum1 += temp1;
+		sum2 += temp2;
+		sum3 += temp3;
+
+		sum0 += temp4;
+		sum1 += temp5;
+		sum2 += temp6;
+		sum3 += temp7;
+
+		x+=8;
+		i+=8;
+
+	}
+
+	svec[0] = sum0+sum1+sum2+sum3;
+	svec[1] = 0.0;
+
+}
+
+#endif
+
+FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x)
+{
+	BLASLONG i=0;
+	FLOAT sumf = 0.0;
+	FLOAT svec[2] __attribute__ ((aligned (16)));;
+	BLASLONG n1;
+
+	if (n <= 0 || inc_x <= 0) return(sumf);
+
+	if ( inc_x == 1 )
+	{
+
+		n1 = n & -16;
+		if ( n1 > 0 )
+		{
+
+			dasum_kernel_16(n1, x, svec);
+			sumf = svec[0] + svec[1];
+			i=n1;
+		}
+
+		while(i < n)
+		{
+			sumf += ABS(x[i]);
+			i++;
+		}
+
+	}
+	else
+	{
+
+		n *= inc_x;
+		while(i < n)
+		{
+			sumf += ABS(x[i]);
+			i += inc_x;
+		}
+
+	}
+	return(sumf);
+}
+
+

kernel/power/dasum_microk_power8.c

@@ -0,0 +1,177 @@
+/***************************************************************************
+Copyright (c) 2013-2016, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+/**************************************************************************************
+* 2016/03/28 Werner Saar (wernsaar@googlemail.com)
+* 	 BLASTEST 		: OK
+* 	 CTEST			: OK
+* 	 TEST			: OK
+*	 LAPACK-TEST		: OK
+**************************************************************************************/
+
+#define HAVE_KERNEL_16 1
+static void dasum_kernel_16( BLASLONG n, FLOAT *x, FLOAT *svec) __attribute__ ((noinline));
+
+static void dasum_kernel_16( BLASLONG n, FLOAT *x, FLOAT *svec)
+{
+
+
+	BLASLONG i = n;
+	BLASLONG o16 = 16;
+	BLASLONG o32 = 32;
+	BLASLONG o48 = 48;
+	BLASLONG o64 = 64;
+	BLASLONG o80 = 80;
+	BLASLONG o96 = 96;
+	BLASLONG o112 = 112;
+	FLOAT *x1=x;
+	BLASLONG pre = 384;
+
+	__asm__  __volatile__
+	(
+
+	"dcbt		%2 , %4				    \n\t"
+
+	"xxlxor		32,32,32			    \n\t"
+	"xxlxor		33,33,33			    \n\t"
+	"xxlxor		34,34,34			    \n\t"
+	"xxlxor		35,35,35			    \n\t"
+	"xxlxor		36,36,36			    \n\t"
+	"xxlxor		37,37,37			    \n\t"
+	"xxlxor		38,38,38			    \n\t"
+	"xxlxor		39,39,39			    \n\t"
+
+	"lxvd2x		40, 0, %2			    \n\t"
+	"lxvd2x		41, %5, %2			    \n\t"
+	"lxvd2x		42, %6, %2			    \n\t"
+	"lxvd2x		43, %7, %2			    \n\t"
+	"lxvd2x		44, %8, %2			    \n\t"
+	"lxvd2x		45, %9, %2			    \n\t"
+	"lxvd2x		46, %10, %2			    \n\t"
+	"lxvd2x		47, %11, %2			    \n\t"
+
+	"addi		%2, %2, 128			    \n\t"
+
+	"addic.		%0 , %0	, -16  	 	             \n\t"
+	"ble		2f		             	     \n\t"
+
+	".align 5				            \n\t"
+	"1:				                    \n\t"
+
+	"dcbt		%2 , %4				    \n\t"
+
+	"xvabsdp	48, 40				    \n\t"
+	"xvabsdp	49, 41				    \n\t"
+	"xvabsdp	50, 42				    \n\t"
+	"xvabsdp	51, 43				    \n\t"
+
+	"lxvd2x		40, 0, %2			    \n\t"
+	"lxvd2x		41, %5, %2			    \n\t"
+
+	"xvabsdp	52, 44				    \n\t"
+	"xvabsdp	53, 45				    \n\t"
+
+	"lxvd2x		42, %6, %2			    \n\t"
+	"lxvd2x		43, %7, %2			    \n\t"
+
+	"xvabsdp	54, 46				    \n\t"
+	"xvabsdp	55, 47				    \n\t"
+
+	"lxvd2x		44, %8, %2			    \n\t"
+	"lxvd2x		45, %9, %2			    \n\t"
+
+	"xvadddp	32, 32, 48		    \n\t"
+	"xvadddp	33, 33, 49		    \n\t"
+
+	"lxvd2x		46, %10, %2			    \n\t"
+	"lxvd2x		47, %11, %2			    \n\t"
+
+	"xvadddp	34, 34, 50		    \n\t"
+	"xvadddp	35, 35, 51		    \n\t"
+	"addi		%2, %2, 128			    \n\t"
+	"xvadddp	36, 36, 52		    \n\t"
+	"xvadddp	37, 37, 53		    \n\t"
+	"addic.		%0 , %0	, -16  	 	             \n\t"
+	"xvadddp	38, 38, 54		    \n\t"
+	"xvadddp	39, 39, 55		    \n\t"
+
+	"bgt		1b		             	     \n\t"
+
+	"2:						     \n\t"
+
+
+	"xvabsdp	48, 40				    \n\t"
+	"xvabsdp	49, 41				    \n\t"
+	"xvabsdp	50, 42				    \n\t"
+	"xvabsdp	51, 43				    \n\t"
+	"xvabsdp	52, 44				    \n\t"
+	"xvabsdp	53, 45				    \n\t"
+	"xvabsdp	54, 46				    \n\t"
+	"xvabsdp	55, 47				    \n\t"
+
+	"xvadddp	32, 32, 48		    \n\t"
+	"xvadddp	33, 33, 49		    \n\t"
+	"xvadddp	34, 34, 50		    \n\t"
+	"xvadddp	35, 35, 51		    \n\t"
+	"xvadddp	36, 36, 52		    \n\t"
+	"xvadddp	37, 37, 53		    \n\t"
+	"xvadddp	38, 38, 54		    \n\t"
+	"xvadddp	39, 39, 55		    \n\t"
+
+	"xvadddp	32, 32, 33		     \n\t"
+	"xvadddp	34, 34, 35		     \n\t"
+	"xvadddp	36, 36, 37		     \n\t"
+	"xvadddp	38, 38, 39		     \n\t"
+
+	"xvadddp	32, 32, 34		     \n\t"
+	"xvadddp	36, 36, 38		     \n\t"
+
+	"xvadddp	32, 32, 36		     \n\t"
+
+
+	"stxvd2x	32, 0, %3		     \n\t"
+
+	:
+        : 
+          "r" (i),	// 0	
+	  "r" (n),  	// 1
+          "r" (x1),     // 2
+          "r" (svec),   // 3
+          "r" (pre),    // 4
+	  "r" (o16),	// 5
+	  "r" (o32),	// 6
+	  "r" (o48),    // 7
+          "r" (o64),    // 8
+          "r" (o80),    // 9
+          "r" (o96),    // 10
+          "r" (o112)   // 11
+	: "cr0", "%0", "%2",  "memory"
+	);
+
+} 
+
+

kernel/power/daxpy.c

@@ -0,0 +1,136 @@
+/***************************************************************************
+Copyright (c) 2013-2016, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+/**************************************************************************************
+* 2016/03/22 Werner Saar (wernsaar@googlemail.com)
+* 	 BLASTEST 		: OK
+* 	 CTEST			: OK
+* 	 TEST			: OK
+*	 LAPACK-TEST		: OK
+**************************************************************************************/
+
+#include "common.h"
+
+
+#if defined(POWER8)
+#include "daxpy_microk_power8.c"
+#endif
+
+
+#ifndef HAVE_KERNEL_8
+
+static void daxpy_kernel_8(BLASLONG n, FLOAT *x, FLOAT *y, FLOAT *alpha)
+{
+	BLASLONG register i = 0;
+	FLOAT a = *alpha;
+
+	while(i < n)
+        {
+              y[i]   += a * x[i];
+              y[i+1] += a * x[i+1];
+              y[i+2] += a * x[i+2];
+              y[i+3] += a * x[i+3];
+              y[i+4] += a * x[i+4];
+              y[i+5] += a * x[i+5];
+              y[i+6] += a * x[i+6];
+              y[i+7] += a * x[i+7];
+              i+=8 ;
+
+       }
+
+}
+
+#endif
+
+int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y, FLOAT *dummy, BLASLONG dummy2)
+{
+	BLASLONG i=0;
+	BLASLONG ix=0,iy=0;
+	FLOAT a2[4];
+	a2[0]=da;
+	a2[1]=da;
+	a2[2]=da;
+	a2[3]=da;
+
+	if ( n <= 0 )  return(0);
+
+	if ( (inc_x == 1) && (inc_y == 1) )
+	{
+
+		BLASLONG n1 = n & -16;
+
+		if ( n1 )
+			daxpy_kernel_8(n1, x, y , a2 );
+
+		i = n1;
+		while(i < n)
+		{
+
+			y[i] += da * x[i] ;
+			i++ ;
+
+		}
+		return(0);
+
+
+	}
+
+	BLASLONG n1 = n & -4;
+
+	while(i < n1)
+	{
+
+		FLOAT m1      = da * x[ix] ;
+		FLOAT m2      = da * x[ix+inc_x] ;
+		FLOAT m3      = da * x[ix+2*inc_x] ;
+		FLOAT m4      = da * x[ix+3*inc_x] ;
+
+		y[iy]         += m1 ;
+		y[iy+inc_y]   += m2 ;
+		y[iy+2*inc_y] += m3 ;
+		y[iy+3*inc_y] += m4 ;
+
+		ix  += inc_x*4 ;
+		iy  += inc_y*4 ;
+		i+=4 ;
+
+	}
+
+	while(i < n)
+	{
+
+		y[iy] += da * x[ix] ;
+		ix  += inc_x ;
+		iy  += inc_y ;
+		i++ ;
+
+	}
+	return(0);
+
+}
+
+

kernel/power/daxpy_microk_power8.c

@@ -0,0 +1,201 @@
+/***************************************************************************
+Copyright (c) 2013-2016, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+/**************************************************************************************
+* 2016/03/22 Werner Saar (wernsaar@googlemail.com)
+* 	 BLASTEST 		: OK
+* 	 CTEST			: OK
+* 	 TEST			: OK
+*	 LAPACK-TEST		: OK
+**************************************************************************************/
+
+
+#define HAVE_KERNEL_8 1
+static void daxpy_kernel_8( BLASLONG n, FLOAT *x, FLOAT *y , FLOAT *alpha) __attribute__ ((noinline));
+
+static void daxpy_kernel_8( BLASLONG n, FLOAT *x, FLOAT *y, FLOAT *alpha)
+{
+
+
+	BLASLONG i = n;
+	BLASLONG o16 = 16;
+	BLASLONG o32 = 32;
+	BLASLONG o48 = 48;
+	FLOAT *x1=x;
+	FLOAT *y1=y;
+	FLOAT *y2=y+1;
+	BLASLONG pre = 384;
+
+	__asm__  __volatile__
+	(
+
+	"lxsdx		33, %5, %4			    \n\t"
+	"xxspltd	32, 33, 0			    \n\t"
+	"addi		%8, %8, -8			    \n\t"
+
+	"dcbt		%2, %9				    \n\t"
+	"dcbt		%3, %9				    \n\t"
+
+	"lxvd2x		40, 0, %2			    \n\t"
+	"lxvd2x		41, %5, %2			    \n\t"
+	"lxvd2x		42, %6, %2			    \n\t"
+	"lxvd2x		43, %7, %2			    \n\t"
+
+	"lxvd2x		48, 0, %3			    \n\t"
+	"lxvd2x		49, %5, %3			    \n\t"
+	"lxvd2x		50, %6, %3			    \n\t"
+	"lxvd2x		51, %7, %3			    \n\t"
+
+	"addi		%2, %2, 64			    \n\t"
+	"addi		%3, %3, 64			    \n\t"
+
+	"lxvd2x		44, 0, %2			    \n\t"
+	"lxvd2x		45, %5, %2			    \n\t"
+	"lxvd2x		46, %6, %2			    \n\t"
+	"lxvd2x		47, %7, %2			    \n\t"
+
+	"lxvd2x		52, 0, %3			    \n\t"
+	"lxvd2x		53, %5, %3			    \n\t"
+	"lxvd2x		54, %6, %3			    \n\t"
+	"lxvd2x		55, %7, %3			    \n\t"
+
+	"addi		%2, %2, 64			    \n\t"
+	"addi		%3, %3, 64			    \n\t"
+
+	"addic.		%0 , %0	, -16  	 	             \n\t"
+	"ble		2f		             	     \n\t"
+
+	".align 5				            \n\t"
+	"1:				                    \n\t"
+
+	"dcbt		%2, %9				    \n\t"
+	"dcbt		%3, %9				    \n\t"
+
+	"xvmaddadp	48, 40, 32		    	    \n\t"
+	"xvmaddadp	49, 41, 32		    	    \n\t"
+
+	"lxvd2x		40, 0, %2			    \n\t"
+	"lxvd2x		41, %5, %2			    \n\t"
+
+	"stxvd2x	48,  0, %8			    \n\t"
+	"stxvd2x	49, %5, %8			    \n\t"
+
+	"xvmaddadp	50, 42, 32		    	    \n\t"
+	"xvmaddadp	51, 43, 32		    	    \n\t"
+
+	"lxvd2x		42, %6, %2			    \n\t"
+	"lxvd2x		43, %7, %2			    \n\t"
+
+	"stxvd2x	50, %6, %8			    \n\t"
+	"stxvd2x	51, %7, %8			    \n\t"
+
+	"lxvd2x		48, 0, %3			    \n\t"
+	"lxvd2x		49, %5, %3			    \n\t"
+	"lxvd2x		50, %6, %3			    \n\t"
+	"lxvd2x		51, %7, %3			    \n\t"
+
+	"addi		%2, %2, 64			    \n\t"
+	"addi		%8, %8, 64			    \n\t"
+
+	"xvmaddadp	52, 44, 32		    	    \n\t"
+	"addi		%3, %3, 64			    \n\t"
+	"xvmaddadp	53, 45, 32		    	    \n\t"
+
+	"lxvd2x		44, 0, %2			    \n\t"
+	"lxvd2x		45, %5, %2			    \n\t"
+
+	"stxvd2x	52,  0, %8			    \n\t"
+	"stxvd2x	53, %5, %8			    \n\t"
+
+	"xvmaddadp	54, 46, 32		    	    \n\t"
+	"xvmaddadp	55, 47, 32		    	    \n\t"
+
+	"lxvd2x		46, %6, %2			    \n\t"
+	"lxvd2x		47, %7, %2			    \n\t"
+
+	"stxvd2x	54, %6, %8			    \n\t"
+	"stxvd2x	55, %7, %8			    \n\t"
+
+	"addi		%2, %2, 64			    \n\t"
+	"addi		%8, %8, 64			    \n\t"
+
+	"lxvd2x		52, 0, %3			    \n\t"
+	"lxvd2x		53, %5, %3			    \n\t"
+	"lxvd2x		54, %6, %3			    \n\t"
+	"lxvd2x		55, %7, %3			    \n\t"
+
+	"addi		%3, %3, 64			    \n\t"
+
+
+	"addic.		%0 , %0	, -16  	 	             \n\t"
+	"bgt		1b		             	     \n\t"
+
+	"2:						     \n\t"
+
+	
+	"xvmaddadp	48, 40, 32		    	    \n\t"
+	"xvmaddadp	49, 41, 32		    	    \n\t"
+	"xvmaddadp	50, 42, 32		    	    \n\t"
+	"xvmaddadp	51, 43, 32		    	    \n\t"
+
+	"xvmaddadp	52, 44, 32		    	    \n\t"
+	"xvmaddadp	53, 45, 32		    	    \n\t"
+	"xvmaddadp	54, 46, 32		    	    \n\t"
+	"xvmaddadp	55, 47, 32		    	    \n\t"
+
+	"stxvd2x	48,  0, %8			    \n\t"
+	"stxvd2x	49, %5, %8			    \n\t"
+	"stxvd2x	50, %6, %8			    \n\t"
+	"stxvd2x	51, %7, %8			    \n\t"
+
+	"addi		%8, %8, 64			    \n\t"
+
+	"stxvd2x	52,  0, %8			    \n\t"
+	"stxvd2x	53, %5, %8			    \n\t"
+	"stxvd2x	54, %6, %8			    \n\t"
+	"stxvd2x	55, %7, %8			    \n\t"
+
+	"addi		%8, %8, 64			    \n\t"
+
+	:
+        : 
+          "r" (i),	// 0	
+	  "r" (n),  	// 1
+          "r" (x1),     // 2
+          "r" (y1),     // 3
+          "r" (alpha),    // 4
+	  "r" (o16),	// 5
+	  "r" (o32),	// 6
+	  "r" (o48),    // 7
+	  "r" (y2),     // 8
+	  "r" (pre)	// 9
+	: "cr0", "%0", "%2" , "%3", "%8", "memory"
+	);
+
+} 
+
+

kernel/power/dcopy.c

@@ -0,0 +1,131 @@
+/***************************************************************************
+Copyright (c) 2013-2016, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+/**************************************************************************************
+* 2016/03/25 Werner Saar (wernsaar@googlemail.com)
+* 	 BLASTEST 		: OK
+* 	 CTEST			: OK
+* 	 TEST			: OK
+*	 LAPACK-TEST		: OK
+**************************************************************************************/
+
+#include "common.h"
+
+#if defined(POWER8)
+#include "dcopy_microk_power8.c"
+#endif
+
+#ifndef HAVE_KERNEL_32
+
+static void dcopy_kernel_32(BLASLONG n, FLOAT *x, FLOAT *y)
+{
+
+	BLASLONG i=0;
+	FLOAT f0, f1, f2, f3, f4, f5, f6, f7;
+	FLOAT *x1=x;
+	FLOAT *y1=y;
+
+	while ( i<n )
+	{
+
+		f0 = x1[0];
+		f1 = x1[1];
+		f2 = x1[2];
+		f3 = x1[3];
+		f4 = x1[4];
+		f5 = x1[5];
+		f6 = x1[6];
+		f7 = x1[7];
+
+		y1[0] = f0;
+		y1[1] = f1;
+		y1[2] = f2;
+		y1[3] = f3;
+		y1[4] = f4;
+		y1[5] = f5;
+		y1[6] = f6;
+		y1[7] = f7;
+
+		x1 += 8;
+		y1 += 8;
+
+		i+=8;
+	}
+	return;
+
+}
+
+
+#endif
+
+
+
+int CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y)
+{
+	BLASLONG i=0;
+	BLASLONG ix=0,iy=0;
+
+	if ( n <= 0     )  return(0);
+
+	if ( (inc_x == 1) && (inc_y == 1 ))
+	{
+
+		BLASLONG n1 = n & -32;
+		if ( n1 > 0 )
+		{
+			dcopy_kernel_32(n1, x, y);
+			i=n1;
+		}
+
+		while(i < n)
+		{
+			y[i] = x[i] ;
+			i++ ;
+
+		}
+
+
+	}
+	else
+	{
+
+		while(i < n)
+		{
+			y[iy] = x[ix] ;
+			ix += inc_x ;
+			iy += inc_y ;
+			i++ ;
+
+		}
+
+	}
+	return(0);
+	
+
+}
+
+

kernel/power/dcopy_microk_power8.c

@@ -0,0 +1,174 @@
+/***************************************************************************
+Copyright (c) 2013-2016, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+/**************************************************************************************
+* 2016/03/25 Werner Saar (wernsaar@googlemail.com)
+* 	 BLASTEST 		: OK
+* 	 CTEST			: OK
+* 	 TEST			: OK
+*	 LAPACK-TEST		: OK
+**************************************************************************************/
+
+#define HAVE_KERNEL_32 1
+
+static void dcopy_kernel_32( BLASLONG n, FLOAT *x, FLOAT *y) __attribute__ ((noinline));
+
+static void dcopy_kernel_32( BLASLONG n, FLOAT *x, FLOAT *y)
+{
+
+
+	BLASLONG i = n;
+	BLASLONG o16 = 16;
+	BLASLONG o32 = 32;
+	BLASLONG o48 = 48;
+	BLASLONG o64 = 64;
+	BLASLONG o80 = 80;
+	BLASLONG o96 = 96;
+	BLASLONG o112 = 112;
+	FLOAT *x1=x;
+	FLOAT *y1=y;
+	BLASLONG pre = 384;
+	BLASLONG alpha=0;
+
+	__asm__  __volatile__
+	(
+
+	"lxvd2x		40, 0, %2			    \n\t"
+	"lxvd2x		41, %5, %2			    \n\t"
+	"lxvd2x		42, %6, %2			    \n\t"
+	"lxvd2x		43, %7, %2			    \n\t"
+	"lxvd2x		44, %8, %2			    \n\t"
+	"lxvd2x		45, %9, %2			    \n\t"
+	"lxvd2x		46, %10, %2			    \n\t"
+	"lxvd2x		47, %11, %2			    \n\t"
+
+	"addi		%2, %2, 128			    \n\t"
+
+	"lxvd2x		50, 0, %2			    \n\t"
+	"lxvd2x		51, %5, %2			    \n\t"
+	"lxvd2x		52, %6, %2			    \n\t"
+	"lxvd2x		53, %7, %2			    \n\t"
+	"lxvd2x		54, %8, %2			    \n\t"
+	"lxvd2x		55, %9, %2			    \n\t"
+	"lxvd2x		56, %10, %2			    \n\t"
+	"lxvd2x		57, %11, %2			    \n\t"
+
+	"addi		%2, %2, 128			    \n\t"
+
+	"addic.		%0 , %0	, -32  	 	             \n\t"
+	"ble		2f		             	     \n\t"
+
+	".align 5				            \n\t"
+	"1:				                    \n\t"
+
+	"stxvd2x		40, 0, %1			    \n\t"
+	"stxvd2x		41, %5, %1			    \n\t"
+	"lxvd2x		40, 0, %2			    \n\t"
+	"lxvd2x		41, %5, %2			    \n\t"
+	"stxvd2x		42, %6, %1			    \n\t"
+	"stxvd2x		43, %7, %1			    \n\t"
+	"lxvd2x		42, %6, %2			    \n\t"
+	"lxvd2x		43, %7, %2			    \n\t"
+	"stxvd2x		44, %8, %1			    \n\t"
+	"stxvd2x		45, %9, %1			    \n\t"
+	"lxvd2x		44, %8, %2			    \n\t"
+	"lxvd2x		45, %9, %2			    \n\t"
+	"stxvd2x		46, %10, %1			    \n\t"
+	"stxvd2x		47, %11, %1			    \n\t"
+	"lxvd2x		46, %10, %2			    \n\t"
+	"lxvd2x		47, %11, %2			    \n\t"
+
+
+	"addi		%1, %1, 128			    \n\t"
+	"addi		%2, %2, 128			    \n\t"
+
+	"stxvd2x		50, 0, %1			    \n\t"
+	"stxvd2x		51, %5, %1			    \n\t"
+	"lxvd2x		50, 0, %2			    \n\t"
+	"lxvd2x		51, %5, %2			    \n\t"
+	"stxvd2x		52, %6, %1			    \n\t"
+	"stxvd2x		53, %7, %1			    \n\t"
+	"lxvd2x		52, %6, %2			    \n\t"
+	"lxvd2x		53, %7, %2			    \n\t"
+	"stxvd2x		54, %8, %1			    \n\t"
+	"stxvd2x		55, %9, %1			    \n\t"
+	"lxvd2x		54, %8, %2			    \n\t"
+	"lxvd2x		55, %9, %2			    \n\t"
+	"stxvd2x		56, %10, %1			    \n\t"
+	"stxvd2x		57, %11, %1			    \n\t"
+	"lxvd2x		56, %10, %2			    \n\t"
+	"lxvd2x		57, %11, %2			    \n\t"
+
+	"addi		%1, %1, 128			    \n\t"
+	"addi		%2, %2, 128			    \n\t"
+
+	"addic.		%0 , %0	, -32  	 	             \n\t"
+	"bgt		1b		             	     \n\t"
+
+	"2:						     \n\t"
+
+	"stxvd2x		40, 0, %1			    \n\t"
+	"stxvd2x		41, %5, %1			    \n\t"
+	"stxvd2x		42, %6, %1			    \n\t"
+	"stxvd2x		43, %7, %1			    \n\t"
+	"stxvd2x		44, %8, %1			    \n\t"
+	"stxvd2x		45, %9, %1			    \n\t"
+	"stxvd2x		46, %10, %1			    \n\t"
+	"stxvd2x		47, %11, %1			    \n\t"
+
+	"addi		%1, %1, 128			    \n\t"
+
+	"stxvd2x		50, 0, %1			    \n\t"
+	"stxvd2x		51, %5, %1			    \n\t"
+	"stxvd2x		52, %6, %1			    \n\t"
+	"stxvd2x		53, %7, %1			    \n\t"
+	"stxvd2x		54, %8, %1			    \n\t"
+	"stxvd2x		55, %9, %1			    \n\t"
+	"stxvd2x		56, %10, %1			    \n\t"
+	"stxvd2x		57, %11, %1			    \n\t"
+
+
+	:
+        : 
+          "r" (i),	// 0	
+	  "r" (y1),  	// 1
+          "r" (x1),     // 2
+          "r" (alpha),  // 3
+          "r" (pre),    // 4
+	  "r" (o16),	// 5
+	  "r" (o32),	// 6
+	  "r" (o48),    // 7
+          "r" (o64),    // 8
+          "r" (o80),    // 9
+          "r" (o96),    // 10
+          "r" (o112)    // 11
+	: "cr0", "%0", "%2" , "%1", "memory"
+	);
+
+} 
+
+

kernel/power/ddot.c

@@ -0,0 +1,139 @@
+/***************************************************************************
+Copyright (c) 2013-2016, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+/**************************************************************************************
+* 2016/03/20 Werner Saar (wernsaar@googlemail.com)
+* 	 BLASTEST 		: OK
+* 	 CTEST			: OK
+* 	 TEST			: OK
+*	 LAPACK-TEST		: OK
+**************************************************************************************/
+
+#include "common.h"
+
+
+#if defined(POWER8) 
+#include "ddot_microk_power8.c"
+#endif
+
+
+#ifndef HAVE_KERNEL_8
+
+static void ddot_kernel_8(BLASLONG n, FLOAT *x, FLOAT *y, FLOAT *d)
+{
+	BLASLONG register i = 0;
+	FLOAT dot = 0.0;
+
+	while(i < n)
+        {
+              dot += y[i]  * x[i]
+                  + y[i+1] * x[i+1]
+                  + y[i+2] * x[i+2]
+                  + y[i+3] * x[i+3]
+                  + y[i+4] * x[i+4]
+                  + y[i+5] * x[i+5]
+                  + y[i+6] * x[i+6]
+                  + y[i+7] * x[i+7] ;
+
+              i+=8 ;
+
+       }
+       *d += dot;
+
+}
+
+#endif
+
+FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y)
+{
+	BLASLONG i=0;
+	BLASLONG ix=0,iy=0;
+
+	FLOAT  dot = 0.0 ;
+
+	if ( n <= 0 )  return(dot);
+
+	if ( (inc_x == 1) && (inc_y == 1) )
+	{
+
+		BLASLONG n1 = n & -16;
+
+		if ( n1 )
+			ddot_kernel_8(n1, x, y , &dot );
+
+		i = n1;
+		while(i < n)
+		{
+
+			dot += y[i] * x[i] ;
+			i++ ;
+
+		}
+		return(dot);
+
+
+	}
+
+	FLOAT temp1 = 0.0;
+	FLOAT temp2 = 0.0;
+
+        BLASLONG n1 = n & -4;	
+
+	while(i < n1)
+	{
+
+		FLOAT m1 = y[iy]       * x[ix] ;
+		FLOAT m2 = y[iy+inc_y] * x[ix+inc_x] ;
+
+		FLOAT m3 = y[iy+2*inc_y] * x[ix+2*inc_x] ;
+		FLOAT m4 = y[iy+3*inc_y] * x[ix+3*inc_x] ;
+
+		ix  += inc_x*4 ;
+		iy  += inc_y*4 ;
+
+		temp1 += m1+m3;
+		temp2 += m2+m4;
+
+		i+=4 ;
+
+	}
+
+	while(i < n)
+	{
+
+		temp1 += y[iy] * x[ix] ;
+		ix  += inc_x ;
+		iy  += inc_y ;
+		i++ ;
+
+	}
+	dot = temp1 + temp2;
+	return(dot);
+
+}
+
+

kernel/power/ddot_microk_power8.c

@@ -0,0 +1,178 @@
+/***************************************************************************
+Copyright (c) 2013-2016, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+/**************************************************************************************
+* 2016/03/20 Werner Saar (wernsaar@googlemail.com)
+* 	 BLASTEST 		: OK
+* 	 CTEST			: OK
+* 	 TEST			: OK
+*	 LAPACK-TEST		: OK
+**************************************************************************************/
+
+#define HAVE_KERNEL_8 1
+static void ddot_kernel_8( BLASLONG n, FLOAT *x, FLOAT *y , FLOAT *dot) __attribute__ ((noinline));
+
+static void ddot_kernel_8( BLASLONG n, FLOAT *x, FLOAT *y, FLOAT *dot)
+{
+
+
+	BLASLONG i = n;
+	BLASLONG o16 = 16;
+	BLASLONG o32 = 32;
+	BLASLONG o48 = 48;
+	BLASLONG o64 = 64;
+	BLASLONG o80 = 80;
+	BLASLONG o96 = 96;
+	BLASLONG o112 = 112;
+	FLOAT *x1=x;
+	FLOAT *y1=y;
+	BLASLONG pre = 384;
+
+	__asm__  __volatile__
+	(
+	"xxlxor		32,32,32			    \n\t"
+	"xxlxor		33,33,33			    \n\t"
+	"xxlxor		34,34,34			    \n\t"
+	"xxlxor		35,35,35			    \n\t"
+	"xxlxor		36,36,36			    \n\t"
+	"xxlxor		37,37,37			    \n\t"
+	"xxlxor		38,38,38			    \n\t"
+	"xxlxor		39,39,39			    \n\t"
+
+	"dcbt		%2, %12				    \n\t"
+	"dcbt		%3, %12				    \n\t"
+
+	"lxvd2x		40, 0, %2			    \n\t"
+	"lxvd2x		48, 0, %3			    \n\t"
+	"lxvd2x		41, %5, %2			    \n\t"
+	"lxvd2x		49, %5, %3			    \n\t"
+	"lxvd2x		42, %6, %2			    \n\t"
+	"lxvd2x		50, %6, %3			    \n\t"
+	"lxvd2x		43, %7, %2			    \n\t"
+	"lxvd2x		51, %7, %3			    \n\t"
+	"lxvd2x		44, %8, %2			    \n\t"
+	"lxvd2x		52, %8, %3			    \n\t"
+	"lxvd2x		45, %9, %2			    \n\t"
+	"lxvd2x		53, %9, %3			    \n\t"
+	"lxvd2x		46, %10, %2			    \n\t"
+	"lxvd2x		54, %10, %3			    \n\t"
+	"lxvd2x		47, %11, %2			    \n\t"
+	"lxvd2x		55, %11, %3			    \n\t"
+
+	"addi		%2, %2, 128			    \n\t"
+	"addi		%3, %3, 128			    \n\t"
+
+	"addic.		%0 , %0	, -16  	 	             \n\t"
+	"ble		2f		             	     \n\t"
+
+	".align 5				            \n\t"
+	"1:				                    \n\t"
+
+	"dcbt		%2, %12				    \n\t"
+	"dcbt		%3, %12				    \n\t"
+
+	"xvmaddadp	32, 40, 48		    \n\t"
+	"lxvd2x		40, 0, %2			    \n\t"
+	"lxvd2x		48, 0, %3			    \n\t"
+	"xvmaddadp	33, 41, 49		    \n\t"
+	"lxvd2x		41, %5, %2			    \n\t"
+	"lxvd2x		49, %5, %3			    \n\t"
+	"xvmaddadp	34, 42, 50		    \n\t"
+	"lxvd2x		42, %6, %2			    \n\t"
+	"lxvd2x		50, %6, %3			    \n\t"
+	"xvmaddadp	35, 43, 51		    \n\t"
+	"lxvd2x		43, %7, %2			    \n\t"
+	"lxvd2x		51, %7, %3			    \n\t"
+	"xvmaddadp	36, 44, 52		    \n\t"
+	"lxvd2x		44, %8, %2			    \n\t"
+	"lxvd2x		52, %8, %3			    \n\t"
+	"xvmaddadp	37, 45, 53		    \n\t"
+	"lxvd2x		45, %9, %2			    \n\t"
+	"lxvd2x		53, %9, %3			    \n\t"
+	"xvmaddadp	38, 46, 54		    \n\t"
+	"lxvd2x		46, %10, %2			    \n\t"
+	"lxvd2x		54, %10, %3			    \n\t"
+	"xvmaddadp	39, 47, 55		    \n\t"
+
+	"lxvd2x		47, %11, %2			    \n\t"
+	"lxvd2x		55, %11, %3			    \n\t"
+
+
+	"addi		%2, %2, 128			    \n\t"
+	"addi		%3, %3, 128			    \n\t"
+
+	"addic.		%0 , %0	, -16  	 	             \n\t"
+	"bgt		1b		             	     \n\t"
+
+	"2:						     \n\t"
+
+	"xvmaddadp	32, 40, 48		    \n\t"
+	"xvmaddadp	33, 41, 49		    \n\t"
+	"xvmaddadp	34, 42, 50		    \n\t"
+	"xvmaddadp	35, 43, 51		    \n\t"
+	"xvmaddadp	36, 44, 52		    \n\t"
+	"xvmaddadp	37, 45, 53		    \n\t"
+	"xvmaddadp	38, 46, 54		    \n\t"
+	"xvmaddadp	39, 47, 55		    \n\t"
+
+	"xvadddp	32, 32, 33		     \n\t"
+	"xvadddp	34, 34, 35		     \n\t"
+	"xvadddp	36, 36, 37		     \n\t"
+	"xvadddp	38, 38, 39		     \n\t"
+
+	"xvadddp	32, 32, 34		     \n\t"
+	"xvadddp	36, 36, 38		     \n\t"
+
+	"xvadddp	32, 32, 36		     \n\t"
+
+	"xxswapd	33, 32			     \n\t"
+
+	"xsadddp	32, 32, 33		     \n\t"
+
+	"stxsdx		32, 0, %4			     \n\t"
+
+	:
+        : 
+          "r" (i),	// 0	
+	  "r" (n),  	// 1
+          "r" (x1),     // 2
+          "r" (y1),     // 3
+          "r" (dot),    // 4
+	  "r" (o16),	// 5
+	  "r" (o32),	// 6
+	  "r" (o48),    // 7
+          "r" (o64),    // 8
+          "r" (o80),    // 9
+          "r" (o96),    // 10
+          "r" (o112),   // 11
+	  "r" (pre)	// 12
+	: "cr0", "%0", "%2" , "%3", "memory"
+	);
+
+} 
+
+

kernel/power/dgemv_n.c

@@ -0,0 +1,426 @@
+/***************************************************************************
+Copyright (c) 2013-2016, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+/**************************************************************************************
+* 2016/03/30 Werner Saar (wernsaar@googlemail.com)
+* 	 BLASTEST 		: OK
+* 	 CTEST			: OK
+* 	 TEST			: OK
+*	 LAPACK-TEST		: OK
+**************************************************************************************/
+
+
+
+#include "common.h"
+
+
+#if defined(POWER8)
+#include "dgemv_n_microk_power8.c"
+#endif
+
+
+#define NBMAX 4096
+
+#ifndef HAVE_KERNEL_4x4
+
+static void dgemv_kernel_4x4(BLASLONG n, FLOAT **ap, FLOAT *xo, FLOAT *y, FLOAT *alpha)
+{
+	BLASLONG i;
+	FLOAT *a0,*a1,*a2,*a3;
+	FLOAT x[4]  __attribute__ ((aligned (16)));;
+	a0 = ap[0];
+	a1 = ap[1];
+	a2 = ap[2];
+	a3 = ap[3];
+
+	for ( i=0; i<4; i++)
+		x[i] = xo[i] * *alpha;
+
+	for ( i=0; i< n; i+=4 )
+	{
+		y[i] += a0[i]*x[0] + a1[i]*x[1] + a2[i]*x[2] + a3[i]*x[3];		
+		y[i+1] += a0[i+1]*x[0] + a1[i+1]*x[1] + a2[i+1]*x[2] + a3[i+1]*x[3];		
+		y[i+2] += a0[i+2]*x[0] + a1[i+2]*x[1] + a2[i+2]*x[2] + a3[i+2]*x[3];		
+		y[i+3] += a0[i+3]*x[0] + a1[i+3]*x[1] + a2[i+3]*x[2] + a3[i+3]*x[3];		
+	}
+}
+
+#endif
+
+#ifndef HAVE_KERNEL_4x2
+
+static void dgemv_kernel_4x2(BLASLONG n, FLOAT **ap, FLOAT *xo, FLOAT *y, FLOAT *alpha)
+{
+	BLASLONG i;
+	FLOAT *a0,*a1;
+	FLOAT x[4]  __attribute__ ((aligned (16)));;
+	a0 = ap[0];
+	a1 = ap[1];
+
+	for ( i=0; i<2; i++)
+		x[i] = xo[i] * *alpha;
+
+	for ( i=0; i< n; i+=4 )
+	{
+		y[i] += a0[i]*x[0] + a1[i]*x[1];		
+		y[i+1] += a0[i+1]*x[0] + a1[i+1]*x[1];		
+		y[i+2] += a0[i+2]*x[0] + a1[i+2]*x[1];		
+		y[i+3] += a0[i+3]*x[0] + a1[i+3]*x[1];		
+	}
+}
+
+
+#endif
+
+#ifndef HAVE_KERNEL_4x1
+
+static void dgemv_kernel_4x1(BLASLONG n, FLOAT *ap, FLOAT *xo, FLOAT *y, FLOAT *alpha)
+{
+	BLASLONG i;
+	FLOAT *a0;
+	FLOAT x[4]  __attribute__ ((aligned (16)));;
+	a0 = ap;
+
+	for ( i=0; i<1; i++)
+		x[i] = xo[i] * *alpha;
+
+	for ( i=0; i< n; i+=4 )
+	{
+		y[i] += a0[i]*x[0];		
+		y[i+1] += a0[i+1]*x[0];		
+		y[i+2] += a0[i+2]*x[0];		
+		y[i+3] += a0[i+3]*x[0];		
+	}
+}
+
+
+#endif
+
+
+static void add_y(BLASLONG n, FLOAT *src, FLOAT *dest, BLASLONG inc_dest)
+{
+	BLASLONG i;
+	if ( inc_dest != 1 )
+	{
+		for ( i=0; i<n; i++ )
+		{
+			*dest += *src;
+			src++;
+			dest += inc_dest;
+		}
+		return;
+	}
+
+}
+
+int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha, FLOAT *a, BLASLONG lda, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y, FLOAT *buffer)
+{
+
+	BLASLONG i;
+	BLASLONG j;
+	FLOAT *a_ptr;
+	FLOAT *x_ptr;
+	FLOAT *y_ptr;
+	BLASLONG n1;
+	BLASLONG m1;
+	BLASLONG m2;
+	BLASLONG m3;
+	BLASLONG n2;
+	BLASLONG lda4 =  lda << 2;
+	FLOAT *ap[4] __attribute__ ((aligned (16)));;
+	FLOAT xbuffer[8] __attribute__ ((aligned (16)));;
+	FLOAT alpha_r[4] __attribute__ ((aligned (16)));;
+	FLOAT *ybuffer;
+
+	alpha_r[0] = alpha;
+
+        if ( m < 1 ) return(0);
+        if ( n < 1 ) return(0);
+
+	ybuffer = buffer;
+	
+	n1 = n >> 2 ;
+	n2 = n &  3 ;
+
+        m3 = m & 3  ;
+        m1 = m & -4 ;
+        m2 = (m & (NBMAX-1)) - m3 ;
+
+	y_ptr = y;
+
+	BLASLONG NB = NBMAX;
+
+	while ( NB == NBMAX )
+	{
+		
+		m1 -= NB;
+		if ( m1 < 0)
+		{
+			if ( m2 == 0 ) break;	
+			NB = m2;
+		}
+		
+		a_ptr = a;
+		x_ptr = x;
+		
+		ap[0] = a_ptr;
+		ap[1] = a_ptr + lda;
+		ap[2] = ap[1] + lda;
+		ap[3] = ap[2] + lda;
+
+		if ( inc_y != 1 )
+			memset(ybuffer,0,NB*8);
+		else
+			ybuffer = y_ptr;
+
+		if ( inc_x == 1 )
+		{
+
+
+			for( i = 0; i < n1 ; i++)
+			{
+				dgemv_kernel_4x4(NB,ap,x_ptr,ybuffer,alpha_r);
+				ap[0] += lda4; 
+				ap[1] += lda4; 
+				ap[2] += lda4; 
+				ap[3] += lda4; 
+				a_ptr += lda4;
+				x_ptr += 4;	
+			}
+
+			if ( n2 & 2 )
+			{
+				dgemv_kernel_4x2(NB,ap,x_ptr,ybuffer,alpha_r);
+				a_ptr += lda*2;
+				x_ptr += 2;	
+			}
+
+
+			if ( n2 & 1 )
+			{
+				dgemv_kernel_4x1(NB,a_ptr,x_ptr,ybuffer,alpha_r);
+				a_ptr += lda;
+				x_ptr += 1;	
+
+			}
+
+
+		}
+		else
+		{
+
+			for( i = 0; i < n1 ; i++)
+			{
+				xbuffer[0] = x_ptr[0];
+				x_ptr += inc_x;	
+				xbuffer[1] =  x_ptr[0];
+				x_ptr += inc_x;	
+				xbuffer[2] =  x_ptr[0];
+				x_ptr += inc_x;	
+				xbuffer[3] = x_ptr[0];
+				x_ptr += inc_x;	
+				dgemv_kernel_4x4(NB,ap,xbuffer,ybuffer,alpha_r);
+				ap[0] += lda4; 
+				ap[1] += lda4; 
+				ap[2] += lda4; 
+				ap[3] += lda4; 
+				a_ptr += lda4;
+			}
+
+			for( i = 0; i < n2 ; i++)
+			{
+				xbuffer[0] = x_ptr[0];
+				x_ptr += inc_x;	
+				dgemv_kernel_4x1(NB,a_ptr,xbuffer,ybuffer,alpha_r);
+				a_ptr += lda;
+
+			}
+
+		}
+
+		a     += NB;
+		if ( inc_y != 1 )
+		{
+			add_y(NB,ybuffer,y_ptr,inc_y);
+			y_ptr += NB * inc_y;
+		}
+		else
+			y_ptr += NB ;
+
+	}
+
+	if ( m3 == 0 ) return(0);
+
+	if ( m3 == 3 )
+	{
+		a_ptr = a;
+		x_ptr = x;
+		FLOAT temp0 = 0.0;
+		FLOAT temp1 = 0.0;
+		FLOAT temp2 = 0.0;
+		if ( lda == 3 && inc_x ==1 )
+		{
+
+			for( i = 0; i < ( n & -4 ); i+=4 )
+			{
+
+				temp0 += a_ptr[0] * x_ptr[0] + a_ptr[3] * x_ptr[1];
+				temp1 += a_ptr[1] * x_ptr[0] + a_ptr[4] * x_ptr[1];
+				temp2 += a_ptr[2] * x_ptr[0] + a_ptr[5] * x_ptr[1];
+
+				temp0 += a_ptr[6] * x_ptr[2] + a_ptr[9]  * x_ptr[3];
+				temp1 += a_ptr[7] * x_ptr[2] + a_ptr[10] * x_ptr[3];
+				temp2 += a_ptr[8] * x_ptr[2] + a_ptr[11] * x_ptr[3];
+
+				a_ptr += 12;
+				x_ptr += 4;
+			}
+
+			for( ; i < n; i++ )
+			{
+				temp0 += a_ptr[0] * x_ptr[0];
+				temp1 += a_ptr[1] * x_ptr[0];
+				temp2 += a_ptr[2] * x_ptr[0];
+				a_ptr += 3;
+				x_ptr ++;
+			}
+
+		}
+		else
+		{
+
+			for( i = 0; i < n; i++ )
+			{
+				temp0 += a_ptr[0] * x_ptr[0];
+				temp1 += a_ptr[1] * x_ptr[0];
+				temp2 += a_ptr[2] * x_ptr[0];
+				a_ptr += lda;
+				x_ptr += inc_x;
+
+
+			}
+
+		}
+		y_ptr[0] += alpha * temp0;
+		y_ptr += inc_y;
+		y_ptr[0] += alpha * temp1;
+		y_ptr += inc_y;
+		y_ptr[0] += alpha * temp2;
+		return(0);
+	}
+
+
+	if ( m3 == 2 )
+	{
+		a_ptr = a;
+		x_ptr = x;
+		FLOAT temp0 = 0.0;
+		FLOAT temp1 = 0.0;
+		if ( lda == 2 && inc_x ==1 )
+		{
+
+			for( i = 0; i < (n & -4) ; i+=4 )
+			{
+				temp0 += a_ptr[0] * x_ptr[0] + a_ptr[2] * x_ptr[1];
+				temp1 += a_ptr[1] * x_ptr[0] + a_ptr[3] * x_ptr[1];
+				temp0 += a_ptr[4] * x_ptr[2] + a_ptr[6] * x_ptr[3];
+				temp1 += a_ptr[5] * x_ptr[2] + a_ptr[7] * x_ptr[3];
+				a_ptr += 8;
+				x_ptr += 4;
+
+			}
+
+
+			for( ; i < n; i++ )
+			{
+				temp0 += a_ptr[0]   * x_ptr[0];
+				temp1 += a_ptr[1]   * x_ptr[0];
+				a_ptr += 2;
+				x_ptr ++;
+			}
+
+		}
+		else
+		{
+
+			for( i = 0; i < n; i++ )
+			{
+				temp0 += a_ptr[0] * x_ptr[0];
+				temp1 += a_ptr[1] * x_ptr[0];
+				a_ptr += lda;
+				x_ptr += inc_x;
+
+
+			}
+
+		}
+		y_ptr[0] += alpha * temp0;
+		y_ptr += inc_y;
+		y_ptr[0] += alpha * temp1;
+		return(0);
+	}
+
+	if ( m3 == 1 )
+	{
+		a_ptr = a;
+		x_ptr = x;
+		FLOAT temp = 0.0;
+		if ( lda == 1 && inc_x ==1 )
+		{
+
+			for( i = 0; i < (n & -4); i+=4 )
+			{
+				temp += a_ptr[i] * x_ptr[i] + a_ptr[i+1] * x_ptr[i+1] + a_ptr[i+2] * x_ptr[i+2] + a_ptr[i+3] * x_ptr[i+3];
+	
+			}
+
+			for( ; i < n; i++ )
+			{
+				temp += a_ptr[i] * x_ptr[i];
+			}
+
+		}
+		else
+		{
+
+			for( i = 0; i < n; i++ )
+			{
+				temp += a_ptr[0] * x_ptr[0];
+				a_ptr += lda;
+				x_ptr += inc_x;
+			}
+
+		}
+		y_ptr[0] += alpha * temp;
+		return(0);
+	}
+
+
+	return(0);
+}
+
+

kernel/power/dgemv_n_microk_power8.c

@@ -0,0 +1,301 @@
+/***************************************************************************
+Copyright (c) 2013-2016, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+/**************************************************************************************
+* 2016/03/30 Werner Saar (wernsaar@googlemail.com)
+* 	 BLASTEST 		: OK
+* 	 CTEST			: OK
+* 	 TEST			: OK
+*	 LAPACK-TEST		: OK
+**************************************************************************************/
+
+#define HAVE_KERNEL_4x4 1
+
+static void dgemv_kernel_4x4(BLASLONG n, FLOAT **ap, FLOAT *xo, FLOAT *y, FLOAT *alpha) __attribute__ ((noinline));
+
+static void dgemv_kernel_4x4(BLASLONG n, FLOAT **ap, FLOAT *xo, FLOAT *y, FLOAT *alpha)
+{
+        BLASLONG i=n;
+	BLASLONG o8  = 8;
+	BLASLONG o16 = 16;
+	BLASLONG o24 = 24;
+	BLASLONG pre = 384;
+
+        FLOAT *a0,*a1,*a2,*a3;
+	FLOAT *y1=y+1;
+        FLOAT x[4]  __attribute__ ((aligned (16)));;
+        a0 = ap[0]+1;
+        a1 = ap[1]+1;
+        a2 = ap[2]+1;
+        a3 = ap[3]+1;
+
+	x[0]=xo[0] * *alpha;
+	x[1]=xo[1] * *alpha;
+	x[2]=xo[2] * *alpha;
+	x[3]=xo[3] * *alpha;
+
+
+	__asm__  __volatile__
+	(
+	"lxvdsx		32, 0 , %1			    \n\t"	// x0
+	"lxvdsx		33,%3 , %1			    \n\t"	// x1
+	"lxvdsx		34,%4 , %1			    \n\t"	// x2
+	"lxvdsx		35,%5 , %1			    \n\t"	// x3
+	"addi		%2 , %2 , -8			    \n\t"
+	"addi		%6 , %6 , -8			    \n\t"
+	"addi		%7 , %7 , -8			    \n\t"
+	"addi		%8 , %8 , -8			    \n\t"
+	"addi		%9 , %9 , -8			    \n\t"
+	
+	"lxvd2x		48, 0, %6			    \n\t"	// a0[0], a0[1] 
+	"lxvd2x		49,%4, %6			    \n\t"	// a0[2], a0[3] 
+
+	"lxvd2x		50, 0, %7			    \n\t"	// a1[0], a1[1] 
+	"lxvd2x		51,%4, %7			    \n\t"	// a1[2], a1[3] 
+
+	"lxvd2x		52, 0, %8			    \n\t"	// a2[0], a2[1] 
+	"lxvd2x		53,%4, %8			    \n\t"	// a2[2], a2[3] 
+
+	"lxvd2x		54, 0, %9			    \n\t"	// a3[0], a3[1] 
+	"lxvd2x		55,%4, %9			    \n\t"	// a3[2], a3[3] 
+
+	"addi		%6, %6, 32			    \n\t"
+	"addi		%7, %7, 32			    \n\t"
+	"addi		%8, %8, 32			    \n\t"
+	"addi		%9, %9, 32			    \n\t"
+
+	"addic.		%0 , %0	, -4  	 	             \n\t"
+	"ble		2f		             	     \n\t"
+
+	".align 5				            \n\t"
+	"1:				                    \n\t"
+
+	"dcbt		%2, %10				    \n\t"
+
+	"lxvd2x		40, 0, %2			    \n\t"	// y0, y1
+	"lxvd2x		41,%4, %2			    \n\t"	// y2, y3
+	
+	"dcbt		%6, %10				    \n\t"
+	"dcbt		%7, %10				    \n\t"
+	"dcbt		%8, %10				    \n\t"
+	"dcbt		%9, %10				    \n\t"
+
+	"xvmaddadp	40, 48, 32			    \n\t"
+	"xvmaddadp	41, 49, 32			    \n\t"
+
+	"lxvd2x		48, 0, %6			    \n\t"	// a0[0], a0[1] 
+	"lxvd2x		49,%4, %6			    \n\t"	// a0[2], a0[3] 
+
+	"xvmaddadp	40, 50, 33			    \n\t"
+	"addi		%6, %6, 32			    \n\t"
+	"xvmaddadp	41, 51, 33			    \n\t"
+
+	"lxvd2x		50, 0, %7			    \n\t"	// a1[0], a1[1] 
+	"lxvd2x		51,%4, %7			    \n\t"	// a1[2], a1[3] 
+
+	"xvmaddadp	40, 52, 34			    \n\t"
+	"addi		%7, %7, 32			    \n\t"
+	"xvmaddadp	41, 53, 34			    \n\t"
+
+	"lxvd2x		52, 0, %8			    \n\t"	// a2[0], a2[1] 
+	"lxvd2x		53,%4, %8			    \n\t"	// a2[2], a2[3] 
+
+	"xvmaddadp	40, 54, 35			    \n\t"
+	"addi		%8, %8, 32			    \n\t"
+	"xvmaddadp	41, 55, 35			    \n\t"
+
+	"stxvd2x	40, 0, %2			    \n\t"	// y0, y1
+	"stxvd2x	41,%4, %2			    \n\t"	// y2, y3
+
+	"lxvd2x		54, 0, %9			    \n\t"	// a3[0], a3[1] 
+	"lxvd2x		55,%4, %9			    \n\t"	// a3[2], a3[3] 
+
+	"addi		%9, %9, 32			    \n\t"
+	"addi		%2, %2, 32			    \n\t"
+
+	"addic.		%0 , %0	, -4  	 	             \n\t"
+	"ble		2f		             	     \n\t"
+
+
+	"lxvd2x		40, 0, %2			    \n\t"	// y0, y1
+	"lxvd2x		41,%4, %2			    \n\t"	// y2, y3
+	
+	"xvmaddadp	40, 48, 32			    \n\t"
+	"xvmaddadp	41, 49, 32			    \n\t"
+
+	"lxvd2x		48, 0, %6			    \n\t"	// a0[0], a0[1] 
+	"lxvd2x		49,%4, %6			    \n\t"	// a0[2], a0[3] 
+
+	"xvmaddadp	40, 50, 33			    \n\t"
+	"addi		%6, %6, 32			    \n\t"
+	"xvmaddadp	41, 51, 33			    \n\t"
+
+	"lxvd2x		50, 0, %7			    \n\t"	// a1[0], a1[1] 
+	"lxvd2x		51,%4, %7			    \n\t"	// a1[2], a1[3] 
+
+	"xvmaddadp	40, 52, 34			    \n\t"
+	"addi		%7, %7, 32			    \n\t"
+	"xvmaddadp	41, 53, 34			    \n\t"
+
+	"lxvd2x		52, 0, %8			    \n\t"	// a2[0], a2[1] 
+	"lxvd2x		53,%4, %8			    \n\t"	// a2[2], a2[3] 
+
+	"xvmaddadp	40, 54, 35			    \n\t"
+	"addi		%8, %8, 32			    \n\t"
+	"xvmaddadp	41, 55, 35			    \n\t"
+
+	"stxvd2x	40, 0, %2			    \n\t"	// y0, y1
+	"stxvd2x	41,%4, %2			    \n\t"	// y2, y3
+
+	"lxvd2x		54, 0, %9			    \n\t"	// a3[0], a3[1] 
+	"lxvd2x		55,%4, %9			    \n\t"	// a3[2], a3[3] 
+
+	"addi		%9, %9, 32			    \n\t"
+	"addi		%2, %2, 32			    \n\t"
+
+	"addic.		%0 , %0	, -4  	 	             \n\t"
+	"ble		2f		             	     \n\t"
+
+
+	"lxvd2x		40, 0, %2			    \n\t"	// y0, y1
+	"lxvd2x		41,%4, %2			    \n\t"	// y2, y3
+	
+	"xvmaddadp	40, 48, 32			    \n\t"
+	"xvmaddadp	41, 49, 32			    \n\t"
+
+	"lxvd2x		48, 0, %6			    \n\t"	// a0[0], a0[1] 
+	"lxvd2x		49,%4, %6			    \n\t"	// a0[2], a0[3] 
+
+	"xvmaddadp	40, 50, 33			    \n\t"
+	"addi		%6, %6, 32			    \n\t"
+	"xvmaddadp	41, 51, 33			    \n\t"
+
+	"lxvd2x		50, 0, %7			    \n\t"	// a1[0], a1[1] 
+	"lxvd2x		51,%4, %7			    \n\t"	// a1[2], a1[3] 
+
+	"xvmaddadp	40, 52, 34			    \n\t"
+	"addi		%7, %7, 32			    \n\t"
+	"xvmaddadp	41, 53, 34			    \n\t"
+
+	"lxvd2x		52, 0, %8			    \n\t"	// a2[0], a2[1] 
+	"lxvd2x		53,%4, %8			    \n\t"	// a2[2], a2[3] 
+
+	"xvmaddadp	40, 54, 35			    \n\t"
+	"addi		%8, %8, 32			    \n\t"
+	"xvmaddadp	41, 55, 35			    \n\t"
+
+	"stxvd2x	40, 0, %2			    \n\t"	// y0, y1
+	"stxvd2x	41,%4, %2			    \n\t"	// y2, y3
+
+	"lxvd2x		54, 0, %9			    \n\t"	// a3[0], a3[1] 
+	"lxvd2x		55,%4, %9			    \n\t"	// a3[2], a3[3] 
+
+	"addi		%9, %9, 32			    \n\t"
+	"addi		%2, %2, 32			    \n\t"
+
+	"addic.		%0 , %0	, -4  	 	             \n\t"
+	"ble		2f		             	     \n\t"
+
+
+	"lxvd2x		40, 0, %2			    \n\t"	// y0, y1
+	"lxvd2x		41,%4, %2			    \n\t"	// y2, y3
+	
+	"xvmaddadp	40, 48, 32			    \n\t"
+	"xvmaddadp	41, 49, 32			    \n\t"
+
+	"lxvd2x		48, 0, %6			    \n\t"	// a0[0], a0[1] 
+	"lxvd2x		49,%4, %6			    \n\t"	// a0[2], a0[3] 
+
+	"xvmaddadp	40, 50, 33			    \n\t"
+	"addi		%6, %6, 32			    \n\t"
+	"xvmaddadp	41, 51, 33			    \n\t"
+
+	"lxvd2x		50, 0, %7			    \n\t"	// a1[0], a1[1] 
+	"lxvd2x		51,%4, %7			    \n\t"	// a1[2], a1[3] 
+
+	"xvmaddadp	40, 52, 34			    \n\t"
+	"addi		%7, %7, 32			    \n\t"
+	"xvmaddadp	41, 53, 34			    \n\t"
+
+	"lxvd2x		52, 0, %8			    \n\t"	// a2[0], a2[1] 
+	"lxvd2x		53,%4, %8			    \n\t"	// a2[2], a2[3] 
+
+	"xvmaddadp	40, 54, 35			    \n\t"
+	"addi		%8, %8, 32			    \n\t"
+	"xvmaddadp	41, 55, 35			    \n\t"
+
+	"stxvd2x	40, 0, %2			    \n\t"	// y0, y1
+	"stxvd2x	41,%4, %2			    \n\t"	// y2, y3
+
+	"lxvd2x		54, 0, %9			    \n\t"	// a3[0], a3[1] 
+	"lxvd2x		55,%4, %9			    \n\t"	// a3[2], a3[3] 
+
+	"addi		%9, %9, 32			    \n\t"
+	"addi		%2, %2, 32			    \n\t"
+
+	"addic.		%0 , %0	, -4  	 	             \n\t"
+	"bgt		1b		             	     \n\t"
+
+	"2:						     \n\t"
+
+	"lxvd2x		40, 0, %2			    \n\t"	// y0, y1
+	"lxvd2x		41,%4, %2			    \n\t"	// y2, y3
+
+	"xvmaddadp	40, 48, 32			    \n\t"
+	"xvmaddadp	41, 49, 32			    \n\t"
+
+	"xvmaddadp	40, 50, 33			    \n\t"
+	"xvmaddadp	41, 51, 33			    \n\t"
+
+	"xvmaddadp	40, 52, 34			    \n\t"
+	"xvmaddadp	41, 53, 34			    \n\t"
+
+	"xvmaddadp	40, 54, 35			    \n\t"
+	"xvmaddadp	41, 55, 35			    \n\t"
+
+	"stxvd2x	40, 0, %2			    \n\t"	// y0, y1
+	"stxvd2x	41,%4, %2			    \n\t"	// y2, y3
+
+	:
+        : 
+          "r" (i),	// 0	
+          "r" (x),      // 1
+          "r" (y1),     // 2
+	  "r" (o8),	// 3
+	  "r" (o16),	// 4
+	  "r" (o24),	// 5
+	  "r" (a0),	// 6
+	  "r" (a1),	// 7
+	  "r" (a2),	// 8
+	  "r" (a3),	// 9
+	  "r" (pre)	// 10
+	: "cr0", "%0", "%2" , "%6", "%7", "%8", "%9", "memory"
+	);
+
+} 
+
+

kernel/power/drot.c

@@ -0,0 +1,167 @@
+/***************************************************************************
+Copyright (c) 2013-2016, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+/**************************************************************************************
+* 2016/03/27 Werner Saar (wernsaar@googlemail.com)
+* 	 BLASTEST 		: OK
+* 	 CTEST			: OK
+* 	 TEST			: OK
+*	 LAPACK-TEST		: OK
+**************************************************************************************/
+
+
+
+#include "common.h"
+
+#pragma GCC optimize "O1"
+
+#if defined(POWER8)
+#include "drot_microk_power8.c"
+#endif
+
+
+#ifndef HAVE_KERNEL_16
+
+static void drot_kernel_16(BLASLONG n, FLOAT *x, FLOAT *y, FLOAT *c, FLOAT *s)
+{
+
+	BLASLONG i=0;
+	FLOAT f0, f1, f2, f3;
+	FLOAT x00, x01, x02, x03;
+	FLOAT g0, g1, g2, g3;
+	FLOAT y00, y01, y02, y03;
+	FLOAT *x1=x;
+	FLOAT *y1=y;
+	FLOAT c1=*c;
+	FLOAT s1=*s;
+
+	while ( i<n )
+	{
+
+		x00 = x1[0];
+		y00 = y1[0];
+		x01 = x1[1];
+		y01 = y1[1];
+		x02 = x1[2];
+		y02 = y1[2];
+		x03 = x1[3];
+		y03 = y1[3];
+
+		f0 = c1*x00 + s1*y00;
+		g0 = c1*y00 - s1*x00;
+		f1 = c1*x01 + s1*y01;
+		g1 = c1*y01 - s1*x01;
+		f2 = c1*x02 + s1*y02;
+		g2 = c1*y02 - s1*x02;
+		f3 = c1*x03 + s1*y03;
+		g3 = c1*y03 - s1*x03;
+
+		x1[0] = f0;
+		y1[0] = g0;
+		x1[1] = f1;
+		y1[1] = g1;
+		x1[2] = f2;
+		y1[2] = g2;
+		x1[3] = f3;
+		y1[3] = g3;
+
+		x1 += 4;
+		y1 += 4;
+
+		i+=4;
+	}
+	return;
+
+}
+
+
+#endif
+
+
+int CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y, FLOAT c, FLOAT s)
+{
+	BLASLONG i=0;
+	BLASLONG ix=0,iy=0;
+	FLOAT c1[4] __attribute__ ((aligned (16)));;
+	FLOAT s1[4] __attribute__ ((aligned (16)));;
+	FLOAT *x1=x;
+	FLOAT *y1=y;
+	FLOAT temp;
+
+	if ( n <= 0     )  return(0);
+
+	if ( (inc_x == 1) && (inc_y == 1) )
+	{
+
+		BLASLONG n1 = n & -16;
+		if ( n1 > 0 )
+		{
+			c1[0]=c;
+			c1[1]=c;
+			c1[2]=c;
+			c1[3]=c;
+			s1[0]=s;
+			s1[1]=s;
+			s1[2]=s;
+			s1[3]=s;
+			drot_kernel_16(n1, x1, y1, c1, s1);
+			i=n1;
+		}
+
+		while(i < n)
+		{
+			temp  = c*x[i] + s*y[i] ;
+			y[i]  = c*y[i] - s*x[i] ;
+			x[i]  = temp ;
+
+			i++ ;
+
+		}
+
+
+	}
+	else
+	{
+
+		while(i < n)
+		{
+			temp   = c*x[ix] + s*y[iy] ;
+			y[iy]  = c*y[iy] - s*x[ix] ;
+			x[ix]  = temp ;
+
+			ix += inc_x ;
+			iy += inc_y ;
+			i++ ;
+
+		}
+
+	}
+	return(0);
+
+}
+
+

kernel/power/drot_microk_power8.c

@@ -0,0 +1,211 @@
+/***************************************************************************
+Copyright (c) 2013-2016, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+/**************************************************************************************
+* 2016/03/27 Werner Saar (wernsaar@googlemail.com)
+*
+* I don't use fused multiply-add ( precision problems with lapack )
+*
+* 	 BLASTEST 		: OK
+* 	 CTEST			: OK
+* 	 TEST			: OK
+*	 LAPACK-TEST		: OK
+**************************************************************************************/
+
+#define HAVE_KERNEL_16 1
+
+static void drot_kernel_16( BLASLONG n, FLOAT *x, FLOAT *y, FLOAT *c, FLOAT *s) __attribute__ ((noinline));
+
+static void drot_kernel_16( BLASLONG n, FLOAT *x, FLOAT *y, FLOAT *c, FLOAT *s)
+{
+
+
+	BLASLONG i = n;
+	BLASLONG o16 = 16;
+	BLASLONG o32 = 32;
+	BLASLONG o48 = 48;
+	FLOAT *x1=x;
+	FLOAT *y1=y;
+	FLOAT *x2=x+1;
+	FLOAT *y2=y+1;
+
+	__asm__  __volatile__
+	(
+
+        "lxsdx          36 , %5, %3                          \n\t"	// load c
+        "lxsdx          37 , %5, %4                          \n\t"	// load s
+	"addi		%8 , %8, -8			     \n\t"
+	"addi		%9 , %9, -8			     \n\t"
+
+        "xxspltd        36 , 36, 0                           \n\t"
+        "xxspltd        37 , 37, 0                           \n\t"
+
+	"lxvd2x		32, 0, %1			    \n\t"	// load x
+	"lxvd2x		33, %5, %1			    \n\t"
+	"lxvd2x		34, %6, %1			    \n\t"
+	"lxvd2x		35, %7, %1			    \n\t"
+
+	"lxvd2x		40, 0, %2			    \n\t"	// load y
+	"lxvd2x		41, %5, %2			    \n\t"
+	"lxvd2x		42, %6, %2			    \n\t"
+	"lxvd2x		43, %7, %2			    \n\t"
+
+	"addi		%1, %1, 64			    \n\t"
+	"addi		%2, %2, 64			    \n\t"
+
+	"addic.		%0 , %0	, -8  	 	             \n\t"
+	"ble		2f		             	     \n\t"
+
+	".align 5				            \n\t"
+	"1:				                    \n\t"
+
+	"xvmuldp	48, 32, 36		    	    \n\t"	// c * x
+	"xvmuldp	49, 33, 36		    	    \n\t"
+	"xvmuldp	50, 34, 36		    	    \n\t"
+	"xvmuldp	51, 35, 36		    	    \n\t"
+
+	"xvmuldp	56, 40, 36		    	    \n\t"	// c * y
+	"xvmuldp	57, 41, 36		    	    \n\t"
+	"xvmuldp	58, 42, 36		    	    \n\t"
+	"xvmuldp	59, 43, 36		    	    \n\t"
+
+	"xvmuldp	52, 32, 37		    	    \n\t"	// s * x
+	"xvmuldp	53, 33, 37		    	    \n\t"
+
+	"lxvd2x		32, 0, %1			    \n\t"	// load x
+	"lxvd2x		33, %5, %1			    \n\t"
+
+	"xvmuldp	54, 34, 37		    	    \n\t"
+	"xvmuldp	55, 35, 37		    	    \n\t"
+
+	"lxvd2x		34, %6, %1			    \n\t"
+	"lxvd2x		35, %7, %1			    \n\t"
+
+	"xvmuldp	60, 40, 37		    	    \n\t"	// s * y
+	"xvmuldp	61, 41, 37		    	    \n\t"
+
+	"lxvd2x		40, 0, %2			    \n\t"	// load y
+	"lxvd2x		41, %5, %2			    \n\t"
+
+	"xvmuldp	62, 42, 37		    	    \n\t"
+	"xvmuldp	63, 43, 37		    	    \n\t"
+
+	"lxvd2x		42, %6, %2			    \n\t"
+	"lxvd2x		43, %7, %2			    \n\t"
+
+	"xvadddp	48, 48 , 60			    \n\t"	// c * x + s * y 
+	"xvadddp	49, 49 , 61			    \n\t"	// c * x + s * y 
+
+	"addi		%1, %1, 64			    \n\t"
+	"addi		%2, %2, 64			    \n\t"
+
+	"xvadddp	50, 50 , 62			    \n\t"	// c * x + s * y 
+	"xvadddp	51, 51 , 63			    \n\t"	// c * x + s * y 
+
+	"xvsubdp	56, 56 , 52			    \n\t"	// c * y - s * x
+	"xvsubdp	57, 57 , 53			    \n\t"	// c * y - s * x
+	"xvsubdp	58, 58 , 54			    \n\t"	// c * y - s * x
+	"xvsubdp	59, 59 , 55			    \n\t"	// c * y - s * x
+
+	"stxvd2x	48, 0, %8			    \n\t"	// store x
+	"stxvd2x	49, %5, %8			    \n\t"
+	"stxvd2x	50, %6, %8			    \n\t"
+	"stxvd2x	51, %7, %8			    \n\t"
+
+	"stxvd2x	56, 0, %9			    \n\t"	// store y
+	"stxvd2x	57, %5, %9			    \n\t"
+	"stxvd2x	58, %6, %9			    \n\t"
+	"stxvd2x	59, %7, %9			    \n\t"
+
+	"addi		%8, %8, 64			    \n\t"
+	"addi		%9, %9, 64			    \n\t"
+
+	"addic.		%0 , %0	, -8  	 	             \n\t"
+	"bgt		1b		             	     \n\t"
+
+	"2:						     \n\t"
+
+	"xvmuldp	48, 32, 36		    	    \n\t"	// c * x
+	"xvmuldp	49, 33, 36		    	    \n\t"
+	"xvmuldp	50, 34, 36		    	    \n\t"
+	"xvmuldp	51, 35, 36		    	    \n\t"
+
+	"xvmuldp	56, 40, 36		    	    \n\t"	// c * y
+	"xvmuldp	57, 41, 36		    	    \n\t"
+	"xvmuldp	58, 42, 36		    	    \n\t"
+	"xvmuldp	59, 43, 36		    	    \n\t"
+
+	"xvmuldp	52, 32, 37		    	    \n\t"	// s * x
+	"xvmuldp	53, 33, 37		    	    \n\t"
+	"xvmuldp	54, 34, 37		    	    \n\t"
+	"xvmuldp	55, 35, 37		    	    \n\t"
+
+	"xvmuldp	60, 40, 37		    	    \n\t"	// s * y
+	"xvmuldp	61, 41, 37		    	    \n\t"
+	"xvmuldp	62, 42, 37		    	    \n\t"
+	"xvmuldp	63, 43, 37		    	    \n\t"
+
+	"xvadddp	48, 48 , 60			    \n\t"	// c * x + s * y 
+	"xvadddp	49, 49 , 61			    \n\t"	// c * x + s * y 
+	"xvadddp	50, 50 , 62			    \n\t"	// c * x + s * y 
+	"xvadddp	51, 51 , 63			    \n\t"	// c * x + s * y 
+
+	"xvsubdp	56, 56 , 52			    \n\t"	// c * y - s * x
+	"xvsubdp	57, 57 , 53			    \n\t"	// c * y - s * x
+	"xvsubdp	58, 58 , 54			    \n\t"	// c * y - s * x
+	"xvsubdp	59, 59 , 55			    \n\t"	// c * y - s * x
+
+	"stxvd2x	48, 0, %8			    \n\t"	// store x
+	"stxvd2x	49, %5, %8			    \n\t"
+	"stxvd2x	50, %6, %8			    \n\t"
+	"stxvd2x	51, %7, %8			    \n\t"
+
+	"stxvd2x	56, 0, %9			    \n\t"	// store y
+	"stxvd2x	57, %5, %9			    \n\t"
+	"stxvd2x	58, %6, %9			    \n\t"
+	"stxvd2x	59, %7, %9			    \n\t"
+
+
+
+	:
+        : 
+          "r" (i),	// 0	
+	  "r" (x1),  	// 1
+          "r" (y1),     // 2
+          "r" (c),      // 3
+          "r" (s),      // 4
+	  "r" (o16),	// 5
+	  "r" (o32),	// 6
+	  "r" (o48),    // 7
+	  "r" (x2),     // 8
+	  "r" (y2)      // 9
+	: "cr0", "%0", "%1" , "%2", "%8", "%9", "memory"
+	);
+
+} 
+
+

kernel/power/dscal.c

@@ -0,0 +1,174 @@
+/***************************************************************************
+Copyright (c) 2013-2016, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+/**************************************************************************************
+* 2016/03/25 Werner Saar (wernsaar@googlemail.com)
+* 	 BLASTEST 		: OK
+* 	 CTEST			: OK
+* 	 TEST			: OK
+*	 LAPACK-TEST		: OK
+**************************************************************************************/
+
+#include "common.h"
+
+#if defined(POWER8) 
+#include "dscal_microk_power8.c"
+#endif
+
+#if !defined(HAVE_KERNEL_8)
+
+static void dscal_kernel_8( BLASLONG n, FLOAT *da , FLOAT *x )
+{
+
+        BLASLONG i;
+        FLOAT alpha = *da;
+
+        for( i=0; i<n; i+=8 )
+        {
+                x[0] *= alpha;
+                x[1] *= alpha;
+                x[2] *= alpha;
+                x[3] *= alpha;
+                x[4] *= alpha;
+                x[5] *= alpha;
+                x[6] *= alpha;
+                x[7] *= alpha;
+                x+=8;
+        }
+
+}
+
+static void dscal_kernel_8_zero( BLASLONG n, FLOAT *da , FLOAT *x )
+{
+
+        BLASLONG i;
+	FLOAT alpha=0.0;
+
+        for( i=0; i<n; i+=8 )
+        {
+                x[0] = alpha;
+                x[1] = alpha;
+                x[2] = alpha;
+                x[3] = alpha;
+                x[4] = alpha;
+                x[5] = alpha;
+                x[6] = alpha;
+                x[7] = alpha;
+                x+=8;
+        }
+
+}
+
+
+#endif
+
+int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y, FLOAT *dummy, BLASLONG dummy2)
+{
+	BLASLONG i=0,j=0;
+	if ( n <= 0 || inc_x <=0 )
+		return(0);
+
+
+	if ( inc_x == 1 )
+	{
+
+		if ( da == 0.0 )
+		{		
+
+			BLASLONG n1 = n & -16;
+			if ( n1 > 0 )
+			{
+				FLOAT alpha[2];
+				alpha[0]=da;
+				alpha[1]=da;
+				dscal_kernel_8_zero(n1 , alpha , x);
+				j=n1;
+			}
+
+			while(j < n)
+			{
+
+				x[j]=0.0;
+				j++;
+			}
+
+		}
+		else
+		{
+
+			BLASLONG n1 = n & -16;
+			if ( n1 > 0 )
+			{
+				FLOAT alpha[2];
+				alpha[0]=da;
+				alpha[1]=da;
+				dscal_kernel_8(n1 , alpha , x);
+				j=n1;
+			}
+			while(j < n)
+			{
+
+				x[j] = da * x[j] ;
+				j++;
+			}
+		}
+
+
+	}
+	else
+	{
+
+		if ( da == 0.0 )
+		{		
+
+			while(j < n)
+			{
+
+				x[i]=0.0;
+				i += inc_x ;
+				j++;
+			}
+
+		}
+		else
+		{
+
+			while(j < n)
+			{
+
+				x[i] = da * x[i] ;
+				i += inc_x ;
+				j++;
+			}
+		}
+
+	}
+	return 0;
+
+}
+
+

kernel/power/dscal_microk_power8.c

@@ -0,0 +1,219 @@
+/***************************************************************************
+Copyright (c) 2013-2016, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+/**************************************************************************************
+* 2016/03/25 Werner Saar (wernsaar@googlemail.com)
+* 	 BLASTEST 		: OK
+* 	 CTEST			: OK
+* 	 TEST			: OK
+*	 LAPACK-TEST		: OK
+**************************************************************************************/
+
+#define HAVE_KERNEL_8 1
+
+static void dscal_kernel_8( BLASLONG n, FLOAT *alpha, FLOAT *x) __attribute__ ((noinline));
+
+static void dscal_kernel_8( BLASLONG n, FLOAT *alpha, FLOAT *x)
+{
+
+
+	BLASLONG i = n;
+	BLASLONG o16 = 16;
+	BLASLONG o32 = 32;
+	BLASLONG o48 = 48;
+	BLASLONG o64 = 64;
+	BLASLONG o80 = 80;
+	BLASLONG o96 = 96;
+	BLASLONG o112 = 112;
+	FLOAT *x1=x;
+	FLOAT *x2=x+1;
+	BLASLONG pre = 384;
+
+	__asm__  __volatile__
+	(
+
+        "lxsdx          33, 0, %3                           \n\t"
+        "xxspltd        32, 33, 0                           \n\t"
+        "addi           %1, %1, -8                          \n\t"
+
+	"dcbt		%2, %4				    \n\t"
+
+	"lxvd2x		40, 0, %2			    \n\t"
+	"lxvd2x		41, %5, %2			    \n\t"
+	"lxvd2x		42, %6, %2			    \n\t"
+	"lxvd2x		43, %7, %2			    \n\t"
+	"lxvd2x		44, %8, %2			    \n\t"
+	"lxvd2x		45, %9, %2			    \n\t"
+	"lxvd2x		46, %10, %2			    \n\t"
+	"lxvd2x		47, %11, %2			    \n\t"
+
+	"addi		%2, %2, 128			    \n\t"
+
+	"addic.		%0 , %0	, -16  	 	             \n\t"
+	"ble		2f		             	     \n\t"
+
+	".align 5				            \n\t"
+	"1:				                    \n\t"
+
+	"dcbt		%2, %4				    \n\t"
+
+	"xvmuldp	48, 40, 32		    	    \n\t"
+	"xvmuldp	49, 41, 32		    	    \n\t"
+	"lxvd2x		40, 0, %2			    \n\t"
+	"lxvd2x		41, %5, %2			    \n\t"
+	"xvmuldp	50, 42, 32		    	    \n\t"
+	"xvmuldp	51, 43, 32		    	    \n\t"
+	"lxvd2x		42, %6, %2			    \n\t"
+	"lxvd2x		43, %7, %2			    \n\t"
+	"xvmuldp	52, 44, 32		    	    \n\t"
+	"xvmuldp	53, 45, 32		    	    \n\t"
+	"lxvd2x		44, %8, %2			    \n\t"
+	"lxvd2x		45, %9, %2			    \n\t"
+	"xvmuldp	54, 46, 32		    	    \n\t"
+	"xvmuldp	55, 47, 32		    	    \n\t"
+	"lxvd2x		46, %10, %2			    \n\t"
+	"lxvd2x		47, %11, %2			    \n\t"
+
+	"stxvd2x	48, 0, %1			    \n\t"
+	"stxvd2x	49, %5, %1			    \n\t"
+	"stxvd2x	50, %6, %1			    \n\t"
+	"stxvd2x	51, %7, %1			    \n\t"
+	"stxvd2x	52, %8, %1			    \n\t"
+	"stxvd2x	53, %9, %1			    \n\t"
+	"stxvd2x	54, %10, %1			    \n\t"
+	"stxvd2x	55, %11, %1			    \n\t"
+
+	"addi		%1, %1, 128			    \n\t"
+	"addi		%2, %2, 128			    \n\t"
+
+	"addic.		%0 , %0	, -16  	 	             \n\t"
+	"bgt		1b		             	     \n\t"
+
+	"2:						     \n\t"
+
+	"xvmuldp	48, 40, 32		    	    \n\t"
+	"xvmuldp	49, 41, 32		    	    \n\t"
+	"xvmuldp	50, 42, 32		    	    \n\t"
+	"xvmuldp	51, 43, 32		    	    \n\t"
+	"xvmuldp	52, 44, 32		    	    \n\t"
+	"xvmuldp	53, 45, 32		    	    \n\t"
+	"xvmuldp	54, 46, 32		    	    \n\t"
+	"xvmuldp	55, 47, 32		    	    \n\t"
+
+	"stxvd2x	48, 0, %1			    \n\t"
+	"stxvd2x	49, %5, %1			    \n\t"
+	"stxvd2x	50, %6, %1			    \n\t"
+	"stxvd2x	51, %7, %1			    \n\t"
+	"stxvd2x	52, %8, %1			    \n\t"
+	"stxvd2x	53, %9, %1			    \n\t"
+	"stxvd2x	54, %10, %1			    \n\t"
+	"stxvd2x	55, %11, %1			    \n\t"
+
+	:
+        : 
+          "r" (i),	// 0	
+	  "r" (x2),  	// 1
+          "r" (x1),     // 2
+          "r" (alpha),  // 3
+          "r" (pre),    // 4
+	  "r" (o16),	// 5
+	  "r" (o32),	// 6
+	  "r" (o48),    // 7
+          "r" (o64),    // 8
+          "r" (o80),    // 9
+          "r" (o96),    // 10
+          "r" (o112)    // 11
+	: "cr0", "%0", "%2" , "%1", "memory"
+	);
+
+} 
+
+
+static void dscal_kernel_8_zero( BLASLONG n, FLOAT *alpha, FLOAT *x) __attribute__ ((noinline));
+
+static void dscal_kernel_8_zero( BLASLONG n, FLOAT *alpha, FLOAT *x)
+{
+
+
+	BLASLONG i = n;
+	BLASLONG o16 = 16;
+	BLASLONG o32 = 32;
+	BLASLONG o48 = 48;
+	BLASLONG o64 = 64;
+	BLASLONG o80 = 80;
+	BLASLONG o96 = 96;
+	BLASLONG o112 = 112;
+	FLOAT *x1=x;
+	FLOAT *x2=x+1;
+	BLASLONG pre = 384;
+
+	__asm__  __volatile__
+	(
+
+	"xxlxor		32 , 32 , 32			    \n\t"
+        "addi           %1, %1, -8                          \n\t"
+
+
+	".align 5				            \n\t"
+	"1:				                    \n\t"
+
+	"stxvd2x	32, 0, %1			    \n\t"
+	"stxvd2x	32, %5, %1			    \n\t"
+	"stxvd2x	32, %6, %1			    \n\t"
+	"stxvd2x	32, %7, %1			    \n\t"
+	"stxvd2x	32, %8, %1			    \n\t"
+	"stxvd2x	32, %9, %1			    \n\t"
+	"stxvd2x	32, %10, %1			    \n\t"
+	"stxvd2x	32, %11, %1			    \n\t"
+
+	"addi		%1, %1, 128			    \n\t"
+
+	"addic.		%0 , %0	, -16  	 	             \n\t"
+	"bgt		1b		             	     \n\t"
+
+	"2:						     \n\t"
+
+	:
+        : 
+          "r" (i),	// 0	
+	  "r" (x2),  	// 1
+          "r" (x1),     // 2
+          "r" (alpha),  // 3
+          "r" (pre),    // 4
+	  "r" (o16),	// 5
+	  "r" (o32),	// 6
+	  "r" (o48),    // 7
+          "r" (o64),    // 8
+          "r" (o80),    // 9
+          "r" (o96),    // 10
+          "r" (o112)    // 11
+	: "cr0", "%0", "%2" , "%1", "memory"
+	);
+
+} 
+
+

kernel/power/dswap.c

@@ -0,0 +1,154 @@
+/***************************************************************************
+Copyright (c) 2013-2016, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+/**************************************************************************************
+* 2016/03/25 Werner Saar (wernsaar@googlemail.com)
+* 	 BLASTEST 		: OK
+* 	 CTEST			: OK
+* 	 TEST			: OK
+*	 LAPACK-TEST		: OK
+**************************************************************************************/
+
+#include "common.h"
+
+#if defined(POWER8)
+#include "dswap_microk_power8.c"
+#endif
+
+#ifndef HAVE_KERNEL_32
+
+static void dswap_kernel_32(BLASLONG n, FLOAT *x, FLOAT *y)
+{
+
+	BLASLONG i=0;
+	FLOAT f0, f1, f2, f3, f4, f5, f6, f7;
+	FLOAT g0, g1, g2, g3, g4, g5, g6, g7;
+	FLOAT *x1=x;
+	FLOAT *y1=y;
+
+	while ( i<n )
+	{
+
+		f0 = x1[0];
+		f1 = x1[1];
+		f2 = x1[2];
+		f3 = x1[3];
+		f4 = x1[4];
+		f5 = x1[5];
+		f6 = x1[6];
+		f7 = x1[7];
+
+		g0 = y1[0];
+		g1 = y1[1];
+		g2 = y1[2];
+		g3 = y1[3];
+		g4 = y1[4];
+		g5 = y1[5];
+		g6 = y1[6];
+		g7 = y1[7];
+
+		y1[0] = f0;
+		y1[1] = f1;
+		y1[2] = f2;
+		y1[3] = f3;
+		y1[4] = f4;
+		y1[5] = f5;
+		y1[6] = f6;
+		y1[7] = f7;
+
+		x1[0] = g0;
+		x1[1] = g1;
+		x1[2] = g2;
+		x1[3] = g3;
+		x1[4] = g4;
+		x1[5] = g5;
+		x1[6] = g6;
+		x1[7] = g7;
+
+		x1 += 8;
+		y1 += 8;
+
+		i+=8;
+	}
+	return;
+
+}
+
+
+#endif
+
+
+int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT dummy3, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y, FLOAT *dummy, BLASLONG dummy2)
+{
+	BLASLONG i=0;
+	BLASLONG ix=0,iy=0;
+	FLOAT temp;
+
+	if ( n <= 0     )  return(0);
+
+	if ( (inc_x == 1) && (inc_y == 1 ))
+	{
+
+		BLASLONG n1 = n & -32;
+		if ( n1 > 0 )
+		{
+			dswap_kernel_32(n1, x, y);
+			i=n1;
+		}
+
+		while(i < n)
+		{
+			temp = y[i];	
+			y[i] = x[i] ;
+			x[i] = temp;
+			i++ ;
+
+		}
+
+
+	}
+	else
+	{
+
+		while(i < n)
+		{
+			temp  = y[iy];
+			y[iy] = x[ix] ;
+			x[ix] = temp;
+			ix += inc_x ;
+			iy += inc_y ;
+			i++ ;
+
+		}
+
+	}
+	return(0);
+	
+
+}
+
+

kernel/power/dswap_microk_power8.c

@@ -0,0 +1,180 @@
+/***************************************************************************
+Copyright (c) 2013-2016, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+/**************************************************************************************
+* 2016/03/25 Werner Saar (wernsaar@googlemail.com)
+* 	 BLASTEST 		: OK
+* 	 CTEST			: OK
+* 	 TEST			: OK
+*	 LAPACK-TEST		: OK
+**************************************************************************************/
+
+#define HAVE_KERNEL_32 1
+
+static void dswap_kernel_32( BLASLONG n, FLOAT *x, FLOAT *y) __attribute__ ((noinline));
+
+static void dswap_kernel_32( BLASLONG n, FLOAT *x, FLOAT *y)
+{
+
+
+	BLASLONG i = n;
+	BLASLONG o16 = 16;
+	BLASLONG o32 = 32;
+	BLASLONG o48 = 48;
+	BLASLONG o64 = 64;
+	BLASLONG o80 = 80;
+	BLASLONG o96 = 96;
+	BLASLONG o112 = 112;
+	FLOAT *x1=x;
+	FLOAT *y1=y;
+	FLOAT *x2=x+1;
+	FLOAT *y2=y+1;
+	BLASLONG pre = 384;
+	BLASLONG alpha=0;
+
+	__asm__  __volatile__
+	(
+
+	"addi		%3, %3, -8			    \n\t"	
+	"addi		%4, %4, -8			    \n\t"	
+
+	".align 5				            \n\t"
+	"1:				                    \n\t"
+
+	"lxvd2x		32, 0, %2			    \n\t"
+	"lxvd2x		33, %5, %2			    \n\t"
+	"lxvd2x		34, %6, %2			    \n\t"
+	"lxvd2x		35, %7, %2			    \n\t"
+	"lxvd2x		36, %8, %2			    \n\t"
+	"lxvd2x		37, %9, %2			    \n\t"
+	"lxvd2x		38, %10, %2			    \n\t"
+	"lxvd2x		39, %11, %2			    \n\t"
+
+	"addi		%2, %2, 128			    \n\t"
+
+	"lxvd2x		40, 0, %2			    \n\t"
+	"lxvd2x		41, %5, %2			    \n\t"
+	"lxvd2x		42, %6, %2			    \n\t"
+	"lxvd2x		43, %7, %2			    \n\t"
+	"lxvd2x		44, %8, %2			    \n\t"
+	"lxvd2x		45, %9, %2			    \n\t"
+	"lxvd2x		46, %10, %2			    \n\t"
+	"lxvd2x		47, %11, %2			    \n\t"
+
+	"addi		%2, %2, 128			    \n\t"
+
+	"lxvd2x		48, 0, %1			    \n\t"
+	"lxvd2x		49, %5, %1			    \n\t"
+	"lxvd2x		50, %6, %1			    \n\t"
+	"lxvd2x		51, %7, %1			    \n\t"
+	"lxvd2x		52, %8, %1			    \n\t"
+	"lxvd2x		53, %9, %1			    \n\t"
+	"lxvd2x		54, %10, %1			    \n\t"
+	"lxvd2x		55, %11, %1			    \n\t"
+
+	"addi		%1, %1, 128			    \n\t"
+
+	"lxvd2x		56, 0, %1			    \n\t"
+	"lxvd2x		57, %5, %1			    \n\t"
+	"lxvd2x		58, %6, %1			    \n\t"
+	"lxvd2x		59, %7, %1			    \n\t"
+	"lxvd2x		60, %8, %1			    \n\t"
+	"lxvd2x		61, %9, %1			    \n\t"
+	"lxvd2x		62, %10, %1			    \n\t"
+	"lxvd2x		63, %11, %1			    \n\t"
+
+	"addi		%1, %1, 128			    \n\t"
+
+	"stxvd2x		32, 0, %3			    \n\t"
+	"stxvd2x		33, %5, %3			    \n\t"
+	"stxvd2x		34, %6, %3			    \n\t"
+	"stxvd2x		35, %7, %3			    \n\t"
+	"stxvd2x		36, %8, %3			    \n\t"
+	"stxvd2x		37, %9, %3			    \n\t"
+	"stxvd2x		38, %10, %3			    \n\t"
+	"stxvd2x		39, %11, %3			    \n\t"
+
+	"addi		%3, %3, 128			    \n\t"
+
+	"stxvd2x		40, 0, %3			    \n\t"
+	"stxvd2x		41, %5, %3			    \n\t"
+	"stxvd2x		42, %6, %3			    \n\t"
+	"stxvd2x		43, %7, %3			    \n\t"
+	"stxvd2x		44, %8, %3			    \n\t"
+	"stxvd2x		45, %9, %3			    \n\t"
+	"stxvd2x		46, %10, %3			    \n\t"
+	"stxvd2x		47, %11, %3			    \n\t"
+
+	"addi		%3, %3, 128			    \n\t"
+
+	"stxvd2x		48, 0, %4			    \n\t"
+	"stxvd2x		49, %5, %4			    \n\t"
+	"stxvd2x		50, %6, %4			    \n\t"
+	"stxvd2x		51, %7, %4			    \n\t"
+	"stxvd2x		52, %8, %4			    \n\t"
+	"stxvd2x		53, %9, %4			    \n\t"
+	"stxvd2x		54, %10, %4			    \n\t"
+	"stxvd2x		55, %11, %4			    \n\t"
+
+	"addi		%4, %4, 128			    \n\t"
+
+	"stxvd2x		56, 0, %4			    \n\t"
+	"stxvd2x		57, %5, %4			    \n\t"
+	"stxvd2x		58, %6, %4			    \n\t"
+	"stxvd2x		59, %7, %4			    \n\t"
+	"stxvd2x		60, %8, %4			    \n\t"
+	"stxvd2x		61, %9, %4			    \n\t"
+	"stxvd2x		62, %10, %4			    \n\t"
+	"stxvd2x		63, %11, %4			    \n\t"
+
+	"addi		%4, %4, 128			    \n\t"
+
+	"addic.		%0 , %0	, -32  	 	             \n\t"
+	"bgt		1b		             	     \n\t"
+
+	"2:						     \n\t"
+
+	:
+        : 
+          "r" (i),	// 0	
+	  "r" (y1),  	// 1
+          "r" (x1),     // 2
+          "r" (y2),     // 3
+          "r" (x2),     // 4
+	  "r" (o16),	// 5
+	  "r" (o32),	// 6
+	  "r" (o48),    // 7
+          "r" (o64),    // 8
+          "r" (o80),    // 9
+          "r" (o96),    // 10
+          "r" (o112)    // 11
+	: "cr0", "%0", "%2" , "%1", "%3", "%4", "memory"
+	);
+
+} 
+
+

kernel/power/sasum.c

@@ -0,0 +1,146 @@
+/***************************************************************************
+Copyright (c) 2013-2016, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+/**************************************************************************************
+* 2016/03/28 Werner Saar (wernsaar@googlemail.com)
+* 	 BLASTEST 		: OK
+* 	 CTEST			: OK
+* 	 TEST			: OK
+*	 LAPACK-TEST		: OK
+**************************************************************************************/
+
+#include "common.h"
+#include <math.h>
+
+#if defined(DOUBLE)
+
+#define ABS fabs
+
+#else
+
+#define ABS fabsf
+
+#endif
+
+#if defined(POWER8)
+#include "sasum_microk_power8.c"
+#endif
+
+
+#ifndef HAVE_KERNEL_32
+
+static void sasum_kernel_32(BLASLONG n, FLOAT *x1, FLOAT *svec)
+{
+
+	BLASLONG i=0;
+	FLOAT *x = x1;
+	FLOAT temp0, temp1, temp2, temp3;
+	FLOAT temp4, temp5, temp6, temp7;
+	FLOAT sum0 = 0.0;
+	FLOAT sum1 = 0.0;
+	FLOAT sum2 = 0.0;
+	FLOAT sum3 = 0.0;
+
+	while ( i< n )
+	{
+
+		temp0 = ABS(x[0]);
+		temp1 = ABS(x[1]);
+		temp2 = ABS(x[2]);
+		temp3 = ABS(x[3]);
+		temp4 = ABS(x[4]);
+		temp5 = ABS(x[5]);
+		temp6 = ABS(x[6]);
+		temp7 = ABS(x[7]);
+
+		sum0 += temp0;
+		sum1 += temp1;
+		sum2 += temp2;
+		sum3 += temp3;
+
+		sum0 += temp4;
+		sum1 += temp5;
+		sum2 += temp6;
+		sum3 += temp7;
+
+		x+=8;
+		i+=8;
+
+	}
+
+	svec[0] = sum0+sum1+sum2+sum3;
+	svec[1] = 0.0;
+	svec[2] = 0.0;
+	svec[3] = 0.0;
+
+}
+
+#endif
+
+FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x)
+{
+	BLASLONG i=0;
+	FLOAT sumf = 0.0;
+	FLOAT svec[4] __attribute__ ((aligned (16)));;
+	BLASLONG n1;
+
+	if (n <= 0 || inc_x <= 0) return(sumf);
+
+	if ( inc_x == 1 )
+	{
+
+		n1 = n & -32;
+		if ( n1 > 0 )
+		{
+
+			sasum_kernel_32(n1, x, svec);
+			sumf = svec[0] + svec[1]+svec[2]+svec[3];
+			i=n1;
+		}
+
+		while(i < n)
+		{
+			sumf += ABS(x[i]);
+			i++;
+		}
+
+	}
+	else
+	{
+
+		n *= inc_x;
+		while(i < n)
+		{
+			sumf += ABS(x[i]);
+			i += inc_x;
+		}
+
+	}
+	return(sumf);
+}
+
+

kernel/power/sasum_microk_power8.c

@@ -0,0 +1,177 @@
+/***************************************************************************
+Copyright (c) 2013-2016, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+/**************************************************************************************
+* 2016/03/28 Werner Saar (wernsaar@googlemail.com)
+* 	 BLASTEST 		: OK
+* 	 CTEST			: OK
+* 	 TEST			: OK
+*	 LAPACK-TEST		: OK
+**************************************************************************************/
+
+#define HAVE_KERNEL_32 1
+static void sasum_kernel_32( BLASLONG n, FLOAT *x, FLOAT *svec) __attribute__ ((noinline));
+
+static void sasum_kernel_32( BLASLONG n, FLOAT *x, FLOAT *svec)
+{
+
+
+	BLASLONG i = n;
+	BLASLONG o16 = 16;
+	BLASLONG o32 = 32;
+	BLASLONG o48 = 48;
+	BLASLONG o64 = 64;
+	BLASLONG o80 = 80;
+	BLASLONG o96 = 96;
+	BLASLONG o112 = 112;
+	FLOAT *x1=x;
+	BLASLONG pre = 384;
+
+	__asm__  __volatile__
+	(
+
+	"dcbt		%2 , %4				    \n\t"
+
+	"xxlxor		32,32,32			    \n\t"
+	"xxlxor		33,33,33			    \n\t"
+	"xxlxor		34,34,34			    \n\t"
+	"xxlxor		35,35,35			    \n\t"
+	"xxlxor		36,36,36			    \n\t"
+	"xxlxor		37,37,37			    \n\t"
+	"xxlxor		38,38,38			    \n\t"
+	"xxlxor		39,39,39			    \n\t"
+
+	"lxvw4x		40, 0, %2			    \n\t"
+	"lxvw4x		41, %5, %2			    \n\t"
+	"lxvw4x		42, %6, %2			    \n\t"
+	"lxvw4x		43, %7, %2			    \n\t"
+	"lxvw4x		44, %8, %2			    \n\t"
+	"lxvw4x		45, %9, %2			    \n\t"
+	"lxvw4x		46, %10, %2			    \n\t"
+	"lxvw4x		47, %11, %2			    \n\t"
+
+	"addi		%2, %2, 128			    \n\t"
+
+	"addic.		%0 , %0	, -32  	 	             \n\t"
+	"ble		2f		             	     \n\t"
+
+	".align 5				            \n\t"
+	"1:				                    \n\t"
+
+	"dcbt		%2 , %4				    \n\t"
+
+	"xvabssp	48, 40				    \n\t"
+	"xvabssp	49, 41				    \n\t"
+	"xvabssp	50, 42				    \n\t"
+	"xvabssp	51, 43				    \n\t"
+
+	"lxvw4x		40, 0, %2			    \n\t"
+	"lxvw4x		41, %5, %2			    \n\t"
+
+	"xvabssp	52, 44				    \n\t"
+	"xvabssp	53, 45				    \n\t"
+
+	"lxvw4x		42, %6, %2			    \n\t"
+	"lxvw4x		43, %7, %2			    \n\t"
+
+	"xvabssp	54, 46				    \n\t"
+	"xvabssp	55, 47				    \n\t"
+
+	"lxvw4x		44, %8, %2			    \n\t"
+	"lxvw4x		45, %9, %2			    \n\t"
+
+	"xvaddsp	32, 32, 48		    \n\t"
+	"xvaddsp	33, 33, 49		    \n\t"
+
+	"lxvw4x		46, %10, %2			    \n\t"
+	"lxvw4x		47, %11, %2			    \n\t"
+
+	"xvaddsp	34, 34, 50		    \n\t"
+	"xvaddsp	35, 35, 51		    \n\t"
+	"addi		%2, %2, 128			    \n\t"
+	"xvaddsp	36, 36, 52		    \n\t"
+	"xvaddsp	37, 37, 53		    \n\t"
+	"addic.		%0 , %0	, -32  	 	             \n\t"
+	"xvaddsp	38, 38, 54		    \n\t"
+	"xvaddsp	39, 39, 55		    \n\t"
+
+	"bgt		1b		             	     \n\t"
+
+	"2:						     \n\t"
+
+
+	"xvabssp	48, 40				    \n\t"
+	"xvabssp	49, 41				    \n\t"
+	"xvabssp	50, 42				    \n\t"
+	"xvabssp	51, 43				    \n\t"
+	"xvabssp	52, 44				    \n\t"
+	"xvabssp	53, 45				    \n\t"
+	"xvabssp	54, 46				    \n\t"
+	"xvabssp	55, 47				    \n\t"
+
+	"xvaddsp	32, 32, 48		    \n\t"
+	"xvaddsp	33, 33, 49		    \n\t"
+	"xvaddsp	34, 34, 50		    \n\t"
+	"xvaddsp	35, 35, 51		    \n\t"
+	"xvaddsp	36, 36, 52		    \n\t"
+	"xvaddsp	37, 37, 53		    \n\t"
+	"xvaddsp	38, 38, 54		    \n\t"
+	"xvaddsp	39, 39, 55		    \n\t"
+
+	"xvaddsp	32, 32, 33		     \n\t"
+	"xvaddsp	34, 34, 35		     \n\t"
+	"xvaddsp	36, 36, 37		     \n\t"
+	"xvaddsp	38, 38, 39		     \n\t"
+
+	"xvaddsp	32, 32, 34		     \n\t"
+	"xvaddsp	36, 36, 38		     \n\t"
+
+	"xvaddsp	32, 32, 36		     \n\t"
+
+
+	"stxvw4x	32, 0, %3		     \n\t"
+
+	:
+        : 
+          "r" (i),	// 0	
+	  "r" (n),  	// 1
+          "r" (x1),     // 2
+          "r" (svec),   // 3
+          "r" (pre),    // 4
+	  "r" (o16),	// 5
+	  "r" (o32),	// 6
+	  "r" (o48),    // 7
+          "r" (o64),    // 8
+          "r" (o80),    // 9
+          "r" (o96),    // 10
+          "r" (o112)   // 11
+	: "cr0", "%0", "%2",  "memory"
+	);
+
+} 
+
+

kernel/power/scopy.c

@@ -0,0 +1,131 @@
+/***************************************************************************
+Copyright (c) 2013-2016, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+/**************************************************************************************
+* 2016/03/25 Werner Saar (wernsaar@googlemail.com)
+* 	 BLASTEST 		: OK
+* 	 CTEST			: OK
+* 	 TEST			: OK
+*	 LAPACK-TEST		: OK
+**************************************************************************************/
+
+#include "common.h"
+
+#if defined(POWER8)
+#include "scopy_microk_power8.c"
+#endif
+
+#ifndef HAVE_KERNEL_32
+
+static void scopy_kernel_32(BLASLONG n, FLOAT *x, FLOAT *y)
+{
+
+	BLASLONG i=0;
+	FLOAT f0, f1, f2, f3, f4, f5, f6, f7;
+	FLOAT *x1=x;
+	FLOAT *y1=y;
+
+	while ( i<n )
+	{
+
+		f0 = x1[0];
+		f1 = x1[1];
+		f2 = x1[2];
+		f3 = x1[3];
+		f4 = x1[4];
+		f5 = x1[5];
+		f6 = x1[6];
+		f7 = x1[7];
+
+		y1[0] = f0;
+		y1[1] = f1;
+		y1[2] = f2;
+		y1[3] = f3;
+		y1[4] = f4;
+		y1[5] = f5;
+		y1[6] = f6;
+		y1[7] = f7;
+
+		x1 += 8;
+		y1 += 8;
+
+		i+=8;
+	}
+	return;
+
+}
+
+
+#endif
+
+
+
+int CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y)
+{
+	BLASLONG i=0;
+	BLASLONG ix=0,iy=0;
+
+	if ( n <= 0     )  return(0);
+
+	if ( (inc_x == 1) && (inc_y == 1 ))
+	{
+
+		BLASLONG n1 = n & -32;
+		if ( n1 > 0 )
+		{
+			scopy_kernel_32(n1, x, y);
+			i=n1;
+		}
+
+		while(i < n)
+		{
+			y[i] = x[i] ;
+			i++ ;
+
+		}
+
+
+	}
+	else
+	{
+
+		while(i < n)
+		{
+			y[iy] = x[ix] ;
+			ix += inc_x ;
+			iy += inc_y ;
+			i++ ;
+
+		}
+
+	}
+	return(0);
+	
+
+}
+
+

kernel/power/scopy_microk_power8.c

@@ -0,0 +1,131 @@
+/***************************************************************************
+Copyright (c) 2013-2016, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+/**************************************************************************************
+* 2016/03/25 Werner Saar (wernsaar@googlemail.com)
+* 	 BLASTEST 		: OK
+* 	 CTEST			: OK
+* 	 TEST			: OK
+*	 LAPACK-TEST		: OK
+**************************************************************************************/
+
+#define HAVE_KERNEL_32 1
+
+static void scopy_kernel_32( BLASLONG n, FLOAT *x, FLOAT *y) __attribute__ ((noinline));
+
+static void scopy_kernel_32( BLASLONG n, FLOAT *x, FLOAT *y)
+{
+
+
+	BLASLONG i = n;
+	BLASLONG o16 = 16;
+	BLASLONG o32 = 32;
+	BLASLONG o48 = 48;
+	BLASLONG o64 = 64;
+	BLASLONG o80 = 80;
+	BLASLONG o96 = 96;
+	BLASLONG o112 = 112;
+	FLOAT *x1=x;
+	FLOAT *y1=y;
+	BLASLONG pre = 384;
+	BLASLONG alpha=0;
+
+	__asm__  __volatile__
+	(
+
+	"lxvw4x		40, 0, %2			    \n\t"
+	"lxvw4x		41, %5, %2			    \n\t"
+	"lxvw4x		42, %6, %2			    \n\t"
+	"lxvw4x		43, %7, %2			    \n\t"
+	"lxvw4x		44, %8, %2			    \n\t"
+	"lxvw4x		45, %9, %2			    \n\t"
+	"lxvw4x		46, %10, %2			    \n\t"
+	"lxvw4x		47, %11, %2			    \n\t"
+
+	"addi		%2, %2, 128			    \n\t"
+
+	"addic.		%0 , %0	, -32  	 	             \n\t"
+	"ble		2f		             	     \n\t"
+
+	".align 5				            \n\t"
+	"1:				                    \n\t"
+
+	"stxvw4x		40, 0, %1			    \n\t"
+	"stxvw4x		41, %5, %1			    \n\t"
+	"lxvw4x		40, 0, %2			    \n\t"
+	"lxvw4x		41, %5, %2			    \n\t"
+	"stxvw4x		42, %6, %1			    \n\t"
+	"stxvw4x		43, %7, %1			    \n\t"
+	"lxvw4x		42, %6, %2			    \n\t"
+	"lxvw4x		43, %7, %2			    \n\t"
+	"stxvw4x		44, %8, %1			    \n\t"
+	"stxvw4x		45, %9, %1			    \n\t"
+	"lxvw4x		44, %8, %2			    \n\t"
+	"lxvw4x		45, %9, %2			    \n\t"
+	"stxvw4x		46, %10, %1			    \n\t"
+	"stxvw4x		47, %11, %1			    \n\t"
+	"lxvw4x		46, %10, %2			    \n\t"
+	"lxvw4x		47, %11, %2			    \n\t"
+
+
+	"addi		%1, %1, 128			    \n\t"
+	"addi		%2, %2, 128			    \n\t"
+
+	"addic.		%0 , %0	, -32  	 	             \n\t"
+	"bgt		1b		             	     \n\t"
+
+	"2:						     \n\t"
+
+	"stxvw4x		40, 0, %1			    \n\t"
+	"stxvw4x		41, %5, %1			    \n\t"
+	"stxvw4x		42, %6, %1			    \n\t"
+	"stxvw4x		43, %7, %1			    \n\t"
+	"stxvw4x		44, %8, %1			    \n\t"
+	"stxvw4x		45, %9, %1			    \n\t"
+	"stxvw4x		46, %10, %1			    \n\t"
+	"stxvw4x		47, %11, %1			    \n\t"
+
+	:
+        : 
+          "r" (i),	// 0	
+	  "r" (y1),  	// 1
+          "r" (x1),     // 2
+          "r" (alpha),  // 3
+          "r" (pre),    // 4
+	  "r" (o16),	// 5
+	  "r" (o32),	// 6
+	  "r" (o48),    // 7
+          "r" (o64),    // 8
+          "r" (o80),    // 9
+          "r" (o96),    // 10
+          "r" (o112)    // 11
+	: "cr0", "%0", "%2" , "%1", "memory"
+	);
+
+} 
+
+

kernel/power/sdot.c

@@ -0,0 +1,126 @@
+/***************************************************************************
+Copyright (c) 2013-2016, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+/**************************************************************************************
+* 2016/03/21 Werner Saar (wernsaar@googlemail.com)
+* 	 BLASTEST 		: OK
+* 	 CTEST			: OK
+* 	 TEST			: OK
+*	 LAPACK-TEST		: OK
+**************************************************************************************/
+
+#include "common.h"
+
+#if defined(POWER8) 
+#include "sdot_microk_power8.c"
+#endif
+
+
+#ifndef HAVE_KERNEL_16
+
+static void sdot_kernel_16(BLASLONG n, FLOAT *x, FLOAT *y, FLOAT *d)
+{
+	BLASLONG register i = 0;
+	FLOAT dot = 0.0;
+
+	while(i < n)
+        {
+              dot += y[i]  * x[i]
+                  + y[i+1] * x[i+1]
+                  + y[i+2] * x[i+2]
+                  + y[i+3] * x[i+3]
+                  + y[i+4] * x[i+4]
+                  + y[i+5] * x[i+5]
+                  + y[i+6] * x[i+6]
+                  + y[i+7] * x[i+7] ;
+
+              i+=8 ;
+
+       }
+       *d += dot;
+
+}
+
+#endif
+
+FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y)
+{
+	BLASLONG i=0;
+	BLASLONG ix=0,iy=0;
+
+	FLOAT  dot = 0.0 ;
+
+	if ( n <= 0 )  return(dot);
+
+	if ( (inc_x == 1) && (inc_y == 1) )
+	{
+
+		BLASLONG n1 = n & -32;
+
+		if ( n1 )
+			sdot_kernel_16(n1, x, y , &dot );
+
+
+		i = n1;
+		while(i < n)
+		{
+
+			dot += y[i] * x[i] ;
+			i++ ;
+
+		}
+		return(dot);
+
+
+	}
+
+	BLASLONG n1 = n & -2;
+
+	while(i < n1)
+	{
+
+		dot += y[iy] * x[ix] + y[iy+inc_y] * x[ix+inc_x];
+		ix  += inc_x*2 ;
+		iy  += inc_y*2 ;
+		i+=2 ;
+
+	}
+
+	while(i < n)
+	{
+
+		dot += y[iy] * x[ix] ;
+		ix  += inc_x ;
+		iy  += inc_y ;
+		i++ ;
+
+	}
+	return(dot);
+
+}
+
+

kernel/power/sdot_microk_power8.c

@@ -0,0 +1,179 @@
+/***************************************************************************
+Copyright (c) 2013-2016, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+/**************************************************************************************
+* 2016/03/21 Werner Saar (wernsaar@googlemail.com)
+* 	 BLASTEST 		: OK
+* 	 CTEST			: OK
+* 	 TEST			: OK
+*	 LAPACK-TEST		: OK
+**************************************************************************************/
+
+#define HAVE_KERNEL_16 1
+static void sdot_kernel_16( BLASLONG n, FLOAT *x, FLOAT *y , FLOAT *dot) __attribute__ ((noinline));
+
+static void sdot_kernel_16( BLASLONG n, FLOAT *x, FLOAT *y, FLOAT *dot)
+{
+
+
+	BLASLONG i = n;
+	BLASLONG o16 = 16;
+	BLASLONG o32 = 32;
+	BLASLONG o48 = 48;
+	BLASLONG o64 = 64;
+	BLASLONG o80 = 80;
+	BLASLONG o96 = 96;
+	BLASLONG o112 = 112;
+	FLOAT *x1=x;
+	FLOAT *y1=y;
+	BLASLONG pre = 384;
+	FLOAT tempdot[4];
+
+
+	__asm__  __volatile__
+	(
+	"xxlxor		32,32,32			    \n\t"
+	"xxlxor		33,33,33			    \n\t"
+	"xxlxor		34,34,34			    \n\t"
+	"xxlxor		35,35,35			    \n\t"
+	"xxlxor		36,36,36			    \n\t"
+	"xxlxor		37,37,37			    \n\t"
+	"xxlxor		38,38,38			    \n\t"
+	"xxlxor		39,39,39			    \n\t"
+
+	"dcbt		%2, %12				    \n\t"
+	"dcbt		%3, %12				    \n\t"
+
+	"lxvw4x		40, 0, %2			    \n\t"
+	"lxvw4x		48, 0, %3			    \n\t"
+	"lxvw4x		41, %5, %2			    \n\t"
+	"lxvw4x		49, %5, %3			    \n\t"
+	"lxvw4x		42, %6, %2			    \n\t"
+	"lxvw4x		50, %6, %3			    \n\t"
+	"lxvw4x		43, %7, %2			    \n\t"
+	"lxvw4x		51, %7, %3			    \n\t"
+	"lxvw4x		44, %8, %2			    \n\t"
+	"lxvw4x		52, %8, %3			    \n\t"
+	"lxvw4x		45, %9, %2			    \n\t"
+	"lxvw4x		53, %9, %3			    \n\t"
+	"lxvw4x		46, %10, %2			    \n\t"
+	"lxvw4x		54, %10, %3			    \n\t"
+	"lxvw4x		47, %11, %2			    \n\t"
+	"lxvw4x		55, %11, %3			    \n\t"
+
+	"addi		%2, %2, 128			    \n\t"
+	"addi		%3, %3, 128			    \n\t"
+
+	"addic.		%0 , %0	, -32  	 	             \n\t"
+	"ble		2f		             	     \n\t"
+
+	".align 5				            \n\t"
+	"1:				                    \n\t"
+
+	"dcbt		%2, %12				    \n\t"
+	"dcbt		%3, %12				    \n\t"
+
+	"xvmaddasp	32, 40, 48		    \n\t"
+	"lxvw4x		40, 0, %2			    \n\t"
+	"lxvw4x		48, 0, %3			    \n\t"
+	"xvmaddasp	33, 41, 49		    \n\t"
+	"lxvw4x		41, %5, %2			    \n\t"
+	"lxvw4x		49, %5, %3			    \n\t"
+	"xvmaddasp	34, 42, 50		    \n\t"
+	"lxvw4x		42, %6, %2			    \n\t"
+	"lxvw4x		50, %6, %3			    \n\t"
+	"xvmaddasp	35, 43, 51		    \n\t"
+	"lxvw4x		43, %7, %2			    \n\t"
+	"lxvw4x		51, %7, %3			    \n\t"
+	"xvmaddasp	36, 44, 52		    \n\t"
+	"lxvw4x		44, %8, %2			    \n\t"
+	"lxvw4x		52, %8, %3			    \n\t"
+	"xvmaddasp	37, 45, 53		    \n\t"
+	"lxvw4x		45, %9, %2			    \n\t"
+	"lxvw4x		53, %9, %3			    \n\t"
+	"xvmaddasp	38, 46, 54		    \n\t"
+	"lxvw4x		46, %10, %2			    \n\t"
+	"lxvw4x		54, %10, %3			    \n\t"
+	"xvmaddasp	39, 47, 55		    \n\t"
+
+	"lxvw4x		47, %11, %2			    \n\t"
+	"lxvw4x		55, %11, %3			    \n\t"
+
+
+	"addi		%2, %2, 128			    \n\t"
+	"addi		%3, %3, 128			    \n\t"
+
+	"addic.		%0 , %0	, -32  	 	             \n\t"
+	"bgt		1b		             	     \n\t"
+
+	"2:						     \n\t"
+
+	"xvmaddasp	32, 40, 48		    \n\t"
+	"xvmaddasp	33, 41, 49		    \n\t"
+	"xvmaddasp	34, 42, 50		    \n\t"
+	"xvmaddasp	35, 43, 51		    \n\t"
+	"xvmaddasp	36, 44, 52		    \n\t"
+	"xvmaddasp	37, 45, 53		    \n\t"
+	"xvmaddasp	38, 46, 54		    \n\t"
+	"xvmaddasp	39, 47, 55		    \n\t"
+
+	"xvaddsp	32, 32 , 33		    \n\t"
+	"xvaddsp	34, 34 , 35		    \n\t"
+	"xvaddsp	36, 36 , 37		    \n\t"
+	"xvaddsp	38, 38 , 39		    \n\t"
+
+	"xvaddsp	32, 32 , 34		    \n\t"
+	"xvaddsp	36, 36 , 38		    \n\t"
+
+	"xvaddsp	32, 32 , 36		    \n\t"
+
+	"stxvw4x	32, 0 , %4		    \n\t"
+
+	:
+        : 
+          "r" (i),	// 0	
+	  "r" (n),  	// 1
+          "r" (x1),     // 2
+          "r" (y1),     // 3
+          "r" (tempdot),    // 4
+	  "r" (o16),	// 5
+	  "r" (o32),	// 6
+	  "r" (o48),    // 7
+          "r" (o64),    // 8
+          "r" (o80),    // 9
+          "r" (o96),    // 10
+          "r" (o112),   // 11
+	  "r" (pre)	// 12
+	: "cr0", "%0", "%2" , "%3", "memory"
+	);
+
+	*dot = tempdot[0] + tempdot[1] + tempdot[2] + tempdot[3];
+
+
+} 
+
+

kernel/power/sgemm_kernel_16x8_power8.S

@@ -0,0 +1,371 @@
+/***************************************************************************
+Copyright (c) 2013-2016, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+/**************************************************************************************
+* 2016/04/02 Werner Saar (wernsaar@googlemail.com)
+* 	 BLASTEST 		: OK
+* 	 CTEST			: OK
+* 	 TEST			: OK
+* 	 LAPACK-TEST		: OK
+**************************************************************************************/
+
+/*********************************************************************/
+/* Copyright 2009, 2010 The University of Texas at Austin.           */
+/* All rights reserved.                                              */
+/*                                                                   */
+/* Redistribution and use in source and binary forms, with or        */
+/* without modification, are permitted provided that the following   */
+/* conditions are met:                                               */
+/*                                                                   */
+/*   1. Redistributions of source code must retain the above         */
+/*      copyright notice, this list of conditions and the following  */
+/*      disclaimer.                                                  */
+/*                                                                   */
+/*   2. Redistributions in binary form must reproduce the above      */
+/*      copyright notice, this list of conditions and the following  */
+/*      disclaimer in the documentation and/or other materials       */
+/*      provided with the distribution.                              */
+/*                                                                   */
+/*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */
+/*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */
+/*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */
+/*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */
+/*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */
+/*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */
+/*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */
+/*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */
+/*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */
+/*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */
+/*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */
+/*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */
+/*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */
+/*    POSSIBILITY OF SUCH DAMAGE.                                    */
+/*                                                                   */
+/* The views and conclusions contained in the software and           */
+/* documentation are those of the authors and should not be          */
+/* interpreted as representing official policies, either expressed   */
+/* or implied, of The University of Texas at Austin.                 */
+/*********************************************************************/
+
+#define ASSEMBLER
+#include "common.h"
+#include "def_vsx.h"
+
+#ifndef __64BIT__
+#define LOAD	lwz
+#else
+#define LOAD	ld
+#endif
+
+#ifdef __64BIT__
+#define STACKSIZE 32752
+#define ALPHA_SP   296(SP)
+#define FZERO	304(SP)
+#else
+#define STACKSIZE 240
+#define ALPHA_SP   224(SP)
+#define FZERO	232(SP)
+#endif
+
+#define	M	r3
+#define	N	r4
+#define	K	r5
+
+#ifdef linux
+#ifndef __64BIT__
+#define A	r6
+#define	B	r7
+#define	C	r8
+#define	LDC	r9
+#define OFFSET	r10
+#else
+#define A	r7
+#define	B	r8
+#define	C	r9
+#define	LDC	r10
+#define OFFSET	r6
+#endif
+#endif
+
+#if defined(_AIX) || defined(__APPLE__)
+#if !defined(__64BIT__) && defined(DOUBLE)
+#define A	r8
+#define	B	r9
+#define	C	r10
+#define	LDC	r7
+#define OFFSET	r6
+#else
+#define A	r7
+#define	B	r8
+#define	C	r9
+#define	LDC	r10
+#define OFFSET	r6
+#endif
+#endif
+
+#define alpha_r vs30
+#define alpha_vr vs31
+
+#define o0	0
+
+#define FRAMEPOINTER r12
+
+#define BBUFFER r14
+#define o4	r15
+#define o12	r16
+#define o8	r17
+#define L	r18
+#define T1	r19
+#define KK	r20
+#define BBO	r21
+#define	I	r22
+#define J	r23
+#define AO	r24
+#define	BO	r25
+#define	CO 	r26
+#define o16	r27
+#define	o32	r28
+#define	o48	r29
+
+#define PRE	r30
+#define T2	r31
+
+#include "sgemm_macros_16x8_power8.S"
+
+
+#ifndef NEEDPARAM
+
+	PROLOGUE
+	PROFCODE
+
+	mr	FRAMEPOINTER, SP
+	addi	SP, SP, -STACKSIZE
+	addi	SP, SP, -STACKSIZE
+	addi	SP, SP, -STACKSIZE
+	addi	SP, SP, -STACKSIZE
+	li	r0, 0
+
+	stfd	f14,    0(SP)
+	stfd	f15,    8(SP)
+	stfd	f16,   16(SP)
+	stfd	f17,   24(SP)
+
+	stfd	f18,   32(SP)
+	stfd	f19,   40(SP)
+	stfd	f20,   48(SP)
+	stfd	f21,   56(SP)
+
+	stfd	f22,   64(SP)
+	stfd	f23,   72(SP)
+	stfd	f24,   80(SP)
+	stfd	f25,   88(SP)
+
+	stfd	f26,   96(SP)
+	stfd	f27,  104(SP)
+	stfd	f28,  112(SP)
+	stfd	f29,  120(SP)
+
+	stfd	f30,  128(SP)
+	stfd	f31,  136(SP)
+
+#ifdef __64BIT__
+	std	r31,  144(SP)
+	std	r30,  152(SP)
+	std	r29,  160(SP)
+	std	r28,  168(SP)
+	std	r27,  176(SP)
+	std	r26,  184(SP)
+	std	r25,  192(SP)
+	std	r24,  200(SP)
+	std	r23,  208(SP)
+	std	r22,  216(SP)
+	std	r21,  224(SP)
+	std	r20,  232(SP)
+	std	r19,  240(SP)
+	std	r18,  248(SP)
+	std	r17,  256(SP)
+	std	r16,  264(SP)
+	std	r15,  272(SP)
+	std	r14,  280(SP)
+#else
+	stw	r31,  144(SP)
+	stw	r30,  148(SP)
+	stw	r29,  152(SP)
+	stw	r28,  156(SP)
+	stw	r27,  160(SP)
+	stw	r26,  164(SP)
+	stw	r25,  168(SP)
+	stw	r24,  172(SP)
+	stw	r23,  176(SP)
+	stw	r22,  180(SP)
+	stw	r21,  184(SP)
+	stw	r20,  188(SP)
+	stw	r19,  192(SP)
+	stw	r18,  196(SP)
+	stw	r17,  200(SP)
+	stw	r16,  204(SP)
+	stw	r15,  208(SP)
+	stw	r14,  212(SP)
+#endif
+
+	// stfd	f1,  ALPHA_SP
+	// stw	r0,  FZERO
+
+#if defined(_AIX) || defined(__APPLE__)
+#if !defined(__64BIT__) && defined(DOUBLE)
+	lwz	LDC, FRAMESLOT(0) + 0(FRAMEPOINTER)
+#endif
+#endif
+
+	slwi	LDC, LDC, 2
+
+#if defined(TRMMKERNEL)
+#if defined(linux) && defined(__64BIT__)
+	ld	OFFSET,  FRAMESLOT(0) + 0(FRAMEPOINTER)
+#endif
+
+#if defined(_AIX) || defined(__APPLE__)
+#ifdef __64BIT__
+	ld	OFFSET,  FRAMESLOT(0) + 0(FRAMEPOINTER)
+#else
+#ifdef DOUBLE
+	lwz	OFFSET,  FRAMESLOT(1) + 0(FRAMEPOINTER)
+#else
+	lwz	OFFSET,  FRAMESLOT(0) + 0(FRAMEPOINTER)
+#endif
+#endif
+#endif
+#endif
+
+
+	cmpwi	cr0, M, 0
+	ble	L999_H1
+	cmpwi	cr0, N, 0
+	ble	L999_H1
+	cmpwi	cr0, K, 0
+	ble	L999_H1
+
+	li	PRE, 256 
+	li	o4 , 4
+	li	o8 , 8
+	li	o12, 12
+	li	o16, 16
+	li	o32, 32
+	li	o48, 48
+
+        addi    BBUFFER, SP, 512+4096
+	li	T1, -4096
+	and	BBUFFER, BBUFFER, T1
+
+        addi    T1, SP, 300
+        stxsspx    f1, o0 , T1
+        stxsspx    f1, o4 , T1
+        stxsspx    f1, o8 , T1
+        stxsspx    f1, o12 , T1
+
+	lxsspx	   alpha_r,  o0, T1
+        lxvw4x     alpha_vr, o0, T1
+
+
+
+#include "sgemm_logic_16x8_power8.S"
+
+L999:
+	addi	r3, 0, 0
+
+	lfd	f14,    0(SP)
+	lfd	f15,    8(SP)
+	lfd	f16,   16(SP)
+	lfd	f17,   24(SP)
+
+	lfd	f18,   32(SP)
+	lfd	f19,   40(SP)
+	lfd	f20,   48(SP)
+	lfd	f21,   56(SP)
+
+	lfd	f22,   64(SP)
+	lfd	f23,   72(SP)
+	lfd	f24,   80(SP)
+	lfd	f25,   88(SP)
+
+	lfd	f26,   96(SP)
+	lfd	f27,  104(SP)
+	lfd	f28,  112(SP)
+	lfd	f29,  120(SP)
+
+	lfd	f30,  128(SP)
+	lfd	f31,  136(SP)
+
+#ifdef __64BIT__
+	ld	r31,  144(SP)
+	ld	r30,  152(SP)
+	ld	r29,  160(SP)
+	ld	r28,  168(SP)
+	ld	r27,  176(SP)
+	ld	r26,  184(SP)
+	ld	r25,  192(SP)
+	ld	r24,  200(SP)
+	ld	r23,  208(SP)
+	ld	r22,  216(SP)
+	ld	r21,  224(SP)
+	ld	r20,  232(SP)
+	ld	r19,  240(SP)
+	ld	r18,  248(SP)
+	ld	r17,  256(SP)
+	ld	r16,  264(SP)
+	ld	r15,  272(SP)
+	ld	r14,  280(SP)
+#else
+	lwz	r31,  144(SP)
+	lwz	r30,  148(SP)
+	lwz	r29,  152(SP)
+	lwz	r28,  156(SP)
+	lwz	r27,  160(SP)
+	lwz	r26,  164(SP)
+	lwz	r25,  168(SP)
+	lwz	r24,  172(SP)
+	lwz	r23,  176(SP)
+	lwz	r22,  180(SP)
+	lwz	r21,  184(SP)
+	lwz	r20,  188(SP)
+	lwz	r19,  192(SP)
+	lwz	r18,  196(SP)
+	lwz	r17,  200(SP)
+	lwz	r16,  204(SP)
+	lwz	r15,  208(SP)
+	lwz	r14,  212(SP)
+#endif
+
+	addi	SP, SP, STACKSIZE
+	addi	SP, SP, STACKSIZE
+	addi	SP, SP, STACKSIZE
+	addi	SP, SP, STACKSIZE
+
+	blr
+
+	EPILOGUE
+#endif

kernel/power/srot.c

@@ -0,0 +1,167 @@
+/***************************************************************************
+Copyright (c) 2013-2016, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+/**************************************************************************************
+* 2016/03/26 Werner Saar (wernsaar@googlemail.com)
+* 	 BLASTEST 		: OK
+* 	 CTEST			: OK
+* 	 TEST			: OK
+*	 LAPACK-TEST		: OK
+**************************************************************************************/
+
+
+
+#include "common.h"
+
+#pragma GCC optimize "O1"
+
+#if defined(POWER8)
+#include "srot_microk_power8.c"
+#endif
+
+
+#ifndef HAVE_KERNEL_16
+
+static void srot_kernel_16(BLASLONG n, FLOAT *x, FLOAT *y, FLOAT *c, FLOAT *s)
+{
+
+	BLASLONG i=0;
+	FLOAT f0, f1, f2, f3;
+	FLOAT x00, x01, x02, x03;
+	FLOAT g0, g1, g2, g3;
+	FLOAT y00, y01, y02, y03;
+	FLOAT *x1=x;
+	FLOAT *y1=y;
+	FLOAT c1=*c;
+	FLOAT s1=*s;
+
+	while ( i<n )
+	{
+
+		x00 = x1[0];
+		y00 = y1[0];
+		x01 = x1[1];
+		y01 = y1[1];
+		x02 = x1[2];
+		y02 = y1[2];
+		x03 = x1[3];
+		y03 = y1[3];
+
+		f0 = c1*x00 + s1*y00;
+		g0 = c1*y00 - s1*x00;
+		f1 = c1*x01 + s1*y01;
+		g1 = c1*y01 - s1*x01;
+		f2 = c1*x02 + s1*y02;
+		g2 = c1*y02 - s1*x02;
+		f3 = c1*x03 + s1*y03;
+		g3 = c1*y03 - s1*x03;
+
+		x1[0] = f0;
+		y1[0] = g0;
+		x1[1] = f1;
+		y1[1] = g1;
+		x1[2] = f2;
+		y1[2] = g2;
+		x1[3] = f3;
+		y1[3] = g3;
+
+		x1 += 4;
+		y1 += 4;
+
+		i+=4;
+	}
+	return;
+
+}
+
+
+#endif
+
+
+int CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y, FLOAT c, FLOAT s)
+{
+	BLASLONG i=0;
+	BLASLONG ix=0,iy=0;
+	FLOAT c1[4] __attribute__ ((aligned (16)));;
+	FLOAT s1[4] __attribute__ ((aligned (16)));;
+	FLOAT *x1=x;
+	FLOAT *y1=y;
+	FLOAT temp;
+
+	if ( n <= 0     )  return(0);
+
+	if ( (inc_x == 1) && (inc_y == 1) )
+	{
+
+		BLASLONG n1 = n & -16;
+		if ( n1 > 0 )
+		{
+			c1[0]=c;
+			c1[1]=c;
+			c1[2]=c;
+			c1[3]=c;
+			s1[0]=s;
+			s1[1]=s;
+			s1[2]=s;
+			s1[3]=s;
+			srot_kernel_16(n1, x1, y1, c1, s1);
+			i=n1;
+		}
+
+		while(i < n)
+		{
+			temp  = c*x[i] + s*y[i] ;
+			y[i]  = c*y[i] - s*x[i] ;
+			x[i]  = temp ;
+
+			i++ ;
+
+		}
+
+
+	}
+	else
+	{
+
+		while(i < n)
+		{
+			temp   = c*x[ix] + s*y[iy] ;
+			y[iy]  = c*y[iy] - s*x[ix] ;
+			x[ix]  = temp ;
+
+			ix += inc_x ;
+			iy += inc_y ;
+			i++ ;
+
+		}
+
+	}
+	return(0);
+
+}
+
+

kernel/power/srot_microk_power8.c

@@ -0,0 +1,208 @@
+/***************************************************************************
+Copyright (c) 2013-2016, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+/**************************************************************************************
+* 2016/03/27 Werner Saar (wernsaar@googlemail.com)
+*
+* I don't use fused multiply-add ( precision problems with lapack )
+*
+* 	 BLASTEST 		: OK
+* 	 CTEST			: OK
+* 	 TEST			: OK
+*	 LAPACK-TEST		: OK
+**************************************************************************************/
+
+#define HAVE_KERNEL_16 1
+
+static void srot_kernel_16( BLASLONG n, FLOAT *x, FLOAT *y, FLOAT *c, FLOAT *s) __attribute__ ((noinline));
+
+static void srot_kernel_16( BLASLONG n, FLOAT *x, FLOAT *y, FLOAT *c, FLOAT *s)
+{
+
+
+	BLASLONG i = n;
+	BLASLONG o16 = 16;
+	BLASLONG o32 = 32;
+	BLASLONG o48 = 48;
+	FLOAT *x1=x;
+	FLOAT *y1=y;
+	FLOAT *x2=x+1;
+	FLOAT *y2=y+1;
+
+	__asm__  __volatile__
+	(
+
+        "lxvw4x         36 , 0, %3                          \n\t"	// load c
+        "lxvw4x         37 , 0, %4                          \n\t"	// load s
+	"addi		%8 , %8, -4			     \n\t"
+	"addi		%9 , %9, -4			     \n\t"
+
+	"lxvw4x		32, 0, %1			    \n\t"	// load x
+	"lxvw4x		33, %5, %1			    \n\t"
+	"lxvw4x		34, %6, %1			    \n\t"
+	"lxvw4x		35, %7, %1			    \n\t"
+
+	"lxvw4x		40, 0, %2			    \n\t"	// load y
+	"lxvw4x		41, %5, %2			    \n\t"
+	"lxvw4x		42, %6, %2			    \n\t"
+	"lxvw4x		43, %7, %2			    \n\t"
+
+	"addi		%1, %1, 64			    \n\t"
+	"addi		%2, %2, 64			    \n\t"
+
+	"addic.		%0 , %0	, -16  	 	             \n\t"
+	"ble		2f		             	     \n\t"
+
+	".align 5				            \n\t"
+	"1:				                    \n\t"
+
+	"xvmulsp	48, 32, 36		    	    \n\t"	// c * x
+	"xvmulsp	49, 33, 36		    	    \n\t"
+	"xvmulsp	50, 34, 36		    	    \n\t"
+	"xvmulsp	51, 35, 36		    	    \n\t"
+
+	"xvmulsp	56, 40, 36		    	    \n\t"	// c * y
+	"xvmulsp	57, 41, 36		    	    \n\t"
+	"xvmulsp	58, 42, 36		    	    \n\t"
+	"xvmulsp	59, 43, 36		    	    \n\t"
+
+	"xvmulsp	52, 32, 37		    	    \n\t"	// s * x
+	"xvmulsp	53, 33, 37		    	    \n\t"
+
+	"lxvw4x		32, 0, %1			    \n\t"	// load x
+	"lxvw4x		33, %5, %1			    \n\t"
+
+	"xvmulsp	54, 34, 37		    	    \n\t"
+	"xvmulsp	55, 35, 37		    	    \n\t"
+
+	"lxvw4x		34, %6, %1			    \n\t"
+	"lxvw4x		35, %7, %1			    \n\t"
+
+	"xvmulsp	60, 40, 37		    	    \n\t"	// s * y
+	"xvmulsp	61, 41, 37		    	    \n\t"
+
+	"lxvw4x		40, 0, %2			    \n\t"	// load y
+	"lxvw4x		41, %5, %2			    \n\t"
+
+	"xvmulsp	62, 42, 37		    	    \n\t"
+	"xvmulsp	63, 43, 37		    	    \n\t"
+
+	"lxvw4x		42, %6, %2			    \n\t"
+	"lxvw4x		43, %7, %2			    \n\t"
+
+	"xvaddsp	48, 48 , 60			    \n\t"	// c * x + s * y 
+	"xvaddsp	49, 49 , 61			    \n\t"	// c * x + s * y 
+
+	"addi		%1, %1, 64			    \n\t"
+	"addi		%2, %2, 64			    \n\t"
+
+	"xvaddsp	50, 50 , 62			    \n\t"	// c * x + s * y 
+	"xvaddsp	51, 51 , 63			    \n\t"	// c * x + s * y 
+
+	"xvsubsp	56, 56 , 52			    \n\t"	// c * y - s * x
+	"xvsubsp	57, 57 , 53			    \n\t"	// c * y - s * x
+	"xvsubsp	58, 58 , 54			    \n\t"	// c * y - s * x
+	"xvsubsp	59, 59 , 55			    \n\t"	// c * y - s * x
+
+	"stxvw4x	48, 0, %8			    \n\t"	// store x
+	"stxvw4x	49, %5, %8			    \n\t"
+	"stxvw4x	50, %6, %8			    \n\t"
+	"stxvw4x	51, %7, %8			    \n\t"
+
+	"stxvw4x	56, 0, %9			    \n\t"	// store y
+	"stxvw4x	57, %5, %9			    \n\t"
+	"stxvw4x	58, %6, %9			    \n\t"
+	"stxvw4x	59, %7, %9			    \n\t"
+
+	"addi		%8, %8, 64			    \n\t"
+	"addi		%9, %9, 64			    \n\t"
+
+	"addic.		%0 , %0	, -16  	 	             \n\t"
+	"bgt		1b		             	     \n\t"
+
+	"2:						     \n\t"
+
+	"xvmulsp	48, 32, 36		    	    \n\t"	// c * x
+	"xvmulsp	49, 33, 36		    	    \n\t"
+	"xvmulsp	50, 34, 36		    	    \n\t"
+	"xvmulsp	51, 35, 36		    	    \n\t"
+
+	"xvmulsp	56, 40, 36		    	    \n\t"	// c * y
+	"xvmulsp	57, 41, 36		    	    \n\t"
+	"xvmulsp	58, 42, 36		    	    \n\t"
+	"xvmulsp	59, 43, 36		    	    \n\t"
+
+	"xvmulsp	52, 32, 37		    	    \n\t"	// s * x
+	"xvmulsp	53, 33, 37		    	    \n\t"
+	"xvmulsp	54, 34, 37		    	    \n\t"
+	"xvmulsp	55, 35, 37		    	    \n\t"
+
+	"xvmulsp	60, 40, 37		    	    \n\t"	// s * y
+	"xvmulsp	61, 41, 37		    	    \n\t"
+	"xvmulsp	62, 42, 37		    	    \n\t"
+	"xvmulsp	63, 43, 37		    	    \n\t"
+
+	"xvaddsp	48, 48 , 60			    \n\t"	// c * x + s * y 
+	"xvaddsp	49, 49 , 61			    \n\t"	// c * x + s * y 
+	"xvaddsp	50, 50 , 62			    \n\t"	// c * x + s * y 
+	"xvaddsp	51, 51 , 63			    \n\t"	// c * x + s * y 
+
+	"xvsubsp	56, 56 , 52			    \n\t"	// c * y - s * x
+	"xvsubsp	57, 57 , 53			    \n\t"	// c * y - s * x
+	"xvsubsp	58, 58 , 54			    \n\t"	// c * y - s * x
+	"xvsubsp	59, 59 , 55			    \n\t"	// c * y - s * x
+
+	"stxvw4x	48, 0, %8			    \n\t"	// store x
+	"stxvw4x	49, %5, %8			    \n\t"
+	"stxvw4x	50, %6, %8			    \n\t"
+	"stxvw4x	51, %7, %8			    \n\t"
+
+	"stxvw4x	56, 0, %9			    \n\t"	// store y
+	"stxvw4x	57, %5, %9			    \n\t"
+	"stxvw4x	58, %6, %9			    \n\t"
+	"stxvw4x	59, %7, %9			    \n\t"
+
+
+
+	:
+        : 
+          "r" (i),	// 0	
+	  "r" (x1),  	// 1
+          "r" (y1),     // 2
+          "r" (c),      // 3
+          "r" (s),      // 4
+	  "r" (o16),	// 5
+	  "r" (o32),	// 6
+	  "r" (o48),    // 7
+	  "r" (x2),     // 8
+	  "r" (y2)      // 9
+	: "cr0", "%0", "%1" , "%2", "%8", "%9", "memory"
+	);
+
+} 
+
+

kernel/power/sscal.c

@@ -0,0 +1,179 @@
+/***************************************************************************
+Copyright (c) 2013-2016, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+/**************************************************************************************
+* 2016/03/27 Werner Saar (wernsaar@googlemail.com)
+* 	 BLASTEST 		: OK
+* 	 CTEST			: OK
+* 	 TEST			: OK
+*	 LAPACK-TEST		: OK
+**************************************************************************************/
+
+#include "common.h"
+
+#if defined(POWER8) 
+#include "sscal_microk_power8.c"
+#endif
+
+
+#if !defined(HAVE_KERNEL_16)
+
+static void sscal_kernel_16( BLASLONG n, FLOAT *da , FLOAT *x )
+{
+
+        BLASLONG i;
+        FLOAT alpha = *da;
+
+        for( i=0; i<n; i+=8 )
+        {
+                x[0] *= alpha;
+                x[1] *= alpha;
+                x[2] *= alpha;
+                x[3] *= alpha;
+                x[4] *= alpha;
+                x[5] *= alpha;
+                x[6] *= alpha;
+                x[7] *= alpha;
+                x+=8;
+        }
+
+}
+
+static void sscal_kernel_16_zero( BLASLONG n, FLOAT *da , FLOAT *x )
+{
+
+        BLASLONG i;
+	FLOAT alpha=0.0;
+
+        for( i=0; i<n; i+=8 )
+        {
+                x[0] = alpha;
+                x[1] = alpha;
+                x[2] = alpha;
+                x[3] = alpha;
+                x[4] = alpha;
+                x[5] = alpha;
+                x[6] = alpha;
+                x[7] = alpha;
+                x+=8;
+        }
+
+}
+
+
+#endif
+
+int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y, FLOAT *dummy, BLASLONG dummy2)
+{
+	BLASLONG i=0,j=0;
+	FLOAT alpha[4] __attribute__ ((aligned (16)));;
+
+	if ( n <= 0 || inc_x <=0 )
+		return(0);
+
+
+	if ( inc_x == 1 )
+	{
+
+		if ( da == 0.0 )
+		{		
+
+			BLASLONG n1 = n & -32;
+			if ( n1 > 0 )
+			{
+				alpha[0]=da;
+				alpha[1]=da;
+				alpha[2]=da;
+				alpha[3]=da;
+				sscal_kernel_16_zero(n1 , alpha , x);
+				j=n1;
+			}
+
+			while(j < n)
+			{
+
+				x[j]=0.0;
+				j++;
+			}
+
+		}
+		else
+		{
+
+			BLASLONG n1 = n & -32;
+			if ( n1 > 0 )
+			{
+				alpha[0]=da;
+				alpha[1]=da;
+				alpha[2]=da;
+				alpha[3]=da;
+				sscal_kernel_16(n1 , alpha , x);
+				j=n1;
+			}
+			while(j < n)
+			{
+
+				x[j] = da * x[j] ;
+				j++;
+			}
+		}
+
+
+	}
+	else
+	{
+
+		if ( da == 0.0 )
+		{		
+
+			while(j < n)
+			{
+
+				x[i]=0.0;
+				i += inc_x ;
+				j++;
+			}
+
+		}
+		else
+		{
+
+			while(j < n)
+			{
+
+				x[i] = da * x[i] ;
+				i += inc_x ;
+				j++;
+			}
+		}
+
+	}
+	return 0;
+
+}
+
+

kernel/power/sscal_microk_power8.c

@@ -0,0 +1,218 @@
+/***************************************************************************
+Copyright (c) 2013-2016, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+/**************************************************************************************
+* 2016/03/27 Werner Saar (wernsaar@googlemail.com)
+* 	 BLASTEST 		: OK
+* 	 CTEST			: OK
+* 	 TEST			: OK
+*	 LAPACK-TEST		: OK
+**************************************************************************************/
+
+#define HAVE_KERNEL_16 1
+
+static void sscal_kernel_16( BLASLONG n, FLOAT *alpha, FLOAT *x) __attribute__ ((noinline));
+
+static void sscal_kernel_16( BLASLONG n, FLOAT *alpha, FLOAT *x)
+{
+
+
+	BLASLONG i = n;
+	BLASLONG o16 = 16;
+	BLASLONG o32 = 32;
+	BLASLONG o48 = 48;
+	BLASLONG o64 = 64;
+	BLASLONG o80 = 80;
+	BLASLONG o96 = 96;
+	BLASLONG o112 = 112;
+	FLOAT *x1=x;
+	FLOAT *x2=x+1;
+	BLASLONG pre = 384;
+
+	__asm__  __volatile__
+	(
+
+        "lxvw4x         32, 0, %3                           \n\t"
+        "addi           %1, %1, -4                          \n\t"
+
+	"dcbt		%2, %4				    \n\t"
+
+	"lxvw4x		40, 0, %2			    \n\t"
+	"lxvw4x		41, %5, %2			    \n\t"
+	"lxvw4x		42, %6, %2			    \n\t"
+	"lxvw4x		43, %7, %2			    \n\t"
+	"lxvw4x		44, %8, %2			    \n\t"
+	"lxvw4x		45, %9, %2			    \n\t"
+	"lxvw4x		46, %10, %2			    \n\t"
+	"lxvw4x		47, %11, %2			    \n\t"
+
+	"addi		%2, %2, 128			    \n\t"
+
+	"addic.		%0 , %0	, -32  	 	             \n\t"
+	"ble		2f		             	     \n\t"
+
+	".align 5				            \n\t"
+	"1:				                    \n\t"
+
+	"dcbt		%2, %4				    \n\t"
+
+	"xvmulsp	48, 40, 32		    	    \n\t"
+	"xvmulsp	49, 41, 32		    	    \n\t"
+	"lxvw4x		40, 0, %2			    \n\t"
+	"lxvw4x		41, %5, %2			    \n\t"
+	"xvmulsp	50, 42, 32		    	    \n\t"
+	"xvmulsp	51, 43, 32		    	    \n\t"
+	"lxvw4x		42, %6, %2			    \n\t"
+	"lxvw4x		43, %7, %2			    \n\t"
+	"xvmulsp	52, 44, 32		    	    \n\t"
+	"xvmulsp	53, 45, 32		    	    \n\t"
+	"lxvw4x		44, %8, %2			    \n\t"
+	"lxvw4x		45, %9, %2			    \n\t"
+	"xvmulsp	54, 46, 32		    	    \n\t"
+	"xvmulsp	55, 47, 32		    	    \n\t"
+	"lxvw4x		46, %10, %2			    \n\t"
+	"lxvw4x		47, %11, %2			    \n\t"
+
+	"stxvw4x	48, 0, %1			    \n\t"
+	"stxvw4x	49, %5, %1			    \n\t"
+	"stxvw4x	50, %6, %1			    \n\t"
+	"stxvw4x	51, %7, %1			    \n\t"
+	"stxvw4x	52, %8, %1			    \n\t"
+	"stxvw4x	53, %9, %1			    \n\t"
+	"stxvw4x	54, %10, %1			    \n\t"
+	"stxvw4x	55, %11, %1			    \n\t"
+
+	"addi		%1, %1, 128			    \n\t"
+	"addi		%2, %2, 128			    \n\t"
+
+	"addic.		%0 , %0	, -32  	 	             \n\t"
+	"bgt		1b		             	     \n\t"
+
+	"2:						     \n\t"
+
+	"xvmulsp	48, 40, 32		    	    \n\t"
+	"xvmulsp	49, 41, 32		    	    \n\t"
+	"xvmulsp	50, 42, 32		    	    \n\t"
+	"xvmulsp	51, 43, 32		    	    \n\t"
+	"xvmulsp	52, 44, 32		    	    \n\t"
+	"xvmulsp	53, 45, 32		    	    \n\t"
+	"xvmulsp	54, 46, 32		    	    \n\t"
+	"xvmulsp	55, 47, 32		    	    \n\t"
+
+	"stxvw4x	48, 0, %1			    \n\t"
+	"stxvw4x	49, %5, %1			    \n\t"
+	"stxvw4x	50, %6, %1			    \n\t"
+	"stxvw4x	51, %7, %1			    \n\t"
+	"stxvw4x	52, %8, %1			    \n\t"
+	"stxvw4x	53, %9, %1			    \n\t"
+	"stxvw4x	54, %10, %1			    \n\t"
+	"stxvw4x	55, %11, %1			    \n\t"
+
+	:
+        : 
+          "r" (i),	// 0	
+	  "r" (x2),  	// 1
+          "r" (x1),     // 2
+          "r" (alpha),  // 3
+          "r" (pre),    // 4
+	  "r" (o16),	// 5
+	  "r" (o32),	// 6
+	  "r" (o48),    // 7
+          "r" (o64),    // 8
+          "r" (o80),    // 9
+          "r" (o96),    // 10
+          "r" (o112)    // 11
+	: "cr0", "%0", "%2" , "%1", "memory"
+	);
+
+} 
+
+
+static void sscal_kernel_16_zero( BLASLONG n, FLOAT *alpha, FLOAT *x) __attribute__ ((noinline));
+
+static void sscal_kernel_16_zero( BLASLONG n, FLOAT *alpha, FLOAT *x)
+{
+
+
+	BLASLONG i = n;
+	BLASLONG o16 = 16;
+	BLASLONG o32 = 32;
+	BLASLONG o48 = 48;
+	BLASLONG o64 = 64;
+	BLASLONG o80 = 80;
+	BLASLONG o96 = 96;
+	BLASLONG o112 = 112;
+	FLOAT *x1=x;
+	FLOAT *x2=x+1;
+	BLASLONG pre = 384;
+
+	__asm__  __volatile__
+	(
+
+	"xxlxor		32 , 32 , 32			    \n\t"
+        "addi           %1, %1, -4                          \n\t"
+
+
+	".align 5				            \n\t"
+	"1:				                    \n\t"
+
+	"stxvw4x	32, 0, %1			    \n\t"
+	"stxvw4x	32, %5, %1			    \n\t"
+	"stxvw4x	32, %6, %1			    \n\t"
+	"stxvw4x	32, %7, %1			    \n\t"
+	"stxvw4x	32, %8, %1			    \n\t"
+	"stxvw4x	32, %9, %1			    \n\t"
+	"stxvw4x	32, %10, %1			    \n\t"
+	"stxvw4x	32, %11, %1			    \n\t"
+
+	"addi		%1, %1, 128			    \n\t"
+
+	"addic.		%0 , %0	, -32  	 	             \n\t"
+	"bgt		1b		             	     \n\t"
+
+	"2:						     \n\t"
+
+	:
+        : 
+          "r" (i),	// 0	
+	  "r" (x2),  	// 1
+          "r" (x1),     // 2
+          "r" (alpha),  // 3
+          "r" (pre),    // 4
+	  "r" (o16),	// 5
+	  "r" (o32),	// 6
+	  "r" (o48),    // 7
+          "r" (o64),    // 8
+          "r" (o80),    // 9
+          "r" (o96),    // 10
+          "r" (o112)    // 11
+	: "cr0", "%0", "%2" , "%1", "memory"
+	);
+
+} 
+
+

kernel/power/sswap.c

@@ -0,0 +1,154 @@
+/***************************************************************************
+Copyright (c) 2013-2016, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+/**************************************************************************************
+* 2016/03/25 Werner Saar (wernsaar@googlemail.com)
+* 	 BLASTEST 		: OK
+* 	 CTEST			: OK
+* 	 TEST			: OK
+*	 LAPACK-TEST		: OK
+**************************************************************************************/
+
+#include "common.h"
+
+#if defined(POWER8)
+#include "sswap_microk_power8.c"
+#endif
+
+#ifndef HAVE_KERNEL_32
+
+static void sswap_kernel_32(BLASLONG n, FLOAT *x, FLOAT *y)
+{
+
+	BLASLONG i=0;
+	FLOAT f0, f1, f2, f3, f4, f5, f6, f7;
+	FLOAT g0, g1, g2, g3, g4, g5, g6, g7;
+	FLOAT *x1=x;
+	FLOAT *y1=y;
+
+	while ( i<n )
+	{
+
+		f0 = x1[0];
+		f1 = x1[1];
+		f2 = x1[2];
+		f3 = x1[3];
+		f4 = x1[4];
+		f5 = x1[5];
+		f6 = x1[6];
+		f7 = x1[7];
+
+		g0 = y1[0];
+		g1 = y1[1];
+		g2 = y1[2];
+		g3 = y1[3];
+		g4 = y1[4];
+		g5 = y1[5];
+		g6 = y1[6];
+		g7 = y1[7];
+
+		y1[0] = f0;
+		y1[1] = f1;
+		y1[2] = f2;
+		y1[3] = f3;
+		y1[4] = f4;
+		y1[5] = f5;
+		y1[6] = f6;
+		y1[7] = f7;
+
+		x1[0] = g0;
+		x1[1] = g1;
+		x1[2] = g2;
+		x1[3] = g3;
+		x1[4] = g4;
+		x1[5] = g5;
+		x1[6] = g6;
+		x1[7] = g7;
+
+		x1 += 8;
+		y1 += 8;
+
+		i+=8;
+	}
+	return;
+
+}
+
+
+#endif
+
+
+int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT dummy3, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y, FLOAT *dummy, BLASLONG dummy2)
+{
+	BLASLONG i=0;
+	BLASLONG ix=0,iy=0;
+	FLOAT temp;
+
+	if ( n <= 0     )  return(0);
+
+	if ( (inc_x == 1) && (inc_y == 1 ))
+	{
+
+		BLASLONG n1 = n & -32;
+		if ( n1 > 0 )
+		{
+			sswap_kernel_32(n1, x, y);
+			i=n1;
+		}
+
+		while(i < n)
+		{
+			temp = y[i];	
+			y[i] = x[i] ;
+			x[i] = temp;
+			i++ ;
+
+		}
+
+
+	}
+	else
+	{
+
+		while(i < n)
+		{
+			temp  = y[iy];
+			y[iy] = x[ix] ;
+			x[ix] = temp;
+			ix += inc_x ;
+			iy += inc_y ;
+			i++ ;
+
+		}
+
+	}
+	return(0);
+	
+
+}
+
+

kernel/power/sswap_microk_power8.c

@@ -0,0 +1,136 @@
+/***************************************************************************
+Copyright (c) 2013-2016, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+/**************************************************************************************
+* 2016/03/25 Werner Saar (wernsaar@googlemail.com)
+* 	 BLASTEST 		: OK
+* 	 CTEST			: OK
+* 	 TEST			: OK
+*	 LAPACK-TEST		: OK
+**************************************************************************************/
+
+#define HAVE_KERNEL_32 1
+
+static void sswap_kernel_32( BLASLONG n, FLOAT *x, FLOAT *y) __attribute__ ((noinline));
+
+static void sswap_kernel_32( BLASLONG n, FLOAT *x, FLOAT *y)
+{
+
+
+	BLASLONG i = n;
+	BLASLONG o16 = 16;
+	BLASLONG o32 = 32;
+	BLASLONG o48 = 48;
+	BLASLONG o64 = 64;
+	BLASLONG o80 = 80;
+	BLASLONG o96 = 96;
+	BLASLONG o112 = 112;
+	FLOAT *x1=x;
+	FLOAT *y1=y;
+	FLOAT *x2=x+1;
+	FLOAT *y2=y+1;
+	BLASLONG pre = 384;
+	BLASLONG alpha=0;
+
+	__asm__  __volatile__
+	(
+
+	"addi		%3, %3, -4			    \n\t"	
+	"addi		%4, %4, -4			    \n\t"	
+
+	".align 5				            \n\t"
+	"1:				                    \n\t"
+
+	"lxvw4x		32, 0, %2			    \n\t"
+	"lxvw4x		33, %5, %2			    \n\t"
+	"lxvw4x		34, %6, %2			    \n\t"
+	"lxvw4x		35, %7, %2			    \n\t"
+	"lxvw4x		36, %8, %2			    \n\t"
+	"lxvw4x		37, %9, %2			    \n\t"
+	"lxvw4x		38, %10, %2			    \n\t"
+	"lxvw4x		39, %11, %2			    \n\t"
+
+	"addi		%2, %2, 128			    \n\t"
+
+	"lxvw4x		48, 0, %1			    \n\t"
+	"lxvw4x		49, %5, %1			    \n\t"
+	"lxvw4x		50, %6, %1			    \n\t"
+	"lxvw4x		51, %7, %1			    \n\t"
+	"lxvw4x		52, %8, %1			    \n\t"
+	"lxvw4x		53, %9, %1			    \n\t"
+	"lxvw4x		54, %10, %1			    \n\t"
+	"lxvw4x		55, %11, %1			    \n\t"
+
+	"addi		%1, %1, 128			    \n\t"
+
+	"stxvw4x		32, 0, %3			    \n\t"
+	"stxvw4x		33, %5, %3			    \n\t"
+	"stxvw4x		34, %6, %3			    \n\t"
+	"stxvw4x		35, %7, %3			    \n\t"
+	"stxvw4x		36, %8, %3			    \n\t"
+	"stxvw4x		37, %9, %3			    \n\t"
+	"stxvw4x		38, %10, %3			    \n\t"
+	"stxvw4x		39, %11, %3			    \n\t"
+
+	"addi		%3, %3, 128			    \n\t"
+
+	"stxvw4x		48, 0, %4			    \n\t"
+	"stxvw4x		49, %5, %4			    \n\t"
+	"stxvw4x		50, %6, %4			    \n\t"
+	"stxvw4x		51, %7, %4			    \n\t"
+	"stxvw4x		52, %8, %4			    \n\t"
+	"stxvw4x		53, %9, %4			    \n\t"
+	"stxvw4x		54, %10, %4			    \n\t"
+	"stxvw4x		55, %11, %4			    \n\t"
+
+	"addi		%4, %4, 128			    \n\t"
+
+	"addic.		%0 , %0	, -32  	 	             \n\t"
+	"bgt		1b		             	     \n\t"
+
+	"2:						     \n\t"
+
+	:
+        : 
+          "r" (i),	// 0	
+	  "r" (y1),  	// 1
+          "r" (x1),     // 2
+          "r" (y2),     // 3
+          "r" (x2),     // 4
+	  "r" (o16),	// 5
+	  "r" (o32),	// 6
+	  "r" (o48),    // 7
+          "r" (o64),    // 8
+          "r" (o80),    // 9
+          "r" (o96),    // 10
+          "r" (o112)    // 11
+	: "cr0", "%0", "%2" , "%1", "%3", "%4", "memory"
+	);
+
+} 
+
+

kernel/power/strmm_kernel_16x8_power8.S

@@ -0,0 +1,369 @@
+/***************************************************************************
+Copyright (c) 2013-2016, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+/**************************************************************************************
+* 2016/04/02 Werner Saar (wernsaar@googlemail.com)
+* 	 BLASTEST 		: OK
+* 	 CTEST			: OK
+* 	 TEST			: OK
+* 	 LAPACK-TEST		: OK
+**************************************************************************************/
+
+/*********************************************************************/
+/* Copyright 2009, 2010 The University of Texas at Austin.           */
+/* All rights reserved.                                              */
+/*                                                                   */
+/* Redistribution and use in source and binary forms, with or        */
+/* without modification, are permitted provided that the following   */
+/* conditions are met:                                               */
+/*                                                                   */
+/*   1. Redistributions of source code must retain the above         */
+/*      copyright notice, this list of conditions and the following  */
+/*      disclaimer.                                                  */
+/*                                                                   */
+/*   2. Redistributions in binary form must reproduce the above      */
+/*      copyright notice, this list of conditions and the following  */
+/*      disclaimer in the documentation and/or other materials       */
+/*      provided with the distribution.                              */
+/*                                                                   */
+/*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */
+/*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */
+/*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */
+/*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */
+/*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */
+/*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */
+/*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */
+/*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */
+/*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */
+/*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */
+/*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */
+/*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */
+/*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */
+/*    POSSIBILITY OF SUCH DAMAGE.                                    */
+/*                                                                   */
+/* The views and conclusions contained in the software and           */
+/* documentation are those of the authors and should not be          */
+/* interpreted as representing official policies, either expressed   */
+/* or implied, of The University of Texas at Austin.                 */
+/*********************************************************************/
+
+#define ASSEMBLER
+#include "common.h"
+#include "def_vsx.h"
+
+#ifndef __64BIT__
+#define LOAD	lwz
+#else
+#define LOAD	ld
+#endif
+
+#ifdef __64BIT__
+#define STACKSIZE 340
+#define ALPHA_SP   296(SP)
+#define FZERO	304(SP)
+#else
+#define STACKSIZE 240
+#define ALPHA_SP   224(SP)
+#define FZERO	232(SP)
+#endif
+
+#define	M	r3
+#define	N	r4
+#define	K	r5
+
+#ifdef linux
+#ifndef __64BIT__
+#define A	r6
+#define	B	r7
+#define	C	r8
+#define	LDC	r9
+#define OFFSET	r10
+#else
+#define A	r7
+#define	B	r8
+#define	C	r9
+#define	LDC	r10
+#define OFFSET	r6
+#endif
+#endif
+
+#if defined(_AIX) || defined(__APPLE__)
+#if !defined(__64BIT__) && defined(DOUBLE)
+#define A	r8
+#define	B	r9
+#define	C	r10
+#define	LDC	r7
+#define OFFSET	r6
+#else
+#define A	r7
+#define	B	r8
+#define	C	r9
+#define	LDC	r10
+#define OFFSET	r6
+#endif
+#endif
+
+#define alpha_r vs30
+#define alpha_vr vs31
+
+#define o0	0
+
+#define TBUFFER r13
+#define o12	r14
+#define o4	r15
+#define K1	r16
+#define o8	r17
+#define L	r18
+#define T1	r19
+#define KK	r20
+#define KKK	r21
+#define	I	r22
+#define J	r23
+#define AO	r24
+#define	BO	r25
+#define	CO 	r26
+#define o16	r27
+#define	o32	r28
+#define	o48	r29
+
+#define PRE	r30
+#define T2	r31
+
+#include "strmm_macros_16x8_power8.S"
+
+
+#ifndef NEEDPARAM
+
+	PROLOGUE
+	PROFCODE
+
+	addi	SP, SP, -STACKSIZE
+	li	r0, 0
+
+	stfd	f14,    0(SP)
+	stfd	f15,    8(SP)
+	stfd	f16,   16(SP)
+	stfd	f17,   24(SP)
+
+	stfd	f18,   32(SP)
+	stfd	f19,   40(SP)
+	stfd	f20,   48(SP)
+	stfd	f21,   56(SP)
+
+	stfd	f22,   64(SP)
+	stfd	f23,   72(SP)
+	stfd	f24,   80(SP)
+	stfd	f25,   88(SP)
+
+	stfd	f26,   96(SP)
+	stfd	f27,  104(SP)
+	stfd	f28,  112(SP)
+	stfd	f29,  120(SP)
+
+	stfd	f30,  128(SP)
+	stfd	f31,  136(SP)
+
+#ifdef __64BIT__
+	std	r31,  144(SP)
+	std	r30,  152(SP)
+	std	r29,  160(SP)
+	std	r28,  168(SP)
+	std	r27,  176(SP)
+	std	r26,  184(SP)
+	std	r25,  192(SP)
+	std	r24,  200(SP)
+	std	r23,  208(SP)
+	std	r22,  216(SP)
+	std	r21,  224(SP)
+	std	r20,  232(SP)
+	std	r19,  240(SP)
+	std	r18,  248(SP)
+	std	r17,  256(SP)
+	std	r16,  264(SP)
+	std	r15,  272(SP)
+	std	r14,  280(SP)
+	std	r13,  288(SP)
+#else
+	stw	r31,  144(SP)
+	stw	r30,  148(SP)
+	stw	r29,  152(SP)
+	stw	r28,  156(SP)
+	stw	r27,  160(SP)
+	stw	r26,  164(SP)
+	stw	r25,  168(SP)
+	stw	r24,  172(SP)
+	stw	r23,  176(SP)
+	stw	r22,  180(SP)
+	stw	r21,  184(SP)
+	stw	r20,  188(SP)
+	stw	r19,  192(SP)
+	stw	r18,  196(SP)
+	stw	r17,  200(SP)
+	stw	r16,  204(SP)
+	stw	r15,  208(SP)
+	stw	r14,  212(SP)
+	stw	r13,  216(SP)
+#endif
+
+	// stfd	f1,  ALPHA_SP
+	// stw	r0,  FZERO
+
+#if defined(_AIX) || defined(__APPLE__)
+#if !defined(__64BIT__) && defined(DOUBLE)
+	lwz	LDC, FRAMESLOT(0) + STACKSIZE(SP)
+#endif
+#endif
+
+	slwi	LDC, LDC, BASE_SHIFT
+
+#if defined(TRMMKERNEL)
+#if defined(linux) && defined(__64BIT__)
+	ld	OFFSET,  FRAMESLOT(0) + STACKSIZE(SP)
+#endif
+
+#if defined(_AIX) || defined(__APPLE__)
+#ifdef __64BIT__
+	ld	OFFSET,  FRAMESLOT(0) + STACKSIZE(SP)
+#else
+#ifdef DOUBLE
+	lwz	OFFSET,  FRAMESLOT(1) + STACKSIZE(SP)
+#else
+	lwz	OFFSET,  FRAMESLOT(0) + STACKSIZE(SP)
+#endif
+#endif
+#endif
+#endif
+
+        mr      KK, OFFSET
+#if defined(TRMMKERNEL) && !defined(LEFT)
+        neg     KK, KK
+#endif
+
+
+	cmpwi	cr0, M, 0
+	ble	L999_H1
+	cmpwi	cr0, N, 0
+	ble	L999_H1
+	cmpwi	cr0, K, 0
+	ble	L999_H1
+
+	li	PRE, 256 
+	li	o4 , 4
+	li	o8 , 8
+	li	o12, 12
+	li	o16, 16
+	li	o32, 32
+	li	o48, 48
+	addi	TBUFFER, SP, 320
+
+        addi    T1, SP, 300
+        stxsspx    f1, o0 , T1
+        stxsspx    f1, o4 , T1
+        stxsspx    f1, o8 , T1
+        stxsspx    f1, o12 , T1
+
+	lxsspx	   alpha_r, o0,  T1
+        lxvw4x     alpha_vr, o0, T1
+
+
+
+#include "strmm_logic_16x8_power8.S"
+
+L999:
+	addi	r3, 0, 0
+
+	lfd	f14,    0(SP)
+	lfd	f15,    8(SP)
+	lfd	f16,   16(SP)
+	lfd	f17,   24(SP)
+
+	lfd	f18,   32(SP)
+	lfd	f19,   40(SP)
+	lfd	f20,   48(SP)
+	lfd	f21,   56(SP)
+
+	lfd	f22,   64(SP)
+	lfd	f23,   72(SP)
+	lfd	f24,   80(SP)
+	lfd	f25,   88(SP)
+
+	lfd	f26,   96(SP)
+	lfd	f27,  104(SP)
+	lfd	f28,  112(SP)
+	lfd	f29,  120(SP)
+
+	lfd	f30,  128(SP)
+	lfd	f31,  136(SP)
+
+#ifdef __64BIT__
+	ld	r31,  144(SP)
+	ld	r30,  152(SP)
+	ld	r29,  160(SP)
+	ld	r28,  168(SP)
+	ld	r27,  176(SP)
+	ld	r26,  184(SP)
+	ld	r25,  192(SP)
+	ld	r24,  200(SP)
+	ld	r23,  208(SP)
+	ld	r22,  216(SP)
+	ld	r21,  224(SP)
+	ld	r20,  232(SP)
+	ld	r19,  240(SP)
+	ld	r18,  248(SP)
+	ld	r17,  256(SP)
+	ld	r16,  264(SP)
+	ld	r15,  272(SP)
+	ld	r14,  280(SP)
+	ld	r13,  288(SP)
+#else
+	lwz	r31,  144(SP)
+	lwz	r30,  148(SP)
+	lwz	r29,  152(SP)
+	lwz	r28,  156(SP)
+	lwz	r27,  160(SP)
+	lwz	r26,  164(SP)
+	lwz	r25,  168(SP)
+	lwz	r24,  172(SP)
+	lwz	r23,  176(SP)
+	lwz	r22,  180(SP)
+	lwz	r21,  184(SP)
+	lwz	r20,  188(SP)
+	lwz	r19,  192(SP)
+	lwz	r18,  196(SP)
+	lwz	r17,  200(SP)
+	lwz	r16,  204(SP)
+	lwz	r15,  208(SP)
+	lwz	r14,  212(SP)
+	lwz	r13,  216(SP)
+#endif
+
+	addi	SP, SP, STACKSIZE
+
+	blr
+
+	EPILOGUE
+#endif

kernel/power/zasum.c

@@ -0,0 +1,149 @@
+/***************************************************************************
+Copyright (c) 2013-2016, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+/**************************************************************************************
+* 2016/03/28 Werner Saar (wernsaar@googlemail.com)
+* 	 BLASTEST 		: OK
+* 	 CTEST			: OK
+* 	 TEST			: OK
+*	 LAPACK-TEST		: OK
+**************************************************************************************/
+
+#include "common.h"
+#include <math.h>
+
+#if defined(DOUBLE)
+
+#define ABS fabs
+
+#else
+
+#define ABS fabsf
+
+#endif
+
+#if defined(POWER8)
+#include "zasum_microk_power8.c"
+#endif
+
+
+#ifndef HAVE_KERNEL_8
+
+static void zasum_kernel_8(BLASLONG n, FLOAT *x1, FLOAT *svec)
+{
+
+	BLASLONG i=0;
+	FLOAT *x = x1;
+	FLOAT temp0, temp1, temp2, temp3;
+	FLOAT temp4, temp5, temp6, temp7;
+	FLOAT sum0 = 0.0;
+	FLOAT sum1 = 0.0;
+	FLOAT sum2 = 0.0;
+	FLOAT sum3 = 0.0;
+
+	while ( i< n )
+	{
+
+		temp0 = ABS(x[0]);
+		temp1 = ABS(x[1]);
+		temp2 = ABS(x[2]);
+		temp3 = ABS(x[3]);
+		temp4 = ABS(x[4]);
+		temp5 = ABS(x[5]);
+		temp6 = ABS(x[6]);
+		temp7 = ABS(x[7]);
+
+		sum0 += temp0;
+		sum1 += temp1;
+		sum2 += temp2;
+		sum3 += temp3;
+
+		sum0 += temp4;
+		sum1 += temp5;
+		sum2 += temp6;
+		sum3 += temp7;
+
+		x+=8;
+		i+=4;
+
+	}
+
+	svec[0] = sum0+sum1+sum2+sum3;
+	svec[1] = 0.0;
+
+}
+
+#endif
+
+FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x)
+{
+	BLASLONG i=0;
+	BLASLONG ip=0;
+	FLOAT sumf = 0.0;
+	FLOAT svec[2] __attribute__ ((aligned (16)));;
+	BLASLONG n1;
+	BLASLONG inc_x2;
+
+	if (n <= 0 || inc_x <= 0) return(sumf);
+
+	if ( inc_x == 1 )
+	{
+
+		n1 = n & -8;
+		if ( n1 > 0 )
+		{
+
+			zasum_kernel_8(n1, x, svec);
+			sumf = svec[0] + svec[1];
+			i=n1;
+			ip=2*n1;
+		}
+
+		while(i < n)
+		{
+			sumf += ABS(x[ip]) + ABS(x[ip+1]);
+			i++;
+			ip+=2;
+		}
+
+	}
+	else
+	{
+		inc_x2 = 2* inc_x;
+
+		while(i < n)
+		{
+			sumf += ABS(x[ip]) + ABS(x[ip+1]);
+			ip+=inc_x2;
+			i++;
+		}
+
+	}
+	return(sumf);
+}
+
+

kernel/power/zasum_microk_power8.c

@@ -0,0 +1,177 @@
+/***************************************************************************
+Copyright (c) 2013-2016, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+/**************************************************************************************
+* 2016/03/28 Werner Saar (wernsaar@googlemail.com)
+* 	 BLASTEST 		: OK
+* 	 CTEST			: OK
+* 	 TEST			: OK
+*	 LAPACK-TEST		: OK
+**************************************************************************************/
+
+#define HAVE_KERNEL_8 1
+static void zasum_kernel_8( BLASLONG n, FLOAT *x, FLOAT *svec) __attribute__ ((noinline));
+
+static void zasum_kernel_8( BLASLONG n, FLOAT *x, FLOAT *svec)
+{
+
+
+	BLASLONG i = n;
+	BLASLONG o16 = 16;
+	BLASLONG o32 = 32;
+	BLASLONG o48 = 48;
+	BLASLONG o64 = 64;
+	BLASLONG o80 = 80;
+	BLASLONG o96 = 96;
+	BLASLONG o112 = 112;
+	FLOAT *x1=x;
+	BLASLONG pre = 384;
+
+	__asm__  __volatile__
+	(
+
+	"dcbt		%2 , %4				    \n\t"
+
+	"xxlxor		32,32,32			    \n\t"
+	"xxlxor		33,33,33			    \n\t"
+	"xxlxor		34,34,34			    \n\t"
+	"xxlxor		35,35,35			    \n\t"
+	"xxlxor		36,36,36			    \n\t"
+	"xxlxor		37,37,37			    \n\t"
+	"xxlxor		38,38,38			    \n\t"
+	"xxlxor		39,39,39			    \n\t"
+
+	"lxvd2x		40, 0, %2			    \n\t"
+	"lxvd2x		41, %5, %2			    \n\t"
+	"lxvd2x		42, %6, %2			    \n\t"
+	"lxvd2x		43, %7, %2			    \n\t"
+	"lxvd2x		44, %8, %2			    \n\t"
+	"lxvd2x		45, %9, %2			    \n\t"
+	"lxvd2x		46, %10, %2			    \n\t"
+	"lxvd2x		47, %11, %2			    \n\t"
+
+	"addi		%2, %2, 128			    \n\t"
+
+	"addic.		%0 , %0	, -8  	 	             \n\t"
+	"ble		2f		             	     \n\t"
+
+	".align 5				            \n\t"
+	"1:				                    \n\t"
+
+	"dcbt		%2 , %4				    \n\t"
+
+	"xvabsdp	48, 40				    \n\t"
+	"xvabsdp	49, 41				    \n\t"
+	"xvabsdp	50, 42				    \n\t"
+	"xvabsdp	51, 43				    \n\t"
+
+	"lxvd2x		40, 0, %2			    \n\t"
+	"lxvd2x		41, %5, %2			    \n\t"
+
+	"xvabsdp	52, 44				    \n\t"
+	"xvabsdp	53, 45				    \n\t"
+
+	"lxvd2x		42, %6, %2			    \n\t"
+	"lxvd2x		43, %7, %2			    \n\t"
+
+	"xvabsdp	54, 46				    \n\t"
+	"xvabsdp	55, 47				    \n\t"
+
+	"lxvd2x		44, %8, %2			    \n\t"
+	"lxvd2x		45, %9, %2			    \n\t"
+
+	"xvadddp	32, 32, 48		    \n\t"
+	"xvadddp	33, 33, 49		    \n\t"
+
+	"lxvd2x		46, %10, %2			    \n\t"
+	"lxvd2x		47, %11, %2			    \n\t"
+
+	"xvadddp	34, 34, 50		    \n\t"
+	"xvadddp	35, 35, 51		    \n\t"
+	"addi		%2, %2, 128			    \n\t"
+	"xvadddp	36, 36, 52		    \n\t"
+	"xvadddp	37, 37, 53		    \n\t"
+	"addic.		%0 , %0	, -8  	 	             \n\t"
+	"xvadddp	38, 38, 54		    \n\t"
+	"xvadddp	39, 39, 55		    \n\t"
+
+	"bgt		1b		             	     \n\t"
+
+	"2:						     \n\t"
+
+
+	"xvabsdp	48, 40				    \n\t"
+	"xvabsdp	49, 41				    \n\t"
+	"xvabsdp	50, 42				    \n\t"
+	"xvabsdp	51, 43				    \n\t"
+	"xvabsdp	52, 44				    \n\t"
+	"xvabsdp	53, 45				    \n\t"
+	"xvabsdp	54, 46				    \n\t"
+	"xvabsdp	55, 47				    \n\t"
+
+	"xvadddp	32, 32, 48		    \n\t"
+	"xvadddp	33, 33, 49		    \n\t"
+	"xvadddp	34, 34, 50		    \n\t"
+	"xvadddp	35, 35, 51		    \n\t"
+	"xvadddp	36, 36, 52		    \n\t"
+	"xvadddp	37, 37, 53		    \n\t"
+	"xvadddp	38, 38, 54		    \n\t"
+	"xvadddp	39, 39, 55		    \n\t"
+
+	"xvadddp	32, 32, 33		     \n\t"
+	"xvadddp	34, 34, 35		     \n\t"
+	"xvadddp	36, 36, 37		     \n\t"
+	"xvadddp	38, 38, 39		     \n\t"
+
+	"xvadddp	32, 32, 34		     \n\t"
+	"xvadddp	36, 36, 38		     \n\t"
+
+	"xvadddp	32, 32, 36		     \n\t"
+
+
+	"stxvd2x	32, 0, %3		     \n\t"
+
+	:
+        : 
+          "r" (i),	// 0	
+	  "r" (n),  	// 1
+          "r" (x1),     // 2
+          "r" (svec),   // 3
+          "r" (pre),    // 4
+	  "r" (o16),	// 5
+	  "r" (o32),	// 6
+	  "r" (o48),    // 7
+          "r" (o64),    // 8
+          "r" (o80),    // 9
+          "r" (o96),    // 10
+          "r" (o112)   // 11
+	: "cr0", "%0", "%2",  "memory"
+	);
+
+} 
+
+

kernel/power/zaxpy.c

@@ -0,0 +1,140 @@
+/***************************************************************************
+Copyright (c) 2013-2016, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+/**************************************************************************************
+* 2016/03/23 Werner Saar (wernsaar@googlemail.com)
+* 	 BLASTEST 		: OK
+* 	 CTEST			: OK
+* 	 TEST			: OK
+*	 LAPACK-TEST		: OK
+**************************************************************************************/
+
+#include "common.h"
+
+
+#if defined(POWER8)
+#include "zaxpy_microk_power8.c"
+#endif
+
+
+#ifndef HAVE_KERNEL_4
+
+static void zaxpy_kernel_4(BLASLONG n, FLOAT *x, FLOAT *y, FLOAT *alpha)
+{
+	BLASLONG register i  = 0;
+	BLASLONG register ix = 0;
+	FLOAT da_r = alpha[0];
+	FLOAT da_i = alpha[1];
+	
+
+	while(i < n)
+        {
+#if !defined(CONJ)
+              y[ix]   += ( da_r * x[ix]   - da_i * x[ix+1] ) ;
+              y[ix+1] += ( da_r * x[ix+1] + da_i * x[ix]   ) ;
+              y[ix+2] += ( da_r * x[ix+2] - da_i * x[ix+3] ) ;
+              y[ix+3] += ( da_r * x[ix+3] + da_i * x[ix+2] ) ;
+#else
+              y[ix]   += ( da_r * x[ix]   + da_i * x[ix+1] ) ;
+              y[ix+1] -= ( da_r * x[ix+1] - da_i * x[ix]   ) ;
+              y[ix+2] += ( da_r * x[ix+2] + da_i * x[ix+3] ) ;
+              y[ix+3] -= ( da_r * x[ix+3] - da_i * x[ix+2] ) ;
+#endif
+
+              ix+=4 ;
+              i+=2 ;
+
+       }
+
+}
+
+#endif
+
+int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da_r, FLOAT da_i, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y, FLOAT *dummy, BLASLONG dummy2)
+{
+	BLASLONG i=0;
+	BLASLONG ix=0,iy=0;
+	FLOAT da[4];
+
+	if ( n <= 0 )  return(0);
+
+	if ( (inc_x == 1) && (inc_y == 1) )
+	{
+
+		BLASLONG n1 = n & -16;
+
+		if ( n1 )
+		{
+			da[0] = da_r;
+			da[1] = da_r;
+			da[2] = da_i;
+			da[3] = da_i;
+			zaxpy_kernel_4(n1, x, y , da );
+			ix = 2 * n1;
+		}
+		i = n1;
+		while(i < n)
+		{
+#if !defined(CONJ)
+                	y[ix]   += ( da_r * x[ix]   - da_i * x[ix+1] ) ;
+                	y[ix+1] += ( da_r * x[ix+1] + da_i * x[ix]   ) ;
+#else
+                	y[ix]   += ( da_r * x[ix]   + da_i * x[ix+1] ) ;
+                	y[ix+1] -= ( da_r * x[ix+1] - da_i * x[ix]   ) ;
+#endif
+			i++ ;
+			ix += 2;
+
+		}
+		return(0);
+
+
+	}
+
+	inc_x *=2;
+	inc_y *=2;
+
+	while(i < n)
+	{
+
+#if !defined(CONJ)
+                y[iy]   += ( da_r * x[ix]   - da_i * x[ix+1] ) ;
+                y[iy+1] += ( da_r * x[ix+1] + da_i * x[ix]   ) ;
+#else
+                y[iy]   += ( da_r * x[ix]   + da_i * x[ix+1] ) ;
+                y[iy+1] -= ( da_r * x[ix+1] - da_i * x[ix]   ) ;
+#endif
+		ix  += inc_x ;
+		iy  += inc_y ;
+		i++ ;
+
+	}
+	return(0);
+
+}
+
+

kernel/power/zaxpy_microk_power8.c

@@ -0,0 +1,250 @@
+/***************************************************************************
+Copyright (c) 2013-2016, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+/**************************************************************************************
+* 2016/03/23 Werner Saar (wernsaar@googlemail.com)
+* 	 BLASTEST 		: OK
+* 	 CTEST			: OK
+* 	 TEST			: OK
+*	 LAPACK-TEST		: OK
+**************************************************************************************/
+
+
+#define HAVE_KERNEL_4 1
+static void zaxpy_kernel_4( BLASLONG n, FLOAT *x, FLOAT *y , FLOAT *alpha) __attribute__ ((noinline));
+
+static void zaxpy_kernel_4( BLASLONG n, FLOAT *x, FLOAT *y, FLOAT *alpha)
+{
+
+
+	BLASLONG i = n;
+	BLASLONG o16 = 16;
+	BLASLONG o32 = 32;
+	BLASLONG o48 = 48;
+	FLOAT *x1=x;
+	FLOAT *y1=y;
+	FLOAT *y2=y+1;
+	BLASLONG pre = 384;
+
+#if !defined(CONJ)
+        FLOAT mvec[2] = { -1.0, 1.0 };
+#else
+        FLOAT mvec[2] = { 1.0, -1.0 };
+#endif
+
+
+	__asm__  __volatile__
+	(
+
+	"lxsdx		34, 0 , %4			    \n\t"	// alpha_r
+	"lxsdx		35, %5, %4			    \n\t"	// alpha_i
+	"xxspltd	32, 34, 0			    \n\t"
+	"xxspltd	33, 35, 0			    \n\t"
+
+	"lxvd2x		36, 0,	%9			    \n\t"	// mvec
+
+#if !defined(CONJ)
+        "xvmuldp         33, 33  , 36               	    \n\t"	// alpha_i * mvec
+#else
+        "xvmuldp         32, 32  , 36               	    \n\t"	// alpha_r * mvec
+#endif
+
+	"addi		%8, %8, -8			    \n\t"
+
+	"dcbt		%2, %10				    \n\t"
+	"dcbt		%3, %10				    \n\t"
+
+
+	"lxvd2x		40, 0, %2			    \n\t"       // x0
+	"lxvd2x		41, %5, %2			    \n\t"	// x1
+	"lxvd2x		42, %6, %2			    \n\t"	// x2
+	"lxvd2x		43, %7, %2			    \n\t"	// x3
+
+	"lxvd2x		48, 0, %3			    \n\t"	// y0
+	"lxvd2x		49, %5, %3			    \n\t"	// y1
+	"lxvd2x		50, %6, %3			    \n\t"	// y2
+	"lxvd2x		51, %7, %3			    \n\t"	// y3
+
+	"xxswapd	56, 40				    \n\t"	// exchange real and imag part
+	"xxswapd	57, 41				    \n\t"	// exchange real and imag part
+	"xxswapd	58, 42				    \n\t"	// exchange real and imag part
+	"xxswapd	59, 43				    \n\t"	// exchange real and imag part
+
+	"addi		%2, %2, 64			    \n\t"
+	"addi		%3, %3, 64			    \n\t"
+
+	"lxvd2x		44, 0, %2			    \n\t"	// x4
+	"lxvd2x		45, %5, %2			    \n\t"	// x5
+	"lxvd2x		46, %6, %2			    \n\t"	// x6
+	"lxvd2x		47, %7, %2			    \n\t"	// x7
+
+	"lxvd2x		52, 0, %3			    \n\t"	// y4
+	"lxvd2x		53, %5, %3			    \n\t"	// y5
+	"lxvd2x		54, %6, %3			    \n\t"	// y6
+	"lxvd2x		55, %7, %3			    \n\t"	// y7
+
+	"xxswapd	60, 44				    \n\t"	// exchange real and imag part
+	"xxswapd	61, 45				    \n\t"	// exchange real and imag part
+	"xxswapd	62, 46				    \n\t"	// exchange real and imag part
+	"xxswapd	63, 47				    \n\t"	// exchange real and imag part
+
+	"addi		%2, %2, 64			    \n\t"
+	"addi		%3, %3, 64			    \n\t"
+
+	"addic.		%0 , %0	, -8  	 	             \n\t"
+	"ble		2f		             	     \n\t"
+
+	".align 5				            \n\t"
+	"1:				                    \n\t"
+
+	"dcbt		%2, %10				    \n\t"
+	"dcbt		%3, %10				    \n\t"
+
+	"xvmaddadp	48, 40, 32		    	    \n\t"	// alpha_r * x0_r , alpha_r * x0_i
+	"xvmaddadp	49, 41, 32		    	    \n\t"
+	"lxvd2x		40, 0, %2			    \n\t"       // x0
+	"lxvd2x		41, %5, %2			    \n\t"	// x1
+	"xvmaddadp	50, 42, 32		    	    \n\t"
+	"xvmaddadp	51, 43, 32		    	    \n\t"
+	"lxvd2x		42, %6, %2			    \n\t"	// x2
+	"lxvd2x		43, %7, %2			    \n\t"	// x3
+
+	"xvmaddadp	52, 44, 32		    	    \n\t"
+	"addi		%2, %2, 64			    \n\t"
+	"xvmaddadp	53, 45, 32		    	    \n\t"
+	"lxvd2x		44, 0, %2			    \n\t"	// x4
+	"lxvd2x		45, %5, %2			    \n\t"	// x5
+	"xvmaddadp	54, 46, 32		    	    \n\t"
+	"xvmaddadp	55, 47, 32		    	    \n\t"
+	"lxvd2x		46, %6, %2			    \n\t"	// x6
+	"lxvd2x		47, %7, %2			    \n\t"	// x7
+
+	"xvmaddadp	48, 56, 33		    	    \n\t"	// alpha_i * x0_i , alpha_i * x0_r
+	"addi		%2, %2, 64			    \n\t"
+	"xvmaddadp	49, 57, 33		    	    \n\t"
+	"xvmaddadp	50, 58, 33		    	    \n\t"
+	"xvmaddadp	51, 59, 33		    	    \n\t"
+
+	"xvmaddadp	52, 60, 33		    	    \n\t"
+	"xvmaddadp	53, 61, 33		    	    \n\t"
+	"xvmaddadp	54, 62, 33		    	    \n\t"
+	"xvmaddadp	55, 63, 33		    	    \n\t"
+
+	"stxvd2x	48,  0, %8			    \n\t"
+	"stxvd2x	49, %5, %8			    \n\t"
+	"stxvd2x	50, %6, %8			    \n\t"
+	"stxvd2x	51, %7, %8			    \n\t"
+
+	"addi		%8, %8, 64			    \n\t"
+
+	"stxvd2x	52,  0, %8			    \n\t"
+	"stxvd2x	53, %5, %8			    \n\t"
+	"stxvd2x	54, %6, %8			    \n\t"
+	"stxvd2x	55, %7, %8			    \n\t"
+
+	"addi		%8, %8, 64			    \n\t"
+
+	"xxswapd	56, 40				    \n\t"	// exchange real and imag part
+	"xxswapd	57, 41				    \n\t"	// exchange real and imag part
+	"lxvd2x		48, 0, %3			    \n\t"	// y0
+	"lxvd2x		49, %5, %3			    \n\t"	// y1
+	"xxswapd	58, 42				    \n\t"	// exchange real and imag part
+	"xxswapd	59, 43				    \n\t"	// exchange real and imag part
+	"lxvd2x		50, %6, %3			    \n\t"	// y2
+	"lxvd2x		51, %7, %3			    \n\t"	// y3
+
+	"xxswapd	60, 44				    \n\t"	// exchange real and imag part
+	"addi		%3, %3, 64			    \n\t"
+	"xxswapd	61, 45				    \n\t"	// exchange real and imag part
+	"lxvd2x		52, 0, %3			    \n\t"	// y4
+	"lxvd2x		53, %5, %3			    \n\t"	// y5
+	"xxswapd	62, 46				    \n\t"	// exchange real and imag part
+	"xxswapd	63, 47				    \n\t"	// exchange real and imag part
+	"lxvd2x		54, %6, %3			    \n\t"	// y6
+	"lxvd2x		55, %7, %3			    \n\t"	// y7
+
+	"addi		%3, %3, 64			    \n\t"
+
+	"addic.		%0 , %0	, -8  	 	             \n\t"
+	"bgt		1b		             	     \n\t"
+
+	"2:						     \n\t"
+
+	"xvmaddadp	48, 40, 32		    	    \n\t"	// alpha_r * x0_r , alpha_r * x0_i
+	"xvmaddadp	49, 41, 32		    	    \n\t"
+	"xvmaddadp	50, 42, 32		    	    \n\t"
+	"xvmaddadp	51, 43, 32		    	    \n\t"
+
+	"xvmaddadp	52, 44, 32		    	    \n\t"
+	"xvmaddadp	53, 45, 32		    	    \n\t"
+	"xvmaddadp	54, 46, 32		    	    \n\t"
+	"xvmaddadp	55, 47, 32		    	    \n\t"
+
+	"xvmaddadp	48, 56, 33		    	    \n\t"	// alpha_i * x0_i , alpha_i * x0_r
+	"xvmaddadp	49, 57, 33		    	    \n\t"
+	"xvmaddadp	50, 58, 33		    	    \n\t"
+	"xvmaddadp	51, 59, 33		    	    \n\t"
+
+	"xvmaddadp	52, 60, 33		    	    \n\t"
+	"xvmaddadp	53, 61, 33		    	    \n\t"
+	"xvmaddadp	54, 62, 33		    	    \n\t"
+	"xvmaddadp	55, 63, 33		    	    \n\t"
+
+
+	"stxvd2x	48,  0, %8			    \n\t"
+	"stxvd2x	49, %5, %8			    \n\t"
+	"stxvd2x	50, %6, %8			    \n\t"
+	"stxvd2x	51, %7, %8			    \n\t"
+
+	"addi		%8, %8, 64			    \n\t"
+
+	"stxvd2x	52,  0, %8			    \n\t"
+	"stxvd2x	53, %5, %8			    \n\t"
+	"stxvd2x	54, %6, %8			    \n\t"
+	"stxvd2x	55, %7, %8			    \n\t"
+
+	"addi		%8, %8, 64			    \n\t"
+
+	:
+        : 
+          "r" (i),	// 0	
+	  "r" (n),  	// 1
+          "r" (x1),     // 2
+          "r" (y1),     // 3
+          "r" (alpha),  // 4
+	  "r" (o16),	// 5
+	  "r" (o32),	// 6
+	  "r" (o48),    // 7
+	  "r" (y2),     // 8
+	  "r" (mvec),   // 9
+	  "r" (pre)	// 10
+	: "cr0", "%0", "%2" , "%3", "%8", "memory"
+	);
+
+} 
+
+

kernel/power/zcopy.c

@@ -0,0 +1,140 @@
+/***************************************************************************
+Copyright (c) 2013-2016, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+/**************************************************************************************
+* 2016/03/25 Werner Saar (wernsaar@googlemail.com)
+* 	 BLASTEST 		: OK
+* 	 CTEST			: OK
+* 	 TEST			: OK
+*	 LAPACK-TEST		: OK
+**************************************************************************************/
+
+#include "common.h"
+
+#if defined(POWER8)
+#include "zcopy_microk_power8.c"
+#endif
+
+#ifndef HAVE_KERNEL_16
+
+static void zcopy_kernel_16(BLASLONG n, FLOAT *x, FLOAT *y)
+{
+
+	BLASLONG i=0;
+	FLOAT f0, f1, f2, f3, f4, f5, f6, f7;
+	FLOAT *x1=x;
+	FLOAT *y1=y;
+
+	while ( i<n )
+	{
+
+		f0 = x1[0];
+		f1 = x1[1];
+		f2 = x1[2];
+		f3 = x1[3];
+		f4 = x1[4];
+		f5 = x1[5];
+		f6 = x1[6];
+		f7 = x1[7];
+
+		y1[0] = f0;
+		y1[1] = f1;
+		y1[2] = f2;
+		y1[3] = f3;
+		y1[4] = f4;
+		y1[5] = f5;
+		y1[6] = f6;
+		y1[7] = f7;
+
+		x1 += 8;
+		y1 += 8;
+
+		i+=4;
+	}
+	return;
+
+}
+
+
+#endif
+
+
+
+int CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y)
+{
+	BLASLONG i=0;
+	BLASLONG ix=0,iy=0;
+
+	if ( n <= 0     )  return(0);
+
+	if ( (inc_x == 1) && (inc_y == 1 ))
+	{
+
+		BLASLONG n1 = n & -16;
+		if ( n1 > 0 )
+		{
+			zcopy_kernel_16(n1, x, y);
+			i=n1;
+			ix=n1*2;
+			iy=n1*2;
+		}
+
+		while(i < n)
+		{
+			y[iy] = x[iy] ;
+			y[iy+1] = x[ix+1] ;
+			ix+=2;
+			iy+=2;
+			i++ ;
+
+		}
+
+
+	}
+	else
+	{
+
+		BLASLONG inc_x2 = 2 * inc_x;
+		BLASLONG inc_y2 = 2 * inc_y;
+
+		while(i < n)
+		{
+			y[iy] = x[ix] ;
+			y[iy+1] = x[ix+1] ;
+			ix += inc_x2 ;
+			iy += inc_y2 ;
+			i++ ;
+
+		}
+
+	}
+	return(0);
+	
+
+}
+
+

kernel/power/zcopy_microk_power8.c

@@ -0,0 +1,174 @@
+/***************************************************************************
+Copyright (c) 2013-2016, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+/**************************************************************************************
+* 2016/03/25 Werner Saar (wernsaar@googlemail.com)
+* 	 BLASTEST 		: OK
+* 	 CTEST			: OK
+* 	 TEST			: OK
+*	 LAPACK-TEST		: OK
+**************************************************************************************/
+
+#define HAVE_KERNEL_16 1
+
+static void zcopy_kernel_16( BLASLONG n, FLOAT *x, FLOAT *y) __attribute__ ((noinline));
+
+static void zcopy_kernel_16( BLASLONG n, FLOAT *x, FLOAT *y)
+{
+
+
+	BLASLONG i = n;
+	BLASLONG o16 = 16;
+	BLASLONG o32 = 32;
+	BLASLONG o48 = 48;
+	BLASLONG o64 = 64;
+	BLASLONG o80 = 80;
+	BLASLONG o96 = 96;
+	BLASLONG o112 = 112;
+	FLOAT *x1=x;
+	FLOAT *y1=y;
+	BLASLONG pre = 384;
+	BLASLONG alpha=0;
+
+	__asm__  __volatile__
+	(
+
+	"lxvd2x		40, 0, %2			    \n\t"
+	"lxvd2x		41, %5, %2			    \n\t"
+	"lxvd2x		42, %6, %2			    \n\t"
+	"lxvd2x		43, %7, %2			    \n\t"
+	"lxvd2x		44, %8, %2			    \n\t"
+	"lxvd2x		45, %9, %2			    \n\t"
+	"lxvd2x		46, %10, %2			    \n\t"
+	"lxvd2x		47, %11, %2			    \n\t"
+
+	"addi		%2, %2, 128			    \n\t"
+
+	"lxvd2x		50, 0, %2			    \n\t"
+	"lxvd2x		51, %5, %2			    \n\t"
+	"lxvd2x		52, %6, %2			    \n\t"
+	"lxvd2x		53, %7, %2			    \n\t"
+	"lxvd2x		54, %8, %2			    \n\t"
+	"lxvd2x		55, %9, %2			    \n\t"
+	"lxvd2x		56, %10, %2			    \n\t"
+	"lxvd2x		57, %11, %2			    \n\t"
+
+	"addi		%2, %2, 128			    \n\t"
+
+	"addic.		%0 , %0	, -16  	 	             \n\t"
+	"ble		2f		             	     \n\t"
+
+	".align 5				            \n\t"
+	"1:				                    \n\t"
+
+	"stxvd2x		40, 0, %1			    \n\t"
+	"stxvd2x		41, %5, %1			    \n\t"
+	"lxvd2x		40, 0, %2			    \n\t"
+	"lxvd2x		41, %5, %2			    \n\t"
+	"stxvd2x		42, %6, %1			    \n\t"
+	"stxvd2x		43, %7, %1			    \n\t"
+	"lxvd2x		42, %6, %2			    \n\t"
+	"lxvd2x		43, %7, %2			    \n\t"
+	"stxvd2x		44, %8, %1			    \n\t"
+	"stxvd2x		45, %9, %1			    \n\t"
+	"lxvd2x		44, %8, %2			    \n\t"
+	"lxvd2x		45, %9, %2			    \n\t"
+	"stxvd2x		46, %10, %1			    \n\t"
+	"stxvd2x		47, %11, %1			    \n\t"
+	"lxvd2x		46, %10, %2			    \n\t"
+	"lxvd2x		47, %11, %2			    \n\t"
+
+
+	"addi		%1, %1, 128			    \n\t"
+	"addi		%2, %2, 128			    \n\t"
+
+	"stxvd2x		50, 0, %1			    \n\t"
+	"stxvd2x		51, %5, %1			    \n\t"
+	"lxvd2x		50, 0, %2			    \n\t"
+	"lxvd2x		51, %5, %2			    \n\t"
+	"stxvd2x		52, %6, %1			    \n\t"
+	"stxvd2x		53, %7, %1			    \n\t"
+	"lxvd2x		52, %6, %2			    \n\t"
+	"lxvd2x		53, %7, %2			    \n\t"
+	"stxvd2x		54, %8, %1			    \n\t"
+	"stxvd2x		55, %9, %1			    \n\t"
+	"lxvd2x		54, %8, %2			    \n\t"
+	"lxvd2x		55, %9, %2			    \n\t"
+	"stxvd2x		56, %10, %1			    \n\t"
+	"stxvd2x		57, %11, %1			    \n\t"
+	"lxvd2x		56, %10, %2			    \n\t"
+	"lxvd2x		57, %11, %2			    \n\t"
+
+	"addi		%1, %1, 128			    \n\t"
+	"addi		%2, %2, 128			    \n\t"
+
+	"addic.		%0 , %0	, -16  	 	             \n\t"
+	"bgt		1b		             	     \n\t"
+
+	"2:						     \n\t"
+
+	"stxvd2x		40, 0, %1			    \n\t"
+	"stxvd2x		41, %5, %1			    \n\t"
+	"stxvd2x		42, %6, %1			    \n\t"
+	"stxvd2x		43, %7, %1			    \n\t"
+	"stxvd2x		44, %8, %1			    \n\t"
+	"stxvd2x		45, %9, %1			    \n\t"
+	"stxvd2x		46, %10, %1			    \n\t"
+	"stxvd2x		47, %11, %1			    \n\t"
+
+	"addi		%1, %1, 128			    \n\t"
+
+	"stxvd2x		50, 0, %1			    \n\t"
+	"stxvd2x		51, %5, %1			    \n\t"
+	"stxvd2x		52, %6, %1			    \n\t"
+	"stxvd2x		53, %7, %1			    \n\t"
+	"stxvd2x		54, %8, %1			    \n\t"
+	"stxvd2x		55, %9, %1			    \n\t"
+	"stxvd2x		56, %10, %1			    \n\t"
+	"stxvd2x		57, %11, %1			    \n\t"
+
+
+	:
+        : 
+          "r" (i),	// 0	
+	  "r" (y1),  	// 1
+          "r" (x1),     // 2
+          "r" (alpha),  // 3
+          "r" (pre),    // 4
+	  "r" (o16),	// 5
+	  "r" (o32),	// 6
+	  "r" (o48),    // 7
+          "r" (o64),    // 8
+          "r" (o80),    // 9
+          "r" (o96),    // 10
+          "r" (o112)    // 11
+	: "cr0", "%0", "%2" , "%1", "memory"
+	);
+
+} 
+
+

kernel/power/zdot.c

@@ -0,0 +1,167 @@
+/***************************************************************************
+Copyright (c) 2013-2016, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+/**************************************************************************************
+* 2016/03/21 Werner Saar (wernsaar@googlemail.com)
+* 	 BLASTEST 		: OK
+* 	 CTEST			: OK
+* 	 TEST			: OK
+*	 LAPACK-TEST		: OK
+**************************************************************************************/
+
+#include "common.h"
+#include <complex.h>
+
+
+#if defined(POWER8) 
+#include "zdot_microk_power8.c"
+#endif
+
+
+#ifndef HAVE_KERNEL_8
+
+static void zdot_kernel_8(BLASLONG n, FLOAT *x, FLOAT *y, FLOAT *d) __attribute__ ((noinline));
+
+static void zdot_kernel_8(BLASLONG n, FLOAT *x, FLOAT *y, FLOAT *d)
+{
+	BLASLONG register i = 0;
+	FLOAT dot[4] = { 0.0, 0.0, 0.0, 0.0 };
+	BLASLONG j=0;
+
+	while( i < n )
+        {
+
+            dot[0] += x[j]   * y[j]   ;
+            dot[1] += x[j+1] * y[j+1] ;
+            dot[2] += x[j]   * y[j+1] ;
+            dot[3] += x[j+1] * y[j]   ;
+
+            dot[0] += x[j+2] * y[j+2] ;
+            dot[1] += x[j+3] * y[j+3] ;
+            dot[2] += x[j+2] * y[j+3] ;
+            dot[3] += x[j+3] * y[j+2] ;
+
+            dot[0] += x[j+4] * y[j+4] ;
+            dot[1] += x[j+5] * y[j+5] ;
+            dot[2] += x[j+4] * y[j+5] ;
+            dot[3] += x[j+5] * y[j+4] ;
+
+            dot[0] += x[j+6] * y[j+6] ;
+            dot[1] += x[j+7] * y[j+7] ;
+            dot[2] += x[j+6] * y[j+7] ;
+            dot[3] += x[j+7] * y[j+6] ;
+
+	    j+=8;
+            i+=4;
+
+        }
+	d[0] = dot[0];
+	d[1] = dot[1];
+	d[2] = dot[2];
+	d[3] = dot[3];
+
+}
+
+#endif
+
+FLOAT _Complex CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y)
+{
+	BLASLONG i;
+	BLASLONG ix,iy;
+	FLOAT _Complex result;
+	FLOAT  dot[4] = { 0.0, 0.0, 0.0 , 0.0 } ; 
+
+	if ( n <= 0 ) 
+	{
+	        __real__ result = 0.0 ;
+        	__imag__ result = 0.0 ;
+		return(result);
+
+	}
+
+	if ( (inc_x == 1) && (inc_y == 1) )
+	{
+
+		BLASLONG n1 = n & -8;
+
+		if ( n1 )
+			zdot_kernel_8(n1, x, y , dot );
+
+		i = n1;
+		BLASLONG j = i * 2;
+
+		while( i < n )
+		{
+
+			dot[0] += x[j]   * y[j]   ;
+			dot[1] += x[j+1] * y[j+1] ;
+			dot[2] += x[j]   * y[j+1] ;
+			dot[3] += x[j+1] * y[j]   ;
+
+			j+=2;
+			i++ ;
+
+		}
+
+
+	}
+	else
+	{
+		i=0;
+		ix=0;
+		iy=0;
+		inc_x <<= 1;
+		inc_y <<= 1;
+		while(i < n)
+		{
+
+			dot[0] += x[ix]   * y[iy]   ;
+			dot[1] += x[ix+1] * y[iy+1] ;
+			dot[2] += x[ix]   * y[iy+1] ;
+			dot[3] += x[ix+1] * y[iy]   ;
+
+			ix  += inc_x ;
+			iy  += inc_y ;
+			i++ ;
+
+		}
+	}
+
+#if !defined(CONJ)
+	__real__ result = dot[0] - dot[1];
+	__imag__ result = dot[2] + dot[3];
+#else
+	__real__ result = dot[0] + dot[1];
+	__imag__ result = dot[2] - dot[3];
+
+#endif
+
+	return(result);
+
+}
+
+

kernel/power/zdot_microk_power8.c

@@ -0,0 +1,219 @@
+/***************************************************************************
+Copyright (c) 2013-2016, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+/**************************************************************************************
+* 2016/03/21 Werner Saar (wernsaar@googlemail.com)
+* 	 BLASTEST 		: OK
+* 	 CTEST			: OK
+* 	 TEST			: OK
+*	 LAPACK-TEST		: OK
+**************************************************************************************/
+
+#define HAVE_KERNEL_8 1
+static void zdot_kernel_8( BLASLONG n, FLOAT *x, FLOAT *y , FLOAT *dot) __attribute__ ((noinline));
+
+static void zdot_kernel_8( BLASLONG n, FLOAT *x, FLOAT *y, FLOAT *dot)
+{
+
+
+	BLASLONG i = n;
+	BLASLONG o16 = 16;
+	BLASLONG o32 = 32;
+	BLASLONG o48 = 48;
+	FLOAT *x1=x;
+	FLOAT *y1=y;
+	BLASLONG pre = 384;
+
+	__asm__  __volatile__
+	(
+	"xxlxor		32,32,32			    \n\t"
+	"xxlxor		33,33,33			    \n\t"
+	"xxlxor		34,34,34			    \n\t"
+	"xxlxor		35,35,35			    \n\t"
+	"xxlxor		36,36,36			    \n\t"
+	"xxlxor		37,37,37			    \n\t"
+	"xxlxor		38,38,38			    \n\t"
+	"xxlxor		39,39,39			    \n\t"
+
+	"dcbt		%2, %8				    \n\t"
+	"dcbt		%3, %8				    \n\t"
+
+	"lxvd2x		40, 0, %2			    \n\t"	// x0_r, x0_i
+	"lxvd2x		48, 0, %3			    \n\t"	// y0_r, y0_i
+	"lxvd2x		41, %5, %2			    \n\t"	// x1_r, x1_i
+	"lxvd2x		49, %5, %3			    \n\t"	// y1_r, y1_i
+	"lxvd2x		42, %6, %2			    \n\t"	// x2_r, x2_i
+	"lxvd2x		50, %6, %3			    \n\t"	// y2_r, y2_i
+	"lxvd2x		43, %7, %2			    \n\t"	// x3_r, x3_i
+	"lxvd2x		51, %7, %3			    \n\t"	// y3_r, y3_i
+
+	"xxswapd	52,48				    \n\t"	// y0_i, y0_r
+	"xxswapd	53,49				    \n\t"	// y1_i, y1_r
+	"xxswapd	54,50				    \n\t"	// y2_i, y2_r
+	"xxswapd	55,51				    \n\t"	// y3_i, y3_r
+
+	"addi		%2, %2, 64			    \n\t"
+	"addi		%3, %3, 64			    \n\t"
+
+
+	"lxvd2x		44, 0, %2			    \n\t"	// x0_r, x0_i
+	"lxvd2x		56, 0, %3			    \n\t"	// y0_r, y0_i
+	"lxvd2x		45, %5, %2			    \n\t"	// x1_r, x1_i
+	"lxvd2x		57, %5, %3			    \n\t"	// y1_r, y1_i
+	"lxvd2x		46, %6, %2			    \n\t"	// x2_r, x2_i
+	"lxvd2x		58, %6, %3			    \n\t"	// y2_r, y2_i
+	"lxvd2x		47, %7, %2			    \n\t"	// x3_r, x3_i
+	"lxvd2x		59, %7, %3			    \n\t"	// y3_r, y3_i
+
+	"xxswapd	60,56				    \n\t"	// y0_i, y0_r
+	"xxswapd	61,57				    \n\t"	// y1_i, y1_r
+	"xxswapd	62,58				    \n\t"	// y2_i, y2_r
+	"xxswapd	63,59				    \n\t"	// y3_i, y3_r
+
+	"addi		%2, %2, 64			    \n\t"
+	"addi		%3, %3, 64			    \n\t"
+
+	"addic.		%0 , %0	, -8  	 	             \n\t"
+	"ble		2f		             	     \n\t"
+
+	".align 5				            \n\t"
+	"1:				                    \n\t"
+
+	"dcbt		%2, %8				    \n\t"
+	"dcbt		%3, %8				    \n\t"
+
+	"xvmaddadp	32, 40, 48		    	    \n\t"	// x0_r * y0_r , x0_i * y0_i
+	"lxvd2x		48, 0, %3			    \n\t"	// y0_r, y0_i
+	"xvmaddadp	34, 41, 49		    	    \n\t"	// x1_r * y1_r , x1_i * y1_i
+	"lxvd2x		49, %5, %3			    \n\t"	// y1_r, y1_i
+
+	"xvmaddadp	36, 42, 50		    	    \n\t"	// x2_r * y2_r , x2_i * y2_i
+	"lxvd2x		50, %6, %3			    \n\t"	// y2_r, y2_i
+	"xvmaddadp	38, 43, 51		    	    \n\t"	// x3_r * y3_r , x3_i * y3_i
+	"lxvd2x		51, %7, %3			    \n\t"	// y3_r, y3_i
+
+	"xvmaddadp	33, 40, 52		    	    \n\t"	// x0_r * y0_i , x0_i * y0_r
+	"lxvd2x		40, 0, %2			    \n\t"	// x0_r, x0_i
+	"xvmaddadp	35, 41, 53		    	    \n\t"	// x1_r * y1_i , x1_i * y1_r
+	"lxvd2x		41, %5, %2			    \n\t"	// x1_r, x1_i
+
+	"xvmaddadp	37, 42, 54		    	    \n\t"	// x2_r * y2_i , x2_i * y2_r
+	"lxvd2x		42, %6, %2			    \n\t"	// x2_r, x2_i
+	"xvmaddadp	39, 43, 55		    	    \n\t"	// x3_r * y3_i , x3_i * y3_r
+	"lxvd2x		43, %7, %2			    \n\t"	// x3_r, x3_i
+
+	"xxswapd	52,48				    \n\t"	// y0_i, y0_r
+	"xxswapd	53,49				    \n\t"	// y1_i, y1_r
+
+	"addi		%2, %2, 64			    \n\t"
+	"addi		%3, %3, 64			    \n\t"
+
+	"xxswapd	54,50				    \n\t"	// y2_i, y2_r
+	"xxswapd	55,51				    \n\t"	// y3_i, y3_r
+
+	"xvmaddadp	32, 44, 56		    	    \n\t"	// x0_r * y0_r , x0_i * y0_i
+	"lxvd2x		56, 0, %3			    \n\t"	// y0_r, y0_i
+	"xvmaddadp	34, 45, 57		    	    \n\t"	// x1_r * y1_r , x1_i * y1_i
+	"lxvd2x		57, %5, %3			    \n\t"	// y1_r, y1_i
+	"xvmaddadp	36, 46, 58		    	    \n\t"	// x2_r * y2_r , x2_i * y2_i
+	"lxvd2x		58, %6, %3			    \n\t"	// y2_r, y2_i
+	"xvmaddadp	38, 47, 59		    	    \n\t"	// x3_r * y3_r , x3_i * y3_i
+	"lxvd2x		59, %7, %3			    \n\t"	// y3_r, y3_i
+
+	"xvmaddadp	33, 44, 60		    	    \n\t"	// x0_r * y0_i , x0_i * y0_r
+	"lxvd2x		44, 0, %2			    \n\t"	// x0_r, x0_i
+	"xvmaddadp	35, 45, 61		    	    \n\t"	// x1_r * y1_i , x1_i * y1_r
+	"lxvd2x		45, %5, %2			    \n\t"	// x1_r, x1_i
+	"xvmaddadp	37, 46, 62		    	    \n\t"	// x2_r * y2_i , x2_i * y2_r
+	"lxvd2x		46, %6, %2			    \n\t"	// x2_r, x2_i
+	"xvmaddadp	39, 47, 63		    	    \n\t"	// x3_r * y3_i , x3_i * y3_r
+	"lxvd2x		47, %7, %2			    \n\t"	// x3_r, x3_i
+
+	"xxswapd	60,56				    \n\t"	// y0_i, y0_r
+	"xxswapd	61,57				    \n\t"	// y1_i, y1_r
+
+	"addi		%2, %2, 64			    \n\t"
+	"addi		%3, %3, 64			    \n\t"
+
+	"xxswapd	62,58				    \n\t"	// y2_i, y2_r
+	"xxswapd	63,59				    \n\t"	// y3_i, y3_r
+
+	"addic.		%0 , %0	, -8  	 	             \n\t"
+	"bgt		1b		             	     \n\t"
+
+	"2:						     \n\t"
+
+	"xvmaddadp	32, 40, 48		    	    \n\t"	// x0_r * y0_r , x0_i * y0_i
+	"xvmaddadp	34, 41, 49		    	    \n\t"	// x1_r * y1_r , x1_i * y1_i
+	"xvmaddadp	36, 42, 50		    	    \n\t"	// x2_r * y2_r , x2_i * y2_i
+	"xvmaddadp	38, 43, 51		    	    \n\t"	// x3_r * y3_r , x3_i * y3_i
+
+	"xvmaddadp	33, 40, 52		    	    \n\t"	// x0_r * y0_i , x0_i * y0_r
+	"xvmaddadp	35, 41, 53		    	    \n\t"	// x1_r * y1_i , x1_i * y1_r
+	"xvmaddadp	37, 42, 54		    	    \n\t"	// x2_r * y2_i , x2_i * y2_r
+	"xvmaddadp	39, 43, 55		    	    \n\t"	// x3_r * y3_i , x3_i * y3_r
+
+	"xvmaddadp	32, 44, 56		    	    \n\t"	// x0_r * y0_r , x0_i * y0_i
+	"xvmaddadp	34, 45, 57		    	    \n\t"	// x1_r * y1_r , x1_i * y1_i
+	"xvmaddadp	36, 46, 58		    	    \n\t"	// x2_r * y2_r , x2_i * y2_i
+	"xvmaddadp	38, 47, 59		    	    \n\t"	// x3_r * y3_r , x3_i * y3_i
+
+	"xvmaddadp	33, 44, 60		    	    \n\t"	// x0_r * y0_i , x0_i * y0_r
+	"xvmaddadp	35, 45, 61		    	    \n\t"	// x1_r * y1_i , x1_i * y1_r
+	"xvmaddadp	37, 46, 62		    	    \n\t"	// x2_r * y2_i , x2_i * y2_r
+	"xvmaddadp	39, 47, 63		    	    \n\t"	// x3_r * y3_i , x3_i * y3_r
+
+
+	"xvadddp	32, 32, 34		     \n\t"
+	"xvadddp	36, 36, 38		     \n\t"
+
+	"xvadddp	33, 33, 35		     \n\t"
+	"xvadddp	37, 37, 39		     \n\t"
+
+	"xvadddp	32, 32, 36		     \n\t"
+	"xvadddp	33, 33, 37		     \n\t"
+
+	"stxvd2x	32, 0, %4		     \n\t"
+	"stxvd2x	33, %5, %4		     \n\t"
+
+	:
+        : 
+          "r" (i),	// 0	
+	  "r" (n),  	// 1
+          "r" (x1),     // 2
+          "r" (y1),     // 3
+          "r" (dot),    // 4
+	  "r" (o16),	// 5
+	  "r" (o32),	// 6
+	  "r" (o48),    // 7
+	  "r" (pre)	// 8
+	: "cr0", "%0", "%2" , "%3", "memory"
+	);
+
+} 
+
+

kernel/power/zgemm_kernel_8x2_power8.S

@@ -1,38 +1,3 @@
-/***************************************************************************
-Copyright (c) 2013-2016, The OpenBLAS Project
-All rights reserved.
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in
-the documentation and/or other materials provided with the
-distribution.
-3. Neither the name of the OpenBLAS project nor the names of
-its contributors may be used to endorse or promote products
-derived from this software without specific prior written permission.
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
-USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*****************************************************************************/
-
-/**************************************************************************************
-* 2016/03/05 Werner Saar (wernsaar@googlemail.com)
-* 	 BLASTEST 		: OK
-* 	 CTEST			: OK
-* 	 TEST			: OK
-*	 LAPACK-TEST		: OK
-**************************************************************************************/
-
 /*********************************************************************/
 /* Copyright 2009, 2010 The University of Texas at Austin.           */
 /* All rights reserved.                                              */
@@ -82,7 +47,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #endif
 
 #ifdef __64BIT__
-#define STACKSIZE 320
+#define STACKSIZE 32000
 #define ALPHA_R_SP 296(SP)
 #define ALPHA_I_SP 304(SP)
 #define FZERO	312(SP)
@@ -133,11 +98,16 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define alpha_r vs30
 #define alpha_i vs31
 
+
+#define FRAMEPOINTER r12
+
+#define BBUFFER r14
+
 #define L	r15
 #define ALPHA	r16
 #define o24	r17
 #define T2	r19
-#define KK	r20
+#define BBO	r20
 #define	o8	r21
 #define	I	r22
 #define J	r23
@@ -156,8 +126,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	PROLOGUE
 	PROFCODE
 
-	addi	SP, SP, -STACKSIZE
-	li	r0, 0
+	mr      FRAMEPOINTER, SP
+        addi    SP, SP, -STACKSIZE
+        addi    SP, SP, -STACKSIZE
+        addi    SP, SP, -STACKSIZE
+        addi    SP, SP, -STACKSIZE
+        li      r0, 0
 
 	stfd	f14,    0(SP)
 	stfd	f15,    8(SP)
@@ -200,6 +174,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	std	r17,  256(SP)
 	std	r16,  264(SP)
 	std	r15,  272(SP)
+	std	r14,  280(SP)
 #else
 	stw	r31,  144(SP)
 	stw	r30,  148(SP)
@@ -226,37 +201,37 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #ifdef linux
 #ifdef __64BIT__
-	ld	LDC, FRAMESLOT(0) + STACKSIZE(SP)
+	ld	LDC, FRAMESLOT(0) + 0(FRAMEPOINTER)
 #endif
 #endif
 
 #if defined(_AIX) || defined(__APPLE__)
 #ifdef __64BIT__
-	ld	LDC, FRAMESLOT(0) + STACKSIZE(SP)
+	ld	LDC, FRAMESLOT(0) + 0(FRAMEPOINTER)
 #else
 #ifdef DOUBLE
-	lwz	B,   FRAMESLOT(0) + STACKSIZE(SP)
-	lwz	C,   FRAMESLOT(1) + STACKSIZE(SP)
-	lwz	LDC, FRAMESLOT(2) + STACKSIZE(SP)
+	lwz	B,   FRAMESLOT(0) + 0(FRAMEPOINTER)
+	lwz	C,   FRAMESLOT(1) + 0(FRAMEPOINTER)
+	lwz	LDC, FRAMESLOT(2) + 0(FRAMEPOINTER)
 #else
-	lwz	LDC, FRAMESLOT(0) + STACKSIZE(SP)
+	lwz	LDC, FRAMESLOT(0) + 0(FRAMEPOINTER)
 #endif
 #endif
 #endif
 
 #ifdef TRMMKERNEL
 #if defined(linux) && defined(__64BIT__)
-	ld	OFFSET,  FRAMESLOT(1) + STACKSIZE(SP)
+	ld	OFFSET,  FRAMESLOT(1) + 0(FRAMEPOINTER)
 #endif
 
 #if defined(_AIX) || defined(__APPLE__)
 #ifdef __64BIT__
-	ld	OFFSET,  FRAMESLOT(1) + STACKSIZE(SP)
+	ld	OFFSET,  FRAMESLOT(1) + 0(FRAMEPOINTER)
 #else
 #ifdef DOUBLE
-	lwz	OFFSET,  FRAMESLOT(3) + STACKSIZE(SP)
+	lwz	OFFSET,  FRAMESLOT(3) + 0(FRAMEPOINTER)
 #else
-	lwz	OFFSET,  FRAMESLOT(1) + STACKSIZE(SP)
+	lwz	OFFSET,  FRAMESLOT(1) + 0(FRAMEPOINTER)
 #endif
 #endif
 #endif
@@ -268,34 +243,38 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "zgemm_macros_8x2_power8.S"
 
 	cmpwi	cr0, M, 0
-	ble	.L999
+	ble	L999
 	cmpwi	cr0, N, 0
-	ble	.L999
+	ble	L999
 	cmpwi	cr0, K, 0
-	ble	.L999
+	ble	L999
 
 	slwi	LDC, LDC, ZBASE_SHIFT
-	li	PRE, 256 
+	li	PRE, 384 
 	li	o8  , 8
 	li	o16 , 16
 	li	o24 , 24
 	li	o32 , 32
 	li	o48 , 48
 
+        addi    BBUFFER, SP, 512+4096
+        li      T1, -4096
+        and     BBUFFER, BBUFFER, T1
+
 #ifdef __64BIT__
 	addi	ALPHA, SP, 296
 #else
 	addi	ALPHA, SP, 224
 #endif
 
-	lxvdsx	alpha_r, 0, ALPHA
-	lxvdsx	alpha_i, o8, ALPHA
+	lxsdx	alpha_r, 0, ALPHA
+	lxsdx	alpha_i, o8, ALPHA
 
-	.align 5
+	.align 4
 
 #include "zgemm_logic_8x2_power8.S"
 
-.L999:
+L999:
 	addi	r3, 0, 0
 
 	lfd	f14,    0(SP)
@@ -339,6 +318,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	ld	r17,  256(SP)
 	ld	r16,  264(SP)
 	ld	r15,  272(SP)
+	ld	r14,  280(SP)
 #else
 	lwz	r31,  144(SP)
 	lwz	r30,  148(SP)
@@ -360,6 +340,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #endif
 
 	addi	SP, SP, STACKSIZE
+	addi	SP, SP, STACKSIZE
+	addi	SP, SP, STACKSIZE
+	addi	SP, SP, STACKSIZE
 
 	blr
 

kernel/power/zgemm_logic_8x2_power8.S

@@ -1,83 +1,111 @@
 	srawi.		J,	N,	1
-	ble		.LZGEMM_L2_END
+	ble		ZGEMM_L2_END
+
+ZGEMM_L2_BEGIN:
+
+	mr		BO,	B
+	mr		BBO,	BBUFFER
+	slwi		T1,	K,	1
+
+ZGEMM_L2_COPYB:
+
+	lxvdsx		vs4,	o0,	BO              // b0_r
+	lxvdsx		vs5,	o8,	BO              // b0_i
+	addi		BO,	BO,	16
+	stxvd2x		vs4,	o0,	BBO
+	stxvd2x		vs5,	o16,	BBO
+	addic.		T1,	T1,	-1
+	addi		BBO,	BBO,	32
+
+	bge		ZGEMM_L2_COPYB
 
-.LZGEMM_L2_BEGIN:
 
 	mr		CO,	C
 	mr		AO,	A
 	slwi		T1,	LDC	,	1
 	add		C,	C,	T1
 	srawi.		I,	M,	3
-	ble		.LZGEMM_L2x8_END
+	ble		ZGEMM_L2x8_END
 
-.LZGEMM_L2x8_BEGIN:
+ZGEMM_L2x8_BEGIN:
 
 
-	mr		BO,	B
+	mr		BO,	BBUFFER
 	srawi.		L,	K,	3
-	ble		.LZGEMM_L2x8_SUB0
+	ble		ZGEMM_L2x8_SUB0
 	cmpwi		cr0,	L,	1
-	ble		.LZGEMM_L2x8_SUB4
+	ble		ZGEMM_L2x8_SUB4
 
-.LZGEMM_L2x8_LOOP_START:
+ZGEMM_L2x8_LOOP_START:
 
 	dcbt		AO,	PRE
+	dcbt		BO,	PRE
 	LOAD2x8_1
 	dcbt		AO,	PRE
 	KERNEL2x8_I1
 	dcbt		AO,	PRE
+	dcbt		BO,	PRE
 	KERNEL2x8_2
 	dcbt		AO,	PRE
 	KERNEL2x8_1
 	dcbt		AO,	PRE
+	dcbt		BO,	PRE
 	KERNEL2x8_2
 
 	dcbt		AO,	PRE
 	KERNEL2x8_1
 	dcbt		AO,	PRE
+	dcbt		BO,	PRE
 	KERNEL2x8_2
 	dcbt		AO,	PRE
 	KERNEL2x8_1
 	dcbt		AO,	PRE
+	dcbt		BO,	PRE
 	KERNEL2x8_2
 
 	addic.		L,	L,	-2
-	ble		.LZGEMM_L2x8_LOOP_END
+	ble		ZGEMM_L2x8_LOOP_END
 
 	.align 5
 
-.LZGEMM_L2x8_LOOP:
+ZGEMM_L2x8_LOOP:
 
 	dcbt		AO,	PRE
 	KERNEL2x8_1
 	dcbt		AO,	PRE
+	dcbt		BO,	PRE
 	KERNEL2x8_2
 	dcbt		AO,	PRE
 	KERNEL2x8_1
 	dcbt		AO,	PRE
+	dcbt		BO,	PRE
 	KERNEL2x8_2
 
 	dcbt		AO,	PRE
 	KERNEL2x8_1
 	dcbt		AO,	PRE
+	dcbt		BO,	PRE
 	KERNEL2x8_2
 	dcbt		AO,	PRE
 	KERNEL2x8_1
 	dcbt		AO,	PRE
+	dcbt		BO,	PRE
 	KERNEL2x8_2
 
 	addic.		L,	L,	-1
-	bgt		.LZGEMM_L2x8_LOOP
+	bgt		ZGEMM_L2x8_LOOP
 
-.LZGEMM_L2x8_LOOP_END:
+ZGEMM_L2x8_LOOP_END:
 
 	dcbt		AO,	PRE
 	KERNEL2x8_1
 	dcbt		AO,	PRE
+	dcbt		BO,	PRE
 	KERNEL2x8_2
 	dcbt		AO,	PRE
 	KERNEL2x8_1
 	dcbt		AO,	PRE
+	dcbt		BO,	PRE
 	KERNEL2x8_2
 
 	dcbt		AO,	PRE
@@ -88,9 +116,9 @@
 	KERNEL2x8_1
 	KERNEL2x8_E2
 
-	b		.LZGEMM_L2x8_SUB1
+	b		ZGEMM_L2x8_SUB1
 
-.LZGEMM_L2x8_SUB4:
+ZGEMM_L2x8_SUB4:
 
 	dcbt		AO,	PRE
 	KERNEL2x8_SUBI1
@@ -106,53 +134,53 @@
 	KERNEL2x8_SUB1
 	KERNEL2x8_SUB1
 
-	b		.LZGEMM_L2x8_SUB1
+	b		ZGEMM_L2x8_SUB1
 
-.LZGEMM_L2x8_SUB0:
+ZGEMM_L2x8_SUB0:
 
 	andi.		L,	K,	7
 
 	KERNEL2x8_SUBI1
 
 	addic.		L,	L,	-1
-	ble		.LZGEMM_L2x8_SAVE
-	b		.LZGEMM_L2x8_SUB2
+	ble		ZGEMM_L2x8_SAVE
+	b		ZGEMM_L2x8_SUB2
 
-.LZGEMM_L2x8_SUB1:
+ZGEMM_L2x8_SUB1:
 
 	andi.		L,	K,	7
-	ble		.LZGEMM_L2x8_SAVE
+	ble		ZGEMM_L2x8_SAVE
 
-.LZGEMM_L2x8_SUB2:
+ZGEMM_L2x8_SUB2:
 
 	KERNEL2x8_SUB1
 
 	addic.		L,	L,	-1
-	bgt		.LZGEMM_L2x8_SUB2
+	bgt		ZGEMM_L2x8_SUB2
 
-.LZGEMM_L2x8_SAVE:
+ZGEMM_L2x8_SAVE:
 
 	SAVE2x8
 
 	addic.		I,	I,	-1
-	bgt		.LZGEMM_L2x8_BEGIN
+	bgt		ZGEMM_L2x8_BEGIN
 
-.LZGEMM_L2x8_END:
+ZGEMM_L2x8_END:
 
-.LZGEMM_L2x4_BEGIN:
+ZGEMM_L2x4_BEGIN:
 
 	andi.		T2,	M,	7
-	ble		.LZGEMM_L2x1_END
+	ble		ZGEMM_L2x1_END
 
 	andi.		T1,	M,	4
-	ble		.LZGEMM_L2x4_END
-	mr		BO,	B
+	ble		ZGEMM_L2x4_END
+	mr		BO,	BBUFFER
 	srawi.		L,	K,	3
-	ble		.LZGEMM_L2x4_SUB0
+	ble		ZGEMM_L2x4_SUB0
 	cmpwi		cr0,	L,	1
-	ble		.LZGEMM_L2x4_SUB4
+	ble		ZGEMM_L2x4_SUB4
 
-.LZGEMM_L2x4_LOOP_START:
+ZGEMM_L2x4_LOOP_START:
 
 	LOAD2x4_1
 	KERNEL2x4_I1
@@ -166,11 +194,11 @@
 	KERNEL2x4_2
 
 	addic.		L,	L,	-2
-	ble		.LZGEMM_L2x4_LOOP_END
+	ble		ZGEMM_L2x4_LOOP_END
 
 	.align 5
 
-.LZGEMM_L2x4_LOOP:
+ZGEMM_L2x4_LOOP:
 
 	KERNEL2x4_1
 	KERNEL2x4_2
@@ -183,9 +211,9 @@
 	KERNEL2x4_2
 
 	addic.		L,	L,	-1
-	bgt		.LZGEMM_L2x4_LOOP
+	bgt		ZGEMM_L2x4_LOOP
 
-.LZGEMM_L2x4_LOOP_END:
+ZGEMM_L2x4_LOOP_END:
 
 	KERNEL2x4_1
 	KERNEL2x4_2
@@ -197,9 +225,9 @@
 	KERNEL2x4_1
 	KERNEL2x4_E2
 
-	b		.LZGEMM_L2x4_SUB1
+	b		ZGEMM_L2x4_SUB1
 
-.LZGEMM_L2x4_SUB4:
+ZGEMM_L2x4_SUB4:
 
 	KERNEL2x4_SUBI1
 	KERNEL2x4_SUB1
@@ -211,48 +239,48 @@
 	KERNEL2x4_SUB1
 	KERNEL2x4_SUB1
 
-	b		.LZGEMM_L2x4_SUB1
+	b		ZGEMM_L2x4_SUB1
 
-.LZGEMM_L2x4_SUB0:
+ZGEMM_L2x4_SUB0:
 
 	andi.		L,	K,	7
 
 	KERNEL2x4_SUBI1
 
 	addic.		L,	L,	-1
-	ble		.LZGEMM_L2x4_SAVE
-	b		.LZGEMM_L2x4_SUB2
+	ble		ZGEMM_L2x4_SAVE
+	b		ZGEMM_L2x4_SUB2
 
-.LZGEMM_L2x4_SUB1:
+ZGEMM_L2x4_SUB1:
 
 	andi.		L,	K,	7
-	ble		.LZGEMM_L2x4_SAVE
+	ble		ZGEMM_L2x4_SAVE
 
-.LZGEMM_L2x4_SUB2:
+ZGEMM_L2x4_SUB2:
 
 	KERNEL2x4_SUB1
 
 	addic.		L,	L,	-1
-	bgt		.LZGEMM_L2x4_SUB2
+	bgt		ZGEMM_L2x4_SUB2
 
-.LZGEMM_L2x4_SAVE:
+ZGEMM_L2x4_SAVE:
 
 	SAVE2x4
 
-.LZGEMM_L2x4_END:
+ZGEMM_L2x4_END:
 
-.LZGEMM_L2x2_BEGIN:
+ZGEMM_L2x2_BEGIN:
 
 
 	andi.		T1,	M,	2
-	ble		.LZGEMM_L2x2_END
-	mr		BO,	B
+	ble		ZGEMM_L2x2_END
+	mr		BO,	BBUFFER
 	srawi.		L,	K,	3
-	ble		.LZGEMM_L2x2_SUB0
+	ble		ZGEMM_L2x2_SUB0
 	cmpwi		cr0,	L,	1
-	ble		.LZGEMM_L2x2_SUB4
+	ble		ZGEMM_L2x2_SUB4
 
-.LZGEMM_L2x2_LOOP_START:
+ZGEMM_L2x2_LOOP_START:
 
 	LOAD2x2_1
 	KERNEL2x2_I1
@@ -266,11 +294,11 @@
 	KERNEL2x2_2
 
 	addic.		L,	L,	-2
-	ble		.LZGEMM_L2x2_LOOP_END
+	ble		ZGEMM_L2x2_LOOP_END
 
 	.align 5
 
-.LZGEMM_L2x2_LOOP:
+ZGEMM_L2x2_LOOP:
 
 	KERNEL2x2_1
 	KERNEL2x2_2
@@ -283,9 +311,9 @@
 	KERNEL2x2_2
 
 	addic.		L,	L,	-1
-	bgt		.LZGEMM_L2x2_LOOP
+	bgt		ZGEMM_L2x2_LOOP
 
-.LZGEMM_L2x2_LOOP_END:
+ZGEMM_L2x2_LOOP_END:
 
 	KERNEL2x2_1
 	KERNEL2x2_2
@@ -297,9 +325,9 @@
 	KERNEL2x2_1
 	KERNEL2x2_E2
 
-	b		.LZGEMM_L2x2_SUB1
+	b		ZGEMM_L2x2_SUB1
 
-.LZGEMM_L2x2_SUB4:
+ZGEMM_L2x2_SUB4:
 
 	KERNEL2x2_SUBI1
 	KERNEL2x2_SUB1
@@ -311,48 +339,48 @@
 	KERNEL2x2_SUB1
 	KERNEL2x2_SUB1
 
-	b		.LZGEMM_L2x2_SUB1
+	b		ZGEMM_L2x2_SUB1
 
-.LZGEMM_L2x2_SUB0:
+ZGEMM_L2x2_SUB0:
 
 	andi.		L,	K,	7
 
 	KERNEL2x2_SUBI1
 
 	addic.		L,	L,	-1
-	ble		.LZGEMM_L2x2_SAVE
-	b		.LZGEMM_L2x2_SUB2
+	ble		ZGEMM_L2x2_SAVE
+	b		ZGEMM_L2x2_SUB2
 
-.LZGEMM_L2x2_SUB1:
+ZGEMM_L2x2_SUB1:
 
 	andi.		L,	K,	7
-	ble		.LZGEMM_L2x2_SAVE
+	ble		ZGEMM_L2x2_SAVE
 
-.LZGEMM_L2x2_SUB2:
+ZGEMM_L2x2_SUB2:
 
 	KERNEL2x2_SUB1
 
 	addic.		L,	L,	-1
-	bgt		.LZGEMM_L2x2_SUB2
+	bgt		ZGEMM_L2x2_SUB2
 
-.LZGEMM_L2x2_SAVE:
+ZGEMM_L2x2_SAVE:
 
 	SAVE2x2
 
-.LZGEMM_L2x2_END:
+ZGEMM_L2x2_END:
 
-.LZGEMM_L2x1_BEGIN:
+ZGEMM_L2x1_BEGIN:
 
 
 	andi.		T1,	M,	1
-	ble		.LZGEMM_L2x1_END
-	mr		BO,	B
+	ble		ZGEMM_L2x1_END
+	mr		BO,	BBUFFER
 	srawi.		L,	K,	3
-	ble		.LZGEMM_L2x1_SUB0
+	ble		ZGEMM_L2x1_SUB0
 	cmpwi		cr0,	L,	1
-	ble		.LZGEMM_L2x1_SUB4
+	ble		ZGEMM_L2x1_SUB4
 
-.LZGEMM_L2x1_LOOP_START:
+ZGEMM_L2x1_LOOP_START:
 
 	LOAD2x1_1
 	KERNEL2x1_I1
@@ -366,11 +394,11 @@
 	KERNEL2x1_2
 
 	addic.		L,	L,	-2
-	ble		.LZGEMM_L2x1_LOOP_END
+	ble		ZGEMM_L2x1_LOOP_END
 
 	.align 5
 
-.LZGEMM_L2x1_LOOP:
+ZGEMM_L2x1_LOOP:
 
 	KERNEL2x1_1
 	KERNEL2x1_2
@@ -383,9 +411,9 @@
 	KERNEL2x1_2
 
 	addic.		L,	L,	-1
-	bgt		.LZGEMM_L2x1_LOOP
+	bgt		ZGEMM_L2x1_LOOP
 
-.LZGEMM_L2x1_LOOP_END:
+ZGEMM_L2x1_LOOP_END:
 
 	KERNEL2x1_1
 	KERNEL2x1_2
@@ -397,9 +425,9 @@
 	KERNEL2x1_1
 	KERNEL2x1_E2
 
-	b		.LZGEMM_L2x1_SUB1
+	b		ZGEMM_L2x1_SUB1
 
-.LZGEMM_L2x1_SUB4:
+ZGEMM_L2x1_SUB4:
 
 	KERNEL2x1_SUBI1
 	KERNEL2x1_SUB1
@@ -411,72 +439,89 @@
 	KERNEL2x1_SUB1
 	KERNEL2x1_SUB1
 
-	b		.LZGEMM_L2x1_SUB1
+	b		ZGEMM_L2x1_SUB1
 
-.LZGEMM_L2x1_SUB0:
+ZGEMM_L2x1_SUB0:
 
 	andi.		L,	K,	7
 
 	KERNEL2x1_SUBI1
 
 	addic.		L,	L,	-1
-	ble		.LZGEMM_L2x1_SAVE
-	b		.LZGEMM_L2x1_SUB2
+	ble		ZGEMM_L2x1_SAVE
+	b		ZGEMM_L2x1_SUB2
 
-.LZGEMM_L2x1_SUB1:
+ZGEMM_L2x1_SUB1:
 
 	andi.		L,	K,	7
-	ble		.LZGEMM_L2x1_SAVE
+	ble		ZGEMM_L2x1_SAVE
 
-.LZGEMM_L2x1_SUB2:
+ZGEMM_L2x1_SUB2:
 
 	KERNEL2x1_SUB1
 
 	addic.		L,	L,	-1
-	bgt		.LZGEMM_L2x1_SUB2
+	bgt		ZGEMM_L2x1_SUB2
 
-.LZGEMM_L2x1_SAVE:
+ZGEMM_L2x1_SAVE:
 
 	SAVE2x1
 
-.LZGEMM_L2x1_END:
+ZGEMM_L2x1_END:
 
 	slwi		T1,	K,	5
 	add		B,	B,	T1
 
 	addic.		J,	J,	-1
-	bgt		.LZGEMM_L2_BEGIN
+	bgt		ZGEMM_L2_BEGIN
 
 	andi.		T2,	N,	1
-	ble		.L999
+	ble		L999
 
-.LZGEMM_L2_END:
+ZGEMM_L2_END:
 
-	b		.LZGEMM_L1_BEGIN
+	b		ZGEMM_L1_BEGIN
 
-.L999_H1:
+L999_H1:
 
-	b		.L999
+	b		L999
+
+ZGEMM_L1_BEGIN:
+
+	mr		BO,	B
+	mr		BBO,	BBUFFER
+	slwi		T1,	K,	0
+
+ZGEMM_L1_COPYB:
+
+	lxvdsx		vs4,	o0,	BO              // b0_r
+	lxvdsx		vs5,	o8,	BO              // b0_i
+	addi		BO,	BO,	16
+	stxvd2x		vs4,	o0,	BBO
+	stxvd2x		vs5,	o16,	BBO
+	addic.		T1,	T1,	-1
+	addi		BBO,	BBO,	32
+
+	bge		ZGEMM_L1_COPYB
 
-.LZGEMM_L1_BEGIN:
 
 	andi.		T1,	N,	1
-	ble		.LZGEMM_L1_END
+	ble		ZGEMM_L1_END
 	mr		CO,	C
 	mr		AO,	A
 	srawi.		I,	M,	3
-	ble		.LZGEMM_L1x8_END
+	ble		ZGEMM_L1x8_END
 
-.LZGEMM_L1x8_BEGIN:
+ZGEMM_L1x8_BEGIN:
 
 
-	mr		BO,	B
+	mr		BO,	BBUFFER
 	srawi.		L,	K,	3
-	ble		.LZGEMM_L1x8_SUB0
+	ble		ZGEMM_L1x8_SUB0
 	cmpwi		cr0,	L,	1
-	ble		.LZGEMM_L1x8_SUB4
+	ble		ZGEMM_L1x8_SUB4
 
-.LZGEMM_L1x8_LOOP_START:
+ZGEMM_L1x8_LOOP_START:
 
 	dcbt		AO,	PRE
 	LOAD1x8_1
@@ -499,11 +544,11 @@
 	KERNEL1x8_2
 
 	addic.		L,	L,	-2
-	ble		.LZGEMM_L1x8_LOOP_END
+	ble		ZGEMM_L1x8_LOOP_END
 
 	.align 5
 
-.LZGEMM_L1x8_LOOP:
+ZGEMM_L1x8_LOOP:
 
 	dcbt		AO,	PRE
 	KERNEL1x8_1
@@ -524,9 +569,9 @@
 	KERNEL1x8_2
 
 	addic.		L,	L,	-1
-	bgt		.LZGEMM_L1x8_LOOP
+	bgt		ZGEMM_L1x8_LOOP
 
-.LZGEMM_L1x8_LOOP_END:
+ZGEMM_L1x8_LOOP_END:
 
 	dcbt		AO,	PRE
 	KERNEL1x8_1
@@ -545,9 +590,9 @@
 	KERNEL1x8_1
 	KERNEL1x8_E2
 
-	b		.LZGEMM_L1x8_SUB1
+	b		ZGEMM_L1x8_SUB1
 
-.LZGEMM_L1x8_SUB4:
+ZGEMM_L1x8_SUB4:
 
 	dcbt		AO,	PRE
 	KERNEL1x8_SUBI1
@@ -563,53 +608,53 @@
 	KERNEL1x8_SUB1
 	KERNEL1x8_SUB1
 
-	b		.LZGEMM_L1x8_SUB1
+	b		ZGEMM_L1x8_SUB1
 
-.LZGEMM_L1x8_SUB0:
+ZGEMM_L1x8_SUB0:
 
 	andi.		L,	K,	7
 
 	KERNEL1x8_SUBI1
 
 	addic.		L,	L,	-1
-	ble		.LZGEMM_L1x8_SAVE
-	b		.LZGEMM_L1x8_SUB2
+	ble		ZGEMM_L1x8_SAVE
+	b		ZGEMM_L1x8_SUB2
 
-.LZGEMM_L1x8_SUB1:
+ZGEMM_L1x8_SUB1:
 
 	andi.		L,	K,	7
-	ble		.LZGEMM_L1x8_SAVE
+	ble		ZGEMM_L1x8_SAVE
 
-.LZGEMM_L1x8_SUB2:
+ZGEMM_L1x8_SUB2:
 
 	KERNEL1x8_SUB1
 
 	addic.		L,	L,	-1
-	bgt		.LZGEMM_L1x8_SUB2
+	bgt		ZGEMM_L1x8_SUB2
 
-.LZGEMM_L1x8_SAVE:
+ZGEMM_L1x8_SAVE:
 
 	SAVE1x8
 
 	addic.		I,	I,	-1
-	bgt		.LZGEMM_L1x8_BEGIN
+	bgt		ZGEMM_L1x8_BEGIN
 
-.LZGEMM_L1x8_END:
+ZGEMM_L1x8_END:
 
-.LZGEMM_L1x4_BEGIN:
+ZGEMM_L1x4_BEGIN:
 
 	andi.		T2,	M,	7
-	ble		.LZGEMM_L1x1_END
+	ble		ZGEMM_L1x1_END
 
 	andi.		T1,	M,	4
-	ble		.LZGEMM_L1x4_END
-	mr		BO,	B
+	ble		ZGEMM_L1x4_END
+	mr		BO,	BBUFFER
 	srawi.		L,	K,	3
-	ble		.LZGEMM_L1x4_SUB0
+	ble		ZGEMM_L1x4_SUB0
 	cmpwi		cr0,	L,	1
-	ble		.LZGEMM_L1x4_SUB4
+	ble		ZGEMM_L1x4_SUB4
 
-.LZGEMM_L1x4_LOOP_START:
+ZGEMM_L1x4_LOOP_START:
 
 	LOAD1x4_1
 	KERNEL1x4_I1
@@ -623,11 +668,11 @@
 	KERNEL1x4_2
 
 	addic.		L,	L,	-2
-	ble		.LZGEMM_L1x4_LOOP_END
+	ble		ZGEMM_L1x4_LOOP_END
 
 	.align 5
 
-.LZGEMM_L1x4_LOOP:
+ZGEMM_L1x4_LOOP:
 
 	KERNEL1x4_1
 	KERNEL1x4_2
@@ -640,9 +685,9 @@
 	KERNEL1x4_2
 
 	addic.		L,	L,	-1
-	bgt		.LZGEMM_L1x4_LOOP
+	bgt		ZGEMM_L1x4_LOOP
 
-.LZGEMM_L1x4_LOOP_END:
+ZGEMM_L1x4_LOOP_END:
 
 	KERNEL1x4_1
 	KERNEL1x4_2
@@ -654,9 +699,9 @@
 	KERNEL1x4_1
 	KERNEL1x4_E2
 
-	b		.LZGEMM_L1x4_SUB1
+	b		ZGEMM_L1x4_SUB1
 
-.LZGEMM_L1x4_SUB4:
+ZGEMM_L1x4_SUB4:
 
 	KERNEL1x4_SUBI1
 	KERNEL1x4_SUB1
@@ -668,48 +713,48 @@
 	KERNEL1x4_SUB1
 	KERNEL1x4_SUB1
 
-	b		.LZGEMM_L1x4_SUB1
+	b		ZGEMM_L1x4_SUB1
 
-.LZGEMM_L1x4_SUB0:
+ZGEMM_L1x4_SUB0:
 
 	andi.		L,	K,	7
 
 	KERNEL1x4_SUBI1
 
 	addic.		L,	L,	-1
-	ble		.LZGEMM_L1x4_SAVE
-	b		.LZGEMM_L1x4_SUB2
+	ble		ZGEMM_L1x4_SAVE
+	b		ZGEMM_L1x4_SUB2
 
-.LZGEMM_L1x4_SUB1:
+ZGEMM_L1x4_SUB1:
 
 	andi.		L,	K,	7
-	ble		.LZGEMM_L1x4_SAVE
+	ble		ZGEMM_L1x4_SAVE
 
-.LZGEMM_L1x4_SUB2:
+ZGEMM_L1x4_SUB2:
 
 	KERNEL1x4_SUB1
 
 	addic.		L,	L,	-1
-	bgt		.LZGEMM_L1x4_SUB2
+	bgt		ZGEMM_L1x4_SUB2
 
-.LZGEMM_L1x4_SAVE:
+ZGEMM_L1x4_SAVE:
 
 	SAVE1x4
 
-.LZGEMM_L1x4_END:
+ZGEMM_L1x4_END:
 
-.LZGEMM_L1x2_BEGIN:
+ZGEMM_L1x2_BEGIN:
 
 
 	andi.		T1,	M,	2
-	ble		.LZGEMM_L1x2_END
-	mr		BO,	B
+	ble		ZGEMM_L1x2_END
+	mr		BO,	BBUFFER
 	srawi.		L,	K,	3
-	ble		.LZGEMM_L1x2_SUB0
+	ble		ZGEMM_L1x2_SUB0
 	cmpwi		cr0,	L,	1
-	ble		.LZGEMM_L1x2_SUB4
+	ble		ZGEMM_L1x2_SUB4
 
-.LZGEMM_L1x2_LOOP_START:
+ZGEMM_L1x2_LOOP_START:
 
 	LOAD1x2_1
 	KERNEL1x2_I1
@@ -723,11 +768,11 @@
 	KERNEL1x2_2
 
 	addic.		L,	L,	-2
-	ble		.LZGEMM_L1x2_LOOP_END
+	ble		ZGEMM_L1x2_LOOP_END
 
 	.align 5
 
-.LZGEMM_L1x2_LOOP:
+ZGEMM_L1x2_LOOP:
 
 	KERNEL1x2_1
 	KERNEL1x2_2
@@ -740,9 +785,9 @@
 	KERNEL1x2_2
 
 	addic.		L,	L,	-1
-	bgt		.LZGEMM_L1x2_LOOP
+	bgt		ZGEMM_L1x2_LOOP
 
-.LZGEMM_L1x2_LOOP_END:
+ZGEMM_L1x2_LOOP_END:
 
 	KERNEL1x2_1
 	KERNEL1x2_2
@@ -754,9 +799,9 @@
 	KERNEL1x2_1
 	KERNEL1x2_E2
 
-	b		.LZGEMM_L1x2_SUB1
+	b		ZGEMM_L1x2_SUB1
 
-.LZGEMM_L1x2_SUB4:
+ZGEMM_L1x2_SUB4:
 
 	KERNEL1x2_SUBI1
 	KERNEL1x2_SUB1
@@ -768,48 +813,48 @@
 	KERNEL1x2_SUB1
 	KERNEL1x2_SUB1
 
-	b		.LZGEMM_L1x2_SUB1
+	b		ZGEMM_L1x2_SUB1
 
-.LZGEMM_L1x2_SUB0:
+ZGEMM_L1x2_SUB0:
 
 	andi.		L,	K,	7
 
 	KERNEL1x2_SUBI1
 
 	addic.		L,	L,	-1
-	ble		.LZGEMM_L1x2_SAVE
-	b		.LZGEMM_L1x2_SUB2
+	ble		ZGEMM_L1x2_SAVE
+	b		ZGEMM_L1x2_SUB2
 
-.LZGEMM_L1x2_SUB1:
+ZGEMM_L1x2_SUB1:
 
 	andi.		L,	K,	7
-	ble		.LZGEMM_L1x2_SAVE
+	ble		ZGEMM_L1x2_SAVE
 
-.LZGEMM_L1x2_SUB2:
+ZGEMM_L1x2_SUB2:
 
 	KERNEL1x2_SUB1
 
 	addic.		L,	L,	-1
-	bgt		.LZGEMM_L1x2_SUB2
+	bgt		ZGEMM_L1x2_SUB2
 
-.LZGEMM_L1x2_SAVE:
+ZGEMM_L1x2_SAVE:
 
 	SAVE1x2
 
-.LZGEMM_L1x2_END:
+ZGEMM_L1x2_END:
 
-.LZGEMM_L1x1_BEGIN:
+ZGEMM_L1x1_BEGIN:
 
 
 	andi.		T1,	M,	1
-	ble		.LZGEMM_L1x1_END
-	mr		BO,	B
+	ble		ZGEMM_L1x1_END
+	mr		BO,	BBUFFER
 	srawi.		L,	K,	3
-	ble		.LZGEMM_L1x1_SUB0
+	ble		ZGEMM_L1x1_SUB0
 	cmpwi		cr0,	L,	1
-	ble		.LZGEMM_L1x1_SUB4
+	ble		ZGEMM_L1x1_SUB4
 
-.LZGEMM_L1x1_LOOP_START:
+ZGEMM_L1x1_LOOP_START:
 
 	LOAD1x1_1
 	KERNEL1x1_I1
@@ -823,11 +868,11 @@
 	KERNEL1x1_2
 
 	addic.		L,	L,	-2
-	ble		.LZGEMM_L1x1_LOOP_END
+	ble		ZGEMM_L1x1_LOOP_END
 
 	.align 5
 
-.LZGEMM_L1x1_LOOP:
+ZGEMM_L1x1_LOOP:
 
 	KERNEL1x1_1
 	KERNEL1x1_2
@@ -840,9 +885,9 @@
 	KERNEL1x1_2
 
 	addic.		L,	L,	-1
-	bgt		.LZGEMM_L1x1_LOOP
+	bgt		ZGEMM_L1x1_LOOP
 
-.LZGEMM_L1x1_LOOP_END:
+ZGEMM_L1x1_LOOP_END:
 
 	KERNEL1x1_1
 	KERNEL1x1_2
@@ -854,9 +899,9 @@
 	KERNEL1x1_1
 	KERNEL1x1_E2
 
-	b		.LZGEMM_L1x1_SUB1
+	b		ZGEMM_L1x1_SUB1
 
-.LZGEMM_L1x1_SUB4:
+ZGEMM_L1x1_SUB4:
 
 	KERNEL1x1_SUBI1
 	KERNEL1x1_SUB1
@@ -868,34 +913,34 @@
 	KERNEL1x1_SUB1
 	KERNEL1x1_SUB1
 
-	b		.LZGEMM_L1x1_SUB1
+	b		ZGEMM_L1x1_SUB1
 
-.LZGEMM_L1x1_SUB0:
+ZGEMM_L1x1_SUB0:
 
 	andi.		L,	K,	7
 
 	KERNEL1x1_SUBI1
 
 	addic.		L,	L,	-1
-	ble		.LZGEMM_L1x1_SAVE
-	b		.LZGEMM_L1x1_SUB2
+	ble		ZGEMM_L1x1_SAVE
+	b		ZGEMM_L1x1_SUB2
 
-.LZGEMM_L1x1_SUB1:
+ZGEMM_L1x1_SUB1:
 
 	andi.		L,	K,	7
-	ble		.LZGEMM_L1x1_SAVE
+	ble		ZGEMM_L1x1_SAVE
 
-.LZGEMM_L1x1_SUB2:
+ZGEMM_L1x1_SUB2:
 
 	KERNEL1x1_SUB1
 
 	addic.		L,	L,	-1
-	bgt		.LZGEMM_L1x1_SUB2
+	bgt		ZGEMM_L1x1_SUB2
 
-.LZGEMM_L1x1_SAVE:
+ZGEMM_L1x1_SAVE:
 
 	SAVE1x1
 
-.LZGEMM_L1x1_END:
+ZGEMM_L1x1_END:
 
-.LZGEMM_L1_END:
+ZGEMM_L1_END:

kernel/power/zgemm_macros_8x2_power8.S

@@ -1,39 +1,3 @@
-/***************************************************************************
-Copyright (c) 2013-2016, The OpenBLAS Project
-All rights reserved.
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in
-the documentation and/or other materials provided with the
-distribution.
-3. Neither the name of the OpenBLAS project nor the names of
-its contributors may be used to endorse or promote products
-derived from this software without specific prior written permission.
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
-USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*****************************************************************************/
-
-/**************************************************************************************
-* 2016/03/05 Werner Saar (wernsaar@googlemail.com)
-* 	 BLASTEST 		: OK
-* 	 CTEST			: OK
-* 	 TEST			: OK
-*	 LAPACK-TEST		: OK
-**************************************************************************************/
-
-
 #if   defined(NN) || defined(NT) || defined(TN) || defined(TT)
 
 	#define	XSFADD_R1	xsadddp
@@ -70,12 +34,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro LOAD2x8_1
 
-	lxvdsx		vs16,	o0,	BO		// load real part from B
-	lxvdsx		vs17,	o8,	BO		// load imag part from B
-	lxvdsx		vs18,	o16,	BO		// load real part from B
-	lxvdsx		vs19,	o24,	BO		// load imag part from B
+	lxvd2x		vs16,	o0,	BO		// load real part from B
+	lxvd2x		vs17,	o16,	BO		// load imag part from B
+	lxvd2x		vs18,	o32,	BO		// load real part from B
+	lxvd2x		vs19,	o48,	BO		// load imag part from B
 
-	addi		BO,	BO,	32
+	addi		BO,	BO,	64
 
 	lxvd2x		vs0,	o0,	AO		// load real,imag from A
 	lxvd2x		vs1,	o16,	AO		// load real,imag from A
@@ -110,12 +74,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 	addi		AO,	AO,	64
 
-	lxvdsx		vs20,	o0,	BO		// load real part from B
-	lxvdsx		vs21,	o8,	BO		// load imag part from B
-	lxvdsx		vs22,	o16,	BO		// load real part from B
-	lxvdsx		vs23,	o24,	BO		// load imag part from B
+	lxvd2x		vs20,	o0,	BO		// load real part from B
+	lxvd2x		vs21,	o16,	BO		// load imag part from B
+	lxvd2x		vs22,	o32,	BO		// load real part from B
+	lxvd2x		vs23,	o48,	BO		// load imag part from B
 
-	addi		BO,	BO,	32
+	addi		BO,	BO,	64
 
 	xvmuldp		vs32,	vs0,	vs16		// real*real, imag*real
 	xvmuldp		vs33,	vs0,	vs17		// real*imag, imag*imag
@@ -156,36 +120,41 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro KERNEL2x8_1
 
+	lxvd2x		vs8,	o0,	AO		// load real,imag from A
+	lxvd2x		vs9,	o16,	AO		// load real,imag from A
+	lxvd2x		vs10,	o32,	AO		// load real,imag from A
+	lxvd2x		vs11,	o48,	AO		// load real,imag from A
+
+	addi		AO,	AO,	64
+
+	lxvd2x		vs12,	o0,	AO		// load real,imag from A
+	lxvd2x		vs13,	o16,	AO		// load real,imag from A
+	lxvd2x		vs14,	o32,	AO		// load real,imag from A
+	lxvd2x		vs15,	o48,	AO		// load real,imag from A
+
+	addi		AO,	AO,	64
+
+	lxvd2x		vs20,	o0,	BO		// load real part from B
+	lxvd2x		vs21,	o16,	BO		// load imag part from B
+	lxvd2x		vs22,	o32,	BO		// load real part from B
+	lxvd2x		vs23,	o48,	BO		// load imag part from B
+
+	addi		BO,	BO,	64
 
 	xvmaddadp	vs32,	vs0,	vs16		// real*real, imag*real
 	xvmaddadp	vs33,	vs0,	vs17		// real*imag, imag*imag
 	xvmaddadp	vs34,	vs1,	vs16		// real*real, imag*real
 	xvmaddadp	vs35,	vs1,	vs17		// real*imag, imag*imag
-
-	lxvdsx		vs22,	o16,	BO		// load real part from B
-	lxvdsx		vs23,	o24,	BO		// load imag part from B
-
 	xvmaddadp	vs36,	vs2,	vs16		// real*real, imag*real
 	xvmaddadp	vs37,	vs2,	vs17		// real*imag, imag*imag
 	xvmaddadp	vs38,	vs3,	vs16		// real*real, imag*real
 	xvmaddadp	vs39,	vs3,	vs17		// real*imag, imag*imag
-
-	lxvd2x		vs8,	o0,	AO		// load real,imag from A
-	lxvd2x		vs9,	o16,	AO		// load real,imag from A
-
 	xvmaddadp	vs40,	vs4,	vs16		// real*real, imag*real
 	xvmaddadp	vs41,	vs4,	vs17		// real*imag, imag*imag
 	xvmaddadp	vs42,	vs5,	vs16		// real*real, imag*real
 	xvmaddadp	vs43,	vs5,	vs17		// real*imag, imag*imag
-
-	lxvd2x		vs10,	o32,	AO		// load real,imag from A
-	lxvd2x		vs11,	o48,	AO		// load real,imag from A
-
 	xvmaddadp	vs44,	vs6,	vs16		// real*real, imag*real
 	xvmaddadp	vs45,	vs6,	vs17		// real*imag, imag*imag
-
-	addi		AO,	AO,	64
-
 	xvmaddadp	vs46,	vs7,	vs16		// real*real, imag*real
 	xvmaddadp	vs47,	vs7,	vs17		// real*imag, imag*imag
 
@@ -193,101 +162,79 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 	xvmaddadp	vs49,	vs0,	vs19		// real*imag, imag*imag
 	xvmaddadp	vs50,	vs1,	vs18		// real*real, imag*real
 	xvmaddadp	vs51,	vs1,	vs19		// real*imag, imag*imag
-
-	lxvd2x		vs12,	o0,	AO		// load real,imag from A
-	lxvd2x		vs13,	o16,	AO		// load real,imag from A
-
 	xvmaddadp	vs52,	vs2,	vs18		// real*real, imag*real
 	xvmaddadp	vs53,	vs2,	vs19		// real*imag, imag*imag
 	xvmaddadp	vs54,	vs3,	vs18		// real*real, imag*real
 	xvmaddadp	vs55,	vs3,	vs19		// real*imag, imag*imag
-
-	lxvd2x		vs14,	o32,	AO		// load real,imag from A
-	lxvd2x		vs15,	o48,	AO		// load real,imag from A
-
 	xvmaddadp	vs56,	vs4,	vs18		// real*real, imag*real
 	xvmaddadp	vs57,	vs4,	vs19		// real*imag, imag*imag
 	xvmaddadp	vs58,	vs5,	vs18		// real*real, imag*real
 	xvmaddadp	vs59,	vs5,	vs19		// real*imag, imag*imag
-
-	lxvdsx		vs20,	o0,	BO		// load real part from B
-	lxvdsx		vs21,	o8,	BO		// load imag part from B
-
 	xvmaddadp	vs60,	vs6,	vs18		// real*real, imag*real
 	xvmaddadp	vs61,	vs6,	vs19		// real*imag, imag*imag
 	xvmaddadp	vs62,	vs7,	vs18		// real*real, imag*real
 	xvmaddadp	vs63,	vs7,	vs19		// real*imag, imag*imag
 
-	addi		AO,	AO,	64
-	addi		BO,	BO,	32
 
 .endm
 
 .macro KERNEL2x8_2
 
+	lxvd2x		vs0,	o0,	AO		// load real,imag from A
+	lxvd2x		vs1,	o16,	AO		// load real,imag from A
+	lxvd2x		vs2,	o32,	AO		// load real,imag from A
+	lxvd2x		vs3,	o48,	AO		// load real,imag from A
+
+	addi		AO,	AO,	64
+
+	lxvd2x		vs4,	o0,	AO		// load real,imag from A
+	lxvd2x		vs5,	o16,	AO		// load real,imag from A
+	lxvd2x		vs6,	o32,	AO		// load real,imag from A
+	lxvd2x		vs7,	o48,	AO		// load real,imag from A
+
+	addi		AO,	AO,	64
+
+	lxvd2x		vs16,	o0,	BO		// load real part from B
+	lxvd2x		vs17,	o16,	BO		// load imag part from B
+	lxvd2x		vs18,	o32,	BO		// load real part from B
+	lxvd2x		vs19,	o48,	BO		// load imag part from B
+
+	addi		BO,	BO,	64
 
 	xvmaddadp	vs32,	vs8,	vs20		// real*real, imag*real
 	xvmaddadp	vs33,	vs8,	vs21		// real*imag, imag*imag
 	xvmaddadp	vs34,	vs9,	vs20		// real*real, imag*real
 	xvmaddadp	vs35,	vs9,	vs21		// real*imag, imag*imag
-
-	lxvdsx		vs16,	o0,	BO		// load real part from B
-	lxvdsx		vs17,	o8,	BO		// load imag part from B
-
 	xvmaddadp	vs36,	vs10,	vs20		// real*real, imag*real
 	xvmaddadp	vs37,	vs10,	vs21		// real*imag, imag*imag
 	xvmaddadp	vs38,	vs11,	vs20		// real*real, imag*real
 	xvmaddadp	vs39,	vs11,	vs21		// real*imag, imag*imag
-
-	lxvd2x		vs0,	o0,	AO		// load real,imag from A
-	lxvd2x		vs1,	o16,	AO		// load real,imag from A
-
 	xvmaddadp	vs40,	vs12,	vs20		// real*real, imag*real
 	xvmaddadp	vs41,	vs12,	vs21		// real*imag, imag*imag
 	xvmaddadp	vs42,	vs13,	vs20		// real*real, imag*real
 	xvmaddadp	vs43,	vs13,	vs21		// real*imag, imag*imag
-
-	lxvd2x		vs2,	o32,	AO		// load real,imag from A
-	lxvd2x		vs3,	o48,	AO		// load real,imag from A
-
 	xvmaddadp	vs44,	vs14,	vs20		// real*real, imag*real
 	xvmaddadp	vs45,	vs14,	vs21		// real*imag, imag*imag
 	xvmaddadp	vs46,	vs15,	vs20		// real*real, imag*real
 	xvmaddadp	vs47,	vs15,	vs21		// real*imag, imag*imag
 
-	addi		AO,	AO,	64
-
 	xvmaddadp	vs48,	vs8,	vs22		// real*real, imag*real
 	xvmaddadp	vs49,	vs8,	vs23		// real*imag, imag*imag
 	xvmaddadp	vs50,	vs9,	vs22		// real*real, imag*real
 	xvmaddadp	vs51,	vs9,	vs23		// real*imag, imag*imag
-
-	lxvd2x		vs4,	o0,	AO		// load real,imag from A
-	lxvd2x		vs5,	o16,	AO		// load real,imag from A
-
 	xvmaddadp	vs52,	vs10,	vs22		// real*real, imag*real
 	xvmaddadp	vs53,	vs10,	vs23		// real*imag, imag*imag
 	xvmaddadp	vs54,	vs11,	vs22		// real*real, imag*real
 	xvmaddadp	vs55,	vs11,	vs23		// real*imag, imag*imag
-
-	lxvd2x		vs6,	o32,	AO		// load real,imag from A
-	lxvd2x		vs7,	o48,	AO		// load real,imag from A
-
 	xvmaddadp	vs56,	vs12,	vs22		// real*real, imag*real
 	xvmaddadp	vs57,	vs12,	vs23		// real*imag, imag*imag
 	xvmaddadp	vs58,	vs13,	vs22		// real*real, imag*real
 	xvmaddadp	vs59,	vs13,	vs23		// real*imag, imag*imag
-
-	lxvdsx		vs18,	o16,	BO		// load real part from B
-	lxvdsx		vs19,	o24,	BO		// load imag part from B
-
 	xvmaddadp	vs60,	vs14,	vs22		// real*real, imag*real
 	xvmaddadp	vs61,	vs14,	vs23		// real*imag, imag*imag
 	xvmaddadp	vs62,	vs15,	vs22		// real*real, imag*real
 	xvmaddadp	vs63,	vs15,	vs23		// real*imag, imag*imag
 
-	addi		AO,	AO,	64
-	addi		BO,	BO,	32
 
 .endm
 
@@ -347,12 +294,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 	addi		AO,	AO,	64
 
-	lxvdsx		vs16,	o0,	BO		// load real part from B
-	lxvdsx		vs17,	o8,	BO		// load imag part from B
-	lxvdsx		vs18,	o16,	BO		// load real part from B
-	lxvdsx		vs19,	o24,	BO		// load imag part from B
+	lxvd2x		vs16,	o0,	BO		// load real part from B
+	lxvd2x		vs17,	o16,	BO		// load imag part from B
+	lxvd2x		vs18,	o32,	BO		// load real part from B
+	lxvd2x		vs19,	o48,	BO		// load imag part from B
 
-	addi		BO,	BO,	32
+	addi		BO,	BO,	64
 
 	xvmuldp		vs32,	vs0,	vs16		// real*real, imag*real
 	xvmuldp		vs33,	vs0,	vs17		// real*imag, imag*imag
@@ -407,12 +354,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 	addi		AO,	AO,	64
 
-	lxvdsx		vs16,	o0,	BO		// load real part from B
-	lxvdsx		vs17,	o8,	BO		// load imag part from B
-	lxvdsx		vs18,	o16,	BO		// load real part from B
-	lxvdsx		vs19,	o24,	BO		// load imag part from B
+	lxvd2x		vs16,	o0,	BO		// load real part from B
+	lxvd2x		vs17,	o16,	BO		// load imag part from B
+	lxvd2x		vs18,	o32,	BO		// load real part from B
+	lxvd2x		vs19,	o48,	BO		// load imag part from B
 
-	addi		BO,	BO,	32
+	addi		BO,	BO,	64
 
 	xvmaddadp	vs32,	vs0,	vs16		// real*real, imag*real
 	xvmaddadp	vs33,	vs0,	vs17		// real*imag, imag*imag
@@ -927,12 +874,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro LOAD2x4_1
 
-	lxvdsx		vs16,	o0,	BO		// load real part from B
-	lxvdsx		vs17,	o8,	BO		// load imag part from B
-	lxvdsx		vs18,	o16,	BO		// load real part from B
-	lxvdsx		vs19,	o24,	BO		// load imag part from B
+	lxvd2x		vs16,	o0,	BO		// load real part from B
+	lxvd2x		vs17,	o16,	BO		// load imag part from B
+	lxvd2x		vs18,	o32,	BO		// load real part from B
+	lxvd2x		vs19,	o48,	BO		// load imag part from B
 
-	addi		BO,	BO,	32
+	addi		BO,	BO,	64
 
 	lxvd2x		vs0,	o0,	AO		// load real,imag from A
 	lxvd2x		vs1,	o16,	AO		// load real,imag from A
@@ -953,12 +900,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 	addi		AO,	AO,	64
 
-	lxvdsx		vs20,	o0,	BO		// load real part from B
-	lxvdsx		vs21,	o8,	BO		// load imag part from B
-	lxvdsx		vs22,	o16,	BO		// load real part from B
-	lxvdsx		vs23,	o24,	BO		// load imag part from B
+	lxvd2x		vs20,	o0,	BO		// load real part from B
+	lxvd2x		vs21,	o16,	BO		// load imag part from B
+	lxvd2x		vs22,	o32,	BO		// load real part from B
+	lxvd2x		vs23,	o48,	BO		// load imag part from B
 
-	addi		BO,	BO,	32
+	addi		BO,	BO,	64
 
 	xvmuldp		vs32,	vs0,	vs16		// real*real, imag*real
 	xvmuldp		vs33,	vs0,	vs17		// real*imag, imag*imag
@@ -990,12 +937,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 	addi		AO,	AO,	64
 
-	lxvdsx		vs20,	o0,	BO		// load real part from B
-	lxvdsx		vs21,	o8,	BO		// load imag part from B
-	lxvdsx		vs22,	o16,	BO		// load real part from B
-	lxvdsx		vs23,	o24,	BO		// load imag part from B
+	lxvd2x		vs20,	o0,	BO		// load real part from B
+	lxvd2x		vs21,	o16,	BO		// load imag part from B
+	lxvd2x		vs22,	o32,	BO		// load real part from B
+	lxvd2x		vs23,	o48,	BO		// load imag part from B
 
-	addi		BO,	BO,	32
+	addi		BO,	BO,	64
 
 	xvmaddadp	vs32,	vs0,	vs16		// real*real, imag*real
 	xvmaddadp	vs33,	vs0,	vs17		// real*imag, imag*imag
@@ -1027,12 +974,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 	addi		AO,	AO,	64
 
-	lxvdsx		vs16,	o0,	BO		// load real part from B
-	lxvdsx		vs17,	o8,	BO		// load imag part from B
-	lxvdsx		vs18,	o16,	BO		// load real part from B
-	lxvdsx		vs19,	o24,	BO		// load imag part from B
+	lxvd2x		vs16,	o0,	BO		// load real part from B
+	lxvd2x		vs17,	o16,	BO		// load imag part from B
+	lxvd2x		vs18,	o32,	BO		// load real part from B
+	lxvd2x		vs19,	o48,	BO		// load imag part from B
 
-	addi		BO,	BO,	32
+	addi		BO,	BO,	64
 
 	xvmaddadp	vs32,	vs8,	vs20		// real*real, imag*real
 	xvmaddadp	vs33,	vs8,	vs21		// real*imag, imag*imag
@@ -1088,12 +1035,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 	addi		AO,	AO,	64
 
-	lxvdsx		vs16,	o0,	BO		// load real part from B
-	lxvdsx		vs17,	o8,	BO		// load imag part from B
-	lxvdsx		vs18,	o16,	BO		// load real part from B
-	lxvdsx		vs19,	o24,	BO		// load imag part from B
+	lxvd2x		vs16,	o0,	BO		// load real part from B
+	lxvd2x		vs17,	o16,	BO		// load imag part from B
+	lxvd2x		vs18,	o32,	BO		// load real part from B
+	lxvd2x		vs19,	o48,	BO		// load imag part from B
 
-	addi		BO,	BO,	32
+	addi		BO,	BO,	64
 
 	xvmuldp		vs32,	vs0,	vs16		// real*real, imag*real
 	xvmuldp		vs33,	vs0,	vs17		// real*imag, imag*imag
@@ -1125,12 +1072,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 	addi		AO,	AO,	64
 
-	lxvdsx		vs16,	o0,	BO		// load real part from B
-	lxvdsx		vs17,	o8,	BO		// load imag part from B
-	lxvdsx		vs18,	o16,	BO		// load real part from B
-	lxvdsx		vs19,	o24,	BO		// load imag part from B
+	lxvd2x		vs16,	o0,	BO		// load real part from B
+	lxvd2x		vs17,	o16,	BO		// load imag part from B
+	lxvd2x		vs18,	o32,	BO		// load real part from B
+	lxvd2x		vs19,	o48,	BO		// load imag part from B
 
-	addi		BO,	BO,	32
+	addi		BO,	BO,	64
 
 	xvmaddadp	vs32,	vs0,	vs16		// real*real, imag*real
 	xvmaddadp	vs33,	vs0,	vs17		// real*imag, imag*imag
@@ -1410,12 +1357,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro LOAD2x2_1
 
-	lxvdsx		vs16,	o0,	BO		// load real part from B
-	lxvdsx		vs17,	o8,	BO		// load imag part from B
-	lxvdsx		vs18,	o16,	BO		// load real part from B
-	lxvdsx		vs19,	o24,	BO		// load imag part from B
+	lxvd2x		vs16,	o0,	BO		// load real part from B
+	lxvd2x		vs17,	o16,	BO		// load imag part from B
+	lxvd2x		vs18,	o32,	BO		// load real part from B
+	lxvd2x		vs19,	o48,	BO		// load imag part from B
 
-	addi		BO,	BO,	32
+	addi		BO,	BO,	64
 
 	lxvd2x		vs0,	o0,	AO		// load real,imag from A
 	lxvd2x		vs1,	o16,	AO		// load real,imag from A
@@ -1432,12 +1379,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 	addi		AO,	AO,	32
 
-	lxvdsx		vs20,	o0,	BO		// load real part from B
-	lxvdsx		vs21,	o8,	BO		// load imag part from B
-	lxvdsx		vs22,	o16,	BO		// load real part from B
-	lxvdsx		vs23,	o24,	BO		// load imag part from B
+	lxvd2x		vs20,	o0,	BO		// load real part from B
+	lxvd2x		vs21,	o16,	BO		// load imag part from B
+	lxvd2x		vs22,	o32,	BO		// load real part from B
+	lxvd2x		vs23,	o48,	BO		// load imag part from B
 
-	addi		BO,	BO,	32
+	addi		BO,	BO,	64
 
 	xvmuldp		vs32,	vs0,	vs16		// real*real, imag*real
 	xvmuldp		vs33,	vs0,	vs17		// real*imag, imag*imag
@@ -1459,12 +1406,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 	addi		AO,	AO,	32
 
-	lxvdsx		vs20,	o0,	BO		// load real part from B
-	lxvdsx		vs21,	o8,	BO		// load imag part from B
-	lxvdsx		vs22,	o16,	BO		// load real part from B
-	lxvdsx		vs23,	o24,	BO		// load imag part from B
+	lxvd2x		vs20,	o0,	BO		// load real part from B
+	lxvd2x		vs21,	o16,	BO		// load imag part from B
+	lxvd2x		vs22,	o32,	BO		// load real part from B
+	lxvd2x		vs23,	o48,	BO		// load imag part from B
 
-	addi		BO,	BO,	32
+	addi		BO,	BO,	64
 
 	xvmaddadp	vs32,	vs0,	vs16		// real*real, imag*real
 	xvmaddadp	vs33,	vs0,	vs17		// real*imag, imag*imag
@@ -1486,12 +1433,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 	addi		AO,	AO,	32
 
-	lxvdsx		vs16,	o0,	BO		// load real part from B
-	lxvdsx		vs17,	o8,	BO		// load imag part from B
-	lxvdsx		vs18,	o16,	BO		// load real part from B
-	lxvdsx		vs19,	o24,	BO		// load imag part from B
+	lxvd2x		vs16,	o0,	BO		// load real part from B
+	lxvd2x		vs17,	o16,	BO		// load imag part from B
+	lxvd2x		vs18,	o32,	BO		// load real part from B
+	lxvd2x		vs19,	o48,	BO		// load imag part from B
 
-	addi		BO,	BO,	32
+	addi		BO,	BO,	64
 
 	xvmaddadp	vs32,	vs8,	vs20		// real*real, imag*real
 	xvmaddadp	vs33,	vs8,	vs21		// real*imag, imag*imag
@@ -1529,12 +1476,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 	addi		AO,	AO,	32
 
-	lxvdsx		vs16,	o0,	BO		// load real part from B
-	lxvdsx		vs17,	o8,	BO		// load imag part from B
-	lxvdsx		vs18,	o16,	BO		// load real part from B
-	lxvdsx		vs19,	o24,	BO		// load imag part from B
+	lxvd2x		vs16,	o0,	BO		// load real part from B
+	lxvd2x		vs17,	o16,	BO		// load imag part from B
+	lxvd2x		vs18,	o32,	BO		// load real part from B
+	lxvd2x		vs19,	o48,	BO		// load imag part from B
 
-	addi		BO,	BO,	32
+	addi		BO,	BO,	64
 
 	xvmuldp		vs32,	vs0,	vs16		// real*real, imag*real
 	xvmuldp		vs33,	vs0,	vs17		// real*imag, imag*imag
@@ -1556,12 +1503,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 	addi		AO,	AO,	32
 
-	lxvdsx		vs16,	o0,	BO		// load real part from B
-	lxvdsx		vs17,	o8,	BO		// load imag part from B
-	lxvdsx		vs18,	o16,	BO		// load real part from B
-	lxvdsx		vs19,	o24,	BO		// load imag part from B
+	lxvd2x		vs16,	o0,	BO		// load real part from B
+	lxvd2x		vs17,	o16,	BO		// load imag part from B
+	lxvd2x		vs18,	o32,	BO		// load real part from B
+	lxvd2x		vs19,	o48,	BO		// load imag part from B
 
-	addi		BO,	BO,	32
+	addi		BO,	BO,	64
 
 	xvmaddadp	vs32,	vs0,	vs16		// real*real, imag*real
 	xvmaddadp	vs33,	vs0,	vs17		// real*imag, imag*imag
@@ -1725,12 +1672,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro LOAD2x1_1
 
-	lxvdsx		vs16,	o0,	BO		// load real part from B
-	lxvdsx		vs17,	o8,	BO		// load imag part from B
-	lxvdsx		vs18,	o16,	BO		// load real part from B
-	lxvdsx		vs19,	o24,	BO		// load imag part from B
+	lxvd2x		vs16,	o0,	BO		// load real part from B
+	lxvd2x		vs17,	o16,	BO		// load imag part from B
+	lxvd2x		vs18,	o32,	BO		// load real part from B
+	lxvd2x		vs19,	o48,	BO		// load imag part from B
 
-	addi		BO,	BO,	32
+	addi		BO,	BO,	64
 
 	lxvd2x		vs0,	o0,	AO		// load real,imag from A
 
@@ -1745,12 +1692,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 	addi		AO,	AO,	16
 
-	lxvdsx		vs20,	o0,	BO		// load real part from B
-	lxvdsx		vs21,	o8,	BO		// load imag part from B
-	lxvdsx		vs22,	o16,	BO		// load real part from B
-	lxvdsx		vs23,	o24,	BO		// load imag part from B
+	lxvd2x		vs20,	o0,	BO		// load real part from B
+	lxvd2x		vs21,	o16,	BO		// load imag part from B
+	lxvd2x		vs22,	o32,	BO		// load real part from B
+	lxvd2x		vs23,	o48,	BO		// load imag part from B
 
-	addi		BO,	BO,	32
+	addi		BO,	BO,	64
 
 	xvmuldp		vs32,	vs0,	vs16		// real*real, imag*real
 	xvmuldp		vs33,	vs0,	vs17		// real*imag, imag*imag
@@ -1767,12 +1714,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 	addi		AO,	AO,	16
 
-	lxvdsx		vs20,	o0,	BO		// load real part from B
-	lxvdsx		vs21,	o8,	BO		// load imag part from B
-	lxvdsx		vs22,	o16,	BO		// load real part from B
-	lxvdsx		vs23,	o24,	BO		// load imag part from B
+	lxvd2x		vs20,	o0,	BO		// load real part from B
+	lxvd2x		vs21,	o16,	BO		// load imag part from B
+	lxvd2x		vs22,	o32,	BO		// load real part from B
+	lxvd2x		vs23,	o48,	BO		// load imag part from B
 
-	addi		BO,	BO,	32
+	addi		BO,	BO,	64
 
 	xvmaddadp	vs32,	vs0,	vs16		// real*real, imag*real
 	xvmaddadp	vs33,	vs0,	vs17		// real*imag, imag*imag
@@ -1789,12 +1736,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 	addi		AO,	AO,	16
 
-	lxvdsx		vs16,	o0,	BO		// load real part from B
-	lxvdsx		vs17,	o8,	BO		// load imag part from B
-	lxvdsx		vs18,	o16,	BO		// load real part from B
-	lxvdsx		vs19,	o24,	BO		// load imag part from B
+	lxvd2x		vs16,	o0,	BO		// load real part from B
+	lxvd2x		vs17,	o16,	BO		// load imag part from B
+	lxvd2x		vs18,	o32,	BO		// load real part from B
+	lxvd2x		vs19,	o48,	BO		// load imag part from B
 
-	addi		BO,	BO,	32
+	addi		BO,	BO,	64
 
 	xvmaddadp	vs32,	vs8,	vs20		// real*real, imag*real
 	xvmaddadp	vs33,	vs8,	vs21		// real*imag, imag*imag
@@ -1823,12 +1770,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 	addi		AO,	AO,	16
 
-	lxvdsx		vs16,	o0,	BO		// load real part from B
-	lxvdsx		vs17,	o8,	BO		// load imag part from B
-	lxvdsx		vs18,	o16,	BO		// load real part from B
-	lxvdsx		vs19,	o24,	BO		// load imag part from B
+	lxvd2x		vs16,	o0,	BO		// load real part from B
+	lxvd2x		vs17,	o16,	BO		// load imag part from B
+	lxvd2x		vs18,	o32,	BO		// load real part from B
+	lxvd2x		vs19,	o48,	BO		// load imag part from B
 
-	addi		BO,	BO,	32
+	addi		BO,	BO,	64
 
 	xvmuldp		vs32,	vs0,	vs16		// real*real, imag*real
 	xvmuldp		vs33,	vs0,	vs17		// real*imag, imag*imag
@@ -1845,12 +1792,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 	addi		AO,	AO,	16
 
-	lxvdsx		vs16,	o0,	BO		// load real part from B
-	lxvdsx		vs17,	o8,	BO		// load imag part from B
-	lxvdsx		vs18,	o16,	BO		// load real part from B
-	lxvdsx		vs19,	o24,	BO		// load imag part from B
+	lxvd2x		vs16,	o0,	BO		// load real part from B
+	lxvd2x		vs17,	o16,	BO		// load imag part from B
+	lxvd2x		vs18,	o32,	BO		// load real part from B
+	lxvd2x		vs19,	o48,	BO		// load imag part from B
 
-	addi		BO,	BO,	32
+	addi		BO,	BO,	64
 
 	xvmaddadp	vs32,	vs0,	vs16		// real*real, imag*real
 	xvmaddadp	vs33,	vs0,	vs17		// real*imag, imag*imag
@@ -1956,10 +1903,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro LOAD1x8_1
 
-	lxvdsx		vs16,	o0,	BO		// load real part from B
-	lxvdsx		vs17,	o8,	BO		// load imag part from B
+	lxvd2x		vs16,	o0,	BO		// load real part from B
+	lxvd2x		vs17,	o16,	BO		// load imag part from B
 
-	addi		BO,	BO,	16
+	addi		BO,	BO,	32
 
 	lxvd2x		vs0,	o0,	AO		// load real,imag from A
 	lxvd2x		vs1,	o16,	AO		// load real,imag from A
@@ -1994,10 +1941,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 	addi		AO,	AO,	64
 
-	lxvdsx		vs20,	o0,	BO		// load real part from B
-	lxvdsx		vs21,	o8,	BO		// load imag part from B
+	lxvd2x		vs20,	o0,	BO		// load real part from B
+	lxvd2x		vs21,	o16,	BO		// load imag part from B
 
-	addi		BO,	BO,	16
+	addi		BO,	BO,	32
 
 	xvmuldp		vs32,	vs0,	vs16		// real*real, imag*real
 	xvmuldp		vs33,	vs0,	vs17		// real*imag, imag*imag
@@ -2035,10 +1982,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 	addi		AO,	AO,	64
 
-	lxvdsx		vs20,	o0,	BO		// load real part from B
-	lxvdsx		vs21,	o8,	BO		// load imag part from B
+	lxvd2x		vs20,	o0,	BO		// load real part from B
+	lxvd2x		vs21,	o16,	BO		// load imag part from B
 
-	addi		BO,	BO,	16
+	addi		BO,	BO,	32
 
 	xvmaddadp	vs32,	vs0,	vs16		// real*real, imag*real
 	xvmaddadp	vs33,	vs0,	vs17		// real*imag, imag*imag
@@ -2076,10 +2023,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 	addi		AO,	AO,	64
 
-	lxvdsx		vs16,	o0,	BO		// load real part from B
-	lxvdsx		vs17,	o8,	BO		// load imag part from B
+	lxvd2x		vs16,	o0,	BO		// load real part from B
+	lxvd2x		vs17,	o16,	BO		// load imag part from B
 
-	addi		BO,	BO,	16
+	addi		BO,	BO,	32
 
 	xvmaddadp	vs32,	vs8,	vs20		// real*real, imag*real
 	xvmaddadp	vs33,	vs8,	vs21		// real*imag, imag*imag
@@ -2140,10 +2087,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 	addi		AO,	AO,	64
 
-	lxvdsx		vs16,	o0,	BO		// load real part from B
-	lxvdsx		vs17,	o8,	BO		// load imag part from B
+	lxvd2x		vs16,	o0,	BO		// load real part from B
+	lxvd2x		vs17,	o16,	BO		// load imag part from B
 
-	addi		BO,	BO,	16
+	addi		BO,	BO,	32
 
 	xvmuldp		vs32,	vs0,	vs16		// real*real, imag*real
 	xvmuldp		vs33,	vs0,	vs17		// real*imag, imag*imag
@@ -2181,10 +2128,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 	addi		AO,	AO,	64
 
-	lxvdsx		vs16,	o0,	BO		// load real part from B
-	lxvdsx		vs17,	o8,	BO		// load imag part from B
+	lxvd2x		vs16,	o0,	BO		// load real part from B
+	lxvd2x		vs17,	o16,	BO		// load imag part from B
 
-	addi		BO,	BO,	16
+	addi		BO,	BO,	32
 
 	xvmaddadp	vs32,	vs0,	vs16		// real*real, imag*real
 	xvmaddadp	vs33,	vs0,	vs17		// real*imag, imag*imag
@@ -2452,10 +2399,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro LOAD1x4_1
 
-	lxvdsx		vs16,	o0,	BO		// load real part from B
-	lxvdsx		vs17,	o8,	BO		// load imag part from B
+	lxvd2x		vs16,	o0,	BO		// load real part from B
+	lxvd2x		vs17,	o16,	BO		// load imag part from B
 
-	addi		BO,	BO,	16
+	addi		BO,	BO,	32
 
 	lxvd2x		vs0,	o0,	AO		// load real,imag from A
 	lxvd2x		vs1,	o16,	AO		// load real,imag from A
@@ -2476,10 +2423,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 	addi		AO,	AO,	64
 
-	lxvdsx		vs20,	o0,	BO		// load real part from B
-	lxvdsx		vs21,	o8,	BO		// load imag part from B
+	lxvd2x		vs20,	o0,	BO		// load real part from B
+	lxvd2x		vs21,	o16,	BO		// load imag part from B
 
-	addi		BO,	BO,	16
+	addi		BO,	BO,	32
 
 	xvmuldp		vs32,	vs0,	vs16		// real*real, imag*real
 	xvmuldp		vs33,	vs0,	vs17		// real*imag, imag*imag
@@ -2502,10 +2449,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 	addi		AO,	AO,	64
 
-	lxvdsx		vs20,	o0,	BO		// load real part from B
-	lxvdsx		vs21,	o8,	BO		// load imag part from B
+	lxvd2x		vs20,	o0,	BO		// load real part from B
+	lxvd2x		vs21,	o16,	BO		// load imag part from B
 
-	addi		BO,	BO,	16
+	addi		BO,	BO,	32
 
 	xvmaddadp	vs32,	vs0,	vs16		// real*real, imag*real
 	xvmaddadp	vs33,	vs0,	vs17		// real*imag, imag*imag
@@ -2528,10 +2475,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 	addi		AO,	AO,	64
 
-	lxvdsx		vs16,	o0,	BO		// load real part from B
-	lxvdsx		vs17,	o8,	BO		// load imag part from B
+	lxvd2x		vs16,	o0,	BO		// load real part from B
+	lxvd2x		vs17,	o16,	BO		// load imag part from B
 
-	addi		BO,	BO,	16
+	addi		BO,	BO,	32
 
 	xvmaddadp	vs32,	vs8,	vs20		// real*real, imag*real
 	xvmaddadp	vs33,	vs8,	vs21		// real*imag, imag*imag
@@ -2569,10 +2516,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 	addi		AO,	AO,	64
 
-	lxvdsx		vs16,	o0,	BO		// load real part from B
-	lxvdsx		vs17,	o8,	BO		// load imag part from B
+	lxvd2x		vs16,	o0,	BO		// load real part from B
+	lxvd2x		vs17,	o16,	BO		// load imag part from B
 
-	addi		BO,	BO,	16
+	addi		BO,	BO,	32
 
 	xvmuldp		vs32,	vs0,	vs16		// real*real, imag*real
 	xvmuldp		vs33,	vs0,	vs17		// real*imag, imag*imag
@@ -2595,10 +2542,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 	addi		AO,	AO,	64
 
-	lxvdsx		vs16,	o0,	BO		// load real part from B
-	lxvdsx		vs17,	o8,	BO		// load imag part from B
+	lxvd2x		vs16,	o0,	BO		// load real part from B
+	lxvd2x		vs17,	o16,	BO		// load imag part from B
 
-	addi		BO,	BO,	16
+	addi		BO,	BO,	32
 
 	xvmaddadp	vs32,	vs0,	vs16		// real*real, imag*real
 	xvmaddadp	vs33,	vs0,	vs17		// real*imag, imag*imag
@@ -2748,10 +2695,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro LOAD1x2_1
 
-	lxvdsx		vs16,	o0,	BO		// load real part from B
-	lxvdsx		vs17,	o8,	BO		// load imag part from B
+	lxvd2x		vs16,	o0,	BO		// load real part from B
+	lxvd2x		vs17,	o16,	BO		// load imag part from B
 
-	addi		BO,	BO,	16
+	addi		BO,	BO,	32
 
 	lxvd2x		vs0,	o0,	AO		// load real,imag from A
 	lxvd2x		vs1,	o16,	AO		// load real,imag from A
@@ -2768,10 +2715,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 	addi		AO,	AO,	32
 
-	lxvdsx		vs20,	o0,	BO		// load real part from B
-	lxvdsx		vs21,	o8,	BO		// load imag part from B
+	lxvd2x		vs20,	o0,	BO		// load real part from B
+	lxvd2x		vs21,	o16,	BO		// load imag part from B
 
-	addi		BO,	BO,	16
+	addi		BO,	BO,	32
 
 	xvmuldp		vs32,	vs0,	vs16		// real*real, imag*real
 	xvmuldp		vs33,	vs0,	vs17		// real*imag, imag*imag
@@ -2788,10 +2735,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 	addi		AO,	AO,	32
 
-	lxvdsx		vs20,	o0,	BO		// load real part from B
-	lxvdsx		vs21,	o8,	BO		// load imag part from B
+	lxvd2x		vs20,	o0,	BO		// load real part from B
+	lxvd2x		vs21,	o16,	BO		// load imag part from B
 
-	addi		BO,	BO,	16
+	addi		BO,	BO,	32
 
 	xvmaddadp	vs32,	vs0,	vs16		// real*real, imag*real
 	xvmaddadp	vs33,	vs0,	vs17		// real*imag, imag*imag
@@ -2808,10 +2755,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 	addi		AO,	AO,	32
 
-	lxvdsx		vs16,	o0,	BO		// load real part from B
-	lxvdsx		vs17,	o8,	BO		// load imag part from B
+	lxvd2x		vs16,	o0,	BO		// load real part from B
+	lxvd2x		vs17,	o16,	BO		// load imag part from B
 
-	addi		BO,	BO,	16
+	addi		BO,	BO,	32
 
 	xvmaddadp	vs32,	vs8,	vs20		// real*real, imag*real
 	xvmaddadp	vs33,	vs8,	vs21		// real*imag, imag*imag
@@ -2839,10 +2786,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 	addi		AO,	AO,	32
 
-	lxvdsx		vs16,	o0,	BO		// load real part from B
-	lxvdsx		vs17,	o8,	BO		// load imag part from B
+	lxvd2x		vs16,	o0,	BO		// load real part from B
+	lxvd2x		vs17,	o16,	BO		// load imag part from B
 
-	addi		BO,	BO,	16
+	addi		BO,	BO,	32
 
 	xvmuldp		vs32,	vs0,	vs16		// real*real, imag*real
 	xvmuldp		vs33,	vs0,	vs17		// real*imag, imag*imag
@@ -2859,10 +2806,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 	addi		AO,	AO,	32
 
-	lxvdsx		vs16,	o0,	BO		// load real part from B
-	lxvdsx		vs17,	o8,	BO		// load imag part from B
+	lxvd2x		vs16,	o0,	BO		// load real part from B
+	lxvd2x		vs17,	o16,	BO		// load imag part from B
 
-	addi		BO,	BO,	16
+	addi		BO,	BO,	32
 
 	xvmaddadp	vs32,	vs0,	vs16		// real*real, imag*real
 	xvmaddadp	vs33,	vs0,	vs17		// real*imag, imag*imag
@@ -2954,10 +2901,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 .macro LOAD1x1_1
 
-	lxvdsx		vs16,	o0,	BO		// load real part from B
-	lxvdsx		vs17,	o8,	BO		// load imag part from B
+	lxvd2x		vs16,	o0,	BO		// load real part from B
+	lxvd2x		vs17,	o16,	BO		// load imag part from B
 
-	addi		BO,	BO,	16
+	addi		BO,	BO,	32
 
 	lxvd2x		vs0,	o0,	AO		// load real,imag from A
 
@@ -2972,10 +2919,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 	addi		AO,	AO,	16
 
-	lxvdsx		vs20,	o0,	BO		// load real part from B
-	lxvdsx		vs21,	o8,	BO		// load imag part from B
+	lxvd2x		vs20,	o0,	BO		// load real part from B
+	lxvd2x		vs21,	o16,	BO		// load imag part from B
 
-	addi		BO,	BO,	16
+	addi		BO,	BO,	32
 
 	xvmuldp		vs32,	vs0,	vs16		// real*real, imag*real
 	xvmuldp		vs33,	vs0,	vs17		// real*imag, imag*imag
@@ -2989,10 +2936,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 	addi		AO,	AO,	16
 
-	lxvdsx		vs20,	o0,	BO		// load real part from B
-	lxvdsx		vs21,	o8,	BO		// load imag part from B
+	lxvd2x		vs20,	o0,	BO		// load real part from B
+	lxvd2x		vs21,	o16,	BO		// load imag part from B
 
-	addi		BO,	BO,	16
+	addi		BO,	BO,	32
 
 	xvmaddadp	vs32,	vs0,	vs16		// real*real, imag*real
 	xvmaddadp	vs33,	vs0,	vs17		// real*imag, imag*imag
@@ -3006,10 +2953,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 	addi		AO,	AO,	16
 
-	lxvdsx		vs16,	o0,	BO		// load real part from B
-	lxvdsx		vs17,	o8,	BO		// load imag part from B
+	lxvd2x		vs16,	o0,	BO		// load real part from B
+	lxvd2x		vs17,	o16,	BO		// load imag part from B
 
-	addi		BO,	BO,	16
+	addi		BO,	BO,	32
 
 	xvmaddadp	vs32,	vs8,	vs20		// real*real, imag*real
 	xvmaddadp	vs33,	vs8,	vs21		// real*imag, imag*imag
@@ -3032,10 +2979,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 	addi		AO,	AO,	16
 
-	lxvdsx		vs16,	o0,	BO		// load real part from B
-	lxvdsx		vs17,	o8,	BO		// load imag part from B
+	lxvd2x		vs16,	o0,	BO		// load real part from B
+	lxvd2x		vs17,	o16,	BO		// load imag part from B
 
-	addi		BO,	BO,	16
+	addi		BO,	BO,	32
 
 	xvmuldp		vs32,	vs0,	vs16		// real*real, imag*real
 	xvmuldp		vs33,	vs0,	vs17		// real*imag, imag*imag
@@ -3049,10 +2996,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 	addi		AO,	AO,	16
 
-	lxvdsx		vs16,	o0,	BO		// load real part from B
-	lxvdsx		vs17,	o8,	BO		// load imag part from B
+	lxvd2x		vs16,	o0,	BO		// load real part from B
+	lxvd2x		vs17,	o16,	BO		// load imag part from B
 
-	addi		BO,	BO,	16
+	addi		BO,	BO,	32
 
 	xvmaddadp	vs32,	vs0,	vs16		// real*real, imag*real
 	xvmaddadp	vs33,	vs0,	vs17		// real*imag, imag*imag

kernel/power/zscal.c

@@ -0,0 +1,176 @@
+/***************************************************************************
+Copyright (c) 2013-2016, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+/**************************************************************************************
+* 2016/03/27 Werner Saar (wernsaar@googlemail.com)
+* 	 BLASTEST 		: OK
+* 	 CTEST			: OK
+* 	 TEST			: OK
+*	 LAPACK-TEST		: OK
+**************************************************************************************/
+
+
+#include "common.h"
+
+#pragma GCC optimize "O1"
+
+#if defined(POWER8)
+#include "zscal_microk_power8.c"
+#endif
+
+
+#ifndef HAVE_KERNEL_8
+
+static void zscal_kernel_8(BLASLONG n, FLOAT *x, FLOAT *alpha)
+{
+
+	BLASLONG i=0;
+	FLOAT *x1=x;
+	FLOAT  alpha_r1=alpha[0];
+	FLOAT  alpha_r2=alpha[1];
+	FLOAT  alpha_i1=alpha[2];
+	FLOAT  alpha_i2=alpha[3];
+	FLOAT  temp00, temp01, temp10, temp11, temp20, temp21, temp30, temp31;
+	FLOAT  x0_r, x0_i, x1_r, x1_i, x2_r, x2_i, x3_r, x3_i;
+
+	while ( i<n )
+	{
+		x0_r = x1[0];
+		x0_i = x1[1];
+		x1_r = x1[2];
+		x1_i = x1[3];
+		x2_r = x1[4];
+		x2_i = x1[5];
+		x3_r = x1[6];
+		x3_i = x1[7];
+
+		temp00  = x0_r * alpha_r1;
+		temp10  = x1_r * alpha_r1;
+		temp20  = x2_r * alpha_r1;
+		temp30  = x3_r * alpha_r1;
+
+		temp01  = x0_i * alpha_r2;
+		temp11  = x1_i * alpha_r2;
+		temp21  = x2_i * alpha_r2;
+		temp31  = x3_i * alpha_r2;
+
+		temp00 += x0_i * alpha_i1;
+		temp10 += x1_i * alpha_i1;
+		temp20 += x2_i * alpha_i1;
+		temp30 += x3_i * alpha_i1;
+
+		temp01 += x0_r * alpha_i2;
+		temp11 += x1_r * alpha_i2;
+		temp21 += x2_r * alpha_i2;
+		temp31 += x3_r * alpha_i2;
+
+		x1[0] = temp00;
+		x1[1] = temp01;
+		x1[2] = temp10;
+		x1[3] = temp11;
+		x1[4] = temp20;
+		x1[5] = temp21;
+		x1[6] = temp30;
+		x1[7] = temp31;
+
+		x1 += 8;
+		i+=4;
+
+	}
+	return;
+
+
+}
+
+#endif
+
+int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da_r,FLOAT da_i, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y, FLOAT *dummy, BLASLONG dummy2)
+{
+	BLASLONG i=0;
+	BLASLONG inc_x2;
+	BLASLONG ip = 0;
+	FLOAT temp;
+	FLOAT alpha[4] __attribute__ ((aligned (16)));;
+	BLASLONG n1;
+
+	if ( n <= 0 )
+		return(0);
+
+	if ( inc_x <= 0 )
+		return(0);
+
+	if ( inc_x == 1 )
+	{
+
+
+		n1 = n & -8;
+		if ( n1 > 0 )
+		{
+			alpha[0] = da_r;
+			alpha[1] = da_r;
+			alpha[2] = -da_i;
+			alpha[3] = da_i;
+			zscal_kernel_8(n1, x, alpha);
+			i=n1;
+			ip = n1 * 2;
+
+		}
+
+		while ( i < n )
+		{
+
+				temp    = da_r * x[ip]   - da_i * x[ip+1] ;
+				x[ip+1] = da_r * x[ip+1] + da_i * x[ip]   ;
+				x[ip]   = temp;
+				ip += 2;
+				i++;
+		}
+
+	}
+	else
+	{
+
+		inc_x2 = 2 * inc_x;
+
+		while ( i < n )
+		{
+
+				temp    = da_r * x[ip]   - da_i * x[ip+1] ;
+				x[ip+1] = da_r * x[ip+1] + da_i * x[ip]   ;
+				x[ip]   = temp;
+				ip += inc_x2;
+				i++;
+		}
+
+
+	}
+
+	return(0);
+
+}
+
+

kernel/power/zscal_microk_power8.c

@@ -0,0 +1,224 @@
+/***************************************************************************
+Copyright (c) 2013-2016, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+/**************************************************************************************
+* 2016/03/25 Werner Saar (wernsaar@googlemail.com)
+*
+* I don't use fused multipy-add ( lapack precision problems )
+*
+* 	 BLASTEST 		: OK
+* 	 CTEST			: OK
+* 	 TEST			: OK
+*	 LAPACK-TEST		: OK
+**************************************************************************************/
+
+#define HAVE_KERNEL_8 1
+
+static void zscal_kernel_8( BLASLONG n, FLOAT *x, FLOAT *alpha) __attribute__ ((noinline));
+
+static void zscal_kernel_8( BLASLONG n, FLOAT *x, FLOAT *alpha)
+{
+
+
+	BLASLONG i = n;
+	BLASLONG o16 = 16;
+	BLASLONG o32 = 32;
+	BLASLONG o48 = 48;
+	BLASLONG o64 = 64;
+	BLASLONG o80 = 80;
+	BLASLONG o96 = 96;
+	BLASLONG o112 = 112;
+	FLOAT *x1=x;
+	FLOAT *x2=x+1;
+	BLASLONG pre = 384;
+
+	__asm__  __volatile__
+	(
+
+        "lxvd2x         32, 0, %3                           \n\t"	// alpha_r , alpha_r
+        "lxvd2x         33, %5, %3                          \n\t"	// -alpha_i , alpha_i
+        "addi           %1, %1, -8                          \n\t"
+
+	"dcbt		%2, %4				    \n\t"
+
+	"lxvd2x		40, 0, %2			    \n\t"	// x0_r, x0_i
+	"lxvd2x		41, %5, %2			    \n\t"
+	"lxvd2x		42, %6, %2			    \n\t"
+	"lxvd2x		43, %7, %2			    \n\t"
+	"lxvd2x		44, %8, %2			    \n\t"
+	"lxvd2x		45, %9, %2			    \n\t"
+	"lxvd2x		46, %10, %2			    \n\t"
+	"lxvd2x		47, %11, %2			    \n\t"
+
+	"addi		%2, %2, 128			    \n\t"
+
+	"addic.		%0 , %0	, -8  	 	             \n\t"
+	"ble		2f		             	     \n\t"
+
+	".align 5				            \n\t"
+	"1:				                    \n\t"
+
+	"dcbt		%2, %4				    \n\t"
+
+	"xvmuldp	48, 40, 32		    	    \n\t"	// x0_r * alpha_r, x0_i * alpha_r
+	"xvmuldp	49, 41, 32		    	    \n\t"
+	"xvmuldp	50, 42, 32		    	    \n\t"
+	"xvmuldp	51, 43, 32		    	    \n\t"
+	"xvmuldp	52, 44, 32		    	    \n\t"
+	"xvmuldp	53, 45, 32		    	    \n\t"
+	"xvmuldp	54, 46, 32		    	    \n\t"
+	"xvmuldp	55, 47, 32		    	    \n\t"
+
+	"xxswapd	56, 40				    \n\t"
+	"xxswapd	57, 41				    \n\t"
+	"xxswapd	58, 42				    \n\t"
+	"xxswapd	59, 43				    \n\t"
+	"xxswapd	60, 44				    \n\t"
+	"xxswapd	61, 45				    \n\t"
+	"xxswapd	62, 46				    \n\t"
+	"xxswapd	63, 47				    \n\t"
+
+	"xvmuldp	56, 56, 33		    	    \n\t"	// x0_i * -alpha_i, x0_r * alpha_i
+	"xvmuldp	57, 57, 33		    	    \n\t"
+
+	"lxvd2x		40, 0, %2			    \n\t"	// x0_r, x0_i
+	"lxvd2x		41, %5, %2			    \n\t"
+
+	"xvmuldp	58, 58, 33		    	    \n\t"
+	"xvmuldp	59, 59, 33		    	    \n\t"
+
+	"lxvd2x		42, %6, %2			    \n\t"
+	"lxvd2x		43, %7, %2			    \n\t"
+
+	"xvmuldp	60, 60, 33		    	    \n\t"
+	"xvmuldp	61, 61, 33		    	    \n\t"
+
+	"lxvd2x		44, %8, %2			    \n\t"
+	"lxvd2x		45, %9, %2			    \n\t"
+
+	"xvmuldp	62, 62, 33		    	    \n\t"
+	"xvmuldp	63, 63, 33		    	    \n\t"
+
+	"lxvd2x		46, %10, %2			    \n\t"
+	"lxvd2x		47, %11, %2			    \n\t"
+
+	"xvadddp	48, 48 , 56			    \n\t"
+	"xvadddp	49, 49 , 57			    \n\t"
+	"xvadddp	50, 50 , 58			    \n\t"
+	"xvadddp	51, 51 , 59			    \n\t"
+
+	"stxvd2x	48, 0, %1			    \n\t"
+	"stxvd2x	49, %5, %1			    \n\t"
+
+	"xvadddp	52, 52 , 60			    \n\t"
+	"xvadddp	53, 53 , 61			    \n\t"
+
+	"stxvd2x	50, %6, %1			    \n\t"
+	"stxvd2x	51, %7, %1			    \n\t"
+
+	"xvadddp	54, 54 , 62			    \n\t"
+	"xvadddp	55, 55 , 63			    \n\t"
+
+	"stxvd2x	52, %8, %1			    \n\t"
+	"stxvd2x	53, %9, %1			    \n\t"
+	"stxvd2x	54, %10, %1			    \n\t"
+	"stxvd2x	55, %11, %1			    \n\t"
+
+	"addi		%1, %1, 128			    \n\t"
+	"addi		%2, %2, 128			    \n\t"
+
+	"addic.		%0 , %0	, -8  	 	             \n\t"
+	"bgt		1b		             	     \n\t"
+
+	"2:						     \n\t"
+
+	"xvmuldp	48, 40, 32		    	    \n\t"	// x0_r * alpha_r, x0_i * alpha_r
+	"xvmuldp	49, 41, 32		    	    \n\t"
+	"xvmuldp	50, 42, 32		    	    \n\t"
+	"xvmuldp	51, 43, 32		    	    \n\t"
+	"xvmuldp	52, 44, 32		    	    \n\t"
+	"xvmuldp	53, 45, 32		    	    \n\t"
+	"xvmuldp	54, 46, 32		    	    \n\t"
+	"xvmuldp	55, 47, 32		    	    \n\t"
+
+	"xxswapd	56, 40				    \n\t"
+	"xxswapd	57, 41				    \n\t"
+	"xxswapd	58, 42				    \n\t"
+	"xxswapd	59, 43				    \n\t"
+	"xxswapd	60, 44				    \n\t"
+	"xxswapd	61, 45				    \n\t"
+	"xxswapd	62, 46				    \n\t"
+	"xxswapd	63, 47				    \n\t"
+
+	"xvmuldp	56, 56, 33		    	    \n\t"	// x0_i * -alpha_i, x0_r * alpha_i
+	"xvmuldp	57, 57, 33		    	    \n\t"
+	"xvmuldp	58, 58, 33		    	    \n\t"
+	"xvmuldp	59, 59, 33		    	    \n\t"
+	"xvmuldp	60, 60, 33		    	    \n\t"
+	"xvmuldp	61, 61, 33		    	    \n\t"
+	"xvmuldp	62, 62, 33		    	    \n\t"
+	"xvmuldp	63, 63, 33		    	    \n\t"
+
+	"xvadddp	48, 48 , 56			    \n\t"
+	"xvadddp	49, 49 , 57			    \n\t"
+	"xvadddp	50, 50 , 58			    \n\t"
+	"xvadddp	51, 51 , 59			    \n\t"
+	"xvadddp	52, 52 , 60			    \n\t"
+	"xvadddp	53, 53 , 61			    \n\t"
+	"xvadddp	54, 54 , 62			    \n\t"
+	"xvadddp	55, 55 , 63			    \n\t"
+
+	"stxvd2x	48, 0, %1			    \n\t"
+	"stxvd2x	49, %5, %1			    \n\t"
+	"stxvd2x	50, %6, %1			    \n\t"
+	"stxvd2x	51, %7, %1			    \n\t"
+	"stxvd2x	52, %8, %1			    \n\t"
+	"stxvd2x	53, %9, %1			    \n\t"
+	"stxvd2x	54, %10, %1			    \n\t"
+	"stxvd2x	55, %11, %1			    \n\t"
+
+
+	:
+        : 
+          "r" (i),	// 0	
+	  "r" (x2),  	// 1
+          "r" (x1),     // 2
+          "r" (alpha),  // 3
+          "r" (pre),    // 4
+	  "r" (o16),	// 5
+	  "r" (o32),	// 6
+	  "r" (o48),    // 7
+          "r" (o64),    // 8
+          "r" (o80),    // 9
+          "r" (o96),    // 10
+          "r" (o112)    // 11
+	: "cr0", "%0", "%2" , "%1", "memory"
+	);
+
+} 
+
+

kernel/power/zswap.c

@@ -0,0 +1,175 @@
+/***************************************************************************
+Copyright (c) 2013-2016, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+/**************************************************************************************
+* 2016/03/27 Werner Saar (wernsaar@googlemail.com)
+* 	 BLASTEST 		: OK
+* 	 CTEST			: OK
+* 	 TEST			: OK
+*	 LAPACK-TEST		: OK
+**************************************************************************************/
+
+#include "common.h"
+
+
+#if defined(POWER8)
+#include "zswap_microk_power8.c"
+#endif
+
+
+#ifndef HAVE_KERNEL_16
+
+static void zswap_kernel_16(BLASLONG n, FLOAT *x, FLOAT *y)
+{
+
+	BLASLONG i=0;
+	FLOAT f0, f1, f2, f3, f4, f5, f6, f7;
+	FLOAT g0, g1, g2, g3, g4, g5, g6, g7;
+	FLOAT *x1=x;
+	FLOAT *y1=y;
+
+	while ( i<n )
+	{
+
+		f0 = x1[0];
+		f1 = x1[1];
+		f2 = x1[2];
+		f3 = x1[3];
+		f4 = x1[4];
+		f5 = x1[5];
+		f6 = x1[6];
+		f7 = x1[7];
+
+		g0 = y1[0];
+		g1 = y1[1];
+		g2 = y1[2];
+		g3 = y1[3];
+		g4 = y1[4];
+		g5 = y1[5];
+		g6 = y1[6];
+		g7 = y1[7];
+
+		y1[0] = f0;
+		y1[1] = f1;
+		y1[2] = f2;
+		y1[3] = f3;
+		y1[4] = f4;
+		y1[5] = f5;
+		y1[6] = f6;
+		y1[7] = f7;
+
+		x1[0] = g0;
+		x1[1] = g1;
+		x1[2] = g2;
+		x1[3] = g3;
+		x1[4] = g4;
+		x1[5] = g5;
+		x1[6] = g6;
+		x1[7] = g7;
+
+		x1 += 8;
+		y1 += 8;
+
+		i+=4;
+	}
+	return;
+
+}
+
+
+#endif
+
+
+int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT dummy3, FLOAT dummy4, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y, FLOAT *dummy, BLASLONG dummy2)
+{
+	BLASLONG i=0;
+	BLASLONG ix=0,iy=0;
+	FLOAT temp[2];
+	BLASLONG inc_x2, inc_y2;
+
+	if ( n <= 0     )  return(0);
+
+	if ( (inc_x == 1) && (inc_y == 1 ))
+	{
+
+		BLASLONG n1 = n & -16;
+		if ( n1 > 0 )
+		{
+			zswap_kernel_16(n1, x, y);
+			i=n1;
+			ix = 2* n1;
+			iy = 2* n1;
+		}
+
+		while(i < n)
+		{
+
+	                temp[0]  = x[ix]   ;
+        	        temp[1]  = x[ix+1] ;
+                	x[ix]    = y[iy]   ;
+                	x[ix+1]  = y[iy+1] ;
+                	y[iy]    = temp[0] ;
+                	y[iy+1]  = temp[1] ;
+
+                	ix += 2 ;
+                	iy += 2 ;
+                	i++ ;
+
+
+		}
+
+
+	}
+	else
+	{
+
+	        inc_x2 = 2 * inc_x;
+	        inc_y2 = 2 * inc_y;
+
+		while(i < n)
+		{
+
+	                temp[0]  = x[ix]   ;
+        	        temp[1]  = x[ix+1] ;
+                	x[ix]    = y[iy]   ;
+                	x[ix+1]  = y[iy+1] ;
+                	y[iy]    = temp[0] ;
+                	y[iy+1]  = temp[1] ;
+
+                	ix += inc_x2 ;
+                	iy += inc_y2 ;
+                	i++ ;
+
+		}
+
+	}
+	return(0);
+	
+
+}
+
+

kernel/power/zswap_microk_power8.c

@@ -0,0 +1,180 @@
+/***************************************************************************
+Copyright (c) 2013-2016, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+/**************************************************************************************
+* 2016/03/27 Werner Saar (wernsaar@googlemail.com)
+* 	 BLASTEST 		: OK
+* 	 CTEST			: OK
+* 	 TEST			: OK
+*	 LAPACK-TEST		: OK
+**************************************************************************************/
+
+#define HAVE_KERNEL_16 1
+
+static void zswap_kernel_16( BLASLONG n, FLOAT *x, FLOAT *y) __attribute__ ((noinline));
+
+static void zswap_kernel_16( BLASLONG n, FLOAT *x, FLOAT *y)
+{
+
+
+	BLASLONG i = n;
+	BLASLONG o16 = 16;
+	BLASLONG o32 = 32;
+	BLASLONG o48 = 48;
+	BLASLONG o64 = 64;
+	BLASLONG o80 = 80;
+	BLASLONG o96 = 96;
+	BLASLONG o112 = 112;
+	FLOAT *x1=x;
+	FLOAT *y1=y;
+	FLOAT *x2=x+1;
+	FLOAT *y2=y+1;
+	BLASLONG pre = 384;
+	BLASLONG alpha=0;
+
+	__asm__  __volatile__
+	(
+
+	"addi		%3, %3, -8			    \n\t"	
+	"addi		%4, %4, -8			    \n\t"	
+
+	".align 5				            \n\t"
+	"1:				                    \n\t"
+
+	"lxvd2x		32, 0, %2			    \n\t"
+	"lxvd2x		33, %5, %2			    \n\t"
+	"lxvd2x		34, %6, %2			    \n\t"
+	"lxvd2x		35, %7, %2			    \n\t"
+	"lxvd2x		36, %8, %2			    \n\t"
+	"lxvd2x		37, %9, %2			    \n\t"
+	"lxvd2x		38, %10, %2			    \n\t"
+	"lxvd2x		39, %11, %2			    \n\t"
+
+	"addi		%2, %2, 128			    \n\t"
+
+	"lxvd2x		40, 0, %2			    \n\t"
+	"lxvd2x		41, %5, %2			    \n\t"
+	"lxvd2x		42, %6, %2			    \n\t"
+	"lxvd2x		43, %7, %2			    \n\t"
+	"lxvd2x		44, %8, %2			    \n\t"
+	"lxvd2x		45, %9, %2			    \n\t"
+	"lxvd2x		46, %10, %2			    \n\t"
+	"lxvd2x		47, %11, %2			    \n\t"
+
+	"addi		%2, %2, 128			    \n\t"
+
+	"lxvd2x		48, 0, %1			    \n\t"
+	"lxvd2x		49, %5, %1			    \n\t"
+	"lxvd2x		50, %6, %1			    \n\t"
+	"lxvd2x		51, %7, %1			    \n\t"
+	"lxvd2x		52, %8, %1			    \n\t"
+	"lxvd2x		53, %9, %1			    \n\t"
+	"lxvd2x		54, %10, %1			    \n\t"
+	"lxvd2x		55, %11, %1			    \n\t"
+
+	"addi		%1, %1, 128			    \n\t"
+
+	"lxvd2x		56, 0, %1			    \n\t"
+	"lxvd2x		57, %5, %1			    \n\t"
+	"lxvd2x		58, %6, %1			    \n\t"
+	"lxvd2x		59, %7, %1			    \n\t"
+	"lxvd2x		60, %8, %1			    \n\t"
+	"lxvd2x		61, %9, %1			    \n\t"
+	"lxvd2x		62, %10, %1			    \n\t"
+	"lxvd2x		63, %11, %1			    \n\t"
+
+	"addi		%1, %1, 128			    \n\t"
+
+	"stxvd2x		32, 0, %3			    \n\t"
+	"stxvd2x		33, %5, %3			    \n\t"
+	"stxvd2x		34, %6, %3			    \n\t"
+	"stxvd2x		35, %7, %3			    \n\t"
+	"stxvd2x		36, %8, %3			    \n\t"
+	"stxvd2x		37, %9, %3			    \n\t"
+	"stxvd2x		38, %10, %3			    \n\t"
+	"stxvd2x		39, %11, %3			    \n\t"
+
+	"addi		%3, %3, 128			    \n\t"
+
+	"stxvd2x		40, 0, %3			    \n\t"
+	"stxvd2x		41, %5, %3			    \n\t"
+	"stxvd2x		42, %6, %3			    \n\t"
+	"stxvd2x		43, %7, %3			    \n\t"
+	"stxvd2x		44, %8, %3			    \n\t"
+	"stxvd2x		45, %9, %3			    \n\t"
+	"stxvd2x		46, %10, %3			    \n\t"
+	"stxvd2x		47, %11, %3			    \n\t"
+
+	"addi		%3, %3, 128			    \n\t"
+
+	"stxvd2x		48, 0, %4			    \n\t"
+	"stxvd2x		49, %5, %4			    \n\t"
+	"stxvd2x		50, %6, %4			    \n\t"
+	"stxvd2x		51, %7, %4			    \n\t"
+	"stxvd2x		52, %8, %4			    \n\t"
+	"stxvd2x		53, %9, %4			    \n\t"
+	"stxvd2x		54, %10, %4			    \n\t"
+	"stxvd2x		55, %11, %4			    \n\t"
+
+	"addi		%4, %4, 128			    \n\t"
+
+	"stxvd2x		56, 0, %4			    \n\t"
+	"stxvd2x		57, %5, %4			    \n\t"
+	"stxvd2x		58, %6, %4			    \n\t"
+	"stxvd2x		59, %7, %4			    \n\t"
+	"stxvd2x		60, %8, %4			    \n\t"
+	"stxvd2x		61, %9, %4			    \n\t"
+	"stxvd2x		62, %10, %4			    \n\t"
+	"stxvd2x		63, %11, %4			    \n\t"
+
+	"addi		%4, %4, 128			    \n\t"
+
+	"addic.		%0 , %0	, -16  	 	             \n\t"
+	"bgt		1b		             	     \n\t"
+
+	"2:						     \n\t"
+
+	:
+        : 
+          "r" (i),	// 0	
+	  "r" (y1),  	// 1
+          "r" (x1),     // 2
+          "r" (y2),     // 3
+          "r" (x2),     // 4
+	  "r" (o16),	// 5
+	  "r" (o32),	// 6
+	  "r" (o48),    // 7
+          "r" (o64),    // 8
+          "r" (o80),    // 9
+          "r" (o96),    // 10
+          "r" (o112)    // 11
+	: "cr0", "%0", "%2" , "%1", "%3", "%4", "memory"
+	);
+
+} 
+
+

kernel/power/ztrmm_kernel_8x2_power8.S

@@ -271,7 +271,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #endif
 #endif
 
-#include "zgemm_macros_8x2_power8.S"
+#include "ztrmm_macros_8x2_power8.S"
 
 	cmpwi	cr0, M, 0
 	ble	.L999

kernel/x86_64/KERNEL.STEAMROLLER

@@ -24,7 +24,7 @@ SGEMVTKERNEL = sgemv_t_4.c
 DGEMVNKERNEL = dgemv_n_4.c
 DGEMVTKERNEL = dgemv_t_4.c
 
-ZGEMVNKERNEL = zgemv_t_4.c
+ZGEMVNKERNEL = zgemv_n_4.c
 ZGEMVTKERNEL = zgemv_t_4.c
 
 DCOPYKERNEL  = dcopy_bulldozer.S

kernel/x86_64/sdot.c

@@ -72,18 +72,20 @@ FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y)
 {
 	BLASLONG i=0;
 	BLASLONG ix=0,iy=0;
+	double dot = 0.0 ;
 
-	FLOAT  dot = 0.0 ;
+	FLOAT mydot=0.0;
+	BLASLONG n1;
 
 	if ( n <= 0 )  return(dot);
 
 	if ( (inc_x == 1) && (inc_y == 1) )
 	{
 
-		BLASLONG n1 = n & -32;
+	        n1 = n & (BLASLONG)(-32);
 
 		if ( n1 )
-			sdot_kernel_16(n1, x, y , &dot );
+			sdot_kernel_16(n1, x, y , &mydot );
 
 
 		i = n1;
@@ -94,12 +96,13 @@ FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y)
 			i++ ;
 
 		}
+		dot+=mydot;
 		return(dot);
 
 
 	}
 
-	BLASLONG n1 = n & -2;
+	n1 = n & (BLASLONG)(-2);
 
 	while(i < n1)
 	{
@@ -124,4 +127,3 @@ FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y)
 
 }
 
-

param.h

@@ -1961,35 +1961,36 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #if defined(POWER8)
 
-#define SNUMOPT		4
+#define SNUMOPT		16
 #define DNUMOPT		8
 
-#define GEMM_DEFAULT_OFFSET_A  384
-#define GEMM_DEFAULT_OFFSET_B 1024
+#define GEMM_DEFAULT_OFFSET_A 4096 
+#define GEMM_DEFAULT_OFFSET_B 4096
 #define GEMM_DEFAULT_ALIGN 0x03fffUL
 
-#define SGEMM_DEFAULT_UNROLL_M 4
-#define SGEMM_DEFAULT_UNROLL_N 4
+#define SGEMM_DEFAULT_UNROLL_M 16
+#define SGEMM_DEFAULT_UNROLL_N 8
 #define DGEMM_DEFAULT_UNROLL_M 16
 #define DGEMM_DEFAULT_UNROLL_N 4
-#define CGEMM_DEFAULT_UNROLL_M 2
-#define CGEMM_DEFAULT_UNROLL_N 2
+#define CGEMM_DEFAULT_UNROLL_M 8
+#define CGEMM_DEFAULT_UNROLL_N 4
 #define ZGEMM_DEFAULT_UNROLL_M 8
 #define ZGEMM_DEFAULT_UNROLL_N 2
 
-#define SGEMM_DEFAULT_P  992
+#define SGEMM_DEFAULT_P  960
 #define DGEMM_DEFAULT_P  480
-#define CGEMM_DEFAULT_P  488
-#define ZGEMM_DEFAULT_P  240
+#define CGEMM_DEFAULT_P  720
+#define ZGEMM_DEFAULT_P  480
 
-#define SGEMM_DEFAULT_Q  504
+#define SGEMM_DEFAULT_Q  720
 #define DGEMM_DEFAULT_Q  720
-#define CGEMM_DEFAULT_Q  400
-#define ZGEMM_DEFAULT_Q  360
+#define CGEMM_DEFAULT_Q  720
+#define ZGEMM_DEFAULT_Q  720
 
-#define SGEMM_DEFAULT_R 28800
+#define SGEMM_DEFAULT_R 21600
 #define DGEMM_DEFAULT_R 14400
-#define ZGEMM_DEFAULT_R 7200
+#define CGEMM_DEFAULT_R 16200
+#define ZGEMM_DEFAULT_R 21600
 
 #define SYMV_P	 8