Merge pull request #3793 from imzhuhl/new_sbgemm

New sbgemm implementation for Neoverse N2

common_param.h

@@ -50,6 +50,7 @@ typedef struct {
 #ifdef BUILD_BFLOAT16
   int sbgemm_p, sbgemm_q, sbgemm_r;
   int sbgemm_unroll_m, sbgemm_unroll_n, sbgemm_unroll_mn;
+  int sbgemm_align_k;
 
   void   (*sbstobf16_k) (BLASLONG, float    *, BLASLONG, bfloat16 *, BLASLONG);
   void   (*sbdtobf16_k) (BLASLONG, double   *, BLASLONG, bfloat16 *, BLASLONG);

driver/level3/level3.c

@@ -304,6 +304,15 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n,
 	while (gemm_p * min_l > l2size) gemm_p -= GEMM_UNROLL_M;
       }
 
+      BLASLONG pad_min_l = min_l;
+#if defined(HALF)
+#if defined(DYNAMIC_ARCH)
+      pad_min_l = (min_l + gotoblas->sbgemm_align_k - 1) & ~(gotoblas->sbgemm_align_k-1);
+#else
+      pad_min_l = (min_l + SBGEMM_ALIGN_K - 1) & ~(SBGEMM_ALIGN_K - 1);;
+#endif
+#endif
+
       /* First, we have to move data A to L2 cache */
       min_i = m_to - m_from;
       l1stride = 1;
@@ -350,7 +359,7 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n,
 	START_RPCC();
 
 	OCOPY_OPERATION(min_l, min_jj, b, ldb, ls, jjs,
-			sb + min_l * (jjs - js) * COMPSIZE * l1stride);
+			sb + pad_min_l * (jjs - js) * COMPSIZE * l1stride);
 
 	STOP_RPCC(outercost);
 
@@ -358,10 +367,10 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n,
 
 #if !defined(XDOUBLE)  || !defined(QUAD_PRECISION)
 	KERNEL_OPERATION(min_i, min_jj, min_l, alpha,
-			 sa, sb + min_l * (jjs - js)  * COMPSIZE * l1stride, c, ldc, m_from, jjs);
+			 sa, sb + pad_min_l * (jjs - js)  * COMPSIZE * l1stride, c, ldc, m_from, jjs);
 #else
 	KERNEL_OPERATION(min_i, min_jj, min_l, (void *)&xalpha,
-			 sa, sb + min_l * (jjs - js)  * COMPSIZE * l1stride, c, ldc, m_from, jjs);
+			 sa, sb + pad_min_l * (jjs - js)  * COMPSIZE * l1stride, c, ldc, m_from, jjs);
 #endif
 
 	STOP_RPCC(kernelcost);

driver/level3/level3_thread.c

@@ -324,6 +324,16 @@ static int inner_thread(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n,
     } else {
       if (min_l > GEMM_Q) min_l = (min_l + 1) / 2;
     }
+    
+    BLASLONG pad_min_l = min_l;
+
+#if defined(HALF)
+#if defined(DYNAMIC_ARCH)
+    pad_min_l = (min_l + gotoblas->sbgemm_align_k - 1) & ~(gotoblas->sbgemm_align_k-1);
+#else
+    pad_min_l = (min_l + SBGEMM_ALIGN_K - 1) & ~(SBGEMM_ALIGN_K - 1);;
+#endif
+#endif
 
     /* Determine step size in m
      * Note: We are currently on the first step in m
@@ -382,13 +392,13 @@ static int inner_thread(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n,
         /* Copy part of local region of B into workspace */
 	START_RPCC();
 	OCOPY_OPERATION(min_l, min_jj, b, ldb, ls, jjs,
-			buffer[bufferside] + min_l * (jjs - js) * COMPSIZE * l1stride);
+			buffer[bufferside] + pad_min_l * (jjs - js) * COMPSIZE * l1stride);
 	STOP_RPCC(copy_B);
 
         /* Apply kernel with local region of A and part of local region of B */
 	START_RPCC();
 	KERNEL_OPERATION(min_i, min_jj, min_l, alpha,
-			 sa, buffer[bufferside] + min_l * (jjs - js) * COMPSIZE * l1stride,
+			 sa, buffer[bufferside] + pad_min_l * (jjs - js) * COMPSIZE * l1stride,
 			 c, ldc, m_from, jjs);
 	STOP_RPCC(kernel);
 

kernel/arm64/KERNEL.NEOVERSEN2

@@ -190,10 +190,10 @@ ZGEMMOTCOPYOBJ =  zgemm_otcopy$(TSUFFIX).$(SUFFIX)
 
 SBGEMM_BETA    =  sbgemm_beta_neoversen2.c
 SBGEMMKERNEL    = sbgemm_kernel_$(SBGEMM_UNROLL_M)x$(SBGEMM_UNROLL_N)_neoversen2.c
-SBGEMMINCOPY    = sbgemm_ncopy_neoversen2.c
-SBGEMMITCOPY    = sbgemm_tcopy_neoversen2.c
-SBGEMMONCOPY    = sbgemm_ncopy_neoversen2.c
-SBGEMMOTCOPY    = sbgemm_tcopy_neoversen2.c
+SBGEMMINCOPY    = sbgemm_ncopy_$(SBGEMM_UNROLL_N)_neoversen2.c
+SBGEMMITCOPY    = sbgemm_tcopy_$(SBGEMM_UNROLL_M)_neoversen2.c
+SBGEMMONCOPY    = sbgemm_ncopy_$(SBGEMM_UNROLL_N)_neoversen2.c
+SBGEMMOTCOPY    = sbgemm_tcopy_$(SBGEMM_UNROLL_M)_neoversen2.c
 SBGEMMINCOPYOBJ =  sbgemm_incopy$(TSUFFIX).$(SUFFIX)
 SBGEMMITCOPYOBJ =  sbgemm_itcopy$(TSUFFIX).$(SUFFIX)
 SBGEMMONCOPYOBJ =  sbgemm_oncopy$(TSUFFIX).$(SUFFIX)

kernel/arm64/sbgemm_kernel_8x4_neoversen2.c

@@ -37,9 +37,9 @@
 
 int CNAME(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT *A, IFLOAT *B,
           FLOAT *C, BLASLONG ldc) {
-    if (alpha == 1.0f)
-        return sbgemm_kernel_neoversen2_alpha_one(m, n, k, alpha, A, B, C, ldc);
-    else
-        return sbgemm_kernel_neoversen2_alpha(m, n, k, alpha, A, B, C, ldc);
-    return 0;
+  if (alpha == 1.0f)
+    return sbgemm_kernel_neoversen2_alpha_one(m, n, k, alpha, A, B, C, ldc);
+  else
+    return sbgemm_kernel_neoversen2_alpha(m, n, k, alpha, A, B, C, ldc);
+  return 0;
 }

kernel/arm64/sbgemm_ncopy_4_neoversen2.c

@@ -0,0 +1,126 @@
+/***************************************************************************
+ * Copyright (c) 2022, The OpenBLAS Project
+ * All rights reserved.
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * 3. Neither the name of the OpenBLAS project nor the names of
+ * its contributors may be used to endorse or promote products
+ * derived from this software without specific prior written permission.
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ * *****************************************************************************/
+
+#include <arm_sve.h>
+
+#include "common.h"
+
+int CNAME(BLASLONG m, BLASLONG n, IFLOAT *a, BLASLONG lda, IFLOAT *b) {
+  IFLOAT *a_offset;
+  IFLOAT *a_offsetx[4];
+  IFLOAT *b_offset;
+  a_offset = a;
+  b_offset = b;
+
+  svbool_t pg16 = svdupq_b16(1, 1, 1, 1, 0, 0, 0, 0);
+  svbfloat16_t v0, v1, v2, v3;
+
+  for (BLASLONG j = 0; j < n / 4; j++) {
+    a_offsetx[0] = a_offset;
+    a_offsetx[1] = a_offsetx[0] + lda;
+    a_offsetx[2] = a_offsetx[1] + lda;
+    a_offsetx[3] = a_offsetx[2] + lda;
+    a_offset += 4 * lda;
+
+    for (BLASLONG i = 0; i < m / 4; i++) {
+      v0 = svld1_bf16(pg16, (bfloat16_t *)a_offsetx[0]);
+      v1 = svld1_bf16(pg16, (bfloat16_t *)a_offsetx[1]);
+      v2 = svld1_bf16(pg16, (bfloat16_t *)a_offsetx[2]);
+      v3 = svld1_bf16(pg16, (bfloat16_t *)a_offsetx[3]);
+
+      svst1_bf16(pg16, (bfloat16_t *)b_offset, v0);
+      svst1_bf16(pg16, (bfloat16_t *)b_offset + 4, v1);
+      svst1_bf16(pg16, (bfloat16_t *)b_offset + 8, v2);
+      svst1_bf16(pg16, (bfloat16_t *)b_offset + 12, v3);
+
+      b_offset += 16;
+      a_offsetx[0] += 4;
+      a_offsetx[1] += 4;
+      a_offsetx[2] += 4;
+      a_offsetx[3] += 4;
+    }
+
+    if (m & 3) {
+      BLASLONG rest = m & 3;
+      for (BLASLONG col = 0; col < 4; col++) {
+        b_offset[4 * col] = a_offsetx[col][0];
+        b_offset[4 * col + 1] = rest == 1 ? 0 : a_offsetx[col][1];
+        b_offset[4 * col + 2] = rest <= 2 ? 0 : a_offsetx[col][2];
+        b_offset[4 * col + 3] = rest <= 3 ? 0 : a_offsetx[col][3];
+      }
+      b_offset += 16;
+    }
+  }
+
+  if (n & 2) {
+    a_offsetx[0] = a_offset;
+    a_offsetx[1] = a_offsetx[0] + lda;
+    a_offset += 2 * lda;
+
+    for (BLASLONG i = 0; i < m / 4; i++) {
+      v0 = svld1_bf16(pg16, (bfloat16_t *)a_offsetx[0]);
+      v1 = svld1_bf16(pg16, (bfloat16_t *)a_offsetx[1]);
+      svst1_bf16(pg16, (bfloat16_t *)b_offset, v0);
+      svst1_bf16(pg16, (bfloat16_t *)b_offset + 4, v1);
+
+      b_offset += 8;
+      a_offsetx[0] += 4;
+      a_offsetx[1] += 4;
+    }
+
+    if (m & 3) {
+      BLASLONG rest = m & 3;
+      for (BLASLONG col = 0; col < 2; col++) {
+        b_offset[4 * col] = a_offsetx[col][0];
+        b_offset[4 * col + 1] = rest == 1 ? 0 : a_offsetx[col][1];
+        b_offset[4 * col + 2] = rest <= 2 ? 0 : a_offsetx[col][2];
+        b_offset[4 * col + 3] = rest <= 3 ? 0 : a_offsetx[col][3];
+      }
+      b_offset += 8;
+    }
+  }
+
+  if (n & 1) {
+    a_offsetx[0] = a_offset;
+    for (BLASLONG i = 0; i < m / 4; i++) {
+      v0 = svld1_bf16(pg16, (bfloat16_t *)a_offsetx[0]);
+      svst1_bf16(pg16, (bfloat16_t *)b_offset, v0);
+      b_offset += 4;
+      a_offsetx[0] += 4;
+    }
+    if (m & 3) {
+      BLASLONG rest = m & 3;
+      b_offset[0] = a_offsetx[0][0];
+      b_offset[1] = rest == 1 ? 0 : a_offsetx[0][1];
+      b_offset[2] = rest <= 2 ? 0 : a_offsetx[0][2];
+      b_offset[3] = rest <= 3 ? 0 : a_offsetx[0][3];
+    }
+  }
+
+  return 0;
+}

kernel/arm64/sbgemm_ncopy_neoversen2.c

@@ -1,101 +0,0 @@
-/***************************************************************************
- * Copyright (c) 2022, The OpenBLAS Project
- * All rights reserved.
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- * 1. Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * 3. Neither the name of the OpenBLAS project nor the names of
- * its contributors may be used to endorse or promote products
- * derived from this software without specific prior written permission.
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- * *****************************************************************************/
-
-#include "common.h"
-
-int CNAME(BLASLONG m, BLASLONG n, IFLOAT *a, BLASLONG lda, IFLOAT *b) {
-    IFLOAT *a_offset, *a_offset1, *a_offset2;
-    IFLOAT *b_offset;
-
-    a_offset = a;
-    b_offset = b;
-
-    for (BLASLONG j = 0; j < n / 2; j++) {
-        a_offset1 = a_offset;
-        a_offset2 = a_offset1 + lda;
-        a_offset += 2 * lda;
-        for (BLASLONG i = 0; i < m / 4; i++) {
-            *(b_offset + 0) = *(a_offset1 + 0);
-            *(b_offset + 1) = *(a_offset1 + 1);
-            *(b_offset + 2) = *(a_offset1 + 2);
-            *(b_offset + 3) = *(a_offset1 + 3);
-            *(b_offset + 4) = *(a_offset2 + 0);
-            *(b_offset + 5) = *(a_offset2 + 1);
-            *(b_offset + 6) = *(a_offset2 + 2);
-            *(b_offset + 7) = *(a_offset2 + 3);
-
-            a_offset1 += 4;
-            a_offset2 += 4;
-            b_offset += 8;
-        }
-        BLASLONG rest = m & 3;
-        if (rest == 3) {
-            *(b_offset + 0) = *(a_offset1 + 0);
-            *(b_offset + 1) = *(a_offset1 + 1);
-            *(b_offset + 2) = *(a_offset1 + 2);
-            *(b_offset + 3) = *(a_offset2 + 0);
-            *(b_offset + 4) = *(a_offset2 + 1);
-            *(b_offset + 5) = *(a_offset2 + 2);
-            b_offset += 6;
-        } else if (rest == 2) {
-            *(b_offset + 0) = *(a_offset1 + 0);
-            *(b_offset + 1) = *(a_offset1 + 1);
-            *(b_offset + 2) = *(a_offset2 + 0);
-            *(b_offset + 3) = *(a_offset2 + 1);
-            b_offset += 4;
-        } else if (rest == 1) {
-            *(b_offset + 0) = *(a_offset1 + 0);
-            *(b_offset + 1) = *(a_offset2 + 0);
-            b_offset += 2;
-        }
-    }
-    if (n & 1) {
-        for (BLASLONG i = 0; i < m / 4; i++) {
-            *(b_offset + 0) = *(a_offset + 0);
-            *(b_offset + 1) = *(a_offset + 1);
-            *(b_offset + 2) = *(a_offset + 2);
-            *(b_offset + 3) = *(a_offset + 3);
-
-            b_offset += 4;
-            a_offset += 4;
-        }
-        BLASLONG rest = m & 3;
-        if (rest == 3) {
-            *(b_offset + 0) = *(a_offset + 0);
-            *(b_offset + 1) = *(a_offset + 1);
-            *(b_offset + 2) = *(a_offset + 2);
-        } else if (rest == 2) {
-            *(b_offset + 0) = *(a_offset + 0);
-            *(b_offset + 1) = *(a_offset + 1);
-        } else if (rest == 1) {
-            *(b_offset + 0) = *(a_offset + 0);
-        }
-    }
-
-    return 0;
-}

kernel/arm64/sbgemm_tcopy_8_neoversen2.c

@@ -0,0 +1,165 @@
+/***************************************************************************
+ * Copyright (c) 2022, The OpenBLAS Project
+ * All rights reserved.
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * 3. Neither the name of the OpenBLAS project nor the names of
+ * its contributors may be used to endorse or promote products
+ * derived from this software without specific prior written permission.
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ * *****************************************************************************/
+
+#include "common.h"
+
+int CNAME(BLASLONG m, BLASLONG n, IFLOAT *a, BLASLONG lda, IFLOAT *b) {
+  IFLOAT *a_offset, *a_offset0, *a_offset1, *a_offset2, *a_offset3;
+  IFLOAT *b_offset;
+  a_offset = a;
+  b_offset = b;
+
+  for (BLASLONG j = 0; j < n / 8; j++) {
+    a_offset0 = a_offset;
+    a_offset1 = a_offset0 + lda;
+    a_offset2 = a_offset1 + lda;
+    a_offset3 = a_offset2 + lda;
+    a_offset += 8;
+
+    for (BLASLONG i = 0; i < m / 4; i++) {
+      for (BLASLONG line = 0; line < 8; line++) {
+        b_offset[line * 4] = a_offset0[line];
+        b_offset[line * 4 + 1] = a_offset1[line];
+        b_offset[line * 4 + 2] = a_offset2[line];
+        b_offset[line * 4 + 3] = a_offset3[line];
+      }
+
+      b_offset += 32;
+      a_offset0 += 4 * lda;
+      a_offset1 += 4 * lda;
+      a_offset2 += 4 * lda;
+      a_offset3 += 4 * lda;
+    }
+
+    if (m & 3) {
+      BLASLONG rest = m & 3;
+      for (BLASLONG line = 0; line < 8; line++) {
+        b_offset[line * 4] = a_offset0[line];
+        b_offset[line * 4 + 1] = rest == 1 ? 0 : a_offset1[line];
+        b_offset[line * 4 + 2] = rest <= 2 ? 0 : a_offset2[line];
+        b_offset[line * 4 + 3] = rest <= 3 ? 0 : a_offset3[line];
+      }
+      b_offset += 32;
+    }
+  }
+
+  if (n & 4) {
+    a_offset0 = a_offset;
+    a_offset1 = a_offset0 + lda;
+    a_offset2 = a_offset1 + lda;
+    a_offset3 = a_offset2 + lda;
+    a_offset += 4;
+
+    for (BLASLONG i = 0; i < m / 4; i++) {
+      for (BLASLONG line = 0; line < 4; line++) {
+        b_offset[line * 4] = a_offset0[line];
+        b_offset[line * 4 + 1] = a_offset1[line];
+        b_offset[line * 4 + 2] = a_offset2[line];
+        b_offset[line * 4 + 3] = a_offset3[line];
+      }
+
+      b_offset += 16;
+      a_offset0 += 4 * lda;
+      a_offset1 += 4 * lda;
+      a_offset2 += 4 * lda;
+      a_offset3 += 4 * lda;
+    }
+
+    if (m & 3) {
+      BLASLONG rest = m & 3;
+      for (BLASLONG line = 0; line < 4; line++) {
+        b_offset[line * 4] = a_offset0[line];
+        b_offset[line * 4 + 1] = rest == 1 ? 0 : a_offset1[line];
+        b_offset[line * 4 + 2] = rest <= 2 ? 0 : a_offset2[line];
+        b_offset[line * 4 + 3] = rest <= 3 ? 0 : a_offset3[line];
+      }
+      b_offset += 16;
+    }
+  }
+
+  if (n & 2) {
+    a_offset0 = a_offset;
+    a_offset1 = a_offset0 + lda;
+    a_offset2 = a_offset1 + lda;
+    a_offset3 = a_offset2 + lda;
+    a_offset += 2;
+
+    for (BLASLONG i = 0; i < m / 4; i++) {
+      for (BLASLONG line = 0; line < 2; line++) {
+        b_offset[line * 4] = a_offset0[line];
+        b_offset[line * 4 + 1] = a_offset1[line];
+        b_offset[line * 4 + 2] = a_offset2[line];
+        b_offset[line * 4 + 3] = a_offset3[line];
+      }
+      b_offset += 8;
+      a_offset0 += 4 * lda;
+      a_offset1 += 4 * lda;
+      a_offset2 += 4 * lda;
+      a_offset3 += 4 * lda;
+    }
+
+    if (m & 3) {
+      BLASLONG rest = m & 3;
+      for (BLASLONG line = 0; line < 2; line++) {
+        b_offset[line * 4] = a_offset0[line];
+        b_offset[line * 4 + 1] = rest == 1 ? 0 : a_offset1[line];
+        b_offset[line * 4 + 2] = rest <= 2 ? 0 : a_offset2[line];
+        b_offset[line * 4 + 3] = rest <= 3 ? 0 : a_offset3[line];
+      }
+      b_offset += 8;
+    }
+  }
+
+  if (n & 1) {
+    a_offset0 = a_offset;
+    a_offset1 = a_offset0 + lda;
+    a_offset2 = a_offset1 + lda;
+    a_offset3 = a_offset2 + lda;
+
+    for (BLASLONG i = 0; i < m / 4; i++) {
+      b_offset[0] = *a_offset0;
+      b_offset[1] = *a_offset1;
+      b_offset[2] = *a_offset2;
+      b_offset[3] = *a_offset3;
+      b_offset += 4;
+      a_offset0 += 4 * lda;
+      a_offset1 += 4 * lda;
+      a_offset2 += 4 * lda;
+      a_offset3 += 4 * lda;
+    }
+
+    if (m & 3) {
+      BLASLONG rest = m & 3;
+      b_offset[0] = *a_offset0;
+      b_offset[1] = rest == 1 ? 0 : *a_offset1;
+      b_offset[2] = rest <= 2 ? 0 : *a_offset2;
+      b_offset[3] = rest <= 3 ? 0 : *a_offset3;
+    }
+  }
+  return 0;
+}

kernel/arm64/sbgemm_tcopy_neoversen2.c

@@ -1,109 +0,0 @@
-/***************************************************************************
- * Copyright (c) 2022, The OpenBLAS Project
- * All rights reserved.
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- * 1. Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * 3. Neither the name of the OpenBLAS project nor the names of
- * its contributors may be used to endorse or promote products
- * derived from this software without specific prior written permission.
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- * *****************************************************************************/
-
-#include "common.h"
-
-
-int CNAME(BLASLONG m, BLASLONG n, IFLOAT *a, BLASLONG lda, IFLOAT *b) {
-    IFLOAT *a_offset, *a_offset1, *a_offset2, *a_offset3, *a_offset4;
-    IFLOAT *b_offset;
-    a_offset = a;
-    b_offset = b;
-
-    for (BLASLONG j = 0; j < n / 2; j++) {
-        a_offset1 = a_offset;
-        a_offset2 = a_offset1 + lda;
-        a_offset3 = a_offset2 + lda;
-        a_offset4 = a_offset3 + lda;
-        a_offset += 2;
-
-        for (BLASLONG i = 0; i < m / 4; i++) {
-            *(b_offset + 0) = *(a_offset1 + 0);
-            *(b_offset + 1) = *(a_offset2 + 0);
-            *(b_offset + 2) = *(a_offset3 + 0);
-            *(b_offset + 3) = *(a_offset4 + 0);
-            *(b_offset + 4) = *(a_offset1 + 1);
-            *(b_offset + 5) = *(a_offset2 + 1);
-            *(b_offset + 6) = *(a_offset3 + 1);
-            *(b_offset + 7) = *(a_offset4 + 1);
-
-            b_offset += 8;
-            a_offset1 += 4 * lda;
-            a_offset2 += 4 * lda;
-            a_offset3 += 4 * lda;
-            a_offset4 += 4 * lda;
-        }
-        
-        if (m & 3) {
-            BLASLONG rest = m & 3;
-            if (rest == 3) {
-                *(b_offset + 0) = *(a_offset1 + 0);
-                *(b_offset + 1) = *(a_offset2 + 0);
-                *(b_offset + 2) = *(a_offset3 + 0);
-                *(b_offset + 3) = *(a_offset1 + 1);
-                *(b_offset + 4) = *(a_offset2 + 1);
-                *(b_offset + 5) = *(a_offset3 + 1);
-                b_offset += 6;
-            } else if (rest == 2) {
-                *(b_offset + 0) = *(a_offset1 + 0);
-                *(b_offset + 1) = *(a_offset2 + 0);
-                *(b_offset + 2) = *(a_offset1 + 1);
-                *(b_offset + 3) = *(a_offset2 + 1);
-                b_offset += 4;
-            } else if (rest == 1) {
-                *(b_offset + 0) = *(a_offset1 + 0);
-                *(b_offset + 1) = *(a_offset1 + 1);
-                b_offset += 2;
-            }
-        }
-    }
-    if (n & 1) {
-        for (BLASLONG i = 0; i < m / 4; i++) {
-            *(b_offset + 0) = *(a_offset);
-            *(b_offset + 1) = *(a_offset + lda);
-            *(b_offset + 2) = *(a_offset + lda * 2);
-            *(b_offset + 3) = *(a_offset + lda * 3);
-
-            b_offset += 4;
-            a_offset += 4 * lda;
-        }
-        BLASLONG rest = m & 3;
-        if (rest == 3) {
-            *(b_offset + 0) = *(a_offset);
-            *(b_offset + 1) = *(a_offset + lda);
-            *(b_offset + 2) = *(a_offset + lda * 2);
-        } else if (rest == 2) {
-            *(b_offset + 0) = *(a_offset);
-            *(b_offset + 1) = *(a_offset + lda);
-        } else if (rest == 1) {
-            *(b_offset + 0) = *(a_offset);
-        }
-    }
-
-    return 0;
-}

kernel/setparam-ref.c

@@ -62,6 +62,8 @@ gotoblas_t TABLE_NAME = {
  MAX(SBGEMM_DEFAULT_UNROLL_M, SBGEMM_DEFAULT_UNROLL_N),
 #endif
 
+  SBGEMM_ALIGN_K,
+
   sbstobf16_kTS, sbdtobf16_kTS, sbf16tos_kTS, dbf16tod_kTS,
 
   samax_kTS,  samin_kTS,  smax_kTS,  smin_kTS,
@@ -866,8 +868,9 @@ gotoblas_t TABLE_NAME = {
   cgeadd_kTS,
 #endif
 #if BUILD_COMPLEX16==1
-  zgeadd_kTS
+  zgeadd_kTS,
 #endif
+  1,  // align_k
 };
 
 #if (ARCH_ARM64)

param.h

@@ -79,6 +79,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define SBGEMM_DEFAULT_P 256
 #define SBGEMM_DEFAULT_R 256
 #define SBGEMM_DEFAULT_Q 256
+#define SBGEMM_ALIGN_K 1  // must be 2^x
+
 #ifdef OPTERON
 
 #define SNUMOPT		4
@@ -3394,6 +3396,9 @@ is a big desktop or server with abundant cache rather than a phone or embedded d
 
 #elif defined(NEOVERSEN2)
 
+#undef SBGEMM_ALIGN_K
+#define SBGEMM_ALIGN_K 4
+
 #undef SBGEMM_DEFAULT_UNROLL_M
 #undef SBGEMM_DEFAULT_UNROLL_N
 #define SBGEMM_DEFAULT_UNROLL_M 8