OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
Browse Source

* checkpoint sbgemm for SVE-256

tags/v0.3.30
Aditya Tewari Ye Tao 1 year ago
parent
c139b63342
commit
4379a6fbe3
10 changed files with 1277 additions and 4 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +2
    -2
      cmake/system.cmake
  2. +12
    -2
      kernel/arm64/KERNEL.NEOVERSEV1
  3. +83
    -0
      kernel/arm64/sbgemm_beta_neoversev1.c
  4. +46
    -0
      kernel/arm64/sbgemm_kernel_8x4_neoversev1.c
  5. +472
    -0
      kernel/arm64/sbgemm_kernel_8x4_neoversev1_impl.c
  6. +127
    -0
      kernel/arm64/sbgemm_ncopy_4_neoversev1.c
  7. +180
    -0
      kernel/arm64/sbgemm_ncopy_8_neoversev1.c
  8. +148
    -0
      kernel/arm64/sbgemm_tcopy_4_neoversev1.c
  9. +200
    -0
      kernel/arm64/sbgemm_tcopy_8_neoversev1.c
  10. +7
    -0
      param.h

+ 2
- 2
cmake/system.cmake View File

@@ -291,10 +291,10 @@ if (DEFINED TARGET)

if (${TARGET} STREQUAL NEOVERSEV1)
if (${CMAKE_C_COMPILER_ID} STREQUAL "PGI" AND NOT NO_SVE)
set (KERNEL_DEFINITIONS "${KERNEL_DEFINITIONS} -Msve_intrinsics -march=armv8.4-a+sve -mtune=neoverse-v1")
set (KERNEL_DEFINITIONS "${KERNEL_DEFINITIONS} -Msve_intrinsics -march=armv8.4-a+sve+bf16 -mtune=neoverse-v1")
else ()
if (CMAKE_C_COMPILER_VERSION VERSION_GREATER 10.4 OR CMAKE_C_COMPILER_VERSION VERSION_EQUAL 10.4)
set (KERNEL_DEFINITIONS "${KERNEL_DEFINITIONS} -march=armv8.4-a+sve -mtune=neoverse-v1")
set (KERNEL_DEFINITIONS "${KERNEL_DEFINITIONS} -march=armv8.4-a+sve+bf16 -mtune=neoverse-v1")
else ()
message(FATAL_ERROR "Compiler ${CMAKE_C_COMPILER} ${CMAKE_C_COMPILER_VERSION} does not support Neoverse V1.")
endif()


+ 12
- 2
kernel/arm64/KERNEL.NEOVERSEV1 View File

@@ -1,4 +1,14 @@
include $(KERNELDIR)/KERNEL.ARMV8SVE

SGEMVTKERNEL = gemv_t_sve_v1x3.c
DGEMVTKERNEL = gemv_t_sve_v1x3.c
SGEMVTKERNEL = gemv_t_sve.c
DGEMVTKERNEL = gemv_t_sve.c
SBGEMM_BETA = sbgemm_beta_neoversev1.c
SBGEMMKERNEL = sbgemm_kernel_$(SBGEMM_UNROLL_M)x$(SBGEMM_UNROLL_N)_neoversev1.c
SBGEMMINCOPY = sbgemm_ncopy_$(SBGEMM_UNROLL_M)_neoversev1.c
SBGEMMITCOPY = sbgemm_tcopy_$(SBGEMM_UNROLL_M)_neoversev1.c
SBGEMMONCOPY = sbgemm_ncopy_$(SBGEMM_UNROLL_N)_neoversev1.c
SBGEMMOTCOPY = sbgemm_tcopy_$(SBGEMM_UNROLL_N)_neoversev1.c
SBGEMMINCOPYOBJ = sbgemm_incopy$(TSUFFIX).$(SUFFIX)
SBGEMMITCOPYOBJ = sbgemm_itcopy$(TSUFFIX).$(SUFFIX)
SBGEMMONCOPYOBJ = sbgemm_oncopy$(TSUFFIX).$(SUFFIX)
SBGEMMOTCOPYOBJ = sbgemm_otcopy$(TSUFFIX).$(SUFFIX)

+ 83
- 0
kernel/arm64/sbgemm_beta_neoversev1.c View File

@@ -0,0 +1,83 @@
/***************************************************************************
* Copyright (c) 2024, The OpenBLAS Project
* All rights reserved.
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
* met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* 3. Neither the name of the OpenBLAS project nor the names of
* its contributors may be used to endorse or promote products
* derived from this software without specific prior written permission.
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
* *****************************************************************************/

#include "common.h"

int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT beta, IFLOAT *dummy2,
BLASLONG dummy3, IFLOAT *dummy4, BLASLONG dummy5, FLOAT *c,
BLASLONG ldc) {

BLASLONG i, j;
BLASLONG chunk, remain;
FLOAT *c_offset1, *c_offset;
c_offset = c;
chunk = m >> 3;
remain = m & 7;
if (beta == ZERO) {
for (j = n; j > 0; j--) {
c_offset1 = c_offset;
c_offset += ldc;
for (i = chunk; i > 0; i--) {
*(c_offset1 + 0) = ZERO;
*(c_offset1 + 1) = ZERO;
*(c_offset1 + 2) = ZERO;
*(c_offset1 + 3) = ZERO;
*(c_offset1 + 4) = ZERO;
*(c_offset1 + 5) = ZERO;
*(c_offset1 + 6) = ZERO;
*(c_offset1 + 7) = ZERO;
c_offset1 += 8;
}
for (i = remain; i > 0; i--) {
*c_offset1 = ZERO;
c_offset1++;
}
}
} else {
for (j = n; j > 0; j--) {
c_offset1 = c_offset;
c_offset += ldc;
for (i = chunk; i > 0; i--) {
*(c_offset1 + 0) *= beta;
*(c_offset1 + 1) *= beta;
*(c_offset1 + 2) *= beta;
*(c_offset1 + 3) *= beta;
*(c_offset1 + 4) *= beta;
*(c_offset1 + 5) *= beta;
*(c_offset1 + 6) *= beta;
*(c_offset1 + 7) *= beta;
c_offset1 += 8;
}
for (i = remain; i > 0; i--) {
*c_offset1 *= beta;
c_offset1++;
}
}
}
return 0;
};

+ 46
- 0
kernel/arm64/sbgemm_kernel_8x4_neoversev1.c View File

@@ -0,0 +1,46 @@
/***************************************************************************
* Copyright (c) 2024, The OpenBLAS Project
* All rights reserved.
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
* met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* 3. Neither the name of the OpenBLAS project nor the names of
* its contributors may be used to endorse or promote products
* derived from this software without specific prior written permission.
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
* *****************************************************************************/

#include <arm_sve.h>

#include "common.h"

#define ALPHA_ONE
#include "sbgemm_kernel_8x4_neoversev1_impl.c"
#undef ALPHA_ONE
#include "sbgemm_kernel_8x4_neoversev1_impl.c"

int CNAME(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT *A, IFLOAT *B,
FLOAT *C, BLASLONG ldc) {
if (alpha == 1.0f)
return sbgemm_kernel_neoversev1_alpha_one(m, n, k, alpha, A, B, C, ldc);
else
return sbgemm_kernel_neoversev1_alpha(m, n, k, alpha, A, B, C, ldc);
return 0;
}


+ 472
- 0
kernel/arm64/sbgemm_kernel_8x4_neoversev1_impl.c View File

@@ -0,0 +1,472 @@
/***************************************************************************
* Copyright (c) 2024, The OpenBLAS Project
* All rights reserved.
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
* met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* 3. Neither the name of the OpenBLAS project nor the names of
* its contributors may be used to endorse or promote products
* derived from this software without specific prior written permission.
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
* *****************************************************************************/

#include <arm_sve.h>

#include "common.h"

#define INIT_C(M, N) mc##M##N = svdup_f32(0);

#define MATMUL(M, N) mc##M##N = svbfmmla(mc##M##N, ma##M, mb##N);

#define INIT_C_8x4 \
do { \
INIT_C(0, 0); \
INIT_C(0, 1); \
INIT_C(1, 0); \
INIT_C(1, 1); \
INIT_C(2, 0); \
INIT_C(2, 1); \
INIT_C(3, 0); \
INIT_C(3, 1); \
} while (0);

#ifdef ALPHA_ONE
#define UPDATE_C(PG, PTR, DST, SRC) \
do { \
DST = svld1_f32((PG), (PTR)); \
DST = svadd_z((PG), SRC, DST); \
svst1_f32((PG), (PTR), DST); \
} while (0);
#else
#define UPDATE_C(PG, PTR, DST, SRC) \
do { \
DST = svld1_f32((PG), (PTR)); \
DST = svmad_z((PG), svalpha, SRC, DST); \
svst1_f32((PG), (PTR), DST); \
} while (0);
#endif

#ifdef ALPHA_ONE
int sbgemm_kernel_neoversev1_alpha_one(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A, IFLOAT * B, FLOAT * C, BLASLONG ldc)
#else
int sbgemm_kernel_neoversev1_alpha(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A, IFLOAT * B, FLOAT * C, BLASLONG ldc)
#endif
{
BLASLONG pad_k = (k + 3) & ~3;

svbfloat16_t ma0, ma1, ma2, ma3, mb0, mb1;
svfloat32_t mc00, mc01, mc10, mc11, mc20, mc21, mc30, mc31,
vc0, vc1, vc2, vc3, vc4, vc5, vc6, vc7,
oc0, oc1, oc2, oc3, oc4, oc5, oc6, oc7;
svfloat32_t svalpha = svdup_f32(alpha);

svbool_t pg16 = svptrue_b16();
svbool_t pg16_low = svdupq_b16(1, 1, 1, 1, 0, 0, 0, 0);
svbool_t pg32 = svptrue_b32();
svbool_t pg32_low = svdupq_b32(1, 1, 0, 0);
svbool_t pg32_first = svdupq_b32(1, 0, 0, 0);

bfloat16_t *ptr_a = (bfloat16_t *)A;
bfloat16_t *ptr_b = (bfloat16_t *)B;
FLOAT *ptr_c = C;

bfloat16_t *ptr_a0, *ptr_a1, *ptr_a2, *ptr_a3;
bfloat16_t *ptr_b0, *ptr_b1;
FLOAT *ptr_c0, *ptr_c1, *ptr_c2, *ptr_c3;

for (BLASLONG j = 0; j < n / 4; j++) {
ptr_c0 = ptr_c;
ptr_c1 = ptr_c0 + ldc;
ptr_c2 = ptr_c1 + ldc;
ptr_c3 = ptr_c2 + ldc;
ptr_c += 4 * ldc;
ptr_a = (bfloat16_t *)A;

for (BLASLONG i = 0; i < m / 8; i++) {
ptr_a0 = ptr_a;
ptr_a += 8 * pad_k;

ptr_b0 = ptr_b;

INIT_C_8x4;

for (BLASLONG p = 0; p < pad_k; p += 4) {
ma0 = svld1_bf16(pg16, ptr_a0);
ma1 = svld1_bf16(pg16, ptr_a0 + 8);
ma2 = svld1_bf16(pg16, ptr_a0 + 16);
ma3 = svld1_bf16(pg16, ptr_a0 + 24);

mb0 = svld1_bf16(pg16, ptr_b0);
mb1 = svld1_bf16(pg16, ptr_b0 + 8);

MATMUL(0, 0); MATMUL(0, 1);
MATMUL(1, 0); MATMUL(1, 1);
MATMUL(2, 0); MATMUL(2, 1);
MATMUL(3, 0); MATMUL(3, 1);

ptr_a0 += 32;
ptr_b0 += 16;
}

vc0 = svuzp1(mc00, mc10);
vc1 = svuzp1(mc20, mc30);
vc2 = svuzp2(mc00, mc10);
vc3 = svuzp2(mc20, mc30);
vc4 = svuzp1(mc01, mc11);
vc5 = svuzp1(mc21, mc31);
vc6 = svuzp2(mc01, mc11);
vc7 = svuzp2(mc21, mc31);

UPDATE_C(pg32, ptr_c0, oc0, vc0);
UPDATE_C(pg32, ptr_c0+4, oc1, vc1);
UPDATE_C(pg32, ptr_c1, oc2, vc2);
UPDATE_C(pg32, ptr_c1+4, oc3, vc3);
UPDATE_C(pg32, ptr_c2, oc4, vc4)
UPDATE_C(pg32, ptr_c2+4, oc5, vc5);
UPDATE_C(pg32, ptr_c3, oc6, vc6)
UPDATE_C(pg32, ptr_c3+4, oc7, vc7);

ptr_c0 += 8;
ptr_c1 += 8;
ptr_c2 += 8;
ptr_c3 += 8;
}

if (m & 4) {
ptr_a0 = ptr_a;
ptr_a += 4 * pad_k;
ptr_b0 = ptr_b;

INIT_C(0, 0); INIT_C(0, 1);
INIT_C(1, 0); INIT_C(1, 1);

for (BLASLONG p = 0; p < pad_k; p += 4) {
ma0 = svld1_bf16(pg16, ptr_a0);
ma1 = svld1_bf16(pg16, ptr_a0 + 8);
mb0 = svld1_bf16(pg16, ptr_b0);
mb1 = svld1_bf16(pg16, ptr_b0 + 8);

MATMUL(0, 0); MATMUL(0, 1);
MATMUL(1, 0); MATMUL(1, 1);

ptr_a0 += 16;
ptr_b0 += 16;
}

vc0 = svuzp1(mc00, mc10);
vc1 = svuzp2(mc00, mc10);
vc2 = svuzp1(mc01, mc11);
vc3 = svuzp2(mc01, mc11);

UPDATE_C(pg32, ptr_c0, oc0, vc0);
UPDATE_C(pg32, ptr_c1, oc1, vc1);
UPDATE_C(pg32, ptr_c2, oc2, vc2);
UPDATE_C(pg32, ptr_c3, oc3, vc3);

ptr_c0 += 4;
ptr_c1 += 4;
ptr_c2 += 4;
ptr_c3 += 4;
}

if (m & 2) {
ptr_a0 = ptr_a;
ptr_a += 2 * pad_k;
ptr_b0 = ptr_b;

INIT_C(0, 0); INIT_C(0, 1);
for (BLASLONG p = 0; p < pad_k; p += 4) {
ma0 = svld1_bf16(pg16, ptr_a0);
mb0 = svld1_bf16(pg16, ptr_b0);
mb1 = svld1_bf16(pg16, ptr_b0 + 8);

MATMUL(0, 0); MATMUL(0, 1);

ptr_a0 += 8;
ptr_b0 += 16;
}

vc0 = svuzp1(mc00, mc00);
vc1 = svuzp2(mc00, mc00);
vc2 = svuzp1(mc01, mc01);
vc3 = svuzp2(mc01, mc01);

UPDATE_C(pg32_low, ptr_c0, oc0, vc0);
UPDATE_C(pg32_low, ptr_c1, oc1, vc1);
UPDATE_C(pg32_low, ptr_c2, oc2, vc2);
UPDATE_C(pg32_low, ptr_c3, oc3, vc3);

ptr_c0 += 2;
ptr_c1 += 2;
ptr_c2 += 2;
ptr_c3 += 2;
}

if (m & 1) {
ptr_a0 = ptr_a;
ptr_b0 = ptr_b;

INIT_C(0, 0); INIT_C(0, 1);
for (BLASLONG p = 0; p < pad_k; p += 4) {
ma0 = svld1_bf16(pg16_low, ptr_a0);
mb0 = svld1_bf16(pg16, ptr_b0);
mb1 = svld1_bf16(pg16, ptr_b0 + 8);

MATMUL(0, 0); MATMUL(0, 1);

ptr_a0 += 4;
ptr_b0 += 16;
}

vc1 = svuzp2(mc00, mc00);
vc3 = svuzp2(mc01, mc01);

UPDATE_C(pg32_first, ptr_c0, oc0, mc00);
UPDATE_C(pg32_first, ptr_c1, oc1, vc1);
UPDATE_C(pg32_first, ptr_c2, oc2, mc01);
UPDATE_C(pg32_first, ptr_c3, oc3, vc3);

}

ptr_b += 4 * pad_k;
}

if (n & 2) {
ptr_c0 = ptr_c;
ptr_c1 = ptr_c0 + ldc;
ptr_c += 2 * ldc;
ptr_a = (bfloat16_t *)A;

for (BLASLONG i = 0; i < m / 8; i++) {
ptr_a0 = ptr_a;
ptr_a += 8 * pad_k;

ptr_b0 = ptr_b;

INIT_C(0, 0);
INIT_C(1, 0);
INIT_C(2, 0);
INIT_C(3, 0);

for (BLASLONG p = 0; p < pad_k; p += 4) {
ma0 = svld1_bf16(pg16, ptr_a0);
ma1 = svld1_bf16(pg16, ptr_a0 + 8);
ma2 = svld1_bf16(pg16, ptr_a0 + 16);
ma3 = svld1_bf16(pg16, ptr_a0 + 24);

mb0 = svld1_bf16(pg16, ptr_b0);

MATMUL(0, 0);
MATMUL(1, 0);
MATMUL(2, 0);
MATMUL(3, 0);

ptr_a0 += 32;
ptr_b0 += 8;
}

vc0 = svuzp1(mc00, mc10);
vc1 = svuzp1(mc20, mc30);
vc2 = svuzp2(mc00, mc10);
vc3 = svuzp2(mc20, mc30);

UPDATE_C(pg32, ptr_c0, oc0, vc0);
UPDATE_C(pg32, ptr_c0 + 4, oc1, vc1);
UPDATE_C(pg32, ptr_c1, oc2, vc2);
UPDATE_C(pg32, ptr_c1 + 4, oc3, vc3);

ptr_c0 += 8;
ptr_c1 += 8;
}

if (m & 4) {
ptr_a0 = ptr_a;
ptr_a += 4 * pad_k;
ptr_b0 = ptr_b;

INIT_C(0, 0);
INIT_C(1, 0);

for (BLASLONG p = 0; p < pad_k; p += 4) {
ma0 = svld1_bf16(pg16, ptr_a0);
ma1 = svld1_bf16(pg16, ptr_a0 + 8);
mb0 = svld1_bf16(pg16, ptr_b0);
MATMUL(0, 0);
MATMUL(1, 0);
ptr_a0 += 16;
ptr_b0 += 8;
}

vc0 = svuzp1(mc00, mc10);
vc1 = svuzp2(mc00, mc10);

UPDATE_C(pg32, ptr_c0, oc0, vc0);
UPDATE_C(pg32, ptr_c1, oc1, vc1);

ptr_c0 += 4;
ptr_c1 += 4;
}

if (m & 2) {
ptr_a0 = ptr_a;
ptr_a += 2 * pad_k;
ptr_b0 = ptr_b;

INIT_C(0, 0);

for (BLASLONG p = 0; p < pad_k; p += 4) {
ma0 = svld1_bf16(pg16, ptr_a0);
mb0 = svld1_bf16(pg16, ptr_b0);

MATMUL(0, 0);

ptr_a0 += 8;
ptr_b0 += 8;
}

vc0 = svuzp1(mc00, mc00);
vc1 = svuzp2(mc00, mc00);
UPDATE_C(pg32_low, ptr_c0, oc0, vc0);
UPDATE_C(pg32_low, ptr_c1, oc1, vc1);

ptr_c0 += 2;
ptr_c1 += 2;

}

if (m & 1) {
ptr_a0 = ptr_a;
ptr_b0 = ptr_b;
INIT_C(0, 0);
for (BLASLONG p = 0; p < pad_k; p += 4) {
ma0 = svld1_bf16(pg16_low, ptr_a0);
mb0 = svld1_bf16(pg16, ptr_b0);
MATMUL(0, 0);
ptr_a0 += 4;
ptr_b0 += 8;
}
vc1 = svuzp2(mc00, mc00);

UPDATE_C(pg32_first, ptr_c0, oc0, mc00);
UPDATE_C(pg32_first, ptr_c1, oc1, vc1);
}

ptr_b += 2 * pad_k;
}

if (n & 1) {
ptr_c0 = ptr_c;
ptr_a = (bfloat16_t *)A;

for (BLASLONG i = 0; i < m / 8; i++) {
ptr_a0 = ptr_a;
ptr_a += 8 * pad_k;

ptr_b0 = ptr_b;

INIT_C(0, 0);
INIT_C(1, 0);
INIT_C(2, 0);
INIT_C(3, 0);

for (BLASLONG p = 0; p < pad_k; p += 4) {
ma0 = svld1_bf16(pg16, ptr_a0);
ma1 = svld1_bf16(pg16, ptr_a0 + 8);
ma2 = svld1_bf16(pg16, ptr_a0 + 16);
ma3 = svld1_bf16(pg16, ptr_a0 + 24);

mb0 = svld1_bf16(pg16_low, ptr_b0);

MATMUL(0, 0);
MATMUL(1, 0);
MATMUL(2, 0);
MATMUL(3, 0);

ptr_a0 += 32;
ptr_b0 += 4;
}

vc0 = svuzp1(mc00, mc10);
vc1 = svuzp1(mc20, mc30);

UPDATE_C(pg32, ptr_c0, oc0, vc0);
UPDATE_C(pg32, ptr_c0 + 4, oc1, vc1);

ptr_c0 += 8;
}

if (m & 4) {
ptr_a0 = ptr_a;
ptr_a += 4 * pad_k;
ptr_b0 = ptr_b;
INIT_C(0, 0);
INIT_C(1, 0);
for (BLASLONG p = 0; p < pad_k; p += 4) {
ma0 = svld1_bf16(pg16, ptr_a0);
ma1 = svld1_bf16(pg16, ptr_a0 + 8);
mb0 = svld1_bf16(pg16_low, ptr_b0);
MATMUL(0, 0);
MATMUL(1, 0);
ptr_a0 += 16;
ptr_b0 += 4;
}
vc0 = svuzp1(mc00, mc10);
UPDATE_C(pg32, ptr_c0, oc0, vc0);
ptr_c0 += 4;
}

if (m & 2) {
ptr_a0 = ptr_a;
ptr_a += 2 * pad_k;
ptr_b0 = ptr_b;

INIT_C(0, 0);

for (BLASLONG p = 0; p < pad_k; p += 4) {
ma0 = svld1_bf16(pg16, ptr_a0);
mb0 = svld1_bf16(pg16_low, ptr_b0);

MATMUL(0, 0);

ptr_a0 += 8;
ptr_b0 += 4;
}
vc0 = svuzp1(mc00, mc00);
UPDATE_C(pg32_low, ptr_c0, oc0, vc0);
ptr_c0 += 2;
}

if (m & 1) {
ptr_a0 = ptr_a;
ptr_b0 = ptr_b;
INIT_C(0, 0);
for (BLASLONG p = 0; p < pad_k; p += 4) {
ma0 = svld1_bf16(pg16_low, ptr_a0);
mb0 = svld1_bf16(pg16_low, ptr_b0);
MATMUL(0, 0);
ptr_a0 += 4;
ptr_b0 += 4;
}
UPDATE_C(pg32_first, ptr_c0, oc0, mc00);
}
}

return 0;
}


+ 127
- 0
kernel/arm64/sbgemm_ncopy_4_neoversev1.c View File

@@ -0,0 +1,127 @@
/***************************************************************************
* Copyright (c) 2024, The OpenBLAS Project
* All rights reserved.
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
* met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* 3. Neither the name of the OpenBLAS project nor the names of
* its contributors may be used to endorse or promote products
* derived from this software without specific prior written permission.
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
* *****************************************************************************/

#include <arm_sve.h>

#include "common.h"

int CNAME(BLASLONG m, BLASLONG n, IFLOAT *a, BLASLONG lda, IFLOAT *b) {
IFLOAT *a_offset;
IFLOAT *a_offsetx[4];
IFLOAT *b_offset;
a_offset = a;
b_offset = b;

svbool_t pg16 = svdupq_b16(1, 1, 1, 1, 0, 0, 0, 0);
svbfloat16_t v0, v1, v2, v3;

for (BLASLONG j = 0; j < n / 4; j++) {
a_offsetx[0] = a_offset;
a_offsetx[1] = a_offsetx[0] + lda;
a_offsetx[2] = a_offsetx[1] + lda;
a_offsetx[3] = a_offsetx[2] + lda;
a_offset += 4 * lda;

for (BLASLONG i = 0; i < m / 4; i++) {
v0 = svld1_bf16(pg16, (bfloat16_t *)a_offsetx[0]);
v1 = svld1_bf16(pg16, (bfloat16_t *)a_offsetx[1]);
v2 = svld1_bf16(pg16, (bfloat16_t *)a_offsetx[2]);
v3 = svld1_bf16(pg16, (bfloat16_t *)a_offsetx[3]);

svst1_bf16(pg16, (bfloat16_t *)b_offset, v0);
svst1_bf16(pg16, (bfloat16_t *)b_offset + 4, v1);
svst1_bf16(pg16, (bfloat16_t *)b_offset + 8, v2);
svst1_bf16(pg16, (bfloat16_t *)b_offset + 12, v3);

b_offset += 16;
a_offsetx[0] += 4;
a_offsetx[1] += 4;
a_offsetx[2] += 4;
a_offsetx[3] += 4;
}

if (m & 3) {
BLASLONG rest = m & 3;
for (BLASLONG col = 0; col < 4; col++) {
b_offset[4 * col] = a_offsetx[col][0];
b_offset[4 * col + 1] = rest == 1 ? 0 : a_offsetx[col][1];
b_offset[4 * col + 2] = rest <= 2 ? 0 : a_offsetx[col][2];
b_offset[4 * col + 3] = rest <= 3 ? 0 : a_offsetx[col][3];
}
b_offset += 16;
}
}

if (n & 2) {
a_offsetx[0] = a_offset;
a_offsetx[1] = a_offsetx[0] + lda;
a_offset += 2 * lda;

for (BLASLONG i = 0; i < m / 4; i++) {
v0 = svld1_bf16(pg16, (bfloat16_t *)a_offsetx[0]);
v1 = svld1_bf16(pg16, (bfloat16_t *)a_offsetx[1]);
svst1_bf16(pg16, (bfloat16_t *)b_offset, v0);
svst1_bf16(pg16, (bfloat16_t *)b_offset + 4, v1);

b_offset += 8;
a_offsetx[0] += 4;
a_offsetx[1] += 4;
}

if (m & 3) {
BLASLONG rest = m & 3;
for (BLASLONG col = 0; col < 2; col++) {
b_offset[4 * col] = a_offsetx[col][0];
b_offset[4 * col + 1] = rest == 1 ? 0 : a_offsetx[col][1];
b_offset[4 * col + 2] = rest <= 2 ? 0 : a_offsetx[col][2];
b_offset[4 * col + 3] = rest <= 3 ? 0 : a_offsetx[col][3];
}
b_offset += 8;
}
}

if (n & 1) {
a_offsetx[0] = a_offset;
for (BLASLONG i = 0; i < m / 4; i++) {
v0 = svld1_bf16(pg16, (bfloat16_t *)a_offsetx[0]);
svst1_bf16(pg16, (bfloat16_t *)b_offset, v0);
b_offset += 4;
a_offsetx[0] += 4;
}
if (m & 3) {
BLASLONG rest = m & 3;
b_offset[0] = a_offsetx[0][0];
b_offset[1] = rest == 1 ? 0 : a_offsetx[0][1];
b_offset[2] = rest <= 2 ? 0 : a_offsetx[0][2];
b_offset[3] = rest <= 3 ? 0 : a_offsetx[0][3];
}
}

return 0;
}


+ 180
- 0
kernel/arm64/sbgemm_ncopy_8_neoversev1.c View File

@@ -0,0 +1,180 @@
/***************************************************************************
* Copyright (c) 2024, The OpenBLAS Project
* All rights reserved.
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
* met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* 3. Neither the name of the OpenBLAS project nor the names of
* its contributors may be used to endorse or promote products
* derived from this software without specific prior written permission.
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
* *****************************************************************************/

#include <arm_sve.h>

#include "common.h"

int CNAME(BLASLONG m, BLASLONG n, IFLOAT *a, BLASLONG lda, IFLOAT *b) {
IFLOAT *a_offset;
IFLOAT *a_offsetx[8];
IFLOAT *b_offset;
a_offset = a;
b_offset = b;

svbool_t pg16 = svdupq_b16(1, 1, 1, 1, 0, 0, 0, 0);
svbfloat16_t v0, v1, v2, v3, v4, v5, v6, v7;

for (BLASLONG j = 0; j < n / 8; j++) {
a_offsetx[0] = a_offset;
a_offsetx[1] = a_offsetx[0] + lda;
a_offsetx[2] = a_offsetx[1] + lda;
a_offsetx[3] = a_offsetx[2] + lda;
a_offsetx[4] = a_offsetx[3] + lda;
a_offsetx[5] = a_offsetx[4] + lda;
a_offsetx[6] = a_offsetx[5] + lda;
a_offsetx[7] = a_offsetx[6] + lda;
a_offset += 8 * lda;

for (BLASLONG i = 0; i < m / 4; i++) {
v0 = svld1_bf16(pg16, (bfloat16_t *)a_offsetx[0]);
v1 = svld1_bf16(pg16, (bfloat16_t *)a_offsetx[1]);
v2 = svld1_bf16(pg16, (bfloat16_t *)a_offsetx[2]);
v3 = svld1_bf16(pg16, (bfloat16_t *)a_offsetx[3]);
v4 = svld1_bf16(pg16, (bfloat16_t *)a_offsetx[4]);
v5 = svld1_bf16(pg16, (bfloat16_t *)a_offsetx[5]);
v6 = svld1_bf16(pg16, (bfloat16_t *)a_offsetx[6]);
v7 = svld1_bf16(pg16, (bfloat16_t *)a_offsetx[7]);

svst1_bf16(pg16, (bfloat16_t *)b_offset, v0);
svst1_bf16(pg16, (bfloat16_t *)b_offset + 4, v1);
svst1_bf16(pg16, (bfloat16_t *)b_offset + 8, v2);
svst1_bf16(pg16, (bfloat16_t *)b_offset + 12, v3);
svst1_bf16(pg16, (bfloat16_t *)b_offset + 16, v4);
svst1_bf16(pg16, (bfloat16_t *)b_offset + 20, v5);
svst1_bf16(pg16, (bfloat16_t *)b_offset + 24, v6);
svst1_bf16(pg16, (bfloat16_t *)b_offset + 28, v7);

b_offset += 32;
a_offsetx[0] += 4;
a_offsetx[1] += 4;
a_offsetx[2] += 4;
a_offsetx[3] += 4;
a_offsetx[4] += 4;
a_offsetx[5] += 4;
a_offsetx[6] += 4;
a_offsetx[7] += 4;
}

if (m & 3) {
BLASLONG rest = m & 3;
for (BLASLONG col = 0; col < 8; col++) {
b_offset[4 * col] = a_offsetx[col][0];
b_offset[4 * col + 1] = rest == 1 ? 0 : a_offsetx[col][1];
b_offset[4 * col + 2] = rest <= 2 ? 0 : a_offsetx[col][2];
b_offset[4 * col + 3] = rest <= 3 ? 0 : a_offsetx[col][3];
}
b_offset += 32;
}
}

if (n & 4) {
a_offsetx[0] = a_offset;
a_offsetx[1] = a_offsetx[0] + lda;
a_offsetx[2] = a_offsetx[1] + lda;
a_offsetx[3] = a_offsetx[2] + lda;
a_offset += 4 * lda;

for (BLASLONG i = 0; i < m / 4; i++) {
v0 = svld1_bf16(pg16, (bfloat16_t *)a_offsetx[0]);
v1 = svld1_bf16(pg16, (bfloat16_t *)a_offsetx[1]);
v2 = svld1_bf16(pg16, (bfloat16_t *)a_offsetx[2]);
v3 = svld1_bf16(pg16, (bfloat16_t *)a_offsetx[3]);

svst1_bf16(pg16, (bfloat16_t *)b_offset, v0);
svst1_bf16(pg16, (bfloat16_t *)b_offset + 4, v1);
svst1_bf16(pg16, (bfloat16_t *)b_offset + 8, v2);
svst1_bf16(pg16, (bfloat16_t *)b_offset + 12, v3);

b_offset += 16;
a_offsetx[0] += 4;
a_offsetx[1] += 4;
a_offsetx[2] += 4;
a_offsetx[3] += 4;
}

if (m & 3) {
BLASLONG rest = m & 3;
for (BLASLONG col = 0; col < 4; col++) {
b_offset[4 * col] = a_offsetx[col][0];
b_offset[4 * col + 1] = rest == 1 ? 0 : a_offsetx[col][1];
b_offset[4 * col + 2] = rest <= 2 ? 0 : a_offsetx[col][2];
b_offset[4 * col + 3] = rest <= 3 ? 0 : a_offsetx[col][3];
}
b_offset += 16;
}
}

if (n & 2) {
a_offsetx[0] = a_offset;
a_offsetx[1] = a_offsetx[0] + lda;
a_offset += 2 * lda;

for (BLASLONG i = 0; i < m / 4; i++) {
v0 = svld1_bf16(pg16, (bfloat16_t *)a_offsetx[0]);
v1 = svld1_bf16(pg16, (bfloat16_t *)a_offsetx[1]);
svst1_bf16(pg16, (bfloat16_t *)b_offset, v0);
svst1_bf16(pg16, (bfloat16_t *)b_offset + 4, v1);

b_offset += 8;
a_offsetx[0] += 4;
a_offsetx[1] += 4;
}

if (m & 3) {
BLASLONG rest = m & 3;
for (BLASLONG col = 0; col < 2; col++) {
b_offset[4 * col] = a_offsetx[col][0];
b_offset[4 * col + 1] = rest == 1 ? 0 : a_offsetx[col][1];
b_offset[4 * col + 2] = rest <= 2 ? 0 : a_offsetx[col][2];
b_offset[4 * col + 3] = rest <= 3 ? 0 : a_offsetx[col][3];
}
b_offset += 8;
}
}

if (n & 1) {
a_offsetx[0] = a_offset;
for (BLASLONG i = 0; i < m / 4; i++) {
v0 = svld1_bf16(pg16, (bfloat16_t *)a_offsetx[0]);
svst1_bf16(pg16, (bfloat16_t *)b_offset, v0);
b_offset += 4;
a_offsetx[0] += 4;
}
if (m & 3) {
BLASLONG rest = m & 3;
b_offset[0] = a_offsetx[0][0];
b_offset[1] = rest == 1 ? 0 : a_offsetx[0][1];
b_offset[2] = rest <= 2 ? 0 : a_offsetx[0][2];
b_offset[3] = rest <= 3 ? 0 : a_offsetx[0][3];
}
}

return 0;
}


+ 148
- 0
kernel/arm64/sbgemm_tcopy_4_neoversev1.c View File

@@ -0,0 +1,148 @@
/***************************************************************************
* Copyright (c) 2024, The OpenBLAS Project
* All rights reserved.
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
* met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* 3. Neither the name of the OpenBLAS project nor the names of
* its contributors may be used to endorse or promote products
* derived from this software without specific prior written permission.
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
* *****************************************************************************/
#include <arm_neon.h>

#include "common.h"

int CNAME(BLASLONG m, BLASLONG n, IFLOAT *a, BLASLONG lda, IFLOAT *b) {
IFLOAT *a_offset, *a_offset0, *a_offset1, *a_offset2, *a_offset3;
IFLOAT *b_offset;
a_offset = a;
b_offset = b;

uint16x4_t v0_h, v1_h, v2_h, v3_h, v4_h, v5_h, v6_h, v7_h;

for (BLASLONG j = 0; j < n / 4; j++) {
a_offset0 = a_offset;
a_offset1 = a_offset0 + lda;
a_offset2 = a_offset1 + lda;
a_offset3 = a_offset2 + lda;
a_offset += 4;

for (BLASLONG i = 0; i < m / 4; i++) {
v0_h = vld1_u16(a_offset0);
v1_h = vld1_u16(a_offset1);
v2_h = vld1_u16(a_offset2);
v3_h = vld1_u16(a_offset3);

v4_h = vtrn1_u16(v0_h, v1_h);
v5_h = vtrn2_u16(v0_h, v1_h);
v6_h = vtrn1_u16(v2_h, v3_h);
v7_h = vtrn2_u16(v2_h, v3_h);

v0_h = (uint16x4_t)vtrn1_u32((uint32x2_t)v4_h, (uint32x2_t)v6_h);
v1_h = (uint16x4_t)vtrn1_u32((uint32x2_t)v5_h, (uint32x2_t)v7_h);
v2_h = (uint16x4_t)vtrn2_u32((uint32x2_t)v4_h, (uint32x2_t)v6_h);
v3_h = (uint16x4_t)vtrn2_u32((uint32x2_t)v5_h, (uint32x2_t)v7_h);

vst1_u16(b_offset, v0_h);
vst1_u16(b_offset + 4, v1_h);
vst1_u16(b_offset + 8, v2_h);
vst1_u16(b_offset + 12, v3_h);

b_offset += 16;
a_offset0 += 4 * lda;
a_offset1 += 4 * lda;
a_offset2 += 4 * lda;
a_offset3 += 4 * lda;
}

if (m & 3) {
BLASLONG rest = m & 3;
for (BLASLONG line = 0; line < 4; line++) {
b_offset[line * 4] = a_offset0[line];
b_offset[line * 4 + 1] = rest == 1 ? 0 : a_offset1[line];
b_offset[line * 4 + 2] = rest <= 2 ? 0 : a_offset2[line];
b_offset[line * 4 + 3] = rest <= 3 ? 0 : a_offset3[line];
}
b_offset += 16;
}
}

if (n & 2) {
a_offset0 = a_offset;
a_offset1 = a_offset0 + lda;
a_offset2 = a_offset1 + lda;
a_offset3 = a_offset2 + lda;
a_offset += 2;

for (BLASLONG i = 0; i < m / 4; i++) {
for (BLASLONG line = 0; line < 2; line++) {
b_offset[line * 4] = a_offset0[line];
b_offset[line * 4 + 1] = a_offset1[line];
b_offset[line * 4 + 2] = a_offset2[line];
b_offset[line * 4 + 3] = a_offset3[line];
}
b_offset += 8;
a_offset0 += 4 * lda;
a_offset1 += 4 * lda;
a_offset2 += 4 * lda;
a_offset3 += 4 * lda;
}

if (m & 3) {
BLASLONG rest = m & 3;
for (BLASLONG line = 0; line < 2; line++) {
b_offset[line * 4] = a_offset0[line];
b_offset[line * 4 + 1] = rest == 1 ? 0 : a_offset1[line];
b_offset[line * 4 + 2] = rest <= 2 ? 0 : a_offset2[line];
b_offset[line * 4 + 3] = rest <= 3 ? 0 : a_offset3[line];
}
b_offset += 8;
}
}

if (n & 1) {
a_offset0 = a_offset;
a_offset1 = a_offset0 + lda;
a_offset2 = a_offset1 + lda;
a_offset3 = a_offset2 + lda;

for (BLASLONG i = 0; i < m / 4; i++) {
b_offset[0] = *a_offset0;
b_offset[1] = *a_offset1;
b_offset[2] = *a_offset2;
b_offset[3] = *a_offset3;
b_offset += 4;
a_offset0 += 4 * lda;
a_offset1 += 4 * lda;
a_offset2 += 4 * lda;
a_offset3 += 4 * lda;
}

if (m & 3) {
BLASLONG rest = m & 3;
b_offset[0] = *a_offset0;
b_offset[1] = rest == 1 ? 0 : *a_offset1;
b_offset[2] = rest <= 2 ? 0 : *a_offset2;
b_offset[3] = rest <= 3 ? 0 : *a_offset3;
}
}
return 0;
}


+ 200
- 0
kernel/arm64/sbgemm_tcopy_8_neoversev1.c View File

@@ -0,0 +1,200 @@
/***************************************************************************
* Copyright (c) 2024, The OpenBLAS Project
* All rights reserved.
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
* met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* 3. Neither the name of the OpenBLAS project nor the names of
* its contributors may be used to endorse or promote products
* derived from this software without specific prior written permission.
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
* *****************************************************************************/
#include <arm_neon.h>

#include "common.h"

int CNAME(BLASLONG m, BLASLONG n, IFLOAT *a, BLASLONG lda, IFLOAT *b) {
IFLOAT *a_offset, *a_offset0, *a_offset1, *a_offset2, *a_offset3;
IFLOAT *b_offset;
a_offset = a;
b_offset = b;

uint16x8_t v0, v1, v2, v3, v4, v5, v6, v7;
uint16x4_t v0_h, v1_h, v2_h, v3_h, v4_h, v5_h, v6_h, v7_h;

for (BLASLONG j = 0; j < n / 8; j++) {
a_offset0 = a_offset;
a_offset1 = a_offset0 + lda;
a_offset2 = a_offset1 + lda;
a_offset3 = a_offset2 + lda;
a_offset += 8;

for (BLASLONG i = 0; i < m / 4; i++) {
v0 = vld1q_u16(a_offset0);
v1 = vld1q_u16(a_offset1);
v2 = vld1q_u16(a_offset2);
v3 = vld1q_u16(a_offset3);

v4 = vtrn1q_u16(v0, v1);
v5 = vtrn2q_u16(v0, v1);
v6 = vtrn1q_u16(v2, v3);
v7 = vtrn2q_u16(v2, v3);

v0 = (uint16x8_t)vtrn1q_u32((uint32x4_t)v4, (uint32x4_t)v6);
v1 = (uint16x8_t)vtrn1q_u32((uint32x4_t)v5, (uint32x4_t)v7);
v2 = (uint16x8_t)vtrn2q_u32((uint32x4_t)v4, (uint32x4_t)v6);
v3 = (uint16x8_t)vtrn2q_u32((uint32x4_t)v5, (uint32x4_t)v7);

vst1_u16(b_offset, vget_low_u16(v0));
vst1_u16(b_offset + 4, vget_low_u16(v1));
vst1_u16(b_offset + 8, vget_low_u16(v2));
vst1_u16(b_offset + 12, vget_low_u16(v3));
vst1_u16(b_offset + 16, vget_high_u16(v0));
vst1_u16(b_offset + 20, vget_high_u16(v1));
vst1_u16(b_offset + 24, vget_high_u16(v2));
vst1_u16(b_offset + 28, vget_high_u16(v3));

b_offset += 32;
a_offset0 += 4 * lda;
a_offset1 += 4 * lda;
a_offset2 += 4 * lda;
a_offset3 += 4 * lda;
}

if (m & 3) {
BLASLONG rest = m & 3;
for (BLASLONG line = 0; line < 8; line++) {
b_offset[line * 4] = a_offset0[line];
b_offset[line * 4 + 1] = rest == 1 ? 0 : a_offset1[line];
b_offset[line * 4 + 2] = rest <= 2 ? 0 : a_offset2[line];
b_offset[line * 4 + 3] = rest <= 3 ? 0 : a_offset3[line];
}
b_offset += 32;
}
}

if (n & 4) {
a_offset0 = a_offset;
a_offset1 = a_offset0 + lda;
a_offset2 = a_offset1 + lda;
a_offset3 = a_offset2 + lda;
a_offset += 4;

for (BLASLONG i = 0; i < m / 4; i++) {
v0_h = vld1_u16(a_offset0);
v1_h = vld1_u16(a_offset1);
v2_h = vld1_u16(a_offset2);
v3_h = vld1_u16(a_offset3);

v4_h = vtrn1_u16(v0_h, v1_h);
v5_h = vtrn2_u16(v0_h, v1_h);
v6_h = vtrn1_u16(v2_h, v3_h);
v7_h = vtrn2_u16(v2_h, v3_h);

v0_h = (uint16x4_t)vtrn1_u32((uint32x2_t)v4_h, (uint32x2_t)v6_h);
v1_h = (uint16x4_t)vtrn1_u32((uint32x2_t)v5_h, (uint32x2_t)v7_h);
v2_h = (uint16x4_t)vtrn2_u32((uint32x2_t)v4_h, (uint32x2_t)v6_h);
v3_h = (uint16x4_t)vtrn2_u32((uint32x2_t)v5_h, (uint32x2_t)v7_h);

vst1_u16(b_offset, v0_h);
vst1_u16(b_offset + 4, v1_h);
vst1_u16(b_offset + 8, v2_h);
vst1_u16(b_offset + 12, v3_h);

b_offset += 16;
a_offset0 += 4 * lda;
a_offset1 += 4 * lda;
a_offset2 += 4 * lda;
a_offset3 += 4 * lda;
}

if (m & 3) {
BLASLONG rest = m & 3;
for (BLASLONG line = 0; line < 4; line++) {
b_offset[line * 4] = a_offset0[line];
b_offset[line * 4 + 1] = rest == 1 ? 0 : a_offset1[line];
b_offset[line * 4 + 2] = rest <= 2 ? 0 : a_offset2[line];
b_offset[line * 4 + 3] = rest <= 3 ? 0 : a_offset3[line];
}
b_offset += 16;
}
}

if (n & 2) {
a_offset0 = a_offset;
a_offset1 = a_offset0 + lda;
a_offset2 = a_offset1 + lda;
a_offset3 = a_offset2 + lda;
a_offset += 2;

for (BLASLONG i = 0; i < m / 4; i++) {
for (BLASLONG line = 0; line < 2; line++) {
b_offset[line * 4] = a_offset0[line];
b_offset[line * 4 + 1] = a_offset1[line];
b_offset[line * 4 + 2] = a_offset2[line];
b_offset[line * 4 + 3] = a_offset3[line];
}
b_offset += 8;
a_offset0 += 4 * lda;
a_offset1 += 4 * lda;
a_offset2 += 4 * lda;
a_offset3 += 4 * lda;
}

if (m & 3) {
BLASLONG rest = m & 3;
for (BLASLONG line = 0; line < 2; line++) {
b_offset[line * 4] = a_offset0[line];
b_offset[line * 4 + 1] = rest == 1 ? 0 : a_offset1[line];
b_offset[line * 4 + 2] = rest <= 2 ? 0 : a_offset2[line];
b_offset[line * 4 + 3] = rest <= 3 ? 0 : a_offset3[line];
}
b_offset += 8;
}
}

if (n & 1) {
a_offset0 = a_offset;
a_offset1 = a_offset0 + lda;
a_offset2 = a_offset1 + lda;
a_offset3 = a_offset2 + lda;

for (BLASLONG i = 0; i < m / 4; i++) {
b_offset[0] = *a_offset0;
b_offset[1] = *a_offset1;
b_offset[2] = *a_offset2;
b_offset[3] = *a_offset3;
b_offset += 4;
a_offset0 += 4 * lda;
a_offset1 += 4 * lda;
a_offset2 += 4 * lda;
a_offset3 += 4 * lda;
}

if (m & 3) {
BLASLONG rest = m & 3;
b_offset[0] = *a_offset0;
b_offset[1] = rest == 1 ? 0 : *a_offset1;
b_offset[2] = rest <= 2 ? 0 : *a_offset2;
b_offset[3] = rest <= 3 ? 0 : *a_offset3;
}
}
return 0;
}


+ 7
- 0
param.h View File

@@ -3553,6 +3553,13 @@ is a big desktop or server with abundant cache rather than a phone or embedded d
#define SWITCH_RATIO 16
#define GEMM_PREFERED_SIZE 8
#endif
#undef SBGEMM_ALIGN_K
#define SBGEMM_ALIGN_K 4

#undef SBGEMM_DEFAULT_UNROLL_M
#undef SBGEMM_DEFAULT_UNROLL_N
#define SBGEMM_DEFAULT_UNROLL_M 8
#define SBGEMM_DEFAULT_UNROLL_N 4

#define SGEMM_DEFAULT_UNROLL_M 16
#define SGEMM_DEFAULT_UNROLL_N 8


Write Preview
Loading…
Cancel
Save