From 8279e68805e779cfe7f599ebc9ab1e9a427b5eaf Mon Sep 17 00:00:00 2001 From: Sharif Inamdar Date: Mon, 9 Jun 2025 09:25:45 +0000 Subject: [PATCH 01/36] Optimize gemv_n_sve_v1x3 kernel - Calculate predicate outside the loop - Divide matrix in blocks of 3 --- CONTRIBUTORS.md | 5 +- kernel/arm64/gemv_n_sve_v1x3.c | 96 ++++++++++++++++++++-------------- 2 files changed, 61 insertions(+), 40 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index d8f57ef60..35519e900 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -253,4 +253,7 @@ In chronological order: * [2025-02-27] Add sbgemv_n_neon kernel * Abhishek Kumar - * [2025-04-22] Optimise dot kernel for NEOVERSE V1 \ No newline at end of file + * [2025-04-22] Optimise dot kernel for NEOVERSE V1 + +* Sharif Inamdar + * [2025-06-05] Optimize gemv_n_sve_v1x3 kernel \ No newline at end of file diff --git a/kernel/arm64/gemv_n_sve_v1x3.c b/kernel/arm64/gemv_n_sve_v1x3.c index d6aa3d389..5ab8d3a16 100644 --- a/kernel/arm64/gemv_n_sve_v1x3.c +++ b/kernel/arm64/gemv_n_sve_v1x3.c @@ -52,17 +52,17 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha, FLOAT *a, BLASLONG lda, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y, FLOAT *buffer) { - BLASLONG i; - BLASLONG ix,iy; - BLASLONG j; - FLOAT *a_ptr; + BLASLONG i, j; + BLASLONG ix = 0; + BLASLONG iy; + FLOAT *a_ptr = a; FLOAT temp; - ix = 0; - a_ptr = a; - if (inc_y == 1) { - BLASLONG width = (n + 3 - 1) / 3; + BLASLONG width = n / 3; // Only process full 3-column blocks + BLASLONG sve_size = SV_COUNT(); + svbool_t pg_full = SV_TRUE(); + svbool_t pg_tail = SV_WHILE(0, m % sve_size); FLOAT *a0_ptr = a_ptr + lda * width * 0; FLOAT *a1_ptr = a_ptr + lda * width * 1; @@ -73,57 +73,75 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha, FLOAT *a, FLOAT *x2_ptr = x + inc_x * width * 2; for (j = 0; j < width; j++) { - svbool_t pg00 = ((j + width * 0) < n) ? SV_TRUE() : svpfalse(); - svbool_t pg01 = ((j + width * 1) < n) ? SV_TRUE() : svpfalse(); - svbool_t pg02 = ((j + width * 2) < n) ? SV_TRUE() : svpfalse(); + SV_TYPE temp0_vec = SV_DUP(alpha * x0_ptr[ix]); + SV_TYPE temp1_vec = SV_DUP(alpha * x1_ptr[ix]); + SV_TYPE temp2_vec = SV_DUP(alpha * x2_ptr[ix]); - SV_TYPE temp0_vec = ((j + width * 0) < n) ? SV_DUP(alpha * x0_ptr[ix]) : SV_DUP(0.0); - SV_TYPE temp1_vec = ((j + width * 1) < n) ? SV_DUP(alpha * x1_ptr[ix]) : SV_DUP(0.0); - SV_TYPE temp2_vec = ((j + width * 2) < n) ? SV_DUP(alpha * x2_ptr[ix]) : SV_DUP(0.0); i = 0; - BLASLONG sve_size = SV_COUNT(); - while ((i + sve_size * 1 - 1) < m) { - SV_TYPE y0_vec = svld1_vnum(SV_TRUE(), y + i, 0); + while ((i + sve_size - 1) < m) { + SV_TYPE y0_vec = svld1(pg_full, y + i); - SV_TYPE a00_vec = svld1_vnum(pg00, a0_ptr + i, 0); - SV_TYPE a01_vec = svld1_vnum(pg01, a1_ptr + i, 0); - SV_TYPE a02_vec = svld1_vnum(pg02, a2_ptr + i, 0); + SV_TYPE a00_vec = svld1(pg_full, a0_ptr + i); + SV_TYPE a01_vec = svld1(pg_full, a1_ptr + i); + SV_TYPE a02_vec = svld1(pg_full, a2_ptr + i); - y0_vec = svmla_m(pg00, y0_vec, temp0_vec, a00_vec); - y0_vec = svmla_m(pg01, y0_vec, temp1_vec, a01_vec); - y0_vec = svmla_m(pg02, y0_vec, temp2_vec, a02_vec); + y0_vec = svmla_x(pg_full, y0_vec, temp0_vec, a00_vec); + y0_vec = svmla_x(pg_full, y0_vec, temp1_vec, a01_vec); + y0_vec = svmla_x(pg_full, y0_vec, temp2_vec, a02_vec); - svst1_vnum(SV_TRUE(), y + i, 0, y0_vec); - i += sve_size * 1; + svst1(pg_full, y + i, y0_vec); + i += sve_size; } if (i < m) { - svbool_t pg0 = SV_WHILE(i + sve_size * 0, m); - - pg00 = svand_z(SV_TRUE(), pg0, pg00); - pg01 = svand_z(SV_TRUE(), pg0, pg01); - pg02 = svand_z(SV_TRUE(), pg0, pg02); + SV_TYPE y0_vec = svld1(pg_tail, y + i); - SV_TYPE y0_vec = svld1_vnum(pg0, y + i, 0); + SV_TYPE a00_vec = svld1(pg_tail, a0_ptr + i); + SV_TYPE a01_vec = svld1(pg_tail, a1_ptr + i); + SV_TYPE a02_vec = svld1(pg_tail, a2_ptr + i); - SV_TYPE a00_vec = svld1_vnum(pg00, a0_ptr + i, 0); - SV_TYPE a01_vec = svld1_vnum(pg01, a1_ptr + i, 0); - SV_TYPE a02_vec = svld1_vnum(pg02, a2_ptr + i, 0); + y0_vec = svmla_m(pg_tail, y0_vec, temp0_vec, a00_vec); + y0_vec = svmla_m(pg_tail, y0_vec, temp1_vec, a01_vec); + y0_vec = svmla_m(pg_tail, y0_vec, temp2_vec, a02_vec); - y0_vec = svmla_m(pg00, y0_vec, temp0_vec, a00_vec); - y0_vec = svmla_m(pg01, y0_vec, temp1_vec, a01_vec); - y0_vec = svmla_m(pg02, y0_vec, temp2_vec, a02_vec); - - svst1_vnum(pg0, y + i, 0, y0_vec); + svst1(pg_tail, y + i, y0_vec); } a0_ptr += lda; a1_ptr += lda; a2_ptr += lda; ix += inc_x; } + // Handle remaining n % 3 columns + for (j = width * 3; j < n; j++) { + FLOAT *a_col = a + j * lda; + temp = alpha * x[j * inc_x]; + SV_TYPE temp_vec = SV_DUP(temp); + + i = 0; + while ((i + sve_size - 1) < m) { + SV_TYPE y_vec = svld1(pg_full, y + i); + + SV_TYPE a_vec = svld1(pg_full, a_col + i); + + y_vec = svmla_x(pg_full, y_vec, temp_vec, a_vec); + + svst1(pg_full, y + i, y_vec); + i += sve_size; + } + if (i < m) { + SV_TYPE y_vec = svld1(pg_tail, y + i); + + SV_TYPE a_vec = svld1(pg_tail, a_col + i); + + y_vec = svmla_m(pg_tail, y_vec, temp_vec, a_vec); + + svst1(pg_tail, y + i, y_vec); + } + } return(0); } + // Fallback scalar loop for (j = 0; j < n; j++) { temp = alpha * x[ix]; iy = 0; From 1dde4a13c0d5749ac9befaacab6f5ec9a4df6c35 Mon Sep 17 00:00:00 2001 From: pratiklp00 Date: Thu, 26 Jun 2025 00:03:38 -0500 Subject: [PATCH 02/36] p11 changes --- TargetList.txt | 1 + cpuid_power.c | 4 ++++ driver/others/dynamic_power.c | 10 ++++++++++ getarch.c | 2 +- 4 files changed, 16 insertions(+), 1 deletion(-) diff --git a/TargetList.txt b/TargetList.txt index 232e12ffa..0c94869e7 100644 --- a/TargetList.txt +++ b/TargetList.txt @@ -52,6 +52,7 @@ POWER7 POWER8 POWER9 POWER10 +POWER11 PPCG4 PPC970 PPC970MP diff --git a/cpuid_power.c b/cpuid_power.c index 1ced8930a..2b2c32eea 100644 --- a/cpuid_power.c +++ b/cpuid_power.c @@ -131,6 +131,7 @@ int detect(void){ if (!strncasecmp(p, "POWER8", 6)) return CPUTYPE_POWER8; if (!strncasecmp(p, "POWER9", 6)) return CPUTYPE_POWER9; if (!strncasecmp(p, "POWER10", 7)) return CPUTYPE_POWER10; + if (!strncasecmp(p, "POWER11", 7)) return CPUTYPE_POWER10; if (!strncasecmp(p, "Cell", 4)) return CPUTYPE_CELL; if (!strncasecmp(p, "7447", 4)) return CPUTYPE_PPCG4; @@ -171,6 +172,9 @@ int detect(void){ int id; __asm __volatile("mfpvr %0" : "=r"(id)); switch ( id >> 16 ) { + case 0x82: // POWER11 + return CPUTYPE_POWER10; + break; case 0x80: // POWER10 return CPUTYPE_POWER10; break; diff --git a/driver/others/dynamic_power.c b/driver/others/dynamic_power.c index 4c1f4a26e..f0afe3eb6 100644 --- a/driver/others/dynamic_power.c +++ b/driver/others/dynamic_power.c @@ -70,6 +70,9 @@ static int cpuid(void) #endif #ifdef POWER_10 else if (arch >= POWER_10) return CPU_POWER10; +#endif +#ifdef POWER_11 + else if (arch >= POWER_11) return CPU_POWER10; #endif return CPU_UNKNOWN; } @@ -173,6 +176,13 @@ static struct { .cpu_type = CPU_POWER10, }, + { /* Power11 */ + .pvr_mask = 0xffff0000, + .pvr_value = 0x00820000, + .cpu_name = "POWER11 (raw)", + .cpu_type = CPU_POWER10, + }, + { /* End of table, pvr_mask and pvr_value must be zero */ .pvr_mask = 0x0, .pvr_value = 0x0, diff --git a/getarch.c b/getarch.c index cb0b3cd7c..76c1aaff8 100644 --- a/getarch.c +++ b/getarch.c @@ -836,7 +836,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define CORENAME "POWER9" #endif -#if defined(FORCE_POWER10) +#if defined(FORCE_POWER10) || (FORCE_POWER11) #define FORCE #define ARCHITECTURE "POWER" #define SUBARCHITECTURE "POWER10" From 7f360001f9261350be0d69c5fbe5c7d5b8aaf768 Mon Sep 17 00:00:00 2001 From: Lina Iyer <140435362+ilina-linaro@users.noreply.github.com> Date: Thu, 3 Jul 2025 07:15:20 -0600 Subject: [PATCH 03/36] Update README.md to include Windows on Arm64 Update README.md to indicate that binaries are available for Windows on ARM64 --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index cc9325d39..6a4fc6a7f 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ For a general introduction to the BLAS routines, please refer to the extensive d We provide official binary packages for the following platform: * Windows x86/x86_64 + * Windows arm64 (woa) You can download them from [file hosting on sourceforge.net](https://sourceforge.net/projects/openblas/files/) or from the [Releases section of the GitHub project page](https://github.com/OpenMathLib/OpenBLAS/releases). From f7ad906b49c23efaa712991eacc4ce368161c0ca Mon Sep 17 00:00:00 2001 From: "Iha, Taisei" Date: Fri, 4 Jul 2025 22:57:44 +0900 Subject: [PATCH 04/36] Performance improvements of [SD]DOT with loop-unrolling on A64FX --- kernel/arm64/KERNEL.A64FX | 3 + kernel/arm64/dot.c | 4 + kernel/arm64/dot_kernel_sve_v8.c | 146 +++++++++++++++++++++++++++++++ kernel/arm64/dot_sve_v8.c | 34 +++++++ 4 files changed, 187 insertions(+) create mode 100644 kernel/arm64/dot_kernel_sve_v8.c create mode 100644 kernel/arm64/dot_sve_v8.c diff --git a/kernel/arm64/KERNEL.A64FX b/kernel/arm64/KERNEL.A64FX index 3d68271da..e5e6a9d69 100644 --- a/kernel/arm64/KERNEL.A64FX +++ b/kernel/arm64/KERNEL.A64FX @@ -4,3 +4,6 @@ SGEMVNKERNEL = gemv_n_sve_v4x3.c DGEMVNKERNEL = gemv_n_sve_v4x3.c SGEMVTKERNEL = gemv_t_sve_v4x3.c DGEMVTKERNEL = gemv_t_sve_v4x3.c + +DDOTKERNEL = dot_sve_v8.c +SDOTKERNEL = dot_sve_v8.c diff --git a/kernel/arm64/dot.c b/kernel/arm64/dot.c index ece31bccd..56a8a36dc 100644 --- a/kernel/arm64/dot.c +++ b/kernel/arm64/dot.c @@ -40,8 +40,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif #ifdef USE_SVE +#ifdef DOT_KERNEL_SVE +#include DOT_KERNEL_SVE +#else #include "dot_kernel_sve.c" #endif +#endif #include "dot_kernel_asimd.c" #if defined(SMP) diff --git a/kernel/arm64/dot_kernel_sve_v8.c b/kernel/arm64/dot_kernel_sve_v8.c new file mode 100644 index 000000000..fa18553b1 --- /dev/null +++ b/kernel/arm64/dot_kernel_sve_v8.c @@ -0,0 +1,146 @@ +/*************************************************************************** +Copyright (c) 2025, The OpenBLAS Project +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + 3. Neither the name of the OpenBLAS project nor the names of + its contributors may be used to endorse or promote products + derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#include +#include "common.h" + +#ifdef DOUBLE +#define SV_COUNT svcntd +#define SV_TYPE svfloat64_t +#define SV_TRUE svptrue_b64 +#define SV_WHILE svwhilelt_b64_s64 +#define SV_DUP svdup_f64 +#else +#define SV_COUNT svcntw +#define SV_TYPE svfloat32_t +#define SV_TRUE svptrue_b32 +#define SV_WHILE svwhilelt_b32_s64 +#define SV_DUP svdup_f32 +#endif + +static FLOAT dot_kernel_sve(BLASLONG n, FLOAT* x, FLOAT* y) +{ + SV_TYPE temp0_vec = SV_DUP(0.0); + SV_TYPE temp1_vec = SV_DUP(0.0); + SV_TYPE temp2_vec = SV_DUP(0.0); + SV_TYPE temp3_vec = SV_DUP(0.0); + SV_TYPE temp4_vec = SV_DUP(0.0); + SV_TYPE temp5_vec = SV_DUP(0.0); + SV_TYPE temp6_vec = SV_DUP(0.0); + SV_TYPE temp7_vec = SV_DUP(0.0); + + BLASLONG i = 0; + BLASLONG sve_size = SV_COUNT(); + + while ((i + sve_size * 8 - 1) < n) { + FLOAT *x0_ptr = x + i; + SV_TYPE x0_vec = svld1_vnum(SV_TRUE(), x0_ptr, 0); + SV_TYPE x1_vec = svld1_vnum(SV_TRUE(), x0_ptr, 1); + SV_TYPE x2_vec = svld1_vnum(SV_TRUE(), x0_ptr, 2); + SV_TYPE x3_vec = svld1_vnum(SV_TRUE(), x0_ptr, 3); + SV_TYPE x4_vec = svld1_vnum(SV_TRUE(), x0_ptr, 4); + SV_TYPE x5_vec = svld1_vnum(SV_TRUE(), x0_ptr, 5); + SV_TYPE x6_vec = svld1_vnum(SV_TRUE(), x0_ptr, 6); + SV_TYPE x7_vec = svld1_vnum(SV_TRUE(), x0_ptr, 7); + + FLOAT *y0_ptr = y + i; + SV_TYPE y0_vec = svld1_vnum(SV_TRUE(), y0_ptr, 0); + SV_TYPE y1_vec = svld1_vnum(SV_TRUE(), y0_ptr, 1); + SV_TYPE y2_vec = svld1_vnum(SV_TRUE(), y0_ptr, 2); + SV_TYPE y3_vec = svld1_vnum(SV_TRUE(), y0_ptr, 3); + SV_TYPE y4_vec = svld1_vnum(SV_TRUE(), y0_ptr, 4); + SV_TYPE y5_vec = svld1_vnum(SV_TRUE(), y0_ptr, 5); + SV_TYPE y6_vec = svld1_vnum(SV_TRUE(), y0_ptr, 6); + SV_TYPE y7_vec = svld1_vnum(SV_TRUE(), y0_ptr, 7); + + temp0_vec = svmla_x(SV_TRUE(), temp0_vec, x0_vec, y0_vec); + temp1_vec = svmla_x(SV_TRUE(), temp1_vec, x1_vec, y1_vec); + temp2_vec = svmla_x(SV_TRUE(), temp2_vec, x2_vec, y2_vec); + temp3_vec = svmla_x(SV_TRUE(), temp3_vec, x3_vec, y3_vec); + temp4_vec = svmla_x(SV_TRUE(), temp4_vec, x4_vec, y4_vec); + temp5_vec = svmla_x(SV_TRUE(), temp5_vec, x5_vec, y5_vec); + temp6_vec = svmla_x(SV_TRUE(), temp6_vec, x6_vec, y6_vec); + temp7_vec = svmla_x(SV_TRUE(), temp7_vec, x7_vec, y7_vec); + + i += sve_size * 8; + } + + if (i < n) { + svbool_t pg0 = SV_WHILE(i + sve_size * 0, n); + svbool_t pg1 = SV_WHILE(i + sve_size * 1, n); + svbool_t pg2 = SV_WHILE(i + sve_size * 2, n); + svbool_t pg3 = SV_WHILE(i + sve_size * 3, n); + svbool_t pg4 = SV_WHILE(i + sve_size * 4, n); + svbool_t pg5 = SV_WHILE(i + sve_size * 5, n); + svbool_t pg6 = SV_WHILE(i + sve_size * 6, n); + svbool_t pg7 = SV_WHILE(i + sve_size * 7, n); + + FLOAT *x0_ptr = x + i; + SV_TYPE x0_vec = svld1_vnum(pg0, x0_ptr, 0); + SV_TYPE x1_vec = svld1_vnum(pg1, x0_ptr, 1); + SV_TYPE x2_vec = svld1_vnum(pg2, x0_ptr, 2); + SV_TYPE x3_vec = svld1_vnum(pg3, x0_ptr, 3); + SV_TYPE x4_vec = svld1_vnum(pg4, x0_ptr, 4); + SV_TYPE x5_vec = svld1_vnum(pg5, x0_ptr, 5); + SV_TYPE x6_vec = svld1_vnum(pg6, x0_ptr, 6); + SV_TYPE x7_vec = svld1_vnum(pg7, x0_ptr, 7); + + FLOAT *y0_ptr = y + i; + SV_TYPE y0_vec = svld1_vnum(pg0, y0_ptr, 0); + SV_TYPE y1_vec = svld1_vnum(pg1, y0_ptr, 1); + SV_TYPE y2_vec = svld1_vnum(pg2, y0_ptr, 2); + SV_TYPE y3_vec = svld1_vnum(pg3, y0_ptr, 3); + SV_TYPE y4_vec = svld1_vnum(pg4, y0_ptr, 4); + SV_TYPE y5_vec = svld1_vnum(pg5, y0_ptr, 5); + SV_TYPE y6_vec = svld1_vnum(pg6, y0_ptr, 6); + SV_TYPE y7_vec = svld1_vnum(pg7, y0_ptr, 7); + + temp0_vec = svmla_m(pg0, temp0_vec, x0_vec, y0_vec); + temp1_vec = svmla_m(pg1, temp1_vec, x1_vec, y1_vec); + temp2_vec = svmla_m(pg2, temp2_vec, x2_vec, y2_vec); + temp3_vec = svmla_m(pg3, temp3_vec, x3_vec, y3_vec); + temp4_vec = svmla_m(pg4, temp4_vec, x4_vec, y4_vec); + temp5_vec = svmla_m(pg5, temp5_vec, x5_vec, y5_vec); + temp6_vec = svmla_m(pg6, temp6_vec, x6_vec, y6_vec); + temp7_vec = svmla_m(pg7, temp7_vec, x7_vec, y7_vec); + } + + temp0_vec = svadd_x(SV_TRUE(), temp0_vec, temp1_vec); + temp2_vec = svadd_x(SV_TRUE(), temp2_vec, temp3_vec); + temp4_vec = svadd_x(SV_TRUE(), temp4_vec, temp5_vec); + temp6_vec = svadd_x(SV_TRUE(), temp6_vec, temp7_vec); + temp0_vec = svadd_x(SV_TRUE(), temp0_vec, temp2_vec); + temp4_vec = svadd_x(SV_TRUE(), temp4_vec, temp6_vec); + temp0_vec = svadd_x(SV_TRUE(), temp0_vec, temp4_vec); + + return svaddv(SV_TRUE(), temp0_vec); +} diff --git a/kernel/arm64/dot_sve_v8.c b/kernel/arm64/dot_sve_v8.c new file mode 100644 index 000000000..ace47b4e7 --- /dev/null +++ b/kernel/arm64/dot_sve_v8.c @@ -0,0 +1,34 @@ +/*************************************************************************** +Copyright (c) 2025, The OpenBLAS Project +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + 3. Neither the name of the OpenBLAS project nor the names of + its contributors may be used to endorse or promote products + derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#define DOT_KERNEL_SVE "dot_kernel_sve_v8.c" +#include "dot.c" From 901de8f33aa14efc5ae836c9b04b38d0e2e5a0c1 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Sat, 5 Jul 2025 20:35:16 +0200 Subject: [PATCH 05/36] remove lapacke_mangling.h and add la_xisnan.mod --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 8294da4d4..d14d438a2 100644 --- a/.gitignore +++ b/.gitignore @@ -13,8 +13,8 @@ lapack-3.4.1.tgz lapack-3.4.2 lapack-3.4.2.tgz lapack-netlib/make.inc -lapack-netlib/lapacke/include/lapacke_mangling.h lapack-netlib/SRC/la_constants.mod +lapack-netlib/SRC/la_xisnan.mod lapack-netlib/TESTING/testing_results.txt lapack-netlib/INSTALL/test* lapack-netlib/TESTING/xeigtstc From 8d11e4630c1e92437636d016f35ad48395b62b4f Mon Sep 17 00:00:00 2001 From: Sertonix Date: Sun, 6 Jul 2025 23:48:10 +0200 Subject: [PATCH 06/36] Fix cpuid.S on arm The ARM assembly syntax differs a bit Fixes 61b9339d3a1f getarch/cpuid.S: Fix warning about executable stack Signed-off-by: Sertonix --- cpuid.S | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cpuid.S b/cpuid.S index 295917bdb..a8f5f8b57 100644 --- a/cpuid.S +++ b/cpuid.S @@ -66,5 +66,9 @@ _cpuid: #endif #if defined(__ELF__) && defined(__linux__) +#if defined(__arm__) + .section .note.GNU-stack,"",%progbits +#else .section .note.GNU-stack,"",@progbits #endif +#endif From 7a97c4ca9770b92f9e8186f520d2a156d53036d5 Mon Sep 17 00:00:00 2001 From: Chris Sidebottom Date: Mon, 7 Jul 2025 10:03:26 +0000 Subject: [PATCH 07/36] Rename HALF -> BFLOAT16 in some more places --- driver/level3/level3.c | 3 ++- driver/level3/level3_thread.c | 4 ++-- lapack/CMakeLists.txt | 2 +- lapack/potrf/potrf_parallel.c | 3 ++- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/driver/level3/level3.c b/driver/level3/level3.c index b7328876b..5d3438450 100644 --- a/driver/level3/level3.c +++ b/driver/level3/level3.c @@ -1,5 +1,6 @@ /*********************************************************************/ /* Copyright 2009, 2010 The University of Texas at Austin. */ +/* Copyright 2025 The OpenBLAS Project. */ /* All rights reserved. */ /* */ /* Redistribution and use in source and binary forms, with or */ @@ -305,7 +306,7 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, } BLASLONG pad_min_l = min_l; -#if defined(HALF) +#if defined(BFLOAT16) #if defined(DYNAMIC_ARCH) pad_min_l = (min_l + gotoblas->sbgemm_align_k - 1) & ~(gotoblas->sbgemm_align_k-1); #else diff --git a/driver/level3/level3_thread.c b/driver/level3/level3_thread.c index db3bffc10..5ede6153e 100644 --- a/driver/level3/level3_thread.c +++ b/driver/level3/level3_thread.c @@ -1,6 +1,6 @@ /*********************************************************************/ /* Copyright 2009, 2010 The University of Texas at Austin. */ -/* Copyright 2023 The OpenBLAS Project. */ +/* Copyright 2023, 2025 The OpenBLAS Project. */ /* All rights reserved. */ /* */ /* Redistribution and use in source and binary forms, with or */ @@ -324,7 +324,7 @@ static int inner_thread(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, BLASLONG pad_min_l = min_l; -#if defined(HALF) +#if defined(BFLOAT16) #if defined(DYNAMIC_ARCH) pad_min_l = (min_l + gotoblas->sbgemm_align_k - 1) & ~(gotoblas->sbgemm_align_k-1); #else diff --git a/lapack/CMakeLists.txt b/lapack/CMakeLists.txt index 71df986af..f4a4ee5be 100644 --- a/lapack/CMakeLists.txt +++ b/lapack/CMakeLists.txt @@ -52,7 +52,7 @@ GenerateNamedObjects("laswp/generic/laswp_k_4.c" "" "laswp_plus" false "" "" fa GenerateNamedObjects("laswp/generic/laswp_k_4.c" "MINUS" "laswp_minus" false "" "" false 3) foreach (float_type ${FLOAT_TYPES}) -if (${float_type} STREQUAL "HALF") +if (${float_type} STREQUAL "BFLOAT16") continue() endif() GenerateNamedObjects("getrf/getrf_single.c" "UNIT" "getrf_single" false "" "" false ${float_type}) diff --git a/lapack/potrf/potrf_parallel.c b/lapack/potrf/potrf_parallel.c index c38a2632d..2bbbc7bb3 100644 --- a/lapack/potrf/potrf_parallel.c +++ b/lapack/potrf/potrf_parallel.c @@ -1,5 +1,6 @@ /*********************************************************************/ /* Copyright 2009, 2010 The University of Texas at Austin. */ +/* Copyright 2025 The OpenBLAS Project. */ /* All rights reserved. */ /* */ /* Redistribution and use in source and binary forms, with or */ @@ -405,7 +406,7 @@ static int thread_driver(blas_arg_t *args, FLOAT *sa, FLOAT *sb){ #elif defined(DOUBLE) mode = BLAS_DOUBLE | BLAS_REAL; mask = MAX(DGEMM_UNROLL_M, DGEMM_UNROLL_N) - 1; -#elif defined(HALF) +#elif defined(BFLOAT16) mode = BLAS_HALF | BLAS_REAL; mask = MAX(SBGEMM_UNROLL_M, SBGEMM_UNROLL_N) - 1; #else From eaaa628af26902ccde17c1b8a84e1df905b50b85 Mon Sep 17 00:00:00 2001 From: Chris Sidebottom Date: Mon, 7 Jul 2025 10:20:17 +0000 Subject: [PATCH 08/36] Enable BUILD_BFLOAT16 in cirun --- .github/workflows/arm64_graviton.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/arm64_graviton.yml b/.github/workflows/arm64_graviton.yml index 6928312b5..b4b189763 100644 --- a/.github/workflows/arm64_graviton.yml +++ b/.github/workflows/arm64_graviton.yml @@ -95,6 +95,7 @@ jobs: cmake -DDYNAMIC_ARCH=1 \ -DNOFORTRAN=0 \ -DBUILD_WITHOUT_LAPACK=0 \ + -DBUILD_BFLOAT16=1 \ -DCMAKE_VERBOSE_MAKEFILE=ON \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_Fortran_COMPILER=${{ matrix.fortran }} \ From 46b9b7a08075e0a701aa5d8940b834d246d3ef70 Mon Sep 17 00:00:00 2001 From: Chris Sidebottom Date: Mon, 7 Jul 2025 10:41:12 +0000 Subject: [PATCH 09/36] Also enable BFLOAT16 for make cirun --- .github/workflows/arm64_graviton.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/arm64_graviton.yml b/.github/workflows/arm64_graviton.yml index b4b189763..4b4e15167 100644 --- a/.github/workflows/arm64_graviton.yml +++ b/.github/workflows/arm64_graviton.yml @@ -88,7 +88,7 @@ jobs: run: | case "${{ matrix.build }}" in "make") - make -j$(nproc) DYNAMIC_ARCH=1 USE_OPENMP=0 FC="ccache ${{ matrix.fortran }}" + make -j$(nproc) DYNAMIC_ARCH=1 BUILD_BFLOAT16=1 USE_OPENMP=0 FC="ccache ${{ matrix.fortran }}" ;; "cmake") mkdir build && cd build From 552e1c7a7aff6163dbb19c5375d2d966e7f66a2a Mon Sep 17 00:00:00 2001 From: Chris Sidebottom Date: Mon, 7 Jul 2025 11:26:36 +0000 Subject: [PATCH 10/36] Correct compiler flags for NEOVERSEV1 target --- cmake/cc.cmake | 6 +++--- cmake/system.cmake | 4 ++-- getarch.c | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cmake/cc.cmake b/cmake/cc.cmake index 91037e851..3679278b0 100644 --- a/cmake/cc.cmake +++ b/cmake/cc.cmake @@ -211,14 +211,14 @@ endif () if (${CORE} STREQUAL NEOVERSEV1) if (NOT DYNAMIC_ARCH) if (${CMAKE_C_COMPILER_ID} STREQUAL "PGI" AND NOT NO_SVE) - set (CCOMMON_OPT "${CCOMMON_OPT} -Msve_intrinsics -march=armv8.4-a+sve -mtune=neoverse-v1") + set (CCOMMON_OPT "${CCOMMON_OPT} -Msve_intrinsics -march=armv8.4-a+sve+bf16 -mtune=neoverse-v1") elseif (${CMAKE_C_COMPILER_ID} STREQUAL "NVC" AND NOT NO_SVE) set (CCOMMON_OPT "${CCOMMON_OPT} -tp=neoverse-v1") else () if (${GCC_VERSION} VERSION_GREATER 10.4 OR ${GCC_VERSION} VERSION_EQUAL 10.4) - set (CCOMMON_OPT "${CCOMMON_OPT} -march=armv8.4-a+sve -mtune=neoverse-v1") + set (CCOMMON_OPT "${CCOMMON_OPT} -march=armv8.4-a+sve+bf16 -mtune=neoverse-v1") else () - set (CCOMMON_OPT "${CCOMMON_OPT} -march=armv8.2-a+sve") + set (CCOMMON_OPT "${CCOMMON_OPT} -march=armv8.2-a+sve+bf16") endif() endif() endif () diff --git a/cmake/system.cmake b/cmake/system.cmake index bac756901..6ad73525a 100644 --- a/cmake/system.cmake +++ b/cmake/system.cmake @@ -291,10 +291,10 @@ if (DEFINED TARGET) if (${TARGET} STREQUAL NEOVERSEV1) if (${CMAKE_C_COMPILER_ID} STREQUAL "PGI" AND NOT NO_SVE) - set (KERNEL_DEFINITIONS "${KERNEL_DEFINITIONS} -Msve_intrinsics -march=armv8.4-a+sve -mtune=neoverse-v1") + set (KERNEL_DEFINITIONS "${KERNEL_DEFINITIONS} -Msve_intrinsics -march=armv8.4-a+sve+bf16 -mtune=neoverse-v1") else () if (CMAKE_C_COMPILER_VERSION VERSION_GREATER 10.4 OR CMAKE_C_COMPILER_VERSION VERSION_EQUAL 10.4) - set (KERNEL_DEFINITIONS "${KERNEL_DEFINITIONS} -march=armv8.4-a+sve -mtune=neoverse-v1") + set (KERNEL_DEFINITIONS "${KERNEL_DEFINITIONS} -march=armv8.4-a+sve+bf16 -mtune=neoverse-v1") else () message(FATAL_ERROR "Compiler ${CMAKE_C_COMPILER} ${CMAKE_C_COMPILER_VERSION} does not support Neoverse V1.") endif() diff --git a/getarch.c b/getarch.c index cb0b3cd7c..3fdbc9799 100644 --- a/getarch.c +++ b/getarch.c @@ -1,5 +1,5 @@ /***************************************************************************** -Copyright (c) 2011-2014, The OpenBLAS Project +Copyright (c) 2011-2014, 2025 The OpenBLAS Project All rights reserved. Redistribution and use in source and binary forms, with or without @@ -1476,7 +1476,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. "-DL2_SIZE=1048576 -DL2_LINESIZE=64 -DL2_ASSOCIATIVE=16 " \ "-DDTB_DEFAULT_ENTRIES=64 -DDTB_SIZE=4096 " \ "-DHAVE_VFPV4 -DHAVE_VFPV3 -DHAVE_VFP -DHAVE_NEON -DHAVE_SVE -DARMV8 " \ - "-march=armv8.4-a+sve -mtune=neoverse-v1" + "-march=armv8.4-a+sve+bf16 -mtune=neoverse-v1" #define LIBNAME "neoversev1" #define CORENAME "NEOVERSEV1" #endif From 309c48e3270a3e318d779337270f86f73ba6025f Mon Sep 17 00:00:00 2001 From: guoyuanplct Date: Tue, 8 Jul 2025 17:13:27 +0800 Subject: [PATCH 11/36] Update CONTRIBUTORS.md --- CONTRIBUTORS.md | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index d8f57ef60..9ced02b30 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -250,7 +250,14 @@ In chronological order: * Ye Tao * [2025-02-03] Optimize SBGEMM kernel on NEOVERSEV1 - * [2025-02-27] Add sbgemv_n_neon kernel + * [2025-02-27] Add sbgemv_n_neon kernel2 * Abhishek Kumar - * [2025-04-22] Optimise dot kernel for NEOVERSE V1 \ No newline at end of file + * [2025-04-22] Optimise dot kernel for NEOVERSE V1 + +* Guoyuan Li + * [2025-04-11] Optimise gemv kernel for RISCV64_ZVL256B + * [2025-05-01] Optimise zgemv kernel for RISCV64_ZVL256B + * [2025-05-17] Optimise omatcopy/zomatcopy kernel for RISCV64_ZVL256B + * [2025-05-29] Optimise axpby kernel for RISCV64_ZVL256B + * [2025-06-05] Optimise hbmv kernel for RISCV64_ZVL256B From 4ff549a450cf5955a3053b43a1542f19c9e8ab69 Mon Sep 17 00:00:00 2001 From: guoyuanplct Date: Tue, 8 Jul 2025 17:16:51 +0800 Subject: [PATCH 12/36] Update CONTRIBUTORS.md --- CONTRIBUTORS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 9ced02b30..d26c3d534 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -250,7 +250,7 @@ In chronological order: * Ye Tao * [2025-02-03] Optimize SBGEMM kernel on NEOVERSEV1 - * [2025-02-27] Add sbgemv_n_neon kernel2 + * [2025-02-27] Add sbgemv_n_neon kernel * Abhishek Kumar * [2025-04-22] Optimise dot kernel for NEOVERSE V1 From 04bb5acd7971d8a47558932e5610d672f35ef7ce Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Tue, 8 Jul 2025 14:40:22 +0200 Subject: [PATCH 13/36] change BLAS_HALF to BLAS_BFLOAT16 (another missed rename) --- lapack/potrf/potrf_parallel.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lapack/potrf/potrf_parallel.c b/lapack/potrf/potrf_parallel.c index 2bbbc7bb3..1e512848b 100644 --- a/lapack/potrf/potrf_parallel.c +++ b/lapack/potrf/potrf_parallel.c @@ -407,7 +407,7 @@ static int thread_driver(blas_arg_t *args, FLOAT *sa, FLOAT *sb){ mode = BLAS_DOUBLE | BLAS_REAL; mask = MAX(DGEMM_UNROLL_M, DGEMM_UNROLL_N) - 1; #elif defined(BFLOAT16) - mode = BLAS_HALF | BLAS_REAL; + mode = BLAS_BFLOAT16 | BLAS_REAL; mask = MAX(SBGEMM_UNROLL_M, SBGEMM_UNROLL_N) - 1; #else mode = BLAS_SINGLE | BLAS_REAL; From f95e7b0e3279b0ca443b8ca4850b612df19343bb Mon Sep 17 00:00:00 2001 From: Chris Sidebottom Date: Thu, 3 Jul 2025 17:47:08 +0000 Subject: [PATCH 14/36] Add infrastructure for BGEMM Setting up all the infrastructure for BGEMM support in OpenBLAS, hopefully I found all the right places. Derived mostly from the previous work done in https://github.com/OpenMathLib/OpenBLAS/pull/5287 Co-authored-by: Ye Tao --- .gitignore | 2 + CONTRIBUTORS.md | 1 + Makefile.system | 10 +- Makefile.tail | 7 +- cblas.h | 30 +++++ cmake/system.cmake | 4 +- common.h | 7 +- common_b.h | 85 ++++++++++++ common_interface.h | 3 + common_level3.h | 17 +++ common_macro.h | 51 +++++++- common_param.h | 44 ++++++- driver/level3/Makefile | 55 ++++++++ getarch_2nd.c | 32 +++++ interface/Makefile | 10 +- interface/gemm.c | 7 +- kernel/Makefile.L3 | 68 +++++++++- kernel/generic/gemm_beta.c | 69 +++++++--- kernel/generic/gemmkernel_2x2.c | 60 ++++++--- kernel/setparam-ref.c | 15 ++- param.h | 10 +- test/Makefile | 42 +++++- test/compare_sgemm_bgemm.c | 224 ++++++++++++++++++++++++++++++++ 23 files changed, 797 insertions(+), 56 deletions(-) create mode 100644 common_b.h create mode 100644 test/compare_sgemm_bgemm.c diff --git a/.gitignore b/.gitignore index d14d438a2..c0885d466 100644 --- a/.gitignore +++ b/.gitignore @@ -82,6 +82,7 @@ test/ZBLAT3.SUMM test/ZBLAT3_3M.SUMM test/SHBLAT3.SUMM test/SBBLAT3.SUMM +test/BBLAT3.SUMM test/cblat1 test/cblat2 test/cblat3 @@ -96,6 +97,7 @@ test/sblat3 test/sblat3_3m test/test_shgemm test/test_sbgemm +test/test_bgemm test/zblat1 test/zblat2 test/zblat3 diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index d26c3d534..8391862b1 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -251,6 +251,7 @@ In chronological order: * Ye Tao * [2025-02-03] Optimize SBGEMM kernel on NEOVERSEV1 * [2025-02-27] Add sbgemv_n_neon kernel + * [2025-05-17] Impl prototype of BGEMM inferface * Abhishek Kumar * [2025-04-22] Optimise dot kernel for NEOVERSE V1 diff --git a/Makefile.system b/Makefile.system index bde3014cc..9e20c132c 100644 --- a/Makefile.system +++ b/Makefile.system @@ -276,14 +276,14 @@ SMALL_MATRIX_OPT = 1 endif ifeq ($(ARCH), arm64) GEMM_GEMV_FORWARD = 1 -GEMM_GEMV_FORWARD_BF16 = 1 +SBGEMM_GEMV_FORWARD = 1 endif ifeq ($(ARCH), riscv) GEMM_GEMV_FORWARD = 1 endif ifeq ($(ARCH), power) GEMM_GEMV_FORWARD = 1 -GEMM_GEMV_FORWARD_BF16 = 1 +SBGEMM_GEMV_FORWARD = 1 endif ifeq ($(SMALL_MATRIX_OPT), 1) @@ -293,8 +293,8 @@ ifneq ($(ONLY_CBLAS), 1) ifeq ($(GEMM_GEMV_FORWARD), 1) CCOMMON_OPT += -DGEMM_GEMV_FORWARD endif -ifeq ($(GEMM_GEMV_FORWARD_BF16), 1) -CCOMMON_OPT += -DGEMM_GEMV_FORWARD_BF16 +ifeq ($(SBGEMM_GEMV_FORWARD), 1) +CCOMMON_OPT += -DSBGEMM_GEMV_FORWARD endif endif @@ -1905,6 +1905,8 @@ export BUILD_HFLOAT16 export NO_LSX export NO_LASX +export BGEMM_UNROLL_M +export BGEMM_UNROLL_N export SBGEMM_UNROLL_M export SBGEMM_UNROLL_N export SHGEMM_UNROLL_M diff --git a/Makefile.tail b/Makefile.tail index ed2c0e507..ddd74dcad 100644 --- a/Makefile.tail +++ b/Makefile.tail @@ -1,3 +1,4 @@ +BBLASOBJS_P = $(BBLASOBJS:.$(SUFFIX)=.$(PSUFFIX)) SBBLASOBJS_P = $(SBBLASOBJS:.$(SUFFIX)=.$(PSUFFIX)) SHBLASPBJS_P = $(SHBLASOBJS:.$(SUFFIX)=.$(PSUFFIX)) SBLASOBJS_P = $(SBLASOBJS:.$(SUFFIX)=.$(PSUFFIX)) @@ -12,8 +13,8 @@ COMMONOBJS_P = $(COMMONOBJS:.$(SUFFIX)=.$(PSUFFIX)) HPLOBJS_P = $(HPLOBJS:.$(SUFFIX)=.$(PSUFFIX)) -BLASOBJS = $(SHBLASOBJS) $(SBEXTOBJS) $(SBBLASOBJS) $(SBLASOBJS) $(DBLASOBJS) $(CBLASOBJS) $(ZBLASOBJS) $(CBAUXOBJS) -BLASOBJS_P = $(SHBLASPBJS_P) $(SBEXTOBJS_P) $(SBBLASOBJS_P) $(SBLASOBJS_P) $(DBLASOBJS_P) $(CBLASOBJS_P) $(ZBLASOBJS_P) $(CBAUXOBJS_P) +BLASOBJS = $(SHBLASOBJS) $(BBLASOBJS) $(SBEXTOBJS) $(SBBLASOBJS) $(SBLASOBJS) $(DBLASOBJS) $(CBLASOBJS) $(ZBLASOBJS) $(CBAUXOBJS) +BLASOBJS_P = $(SHBLASPBJS_P) $(BBLASOBJS_P) $(SBEXTOBJS_P) $(SBBLASOBJS_P) $(SBLASOBJS_P) $(DBLASOBJS_P) $(CBLASOBJS_P) $(ZBLASOBJS_P) $(CBAUXOBJS_P) ifdef EXPRECISION BLASOBJS += $(QBLASOBJS) $(XBLASOBJS) @@ -26,6 +27,7 @@ BLASOBJS_P += $(QBLASOBJS_P) $(XBLASOBJS_P) endif $(SHBLASOBJS) $(SHBLASOBJS_P) : override CFLAGS += -DHFLOAT16 -UDOUBLE -UCOMPLEX +$(BBLASOBJS) $(BBLASOBJS_P) : override CFLAGS += -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX -USMALL_MATRIX_OPT $(SBBLASOBJS) $(SBBLASOBJS_P) : override CFLAGS += -DBFLOAT16 -UDOUBLE -UCOMPLEX $(SBLASOBJS) $(SBLASOBJS_P) : override CFLAGS += -UDOUBLE -UCOMPLEX $(DBLASOBJS) $(DBLASOBJS_P) : override CFLAGS += -DDOUBLE -UCOMPLEX @@ -36,6 +38,7 @@ $(XBLASOBJS) $(XBLASOBJS_P) : override CFLAGS += -DXDOUBLE -DCOMPLEX $(SBEXTOBJS) $(SBEXTOBJS_P) : override CFLAGS += -DBFLOAT16 -UDOUBLE -UCOMPLEX $(SHBLASOBJS_P) : override CFLAGS += -DPROFILE $(COMMON_PROF) +$(BBLASOBJS_P) : override CFLAGS += -DPROFILE $(COMMON_PROF) $(SBBLASOBJS_P) : override CFLAGS += -DPROFILE $(COMMON_PROF) $(SBLASOBJS_P) : override CFLAGS += -DPROFILE $(COMMON_PROF) $(DBLASOBJS_P) : override CFLAGS += -DPROFILE $(COMMON_PROF) diff --git a/cblas.h b/cblas.h index 0364b216f..f48d5da1e 100644 --- a/cblas.h +++ b/cblas.h @@ -1,3 +1,31 @@ +/*************************************************************************** + * Copyright (c) 2025, The OpenBLAS Project + * All rights reserved. + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of the OpenBLAS project nor the names of + * its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * *****************************************************************************/ + #ifndef CBLAS_H #define CBLAS_H @@ -441,6 +469,8 @@ void cblas_dbf16tod(OPENBLAS_CONST blasint n, OPENBLAS_CONST bfloat16 *in, OPE float cblas_sbdot(OPENBLAS_CONST blasint n, OPENBLAS_CONST bfloat16 *x, OPENBLAS_CONST blasint incx, OPENBLAS_CONST bfloat16 *y, OPENBLAS_CONST blasint incy); void cblas_sbgemv(OPENBLAS_CONST enum CBLAS_ORDER order, OPENBLAS_CONST enum CBLAS_TRANSPOSE trans, OPENBLAS_CONST blasint m, OPENBLAS_CONST blasint n, OPENBLAS_CONST float alpha, OPENBLAS_CONST bfloat16 *a, OPENBLAS_CONST blasint lda, OPENBLAS_CONST bfloat16 *x, OPENBLAS_CONST blasint incx, OPENBLAS_CONST float beta, float *y, OPENBLAS_CONST blasint incy); +void cblas_bgemm(OPENBLAS_CONST enum CBLAS_ORDER Order, OPENBLAS_CONST enum CBLAS_TRANSPOSE TransA, OPENBLAS_CONST enum CBLAS_TRANSPOSE TransB, OPENBLAS_CONST blasint M, OPENBLAS_CONST blasint N, OPENBLAS_CONST blasint K, + OPENBLAS_CONST bfloat16 alpha, OPENBLAS_CONST bfloat16 *A, OPENBLAS_CONST blasint lda, OPENBLAS_CONST bfloat16 *B, OPENBLAS_CONST blasint ldb, OPENBLAS_CONST bfloat16 beta, bfloat16 *C, OPENBLAS_CONST blasint ldc); void cblas_sbgemm(OPENBLAS_CONST enum CBLAS_ORDER Order, OPENBLAS_CONST enum CBLAS_TRANSPOSE TransA, OPENBLAS_CONST enum CBLAS_TRANSPOSE TransB, OPENBLAS_CONST blasint M, OPENBLAS_CONST blasint N, OPENBLAS_CONST blasint K, OPENBLAS_CONST float alpha, OPENBLAS_CONST bfloat16 *A, OPENBLAS_CONST blasint lda, OPENBLAS_CONST bfloat16 *B, OPENBLAS_CONST blasint ldb, OPENBLAS_CONST float beta, float *C, OPENBLAS_CONST blasint ldc); void cblas_sbgemm_batch(OPENBLAS_CONST enum CBLAS_ORDER Order, OPENBLAS_CONST enum CBLAS_TRANSPOSE * TransA_array, OPENBLAS_CONST enum CBLAS_TRANSPOSE * TransB_array, OPENBLAS_CONST blasint * M_array, OPENBLAS_CONST blasint * N_array, OPENBLAS_CONST blasint * K_array, diff --git a/cmake/system.cmake b/cmake/system.cmake index 6ad73525a..81f1a67ad 100644 --- a/cmake/system.cmake +++ b/cmake/system.cmake @@ -425,8 +425,8 @@ endif () if (GEMM_GEMV_FORWARD AND NOT ONLY_CBLAS) set(CCOMMON_OPT "${CCOMMON_OPT} -DGEMM_GEMV_FORWARD") endif () -if (GEMM_GEMV_FORWARD_BF16 AND NOT ONLY_CBLAS) - set(CCOMMON_OPT "${CCOMMON_OPT} -DGEMM_GEMV_FORWARD_BF16") +if (SBGEMM_GEMV_FORWARD AND NOT ONLY_CBLAS) + set(CCOMMON_OPT "${CCOMMON_OPT} -DSBGEMM_GEMV_FORWARD") endif () if (SMALL_MATRIX_OPT) set(CCOMMON_OPT "${CCOMMON_OPT} -DSMALL_MATRIX_OPT") diff --git a/common.h b/common.h index 23a08aaa9..4984d727c 100644 --- a/common.h +++ b/common.h @@ -1,5 +1,6 @@ /*********************************************************************/ /* Copyright 2009, 2010 The University of Texas at Austin. */ +/* Copyright 2025 The OpenBLAS Project. */ /* All rights reserved. */ /* */ /* Redistribution and use in source and binary forms, with or */ @@ -317,7 +318,11 @@ typedef int blasint; #elif defined(BFLOAT16) #define IFLOAT bfloat16 #define XFLOAT IFLOAT -#define FLOAT float +#ifdef BGEMM +#define FLOAT bfloat16 +#else +#define FLOAT float +#endif #define SIZE 2 #define BASE_SHIFT 1 #define ZBASE_SHIFT 2 diff --git a/common_b.h b/common_b.h new file mode 100644 index 000000000..e03f6800d --- /dev/null +++ b/common_b.h @@ -0,0 +1,85 @@ +/*************************************************************************** + * Copyright (c) 2025, The OpenBLAS Project + * All rights reserved. + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of the OpenBLAS project nor the names of + * its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * *****************************************************************************/ + +#ifndef COMMON_B_H +#define COMMON_B_H + +#ifndef DYNAMIC_ARCH +#define BGEMM_ONCOPY bgemm_oncopy +#define BGEMM_OTCOPY bgemm_otcopy +#define BGEMM_INCOPY bgemm_incopy +#define BGEMM_ITCOPY bgemm_itcopy + +#define BGEMM_BETA bgemm_beta +#define BGEMM_KERNEL bgemm_kernel + +#else + +#define BGEMM_ONCOPY gotoblas->bgemm_oncopy +#define BGEMM_OTCOPY gotoblas->bgemm_otcopy +#define BGEMM_INCOPY gotoblas->bgemm_incopy +#define BGEMM_ITCOPY gotoblas->bgemm_itcopy +#define BGEMM_BETA gotoblas->bgemm_beta +#define BGEMM_KERNEL gotoblas->bgemm_kernel + +#endif + +#define BGEMM_NN bgemm_nn +#define BGEMM_CN bgemm_tn +#define BGEMM_TN bgemm_tn +#define BGEMM_NC bgemm_nt +#define BGEMM_NT bgemm_nt +#define BGEMM_CC bgemm_tt +#define BGEMM_CT bgemm_tt +#define BGEMM_TC bgemm_tt +#define BGEMM_TT bgemm_tt +#define BGEMM_NR bgemm_nn +#define BGEMM_TR bgemm_tn +#define BGEMM_CR bgemm_tn +#define BGEMM_RN bgemm_nn +#define BGEMM_RT bgemm_nt +#define BGEMM_RC bgemm_nt +#define BGEMM_RR bgemm_nn + +#define BGEMM_THREAD_NN bgemm_thread_nn +#define BGEMM_THREAD_CN bgemm_thread_tn +#define BGEMM_THREAD_TN bgemm_thread_tn +#define BGEMM_THREAD_NC bgemm_thread_nt +#define BGEMM_THREAD_NT bgemm_thread_nt +#define BGEMM_THREAD_CC bgemm_thread_tt +#define BGEMM_THREAD_CT bgemm_thread_tt +#define BGEMM_THREAD_TC bgemm_thread_tt +#define BGEMM_THREAD_TT bgemm_thread_tt +#define BGEMM_THREAD_NR bgemm_thread_nn +#define BGEMM_THREAD_TR bgemm_thread_tn +#define BGEMM_THREAD_CR bgemm_thread_tn +#define BGEMM_THREAD_RN bgemm_thread_nn +#define BGEMM_THREAD_RT bgemm_thread_nt +#define BGEMM_THREAD_RC bgemm_thread_nt +#define BGEMM_THREAD_RR bgemm_thread_nn +#endif diff --git a/common_interface.h b/common_interface.h index 23d86871f..f69baab1c 100644 --- a/common_interface.h +++ b/common_interface.h @@ -1,5 +1,6 @@ /*********************************************************************/ /* Copyright 2009, 2010 The University of Texas at Austin. */ +/* Copyright 2025 The OpenBLAS Project. */ /* All rights reserved. */ /* */ /* Redistribution and use in source and binary forms, with or */ @@ -483,6 +484,8 @@ void BLASFUNC(xhbmv)(char *, blasint *, blasint *, xdouble *, xdouble *, blasint void BLASFUNC(shgemm)(char *, char *, blasint *, blasint *, blasint *, float *, hfloat16 *, blasint *, hfloat16 *, blasint *, float *, float *, blasint *); +void BLASFUNC(bgemm)(char *, char *, blasint *, blasint *, blasint *, bfloat16 *, + bfloat16 *, blasint *, bfloat16 *, blasint *, bfloat16 *, bfloat16 *, blasint *); void BLASFUNC(sbgemm)(char *, char *, blasint *, blasint *, blasint *, float *, bfloat16 *, blasint *, bfloat16 *, blasint *, float *, float *, blasint *); void BLASFUNC(sgemm)(char *, char *, blasint *, blasint *, blasint *, float *, diff --git a/common_level3.h b/common_level3.h index 1838b4bf6..607eaf940 100644 --- a/common_level3.h +++ b/common_level3.h @@ -56,6 +56,8 @@ int sgemm_direct_performant(BLASLONG M, BLASLONG N, BLASLONG K); int shgemm_beta(BLASLONG, BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BLASLONG, float *, BLASLONG); +int bgemm_beta(BLASLONG, BLASLONG, BLASLONG, bfloat16, + bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG); int sbgemm_beta(BLASLONG, BLASLONG, BLASLONG, float, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, float *, BLASLONG); int sgemm_beta(BLASLONG, BLASLONG, BLASLONG, float, @@ -83,6 +85,10 @@ int shgemm_incopy(BLASLONG m, BLASLONG n, hfloat16 *a, BLASLONG lda, hfloat16 *b int shgemm_itcopy(BLASLONG m, BLASLONG n, hfloat16 *a, BLASLONG lda, hfloat16 *b); int shgemm_oncopy(BLASLONG m, BLASLONG n, hfloat16 *a, BLASLONG lda, hfloat16 *b); int shgemm_otcopy(BLASLONG m, BLASLONG n, hfloat16 *a, BLASLONG lda, hfloat16 *b); +int bgemm_incopy(BLASLONG m, BLASLONG n, bfloat16 *a, BLASLONG lda, bfloat16 *b); +int bgemm_itcopy(BLASLONG m, BLASLONG n, bfloat16 *a, BLASLONG lda, bfloat16 *b); +int bgemm_oncopy(BLASLONG m, BLASLONG n, bfloat16 *a, BLASLONG lda, bfloat16 *b); +int bgemm_otcopy(BLASLONG m, BLASLONG n, bfloat16 *a, BLASLONG lda, bfloat16 *b); int sbgemm_incopy(BLASLONG m, BLASLONG n, bfloat16 *a, BLASLONG lda, bfloat16 *b); int sbgemm_itcopy(BLASLONG m, BLASLONG n, bfloat16 *a, BLASLONG lda, bfloat16 *b); int sbgemm_oncopy(BLASLONG m, BLASLONG n, bfloat16 *a, BLASLONG lda, bfloat16 *b); @@ -511,6 +517,7 @@ int xher2k_kernel_LN(BLASLONG m, BLASLONG n, BLASLONG k, xdouble alpha_r, xdoubl int xher2k_kernel_LC(BLASLONG m, BLASLONG n, BLASLONG k, xdouble alpha_r, xdouble alpha_i, xdouble *a, xdouble *b, xdouble *c, BLASLONG ldc, BLASLONG offset, int flag); int shgemm_kernel(BLASLONG, BLASLONG, BLASLONG, float, hfloat16 *, hfloat16 *, float *, BLASLONG); +int bgemm_kernel(BLASLONG, BLASLONG, BLASLONG, bfloat16, bfloat16 *, bfloat16 *, bfloat16 *, BLASLONG); int sbgemm_kernel(BLASLONG, BLASLONG, BLASLONG, float, bfloat16 *, bfloat16 *, float *, BLASLONG); int sgemm_kernel(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG); int dgemm_kernel(BLASLONG, BLASLONG, BLASLONG, double, double *, double *, double *, BLASLONG); @@ -668,6 +675,11 @@ int shgemm_nt(blas_arg_t *, BLASLONG *, BLASLONG *, hfloat16 *, hfloat16 *, BLAS int shgemm_tn(blas_arg_t *, BLASLONG *, BLASLONG *, hfloat16 *, hfloat16 *, BLASLONG); int shgemm_tt(blas_arg_t *, BLASLONG *, BLASLONG *, hfloat16 *, hfloat16 *, BLASLONG); +int bgemm_nn(blas_arg_t *, BLASLONG *, BLASLONG *, bfloat16 *, bfloat16 *, BLASLONG); +int bgemm_nt(blas_arg_t *, BLASLONG *, BLASLONG *, bfloat16 *, bfloat16 *, BLASLONG); +int bgemm_tn(blas_arg_t *, BLASLONG *, BLASLONG *, bfloat16 *, bfloat16 *, BLASLONG); +int bgemm_tt(blas_arg_t *, BLASLONG *, BLASLONG *, bfloat16 *, bfloat16 *, BLASLONG); + int sbgemm_nn(blas_arg_t *, BLASLONG *, BLASLONG *, bfloat16 *, bfloat16 *, BLASLONG); int sbgemm_nt(blas_arg_t *, BLASLONG *, BLASLONG *, bfloat16 *, bfloat16 *, BLASLONG); int sbgemm_tn(blas_arg_t *, BLASLONG *, BLASLONG *, bfloat16 *, bfloat16 *, BLASLONG); @@ -770,6 +782,11 @@ int shgemm_thread_nt(blas_arg_t *, BLASLONG *, BLASLONG *, hfloat16 *, hfloat16 int shgemm_thread_tn(blas_arg_t *, BLASLONG *, BLASLONG *, hfloat16 *, hfloat16 *, BLASLONG); int shgemm_thread_tt(blas_arg_t *, BLASLONG *, BLASLONG *, hfloat16 *, hfloat16 *, BLASLONG); +int bgemm_thread_nn(blas_arg_t *, BLASLONG *, BLASLONG *, bfloat16 *, bfloat16 *, BLASLONG); +int bgemm_thread_nt(blas_arg_t *, BLASLONG *, BLASLONG *, bfloat16 *, bfloat16 *, BLASLONG); +int bgemm_thread_tn(blas_arg_t *, BLASLONG *, BLASLONG *, bfloat16 *, bfloat16 *, BLASLONG); +int bgemm_thread_tt(blas_arg_t *, BLASLONG *, BLASLONG *, bfloat16 *, bfloat16 *, BLASLONG); + int sbgemm_thread_nn(blas_arg_t *, BLASLONG *, BLASLONG *, bfloat16 *, bfloat16 *, BLASLONG); int sbgemm_thread_nt(blas_arg_t *, BLASLONG *, BLASLONG *, bfloat16 *, bfloat16 *, BLASLONG); int sbgemm_thread_tn(blas_arg_t *, BLASLONG *, BLASLONG *, bfloat16 *, bfloat16 *, BLASLONG); diff --git a/common_macro.h b/common_macro.h index b29a9c08d..22c1e14a2 100644 --- a/common_macro.h +++ b/common_macro.h @@ -1,5 +1,6 @@ /*********************************************************************/ /* Copyright 2009, 2010 The University of Texas at Austin. */ +/* Copyright 2025 The OpenBLAS Project. */ /* All rights reserved. */ /* */ /* Redistribution and use in source and binary forms, with or */ @@ -40,6 +41,7 @@ #define COMMON_MACRO #include "common_sh.h" +#include "common_b.h" #include "common_sb.h" #include "common_s.h" #include "common_d.h" @@ -702,8 +704,52 @@ #define GEMM_THREAD_RR SHGEMM_THREAD_NN -#elif defined(BFLOAT16) +#elif defined(BFLOAT16) && defined(BGEMM) +#define GEMM_BETA BGEMM_BETA +#define GEMM_KERNEL_N BGEMM_KERNEL +#define GEMM_KERNEL_L BGEMM_KERNEL +#define GEMM_KERNEL_R BGEMM_KERNEL +#define GEMM_KERNEL_B BGEMM_KERNEL + +#define GEMM_NN BGEMM_NN +#define GEMM_CN BGEMM_TN +#define GEMM_TN BGEMM_TN +#define GEMM_NC BGEMM_NT +#define GEMM_NT BGEMM_NT +#define GEMM_CC BGEMM_TT +#define GEMM_CT BGEMM_TT +#define GEMM_TC BGEMM_TT +#define GEMM_TT BGEMM_TT +#define GEMM_NR BGEMM_NN +#define GEMM_TR BGEMM_TN +#define GEMM_CR BGEMM_TN +#define GEMM_RN BGEMM_NN +#define GEMM_RT BGEMM_NT +#define GEMM_RC BGEMM_NT +#define GEMM_RR BGEMM_NN +#define GEMM_ONCOPY BGEMM_ONCOPY +#define GEMM_OTCOPY BGEMM_OTCOPY +#define GEMM_INCOPY BGEMM_INCOPY +#define GEMM_ITCOPY BGEMM_ITCOPY + +#define GEMM_THREAD_NN BGEMM_THREAD_NN +#define GEMM_THREAD_CN BGEMM_THREAD_TN +#define GEMM_THREAD_TN BGEMM_THREAD_TN +#define GEMM_THREAD_NC BGEMM_THREAD_NT +#define GEMM_THREAD_NT BGEMM_THREAD_NT +#define GEMM_THREAD_CC BGEMM_THREAD_TT +#define GEMM_THREAD_CT BGEMM_THREAD_TT +#define GEMM_THREAD_TC BGEMM_THREAD_TT +#define GEMM_THREAD_TT BGEMM_THREAD_TT +#define GEMM_THREAD_NR BGEMM_THREAD_NN +#define GEMM_THREAD_TR BGEMM_THREAD_TN +#define GEMM_THREAD_CR BGEMM_THREAD_TN +#define GEMM_THREAD_RN BGEMM_THREAD_NN +#define GEMM_THREAD_RT BGEMM_THREAD_NT +#define GEMM_THREAD_RC BGEMM_THREAD_NT +#define GEMM_THREAD_RR BGEMM_THREAD_NN +#elif defined(BFLOAT16) #define D_TO_BF16_K SBDTOBF16_K #define D_BF16_TO_K DBF16TOD_K #define S_TO_BF16_K SBSTOBF16_K @@ -2663,6 +2709,9 @@ || defined(ARCH_LOONGARCH64) || defined(ARCH_E2K) || defined(ARCH_ALPHA)) extern BLASLONG gemm_offset_a; extern BLASLONG gemm_offset_b; +extern BLASLONG bgemm_p; +extern BLASLONG bgemm_q; +extern BLASLONG bgemm_r; extern BLASLONG sbgemm_p; extern BLASLONG sbgemm_q; extern BLASLONG sbgemm_r; diff --git a/common_param.h b/common_param.h index f82b73a72..503525dd2 100644 --- a/common_param.h +++ b/common_param.h @@ -1,6 +1,6 @@ /*********************************************************************/ /* Copyright 2009, 2010 The University of Texas at Austin. */ -/* Copyright 2023 The OpenBLAS Project. */ +/* Copyright 2023, 2025 The OpenBLAS Project. */ /* All rights reserved. */ /* */ /* Redistribution and use in source and binary forms, with or */ @@ -65,6 +65,10 @@ int (*shgemm_otcopy )(BLASLONG, BLASLONG, hfloat16 *, BLASLONG, hfloat16 *); #if BUILD_BFLOAT16 == 1 + int bgemm_p, bgemm_q, bgemm_r; + int bgemm_unroll_m, bgemm_unroll_n, bgemm_unroll_mn; + int bgemm_align_k; + int sbgemm_p, sbgemm_q, sbgemm_r; int sbgemm_unroll_m, sbgemm_unroll_n, sbgemm_unroll_mn; int sbgemm_align_k; @@ -105,6 +109,14 @@ int (*shgemm_otcopy )(BLASLONG, BLASLONG, hfloat16 *, BLASLONG, hfloat16 *); int (*sbsymv_L) (BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); int (*sbsymv_U) (BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); + int (*bgemm_kernel )(BLASLONG, BLASLONG, BLASLONG, bfloat16, bfloat16 *, bfloat16 *, bfloat16 *, BLASLONG); + int (*bgemm_beta )(BLASLONG, BLASLONG, BLASLONG, bfloat16, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG); + + int (*bgemm_incopy )(BLASLONG, BLASLONG, bfloat16 *, BLASLONG, bfloat16 *); + int (*bgemm_itcopy )(BLASLONG, BLASLONG, bfloat16 *, BLASLONG, bfloat16 *); + int (*bgemm_oncopy )(BLASLONG, BLASLONG, bfloat16 *, BLASLONG, bfloat16 *); + int (*bgemm_otcopy )(BLASLONG, BLASLONG, bfloat16 *, BLASLONG, bfloat16 *); + int (*sbgemm_kernel )(BLASLONG, BLASLONG, BLASLONG, float, bfloat16 *, bfloat16 *, float *, BLASLONG); int (*sbgemm_beta )(BLASLONG, BLASLONG, BLASLONG, float, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, float *, BLASLONG); @@ -1254,6 +1266,13 @@ extern gotoblas_t *gotoblas; #endif #if (BUILD_BFLOAT16==1) +#define BGEMM_P gotoblas -> bgemm_p +#define BGEMM_Q gotoblas -> bgemm_q +#define BGEMM_R gotoblas -> bgemm_r +#define BGEMM_UNROLL_M gotoblas -> bgemm_unroll_m +#define BGEMM_UNROLL_N gotoblas -> bgemm_unroll_n +#define BGEMM_UNROLL_MN gotoblas -> bgemm_unroll_mn + #define SBGEMM_P gotoblas -> sbgemm_p #define SBGEMM_Q gotoblas -> sbgemm_q #define SBGEMM_R gotoblas -> sbgemm_r @@ -1395,6 +1414,17 @@ extern gotoblas_t *gotoblas; #endif #if (BUILD_BFLOAT16 == 1) +#define BGEMM_P BGEMM_DEFAULT_P +#define BGEMM_Q BGEMM_DEFAULT_Q +#define BGEMM_R BGEMM_DEFAULT_R +#define BGEMM_UNROLL_M BGEMM_DEFAULT_UNROLL_M +#define BGEMM_UNROLL_N BGEMM_DEFAULT_UNROLL_N +#ifdef BGEMM_DEFAULT_UNROLL_MN +#define BGEMM_UNROLL_MN BGEMM_DEFAULT_UNROLL_MN +#else +#define BGEMM_UNROLL_MN MAX((BGEMM_UNROLL_M), (BGEMM_UNROLL_N)) +#endif + #define SBGEMM_P SBGEMM_DEFAULT_P #define SBGEMM_Q SBGEMM_DEFAULT_Q #define SBGEMM_R SBGEMM_DEFAULT_R @@ -1555,6 +1585,18 @@ extern gotoblas_t *gotoblas; #define GEMM_DEFAULT_R SHGEMM_DEFAULT_R #define GEMM_DEFAULT_UNROLL_M SHGEMM_DEFAULT_UNROLL_M #define GEMM_DEFAULT_UNROLL_N SHGEMM_DEFAULT_UNROLL_N +#elif defined(BFLOAT16) && defined(BGEMM) +#define GEMM_P BGEMM_P +#define GEMM_Q BGEMM_Q +#define GEMM_R BGEMM_R +#define GEMM_UNROLL_M BGEMM_UNROLL_M +#define GEMM_UNROLL_N BGEMM_UNROLL_N +#define GEMM_UNROLL_MN BGEMM_UNROLL_MN +#define GEMM_DEFAULT_P BGEMM_DEFAULT_P +#define GEMM_DEFAULT_Q BGEMM_DEFAULT_Q +#define GEMM_DEFAULT_R BGEMM_DEFAULT_R +#define GEMM_DEFAULT_UNROLL_M BGEMM_DEFAULT_UNROLL_M +#define GEMM_DEFAULT_UNROLL_N BGEMM_DEFAULT_UNROLL_N #elif defined(BFLOAT16) #define GEMM_P SBGEMM_P #define GEMM_Q SBGEMM_Q diff --git a/driver/level3/Makefile b/driver/level3/Makefile index 132645a87..622996c3b 100644 --- a/driver/level3/Makefile +++ b/driver/level3/Makefile @@ -1,3 +1,32 @@ +############################################################################### +# Copyright (c) 2025, The OpenBLAS Project +# All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# 3. Neither the name of the OpenBLAS project nor the names of +# its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +############################################################################### + TOPDIR = ../.. include ../../Makefile.system @@ -20,6 +49,7 @@ USE_GEMM3M = 1 endif ifeq ($(BUILD_BFLOAT16),1) +BBLASOBJS += bgemm_nn.$(SUFFIX) bgemm_nt.$(SUFFIX) bgemm_tn.$(SUFFIX) bgemm_tt.$(SUFFIX) SBBLASOBJS += sbgemm_nn.$(SUFFIX) sbgemm_nt.$(SUFFIX) sbgemm_tn.$(SUFFIX) sbgemm_tt.$(SUFFIX) endif @@ -212,6 +242,7 @@ COMMONOBJS += syrk_thread.$(SUFFIX) ifneq ($(USE_SIMPLE_THREADED_LEVEL3), 1) ifeq ($(BUILD_BFLOAT16),1) +BBLASOBJS += bgemm_thread_nn.$(SUFFIX) bgemm_thread_nt.$(SUFFIX) bgemm_thread_tn.$(SUFFIX) bgemm_thread_tt.$(SUFFIX) SBBLASOBJS += sbgemm_thread_nn.$(SUFFIX) sbgemm_thread_nt.$(SUFFIX) sbgemm_thread_tn.$(SUFFIX) sbgemm_thread_tt.$(SUFFIX) endif ifeq ($(BUILD_HFLOAT16),1) @@ -350,6 +381,18 @@ endif all :: +bgemm_nn.$(SUFFIX) : gemm.c level3.c ../../param.h + $(CC) $(CFLAGS) $(BLOCKS) -c -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX -DNN $< -o $(@F) + +bgemm_nt.$(SUFFIX) : gemm.c level3.c ../../param.h + $(CC) $(CFLAGS) $(BLOCKS) -c -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX -DNT $< -o $(@F) + +bgemm_tn.$(SUFFIX) : gemm.c level3.c ../../param.h + $(CC) $(CFLAGS) $(BLOCKS) -c -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX -DTN $< -o $(@F) + +bgemm_tt.$(SUFFIX) : gemm.c level3.c ../../param.h + $(CC) $(CFLAGS) $(BLOCKS) -c -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX -DTT $< -o $(@F) + sbgemm_nn.$(SUFFIX) : gemm.c level3.c ../../param.h $(CC) $(CFLAGS) $(BLOCKS) -c -DBFLOAT16 -UDOUBLE -UCOMPLEX -DNN $< -o $(@F) @@ -569,6 +612,18 @@ gemm_thread_variable.$(SUFFIX) : gemm_thread_variable.c ../../common.h beta_thread.$(SUFFIX) : beta_thread.c ../../common.h $(CC) -c $(CFLAGS) $< -o $(@F) +bgemm_thread_nn.$(SUFFIX) : gemm.c level3_thread.c ../../param.h + $(CC) $(CFLAGS) $(BLOCKS) -c -DTHREADED_LEVEL3 -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX -DNN $< -o $(@F) + +bgemm_thread_nt.$(SUFFIX) : gemm.c level3_thread.c ../../param.h + $(CC) $(CFLAGS) $(BLOCKS) -c -DTHREADED_LEVEL3 -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX -DNT $< -o $(@F) + +bgemm_thread_tn.$(SUFFIX) : gemm.c level3_thread.c ../../param.h + $(CC) $(CFLAGS) $(BLOCKS) -c -DTHREADED_LEVEL3 -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX -DTN $< -o $(@F) + +bgemm_thread_tt.$(SUFFIX) : gemm.c level3_thread.c ../../param.h + $(CC) $(CFLAGS) $(BLOCKS) -c -DTHREADED_LEVEL3 -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX -DTT $< -o $(@F) + sbgemm_thread_nn.$(SUFFIX) : gemm.c level3_thread.c ../../param.h $(CC) $(CFLAGS) $(BLOCKS) -c -DTHREADED_LEVEL3 -DBFLOAT16 -UDOUBLE -UCOMPLEX -DNN $< -o $(@F) diff --git a/getarch_2nd.c b/getarch_2nd.c index 8170e9cf3..2085556bd 100644 --- a/getarch_2nd.c +++ b/getarch_2nd.c @@ -1,3 +1,31 @@ +/*************************************************************************** + * Copyright (c) 2025, The OpenBLAS Project + * All rights reserved. + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of the OpenBLAS project nor the names of + * its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * *****************************************************************************/ + #include #ifndef BUILD_KERNEL #include "config.h" @@ -17,6 +45,10 @@ typedef unsigned long BLASULONG; int main(int argc, char **argv) { if ( (argc <= 1) || ((argc >= 2) && (*argv[1] == '0'))) { + printf("BGEMM_UNROLL_M=%d\n", BGEMM_DEFAULT_UNROLL_M); + printf("BGEMM_UNROLL_N=%d\n", BGEMM_DEFAULT_UNROLL_N); + printf("BGEMM_UNROLL_M=%d\n", BGEMM_DEFAULT_UNROLL_M); + printf("BGEMM_UNROLL_N=%d\n", BGEMM_DEFAULT_UNROLL_N); printf("SBGEMM_UNROLL_M=%d\n", SBGEMM_DEFAULT_UNROLL_M); printf("SBGEMM_UNROLL_N=%d\n", SBGEMM_DEFAULT_UNROLL_N); printf("SHGEMM_UNROLL_M=%d\n", SHGEMM_DEFAULT_UNROLL_M); diff --git a/interface/Makefile b/interface/Makefile index dc154a3c4..e14796cbb 100644 --- a/interface/Makefile +++ b/interface/Makefile @@ -47,6 +47,7 @@ SBLAS3OBJS = \ sgeadd.$(SUFFIX) sgemmt.$(SUFFIX) sgemmtr.$(SUFFIX) ifeq ($(BUILD_BFLOAT16),1) +BBLAS3OBJ = bgemm.$(SUFFIX) SBBLAS1OBJS = sbdot.$(SUFFIX) SBBLAS2OBJS = sbgemv.$(SUFFIX) SBBLAS3OBJS = sbgemm.$(SUFFIX) sbgemmt.$(SUFFIX) sbgemmtr.$(SUFFIX) @@ -289,6 +290,7 @@ CSBLAS3OBJS = \ cblas_sgeadd.$(SUFFIX) cblas_sgemmt.$(SUFFIX) cblas_sgemmtr.$(SUFFIX) cblas_sgemm_batch.$(SUFFIX) ifeq ($(BUILD_BFLOAT16),1) +CBBLAS3OBJS = cblas_bgemm.$(SUFFIX) CSBBLAS1OBJS = cblas_sbdot.$(SUFFIX) CSBBLAS2OBJS = cblas_sbgemv.$(SUFFIX) CSBBLAS3OBJS = cblas_sbgemm.$(SUFFIX) cblas_sbgemmt.$(SUFFIX) cblas_sbgemmtr.$(SUFFIX) cblas_sbgemm_batch.$(SUFFIX) @@ -393,6 +395,7 @@ override CFLAGS += -I. SBLAS1OBJS += $(CSBLAS1OBJS) SBLAS2OBJS += $(CSBLAS2OBJS) SBLAS3OBJS += $(CSBLAS3OBJS) +BBLAS3OBJ += $(CBBLAS3OBJS) SBBLAS1OBJS += $(CSBBLAS1OBJS) SBBLAS2OBJS += $(CSBBLAS2OBJS) SBBLAS3OBJS += $(CSBBLAS3OBJS) @@ -412,6 +415,7 @@ SBEXTOBJS += $(CSBEXTOBJS) CBAUXOBJS += $(CXERBLAOBJ) endif +BBLASOBJS = $(BBLAS3OBJ) SBLASOBJS = $(SBLAS1OBJS) $(SBLAS2OBJS) $(SBLAS3OBJS) SBBLASOBJS = $(SBBLAS1OBJS) $(SBBLAS2OBJS) $(SBBLAS3OBJS) SHBLASOBJS = $(SHBLAS3OBJS) @@ -560,7 +564,7 @@ level1 : $(SBEXTOBJS) $(SBBLAS1OBJS) $(SBLAS1OBJS) $(DBLAS1OBJS) $(QBLAS1OBJS) $ level2 : $(SBBLAS2OBJS) $(SBLAS2OBJS) $(DBLAS2OBJS) $(QBLAS2OBJS) $(CBLAS2OBJS) $(ZBLAS2OBJS) $(XBLAS2OBJS) $(AR) $(ARFLAGS) -ru $(TOPDIR)/$(LIBNAME) $^ -level3 : $(SBBLAS3OBJS) $(SBLAS3OBJS) $(DBLAS3OBJS) $(QBLAS3OBJS) $(CBLAS3OBJS) $(ZBLAS3OBJS) $(XBLAS3OBJS) $(SHBLAS3OBJS) +level3 : $(SBBLAS3OBJS) $(BBLAS3OBJ) $(SBLAS3OBJS) $(DBLAS3OBJS) $(QBLAS3OBJS) $(CBLAS3OBJS) $(ZBLAS3OBJS) $(XBLAS3OBJS) $(SHBLAS3OBJS) $(AR) $(ARFLAGS) -ru $(TOPDIR)/$(LIBNAME) $^ aux : $(CBAUXOBJS) @@ -1311,6 +1315,8 @@ xhpr2.$(SUFFIX) xhpr2.$(PSUFFIX) : zhpr2.c $(CC) -c $(CFLAGS) $< -o $(@F) ifeq ($(BUILD_BFLOAT16),1) +bgemm.$(SUFFIX) bgemm.$(PSUFFIX) : gemm.c ../param.h + $(CC) -c $(CFLAGS) $< -o $(@F) sbgemm.$(SUFFIX) sbgemm.$(PSUFFIX) : gemm.c ../param.h $(CC) -c $(CFLAGS) $< -o $(@F) sbgemmt.$(SUFFIX) sbgemmt.$(PSUFFIX) : sbgemmt.c ../param.h @@ -1979,6 +1985,8 @@ cblas_sgemm.$(SUFFIX) cblas_sgemm.$(PSUFFIX) : gemm.c ../param.h $(CC) -DCBLAS -c $(CFLAGS) $< -o $(@F) ifeq ($(BUILD_BFLOAT16),1) +cblas_bgemm.$(SUFFIX) cblas_bgemm.$(PSUFFIX) : gemm.c ../param.h + $(CC) -DCBLAS -DBGEMM -c $(CFLAGS) $< -o $(@F) cblas_sbgemm.$(SUFFIX) cblas_sbgemm.$(PSUFFIX) : gemm.c ../param.h $(CC) -DCBLAS -c $(CFLAGS) $< -o $(@F) endif diff --git a/interface/gemm.c b/interface/gemm.c index d79282e13..c21fd988b 100644 --- a/interface/gemm.c +++ b/interface/gemm.c @@ -54,8 +54,13 @@ #define ERROR_NAME "DGEMM " #define GEMV BLASFUNC(dgemv) #elif defined(BFLOAT16) +#ifdef BGEMM +#define ERROR_NAME "BGEMM " +#define GEMV BLASFUNC(bgemv) +#else #define ERROR_NAME "SBGEMM " #define GEMV BLASFUNC(sbgemv) +#endif #elif defined(HFLOAT16) #define ERROR_NAME "SHGEMM " #else @@ -579,7 +584,7 @@ void CNAME(enum CBLAS_ORDER order, enum CBLAS_TRANSPOSE TransA, enum CBLAS_TRANS args.m, args.n, args.k, args.lda, args.ldb, args.ldc); #endif -#if defined(GEMM_GEMV_FORWARD) && !defined(GEMM3M) && !defined(COMPLEX) && !defined(HFLOAT16) && (!defined(BFLOAT16) || defined(GEMM_GEMV_FORWARD_BF16)) +#if defined(GEMM_GEMV_FORWARD) && !defined(GEMM3M) && !defined(COMPLEX) && !defined(HFLOAT16) && (!defined(BFLOAT16) || (!defined(BGEMM) && defined(SBGEMM_GEMV_FORWARD)) || (defined(BGEMM) && defined(BGEMM_GEMV_FORWARD))) #if defined(ARCH_ARM64) // The gemv kernels in arm64/{gemv_n.S,gemv_n_sve.c,gemv_t.S,gemv_t_sve.c} // perform poorly in certain circumstances. We use the following boolean diff --git a/kernel/Makefile.L3 b/kernel/Makefile.L3 index 6afb49a77..06f18e6be 100644 --- a/kernel/Makefile.L3 +++ b/kernel/Makefile.L3 @@ -1,3 +1,30 @@ +############################################################################### +# Copyright (c) 2025, The OpenBLAS Project +# All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# 3. Neither the name of the OpenBLAS project nor the names of +# its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +############################################################################### USE_GEMM3M = 0 OS := $(shell uname) @@ -110,6 +137,23 @@ endif endif ifeq ($(BUILD_BFLOAT16), 1) +ifndef BGEMMKERNEL +BGEMM_BETA = ../generic/gemm_beta.c +BGEMMKERNEL = ../generic/gemmkernel_2x2.c +BGEMMINCOPY = ../generic/gemm_ncopy_2.c +BGEMMITCOPY = ../generic/gemm_tcopy_2.c +BGEMMONCOPY = ../generic/gemm_ncopy_2.c +BGEMMOTCOPY = ../generic/gemm_tcopy_2.c +BGEMMINCOPYOBJ = bgemm_incopy$(TSUFFIX).$(SUFFIX) +BGEMMITCOPYOBJ = bgemm_itcopy$(TSUFFIX).$(SUFFIX) +BGEMMONCOPYOBJ = bgemm_oncopy$(TSUFFIX).$(SUFFIX) +BGEMMOTCOPYOBJ = bgemm_otcopy$(TSUFFIX).$(SUFFIX) +endif +BKERNELOBJS += \ + bgemm_kernel$(TSUFFIX).$(SUFFIX) \ + $(BGEMMINCOPYOBJ) $(BGEMMITCOPYOBJ) \ + $(BGEMMONCOPYOBJ) $(BGEMMOTCOPYOBJ) + ifndef SBGEMMKERNEL SBGEMM_BETA = ../generic/gemm_beta.c SBGEMMKERNEL = ../generic/gemmkernel_2x2.c @@ -210,6 +254,7 @@ XKERNELOBJS += \ $(XGEMMONCOPYOBJ) $(XGEMMOTCOPYOBJ) ifeq ($(BUILD_BFLOAT16),1) +BBLASOBJS += $(BKERNELOBJS) SBBLASOBJS += $(SBKERNELOBJS) endif ifeq ($(BUILD_HFLOAT16),1) @@ -223,6 +268,7 @@ ZBLASOBJS += $(ZKERNELOBJS) XBLASOBJS += $(XKERNELOBJS) ifeq ($(BUILD_BFLOAT16),1) +BBLASOBJS += bgemm_beta$(TSUFFIX).$(SUFFIX) SBBLASOBJS += sbgemm_beta$(TSUFFIX).$(SUFFIX) endif ifeq ($(BUILD_HFLOAT16),1) @@ -667,6 +713,8 @@ XGEMMONCOPYOBJ_P = $(XGEMMONCOPYOBJ:.$(SUFFIX)=.$(PSUFFIX)) XGEMMOTCOPYOBJ_P = $(XGEMMOTCOPYOBJ:.$(SUFFIX)=.$(PSUFFIX)) ifeq ($(BUILD_BFLOAT16),1) +$(KDIR)bgemm_beta$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(BGEMM_BETA) + $(CC) $(CFLAGS) -c -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX $< -o $@ $(KDIR)sbgemm_beta$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SBGEMM_BETA) $(CC) $(CFLAGS) -c -DBFLOAT16 -UDOUBLE -UCOMPLEX $< -o $@ endif @@ -698,9 +746,22 @@ ifeq ($(ARCH), E2K) USE_TRMM = 1 endif - ifeq ($(BUILD_BFLOAT16), 1) +$(KDIR)$(BGEMMONCOPYOBJ) : $(KERNELDIR)/$(BGEMMONCOPY) + $(CC) $(CFLAGS) -c -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX $< -o $@ + +$(KDIR)$(BGEMMOTCOPYOBJ) : $(KERNELDIR)/$(BGEMMOTCOPY) + $(CC) $(CFLAGS) -c -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX $< -o $@ + +ifneq ($(BGEMM_UNROLL_M), $(BGEMM_UNROLL_N)) +$(KDIR)$(BGEMMINCOPYOBJ) : $(KERNELDIR)/$(BGEMMINCOPY) + $(CC) $(CFLAGS) -c -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX $< -o $@ + +$(KDIR)$(BGEMMITCOPYOBJ) : $(KERNELDIR)/$(BGEMMITCOPY) + $(CC) $(CFLAGS) -c -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX $< -o $@ +endif + $(KDIR)$(SBGEMMONCOPYOBJ) : $(KERNELDIR)/$(SBGEMMONCOPY) $(CC) $(CFLAGS) -c -DBFLOAT16 -UDOUBLE -UCOMPLEX $< -o $@ @@ -914,7 +975,8 @@ endif endif ifeq ($(BUILD_BFLOAT16), 1) - +$(KDIR)bgemm_kernel$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(BGEMMKERNEL) + $(CC) $(CFLAGS) -c -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX $< -o $@ $(KDIR)sbgemm_kernel$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SBGEMMKERNEL) $(SBGEMMDEPEND) $(CC) $(CFLAGS) -c -DBFLOAT16 -UDOUBLE -UCOMPLEX $< -o $@ endif @@ -2908,6 +2970,8 @@ $(KDIR)sgemm_beta$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SGEMM_BETA) $(CC) $(PFLAGS) -c -UDOUBLE -UCOMPLEX $< -o $@ ifeq ($(BUILD_BFLOAT16),1) +$(KDIR)bgemm_beta$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(BGEMM_BETA) + $(CC) $(PFLAGS) -c -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX $< -o $@ $(KDIR)sbgemm_beta$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SBGEMM_BETA) $(CC) $(PFLAGS) -c -DBFLOAT16 -UDOUBLE -UCOMPLEX $< -o $@ endif diff --git a/kernel/generic/gemm_beta.c b/kernel/generic/gemm_beta.c index ccb772cc7..74e9bf9a9 100644 --- a/kernel/generic/gemm_beta.c +++ b/kernel/generic/gemm_beta.c @@ -1,5 +1,6 @@ /*********************************************************************/ /* Copyright 2009, 2010 The University of Texas at Austin. */ +/* Copyright 2025 The OpenBLAS Project. */ /* All rights reserved. */ /* */ /* Redistribution and use in source and binary forms, with or */ @@ -38,11 +39,41 @@ #include "common.h" +#if defined(BFLOAT16) && defined(BGEMM) && defined(BFLOAT16CONVERSION) +static float +bfloat16tof32 (bfloat16 f16) +{ + float result = 0; + unsigned short* q = (unsigned short*)(&result); +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + q[0] = f16; +#else + q[1] = f16; +#endif + return result; +} +static bfloat16 +f32tobfloat16(float f32) +{ + unsigned short* q = (unsigned short*)(&f32); +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + return q[0]; +#else + return q[1]; +#endif +} + +#define BF16TOF32(x) (bfloat16tof32(x)) +#define F32TOBF16(x) (f32tobfloat16(x)) +#else +#define BF16TOF32(x) x +#define F32TOBF16(x) x +#endif + int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT beta, IFLOAT *dummy2, BLASLONG dummy3, IFLOAT *dummy4, BLASLONG dummy5, FLOAT *c, BLASLONG ldc){ - BLASLONG i, j; BLASLONG chunk, remain; FLOAT *c_offset1, *c_offset; @@ -54,18 +85,18 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT beta, c_offset1 = c_offset; c_offset += ldc; for(i=chunk; i>0; i--){ - *(c_offset1 + 0) = ZERO; - *(c_offset1 + 1) = ZERO; - *(c_offset1 + 2) = ZERO; - *(c_offset1 + 3) = ZERO; - *(c_offset1 + 4) = ZERO; - *(c_offset1 + 5) = ZERO; - *(c_offset1 + 6) = ZERO; - *(c_offset1 + 7) = ZERO; + *(c_offset1 + 0) = F32TOBF16(ZERO); + *(c_offset1 + 1) = F32TOBF16(ZERO); + *(c_offset1 + 2) = F32TOBF16(ZERO); + *(c_offset1 + 3) = F32TOBF16(ZERO); + *(c_offset1 + 4) = F32TOBF16(ZERO); + *(c_offset1 + 5) = F32TOBF16(ZERO); + *(c_offset1 + 6) = F32TOBF16(ZERO); + *(c_offset1 + 7) = F32TOBF16(ZERO); c_offset1 += 8; } for(i=remain; i>0; i--){ - *c_offset1 = ZERO; + *c_offset1 = F32TOBF16(ZERO); c_offset1 ++; } } @@ -74,18 +105,18 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT beta, c_offset1 = c_offset; c_offset += ldc; for(i=chunk; i>0; i--){ - *(c_offset1 + 0) *= beta; - *(c_offset1 + 1) *= beta; - *(c_offset1 + 2) *= beta; - *(c_offset1 + 3) *= beta; - *(c_offset1 + 4) *= beta; - *(c_offset1 + 5) *= beta; - *(c_offset1 + 6) *= beta; - *(c_offset1 + 7) *= beta; + *(c_offset1 + 0) = F32TOBF16(BF16TOF32(beta) * BF16TOF32(c_offset1[0])); + *(c_offset1 + 1) = F32TOBF16(BF16TOF32(beta) * BF16TOF32(c_offset1[1])); + *(c_offset1 + 2) = F32TOBF16(BF16TOF32(beta) * BF16TOF32(c_offset1[2])); + *(c_offset1 + 3) = F32TOBF16(BF16TOF32(beta) * BF16TOF32(c_offset1[3])); + *(c_offset1 + 4) = F32TOBF16(BF16TOF32(beta) * BF16TOF32(c_offset1[4])); + *(c_offset1 + 5) = F32TOBF16(BF16TOF32(beta) * BF16TOF32(c_offset1[5])); + *(c_offset1 + 6) = F32TOBF16(BF16TOF32(beta) * BF16TOF32(c_offset1[6])); + *(c_offset1 + 7) = F32TOBF16(BF16TOF32(beta) * BF16TOF32(c_offset1[7])); c_offset1 += 8; } for(i=remain; i>0; i--){ - *c_offset1 *= beta; + *c_offset1 = F32TOBF16(BF16TOF32(beta) * BF16TOF32(c_offset1[0])); c_offset1 ++; } } diff --git a/kernel/generic/gemmkernel_2x2.c b/kernel/generic/gemmkernel_2x2.c index bf1c3ae38..add84f043 100644 --- a/kernel/generic/gemmkernel_2x2.c +++ b/kernel/generic/gemmkernel_2x2.c @@ -12,9 +12,29 @@ bfloat16tof32 (bfloat16 f16) #endif return result; } + +static bfloat16 f32tobfloat16(float f32) { + unsigned short *q = (unsigned short *)(&f32); +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + return q[0]; +#else + return q[1]; +#endif +} + +#ifdef BGEMM +#define ALPHA bfloat16tof32(alpha) #define BF16TOF32(x) (bfloat16tof32(x)) +#define F32TOBF16(x) (f32tobfloat16(x)) #else +#define ALPHA alpha +#define BF16TOF32(x) (bfloat16tof32(x)) +#define F32TOBF16(x) x +#endif +#else +#define ALPHA alpha #define BF16TOF32(x) x +#define F32TOBF16(x) x #endif int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alpha,IFLOAT* ba,IFLOAT* bb,FLOAT* C,BLASLONG ldc #ifdef TRMMKERNEL @@ -25,7 +45,11 @@ int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alpha,IFLOAT* ba,IFLOAT* bb, BLASLONG i,j,k; FLOAT *C0,*C1; IFLOAT *ptrba,*ptrbb; +#ifdef BGEMM + float res0,res1,res2,res3; +#else FLOAT res0,res1,res2,res3; +#endif IFLOAT load0,load1,load2,load3,load4,load5,load6,load7; for (j=0; j SBBLAT3.SUMM @$(GREP) -q FATAL SBBLAT3.SUMM && cat SBBLAT3.SUMM || exit 0 + OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 ./test_bgemm > BBLAT3.SUMM + @$(GREP) -q FATAL BBLAT3.SUMM && cat BBLAT3.SUMM || exit 0 endif ifeq ($(BUILD_SINGLE),1) OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 ./sblat3 < ./sblat3.dat @@ -223,6 +254,8 @@ ifeq ($(USE_OPENMP), 1) ifeq ($(BUILD_BFLOAT16),1) OMP_NUM_THREADS=2 ./test_sbgemm > SBBLAT3.SUMM @$(GREP) -q FATAL SBBLAT3.SUMM && cat SBBLAT3.SUMM || exit 0 + OMP_NUM_THREADS=2 ./test_bgemm > BBLAT3.SUMM + @$(GREP) -q FATAL BBLAT3.SUMM && cat BBLAT3.SUMM || exit 0 endif ifeq ($(BUILD_SINGLE),1) OMP_NUM_THREADS=2 ./sblat3 < ./sblat3.dat @@ -244,6 +277,8 @@ else ifeq ($(BUILD_BFLOAT16),1) OPENBLAS_NUM_THREADS=2 ./test_sbgemm > SBBLAT3.SUMM @$(GREP) -q FATAL SBBLAT3.SUMM && cat SBBLAT3.SUMM || exit 0 + OPENBLAS_NUM_THREADS=2 ./test_bgemm > BBLAT3.SUMM + @$(GREP) -q FATAL BBLAT3.SUMM && cat BBLAT3.SUMM || exit 0 endif ifeq ($(BUILD_SINGLE),1) OPENBLAS_NUM_THREADS=2 ./sblat3 < ./sblat3.dat @@ -367,6 +402,9 @@ zblat3 : zblat3.$(SUFFIX) ../$(LIBNAME) endif ifeq ($(BUILD_BFLOAT16),1) +test_bgemm : compare_sgemm_bgemm.c ../$(LIBNAME) + $(CC) $(CLDFLAGS) -o test_bgemm compare_sgemm_bgemm.c ../$(LIBNAME) $(EXTRALIB) $(CEXTRALIB) + test_sbgemm : compare_sgemm_sbgemm.c ../$(LIBNAME) $(CC) $(CLDFLAGS) -o test_sbgemm compare_sgemm_sbgemm.c ../$(LIBNAME) $(EXTRALIB) $(CEXTRALIB) endif @@ -387,7 +425,7 @@ clean: @rm -f *.$(SUFFIX) *.$(PSUFFIX) gmon.$(SUFFIX)ut *.SUMM *.cxml *.exe *.pdb *.dwf \ sblat1 dblat1 cblat1 zblat1 \ sblat2 dblat2 cblat2 zblat2 \ - test_sbgemm sblat3 dblat3 cblat3 zblat3 \ + test_bgemm test_sbgemm sblat3 dblat3 cblat3 zblat3 \ sblat1p dblat1p cblat1p zblat1p \ sblat2p dblat2p cblat2p zblat2p \ sblat3p dblat3p cblat3p zblat3p \ diff --git a/test/compare_sgemm_bgemm.c b/test/compare_sgemm_bgemm.c new file mode 100644 index 000000000..2858b782d --- /dev/null +++ b/test/compare_sgemm_bgemm.c @@ -0,0 +1,224 @@ +/*************************************************************************** +Copyright (c) 2025 The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ +#include "../common.h" +#include +#include + +#include + +#define SGEMM BLASFUNC(sgemm) +#define BGEMM BLASFUNC(bgemm) +#define BGEMM_LARGEST 256 + +typedef union +{ + unsigned short v; +#if defined(_AIX) + struct __attribute__((packed)) +#else + struct +#endif + { +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + unsigned short s:1; + unsigned short e:8; + unsigned short m:7; +#else + unsigned short m:7; + unsigned short e:8; + unsigned short s:1; +#endif + } bits; +} bfloat16_bits; + +typedef union +{ + float v; +#if defined(_AIX) + struct __attribute__((packed)) +#else + struct +#endif + { +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + uint32_t s:1; + uint32_t e:8; + uint32_t m:23; +#else + uint32_t m:23; + uint32_t e:8; + uint32_t s:1; +#endif + } bits; +} float32_bits; + +float +float16to32 (bfloat16_bits f16) +{ + float32_bits f32; + f32.bits.s = f16.bits.s; + f32.bits.e = f16.bits.e; + f32.bits.m = (uint32_t) f16.bits.m << 16; + return f32.v; +} + +bfloat16 +float32to16 (float32_bits f32) +{ + bfloat16_bits f16; + f16.bits.s = f32.bits.s; + f16.bits.e = f32.bits.e; + f16.bits.m = (f32.bits.m >> 16) & 0x7f; + return f16.v; +} + +static float truncate_float(float value) { + bfloat16_bits f16 = (bfloat16_bits)float32to16((float32_bits)value); + return float16to32(f16); +} + +void *malloc_safe(size_t size) { + if (size == 0) + return malloc(1); + else + return malloc(size); +} + +int +main (int argc, char *argv[]) +{ + blasint m, n, k; + int i, j, l; + blasint x, y; + int ret = 0; + int loop = BGEMM_LARGEST; + char transA = 'N', transB = 'N'; + float alpha = 1.0, beta = 0.0; + bfloat16 alpha_bf16 = float32to16((float32_bits)alpha); + bfloat16 beta_bf16 = float32to16((float32_bits)beta); + + for (x = 0; x <= loop; x++) + { + if ((x > 100) && (x != BGEMM_LARGEST)) continue; + m = k = n = x; + float *A = (float *)malloc_safe(m * k * sizeof(FLOAT)); + float *B = (float *)malloc_safe(k * n * sizeof(FLOAT)); + float *C = (float *)malloc_safe(m * n * sizeof(FLOAT)); + bfloat16_bits *AA = (bfloat16_bits *)malloc_safe(m * k * sizeof(bfloat16_bits)); + bfloat16_bits *BB = (bfloat16_bits *)malloc_safe(k * n * sizeof(bfloat16_bits)); + bfloat16_bits *CC = (bfloat16_bits *)malloc_safe(k * n * sizeof(bfloat16_bits)); + FLOAT *DD = (FLOAT *)malloc_safe(m * n * sizeof(FLOAT)); + if ((A == NULL) || (B == NULL) || (C == NULL) || (AA == NULL) || (BB == NULL) || + (DD == NULL) || (CC == NULL)) + return 1; + bfloat16 atmp,btmp; + blasint one=1; + + for (j = 0; j < m; j++) + { + for (i = 0; i < k; i++) + { + A[j * k + i] = ((FLOAT) rand () / (FLOAT) RAND_MAX) + 0.5; + sbstobf16_(&one, &A[j*k+i], &one, &atmp, &one); + AA[j * k + i].v = atmp; + } + } + for (j = 0; j < n; j++) + { + for (i = 0; i < k; i++) + { + B[j * k + i] = ((FLOAT) rand () / (FLOAT) RAND_MAX) + 0.5; + sbstobf16_(&one, &B[j*k+i], &one, &btmp, &one); + BB[j * k + i].v = btmp; + } + } + for (y = 0; y < 4; y++) + { + if ((y == 0) || (y == 2)) { + transA = 'N'; + } else { + transA = 'T'; + } + if ((y == 0) || (y == 1)) { + transB = 'N'; + } else { + transB = 'T'; + } + + memset(CC, 0, m * n * sizeof(bfloat16_bits)); + memset(DD, 0, m * n * sizeof(FLOAT)); + memset(C, 0, m * n * sizeof(FLOAT)); + + SGEMM (&transA, &transB, &m, &n, &k, &alpha, A, + &m, B, &k, &beta, C, &m); + BGEMM (&transA, &transB, &m, &n, &k, &alpha_bf16, (bfloat16*) AA, + &m, (bfloat16*)BB, &k, &beta_bf16, (bfloat16*)CC, &m); + + for (i = 0; i < n; i++) + for (j = 0; j < m; j++) + { + for (l = 0; l < k; l++) + if (transA == 'N' && transB == 'N') + { + DD[i * m + j] += + float16to32 (AA[l * m + j]) * float16to32 (BB[l + k * i]); + } else if (transA == 'T' && transB == 'N') + { + DD[i * m + j] += + float16to32 (AA[k * j + l]) * float16to32 (BB[l + k * i]); + } else if (transA == 'N' && transB == 'T') + { + DD[i * m + j] += + float16to32 (AA[l * m + j]) * float16to32 (BB[i + l * n]); + } else if (transA == 'T' && transB == 'T') + { + DD[i * m + j] += + float16to32 (AA[k * j + l]) * float16to32 (BB[i + l * n]); + } + if (fabs(float16to32(CC[i * m + j]) - truncate_float(C[i * m + j])) > 2.0) { + ret++; + } + if (fabs(float16to32(CC[i * m + j]) - truncate_float(DD[i * m + j])) > 1.0) { + ret++; + } + + } + } + free(A); + free(B); + free(C); + free(AA); + free(BB); + free(CC); + free(DD); + } + + if (ret != 0) { + fprintf (stderr, "FATAL ERROR BGEMM - Return code: %d\n", ret); + return ret; + } +} From 73bf0b941a8a3ccd6cceb8fca70b1f7b15595d9f Mon Sep 17 00:00:00 2001 From: Chris Sidebottom Date: Mon, 7 Jul 2025 10:40:33 +0000 Subject: [PATCH 15/36] Add bgemm to gensymbol --- exports/gensymbol | 2 +- exports/gensymbol.pl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/exports/gensymbol b/exports/gensymbol index 3719574ea..17fbd2877 100755 --- a/exports/gensymbol +++ b/exports/gensymbol @@ -51,7 +51,7 @@ blasobjsz=" zgeadd dzsum zgemmt zgemmtr" blasobjs="lsame xerbla" -bfblasobjs="sbgemm sbgemmt sbgemmtr sbgemv sbdot sbstobf16 sbdtobf16 sbf16tos dbf16tod" +bfblasobjs="bgemm sbgemm sbgemmt sbgemmtr sbgemv sbdot sbstobf16 sbdtobf16 sbf16tos dbf16tod" hfblasobjs="shgemm" cblasobjsc=" cblas_caxpy cblas_ccopy cblas_cdotc cblas_cdotu cblas_cgbmv cblas_cgemm cblas_cgemv diff --git a/exports/gensymbol.pl b/exports/gensymbol.pl index 5a8423697..01f68fbb3 100644 --- a/exports/gensymbol.pl +++ b/exports/gensymbol.pl @@ -51,7 +51,7 @@ zgeadd, dzsum, zgemmt,zgemmtr); @blasobjs = (lsame, xerbla); -@bfblasobjs = (sbgemm, sbgemmt, sbgemmtr, sbgemv, sbdot, sbstobf16, sbdtobf16, sbf16tos, dbf16tod); +@bfblasobjs = (bgemm, sbgemm, sbgemmt, sbgemmtr, sbgemv, sbdot, sbstobf16, sbdtobf16, sbf16tos, dbf16tod); @hfblasobjs = (shgemm); @cblasobjsc = ( cblas_caxpy, cblas_ccopy, cblas_cdotc, cblas_cdotu, cblas_cgbmv, cblas_cgemm, cblas_cgemv, From 48394384ef3bcd1eb6ca96ea5b690bc75d459a49 Mon Sep 17 00:00:00 2001 From: Chris Sidebottom Date: Mon, 7 Jul 2025 11:09:26 +0000 Subject: [PATCH 16/36] Use correct constants for per-target BGEMM/SBGEMM This fixes the build and tests on `NEOVERSEV1` target, which was failing with specific constants for `SBGEMM` Co-authored-by: Ye Tao --- driver/level3/level3.c | 22 +++++++++++++++++----- driver/level3/level3_thread.c | 22 +++++++++++++++++----- 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/driver/level3/level3.c b/driver/level3/level3.c index 5d3438450..78bc6aa52 100644 --- a/driver/level3/level3.c +++ b/driver/level3/level3.c @@ -170,6 +170,22 @@ #define STOP_RPCC(COUNTER) #endif +#if defined(BUILD_BFLOAT16) +#if defined(DYNAMIC_ARCH) + #if defined(BGEMM) + #define BFLOAT16_ALIGN_K gotoblas->bgemm_align_k + #else + #define BFLOAT16_ALIGN_K gotoblas->sbgemm_align_k + #endif +#else + #if defined(BGEMM) + #define BFLOAT16_ALIGN_K BGEMM_ALIGN_K + #else + #define BFLOAT16_ALIGN_K SBGEMM_ALIGN_K + #endif +#endif +#endif + int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, XFLOAT *sa, XFLOAT *sb, BLASLONG dummy){ BLASLONG k, lda, ldb, ldc; @@ -307,11 +323,7 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, BLASLONG pad_min_l = min_l; #if defined(BFLOAT16) -#if defined(DYNAMIC_ARCH) - pad_min_l = (min_l + gotoblas->sbgemm_align_k - 1) & ~(gotoblas->sbgemm_align_k-1); -#else - pad_min_l = (min_l + SBGEMM_ALIGN_K - 1) & ~(SBGEMM_ALIGN_K - 1);; -#endif + pad_min_l = (min_l + BFLOAT16_ALIGN_K - 1) & ~(BFLOAT16_ALIGN_K - 1); #endif /* First, we have to move data A to L2 cache */ diff --git a/driver/level3/level3_thread.c b/driver/level3/level3_thread.c index 5ede6153e..cb93591ab 100644 --- a/driver/level3/level3_thread.c +++ b/driver/level3/level3_thread.c @@ -216,6 +216,22 @@ typedef struct { #define STOP_RPCC(COUNTER) #endif +#if defined(BUILD_BFLOAT16) +#if defined(DYNAMIC_ARCH) + #if defined(BGEMM) + #define BFLOAT16_ALIGN_K gotoblas->bgemm_align_k + #else + #define BFLOAT16_ALIGN_K gotoblas->sbgemm_align_k + #endif +#else + #if defined(BGEMM) + #define BFLOAT16_ALIGN_K BGEMM_ALIGN_K + #else + #define BFLOAT16_ALIGN_K SBGEMM_ALIGN_K + #endif +#endif +#endif + static int inner_thread(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, IFLOAT *sa, IFLOAT *sb, BLASLONG mypos){ IFLOAT *buffer[DIVIDE_RATE]; @@ -325,11 +341,7 @@ static int inner_thread(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, BLASLONG pad_min_l = min_l; #if defined(BFLOAT16) -#if defined(DYNAMIC_ARCH) - pad_min_l = (min_l + gotoblas->sbgemm_align_k - 1) & ~(gotoblas->sbgemm_align_k-1); -#else - pad_min_l = (min_l + SBGEMM_ALIGN_K - 1) & ~(SBGEMM_ALIGN_K - 1);; -#endif + pad_min_l = (min_l + BFLOAT16_ALIGN_K - 1) & ~(BFLOAT16_ALIGN_K - 1); #endif /* Determine step size in m From 66d9185ebe80cb65935f098865651b9743efebc7 Mon Sep 17 00:00:00 2001 From: Chris Sidebottom Date: Tue, 8 Jul 2025 22:47:20 +0000 Subject: [PATCH 17/36] Fix CMake support --- Makefile.tail | 30 ++++++++++++++++++++++++++++- cmake/kernel.cmake | 38 +++++++++++++++++++++++++++++++++++++ cmake/prebuild.cmake | 30 +++++++++++++++++++++++++++++ cmake/utils.cmake | 30 ++++++++++++++++++++++++++++- interface/CMakeLists.txt | 28 +++++++++++++++++++++++++++ interface/gemm.c | 2 +- kernel/CMakeLists.txt | 41 ++++++++++++++++++++++++++++++++++++++++ 7 files changed, 196 insertions(+), 3 deletions(-) diff --git a/Makefile.tail b/Makefile.tail index ddd74dcad..4a12f531e 100644 --- a/Makefile.tail +++ b/Makefile.tail @@ -1,3 +1,31 @@ +############################################################################### +# Copyright (c) 2025, The OpenBLAS Project +# All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# 3. Neither the name of the OpenBLAS project nor the names of +# its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +############################################################################### + BBLASOBJS_P = $(BBLASOBJS:.$(SUFFIX)=.$(PSUFFIX)) SBBLASOBJS_P = $(SBBLASOBJS:.$(SUFFIX)=.$(PSUFFIX)) SHBLASPBJS_P = $(SHBLASOBJS:.$(SUFFIX)=.$(PSUFFIX)) @@ -27,7 +55,7 @@ BLASOBJS_P += $(QBLASOBJS_P) $(XBLASOBJS_P) endif $(SHBLASOBJS) $(SHBLASOBJS_P) : override CFLAGS += -DHFLOAT16 -UDOUBLE -UCOMPLEX -$(BBLASOBJS) $(BBLASOBJS_P) : override CFLAGS += -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX -USMALL_MATRIX_OPT +$(BBLASOBJS) $(BBLASOBJS_P) : override CFLAGS += -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX $(SBBLASOBJS) $(SBBLASOBJS_P) : override CFLAGS += -DBFLOAT16 -UDOUBLE -UCOMPLEX $(SBLASOBJS) $(SBLASOBJS_P) : override CFLAGS += -UDOUBLE -UCOMPLEX $(DBLASOBJS) $(DBLASOBJS_P) : override CFLAGS += -DDOUBLE -UCOMPLEX diff --git a/cmake/kernel.cmake b/cmake/kernel.cmake index 2cea6d9e6..c0dd32ca0 100644 --- a/cmake/kernel.cmake +++ b/cmake/kernel.cmake @@ -1,3 +1,31 @@ +############################################################################### +# Copyright (c) 2025, The OpenBLAS Project +# All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# 3. Neither the name of the OpenBLAS project nor the names of +# its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +############################################################################### + # helper functions for the kernel CMakeLists.txt function(SetFallback KERNEL SOURCE_PATH) @@ -206,6 +234,16 @@ macro(SetDefaultL3) SetFallback(ZGEADD_KERNEL ../generic/zgeadd.c) if (BUILD_BFLOAT16) SetFallback(SHGEADD_KERNEL ../generic/geadd.c) + SetFallback(BGEMMKERNEL ../generic/gemmkernel_2x2.c) + SetFallback(BGEMM_BETA ../generic/gemm_beta.c) + SetFallback(BGEMMINCOPY ../generic/gemm_ncopy_2.c) + SetFallback(BGEMMITCOPY ../generic/gemm_tcopy_2.c) + SetFallback(BGEMMONCOPY ../generic/gemm_ncopy_2.c) + SetFallback(BGEMMOTCOPY ../generic/gemm_tcopy_2.c) + SetFallback(BGEMMINCOPYOBJ bgemm_incopy.o) + SetFallback(BGEMMITCOPYOBJ bgemm_itcopy.o) + SetFallback(BGEMMONCOPYOBJ bgemm_oncopy.o) + SetFallback(BGEMMOTCOPYOBJ bgemm_otcopy.o) SetFallback(SBGEMMKERNEL ../generic/gemmkernel_2x2.c) SetFallback(SBGEMM_BETA ../generic/gemm_beta.c) SetFallback(SBGEMMINCOPY ../generic/gemm_ncopy_2.c) diff --git a/cmake/prebuild.cmake b/cmake/prebuild.cmake index 8afe8b387..fe592ed73 100644 --- a/cmake/prebuild.cmake +++ b/cmake/prebuild.cmake @@ -1,3 +1,31 @@ +############################################################################### +# Copyright (c) 2025, The OpenBLAS Project +# All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# 3. Neither the name of the OpenBLAS project nor the names of +# its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +############################################################################### + ## ## Author: Hank Anderson ## Description: Ported from OpenBLAS/Makefile.prebuild @@ -131,6 +159,8 @@ if (DEFINED CORE AND CMAKE_CROSSCOMPILING AND NOT (${HOST_OS} STREQUAL "WINDOWSS set(HAVE_SSE2 1) set(HAVE_SSE3 1) set(HAVE_SSSE3 1) + set(BGEMM_UNROLL_M 8) + set(BGEMM_UNROLL_N 4) set(SBGEMM_UNROLL_M 8) set(SBGEMM_UNROLL_N 4) set(SGEMM_UNROLL_M 8) diff --git a/cmake/utils.cmake b/cmake/utils.cmake index a93f21686..1b19f41bc 100644 --- a/cmake/utils.cmake +++ b/cmake/utils.cmake @@ -1,3 +1,31 @@ +############################################################################### +# Copyright (c) 2025, The OpenBLAS Project +# All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# 3. Neither the name of the OpenBLAS project nor the names of +# its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +############################################################################### + # Functions to help with the OpenBLAS build # Reads string from getarch into CMake vars. Format of getarch vars is VARNAME=VALUE @@ -347,7 +375,7 @@ function(GenerateNamedObjects sources_in) if (NOT no_float_type) string(SUBSTRING ${float_type} 0 1 float_char) string(TOLOWER ${float_char} float_char) - if (${float_type} STREQUAL "BFLOAT16") + if (${float_type} STREQUAL "BFLOAT16" AND NOT "${defines_in}" MATCHES "BGEMM") set (float_char "sb") endif () endif () diff --git a/interface/CMakeLists.txt b/interface/CMakeLists.txt index c4306d8ee..995bebec3 100644 --- a/interface/CMakeLists.txt +++ b/interface/CMakeLists.txt @@ -1,3 +1,30 @@ +############################################################################### +# Copyright (c) 2025, The OpenBLAS Project +# All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# 3. Neither the name of the OpenBLAS project nor the names of +# its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +############################################################################### include_directories(${PROJECT_SOURCE_DIR}) include_directories(${PROJECT_BINARY_DIR}) @@ -124,6 +151,7 @@ endif () if (BUILD_BFLOAT16) GenerateNamedObjects("bf16dot.c" "" "sbdot" ${CBLAS_FLAG} "" "" true "BFLOAT16") + GenerateNamedObjects("gemm.c" "BGEMM" "bgemm" ${CBLAS_FLAG} "" "" true "BFLOAT16") GenerateNamedObjects("gemm.c" "" "sbgemm" ${CBLAS_FLAG} "" "" true "BFLOAT16") GenerateNamedObjects("sbgemmt.c" "" "sbgemmt" ${CBLAS_FLAG} "" "" true "BFLOAT16") GenerateNamedObjects("sbgemmt.c" "RNAME" "sbgemmtr" ${CBLAS_FLAG} "" "" true "BFLOAT16") diff --git a/interface/gemm.c b/interface/gemm.c index c21fd988b..a64d78d97 100644 --- a/interface/gemm.c +++ b/interface/gemm.c @@ -118,7 +118,7 @@ static int (*gemm[])(blas_arg_t *, BLASLONG *, BLASLONG *, IFLOAT *, IFLOAT *, B #endif }; -#if defined(SMALL_MATRIX_OPT) && !defined(GEMM3M) && !defined(XDOUBLE) &&!defined(HFLOAT16) +#if defined(SMALL_MATRIX_OPT) && !defined(GEMM3M) && !defined(XDOUBLE) && !defined(HFLOAT16) && !defined(BGEMM) #define USE_SMALL_MATRIX_OPT 1 #else #define USE_SMALL_MATRIX_OPT 0 diff --git a/kernel/CMakeLists.txt b/kernel/CMakeLists.txt index 9434f114e..1a5d32e07 100644 --- a/kernel/CMakeLists.txt +++ b/kernel/CMakeLists.txt @@ -1,3 +1,30 @@ +############################################################################### +# Copyright (c) 2025, The OpenBLAS Project +# All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# 3. Neither the name of the OpenBLAS project nor the names of +# its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +############################################################################### include_directories(${PROJECT_SOURCE_DIR}) @@ -336,6 +363,20 @@ function (build_core TARGET_CORE KDIR TSUFFIX KERNEL_DEFINITIONS) endif () if (BUILD_BFLOAT16) + if (BGEMMINCOPY) + GenerateNamedObjects("${KERNELDIR}/${BGEMMINCOPY}" "BGEMM" "${BGEMMINCOPYOBJ}" false "" "" true "BFLOAT16") + endif () + if (BGEMMITCOPY) + GenerateNamedObjects("${KERNELDIR}/${BGEMMITCOPY}" "BGEMM" "${BGEMMITCOPYOBJ}" false "" "" true "BFLOAT16") + endif () + if (BGEMMONCOPY) + GenerateNamedObjects("${KERNELDIR}/${BGEMMONCOPY}" "BGEMM" "${BGEMMONCOPYOBJ}" false "" "" true "BFLOAT16") + endif () + if (BGEMMOTCOPY) + GenerateNamedObjects("${KERNELDIR}/${BGEMMOTCOPY}" "BGEMM" "${BGEMMOTCOPYOBJ}" false "" "" true "BFLOAT16") + endif () + GenerateNamedObjects("${KERNELDIR}/${BGEMMKERNEL}" "BGEMM" "gemm_kernel" false "" "" false "BFLOAT16") + GenerateNamedObjects("${KERNELDIR}/${BGEMM_BETA}" "BGEMM" "gemm_beta" false "" "" false "BFLOAT16") if (SBGEMMINCOPY) GenerateNamedObjects("${KERNELDIR}/${SBGEMMINCOPY}" "" "${SBGEMMINCOPYOBJ}" false "" "" true "BFLOAT16") endif () From 8cd4be8d47a797f7912ffbe6d377732c859a6501 Mon Sep 17 00:00:00 2001 From: Chris Sidebottom Date: Wed, 9 Jul 2025 08:27:18 +0100 Subject: [PATCH 18/36] Temporarily disable test_bgemm --- test/Makefile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/Makefile b/test/Makefile index 7ac87f7f6..cb7efea20 100644 --- a/test/Makefile +++ b/test/Makefile @@ -229,8 +229,8 @@ ifneq ($(CROSS), 1) ifeq ($(BUILD_BFLOAT16),1) OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 ./test_sbgemm > SBBLAT3.SUMM @$(GREP) -q FATAL SBBLAT3.SUMM && cat SBBLAT3.SUMM || exit 0 - OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 ./test_bgemm > BBLAT3.SUMM - @$(GREP) -q FATAL BBLAT3.SUMM && cat BBLAT3.SUMM || exit 0 + # OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 ./test_bgemm > BBLAT3.SUMM + # @$(GREP) -q FATAL BBLAT3.SUMM && cat BBLAT3.SUMM || exit 0 endif ifeq ($(BUILD_SINGLE),1) OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 ./sblat3 < ./sblat3.dat @@ -254,8 +254,8 @@ ifeq ($(USE_OPENMP), 1) ifeq ($(BUILD_BFLOAT16),1) OMP_NUM_THREADS=2 ./test_sbgemm > SBBLAT3.SUMM @$(GREP) -q FATAL SBBLAT3.SUMM && cat SBBLAT3.SUMM || exit 0 - OMP_NUM_THREADS=2 ./test_bgemm > BBLAT3.SUMM - @$(GREP) -q FATAL BBLAT3.SUMM && cat BBLAT3.SUMM || exit 0 + # OMP_NUM_THREADS=2 ./test_bgemm > BBLAT3.SUMM + # @$(GREP) -q FATAL BBLAT3.SUMM && cat BBLAT3.SUMM || exit 0 endif ifeq ($(BUILD_SINGLE),1) OMP_NUM_THREADS=2 ./sblat3 < ./sblat3.dat @@ -277,8 +277,8 @@ else ifeq ($(BUILD_BFLOAT16),1) OPENBLAS_NUM_THREADS=2 ./test_sbgemm > SBBLAT3.SUMM @$(GREP) -q FATAL SBBLAT3.SUMM && cat SBBLAT3.SUMM || exit 0 - OPENBLAS_NUM_THREADS=2 ./test_bgemm > BBLAT3.SUMM - @$(GREP) -q FATAL BBLAT3.SUMM && cat BBLAT3.SUMM || exit 0 + # OPENBLAS_NUM_THREADS=2 ./test_bgemm > BBLAT3.SUMM + # @$(GREP) -q FATAL BBLAT3.SUMM && cat BBLAT3.SUMM || exit 0 endif ifeq ($(BUILD_SINGLE),1) OPENBLAS_NUM_THREADS=2 ./sblat3 < ./sblat3.dat From ff614575c96a6850948ff69f33d66e8f013a6318 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Wed, 9 Jul 2025 14:44:25 +0200 Subject: [PATCH 19/36] Fix arm64 HAVE_SME setting for DYNAMIC_ARCH builds --- kernel/CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/kernel/CMakeLists.txt b/kernel/CMakeLists.txt index 1a5d32e07..0ec08827d 100644 --- a/kernel/CMakeLists.txt +++ b/kernel/CMakeLists.txt @@ -238,6 +238,9 @@ function (build_core TARGET_CORE KDIR TSUFFIX KERNEL_DEFINITIONS) if (X86_64 OR ARM64) set(USE_DIRECT_SGEMM true) endif() + if (UC_TARGET_CORE MATCHES ARMV9SME) + set (HAVE_SME true) + endif () if (USE_DIRECT_SGEMM) # if (NOT DEFINED SGEMMDIRECTKERNEL) From b746f0eda300cba8cd2ead42fd5911064b9a0693 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Thu, 10 Jul 2025 08:58:16 +0200 Subject: [PATCH 20/36] Allocate IWORK to hold at least the one element for workspace queries --- lapack-netlib/LAPACKE/src/lapacke_dtrsen.c | 6 ++++++ lapack-netlib/LAPACKE/src/lapacke_strsen.c | 7 +++++++ 2 files changed, 13 insertions(+) diff --git a/lapack-netlib/LAPACKE/src/lapacke_dtrsen.c b/lapack-netlib/LAPACKE/src/lapacke_dtrsen.c index 255ea34ea..9e00865df 100644 --- a/lapack-netlib/LAPACKE/src/lapacke_dtrsen.c +++ b/lapack-netlib/LAPACKE/src/lapacke_dtrsen.c @@ -78,6 +78,12 @@ lapack_int LAPACKE_dtrsen( int matrix_layout, char job, char compq, info = LAPACK_WORK_MEMORY_ERROR; goto exit_level_0; } + } else { + iwork = (lapack_int*)LAPACKE_malloc( sizeof(lapack_int) ); + if( iwork == NULL ) { + info = LAPACK_WORK_MEMORY_ERROR; + goto exit_level_0; + } } work = (double*)LAPACKE_malloc( sizeof(double) * lwork ); if( work == NULL ) { diff --git a/lapack-netlib/LAPACKE/src/lapacke_strsen.c b/lapack-netlib/LAPACKE/src/lapacke_strsen.c index c9a5b845c..8f984ad0e 100644 --- a/lapack-netlib/LAPACKE/src/lapacke_strsen.c +++ b/lapack-netlib/LAPACKE/src/lapacke_strsen.c @@ -77,7 +77,14 @@ lapack_int LAPACKE_strsen( int matrix_layout, char job, char compq, info = LAPACK_WORK_MEMORY_ERROR; goto exit_level_0; } + } else { + iwork = (lapack_int*)LAPACKE_malloc( sizeof(lapack_int) ); + if( iwork == NULL ) { + info = LAPACK_WORK_MEMORY_ERROR; + goto exit_level_0; + } } + work = (float*)LAPACKE_malloc( sizeof(float) * lwork ); if( work == NULL ) { info = LAPACK_WORK_MEMORY_ERROR; From b37516add6e1d39ddeeb7139e6288e5259794e17 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Thu, 10 Jul 2025 14:59:01 +0200 Subject: [PATCH 21/36] Add BGEMM parameters --- driver/others/parameter.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/driver/others/parameter.c b/driver/others/parameter.c index 3bcb0d434..9a1ff5735 100644 --- a/driver/others/parameter.c +++ b/driver/others/parameter.c @@ -72,6 +72,11 @@ BLASLONG shgemm_p = DEFAULT_GEMM_P; #else BLASLONG shgemm_p = SHGEMM_P; #endif +#if BGEMM_P == bgemm_p +BLASLONG bgemm_p = DEFAULT_GEMM_P; +#else +BLASLONG bgemm_p = BGEMM_P; +#endif #if SGEMM_P == sgemm_p BLASLONG sgemm_p = DEFAULT_GEMM_P; #else @@ -103,6 +108,11 @@ BLASLONG shgemm_q = DEFAULT_GEMM_Q; #else BLASLONG shgemm_q = SHGEMM_Q; #endif +#if BGEMM_Q == bgemm_q +BLASLONG bgemm_q = DEFAULT_GEMM_Q; +#else +BLASLONG bgemm_q = BGEMM_Q; +#endif #if SGEMM_Q == sgemm_q BLASLONG sgemm_q = DEFAULT_GEMM_Q; #else @@ -134,6 +144,11 @@ BLASLONG shgemm_r = DEFAULT_GEMM_R; #else BLASLONG shgemm_r = SHGEMM_R; #endif +#if BGEMM_R == bgemm_r +BLASLONG bgemm_r = DEFAULT_GEMM_R; +#else +BLASLONG bgemm_r = BGEMM_R; +#endif #if SGEMM_R == sgemm_r BLASLONG sgemm_r = DEFAULT_GEMM_R; #else @@ -541,6 +556,7 @@ void blas_set_parameter(void){ #ifdef BUILD_BFLOAT16 sbgemm_r = (((BUFFER_SIZE - ((SBGEMM_P * SBGEMM_Q * 4 + GEMM_OFFSET_A + GEMM_ALIGN) & ~GEMM_ALIGN)) / (SBGEMM_Q * 4)) - 15) & ~15; + bgemm_r = (((BUFFER_SIZE - ((BGEMM_P * BGEMM_Q * 4 + GEMM_OFFSET_A + GEMM_ALIGN) & ~GEMM_ALIGN)) / (BGEMM_Q * 4)) - 15) & ~15; #endif #ifdef BUILD_HFLOAT16 shgemm_r = (((BUFFER_SIZE - ((SHGEMM_P * SHGEMM_Q * 4 + GEMM_OFFSET_A + GEMM_ALIGN) & ~GEMM_ALIGN)) / (SHGEMM_Q * 4)) - 15) & ~15; @@ -653,6 +669,7 @@ void blas_set_parameter(void){ #ifdef BUILD_BFLOAT16 sbgemm_r = (((BUFFER_SIZE - ((SBGEMM_P * SBGEMM_Q * 4 + GEMM_OFFSET_A + GEMM_ALIGN) & ~GEMM_ALIGN)) / (SBGEMM_Q * 4)) - 15) & ~15; + bgemm_r = (((BUFFER_SIZE - ((BGEMM_P * BGEMM_Q * 4 + GEMM_OFFSET_A + GEMM_ALIGN) & ~GEMM_ALIGN)) / (BGEMM_Q * 4)) - 15) & ~15; #endif #ifdef BUILD_HFLOAT16 shgemm_r = (((BUFFER_SIZE - ((SHGEMM_P * SHGEMM_Q * 4 + GEMM_OFFSET_A + GEMM_ALIGN) & ~GEMM_ALIGN)) / (SHGEMM_Q * 4)) - 15) & ~15; From 343830c26f70c169c0b6064ffca611609929d531 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Thu, 10 Jul 2025 14:59:46 +0200 Subject: [PATCH 22/36] Add BGEMM parameter tables --- kernel/setparam-ref.c | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/kernel/setparam-ref.c b/kernel/setparam-ref.c index 3b2fd42cc..9eb5959dc 100644 --- a/kernel/setparam-ref.c +++ b/kernel/setparam-ref.c @@ -926,6 +926,7 @@ gotoblas_t TABLE_NAME = { static void init_parameter(void) { #if (BUILD_BFLOAT16) TABLE_NAME.sbgemm_p = SBGEMM_DEFAULT_P; + TABLE_NAME.bgemm_p = BGEMM_DEFAULT_P; #endif #if (BUILD_SINGLE==1) || (BUILD_COMPLEX==1) TABLE_NAME.sgemm_p = SGEMM_DEFAULT_P; @@ -942,6 +943,7 @@ static void init_parameter(void) { #if (BUILD_BFLOAT16) TABLE_NAME.sbgemm_q = SBGEMM_DEFAULT_Q; + TABLE_NAME.bgemm_q = BGEMM_DEFAULT_Q; #endif #if BUILD_SINGLE == 1 || (BUILD_COMPLEX==1) TABLE_NAME.sgemm_q = SGEMM_DEFAULT_Q; @@ -958,6 +960,7 @@ static void init_parameter(void) { #if (BUILD_BFLOAT16) TABLE_NAME.sbgemm_r = SBGEMM_DEFAULT_R; + TABLE_NAME.bgemm_r = BGEMM_DEFAULT_R; #endif #if BUILD_SINGLE == 1 || (BUILD_COMPLEX==1) TABLE_NAME.sgemm_r = SGEMM_DEFAULT_R; @@ -1113,10 +1116,12 @@ static void init_parameter(void) { #ifdef BUILD_BFLOAT16 TABLE_NAME.sbgemm_p = SBGEMM_DEFAULT_P; + TABLE_NAME.bgemm_p = BGEMM_DEFAULT_P; #endif #ifdef BUILD_BFLOAT16 TABLE_NAME.sbgemm_r = SBGEMM_DEFAULT_R; + TABLE_NAME.bgemm_r = BGEMM_DEFAULT_R; #endif #if defined(LA464) @@ -1215,6 +1220,7 @@ static void init_parameter(void) { #ifdef BUILD_BFLOAT16 TABLE_NAME.sbgemm_q = SBGEMM_DEFAULT_Q; + TABLE_NAME.bgemm_q = BGEMM_DEFAULT_Q; #endif } #else // (ARCH_LOONGARCH64) @@ -1223,6 +1229,7 @@ static void init_parameter(void) { #ifdef BUILD_BFLOAT16 TABLE_NAME.sbgemm_p = SBGEMM_DEFAULT_P; + TABLE_NAME.bgemm_p = BGEMM_DEFAULT_P; #endif TABLE_NAME.sgemm_p = SGEMM_DEFAULT_P; TABLE_NAME.dgemm_p = DGEMM_DEFAULT_P; @@ -1231,6 +1238,7 @@ static void init_parameter(void) { #ifdef BUILD_BFLOAT16 TABLE_NAME.sbgemm_r = SBGEMM_DEFAULT_R; + TABLE_NAME.bgemm_r = BGEMM_DEFAULT_R; #endif TABLE_NAME.sgemm_r = SGEMM_DEFAULT_R; TABLE_NAME.dgemm_r = DGEMM_DEFAULT_R; @@ -1240,6 +1248,7 @@ static void init_parameter(void) { #ifdef BUILD_BFLOAT16 TABLE_NAME.sbgemm_q = SBGEMM_DEFAULT_Q; + TABLE_NAME.bgemm_q = BGEMM_DEFAULT_Q; #endif TABLE_NAME.sgemm_q = SGEMM_DEFAULT_Q; TABLE_NAME.dgemm_q = DGEMM_DEFAULT_Q; @@ -1252,6 +1261,7 @@ static void init_parameter(void) { static void init_parameter(void) { #ifdef BUILD_BFLOAT16 TABLE_NAME.sbgemm_p = SBGEMM_DEFAULT_P; + TABLE_NAME.bgemm_p = BGEMM_DEFAULT_P; #endif TABLE_NAME.sgemm_p = SGEMM_DEFAULT_P; TABLE_NAME.dgemm_p = DGEMM_DEFAULT_P; @@ -1260,6 +1270,7 @@ static void init_parameter(void) { #ifdef BUILD_BFLOAT16 TABLE_NAME.sbgemm_r = SBGEMM_DEFAULT_R; + TABLE_NAME.bgemm_r = BGEMM_DEFAULT_R; #endif TABLE_NAME.sgemm_r = SGEMM_DEFAULT_R; TABLE_NAME.dgemm_r = DGEMM_DEFAULT_R; @@ -1269,6 +1280,7 @@ static void init_parameter(void) { #ifdef BUILD_BFLOAT16 TABLE_NAME.sbgemm_q = SBGEMM_DEFAULT_Q; + TABLE_NAME.bgemm_q = BGEMM_DEFAULT_Q; #endif TABLE_NAME.sgemm_q = SGEMM_DEFAULT_Q; TABLE_NAME.dgemm_q = DGEMM_DEFAULT_Q; @@ -1282,6 +1294,7 @@ static void init_parameter(void) { #ifdef BUILD_BFLOAT16 TABLE_NAME.sbgemm_p = SBGEMM_DEFAULT_P; + TABLE_NAME.bgemm_p = BGEMM_DEFAULT_P; #endif #ifdef BUILD_HFLOAT16 TABLE_NAME.shgemm_p = SHGEMM_DEFAULT_P; @@ -1293,6 +1306,7 @@ static void init_parameter(void) { #ifdef BUILD_BFLOAT16 TABLE_NAME.sbgemm_r = SBGEMM_DEFAULT_R; + TABLE_NAME.bgemm_r = BGEMM_DEFAULT_R; #endif #ifdef BUILD_HFLOAT16 TABLE_NAME.shgemm_r = SHGEMM_DEFAULT_R; @@ -1305,6 +1319,7 @@ static void init_parameter(void) { #ifdef BUILD_BFLOAT16 TABLE_NAME.sbgemm_q = SBGEMM_DEFAULT_Q; + TABLE_NAME.bgemm_q = BGEMM_DEFAULT_Q; #endif #ifdef BUILD_HFLOAT16 TABLE_NAME.shgemm_q = SHGEMM_DEFAULT_Q; @@ -1455,6 +1470,8 @@ static void init_parameter(void) { #ifdef BUILD_BFLOAT16 TABLE_NAME.sbgemm_p = SBGEMM_DEFAULT_P; TABLE_NAME.sbgemm_q = SBGEMM_DEFAULT_Q; + TABLE_NAME.bgemm_p = BGEMM_DEFAULT_P; + TABLE_NAME.bgemm_q = BGEMM_DEFAULT_Q; #endif #ifdef BUILD_HFLOAT16 TABLE_NAME.shgemm_p = SHGEMM_DEFAULT_P; @@ -2053,6 +2070,10 @@ static void init_parameter(void) { ((TABLE_NAME.sbgemm_p * TABLE_NAME.sbgemm_q * 4 + TABLE_NAME.offsetA + TABLE_NAME.align) & ~TABLE_NAME.align) ) / (TABLE_NAME.sbgemm_q * 4) - 15) & ~15); + TABLE_NAME.bgemm_r = (((BUFFER_SIZE - + ((TABLE_NAME.bgemm_p * TABLE_NAME.bgemm_q * 4 + TABLE_NAME.offsetA + + TABLE_NAME.align) & ~TABLE_NAME.align) + ) / (TABLE_NAME.bgemm_q * 4) - 15) & ~15); #endif #if BUILD_HFLOAT16==1 From b54aec804ebc7c9bedfa29bdb45f4065516494c8 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Thu, 10 Jul 2025 15:00:30 +0200 Subject: [PATCH 23/36] remove spurious include --- test/compare_sgemm_bgemm.c | 1 - 1 file changed, 1 deletion(-) diff --git a/test/compare_sgemm_bgemm.c b/test/compare_sgemm_bgemm.c index 2858b782d..8cddcba97 100644 --- a/test/compare_sgemm_bgemm.c +++ b/test/compare_sgemm_bgemm.c @@ -28,7 +28,6 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include -#include #define SGEMM BLASFUNC(sgemm) #define BGEMM BLASFUNC(bgemm) From 9a272fece67d6bf865943b4e3505ef0cad0e7d47 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Thu, 10 Jul 2025 15:02:59 +0200 Subject: [PATCH 24/36] Re-enable the BGEMM tests --- test/Makefile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/Makefile b/test/Makefile index cb7efea20..7ac87f7f6 100644 --- a/test/Makefile +++ b/test/Makefile @@ -229,8 +229,8 @@ ifneq ($(CROSS), 1) ifeq ($(BUILD_BFLOAT16),1) OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 ./test_sbgemm > SBBLAT3.SUMM @$(GREP) -q FATAL SBBLAT3.SUMM && cat SBBLAT3.SUMM || exit 0 - # OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 ./test_bgemm > BBLAT3.SUMM - # @$(GREP) -q FATAL BBLAT3.SUMM && cat BBLAT3.SUMM || exit 0 + OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 ./test_bgemm > BBLAT3.SUMM + @$(GREP) -q FATAL BBLAT3.SUMM && cat BBLAT3.SUMM || exit 0 endif ifeq ($(BUILD_SINGLE),1) OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 ./sblat3 < ./sblat3.dat @@ -254,8 +254,8 @@ ifeq ($(USE_OPENMP), 1) ifeq ($(BUILD_BFLOAT16),1) OMP_NUM_THREADS=2 ./test_sbgemm > SBBLAT3.SUMM @$(GREP) -q FATAL SBBLAT3.SUMM && cat SBBLAT3.SUMM || exit 0 - # OMP_NUM_THREADS=2 ./test_bgemm > BBLAT3.SUMM - # @$(GREP) -q FATAL BBLAT3.SUMM && cat BBLAT3.SUMM || exit 0 + OMP_NUM_THREADS=2 ./test_bgemm > BBLAT3.SUMM + @$(GREP) -q FATAL BBLAT3.SUMM && cat BBLAT3.SUMM || exit 0 endif ifeq ($(BUILD_SINGLE),1) OMP_NUM_THREADS=2 ./sblat3 < ./sblat3.dat @@ -277,8 +277,8 @@ else ifeq ($(BUILD_BFLOAT16),1) OPENBLAS_NUM_THREADS=2 ./test_sbgemm > SBBLAT3.SUMM @$(GREP) -q FATAL SBBLAT3.SUMM && cat SBBLAT3.SUMM || exit 0 - # OPENBLAS_NUM_THREADS=2 ./test_bgemm > BBLAT3.SUMM - # @$(GREP) -q FATAL BBLAT3.SUMM && cat BBLAT3.SUMM || exit 0 + OPENBLAS_NUM_THREADS=2 ./test_bgemm > BBLAT3.SUMM + @$(GREP) -q FATAL BBLAT3.SUMM && cat BBLAT3.SUMM || exit 0 endif ifeq ($(BUILD_SINGLE),1) OPENBLAS_NUM_THREADS=2 ./sblat3 < ./sblat3.dat From 740efd71c4ad0f6f56371cabd23f086b985e0602 Mon Sep 17 00:00:00 2001 From: Chris Sidebottom Date: Thu, 10 Jul 2025 23:23:27 +0000 Subject: [PATCH 25/36] Add optimized BGEMM kernel for NEOVERSEV1 target This also improves the testing and generic kernel by re-using the BF16 conversion functions. Built on top of https://github.com/OpenMathLib/OpenBLAS/pull/5357 and derived from https://github.com/OpenMathLib/OpenBLAS/pull/5287 Co-authored-by: Ye Tao --- benchmark/Makefile | 34 +- benchmark/gemm.c | 12 + common_b.h | 14 +- interface/Makefile | 30 +- kernel/Makefile.L3 | 21 +- kernel/arm64/KERNEL.NEOVERSEV1 | 43 +- kernel/arm64/bgemm_beta_neon.c | 107 +++++ kernel/arm64/bgemm_kernel_4x4_neoversev1.c | 50 ++ .../arm64/bgemm_kernel_4x4_neoversev1_impl.c | 429 ++++++++++++++++++ kernel/generic/gemmkernel_2x2.c | 54 ++- kernel/setparam-ref.c | 4 + param.h | 7 + test/compare_sgemm_bgemm.c | 110 ++--- 13 files changed, 816 insertions(+), 99 deletions(-) create mode 100644 kernel/arm64/bgemm_beta_neon.c create mode 100644 kernel/arm64/bgemm_kernel_4x4_neoversev1.c create mode 100644 kernel/arm64/bgemm_kernel_4x4_neoversev1_impl.c diff --git a/benchmark/Makefile b/benchmark/Makefile index 9316921e1..cdf87c0ab 100644 --- a/benchmark/Makefile +++ b/benchmark/Makefile @@ -1,3 +1,31 @@ +############################################################################### +# Copyright (c) 2025, The OpenBLAS Project +# All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# 3. Neither the name of the OpenBLAS project nor the names of +# its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +############################################################################### + TOPDIR = .. include $(TOPDIR)/Makefile.system @@ -56,7 +84,7 @@ GOTO_LAPACK_TARGETS= endif ifeq ($(BUILD_BFLOAT16),1) -GOTO_BFLOAT_TARGETS=sbgemm.goto +GOTO_BFLOAT_TARGETS=bgemm.goto sbgemm.goto else GOTO_BFLOAT_TARGETS= endif @@ -635,6 +663,8 @@ zcholesky.essl : zcholesky.$(SUFFIX) ##################################### Sgemm #################################################### ifeq ($(BUILD_BFLOAT16),1) +bgemm.goto : bgemm.$(SUFFIX) ../$(LIBNAME) + $(CC) $(CFLAGS) -o $(@F) $^ $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB) -lm sbgemm.goto : sbgemm.$(SUFFIX) ../$(LIBNAME) $(CC) $(CFLAGS) -o $(@F) $^ $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB) -lm endif @@ -2970,6 +3000,8 @@ zcholesky.$(SUFFIX) : cholesky.c $(CC) $(CFLAGS) -c -DCOMPLEX -DDOUBLE -o $(@F) $^ ifeq ($(BUILD_BFLOAT16),1) +bgemm.$(SUFFIX) : gemm.c + $(CC) $(CFLAGS) -c -DBFLOAT16 -DBGEMM -UCOMPLEX -UDOUBLE -o $(@F) $^ sbgemm.$(SUFFIX) : gemm.c $(CC) $(CFLAGS) -c -DBFLOAT16 -UCOMPLEX -UDOUBLE -o $(@F) $^ endif diff --git a/benchmark/gemm.c b/benchmark/gemm.c index a138bfe1e..704e33225 100644 --- a/benchmark/gemm.c +++ b/benchmark/gemm.c @@ -33,6 +33,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifdef DOUBLE #define GEMM BLASFUNC(dgemm) +#elif defined(BFLOAT16) && defined(BGEMM) +#define GEMM BLASFUNC(bgemm) #elif defined(BFLOAT16) #define GEMM BLASFUNC(sbgemm) #undef IFLOAT @@ -60,8 +62,18 @@ int main(int argc, char *argv[]){ IFLOAT *a, *b; FLOAT *c; +#ifdef BGEMM + blasint one=1; + blasint two=2; + float alpha_in[] = {1.0, 0.0}; + float beta_in[] = {0.0, 0.0}; + FLOAT alpha[2], beta[2]; + sbstobf16_(&two, alpha_in, &one, alpha, &one); + sbstobf16_(&two, beta_in, &one, beta, &one); +#else FLOAT alpha[] = {1.0, 0.0}; FLOAT beta [] = {0.0, 0.0}; +#endif char transa = 'N'; char transb = 'N'; blasint m, n, k, i, j, lda, ldb, ldc; diff --git a/common_b.h b/common_b.h index e03f6800d..4d77ec4fa 100644 --- a/common_b.h +++ b/common_b.h @@ -30,10 +30,16 @@ #define COMMON_B_H #ifndef DYNAMIC_ARCH -#define BGEMM_ONCOPY bgemm_oncopy -#define BGEMM_OTCOPY bgemm_otcopy -#define BGEMM_INCOPY bgemm_incopy -#define BGEMM_ITCOPY bgemm_itcopy +#define BGEMM_ONCOPY bgemm_oncopy +#define BGEMM_OTCOPY bgemm_otcopy + +#if BGEMM_DEFAULT_UNROLL_M == BGEMM_DEFAULT_UNROLL_N +#define BGEMM_INCOPY bgemm_oncopy +#define BGEMM_ITCOPY bgemm_otcopy +#else +#define BGEMM_INCOPY bgemm_incopy +#define BGEMM_ITCOPY bgemm_itcopy +#endif #define BGEMM_BETA bgemm_beta #define BGEMM_KERNEL bgemm_kernel diff --git a/interface/Makefile b/interface/Makefile index e14796cbb..3af12748f 100644 --- a/interface/Makefile +++ b/interface/Makefile @@ -1,3 +1,31 @@ +############################################################################### +# Copyright (c) 2025, The OpenBLAS Project +# All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# 3. Neither the name of the OpenBLAS project nor the names of +# its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +############################################################################### + TOPDIR = .. include $(TOPDIR)/Makefile.system @@ -526,7 +554,7 @@ ifneq ($(BUILD_COMPLEX16),1) ZBLASOBJS= endif -FUNCOBJS = $(SBEXTOBJS) $(CXERBLAOBJS) $(SBBLASOBJS) $(SBLASOBJS) $(DBLASOBJS) $(CBLASOBJS) $(ZBLASOBJS) $(SHBLASOBJS) +FUNCOBJS = $(SBEXTOBJS) $(CXERBLAOBJS) $(BBLASOBJS) $(SBBLASOBJS) $(SBLASOBJS) $(DBLASOBJS) $(CBLASOBJS) $(ZBLASOBJS) $(SHBLASOBJS) ifeq ($(EXPRECISION), 1) FUNCOBJS += $(QBLASOBJS) $(XBLASOBJS) diff --git a/kernel/Makefile.L3 b/kernel/Makefile.L3 index 06f18e6be..f45ccc42d 100644 --- a/kernel/Makefile.L3 +++ b/kernel/Makefile.L3 @@ -674,6 +674,10 @@ ZBLASOBJS += \ endif ifeq ($(BUILD_BFLOAT16), 1) +BGEMMINCOPYOBJ_P = $(BGEMMINCOPYOBJ:.$(SUFFIX)=.$(PSUFFIX)) +BGEMMITCOPYOBJ_P = $(BGEMMITCOPYOBJ:.$(SUFFIX)=.$(PSUFFIX)) +BGEMMONCOPYOBJ_P = $(BGEMMONCOPYOBJ:.$(SUFFIX)=.$(PSUFFIX)) +BGEMMOTCOPYOBJ_P = $(BGEMMOTCOPYOBJ:.$(SUFFIX)=.$(PSUFFIX)) SBGEMMINCOPYOBJ_P = $(SBGEMMINCOPYOBJ:.$(SUFFIX)=.$(PSUFFIX)) SBGEMMITCOPYOBJ_P = $(SBGEMMITCOPYOBJ:.$(SUFFIX)=.$(PSUFFIX)) SBGEMMONCOPYOBJ_P = $(SBGEMMONCOPYOBJ:.$(SUFFIX)=.$(PSUFFIX)) @@ -2998,6 +3002,20 @@ $(KDIR)xgemm_beta$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(XGEMM_BETA) ifeq ($(BUILD_BFLOAT16), 1) +$(BGEMMONCOPYOBJ_P) : $(KERNELDIR)/$(BGEMMONCOPY) + $(CC) $(PFLAGS) -c -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX $< -o $@ + +$(BGEMMOTCOPYOBJ_P) : $(KERNELDIR)/$(BGEMMOTCOPY) + $(CC) $(PFLAGS) -c -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX $< -o $@ + +ifneq ($(BGEMM_UNROLL_M), $(BGEMM_UNROLL_N)) +$(BGEMMINCOPYOBJ_P) : $(KERNELDIR)/$(BGEMMINCOPY) + $(CC) $(PFLAGS) -c -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX $< -o $@ + +$(BGEMMITCOPYOBJ_P) : $(KERNELDIR)/$(BGEMMITCOPY) + $(CC) $(PFLAGS) -c -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX $< -o $@ +endif + $(SBGEMMONCOPYOBJ_P) : $(KERNELDIR)/$(SBGEMMONCOPY) $(CC) $(PFLAGS) -c -DBFLOAT16 -UDOUBLE -UCOMPLEX $< -o $@ @@ -3010,7 +3028,6 @@ $(SBGEMMINCOPYOBJ_P) : $(KERNELDIR)/$(SBGEMMINCOPY) $(SBGEMMITCOPYOBJ_P) : $(KERNELDIR)/$(SBGEMMITCOPY) $(CC) $(PFLAGS) -c -DBFLOAT16 -UDOUBLE -UCOMPLEX $< -o $@ - endif endif @@ -3137,6 +3154,8 @@ endif ifeq ($(BUILD_BFLOAT16), 1) +$(KDIR)bgemm_kernel$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(BGEMMKERNEL) $(BGEMMDEPEND) + $(CC) $(PFLAGS) -c -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX $< -o $@ $(KDIR)sbgemm_kernel$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SBGEMMKERNEL) $(SBGEMMDEPEND) $(CC) $(PFLAGS) -c -DBFLOAT16 -UDOUBLE -UCOMPLEX $< -o $@ endif diff --git a/kernel/arm64/KERNEL.NEOVERSEV1 b/kernel/arm64/KERNEL.NEOVERSEV1 index 3e622bcbf..8bc0f35e5 100644 --- a/kernel/arm64/KERNEL.NEOVERSEV1 +++ b/kernel/arm64/KERNEL.NEOVERSEV1 @@ -1,3 +1,31 @@ +############################################################################### +# Copyright (c) 2025, The OpenBLAS Project +# All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# 3. Neither the name of the OpenBLAS project nor the names of +# its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +############################################################################### + include $(KERNELDIR)/KERNEL.ARMV8SVE SGEMVNKERNEL = gemv_n_sve_v1x3.c @@ -5,6 +33,19 @@ DGEMVNKERNEL = gemv_n_sve_v1x3.c SGEMVTKERNEL = gemv_t_sve_v1x3.c DGEMVTKERNEL = gemv_t_sve_v1x3.c ifeq ($(BUILD_BFLOAT16), 1) +BGEMM_BETA = bgemm_beta_neon.c +BGEMMKERNEL = bgemm_kernel_$(BGEMM_UNROLL_M)x$(BGEMM_UNROLL_N)_neoversev1.c +ifneq ($(BGEMM_UNROLL_M), $(BGEMM_UNROLL_N)) +BGEMMINCOPY = sbgemm_ncopy_$(SBGEMM_UNROLL_M)_neoversev1.c +BGEMMITCOPY = sbgemm_tcopy_$(SBGEMM_UNROLL_M)_neoversev1.c +BGEMMINCOPYOBJ = bgemm_incopy$(TSUFFIX).$(SUFFIX) +BGEMMITCOPYOBJ = bgemm_itcopy$(TSUFFIX).$(SUFFIX) +endif +BGEMMONCOPY = sbgemm_ncopy_$(BGEMM_UNROLL_N)_neoversev1.c +BGEMMOTCOPY = sbgemm_tcopy_$(BGEMM_UNROLL_N)_neoversev1.c +BGEMMONCOPYOBJ = bgemm_oncopy$(TSUFFIX).$(SUFFIX) +BGEMMOTCOPYOBJ = bgemm_otcopy$(TSUFFIX).$(SUFFIX) + SBGEMM_BETA = sbgemm_beta_neoversev1.c SBGEMMKERNEL = sbgemm_kernel_$(SBGEMM_UNROLL_M)x$(SBGEMM_UNROLL_N)_neoversev1.c ifneq ($(SBGEMM_UNROLL_M), $(SBGEMM_UNROLL_N)) @@ -21,4 +62,4 @@ SBGEMMOTCOPYOBJ = sbgemm_otcopy$(TSUFFIX).$(SUFFIX) SBGEMVNKERNEL = sbgemv_n_neon.c SBGEMVTKERNEL = sbgemv_t_bfdot.c -endif \ No newline at end of file +endif diff --git a/kernel/arm64/bgemm_beta_neon.c b/kernel/arm64/bgemm_beta_neon.c new file mode 100644 index 000000000..603377f8f --- /dev/null +++ b/kernel/arm64/bgemm_beta_neon.c @@ -0,0 +1,107 @@ +/*************************************************************************** + * Copyright (c) 2025, The OpenBLAS Project + * All rights reserved. + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of the OpenBLAS project nor the names of + * its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * *****************************************************************************/ + +#include "common.h" + +#include + +int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT beta_in, IFLOAT *dummy2, + BLASLONG dummy3, IFLOAT *dummy4, BLASLONG dummy5, FLOAT *c, + BLASLONG ldc) { + BLASLONG i, j; + BLASLONG chunk, remain; + + bfloat16_t *ptr_c, *ptr_c0; + + bfloat16x8_t x0, z0; + float32x4_t y0, y1; + + float x; + bfloat16_t z; + + bfloat16_t zero_bf16 = vcvth_bf16_f32(0.0f); + bfloat16x8_t zeros = vdupq_n_bf16(zero_bf16); + + bfloat16_t beta_bf16; + memcpy(&beta_bf16, &beta_in, sizeof(bfloat16_t)); + float beta = vcvtah_f32_bf16(beta_bf16); + float32x4_t beta_neon = vdupq_n_f32(beta); + + ptr_c = (bfloat16_t *)c; + + chunk = m >> 3; + remain = m & 7; + + if (beta == 0.0f){ + for (j = 0; j < n; j ++){ + ptr_c0 = ptr_c; + ptr_c += ldc; + + for (i = 0; i < chunk; i ++){ + vst1q_bf16(ptr_c0, zeros); + ptr_c0 += 8; + } + + for (i = 0; i < remain; i ++){ + ptr_c0[0] = zero_bf16; + ptr_c0 ++; + } + } + } else { + for (j = 0; j < n; j ++){ + ptr_c0 = ptr_c; + ptr_c += ldc; + + for (i = 0; i < chunk; i ++){ + x0 = vld1q_bf16(ptr_c0); + + y0 = vcvtq_low_f32_bf16(x0); + y1 = vcvtq_high_f32_bf16(x0); + + y0 = vmulq_f32(y0, beta_neon); + y1 = vmulq_f32(y1, beta_neon); + + z0 = vcvtq_low_bf16_f32(y0); + z0 = vcvtq_high_bf16_f32(z0, y1); + + vst1q_bf16(ptr_c0, z0); + + ptr_c0 += 8; + } + + for (i = 0; i < remain; i ++){ + x = vcvtah_f32_bf16(ptr_c0[0]); + z = vcvth_bf16_f32(x * beta); + + ptr_c0[0] = z; + ptr_c0 ++; + } + } + } + return 0; +}; diff --git a/kernel/arm64/bgemm_kernel_4x4_neoversev1.c b/kernel/arm64/bgemm_kernel_4x4_neoversev1.c new file mode 100644 index 000000000..7af31bb2c --- /dev/null +++ b/kernel/arm64/bgemm_kernel_4x4_neoversev1.c @@ -0,0 +1,50 @@ +/*************************************************************************** + * Copyright (c) 2025, The OpenBLAS Project + * All rights reserved. + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of the OpenBLAS project nor the names of + * its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * *****************************************************************************/ + +#include + +#include "common.h" + +#define ALPHA_ONE +#include "bgemm_kernel_4x4_neoversev1_impl.c" +#undef ALPHA_ONE +#undef UPDATE_C +#include "bgemm_kernel_4x4_neoversev1_impl.c" + +int CNAME(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT *A, IFLOAT *B, + FLOAT *C, BLASLONG ldc) { + bfloat16_t alpha_bf16; + memcpy(&alpha_bf16, &alpha, sizeof(bfloat16_t)); + float alpha_f32 = vcvtah_f32_bf16(alpha_bf16); + + if (alpha_f32 == 1.0f) + return bgemm_kernel_neoversev1_alpha_one(m, n, k, alpha, A, B, C, ldc); + else + return bgemm_kernel_neoversev1_alpha(m, n, k, alpha, A, B, C, ldc); + return 0; +} diff --git a/kernel/arm64/bgemm_kernel_4x4_neoversev1_impl.c b/kernel/arm64/bgemm_kernel_4x4_neoversev1_impl.c new file mode 100644 index 000000000..2477da9c0 --- /dev/null +++ b/kernel/arm64/bgemm_kernel_4x4_neoversev1_impl.c @@ -0,0 +1,429 @@ +/*************************************************************************** + * Copyright (c) 2025, The OpenBLAS Project + * All rights reserved. + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of the OpenBLAS project nor the names of + * its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * *****************************************************************************/ + +#include + +#include "common.h" + +#define INIT_C(M, N) mc##M##N = svdup_f32(0); + +#define MATMUL(M, N) mc##M##N = svbfmmla(mc##M##N, ma##M, mb##N); + +#define INIT_C_4x4 \ + do { \ + INIT_C(0, 0); \ + INIT_C(0, 1); \ + INIT_C(1, 0); \ + INIT_C(1, 1); \ + } while (0); + +#ifdef ALPHA_ONE +#define UPDATE_C(PG16, PG32, PTR, TMP32, TMP16, SRC32) \ + do { \ + TMP32 = svreinterpret_f32_u32(svld1uh_u32((PG16), (uint16_t*)PTR)); \ + TMP32 = svadd_z((PG32), SRC32, TMP32); \ + TMP16 = svcvt_bf16_f32_z((PG32), TMP32); \ + TMP16 = svuzp1_bf16(TMP16, TMP16); \ + svst1_bf16((PG16), (PTR), TMP16); \ + } while (0) +#else +#define UPDATE_C(PG16, PG32, PTR, TMP32, TMP16, SRC32) \ + do { \ + TMP32 = svreinterpret_f32_u32(svld1uh_u32((PG16), (uint16_t*)PTR)); \ + TMP32 = svmad_z((PG32), svalpha, SRC32, TMP32); \ + TMP16 = svcvt_bf16_f32_z((PG32), TMP32); \ + TMP16 = svuzp1_bf16(TMP16, TMP16); \ + svst1_bf16((PG16), (PTR), TMP16); \ + } while (0) +#endif + +#define ZIP_EVEN_ELEMENTS(PG, mc0, mc1, tmp, vc) \ + do { \ + (tmp) = svuzp1_f32((mc0), (mc1)); \ + (vc) = svcompact_f32((PG), (tmp)); \ + } while (0) + +#define ZIP_ODD_ELEMENTS(PG, mc0, mc1, tmp, vc) \ + do { \ + (tmp) = svuzp2_f32((mc0), (mc1)); \ + (vc) = svcompact_f32((PG), (tmp)); \ + } while (0) + +#define ACCUMULATE_LAST4_TO_FIRST4(M, N, TMP) \ + do { \ + TMP = svext_f32(mc##M##N, mc##M##N, 4); \ + mc##M##N = svadd_f32_z(svptrue_b32(), mc##M##N, (TMP)); \ + } while (0) + +#ifdef ALPHA_ONE +int bgemm_kernel_neoversev1_alpha_one(BLASLONG m, BLASLONG n, BLASLONG k, + FLOAT alpha, IFLOAT *A, IFLOAT *B, + FLOAT *C, BLASLONG ldc) +#else +int bgemm_kernel_neoversev1_alpha(BLASLONG m, BLASLONG n, BLASLONG k, + FLOAT alpha, IFLOAT *A, IFLOAT *B, FLOAT *C, + BLASLONG ldc) +#endif +{ + BLASLONG pad_k = (k + 7) & ~7; + svbfloat16_t ma0, ma1, mb0, mb1; + svfloat32_t mc00, mc01, mc10, mc11, vc0, vc1, vc2, vc3; + svfloat32_t tmp; +#ifndef ALPHA_ONE + bfloat16_t alpha_bf16; + memcpy(&alpha_bf16, &alpha, sizeof(bfloat16_t)); + svfloat32_t svalpha = svdup_f32(vcvtah_f32_bf16(alpha_bf16)); +#endif + + svbool_t pg16_all = svptrue_b16(); + + svbool_t pg32_first_1 = svwhilelt_b32(0, 1); + svbool_t pg32_first_2 = svwhilelt_b32(0, 2); + svbool_t pg32_first_4 = svwhilelt_b32(0, 4); + + svbool_t pg16_first_1 = svwhilelt_b16(0, 1); + svbool_t pg16_first_2 = svwhilelt_b16(0, 2); + svbool_t pg16_first_4 = svwhilelt_b16(0, 4); + + svbool_t pg32_select_first_2_per_quadword = svdupq_b32(1, 1, 0, 0); + + bfloat16_t *ptr_a = (bfloat16_t *)A; + bfloat16_t *ptr_b = (bfloat16_t *)B; + bfloat16_t *ptr_c = (bfloat16_t *)C; + + bfloat16_t *ptr_a0; + bfloat16_t *ptr_b0; + bfloat16_t *ptr_c0, *ptr_c1, *ptr_c2, *ptr_c3; + + svfloat32_t tmp32; + svbfloat16_t tmp16; + + for (BLASLONG j = 0; j < n / 4; j++) { + ptr_c0 = ptr_c; + ptr_c1 = ptr_c0 + ldc; + ptr_c2 = ptr_c1 + ldc; + ptr_c3 = ptr_c2 + ldc; + ptr_c += 4 * ldc; + ptr_a = (bfloat16_t *)A; + + for (BLASLONG i = 0; i < m / 4; i++) { + ptr_a0 = ptr_a; + ptr_a += 4 * pad_k; + + ptr_b0 = ptr_b; + + INIT_C_4x4; + + for (BLASLONG p = 0; p < pad_k; p += 8) { + ma0 = svld1_bf16(pg16_all, ptr_a0); + ma1 = svld1_bf16(pg16_all, ptr_a0 + 16); + + mb0 = svld1_bf16(pg16_all, ptr_b0); + mb1 = svld1_bf16(pg16_all, ptr_b0 + 16); + + MATMUL(0, 0); + MATMUL(0, 1); + MATMUL(1, 0); + MATMUL(1, 1); + + ptr_a0 += 32; + ptr_b0 += 32; + } + + ACCUMULATE_LAST4_TO_FIRST4(0, 0, tmp); + ACCUMULATE_LAST4_TO_FIRST4(0, 1, tmp); + ACCUMULATE_LAST4_TO_FIRST4(1, 0, tmp); + ACCUMULATE_LAST4_TO_FIRST4(1, 1, tmp); + + ZIP_EVEN_ELEMENTS(pg32_select_first_2_per_quadword, mc00, mc10, tmp, vc0); + ZIP_ODD_ELEMENTS(pg32_select_first_2_per_quadword, mc00, mc10, tmp, vc1); + + ZIP_EVEN_ELEMENTS(pg32_select_first_2_per_quadword, mc01, mc11, tmp, vc2); + ZIP_ODD_ELEMENTS(pg32_select_first_2_per_quadword, mc01, mc11, tmp, vc3); + + UPDATE_C(pg16_first_4, pg32_first_4, ptr_c0, tmp32, tmp16, vc0); + UPDATE_C(pg16_first_4, pg32_first_4, ptr_c1, tmp32, tmp16, vc1); + UPDATE_C(pg16_first_4, pg32_first_4, ptr_c2, tmp32, tmp16, vc2); + UPDATE_C(pg16_first_4, pg32_first_4, ptr_c3, tmp32, tmp16, vc3); + + ptr_c0 += 4; + ptr_c1 += 4; + ptr_c2 += 4; + ptr_c3 += 4; + } + + if (m & 2) { + ptr_a0 = ptr_a; + ptr_a += 2 * pad_k; + + ptr_b0 = ptr_b; + INIT_C(0, 0); + INIT_C(0, 1); + for (BLASLONG p = 0; p < pad_k; p += 8) { + ma0 = svld1_bf16(pg16_all, ptr_a0); + mb0 = svld1_bf16(pg16_all, ptr_b0); + mb1 = svld1_bf16(pg16_all, ptr_b0 + 16); + + MATMUL(0, 0); + MATMUL(0, 1); + + ptr_a0 += 16; + ptr_b0 += 32; + } + + ACCUMULATE_LAST4_TO_FIRST4(0, 0, tmp); + ACCUMULATE_LAST4_TO_FIRST4(0, 1, tmp); + + vc0 = svuzp1(mc00, mc00); + vc1 = svuzp2(mc00, mc00); + vc2 = svuzp1(mc01, mc01); + vc3 = svuzp2(mc01, mc01); + + UPDATE_C(pg16_first_2, pg32_first_2, ptr_c0, tmp32, tmp16, vc0); + UPDATE_C(pg16_first_2, pg32_first_2, ptr_c1, tmp32, tmp16, vc1); + UPDATE_C(pg16_first_2, pg32_first_2, ptr_c2, tmp32, tmp16, vc2); + UPDATE_C(pg16_first_2, pg32_first_2, ptr_c3, tmp32, tmp16, vc3); + + ptr_c0 += 2; + ptr_c1 += 2; + ptr_c2 += 2; + ptr_c3 += 2; + } + + if (m & 1) { + ptr_a0 = ptr_a; + ptr_b0 = ptr_b; + + INIT_C(0, 0); + INIT_C(0, 1); + for (BLASLONG p = 0; p < pad_k; p += 8) { + ma0 = svld1_bf16(pg16_all, ptr_a0); + mb0 = svld1_bf16(pg16_all, ptr_b0); + mb1 = svld1_bf16(pg16_all, ptr_b0 + 16); + + MATMUL(0, 0); + MATMUL(0, 1); + + ptr_a0 += 16; + ptr_b0 += 32; + } + + ACCUMULATE_LAST4_TO_FIRST4(0, 0, tmp); + ACCUMULATE_LAST4_TO_FIRST4(0, 1, tmp); + + // use compact is more straightforward + vc1 = svuzp2(mc00, mc00); + vc3 = svuzp2(mc01, mc01); + + UPDATE_C(pg16_first_1, pg32_first_1, ptr_c0, tmp32, tmp16, mc00); + UPDATE_C(pg16_first_1, pg32_first_1, ptr_c1, tmp32, tmp16, vc1); + UPDATE_C(pg16_first_1, pg32_first_1, ptr_c2, tmp32, tmp16, mc01); + UPDATE_C(pg16_first_1, pg32_first_1, ptr_c3, tmp32, tmp16, vc3); + } + + ptr_b += 4 * pad_k; + } + + if (n & 2) { + ptr_c0 = ptr_c; + ptr_c1 = ptr_c0 + ldc; + ptr_c += 2 * ldc; + ptr_a = (bfloat16_t *)A; + + for (BLASLONG i = 0; i < m / 4; i++) { + ptr_a0 = ptr_a; + ptr_a += 4 * pad_k; + + ptr_b0 = ptr_b; + + INIT_C(0, 0); + INIT_C(1, 0); + + for (BLASLONG p = 0; p < pad_k; p += 8) { + ma0 = svld1_bf16(pg16_all, ptr_a0); + ma1 = svld1_bf16(pg16_all, ptr_a0 + 16); + + mb0 = svld1_bf16(pg16_all, ptr_b0); + + MATMUL(0, 0); + MATMUL(1, 0); + + ptr_a0 += 32; + ptr_b0 += 16; + } + + ACCUMULATE_LAST4_TO_FIRST4(0, 0, tmp); + ACCUMULATE_LAST4_TO_FIRST4(1, 0, tmp); + + ZIP_EVEN_ELEMENTS(pg32_select_first_2_per_quadword, mc00, mc10, tmp, vc0); + ZIP_ODD_ELEMENTS(pg32_select_first_2_per_quadword, mc00, mc10, tmp, vc2); + + UPDATE_C(pg16_first_4, pg32_first_4, ptr_c0, tmp32, tmp16, vc0); + UPDATE_C(pg16_first_4, pg32_first_4, ptr_c1, tmp32, tmp16, vc2); + + ptr_c0 += 4; + ptr_c1 += 4; + } + + if (m & 2) { + ptr_a0 = ptr_a; + ptr_a += 2 * pad_k; + ptr_b0 = ptr_b; + + INIT_C(0, 0); + + for (BLASLONG p = 0; p < pad_k; p += 8) { + ma0 = svld1_bf16(pg16_all, ptr_a0); + mb0 = svld1_bf16(pg16_all, ptr_b0); + + MATMUL(0, 0); + + ptr_a0 += 16; + ptr_b0 += 16; + } + + ACCUMULATE_LAST4_TO_FIRST4(0, 0, tmp); + vc0 = svuzp1(mc00, mc00); + vc1 = svuzp2(mc00, mc00); + + + UPDATE_C(pg16_first_2, pg32_first_2, ptr_c0, tmp32, tmp16, vc0); + UPDATE_C(pg16_first_2, pg32_first_2, ptr_c1, tmp32, tmp16, vc1); + + ptr_c0 += 2; + ptr_c1 += 2; + } + + if (m & 1) { + ptr_a0 = ptr_a; + ptr_b0 = ptr_b; + INIT_C(0, 0); + for (BLASLONG p = 0; p < pad_k; p += 8) { + ma0 = svld1_bf16(pg16_all, ptr_a0); + mb0 = svld1_bf16(pg16_all, ptr_b0); + MATMUL(0, 0); + ptr_a0 += 16; + ptr_b0 += 16; + } + + ACCUMULATE_LAST4_TO_FIRST4(0, 0, tmp); + vc1 = svuzp2(mc00, mc00); + + UPDATE_C(pg16_first_1, pg32_first_1, ptr_c0, tmp32, tmp16, mc00); + UPDATE_C(pg16_first_1, pg32_first_1, ptr_c1, tmp32, tmp16, vc1); + } + + ptr_b += 2 * pad_k; + } + + if (n & 1) { // TODO: this case seems a overhead. find out whether it's in our + // case. + ptr_c0 = ptr_c; + ptr_a = (bfloat16_t *)A; + + for (BLASLONG i = 0; i < m / 4; i++) { + ptr_a0 = ptr_a; + ptr_a += 4 * pad_k; + + ptr_b0 = ptr_b; + + INIT_C(0, 0); + INIT_C(1, 0); + + for (BLASLONG p = 0; p < pad_k; p += 8) { + ma0 = svld1_bf16(pg16_all, ptr_a0); + ma1 = svld1_bf16(pg16_all, ptr_a0 + 16); + + mb0 = svld1_bf16(pg16_all, ptr_b0); + + MATMUL(0, 0); + MATMUL(1, 0); + + ptr_a0 += 32; + ptr_b0 += 16; + } + + ACCUMULATE_LAST4_TO_FIRST4(0, 0, tmp); + ACCUMULATE_LAST4_TO_FIRST4(1, 0, tmp); + + ZIP_EVEN_ELEMENTS(pg32_select_first_2_per_quadword, mc00, mc10, tmp, vc0); + + UPDATE_C(pg16_first_4, pg32_first_4, ptr_c0, tmp32, tmp16, vc0); + + ptr_c0 += 4; + } + + if (m & 2) { + ptr_a0 = ptr_a; + ptr_a += 2 * pad_k; + ptr_b0 = ptr_b; + + INIT_C(0, 0); + + for (BLASLONG p = 0; p < pad_k; p += 8) { + ma0 = svld1_bf16(pg16_all, ptr_a0); + mb0 = svld1_bf16(pg16_all, ptr_b0); + + MATMUL(0, 0); + + ptr_a0 += 16; + ptr_b0 += 16; + } + + ACCUMULATE_LAST4_TO_FIRST4(0, 0, tmp); + + vc0 = svuzp1(mc00, mc00); + + UPDATE_C(pg16_first_2, pg32_first_2, ptr_c0, tmp32, tmp16, vc0); + + ptr_c0 += 2; + } + + if (m & 1) { + ptr_a0 = ptr_a; + ptr_b0 = ptr_b; + + INIT_C(0, 0); + for (BLASLONG p = 0; p < pad_k; p += 8) { + + ma0 = svld1_bf16(pg16_all, ptr_a0); + mb0 = svld1_bf16(pg16_all, ptr_b0); + + MATMUL(0, 0); + ptr_a0 += 16; + ptr_b0 += 16; + } + + ACCUMULATE_LAST4_TO_FIRST4(0, 0, tmp); + + UPDATE_C(pg16_first_1, pg32_first_1, ptr_c0, tmp32, tmp16, mc00); + } + } + + return 0; +} diff --git a/kernel/generic/gemmkernel_2x2.c b/kernel/generic/gemmkernel_2x2.c index add84f043..8872f2f56 100644 --- a/kernel/generic/gemmkernel_2x2.c +++ b/kernel/generic/gemmkernel_2x2.c @@ -1,26 +1,50 @@ +/*************************************************************************** + * Copyright (c) 2025, The OpenBLAS Project + * All rights reserved. + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of the OpenBLAS project nor the names of + * its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * *****************************************************************************/ + #include "common.h" #if defined(BFLOAT16) && defined(BFLOAT16CONVERSION) static float -bfloat16tof32 (bfloat16 f16) +bfloat16tof32 (bfloat16 value) { - float result = 0; - unsigned short* q = (unsigned short*)(&result); -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ - q[0] = f16; -#else - q[1] = f16; -#endif + blasint one = 1; + float result; + sbf16tos_(&one, &value, &one, &result, &one); return result; } -static bfloat16 f32tobfloat16(float f32) { - unsigned short *q = (unsigned short *)(&f32); -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ - return q[0]; -#else - return q[1]; -#endif +#ifdef BGEMM +static bfloat16 f32tobfloat16(float value) { + blasint one = 1; + bfloat16 result; + sbstobf16_(&one, &value, &one, &result, &one); + return result; } +#endif #ifdef BGEMM #define ALPHA bfloat16tof32(alpha) diff --git a/kernel/setparam-ref.c b/kernel/setparam-ref.c index 9eb5959dc..886895acc 100644 --- a/kernel/setparam-ref.c +++ b/kernel/setparam-ref.c @@ -88,7 +88,11 @@ gotoblas_t TABLE_NAME = { ssymv_LTS, ssymv_UTS, bgemm_kernelTS, bgemm_betaTS, +#if BGEMM_DEFAULT_UNROLL_M != BGEMM_DEFAULT_UNROLL_N bgemm_incopyTS, bgemm_itcopyTS, +#else + bgemm_oncopyTS, bgemm_otcopyTS, +#endif bgemm_oncopyTS, bgemm_otcopyTS, sbgemm_kernelTS, sbgemm_betaTS, diff --git a/param.h b/param.h index 681ff598e..662d08449 100644 --- a/param.h +++ b/param.h @@ -3593,6 +3593,13 @@ is a big desktop or server with abundant cache rather than a phone or embedded d #define GEMM_PREFERED_SIZE 8 #endif +#undef BGEMM_ALIGN_K +#undef BGEMM_DEFAULT_UNROLL_M +#undef BGEMM_DEFAULT_UNROLL_N +#define BGEMM_ALIGN_K 8 +#define BGEMM_DEFAULT_UNROLL_N 4 +#define BGEMM_DEFAULT_UNROLL_M 4 + #undef SBGEMM_ALIGN_K #undef SBGEMM_DEFAULT_UNROLL_M #undef SBGEMM_DEFAULT_UNROLL_N diff --git a/test/compare_sgemm_bgemm.c b/test/compare_sgemm_bgemm.c index 8cddcba97..51d26adbc 100644 --- a/test/compare_sgemm_bgemm.c +++ b/test/compare_sgemm_bgemm.c @@ -33,92 +33,49 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define BGEMM BLASFUNC(bgemm) #define BGEMM_LARGEST 256 -typedef union +static float float16to32(bfloat16 value) { - unsigned short v; -#if defined(_AIX) - struct __attribute__((packed)) -#else - struct -#endif - { -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ - unsigned short s:1; - unsigned short e:8; - unsigned short m:7; -#else - unsigned short m:7; - unsigned short e:8; - unsigned short s:1; -#endif - } bits; -} bfloat16_bits; - -typedef union -{ - float v; -#if defined(_AIX) - struct __attribute__((packed)) -#else - struct -#endif - { -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ - uint32_t s:1; - uint32_t e:8; - uint32_t m:23; -#else - uint32_t m:23; - uint32_t e:8; - uint32_t s:1; -#endif - } bits; -} float32_bits; - -float -float16to32 (bfloat16_bits f16) -{ - float32_bits f32; - f32.bits.s = f16.bits.s; - f32.bits.e = f16.bits.e; - f32.bits.m = (uint32_t) f16.bits.m << 16; - return f32.v; -} - -bfloat16 -float32to16 (float32_bits f32) -{ - bfloat16_bits f16; - f16.bits.s = f32.bits.s; - f16.bits.e = f32.bits.e; - f16.bits.m = (f32.bits.m >> 16) & 0x7f; - return f16.v; + blasint one = 1; + float result; + sbf16tos_(&one, &value, &one, &result, &one); + return result; } static float truncate_float(float value) { - bfloat16_bits f16 = (bfloat16_bits)float32to16((float32_bits)value); - return float16to32(f16); + blasint one = 1; + bfloat16 tmp; + float result; + sbstobf16_(&one, &value, &one, &tmp, &one); + sbf16tos_(&one, &tmp, &one, &result, &one); + return result; } -void *malloc_safe(size_t size) { +static void *malloc_safe(size_t size) { if (size == 0) return malloc(1); else return malloc(size); } +static is_close(float a, float b, float rtol, float atol) { + return fabs(a - b) <= (atol + rtol*fabs(b)); +} + int main (int argc, char *argv[]) { blasint m, n, k; int i, j, l; blasint x, y; + blasint one = 1; int ret = 0; int loop = BGEMM_LARGEST; char transA = 'N', transB = 'N'; float alpha = 1.0, beta = 0.0; - bfloat16 alpha_bf16 = float32to16((float32_bits)alpha); - bfloat16 beta_bf16 = float32to16((float32_bits)beta); + bfloat16 alpha_bf16; + sbstobf16_(&one, &alpha, &one, &alpha_bf16, &one); + bfloat16 beta_bf16; + sbstobf16_(&one, &beta, &one, &beta_bf16, &one); for (x = 0; x <= loop; x++) { @@ -127,23 +84,20 @@ main (int argc, char *argv[]) float *A = (float *)malloc_safe(m * k * sizeof(FLOAT)); float *B = (float *)malloc_safe(k * n * sizeof(FLOAT)); float *C = (float *)malloc_safe(m * n * sizeof(FLOAT)); - bfloat16_bits *AA = (bfloat16_bits *)malloc_safe(m * k * sizeof(bfloat16_bits)); - bfloat16_bits *BB = (bfloat16_bits *)malloc_safe(k * n * sizeof(bfloat16_bits)); - bfloat16_bits *CC = (bfloat16_bits *)malloc_safe(k * n * sizeof(bfloat16_bits)); + bfloat16 *AA = (bfloat16 *)malloc_safe(m * k * sizeof(bfloat16)); + bfloat16 *BB = (bfloat16 *)malloc_safe(k * n * sizeof(bfloat16)); + bfloat16 *CC = (bfloat16 *)malloc_safe(k * n * sizeof(bfloat16)); FLOAT *DD = (FLOAT *)malloc_safe(m * n * sizeof(FLOAT)); if ((A == NULL) || (B == NULL) || (C == NULL) || (AA == NULL) || (BB == NULL) || (DD == NULL) || (CC == NULL)) return 1; - bfloat16 atmp,btmp; - blasint one=1; for (j = 0; j < m; j++) { for (i = 0; i < k; i++) { A[j * k + i] = ((FLOAT) rand () / (FLOAT) RAND_MAX) + 0.5; - sbstobf16_(&one, &A[j*k+i], &one, &atmp, &one); - AA[j * k + i].v = atmp; + sbstobf16_(&one, &A[j*k+i], &one, &AA[j * k + i], &one); } } for (j = 0; j < n; j++) @@ -151,8 +105,7 @@ main (int argc, char *argv[]) for (i = 0; i < k; i++) { B[j * k + i] = ((FLOAT) rand () / (FLOAT) RAND_MAX) + 0.5; - sbstobf16_(&one, &B[j*k+i], &one, &btmp, &one); - BB[j * k + i].v = btmp; + sbstobf16_(&one, &B[j*k+i], &one, &BB[j * k + i], &one); } } for (y = 0; y < 4; y++) @@ -168,7 +121,7 @@ main (int argc, char *argv[]) transB = 'T'; } - memset(CC, 0, m * n * sizeof(bfloat16_bits)); + memset(CC, 0, m * n * sizeof(bfloat16)); memset(DD, 0, m * n * sizeof(FLOAT)); memset(C, 0, m * n * sizeof(FLOAT)); @@ -198,10 +151,15 @@ main (int argc, char *argv[]) DD[i * m + j] += float16to32 (AA[k * j + l]) * float16to32 (BB[i + l * n]); } - if (fabs(float16to32(CC[i * m + j]) - truncate_float(C[i * m + j])) > 2.0) { + if (!is_close(float16to32(CC[i * m + j]), truncate_float(C[i * m + j]), 0.01, 0.001)) { + printf("Mismatch at i=%d, j=%d, k=%d: CC=%.6f, C=%.6f\n", + i, j, k, float16to32(CC[i * m + j]), truncate_float(C[i * m + j])); ret++; } - if (fabs(float16to32(CC[i * m + j]) - truncate_float(DD[i * m + j])) > 1.0) { + + if (!is_close(float16to32(CC[i * m + j]), truncate_float(DD[i * m + j]), 0.0001, 0.00001)) { + printf("Mismatch at i=%d, j=%d, k=%d: CC=%.6f, DD=%.6f\n", + i, j, k, float16to32(CC[i * m + j]), truncate_float(DD[i * m + j])); ret++; } From aad97c7763e35c83b1cf3d3c4fc1610e6b061388 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Fri, 11 Jul 2025 15:32:41 +0200 Subject: [PATCH 26/36] Fix return type declaration --- test/compare_sgemm_bgemm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/compare_sgemm_bgemm.c b/test/compare_sgemm_bgemm.c index 51d26adbc..69210b98e 100644 --- a/test/compare_sgemm_bgemm.c +++ b/test/compare_sgemm_bgemm.c @@ -57,7 +57,7 @@ static void *malloc_safe(size_t size) { return malloc(size); } -static is_close(float a, float b, float rtol, float atol) { +static float is_close(float a, float b, float rtol, float atol) { return fabs(a - b) <= (atol + rtol*fabs(b)); } From c8d41e4a32fdc55751e18242ba8c45cd406b73e3 Mon Sep 17 00:00:00 2001 From: newyork_loki <137760120+lowkeyrossi@users.noreply.github.com> Date: Sat, 12 Jul 2025 14:34:29 +0530 Subject: [PATCH 27/36] Add CI support for OpenBLAS on WoA --- .github/workflows/windows_arm64.yml | 82 +++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 .github/workflows/windows_arm64.yml diff --git a/.github/workflows/windows_arm64.yml b/.github/workflows/windows_arm64.yml new file mode 100644 index 000000000..bd23f955b --- /dev/null +++ b/.github/workflows/windows_arm64.yml @@ -0,0 +1,82 @@ +name: Windows ARM64 CI + +on: + push: + branches: + - develop + - release-** + pull_request: + branches: + - develop + - release-** + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read # to fetch code (actions/checkout) + +jobs: + build: + # if: "github.repository == 'OpenMathLib/OpenBLAS'" + runs-on: windows-11-arm + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Install LLVM for Win-ARM64 + shell: pwsh + run: | + Invoke-WebRequest https://github.com/llvm/llvm-project/releases/download/llvmorg-20.1.6/LLVM-20.1.6-woa64.exe -UseBasicParsing -OutFile LLVM-woa64.exe + Start-Process -FilePath ".\LLVM-woa64.exe" -ArgumentList "/S" -Wait + echo "C:\Program Files\LLVM\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + + - name: Install CMake and Ninja for Win-ARM64 + shell: pwsh + run: | + Invoke-WebRequest https://github.com/Kitware/CMake/releases/download/v3.29.4/cmake-3.29.4-windows-arm64.msi -OutFile cmake-arm64.msi + Start-Process msiexec.exe -ArgumentList "/i cmake-arm64.msi /quiet /norestart" -Wait + echo "C:\Program Files\CMake\bin" >> $env:GITHUB_PATH + + Invoke-WebRequest https://github.com/ninja-build/ninja/releases/download/v1.13.1/ninja-winarm64.zip -OutFile ninja-winarm64.zip + Expand-Archive ninja-winarm64.zip -DestinationPath ninja + Copy-Item ninja\ninja.exe -Destination "C:\Windows\System32" + + - name: Configure OpenBLAS + shell: cmd + run: | + CALL "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsarm64.bat" + mkdir build + cd build + cmake .. -G Ninja ^ + -DCMAKE_BUILD_TYPE=Release ^ + -DTARGET=ARMV8 ^ + -DBINARY=64 ^ + -DCMAKE_C_COMPILER=clang-cl ^ + -DCMAKE_Fortran_COMPILER=flang-new ^ + -DBUILD_SHARED_LIBS=ON ^ + -DCMAKE_SYSTEM_PROCESSOR=arm64 ^ + -DCMAKE_SYSTEM_NAME=Windows ^ + -DCMAKE_INSTALL_PREFIX=C:/opt + + - name: Build OpenBLAS + shell: cmd + run: | + cd build + ninja -j16 + + - name: Install OpenBLAS + shell: cmd + run: | + cd build + cmake --install . + + - name: Run ctests + shell: pwsh + run: | + $env:PATH = "C:\opt\bin;$env:PATH" + cd build + ctest + + From cb2c726716be1931426fbbf7c8ceaa33adcc37a1 Mon Sep 17 00:00:00 2001 From: newyork_loki <137760120+lowkeyrossi@users.noreply.github.com> Date: Sat, 12 Jul 2025 14:37:30 +0530 Subject: [PATCH 28/36] Add CI support for OpenBLAS on WoA --- .github/workflows/windows_arm64.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/windows_arm64.yml b/.github/workflows/windows_arm64.yml index bd23f955b..d1b5f94e0 100644 --- a/.github/workflows/windows_arm64.yml +++ b/.github/workflows/windows_arm64.yml @@ -19,7 +19,7 @@ permissions: jobs: build: - # if: "github.repository == 'OpenMathLib/OpenBLAS'" + if: "github.repository == 'OpenMathLib/OpenBLAS'" runs-on: windows-11-arm steps: - name: Checkout repository From 3f110c827294b54ba7438b1763584c7c01b1ee64 Mon Sep 17 00:00:00 2001 From: Chris Sidebottom Date: Sun, 13 Jul 2025 12:48:09 +0000 Subject: [PATCH 29/36] Improve bgemm and sbgemm testing - Fixes wrong return type for `is_close` - Adds stricter compiler flags for test files so we don't see the above issue again - Re-uses test helper functions between compare_sgemm_sbgemm/bgemm.c --- test/Makefile | 7 +-- test/compare_sgemm_bgemm.c | 30 +++--------- test/compare_sgemm_sbgemm.c | 97 ++++++++----------------------------- test/test_helpers.h | 55 +++++++++++++++++++++ 4 files changed, 84 insertions(+), 105 deletions(-) create mode 100644 test/test_helpers.h diff --git a/test/Makefile b/test/Makefile index 7ac87f7f6..cd8006c04 100644 --- a/test/Makefile +++ b/test/Makefile @@ -13,7 +13,7 @@ # 3. Neither the name of the OpenBLAS project nor the names of # its contributors may be used to endorse or promote products # derived from this software without specific prior written permission. -# +# # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE @@ -34,6 +34,7 @@ ifneq (, $(filter $(CORE),LOONGSON3R3 LOONGSON3R4)) endif override FFLAGS += -fno-tree-vectorize endif +override CFLAGS += -std=c11 -Wall -Werror SUPPORT_GEMM3M = 0 @@ -402,10 +403,10 @@ zblat3 : zblat3.$(SUFFIX) ../$(LIBNAME) endif ifeq ($(BUILD_BFLOAT16),1) -test_bgemm : compare_sgemm_bgemm.c ../$(LIBNAME) +test_bgemm : compare_sgemm_bgemm.c test_helpers.h ../$(LIBNAME) $(CC) $(CLDFLAGS) -o test_bgemm compare_sgemm_bgemm.c ../$(LIBNAME) $(EXTRALIB) $(CEXTRALIB) -test_sbgemm : compare_sgemm_sbgemm.c ../$(LIBNAME) +test_sbgemm : compare_sgemm_sbgemm.c test_helpers.h ../$(LIBNAME) $(CC) $(CLDFLAGS) -o test_sbgemm compare_sgemm_sbgemm.c ../$(LIBNAME) $(EXTRALIB) $(CEXTRALIB) endif diff --git a/test/compare_sgemm_bgemm.c b/test/compare_sgemm_bgemm.c index 69210b98e..8ece63841 100644 --- a/test/compare_sgemm_bgemm.c +++ b/test/compare_sgemm_bgemm.c @@ -28,20 +28,13 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include +#include "test_helpers.h" #define SGEMM BLASFUNC(sgemm) #define BGEMM BLASFUNC(bgemm) #define BGEMM_LARGEST 256 -static float float16to32(bfloat16 value) -{ - blasint one = 1; - float result; - sbf16tos_(&one, &value, &one, &result, &one); - return result; -} - -static float truncate_float(float value) { +static float truncate_float32_to_bfloat16(float value) { blasint one = 1; bfloat16 tmp; float result; @@ -50,17 +43,6 @@ static float truncate_float(float value) { return result; } -static void *malloc_safe(size_t size) { - if (size == 0) - return malloc(1); - else - return malloc(size); -} - -static float is_close(float a, float b, float rtol, float atol) { - return fabs(a - b) <= (atol + rtol*fabs(b)); -} - int main (int argc, char *argv[]) { @@ -151,15 +133,15 @@ main (int argc, char *argv[]) DD[i * m + j] += float16to32 (AA[k * j + l]) * float16to32 (BB[i + l * n]); } - if (!is_close(float16to32(CC[i * m + j]), truncate_float(C[i * m + j]), 0.01, 0.001)) { + if (!is_close(float16to32(CC[i * m + j]), truncate_float32_to_bfloat16(C[i * m + j]), 0.01, 0.001)) { printf("Mismatch at i=%d, j=%d, k=%d: CC=%.6f, C=%.6f\n", - i, j, k, float16to32(CC[i * m + j]), truncate_float(C[i * m + j])); + i, j, k, float16to32(CC[i * m + j]), truncate_float32_to_bfloat16(C[i * m + j])); ret++; } - if (!is_close(float16to32(CC[i * m + j]), truncate_float(DD[i * m + j]), 0.0001, 0.00001)) { + if (!is_close(float16to32(CC[i * m + j]), truncate_float32_to_bfloat16(DD[i * m + j]), 0.0001, 0.00001)) { printf("Mismatch at i=%d, j=%d, k=%d: CC=%.6f, DD=%.6f\n", - i, j, k, float16to32(CC[i * m + j]), truncate_float(DD[i * m + j])); + i, j, k, float16to32(CC[i * m + j]), truncate_float32_to_bfloat16(DD[i * m + j])); ret++; } diff --git a/test/compare_sgemm_sbgemm.c b/test/compare_sgemm_sbgemm.c index ae109c1a5..4fa24b9ce 100644 --- a/test/compare_sgemm_sbgemm.c +++ b/test/compare_sgemm_sbgemm.c @@ -27,72 +27,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include "../common.h" + +#include "test_helpers.h" + #define SGEMM BLASFUNC(sgemm) #define SBGEMM BLASFUNC(sbgemm) #define SGEMV BLASFUNC(sgemv) #define SBGEMV BLASFUNC(sbgemv) -typedef union -{ - unsigned short v; -#if defined(_AIX) - struct __attribute__((packed)) -#else - struct -#endif - { -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ - unsigned short s:1; - unsigned short e:8; - unsigned short m:7; -#else - unsigned short m:7; - unsigned short e:8; - unsigned short s:1; -#endif - } bits; -} bfloat16_bits; - -typedef union -{ - float v; -#if defined(_AIX) - struct __attribute__((packed)) -#else - struct -#endif - { -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ - uint32_t s:1; - uint32_t e:8; - uint32_t m:23; -#else - uint32_t m:23; - uint32_t e:8; - uint32_t s:1; -#endif - } bits; -} float32_bits; - -float -float16to32 (bfloat16_bits f16) -{ - float32_bits f32; - f32.bits.s = f16.bits.s; - f32.bits.e = f16.bits.e; - f32.bits.m = (uint32_t) f16.bits.m << 16; - return f32.v; -} - #define SBGEMM_LARGEST 256 -void *malloc_safe(size_t size) -{ - if (size == 0) - return malloc(1); - else - return malloc(size); -} - int main (int argc, char *argv[]) { @@ -111,14 +54,13 @@ main (int argc, char *argv[]) float *A = (float *)malloc_safe(m * k * sizeof(FLOAT)); float *B = (float *)malloc_safe(k * n * sizeof(FLOAT)); float *C = (float *)malloc_safe(m * n * sizeof(FLOAT)); - bfloat16_bits *AA = (bfloat16_bits *)malloc_safe(m * k * sizeof(bfloat16_bits)); - bfloat16_bits *BB = (bfloat16_bits *)malloc_safe(k * n * sizeof(bfloat16_bits)); + bfloat16 *AA = (bfloat16 *)malloc_safe(m * k * sizeof(bfloat16)); + bfloat16 *BB = (bfloat16 *)malloc_safe(k * n * sizeof(bfloat16)); float *DD = (float *)malloc_safe(m * n * sizeof(FLOAT)); float *CC = (float *)malloc_safe(m * n * sizeof(FLOAT)); if ((A == NULL) || (B == NULL) || (C == NULL) || (AA == NULL) || (BB == NULL) || (DD == NULL) || (CC == NULL)) return 1; - bfloat16 atmp,btmp; blasint one=1; for (j = 0; j < m; j++) @@ -126,8 +68,7 @@ main (int argc, char *argv[]) for (i = 0; i < k; i++) { A[j * k + i] = ((FLOAT) rand () / (FLOAT) RAND_MAX) + 0.5; - sbstobf16_(&one, &A[j*k+i], &one, &atmp, &one); - AA[j * k + i].v = atmp; + sbstobf16_(&one, &A[j*k+i], &one, &AA[j * k + i], &one); } } for (j = 0; j < n; j++) @@ -135,8 +76,7 @@ main (int argc, char *argv[]) for (i = 0; i < k; i++) { B[j * k + i] = ((FLOAT) rand () / (FLOAT) RAND_MAX) + 0.5; - sbstobf16_(&one, &B[j*k+i], &one, &btmp, &one); - BB[j * k + i].v = btmp; + sbstobf16_(&one, &B[j*k+i], &one, &BB[j * k + i], &one); } } for (y = 0; y < 4; y++) @@ -182,10 +122,12 @@ main (int argc, char *argv[]) DD[i * m + j] += float16to32 (AA[k * j + l]) * float16to32 (BB[i + l * n]); } - if (fabs (CC[i * m + j] - C[i * m + j]) > 1.0) + if (!is_close(CC[i * m + j], C[i * m + j], 0.01, 0.001)) { ret++; - if (fabs (CC[i * m + j] - DD[i * m + j]) > 1.0) + } + if (!is_close(CC[i * m + j], DD[i * m + j], 0.001, 0.0001)) { ret++; + } } } free(A); @@ -211,14 +153,13 @@ main (int argc, char *argv[]) float *A = (float *)malloc_safe(x * x * sizeof(FLOAT)); float *B = (float *)malloc_safe(x * sizeof(FLOAT) << l); float *C = (float *)malloc_safe(x * sizeof(FLOAT) << l); - bfloat16_bits *AA = (bfloat16_bits *)malloc_safe(x * x * sizeof(bfloat16_bits)); - bfloat16_bits *BB = (bfloat16_bits *)malloc_safe(x * sizeof(bfloat16_bits) << l); + bfloat16 *AA = (bfloat16 *)malloc_safe(x * x * sizeof(bfloat16)); + bfloat16 *BB = (bfloat16 *)malloc_safe(x * sizeof(bfloat16) << l); float *DD = (float *)malloc_safe(x * sizeof(FLOAT)); float *CC = (float *)malloc_safe(x * sizeof(FLOAT) << l); if ((A == NULL) || (B == NULL) || (C == NULL) || (AA == NULL) || (BB == NULL) || (DD == NULL) || (CC == NULL)) return 1; - bfloat16 atmp, btmp; blasint one = 1; for (j = 0; j < x; j++) @@ -226,12 +167,10 @@ main (int argc, char *argv[]) for (i = 0; i < x; i++) { A[j * x + i] = ((FLOAT) rand () / (FLOAT) RAND_MAX) + 0.5; - sbstobf16_(&one, &A[j*x+i], &one, &atmp, &one); - AA[j * x + i].v = atmp; + sbstobf16_(&one, &A[j*x+i], &one, &AA[j * x + i], &one); } B[j << l] = ((FLOAT) rand () / (FLOAT) RAND_MAX) + 0.5; - sbstobf16_(&one, &B[j << l], &one, &btmp, &one); - BB[j << l].v = btmp; + sbstobf16_(&one, &B[j << l], &one, &BB[j << l], &one); CC[j << l] = C[j << l] = ((FLOAT) rand () / (FLOAT) RAND_MAX) + 0.5; } @@ -262,10 +201,12 @@ main (int argc, char *argv[]) } for (j = 0; j < x; j++) { - if (fabs (CC[j << l] - C[j << l]) > 1.0) + if (!is_close(CC[j << l], C[j << l], 0.01, 0.001)) { ret++; - if (fabs (CC[j << l] - DD[j]) > 1.0) + } + if (!is_close(CC[j << l], DD[j], 0.001, 0.0001)) { ret++; + } } } free(A); diff --git a/test/test_helpers.h b/test/test_helpers.h new file mode 100644 index 000000000..2bb3f7acd --- /dev/null +++ b/test/test_helpers.h @@ -0,0 +1,55 @@ +/*************************************************************************** +Copyright (c) 2025 The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#ifndef TEST_HELPERS_H +#define TEST_HELPERS_H +#include + +#include "../common.h" + +#if IFLOAT == bfloat16 +static float float16to32(bfloat16 value) +{ + blasint one = 1; + float result; + sbf16tos_(&one, &value, &one, &result, &one); + return result; +} +#endif + +static void *malloc_safe(size_t size) { + if (size == 0) + return malloc(1); + else + return malloc(size); +} + +static bool is_close(float a, float b, float rtol, float atol) { + return fabs(a - b) <= (atol + rtol*fabs(b)); +} + +#endif From 09a016fdf69828a26f18757bd99f60011a27ee5b Mon Sep 17 00:00:00 2001 From: Chris Sidebottom Date: Sun, 13 Jul 2025 13:01:27 +0000 Subject: [PATCH 30/36] Split sbgemv test from sbgemm test --- .gitignore | 2 + test/Makefile | 24 ++++++- test/compare_sgemm_bgemm.c | 3 +- test/compare_sgemm_sbgemm.c | 80 ---------------------- test/compare_sgemv_sbgemv.c | 128 ++++++++++++++++++++++++++++++++++++ 5 files changed, 153 insertions(+), 84 deletions(-) create mode 100644 test/compare_sgemv_sbgemv.c diff --git a/.gitignore b/.gitignore index c0885d466..1807d4496 100644 --- a/.gitignore +++ b/.gitignore @@ -81,6 +81,7 @@ test/ZBLAT2.SUMM test/ZBLAT3.SUMM test/ZBLAT3_3M.SUMM test/SHBLAT3.SUMM +test/SBBLAT2.SUMM test/SBBLAT3.SUMM test/BBLAT3.SUMM test/cblat1 @@ -97,6 +98,7 @@ test/sblat3 test/sblat3_3m test/test_shgemm test/test_sbgemm +test/test_sbgemv test/test_bgemm test/zblat1 test/zblat2 diff --git a/test/Makefile b/test/Makefile index cd8006c04..144738eb2 100644 --- a/test/Makefile +++ b/test/Makefile @@ -119,6 +119,9 @@ endif endif endif +ifeq ($(BUILD_BFLOAT16), 1) +B2 = test_sbgemv +endif ifeq ($(BUILD_SINGLE),1) S2=sblat2 endif @@ -132,11 +135,15 @@ ifeq ($(BUILD_COMPLEX16),1) Z2=zblat2 endif -level2: $(S2) $(D2) $(C2) $(Z2) +level2: $(B2) $(S2) $(D2) $(C2) $(Z2) ifneq ($(CROSS), 1) rm -f ?BLAT2.SUMM +ifeq ($(BUILD_BFLOAT16),1) + OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 ./test_sbgemv > SBBLAT2.SUMM + @$(GREP) -q FATAL SBBLAT2.SUMM && cat SBBLAT2.SUMM || exit 0 +endif ifeq ($(BUILD_SINGLE),1) OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 ./sblat2 < ./sblat2.dat @$(GREP) -q FATAL SBLAT2.SUMM && cat SBLAT2.SUMM || exit 0 @@ -156,6 +163,10 @@ endif ifdef SMP rm -f ?BLAT2.SUMM ifeq ($(USE_OPENMP), 1) +ifeq ($(BUILD_BFLOAT16),1) + OMP_NUM_THREADS=2 ./test_sbgemv > SBBLAT2.SUMM + @$(GREP) -q FATAL SBBLAT2.SUMM && cat SBBLAT2.SUMM || exit 0 +endif ifeq ($(BUILD_SINGLE),1) OMP_NUM_THREADS=2 ./sblat2 < ./sblat2.dat @$(GREP) -q FATAL SBLAT2.SUMM && cat SBLAT2.SUMM || exit 0 @@ -173,6 +184,10 @@ ifeq ($(BUILD_COMPLEX16),1) @$(GREP) -q FATAL ZBLAT2.SUMM && cat ZBLAT2.SUMM || exit 0 endif else +ifeq ($(BUILD_BFLOAT16),1) + OMP_NUM_THREADS=2 ./test_sbgemv > SBBLAT2.SUMM + @$(GREP) -q FATAL SBBLAT2.SUMM && cat SBBLAT2.SUMM || exit 0 +endif ifeq ($(BUILD_SINGLE),1) OPENBLAS_NUM_THREADS=2 ./sblat2 < ./sblat2.dat @$(GREP) -q FATAL SBLAT2.SUMM && cat SBLAT2.SUMM || exit 0 @@ -195,7 +210,7 @@ endif ifeq ($(BUILD_BFLOAT16),1) BF3= test_bgemm -B3= test_sbgemm +B3 = test_sbgemm endif ifeq ($(BUILD_SINGLE),1) S3=sblat3 @@ -408,6 +423,9 @@ test_bgemm : compare_sgemm_bgemm.c test_helpers.h ../$(LIBNAME) test_sbgemm : compare_sgemm_sbgemm.c test_helpers.h ../$(LIBNAME) $(CC) $(CLDFLAGS) -o test_sbgemm compare_sgemm_sbgemm.c ../$(LIBNAME) $(EXTRALIB) $(CEXTRALIB) + +test_sbgemv : compare_sgemv_sbgemv.c ../$(LIBNAME) + $(CC) $(CLDFLAGS) -o test_sbgemv compare_sgemv_sbgemv.c ../$(LIBNAME) $(EXTRALIB) $(CEXTRALIB) endif ifeq ($(BUILD_COMPLEX),1) @@ -426,7 +444,7 @@ clean: @rm -f *.$(SUFFIX) *.$(PSUFFIX) gmon.$(SUFFIX)ut *.SUMM *.cxml *.exe *.pdb *.dwf \ sblat1 dblat1 cblat1 zblat1 \ sblat2 dblat2 cblat2 zblat2 \ - test_bgemm test_sbgemm sblat3 dblat3 cblat3 zblat3 \ + test_bgemm test_sbgemm test_sbgemv sblat3 dblat3 cblat3 zblat3 \ sblat1p dblat1p cblat1p zblat1p \ sblat2p dblat2p cblat2p zblat2p \ sblat3p dblat3p cblat3p zblat3p \ diff --git a/test/compare_sgemm_bgemm.c b/test/compare_sgemm_bgemm.c index 8ece63841..bc8a0b468 100644 --- a/test/compare_sgemm_bgemm.c +++ b/test/compare_sgemm_bgemm.c @@ -158,6 +158,7 @@ main (int argc, char *argv[]) if (ret != 0) { fprintf (stderr, "FATAL ERROR BGEMM - Return code: %d\n", ret); - return ret; } + + return ret; } diff --git a/test/compare_sgemm_sbgemm.c b/test/compare_sgemm_sbgemm.c index 4fa24b9ce..489222516 100644 --- a/test/compare_sgemm_sbgemm.c +++ b/test/compare_sgemm_sbgemm.c @@ -141,87 +141,7 @@ main (int argc, char *argv[]) if (ret != 0) { fprintf (stderr, "FATAL ERROR SBGEMM - Return code: %d\n", ret); - return ret; } - for (beta = 0; beta < 3; beta += 1) { - for (alpha = 0; alpha < 3; alpha += 1) { - for (l = 0; l < 2; l++) { // l = 1 to test inc_x & inc_y not equal to one. - for (x = 1; x <= loop; x++) - { - k = (x == 0) ? 0 : l + 1; - float *A = (float *)malloc_safe(x * x * sizeof(FLOAT)); - float *B = (float *)malloc_safe(x * sizeof(FLOAT) << l); - float *C = (float *)malloc_safe(x * sizeof(FLOAT) << l); - bfloat16 *AA = (bfloat16 *)malloc_safe(x * x * sizeof(bfloat16)); - bfloat16 *BB = (bfloat16 *)malloc_safe(x * sizeof(bfloat16) << l); - float *DD = (float *)malloc_safe(x * sizeof(FLOAT)); - float *CC = (float *)malloc_safe(x * sizeof(FLOAT) << l); - if ((A == NULL) || (B == NULL) || (C == NULL) || (AA == NULL) || (BB == NULL) || - (DD == NULL) || (CC == NULL)) - return 1; - blasint one = 1; - - for (j = 0; j < x; j++) - { - for (i = 0; i < x; i++) - { - A[j * x + i] = ((FLOAT) rand () / (FLOAT) RAND_MAX) + 0.5; - sbstobf16_(&one, &A[j*x+i], &one, &AA[j * x + i], &one); - } - B[j << l] = ((FLOAT) rand () / (FLOAT) RAND_MAX) + 0.5; - sbstobf16_(&one, &B[j << l], &one, &BB[j << l], &one); - - CC[j << l] = C[j << l] = ((FLOAT) rand () / (FLOAT) RAND_MAX) + 0.5; - } - - for (y = 0; y < 2; y++) - { - if (y == 0) { - transA = 'N'; - } else { - transA = 'T'; - } - - memset(CC, 0, x * sizeof(FLOAT) << l); - memset(DD, 0, x * sizeof(FLOAT)); - memset(C, 0, x * sizeof(FLOAT) << l); - - SGEMV (&transA, &x, &x, &alpha, A, &x, B, &k, &beta, C, &k); - SBGEMV (&transA, &x, &x, &alpha, (bfloat16*) AA, &x, (bfloat16*) BB, &k, &beta, CC, &k); - - for (int i = 0; i < x; i ++) DD[i] *= beta; - - for (j = 0; j < x; j++) - for (i = 0; i < x; i++) - if (transA == 'N') { - DD[i] += alpha * float16to32 (AA[j * x + i]) * float16to32 (BB[j << l]); - } else if (transA == 'T') { - DD[j] += alpha * float16to32 (AA[j * x + i]) * float16to32 (BB[i << l]); - } - - for (j = 0; j < x; j++) { - if (!is_close(CC[j << l], C[j << l], 0.01, 0.001)) { - ret++; - } - if (!is_close(CC[j << l], DD[j], 0.001, 0.0001)) { - ret++; - } - } - } - free(A); - free(B); - free(C); - free(AA); - free(BB); - free(DD); - free(CC); - } // x - } // l - } // alpha - } // beta - - if (ret != 0) - fprintf (stderr, "FATAL ERROR SBGEMV - Return code: %d\n", ret); return ret; } diff --git a/test/compare_sgemv_sbgemv.c b/test/compare_sgemv_sbgemv.c new file mode 100644 index 000000000..5fa2d5f66 --- /dev/null +++ b/test/compare_sgemv_sbgemv.c @@ -0,0 +1,128 @@ +/*************************************************************************** +Copyright (c) 2020,2025 The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ +#include +#include +#include "../common.h" + +#include "test_helpers.h" + +#define SGEMV BLASFUNC(sgemv) +#define SBGEMV BLASFUNC(sbgemv) +#define SBGEMV_LARGEST 256 + +int +main (int argc, char *argv[]) +{ + blasint k; + int i, j, l; + blasint x, y; + int ret = 0; + int loop = SBGEMV_LARGEST; + char transA = 'N'; + float alpha = 1.0, beta = 0.0; + + for (beta = 0; beta < 3; beta += 1) { + for (alpha = 0; alpha < 3; alpha += 1) { + for (l = 0; l < 2; l++) { // l = 1 to test inc_x & inc_y not equal to one. + for (x = 1; x <= loop; x++) + { + k = (x == 0) ? 0 : l + 1; + float *A = (float *)malloc_safe(x * x * sizeof(FLOAT)); + float *B = (float *)malloc_safe(x * sizeof(FLOAT) << l); + float *C = (float *)malloc_safe(x * sizeof(FLOAT) << l); + bfloat16 *AA = (bfloat16 *)malloc_safe(x * x * sizeof(bfloat16)); + bfloat16 *BB = (bfloat16 *)malloc_safe(x * sizeof(bfloat16) << l); + float *DD = (float *)malloc_safe(x * sizeof(FLOAT)); + float *CC = (float *)malloc_safe(x * sizeof(FLOAT) << l); + if ((A == NULL) || (B == NULL) || (C == NULL) || (AA == NULL) || (BB == NULL) || + (DD == NULL) || (CC == NULL)) + return 1; + blasint one = 1; + + for (j = 0; j < x; j++) + { + for (i = 0; i < x; i++) + { + A[j * x + i] = ((FLOAT) rand () / (FLOAT) RAND_MAX) + 0.5; + sbstobf16_(&one, &A[j*x+i], &one, &AA[j * x + i], &one); + } + B[j << l] = ((FLOAT) rand () / (FLOAT) RAND_MAX) + 0.5; + sbstobf16_(&one, &B[j << l], &one, &BB[j << l], &one); + + CC[j << l] = C[j << l] = ((FLOAT) rand () / (FLOAT) RAND_MAX) + 0.5; + } + + for (y = 0; y < 2; y++) + { + if (y == 0) { + transA = 'N'; + } else { + transA = 'T'; + } + + memset(CC, 0, x * sizeof(FLOAT) << l); + memset(DD, 0, x * sizeof(FLOAT)); + memset(C, 0, x * sizeof(FLOAT) << l); + + SGEMV (&transA, &x, &x, &alpha, A, &x, B, &k, &beta, C, &k); + SBGEMV (&transA, &x, &x, &alpha, (bfloat16*) AA, &x, (bfloat16*) BB, &k, &beta, CC, &k); + + for (int i = 0; i < x; i ++) DD[i] *= beta; + + for (j = 0; j < x; j++) + for (i = 0; i < x; i++) + if (transA == 'N') { + DD[i] += alpha * float16to32 (AA[j * x + i]) * float16to32 (BB[j << l]); + } else if (transA == 'T') { + DD[j] += alpha * float16to32 (AA[j * x + i]) * float16to32 (BB[i << l]); + } + + for (j = 0; j < x; j++) { + if (!is_close(CC[j << l], C[j << l], 0.01, 0.001)) { + ret++; + } + if (!is_close(CC[j << l], DD[j], 0.001, 0.0001)) { + ret++; + } + } + } + free(A); + free(B); + free(C); + free(AA); + free(BB); + free(DD); + free(CC); + } // x + } // l + } // alpha + } // beta + + if (ret != 0) + fprintf (stderr, "FATAL ERROR SBGEMV - Return code: %d\n", ret); + return ret; +} From 848e9e6ba7c7eed58070b243867e328402b85443 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Sun, 13 Jul 2025 20:34:19 +0200 Subject: [PATCH 31/36] Add ID data for Intel Lunar Lake ("Core Ultra 200V series") --- cpuid_x86.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cpuid_x86.c b/cpuid_x86.c index 1b09c7217..8e1438e50 100644 --- a/cpuid_x86.c +++ b/cpuid_x86.c @@ -1567,6 +1567,7 @@ int get_cpuname(void){ case 10: case 15: case 14: // Alder Lake N + case 13: // Lunar Lake if(support_avx2()) return CPUTYPE_HASWELL; if(support_avx()) @@ -2412,7 +2413,8 @@ int get_coretype(void){ case 7: // Raptor Lake case 10: case 15: - case 14: // Alder Lake N + case 14: // Alder Lake N + case 13: // Lunar Lake #ifndef NO_AVX2 if(support_avx2()) return CORE_HASWELL; From e1054114601c972634ef692ed0ba150d7d5ede5d Mon Sep 17 00:00:00 2001 From: Chris Sidebottom Date: Sun, 13 Jul 2025 15:25:07 +0000 Subject: [PATCH 32/36] Add infrastructure for bgemv/bscal - Sets up all the various entrypoints for `bgemv` - Adds `bscal` for use in the `bgemv` interface - Adds test cases for comparing `sgemv` and `bgemv` - Adds generic kernels for `bgemv_n` and `bgemv_t` which are accurate enough to pass above tests --- .gitignore | 2 + cblas.h | 1 + cmake/kernel.cmake | 4 +- cmake/utils.cmake | 2 +- common_b.h | 9 ++ common_interface.h | 3 + common_level1.h | 4 + common_level2.h | 6 ++ common_macro.h | 11 ++- common_param.h | 4 + driver/level2/Makefile | 35 ++++++++ driver/level2/sbgemv_thread.c | 19 ++-- exports/gensymbol | 30 ++++++- exports/gensymbol.pl | 30 ++++++- interface/CMakeLists.txt | 1 + interface/Makefile | 22 ++++- interface/sbgemmt.c | 4 +- interface/sbgemv.c | 46 +++++++--- interface/scal.c | 10 ++- kernel/Makefile.L1 | 36 ++++++++ kernel/Makefile.L2 | 43 +++++++++ kernel/generic/bf16_macros.h | 64 ++++++++++++++ kernel/generic/gemmkernel_2x2.c | 33 +------ kernel/generic/gemv_n.c | 70 +++++++++++++++ kernel/generic/gemv_t.c | 60 +++++++++++++ kernel/generic/scal.c | 106 +++++++++++++++++++++++ kernel/setparam-ref.c | 4 +- test/Makefile | 20 +++-- test/compare_sgemm_bgemm.c | 9 -- test/compare_sgemv_bgemv.c | 149 ++++++++++++++++++++++++++++++++ test/compare_sgemv_sbgemv.c | 2 +- test/test_helpers.h | 13 ++- 32 files changed, 763 insertions(+), 89 deletions(-) create mode 100644 kernel/generic/bf16_macros.h create mode 100644 kernel/generic/gemv_n.c create mode 100644 kernel/generic/gemv_t.c create mode 100644 kernel/generic/scal.c create mode 100644 test/compare_sgemv_bgemv.c diff --git a/.gitignore b/.gitignore index 1807d4496..dcbc73dc8 100644 --- a/.gitignore +++ b/.gitignore @@ -83,6 +83,7 @@ test/ZBLAT3_3M.SUMM test/SHBLAT3.SUMM test/SBBLAT2.SUMM test/SBBLAT3.SUMM +test/BBLAT2.SUMM test/BBLAT3.SUMM test/cblat1 test/cblat2 @@ -100,6 +101,7 @@ test/test_shgemm test/test_sbgemm test/test_sbgemv test/test_bgemm +test/test_bgemv test/zblat1 test/zblat2 test/zblat3 diff --git a/cblas.h b/cblas.h index f48d5da1e..7503e43f7 100644 --- a/cblas.h +++ b/cblas.h @@ -465,6 +465,7 @@ void cblas_sbdtobf16(OPENBLAS_CONST blasint n, OPENBLAS_CONST double *in, OPEN void cblas_sbf16tos(OPENBLAS_CONST blasint n, OPENBLAS_CONST bfloat16 *in, OPENBLAS_CONST blasint incin, float *out, OPENBLAS_CONST blasint incout); /* convert BFLOAT16 array to double array */ void cblas_dbf16tod(OPENBLAS_CONST blasint n, OPENBLAS_CONST bfloat16 *in, OPENBLAS_CONST blasint incin, double *out, OPENBLAS_CONST blasint incout); +void cblas_bgemv(OPENBLAS_CONST enum CBLAS_ORDER order, OPENBLAS_CONST enum CBLAS_TRANSPOSE trans, OPENBLAS_CONST blasint m, OPENBLAS_CONST blasint n, OPENBLAS_CONST bfloat16 alpha, OPENBLAS_CONST bfloat16 *a, OPENBLAS_CONST blasint lda, OPENBLAS_CONST bfloat16 *x, OPENBLAS_CONST blasint incx, OPENBLAS_CONST bfloat16 beta, bfloat16 *y, OPENBLAS_CONST blasint incy); /* dot production of BFLOAT16 input arrays, and output as float */ float cblas_sbdot(OPENBLAS_CONST blasint n, OPENBLAS_CONST bfloat16 *x, OPENBLAS_CONST blasint incx, OPENBLAS_CONST bfloat16 *y, OPENBLAS_CONST blasint incy); void cblas_sbgemv(OPENBLAS_CONST enum CBLAS_ORDER order, OPENBLAS_CONST enum CBLAS_TRANSPOSE trans, OPENBLAS_CONST blasint m, OPENBLAS_CONST blasint n, OPENBLAS_CONST float alpha, OPENBLAS_CONST bfloat16 *a, OPENBLAS_CONST blasint lda, OPENBLAS_CONST bfloat16 *x, OPENBLAS_CONST blasint incx, OPENBLAS_CONST float beta, float *y, OPENBLAS_CONST blasint incy); diff --git a/cmake/kernel.cmake b/cmake/kernel.cmake index c0dd32ca0..ab548cfb0 100644 --- a/cmake/kernel.cmake +++ b/cmake/kernel.cmake @@ -169,8 +169,6 @@ if (BUILD_BFLOAT16) SetFallback(SHSWAPKERNEL ../arm/swap.c) SetFallback(TOBF16KERNEL ../x86_64/tobf16.c) SetFallback(BF16TOKERNEL ../x86_64/bf16to.c) - SetFallback(SBGEMVNKERNEL ../x86_64/sbgemv_n.c) - SetFallback(SBGEMVTKERNEL ../x86_64/sbgemv_t.c) endif () endmacro () @@ -221,6 +219,8 @@ macro(SetDefaultL2) SetFallback(XHEMV_V_KERNEL ../generic/zhemv_k.c) SetFallback(XHEMV_M_KERNEL ../generic/zhemv_k.c) if (BUILD_BFLOAT16) + SetFallback(BGEMVNKERNEL ../generic/gemv_n.c) + SetFallback(BGEMVTKERNEL ../generic/gemv_t.c) SetFallback(SBGEMVNKERNEL ../x86_64/sbgemv_n.c) SetFallback(SBGEMVTKERNEL ../x86_64/sbgemv_t.c) SetFallback(SHGERKERNEL ../generic/ger.c) diff --git a/cmake/utils.cmake b/cmake/utils.cmake index 1b19f41bc..52e5b5ee3 100644 --- a/cmake/utils.cmake +++ b/cmake/utils.cmake @@ -375,7 +375,7 @@ function(GenerateNamedObjects sources_in) if (NOT no_float_type) string(SUBSTRING ${float_type} 0 1 float_char) string(TOLOWER ${float_char} float_char) - if (${float_type} STREQUAL "BFLOAT16" AND NOT "${defines_in}" MATCHES "BGEMM") + if (${float_type} STREQUAL "BFLOAT16" AND NOT "${defines_in}" MATCHES "BGEM") set (float_char "sb") endif () endif () diff --git a/common_b.h b/common_b.h index 4d77ec4fa..1921c3a69 100644 --- a/common_b.h +++ b/common_b.h @@ -30,6 +30,11 @@ #define COMMON_B_H #ifndef DYNAMIC_ARCH +#define BGEMV_N_K bgemv_n +#define BGEMV_T_K bgemv_t + +#define BSCAL_K bscal_k + #define BGEMM_ONCOPY bgemm_oncopy #define BGEMM_OTCOPY bgemm_otcopy @@ -45,6 +50,10 @@ #define BGEMM_KERNEL bgemm_kernel #else +#define BGEMV_N_K gotoblas->bgemv_n +#define BGEMV_T_K gotoblas->bgemv_t + +#define BSCAL_K gotoblas->bscal_k #define BGEMM_ONCOPY gotoblas->bgemm_oncopy #define BGEMM_OTCOPY gotoblas->bgemm_otcopy diff --git a/common_interface.h b/common_interface.h index f69baab1c..945b6c8a1 100644 --- a/common_interface.h +++ b/common_interface.h @@ -60,6 +60,7 @@ double BLASFUNC(dsdot) (blasint *, float *, blasint *, float *, blasint *); double BLASFUNC(ddot) (blasint *, double *, blasint *, double *, blasint *); xdouble BLASFUNC(qdot) (blasint *, xdouble *, blasint *, xdouble *, blasint *); +void BLASFUNC(bscal) (blasint *, bfloat16 *, bfloat16 *, blasint *); float BLASFUNC(sbdot) (blasint *, bfloat16 *, blasint *, bfloat16 *, blasint *); void BLASFUNC(sbstobf16) (blasint *, float *, blasint *, bfloat16 *, blasint *); void BLASFUNC(sbdtobf16) (blasint *, double *, blasint *, bfloat16 *, blasint *); @@ -256,6 +257,8 @@ void BLASFUNC(xgeru)(blasint *, blasint *, xdouble *, xdouble *, blasint *, void BLASFUNC(xgerc)(blasint *, blasint *, xdouble *, xdouble *, blasint *, xdouble *, blasint *, xdouble *, blasint *); +void BLASFUNC(bgemv)(char *, blasint *, blasint *, bfloat16 *, bfloat16 *, blasint *, + bfloat16 *, blasint *, bfloat16 *, bfloat16 *, blasint *); void BLASFUNC(sbgemv)(char *, blasint *, blasint *, float *, bfloat16 *, blasint *, bfloat16 *, blasint *, float *, float *, blasint *); void BLASFUNC(sgemv)(char *, blasint *, blasint *, float *, float *, blasint *, diff --git a/common_level1.h b/common_level1.h index 85b39f7a7..7ab45a472 100644 --- a/common_level1.h +++ b/common_level1.h @@ -1,4 +1,5 @@ /*********************************************************************/ +/* Copyright 2025 The OpenBLAS Project. */ /* Copyright 2009, 2010 The University of Texas at Austin. */ /* All rights reserved. */ /* */ @@ -169,6 +170,9 @@ BLASLONG icmin_k(BLASLONG, float *, BLASLONG); BLASLONG izmin_k(BLASLONG, double *, BLASLONG); BLASLONG ixmin_k(BLASLONG, xdouble *, BLASLONG); + +int bscal_k(BLASLONG, BLASLONG, BLASLONG, bfloat16, + bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG); int sscal_k(BLASLONG, BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG); int dscal_k(BLASLONG, BLASLONG, BLASLONG, double, diff --git a/common_level2.h b/common_level2.h index 9a5ebb4d9..eea5e43f3 100644 --- a/common_level2.h +++ b/common_level2.h @@ -1,4 +1,5 @@ /*********************************************************************/ +/* Copyright 2025 The OpenBLAS Project */ /* Copyright 2009, 2010 The University of Texas at Austin. */ /* All rights reserved. */ /* */ @@ -44,6 +45,11 @@ extern "C" { #endif + +int bgemv_n(BLASLONG, BLASLONG, bfloat16, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, bfloat16, bfloat16 *, BLASLONG); +int bgemv_t(BLASLONG, BLASLONG, bfloat16, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, bfloat16, bfloat16 *, BLASLONG); +int bgemv_thread_n(BLASLONG, BLASLONG, bfloat16, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, bfloat16, bfloat16 *, BLASLONG, int); +int bgemv_thread_t(BLASLONG, BLASLONG, bfloat16, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, bfloat16, bfloat16 *, BLASLONG, int); int sbgemv_n(BLASLONG, BLASLONG, float, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, float, float *, BLASLONG); int sbgemv_t(BLASLONG, BLASLONG, float, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, float, float *, BLASLONG); int sbgemv_thread_n(BLASLONG, BLASLONG, float, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, float, float *, BLASLONG, int); diff --git a/common_macro.h b/common_macro.h index 22c1e14a2..f9c22089b 100644 --- a/common_macro.h +++ b/common_macro.h @@ -705,6 +705,11 @@ #elif defined(BFLOAT16) && defined(BGEMM) +#define SCAL_K BSCAL_K + +#define GEMV_N BGEMV_N_K +#define GEMV_T BGEMV_T_K + #define GEMM_BETA BGEMM_BETA #define GEMM_KERNEL_N BGEMM_KERNEL #define GEMM_KERNEL_L BGEMM_KERNEL @@ -754,8 +759,8 @@ #define D_BF16_TO_K DBF16TOD_K #define S_TO_BF16_K SBSTOBF16_K #define S_BF16_TO_K SBF16TOS_K -#define SBGEMV_N SBGEMV_N_K -#define SBGEMV_T SBGEMV_T_K +#define GEMV_N SBGEMV_N_K +#define GEMV_T SBGEMV_T_K #define AMAX_K SAMAX_K #define AMIN_K SAMIN_K @@ -773,8 +778,6 @@ #define AXPYC_K SAXPYC_K #define AXPBY_K SAXPBY_K #define SCAL_K SSCAL_K -#define GEMV_N SGEMV_N -#define GEMV_T SGEMV_T #define SYMV_U SSYMV_U #define SYMV_L SSYMV_L #define GERU_K SGERU_K diff --git a/common_param.h b/common_param.h index 503525dd2..d6b8d9bad 100644 --- a/common_param.h +++ b/common_param.h @@ -98,10 +98,14 @@ int (*shgemm_otcopy )(BLASLONG, BLASLONG, hfloat16 *, BLASLONG, hfloat16 *); int (*sbrot_k) (BLASLONG, float *, BLASLONG, float *, BLASLONG, float, float); int (*sbrotm_k) (BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); + int (*bscal_k) (BLASLONG, BLASLONG, BLASLONG, bfloat16, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG); int (*sbaxpy_k) (BLASLONG, BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG); int (*sbscal_k) (BLASLONG, BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG); int (*sbswap_k) (BLASLONG, BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG); + int (*bgemv_n) (BLASLONG, BLASLONG, bfloat16, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, bfloat16, bfloat16 *, BLASLONG); + int (*bgemv_t) (BLASLONG, BLASLONG, bfloat16, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, bfloat16, bfloat16 *, BLASLONG); + int (*sbgemv_n) (BLASLONG, BLASLONG, float, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, float, float *, BLASLONG); int (*sbgemv_t) (BLASLONG, BLASLONG, float, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, float, float *, BLASLONG); int (*sbger_k) (BLASLONG, BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); diff --git a/driver/level2/Makefile b/driver/level2/Makefile index 5f8c712a8..3f3731d3f 100644 --- a/driver/level2/Makefile +++ b/driver/level2/Makefile @@ -1,3 +1,31 @@ +############################################################################### +# Copyright (c) 2025 The OpenBLAS Project +# All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# 3. Neither the name of the OpenBLAS project nor the names of +# its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +############################################################################### + TOPDIR = ../.. include ../../Makefile.system @@ -423,6 +451,9 @@ XBLASOBJS += \ xtbmv_thread_CLU.$(SUFFIX) xtbmv_thread_CLN.$(SUFFIX) ifeq ($(BUILD_BFLOAT16),1) +BBLASOBJS += \ + bgemv_thread_n$(TSUFFIX).$(SUFFIX) \ + bgemv_thread_t$(TSUFFIX).$(SUFFIX) SBBLASOBJS += \ sbgemv_thread_n$(TSUFFIX).$(SUFFIX) \ sbgemv_thread_t$(TSUFFIX).$(SUFFIX) @@ -3707,6 +3738,10 @@ xtrsv_CUN.$(SUFFIX) xtrsv_CUN.$(PSUFFIX) : ztrsv_L.c ../../param.h $(CC) -c $(CFLAGS) -DXDOUBLE -DCOMPLEX -DTRANSA=4 -UUNIT $< -o $(@F) ifeq ($(BUILD_BFLOAT16),1) +bgemv_thread_n.$(SUFFIX) bgemv_thread_n.$(PSUFFIX) : sbgemv_thread.c ../../common.h + $(CC) -c $(CFLAGS) -DBGEMM -UCOMPLEX -UDOUBLE -UTRANSA -UCONJ -UXCONJ $< -o $(@F) +bgemv_thread_t.$(SUFFIX) bgemv_thread_t.$(PSUFFIX) : sbgemv_thread.c ../../common.h + $(CC) -c $(CFLAGS) -DBGEMM -UCOMPLEX -UDOUBLE -DTRANSA -UCONJ -UXCONJ $< -o $(@F) sbgemv_thread_n.$(SUFFIX) sbgemv_thread_n.$(PSUFFIX) : sbgemv_thread.c ../../common.h $(CC) -c $(CFLAGS) -UCOMPLEX -UDOUBLE -UTRANSA -UCONJ -UXCONJ $< -o $(@F) sbgemv_thread_t.$(SUFFIX) sbgemv_thread_t.$(PSUFFIX) : sbgemv_thread.c ../../common.h diff --git a/driver/level2/sbgemv_thread.c b/driver/level2/sbgemv_thread.c index 534c60f95..c7fc90a35 100644 --- a/driver/level2/sbgemv_thread.c +++ b/driver/level2/sbgemv_thread.c @@ -1,4 +1,5 @@ /*********************************************************************/ +/* Copyright 2025 The OpenBLAS Project. */ /* Copyright 2009, 2010 The University of Texas at Austin. */ /* All rights reserved. */ /* */ @@ -41,21 +42,21 @@ #include "common.h" #ifndef TRANSA -#define SBGEMV SBGEMV_N +#define GEMV GEMV_N #else -#define SBGEMV SBGEMV_T +#define GEMV GEMV_T #endif static int sbgemv_kernel(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *dummy1, FLOAT *dummy2, BLASLONG dummy3){ - bfloat16 *a, *x; - float *y; + IFLOAT *a, *x; + FLOAT *y; BLASLONG lda, incx, incy; BLASLONG m_from, m_to, n_from, n_to; - a = (bfloat16 *)args->a; - x = (bfloat16 *)args->b; - y = (float *)args->c; + a = (IFLOAT *)args->a; + x = (IFLOAT *)args->b; + y = (FLOAT *)args->c; lda = args->lda; incx = args->ldb; @@ -77,12 +78,12 @@ static int sbgemv_kernel(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, y += n_from * incy; #endif - SBGEMV(m_to - m_from, n_to - n_from, *((FLOAT *)(args->alpha)), a, lda, x, incx, *((FLOAT *)(args->beta)), y, incy); + GEMV(m_to - m_from, n_to - n_from, *((FLOAT *)(args->alpha)), a, lda, x, incx, *((FLOAT *)(args->beta)), y, incy); return 0; } -int CNAME(BLASLONG m, BLASLONG n, float alpha, bfloat16 *a, BLASLONG lda, bfloat16 *x, BLASLONG incx, float beta, float *y, BLASLONG incy, int threads) +int CNAME(BLASLONG m, BLASLONG n, FLOAT alpha, IFLOAT *a, BLASLONG lda, IFLOAT *x, BLASLONG incx, FLOAT beta, FLOAT *y, BLASLONG incy, int threads) { blas_arg_t args; blas_queue_t queue[MAX_CPU_NUMBER]; diff --git a/exports/gensymbol b/exports/gensymbol index 17fbd2877..40e13e623 100755 --- a/exports/gensymbol +++ b/exports/gensymbol @@ -1,5 +1,33 @@ #!/bin/sh +############################################################################### +# Copyright (c) 2025, The OpenBLAS Project +# All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# 3. Neither the name of the OpenBLAS project nor the names of +# its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +############################################################################### + # Changelog # 2017/09/03 staticfloat # Added zsymv and csymv into @lapackobjs2 so they are properly renamed @@ -51,7 +79,7 @@ blasobjsz=" zgeadd dzsum zgemmt zgemmtr" blasobjs="lsame xerbla" -bfblasobjs="bgemm sbgemm sbgemmt sbgemmtr sbgemv sbdot sbstobf16 sbdtobf16 sbf16tos dbf16tod" +bfblasobjs="bgemm bgemv sbgemm sbgemmt sbgemmtr sbgemv sbdot sbstobf16 sbdtobf16 sbf16tos dbf16tod" hfblasobjs="shgemm" cblasobjsc=" cblas_caxpy cblas_ccopy cblas_cdotc cblas_cdotu cblas_cgbmv cblas_cgemm cblas_cgemv diff --git a/exports/gensymbol.pl b/exports/gensymbol.pl index 01f68fbb3..3447a4e51 100644 --- a/exports/gensymbol.pl +++ b/exports/gensymbol.pl @@ -1,5 +1,33 @@ #!/usr/bin/env perl +############################################################################### +# Copyright (c) 2025, The OpenBLAS Project +# All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# 3. Neither the name of the OpenBLAS project nor the names of +# its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +############################################################################### + # Changelog # 2017/09/03 staticfloat # Added zsymv and csymv into @lapackobjs2 so they are properly renamed @@ -51,7 +79,7 @@ zgeadd, dzsum, zgemmt,zgemmtr); @blasobjs = (lsame, xerbla); -@bfblasobjs = (bgemm, sbgemm, sbgemmt, sbgemmtr, sbgemv, sbdot, sbstobf16, sbdtobf16, sbf16tos, dbf16tod); +@bfblasobjs = (bgemm, bgemv, sbgemm, sbgemmt, sbgemmtr, sbgemv, sbdot, sbstobf16, sbdtobf16, sbf16tos, dbf16tod); @hfblasobjs = (shgemm); @cblasobjsc = ( cblas_caxpy, cblas_ccopy, cblas_cdotc, cblas_cdotu, cblas_cgbmv, cblas_cgemm, cblas_cgemv, diff --git a/interface/CMakeLists.txt b/interface/CMakeLists.txt index 995bebec3..0fcc79bfc 100644 --- a/interface/CMakeLists.txt +++ b/interface/CMakeLists.txt @@ -155,6 +155,7 @@ if (BUILD_BFLOAT16) GenerateNamedObjects("gemm.c" "" "sbgemm" ${CBLAS_FLAG} "" "" true "BFLOAT16") GenerateNamedObjects("sbgemmt.c" "" "sbgemmt" ${CBLAS_FLAG} "" "" true "BFLOAT16") GenerateNamedObjects("sbgemmt.c" "RNAME" "sbgemmtr" ${CBLAS_FLAG} "" "" true "BFLOAT16") + GenerateNamedObjects("bgemv.c" "BGEMV" "bgemv" ${CBLAS_FLAG} "" "" true "BFLOAT16") GenerateNamedObjects("sbgemv.c" "" "sbgemv" ${CBLAS_FLAG} "" "" true "BFLOAT16") GenerateNamedObjects("tobf16.c" "SINGLE_PREC" "sbstobf16" ${CBLAS_FLAG} "" "" true "BFLOAT16") GenerateNamedObjects("tobf16.c" "DOUBLE_PREC" "sbdtobf16" ${CBLAS_FLAG} "" "" true "BFLOAT16") diff --git a/interface/Makefile b/interface/Makefile index 3af12748f..999d23e9a 100644 --- a/interface/Makefile +++ b/interface/Makefile @@ -75,7 +75,9 @@ SBLAS3OBJS = \ sgeadd.$(SUFFIX) sgemmt.$(SUFFIX) sgemmtr.$(SUFFIX) ifeq ($(BUILD_BFLOAT16),1) -BBLAS3OBJ = bgemm.$(SUFFIX) +BBLAS3OBJS = bgemm.$(SUFFIX) +BBLAS2OBJS = bgemv.$(SUFFIX) +BBLAS1OBJS = bscal.$(SUFFIX) SBBLAS1OBJS = sbdot.$(SUFFIX) SBBLAS2OBJS = sbgemv.$(SUFFIX) SBBLAS3OBJS = sbgemm.$(SUFFIX) sbgemmt.$(SUFFIX) sbgemmtr.$(SUFFIX) @@ -319,6 +321,8 @@ CSBLAS3OBJS = \ ifeq ($(BUILD_BFLOAT16),1) CBBLAS3OBJS = cblas_bgemm.$(SUFFIX) +CBBLAS2OBJS = cblas_bgemv.$(SUFFIX) +CBBLAS1OBJS = cblas_bscal.$(SUFFIX) CSBBLAS1OBJS = cblas_sbdot.$(SUFFIX) CSBBLAS2OBJS = cblas_sbgemv.$(SUFFIX) CSBBLAS3OBJS = cblas_sbgemm.$(SUFFIX) cblas_sbgemmt.$(SUFFIX) cblas_sbgemmtr.$(SUFFIX) cblas_sbgemm_batch.$(SUFFIX) @@ -423,7 +427,9 @@ override CFLAGS += -I. SBLAS1OBJS += $(CSBLAS1OBJS) SBLAS2OBJS += $(CSBLAS2OBJS) SBLAS3OBJS += $(CSBLAS3OBJS) -BBLAS3OBJ += $(CBBLAS3OBJS) +BBLAS3OBJS += $(CBBLAS3OBJS) +BBLAS2OBJS += $(CBBLAS2OBJS) +BBLAS1OBJS += $(CBBLAS1OBJS) SBBLAS1OBJS += $(CSBBLAS1OBJS) SBBLAS2OBJS += $(CSBBLAS2OBJS) SBBLAS3OBJS += $(CSBBLAS3OBJS) @@ -443,7 +449,7 @@ SBEXTOBJS += $(CSBEXTOBJS) CBAUXOBJS += $(CXERBLAOBJ) endif -BBLASOBJS = $(BBLAS3OBJ) +BBLASOBJS = $(BBLAS3OBJS) $(BBLAS2OBJS) $(BBLAS1OBJS) SBLASOBJS = $(SBLAS1OBJS) $(SBLAS2OBJS) $(SBLAS3OBJS) SBBLASOBJS = $(SBBLAS1OBJS) $(SBBLAS2OBJS) $(SBBLAS3OBJS) SHBLASOBJS = $(SHBLAS3OBJS) @@ -589,7 +595,7 @@ clean :: level1 : $(SBEXTOBJS) $(SBBLAS1OBJS) $(SBLAS1OBJS) $(DBLAS1OBJS) $(QBLAS1OBJS) $(CBLAS1OBJS) $(ZBLAS1OBJS) $(XBLAS1OBJS) $(AR) $(ARFLAGS) -ru $(TOPDIR)/$(LIBNAME) $^ -level2 : $(SBBLAS2OBJS) $(SBLAS2OBJS) $(DBLAS2OBJS) $(QBLAS2OBJS) $(CBLAS2OBJS) $(ZBLAS2OBJS) $(XBLAS2OBJS) +level2 : $(SBBLAS2OBJS) $(BBLAS2OBJS) $(SBLAS2OBJS) $(DBLAS2OBJS) $(QBLAS2OBJS) $(CBLAS2OBJS) $(ZBLAS2OBJS) $(XBLAS2OBJS) $(AR) $(ARFLAGS) -ru $(TOPDIR)/$(LIBNAME) $^ level3 : $(SBBLAS3OBJS) $(BBLAS3OBJ) $(SBLAS3OBJS) $(DBLAS3OBJS) $(QBLAS3OBJS) $(CBLAS3OBJS) $(ZBLAS3OBJS) $(XBLAS3OBJS) $(SHBLAS3OBJS) @@ -824,6 +830,8 @@ dsdot.$(SUFFIX) dsdot.$(PSUFFIX) : dsdot.c $(CC) $(CFLAGS) -c $< -o $(@F) ifeq ($(BUILD_BFLOAT16),1) +bscal.$(SUFFIX) bscal.$(PSUFFIX) : scal.c + $(CC) $(CFLAGS) -DBGEMM -c $< -o $(@F) sbdot.$(SUFFIX) sbdot.$(PSUFFIX) : bf16dot.c $(CC) $(CFLAGS) -c $< -o $(@F) sbstobf16.$(SUFFIX) sbstobf16.$(PSUFFIX) : tobf16.c @@ -981,6 +989,8 @@ xgerc.$(SUFFIX) xgerc.$(PSUFFIX) : zger.c $(CC) -c $(CFLAGS) -DCONJ $< -o $(@F) ifeq ($(BUILD_BFLOAT16),1) +bgemv.$(SUFFIX) bgemv.$(PSUFFIX) : sbgemv.c + $(CC) $(CFLAGS) -DBGEMM -c $< -o $(@F) sbgemv.$(SUFFIX) sbgemv.$(PSUFFIX) : sbgemv.c $(CC) $(CFLAGS) -c $< -o $(@F) endif @@ -1653,6 +1663,8 @@ cblas_dsdot.$(SUFFIX) cblas_dsdot.$(PSUFFIX) : dsdot.c $(CC) $(CFLAGS) -DCBLAS -c $< -o $(@F) ifeq ($(BUILD_BFLOAT16),1) +cblas_bscal.$(SUFFIX) cblas_bscal.$(PSUFFIX) : scal.c + $(CC) $(CFLAGS) -DCBLAS -c $< -o $(@F) cblas_sbdot.$(SUFFIX) cblas_sbdot.$(PSUFFIX) : bf16dot.c $(CC) $(CFLAGS) -DCBLAS -c $< -o $(@F) cblas_sbstobf16.$(SUFFIX) cblas_sbstobf16.$(PSUFFIX) : tobf16.c @@ -1807,6 +1819,8 @@ cblas_zdrot.$(SUFFIX) cblas_zdrot.$(PSUFFIX) : zrot.c $(CC) $(CFLAGS) -DCBLAS -c $< -o $(@F) ifeq ($(BUILD_BFLOAT16),1) +cblas_bgemv.$(SUFFIX) cblas_bgemv.$(PSUFFIX) : sbgemv.c + $(CC) -DCBLAS -DBGEMM -c $(CFLAGS) $< -o $(@F) cblas_sbgemv.$(SUFFIX) cblas_sbgemv.$(PSUFFIX) : sbgemv.c $(CC) -DCBLAS -c $(CFLAGS) $< -o $(@F) endif diff --git a/interface/sbgemmt.c b/interface/sbgemmt.c index 759af4bfb..67914fe65 100644 --- a/interface/sbgemmt.c +++ b/interface/sbgemmt.c @@ -1,5 +1,5 @@ /*********************************************************************/ -/* Copyright 2024, The OpenBLAS Project. */ +/* Copyright 2024-2025 The OpenBLAS Project. */ /* All rights reserved. */ /* */ /* Redistribution and use in source and binary forms, with or */ @@ -305,7 +305,7 @@ void CNAME(enum CBLAS_ORDER order, enum CBLAS_UPLO Uplo, #endif int (*gemv[]) (BLASLONG, BLASLONG, FLOAT, IFLOAT *, BLASLONG, IFLOAT *, BLASLONG, FLOAT, FLOAT *, BLASLONG) = { - SBGEMV_N, SBGEMV_T,}; + GEMV_N, GEMV_T,}; if (m == 0) diff --git a/interface/sbgemv.c b/interface/sbgemv.c index fce86f8e4..cee3e80fc 100644 --- a/interface/sbgemv.c +++ b/interface/sbgemv.c @@ -1,4 +1,5 @@ /*********************************************************************/ +/* Copyright 2025 The OpenBLAS Project. */ /* Copyright 2009, 2010 The University of Texas at Austin. */ /* All rights reserved. */ /* */ @@ -43,17 +44,25 @@ #include "functable.h" #endif +#ifdef BGEMM +#define GEMV_THREAD_N bgemv_thread_n +#define GEMV_THREAD_T bgemv_thread_t +#define ERROR_NAME "BGEMV " +#else +#define GEMV_THREAD_N sbgemv_thread_n +#define GEMV_THREAD_T sbgemv_thread_t #define ERROR_NAME "SBGEMV " +#endif #ifdef SMP -static int (*sbgemv_thread[])(BLASLONG, BLASLONG, float, bfloat16 *, BLASLONG, bfloat16 * , BLASLONG, float, float *, BLASLONG, int) = { - sbgemv_thread_n, sbgemv_thread_t, +static int (*gemv_thread[])(BLASLONG, BLASLONG, FLOAT, IFLOAT *, BLASLONG, IFLOAT * , BLASLONG, FLOAT, FLOAT *, BLASLONG, int) = { + GEMV_THREAD_N, GEMV_THREAD_T, }; #endif #ifndef CBLAS -void NAME(char *TRANS, blasint *M, blasint *N, float *ALPHA, bfloat16 *a, blasint *LDA, bfloat16 *x, blasint *INCX, float *BETA, float *y, blasint *INCY) +void NAME(char *TRANS, blasint *M, blasint *N, FLOAT *ALPHA, IFLOAT *a, blasint *LDA, IFLOAT *x, blasint *INCX, FLOAT *BETA, FLOAT *y, blasint *INCY) { char trans = *TRANS; blasint m = *M; @@ -61,14 +70,14 @@ void NAME(char *TRANS, blasint *M, blasint *N, float *ALPHA, bfloat16 *a, blasin blasint lda = *LDA; blasint incx = *INCX; blasint incy = *INCY; - float alpha = *ALPHA; - float beta = *BETA; + FLOAT alpha = *ALPHA; + FLOAT beta = *BETA; #ifdef SMP int nthreads; #endif - int (*sbgemv[])(BLASLONG, BLASLONG, float, bfloat16 *, BLASLONG, bfloat16 * , BLASLONG, float, float *, BLASLONG) = { - SBGEMV_N, SBGEMV_T, + int (*gemv[])(BLASLONG, BLASLONG, FLOAT, IFLOAT *, BLASLONG, IFLOAT * , BLASLONG, FLOAT, FLOAT *, BLASLONG) = { + GEMV_N, GEMV_T, }; blasint info; @@ -104,7 +113,7 @@ void NAME(char *TRANS, blasint *M, blasint *N, float *ALPHA, bfloat16 *a, blasin #else -void CNAME(enum CBLAS_ORDER order, enum CBLAS_TRANSPOSE TransA, blasint m, blasint n, float alpha, bfloat16 *a, blasint lda, bfloat16 *x, blasint incx, float beta, float *y, blasint incy) +void CNAME(enum CBLAS_ORDER order, enum CBLAS_TRANSPOSE TransA, blasint m, blasint n, FLOAT alpha, IFLOAT *a, blasint lda, IFLOAT *x, blasint incx, FLOAT beta, FLOAT *y, blasint incy) { blasint lenx, leny; int trans; @@ -113,8 +122,8 @@ void CNAME(enum CBLAS_ORDER order, enum CBLAS_TRANSPOSE TransA, blasint m, blasi int nthreads; #endif - int (*sbgemv[])(BLASLONG, BLASLONG, float, bfloat16 *, BLASLONG, bfloat16 * , BLASLONG, float, float *, BLASLONG) = { - SBGEMV_N, SBGEMV_T, + int (*gemv[])(BLASLONG, BLASLONG, FLOAT, IFLOAT *, BLASLONG, IFLOAT * , BLASLONG, FLOAT, FLOAT *, BLASLONG) = { + GEMV_N, GEMV_T, }; PRINT_DEBUG_CNAME; @@ -166,8 +175,17 @@ void CNAME(enum CBLAS_ORDER order, enum CBLAS_TRANSPOSE TransA, blasint m, blasi leny = m; } - if (alpha == ZERO) { - if (beta != ONE) SCAL_K(leny, 0, 0, beta, y, blasabs(incy), NULL, 0, NULL, 0); +#ifdef BGEMM + float alpha_float, beta_float; + SBF16TOS_K(1, &alpha, 1, &alpha_float, 1); + SBF16TOS_K(1, &beta, 1, &beta_float, 1); +#else + float alpha_float = alpha; + float beta_float = beta; +#endif + + if (alpha_float == ZERO) { + if (beta_float != ONE) SCAL_K(leny, 0, 0, beta, y, blasabs(incy), NULL, 0, NULL, 0); return; } @@ -185,10 +203,10 @@ void CNAME(enum CBLAS_ORDER order, enum CBLAS_TRANSPOSE TransA, blasint m, blasi if (nthreads == 1) { #endif - (sbgemv[(int)trans])(m, n, alpha, a, lda, x, incx, beta, y, incy); + (gemv[(int)trans])(m, n, alpha, a, lda, x, incx, beta, y, incy); #ifdef SMP } else { - (sbgemv_thread[(int)trans])(m, n, alpha, a, lda, x, incx, beta, y, incy, nthreads); + (gemv_thread[(int)trans])(m, n, alpha, a, lda, x, incx, beta, y, incy, nthreads); } #endif diff --git a/interface/scal.c b/interface/scal.c index c6638a62d..4f12df7c0 100644 --- a/interface/scal.c +++ b/interface/scal.c @@ -1,4 +1,5 @@ /*********************************************************************/ +/* Copyright 2025 The OpenBLAS Project. */ /* Copyright 2009, 2010 The University of Texas at Austin. */ /* All rights reserved. */ /* */ @@ -68,7 +69,14 @@ void CNAME(blasint n, FLOAT alpha, FLOAT *x, blasint incx){ if (incx <= 0 || n <= 0) return; - if (alpha == ONE) return; +#ifdef BGEMM + float alpha_float; + SBF16TOS_K(1, &alpha, 1, &alpha_float, 1); +#else + float alpha_float = alpha; +#endif + + if (alpha_float == ONE) return; IDEBUG_START; diff --git a/kernel/Makefile.L1 b/kernel/Makefile.L1 index 0fc672094..221cc5127 100644 --- a/kernel/Makefile.L1 +++ b/kernel/Makefile.L1 @@ -1,3 +1,31 @@ +############################################################################### +# Copyright (c) 2025 The OpenBLAS Project +# All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# 3. Neither the name of the OpenBLAS project nor the names of +# its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +############################################################################### + FMAFLAG= ifndef OLDGCC ifdef HAVE_FMA3 @@ -271,6 +299,10 @@ XDOTKERNEL = zdot.S endif ifeq ($(BUILD_BFLOAT16),1) +ifndef BSCALKERNEL +BSCALKERNEL = ../generic/scal.c +endif + ifndef SBDOTKERNEL SBDOTKERNEL = ../x86_64/sbdot.c endif @@ -551,6 +583,8 @@ XBLASOBJS += \ xscal_k$(TSUFFIX).$(SUFFIX) xswap_k$(TSUFFIX).$(SUFFIX) xsum_k$(TSUFFIX).$(SUFFIX) ifeq ($(BUILD_BFLOAT16),1) +BBLASOBJS += \ + bscal_k$(TSUFFIX).$(SUFFIX) SBBLASOBJS += \ sbdot_k$(TSUFFIX).$(SUFFIX) SBEXTOBJS += \ @@ -778,6 +812,8 @@ $(KDIR)qdot_k$(TSUFFIX).$(SUFFIX) $(KDIR)qdot_k$(TPSUFFIX).$(PSUFFIX) : $(KERNEL $(CC) -c $(CFLAGS) -UCOMPLEX -DXDOUBLE $< -o $@ ifeq ($(BUILD_BFLOAT16),1) +$(KDIR)bscal_k$(TSUFFIX).$(SUFFIX) $(KDIR)bscal_k$(TPSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(BSCALKERNEL) + $(CC) -c $(CFLAGS) -UCOMPLEX -UDOUBLE $< -o $@ $(KDIR)sbdot_k$(TSUFFIX).$(SUFFIX) $(KDIR)sbdot_k$(TPSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SBDOTKERNEL) $(CC) -c $(CFLAGS) -UCOMPLEX $< -o $@ $(KDIR)sbstobf16_k$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(TOBF16KERNEL) diff --git a/kernel/Makefile.L2 b/kernel/Makefile.L2 index 0332ba722..a9fcf9225 100644 --- a/kernel/Makefile.L2 +++ b/kernel/Makefile.L2 @@ -1,3 +1,31 @@ +############################################################################### +# Copyright (c) 2025 The OpenBLAS Project +# All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# 3. Neither the name of the OpenBLAS project nor the names of +# its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +############################################################################### + FMAFLAG= ifndef OLDGCC ifdef HAVE_FMA3 @@ -56,6 +84,14 @@ XGEMVTKERNEL = zgemv_t.S endif ifeq ($(BUILD_BFLOAT16),1) +ifndef BGEMVNKERNEL +BGEMVNKERNEL = ../generic/gemv_n.c +endif + +ifndef BGEMVTKERNEL +BGEMVTKERNEL = ../generic/gemv_t.c +endif + ifndef SBGEMVNKERNEL SBGEMVNKERNEL = ../x86_64/sbgemv_n.c endif @@ -255,6 +291,9 @@ XBLASOBJS += \ xgeru_k$(TSUFFIX).$(SUFFIX) xgerc_k$(TSUFFIX).$(SUFFIX) xgerv_k$(TSUFFIX).$(SUFFIX) xgerd_k$(TSUFFIX).$(SUFFIX) ifeq ($(BUILD_BFLOAT16),1) +BBLASOBJS += \ + bgemv_n$(TSUFFIX).$(SUFFIX) \ + bgemv_t$(TSUFFIX).$(SUFFIX) SBBLASOBJS += \ sbgemv_n$(TSUFFIX).$(SUFFIX) \ sbgemv_t$(TSUFFIX).$(SUFFIX) @@ -513,5 +552,9 @@ $(KDIR)sbgemv_n$(TSUFFIX).$(SUFFIX) $(KDIR)sbgemv_n$(TPSUFFIX).$(PSUFFIX) : $(KE $(CC) -c $(CFLAGS) -UCOMPLEX $< -o $@ $(KDIR)sbgemv_t$(TSUFFIX).$(SUFFIX) $(KDIR)sbgemv_t$(TPSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SBGEMVTKERNEL) $(CC) -c $(CFLAGS) -UCOMPLEX $< -o $@ +$(KDIR)bgemv_n$(TSUFFIX).$(SUFFIX) $(KDIR)bgemv_n$(TPSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(BGEMVNKERNEL) + $(CC) -c $(CFLAGS) -DBGEMM -UCOMPLEX $< -o $@ +$(KDIR)bgemv_t$(TSUFFIX).$(SUFFIX) $(KDIR)bgemv_t$(TPSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(BGEMVTKERNEL) + $(CC) -c $(CFLAGS) -DBGEMM -UCOMPLEX $< -o $@ endif diff --git a/kernel/generic/bf16_macros.h b/kernel/generic/bf16_macros.h new file mode 100644 index 000000000..f1b02cea4 --- /dev/null +++ b/kernel/generic/bf16_macros.h @@ -0,0 +1,64 @@ +/*************************************************************************** + * Copyright (c) 2025, The OpenBLAS Project + * All rights reserved. + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of the OpenBLAS project nor the names of + * its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * *****************************************************************************/ + +#if defined(BFLOAT16) && defined(BFLOAT16CONVERSION) +static float +bfloat16tof32 (bfloat16 value) +{ + blasint one = 1; + float result; + sbf16tos_(&one, &value, &one, &result, &one); + return result; +} + +#ifdef BGEMM +static bfloat16 f32tobfloat16(float value) { + blasint one = 1; + bfloat16 result; + sbstobf16_(&one, &value, &one, &result, &one); + return result; +} +#endif + +#ifdef BGEMM +#define ALPHA bfloat16tof32(alpha) +#define BETA bfloat16tof32(beta) +#define BF16TOF32(x) (bfloat16tof32(x)) +#define F32TOBF16(x) (f32tobfloat16(x)) +#else +#define ALPHA alpha +#define BETA beta +#define BF16TOF32(x) (bfloat16tof32(x)) +#define F32TOBF16(x) x +#endif +#else +#define ALPHA alpha +#define BETA beta +#define BF16TOF32(x) x +#define F32TOBF16(x) x +#endif diff --git a/kernel/generic/gemmkernel_2x2.c b/kernel/generic/gemmkernel_2x2.c index 8872f2f56..c24370c89 100644 --- a/kernel/generic/gemmkernel_2x2.c +++ b/kernel/generic/gemmkernel_2x2.c @@ -27,39 +27,8 @@ * *****************************************************************************/ #include "common.h" -#if defined(BFLOAT16) && defined(BFLOAT16CONVERSION) -static float -bfloat16tof32 (bfloat16 value) -{ - blasint one = 1; - float result; - sbf16tos_(&one, &value, &one, &result, &one); - return result; -} - -#ifdef BGEMM -static bfloat16 f32tobfloat16(float value) { - blasint one = 1; - bfloat16 result; - sbstobf16_(&one, &value, &one, &result, &one); - return result; -} -#endif +#include "bf16_macros.h" -#ifdef BGEMM -#define ALPHA bfloat16tof32(alpha) -#define BF16TOF32(x) (bfloat16tof32(x)) -#define F32TOBF16(x) (f32tobfloat16(x)) -#else -#define ALPHA alpha -#define BF16TOF32(x) (bfloat16tof32(x)) -#define F32TOBF16(x) x -#endif -#else -#define ALPHA alpha -#define BF16TOF32(x) x -#define F32TOBF16(x) x -#endif int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alpha,IFLOAT* ba,IFLOAT* bb,FLOAT* C,BLASLONG ldc #ifdef TRMMKERNEL ,BLASLONG offset diff --git a/kernel/generic/gemv_n.c b/kernel/generic/gemv_n.c new file mode 100644 index 000000000..1c72b07af --- /dev/null +++ b/kernel/generic/gemv_n.c @@ -0,0 +1,70 @@ +/*************************************************************************** +Copyright (c) 2013-2014, 2025 The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#include "common.h" +#include "bf16_macros.h" + +int CNAME(BLASLONG m, BLASLONG n, FLOAT alpha, IFLOAT *a, BLASLONG lda, IFLOAT *x, BLASLONG inc_x, FLOAT beta, FLOAT *y, BLASLONG inc_y) +{ + BLASLONG i; + BLASLONG ix, iy; + BLASLONG j; + FLOAT *a_ptr; +#ifdef BGEMM + float temp; +#else + FLOAT temp; +#endif + + iy = 0; + for (BLASLONG i = 0; i < m; i++) + { + temp = 0.0; + + ix = 0; + a_ptr = a; + for (BLASLONG j = 0; j < n; j++) + { + temp += BF16TOF32(a_ptr[i]) * BF16TOF32(x[ix]); + ix += inc_x; + a_ptr += lda; + } + + if (BETA == ZERO) + { + y[iy] = F32TOBF16(ALPHA * temp); + } + else + { + y[iy] = F32TOBF16(ALPHA * temp + BETA * BF16TOF32(y[iy])); + } + + iy += inc_y; + } + + return (0); +} diff --git a/kernel/generic/gemv_t.c b/kernel/generic/gemv_t.c new file mode 100644 index 000000000..3b651b5c1 --- /dev/null +++ b/kernel/generic/gemv_t.c @@ -0,0 +1,60 @@ +/*************************************************************************** +Copyright (c) 2013, 2025 The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#include "common.h" +#include "bf16_macros.h" + +int CNAME(BLASLONG m, BLASLONG n, FLOAT alpha, IFLOAT *a, BLASLONG lda, IFLOAT *x, BLASLONG inc_x, FLOAT beta, FLOAT *y, BLASLONG inc_y) +{ + BLASLONG i; + BLASLONG ix, iy; + BLASLONG j; + FLOAT *a_ptr; +#ifdef BGEMM + float temp; +#else + FLOAT temp; +#endif + + iy = 0; + a_ptr = a; + + for (j = 0; j < n; j++) + { + temp = 0.0; + ix = 0; + for (i = 0; i < m; i++) + { + temp += BF16TOF32(a_ptr[i]) * BF16TOF32(x[ix]); + ix += inc_x; + } + y[iy] += F32TOBF16(ALPHA * temp); + iy += inc_y; + a_ptr += lda; + } + return (0); +} diff --git a/kernel/generic/scal.c b/kernel/generic/scal.c new file mode 100644 index 000000000..fef0c7bf9 --- /dev/null +++ b/kernel/generic/scal.c @@ -0,0 +1,106 @@ +/*************************************************************************** +Copyright (c) 2013, 2025 The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#include "common.h" + +int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y, FLOAT *dummy, BLASLONG dummy2) +{ + BLASLONG i = 0, j = 0; +#if defined(BFLOAT16) + float x_float, da_float; + SBF16TOS_K(1, &da, 1, &da_float, 1); +#else + float x_float; + float da_float = da; +#endif + + if ((n <= 0) || (inc_x <= 0)) + return (0); + + if (dummy2 == 0) + { + while (j < n) + { + + if (da_float == 0.0) + x_float = 0.0; + else + { +#if defined(BFLOAT16) + SBF16TOS_K(1, &x[i], 1, &x_float, 1); +#else + float x_float = x[i]; +#endif + x_float = da_float * x_float; + } + +#if defined(BFLOAT16) + SBSTOBF16_K(1, &x_float, 1, &x[i], 1); +#else + x[i] = x_float; +#endif + + i += inc_x; + j++; + } + } + else + { + + while (j < n) + { +#if defined(BFLOAT16) + SBF16TOS_K(1, &x[i], 1, &x_float, 1); +#else + float x_float = x[i]; +#endif + if (da == 0.0) + if (!isnan(x_float) && !isinf(x_float)) + { + x_float = 0.0; + } + else + { + x_float = NAN; + } + else + { + x_float = da_float * x_float; + } + +#if defined(BFLOAT16) + SBSTOBF16_K(1, &x_float, 1, &x[i], 1); +#else + x[i] = x_float; +#endif + + i += inc_x; + j++; + } + } + return 0; +} diff --git a/kernel/setparam-ref.c b/kernel/setparam-ref.c index 886895acc..c09472e76 100644 --- a/kernel/setparam-ref.c +++ b/kernel/setparam-ref.c @@ -83,8 +83,8 @@ gotoblas_t TABLE_NAME = { isamax_kTS, isamin_kTS, ismax_kTS, ismin_kTS, snrm2_kTS, sasum_kTS, ssum_kTS, scopy_kTS, sbdot_kTS, dsdot_kTS, - srot_kTS, srotm_kTS, saxpy_kTS, sscal_kTS, sswap_kTS, - sbgemv_nTS, sbgemv_tTS, sger_kTS, + srot_kTS, srotm_kTS, bscal_kTS, saxpy_kTS, sscal_kTS, sswap_kTS, + bgemv_nTS, bgemv_tTS, sbgemv_nTS, sbgemv_tTS, sger_kTS, ssymv_LTS, ssymv_UTS, bgemm_kernelTS, bgemm_betaTS, diff --git a/test/Makefile b/test/Makefile index 144738eb2..d8e12058a 100644 --- a/test/Makefile +++ b/test/Makefile @@ -120,6 +120,7 @@ endif endif ifeq ($(BUILD_BFLOAT16), 1) +BB2 = test_bgemv B2 = test_sbgemv endif ifeq ($(BUILD_SINGLE),1) @@ -135,12 +136,14 @@ ifeq ($(BUILD_COMPLEX16),1) Z2=zblat2 endif -level2: $(B2) $(S2) $(D2) $(C2) $(Z2) +level2: $(BB2) $(B2) $(S2) $(D2) $(C2) $(Z2) ifneq ($(CROSS), 1) rm -f ?BLAT2.SUMM ifeq ($(BUILD_BFLOAT16),1) + OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 ./test_bgemv > BBLAT2.SUMM + @$(GREP) -q FATAL BBLAT2.SUMM && cat BBLAT2.SUMM || exit 0 OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 ./test_sbgemv > SBBLAT2.SUMM @$(GREP) -q FATAL SBBLAT2.SUMM && cat SBBLAT2.SUMM || exit 0 endif @@ -164,6 +167,8 @@ ifdef SMP rm -f ?BLAT2.SUMM ifeq ($(USE_OPENMP), 1) ifeq ($(BUILD_BFLOAT16),1) + OMP_NUM_THREADS=2 ./test_bgemv > BBLAT2.SUMM + @$(GREP) -q FATAL BBLAT2.SUMM && cat BBLAT2.SUMM || exit 0 OMP_NUM_THREADS=2 ./test_sbgemv > SBBLAT2.SUMM @$(GREP) -q FATAL SBBLAT2.SUMM && cat SBBLAT2.SUMM || exit 0 endif @@ -185,6 +190,8 @@ ifeq ($(BUILD_COMPLEX16),1) endif else ifeq ($(BUILD_BFLOAT16),1) + OMP_NUM_THREADS=2 ./test_bgemv > BBLAT2.SUMM + @$(GREP) -q FATAL BBLAT2.SUMM && cat BBLAT2.SUMM || exit 0 OMP_NUM_THREADS=2 ./test_sbgemv > SBBLAT2.SUMM @$(GREP) -q FATAL SBBLAT2.SUMM && cat SBBLAT2.SUMM || exit 0 endif @@ -419,13 +426,16 @@ endif ifeq ($(BUILD_BFLOAT16),1) test_bgemm : compare_sgemm_bgemm.c test_helpers.h ../$(LIBNAME) - $(CC) $(CLDFLAGS) -o test_bgemm compare_sgemm_bgemm.c ../$(LIBNAME) $(EXTRALIB) $(CEXTRALIB) + $(CC) $(CLDFLAGS) -DIBFLOAT16 -DOBFLOAT16 -o test_bgemm compare_sgemm_bgemm.c ../$(LIBNAME) $(EXTRALIB) $(CEXTRALIB) + +test_bgemv : compare_sgemv_bgemv.c ../$(LIBNAME) + $(CC) $(CLDFLAGS) -DIBFLOAT16 -DOBFLOAT16 -o test_bgemv compare_sgemv_bgemv.c ../$(LIBNAME) $(EXTRALIB) $(CEXTRALIB) test_sbgemm : compare_sgemm_sbgemm.c test_helpers.h ../$(LIBNAME) - $(CC) $(CLDFLAGS) -o test_sbgemm compare_sgemm_sbgemm.c ../$(LIBNAME) $(EXTRALIB) $(CEXTRALIB) + $(CC) $(CLDFLAGS) -DIBFLOAT16 -o test_sbgemm compare_sgemm_sbgemm.c ../$(LIBNAME) $(EXTRALIB) $(CEXTRALIB) test_sbgemv : compare_sgemv_sbgemv.c ../$(LIBNAME) - $(CC) $(CLDFLAGS) -o test_sbgemv compare_sgemv_sbgemv.c ../$(LIBNAME) $(EXTRALIB) $(CEXTRALIB) + $(CC) $(CLDFLAGS) -DIBFLOAT16 -o test_sbgemv compare_sgemv_sbgemv.c ../$(LIBNAME) $(EXTRALIB) $(CEXTRALIB) endif ifeq ($(BUILD_COMPLEX),1) @@ -444,7 +454,7 @@ clean: @rm -f *.$(SUFFIX) *.$(PSUFFIX) gmon.$(SUFFIX)ut *.SUMM *.cxml *.exe *.pdb *.dwf \ sblat1 dblat1 cblat1 zblat1 \ sblat2 dblat2 cblat2 zblat2 \ - test_bgemm test_sbgemm test_sbgemv sblat3 dblat3 cblat3 zblat3 \ + test_bgemm test_bgemv test_sbgemm test_sbgemv sblat3 dblat3 cblat3 zblat3 \ sblat1p dblat1p cblat1p zblat1p \ sblat2p dblat2p cblat2p zblat2p \ sblat3p dblat3p cblat3p zblat3p \ diff --git a/test/compare_sgemm_bgemm.c b/test/compare_sgemm_bgemm.c index bc8a0b468..f18fe1201 100644 --- a/test/compare_sgemm_bgemm.c +++ b/test/compare_sgemm_bgemm.c @@ -34,15 +34,6 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define BGEMM BLASFUNC(bgemm) #define BGEMM_LARGEST 256 -static float truncate_float32_to_bfloat16(float value) { - blasint one = 1; - bfloat16 tmp; - float result; - sbstobf16_(&one, &value, &one, &tmp, &one); - sbf16tos_(&one, &tmp, &one, &result, &one); - return result; -} - int main (int argc, char *argv[]) { diff --git a/test/compare_sgemv_bgemv.c b/test/compare_sgemv_bgemv.c new file mode 100644 index 000000000..aac98760f --- /dev/null +++ b/test/compare_sgemv_bgemv.c @@ -0,0 +1,149 @@ +/*************************************************************************** +Copyright (c) 2020,2025 The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ +#include +#include +#include "../common.h" + +#include "test_helpers.h" + +#define SGEMV BLASFUNC(sgemv) +#define BGEMV BLASFUNC(bgemv) +#define BGEMV_LARGEST 256 + +int main(int argc, char *argv[]) +{ + blasint k; + int i, j, l; + blasint x, y; + blasint one = 1; + int ret = 0; + int loop = BGEMV_LARGEST; + char transA = 'N'; + float alpha = 1.0, beta = 0.0; + bfloat16 alpha_bf16, beta_bf16; + + for (beta = 0; beta < 3; beta += 1) + { + for (alpha = 0; alpha < 3; alpha += 1) + { + for (l = 0; l < 2; l++) + { // l = 1 to test inc_x & inc_y not equal to one. + for (x = 1; x <= loop; x++) + { + k = (x == 0) ? 0 : l + 1; + float *A = (float *)malloc_safe(x * x * sizeof(FLOAT)); + float *B = (float *)malloc_safe(x * sizeof(FLOAT) << l); + float *C = (float *)malloc_safe(x * sizeof(FLOAT) << l); + bfloat16 *AA = (bfloat16 *)malloc_safe(x * x * sizeof(bfloat16)); + bfloat16 *BB = (bfloat16 *)malloc_safe(x * sizeof(bfloat16) << l); + bfloat16 *CC = (bfloat16 *)malloc_safe(x * sizeof(bfloat16) << l); + float *DD = (float *)malloc_safe(x * sizeof(FLOAT)); + if ((A == NULL) || (B == NULL) || (C == NULL) || (AA == NULL) || (BB == NULL) || + (CC == NULL) || (DD == NULL)) + return 1; + + for (j = 0; j < x; j++) + { + for (i = 0; i < x; i++) + { + A[j * x + i] = ((FLOAT)rand() / (FLOAT)RAND_MAX) + 0.5; + sbstobf16_(&one, &A[j * x + i], &one, &AA[j * x + i], &one); + } + B[j << l] = ((FLOAT)rand() / (FLOAT)RAND_MAX) + 0.5; + sbstobf16_(&one, &B[j << l], &one, &BB[j << l], &one); + + C[j << l] = ((FLOAT)rand() / (FLOAT)RAND_MAX) + 0.5; + sbstobf16_(&one, &B[j << l], &one, &CC[j << l], &one); + } + + for (y = 0; y < 2; y++) + { + if (y == 0) + { + transA = 'N'; + } + else + { + transA = 'T'; + } + + memset(C, 0, x * sizeof(FLOAT) << l); + memset(CC, 0, x * sizeof(bfloat16) << l); + memset(DD, 0, x * sizeof(FLOAT)); + + sbstobf16_(&one, &alpha, &one, &alpha_bf16, &one); + sbstobf16_(&one, &beta, &one, &beta_bf16, &one); + SGEMV(&transA, &x, &x, &alpha, A, &x, B, &k, &beta, C, &k); + BGEMV(&transA, &x, &x, &alpha_bf16, AA, &x, BB, &k, &beta_bf16, CC, &k); + + for (int i = 0; i < x; i++) + DD[i] *= beta; + + for (j = 0; j < x; j++) + for (i = 0; i < x; i++) + if (transA == 'N') + { + DD[i] += alpha * float16to32(AA[j * x + i]) * float16to32(BB[j << l]); + } + else if (transA == 'T') + { + DD[j] += alpha * float16to32(AA[j * x + i]) * float16to32(BB[i << l]); + } + + for (j = 0; j < x; j++) + { + if (!is_close(float16to32(CC[j << l]), truncate_float32_to_bfloat16(C[j << l]), 0.01, 0.001)) + { + printf("Mismatch at trans=%c, alpha=%.2f, beta=%.2f, i=%d, j=%d, k=%d: CC=%.6f, C=%.6f\n", + transA, alpha, beta, i, j, k, float16to32(CC[j << l]), truncate_float32_to_bfloat16(C[j << l])); + ret++; + } + if (!is_close(float16to32(CC[j << l]), truncate_float32_to_bfloat16(DD[j]), 0.001, 0.0001)) + { + printf("Mismatch at trans=%c, alpha=%.2f, beta=%.2f, i=%d, j=%d, k=%d: CC=%.6f, C=%.6f\n", + transA, alpha, beta, i, j, k, float16to32(CC[j << l]), truncate_float32_to_bfloat16(DD[j])); + ret++; + } + } + } + + free(A); + free(B); + free(C); + free(AA); + free(BB); + free(CC); + free(DD); + } // x + } // l + } // alpha + } // beta + + if (ret != 0) + fprintf(stderr, "FATAL ERROR BGEMV - Return code: %d\n", ret); + return ret; +} diff --git a/test/compare_sgemv_sbgemv.c b/test/compare_sgemv_sbgemv.c index 5fa2d5f66..15cdce6cb 100644 --- a/test/compare_sgemv_sbgemv.c +++ b/test/compare_sgemv_sbgemv.c @@ -56,8 +56,8 @@ main (int argc, char *argv[]) float *C = (float *)malloc_safe(x * sizeof(FLOAT) << l); bfloat16 *AA = (bfloat16 *)malloc_safe(x * x * sizeof(bfloat16)); bfloat16 *BB = (bfloat16 *)malloc_safe(x * sizeof(bfloat16) << l); - float *DD = (float *)malloc_safe(x * sizeof(FLOAT)); float *CC = (float *)malloc_safe(x * sizeof(FLOAT) << l); + float *DD = (float *)malloc_safe(x * sizeof(FLOAT)); if ((A == NULL) || (B == NULL) || (C == NULL) || (AA == NULL) || (BB == NULL) || (DD == NULL) || (CC == NULL)) return 1; diff --git a/test/test_helpers.h b/test/test_helpers.h index 2bb3f7acd..fcec86e10 100644 --- a/test/test_helpers.h +++ b/test/test_helpers.h @@ -31,7 +31,7 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "../common.h" -#if IFLOAT == bfloat16 +#ifdef IBFLOAT16 static float float16to32(bfloat16 value) { blasint one = 1; @@ -41,6 +41,17 @@ static float float16to32(bfloat16 value) } #endif +#ifdef OBFLOAT16 +static float truncate_float32_to_bfloat16(float value) { + blasint one = 1; + bfloat16 tmp; + float result; + sbstobf16_(&one, &value, &one, &tmp, &one); + sbf16tos_(&one, &tmp, &one, &result, &one); + return result; +} +#endif + static void *malloc_safe(size_t size) { if (size == 0) return malloc(1); From 72d2ebb4ddb7c44141025fb3c3223415554a5666 Mon Sep 17 00:00:00 2001 From: Chris Sidebottom Date: Tue, 15 Jul 2025 15:00:20 +0100 Subject: [PATCH 33/36] Re-add GEMV fallback for Level3 --- cmake/kernel.cmake | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmake/kernel.cmake b/cmake/kernel.cmake index ab548cfb0..b9c47206e 100644 --- a/cmake/kernel.cmake +++ b/cmake/kernel.cmake @@ -169,6 +169,10 @@ if (BUILD_BFLOAT16) SetFallback(SHSWAPKERNEL ../arm/swap.c) SetFallback(TOBF16KERNEL ../x86_64/tobf16.c) SetFallback(BF16TOKERNEL ../x86_64/bf16to.c) + SetFallback(BGEMVNKERNEL ../generic/gemv_n.c) + SetFallback(BGEMVTKERNEL ../generic/gemv_t.c) + SetFallback(SBGEMVNKERNEL ../x86_64/sbgemv_n.c) + SetFallback(SBGEMVTKERNEL ../x86_64/sbgemv_t.c) endif () endmacro () From 947d7af4c9c4ea7e6edeb42ceeb865eab05d5c37 Mon Sep 17 00:00:00 2001 From: Chris Sidebottom Date: Tue, 15 Jul 2025 14:41:19 +0000 Subject: [PATCH 34/36] Fix CMake references to bscal and bgemv --- cmake/kernel.cmake | 1 + interface/CMakeLists.txt | 3 ++- kernel/CMakeLists.txt | 3 +++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/cmake/kernel.cmake b/cmake/kernel.cmake index b9c47206e..6d752ac51 100644 --- a/cmake/kernel.cmake +++ b/cmake/kernel.cmake @@ -110,6 +110,7 @@ macro(SetDefaultL1) SetFallback(SROTMKERNEL rotm.S) SetFallback(DROTMKERNEL rotm.S) SetFallback(QROTMKERNEL rotm.S) + SetFallback(BSCALKERNEL ../generic/scal.c) SetFallback(SSCALKERNEL scal.S) SetFallback(DSCALKERNEL scal.S) SetFallback(CSCALKERNEL zscal.S) diff --git a/interface/CMakeLists.txt b/interface/CMakeLists.txt index 0fcc79bfc..afb45a420 100644 --- a/interface/CMakeLists.txt +++ b/interface/CMakeLists.txt @@ -150,12 +150,13 @@ endif () GenerateNamedObjects("imax.c" "USE_MIN" "i*min" ${CBLAS_FLAG}) if (BUILD_BFLOAT16) + GenerateNamedObjects("scal.c" "BGEMM" "bscal" ${CBLAS_FLAG} "" "" true "BFLOAT16") GenerateNamedObjects("bf16dot.c" "" "sbdot" ${CBLAS_FLAG} "" "" true "BFLOAT16") GenerateNamedObjects("gemm.c" "BGEMM" "bgemm" ${CBLAS_FLAG} "" "" true "BFLOAT16") GenerateNamedObjects("gemm.c" "" "sbgemm" ${CBLAS_FLAG} "" "" true "BFLOAT16") GenerateNamedObjects("sbgemmt.c" "" "sbgemmt" ${CBLAS_FLAG} "" "" true "BFLOAT16") GenerateNamedObjects("sbgemmt.c" "RNAME" "sbgemmtr" ${CBLAS_FLAG} "" "" true "BFLOAT16") - GenerateNamedObjects("bgemv.c" "BGEMV" "bgemv" ${CBLAS_FLAG} "" "" true "BFLOAT16") + GenerateNamedObjects("sbgemv.c" "BGEMM" "bgemv" ${CBLAS_FLAG} "" "" true "BFLOAT16") GenerateNamedObjects("sbgemv.c" "" "sbgemv" ${CBLAS_FLAG} "" "" true "BFLOAT16") GenerateNamedObjects("tobf16.c" "SINGLE_PREC" "sbstobf16" ${CBLAS_FLAG} "" "" true "BFLOAT16") GenerateNamedObjects("tobf16.c" "DOUBLE_PREC" "sbdtobf16" ${CBLAS_FLAG} "" "" true "BFLOAT16") diff --git a/kernel/CMakeLists.txt b/kernel/CMakeLists.txt index 0ec08827d..8330f5054 100644 --- a/kernel/CMakeLists.txt +++ b/kernel/CMakeLists.txt @@ -121,6 +121,7 @@ function (build_core TARGET_CORE KDIR TSUFFIX KERNEL_DEFINITIONS) # sbdot if (BUILD_BFLOAT16) + GenerateNamedObjects("${KERNELDIR}/${BSCALKERNEL}" "BGEMM" "scal_k" false "" "" false "BFLOAT16") GenerateNamedObjects("${KERNELDIR}/${SBDOTKERNEL}" "SBDOT" "dot_k" false "" "" false "BFLOAT16") GenerateNamedObjects("${KERNELDIR}/${BF16TOKERNEL}" "SINGLE" "f16tos_k" false "" "" false "BFLOAT16") GenerateNamedObjects("${KERNELDIR}/${BF16TOKERNEL}" "DOUBLE" "bf16tod_k" false "" "" false "DOUBLE") @@ -222,6 +223,8 @@ function (build_core TARGET_CORE KDIR TSUFFIX KERNEL_DEFINITIONS) GenerateNamedObjects("${KERNELDIR}/${SGEMVTKERNEL}" "TRANS" "gemv_t" false "" "" false "SINGLE") endif () if (BUILD_BFLOAT16) + GenerateNamedObjects("${KERNELDIR}/${BGEMVNKERNEL}" "BGEMM" "gemv_n" false "" "" false "BFLOAT16") + GenerateNamedObjects("${KERNELDIR}/${BGEMVTKERNEL}" "BGEMM" "gemv_t" false "" "" false "BFLOAT16") GenerateNamedObjects("${KERNELDIR}/${SBGEMVNKERNEL}" "" "gemv_n" false "" "" false "BFLOAT16") GenerateNamedObjects("${KERNELDIR}/${SBGEMVTKERNEL}" "" "gemv_t" false "" "" false "BFLOAT16") endif () From 7d7757acd182e82992e0258f86c13d67fda614a3 Mon Sep 17 00:00:00 2001 From: Martin Kroeker Date: Tue, 15 Jul 2025 18:25:55 +0200 Subject: [PATCH 35/36] Update cross-compilation instructions for the Android NDK --- docs/install.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/install.md b/docs/install.md index 656c6a121..73ce6df45 100644 --- a/docs/install.md +++ b/docs/install.md @@ -623,14 +623,22 @@ Note: using `TARGET=CORTEXA57` in place of `ARMV8` will pick up better optimized routines. Implementations for the `CORTEXA57` target are compatible with all other `ARMV8` targets. -Note: for NDK 23b, something as simple as: +Note: for NDK 23b and later, something as simple as: ```bash export PATH=/opt/android-ndk-r23b/toolchains/llvm/prebuilt/linux-x86_64/bin/:$PATH -make HOSTCC=gcc CC=/opt/android-ndk-r23b/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android31-clang ONLY_CBLAS=1 TARGET=ARMV8 +make HOSTCC=gcc CC=/opt/android-ndk-r23b/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android31-clang ONLY_CBLAS=1 TARGET=ARMV8 RANLIB=echo ``` appears to be sufficient on Linux. On OSX, setting AR to the ar provided in the "bin" path of the NDK (probably `llvm-ar`) is also necessary. +If you prefer building with CMake, running +```bash +cmake -DANDROID_ABI=arm64-v8a -DTARGET=ARMV8 -DCMAKE_TOOLCHAIN_FILE=/opt/android-ndk-r27/build/cmake/android.toolchain.cmake -DNOFORTRAN=1 -DANDROID_PLATFORM=android-23 .. +cmake --build . +``` +in your build directory should work (be sure to adjust the toolchain_file argument according to where you installed the NDK, and the ANDROID_PLATFORM +according to the minimum version of Android you want to support. (If you leave out the ANDROID_PLATFORM parameter, the build will fail with an error +message about a missing declaration or missing header file complex.h) ??? note "Alternative build script for 3 architectures" From eae0abfdb6153a4f8619927e46c797859e55d48c Mon Sep 17 00:00:00 2001 From: Rajendra Prasad Matcha Date: Fri, 11 Jul 2025 14:51:16 +0530 Subject: [PATCH 36/36] SME1 based direct kernel with alpha and beta for cblas_sgemm level 3 API. --- common_level3.h | 7 + common_param.h | 1 + common_s.h | 2 + interface/gemm.c | 3 + kernel/CMakeLists.txt | 2 + kernel/Makefile.L3 | 6 +- .../sgemm_direct_alpha_beta_arm64_sme1.c | 199 ++++++++++++++++++ kernel/arm64/sme_abi.h | 46 ++++ kernel/setparam-ref.c | 1 + 9 files changed, 266 insertions(+), 1 deletion(-) create mode 100644 kernel/arm64/sgemm_direct_alpha_beta_arm64_sme1.c create mode 100644 kernel/arm64/sme_abi.h diff --git a/common_level3.h b/common_level3.h index 1838b4bf6..6ca62dc88 100644 --- a/common_level3.h +++ b/common_level3.h @@ -52,6 +52,13 @@ void sgemm_direct(BLASLONG M, BLASLONG N, BLASLONG K, float * B, BLASLONG strideB, float * R, BLASLONG strideR); +void sgemm_direct_alpha_beta(BLASLONG M, BLASLONG N, BLASLONG K, + float alpha, + float * A, BLASLONG strideA, + float * B, BLASLONG strideB, + float beta, + float * R, BLASLONG strideR); + int sgemm_direct_performant(BLASLONG M, BLASLONG N, BLASLONG K); int shgemm_beta(BLASLONG, BLASLONG, BLASLONG, float, diff --git a/common_param.h b/common_param.h index f82b73a72..8923fa430 100644 --- a/common_param.h +++ b/common_param.h @@ -240,6 +240,7 @@ int (*shgemm_otcopy )(BLASLONG, BLASLONG, hfloat16 *, BLASLONG, hfloat16 *); #endif #ifdef ARCH_ARM64 void (*sgemm_direct) (BLASLONG, BLASLONG, BLASLONG, float *, BLASLONG , float *, BLASLONG , float * , BLASLONG); + void (*sgemm_direct_alpha_beta) (BLASLONG, BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float, float * , BLASLONG); #endif diff --git a/common_s.h b/common_s.h index 1dede1e36..88b4732f5 100644 --- a/common_s.h +++ b/common_s.h @@ -49,6 +49,7 @@ #define SGEMM_DIRECT_PERFORMANT sgemm_direct_performant #define SGEMM_DIRECT sgemm_direct +#define SGEMM_DIRECT_ALPHA_BETA sgemm_direct_alpha_beta #define SGEMM_ONCOPY sgemm_oncopy #define SGEMM_OTCOPY sgemm_otcopy @@ -218,6 +219,7 @@ #elif ARCH_ARM64 #define SGEMM_DIRECT_PERFORMANT sgemm_direct_performant #define SGEMM_DIRECT gotoblas -> sgemm_direct +#define SGEMM_DIRECT_ALPHA_BETA gotoblas -> sgemm_direct_alpha_beta #endif #define SGEMM_ONCOPY gotoblas -> sgemm_oncopy diff --git a/interface/gemm.c b/interface/gemm.c index d79282e13..0b29a4dbb 100644 --- a/interface/gemm.c +++ b/interface/gemm.c @@ -436,6 +436,9 @@ void CNAME(enum CBLAS_ORDER order, enum CBLAS_TRANSPOSE TransA, enum CBLAS_TRANS if (beta == 0 && alpha == 1.0 && order == CblasRowMajor && TransA == CblasNoTrans && TransB == CblasNoTrans) { SGEMM_DIRECT(m, n, k, a, lda, b, ldb, c, ldc); return; + }else if (order == CblasRowMajor && TransA == CblasNoTrans && TransB == CblasNoTrans) { + SGEMM_DIRECT_ALPHA_BETA(m, n, k, alpha, a, lda, b, ldb, beta, c, ldc); + return; } #endif #endif diff --git a/kernel/CMakeLists.txt b/kernel/CMakeLists.txt index 9434f114e..2ec3bf77c 100644 --- a/kernel/CMakeLists.txt +++ b/kernel/CMakeLists.txt @@ -222,9 +222,11 @@ function (build_core TARGET_CORE KDIR TSUFFIX KERNEL_DEFINITIONS) GenerateNamedObjects("${KERNELDIR}/${SGEMMDIRECTPERFORMANT}" "" "gemm_direct_performant" false "" "" false SINGLE) elseif (ARM64) set (SGEMMDIRECTKERNEL sgemm_direct_arm64_sme1.c) + set (SGEMMDIRECTKERNEL_ALPHA_BETA sgemm_direct_alpha_beta_arm64_sme1.c) set (SGEMMDIRECTSMEKERNEL sgemm_direct_sme1.S) set (SGEMMDIRECTPREKERNEL sgemm_direct_sme1_preprocess.S) GenerateNamedObjects("${KERNELDIR}/${SGEMMDIRECTKERNEL}" "" "gemm_direct" false "" "" false SINGLE) + GenerateNamedObjects("${KERNELDIR}/${SGEMMDIRECTKERNEL_ALPHA_BETA}" "" "gemm_direct_alpha_beta" false "" "" false SINGLE) if (HAVE_SME) GenerateNamedObjects("${KERNELDIR}/${SGEMMDIRECTSMEKERNEL}" "" "gemm_direct_sme1" false "" "" false SINGLE) GenerateNamedObjects("${KERNELDIR}/${SGEMMDIRECTPREKERNEL}" "" "gemm_direct_sme1_preprocess" false "" "" false SINGLE) diff --git a/kernel/Makefile.L3 b/kernel/Makefile.L3 index 6afb49a77..ebdfe7b06 100644 --- a/kernel/Makefile.L3 +++ b/kernel/Makefile.L3 @@ -105,6 +105,7 @@ ifeq ($(TARGET_CORE), ARMV9SME) HAVE_SME = 1 endif SGEMMDIRECTKERNEL = sgemm_direct_arm64_sme1.c +SGEMMDIRECTKERNEL_ALPHA_BETA = sgemm_direct_alpha_beta_arm64_sme1.c endif endif endif @@ -164,7 +165,8 @@ SKERNELOBJS += \ endif ifeq ($(ARCH), arm64) SKERNELOBJS += \ - sgemm_direct$(TSUFFIX).$(SUFFIX) + sgemm_direct$(TSUFFIX).$(SUFFIX) \ + sgemm_direct_alpha_beta$(TSUFFIX).$(SUFFIX) ifdef HAVE_SME SKERNELOBJS += \ sgemm_direct_sme1$(TSUFFIX).$(SUFFIX) \ @@ -904,6 +906,8 @@ endif ifeq ($(ARCH), arm64) $(KDIR)sgemm_direct$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMMDIRECTKERNEL) $(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX $< -o $@ +$(KDIR)sgemm_direct_alpha_beta$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMMDIRECTKERNEL_ALPHA_BETA) + $(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX $< -o $@ ifdef HAVE_SME $(KDIR)sgemm_direct_sme1$(TSUFFIX).$(SUFFIX) : $(CC) $(CFLAGS) -c $(KERNELDIR)/sgemm_direct_sme1.S -UDOUBLE -UCOMPLEX -o $@ diff --git a/kernel/arm64/sgemm_direct_alpha_beta_arm64_sme1.c b/kernel/arm64/sgemm_direct_alpha_beta_arm64_sme1.c new file mode 100644 index 000000000..d9de3ace3 --- /dev/null +++ b/kernel/arm64/sgemm_direct_alpha_beta_arm64_sme1.c @@ -0,0 +1,199 @@ +/* + Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + SPDX-License-Identifier: BSD-3-Clause-Clear +*/ + +#include "common.h" +#include +#include +#include +#include "sme_abi.h" +#if defined(HAVE_SME) + +#if defined(__ARM_FEATURE_SME) && defined(__clang__) && __clang_major__ >= 16 +#include +#endif + +/* Function prototypes */ +extern void sgemm_direct_sme1_preprocess(uint64_t nbr, uint64_t nbc,\ + const float * restrict a, float * a_mod) __asm__("sgemm_direct_sme1_preprocess"); + +/* Function Definitions */ +static uint64_t sve_cntw() { + uint64_t cnt; + asm volatile( + "rdsvl %[res], #1\n" + "lsr %[res], %[res], #2\n" + : [res] "=r" (cnt) :: + ); + return cnt; +} + +#if defined(__ARM_FEATURE_SME) && defined(__ARM_FEATURE_LOCALLY_STREAMING) && defined(__clang__) && __clang_major__ >= 16 +// Outer product kernel. +// Computes a 2SVL x 2SVL block of C, utilizing all four FP32 tiles of ZA. +__attribute__((always_inline)) inline void +kernel_2x2(const float *A, const float *B, float *C, size_t shared_dim, + size_t ldc, size_t block_rows, size_t block_cols, float alpha, float beta) + __arm_out("za") __arm_streaming { + + const uint64_t svl = svcntw(); + size_t ldb = ldc; + // Predicate set-up + svbool_t pg = svptrue_b32(); + svbool_t pg_a_0 = svwhilelt_b32_u64(0, block_rows); + svbool_t pg_a_1 = svwhilelt_b32_u64(svl, block_rows); + + svbool_t pg_b_0 = svwhilelt_b32_u64(0, block_cols); + svbool_t pg_b_1 = svwhilelt_b32_u64(svl, block_cols); + +#define pg_c_0 pg_b_0 +#define pg_c_1 pg_b_1 + + svzero_za(); + svfloat32_t beta_vec = svdup_f32(beta); + // Load C to ZA + for (size_t i = 0; i < MIN(svl, block_rows); i++) { + svfloat32_t row_c_0 = svld1(pg_c_0, &C[i * ldc]); + row_c_0 = svmul_x(pg, beta_vec, row_c_0); + svwrite_hor_za32_f32_m(/*tile*/0, /*slice*/i, pg_c_0, row_c_0); + + svfloat32_t row_c_1 = svld1(pg_c_1, &C[i * ldc + svl]); + row_c_1 = svmul_x(pg, beta_vec, row_c_1); + svwrite_hor_za32_f32_m(/*tile*/1, /*slice*/i, pg_c_1, row_c_1); + } + for (size_t i = svl; i < block_rows; i++) { + svfloat32_t row_c_0 = svld1(pg_c_0, &C[i * ldc]); + row_c_0 = svmul_x(pg, beta_vec, row_c_0); + svwrite_hor_za32_f32_m(/*tile*/2, /*slice*/i, pg_c_0, row_c_0); + + svfloat32_t row_c_1 = svld1(pg_c_1, &C[i * ldc + svl]); + row_c_1 = svmul_x(pg, beta_vec, row_c_1); + svwrite_hor_za32_f32_m(/*tile*/3, /*slice*/i, pg_c_1, row_c_1); + } + + svfloat32_t alpha_vec = svdup_f32(alpha); + // Iterate through shared dimension (K) + for (size_t k = 0; k < shared_dim; k++) { + // Load column of A + svfloat32_t col_a_0 = svld1(pg_a_0, &A[k * svl]); + col_a_0 = svmul_x(pg, alpha_vec, col_a_0); + svfloat32_t col_a_1 = svld1(pg_a_1, &A[(k + shared_dim) * svl]); + col_a_1 = svmul_x(pg, alpha_vec, col_a_1); + // Load row of B + svfloat32_t row_b_0 = svld1(pg_b_0, &B[k * ldb]); + svfloat32_t row_b_1 = svld1(pg_b_1, &B[k * ldb + svl]); + // Perform outer product + svmopa_za32_m(/*tile*/0, pg, pg, col_a_0, row_b_0); + svmopa_za32_m(/*tile*/1, pg, pg, col_a_0, row_b_1); + svmopa_za32_m(/*tile*/2, pg, pg, col_a_1, row_b_0); + svmopa_za32_m(/*tile*/3, pg, pg, col_a_1, row_b_1); + } + + // Store to C from ZA + for (size_t i = 0; i < MIN(svl, block_rows); i++) { + svst1_hor_za32(/*tile*/0, /*slice*/i, pg_c_0, &C[i * ldc]); + svst1_hor_za32(/*tile*/1, /*slice*/i, pg_c_1, &C[i * ldc + svl]); + } + for (size_t i = svl; i < block_rows; i++) { + svst1_hor_za32(/*tile*/2, /*slice*/i, pg_c_0, &C[i * ldc]); + svst1_hor_za32(/*tile*/3, /*slice*/i, pg_c_1, &C[i * ldc + svl]); + } +} + +__arm_new("za") __arm_locally_streaming +void sgemm_direct_alpha_beta_sme1_2VLx2VL(uint64_t m, uint64_t k, uint64_t n, const float* alpha,\ + const float *ba, const float *restrict bb, const float* beta,\ + float *restrict C) { + + const uint64_t num_rows = m; + const uint64_t num_cols = n; + + const float *restrict a_ptr = ba; + const float *restrict b_ptr = bb; + float *restrict c_ptr = C; + + const uint64_t svl = svcntw(); + const uint64_t ldc = n; + + // Block over rows of C (panels of A) + uint64_t row_idx = 0; + + // 2x2 loop + uint64_t row_batch = 2*svl; + + // Block over row dimension of C + for (; row_idx < num_rows; row_idx += row_batch) { + row_batch = MIN(row_batch, num_rows - row_idx); + uint64_t col_idx = 0; + uint64_t col_batch = 2*svl; + + // Block over column dimension of C + for (; col_idx < num_cols; col_idx += col_batch) { + col_batch = MIN(col_batch, num_cols - col_idx); + + kernel_2x2(&a_ptr[row_idx * k], &b_ptr[col_idx], + &c_ptr[row_idx * ldc + col_idx], k, + ldc, row_batch, col_batch, *alpha, *beta); + } + } + return; +} + +#else +void sgemm_direct_alpha_beta_sme1_2VLx2VL(uint64_t m, uint64_t k, uint64_t n, const float* alpha,\ + const float *ba, const float *restrict bb, const float* beta,\ + float *restrict C){} +#endif + +/*void sgemm_kernel_direct (BLASLONG M, BLASLONG N, BLASLONG K,\ + float * __restrict A, BLASLONG strideA, float * __restrict B,\ + BLASLONG strideB , float * __restrict R, BLASLONG strideR) +*/ +void CNAME (BLASLONG M, BLASLONG N, BLASLONG K, float alpha, float * __restrict A,\ + BLASLONG strideA, float * __restrict B, BLASLONG strideB ,\ + float beta, float * __restrict R, BLASLONG strideR){ + + uint64_t m_mod, vl_elms; + + vl_elms = sve_cntw(); + + m_mod = ceil((double)M/(double)vl_elms) * vl_elms; + + float *A_mod = (float *) malloc(m_mod*K*sizeof(float)); + + /* Prevent compiler optimization by reading from memory instead + * of reading directly from vector (z) registers. + * */ + asm volatile("" : : :"p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", + "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", + "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", + "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", + "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", + "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"); + + /* Pre-process the left matrix to make it suitable for + matrix sum of outer-product calculation + */ + sgemm_direct_sme1_preprocess(M, K, A, A_mod); + + asm volatile("" : : :"p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", + "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", + "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", + "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", + "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", + "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"); + + /* Calculate C = alpha*A*B + beta*C */ + sgemm_direct_alpha_beta_sme1_2VLx2VL(M, K, N, &alpha, A_mod, B, &beta, R); + + free(A_mod); +} + +#else + +void CNAME (BLASLONG M, BLASLONG N, BLASLONG K, float alpha, float * __restrict A,\ + BLASLONG strideA, float * __restrict B, BLASLONG strideB ,\ + float beta, float * __restrict R, BLASLONG strideR){} + +#endif diff --git a/kernel/arm64/sme_abi.h b/kernel/arm64/sme_abi.h new file mode 100644 index 000000000..07bba4895 --- /dev/null +++ b/kernel/arm64/sme_abi.h @@ -0,0 +1,46 @@ +/*************************************************************************** + * Copyright (c) 2024, The OpenBLAS Project + * All rights reserved. + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of the OpenBLAS project nor the names of + * its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE + * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF + * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * *****************************************************************************/ + +#pragma once + +#include + +/** + * * These are SME ABI routines for saving & restoring SME state. + * * They are typically provided by a compiler runtime library such + * * as libgcc or compiler-rt, but support for these routines is not + * * yet available on all platforms. + * * + * * Define these as aborting stubs so that we loudly fail on nested + * * usage of SME state. + * * + * * These are defined as weak symbols so that a compiler runtime can + * * override them if supported. + * */ +__attribute__((weak)) void __arm_tpidr2_save() { abort(); } +__attribute__((weak)) void __arm_tpidr2_restore() { abort(); } + diff --git a/kernel/setparam-ref.c b/kernel/setparam-ref.c index 20a1df261..d807071a4 100644 --- a/kernel/setparam-ref.c +++ b/kernel/setparam-ref.c @@ -198,6 +198,7 @@ gotoblas_t TABLE_NAME = { #endif #ifdef ARCH_ARM64 sgemm_directTS, + sgemm_direct_alpha_betaTS, #endif sgemm_kernelTS, sgemm_betaTS,