Merge branch 'OpenMathLib:develop' into gemmt_tests

.github/workflows/arm64_graviton.yml

@@ -88,13 +88,14 @@ jobs:
         run: |
           case "${{ matrix.build }}" in
             "make")
-              make -j$(nproc) DYNAMIC_ARCH=1 USE_OPENMP=0 FC="ccache ${{ matrix.fortran }}"
+              make -j$(nproc) DYNAMIC_ARCH=1 BUILD_BFLOAT16=1 USE_OPENMP=0 FC="ccache ${{ matrix.fortran }}"
               ;;
             "cmake")
               mkdir build && cd build
               cmake -DDYNAMIC_ARCH=1 \
                     -DNOFORTRAN=0 \
                     -DBUILD_WITHOUT_LAPACK=0 \
+                    -DBUILD_BFLOAT16=1 \
                     -DCMAKE_VERBOSE_MAKEFILE=ON \
                     -DCMAKE_BUILD_TYPE=Release \
                     -DCMAKE_Fortran_COMPILER=${{ matrix.fortran }} \

.github/workflows/windows_arm64.yml

@@ -0,0 +1,82 @@
+name: Windows ARM64 CI
+
+on:
+  push:
+    branches:
+      - develop
+      - release-**
+  pull_request:
+    branches:
+      - develop
+      - release-**
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read # to fetch code (actions/checkout)
+
+jobs:
+  build:
+    if: "github.repository == 'OpenMathLib/OpenBLAS'"
+    runs-on: windows-11-arm
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Install LLVM for Win-ARM64
+        shell: pwsh
+        run: |
+          Invoke-WebRequest https://github.com/llvm/llvm-project/releases/download/llvmorg-20.1.6/LLVM-20.1.6-woa64.exe -UseBasicParsing -OutFile LLVM-woa64.exe
+          Start-Process -FilePath ".\LLVM-woa64.exe" -ArgumentList "/S" -Wait
+          echo "C:\Program Files\LLVM\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+
+      - name: Install CMake and Ninja for Win-ARM64
+        shell: pwsh
+        run: |
+          Invoke-WebRequest https://github.com/Kitware/CMake/releases/download/v3.29.4/cmake-3.29.4-windows-arm64.msi -OutFile cmake-arm64.msi
+          Start-Process msiexec.exe -ArgumentList "/i cmake-arm64.msi /quiet /norestart" -Wait
+          echo "C:\Program Files\CMake\bin" >> $env:GITHUB_PATH
+          
+          Invoke-WebRequest https://github.com/ninja-build/ninja/releases/download/v1.13.1/ninja-winarm64.zip -OutFile ninja-winarm64.zip
+          Expand-Archive ninja-winarm64.zip -DestinationPath ninja
+          Copy-Item ninja\ninja.exe -Destination "C:\Windows\System32"
+
+      - name: Configure OpenBLAS
+        shell: cmd
+        run: |
+          CALL "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsarm64.bat"
+          mkdir build
+          cd build
+          cmake .. -G Ninja ^
+            -DCMAKE_BUILD_TYPE=Release ^
+            -DTARGET=ARMV8 ^
+            -DBINARY=64 ^
+            -DCMAKE_C_COMPILER=clang-cl ^
+            -DCMAKE_Fortran_COMPILER=flang-new ^
+            -DBUILD_SHARED_LIBS=ON ^
+            -DCMAKE_SYSTEM_PROCESSOR=arm64 ^
+            -DCMAKE_SYSTEM_NAME=Windows ^
+            -DCMAKE_INSTALL_PREFIX=C:/opt
+
+      - name: Build OpenBLAS
+        shell: cmd
+        run: |
+          cd build
+          ninja -j16
+
+      - name: Install OpenBLAS
+        shell: cmd
+        run: |
+          cd build
+          cmake --install .
+
+      - name: Run ctests 
+        shell: pwsh
+        run: |
+          $env:PATH = "C:\opt\bin;$env:PATH"
+          cd build
+          ctest
+
+

.gitignore

@@ -13,8 +13,8 @@ lapack-3.4.1.tgz
 lapack-3.4.2
 lapack-3.4.2.tgz
 lapack-netlib/make.inc
-lapack-netlib/lapacke/include/lapacke_mangling.h
 lapack-netlib/SRC/la_constants.mod
+lapack-netlib/SRC/la_xisnan.mod
 lapack-netlib/TESTING/testing_results.txt
 lapack-netlib/INSTALL/test*
 lapack-netlib/TESTING/xeigtstc
@@ -81,7 +81,10 @@ test/ZBLAT2.SUMM
 test/ZBLAT3.SUMM
 test/ZBLAT3_3M.SUMM
 test/SHBLAT3.SUMM
+test/SBBLAT2.SUMM
 test/SBBLAT3.SUMM
+test/BBLAT2.SUMM
+test/BBLAT3.SUMM
 test/cblat1
 test/cblat2
 test/cblat3
@@ -96,6 +99,9 @@ test/sblat3
 test/sblat3_3m
 test/test_shgemm
 test/test_sbgemm
+test/test_sbgemv
+test/test_bgemm
+test/test_bgemv
 test/zblat1
 test/zblat2
 test/zblat3

CONTRIBUTORS.md

@@ -251,6 +251,18 @@ In chronological order:
 * Ye Tao <ye.tao@arm.com>
   * [2025-02-03] Optimize SBGEMM kernel on NEOVERSEV1
   * [2025-02-27] Add sbgemv_n_neon kernel
+  * [2025-05-17] Impl prototype of BGEMM inferface
 
 * Abhishek Kumar <https://github.com/abhishek-iitmadras>
-  * [2025-04-22] Optimise dot kernel for NEOVERSE V1
+  * [2025-04-22] Optimise dot kernel for NEOVERSE V1
+
+* Sharif Inamdar <sharif.inamdar@arm.com>
+  * [2025-06-05] Optimize gemv_n_sve_v1x3 kernel
+
+* Guoyuan Li <https://github.com/guoyuanplct>
+  * [2025-04-11] Optimise gemv kernel for RISCV64_ZVL256B
+  * [2025-05-01] Optimise zgemv kernel for RISCV64_ZVL256B
+  * [2025-05-17] Optimise omatcopy/zomatcopy kernel for RISCV64_ZVL256B
+  * [2025-05-29] Optimise axpby kernel for RISCV64_ZVL256B
+  * [2025-06-05] Optimise hbmv kernel for RISCV64_ZVL256B
+

Makefile.system

@@ -276,14 +276,14 @@ SMALL_MATRIX_OPT = 1
 endif
 ifeq ($(ARCH), arm64)
 GEMM_GEMV_FORWARD = 1
-GEMM_GEMV_FORWARD_BF16 = 1
+SBGEMM_GEMV_FORWARD = 1
 endif
 ifeq ($(ARCH), riscv)
 GEMM_GEMV_FORWARD = 1
 endif
 ifeq ($(ARCH), power)
 GEMM_GEMV_FORWARD = 1
-GEMM_GEMV_FORWARD_BF16 = 1
+SBGEMM_GEMV_FORWARD = 1
 endif
 
 ifeq ($(SMALL_MATRIX_OPT), 1)
@@ -293,8 +293,8 @@ ifneq ($(ONLY_CBLAS), 1)
 ifeq ($(GEMM_GEMV_FORWARD), 1)
 CCOMMON_OPT += -DGEMM_GEMV_FORWARD
 endif
-ifeq ($(GEMM_GEMV_FORWARD_BF16), 1)
-CCOMMON_OPT += -DGEMM_GEMV_FORWARD_BF16
+ifeq ($(SBGEMM_GEMV_FORWARD), 1)
+CCOMMON_OPT += -DSBGEMM_GEMV_FORWARD
 endif
 endif
 
@@ -1905,6 +1905,8 @@ export BUILD_HFLOAT16
 export NO_LSX
 export NO_LASX
 
+export BGEMM_UNROLL_M
+export BGEMM_UNROLL_N
 export SBGEMM_UNROLL_M
 export SBGEMM_UNROLL_N
 export SHGEMM_UNROLL_M

Makefile.tail

@@ -1,3 +1,32 @@
+###############################################################################
+# Copyright (c) 2025, The OpenBLAS Project
+# All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in
+#    the documentation and/or other materials provided with the
+#    distribution.
+# 3. Neither the name of the OpenBLAS project nor the names of
+#    its contributors may be used to endorse or promote products
+#    derived from this software without specific prior written permission.
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+###############################################################################
+
+BBLASOBJS_P = $(BBLASOBJS:.$(SUFFIX)=.$(PSUFFIX))
 SBBLASOBJS_P = $(SBBLASOBJS:.$(SUFFIX)=.$(PSUFFIX))
 SHBLASPBJS_P = $(SHBLASOBJS:.$(SUFFIX)=.$(PSUFFIX))
 SBLASOBJS_P = $(SBLASOBJS:.$(SUFFIX)=.$(PSUFFIX))
@@ -12,8 +41,8 @@ COMMONOBJS_P = $(COMMONOBJS:.$(SUFFIX)=.$(PSUFFIX))
 
 HPLOBJS_P   = $(HPLOBJS:.$(SUFFIX)=.$(PSUFFIX))
 
-BLASOBJS    = $(SHBLASOBJS) $(SBEXTOBJS) $(SBBLASOBJS)  $(SBLASOBJS)   $(DBLASOBJS)   $(CBLASOBJS)   $(ZBLASOBJS) $(CBAUXOBJS)
-BLASOBJS_P  = $(SHBLASPBJS_P) $(SBEXTOBJS_P) $(SBBLASOBJS_P) $(SBLASOBJS_P) $(DBLASOBJS_P) $(CBLASOBJS_P) $(ZBLASOBJS_P) $(CBAUXOBJS_P)
+BLASOBJS    = $(SHBLASOBJS) $(BBLASOBJS) $(SBEXTOBJS) $(SBBLASOBJS)  $(SBLASOBJS)   $(DBLASOBJS)   $(CBLASOBJS)   $(ZBLASOBJS) $(CBAUXOBJS)
+BLASOBJS_P  = $(SHBLASPBJS_P) $(BBLASOBJS_P) $(SBEXTOBJS_P) $(SBBLASOBJS_P) $(SBLASOBJS_P) $(DBLASOBJS_P) $(CBLASOBJS_P) $(ZBLASOBJS_P) $(CBAUXOBJS_P)
 
 ifdef EXPRECISION
 BLASOBJS   += $(QBLASOBJS)   $(XBLASOBJS)
@@ -26,6 +55,7 @@ BLASOBJS_P += $(QBLASOBJS_P) $(XBLASOBJS_P)
 endif
 
 $(SHBLASOBJS) $(SHBLASOBJS_P) : override CFLAGS += -DHFLOAT16 -UDOUBLE  -UCOMPLEX
+$(BBLASOBJS) $(BBLASOBJS_P) : override CFLAGS += -DBFLOAT16 -DBGEMM -UDOUBLE  -UCOMPLEX
 $(SBBLASOBJS) $(SBBLASOBJS_P) : override CFLAGS += -DBFLOAT16 -UDOUBLE  -UCOMPLEX
 $(SBLASOBJS) $(SBLASOBJS_P) : override CFLAGS += -UDOUBLE  -UCOMPLEX
 $(DBLASOBJS) $(DBLASOBJS_P) : override CFLAGS += -DDOUBLE  -UCOMPLEX
@@ -36,6 +66,7 @@ $(XBLASOBJS) $(XBLASOBJS_P) : override CFLAGS += -DXDOUBLE -DCOMPLEX
 $(SBEXTOBJS) $(SBEXTOBJS_P) : override CFLAGS += -DBFLOAT16 -UDOUBLE  -UCOMPLEX
 
 $(SHBLASOBJS_P) : override CFLAGS += -DPROFILE $(COMMON_PROF)
+$(BBLASOBJS_P) : override CFLAGS += -DPROFILE $(COMMON_PROF)
 $(SBBLASOBJS_P) : override CFLAGS += -DPROFILE $(COMMON_PROF)
 $(SBLASOBJS_P) : override CFLAGS += -DPROFILE $(COMMON_PROF)
 $(DBLASOBJS_P) : override CFLAGS += -DPROFILE $(COMMON_PROF)

README.md

@@ -29,6 +29,7 @@ For a general introduction to the BLAS routines, please refer to the extensive d
 We provide official binary packages for the following platform:
 
   * Windows x86/x86_64
+  * Windows arm64 (woa)
 
 You can download them from [file hosting on sourceforge.net](https://sourceforge.net/projects/openblas/files/) or from the [Releases section of the GitHub project page](https://github.com/OpenMathLib/OpenBLAS/releases).
 

TargetList.txt

@@ -52,6 +52,7 @@ POWER7
 POWER8
 POWER9
 POWER10
+POWER11
 PPCG4
 PPC970
 PPC970MP

benchmark/Makefile

@@ -1,3 +1,31 @@
+###############################################################################
+# Copyright (c) 2025, The OpenBLAS Project
+# All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in
+#    the documentation and/or other materials provided with the
+#    distribution.
+# 3. Neither the name of the OpenBLAS project nor the names of
+#    its contributors may be used to endorse or promote products
+#    derived from this software without specific prior written permission.
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+###############################################################################
+
 TOPDIR	= ..
 include $(TOPDIR)/Makefile.system
 
@@ -56,7 +84,7 @@ GOTO_LAPACK_TARGETS=
 endif
 
 ifeq ($(BUILD_BFLOAT16),1)
-GOTO_BFLOAT_TARGETS=sbgemm.goto
+GOTO_BFLOAT_TARGETS=bgemm.goto sbgemm.goto
 else
 GOTO_BFLOAT_TARGETS=
 endif
@@ -635,6 +663,8 @@ zcholesky.essl : zcholesky.$(SUFFIX)
 
 ##################################### Sgemm ####################################################
 ifeq ($(BUILD_BFLOAT16),1)
+bgemm.goto : bgemm.$(SUFFIX) ../$(LIBNAME)
+	$(CC) $(CFLAGS) -o $(@F) $^ $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB) -lm
 sbgemm.goto : sbgemm.$(SUFFIX) ../$(LIBNAME)
 	$(CC) $(CFLAGS) -o $(@F) $^ $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB) -lm
 endif
@@ -2970,6 +3000,8 @@ zcholesky.$(SUFFIX) : cholesky.c
 	$(CC) $(CFLAGS) -c -DCOMPLEX -DDOUBLE -o $(@F) $^
 
 ifeq ($(BUILD_BFLOAT16),1)
+bgemm.$(SUFFIX) : gemm.c
+	$(CC) $(CFLAGS) -c -DBFLOAT16 -DBGEMM -UCOMPLEX -UDOUBLE -o $(@F) $^
 sbgemm.$(SUFFIX) : gemm.c
 	$(CC) $(CFLAGS) -c -DBFLOAT16 -UCOMPLEX -UDOUBLE -o $(@F) $^
 endif

benchmark/gemm.c

@@ -33,6 +33,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #ifdef DOUBLE
 #define GEMM   BLASFUNC(dgemm)
+#elif defined(BFLOAT16) && defined(BGEMM)
+#define GEMM   BLASFUNC(bgemm)
 #elif defined(BFLOAT16)
 #define GEMM   BLASFUNC(sbgemm)
 #undef IFLOAT
@@ -60,8 +62,18 @@ int main(int argc, char *argv[]){
 
   IFLOAT *a, *b;
   FLOAT *c;
+#ifdef BGEMM
+  blasint one=1;
+  blasint two=2;
+  float alpha_in[] = {1.0, 0.0};
+  float beta_in[] = {0.0, 0.0};
+  FLOAT alpha[2], beta[2];
+  sbstobf16_(&two, alpha_in, &one, alpha, &one);
+  sbstobf16_(&two, beta_in, &one, beta, &one);
+#else
   FLOAT alpha[] = {1.0, 0.0};
   FLOAT beta [] = {0.0, 0.0};
+#endif
   char transa = 'N';
   char transb = 'N';
   blasint m, n, k, i, j, lda, ldb, ldc;

cblas.h

@@ -1,3 +1,31 @@
+/***************************************************************************
+ * Copyright (c) 2025, The OpenBLAS Project
+ * All rights reserved.
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * 3. Neither the name of the OpenBLAS project nor the names of
+ * its contributors may be used to endorse or promote products
+ * derived from this software without specific prior written permission.
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ * *****************************************************************************/
+
 #ifndef CBLAS_H
 #define CBLAS_H
 
@@ -445,10 +473,13 @@ void   cblas_sbdtobf16(OPENBLAS_CONST blasint n, OPENBLAS_CONST double *in, OPEN
 void   cblas_sbf16tos(OPENBLAS_CONST blasint n, OPENBLAS_CONST bfloat16 *in, OPENBLAS_CONST blasint incin, float  *out, OPENBLAS_CONST blasint incout);
 /* convert BFLOAT16 array to double array */
 void   cblas_dbf16tod(OPENBLAS_CONST blasint n, OPENBLAS_CONST bfloat16 *in, OPENBLAS_CONST blasint incin, double *out, OPENBLAS_CONST blasint incout);
+void   cblas_bgemv(OPENBLAS_CONST enum CBLAS_ORDER order,  OPENBLAS_CONST enum CBLAS_TRANSPOSE trans,  OPENBLAS_CONST blasint m, OPENBLAS_CONST blasint n, OPENBLAS_CONST bfloat16 alpha, OPENBLAS_CONST bfloat16 *a, OPENBLAS_CONST blasint lda, OPENBLAS_CONST bfloat16 *x, OPENBLAS_CONST blasint incx, OPENBLAS_CONST bfloat16 beta, bfloat16 *y, OPENBLAS_CONST blasint incy);
 /* dot production of BFLOAT16 input arrays, and output as float */
 float  cblas_sbdot(OPENBLAS_CONST blasint n, OPENBLAS_CONST bfloat16 *x, OPENBLAS_CONST blasint incx, OPENBLAS_CONST bfloat16 *y, OPENBLAS_CONST blasint incy);
 void   cblas_sbgemv(OPENBLAS_CONST enum CBLAS_ORDER order,  OPENBLAS_CONST enum CBLAS_TRANSPOSE trans,  OPENBLAS_CONST blasint m, OPENBLAS_CONST blasint n, OPENBLAS_CONST float alpha, OPENBLAS_CONST bfloat16 *a, OPENBLAS_CONST blasint lda, OPENBLAS_CONST bfloat16 *x, OPENBLAS_CONST blasint incx, OPENBLAS_CONST float beta, float *y, OPENBLAS_CONST blasint incy);
 
+void cblas_bgemm(OPENBLAS_CONST enum CBLAS_ORDER Order, OPENBLAS_CONST enum CBLAS_TRANSPOSE TransA, OPENBLAS_CONST enum CBLAS_TRANSPOSE TransB, OPENBLAS_CONST blasint M, OPENBLAS_CONST blasint N, OPENBLAS_CONST blasint K,
+		    OPENBLAS_CONST bfloat16 alpha, OPENBLAS_CONST bfloat16 *A, OPENBLAS_CONST blasint lda, OPENBLAS_CONST bfloat16 *B, OPENBLAS_CONST blasint ldb, OPENBLAS_CONST bfloat16 beta, bfloat16 *C, OPENBLAS_CONST blasint ldc);
 void   cblas_sbgemm(OPENBLAS_CONST enum CBLAS_ORDER Order, OPENBLAS_CONST enum CBLAS_TRANSPOSE TransA, OPENBLAS_CONST enum CBLAS_TRANSPOSE TransB, OPENBLAS_CONST blasint M, OPENBLAS_CONST blasint N, OPENBLAS_CONST blasint K,
 		    OPENBLAS_CONST float alpha, OPENBLAS_CONST bfloat16 *A, OPENBLAS_CONST blasint lda, OPENBLAS_CONST bfloat16 *B, OPENBLAS_CONST blasint ldb, OPENBLAS_CONST float beta, float *C, OPENBLAS_CONST blasint ldc);
 void cblas_sbgemm_batch(OPENBLAS_CONST enum CBLAS_ORDER Order, OPENBLAS_CONST enum CBLAS_TRANSPOSE * TransA_array, OPENBLAS_CONST enum CBLAS_TRANSPOSE * TransB_array, OPENBLAS_CONST blasint * M_array, OPENBLAS_CONST blasint * N_array, OPENBLAS_CONST blasint * K_array,

cmake/cc.cmake

@@ -211,14 +211,14 @@ endif ()
 if (${CORE} STREQUAL NEOVERSEV1)
   if (NOT DYNAMIC_ARCH)
     if (${CMAKE_C_COMPILER_ID} STREQUAL "PGI" AND NOT NO_SVE)
-	set (CCOMMON_OPT  "${CCOMMON_OPT} -Msve_intrinsics -march=armv8.4-a+sve -mtune=neoverse-v1")
+	set (CCOMMON_OPT  "${CCOMMON_OPT} -Msve_intrinsics -march=armv8.4-a+sve+bf16 -mtune=neoverse-v1")
     elseif (${CMAKE_C_COMPILER_ID} STREQUAL "NVC" AND NOT NO_SVE)
 	set (CCOMMON_OPT  "${CCOMMON_OPT} -tp=neoverse-v1")
     else ()
       if (${GCC_VERSION} VERSION_GREATER 10.4 OR ${GCC_VERSION} VERSION_EQUAL 10.4)
-        set (CCOMMON_OPT  "${CCOMMON_OPT} -march=armv8.4-a+sve -mtune=neoverse-v1")
+        set (CCOMMON_OPT  "${CCOMMON_OPT} -march=armv8.4-a+sve+bf16 -mtune=neoverse-v1")
       else ()
-        set (CCOMMON_OPT "${CCOMMON_OPT} -march=armv8.2-a+sve")
+        set (CCOMMON_OPT "${CCOMMON_OPT} -march=armv8.2-a+sve+bf16")
       endif()
     endif()
   endif ()

cmake/kernel.cmake

@@ -1,3 +1,31 @@
+###############################################################################
+# Copyright (c) 2025, The OpenBLAS Project
+# All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in
+#    the documentation and/or other materials provided with the
+#    distribution.
+# 3. Neither the name of the OpenBLAS project nor the names of
+#    its contributors may be used to endorse or promote products
+#    derived from this software without specific prior written permission.
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+###############################################################################
+
 # helper functions for the kernel CMakeLists.txt
 
 function(SetFallback KERNEL SOURCE_PATH)
@@ -82,6 +110,7 @@ macro(SetDefaultL1)
   SetFallback(SROTMKERNEL rotm.S)
   SetFallback(DROTMKERNEL rotm.S)
   SetFallback(QROTMKERNEL rotm.S)
+  SetFallback(BSCALKERNEL ../generic/scal.c)
   SetFallback(SSCALKERNEL scal.S)
   SetFallback(DSCALKERNEL scal.S)
   SetFallback(CSCALKERNEL zscal.S)
@@ -141,6 +170,8 @@ if (BUILD_BFLOAT16)
   SetFallback(SHSWAPKERNEL ../arm/swap.c)
   SetFallback(TOBF16KERNEL ../x86_64/tobf16.c)
   SetFallback(BF16TOKERNEL ../x86_64/bf16to.c)
+  SetFallback(BGEMVNKERNEL ../generic/gemv_n.c)
+  SetFallback(BGEMVTKERNEL ../generic/gemv_t.c)
   SetFallback(SBGEMVNKERNEL ../x86_64/sbgemv_n.c)
   SetFallback(SBGEMVTKERNEL ../x86_64/sbgemv_t.c)
 endif ()
@@ -193,6 +224,8 @@ macro(SetDefaultL2)
   SetFallback(XHEMV_V_KERNEL ../generic/zhemv_k.c)
   SetFallback(XHEMV_M_KERNEL ../generic/zhemv_k.c)
 if (BUILD_BFLOAT16)
+  SetFallback(BGEMVNKERNEL ../generic/gemv_n.c)
+  SetFallback(BGEMVTKERNEL ../generic/gemv_t.c)
   SetFallback(SBGEMVNKERNEL ../x86_64/sbgemv_n.c)
   SetFallback(SBGEMVTKERNEL ../x86_64/sbgemv_t.c)
   SetFallback(SHGERKERNEL ../generic/ger.c)
@@ -206,6 +239,16 @@ macro(SetDefaultL3)
   SetFallback(ZGEADD_KERNEL ../generic/zgeadd.c)
 if (BUILD_BFLOAT16)
   SetFallback(SHGEADD_KERNEL ../generic/geadd.c)
+  SetFallback(BGEMMKERNEL ../generic/gemmkernel_2x2.c)
+  SetFallback(BGEMM_BETA  ../generic/gemm_beta.c)
+  SetFallback(BGEMMINCOPY ../generic/gemm_ncopy_2.c)
+  SetFallback(BGEMMITCOPY ../generic/gemm_tcopy_2.c)
+  SetFallback(BGEMMONCOPY ../generic/gemm_ncopy_2.c)
+  SetFallback(BGEMMOTCOPY ../generic/gemm_tcopy_2.c)
+  SetFallback(BGEMMINCOPYOBJ bgemm_incopy.o)
+  SetFallback(BGEMMITCOPYOBJ bgemm_itcopy.o)
+  SetFallback(BGEMMONCOPYOBJ bgemm_oncopy.o)
+  SetFallback(BGEMMOTCOPYOBJ bgemm_otcopy.o)
   SetFallback(SBGEMMKERNEL ../generic/gemmkernel_2x2.c)
   SetFallback(SBGEMM_BETA  ../generic/gemm_beta.c)
   SetFallback(SBGEMMINCOPY ../generic/gemm_ncopy_2.c)

cmake/prebuild.cmake

@@ -1,3 +1,31 @@
+###############################################################################
+# Copyright (c) 2025, The OpenBLAS Project
+# All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in
+#    the documentation and/or other materials provided with the
+#    distribution.
+# 3. Neither the name of the OpenBLAS project nor the names of
+#    its contributors may be used to endorse or promote products
+#    derived from this software without specific prior written permission.
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+###############################################################################
+
 ##
 ## Author: Hank Anderson <hank@statease.com>
 ## Description: Ported from OpenBLAS/Makefile.prebuild
@@ -131,6 +159,8 @@ if (DEFINED CORE AND CMAKE_CROSSCOMPILING AND NOT (${HOST_OS} STREQUAL "WINDOWSS
       set(HAVE_SSE2 1)
       set(HAVE_SSE3 1)
       set(HAVE_SSSE3 1)
+      set(BGEMM_UNROLL_M 8)
+      set(BGEMM_UNROLL_N 4)
       set(SBGEMM_UNROLL_M 8)
       set(SBGEMM_UNROLL_N 4)
       set(SGEMM_UNROLL_M 8)

cmake/system.cmake

@@ -291,10 +291,10 @@ if (DEFINED TARGET)
 
 if (${TARGET} STREQUAL NEOVERSEV1)
     if (${CMAKE_C_COMPILER_ID} STREQUAL "PGI" AND NOT NO_SVE)
-	set (KERNEL_DEFINITIONS  "${KERNEL_DEFINITIONS} -Msve_intrinsics -march=armv8.4-a+sve -mtune=neoverse-v1")
+	set (KERNEL_DEFINITIONS  "${KERNEL_DEFINITIONS} -Msve_intrinsics -march=armv8.4-a+sve+bf16 -mtune=neoverse-v1")
     else ()
     if (CMAKE_C_COMPILER_VERSION VERSION_GREATER 10.4 OR CMAKE_C_COMPILER_VERSION VERSION_EQUAL 10.4)
-      set (KERNEL_DEFINITIONS  "${KERNEL_DEFINITIONS} -march=armv8.4-a+sve -mtune=neoverse-v1")
+      set (KERNEL_DEFINITIONS  "${KERNEL_DEFINITIONS} -march=armv8.4-a+sve+bf16 -mtune=neoverse-v1")
     else ()
 	    message(FATAL_ERROR "Compiler ${CMAKE_C_COMPILER} ${CMAKE_C_COMPILER_VERSION} does not support Neoverse V1.")
     endif()
@@ -425,8 +425,8 @@ endif ()
 if (GEMM_GEMV_FORWARD AND NOT ONLY_CBLAS)
   set(CCOMMON_OPT "${CCOMMON_OPT} -DGEMM_GEMV_FORWARD")
 endif ()
-if (GEMM_GEMV_FORWARD_BF16 AND NOT ONLY_CBLAS)
-    set(CCOMMON_OPT "${CCOMMON_OPT} -DGEMM_GEMV_FORWARD_BF16")
+if (SBGEMM_GEMV_FORWARD AND NOT ONLY_CBLAS)
+    set(CCOMMON_OPT "${CCOMMON_OPT} -DSBGEMM_GEMV_FORWARD")
 endif ()
 if (SMALL_MATRIX_OPT)
   set(CCOMMON_OPT "${CCOMMON_OPT} -DSMALL_MATRIX_OPT")

cmake/utils.cmake

@@ -1,3 +1,31 @@
+###############################################################################
+# Copyright (c) 2025, The OpenBLAS Project
+# All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in
+#    the documentation and/or other materials provided with the
+#    distribution.
+# 3. Neither the name of the OpenBLAS project nor the names of
+#    its contributors may be used to endorse or promote products
+#    derived from this software without specific prior written permission.
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+###############################################################################
+
 # Functions to help with the OpenBLAS build
 
 # Reads string from getarch into CMake vars. Format of getarch vars is VARNAME=VALUE
@@ -347,7 +375,7 @@ function(GenerateNamedObjects sources_in)
       if (NOT no_float_type)
         string(SUBSTRING ${float_type} 0 1 float_char)
         string(TOLOWER ${float_char} float_char)
-	if (${float_type} STREQUAL "BFLOAT16")
+	if (${float_type} STREQUAL "BFLOAT16" AND NOT "${defines_in}" MATCHES "BGEM")
 		set (float_char "sb")
 	endif ()
       endif ()

common.h

@@ -1,5 +1,6 @@
 /*********************************************************************/
 /* Copyright 2009, 2010 The University of Texas at Austin.           */
+/* Copyright 2025 The OpenBLAS Project.                              */
 /* All rights reserved.                                              */
 /*                                                                   */
 /* Redistribution and use in source and binary forms, with or        */
@@ -317,7 +318,11 @@ typedef int blasint;
 #elif defined(BFLOAT16)
 #define IFLOAT	bfloat16
 #define XFLOAT IFLOAT
-#define FLOAT	float
+#ifdef BGEMM
+#define FLOAT	bfloat16
+#else
+#define FLOAT float
+#endif
 #define SIZE   2
 #define BASE_SHIFT 1
 #define ZBASE_SHIFT 2

common_b.h

@@ -0,0 +1,100 @@
+/***************************************************************************
+ * Copyright (c) 2025, The OpenBLAS Project
+ * All rights reserved.
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * 3. Neither the name of the OpenBLAS project nor the names of
+ * its contributors may be used to endorse or promote products
+ * derived from this software without specific prior written permission.
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ * *****************************************************************************/
+
+#ifndef COMMON_B_H
+#define COMMON_B_H
+
+#ifndef DYNAMIC_ARCH
+#define BGEMV_N_K          bgemv_n
+#define BGEMV_T_K          bgemv_t
+
+#define	BSCAL_K			bscal_k
+
+#define	BGEMM_ONCOPY		bgemm_oncopy
+#define	BGEMM_OTCOPY		bgemm_otcopy
+
+#if BGEMM_DEFAULT_UNROLL_M == BGEMM_DEFAULT_UNROLL_N
+#define	BGEMM_INCOPY		bgemm_oncopy
+#define	BGEMM_ITCOPY		bgemm_otcopy
+#else
+#define	BGEMM_INCOPY		bgemm_incopy
+#define	BGEMM_ITCOPY		bgemm_itcopy
+#endif
+
+#define BGEMM_BETA bgemm_beta
+#define BGEMM_KERNEL bgemm_kernel
+
+#else
+#define BGEMV_N_K          gotoblas->bgemv_n
+#define BGEMV_T_K          gotoblas->bgemv_t
+
+#define	BSCAL_K            gotoblas->bscal_k
+
+#define BGEMM_ONCOPY gotoblas->bgemm_oncopy
+#define BGEMM_OTCOPY gotoblas->bgemm_otcopy
+#define BGEMM_INCOPY gotoblas->bgemm_incopy
+#define BGEMM_ITCOPY gotoblas->bgemm_itcopy
+#define BGEMM_BETA gotoblas->bgemm_beta
+#define BGEMM_KERNEL gotoblas->bgemm_kernel
+
+#endif
+
+#define BGEMM_NN bgemm_nn
+#define BGEMM_CN bgemm_tn
+#define BGEMM_TN bgemm_tn
+#define BGEMM_NC bgemm_nt
+#define BGEMM_NT bgemm_nt
+#define BGEMM_CC bgemm_tt
+#define BGEMM_CT bgemm_tt
+#define BGEMM_TC bgemm_tt
+#define BGEMM_TT bgemm_tt
+#define BGEMM_NR bgemm_nn
+#define BGEMM_TR bgemm_tn
+#define BGEMM_CR bgemm_tn
+#define BGEMM_RN bgemm_nn
+#define BGEMM_RT bgemm_nt
+#define BGEMM_RC bgemm_nt
+#define BGEMM_RR bgemm_nn
+
+#define BGEMM_THREAD_NN bgemm_thread_nn
+#define BGEMM_THREAD_CN bgemm_thread_tn
+#define BGEMM_THREAD_TN bgemm_thread_tn
+#define BGEMM_THREAD_NC bgemm_thread_nt
+#define BGEMM_THREAD_NT bgemm_thread_nt
+#define BGEMM_THREAD_CC bgemm_thread_tt
+#define BGEMM_THREAD_CT bgemm_thread_tt
+#define BGEMM_THREAD_TC bgemm_thread_tt
+#define BGEMM_THREAD_TT bgemm_thread_tt
+#define BGEMM_THREAD_NR bgemm_thread_nn
+#define BGEMM_THREAD_TR bgemm_thread_tn
+#define BGEMM_THREAD_CR bgemm_thread_tn
+#define BGEMM_THREAD_RN bgemm_thread_nn
+#define BGEMM_THREAD_RT bgemm_thread_nt
+#define BGEMM_THREAD_RC bgemm_thread_nt
+#define BGEMM_THREAD_RR bgemm_thread_nn
+#endif

common_interface.h

@@ -1,5 +1,6 @@
 /*********************************************************************/
 /* Copyright 2009, 2010 The University of Texas at Austin.           */
+/* Copyright 2025 The OpenBLAS Project.                              */
 /* All rights reserved.                                              */
 /*                                                                   */
 /* Redistribution and use in source and binary forms, with or        */
@@ -59,6 +60,7 @@ double BLASFUNC(dsdot) (blasint *, float  *, blasint *, float  *, blasint *);
 double BLASFUNC(ddot)  (blasint *, double *, blasint *, double *, blasint *);
 xdouble BLASFUNC(qdot)  (blasint *, xdouble *, blasint *, xdouble *, blasint *);
 
+void   BLASFUNC(bscal) (blasint *,  bfloat16  *, bfloat16  *, blasint *);
 float  BLASFUNC(sbdot)     (blasint *, bfloat16 *, blasint *, bfloat16 *, blasint *);
 void   BLASFUNC(sbstobf16) (blasint *, float *,    blasint *, bfloat16 *, blasint *);
 void   BLASFUNC(sbdtobf16) (blasint *, double *,   blasint *, bfloat16 *, blasint *);
@@ -255,6 +257,8 @@ void BLASFUNC(xgeru)(blasint *,    blasint *, xdouble *, xdouble *, blasint *,
 void BLASFUNC(xgerc)(blasint *,    blasint *, xdouble *, xdouble *, blasint *,
 		    xdouble *, blasint *, xdouble *, blasint *);
 
+void BLASFUNC(bgemv)(char *, blasint *, blasint *, bfloat16  *, bfloat16 *, blasint *,
+            bfloat16  *, blasint *, bfloat16  *, bfloat16  *, blasint *);
 void BLASFUNC(sbgemv)(char *, blasint *, blasint *, float  *, bfloat16 *, blasint *,
             bfloat16  *, blasint *, float  *, float  *, blasint *);
 void BLASFUNC(sgemv)(char *, blasint *, blasint *, float  *, float  *, blasint *,
@@ -483,6 +487,8 @@ void BLASFUNC(xhbmv)(char *, blasint *, blasint *, xdouble *, xdouble *, blasint
 
 void BLASFUNC(shgemm)(char *, char *, blasint *, blasint *, blasint *, float *,
 	   hfloat16  *, blasint *, hfloat16 *, blasint *, float  *, float  *, blasint *);
+void BLASFUNC(bgemm)(char *, char *, blasint *, blasint *, blasint *, bfloat16 *,
+	   bfloat16 *, blasint *, bfloat16 *, blasint *, bfloat16 *, bfloat16 *, blasint *);
 void BLASFUNC(sbgemm)(char *, char *, blasint *, blasint *, blasint *, float *,
 	   bfloat16 *, blasint *, bfloat16 *, blasint *, float *, float *, blasint *);
 void BLASFUNC(sgemm)(char *, char *, blasint *, blasint *, blasint *, float *,

common_level1.h

@@ -1,4 +1,5 @@
 /*********************************************************************/
+/* Copyright 2025 The OpenBLAS Project.                              */
 /* Copyright 2009, 2010 The University of Texas at Austin.           */
 /* All rights reserved.                                              */
 /*                                                                   */
@@ -169,6 +170,9 @@ BLASLONG   icmin_k(BLASLONG, float  *, BLASLONG);
 BLASLONG   izmin_k(BLASLONG, double *, BLASLONG);
 BLASLONG   ixmin_k(BLASLONG, xdouble *, BLASLONG);
 
+
+int    bscal_k(BLASLONG, BLASLONG, BLASLONG, bfloat16,
+	      bfloat16 *, BLASLONG, bfloat16 *, BLASLONG,  bfloat16  *, BLASLONG);
 int    sscal_k(BLASLONG, BLASLONG, BLASLONG, float,
 	      float *, BLASLONG, float *, BLASLONG,  float  *, BLASLONG);
 int    dscal_k(BLASLONG, BLASLONG, BLASLONG, double,

common_level2.h

@@ -1,4 +1,5 @@
 /*********************************************************************/
+/* Copyright 2025 The OpenBLAS Project                               */
 /* Copyright 2009, 2010 The University of Texas at Austin.           */
 /* All rights reserved.                                              */
 /*                                                                   */
@@ -44,6 +45,11 @@
 extern "C" {
 #endif
 
+
+int bgemv_n(BLASLONG, BLASLONG, bfloat16, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, bfloat16, bfloat16 *, BLASLONG);
+int bgemv_t(BLASLONG, BLASLONG, bfloat16, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, bfloat16, bfloat16 *, BLASLONG);
+int bgemv_thread_n(BLASLONG, BLASLONG, bfloat16, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, bfloat16, bfloat16 *, BLASLONG, int);
+int bgemv_thread_t(BLASLONG, BLASLONG, bfloat16, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, bfloat16, bfloat16 *, BLASLONG, int);
 int sbgemv_n(BLASLONG, BLASLONG, float, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, float, float *, BLASLONG);
 int sbgemv_t(BLASLONG, BLASLONG, float, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, float, float *, BLASLONG);
 int sbgemv_thread_n(BLASLONG, BLASLONG, float, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, float, float *, BLASLONG, int);

common_level3.h

@@ -52,10 +52,19 @@ void sgemm_direct(BLASLONG M, BLASLONG N, BLASLONG K,
 	float * B, BLASLONG strideB,
 	float * R, BLASLONG strideR);
 
+void sgemm_direct_alpha_beta(BLASLONG M, BLASLONG N, BLASLONG K,
+	float alpha,
+	float * A, BLASLONG strideA,
+	float * B, BLASLONG strideB,
+	float beta,
+	float * R, BLASLONG strideR);
+
 int sgemm_direct_performant(BLASLONG M, BLASLONG N, BLASLONG K);
 
 int shgemm_beta(BLASLONG, BLASLONG, BLASLONG, float,
 	       hfloat16 *, BLASLONG, hfloat16 *, BLASLONG, float *, BLASLONG);
+int bgemm_beta(BLASLONG, BLASLONG, BLASLONG, bfloat16,
+	       bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG);
 int sbgemm_beta(BLASLONG, BLASLONG, BLASLONG, float,
 	       bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, float *, BLASLONG);
 int sgemm_beta(BLASLONG, BLASLONG, BLASLONG, float,
@@ -83,6 +92,10 @@ int shgemm_incopy(BLASLONG m, BLASLONG n, hfloat16 *a, BLASLONG lda, hfloat16 *b
 int shgemm_itcopy(BLASLONG m, BLASLONG n, hfloat16 *a, BLASLONG lda, hfloat16 *b);
 int shgemm_oncopy(BLASLONG m, BLASLONG n, hfloat16 *a, BLASLONG lda, hfloat16 *b);
 int shgemm_otcopy(BLASLONG m, BLASLONG n, hfloat16 *a, BLASLONG lda, hfloat16 *b);
+int bgemm_incopy(BLASLONG m, BLASLONG n, bfloat16 *a, BLASLONG lda, bfloat16 *b);
+int bgemm_itcopy(BLASLONG m, BLASLONG n, bfloat16 *a, BLASLONG lda, bfloat16 *b);
+int bgemm_oncopy(BLASLONG m, BLASLONG n, bfloat16 *a, BLASLONG lda, bfloat16 *b);
+int bgemm_otcopy(BLASLONG m, BLASLONG n, bfloat16 *a, BLASLONG lda, bfloat16 *b);
 int sbgemm_incopy(BLASLONG m, BLASLONG n, bfloat16 *a, BLASLONG lda, bfloat16 *b);
 int sbgemm_itcopy(BLASLONG m, BLASLONG n, bfloat16 *a, BLASLONG lda, bfloat16 *b);
 int sbgemm_oncopy(BLASLONG m, BLASLONG n, bfloat16 *a, BLASLONG lda, bfloat16 *b);
@@ -511,6 +524,7 @@ int xher2k_kernel_LN(BLASLONG m, BLASLONG n, BLASLONG k, xdouble alpha_r, xdoubl
 int xher2k_kernel_LC(BLASLONG m, BLASLONG n, BLASLONG k, xdouble alpha_r, xdouble alpha_i, xdouble *a, xdouble *b, xdouble *c, BLASLONG ldc, BLASLONG offset, int flag);
 
 int shgemm_kernel(BLASLONG, BLASLONG, BLASLONG, float,  hfloat16 *, hfloat16 *, float *, BLASLONG);
+int bgemm_kernel(BLASLONG, BLASLONG, BLASLONG, bfloat16,  bfloat16 *, bfloat16 *, bfloat16 *, BLASLONG);
 int sbgemm_kernel(BLASLONG, BLASLONG, BLASLONG, float,  bfloat16 *, bfloat16 *, float *, BLASLONG);
 int sgemm_kernel(BLASLONG, BLASLONG, BLASLONG, float,  float  *, float  *, float  *, BLASLONG);
 int dgemm_kernel(BLASLONG, BLASLONG, BLASLONG, double, double *, double *, double *, BLASLONG);
@@ -668,6 +682,11 @@ int shgemm_nt(blas_arg_t *, BLASLONG *, BLASLONG *, hfloat16 *, hfloat16 *, BLAS
 int shgemm_tn(blas_arg_t *, BLASLONG *, BLASLONG *, hfloat16 *, hfloat16 *, BLASLONG);
 int shgemm_tt(blas_arg_t *, BLASLONG *, BLASLONG *, hfloat16 *, hfloat16 *, BLASLONG);
 
+int bgemm_nn(blas_arg_t *, BLASLONG *, BLASLONG *, bfloat16 *, bfloat16 *, BLASLONG);
+int bgemm_nt(blas_arg_t *, BLASLONG *, BLASLONG *, bfloat16 *, bfloat16 *, BLASLONG);
+int bgemm_tn(blas_arg_t *, BLASLONG *, BLASLONG *, bfloat16 *, bfloat16 *, BLASLONG);
+int bgemm_tt(blas_arg_t *, BLASLONG *, BLASLONG *, bfloat16 *, bfloat16 *, BLASLONG);
+
 int sbgemm_nn(blas_arg_t *, BLASLONG *, BLASLONG *, bfloat16 *, bfloat16 *, BLASLONG);
 int sbgemm_nt(blas_arg_t *, BLASLONG *, BLASLONG *, bfloat16 *, bfloat16 *, BLASLONG);
 int sbgemm_tn(blas_arg_t *, BLASLONG *, BLASLONG *, bfloat16 *, bfloat16 *, BLASLONG);
@@ -770,6 +789,11 @@ int shgemm_thread_nt(blas_arg_t *, BLASLONG *, BLASLONG *, hfloat16 *, hfloat16
 int shgemm_thread_tn(blas_arg_t *, BLASLONG *, BLASLONG *, hfloat16 *, hfloat16 *, BLASLONG);
 int shgemm_thread_tt(blas_arg_t *, BLASLONG *, BLASLONG *, hfloat16 *, hfloat16 *, BLASLONG);
 
+int bgemm_thread_nn(blas_arg_t *, BLASLONG *, BLASLONG *, bfloat16 *, bfloat16 *, BLASLONG);
+int bgemm_thread_nt(blas_arg_t *, BLASLONG *, BLASLONG *, bfloat16 *, bfloat16 *, BLASLONG);
+int bgemm_thread_tn(blas_arg_t *, BLASLONG *, BLASLONG *, bfloat16 *, bfloat16 *, BLASLONG);
+int bgemm_thread_tt(blas_arg_t *, BLASLONG *, BLASLONG *, bfloat16 *, bfloat16 *, BLASLONG);
+
 int sbgemm_thread_nn(blas_arg_t *, BLASLONG *, BLASLONG *, bfloat16 *, bfloat16 *, BLASLONG);
 int sbgemm_thread_nt(blas_arg_t *, BLASLONG *, BLASLONG *, bfloat16 *, bfloat16 *, BLASLONG);
 int sbgemm_thread_tn(blas_arg_t *, BLASLONG *, BLASLONG *, bfloat16 *, bfloat16 *, BLASLONG);

common_macro.h

@@ -1,5 +1,6 @@
 /*********************************************************************/
 /* Copyright 2009, 2010 The University of Texas at Austin.           */
+/* Copyright 2025 The OpenBLAS Project.                              */
 /* All rights reserved.                                              */
 /*                                                                   */
 /* Redistribution and use in source and binary forms, with or        */
@@ -40,6 +41,7 @@
 #define COMMON_MACRO
 
 #include "common_sh.h"
+#include "common_b.h"
 #include "common_sb.h"
 #include "common_s.h"
 #include "common_d.h"
@@ -702,14 +704,63 @@
 #define GEMM_THREAD_RR   SHGEMM_THREAD_NN
 
 
-#elif defined(BFLOAT16)
+#elif defined(BFLOAT16) && defined(BGEMM)
+#define SCAL_K        BSCAL_K
+
+#define GEMV_N        BGEMV_N_K
+#define GEMV_T        BGEMV_T_K
+
+#define GEMM_BETA               BGEMM_BETA
+#define	GEMM_KERNEL_N		BGEMM_KERNEL
+#define	GEMM_KERNEL_L		BGEMM_KERNEL
+#define	GEMM_KERNEL_R		BGEMM_KERNEL
+#define	GEMM_KERNEL_B		BGEMM_KERNEL
+
+#define	GEMM_NN			BGEMM_NN
+#define	GEMM_CN			BGEMM_TN
+#define	GEMM_TN			BGEMM_TN
+#define	GEMM_NC			BGEMM_NT
+#define	GEMM_NT			BGEMM_NT
+#define	GEMM_CC			BGEMM_TT
+#define	GEMM_CT			BGEMM_TT
+#define	GEMM_TC			BGEMM_TT
+#define	GEMM_TT			BGEMM_TT
+#define	GEMM_NR			BGEMM_NN
+#define	GEMM_TR			BGEMM_TN
+#define	GEMM_CR			BGEMM_TN
+#define	GEMM_RN			BGEMM_NN
+#define	GEMM_RT			BGEMM_NT
+#define	GEMM_RC			BGEMM_NT
+#define	GEMM_RR			BGEMM_NN
+#define	GEMM_ONCOPY		BGEMM_ONCOPY
+#define	GEMM_OTCOPY		BGEMM_OTCOPY
+#define	GEMM_INCOPY		BGEMM_INCOPY
+#define	GEMM_ITCOPY		BGEMM_ITCOPY
+
+#define	GEMM_THREAD_NN		BGEMM_THREAD_NN
+#define	GEMM_THREAD_CN		BGEMM_THREAD_TN
+#define	GEMM_THREAD_TN		BGEMM_THREAD_TN
+#define	GEMM_THREAD_NC		BGEMM_THREAD_NT
+#define	GEMM_THREAD_NT		BGEMM_THREAD_NT
+#define	GEMM_THREAD_CC		BGEMM_THREAD_TT
+#define	GEMM_THREAD_CT		BGEMM_THREAD_TT
+#define	GEMM_THREAD_TC		BGEMM_THREAD_TT
+#define	GEMM_THREAD_TT		BGEMM_THREAD_TT
+#define	GEMM_THREAD_NR		BGEMM_THREAD_NN
+#define	GEMM_THREAD_TR		BGEMM_THREAD_TN
+#define	GEMM_THREAD_CR		BGEMM_THREAD_TN
+#define	GEMM_THREAD_RN		BGEMM_THREAD_NN
+#define	GEMM_THREAD_RT		BGEMM_THREAD_NT
+#define	GEMM_THREAD_RC		BGEMM_THREAD_NT
+#define	GEMM_THREAD_RR		BGEMM_THREAD_NN
 
+#elif defined(BFLOAT16)
 #define D_TO_BF16_K     SBDTOBF16_K
 #define D_BF16_TO_K     DBF16TOD_K
 #define S_TO_BF16_K     SBSTOBF16_K
 #define S_BF16_TO_K     SBF16TOS_K
-#define SBGEMV_N        SBGEMV_N_K
-#define SBGEMV_T        SBGEMV_T_K
+#define GEMV_N          SBGEMV_N_K
+#define GEMV_T          SBGEMV_T_K
 
 #define	AMAX_K			SAMAX_K
 #define	AMIN_K			SAMIN_K
@@ -727,8 +778,6 @@
 #define	AXPYC_K			SAXPYC_K
 #define AXPBY_K     SAXPBY_K
 #define SCAL_K			SSCAL_K
-#define GEMV_N			SGEMV_N
-#define GEMV_T			SGEMV_T
 #define SYMV_U			SSYMV_U
 #define SYMV_L			SSYMV_L
 #define	GERU_K			SGERU_K
@@ -2663,6 +2712,9 @@
       || defined(ARCH_LOONGARCH64) || defined(ARCH_E2K) || defined(ARCH_ALPHA))
 extern BLASLONG gemm_offset_a;
 extern BLASLONG gemm_offset_b;
+extern BLASLONG bgemm_p;
+extern BLASLONG bgemm_q;
+extern BLASLONG bgemm_r;
 extern BLASLONG sbgemm_p;
 extern BLASLONG sbgemm_q;
 extern BLASLONG sbgemm_r;

common_param.h

@@ -1,6 +1,6 @@
 /*********************************************************************/
 /* Copyright 2009, 2010 The University of Texas at Austin.           */
-/* Copyright 2023 The OpenBLAS Project.                              */
+/* Copyright 2023, 2025 The OpenBLAS Project.                        */
 /* All rights reserved.                                              */
 /*                                                                   */
 /* Redistribution and use in source and binary forms, with or        */
@@ -65,6 +65,10 @@ int (*shgemm_otcopy   )(BLASLONG, BLASLONG, hfloat16 *, BLASLONG, hfloat16 *);
 
 
 #if BUILD_BFLOAT16 == 1
+  int bgemm_p, bgemm_q, bgemm_r;
+  int bgemm_unroll_m, bgemm_unroll_n, bgemm_unroll_mn;
+  int bgemm_align_k;
+
   int sbgemm_p, sbgemm_q, sbgemm_r;
   int sbgemm_unroll_m, sbgemm_unroll_n, sbgemm_unroll_mn;
   int sbgemm_align_k;
@@ -94,10 +98,14 @@ int (*shgemm_otcopy   )(BLASLONG, BLASLONG, hfloat16 *, BLASLONG, hfloat16 *);
   int    (*sbrot_k)  (BLASLONG, float *, BLASLONG, float *, BLASLONG, float, float);
   int    (*sbrotm_k) (BLASLONG, float *, BLASLONG, float *, BLASLONG, float *);
 
+  int    (*bscal_k) (BLASLONG, BLASLONG, BLASLONG, bfloat16, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG);
   int    (*sbaxpy_k) (BLASLONG, BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG);
   int    (*sbscal_k) (BLASLONG, BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG);
   int    (*sbswap_k) (BLASLONG, BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG);
 
+  int    (*bgemv_n) (BLASLONG, BLASLONG, bfloat16, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, bfloat16, bfloat16 *, BLASLONG);
+  int    (*bgemv_t) (BLASLONG, BLASLONG, bfloat16, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, bfloat16, bfloat16 *, BLASLONG);
+
   int    (*sbgemv_n) (BLASLONG, BLASLONG, float, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, float, float *, BLASLONG);
   int    (*sbgemv_t) (BLASLONG, BLASLONG, float, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, float, float *, BLASLONG);
   int    (*sbger_k)  (BLASLONG, BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG, float *);
@@ -105,6 +113,14 @@ int (*shgemm_otcopy   )(BLASLONG, BLASLONG, hfloat16 *, BLASLONG, hfloat16 *);
   int    (*sbsymv_L) (BLASLONG, BLASLONG, float,  float  *, BLASLONG, float  *, BLASLONG, float  *, BLASLONG, float *);
   int    (*sbsymv_U) (BLASLONG, BLASLONG, float,  float  *, BLASLONG, float  *, BLASLONG, float  *, BLASLONG, float *);
 
+  int    (*bgemm_kernel   )(BLASLONG, BLASLONG, BLASLONG, bfloat16, bfloat16 *, bfloat16 *, bfloat16 *, BLASLONG);
+  int    (*bgemm_beta     )(BLASLONG, BLASLONG, BLASLONG, bfloat16, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG);
+
+  int    (*bgemm_incopy   )(BLASLONG, BLASLONG, bfloat16 *, BLASLONG, bfloat16 *);
+  int    (*bgemm_itcopy   )(BLASLONG, BLASLONG, bfloat16 *, BLASLONG, bfloat16 *);
+  int    (*bgemm_oncopy   )(BLASLONG, BLASLONG, bfloat16 *, BLASLONG, bfloat16 *);
+  int    (*bgemm_otcopy   )(BLASLONG, BLASLONG, bfloat16 *, BLASLONG, bfloat16 *);
+
   int    (*sbgemm_kernel   )(BLASLONG, BLASLONG, BLASLONG, float, bfloat16 *, bfloat16 *, float *, BLASLONG);
   int    (*sbgemm_beta     )(BLASLONG, BLASLONG, BLASLONG, float, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, float *, BLASLONG);
 
@@ -240,6 +256,7 @@ int (*shgemm_otcopy   )(BLASLONG, BLASLONG, hfloat16 *, BLASLONG, hfloat16 *);
 #endif
 #ifdef ARCH_ARM64
   void (*sgemm_direct) (BLASLONG, BLASLONG, BLASLONG, float *, BLASLONG , float *, BLASLONG , float * , BLASLONG);
+  void (*sgemm_direct_alpha_beta) (BLASLONG, BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float, float * , BLASLONG);
 #endif
 
   
@@ -1254,6 +1271,13 @@ extern gotoblas_t *gotoblas;
 #endif
 
 #if (BUILD_BFLOAT16==1)
+#define	BGEMM_P		gotoblas -> bgemm_p
+#define	BGEMM_Q		gotoblas -> bgemm_q
+#define	BGEMM_R		gotoblas -> bgemm_r
+#define	BGEMM_UNROLL_M	gotoblas -> bgemm_unroll_m
+#define	BGEMM_UNROLL_N	gotoblas -> bgemm_unroll_n
+#define	BGEMM_UNROLL_MN	gotoblas -> bgemm_unroll_mn
+
 #define	SBGEMM_P		gotoblas -> sbgemm_p
 #define	SBGEMM_Q		gotoblas -> sbgemm_q
 #define	SBGEMM_R		gotoblas -> sbgemm_r
@@ -1395,6 +1419,17 @@ extern gotoblas_t *gotoblas;
 #endif
 
 #if (BUILD_BFLOAT16 == 1)
+#define	BGEMM_P			BGEMM_DEFAULT_P
+#define	BGEMM_Q			BGEMM_DEFAULT_Q
+#define	BGEMM_R			BGEMM_DEFAULT_R
+#define	BGEMM_UNROLL_M	BGEMM_DEFAULT_UNROLL_M
+#define	BGEMM_UNROLL_N	BGEMM_DEFAULT_UNROLL_N
+#ifdef  BGEMM_DEFAULT_UNROLL_MN
+#define	BGEMM_UNROLL_MN	BGEMM_DEFAULT_UNROLL_MN
+#else
+#define	BGEMM_UNROLL_MN	MAX((BGEMM_UNROLL_M), (BGEMM_UNROLL_N))
+#endif
+
 #define	SBGEMM_P		SBGEMM_DEFAULT_P
 #define	SBGEMM_Q		SBGEMM_DEFAULT_Q
 #define	SBGEMM_R		SBGEMM_DEFAULT_R
@@ -1555,6 +1590,18 @@ extern gotoblas_t *gotoblas;
 #define GEMM_DEFAULT_R		SHGEMM_DEFAULT_R
 #define GEMM_DEFAULT_UNROLL_M	SHGEMM_DEFAULT_UNROLL_M
 #define GEMM_DEFAULT_UNROLL_N	SHGEMM_DEFAULT_UNROLL_N
+#elif defined(BFLOAT16) && defined(BGEMM)
+#define GEMM_P			BGEMM_P
+#define GEMM_Q			BGEMM_Q
+#define GEMM_R			BGEMM_R
+#define GEMM_UNROLL_M		BGEMM_UNROLL_M
+#define GEMM_UNROLL_N		BGEMM_UNROLL_N
+#define GEMM_UNROLL_MN		BGEMM_UNROLL_MN
+#define GEMM_DEFAULT_P		BGEMM_DEFAULT_P
+#define GEMM_DEFAULT_Q		BGEMM_DEFAULT_Q
+#define GEMM_DEFAULT_R		BGEMM_DEFAULT_R
+#define GEMM_DEFAULT_UNROLL_M	BGEMM_DEFAULT_UNROLL_M
+#define GEMM_DEFAULT_UNROLL_N	BGEMM_DEFAULT_UNROLL_N
 #elif defined(BFLOAT16)
 #define GEMM_P			SBGEMM_P
 #define GEMM_Q			SBGEMM_Q

common_s.h

@@ -49,6 +49,7 @@
 
 #define SGEMM_DIRECT_PERFORMANT    sgemm_direct_performant
 #define SGEMM_DIRECT		sgemm_direct
+#define  SGEMM_DIRECT_ALPHA_BETA		sgemm_direct_alpha_beta
 
 #define	SGEMM_ONCOPY		sgemm_oncopy
 #define	SGEMM_OTCOPY		sgemm_otcopy
@@ -218,6 +219,7 @@
 #elif ARCH_ARM64
 #define SGEMM_DIRECT_PERFORMANT    sgemm_direct_performant
 #define  SGEMM_DIRECT		gotoblas -> sgemm_direct
+#define  SGEMM_DIRECT_ALPHA_BETA		gotoblas -> sgemm_direct_alpha_beta
 #endif
 
 #define	SGEMM_ONCOPY		gotoblas -> sgemm_oncopy

cpuid.S

@@ -66,5 +66,9 @@ _cpuid:
 
 #endif
 #if defined(__ELF__) && defined(__linux__)
+#if defined(__arm__)
+        .section        .note.GNU-stack,"",%progbits
+#else
         .section        .note.GNU-stack,"",@progbits
 #endif
+#endif

cpuid_power.c

@@ -131,6 +131,7 @@ int detect(void){
   if (!strncasecmp(p, "POWER8", 6)) return CPUTYPE_POWER8;
   if (!strncasecmp(p, "POWER9", 6)) return CPUTYPE_POWER9;
   if (!strncasecmp(p, "POWER10", 7)) return CPUTYPE_POWER10;
+  if (!strncasecmp(p, "POWER11", 7)) return CPUTYPE_POWER10;
   if (!strncasecmp(p, "Cell",   4)) return CPUTYPE_CELL;
   if (!strncasecmp(p, "7447",   4)) return CPUTYPE_PPCG4;
 
@@ -171,6 +172,9 @@ int detect(void){
 int id;
 __asm __volatile("mfpvr %0" : "=r"(id));
 switch ( id >> 16 ) {
+  case 0x82: // POWER11
+    return CPUTYPE_POWER10;
+    break;
   case 0x80: // POWER10
     return CPUTYPE_POWER10;
     break;

cpuid_x86.c

@@ -1567,6 +1567,7 @@ int get_cpuname(void){
           case 10:
           case 15:
 	  case 14: // Alder Lake N
+	  case 13: // Lunar Lake
             if(support_avx2())
               return CPUTYPE_HASWELL;
 	    if(support_avx())
@@ -2412,7 +2413,8 @@ int get_coretype(void){
 	  case 7: // Raptor Lake
           case 10:
           case 15:
-	  case 14: // Alder Lake N	  
+	  case 14: // Alder Lake N
+	  case 13: // Lunar Lake
 #ifndef NO_AVX2
 	  if(support_avx2())
             return CORE_HASWELL;

docs/install.md

@@ -623,14 +623,22 @@ Note: using `TARGET=CORTEXA57` in place of `ARMV8` will pick up better
 optimized routines. Implementations for the `CORTEXA57` target are compatible
 with all other `ARMV8` targets.
 
-Note: for NDK 23b, something as simple as:
+Note: for NDK 23b and later, something as simple as:
 ```bash
 export PATH=/opt/android-ndk-r23b/toolchains/llvm/prebuilt/linux-x86_64/bin/:$PATH
-make HOSTCC=gcc CC=/opt/android-ndk-r23b/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android31-clang ONLY_CBLAS=1 TARGET=ARMV8
+make HOSTCC=gcc CC=/opt/android-ndk-r23b/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android31-clang ONLY_CBLAS=1 TARGET=ARMV8 RANLIB=echo
 ```
 appears to be sufficient on Linux. On OSX, setting AR to the ar provided in the
 "bin" path of the NDK (probably `llvm-ar`) is also necessary.
 
+If you prefer building with CMake, running 
+```bash
+cmake -DANDROID_ABI=arm64-v8a -DTARGET=ARMV8 -DCMAKE_TOOLCHAIN_FILE=/opt/android-ndk-r27/build/cmake/android.toolchain.cmake -DNOFORTRAN=1 -DANDROID_PLATFORM=android-23 ..
+cmake --build .
+```
+in your build directory should work (be sure to adjust the toolchain_file argument according to where you installed the NDK, and the ANDROID_PLATFORM
+according to the minimum version of Android you want to support. (If you leave out the ANDROID_PLATFORM parameter, the build will fail with an error 
+message about a missing declaration or missing header file complex.h)
 
 ??? note "Alternative build script for 3 architectures"
 

driver/level2/Makefile

@@ -1,3 +1,31 @@
+###############################################################################
+# Copyright (c) 2025 The OpenBLAS Project
+# All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in
+#    the documentation and/or other materials provided with the
+#    distribution.
+# 3. Neither the name of the OpenBLAS project nor the names of
+#    its contributors may be used to endorse or promote products
+#    derived from this software without specific prior written permission.
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+###############################################################################
+
 TOPDIR	= ../..
 include ../../Makefile.system
 
@@ -423,6 +451,9 @@ XBLASOBJS   += \
 	xtbmv_thread_CLU.$(SUFFIX)	xtbmv_thread_CLN.$(SUFFIX)
 
 ifeq ($(BUILD_BFLOAT16),1)
+BBLASOBJS      += \
+        bgemv_thread_n$(TSUFFIX).$(SUFFIX) \
+        bgemv_thread_t$(TSUFFIX).$(SUFFIX)
 SBBLASOBJS     += \
         sbgemv_thread_n$(TSUFFIX).$(SUFFIX) \
         sbgemv_thread_t$(TSUFFIX).$(SUFFIX)
@@ -3707,6 +3738,10 @@ xtrsv_CUN.$(SUFFIX)  xtrsv_CUN.$(PSUFFIX)  : ztrsv_L.c ../../param.h
 	$(CC) -c $(CFLAGS) -DXDOUBLE -DCOMPLEX -DTRANSA=4 -UUNIT $< -o $(@F)
 
 ifeq ($(BUILD_BFLOAT16),1)
+bgemv_thread_n.$(SUFFIX) bgemv_thread_n.$(PSUFFIX) : sbgemv_thread.c ../../common.h
+	$(CC) -c $(CFLAGS) -DBGEMM -UCOMPLEX -UDOUBLE  -UTRANSA -UCONJ -UXCONJ $< -o $(@F)
+bgemv_thread_t.$(SUFFIX) bgemv_thread_t.$(PSUFFIX) : sbgemv_thread.c ../../common.h
+	$(CC) -c $(CFLAGS) -DBGEMM -UCOMPLEX -UDOUBLE  -DTRANSA -UCONJ -UXCONJ $< -o $(@F)
 sbgemv_thread_n.$(SUFFIX) sbgemv_thread_n.$(PSUFFIX) : sbgemv_thread.c ../../common.h
 	$(CC) -c $(CFLAGS) -UCOMPLEX -UDOUBLE  -UTRANSA -UCONJ -UXCONJ $< -o $(@F)
 sbgemv_thread_t.$(SUFFIX) sbgemv_thread_t.$(PSUFFIX) : sbgemv_thread.c ../../common.h

driver/level2/sbgemv_thread.c

@@ -1,4 +1,5 @@
 /*********************************************************************/
+/* Copyright 2025 The OpenBLAS Project.                              */
 /* Copyright 2009, 2010 The University of Texas at Austin.           */
 /* All rights reserved.                                              */
 /*                                                                   */
@@ -41,21 +42,21 @@
 #include "common.h"
 
 #ifndef TRANSA
-#define SBGEMV	SBGEMV_N
+#define GEMV	GEMV_N
 #else
-#define SBGEMV	SBGEMV_T
+#define GEMV	GEMV_T
 #endif
 
 static int sbgemv_kernel(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *dummy1, FLOAT *dummy2, BLASLONG dummy3){
 
-    bfloat16 *a, *x;
-    float    *y;
+    IFLOAT *a, *x;
+    FLOAT    *y;
     BLASLONG lda, incx, incy;
     BLASLONG m_from, m_to, n_from, n_to;
 
-    a = (bfloat16 *)args->a;
-    x = (bfloat16 *)args->b;
-    y = (float *)args->c;
+    a = (IFLOAT *)args->a;
+    x = (IFLOAT *)args->b;
+    y = (FLOAT *)args->c;
 
     lda  = args->lda;
     incx = args->ldb;
@@ -77,12 +78,12 @@ static int sbgemv_kernel(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n,
     y += n_from * incy;
 #endif
 
-    SBGEMV(m_to - m_from, n_to - n_from, *((FLOAT *)(args->alpha)), a, lda, x, incx, *((FLOAT *)(args->beta)), y, incy);
+    GEMV(m_to - m_from, n_to - n_from, *((FLOAT *)(args->alpha)), a, lda, x, incx, *((FLOAT *)(args->beta)), y, incy);
 
     return 0;
 }
 
-int CNAME(BLASLONG m, BLASLONG n, float alpha, bfloat16 *a, BLASLONG lda, bfloat16 *x, BLASLONG incx, float beta, float *y, BLASLONG incy, int threads)
+int CNAME(BLASLONG m, BLASLONG n, FLOAT alpha, IFLOAT *a, BLASLONG lda, IFLOAT *x, BLASLONG incx, FLOAT beta, FLOAT *y, BLASLONG incy, int threads)
 {
     blas_arg_t args;
     blas_queue_t queue[MAX_CPU_NUMBER];

driver/level3/Makefile

@@ -1,3 +1,32 @@
+###############################################################################
+# Copyright (c) 2025, The OpenBLAS Project
+# All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in
+#    the documentation and/or other materials provided with the
+#    distribution.
+# 3. Neither the name of the OpenBLAS project nor the names of
+#    its contributors may be used to endorse or promote products
+#    derived from this software without specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+###############################################################################
+
 TOPDIR	= ../..
 include ../../Makefile.system
 
@@ -20,6 +49,7 @@ USE_GEMM3M = 1
 endif
 
 ifeq ($(BUILD_BFLOAT16),1)
+BBLASOBJS       += bgemm_nn.$(SUFFIX) bgemm_nt.$(SUFFIX) bgemm_tn.$(SUFFIX) bgemm_tt.$(SUFFIX)
 SBBLASOBJS       += sbgemm_nn.$(SUFFIX) sbgemm_nt.$(SUFFIX) sbgemm_tn.$(SUFFIX) sbgemm_tt.$(SUFFIX)
 endif
 
@@ -212,6 +242,7 @@ COMMONOBJS  += syrk_thread.$(SUFFIX)
 
 ifneq ($(USE_SIMPLE_THREADED_LEVEL3), 1)
 ifeq ($(BUILD_BFLOAT16),1)
+BBLASOBJS     += bgemm_thread_nn.$(SUFFIX) bgemm_thread_nt.$(SUFFIX) bgemm_thread_tn.$(SUFFIX) bgemm_thread_tt.$(SUFFIX)
 SBBLASOBJS    += sbgemm_thread_nn.$(SUFFIX) sbgemm_thread_nt.$(SUFFIX) sbgemm_thread_tn.$(SUFFIX) sbgemm_thread_tt.$(SUFFIX)
 endif
 ifeq ($(BUILD_HFLOAT16),1)
@@ -350,6 +381,18 @@ endif
 
 all ::
 
+bgemm_nn.$(SUFFIX) : gemm.c level3.c ../../param.h
+	$(CC) $(CFLAGS) $(BLOCKS) -c -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX -DNN $< -o $(@F)
+
+bgemm_nt.$(SUFFIX) : gemm.c level3.c ../../param.h
+	$(CC) $(CFLAGS) $(BLOCKS) -c -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX -DNT $< -o $(@F)
+
+bgemm_tn.$(SUFFIX) : gemm.c level3.c ../../param.h
+	$(CC) $(CFLAGS) $(BLOCKS) -c -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX -DTN $< -o $(@F)
+
+bgemm_tt.$(SUFFIX) : gemm.c level3.c ../../param.h
+	$(CC) $(CFLAGS) $(BLOCKS) -c -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX -DTT $< -o $(@F)
+
 sbgemm_nn.$(SUFFIX) : gemm.c level3.c ../../param.h
 	$(CC) $(CFLAGS) $(BLOCKS) -c -DBFLOAT16 -UDOUBLE -UCOMPLEX -DNN $< -o $(@F)
 
@@ -569,6 +612,18 @@ gemm_thread_variable.$(SUFFIX) : gemm_thread_variable.c ../../common.h
 beta_thread.$(SUFFIX) : beta_thread.c ../../common.h
 	$(CC) -c $(CFLAGS) $< -o $(@F)
 
+bgemm_thread_nn.$(SUFFIX) : gemm.c level3_thread.c ../../param.h
+	$(CC) $(CFLAGS) $(BLOCKS) -c -DTHREADED_LEVEL3 -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX -DNN $< -o $(@F)
+
+bgemm_thread_nt.$(SUFFIX) : gemm.c level3_thread.c ../../param.h
+	$(CC) $(CFLAGS) $(BLOCKS) -c -DTHREADED_LEVEL3 -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX -DNT $< -o $(@F)
+
+bgemm_thread_tn.$(SUFFIX) : gemm.c level3_thread.c ../../param.h
+	$(CC) $(CFLAGS) $(BLOCKS) -c -DTHREADED_LEVEL3 -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX -DTN $< -o $(@F)
+
+bgemm_thread_tt.$(SUFFIX) : gemm.c level3_thread.c ../../param.h
+	$(CC) $(CFLAGS) $(BLOCKS) -c -DTHREADED_LEVEL3 -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX -DTT $< -o $(@F)
+
 sbgemm_thread_nn.$(SUFFIX) : gemm.c level3_thread.c ../../param.h
 	$(CC) $(CFLAGS) $(BLOCKS) -c -DTHREADED_LEVEL3 -DBFLOAT16 -UDOUBLE -UCOMPLEX -DNN $< -o $(@F)
 

driver/level3/level3.c

@@ -1,5 +1,6 @@
 /*********************************************************************/
 /* Copyright 2009, 2010 The University of Texas at Austin.           */
+/* Copyright 2025 The OpenBLAS Project.                              */
 /* All rights reserved.                                              */
 /*                                                                   */
 /* Redistribution and use in source and binary forms, with or        */
@@ -169,6 +170,22 @@
 #define STOP_RPCC(COUNTER)
 #endif
 
+#if defined(BUILD_BFLOAT16)
+#if defined(DYNAMIC_ARCH)
+  #if defined(BGEMM)
+    #define BFLOAT16_ALIGN_K gotoblas->bgemm_align_k
+  #else
+    #define BFLOAT16_ALIGN_K gotoblas->sbgemm_align_k
+  #endif
+#else
+  #if defined(BGEMM)
+    #define BFLOAT16_ALIGN_K BGEMM_ALIGN_K
+  #else
+    #define BFLOAT16_ALIGN_K SBGEMM_ALIGN_K
+  #endif
+#endif
+#endif
+
 int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n,
 		  XFLOAT *sa, XFLOAT *sb, BLASLONG dummy){
   BLASLONG k, lda, ldb, ldc;
@@ -305,12 +322,8 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n,
       }
 
       BLASLONG pad_min_l = min_l;
-#if defined(HALF)
-#if defined(DYNAMIC_ARCH)
-      pad_min_l = (min_l + gotoblas->sbgemm_align_k - 1) & ~(gotoblas->sbgemm_align_k-1);
-#else
-      pad_min_l = (min_l + SBGEMM_ALIGN_K - 1) & ~(SBGEMM_ALIGN_K - 1);;
-#endif
+#if defined(BFLOAT16)
+    pad_min_l = (min_l + BFLOAT16_ALIGN_K - 1) & ~(BFLOAT16_ALIGN_K - 1);
 #endif
 
       /* First, we have to move data A to L2 cache */

driver/level3/level3_thread.c

@@ -1,6 +1,6 @@
 /*********************************************************************/
 /* Copyright 2009, 2010 The University of Texas at Austin.           */
-/* Copyright 2023 The OpenBLAS Project.                              */
+/* Copyright 2023, 2025 The OpenBLAS Project.                        */
 /* All rights reserved.                                              */
 /*                                                                   */
 /* Redistribution and use in source and binary forms, with or        */
@@ -216,6 +216,22 @@ typedef struct {
 #define STOP_RPCC(COUNTER)
 #endif
 
+#if defined(BUILD_BFLOAT16)
+#if defined(DYNAMIC_ARCH)
+  #if defined(BGEMM)
+    #define BFLOAT16_ALIGN_K gotoblas->bgemm_align_k
+  #else
+    #define BFLOAT16_ALIGN_K gotoblas->sbgemm_align_k
+  #endif
+#else
+  #if defined(BGEMM)
+    #define BFLOAT16_ALIGN_K BGEMM_ALIGN_K
+  #else
+    #define BFLOAT16_ALIGN_K SBGEMM_ALIGN_K
+  #endif
+#endif
+#endif
+
 static int inner_thread(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, IFLOAT *sa, IFLOAT *sb, BLASLONG mypos){
 
   IFLOAT *buffer[DIVIDE_RATE];
@@ -324,12 +340,8 @@ static int inner_thread(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n,
     
     BLASLONG pad_min_l = min_l;
 
-#if defined(HALF)
-#if defined(DYNAMIC_ARCH)
-    pad_min_l = (min_l + gotoblas->sbgemm_align_k - 1) & ~(gotoblas->sbgemm_align_k-1);
-#else
-    pad_min_l = (min_l + SBGEMM_ALIGN_K - 1) & ~(SBGEMM_ALIGN_K - 1);;
-#endif
+#if defined(BFLOAT16)
+    pad_min_l = (min_l + BFLOAT16_ALIGN_K - 1) & ~(BFLOAT16_ALIGN_K - 1);
 #endif
 
     /* Determine step size in m

driver/others/dynamic_power.c

@@ -70,6 +70,9 @@ static int cpuid(void)
 #endif
 #ifdef POWER_10
     else if (arch >= POWER_10) return CPU_POWER10;
+#endif
+#ifdef POWER_11
+    else if (arch >= POWER_11) return CPU_POWER10;
 #endif
     return CPU_UNKNOWN;
 }
@@ -173,6 +176,13 @@ static  struct {
         .cpu_type       = CPU_POWER10,
     },
 
+    {   /* Power11 */
+        .pvr_mask       = 0xffff0000,
+        .pvr_value      = 0x00820000,
+        .cpu_name       = "POWER11 (raw)",
+        .cpu_type       = CPU_POWER10,
+    },
+
     {   /* End of table, pvr_mask and pvr_value must be zero */
         .pvr_mask       = 0x0,
         .pvr_value      = 0x0,

driver/others/parameter.c

@@ -72,6 +72,11 @@ BLASLONG shgemm_p = DEFAULT_GEMM_P;
 #else
 BLASLONG shgemm_p = SHGEMM_P;
 #endif
+#if BGEMM_P == bgemm_p
+BLASLONG bgemm_p = DEFAULT_GEMM_P;
+#else
+BLASLONG bgemm_p = BGEMM_P;
+#endif
 #if SGEMM_P == sgemm_p
 BLASLONG sgemm_p = DEFAULT_GEMM_P;
 #else
@@ -103,6 +108,11 @@ BLASLONG shgemm_q = DEFAULT_GEMM_Q;
 #else
 BLASLONG shgemm_q = SHGEMM_Q;
 #endif
+#if BGEMM_Q == bgemm_q
+BLASLONG bgemm_q = DEFAULT_GEMM_Q;
+#else
+BLASLONG bgemm_q = BGEMM_Q;
+#endif
 #if SGEMM_Q == sgemm_q
 BLASLONG sgemm_q = DEFAULT_GEMM_Q;
 #else
@@ -134,6 +144,11 @@ BLASLONG shgemm_r = DEFAULT_GEMM_R;
 #else
 BLASLONG shgemm_r = SHGEMM_R;
 #endif
+#if BGEMM_R == bgemm_r
+BLASLONG bgemm_r = DEFAULT_GEMM_R;
+#else
+BLASLONG bgemm_r = BGEMM_R;
+#endif
 #if SGEMM_R == sgemm_r
 BLASLONG sgemm_r = DEFAULT_GEMM_R;
 #else
@@ -541,6 +556,7 @@ void blas_set_parameter(void){
 
 #ifdef BUILD_BFLOAT16
   sbgemm_r = (((BUFFER_SIZE - ((SBGEMM_P * SBGEMM_Q *  4 + GEMM_OFFSET_A + GEMM_ALIGN) & ~GEMM_ALIGN)) / (SBGEMM_Q *  4)) - 15) & ~15;
+  bgemm_r = (((BUFFER_SIZE - ((BGEMM_P * BGEMM_Q *  4 + GEMM_OFFSET_A + GEMM_ALIGN) & ~GEMM_ALIGN)) / (BGEMM_Q *  4)) - 15) & ~15;
 #endif
 #ifdef BUILD_HFLOAT16
   shgemm_r = (((BUFFER_SIZE - ((SHGEMM_P * SHGEMM_Q *  4 + GEMM_OFFSET_A + GEMM_ALIGN) & ~GEMM_ALIGN)) / (SHGEMM_Q *  4)) - 15) & ~15;
@@ -653,6 +669,7 @@ void blas_set_parameter(void){
 
 #ifdef BUILD_BFLOAT16
   sbgemm_r = (((BUFFER_SIZE - ((SBGEMM_P * SBGEMM_Q *  4 + GEMM_OFFSET_A + GEMM_ALIGN) & ~GEMM_ALIGN)) / (SBGEMM_Q *  4)) - 15) & ~15;
+  bgemm_r = (((BUFFER_SIZE - ((BGEMM_P * BGEMM_Q *  4 + GEMM_OFFSET_A + GEMM_ALIGN) & ~GEMM_ALIGN)) / (BGEMM_Q *  4)) - 15) & ~15;
 #endif
 #ifdef BUILD_HFLOAT16
   shgemm_r = (((BUFFER_SIZE - ((SHGEMM_P * SHGEMM_Q *  4 + GEMM_OFFSET_A + GEMM_ALIGN) & ~GEMM_ALIGN)) / (SHGEMM_Q *  4)) - 15) & ~15;

exports/gensymbol

@@ -1,5 +1,33 @@
 #!/bin/sh
 
+###############################################################################
+# Copyright (c) 2025, The OpenBLAS Project
+# All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in
+#    the documentation and/or other materials provided with the
+#    distribution.
+# 3. Neither the name of the OpenBLAS project nor the names of
+#    its contributors may be used to endorse or promote products
+#    derived from this software without specific prior written permission.
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+###############################################################################
+
 # Changelog
 # 2017/09/03 staticfloat
 # Added zsymv and csymv into @lapackobjs2 so they are properly renamed
@@ -51,7 +79,7 @@ blasobjsz="
     zgeadd  dzsum zgemmt zgemmtr"
 
 blasobjs="lsame xerbla"
-bfblasobjs="sbgemm sbgemmt sbgemmtr sbgemv sbdot sbstobf16 sbdtobf16 sbf16tos dbf16tod"
+bfblasobjs="bgemm bgemv sbgemm sbgemmt sbgemmtr sbgemv sbdot sbstobf16 sbdtobf16 sbf16tos dbf16tod"
 hfblasobjs="shgemm"
 cblasobjsc="
     cblas_caxpy cblas_ccopy cblas_cdotc cblas_cdotu cblas_cgbmv cblas_cgemm cblas_cgemv

exports/gensymbol.pl

@@ -1,5 +1,33 @@
 #!/usr/bin/env perl
 
+###############################################################################
+# Copyright (c) 2025, The OpenBLAS Project
+# All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in
+#    the documentation and/or other materials provided with the
+#    distribution.
+# 3. Neither the name of the OpenBLAS project nor the names of
+#    its contributors may be used to endorse or promote products
+#    derived from this software without specific prior written permission.
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+###############################################################################
+
 # Changelog
 # 2017/09/03 staticfloat
 # Added zsymv and csymv into @lapackobjs2 so they are properly renamed
@@ -51,7 +79,7 @@
     zgeadd, dzsum, zgemmt,zgemmtr);
 
 @blasobjs = (lsame, xerbla);
-@bfblasobjs = (sbgemm, sbgemmt, sbgemmtr, sbgemv, sbdot, sbstobf16, sbdtobf16, sbf16tos, dbf16tod);
+@bfblasobjs = (bgemm, bgemv, sbgemm, sbgemmt, sbgemmtr, sbgemv, sbdot, sbstobf16, sbdtobf16, sbf16tos, dbf16tod);
 @hfblasobjs = (shgemm);
 @cblasobjsc = (
     cblas_caxpy, cblas_ccopy, cblas_cdotc, cblas_cdotu, cblas_cgbmv, cblas_cgemm, cblas_cgemv,

getarch.c

@@ -1,5 +1,5 @@
 /*****************************************************************************
-Copyright (c) 2011-2014, The OpenBLAS Project
+Copyright (c) 2011-2014, 2025 The OpenBLAS Project
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
@@ -836,7 +836,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define CORENAME  "POWER9"
 #endif
 
-#if defined(FORCE_POWER10)
+#if defined(FORCE_POWER10) || (FORCE_POWER11)
 #define FORCE
 #define ARCHITECTURE    "POWER"
 #define SUBARCHITECTURE "POWER10"
@@ -1476,7 +1476,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
        "-DL2_SIZE=1048576 -DL2_LINESIZE=64 -DL2_ASSOCIATIVE=16 " \
        "-DDTB_DEFAULT_ENTRIES=64 -DDTB_SIZE=4096 " \
        "-DHAVE_VFPV4 -DHAVE_VFPV3 -DHAVE_VFP -DHAVE_NEON -DHAVE_SVE -DARMV8 " \
-       "-march=armv8.4-a+sve -mtune=neoverse-v1"
+       "-march=armv8.4-a+sve+bf16 -mtune=neoverse-v1"
 #define LIBNAME   "neoversev1"
 #define CORENAME  "NEOVERSEV1"
 #endif

getarch_2nd.c

@@ -1,3 +1,31 @@
+/***************************************************************************
+ * Copyright (c) 2025, The OpenBLAS Project
+ * All rights reserved.
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * 3. Neither the name of the OpenBLAS project nor the names of
+ * its contributors may be used to endorse or promote products
+ * derived from this software without specific prior written permission.
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ * *****************************************************************************/
+
 #include <stdio.h>
 #ifndef BUILD_KERNEL
 #include "config.h"
@@ -17,6 +45,10 @@ typedef unsigned long BLASULONG;
 int main(int argc, char **argv) {
 
   if ( (argc <= 1) || ((argc >= 2) && (*argv[1] == '0'))) {
+    printf("BGEMM_UNROLL_M=%d\n", BGEMM_DEFAULT_UNROLL_M);
+    printf("BGEMM_UNROLL_N=%d\n", BGEMM_DEFAULT_UNROLL_N);
+    printf("BGEMM_UNROLL_M=%d\n", BGEMM_DEFAULT_UNROLL_M);
+    printf("BGEMM_UNROLL_N=%d\n", BGEMM_DEFAULT_UNROLL_N);
     printf("SBGEMM_UNROLL_M=%d\n", SBGEMM_DEFAULT_UNROLL_M);
     printf("SBGEMM_UNROLL_N=%d\n", SBGEMM_DEFAULT_UNROLL_N);
     printf("SHGEMM_UNROLL_M=%d\n", SHGEMM_DEFAULT_UNROLL_M);

interface/CMakeLists.txt

@@ -1,3 +1,30 @@
+###############################################################################
+# Copyright (c) 2025, The OpenBLAS Project
+# All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in
+#    the documentation and/or other materials provided with the
+#    distribution.
+# 3. Neither the name of the OpenBLAS project nor the names of
+#    its contributors may be used to endorse or promote products
+#    derived from this software without specific prior written permission.
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+###############################################################################
 
 include_directories(${PROJECT_SOURCE_DIR})
 include_directories(${PROJECT_BINARY_DIR})
@@ -123,10 +150,13 @@ endif ()
   GenerateNamedObjects("imax.c" "USE_MIN" "i*min" ${CBLAS_FLAG})
 
 if (BUILD_BFLOAT16)
+  GenerateNamedObjects("scal.c" "BGEMM" "bscal" ${CBLAS_FLAG} "" "" true "BFLOAT16")
 	GenerateNamedObjects("bf16dot.c" "" "sbdot" ${CBLAS_FLAG} "" "" true "BFLOAT16")
+	GenerateNamedObjects("gemm.c" "BGEMM" "bgemm" ${CBLAS_FLAG} "" "" true "BFLOAT16")
 	GenerateNamedObjects("gemm.c" "" "sbgemm" ${CBLAS_FLAG} "" "" true "BFLOAT16")
 	GenerateNamedObjects("sbgemmt.c" "" "sbgemmt" ${CBLAS_FLAG} "" "" true "BFLOAT16")
 	GenerateNamedObjects("sbgemmt.c" "RNAME" "sbgemmtr" ${CBLAS_FLAG} "" "" true "BFLOAT16")
+	GenerateNamedObjects("sbgemv.c" "BGEMM" "bgemv" ${CBLAS_FLAG} "" "" true "BFLOAT16")
 	GenerateNamedObjects("sbgemv.c" "" "sbgemv" ${CBLAS_FLAG} "" "" true "BFLOAT16")
 	GenerateNamedObjects("tobf16.c" "SINGLE_PREC" "sbstobf16" ${CBLAS_FLAG} "" "" true "BFLOAT16")
 	GenerateNamedObjects("tobf16.c" "DOUBLE_PREC" "sbdtobf16" ${CBLAS_FLAG} "" "" true "BFLOAT16")

interface/Makefile

@@ -1,3 +1,31 @@
+###############################################################################
+# Copyright (c) 2025, The OpenBLAS Project
+# All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in
+#    the documentation and/or other materials provided with the
+#    distribution.
+# 3. Neither the name of the OpenBLAS project nor the names of
+#    its contributors may be used to endorse or promote products
+#    derived from this software without specific prior written permission.
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+###############################################################################
+
 TOPDIR	= ..
 include $(TOPDIR)/Makefile.system
 
@@ -47,6 +75,9 @@ SBLAS3OBJS    = \
 		sgeadd.$(SUFFIX) sgemmt.$(SUFFIX) sgemmtr.$(SUFFIX)
 
 ifeq ($(BUILD_BFLOAT16),1)
+BBLAS3OBJS     = bgemm.$(SUFFIX)
+BBLAS2OBJS     = bgemv.$(SUFFIX)
+BBLAS1OBJS     = bscal.$(SUFFIX)
 SBBLAS1OBJS    = sbdot.$(SUFFIX)
 SBBLAS2OBJS    = sbgemv.$(SUFFIX)
 SBBLAS3OBJS    = sbgemm.$(SUFFIX) sbgemmt.$(SUFFIX) sbgemmtr.$(SUFFIX)
@@ -289,6 +320,9 @@ CSBLAS3OBJS   = \
 	cblas_sgeadd.$(SUFFIX) cblas_sgemmt.$(SUFFIX) cblas_sgemmtr.$(SUFFIX) cblas_sgemm_batch.$(SUFFIX)
 
 ifeq ($(BUILD_BFLOAT16),1)
+CBBLAS3OBJS = cblas_bgemm.$(SUFFIX)
+CBBLAS2OBJS = cblas_bgemv.$(SUFFIX)
+CBBLAS1OBJS = cblas_bscal.$(SUFFIX)
 CSBBLAS1OBJS = cblas_sbdot.$(SUFFIX)
 CSBBLAS2OBJS = cblas_sbgemv.$(SUFFIX)
 CSBBLAS3OBJS = cblas_sbgemm.$(SUFFIX) cblas_sbgemmt.$(SUFFIX) cblas_sbgemmtr.$(SUFFIX) cblas_sbgemm_batch.$(SUFFIX)
@@ -393,6 +427,9 @@ override CFLAGS += -I.
 SBLAS1OBJS   += $(CSBLAS1OBJS)
 SBLAS2OBJS   += $(CSBLAS2OBJS)
 SBLAS3OBJS   += $(CSBLAS3OBJS)
+BBLAS3OBJS   += $(CBBLAS3OBJS)
+BBLAS2OBJS   += $(CBBLAS2OBJS)
+BBLAS1OBJS   += $(CBBLAS1OBJS)
 SBBLAS1OBJS  += $(CSBBLAS1OBJS)
 SBBLAS2OBJS  += $(CSBBLAS2OBJS)
 SBBLAS3OBJS  += $(CSBBLAS3OBJS)
@@ -412,6 +449,7 @@ SBEXTOBJS     += $(CSBEXTOBJS)
 CBAUXOBJS += $(CXERBLAOBJ)
 endif
 
+BBLASOBJS    = $(BBLAS3OBJS) $(BBLAS2OBJS) $(BBLAS1OBJS)
 SBLASOBJS    = $(SBLAS1OBJS) $(SBLAS2OBJS) $(SBLAS3OBJS)
 SBBLASOBJS   = $(SBBLAS1OBJS) $(SBBLAS2OBJS) $(SBBLAS3OBJS)
 SHBLASOBJS   = $(SHBLAS3OBJS)
@@ -522,7 +560,7 @@ ifneq ($(BUILD_COMPLEX16),1)
 	ZBLASOBJS=
 endif
 
-FUNCOBJS    = $(SBEXTOBJS) $(CXERBLAOBJS) $(SBBLASOBJS) $(SBLASOBJS) $(DBLASOBJS) $(CBLASOBJS) $(ZBLASOBJS) $(SHBLASOBJS)
+FUNCOBJS    = $(SBEXTOBJS) $(CXERBLAOBJS) $(BBLASOBJS) $(SBBLASOBJS) $(SBLASOBJS) $(DBLASOBJS) $(CBLASOBJS) $(ZBLASOBJS) $(SHBLASOBJS)
 
 ifeq ($(EXPRECISION), 1)
 FUNCOBJS   += $(QBLASOBJS) $(XBLASOBJS)
@@ -557,10 +595,10 @@ clean ::
 level1 : $(SBEXTOBJS) $(SBBLAS1OBJS) $(SBLAS1OBJS) $(DBLAS1OBJS) $(QBLAS1OBJS) $(CBLAS1OBJS) $(ZBLAS1OBJS) $(XBLAS1OBJS)
 	$(AR) $(ARFLAGS) -ru $(TOPDIR)/$(LIBNAME) $^
 
-level2 : $(SBBLAS2OBJS) $(SBLAS2OBJS) $(DBLAS2OBJS) $(QBLAS2OBJS) $(CBLAS2OBJS) $(ZBLAS2OBJS) $(XBLAS2OBJS)
+level2 : $(SBBLAS2OBJS) $(BBLAS2OBJS) $(SBLAS2OBJS) $(DBLAS2OBJS) $(QBLAS2OBJS) $(CBLAS2OBJS) $(ZBLAS2OBJS) $(XBLAS2OBJS)
 	$(AR) $(ARFLAGS) -ru $(TOPDIR)/$(LIBNAME) $^
 
-level3 : $(SBBLAS3OBJS) $(SBLAS3OBJS) $(DBLAS3OBJS) $(QBLAS3OBJS) $(CBLAS3OBJS) $(ZBLAS3OBJS) $(XBLAS3OBJS) $(SHBLAS3OBJS)
+level3 : $(SBBLAS3OBJS) $(BBLAS3OBJ) $(SBLAS3OBJS) $(DBLAS3OBJS) $(QBLAS3OBJS) $(CBLAS3OBJS) $(ZBLAS3OBJS) $(XBLAS3OBJS) $(SHBLAS3OBJS)
 	$(AR) $(ARFLAGS) -ru $(TOPDIR)/$(LIBNAME) $^
 
 aux :	$(CBAUXOBJS)
@@ -792,6 +830,8 @@ dsdot.$(SUFFIX) dsdot.$(PSUFFIX) : dsdot.c
 	$(CC) $(CFLAGS) -c $< -o $(@F)
 
 ifeq ($(BUILD_BFLOAT16),1)
+bscal.$(SUFFIX) bscal.$(PSUFFIX) : scal.c
+	$(CC) $(CFLAGS) -DBGEMM -c $< -o $(@F)
 sbdot.$(SUFFIX) sbdot.$(PSUFFIX) : bf16dot.c
 	$(CC) $(CFLAGS) -c $< -o $(@F)
 sbstobf16.$(SUFFIX) sbstobf16.$(PSUFFIX) : tobf16.c
@@ -949,6 +989,8 @@ xgerc.$(SUFFIX) xgerc.$(PSUFFIX) : zger.c
 	$(CC) -c $(CFLAGS) -DCONJ $< -o $(@F)
 
 ifeq ($(BUILD_BFLOAT16),1)
+bgemv.$(SUFFIX) bgemv.$(PSUFFIX) : sbgemv.c
+	$(CC) $(CFLAGS) -DBGEMM -c $< -o $(@F)
 sbgemv.$(SUFFIX) sbgemv.$(PSUFFIX) : sbgemv.c
 	$(CC) $(CFLAGS) -c $< -o $(@F)
 endif
@@ -1311,6 +1353,8 @@ xhpr2.$(SUFFIX) xhpr2.$(PSUFFIX) : zhpr2.c
 	$(CC) -c $(CFLAGS) $< -o $(@F)
 
 ifeq ($(BUILD_BFLOAT16),1)
+bgemm.$(SUFFIX) bgemm.$(PSUFFIX) : gemm.c ../param.h
+	$(CC) -c $(CFLAGS) $< -o $(@F)
 sbgemm.$(SUFFIX) sbgemm.$(PSUFFIX) : gemm.c ../param.h
 	$(CC) -c $(CFLAGS) $< -o $(@F)
 sbgemmt.$(SUFFIX) sbgemmt.$(PSUFFIX) : sbgemmt.c ../param.h
@@ -1619,6 +1663,8 @@ cblas_dsdot.$(SUFFIX) cblas_dsdot.$(PSUFFIX) : dsdot.c
 	$(CC) $(CFLAGS) -DCBLAS -c $< -o $(@F)
 
 ifeq ($(BUILD_BFLOAT16),1)
+cblas_bscal.$(SUFFIX) cblas_bscal.$(PSUFFIX) : scal.c
+	$(CC) $(CFLAGS) -DCBLAS -c $< -o $(@F)
 cblas_sbdot.$(SUFFIX) cblas_sbdot.$(PSUFFIX) : bf16dot.c
 	$(CC) $(CFLAGS) -DCBLAS -c $< -o $(@F)
 cblas_sbstobf16.$(SUFFIX) cblas_sbstobf16.$(PSUFFIX) : tobf16.c
@@ -1773,6 +1819,8 @@ cblas_zdrot.$(SUFFIX) cblas_zdrot.$(PSUFFIX) : zrot.c
 	$(CC) $(CFLAGS) -DCBLAS -c $< -o $(@F)
 
 ifeq ($(BUILD_BFLOAT16),1)
+cblas_bgemv.$(SUFFIX) cblas_bgemv.$(PSUFFIX) : sbgemv.c
+	$(CC) -DCBLAS -DBGEMM -c $(CFLAGS) $< -o $(@F)
 cblas_sbgemv.$(SUFFIX) cblas_sbgemv.$(PSUFFIX) : sbgemv.c
 	$(CC) -DCBLAS -c $(CFLAGS) $< -o $(@F)
 endif
@@ -1979,6 +2027,8 @@ cblas_sgemm.$(SUFFIX) cblas_sgemm.$(PSUFFIX) : gemm.c ../param.h
 	$(CC) -DCBLAS -c $(CFLAGS) $< -o $(@F)
 
 ifeq ($(BUILD_BFLOAT16),1)
+cblas_bgemm.$(SUFFIX) cblas_bgemm.$(PSUFFIX) : gemm.c ../param.h
+	$(CC) -DCBLAS -DBGEMM -c $(CFLAGS) $< -o $(@F)
 cblas_sbgemm.$(SUFFIX) cblas_sbgemm.$(PSUFFIX) : gemm.c ../param.h
 	$(CC) -DCBLAS -c $(CFLAGS) $< -o $(@F)
 endif

interface/gemm.c

@@ -54,8 +54,13 @@
 #define ERROR_NAME "DGEMM "
 #define GEMV BLASFUNC(dgemv)
 #elif defined(BFLOAT16)
+#ifdef BGEMM
+#define ERROR_NAME "BGEMM "
+#define GEMV BLASFUNC(bgemv)
+#else
 #define ERROR_NAME "SBGEMM "
 #define GEMV BLASFUNC(sbgemv)
+#endif
 #elif defined(HFLOAT16)
 #define ERROR_NAME "SHGEMM "
 #else
@@ -113,7 +118,7 @@ static int (*gemm[])(blas_arg_t *, BLASLONG *, BLASLONG *, IFLOAT *, IFLOAT *, B
 #endif
 };
 
-#if defined(SMALL_MATRIX_OPT) && !defined(GEMM3M) && !defined(XDOUBLE) &&!defined(HFLOAT16)
+#if defined(SMALL_MATRIX_OPT) && !defined(GEMM3M) && !defined(XDOUBLE) && !defined(HFLOAT16) && !defined(BGEMM)
 #define USE_SMALL_MATRIX_OPT 1
 #else
 #define USE_SMALL_MATRIX_OPT 0
@@ -436,6 +441,9 @@ void CNAME(enum CBLAS_ORDER order, enum CBLAS_TRANSPOSE TransA, enum CBLAS_TRANS
   if (beta == 0 && alpha == 1.0 && order == CblasRowMajor && TransA == CblasNoTrans && TransB == CblasNoTrans) {
 	SGEMM_DIRECT(m, n, k, a, lda, b, ldb, c, ldc);
 	return;
+  }else if (order == CblasRowMajor && TransA == CblasNoTrans && TransB == CblasNoTrans) {
+	SGEMM_DIRECT_ALPHA_BETA(m, n, k, alpha, a, lda, b, ldb, beta, c, ldc);
+	return;
   }
 #endif
 #endif
@@ -579,7 +587,7 @@ void CNAME(enum CBLAS_ORDER order, enum CBLAS_TRANSPOSE TransA, enum CBLAS_TRANS
 	 args.m, args.n, args.k, args.lda, args.ldb, args.ldc);
 #endif
 
-#if defined(GEMM_GEMV_FORWARD) && !defined(GEMM3M) && !defined(COMPLEX) && !defined(HFLOAT16) && (!defined(BFLOAT16) || defined(GEMM_GEMV_FORWARD_BF16))
+#if defined(GEMM_GEMV_FORWARD) && !defined(GEMM3M) && !defined(COMPLEX) && !defined(HFLOAT16) && (!defined(BFLOAT16) || (!defined(BGEMM) && defined(SBGEMM_GEMV_FORWARD)) || (defined(BGEMM) && defined(BGEMM_GEMV_FORWARD)))
 #if defined(ARCH_ARM64)
   // The gemv kernels in arm64/{gemv_n.S,gemv_n_sve.c,gemv_t.S,gemv_t_sve.c}
   // perform poorly in certain circumstances. We use the following boolean

interface/sbgemmt.c

@@ -1,5 +1,5 @@
 /*********************************************************************/
-/* Copyright 2024, The OpenBLAS Project.                             */
+/* Copyright 2024-2025 The OpenBLAS Project.                         */
 /* All rights reserved.                                              */
 /*                                                                   */
 /* Redistribution and use in source and binary forms, with or        */
@@ -305,7 +305,7 @@ void CNAME(enum CBLAS_ORDER order, enum CBLAS_UPLO Uplo,
 #endif
 	int (*gemv[]) (BLASLONG, BLASLONG, FLOAT, IFLOAT *, BLASLONG,
 		       IFLOAT *, BLASLONG, FLOAT, FLOAT *, BLASLONG) = {
-	SBGEMV_N, SBGEMV_T,};
+	GEMV_N, GEMV_T,};
 
 
 	if (m == 0)

interface/sbgemv.c

@@ -1,4 +1,5 @@
 /*********************************************************************/
+/* Copyright 2025 The OpenBLAS Project.                              */
 /* Copyright 2009, 2010 The University of Texas at Austin.           */
 /* All rights reserved.                                              */
 /*                                                                   */
@@ -43,17 +44,25 @@
 #include "functable.h"
 #endif
 
+#ifdef BGEMM
+#define GEMV_THREAD_N bgemv_thread_n
+#define GEMV_THREAD_T bgemv_thread_t
+#define ERROR_NAME "BGEMV "
+#else
+#define GEMV_THREAD_N sbgemv_thread_n
+#define GEMV_THREAD_T sbgemv_thread_t
 #define ERROR_NAME "SBGEMV "
+#endif 
 
 #ifdef SMP
-static int (*sbgemv_thread[])(BLASLONG, BLASLONG, float, bfloat16 *, BLASLONG, bfloat16 * , BLASLONG, float, float *, BLASLONG, int) = {
-    sbgemv_thread_n, sbgemv_thread_t,
+static int (*gemv_thread[])(BLASLONG, BLASLONG, FLOAT, IFLOAT *, BLASLONG, IFLOAT * , BLASLONG, FLOAT, FLOAT *, BLASLONG, int) = {
+    GEMV_THREAD_N, GEMV_THREAD_T,
 };
 #endif
 
 #ifndef CBLAS
 
-void NAME(char *TRANS, blasint *M, blasint *N, float *ALPHA, bfloat16 *a, blasint *LDA, bfloat16 *x, blasint *INCX, float *BETA, float *y, blasint *INCY)
+void NAME(char *TRANS, blasint *M, blasint *N, FLOAT *ALPHA, IFLOAT *a, blasint *LDA, IFLOAT *x, blasint *INCX, FLOAT *BETA, FLOAT *y, blasint *INCY)
 {
     char trans = *TRANS;
     blasint m = *M;
@@ -61,14 +70,14 @@ void NAME(char *TRANS, blasint *M, blasint *N, float *ALPHA, bfloat16 *a, blasin
     blasint lda = *LDA;
     blasint incx = *INCX;
     blasint incy = *INCY;
-    float alpha = *ALPHA;
-    float beta  = *BETA;
+    FLOAT alpha = *ALPHA;
+    FLOAT beta  = *BETA;
 #ifdef SMP
     int nthreads;
 #endif
 
-    int (*sbgemv[])(BLASLONG, BLASLONG, float, bfloat16 *, BLASLONG, bfloat16 * , BLASLONG, float, float *, BLASLONG) = {
-        SBGEMV_N, SBGEMV_T,
+    int (*gemv[])(BLASLONG, BLASLONG, FLOAT, IFLOAT *, BLASLONG, IFLOAT * , BLASLONG, FLOAT, FLOAT *, BLASLONG) = {
+        GEMV_N, GEMV_T,
     };
 
     blasint info;
@@ -104,7 +113,7 @@ void NAME(char *TRANS, blasint *M, blasint *N, float *ALPHA, bfloat16 *a, blasin
 
 #else
 
-void CNAME(enum CBLAS_ORDER order, enum CBLAS_TRANSPOSE TransA, blasint m, blasint n, float alpha, bfloat16 *a, blasint lda, bfloat16 *x, blasint incx, float beta, float *y, blasint incy)
+void CNAME(enum CBLAS_ORDER order, enum CBLAS_TRANSPOSE TransA, blasint m, blasint n, FLOAT alpha, IFLOAT *a, blasint lda, IFLOAT *x, blasint incx, FLOAT beta, FLOAT *y, blasint incy)
 {
     blasint lenx,  leny;
     int     trans;
@@ -113,8 +122,8 @@ void CNAME(enum CBLAS_ORDER order, enum CBLAS_TRANSPOSE TransA, blasint m, blasi
     int     nthreads;
 #endif
 
-    int (*sbgemv[])(BLASLONG, BLASLONG, float, bfloat16 *, BLASLONG,  bfloat16 * , BLASLONG, float, float *, BLASLONG) = {
-        SBGEMV_N, SBGEMV_T,
+    int (*gemv[])(BLASLONG, BLASLONG, FLOAT, IFLOAT *, BLASLONG,  IFLOAT * , BLASLONG, FLOAT, FLOAT *, BLASLONG) = {
+        GEMV_N, GEMV_T,
     };
 
     PRINT_DEBUG_CNAME;
@@ -166,8 +175,17 @@ void CNAME(enum CBLAS_ORDER order, enum CBLAS_TRANSPOSE TransA, blasint m, blasi
         leny = m;
     }
 
-    if (alpha == ZERO) {
-        if (beta != ONE) SCAL_K(leny, 0, 0, beta, y, blasabs(incy), NULL, 0, NULL, 0);
+#ifdef BGEMM
+  float alpha_float, beta_float;
+  SBF16TOS_K(1, &alpha, 1, &alpha_float, 1);
+  SBF16TOS_K(1, &beta, 1, &beta_float, 1);
+#else
+  float alpha_float = alpha;
+  float beta_float = beta;
+#endif
+
+    if (alpha_float == ZERO) {
+        if (beta_float != ONE) SCAL_K(leny, 0, 0, beta, y, blasabs(incy), NULL, 0, NULL, 0);
         return;
     }
 
@@ -185,10 +203,10 @@ void CNAME(enum CBLAS_ORDER order, enum CBLAS_TRANSPOSE TransA, blasint m, blasi
 
     if (nthreads == 1) {
 #endif
-        (sbgemv[(int)trans])(m, n, alpha, a, lda, x, incx, beta, y, incy);
+        (gemv[(int)trans])(m, n, alpha, a, lda, x, incx, beta, y, incy);
 #ifdef SMP
     } else {
-        (sbgemv_thread[(int)trans])(m, n, alpha, a, lda, x, incx, beta, y, incy, nthreads);
+        (gemv_thread[(int)trans])(m, n, alpha, a, lda, x, incx, beta, y, incy, nthreads);
     }
 #endif
 

interface/scal.c

@@ -1,4 +1,5 @@
 /*********************************************************************/
+/* Copyright 2025 The OpenBLAS Project.                              */
 /* Copyright 2009, 2010 The University of Texas at Austin.           */
 /* All rights reserved.                                              */
 /*                                                                   */
@@ -68,7 +69,14 @@ void CNAME(blasint n, FLOAT alpha, FLOAT *x, blasint incx){
 
   if (incx <= 0 || n <= 0) return;
 
-  if (alpha == ONE) return;
+#ifdef BGEMM
+  float alpha_float;
+  SBF16TOS_K(1, &alpha, 1, &alpha_float, 1);
+#else
+  float alpha_float = alpha;
+#endif
+
+  if (alpha_float == ONE) return;
 
   IDEBUG_START;
 

kernel/CMakeLists.txt

@@ -1,3 +1,30 @@
+###############################################################################
+# Copyright (c) 2025, The OpenBLAS Project
+# All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in
+#    the documentation and/or other materials provided with the
+#    distribution.
+# 3. Neither the name of the OpenBLAS project nor the names of
+#    its contributors may be used to endorse or promote products
+#    derived from this software without specific prior written permission.
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+###############################################################################
 
 include_directories(${PROJECT_SOURCE_DIR})
 
@@ -94,6 +121,7 @@ function (build_core TARGET_CORE KDIR TSUFFIX KERNEL_DEFINITIONS)
 
     # sbdot
     if (BUILD_BFLOAT16)
+      GenerateNamedObjects("${KERNELDIR}/${BSCALKERNEL}" "BGEMM" "scal_k" false "" "" false "BFLOAT16")
 	    GenerateNamedObjects("${KERNELDIR}/${SBDOTKERNEL}" "SBDOT" "dot_k" false "" "" false "BFLOAT16")
 	    GenerateNamedObjects("${KERNELDIR}/${BF16TOKERNEL}" "SINGLE" "f16tos_k" false "" "" false "BFLOAT16")
 	    GenerateNamedObjects("${KERNELDIR}/${BF16TOKERNEL}" "DOUBLE" "bf16tod_k" false "" "" false "DOUBLE")
@@ -195,6 +223,8 @@ function (build_core TARGET_CORE KDIR TSUFFIX KERNEL_DEFINITIONS)
 	    GenerateNamedObjects("${KERNELDIR}/${SGEMVTKERNEL}" "TRANS" "gemv_t" false "" "" false "SINGLE")
     endif ()
     if (BUILD_BFLOAT16)
+	    GenerateNamedObjects("${KERNELDIR}/${BGEMVNKERNEL}" "BGEMM" "gemv_n" false "" "" false "BFLOAT16")
+	    GenerateNamedObjects("${KERNELDIR}/${BGEMVTKERNEL}" "BGEMM" "gemv_t" false "" "" false "BFLOAT16")
 	    GenerateNamedObjects("${KERNELDIR}/${SBGEMVNKERNEL}" "" "gemv_n" false "" "" false "BFLOAT16")
 	    GenerateNamedObjects("${KERNELDIR}/${SBGEMVTKERNEL}" "" "gemv_t" false "" "" false "BFLOAT16")
     endif ()
@@ -211,6 +241,9 @@ function (build_core TARGET_CORE KDIR TSUFFIX KERNEL_DEFINITIONS)
     if (X86_64 OR ARM64)
        set(USE_DIRECT_SGEMM true)
     endif()
+    if (UC_TARGET_CORE MATCHES ARMV9SME)
+          set (HAVE_SME true)
+    endif ()
 
     if (USE_DIRECT_SGEMM)
 	    #	    if (NOT DEFINED SGEMMDIRECTKERNEL)
@@ -222,9 +255,11 @@ function (build_core TARGET_CORE KDIR TSUFFIX KERNEL_DEFINITIONS)
 	  GenerateNamedObjects("${KERNELDIR}/${SGEMMDIRECTPERFORMANT}" "" "gemm_direct_performant" false "" "" false  SINGLE)
 	    elseif (ARM64)
 	  set (SGEMMDIRECTKERNEL sgemm_direct_arm64_sme1.c)
+	  set (SGEMMDIRECTKERNEL_ALPHA_BETA sgemm_direct_alpha_beta_arm64_sme1.c)
 	  set (SGEMMDIRECTSMEKERNEL sgemm_direct_sme1.S)
 	  set (SGEMMDIRECTPREKERNEL sgemm_direct_sme1_preprocess.S)
 	  GenerateNamedObjects("${KERNELDIR}/${SGEMMDIRECTKERNEL}" "" "gemm_direct" false "" "" false SINGLE)
+	  GenerateNamedObjects("${KERNELDIR}/${SGEMMDIRECTKERNEL_ALPHA_BETA}" "" "gemm_direct_alpha_beta" false "" "" false SINGLE)
    	  if (HAVE_SME)
 	  GenerateNamedObjects("${KERNELDIR}/${SGEMMDIRECTSMEKERNEL}" "" "gemm_direct_sme1" false "" "" false SINGLE)
 	  GenerateNamedObjects("${KERNELDIR}/${SGEMMDIRECTPREKERNEL}" "" "gemm_direct_sme1_preprocess" false "" "" false SINGLE)
@@ -336,6 +371,20 @@ function (build_core TARGET_CORE KDIR TSUFFIX KERNEL_DEFINITIONS)
     endif ()
 
     if (BUILD_BFLOAT16)
+        if (BGEMMINCOPY)
+		GenerateNamedObjects("${KERNELDIR}/${BGEMMINCOPY}" "BGEMM" "${BGEMMINCOPYOBJ}" false "" "" true "BFLOAT16")
+        endif ()
+        if (BGEMMITCOPY)
+		GenerateNamedObjects("${KERNELDIR}/${BGEMMITCOPY}" "BGEMM" "${BGEMMITCOPYOBJ}" false "" "" true "BFLOAT16")
+        endif ()
+        if (BGEMMONCOPY)
+		GenerateNamedObjects("${KERNELDIR}/${BGEMMONCOPY}" "BGEMM" "${BGEMMONCOPYOBJ}" false "" "" true "BFLOAT16")
+        endif ()
+        if (BGEMMOTCOPY)
+		GenerateNamedObjects("${KERNELDIR}/${BGEMMOTCOPY}" "BGEMM" "${BGEMMOTCOPYOBJ}" false "" "" true "BFLOAT16")
+        endif ()
+	GenerateNamedObjects("${KERNELDIR}/${BGEMMKERNEL}" "BGEMM" "gemm_kernel" false "" "" false "BFLOAT16")
+	GenerateNamedObjects("${KERNELDIR}/${BGEMM_BETA}" "BGEMM" "gemm_beta" false "" "" false "BFLOAT16")
         if (SBGEMMINCOPY)
 		GenerateNamedObjects("${KERNELDIR}/${SBGEMMINCOPY}" "" "${SBGEMMINCOPYOBJ}" false "" "" true "BFLOAT16")
         endif ()

kernel/Makefile.L1

@@ -1,3 +1,31 @@
+###############################################################################
+# Copyright (c) 2025 The OpenBLAS Project
+# All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in
+#    the documentation and/or other materials provided with the
+#    distribution.
+# 3. Neither the name of the OpenBLAS project nor the names of
+#    its contributors may be used to endorse or promote products
+#    derived from this software without specific prior written permission.
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+###############################################################################
+
 FMAFLAG=
 ifndef OLDGCC
 ifdef HAVE_FMA3
@@ -271,6 +299,10 @@ XDOTKERNEL = zdot.S
 endif
 
 ifeq ($(BUILD_BFLOAT16),1)
+ifndef BSCALKERNEL
+BSCALKERNEL =  ../generic/scal.c
+endif
+
 ifndef SBDOTKERNEL
 SBDOTKERNEL = ../x86_64/sbdot.c
 endif
@@ -551,6 +583,8 @@ XBLASOBJS	+= \
 	xscal_k$(TSUFFIX).$(SUFFIX) xswap_k$(TSUFFIX).$(SUFFIX) xsum_k$(TSUFFIX).$(SUFFIX)
 
 ifeq ($(BUILD_BFLOAT16),1)
+BBLASOBJS += \
+	bscal_k$(TSUFFIX).$(SUFFIX)
 SBBLASOBJS	+= \
 	 sbdot_k$(TSUFFIX).$(SUFFIX)
 SBEXTOBJS        += \
@@ -778,6 +812,8 @@ $(KDIR)qdot_k$(TSUFFIX).$(SUFFIX) $(KDIR)qdot_k$(TPSUFFIX).$(PSUFFIX) : $(KERNEL
 	$(CC) -c $(CFLAGS) -UCOMPLEX -DXDOUBLE $< -o $@
 
 ifeq ($(BUILD_BFLOAT16),1)
+$(KDIR)bscal_k$(TSUFFIX).$(SUFFIX)  $(KDIR)bscal_k$(TPSUFFIX).$(PSUFFIX)  : $(KERNELDIR)/$(BSCALKERNEL)
+	$(CC) -c $(CFLAGS) -UCOMPLEX -UDOUBLE $< -o $@
 $(KDIR)sbdot_k$(TSUFFIX).$(SUFFIX) $(KDIR)sbdot_k$(TPSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SBDOTKERNEL)
 	$(CC) -c $(CFLAGS) -UCOMPLEX $< -o $@
 $(KDIR)sbstobf16_k$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(TOBF16KERNEL)

kernel/Makefile.L2

@@ -1,3 +1,31 @@
+###############################################################################
+# Copyright (c) 2025 The OpenBLAS Project
+# All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in
+#    the documentation and/or other materials provided with the
+#    distribution.
+# 3. Neither the name of the OpenBLAS project nor the names of
+#    its contributors may be used to endorse or promote products
+#    derived from this software without specific prior written permission.
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+###############################################################################
+
 FMAFLAG=
 ifndef OLDGCC
 ifdef HAVE_FMA3
@@ -56,6 +84,14 @@ XGEMVTKERNEL = zgemv_t.S
 endif
 
 ifeq ($(BUILD_BFLOAT16),1)
+ifndef BGEMVNKERNEL
+BGEMVNKERNEL = ../generic/gemv_n.c
+endif
+
+ifndef BGEMVTKERNEL
+BGEMVTKERNEL = ../generic/gemv_t.c
+endif
+
 ifndef SBGEMVNKERNEL
 SBGEMVNKERNEL = ../x86_64/sbgemv_n.c
 endif
@@ -255,6 +291,9 @@ XBLASOBJS	+= \
 	xgeru_k$(TSUFFIX).$(SUFFIX) xgerc_k$(TSUFFIX).$(SUFFIX) xgerv_k$(TSUFFIX).$(SUFFIX) xgerd_k$(TSUFFIX).$(SUFFIX)
 
 ifeq ($(BUILD_BFLOAT16),1)
+BBLASOBJS     += \
+        bgemv_n$(TSUFFIX).$(SUFFIX) \
+        bgemv_t$(TSUFFIX).$(SUFFIX)
 SBBLASOBJS     += \
         sbgemv_n$(TSUFFIX).$(SUFFIX) \
         sbgemv_t$(TSUFFIX).$(SUFFIX)
@@ -513,5 +552,9 @@ $(KDIR)sbgemv_n$(TSUFFIX).$(SUFFIX) $(KDIR)sbgemv_n$(TPSUFFIX).$(PSUFFIX) : $(KE
 	$(CC) -c $(CFLAGS) -UCOMPLEX $< -o $@
 $(KDIR)sbgemv_t$(TSUFFIX).$(SUFFIX) $(KDIR)sbgemv_t$(TPSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SBGEMVTKERNEL)
 	$(CC) -c $(CFLAGS) -UCOMPLEX $< -o $@
+$(KDIR)bgemv_n$(TSUFFIX).$(SUFFIX) $(KDIR)bgemv_n$(TPSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(BGEMVNKERNEL)
+	$(CC) -c $(CFLAGS) -DBGEMM -UCOMPLEX $< -o $@
+$(KDIR)bgemv_t$(TSUFFIX).$(SUFFIX) $(KDIR)bgemv_t$(TPSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(BGEMVTKERNEL)
+	$(CC) -c $(CFLAGS) -DBGEMM -UCOMPLEX $< -o $@
 endif
 

kernel/Makefile.L3

@@ -1,3 +1,30 @@
+###############################################################################
+# Copyright (c) 2025, The OpenBLAS Project
+# All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in
+#    the documentation and/or other materials provided with the
+#    distribution.
+# 3. Neither the name of the OpenBLAS project nor the names of
+#    its contributors may be used to endorse or promote products
+#    derived from this software without specific prior written permission.
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+###############################################################################
 USE_GEMM3M = 0
 OS := $(shell uname)
 
@@ -105,11 +132,29 @@ ifeq ($(TARGET_CORE), ARMV9SME)
 HAVE_SME = 1
 endif
 SGEMMDIRECTKERNEL = sgemm_direct_arm64_sme1.c
+SGEMMDIRECTKERNEL_ALPHA_BETA = sgemm_direct_alpha_beta_arm64_sme1.c
 endif
 endif
 endif
 
 ifeq ($(BUILD_BFLOAT16), 1)
+ifndef BGEMMKERNEL
+BGEMM_BETA = ../generic/gemm_beta.c
+BGEMMKERNEL    = ../generic/gemmkernel_2x2.c
+BGEMMINCOPY    = ../generic/gemm_ncopy_2.c
+BGEMMITCOPY    = ../generic/gemm_tcopy_2.c
+BGEMMONCOPY    = ../generic/gemm_ncopy_2.c
+BGEMMOTCOPY    = ../generic/gemm_tcopy_2.c
+BGEMMINCOPYOBJ =  bgemm_incopy$(TSUFFIX).$(SUFFIX)
+BGEMMITCOPYOBJ =  bgemm_itcopy$(TSUFFIX).$(SUFFIX)
+BGEMMONCOPYOBJ =  bgemm_oncopy$(TSUFFIX).$(SUFFIX)
+BGEMMOTCOPYOBJ =  bgemm_otcopy$(TSUFFIX).$(SUFFIX)
+endif
+BKERNELOBJS += \
+	bgemm_kernel$(TSUFFIX).$(SUFFIX) \
+	$(BGEMMINCOPYOBJ) $(BGEMMITCOPYOBJ) \
+	$(BGEMMONCOPYOBJ) $(BGEMMOTCOPYOBJ)	
+
 ifndef SBGEMMKERNEL
 SBGEMM_BETA = ../generic/gemm_beta.c
 SBGEMMKERNEL    = ../generic/gemmkernel_2x2.c
@@ -164,7 +209,8 @@ SKERNELOBJS += \
 endif
 ifeq ($(ARCH), arm64)
 SKERNELOBJS += \
-	sgemm_direct$(TSUFFIX).$(SUFFIX)
+	sgemm_direct$(TSUFFIX).$(SUFFIX) \
+	sgemm_direct_alpha_beta$(TSUFFIX).$(SUFFIX)
 ifdef HAVE_SME
 SKERNELOBJS += \
 	sgemm_direct_sme1$(TSUFFIX).$(SUFFIX) \
@@ -210,6 +256,7 @@ XKERNELOBJS	+= \
 	$(XGEMMONCOPYOBJ) $(XGEMMOTCOPYOBJ)
 
 ifeq ($(BUILD_BFLOAT16),1)
+BBLASOBJS += $(BKERNELOBJS)
 SBBLASOBJS      += $(SBKERNELOBJS)
 endif
 ifeq ($(BUILD_HFLOAT16),1)
@@ -223,6 +270,7 @@ ZBLASOBJS	+= $(ZKERNELOBJS)
 XBLASOBJS	+= $(XKERNELOBJS)
 
 ifeq ($(BUILD_BFLOAT16),1)
+BBLASOBJS += bgemm_beta$(TSUFFIX).$(SUFFIX)
 SBBLASOBJS += sbgemm_beta$(TSUFFIX).$(SUFFIX)
 endif
 ifeq ($(BUILD_HFLOAT16),1)
@@ -628,6 +676,10 @@ ZBLASOBJS += \
 endif
 
 ifeq ($(BUILD_BFLOAT16), 1)
+BGEMMINCOPYOBJ_P = $(BGEMMINCOPYOBJ:.$(SUFFIX)=.$(PSUFFIX))
+BGEMMITCOPYOBJ_P = $(BGEMMITCOPYOBJ:.$(SUFFIX)=.$(PSUFFIX))
+BGEMMONCOPYOBJ_P = $(BGEMMONCOPYOBJ:.$(SUFFIX)=.$(PSUFFIX))
+BGEMMOTCOPYOBJ_P = $(BGEMMOTCOPYOBJ:.$(SUFFIX)=.$(PSUFFIX))
 SBGEMMINCOPYOBJ_P = $(SBGEMMINCOPYOBJ:.$(SUFFIX)=.$(PSUFFIX))
 SBGEMMITCOPYOBJ_P = $(SBGEMMITCOPYOBJ:.$(SUFFIX)=.$(PSUFFIX))
 SBGEMMONCOPYOBJ_P = $(SBGEMMONCOPYOBJ:.$(SUFFIX)=.$(PSUFFIX))
@@ -667,6 +719,8 @@ XGEMMONCOPYOBJ_P = $(XGEMMONCOPYOBJ:.$(SUFFIX)=.$(PSUFFIX))
 XGEMMOTCOPYOBJ_P = $(XGEMMOTCOPYOBJ:.$(SUFFIX)=.$(PSUFFIX))
 
 ifeq ($(BUILD_BFLOAT16),1)
+$(KDIR)bgemm_beta$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(BGEMM_BETA)
+	$(CC) $(CFLAGS) -c -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX $< -o $@
 $(KDIR)sbgemm_beta$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SBGEMM_BETA)
 	$(CC) $(CFLAGS) -c -DBFLOAT16 -UDOUBLE -UCOMPLEX $< -o $@
 endif
@@ -698,9 +752,22 @@ ifeq ($(ARCH), E2K)
 USE_TRMM = 1
 endif
 
-
 ifeq ($(BUILD_BFLOAT16), 1)
 
+$(KDIR)$(BGEMMONCOPYOBJ) : $(KERNELDIR)/$(BGEMMONCOPY)
+	$(CC) $(CFLAGS) -c -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX $< -o $@
+
+$(KDIR)$(BGEMMOTCOPYOBJ) : $(KERNELDIR)/$(BGEMMOTCOPY)
+	$(CC) $(CFLAGS) -c -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX $< -o $@
+
+ifneq ($(BGEMM_UNROLL_M), $(BGEMM_UNROLL_N))
+$(KDIR)$(BGEMMINCOPYOBJ) : $(KERNELDIR)/$(BGEMMINCOPY)
+	$(CC) $(CFLAGS) -c -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX $< -o $@
+
+$(KDIR)$(BGEMMITCOPYOBJ) : $(KERNELDIR)/$(BGEMMITCOPY)
+	$(CC) $(CFLAGS) -c -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX $< -o $@
+endif
+
 $(KDIR)$(SBGEMMONCOPYOBJ) : $(KERNELDIR)/$(SBGEMMONCOPY)
 	$(CC) $(CFLAGS) -c -DBFLOAT16 -UDOUBLE -UCOMPLEX $< -o $@
 
@@ -904,6 +971,8 @@ endif
 ifeq ($(ARCH), arm64)
 $(KDIR)sgemm_direct$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMMDIRECTKERNEL)
 	$(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX $< -o $@
+$(KDIR)sgemm_direct_alpha_beta$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMMDIRECTKERNEL_ALPHA_BETA)
+	$(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX $< -o $@
 ifdef HAVE_SME
 $(KDIR)sgemm_direct_sme1$(TSUFFIX).$(SUFFIX) : 
 	$(CC) $(CFLAGS) -c $(KERNELDIR)/sgemm_direct_sme1.S -UDOUBLE -UCOMPLEX  -o $@
@@ -914,7 +983,8 @@ endif
 endif
 
 ifeq ($(BUILD_BFLOAT16), 1)
-
+$(KDIR)bgemm_kernel$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(BGEMMKERNEL) 
+	$(CC) $(CFLAGS) -c -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX $< -o $@
 $(KDIR)sbgemm_kernel$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SBGEMMKERNEL) $(SBGEMMDEPEND)
 	$(CC) $(CFLAGS) -c -DBFLOAT16 -UDOUBLE -UCOMPLEX $< -o $@
 endif
@@ -2908,6 +2978,8 @@ $(KDIR)sgemm_beta$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SGEMM_BETA)
 	$(CC) $(PFLAGS) -c -UDOUBLE -UCOMPLEX $< -o $@
 
 ifeq ($(BUILD_BFLOAT16),1)
+$(KDIR)bgemm_beta$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(BGEMM_BETA)
+	$(CC) $(PFLAGS) -c -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX $< -o $@
 $(KDIR)sbgemm_beta$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SBGEMM_BETA)
 	$(CC) $(PFLAGS) -c -DBFLOAT16 -UDOUBLE -UCOMPLEX $< -o $@
 endif
@@ -2934,6 +3006,20 @@ $(KDIR)xgemm_beta$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(XGEMM_BETA)
 
 
 ifeq ($(BUILD_BFLOAT16), 1)
+$(BGEMMONCOPYOBJ_P) : $(KERNELDIR)/$(BGEMMONCOPY)
+	$(CC) $(PFLAGS) -c -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX $< -o $@
+
+$(BGEMMOTCOPYOBJ_P) : $(KERNELDIR)/$(BGEMMOTCOPY)
+	$(CC) $(PFLAGS) -c -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX $< -o $@
+
+ifneq ($(BGEMM_UNROLL_M), $(BGEMM_UNROLL_N))
+$(BGEMMINCOPYOBJ_P) : $(KERNELDIR)/$(BGEMMINCOPY)
+	$(CC) $(PFLAGS) -c -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX $< -o $@
+
+$(BGEMMITCOPYOBJ_P) : $(KERNELDIR)/$(BGEMMITCOPY)
+	$(CC) $(PFLAGS) -c -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX $< -o $@
+endif
+
 $(SBGEMMONCOPYOBJ_P) : $(KERNELDIR)/$(SBGEMMONCOPY)
 	$(CC) $(PFLAGS) -c -DBFLOAT16 -UDOUBLE -UCOMPLEX $< -o $@
 
@@ -2946,7 +3032,6 @@ $(SBGEMMINCOPYOBJ_P) : $(KERNELDIR)/$(SBGEMMINCOPY)
 
 $(SBGEMMITCOPYOBJ_P) : $(KERNELDIR)/$(SBGEMMITCOPY)
 	$(CC) $(PFLAGS) -c -DBFLOAT16 -UDOUBLE -UCOMPLEX $< -o $@
-
 endif
 endif
 
@@ -3073,6 +3158,8 @@ endif
 
 
 ifeq ($(BUILD_BFLOAT16), 1)
+$(KDIR)bgemm_kernel$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(BGEMMKERNEL) $(BGEMMDEPEND)
+	$(CC) $(PFLAGS) -c -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX $< -o $@
 $(KDIR)sbgemm_kernel$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SBGEMMKERNEL) $(SBGEMMDEPEND)
 	$(CC) $(PFLAGS) -c -DBFLOAT16 -UDOUBLE -UCOMPLEX $< -o $@
 endif

kernel/arm64/KERNEL.A64FX

@@ -4,3 +4,6 @@ SGEMVNKERNEL = gemv_n_sve_v4x3.c
 DGEMVNKERNEL = gemv_n_sve_v4x3.c
 SGEMVTKERNEL = gemv_t_sve_v4x3.c
 DGEMVTKERNEL = gemv_t_sve_v4x3.c
+
+DDOTKERNEL   = dot_sve_v8.c
+SDOTKERNEL   = dot_sve_v8.c

kernel/arm64/KERNEL.NEOVERSEV1

@@ -1,3 +1,31 @@
+###############################################################################
+# Copyright (c) 2025, The OpenBLAS Project
+# All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in
+#    the documentation and/or other materials provided with the
+#    distribution.
+# 3. Neither the name of the OpenBLAS project nor the names of
+#    its contributors may be used to endorse or promote products
+#    derived from this software without specific prior written permission.
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+###############################################################################
+
 include $(KERNELDIR)/KERNEL.ARMV8SVE
 
 SGEMVNKERNEL = gemv_n_sve_v1x3.c
@@ -5,6 +33,19 @@ DGEMVNKERNEL = gemv_n_sve_v1x3.c
 SGEMVTKERNEL = gemv_t_sve_v1x3.c
 DGEMVTKERNEL = gemv_t_sve_v1x3.c
 ifeq ($(BUILD_BFLOAT16), 1)
+BGEMM_BETA    =  bgemm_beta_neon.c
+BGEMMKERNEL    = bgemm_kernel_$(BGEMM_UNROLL_M)x$(BGEMM_UNROLL_N)_neoversev1.c
+ifneq ($(BGEMM_UNROLL_M), $(BGEMM_UNROLL_N))
+BGEMMINCOPY    = sbgemm_ncopy_$(SBGEMM_UNROLL_M)_neoversev1.c
+BGEMMITCOPY    = sbgemm_tcopy_$(SBGEMM_UNROLL_M)_neoversev1.c
+BGEMMINCOPYOBJ =  bgemm_incopy$(TSUFFIX).$(SUFFIX)
+BGEMMITCOPYOBJ =  bgemm_itcopy$(TSUFFIX).$(SUFFIX)
+endif
+BGEMMONCOPY    = sbgemm_ncopy_$(BGEMM_UNROLL_N)_neoversev1.c
+BGEMMOTCOPY    = sbgemm_tcopy_$(BGEMM_UNROLL_N)_neoversev1.c
+BGEMMONCOPYOBJ =  bgemm_oncopy$(TSUFFIX).$(SUFFIX)
+BGEMMOTCOPYOBJ =  bgemm_otcopy$(TSUFFIX).$(SUFFIX)
+
 SBGEMM_BETA    =  sbgemm_beta_neoversev1.c
 SBGEMMKERNEL    = sbgemm_kernel_$(SBGEMM_UNROLL_M)x$(SBGEMM_UNROLL_N)_neoversev1.c
 ifneq ($(SBGEMM_UNROLL_M), $(SBGEMM_UNROLL_N))
@@ -21,4 +62,4 @@ SBGEMMOTCOPYOBJ =  sbgemm_otcopy$(TSUFFIX).$(SUFFIX)
 SBGEMVNKERNEL = sbgemv_n_neon.c
 SBGEMVTKERNEL = sbgemv_t_bfdot.c
 
-endif
+endif

kernel/arm64/bgemm_beta_neon.c

@@ -0,0 +1,107 @@
+/***************************************************************************
+ * Copyright (c) 2025, The OpenBLAS Project
+ * All rights reserved.
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * 3. Neither the name of the OpenBLAS project nor the names of
+ * its contributors may be used to endorse or promote products
+ * derived from this software without specific prior written permission.
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ * *****************************************************************************/
+
+#include "common.h"
+
+#include <arm_neon.h>
+
+int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT beta_in, IFLOAT *dummy2,
+          BLASLONG dummy3, IFLOAT *dummy4, BLASLONG dummy5, FLOAT *c,
+          BLASLONG ldc) {
+    BLASLONG i, j;
+    BLASLONG chunk, remain;
+    
+    bfloat16_t *ptr_c, *ptr_c0;
+
+    bfloat16x8_t x0, z0;
+    float32x4_t y0, y1;
+      
+    float x;
+    bfloat16_t z;
+
+    bfloat16_t zero_bf16 = vcvth_bf16_f32(0.0f); 
+    bfloat16x8_t zeros = vdupq_n_bf16(zero_bf16);
+
+    bfloat16_t beta_bf16;
+    memcpy(&beta_bf16, &beta_in, sizeof(bfloat16_t));
+    float beta = vcvtah_f32_bf16(beta_bf16);
+    float32x4_t beta_neon = vdupq_n_f32(beta);
+
+    ptr_c = (bfloat16_t *)c;
+    
+    chunk = m >> 3;
+    remain = m & 7;
+
+    if (beta == 0.0f){
+      for (j = 0; j < n; j ++){
+        ptr_c0 = ptr_c;
+        ptr_c += ldc;
+
+        for (i = 0; i < chunk; i ++){
+          vst1q_bf16(ptr_c0, zeros);
+          ptr_c0 += 8;
+        }
+
+        for (i = 0; i < remain; i ++){
+          ptr_c0[0] = zero_bf16;
+          ptr_c0 ++;
+        }
+      }
+    } else {
+      for (j = 0; j < n; j ++){
+        ptr_c0 = ptr_c;
+        ptr_c += ldc;
+
+        for (i = 0; i < chunk; i ++){
+          x0 = vld1q_bf16(ptr_c0);
+
+          y0 = vcvtq_low_f32_bf16(x0);
+          y1 = vcvtq_high_f32_bf16(x0);
+
+          y0 = vmulq_f32(y0, beta_neon);
+          y1 = vmulq_f32(y1, beta_neon);
+
+          z0 = vcvtq_low_bf16_f32(y0);
+          z0 = vcvtq_high_bf16_f32(z0, y1);
+          
+          vst1q_bf16(ptr_c0, z0);
+
+          ptr_c0 += 8;
+        }
+
+        for (i = 0; i < remain; i ++){
+          x = vcvtah_f32_bf16(ptr_c0[0]);
+          z = vcvth_bf16_f32(x * beta);
+
+          ptr_c0[0] = z;
+          ptr_c0 ++;
+        }
+      } 
+    }
+    return 0;
+};

kernel/arm64/bgemm_kernel_4x4_neoversev1.c

@@ -0,0 +1,50 @@
+/***************************************************************************
+ * Copyright (c) 2025, The OpenBLAS Project
+ * All rights reserved.
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * 3. Neither the name of the OpenBLAS project nor the names of
+ * its contributors may be used to endorse or promote products
+ * derived from this software without specific prior written permission.
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ * *****************************************************************************/
+
+#include <arm_sve.h>
+
+#include "common.h"
+
+#define ALPHA_ONE
+#include "bgemm_kernel_4x4_neoversev1_impl.c"
+#undef ALPHA_ONE
+#undef UPDATE_C
+#include "bgemm_kernel_4x4_neoversev1_impl.c"
+
+int CNAME(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT *A, IFLOAT *B,
+          FLOAT *C, BLASLONG ldc) {
+  bfloat16_t alpha_bf16;
+  memcpy(&alpha_bf16, &alpha, sizeof(bfloat16_t));
+  float alpha_f32 = vcvtah_f32_bf16(alpha_bf16);
+
+  if (alpha_f32 == 1.0f)
+    return bgemm_kernel_neoversev1_alpha_one(m, n, k, alpha, A, B, C, ldc);
+  else
+    return bgemm_kernel_neoversev1_alpha(m, n, k, alpha, A, B, C, ldc);
+  return 0;
+}

kernel/arm64/bgemm_kernel_4x4_neoversev1_impl.c

@@ -0,0 +1,429 @@
+/***************************************************************************
+ * Copyright (c) 2025, The OpenBLAS Project
+ * All rights reserved.
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * 3. Neither the name of the OpenBLAS project nor the names of
+ * its contributors may be used to endorse or promote products
+ * derived from this software without specific prior written permission.
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ * *****************************************************************************/
+
+#include <arm_sve.h>
+
+#include "common.h"
+
+#define INIT_C(M, N) mc##M##N = svdup_f32(0);
+
+#define MATMUL(M, N) mc##M##N = svbfmmla(mc##M##N, ma##M, mb##N);
+
+#define INIT_C_4x4                                                             \
+  do {                                                                         \
+    INIT_C(0, 0);                                                              \
+    INIT_C(0, 1);                                                              \
+    INIT_C(1, 0);                                                              \
+    INIT_C(1, 1);                                                              \
+  } while (0);
+
+#ifdef ALPHA_ONE
+#define UPDATE_C(PG16, PG32, PTR, TMP32, TMP16, SRC32)                         \
+  do {                                                                         \
+    TMP32 = svreinterpret_f32_u32(svld1uh_u32((PG16), (uint16_t*)PTR));        \
+    TMP32 = svadd_z((PG32), SRC32, TMP32);                                     \
+    TMP16 = svcvt_bf16_f32_z((PG32), TMP32);                                   \
+    TMP16 = svuzp1_bf16(TMP16, TMP16);                                         \
+    svst1_bf16((PG16), (PTR), TMP16);                                          \
+  } while (0)
+#else
+#define UPDATE_C(PG16, PG32, PTR, TMP32, TMP16, SRC32)                         \
+  do {                                                                         \
+    TMP32 = svreinterpret_f32_u32(svld1uh_u32((PG16), (uint16_t*)PTR));        \
+    TMP32 = svmad_z((PG32), svalpha, SRC32, TMP32);                            \
+    TMP16 = svcvt_bf16_f32_z((PG32), TMP32);                                   \
+    TMP16 = svuzp1_bf16(TMP16, TMP16);                                         \
+    svst1_bf16((PG16), (PTR), TMP16);                                          \
+  } while (0)
+#endif
+
+#define ZIP_EVEN_ELEMENTS(PG, mc0, mc1, tmp, vc)                               \
+  do {                                                                         \
+    (tmp) = svuzp1_f32((mc0), (mc1));                                          \
+    (vc) = svcompact_f32((PG), (tmp));                                         \
+  } while (0)
+
+#define ZIP_ODD_ELEMENTS(PG, mc0, mc1, tmp, vc)                                \
+  do {                                                                         \
+    (tmp) = svuzp2_f32((mc0), (mc1));                                          \
+    (vc) = svcompact_f32((PG), (tmp));                                         \
+  } while (0)
+
+#define ACCUMULATE_LAST4_TO_FIRST4(M, N, TMP)                                  \
+  do {                                                                         \
+    TMP = svext_f32(mc##M##N, mc##M##N, 4);                                    \
+    mc##M##N = svadd_f32_z(svptrue_b32(), mc##M##N, (TMP));                    \
+  } while (0)
+
+#ifdef ALPHA_ONE
+int bgemm_kernel_neoversev1_alpha_one(BLASLONG m, BLASLONG n, BLASLONG k,
+                                       FLOAT alpha, IFLOAT *A, IFLOAT *B,
+                                       FLOAT *C, BLASLONG ldc)
+#else
+int bgemm_kernel_neoversev1_alpha(BLASLONG m, BLASLONG n, BLASLONG k,
+                                   FLOAT alpha, IFLOAT *A, IFLOAT *B, FLOAT *C,
+                                   BLASLONG ldc)
+#endif
+{
+  BLASLONG pad_k = (k + 7) & ~7;
+  svbfloat16_t ma0, ma1, mb0, mb1;
+  svfloat32_t mc00, mc01, mc10, mc11, vc0, vc1, vc2, vc3;
+  svfloat32_t tmp;
+#ifndef ALPHA_ONE
+  bfloat16_t alpha_bf16;
+  memcpy(&alpha_bf16, &alpha, sizeof(bfloat16_t));
+  svfloat32_t svalpha = svdup_f32(vcvtah_f32_bf16(alpha_bf16));
+#endif
+
+  svbool_t pg16_all = svptrue_b16();
+
+  svbool_t pg32_first_1 = svwhilelt_b32(0, 1);
+  svbool_t pg32_first_2 = svwhilelt_b32(0, 2);
+  svbool_t pg32_first_4 = svwhilelt_b32(0, 4);
+
+  svbool_t pg16_first_1 = svwhilelt_b16(0, 1);
+  svbool_t pg16_first_2 = svwhilelt_b16(0, 2);
+  svbool_t pg16_first_4 = svwhilelt_b16(0, 4);
+
+  svbool_t pg32_select_first_2_per_quadword = svdupq_b32(1, 1, 0, 0);
+
+  bfloat16_t *ptr_a = (bfloat16_t *)A;
+  bfloat16_t *ptr_b = (bfloat16_t *)B;
+  bfloat16_t *ptr_c = (bfloat16_t *)C;
+
+  bfloat16_t *ptr_a0;
+  bfloat16_t *ptr_b0;
+  bfloat16_t *ptr_c0, *ptr_c1, *ptr_c2, *ptr_c3;
+
+  svfloat32_t tmp32;
+  svbfloat16_t tmp16;
+
+  for (BLASLONG j = 0; j < n / 4; j++) {
+    ptr_c0 = ptr_c;
+    ptr_c1 = ptr_c0 + ldc;
+    ptr_c2 = ptr_c1 + ldc;
+    ptr_c3 = ptr_c2 + ldc;
+    ptr_c += 4 * ldc;
+    ptr_a = (bfloat16_t *)A;
+
+    for (BLASLONG i = 0; i < m / 4; i++) {
+      ptr_a0 = ptr_a;
+      ptr_a += 4 * pad_k;
+
+      ptr_b0 = ptr_b;
+
+      INIT_C_4x4;
+
+      for (BLASLONG p = 0; p < pad_k; p += 8) {
+        ma0 = svld1_bf16(pg16_all, ptr_a0);
+        ma1 = svld1_bf16(pg16_all, ptr_a0 + 16);
+
+        mb0 = svld1_bf16(pg16_all, ptr_b0);
+        mb1 = svld1_bf16(pg16_all, ptr_b0 + 16);
+
+        MATMUL(0, 0);
+        MATMUL(0, 1);
+        MATMUL(1, 0);
+        MATMUL(1, 1);
+
+        ptr_a0 += 32;
+        ptr_b0 += 32;
+      }
+
+      ACCUMULATE_LAST4_TO_FIRST4(0, 0, tmp);
+      ACCUMULATE_LAST4_TO_FIRST4(0, 1, tmp);
+      ACCUMULATE_LAST4_TO_FIRST4(1, 0, tmp);
+      ACCUMULATE_LAST4_TO_FIRST4(1, 1, tmp);
+
+      ZIP_EVEN_ELEMENTS(pg32_select_first_2_per_quadword, mc00, mc10, tmp, vc0);
+      ZIP_ODD_ELEMENTS(pg32_select_first_2_per_quadword, mc00, mc10, tmp, vc1);
+
+      ZIP_EVEN_ELEMENTS(pg32_select_first_2_per_quadword, mc01, mc11, tmp, vc2);
+      ZIP_ODD_ELEMENTS(pg32_select_first_2_per_quadword, mc01, mc11, tmp, vc3);
+
+      UPDATE_C(pg16_first_4, pg32_first_4, ptr_c0, tmp32, tmp16, vc0);
+      UPDATE_C(pg16_first_4, pg32_first_4, ptr_c1, tmp32, tmp16, vc1);
+      UPDATE_C(pg16_first_4, pg32_first_4, ptr_c2, tmp32, tmp16, vc2);
+      UPDATE_C(pg16_first_4, pg32_first_4, ptr_c3, tmp32, tmp16, vc3);
+
+      ptr_c0 += 4;
+      ptr_c1 += 4;
+      ptr_c2 += 4;
+      ptr_c3 += 4;
+    }
+
+    if (m & 2) {
+      ptr_a0 = ptr_a;
+      ptr_a += 2 * pad_k;
+
+      ptr_b0 = ptr_b;
+      INIT_C(0, 0);
+      INIT_C(0, 1);
+      for (BLASLONG p = 0; p < pad_k; p += 8) {
+        ma0 = svld1_bf16(pg16_all, ptr_a0);
+        mb0 = svld1_bf16(pg16_all, ptr_b0);
+        mb1 = svld1_bf16(pg16_all, ptr_b0 + 16);
+
+        MATMUL(0, 0);
+        MATMUL(0, 1);
+
+        ptr_a0 += 16;
+        ptr_b0 += 32;
+      }
+
+      ACCUMULATE_LAST4_TO_FIRST4(0, 0, tmp);
+      ACCUMULATE_LAST4_TO_FIRST4(0, 1, tmp);
+
+      vc0 = svuzp1(mc00, mc00);
+      vc1 = svuzp2(mc00, mc00);
+      vc2 = svuzp1(mc01, mc01);
+      vc3 = svuzp2(mc01, mc01);
+
+      UPDATE_C(pg16_first_2, pg32_first_2, ptr_c0, tmp32, tmp16, vc0);
+      UPDATE_C(pg16_first_2, pg32_first_2, ptr_c1, tmp32, tmp16, vc1);
+      UPDATE_C(pg16_first_2, pg32_first_2, ptr_c2, tmp32, tmp16, vc2);
+      UPDATE_C(pg16_first_2, pg32_first_2, ptr_c3, tmp32, tmp16, vc3);
+
+      ptr_c0 += 2;
+      ptr_c1 += 2;
+      ptr_c2 += 2;
+      ptr_c3 += 2;
+    }
+
+    if (m & 1) {
+      ptr_a0 = ptr_a;
+      ptr_b0 = ptr_b;
+
+      INIT_C(0, 0);
+      INIT_C(0, 1);
+      for (BLASLONG p = 0; p < pad_k; p += 8) {
+        ma0 = svld1_bf16(pg16_all, ptr_a0);
+        mb0 = svld1_bf16(pg16_all, ptr_b0);
+        mb1 = svld1_bf16(pg16_all, ptr_b0 + 16);
+
+        MATMUL(0, 0);
+        MATMUL(0, 1);
+
+        ptr_a0 += 16;
+        ptr_b0 += 32;
+      }
+
+      ACCUMULATE_LAST4_TO_FIRST4(0, 0, tmp);
+      ACCUMULATE_LAST4_TO_FIRST4(0, 1, tmp);
+
+      // use compact is more straightforward
+      vc1 = svuzp2(mc00, mc00);
+      vc3 = svuzp2(mc01, mc01);
+
+      UPDATE_C(pg16_first_1, pg32_first_1, ptr_c0, tmp32, tmp16, mc00);
+      UPDATE_C(pg16_first_1, pg32_first_1, ptr_c1, tmp32, tmp16, vc1);
+      UPDATE_C(pg16_first_1, pg32_first_1, ptr_c2, tmp32, tmp16, mc01);
+      UPDATE_C(pg16_first_1, pg32_first_1, ptr_c3, tmp32, tmp16, vc3);
+    }
+
+    ptr_b += 4 * pad_k;
+  }
+
+  if (n & 2) {
+    ptr_c0 = ptr_c;
+    ptr_c1 = ptr_c0 + ldc;
+    ptr_c += 2 * ldc;
+    ptr_a = (bfloat16_t *)A;
+
+    for (BLASLONG i = 0; i < m / 4; i++) {
+      ptr_a0 = ptr_a;
+      ptr_a += 4 * pad_k;
+
+      ptr_b0 = ptr_b;
+
+      INIT_C(0, 0);
+      INIT_C(1, 0);
+
+      for (BLASLONG p = 0; p < pad_k; p += 8) {
+        ma0 = svld1_bf16(pg16_all, ptr_a0);
+        ma1 = svld1_bf16(pg16_all, ptr_a0 + 16);
+
+        mb0 = svld1_bf16(pg16_all, ptr_b0);
+
+        MATMUL(0, 0);
+        MATMUL(1, 0);
+
+        ptr_a0 += 32;
+        ptr_b0 += 16;
+      }
+
+      ACCUMULATE_LAST4_TO_FIRST4(0, 0, tmp);
+      ACCUMULATE_LAST4_TO_FIRST4(1, 0, tmp);
+
+      ZIP_EVEN_ELEMENTS(pg32_select_first_2_per_quadword, mc00, mc10, tmp, vc0);
+      ZIP_ODD_ELEMENTS(pg32_select_first_2_per_quadword, mc00, mc10, tmp, vc2);
+
+      UPDATE_C(pg16_first_4, pg32_first_4, ptr_c0, tmp32, tmp16, vc0);
+      UPDATE_C(pg16_first_4, pg32_first_4, ptr_c1, tmp32, tmp16, vc2);
+
+      ptr_c0 += 4;
+      ptr_c1 += 4;
+    }
+
+    if (m & 2) {
+      ptr_a0 = ptr_a;
+      ptr_a += 2 * pad_k;
+      ptr_b0 = ptr_b;
+
+      INIT_C(0, 0);
+
+      for (BLASLONG p = 0; p < pad_k; p += 8) {
+        ma0 = svld1_bf16(pg16_all, ptr_a0);
+        mb0 = svld1_bf16(pg16_all, ptr_b0);
+
+        MATMUL(0, 0);
+
+        ptr_a0 += 16;
+        ptr_b0 += 16;
+      }
+
+      ACCUMULATE_LAST4_TO_FIRST4(0, 0, tmp);
+      vc0 = svuzp1(mc00, mc00);
+      vc1 = svuzp2(mc00, mc00);
+
+      
+      UPDATE_C(pg16_first_2, pg32_first_2, ptr_c0, tmp32, tmp16, vc0);
+      UPDATE_C(pg16_first_2, pg32_first_2, ptr_c1, tmp32, tmp16, vc1);
+
+      ptr_c0 += 2;
+      ptr_c1 += 2;
+    }
+
+    if (m & 1) {
+      ptr_a0 = ptr_a;
+      ptr_b0 = ptr_b;
+      INIT_C(0, 0);
+      for (BLASLONG p = 0; p < pad_k; p += 8) {
+        ma0 = svld1_bf16(pg16_all, ptr_a0);
+        mb0 = svld1_bf16(pg16_all, ptr_b0);
+        MATMUL(0, 0);
+        ptr_a0 += 16;
+        ptr_b0 += 16;
+      }
+
+      ACCUMULATE_LAST4_TO_FIRST4(0, 0, tmp);
+      vc1 = svuzp2(mc00, mc00);
+
+      UPDATE_C(pg16_first_1, pg32_first_1, ptr_c0, tmp32, tmp16, mc00);
+      UPDATE_C(pg16_first_1, pg32_first_1, ptr_c1, tmp32, tmp16, vc1);
+    }
+
+    ptr_b += 2 * pad_k;
+  }
+
+  if (n & 1) { // TODO: this case seems a overhead. find out whether it's in our
+               // case.
+    ptr_c0 = ptr_c;
+    ptr_a = (bfloat16_t *)A;
+
+    for (BLASLONG i = 0; i < m / 4; i++) {
+      ptr_a0 = ptr_a;
+      ptr_a += 4 * pad_k;
+
+      ptr_b0 = ptr_b;
+
+      INIT_C(0, 0);
+      INIT_C(1, 0);
+
+      for (BLASLONG p = 0; p < pad_k; p += 8) {
+        ma0 = svld1_bf16(pg16_all, ptr_a0);
+        ma1 = svld1_bf16(pg16_all, ptr_a0 + 16);
+
+        mb0 = svld1_bf16(pg16_all, ptr_b0);
+
+        MATMUL(0, 0);
+        MATMUL(1, 0);
+
+        ptr_a0 += 32;
+        ptr_b0 += 16;
+      }
+
+      ACCUMULATE_LAST4_TO_FIRST4(0, 0, tmp);
+      ACCUMULATE_LAST4_TO_FIRST4(1, 0, tmp);
+
+      ZIP_EVEN_ELEMENTS(pg32_select_first_2_per_quadword, mc00, mc10, tmp, vc0);
+
+      UPDATE_C(pg16_first_4, pg32_first_4, ptr_c0, tmp32, tmp16, vc0);
+
+      ptr_c0 += 4;
+    }
+
+    if (m & 2) {
+      ptr_a0 = ptr_a;
+      ptr_a += 2 * pad_k;
+      ptr_b0 = ptr_b;
+
+      INIT_C(0, 0);
+
+      for (BLASLONG p = 0; p < pad_k; p += 8) {
+        ma0 = svld1_bf16(pg16_all, ptr_a0);
+        mb0 = svld1_bf16(pg16_all, ptr_b0);
+
+        MATMUL(0, 0);
+
+        ptr_a0 += 16;
+        ptr_b0 += 16;
+      }
+
+      ACCUMULATE_LAST4_TO_FIRST4(0, 0, tmp);
+
+      vc0 = svuzp1(mc00, mc00);
+
+      UPDATE_C(pg16_first_2, pg32_first_2, ptr_c0, tmp32, tmp16, vc0);
+
+      ptr_c0 += 2;
+    }
+
+    if (m & 1) {
+      ptr_a0 = ptr_a;
+      ptr_b0 = ptr_b;
+
+      INIT_C(0, 0);
+      for (BLASLONG p = 0; p < pad_k; p += 8) {
+
+        ma0 = svld1_bf16(pg16_all, ptr_a0);
+        mb0 = svld1_bf16(pg16_all, ptr_b0);
+
+        MATMUL(0, 0);
+        ptr_a0 += 16;
+        ptr_b0 += 16;
+      }
+
+      ACCUMULATE_LAST4_TO_FIRST4(0, 0, tmp);
+
+      UPDATE_C(pg16_first_1, pg32_first_1, ptr_c0, tmp32, tmp16, mc00);
+    }
+  }
+
+  return 0;
+}

kernel/arm64/dot.c

@@ -40,8 +40,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #endif
 
 #ifdef USE_SVE
+#ifdef DOT_KERNEL_SVE
+#include DOT_KERNEL_SVE
+#else
 #include "dot_kernel_sve.c"
 #endif
+#endif
 #include "dot_kernel_asimd.c"
 
 #if defined(SMP)

kernel/arm64/dot_kernel_sve_v8.c

@@ -0,0 +1,146 @@
+/***************************************************************************
+Copyright (c) 2025, The OpenBLAS Project
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   1. Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+
+   2. Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in
+      the documentation and/or other materials provided with the
+      distribution.
+   3. Neither the name of the OpenBLAS project nor the names of
+      its contributors may be used to endorse or promote products
+      derived from this software without specific prior written
+      permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+#include <arm_sve.h>
+#include "common.h"
+
+#ifdef DOUBLE
+#define SV_COUNT svcntd
+#define SV_TYPE svfloat64_t
+#define SV_TRUE svptrue_b64
+#define SV_WHILE svwhilelt_b64_s64
+#define SV_DUP svdup_f64
+#else
+#define SV_COUNT svcntw
+#define SV_TYPE svfloat32_t
+#define SV_TRUE svptrue_b32
+#define SV_WHILE svwhilelt_b32_s64
+#define SV_DUP svdup_f32
+#endif
+
+static FLOAT dot_kernel_sve(BLASLONG n, FLOAT* x, FLOAT* y)
+{
+  SV_TYPE temp0_vec = SV_DUP(0.0);
+  SV_TYPE temp1_vec = SV_DUP(0.0);
+  SV_TYPE temp2_vec = SV_DUP(0.0);
+  SV_TYPE temp3_vec = SV_DUP(0.0);
+  SV_TYPE temp4_vec = SV_DUP(0.0);
+  SV_TYPE temp5_vec = SV_DUP(0.0);
+  SV_TYPE temp6_vec = SV_DUP(0.0);
+  SV_TYPE temp7_vec = SV_DUP(0.0);
+
+  BLASLONG i = 0;
+  BLASLONG sve_size = SV_COUNT();
+
+  while ((i + sve_size * 8 - 1) < n) {
+    FLOAT *x0_ptr = x + i;
+    SV_TYPE x0_vec = svld1_vnum(SV_TRUE(), x0_ptr, 0);
+    SV_TYPE x1_vec = svld1_vnum(SV_TRUE(), x0_ptr, 1);
+    SV_TYPE x2_vec = svld1_vnum(SV_TRUE(), x0_ptr, 2);
+    SV_TYPE x3_vec = svld1_vnum(SV_TRUE(), x0_ptr, 3);
+    SV_TYPE x4_vec = svld1_vnum(SV_TRUE(), x0_ptr, 4);
+    SV_TYPE x5_vec = svld1_vnum(SV_TRUE(), x0_ptr, 5);
+    SV_TYPE x6_vec = svld1_vnum(SV_TRUE(), x0_ptr, 6);
+    SV_TYPE x7_vec = svld1_vnum(SV_TRUE(), x0_ptr, 7);
+
+    FLOAT *y0_ptr = y + i;
+    SV_TYPE y0_vec = svld1_vnum(SV_TRUE(), y0_ptr, 0);
+    SV_TYPE y1_vec = svld1_vnum(SV_TRUE(), y0_ptr, 1);
+    SV_TYPE y2_vec = svld1_vnum(SV_TRUE(), y0_ptr, 2);
+    SV_TYPE y3_vec = svld1_vnum(SV_TRUE(), y0_ptr, 3);
+    SV_TYPE y4_vec = svld1_vnum(SV_TRUE(), y0_ptr, 4);
+    SV_TYPE y5_vec = svld1_vnum(SV_TRUE(), y0_ptr, 5);
+    SV_TYPE y6_vec = svld1_vnum(SV_TRUE(), y0_ptr, 6);
+    SV_TYPE y7_vec = svld1_vnum(SV_TRUE(), y0_ptr, 7);
+
+    temp0_vec = svmla_x(SV_TRUE(), temp0_vec, x0_vec, y0_vec);
+    temp1_vec = svmla_x(SV_TRUE(), temp1_vec, x1_vec, y1_vec);
+    temp2_vec = svmla_x(SV_TRUE(), temp2_vec, x2_vec, y2_vec);
+    temp3_vec = svmla_x(SV_TRUE(), temp3_vec, x3_vec, y3_vec);
+    temp4_vec = svmla_x(SV_TRUE(), temp4_vec, x4_vec, y4_vec);
+    temp5_vec = svmla_x(SV_TRUE(), temp5_vec, x5_vec, y5_vec);
+    temp6_vec = svmla_x(SV_TRUE(), temp6_vec, x6_vec, y6_vec);
+    temp7_vec = svmla_x(SV_TRUE(), temp7_vec, x7_vec, y7_vec);
+
+    i += sve_size * 8;
+  }
+
+  if (i < n) {
+    svbool_t pg0 = SV_WHILE(i + sve_size * 0, n);
+    svbool_t pg1 = SV_WHILE(i + sve_size * 1, n);
+    svbool_t pg2 = SV_WHILE(i + sve_size * 2, n);
+    svbool_t pg3 = SV_WHILE(i + sve_size * 3, n);
+    svbool_t pg4 = SV_WHILE(i + sve_size * 4, n);
+    svbool_t pg5 = SV_WHILE(i + sve_size * 5, n);
+    svbool_t pg6 = SV_WHILE(i + sve_size * 6, n);
+    svbool_t pg7 = SV_WHILE(i + sve_size * 7, n);
+
+    FLOAT *x0_ptr = x + i;
+    SV_TYPE x0_vec = svld1_vnum(pg0, x0_ptr, 0);
+    SV_TYPE x1_vec = svld1_vnum(pg1, x0_ptr, 1);
+    SV_TYPE x2_vec = svld1_vnum(pg2, x0_ptr, 2);
+    SV_TYPE x3_vec = svld1_vnum(pg3, x0_ptr, 3);
+    SV_TYPE x4_vec = svld1_vnum(pg4, x0_ptr, 4);
+    SV_TYPE x5_vec = svld1_vnum(pg5, x0_ptr, 5);
+    SV_TYPE x6_vec = svld1_vnum(pg6, x0_ptr, 6);
+    SV_TYPE x7_vec = svld1_vnum(pg7, x0_ptr, 7);
+
+    FLOAT *y0_ptr = y + i;
+    SV_TYPE y0_vec = svld1_vnum(pg0, y0_ptr, 0);
+    SV_TYPE y1_vec = svld1_vnum(pg1, y0_ptr, 1);
+    SV_TYPE y2_vec = svld1_vnum(pg2, y0_ptr, 2);
+    SV_TYPE y3_vec = svld1_vnum(pg3, y0_ptr, 3);
+    SV_TYPE y4_vec = svld1_vnum(pg4, y0_ptr, 4);
+    SV_TYPE y5_vec = svld1_vnum(pg5, y0_ptr, 5);
+    SV_TYPE y6_vec = svld1_vnum(pg6, y0_ptr, 6);
+    SV_TYPE y7_vec = svld1_vnum(pg7, y0_ptr, 7);
+
+    temp0_vec = svmla_m(pg0, temp0_vec, x0_vec, y0_vec);
+    temp1_vec = svmla_m(pg1, temp1_vec, x1_vec, y1_vec);
+    temp2_vec = svmla_m(pg2, temp2_vec, x2_vec, y2_vec);
+    temp3_vec = svmla_m(pg3, temp3_vec, x3_vec, y3_vec);
+    temp4_vec = svmla_m(pg4, temp4_vec, x4_vec, y4_vec);
+    temp5_vec = svmla_m(pg5, temp5_vec, x5_vec, y5_vec);
+    temp6_vec = svmla_m(pg6, temp6_vec, x6_vec, y6_vec);
+    temp7_vec = svmla_m(pg7, temp7_vec, x7_vec, y7_vec);
+  }
+
+  temp0_vec = svadd_x(SV_TRUE(), temp0_vec, temp1_vec);
+  temp2_vec = svadd_x(SV_TRUE(), temp2_vec, temp3_vec);
+  temp4_vec = svadd_x(SV_TRUE(), temp4_vec, temp5_vec);
+  temp6_vec = svadd_x(SV_TRUE(), temp6_vec, temp7_vec);
+  temp0_vec = svadd_x(SV_TRUE(), temp0_vec, temp2_vec);
+  temp4_vec = svadd_x(SV_TRUE(), temp4_vec, temp6_vec);
+  temp0_vec = svadd_x(SV_TRUE(), temp0_vec, temp4_vec);
+
+  return svaddv(SV_TRUE(), temp0_vec);
+}

kernel/arm64/dot_sve_v8.c

@@ -0,0 +1,34 @@
+/***************************************************************************
+Copyright (c) 2025, The OpenBLAS Project
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   1. Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+
+   2. Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in
+      the documentation and/or other materials provided with the
+      distribution.
+   3. Neither the name of the OpenBLAS project nor the names of
+      its contributors may be used to endorse or promote products
+      derived from this software without specific prior written
+      permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+#define DOT_KERNEL_SVE "dot_kernel_sve_v8.c"
+#include "dot.c"

kernel/arm64/gemv_n_sve_v1x3.c

@@ -52,17 +52,17 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha, FLOAT *a,
           BLASLONG lda, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y,
           FLOAT *buffer)
 {
-  BLASLONG i;
-  BLASLONG ix,iy;
-  BLASLONG j;
-  FLOAT *a_ptr;
+  BLASLONG i, j;
+  BLASLONG ix = 0;
+  BLASLONG iy;
+  FLOAT *a_ptr = a;
   FLOAT temp;
 
-  ix = 0;
-  a_ptr = a;
-
   if (inc_y == 1) {
-    BLASLONG width = (n + 3 - 1) / 3;
+    BLASLONG width = n / 3; // Only process full 3-column blocks
+    BLASLONG sve_size = SV_COUNT();
+    svbool_t pg_full = SV_TRUE();
+    svbool_t pg_tail = SV_WHILE(0, m % sve_size);
 
     FLOAT *a0_ptr = a_ptr + lda * width * 0;
     FLOAT *a1_ptr = a_ptr + lda * width * 1;
@@ -73,57 +73,75 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha, FLOAT *a,
     FLOAT *x2_ptr = x + inc_x * width * 2;
 
     for (j = 0; j < width; j++) {
-      svbool_t pg00 = ((j + width * 0) < n) ? SV_TRUE() : svpfalse();
-      svbool_t pg01 = ((j + width * 1) < n) ? SV_TRUE() : svpfalse();
-      svbool_t pg02 = ((j + width * 2) < n) ? SV_TRUE() : svpfalse();
+      SV_TYPE temp0_vec = SV_DUP(alpha * x0_ptr[ix]);
+      SV_TYPE temp1_vec = SV_DUP(alpha * x1_ptr[ix]);
+      SV_TYPE temp2_vec = SV_DUP(alpha * x2_ptr[ix]);
 
-      SV_TYPE temp0_vec = ((j + width * 0) < n) ? SV_DUP(alpha * x0_ptr[ix]) : SV_DUP(0.0);
-      SV_TYPE temp1_vec = ((j + width * 1) < n) ? SV_DUP(alpha * x1_ptr[ix]) : SV_DUP(0.0);
-      SV_TYPE temp2_vec = ((j + width * 2) < n) ? SV_DUP(alpha * x2_ptr[ix]) : SV_DUP(0.0);
       i = 0;
-      BLASLONG sve_size = SV_COUNT();
-      while ((i + sve_size * 1 - 1) < m) {
-        SV_TYPE y0_vec = svld1_vnum(SV_TRUE(), y + i, 0);
+      while ((i + sve_size - 1) < m) {
+        SV_TYPE y0_vec = svld1(pg_full, y + i);
 
-        SV_TYPE a00_vec = svld1_vnum(pg00, a0_ptr + i, 0);
-        SV_TYPE a01_vec = svld1_vnum(pg01, a1_ptr + i, 0);
-        SV_TYPE a02_vec = svld1_vnum(pg02, a2_ptr + i, 0);
+        SV_TYPE a00_vec = svld1(pg_full, a0_ptr + i);
+        SV_TYPE a01_vec = svld1(pg_full, a1_ptr + i);
+        SV_TYPE a02_vec = svld1(pg_full, a2_ptr + i);
 
-        y0_vec = svmla_m(pg00, y0_vec, temp0_vec, a00_vec);
-        y0_vec = svmla_m(pg01, y0_vec, temp1_vec, a01_vec);
-        y0_vec = svmla_m(pg02, y0_vec, temp2_vec, a02_vec);
+        y0_vec = svmla_x(pg_full, y0_vec, temp0_vec, a00_vec);
+        y0_vec = svmla_x(pg_full, y0_vec, temp1_vec, a01_vec);
+        y0_vec = svmla_x(pg_full, y0_vec, temp2_vec, a02_vec);
 
-        svst1_vnum(SV_TRUE(), y + i, 0, y0_vec);
-        i += sve_size * 1;
+        svst1(pg_full, y + i, y0_vec);
+        i += sve_size;
       }
 
       if (i < m) {
-        svbool_t pg0 = SV_WHILE(i + sve_size * 0, m);
-
-        pg00 = svand_z(SV_TRUE(), pg0, pg00);
-        pg01 = svand_z(SV_TRUE(), pg0, pg01);
-        pg02 = svand_z(SV_TRUE(), pg0, pg02);
+        SV_TYPE y0_vec = svld1(pg_tail, y + i);
 
-        SV_TYPE y0_vec = svld1_vnum(pg0, y + i, 0);
+        SV_TYPE a00_vec = svld1(pg_tail, a0_ptr + i);
+        SV_TYPE a01_vec = svld1(pg_tail, a1_ptr + i);
+        SV_TYPE a02_vec = svld1(pg_tail, a2_ptr + i);
 
-        SV_TYPE a00_vec = svld1_vnum(pg00, a0_ptr + i, 0);
-        SV_TYPE a01_vec = svld1_vnum(pg01, a1_ptr + i, 0);
-        SV_TYPE a02_vec = svld1_vnum(pg02, a2_ptr + i, 0);
+        y0_vec = svmla_m(pg_tail, y0_vec, temp0_vec, a00_vec);
+        y0_vec = svmla_m(pg_tail, y0_vec, temp1_vec, a01_vec);
+        y0_vec = svmla_m(pg_tail, y0_vec, temp2_vec, a02_vec);
 
-        y0_vec = svmla_m(pg00, y0_vec, temp0_vec, a00_vec);
-        y0_vec = svmla_m(pg01, y0_vec, temp1_vec, a01_vec);
-        y0_vec = svmla_m(pg02, y0_vec, temp2_vec, a02_vec);
-
-        svst1_vnum(pg0, y + i, 0, y0_vec);
+        svst1(pg_tail, y + i, y0_vec);
       }
       a0_ptr += lda;
       a1_ptr += lda;
       a2_ptr += lda;
       ix += inc_x;
     }
+    // Handle remaining n % 3 columns
+    for (j = width * 3; j < n; j++) {
+      FLOAT *a_col = a + j * lda;
+      temp = alpha * x[j * inc_x];
+      SV_TYPE temp_vec = SV_DUP(temp);
+
+      i = 0;
+      while ((i + sve_size - 1) < m) {
+        SV_TYPE y_vec = svld1(pg_full, y + i);
+
+        SV_TYPE a_vec = svld1(pg_full, a_col + i);
+
+        y_vec = svmla_x(pg_full, y_vec, temp_vec, a_vec);
+
+        svst1(pg_full, y + i, y_vec);
+        i += sve_size;
+      }
+      if (i < m) {
+        SV_TYPE y_vec = svld1(pg_tail, y + i);
+
+        SV_TYPE a_vec = svld1(pg_tail, a_col + i);
+
+        y_vec = svmla_m(pg_tail, y_vec, temp_vec, a_vec);
+
+        svst1(pg_tail, y + i, y_vec);
+      }
+    }
     return(0);
   }
 
+  // Fallback scalar loop
   for (j = 0; j < n; j++) {
     temp = alpha * x[ix];
     iy = 0;

kernel/arm64/sgemm_direct_alpha_beta_arm64_sme1.c

@@ -0,0 +1,199 @@
+/*
+ Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ SPDX-License-Identifier: BSD-3-Clause-Clear
+*/
+
+#include "common.h"
+#include <stdlib.h>
+#include <inttypes.h>
+#include <math.h>
+#include "sme_abi.h"
+#if defined(HAVE_SME)
+
+#if defined(__ARM_FEATURE_SME) && defined(__clang__) && __clang_major__ >= 16
+#include <arm_sme.h>
+#endif
+
+/* Function prototypes */
+extern void sgemm_direct_sme1_preprocess(uint64_t nbr, uint64_t nbc,\
+                                  const float * restrict a, float *  a_mod) __asm__("sgemm_direct_sme1_preprocess");
+
+/* Function Definitions */
+static uint64_t sve_cntw() {
+    uint64_t cnt;
+    asm volatile(
+        "rdsvl  %[res], #1\n"
+        "lsr    %[res], %[res], #2\n"
+        : [res] "=r" (cnt) ::
+    );
+    return cnt;
+}
+
+#if defined(__ARM_FEATURE_SME) && defined(__ARM_FEATURE_LOCALLY_STREAMING) && defined(__clang__) && __clang_major__ >= 16
+// Outer product kernel.
+// Computes a 2SVL x 2SVL block of C, utilizing all four FP32 tiles of ZA.
+__attribute__((always_inline)) inline void
+kernel_2x2(const float *A, const float *B, float *C, size_t shared_dim,
+           size_t ldc, size_t block_rows, size_t block_cols, float alpha, float beta)
+    __arm_out("za") __arm_streaming {
+
+  const uint64_t svl = svcntw();
+  size_t ldb = ldc;
+  // Predicate set-up
+  svbool_t pg = svptrue_b32();
+  svbool_t pg_a_0 = svwhilelt_b32_u64(0, block_rows);
+  svbool_t pg_a_1 = svwhilelt_b32_u64(svl, block_rows);
+
+  svbool_t pg_b_0 = svwhilelt_b32_u64(0, block_cols);
+  svbool_t pg_b_1 = svwhilelt_b32_u64(svl, block_cols);
+
+#define pg_c_0 pg_b_0
+#define pg_c_1 pg_b_1
+
+  svzero_za();
+  svfloat32_t beta_vec = svdup_f32(beta);
+    // Load C to ZA
+  for (size_t i = 0; i < MIN(svl, block_rows); i++) {
+	svfloat32_t row_c_0 = svld1(pg_c_0, &C[i * ldc]);
+	row_c_0 = svmul_x(pg, beta_vec, row_c_0);
+    svwrite_hor_za32_f32_m(/*tile*/0, /*slice*/i, pg_c_0, row_c_0);
+    
+	svfloat32_t row_c_1 = svld1(pg_c_1, &C[i * ldc + svl]);
+	row_c_1 = svmul_x(pg, beta_vec, row_c_1);
+    svwrite_hor_za32_f32_m(/*tile*/1, /*slice*/i, pg_c_1, row_c_1);
+  }
+  for (size_t i = svl; i < block_rows; i++) {
+    svfloat32_t row_c_0 = svld1(pg_c_0, &C[i * ldc]);
+	row_c_0 = svmul_x(pg, beta_vec, row_c_0);
+	svwrite_hor_za32_f32_m(/*tile*/2, /*slice*/i, pg_c_0, row_c_0);
+
+    svfloat32_t row_c_1 = svld1(pg_c_1, &C[i * ldc + svl]);
+	row_c_1 = svmul_x(pg, beta_vec, row_c_1);
+    svwrite_hor_za32_f32_m(/*tile*/3, /*slice*/i, pg_c_1, row_c_1);
+  }
+  
+  svfloat32_t alpha_vec = svdup_f32(alpha);
+  // Iterate through shared dimension (K)
+  for (size_t k = 0; k < shared_dim; k++) {
+    // Load column of A
+    svfloat32_t col_a_0 = svld1(pg_a_0, &A[k * svl]);
+	col_a_0 = svmul_x(pg, alpha_vec, col_a_0);
+    svfloat32_t col_a_1 = svld1(pg_a_1, &A[(k + shared_dim) * svl]);
+    col_a_1 = svmul_x(pg, alpha_vec, col_a_1);
+    // Load row of B
+    svfloat32_t row_b_0 = svld1(pg_b_0, &B[k * ldb]);
+    svfloat32_t row_b_1 = svld1(pg_b_1, &B[k * ldb + svl]);
+    // Perform outer product
+    svmopa_za32_m(/*tile*/0, pg, pg, col_a_0, row_b_0);
+    svmopa_za32_m(/*tile*/1, pg, pg, col_a_0, row_b_1);
+    svmopa_za32_m(/*tile*/2, pg, pg, col_a_1, row_b_0);
+    svmopa_za32_m(/*tile*/3, pg, pg, col_a_1, row_b_1);
+  }
+
+  // Store to C from ZA
+  for (size_t i = 0; i < MIN(svl, block_rows); i++) {
+    svst1_hor_za32(/*tile*/0, /*slice*/i, pg_c_0, &C[i * ldc]);
+    svst1_hor_za32(/*tile*/1, /*slice*/i, pg_c_1, &C[i * ldc + svl]);
+  }
+  for (size_t i = svl; i < block_rows; i++) {
+    svst1_hor_za32(/*tile*/2, /*slice*/i, pg_c_0, &C[i * ldc]);
+    svst1_hor_za32(/*tile*/3, /*slice*/i, pg_c_1, &C[i * ldc + svl]);
+  }
+}
+
+__arm_new("za") __arm_locally_streaming
+void sgemm_direct_alpha_beta_sme1_2VLx2VL(uint64_t m, uint64_t k, uint64_t n, const float* alpha,\
+                                   const float *ba, const float *restrict bb, const float* beta,\
+                                   float *restrict C) {
+
+  const uint64_t num_rows = m;
+  const uint64_t num_cols = n;
+
+  const float *restrict a_ptr = ba;
+  const float *restrict b_ptr = bb;
+  float *restrict c_ptr = C;
+
+  const uint64_t svl = svcntw();
+  const uint64_t ldc = n;
+
+  // Block over rows of C (panels of A)
+  uint64_t row_idx = 0;
+
+  // 2x2 loop
+  uint64_t row_batch = 2*svl;
+
+  // Block over row dimension of C
+  for (; row_idx < num_rows; row_idx += row_batch) {
+    row_batch = MIN(row_batch, num_rows - row_idx);
+    uint64_t col_idx = 0;
+    uint64_t col_batch = 2*svl;
+
+    // Block over column dimension of C
+    for (; col_idx < num_cols; col_idx += col_batch) {
+      col_batch = MIN(col_batch, num_cols - col_idx);
+
+      kernel_2x2(&a_ptr[row_idx * k], &b_ptr[col_idx],
+                 &c_ptr[row_idx * ldc + col_idx], k,
+				 ldc, row_batch, col_batch, *alpha, *beta);
+    }
+  }
+  return;
+}
+
+#else
+void sgemm_direct_alpha_beta_sme1_2VLx2VL(uint64_t m, uint64_t k, uint64_t n, const float* alpha,\
+                                   const float *ba, const float *restrict bb, const float* beta,\
+                                   float *restrict C){}
+#endif
+
+/*void sgemm_kernel_direct (BLASLONG M, BLASLONG N, BLASLONG K,\
+       float * __restrict A, BLASLONG strideA, float * __restrict B,\
+       BLASLONG strideB , float * __restrict R, BLASLONG strideR)
+*/
+void CNAME (BLASLONG M, BLASLONG N, BLASLONG K, float alpha, float * __restrict A,\
+            BLASLONG strideA, float * __restrict B, BLASLONG strideB ,\
+            float beta, float * __restrict R, BLASLONG strideR){
+                
+        uint64_t m_mod, vl_elms;
+        
+        vl_elms = sve_cntw();
+
+        m_mod = ceil((double)M/(double)vl_elms) * vl_elms;
+
+        float *A_mod = (float *) malloc(m_mod*K*sizeof(float));
+	    
+	    /* Prevent compiler optimization by reading from memory instead
+	     * of reading directly from vector (z) registers.
+	     * */
+        asm volatile("" : : :"p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7",
+                         "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15",
+                         "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7",
+                         "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15",
+                         "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23",
+                         "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31");
+      
+        /* Pre-process the left matrix to make it suitable for 
+           matrix sum of outer-product calculation
+         */
+        sgemm_direct_sme1_preprocess(M, K, A, A_mod);
+       
+        asm volatile("" : : :"p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7",
+                         "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15",
+                         "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7",
+                         "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15",
+                         "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23",
+                         "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31");
+
+        /* Calculate C = alpha*A*B + beta*C */
+        sgemm_direct_alpha_beta_sme1_2VLx2VL(M, K, N, &alpha, A_mod, B, &beta, R);
+
+        free(A_mod);
+}
+
+#else
+
+void CNAME (BLASLONG M, BLASLONG N, BLASLONG K, float alpha, float * __restrict A,\
+            BLASLONG strideA, float * __restrict B, BLASLONG strideB ,\
+            float beta, float * __restrict R, BLASLONG strideR){}
+ 
+#endif

kernel/arm64/sme_abi.h

@@ -0,0 +1,46 @@
+/***************************************************************************
+ * Copyright (c) 2024, The OpenBLAS Project
+ * All rights reserved.
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * 3. Neither the name of the OpenBLAS project nor the names of
+ * its contributors may be used to endorse or promote products
+ * derived from this software without specific prior written permission.
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
+ * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+ * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ * *****************************************************************************/
+
+#pragma once
+
+#include <stdlib.h>
+
+/**
+ *  * These are SME ABI routines for saving & restoring SME state.
+ *   * They are typically provided by a compiler runtime library such
+ *    * as libgcc or compiler-rt, but support for these routines is not
+ *     * yet available on all platforms.
+ *      *
+ *       * Define these as aborting stubs so that we loudly fail on nested
+ *        * usage of SME state.
+ *         *
+ *          * These are defined as weak symbols so that a compiler runtime can
+ *           * override them if supported.
+ *            */
+__attribute__((weak)) void __arm_tpidr2_save() { abort(); }
+__attribute__((weak)) void __arm_tpidr2_restore() { abort(); }
+

kernel/generic/bf16_macros.h

@@ -0,0 +1,64 @@
+/***************************************************************************
+ * Copyright (c) 2025, The OpenBLAS Project
+ * All rights reserved.
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * 3. Neither the name of the OpenBLAS project nor the names of
+ * its contributors may be used to endorse or promote products
+ * derived from this software without specific prior written permission.
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ * *****************************************************************************/
+
+#if defined(BFLOAT16) && defined(BFLOAT16CONVERSION)
+static float
+bfloat16tof32 (bfloat16 value)
+{
+  blasint one = 1;
+  float result;
+  sbf16tos_(&one, &value, &one, &result, &one);
+  return result;
+}
+
+#ifdef BGEMM
+static bfloat16 f32tobfloat16(float value) {
+  blasint one = 1;
+  bfloat16 result;
+  sbstobf16_(&one, &value, &one, &result, &one);
+  return result;
+}
+#endif
+
+#ifdef BGEMM
+#define ALPHA bfloat16tof32(alpha)
+#define BETA bfloat16tof32(beta)
+#define BF16TOF32(x) (bfloat16tof32(x))
+#define F32TOBF16(x) (f32tobfloat16(x))
+#else
+#define ALPHA alpha
+#define BETA beta
+#define BF16TOF32(x) (bfloat16tof32(x))
+#define F32TOBF16(x) x
+#endif
+#else
+#define ALPHA alpha
+#define BETA beta
+#define BF16TOF32(x) x
+#define F32TOBF16(x) x
+#endif

kernel/generic/gemm_beta.c

@@ -1,5 +1,6 @@
 /*********************************************************************/
 /* Copyright 2009, 2010 The University of Texas at Austin.           */
+/* Copyright 2025 The OpenBLAS Project.						         */
 /* All rights reserved.                                              */
 /*                                                                   */
 /* Redistribution and use in source and binary forms, with or        */
@@ -38,11 +39,41 @@
 
 #include "common.h"
 
+#if defined(BFLOAT16) && defined(BGEMM) && defined(BFLOAT16CONVERSION)
+static float
+bfloat16tof32 (bfloat16 f16)
+{
+  float result = 0;
+  unsigned short* q = (unsigned short*)(&result);
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+  q[0] = f16;
+#else
+  q[1] = f16;
+#endif
+  return result;
+}
+static bfloat16
+f32tobfloat16(float f32)
+{
+    unsigned short* q = (unsigned short*)(&f32);
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+    return q[0];
+#else
+    return q[1];
+#endif
+}
+
+#define BF16TOF32(x) (bfloat16tof32(x))
+#define F32TOBF16(x) (f32tobfloat16(x))
+#else
+#define BF16TOF32(x) x
+#define F32TOBF16(x) x
+#endif
+
 int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT beta,
 	  IFLOAT *dummy2, BLASLONG dummy3, IFLOAT *dummy4, BLASLONG dummy5,
 	  FLOAT *c, BLASLONG ldc){
 
-
   BLASLONG i, j;
   BLASLONG chunk, remain;
   FLOAT *c_offset1, *c_offset;
@@ -54,18 +85,18 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT beta,
 		c_offset1 = c_offset;
 		c_offset += ldc;
 		for(i=chunk; i>0; i--){
-			*(c_offset1 + 0) = ZERO;
-			*(c_offset1 + 1) = ZERO;
-			*(c_offset1 + 2) = ZERO;
-			*(c_offset1 + 3) = ZERO;
-			*(c_offset1 + 4) = ZERO;
-			*(c_offset1 + 5) = ZERO;
-			*(c_offset1 + 6) = ZERO;
-			*(c_offset1 + 7) = ZERO;
+			*(c_offset1 + 0) = F32TOBF16(ZERO);
+			*(c_offset1 + 1) = F32TOBF16(ZERO);
+			*(c_offset1 + 2) = F32TOBF16(ZERO);
+			*(c_offset1 + 3) = F32TOBF16(ZERO);
+			*(c_offset1 + 4) = F32TOBF16(ZERO);
+			*(c_offset1 + 5) = F32TOBF16(ZERO);
+			*(c_offset1 + 6) = F32TOBF16(ZERO);
+			*(c_offset1 + 7) = F32TOBF16(ZERO);
 			c_offset1 += 8;
 		}
 		for(i=remain; i>0; i--){
-			*c_offset1 = ZERO;
+			*c_offset1 = F32TOBF16(ZERO);
 			c_offset1 ++;
 		}
 	  }
@@ -74,18 +105,18 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT beta,
 		c_offset1 = c_offset;
 		c_offset += ldc;
 		for(i=chunk; i>0; i--){
-			*(c_offset1 + 0) *= beta;
-			*(c_offset1 + 1) *= beta;
-			*(c_offset1 + 2) *= beta;
-			*(c_offset1 + 3) *= beta;
-			*(c_offset1 + 4) *= beta;
-			*(c_offset1 + 5) *= beta;
-			*(c_offset1 + 6) *= beta;
-			*(c_offset1 + 7) *= beta;
+			*(c_offset1 + 0) = F32TOBF16(BF16TOF32(beta) * BF16TOF32(c_offset1[0]));
+			*(c_offset1 + 1) = F32TOBF16(BF16TOF32(beta) * BF16TOF32(c_offset1[1]));
+			*(c_offset1 + 2) = F32TOBF16(BF16TOF32(beta) * BF16TOF32(c_offset1[2]));
+			*(c_offset1 + 3) = F32TOBF16(BF16TOF32(beta) * BF16TOF32(c_offset1[3]));
+			*(c_offset1 + 4) = F32TOBF16(BF16TOF32(beta) * BF16TOF32(c_offset1[4]));
+			*(c_offset1 + 5) = F32TOBF16(BF16TOF32(beta) * BF16TOF32(c_offset1[5]));
+			*(c_offset1 + 6) = F32TOBF16(BF16TOF32(beta) * BF16TOF32(c_offset1[6]));
+			*(c_offset1 + 7) = F32TOBF16(BF16TOF32(beta) * BF16TOF32(c_offset1[7]));
 			c_offset1 += 8;
 		}
 		for(i=remain; i>0; i--){
-			*c_offset1 *= beta;
+			*c_offset1 = F32TOBF16(BF16TOF32(beta) * BF16TOF32(c_offset1[0]));
 			c_offset1 ++;
 		}
 	  }

kernel/generic/gemmkernel_2x2.c

@@ -1,21 +1,34 @@
+/***************************************************************************
+ * Copyright (c) 2025, The OpenBLAS Project
+ * All rights reserved.
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * 3. Neither the name of the OpenBLAS project nor the names of
+ * its contributors may be used to endorse or promote products
+ * derived from this software without specific prior written permission.
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ * *****************************************************************************/
+
 #include "common.h"
-#if defined(BFLOAT16) && defined(BFLOAT16CONVERSION)
-static float
-bfloat16tof32 (bfloat16 f16)
-{
-  float result = 0;
-  unsigned short* q = (unsigned short*)(&result);
-#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-  q[0] = f16;
-#else
-  q[1] = f16;
-#endif
-  return result;
-}
-#define BF16TOF32(x) (bfloat16tof32(x))
-#else
-#define BF16TOF32(x) x
-#endif
+#include "bf16_macros.h"
+
 int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alpha,IFLOAT* ba,IFLOAT* bb,FLOAT* C,BLASLONG ldc
 #ifdef TRMMKERNEL
 		,BLASLONG offset
@@ -25,7 +38,11 @@ int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alpha,IFLOAT* ba,IFLOAT* bb,
    BLASLONG i,j,k;
    FLOAT *C0,*C1;
    IFLOAT *ptrba,*ptrbb;
+#ifdef BGEMM
+   float res0,res1,res2,res3;
+#else
    FLOAT res0,res1,res2,res3;
+#endif
    IFLOAT load0,load1,load2,load3,load4,load5,load6,load7;
    for (j=0; j<bn/2; j+=1)
      {
@@ -89,14 +106,14 @@ int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alpha,IFLOAT* ba,IFLOAT* bb,
                   ptrba = ptrba+2;
                   ptrbb = ptrbb+2;
                }
-             res0 = res0*alpha;
-             C0[0] = C0[0]+res0;
-             res1 = res1*alpha;
-             C0[1] = C0[1]+res1;
-             res2 = res2*alpha;
-             C1[0] = C1[0]+res2;
-             res3 = res3*alpha;
-             C1[1] = C1[1]+res3;
+             res0 = res0*ALPHA;
+             C0[0] = F32TOBF16(BF16TOF32(C0[0])+res0);
+             res1 = res1*ALPHA;
+             C0[1] = F32TOBF16(BF16TOF32(C0[1])+res1);
+             res2 = res2*ALPHA;
+             C1[0] = F32TOBF16(BF16TOF32(C1[0])+res2);
+             res3 = res3*ALPHA;
+             C1[1] = F32TOBF16(BF16TOF32(C1[1])+res3);
              C0 = C0+2;
              C1 = C1+2;
           }
@@ -115,10 +132,10 @@ int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alpha,IFLOAT* ba,IFLOAT* bb,
                   ptrba = ptrba+1;
                   ptrbb = ptrbb+2;
                }
-             res0 = res0*alpha;
-             C0[0] = C0[0]+res0;
-             res1 = res1*alpha;
-             C1[0] = C1[0]+res1;
+             res0 = res0*ALPHA;
+             C0[0] = F32TOBF16(BF16TOF32(C0[0])+res0);
+             res1 = res1*ALPHA;
+             C1[0] = F32TOBF16(BF16TOF32(C1[0])+res1);
              C0 = C0+1;
              C1 = C1+1;
           }
@@ -146,10 +163,10 @@ int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alpha,IFLOAT* ba,IFLOAT* bb,
                   ptrba = ptrba+2;
                   ptrbb = ptrbb+1;
                }
-             res0 = res0*alpha;
-             C0[0] = C0[0]+res0;
-             res1 = res1*alpha;
-             C0[1] = C0[1]+res1;
+             res0 = res0*ALPHA;
+             C0[0] = F32TOBF16(BF16TOF32(C0[0])+res0);
+             res1 = res1*ALPHA;
+             C0[1] = F32TOBF16(BF16TOF32(C0[1])+res1);
              C0 = C0+2;
           }
         for (i=0; i<(bm&1); i+=1)
@@ -164,8 +181,8 @@ int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alpha,IFLOAT* ba,IFLOAT* bb,
                   ptrba = ptrba+1;
                   ptrbb = ptrbb+1;
                }
-             res0 = res0*alpha;
-             C0[0] = C0[0]+res0;
+             res0 = res0*ALPHA;
+             C0[0] = F32TOBF16(BF16TOF32(C0[0])+res0);
              C0 = C0+1;
           }
         k = (bk<<0);

kernel/generic/gemv_n.c

@@ -0,0 +1,70 @@
+/***************************************************************************
+Copyright (c) 2013-2014, 2025 The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+#include "common.h"
+#include "bf16_macros.h"
+
+int CNAME(BLASLONG m, BLASLONG n, FLOAT alpha, IFLOAT *a, BLASLONG lda, IFLOAT *x, BLASLONG inc_x, FLOAT beta, FLOAT *y, BLASLONG inc_y)
+{
+    BLASLONG i;
+    BLASLONG ix, iy;
+    BLASLONG j;
+    FLOAT *a_ptr;
+#ifdef BGEMM
+    float temp;
+#else
+    FLOAT temp;
+#endif
+
+    iy = 0;
+    for (BLASLONG i = 0; i < m; i++)
+    {
+        temp = 0.0;
+
+        ix = 0;
+        a_ptr = a;
+        for (BLASLONG j = 0; j < n; j++)
+        {
+            temp += BF16TOF32(a_ptr[i]) * BF16TOF32(x[ix]);
+            ix += inc_x;
+            a_ptr += lda;
+        }
+
+        if (BETA == ZERO)
+        {
+            y[iy] = F32TOBF16(ALPHA * temp);
+        }
+        else
+        {
+            y[iy] = F32TOBF16(ALPHA * temp + BETA * BF16TOF32(y[iy]));
+        }
+
+        iy += inc_y;
+    }
+
+    return (0);
+}

kernel/generic/gemv_t.c

@@ -0,0 +1,60 @@
+/***************************************************************************
+Copyright (c) 2013, 2025 The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+#include "common.h"
+#include "bf16_macros.h"
+
+int CNAME(BLASLONG m, BLASLONG n, FLOAT alpha, IFLOAT *a, BLASLONG lda, IFLOAT *x, BLASLONG inc_x, FLOAT beta, FLOAT *y, BLASLONG inc_y)
+{
+    BLASLONG i;
+    BLASLONG ix, iy;
+    BLASLONG j;
+    FLOAT *a_ptr;
+#ifdef BGEMM
+    float temp;
+#else
+    FLOAT temp;
+#endif
+
+    iy = 0;
+    a_ptr = a;
+
+    for (j = 0; j < n; j++)
+    {
+        temp = 0.0;
+        ix = 0;
+        for (i = 0; i < m; i++)
+        {
+            temp += BF16TOF32(a_ptr[i]) * BF16TOF32(x[ix]);
+            ix += inc_x;
+        }
+        y[iy] += F32TOBF16(ALPHA * temp);
+        iy += inc_y;
+        a_ptr += lda;
+    }
+    return (0);
+}

kernel/generic/scal.c

@@ -0,0 +1,106 @@
+/***************************************************************************
+Copyright (c) 2013, 2025 The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+#include "common.h"
+
+int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y, FLOAT *dummy, BLASLONG dummy2)
+{
+    BLASLONG i = 0, j = 0;
+#if defined(BFLOAT16)
+    float x_float, da_float;
+    SBF16TOS_K(1, &da, 1, &da_float, 1);
+#else
+    float x_float;
+    float da_float = da;
+#endif
+
+    if ((n <= 0) || (inc_x <= 0))
+        return (0);
+
+    if (dummy2 == 0)
+    {
+        while (j < n)
+        {
+
+            if (da_float == 0.0)
+                x_float = 0.0;
+            else
+            {
+#if defined(BFLOAT16)
+                SBF16TOS_K(1, &x[i], 1, &x_float, 1);
+#else
+                float x_float = x[i];
+#endif
+                x_float = da_float * x_float;
+            }
+
+#if defined(BFLOAT16)
+            SBSTOBF16_K(1, &x_float, 1, &x[i], 1);
+#else
+            x[i] = x_float;
+#endif
+
+            i += inc_x;
+            j++;
+        }
+    }
+    else
+    {
+
+        while (j < n)
+        {
+#if defined(BFLOAT16)
+            SBF16TOS_K(1, &x[i], 1, &x_float, 1);
+#else
+            float x_float = x[i];
+#endif
+            if (da == 0.0)
+                if (!isnan(x_float) && !isinf(x_float))
+                {
+                    x_float = 0.0;
+                }
+                else
+                {
+                    x_float = NAN;
+                }
+            else
+            {
+                x_float = da_float * x_float;
+            }
+
+#if defined(BFLOAT16)
+            SBSTOBF16_K(1, &x_float, 1, &x[i], 1);
+#else
+            x[i] = x_float;
+#endif
+
+            i += inc_x;
+            j++;
+        }
+    }
+    return 0;
+}

kernel/setparam-ref.c

@@ -1,6 +1,6 @@
 /*********************************************************************/
 /* Copyright 2009, 2010 The University of Texas at Austin.           */
-/* Copyright 2023 The OpenBLAS Project.                              */
+/* Copyright 2023, 2025 The OpenBLAS Project.                        */
 /* All rights reserved.                                              */
 /*                                                                   */
 /* Redistribution and use in source and binary forms, with or        */
@@ -57,6 +57,15 @@ gotoblas_t TABLE_NAME = {
   GEMM_DEFAULT_OFFSET_A, GEMM_DEFAULT_OFFSET_B, GEMM_DEFAULT_ALIGN,
 
 #ifdef BUILD_BFLOAT16
+  0, 0, 0,
+  BGEMM_DEFAULT_UNROLL_M, BGEMM_DEFAULT_UNROLL_N,
+#ifdef BGEMM_DEFAULT_UNROLL_MN
+ BGEMM_DEFAULT_UNROLL_MN,
+#else
+ MAX(BGEMM_DEFAULT_UNROLL_M, BGEMM_DEFAULT_UNROLL_N),
+#endif
+  BGEMM_ALIGN_K,
+
   0, 0, 0,
   SBGEMM_DEFAULT_UNROLL_M, SBGEMM_DEFAULT_UNROLL_N,
 #ifdef SBGEMM_DEFAULT_UNROLL_MN
@@ -74,10 +83,18 @@ gotoblas_t TABLE_NAME = {
   isamax_kTS, isamin_kTS, ismax_kTS, ismin_kTS,
   snrm2_kTS,  sasum_kTS,  ssum_kTS, scopy_kTS, sbdot_kTS,
   dsdot_kTS,
-  srot_kTS,   srotm_kTS,  saxpy_kTS,  sscal_kTS, sswap_kTS,
-  sbgemv_nTS, sbgemv_tTS, sger_kTS,
+  srot_kTS,   srotm_kTS, bscal_kTS, saxpy_kTS,  sscal_kTS, sswap_kTS,
+  bgemv_nTS, bgemv_tTS, sbgemv_nTS, sbgemv_tTS, sger_kTS,
   ssymv_LTS, ssymv_UTS,
 
+  bgemm_kernelTS, bgemm_betaTS,
+#if BGEMM_DEFAULT_UNROLL_M != BGEMM_DEFAULT_UNROLL_N
+  bgemm_incopyTS, bgemm_itcopyTS,
+#else
+  bgemm_oncopyTS, bgemm_otcopyTS,
+#endif
+  bgemm_oncopyTS, bgemm_otcopyTS,
+
   sbgemm_kernelTS, sbgemm_betaTS,
 #if SBGEMM_DEFAULT_UNROLL_M != SBGEMM_DEFAULT_UNROLL_N
   sbgemm_incopyTS, sbgemm_itcopyTS,
@@ -198,6 +215,7 @@ gotoblas_t TABLE_NAME = {
 #endif
 #ifdef ARCH_ARM64
   sgemm_directTS,
+  sgemm_direct_alpha_betaTS,
 #endif
 
   sgemm_kernelTS, sgemm_betaTS,
@@ -913,6 +931,7 @@ gotoblas_t TABLE_NAME = {
 static void init_parameter(void) {
 #if (BUILD_BFLOAT16)
   TABLE_NAME.sbgemm_p = SBGEMM_DEFAULT_P;
+  TABLE_NAME.bgemm_p = BGEMM_DEFAULT_P;
 #endif
 #if (BUILD_SINGLE==1) || (BUILD_COMPLEX==1)
   TABLE_NAME.sgemm_p = SGEMM_DEFAULT_P;
@@ -929,6 +948,7 @@ static void init_parameter(void) {
 
 #if (BUILD_BFLOAT16)
   TABLE_NAME.sbgemm_q = SBGEMM_DEFAULT_Q;
+  TABLE_NAME.bgemm_q = BGEMM_DEFAULT_Q;
 #endif
 #if BUILD_SINGLE == 1 || (BUILD_COMPLEX==1)
   TABLE_NAME.sgemm_q = SGEMM_DEFAULT_Q;
@@ -945,6 +965,7 @@ static void init_parameter(void) {
 
 #if (BUILD_BFLOAT16)
   TABLE_NAME.sbgemm_r = SBGEMM_DEFAULT_R;
+  TABLE_NAME.bgemm_r = BGEMM_DEFAULT_R;
 #endif
 #if BUILD_SINGLE == 1 || (BUILD_COMPLEX==1)
   TABLE_NAME.sgemm_r = SGEMM_DEFAULT_R;
@@ -1100,10 +1121,12 @@ static void init_parameter(void) {
 
 #ifdef BUILD_BFLOAT16
   TABLE_NAME.sbgemm_p = SBGEMM_DEFAULT_P;
+  TABLE_NAME.bgemm_p = BGEMM_DEFAULT_P;
 #endif
 
 #ifdef BUILD_BFLOAT16
   TABLE_NAME.sbgemm_r = SBGEMM_DEFAULT_R;
+  TABLE_NAME.bgemm_r = BGEMM_DEFAULT_R;
 #endif
 
 #if defined(LA464)
@@ -1202,6 +1225,7 @@ static void init_parameter(void) {
 
 #ifdef BUILD_BFLOAT16
   TABLE_NAME.sbgemm_q = SBGEMM_DEFAULT_Q;
+  TABLE_NAME.bgemm_q = BGEMM_DEFAULT_Q;
 #endif
 }
 #else // (ARCH_LOONGARCH64)
@@ -1210,6 +1234,7 @@ static void init_parameter(void) {
 
 #ifdef BUILD_BFLOAT16
   TABLE_NAME.sbgemm_p = SBGEMM_DEFAULT_P;
+  TABLE_NAME.bgemm_p = BGEMM_DEFAULT_P;
 #endif
   TABLE_NAME.sgemm_p = SGEMM_DEFAULT_P;
   TABLE_NAME.dgemm_p = DGEMM_DEFAULT_P;
@@ -1218,6 +1243,7 @@ static void init_parameter(void) {
 
 #ifdef BUILD_BFLOAT16
   TABLE_NAME.sbgemm_r = SBGEMM_DEFAULT_R;
+  TABLE_NAME.bgemm_r = BGEMM_DEFAULT_R;
 #endif
   TABLE_NAME.sgemm_r = SGEMM_DEFAULT_R;
   TABLE_NAME.dgemm_r = DGEMM_DEFAULT_R;
@@ -1227,6 +1253,7 @@ static void init_parameter(void) {
 
 #ifdef BUILD_BFLOAT16
   TABLE_NAME.sbgemm_q = SBGEMM_DEFAULT_Q;
+  TABLE_NAME.bgemm_q = BGEMM_DEFAULT_Q;
 #endif
   TABLE_NAME.sgemm_q = SGEMM_DEFAULT_Q;
   TABLE_NAME.dgemm_q = DGEMM_DEFAULT_Q;
@@ -1239,6 +1266,7 @@ static void init_parameter(void) {
 static void init_parameter(void) {
 #ifdef BUILD_BFLOAT16
 	TABLE_NAME.sbgemm_p = SBGEMM_DEFAULT_P;
+	TABLE_NAME.bgemm_p = BGEMM_DEFAULT_P;
 #endif
 	TABLE_NAME.sgemm_p = SGEMM_DEFAULT_P;
 	TABLE_NAME.dgemm_p = DGEMM_DEFAULT_P;
@@ -1247,6 +1275,7 @@ static void init_parameter(void) {
 
 #ifdef BUILD_BFLOAT16
 	TABLE_NAME.sbgemm_r = SBGEMM_DEFAULT_R;
+	TABLE_NAME.bgemm_r = BGEMM_DEFAULT_R;
 #endif
 	TABLE_NAME.sgemm_r = SGEMM_DEFAULT_R;
 	TABLE_NAME.dgemm_r = DGEMM_DEFAULT_R;
@@ -1256,6 +1285,7 @@ static void init_parameter(void) {
 
 #ifdef BUILD_BFLOAT16
 	TABLE_NAME.sbgemm_q = SBGEMM_DEFAULT_Q;
+	TABLE_NAME.bgemm_q = BGEMM_DEFAULT_Q;
 #endif
 	TABLE_NAME.sgemm_q = SGEMM_DEFAULT_Q;
 	TABLE_NAME.dgemm_q = DGEMM_DEFAULT_Q;
@@ -1269,6 +1299,7 @@ static void init_parameter(void) {
 
 #ifdef BUILD_BFLOAT16
   TABLE_NAME.sbgemm_p = SBGEMM_DEFAULT_P;
+  TABLE_NAME.bgemm_p = BGEMM_DEFAULT_P;
 #endif
 #ifdef BUILD_HFLOAT16
   TABLE_NAME.shgemm_p = SHGEMM_DEFAULT_P;
@@ -1280,6 +1311,7 @@ static void init_parameter(void) {
 
 #ifdef BUILD_BFLOAT16
   TABLE_NAME.sbgemm_r = SBGEMM_DEFAULT_R;
+  TABLE_NAME.bgemm_r = BGEMM_DEFAULT_R;
 #endif
 #ifdef BUILD_HFLOAT16
   TABLE_NAME.shgemm_r = SHGEMM_DEFAULT_R;
@@ -1292,6 +1324,7 @@ static void init_parameter(void) {
 
 #ifdef BUILD_BFLOAT16
   TABLE_NAME.sbgemm_q = SBGEMM_DEFAULT_Q;
+  TABLE_NAME.bgemm_q = BGEMM_DEFAULT_Q;
 #endif
 #ifdef BUILD_HFLOAT16
   TABLE_NAME.shgemm_q = SHGEMM_DEFAULT_Q;
@@ -1442,6 +1475,8 @@ static void init_parameter(void) {
 #ifdef BUILD_BFLOAT16
   TABLE_NAME.sbgemm_p = SBGEMM_DEFAULT_P;
   TABLE_NAME.sbgemm_q = SBGEMM_DEFAULT_Q;
+  TABLE_NAME.bgemm_p = BGEMM_DEFAULT_P;
+  TABLE_NAME.bgemm_q = BGEMM_DEFAULT_Q;
 #endif
 #ifdef BUILD_HFLOAT16
   TABLE_NAME.shgemm_p = SHGEMM_DEFAULT_P;
@@ -2040,6 +2075,10 @@ static void init_parameter(void) {
 			       ((TABLE_NAME.sbgemm_p * TABLE_NAME.sbgemm_q *  4 + TABLE_NAME.offsetA
 				 + TABLE_NAME.align) & ~TABLE_NAME.align)
 			       ) / (TABLE_NAME.sbgemm_q *  4) - 15) & ~15);
+  TABLE_NAME.bgemm_r = (((BUFFER_SIZE -
+			       ((TABLE_NAME.bgemm_p * TABLE_NAME.bgemm_q *  4 + TABLE_NAME.offsetA
+				 + TABLE_NAME.align) & ~TABLE_NAME.align)
+			       ) / (TABLE_NAME.bgemm_q *  4) - 15) & ~15);
 #endif
 
 #if BUILD_HFLOAT16==1

lapack-netlib/LAPACKE/src/lapacke_dtrsen.c

@@ -78,6 +78,12 @@ lapack_int LAPACKE_dtrsen( int matrix_layout, char job, char compq,
             info = LAPACK_WORK_MEMORY_ERROR;
             goto exit_level_0;
         }
+    } else {
+        iwork = (lapack_int*)LAPACKE_malloc( sizeof(lapack_int) );
+        if( iwork == NULL ) {
+            info = LAPACK_WORK_MEMORY_ERROR;
+            goto exit_level_0;
+        }
     }
     work = (double*)LAPACKE_malloc( sizeof(double) * lwork );
     if( work == NULL ) {

lapack-netlib/LAPACKE/src/lapacke_strsen.c

@@ -77,7 +77,14 @@ lapack_int LAPACKE_strsen( int matrix_layout, char job, char compq,
             info = LAPACK_WORK_MEMORY_ERROR;
             goto exit_level_0;
         }
+    } else {
+        iwork = (lapack_int*)LAPACKE_malloc( sizeof(lapack_int) );
+        if( iwork == NULL ) {
+            info = LAPACK_WORK_MEMORY_ERROR;
+            goto exit_level_0;
+        }
     }
+
     work = (float*)LAPACKE_malloc( sizeof(float) * lwork );
     if( work == NULL ) {
         info = LAPACK_WORK_MEMORY_ERROR;

lapack/CMakeLists.txt

@@ -52,7 +52,7 @@ GenerateNamedObjects("laswp/generic/laswp_k_4.c" "" "laswp_plus" false "" ""  fa
 GenerateNamedObjects("laswp/generic/laswp_k_4.c" "MINUS" "laswp_minus" false "" ""  false 3)
 
 foreach (float_type ${FLOAT_TYPES})
-if (${float_type} STREQUAL "HALF")
+if (${float_type} STREQUAL "BFLOAT16")
  continue()
 endif()
 GenerateNamedObjects("getrf/getrf_single.c" "UNIT" "getrf_single" false "" "" false ${float_type})

lapack/potrf/potrf_parallel.c

@@ -1,5 +1,6 @@
 /*********************************************************************/
 /* Copyright 2009, 2010 The University of Texas at Austin.           */
+/* Copyright 2025 The OpenBLAS Project.                              */
 /* All rights reserved.                                              */
 /*                                                                   */
 /* Redistribution and use in source and binary forms, with or        */
@@ -405,8 +406,8 @@ static int thread_driver(blas_arg_t *args, FLOAT *sa, FLOAT *sb){
 #elif defined(DOUBLE)
   mode  =  BLAS_DOUBLE  | BLAS_REAL;
   mask  = MAX(DGEMM_UNROLL_M, DGEMM_UNROLL_N) - 1;
-#elif defined(HALF)
-  mode  =  BLAS_HALF  | BLAS_REAL;
+#elif defined(BFLOAT16)
+  mode  =  BLAS_BFLOAT16  | BLAS_REAL;
   mask  = MAX(SBGEMM_UNROLL_M, SBGEMM_UNROLL_N) - 1;
 #else
   mode  =  BLAS_SINGLE  | BLAS_REAL;

param.h

@@ -1,5 +1,5 @@
 /*****************************************************************************
-Copyright (c) 2011-2023, The OpenBLAS Project
+Copyright (c) 2011-2023, 2025 The OpenBLAS Project
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
@@ -79,6 +79,14 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define SHGEMM_DEFAULT_R 240
 #define SHGEMM_DEFAULT_Q 12288
 
+#define BGEMM_DEFAULT_UNROLL_N 4
+#define BGEMM_DEFAULT_UNROLL_M 8
+#define BGEMM_DEFAULT_UNROLL_MN 32
+#define BGEMM_DEFAULT_P 256
+#define BGEMM_DEFAULT_R 256
+#define BGEMM_DEFAULT_Q 256
+#define BGEMM_ALIGN_K 1  // must be 2^x
+
 #define SBGEMM_DEFAULT_UNROLL_N 4
 #define SBGEMM_DEFAULT_UNROLL_M 8
 #define SBGEMM_DEFAULT_UNROLL_MN 32
@@ -3585,6 +3593,13 @@ is a big desktop or server with abundant cache rather than a phone or embedded d
 #define GEMM_PREFERED_SIZE      8
 #endif
 
+#undef BGEMM_ALIGN_K
+#undef BGEMM_DEFAULT_UNROLL_M
+#undef BGEMM_DEFAULT_UNROLL_N
+#define BGEMM_ALIGN_K 8
+#define BGEMM_DEFAULT_UNROLL_N 4
+#define BGEMM_DEFAULT_UNROLL_M 4
+
 #undef SBGEMM_ALIGN_K
 #undef SBGEMM_DEFAULT_UNROLL_M
 #undef SBGEMM_DEFAULT_UNROLL_N

test/Makefile

@@ -1,3 +1,31 @@
+###############################################################################
+# Copyright (c) 2025, The OpenBLAS Project
+# All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in
+#    the documentation and/or other materials provided with the
+#    distribution.
+# 3. Neither the name of the OpenBLAS project nor the names of
+#    its contributors may be used to endorse or promote products
+#    derived from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+###############################################################################
 TOPDIR	= ..
 include ../Makefile.system
 ifeq ($(F_COMPILER),GFORTRAN)
@@ -6,6 +34,7 @@ ifneq (, $(filter $(CORE),LOONGSON3R3 LOONGSON3R4))
 endif
         override FFLAGS += -fno-tree-vectorize
 endif
+override CFLAGS += -std=c11 -Wall -Werror
 
 SUPPORT_GEMM3M = 0
 
@@ -90,6 +119,10 @@ endif
 endif
 endif
 
+ifeq ($(BUILD_BFLOAT16), 1)
+BB2 = test_bgemv
+B2 = test_sbgemv
+endif
 ifeq ($(BUILD_SINGLE),1)
 S2=sblat2
 endif
@@ -103,11 +136,17 @@ ifeq ($(BUILD_COMPLEX16),1)
 Z2=zblat2
 endif
 
-level2: $(S2) $(D2) $(C2) $(Z2)
+level2: $(BB2) $(B2) $(S2) $(D2) $(C2) $(Z2)
 
 
 ifneq ($(CROSS), 1)
 	rm -f ?BLAT2.SUMM
+ifeq ($(BUILD_BFLOAT16),1)
+	OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 ./test_bgemv > BBLAT2.SUMM
+	@$(GREP) -q FATAL BBLAT2.SUMM && cat BBLAT2.SUMM || exit 0
+	OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 ./test_sbgemv > SBBLAT2.SUMM
+	@$(GREP) -q FATAL SBBLAT2.SUMM && cat SBBLAT2.SUMM || exit 0
+endif
 ifeq ($(BUILD_SINGLE),1)
 	OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 ./sblat2 < ./sblat2.dat
 	@$(GREP) -q FATAL SBLAT2.SUMM && cat SBLAT2.SUMM || exit 0
@@ -127,6 +166,12 @@ endif
 ifdef SMP
 	rm -f ?BLAT2.SUMM
 ifeq ($(USE_OPENMP), 1)
+ifeq ($(BUILD_BFLOAT16),1)
+	OMP_NUM_THREADS=2 ./test_bgemv > BBLAT2.SUMM
+	@$(GREP) -q FATAL BBLAT2.SUMM && cat BBLAT2.SUMM || exit 0
+	OMP_NUM_THREADS=2 ./test_sbgemv > SBBLAT2.SUMM
+	@$(GREP) -q FATAL SBBLAT2.SUMM && cat SBBLAT2.SUMM || exit 0
+endif
 ifeq ($(BUILD_SINGLE),1)
 	OMP_NUM_THREADS=2 ./sblat2 < ./sblat2.dat
 	@$(GREP) -q FATAL SBLAT2.SUMM && cat SBLAT2.SUMM || exit 0
@@ -144,6 +189,12 @@ ifeq ($(BUILD_COMPLEX16),1)
 	@$(GREP) -q FATAL ZBLAT2.SUMM && cat ZBLAT2.SUMM || exit 0
 endif
 else
+ifeq ($(BUILD_BFLOAT16),1)
+	OMP_NUM_THREADS=2 ./test_bgemv > BBLAT2.SUMM
+	@$(GREP) -q FATAL BBLAT2.SUMM && cat BBLAT2.SUMM || exit 0
+	OMP_NUM_THREADS=2 ./test_sbgemv > SBBLAT2.SUMM
+	@$(GREP) -q FATAL SBBLAT2.SUMM && cat SBBLAT2.SUMM || exit 0
+endif
 ifeq ($(BUILD_SINGLE),1)
 	OPENBLAS_NUM_THREADS=2 ./sblat2 < ./sblat2.dat
 	@$(GREP) -q FATAL SBLAT2.SUMM && cat SBLAT2.SUMM || exit 0
@@ -165,7 +216,8 @@ endif
 endif
 
 ifeq ($(BUILD_BFLOAT16),1)
-B3= test_sbgemm
+BF3= test_bgemm
+B3 = test_sbgemm
 endif
 ifeq ($(BUILD_SINGLE),1)
 S3=sblat3
@@ -192,7 +244,7 @@ endif
 ifeq ($(SUPPORT_GEMM3M),1)
 level3: $(B3) $(S3) $(D3) $(C3) $(Z3) level3_3m
 else
-level3: $(B3) $(S3) $(D3) $(C3) $(Z3)
+level3: $(BF3) $(B3) $(S3) $(D3) $(C3) $(Z3)
 endif
 
 ifneq ($(CROSS), 1)
@@ -200,6 +252,8 @@ ifneq ($(CROSS), 1)
 ifeq ($(BUILD_BFLOAT16),1)
 	OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 ./test_sbgemm > SBBLAT3.SUMM
 	@$(GREP) -q FATAL SBBLAT3.SUMM && cat SBBLAT3.SUMM || exit 0
+	OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 ./test_bgemm > BBLAT3.SUMM
+	@$(GREP) -q FATAL BBLAT3.SUMM && cat BBLAT3.SUMM || exit 0
 endif
 ifeq ($(BUILD_SINGLE),1)
 	OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 ./sblat3 < ./sblat3.dat
@@ -223,6 +277,8 @@ ifeq ($(USE_OPENMP), 1)
 ifeq ($(BUILD_BFLOAT16),1)
 	OMP_NUM_THREADS=2 ./test_sbgemm > SBBLAT3.SUMM
 	@$(GREP) -q FATAL SBBLAT3.SUMM && cat SBBLAT3.SUMM || exit 0
+	OMP_NUM_THREADS=2 ./test_bgemm > BBLAT3.SUMM
+	@$(GREP) -q FATAL BBLAT3.SUMM && cat BBLAT3.SUMM || exit 0
 endif
 ifeq ($(BUILD_SINGLE),1)
 	OMP_NUM_THREADS=2 ./sblat3 < ./sblat3.dat
@@ -244,6 +300,8 @@ else
 ifeq ($(BUILD_BFLOAT16),1)
 	OPENBLAS_NUM_THREADS=2 ./test_sbgemm > SBBLAT3.SUMM
 	@$(GREP) -q FATAL SBBLAT3.SUMM && cat SBBLAT3.SUMM || exit 0
+	OPENBLAS_NUM_THREADS=2 ./test_bgemm > BBLAT3.SUMM
+	@$(GREP) -q FATAL BBLAT3.SUMM && cat BBLAT3.SUMM || exit 0
 endif
 ifeq ($(BUILD_SINGLE),1)
 	OPENBLAS_NUM_THREADS=2 ./sblat3 < ./sblat3.dat
@@ -367,8 +425,17 @@ zblat3 : zblat3.$(SUFFIX) ../$(LIBNAME)
 endif
 
 ifeq ($(BUILD_BFLOAT16),1)
-test_sbgemm : compare_sgemm_sbgemm.c ../$(LIBNAME)
-	$(CC) $(CLDFLAGS) -o test_sbgemm compare_sgemm_sbgemm.c ../$(LIBNAME) $(EXTRALIB) $(CEXTRALIB)
+test_bgemm : compare_sgemm_bgemm.c test_helpers.h ../$(LIBNAME)
+	$(CC) $(CLDFLAGS) -DIBFLOAT16 -DOBFLOAT16 -o test_bgemm compare_sgemm_bgemm.c ../$(LIBNAME) $(EXTRALIB) $(CEXTRALIB)
+
+test_bgemv : compare_sgemv_bgemv.c ../$(LIBNAME)
+	$(CC) $(CLDFLAGS) -DIBFLOAT16 -DOBFLOAT16 -o test_bgemv compare_sgemv_bgemv.c ../$(LIBNAME) $(EXTRALIB) $(CEXTRALIB)
+
+test_sbgemm : compare_sgemm_sbgemm.c test_helpers.h ../$(LIBNAME)
+	$(CC) $(CLDFLAGS) -DIBFLOAT16 -o test_sbgemm compare_sgemm_sbgemm.c ../$(LIBNAME) $(EXTRALIB) $(CEXTRALIB)
+
+test_sbgemv : compare_sgemv_sbgemv.c ../$(LIBNAME)
+	$(CC) $(CLDFLAGS) -DIBFLOAT16 -o test_sbgemv compare_sgemv_sbgemv.c ../$(LIBNAME) $(EXTRALIB) $(CEXTRALIB)
 endif
 
 ifeq ($(BUILD_COMPLEX),1)
@@ -387,7 +454,7 @@ clean:
 	@rm -f *.$(SUFFIX) *.$(PSUFFIX) gmon.$(SUFFIX)ut *.SUMM *.cxml *.exe *.pdb *.dwf \
 	sblat1 dblat1 cblat1 zblat1 \
 	sblat2 dblat2 cblat2 zblat2 \
-	test_sbgemm sblat3 dblat3 cblat3 zblat3 \
+	test_bgemm test_bgemv test_sbgemm test_sbgemv sblat3 dblat3 cblat3 zblat3 \
 	sblat1p dblat1p cblat1p zblat1p \
 	sblat2p dblat2p cblat2p zblat2p \
 	sblat3p dblat3p cblat3p zblat3p \

test/compare_sgemm_bgemm.c

@@ -0,0 +1,155 @@
+/***************************************************************************
+Copyright (c) 2025 The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
+GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+#include "../common.h"
+#include <stdint.h>
+#include <stdio.h>
+
+#include "test_helpers.h"
+
+#define SGEMM BLASFUNC(sgemm)
+#define BGEMM BLASFUNC(bgemm)
+#define BGEMM_LARGEST 256
+
+int
+main (int argc, char *argv[])
+{
+  blasint m, n, k;
+  int i, j, l;
+  blasint x, y;
+  blasint one = 1;
+  int ret = 0;
+  int loop = BGEMM_LARGEST;
+  char transA = 'N', transB = 'N';
+  float alpha = 1.0, beta = 0.0;
+  bfloat16 alpha_bf16;
+  sbstobf16_(&one, &alpha, &one, &alpha_bf16, &one);
+  bfloat16 beta_bf16;
+  sbstobf16_(&one, &beta, &one, &beta_bf16, &one);
+
+  for (x = 0; x <= loop; x++)
+  {
+    if ((x > 100) && (x != BGEMM_LARGEST)) continue;
+    m = k = n = x;
+    float *A = (float *)malloc_safe(m * k * sizeof(FLOAT));
+    float *B = (float *)malloc_safe(k * n * sizeof(FLOAT));
+    float *C = (float *)malloc_safe(m * n * sizeof(FLOAT));
+    bfloat16 *AA = (bfloat16 *)malloc_safe(m * k * sizeof(bfloat16));
+    bfloat16 *BB = (bfloat16 *)malloc_safe(k * n * sizeof(bfloat16));
+    bfloat16 *CC = (bfloat16 *)malloc_safe(k * n * sizeof(bfloat16));
+    FLOAT *DD = (FLOAT *)malloc_safe(m * n * sizeof(FLOAT));
+    if ((A == NULL) || (B == NULL) || (C == NULL) || (AA == NULL) || (BB == NULL) ||
+        (DD == NULL) || (CC == NULL))
+      return 1;
+
+    for (j = 0; j < m; j++)
+    {
+      for (i = 0; i < k; i++)
+      {
+        A[j * k + i] = ((FLOAT) rand () / (FLOAT) RAND_MAX) + 0.5;
+        sbstobf16_(&one, &A[j*k+i], &one, &AA[j * k + i], &one);
+      }
+    }
+    for (j = 0; j < n; j++)
+    {
+      for (i = 0; i < k; i++)
+      {
+        B[j * k + i] = ((FLOAT) rand () / (FLOAT) RAND_MAX) + 0.5;
+        sbstobf16_(&one, &B[j*k+i], &one, &BB[j * k + i], &one);
+      }
+    }
+    for (y = 0; y < 4; y++)
+    {
+      if ((y == 0) || (y == 2)) {
+        transA = 'N';
+      } else {
+        transA = 'T';
+      }
+      if ((y == 0) || (y == 1)) {
+        transB = 'N';
+      } else {
+        transB = 'T';
+      }
+
+      memset(CC, 0, m * n * sizeof(bfloat16));
+      memset(DD, 0, m * n * sizeof(FLOAT));
+      memset(C, 0, m * n * sizeof(FLOAT));
+
+      SGEMM (&transA, &transB, &m, &n, &k, &alpha, A,
+        &m, B, &k, &beta, C, &m);
+      BGEMM (&transA, &transB, &m, &n, &k, &alpha_bf16, (bfloat16*) AA,
+        &m, (bfloat16*)BB, &k, &beta_bf16, (bfloat16*)CC, &m);
+
+      for (i = 0; i < n; i++)
+        for (j = 0; j < m; j++)
+        {
+          for (l = 0; l < k; l++)
+            if (transA == 'N' && transB == 'N')
+            {
+              DD[i * m + j] +=
+                float16to32 (AA[l * m + j]) * float16to32 (BB[l + k * i]);
+            } else if (transA == 'T' && transB == 'N')
+            {
+              DD[i * m + j] +=
+                float16to32 (AA[k * j + l]) * float16to32 (BB[l + k * i]);
+            } else if (transA == 'N' && transB == 'T')
+            {
+              DD[i * m + j] +=
+                float16to32 (AA[l * m + j]) * float16to32 (BB[i + l * n]);
+            } else if (transA == 'T' && transB == 'T')
+            {
+              DD[i * m + j] +=
+                float16to32 (AA[k * j + l]) * float16to32 (BB[i + l * n]);
+            }
+          if (!is_close(float16to32(CC[i * m + j]), truncate_float32_to_bfloat16(C[i * m + j]), 0.01, 0.001)) {
+            printf("Mismatch at i=%d, j=%d, k=%d: CC=%.6f, C=%.6f\n",
+                    i, j, k, float16to32(CC[i * m + j]), truncate_float32_to_bfloat16(C[i * m + j]));
+            ret++;
+          }
+
+          if (!is_close(float16to32(CC[i * m + j]), truncate_float32_to_bfloat16(DD[i * m + j]), 0.0001, 0.00001)) {
+            printf("Mismatch at i=%d, j=%d, k=%d: CC=%.6f, DD=%.6f\n",
+                    i, j, k, float16to32(CC[i * m + j]), truncate_float32_to_bfloat16(DD[i * m + j]));
+            ret++;
+          }
+            
+        }
+    }
+    free(A);
+    free(B);
+    free(C);
+    free(AA);
+    free(BB);
+    free(CC);
+    free(DD);
+  }
+
+  if (ret != 0) {
+    fprintf (stderr, "FATAL ERROR BGEMM - Return code: %d\n", ret);
+  }
+
+  return ret;
+}

test/compare_sgemm_sbgemm.c

@@ -27,72 +27,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <stdio.h>
 #include <stdint.h>
 #include "../common.h"
+
+#include "test_helpers.h"
+
 #define SGEMM   BLASFUNC(sgemm)
 #define SBGEMM   BLASFUNC(sbgemm)
 #define SGEMV   BLASFUNC(sgemv)
 #define SBGEMV   BLASFUNC(sbgemv)
-typedef union
-{
-  unsigned short v;
-#if defined(_AIX)
-  struct __attribute__((packed))
-#else
-  struct
-#endif
-  {
-#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-    unsigned short s:1;
-    unsigned short e:8;
-    unsigned short m:7;
-#else
-    unsigned short m:7;
-    unsigned short e:8;
-    unsigned short s:1;
-#endif
-  } bits;
-} bfloat16_bits;
-
-typedef union
-{
-  float v;
-#if defined(_AIX)
-  struct __attribute__((packed))
-#else
-  struct
-#endif
-  {
-#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-    uint32_t s:1;
-    uint32_t e:8;
-    uint32_t m:23;
-#else
-    uint32_t m:23;
-    uint32_t e:8;
-    uint32_t s:1;
-#endif
-  } bits;
-} float32_bits;
-
-float
-float16to32 (bfloat16_bits f16)
-{
-  float32_bits f32;
-  f32.bits.s = f16.bits.s;
-  f32.bits.e = f16.bits.e;
-  f32.bits.m = (uint32_t) f16.bits.m << 16;
-  return f32.v;
-}
-
 #define SBGEMM_LARGEST  256
 
-void *malloc_safe(size_t size)
-{
-  if (size == 0)
-    return malloc(1);
-  else
-    return malloc(size);
-}
-
 int
 main (int argc, char *argv[])
 {
@@ -111,14 +54,13 @@ main (int argc, char *argv[])
     float *A = (float *)malloc_safe(m * k * sizeof(FLOAT));
     float *B = (float *)malloc_safe(k * n * sizeof(FLOAT));
     float *C = (float *)malloc_safe(m * n * sizeof(FLOAT));
-    bfloat16_bits *AA = (bfloat16_bits *)malloc_safe(m * k * sizeof(bfloat16_bits));
-    bfloat16_bits *BB = (bfloat16_bits *)malloc_safe(k * n * sizeof(bfloat16_bits));
+    bfloat16 *AA = (bfloat16 *)malloc_safe(m * k * sizeof(bfloat16));
+    bfloat16 *BB = (bfloat16 *)malloc_safe(k * n * sizeof(bfloat16));
     float *DD = (float *)malloc_safe(m * n * sizeof(FLOAT));
     float *CC = (float *)malloc_safe(m * n * sizeof(FLOAT));
     if ((A == NULL) || (B == NULL) || (C == NULL) || (AA == NULL) || (BB == NULL) ||
         (DD == NULL) || (CC == NULL))
       return 1;
-    bfloat16 atmp,btmp;
     blasint one=1;
 
     for (j = 0; j < m; j++)
@@ -126,8 +68,7 @@ main (int argc, char *argv[])
       for (i = 0; i < k; i++)
       {
         A[j * k + i] = ((FLOAT) rand () / (FLOAT) RAND_MAX) + 0.5;
-        sbstobf16_(&one, &A[j*k+i], &one, &atmp, &one);
-        AA[j * k + i].v = atmp;
+        sbstobf16_(&one, &A[j*k+i], &one, &AA[j * k + i], &one);
       }
     }
     for (j = 0; j < n; j++)
@@ -135,8 +76,7 @@ main (int argc, char *argv[])
       for (i = 0; i < k; i++)
       {
         B[j * k + i] = ((FLOAT) rand () / (FLOAT) RAND_MAX) + 0.5;
-        sbstobf16_(&one, &B[j*k+i], &one, &btmp, &one);
-        BB[j * k + i].v = btmp;
+        sbstobf16_(&one, &B[j*k+i], &one, &BB[j * k + i], &one);
       }
     }
     for (y = 0; y < 4; y++)
@@ -182,10 +122,12 @@ main (int argc, char *argv[])
               DD[i * m + j] +=
                 float16to32 (AA[k * j + l]) * float16to32 (BB[i + l * n]);
             }
-          if (fabs (CC[i * m + j] - C[i * m + j]) > 1.0)
+          if (!is_close(CC[i * m + j], C[i * m + j], 0.01, 0.001)) {
             ret++;
-          if (fabs (CC[i * m + j] - DD[i * m + j]) > 1.0)
+          }
+          if (!is_close(CC[i * m + j], DD[i * m + j], 0.001, 0.0001)) {
             ret++;
+          }
         }
     }
     free(A);
@@ -199,88 +141,7 @@ main (int argc, char *argv[])
 
   if (ret != 0) {
     fprintf (stderr, "FATAL ERROR SBGEMM - Return code: %d\n", ret);
-    return ret;
   }
 
-  for (beta = 0; beta < 3; beta += 1) {
-  for (alpha = 0; alpha < 3; alpha += 1) {
-  for (l = 0; l < 2; l++) {  // l = 1 to test inc_x & inc_y not equal to one.
-  for (x = 1; x <= loop; x++)
-  {
-    k = (x == 0) ? 0 : l + 1;
-    float *A = (float *)malloc_safe(x * x * sizeof(FLOAT));
-    float *B = (float *)malloc_safe(x * sizeof(FLOAT) << l);
-    float *C = (float *)malloc_safe(x * sizeof(FLOAT) << l);
-    bfloat16_bits *AA = (bfloat16_bits *)malloc_safe(x * x * sizeof(bfloat16_bits));
-    bfloat16_bits *BB = (bfloat16_bits *)malloc_safe(x * sizeof(bfloat16_bits) << l);
-    float *DD = (float *)malloc_safe(x * sizeof(FLOAT));
-    float *CC = (float *)malloc_safe(x * sizeof(FLOAT) << l);
-    if ((A == NULL) || (B == NULL) || (C == NULL) || (AA == NULL) || (BB == NULL) ||
-        (DD == NULL) || (CC == NULL))
-      return 1;
-    bfloat16 atmp, btmp;
-    blasint one = 1;
-
-    for (j = 0; j < x; j++)
-    {
-      for (i = 0; i < x; i++)
-      {
-        A[j * x + i] = ((FLOAT) rand () / (FLOAT) RAND_MAX) + 0.5;
-        sbstobf16_(&one, &A[j*x+i], &one, &atmp, &one);
-        AA[j * x + i].v = atmp;
-      }
-      B[j << l] = ((FLOAT) rand () / (FLOAT) RAND_MAX) + 0.5;
-      sbstobf16_(&one, &B[j << l], &one, &btmp, &one);
-      BB[j << l].v = btmp;
-      
-      CC[j << l] = C[j << l] = ((FLOAT) rand () / (FLOAT) RAND_MAX) + 0.5;
-    }
-
-    for (y = 0; y < 2; y++)
-    {
-      if (y == 0) {
-        transA = 'N';
-      } else {
-        transA = 'T';
-      }
-
-      memset(CC, 0, x * sizeof(FLOAT) << l);
-      memset(DD, 0, x * sizeof(FLOAT));
-      memset(C, 0, x * sizeof(FLOAT) << l);
-
-      SGEMV (&transA, &x, &x, &alpha, A, &x, B, &k, &beta, C, &k);
-      SBGEMV (&transA, &x, &x, &alpha, (bfloat16*) AA, &x, (bfloat16*) BB, &k, &beta, CC, &k);
-
-      for (int i = 0; i < x; i ++) DD[i] *= beta;
-
-      for (j = 0; j < x; j++)
-        for (i = 0; i < x; i++)
-          if (transA == 'N') {
-            DD[i] += alpha * float16to32 (AA[j * x + i]) * float16to32 (BB[j << l]);
-          } else if (transA == 'T') {
-            DD[j] += alpha * float16to32 (AA[j * x + i]) * float16to32 (BB[i << l]);
-          }
-
-      for (j = 0; j < x; j++) {
-        if (fabs (CC[j << l] - C[j << l]) > 1.0)
-          ret++;
-        if (fabs (CC[j << l] - DD[j]) > 1.0)
-          ret++;
-      }
-    }
-    free(A);
-    free(B);
-    free(C);
-    free(AA);
-    free(BB);
-    free(DD);
-    free(CC);
-  } // x
-  } // l
-  } // alpha
-  } // beta
-
-  if (ret != 0)
-    fprintf (stderr, "FATAL ERROR SBGEMV - Return code: %d\n", ret);
   return ret;
 }

test/compare_sgemv_bgemv.c

@@ -0,0 +1,149 @@
+/***************************************************************************
+Copyright (c) 2020,2025 The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+#include <stdio.h>
+#include <stdint.h>
+#include "../common.h"
+
+#include "test_helpers.h"
+
+#define SGEMV BLASFUNC(sgemv)
+#define BGEMV BLASFUNC(bgemv)
+#define BGEMV_LARGEST 256
+
+int main(int argc, char *argv[])
+{
+  blasint k;
+  int i, j, l;
+  blasint x, y;
+  blasint one = 1;
+  int ret = 0;
+  int loop = BGEMV_LARGEST;
+  char transA = 'N';
+  float alpha = 1.0, beta = 0.0;
+  bfloat16 alpha_bf16, beta_bf16;
+
+  for (beta = 0; beta < 3; beta += 1)
+  {
+    for (alpha = 0; alpha < 3; alpha += 1)
+    {
+      for (l = 0; l < 2; l++)
+      { // l = 1 to test inc_x & inc_y not equal to one.
+        for (x = 1; x <= loop; x++)
+        {
+          k = (x == 0) ? 0 : l + 1;
+          float *A = (float *)malloc_safe(x * x * sizeof(FLOAT));
+          float *B = (float *)malloc_safe(x * sizeof(FLOAT) << l);
+          float *C = (float *)malloc_safe(x * sizeof(FLOAT) << l);
+          bfloat16 *AA = (bfloat16 *)malloc_safe(x * x * sizeof(bfloat16));
+          bfloat16 *BB = (bfloat16 *)malloc_safe(x * sizeof(bfloat16) << l);
+          bfloat16 *CC = (bfloat16 *)malloc_safe(x * sizeof(bfloat16) << l);
+          float *DD = (float *)malloc_safe(x * sizeof(FLOAT));
+          if ((A == NULL) || (B == NULL) || (C == NULL) || (AA == NULL) || (BB == NULL) ||
+              (CC == NULL) || (DD == NULL))
+            return 1;
+
+          for (j = 0; j < x; j++)
+          {
+            for (i = 0; i < x; i++)
+            {
+              A[j * x + i] = ((FLOAT)rand() / (FLOAT)RAND_MAX) + 0.5;
+              sbstobf16_(&one, &A[j * x + i], &one, &AA[j * x + i], &one);
+            }
+            B[j << l] = ((FLOAT)rand() / (FLOAT)RAND_MAX) + 0.5;
+            sbstobf16_(&one, &B[j << l], &one, &BB[j << l], &one);
+
+            C[j << l] = ((FLOAT)rand() / (FLOAT)RAND_MAX) + 0.5;
+            sbstobf16_(&one, &B[j << l], &one, &CC[j << l], &one);
+          }
+
+          for (y = 0; y < 2; y++)
+          {
+            if (y == 0)
+            {
+              transA = 'N';
+            }
+            else
+            {
+              transA = 'T';
+            }
+
+            memset(C, 0, x * sizeof(FLOAT) << l);
+            memset(CC, 0, x * sizeof(bfloat16) << l);
+            memset(DD, 0, x * sizeof(FLOAT));
+
+            sbstobf16_(&one, &alpha, &one, &alpha_bf16, &one);
+            sbstobf16_(&one, &beta, &one, &beta_bf16, &one);
+            SGEMV(&transA, &x, &x, &alpha, A, &x, B, &k, &beta, C, &k);
+            BGEMV(&transA, &x, &x, &alpha_bf16, AA, &x, BB, &k, &beta_bf16, CC, &k);
+
+            for (int i = 0; i < x; i++)
+              DD[i] *= beta;
+
+            for (j = 0; j < x; j++)
+              for (i = 0; i < x; i++)
+                if (transA == 'N')
+                {
+                  DD[i] += alpha * float16to32(AA[j * x + i]) * float16to32(BB[j << l]);
+                }
+                else if (transA == 'T')
+                {
+                  DD[j] += alpha * float16to32(AA[j * x + i]) * float16to32(BB[i << l]);
+                }
+
+            for (j = 0; j < x; j++)
+            {
+              if (!is_close(float16to32(CC[j << l]), truncate_float32_to_bfloat16(C[j << l]), 0.01, 0.001))
+              {
+                printf("Mismatch at trans=%c, alpha=%.2f, beta=%.2f, i=%d, j=%d, k=%d: CC=%.6f, C=%.6f\n",
+                       transA, alpha, beta, i, j, k, float16to32(CC[j << l]), truncate_float32_to_bfloat16(C[j << l]));
+                ret++;
+              }
+              if (!is_close(float16to32(CC[j << l]), truncate_float32_to_bfloat16(DD[j]), 0.001, 0.0001))
+              {
+                printf("Mismatch at trans=%c, alpha=%.2f, beta=%.2f, i=%d, j=%d, k=%d: CC=%.6f, C=%.6f\n",
+                       transA, alpha, beta, i, j, k, float16to32(CC[j << l]), truncate_float32_to_bfloat16(DD[j]));
+                ret++;
+              }
+            }
+          }
+
+          free(A);
+          free(B);
+          free(C);
+          free(AA);
+          free(BB);
+          free(CC);
+          free(DD);
+        } // x
+      } // l
+    } // alpha
+  } // beta
+
+  if (ret != 0)
+    fprintf(stderr, "FATAL ERROR BGEMV - Return code: %d\n", ret);
+  return ret;
+}

test/compare_sgemv_sbgemv.c

@@ -0,0 +1,128 @@
+/***************************************************************************
+Copyright (c) 2020,2025 The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+#include <stdio.h>
+#include <stdint.h>
+#include "../common.h"
+
+#include "test_helpers.h"
+
+#define SGEMV   BLASFUNC(sgemv)
+#define SBGEMV   BLASFUNC(sbgemv)
+#define SBGEMV_LARGEST  256
+
+int
+main (int argc, char *argv[])
+{
+  blasint k;
+  int i, j, l;
+  blasint x, y;
+  int ret = 0;
+  int loop = SBGEMV_LARGEST;
+  char transA = 'N';
+  float alpha = 1.0, beta = 0.0;
+
+  for (beta = 0; beta < 3; beta += 1) {
+  for (alpha = 0; alpha < 3; alpha += 1) {
+  for (l = 0; l < 2; l++) {  // l = 1 to test inc_x & inc_y not equal to one.
+  for (x = 1; x <= loop; x++)
+  {
+    k = (x == 0) ? 0 : l + 1;
+    float *A = (float *)malloc_safe(x * x * sizeof(FLOAT));
+    float *B = (float *)malloc_safe(x * sizeof(FLOAT) << l);
+    float *C = (float *)malloc_safe(x * sizeof(FLOAT) << l);
+    bfloat16 *AA = (bfloat16 *)malloc_safe(x * x * sizeof(bfloat16));
+    bfloat16 *BB = (bfloat16 *)malloc_safe(x * sizeof(bfloat16) << l);
+    float *CC = (float *)malloc_safe(x * sizeof(FLOAT) << l);
+    float *DD = (float *)malloc_safe(x * sizeof(FLOAT));
+    if ((A == NULL) || (B == NULL) || (C == NULL) || (AA == NULL) || (BB == NULL) ||
+        (DD == NULL) || (CC == NULL))
+      return 1;
+    blasint one = 1;
+
+    for (j = 0; j < x; j++)
+    {
+      for (i = 0; i < x; i++)
+      {
+        A[j * x + i] = ((FLOAT) rand () / (FLOAT) RAND_MAX) + 0.5;
+        sbstobf16_(&one, &A[j*x+i], &one, &AA[j * x + i], &one);
+      }
+      B[j << l] = ((FLOAT) rand () / (FLOAT) RAND_MAX) + 0.5;
+      sbstobf16_(&one, &B[j << l], &one, &BB[j << l], &one);
+      
+      CC[j << l] = C[j << l] = ((FLOAT) rand () / (FLOAT) RAND_MAX) + 0.5;
+    }
+
+    for (y = 0; y < 2; y++)
+    {
+      if (y == 0) {
+        transA = 'N';
+      } else {
+        transA = 'T';
+      }
+
+      memset(CC, 0, x * sizeof(FLOAT) << l);
+      memset(DD, 0, x * sizeof(FLOAT));
+      memset(C, 0, x * sizeof(FLOAT) << l);
+
+      SGEMV (&transA, &x, &x, &alpha, A, &x, B, &k, &beta, C, &k);
+      SBGEMV (&transA, &x, &x, &alpha, (bfloat16*) AA, &x, (bfloat16*) BB, &k, &beta, CC, &k);
+
+      for (int i = 0; i < x; i ++) DD[i] *= beta;
+
+      for (j = 0; j < x; j++)
+        for (i = 0; i < x; i++)
+          if (transA == 'N') {
+            DD[i] += alpha * float16to32 (AA[j * x + i]) * float16to32 (BB[j << l]);
+          } else if (transA == 'T') {
+            DD[j] += alpha * float16to32 (AA[j * x + i]) * float16to32 (BB[i << l]);
+          }
+
+      for (j = 0; j < x; j++) {
+        if (!is_close(CC[j << l], C[j << l], 0.01, 0.001)) {
+          ret++;
+        }
+        if (!is_close(CC[j << l], DD[j], 0.001, 0.0001)) {
+          ret++;
+        }
+      }
+    }
+    free(A);
+    free(B);
+    free(C);
+    free(AA);
+    free(BB);
+    free(DD);
+    free(CC);
+  } // x
+  } // l
+  } // alpha
+  } // beta
+
+  if (ret != 0)
+    fprintf (stderr, "FATAL ERROR SBGEMV - Return code: %d\n", ret);
+  return ret;
+}

test/test_helpers.h

@@ -0,0 +1,66 @@
+/***************************************************************************
+Copyright (c) 2025 The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
+GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+#ifndef TEST_HELPERS_H
+#define TEST_HELPERS_H
+#include <stdbool.h>
+
+#include "../common.h"
+
+#ifdef IBFLOAT16
+static float float16to32(bfloat16 value)
+{
+  blasint one = 1;
+  float result;
+  sbf16tos_(&one, &value, &one, &result, &one);
+  return result;
+}
+#endif
+
+#ifdef OBFLOAT16
+static float truncate_float32_to_bfloat16(float value) {
+  blasint one = 1;
+  bfloat16 tmp;
+  float result;
+  sbstobf16_(&one, &value, &one, &tmp, &one);
+  sbf16tos_(&one, &tmp, &one, &result, &one);
+  return result;
+}
+#endif
+
+static void *malloc_safe(size_t size) {
+  if (size == 0)
+    return malloc(1);
+  else
+    return malloc(size);
+}
+
+static bool is_close(float a, float b, float rtol, float atol) {
+  return fabs(a - b) <= (atol + rtol*fabs(b));
+}
+
+#endif