diff --git a/.github/workflows/arm64_graviton.yml b/.github/workflows/arm64_graviton.yml index 6928312b5..4b4e15167 100644 --- a/.github/workflows/arm64_graviton.yml +++ b/.github/workflows/arm64_graviton.yml @@ -88,13 +88,14 @@ jobs: run: | case "${{ matrix.build }}" in "make") - make -j$(nproc) DYNAMIC_ARCH=1 USE_OPENMP=0 FC="ccache ${{ matrix.fortran }}" + make -j$(nproc) DYNAMIC_ARCH=1 BUILD_BFLOAT16=1 USE_OPENMP=0 FC="ccache ${{ matrix.fortran }}" ;; "cmake") mkdir build && cd build cmake -DDYNAMIC_ARCH=1 \ -DNOFORTRAN=0 \ -DBUILD_WITHOUT_LAPACK=0 \ + -DBUILD_BFLOAT16=1 \ -DCMAKE_VERBOSE_MAKEFILE=ON \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_Fortran_COMPILER=${{ matrix.fortran }} \ diff --git a/.github/workflows/windows_arm64.yml b/.github/workflows/windows_arm64.yml new file mode 100644 index 000000000..d1b5f94e0 --- /dev/null +++ b/.github/workflows/windows_arm64.yml @@ -0,0 +1,82 @@ +name: Windows ARM64 CI + +on: + push: + branches: + - develop + - release-** + pull_request: + branches: + - develop + - release-** + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read # to fetch code (actions/checkout) + +jobs: + build: + if: "github.repository == 'OpenMathLib/OpenBLAS'" + runs-on: windows-11-arm + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Install LLVM for Win-ARM64 + shell: pwsh + run: | + Invoke-WebRequest https://github.com/llvm/llvm-project/releases/download/llvmorg-20.1.6/LLVM-20.1.6-woa64.exe -UseBasicParsing -OutFile LLVM-woa64.exe + Start-Process -FilePath ".\LLVM-woa64.exe" -ArgumentList "/S" -Wait + echo "C:\Program Files\LLVM\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + + - name: Install CMake and Ninja for Win-ARM64 + shell: pwsh + run: | + Invoke-WebRequest https://github.com/Kitware/CMake/releases/download/v3.29.4/cmake-3.29.4-windows-arm64.msi -OutFile cmake-arm64.msi + Start-Process msiexec.exe -ArgumentList "/i cmake-arm64.msi /quiet /norestart" -Wait + echo "C:\Program Files\CMake\bin" >> $env:GITHUB_PATH + + Invoke-WebRequest https://github.com/ninja-build/ninja/releases/download/v1.13.1/ninja-winarm64.zip -OutFile ninja-winarm64.zip + Expand-Archive ninja-winarm64.zip -DestinationPath ninja + Copy-Item ninja\ninja.exe -Destination "C:\Windows\System32" + + - name: Configure OpenBLAS + shell: cmd + run: | + CALL "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsarm64.bat" + mkdir build + cd build + cmake .. -G Ninja ^ + -DCMAKE_BUILD_TYPE=Release ^ + -DTARGET=ARMV8 ^ + -DBINARY=64 ^ + -DCMAKE_C_COMPILER=clang-cl ^ + -DCMAKE_Fortran_COMPILER=flang-new ^ + -DBUILD_SHARED_LIBS=ON ^ + -DCMAKE_SYSTEM_PROCESSOR=arm64 ^ + -DCMAKE_SYSTEM_NAME=Windows ^ + -DCMAKE_INSTALL_PREFIX=C:/opt + + - name: Build OpenBLAS + shell: cmd + run: | + cd build + ninja -j16 + + - name: Install OpenBLAS + shell: cmd + run: | + cd build + cmake --install . + + - name: Run ctests + shell: pwsh + run: | + $env:PATH = "C:\opt\bin;$env:PATH" + cd build + ctest + + diff --git a/.gitignore b/.gitignore index 8294da4d4..dcbc73dc8 100644 --- a/.gitignore +++ b/.gitignore @@ -13,8 +13,8 @@ lapack-3.4.1.tgz lapack-3.4.2 lapack-3.4.2.tgz lapack-netlib/make.inc -lapack-netlib/lapacke/include/lapacke_mangling.h lapack-netlib/SRC/la_constants.mod +lapack-netlib/SRC/la_xisnan.mod lapack-netlib/TESTING/testing_results.txt lapack-netlib/INSTALL/test* lapack-netlib/TESTING/xeigtstc @@ -81,7 +81,10 @@ test/ZBLAT2.SUMM test/ZBLAT3.SUMM test/ZBLAT3_3M.SUMM test/SHBLAT3.SUMM +test/SBBLAT2.SUMM test/SBBLAT3.SUMM +test/BBLAT2.SUMM +test/BBLAT3.SUMM test/cblat1 test/cblat2 test/cblat3 @@ -96,6 +99,9 @@ test/sblat3 test/sblat3_3m test/test_shgemm test/test_sbgemm +test/test_sbgemv +test/test_bgemm +test/test_bgemv test/zblat1 test/zblat2 test/zblat3 diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index d8f57ef60..57993889a 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -251,6 +251,18 @@ In chronological order: * Ye Tao * [2025-02-03] Optimize SBGEMM kernel on NEOVERSEV1 * [2025-02-27] Add sbgemv_n_neon kernel + * [2025-05-17] Impl prototype of BGEMM inferface * Abhishek Kumar - * [2025-04-22] Optimise dot kernel for NEOVERSE V1 \ No newline at end of file + * [2025-04-22] Optimise dot kernel for NEOVERSE V1 + +* Sharif Inamdar + * [2025-06-05] Optimize gemv_n_sve_v1x3 kernel + +* Guoyuan Li + * [2025-04-11] Optimise gemv kernel for RISCV64_ZVL256B + * [2025-05-01] Optimise zgemv kernel for RISCV64_ZVL256B + * [2025-05-17] Optimise omatcopy/zomatcopy kernel for RISCV64_ZVL256B + * [2025-05-29] Optimise axpby kernel for RISCV64_ZVL256B + * [2025-06-05] Optimise hbmv kernel for RISCV64_ZVL256B + diff --git a/Makefile.system b/Makefile.system index bde3014cc..9e20c132c 100644 --- a/Makefile.system +++ b/Makefile.system @@ -276,14 +276,14 @@ SMALL_MATRIX_OPT = 1 endif ifeq ($(ARCH), arm64) GEMM_GEMV_FORWARD = 1 -GEMM_GEMV_FORWARD_BF16 = 1 +SBGEMM_GEMV_FORWARD = 1 endif ifeq ($(ARCH), riscv) GEMM_GEMV_FORWARD = 1 endif ifeq ($(ARCH), power) GEMM_GEMV_FORWARD = 1 -GEMM_GEMV_FORWARD_BF16 = 1 +SBGEMM_GEMV_FORWARD = 1 endif ifeq ($(SMALL_MATRIX_OPT), 1) @@ -293,8 +293,8 @@ ifneq ($(ONLY_CBLAS), 1) ifeq ($(GEMM_GEMV_FORWARD), 1) CCOMMON_OPT += -DGEMM_GEMV_FORWARD endif -ifeq ($(GEMM_GEMV_FORWARD_BF16), 1) -CCOMMON_OPT += -DGEMM_GEMV_FORWARD_BF16 +ifeq ($(SBGEMM_GEMV_FORWARD), 1) +CCOMMON_OPT += -DSBGEMM_GEMV_FORWARD endif endif @@ -1905,6 +1905,8 @@ export BUILD_HFLOAT16 export NO_LSX export NO_LASX +export BGEMM_UNROLL_M +export BGEMM_UNROLL_N export SBGEMM_UNROLL_M export SBGEMM_UNROLL_N export SHGEMM_UNROLL_M diff --git a/Makefile.tail b/Makefile.tail index ed2c0e507..4a12f531e 100644 --- a/Makefile.tail +++ b/Makefile.tail @@ -1,3 +1,32 @@ +############################################################################### +# Copyright (c) 2025, The OpenBLAS Project +# All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# 3. Neither the name of the OpenBLAS project nor the names of +# its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +############################################################################### + +BBLASOBJS_P = $(BBLASOBJS:.$(SUFFIX)=.$(PSUFFIX)) SBBLASOBJS_P = $(SBBLASOBJS:.$(SUFFIX)=.$(PSUFFIX)) SHBLASPBJS_P = $(SHBLASOBJS:.$(SUFFIX)=.$(PSUFFIX)) SBLASOBJS_P = $(SBLASOBJS:.$(SUFFIX)=.$(PSUFFIX)) @@ -12,8 +41,8 @@ COMMONOBJS_P = $(COMMONOBJS:.$(SUFFIX)=.$(PSUFFIX)) HPLOBJS_P = $(HPLOBJS:.$(SUFFIX)=.$(PSUFFIX)) -BLASOBJS = $(SHBLASOBJS) $(SBEXTOBJS) $(SBBLASOBJS) $(SBLASOBJS) $(DBLASOBJS) $(CBLASOBJS) $(ZBLASOBJS) $(CBAUXOBJS) -BLASOBJS_P = $(SHBLASPBJS_P) $(SBEXTOBJS_P) $(SBBLASOBJS_P) $(SBLASOBJS_P) $(DBLASOBJS_P) $(CBLASOBJS_P) $(ZBLASOBJS_P) $(CBAUXOBJS_P) +BLASOBJS = $(SHBLASOBJS) $(BBLASOBJS) $(SBEXTOBJS) $(SBBLASOBJS) $(SBLASOBJS) $(DBLASOBJS) $(CBLASOBJS) $(ZBLASOBJS) $(CBAUXOBJS) +BLASOBJS_P = $(SHBLASPBJS_P) $(BBLASOBJS_P) $(SBEXTOBJS_P) $(SBBLASOBJS_P) $(SBLASOBJS_P) $(DBLASOBJS_P) $(CBLASOBJS_P) $(ZBLASOBJS_P) $(CBAUXOBJS_P) ifdef EXPRECISION BLASOBJS += $(QBLASOBJS) $(XBLASOBJS) @@ -26,6 +55,7 @@ BLASOBJS_P += $(QBLASOBJS_P) $(XBLASOBJS_P) endif $(SHBLASOBJS) $(SHBLASOBJS_P) : override CFLAGS += -DHFLOAT16 -UDOUBLE -UCOMPLEX +$(BBLASOBJS) $(BBLASOBJS_P) : override CFLAGS += -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX $(SBBLASOBJS) $(SBBLASOBJS_P) : override CFLAGS += -DBFLOAT16 -UDOUBLE -UCOMPLEX $(SBLASOBJS) $(SBLASOBJS_P) : override CFLAGS += -UDOUBLE -UCOMPLEX $(DBLASOBJS) $(DBLASOBJS_P) : override CFLAGS += -DDOUBLE -UCOMPLEX @@ -36,6 +66,7 @@ $(XBLASOBJS) $(XBLASOBJS_P) : override CFLAGS += -DXDOUBLE -DCOMPLEX $(SBEXTOBJS) $(SBEXTOBJS_P) : override CFLAGS += -DBFLOAT16 -UDOUBLE -UCOMPLEX $(SHBLASOBJS_P) : override CFLAGS += -DPROFILE $(COMMON_PROF) +$(BBLASOBJS_P) : override CFLAGS += -DPROFILE $(COMMON_PROF) $(SBBLASOBJS_P) : override CFLAGS += -DPROFILE $(COMMON_PROF) $(SBLASOBJS_P) : override CFLAGS += -DPROFILE $(COMMON_PROF) $(DBLASOBJS_P) : override CFLAGS += -DPROFILE $(COMMON_PROF) diff --git a/README.md b/README.md index cc9325d39..6a4fc6a7f 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ For a general introduction to the BLAS routines, please refer to the extensive d We provide official binary packages for the following platform: * Windows x86/x86_64 + * Windows arm64 (woa) You can download them from [file hosting on sourceforge.net](https://sourceforge.net/projects/openblas/files/) or from the [Releases section of the GitHub project page](https://github.com/OpenMathLib/OpenBLAS/releases). diff --git a/TargetList.txt b/TargetList.txt index 232e12ffa..0c94869e7 100644 --- a/TargetList.txt +++ b/TargetList.txt @@ -52,6 +52,7 @@ POWER7 POWER8 POWER9 POWER10 +POWER11 PPCG4 PPC970 PPC970MP diff --git a/benchmark/Makefile b/benchmark/Makefile index 9316921e1..cdf87c0ab 100644 --- a/benchmark/Makefile +++ b/benchmark/Makefile @@ -1,3 +1,31 @@ +############################################################################### +# Copyright (c) 2025, The OpenBLAS Project +# All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# 3. Neither the name of the OpenBLAS project nor the names of +# its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +############################################################################### + TOPDIR = .. include $(TOPDIR)/Makefile.system @@ -56,7 +84,7 @@ GOTO_LAPACK_TARGETS= endif ifeq ($(BUILD_BFLOAT16),1) -GOTO_BFLOAT_TARGETS=sbgemm.goto +GOTO_BFLOAT_TARGETS=bgemm.goto sbgemm.goto else GOTO_BFLOAT_TARGETS= endif @@ -635,6 +663,8 @@ zcholesky.essl : zcholesky.$(SUFFIX) ##################################### Sgemm #################################################### ifeq ($(BUILD_BFLOAT16),1) +bgemm.goto : bgemm.$(SUFFIX) ../$(LIBNAME) + $(CC) $(CFLAGS) -o $(@F) $^ $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB) -lm sbgemm.goto : sbgemm.$(SUFFIX) ../$(LIBNAME) $(CC) $(CFLAGS) -o $(@F) $^ $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB) -lm endif @@ -2970,6 +3000,8 @@ zcholesky.$(SUFFIX) : cholesky.c $(CC) $(CFLAGS) -c -DCOMPLEX -DDOUBLE -o $(@F) $^ ifeq ($(BUILD_BFLOAT16),1) +bgemm.$(SUFFIX) : gemm.c + $(CC) $(CFLAGS) -c -DBFLOAT16 -DBGEMM -UCOMPLEX -UDOUBLE -o $(@F) $^ sbgemm.$(SUFFIX) : gemm.c $(CC) $(CFLAGS) -c -DBFLOAT16 -UCOMPLEX -UDOUBLE -o $(@F) $^ endif diff --git a/benchmark/gemm.c b/benchmark/gemm.c index a138bfe1e..704e33225 100644 --- a/benchmark/gemm.c +++ b/benchmark/gemm.c @@ -33,6 +33,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifdef DOUBLE #define GEMM BLASFUNC(dgemm) +#elif defined(BFLOAT16) && defined(BGEMM) +#define GEMM BLASFUNC(bgemm) #elif defined(BFLOAT16) #define GEMM BLASFUNC(sbgemm) #undef IFLOAT @@ -60,8 +62,18 @@ int main(int argc, char *argv[]){ IFLOAT *a, *b; FLOAT *c; +#ifdef BGEMM + blasint one=1; + blasint two=2; + float alpha_in[] = {1.0, 0.0}; + float beta_in[] = {0.0, 0.0}; + FLOAT alpha[2], beta[2]; + sbstobf16_(&two, alpha_in, &one, alpha, &one); + sbstobf16_(&two, beta_in, &one, beta, &one); +#else FLOAT alpha[] = {1.0, 0.0}; FLOAT beta [] = {0.0, 0.0}; +#endif char transa = 'N'; char transb = 'N'; blasint m, n, k, i, j, lda, ldb, ldc; diff --git a/cblas.h b/cblas.h index 7953c820e..13b1585a0 100644 --- a/cblas.h +++ b/cblas.h @@ -1,3 +1,31 @@ +/*************************************************************************** + * Copyright (c) 2025, The OpenBLAS Project + * All rights reserved. + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of the OpenBLAS project nor the names of + * its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * *****************************************************************************/ + #ifndef CBLAS_H #define CBLAS_H @@ -445,10 +473,13 @@ void cblas_sbdtobf16(OPENBLAS_CONST blasint n, OPENBLAS_CONST double *in, OPEN void cblas_sbf16tos(OPENBLAS_CONST blasint n, OPENBLAS_CONST bfloat16 *in, OPENBLAS_CONST blasint incin, float *out, OPENBLAS_CONST blasint incout); /* convert BFLOAT16 array to double array */ void cblas_dbf16tod(OPENBLAS_CONST blasint n, OPENBLAS_CONST bfloat16 *in, OPENBLAS_CONST blasint incin, double *out, OPENBLAS_CONST blasint incout); +void cblas_bgemv(OPENBLAS_CONST enum CBLAS_ORDER order, OPENBLAS_CONST enum CBLAS_TRANSPOSE trans, OPENBLAS_CONST blasint m, OPENBLAS_CONST blasint n, OPENBLAS_CONST bfloat16 alpha, OPENBLAS_CONST bfloat16 *a, OPENBLAS_CONST blasint lda, OPENBLAS_CONST bfloat16 *x, OPENBLAS_CONST blasint incx, OPENBLAS_CONST bfloat16 beta, bfloat16 *y, OPENBLAS_CONST blasint incy); /* dot production of BFLOAT16 input arrays, and output as float */ float cblas_sbdot(OPENBLAS_CONST blasint n, OPENBLAS_CONST bfloat16 *x, OPENBLAS_CONST blasint incx, OPENBLAS_CONST bfloat16 *y, OPENBLAS_CONST blasint incy); void cblas_sbgemv(OPENBLAS_CONST enum CBLAS_ORDER order, OPENBLAS_CONST enum CBLAS_TRANSPOSE trans, OPENBLAS_CONST blasint m, OPENBLAS_CONST blasint n, OPENBLAS_CONST float alpha, OPENBLAS_CONST bfloat16 *a, OPENBLAS_CONST blasint lda, OPENBLAS_CONST bfloat16 *x, OPENBLAS_CONST blasint incx, OPENBLAS_CONST float beta, float *y, OPENBLAS_CONST blasint incy); +void cblas_bgemm(OPENBLAS_CONST enum CBLAS_ORDER Order, OPENBLAS_CONST enum CBLAS_TRANSPOSE TransA, OPENBLAS_CONST enum CBLAS_TRANSPOSE TransB, OPENBLAS_CONST blasint M, OPENBLAS_CONST blasint N, OPENBLAS_CONST blasint K, + OPENBLAS_CONST bfloat16 alpha, OPENBLAS_CONST bfloat16 *A, OPENBLAS_CONST blasint lda, OPENBLAS_CONST bfloat16 *B, OPENBLAS_CONST blasint ldb, OPENBLAS_CONST bfloat16 beta, bfloat16 *C, OPENBLAS_CONST blasint ldc); void cblas_sbgemm(OPENBLAS_CONST enum CBLAS_ORDER Order, OPENBLAS_CONST enum CBLAS_TRANSPOSE TransA, OPENBLAS_CONST enum CBLAS_TRANSPOSE TransB, OPENBLAS_CONST blasint M, OPENBLAS_CONST blasint N, OPENBLAS_CONST blasint K, OPENBLAS_CONST float alpha, OPENBLAS_CONST bfloat16 *A, OPENBLAS_CONST blasint lda, OPENBLAS_CONST bfloat16 *B, OPENBLAS_CONST blasint ldb, OPENBLAS_CONST float beta, float *C, OPENBLAS_CONST blasint ldc); void cblas_sbgemm_batch(OPENBLAS_CONST enum CBLAS_ORDER Order, OPENBLAS_CONST enum CBLAS_TRANSPOSE * TransA_array, OPENBLAS_CONST enum CBLAS_TRANSPOSE * TransB_array, OPENBLAS_CONST blasint * M_array, OPENBLAS_CONST blasint * N_array, OPENBLAS_CONST blasint * K_array, diff --git a/cmake/cc.cmake b/cmake/cc.cmake index 91037e851..3679278b0 100644 --- a/cmake/cc.cmake +++ b/cmake/cc.cmake @@ -211,14 +211,14 @@ endif () if (${CORE} STREQUAL NEOVERSEV1) if (NOT DYNAMIC_ARCH) if (${CMAKE_C_COMPILER_ID} STREQUAL "PGI" AND NOT NO_SVE) - set (CCOMMON_OPT "${CCOMMON_OPT} -Msve_intrinsics -march=armv8.4-a+sve -mtune=neoverse-v1") + set (CCOMMON_OPT "${CCOMMON_OPT} -Msve_intrinsics -march=armv8.4-a+sve+bf16 -mtune=neoverse-v1") elseif (${CMAKE_C_COMPILER_ID} STREQUAL "NVC" AND NOT NO_SVE) set (CCOMMON_OPT "${CCOMMON_OPT} -tp=neoverse-v1") else () if (${GCC_VERSION} VERSION_GREATER 10.4 OR ${GCC_VERSION} VERSION_EQUAL 10.4) - set (CCOMMON_OPT "${CCOMMON_OPT} -march=armv8.4-a+sve -mtune=neoverse-v1") + set (CCOMMON_OPT "${CCOMMON_OPT} -march=armv8.4-a+sve+bf16 -mtune=neoverse-v1") else () - set (CCOMMON_OPT "${CCOMMON_OPT} -march=armv8.2-a+sve") + set (CCOMMON_OPT "${CCOMMON_OPT} -march=armv8.2-a+sve+bf16") endif() endif() endif () diff --git a/cmake/kernel.cmake b/cmake/kernel.cmake index 2cea6d9e6..6d752ac51 100644 --- a/cmake/kernel.cmake +++ b/cmake/kernel.cmake @@ -1,3 +1,31 @@ +############################################################################### +# Copyright (c) 2025, The OpenBLAS Project +# All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# 3. Neither the name of the OpenBLAS project nor the names of +# its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +############################################################################### + # helper functions for the kernel CMakeLists.txt function(SetFallback KERNEL SOURCE_PATH) @@ -82,6 +110,7 @@ macro(SetDefaultL1) SetFallback(SROTMKERNEL rotm.S) SetFallback(DROTMKERNEL rotm.S) SetFallback(QROTMKERNEL rotm.S) + SetFallback(BSCALKERNEL ../generic/scal.c) SetFallback(SSCALKERNEL scal.S) SetFallback(DSCALKERNEL scal.S) SetFallback(CSCALKERNEL zscal.S) @@ -141,6 +170,8 @@ if (BUILD_BFLOAT16) SetFallback(SHSWAPKERNEL ../arm/swap.c) SetFallback(TOBF16KERNEL ../x86_64/tobf16.c) SetFallback(BF16TOKERNEL ../x86_64/bf16to.c) + SetFallback(BGEMVNKERNEL ../generic/gemv_n.c) + SetFallback(BGEMVTKERNEL ../generic/gemv_t.c) SetFallback(SBGEMVNKERNEL ../x86_64/sbgemv_n.c) SetFallback(SBGEMVTKERNEL ../x86_64/sbgemv_t.c) endif () @@ -193,6 +224,8 @@ macro(SetDefaultL2) SetFallback(XHEMV_V_KERNEL ../generic/zhemv_k.c) SetFallback(XHEMV_M_KERNEL ../generic/zhemv_k.c) if (BUILD_BFLOAT16) + SetFallback(BGEMVNKERNEL ../generic/gemv_n.c) + SetFallback(BGEMVTKERNEL ../generic/gemv_t.c) SetFallback(SBGEMVNKERNEL ../x86_64/sbgemv_n.c) SetFallback(SBGEMVTKERNEL ../x86_64/sbgemv_t.c) SetFallback(SHGERKERNEL ../generic/ger.c) @@ -206,6 +239,16 @@ macro(SetDefaultL3) SetFallback(ZGEADD_KERNEL ../generic/zgeadd.c) if (BUILD_BFLOAT16) SetFallback(SHGEADD_KERNEL ../generic/geadd.c) + SetFallback(BGEMMKERNEL ../generic/gemmkernel_2x2.c) + SetFallback(BGEMM_BETA ../generic/gemm_beta.c) + SetFallback(BGEMMINCOPY ../generic/gemm_ncopy_2.c) + SetFallback(BGEMMITCOPY ../generic/gemm_tcopy_2.c) + SetFallback(BGEMMONCOPY ../generic/gemm_ncopy_2.c) + SetFallback(BGEMMOTCOPY ../generic/gemm_tcopy_2.c) + SetFallback(BGEMMINCOPYOBJ bgemm_incopy.o) + SetFallback(BGEMMITCOPYOBJ bgemm_itcopy.o) + SetFallback(BGEMMONCOPYOBJ bgemm_oncopy.o) + SetFallback(BGEMMOTCOPYOBJ bgemm_otcopy.o) SetFallback(SBGEMMKERNEL ../generic/gemmkernel_2x2.c) SetFallback(SBGEMM_BETA ../generic/gemm_beta.c) SetFallback(SBGEMMINCOPY ../generic/gemm_ncopy_2.c) diff --git a/cmake/prebuild.cmake b/cmake/prebuild.cmake index 8afe8b387..fe592ed73 100644 --- a/cmake/prebuild.cmake +++ b/cmake/prebuild.cmake @@ -1,3 +1,31 @@ +############################################################################### +# Copyright (c) 2025, The OpenBLAS Project +# All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# 3. Neither the name of the OpenBLAS project nor the names of +# its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +############################################################################### + ## ## Author: Hank Anderson ## Description: Ported from OpenBLAS/Makefile.prebuild @@ -131,6 +159,8 @@ if (DEFINED CORE AND CMAKE_CROSSCOMPILING AND NOT (${HOST_OS} STREQUAL "WINDOWSS set(HAVE_SSE2 1) set(HAVE_SSE3 1) set(HAVE_SSSE3 1) + set(BGEMM_UNROLL_M 8) + set(BGEMM_UNROLL_N 4) set(SBGEMM_UNROLL_M 8) set(SBGEMM_UNROLL_N 4) set(SGEMM_UNROLL_M 8) diff --git a/cmake/system.cmake b/cmake/system.cmake index bac756901..81f1a67ad 100644 --- a/cmake/system.cmake +++ b/cmake/system.cmake @@ -291,10 +291,10 @@ if (DEFINED TARGET) if (${TARGET} STREQUAL NEOVERSEV1) if (${CMAKE_C_COMPILER_ID} STREQUAL "PGI" AND NOT NO_SVE) - set (KERNEL_DEFINITIONS "${KERNEL_DEFINITIONS} -Msve_intrinsics -march=armv8.4-a+sve -mtune=neoverse-v1") + set (KERNEL_DEFINITIONS "${KERNEL_DEFINITIONS} -Msve_intrinsics -march=armv8.4-a+sve+bf16 -mtune=neoverse-v1") else () if (CMAKE_C_COMPILER_VERSION VERSION_GREATER 10.4 OR CMAKE_C_COMPILER_VERSION VERSION_EQUAL 10.4) - set (KERNEL_DEFINITIONS "${KERNEL_DEFINITIONS} -march=armv8.4-a+sve -mtune=neoverse-v1") + set (KERNEL_DEFINITIONS "${KERNEL_DEFINITIONS} -march=armv8.4-a+sve+bf16 -mtune=neoverse-v1") else () message(FATAL_ERROR "Compiler ${CMAKE_C_COMPILER} ${CMAKE_C_COMPILER_VERSION} does not support Neoverse V1.") endif() @@ -425,8 +425,8 @@ endif () if (GEMM_GEMV_FORWARD AND NOT ONLY_CBLAS) set(CCOMMON_OPT "${CCOMMON_OPT} -DGEMM_GEMV_FORWARD") endif () -if (GEMM_GEMV_FORWARD_BF16 AND NOT ONLY_CBLAS) - set(CCOMMON_OPT "${CCOMMON_OPT} -DGEMM_GEMV_FORWARD_BF16") +if (SBGEMM_GEMV_FORWARD AND NOT ONLY_CBLAS) + set(CCOMMON_OPT "${CCOMMON_OPT} -DSBGEMM_GEMV_FORWARD") endif () if (SMALL_MATRIX_OPT) set(CCOMMON_OPT "${CCOMMON_OPT} -DSMALL_MATRIX_OPT") diff --git a/cmake/utils.cmake b/cmake/utils.cmake index a93f21686..52e5b5ee3 100644 --- a/cmake/utils.cmake +++ b/cmake/utils.cmake @@ -1,3 +1,31 @@ +############################################################################### +# Copyright (c) 2025, The OpenBLAS Project +# All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# 3. Neither the name of the OpenBLAS project nor the names of +# its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +############################################################################### + # Functions to help with the OpenBLAS build # Reads string from getarch into CMake vars. Format of getarch vars is VARNAME=VALUE @@ -347,7 +375,7 @@ function(GenerateNamedObjects sources_in) if (NOT no_float_type) string(SUBSTRING ${float_type} 0 1 float_char) string(TOLOWER ${float_char} float_char) - if (${float_type} STREQUAL "BFLOAT16") + if (${float_type} STREQUAL "BFLOAT16" AND NOT "${defines_in}" MATCHES "BGEM") set (float_char "sb") endif () endif () diff --git a/common.h b/common.h index 23a08aaa9..4984d727c 100644 --- a/common.h +++ b/common.h @@ -1,5 +1,6 @@ /*********************************************************************/ /* Copyright 2009, 2010 The University of Texas at Austin. */ +/* Copyright 2025 The OpenBLAS Project. */ /* All rights reserved. */ /* */ /* Redistribution and use in source and binary forms, with or */ @@ -317,7 +318,11 @@ typedef int blasint; #elif defined(BFLOAT16) #define IFLOAT bfloat16 #define XFLOAT IFLOAT -#define FLOAT float +#ifdef BGEMM +#define FLOAT bfloat16 +#else +#define FLOAT float +#endif #define SIZE 2 #define BASE_SHIFT 1 #define ZBASE_SHIFT 2 diff --git a/common_b.h b/common_b.h new file mode 100644 index 000000000..1921c3a69 --- /dev/null +++ b/common_b.h @@ -0,0 +1,100 @@ +/*************************************************************************** + * Copyright (c) 2025, The OpenBLAS Project + * All rights reserved. + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of the OpenBLAS project nor the names of + * its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * *****************************************************************************/ + +#ifndef COMMON_B_H +#define COMMON_B_H + +#ifndef DYNAMIC_ARCH +#define BGEMV_N_K bgemv_n +#define BGEMV_T_K bgemv_t + +#define BSCAL_K bscal_k + +#define BGEMM_ONCOPY bgemm_oncopy +#define BGEMM_OTCOPY bgemm_otcopy + +#if BGEMM_DEFAULT_UNROLL_M == BGEMM_DEFAULT_UNROLL_N +#define BGEMM_INCOPY bgemm_oncopy +#define BGEMM_ITCOPY bgemm_otcopy +#else +#define BGEMM_INCOPY bgemm_incopy +#define BGEMM_ITCOPY bgemm_itcopy +#endif + +#define BGEMM_BETA bgemm_beta +#define BGEMM_KERNEL bgemm_kernel + +#else +#define BGEMV_N_K gotoblas->bgemv_n +#define BGEMV_T_K gotoblas->bgemv_t + +#define BSCAL_K gotoblas->bscal_k + +#define BGEMM_ONCOPY gotoblas->bgemm_oncopy +#define BGEMM_OTCOPY gotoblas->bgemm_otcopy +#define BGEMM_INCOPY gotoblas->bgemm_incopy +#define BGEMM_ITCOPY gotoblas->bgemm_itcopy +#define BGEMM_BETA gotoblas->bgemm_beta +#define BGEMM_KERNEL gotoblas->bgemm_kernel + +#endif + +#define BGEMM_NN bgemm_nn +#define BGEMM_CN bgemm_tn +#define BGEMM_TN bgemm_tn +#define BGEMM_NC bgemm_nt +#define BGEMM_NT bgemm_nt +#define BGEMM_CC bgemm_tt +#define BGEMM_CT bgemm_tt +#define BGEMM_TC bgemm_tt +#define BGEMM_TT bgemm_tt +#define BGEMM_NR bgemm_nn +#define BGEMM_TR bgemm_tn +#define BGEMM_CR bgemm_tn +#define BGEMM_RN bgemm_nn +#define BGEMM_RT bgemm_nt +#define BGEMM_RC bgemm_nt +#define BGEMM_RR bgemm_nn + +#define BGEMM_THREAD_NN bgemm_thread_nn +#define BGEMM_THREAD_CN bgemm_thread_tn +#define BGEMM_THREAD_TN bgemm_thread_tn +#define BGEMM_THREAD_NC bgemm_thread_nt +#define BGEMM_THREAD_NT bgemm_thread_nt +#define BGEMM_THREAD_CC bgemm_thread_tt +#define BGEMM_THREAD_CT bgemm_thread_tt +#define BGEMM_THREAD_TC bgemm_thread_tt +#define BGEMM_THREAD_TT bgemm_thread_tt +#define BGEMM_THREAD_NR bgemm_thread_nn +#define BGEMM_THREAD_TR bgemm_thread_tn +#define BGEMM_THREAD_CR bgemm_thread_tn +#define BGEMM_THREAD_RN bgemm_thread_nn +#define BGEMM_THREAD_RT bgemm_thread_nt +#define BGEMM_THREAD_RC bgemm_thread_nt +#define BGEMM_THREAD_RR bgemm_thread_nn +#endif diff --git a/common_interface.h b/common_interface.h index 23d86871f..945b6c8a1 100644 --- a/common_interface.h +++ b/common_interface.h @@ -1,5 +1,6 @@ /*********************************************************************/ /* Copyright 2009, 2010 The University of Texas at Austin. */ +/* Copyright 2025 The OpenBLAS Project. */ /* All rights reserved. */ /* */ /* Redistribution and use in source and binary forms, with or */ @@ -59,6 +60,7 @@ double BLASFUNC(dsdot) (blasint *, float *, blasint *, float *, blasint *); double BLASFUNC(ddot) (blasint *, double *, blasint *, double *, blasint *); xdouble BLASFUNC(qdot) (blasint *, xdouble *, blasint *, xdouble *, blasint *); +void BLASFUNC(bscal) (blasint *, bfloat16 *, bfloat16 *, blasint *); float BLASFUNC(sbdot) (blasint *, bfloat16 *, blasint *, bfloat16 *, blasint *); void BLASFUNC(sbstobf16) (blasint *, float *, blasint *, bfloat16 *, blasint *); void BLASFUNC(sbdtobf16) (blasint *, double *, blasint *, bfloat16 *, blasint *); @@ -255,6 +257,8 @@ void BLASFUNC(xgeru)(blasint *, blasint *, xdouble *, xdouble *, blasint *, void BLASFUNC(xgerc)(blasint *, blasint *, xdouble *, xdouble *, blasint *, xdouble *, blasint *, xdouble *, blasint *); +void BLASFUNC(bgemv)(char *, blasint *, blasint *, bfloat16 *, bfloat16 *, blasint *, + bfloat16 *, blasint *, bfloat16 *, bfloat16 *, blasint *); void BLASFUNC(sbgemv)(char *, blasint *, blasint *, float *, bfloat16 *, blasint *, bfloat16 *, blasint *, float *, float *, blasint *); void BLASFUNC(sgemv)(char *, blasint *, blasint *, float *, float *, blasint *, @@ -483,6 +487,8 @@ void BLASFUNC(xhbmv)(char *, blasint *, blasint *, xdouble *, xdouble *, blasint void BLASFUNC(shgemm)(char *, char *, blasint *, blasint *, blasint *, float *, hfloat16 *, blasint *, hfloat16 *, blasint *, float *, float *, blasint *); +void BLASFUNC(bgemm)(char *, char *, blasint *, blasint *, blasint *, bfloat16 *, + bfloat16 *, blasint *, bfloat16 *, blasint *, bfloat16 *, bfloat16 *, blasint *); void BLASFUNC(sbgemm)(char *, char *, blasint *, blasint *, blasint *, float *, bfloat16 *, blasint *, bfloat16 *, blasint *, float *, float *, blasint *); void BLASFUNC(sgemm)(char *, char *, blasint *, blasint *, blasint *, float *, diff --git a/common_level1.h b/common_level1.h index 85b39f7a7..7ab45a472 100644 --- a/common_level1.h +++ b/common_level1.h @@ -1,4 +1,5 @@ /*********************************************************************/ +/* Copyright 2025 The OpenBLAS Project. */ /* Copyright 2009, 2010 The University of Texas at Austin. */ /* All rights reserved. */ /* */ @@ -169,6 +170,9 @@ BLASLONG icmin_k(BLASLONG, float *, BLASLONG); BLASLONG izmin_k(BLASLONG, double *, BLASLONG); BLASLONG ixmin_k(BLASLONG, xdouble *, BLASLONG); + +int bscal_k(BLASLONG, BLASLONG, BLASLONG, bfloat16, + bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG); int sscal_k(BLASLONG, BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG); int dscal_k(BLASLONG, BLASLONG, BLASLONG, double, diff --git a/common_level2.h b/common_level2.h index 9a5ebb4d9..eea5e43f3 100644 --- a/common_level2.h +++ b/common_level2.h @@ -1,4 +1,5 @@ /*********************************************************************/ +/* Copyright 2025 The OpenBLAS Project */ /* Copyright 2009, 2010 The University of Texas at Austin. */ /* All rights reserved. */ /* */ @@ -44,6 +45,11 @@ extern "C" { #endif + +int bgemv_n(BLASLONG, BLASLONG, bfloat16, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, bfloat16, bfloat16 *, BLASLONG); +int bgemv_t(BLASLONG, BLASLONG, bfloat16, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, bfloat16, bfloat16 *, BLASLONG); +int bgemv_thread_n(BLASLONG, BLASLONG, bfloat16, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, bfloat16, bfloat16 *, BLASLONG, int); +int bgemv_thread_t(BLASLONG, BLASLONG, bfloat16, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, bfloat16, bfloat16 *, BLASLONG, int); int sbgemv_n(BLASLONG, BLASLONG, float, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, float, float *, BLASLONG); int sbgemv_t(BLASLONG, BLASLONG, float, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, float, float *, BLASLONG); int sbgemv_thread_n(BLASLONG, BLASLONG, float, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, float, float *, BLASLONG, int); diff --git a/common_level3.h b/common_level3.h index 1838b4bf6..fc4909e69 100644 --- a/common_level3.h +++ b/common_level3.h @@ -52,10 +52,19 @@ void sgemm_direct(BLASLONG M, BLASLONG N, BLASLONG K, float * B, BLASLONG strideB, float * R, BLASLONG strideR); +void sgemm_direct_alpha_beta(BLASLONG M, BLASLONG N, BLASLONG K, + float alpha, + float * A, BLASLONG strideA, + float * B, BLASLONG strideB, + float beta, + float * R, BLASLONG strideR); + int sgemm_direct_performant(BLASLONG M, BLASLONG N, BLASLONG K); int shgemm_beta(BLASLONG, BLASLONG, BLASLONG, float, hfloat16 *, BLASLONG, hfloat16 *, BLASLONG, float *, BLASLONG); +int bgemm_beta(BLASLONG, BLASLONG, BLASLONG, bfloat16, + bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG); int sbgemm_beta(BLASLONG, BLASLONG, BLASLONG, float, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, float *, BLASLONG); int sgemm_beta(BLASLONG, BLASLONG, BLASLONG, float, @@ -83,6 +92,10 @@ int shgemm_incopy(BLASLONG m, BLASLONG n, hfloat16 *a, BLASLONG lda, hfloat16 *b int shgemm_itcopy(BLASLONG m, BLASLONG n, hfloat16 *a, BLASLONG lda, hfloat16 *b); int shgemm_oncopy(BLASLONG m, BLASLONG n, hfloat16 *a, BLASLONG lda, hfloat16 *b); int shgemm_otcopy(BLASLONG m, BLASLONG n, hfloat16 *a, BLASLONG lda, hfloat16 *b); +int bgemm_incopy(BLASLONG m, BLASLONG n, bfloat16 *a, BLASLONG lda, bfloat16 *b); +int bgemm_itcopy(BLASLONG m, BLASLONG n, bfloat16 *a, BLASLONG lda, bfloat16 *b); +int bgemm_oncopy(BLASLONG m, BLASLONG n, bfloat16 *a, BLASLONG lda, bfloat16 *b); +int bgemm_otcopy(BLASLONG m, BLASLONG n, bfloat16 *a, BLASLONG lda, bfloat16 *b); int sbgemm_incopy(BLASLONG m, BLASLONG n, bfloat16 *a, BLASLONG lda, bfloat16 *b); int sbgemm_itcopy(BLASLONG m, BLASLONG n, bfloat16 *a, BLASLONG lda, bfloat16 *b); int sbgemm_oncopy(BLASLONG m, BLASLONG n, bfloat16 *a, BLASLONG lda, bfloat16 *b); @@ -511,6 +524,7 @@ int xher2k_kernel_LN(BLASLONG m, BLASLONG n, BLASLONG k, xdouble alpha_r, xdoubl int xher2k_kernel_LC(BLASLONG m, BLASLONG n, BLASLONG k, xdouble alpha_r, xdouble alpha_i, xdouble *a, xdouble *b, xdouble *c, BLASLONG ldc, BLASLONG offset, int flag); int shgemm_kernel(BLASLONG, BLASLONG, BLASLONG, float, hfloat16 *, hfloat16 *, float *, BLASLONG); +int bgemm_kernel(BLASLONG, BLASLONG, BLASLONG, bfloat16, bfloat16 *, bfloat16 *, bfloat16 *, BLASLONG); int sbgemm_kernel(BLASLONG, BLASLONG, BLASLONG, float, bfloat16 *, bfloat16 *, float *, BLASLONG); int sgemm_kernel(BLASLONG, BLASLONG, BLASLONG, float, float *, float *, float *, BLASLONG); int dgemm_kernel(BLASLONG, BLASLONG, BLASLONG, double, double *, double *, double *, BLASLONG); @@ -668,6 +682,11 @@ int shgemm_nt(blas_arg_t *, BLASLONG *, BLASLONG *, hfloat16 *, hfloat16 *, BLAS int shgemm_tn(blas_arg_t *, BLASLONG *, BLASLONG *, hfloat16 *, hfloat16 *, BLASLONG); int shgemm_tt(blas_arg_t *, BLASLONG *, BLASLONG *, hfloat16 *, hfloat16 *, BLASLONG); +int bgemm_nn(blas_arg_t *, BLASLONG *, BLASLONG *, bfloat16 *, bfloat16 *, BLASLONG); +int bgemm_nt(blas_arg_t *, BLASLONG *, BLASLONG *, bfloat16 *, bfloat16 *, BLASLONG); +int bgemm_tn(blas_arg_t *, BLASLONG *, BLASLONG *, bfloat16 *, bfloat16 *, BLASLONG); +int bgemm_tt(blas_arg_t *, BLASLONG *, BLASLONG *, bfloat16 *, bfloat16 *, BLASLONG); + int sbgemm_nn(blas_arg_t *, BLASLONG *, BLASLONG *, bfloat16 *, bfloat16 *, BLASLONG); int sbgemm_nt(blas_arg_t *, BLASLONG *, BLASLONG *, bfloat16 *, bfloat16 *, BLASLONG); int sbgemm_tn(blas_arg_t *, BLASLONG *, BLASLONG *, bfloat16 *, bfloat16 *, BLASLONG); @@ -770,6 +789,11 @@ int shgemm_thread_nt(blas_arg_t *, BLASLONG *, BLASLONG *, hfloat16 *, hfloat16 int shgemm_thread_tn(blas_arg_t *, BLASLONG *, BLASLONG *, hfloat16 *, hfloat16 *, BLASLONG); int shgemm_thread_tt(blas_arg_t *, BLASLONG *, BLASLONG *, hfloat16 *, hfloat16 *, BLASLONG); +int bgemm_thread_nn(blas_arg_t *, BLASLONG *, BLASLONG *, bfloat16 *, bfloat16 *, BLASLONG); +int bgemm_thread_nt(blas_arg_t *, BLASLONG *, BLASLONG *, bfloat16 *, bfloat16 *, BLASLONG); +int bgemm_thread_tn(blas_arg_t *, BLASLONG *, BLASLONG *, bfloat16 *, bfloat16 *, BLASLONG); +int bgemm_thread_tt(blas_arg_t *, BLASLONG *, BLASLONG *, bfloat16 *, bfloat16 *, BLASLONG); + int sbgemm_thread_nn(blas_arg_t *, BLASLONG *, BLASLONG *, bfloat16 *, bfloat16 *, BLASLONG); int sbgemm_thread_nt(blas_arg_t *, BLASLONG *, BLASLONG *, bfloat16 *, bfloat16 *, BLASLONG); int sbgemm_thread_tn(blas_arg_t *, BLASLONG *, BLASLONG *, bfloat16 *, bfloat16 *, BLASLONG); diff --git a/common_macro.h b/common_macro.h index b29a9c08d..f9c22089b 100644 --- a/common_macro.h +++ b/common_macro.h @@ -1,5 +1,6 @@ /*********************************************************************/ /* Copyright 2009, 2010 The University of Texas at Austin. */ +/* Copyright 2025 The OpenBLAS Project. */ /* All rights reserved. */ /* */ /* Redistribution and use in source and binary forms, with or */ @@ -40,6 +41,7 @@ #define COMMON_MACRO #include "common_sh.h" +#include "common_b.h" #include "common_sb.h" #include "common_s.h" #include "common_d.h" @@ -702,14 +704,63 @@ #define GEMM_THREAD_RR SHGEMM_THREAD_NN -#elif defined(BFLOAT16) +#elif defined(BFLOAT16) && defined(BGEMM) +#define SCAL_K BSCAL_K + +#define GEMV_N BGEMV_N_K +#define GEMV_T BGEMV_T_K + +#define GEMM_BETA BGEMM_BETA +#define GEMM_KERNEL_N BGEMM_KERNEL +#define GEMM_KERNEL_L BGEMM_KERNEL +#define GEMM_KERNEL_R BGEMM_KERNEL +#define GEMM_KERNEL_B BGEMM_KERNEL + +#define GEMM_NN BGEMM_NN +#define GEMM_CN BGEMM_TN +#define GEMM_TN BGEMM_TN +#define GEMM_NC BGEMM_NT +#define GEMM_NT BGEMM_NT +#define GEMM_CC BGEMM_TT +#define GEMM_CT BGEMM_TT +#define GEMM_TC BGEMM_TT +#define GEMM_TT BGEMM_TT +#define GEMM_NR BGEMM_NN +#define GEMM_TR BGEMM_TN +#define GEMM_CR BGEMM_TN +#define GEMM_RN BGEMM_NN +#define GEMM_RT BGEMM_NT +#define GEMM_RC BGEMM_NT +#define GEMM_RR BGEMM_NN +#define GEMM_ONCOPY BGEMM_ONCOPY +#define GEMM_OTCOPY BGEMM_OTCOPY +#define GEMM_INCOPY BGEMM_INCOPY +#define GEMM_ITCOPY BGEMM_ITCOPY + +#define GEMM_THREAD_NN BGEMM_THREAD_NN +#define GEMM_THREAD_CN BGEMM_THREAD_TN +#define GEMM_THREAD_TN BGEMM_THREAD_TN +#define GEMM_THREAD_NC BGEMM_THREAD_NT +#define GEMM_THREAD_NT BGEMM_THREAD_NT +#define GEMM_THREAD_CC BGEMM_THREAD_TT +#define GEMM_THREAD_CT BGEMM_THREAD_TT +#define GEMM_THREAD_TC BGEMM_THREAD_TT +#define GEMM_THREAD_TT BGEMM_THREAD_TT +#define GEMM_THREAD_NR BGEMM_THREAD_NN +#define GEMM_THREAD_TR BGEMM_THREAD_TN +#define GEMM_THREAD_CR BGEMM_THREAD_TN +#define GEMM_THREAD_RN BGEMM_THREAD_NN +#define GEMM_THREAD_RT BGEMM_THREAD_NT +#define GEMM_THREAD_RC BGEMM_THREAD_NT +#define GEMM_THREAD_RR BGEMM_THREAD_NN +#elif defined(BFLOAT16) #define D_TO_BF16_K SBDTOBF16_K #define D_BF16_TO_K DBF16TOD_K #define S_TO_BF16_K SBSTOBF16_K #define S_BF16_TO_K SBF16TOS_K -#define SBGEMV_N SBGEMV_N_K -#define SBGEMV_T SBGEMV_T_K +#define GEMV_N SBGEMV_N_K +#define GEMV_T SBGEMV_T_K #define AMAX_K SAMAX_K #define AMIN_K SAMIN_K @@ -727,8 +778,6 @@ #define AXPYC_K SAXPYC_K #define AXPBY_K SAXPBY_K #define SCAL_K SSCAL_K -#define GEMV_N SGEMV_N -#define GEMV_T SGEMV_T #define SYMV_U SSYMV_U #define SYMV_L SSYMV_L #define GERU_K SGERU_K @@ -2663,6 +2712,9 @@ || defined(ARCH_LOONGARCH64) || defined(ARCH_E2K) || defined(ARCH_ALPHA)) extern BLASLONG gemm_offset_a; extern BLASLONG gemm_offset_b; +extern BLASLONG bgemm_p; +extern BLASLONG bgemm_q; +extern BLASLONG bgemm_r; extern BLASLONG sbgemm_p; extern BLASLONG sbgemm_q; extern BLASLONG sbgemm_r; diff --git a/common_param.h b/common_param.h index f82b73a72..0145f667a 100644 --- a/common_param.h +++ b/common_param.h @@ -1,6 +1,6 @@ /*********************************************************************/ /* Copyright 2009, 2010 The University of Texas at Austin. */ -/* Copyright 2023 The OpenBLAS Project. */ +/* Copyright 2023, 2025 The OpenBLAS Project. */ /* All rights reserved. */ /* */ /* Redistribution and use in source and binary forms, with or */ @@ -65,6 +65,10 @@ int (*shgemm_otcopy )(BLASLONG, BLASLONG, hfloat16 *, BLASLONG, hfloat16 *); #if BUILD_BFLOAT16 == 1 + int bgemm_p, bgemm_q, bgemm_r; + int bgemm_unroll_m, bgemm_unroll_n, bgemm_unroll_mn; + int bgemm_align_k; + int sbgemm_p, sbgemm_q, sbgemm_r; int sbgemm_unroll_m, sbgemm_unroll_n, sbgemm_unroll_mn; int sbgemm_align_k; @@ -94,10 +98,14 @@ int (*shgemm_otcopy )(BLASLONG, BLASLONG, hfloat16 *, BLASLONG, hfloat16 *); int (*sbrot_k) (BLASLONG, float *, BLASLONG, float *, BLASLONG, float, float); int (*sbrotm_k) (BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); + int (*bscal_k) (BLASLONG, BLASLONG, BLASLONG, bfloat16, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG); int (*sbaxpy_k) (BLASLONG, BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG); int (*sbscal_k) (BLASLONG, BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG); int (*sbswap_k) (BLASLONG, BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG); + int (*bgemv_n) (BLASLONG, BLASLONG, bfloat16, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, bfloat16, bfloat16 *, BLASLONG); + int (*bgemv_t) (BLASLONG, BLASLONG, bfloat16, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, bfloat16, bfloat16 *, BLASLONG); + int (*sbgemv_n) (BLASLONG, BLASLONG, float, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, float, float *, BLASLONG); int (*sbgemv_t) (BLASLONG, BLASLONG, float, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, float, float *, BLASLONG); int (*sbger_k) (BLASLONG, BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); @@ -105,6 +113,14 @@ int (*shgemm_otcopy )(BLASLONG, BLASLONG, hfloat16 *, BLASLONG, hfloat16 *); int (*sbsymv_L) (BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); int (*sbsymv_U) (BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float *, BLASLONG, float *); + int (*bgemm_kernel )(BLASLONG, BLASLONG, BLASLONG, bfloat16, bfloat16 *, bfloat16 *, bfloat16 *, BLASLONG); + int (*bgemm_beta )(BLASLONG, BLASLONG, BLASLONG, bfloat16, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG); + + int (*bgemm_incopy )(BLASLONG, BLASLONG, bfloat16 *, BLASLONG, bfloat16 *); + int (*bgemm_itcopy )(BLASLONG, BLASLONG, bfloat16 *, BLASLONG, bfloat16 *); + int (*bgemm_oncopy )(BLASLONG, BLASLONG, bfloat16 *, BLASLONG, bfloat16 *); + int (*bgemm_otcopy )(BLASLONG, BLASLONG, bfloat16 *, BLASLONG, bfloat16 *); + int (*sbgemm_kernel )(BLASLONG, BLASLONG, BLASLONG, float, bfloat16 *, bfloat16 *, float *, BLASLONG); int (*sbgemm_beta )(BLASLONG, BLASLONG, BLASLONG, float, bfloat16 *, BLASLONG, bfloat16 *, BLASLONG, float *, BLASLONG); @@ -240,6 +256,7 @@ int (*shgemm_otcopy )(BLASLONG, BLASLONG, hfloat16 *, BLASLONG, hfloat16 *); #endif #ifdef ARCH_ARM64 void (*sgemm_direct) (BLASLONG, BLASLONG, BLASLONG, float *, BLASLONG , float *, BLASLONG , float * , BLASLONG); + void (*sgemm_direct_alpha_beta) (BLASLONG, BLASLONG, BLASLONG, float, float *, BLASLONG, float *, BLASLONG, float, float * , BLASLONG); #endif @@ -1254,6 +1271,13 @@ extern gotoblas_t *gotoblas; #endif #if (BUILD_BFLOAT16==1) +#define BGEMM_P gotoblas -> bgemm_p +#define BGEMM_Q gotoblas -> bgemm_q +#define BGEMM_R gotoblas -> bgemm_r +#define BGEMM_UNROLL_M gotoblas -> bgemm_unroll_m +#define BGEMM_UNROLL_N gotoblas -> bgemm_unroll_n +#define BGEMM_UNROLL_MN gotoblas -> bgemm_unroll_mn + #define SBGEMM_P gotoblas -> sbgemm_p #define SBGEMM_Q gotoblas -> sbgemm_q #define SBGEMM_R gotoblas -> sbgemm_r @@ -1395,6 +1419,17 @@ extern gotoblas_t *gotoblas; #endif #if (BUILD_BFLOAT16 == 1) +#define BGEMM_P BGEMM_DEFAULT_P +#define BGEMM_Q BGEMM_DEFAULT_Q +#define BGEMM_R BGEMM_DEFAULT_R +#define BGEMM_UNROLL_M BGEMM_DEFAULT_UNROLL_M +#define BGEMM_UNROLL_N BGEMM_DEFAULT_UNROLL_N +#ifdef BGEMM_DEFAULT_UNROLL_MN +#define BGEMM_UNROLL_MN BGEMM_DEFAULT_UNROLL_MN +#else +#define BGEMM_UNROLL_MN MAX((BGEMM_UNROLL_M), (BGEMM_UNROLL_N)) +#endif + #define SBGEMM_P SBGEMM_DEFAULT_P #define SBGEMM_Q SBGEMM_DEFAULT_Q #define SBGEMM_R SBGEMM_DEFAULT_R @@ -1555,6 +1590,18 @@ extern gotoblas_t *gotoblas; #define GEMM_DEFAULT_R SHGEMM_DEFAULT_R #define GEMM_DEFAULT_UNROLL_M SHGEMM_DEFAULT_UNROLL_M #define GEMM_DEFAULT_UNROLL_N SHGEMM_DEFAULT_UNROLL_N +#elif defined(BFLOAT16) && defined(BGEMM) +#define GEMM_P BGEMM_P +#define GEMM_Q BGEMM_Q +#define GEMM_R BGEMM_R +#define GEMM_UNROLL_M BGEMM_UNROLL_M +#define GEMM_UNROLL_N BGEMM_UNROLL_N +#define GEMM_UNROLL_MN BGEMM_UNROLL_MN +#define GEMM_DEFAULT_P BGEMM_DEFAULT_P +#define GEMM_DEFAULT_Q BGEMM_DEFAULT_Q +#define GEMM_DEFAULT_R BGEMM_DEFAULT_R +#define GEMM_DEFAULT_UNROLL_M BGEMM_DEFAULT_UNROLL_M +#define GEMM_DEFAULT_UNROLL_N BGEMM_DEFAULT_UNROLL_N #elif defined(BFLOAT16) #define GEMM_P SBGEMM_P #define GEMM_Q SBGEMM_Q diff --git a/common_s.h b/common_s.h index 1dede1e36..88b4732f5 100644 --- a/common_s.h +++ b/common_s.h @@ -49,6 +49,7 @@ #define SGEMM_DIRECT_PERFORMANT sgemm_direct_performant #define SGEMM_DIRECT sgemm_direct +#define SGEMM_DIRECT_ALPHA_BETA sgemm_direct_alpha_beta #define SGEMM_ONCOPY sgemm_oncopy #define SGEMM_OTCOPY sgemm_otcopy @@ -218,6 +219,7 @@ #elif ARCH_ARM64 #define SGEMM_DIRECT_PERFORMANT sgemm_direct_performant #define SGEMM_DIRECT gotoblas -> sgemm_direct +#define SGEMM_DIRECT_ALPHA_BETA gotoblas -> sgemm_direct_alpha_beta #endif #define SGEMM_ONCOPY gotoblas -> sgemm_oncopy diff --git a/cpuid.S b/cpuid.S index 295917bdb..a8f5f8b57 100644 --- a/cpuid.S +++ b/cpuid.S @@ -66,5 +66,9 @@ _cpuid: #endif #if defined(__ELF__) && defined(__linux__) +#if defined(__arm__) + .section .note.GNU-stack,"",%progbits +#else .section .note.GNU-stack,"",@progbits #endif +#endif diff --git a/cpuid_power.c b/cpuid_power.c index 1ced8930a..2b2c32eea 100644 --- a/cpuid_power.c +++ b/cpuid_power.c @@ -131,6 +131,7 @@ int detect(void){ if (!strncasecmp(p, "POWER8", 6)) return CPUTYPE_POWER8; if (!strncasecmp(p, "POWER9", 6)) return CPUTYPE_POWER9; if (!strncasecmp(p, "POWER10", 7)) return CPUTYPE_POWER10; + if (!strncasecmp(p, "POWER11", 7)) return CPUTYPE_POWER10; if (!strncasecmp(p, "Cell", 4)) return CPUTYPE_CELL; if (!strncasecmp(p, "7447", 4)) return CPUTYPE_PPCG4; @@ -171,6 +172,9 @@ int detect(void){ int id; __asm __volatile("mfpvr %0" : "=r"(id)); switch ( id >> 16 ) { + case 0x82: // POWER11 + return CPUTYPE_POWER10; + break; case 0x80: // POWER10 return CPUTYPE_POWER10; break; diff --git a/cpuid_x86.c b/cpuid_x86.c index 1b09c7217..8e1438e50 100644 --- a/cpuid_x86.c +++ b/cpuid_x86.c @@ -1567,6 +1567,7 @@ int get_cpuname(void){ case 10: case 15: case 14: // Alder Lake N + case 13: // Lunar Lake if(support_avx2()) return CPUTYPE_HASWELL; if(support_avx()) @@ -2412,7 +2413,8 @@ int get_coretype(void){ case 7: // Raptor Lake case 10: case 15: - case 14: // Alder Lake N + case 14: // Alder Lake N + case 13: // Lunar Lake #ifndef NO_AVX2 if(support_avx2()) return CORE_HASWELL; diff --git a/docs/install.md b/docs/install.md index 656c6a121..73ce6df45 100644 --- a/docs/install.md +++ b/docs/install.md @@ -623,14 +623,22 @@ Note: using `TARGET=CORTEXA57` in place of `ARMV8` will pick up better optimized routines. Implementations for the `CORTEXA57` target are compatible with all other `ARMV8` targets. -Note: for NDK 23b, something as simple as: +Note: for NDK 23b and later, something as simple as: ```bash export PATH=/opt/android-ndk-r23b/toolchains/llvm/prebuilt/linux-x86_64/bin/:$PATH -make HOSTCC=gcc CC=/opt/android-ndk-r23b/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android31-clang ONLY_CBLAS=1 TARGET=ARMV8 +make HOSTCC=gcc CC=/opt/android-ndk-r23b/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android31-clang ONLY_CBLAS=1 TARGET=ARMV8 RANLIB=echo ``` appears to be sufficient on Linux. On OSX, setting AR to the ar provided in the "bin" path of the NDK (probably `llvm-ar`) is also necessary. +If you prefer building with CMake, running +```bash +cmake -DANDROID_ABI=arm64-v8a -DTARGET=ARMV8 -DCMAKE_TOOLCHAIN_FILE=/opt/android-ndk-r27/build/cmake/android.toolchain.cmake -DNOFORTRAN=1 -DANDROID_PLATFORM=android-23 .. +cmake --build . +``` +in your build directory should work (be sure to adjust the toolchain_file argument according to where you installed the NDK, and the ANDROID_PLATFORM +according to the minimum version of Android you want to support. (If you leave out the ANDROID_PLATFORM parameter, the build will fail with an error +message about a missing declaration or missing header file complex.h) ??? note "Alternative build script for 3 architectures" diff --git a/driver/level2/Makefile b/driver/level2/Makefile index 5f8c712a8..3f3731d3f 100644 --- a/driver/level2/Makefile +++ b/driver/level2/Makefile @@ -1,3 +1,31 @@ +############################################################################### +# Copyright (c) 2025 The OpenBLAS Project +# All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# 3. Neither the name of the OpenBLAS project nor the names of +# its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +############################################################################### + TOPDIR = ../.. include ../../Makefile.system @@ -423,6 +451,9 @@ XBLASOBJS += \ xtbmv_thread_CLU.$(SUFFIX) xtbmv_thread_CLN.$(SUFFIX) ifeq ($(BUILD_BFLOAT16),1) +BBLASOBJS += \ + bgemv_thread_n$(TSUFFIX).$(SUFFIX) \ + bgemv_thread_t$(TSUFFIX).$(SUFFIX) SBBLASOBJS += \ sbgemv_thread_n$(TSUFFIX).$(SUFFIX) \ sbgemv_thread_t$(TSUFFIX).$(SUFFIX) @@ -3707,6 +3738,10 @@ xtrsv_CUN.$(SUFFIX) xtrsv_CUN.$(PSUFFIX) : ztrsv_L.c ../../param.h $(CC) -c $(CFLAGS) -DXDOUBLE -DCOMPLEX -DTRANSA=4 -UUNIT $< -o $(@F) ifeq ($(BUILD_BFLOAT16),1) +bgemv_thread_n.$(SUFFIX) bgemv_thread_n.$(PSUFFIX) : sbgemv_thread.c ../../common.h + $(CC) -c $(CFLAGS) -DBGEMM -UCOMPLEX -UDOUBLE -UTRANSA -UCONJ -UXCONJ $< -o $(@F) +bgemv_thread_t.$(SUFFIX) bgemv_thread_t.$(PSUFFIX) : sbgemv_thread.c ../../common.h + $(CC) -c $(CFLAGS) -DBGEMM -UCOMPLEX -UDOUBLE -DTRANSA -UCONJ -UXCONJ $< -o $(@F) sbgemv_thread_n.$(SUFFIX) sbgemv_thread_n.$(PSUFFIX) : sbgemv_thread.c ../../common.h $(CC) -c $(CFLAGS) -UCOMPLEX -UDOUBLE -UTRANSA -UCONJ -UXCONJ $< -o $(@F) sbgemv_thread_t.$(SUFFIX) sbgemv_thread_t.$(PSUFFIX) : sbgemv_thread.c ../../common.h diff --git a/driver/level2/sbgemv_thread.c b/driver/level2/sbgemv_thread.c index 534c60f95..c7fc90a35 100644 --- a/driver/level2/sbgemv_thread.c +++ b/driver/level2/sbgemv_thread.c @@ -1,4 +1,5 @@ /*********************************************************************/ +/* Copyright 2025 The OpenBLAS Project. */ /* Copyright 2009, 2010 The University of Texas at Austin. */ /* All rights reserved. */ /* */ @@ -41,21 +42,21 @@ #include "common.h" #ifndef TRANSA -#define SBGEMV SBGEMV_N +#define GEMV GEMV_N #else -#define SBGEMV SBGEMV_T +#define GEMV GEMV_T #endif static int sbgemv_kernel(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, FLOAT *dummy1, FLOAT *dummy2, BLASLONG dummy3){ - bfloat16 *a, *x; - float *y; + IFLOAT *a, *x; + FLOAT *y; BLASLONG lda, incx, incy; BLASLONG m_from, m_to, n_from, n_to; - a = (bfloat16 *)args->a; - x = (bfloat16 *)args->b; - y = (float *)args->c; + a = (IFLOAT *)args->a; + x = (IFLOAT *)args->b; + y = (FLOAT *)args->c; lda = args->lda; incx = args->ldb; @@ -77,12 +78,12 @@ static int sbgemv_kernel(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, y += n_from * incy; #endif - SBGEMV(m_to - m_from, n_to - n_from, *((FLOAT *)(args->alpha)), a, lda, x, incx, *((FLOAT *)(args->beta)), y, incy); + GEMV(m_to - m_from, n_to - n_from, *((FLOAT *)(args->alpha)), a, lda, x, incx, *((FLOAT *)(args->beta)), y, incy); return 0; } -int CNAME(BLASLONG m, BLASLONG n, float alpha, bfloat16 *a, BLASLONG lda, bfloat16 *x, BLASLONG incx, float beta, float *y, BLASLONG incy, int threads) +int CNAME(BLASLONG m, BLASLONG n, FLOAT alpha, IFLOAT *a, BLASLONG lda, IFLOAT *x, BLASLONG incx, FLOAT beta, FLOAT *y, BLASLONG incy, int threads) { blas_arg_t args; blas_queue_t queue[MAX_CPU_NUMBER]; diff --git a/driver/level3/Makefile b/driver/level3/Makefile index 132645a87..622996c3b 100644 --- a/driver/level3/Makefile +++ b/driver/level3/Makefile @@ -1,3 +1,32 @@ +############################################################################### +# Copyright (c) 2025, The OpenBLAS Project +# All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# 3. Neither the name of the OpenBLAS project nor the names of +# its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +############################################################################### + TOPDIR = ../.. include ../../Makefile.system @@ -20,6 +49,7 @@ USE_GEMM3M = 1 endif ifeq ($(BUILD_BFLOAT16),1) +BBLASOBJS += bgemm_nn.$(SUFFIX) bgemm_nt.$(SUFFIX) bgemm_tn.$(SUFFIX) bgemm_tt.$(SUFFIX) SBBLASOBJS += sbgemm_nn.$(SUFFIX) sbgemm_nt.$(SUFFIX) sbgemm_tn.$(SUFFIX) sbgemm_tt.$(SUFFIX) endif @@ -212,6 +242,7 @@ COMMONOBJS += syrk_thread.$(SUFFIX) ifneq ($(USE_SIMPLE_THREADED_LEVEL3), 1) ifeq ($(BUILD_BFLOAT16),1) +BBLASOBJS += bgemm_thread_nn.$(SUFFIX) bgemm_thread_nt.$(SUFFIX) bgemm_thread_tn.$(SUFFIX) bgemm_thread_tt.$(SUFFIX) SBBLASOBJS += sbgemm_thread_nn.$(SUFFIX) sbgemm_thread_nt.$(SUFFIX) sbgemm_thread_tn.$(SUFFIX) sbgemm_thread_tt.$(SUFFIX) endif ifeq ($(BUILD_HFLOAT16),1) @@ -350,6 +381,18 @@ endif all :: +bgemm_nn.$(SUFFIX) : gemm.c level3.c ../../param.h + $(CC) $(CFLAGS) $(BLOCKS) -c -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX -DNN $< -o $(@F) + +bgemm_nt.$(SUFFIX) : gemm.c level3.c ../../param.h + $(CC) $(CFLAGS) $(BLOCKS) -c -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX -DNT $< -o $(@F) + +bgemm_tn.$(SUFFIX) : gemm.c level3.c ../../param.h + $(CC) $(CFLAGS) $(BLOCKS) -c -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX -DTN $< -o $(@F) + +bgemm_tt.$(SUFFIX) : gemm.c level3.c ../../param.h + $(CC) $(CFLAGS) $(BLOCKS) -c -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX -DTT $< -o $(@F) + sbgemm_nn.$(SUFFIX) : gemm.c level3.c ../../param.h $(CC) $(CFLAGS) $(BLOCKS) -c -DBFLOAT16 -UDOUBLE -UCOMPLEX -DNN $< -o $(@F) @@ -569,6 +612,18 @@ gemm_thread_variable.$(SUFFIX) : gemm_thread_variable.c ../../common.h beta_thread.$(SUFFIX) : beta_thread.c ../../common.h $(CC) -c $(CFLAGS) $< -o $(@F) +bgemm_thread_nn.$(SUFFIX) : gemm.c level3_thread.c ../../param.h + $(CC) $(CFLAGS) $(BLOCKS) -c -DTHREADED_LEVEL3 -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX -DNN $< -o $(@F) + +bgemm_thread_nt.$(SUFFIX) : gemm.c level3_thread.c ../../param.h + $(CC) $(CFLAGS) $(BLOCKS) -c -DTHREADED_LEVEL3 -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX -DNT $< -o $(@F) + +bgemm_thread_tn.$(SUFFIX) : gemm.c level3_thread.c ../../param.h + $(CC) $(CFLAGS) $(BLOCKS) -c -DTHREADED_LEVEL3 -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX -DTN $< -o $(@F) + +bgemm_thread_tt.$(SUFFIX) : gemm.c level3_thread.c ../../param.h + $(CC) $(CFLAGS) $(BLOCKS) -c -DTHREADED_LEVEL3 -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX -DTT $< -o $(@F) + sbgemm_thread_nn.$(SUFFIX) : gemm.c level3_thread.c ../../param.h $(CC) $(CFLAGS) $(BLOCKS) -c -DTHREADED_LEVEL3 -DBFLOAT16 -UDOUBLE -UCOMPLEX -DNN $< -o $(@F) diff --git a/driver/level3/level3.c b/driver/level3/level3.c index b7328876b..78bc6aa52 100644 --- a/driver/level3/level3.c +++ b/driver/level3/level3.c @@ -1,5 +1,6 @@ /*********************************************************************/ /* Copyright 2009, 2010 The University of Texas at Austin. */ +/* Copyright 2025 The OpenBLAS Project. */ /* All rights reserved. */ /* */ /* Redistribution and use in source and binary forms, with or */ @@ -169,6 +170,22 @@ #define STOP_RPCC(COUNTER) #endif +#if defined(BUILD_BFLOAT16) +#if defined(DYNAMIC_ARCH) + #if defined(BGEMM) + #define BFLOAT16_ALIGN_K gotoblas->bgemm_align_k + #else + #define BFLOAT16_ALIGN_K gotoblas->sbgemm_align_k + #endif +#else + #if defined(BGEMM) + #define BFLOAT16_ALIGN_K BGEMM_ALIGN_K + #else + #define BFLOAT16_ALIGN_K SBGEMM_ALIGN_K + #endif +#endif +#endif + int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, XFLOAT *sa, XFLOAT *sb, BLASLONG dummy){ BLASLONG k, lda, ldb, ldc; @@ -305,12 +322,8 @@ int CNAME(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, } BLASLONG pad_min_l = min_l; -#if defined(HALF) -#if defined(DYNAMIC_ARCH) - pad_min_l = (min_l + gotoblas->sbgemm_align_k - 1) & ~(gotoblas->sbgemm_align_k-1); -#else - pad_min_l = (min_l + SBGEMM_ALIGN_K - 1) & ~(SBGEMM_ALIGN_K - 1);; -#endif +#if defined(BFLOAT16) + pad_min_l = (min_l + BFLOAT16_ALIGN_K - 1) & ~(BFLOAT16_ALIGN_K - 1); #endif /* First, we have to move data A to L2 cache */ diff --git a/driver/level3/level3_thread.c b/driver/level3/level3_thread.c index db3bffc10..cb93591ab 100644 --- a/driver/level3/level3_thread.c +++ b/driver/level3/level3_thread.c @@ -1,6 +1,6 @@ /*********************************************************************/ /* Copyright 2009, 2010 The University of Texas at Austin. */ -/* Copyright 2023 The OpenBLAS Project. */ +/* Copyright 2023, 2025 The OpenBLAS Project. */ /* All rights reserved. */ /* */ /* Redistribution and use in source and binary forms, with or */ @@ -216,6 +216,22 @@ typedef struct { #define STOP_RPCC(COUNTER) #endif +#if defined(BUILD_BFLOAT16) +#if defined(DYNAMIC_ARCH) + #if defined(BGEMM) + #define BFLOAT16_ALIGN_K gotoblas->bgemm_align_k + #else + #define BFLOAT16_ALIGN_K gotoblas->sbgemm_align_k + #endif +#else + #if defined(BGEMM) + #define BFLOAT16_ALIGN_K BGEMM_ALIGN_K + #else + #define BFLOAT16_ALIGN_K SBGEMM_ALIGN_K + #endif +#endif +#endif + static int inner_thread(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, IFLOAT *sa, IFLOAT *sb, BLASLONG mypos){ IFLOAT *buffer[DIVIDE_RATE]; @@ -324,12 +340,8 @@ static int inner_thread(blas_arg_t *args, BLASLONG *range_m, BLASLONG *range_n, BLASLONG pad_min_l = min_l; -#if defined(HALF) -#if defined(DYNAMIC_ARCH) - pad_min_l = (min_l + gotoblas->sbgemm_align_k - 1) & ~(gotoblas->sbgemm_align_k-1); -#else - pad_min_l = (min_l + SBGEMM_ALIGN_K - 1) & ~(SBGEMM_ALIGN_K - 1);; -#endif +#if defined(BFLOAT16) + pad_min_l = (min_l + BFLOAT16_ALIGN_K - 1) & ~(BFLOAT16_ALIGN_K - 1); #endif /* Determine step size in m diff --git a/driver/others/dynamic_power.c b/driver/others/dynamic_power.c index 4c1f4a26e..f0afe3eb6 100644 --- a/driver/others/dynamic_power.c +++ b/driver/others/dynamic_power.c @@ -70,6 +70,9 @@ static int cpuid(void) #endif #ifdef POWER_10 else if (arch >= POWER_10) return CPU_POWER10; +#endif +#ifdef POWER_11 + else if (arch >= POWER_11) return CPU_POWER10; #endif return CPU_UNKNOWN; } @@ -173,6 +176,13 @@ static struct { .cpu_type = CPU_POWER10, }, + { /* Power11 */ + .pvr_mask = 0xffff0000, + .pvr_value = 0x00820000, + .cpu_name = "POWER11 (raw)", + .cpu_type = CPU_POWER10, + }, + { /* End of table, pvr_mask and pvr_value must be zero */ .pvr_mask = 0x0, .pvr_value = 0x0, diff --git a/driver/others/parameter.c b/driver/others/parameter.c index 3bcb0d434..9a1ff5735 100644 --- a/driver/others/parameter.c +++ b/driver/others/parameter.c @@ -72,6 +72,11 @@ BLASLONG shgemm_p = DEFAULT_GEMM_P; #else BLASLONG shgemm_p = SHGEMM_P; #endif +#if BGEMM_P == bgemm_p +BLASLONG bgemm_p = DEFAULT_GEMM_P; +#else +BLASLONG bgemm_p = BGEMM_P; +#endif #if SGEMM_P == sgemm_p BLASLONG sgemm_p = DEFAULT_GEMM_P; #else @@ -103,6 +108,11 @@ BLASLONG shgemm_q = DEFAULT_GEMM_Q; #else BLASLONG shgemm_q = SHGEMM_Q; #endif +#if BGEMM_Q == bgemm_q +BLASLONG bgemm_q = DEFAULT_GEMM_Q; +#else +BLASLONG bgemm_q = BGEMM_Q; +#endif #if SGEMM_Q == sgemm_q BLASLONG sgemm_q = DEFAULT_GEMM_Q; #else @@ -134,6 +144,11 @@ BLASLONG shgemm_r = DEFAULT_GEMM_R; #else BLASLONG shgemm_r = SHGEMM_R; #endif +#if BGEMM_R == bgemm_r +BLASLONG bgemm_r = DEFAULT_GEMM_R; +#else +BLASLONG bgemm_r = BGEMM_R; +#endif #if SGEMM_R == sgemm_r BLASLONG sgemm_r = DEFAULT_GEMM_R; #else @@ -541,6 +556,7 @@ void blas_set_parameter(void){ #ifdef BUILD_BFLOAT16 sbgemm_r = (((BUFFER_SIZE - ((SBGEMM_P * SBGEMM_Q * 4 + GEMM_OFFSET_A + GEMM_ALIGN) & ~GEMM_ALIGN)) / (SBGEMM_Q * 4)) - 15) & ~15; + bgemm_r = (((BUFFER_SIZE - ((BGEMM_P * BGEMM_Q * 4 + GEMM_OFFSET_A + GEMM_ALIGN) & ~GEMM_ALIGN)) / (BGEMM_Q * 4)) - 15) & ~15; #endif #ifdef BUILD_HFLOAT16 shgemm_r = (((BUFFER_SIZE - ((SHGEMM_P * SHGEMM_Q * 4 + GEMM_OFFSET_A + GEMM_ALIGN) & ~GEMM_ALIGN)) / (SHGEMM_Q * 4)) - 15) & ~15; @@ -653,6 +669,7 @@ void blas_set_parameter(void){ #ifdef BUILD_BFLOAT16 sbgemm_r = (((BUFFER_SIZE - ((SBGEMM_P * SBGEMM_Q * 4 + GEMM_OFFSET_A + GEMM_ALIGN) & ~GEMM_ALIGN)) / (SBGEMM_Q * 4)) - 15) & ~15; + bgemm_r = (((BUFFER_SIZE - ((BGEMM_P * BGEMM_Q * 4 + GEMM_OFFSET_A + GEMM_ALIGN) & ~GEMM_ALIGN)) / (BGEMM_Q * 4)) - 15) & ~15; #endif #ifdef BUILD_HFLOAT16 shgemm_r = (((BUFFER_SIZE - ((SHGEMM_P * SHGEMM_Q * 4 + GEMM_OFFSET_A + GEMM_ALIGN) & ~GEMM_ALIGN)) / (SHGEMM_Q * 4)) - 15) & ~15; diff --git a/exports/gensymbol b/exports/gensymbol index 3719574ea..40e13e623 100755 --- a/exports/gensymbol +++ b/exports/gensymbol @@ -1,5 +1,33 @@ #!/bin/sh +############################################################################### +# Copyright (c) 2025, The OpenBLAS Project +# All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# 3. Neither the name of the OpenBLAS project nor the names of +# its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +############################################################################### + # Changelog # 2017/09/03 staticfloat # Added zsymv and csymv into @lapackobjs2 so they are properly renamed @@ -51,7 +79,7 @@ blasobjsz=" zgeadd dzsum zgemmt zgemmtr" blasobjs="lsame xerbla" -bfblasobjs="sbgemm sbgemmt sbgemmtr sbgemv sbdot sbstobf16 sbdtobf16 sbf16tos dbf16tod" +bfblasobjs="bgemm bgemv sbgemm sbgemmt sbgemmtr sbgemv sbdot sbstobf16 sbdtobf16 sbf16tos dbf16tod" hfblasobjs="shgemm" cblasobjsc=" cblas_caxpy cblas_ccopy cblas_cdotc cblas_cdotu cblas_cgbmv cblas_cgemm cblas_cgemv diff --git a/exports/gensymbol.pl b/exports/gensymbol.pl index 5a8423697..3447a4e51 100644 --- a/exports/gensymbol.pl +++ b/exports/gensymbol.pl @@ -1,5 +1,33 @@ #!/usr/bin/env perl +############################################################################### +# Copyright (c) 2025, The OpenBLAS Project +# All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# 3. Neither the name of the OpenBLAS project nor the names of +# its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +############################################################################### + # Changelog # 2017/09/03 staticfloat # Added zsymv and csymv into @lapackobjs2 so they are properly renamed @@ -51,7 +79,7 @@ zgeadd, dzsum, zgemmt,zgemmtr); @blasobjs = (lsame, xerbla); -@bfblasobjs = (sbgemm, sbgemmt, sbgemmtr, sbgemv, sbdot, sbstobf16, sbdtobf16, sbf16tos, dbf16tod); +@bfblasobjs = (bgemm, bgemv, sbgemm, sbgemmt, sbgemmtr, sbgemv, sbdot, sbstobf16, sbdtobf16, sbf16tos, dbf16tod); @hfblasobjs = (shgemm); @cblasobjsc = ( cblas_caxpy, cblas_ccopy, cblas_cdotc, cblas_cdotu, cblas_cgbmv, cblas_cgemm, cblas_cgemv, diff --git a/getarch.c b/getarch.c index cb0b3cd7c..51c6705fe 100644 --- a/getarch.c +++ b/getarch.c @@ -1,5 +1,5 @@ /***************************************************************************** -Copyright (c) 2011-2014, The OpenBLAS Project +Copyright (c) 2011-2014, 2025 The OpenBLAS Project All rights reserved. Redistribution and use in source and binary forms, with or without @@ -836,7 +836,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define CORENAME "POWER9" #endif -#if defined(FORCE_POWER10) +#if defined(FORCE_POWER10) || (FORCE_POWER11) #define FORCE #define ARCHITECTURE "POWER" #define SUBARCHITECTURE "POWER10" @@ -1476,7 +1476,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. "-DL2_SIZE=1048576 -DL2_LINESIZE=64 -DL2_ASSOCIATIVE=16 " \ "-DDTB_DEFAULT_ENTRIES=64 -DDTB_SIZE=4096 " \ "-DHAVE_VFPV4 -DHAVE_VFPV3 -DHAVE_VFP -DHAVE_NEON -DHAVE_SVE -DARMV8 " \ - "-march=armv8.4-a+sve -mtune=neoverse-v1" + "-march=armv8.4-a+sve+bf16 -mtune=neoverse-v1" #define LIBNAME "neoversev1" #define CORENAME "NEOVERSEV1" #endif diff --git a/getarch_2nd.c b/getarch_2nd.c index 8170e9cf3..2085556bd 100644 --- a/getarch_2nd.c +++ b/getarch_2nd.c @@ -1,3 +1,31 @@ +/*************************************************************************** + * Copyright (c) 2025, The OpenBLAS Project + * All rights reserved. + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of the OpenBLAS project nor the names of + * its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * *****************************************************************************/ + #include #ifndef BUILD_KERNEL #include "config.h" @@ -17,6 +45,10 @@ typedef unsigned long BLASULONG; int main(int argc, char **argv) { if ( (argc <= 1) || ((argc >= 2) && (*argv[1] == '0'))) { + printf("BGEMM_UNROLL_M=%d\n", BGEMM_DEFAULT_UNROLL_M); + printf("BGEMM_UNROLL_N=%d\n", BGEMM_DEFAULT_UNROLL_N); + printf("BGEMM_UNROLL_M=%d\n", BGEMM_DEFAULT_UNROLL_M); + printf("BGEMM_UNROLL_N=%d\n", BGEMM_DEFAULT_UNROLL_N); printf("SBGEMM_UNROLL_M=%d\n", SBGEMM_DEFAULT_UNROLL_M); printf("SBGEMM_UNROLL_N=%d\n", SBGEMM_DEFAULT_UNROLL_N); printf("SHGEMM_UNROLL_M=%d\n", SHGEMM_DEFAULT_UNROLL_M); diff --git a/interface/CMakeLists.txt b/interface/CMakeLists.txt index 3f7672baf..f2ca39479 100644 --- a/interface/CMakeLists.txt +++ b/interface/CMakeLists.txt @@ -1,3 +1,30 @@ +############################################################################### +# Copyright (c) 2025, The OpenBLAS Project +# All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# 3. Neither the name of the OpenBLAS project nor the names of +# its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +############################################################################### include_directories(${PROJECT_SOURCE_DIR}) include_directories(${PROJECT_BINARY_DIR}) @@ -123,10 +150,13 @@ endif () GenerateNamedObjects("imax.c" "USE_MIN" "i*min" ${CBLAS_FLAG}) if (BUILD_BFLOAT16) + GenerateNamedObjects("scal.c" "BGEMM" "bscal" ${CBLAS_FLAG} "" "" true "BFLOAT16") GenerateNamedObjects("bf16dot.c" "" "sbdot" ${CBLAS_FLAG} "" "" true "BFLOAT16") + GenerateNamedObjects("gemm.c" "BGEMM" "bgemm" ${CBLAS_FLAG} "" "" true "BFLOAT16") GenerateNamedObjects("gemm.c" "" "sbgemm" ${CBLAS_FLAG} "" "" true "BFLOAT16") GenerateNamedObjects("sbgemmt.c" "" "sbgemmt" ${CBLAS_FLAG} "" "" true "BFLOAT16") GenerateNamedObjects("sbgemmt.c" "RNAME" "sbgemmtr" ${CBLAS_FLAG} "" "" true "BFLOAT16") + GenerateNamedObjects("sbgemv.c" "BGEMM" "bgemv" ${CBLAS_FLAG} "" "" true "BFLOAT16") GenerateNamedObjects("sbgemv.c" "" "sbgemv" ${CBLAS_FLAG} "" "" true "BFLOAT16") GenerateNamedObjects("tobf16.c" "SINGLE_PREC" "sbstobf16" ${CBLAS_FLAG} "" "" true "BFLOAT16") GenerateNamedObjects("tobf16.c" "DOUBLE_PREC" "sbdtobf16" ${CBLAS_FLAG} "" "" true "BFLOAT16") diff --git a/interface/Makefile b/interface/Makefile index faac7c05e..d7676004a 100644 --- a/interface/Makefile +++ b/interface/Makefile @@ -1,3 +1,31 @@ +############################################################################### +# Copyright (c) 2025, The OpenBLAS Project +# All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# 3. Neither the name of the OpenBLAS project nor the names of +# its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +############################################################################### + TOPDIR = .. include $(TOPDIR)/Makefile.system @@ -47,6 +75,9 @@ SBLAS3OBJS = \ sgeadd.$(SUFFIX) sgemmt.$(SUFFIX) sgemmtr.$(SUFFIX) ifeq ($(BUILD_BFLOAT16),1) +BBLAS3OBJS = bgemm.$(SUFFIX) +BBLAS2OBJS = bgemv.$(SUFFIX) +BBLAS1OBJS = bscal.$(SUFFIX) SBBLAS1OBJS = sbdot.$(SUFFIX) SBBLAS2OBJS = sbgemv.$(SUFFIX) SBBLAS3OBJS = sbgemm.$(SUFFIX) sbgemmt.$(SUFFIX) sbgemmtr.$(SUFFIX) @@ -289,6 +320,9 @@ CSBLAS3OBJS = \ cblas_sgeadd.$(SUFFIX) cblas_sgemmt.$(SUFFIX) cblas_sgemmtr.$(SUFFIX) cblas_sgemm_batch.$(SUFFIX) ifeq ($(BUILD_BFLOAT16),1) +CBBLAS3OBJS = cblas_bgemm.$(SUFFIX) +CBBLAS2OBJS = cblas_bgemv.$(SUFFIX) +CBBLAS1OBJS = cblas_bscal.$(SUFFIX) CSBBLAS1OBJS = cblas_sbdot.$(SUFFIX) CSBBLAS2OBJS = cblas_sbgemv.$(SUFFIX) CSBBLAS3OBJS = cblas_sbgemm.$(SUFFIX) cblas_sbgemmt.$(SUFFIX) cblas_sbgemmtr.$(SUFFIX) cblas_sbgemm_batch.$(SUFFIX) @@ -393,6 +427,9 @@ override CFLAGS += -I. SBLAS1OBJS += $(CSBLAS1OBJS) SBLAS2OBJS += $(CSBLAS2OBJS) SBLAS3OBJS += $(CSBLAS3OBJS) +BBLAS3OBJS += $(CBBLAS3OBJS) +BBLAS2OBJS += $(CBBLAS2OBJS) +BBLAS1OBJS += $(CBBLAS1OBJS) SBBLAS1OBJS += $(CSBBLAS1OBJS) SBBLAS2OBJS += $(CSBBLAS2OBJS) SBBLAS3OBJS += $(CSBBLAS3OBJS) @@ -412,6 +449,7 @@ SBEXTOBJS += $(CSBEXTOBJS) CBAUXOBJS += $(CXERBLAOBJ) endif +BBLASOBJS = $(BBLAS3OBJS) $(BBLAS2OBJS) $(BBLAS1OBJS) SBLASOBJS = $(SBLAS1OBJS) $(SBLAS2OBJS) $(SBLAS3OBJS) SBBLASOBJS = $(SBBLAS1OBJS) $(SBBLAS2OBJS) $(SBBLAS3OBJS) SHBLASOBJS = $(SHBLAS3OBJS) @@ -522,7 +560,7 @@ ifneq ($(BUILD_COMPLEX16),1) ZBLASOBJS= endif -FUNCOBJS = $(SBEXTOBJS) $(CXERBLAOBJS) $(SBBLASOBJS) $(SBLASOBJS) $(DBLASOBJS) $(CBLASOBJS) $(ZBLASOBJS) $(SHBLASOBJS) +FUNCOBJS = $(SBEXTOBJS) $(CXERBLAOBJS) $(BBLASOBJS) $(SBBLASOBJS) $(SBLASOBJS) $(DBLASOBJS) $(CBLASOBJS) $(ZBLASOBJS) $(SHBLASOBJS) ifeq ($(EXPRECISION), 1) FUNCOBJS += $(QBLASOBJS) $(XBLASOBJS) @@ -557,10 +595,10 @@ clean :: level1 : $(SBEXTOBJS) $(SBBLAS1OBJS) $(SBLAS1OBJS) $(DBLAS1OBJS) $(QBLAS1OBJS) $(CBLAS1OBJS) $(ZBLAS1OBJS) $(XBLAS1OBJS) $(AR) $(ARFLAGS) -ru $(TOPDIR)/$(LIBNAME) $^ -level2 : $(SBBLAS2OBJS) $(SBLAS2OBJS) $(DBLAS2OBJS) $(QBLAS2OBJS) $(CBLAS2OBJS) $(ZBLAS2OBJS) $(XBLAS2OBJS) +level2 : $(SBBLAS2OBJS) $(BBLAS2OBJS) $(SBLAS2OBJS) $(DBLAS2OBJS) $(QBLAS2OBJS) $(CBLAS2OBJS) $(ZBLAS2OBJS) $(XBLAS2OBJS) $(AR) $(ARFLAGS) -ru $(TOPDIR)/$(LIBNAME) $^ -level3 : $(SBBLAS3OBJS) $(SBLAS3OBJS) $(DBLAS3OBJS) $(QBLAS3OBJS) $(CBLAS3OBJS) $(ZBLAS3OBJS) $(XBLAS3OBJS) $(SHBLAS3OBJS) +level3 : $(SBBLAS3OBJS) $(BBLAS3OBJ) $(SBLAS3OBJS) $(DBLAS3OBJS) $(QBLAS3OBJS) $(CBLAS3OBJS) $(ZBLAS3OBJS) $(XBLAS3OBJS) $(SHBLAS3OBJS) $(AR) $(ARFLAGS) -ru $(TOPDIR)/$(LIBNAME) $^ aux : $(CBAUXOBJS) @@ -792,6 +830,8 @@ dsdot.$(SUFFIX) dsdot.$(PSUFFIX) : dsdot.c $(CC) $(CFLAGS) -c $< -o $(@F) ifeq ($(BUILD_BFLOAT16),1) +bscal.$(SUFFIX) bscal.$(PSUFFIX) : scal.c + $(CC) $(CFLAGS) -DBGEMM -c $< -o $(@F) sbdot.$(SUFFIX) sbdot.$(PSUFFIX) : bf16dot.c $(CC) $(CFLAGS) -c $< -o $(@F) sbstobf16.$(SUFFIX) sbstobf16.$(PSUFFIX) : tobf16.c @@ -949,6 +989,8 @@ xgerc.$(SUFFIX) xgerc.$(PSUFFIX) : zger.c $(CC) -c $(CFLAGS) -DCONJ $< -o $(@F) ifeq ($(BUILD_BFLOAT16),1) +bgemv.$(SUFFIX) bgemv.$(PSUFFIX) : sbgemv.c + $(CC) $(CFLAGS) -DBGEMM -c $< -o $(@F) sbgemv.$(SUFFIX) sbgemv.$(PSUFFIX) : sbgemv.c $(CC) $(CFLAGS) -c $< -o $(@F) endif @@ -1311,6 +1353,8 @@ xhpr2.$(SUFFIX) xhpr2.$(PSUFFIX) : zhpr2.c $(CC) -c $(CFLAGS) $< -o $(@F) ifeq ($(BUILD_BFLOAT16),1) +bgemm.$(SUFFIX) bgemm.$(PSUFFIX) : gemm.c ../param.h + $(CC) -c $(CFLAGS) $< -o $(@F) sbgemm.$(SUFFIX) sbgemm.$(PSUFFIX) : gemm.c ../param.h $(CC) -c $(CFLAGS) $< -o $(@F) sbgemmt.$(SUFFIX) sbgemmt.$(PSUFFIX) : sbgemmt.c ../param.h @@ -1619,6 +1663,8 @@ cblas_dsdot.$(SUFFIX) cblas_dsdot.$(PSUFFIX) : dsdot.c $(CC) $(CFLAGS) -DCBLAS -c $< -o $(@F) ifeq ($(BUILD_BFLOAT16),1) +cblas_bscal.$(SUFFIX) cblas_bscal.$(PSUFFIX) : scal.c + $(CC) $(CFLAGS) -DCBLAS -c $< -o $(@F) cblas_sbdot.$(SUFFIX) cblas_sbdot.$(PSUFFIX) : bf16dot.c $(CC) $(CFLAGS) -DCBLAS -c $< -o $(@F) cblas_sbstobf16.$(SUFFIX) cblas_sbstobf16.$(PSUFFIX) : tobf16.c @@ -1773,6 +1819,8 @@ cblas_zdrot.$(SUFFIX) cblas_zdrot.$(PSUFFIX) : zrot.c $(CC) $(CFLAGS) -DCBLAS -c $< -o $(@F) ifeq ($(BUILD_BFLOAT16),1) +cblas_bgemv.$(SUFFIX) cblas_bgemv.$(PSUFFIX) : sbgemv.c + $(CC) -DCBLAS -DBGEMM -c $(CFLAGS) $< -o $(@F) cblas_sbgemv.$(SUFFIX) cblas_sbgemv.$(PSUFFIX) : sbgemv.c $(CC) -DCBLAS -c $(CFLAGS) $< -o $(@F) endif @@ -1979,6 +2027,8 @@ cblas_sgemm.$(SUFFIX) cblas_sgemm.$(PSUFFIX) : gemm.c ../param.h $(CC) -DCBLAS -c $(CFLAGS) $< -o $(@F) ifeq ($(BUILD_BFLOAT16),1) +cblas_bgemm.$(SUFFIX) cblas_bgemm.$(PSUFFIX) : gemm.c ../param.h + $(CC) -DCBLAS -DBGEMM -c $(CFLAGS) $< -o $(@F) cblas_sbgemm.$(SUFFIX) cblas_sbgemm.$(PSUFFIX) : gemm.c ../param.h $(CC) -DCBLAS -c $(CFLAGS) $< -o $(@F) endif diff --git a/interface/gemm.c b/interface/gemm.c index d79282e13..c5182c266 100644 --- a/interface/gemm.c +++ b/interface/gemm.c @@ -54,8 +54,13 @@ #define ERROR_NAME "DGEMM " #define GEMV BLASFUNC(dgemv) #elif defined(BFLOAT16) +#ifdef BGEMM +#define ERROR_NAME "BGEMM " +#define GEMV BLASFUNC(bgemv) +#else #define ERROR_NAME "SBGEMM " #define GEMV BLASFUNC(sbgemv) +#endif #elif defined(HFLOAT16) #define ERROR_NAME "SHGEMM " #else @@ -113,7 +118,7 @@ static int (*gemm[])(blas_arg_t *, BLASLONG *, BLASLONG *, IFLOAT *, IFLOAT *, B #endif }; -#if defined(SMALL_MATRIX_OPT) && !defined(GEMM3M) && !defined(XDOUBLE) &&!defined(HFLOAT16) +#if defined(SMALL_MATRIX_OPT) && !defined(GEMM3M) && !defined(XDOUBLE) && !defined(HFLOAT16) && !defined(BGEMM) #define USE_SMALL_MATRIX_OPT 1 #else #define USE_SMALL_MATRIX_OPT 0 @@ -436,6 +441,9 @@ void CNAME(enum CBLAS_ORDER order, enum CBLAS_TRANSPOSE TransA, enum CBLAS_TRANS if (beta == 0 && alpha == 1.0 && order == CblasRowMajor && TransA == CblasNoTrans && TransB == CblasNoTrans) { SGEMM_DIRECT(m, n, k, a, lda, b, ldb, c, ldc); return; + }else if (order == CblasRowMajor && TransA == CblasNoTrans && TransB == CblasNoTrans) { + SGEMM_DIRECT_ALPHA_BETA(m, n, k, alpha, a, lda, b, ldb, beta, c, ldc); + return; } #endif #endif @@ -579,7 +587,7 @@ void CNAME(enum CBLAS_ORDER order, enum CBLAS_TRANSPOSE TransA, enum CBLAS_TRANS args.m, args.n, args.k, args.lda, args.ldb, args.ldc); #endif -#if defined(GEMM_GEMV_FORWARD) && !defined(GEMM3M) && !defined(COMPLEX) && !defined(HFLOAT16) && (!defined(BFLOAT16) || defined(GEMM_GEMV_FORWARD_BF16)) +#if defined(GEMM_GEMV_FORWARD) && !defined(GEMM3M) && !defined(COMPLEX) && !defined(HFLOAT16) && (!defined(BFLOAT16) || (!defined(BGEMM) && defined(SBGEMM_GEMV_FORWARD)) || (defined(BGEMM) && defined(BGEMM_GEMV_FORWARD))) #if defined(ARCH_ARM64) // The gemv kernels in arm64/{gemv_n.S,gemv_n_sve.c,gemv_t.S,gemv_t_sve.c} // perform poorly in certain circumstances. We use the following boolean diff --git a/interface/sbgemmt.c b/interface/sbgemmt.c index 759af4bfb..67914fe65 100644 --- a/interface/sbgemmt.c +++ b/interface/sbgemmt.c @@ -1,5 +1,5 @@ /*********************************************************************/ -/* Copyright 2024, The OpenBLAS Project. */ +/* Copyright 2024-2025 The OpenBLAS Project. */ /* All rights reserved. */ /* */ /* Redistribution and use in source and binary forms, with or */ @@ -305,7 +305,7 @@ void CNAME(enum CBLAS_ORDER order, enum CBLAS_UPLO Uplo, #endif int (*gemv[]) (BLASLONG, BLASLONG, FLOAT, IFLOAT *, BLASLONG, IFLOAT *, BLASLONG, FLOAT, FLOAT *, BLASLONG) = { - SBGEMV_N, SBGEMV_T,}; + GEMV_N, GEMV_T,}; if (m == 0) diff --git a/interface/sbgemv.c b/interface/sbgemv.c index fce86f8e4..cee3e80fc 100644 --- a/interface/sbgemv.c +++ b/interface/sbgemv.c @@ -1,4 +1,5 @@ /*********************************************************************/ +/* Copyright 2025 The OpenBLAS Project. */ /* Copyright 2009, 2010 The University of Texas at Austin. */ /* All rights reserved. */ /* */ @@ -43,17 +44,25 @@ #include "functable.h" #endif +#ifdef BGEMM +#define GEMV_THREAD_N bgemv_thread_n +#define GEMV_THREAD_T bgemv_thread_t +#define ERROR_NAME "BGEMV " +#else +#define GEMV_THREAD_N sbgemv_thread_n +#define GEMV_THREAD_T sbgemv_thread_t #define ERROR_NAME "SBGEMV " +#endif #ifdef SMP -static int (*sbgemv_thread[])(BLASLONG, BLASLONG, float, bfloat16 *, BLASLONG, bfloat16 * , BLASLONG, float, float *, BLASLONG, int) = { - sbgemv_thread_n, sbgemv_thread_t, +static int (*gemv_thread[])(BLASLONG, BLASLONG, FLOAT, IFLOAT *, BLASLONG, IFLOAT * , BLASLONG, FLOAT, FLOAT *, BLASLONG, int) = { + GEMV_THREAD_N, GEMV_THREAD_T, }; #endif #ifndef CBLAS -void NAME(char *TRANS, blasint *M, blasint *N, float *ALPHA, bfloat16 *a, blasint *LDA, bfloat16 *x, blasint *INCX, float *BETA, float *y, blasint *INCY) +void NAME(char *TRANS, blasint *M, blasint *N, FLOAT *ALPHA, IFLOAT *a, blasint *LDA, IFLOAT *x, blasint *INCX, FLOAT *BETA, FLOAT *y, blasint *INCY) { char trans = *TRANS; blasint m = *M; @@ -61,14 +70,14 @@ void NAME(char *TRANS, blasint *M, blasint *N, float *ALPHA, bfloat16 *a, blasin blasint lda = *LDA; blasint incx = *INCX; blasint incy = *INCY; - float alpha = *ALPHA; - float beta = *BETA; + FLOAT alpha = *ALPHA; + FLOAT beta = *BETA; #ifdef SMP int nthreads; #endif - int (*sbgemv[])(BLASLONG, BLASLONG, float, bfloat16 *, BLASLONG, bfloat16 * , BLASLONG, float, float *, BLASLONG) = { - SBGEMV_N, SBGEMV_T, + int (*gemv[])(BLASLONG, BLASLONG, FLOAT, IFLOAT *, BLASLONG, IFLOAT * , BLASLONG, FLOAT, FLOAT *, BLASLONG) = { + GEMV_N, GEMV_T, }; blasint info; @@ -104,7 +113,7 @@ void NAME(char *TRANS, blasint *M, blasint *N, float *ALPHA, bfloat16 *a, blasin #else -void CNAME(enum CBLAS_ORDER order, enum CBLAS_TRANSPOSE TransA, blasint m, blasint n, float alpha, bfloat16 *a, blasint lda, bfloat16 *x, blasint incx, float beta, float *y, blasint incy) +void CNAME(enum CBLAS_ORDER order, enum CBLAS_TRANSPOSE TransA, blasint m, blasint n, FLOAT alpha, IFLOAT *a, blasint lda, IFLOAT *x, blasint incx, FLOAT beta, FLOAT *y, blasint incy) { blasint lenx, leny; int trans; @@ -113,8 +122,8 @@ void CNAME(enum CBLAS_ORDER order, enum CBLAS_TRANSPOSE TransA, blasint m, blasi int nthreads; #endif - int (*sbgemv[])(BLASLONG, BLASLONG, float, bfloat16 *, BLASLONG, bfloat16 * , BLASLONG, float, float *, BLASLONG) = { - SBGEMV_N, SBGEMV_T, + int (*gemv[])(BLASLONG, BLASLONG, FLOAT, IFLOAT *, BLASLONG, IFLOAT * , BLASLONG, FLOAT, FLOAT *, BLASLONG) = { + GEMV_N, GEMV_T, }; PRINT_DEBUG_CNAME; @@ -166,8 +175,17 @@ void CNAME(enum CBLAS_ORDER order, enum CBLAS_TRANSPOSE TransA, blasint m, blasi leny = m; } - if (alpha == ZERO) { - if (beta != ONE) SCAL_K(leny, 0, 0, beta, y, blasabs(incy), NULL, 0, NULL, 0); +#ifdef BGEMM + float alpha_float, beta_float; + SBF16TOS_K(1, &alpha, 1, &alpha_float, 1); + SBF16TOS_K(1, &beta, 1, &beta_float, 1); +#else + float alpha_float = alpha; + float beta_float = beta; +#endif + + if (alpha_float == ZERO) { + if (beta_float != ONE) SCAL_K(leny, 0, 0, beta, y, blasabs(incy), NULL, 0, NULL, 0); return; } @@ -185,10 +203,10 @@ void CNAME(enum CBLAS_ORDER order, enum CBLAS_TRANSPOSE TransA, blasint m, blasi if (nthreads == 1) { #endif - (sbgemv[(int)trans])(m, n, alpha, a, lda, x, incx, beta, y, incy); + (gemv[(int)trans])(m, n, alpha, a, lda, x, incx, beta, y, incy); #ifdef SMP } else { - (sbgemv_thread[(int)trans])(m, n, alpha, a, lda, x, incx, beta, y, incy, nthreads); + (gemv_thread[(int)trans])(m, n, alpha, a, lda, x, incx, beta, y, incy, nthreads); } #endif diff --git a/interface/scal.c b/interface/scal.c index c6638a62d..4f12df7c0 100644 --- a/interface/scal.c +++ b/interface/scal.c @@ -1,4 +1,5 @@ /*********************************************************************/ +/* Copyright 2025 The OpenBLAS Project. */ /* Copyright 2009, 2010 The University of Texas at Austin. */ /* All rights reserved. */ /* */ @@ -68,7 +69,14 @@ void CNAME(blasint n, FLOAT alpha, FLOAT *x, blasint incx){ if (incx <= 0 || n <= 0) return; - if (alpha == ONE) return; +#ifdef BGEMM + float alpha_float; + SBF16TOS_K(1, &alpha, 1, &alpha_float, 1); +#else + float alpha_float = alpha; +#endif + + if (alpha_float == ONE) return; IDEBUG_START; diff --git a/kernel/CMakeLists.txt b/kernel/CMakeLists.txt index 9434f114e..a2e349d32 100644 --- a/kernel/CMakeLists.txt +++ b/kernel/CMakeLists.txt @@ -1,3 +1,30 @@ +############################################################################### +# Copyright (c) 2025, The OpenBLAS Project +# All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# 3. Neither the name of the OpenBLAS project nor the names of +# its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +############################################################################### include_directories(${PROJECT_SOURCE_DIR}) @@ -94,6 +121,7 @@ function (build_core TARGET_CORE KDIR TSUFFIX KERNEL_DEFINITIONS) # sbdot if (BUILD_BFLOAT16) + GenerateNamedObjects("${KERNELDIR}/${BSCALKERNEL}" "BGEMM" "scal_k" false "" "" false "BFLOAT16") GenerateNamedObjects("${KERNELDIR}/${SBDOTKERNEL}" "SBDOT" "dot_k" false "" "" false "BFLOAT16") GenerateNamedObjects("${KERNELDIR}/${BF16TOKERNEL}" "SINGLE" "f16tos_k" false "" "" false "BFLOAT16") GenerateNamedObjects("${KERNELDIR}/${BF16TOKERNEL}" "DOUBLE" "bf16tod_k" false "" "" false "DOUBLE") @@ -195,6 +223,8 @@ function (build_core TARGET_CORE KDIR TSUFFIX KERNEL_DEFINITIONS) GenerateNamedObjects("${KERNELDIR}/${SGEMVTKERNEL}" "TRANS" "gemv_t" false "" "" false "SINGLE") endif () if (BUILD_BFLOAT16) + GenerateNamedObjects("${KERNELDIR}/${BGEMVNKERNEL}" "BGEMM" "gemv_n" false "" "" false "BFLOAT16") + GenerateNamedObjects("${KERNELDIR}/${BGEMVTKERNEL}" "BGEMM" "gemv_t" false "" "" false "BFLOAT16") GenerateNamedObjects("${KERNELDIR}/${SBGEMVNKERNEL}" "" "gemv_n" false "" "" false "BFLOAT16") GenerateNamedObjects("${KERNELDIR}/${SBGEMVTKERNEL}" "" "gemv_t" false "" "" false "BFLOAT16") endif () @@ -211,6 +241,9 @@ function (build_core TARGET_CORE KDIR TSUFFIX KERNEL_DEFINITIONS) if (X86_64 OR ARM64) set(USE_DIRECT_SGEMM true) endif() + if (UC_TARGET_CORE MATCHES ARMV9SME) + set (HAVE_SME true) + endif () if (USE_DIRECT_SGEMM) # if (NOT DEFINED SGEMMDIRECTKERNEL) @@ -222,9 +255,11 @@ function (build_core TARGET_CORE KDIR TSUFFIX KERNEL_DEFINITIONS) GenerateNamedObjects("${KERNELDIR}/${SGEMMDIRECTPERFORMANT}" "" "gemm_direct_performant" false "" "" false SINGLE) elseif (ARM64) set (SGEMMDIRECTKERNEL sgemm_direct_arm64_sme1.c) + set (SGEMMDIRECTKERNEL_ALPHA_BETA sgemm_direct_alpha_beta_arm64_sme1.c) set (SGEMMDIRECTSMEKERNEL sgemm_direct_sme1.S) set (SGEMMDIRECTPREKERNEL sgemm_direct_sme1_preprocess.S) GenerateNamedObjects("${KERNELDIR}/${SGEMMDIRECTKERNEL}" "" "gemm_direct" false "" "" false SINGLE) + GenerateNamedObjects("${KERNELDIR}/${SGEMMDIRECTKERNEL_ALPHA_BETA}" "" "gemm_direct_alpha_beta" false "" "" false SINGLE) if (HAVE_SME) GenerateNamedObjects("${KERNELDIR}/${SGEMMDIRECTSMEKERNEL}" "" "gemm_direct_sme1" false "" "" false SINGLE) GenerateNamedObjects("${KERNELDIR}/${SGEMMDIRECTPREKERNEL}" "" "gemm_direct_sme1_preprocess" false "" "" false SINGLE) @@ -336,6 +371,20 @@ function (build_core TARGET_CORE KDIR TSUFFIX KERNEL_DEFINITIONS) endif () if (BUILD_BFLOAT16) + if (BGEMMINCOPY) + GenerateNamedObjects("${KERNELDIR}/${BGEMMINCOPY}" "BGEMM" "${BGEMMINCOPYOBJ}" false "" "" true "BFLOAT16") + endif () + if (BGEMMITCOPY) + GenerateNamedObjects("${KERNELDIR}/${BGEMMITCOPY}" "BGEMM" "${BGEMMITCOPYOBJ}" false "" "" true "BFLOAT16") + endif () + if (BGEMMONCOPY) + GenerateNamedObjects("${KERNELDIR}/${BGEMMONCOPY}" "BGEMM" "${BGEMMONCOPYOBJ}" false "" "" true "BFLOAT16") + endif () + if (BGEMMOTCOPY) + GenerateNamedObjects("${KERNELDIR}/${BGEMMOTCOPY}" "BGEMM" "${BGEMMOTCOPYOBJ}" false "" "" true "BFLOAT16") + endif () + GenerateNamedObjects("${KERNELDIR}/${BGEMMKERNEL}" "BGEMM" "gemm_kernel" false "" "" false "BFLOAT16") + GenerateNamedObjects("${KERNELDIR}/${BGEMM_BETA}" "BGEMM" "gemm_beta" false "" "" false "BFLOAT16") if (SBGEMMINCOPY) GenerateNamedObjects("${KERNELDIR}/${SBGEMMINCOPY}" "" "${SBGEMMINCOPYOBJ}" false "" "" true "BFLOAT16") endif () diff --git a/kernel/Makefile.L1 b/kernel/Makefile.L1 index 0fc672094..221cc5127 100644 --- a/kernel/Makefile.L1 +++ b/kernel/Makefile.L1 @@ -1,3 +1,31 @@ +############################################################################### +# Copyright (c) 2025 The OpenBLAS Project +# All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# 3. Neither the name of the OpenBLAS project nor the names of +# its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +############################################################################### + FMAFLAG= ifndef OLDGCC ifdef HAVE_FMA3 @@ -271,6 +299,10 @@ XDOTKERNEL = zdot.S endif ifeq ($(BUILD_BFLOAT16),1) +ifndef BSCALKERNEL +BSCALKERNEL = ../generic/scal.c +endif + ifndef SBDOTKERNEL SBDOTKERNEL = ../x86_64/sbdot.c endif @@ -551,6 +583,8 @@ XBLASOBJS += \ xscal_k$(TSUFFIX).$(SUFFIX) xswap_k$(TSUFFIX).$(SUFFIX) xsum_k$(TSUFFIX).$(SUFFIX) ifeq ($(BUILD_BFLOAT16),1) +BBLASOBJS += \ + bscal_k$(TSUFFIX).$(SUFFIX) SBBLASOBJS += \ sbdot_k$(TSUFFIX).$(SUFFIX) SBEXTOBJS += \ @@ -778,6 +812,8 @@ $(KDIR)qdot_k$(TSUFFIX).$(SUFFIX) $(KDIR)qdot_k$(TPSUFFIX).$(PSUFFIX) : $(KERNEL $(CC) -c $(CFLAGS) -UCOMPLEX -DXDOUBLE $< -o $@ ifeq ($(BUILD_BFLOAT16),1) +$(KDIR)bscal_k$(TSUFFIX).$(SUFFIX) $(KDIR)bscal_k$(TPSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(BSCALKERNEL) + $(CC) -c $(CFLAGS) -UCOMPLEX -UDOUBLE $< -o $@ $(KDIR)sbdot_k$(TSUFFIX).$(SUFFIX) $(KDIR)sbdot_k$(TPSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SBDOTKERNEL) $(CC) -c $(CFLAGS) -UCOMPLEX $< -o $@ $(KDIR)sbstobf16_k$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(TOBF16KERNEL) diff --git a/kernel/Makefile.L2 b/kernel/Makefile.L2 index 0332ba722..a9fcf9225 100644 --- a/kernel/Makefile.L2 +++ b/kernel/Makefile.L2 @@ -1,3 +1,31 @@ +############################################################################### +# Copyright (c) 2025 The OpenBLAS Project +# All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# 3. Neither the name of the OpenBLAS project nor the names of +# its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +############################################################################### + FMAFLAG= ifndef OLDGCC ifdef HAVE_FMA3 @@ -56,6 +84,14 @@ XGEMVTKERNEL = zgemv_t.S endif ifeq ($(BUILD_BFLOAT16),1) +ifndef BGEMVNKERNEL +BGEMVNKERNEL = ../generic/gemv_n.c +endif + +ifndef BGEMVTKERNEL +BGEMVTKERNEL = ../generic/gemv_t.c +endif + ifndef SBGEMVNKERNEL SBGEMVNKERNEL = ../x86_64/sbgemv_n.c endif @@ -255,6 +291,9 @@ XBLASOBJS += \ xgeru_k$(TSUFFIX).$(SUFFIX) xgerc_k$(TSUFFIX).$(SUFFIX) xgerv_k$(TSUFFIX).$(SUFFIX) xgerd_k$(TSUFFIX).$(SUFFIX) ifeq ($(BUILD_BFLOAT16),1) +BBLASOBJS += \ + bgemv_n$(TSUFFIX).$(SUFFIX) \ + bgemv_t$(TSUFFIX).$(SUFFIX) SBBLASOBJS += \ sbgemv_n$(TSUFFIX).$(SUFFIX) \ sbgemv_t$(TSUFFIX).$(SUFFIX) @@ -513,5 +552,9 @@ $(KDIR)sbgemv_n$(TSUFFIX).$(SUFFIX) $(KDIR)sbgemv_n$(TPSUFFIX).$(PSUFFIX) : $(KE $(CC) -c $(CFLAGS) -UCOMPLEX $< -o $@ $(KDIR)sbgemv_t$(TSUFFIX).$(SUFFIX) $(KDIR)sbgemv_t$(TPSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SBGEMVTKERNEL) $(CC) -c $(CFLAGS) -UCOMPLEX $< -o $@ +$(KDIR)bgemv_n$(TSUFFIX).$(SUFFIX) $(KDIR)bgemv_n$(TPSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(BGEMVNKERNEL) + $(CC) -c $(CFLAGS) -DBGEMM -UCOMPLEX $< -o $@ +$(KDIR)bgemv_t$(TSUFFIX).$(SUFFIX) $(KDIR)bgemv_t$(TPSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(BGEMVTKERNEL) + $(CC) -c $(CFLAGS) -DBGEMM -UCOMPLEX $< -o $@ endif diff --git a/kernel/Makefile.L3 b/kernel/Makefile.L3 index 6afb49a77..79c88d76c 100644 --- a/kernel/Makefile.L3 +++ b/kernel/Makefile.L3 @@ -1,3 +1,30 @@ +############################################################################### +# Copyright (c) 2025, The OpenBLAS Project +# All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# 3. Neither the name of the OpenBLAS project nor the names of +# its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +############################################################################### USE_GEMM3M = 0 OS := $(shell uname) @@ -105,11 +132,29 @@ ifeq ($(TARGET_CORE), ARMV9SME) HAVE_SME = 1 endif SGEMMDIRECTKERNEL = sgemm_direct_arm64_sme1.c +SGEMMDIRECTKERNEL_ALPHA_BETA = sgemm_direct_alpha_beta_arm64_sme1.c endif endif endif ifeq ($(BUILD_BFLOAT16), 1) +ifndef BGEMMKERNEL +BGEMM_BETA = ../generic/gemm_beta.c +BGEMMKERNEL = ../generic/gemmkernel_2x2.c +BGEMMINCOPY = ../generic/gemm_ncopy_2.c +BGEMMITCOPY = ../generic/gemm_tcopy_2.c +BGEMMONCOPY = ../generic/gemm_ncopy_2.c +BGEMMOTCOPY = ../generic/gemm_tcopy_2.c +BGEMMINCOPYOBJ = bgemm_incopy$(TSUFFIX).$(SUFFIX) +BGEMMITCOPYOBJ = bgemm_itcopy$(TSUFFIX).$(SUFFIX) +BGEMMONCOPYOBJ = bgemm_oncopy$(TSUFFIX).$(SUFFIX) +BGEMMOTCOPYOBJ = bgemm_otcopy$(TSUFFIX).$(SUFFIX) +endif +BKERNELOBJS += \ + bgemm_kernel$(TSUFFIX).$(SUFFIX) \ + $(BGEMMINCOPYOBJ) $(BGEMMITCOPYOBJ) \ + $(BGEMMONCOPYOBJ) $(BGEMMOTCOPYOBJ) + ifndef SBGEMMKERNEL SBGEMM_BETA = ../generic/gemm_beta.c SBGEMMKERNEL = ../generic/gemmkernel_2x2.c @@ -164,7 +209,8 @@ SKERNELOBJS += \ endif ifeq ($(ARCH), arm64) SKERNELOBJS += \ - sgemm_direct$(TSUFFIX).$(SUFFIX) + sgemm_direct$(TSUFFIX).$(SUFFIX) \ + sgemm_direct_alpha_beta$(TSUFFIX).$(SUFFIX) ifdef HAVE_SME SKERNELOBJS += \ sgemm_direct_sme1$(TSUFFIX).$(SUFFIX) \ @@ -210,6 +256,7 @@ XKERNELOBJS += \ $(XGEMMONCOPYOBJ) $(XGEMMOTCOPYOBJ) ifeq ($(BUILD_BFLOAT16),1) +BBLASOBJS += $(BKERNELOBJS) SBBLASOBJS += $(SBKERNELOBJS) endif ifeq ($(BUILD_HFLOAT16),1) @@ -223,6 +270,7 @@ ZBLASOBJS += $(ZKERNELOBJS) XBLASOBJS += $(XKERNELOBJS) ifeq ($(BUILD_BFLOAT16),1) +BBLASOBJS += bgemm_beta$(TSUFFIX).$(SUFFIX) SBBLASOBJS += sbgemm_beta$(TSUFFIX).$(SUFFIX) endif ifeq ($(BUILD_HFLOAT16),1) @@ -628,6 +676,10 @@ ZBLASOBJS += \ endif ifeq ($(BUILD_BFLOAT16), 1) +BGEMMINCOPYOBJ_P = $(BGEMMINCOPYOBJ:.$(SUFFIX)=.$(PSUFFIX)) +BGEMMITCOPYOBJ_P = $(BGEMMITCOPYOBJ:.$(SUFFIX)=.$(PSUFFIX)) +BGEMMONCOPYOBJ_P = $(BGEMMONCOPYOBJ:.$(SUFFIX)=.$(PSUFFIX)) +BGEMMOTCOPYOBJ_P = $(BGEMMOTCOPYOBJ:.$(SUFFIX)=.$(PSUFFIX)) SBGEMMINCOPYOBJ_P = $(SBGEMMINCOPYOBJ:.$(SUFFIX)=.$(PSUFFIX)) SBGEMMITCOPYOBJ_P = $(SBGEMMITCOPYOBJ:.$(SUFFIX)=.$(PSUFFIX)) SBGEMMONCOPYOBJ_P = $(SBGEMMONCOPYOBJ:.$(SUFFIX)=.$(PSUFFIX)) @@ -667,6 +719,8 @@ XGEMMONCOPYOBJ_P = $(XGEMMONCOPYOBJ:.$(SUFFIX)=.$(PSUFFIX)) XGEMMOTCOPYOBJ_P = $(XGEMMOTCOPYOBJ:.$(SUFFIX)=.$(PSUFFIX)) ifeq ($(BUILD_BFLOAT16),1) +$(KDIR)bgemm_beta$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(BGEMM_BETA) + $(CC) $(CFLAGS) -c -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX $< -o $@ $(KDIR)sbgemm_beta$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SBGEMM_BETA) $(CC) $(CFLAGS) -c -DBFLOAT16 -UDOUBLE -UCOMPLEX $< -o $@ endif @@ -698,9 +752,22 @@ ifeq ($(ARCH), E2K) USE_TRMM = 1 endif - ifeq ($(BUILD_BFLOAT16), 1) +$(KDIR)$(BGEMMONCOPYOBJ) : $(KERNELDIR)/$(BGEMMONCOPY) + $(CC) $(CFLAGS) -c -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX $< -o $@ + +$(KDIR)$(BGEMMOTCOPYOBJ) : $(KERNELDIR)/$(BGEMMOTCOPY) + $(CC) $(CFLAGS) -c -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX $< -o $@ + +ifneq ($(BGEMM_UNROLL_M), $(BGEMM_UNROLL_N)) +$(KDIR)$(BGEMMINCOPYOBJ) : $(KERNELDIR)/$(BGEMMINCOPY) + $(CC) $(CFLAGS) -c -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX $< -o $@ + +$(KDIR)$(BGEMMITCOPYOBJ) : $(KERNELDIR)/$(BGEMMITCOPY) + $(CC) $(CFLAGS) -c -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX $< -o $@ +endif + $(KDIR)$(SBGEMMONCOPYOBJ) : $(KERNELDIR)/$(SBGEMMONCOPY) $(CC) $(CFLAGS) -c -DBFLOAT16 -UDOUBLE -UCOMPLEX $< -o $@ @@ -904,6 +971,8 @@ endif ifeq ($(ARCH), arm64) $(KDIR)sgemm_direct$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMMDIRECTKERNEL) $(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX $< -o $@ +$(KDIR)sgemm_direct_alpha_beta$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMMDIRECTKERNEL_ALPHA_BETA) + $(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX $< -o $@ ifdef HAVE_SME $(KDIR)sgemm_direct_sme1$(TSUFFIX).$(SUFFIX) : $(CC) $(CFLAGS) -c $(KERNELDIR)/sgemm_direct_sme1.S -UDOUBLE -UCOMPLEX -o $@ @@ -914,7 +983,8 @@ endif endif ifeq ($(BUILD_BFLOAT16), 1) - +$(KDIR)bgemm_kernel$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(BGEMMKERNEL) + $(CC) $(CFLAGS) -c -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX $< -o $@ $(KDIR)sbgemm_kernel$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SBGEMMKERNEL) $(SBGEMMDEPEND) $(CC) $(CFLAGS) -c -DBFLOAT16 -UDOUBLE -UCOMPLEX $< -o $@ endif @@ -2908,6 +2978,8 @@ $(KDIR)sgemm_beta$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SGEMM_BETA) $(CC) $(PFLAGS) -c -UDOUBLE -UCOMPLEX $< -o $@ ifeq ($(BUILD_BFLOAT16),1) +$(KDIR)bgemm_beta$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(BGEMM_BETA) + $(CC) $(PFLAGS) -c -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX $< -o $@ $(KDIR)sbgemm_beta$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SBGEMM_BETA) $(CC) $(PFLAGS) -c -DBFLOAT16 -UDOUBLE -UCOMPLEX $< -o $@ endif @@ -2934,6 +3006,20 @@ $(KDIR)xgemm_beta$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(XGEMM_BETA) ifeq ($(BUILD_BFLOAT16), 1) +$(BGEMMONCOPYOBJ_P) : $(KERNELDIR)/$(BGEMMONCOPY) + $(CC) $(PFLAGS) -c -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX $< -o $@ + +$(BGEMMOTCOPYOBJ_P) : $(KERNELDIR)/$(BGEMMOTCOPY) + $(CC) $(PFLAGS) -c -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX $< -o $@ + +ifneq ($(BGEMM_UNROLL_M), $(BGEMM_UNROLL_N)) +$(BGEMMINCOPYOBJ_P) : $(KERNELDIR)/$(BGEMMINCOPY) + $(CC) $(PFLAGS) -c -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX $< -o $@ + +$(BGEMMITCOPYOBJ_P) : $(KERNELDIR)/$(BGEMMITCOPY) + $(CC) $(PFLAGS) -c -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX $< -o $@ +endif + $(SBGEMMONCOPYOBJ_P) : $(KERNELDIR)/$(SBGEMMONCOPY) $(CC) $(PFLAGS) -c -DBFLOAT16 -UDOUBLE -UCOMPLEX $< -o $@ @@ -2946,7 +3032,6 @@ $(SBGEMMINCOPYOBJ_P) : $(KERNELDIR)/$(SBGEMMINCOPY) $(SBGEMMITCOPYOBJ_P) : $(KERNELDIR)/$(SBGEMMITCOPY) $(CC) $(PFLAGS) -c -DBFLOAT16 -UDOUBLE -UCOMPLEX $< -o $@ - endif endif @@ -3073,6 +3158,8 @@ endif ifeq ($(BUILD_BFLOAT16), 1) +$(KDIR)bgemm_kernel$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(BGEMMKERNEL) $(BGEMMDEPEND) + $(CC) $(PFLAGS) -c -DBFLOAT16 -DBGEMM -UDOUBLE -UCOMPLEX $< -o $@ $(KDIR)sbgemm_kernel$(TSUFFIX).$(PSUFFIX) : $(KERNELDIR)/$(SBGEMMKERNEL) $(SBGEMMDEPEND) $(CC) $(PFLAGS) -c -DBFLOAT16 -UDOUBLE -UCOMPLEX $< -o $@ endif diff --git a/kernel/arm64/KERNEL.A64FX b/kernel/arm64/KERNEL.A64FX index 3d68271da..e5e6a9d69 100644 --- a/kernel/arm64/KERNEL.A64FX +++ b/kernel/arm64/KERNEL.A64FX @@ -4,3 +4,6 @@ SGEMVNKERNEL = gemv_n_sve_v4x3.c DGEMVNKERNEL = gemv_n_sve_v4x3.c SGEMVTKERNEL = gemv_t_sve_v4x3.c DGEMVTKERNEL = gemv_t_sve_v4x3.c + +DDOTKERNEL = dot_sve_v8.c +SDOTKERNEL = dot_sve_v8.c diff --git a/kernel/arm64/KERNEL.NEOVERSEV1 b/kernel/arm64/KERNEL.NEOVERSEV1 index 3e622bcbf..8bc0f35e5 100644 --- a/kernel/arm64/KERNEL.NEOVERSEV1 +++ b/kernel/arm64/KERNEL.NEOVERSEV1 @@ -1,3 +1,31 @@ +############################################################################### +# Copyright (c) 2025, The OpenBLAS Project +# All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# 3. Neither the name of the OpenBLAS project nor the names of +# its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +############################################################################### + include $(KERNELDIR)/KERNEL.ARMV8SVE SGEMVNKERNEL = gemv_n_sve_v1x3.c @@ -5,6 +33,19 @@ DGEMVNKERNEL = gemv_n_sve_v1x3.c SGEMVTKERNEL = gemv_t_sve_v1x3.c DGEMVTKERNEL = gemv_t_sve_v1x3.c ifeq ($(BUILD_BFLOAT16), 1) +BGEMM_BETA = bgemm_beta_neon.c +BGEMMKERNEL = bgemm_kernel_$(BGEMM_UNROLL_M)x$(BGEMM_UNROLL_N)_neoversev1.c +ifneq ($(BGEMM_UNROLL_M), $(BGEMM_UNROLL_N)) +BGEMMINCOPY = sbgemm_ncopy_$(SBGEMM_UNROLL_M)_neoversev1.c +BGEMMITCOPY = sbgemm_tcopy_$(SBGEMM_UNROLL_M)_neoversev1.c +BGEMMINCOPYOBJ = bgemm_incopy$(TSUFFIX).$(SUFFIX) +BGEMMITCOPYOBJ = bgemm_itcopy$(TSUFFIX).$(SUFFIX) +endif +BGEMMONCOPY = sbgemm_ncopy_$(BGEMM_UNROLL_N)_neoversev1.c +BGEMMOTCOPY = sbgemm_tcopy_$(BGEMM_UNROLL_N)_neoversev1.c +BGEMMONCOPYOBJ = bgemm_oncopy$(TSUFFIX).$(SUFFIX) +BGEMMOTCOPYOBJ = bgemm_otcopy$(TSUFFIX).$(SUFFIX) + SBGEMM_BETA = sbgemm_beta_neoversev1.c SBGEMMKERNEL = sbgemm_kernel_$(SBGEMM_UNROLL_M)x$(SBGEMM_UNROLL_N)_neoversev1.c ifneq ($(SBGEMM_UNROLL_M), $(SBGEMM_UNROLL_N)) @@ -21,4 +62,4 @@ SBGEMMOTCOPYOBJ = sbgemm_otcopy$(TSUFFIX).$(SUFFIX) SBGEMVNKERNEL = sbgemv_n_neon.c SBGEMVTKERNEL = sbgemv_t_bfdot.c -endif \ No newline at end of file +endif diff --git a/kernel/arm64/bgemm_beta_neon.c b/kernel/arm64/bgemm_beta_neon.c new file mode 100644 index 000000000..603377f8f --- /dev/null +++ b/kernel/arm64/bgemm_beta_neon.c @@ -0,0 +1,107 @@ +/*************************************************************************** + * Copyright (c) 2025, The OpenBLAS Project + * All rights reserved. + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of the OpenBLAS project nor the names of + * its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * *****************************************************************************/ + +#include "common.h" + +#include + +int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT beta_in, IFLOAT *dummy2, + BLASLONG dummy3, IFLOAT *dummy4, BLASLONG dummy5, FLOAT *c, + BLASLONG ldc) { + BLASLONG i, j; + BLASLONG chunk, remain; + + bfloat16_t *ptr_c, *ptr_c0; + + bfloat16x8_t x0, z0; + float32x4_t y0, y1; + + float x; + bfloat16_t z; + + bfloat16_t zero_bf16 = vcvth_bf16_f32(0.0f); + bfloat16x8_t zeros = vdupq_n_bf16(zero_bf16); + + bfloat16_t beta_bf16; + memcpy(&beta_bf16, &beta_in, sizeof(bfloat16_t)); + float beta = vcvtah_f32_bf16(beta_bf16); + float32x4_t beta_neon = vdupq_n_f32(beta); + + ptr_c = (bfloat16_t *)c; + + chunk = m >> 3; + remain = m & 7; + + if (beta == 0.0f){ + for (j = 0; j < n; j ++){ + ptr_c0 = ptr_c; + ptr_c += ldc; + + for (i = 0; i < chunk; i ++){ + vst1q_bf16(ptr_c0, zeros); + ptr_c0 += 8; + } + + for (i = 0; i < remain; i ++){ + ptr_c0[0] = zero_bf16; + ptr_c0 ++; + } + } + } else { + for (j = 0; j < n; j ++){ + ptr_c0 = ptr_c; + ptr_c += ldc; + + for (i = 0; i < chunk; i ++){ + x0 = vld1q_bf16(ptr_c0); + + y0 = vcvtq_low_f32_bf16(x0); + y1 = vcvtq_high_f32_bf16(x0); + + y0 = vmulq_f32(y0, beta_neon); + y1 = vmulq_f32(y1, beta_neon); + + z0 = vcvtq_low_bf16_f32(y0); + z0 = vcvtq_high_bf16_f32(z0, y1); + + vst1q_bf16(ptr_c0, z0); + + ptr_c0 += 8; + } + + for (i = 0; i < remain; i ++){ + x = vcvtah_f32_bf16(ptr_c0[0]); + z = vcvth_bf16_f32(x * beta); + + ptr_c0[0] = z; + ptr_c0 ++; + } + } + } + return 0; +}; diff --git a/kernel/arm64/bgemm_kernel_4x4_neoversev1.c b/kernel/arm64/bgemm_kernel_4x4_neoversev1.c new file mode 100644 index 000000000..7af31bb2c --- /dev/null +++ b/kernel/arm64/bgemm_kernel_4x4_neoversev1.c @@ -0,0 +1,50 @@ +/*************************************************************************** + * Copyright (c) 2025, The OpenBLAS Project + * All rights reserved. + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of the OpenBLAS project nor the names of + * its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * *****************************************************************************/ + +#include + +#include "common.h" + +#define ALPHA_ONE +#include "bgemm_kernel_4x4_neoversev1_impl.c" +#undef ALPHA_ONE +#undef UPDATE_C +#include "bgemm_kernel_4x4_neoversev1_impl.c" + +int CNAME(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT *A, IFLOAT *B, + FLOAT *C, BLASLONG ldc) { + bfloat16_t alpha_bf16; + memcpy(&alpha_bf16, &alpha, sizeof(bfloat16_t)); + float alpha_f32 = vcvtah_f32_bf16(alpha_bf16); + + if (alpha_f32 == 1.0f) + return bgemm_kernel_neoversev1_alpha_one(m, n, k, alpha, A, B, C, ldc); + else + return bgemm_kernel_neoversev1_alpha(m, n, k, alpha, A, B, C, ldc); + return 0; +} diff --git a/kernel/arm64/bgemm_kernel_4x4_neoversev1_impl.c b/kernel/arm64/bgemm_kernel_4x4_neoversev1_impl.c new file mode 100644 index 000000000..2477da9c0 --- /dev/null +++ b/kernel/arm64/bgemm_kernel_4x4_neoversev1_impl.c @@ -0,0 +1,429 @@ +/*************************************************************************** + * Copyright (c) 2025, The OpenBLAS Project + * All rights reserved. + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of the OpenBLAS project nor the names of + * its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * *****************************************************************************/ + +#include + +#include "common.h" + +#define INIT_C(M, N) mc##M##N = svdup_f32(0); + +#define MATMUL(M, N) mc##M##N = svbfmmla(mc##M##N, ma##M, mb##N); + +#define INIT_C_4x4 \ + do { \ + INIT_C(0, 0); \ + INIT_C(0, 1); \ + INIT_C(1, 0); \ + INIT_C(1, 1); \ + } while (0); + +#ifdef ALPHA_ONE +#define UPDATE_C(PG16, PG32, PTR, TMP32, TMP16, SRC32) \ + do { \ + TMP32 = svreinterpret_f32_u32(svld1uh_u32((PG16), (uint16_t*)PTR)); \ + TMP32 = svadd_z((PG32), SRC32, TMP32); \ + TMP16 = svcvt_bf16_f32_z((PG32), TMP32); \ + TMP16 = svuzp1_bf16(TMP16, TMP16); \ + svst1_bf16((PG16), (PTR), TMP16); \ + } while (0) +#else +#define UPDATE_C(PG16, PG32, PTR, TMP32, TMP16, SRC32) \ + do { \ + TMP32 = svreinterpret_f32_u32(svld1uh_u32((PG16), (uint16_t*)PTR)); \ + TMP32 = svmad_z((PG32), svalpha, SRC32, TMP32); \ + TMP16 = svcvt_bf16_f32_z((PG32), TMP32); \ + TMP16 = svuzp1_bf16(TMP16, TMP16); \ + svst1_bf16((PG16), (PTR), TMP16); \ + } while (0) +#endif + +#define ZIP_EVEN_ELEMENTS(PG, mc0, mc1, tmp, vc) \ + do { \ + (tmp) = svuzp1_f32((mc0), (mc1)); \ + (vc) = svcompact_f32((PG), (tmp)); \ + } while (0) + +#define ZIP_ODD_ELEMENTS(PG, mc0, mc1, tmp, vc) \ + do { \ + (tmp) = svuzp2_f32((mc0), (mc1)); \ + (vc) = svcompact_f32((PG), (tmp)); \ + } while (0) + +#define ACCUMULATE_LAST4_TO_FIRST4(M, N, TMP) \ + do { \ + TMP = svext_f32(mc##M##N, mc##M##N, 4); \ + mc##M##N = svadd_f32_z(svptrue_b32(), mc##M##N, (TMP)); \ + } while (0) + +#ifdef ALPHA_ONE +int bgemm_kernel_neoversev1_alpha_one(BLASLONG m, BLASLONG n, BLASLONG k, + FLOAT alpha, IFLOAT *A, IFLOAT *B, + FLOAT *C, BLASLONG ldc) +#else +int bgemm_kernel_neoversev1_alpha(BLASLONG m, BLASLONG n, BLASLONG k, + FLOAT alpha, IFLOAT *A, IFLOAT *B, FLOAT *C, + BLASLONG ldc) +#endif +{ + BLASLONG pad_k = (k + 7) & ~7; + svbfloat16_t ma0, ma1, mb0, mb1; + svfloat32_t mc00, mc01, mc10, mc11, vc0, vc1, vc2, vc3; + svfloat32_t tmp; +#ifndef ALPHA_ONE + bfloat16_t alpha_bf16; + memcpy(&alpha_bf16, &alpha, sizeof(bfloat16_t)); + svfloat32_t svalpha = svdup_f32(vcvtah_f32_bf16(alpha_bf16)); +#endif + + svbool_t pg16_all = svptrue_b16(); + + svbool_t pg32_first_1 = svwhilelt_b32(0, 1); + svbool_t pg32_first_2 = svwhilelt_b32(0, 2); + svbool_t pg32_first_4 = svwhilelt_b32(0, 4); + + svbool_t pg16_first_1 = svwhilelt_b16(0, 1); + svbool_t pg16_first_2 = svwhilelt_b16(0, 2); + svbool_t pg16_first_4 = svwhilelt_b16(0, 4); + + svbool_t pg32_select_first_2_per_quadword = svdupq_b32(1, 1, 0, 0); + + bfloat16_t *ptr_a = (bfloat16_t *)A; + bfloat16_t *ptr_b = (bfloat16_t *)B; + bfloat16_t *ptr_c = (bfloat16_t *)C; + + bfloat16_t *ptr_a0; + bfloat16_t *ptr_b0; + bfloat16_t *ptr_c0, *ptr_c1, *ptr_c2, *ptr_c3; + + svfloat32_t tmp32; + svbfloat16_t tmp16; + + for (BLASLONG j = 0; j < n / 4; j++) { + ptr_c0 = ptr_c; + ptr_c1 = ptr_c0 + ldc; + ptr_c2 = ptr_c1 + ldc; + ptr_c3 = ptr_c2 + ldc; + ptr_c += 4 * ldc; + ptr_a = (bfloat16_t *)A; + + for (BLASLONG i = 0; i < m / 4; i++) { + ptr_a0 = ptr_a; + ptr_a += 4 * pad_k; + + ptr_b0 = ptr_b; + + INIT_C_4x4; + + for (BLASLONG p = 0; p < pad_k; p += 8) { + ma0 = svld1_bf16(pg16_all, ptr_a0); + ma1 = svld1_bf16(pg16_all, ptr_a0 + 16); + + mb0 = svld1_bf16(pg16_all, ptr_b0); + mb1 = svld1_bf16(pg16_all, ptr_b0 + 16); + + MATMUL(0, 0); + MATMUL(0, 1); + MATMUL(1, 0); + MATMUL(1, 1); + + ptr_a0 += 32; + ptr_b0 += 32; + } + + ACCUMULATE_LAST4_TO_FIRST4(0, 0, tmp); + ACCUMULATE_LAST4_TO_FIRST4(0, 1, tmp); + ACCUMULATE_LAST4_TO_FIRST4(1, 0, tmp); + ACCUMULATE_LAST4_TO_FIRST4(1, 1, tmp); + + ZIP_EVEN_ELEMENTS(pg32_select_first_2_per_quadword, mc00, mc10, tmp, vc0); + ZIP_ODD_ELEMENTS(pg32_select_first_2_per_quadword, mc00, mc10, tmp, vc1); + + ZIP_EVEN_ELEMENTS(pg32_select_first_2_per_quadword, mc01, mc11, tmp, vc2); + ZIP_ODD_ELEMENTS(pg32_select_first_2_per_quadword, mc01, mc11, tmp, vc3); + + UPDATE_C(pg16_first_4, pg32_first_4, ptr_c0, tmp32, tmp16, vc0); + UPDATE_C(pg16_first_4, pg32_first_4, ptr_c1, tmp32, tmp16, vc1); + UPDATE_C(pg16_first_4, pg32_first_4, ptr_c2, tmp32, tmp16, vc2); + UPDATE_C(pg16_first_4, pg32_first_4, ptr_c3, tmp32, tmp16, vc3); + + ptr_c0 += 4; + ptr_c1 += 4; + ptr_c2 += 4; + ptr_c3 += 4; + } + + if (m & 2) { + ptr_a0 = ptr_a; + ptr_a += 2 * pad_k; + + ptr_b0 = ptr_b; + INIT_C(0, 0); + INIT_C(0, 1); + for (BLASLONG p = 0; p < pad_k; p += 8) { + ma0 = svld1_bf16(pg16_all, ptr_a0); + mb0 = svld1_bf16(pg16_all, ptr_b0); + mb1 = svld1_bf16(pg16_all, ptr_b0 + 16); + + MATMUL(0, 0); + MATMUL(0, 1); + + ptr_a0 += 16; + ptr_b0 += 32; + } + + ACCUMULATE_LAST4_TO_FIRST4(0, 0, tmp); + ACCUMULATE_LAST4_TO_FIRST4(0, 1, tmp); + + vc0 = svuzp1(mc00, mc00); + vc1 = svuzp2(mc00, mc00); + vc2 = svuzp1(mc01, mc01); + vc3 = svuzp2(mc01, mc01); + + UPDATE_C(pg16_first_2, pg32_first_2, ptr_c0, tmp32, tmp16, vc0); + UPDATE_C(pg16_first_2, pg32_first_2, ptr_c1, tmp32, tmp16, vc1); + UPDATE_C(pg16_first_2, pg32_first_2, ptr_c2, tmp32, tmp16, vc2); + UPDATE_C(pg16_first_2, pg32_first_2, ptr_c3, tmp32, tmp16, vc3); + + ptr_c0 += 2; + ptr_c1 += 2; + ptr_c2 += 2; + ptr_c3 += 2; + } + + if (m & 1) { + ptr_a0 = ptr_a; + ptr_b0 = ptr_b; + + INIT_C(0, 0); + INIT_C(0, 1); + for (BLASLONG p = 0; p < pad_k; p += 8) { + ma0 = svld1_bf16(pg16_all, ptr_a0); + mb0 = svld1_bf16(pg16_all, ptr_b0); + mb1 = svld1_bf16(pg16_all, ptr_b0 + 16); + + MATMUL(0, 0); + MATMUL(0, 1); + + ptr_a0 += 16; + ptr_b0 += 32; + } + + ACCUMULATE_LAST4_TO_FIRST4(0, 0, tmp); + ACCUMULATE_LAST4_TO_FIRST4(0, 1, tmp); + + // use compact is more straightforward + vc1 = svuzp2(mc00, mc00); + vc3 = svuzp2(mc01, mc01); + + UPDATE_C(pg16_first_1, pg32_first_1, ptr_c0, tmp32, tmp16, mc00); + UPDATE_C(pg16_first_1, pg32_first_1, ptr_c1, tmp32, tmp16, vc1); + UPDATE_C(pg16_first_1, pg32_first_1, ptr_c2, tmp32, tmp16, mc01); + UPDATE_C(pg16_first_1, pg32_first_1, ptr_c3, tmp32, tmp16, vc3); + } + + ptr_b += 4 * pad_k; + } + + if (n & 2) { + ptr_c0 = ptr_c; + ptr_c1 = ptr_c0 + ldc; + ptr_c += 2 * ldc; + ptr_a = (bfloat16_t *)A; + + for (BLASLONG i = 0; i < m / 4; i++) { + ptr_a0 = ptr_a; + ptr_a += 4 * pad_k; + + ptr_b0 = ptr_b; + + INIT_C(0, 0); + INIT_C(1, 0); + + for (BLASLONG p = 0; p < pad_k; p += 8) { + ma0 = svld1_bf16(pg16_all, ptr_a0); + ma1 = svld1_bf16(pg16_all, ptr_a0 + 16); + + mb0 = svld1_bf16(pg16_all, ptr_b0); + + MATMUL(0, 0); + MATMUL(1, 0); + + ptr_a0 += 32; + ptr_b0 += 16; + } + + ACCUMULATE_LAST4_TO_FIRST4(0, 0, tmp); + ACCUMULATE_LAST4_TO_FIRST4(1, 0, tmp); + + ZIP_EVEN_ELEMENTS(pg32_select_first_2_per_quadword, mc00, mc10, tmp, vc0); + ZIP_ODD_ELEMENTS(pg32_select_first_2_per_quadword, mc00, mc10, tmp, vc2); + + UPDATE_C(pg16_first_4, pg32_first_4, ptr_c0, tmp32, tmp16, vc0); + UPDATE_C(pg16_first_4, pg32_first_4, ptr_c1, tmp32, tmp16, vc2); + + ptr_c0 += 4; + ptr_c1 += 4; + } + + if (m & 2) { + ptr_a0 = ptr_a; + ptr_a += 2 * pad_k; + ptr_b0 = ptr_b; + + INIT_C(0, 0); + + for (BLASLONG p = 0; p < pad_k; p += 8) { + ma0 = svld1_bf16(pg16_all, ptr_a0); + mb0 = svld1_bf16(pg16_all, ptr_b0); + + MATMUL(0, 0); + + ptr_a0 += 16; + ptr_b0 += 16; + } + + ACCUMULATE_LAST4_TO_FIRST4(0, 0, tmp); + vc0 = svuzp1(mc00, mc00); + vc1 = svuzp2(mc00, mc00); + + + UPDATE_C(pg16_first_2, pg32_first_2, ptr_c0, tmp32, tmp16, vc0); + UPDATE_C(pg16_first_2, pg32_first_2, ptr_c1, tmp32, tmp16, vc1); + + ptr_c0 += 2; + ptr_c1 += 2; + } + + if (m & 1) { + ptr_a0 = ptr_a; + ptr_b0 = ptr_b; + INIT_C(0, 0); + for (BLASLONG p = 0; p < pad_k; p += 8) { + ma0 = svld1_bf16(pg16_all, ptr_a0); + mb0 = svld1_bf16(pg16_all, ptr_b0); + MATMUL(0, 0); + ptr_a0 += 16; + ptr_b0 += 16; + } + + ACCUMULATE_LAST4_TO_FIRST4(0, 0, tmp); + vc1 = svuzp2(mc00, mc00); + + UPDATE_C(pg16_first_1, pg32_first_1, ptr_c0, tmp32, tmp16, mc00); + UPDATE_C(pg16_first_1, pg32_first_1, ptr_c1, tmp32, tmp16, vc1); + } + + ptr_b += 2 * pad_k; + } + + if (n & 1) { // TODO: this case seems a overhead. find out whether it's in our + // case. + ptr_c0 = ptr_c; + ptr_a = (bfloat16_t *)A; + + for (BLASLONG i = 0; i < m / 4; i++) { + ptr_a0 = ptr_a; + ptr_a += 4 * pad_k; + + ptr_b0 = ptr_b; + + INIT_C(0, 0); + INIT_C(1, 0); + + for (BLASLONG p = 0; p < pad_k; p += 8) { + ma0 = svld1_bf16(pg16_all, ptr_a0); + ma1 = svld1_bf16(pg16_all, ptr_a0 + 16); + + mb0 = svld1_bf16(pg16_all, ptr_b0); + + MATMUL(0, 0); + MATMUL(1, 0); + + ptr_a0 += 32; + ptr_b0 += 16; + } + + ACCUMULATE_LAST4_TO_FIRST4(0, 0, tmp); + ACCUMULATE_LAST4_TO_FIRST4(1, 0, tmp); + + ZIP_EVEN_ELEMENTS(pg32_select_first_2_per_quadword, mc00, mc10, tmp, vc0); + + UPDATE_C(pg16_first_4, pg32_first_4, ptr_c0, tmp32, tmp16, vc0); + + ptr_c0 += 4; + } + + if (m & 2) { + ptr_a0 = ptr_a; + ptr_a += 2 * pad_k; + ptr_b0 = ptr_b; + + INIT_C(0, 0); + + for (BLASLONG p = 0; p < pad_k; p += 8) { + ma0 = svld1_bf16(pg16_all, ptr_a0); + mb0 = svld1_bf16(pg16_all, ptr_b0); + + MATMUL(0, 0); + + ptr_a0 += 16; + ptr_b0 += 16; + } + + ACCUMULATE_LAST4_TO_FIRST4(0, 0, tmp); + + vc0 = svuzp1(mc00, mc00); + + UPDATE_C(pg16_first_2, pg32_first_2, ptr_c0, tmp32, tmp16, vc0); + + ptr_c0 += 2; + } + + if (m & 1) { + ptr_a0 = ptr_a; + ptr_b0 = ptr_b; + + INIT_C(0, 0); + for (BLASLONG p = 0; p < pad_k; p += 8) { + + ma0 = svld1_bf16(pg16_all, ptr_a0); + mb0 = svld1_bf16(pg16_all, ptr_b0); + + MATMUL(0, 0); + ptr_a0 += 16; + ptr_b0 += 16; + } + + ACCUMULATE_LAST4_TO_FIRST4(0, 0, tmp); + + UPDATE_C(pg16_first_1, pg32_first_1, ptr_c0, tmp32, tmp16, mc00); + } + } + + return 0; +} diff --git a/kernel/arm64/dot.c b/kernel/arm64/dot.c index ece31bccd..56a8a36dc 100644 --- a/kernel/arm64/dot.c +++ b/kernel/arm64/dot.c @@ -40,8 +40,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #endif #ifdef USE_SVE +#ifdef DOT_KERNEL_SVE +#include DOT_KERNEL_SVE +#else #include "dot_kernel_sve.c" #endif +#endif #include "dot_kernel_asimd.c" #if defined(SMP) diff --git a/kernel/arm64/dot_kernel_sve_v8.c b/kernel/arm64/dot_kernel_sve_v8.c new file mode 100644 index 000000000..fa18553b1 --- /dev/null +++ b/kernel/arm64/dot_kernel_sve_v8.c @@ -0,0 +1,146 @@ +/*************************************************************************** +Copyright (c) 2025, The OpenBLAS Project +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + 3. Neither the name of the OpenBLAS project nor the names of + its contributors may be used to endorse or promote products + derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#include +#include "common.h" + +#ifdef DOUBLE +#define SV_COUNT svcntd +#define SV_TYPE svfloat64_t +#define SV_TRUE svptrue_b64 +#define SV_WHILE svwhilelt_b64_s64 +#define SV_DUP svdup_f64 +#else +#define SV_COUNT svcntw +#define SV_TYPE svfloat32_t +#define SV_TRUE svptrue_b32 +#define SV_WHILE svwhilelt_b32_s64 +#define SV_DUP svdup_f32 +#endif + +static FLOAT dot_kernel_sve(BLASLONG n, FLOAT* x, FLOAT* y) +{ + SV_TYPE temp0_vec = SV_DUP(0.0); + SV_TYPE temp1_vec = SV_DUP(0.0); + SV_TYPE temp2_vec = SV_DUP(0.0); + SV_TYPE temp3_vec = SV_DUP(0.0); + SV_TYPE temp4_vec = SV_DUP(0.0); + SV_TYPE temp5_vec = SV_DUP(0.0); + SV_TYPE temp6_vec = SV_DUP(0.0); + SV_TYPE temp7_vec = SV_DUP(0.0); + + BLASLONG i = 0; + BLASLONG sve_size = SV_COUNT(); + + while ((i + sve_size * 8 - 1) < n) { + FLOAT *x0_ptr = x + i; + SV_TYPE x0_vec = svld1_vnum(SV_TRUE(), x0_ptr, 0); + SV_TYPE x1_vec = svld1_vnum(SV_TRUE(), x0_ptr, 1); + SV_TYPE x2_vec = svld1_vnum(SV_TRUE(), x0_ptr, 2); + SV_TYPE x3_vec = svld1_vnum(SV_TRUE(), x0_ptr, 3); + SV_TYPE x4_vec = svld1_vnum(SV_TRUE(), x0_ptr, 4); + SV_TYPE x5_vec = svld1_vnum(SV_TRUE(), x0_ptr, 5); + SV_TYPE x6_vec = svld1_vnum(SV_TRUE(), x0_ptr, 6); + SV_TYPE x7_vec = svld1_vnum(SV_TRUE(), x0_ptr, 7); + + FLOAT *y0_ptr = y + i; + SV_TYPE y0_vec = svld1_vnum(SV_TRUE(), y0_ptr, 0); + SV_TYPE y1_vec = svld1_vnum(SV_TRUE(), y0_ptr, 1); + SV_TYPE y2_vec = svld1_vnum(SV_TRUE(), y0_ptr, 2); + SV_TYPE y3_vec = svld1_vnum(SV_TRUE(), y0_ptr, 3); + SV_TYPE y4_vec = svld1_vnum(SV_TRUE(), y0_ptr, 4); + SV_TYPE y5_vec = svld1_vnum(SV_TRUE(), y0_ptr, 5); + SV_TYPE y6_vec = svld1_vnum(SV_TRUE(), y0_ptr, 6); + SV_TYPE y7_vec = svld1_vnum(SV_TRUE(), y0_ptr, 7); + + temp0_vec = svmla_x(SV_TRUE(), temp0_vec, x0_vec, y0_vec); + temp1_vec = svmla_x(SV_TRUE(), temp1_vec, x1_vec, y1_vec); + temp2_vec = svmla_x(SV_TRUE(), temp2_vec, x2_vec, y2_vec); + temp3_vec = svmla_x(SV_TRUE(), temp3_vec, x3_vec, y3_vec); + temp4_vec = svmla_x(SV_TRUE(), temp4_vec, x4_vec, y4_vec); + temp5_vec = svmla_x(SV_TRUE(), temp5_vec, x5_vec, y5_vec); + temp6_vec = svmla_x(SV_TRUE(), temp6_vec, x6_vec, y6_vec); + temp7_vec = svmla_x(SV_TRUE(), temp7_vec, x7_vec, y7_vec); + + i += sve_size * 8; + } + + if (i < n) { + svbool_t pg0 = SV_WHILE(i + sve_size * 0, n); + svbool_t pg1 = SV_WHILE(i + sve_size * 1, n); + svbool_t pg2 = SV_WHILE(i + sve_size * 2, n); + svbool_t pg3 = SV_WHILE(i + sve_size * 3, n); + svbool_t pg4 = SV_WHILE(i + sve_size * 4, n); + svbool_t pg5 = SV_WHILE(i + sve_size * 5, n); + svbool_t pg6 = SV_WHILE(i + sve_size * 6, n); + svbool_t pg7 = SV_WHILE(i + sve_size * 7, n); + + FLOAT *x0_ptr = x + i; + SV_TYPE x0_vec = svld1_vnum(pg0, x0_ptr, 0); + SV_TYPE x1_vec = svld1_vnum(pg1, x0_ptr, 1); + SV_TYPE x2_vec = svld1_vnum(pg2, x0_ptr, 2); + SV_TYPE x3_vec = svld1_vnum(pg3, x0_ptr, 3); + SV_TYPE x4_vec = svld1_vnum(pg4, x0_ptr, 4); + SV_TYPE x5_vec = svld1_vnum(pg5, x0_ptr, 5); + SV_TYPE x6_vec = svld1_vnum(pg6, x0_ptr, 6); + SV_TYPE x7_vec = svld1_vnum(pg7, x0_ptr, 7); + + FLOAT *y0_ptr = y + i; + SV_TYPE y0_vec = svld1_vnum(pg0, y0_ptr, 0); + SV_TYPE y1_vec = svld1_vnum(pg1, y0_ptr, 1); + SV_TYPE y2_vec = svld1_vnum(pg2, y0_ptr, 2); + SV_TYPE y3_vec = svld1_vnum(pg3, y0_ptr, 3); + SV_TYPE y4_vec = svld1_vnum(pg4, y0_ptr, 4); + SV_TYPE y5_vec = svld1_vnum(pg5, y0_ptr, 5); + SV_TYPE y6_vec = svld1_vnum(pg6, y0_ptr, 6); + SV_TYPE y7_vec = svld1_vnum(pg7, y0_ptr, 7); + + temp0_vec = svmla_m(pg0, temp0_vec, x0_vec, y0_vec); + temp1_vec = svmla_m(pg1, temp1_vec, x1_vec, y1_vec); + temp2_vec = svmla_m(pg2, temp2_vec, x2_vec, y2_vec); + temp3_vec = svmla_m(pg3, temp3_vec, x3_vec, y3_vec); + temp4_vec = svmla_m(pg4, temp4_vec, x4_vec, y4_vec); + temp5_vec = svmla_m(pg5, temp5_vec, x5_vec, y5_vec); + temp6_vec = svmla_m(pg6, temp6_vec, x6_vec, y6_vec); + temp7_vec = svmla_m(pg7, temp7_vec, x7_vec, y7_vec); + } + + temp0_vec = svadd_x(SV_TRUE(), temp0_vec, temp1_vec); + temp2_vec = svadd_x(SV_TRUE(), temp2_vec, temp3_vec); + temp4_vec = svadd_x(SV_TRUE(), temp4_vec, temp5_vec); + temp6_vec = svadd_x(SV_TRUE(), temp6_vec, temp7_vec); + temp0_vec = svadd_x(SV_TRUE(), temp0_vec, temp2_vec); + temp4_vec = svadd_x(SV_TRUE(), temp4_vec, temp6_vec); + temp0_vec = svadd_x(SV_TRUE(), temp0_vec, temp4_vec); + + return svaddv(SV_TRUE(), temp0_vec); +} diff --git a/kernel/arm64/dot_sve_v8.c b/kernel/arm64/dot_sve_v8.c new file mode 100644 index 000000000..ace47b4e7 --- /dev/null +++ b/kernel/arm64/dot_sve_v8.c @@ -0,0 +1,34 @@ +/*************************************************************************** +Copyright (c) 2025, The OpenBLAS Project +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + 3. Neither the name of the OpenBLAS project nor the names of + its contributors may be used to endorse or promote products + derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#define DOT_KERNEL_SVE "dot_kernel_sve_v8.c" +#include "dot.c" diff --git a/kernel/arm64/gemv_n_sve_v1x3.c b/kernel/arm64/gemv_n_sve_v1x3.c index d6aa3d389..5ab8d3a16 100644 --- a/kernel/arm64/gemv_n_sve_v1x3.c +++ b/kernel/arm64/gemv_n_sve_v1x3.c @@ -52,17 +52,17 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha, FLOAT *a, BLASLONG lda, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y, FLOAT *buffer) { - BLASLONG i; - BLASLONG ix,iy; - BLASLONG j; - FLOAT *a_ptr; + BLASLONG i, j; + BLASLONG ix = 0; + BLASLONG iy; + FLOAT *a_ptr = a; FLOAT temp; - ix = 0; - a_ptr = a; - if (inc_y == 1) { - BLASLONG width = (n + 3 - 1) / 3; + BLASLONG width = n / 3; // Only process full 3-column blocks + BLASLONG sve_size = SV_COUNT(); + svbool_t pg_full = SV_TRUE(); + svbool_t pg_tail = SV_WHILE(0, m % sve_size); FLOAT *a0_ptr = a_ptr + lda * width * 0; FLOAT *a1_ptr = a_ptr + lda * width * 1; @@ -73,57 +73,75 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha, FLOAT *a, FLOAT *x2_ptr = x + inc_x * width * 2; for (j = 0; j < width; j++) { - svbool_t pg00 = ((j + width * 0) < n) ? SV_TRUE() : svpfalse(); - svbool_t pg01 = ((j + width * 1) < n) ? SV_TRUE() : svpfalse(); - svbool_t pg02 = ((j + width * 2) < n) ? SV_TRUE() : svpfalse(); + SV_TYPE temp0_vec = SV_DUP(alpha * x0_ptr[ix]); + SV_TYPE temp1_vec = SV_DUP(alpha * x1_ptr[ix]); + SV_TYPE temp2_vec = SV_DUP(alpha * x2_ptr[ix]); - SV_TYPE temp0_vec = ((j + width * 0) < n) ? SV_DUP(alpha * x0_ptr[ix]) : SV_DUP(0.0); - SV_TYPE temp1_vec = ((j + width * 1) < n) ? SV_DUP(alpha * x1_ptr[ix]) : SV_DUP(0.0); - SV_TYPE temp2_vec = ((j + width * 2) < n) ? SV_DUP(alpha * x2_ptr[ix]) : SV_DUP(0.0); i = 0; - BLASLONG sve_size = SV_COUNT(); - while ((i + sve_size * 1 - 1) < m) { - SV_TYPE y0_vec = svld1_vnum(SV_TRUE(), y + i, 0); + while ((i + sve_size - 1) < m) { + SV_TYPE y0_vec = svld1(pg_full, y + i); - SV_TYPE a00_vec = svld1_vnum(pg00, a0_ptr + i, 0); - SV_TYPE a01_vec = svld1_vnum(pg01, a1_ptr + i, 0); - SV_TYPE a02_vec = svld1_vnum(pg02, a2_ptr + i, 0); + SV_TYPE a00_vec = svld1(pg_full, a0_ptr + i); + SV_TYPE a01_vec = svld1(pg_full, a1_ptr + i); + SV_TYPE a02_vec = svld1(pg_full, a2_ptr + i); - y0_vec = svmla_m(pg00, y0_vec, temp0_vec, a00_vec); - y0_vec = svmla_m(pg01, y0_vec, temp1_vec, a01_vec); - y0_vec = svmla_m(pg02, y0_vec, temp2_vec, a02_vec); + y0_vec = svmla_x(pg_full, y0_vec, temp0_vec, a00_vec); + y0_vec = svmla_x(pg_full, y0_vec, temp1_vec, a01_vec); + y0_vec = svmla_x(pg_full, y0_vec, temp2_vec, a02_vec); - svst1_vnum(SV_TRUE(), y + i, 0, y0_vec); - i += sve_size * 1; + svst1(pg_full, y + i, y0_vec); + i += sve_size; } if (i < m) { - svbool_t pg0 = SV_WHILE(i + sve_size * 0, m); - - pg00 = svand_z(SV_TRUE(), pg0, pg00); - pg01 = svand_z(SV_TRUE(), pg0, pg01); - pg02 = svand_z(SV_TRUE(), pg0, pg02); + SV_TYPE y0_vec = svld1(pg_tail, y + i); - SV_TYPE y0_vec = svld1_vnum(pg0, y + i, 0); + SV_TYPE a00_vec = svld1(pg_tail, a0_ptr + i); + SV_TYPE a01_vec = svld1(pg_tail, a1_ptr + i); + SV_TYPE a02_vec = svld1(pg_tail, a2_ptr + i); - SV_TYPE a00_vec = svld1_vnum(pg00, a0_ptr + i, 0); - SV_TYPE a01_vec = svld1_vnum(pg01, a1_ptr + i, 0); - SV_TYPE a02_vec = svld1_vnum(pg02, a2_ptr + i, 0); + y0_vec = svmla_m(pg_tail, y0_vec, temp0_vec, a00_vec); + y0_vec = svmla_m(pg_tail, y0_vec, temp1_vec, a01_vec); + y0_vec = svmla_m(pg_tail, y0_vec, temp2_vec, a02_vec); - y0_vec = svmla_m(pg00, y0_vec, temp0_vec, a00_vec); - y0_vec = svmla_m(pg01, y0_vec, temp1_vec, a01_vec); - y0_vec = svmla_m(pg02, y0_vec, temp2_vec, a02_vec); - - svst1_vnum(pg0, y + i, 0, y0_vec); + svst1(pg_tail, y + i, y0_vec); } a0_ptr += lda; a1_ptr += lda; a2_ptr += lda; ix += inc_x; } + // Handle remaining n % 3 columns + for (j = width * 3; j < n; j++) { + FLOAT *a_col = a + j * lda; + temp = alpha * x[j * inc_x]; + SV_TYPE temp_vec = SV_DUP(temp); + + i = 0; + while ((i + sve_size - 1) < m) { + SV_TYPE y_vec = svld1(pg_full, y + i); + + SV_TYPE a_vec = svld1(pg_full, a_col + i); + + y_vec = svmla_x(pg_full, y_vec, temp_vec, a_vec); + + svst1(pg_full, y + i, y_vec); + i += sve_size; + } + if (i < m) { + SV_TYPE y_vec = svld1(pg_tail, y + i); + + SV_TYPE a_vec = svld1(pg_tail, a_col + i); + + y_vec = svmla_m(pg_tail, y_vec, temp_vec, a_vec); + + svst1(pg_tail, y + i, y_vec); + } + } return(0); } + // Fallback scalar loop for (j = 0; j < n; j++) { temp = alpha * x[ix]; iy = 0; diff --git a/kernel/arm64/sgemm_direct_alpha_beta_arm64_sme1.c b/kernel/arm64/sgemm_direct_alpha_beta_arm64_sme1.c new file mode 100644 index 000000000..d9de3ace3 --- /dev/null +++ b/kernel/arm64/sgemm_direct_alpha_beta_arm64_sme1.c @@ -0,0 +1,199 @@ +/* + Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + SPDX-License-Identifier: BSD-3-Clause-Clear +*/ + +#include "common.h" +#include +#include +#include +#include "sme_abi.h" +#if defined(HAVE_SME) + +#if defined(__ARM_FEATURE_SME) && defined(__clang__) && __clang_major__ >= 16 +#include +#endif + +/* Function prototypes */ +extern void sgemm_direct_sme1_preprocess(uint64_t nbr, uint64_t nbc,\ + const float * restrict a, float * a_mod) __asm__("sgemm_direct_sme1_preprocess"); + +/* Function Definitions */ +static uint64_t sve_cntw() { + uint64_t cnt; + asm volatile( + "rdsvl %[res], #1\n" + "lsr %[res], %[res], #2\n" + : [res] "=r" (cnt) :: + ); + return cnt; +} + +#if defined(__ARM_FEATURE_SME) && defined(__ARM_FEATURE_LOCALLY_STREAMING) && defined(__clang__) && __clang_major__ >= 16 +// Outer product kernel. +// Computes a 2SVL x 2SVL block of C, utilizing all four FP32 tiles of ZA. +__attribute__((always_inline)) inline void +kernel_2x2(const float *A, const float *B, float *C, size_t shared_dim, + size_t ldc, size_t block_rows, size_t block_cols, float alpha, float beta) + __arm_out("za") __arm_streaming { + + const uint64_t svl = svcntw(); + size_t ldb = ldc; + // Predicate set-up + svbool_t pg = svptrue_b32(); + svbool_t pg_a_0 = svwhilelt_b32_u64(0, block_rows); + svbool_t pg_a_1 = svwhilelt_b32_u64(svl, block_rows); + + svbool_t pg_b_0 = svwhilelt_b32_u64(0, block_cols); + svbool_t pg_b_1 = svwhilelt_b32_u64(svl, block_cols); + +#define pg_c_0 pg_b_0 +#define pg_c_1 pg_b_1 + + svzero_za(); + svfloat32_t beta_vec = svdup_f32(beta); + // Load C to ZA + for (size_t i = 0; i < MIN(svl, block_rows); i++) { + svfloat32_t row_c_0 = svld1(pg_c_0, &C[i * ldc]); + row_c_0 = svmul_x(pg, beta_vec, row_c_0); + svwrite_hor_za32_f32_m(/*tile*/0, /*slice*/i, pg_c_0, row_c_0); + + svfloat32_t row_c_1 = svld1(pg_c_1, &C[i * ldc + svl]); + row_c_1 = svmul_x(pg, beta_vec, row_c_1); + svwrite_hor_za32_f32_m(/*tile*/1, /*slice*/i, pg_c_1, row_c_1); + } + for (size_t i = svl; i < block_rows; i++) { + svfloat32_t row_c_0 = svld1(pg_c_0, &C[i * ldc]); + row_c_0 = svmul_x(pg, beta_vec, row_c_0); + svwrite_hor_za32_f32_m(/*tile*/2, /*slice*/i, pg_c_0, row_c_0); + + svfloat32_t row_c_1 = svld1(pg_c_1, &C[i * ldc + svl]); + row_c_1 = svmul_x(pg, beta_vec, row_c_1); + svwrite_hor_za32_f32_m(/*tile*/3, /*slice*/i, pg_c_1, row_c_1); + } + + svfloat32_t alpha_vec = svdup_f32(alpha); + // Iterate through shared dimension (K) + for (size_t k = 0; k < shared_dim; k++) { + // Load column of A + svfloat32_t col_a_0 = svld1(pg_a_0, &A[k * svl]); + col_a_0 = svmul_x(pg, alpha_vec, col_a_0); + svfloat32_t col_a_1 = svld1(pg_a_1, &A[(k + shared_dim) * svl]); + col_a_1 = svmul_x(pg, alpha_vec, col_a_1); + // Load row of B + svfloat32_t row_b_0 = svld1(pg_b_0, &B[k * ldb]); + svfloat32_t row_b_1 = svld1(pg_b_1, &B[k * ldb + svl]); + // Perform outer product + svmopa_za32_m(/*tile*/0, pg, pg, col_a_0, row_b_0); + svmopa_za32_m(/*tile*/1, pg, pg, col_a_0, row_b_1); + svmopa_za32_m(/*tile*/2, pg, pg, col_a_1, row_b_0); + svmopa_za32_m(/*tile*/3, pg, pg, col_a_1, row_b_1); + } + + // Store to C from ZA + for (size_t i = 0; i < MIN(svl, block_rows); i++) { + svst1_hor_za32(/*tile*/0, /*slice*/i, pg_c_0, &C[i * ldc]); + svst1_hor_za32(/*tile*/1, /*slice*/i, pg_c_1, &C[i * ldc + svl]); + } + for (size_t i = svl; i < block_rows; i++) { + svst1_hor_za32(/*tile*/2, /*slice*/i, pg_c_0, &C[i * ldc]); + svst1_hor_za32(/*tile*/3, /*slice*/i, pg_c_1, &C[i * ldc + svl]); + } +} + +__arm_new("za") __arm_locally_streaming +void sgemm_direct_alpha_beta_sme1_2VLx2VL(uint64_t m, uint64_t k, uint64_t n, const float* alpha,\ + const float *ba, const float *restrict bb, const float* beta,\ + float *restrict C) { + + const uint64_t num_rows = m; + const uint64_t num_cols = n; + + const float *restrict a_ptr = ba; + const float *restrict b_ptr = bb; + float *restrict c_ptr = C; + + const uint64_t svl = svcntw(); + const uint64_t ldc = n; + + // Block over rows of C (panels of A) + uint64_t row_idx = 0; + + // 2x2 loop + uint64_t row_batch = 2*svl; + + // Block over row dimension of C + for (; row_idx < num_rows; row_idx += row_batch) { + row_batch = MIN(row_batch, num_rows - row_idx); + uint64_t col_idx = 0; + uint64_t col_batch = 2*svl; + + // Block over column dimension of C + for (; col_idx < num_cols; col_idx += col_batch) { + col_batch = MIN(col_batch, num_cols - col_idx); + + kernel_2x2(&a_ptr[row_idx * k], &b_ptr[col_idx], + &c_ptr[row_idx * ldc + col_idx], k, + ldc, row_batch, col_batch, *alpha, *beta); + } + } + return; +} + +#else +void sgemm_direct_alpha_beta_sme1_2VLx2VL(uint64_t m, uint64_t k, uint64_t n, const float* alpha,\ + const float *ba, const float *restrict bb, const float* beta,\ + float *restrict C){} +#endif + +/*void sgemm_kernel_direct (BLASLONG M, BLASLONG N, BLASLONG K,\ + float * __restrict A, BLASLONG strideA, float * __restrict B,\ + BLASLONG strideB , float * __restrict R, BLASLONG strideR) +*/ +void CNAME (BLASLONG M, BLASLONG N, BLASLONG K, float alpha, float * __restrict A,\ + BLASLONG strideA, float * __restrict B, BLASLONG strideB ,\ + float beta, float * __restrict R, BLASLONG strideR){ + + uint64_t m_mod, vl_elms; + + vl_elms = sve_cntw(); + + m_mod = ceil((double)M/(double)vl_elms) * vl_elms; + + float *A_mod = (float *) malloc(m_mod*K*sizeof(float)); + + /* Prevent compiler optimization by reading from memory instead + * of reading directly from vector (z) registers. + * */ + asm volatile("" : : :"p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", + "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", + "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", + "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", + "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", + "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"); + + /* Pre-process the left matrix to make it suitable for + matrix sum of outer-product calculation + */ + sgemm_direct_sme1_preprocess(M, K, A, A_mod); + + asm volatile("" : : :"p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", + "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15", + "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", + "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", + "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", + "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"); + + /* Calculate C = alpha*A*B + beta*C */ + sgemm_direct_alpha_beta_sme1_2VLx2VL(M, K, N, &alpha, A_mod, B, &beta, R); + + free(A_mod); +} + +#else + +void CNAME (BLASLONG M, BLASLONG N, BLASLONG K, float alpha, float * __restrict A,\ + BLASLONG strideA, float * __restrict B, BLASLONG strideB ,\ + float beta, float * __restrict R, BLASLONG strideR){} + +#endif diff --git a/kernel/arm64/sme_abi.h b/kernel/arm64/sme_abi.h new file mode 100644 index 000000000..07bba4895 --- /dev/null +++ b/kernel/arm64/sme_abi.h @@ -0,0 +1,46 @@ +/*************************************************************************** + * Copyright (c) 2024, The OpenBLAS Project + * All rights reserved. + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of the OpenBLAS project nor the names of + * its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE + * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF + * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * *****************************************************************************/ + +#pragma once + +#include + +/** + * * These are SME ABI routines for saving & restoring SME state. + * * They are typically provided by a compiler runtime library such + * * as libgcc or compiler-rt, but support for these routines is not + * * yet available on all platforms. + * * + * * Define these as aborting stubs so that we loudly fail on nested + * * usage of SME state. + * * + * * These are defined as weak symbols so that a compiler runtime can + * * override them if supported. + * */ +__attribute__((weak)) void __arm_tpidr2_save() { abort(); } +__attribute__((weak)) void __arm_tpidr2_restore() { abort(); } + diff --git a/kernel/generic/bf16_macros.h b/kernel/generic/bf16_macros.h new file mode 100644 index 000000000..f1b02cea4 --- /dev/null +++ b/kernel/generic/bf16_macros.h @@ -0,0 +1,64 @@ +/*************************************************************************** + * Copyright (c) 2025, The OpenBLAS Project + * All rights reserved. + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of the OpenBLAS project nor the names of + * its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * *****************************************************************************/ + +#if defined(BFLOAT16) && defined(BFLOAT16CONVERSION) +static float +bfloat16tof32 (bfloat16 value) +{ + blasint one = 1; + float result; + sbf16tos_(&one, &value, &one, &result, &one); + return result; +} + +#ifdef BGEMM +static bfloat16 f32tobfloat16(float value) { + blasint one = 1; + bfloat16 result; + sbstobf16_(&one, &value, &one, &result, &one); + return result; +} +#endif + +#ifdef BGEMM +#define ALPHA bfloat16tof32(alpha) +#define BETA bfloat16tof32(beta) +#define BF16TOF32(x) (bfloat16tof32(x)) +#define F32TOBF16(x) (f32tobfloat16(x)) +#else +#define ALPHA alpha +#define BETA beta +#define BF16TOF32(x) (bfloat16tof32(x)) +#define F32TOBF16(x) x +#endif +#else +#define ALPHA alpha +#define BETA beta +#define BF16TOF32(x) x +#define F32TOBF16(x) x +#endif diff --git a/kernel/generic/gemm_beta.c b/kernel/generic/gemm_beta.c index ccb772cc7..74e9bf9a9 100644 --- a/kernel/generic/gemm_beta.c +++ b/kernel/generic/gemm_beta.c @@ -1,5 +1,6 @@ /*********************************************************************/ /* Copyright 2009, 2010 The University of Texas at Austin. */ +/* Copyright 2025 The OpenBLAS Project. */ /* All rights reserved. */ /* */ /* Redistribution and use in source and binary forms, with or */ @@ -38,11 +39,41 @@ #include "common.h" +#if defined(BFLOAT16) && defined(BGEMM) && defined(BFLOAT16CONVERSION) +static float +bfloat16tof32 (bfloat16 f16) +{ + float result = 0; + unsigned short* q = (unsigned short*)(&result); +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + q[0] = f16; +#else + q[1] = f16; +#endif + return result; +} +static bfloat16 +f32tobfloat16(float f32) +{ + unsigned short* q = (unsigned short*)(&f32); +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + return q[0]; +#else + return q[1]; +#endif +} + +#define BF16TOF32(x) (bfloat16tof32(x)) +#define F32TOBF16(x) (f32tobfloat16(x)) +#else +#define BF16TOF32(x) x +#define F32TOBF16(x) x +#endif + int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT beta, IFLOAT *dummy2, BLASLONG dummy3, IFLOAT *dummy4, BLASLONG dummy5, FLOAT *c, BLASLONG ldc){ - BLASLONG i, j; BLASLONG chunk, remain; FLOAT *c_offset1, *c_offset; @@ -54,18 +85,18 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT beta, c_offset1 = c_offset; c_offset += ldc; for(i=chunk; i>0; i--){ - *(c_offset1 + 0) = ZERO; - *(c_offset1 + 1) = ZERO; - *(c_offset1 + 2) = ZERO; - *(c_offset1 + 3) = ZERO; - *(c_offset1 + 4) = ZERO; - *(c_offset1 + 5) = ZERO; - *(c_offset1 + 6) = ZERO; - *(c_offset1 + 7) = ZERO; + *(c_offset1 + 0) = F32TOBF16(ZERO); + *(c_offset1 + 1) = F32TOBF16(ZERO); + *(c_offset1 + 2) = F32TOBF16(ZERO); + *(c_offset1 + 3) = F32TOBF16(ZERO); + *(c_offset1 + 4) = F32TOBF16(ZERO); + *(c_offset1 + 5) = F32TOBF16(ZERO); + *(c_offset1 + 6) = F32TOBF16(ZERO); + *(c_offset1 + 7) = F32TOBF16(ZERO); c_offset1 += 8; } for(i=remain; i>0; i--){ - *c_offset1 = ZERO; + *c_offset1 = F32TOBF16(ZERO); c_offset1 ++; } } @@ -74,18 +105,18 @@ int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT beta, c_offset1 = c_offset; c_offset += ldc; for(i=chunk; i>0; i--){ - *(c_offset1 + 0) *= beta; - *(c_offset1 + 1) *= beta; - *(c_offset1 + 2) *= beta; - *(c_offset1 + 3) *= beta; - *(c_offset1 + 4) *= beta; - *(c_offset1 + 5) *= beta; - *(c_offset1 + 6) *= beta; - *(c_offset1 + 7) *= beta; + *(c_offset1 + 0) = F32TOBF16(BF16TOF32(beta) * BF16TOF32(c_offset1[0])); + *(c_offset1 + 1) = F32TOBF16(BF16TOF32(beta) * BF16TOF32(c_offset1[1])); + *(c_offset1 + 2) = F32TOBF16(BF16TOF32(beta) * BF16TOF32(c_offset1[2])); + *(c_offset1 + 3) = F32TOBF16(BF16TOF32(beta) * BF16TOF32(c_offset1[3])); + *(c_offset1 + 4) = F32TOBF16(BF16TOF32(beta) * BF16TOF32(c_offset1[4])); + *(c_offset1 + 5) = F32TOBF16(BF16TOF32(beta) * BF16TOF32(c_offset1[5])); + *(c_offset1 + 6) = F32TOBF16(BF16TOF32(beta) * BF16TOF32(c_offset1[6])); + *(c_offset1 + 7) = F32TOBF16(BF16TOF32(beta) * BF16TOF32(c_offset1[7])); c_offset1 += 8; } for(i=remain; i>0; i--){ - *c_offset1 *= beta; + *c_offset1 = F32TOBF16(BF16TOF32(beta) * BF16TOF32(c_offset1[0])); c_offset1 ++; } } diff --git a/kernel/generic/gemmkernel_2x2.c b/kernel/generic/gemmkernel_2x2.c index bf1c3ae38..c24370c89 100644 --- a/kernel/generic/gemmkernel_2x2.c +++ b/kernel/generic/gemmkernel_2x2.c @@ -1,21 +1,34 @@ +/*************************************************************************** + * Copyright (c) 2025, The OpenBLAS Project + * All rights reserved. + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of the OpenBLAS project nor the names of + * its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * *****************************************************************************/ + #include "common.h" -#if defined(BFLOAT16) && defined(BFLOAT16CONVERSION) -static float -bfloat16tof32 (bfloat16 f16) -{ - float result = 0; - unsigned short* q = (unsigned short*)(&result); -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ - q[0] = f16; -#else - q[1] = f16; -#endif - return result; -} -#define BF16TOF32(x) (bfloat16tof32(x)) -#else -#define BF16TOF32(x) x -#endif +#include "bf16_macros.h" + int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alpha,IFLOAT* ba,IFLOAT* bb,FLOAT* C,BLASLONG ldc #ifdef TRMMKERNEL ,BLASLONG offset @@ -25,7 +38,11 @@ int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alpha,IFLOAT* ba,IFLOAT* bb, BLASLONG i,j,k; FLOAT *C0,*C1; IFLOAT *ptrba,*ptrbb; +#ifdef BGEMM + float res0,res1,res2,res3; +#else FLOAT res0,res1,res2,res3; +#endif IFLOAT load0,load1,load2,load3,load4,load5,load6,load7; for (j=0; j BBLAT2.SUMM + @$(GREP) -q FATAL BBLAT2.SUMM && cat BBLAT2.SUMM || exit 0 + OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 ./test_sbgemv > SBBLAT2.SUMM + @$(GREP) -q FATAL SBBLAT2.SUMM && cat SBBLAT2.SUMM || exit 0 +endif ifeq ($(BUILD_SINGLE),1) OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 ./sblat2 < ./sblat2.dat @$(GREP) -q FATAL SBLAT2.SUMM && cat SBLAT2.SUMM || exit 0 @@ -127,6 +166,12 @@ endif ifdef SMP rm -f ?BLAT2.SUMM ifeq ($(USE_OPENMP), 1) +ifeq ($(BUILD_BFLOAT16),1) + OMP_NUM_THREADS=2 ./test_bgemv > BBLAT2.SUMM + @$(GREP) -q FATAL BBLAT2.SUMM && cat BBLAT2.SUMM || exit 0 + OMP_NUM_THREADS=2 ./test_sbgemv > SBBLAT2.SUMM + @$(GREP) -q FATAL SBBLAT2.SUMM && cat SBBLAT2.SUMM || exit 0 +endif ifeq ($(BUILD_SINGLE),1) OMP_NUM_THREADS=2 ./sblat2 < ./sblat2.dat @$(GREP) -q FATAL SBLAT2.SUMM && cat SBLAT2.SUMM || exit 0 @@ -144,6 +189,12 @@ ifeq ($(BUILD_COMPLEX16),1) @$(GREP) -q FATAL ZBLAT2.SUMM && cat ZBLAT2.SUMM || exit 0 endif else +ifeq ($(BUILD_BFLOAT16),1) + OMP_NUM_THREADS=2 ./test_bgemv > BBLAT2.SUMM + @$(GREP) -q FATAL BBLAT2.SUMM && cat BBLAT2.SUMM || exit 0 + OMP_NUM_THREADS=2 ./test_sbgemv > SBBLAT2.SUMM + @$(GREP) -q FATAL SBBLAT2.SUMM && cat SBBLAT2.SUMM || exit 0 +endif ifeq ($(BUILD_SINGLE),1) OPENBLAS_NUM_THREADS=2 ./sblat2 < ./sblat2.dat @$(GREP) -q FATAL SBLAT2.SUMM && cat SBLAT2.SUMM || exit 0 @@ -165,7 +216,8 @@ endif endif ifeq ($(BUILD_BFLOAT16),1) -B3= test_sbgemm +BF3= test_bgemm +B3 = test_sbgemm endif ifeq ($(BUILD_SINGLE),1) S3=sblat3 @@ -192,7 +244,7 @@ endif ifeq ($(SUPPORT_GEMM3M),1) level3: $(B3) $(S3) $(D3) $(C3) $(Z3) level3_3m else -level3: $(B3) $(S3) $(D3) $(C3) $(Z3) +level3: $(BF3) $(B3) $(S3) $(D3) $(C3) $(Z3) endif ifneq ($(CROSS), 1) @@ -200,6 +252,8 @@ ifneq ($(CROSS), 1) ifeq ($(BUILD_BFLOAT16),1) OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 ./test_sbgemm > SBBLAT3.SUMM @$(GREP) -q FATAL SBBLAT3.SUMM && cat SBBLAT3.SUMM || exit 0 + OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 ./test_bgemm > BBLAT3.SUMM + @$(GREP) -q FATAL BBLAT3.SUMM && cat BBLAT3.SUMM || exit 0 endif ifeq ($(BUILD_SINGLE),1) OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 ./sblat3 < ./sblat3.dat @@ -223,6 +277,8 @@ ifeq ($(USE_OPENMP), 1) ifeq ($(BUILD_BFLOAT16),1) OMP_NUM_THREADS=2 ./test_sbgemm > SBBLAT3.SUMM @$(GREP) -q FATAL SBBLAT3.SUMM && cat SBBLAT3.SUMM || exit 0 + OMP_NUM_THREADS=2 ./test_bgemm > BBLAT3.SUMM + @$(GREP) -q FATAL BBLAT3.SUMM && cat BBLAT3.SUMM || exit 0 endif ifeq ($(BUILD_SINGLE),1) OMP_NUM_THREADS=2 ./sblat3 < ./sblat3.dat @@ -244,6 +300,8 @@ else ifeq ($(BUILD_BFLOAT16),1) OPENBLAS_NUM_THREADS=2 ./test_sbgemm > SBBLAT3.SUMM @$(GREP) -q FATAL SBBLAT3.SUMM && cat SBBLAT3.SUMM || exit 0 + OPENBLAS_NUM_THREADS=2 ./test_bgemm > BBLAT3.SUMM + @$(GREP) -q FATAL BBLAT3.SUMM && cat BBLAT3.SUMM || exit 0 endif ifeq ($(BUILD_SINGLE),1) OPENBLAS_NUM_THREADS=2 ./sblat3 < ./sblat3.dat @@ -367,8 +425,17 @@ zblat3 : zblat3.$(SUFFIX) ../$(LIBNAME) endif ifeq ($(BUILD_BFLOAT16),1) -test_sbgemm : compare_sgemm_sbgemm.c ../$(LIBNAME) - $(CC) $(CLDFLAGS) -o test_sbgemm compare_sgemm_sbgemm.c ../$(LIBNAME) $(EXTRALIB) $(CEXTRALIB) +test_bgemm : compare_sgemm_bgemm.c test_helpers.h ../$(LIBNAME) + $(CC) $(CLDFLAGS) -DIBFLOAT16 -DOBFLOAT16 -o test_bgemm compare_sgemm_bgemm.c ../$(LIBNAME) $(EXTRALIB) $(CEXTRALIB) + +test_bgemv : compare_sgemv_bgemv.c ../$(LIBNAME) + $(CC) $(CLDFLAGS) -DIBFLOAT16 -DOBFLOAT16 -o test_bgemv compare_sgemv_bgemv.c ../$(LIBNAME) $(EXTRALIB) $(CEXTRALIB) + +test_sbgemm : compare_sgemm_sbgemm.c test_helpers.h ../$(LIBNAME) + $(CC) $(CLDFLAGS) -DIBFLOAT16 -o test_sbgemm compare_sgemm_sbgemm.c ../$(LIBNAME) $(EXTRALIB) $(CEXTRALIB) + +test_sbgemv : compare_sgemv_sbgemv.c ../$(LIBNAME) + $(CC) $(CLDFLAGS) -DIBFLOAT16 -o test_sbgemv compare_sgemv_sbgemv.c ../$(LIBNAME) $(EXTRALIB) $(CEXTRALIB) endif ifeq ($(BUILD_COMPLEX),1) @@ -387,7 +454,7 @@ clean: @rm -f *.$(SUFFIX) *.$(PSUFFIX) gmon.$(SUFFIX)ut *.SUMM *.cxml *.exe *.pdb *.dwf \ sblat1 dblat1 cblat1 zblat1 \ sblat2 dblat2 cblat2 zblat2 \ - test_sbgemm sblat3 dblat3 cblat3 zblat3 \ + test_bgemm test_bgemv test_sbgemm test_sbgemv sblat3 dblat3 cblat3 zblat3 \ sblat1p dblat1p cblat1p zblat1p \ sblat2p dblat2p cblat2p zblat2p \ sblat3p dblat3p cblat3p zblat3p \ diff --git a/test/compare_sgemm_bgemm.c b/test/compare_sgemm_bgemm.c new file mode 100644 index 000000000..f18fe1201 --- /dev/null +++ b/test/compare_sgemm_bgemm.c @@ -0,0 +1,155 @@ +/*************************************************************************** +Copyright (c) 2025 The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ +#include "../common.h" +#include +#include + +#include "test_helpers.h" + +#define SGEMM BLASFUNC(sgemm) +#define BGEMM BLASFUNC(bgemm) +#define BGEMM_LARGEST 256 + +int +main (int argc, char *argv[]) +{ + blasint m, n, k; + int i, j, l; + blasint x, y; + blasint one = 1; + int ret = 0; + int loop = BGEMM_LARGEST; + char transA = 'N', transB = 'N'; + float alpha = 1.0, beta = 0.0; + bfloat16 alpha_bf16; + sbstobf16_(&one, &alpha, &one, &alpha_bf16, &one); + bfloat16 beta_bf16; + sbstobf16_(&one, &beta, &one, &beta_bf16, &one); + + for (x = 0; x <= loop; x++) + { + if ((x > 100) && (x != BGEMM_LARGEST)) continue; + m = k = n = x; + float *A = (float *)malloc_safe(m * k * sizeof(FLOAT)); + float *B = (float *)malloc_safe(k * n * sizeof(FLOAT)); + float *C = (float *)malloc_safe(m * n * sizeof(FLOAT)); + bfloat16 *AA = (bfloat16 *)malloc_safe(m * k * sizeof(bfloat16)); + bfloat16 *BB = (bfloat16 *)malloc_safe(k * n * sizeof(bfloat16)); + bfloat16 *CC = (bfloat16 *)malloc_safe(k * n * sizeof(bfloat16)); + FLOAT *DD = (FLOAT *)malloc_safe(m * n * sizeof(FLOAT)); + if ((A == NULL) || (B == NULL) || (C == NULL) || (AA == NULL) || (BB == NULL) || + (DD == NULL) || (CC == NULL)) + return 1; + + for (j = 0; j < m; j++) + { + for (i = 0; i < k; i++) + { + A[j * k + i] = ((FLOAT) rand () / (FLOAT) RAND_MAX) + 0.5; + sbstobf16_(&one, &A[j*k+i], &one, &AA[j * k + i], &one); + } + } + for (j = 0; j < n; j++) + { + for (i = 0; i < k; i++) + { + B[j * k + i] = ((FLOAT) rand () / (FLOAT) RAND_MAX) + 0.5; + sbstobf16_(&one, &B[j*k+i], &one, &BB[j * k + i], &one); + } + } + for (y = 0; y < 4; y++) + { + if ((y == 0) || (y == 2)) { + transA = 'N'; + } else { + transA = 'T'; + } + if ((y == 0) || (y == 1)) { + transB = 'N'; + } else { + transB = 'T'; + } + + memset(CC, 0, m * n * sizeof(bfloat16)); + memset(DD, 0, m * n * sizeof(FLOAT)); + memset(C, 0, m * n * sizeof(FLOAT)); + + SGEMM (&transA, &transB, &m, &n, &k, &alpha, A, + &m, B, &k, &beta, C, &m); + BGEMM (&transA, &transB, &m, &n, &k, &alpha_bf16, (bfloat16*) AA, + &m, (bfloat16*)BB, &k, &beta_bf16, (bfloat16*)CC, &m); + + for (i = 0; i < n; i++) + for (j = 0; j < m; j++) + { + for (l = 0; l < k; l++) + if (transA == 'N' && transB == 'N') + { + DD[i * m + j] += + float16to32 (AA[l * m + j]) * float16to32 (BB[l + k * i]); + } else if (transA == 'T' && transB == 'N') + { + DD[i * m + j] += + float16to32 (AA[k * j + l]) * float16to32 (BB[l + k * i]); + } else if (transA == 'N' && transB == 'T') + { + DD[i * m + j] += + float16to32 (AA[l * m + j]) * float16to32 (BB[i + l * n]); + } else if (transA == 'T' && transB == 'T') + { + DD[i * m + j] += + float16to32 (AA[k * j + l]) * float16to32 (BB[i + l * n]); + } + if (!is_close(float16to32(CC[i * m + j]), truncate_float32_to_bfloat16(C[i * m + j]), 0.01, 0.001)) { + printf("Mismatch at i=%d, j=%d, k=%d: CC=%.6f, C=%.6f\n", + i, j, k, float16to32(CC[i * m + j]), truncate_float32_to_bfloat16(C[i * m + j])); + ret++; + } + + if (!is_close(float16to32(CC[i * m + j]), truncate_float32_to_bfloat16(DD[i * m + j]), 0.0001, 0.00001)) { + printf("Mismatch at i=%d, j=%d, k=%d: CC=%.6f, DD=%.6f\n", + i, j, k, float16to32(CC[i * m + j]), truncate_float32_to_bfloat16(DD[i * m + j])); + ret++; + } + + } + } + free(A); + free(B); + free(C); + free(AA); + free(BB); + free(CC); + free(DD); + } + + if (ret != 0) { + fprintf (stderr, "FATAL ERROR BGEMM - Return code: %d\n", ret); + } + + return ret; +} diff --git a/test/compare_sgemm_sbgemm.c b/test/compare_sgemm_sbgemm.c index ae109c1a5..489222516 100644 --- a/test/compare_sgemm_sbgemm.c +++ b/test/compare_sgemm_sbgemm.c @@ -27,72 +27,15 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include "../common.h" + +#include "test_helpers.h" + #define SGEMM BLASFUNC(sgemm) #define SBGEMM BLASFUNC(sbgemm) #define SGEMV BLASFUNC(sgemv) #define SBGEMV BLASFUNC(sbgemv) -typedef union -{ - unsigned short v; -#if defined(_AIX) - struct __attribute__((packed)) -#else - struct -#endif - { -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ - unsigned short s:1; - unsigned short e:8; - unsigned short m:7; -#else - unsigned short m:7; - unsigned short e:8; - unsigned short s:1; -#endif - } bits; -} bfloat16_bits; - -typedef union -{ - float v; -#if defined(_AIX) - struct __attribute__((packed)) -#else - struct -#endif - { -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ - uint32_t s:1; - uint32_t e:8; - uint32_t m:23; -#else - uint32_t m:23; - uint32_t e:8; - uint32_t s:1; -#endif - } bits; -} float32_bits; - -float -float16to32 (bfloat16_bits f16) -{ - float32_bits f32; - f32.bits.s = f16.bits.s; - f32.bits.e = f16.bits.e; - f32.bits.m = (uint32_t) f16.bits.m << 16; - return f32.v; -} - #define SBGEMM_LARGEST 256 -void *malloc_safe(size_t size) -{ - if (size == 0) - return malloc(1); - else - return malloc(size); -} - int main (int argc, char *argv[]) { @@ -111,14 +54,13 @@ main (int argc, char *argv[]) float *A = (float *)malloc_safe(m * k * sizeof(FLOAT)); float *B = (float *)malloc_safe(k * n * sizeof(FLOAT)); float *C = (float *)malloc_safe(m * n * sizeof(FLOAT)); - bfloat16_bits *AA = (bfloat16_bits *)malloc_safe(m * k * sizeof(bfloat16_bits)); - bfloat16_bits *BB = (bfloat16_bits *)malloc_safe(k * n * sizeof(bfloat16_bits)); + bfloat16 *AA = (bfloat16 *)malloc_safe(m * k * sizeof(bfloat16)); + bfloat16 *BB = (bfloat16 *)malloc_safe(k * n * sizeof(bfloat16)); float *DD = (float *)malloc_safe(m * n * sizeof(FLOAT)); float *CC = (float *)malloc_safe(m * n * sizeof(FLOAT)); if ((A == NULL) || (B == NULL) || (C == NULL) || (AA == NULL) || (BB == NULL) || (DD == NULL) || (CC == NULL)) return 1; - bfloat16 atmp,btmp; blasint one=1; for (j = 0; j < m; j++) @@ -126,8 +68,7 @@ main (int argc, char *argv[]) for (i = 0; i < k; i++) { A[j * k + i] = ((FLOAT) rand () / (FLOAT) RAND_MAX) + 0.5; - sbstobf16_(&one, &A[j*k+i], &one, &atmp, &one); - AA[j * k + i].v = atmp; + sbstobf16_(&one, &A[j*k+i], &one, &AA[j * k + i], &one); } } for (j = 0; j < n; j++) @@ -135,8 +76,7 @@ main (int argc, char *argv[]) for (i = 0; i < k; i++) { B[j * k + i] = ((FLOAT) rand () / (FLOAT) RAND_MAX) + 0.5; - sbstobf16_(&one, &B[j*k+i], &one, &btmp, &one); - BB[j * k + i].v = btmp; + sbstobf16_(&one, &B[j*k+i], &one, &BB[j * k + i], &one); } } for (y = 0; y < 4; y++) @@ -182,10 +122,12 @@ main (int argc, char *argv[]) DD[i * m + j] += float16to32 (AA[k * j + l]) * float16to32 (BB[i + l * n]); } - if (fabs (CC[i * m + j] - C[i * m + j]) > 1.0) + if (!is_close(CC[i * m + j], C[i * m + j], 0.01, 0.001)) { ret++; - if (fabs (CC[i * m + j] - DD[i * m + j]) > 1.0) + } + if (!is_close(CC[i * m + j], DD[i * m + j], 0.001, 0.0001)) { ret++; + } } } free(A); @@ -199,88 +141,7 @@ main (int argc, char *argv[]) if (ret != 0) { fprintf (stderr, "FATAL ERROR SBGEMM - Return code: %d\n", ret); - return ret; } - for (beta = 0; beta < 3; beta += 1) { - for (alpha = 0; alpha < 3; alpha += 1) { - for (l = 0; l < 2; l++) { // l = 1 to test inc_x & inc_y not equal to one. - for (x = 1; x <= loop; x++) - { - k = (x == 0) ? 0 : l + 1; - float *A = (float *)malloc_safe(x * x * sizeof(FLOAT)); - float *B = (float *)malloc_safe(x * sizeof(FLOAT) << l); - float *C = (float *)malloc_safe(x * sizeof(FLOAT) << l); - bfloat16_bits *AA = (bfloat16_bits *)malloc_safe(x * x * sizeof(bfloat16_bits)); - bfloat16_bits *BB = (bfloat16_bits *)malloc_safe(x * sizeof(bfloat16_bits) << l); - float *DD = (float *)malloc_safe(x * sizeof(FLOAT)); - float *CC = (float *)malloc_safe(x * sizeof(FLOAT) << l); - if ((A == NULL) || (B == NULL) || (C == NULL) || (AA == NULL) || (BB == NULL) || - (DD == NULL) || (CC == NULL)) - return 1; - bfloat16 atmp, btmp; - blasint one = 1; - - for (j = 0; j < x; j++) - { - for (i = 0; i < x; i++) - { - A[j * x + i] = ((FLOAT) rand () / (FLOAT) RAND_MAX) + 0.5; - sbstobf16_(&one, &A[j*x+i], &one, &atmp, &one); - AA[j * x + i].v = atmp; - } - B[j << l] = ((FLOAT) rand () / (FLOAT) RAND_MAX) + 0.5; - sbstobf16_(&one, &B[j << l], &one, &btmp, &one); - BB[j << l].v = btmp; - - CC[j << l] = C[j << l] = ((FLOAT) rand () / (FLOAT) RAND_MAX) + 0.5; - } - - for (y = 0; y < 2; y++) - { - if (y == 0) { - transA = 'N'; - } else { - transA = 'T'; - } - - memset(CC, 0, x * sizeof(FLOAT) << l); - memset(DD, 0, x * sizeof(FLOAT)); - memset(C, 0, x * sizeof(FLOAT) << l); - - SGEMV (&transA, &x, &x, &alpha, A, &x, B, &k, &beta, C, &k); - SBGEMV (&transA, &x, &x, &alpha, (bfloat16*) AA, &x, (bfloat16*) BB, &k, &beta, CC, &k); - - for (int i = 0; i < x; i ++) DD[i] *= beta; - - for (j = 0; j < x; j++) - for (i = 0; i < x; i++) - if (transA == 'N') { - DD[i] += alpha * float16to32 (AA[j * x + i]) * float16to32 (BB[j << l]); - } else if (transA == 'T') { - DD[j] += alpha * float16to32 (AA[j * x + i]) * float16to32 (BB[i << l]); - } - - for (j = 0; j < x; j++) { - if (fabs (CC[j << l] - C[j << l]) > 1.0) - ret++; - if (fabs (CC[j << l] - DD[j]) > 1.0) - ret++; - } - } - free(A); - free(B); - free(C); - free(AA); - free(BB); - free(DD); - free(CC); - } // x - } // l - } // alpha - } // beta - - if (ret != 0) - fprintf (stderr, "FATAL ERROR SBGEMV - Return code: %d\n", ret); return ret; } diff --git a/test/compare_sgemv_bgemv.c b/test/compare_sgemv_bgemv.c new file mode 100644 index 000000000..aac98760f --- /dev/null +++ b/test/compare_sgemv_bgemv.c @@ -0,0 +1,149 @@ +/*************************************************************************** +Copyright (c) 2020,2025 The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ +#include +#include +#include "../common.h" + +#include "test_helpers.h" + +#define SGEMV BLASFUNC(sgemv) +#define BGEMV BLASFUNC(bgemv) +#define BGEMV_LARGEST 256 + +int main(int argc, char *argv[]) +{ + blasint k; + int i, j, l; + blasint x, y; + blasint one = 1; + int ret = 0; + int loop = BGEMV_LARGEST; + char transA = 'N'; + float alpha = 1.0, beta = 0.0; + bfloat16 alpha_bf16, beta_bf16; + + for (beta = 0; beta < 3; beta += 1) + { + for (alpha = 0; alpha < 3; alpha += 1) + { + for (l = 0; l < 2; l++) + { // l = 1 to test inc_x & inc_y not equal to one. + for (x = 1; x <= loop; x++) + { + k = (x == 0) ? 0 : l + 1; + float *A = (float *)malloc_safe(x * x * sizeof(FLOAT)); + float *B = (float *)malloc_safe(x * sizeof(FLOAT) << l); + float *C = (float *)malloc_safe(x * sizeof(FLOAT) << l); + bfloat16 *AA = (bfloat16 *)malloc_safe(x * x * sizeof(bfloat16)); + bfloat16 *BB = (bfloat16 *)malloc_safe(x * sizeof(bfloat16) << l); + bfloat16 *CC = (bfloat16 *)malloc_safe(x * sizeof(bfloat16) << l); + float *DD = (float *)malloc_safe(x * sizeof(FLOAT)); + if ((A == NULL) || (B == NULL) || (C == NULL) || (AA == NULL) || (BB == NULL) || + (CC == NULL) || (DD == NULL)) + return 1; + + for (j = 0; j < x; j++) + { + for (i = 0; i < x; i++) + { + A[j * x + i] = ((FLOAT)rand() / (FLOAT)RAND_MAX) + 0.5; + sbstobf16_(&one, &A[j * x + i], &one, &AA[j * x + i], &one); + } + B[j << l] = ((FLOAT)rand() / (FLOAT)RAND_MAX) + 0.5; + sbstobf16_(&one, &B[j << l], &one, &BB[j << l], &one); + + C[j << l] = ((FLOAT)rand() / (FLOAT)RAND_MAX) + 0.5; + sbstobf16_(&one, &B[j << l], &one, &CC[j << l], &one); + } + + for (y = 0; y < 2; y++) + { + if (y == 0) + { + transA = 'N'; + } + else + { + transA = 'T'; + } + + memset(C, 0, x * sizeof(FLOAT) << l); + memset(CC, 0, x * sizeof(bfloat16) << l); + memset(DD, 0, x * sizeof(FLOAT)); + + sbstobf16_(&one, &alpha, &one, &alpha_bf16, &one); + sbstobf16_(&one, &beta, &one, &beta_bf16, &one); + SGEMV(&transA, &x, &x, &alpha, A, &x, B, &k, &beta, C, &k); + BGEMV(&transA, &x, &x, &alpha_bf16, AA, &x, BB, &k, &beta_bf16, CC, &k); + + for (int i = 0; i < x; i++) + DD[i] *= beta; + + for (j = 0; j < x; j++) + for (i = 0; i < x; i++) + if (transA == 'N') + { + DD[i] += alpha * float16to32(AA[j * x + i]) * float16to32(BB[j << l]); + } + else if (transA == 'T') + { + DD[j] += alpha * float16to32(AA[j * x + i]) * float16to32(BB[i << l]); + } + + for (j = 0; j < x; j++) + { + if (!is_close(float16to32(CC[j << l]), truncate_float32_to_bfloat16(C[j << l]), 0.01, 0.001)) + { + printf("Mismatch at trans=%c, alpha=%.2f, beta=%.2f, i=%d, j=%d, k=%d: CC=%.6f, C=%.6f\n", + transA, alpha, beta, i, j, k, float16to32(CC[j << l]), truncate_float32_to_bfloat16(C[j << l])); + ret++; + } + if (!is_close(float16to32(CC[j << l]), truncate_float32_to_bfloat16(DD[j]), 0.001, 0.0001)) + { + printf("Mismatch at trans=%c, alpha=%.2f, beta=%.2f, i=%d, j=%d, k=%d: CC=%.6f, C=%.6f\n", + transA, alpha, beta, i, j, k, float16to32(CC[j << l]), truncate_float32_to_bfloat16(DD[j])); + ret++; + } + } + } + + free(A); + free(B); + free(C); + free(AA); + free(BB); + free(CC); + free(DD); + } // x + } // l + } // alpha + } // beta + + if (ret != 0) + fprintf(stderr, "FATAL ERROR BGEMV - Return code: %d\n", ret); + return ret; +} diff --git a/test/compare_sgemv_sbgemv.c b/test/compare_sgemv_sbgemv.c new file mode 100644 index 000000000..15cdce6cb --- /dev/null +++ b/test/compare_sgemv_sbgemv.c @@ -0,0 +1,128 @@ +/*************************************************************************** +Copyright (c) 2020,2025 The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ +#include +#include +#include "../common.h" + +#include "test_helpers.h" + +#define SGEMV BLASFUNC(sgemv) +#define SBGEMV BLASFUNC(sbgemv) +#define SBGEMV_LARGEST 256 + +int +main (int argc, char *argv[]) +{ + blasint k; + int i, j, l; + blasint x, y; + int ret = 0; + int loop = SBGEMV_LARGEST; + char transA = 'N'; + float alpha = 1.0, beta = 0.0; + + for (beta = 0; beta < 3; beta += 1) { + for (alpha = 0; alpha < 3; alpha += 1) { + for (l = 0; l < 2; l++) { // l = 1 to test inc_x & inc_y not equal to one. + for (x = 1; x <= loop; x++) + { + k = (x == 0) ? 0 : l + 1; + float *A = (float *)malloc_safe(x * x * sizeof(FLOAT)); + float *B = (float *)malloc_safe(x * sizeof(FLOAT) << l); + float *C = (float *)malloc_safe(x * sizeof(FLOAT) << l); + bfloat16 *AA = (bfloat16 *)malloc_safe(x * x * sizeof(bfloat16)); + bfloat16 *BB = (bfloat16 *)malloc_safe(x * sizeof(bfloat16) << l); + float *CC = (float *)malloc_safe(x * sizeof(FLOAT) << l); + float *DD = (float *)malloc_safe(x * sizeof(FLOAT)); + if ((A == NULL) || (B == NULL) || (C == NULL) || (AA == NULL) || (BB == NULL) || + (DD == NULL) || (CC == NULL)) + return 1; + blasint one = 1; + + for (j = 0; j < x; j++) + { + for (i = 0; i < x; i++) + { + A[j * x + i] = ((FLOAT) rand () / (FLOAT) RAND_MAX) + 0.5; + sbstobf16_(&one, &A[j*x+i], &one, &AA[j * x + i], &one); + } + B[j << l] = ((FLOAT) rand () / (FLOAT) RAND_MAX) + 0.5; + sbstobf16_(&one, &B[j << l], &one, &BB[j << l], &one); + + CC[j << l] = C[j << l] = ((FLOAT) rand () / (FLOAT) RAND_MAX) + 0.5; + } + + for (y = 0; y < 2; y++) + { + if (y == 0) { + transA = 'N'; + } else { + transA = 'T'; + } + + memset(CC, 0, x * sizeof(FLOAT) << l); + memset(DD, 0, x * sizeof(FLOAT)); + memset(C, 0, x * sizeof(FLOAT) << l); + + SGEMV (&transA, &x, &x, &alpha, A, &x, B, &k, &beta, C, &k); + SBGEMV (&transA, &x, &x, &alpha, (bfloat16*) AA, &x, (bfloat16*) BB, &k, &beta, CC, &k); + + for (int i = 0; i < x; i ++) DD[i] *= beta; + + for (j = 0; j < x; j++) + for (i = 0; i < x; i++) + if (transA == 'N') { + DD[i] += alpha * float16to32 (AA[j * x + i]) * float16to32 (BB[j << l]); + } else if (transA == 'T') { + DD[j] += alpha * float16to32 (AA[j * x + i]) * float16to32 (BB[i << l]); + } + + for (j = 0; j < x; j++) { + if (!is_close(CC[j << l], C[j << l], 0.01, 0.001)) { + ret++; + } + if (!is_close(CC[j << l], DD[j], 0.001, 0.0001)) { + ret++; + } + } + } + free(A); + free(B); + free(C); + free(AA); + free(BB); + free(DD); + free(CC); + } // x + } // l + } // alpha + } // beta + + if (ret != 0) + fprintf (stderr, "FATAL ERROR SBGEMV - Return code: %d\n", ret); + return ret; +} diff --git a/test/test_helpers.h b/test/test_helpers.h new file mode 100644 index 000000000..fcec86e10 --- /dev/null +++ b/test/test_helpers.h @@ -0,0 +1,66 @@ +/*************************************************************************** +Copyright (c) 2025 The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + +#ifndef TEST_HELPERS_H +#define TEST_HELPERS_H +#include + +#include "../common.h" + +#ifdef IBFLOAT16 +static float float16to32(bfloat16 value) +{ + blasint one = 1; + float result; + sbf16tos_(&one, &value, &one, &result, &one); + return result; +} +#endif + +#ifdef OBFLOAT16 +static float truncate_float32_to_bfloat16(float value) { + blasint one = 1; + bfloat16 tmp; + float result; + sbstobf16_(&one, &value, &one, &tmp, &one); + sbf16tos_(&one, &tmp, &one, &result, &one); + return result; +} +#endif + +static void *malloc_safe(size_t size) { + if (size == 0) + return malloc(1); + else + return malloc(size); +} + +static bool is_close(float a, float b, float rtol, float atol) { + return fabs(a - b) <= (atol + rtol*fabs(b)); +} + +#endif