OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
Browse Source

Merge pull request #99 from xianyi/develop 

rebase
tags/v0.3.11^2
Martin Kroeker GitHub 5 years ago
parent
a0ada4bcb8 756802df61
commit
a897bc3bd2
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 1993 additions and 396 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +5
    -1
      Makefile.x86_64
  2. +16
    -1
      cmake/cc.cmake
  3. +1
    -1
      ctest/CMakeLists.txt
  4. +5
    -2
      kernel/Makefile
  5. +4
    -4
      kernel/power/KERNEL.POWER10
  6. +196
    -281
      kernel/power/sbgemm_kernel_power10.c
  7. +437
    -0
      kernel/power/sbgemm_ncopy_16_power10.c
  8. +383
    -0
      kernel/power/sbgemm_ncopy_8_power10.c
  9. +244
    -0
      kernel/power/sbgemm_tcopy_16_power10.c
  10. +659
    -0
      kernel/power/sbgemm_tcopy_8_power10.c
  11. +3
    -8
      kernel/x86_64/KERNEL
  12. +1
    -1
      lapack/CMakeLists.txt
  13. +39
    -97
      test/Makefile

+ 5
- 1
Makefile.x86_64 View File

@@ -12,6 +12,10 @@ ifdef HAVE_SSE3
ifndef DYNAMIC_ARCH
CCOMMON_OPT += -msse3
FCOMMON_OPT += -msse3
ifdef HAVE_SSSE3
CCOMMON_OPT += -mssse3
FCOMMON_OPT += -mssse3
endif
endif
endif

@@ -60,7 +64,7 @@ endif
endif
endif

ifeq ($(CORE), HASWELL)
ifeq ($(CORE), $(filter $(CORE), HASWELL ZEN SKYLAKEX COOPERLAKE))
ifndef DYNAMIC_ARCH
ifndef NO_AVX2
ifeq ($(C_COMPILER), GCC)


+ 16
- 1
cmake/cc.cmake View File

@@ -109,10 +109,25 @@ if (${CORE} STREQUAL "COOPERLAKE")
if (NOT NO_AVX512)
execute_process(COMMAND ${CMAKE_C_COMPILER} -dumpversion OUTPUT_VARIABLE GCC_VERSION)
if (${GCC_VERSION} VERSION_GREATER 10.1 OR ${GCC_VERSION} VERSION_EQUAL 10.1)
set (CCOMMON_OPT = "${CCOMMON_OPT} -march=cooperlake")
set (CCOMMON_OPT "${CCOMMON_OPT} -march=cooperlake")
else ()
set (CCOMMON_OPT "${CCOMMON_OPT} -march=skylake-avx512")
endif()
endif ()
endif ()
endif ()

if (NOT DYNAMIC_ARCH)
if (HAVE_AVX2)
set (CCOMMON_OPT "${CCOMMON_OPT} -mavx2")
endif ()
if (HAVE_AVX)
set (CCOMMON_OPT "${CCOMMON_OPT} -mavx")
endif ()
if (HAVE_SSE3)
set (CCOMMON_OPT "${CCOMMON_OPT} -msse3")
endif ()
if (HAVE_SSSE3)
set (CCOMMON_OPT "${CCOMMON_OPT} -mssse3")
endif ()
endif()

+ 1
- 1
ctest/CMakeLists.txt View File

@@ -12,7 +12,7 @@ FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/test_cblas_helper.sh
foreach(float_type ${FLOAT_TYPES})
string(SUBSTRING ${float_type} 0 1 float_char_upper)
string(TOLOWER ${float_char_upper} float_char)
if (${float_char} STREQUAL "h")
if (${float_char} STREQUAL "b")
continue()
endif()
#level1


+ 5
- 2
kernel/Makefile View File

@@ -8,6 +8,9 @@ include $(TOPDIR)/Makefile.system
ifdef HAVE_SSE3
CFLAGS += -msse3
endif
ifdef HAVE_SSSE3
CFLAGS += -mssse3
endif

ifeq ($(C_COMPILER), GCC)
GCCVERSIONGTEQ9 := $(shell expr `$(CC) -dumpversion | cut -f1 -d.` \>= 9)
@@ -41,8 +44,8 @@ ifdef NO_AVX2
endif

ifdef TARGET_CORE
ifeq ($(TARGET_CORE), $(filter $(TARGET_CORE),PRESCOTT CORE2 PENRYN DUNNINGTON ATOM NANO NEHALEM BARCELONA BOBCAT BULLDOZER PILEDRIVER EXCAVATOR STEAMROLLER OPTERON_SSE3))
override CFLAGS += -msse3
ifeq ($(TARGET_CORE), $(filter $(TARGET_CORE),PRESCOTT CORE2 PENRYN DUNNINGTON ATOM NANO SANDYBRIDGE HASWELL NEHALEM ZEN BARCELONA BOBCAT BULLDOZER PILEDRIVER EXCAVATOR STEAMROLLER OPTERON_SSE3))
override CFLAGS += -msse3 -mssse3
endif
ifeq ($(TARGET_CORE), COOPERLAKE)
override CFLAGS += -DBUILD_KERNEL -DTABLE_NAME=gotoblas_$(TARGET_CORE)


+ 4
- 4
kernel/power/KERNEL.POWER10 View File

@@ -9,10 +9,10 @@ else

SBGEMM_BETA = ../generic/gemm_beta.c
SBGEMMKERNEL = sbgemm_kernel_power10.c
SBGEMMINCOPY = ../generic/gemm_ncopy_16.c
SBGEMMITCOPY = ../generic/gemm_tcopy_16.c
SBGEMMONCOPY = ../generic/gemm_ncopy_8.c
SBGEMMOTCOPY = ../generic/gemm_tcopy_8.c
SBGEMMINCOPY = sbgemm_ncopy_16_power10.c
SBGEMMITCOPY = sbgemm_tcopy_16_power10.c
SBGEMMONCOPY = sbgemm_ncopy_8_power10.c
SBGEMMOTCOPY = sbgemm_tcopy_8_power10.c
SBGEMMINCOPYOBJ = sbgemm_incopy$(TSUFFIX).$(SUFFIX)
SBGEMMITCOPYOBJ = sbgemm_itcopy$(TSUFFIX).$(SUFFIX)
SBGEMMONCOPYOBJ = sbgemm_oncopy$(TSUFFIX).$(SUFFIX)


+ 196
- 281
kernel/power/sbgemm_kernel_power10.c View File

@@ -137,15 +137,13 @@ int
CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
IFLOAT * B, FLOAT * C, BLASLONG ldc)
{
BLASLONG N = n;
BLASLONG i1;
v4sf_t valpha = { alpha, alpha, alpha, alpha };
vector short vzero = { 0, 0, 0, 0, 0, 0, 0, 0 };
N = n >> 3;
/* Loop for n >= 8. */
for (i1 = 0; i1 < N; i1++)
for (i1 = 0; i1 < (n >> 3); i1++)
{
BLASLONG i, j;
BLASLONG j;
FLOAT *CO;
IFLOAT *AO;
CO = C;
@@ -153,9 +151,8 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
AO = A;
PREFETCH1 (A, 128);
PREFETCH1 (A, 256);
i = m >> 4;
/* Loop for m >= 16. */
for (j = 0; j < i; j++)
for (j = 0; j < (m >> 4); j++)
{
IFLOAT *BO = B;
v4sf_t *rowC;
@@ -167,20 +164,14 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
{
vec_t *rowA = (vec_t *) & (AO[l << 5]);
vec_t *rowB = (vec_t *) & (BO[l << 4]);
vec_t rowB_h = MERGE_HIGH (rowB[0], rowB[1]);
vec_t rowB_l = MERGE_LOW (rowB[0], rowB[1]);
vec_t rowA_h = MERGE_HIGH (rowA[0], rowA[2]);
vec_t rowA_l = MERGE_LOW (rowA[0], rowA[2]);
vec_t rowA2_h = MERGE_HIGH (rowA[1], rowA[3]);
vec_t rowA2_l = MERGE_LOW (rowA[1], rowA[3]);
MMA (&acc0, rowB_h, rowA_h);
MMA (&acc1, rowB_l, rowA_h);
MMA (&acc2, rowB_h, rowA_l);
MMA (&acc3, rowB_l, rowA_l);
MMA (&acc4, rowB_h, rowA2_h);
MMA (&acc5, rowB_l, rowA2_h);
MMA (&acc6, rowB_h, rowA2_l);
MMA (&acc7, rowB_l, rowA2_l);
MMA (&acc0, rowB[0], rowA[0]);
MMA (&acc1, rowB[1], rowA[0]);
MMA (&acc2, rowB[0], rowA[1]);
MMA (&acc3, rowB[1], rowA[1]);
MMA (&acc4, rowB[0], rowA[2]);
MMA (&acc5, rowB[1], rowA[2]);
MMA (&acc6, rowB[0], rowA[3]);
MMA (&acc7, rowB[1], rowA[3]);
}
if (k % 2 == 1)
{
@@ -216,9 +207,7 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
AO += (k << 4);
BO += (k << 3);
}
i = (m & 15) >> 3;
/* Loop for m >= 8. */
for (j = 0; j < i; j++)
if (m & 8)
{
IFLOAT *BO = B;
v4sf_t *rowC;
@@ -230,14 +219,11 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
{
vec_t *rowA = (vec_t *) & (AO[l << 4]);
vec_t *rowB = (vec_t *) & (BO[l << 4]);
vec_t rowB_h = MERGE_HIGH (rowB[0], rowB[1]);
vec_t rowB_l = MERGE_LOW (rowB[0], rowB[1]);
vec_t rowA_h = MERGE_HIGH (rowA[0], rowA[1]);
vec_t rowA_l = MERGE_LOW (rowA[0], rowA[1]);
MMA (&acc0, rowB_h, rowA_h);
MMA (&acc1, rowB_l, rowA_h);
MMA (&acc2, rowB_h, rowA_l);
MMA (&acc3, rowB_l, rowA_l);

MMA (&acc0, rowB[0], rowA[0]);
MMA (&acc1, rowB[1], rowA[0]);
MMA (&acc2, rowB[0], rowA[1]);
MMA (&acc3, rowB[1], rowA[1]);
}
if (k % 2 == 1)
{
@@ -262,9 +248,7 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
AO += (k << 3);
BO += (k << 3);
}
i = (m & 7) >> 2;
/* Loop for m >= 4. */
for (j = 0; j < i; j++)
if (m & 4)
{
IFLOAT *BO = B;
v4sf_t *rowC;
@@ -277,9 +261,8 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
{
vec_t *rowA = (vec_t *) & (AO[l << 3]);
vec_t *rowB = (vec_t *) & (BO[l << 4]);
vec_t rowA_mrg = MERGE_ROW (rowA[0]);
MMA (&acc0, MERGE_HIGH (rowB[0], rowB[1]), rowA_mrg);
MMA (&acc1, MERGE_LOW (rowB[0], rowB[1]), rowA_mrg);
MMA (&acc0, rowB[0], rowA[0]);
MMA (&acc1, rowB[1], rowA[0]);
}
if (k % 2 == 1)
{
@@ -297,9 +280,7 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
AO += (k << 2);
BO += (k << 3);
}
i = (m & 3) >> 1;
/* Loop for m >= 2. */
for (j = 0; j < i; j++)
if (m & 2)
{
IFLOAT *BO = B;
v2sf_t *rowC;
@@ -316,8 +297,8 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
0, 0, 0, 0
};
vec_t *rowB = (vec_t *) & (BO[l << 4]);
MMA (&acc0, MERGE_HIGH (rowB[0], rowB[1]), (vec_t) rowA);
MMA (&acc1, MERGE_LOW (rowB[0], rowB[1]), (vec_t) rowA);
MMA (&acc0, rowB[0], (vec_t) rowA);
MMA (&acc1, rowB[1], (vec_t) rowA);
}
if (k % 2 == 1)
{
@@ -334,64 +315,50 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
AO += (k << 1);
BO += (k << 3);
}
i = (m & 1) >> 0;
/* Loop for m = 1. */
for (j = 0; j < i; j++)
if (m & 1)
{
IFLOAT *BO = B;
v2sf_t *rowC;
v2sf_t result[8];
__vector_quad acc0, acc1;
__builtin_mma_xxsetaccz (&acc0);
__builtin_mma_xxsetaccz (&acc1);
BLASLONG l = 0;
v4sf_t t = { 0, 0, 0, 0 }
, t1 =
{
0, 0, 0, 0};
for (l = 0; l < k; l++)
for (l = 0; l < k / 2; l++)
{
v4sf_t rowA =
{ BF16TOF32 (AO[l]), BF16TOF32 (AO[l]), BF16TOF32 (AO[l]),
BF16TOF32 (AO[l])
};
v4sf_t rowB =
{ BF16TOF32 (BO[l << 3]), BF16TOF32 (BO[(l << 3) + 1]),
BF16TOF32 (BO[(l << 3) + 2]),
BF16TOF32 (BO[(l << 3) + 3])
};
v4sf_t rowB1 =
{ BF16TOF32 (BO[(l << 3) + 4]), BF16TOF32 (BO[(l << 3) + 5]),
BF16TOF32 (BO[(l << 3) + 6]),
BF16TOF32 (BO[(l << 3) + 7])
};
t += rowA * rowB;
t1 += rowA * rowB1;
vector short rowA =
{ AO[(l << 1) + 0], AO[(l << 1) + 1], 0, 0, 0, 0, 0, 0};
vec_t *rowB = (vec_t *) & (BO[l << 4]);
MMA (&acc0, rowB[0], (vec_t) rowA);
MMA (&acc1, rowB[1], (vec_t) rowA);
}
t = t * valpha;
t1 = t1 * valpha;
CO[0 * ldc] += t[0];
CO[1 * ldc] += t[1];
CO[2 * ldc] += t[2];
CO[3 * ldc] += t[3];
CO[4 * ldc] += t1[0];
CO[5 * ldc] += t1[1];
CO[6 * ldc] += t1[2];
CO[7 * ldc] += t1[3];
if (k % 2 == 1)
{
if (k > 1)
l = (k / 2) << 1;
vector short rowA = { AO[l], 0, 0, 0, 0, 0, 0, 0 };
vec_t *rowB = (vec_t *) & (BO[(l << 3)]);
MMA (&acc0, MERGE_HIGH (rowB[0], rowB[1]), (vec_t) rowA);
MMA (&acc1, MERGE_LOW (rowB[0], rowB[1]), (vec_t) rowA);
}
SAVE4x2_ACC (&acc0, 0);
SAVE4x2_ACC1 (&acc1, 0);
CO += 1;
AO += k;
BO += (k << 3);
}
B += k << 3;
}
N = (n & 7) >> 2;
/* Loop for n >= 4. */
for (i1 = 0; i1 < N; i1++)
if (n & 4)
{
BLASLONG i, j;
BLASLONG j;
FLOAT *CO;
IFLOAT *AO;
CO = C;
C += ldc << 2;
AO = A;
i = m >> 5;
/* Loop for m >= 32. */
for (j = 0; j < i; j++)
for (j = 0; j < (m >> 5); j++)
{
IFLOAT *BO = B;
IFLOAT *A1 = AO + (16 * k);
@@ -405,15 +372,14 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
vec_t *rowA = (vec_t *) & (AO[l << 5]);
vec_t *rowA1 = (vec_t *) & (A1[l << 5]);
vec_t *rowB = (vec_t *) & (BO[l << 3]);
vec_t rowB_mrg = MERGE_ROW (rowB[0]);
MMA (&acc0, rowB_mrg, MERGE_HIGH (rowA[0], rowA[2]));
MMA (&acc1, rowB_mrg, MERGE_LOW (rowA[0], rowA[2]));
MMA (&acc2, rowB_mrg, MERGE_HIGH (rowA[1], rowA[3]));
MMA (&acc3, rowB_mrg, MERGE_LOW (rowA[1], rowA[3]));
MMA (&acc4, rowB_mrg, MERGE_HIGH (rowA1[0], rowA1[2]));
MMA (&acc5, rowB_mrg, MERGE_LOW (rowA1[0], rowA1[2]));
MMA (&acc6, rowB_mrg, MERGE_HIGH (rowA1[1], rowA1[3]));
MMA (&acc7, rowB_mrg, MERGE_LOW (rowA1[1], rowA1[3]));
MMA (&acc0, rowB[0], rowA[0]);
MMA (&acc1, rowB[0], rowA[1]);
MMA (&acc2, rowB[0], rowA[2]);
MMA (&acc3, rowB[0], rowA[3]);
MMA (&acc4, rowB[0], rowA1[0]);
MMA (&acc5, rowB[0], rowA1[1]);
MMA (&acc6, rowB[0], rowA1[2]);
MMA (&acc7, rowB[0], rowA1[3]);
}
if (k % 2 == 1)
{
@@ -448,9 +414,7 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
AO += k << 5;
BO += k << 2;
}
i = (m & 31) >> 4;
/* Loop for m >= 16. */
for (j = 0; j < i; j++)
if (m & 16)
{
IFLOAT *BO = B;
v4sf_t *rowC;
@@ -462,11 +426,10 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
{
vec_t *rowA = (vec_t *) & (AO[l << 5]);
vec_t *rowB = (vec_t *) & (BO[l << 3]);
vec_t rowB_mrg = MERGE_ROW (rowB[0]);
MMA (&acc0, rowB_mrg, MERGE_HIGH (rowA[0], rowA[2]));
MMA (&acc1, rowB_mrg, MERGE_LOW (rowA[0], rowA[2]));
MMA (&acc2, rowB_mrg, MERGE_HIGH (rowA[1], rowA[3]));
MMA (&acc3, rowB_mrg, MERGE_LOW (rowA[1], rowA[3]));
MMA (&acc0, rowB[0], rowA[0]);
MMA (&acc1, rowB[0], rowA[1]);
MMA (&acc2, rowB[0], rowA[2]);
MMA (&acc3, rowB[0], rowA[3]);
}
if (k % 2 == 1)
{
@@ -490,9 +453,7 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
AO += k << 4;
BO += k << 2;
}
i = (m & 15) >> 3;
/* Loop for m >= 8. */
for (j = 0; j < i; j++)
if (m & 8)
{
IFLOAT *BO = B;
v4sf_t *rowC;
@@ -505,9 +466,8 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
{
vec_t *rowA = (vec_t *) & (AO[l << 4]);
vec_t *rowB = (vec_t *) & (BO[l << 3]);
vec_t rowB_mrg = MERGE_ROW (rowB[0]);
MMA (&acc0, rowB_mrg, MERGE_HIGH (rowA[0], rowA[1]));
MMA (&acc1, rowB_mrg, MERGE_LOW (rowA[0], rowA[1]));
MMA (&acc0, rowB[0], rowA[0]);
MMA (&acc1, rowB[0], rowA[1]);
}
if (k % 2 == 1)
{
@@ -525,9 +485,7 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
AO += k << 3;
BO += k << 2;
}
i = (m & 7) >> 2;
/* Loop for m >= 4. */
for (j = 0; j < i; j++)
if (m & 4)
{
IFLOAT *BO = B;
v4sf_t *rowC;
@@ -539,7 +497,7 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
{
vec_t *rowA = (vec_t *) & (AO[l << 3]);
vec_t *rowB = (vec_t *) & (BO[l << 3]);
MMA (&acc0, MERGE_ROW (rowB[0]), MERGE_ROW (rowA[0]));
MMA (&acc0, rowB[0], rowA[0]);
}
if (k % 2 == 1)
{
@@ -555,9 +513,7 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
AO += k << 2;
BO += k << 2;
}
i = (m & 3) >> 1;
/* Loop for m >= 2. */
for (j = 0; j < i; j++)
if (m & 2)
{
IFLOAT *BO = B;
v2sf_t *rowC;
@@ -573,7 +529,7 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
0, 0, 0, 0
};
vec_t *rowB = (vec_t *) & (BO[l << 3]);
MMA (&acc0, MERGE_ROW (rowB[0]), (vec_t) rowA);
MMA (&acc0, rowB[0], (vec_t) rowA);
}
if (k % 2 == 1)
{
@@ -588,31 +544,32 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
AO += k << 1;
BO += k << 2;
}
i = (m & 1) >> 0;
/* Loop for m = 1. */
for (j = 0; j < i; j++)
if (m & 1)
{
IFLOAT *BO = B;
v2sf_t *rowC;
v2sf_t result[8];
__vector_quad acc0;
BLASLONG l = 0;
v4sf_t t = { 0, 0, 0, 0 };
for (l = 0; l < k; l++)
__builtin_mma_xxsetaccz (&acc0);
for (l = 0; l < k / 2; l++)
{
v4sf_t rowA =
{ BF16TOF32 (AO[l]), BF16TOF32 (AO[l]), BF16TOF32 (AO[l]),
BF16TOF32 (AO[l])
};
v4sf_t rowB =
{ BF16TOF32 (BO[l << 2]), BF16TOF32 (BO[(l << 2) + 1]),
BF16TOF32 (BO[(l << 2) + 2]),
BF16TOF32 (BO[(l << 2) + 3])
vector short rowA =
{ AO[(l << 1) + 0], AO[(l << 1) + 1], 0,
0, 0, 0, 0
};
t += rowA * rowB;
vec_t *rowB = (vec_t *) & (BO[l << 3]);
MMA (&acc0, rowB[0], (vec_t) rowA);
}
t = t * valpha;
CO[0 * ldc] += t[0];
CO[1 * ldc] += t[1];
CO[2 * ldc] += t[2];
CO[3 * ldc] += t[3];
if (k % 2 == 1)
{
if (k > 1)
l = (k / 2) << 1;
vector short rowA = { AO[l], 0, 0, 0, 0, 0, 0, 0 };
vec_t *rowB = (vec_t *) & (BO[l << 2]);
MMA (&acc0, MERGE_ROW (rowB[0]), (vec_t) rowA);
}
SAVE4x2_ACC (&acc0, 0);
AO += k;
BO += (k << 2);
CO += 1;
@@ -620,19 +577,16 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,

B += k << 2;
}
N = (n & 3) >> 1;
/* Loop for n >= 2. */
for (i1 = 0; i1 < N; i1++)
if (n & 2)
{
BLASLONG i, j;
BLASLONG j;
FLOAT *CO;
IFLOAT *AO;
CO = C;
C += ldc << 1;
AO = A;
i = m >> 5;
/* Loop for m >= 32. */
for (j = 0; j < i; j++)
for (j = 0; j < (m >> 5); j++)
{
IFLOAT *BO = B;
v4sf_t *rowC;
@@ -650,14 +604,14 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
};
vec_t *rowA = (vec_t *) & (AO[l << 5]);
vec_t *rowA1 = (vec_t *) & (A1[l << 5]);
MMA (&acc0, (vec_t) rowB, MERGE_HIGH (rowA[0], rowA[2]));
MMA (&acc1, (vec_t) rowB, MERGE_LOW (rowA[0], rowA[2]));
MMA (&acc2, (vec_t) rowB, MERGE_HIGH (rowA[1], rowA[3]));
MMA (&acc3, (vec_t) rowB, MERGE_LOW (rowA[1], rowA[3]));
MMA (&acc4, (vec_t) rowB, MERGE_HIGH (rowA1[0], rowA1[2]));
MMA (&acc5, (vec_t) rowB, MERGE_LOW (rowA1[0], rowA1[2]));
MMA (&acc6, (vec_t) rowB, MERGE_HIGH (rowA1[1], rowA1[3]));
MMA (&acc7, (vec_t) rowB, MERGE_LOW (rowA1[1], rowA1[3]));
MMA (&acc0, (vec_t) rowB, rowA[0]);
MMA (&acc1, (vec_t) rowB, rowA[1]);
MMA (&acc2, (vec_t) rowB, rowA[2]);
MMA (&acc3, (vec_t) rowB, rowA[3]);
MMA (&acc4, (vec_t) rowB, rowA1[0]);
MMA (&acc5, (vec_t) rowB, rowA1[1]);
MMA (&acc6, (vec_t) rowB, rowA1[2]);
MMA (&acc7, (vec_t) rowB, rowA1[3]);
}
if (k % 2 == 1)
{
@@ -688,9 +642,7 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
AO += k << 5;
BO += k << 1;
}
i = (m & 31) >> 4;
/* Loop for m >= 16. */
for (j = 0; j < i; j++)
if (m & 16)
{
IFLOAT *BO = B;
v4sf_t *rowC;
@@ -706,10 +658,10 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
0, 0, 0, 0
};
vec_t *rowA = (vec_t *) & (AO[l << 5]);
MMA (&acc0, (vec_t) rowB, MERGE_HIGH (rowA[0], rowA[2]));
MMA (&acc1, (vec_t) rowB, MERGE_LOW (rowA[0], rowA[2]));
MMA (&acc2, (vec_t) rowB, MERGE_HIGH (rowA[1], rowA[3]));
MMA (&acc3, (vec_t) rowB, MERGE_LOW (rowA[1], rowA[3]));
MMA (&acc0, (vec_t) rowB, rowA[0]);
MMA (&acc1, (vec_t) rowB, rowA[1]);
MMA (&acc2, (vec_t) rowB, rowA[2]);
MMA (&acc3, (vec_t) rowB, rowA[3]);
}
if (k % 2 == 1)
{
@@ -730,9 +682,7 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
AO += k << 4;
BO += k << 1;
}
i = (m & 15) >> 3;
/* Loop for m >= 8. */
for (j = 0; j < i; j++)
if (m & 8)
{
IFLOAT *BO = B;
v4sf_t *rowC;
@@ -749,8 +699,8 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
0, 0, 0, 0
};
vec_t *rowA = (vec_t *) & (AO[l << 4]);
MMA (&acc0, (vec_t) rowB, MERGE_HIGH (rowA[0], rowA[1]));
MMA (&acc1, (vec_t) rowB, MERGE_LOW (rowA[0], rowA[1]));
MMA (&acc0, (vec_t) rowB, rowA[0]);
MMA (&acc1, (vec_t) rowB, rowA[1]);
}
if (k % 2 == 1)
{
@@ -767,9 +717,7 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
AO += k << 3;
BO += k << 1;
}
i = (m & 7) >> 2;
/* Loop for m >= 4. */
for (j = 0; j < i; j++)
if (m & 4)
{
IFLOAT *BO = B;
v4sf_t *rowC;
@@ -785,7 +733,7 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
0, 0, 0, 0
};
vec_t *rowA = (vec_t *) & (AO[l << 3]);
MMA (&acc0, (vec_t) rowB, MERGE_ROW (rowA[0]));
MMA (&acc0, (vec_t) rowB, rowA[0]);
}
if (k % 2 == 1)
{
@@ -800,9 +748,7 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
AO += k << 2;
BO += k << 1;
}
i = (m & 3) >> 1;
/* Loop for m >= 2. */
for (j = 0; j < i; j++)
if (m & 2)
{
IFLOAT *BO = B;
BLASLONG l = 0;
@@ -828,9 +774,7 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
AO += k << 1;
BO += k << 1;
}
i = (m & 1) >> 0;
/* Loop for m = 1. */
for (j = 0; j < i; j++)
if (m & 1)
{
IFLOAT *BO = B;
BLASLONG l = 0;
@@ -852,153 +796,126 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
}
B += k << 1;
}
N = (n & 1) >> 0;
/* Loop for n = 1. */
for (i1 = 0; i1 < N; i1++)
if (n & 1)
{
BLASLONG i;
BLASLONG j;
FLOAT *CO;
IFLOAT *AO;
CO = C;
C += ldc;
AO = A;
i = m;
/* Loop for m >= 16. */
while (i >= 16)
for (j = 0; j < (m >> 4); j++)
{
IFLOAT *BO = B;
v4sf_t *rowC;
v4sf_t result[4];
__vector_quad acc0, acc1, acc2, acc3;
SET_ACC_ZERO4 ();
BLASLONG l = 0;
v4sf_t t = { 0, 0, 0, 0 };
v4sf_t t1 = { 0, 0, 0, 0 };
v4sf_t t2 = { 0, 0, 0, 0 };
v4sf_t t3 = { 0, 0, 0, 0 };
for (l = 0; l < k; l++)
for (l = 0; l < k / 2; l++)
{
v4sf_t rowB =
{ BF16TOF32 (BO[l]), BF16TOF32 (BO[l]), BF16TOF32 (BO[l]),
BF16TOF32 (BO[l])
};
v4sf_t rowA =
{ BF16TOF32 (AO[l << 4]), BF16TOF32 (AO[(l << 4) + 1]),
BF16TOF32 (AO[(l << 4) + 2]),
BF16TOF32 (AO[(l << 4) + 3])
};
v4sf_t rowA1 =
{ BF16TOF32 (AO[(l << 4) + 4]), BF16TOF32 (AO[(l << 4) + 5]),
BF16TOF32 (AO[(l << 4) + 6]),
BF16TOF32 (AO[(l << 4) + 7])
};
v4sf_t rowA2 =
{ BF16TOF32 (AO[(l << 4) + 8]), BF16TOF32 (AO[(l << 4) + 9]),
BF16TOF32 (AO[(l << 4) + 10]),
BF16TOF32 (AO[(l << 4) + 11])
};
v4sf_t rowA3 = { BF16TOF32 (AO[(l << 4) + 12]),
BF16TOF32 (AO[(l << 4) + 13]), BF16TOF32 (AO[(l << 4) + 14]),
BF16TOF32 (AO[(l << 4) + 15])
};
t += rowA * rowB;
t1 += rowA1 * rowB;
t2 += rowA2 * rowB;
t3 += rowA3 * rowB;
vector short rowB =
{ BO[l << 1], BO[(l << 1) + 1], 0, 0, 0, 0, 0, 0};
vec_t *rowA = (vec_t *) & (AO[l << 5]);
MMA (&acc0, (vec_t) rowB, rowA[0]);
MMA (&acc1, (vec_t) rowB, rowA[1]);
MMA (&acc2, (vec_t) rowB, rowA[2]);
MMA (&acc3, (vec_t) rowB, rowA[3]);
}
t = t * valpha;
t1 = t1 * valpha;
t2 = t2 * valpha;
t3 = t3 * valpha;
CO[0] += t[0];
CO[1] += t[1];
CO[2] += t[2];
CO[3] += t[3];
CO[4] += t1[0];
CO[5] += t1[1];
CO[6] += t1[2];
CO[7] += t1[3];
CO[8] += t2[0];
CO[9] += t2[1];
CO[10] += t2[2];
CO[11] += t2[3];
CO[12] += t3[0];
CO[13] += t3[1];
CO[14] += t3[2];
CO[15] += t3[3];
if (k % 2 == 1)
{
if (k > 1)
l = (k / 2) << 1;
vector short rowB = { BO[l], 0, 0, 0, 0, 0, 0, 0 };
vec_t *rowA = (vec_t *) & (AO[(l << 4)]);
MMA (&acc0, (vec_t) rowB, MERGE_HIGH (rowA[0], rowA[2]));
MMA (&acc1, (vec_t) rowB, MERGE_LOW (rowA[0], rowA[2]));
MMA (&acc2, (vec_t) rowB, MERGE_HIGH (rowA[1], rowA[3]));
MMA (&acc3, (vec_t) rowB, MERGE_LOW (rowA[1], rowA[3]));
}
rowC = (v4sf_t *) &CO[0];
__builtin_mma_disassemble_acc ((void *)result, &acc0);
rowC[0] += result[0] * alpha;
__builtin_mma_disassemble_acc ((void *)result, &acc1);
rowC[1] += result[0] * alpha;
__builtin_mma_disassemble_acc ((void *)result, &acc2);
rowC[2] += result[0] * alpha;
__builtin_mma_disassemble_acc ((void *)result, &acc3);
rowC[3] += result[0] * alpha;
AO += k << 4;
BO += k;
CO += 16;
i -= 16;
}
/* Loop for m >= 8. */
while (i >= 8)
if (m & 8)
{
IFLOAT *BO = B;
v4sf_t *rowC;
v4sf_t result[4];
__vector_quad acc0, acc1;
__builtin_mma_xxsetaccz (&acc0);
__builtin_mma_xxsetaccz (&acc1);
BLASLONG l = 0;
v4sf_t t = { 0, 0, 0, 0 };
v4sf_t t1 = { 0, 0, 0, 0 };
for (l = 0; l < k; l++)
for (l = 0; l < k / 2; l++)
{
v4sf_t rowB =
{ BF16TOF32 (BO[l]), BF16TOF32 (BO[l]), BF16TOF32 (BO[l]),
BF16TOF32 (BO[l])
};
v4sf_t rowA =
{ BF16TOF32 (AO[l << 3]), BF16TOF32 (AO[(l << 3) + 1]),
BF16TOF32 (AO[(l << 3) + 2]),
BF16TOF32 (AO[(l << 3) + 3])
};
v4sf_t rowA1 =
{ BF16TOF32 (AO[(l << 3) + 4]), BF16TOF32 (AO[(l << 3) + 5]),
BF16TOF32 (AO[(l << 3) + 6]),
BF16TOF32 (AO[(l << 3) + 7])
};
t += rowA * rowB;
t1 += rowA1 * rowB;
vector short rowB =
{ BO[l << 1], BO[(l << 1) + 1], 0, 0, 0, 0, 0, 0};
vec_t *rowA = (vec_t *) & (AO[l << 4]);
MMA (&acc0, (vec_t) rowB, rowA[0]);
MMA (&acc1, (vec_t) rowB, rowA[1]);
}
t = t * valpha;
t1 = t1 * valpha;
CO[0] += t[0];
CO[1] += t[1];
CO[2] += t[2];
CO[3] += t[3];
CO[4] += t1[0];
CO[5] += t1[1];
CO[6] += t1[2];
CO[7] += t1[3];
if (k % 2 == 1)
{
if (k > 1)
l = (k / 2) << 1;
vector short rowB = { BO[l], 0, 0, 0, 0, 0, 0, 0 };
vec_t *rowA = (vec_t *) & (AO[(l << 3)]);
MMA (&acc0, (vec_t) rowB, MERGE_HIGH (rowA[0], rowA[1]));
MMA (&acc1, (vec_t) rowB, MERGE_LOW (rowA[0], rowA[1]));
}
rowC = (v4sf_t *) &CO[0];
__builtin_mma_disassemble_acc ((void *)result, &acc0);
rowC[0] += result[0] * alpha;
__builtin_mma_disassemble_acc ((void *)result, &acc1);
rowC[1] += result[0] * alpha;
AO += k << 3;
BO += k;
CO += 8;
i -= 8;
}
/* Loop for m >= 4. */
while (i >= 4)
if (m & 4)
{
IFLOAT *BO = B;
v4sf_t *rowC;
v4sf_t result[4];
__vector_quad acc0;
__builtin_mma_xxsetaccz (&acc0);
BLASLONG l = 0;
v4sf_t t = { 0, 0, 0, 0 };
for (l = 0; l < k; l++)
for (l = 0; l < k / 2; l++)
{
v4sf_t rowB =
{ BF16TOF32 (BO[l]), BF16TOF32 (BO[l]), BF16TOF32 (BO[l]),
BF16TOF32 (BO[l])
};
v4sf_t rowA =
{ BF16TOF32 (AO[l << 2]), BF16TOF32 (AO[(l << 2) + 1]),
BF16TOF32 (AO[(l << 2) + 2]),
BF16TOF32 (AO[(l << 2) + 3])
};
t += rowA * rowB;
vector short rowB =
{ BO[l << 1], BO[(l << 1) + 1], 0, 0, 0, 0, 0, 0};
vec_t *rowA = (vec_t *) & (AO[l << 3]);
MMA (&acc0, (vec_t) rowB, rowA[0]);
}
t = t * valpha;
CO[0] += t[0];
CO[1] += t[1];
CO[2] += t[2];
CO[3] += t[3];
if (k % 2 == 1)
{
if (k > 1)
l = (k / 2) << 1;
vector short rowB = { BO[l], 0, 0, 0, 0, 0, 0, 0 };
vec_t *rowA = (vec_t *) & (AO[(l << 2)]);
MMA (&acc0, (vec_t) rowB, MERGE_ROW (rowA[0]));
}
rowC = (v4sf_t *) &CO[0];
__builtin_mma_disassemble_acc ((void *)result, &acc0);
rowC[0] += result[0] * alpha;
AO += k << 2;
BO += k;
CO += 4;
i -= 4;
}
/* Loop for m >= 2. */
while (i >= 2)
if (m & 2)
{
IFLOAT *BO = B;
BLASLONG l = 0;
@@ -1018,10 +935,9 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
AO += k << 1;
BO += k;
CO += 2;
i -= 2;
}
/* Loop for m = 1. */
while (i >= 1)
if (m & 1)
{
IFLOAT *BO = B;
BLASLONG l = 0;
@@ -1034,7 +950,6 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
BO += k;
CO[0] += t * alpha;
CO += 1;
i -= 1;
}

B += k;


+ 437
- 0
kernel/power/sbgemm_ncopy_16_power10.c View File

@@ -0,0 +1,437 @@
/*********************************************************************/
/* Copyright 2009, 2010 The University of Texas at Austin. */
/* All rights reserved. */
/* */
/* Redistribution and use in source and binary forms, with or */
/* without modification, are permitted provided that the following */
/* conditions are met: */
/* */
/* 1. Redistributions of source code must retain the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer. */
/* */
/* 2. Redistributions in binary form must reproduce the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer in the documentation and/or other materials */
/* provided with the distribution. */
/* */
/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */
/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */
/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */
/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */
/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */
/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */
/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */
/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */
/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */
/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */
/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */
/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */
/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */
/* POSSIBILITY OF SUCH DAMAGE. */
/* */
/* The views and conclusions contained in the software and */
/* documentation are those of the authors and should not be */
/* interpreted as representing official policies, either expressed */
/* or implied, of The University of Texas at Austin. */
/*********************************************************************/

#include <stdio.h>
#include "common.h"

int CNAME(BLASLONG m, BLASLONG n, IFLOAT *a, BLASLONG lda, IFLOAT *b){
BLASLONG i, j;

IFLOAT *aoffset;
IFLOAT *aoffset1, *aoffset2, *aoffset3, *aoffset4;
IFLOAT *aoffset5, *aoffset6, *aoffset7, *aoffset8;
IFLOAT *aoffset9, *aoffset10, *aoffset11, *aoffset12;
IFLOAT *aoffset13, *aoffset14, *aoffset15, *aoffset16;

IFLOAT *boffset;
IFLOAT ctemp01, ctemp02, ctemp03, ctemp04;
IFLOAT ctemp05, ctemp06, ctemp07, ctemp08;
IFLOAT ctemp09, ctemp10, ctemp11, ctemp12;
IFLOAT ctemp13, ctemp14, ctemp15, ctemp16;
IFLOAT ctemp17, ctemp18, ctemp19, ctemp20;
IFLOAT ctemp21, ctemp22, ctemp23, ctemp24;
IFLOAT ctemp25, ctemp26, ctemp27, ctemp28;
IFLOAT ctemp29, ctemp30, ctemp31, ctemp32;

aoffset = a;
boffset = b;

j = (n >> 4);
if (j > 0){
do{
aoffset1 = aoffset;
aoffset2 = aoffset1 + lda;
aoffset3 = aoffset2 + lda;
aoffset4 = aoffset3 + lda;
aoffset5 = aoffset4 + lda;
aoffset6 = aoffset5 + lda;
aoffset7 = aoffset6 + lda;
aoffset8 = aoffset7 + lda;
aoffset9 = aoffset8 + lda;
aoffset10 = aoffset9 + lda;
aoffset11 = aoffset10 + lda;
aoffset12 = aoffset11 + lda;
aoffset13 = aoffset12 + lda;
aoffset14 = aoffset13 + lda;
aoffset15 = aoffset14 + lda;
aoffset16 = aoffset15 + lda;
aoffset += 16 * lda;

i = (m >> 1);
if (i > 0){
do{
ctemp01 = *(aoffset1 + 0);
ctemp02 = *(aoffset1 + 1);
ctemp03 = *(aoffset2 + 0);
ctemp04 = *(aoffset2 + 1);

ctemp05 = *(aoffset3 + 0);
ctemp06 = *(aoffset3 + 1);
ctemp07 = *(aoffset4 + 0);
ctemp08 = *(aoffset4 + 1);

ctemp09 = *(aoffset5 + 0);
ctemp10 = *(aoffset5 + 1);
ctemp11 = *(aoffset6 + 0);
ctemp12 = *(aoffset6 + 1);

ctemp13 = *(aoffset7 + 0);
ctemp14 = *(aoffset7 + 1);
ctemp15 = *(aoffset8 + 0);
ctemp16 = *(aoffset8 + 1);

ctemp17 = *(aoffset9 + 0);
ctemp18 = *(aoffset9 + 1);
ctemp19 = *(aoffset10 + 0);
ctemp20 = *(aoffset10 + 1);

ctemp21 = *(aoffset11 + 0);
ctemp22 = *(aoffset11 + 1);
ctemp23 = *(aoffset12 + 0);
ctemp24 = *(aoffset12 + 1);

ctemp25 = *(aoffset13 + 0);
ctemp26 = *(aoffset13 + 1);
ctemp27 = *(aoffset14 + 0);
ctemp28 = *(aoffset14 + 1);

ctemp29 = *(aoffset15 + 0);
ctemp30 = *(aoffset15 + 1);
ctemp31 = *(aoffset16 + 0);
ctemp32 = *(aoffset16 + 1);

*(boffset + 0) = ctemp01;
*(boffset + 1) = ctemp02;
*(boffset + 2) = ctemp03;
*(boffset + 3) = ctemp04;
*(boffset + 4) = ctemp05;
*(boffset + 5) = ctemp06;
*(boffset + 6) = ctemp07;
*(boffset + 7) = ctemp08;

*(boffset + 8) = ctemp09;
*(boffset + 9) = ctemp10;
*(boffset + 10) = ctemp11;
*(boffset + 11) = ctemp12;
*(boffset + 12) = ctemp13;
*(boffset + 13) = ctemp14;
*(boffset + 14) = ctemp15;
*(boffset + 15) = ctemp16;

*(boffset + 16) = ctemp17;
*(boffset + 17) = ctemp18;
*(boffset + 18) = ctemp19;
*(boffset + 19) = ctemp20;
*(boffset + 20) = ctemp21;
*(boffset + 21) = ctemp22;
*(boffset + 22) = ctemp23;
*(boffset + 23) = ctemp24;

*(boffset + 24) = ctemp25;
*(boffset + 25) = ctemp26;
*(boffset + 26) = ctemp27;
*(boffset + 27) = ctemp28;
*(boffset + 28) = ctemp29;
*(boffset + 29) = ctemp30;
*(boffset + 30) = ctemp31;
*(boffset + 31) = ctemp32;

aoffset1 += 2;
aoffset2 += 2;
aoffset3 += 2;
aoffset4 += 2;
aoffset5 += 2;
aoffset6 += 2;
aoffset7 += 2;
aoffset8 += 2;

aoffset9 += 2;
aoffset10 += 2;
aoffset11 += 2;
aoffset12 += 2;
aoffset13 += 2;
aoffset14 += 2;
aoffset15 += 2;
aoffset16 += 2;
boffset += 32;

i --;
}while(i > 0);
}

if (m & 1){
ctemp01 = *(aoffset1 + 0);
ctemp03 = *(aoffset2 + 0);
ctemp05 = *(aoffset3 + 0);
ctemp07 = *(aoffset4 + 0);
ctemp09 = *(aoffset5 + 0);
ctemp11 = *(aoffset6 + 0);
ctemp13 = *(aoffset7 + 0);
ctemp15 = *(aoffset8 + 0);

ctemp17 = *(aoffset9 + 0);
ctemp19 = *(aoffset10 + 0);
ctemp21 = *(aoffset11 + 0);
ctemp23 = *(aoffset12 + 0);
ctemp25 = *(aoffset13 + 0);
ctemp27 = *(aoffset14 + 0);
ctemp29 = *(aoffset15 + 0);
ctemp31 = *(aoffset16 + 0);

*(boffset + 0) = ctemp01;
*(boffset + 1) = ctemp03;
*(boffset + 2) = ctemp05;
*(boffset + 3) = ctemp07;
*(boffset + 4) = ctemp09;
*(boffset + 5) = ctemp11;
*(boffset + 6) = ctemp13;
*(boffset + 7) = ctemp15;

*(boffset + 8) = ctemp17;
*(boffset + 9) = ctemp19;
*(boffset + 10) = ctemp21;
*(boffset + 11) = ctemp23;
*(boffset + 12) = ctemp25;
*(boffset + 13) = ctemp27;
*(boffset + 14) = ctemp29;
*(boffset + 15) = ctemp31;

boffset += 16;
}
j--;
}while(j > 0);
} /* end of if(j > 0) */

if (n & 8){
aoffset1 = aoffset;
aoffset2 = aoffset1 + lda;
aoffset3 = aoffset2 + lda;
aoffset4 = aoffset3 + lda;
aoffset5 = aoffset4 + lda;
aoffset6 = aoffset5 + lda;
aoffset7 = aoffset6 + lda;
aoffset8 = aoffset7 + lda;
aoffset += 8 * lda;

i = (m >> 1);
if (i > 0){
do{
ctemp01 = *(aoffset1 + 0);
ctemp02 = *(aoffset1 + 1);
ctemp03 = *(aoffset2 + 0);
ctemp04 = *(aoffset2 + 1);

ctemp05 = *(aoffset3 + 0);
ctemp06 = *(aoffset3 + 1);
ctemp07 = *(aoffset4 + 0);
ctemp08 = *(aoffset4 + 1);

ctemp09 = *(aoffset5 + 0);
ctemp10 = *(aoffset5 + 1);
ctemp11 = *(aoffset6 + 0);
ctemp12 = *(aoffset6 + 1);

ctemp13 = *(aoffset7 + 0);
ctemp14 = *(aoffset7 + 1);
ctemp15 = *(aoffset8 + 0);
ctemp16 = *(aoffset8 + 1);

*(boffset + 0) = ctemp01;
*(boffset + 1) = ctemp02;
*(boffset + 2) = ctemp03;
*(boffset + 3) = ctemp04;
*(boffset + 4) = ctemp05;
*(boffset + 5) = ctemp06;
*(boffset + 6) = ctemp07;
*(boffset + 7) = ctemp08;

*(boffset + 8) = ctemp09;
*(boffset + 9) = ctemp10;
*(boffset + 10) = ctemp11;
*(boffset + 11) = ctemp12;
*(boffset + 12) = ctemp13;
*(boffset + 13) = ctemp14;
*(boffset + 14) = ctemp15;
*(boffset + 15) = ctemp16;

aoffset1 += 2;
aoffset2 += 2;
aoffset3 += 2;
aoffset4 += 2;
aoffset5 += 2;
aoffset6 += 2;
aoffset7 += 2;
aoffset8 += 2;

boffset += 16;

i --;
}while(i > 0);
}

if (m & 1){
ctemp01 = *(aoffset1 + 0);
ctemp03 = *(aoffset2 + 0);
ctemp05 = *(aoffset3 + 0);
ctemp07 = *(aoffset4 + 0);
ctemp09 = *(aoffset5 + 0);
ctemp11 = *(aoffset6 + 0);
ctemp13 = *(aoffset7 + 0);
ctemp15 = *(aoffset8 + 0);

*(boffset + 0) = ctemp01;
*(boffset + 1) = ctemp03;
*(boffset + 2) = ctemp05;
*(boffset + 3) = ctemp07;
*(boffset + 4) = ctemp09;
*(boffset + 5) = ctemp11;
*(boffset + 6) = ctemp13;
*(boffset + 7) = ctemp15;

boffset += 8;
}
}

if (n & 4){
aoffset1 = aoffset;
aoffset2 = aoffset1 + lda;
aoffset3 = aoffset2 + lda;
aoffset4 = aoffset3 + lda;
aoffset += 4 * lda;

i = (m >> 1);
if (i > 0){
do{
ctemp01 = *(aoffset1 + 0);
ctemp02 = *(aoffset1 + 1);
ctemp03 = *(aoffset2 + 0);
ctemp04 = *(aoffset2 + 1);

ctemp05 = *(aoffset3 + 0);
ctemp06 = *(aoffset3 + 1);
ctemp07 = *(aoffset4 + 0);
ctemp08 = *(aoffset4 + 1);

*(boffset + 0) = ctemp01;
*(boffset + 1) = ctemp02;
*(boffset + 2) = ctemp03;
*(boffset + 3) = ctemp04;
*(boffset + 4) = ctemp05;
*(boffset + 5) = ctemp06;
*(boffset + 6) = ctemp07;
*(boffset + 7) = ctemp08;

aoffset1 += 2;
aoffset2 += 2;
aoffset3 += 2;
aoffset4 += 2;
boffset += 8;

i --;
}while(i > 0);
}

if (m & 1){
ctemp01 = *(aoffset1 + 0);
ctemp03 = *(aoffset2 + 0);
ctemp05 = *(aoffset3 + 0);
ctemp07 = *(aoffset4 + 0);

*(boffset + 0) = ctemp01;
*(boffset + 1) = ctemp03;
*(boffset + 2) = ctemp05;
*(boffset + 3) = ctemp07;
boffset += 4;
}
}

if (n & 2){
aoffset1 = aoffset;
aoffset2 = aoffset1 + lda;
aoffset += 2 * lda;

i = (m >> 1);
if (i > 0){
do{
ctemp01 = *(aoffset1 + 0);
ctemp02 = *(aoffset1 + 1);
ctemp03 = *(aoffset2 + 0);
ctemp04 = *(aoffset2 + 1);

*(boffset + 0) = ctemp01;
*(boffset + 1) = ctemp03;
*(boffset + 2) = ctemp02;
*(boffset + 3) = ctemp04;

aoffset1 += 2;
aoffset2 += 2;
boffset += 4;

i --;
}while(i > 0);
}

if (m & 1){
ctemp01 = *(aoffset1 + 0);
ctemp03 = *(aoffset2 + 0);

*(boffset + 0) = ctemp01;
*(boffset + 1) = ctemp03;
boffset += 2;
}
}

if (n & 1){
aoffset1 = aoffset;

i = (m >> 1);
if (i > 0){
do{
ctemp01 = *(aoffset1 + 0);
ctemp02 = *(aoffset1 + 1);

*(boffset + 0) = ctemp01;
*(boffset + 1) = ctemp02;

aoffset1 += 2;
boffset += 2;

i --;
}while(i > 0);
}

if (m & 1){
ctemp01 = *(aoffset1 + 0);

*(boffset + 0) = ctemp01;
// boffset += 1;
}
}

return 0;
}

+ 383
- 0
kernel/power/sbgemm_ncopy_8_power10.c View File

@@ -0,0 +1,383 @@
/*********************************************************************/
/* Copyright 2009, 2010 The University of Texas at Austin. */
/* All rights reserved. */
/* */
/* Redistribution and use in source and binary forms, with or */
/* without modification, are permitted provided that the following */
/* conditions are met: */
/* */
/* 1. Redistributions of source code must retain the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer. */
/* */
/* 2. Redistributions in binary form must reproduce the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer in the documentation and/or other materials */
/* provided with the distribution. */
/* */
/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */
/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */
/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */
/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */
/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */
/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */
/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */
/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */
/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */
/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */
/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */
/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */
/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */
/* POSSIBILITY OF SUCH DAMAGE. */
/* */
/* The views and conclusions contained in the software and */
/* documentation are those of the authors and should not be */
/* interpreted as representing official policies, either expressed */
/* or implied, of The University of Texas at Austin. */
/*********************************************************************/

#include <stdio.h>
#include <altivec.h>
#include "common.h"

typedef IFLOAT vec_bf16 __attribute__ ((vector_size (16)));
int CNAME(BLASLONG m, BLASLONG n, IFLOAT *a, BLASLONG lda, IFLOAT *b){
BLASLONG i, j;

IFLOAT *aoffset;
IFLOAT *aoffset1, *aoffset2, *aoffset3, *aoffset4;
IFLOAT *aoffset5, *aoffset6, *aoffset7, *aoffset8;

IFLOAT *boffset;
vec_bf16 vtemp01, vtemp02, vtemp03, vtemp04;
vec_bf16 vtemp05, vtemp06, vtemp07, vtemp08;
vec_bf16 vtemp09, vtemp10, vtemp11, vtemp12;
vector char mask =
{ 0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23 };
vector char mask1 =
{ 8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31 };
IFLOAT ctemp01, ctemp02, ctemp03, ctemp04;
IFLOAT ctemp05, ctemp06, ctemp07, ctemp08;
IFLOAT ctemp09, ctemp10, ctemp11, ctemp12;
IFLOAT ctemp13, ctemp14, ctemp15, ctemp16;
IFLOAT ctemp17;
IFLOAT ctemp25;
IFLOAT ctemp33;
IFLOAT ctemp41;
IFLOAT ctemp49;
IFLOAT ctemp57;


aoffset = a;
boffset = b;

j = (n >> 3);
if (j > 0){
do{
aoffset1 = aoffset;
aoffset2 = aoffset1 + lda;
aoffset3 = aoffset2 + lda;
aoffset4 = aoffset3 + lda;
aoffset5 = aoffset4 + lda;
aoffset6 = aoffset5 + lda;
aoffset7 = aoffset6 + lda;
aoffset8 = aoffset7 + lda;
aoffset += 8 * lda;

i = (m >> 3);
if (i > 0){
do{
vtemp01 = *(vec_bf16 *)(aoffset1);
vtemp02 = *(vec_bf16 *)(aoffset2);
vtemp03 = *(vec_bf16 *)(aoffset3);
vtemp04 = *(vec_bf16 *)(aoffset4);
vtemp05 = *(vec_bf16 *)(aoffset5);
vtemp06 = *(vec_bf16 *)(aoffset6);
vtemp07 = *(vec_bf16 *)(aoffset7);
vtemp08 = *(vec_bf16 *)(aoffset8);

vtemp09 = vec_perm(vtemp01, vtemp02, mask);
vtemp10 = vec_perm(vtemp03, vtemp04, mask);
vtemp11 = vec_perm(vtemp05, vtemp06, mask);
vtemp12 = vec_perm(vtemp07, vtemp08, mask);

*(vec_bf16 *)(boffset + 0) = vec_xxpermdi(vtemp09, vtemp10, 0);
*(vec_bf16 *)(boffset + 8) = vec_xxpermdi(vtemp11, vtemp12, 0);
*(vec_bf16 *)(boffset + 16) = vec_xxpermdi(vtemp09, vtemp10, 3);
*(vec_bf16 *)(boffset + 24) = vec_xxpermdi(vtemp11, vtemp12, 3);

vtemp09 = vec_perm(vtemp01, vtemp02, mask1);
vtemp10 = vec_perm(vtemp03, vtemp04, mask1);
vtemp11 = vec_perm(vtemp05, vtemp06, mask1);
vtemp12 = vec_perm(vtemp07, vtemp08, mask1);

*(vec_bf16 *)(boffset + 32) = vec_xxpermdi(vtemp09, vtemp10, 0);
*(vec_bf16 *)(boffset + 40) = vec_xxpermdi(vtemp11, vtemp12, 0);
*(vec_bf16 *)(boffset + 48) = vec_xxpermdi(vtemp09, vtemp10, 3);
*(vec_bf16 *)(boffset + 56) = vec_xxpermdi(vtemp11, vtemp12, 3);

aoffset1 += 8;
aoffset2 += 8;
aoffset3 += 8;
aoffset4 += 8;
aoffset5 += 8;
aoffset6 += 8;
aoffset7 += 8;
aoffset8 += 8;
boffset += 64;
i --;
}while(i > 0);
}

i = (m & 7);
if (i >= 2){
do{
ctemp01 = *(aoffset1 + 0);
ctemp09 = *(aoffset1 + 1);
ctemp17 = *(aoffset2 + 0);
ctemp25 = *(aoffset2 + 1);
ctemp33 = *(aoffset3 + 0);
ctemp41 = *(aoffset3 + 1);
ctemp49 = *(aoffset4 + 0);
ctemp57 = *(aoffset4 + 1);

*(boffset + 0) = ctemp01;
*(boffset + 1) = ctemp09;
*(boffset + 2) = ctemp17;
*(boffset + 3) = ctemp25;
*(boffset + 4) = ctemp33;
*(boffset + 5) = ctemp41;
*(boffset + 6) = ctemp49;
*(boffset + 7) = ctemp57;
aoffset1 += 2;
aoffset2 += 2;
aoffset3 += 2;
aoffset4 += 2;

ctemp01 = *(aoffset5 + 0);
ctemp09 = *(aoffset5 + 1);
ctemp17 = *(aoffset6 + 0);
ctemp25 = *(aoffset6 + 1);
ctemp33 = *(aoffset7 + 0);
ctemp41 = *(aoffset7 + 1);
ctemp49 = *(aoffset8 + 0);
ctemp57 = *(aoffset8 + 1);
*(boffset + 8) = ctemp01;
*(boffset + 9) = ctemp09;
*(boffset + 10) = ctemp17;
*(boffset + 11) = ctemp25;
*(boffset + 12) = ctemp33;
*(boffset + 13) = ctemp41;
*(boffset + 14) = ctemp49;
*(boffset + 15) = ctemp57;

aoffset5 += 2;
aoffset6 += 2;
aoffset7 += 2;
aoffset8 += 2;

boffset += 16;
i -= 2;
}while(i > 1);
}
if (m & 1){
ctemp01 = *(aoffset1 + 0);
ctemp09 = *(aoffset2 + 0);
ctemp17 = *(aoffset3 + 0);
ctemp25 = *(aoffset4 + 0);
ctemp33 = *(aoffset5 + 0);
ctemp41 = *(aoffset6 + 0);
ctemp49 = *(aoffset7 + 0);
ctemp57 = *(aoffset8 + 0);

*(boffset + 0) = ctemp01;
*(boffset + 1) = ctemp09;
*(boffset + 2) = ctemp17;
*(boffset + 3) = ctemp25;
*(boffset + 4) = ctemp33;
*(boffset + 5) = ctemp41;
*(boffset + 6) = ctemp49;
*(boffset + 7) = ctemp57;

aoffset1 ++;
aoffset2 ++;
aoffset3 ++;
aoffset4 ++;
aoffset5 ++;
aoffset6 ++;
aoffset7 ++;
aoffset8 ++;

boffset += 8;
}

j--;
}while(j > 0);
} /* end of if(j > 0) */

if (n & 4){
aoffset1 = aoffset;
aoffset2 = aoffset1 + lda;
aoffset3 = aoffset2 + lda;
aoffset4 = aoffset3 + lda;
aoffset += 4 * lda;

i = (m >> 2);
if (i > 0){
do{
ctemp01 = *(aoffset1 + 0);
ctemp02 = *(aoffset1 + 1);
ctemp03 = *(aoffset1 + 2);
ctemp04 = *(aoffset1 + 3);

ctemp05 = *(aoffset2 + 0);
ctemp06 = *(aoffset2 + 1);
ctemp07 = *(aoffset2 + 2);
ctemp08 = *(aoffset2 + 3);

ctemp09 = *(aoffset3 + 0);
ctemp10 = *(aoffset3 + 1);
ctemp11 = *(aoffset3 + 2);
ctemp12 = *(aoffset3 + 3);

ctemp13 = *(aoffset4 + 0);
ctemp14 = *(aoffset4 + 1);
ctemp15 = *(aoffset4 + 2);
ctemp16 = *(aoffset4 + 3);

*(boffset + 0) = ctemp01;
*(boffset + 1) = ctemp02;
*(boffset + 2) = ctemp05;
*(boffset + 3) = ctemp06;

*(boffset + 4) = ctemp09;
*(boffset + 5) = ctemp10;
*(boffset + 6) = ctemp13;
*(boffset + 7) = ctemp14;

*(boffset + 8) = ctemp03;
*(boffset + 9) = ctemp04;
*(boffset + 10) = ctemp07;
*(boffset + 11) = ctemp08;

*(boffset + 12) = ctemp11;
*(boffset + 13) = ctemp12;
*(boffset + 14) = ctemp15;
*(boffset + 15) = ctemp16;

aoffset1 += 4;
aoffset2 += 4;
aoffset3 += 4;
aoffset4 += 4;
boffset += 16;
i --;
}while(i > 0);
}

i = (m & 3);
if (i >= 2){
do{
ctemp01 = *(aoffset1 + 0);
ctemp09 = *(aoffset1 + 1);
ctemp17 = *(aoffset2 + 0);
ctemp25 = *(aoffset2 + 1);
ctemp33 = *(aoffset3 + 0);
ctemp41 = *(aoffset3 + 1);
ctemp49 = *(aoffset4 + 0);
ctemp57 = *(aoffset4 + 1);

*(boffset + 0) = ctemp01;
*(boffset + 1) = ctemp09;
*(boffset + 2) = ctemp17;
*(boffset + 3) = ctemp25;
*(boffset + 4) = ctemp33;
*(boffset + 5) = ctemp41;
*(boffset + 6) = ctemp49;
*(boffset + 7) = ctemp57;
aoffset1 += 2;
aoffset2 += 2;
aoffset3 += 2;
aoffset4 += 2;

boffset += 8;
i -= 2;
}while(i > 1);
}
if (m & 1){
ctemp01 = *(aoffset1 + 0);
ctemp02 = *(aoffset2 + 0);
ctemp03 = *(aoffset3 + 0);
ctemp04 = *(aoffset4 + 0);

*(boffset + 0) = ctemp01;
*(boffset + 1) = ctemp02;
*(boffset + 2) = ctemp03;
*(boffset + 3) = ctemp04;

aoffset1 ++;
aoffset2 ++;
aoffset3 ++;
aoffset4 ++;

boffset += 4;
}
}

if (n & 2){
aoffset1 = aoffset;
aoffset2 = aoffset1 + lda;
aoffset += 2 * lda;

i = (m >> 1);
if (i > 0){
do{
ctemp01 = *(aoffset1 + 0);
ctemp02 = *(aoffset1 + 1);
ctemp03 = *(aoffset2 + 0);
ctemp04 = *(aoffset2 + 1);

*(boffset + 0) = ctemp01;
*(boffset + 1) = ctemp03;
*(boffset + 2) = ctemp02;
*(boffset + 3) = ctemp04;

aoffset1 += 2;
aoffset2 += 2;
boffset += 4;
i --;
}while(i > 0);
}

if (m & 1){
ctemp01 = *(aoffset1 + 0);
ctemp02 = *(aoffset2 + 0);

*(boffset + 0) = ctemp01;
*(boffset + 1) = ctemp02;

aoffset1 ++;
aoffset2 ++;
boffset += 2;
}
} /* end of if(j > 0) */

if (n & 1){
aoffset1 = aoffset;

i = m;
if (i > 0){
do{
ctemp01 = *(aoffset1 + 0);

*(boffset + 0) = ctemp01;

aoffset1 ++;
boffset ++;
i --;
}while(i > 0);
}

} /* end of if(j > 0) */

return 0;
}

+ 244
- 0
kernel/power/sbgemm_tcopy_16_power10.c View File

@@ -0,0 +1,244 @@
/*********************************************************************/
/* Copyright 2009, 2010 The University of Texas at Austin. */
/* All rights reserved. */
/* */
/* Redistribution and use in source and binary forms, with or */
/* without modification, are permitted provided that the following */
/* conditions are met: */
/* */
/* 1. Redistributions of source code must retain the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer. */
/* */
/* 2. Redistributions in binary form must reproduce the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer in the documentation and/or other materials */
/* provided with the distribution. */
/* */
/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */
/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */
/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */
/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */
/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */
/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */
/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */
/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */
/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */
/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */
/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */
/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */
/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */
/* POSSIBILITY OF SUCH DAMAGE. */
/* */
/* The views and conclusions contained in the software and */
/* documentation are those of the authors and should not be */
/* interpreted as representing official policies, either expressed */
/* or implied, of The University of Texas at Austin. */
/*********************************************************************/

#include <stdio.h>
#include <altivec.h>
#include "common.h"
typedef IFLOAT vec_bf16 __attribute__ ((vector_size (16)));

int CNAME(BLASLONG m, BLASLONG n, IFLOAT *a, BLASLONG lda, IFLOAT *b){

BLASLONG i, j;

IFLOAT *aoffset;
IFLOAT *aoffset1, *aoffset2;
IFLOAT *boffset;

vec_bf16 vtemp01, vtemp02, vtemp03, vtemp04;
IFLOAT ctemp01, ctemp02, ctemp03, ctemp04;
IFLOAT ctemp05, ctemp06, ctemp07, ctemp08;

aoffset = a;
boffset = b;

#if 0
fprintf(stderr, "m = %d n = %d\n", m, n);
#endif

j = (n >> 4);
if (j > 0){
do{
aoffset1 = aoffset;
aoffset2 = aoffset + lda;
aoffset += 16;

i = (m >> 1);
if (i > 0){
do{
vtemp01 = *(vec_bf16 *)(aoffset1);
vtemp02 = *(vec_bf16 *)(aoffset1+8);
vtemp03 = *(vec_bf16 *)(aoffset2);
vtemp04 = *(vec_bf16 *)(aoffset2+8);
*(vec_bf16 *)(boffset + 0) = vec_mergeh(vtemp01, vtemp03);
*(vec_bf16 *)(boffset + 8) = vec_mergel(vtemp01, vtemp03);
*(vec_bf16 *)(boffset + 16) = vec_mergeh(vtemp02, vtemp04);
*(vec_bf16 *)(boffset + 24) = vec_mergel(vtemp02, vtemp04);
aoffset1 += 2 * lda;
aoffset2 += 2 * lda;
boffset += 32;

i --;
}while(i > 0);
}

if (m & 1){
vtemp01 = *(vec_bf16 *)(aoffset1);
vtemp02 = *(vec_bf16 *)(aoffset1+8);
*(vec_bf16 *)(boffset + 0) = vtemp01;
*(vec_bf16 *)(boffset + 8) = vtemp02;
boffset += 16;
}

j--;
}while(j > 0);
} /* end of if(j > 0) */

if (n & 8){
aoffset1 = aoffset;
aoffset2 = aoffset + lda;
aoffset += 8;

i = (m >> 1);
if (i > 0){
do{
vtemp01 = *(vec_bf16 *)(aoffset1);
vtemp03 = *(vec_bf16 *)(aoffset2);
*(vec_bf16 *)(boffset + 0) = vec_mergeh(vtemp01, vtemp03);
*(vec_bf16 *)(boffset + 8) = vec_mergel(vtemp01, vtemp03);

aoffset1 += 2 * lda;
aoffset2 += 2 * lda;
boffset += 16;

i --;
}while(i > 0);
}

if (m & 1){
vtemp01 = *(vec_bf16 *)(aoffset1);
*(vec_bf16 *)(boffset + 0) = vtemp01;
boffset += 8;
}
}

if (n & 4){
aoffset1 = aoffset;
aoffset2 = aoffset + lda;
aoffset += 4;

i = (m >> 1);
if (i > 0){
do{
ctemp01 = *(aoffset1 + 0);
ctemp02 = *(aoffset1 + 1);
ctemp03 = *(aoffset1 + 2);
ctemp04 = *(aoffset1 + 3);

ctemp05 = *(aoffset2 + 0);
ctemp06 = *(aoffset2 + 1);
ctemp07 = *(aoffset2 + 2);
ctemp08 = *(aoffset2 + 3);

*(boffset + 0) = ctemp01;
*(boffset + 1) = ctemp05;
*(boffset + 2) = ctemp02;
*(boffset + 3) = ctemp06;
*(boffset + 4) = ctemp03;
*(boffset + 5) = ctemp07;
*(boffset + 6) = ctemp04;
*(boffset + 7) = ctemp08;

aoffset1 += 2 * lda;
aoffset2 += 2 * lda;
boffset += 8;

i --;
}while(i > 0);
}

if (m & 1){
ctemp01 = *(aoffset1 + 0);
ctemp02 = *(aoffset1 + 1);
ctemp03 = *(aoffset1 + 2);
ctemp04 = *(aoffset1 + 3);

*(boffset + 0) = ctemp01;
*(boffset + 1) = ctemp02;
*(boffset + 2) = ctemp03;
*(boffset + 3) = ctemp04;

boffset += 4;
}
}

if (n & 2){
aoffset1 = aoffset;
aoffset2 = aoffset + lda;
aoffset += 2;

i = (m >> 1);
if (i > 0){
do{
ctemp01 = *(aoffset1 + 0);
ctemp02 = *(aoffset1 + 1);
ctemp03 = *(aoffset2 + 0);
ctemp04 = *(aoffset2 + 1);

*(boffset + 0) = ctemp01;
*(boffset + 1) = ctemp02;
*(boffset + 2) = ctemp03;
*(boffset + 3) = ctemp04;

aoffset1 += 2 * lda;
aoffset2 += 2 * lda;
boffset += 4;

i --;
}while(i > 0);
}

if (m & 1){
ctemp01 = *(aoffset1 + 0);
ctemp02 = *(aoffset1 + 1);

*(boffset + 0) = ctemp01;
*(boffset + 1) = ctemp02;
boffset += 2;
}
}

if (n & 1){
aoffset1 = aoffset;
aoffset2 = aoffset + lda;

i = (m >> 1);
if (i > 0){
do{
ctemp01 = *(aoffset1 + 0);
ctemp02 = *(aoffset2 + 0);

*(boffset + 0) = ctemp01;
*(boffset + 1) = ctemp02;

aoffset1 += 2 * lda;
aoffset2 += 2 * lda;
boffset += 2;

i --;
}while(i > 0);
}

if (m & 1){
ctemp01 = *(aoffset1 + 0);
*(boffset + 0) = ctemp01;
// boffset += 1;
}
}

return 0;
}

+ 659
- 0
kernel/power/sbgemm_tcopy_8_power10.c View File

@@ -0,0 +1,659 @@
/*********************************************************************/
/* Copyright 2009, 2010 The University of Texas at Austin. */
/* All rights reserved. */
/* */
/* Redistribution and use in source and binary forms, with or */
/* without modification, are permitted provided that the following */
/* conditions are met: */
/* */
/* 1. Redistributions of source code must retain the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer. */
/* */
/* 2. Redistributions in binary form must reproduce the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer in the documentation and/or other materials */
/* provided with the distribution. */
/* */
/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */
/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */
/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */
/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */
/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */
/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */
/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */
/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */
/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */
/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */
/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */
/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */
/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */
/* POSSIBILITY OF SUCH DAMAGE. */
/* */
/* The views and conclusions contained in the software and */
/* documentation are those of the authors and should not be */
/* interpreted as representing official policies, either expressed */
/* or implied, of The University of Texas at Austin. */
/*********************************************************************/

#include <stdio.h>
#include "common.h"
#include <altivec.h>

typedef IFLOAT vec_bf16 __attribute__ ((vector_size (16)));

int CNAME(BLASLONG m, BLASLONG n, IFLOAT *a, BLASLONG lda, IFLOAT *b){

BLASLONG i, j;

IFLOAT *aoffset;
IFLOAT *aoffset1, *aoffset2, *aoffset3, *aoffset4;
IFLOAT *aoffset5, *aoffset6, *aoffset7, *aoffset8;

IFLOAT *boffset, *boffset1, *boffset2, *boffset3, *boffset4;
vec_bf16 vtemp01, vtemp02, vtemp03, vtemp04;
vec_bf16 vtemp05, vtemp06, vtemp07, vtemp08;
IFLOAT ctemp01, ctemp02, ctemp03, ctemp04;
IFLOAT ctemp05, ctemp06, ctemp07, ctemp08;
IFLOAT ctemp09, ctemp10, ctemp11, ctemp12;
IFLOAT ctemp13, ctemp14, ctemp15, ctemp16;
IFLOAT ctemp17, ctemp18, ctemp19, ctemp20;
IFLOAT ctemp21, ctemp22, ctemp23, ctemp24;
IFLOAT ctemp25, ctemp26, ctemp27, ctemp28;
IFLOAT ctemp29, ctemp30, ctemp31, ctemp32;

aoffset = a;
boffset = b;

#if 0
fprintf(stderr, "M = %d N = %d\n", m, n);
#endif

boffset2 = b + m * (n & ~7);
boffset3 = b + m * (n & ~3);
boffset4 = b + m * (n & ~1);

j = (m >> 3);
if (j > 0){
do{
aoffset1 = aoffset;
aoffset2 = aoffset1 + lda;
aoffset3 = aoffset2 + lda;
aoffset4 = aoffset3 + lda;
aoffset5 = aoffset4 + lda;
aoffset6 = aoffset5 + lda;
aoffset7 = aoffset6 + lda;
aoffset8 = aoffset7 + lda;
aoffset += 8 * lda;

boffset1 = boffset;
boffset += 64;

i = (n >> 3);
if (i > 0){
do{
vtemp01 = *(vec_bf16 *)(aoffset1);
vtemp02 = *(vec_bf16 *)(aoffset2);
vtemp03 = *(vec_bf16 *)(aoffset3);
vtemp04 = *(vec_bf16 *)(aoffset4);
vtemp05 = *(vec_bf16 *)(aoffset5);
vtemp06 = *(vec_bf16 *)(aoffset6);
vtemp07 = *(vec_bf16 *)(aoffset7);
vtemp08 = *(vec_bf16 *)(aoffset8);
aoffset1 += 8;
aoffset2 += 8;
aoffset3 += 8;
aoffset4 += 8;
aoffset5 += 8;
aoffset6 += 8;
aoffset7 += 8;
aoffset8 += 8;

*(vec_bf16 *)(boffset1 + 0) = vec_mergeh(vtemp01, vtemp02);
*(vec_bf16 *)(boffset1 + 8) = vec_mergel(vtemp01, vtemp02);
*(vec_bf16 *)(boffset1 + 16) = vec_mergeh(vtemp03, vtemp04);
*(vec_bf16 *)(boffset1 + 24) = vec_mergel(vtemp03, vtemp04);
*(vec_bf16 *)(boffset1 + 32) = vec_mergeh(vtemp05, vtemp06);
*(vec_bf16 *)(boffset1 + 40) = vec_mergel(vtemp05, vtemp06);
*(vec_bf16 *)(boffset1 + 48) = vec_mergeh(vtemp07, vtemp08);
*(vec_bf16 *)(boffset1 + 56) = vec_mergel(vtemp07, vtemp08);

boffset1 += m * 8;
i --;
}while(i > 0);
}

if (n & 4){
ctemp01 = *(aoffset1 + 0);
ctemp02 = *(aoffset1 + 1);
ctemp03 = *(aoffset1 + 2);
ctemp04 = *(aoffset1 + 3);
aoffset1 += 4;

ctemp05 = *(aoffset2 + 0);
ctemp06 = *(aoffset2 + 1);
ctemp07 = *(aoffset2 + 2);
ctemp08 = *(aoffset2 + 3);
aoffset2 += 4;

ctemp09 = *(aoffset3 + 0);
ctemp10 = *(aoffset3 + 1);
ctemp11 = *(aoffset3 + 2);
ctemp12 = *(aoffset3 + 3);
aoffset3 += 4;

ctemp13 = *(aoffset4 + 0);
ctemp14 = *(aoffset4 + 1);
ctemp15 = *(aoffset4 + 2);
ctemp16 = *(aoffset4 + 3);
aoffset4 += 4;

ctemp17 = *(aoffset5 + 0);
ctemp18 = *(aoffset5 + 1);
ctemp19 = *(aoffset5 + 2);
ctemp20 = *(aoffset5 + 3);
aoffset5 += 4;

ctemp21 = *(aoffset6 + 0);
ctemp22 = *(aoffset6 + 1);
ctemp23 = *(aoffset6 + 2);
ctemp24 = *(aoffset6 + 3);
aoffset6 += 4;

ctemp25 = *(aoffset7 + 0);
ctemp26 = *(aoffset7 + 1);
ctemp27 = *(aoffset7 + 2);
ctemp28 = *(aoffset7 + 3);
aoffset7 += 4;

ctemp29 = *(aoffset8 + 0);
ctemp30 = *(aoffset8 + 1);
ctemp31 = *(aoffset8 + 2);
ctemp32 = *(aoffset8 + 3);
aoffset8 += 4;

*(boffset2 + 0) = ctemp01;
*(boffset2 + 1) = ctemp05;
*(boffset2 + 2) = ctemp02;
*(boffset2 + 3) = ctemp06;
*(boffset2 + 4) = ctemp03;
*(boffset2 + 5) = ctemp07;
*(boffset2 + 6) = ctemp04;
*(boffset2 + 7) = ctemp08;

*(boffset2 + 8) = ctemp09;
*(boffset2 + 9) = ctemp13;
*(boffset2 + 10) = ctemp10;
*(boffset2 + 11) = ctemp14;
*(boffset2 + 12) = ctemp11;
*(boffset2 + 13) = ctemp15;
*(boffset2 + 14) = ctemp12;
*(boffset2 + 15) = ctemp16;

*(boffset2 + 16) = ctemp17;
*(boffset2 + 17) = ctemp21;
*(boffset2 + 18) = ctemp18;
*(boffset2 + 19) = ctemp22;
*(boffset2 + 20) = ctemp19;
*(boffset2 + 21) = ctemp23;
*(boffset2 + 22) = ctemp20;
*(boffset2 + 23) = ctemp24;

*(boffset2 + 24) = ctemp25;
*(boffset2 + 25) = ctemp29;
*(boffset2 + 26) = ctemp26;
*(boffset2 + 27) = ctemp30;
*(boffset2 + 28) = ctemp27;
*(boffset2 + 29) = ctemp31;
*(boffset2 + 30) = ctemp28;
*(boffset2 + 31) = ctemp32;

boffset2 += 32;
}

if (n & 2){
ctemp01 = *(aoffset1 + 0);
ctemp02 = *(aoffset1 + 1);
aoffset1 += 2;

ctemp03 = *(aoffset2 + 0);
ctemp04 = *(aoffset2 + 1);
aoffset2 += 2;

ctemp05 = *(aoffset3 + 0);
ctemp06 = *(aoffset3 + 1);
aoffset3 += 2;

ctemp07 = *(aoffset4 + 0);
ctemp08 = *(aoffset4 + 1);
aoffset4 += 2;

ctemp09 = *(aoffset5 + 0);
ctemp10 = *(aoffset5 + 1);
aoffset5 += 2;

ctemp11 = *(aoffset6 + 0);
ctemp12 = *(aoffset6 + 1);
aoffset6 += 2;

ctemp13 = *(aoffset7 + 0);
ctemp14 = *(aoffset7 + 1);
aoffset7 += 2;

ctemp15 = *(aoffset8 + 0);
ctemp16 = *(aoffset8 + 1);
aoffset8 += 2;

*(boffset3 + 0) = ctemp01;
*(boffset3 + 1) = ctemp02;
*(boffset3 + 2) = ctemp03;
*(boffset3 + 3) = ctemp04;
*(boffset3 + 4) = ctemp05;
*(boffset3 + 5) = ctemp06;
*(boffset3 + 6) = ctemp07;
*(boffset3 + 7) = ctemp08;
*(boffset3 + 8) = ctemp09;
*(boffset3 + 9) = ctemp10;
*(boffset3 + 10) = ctemp11;
*(boffset3 + 11) = ctemp12;
*(boffset3 + 12) = ctemp13;
*(boffset3 + 13) = ctemp14;
*(boffset3 + 14) = ctemp15;
*(boffset3 + 15) = ctemp16;
boffset3 += 16;
}

if (n & 1){
ctemp01 = *(aoffset1 + 0);
aoffset1 ++;
ctemp02 = *(aoffset2 + 0);
aoffset2 ++;
ctemp03 = *(aoffset3 + 0);
aoffset3 ++;
ctemp04 = *(aoffset4 + 0);
aoffset4 ++;
ctemp05 = *(aoffset5 + 0);
aoffset5 ++;
ctemp06 = *(aoffset6 + 0);
aoffset6 ++;
ctemp07 = *(aoffset7 + 0);
aoffset7 ++;
ctemp08 = *(aoffset8 + 0);
aoffset8 ++;

*(boffset4 + 0) = ctemp01;
*(boffset4 + 1) = ctemp02;
*(boffset4 + 2) = ctemp03;
*(boffset4 + 3) = ctemp04;
*(boffset4 + 4) = ctemp05;
*(boffset4 + 5) = ctemp06;
*(boffset4 + 6) = ctemp07;
*(boffset4 + 7) = ctemp08;
boffset4 += 8;
}

j--;
}while(j > 0);
}

if (m & 4){

aoffset1 = aoffset;
aoffset2 = aoffset1 + lda;
aoffset3 = aoffset2 + lda;
aoffset4 = aoffset3 + lda;
aoffset += 4 * lda;

boffset1 = boffset;
boffset += 32;

i = (n >> 3);
if (i > 0){

do{
ctemp01 = *(aoffset1 + 0);
ctemp02 = *(aoffset1 + 1);
ctemp03 = *(aoffset1 + 2);
ctemp04 = *(aoffset1 + 3);
ctemp05 = *(aoffset1 + 4);
ctemp06 = *(aoffset1 + 5);
ctemp07 = *(aoffset1 + 6);
ctemp08 = *(aoffset1 + 7);
aoffset1 += 8;

ctemp09 = *(aoffset2 + 0);
ctemp10 = *(aoffset2 + 1);
ctemp11 = *(aoffset2 + 2);
ctemp12 = *(aoffset2 + 3);
ctemp13 = *(aoffset2 + 4);
ctemp14 = *(aoffset2 + 5);
ctemp15 = *(aoffset2 + 6);
ctemp16 = *(aoffset2 + 7);
aoffset2 += 8;

ctemp17 = *(aoffset3 + 0);
ctemp18 = *(aoffset3 + 1);
ctemp19 = *(aoffset3 + 2);
ctemp20 = *(aoffset3 + 3);
ctemp21 = *(aoffset3 + 4);
ctemp22 = *(aoffset3 + 5);
ctemp23 = *(aoffset3 + 6);
ctemp24 = *(aoffset3 + 7);
aoffset3 += 8;

ctemp25 = *(aoffset4 + 0);
ctemp26 = *(aoffset4 + 1);
ctemp27 = *(aoffset4 + 2);
ctemp28 = *(aoffset4 + 3);
ctemp29 = *(aoffset4 + 4);
ctemp30 = *(aoffset4 + 5);
ctemp31 = *(aoffset4 + 6);
ctemp32 = *(aoffset4 + 7);
aoffset4 += 8;

*(boffset1 + 0) = ctemp01;
*(boffset1 + 1) = ctemp09;
*(boffset1 + 2) = ctemp02;
*(boffset1 + 3) = ctemp10;
*(boffset1 + 4) = ctemp03;
*(boffset1 + 5) = ctemp11;
*(boffset1 + 6) = ctemp04;
*(boffset1 + 7) = ctemp12;

*(boffset1 + 8) = ctemp05;
*(boffset1 + 9) = ctemp13;
*(boffset1 + 10) = ctemp06;
*(boffset1 + 11) = ctemp14;
*(boffset1 + 12) = ctemp07;
*(boffset1 + 13) = ctemp15;
*(boffset1 + 14) = ctemp08;
*(boffset1 + 15) = ctemp16;

*(boffset1 + 16) = ctemp17;
*(boffset1 + 17) = ctemp25;
*(boffset1 + 18) = ctemp18;
*(boffset1 + 19) = ctemp26;
*(boffset1 + 20) = ctemp19;
*(boffset1 + 21) = ctemp27;
*(boffset1 + 22) = ctemp20;
*(boffset1 + 23) = ctemp28;

*(boffset1 + 24) = ctemp21;
*(boffset1 + 25) = ctemp29;
*(boffset1 + 26) = ctemp22;
*(boffset1 + 27) = ctemp30;
*(boffset1 + 28) = ctemp23;
*(boffset1 + 29) = ctemp31;
*(boffset1 + 30) = ctemp24;
*(boffset1 + 31) = ctemp32;

boffset1 += 8 * m;
i --;
}while(i > 0);
}

if (n & 4) {
ctemp01 = *(aoffset1 + 0);
ctemp02 = *(aoffset1 + 1);
ctemp03 = *(aoffset1 + 2);
ctemp04 = *(aoffset1 + 3);
aoffset1 += 4;

ctemp05 = *(aoffset2 + 0);
ctemp06 = *(aoffset2 + 1);
ctemp07 = *(aoffset2 + 2);
ctemp08 = *(aoffset2 + 3);
aoffset2 += 4;

ctemp09 = *(aoffset3 + 0);
ctemp10 = *(aoffset3 + 1);
ctemp11 = *(aoffset3 + 2);
ctemp12 = *(aoffset3 + 3);
aoffset3 += 4;

ctemp13 = *(aoffset4 + 0);
ctemp14 = *(aoffset4 + 1);
ctemp15 = *(aoffset4 + 2);
ctemp16 = *(aoffset4 + 3);
aoffset4 += 4;

*(boffset2 + 0) = ctemp01;
*(boffset2 + 1) = ctemp05;
*(boffset2 + 2) = ctemp02;
*(boffset2 + 3) = ctemp06;
*(boffset2 + 4) = ctemp03;
*(boffset2 + 5) = ctemp07;
*(boffset2 + 6) = ctemp04;
*(boffset2 + 7) = ctemp08;

*(boffset2 + 8) = ctemp09;
*(boffset2 + 9) = ctemp13;
*(boffset2 + 10) = ctemp10;
*(boffset2 + 11) = ctemp14;
*(boffset2 + 12) = ctemp11;
*(boffset2 + 13) = ctemp15;
*(boffset2 + 14) = ctemp12;
*(boffset2 + 15) = ctemp16;
boffset2 += 16;
}

if (n & 2){
ctemp01 = *(aoffset1 + 0);
ctemp02 = *(aoffset1 + 1);
aoffset1 += 2;

ctemp03 = *(aoffset2 + 0);
ctemp04 = *(aoffset2 + 1);
aoffset2 += 2;

ctemp05 = *(aoffset3 + 0);
ctemp06 = *(aoffset3 + 1);
aoffset3 += 2;

ctemp07 = *(aoffset4 + 0);
ctemp08 = *(aoffset4 + 1);
aoffset4 += 2;

*(boffset3 + 0) = ctemp01;
*(boffset3 + 1) = ctemp02;
*(boffset3 + 2) = ctemp03;
*(boffset3 + 3) = ctemp04;
*(boffset3 + 4) = ctemp05;
*(boffset3 + 5) = ctemp06;
*(boffset3 + 6) = ctemp07;
*(boffset3 + 7) = ctemp08;
boffset3 += 8;
}

if (n & 1){
ctemp01 = *(aoffset1 + 0);
aoffset1 ++;
ctemp02 = *(aoffset2 + 0);
aoffset2 ++;
ctemp03 = *(aoffset3 + 0);
aoffset3 ++;
ctemp04 = *(aoffset4 + 0);
aoffset4 ++;

*(boffset4 + 0) = ctemp01;
*(boffset4 + 1) = ctemp02;
*(boffset4 + 2) = ctemp03;
*(boffset4 + 3) = ctemp04;
boffset4 += 4;
}
}

if (m & 2){
aoffset1 = aoffset;
aoffset2 = aoffset1 + lda;
aoffset += 2 * lda;

boffset1 = boffset;
boffset += 16;

i = (n >> 3);
if (i > 0){
do{
ctemp01 = *(aoffset1 + 0);
ctemp02 = *(aoffset1 + 1);
ctemp03 = *(aoffset1 + 2);
ctemp04 = *(aoffset1 + 3);
ctemp05 = *(aoffset1 + 4);
ctemp06 = *(aoffset1 + 5);
ctemp07 = *(aoffset1 + 6);
ctemp08 = *(aoffset1 + 7);
aoffset1 += 8;

ctemp09 = *(aoffset2 + 0);
ctemp10 = *(aoffset2 + 1);
ctemp11 = *(aoffset2 + 2);
ctemp12 = *(aoffset2 + 3);
ctemp13 = *(aoffset2 + 4);
ctemp14 = *(aoffset2 + 5);
ctemp15 = *(aoffset2 + 6);
ctemp16 = *(aoffset2 + 7);
aoffset2 += 8;

*(boffset1 + 0) = ctemp01;
*(boffset1 + 1) = ctemp09;
*(boffset1 + 2) = ctemp02;
*(boffset1 + 3) = ctemp10;
*(boffset1 + 4) = ctemp03;
*(boffset1 + 5) = ctemp11;
*(boffset1 + 6) = ctemp04;
*(boffset1 + 7) = ctemp12;

*(boffset1 + 8) = ctemp05;
*(boffset1 + 9) = ctemp13;
*(boffset1 + 10) = ctemp06;
*(boffset1 + 11) = ctemp14;
*(boffset1 + 12) = ctemp07;
*(boffset1 + 13) = ctemp15;
*(boffset1 + 14) = ctemp08;
*(boffset1 + 15) = ctemp16;

boffset1 += 8 * m;
i --;
}while(i > 0);
}

if (n & 4){
ctemp01 = *(aoffset1 + 0);
ctemp02 = *(aoffset1 + 1);
ctemp03 = *(aoffset1 + 2);
ctemp04 = *(aoffset1 + 3);
aoffset1 += 4;

ctemp05 = *(aoffset2 + 0);
ctemp06 = *(aoffset2 + 1);
ctemp07 = *(aoffset2 + 2);
ctemp08 = *(aoffset2 + 3);
aoffset2 += 4;

*(boffset2 + 0) = ctemp01;
*(boffset2 + 1) = ctemp05;
*(boffset2 + 2) = ctemp02;
*(boffset2 + 3) = ctemp06;
*(boffset2 + 4) = ctemp03;
*(boffset2 + 5) = ctemp07;
*(boffset2 + 6) = ctemp04;
*(boffset2 + 7) = ctemp08;
boffset2 += 8;
}

if (n & 2){
ctemp01 = *(aoffset1 + 0);
ctemp02 = *(aoffset1 + 1);
aoffset1 += 2;

ctemp03 = *(aoffset2 + 0);
ctemp04 = *(aoffset2 + 1);
aoffset2 += 2;

*(boffset3 + 0) = ctemp01;
*(boffset3 + 1) = ctemp02;
*(boffset3 + 2) = ctemp03;
*(boffset3 + 3) = ctemp04;
boffset3 += 4;
}

if (n & 1){
ctemp01 = *(aoffset1 + 0);
aoffset1 ++;
ctemp02 = *(aoffset2 + 0);
aoffset2 ++;

*(boffset4 + 0) = ctemp01;
*(boffset4 + 1) = ctemp02;
boffset4 += 2;
}
}

if (m & 1){
aoffset1 = aoffset;
// aoffset += lda;

boffset1 = boffset;
// boffset += 8;

i = (n >> 3);
if (i > 0){
do{
ctemp01 = *(aoffset1 + 0);
ctemp02 = *(aoffset1 + 1);
ctemp03 = *(aoffset1 + 2);
ctemp04 = *(aoffset1 + 3);
ctemp05 = *(aoffset1 + 4);
ctemp06 = *(aoffset1 + 5);
ctemp07 = *(aoffset1 + 6);
ctemp08 = *(aoffset1 + 7);
aoffset1 += 8;

*(boffset1 + 0) = ctemp01;
*(boffset1 + 1) = ctemp02;
*(boffset1 + 2) = ctemp03;
*(boffset1 + 3) = ctemp04;
*(boffset1 + 4) = ctemp05;
*(boffset1 + 5) = ctemp06;
*(boffset1 + 6) = ctemp07;
*(boffset1 + 7) = ctemp08;

boffset1 += 8 * m;
i --;
}while(i > 0);
}

if (n & 4){
ctemp01 = *(aoffset1 + 0);
ctemp02 = *(aoffset1 + 1);
ctemp03 = *(aoffset1 + 2);
ctemp04 = *(aoffset1 + 3);
aoffset1 += 4;

*(boffset2 + 0) = ctemp01;
*(boffset2 + 1) = ctemp02;
*(boffset2 + 2) = ctemp03;
*(boffset2 + 3) = ctemp04;
// boffset2 += 4;
}

if (n & 2){
ctemp01 = *(aoffset1 + 0);
ctemp02 = *(aoffset1 + 1);
aoffset1 += 2;

*(boffset3 + 0) = ctemp01;
*(boffset3 + 1) = ctemp02;
// boffset3 += 2;
}

if (n & 1){
ctemp01 = *(aoffset1 + 0);
aoffset1 ++;
*(boffset4 + 0) = ctemp01;
boffset4 ++;
}
}

return 0;
}

+ 3
- 8
kernel/x86_64/KERNEL View File

@@ -259,12 +259,8 @@ SNRM2KERNEL = nrm2_sse.S
endif

ifndef DNRM2KERNEL
ifeq ($(OSNAME),WINNT)
DNRM2KERNEL = ../arm/nrm2.c
else
DNRM2KERNEL = nrm2.S
endif
endif

ifndef QNRM2KERNEL
QNRM2KERNEL = nrm2.S
@@ -275,12 +271,8 @@ CNRM2KERNEL = znrm2_sse.S
endif

ifndef ZNRM2KERNEL
ifeq ($(OSNAME),WINNT)
ZNRM2KERNEL = ../arm/znrm2.c
else
ZNRM2KERNEL = znrm2.S
endif
endif

ifndef XNRM2KERNEL
XNRM2KERNEL = znrm2.S
@@ -486,3 +478,6 @@ XTRSMKERNEL_RN = xtrsm_kernel_LT_1x1.S
XTRSMKERNEL_RT = xtrsm_kernel_LT_1x1.S

XGEMM3MKERNEL = xgemm3m_kernel_2x2.S

SSUMKERNEL = ../arm/sum.c
DSUMKERNEL = ../arm/sum.c

+ 1
- 1
lapack/CMakeLists.txt View File

@@ -2,7 +2,7 @@
include_directories(${PROJECT_SOURCE_DIR})
include_directories(${PROJECT_BINARY_DIR})

list (REMOVE_ITEM FLOAT_TYPES "HALF")
list (REMOVE_ITEM FLOAT_TYPES "BFLOAT16")

set(LAPACK_SOURCES
potrf/potrf_U_single.c


+ 39
- 97
test/Makefile View File

@@ -7,40 +7,22 @@ all ::
else
all :: level1 level2 level3
endif
ifeq ($(BUILD_SINGLE)x$(BUILD_DOUBLE)x$(BUILD_COMPLEX)x$(BUILD_COMPLEX16),1x1x1x1)
level1: sblat1 dblat1 cblat1 zblat1
endif
ifeq ($(BUILD_SINGLE)x$(BUILD_DOUBLE)x$(BUILD_COMPLEX)x$(BUILD_COMPLEX16),x1x1x1)
level1: dblat1 cblat1 zblat1
endif
ifeq ($(BUILD_SINGLE)x$(BUILD_DOUBLE)x$(BUILD_COMPLEX)x$(BUILD_COMPLEX16),1xx1x1)
level1: sblat1 cblat1 zblat1
endif
ifeq ($(BUILD_SINGLE)x$(BUILD_DOUBLE)x$(BUILD_COMPLEX)x$(BUILD_COMPLEX16),xx1x1)
level1: cblat1 zblat1
endif
ifeq ($(BUILD_SINGLE)x$(BUILD_DOUBLE)x$(BUILD_COMPLEX)x$(BUILD_COMPLEX16),xx1x)
level1: cblat1
endif
ifeq ($(BUILD_SINGLE)x$(BUILD_DOUBLE)x$(BUILD_COMPLEX)x$(BUILD_COMPLEX16),xxx1)
level1: zblat1
endif
ifeq ($(BUILD_SINGLE)x$(BUILD_DOUBLE)x$(BUILD_COMPLEX)x$(BUILD_COMPLEX16),1xxx1)
level1: sblat1 zblat1
endif
ifeq ($(BUILD_SINGLE)x$(BUILD_DOUBLE)x$(BUILD_COMPLEX)x$(BUILD_COMPLEX16),1x1xx1)
level1: sblat1 dblat1 zblat1

ifeq ($(BUILD_SINGLE),1)
S1=sblat1
endif
ifeq ($(BUILD_SINGLE)x$(BUILD_DOUBLE)x$(BUILD_COMPLEX)x$(BUILD_COMPLEX16),1x1xx)
level1: sblat1 dblat1
ifeq ($(BUILD_DOUBLE),1)
D1=dblat1
endif
ifeq ($(BUILD_SINGLE)x$(BUILD_DOUBLE)x$(BUILD_COMPLEX)x$(BUILD_COMPLEX16),1xxx)
level1: sblat1
ifeq ($(BUILD_COMPLEX),1)
C1=cblat1
endif
ifeq ($(BUILD_SINGLE)x$(BUILD_DOUBLE)x$(BUILD_COMPLEX)x$(BUILD_COMPLEX16),x1xx)
level1: dblat1
ifeq ($(BUILD_COMPLEX16),1)
Z1=zblat1
endif

level1: $(S1) $(D1) $(C1) $(Z1)

ifndef CROSS
ifeq ($(BUILD_SINGLE),1)
OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 ./sblat1
@@ -85,41 +67,22 @@ endif
endif
endif

#level2: sblat2 dblat2 cblat2 zblat2
ifeq ($(BUILD_SINGLE)x$(BUILD_DOUBLE)x$(BUILD_COMPLEX)x$(BUILD_COMPLEX16),1x1x1x1)
level2: sblat2 dblat2 cblat2 zblat2
endif
ifeq ($(BUILD_SINGLE)x$(BUILD_DOUBLE)x$(BUILD_COMPLEX)x$(BUILD_COMPLEX16),x1x1x1)
level2: dblat2 cblat2 zblat2
endif
ifeq ($(BUILD_SINGLE)x$(BUILD_DOUBLE)x$(BUILD_COMPLEX)x$(BUILD_COMPLEX16),1xx1x1)
level2: sblat2 cblat2 zblat2
endif
ifeq ($(BUILD_SINGLE)x$(BUILD_DOUBLE)x$(BUILD_COMPLEX)x$(BUILD_COMPLEX16),xx1x1)
level2: cblat2 zblat2
endif
ifeq ($(BUILD_SINGLE)x$(BUILD_DOUBLE)x$(BUILD_COMPLEX)x$(BUILD_COMPLEX16),xx1x)
level2: cblat2
endif
ifeq ($(BUILD_SINGLE)x$(BUILD_DOUBLE)x$(BUILD_COMPLEX)x$(BUILD_COMPLEX16),xxx1)
level2: zblat2
endif
ifeq ($(BUILD_SINGLE)x$(BUILD_DOUBLE)x$(BUILD_COMPLEX)x$(BUILD_COMPLEX16),1xxx1)
level2: sblat2 zblat2
endif
ifeq ($(BUILD_SINGLE)x$(BUILD_DOUBLE)x$(BUILD_COMPLEX)x$(BUILD_COMPLEX16),1x1xx1)
level2: sblat2 dblat2 zblat2
ifeq ($(BUILD_SINGLE),1)
S2=sblat2
endif
ifeq ($(BUILD_SINGLE)x$(BUILD_DOUBLE)x$(BUILD_COMPLEX)x$(BUILD_COMPLEX16),1x1xx)
level2: sblat2 dblat2
ifeq ($(BUILD_DOUBLE),1)
D2=dblat2
endif
ifeq ($(BUILD_SINGLE)x$(BUILD_DOUBLE)x$(BUILD_COMPLEX)x$(BUILD_COMPLEX16),1xxx)
level2: sblat2
ifeq ($(BUILD_COMPLEX),1)
C2=cblat2
endif
ifeq ($(BUILD_SINGLE)x$(BUILD_DOUBLE)x$(BUILD_COMPLEX)x$(BUILD_COMPLEX16),x1xx)
level2: dblat2
ifeq ($(BUILD_COMPLEX16),1)
Z2=zblat2
endif

level2: $(S2) $(D2) $(C2) $(Z2)


ifndef CROSS
rm -f ?BLAT2.SUMM
ifeq ($(BUILD_SINGLE),1)
@@ -178,51 +141,30 @@ endif
endif
endif

ifeq ($(BUILD_SINGLE)x$(BUILD_DOUBLE)x$(BUILD_COMPLEX)x$(BUILD_COMPLEX16),1x1x1x1)
level3: sblat3 dblat3 cblat3 zblat3
endif
ifeq ($(BUILD_SINGLE)x$(BUILD_DOUBLE)x$(BUILD_COMPLEX)x$(BUILD_COMPLEX16),x1x1x1)
level3: dblat3 cblat3 zblat3
endif
ifeq ($(BUILD_SINGLE)x$(BUILD_DOUBLE)x$(BUILD_COMPLEX)x$(BUILD_COMPLEX16),1xx1x1)
level3: sblat3 cblat3 zblat3
endif
ifeq ($(BUILD_SINGLE)x$(BUILD_DOUBLE)x$(BUILD_COMPLEX)x$(BUILD_COMPLEX16),xx1x1)
level3: cblat3 zblat3
endif
ifeq ($(BUILD_SINGLE)x$(BUILD_DOUBLE)x$(BUILD_COMPLEX)x$(BUILD_COMPLEX16),xx1x)
level3: cblat3
endif
ifeq ($(BUILD_SINGLE)x$(BUILD_DOUBLE)x$(BUILD_COMPLEX)x$(BUILD_COMPLEX16),xxx1)
level3: zblat3
endif
ifeq ($(BUILD_SINGLE)x$(BUILD_DOUBLE)x$(BUILD_COMPLEX)x$(BUILD_COMPLEX16),1xxx1)
level3: sblat3 zblat3
ifeq ($(BUILD_BFLOAT16),1)
B3= test_sbgemm
endif
ifeq ($(BUILD_SINGLE)x$(BUILD_DOUBLE)x$(BUILD_COMPLEX)x$(BUILD_COMPLEX16),1x1xx1)
level3: sblat3 dblat3 zblat3
ifeq ($(BUILD_SINGLE),1)
S3=sblat3
endif
ifeq ($(BUILD_SINGLE)x$(BUILD_DOUBLE)x$(BUILD_COMPLEX)x$(BUILD_COMPLEX16),1x1xx)
level3: sblat3 dblat3
ifeq ($(BUILD_DOUBLE),1)
D3=dblat3
endif
ifeq ($(BUILD_SINGLE)x$(BUILD_DOUBLE)x$(BUILD_COMPLEX)x$(BUILD_COMPLEX16),1xxx)
level3: sblat3
ifeq ($(BUILD_COMPLEX),1)
C3=cblat3
endif
ifeq ($(BUILD_SINGLE)x$(BUILD_DOUBLE)x$(BUILD_COMPLEX)x$(BUILD_COMPLEX16),x1xx)
level3: dblat3
ifeq ($(BUILD_COMPLEX16),1)
Z3=zblat3
endif

level3: $(B3) $(S3) $(D3) $(C3) $(Z3)


ifeq ($(BUILD_BFLOAT16),1)
level3 : test_sbgemm
endif

ifndef CROSS
rm -f ?BLAT3.SUMM
ifeq ($(BUILD_BFLOAT16),1)
OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 ./test_sbgemm > SHBLAT3.SUMM
@$(GREP) -q FATAL SHBLAT3.SUMM && cat SHBLAT3.SUMM || exit 0
OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 ./test_sbgemm > SBBLAT3.SUMM
@$(GREP) -q FATAL SBBLAT3.SUMM && cat SBBLAT3.SUMM || exit 0
endif
ifeq ($(BUILD_SINGLE),1)
OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 ./sblat3 < ./sblat3.dat
@@ -244,8 +186,8 @@ ifdef SMP
rm -f ?BLAT3.SUMM
ifeq ($(USE_OPENMP), 1)
ifeq ($(BUILD_BFLOAT16),1)
OMP_NUM_THREADS=2 ./test_sbgemm > SHBLAT3.SUMM
@$(GREP) -q FATAL SHBLAT3.SUMM && cat SHBLAT3.SUMM || exit 0
OMP_NUM_THREADS=2 ./test_sbgemm > SBBLAT3.SUMM
@$(GREP) -q FATAL SBBLAT3.SUMM && cat SBBLAT3.SUMM || exit 0
endif
ifeq ($(BUILD_SINGLE),1)
OMP_NUM_THREADS=2 ./sblat3 < ./sblat3.dat
@@ -265,8 +207,8 @@ ifeq ($(BUILD_COMPLEX16),1)
endif
else
ifeq ($(BUILD_BFLOAT16),1)
OPENBLAS_NUM_THREADS=2 ./test_sbgemm > SHBLAT3.SUMM
@$(GREP) -q FATAL SHBLAT3.SUMM && cat SHBLAT3.SUMM || exit 0
OPENBLAS_NUM_THREADS=2 ./test_sbgemm > SBBLAT3.SUMM
@$(GREP) -q FATAL SBBLAT3.SUMM && cat SBBLAT3.SUMM || exit 0
endif
ifeq ($(BUILD_SINGLE),1)
OPENBLAS_NUM_THREADS=2 ./sblat3 < ./sblat3.dat


Write Preview
Loading…
Cancel
Save