OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
Browse Source

POWER10: Change the packing format for bfloat16 

As the new MMA instructions need the inputs in 4x2 order for bfloat16,
changing the format in copy/packing code.  This avoids permute instructions
in the gemm kernel inner loop.
tags/v0.3.11^2
Rajalakshmi Srinivasaraghavan 5 years ago
parent
602a0c7a69
commit
0826d68f93
6 changed files with 1923 additions and 285 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +4
    -4
      kernel/power/KERNEL.POWER10
  2. +196
    -281
      kernel/power/sbgemm_kernel_power10.c
  3. +437
    -0
      kernel/power/sbgemm_ncopy_16_power10.c
  4. +383
    -0
      kernel/power/sbgemm_ncopy_8_power10.c
  5. +244
    -0
      kernel/power/sbgemm_tcopy_16_power10.c
  6. +659
    -0
      kernel/power/sbgemm_tcopy_8_power10.c

+ 4
- 4
kernel/power/KERNEL.POWER10 View File

@@ -9,10 +9,10 @@ else

SBGEMM_BETA = ../generic/gemm_beta.c
SBGEMMKERNEL = sbgemm_kernel_power10.c
SBGEMMINCOPY = ../generic/gemm_ncopy_16.c
SBGEMMITCOPY = ../generic/gemm_tcopy_16.c
SBGEMMONCOPY = ../generic/gemm_ncopy_8.c
SBGEMMOTCOPY = ../generic/gemm_tcopy_8.c
SBGEMMINCOPY = sbgemm_ncopy_16_power10.c
SBGEMMITCOPY = sbgemm_tcopy_16_power10.c
SBGEMMONCOPY = sbgemm_ncopy_8_power10.c
SBGEMMOTCOPY = sbgemm_tcopy_8_power10.c
SBGEMMINCOPYOBJ = sbgemm_incopy$(TSUFFIX).$(SUFFIX)
SBGEMMITCOPYOBJ = sbgemm_itcopy$(TSUFFIX).$(SUFFIX)
SBGEMMONCOPYOBJ = sbgemm_oncopy$(TSUFFIX).$(SUFFIX)


+ 196
- 281
kernel/power/sbgemm_kernel_power10.c View File

@@ -137,15 +137,13 @@ int
CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
IFLOAT * B, FLOAT * C, BLASLONG ldc)
{
BLASLONG N = n;
BLASLONG i1;
v4sf_t valpha = { alpha, alpha, alpha, alpha };
vector short vzero = { 0, 0, 0, 0, 0, 0, 0, 0 };
N = n >> 3;
/* Loop for n >= 8. */
for (i1 = 0; i1 < N; i1++)
for (i1 = 0; i1 < (n >> 3); i1++)
{
BLASLONG i, j;
BLASLONG j;
FLOAT *CO;
IFLOAT *AO;
CO = C;
@@ -153,9 +151,8 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
AO = A;
PREFETCH1 (A, 128);
PREFETCH1 (A, 256);
i = m >> 4;
/* Loop for m >= 16. */
for (j = 0; j < i; j++)
for (j = 0; j < (m >> 4); j++)
{
IFLOAT *BO = B;
v4sf_t *rowC;
@@ -167,20 +164,14 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
{
vec_t *rowA = (vec_t *) & (AO[l << 5]);
vec_t *rowB = (vec_t *) & (BO[l << 4]);
vec_t rowB_h = MERGE_HIGH (rowB[0], rowB[1]);
vec_t rowB_l = MERGE_LOW (rowB[0], rowB[1]);
vec_t rowA_h = MERGE_HIGH (rowA[0], rowA[2]);
vec_t rowA_l = MERGE_LOW (rowA[0], rowA[2]);
vec_t rowA2_h = MERGE_HIGH (rowA[1], rowA[3]);
vec_t rowA2_l = MERGE_LOW (rowA[1], rowA[3]);
MMA (&acc0, rowB_h, rowA_h);
MMA (&acc1, rowB_l, rowA_h);
MMA (&acc2, rowB_h, rowA_l);
MMA (&acc3, rowB_l, rowA_l);
MMA (&acc4, rowB_h, rowA2_h);
MMA (&acc5, rowB_l, rowA2_h);
MMA (&acc6, rowB_h, rowA2_l);
MMA (&acc7, rowB_l, rowA2_l);
MMA (&acc0, rowB[0], rowA[0]);
MMA (&acc1, rowB[1], rowA[0]);
MMA (&acc2, rowB[0], rowA[1]);
MMA (&acc3, rowB[1], rowA[1]);
MMA (&acc4, rowB[0], rowA[2]);
MMA (&acc5, rowB[1], rowA[2]);
MMA (&acc6, rowB[0], rowA[3]);
MMA (&acc7, rowB[1], rowA[3]);
}
if (k % 2 == 1)
{
@@ -216,9 +207,7 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
AO += (k << 4);
BO += (k << 3);
}
i = (m & 15) >> 3;
/* Loop for m >= 8. */
for (j = 0; j < i; j++)
if (m & 8)
{
IFLOAT *BO = B;
v4sf_t *rowC;
@@ -230,14 +219,11 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
{
vec_t *rowA = (vec_t *) & (AO[l << 4]);
vec_t *rowB = (vec_t *) & (BO[l << 4]);
vec_t rowB_h = MERGE_HIGH (rowB[0], rowB[1]);
vec_t rowB_l = MERGE_LOW (rowB[0], rowB[1]);
vec_t rowA_h = MERGE_HIGH (rowA[0], rowA[1]);
vec_t rowA_l = MERGE_LOW (rowA[0], rowA[1]);
MMA (&acc0, rowB_h, rowA_h);
MMA (&acc1, rowB_l, rowA_h);
MMA (&acc2, rowB_h, rowA_l);
MMA (&acc3, rowB_l, rowA_l);

MMA (&acc0, rowB[0], rowA[0]);
MMA (&acc1, rowB[1], rowA[0]);
MMA (&acc2, rowB[0], rowA[1]);
MMA (&acc3, rowB[1], rowA[1]);
}
if (k % 2 == 1)
{
@@ -262,9 +248,7 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
AO += (k << 3);
BO += (k << 3);
}
i = (m & 7) >> 2;
/* Loop for m >= 4. */
for (j = 0; j < i; j++)
if (m & 4)
{
IFLOAT *BO = B;
v4sf_t *rowC;
@@ -277,9 +261,8 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
{
vec_t *rowA = (vec_t *) & (AO[l << 3]);
vec_t *rowB = (vec_t *) & (BO[l << 4]);
vec_t rowA_mrg = MERGE_ROW (rowA[0]);
MMA (&acc0, MERGE_HIGH (rowB[0], rowB[1]), rowA_mrg);
MMA (&acc1, MERGE_LOW (rowB[0], rowB[1]), rowA_mrg);
MMA (&acc0, rowB[0], rowA[0]);
MMA (&acc1, rowB[1], rowA[0]);
}
if (k % 2 == 1)
{
@@ -297,9 +280,7 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
AO += (k << 2);
BO += (k << 3);
}
i = (m & 3) >> 1;
/* Loop for m >= 2. */
for (j = 0; j < i; j++)
if (m & 2)
{
IFLOAT *BO = B;
v2sf_t *rowC;
@@ -316,8 +297,8 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
0, 0, 0, 0
};
vec_t *rowB = (vec_t *) & (BO[l << 4]);
MMA (&acc0, MERGE_HIGH (rowB[0], rowB[1]), (vec_t) rowA);
MMA (&acc1, MERGE_LOW (rowB[0], rowB[1]), (vec_t) rowA);
MMA (&acc0, rowB[0], (vec_t) rowA);
MMA (&acc1, rowB[1], (vec_t) rowA);
}
if (k % 2 == 1)
{
@@ -334,64 +315,50 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
AO += (k << 1);
BO += (k << 3);
}
i = (m & 1) >> 0;
/* Loop for m = 1. */
for (j = 0; j < i; j++)
if (m & 1)
{
IFLOAT *BO = B;
v2sf_t *rowC;
v2sf_t result[8];
__vector_quad acc0, acc1;
__builtin_mma_xxsetaccz (&acc0);
__builtin_mma_xxsetaccz (&acc1);
BLASLONG l = 0;
v4sf_t t = { 0, 0, 0, 0 }
, t1 =
{
0, 0, 0, 0};
for (l = 0; l < k; l++)
for (l = 0; l < k / 2; l++)
{
v4sf_t rowA =
{ BF16TOF32 (AO[l]), BF16TOF32 (AO[l]), BF16TOF32 (AO[l]),
BF16TOF32 (AO[l])
};
v4sf_t rowB =
{ BF16TOF32 (BO[l << 3]), BF16TOF32 (BO[(l << 3) + 1]),
BF16TOF32 (BO[(l << 3) + 2]),
BF16TOF32 (BO[(l << 3) + 3])
};
v4sf_t rowB1 =
{ BF16TOF32 (BO[(l << 3) + 4]), BF16TOF32 (BO[(l << 3) + 5]),
BF16TOF32 (BO[(l << 3) + 6]),
BF16TOF32 (BO[(l << 3) + 7])
};
t += rowA * rowB;
t1 += rowA * rowB1;
vector short rowA =
{ AO[(l << 1) + 0], AO[(l << 1) + 1], 0, 0, 0, 0, 0, 0};
vec_t *rowB = (vec_t *) & (BO[l << 4]);
MMA (&acc0, rowB[0], (vec_t) rowA);
MMA (&acc1, rowB[1], (vec_t) rowA);
}
t = t * valpha;
t1 = t1 * valpha;
CO[0 * ldc] += t[0];
CO[1 * ldc] += t[1];
CO[2 * ldc] += t[2];
CO[3 * ldc] += t[3];
CO[4 * ldc] += t1[0];
CO[5 * ldc] += t1[1];
CO[6 * ldc] += t1[2];
CO[7 * ldc] += t1[3];
if (k % 2 == 1)
{
if (k > 1)
l = (k / 2) << 1;
vector short rowA = { AO[l], 0, 0, 0, 0, 0, 0, 0 };
vec_t *rowB = (vec_t *) & (BO[(l << 3)]);
MMA (&acc0, MERGE_HIGH (rowB[0], rowB[1]), (vec_t) rowA);
MMA (&acc1, MERGE_LOW (rowB[0], rowB[1]), (vec_t) rowA);
}
SAVE4x2_ACC (&acc0, 0);
SAVE4x2_ACC1 (&acc1, 0);
CO += 1;
AO += k;
BO += (k << 3);
}
B += k << 3;
}
N = (n & 7) >> 2;
/* Loop for n >= 4. */
for (i1 = 0; i1 < N; i1++)
if (n & 4)
{
BLASLONG i, j;
BLASLONG j;
FLOAT *CO;
IFLOAT *AO;
CO = C;
C += ldc << 2;
AO = A;
i = m >> 5;
/* Loop for m >= 32. */
for (j = 0; j < i; j++)
for (j = 0; j < (m >> 5); j++)
{
IFLOAT *BO = B;
IFLOAT *A1 = AO + (16 * k);
@@ -405,15 +372,14 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
vec_t *rowA = (vec_t *) & (AO[l << 5]);
vec_t *rowA1 = (vec_t *) & (A1[l << 5]);
vec_t *rowB = (vec_t *) & (BO[l << 3]);
vec_t rowB_mrg = MERGE_ROW (rowB[0]);
MMA (&acc0, rowB_mrg, MERGE_HIGH (rowA[0], rowA[2]));
MMA (&acc1, rowB_mrg, MERGE_LOW (rowA[0], rowA[2]));
MMA (&acc2, rowB_mrg, MERGE_HIGH (rowA[1], rowA[3]));
MMA (&acc3, rowB_mrg, MERGE_LOW (rowA[1], rowA[3]));
MMA (&acc4, rowB_mrg, MERGE_HIGH (rowA1[0], rowA1[2]));
MMA (&acc5, rowB_mrg, MERGE_LOW (rowA1[0], rowA1[2]));
MMA (&acc6, rowB_mrg, MERGE_HIGH (rowA1[1], rowA1[3]));
MMA (&acc7, rowB_mrg, MERGE_LOW (rowA1[1], rowA1[3]));
MMA (&acc0, rowB[0], rowA[0]);
MMA (&acc1, rowB[0], rowA[1]);
MMA (&acc2, rowB[0], rowA[2]);
MMA (&acc3, rowB[0], rowA[3]);
MMA (&acc4, rowB[0], rowA1[0]);
MMA (&acc5, rowB[0], rowA1[1]);
MMA (&acc6, rowB[0], rowA1[2]);
MMA (&acc7, rowB[0], rowA1[3]);
}
if (k % 2 == 1)
{
@@ -448,9 +414,7 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
AO += k << 5;
BO += k << 2;
}
i = (m & 31) >> 4;
/* Loop for m >= 16. */
for (j = 0; j < i; j++)
if (m & 16)
{
IFLOAT *BO = B;
v4sf_t *rowC;
@@ -462,11 +426,10 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
{
vec_t *rowA = (vec_t *) & (AO[l << 5]);
vec_t *rowB = (vec_t *) & (BO[l << 3]);
vec_t rowB_mrg = MERGE_ROW (rowB[0]);
MMA (&acc0, rowB_mrg, MERGE_HIGH (rowA[0], rowA[2]));
MMA (&acc1, rowB_mrg, MERGE_LOW (rowA[0], rowA[2]));
MMA (&acc2, rowB_mrg, MERGE_HIGH (rowA[1], rowA[3]));
MMA (&acc3, rowB_mrg, MERGE_LOW (rowA[1], rowA[3]));
MMA (&acc0, rowB[0], rowA[0]);
MMA (&acc1, rowB[0], rowA[1]);
MMA (&acc2, rowB[0], rowA[2]);
MMA (&acc3, rowB[0], rowA[3]);
}
if (k % 2 == 1)
{
@@ -490,9 +453,7 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
AO += k << 4;
BO += k << 2;
}
i = (m & 15) >> 3;
/* Loop for m >= 8. */
for (j = 0; j < i; j++)
if (m & 8)
{
IFLOAT *BO = B;
v4sf_t *rowC;
@@ -505,9 +466,8 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
{
vec_t *rowA = (vec_t *) & (AO[l << 4]);
vec_t *rowB = (vec_t *) & (BO[l << 3]);
vec_t rowB_mrg = MERGE_ROW (rowB[0]);
MMA (&acc0, rowB_mrg, MERGE_HIGH (rowA[0], rowA[1]));
MMA (&acc1, rowB_mrg, MERGE_LOW (rowA[0], rowA[1]));
MMA (&acc0, rowB[0], rowA[0]);
MMA (&acc1, rowB[0], rowA[1]);
}
if (k % 2 == 1)
{
@@ -525,9 +485,7 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
AO += k << 3;
BO += k << 2;
}
i = (m & 7) >> 2;
/* Loop for m >= 4. */
for (j = 0; j < i; j++)
if (m & 4)
{
IFLOAT *BO = B;
v4sf_t *rowC;
@@ -539,7 +497,7 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
{
vec_t *rowA = (vec_t *) & (AO[l << 3]);
vec_t *rowB = (vec_t *) & (BO[l << 3]);
MMA (&acc0, MERGE_ROW (rowB[0]), MERGE_ROW (rowA[0]));
MMA (&acc0, rowB[0], rowA[0]);
}
if (k % 2 == 1)
{
@@ -555,9 +513,7 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
AO += k << 2;
BO += k << 2;
}
i = (m & 3) >> 1;
/* Loop for m >= 2. */
for (j = 0; j < i; j++)
if (m & 2)
{
IFLOAT *BO = B;
v2sf_t *rowC;
@@ -573,7 +529,7 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
0, 0, 0, 0
};
vec_t *rowB = (vec_t *) & (BO[l << 3]);
MMA (&acc0, MERGE_ROW (rowB[0]), (vec_t) rowA);
MMA (&acc0, rowB[0], (vec_t) rowA);
}
if (k % 2 == 1)
{
@@ -588,31 +544,32 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
AO += k << 1;
BO += k << 2;
}
i = (m & 1) >> 0;
/* Loop for m = 1. */
for (j = 0; j < i; j++)
if (m & 1)
{
IFLOAT *BO = B;
v2sf_t *rowC;
v2sf_t result[8];
__vector_quad acc0;
BLASLONG l = 0;
v4sf_t t = { 0, 0, 0, 0 };
for (l = 0; l < k; l++)
__builtin_mma_xxsetaccz (&acc0);
for (l = 0; l < k / 2; l++)
{
v4sf_t rowA =
{ BF16TOF32 (AO[l]), BF16TOF32 (AO[l]), BF16TOF32 (AO[l]),
BF16TOF32 (AO[l])
};
v4sf_t rowB =
{ BF16TOF32 (BO[l << 2]), BF16TOF32 (BO[(l << 2) + 1]),
BF16TOF32 (BO[(l << 2) + 2]),
BF16TOF32 (BO[(l << 2) + 3])
vector short rowA =
{ AO[(l << 1) + 0], AO[(l << 1) + 1], 0,
0, 0, 0, 0
};
t += rowA * rowB;
vec_t *rowB = (vec_t *) & (BO[l << 3]);
MMA (&acc0, rowB[0], (vec_t) rowA);
}
t = t * valpha;
CO[0 * ldc] += t[0];
CO[1 * ldc] += t[1];
CO[2 * ldc] += t[2];
CO[3 * ldc] += t[3];
if (k % 2 == 1)
{
if (k > 1)
l = (k / 2) << 1;
vector short rowA = { AO[l], 0, 0, 0, 0, 0, 0, 0 };
vec_t *rowB = (vec_t *) & (BO[l << 2]);
MMA (&acc0, MERGE_ROW (rowB[0]), (vec_t) rowA);
}
SAVE4x2_ACC (&acc0, 0);
AO += k;
BO += (k << 2);
CO += 1;
@@ -620,19 +577,16 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,

B += k << 2;
}
N = (n & 3) >> 1;
/* Loop for n >= 2. */
for (i1 = 0; i1 < N; i1++)
if (n & 2)
{
BLASLONG i, j;
BLASLONG j;
FLOAT *CO;
IFLOAT *AO;
CO = C;
C += ldc << 1;
AO = A;
i = m >> 5;
/* Loop for m >= 32. */
for (j = 0; j < i; j++)
for (j = 0; j < (m >> 5); j++)
{
IFLOAT *BO = B;
v4sf_t *rowC;
@@ -650,14 +604,14 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
};
vec_t *rowA = (vec_t *) & (AO[l << 5]);
vec_t *rowA1 = (vec_t *) & (A1[l << 5]);
MMA (&acc0, (vec_t) rowB, MERGE_HIGH (rowA[0], rowA[2]));
MMA (&acc1, (vec_t) rowB, MERGE_LOW (rowA[0], rowA[2]));
MMA (&acc2, (vec_t) rowB, MERGE_HIGH (rowA[1], rowA[3]));
MMA (&acc3, (vec_t) rowB, MERGE_LOW (rowA[1], rowA[3]));
MMA (&acc4, (vec_t) rowB, MERGE_HIGH (rowA1[0], rowA1[2]));
MMA (&acc5, (vec_t) rowB, MERGE_LOW (rowA1[0], rowA1[2]));
MMA (&acc6, (vec_t) rowB, MERGE_HIGH (rowA1[1], rowA1[3]));
MMA (&acc7, (vec_t) rowB, MERGE_LOW (rowA1[1], rowA1[3]));
MMA (&acc0, (vec_t) rowB, rowA[0]);
MMA (&acc1, (vec_t) rowB, rowA[1]);
MMA (&acc2, (vec_t) rowB, rowA[2]);
MMA (&acc3, (vec_t) rowB, rowA[3]);
MMA (&acc4, (vec_t) rowB, rowA1[0]);
MMA (&acc5, (vec_t) rowB, rowA1[1]);
MMA (&acc6, (vec_t) rowB, rowA1[2]);
MMA (&acc7, (vec_t) rowB, rowA1[3]);
}
if (k % 2 == 1)
{
@@ -688,9 +642,7 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
AO += k << 5;
BO += k << 1;
}
i = (m & 31) >> 4;
/* Loop for m >= 16. */
for (j = 0; j < i; j++)
if (m & 16)
{
IFLOAT *BO = B;
v4sf_t *rowC;
@@ -706,10 +658,10 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
0, 0, 0, 0
};
vec_t *rowA = (vec_t *) & (AO[l << 5]);
MMA (&acc0, (vec_t) rowB, MERGE_HIGH (rowA[0], rowA[2]));
MMA (&acc1, (vec_t) rowB, MERGE_LOW (rowA[0], rowA[2]));
MMA (&acc2, (vec_t) rowB, MERGE_HIGH (rowA[1], rowA[3]));
MMA (&acc3, (vec_t) rowB, MERGE_LOW (rowA[1], rowA[3]));
MMA (&acc0, (vec_t) rowB, rowA[0]);
MMA (&acc1, (vec_t) rowB, rowA[1]);
MMA (&acc2, (vec_t) rowB, rowA[2]);
MMA (&acc3, (vec_t) rowB, rowA[3]);
}
if (k % 2 == 1)
{
@@ -730,9 +682,7 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
AO += k << 4;
BO += k << 1;
}
i = (m & 15) >> 3;
/* Loop for m >= 8. */
for (j = 0; j < i; j++)
if (m & 8)
{
IFLOAT *BO = B;
v4sf_t *rowC;
@@ -749,8 +699,8 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
0, 0, 0, 0
};
vec_t *rowA = (vec_t *) & (AO[l << 4]);
MMA (&acc0, (vec_t) rowB, MERGE_HIGH (rowA[0], rowA[1]));
MMA (&acc1, (vec_t) rowB, MERGE_LOW (rowA[0], rowA[1]));
MMA (&acc0, (vec_t) rowB, rowA[0]);
MMA (&acc1, (vec_t) rowB, rowA[1]);
}
if (k % 2 == 1)
{
@@ -767,9 +717,7 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
AO += k << 3;
BO += k << 1;
}
i = (m & 7) >> 2;
/* Loop for m >= 4. */
for (j = 0; j < i; j++)
if (m & 4)
{
IFLOAT *BO = B;
v4sf_t *rowC;
@@ -785,7 +733,7 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
0, 0, 0, 0
};
vec_t *rowA = (vec_t *) & (AO[l << 3]);
MMA (&acc0, (vec_t) rowB, MERGE_ROW (rowA[0]));
MMA (&acc0, (vec_t) rowB, rowA[0]);
}
if (k % 2 == 1)
{
@@ -800,9 +748,7 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
AO += k << 2;
BO += k << 1;
}
i = (m & 3) >> 1;
/* Loop for m >= 2. */
for (j = 0; j < i; j++)
if (m & 2)
{
IFLOAT *BO = B;
BLASLONG l = 0;
@@ -828,9 +774,7 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
AO += k << 1;
BO += k << 1;
}
i = (m & 1) >> 0;
/* Loop for m = 1. */
for (j = 0; j < i; j++)
if (m & 1)
{
IFLOAT *BO = B;
BLASLONG l = 0;
@@ -852,153 +796,126 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
}
B += k << 1;
}
N = (n & 1) >> 0;
/* Loop for n = 1. */
for (i1 = 0; i1 < N; i1++)
if (n & 1)
{
BLASLONG i;
BLASLONG j;
FLOAT *CO;
IFLOAT *AO;
CO = C;
C += ldc;
AO = A;
i = m;
/* Loop for m >= 16. */
while (i >= 16)
for (j = 0; j < (m >> 4); j++)
{
IFLOAT *BO = B;
v4sf_t *rowC;
v4sf_t result[4];
__vector_quad acc0, acc1, acc2, acc3;
SET_ACC_ZERO4 ();
BLASLONG l = 0;
v4sf_t t = { 0, 0, 0, 0 };
v4sf_t t1 = { 0, 0, 0, 0 };
v4sf_t t2 = { 0, 0, 0, 0 };
v4sf_t t3 = { 0, 0, 0, 0 };
for (l = 0; l < k; l++)
for (l = 0; l < k / 2; l++)
{
v4sf_t rowB =
{ BF16TOF32 (BO[l]), BF16TOF32 (BO[l]), BF16TOF32 (BO[l]),
BF16TOF32 (BO[l])
};
v4sf_t rowA =
{ BF16TOF32 (AO[l << 4]), BF16TOF32 (AO[(l << 4) + 1]),
BF16TOF32 (AO[(l << 4) + 2]),
BF16TOF32 (AO[(l << 4) + 3])
};
v4sf_t rowA1 =
{ BF16TOF32 (AO[(l << 4) + 4]), BF16TOF32 (AO[(l << 4) + 5]),
BF16TOF32 (AO[(l << 4) + 6]),
BF16TOF32 (AO[(l << 4) + 7])
};
v4sf_t rowA2 =
{ BF16TOF32 (AO[(l << 4) + 8]), BF16TOF32 (AO[(l << 4) + 9]),
BF16TOF32 (AO[(l << 4) + 10]),
BF16TOF32 (AO[(l << 4) + 11])
};
v4sf_t rowA3 = { BF16TOF32 (AO[(l << 4) + 12]),
BF16TOF32 (AO[(l << 4) + 13]), BF16TOF32 (AO[(l << 4) + 14]),
BF16TOF32 (AO[(l << 4) + 15])
};
t += rowA * rowB;
t1 += rowA1 * rowB;
t2 += rowA2 * rowB;
t3 += rowA3 * rowB;
vector short rowB =
{ BO[l << 1], BO[(l << 1) + 1], 0, 0, 0, 0, 0, 0};
vec_t *rowA = (vec_t *) & (AO[l << 5]);
MMA (&acc0, (vec_t) rowB, rowA[0]);
MMA (&acc1, (vec_t) rowB, rowA[1]);
MMA (&acc2, (vec_t) rowB, rowA[2]);
MMA (&acc3, (vec_t) rowB, rowA[3]);
}
t = t * valpha;
t1 = t1 * valpha;
t2 = t2 * valpha;
t3 = t3 * valpha;
CO[0] += t[0];
CO[1] += t[1];
CO[2] += t[2];
CO[3] += t[3];
CO[4] += t1[0];
CO[5] += t1[1];
CO[6] += t1[2];
CO[7] += t1[3];
CO[8] += t2[0];
CO[9] += t2[1];
CO[10] += t2[2];
CO[11] += t2[3];
CO[12] += t3[0];
CO[13] += t3[1];
CO[14] += t3[2];
CO[15] += t3[3];
if (k % 2 == 1)
{
if (k > 1)
l = (k / 2) << 1;
vector short rowB = { BO[l], 0, 0, 0, 0, 0, 0, 0 };
vec_t *rowA = (vec_t *) & (AO[(l << 4)]);
MMA (&acc0, (vec_t) rowB, MERGE_HIGH (rowA[0], rowA[2]));
MMA (&acc1, (vec_t) rowB, MERGE_LOW (rowA[0], rowA[2]));
MMA (&acc2, (vec_t) rowB, MERGE_HIGH (rowA[1], rowA[3]));
MMA (&acc3, (vec_t) rowB, MERGE_LOW (rowA[1], rowA[3]));
}
rowC = (v4sf_t *) &CO[0];
__builtin_mma_disassemble_acc ((void *)result, &acc0);
rowC[0] += result[0] * alpha;
__builtin_mma_disassemble_acc ((void *)result, &acc1);
rowC[1] += result[0] * alpha;
__builtin_mma_disassemble_acc ((void *)result, &acc2);
rowC[2] += result[0] * alpha;
__builtin_mma_disassemble_acc ((void *)result, &acc3);
rowC[3] += result[0] * alpha;
AO += k << 4;
BO += k;
CO += 16;
i -= 16;
}
/* Loop for m >= 8. */
while (i >= 8)
if (m & 8)
{
IFLOAT *BO = B;
v4sf_t *rowC;
v4sf_t result[4];
__vector_quad acc0, acc1;
__builtin_mma_xxsetaccz (&acc0);
__builtin_mma_xxsetaccz (&acc1);
BLASLONG l = 0;
v4sf_t t = { 0, 0, 0, 0 };
v4sf_t t1 = { 0, 0, 0, 0 };
for (l = 0; l < k; l++)
for (l = 0; l < k / 2; l++)
{
v4sf_t rowB =
{ BF16TOF32 (BO[l]), BF16TOF32 (BO[l]), BF16TOF32 (BO[l]),
BF16TOF32 (BO[l])
};
v4sf_t rowA =
{ BF16TOF32 (AO[l << 3]), BF16TOF32 (AO[(l << 3) + 1]),
BF16TOF32 (AO[(l << 3) + 2]),
BF16TOF32 (AO[(l << 3) + 3])
};
v4sf_t rowA1 =
{ BF16TOF32 (AO[(l << 3) + 4]), BF16TOF32 (AO[(l << 3) + 5]),
BF16TOF32 (AO[(l << 3) + 6]),
BF16TOF32 (AO[(l << 3) + 7])
};
t += rowA * rowB;
t1 += rowA1 * rowB;
vector short rowB =
{ BO[l << 1], BO[(l << 1) + 1], 0, 0, 0, 0, 0, 0};
vec_t *rowA = (vec_t *) & (AO[l << 4]);
MMA (&acc0, (vec_t) rowB, rowA[0]);
MMA (&acc1, (vec_t) rowB, rowA[1]);
}
t = t * valpha;
t1 = t1 * valpha;
CO[0] += t[0];
CO[1] += t[1];
CO[2] += t[2];
CO[3] += t[3];
CO[4] += t1[0];
CO[5] += t1[1];
CO[6] += t1[2];
CO[7] += t1[3];
if (k % 2 == 1)
{
if (k > 1)
l = (k / 2) << 1;
vector short rowB = { BO[l], 0, 0, 0, 0, 0, 0, 0 };
vec_t *rowA = (vec_t *) & (AO[(l << 3)]);
MMA (&acc0, (vec_t) rowB, MERGE_HIGH (rowA[0], rowA[1]));
MMA (&acc1, (vec_t) rowB, MERGE_LOW (rowA[0], rowA[1]));
}
rowC = (v4sf_t *) &CO[0];
__builtin_mma_disassemble_acc ((void *)result, &acc0);
rowC[0] += result[0] * alpha;
__builtin_mma_disassemble_acc ((void *)result, &acc1);
rowC[1] += result[0] * alpha;
AO += k << 3;
BO += k;
CO += 8;
i -= 8;
}
/* Loop for m >= 4. */
while (i >= 4)
if (m & 4)
{
IFLOAT *BO = B;
v4sf_t *rowC;
v4sf_t result[4];
__vector_quad acc0;
__builtin_mma_xxsetaccz (&acc0);
BLASLONG l = 0;
v4sf_t t = { 0, 0, 0, 0 };
for (l = 0; l < k; l++)
for (l = 0; l < k / 2; l++)
{
v4sf_t rowB =
{ BF16TOF32 (BO[l]), BF16TOF32 (BO[l]), BF16TOF32 (BO[l]),
BF16TOF32 (BO[l])
};
v4sf_t rowA =
{ BF16TOF32 (AO[l << 2]), BF16TOF32 (AO[(l << 2) + 1]),
BF16TOF32 (AO[(l << 2) + 2]),
BF16TOF32 (AO[(l << 2) + 3])
};
t += rowA * rowB;
vector short rowB =
{ BO[l << 1], BO[(l << 1) + 1], 0, 0, 0, 0, 0, 0};
vec_t *rowA = (vec_t *) & (AO[l << 3]);
MMA (&acc0, (vec_t) rowB, rowA[0]);
}
t = t * valpha;
CO[0] += t[0];
CO[1] += t[1];
CO[2] += t[2];
CO[3] += t[3];
if (k % 2 == 1)
{
if (k > 1)
l = (k / 2) << 1;
vector short rowB = { BO[l], 0, 0, 0, 0, 0, 0, 0 };
vec_t *rowA = (vec_t *) & (AO[(l << 2)]);
MMA (&acc0, (vec_t) rowB, MERGE_ROW (rowA[0]));
}
rowC = (v4sf_t *) &CO[0];
__builtin_mma_disassemble_acc ((void *)result, &acc0);
rowC[0] += result[0] * alpha;
AO += k << 2;
BO += k;
CO += 4;
i -= 4;
}
/* Loop for m >= 2. */
while (i >= 2)
if (m & 2)
{
IFLOAT *BO = B;
BLASLONG l = 0;
@@ -1018,10 +935,9 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
AO += k << 1;
BO += k;
CO += 2;
i -= 2;
}
/* Loop for m = 1. */
while (i >= 1)
if (m & 1)
{
IFLOAT *BO = B;
BLASLONG l = 0;
@@ -1034,7 +950,6 @@ CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A,
BO += k;
CO[0] += t * alpha;
CO += 1;
i -= 1;
}

B += k;


+ 437
- 0
kernel/power/sbgemm_ncopy_16_power10.c View File

@@ -0,0 +1,437 @@
/*********************************************************************/
/* Copyright 2009, 2010 The University of Texas at Austin. */
/* All rights reserved. */
/* */
/* Redistribution and use in source and binary forms, with or */
/* without modification, are permitted provided that the following */
/* conditions are met: */
/* */
/* 1. Redistributions of source code must retain the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer. */
/* */
/* 2. Redistributions in binary form must reproduce the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer in the documentation and/or other materials */
/* provided with the distribution. */
/* */
/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */
/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */
/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */
/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */
/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */
/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */
/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */
/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */
/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */
/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */
/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */
/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */
/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */
/* POSSIBILITY OF SUCH DAMAGE. */
/* */
/* The views and conclusions contained in the software and */
/* documentation are those of the authors and should not be */
/* interpreted as representing official policies, either expressed */
/* or implied, of The University of Texas at Austin. */
/*********************************************************************/

#include <stdio.h>
#include "common.h"

int CNAME(BLASLONG m, BLASLONG n, IFLOAT *a, BLASLONG lda, IFLOAT *b){
BLASLONG i, j;

IFLOAT *aoffset;
IFLOAT *aoffset1, *aoffset2, *aoffset3, *aoffset4;
IFLOAT *aoffset5, *aoffset6, *aoffset7, *aoffset8;
IFLOAT *aoffset9, *aoffset10, *aoffset11, *aoffset12;
IFLOAT *aoffset13, *aoffset14, *aoffset15, *aoffset16;

IFLOAT *boffset;
IFLOAT ctemp01, ctemp02, ctemp03, ctemp04;
IFLOAT ctemp05, ctemp06, ctemp07, ctemp08;
IFLOAT ctemp09, ctemp10, ctemp11, ctemp12;
IFLOAT ctemp13, ctemp14, ctemp15, ctemp16;
IFLOAT ctemp17, ctemp18, ctemp19, ctemp20;
IFLOAT ctemp21, ctemp22, ctemp23, ctemp24;
IFLOAT ctemp25, ctemp26, ctemp27, ctemp28;
IFLOAT ctemp29, ctemp30, ctemp31, ctemp32;

aoffset = a;
boffset = b;

j = (n >> 4);
if (j > 0){
do{
aoffset1 = aoffset;
aoffset2 = aoffset1 + lda;
aoffset3 = aoffset2 + lda;
aoffset4 = aoffset3 + lda;
aoffset5 = aoffset4 + lda;
aoffset6 = aoffset5 + lda;
aoffset7 = aoffset6 + lda;
aoffset8 = aoffset7 + lda;
aoffset9 = aoffset8 + lda;
aoffset10 = aoffset9 + lda;
aoffset11 = aoffset10 + lda;
aoffset12 = aoffset11 + lda;
aoffset13 = aoffset12 + lda;
aoffset14 = aoffset13 + lda;
aoffset15 = aoffset14 + lda;
aoffset16 = aoffset15 + lda;
aoffset += 16 * lda;

i = (m >> 1);
if (i > 0){
do{
ctemp01 = *(aoffset1 + 0);
ctemp02 = *(aoffset1 + 1);
ctemp03 = *(aoffset2 + 0);
ctemp04 = *(aoffset2 + 1);

ctemp05 = *(aoffset3 + 0);
ctemp06 = *(aoffset3 + 1);
ctemp07 = *(aoffset4 + 0);
ctemp08 = *(aoffset4 + 1);

ctemp09 = *(aoffset5 + 0);
ctemp10 = *(aoffset5 + 1);
ctemp11 = *(aoffset6 + 0);
ctemp12 = *(aoffset6 + 1);

ctemp13 = *(aoffset7 + 0);
ctemp14 = *(aoffset7 + 1);
ctemp15 = *(aoffset8 + 0);
ctemp16 = *(aoffset8 + 1);

ctemp17 = *(aoffset9 + 0);
ctemp18 = *(aoffset9 + 1);
ctemp19 = *(aoffset10 + 0);
ctemp20 = *(aoffset10 + 1);

ctemp21 = *(aoffset11 + 0);
ctemp22 = *(aoffset11 + 1);
ctemp23 = *(aoffset12 + 0);
ctemp24 = *(aoffset12 + 1);

ctemp25 = *(aoffset13 + 0);
ctemp26 = *(aoffset13 + 1);
ctemp27 = *(aoffset14 + 0);
ctemp28 = *(aoffset14 + 1);

ctemp29 = *(aoffset15 + 0);
ctemp30 = *(aoffset15 + 1);
ctemp31 = *(aoffset16 + 0);
ctemp32 = *(aoffset16 + 1);

*(boffset + 0) = ctemp01;
*(boffset + 1) = ctemp02;
*(boffset + 2) = ctemp03;
*(boffset + 3) = ctemp04;
*(boffset + 4) = ctemp05;
*(boffset + 5) = ctemp06;
*(boffset + 6) = ctemp07;
*(boffset + 7) = ctemp08;

*(boffset + 8) = ctemp09;
*(boffset + 9) = ctemp10;
*(boffset + 10) = ctemp11;
*(boffset + 11) = ctemp12;
*(boffset + 12) = ctemp13;
*(boffset + 13) = ctemp14;
*(boffset + 14) = ctemp15;
*(boffset + 15) = ctemp16;

*(boffset + 16) = ctemp17;
*(boffset + 17) = ctemp18;
*(boffset + 18) = ctemp19;
*(boffset + 19) = ctemp20;
*(boffset + 20) = ctemp21;
*(boffset + 21) = ctemp22;
*(boffset + 22) = ctemp23;
*(boffset + 23) = ctemp24;

*(boffset + 24) = ctemp25;
*(boffset + 25) = ctemp26;
*(boffset + 26) = ctemp27;
*(boffset + 27) = ctemp28;
*(boffset + 28) = ctemp29;
*(boffset + 29) = ctemp30;
*(boffset + 30) = ctemp31;
*(boffset + 31) = ctemp32;

aoffset1 += 2;
aoffset2 += 2;
aoffset3 += 2;
aoffset4 += 2;
aoffset5 += 2;
aoffset6 += 2;
aoffset7 += 2;
aoffset8 += 2;

aoffset9 += 2;
aoffset10 += 2;
aoffset11 += 2;
aoffset12 += 2;
aoffset13 += 2;
aoffset14 += 2;
aoffset15 += 2;
aoffset16 += 2;
boffset += 32;

i --;
}while(i > 0);
}

if (m & 1){
ctemp01 = *(aoffset1 + 0);
ctemp03 = *(aoffset2 + 0);
ctemp05 = *(aoffset3 + 0);
ctemp07 = *(aoffset4 + 0);
ctemp09 = *(aoffset5 + 0);
ctemp11 = *(aoffset6 + 0);
ctemp13 = *(aoffset7 + 0);
ctemp15 = *(aoffset8 + 0);

ctemp17 = *(aoffset9 + 0);
ctemp19 = *(aoffset10 + 0);
ctemp21 = *(aoffset11 + 0);
ctemp23 = *(aoffset12 + 0);
ctemp25 = *(aoffset13 + 0);
ctemp27 = *(aoffset14 + 0);
ctemp29 = *(aoffset15 + 0);
ctemp31 = *(aoffset16 + 0);

*(boffset + 0) = ctemp01;
*(boffset + 1) = ctemp03;
*(boffset + 2) = ctemp05;
*(boffset + 3) = ctemp07;
*(boffset + 4) = ctemp09;
*(boffset + 5) = ctemp11;
*(boffset + 6) = ctemp13;
*(boffset + 7) = ctemp15;

*(boffset + 8) = ctemp17;
*(boffset + 9) = ctemp19;
*(boffset + 10) = ctemp21;
*(boffset + 11) = ctemp23;
*(boffset + 12) = ctemp25;
*(boffset + 13) = ctemp27;
*(boffset + 14) = ctemp29;
*(boffset + 15) = ctemp31;

boffset += 16;
}
j--;
}while(j > 0);
} /* end of if(j > 0) */

if (n & 8){
aoffset1 = aoffset;
aoffset2 = aoffset1 + lda;
aoffset3 = aoffset2 + lda;
aoffset4 = aoffset3 + lda;
aoffset5 = aoffset4 + lda;
aoffset6 = aoffset5 + lda;
aoffset7 = aoffset6 + lda;
aoffset8 = aoffset7 + lda;
aoffset += 8 * lda;

i = (m >> 1);
if (i > 0){
do{
ctemp01 = *(aoffset1 + 0);
ctemp02 = *(aoffset1 + 1);
ctemp03 = *(aoffset2 + 0);
ctemp04 = *(aoffset2 + 1);

ctemp05 = *(aoffset3 + 0);
ctemp06 = *(aoffset3 + 1);
ctemp07 = *(aoffset4 + 0);
ctemp08 = *(aoffset4 + 1);

ctemp09 = *(aoffset5 + 0);
ctemp10 = *(aoffset5 + 1);
ctemp11 = *(aoffset6 + 0);
ctemp12 = *(aoffset6 + 1);

ctemp13 = *(aoffset7 + 0);
ctemp14 = *(aoffset7 + 1);
ctemp15 = *(aoffset8 + 0);
ctemp16 = *(aoffset8 + 1);

*(boffset + 0) = ctemp01;
*(boffset + 1) = ctemp02;
*(boffset + 2) = ctemp03;
*(boffset + 3) = ctemp04;
*(boffset + 4) = ctemp05;
*(boffset + 5) = ctemp06;
*(boffset + 6) = ctemp07;
*(boffset + 7) = ctemp08;

*(boffset + 8) = ctemp09;
*(boffset + 9) = ctemp10;
*(boffset + 10) = ctemp11;
*(boffset + 11) = ctemp12;
*(boffset + 12) = ctemp13;
*(boffset + 13) = ctemp14;
*(boffset + 14) = ctemp15;
*(boffset + 15) = ctemp16;

aoffset1 += 2;
aoffset2 += 2;
aoffset3 += 2;
aoffset4 += 2;
aoffset5 += 2;
aoffset6 += 2;
aoffset7 += 2;
aoffset8 += 2;

boffset += 16;

i --;
}while(i > 0);
}

if (m & 1){
ctemp01 = *(aoffset1 + 0);
ctemp03 = *(aoffset2 + 0);
ctemp05 = *(aoffset3 + 0);
ctemp07 = *(aoffset4 + 0);
ctemp09 = *(aoffset5 + 0);
ctemp11 = *(aoffset6 + 0);
ctemp13 = *(aoffset7 + 0);
ctemp15 = *(aoffset8 + 0);

*(boffset + 0) = ctemp01;
*(boffset + 1) = ctemp03;
*(boffset + 2) = ctemp05;
*(boffset + 3) = ctemp07;
*(boffset + 4) = ctemp09;
*(boffset + 5) = ctemp11;
*(boffset + 6) = ctemp13;
*(boffset + 7) = ctemp15;

boffset += 8;
}
}

if (n & 4){
aoffset1 = aoffset;
aoffset2 = aoffset1 + lda;
aoffset3 = aoffset2 + lda;
aoffset4 = aoffset3 + lda;
aoffset += 4 * lda;

i = (m >> 1);
if (i > 0){
do{
ctemp01 = *(aoffset1 + 0);
ctemp02 = *(aoffset1 + 1);
ctemp03 = *(aoffset2 + 0);
ctemp04 = *(aoffset2 + 1);

ctemp05 = *(aoffset3 + 0);
ctemp06 = *(aoffset3 + 1);
ctemp07 = *(aoffset4 + 0);
ctemp08 = *(aoffset4 + 1);

*(boffset + 0) = ctemp01;
*(boffset + 1) = ctemp02;
*(boffset + 2) = ctemp03;
*(boffset + 3) = ctemp04;
*(boffset + 4) = ctemp05;
*(boffset + 5) = ctemp06;
*(boffset + 6) = ctemp07;
*(boffset + 7) = ctemp08;

aoffset1 += 2;
aoffset2 += 2;
aoffset3 += 2;
aoffset4 += 2;
boffset += 8;

i --;
}while(i > 0);
}

if (m & 1){
ctemp01 = *(aoffset1 + 0);
ctemp03 = *(aoffset2 + 0);
ctemp05 = *(aoffset3 + 0);
ctemp07 = *(aoffset4 + 0);

*(boffset + 0) = ctemp01;
*(boffset + 1) = ctemp03;
*(boffset + 2) = ctemp05;
*(boffset + 3) = ctemp07;
boffset += 4;
}
}

if (n & 2){
aoffset1 = aoffset;
aoffset2 = aoffset1 + lda;
aoffset += 2 * lda;

i = (m >> 1);
if (i > 0){
do{
ctemp01 = *(aoffset1 + 0);
ctemp02 = *(aoffset1 + 1);
ctemp03 = *(aoffset2 + 0);
ctemp04 = *(aoffset2 + 1);

*(boffset + 0) = ctemp01;
*(boffset + 1) = ctemp03;
*(boffset + 2) = ctemp02;
*(boffset + 3) = ctemp04;

aoffset1 += 2;
aoffset2 += 2;
boffset += 4;

i --;
}while(i > 0);
}

if (m & 1){
ctemp01 = *(aoffset1 + 0);
ctemp03 = *(aoffset2 + 0);

*(boffset + 0) = ctemp01;
*(boffset + 1) = ctemp03;
boffset += 2;
}
}

if (n & 1){
aoffset1 = aoffset;

i = (m >> 1);
if (i > 0){
do{
ctemp01 = *(aoffset1 + 0);
ctemp02 = *(aoffset1 + 1);

*(boffset + 0) = ctemp01;
*(boffset + 1) = ctemp02;

aoffset1 += 2;
boffset += 2;

i --;
}while(i > 0);
}

if (m & 1){
ctemp01 = *(aoffset1 + 0);

*(boffset + 0) = ctemp01;
// boffset += 1;
}
}

return 0;
}

+ 383
- 0
kernel/power/sbgemm_ncopy_8_power10.c View File

@@ -0,0 +1,383 @@
/*********************************************************************/
/* Copyright 2009, 2010 The University of Texas at Austin. */
/* All rights reserved. */
/* */
/* Redistribution and use in source and binary forms, with or */
/* without modification, are permitted provided that the following */
/* conditions are met: */
/* */
/* 1. Redistributions of source code must retain the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer. */
/* */
/* 2. Redistributions in binary form must reproduce the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer in the documentation and/or other materials */
/* provided with the distribution. */
/* */
/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */
/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */
/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */
/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */
/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */
/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */
/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */
/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */
/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */
/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */
/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */
/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */
/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */
/* POSSIBILITY OF SUCH DAMAGE. */
/* */
/* The views and conclusions contained in the software and */
/* documentation are those of the authors and should not be */
/* interpreted as representing official policies, either expressed */
/* or implied, of The University of Texas at Austin. */
/*********************************************************************/

#include <stdio.h>
#include <altivec.h>
#include "common.h"

typedef IFLOAT vec_bf16 __attribute__ ((vector_size (16)));
int CNAME(BLASLONG m, BLASLONG n, IFLOAT *a, BLASLONG lda, IFLOAT *b){
BLASLONG i, j;

IFLOAT *aoffset;
IFLOAT *aoffset1, *aoffset2, *aoffset3, *aoffset4;
IFLOAT *aoffset5, *aoffset6, *aoffset7, *aoffset8;

IFLOAT *boffset;
vec_bf16 vtemp01, vtemp02, vtemp03, vtemp04;
vec_bf16 vtemp05, vtemp06, vtemp07, vtemp08;
vec_bf16 vtemp09, vtemp10, vtemp11, vtemp12;
vector char mask =
{ 0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23 };
vector char mask1 =
{ 8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31 };
IFLOAT ctemp01, ctemp02, ctemp03, ctemp04;
IFLOAT ctemp05, ctemp06, ctemp07, ctemp08;
IFLOAT ctemp09, ctemp10, ctemp11, ctemp12;
IFLOAT ctemp13, ctemp14, ctemp15, ctemp16;
IFLOAT ctemp17;
IFLOAT ctemp25;
IFLOAT ctemp33;
IFLOAT ctemp41;
IFLOAT ctemp49;
IFLOAT ctemp57;


aoffset = a;
boffset = b;

j = (n >> 3);
if (j > 0){
do{
aoffset1 = aoffset;
aoffset2 = aoffset1 + lda;
aoffset3 = aoffset2 + lda;
aoffset4 = aoffset3 + lda;
aoffset5 = aoffset4 + lda;
aoffset6 = aoffset5 + lda;
aoffset7 = aoffset6 + lda;
aoffset8 = aoffset7 + lda;
aoffset += 8 * lda;

i = (m >> 3);
if (i > 0){
do{
vtemp01 = *(vec_bf16 *)(aoffset1);
vtemp02 = *(vec_bf16 *)(aoffset2);
vtemp03 = *(vec_bf16 *)(aoffset3);
vtemp04 = *(vec_bf16 *)(aoffset4);
vtemp05 = *(vec_bf16 *)(aoffset5);
vtemp06 = *(vec_bf16 *)(aoffset6);
vtemp07 = *(vec_bf16 *)(aoffset7);
vtemp08 = *(vec_bf16 *)(aoffset8);

vtemp09 = vec_perm(vtemp01, vtemp02, mask);
vtemp10 = vec_perm(vtemp03, vtemp04, mask);
vtemp11 = vec_perm(vtemp05, vtemp06, mask);
vtemp12 = vec_perm(vtemp07, vtemp08, mask);

*(vec_bf16 *)(boffset + 0) = vec_xxpermdi(vtemp09, vtemp10, 0);
*(vec_bf16 *)(boffset + 8) = vec_xxpermdi(vtemp11, vtemp12, 0);
*(vec_bf16 *)(boffset + 16) = vec_xxpermdi(vtemp09, vtemp10, 3);
*(vec_bf16 *)(boffset + 24) = vec_xxpermdi(vtemp11, vtemp12, 3);

vtemp09 = vec_perm(vtemp01, vtemp02, mask1);
vtemp10 = vec_perm(vtemp03, vtemp04, mask1);
vtemp11 = vec_perm(vtemp05, vtemp06, mask1);
vtemp12 = vec_perm(vtemp07, vtemp08, mask1);

*(vec_bf16 *)(boffset + 32) = vec_xxpermdi(vtemp09, vtemp10, 0);
*(vec_bf16 *)(boffset + 40) = vec_xxpermdi(vtemp11, vtemp12, 0);
*(vec_bf16 *)(boffset + 48) = vec_xxpermdi(vtemp09, vtemp10, 3);
*(vec_bf16 *)(boffset + 56) = vec_xxpermdi(vtemp11, vtemp12, 3);

aoffset1 += 8;
aoffset2 += 8;
aoffset3 += 8;
aoffset4 += 8;
aoffset5 += 8;
aoffset6 += 8;
aoffset7 += 8;
aoffset8 += 8;
boffset += 64;
i --;
}while(i > 0);
}

i = (m & 7);
if (i >= 2){
do{
ctemp01 = *(aoffset1 + 0);
ctemp09 = *(aoffset1 + 1);
ctemp17 = *(aoffset2 + 0);
ctemp25 = *(aoffset2 + 1);
ctemp33 = *(aoffset3 + 0);
ctemp41 = *(aoffset3 + 1);
ctemp49 = *(aoffset4 + 0);
ctemp57 = *(aoffset4 + 1);

*(boffset + 0) = ctemp01;
*(boffset + 1) = ctemp09;
*(boffset + 2) = ctemp17;
*(boffset + 3) = ctemp25;
*(boffset + 4) = ctemp33;
*(boffset + 5) = ctemp41;
*(boffset + 6) = ctemp49;
*(boffset + 7) = ctemp57;
aoffset1 += 2;
aoffset2 += 2;
aoffset3 += 2;
aoffset4 += 2;

ctemp01 = *(aoffset5 + 0);
ctemp09 = *(aoffset5 + 1);
ctemp17 = *(aoffset6 + 0);
ctemp25 = *(aoffset6 + 1);
ctemp33 = *(aoffset7 + 0);
ctemp41 = *(aoffset7 + 1);
ctemp49 = *(aoffset8 + 0);
ctemp57 = *(aoffset8 + 1);
*(boffset + 8) = ctemp01;
*(boffset + 9) = ctemp09;
*(boffset + 10) = ctemp17;
*(boffset + 11) = ctemp25;
*(boffset + 12) = ctemp33;
*(boffset + 13) = ctemp41;
*(boffset + 14) = ctemp49;
*(boffset + 15) = ctemp57;

aoffset5 += 2;
aoffset6 += 2;
aoffset7 += 2;
aoffset8 += 2;

boffset += 16;
i -= 2;
}while(i > 1);
}
if (m & 1){
ctemp01 = *(aoffset1 + 0);
ctemp09 = *(aoffset2 + 0);
ctemp17 = *(aoffset3 + 0);
ctemp25 = *(aoffset4 + 0);
ctemp33 = *(aoffset5 + 0);
ctemp41 = *(aoffset6 + 0);
ctemp49 = *(aoffset7 + 0);
ctemp57 = *(aoffset8 + 0);

*(boffset + 0) = ctemp01;
*(boffset + 1) = ctemp09;
*(boffset + 2) = ctemp17;
*(boffset + 3) = ctemp25;
*(boffset + 4) = ctemp33;
*(boffset + 5) = ctemp41;
*(boffset + 6) = ctemp49;
*(boffset + 7) = ctemp57;

aoffset1 ++;
aoffset2 ++;
aoffset3 ++;
aoffset4 ++;
aoffset5 ++;
aoffset6 ++;
aoffset7 ++;
aoffset8 ++;

boffset += 8;
}

j--;
}while(j > 0);
} /* end of if(j > 0) */

if (n & 4){
aoffset1 = aoffset;
aoffset2 = aoffset1 + lda;
aoffset3 = aoffset2 + lda;
aoffset4 = aoffset3 + lda;
aoffset += 4 * lda;

i = (m >> 2);
if (i > 0){
do{
ctemp01 = *(aoffset1 + 0);
ctemp02 = *(aoffset1 + 1);
ctemp03 = *(aoffset1 + 2);
ctemp04 = *(aoffset1 + 3);

ctemp05 = *(aoffset2 + 0);
ctemp06 = *(aoffset2 + 1);
ctemp07 = *(aoffset2 + 2);
ctemp08 = *(aoffset2 + 3);

ctemp09 = *(aoffset3 + 0);
ctemp10 = *(aoffset3 + 1);
ctemp11 = *(aoffset3 + 2);
ctemp12 = *(aoffset3 + 3);

ctemp13 = *(aoffset4 + 0);
ctemp14 = *(aoffset4 + 1);
ctemp15 = *(aoffset4 + 2);
ctemp16 = *(aoffset4 + 3);

*(boffset + 0) = ctemp01;
*(boffset + 1) = ctemp02;
*(boffset + 2) = ctemp05;
*(boffset + 3) = ctemp06;

*(boffset + 4) = ctemp09;
*(boffset + 5) = ctemp10;
*(boffset + 6) = ctemp13;
*(boffset + 7) = ctemp14;

*(boffset + 8) = ctemp03;
*(boffset + 9) = ctemp04;
*(boffset + 10) = ctemp07;
*(boffset + 11) = ctemp08;

*(boffset + 12) = ctemp11;
*(boffset + 13) = ctemp12;
*(boffset + 14) = ctemp15;
*(boffset + 15) = ctemp16;

aoffset1 += 4;
aoffset2 += 4;
aoffset3 += 4;
aoffset4 += 4;
boffset += 16;
i --;
}while(i > 0);
}

i = (m & 3);
if (i >= 2){
do{
ctemp01 = *(aoffset1 + 0);
ctemp09 = *(aoffset1 + 1);
ctemp17 = *(aoffset2 + 0);
ctemp25 = *(aoffset2 + 1);
ctemp33 = *(aoffset3 + 0);
ctemp41 = *(aoffset3 + 1);
ctemp49 = *(aoffset4 + 0);
ctemp57 = *(aoffset4 + 1);

*(boffset + 0) = ctemp01;
*(boffset + 1) = ctemp09;
*(boffset + 2) = ctemp17;
*(boffset + 3) = ctemp25;
*(boffset + 4) = ctemp33;
*(boffset + 5) = ctemp41;
*(boffset + 6) = ctemp49;
*(boffset + 7) = ctemp57;
aoffset1 += 2;
aoffset2 += 2;
aoffset3 += 2;
aoffset4 += 2;

boffset += 8;
i -= 2;
}while(i > 1);
}
if (m & 1){
ctemp01 = *(aoffset1 + 0);
ctemp02 = *(aoffset2 + 0);
ctemp03 = *(aoffset3 + 0);
ctemp04 = *(aoffset4 + 0);

*(boffset + 0) = ctemp01;
*(boffset + 1) = ctemp02;
*(boffset + 2) = ctemp03;
*(boffset + 3) = ctemp04;

aoffset1 ++;
aoffset2 ++;
aoffset3 ++;
aoffset4 ++;

boffset += 4;
}
}

if (n & 2){
aoffset1 = aoffset;
aoffset2 = aoffset1 + lda;
aoffset += 2 * lda;

i = (m >> 1);
if (i > 0){
do{
ctemp01 = *(aoffset1 + 0);
ctemp02 = *(aoffset1 + 1);
ctemp03 = *(aoffset2 + 0);
ctemp04 = *(aoffset2 + 1);

*(boffset + 0) = ctemp01;
*(boffset + 1) = ctemp03;
*(boffset + 2) = ctemp02;
*(boffset + 3) = ctemp04;

aoffset1 += 2;
aoffset2 += 2;
boffset += 4;
i --;
}while(i > 0);
}

if (m & 1){
ctemp01 = *(aoffset1 + 0);
ctemp02 = *(aoffset2 + 0);

*(boffset + 0) = ctemp01;
*(boffset + 1) = ctemp02;

aoffset1 ++;
aoffset2 ++;
boffset += 2;
}
} /* end of if(j > 0) */

if (n & 1){
aoffset1 = aoffset;

i = m;
if (i > 0){
do{
ctemp01 = *(aoffset1 + 0);

*(boffset + 0) = ctemp01;

aoffset1 ++;
boffset ++;
i --;
}while(i > 0);
}

} /* end of if(j > 0) */

return 0;
}

+ 244
- 0
kernel/power/sbgemm_tcopy_16_power10.c View File

@@ -0,0 +1,244 @@
/*********************************************************************/
/* Copyright 2009, 2010 The University of Texas at Austin. */
/* All rights reserved. */
/* */
/* Redistribution and use in source and binary forms, with or */
/* without modification, are permitted provided that the following */
/* conditions are met: */
/* */
/* 1. Redistributions of source code must retain the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer. */
/* */
/* 2. Redistributions in binary form must reproduce the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer in the documentation and/or other materials */
/* provided with the distribution. */
/* */
/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */
/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */
/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */
/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */
/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */
/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */
/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */
/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */
/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */
/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */
/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */
/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */
/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */
/* POSSIBILITY OF SUCH DAMAGE. */
/* */
/* The views and conclusions contained in the software and */
/* documentation are those of the authors and should not be */
/* interpreted as representing official policies, either expressed */
/* or implied, of The University of Texas at Austin. */
/*********************************************************************/

#include <stdio.h>
#include <altivec.h>
#include "common.h"
typedef IFLOAT vec_bf16 __attribute__ ((vector_size (16)));

int CNAME(BLASLONG m, BLASLONG n, IFLOAT *a, BLASLONG lda, IFLOAT *b){

BLASLONG i, j;

IFLOAT *aoffset;
IFLOAT *aoffset1, *aoffset2;
IFLOAT *boffset;

vec_bf16 vtemp01, vtemp02, vtemp03, vtemp04;
IFLOAT ctemp01, ctemp02, ctemp03, ctemp04;
IFLOAT ctemp05, ctemp06, ctemp07, ctemp08;

aoffset = a;
boffset = b;

#if 0
fprintf(stderr, "m = %d n = %d\n", m, n);
#endif

j = (n >> 4);
if (j > 0){
do{
aoffset1 = aoffset;
aoffset2 = aoffset + lda;
aoffset += 16;

i = (m >> 1);
if (i > 0){
do{
vtemp01 = *(vec_bf16 *)(aoffset1);
vtemp02 = *(vec_bf16 *)(aoffset1+8);
vtemp03 = *(vec_bf16 *)(aoffset2);
vtemp04 = *(vec_bf16 *)(aoffset2+8);
*(vec_bf16 *)(boffset + 0) = vec_mergeh(vtemp01, vtemp03);
*(vec_bf16 *)(boffset + 8) = vec_mergel(vtemp01, vtemp03);
*(vec_bf16 *)(boffset + 16) = vec_mergeh(vtemp02, vtemp04);
*(vec_bf16 *)(boffset + 24) = vec_mergel(vtemp02, vtemp04);
aoffset1 += 2 * lda;
aoffset2 += 2 * lda;
boffset += 32;

i --;
}while(i > 0);
}

if (m & 1){
vtemp01 = *(vec_bf16 *)(aoffset1);
vtemp02 = *(vec_bf16 *)(aoffset1+8);
*(vec_bf16 *)(boffset + 0) = vtemp01;
*(vec_bf16 *)(boffset + 8) = vtemp02;
boffset += 16;
}

j--;
}while(j > 0);
} /* end of if(j > 0) */

if (n & 8){
aoffset1 = aoffset;
aoffset2 = aoffset + lda;
aoffset += 8;

i = (m >> 1);
if (i > 0){
do{
vtemp01 = *(vec_bf16 *)(aoffset1);
vtemp03 = *(vec_bf16 *)(aoffset2);
*(vec_bf16 *)(boffset + 0) = vec_mergeh(vtemp01, vtemp03);
*(vec_bf16 *)(boffset + 8) = vec_mergel(vtemp01, vtemp03);

aoffset1 += 2 * lda;
aoffset2 += 2 * lda;
boffset += 16;

i --;
}while(i > 0);
}

if (m & 1){
vtemp01 = *(vec_bf16 *)(aoffset1);
*(vec_bf16 *)(boffset + 0) = vtemp01;
boffset += 8;
}
}

if (n & 4){
aoffset1 = aoffset;
aoffset2 = aoffset + lda;
aoffset += 4;

i = (m >> 1);
if (i > 0){
do{
ctemp01 = *(aoffset1 + 0);
ctemp02 = *(aoffset1 + 1);
ctemp03 = *(aoffset1 + 2);
ctemp04 = *(aoffset1 + 3);

ctemp05 = *(aoffset2 + 0);
ctemp06 = *(aoffset2 + 1);
ctemp07 = *(aoffset2 + 2);
ctemp08 = *(aoffset2 + 3);

*(boffset + 0) = ctemp01;
*(boffset + 1) = ctemp05;
*(boffset + 2) = ctemp02;
*(boffset + 3) = ctemp06;
*(boffset + 4) = ctemp03;
*(boffset + 5) = ctemp07;
*(boffset + 6) = ctemp04;
*(boffset + 7) = ctemp08;

aoffset1 += 2 * lda;
aoffset2 += 2 * lda;
boffset += 8;

i --;
}while(i > 0);
}

if (m & 1){
ctemp01 = *(aoffset1 + 0);
ctemp02 = *(aoffset1 + 1);
ctemp03 = *(aoffset1 + 2);
ctemp04 = *(aoffset1 + 3);

*(boffset + 0) = ctemp01;
*(boffset + 1) = ctemp02;
*(boffset + 2) = ctemp03;
*(boffset + 3) = ctemp04;

boffset += 4;
}
}

if (n & 2){
aoffset1 = aoffset;
aoffset2 = aoffset + lda;
aoffset += 2;

i = (m >> 1);
if (i > 0){
do{
ctemp01 = *(aoffset1 + 0);
ctemp02 = *(aoffset1 + 1);
ctemp03 = *(aoffset2 + 0);
ctemp04 = *(aoffset2 + 1);

*(boffset + 0) = ctemp01;
*(boffset + 1) = ctemp02;
*(boffset + 2) = ctemp03;
*(boffset + 3) = ctemp04;

aoffset1 += 2 * lda;
aoffset2 += 2 * lda;
boffset += 4;

i --;
}while(i > 0);
}

if (m & 1){
ctemp01 = *(aoffset1 + 0);
ctemp02 = *(aoffset1 + 1);

*(boffset + 0) = ctemp01;
*(boffset + 1) = ctemp02;
boffset += 2;
}
}

if (n & 1){
aoffset1 = aoffset;
aoffset2 = aoffset + lda;

i = (m >> 1);
if (i > 0){
do{
ctemp01 = *(aoffset1 + 0);
ctemp02 = *(aoffset2 + 0);

*(boffset + 0) = ctemp01;
*(boffset + 1) = ctemp02;

aoffset1 += 2 * lda;
aoffset2 += 2 * lda;
boffset += 2;

i --;
}while(i > 0);
}

if (m & 1){
ctemp01 = *(aoffset1 + 0);
*(boffset + 0) = ctemp01;
// boffset += 1;
}
}

return 0;
}

+ 659
- 0
kernel/power/sbgemm_tcopy_8_power10.c View File

@@ -0,0 +1,659 @@
/*********************************************************************/
/* Copyright 2009, 2010 The University of Texas at Austin. */
/* All rights reserved. */
/* */
/* Redistribution and use in source and binary forms, with or */
/* without modification, are permitted provided that the following */
/* conditions are met: */
/* */
/* 1. Redistributions of source code must retain the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer. */
/* */
/* 2. Redistributions in binary form must reproduce the above */
/* copyright notice, this list of conditions and the following */
/* disclaimer in the documentation and/or other materials */
/* provided with the distribution. */
/* */
/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */
/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */
/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */
/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */
/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */
/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */
/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */
/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */
/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */
/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */
/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */
/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */
/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */
/* POSSIBILITY OF SUCH DAMAGE. */
/* */
/* The views and conclusions contained in the software and */
/* documentation are those of the authors and should not be */
/* interpreted as representing official policies, either expressed */
/* or implied, of The University of Texas at Austin. */
/*********************************************************************/

#include <stdio.h>
#include "common.h"
#include <altivec.h>

typedef IFLOAT vec_bf16 __attribute__ ((vector_size (16)));

int CNAME(BLASLONG m, BLASLONG n, IFLOAT *a, BLASLONG lda, IFLOAT *b){

BLASLONG i, j;

IFLOAT *aoffset;
IFLOAT *aoffset1, *aoffset2, *aoffset3, *aoffset4;
IFLOAT *aoffset5, *aoffset6, *aoffset7, *aoffset8;

IFLOAT *boffset, *boffset1, *boffset2, *boffset3, *boffset4;
vec_bf16 vtemp01, vtemp02, vtemp03, vtemp04;
vec_bf16 vtemp05, vtemp06, vtemp07, vtemp08;
IFLOAT ctemp01, ctemp02, ctemp03, ctemp04;
IFLOAT ctemp05, ctemp06, ctemp07, ctemp08;
IFLOAT ctemp09, ctemp10, ctemp11, ctemp12;
IFLOAT ctemp13, ctemp14, ctemp15, ctemp16;
IFLOAT ctemp17, ctemp18, ctemp19, ctemp20;
IFLOAT ctemp21, ctemp22, ctemp23, ctemp24;
IFLOAT ctemp25, ctemp26, ctemp27, ctemp28;
IFLOAT ctemp29, ctemp30, ctemp31, ctemp32;

aoffset = a;
boffset = b;

#if 0
fprintf(stderr, "M = %d N = %d\n", m, n);
#endif

boffset2 = b + m * (n & ~7);
boffset3 = b + m * (n & ~3);
boffset4 = b + m * (n & ~1);

j = (m >> 3);
if (j > 0){
do{
aoffset1 = aoffset;
aoffset2 = aoffset1 + lda;
aoffset3 = aoffset2 + lda;
aoffset4 = aoffset3 + lda;
aoffset5 = aoffset4 + lda;
aoffset6 = aoffset5 + lda;
aoffset7 = aoffset6 + lda;
aoffset8 = aoffset7 + lda;
aoffset += 8 * lda;

boffset1 = boffset;
boffset += 64;

i = (n >> 3);
if (i > 0){
do{
vtemp01 = *(vec_bf16 *)(aoffset1);
vtemp02 = *(vec_bf16 *)(aoffset2);
vtemp03 = *(vec_bf16 *)(aoffset3);
vtemp04 = *(vec_bf16 *)(aoffset4);
vtemp05 = *(vec_bf16 *)(aoffset5);
vtemp06 = *(vec_bf16 *)(aoffset6);
vtemp07 = *(vec_bf16 *)(aoffset7);
vtemp08 = *(vec_bf16 *)(aoffset8);
aoffset1 += 8;
aoffset2 += 8;
aoffset3 += 8;
aoffset4 += 8;
aoffset5 += 8;
aoffset6 += 8;
aoffset7 += 8;
aoffset8 += 8;

*(vec_bf16 *)(boffset1 + 0) = vec_mergeh(vtemp01, vtemp02);
*(vec_bf16 *)(boffset1 + 8) = vec_mergel(vtemp01, vtemp02);
*(vec_bf16 *)(boffset1 + 16) = vec_mergeh(vtemp03, vtemp04);
*(vec_bf16 *)(boffset1 + 24) = vec_mergel(vtemp03, vtemp04);
*(vec_bf16 *)(boffset1 + 32) = vec_mergeh(vtemp05, vtemp06);
*(vec_bf16 *)(boffset1 + 40) = vec_mergel(vtemp05, vtemp06);
*(vec_bf16 *)(boffset1 + 48) = vec_mergeh(vtemp07, vtemp08);
*(vec_bf16 *)(boffset1 + 56) = vec_mergel(vtemp07, vtemp08);

boffset1 += m * 8;
i --;
}while(i > 0);
}

if (n & 4){
ctemp01 = *(aoffset1 + 0);
ctemp02 = *(aoffset1 + 1);
ctemp03 = *(aoffset1 + 2);
ctemp04 = *(aoffset1 + 3);
aoffset1 += 4;

ctemp05 = *(aoffset2 + 0);
ctemp06 = *(aoffset2 + 1);
ctemp07 = *(aoffset2 + 2);
ctemp08 = *(aoffset2 + 3);
aoffset2 += 4;

ctemp09 = *(aoffset3 + 0);
ctemp10 = *(aoffset3 + 1);
ctemp11 = *(aoffset3 + 2);
ctemp12 = *(aoffset3 + 3);
aoffset3 += 4;

ctemp13 = *(aoffset4 + 0);
ctemp14 = *(aoffset4 + 1);
ctemp15 = *(aoffset4 + 2);
ctemp16 = *(aoffset4 + 3);
aoffset4 += 4;

ctemp17 = *(aoffset5 + 0);
ctemp18 = *(aoffset5 + 1);
ctemp19 = *(aoffset5 + 2);
ctemp20 = *(aoffset5 + 3);
aoffset5 += 4;

ctemp21 = *(aoffset6 + 0);
ctemp22 = *(aoffset6 + 1);
ctemp23 = *(aoffset6 + 2);
ctemp24 = *(aoffset6 + 3);
aoffset6 += 4;

ctemp25 = *(aoffset7 + 0);
ctemp26 = *(aoffset7 + 1);
ctemp27 = *(aoffset7 + 2);
ctemp28 = *(aoffset7 + 3);
aoffset7 += 4;

ctemp29 = *(aoffset8 + 0);
ctemp30 = *(aoffset8 + 1);
ctemp31 = *(aoffset8 + 2);
ctemp32 = *(aoffset8 + 3);
aoffset8 += 4;

*(boffset2 + 0) = ctemp01;
*(boffset2 + 1) = ctemp05;
*(boffset2 + 2) = ctemp02;
*(boffset2 + 3) = ctemp06;
*(boffset2 + 4) = ctemp03;
*(boffset2 + 5) = ctemp07;
*(boffset2 + 6) = ctemp04;
*(boffset2 + 7) = ctemp08;

*(boffset2 + 8) = ctemp09;
*(boffset2 + 9) = ctemp13;
*(boffset2 + 10) = ctemp10;
*(boffset2 + 11) = ctemp14;
*(boffset2 + 12) = ctemp11;
*(boffset2 + 13) = ctemp15;
*(boffset2 + 14) = ctemp12;
*(boffset2 + 15) = ctemp16;

*(boffset2 + 16) = ctemp17;
*(boffset2 + 17) = ctemp21;
*(boffset2 + 18) = ctemp18;
*(boffset2 + 19) = ctemp22;
*(boffset2 + 20) = ctemp19;
*(boffset2 + 21) = ctemp23;
*(boffset2 + 22) = ctemp20;
*(boffset2 + 23) = ctemp24;

*(boffset2 + 24) = ctemp25;
*(boffset2 + 25) = ctemp29;
*(boffset2 + 26) = ctemp26;
*(boffset2 + 27) = ctemp30;
*(boffset2 + 28) = ctemp27;
*(boffset2 + 29) = ctemp31;
*(boffset2 + 30) = ctemp28;
*(boffset2 + 31) = ctemp32;

boffset2 += 32;
}

if (n & 2){
ctemp01 = *(aoffset1 + 0);
ctemp02 = *(aoffset1 + 1);
aoffset1 += 2;

ctemp03 = *(aoffset2 + 0);
ctemp04 = *(aoffset2 + 1);
aoffset2 += 2;

ctemp05 = *(aoffset3 + 0);
ctemp06 = *(aoffset3 + 1);
aoffset3 += 2;

ctemp07 = *(aoffset4 + 0);
ctemp08 = *(aoffset4 + 1);
aoffset4 += 2;

ctemp09 = *(aoffset5 + 0);
ctemp10 = *(aoffset5 + 1);
aoffset5 += 2;

ctemp11 = *(aoffset6 + 0);
ctemp12 = *(aoffset6 + 1);
aoffset6 += 2;

ctemp13 = *(aoffset7 + 0);
ctemp14 = *(aoffset7 + 1);
aoffset7 += 2;

ctemp15 = *(aoffset8 + 0);
ctemp16 = *(aoffset8 + 1);
aoffset8 += 2;

*(boffset3 + 0) = ctemp01;
*(boffset3 + 1) = ctemp02;
*(boffset3 + 2) = ctemp03;
*(boffset3 + 3) = ctemp04;
*(boffset3 + 4) = ctemp05;
*(boffset3 + 5) = ctemp06;
*(boffset3 + 6) = ctemp07;
*(boffset3 + 7) = ctemp08;
*(boffset3 + 8) = ctemp09;
*(boffset3 + 9) = ctemp10;
*(boffset3 + 10) = ctemp11;
*(boffset3 + 11) = ctemp12;
*(boffset3 + 12) = ctemp13;
*(boffset3 + 13) = ctemp14;
*(boffset3 + 14) = ctemp15;
*(boffset3 + 15) = ctemp16;
boffset3 += 16;
}

if (n & 1){
ctemp01 = *(aoffset1 + 0);
aoffset1 ++;
ctemp02 = *(aoffset2 + 0);
aoffset2 ++;
ctemp03 = *(aoffset3 + 0);
aoffset3 ++;
ctemp04 = *(aoffset4 + 0);
aoffset4 ++;
ctemp05 = *(aoffset5 + 0);
aoffset5 ++;
ctemp06 = *(aoffset6 + 0);
aoffset6 ++;
ctemp07 = *(aoffset7 + 0);
aoffset7 ++;
ctemp08 = *(aoffset8 + 0);
aoffset8 ++;

*(boffset4 + 0) = ctemp01;
*(boffset4 + 1) = ctemp02;
*(boffset4 + 2) = ctemp03;
*(boffset4 + 3) = ctemp04;
*(boffset4 + 4) = ctemp05;
*(boffset4 + 5) = ctemp06;
*(boffset4 + 6) = ctemp07;
*(boffset4 + 7) = ctemp08;
boffset4 += 8;
}

j--;
}while(j > 0);
}

if (m & 4){

aoffset1 = aoffset;
aoffset2 = aoffset1 + lda;
aoffset3 = aoffset2 + lda;
aoffset4 = aoffset3 + lda;
aoffset += 4 * lda;

boffset1 = boffset;
boffset += 32;

i = (n >> 3);
if (i > 0){

do{
ctemp01 = *(aoffset1 + 0);
ctemp02 = *(aoffset1 + 1);
ctemp03 = *(aoffset1 + 2);
ctemp04 = *(aoffset1 + 3);
ctemp05 = *(aoffset1 + 4);
ctemp06 = *(aoffset1 + 5);
ctemp07 = *(aoffset1 + 6);
ctemp08 = *(aoffset1 + 7);
aoffset1 += 8;

ctemp09 = *(aoffset2 + 0);
ctemp10 = *(aoffset2 + 1);
ctemp11 = *(aoffset2 + 2);
ctemp12 = *(aoffset2 + 3);
ctemp13 = *(aoffset2 + 4);
ctemp14 = *(aoffset2 + 5);
ctemp15 = *(aoffset2 + 6);
ctemp16 = *(aoffset2 + 7);
aoffset2 += 8;

ctemp17 = *(aoffset3 + 0);
ctemp18 = *(aoffset3 + 1);
ctemp19 = *(aoffset3 + 2);
ctemp20 = *(aoffset3 + 3);
ctemp21 = *(aoffset3 + 4);
ctemp22 = *(aoffset3 + 5);
ctemp23 = *(aoffset3 + 6);
ctemp24 = *(aoffset3 + 7);
aoffset3 += 8;

ctemp25 = *(aoffset4 + 0);
ctemp26 = *(aoffset4 + 1);
ctemp27 = *(aoffset4 + 2);
ctemp28 = *(aoffset4 + 3);
ctemp29 = *(aoffset4 + 4);
ctemp30 = *(aoffset4 + 5);
ctemp31 = *(aoffset4 + 6);
ctemp32 = *(aoffset4 + 7);
aoffset4 += 8;

*(boffset1 + 0) = ctemp01;
*(boffset1 + 1) = ctemp09;
*(boffset1 + 2) = ctemp02;
*(boffset1 + 3) = ctemp10;
*(boffset1 + 4) = ctemp03;
*(boffset1 + 5) = ctemp11;
*(boffset1 + 6) = ctemp04;
*(boffset1 + 7) = ctemp12;

*(boffset1 + 8) = ctemp05;
*(boffset1 + 9) = ctemp13;
*(boffset1 + 10) = ctemp06;
*(boffset1 + 11) = ctemp14;
*(boffset1 + 12) = ctemp07;
*(boffset1 + 13) = ctemp15;
*(boffset1 + 14) = ctemp08;
*(boffset1 + 15) = ctemp16;

*(boffset1 + 16) = ctemp17;
*(boffset1 + 17) = ctemp25;
*(boffset1 + 18) = ctemp18;
*(boffset1 + 19) = ctemp26;
*(boffset1 + 20) = ctemp19;
*(boffset1 + 21) = ctemp27;
*(boffset1 + 22) = ctemp20;
*(boffset1 + 23) = ctemp28;

*(boffset1 + 24) = ctemp21;
*(boffset1 + 25) = ctemp29;
*(boffset1 + 26) = ctemp22;
*(boffset1 + 27) = ctemp30;
*(boffset1 + 28) = ctemp23;
*(boffset1 + 29) = ctemp31;
*(boffset1 + 30) = ctemp24;
*(boffset1 + 31) = ctemp32;

boffset1 += 8 * m;
i --;
}while(i > 0);
}

if (n & 4) {
ctemp01 = *(aoffset1 + 0);
ctemp02 = *(aoffset1 + 1);
ctemp03 = *(aoffset1 + 2);
ctemp04 = *(aoffset1 + 3);
aoffset1 += 4;

ctemp05 = *(aoffset2 + 0);
ctemp06 = *(aoffset2 + 1);
ctemp07 = *(aoffset2 + 2);
ctemp08 = *(aoffset2 + 3);
aoffset2 += 4;

ctemp09 = *(aoffset3 + 0);
ctemp10 = *(aoffset3 + 1);
ctemp11 = *(aoffset3 + 2);
ctemp12 = *(aoffset3 + 3);
aoffset3 += 4;

ctemp13 = *(aoffset4 + 0);
ctemp14 = *(aoffset4 + 1);
ctemp15 = *(aoffset4 + 2);
ctemp16 = *(aoffset4 + 3);
aoffset4 += 4;

*(boffset2 + 0) = ctemp01;
*(boffset2 + 1) = ctemp05;
*(boffset2 + 2) = ctemp02;
*(boffset2 + 3) = ctemp06;
*(boffset2 + 4) = ctemp03;
*(boffset2 + 5) = ctemp07;
*(boffset2 + 6) = ctemp04;
*(boffset2 + 7) = ctemp08;

*(boffset2 + 8) = ctemp09;
*(boffset2 + 9) = ctemp13;
*(boffset2 + 10) = ctemp10;
*(boffset2 + 11) = ctemp14;
*(boffset2 + 12) = ctemp11;
*(boffset2 + 13) = ctemp15;
*(boffset2 + 14) = ctemp12;
*(boffset2 + 15) = ctemp16;
boffset2 += 16;
}

if (n & 2){
ctemp01 = *(aoffset1 + 0);
ctemp02 = *(aoffset1 + 1);
aoffset1 += 2;

ctemp03 = *(aoffset2 + 0);
ctemp04 = *(aoffset2 + 1);
aoffset2 += 2;

ctemp05 = *(aoffset3 + 0);
ctemp06 = *(aoffset3 + 1);
aoffset3 += 2;

ctemp07 = *(aoffset4 + 0);
ctemp08 = *(aoffset4 + 1);
aoffset4 += 2;

*(boffset3 + 0) = ctemp01;
*(boffset3 + 1) = ctemp02;
*(boffset3 + 2) = ctemp03;
*(boffset3 + 3) = ctemp04;
*(boffset3 + 4) = ctemp05;
*(boffset3 + 5) = ctemp06;
*(boffset3 + 6) = ctemp07;
*(boffset3 + 7) = ctemp08;
boffset3 += 8;
}

if (n & 1){
ctemp01 = *(aoffset1 + 0);
aoffset1 ++;
ctemp02 = *(aoffset2 + 0);
aoffset2 ++;
ctemp03 = *(aoffset3 + 0);
aoffset3 ++;
ctemp04 = *(aoffset4 + 0);
aoffset4 ++;

*(boffset4 + 0) = ctemp01;
*(boffset4 + 1) = ctemp02;
*(boffset4 + 2) = ctemp03;
*(boffset4 + 3) = ctemp04;
boffset4 += 4;
}
}

if (m & 2){
aoffset1 = aoffset;
aoffset2 = aoffset1 + lda;
aoffset += 2 * lda;

boffset1 = boffset;
boffset += 16;

i = (n >> 3);
if (i > 0){
do{
ctemp01 = *(aoffset1 + 0);
ctemp02 = *(aoffset1 + 1);
ctemp03 = *(aoffset1 + 2);
ctemp04 = *(aoffset1 + 3);
ctemp05 = *(aoffset1 + 4);
ctemp06 = *(aoffset1 + 5);
ctemp07 = *(aoffset1 + 6);
ctemp08 = *(aoffset1 + 7);
aoffset1 += 8;

ctemp09 = *(aoffset2 + 0);
ctemp10 = *(aoffset2 + 1);
ctemp11 = *(aoffset2 + 2);
ctemp12 = *(aoffset2 + 3);
ctemp13 = *(aoffset2 + 4);
ctemp14 = *(aoffset2 + 5);
ctemp15 = *(aoffset2 + 6);
ctemp16 = *(aoffset2 + 7);
aoffset2 += 8;

*(boffset1 + 0) = ctemp01;
*(boffset1 + 1) = ctemp09;
*(boffset1 + 2) = ctemp02;
*(boffset1 + 3) = ctemp10;
*(boffset1 + 4) = ctemp03;
*(boffset1 + 5) = ctemp11;
*(boffset1 + 6) = ctemp04;
*(boffset1 + 7) = ctemp12;

*(boffset1 + 8) = ctemp05;
*(boffset1 + 9) = ctemp13;
*(boffset1 + 10) = ctemp06;
*(boffset1 + 11) = ctemp14;
*(boffset1 + 12) = ctemp07;
*(boffset1 + 13) = ctemp15;
*(boffset1 + 14) = ctemp08;
*(boffset1 + 15) = ctemp16;

boffset1 += 8 * m;
i --;
}while(i > 0);
}

if (n & 4){
ctemp01 = *(aoffset1 + 0);
ctemp02 = *(aoffset1 + 1);
ctemp03 = *(aoffset1 + 2);
ctemp04 = *(aoffset1 + 3);
aoffset1 += 4;

ctemp05 = *(aoffset2 + 0);
ctemp06 = *(aoffset2 + 1);
ctemp07 = *(aoffset2 + 2);
ctemp08 = *(aoffset2 + 3);
aoffset2 += 4;

*(boffset2 + 0) = ctemp01;
*(boffset2 + 1) = ctemp05;
*(boffset2 + 2) = ctemp02;
*(boffset2 + 3) = ctemp06;
*(boffset2 + 4) = ctemp03;
*(boffset2 + 5) = ctemp07;
*(boffset2 + 6) = ctemp04;
*(boffset2 + 7) = ctemp08;
boffset2 += 8;
}

if (n & 2){
ctemp01 = *(aoffset1 + 0);
ctemp02 = *(aoffset1 + 1);
aoffset1 += 2;

ctemp03 = *(aoffset2 + 0);
ctemp04 = *(aoffset2 + 1);
aoffset2 += 2;

*(boffset3 + 0) = ctemp01;
*(boffset3 + 1) = ctemp02;
*(boffset3 + 2) = ctemp03;
*(boffset3 + 3) = ctemp04;
boffset3 += 4;
}

if (n & 1){
ctemp01 = *(aoffset1 + 0);
aoffset1 ++;
ctemp02 = *(aoffset2 + 0);
aoffset2 ++;

*(boffset4 + 0) = ctemp01;
*(boffset4 + 1) = ctemp02;
boffset4 += 2;
}
}

if (m & 1){
aoffset1 = aoffset;
// aoffset += lda;

boffset1 = boffset;
// boffset += 8;

i = (n >> 3);
if (i > 0){
do{
ctemp01 = *(aoffset1 + 0);
ctemp02 = *(aoffset1 + 1);
ctemp03 = *(aoffset1 + 2);
ctemp04 = *(aoffset1 + 3);
ctemp05 = *(aoffset1 + 4);
ctemp06 = *(aoffset1 + 5);
ctemp07 = *(aoffset1 + 6);
ctemp08 = *(aoffset1 + 7);
aoffset1 += 8;

*(boffset1 + 0) = ctemp01;
*(boffset1 + 1) = ctemp02;
*(boffset1 + 2) = ctemp03;
*(boffset1 + 3) = ctemp04;
*(boffset1 + 4) = ctemp05;
*(boffset1 + 5) = ctemp06;
*(boffset1 + 6) = ctemp07;
*(boffset1 + 7) = ctemp08;

boffset1 += 8 * m;
i --;
}while(i > 0);
}

if (n & 4){
ctemp01 = *(aoffset1 + 0);
ctemp02 = *(aoffset1 + 1);
ctemp03 = *(aoffset1 + 2);
ctemp04 = *(aoffset1 + 3);
aoffset1 += 4;

*(boffset2 + 0) = ctemp01;
*(boffset2 + 1) = ctemp02;
*(boffset2 + 2) = ctemp03;
*(boffset2 + 3) = ctemp04;
// boffset2 += 4;
}

if (n & 2){
ctemp01 = *(aoffset1 + 0);
ctemp02 = *(aoffset1 + 1);
aoffset1 += 2;

*(boffset3 + 0) = ctemp01;
*(boffset3 + 1) = ctemp02;
// boffset3 += 2;
}

if (n & 1){
ctemp01 = *(aoffset1 + 0);
aoffset1 ++;
*(boffset4 + 0) = ctemp01;
boffset4 ++;
}
}

return 0;
}

Write Preview
Loading…
Cancel
Save