OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
Browse Source

Merge pull request #80 from xianyi/develop 

rebase
tags/v0.3.11^2
Martin Kroeker GitHub 5 years ago
parent
b87a77da02 776d005f4c
commit
9d1ea75aa0
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
100 changed files with 380 additions and 311 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +6
    -0
      Makefile.zarch
  2. +1
    -1
      c_check
  3. +5
    -0
      cmake/system.cmake
  4. +2
    -0
      cmake/system_check.cmake
  5. +1
    -1
      kernel/zarch/camax.c
  6. +1
    -1
      kernel/zarch/camin.c
  7. +1
    -1
      kernel/zarch/casum.c
  8. +3
    -3
      kernel/zarch/caxpy.c
  9. +2
    -2
      kernel/zarch/ccopy.c
  10. +3
    -3
      kernel/zarch/cdot.c
  11. +15
    -15
      kernel/zarch/cgemv_n_4.c
  12. +16
    -16
      kernel/zarch/cgemv_t_4.c
  13. +2
    -2
      kernel/zarch/crot.c
  14. +7
    -7
      kernel/zarch/cscal.c
  15. +1
    -1
      kernel/zarch/csum.c
  16. +2
    -2
      kernel/zarch/cswap.c
  17. +9
    -9
      kernel/zarch/ctrmm4x4V.S
  18. +1
    -1
      kernel/zarch/damax.c
  19. +1
    -1
      kernel/zarch/damax_z13.c
  20. +1
    -1
      kernel/zarch/damin.c
  21. +1
    -1
      kernel/zarch/damin_z13.c
  22. +1
    -1
      kernel/zarch/dasum.c
  23. +2
    -2
      kernel/zarch/daxpy.c
  24. +2
    -2
      kernel/zarch/dcopy.c
  25. +2
    -2
      kernel/zarch/ddot.c
  26. +15
    -15
      kernel/zarch/dgemv_n_4.c
  27. +15
    -15
      kernel/zarch/dgemv_t_4.c
  28. +1
    -1
      kernel/zarch/dmax.c
  29. +1
    -1
      kernel/zarch/dmax_z13.c
  30. +1
    -1
      kernel/zarch/dmin.c
  31. +1
    -1
      kernel/zarch/dmin_z13.c
  32. +1
    -1
      kernel/zarch/drot.c
  33. +2
    -2
      kernel/zarch/dscal.c
  34. +2
    -2
      kernel/zarch/dsdot.c
  35. +1
    -1
      kernel/zarch/dsum.c
  36. +1
    -1
      kernel/zarch/dswap.c
  37. +12
    -12
      kernel/zarch/gemm8x4V.S
  38. +2
    -2
      kernel/zarch/gemm_vec.c
  39. +2
    -2
      kernel/zarch/icamax.c
  40. +2
    -2
      kernel/zarch/icamin.c
  41. +2
    -2
      kernel/zarch/idamax.c
  42. +2
    -2
      kernel/zarch/idamin.c
  43. +2
    -2
      kernel/zarch/idmax.c
  44. +2
    -2
      kernel/zarch/idmin.c
  45. +2
    -2
      kernel/zarch/isamax.c
  46. +2
    -2
      kernel/zarch/isamin.c
  47. +2
    -2
      kernel/zarch/ismax.c
  48. +2
    -2
      kernel/zarch/ismin.c
  49. +2
    -2
      kernel/zarch/izamax.c
  50. +2
    -2
      kernel/zarch/izamin.c
  51. +1
    -1
      kernel/zarch/samax.c
  52. +1
    -1
      kernel/zarch/samin.c
  53. +1
    -1
      kernel/zarch/sasum.c
  54. +2
    -2
      kernel/zarch/saxpy.c
  55. +2
    -2
      kernel/zarch/scopy.c
  56. +2
    -2
      kernel/zarch/sdot.c
  57. +15
    -15
      kernel/zarch/sgemv_n_4.c
  58. +15
    -15
      kernel/zarch/sgemv_t_4.c
  59. +1
    -1
      kernel/zarch/smax.c
  60. +1
    -1
      kernel/zarch/smin.c
  61. +1
    -1
      kernel/zarch/srot.c
  62. +2
    -2
      kernel/zarch/sscal.c
  63. +1
    -1
      kernel/zarch/ssum.c
  64. +1
    -1
      kernel/zarch/sswap.c
  65. +12
    -12
      kernel/zarch/strmm8x4V.S
  66. +1
    -1
      kernel/zarch/zamax.c
  67. +1
    -1
      kernel/zarch/zamax_z13.c
  68. +1
    -1
      kernel/zarch/zamin.c
  69. +1
    -1
      kernel/zarch/zamin_z13.c
  70. +1
    -1
      kernel/zarch/zasum.c
  71. +3
    -3
      kernel/zarch/zaxpy.c
  72. +2
    -2
      kernel/zarch/zcopy.c
  73. +3
    -3
      kernel/zarch/zdot.c
  74. +15
    -15
      kernel/zarch/zgemv_n_4.c
  75. +16
    -16
      kernel/zarch/zgemv_t_4.c
  76. +2
    -2
      kernel/zarch/zrot.c
  77. +7
    -7
      kernel/zarch/zscal.c
  78. +1
    -1
      kernel/zarch/zsum.c
  79. +2
    -2
      kernel/zarch/zswap.c
  80. +9
    -9
      kernel/zarch/ztrmm4x4V.S
  81. +6
    -5
      relapack/src/cgbtrf.c
  82. +7
    -2
      relapack/src/cgetrf.c
  83. +2
    -0
      relapack/src/chegst.c
  84. +2
    -2
      relapack/src/chetrf_rook.c
  85. +2
    -0
      relapack/src/clauum.c
  86. +6
    -4
      relapack/src/cpbtrf.c
  87. +3
    -0
      relapack/src/cpotrf.c
  88. +2
    -1
      relapack/src/csytrf.c
  89. +2
    -2
      relapack/src/csytrf_rook.c
  90. +7
    -0
      relapack/src/ctgsyl.c
  91. +5
    -0
      relapack/src/ctrsyl.c
  92. +2
    -0
      relapack/src/ctrtri.c
  93. +4
    -1
      relapack/src/dgbtrf.c
  94. +6
    -6
      relapack/src/dgetrf.c
  95. +6
    -4
      relapack/src/dpbtrf.c
  96. +2
    -1
      relapack/src/dsytrf.c
  97. +2
    -2
      relapack/src/dsytrf_rook.c
  98. +5
    -0
      relapack/src/dtrsyl.c
  99. +7
    -0
      relapack/src/lapack.h
  100. +12
    -4
      relapack/src/sgbtrf.c

+ 6
- 0
Makefile.zarch View File

@@ -8,3 +8,9 @@ ifeq ($(CORE), Z14)
CCOMMON_OPT += -march=z14 -mzvector -O3
FCOMMON_OPT += -march=z14 -mzvector
endif

# Enable floating-point expression contraction for clang, since it is the
# default for gcc
ifeq ($(C_COMPILER), CLANG)
CCOMMON_OPT += -ffp-contract=fast
endif

+ 1
- 1
c_check View File

@@ -8,7 +8,7 @@ $hostos = `uname -s | sed -e s/\-.*//`; chop($hostos);
$hostarch = `uname -m | sed -e s/i.86/x86/`;chop($hostarch);
$hostarch = `uname -p` if ($hostos eq "AIX");
$hostarch = "x86_64" if ($hostarch eq "amd64");
$hostarch = "arm" if ($hostarch =~ /^arm.*/);
$hostarch = "arm" if ($hostarch ne "arm64" && $hostarch =~ /^arm.*/);
$hostarch = "arm64" if ($hostarch eq "aarch64");
$hostarch = "power" if ($hostarch =~ /^(powerpc|ppc).*/);
$hostarch = "zarch" if ($hostarch eq "s390x");


+ 5
- 0
cmake/system.cmake View File

@@ -110,6 +110,11 @@ if (NO_AVX2)
set(GETARCH_FLAGS "${GETARCH_FLAGS} -DNO_AVX2")
endif ()

if (NO_AVX512)
message(STATUS "Disabling Advanced Vector Extensions 512 (AVX512).")
set(GETARCH_FLAGS "${GETARCH_FLAGS} -DNO_AVX512")
endif ()

if (CMAKE_BUILD_TYPE STREQUAL "Debug")
set(GETARCH_FLAGS "${GETARCH_FLAGS} ${CMAKE_C_FLAGS_DEBUG}")
endif ()


+ 2
- 0
cmake/system_check.cmake View File

@@ -109,6 +109,7 @@ else()
endif()

if (X86_64 OR X86)
if (NOT NO_AVX512)
file(WRITE ${PROJECT_BINARY_DIR}/avx512.c "#include <immintrin.h>\n\nint main(void){ __asm__ volatile(\"vbroadcastss -4 * 4(%rsi), %zmm2\"); }")
execute_process(COMMAND ${CMAKE_C_COMPILER} -march=skylake-avx512 -c -v -o ${PROJECT_BINARY_DIR}/avx512.o ${PROJECT_BINARY_DIR}/avx512.c OUTPUT_QUIET ERROR_QUIET RESULT_VARIABLE NO_AVX512)
if (NO_AVX512 EQUAL 1)
@@ -116,6 +117,7 @@ set (CCOMMON_OPT "${CCOMMON_OPT} -DNO_AVX512")
endif()
file(REMOVE "avx512.c" "avx512.o")
endif()
endif()

include(CheckIncludeFile)
CHECK_INCLUDE_FILE("stdatomic.h" HAVE_C11)


+ 1
- 1
kernel/zarch/camax.c View File

@@ -136,7 +136,7 @@ static FLOAT camax_kernel_32(BLASLONG n, FLOAT *x) {
"wfmaxsb %%v0,%%v0,%%v16,0\n\t"
"ler %[amax],%%f0"
: [amax] "=f"(amax),[n] "+&r"(n)
: "m"(*(const struct { FLOAT x[n * 2]; } *) x),[x] "a"(x)
: "m"(*(const FLOAT (*)[n * 2]) x),[x] "a"(x)
: "cc", "r1", "v0", "v1", "v2", "v16", "v17", "v18", "v19", "v20",
"v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30",
"v31");


+ 1
- 1
kernel/zarch/camin.c View File

@@ -136,7 +136,7 @@ static FLOAT camin_kernel_32(BLASLONG n, FLOAT *x) {
"wfminsb %%v0,%%v0,%%v16,0\n\t"
"ler %[amin],%%f0"
: [amin] "=f"(amin),[n] "+&r"(n)
: "m"(*(const struct { FLOAT x[n * 2]; } *) x),[x] "a"(x)
: "m"(*(const FLOAT (*)[n * 2]) x),[x] "a"(x)
: "cc", "r1", "v0", "v1", "v2", "v16", "v17", "v18", "v19", "v20",
"v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30",
"v31");


+ 1
- 1
kernel/zarch/casum.c View File

@@ -108,7 +108,7 @@ static FLOAT casum_kernel_32(BLASLONG n, FLOAT *x) {
"vfasb %%v24,%%v24,%%v25\n\t"
"vstef %%v24,%[asum],0"
: [asum] "=Q"(asum),[n] "+&r"(n)
: "m"(*(const struct { FLOAT x[n * 2]; } *) x),[x] "a"(x)
: "m"(*(const FLOAT (*)[n * 2]) x),[x] "a"(x)
: "cc", "r1", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23",
"v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31");



+ 3
- 3
kernel/zarch/caxpy.c View File

@@ -99,9 +99,9 @@ static void caxpy_kernel_16(BLASLONG n, FLOAT *x, FLOAT *y, FLOAT *alpha) {
"vst %%v19,112(%%r1,%[y])\n\t"
"agfi %%r1,128\n\t"
"brctg %[n],0b"
: "+m"(*(struct { FLOAT x[n * 2]; } *) y),[n] "+&r"(n)
: [y] "a"(y), "m"(*(const struct { FLOAT x[n * 2]; } *) x),[x] "a"(x),
"m"(*(const struct { FLOAT x[2]; } *) alpha),[alpha] "a"(alpha)
: "+m"(*(FLOAT (*)[n * 2]) y),[n] "+&r"(n)
: [y] "a"(y), "m"(*(const FLOAT (*)[n * 2]) x),[x] "a"(x),
"m"(*(const FLOAT (*)[2]) alpha),[alpha] "a"(alpha)
: "cc", "r1", "v0", "v1", "v8", "v9", "v10", "v11", "v12", "v13",
"v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23",
"v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31");


+ 2
- 2
kernel/zarch/ccopy.c View File

@@ -36,9 +36,9 @@ static void ccopy_kernel_32(BLASLONG n, FLOAT *x, FLOAT *y) {
"la %[x],256(%[x])\n\t"
"la %[y],256(%[y])\n\t"
"brctg %[n],0b"
: "=m"(*(struct { FLOAT x[n * 2]; } *) y),[x] "+&a"(x),[y] "+&a"(y),
: "=m"(*(FLOAT (*)[n * 2]) y),[x] "+&a"(x),[y] "+&a"(y),
[n] "+&r"(n)
: "m"(*(const struct { FLOAT x[n * 2]; } *) x)
: "m"(*(const FLOAT (*)[n * 2]) x)
: "cc");
}



+ 3
- 3
kernel/zarch/cdot.c View File

@@ -97,9 +97,9 @@ static void cdot_kernel_16(BLASLONG n, FLOAT *x, FLOAT *y, FLOAT *d) {
"vstef %%v24,4(%[d]),1\n\t"
"vstef %%v25,8(%[d]),1\n\t"
"vstef %%v25,12(%[d]),0"
: "=m"(*(struct { FLOAT x[4]; } *) d),[n] "+&r"(n)
: [d] "a"(d), "m"(*(const struct { FLOAT x[n * 2]; } *) x),[x] "a"(x),
"m"(*(const struct { FLOAT x[n * 2]; } *) y),[y] "a"(y)
: "=m"(*(FLOAT (*)[4]) d),[n] "+&r"(n)
: [d] "a"(d), "m"(*(const FLOAT (*)[n * 2]) x),[x] "a"(x),
"m"(*(const FLOAT (*)[n * 2]) y),[y] "a"(y)
: "cc", "r1", "v0", "v1", "v2", "v3", "v16", "v17", "v18", "v19", "v20",
"v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30",
"v31");


+ 15
- 15
kernel/zarch/cgemv_n_4.c View File

@@ -146,12 +146,12 @@ static void cgemv_kernel_4x4(BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y) {
"vst %%v0,0(%%r1,%[y])\n\t"
"agfi %%r1,16\n\t"
"brctg %[n],0b\n\t"
: "+m"(*(struct { FLOAT x[n * 2]; } *) y),[n] "+&r"(n)
: [y] "a"(y), "m"(*(const struct { FLOAT x[n * 2]; } *) ap0),[ap0] "a"(ap0),
"m"(*(const struct { FLOAT x[n * 2]; } *) ap1),[ap1] "a"(ap1),
"m"(*(const struct { FLOAT x[n * 2]; } *) ap2),[ap2] "a"(ap2),
"m"(*(const struct { FLOAT x[n * 2]; } *) ap3),[ap3] "a"(ap3),
"m"(*(const struct { FLOAT x[8]; } *) x),[x] "a"(x)
: "+m"(*(FLOAT (*)[n * 2]) y),[n] "+&r"(n)
: [y] "a"(y), "m"(*(const FLOAT (*)[n * 2]) ap0),[ap0] "a"(ap0),
"m"(*(const FLOAT (*)[n * 2]) ap1),[ap1] "a"(ap1),
"m"(*(const FLOAT (*)[n * 2]) ap2),[ap2] "a"(ap2),
"m"(*(const FLOAT (*)[n * 2]) ap3),[ap3] "a"(ap3),
"m"(*(const FLOAT (*)[8]) x),[x] "a"(x)
: "cc", "r1", "v0", "v1", "v2", "v16", "v17", "v18", "v19", "v20",
"v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30",
"v31");
@@ -238,10 +238,10 @@ static void cgemv_kernel_4x2(BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y) {
"vst %%v0,0(%%r1,%[y])\n\t"
"agfi %%r1,16\n\t"
"brctg %[n],0b\n\t"
: "+m"(*(struct { FLOAT x[n * 2]; } *) y),[n] "+&r"(n)
: [y] "a"(y), "m"(*(const struct { FLOAT x[n * 2]; } *) ap0),[ap0] "a"(ap0),
"m"(*(const struct { FLOAT x[n * 2]; } *) ap1),[ap1] "a"(ap1),
"m"(*(const struct { FLOAT x[4]; } *) x),[x] "a"(x)
: "+m"(*(FLOAT (*)[n * 2]) y),[n] "+&r"(n)
: [y] "a"(y), "m"(*(const FLOAT (*)[n * 2]) ap0),[ap0] "a"(ap0),
"m"(*(const FLOAT (*)[n * 2]) ap1),[ap1] "a"(ap1),
"m"(*(const FLOAT (*)[4]) x),[x] "a"(x)
: "cc", "r1", "v0", "v1", "v2", "v16", "v17", "v18", "v19", "v20",
"v21", "v22", "v23");
}
@@ -307,9 +307,9 @@ static void cgemv_kernel_4x1(BLASLONG n, FLOAT *ap, FLOAT *x, FLOAT *y) {
"vst %%v0,0(%%r1,%[y])\n\t"
"agfi %%r1,16\n\t"
"brctg %[n],0b\n\t"
: "+m"(*(struct { FLOAT x[n * 2]; } *) y),[n] "+&r"(n)
: [y] "a"(y), "m"(*(const struct { FLOAT x[n * 2]; } *) ap),[ap] "a"(ap),
"m"(*(const struct { FLOAT x[2]; } *) x),[x] "a"(x)
: "+m"(*(FLOAT (*)[n * 2]) y),[n] "+&r"(n)
: [y] "a"(y), "m"(*(const FLOAT (*)[n * 2]) ap),[ap] "a"(ap),
"m"(*(const FLOAT (*)[2]) x),[x] "a"(x)
: "cc", "r1", "v0", "v1", "v2", "v16", "v17", "v18", "v19");
}

@@ -350,8 +350,8 @@ static void add_y_4(BLASLONG n, FLOAT *src, FLOAT *dest, FLOAT alpha_r,
"vst %%v23,16(%%r1,%[dest])\n\t"
"agfi %%r1,32\n\t"
"brctg %[n],0b"
: "+m"(*(struct { FLOAT x[n * 2]; } *) dest),[n] "+&r"(n)
: [dest] "a"(dest), "m"(*(const struct { FLOAT x[n * 2]; } *) src),
: "+m"(*(FLOAT (*)[n * 2]) dest),[n] "+&r"(n)
: [dest] "a"(dest), "m"(*(const FLOAT (*)[n * 2]) src),
[src] "a"(src),[alpha_r] "Q"(alpha_r),[alpha_i] "Q"(alpha_i)
: "cc", "r1", "v0", "v1", "v16", "v17", "v18", "v19", "v20", "v21",
"v22", "v23");


+ 16
- 16
kernel/zarch/cgemv_t_4.c View File

@@ -159,13 +159,13 @@ static void cgemv_kernel_4x4(BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y,
"vfmasb %%v23,%%v19,%%v21,%%v23\n\t"
"vst %%v22,0(%[y])\n\t"
"vst %%v23,16(%[y])"
: "+m"(*(struct { FLOAT x[8]; } *) y),[n] "+&r"(n)
: [y] "a"(y), "m"(*(const struct { FLOAT x[n * 2]; } *) ap0),[ap0] "a"(ap0),
"m"(*(const struct { FLOAT x[n * 2]; } *) ap1),[ap1] "a"(ap1),
"m"(*(const struct { FLOAT x[n * 2]; } *) ap2),[ap2] "a"(ap2),
"m"(*(const struct { FLOAT x[n * 2]; } *) ap3),[ap3] "a"(ap3),
"m"(*(const struct { FLOAT x[n * 2]; } *) x),[x] "a"(x),
"m"(*(const struct { FLOAT x[2]; } *) alpha),[alpha] "a"(alpha)
: "+m"(*(FLOAT (*)[8]) y),[n] "+&r"(n)
: [y] "a"(y), "m"(*(const FLOAT (*)[n * 2]) ap0),[ap0] "a"(ap0),
"m"(*(const FLOAT (*)[n * 2]) ap1),[ap1] "a"(ap1),
"m"(*(const FLOAT (*)[n * 2]) ap2),[ap2] "a"(ap2),
"m"(*(const FLOAT (*)[n * 2]) ap3),[ap3] "a"(ap3),
"m"(*(const FLOAT (*)[n * 2]) x),[x] "a"(x),
"m"(*(const FLOAT (*)[2]) alpha),[alpha] "a"(alpha)
: "cc", "r1", "v0", "v1", "v2", "v3", "v16", "v17", "v18", "v19", "v20",
"v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30",
"v31");
@@ -271,11 +271,11 @@ static void cgemv_kernel_4x2(BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y,
"vfmasb %%v20,%%v16,%%v18,%%v20\n\t"
"vfmasb %%v20,%%v17,%%v19,%%v20\n\t"
"vst %%v20,0(%[y])"
: "+m"(*(struct { FLOAT x[4]; } *) y),[n] "+&r"(n)
: [y] "a"(y), "m"(*(const struct { FLOAT x[n * 2]; } *) ap0),[ap0] "a"(ap0),
"m"(*(const struct { FLOAT x[n * 2]; } *) ap1),[ap1] "a"(ap1),
"m"(*(const struct { FLOAT x[n * 2]; } *) x),[x] "a"(x),
"m"(*(const struct { FLOAT x[2]; } *) alpha),[alpha] "a"(alpha)
: "+m"(*(FLOAT (*)[4]) y),[n] "+&r"(n)
: [y] "a"(y), "m"(*(const FLOAT (*)[n * 2]) ap0),[ap0] "a"(ap0),
"m"(*(const FLOAT (*)[n * 2]) ap1),[ap1] "a"(ap1),
"m"(*(const FLOAT (*)[n * 2]) x),[x] "a"(x),
"m"(*(const FLOAT (*)[2]) alpha),[alpha] "a"(alpha)
: "cc", "r1", "v0", "v1", "v2", "v3", "v16", "v17", "v18", "v19", "v20",
"v21", "v22", "v23");
}
@@ -361,10 +361,10 @@ static void cgemv_kernel_4x1(BLASLONG n, FLOAT *ap, FLOAT *x, FLOAT *y,
"vfmasb %%v0,%%v16,%%v18,%%v0\n\t"
"vfmasb %%v0,%%v17,%%v19,%%v0\n\t"
"vsteg %%v0,0(%[y]),0"
: "+m"(*(struct { FLOAT x[2]; } *) y),[n] "+&r"(n)
: [y] "a"(y), "m"(*(const struct { FLOAT x[n * 2]; } *) ap),[ap] "a"(ap),
"m"(*(const struct { FLOAT x[n * 2]; } *) x),[x] "a"(x),
"m"(*(const struct { FLOAT x[2]; } *) alpha),[alpha] "a"(alpha)
: "+m"(*(FLOAT (*)[2]) y),[n] "+&r"(n)
: [y] "a"(y), "m"(*(const FLOAT (*)[n * 2]) ap),[ap] "a"(ap),
"m"(*(const FLOAT (*)[n * 2]) x),[x] "a"(x),
"m"(*(const FLOAT (*)[2]) alpha),[alpha] "a"(alpha)
: "cc", "r1", "v0", "v1", "v2", "v3", "v16", "v17", "v18", "v19");
}



+ 2
- 2
kernel/zarch/crot.c View File

@@ -169,8 +169,8 @@ static void crot_kernel_32(BLASLONG n, FLOAT *x, FLOAT *y, FLOAT *c, FLOAT *s) {
"vst %%v23, 240(%%r1,%[y])\n\t"
"agfi %%r1,256\n\t"
"brctg %[n],0b"
: "+m"(*(struct { FLOAT x[n * 2]; } *) x),
"+m"(*(struct { FLOAT x[n * 2]; } *) y),[n] "+&r"(n)
: "+m"(*(FLOAT (*)[n * 2]) x),
"+m"(*(FLOAT (*)[n * 2]) y),[n] "+&r"(n)
: [x] "a"(x),[y] "a"(y),[c] "Q"(*c),[s] "Q"(*s)
: "cc", "r1", "v0", "v1", "v16", "v17", "v18", "v19", "v20", "v21",
"v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30",


+ 7
- 7
kernel/zarch/cscal.c View File

@@ -80,8 +80,8 @@ static void cscal_kernel_16(BLASLONG n, FLOAT *alpha, FLOAT *x) {
"vst %%v23,112(%%r1,%[x])\n\t"
"agfi %%r1,128\n\t"
"brctg %[n],0b"
: "+m"(*(struct { FLOAT x[n * 2]; } *) x),[n] "+&r"(n)
: [x] "a"(x), "m"(*(const struct { FLOAT x[2]; } *) alpha),
: "+m"(*(FLOAT (*)[n * 2]) x),[n] "+&r"(n)
: [x] "a"(x), "m"(*(const FLOAT (*)[2]) alpha),
[alpha] "a"(alpha)
: "cc", "r1", "v0", "v1", "v16", "v17", "v18", "v19", "v20", "v21",
"v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30",
@@ -132,8 +132,8 @@ static void cscal_kernel_16_zero_r(BLASLONG n, FLOAT *alpha, FLOAT *x) {
"vst %%v23,112(%%r1,%[x])\n\t"
"agfi %%r1,128\n\t"
"brctg %[n],0b"
: "+m"(*(struct { FLOAT x[n * 2]; } *) x),[n] "+&r"(n)
: [x] "a"(x), "m"(*(const struct { FLOAT x[2]; } *) alpha),
: "+m"(*(FLOAT (*)[n * 2]) x),[n] "+&r"(n)
: [x] "a"(x), "m"(*(const FLOAT (*)[2]) alpha),
[alpha] "a"(alpha)
: "cc", "r1", "v0", "v16", "v17", "v18", "v19", "v20", "v21", "v22",
"v23");
@@ -171,8 +171,8 @@ static void cscal_kernel_16_zero_i(BLASLONG n, FLOAT *alpha, FLOAT *x) {
"vst %%v23,112(%%r1,%[x])\n\t"
"agfi %%r1,128\n\t"
"brctg %[n],0b"
: "+m"(*(struct { FLOAT x[n * 2]; } *) x),[n] "+&r"(n)
: [x] "a"(x), "m"(*(const struct { FLOAT x[2]; } *) alpha),
: "+m"(*(FLOAT (*)[n * 2]) x),[n] "+&r"(n)
: [x] "a"(x), "m"(*(const FLOAT (*)[2]) alpha),
[alpha] "a"(alpha)
: "cc", "r1", "v0", "v16", "v17", "v18", "v19", "v20", "v21", "v22",
"v23");
@@ -194,7 +194,7 @@ static void cscal_kernel_16_zero(BLASLONG n, FLOAT *x) {
"vst %%v0,112(%%r1,%[x])\n\t"
"agfi %%r1,128\n\t"
"brctg %[n],0b"
: "=m"(*(struct { FLOAT x[n * 2]; } *) x),[n] "+&r"(n)
: "=m"(*(FLOAT (*)[n * 2]) x),[n] "+&r"(n)
: [x] "a"(x)
: "cc", "r1", "v0");
}


+ 1
- 1
kernel/zarch/csum.c View File

@@ -90,7 +90,7 @@ static FLOAT csum_kernel_32(BLASLONG n, FLOAT *x) {
"vfasb %%v24,%%v24,%%v25\n\t"
"vstef %%v24,%[sum],0"
: [sum] "=Q"(sum),[n] "+&r"(n)
: "m"(*(const struct { FLOAT x[n * 2]; } *) x),[x] "a"(x)
: "m"(*(const FLOAT (*)[n * 2]) x),[x] "a"(x)
: "cc", "r1", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23",
"v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31");



+ 2
- 2
kernel/zarch/cswap.c View File

@@ -99,8 +99,8 @@ static void cswap_kernel_32(BLASLONG n, FLOAT *x, FLOAT *y) {
"vst %%v31, 240(%%r1,%[y])\n\t"
"agfi %%r1,256\n\t"
"brctg %[n],0b"
: "+m"(*(struct { FLOAT x[n * 2]; } *) x),
"+m"(*(struct { FLOAT x[n * 2]; } *) y),[n] "+&r"(n)
: "+m"(*(FLOAT (*)[n * 2]) x),
"+m"(*(FLOAT (*)[n * 2]) y),[n] "+&r"(n)
: [x] "a"(x),[y] "a"(y)
: "cc", "r1", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16",
"v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26",


+ 9
- 9
kernel/zarch/ctrmm4x4V.S View File

@@ -198,7 +198,7 @@ ALIGN_4
RefreshTempBk LOCAL_VAR1,BK,OFF,4,4
nill LOCAL_VAR1,3
#else
la LOCAL_VAR1,3(0,0)
lghi LOCAL_VAR1,3
NGR LOCAL_VAR1,BK /*refresh BK*/
#endif
jz .L4x4_BK_Store
@@ -254,7 +254,7 @@ ALIGN_4
RefreshTempBk LOCAL_VAR1,BK,OFF,2,4
nill LOCAL_VAR1,3
#else
la LOCAL_VAR1,3(0,0)
lghi LOCAL_VAR1,3
NGR LOCAL_VAR1,BK /*refresh BK*/
#endif
jz .L2x4_BK_Store
@@ -305,7 +305,7 @@ ALIGN_4
RefreshTempBk LOCAL_VAR1,BK,OFF,1,4
nill LOCAL_VAR1,3
#else
la LOCAL_VAR1,3(0,0)
lghi LOCAL_VAR1,3
NGR LOCAL_VAR1,BK /*refresh BK*/
#endif
jz .L1x4_BK_Store
@@ -385,7 +385,7 @@ ALIGN_4
RefreshTempBk LOCAL_VAR1,BK,OFF,4,2
nill LOCAL_VAR1,3
#else
la LOCAL_VAR1,3(0,0)
lghi LOCAL_VAR1,3
NGR LOCAL_VAR1,BK /*refresh BK*/
#endif
jz .L4x2_BK_Store
@@ -442,7 +442,7 @@ ALIGN_4
RefreshTempBk LOCAL_VAR1,BK,OFF,2,2
nill LOCAL_VAR1,3
#else
la LOCAL_VAR1,3(0,0)
lghi LOCAL_VAR1,3
NGR LOCAL_VAR1,BK /*refresh BK*/
#endif
jz .L2x2_BK_Store
@@ -492,7 +492,7 @@ ALIGN_4
RefreshTempBk LOCAL_VAR1,BK,OFF,1,2
nill LOCAL_VAR1,3
#else
la LOCAL_VAR1,3(0,0)
lghi LOCAL_VAR1,3
NGR LOCAL_VAR1,BK /*refresh BK*/
#endif
jz .L1x2_BK_Store
@@ -568,7 +568,7 @@ ALIGN_4
RefreshTempBk LOCAL_VAR1,BK,OFF,4,1
nill LOCAL_VAR1,3
#else
la LOCAL_VAR1,3(0,0)
lghi LOCAL_VAR1,3
NGR LOCAL_VAR1,BK /*refresh BK*/
#endif
jz .L4x1_BK_Store
@@ -620,7 +620,7 @@ ALIGN_4
RefreshTempBk LOCAL_VAR1,BK,OFF,2,1
nill LOCAL_VAR1,3
#else
la LOCAL_VAR1,3(0,0)
lghi LOCAL_VAR1,3
NGR LOCAL_VAR1,BK /*refresh BK*/
#endif
jz .L2x1_BK_Store
@@ -670,7 +670,7 @@ ALIGN_4
RefreshTempBk LOCAL_VAR1,BK,OFF,1,1
nill LOCAL_VAR1,3
#else
la LOCAL_VAR1,3(0,0)
lghi LOCAL_VAR1,3
NGR LOCAL_VAR1,BK /*refresh BK*/
#endif
jz .L1x1_BK_Store


+ 1
- 1
kernel/zarch/damax.c View File

@@ -76,7 +76,7 @@ static FLOAT damax_kernel_32(BLASLONG n, FLOAT *x) {
"wfmaxdb %%v0,%%v0,%%v16,8\n\t"
"lpdr %[amax],%%f0"
: [amax] "=f"(amax),[n] "+&r"(n)
: "m"(*(const struct { FLOAT x[n]; } *) x),[x] "a"(x)
: "m"(*(const FLOAT (*)[n]) x),[x] "a"(x)
: "cc", "r1", "v0", "v16", "v17", "v18", "v19", "v20", "v21", "v22",
"v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31");



+ 1
- 1
kernel/zarch/damax_z13.c View File

@@ -110,7 +110,7 @@ static FLOAT damax_kernel_32(BLASLONG n, FLOAT *x) {
"vsel %%v0,%%v0,%%v16,%%v17\n\t"
"ldr %[amax],%%f0"
: [amax] "=f"(amax),[n] "+&r"(n)
: "m"(*(const struct { FLOAT x[n]; } *) x),[x] "a"(x)
: "m"(*(const FLOAT (*)[n]) x),[x] "a"(x)
: "cc", "r1", "v0", "v16", "v17", "v18", "v19", "v20", "v21", "v22",
"v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31");



+ 1
- 1
kernel/zarch/damin.c View File

@@ -76,7 +76,7 @@ static FLOAT damin_kernel_32(BLASLONG n, FLOAT *x) {
"wfmindb %%v0,%%v0,%%v16,8\n\t"
"lpdr %[amin],%%f0"
: [amin] "=f"(amin),[n] "+&r"(n)
: "m"(*(const struct { FLOAT x[n]; } *) x),[x] "a"(x)
: "m"(*(const FLOAT (*)[n]) x),[x] "a"(x)
: "cc", "r1", "v0", "v16", "v17", "v18", "v19", "v20", "v21", "v22",
"v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31");



+ 1
- 1
kernel/zarch/damin_z13.c View File

@@ -110,7 +110,7 @@ static FLOAT damin_kernel_32(BLASLONG n, FLOAT *x) {
"vsel %%v0,%%v0,%%v16,%%v17\n\t"
"ldr %[amin],%%f0"
: [amin] "=f"(amin),[n] "+&r"(n)
: "m"(*(const struct { FLOAT x[n]; } *) x),[x] "a"(x)
: "m"(*(const FLOAT (*)[n]) x),[x] "a"(x)
: "cc", "r1", "v0", "v16", "v17", "v18", "v19", "v20", "v21", "v22",
"v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31");



+ 1
- 1
kernel/zarch/dasum.c View File

@@ -106,7 +106,7 @@ static FLOAT dasum_kernel_32(BLASLONG n, FLOAT *x) {
"vfadb %%v24,%%v24,%%v25\n\t"
"vsteg %%v24,%[asum],0"
: [asum] "=Q"(asum),[n] "+&r"(n)
: "m"(*(const struct { FLOAT x[n]; } *) x),[x] "a"(x)
: "m"(*(const FLOAT (*)[n]) x),[x] "a"(x)
: "cc", "r1", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23",
"v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31");



+ 2
- 2
kernel/zarch/daxpy.c View File

@@ -100,8 +100,8 @@ static void daxpy_kernel_32(BLASLONG n, FLOAT *x, FLOAT *y, FLOAT *alpha) {
"vst %%v27,240(%%r1,%[y])\n\t"
"agfi %%r1,256\n\t"
"brctg %[n],0b"
: "+m"(*(struct { FLOAT x[n]; } *) y),[n] "+&r"(n)
: [y] "a"(y), "m"(*(const struct { FLOAT x[n]; } *) x),[x] "a"(x),
: "+m"(*(FLOAT (*)[n]) y),[n] "+&r"(n)
: [y] "a"(y), "m"(*(const FLOAT (*)[n]) x),[x] "a"(x),
[alpha] "Q"(*alpha)
: "cc", "r1", "v0", "v16", "v17", "v18", "v19", "v20", "v21", "v22",
"v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31");


+ 2
- 2
kernel/zarch/dcopy.c View File

@@ -36,8 +36,8 @@ static void dcopy_kernel_32(BLASLONG n, FLOAT *x, FLOAT *y) {
"la %[x],256(%[x])\n\t"
"la %[y],256(%[y])\n\t"
"brctg %[n],0b"
: "=m"(*(struct { FLOAT x[n]; } *) y),[x] "+&a"(x),[y] "+&a"(y),[n] "+&r"(n)
: "m"(*(const struct { FLOAT x[n]; } *) x)
: "=m"(*(FLOAT (*)[n]) y),[x] "+&a"(x),[y] "+&a"(y),[n] "+&r"(n)
: "m"(*(const FLOAT (*)[n]) x)
: "cc");
}



+ 2
- 2
kernel/zarch/ddot.c View File

@@ -80,8 +80,8 @@ static FLOAT ddot_kernel_16(BLASLONG n, FLOAT *x, FLOAT *y) {
"adbr %%f0,%%f1\n\t"
"ldr %[dot],%%f0"
: [dot] "=f"(dot),[n] "+&r"(n)
: "m"(*(const struct { FLOAT x[n]; } *) x),[x] "a"(x),
"m"(*(const struct { FLOAT x[n]; } *) y),[y] "a"(y)
: "m"(*(const FLOAT (*)[n]) x),[x] "a"(x),
"m"(*(const FLOAT (*)[n]) y),[y] "a"(y)
: "cc", "r1", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16",
"v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26",
"v27", "v28", "v29", "v30", "v31");


+ 15
- 15
kernel/zarch/dgemv_n_4.c View File

@@ -169,13 +169,13 @@ static void dgemv_kernel_4x4(BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y,
"agfi %%r1,32\n\t"
"brctg %%r0,2b\n\t"
"3:\n\t"
"nop"
: "+m"(*(struct { FLOAT x[n]; } *) y)
: [y] "a"(y), "m"(*(const struct { FLOAT x[n]; } *) ap0),[ap0] "a"(ap0),
"m"(*(const struct { FLOAT x[n]; } *) ap1),[ap1] "a"(ap1),
"m"(*(const struct { FLOAT x[n]; } *) ap2),[ap2] "a"(ap2),
"m"(*(const struct { FLOAT x[n]; } *) ap3),[ap3] "a"(ap3),
"m"(*(const struct { FLOAT x[4]; } *) x),[x] "a"(x),[alpha] "Q"(*alpha),
"nop 0"
: "+m"(*(FLOAT (*)[n]) y)
: [y] "a"(y), "m"(*(const FLOAT (*)[n]) ap0),[ap0] "a"(ap0),
"m"(*(const FLOAT (*)[n]) ap1),[ap1] "a"(ap1),
"m"(*(const FLOAT (*)[n]) ap2),[ap2] "a"(ap2),
"m"(*(const FLOAT (*)[n]) ap3),[ap3] "a"(ap3),
"m"(*(const FLOAT (*)[4]) x),[x] "a"(x),[alpha] "Q"(*alpha),
[n] "r"(n)
: "cc", "r0", "r1", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
"v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25",
@@ -274,11 +274,11 @@ static void dgemv_kernel_4x2(BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y,
"agfi %%r1,32\n\t"
"brctg %%r0,2b\n\t"
"3:\n\t"
"nop"
: "+m"(*(struct { FLOAT x[n]; } *) y)
: [y] "a"(y), "m"(*(const struct { FLOAT x[n]; } *) ap0),[ap0] "a"(ap0),
"m"(*(const struct { FLOAT x[n]; } *) ap1),[ap1] "a"(ap1),
"m"(*(const struct { FLOAT x[2]; } *) x),[x] "a"(x),[alpha] "Q"(*alpha),
"nop 0"
: "+m"(*(FLOAT (*)[n]) y)
: [y] "a"(y), "m"(*(const FLOAT (*)[n]) ap0),[ap0] "a"(ap0),
"m"(*(const FLOAT (*)[n]) ap1),[ap1] "a"(ap1),
"m"(*(const FLOAT (*)[2]) x),[x] "a"(x),[alpha] "Q"(*alpha),
[n] "r"(n)
: "cc", "r0", "r1", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
"v8", "v9", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23",
@@ -351,9 +351,9 @@ static void dgemv_kernel_4x1(BLASLONG n, FLOAT *a0, FLOAT *x, FLOAT *y,
"agfi %%r1,32\n\t"
"brctg %%r0,2b\n\t"
"3:\n\t"
"nop"
: "+m"(*(struct { FLOAT x[n]; } *) y)
: [y] "a"(y), "m"(*(const struct { FLOAT x[n]; } *) a0),[a0] "a"(a0),
"nop 0"
: "+m"(*(FLOAT (*)[n]) y)
: [y] "a"(y), "m"(*(const FLOAT (*)[n]) a0),[a0] "a"(a0),
"m"(*(const FLOAT (*)[1]) x),[x] "a"(x),[alpha] "Q"(*alpha),
[n] "r"(n)
: "cc", "r0", "r1", "v0", "v16", "v17", "v18", "v19", "v20", "v21",


+ 15
- 15
kernel/zarch/dgemv_t_4.c View File

@@ -173,12 +173,12 @@ static void dgemv_kernel_4x4(BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y) {
"vrepg %%v4,%%v3,1\n\t"
"adbr %%f3,%%f4\n\t"
"std %%f3,24(%[y])"
: "=m"(*(struct { FLOAT x[4]; } *) y)
: [y] "a"(y), "m"(*(const struct { FLOAT x[n]; } *) ap0),[ap0] "a"(ap0),
"m"(*(const struct { FLOAT x[n]; } *) ap1),[ap1] "a"(ap1),
"m"(*(const struct { FLOAT x[n]; } *) ap2),[ap2] "a"(ap2),
"m"(*(const struct { FLOAT x[n]; } *) ap3),[ap3] "a"(ap3),
"m"(*(const struct { FLOAT x[n]; } *) x),[x] "a"(x),[n] "r"(n)
: "=m"(*(FLOAT (*)[4]) y)
: [y] "a"(y), "m"(*(const FLOAT (*)[n]) ap0),[ap0] "a"(ap0),
"m"(*(const FLOAT (*)[n]) ap1),[ap1] "a"(ap1),
"m"(*(const FLOAT (*)[n]) ap2),[ap2] "a"(ap2),
"m"(*(const FLOAT (*)[n]) ap3),[ap3] "a"(ap3),
"m"(*(const FLOAT (*)[n]) x),[x] "a"(x),[n] "r"(n)
: "cc", "r0", "r1", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
"v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25",
"v26", "v27", "v28", "v29", "v30", "v31");
@@ -280,10 +280,10 @@ static void dgemv_kernel_4x2(BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y) {
"vrepg %%v2,%%v1,1\n\t"
"adbr %%f1,%%f2\n\t"
"std %%f1,8(%[y])"
: "=m"(*(struct { FLOAT x[2]; } *) y)
: [y] "a"(y), "m"(*(const struct { FLOAT x[n]; } *) ap0),[ap0] "a"(ap0),
"m"(*(const struct { FLOAT x[n]; } *) ap1),[ap1] "a"(ap1),
"m"(*(const struct { FLOAT x[n]; } *) x),[x] "a"(x),[n] "r"(n)
: "=m"(*(FLOAT (*)[2]) y)
: [y] "a"(y), "m"(*(const FLOAT (*)[n]) ap0),[ap0] "a"(ap0),
"m"(*(const FLOAT (*)[n]) ap1),[ap1] "a"(ap1),
"m"(*(const FLOAT (*)[n]) x),[x] "a"(x),[n] "r"(n)
: "cc", "r0", "r1", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
"v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25",
"v26", "v27", "v28", "v29", "v30", "v31");
@@ -360,8 +360,8 @@ static void dgemv_kernel_4x1(BLASLONG n, FLOAT *a0, FLOAT *x, FLOAT *y) {
"adbr %%f0,%%f1\n\t"
"std %%f0,0(%[y])"
: "=m"(*(FLOAT (*)[1]) y)
: [y] "a"(y), "m"(*(const struct { FLOAT x[n]; } *) a0),[a0] "a"(a0),
"m"(*(const struct { FLOAT x[n]; } *) x),[x] "a"(x),[n] "r"(n)
: [y] "a"(y), "m"(*(const FLOAT (*)[n]) a0),[a0] "a"(a0),
"m"(*(const FLOAT (*)[n]) x),[x] "a"(x),[n] "r"(n)
: "cc", "r0", "r1", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
"v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25",
"v26", "v27", "v28", "v29", "v30", "v31");
@@ -438,9 +438,9 @@ static void add_y_kernel_4(BLASLONG n, FLOAT da, FLOAT *src, FLOAT *dest) {
"agfi %%r1,32\n\t"
"brctg %%r0,2b\n\t"
"3:\n\t"
"nop"
: "+m"(*(struct { FLOAT x[n]; } *) dest)
: [dest] "a"(dest),[da] "Q"(da), "m"(*(const struct { FLOAT x[n]; } *) src),
"nop 0"
: "+m"(*(FLOAT (*)[n]) dest)
: [dest] "a"(dest),[da] "Q"(da), "m"(*(const FLOAT (*)[n]) src),
[src] "a"(src),[n] "r"(n)
: "cc", "r0", "r1", "v0", "v16", "v17", "v18", "v19", "v20", "v21",
"v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30",


+ 1
- 1
kernel/zarch/dmax.c View File

@@ -73,7 +73,7 @@ static FLOAT dmax_kernel_32(BLASLONG n, FLOAT *x) {
"wfmaxdb %%v0,%%v0,%%v16,0\n\t"
"ldr %[max],%%f0"
: [max] "=f"(max),[n] "+&r"(n)
: "m"(*(const struct { FLOAT x[n]; } *) x),[x] "a"(x)
: "m"(*(const FLOAT (*)[n]) x),[x] "a"(x)
: "cc", "r1", "v0", "v16", "v17", "v18", "v19", "v20", "v21", "v22",
"v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31");



+ 1
- 1
kernel/zarch/dmax_z13.c View File

@@ -90,7 +90,7 @@ static FLOAT dmax_kernel_32(BLASLONG n, FLOAT *x) {
"vsel %%v0,%%v0,%%v16,%%v17\n\t"
"ldr %[max],%%f0"
: [max] "=f"(max),[n] "+&r"(n)
: "m"(*(const struct { FLOAT x[n]; } *) x),[x] "a"(x)
: "m"(*(const FLOAT (*)[n]) x),[x] "a"(x)
: "cc", "r1", "v0", "v16", "v17", "v18", "v19", "v20", "v21", "v22",
"v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31");



+ 1
- 1
kernel/zarch/dmin.c View File

@@ -73,7 +73,7 @@ static FLOAT dmin_kernel_32(BLASLONG n, FLOAT *x) {
"wfmindb %%v0,%%v0,%%v16,0\n\t"
"ldr %[min],%%f0"
: [min] "=f"(min),[n] "+&r"(n)
: "m"(*(const struct { FLOAT x[n]; } *) x),[x] "a"(x)
: "m"(*(const FLOAT (*)[n]) x),[x] "a"(x)
: "cc", "r1", "v0", "v16", "v17", "v18", "v19", "v20", "v21", "v22",
"v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31");



+ 1
- 1
kernel/zarch/dmin_z13.c View File

@@ -90,7 +90,7 @@ static FLOAT dmin_kernel_32(BLASLONG n, FLOAT *x) {
"vsel %%v0,%%v0,%%v16,%%v17\n\t"
"ldr %[min],%%f0"
: [min] "=f"(min),[n] "+&r"(n)
: "m"(*(const struct { FLOAT x[n]; } *) x),[x] "a"(x)
: "m"(*(const FLOAT (*)[n]) x),[x] "a"(x)
: "cc", "r1", "v0", "v16", "v17", "v18", "v19", "v20", "v21", "v22",
"v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31");



+ 1
- 1
kernel/zarch/drot.c View File

@@ -169,7 +169,7 @@ static void drot_kernel_32(BLASLONG n, FLOAT *x, FLOAT *y, FLOAT *c, FLOAT *s) {
"vst %%v23, 240(%%r1,%[y])\n\t"
"agfi %%r1,256\n\t"
"brctg %[n],0b"
: "+m"(*(struct { FLOAT x[n]; } *) x), "+m"(*(struct { FLOAT x[n]; } *) y),
: "+m"(*(FLOAT (*)[n]) x), "+m"(*(FLOAT (*)[n]) y),
[n] "+&r"(n)
: [x] "a"(x),[y] "a"(y),[c] "Q"(*c),[s] "Q"(*s)
: "cc", "r1", "v0", "v1", "v16", "v17", "v18", "v19", "v20", "v21",


+ 2
- 2
kernel/zarch/dscal.c View File

@@ -59,7 +59,7 @@ static void dscal_kernel_16(BLASLONG n, FLOAT da, FLOAT *x) {
"vst %%v31,112(%%r1,%[x])\n\t"
"agfi %%r1,128\n\t"
"brctg %[n],0b"
: "+m"(*(struct { FLOAT x[n]; } *) x),[n] "+&r"(n)
: "+m"(*(FLOAT (*)[n]) x),[n] "+&r"(n)
: [x] "a"(x),[da] "Q"(da)
: "cc", "r1", "v0", "v24", "v25", "v26", "v27", "v28", "v29", "v30",
"v31");
@@ -81,7 +81,7 @@ static void dscal_kernel_16_zero(BLASLONG n, FLOAT *x) {
"vst %%v0,112(%%r1,%[x])\n\t"
"agfi %%r1,128\n\t"
"brctg %[n],0b"
: "=m"(*(struct { FLOAT x[n]; } *) x),[n] "+&r"(n)
: "=m"(*(FLOAT (*)[n]) x),[n] "+&r"(n)
: [x] "a"(x)
: "cc", "r1", "v0");
}


+ 2
- 2
kernel/zarch/dsdot.c View File

@@ -112,8 +112,8 @@ static double dsdot_kernel_16(BLASLONG n, FLOAT *x, FLOAT *y) {
"adbr %%f0,%%f1\n\t"
"ldr %[dot],%%f0"
: [dot] "=f"(dot),[n] "+&r"(n)
: "m"(*(const struct { FLOAT x[n]; } *) x),[x] "a"(x),
"m"(*(const struct { FLOAT x[n]; } *) y),[y] "a"(y)
: "m"(*(const FLOAT (*)[n]) x),[x] "a"(x),
"m"(*(const FLOAT (*)[n]) y),[y] "a"(y)
: "cc", "r1", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16",
"v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26",
"v27", "v28", "v29", "v30", "v31");


+ 1
- 1
kernel/zarch/dsum.c View File

@@ -88,7 +88,7 @@ static FLOAT dsum_kernel_32(BLASLONG n, FLOAT *x) {
"vfadb %%v24,%%v24,%%v25\n\t"
"vsteg %%v24,%[sum],0"
: [sum] "=Q"(sum),[n] "+&r"(n)
: "m"(*(const struct { FLOAT x[n]; } *) x),[x] "a"(x)
: "m"(*(const FLOAT (*)[n]) x),[x] "a"(x)
: "cc", "r1", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23",
"v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31");



+ 1
- 1
kernel/zarch/dswap.c View File

@@ -99,7 +99,7 @@ static void dswap_kernel_32(BLASLONG n, FLOAT *x, FLOAT *y) {
"vst %%v31, 240(%%r1,%[y])\n\t"
"agfi %%r1,256\n\t"
"brctg %[n],0b"
: "+m"(*(struct { FLOAT x[n]; } *) x), "+m"(*(struct { FLOAT x[n]; } *) y),
: "+m"(*(FLOAT (*)[n]) x), "+m"(*(FLOAT (*)[n]) y),
[n] "+&r"(n)
: [x] "a"(x),[y] "a"(y)
: "cc", "r1", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16",


+ 12
- 12
kernel/zarch/gemm8x4V.S View File

@@ -147,7 +147,7 @@ brctg LOCAL_VAR1,.L8x4_4_BK

ALIGN_4
.L8x4_mod:
la LOCAL_VAR1,3(0,0)
lghi LOCAL_VAR1,3
NGR LOCAL_VAR1,BK /*refresh BK*/
jz .L8x4_BK_Store

@@ -183,7 +183,7 @@ brctg LOCAL_VAR1,.L4x4_4_BK

ALIGN_4
.L4x4_mod:
la LOCAL_VAR1,3(0,0)
lghi LOCAL_VAR1,3
NGR LOCAL_VAR1,BK /*refresh BK*/
jz .L4x4_BK_Store

@@ -217,7 +217,7 @@ brctg LOCAL_VAR1,.L2x4_4_BK

ALIGN_4
.L2x4_mod:
la LOCAL_VAR1,3(0,0)
lghi LOCAL_VAR1,3
NGR LOCAL_VAR1,BK /*refresh BK*/
jz .L2x4_BK_Store

@@ -252,7 +252,7 @@ brctg LOCAL_VAR1,.L1x4_4_BK

ALIGN_4
.L1x4_mod:
la LOCAL_VAR1,3(0,0)
lghi LOCAL_VAR1,3
NGR LOCAL_VAR1,BK /*refresh BK*/
jz .L1x4_BK_Store

@@ -309,7 +309,7 @@ brctg LOCAL_VAR1,.L8x2_4_BK

ALIGN_4
.L8x2_mod:
la LOCAL_VAR1,3(0,0)
lghi LOCAL_VAR1,3
NGR LOCAL_VAR1,BK /*refresh BK*/
jz .L8x2_BK_Store

@@ -346,7 +346,7 @@ brctg LOCAL_VAR1,.L4x2_4_BK

ALIGN_4
.L4x2_mod:
la LOCAL_VAR1,3(0,0)
lghi LOCAL_VAR1,3
NGR LOCAL_VAR1,BK /*refresh BK*/
jz .L4x2_BK_Store

@@ -380,7 +380,7 @@ brctg LOCAL_VAR1,.L2x2_4_BK

ALIGN_4
.L2x2_mod:
la LOCAL_VAR1,3(0,0)
lghi LOCAL_VAR1,3
NGR LOCAL_VAR1,BK /*refresh BK*/
jz .L2x2_BK_Store

@@ -415,7 +415,7 @@ brctg LOCAL_VAR1,.L1x2_4_BK

ALIGN_4
.L1x2_mod:
la LOCAL_VAR1,3(0,0)
lghi LOCAL_VAR1,3
NGR LOCAL_VAR1,BK /*refresh BK*/
jz .L1x2_BK_Store

@@ -471,7 +471,7 @@ brctg LOCAL_VAR1,.L8x1_4_BK

ALIGN_4
.L8x1_mod:
la LOCAL_VAR1,3(0,0)
lghi LOCAL_VAR1,3
NGR LOCAL_VAR1,BK /*refresh BK*/
jz .L8x1_BK_Store

@@ -508,7 +508,7 @@ brctg LOCAL_VAR1,.L4x1_4_BK

ALIGN_4
.L4x1_mod:
la LOCAL_VAR1,3(0,0)
lghi LOCAL_VAR1,3
NGR LOCAL_VAR1,BK /*refresh BK*/
jz .L4x1_BK_Store

@@ -542,7 +542,7 @@ brctg LOCAL_VAR1,.L2x1_4_BK

ALIGN_4
.L2x1_mod:
la LOCAL_VAR1,3(0,0)
lghi LOCAL_VAR1,3
NGR LOCAL_VAR1,BK /*refresh BK*/
jz .L2x1_BK_Store

@@ -577,7 +577,7 @@ brctg LOCAL_VAR1,.L1x1_4_BK

ALIGN_4
.L1x1_mod:
la LOCAL_VAR1,3(0,0)
lghi LOCAL_VAR1,3
NGR LOCAL_VAR1,BK /*refresh BK*/
jz .L1x1_BK_Store



+ 2
- 2
kernel/zarch/gemm_vec.c View File

@@ -172,7 +172,7 @@ static inline vector_float vec_load_hinted(FLOAT const *restrict a) {
vector_float const *restrict addr = (vector_float const *restrict)a;
vector_float y;

#if __GNUC__ < 9
#if __GNUC__ < 9 && !defined(__clang__)
// hex-encode vl %[out],%[addr],3
asm(".insn vrx,0xe70000003006,%[out],%[addr],3"
: [ out ] "=v"(y)
@@ -393,7 +393,7 @@ static inline void GEBP_block_16_4(
* Note that we need to massage this particular "barrier"
* depending on the gcc version.
*/
#if __GNUC__ > 7
#if __GNUC__ > 7 || defined(__clang__)
#define BARRIER_READ_BEFORE_COMPUTE(SUFFIX) \
do { \
asm("" \


+ 2
- 2
kernel/zarch/icamax.c View File

@@ -213,9 +213,9 @@ static BLASLONG icamax_kernel_32(BLASLONG n, FLOAT *x, FLOAT *amax) {
"ste %%f0,%[amax]\n\t"
"vlgvg %[iamax],%%v1,0\n\t"
"2:\n\t"
"nop"
"nop 0"
: [iamax] "=r"(iamax),[amax] "=Q"(*amax),[n] "+&r"(n)
: "m"(*(const struct { FLOAT x[n * 2]; } *) x),[x] "a"(x)
: "m"(*(const FLOAT (*)[n * 2]) x),[x] "a"(x)
: "cc", "r1", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8",
"v9", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24",
"v25", "v26", "v27", "v28", "v29", "v30", "v31");


+ 2
- 2
kernel/zarch/icamin.c View File

@@ -213,9 +213,9 @@ static BLASLONG icamin_kernel_32(BLASLONG n, FLOAT *x, FLOAT *amin) {
"ste %%f0,%[amin]\n\t"
"vlgvg %[iamin],%%v1,0\n\t"
"2:\n\t"
"nop"
"nop 0"
: [iamin] "=r"(iamin),[amin] "=Q"(*amin),[n] "+&r"(n)
: "m"(*(const struct { FLOAT x[n * 2]; } *) x),[x] "a"(x)
: "m"(*(const FLOAT (*)[n * 2]) x),[x] "a"(x)
: "cc", "r1", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8",
"v9", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24",
"v25", "v26", "v27", "v28", "v29", "v30", "v31");


+ 2
- 2
kernel/zarch/idamax.c View File

@@ -160,9 +160,9 @@ static BLASLONG idamax_kernel_32(BLASLONG n, FLOAT *x, FLOAT *amax) {
"std %%f0,%[amax]\n\t"
"vlgvg %[iamax],%%v1,0\n\t"
"2:\n\t"
"nop"
"nop 0"
: [iamax] "=r"(iamax),[amax] "=Q"(*amax),[n] "+&r"(n)
: "m"(*(const struct { FLOAT x[n]; } *) x),[x] "a"(x)
: "m"(*(const FLOAT (*)[n]) x),[x] "a"(x)
: "cc", "r1", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16",
"v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26",
"v27", "v28", "v29", "v30", "v31");


+ 2
- 2
kernel/zarch/idamin.c View File

@@ -160,9 +160,9 @@ static BLASLONG idamin_kernel_32(BLASLONG n, FLOAT *x, FLOAT *amin) {
"std %%f0,%[amin]\n\t"
"vlgvg %[iamin],%%v1,0\n\t"
"2:\n\t"
"nop"
"nop 0"
: [iamin] "=r"(iamin),[amin] "=Q"(*amin),[n] "+&r"(n)
: "m"(*(const struct { FLOAT x[n]; } *) x),[x] "a"(x)
: "m"(*(const FLOAT (*)[n]) x),[x] "a"(x)
: "cc", "r1", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16",
"v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26",
"v27", "v28", "v29", "v30", "v31");


+ 2
- 2
kernel/zarch/idmax.c View File

@@ -140,9 +140,9 @@ static BLASLONG idmax_kernel_32(BLASLONG n, FLOAT *x, FLOAT *max) {
"std %%f0,%[max]\n\t"
"vlgvg %[imax],%%v1,0\n\t"
"2:\n\t"
"nop"
"nop 0"
: [imax] "=r"(imax),[max] "=Q"(*max),[n] "+&r"(n)
: "m"(*(const struct { FLOAT x[n]; } *) x),[x] "a"(x)
: "m"(*(const FLOAT (*)[n]) x),[x] "a"(x)
: "cc", "r1", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16",
"v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26",
"v27", "v28", "v29", "v30", "v31");


+ 2
- 2
kernel/zarch/idmin.c View File

@@ -140,9 +140,9 @@ static BLASLONG idmin_kernel_32(BLASLONG n, FLOAT *x, FLOAT *min) {
"std %%f0,%[min]\n\t"
"vlgvg %[imin],%%v1,0\n\t"
"2:\n\t"
"nop"
"nop 0"
: [imin] "=r"(imin),[min] "=Q"(*min),[n] "+&r"(n)
: "m"(*(const struct { FLOAT x[n]; } *) x),[x] "a"(x)
: "m"(*(const FLOAT (*)[n]) x),[x] "a"(x)
: "cc", "r1", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16",
"v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26",
"v27", "v28", "v29", "v30", "v31");


+ 2
- 2
kernel/zarch/isamax.c View File

@@ -204,9 +204,9 @@ static BLASLONG isamax_kernel_64(BLASLONG n, FLOAT *x, FLOAT *amax) {
"ste %%f0,%[amax]\n\t"
"vlgvg %[iamax],%%v1,0\n\t"
"2:\n\t"
"nop"
"nop 0"
: [iamax] "=r"(iamax),[amax] "=Q"(*amax),[n] "+&r"(n)
: "m"(*(const struct { FLOAT x[n]; } *) x),[x] "a"(x)
: "m"(*(const FLOAT(*)[n]) x),[x] "a"(x)
: "cc", "r1", "v0", "v1", "v2", "v4", "v5", "v6", "v7", "v8", "v16",
"v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26",
"v27", "v28", "v29", "v30", "v31");


+ 2
- 2
kernel/zarch/isamin.c View File

@@ -204,9 +204,9 @@ static BLASLONG isamin_kernel_64(BLASLONG n, FLOAT *x, FLOAT *amin) {
"ste %%f0,%[amin]\n\t"
"vlgvg %[iamin],%%v1,0\n\t"
"2:\n\t"
"nop"
"nop 0"
: [iamin] "=r"(iamin),[amin] "=Q"(*amin),[n] "+&r"(n)
: "m"(*(const struct { FLOAT x[n]; } *) x),[x] "a"(x)
: "m"(*(const FLOAT (*)[n]) x),[x] "a"(x)
: "cc", "r1", "v0", "v1", "v2", "v4", "v5", "v6", "v7", "v8", "v16",
"v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26",
"v27", "v28", "v29", "v30", "v31");


+ 2
- 2
kernel/zarch/ismax.c View File

@@ -184,9 +184,9 @@ static BLASLONG ismax_kernel_64(BLASLONG n, FLOAT *x, FLOAT *max) {
"ste %%f0,%[max]\n\t"
"vlgvg %[imax],%%v1,0\n\t"
"2:\n\t"
"nop"
"nop 0"
: [imax] "=r"(imax),[max] "=Q"(*max),[n] "+&r"(n)
: "m"(*(const struct { FLOAT x[n]; } *) x),[x] "a"(x)
: "m"(*(const FLOAT (*)[n]) x),[x] "a"(x)
: "cc", "r1", "v0", "v1", "v2", "v4", "v5", "v6", "v7", "v8", "v16",
"v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26",
"v27", "v28", "v29", "v30", "v31");


+ 2
- 2
kernel/zarch/ismin.c View File

@@ -184,9 +184,9 @@ static BLASLONG ismin_kernel_64(BLASLONG n, FLOAT *x, FLOAT *min) {
"ste %%f0,%[min]\n\t"
"vlgvg %[imin],%%v1,0\n\t"
"2:\n\t"
"nop"
"nop 0"
: [imin] "=r"(imin),[min] "=Q"(*min),[n] "+&r"(n)
: "m"(*(const struct { FLOAT x[n]; } *) x),[x] "a"(x)
: "m"(*(const FLOAT (*)[n]) x),[x] "a"(x)
: "cc", "r1", "v0", "v1", "v2", "v4", "v5", "v6", "v7", "v8", "v16",
"v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26",
"v27", "v28", "v29", "v30", "v31");


+ 2
- 2
kernel/zarch/izamax.c View File

@@ -157,9 +157,9 @@ static BLASLONG izamax_kernel_16(BLASLONG n, FLOAT *x, FLOAT *amax) {
"std %%f0,%[amax]\n\t"
"vlgvg %[iamax],%%v1,0\n\t"
"2:\n\t"
"nop"
"nop 0"
: [iamax] "=r"(iamax),[amax] "=Q"(*amax),[n] "+&r"(n)
: "m"(*(const struct { FLOAT x[n * 2]; } *) x),[x] "a"(x)
: "m"(*(const FLOAT (*)[n * 2]) x),[x] "a"(x)
: "cc", "r1", "v0", "v1", "v2", "v3", "v4", "v5", "v16", "v17", "v18",
"v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27");



+ 2
- 2
kernel/zarch/izamin.c View File

@@ -157,9 +157,9 @@ static BLASLONG izamin_kernel_16(BLASLONG n, FLOAT *x, FLOAT *amin) {
"std %%f0,%[amin]\n\t"
"vlgvg %[iamin],%%v1,0\n\t"
"2:\n\t"
"nop"
"nop 0"
: [iamin] "=r"(iamin),[amin] "=Q"(*amin),[n] "+&r"(n)
: "m"(*(const struct { FLOAT x[n * 2]; } *) x),[x] "a"(x)
: "m"(*(const FLOAT (*)[n * 2]) x),[x] "a"(x)
: "cc", "r1", "v0", "v1", "v2", "v3", "v4", "v5", "v16", "v17", "v18",
"v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27");



+ 1
- 1
kernel/zarch/samax.c View File

@@ -78,7 +78,7 @@ static FLOAT samax_kernel_64(BLASLONG n, FLOAT *x) {
"wfmaxsb %%v0,%%v0,%%v16,8\n\t"
"lper %[amax],%%f0"
: [amax] "=f"(amax),[n] "+&r"(n)
: "m"(*(const struct { FLOAT x[n]; } *) x),[x] "a"(x)
: "m"(*(const FLOAT (*)[n]) x),[x] "a"(x)
: "cc", "r1", "v0", "v16", "v17", "v18", "v19", "v20", "v21", "v22",
"v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31");



+ 1
- 1
kernel/zarch/samin.c View File

@@ -78,7 +78,7 @@ static FLOAT samin_kernel_64(BLASLONG n, FLOAT *x) {
"wfminsb %%v0,%%v0,%%v16,8\n\t"
"lper %[amin],%%f0"
: [amin] "=f"(amin),[n] "+&r"(n)
: "m"(*(const struct { FLOAT x[n]; } *) x),[x] "a"(x)
: "m"(*(const FLOAT (*)[n]) x),[x] "a"(x)
: "cc", "r1", "v0", "v16", "v17", "v18", "v19", "v20", "v21", "v22",
"v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31");



+ 1
- 1
kernel/zarch/sasum.c View File

@@ -108,7 +108,7 @@ static FLOAT sasum_kernel_64(BLASLONG n, FLOAT *x) {
"vfasb %%v24,%%v24,%%v25\n\t"
"vstef %%v24,%[asum],0"
: [asum] "=Q"(asum),[n] "+&r"(n)
: "m"(*(const struct { FLOAT x[n]; } *) x),[x] "a"(x)
: "m"(*(const FLOAT (*)[n]) x),[x] "a"(x)
: "cc", "r1", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23",
"v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31");



+ 2
- 2
kernel/zarch/saxpy.c View File

@@ -100,8 +100,8 @@ static void saxpy_kernel_64(BLASLONG n, FLOAT *x, FLOAT *y, FLOAT *alpha) {
"vst %%v27,240(%%r1,%[y])\n\t"
"agfi %%r1,256\n\t"
"brctg %[n],0b"
: "+m"(*(struct { FLOAT x[n]; } *) y),[n] "+&r"(n)
: [y] "a"(y), "m"(*(const struct { FLOAT x[n]; } *) x),[x] "a"(x),
: "+m"(*(FLOAT (*)[n]) y),[n] "+&r"(n)
: [y] "a"(y), "m"(*(const FLOAT (*)[n]) x),[x] "a"(x),
[alpha] "Q"(*alpha)
: "cc", "r1", "v0", "v16", "v17", "v18", "v19", "v20", "v21", "v22",
"v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31");


+ 2
- 2
kernel/zarch/scopy.c View File

@@ -36,8 +36,8 @@ static void scopy_kernel_64(BLASLONG n, FLOAT *x, FLOAT *y) {
"la %[x],256(%[x])\n\t"
"la %[y],256(%[y])\n\t"
"brctg %[n],0b"
: "=m"(*(struct { FLOAT x[n]; } *) y),[x] "+&a"(x),[y] "+&a"(y),[n] "+&r"(n)
: "m"(*(const struct { FLOAT x[n]; } *) x)
: "=m"(*(FLOAT (*)[n]) y),[x] "+&a"(x),[y] "+&a"(y),[n] "+&r"(n)
: "m"(*(const FLOAT (*)[n]) x)
: "cc");
}



+ 2
- 2
kernel/zarch/sdot.c View File

@@ -84,8 +84,8 @@ static FLOAT sdot_kernel_32(BLASLONG n, FLOAT *x, FLOAT *y) {
"aebr %%f0,%%f3\n\t"
"ler %[dot],%%f0"
: [dot] "=f"(dot),[n] "+&r"(n)
: "m"(*(const struct { FLOAT x[n]; } *) x),[x] "a"(x),
"m"(*(const struct { FLOAT x[n]; } *) y),[y] "a"(y)
: "m"(*(const FLOAT (*)[n]) x),[x] "a"(x),
"m"(*(const FLOAT (*)[n]) y),[y] "a"(y)
: "cc", "r1", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16",
"v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26",
"v27", "v28", "v29", "v30", "v31");


+ 15
- 15
kernel/zarch/sgemv_n_4.c View File

@@ -159,13 +159,13 @@ static void sgemv_kernel_4x4(BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y,
"agfi %%r1,16\n\t"
"brctg %%r0,2b\n\t"
"3:\n\t"
"nop"
: "+m"(*(struct { FLOAT x[n]; } *) y)
: [y] "a"(y), "m"(*(const struct { FLOAT x[n]; } *) ap0),[ap0] "a"(ap0),
"m"(*(const struct { FLOAT x[n]; } *) ap1),[ap1] "a"(ap1),
"m"(*(const struct { FLOAT x[n]; } *) ap2),[ap2] "a"(ap2),
"m"(*(const struct { FLOAT x[n]; } *) ap3),[ap3] "a"(ap3),
"m"(*(const struct { FLOAT x[4]; } *) x),[x] "a"(x),[alpha] "Q"(*alpha),
"nop 0"
: "+m"(*(FLOAT (*)[n]) y)
: [y] "a"(y), "m"(*(const FLOAT (*)[n]) ap0),[ap0] "a"(ap0),
"m"(*(const FLOAT (*)[n]) ap1),[ap1] "a"(ap1),
"m"(*(const FLOAT (*)[n]) ap2),[ap2] "a"(ap2),
"m"(*(const FLOAT (*)[n]) ap3),[ap3] "a"(ap3),
"m"(*(const FLOAT (*)[4]) x),[x] "a"(x),[alpha] "Q"(*alpha),
[n] "r"(n)
: "cc", "r0", "r1", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
"v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25",
@@ -258,11 +258,11 @@ static void sgemv_kernel_4x2(BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y,
"agfi %%r1,16\n\t"
"brctg %%r0,2b\n\t"
"3:\n\t"
"nop"
: "+m"(*(struct { FLOAT x[n]; } *) y)
: [y] "a"(y), "m"(*(const struct { FLOAT x[n]; } *) ap0),[ap0] "a"(ap0),
"m"(*(const struct { FLOAT x[n]; } *) ap1),[ap1] "a"(ap1),
"m"(*(const struct { FLOAT x[2]; } *) x),[x] "a"(x),[alpha] "Q"(*alpha),
"nop 0"
: "+m"(*(FLOAT (*)[n]) y)
: [y] "a"(y), "m"(*(const FLOAT (*)[n]) ap0),[ap0] "a"(ap0),
"m"(*(const FLOAT (*)[n]) ap1),[ap1] "a"(ap1),
"m"(*(const FLOAT (*)[2]) x),[x] "a"(x),[alpha] "Q"(*alpha),
[n] "r"(n)
: "cc", "r0", "r1", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
"v8", "v9", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23",
@@ -331,9 +331,9 @@ static void sgemv_kernel_4x1(BLASLONG n, FLOAT *a0, FLOAT *x, FLOAT *y,
"agfi %%r1,16\n\t"
"brctg %%r0,2b\n\t"
"3:\n\t"
"nop"
: "+m"(*(struct { FLOAT x[n]; } *) y)
: [y] "a"(y), "m"(*(const struct { FLOAT x[n]; } *) a0),[a0] "a"(a0),
"nop 0"
: "+m"(*(FLOAT (*)[n]) y)
: [y] "a"(y), "m"(*(const FLOAT (*)[n]) a0),[a0] "a"(a0),
"m"(*(const FLOAT (*)[1]) x),[x] "a"(x),[alpha] "Q"(*alpha),
[n] "r"(n)
: "cc", "r0", "r1", "v0", "v16", "v17", "v18", "v19", "v20", "v21",


+ 15
- 15
kernel/zarch/sgemv_t_4.c View File

@@ -172,12 +172,12 @@ static void sgemv_kernel_4x4(BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y) {
"vrepg %%v4,%%v3,1\n\t"
"aebr %%f3,%%f4\n\t"
"ste %%f3,12(%[y])"
: "=m"(*(struct { FLOAT x[4]; } *) y)
: [y] "a"(y), "m"(*(const struct { FLOAT x[n]; } *) ap0),[ap0] "a"(ap0),
"m"(*(const struct { FLOAT x[n]; } *) ap1),[ap1] "a"(ap1),
"m"(*(const struct { FLOAT x[n]; } *) ap2),[ap2] "a"(ap2),
"m"(*(const struct { FLOAT x[n]; } *) ap3),[ap3] "a"(ap3),
"m"(*(const struct { FLOAT x[n]; } *) x),[x] "a"(x),[n] "r"(n)
: "=m"(*(FLOAT (*)[4]) y)
: [y] "a"(y), "m"(*(const FLOAT (*)[n]) ap0),[ap0] "a"(ap0),
"m"(*(const FLOAT (*)[n]) ap1),[ap1] "a"(ap1),
"m"(*(const FLOAT (*)[n]) ap2),[ap2] "a"(ap2),
"m"(*(const FLOAT (*)[n]) ap3),[ap3] "a"(ap3),
"m"(*(const FLOAT (*)[n]) x),[x] "a"(x),[n] "r"(n)
: "cc", "r0", "r1", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
"v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25",
"v26", "v27", "v28", "v29", "v30", "v31");
@@ -278,10 +278,10 @@ static void sgemv_kernel_4x2(BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y) {
"vrepg %%v2,%%v1,1\n\t"
"aebr %%f1,%%f2\n\t"
"ste %%f1,4(%[y])"
: "=m"(*(struct { FLOAT x[2]; } *) y)
: [y] "a"(y), "m"(*(const struct { FLOAT x[n]; } *) ap0),[ap0] "a"(ap0),
"m"(*(const struct { FLOAT x[n]; } *) ap1),[ap1] "a"(ap1),
"m"(*(const struct { FLOAT x[n]; } *) x),[x] "a"(x),[n] "r"(n)
: "=m"(*(FLOAT (*)[2]) y)
: [y] "a"(y), "m"(*(const FLOAT (*)[n]) ap0),[ap0] "a"(ap0),
"m"(*(const FLOAT (*)[n]) ap1),[ap1] "a"(ap1),
"m"(*(const FLOAT (*)[n]) x),[x] "a"(x),[n] "r"(n)
: "cc", "r0", "r1", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
"v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25",
"v26", "v27", "v28", "v29", "v30", "v31");
@@ -357,8 +357,8 @@ static void sgemv_kernel_4x1(BLASLONG n, FLOAT *a0, FLOAT *x, FLOAT *y) {
"aebr %%f0,%%f1\n\t"
"ste %%f0,0(%[y])"
: "=m"(*(FLOAT (*)[1]) y)
: [y] "a"(y), "m"(*(const struct { FLOAT x[n]; } *) a0),[a0] "a"(a0),
"m"(*(const struct { FLOAT x[n]; } *) x),[x] "a"(x),[n] "r"(n)
: [y] "a"(y), "m"(*(const FLOAT (*)[n]) a0),[a0] "a"(a0),
"m"(*(const FLOAT (*)[n]) x),[x] "a"(x),[n] "r"(n)
: "cc", "r0", "r1", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
"v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25",
"v26", "v27", "v28", "v29", "v30", "v31");
@@ -431,9 +431,9 @@ static void add_y_kernel_4(BLASLONG n, FLOAT da, FLOAT *src, FLOAT *dest) {
"agfi %%r1,16\n\t"
"brctg %%r0,2b\n\t"
"3:\n\t"
"nop"
: "+m"(*(struct { FLOAT x[n]; } *) dest)
: [dest] "a"(dest),[da] "Q"(da), "m"(*(const struct { FLOAT x[n]; } *) src),
"nop 0"
: "+m"(*(FLOAT (*)[n]) dest)
: [dest] "a"(dest),[da] "Q"(da), "m"(*(const FLOAT (*)[n]) src),
[src] "a"(src),[n] "r"(n)
: "cc", "r0", "r1", "v0", "v16", "v17", "v18", "v19", "v20", "v21",
"v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30",


+ 1
- 1
kernel/zarch/smax.c View File

@@ -75,7 +75,7 @@ static FLOAT smax_kernel_64(BLASLONG n, FLOAT *x) {
"wfmaxsb %%v0,%%v0,%%v16,0\n\t"
"ler %[max],%%f0"
: [max] "=f"(max),[n] "+&r"(n)
: "m"(*(const struct { FLOAT x[n]; } *) x),[x] "a"(x)
: "m"(*(const FLOAT(*)[n]) x),[x] "a"(x)
: "cc", "r1", "v0", "v16", "v17", "v18", "v19", "v20", "v21", "v22",
"v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31");



+ 1
- 1
kernel/zarch/smin.c View File

@@ -75,7 +75,7 @@ static FLOAT smin_kernel_64(BLASLONG n, FLOAT *x) {
"wfminsb %%v0,%%v0,%%v16,0\n\t"
"ler %[min],%%f0"
: [min] "=f"(min),[n] "+&r"(n)
: "m"(*(const struct { FLOAT x[n]; } *) x),[x] "a"(x)
: "m"(*(const FLOAT (*)[n]) x),[x] "a"(x)
: "cc", "r1", "v0", "v16", "v17", "v18", "v19", "v20", "v21", "v22",
"v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31");



+ 1
- 1
kernel/zarch/srot.c View File

@@ -169,7 +169,7 @@ static void srot_kernel_64(BLASLONG n, FLOAT *x, FLOAT *y, FLOAT *c, FLOAT *s) {
"vst %%v23, 240(%%r1,%[y])\n\t"
"agfi %%r1,256\n\t"
"brctg %[n],0b"
: "+m"(*(struct { FLOAT x[n]; } *) x), "+m"(*(struct { FLOAT x[n]; } *) y),
: "+m"(*(FLOAT (*)[n]) x), "+m"(*(FLOAT (*)[n]) y),
[n] "+&r"(n)
: [x] "a"(x),[y] "a"(y),[c] "Q"(*c),[s] "Q"(*s)
: "cc", "r1", "v0", "v1", "v16", "v17", "v18", "v19", "v20", "v21",


+ 2
- 2
kernel/zarch/sscal.c View File

@@ -59,7 +59,7 @@ static void sscal_kernel_32(BLASLONG n, FLOAT da, FLOAT *x) {
"vst %%v31,112(%%r1,%[x])\n\t"
"agfi %%r1,128\n\t"
"brctg %[n],0b"
: "+m"(*(struct { FLOAT x[n]; } *) x),[n] "+&r"(n)
: "+m"(*(FLOAT (*)[n]) x),[n] "+&r"(n)
: [x] "a"(x),[da] "Q"(da)
: "cc", "r1", "v0", "v24", "v25", "v26", "v27", "v28", "v29", "v30",
"v31");
@@ -81,7 +81,7 @@ static void sscal_kernel_32_zero(BLASLONG n, FLOAT *x) {
"vst %%v0,112(%%r1,%[x])\n\t"
"agfi %%r1,128\n\t"
"brctg %[n],0b"
: "=m"(*(struct { FLOAT x[n]; } *) x),[n] "+&r"(n)
: "=m"(*(FLOAT (*)[n]) x),[n] "+&r"(n)
: [x] "a"(x)
: "cc", "r1", "v0");
}


+ 1
- 1
kernel/zarch/ssum.c View File

@@ -91,7 +91,7 @@ static FLOAT ssum_kernel_64(BLASLONG n, FLOAT *x) {
"vfasb %%v24,%%v24,%%v25\n\t"
"vstef %%v24,%[sum],0"
: [sum] "=Q"(sum),[n] "+&r"(n)
: "m"(*(const struct { FLOAT x[n]; } *) x),[x] "a"(x)
: "m"(*(const FLOAT (*)[n]) x),[x] "a"(x)
: "cc", "r1", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23",
"v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31");



+ 1
- 1
kernel/zarch/sswap.c View File

@@ -99,7 +99,7 @@ static void sswap_kernel_64(BLASLONG n, FLOAT *x, FLOAT *y) {
"vst %%v31, 240(%%r1,%[y])\n\t"
"agfi %%r1,256\n\t"
"brctg %[n],0b"
: "+m"(*(struct { FLOAT x[n]; } *) x), "+m"(*(struct { FLOAT x[n]; } *) y),
: "+m"(*(FLOAT (*)[n]) x), "+m"(*(FLOAT (*)[n]) y),
[n] "+&r"(n)
: [x] "a"(x),[y] "a"(y)
: "cc", "r1", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16",


+ 12
- 12
kernel/zarch/strmm8x4V.S View File

@@ -186,7 +186,7 @@ ALIGN_4
RefreshTempBk LOCAL_VAR1,BK,OFF,8,4
nill LOCAL_VAR1,3
#else
la LOCAL_VAR1,3(0,0)
lghi LOCAL_VAR1,3
NGR LOCAL_VAR1,BK /*refresh BK*/
#endif
jz .L8x4_BK_Store
@@ -239,7 +239,7 @@ ALIGN_4
RefreshTempBk LOCAL_VAR1,BK,OFF,4,4
nill LOCAL_VAR1,3
#else
la LOCAL_VAR1,3(0,0)
lghi LOCAL_VAR1,3
NGR LOCAL_VAR1,BK /*refresh BK*/
#endif
jz .L4x4_BK_Store
@@ -290,7 +290,7 @@ ALIGN_4
RefreshTempBk LOCAL_VAR1,BK,OFF,2,4
nill LOCAL_VAR1,3
#else
la LOCAL_VAR1,3(0,0)
lghi LOCAL_VAR1,3
NGR LOCAL_VAR1,BK /*refresh BK*/
#endif
jz .L2x4_BK_Store
@@ -341,7 +341,7 @@ ALIGN_4
RefreshTempBk LOCAL_VAR1,BK,OFF,1,4
nill LOCAL_VAR1,3
#else
la LOCAL_VAR1,3(0,0)
lghi LOCAL_VAR1,3
NGR LOCAL_VAR1,BK /*refresh BK*/
#endif
jz .L1x4_BK_Store
@@ -423,7 +423,7 @@ ALIGN_4
RefreshTempBk LOCAL_VAR1,BK,OFF,8,2
nill LOCAL_VAR1,3
#else
la LOCAL_VAR1,3(0,0)
lghi LOCAL_VAR1,3
NGR LOCAL_VAR1,BK /*refresh BK*/
#endif
jz .L8x2_BK_Store
@@ -475,7 +475,7 @@ ALIGN_4
RefreshTempBk LOCAL_VAR1,BK,OFF,4,2
nill LOCAL_VAR1,3
#else
la LOCAL_VAR1,3(0,0)
lghi LOCAL_VAR1,3
NGR LOCAL_VAR1,BK /*refresh BK*/
#endif
jz .L4x2_BK_Store
@@ -525,7 +525,7 @@ ALIGN_4
RefreshTempBk LOCAL_VAR1,BK,OFF,2,2
nill LOCAL_VAR1,3
#else
la LOCAL_VAR1,3(0,0)
lghi LOCAL_VAR1,3
NGR LOCAL_VAR1,BK /*refresh BK*/
#endif
jz .L2x2_BK_Store
@@ -575,7 +575,7 @@ ALIGN_4
RefreshTempBk LOCAL_VAR1,BK,OFF,1,2
nill LOCAL_VAR1,3
#else
la LOCAL_VAR1,3(0,0)
lghi LOCAL_VAR1,3
NGR LOCAL_VAR1,BK /*refresh BK*/
#endif
jz .L1x2_BK_Store
@@ -655,7 +655,7 @@ ALIGN_4
RefreshTempBk LOCAL_VAR1,BK,OFF,8,1
nill LOCAL_VAR1,3
#else
la LOCAL_VAR1,3(0,0)
lghi LOCAL_VAR1,3
NGR LOCAL_VAR1,BK /*refresh BK*/
#endif
jz .L8x1_BK_Store
@@ -708,7 +708,7 @@ ALIGN_4
RefreshTempBk LOCAL_VAR1,BK,OFF,4,1
nill LOCAL_VAR1,3
#else
la LOCAL_VAR1,3(0,0)
lghi LOCAL_VAR1,3
NGR LOCAL_VAR1,BK /*refresh BK*/
#endif
jz .L4x1_BK_Store
@@ -757,7 +757,7 @@ ALIGN_4
RefreshTempBk LOCAL_VAR1,BK,OFF,2,1
nill LOCAL_VAR1,3
#else
la LOCAL_VAR1,3(0,0)
lghi LOCAL_VAR1,3
NGR LOCAL_VAR1,BK /*refresh BK*/
#endif
jz .L2x1_BK_Store
@@ -807,7 +807,7 @@ ALIGN_4
RefreshTempBk LOCAL_VAR1,BK,OFF,1,1
nill LOCAL_VAR1,3
#else
la LOCAL_VAR1,3(0,0)
lghi LOCAL_VAR1,3
NGR LOCAL_VAR1,BK /*refresh BK*/
#endif
jz .L1x1_BK_Store


+ 1
- 1
kernel/zarch/zamax.c View File

@@ -114,7 +114,7 @@ static FLOAT zamax_kernel_16(BLASLONG n, FLOAT *x) {
"wfmaxdb %%v0,%%v0,%%v16,0\n\t"
"ldr %[amax],%%f0"
: [amax] "=f"(amax),[n] "+&r"(n)
: "m"(*(const struct { FLOAT x[n * 2]; } *) x),[x] "a"(x)
: "m"(*(const FLOAT (*)[n * 2]) x),[x] "a"(x)
: "cc", "r1", "v0", "v16", "v17", "v18", "v19", "v20", "v21", "v22",
"v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31");



+ 1
- 1
kernel/zarch/zamax_z13.c View File

@@ -123,7 +123,7 @@ static FLOAT zamax_kernel_16(BLASLONG n, FLOAT *x) {
"vsel %%v0,%%v0,%%v16,%%v17\n\t"
"ldr %[amax],%%f0"
: [amax] "=f"(amax),[n] "+&r"(n)
: "m"(*(const struct { FLOAT x[n * 2]; } *) x),[x] "a"(x)
: "m"(*(const FLOAT (*)[n * 2]) x),[x] "a"(x)
: "cc", "r1", "v0", "v16", "v17", "v18", "v19", "v20", "v21", "v22",
"v23", "v24", "v25", "v26", "v27");



+ 1
- 1
kernel/zarch/zamin.c View File

@@ -114,7 +114,7 @@ static FLOAT zamin_kernel_16(BLASLONG n, FLOAT *x) {
"wfmindb %%v0,%%v0,%%v16,0\n\t"
"ldr %[amin],%%f0"
: [amin] "=f"(amin),[n] "+&r"(n)
: "m"(*(const struct { FLOAT x[n * 2]; } *) x),[x] "a"(x)
: "m"(*(const FLOAT (*)[n * 2]) x),[x] "a"(x)
: "cc", "r1", "v0", "v16", "v17", "v18", "v19", "v20", "v21", "v22",
"v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31");



+ 1
- 1
kernel/zarch/zamin_z13.c View File

@@ -123,7 +123,7 @@ static FLOAT zamin_kernel_16(BLASLONG n, FLOAT *x) {
"vsel %%v0,%%v0,%%v16,%%v17\n\t"
"ldr %[amin],%%f0"
: [amin] "=f"(amin),[n] "+&r"(n)
: "m"(*(const struct { FLOAT x[n * 2]; } *) x),[x] "a"(x)
: "m"(*(const FLOAT (*)[n * 2]) x),[x] "a"(x)
: "cc", "r1", "v0", "v16", "v17", "v18", "v19", "v20", "v21", "v22",
"v23", "v24", "v25", "v26", "v27");



+ 1
- 1
kernel/zarch/zasum.c View File

@@ -106,7 +106,7 @@ static FLOAT zasum_kernel_16(BLASLONG n, FLOAT *x) {
"vfadb %%v24,%%v24,%%v25\n\t"
"vsteg %%v24,%[asum],0"
: [asum] "=Q"(asum),[n] "+&r"(n)
: "m"(*(const struct { FLOAT x[n * 2]; } *) x),[x] "a"(x)
: "m"(*(const FLOAT (*)[n * 2]) x),[x] "a"(x)
: "cc", "r1", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23",
"v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31");



+ 3
- 3
kernel/zarch/zaxpy.c View File

@@ -95,9 +95,9 @@ static void zaxpy_kernel_8(BLASLONG n, FLOAT *x, FLOAT *y, FLOAT *alpha) {
"vst %%v19,112(%%r1,%[y])\n\t"
"agfi %%r1,128\n\t"
"brctg %[n],0b"
: "+m"(*(struct { FLOAT x[n * 2]; } *) y),[n] "+&r"(n)
: [y] "a"(y), "m"(*(const struct { FLOAT x[n * 2]; } *) x),[x] "a"(x),
"m"(*(const struct { FLOAT x[2]; } *) alpha),[alpha] "a"(alpha)
: "+m"(*(FLOAT (*)[n * 2]) y),[n] "+&r"(n)
: [y] "a"(y), "m"(*(const FLOAT (*)[n * 2]) x),[x] "a"(x),
"m"(*(const FLOAT (*)[2]) alpha),[alpha] "a"(alpha)
: "cc", "r1", "v0", "v1", "v8", "v9", "v10", "v11", "v12", "v13",
"v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23",
"v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31");


+ 2
- 2
kernel/zarch/zcopy.c View File

@@ -36,9 +36,9 @@ static void zcopy_kernel_16(BLASLONG n, FLOAT *x, FLOAT *y) {
"la %[x],256(%[x])\n\t"
"la %[y],256(%[y])\n\t"
"brctg %[n],0b"
: "=m"(*(struct { FLOAT x[n * 2]; } *) y),[x] "+&a"(x),[y] "+&a"(y),
: "=m"(*(FLOAT (*)[n * 2]) y),[x] "+&a"(x),[y] "+&a"(y),
[n] "+&r"(n)
: "m"(*(const struct { FLOAT x[n * 2]; } *) x)
: "m"(*(const FLOAT (*)[n * 2]) x)
: "cc");
}



+ 3
- 3
kernel/zarch/zdot.c View File

@@ -93,9 +93,9 @@ static void zdot_kernel_8(BLASLONG n, FLOAT *x, FLOAT *y, FLOAT *d) {
"vsteg %%v24,8(%[d]),1\n\t"
"vsteg %%v25,16(%[d]),1\n\t"
"vsteg %%v25,24(%[d]),0"
: "=m"(*(struct { FLOAT x[4]; } *) d),[n] "+&r"(n)
: [d] "a"(d), "m"(*(const struct { FLOAT x[n * 2]; } *) x),[x] "a"(x),
"m"(*(const struct { FLOAT x[n * 2]; } *) y),[y] "a"(y)
: "=m"(*(FLOAT (*)[4]) d),[n] "+&r"(n)
: [d] "a"(d), "m"(*(const FLOAT (*)[n * 2]) x),[x] "a"(x),
"m"(*(const FLOAT (*)[n * 2]) y),[y] "a"(y)
: "cc", "r1", "v0", "v1", "v2", "v3", "v16", "v17", "v18", "v19", "v20",
"v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30",
"v31");


+ 15
- 15
kernel/zarch/zgemv_n_4.c View File

@@ -112,12 +112,12 @@ static void zgemv_kernel_4x4(BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y) {
"vst %%v1,16(%%r1,%[y])\n\t"
"agfi %%r1,32\n\t"
"brctg %[n],0b"
: "+m"(*(struct { FLOAT x[n * 2]; } *) y),[n] "+&r"(n)
: [y] "a"(y), "m"(*(const struct { FLOAT x[n * 2]; } *) ap0),[ap0] "a"(ap0),
"m"(*(const struct { FLOAT x[n * 2]; } *) ap1),[ap1] "a"(ap1),
"m"(*(const struct { FLOAT x[n * 2]; } *) ap2),[ap2] "a"(ap2),
"m"(*(const struct { FLOAT x[n * 2]; } *) ap3),[ap3] "a"(ap3),
"m"(*(const struct { FLOAT x[8]; } *) x),[x] "a"(x)
: "+m"(*(FLOAT (*)[n * 2]) y),[n] "+&r"(n)
: [y] "a"(y), "m"(*(const FLOAT (*)[n * 2]) ap0),[ap0] "a"(ap0),
"m"(*(const FLOAT (*)[n * 2]) ap1),[ap1] "a"(ap1),
"m"(*(const FLOAT (*)[n * 2]) ap2),[ap2] "a"(ap2),
"m"(*(const FLOAT (*)[n * 2]) ap3),[ap3] "a"(ap3),
"m"(*(const FLOAT (*)[8]) x),[x] "a"(x)
: "cc", "r1", "v0", "v1", "v16", "v17", "v18", "v19", "v20", "v21",
"v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30",
"v31");
@@ -172,10 +172,10 @@ static void zgemv_kernel_4x2(BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y) {
"vst %%v1,16(%%r1,%[y])\n\t"
"agfi %%r1,32\n\t"
"brctg %[n],0b"
: "+m"(*(struct { FLOAT x[n * 2]; } *) y),[n] "+&r"(n)
: [y] "a"(y), "m"(*(const struct { FLOAT x[n * 2]; } *) ap0),[ap0] "a"(ap0),
"m"(*(const struct { FLOAT x[n * 2]; } *) ap1),[ap1] "a"(ap1),
"m"(*(const struct { FLOAT x[4]; } *) x),[x] "a"(x)
: "+m"(*(FLOAT (*)[n * 2]) y),[n] "+&r"(n)
: [y] "a"(y), "m"(*(const FLOAT (*)[n * 2]) ap0),[ap0] "a"(ap0),
"m"(*(const FLOAT (*)[n * 2]) ap1),[ap1] "a"(ap1),
"m"(*(const FLOAT (*)[4]) x),[x] "a"(x)
: "cc", "r1", "v0", "v1", "v16", "v17", "v18", "v19", "v20", "v21",
"v22", "v23", "v24", "v25", "v26", "v27");
}
@@ -210,9 +210,9 @@ static void zgemv_kernel_4x1(BLASLONG n, FLOAT *ap, FLOAT *x, FLOAT *y) {
"vst %%v1,16(%%r1,%[y])\n\t"
"agfi %%r1,32\n\t"
"brctg %[n],0b"
: "+m"(*(struct { FLOAT x[n * 2]; } *) y),[n] "+&r"(n)
: [y] "a"(y), "m"(*(const struct { FLOAT x[n * 2]; } *) ap),[ap] "a"(ap),
"m"(*(const struct { FLOAT x[2]; } *) x),[x] "a"(x)
: "+m"(*(FLOAT (*)[n * 2]) y),[n] "+&r"(n)
: [y] "a"(y), "m"(*(const FLOAT (*)[n * 2]) ap),[ap] "a"(ap),
"m"(*(const FLOAT (*)[2]) x),[x] "a"(x)
: "cc", "r1", "v0", "v1", "v16", "v17", "v18", "v19", "v20", "v21");
}

@@ -261,8 +261,8 @@ static void add_y_4(BLASLONG n, FLOAT *src, FLOAT *dest, FLOAT alpha_r,
"vst %%v31,48(%%r1,%[dest])\n\t"
"agfi %%r1,64\n\t"
"brctg %[n],0b"
: "+m"(*(struct { FLOAT x[n * 2]; } *) dest),[n] "+&r"(n)
: [dest] "a"(dest), "m"(*(const struct { FLOAT x[n * 2]; } *) src),
: "+m"(*(FLOAT (*)[n * 2]) dest),[n] "+&r"(n)
: [dest] "a"(dest), "m"(*(const FLOAT (*)[n * 2]) src),
[src] "a"(src),[alpha_r] "Q"(alpha_r),[alpha_i] "Q"(alpha_i)
: "cc", "r1", "v0", "v1", "v16", "v17", "v18", "v19", "v20", "v21",
"v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30",


+ 16
- 16
kernel/zarch/zgemv_t_4.c View File

@@ -141,13 +141,13 @@ static void zgemv_kernel_4x4(BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y,
"vst %%v27,16(%[y])\n\t"
"vst %%v28,32(%[y])\n\t"
"vst %%v29,48(%[y])"
: "+m"(*(struct { FLOAT x[8]; } *) y),[n] "+&r"(n)
: [y] "a"(y), "m"(*(const struct { FLOAT x[n * 2]; } *) ap0),[ap0] "a"(ap0),
"m"(*(const struct { FLOAT x[n * 2]; } *) ap1),[ap1] "a"(ap1),
"m"(*(const struct { FLOAT x[n * 2]; } *) ap2),[ap2] "a"(ap2),
"m"(*(const struct { FLOAT x[n * 2]; } *) ap3),[ap3] "a"(ap3),
"m"(*(const struct { FLOAT x[n * 2]; } *) x),[x] "a"(x),
"m"(*(const struct { FLOAT x[2]; } *) alpha),[alpha] "a"(alpha)
: "+m"(*(FLOAT (*)[8]) y),[n] "+&r"(n)
: [y] "a"(y), "m"(*(const FLOAT (*)[n * 2]) ap0),[ap0] "a"(ap0),
"m"(*(const FLOAT (*)[n * 2]) ap1),[ap1] "a"(ap1),
"m"(*(const FLOAT (*)[n * 2]) ap2),[ap2] "a"(ap2),
"m"(*(const FLOAT (*)[n * 2]) ap3),[ap3] "a"(ap3),
"m"(*(const FLOAT (*)[n * 2]) x),[x] "a"(x),
"m"(*(const FLOAT (*)[2]) alpha),[alpha] "a"(alpha)
: "cc", "r1", "v0", "v1", "v16", "v17", "v18", "v19", "v20", "v21",
"v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30",
"v31");
@@ -229,11 +229,11 @@ static void zgemv_kernel_4x2(BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y,
"vfmadb %%v23,%%v19,%%v21,%%v23\n\t"
"vst %%v22,0(%[y])\n\t"
"vst %%v23,16(%[y])\n\t"
: "+m"(*(struct { FLOAT x[4]; } *) y),[n] "+&r"(n)
: [y] "a"(y), "m"(*(const struct { FLOAT x[n * 2]; } *) ap0),[ap0] "a"(ap0),
"m"(*(const struct { FLOAT x[n * 2]; } *) ap1),[ap1] "a"(ap1),
"m"(*(const struct { FLOAT x[n * 2]; } *) x),[x] "a"(x),
"m"(*(const struct { FLOAT x[2]; } *) alpha),[alpha] "a"(alpha)
: "+m"(*(FLOAT (*)[4]) y),[n] "+&r"(n)
: [y] "a"(y), "m"(*(const FLOAT (*)[n * 2]) ap0),[ap0] "a"(ap0),
"m"(*(const FLOAT (*)[n * 2]) ap1),[ap1] "a"(ap1),
"m"(*(const FLOAT (*)[n * 2]) x),[x] "a"(x),
"m"(*(const FLOAT (*)[2]) alpha),[alpha] "a"(alpha)
: "cc", "r1", "v0", "v1", "v16", "v17", "v18", "v19", "v20", "v21",
"v22", "v23");
}
@@ -294,10 +294,10 @@ static void zgemv_kernel_4x1(BLASLONG n, FLOAT *ap, FLOAT *x, FLOAT *y,
"vfmadb %%v0,%%v16,%%v18,%%v0\n\t"
"vfmadb %%v0,%%v17,%%v19,%%v0\n\t"
"vst %%v0,0(%[y])\n\t"
: "+m"(*(struct { FLOAT x[2]; } *) y),[n] "+&r"(n)
: [y] "a"(y), "m"(*(const struct { FLOAT x[n * 2]; } *) ap),[ap] "a"(ap),
"m"(*(const struct { FLOAT x[n * 2]; } *) x),[x] "a"(x),
"m"(*(const struct { FLOAT x[2]; } *) alpha),[alpha] "a"(alpha)
: "+m"(*(FLOAT (*)[2]) y),[n] "+&r"(n)
: [y] "a"(y), "m"(*(const FLOAT (*)[n * 2]) ap),[ap] "a"(ap),
"m"(*(const FLOAT (*)[n * 2]) x),[x] "a"(x),
"m"(*(const FLOAT (*)[2]) alpha),[alpha] "a"(alpha)
: "cc", "r1", "v0", "v1", "v16", "v17", "v18", "v19");
}



+ 2
- 2
kernel/zarch/zrot.c View File

@@ -169,8 +169,8 @@ static void zrot_kernel_16(BLASLONG n, FLOAT *x, FLOAT *y, FLOAT *c, FLOAT *s) {
"vst %%v23, 240(%%r1,%[y])\n\t"
"agfi %%r1,256\n\t"
"brctg %[n],0b"
: "+m"(*(struct { FLOAT x[n * 2]; } *) x),
"+m"(*(struct { FLOAT x[n * 2]; } *) y),[n] "+&r"(n)
: "+m"(*(FLOAT (*)[n * 2]) x),
"+m"(*(FLOAT (*)[n * 2]) y),[n] "+&r"(n)
: [x] "a"(x),[y] "a"(y),[c] "Q"(*c),[s] "Q"(*s)
: "cc", "r1", "v0", "v1", "v16", "v17", "v18", "v19", "v20", "v21",
"v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30",


+ 7
- 7
kernel/zarch/zscal.c View File

@@ -78,8 +78,8 @@ static void zscal_kernel_8(BLASLONG n, FLOAT *alpha, FLOAT *x) {
"vst %%v23,112(%%r1,%[x])\n\t"
"agfi %%r1,128\n\t"
"brctg %[n],0b"
: "+m"(*(struct { FLOAT x[n * 2]; } *) x),[n] "+&r"(n)
: [x] "a"(x), "m"(*(const struct { FLOAT x[2]; } *) alpha),
: "+m"(*(FLOAT (*)[n * 2]) x),[n] "+&r"(n)
: [x] "a"(x), "m"(*(const FLOAT (*)[2]) alpha),
[alpha] "a"(alpha)
: "cc", "r1", "v0", "v1", "v16", "v17", "v18", "v19", "v20", "v21",
"v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30",
@@ -128,8 +128,8 @@ static void zscal_kernel_8_zero_r(BLASLONG n, FLOAT *alpha, FLOAT *x) {
"vst %%v23,112(%%r1,%[x])\n\t"
"agfi %%r1,128\n\t"
"brctg %[n],0b"
: "+m"(*(struct { FLOAT x[n * 2]; } *) x),[n] "+&r"(n)
: [x] "a"(x), "m"(*(const struct { FLOAT x[2]; } *) alpha),
: "+m"(*(FLOAT (*)[n * 2]) x),[n] "+&r"(n)
: [x] "a"(x), "m"(*(const FLOAT (*)[2]) alpha),
[alpha] "a"(alpha)
: "cc", "r1", "v0", "v16", "v17", "v18", "v19", "v20", "v21", "v22",
"v23");
@@ -167,8 +167,8 @@ static void zscal_kernel_8_zero_i(BLASLONG n, FLOAT *alpha, FLOAT *x) {
"vst %%v23,112(%%r1,%[x])\n\t"
"agfi %%r1,128\n\t"
"brctg %[n],0b"
: "+m"(*(struct { FLOAT x[n * 2]; } *) x),[n] "+&r"(n)
: [x] "a"(x), "m"(*(const struct { FLOAT x[2]; } *) alpha),
: "+m"(*(FLOAT (*)[n * 2]) x),[n] "+&r"(n)
: [x] "a"(x), "m"(*(const FLOAT (*)[2]) alpha),
[alpha] "a"(alpha)
: "cc", "r1", "v0", "v16", "v17", "v18", "v19", "v20", "v21", "v22",
"v23");
@@ -190,7 +190,7 @@ static void zscal_kernel_8_zero(BLASLONG n, FLOAT *x) {
"vst %%v0,112(%%r1,%[x])\n\t"
"agfi %%r1,128\n\t"
"brctg %[n],0b"
: "=m"(*(struct { FLOAT x[n * 2]; } *) x),[n] "+&r"(n)
: "=m"(*(FLOAT (*)[n * 2]) x),[n] "+&r"(n)
: [x] "a"(x)
: "cc", "r1", "v0");
}


+ 1
- 1
kernel/zarch/zsum.c View File

@@ -89,7 +89,7 @@ static FLOAT zsum_kernel_16(BLASLONG n, FLOAT *x) {
"vfadb %%v24,%%v24,%%v25\n\t"
"vsteg %%v24,%[sum],0"
: [sum] "=Q"(sum),[n] "+&r"(n)
: "m"(*(const struct { FLOAT x[n * 2]; } *) x),[x] "a"(x)
: "m"(*(const FLOAT (*)[n * 2]) x),[x] "a"(x)
: "cc", "r1", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23",
"v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31");



+ 2
- 2
kernel/zarch/zswap.c View File

@@ -99,8 +99,8 @@ static void zswap_kernel_16(BLASLONG n, FLOAT *x, FLOAT *y) {
"vst %%v31, 240(%%r1,%[y])\n\t"
"agfi %%r1,256\n\t"
"brctg %[n],0b"
: "+m"(*(struct { FLOAT x[n * 2]; } *) x),
"+m"(*(struct { FLOAT x[n * 2]; } *) y),[n] "+&r"(n)
: "+m"(*(FLOAT (*)[n * 2]) x),
"+m"(*(FLOAT (*)[n * 2]) y),[n] "+&r"(n)
: [x] "a"(x),[y] "a"(y)
: "cc", "r1", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v16",
"v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26",


+ 9
- 9
kernel/zarch/ztrmm4x4V.S View File

@@ -196,7 +196,7 @@ ALIGN_4
RefreshTempBk LOCAL_VAR1,BK,OFF,4,4
nill LOCAL_VAR1,3
#else
la LOCAL_VAR1,3(0,0)
lghi LOCAL_VAR1,3
NGR LOCAL_VAR1,BK /*refresh BK*/
#endif
jz .L4x4_BK_Store
@@ -256,7 +256,7 @@ ALIGN_4
RefreshTempBk LOCAL_VAR1,BK,OFF,2,4
nill LOCAL_VAR1,3
#else
la LOCAL_VAR1,3(0,0)
lghi LOCAL_VAR1,3
NGR LOCAL_VAR1,BK /*refresh BK*/
#endif
jz .L2x4_BK_Store
@@ -307,7 +307,7 @@ ALIGN_4
RefreshTempBk LOCAL_VAR1,BK,OFF,1,4
nill LOCAL_VAR1,3
#else
la LOCAL_VAR1,3(0,0)
lghi LOCAL_VAR1,3
NGR LOCAL_VAR1,BK /*refresh BK*/
#endif
jz .L1x4_BK_Store
@@ -390,7 +390,7 @@ ALIGN_4
RefreshTempBk LOCAL_VAR1,BK,OFF,4,2
nill LOCAL_VAR1,3
#else
la LOCAL_VAR1,3(0,0)
lghi LOCAL_VAR1,3
NGR LOCAL_VAR1,BK /*refresh BK*/
#endif
jz .L4x2_BK_Store
@@ -447,7 +447,7 @@ ALIGN_4
RefreshTempBk LOCAL_VAR1,BK,OFF,2,2
nill LOCAL_VAR1,3
#else
la LOCAL_VAR1,3(0,0)
lghi LOCAL_VAR1,3
NGR LOCAL_VAR1,BK /*refresh BK*/
#endif
jz .L2x2_BK_Store
@@ -497,7 +497,7 @@ ALIGN_4
RefreshTempBk LOCAL_VAR1,BK,OFF,1,2
nill LOCAL_VAR1,3
#else
la LOCAL_VAR1,3(0,0)
lghi LOCAL_VAR1,3
NGR LOCAL_VAR1,BK /*refresh BK*/
#endif
jz .L1x2_BK_Store
@@ -573,7 +573,7 @@ ALIGN_4
RefreshTempBk LOCAL_VAR1,BK,OFF,4,1
nill LOCAL_VAR1,3
#else
la LOCAL_VAR1,3(0,0)
lghi LOCAL_VAR1,3
NGR LOCAL_VAR1,BK /*refresh BK*/
#endif
jz .L4x1_BK_Store
@@ -625,7 +625,7 @@ ALIGN_4
RefreshTempBk LOCAL_VAR1,BK,OFF,2,1
nill LOCAL_VAR1,3
#else
la LOCAL_VAR1,3(0,0)
lghi LOCAL_VAR1,3
NGR LOCAL_VAR1,BK /*refresh BK*/
#endif
jz .L2x1_BK_Store
@@ -675,7 +675,7 @@ ALIGN_4
RefreshTempBk LOCAL_VAR1,BK,OFF,1,1
nill LOCAL_VAR1,3
#else
la LOCAL_VAR1,3(0,0)
lghi LOCAL_VAR1,3
NGR LOCAL_VAR1,BK /*refresh BK*/
#endif
jz .L1x1_BK_Store


+ 6
- 5
relapack/src/cgbtrf.c View File

@@ -36,6 +36,7 @@ void RELAPACK_cgbtrf(
return;
}

if (*m == 0 || *n == 0) return;
// Constant
const float ZERO[] = { 0., 0. };

@@ -56,10 +57,10 @@ void RELAPACK_cgbtrf(

// Allocate work space
const blasint n1 = CREC_SPLIT(*n);
const blasint mWorkl = (kv > n1) ? MAX(1, *m - *kl) : kv;
const blasint nWorkl = (kv > n1) ? n1 : kv;
const blasint mWorku = (*kl > n1) ? n1 : *kl;
const blasint nWorku = (*kl > n1) ? MAX(0, *n - *kl) : *kl;
const blasint mWorkl = abs ( (kv > n1) ? MAX(1, *m - *kl) : kv);
const blasint nWorkl = abs ( (kv > n1) ? n1 : kv);
const blasint mWorku = abs ((*kl > n1) ? n1 : *kl);
const blasint nWorku = abs ((*kl > n1) ? MAX(0, *n - *kl) : *kl);
float *Workl = malloc(mWorkl * nWorkl * 2 * sizeof(float));
float *Worku = malloc(mWorku * nWorku * 2 * sizeof(float));
LAPACK(claset)("L", &mWorkl, &nWorkl, ZERO, ZERO, Workl, &mWorkl);
@@ -82,7 +83,7 @@ static void RELAPACK_cgbtrf_rec(
blasint *info
) {

if (*n <= MAX(CROSSOVER_CGBTRF, 1)) {
if (*n <= MAX(CROSSOVER_CGBTRF, 1)|| *n > *kl || *ldAb == 1) {
// Unblocked
LAPACK(cgbtf2)(m, n, kl, ku, Ab, ldAb, ipiv, info);
return;


+ 7
- 2
relapack/src/cgetrf.c View File

@@ -30,6 +30,8 @@ void RELAPACK_cgetrf(
return;
}

if (*m == 0 || *n == 0) return;

const blasint sn = MIN(*m, *n);

RELAPACK_cgetrf_rec(m, &sn, A, ldA, ipiv, info);
@@ -62,9 +64,11 @@ static void RELAPACK_cgetrf_rec(
blasint *info
) {

if (*n <= MAX(CROSSOVER_CGETRF, 1)) {
if (*m == 0 || *n == 0) return;

if ( *n <= MAX(CROSSOVER_CGETRF, 1)) {
// Unblocked
LAPACK(cgetf2)(m, n, A, ldA, ipiv, info);
LAPACK(cgetrf2)(m, n, A, ldA, ipiv, info);
return;
}

@@ -96,6 +100,7 @@ static void RELAPACK_cgetrf_rec(

// recursion(A_L, ipiv_T)
RELAPACK_cgetrf_rec(m, &n1, A_L, ldA, ipiv_T, info);
if (*info) return;
// apply pivots to A_R
LAPACK(claswp)(&n2, A_R, ldA, iONE, &n1, ipiv_T, iONE);



+ 2
- 0
relapack/src/chegst.c View File

@@ -40,6 +40,8 @@ void RELAPACK_chegst(
return;
}

if (*n == 0) return;

// Clean char * arguments
const char cleanuplo = lower ? 'L' : 'U';



+ 2
- 2
relapack/src/chetrf_rook.c View File

@@ -36,7 +36,7 @@ void RELAPACK_chetrf_rook(
*info = -2;
else if (*ldA < MAX(1, *n))
*info = -4;
else if (*lWork < minlWork && *lWork != -1)
else if ((*lWork < 1 || *lWork < minlWork) && *lWork != -1)
*info = -7;
else if (*lWork == -1) {
// Work size query
@@ -56,7 +56,7 @@ void RELAPACK_chetrf_rook(

if (*info) {
const blasint minfo = -*info;
LAPACK(xerbla)("CHETRF", &minfo, strlen("CHETRF"));
LAPACK(xerbla)("CHETRF_ROOK", &minfo, strlen("CHETRF_ROOK"));
return;
}



+ 2
- 0
relapack/src/clauum.c View File

@@ -32,6 +32,8 @@ void RELAPACK_clauum(
return;
}

if (*n == 0) return;

// Clean char * arguments
const char cleanuplo = lower ? 'L' : 'U';



+ 6
- 4
relapack/src/cpbtrf.c View File

@@ -35,6 +35,8 @@ void RELAPACK_cpbtrf(
return;
}

if (*n == 0) return;

// Clean char * arguments
const char cleanuplo = lower ? 'L' : 'U';

@@ -43,8 +45,8 @@ void RELAPACK_cpbtrf(

// Allocate work space
const blasint n1 = CREC_SPLIT(*n);
const blasint mWork = (*kd > n1) ? (lower ? *n - *kd : n1) : *kd;
const blasint nWork = (*kd > n1) ? (lower ? n1 : *n - *kd) : *kd;
const blasint mWork = abs((*kd > n1) ? (lower ? *n - *kd : n1) : *kd);
const blasint nWork = abs((*kd > n1) ? (lower ? n1 : *n - *kd) : *kd);
float *Work = malloc(mWork * nWork * 2 * sizeof(float));
LAPACK(claset)(uplo, &mWork, &nWork, ZERO, ZERO, Work, &mWork);

@@ -64,7 +66,7 @@ static void RELAPACK_cpbtrf_rec(
blasint *info
){

if (*n <= MAX(CROSSOVER_CPBTRF, 1)) {
if (*n <= MAX(CROSSOVER_CPBTRF, 1) || *ldAb==1) {
// Unblocked
LAPACK(cpbtf2)(uplo, n, kd, Ab, ldAb, info);
return;
@@ -148,7 +150,7 @@ static void RELAPACK_cpbtrf_rec(
}

// recursion(A_BR)
if (*kd > n1)
if (*kd > n1 && ldA != 0)
RELAPACK_cpotrf(uplo, &n2, A_BR, ldA, info);
else
RELAPACK_cpbtrf_rec(uplo, &n2, kd, Ab_BR, ldAb, Work, ldWork, info);


+ 3
- 0
relapack/src/cpotrf.c View File

@@ -32,6 +32,8 @@ void RELAPACK_cpotrf(
return;
}

if (*n == 0) return;

// Clean char * arguments
const char cleanuplo = lower ? 'L' : 'U';

@@ -46,6 +48,7 @@ static void RELAPACK_cpotrf_rec(
float *A, const blasint *ldA,
blasint *info
){
if (*n == 0) return;

if (*n <= MAX(CROSSOVER_CPOTRF, 1)) {
// Unblocked


+ 2
- 1
relapack/src/csytrf.c View File

@@ -36,7 +36,7 @@ void RELAPACK_csytrf(
*info = -2;
else if (*ldA < MAX(1, *n))
*info = -4;
else if (*lWork < minlWork && *lWork != -1)
else if ((*lWork < 1 || *lWork < minlWork) && *lWork != -1)
*info = -7;
else if (*lWork == -1) {
// Work size query
@@ -67,6 +67,7 @@ void RELAPACK_csytrf(
blasint nout;

// Recursive kernel
if (*n != 0)
RELAPACK_csytrf_rec(&cleanuplo, n, n, &nout, A, ldA, ipiv, cleanWork, n, info);

#if XSYTRF_ALLOW_MALLOC


+ 2
- 2
relapack/src/csytrf_rook.c View File

@@ -36,7 +36,7 @@ void RELAPACK_csytrf_rook(
*info = -2;
else if (*ldA < MAX(1, *n))
*info = -4;
else if (*lWork < minlWork && *lWork != -1)
else if ((*lWork < 1 || *lWork < minlWork) && *lWork != -1)
*info = -7;
else if (*lWork == -1) {
// Work size query
@@ -56,7 +56,7 @@ void RELAPACK_csytrf_rook(

if (*info) {
const blasint minfo = -*info;
LAPACK(xerbla)("CSYTRF", &minfo, strlen("CSYTRF"));
LAPACK(xerbla)("CSYTRF_ROOK", &minfo, strlen("CSYTRF_ROOK"));
return;
}



+ 7
- 0
relapack/src/ctgsyl.c View File

@@ -68,6 +68,13 @@ void RELAPACK_ctgsyl(
return;
}

if ( *m == 0 || *n == 0) {
*scale = 1.;
if (notran && (*ijob != 0))
*dif = 0.;
return;
}

// Clean char * arguments
const char cleantrans = notran ? 'N' : 'C';



+ 5
- 0
relapack/src/ctrsyl.c View File

@@ -47,6 +47,11 @@ void RELAPACK_ctrsyl(
return;
}

if (*m == 0 || *n == 0) {
*scale = 1.;
return;
}

// Clean char * arguments
const char cleantranA = notransA ? 'N' : 'C';
const char cleantranB = notransB ? 'N' : 'C';


+ 2
- 0
relapack/src/ctrtri.c View File

@@ -36,6 +36,8 @@ void RELAPACK_ctrtri(
return;
}

if (*n == 0) return;

// Clean char * arguments
const char cleanuplo = lower ? 'L' : 'U';
const char cleandiag = nounit ? 'N' : 'U';


+ 4
- 1
relapack/src/dgbtrf.c View File

@@ -36,6 +36,8 @@ void RELAPACK_dgbtrf(
return;
}

if (*m == 0 || *n == 0) return;

// Constant
const double ZERO[] = { 0. };

@@ -83,7 +85,7 @@ static void RELAPACK_dgbtrf_rec(
blasint *info
) {

if (*n <= MAX(CROSSOVER_DGBTRF, 1)) {
if (*n <= MAX(CROSSOVER_DGBTRF, 1) || *n > *kl || *ldAb == 1) {
// Unblocked
LAPACK(dgbtf2)(m, n, kl, ku, Ab, ldAb, ipiv, info);
return;
@@ -195,6 +197,7 @@ static void RELAPACK_dgbtrf_rec(
// Worku = A_TRr
LAPACK(dlacpy)("L", &m1, &n22, A_TRr, ldA, Worku, ldWorku);
// Worku = A_TL \ Worku
if (ldWorku <= 0) return;
BLAS(dtrsm)("L", "L", "N", "U", &m1, &n22, ONE, A_TL, ldA, Worku, ldWorku);
// A_TRr = Worku
LAPACK(dlacpy)("L", &m1, &n22, Worku, ldWorku, A_TRr, ldA);


+ 6
- 6
relapack/src/dgetrf.c View File

@@ -29,15 +29,16 @@ void RELAPACK_dgetrf(
return;
}

const blasint sn = MIN(*m, *n);
if (*m == 0 || *n == 0) return;

const blasint sn = MIN(*m, *n);
RELAPACK_dgetrf_rec(m, &sn, A, ldA, ipiv, info);

// Right remainder
if (*m < *n) {
// Constants
const double ONE[] = { 1. };
const blasint iONE[] = { 1. };
const blasint iONE[] = { 1 };

// Splitting
const blasint rn = *n - *m;
@@ -60,13 +61,11 @@ static void RELAPACK_dgetrf_rec(
double *A, const blasint *ldA, blasint *ipiv,
blasint *info
) {

if (*n <= MAX(CROSSOVER_DGETRF, 1)) {
if ( *n <= MAX(CROSSOVER_DGETRF, 1)) {
// Unblocked
LAPACK(dgetf2)(m, n, A, ldA, ipiv, info);
LAPACK(dgetrf2)(m, n, A, ldA, ipiv, info);
return;
}

// Constants
const double ONE[] = { 1. };
const double MONE[] = { -1. };
@@ -95,6 +94,7 @@ static void RELAPACK_dgetrf_rec(

// recursion(A_L, ipiv_T)
RELAPACK_dgetrf_rec(m, &n1, A_L, ldA, ipiv_T, info);
if (*info) return;
// apply pivots to A_R
LAPACK(dlaswp)(&n2, A_R, ldA, iONE, &n1, ipiv_T, iONE);



+ 6
- 4
relapack/src/dpbtrf.c View File

@@ -35,6 +35,8 @@ void RELAPACK_dpbtrf(
return;
}

if (*n == 0) return;

// Clean char * arguments
const char cleanuplo = lower ? 'L' : 'U';

@@ -43,8 +45,8 @@ void RELAPACK_dpbtrf(

// Allocate work space
const blasint n1 = DREC_SPLIT(*n);
const blasint mWork = (*kd > n1) ? (lower ? *n - *kd : n1) : *kd;
const blasint nWork = (*kd > n1) ? (lower ? n1 : *n - *kd) : *kd;
const blasint mWork = abs((*kd > n1) ? (lower ? *n - *kd : n1) : *kd);
const blasint nWork = abs((*kd > n1) ? (lower ? n1 : *n - *kd) : *kd);
double *Work = malloc(mWork * nWork * sizeof(double));
LAPACK(dlaset)(uplo, &mWork, &nWork, ZERO, ZERO, Work, &mWork);

@@ -64,7 +66,7 @@ static void RELAPACK_dpbtrf_rec(
blasint *info
){

if (*n <= MAX(CROSSOVER_DPBTRF, 1)) {
if (*n <= MAX(CROSSOVER_DPBTRF, 1) || *ldAb == 1) {
// Unblocked
LAPACK(dpbtf2)(uplo, n, kd, Ab, ldAb, info);
return;
@@ -148,7 +150,7 @@ static void RELAPACK_dpbtrf_rec(
}

// recursion(A_BR)
if (*kd > n1)
if (*kd > n1 && ldA != 0)
RELAPACK_dpotrf(uplo, &n2, A_BR, ldA, info);
else
RELAPACK_dpbtrf_rec(uplo, &n2, kd, Ab_BR, ldAb, Work, ldWork, info);


+ 2
- 1
relapack/src/dsytrf.c View File

@@ -36,7 +36,7 @@ void RELAPACK_dsytrf(
*info = -2;
else if (*ldA < MAX(1, *n))
*info = -4;
else if (*lWork < minlWork && *lWork != -1)
else if ((*lWork < 1 || *lWork < minlWork) && *lWork != -1)
*info = -7;
else if (*lWork == -1) {
// Work size query
@@ -67,6 +67,7 @@ void RELAPACK_dsytrf(
blasint nout;

// Recursive kernel
if (*n != 0)
RELAPACK_dsytrf_rec(&cleanuplo, n, n, &nout, A, ldA, ipiv, cleanWork, n, info);

#if XSYTRF_ALLOW_MALLOC


+ 2
- 2
relapack/src/dsytrf_rook.c View File

@@ -36,7 +36,7 @@ void RELAPACK_dsytrf_rook(
*info = -2;
else if (*ldA < MAX(1, *n))
*info = -4;
else if (*lWork < minlWork && *lWork != -1)
else if ((*lWork <1 || *lWork < minlWork) && *lWork != -1)
*info = -7;
else if (*lWork == -1) {
// Work size query
@@ -56,7 +56,7 @@ void RELAPACK_dsytrf_rook(

if (*info) {
const blasint minfo = -*info;
LAPACK(xerbla)("DSYTRF", &minfo, strlen("DSYTRF"));
LAPACK(xerbla)("DSYTRF_ROOK", &minfo, strlen("DSYTRF_ROOK"));
return;
}



+ 5
- 0
relapack/src/dtrsyl.c View File

@@ -49,6 +49,11 @@ void RELAPACK_dtrsyl(
return;
}

if (*m == 0 || *n == 0) {
*scale = 1.;
return;
}

// Clean char * arguments
const char cleantranA = notransA ? 'N' : (transA ? 'T' : 'C');
const char cleantranB = notransB ? 'N' : (transB ? 'T' : 'C');


+ 7
- 0
relapack/src/lapack.h View File

@@ -4,6 +4,13 @@
extern blasint LAPACK(lsame)(const char *, const char *);
extern blasint LAPACK(xerbla)(const char *, const blasint *, int);

extern const blasint LAPACK(ilaenv)(const blasint *, const char*, const char*, const blasint* , int , int, int );

extern void LAPACK(sgetrf2)(const blasint *, const blasint *, float *, const blasint *, blasint *, blasint *);
extern void LAPACK(dgetrf2)(const blasint *, const blasint *, double *, const blasint *, blasint *, blasint *);
extern void LAPACK(cgetrf2)(const blasint *, const blasint *, float *, const blasint *, blasint *, blasint *);
extern void LAPACK(zgetrf2)(const blasint *, const blasint *, double *, const blasint *, blasint *, blasint *);

extern void LAPACK(slaswp)(const blasint *, float *, const blasint *, const blasint *, const blasint *, const blasint *, const blasint *);
extern void LAPACK(dlaswp)(const blasint *, double *, const blasint *, const blasint *, const blasint *, const blasint *, const blasint *);
extern void LAPACK(claswp)(const blasint *, float *, const blasint *, const blasint *, const blasint *, const blasint *, const blasint *);


+ 12
- 4
relapack/src/sgbtrf.c View File

@@ -35,6 +35,13 @@ void RELAPACK_sgbtrf(
return;
}

if (*m == 0 || *n == 0) return;

if (*ldAb == 1) {
LAPACK(sgbtf2)(m, n, kl, ku, Ab, ldAb, ipiv, info);
return;
}

// Constant
const float ZERO[] = { 0. };

@@ -82,8 +89,9 @@ static void RELAPACK_sgbtrf_rec(
blasint *info
) {

if (*m == 0 || *n == 0) return;

if (*n <= MAX(CROSSOVER_SGBTRF, 1)) {
if ( *n <= MAX(CROSSOVER_SGBTRF, 1) || *n > *kl || *ldAb == 1) {
// Unblocked
LAPACK(sgbtf2)(m, n, kl, ku, Ab, ldAb, ipiv, info);
return;
@@ -160,7 +168,7 @@ static void RELAPACK_sgbtrf_rec(

// recursion(Ab_L, ipiv_T)
RELAPACK_sgbtrf_rec(m, &n1, kl, ku, Ab_L, ldAb, ipiv_T, Workl, ldWorkl, Worku, ldWorku, info);
if (*info) return;
// Workl = A_BLb
LAPACK(slacpy)("U", &m22, &n1, A_BLb, ldA, Workl, ldWorkl);

@@ -222,8 +230,8 @@ static void RELAPACK_sgbtrf_rec(

// recursion(Ab_BR, ipiv_B)
//cause of infinite recursion here ?
// RELAPACK_sgbtrf_rec(&m2, &n2, kl, ku, Ab_BR, ldAb, ipiv_B, Workl, ldWorkl, Worku, ldWorku, info);
LAPACK(sgbtf2)(&m2, &n2, kl, ku, Ab_BR, ldAb, ipiv_B, info);
RELAPACK_sgbtrf_rec(&m2, &n2, kl, ku, Ab_BR, ldAb, ipiv_B, Workl, ldWorkl, Worku, ldWorku, info);
// LAPACK(sgbtf2)(&m2, &n2, kl, ku, Ab_BR, ldAb, ipiv_B, info);
if (*info)
*info += n1;
// shift pivots


Some files were not shown because too many files changed in this diff

Write Preview
Loading…
Cancel
Save