OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
Browse Source

loongarch64: Add optimizations for imin.

tags/v0.3.26
yancheng Shiyou Yin 2 years ago
parent
e46b48e372
commit
e3fb2b5afa
6 changed files with 1148 additions and 0 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +3
    -0
      kernel/loongarch64/KERNEL.LOONGSON2K1000
  2. +3
    -0
      kernel/loongarch64/KERNEL.LOONGSON3R5
  3. +272
    -0
      kernel/loongarch64/idmin_lasx.S
  4. +225
    -0
      kernel/loongarch64/idmin_lsx.S
  5. +374
    -0
      kernel/loongarch64/ismin_lasx.S
  6. +271
    -0
      kernel/loongarch64/ismin_lsx.S

+ 3
- 0
kernel/loongarch64/KERNEL.LOONGSON2K1000 View File

@@ -22,4 +22,7 @@ DMINKERNEL = dmin_lsx.S
ISMAXKERNEL = ismax_lsx.S
IDMAXKERNEL = idmax_lsx.S

ISMINKERNEL = ismin_lsx.S
IDMINKERNEL = idmin_lsx.S

endif

+ 3
- 0
kernel/loongarch64/KERNEL.LOONGSON3R5 View File

@@ -22,6 +22,9 @@ DMINKERNEL = dmin_lasx.S
ISMAXKERNEL = ismax_lasx.S
IDMAXKERNEL = idmax_lasx.S

ISMINKERNEL = ismin_lasx.S
IDMINKERNEL = idmin_lasx.S

DGEMMKERNEL = dgemm_kernel_16x4.S
DGEMMINCOPY = dgemm_ncopy_16.S
DGEMMITCOPY = dgemm_tcopy_16.S


+ 272
- 0
kernel/loongarch64/idmin_lasx.S View File

@@ -0,0 +1,272 @@
#define ASSEMBLER

#include "common.h"

#define N $r4
#define X $r5
#define INCX $r6
#define I $r12
#define t1 $r13
#define t2 $r15
#define t3 $r18
#define t4 $r16
#define i0 $r17
#define i1 $r14
#define TEMP $r19
#define x1 $xr9
#define x2 $xr10
#define x3 $xr11
#define x4 $xr12
#define VX0 $xr13
#define VX1 $xr14
#define VM0 $xr15
#define VM1 $xr16
#define VINC4 $xr17
#define VINC8 $xr18
#define VI0 $xr20
#define VI1 $xr21
#define VI2 $xr22
#define VI3 $xr8
#define VI4 $xr19
#define VT0 $xr23

PROLOGUE
li.d i0, 0
bge $r0, N, .L999
bge $r0, INCX, .L999
li.d TEMP, 1
slli.d TEMP, TEMP, BASE_SHIFT
slli.d INCX, INCX, BASE_SHIFT
bne INCX, TEMP, .L20
xvld VM0, X, 0
addi.d i0, i0, 1
srai.d I, N, 3
bge $r0, I, .L21
slli.d i0, i0, 2 //4
xvreplgr2vr.d VINC4, i0
slli.d i0, i0, 1 //8
xvreplgr2vr.d VINC8, i0
addi.d i0, i0, -15
xvinsgr2vr.d VI1, i0, 0 //initialize the index value for vectorization
addi.d i0, i0, 1
xvinsgr2vr.d VI1, i0, 1
addi.d i0, i0, 1
xvinsgr2vr.d VI1, i0, 2
addi.d i0, i0, 1
xvinsgr2vr.d VI1, i0, 3
addi.d i0, i0, 5
xvinsgr2vr.d VI0, i0, 0 //1
addi.d i0, i0, 1
xvinsgr2vr.d VI0, i0, 1 //2
addi.d i0, i0, 1
xvinsgr2vr.d VI0, i0, 2 //3
addi.d i0, i0, 1
xvinsgr2vr.d VI0, i0, 3 //4
.align 3

.L10:
xvld VX0, X, 0 * SIZE
xvadd.d VI1, VI1, VINC8
xvld VX1, X, 4 * SIZE
xvadd.d VI2, VI1, VINC4
xvfcmp.clt.d VT0, VX1, VX0
addi.d I, I, -1
xvbitsel.v VM1, VX0, VX1, VT0
xvbitsel.v VI2, VI1, VI2, VT0
xvfcmp.clt.d VT0, VM1, VM0
addi.d X, X, 8 * SIZE
xvbitsel.v VM0, VM0, VM1, VT0
xvbitsel.v VI0, VI0, VI2, VT0
blt $r0, I, .L10
.align 3

.L15:
xvpickve.d VI1, VI0, 0
xvpickve.d VI2, VI0, 1
xvpickve.d VI3, VI0, 2
xvpickve.d VI4, VI0, 3
xvpickve.d x1, VM0, 0
xvpickve.d x2, VM0, 1
xvpickve.d x3, VM0, 2
xvpickve.d x4, VM0, 3
xvfcmp.clt.d VT0, x2, x1
xvbitsel.v VM1, x1, x2, VT0
xvbitsel.v VINC4, VI1, VI2, VT0
xvfcmp.clt.d VT0, x4, x3
xvbitsel.v VM0, x3, x4, VT0
xvbitsel.v VINC8, VI3, VI4, VT0
xvfcmp.clt.d VT0, VM1, VM0
xvbitsel.v VM0, VM0, VM1, VT0
xvbitsel.v VI0, VINC8, VINC4, VT0
li.d TEMP, 1 //处理尾数相等时取最小序号
movgr2fr.d $f17, TEMP
ffint.d.l $f17, $f17
xvfcmp.ceq.d VT0, VM0, x1
fcmp.ceq.d $fcc0, $f23, $f17
bceqz $fcc0, .L26
xvfcmp.clt.d VT0, VI1, VI0
xvbitsel.v VI0, VI0, VI1, VT0
b .L26
.align 3

.L20: // INCX!=1
move TEMP, X
addi.d i0, i0, 1
ld.d t1, TEMP, 0 * SIZE
add.d TEMP, TEMP, INCX
xvinsgr2vr.d VM0, t1, 0
srai.d I, N, 3
bge $r0, I, .L21
ld.d t2, TEMP, 0 * SIZE
add.d TEMP, TEMP, INCX
ld.d t3, TEMP, 0 * SIZE
add.d TEMP, TEMP, INCX
ld.d t4, TEMP, 0 * SIZE
add.d TEMP, TEMP, INCX
xvinsgr2vr.d VM0, t2, 1
xvinsgr2vr.d VM0, t3, 2
xvinsgr2vr.d VM0, t4, 3
slli.d i0, i0, 2 //4
xvreplgr2vr.d VINC4, i0
slli.d i0, i0, 1 //8
xvreplgr2vr.d VINC8, i0
addi.d i0, i0, -15
xvinsgr2vr.d VI1, i0, 0 //initialize the index value for vectorization
addi.d i0, i0, 1
xvinsgr2vr.d VI1, i0, 1
addi.d i0, i0, 1
xvinsgr2vr.d VI1, i0, 2
addi.d i0, i0, 1
xvinsgr2vr.d VI1, i0, 3
addi.d i0, i0, 5
xvinsgr2vr.d VI0, i0, 0 //1
addi.d i0, i0, 1
xvinsgr2vr.d VI0, i0, 1 //2
addi.d i0, i0, 1
xvinsgr2vr.d VI0, i0, 2 //3
addi.d i0, i0, 1
xvinsgr2vr.d VI0, i0, 3 //4
.align 3

.L24:
ld.d t1, X, 0 * SIZE
add.d X, X, INCX
ld.d t2, X, 0 * SIZE
add.d X, X, INCX
ld.d t3, X, 0 * SIZE
add.d X, X, INCX
ld.d t4, X, 0 * SIZE
add.d X, X, INCX
xvinsgr2vr.d VX0, t1, 0
xvinsgr2vr.d VX0, t2, 1
xvinsgr2vr.d VX0, t3, 2
xvinsgr2vr.d VX0, t4, 3
xvadd.d VI1, VI1, VINC8
ld.d t1, X, 0 * SIZE
add.d X, X, INCX
ld.d t2, X, 0 * SIZE
add.d X, X, INCX
ld.d t3, X, 0 * SIZE
add.d X, X, INCX
ld.d t4, X, 0 * SIZE
add.d X, X, INCX
xvinsgr2vr.d VX1, t1, 0
xvinsgr2vr.d VX1, t2, 1
xvinsgr2vr.d VX1, t3, 2
xvinsgr2vr.d VX1, t4, 3
xvadd.d VI2, VI1, VINC4
xvfcmp.clt.d VT0, VX1, VX0
addi.d I, I, -1
xvbitsel.v VM1, VX0, VX1, VT0
xvbitsel.v VI2, VI1, VI2, VT0
xvfcmp.clt.d VT0, VM1, VM0
xvbitsel.v VM0, VM0, VM1, VT0
xvbitsel.v VI0, VI0, VI2, VT0
blt $r0, I, .L24
.align 3

.L25:
xvpickve.d VI1, VI0, 0
xvpickve.d VI2, VI0, 1
xvpickve.d VI3, VI0, 2
xvpickve.d VI4, VI0, 3
xvpickve.d x1, VM0, 0
xvpickve.d x2, VM0, 1
xvpickve.d x3, VM0, 2
xvpickve.d x4, VM0, 3
xvfcmp.clt.d VT0, x2, x1
xvbitsel.v VM1, x1, x2, VT0
xvbitsel.v VINC4, VI1, VI2, VT0
xvfcmp.clt.d VT0, x4, x3
xvbitsel.v VM0, x3, x4, VT0
xvbitsel.v VINC8, VI3, VI4, VT0
xvfcmp.clt.d VT0, VM1, VM0
xvbitsel.v VM0, VM0, VM1, VT0
xvbitsel.v VI0, VINC8, VINC4, VT0
li.d TEMP, 1 //处理尾数相等时取最小序号
movgr2fr.d $f17, TEMP
ffint.d.l $f17, $f17
xvfcmp.ceq.d VT0, VM0, x1
fcmp.ceq.d $fcc0, $f23, $f17
bceqz $fcc0, .L26
xvfcmp.clt.d VT0, VI1, VI0
xvbitsel.v VI0, VI0, VI1, VT0
.align 3

.L26:
xvfcmp.ceq.d VT0, VM0, x2
fcmp.ceq.d $fcc0, $f23, $f17
bceqz $fcc0, .L27
xvfcmp.clt.d VT0, VI2, VI0
xvbitsel.v VI0, VI0, VI2, VT0
.align 3

.L27:
xvfcmp.ceq.d VT0, VM0, x3
fcmp.ceq.d $fcc0, $f23, $f17
bceqz $fcc0, .L28
xvfcmp.clt.d VT0, VI3, VI0
xvbitsel.v VI0, VI0, VI3, VT0
.align 3

.L28:
xvfcmp.ceq.d VT0, VM0, x4
fcmp.ceq.d $fcc0, $f23, $f17
bceqz $fcc0, .L29
xvfcmp.clt.d VT0, VI4, VI0
xvbitsel.v VI0, VI0, VI4, VT0
.align 3

.L29:
movfr2gr.d i0, $f20
.align 3

.L21: //N<8
andi I, N, 7
bge $r0, I, .L999
srai.d i1, N, 3
slli.d i1, i1, 3
addi.d i1, i1, 1 //current index
movgr2fr.d $f21, i1
movgr2fr.d $f20, i0
.align 3

.L22:
fld.d $f9, X, 0
addi.d I, I, -1
fcmp.clt.d $fcc0, $f9, $f15
add.d X, X, INCX
fsel $f15, $f15, $f9, $fcc0
fsel $f20, $f20, $f21, $fcc0
addi.d i1, i1, 1
movgr2fr.d $f21, i1
blt $r0, I, .L22
movfr2gr.d i0, $f20
.align 3

.L999:
move $r4, $r17
jirl $r0, $r1, 0x0
.align 3

EPILOGUE

+ 225
- 0
kernel/loongarch64/idmin_lsx.S View File

@@ -0,0 +1,225 @@
#define ASSEMBLER

#include "common.h"

#define N $r4
#define X $r5
#define INCX $r6
#define I $r12
#define t1 $r13
#define t2 $r15
#define t3 $r18
#define t4 $r16
#define i0 $r17
#define i1 $r14
#define TEMP $r19
#define x1 $vr9
#define x2 $vr10
#define x3 $vr11
#define x4 $vr12
#define VX0 $vr13
#define VX1 $vr14
#define VM0 $vr15
#define VM1 $vr16
#define VINC2 $vr17
#define VINC4 $vr18
#define VI0 $vr20
#define VI1 $vr21
#define VI2 $vr22
#define VI3 $vr8
#define VI4 $vr19
#define VT0 $vr23

PROLOGUE
li.d i0, 0
bge $r0, N, .L999
bge $r0, INCX, .L999
li.d TEMP, 1
slli.d TEMP, TEMP, BASE_SHIFT
slli.d INCX, INCX, BASE_SHIFT
bne INCX, TEMP, .L20
vld VM0, X, 0
addi.d i0, i0, 1
srai.d I, N, 3
bge $r0, I, .L21
slli.d i0, i0, 1 //2
vreplgr2vr.d VINC2, i0
slli.d i0, i0, 1 //4
vreplgr2vr.d VINC4, i0
addi.d i0, i0, -7
vinsgr2vr.d VI1, i0, 0 //initialize the index value for vectorization
addi.d i0, i0, 1
vinsgr2vr.d VI1, i0, 1
addi.d i0, i0, 3
vinsgr2vr.d VI0, i0, 0 //1
addi.d i0, i0, 1
vinsgr2vr.d VI0, i0, 1 //2
.align 3

.L10:
vld VX0, X, 0 * SIZE
vadd.d VI1, VI1, VINC4
vld VX1, X, 2 * SIZE
vadd.d VI2, VI1, VINC2
vfcmp.clt.d VT0, VX1, VX0
vbitsel.v x1, VX0, VX1, VT0
vbitsel.v x2, VI1, VI2, VT0
vld VX0, X, 4 * SIZE
vadd.d VI1, VI2, VINC2
vld VX1, X, 6 * SIZE
vadd.d VI2, VI1, VINC2
vfcmp.clt.d VT0, VX1, VX0
addi.d I, I, -1
vbitsel.v x3, VX0, VX1, VT0
vbitsel.v x4, VI1, VI2, VT0
vfcmp.clt.d VT0, x3, x1
addi.d X, X, 8 * SIZE
vbitsel.v x1, x1, x3, VT0
vbitsel.v x2, x2, x4, VT0
vfcmp.clt.d VT0, x1, VM0
vbitsel.v VM0, VM0, x1, VT0
vbitsel.v VI0, VI0, x2, VT0
blt $r0, I, .L10
.align 3

.L15:
vreplvei.d VI1, VI0, 0
vreplvei.d VI2, VI0, 1
vreplvei.d x1, VM0, 0
vreplvei.d x2, VM0, 1
li.d TEMP, 1 //处理尾数相等时取最小序号
movgr2fr.d $f17, TEMP
ffint.d.l $f17, $f17
vfcmp.ceq.d VT0, x2, x1
fcmp.ceq.d $fcc0, $f23, $f17
bceqz $fcc0, .L26
vfcmp.clt.d VT0, VI1, VI0
vbitsel.v VI0, VI0, VI1, VT0
b .L27
.align 3

.L20: // INCX!=1
move TEMP, X
addi.d i0, i0, 1
ld.d t1, TEMP, 0 * SIZE
add.d TEMP, TEMP, INCX
vinsgr2vr.d VM0, t1, 0
srai.d I, N, 3
bge $r0, I, .L21
ld.d t2, TEMP, 0 * SIZE
add.d TEMP, TEMP, INCX
vinsgr2vr.d VM0, t2, 1
slli.d i0, i0, 1 //2
vreplgr2vr.d VINC2, i0
slli.d i0, i0, 1 //4
vreplgr2vr.d VINC4, i0
addi.d i0, i0, -7
vinsgr2vr.d VI1, i0, 0 //initialize the index value for vectorization
addi.d i0, i0, 1
vinsgr2vr.d VI1, i0, 1
addi.d i0, i0, 3
vinsgr2vr.d VI0, i0, 0 //1
addi.d i0, i0, 1
vinsgr2vr.d VI0, i0, 1 //2
.align 3

.L24:
ld.d t1, X, 0 * SIZE
add.d X, X, INCX
ld.d t2, X, 0 * SIZE
add.d X, X, INCX
vinsgr2vr.d VX0, t1, 0
vinsgr2vr.d VX0, t2, 1
vadd.d VI1, VI1, VINC4
ld.d t1, X, 0 * SIZE
add.d X, X, INCX
ld.d t2, X, 0 * SIZE
add.d X, X, INCX
vinsgr2vr.d VX1, t1, 0
vinsgr2vr.d VX1, t2, 1
vadd.d VI2, VI1, VINC2
vfcmp.clt.d VT0, VX1, VX0
vbitsel.v x1, VX0, VX1, VT0
vbitsel.v x2, VI1, VI2, VT0
ld.d t1, X, 0 * SIZE
add.d X, X, INCX
ld.d t2, X, 0 * SIZE
add.d X, X, INCX
vinsgr2vr.d VX0, t1, 0
vinsgr2vr.d VX0, t2, 1
vadd.d VI1, VI2, VINC2
ld.d t1, X, 0 * SIZE
add.d X, X, INCX
ld.d t2, X, 0 * SIZE
add.d X, X, INCX
vinsgr2vr.d VX1, t1, 0
vinsgr2vr.d VX1, t2, 1
vadd.d VI2, VI1, VINC2
vfcmp.clt.d VT0, VX1, VX0
vbitsel.v x3, VX0, VX1, VT0
vbitsel.v x4, VI1, VI2, VT0
vfcmp.clt.d VT0, x3, x1
vbitsel.v x1, x1, x3, VT0
vbitsel.v x2, x2, x4, VT0
vfcmp.clt.d VT0, x1, VM0
addi.d I, I, -1
vbitsel.v VM0, VM0, x1, VT0
vbitsel.v VI0, VI0, x2, VT0
blt $r0, I, .L24
.align 3

.L25:
vreplvei.d VI1, VI0, 0
vreplvei.d VI2, VI0, 1
vreplvei.d x1, VM0, 0
vreplvei.d x2, VM0, 1
li.d TEMP, 1 //处理尾数相等时取最小序号
movgr2fr.d $f17, TEMP
ffint.d.l $f17, $f17
vfcmp.ceq.d VT0, x2, x1
fcmp.ceq.d $fcc0, $f23, $f17
bceqz $fcc0, .L26
vfcmp.clt.d VT0, VI1, VI0
vbitsel.v VI0, VI0, VI1, VT0
b .L27
.align 3

.L26:
vfcmp.clt.d VT0, x2, x1
vbitsel.v VM0, x1, x2, VT0
vbitsel.v VI0, VI1, VI2, VT0
.align 3

.L27:
movfr2gr.d i0, $f20
.align 3

.L21: //N<8
andi I, N, 7
bge $r0, I, .L999
srai.d i1, N, 3
slli.d i1, i1, 3
addi.d i1, i1, 1 //current index
movgr2fr.d $f21, i1
movgr2fr.d $f20, i0
.align 3

.L22:
fld.d $f9, X, 0
addi.d I, I, -1
fcmp.clt.d $fcc0, $f9, $f15
add.d X, X, INCX
fsel $f15, $f15, $f9, $fcc0
fsel $f20, $f20, $f21, $fcc0
addi.d i1, i1, 1
movgr2fr.d $f21, i1
blt $r0, I, .L22
movfr2gr.d i0, $f20
.align 3

.L999:
move $r4, $r17
jirl $r0, $r1, 0x0
.align 3

EPILOGUE

+ 374
- 0
kernel/loongarch64/ismin_lasx.S View File

@@ -0,0 +1,374 @@
#define ASSEMBLER

#include "common.h"

#define N $r4
#define X $r5
#define INCX $r6
#define I $r12
#define t1 $r13
#define t2 $r15
#define t3 $r18
#define t4 $r16
#define i0 $r17
#define i1 $r14
#define TEMP $r19
#define x1 $xr9
#define x2 $xr10
#define x3 $xr11
#define x4 $xr12
#define VX0 $xr13
#define VX1 $xr14
#define VM0 $xr15
#define VM1 $xr16
#define VINC4 $xr17
#define VINC8 $xr18
#define VI0 $xr20
#define VI1 $xr21
#define VI2 $xr22
#define VI3 $xr8
#define VI4 $xr19
#define VT0 $xr23

PROLOGUE
li.d i0, 0
bge $r0, N, .L999
bge $r0, INCX, .L999
li.d TEMP, 1
slli.d TEMP, TEMP, BASE_SHIFT
slli.d INCX, INCX, BASE_SHIFT
bne INCX, TEMP, .L20
xvld VM0, X, 0
addi.w i0, i0, 1
srai.d I, N, 3
bge $r0, I, .L21
slli.w i0, i0, 3 //8
xvreplgr2vr.w VINC8, i0
addi.w i0, i0, -15
xvinsgr2vr.w VI1, i0, 0 //initialize the index value for vectorization
addi.w i0, i0, 1
xvinsgr2vr.w VI1, i0, 1
addi.w i0, i0, 1
xvinsgr2vr.w VI1, i0, 2
addi.w i0, i0, 1
xvinsgr2vr.w VI1, i0, 3
addi.w i0, i0, 1
xvinsgr2vr.w VI1, i0, 4
addi.w i0, i0, 1
xvinsgr2vr.w VI1, i0, 5
addi.w i0, i0, 1
xvinsgr2vr.w VI1, i0, 6
addi.w i0, i0, 1
xvinsgr2vr.w VI1, i0, 7
addi.w i0, i0, 1
xvinsgr2vr.w VI0, i0, 0 //1
addi.w i0, i0, 1
xvinsgr2vr.w VI0, i0, 1 //2
addi.w i0, i0, 1
xvinsgr2vr.w VI0, i0, 2 //3
addi.w i0, i0, 1
xvinsgr2vr.w VI0, i0, 3 //4
addi.w i0, i0, 1
xvinsgr2vr.w VI0, i0, 4 //5
addi.w i0, i0, 1
xvinsgr2vr.w VI0, i0, 5 //6
addi.w i0, i0, 1
xvinsgr2vr.w VI0, i0, 6 //7
addi.w i0, i0, 1
xvinsgr2vr.w VI0, i0, 7 //8
.align 3

.L10:
xvld VX0, X, 0 * SIZE
xvadd.w VI1, VI1, VINC8
xvfcmp.clt.s VT0, VX0, VM0
addi.d I, I, -1
xvbitsel.v VM0, VM0, VX0, VT0
xvbitsel.v VI0, VI0, VI1, VT0
addi.d X, X, 8 * SIZE
blt $r0, I, .L10
.align 3

.L15:
xvxor.v VX0, VX0, VX0
xvor.v VX0, VI0, VX0
xvxor.v VX1, VX1, VX1
xvor.v VX1, VM0, VX1
xvpickve.w VI1, VI0, 0
xvpickve.w VI2, VI0, 1
xvpickve.w VI3, VI0, 2
xvpickve.w VI4, VI0, 3
xvpickve.w x1, VM0, 0
xvpickve.w x2, VM0, 1
xvpickve.w x3, VM0, 2
xvpickve.w x4, VM0, 3
xvfcmp.clt.s VT0, x2, x1
xvbitsel.v VM1, x1, x2, VT0
xvbitsel.v VINC4, VI1, VI2, VT0
xvfcmp.clt.s VT0, x4, x3
xvbitsel.v VM0, x3, x4, VT0
xvbitsel.v VINC8, VI3, VI4, VT0
xvfcmp.clt.s VT0, VM1, VM0
xvbitsel.v VM0, VM0, VM1, VT0
xvbitsel.v VI0, VINC8, VINC4, VT0
li.d TEMP, 1 //处理尾数相等时取最小序号
movgr2fr.w $f17, TEMP
ffint.s.w $f17, $f17
xvfcmp.ceq.s VT0, x1, VM0
fcmp.ceq.s $fcc0, $f23, $f17
bceqz $fcc0, .L26
xvfcmp.clt.s VT0, VI1, VI0
xvbitsel.v VI0, VI0, VI1, VT0
b .L26
.align 3

.L20: // INCX!=1
move TEMP, X
addi.w i0, i0, 1
ld.w t1, TEMP, 0 * SIZE
add.d TEMP, TEMP, INCX
srai.d I, N, 3
bge $r0, I, .L21
ld.w t2, TEMP, 0 * SIZE
add.d TEMP, TEMP, INCX
ld.w t3, TEMP, 0 * SIZE
add.d TEMP, TEMP, INCX
ld.w t4, TEMP, 0 * SIZE
add.d TEMP, TEMP, INCX
xvinsgr2vr.w VM0, t1, 0
xvinsgr2vr.w VM0, t2, 1
xvinsgr2vr.w VM0, t3, 2
xvinsgr2vr.w VM0, t4, 3
ld.w t1, TEMP, 0 * SIZE
add.d TEMP, TEMP, INCX
ld.w t2, TEMP, 0 * SIZE
add.d TEMP, TEMP, INCX
ld.w t3, TEMP, 0 * SIZE
add.d TEMP, TEMP, INCX
ld.w t4, TEMP, 0 * SIZE
add.d TEMP, TEMP, INCX
xvinsgr2vr.w VM0, t1, 4
xvinsgr2vr.w VM0, t2, 5
xvinsgr2vr.w VM0, t3, 6
xvinsgr2vr.w VM0, t4, 7
slli.w i0, i0, 3 //8
xvreplgr2vr.w VINC8, i0
addi.w i0, i0, -15
xvinsgr2vr.w VI1, i0, 0 //initialize the index value for vectorization
addi.w i0, i0, 1
xvinsgr2vr.w VI1, i0, 1
addi.w i0, i0, 1
xvinsgr2vr.w VI1, i0, 2
addi.w i0, i0, 1
xvinsgr2vr.w VI1, i0, 3
addi.w i0, i0, 1
xvinsgr2vr.w VI1, i0, 4
addi.w i0, i0, 1
xvinsgr2vr.w VI1, i0, 5
addi.w i0, i0, 1
xvinsgr2vr.w VI1, i0, 6
addi.w i0, i0, 1
xvinsgr2vr.w VI1, i0, 7
addi.w i0, i0, 1
xvinsgr2vr.w VI0, i0, 0 //1
addi.w i0, i0, 1
xvinsgr2vr.w VI0, i0, 1 //2
addi.w i0, i0, 1
xvinsgr2vr.w VI0, i0, 2 //3
addi.w i0, i0, 1
xvinsgr2vr.w VI0, i0, 3 //4
addi.w i0, i0, 1
xvinsgr2vr.w VI0, i0, 4 //5
addi.w i0, i0, 1
xvinsgr2vr.w VI0, i0, 5 //6
addi.w i0, i0, 1
xvinsgr2vr.w VI0, i0, 6 //7
addi.w i0, i0, 1
xvinsgr2vr.w VI0, i0, 7 //8
.align 3

.L24:
ld.w t1, X, 0 * SIZE
add.d X, X, INCX
ld.w t2, X, 0 * SIZE
add.d X, X, INCX
ld.w t3, X, 0 * SIZE
add.d X, X, INCX
ld.w t4, X, 0 * SIZE
add.d X, X, INCX
xvinsgr2vr.w VX0, t1, 0
xvinsgr2vr.w VX0, t2, 1
xvinsgr2vr.w VX0, t3, 2
xvinsgr2vr.w VX0, t4, 3
ld.w t1, X, 0 * SIZE
add.d X, X, INCX
ld.w t2, X, 0 * SIZE
add.d X, X, INCX
ld.w t3, X, 0 * SIZE
add.d X, X, INCX
ld.w t4, X, 0 * SIZE
add.d X, X, INCX
xvinsgr2vr.w VX0, t1, 4
xvinsgr2vr.w VX0, t2, 5
xvinsgr2vr.w VX0, t3, 6
xvinsgr2vr.w VX0, t4, 7
xvadd.w VI1, VI1, VINC8
xvfcmp.clt.s VT0, VX0, VM0
addi.d I, I, -1
xvbitsel.v VM0, VM0, VX0, VT0
xvbitsel.v VI0, VI0, VI1, VT0
blt $r0, I, .L24
.align 3

.L25:
xvxor.v VX0, VX0, VX0
xvor.v VX0, VI0, VX0
xvxor.v VX1, VX1, VX1
xvor.v VX1, VM0, VX1
xvpickve.w VI1, VI0, 0
xvpickve.w VI2, VI0, 1
xvpickve.w VI3, VI0, 2
xvpickve.w VI4, VI0, 3
xvpickve.w x1, VM0, 0
xvpickve.w x2, VM0, 1
xvpickve.w x3, VM0, 2
xvpickve.w x4, VM0, 3
xvfcmp.clt.s VT0, x2, x1
xvbitsel.v VM1, x1, x2, VT0
xvbitsel.v VINC4, VI1, VI2, VT0
xvfcmp.clt.s VT0, x4, x3
xvbitsel.v VM0, x3, x4, VT0
xvbitsel.v VINC8, VI3, VI4, VT0
xvfcmp.clt.s VT0, VM1, VM0
xvbitsel.v VM0, VM0, VM1, VT0
xvbitsel.v VI0, VINC8, VINC4, VT0
li.d TEMP, 1 //处理尾数相等时取最小序号
movgr2fr.w $f17, TEMP
ffint.s.w $f17, $f17
xvfcmp.ceq.s VT0, VM0, x1
fcmp.ceq.s $fcc0, $f23, $f17
bceqz $fcc0, .L26
xvfcmp.clt.s VT0, VI1, VI0
xvbitsel.v VI0, VI0, VI1, VT0
.align 3

.L26:
xvfcmp.ceq.s VT0, VM0, x2
fcmp.ceq.s $fcc0, $f23, $f17
bceqz $fcc0, .L27
xvfcmp.clt.s VT0, VI2, VI0
xvbitsel.v VI0, VI0, VI2, VT0
.align 3

.L27:
xvfcmp.ceq.s VT0, VM0, x3
fcmp.ceq.s $fcc0, $f23, $f17
bceqz $fcc0, .L28
xvfcmp.clt.s VT0, VI3, VI0
xvbitsel.v VI0, VI0, VI3, VT0
.align 3

.L28:
xvfcmp.ceq.s VT0, VM0, x4
fcmp.ceq.s $fcc0, $f23, $f17
bceqz $fcc0, .L29
xvfcmp.clt.s VT0, VI4, VI0
xvbitsel.v VI0, VI0, VI4, VT0
.align 3

.L29:
fmov.s $f16, $f20
.align 3

.L252:
xvxor.v VI0, VI0, VI0
xvor.v VI0, VI0, VX0
fmov.s $f13, $f15
xvxor.v VM0, VM0, VM0
xvor.v VM0, VM0, VX1
xvpickve.w VI1, VI0, 4
xvpickve.w VI2, VI0, 5
xvpickve.w VI3, VI0, 6
xvpickve.w VI4, VI0, 7
xvpickve.w x1, VM0, 4
xvpickve.w x2, VM0, 5
xvpickve.w x3, VM0, 6
xvpickve.w x4, VM0, 7
xvfcmp.clt.s VT0, x2, x1
xvbitsel.v x1, x1, x2, VT0
xvbitsel.v VINC4, VI1, VI2, VT0
xvfcmp.clt.s VT0, x4, x3
xvbitsel.v VM0, x3, x4, VT0
xvbitsel.v VINC8, VI3, VI4, VT0
xvfcmp.clt.s VT0, x1, VM0
xvbitsel.v VM0, VM0, x1, VT0
xvbitsel.v VI0, VINC8, VINC4, VT0
li.d TEMP, 1 //处理尾数相等时取最小序号
movgr2fr.w $f17, TEMP
ffint.s.w $f17, $f17
xvfcmp.ceq.s VT0, VM0, x1
fcmp.ceq.s $fcc0, $f23, $f17
bceqz $fcc0, .L262
xvfcmp.clt.s VT0, VI1, VI0
xvbitsel.v VI0, VI0, VI1, VT0
.align 3

.L262:
xvfcmp.ceq.s VT0, VM0, x2
fcmp.ceq.s $fcc0, $f23, $f17
bceqz $fcc0, .L272
xvfcmp.clt.s VT0, VI2, VI0
xvbitsel.v VI0, VI0, VI2, VT0
.align 3

.L272:
xvfcmp.ceq.s VT0, VM0, x3
fcmp.ceq.s $fcc0, $f23, $f17
bceqz $fcc0, .L282
xvfcmp.clt.s VT0, VI3, VI0
xvbitsel.v VI0, VI0, VI3, VT0
.align 3

.L282:
xvfcmp.ceq.s VT0, VM0, x4
fcmp.ceq.s $fcc0, $f23, $f17
bceqz $fcc0, .L292
xvfcmp.clt.s VT0, VI4, VI0
xvbitsel.v VI0, VI0, VI4, VT0
.align 3

.L292:
fcmp.clt.s $fcc0, $f13, $f15
fsel $f15, $f15, $f13, $fcc0
fsel $f20, $f20, $f16, $fcc0
movfr2gr.s i0, $f20

.L21: //N<8
andi I, N, 7
bge $r0, I, .L999
srai.d i1, N, 3
slli.d i1, i1, 3
addi.d i1, i1, 1 //current index
movgr2fr.d $f21, i1
movgr2fr.d $f20, i0
.align 3

.L22:
fld.d $f9, X, 0
addi.d I, I, -1
fcmp.clt.s $fcc0, $f9, $f15
fsel $f15, $f15, $f9, $fcc0
fsel $f20, $f20, $f21, $fcc0
addi.d i1, i1, 1
movgr2fr.d $f21, i1
add.d X, X, INCX
blt $r0, I, .L22
movfr2gr.s i0, $f20
.align 3

.L999:
move $r4, $r17
jirl $r0, $r1, 0x0
.align 3

EPILOGUE

+ 271
- 0
kernel/loongarch64/ismin_lsx.S View File

@@ -0,0 +1,271 @@
#define ASSEMBLER

#include "common.h"

#define N $r4
#define X $r5
#define INCX $r6
#define I $r12
#define t1 $r13
#define t2 $r15
#define t3 $r18
#define t4 $r16
#define i0 $r17
#define i1 $r14
#define TEMP $r19
#define x1 $vr9
#define x2 $vr10
#define x3 $vr11
#define x4 $vr12
#define VX0 $vr13
#define VX1 $vr14
#define VM0 $vr15
#define VM1 $vr16
#define VINC4 $vr17
#define VINC8 $vr18
#define VI0 $vr20
#define VI1 $vr21
#define VI2 $vr22
#define VI3 $vr8
#define VI4 $vr19
#define VT0 $vr23

PROLOGUE
li.d i0, 0
bge $r0, N, .L999
bge $r0, INCX, .L999
li.d TEMP, 1
slli.d TEMP, TEMP, BASE_SHIFT
slli.d INCX, INCX, BASE_SHIFT
bne INCX, TEMP, .L20
vld VM0, X, 0
addi.w i0, i0, 1
srai.d I, N, 3
bge $r0, I, .L21
slli.w i0, i0, 2 //4
vreplgr2vr.w VINC4, i0
slli.w i0, i0, 1 //8
vreplgr2vr.w VINC8, i0
addi.w i0, i0, -15
vinsgr2vr.w VI1, i0, 0 //initialize the index value for vectorization
addi.w i0, i0, 1
vinsgr2vr.w VI1, i0, 1
addi.w i0, i0, 1
vinsgr2vr.w VI1, i0, 2
addi.w i0, i0, 1
vinsgr2vr.w VI1, i0, 3
addi.w i0, i0, 5
vinsgr2vr.w VI0, i0, 0 //1
addi.w i0, i0, 1
vinsgr2vr.w VI0, i0, 1 //2
addi.w i0, i0, 1
vinsgr2vr.w VI0, i0, 2 //3
addi.w i0, i0, 1
vinsgr2vr.w VI0, i0, 3 //4
.align 3

.L10:
vld VX0, X, 0 * SIZE
vadd.w VI1, VI1, VINC8
vld VX1, X, 4 * SIZE
vadd.w VI2, VI1, VINC4
vfcmp.clt.s VT0, VX1, VX0
addi.d I, I, -1
vbitsel.v VM1, VX0, VX1, VT0
vbitsel.v VI2, VI1, VI2, VT0
vfcmp.clt.s VT0, VM1, VM0
addi.d X, X, 8 * SIZE
vbitsel.v VM0, VM0, VM1, VT0
vbitsel.v VI0, VI0, VI2, VT0
blt $r0, I, .L10
.align 3

.L15:
vreplvei.w VI1, VI0, 0
vreplvei.w VI2, VI0, 1
vreplvei.w VI3, VI0, 2
vreplvei.w VI4, VI0, 3
vreplvei.w x1, VM0, 0
vreplvei.w x2, VM0, 1
vreplvei.w x3, VM0, 2
vreplvei.w x4, VM0, 3
vfcmp.clt.s VT0, x2, x1
vbitsel.v VM1, x1, x2, VT0
vbitsel.v VINC4, VI1, VI2, VT0
vfcmp.clt.s VT0, x4, x3
vbitsel.v VM0, x3, x4, VT0
vbitsel.v VINC8, VI3, VI4, VT0
vfcmp.clt.s VT0, VM1, VM0
vbitsel.v VM0, VM0, VM1, VT0
vbitsel.v VI0, VINC8, VINC4, VT0
li.d TEMP, 1 //处理尾数相等时取最小序号
movgr2fr.w $f17, TEMP
ffint.s.w $f17, $f17
vfcmp.ceq.s VT0, x1, VM0
fcmp.ceq.s $fcc0, $f23, $f17
bceqz $fcc0, .L26
vfcmp.clt.s VT0, VI1, VI0
vbitsel.v VI0, VI0, VI1, VT0
b .L26
.align 3

.L20: // INCX!=1
move TEMP, X
addi.w i0, i0, 1
ld.w t1, TEMP, 0 * SIZE
add.d TEMP, TEMP, INCX
vinsgr2vr.w VM0, t1, 0
srai.d I, N, 3
bge $r0, I, .L21
ld.w t2, TEMP, 0 * SIZE
add.d TEMP, TEMP, INCX
ld.w t3, TEMP, 0 * SIZE
add.d TEMP, TEMP, INCX
ld.w t4, TEMP, 0 * SIZE
add.d TEMP, TEMP, INCX
vinsgr2vr.w VM0, t2, 1
vinsgr2vr.w VM0, t3, 2
vinsgr2vr.w VM0, t4, 3
slli.w i0, i0, 2 //4
vreplgr2vr.w VINC4, i0
slli.w i0, i0, 1 //8
vreplgr2vr.w VINC8, i0
addi.w i0, i0, -15
vinsgr2vr.w VI1, i0, 0 //initialize the index value for vectorization
addi.w i0, i0, 1
vinsgr2vr.w VI1, i0, 1
addi.w i0, i0, 1
vinsgr2vr.w VI1, i0, 2
addi.w i0, i0, 1
vinsgr2vr.w VI1, i0, 3
addi.w i0, i0, 5
vinsgr2vr.w VI0, i0, 0 //1
addi.w i0, i0, 1
vinsgr2vr.w VI0, i0, 1 //2
addi.w i0, i0, 1
vinsgr2vr.w VI0, i0, 2 //3
addi.w i0, i0, 1
vinsgr2vr.w VI0, i0, 3 //4
.align 3

.L24:
ld.w t1, X, 0 * SIZE
add.d X, X, INCX
ld.w t2, X, 0 * SIZE
add.d X, X, INCX
ld.w t3, X, 0 * SIZE
add.d X, X, INCX
ld.w t4, X, 0 * SIZE
add.d X, X, INCX
vinsgr2vr.w VX0, t1, 0
vinsgr2vr.w VX0, t2, 1
vinsgr2vr.w VX0, t3, 2
vinsgr2vr.w VX0, t4, 3
vadd.w VI1, VI1, VINC8
ld.w t1, X, 0 * SIZE
add.d X, X, INCX
ld.w t2, X, 0 * SIZE
add.d X, X, INCX
ld.w t3, X, 0 * SIZE
add.d X, X, INCX
ld.w t4, X, 0 * SIZE
add.d X, X, INCX
vinsgr2vr.w VX1, t1, 0
vinsgr2vr.w VX1, t2, 1
vinsgr2vr.w VX1, t3, 2
vinsgr2vr.w VX1, t4, 3
vadd.w VI2, VI1, VINC4
vfcmp.clt.s VT0, VX1, VX0
addi.d I, I, -1
vbitsel.v VM1, VX0, VX1, VT0
vbitsel.v VI2, VI1, VI2, VT0
vfcmp.clt.s VT0, VM1, VM0
vbitsel.v VM0, VM0, VM1, VT0
vbitsel.v VI0, VI0, VI2, VT0
blt $r0, I, .L24
.align 3

.L25:
vreplvei.w VI1, VI0, 0
vreplvei.w VI2, VI0, 1
vreplvei.w VI3, VI0, 2
vreplvei.w VI4, VI0, 3
vreplvei.w x1, VM0, 0
vreplvei.w x2, VM0, 1
vreplvei.w x3, VM0, 2
vreplvei.w x4, VM0, 3
vfcmp.clt.s VT0, x2, x1
vbitsel.v VM1, x1, x2, VT0
vbitsel.v VINC4, VI1, VI2, VT0
vfcmp.clt.s VT0, x4, x3
vbitsel.v VM0, x3, x4, VT0
vbitsel.v VINC8, VI3, VI4, VT0
vfcmp.clt.s VT0, VM1, VM0
vbitsel.v VM0, VM0, VM1, VT0
vbitsel.v VI0, VINC8, VINC4, VT0
li.d TEMP, 1 //处理尾数相等时取最小序号
movgr2fr.w $f17, TEMP
ffint.s.w $f17, $f17
vfcmp.ceq.s VT0, x1, VM0
fcmp.ceq.s $fcc0, $f23, $f17
bceqz $fcc0, .L26
vfcmp.clt.s VT0, VI1, VI0
vbitsel.v VI0, VI0, VI1, VT0
.align 3

.L26:
vfcmp.ceq.s VT0, x2, VM0
fcmp.ceq.s $fcc0, $f23, $f17
bceqz $fcc0, .L27
vfcmp.clt.s VT0, VI2, VI0
vbitsel.v VI0, VI0, VI2, VT0
.align 3

.L27:
vfcmp.ceq.s VT0, x3, VM0
fcmp.ceq.s $fcc0, $f23, $f17
bceqz $fcc0, .L28
vfcmp.clt.s VT0, VI3, VI0
vbitsel.v VI0, VI0, VI3, VT0
.align 3

.L28:
vfcmp.ceq.s VT0, x4, VM0
fcmp.ceq.s $fcc0, $f23, $f17
bceqz $fcc0, .L29
vfcmp.clt.s VT0, VI4, VI0
vbitsel.v VI0, VI0, VI4, VT0
.align 3

.L29:
movfr2gr.s i0, $f20

.L21: //N<8
andi I, N, 7
bge $r0, I, .L999
srai.d i1, N, 3
slli.d i1, i1, 3
addi.d i1, i1, 1 //current index
movgr2fr.d $f21, i1
movgr2fr.d $f20, i0
.align 3

.L22:
fld.d $f9, X, 0
fcmp.clt.s $fcc0, $f9, $f15
fsel $f15, $f15, $f9, $fcc0
fsel $f20, $f20, $f21, $fcc0
addi.d I, I, -1
addi.d i1, i1, 1
add.d X, X, INCX
movgr2fr.d $f21, i1
blt $r0, I, .L22
movfr2gr.s i0, $f20
.align 3

.L999:
move $r4, $r17
jirl $r0, $r1, 0x0
.align 3

EPILOGUE

Write Preview
Loading…
Cancel
Save