| @@ -232,6 +232,8 @@ | |||||
| #define CGEADD_K cgeadd_k | #define CGEADD_K cgeadd_k | ||||
| #define CGEMM_SMALL_MATRIX_PERMIT cgemm_small_matrix_permit | |||||
| #define CGEMM_SMALL_KERNEL_NN cgemm_small_kernel_nn | #define CGEMM_SMALL_KERNEL_NN cgemm_small_kernel_nn | ||||
| #define CGEMM_SMALL_KERNEL_NT cgemm_small_kernel_nt | #define CGEMM_SMALL_KERNEL_NT cgemm_small_kernel_nt | ||||
| #define CGEMM_SMALL_KERNEL_NR cgemm_small_kernel_nr | #define CGEMM_SMALL_KERNEL_NR cgemm_small_kernel_nr | ||||
| @@ -157,6 +157,7 @@ | |||||
| #define DIMATCOPY_K_RT dimatcopy_k_rt | #define DIMATCOPY_K_RT dimatcopy_k_rt | ||||
| #define DGEADD_K dgeadd_k | #define DGEADD_K dgeadd_k | ||||
| #define DGEMM_SMALL_MATRIX_PERMIT dgemm_small_matrix_permit | |||||
| #define DGEMM_SMALL_KERNEL_NN dgemm_small_kernel_nn | #define DGEMM_SMALL_KERNEL_NN dgemm_small_kernel_nn | ||||
| #define DGEMM_SMALL_KERNEL_NT dgemm_small_kernel_nt | #define DGEMM_SMALL_KERNEL_NT dgemm_small_kernel_nt | ||||
| @@ -516,11 +516,15 @@ int qgemm_kernel(BLASLONG, BLASLONG, BLASLONG, xdouble, xdouble *, xdouble *, xd | |||||
| #endif | #endif | ||||
| #ifdef SMALL_MATRIX_OPT | #ifdef SMALL_MATRIX_OPT | ||||
| int sgemm_small_matrix_permit(int transa, int transb, BLASLONG m, BLASLONG n, BLASLONG k, float alpha, float beta); | |||||
| int sgemm_small_kernel_nn(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha, float * B, BLASLONG ldb, float beta, float * C, BLASLONG ldc); | int sgemm_small_kernel_nn(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha, float * B, BLASLONG ldb, float beta, float * C, BLASLONG ldc); | ||||
| int sgemm_small_kernel_nt(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha, float * B, BLASLONG ldb, float beta, float * C, BLASLONG ldc); | int sgemm_small_kernel_nt(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha, float * B, BLASLONG ldb, float beta, float * C, BLASLONG ldc); | ||||
| int sgemm_small_kernel_tn(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha, float * B, BLASLONG ldb, float beta, float * C, BLASLONG ldc); | int sgemm_small_kernel_tn(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha, float * B, BLASLONG ldb, float beta, float * C, BLASLONG ldc); | ||||
| int sgemm_small_kernel_tt(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha, float * B, BLASLONG ldb, float beta, float * C, BLASLONG ldc); | int sgemm_small_kernel_tt(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha, float * B, BLASLONG ldb, float beta, float * C, BLASLONG ldc); | ||||
| int dgemm_small_matrix_permit(int transa, int transb, BLASLONG m, BLASLONG n, BLASLONG k, double alpha, double beta); | |||||
| int dgemm_small_kernel_nn(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha, double * B, BLASLONG ldb, double beta, double * C, BLASLONG ldc); | int dgemm_small_kernel_nn(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha, double * B, BLASLONG ldb, double beta, double * C, BLASLONG ldc); | ||||
| int dgemm_small_kernel_nt(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha, double * B, BLASLONG ldb, double beta, double * C, BLASLONG ldc); | int dgemm_small_kernel_nt(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha, double * B, BLASLONG ldb, double beta, double * C, BLASLONG ldc); | ||||
| int dgemm_small_kernel_tn(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha, double * B, BLASLONG ldb, double beta, double * C, BLASLONG ldc); | int dgemm_small_kernel_tn(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha, double * B, BLASLONG ldb, double beta, double * C, BLASLONG ldc); | ||||
| @@ -536,6 +540,8 @@ int dgemm_small_kernel_b0_nt(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLA | |||||
| int dgemm_small_kernel_b0_tn(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha, double * B, BLASLONG ldb, double * C, BLASLONG ldc); | int dgemm_small_kernel_b0_tn(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha, double * B, BLASLONG ldb, double * C, BLASLONG ldc); | ||||
| int dgemm_small_kernel_b0_tt(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha, double * B, BLASLONG ldb, double * C, BLASLONG ldc); | int dgemm_small_kernel_b0_tt(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha, double * B, BLASLONG ldb, double * C, BLASLONG ldc); | ||||
| int cgemm_small_matrix_permit(int transa, int transb, BLASLONG m, BLASLONG n, BLASLONG k, float alpha0, float alpha1, float beta0, float beta1); | |||||
| int cgemm_small_kernel_nn(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc); | int cgemm_small_kernel_nn(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc); | ||||
| int cgemm_small_kernel_nt(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc); | int cgemm_small_kernel_nt(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc); | ||||
| int cgemm_small_kernel_nr(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc); | int cgemm_small_kernel_nr(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc); | ||||
| @@ -556,6 +562,8 @@ int cgemm_small_kernel_ct(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLON | |||||
| int cgemm_small_kernel_cr(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc); | int cgemm_small_kernel_cr(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc); | ||||
| int cgemm_small_kernel_cc(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc); | int cgemm_small_kernel_cc(BLASLONG m, BLASLONG n, BLASLONG k, float * A, BLASLONG lda, float alpha0, float alpha1, float * B, BLASLONG ldb, float beta0, float beta1, float * C, BLASLONG ldc); | ||||
| int zgemm_small_matrix_permit(int transa, int transb, BLASLONG m, BLASLONG n, BLASLONG k, double alpha0, double alpha1, double beta0, double beta1); | |||||
| int zgemm_small_kernel_nn(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc); | int zgemm_small_kernel_nn(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc); | ||||
| int zgemm_small_kernel_nt(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc); | int zgemm_small_kernel_nt(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc); | ||||
| int zgemm_small_kernel_nr(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc); | int zgemm_small_kernel_nr(BLASLONG m, BLASLONG n, BLASLONG k, double * A, BLASLONG lda, double alpha0, double alpha1, double * B, BLASLONG ldb, double beta0, double beta1, double * C, BLASLONG ldc); | ||||
| @@ -644,6 +644,8 @@ | |||||
| #define GEADD_K DGEADD_K | #define GEADD_K DGEADD_K | ||||
| #define GEMM_SMALL_MATRIX_PERMIT DGEMM_SMALL_MATRIX_PERMIT | |||||
| #define GEMM_SMALL_KERNEL_NN DGEMM_SMALL_KERNEL_NN | #define GEMM_SMALL_KERNEL_NN DGEMM_SMALL_KERNEL_NN | ||||
| #define GEMM_SMALL_KERNEL_NT DGEMM_SMALL_KERNEL_NT | #define GEMM_SMALL_KERNEL_NT DGEMM_SMALL_KERNEL_NT | ||||
| #define GEMM_SMALL_KERNEL_TN DGEMM_SMALL_KERNEL_TN | #define GEMM_SMALL_KERNEL_TN DGEMM_SMALL_KERNEL_TN | ||||
| @@ -940,6 +942,8 @@ | |||||
| #define GEADD_K SGEADD_K | #define GEADD_K SGEADD_K | ||||
| #define GEMM_SMALL_MATRIX_PERMIT SGEMM_SMALL_MATRIX_PERMIT | |||||
| #define GEMM_SMALL_KERNEL_NN SGEMM_SMALL_KERNEL_NN | #define GEMM_SMALL_KERNEL_NN SGEMM_SMALL_KERNEL_NN | ||||
| #define GEMM_SMALL_KERNEL_NT SGEMM_SMALL_KERNEL_NT | #define GEMM_SMALL_KERNEL_NT SGEMM_SMALL_KERNEL_NT | ||||
| #define GEMM_SMALL_KERNEL_TN SGEMM_SMALL_KERNEL_TN | #define GEMM_SMALL_KERNEL_TN SGEMM_SMALL_KERNEL_TN | ||||
| @@ -1256,6 +1260,8 @@ | |||||
| #define GEADD_K SGEADD_K | #define GEADD_K SGEADD_K | ||||
| #define GEMM_SMALL_MATRIX_PERMIT SGEMM_SMALL_MATRIX_PERMIT | |||||
| #define GEMM_SMALL_KERNEL_NN SGEMM_SMALL_KERNEL_NN | #define GEMM_SMALL_KERNEL_NN SGEMM_SMALL_KERNEL_NN | ||||
| #define GEMM_SMALL_KERNEL_NT SGEMM_SMALL_KERNEL_NT | #define GEMM_SMALL_KERNEL_NT SGEMM_SMALL_KERNEL_NT | ||||
| #define GEMM_SMALL_KERNEL_TN SGEMM_SMALL_KERNEL_TN | #define GEMM_SMALL_KERNEL_TN SGEMM_SMALL_KERNEL_TN | ||||
| @@ -2093,6 +2099,8 @@ | |||||
| #define GEADD_K ZGEADD_K | #define GEADD_K ZGEADD_K | ||||
| #define GEMM_SMALL_MATRIX_PERMIT ZGEMM_SMALL_MATRIX_PERMIT | |||||
| #define GEMM_SMALL_KERNEL_NN ZGEMM_SMALL_KERNEL_NN | #define GEMM_SMALL_KERNEL_NN ZGEMM_SMALL_KERNEL_NN | ||||
| #define GEMM_SMALL_KERNEL_NT ZGEMM_SMALL_KERNEL_NT | #define GEMM_SMALL_KERNEL_NT ZGEMM_SMALL_KERNEL_NT | ||||
| #define GEMM_SMALL_KERNEL_NR ZGEMM_SMALL_KERNEL_NR | #define GEMM_SMALL_KERNEL_NR ZGEMM_SMALL_KERNEL_NR | ||||
| @@ -2556,6 +2564,8 @@ | |||||
| #define GEADD_K CGEADD_K | #define GEADD_K CGEADD_K | ||||
| #define GEMM_SMALL_MATRIX_PERMIT CGEMM_SMALL_MATRIX_PERMIT | |||||
| #define GEMM_SMALL_KERNEL_NN CGEMM_SMALL_KERNEL_NN | #define GEMM_SMALL_KERNEL_NN CGEMM_SMALL_KERNEL_NN | ||||
| #define GEMM_SMALL_KERNEL_NT CGEMM_SMALL_KERNEL_NT | #define GEMM_SMALL_KERNEL_NT CGEMM_SMALL_KERNEL_NT | ||||
| #define GEMM_SMALL_KERNEL_NR CGEMM_SMALL_KERNEL_NR | #define GEMM_SMALL_KERNEL_NR CGEMM_SMALL_KERNEL_NR | ||||
| @@ -164,6 +164,8 @@ | |||||
| #define SGEADD_K sgeadd_k | #define SGEADD_K sgeadd_k | ||||
| #define SGEMM_SMALL_MATRIX_PERMIT sgemm_small_matrix_permit | |||||
| #define SGEMM_SMALL_KERNEL_NN sgemm_small_kernel_nn | #define SGEMM_SMALL_KERNEL_NN sgemm_small_kernel_nn | ||||
| #define SGEMM_SMALL_KERNEL_NT sgemm_small_kernel_nt | #define SGEMM_SMALL_KERNEL_NT sgemm_small_kernel_nt | ||||
| #define SGEMM_SMALL_KERNEL_TN sgemm_small_kernel_tn | #define SGEMM_SMALL_KERNEL_TN sgemm_small_kernel_tn | ||||
| @@ -232,6 +232,8 @@ | |||||
| #define ZGEADD_K zgeadd_k | #define ZGEADD_K zgeadd_k | ||||
| #define ZGEMM_SMALL_MATRIX_PERMIT zgemm_small_matrix_permit | |||||
| #define ZGEMM_SMALL_KERNEL_NN zgemm_small_kernel_nn | #define ZGEMM_SMALL_KERNEL_NN zgemm_small_kernel_nn | ||||
| #define ZGEMM_SMALL_KERNEL_NT zgemm_small_kernel_nt | #define ZGEMM_SMALL_KERNEL_NT zgemm_small_kernel_nt | ||||
| #define ZGEMM_SMALL_KERNEL_NR zgemm_small_kernel_nr | #define ZGEMM_SMALL_KERNEL_NR zgemm_small_kernel_nr | ||||
| @@ -464,25 +464,26 @@ void CNAME(enum CBLAS_ORDER order, enum CBLAS_TRANSPOSE TransA, enum CBLAS_TRANS | |||||
| #endif | #endif | ||||
| #ifdef SMALL_MATRIX_OPT | #ifdef SMALL_MATRIX_OPT | ||||
| //need to tune small matrices cases. | |||||
| if(MNK <= 100.0*100.0*100.0){ | |||||
| #if !defined(COMPLEX) | #if !defined(COMPLEX) | ||||
| if(GEMM_SMALL_MATRIX_PERMIT(transa, transb, args.m, args.n, args.k, *(FLOAT *)(args.alpha), *(FLOAT *)(args.beta))){ | |||||
| if(*(FLOAT *)(args.beta) == 0.0){ | if(*(FLOAT *)(args.beta) == 0.0){ | ||||
| (gemm_small_kernel_b0[(transb << 2) | transa])(args.m, args.n, args.k, args.a, args.lda, *(FLOAT *)(args.alpha), args.b, args.ldb, args.c, args.ldc); | (gemm_small_kernel_b0[(transb << 2) | transa])(args.m, args.n, args.k, args.a, args.lda, *(FLOAT *)(args.alpha), args.b, args.ldb, args.c, args.ldc); | ||||
| }else{ | }else{ | ||||
| (gemm_small_kernel[(transb << 2) | transa])(args.m, args.n, args.k, args.a, args.lda, *(FLOAT *)(args.alpha), args.b, args.ldb, *(FLOAT *)(args.beta), args.c, args.ldc); | (gemm_small_kernel[(transb << 2) | transa])(args.m, args.n, args.k, args.a, args.lda, *(FLOAT *)(args.alpha), args.b, args.ldb, *(FLOAT *)(args.beta), args.c, args.ldc); | ||||
| } | } | ||||
| return; | |||||
| } | |||||
| #else | #else | ||||
| if(GEMM_SMALL_MATRIX_PERMIT(transa, transb, args.m, args.n, args.k, alpha[0], alpha[1], beta[0], beta[1])){ | |||||
| if(beta[0] == 0.0 && beta[1] == 0.0){ | if(beta[0] == 0.0 && beta[1] == 0.0){ | ||||
| (zgemm_small_kernel_b0[(transb << 2) | transa])(args.m, args.n, args.k, args.a, args.lda, alpha[0], alpha[1], args.b, args.ldb, args.c, args.ldc); | (zgemm_small_kernel_b0[(transb << 2) | transa])(args.m, args.n, args.k, args.a, args.lda, alpha[0], alpha[1], args.b, args.ldb, args.c, args.ldc); | ||||
| }else{ | }else{ | ||||
| (zgemm_small_kernel[(transb << 2) | transa])(args.m, args.n, args.k, args.a, args.lda, alpha[0], alpha[1], args.b, args.ldb, beta[0], beta[1], args.c, args.ldc); | (zgemm_small_kernel[(transb << 2) | transa])(args.m, args.n, args.k, args.a, args.lda, alpha[0], alpha[1], args.b, args.ldb, beta[0], beta[1], args.c, args.ldc); | ||||
| } | } | ||||
| #endif | |||||
| return; | return; | ||||
| } | } | ||||
| #endif | #endif | ||||
| #endif | |||||
| buffer = (XFLOAT *)blas_memory_alloc(0); | buffer = (XFLOAT *)blas_memory_alloc(0); | ||||
| @@ -451,18 +451,21 @@ endif | |||||
| ifeq ($(SMALL_MATRIX_OPT), 1) | ifeq ($(SMALL_MATRIX_OPT), 1) | ||||
| SBLASOBJS += \ | SBLASOBJS += \ | ||||
| sgemm_small_matrix_permit$(TSUFFIX).$(SUFFIX) \ | |||||
| sgemm_small_kernel_nn$(TSUFFIX).$(SUFFIX) sgemm_small_kernel_nt$(TSUFFIX).$(SUFFIX) \ | sgemm_small_kernel_nn$(TSUFFIX).$(SUFFIX) sgemm_small_kernel_nt$(TSUFFIX).$(SUFFIX) \ | ||||
| sgemm_small_kernel_tn$(TSUFFIX).$(SUFFIX) sgemm_small_kernel_tt$(TSUFFIX).$(SUFFIX) \ | sgemm_small_kernel_tn$(TSUFFIX).$(SUFFIX) sgemm_small_kernel_tt$(TSUFFIX).$(SUFFIX) \ | ||||
| sgemm_small_kernel_b0_nn$(TSUFFIX).$(SUFFIX) sgemm_small_kernel_b0_nt$(TSUFFIX).$(SUFFIX) \ | sgemm_small_kernel_b0_nn$(TSUFFIX).$(SUFFIX) sgemm_small_kernel_b0_nt$(TSUFFIX).$(SUFFIX) \ | ||||
| sgemm_small_kernel_b0_tn$(TSUFFIX).$(SUFFIX) sgemm_small_kernel_b0_tt$(TSUFFIX).$(SUFFIX) | sgemm_small_kernel_b0_tn$(TSUFFIX).$(SUFFIX) sgemm_small_kernel_b0_tt$(TSUFFIX).$(SUFFIX) | ||||
| DBLASOBJS += \ | DBLASOBJS += \ | ||||
| dgemm_small_matrix_permit$(TSUFFIX).$(SUFFIX) \ | |||||
| dgemm_small_kernel_nn$(TSUFFIX).$(SUFFIX) dgemm_small_kernel_nt$(TSUFFIX).$(SUFFIX) \ | dgemm_small_kernel_nn$(TSUFFIX).$(SUFFIX) dgemm_small_kernel_nt$(TSUFFIX).$(SUFFIX) \ | ||||
| dgemm_small_kernel_tn$(TSUFFIX).$(SUFFIX) dgemm_small_kernel_tt$(TSUFFIX).$(SUFFIX) \ | dgemm_small_kernel_tn$(TSUFFIX).$(SUFFIX) dgemm_small_kernel_tt$(TSUFFIX).$(SUFFIX) \ | ||||
| dgemm_small_kernel_b0_nn$(TSUFFIX).$(SUFFIX) dgemm_small_kernel_b0_nt$(TSUFFIX).$(SUFFIX) \ | dgemm_small_kernel_b0_nn$(TSUFFIX).$(SUFFIX) dgemm_small_kernel_b0_nt$(TSUFFIX).$(SUFFIX) \ | ||||
| dgemm_small_kernel_b0_tn$(TSUFFIX).$(SUFFIX) dgemm_small_kernel_b0_tt$(TSUFFIX).$(SUFFIX) | dgemm_small_kernel_b0_tn$(TSUFFIX).$(SUFFIX) dgemm_small_kernel_b0_tt$(TSUFFIX).$(SUFFIX) | ||||
| CBLASOBJS += \ | CBLASOBJS += \ | ||||
| cgemm_small_matrix_permit$(TSUFFIX).$(SUFFIX) \ | |||||
| cgemm_small_kernel_nn$(TSUFFIX).$(SUFFIX) cgemm_small_kernel_nt$(TSUFFIX).$(SUFFIX) \ | cgemm_small_kernel_nn$(TSUFFIX).$(SUFFIX) cgemm_small_kernel_nt$(TSUFFIX).$(SUFFIX) \ | ||||
| cgemm_small_kernel_nr$(TSUFFIX).$(SUFFIX) cgemm_small_kernel_nc$(TSUFFIX).$(SUFFIX) \ | cgemm_small_kernel_nr$(TSUFFIX).$(SUFFIX) cgemm_small_kernel_nc$(TSUFFIX).$(SUFFIX) \ | ||||
| cgemm_small_kernel_tn$(TSUFFIX).$(SUFFIX) cgemm_small_kernel_tt$(TSUFFIX).$(SUFFIX) \ | cgemm_small_kernel_tn$(TSUFFIX).$(SUFFIX) cgemm_small_kernel_tt$(TSUFFIX).$(SUFFIX) \ | ||||
| @@ -481,6 +484,7 @@ CBLASOBJS += \ | |||||
| cgemm_small_kernel_b0_cr$(TSUFFIX).$(SUFFIX) cgemm_small_kernel_b0_cc$(TSUFFIX).$(SUFFIX) | cgemm_small_kernel_b0_cr$(TSUFFIX).$(SUFFIX) cgemm_small_kernel_b0_cc$(TSUFFIX).$(SUFFIX) | ||||
| ZBLASOBJS += \ | ZBLASOBJS += \ | ||||
| zgemm_small_matrix_permit$(TSUFFIX).$(SUFFIX) \ | |||||
| zgemm_small_kernel_nn$(TSUFFIX).$(SUFFIX) zgemm_small_kernel_nt$(TSUFFIX).$(SUFFIX) \ | zgemm_small_kernel_nn$(TSUFFIX).$(SUFFIX) zgemm_small_kernel_nt$(TSUFFIX).$(SUFFIX) \ | ||||
| zgemm_small_kernel_nr$(TSUFFIX).$(SUFFIX) zgemm_small_kernel_nc$(TSUFFIX).$(SUFFIX) \ | zgemm_small_kernel_nr$(TSUFFIX).$(SUFFIX) zgemm_small_kernel_nc$(TSUFFIX).$(SUFFIX) \ | ||||
| zgemm_small_kernel_tn$(TSUFFIX).$(SUFFIX) zgemm_small_kernel_tt$(TSUFFIX).$(SUFFIX) \ | zgemm_small_kernel_tn$(TSUFFIX).$(SUFFIX) zgemm_small_kernel_tt$(TSUFFIX).$(SUFFIX) \ | ||||
| @@ -4294,6 +4298,10 @@ $(KDIR)zgeadd_k$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEADD_K) | |||||
| ###### BLAS small matrix optimization ##### | ###### BLAS small matrix optimization ##### | ||||
| ifndef DGEMM_SMALL_M_PERMIT | |||||
| DGEMM_SMALL_M_PERMIT = ../generic/gemm_small_matrix_permit.c | |||||
| endif | |||||
| ifndef DGEMM_SMALL_K_NN | ifndef DGEMM_SMALL_K_NN | ||||
| DGEMM_SMALL_K_NN = ../generic/gemm_small_matrix_kernel_nn.c | DGEMM_SMALL_K_NN = ../generic/gemm_small_matrix_kernel_nn.c | ||||
| endif | endif | ||||
| @@ -4310,6 +4318,9 @@ ifndef DGEMM_SMALL_K_TT | |||||
| DGEMM_SMALL_K_TT = ../generic/gemm_small_matrix_kernel_tt.c | DGEMM_SMALL_K_TT = ../generic/gemm_small_matrix_kernel_tt.c | ||||
| endif | endif | ||||
| $(KDIR)dgemm_small_matrix_permit$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DGEMM_SMALL_M_PERMIT) | |||||
| $(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX $< -o $@ | |||||
| $(KDIR)dgemm_small_kernel_nn$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DGEMM_SMALL_K_NN) | $(KDIR)dgemm_small_kernel_nn$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DGEMM_SMALL_K_NN) | ||||
| $(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX $< -o $@ | $(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX $< -o $@ | ||||
| @@ -4350,6 +4361,9 @@ $(KDIR)dgemm_small_kernel_b0_tn$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DGEMM_SMALL | |||||
| $(KDIR)dgemm_small_kernel_b0_tt$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DGEMM_SMALL_K_B0_TT) | $(KDIR)dgemm_small_kernel_b0_tt$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(DGEMM_SMALL_K_B0_TT) | ||||
| $(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX $< -o $@ | $(CC) $(CFLAGS) -c -DDOUBLE -UCOMPLEX $< -o $@ | ||||
| ifndef SGEMM_SMALL_M_PERMIT | |||||
| SGEMM_SMALL_M_PERMIT = ../generic/gemm_small_matrix_permit.c | |||||
| endif | |||||
| ifndef SGEMM_SMALL_K_NN | ifndef SGEMM_SMALL_K_NN | ||||
| SGEMM_SMALL_K_NN = ../generic/gemm_small_matrix_kernel_nn.c | SGEMM_SMALL_K_NN = ../generic/gemm_small_matrix_kernel_nn.c | ||||
| @@ -4367,6 +4381,9 @@ ifndef SGEMM_SMALL_K_TT | |||||
| SGEMM_SMALL_K_TT = ../generic/gemm_small_matrix_kernel_tt.c | SGEMM_SMALL_K_TT = ../generic/gemm_small_matrix_kernel_tt.c | ||||
| endif | endif | ||||
| $(KDIR)sgemm_small_matrix_permit$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMM_SMALL_M_PERMIT) | |||||
| $(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX $< -o $@ | |||||
| $(KDIR)sgemm_small_kernel_nn$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMM_SMALL_K_NN) | $(KDIR)sgemm_small_kernel_nn$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMM_SMALL_K_NN) | ||||
| $(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX $< -o $@ | $(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX $< -o $@ | ||||
| @@ -4407,6 +4424,9 @@ $(KDIR)sgemm_small_kernel_b0_tn$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMM_SMALL | |||||
| $(KDIR)sgemm_small_kernel_b0_tt$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMM_SMALL_K_B0_TT) | $(KDIR)sgemm_small_kernel_b0_tt$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(SGEMM_SMALL_K_B0_TT) | ||||
| $(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX $< -o $@ | $(CC) $(CFLAGS) -c -UDOUBLE -UCOMPLEX $< -o $@ | ||||
| ifndef CGEMM_SMALL_M_PERMIT | |||||
| CGEMM_SMALL_M_PERMIT = ../generic/zgemm_small_matrix_permit.c | |||||
| endif | |||||
| ifndef CGEMM_SMALL_K_NN | ifndef CGEMM_SMALL_K_NN | ||||
| CGEMM_SMALL_K_NN = ../generic/zgemm_small_matrix_kernel_nn.c | CGEMM_SMALL_K_NN = ../generic/zgemm_small_matrix_kernel_nn.c | ||||
| @@ -4424,6 +4444,9 @@ ifndef CGEMM_SMALL_K_TT | |||||
| CGEMM_SMALL_K_TT = ../generic/zgemm_small_matrix_kernel_tt.c | CGEMM_SMALL_K_TT = ../generic/zgemm_small_matrix_kernel_tt.c | ||||
| endif | endif | ||||
| $(KDIR)cgemm_small_matrix_permit$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMM_SMALL_M_PERMIT) | |||||
| $(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX $< -o $@ | |||||
| $(KDIR)cgemm_small_kernel_nn$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMM_SMALL_K_NN) | $(KDIR)cgemm_small_kernel_nn$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMM_SMALL_K_NN) | ||||
| $(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DNN $< -o $@ | $(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DNN $< -o $@ | ||||
| @@ -4536,6 +4559,10 @@ $(KDIR)cgemm_small_kernel_b0_ct$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMM_SMALL | |||||
| $(KDIR)cgemm_small_kernel_b0_cc$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMM_SMALL_K_B0_TT) | $(KDIR)cgemm_small_kernel_b0_cc$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(CGEMM_SMALL_K_B0_TT) | ||||
| $(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DCC $< -o $@ | $(CC) $(CFLAGS) -c -UDOUBLE -DCOMPLEX -DCC $< -o $@ | ||||
| ifndef ZGEMM_SMALL_M_PERMIT | |||||
| ZGEMM_SMALL_M_PERMIT = ../generic/zgemm_small_matrix_permit.c | |||||
| endif | |||||
| ifndef ZGEMM_SMALL_K_NN | ifndef ZGEMM_SMALL_K_NN | ||||
| ZGEMM_SMALL_K_NN = ../generic/zgemm_small_matrix_kernel_nn.c | ZGEMM_SMALL_K_NN = ../generic/zgemm_small_matrix_kernel_nn.c | ||||
| endif | endif | ||||
| @@ -4552,6 +4579,10 @@ ifndef ZGEMM_SMALL_K_TT | |||||
| ZGEMM_SMALL_K_TT = ../generic/zgemm_small_matrix_kernel_tt.c | ZGEMM_SMALL_K_TT = ../generic/zgemm_small_matrix_kernel_tt.c | ||||
| endif | endif | ||||
| $(KDIR)zgemm_small_matrix_permit$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMM_SMALL_M_PERMIT) | |||||
| $(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX $< -o $@ | |||||
| $(KDIR)zgemm_small_kernel_nn$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMM_SMALL_K_NN) | $(KDIR)zgemm_small_kernel_nn$(TSUFFIX).$(SUFFIX) : $(KERNELDIR)/$(ZGEMM_SMALL_K_NN) | ||||
| $(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DNN $< -o $@ | $(CC) $(CFLAGS) -c -DDOUBLE -DCOMPLEX -DNN $< -o $@ | ||||
| @@ -0,0 +1,37 @@ | |||||
| /*************************************************************************** | |||||
| Copyright (c) 2021, The OpenBLAS Project | |||||
| All rights reserved. | |||||
| Redistribution and use in source and binary forms, with or without | |||||
| modification, are permitted provided that the following conditions are | |||||
| met: | |||||
| 1. Redistributions of source code must retain the above copyright | |||||
| notice, this list of conditions and the following disclaimer. | |||||
| 2. Redistributions in binary form must reproduce the above copyright | |||||
| notice, this list of conditions and the following disclaimer in | |||||
| the documentation and/or other materials provided with the | |||||
| distribution. | |||||
| 3. Neither the name of the OpenBLAS project nor the names of | |||||
| its contributors may be used to endorse or promote products | |||||
| derived from this software without specific prior written permission. | |||||
| THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" | |||||
| AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | |||||
| IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | |||||
| ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE | |||||
| LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL | |||||
| DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR | |||||
| SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER | |||||
| CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, | |||||
| OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE | |||||
| USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |||||
| *****************************************************************************/ | |||||
| #include "common.h" | |||||
| int CNAME(int transa, int transb, BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, FLOAT beta) | |||||
| { | |||||
| double MNK = (double) M * (double) N * (double) K; | |||||
| if (MNK <= 100.0*100.0*100.0) | |||||
| return 1; | |||||
| else | |||||
| return 0; | |||||
| } | |||||
| @@ -0,0 +1,37 @@ | |||||
| /*************************************************************************** | |||||
| Copyright (c) 2021, The OpenBLAS Project | |||||
| All rights reserved. | |||||
| Redistribution and use in source and binary forms, with or without | |||||
| modification, are permitted provided that the following conditions are | |||||
| met: | |||||
| 1. Redistributions of source code must retain the above copyright | |||||
| notice, this list of conditions and the following disclaimer. | |||||
| 2. Redistributions in binary form must reproduce the above copyright | |||||
| notice, this list of conditions and the following disclaimer in | |||||
| the documentation and/or other materials provided with the | |||||
| distribution. | |||||
| 3. Neither the name of the OpenBLAS project nor the names of | |||||
| its contributors may be used to endorse or promote products | |||||
| derived from this software without specific prior written permission. | |||||
| THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" | |||||
| AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | |||||
| IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | |||||
| ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE | |||||
| LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL | |||||
| DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR | |||||
| SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER | |||||
| CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, | |||||
| OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE | |||||
| USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |||||
| *****************************************************************************/ | |||||
| #include "common.h" | |||||
| int CNAME(int transa, int transb, BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha0, FLOAT alpha1, FLOAT beta0, FLOAT beta1) | |||||
| { | |||||
| double MNK = (double) M * (double) N * (double) K; | |||||
| if (MNK <= 100.0*100.0*100.0) | |||||
| return 1; | |||||
| else | |||||
| return 0; | |||||
| } | |||||