Browse Source
Optimize s/dscal function for POWER10
This patch makes use of new POWER10 vector pair instructions for loads and stores.tags/v0.3.14^2
Rajalakshmi Srinivasaraghavan
5 years ago
4 changed files with 339 additions and 2 deletions
Unified View
Diff Options
-
+35 -1kernel/power/dscal.c
-
+134 -0kernel/power/dscal_microk_power10.c
-
+35 -1kernel/power/sscal.c
-
+135 -0kernel/power/sscal_microk_power10.c
+ 35
- 1
kernel/power/dscal.c
View File
| @@ -35,9 +35,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |||||
| #include "common.h" | #include "common.h" | ||||
| #if defined(POWER8) || defined(POWER9) || defined(POWER10) | |||||
| #if defined(__VEC__) || defined(__ALTIVEC__) | #if defined(__VEC__) || defined(__ALTIVEC__) | ||||
| #if defined(POWER8) || defined(POWER9) | |||||
| #include "dscal_microk_power8.c" | #include "dscal_microk_power8.c" | ||||
| #elif defined(POWER10) | |||||
| #include "dscal_microk_power10.c" | |||||
| #endif | #endif | ||||
| #endif | #endif | ||||
| @@ -100,12 +102,28 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLAS | |||||
| if ( da == 0.0 ) | if ( da == 0.0 ) | ||||
| { | { | ||||
| #if defined(POWER10) | |||||
| if ( n >= 16 ) | |||||
| { | |||||
| BLASLONG align = ((32 - ((uintptr_t)x & (uintptr_t)0x1F)) >> 3) & 0x3; | |||||
| for (j = 0; j < align; j++) { | |||||
| x[j] = 0.0; | |||||
| } | |||||
| } | |||||
| BLASLONG n1 = (n-j) & -16; | |||||
| if ( n1 > 0 ) | |||||
| { | |||||
| dscal_kernel_8_zero(n1, &x[j]); | |||||
| j+=n1; | |||||
| } | |||||
| #else | |||||
| BLASLONG n1 = n & -16; | BLASLONG n1 = n & -16; | ||||
| if ( n1 > 0 ) | if ( n1 > 0 ) | ||||
| { | { | ||||
| dscal_kernel_8_zero(n1, x); | dscal_kernel_8_zero(n1, x); | ||||
| j=n1; | j=n1; | ||||
| } | } | ||||
| #endif | |||||
| while(j < n) | while(j < n) | ||||
| { | { | ||||
| @@ -118,12 +136,28 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLAS | |||||
| else | else | ||||
| { | { | ||||
| #if defined(POWER10) | |||||
| if ( n >= 16 ) | |||||
| { | |||||
| BLASLONG align = ((32 - ((uintptr_t)x & (uintptr_t)0x1F)) >> 3) & 0x3; | |||||
| for (j = 0; j < align; j++) { | |||||
| x[j] = da * x[j]; | |||||
| } | |||||
| } | |||||
| BLASLONG n1 = (n-j) & -16; | |||||
| if ( n1 > 0 ) | |||||
| { | |||||
| dscal_kernel_8(n1, &x[j], da); | |||||
| j+=n1; | |||||
| } | |||||
| #else | |||||
| BLASLONG n1 = n & -16; | BLASLONG n1 = n & -16; | ||||
| if ( n1 > 0 ) | if ( n1 > 0 ) | ||||
| { | { | ||||
| dscal_kernel_8(n1, x, da); | dscal_kernel_8(n1, x, da); | ||||
| j=n1; | j=n1; | ||||
| } | } | ||||
| #endif | |||||
| while(j < n) | while(j < n) | ||||
| { | { | ||||
+ 134
- 0
kernel/power/dscal_microk_power10.c
View File
| @@ -0,0 +1,134 @@ | |||||
| /*************************************************************************** | |||||
| Copyright (c) 2021, The OpenBLAS Project | |||||
| All rights reserved. | |||||
| Redistribution and use in source and binary forms, with or without | |||||
| modification, are permitted provided that the following conditions are | |||||
| met: | |||||
| 1. Redistributions of source code must retain the above copyright | |||||
| notice, this list of conditions and the following disclaimer. | |||||
| 2. Redistributions in binary form must reproduce the above copyright | |||||
| notice, this list of conditions and the following disclaimer in | |||||
| the documentation and/or other materials provided with the | |||||
| distribution. | |||||
| 3. Neither the name of the OpenBLAS project nor the names of | |||||
| its contributors may be used to endorse or promote products | |||||
| derived from this software without specific prior written permission. | |||||
| THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" | |||||
| AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | |||||
| IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | |||||
| ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE | |||||
| LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL | |||||
| DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR | |||||
| SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER | |||||
| CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, | |||||
| OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE | |||||
| USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |||||
| *****************************************************************************/ | |||||
| #define HAVE_KERNEL_8 1 | |||||
| static void dscal_kernel_8 (long n, double *x, double alpha) | |||||
| { | |||||
| __asm__ | |||||
| ( | |||||
| "dcbt 0, %2 \n\t" | |||||
| XXSPLTD_S(48,%x3,0) | |||||
| "lxvp 32, 0(%2) \n\t" | |||||
| "lxvp 34, 32(%2) \n\t" | |||||
| "lxvp 36, 64(%2) \n\t" | |||||
| "lxvp 38, 96(%2) \n\t" | |||||
| "addic. %1, %1, -16 \n\t" | |||||
| "ble two%= \n\t" | |||||
| ".align 5 \n" | |||||
| "one%=: \n\t" | |||||
| "xvmuldp 40, 32, 48 \n\t" | |||||
| "xvmuldp 41, 33, 48 \n\t" | |||||
| "xvmuldp 42, 34, 48 \n\t" | |||||
| "xvmuldp 43, 35, 48 \n\t" | |||||
| "lxvp 32, 128(%2) \n\t" | |||||
| "lxvp 34, 160(%2) \n\t" | |||||
| "xvmuldp 44, 36, 48 \n\t" | |||||
| "xvmuldp 45, 37, 48 \n\t" | |||||
| "xvmuldp 46, 38, 48 \n\t" | |||||
| "xvmuldp 47, 39, 48 \n\t" | |||||
| "lxvp 36, 192(%2) \n\t" | |||||
| "lxvp 38, 224(%2) \n\t" | |||||
| "stxvp 40, 0(%2) \n\t" | |||||
| "stxvp 42, 32(%2) \n\t" | |||||
| "stxvp 44, 64(%2) \n\t" | |||||
| "stxvp 46, 96(%2) \n\t" | |||||
| "addi %2, %2, 128 \n\t" | |||||
| "addic. %1, %1, -16 \n\t" | |||||
| "bgt one%= \n" | |||||
| "two%=: \n\t" | |||||
| "xvmuldp 40, 32, 48 \n\t" | |||||
| "xvmuldp 41, 33, 48 \n\t" | |||||
| "xvmuldp 42, 34, 48 \n\t" | |||||
| "xvmuldp 43, 35, 48 \n\t" | |||||
| "xvmuldp 44, 36, 48 \n\t" | |||||
| "xvmuldp 45, 37, 48 \n\t" | |||||
| "xvmuldp 46, 38, 48 \n\t" | |||||
| "xvmuldp 47, 39, 48 \n\t" | |||||
| "stxvp 40, 0(%2) \n\t" | |||||
| "stxvp 42, 32(%2) \n\t" | |||||
| "stxvp 44, 64(%2) \n\t" | |||||
| "stxvp 46, 96(%2) \n\t" | |||||
| "#n=%1 alpha=%3 x=%0=%2" | |||||
| : | |||||
| "+m" (*x), | |||||
| "+r" (n), // 1 | |||||
| "+b" (x) // 2 | |||||
| : | |||||
| "d" (alpha) // 3 | |||||
| : | |||||
| "cr0", | |||||
| "vs32","vs33","vs34","vs35","vs36","vs37","vs38","vs39", | |||||
| "vs40","vs41","vs42","vs43","vs44","vs45","vs46","vs47","vs48" | |||||
| ); | |||||
| } | |||||
| static void dscal_kernel_8_zero (long n, double *x) | |||||
| { | |||||
| __asm__ | |||||
| ( | |||||
| "xxlxor 32, 32, 32 \n\t" | |||||
| "xxlxor 33, 33, 33 \n\t" | |||||
| ".align 5 \n" | |||||
| "one%=: \n\t" | |||||
| "stxvp 32, 0(%2) \n\t" | |||||
| "stxvp 32, 32(%2) \n\t" | |||||
| "stxvp 32, 64(%2) \n\t" | |||||
| "stxvp 32, 96(%2) \n\t" | |||||
| "addi %2, %2, 128 \n\t" | |||||
| "addic. %1, %1, -16 \n\t" | |||||
| "bgt one%= \n" | |||||
| "#n=%1 x=%0=%2 " | |||||
| : | |||||
| "=m" (*x), | |||||
| "+r" (n), // 1 | |||||
| "+b" (x) // 2 | |||||
| : | |||||
| : | |||||
| "cr0","vs32","vs33" | |||||
| ); | |||||
| } | |||||
+ 35
- 1
kernel/power/sscal.c
View File
| @@ -35,9 +35,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |||||
| #include "common.h" | #include "common.h" | ||||
| #if defined(POWER8) || defined(POWER9) || defined(POWER10) | |||||
| #if defined(__VEC__) || defined(__ALTIVEC__) | #if defined(__VEC__) || defined(__ALTIVEC__) | ||||
| #if defined(POWER8) || defined(POWER9) | |||||
| #include "sscal_microk_power8.c" | #include "sscal_microk_power8.c" | ||||
| #elif defined(POWER10) | |||||
| #include "sscal_microk_power10.c" | |||||
| #endif | #endif | ||||
| #endif | #endif | ||||
| @@ -102,12 +104,28 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLAS | |||||
| if ( da == 0.0 ) | if ( da == 0.0 ) | ||||
| { | { | ||||
| #if defined(POWER10) | |||||
| if ( n >= 32 ) | |||||
| { | |||||
| BLASLONG align = ((32 - ((uintptr_t)x & (uintptr_t)0x1F)) >> 2) & 0x7; | |||||
| for (j = 0; j < align; j++) { | |||||
| x[j] = 0.0; | |||||
| } | |||||
| } | |||||
| BLASLONG n1 = (n-j) & -32; | |||||
| if ( n1 > 0 ) | |||||
| { | |||||
| sscal_kernel_16_zero(n1, &x[j]); | |||||
| j+=n1; | |||||
| } | |||||
| #else | |||||
| BLASLONG n1 = n & -32; | BLASLONG n1 = n & -32; | ||||
| if ( n1 > 0 ) | if ( n1 > 0 ) | ||||
| { | { | ||||
| sscal_kernel_16_zero(n1, x); | sscal_kernel_16_zero(n1, x); | ||||
| j=n1; | j=n1; | ||||
| } | } | ||||
| #endif | |||||
| while(j < n) | while(j < n) | ||||
| { | { | ||||
| @@ -120,12 +138,28 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT da, FLOAT *x, BLAS | |||||
| else | else | ||||
| { | { | ||||
| #if defined(POWER10) | |||||
| if ( n >= 32 ) | |||||
| { | |||||
| BLASLONG align = ((32 - ((uintptr_t)x & (uintptr_t)0x1F)) >> 2) & 0x7; | |||||
| for (j = 0; j < align; j++) { | |||||
| x[j] = da * x[j]; | |||||
| } | |||||
| } | |||||
| BLASLONG n1 = (n-j) & -32; | |||||
| if ( n1 > 0 ) | |||||
| { | |||||
| sscal_kernel_16(n1, &x[j], da); | |||||
| j+=n1; | |||||
| } | |||||
| #else | |||||
| BLASLONG n1 = n & -32; | BLASLONG n1 = n & -32; | ||||
| if ( n1 > 0 ) | if ( n1 > 0 ) | ||||
| { | { | ||||
| sscal_kernel_16(n1, x, da); | sscal_kernel_16(n1, x, da); | ||||
| j=n1; | j=n1; | ||||
| } | } | ||||
| #endif | |||||
| while(j < n) | while(j < n) | ||||
| { | { | ||||
+ 135
- 0
kernel/power/sscal_microk_power10.c
View File
| @@ -0,0 +1,135 @@ | |||||
| /*************************************************************************** | |||||
| Copyright (c) 2021, The OpenBLAS Project | |||||
| All rights reserved. | |||||
| Redistribution and use in source and binary forms, with or without | |||||
| modification, are permitted provided that the following conditions are | |||||
| met: | |||||
| 1. Redistributions of source code must retain the above copyright | |||||
| notice, this list of conditions and the following disclaimer. | |||||
| 2. Redistributions in binary form must reproduce the above copyright | |||||
| notice, this list of conditions and the following disclaimer in | |||||
| the documentation and/or other materials provided with the | |||||
| distribution. | |||||
| 3. Neither the name of the OpenBLAS project nor the names of | |||||
| its contributors may be used to endorse or promote products | |||||
| derived from this software without specific prior written permission. | |||||
| THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" | |||||
| AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | |||||
| IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | |||||
| ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE | |||||
| LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL | |||||
| DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR | |||||
| SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER | |||||
| CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, | |||||
| OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE | |||||
| USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |||||
| *****************************************************************************/ | |||||
| #define HAVE_KERNEL_16 1 | |||||
| static void sscal_kernel_16 (long n, float *x, float alpha) | |||||
| { | |||||
| __asm__ | |||||
| ( | |||||
| "dcbt 0, %2 \n\t" | |||||
| "xscvdpspn 48, %x3 \n\t" | |||||
| "xxspltw 48, 48, 0 \n\t" | |||||
| "lxvp 32, 0(%2) \n\t" | |||||
| "lxvp 34, 32(%2) \n\t" | |||||
| "lxvp 36, 64(%2) \n\t" | |||||
| "lxvp 38, 96(%2) \n\t" | |||||
| "addic. %1, %1, -32 \n\t" | |||||
| "ble two%= \n\t" | |||||
| ".align 5 \n" | |||||
| "one%=: \n\t" | |||||
| "xvmulsp 40, 32, 48 \n\t" | |||||
| "xvmulsp 41, 33, 48 \n\t" | |||||
| "xvmulsp 42, 34, 48 \n\t" | |||||
| "xvmulsp 43, 35, 48 \n\t" | |||||
| "lxvp 32, 128(%2) \n\t" | |||||
| "lxvp 34, 160(%2) \n\t" | |||||
| "xvmulsp 44, 36, 48 \n\t" | |||||
| "xvmulsp 45, 37, 48 \n\t" | |||||
| "xvmulsp 46, 38, 48 \n\t" | |||||
| "xvmulsp 47, 39, 48 \n\t" | |||||
| "lxvp 36, 192(%2) \n\t" | |||||
| "lxvp 38, 224(%2) \n\t" | |||||
| "stxvp 40, 0(%2) \n\t" | |||||
| "stxvp 42, 32(%2) \n\t" | |||||
| "stxvp 44, 64(%2) \n\t" | |||||
| "stxvp 46, 96(%2) \n\t" | |||||
| "addi %2, %2, 128 \n\t" | |||||
| "addic. %1, %1, -32 \n\t" | |||||
| "bgt one%= \n" | |||||
| "two%=: \n\t" | |||||
| "xvmulsp 40, 32, 48 \n\t" | |||||
| "xvmulsp 41, 33, 48 \n\t" | |||||
| "xvmulsp 42, 34, 48 \n\t" | |||||
| "xvmulsp 43, 35, 48 \n\t" | |||||
| "xvmulsp 44, 36, 48 \n\t" | |||||
| "xvmulsp 45, 37, 48 \n\t" | |||||
| "xvmulsp 46, 38, 48 \n\t" | |||||
| "xvmulsp 47, 39, 48 \n\t" | |||||
| "stxvp 40, 0(%2) \n\t" | |||||
| "stxvp 42, 32(%2) \n\t" | |||||
| "stxvp 44, 64(%2) \n\t" | |||||
| "stxvp 46, 96(%2) \n\t" | |||||
| "#n=%1 alpha=%3 x=%0=%2" | |||||
| : | |||||
| "+m" (*x), | |||||
| "+r" (n), // 1 | |||||
| "+b" (x) // 2 | |||||
| : | |||||
| "f" (alpha) // 3 | |||||
| : | |||||
| "cr0", | |||||
| "vs32","vs33","vs34","vs35","vs36","vs37","vs38","vs39", | |||||
| "vs40","vs41","vs42","vs43","vs44","vs45","vs46","vs47","vs48" | |||||
| ); | |||||
| } | |||||
| static void sscal_kernel_16_zero (long n, float *x) | |||||
| { | |||||
| __asm__ | |||||
| ( | |||||
| "xxlxor 32, 32, 32 \n\t" | |||||
| "xxlxor 33, 33, 33 \n\t" | |||||
| ".align 5 \n" | |||||
| "one%=: \n\t" | |||||
| "stxvp 32, 0(%2) \n\t" | |||||
| "stxvp 32, 32(%2) \n\t" | |||||
| "stxvp 32, 64(%2) \n\t" | |||||
| "stxvp 32, 96(%2) \n\t" | |||||
| "addi %2, %2, 128 \n\t" | |||||
| "addic. %1, %1, -32 \n\t" | |||||
| "bgt one%= \n" | |||||
| "#n=%1 x=%0=%2 " | |||||
| : | |||||
| "=m" (*x), | |||||
| "+r" (n), // 1 | |||||
| "+b" (x) // 2 | |||||
| : | |||||
| : | |||||
| "cr0","vs32","vs33" | |||||
| ); | |||||
| } | |||||