zzy34407230
/
mindspore2022
Not watched
1
0
Fork 0
Code
Releases 22 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: bc738d4ec3
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'bc738d4ec3'
${ noResults }
mindspore2022/mindspore/lite/nnacl/assembly/arm64/MatmulFp32.S

773 lines
16 kB
Raw Blame History 

  1. #ifdef __aarch64__

  2.     .text

  3.     .align 5

  4.     .global MatmulFloatNeon64

  5. #ifndef __APPLE__

  6.     .type MatmulFloatNeon64, %function

  7. #endif

  8. 

  9. // void MatmulFloatNeon64(const float *a, const float *b, float *c, const float *bias, int act_type, int depth

  10. //                        int row, int col, size_t stride, size_t writeNhwc, size_t WriteWino)

  11. // x0: a

  12. // x1: b

  13. // x2: c

  14. // x3: bias

  15. // w4: act_type

  16. // w5: depth

  17. // w6: row

  18. // w7: col

  19. // w17: stride

  20. // w13: c8_nhwc_c4

  21. 

  22. MatmulFloatNeon64:

  23.   sub sp, sp, #128

  24.   st1 {v8.4s, v9.4s, v10.4s, v11.4s}, [sp], #64

  25.   st1 {v12.4s, v13.4s, v14.4s, v15.4s}, [sp], #64

  26. 

  27.   ldr x9, [sp, #8]

  28.   ldr x14, [sp, #16]

  29. 

  30.   mov w18, #32 // sizeof(float) * 8

  31.   mul w15, w5, w18 // block stride of lhs/rhs: sizeof(float) * 8 * depth

  32.   mov x18, #4

  33.   ldr x17, [sp]

  34.   cbz x14, NoWinoSteps

  35.   mul x8, x7, x17

  36.   mov x11, #8

  37.   mul x11, x11, x17

  38.   mul x8, x8, x18

  39.   mul x11, x11, x18

  40. NoWinoSteps:

  41.   mul x17, x17, x18

  42. 

  43. L1:

  44.   mov w10, w6 // reload lhs row

  45.   mov x12, x0 // reload lhs ptr

  46.   mov x18, x2 // reload dst ptr

  47. 

  48. L2:

  49.   mov x16, x1 // reload rhs ptr

  50.   mov w13, w5 // reload depth

  51.   dup v8.4s, wzr

  52.   dup v9.4s, wzr

  53.   dup v10.4s, wzr

  54.   dup v11.4s, wzr

  55.   dup v12.4s, wzr

  56.   dup v13.4s, wzr

  57.   dup v14.4s, wzr

  58.   dup v15.4s, wzr

  59.   dup v16.4s, wzr

  60.   dup v17.4s, wzr

  61.   dup v18.4s, wzr

  62.   dup v19.4s, wzr

  63.   dup v20.4s, wzr

  64.   dup v21.4s, wzr

  65.   dup v22.4s, wzr

  66.   dup v23.4s, wzr

  67.   dup v24.4s, wzr

  68.   dup v25.4s, wzr

  69.   dup v26.4s, wzr

  70.   dup v27.4s, wzr

  71.   dup v28.4s, wzr

  72.   dup v29.4s, wzr

  73.   dup v30.4s, wzr

  74.   dup v31.4s, wzr

  75. 

  76. LoopStart:

  77.   ld1 {v0.4s, v1.4s, v2.4s}, [x12], #48

  78.   ld1 {v3.4s, v4.4s}, [x16], #32

  79.   fmla v8.4s, v3.4s, v0.s[0]

  80.   fmla v10.4s, v3.4s, v0.s[1]

  81.   fmla v12.4s, v3.4s, v0.s[2]

  82.   fmla v14.4s, v3.4s, v0.s[3]

  83.   fmla v9.4s, v4.4s, v0.s[0]

  84.   fmla v11.4s, v4.4s, v0.s[1]

  85.   fmla v13.4s, v4.4s, v0.s[2]

  86.   fmla v15.4s, v4.4s, v0.s[3]

  87. 

  88.   subs w13, w13, #1

  89.   beq LoopEnd

  90. 

  91. Loop:

  92.   ld1 {v0.4s}, [x12], #16

  93.   fmla v16.4s, v3.4s, v1.s[0]

  94.   fmla v18.4s, v3.4s, v1.s[1]

  95.   fmla v20.4s, v3.4s, v1.s[2]

  96.   fmla v22.4s, v3.4s, v1.s[3]

  97.   fmla v17.4s, v4.4s, v1.s[0]

  98.   fmla v19.4s, v4.4s, v1.s[1]

  99.   fmla v21.4s, v4.4s, v1.s[2]

  100.   fmla v23.4s, v4.4s, v1.s[3]

  101.   ld1 {v1.4s}, [x12], #16

  102.   fmla v24.4s, v3.4s, v2.s[0]

  103.   fmla v26.4s, v3.4s, v2.s[1]

  104.   fmla v28.4s, v3.4s, v2.s[2]

  105.   fmla v30.4s, v3.4s, v2.s[3]

  106.   ld1 {v3.4s}, [x16], #16

  107.   fmla v25.4s, v4.4s, v2.s[0]

  108.   fmla v27.4s, v4.4s, v2.s[1]

  109.   fmla v29.4s, v4.4s, v2.s[2]

  110.   fmla v31.4s, v4.4s, v2.s[3]

  111.   ld1 {v4.4s}, [x16], #16

  112.   fmla v8.4s, v3.4s, v0.s[0]

  113.   fmla v10.4s, v3.4s, v0.s[1]

  114.   fmla v12.4s, v3.4s, v0.s[2]

  115.   fmla v14.4s, v3.4s, v0.s[3]

  116.   ld1 {v2.4s}, [x12], #16

  117.   fmla v9.4s, v4.4s, v0.s[0]

  118.   fmla v11.4s, v4.4s, v0.s[1]

  119.   fmla v13.4s, v4.4s, v0.s[2]

  120.   fmla v15.4s, v4.4s, v0.s[3]

  121. 

  122.   subs w13, w13, #1

  123.   bgt Loop

  124. 

  125. LoopEnd:

  126.   fmla v16.4s, v3.4s, v1.s[0]

  127.   fmla v18.4s, v3.4s, v1.s[1]

  128.   fmla v20.4s, v3.4s, v1.s[2]

  129.   fmla v22.4s, v3.4s, v1.s[3]

  130.   fmla v17.4s, v4.4s, v1.s[0]

  131.   fmla v19.4s, v4.4s, v1.s[1]

  132.   fmla v21.4s, v4.4s, v1.s[2]

  133.   fmla v23.4s, v4.4s, v1.s[3]

  134.   fmla v24.4s, v3.4s, v2.s[0]

  135.   fmla v26.4s, v3.4s, v2.s[1]

  136.   fmla v28.4s, v3.4s, v2.s[2]

  137.   fmla v30.4s, v3.4s, v2.s[3]

  138.   fmla v25.4s, v4.4s, v2.s[0]

  139.   fmla v27.4s, v4.4s, v2.s[1]

  140.   fmla v29.4s, v4.4s, v2.s[2]

  141.   fmla v31.4s, v4.4s, v2.s[3]

  142. 

  143. Bias:

  144.   cbz x3, Activation

  145.   ld1 {v0.4s}, [x3], #16

  146.   ld1 {v1.4s}, [x3]

  147.   sub x3, x3, #16

  148.   fadd v8.4s, v8.4s, v0.4s

  149.   fadd v9.4s, v9.4s, v1.4s

  150.   fadd v10.4s, v10.4s, v0.4s

  151.   fadd v11.4s, v11.4s, v1.4s

  152.   fadd v12.4s, v12.4s, v0.4s

  153.   fadd v13.4s, v13.4s, v1.4s

  154.   fadd v14.4s, v14.4s, v0.4s

  155.   fadd v15.4s, v15.4s, v1.4s

  156.   fadd v16.4s, v16.4s, v0.4s

  157.   fadd v17.4s, v17.4s, v1.4s

  158.   fadd v18.4s, v18.4s, v0.4s

  159.   fadd v19.4s, v19.4s, v1.4s

  160.   fadd v20.4s, v20.4s, v0.4s

  161.   fadd v21.4s, v21.4s, v1.4s

  162.   fadd v22.4s, v22.4s, v0.4s

  163.   fadd v23.4s, v23.4s, v1.4s

  164.   fadd v24.4s, v24.4s, v0.4s

  165.   fadd v25.4s, v25.4s, v1.4s

  166.   fadd v26.4s, v26.4s, v0.4s

  167.   fadd v27.4s, v27.4s, v1.4s

  168.   fadd v28.4s, v28.4s, v0.4s

  169.   fadd v29.4s, v29.4s, v1.4s

  170.   fadd v30.4s, v30.4s, v0.4s

  171.   fadd v31.4s, v31.4s, v1.4s

  172. 

  173. Activation:

  174.   cmp w4, #3

  175.   beq Relu6

  176.   cmp w4, #1

  177.   beq Relu

  178.   b Write

  179. 

  180. Relu6:

  181.   mov w13, #6

  182.   dup v2.4s, w13

  183.   scvtf v2.4s, v2.4s

  184.   fmin v8.4s, v8.4s, v2.4s

  185.   fmin v9.4s, v9.4s, v2.4s

  186.   fmin v10.4s, v10.4s, v2.4s

  187.   fmin v11.4s, v11.4s, v2.4s

  188.   fmin v12.4s, v12.4s, v2.4s

  189.   fmin v13.4s, v13.4s, v2.4s

  190.   fmin v14.4s, v14.4s, v2.4s

  191.   fmin v15.4s, v15.4s, v2.4s

  192.   fmin v16.4s, v16.4s, v2.4s

  193.   fmin v17.4s, v17.4s, v2.4s

  194.   fmin v18.4s, v18.4s, v2.4s

  195.   fmin v19.4s, v19.4s, v2.4s

  196.   fmin v20.4s, v20.4s, v2.4s

  197.   fmin v21.4s, v21.4s, v2.4s

  198.   fmin v22.4s, v22.4s, v2.4s

  199.   fmin v23.4s, v23.4s, v2.4s

  200.   fmin v24.4s, v24.4s, v2.4s

  201.   fmin v25.4s, v25.4s, v2.4s

  202.   fmin v26.4s, v26.4s, v2.4s

  203.   fmin v27.4s, v27.4s, v2.4s

  204.   fmin v28.4s, v28.4s, v2.4s

  205.   fmin v29.4s, v29.4s, v2.4s

  206.   fmin v30.4s, v30.4s, v2.4s

  207.   fmin v31.4s, v31.4s, v2.4s

  208. 

  209. Relu:

  210.   dup v3.4s, wzr

  211.   fmax v8.4s, v8.4s, v3.4s

  212.   fmax v9.4s, v9.4s, v3.4s

  213.   fmax v10.4s, v10.4s, v3.4s

  214.   fmax v11.4s, v11.4s, v3.4s

  215.   fmax v12.4s, v12.4s, v3.4s

  216.   fmax v13.4s, v13.4s, v3.4s

  217.   fmax v14.4s, v14.4s, v3.4s

  218.   fmax v15.4s, v15.4s, v3.4s

  219.   fmax v16.4s, v16.4s, v3.4s

  220.   fmax v17.4s, v17.4s, v3.4s

  221.   fmax v18.4s, v18.4s, v3.4s

  222.   fmax v19.4s, v19.4s, v3.4s

  223.   fmax v20.4s, v20.4s, v3.4s

  224.   fmax v21.4s, v21.4s, v3.4s

  225.   fmax v22.4s, v22.4s, v3.4s

  226.   fmax v23.4s, v23.4s, v3.4s

  227.   fmax v24.4s, v24.4s, v3.4s

  228.   fmax v25.4s, v25.4s, v3.4s

  229.   fmax v26.4s, v26.4s, v3.4s

  230.   fmax v27.4s, v27.4s, v3.4s

  231.   fmax v28.4s, v28.4s, v3.4s

  232.   fmax v29.4s, v29.4s, v3.4s

  233.   fmax v30.4s, v30.4s, v3.4s

  234.   fmax v31.4s, v31.4s, v3.4s

  235. 

  236. Write:

  237.   cbnz x14, WriteWino

  238.   cbz x9, WriteC8

  239.   cmp w7, #1

  240.   beq Write1

  241.   cmp w7, #2

  242.   beq Write2

  243.   cmp w7, #3

  244.   beq Write3

  245.   cmp w7, #4

  246.   beq Write4

  247.   cmp w7, #5

  248.   beq Write5

  249.   cmp w7, #6

  250.   beq Write6

  251.   cmp w7, #7

  252.   beq Write7

  253.   b Write8

  254. 

  255. Write1:

  256.   str s8, [x18]

  257.   cmp w10, #1

  258.   beq WriteEnd

  259.   add x18, x18, x17

  260.   str s10, [x18]

  261.   cmp w10, #2

  262.   beq WriteEnd

  263.   add x18, x18, x17

  264.   str s12, [x18]

  265.   cmp w10, #3

  266.   beq WriteEnd

  267.   add x18, x18, x17

  268.   str s14, [x18]

  269.   cmp w10, #4

  270.   beq WriteEnd

  271.   add x18, x18, x17

  272.   str s16, [x18]

  273.   cmp w10, #5

  274.   beq WriteEnd

  275.   add x18, x18, x17

  276.   str s18, [x18]

  277.   cmp w10, #6

  278.   beq WriteEnd

  279.   add x18, x18, x17

  280.   str s20, [x18]

  281.   cmp w10, #7

  282.   beq WriteEnd

  283.   add x18, x18, x17

  284.   str s22, [x18]

  285.   cmp w10, #8

  286.   beq WriteEnd

  287.   add x18, x18, x17

  288.   str s24, [x18]

  289.   cmp w10, #9

  290.   beq WriteEnd

  291.   add x18, x18, x17

  292.   str s26, [x18]

  293.   cmp w10, #10

  294.   beq WriteEnd

  295.   add x18, x18, x17

  296.   str s28, [x18]

  297.   cmp w10, #11

  298.   beq WriteEnd

  299.   add x18, x18, x17

  300.   str s30, [x18]

  301.   add x18, x18, x17

  302.   b WriteEnd

  303. Write2:

  304.   dup s9, v8.s[1]

  305.   stp s8, s9, [x18]

  306.   cmp w10, #1

  307.   beq WriteEnd

  308.   add x18, x18, x17

  309.   dup s11, v10.s[1]

  310.   stp s10, s11, [x18]

  311.   cmp w10, #2

  312.   beq WriteEnd

  313.   add x18, x18, x17

  314.   dup s13, v12.s[1]

  315.   stp s12, s13, [x18]

  316.   cmp w10, #3

  317.   beq WriteEnd

  318.   add x18, x18, x17

  319.   dup s15, v14.s[1]

  320.   stp s14, s15, [x18]

  321.   cmp w10, #4

  322.   beq WriteEnd

  323.   add x18, x18, x17

  324.   dup s17, v16.s[1]

  325.   stp s16, s17, [x18]

  326.   cmp w10, #5

  327.   beq WriteEnd

  328.   add x18, x18, x17

  329.   dup s19, v18.s[1]

  330.   stp s18, s19, [x18]

  331.   cmp w10, #6

  332.   beq WriteEnd

  333.   add x18, x18, x17

  334.   dup s21, v20.s[1]

  335.   stp s20, s21, [x18]

  336.   cmp w10, #7

  337.   beq WriteEnd

  338.   add x18, x18, x17

  339.   dup s23, v22.s[1]

  340.   stp s22, s23, [x18]

  341.   cmp w10, #8

  342.   beq WriteEnd

  343.   add x18, x18, x17

  344.   dup s25, v24.s[1]

  345.   stp s24, s25, [x18]

  346.   cmp w10, #9

  347.   beq WriteEnd

  348.   add x18, x18, x17

  349.   dup s27, v26.s[1]

  350.   stp s26, s27, [x18]

  351.   cmp w10, #10

  352.   beq WriteEnd

  353.   add x18, x18, x17

  354.   dup s29, v28.s[1]

  355.   stp s28, s29, [x18]

  356.   cmp w10, #11

  357.   beq WriteEnd

  358.   add x18, x18, x17

  359.   dup s31, v30.s[1]

  360.   stp s30, s31, [x18]

  361.   add x18, x18, x17

  362.   b WriteEnd

  363. Write3:

  364.   add x13, x18, #8

  365.   dup s9, v8.s[1]

  366.   stp s8, s9, [x18]

  367.   add x18, x18, x17

  368.   st1 {v8.s}[2], [x13], x17

  369.   cmp w10, #1

  370.   beq WriteEnd

  371.   dup s11, v10.s[1]

  372.   stp s10, s11, [x18]

  373.   add x18, x18, x17

  374.   st1 {v10.s}[2], [x13], x17

  375.   cmp w10, #2

  376.   beq WriteEnd

  377.   dup s13, v12.s[1]

  378.   stp s12, s13, [x18]

  379.   add x18, x18, x17

  380.   st1 {v12.s}[2], [x13], x17

  381.   cmp w10, #3

  382.   beq WriteEnd

  383.   dup s15, v14.s[1]

  384.   stp s14, s15, [x18]

  385.   add x18, x18, x17

  386.   st1 {v14.s}[2], [x13], x17

  387.   cmp w10, #4

  388.   beq WriteEnd

  389.   dup s17, v16.s[1]

  390.   stp s16, s17, [x18]

  391.   add x18, x18, x17

  392.   st1 {v16.s}[2], [x13], x17

  393.   cmp w10, #5

  394.   beq WriteEnd

  395.   dup s19, v18.s[1]

  396.   stp s18, s19, [x18]

  397.   add x18, x18, x17

  398.   st1 {v18.s}[2], [x13], x17

  399.   cmp w10, #6

  400.   beq WriteEnd

  401.   dup s21, v20.s[1]

  402.   stp s20, s21, [x18]

  403.   add x18, x18, x17

  404.   st1 {v20.s}[2], [x13], x17

  405.   cmp w10, #7

  406.   beq WriteEnd

  407.   dup s23, v22.s[1]

  408.   stp s22, s23, [x18]

  409.   add x18, x18, x17

  410.   st1 {v22.s}[2], [x13], x17

  411.   cmp w10, #8

  412.   beq WriteEnd

  413.   dup s25, v24.s[1]

  414.   stp s24, s25, [x18]

  415.   add x18, x18, x17

  416.   st1 {v24.s}[2], [x13], x17

  417.   cmp w10, #9

  418.   beq WriteEnd

  419.   dup s27, v26.s[1]

  420.   stp s26, s27, [x18]

  421.   add x18, x18, x17

  422.   st1 {v26.s}[2], [x13], x17

  423.   cmp w10, #10

  424.   beq WriteEnd

  425.   dup s29, v28.s[1]

  426.   stp s28, s29, [x18]

  427.   add x18, x18, x17

  428.   st1 {v28.s}[2], [x13], x17

  429.   cmp w10, #11

  430.   beq WriteEnd

  431.   dup s31, v30.s[1]

  432.   stp s30, s31, [x18]

  433.   add x18, x18, x17

  434.   st1 {v30.s}[2], [x13]

  435.   b WriteEnd

  436. Write4:

  437.   st1 {v8.4s}, [x18], x17

  438.   cmp w10, #1

  439.   beq WriteEnd

  440.   st1 {v10.4s}, [x18], x17

  441.   cmp w10, #2

  442.   beq WriteEnd

  443.   st1 {v12.4s}, [x18], x17

  444.   cmp w10, #3

  445.   beq WriteEnd

  446.   st1 {v14.4s}, [x18], x17

  447.   cmp w10, #4

  448.   beq WriteEnd

  449.   st1 {v16.4s}, [x18], x17

  450.   cmp w10, #5

  451.   beq WriteEnd

  452.   st1 {v18.4s}, [x18], x17

  453.   cmp w10, #6

  454.   beq WriteEnd

  455.   st1 {v20.4s}, [x18], x17

  456.   cmp w10, #7

  457.   beq WriteEnd

  458.   st1 {v22.4s}, [x18], x17

  459.   cmp w10, #8

  460.   beq WriteEnd

  461.   st1 {v24.4s}, [x18], x17

  462.   cmp w10, #9

  463.   beq WriteEnd

  464.   st1 {v26.4s}, [x18], x17

  465.   cmp w10, #10

  466.   beq WriteEnd

  467.   st1 {v28.4s}, [x18], x17

  468.   cmp w10, #11

  469.   beq WriteEnd

  470.   st1 {v30.4s}, [x18], x17

  471.   b WriteEnd

  472. Write5:

  473.   add x13, x18, #16

  474.   st1 {v8.4s}, [x18], x17

  475.   str s9, [x13]

  476.   cmp w10, #1

  477.   beq WriteEnd

  478.   add x13, x13, x17

  479.   st1 {v10.4s}, [x18], x17

  480.   str s11, [x13]

  481.   cmp w10, #2

  482.   beq WriteEnd

  483.   add x13, x13, x17

  484.   st1 {v12.4s}, [x18], x17

  485.   str s13, [x13]

  486.   cmp w10, #3

  487.   beq WriteEnd

  488.   add x13, x13, x17

  489.   st1 {v14.4s}, [x18], x17

  490.   str s15, [x13]

  491.   cmp w10, #4

  492.   beq WriteEnd

  493.   add x13, x13, x17

  494.   st1 {v16.4s}, [x18], x17

  495.   str s17, [x13]

  496.   cmp w10, #5

  497.   beq WriteEnd

  498.   add x13, x13, x17

  499.   st1 {v18.4s}, [x18], x17

  500.   str s19, [x13]

  501.   cmp w10, #6

  502.   beq WriteEnd

  503.   add x13, x13, x17

  504.   st1 {v20.4s}, [x18], x17

  505.   str s21, [x13]

  506.   cmp w10, #7

  507.   beq WriteEnd

  508.   add x13, x13, x17

  509.   st1 {v22.4s}, [x18], x17

  510.   str s23, [x13]

  511.   cmp w10, #8

  512.   beq WriteEnd

  513.   add x13, x13, x17

  514.   st1 {v24.4s}, [x18], x17

  515.   str s25, [x13]

  516.   cmp w10, #9

  517.   beq WriteEnd

  518.   add x13, x13, x17

  519.   st1 {v26.4s}, [x18], x17

  520.   str s27, [x13]

  521.   cmp w10, #10

  522.   beq WriteEnd

  523.   add x13, x13, x17

  524.   st1 {v28.4s}, [x18], x17

  525.   str s29, [x13]

  526.   cmp w10, #11

  527.   beq WriteEnd

  528.   add x13, x13, x17

  529.   st1 {v30.4s}, [x18], x17

  530.   str s31, [x13]

  531.   b WriteEnd

  532. Write6:

  533.   add x13, x18, #16

  534.   st1 {v8.4s}, [x18], x17

  535.   dup s8, v9.s[1]

  536.   stp s9, s8, [x13]

  537.   cmp w10, #1

  538.   beq WriteEnd

  539.   add x13, x13, x17

  540.   st1 {v10.4s}, [x18], x17

  541.   dup s10, v11.s[1]

  542.   stp s11, s10, [x13]

  543.   cmp w10, #2

  544.   beq WriteEnd

  545.   add x13, x13, x17

  546.   st1 {v12.4s}, [x18], x17

  547.   dup s12, v13.s[1]

  548.   stp s13, s12, [x13]

  549.   cmp w10, #3

  550.   beq WriteEnd

  551.   add x13, x13, x17

  552.   st1 {v14.4s}, [x18], x17

  553.   dup s14, v15.s[1]

  554.   stp s15, s14, [x13]

  555.   cmp w10, #4

  556.   beq WriteEnd

  557.   add x13, x13, x17

  558.   st1 {v16.4s}, [x18], x17

  559.   dup s16, v17.s[1]

  560.   stp s17, s16, [x13]

  561.   cmp w10, #5

  562.   beq WriteEnd

  563.   add x13, x13, x17

  564.   st1 {v18.4s}, [x18], x17

  565.   dup s18, v19.s[1]

  566.   stp s19, s18, [x13]

  567.   cmp w10, #6

  568.   beq WriteEnd

  569.   add x13, x13, x17

  570.   st1 {v20.4s}, [x18], x17

  571.   dup s20, v21.s[1]

  572.   stp s21, s20, [x13]

  573.   cmp w10, #7

  574.   beq WriteEnd

  575.   add x13, x13, x17

  576.   st1 {v22.4s}, [x18], x17

  577.   dup s22, v23.s[1]

  578.   stp s23, s22, [x13]

  579.   cmp w10, #8

  580.   beq WriteEnd

  581.   add x13, x13, x17

  582.   st1 {v24.4s}, [x18], x17

  583.   dup s24, v25.s[1]

  584.   stp s25, s24, [x13]

  585.   cmp w10, #9

  586.   beq WriteEnd

  587.   add x13, x13, x17

  588.   st1 {v26.4s}, [x18], x17

  589.   dup s26, v27.s[1]

  590.   stp s27, s26, [x13]

  591.   cmp w10, #10

  592.   beq WriteEnd

  593.   add x13, x13, x17

  594.   st1 {v28.4s}, [x18], x17

  595.   dup s28, v29.s[1]

  596.   stp s29, s28, [x13]

  597.   cmp w10, #11

  598.   beq WriteEnd

  599.   add x13, x13, x17

  600.   st1 {v30.4s}, [x18], x17

  601.   dup s30, v31.s[1]

  602.   stp s31, s30, [x13]

  603.   b WriteEnd

  604. Write7:

  605.   add x13, x18, #16

  606.   add x16, x18, #24

  607.   st1 {v8.4s}, [x18], x17

  608.   dup s8, v9.s[1]

  609.   stp s9, s8, [x13]

  610.   add x13, x13, x17

  611.   st1 {v9.s}[2], [x16], x17

  612.   cmp w10, #1

  613.   beq WriteEnd

  614.   st1 {v10.4s}, [x18], x17

  615.   dup s10, v11.s[1]

  616.   stp s11, s10, [x13]

  617.   add x13, x13, x17

  618.   st1 {v11.s}[2], [x16], x17

  619.   cmp w10, #2

  620.   beq WriteEnd

  621.   st1 {v12.4s}, [x18], x17

  622.   dup s12, v13.s[1]

  623.   stp s13, s12, [x13]

  624.   add x13, x13, x17

  625.   st1 {v13.s}[2], [x16], x17

  626.   cmp w10, #3

  627.   beq WriteEnd

  628.   st1 {v14.4s}, [x18], x17

  629.   dup s14, v15.s[1]

  630.   stp s15, s14, [x13]

  631.   add x13, x13, x17

  632.   st1 {v15.s}[2], [x16], x17

  633.   cmp w10, #4

  634.   beq WriteEnd

  635.   st1 {v16.4s}, [x18], x17

  636.   dup s16, v17.s[1]

  637.   stp s17, s16, [x13]

  638.   add x13, x13, x17

  639.   st1 {v17.s}[2], [x16], x17

  640.   cmp w10, #5

  641.   beq WriteEnd

  642.   st1 {v18.4s}, [x18], x17

  643.   dup s18, v19.s[1]

  644.   stp s19, s18, [x13]

  645.   add x13, x13, x17

  646.   st1 {v19.s}[2], [x16], x17

  647.   cmp w10, #6

  648.   beq WriteEnd

  649.   st1 {v20.4s}, [x18], x17

  650.   dup s20, v21.s[1]

  651.   stp s21, s20, [x13]

  652.   add x13, x13, x17

  653.   st1 {v21.s}[2], [x16], x17

  654.   cmp w10, #7

  655.   beq WriteEnd

  656.   st1 {v22.4s}, [x18], x17

  657.   dup s22, v23.s[1]

  658.   stp s23, s22, [x13]

  659.   add x13, x13, x17

  660.   st1 {v23.s}[2], [x16], x17

  661.   cmp w10, #8

  662.   beq WriteEnd

  663.   st1 {v24.4s}, [x18], x17

  664.   dup s24, v25.s[1]

  665.   stp s25, s24, [x13]

  666.   add x13, x13, x17

  667.   st1 {v25.s}[2], [x16], x17

  668.   cmp w10, #9

  669.   beq WriteEnd

  670.   st1 {v26.4s}, [x18], x17

  671.   dup s26, v27.s[1]

  672.   stp s27, s26, [x13]

  673.   add x13, x13, x17

  674.   st1 {v27.s}[2], [x16], x17

  675.   cmp w10, #10

  676.   beq WriteEnd

  677.   st1 {v28.4s}, [x18], x17

  678.   dup s28, v29.s[1]

  679.   stp s29, s28, [x13]

  680.   add x13, x13, x17

  681.   st1 {v29.s}[2], [x16], x17

  682.   cmp w10, #11

  683.   beq WriteEnd

  684.   st1 {v30.4s}, [x18], x17

  685.   dup s30, v31.s[1]

  686.   stp s31, s30, [x13]

  687.   add x13, x13, x17

  688.   st1 {v31.s}[2], [x16], x17

  689.   b WriteEnd

  690. WriteC8:

  691.   st1 {v8.4s, v9.4s, v10.4s, v11.4s}, [x2], #64

  692.   st1 {v12.4s, v13.4s, v14.4s, v15.4s}, [x2], #64

  693.   st1 {v16.4s, v17.4s, v18.4s, v19.4s}, [x2], #64

  694.   st1 {v20.4s, v21.4s, v22.4s, v23.4s}, [x2], #64

  695.   st1 {v24.4s, v25.4s, v26.4s, v27.4s}, [x2], #64

  696.   st1 {v28.4s, v29.4s, v30.4s, v31.4s}, [x2], #64

  697.   b WriteEnd

  698. WriteWino:

  699.   st1 {v8.4s, v9.4s}, [x18], x8

  700.   st1 {v10.4s, v11.4s}, [x18], x8

  701.   st1 {v12.4s, v13.4s}, [x18], x8

  702.   st1 {v14.4s, v15.4s}, [x18], x8

  703.   st1 {v16.4s, v17.4s}, [x18], x8

  704.   st1 {v18.4s, v19.4s}, [x18], x8

  705.   st1 {v20.4s, v21.4s}, [x18], x8

  706.   st1 {v22.4s, v23.4s}, [x18], x8

  707.   st1 {v24.4s, v25.4s}, [x18], x8

  708.   st1 {v26.4s, v27.4s}, [x18], x8

  709.   st1 {v28.4s, v29.4s}, [x18], x8

  710.   st1 {v30.4s, v31.4s}, [x18], x8

  711.   b WriteEnd

  712. Write8:

  713.   st1 {v8.4s, v9.4s}, [x18], x17

  714.   cmp w10, #1

  715.   beq WriteEnd

  716.   st1 {v10.4s, v11.4s}, [x18], x17

  717.   cmp w10, #2

  718.   beq WriteEnd

  719.   st1 {v12.4s, v13.4s}, [x18], x17

  720.   cmp w10, #3

  721.   beq WriteEnd

  722.   st1 {v14.4s, v15.4s}, [x18], x17

  723.   cmp w10, #4

  724.   beq WriteEnd

  725.   st1 {v16.4s, v17.4s}, [x18], x17

  726.   cmp w10, #5

  727.   beq WriteEnd

  728.   st1 {v18.4s, v19.4s}, [x18], x17

  729.   cmp w10, #6

  730.   beq WriteEnd

  731.   st1 {v20.4s, v21.4s}, [x18], x17

  732.   cmp w10, #7

  733.   beq WriteEnd

  734.   st1 {v22.4s, v23.4s}, [x18], x17

  735.   cmp w10, #8

  736.   beq WriteEnd

  737.   st1 {v24.4s, v25.4s}, [x18], x17

  738.   cmp w10, #9

  739.   beq WriteEnd

  740.   st1 {v26.4s, v27.4s}, [x18], x17

  741.   cmp w10, #10

  742.   beq WriteEnd

  743.   st1 {v28.4s, v29.4s}, [x18], x17

  744.   cmp w10, #11

  745.   beq WriteEnd

  746.   st1 {v30.4s, v31.4s}, [x18], x17

  747. 

  748. WriteEnd:

  749.   subs w10, w10, #12 // lhs row - 12

  750.   bgt L2

  751. 

  752. End2:

  753.   subs w7, w7, #8 // rhs col - 8

  754.   add x1, x1, x15 // rhs ptr + stride

  755.   cbz x3, NoBiasStep

  756.   add x3, x3, #32 // bias ptr + stride

  757. NoBiasStep:

  758.   cbnz x14, WinoDstStep

  759.   cbz x9, NoDstStep

  760.   add x2, x2, #32 // dst ptr + stride

  761.   b NoDstStep

  762. WinoDstStep:

  763.   add x2, x2, x11

  764. NoDstStep:

  765.   bgt L1

  766. 

  767. End1:

  768.   sub sp, sp, #128

  769.   ld1 {v8.4s, v9.4s, v10.4s, v11.4s}, [sp], #64

  770.   ld1 {v12.4s, v13.4s, v14.4s, v15.4s}, [sp], #64

  771.   ret

  772. #endif