zzy34407230
/
mindspore2022
Not watched
1
0
Fork 0
Code
Releases 22 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: bc738d4ec3
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'bc738d4ec3'
${ noResults }
mindspore2022/mindspore/lite/nnacl/assembly/arm64/MatmulFp32Opt.S

1184 lines
35 kB
Raw Blame History 

  1. #ifdef __aarch64__

  2.     .text

  3.     .align 5

  4.     .global MatmulFloatNeon64Opt

  5. #ifndef __APPLE__

  6.     .type MatmulFloatNeon64Opt, %function

  7. #endif

  8. 

  9. // void MatmulFloatNeon64(const float *a, const float *b, float *c, const float *bias, int act_type, int depth

  10. //                        int row, int col, size_t stride, size_t writeMode)

  11. // x0: a

  12. // x1: b

  13. // x2: c

  14. // x3: bias

  15. // x4: act_type

  16. // x5: depth

  17. // x6: row

  18. // x7: col

  19. // x8: stride

  20. // x9: writeMode

  21. 

  22. MatmulFloatNeon64Opt:

  23.     sub sp, sp, #144

  24.     st1 {v8.4s, v9.4s, v10.4s, v11.4s}, [sp], #64

  25.     st1 {v12.4s, v13.4s, v14.4s, v15.4s}, [sp], #64

  26.     stp x19, x20, [sp], #16

  27. 

  28.     ldr x8, [sp]

  29.     ldr x9, [sp, #8]

  30. 

  31.     mov x18, #48 // sizeof(float) * 12

  32.     mul x17, x5, x18 // block stride of lhs/rhs: sizeof(float) * 12 * depth

  33.     cbnz x9, NoC8Steps

  34.     mov x11, x2

  35.     mov x18, #32

  36.     mul x16, x6, x18 // row * 8 * sizeof(float)

  37. NoC8Steps:

  38.     cmp x9, #2

  39.     bne NoWinoSteps

  40.     mov x18, #4

  41.     mul x15, x7, x8

  42.     mul x15, x15, x18 // kernel_size * col *sizeof(float)

  43.     mov x18, #32

  44.     mul x16, x8, x18 // kernel_size * 8 * sizeof(float)

  45. NoWinoSteps:

  46.     mov x18, #4

  47.     mul x8, x8, x18

  48. 

  49. LoopRowStart:

  50.     cmp x6, #4

  51.     ble LoopRow4

  52.     cmp x6, #8

  53.     blt LoopRow8

  54. 

  55. LoopRow:

  56.     mov x14, x1 // reload rhs ptr

  57.     mov x13, x7 // reload rhs col

  58.     mov x12, x3 // reload bias

  59. 

  60.     LoopCol:

  61.         cbz x9, NoReloadDst

  62.         mov x11, x2

  63.     NoReloadDst:

  64.         mov x10, x0 // reload lhs ptr

  65.         mov x19, x5 // reload depth

  66. 

  67.         cmp x13, #4

  68.         ble LoopDepthStartHalf

  69. 

  70.     LoopDepthStart:

  71.         ld1 {v0.4s, v1.4s, v2.4s}, [x10], #48

  72.         ld1 {v3.4s, v4.4s}, [x14], #32

  73.         fmul v8.4s, v3.4s, v0.s[0]

  74.         fmul v10.4s, v3.4s, v0.s[1]

  75.         fmul v12.4s, v3.4s, v0.s[2]

  76.         fmul v14.4s, v3.4s, v0.s[3]

  77.         fmul v9.4s, v4.4s, v0.s[0]

  78.         fmul v11.4s, v4.4s, v0.s[1]

  79.         fmul v13.4s, v4.4s, v0.s[2]

  80.         fmul v15.4s, v4.4s, v0.s[3]

  81.         fmul v16.4s, v3.4s, v1.s[0]

  82.         fmul v18.4s, v3.4s, v1.s[1]

  83.         fmul v20.4s, v3.4s, v1.s[2]

  84.         fmul v22.4s, v3.4s, v1.s[3]

  85.         fmul v17.4s, v4.4s, v1.s[0]

  86.         fmul v19.4s, v4.4s, v1.s[1]

  87.         fmul v21.4s, v4.4s, v1.s[2]

  88.         fmul v23.4s, v4.4s, v1.s[3]

  89.         fmul v24.4s, v3.4s, v2.s[0]

  90.         fmul v26.4s, v3.4s, v2.s[1]

  91.         fmul v28.4s, v3.4s, v2.s[2]

  92.         fmul v30.4s, v3.4s, v2.s[3]

  93.         fmul v25.4s, v4.4s, v2.s[0]

  94.         fmul v27.4s, v4.4s, v2.s[1]

  95.         fmul v29.4s, v4.4s, v2.s[2]

  96.         fmul v31.4s, v4.4s, v2.s[3]

  97. 

  98.         subs x19, x19, #1

  99.         beq Bias

  100. 

  101.         LoopDepth:

  102.             ld1 {v0.4s, v1.4s, v2.4s}, [x10], #48

  103.             ld1 {v3.4s, v4.4s}, [x14], #32

  104.             fmla v8.4s, v3.4s, v0.s[0]

  105.             fmla v10.4s, v3.4s, v0.s[1]

  106.             fmla v12.4s, v3.4s, v0.s[2]

  107.             fmla v14.4s, v3.4s, v0.s[3]

  108.             fmla v9.4s, v4.4s, v0.s[0]

  109.             fmla v11.4s, v4.4s, v0.s[1]

  110.             fmla v13.4s, v4.4s, v0.s[2]

  111.             fmla v15.4s, v4.4s, v0.s[3]

  112.             fmla v16.4s, v3.4s, v1.s[0]

  113.             fmla v18.4s, v3.4s, v1.s[1]

  114.             fmla v20.4s, v3.4s, v1.s[2]

  115.             fmla v22.4s, v3.4s, v1.s[3]

  116.             fmla v17.4s, v4.4s, v1.s[0]

  117.             fmla v19.4s, v4.4s, v1.s[1]

  118.             fmla v21.4s, v4.4s, v1.s[2]

  119.             fmla v23.4s, v4.4s, v1.s[3]

  120.             fmla v24.4s, v3.4s, v2.s[0]

  121.             fmla v26.4s, v3.4s, v2.s[1]

  122.             fmla v28.4s, v3.4s, v2.s[2]

  123.             fmla v30.4s, v3.4s, v2.s[3]

  124.             fmla v25.4s, v4.4s, v2.s[0]

  125.             fmla v27.4s, v4.4s, v2.s[1]

  126.             fmla v29.4s, v4.4s, v2.s[2]

  127.             fmla v31.4s, v4.4s, v2.s[3]

  128. 

  129.             subs x19, x19, #1

  130.             bgt LoopDepth

  131. 

  132.         Bias:

  133.             cbz x3, Activation

  134.             ld1 {v0.4s}, [x12], #16

  135.             ld1 {v1.4s}, [x12], #16

  136.             fadd v8.4s, v8.4s, v0.4s

  137.             fadd v9.4s, v9.4s, v1.4s

  138.             fadd v10.4s, v10.4s, v0.4s

  139.             fadd v11.4s, v11.4s, v1.4s

  140.             fadd v12.4s, v12.4s, v0.4s

  141.             fadd v13.4s, v13.4s, v1.4s

  142.             fadd v14.4s, v14.4s, v0.4s

  143.             fadd v15.4s, v15.4s, v1.4s

  144.             fadd v16.4s, v16.4s, v0.4s

  145.             fadd v17.4s, v17.4s, v1.4s

  146.             fadd v18.4s, v18.4s, v0.4s

  147.             fadd v19.4s, v19.4s, v1.4s

  148.             fadd v20.4s, v20.4s, v0.4s

  149.             fadd v21.4s, v21.4s, v1.4s

  150.             fadd v22.4s, v22.4s, v0.4s

  151.             fadd v23.4s, v23.4s, v1.4s

  152.             fadd v24.4s, v24.4s, v0.4s

  153.             fadd v25.4s, v25.4s, v1.4s

  154.             fadd v26.4s, v26.4s, v0.4s

  155.             fadd v27.4s, v27.4s, v1.4s

  156.             fadd v28.4s, v28.4s, v0.4s

  157.             fadd v29.4s, v29.4s, v1.4s

  158.             fadd v30.4s, v30.4s, v0.4s

  159.             fadd v31.4s, v31.4s, v1.4s

  160. 

  161.         Activation:

  162.             cmp x4, #3

  163.             beq Relu6

  164.             cmp x4, #1

  165.             beq Relu

  166.             b Write

  167. 

  168.         Relu6:

  169.             mov w19, #6

  170.             dup v2.4s, w19

  171.             scvtf v2.4s, v2.4s

  172.             fmin v8.4s, v8.4s, v2.4s

  173.             fmin v9.4s, v9.4s, v2.4s

  174.             fmin v10.4s, v10.4s, v2.4s

  175.             fmin v11.4s, v11.4s, v2.4s

  176.             fmin v12.4s, v12.4s, v2.4s

  177.             fmin v13.4s, v13.4s, v2.4s

  178.             fmin v14.4s, v14.4s, v2.4s

  179.             fmin v15.4s, v15.4s, v2.4s

  180.             fmin v16.4s, v16.4s, v2.4s

  181.             fmin v17.4s, v17.4s, v2.4s

  182.             fmin v18.4s, v18.4s, v2.4s

  183.             fmin v19.4s, v19.4s, v2.4s

  184.             fmin v20.4s, v20.4s, v2.4s

  185.             fmin v21.4s, v21.4s, v2.4s

  186.             fmin v22.4s, v22.4s, v2.4s

  187.             fmin v23.4s, v23.4s, v2.4s

  188.             fmin v24.4s, v24.4s, v2.4s

  189.             fmin v25.4s, v25.4s, v2.4s

  190.             fmin v26.4s, v26.4s, v2.4s

  191.             fmin v27.4s, v27.4s, v2.4s

  192.             fmin v28.4s, v28.4s, v2.4s

  193.             fmin v29.4s, v29.4s, v2.4s

  194.             fmin v30.4s, v30.4s, v2.4s

  195.             fmin v31.4s, v31.4s, v2.4s

  196.         

  197.         Relu:

  198.             dup v3.4s, wzr

  199.             fmax v8.4s, v8.4s, v3.4s

  200.             fmax v9.4s, v9.4s, v3.4s

  201.             fmax v10.4s, v10.4s, v3.4s

  202.             fmax v11.4s, v11.4s, v3.4s

  203.             fmax v12.4s, v12.4s, v3.4s

  204.             fmax v13.4s, v13.4s, v3.4s

  205.             fmax v14.4s, v14.4s, v3.4s

  206.             fmax v15.4s, v15.4s, v3.4s

  207.             fmax v16.4s, v16.4s, v3.4s

  208.             fmax v17.4s, v17.4s, v3.4s

  209.             fmax v18.4s, v18.4s, v3.4s

  210.             fmax v19.4s, v19.4s, v3.4s

  211.             fmax v20.4s, v20.4s, v3.4s

  212.             fmax v21.4s, v21.4s, v3.4s

  213.             fmax v22.4s, v22.4s, v3.4s

  214.             fmax v23.4s, v23.4s, v3.4s

  215.             fmax v24.4s, v24.4s, v3.4s

  216.             fmax v25.4s, v25.4s, v3.4s

  217.             fmax v26.4s, v26.4s, v3.4s

  218.             fmax v27.4s, v27.4s, v3.4s

  219.             fmax v28.4s, v28.4s, v3.4s

  220.             fmax v29.4s, v29.4s, v3.4s

  221.             fmax v30.4s, v30.4s, v3.4s

  222.             fmax v31.4s, v31.4s, v3.4s

  223.             b Write

  224. 

  225.     LoopDepthStartHalf:

  226.         ld1 {v0.4s, v1.4s, v2.4s}, [x10], #48

  227.         ld1 {v3.4s, v4.4s}, [x14], #32

  228.         fmul v8.4s, v3.4s, v0.s[0]

  229.         fmul v10.4s, v3.4s, v0.s[1]

  230.         fmul v12.4s, v3.4s, v0.s[2]

  231.         fmul v14.4s, v3.4s, v0.s[3]

  232.         fmul v16.4s, v3.4s, v1.s[0]

  233.         fmul v18.4s, v3.4s, v1.s[1]

  234.         fmul v20.4s, v3.4s, v1.s[2]

  235.         fmul v22.4s, v3.4s, v1.s[3]

  236.         fmul v24.4s, v3.4s, v2.s[0]

  237.         fmul v26.4s, v3.4s, v2.s[1]

  238.         fmul v28.4s, v3.4s, v2.s[2]

  239.         fmul v30.4s, v3.4s, v2.s[3]

  240. 

  241.         subs x19, x19, #1

  242.         beq BiasHalf

  243. 

  244.         LoopDepthHalf:

  245.             ld1 {v0.4s, v1.4s, v2.4s}, [x10], #48

  246.             ld1 {v3.4s, v4.4s}, [x14], #32

  247.             fmla v8.4s, v3.4s, v0.s[0]

  248.             fmla v10.4s, v3.4s, v0.s[1]

  249.             fmla v12.4s, v3.4s, v0.s[2]

  250.             fmla v14.4s, v3.4s, v0.s[3]

  251.             fmla v16.4s, v3.4s, v1.s[0]

  252.             fmla v18.4s, v3.4s, v1.s[1]

  253.             fmla v20.4s, v3.4s, v1.s[2]

  254.             fmla v22.4s, v3.4s, v1.s[3]

  255.             fmla v24.4s, v3.4s, v2.s[0]

  256.             fmla v26.4s, v3.4s, v2.s[1]

  257.             fmla v28.4s, v3.4s, v2.s[2]

  258.             fmla v30.4s, v3.4s, v2.s[3]

  259. 

  260.             subs x19, x19, #1

  261.             bgt LoopDepthHalf

  262. 

  263.         BiasHalf:

  264.             cbz x3, ActivationHalf

  265.             ld1 {v0.4s}, [x12], #16

  266.             ld1 {v1.4s}, [x12], #16

  267.             fadd v8.4s, v8.4s, v0.4s

  268.             fadd v10.4s, v10.4s, v0.4s

  269.             fadd v12.4s, v12.4s, v0.4s

  270.             fadd v14.4s, v14.4s, v0.4s

  271.             fadd v16.4s, v16.4s, v0.4s

  272.             fadd v18.4s, v18.4s, v0.4s

  273.             fadd v20.4s, v20.4s, v0.4s

  274.             fadd v22.4s, v22.4s, v0.4s

  275.             fadd v24.4s, v24.4s, v0.4s

  276.             fadd v26.4s, v26.4s, v0.4s

  277.             fadd v28.4s, v28.4s, v0.4s

  278.             fadd v30.4s, v30.4s, v0.4s

  279. 

  280.         ActivationHalf:

  281.             cmp x4, #3

  282.             beq Relu6Half

  283.             cmp x4, #1

  284.             beq ReluHalf

  285.             b Write

  286. 

  287.         Relu6Half:

  288.             mov w19, #6

  289.             dup v2.4s, w19

  290.             scvtf v2.4s, v2.4s

  291.             fmin v8.4s, v8.4s, v2.4s

  292.             fmin v10.4s, v10.4s, v2.4s

  293.             fmin v12.4s, v12.4s, v2.4s

  294.             fmin v14.4s, v14.4s, v2.4s

  295.             fmin v16.4s, v16.4s, v2.4s

  296.             fmin v18.4s, v18.4s, v2.4s

  297.             fmin v20.4s, v20.4s, v2.4s

  298.             fmin v22.4s, v22.4s, v2.4s

  299.             fmin v24.4s, v24.4s, v2.4s

  300.             fmin v26.4s, v26.4s, v2.4s

  301.             fmin v28.4s, v28.4s, v2.4s

  302.             fmin v30.4s, v30.4s, v2.4s

  303. 

  304.         ReluHalf:

  305.             dup v3.4s, wzr

  306.             fmax v8.4s, v8.4s, v3.4s

  307.             fmax v10.4s, v10.4s, v3.4s

  308.             fmax v12.4s, v12.4s, v3.4s

  309.             fmax v14.4s, v14.4s, v3.4s

  310.             fmax v16.4s, v16.4s, v3.4s

  311.             fmax v18.4s, v18.4s, v3.4s

  312.             fmax v20.4s, v20.4s, v3.4s

  313.             fmax v22.4s, v22.4s, v3.4s

  314.             fmax v24.4s, v24.4s, v3.4s

  315.             fmax v26.4s, v26.4s, v3.4s

  316.             fmax v28.4s, v28.4s, v3.4s

  317.             fmax v30.4s, v30.4s, v3.4s

  318.             b Write

  319. 

  320. 

  321. LoopRow8:

  322.     mov x14, x1 // reload rhs ptr

  323.     mov x13, x7 // reload rhs col

  324.     mov x12, x3 // reload bias

  325. 

  326.     LoopCol8:

  327.         cbz x9, NoReloadDst8

  328.         mov x11, x2

  329.     NoReloadDst8:

  330.         mov x10, x0 // reload lhs ptr

  331.         mov x19, x5 // reload depth

  332. 

  333.         cmp x13, #4

  334.         ble LoopDepthStartHalf8

  335. 

  336.     LoopDepthStart8:

  337.         ld1 {v0.4s, v1.4s, v2.4s}, [x10], #48

  338.         ld1 {v3.4s, v4.4s}, [x14], #32

  339.         fmul v8.4s, v3.4s, v0.s[0]

  340.         fmul v10.4s, v3.4s, v0.s[1]

  341.         fmul v12.4s, v3.4s, v0.s[2]

  342.         fmul v14.4s, v3.4s, v0.s[3]

  343.         fmul v9.4s, v4.4s, v0.s[0]

  344.         fmul v11.4s, v4.4s, v0.s[1]

  345.         fmul v13.4s, v4.4s, v0.s[2]

  346.         fmul v15.4s, v4.4s, v0.s[3]

  347.         fmul v16.4s, v3.4s, v1.s[0]

  348.         fmul v18.4s, v3.4s, v1.s[1]

  349.         fmul v20.4s, v3.4s, v1.s[2]

  350.         fmul v22.4s, v3.4s, v1.s[3]

  351.         fmul v17.4s, v4.4s, v1.s[0]

  352.         fmul v19.4s, v4.4s, v1.s[1]

  353.         fmul v21.4s, v4.4s, v1.s[2]

  354.         fmul v23.4s, v4.4s, v1.s[3]

  355. 

  356.         subs x19, x19, #1

  357.         beq Bias8

  358. 

  359.         LoopDepth8:

  360.             ld1 {v0.4s, v1.4s, v2.4s}, [x10], #48

  361.             ld1 {v3.4s, v4.4s}, [x14], #32

  362.             fmla v8.4s, v3.4s, v0.s[0]

  363.             fmla v10.4s, v3.4s, v0.s[1]

  364.             fmla v12.4s, v3.4s, v0.s[2]

  365.             fmla v14.4s, v3.4s, v0.s[3]

  366.             fmla v9.4s, v4.4s, v0.s[0]

  367.             fmla v11.4s, v4.4s, v0.s[1]

  368.             fmla v13.4s, v4.4s, v0.s[2]

  369.             fmla v15.4s, v4.4s, v0.s[3]

  370.             fmla v16.4s, v3.4s, v1.s[0]

  371.             fmla v18.4s, v3.4s, v1.s[1]

  372.             fmla v20.4s, v3.4s, v1.s[2]

  373.             fmla v22.4s, v3.4s, v1.s[3]

  374.             fmla v17.4s, v4.4s, v1.s[0]

  375.             fmla v19.4s, v4.4s, v1.s[1]

  376.             fmla v21.4s, v4.4s, v1.s[2]

  377.             fmla v23.4s, v4.4s, v1.s[3]

  378. 

  379.             subs x19, x19, #1

  380.             bgt LoopDepth8

  381. 

  382.         Bias8:

  383.             cbz x3, Activation8

  384.             ld1 {v0.4s}, [x12], #16

  385.             ld1 {v1.4s}, [x12], #16

  386.             fadd v8.4s, v8.4s, v0.4s

  387.             fadd v9.4s, v9.4s, v1.4s

  388.             fadd v10.4s, v10.4s, v0.4s

  389.             fadd v11.4s, v11.4s, v1.4s

  390.             fadd v12.4s, v12.4s, v0.4s

  391.             fadd v13.4s, v13.4s, v1.4s

  392.             fadd v14.4s, v14.4s, v0.4s

  393.             fadd v15.4s, v15.4s, v1.4s

  394.             fadd v16.4s, v16.4s, v0.4s

  395.             fadd v17.4s, v17.4s, v1.4s

  396.             fadd v18.4s, v18.4s, v0.4s

  397.             fadd v19.4s, v19.4s, v1.4s

  398.             fadd v20.4s, v20.4s, v0.4s

  399.             fadd v21.4s, v21.4s, v1.4s

  400.             fadd v22.4s, v22.4s, v0.4s

  401.             fadd v23.4s, v23.4s, v1.4s

  402. 

  403.         Activation8:

  404.             cmp x4, #3

  405.             beq Relu68

  406.             cmp x4, #1

  407.             beq Relu8

  408.             b Write

  409. 

  410.         Relu68:

  411.             mov w19, #6

  412.             dup v2.4s, w19

  413.             scvtf v2.4s, v2.4s

  414.             fmin v8.4s, v8.4s, v2.4s

  415.             fmin v9.4s, v9.4s, v2.4s

  416.             fmin v10.4s, v10.4s, v2.4s

  417.             fmin v11.4s, v11.4s, v2.4s

  418.             fmin v12.4s, v12.4s, v2.4s

  419.             fmin v13.4s, v13.4s, v2.4s

  420.             fmin v14.4s, v14.4s, v2.4s

  421.             fmin v15.4s, v15.4s, v2.4s

  422.             fmin v16.4s, v16.4s, v2.4s

  423.             fmin v17.4s, v17.4s, v2.4s

  424.             fmin v18.4s, v18.4s, v2.4s

  425.             fmin v19.4s, v19.4s, v2.4s

  426.             fmin v20.4s, v20.4s, v2.4s

  427.             fmin v21.4s, v21.4s, v2.4s

  428.             fmin v22.4s, v22.4s, v2.4s

  429.             fmin v23.4s, v23.4s, v2.4s

  430.         

  431.         Relu8:

  432.             dup v3.4s, wzr

  433.             fmax v8.4s, v8.4s, v3.4s

  434.             fmax v9.4s, v9.4s, v3.4s

  435.             fmax v10.4s, v10.4s, v3.4s

  436.             fmax v11.4s, v11.4s, v3.4s

  437.             fmax v12.4s, v12.4s, v3.4s

  438.             fmax v13.4s, v13.4s, v3.4s

  439.             fmax v14.4s, v14.4s, v3.4s

  440.             fmax v15.4s, v15.4s, v3.4s

  441.             fmax v16.4s, v16.4s, v3.4s

  442.             fmax v17.4s, v17.4s, v3.4s

  443.             fmax v18.4s, v18.4s, v3.4s

  444.             fmax v19.4s, v19.4s, v3.4s

  445.             fmax v20.4s, v20.4s, v3.4s

  446.             fmax v21.4s, v21.4s, v3.4s

  447.             fmax v22.4s, v22.4s, v3.4s

  448.             fmax v23.4s, v23.4s, v3.4s

  449.             b Write

  450. 

  451.     LoopDepthStartHalf8:

  452.         ld1 {v0.4s, v1.4s, v2.4s}, [x10], #48

  453.         ld1 {v3.4s, v4.4s}, [x14], #32

  454.         fmul v8.4s, v3.4s, v0.s[0]

  455.         fmul v10.4s, v3.4s, v0.s[1]

  456.         fmul v12.4s, v3.4s, v0.s[2]

  457.         fmul v14.4s, v3.4s, v0.s[3]

  458.         fmul v16.4s, v3.4s, v1.s[0]

  459.         fmul v18.4s, v3.4s, v1.s[1]

  460.         fmul v20.4s, v3.4s, v1.s[2]

  461.         fmul v22.4s, v3.4s, v1.s[3]

  462. 

  463.         subs x19, x19, #1

  464.         beq BiasHalf8

  465. 

  466.         LoopDepthHalf8:

  467.             ld1 {v0.4s, v1.4s, v2.4s}, [x10], #48

  468.             ld1 {v3.4s, v4.4s}, [x14], #32

  469.             fmla v8.4s, v3.4s, v0.s[0]

  470.             fmla v10.4s, v3.4s, v0.s[1]

  471.             fmla v12.4s, v3.4s, v0.s[2]

  472.             fmla v14.4s, v3.4s, v0.s[3]

  473.             fmla v16.4s, v3.4s, v1.s[0]

  474.             fmla v18.4s, v3.4s, v1.s[1]

  475.             fmla v20.4s, v3.4s, v1.s[2]

  476.             fmla v22.4s, v3.4s, v1.s[3]

  477. 

  478.             subs x19, x19, #1

  479.             bgt LoopDepthHalf8

  480. 

  481.         BiasHalf8:

  482.             cbz x3, ActivationHalf8

  483.             ld1 {v0.4s}, [x12], #16

  484.             ld1 {v1.4s}, [x12], #16

  485.             fadd v8.4s, v8.4s, v0.4s

  486.             fadd v10.4s, v10.4s, v0.4s

  487.             fadd v12.4s, v12.4s, v0.4s

  488.             fadd v14.4s, v14.4s, v0.4s

  489.             fadd v16.4s, v16.4s, v0.4s

  490.             fadd v18.4s, v18.4s, v0.4s

  491.             fadd v20.4s, v20.4s, v0.4s

  492.             fadd v22.4s, v22.4s, v0.4s

  493. 

  494.         ActivationHalf8:

  495.             cmp x4, #3

  496.             beq Relu6Half8

  497.             cmp x4, #1

  498.             beq ReluHalf8

  499.             b Write

  500. 

  501.         Relu6Half8:

  502.             mov w19, #6

  503.             dup v2.4s, w19

  504.             scvtf v2.4s, v2.4s

  505.             fmin v8.4s, v8.4s, v2.4s

  506.             fmin v10.4s, v10.4s, v2.4s

  507.             fmin v12.4s, v12.4s, v2.4s

  508.             fmin v14.4s, v14.4s, v2.4s

  509.             fmin v16.4s, v16.4s, v2.4s

  510.             fmin v18.4s, v18.4s, v2.4s

  511.             fmin v20.4s, v20.4s, v2.4s

  512.             fmin v22.4s, v22.4s, v2.4s

  513. 

  514.         ReluHalf8:

  515.             dup v3.4s, wzr

  516.             fmax v8.4s, v8.4s, v3.4s

  517.             fmax v10.4s, v10.4s, v3.4s

  518.             fmax v12.4s, v12.4s, v3.4s

  519.             fmax v14.4s, v14.4s, v3.4s

  520.             fmax v16.4s, v16.4s, v3.4s

  521.             fmax v18.4s, v18.4s, v3.4s

  522.             fmax v20.4s, v20.4s, v3.4s

  523.             fmax v22.4s, v22.4s, v3.4s

  524.             b Write

  525. 

  526. LoopRow4:

  527.     mov x14, x1 // reload rhs ptr

  528.     mov x13, x7 // reload rhs col

  529.     mov x12, x3 // reload bias

  530. 

  531.     LoopCol4:

  532.         cbz x9, NoReloadDst4

  533.         mov x11, x2

  534.     NoReloadDst4:

  535.         mov x10, x0 // reload lhs ptr

  536.         mov x19, x5 // reload depth

  537. 

  538.         cmp x13, #4

  539.         ble LoopDepthStartHalf4

  540. 

  541.     LoopDepthStart4:

  542.         ld1 {v0.4s, v1.4s, v2.4s}, [x10], #48

  543.         ld1 {v3.4s, v4.4s}, [x14], #32

  544.         fmul v8.4s, v3.4s, v0.s[0]

  545.         fmul v10.4s, v3.4s, v0.s[1]

  546.         fmul v12.4s, v3.4s, v0.s[2]

  547.         fmul v14.4s, v3.4s, v0.s[3]

  548.         fmul v9.4s, v4.4s, v0.s[0]

  549.         fmul v11.4s, v4.4s, v0.s[1]

  550.         fmul v13.4s, v4.4s, v0.s[2]

  551.         fmul v15.4s, v4.4s, v0.s[3]

  552. 

  553.         subs x19, x19, #1

  554.         beq Bias4

  555. 

  556.         LoopDepth4:

  557.             ld1 {v0.4s, v1.4s, v2.4s}, [x10], #48

  558.             ld1 {v3.4s, v4.4s}, [x14], #32

  559.             fmla v8.4s, v3.4s, v0.s[0]

  560.             fmla v10.4s, v3.4s, v0.s[1]

  561.             fmla v12.4s, v3.4s, v0.s[2]

  562.             fmla v14.4s, v3.4s, v0.s[3]

  563.             fmla v9.4s, v4.4s, v0.s[0]

  564.             fmla v11.4s, v4.4s, v0.s[1]

  565.             fmla v13.4s, v4.4s, v0.s[2]

  566.             fmla v15.4s, v4.4s, v0.s[3]

  567. 

  568.             subs x19, x19, #1

  569.             bgt LoopDepth4

  570. 

  571.         Bias4:

  572.             cbz x3, Activation4

  573.             ld1 {v0.4s}, [x12], #16

  574.             ld1 {v1.4s}, [x12], #16

  575.             fadd v8.4s, v8.4s, v0.4s

  576.             fadd v9.4s, v9.4s, v1.4s

  577.             fadd v10.4s, v10.4s, v0.4s

  578.             fadd v11.4s, v11.4s, v1.4s

  579.             fadd v12.4s, v12.4s, v0.4s

  580.             fadd v13.4s, v13.4s, v1.4s

  581.             fadd v14.4s, v14.4s, v0.4s

  582.             fadd v15.4s, v15.4s, v1.4s

  583. 

  584.         Activation4:

  585.             cmp x4, #3

  586.             beq Relu64

  587.             cmp x4, #1

  588.             beq Relu4

  589.             b Write

  590. 

  591.         Relu64:

  592.             mov w19, #6

  593.             dup v2.4s, w19

  594.             scvtf v2.4s, v2.4s

  595.             fmin v8.4s, v8.4s, v2.4s

  596.             fmin v9.4s, v9.4s, v2.4s

  597.             fmin v10.4s, v10.4s, v2.4s

  598.             fmin v11.4s, v11.4s, v2.4s

  599.             fmin v12.4s, v12.4s, v2.4s

  600.             fmin v13.4s, v13.4s, v2.4s

  601.             fmin v14.4s, v14.4s, v2.4s

  602.             fmin v15.4s, v15.4s, v2.4s

  603.         

  604.         Relu4:

  605.             dup v3.4s, wzr

  606.             fmax v8.4s, v8.4s, v3.4s

  607.             fmax v9.4s, v9.4s, v3.4s

  608.             fmax v10.4s, v10.4s, v3.4s

  609.             fmax v11.4s, v11.4s, v3.4s

  610.             fmax v12.4s, v12.4s, v3.4s

  611.             fmax v13.4s, v13.4s, v3.4s

  612.             fmax v14.4s, v14.4s, v3.4s

  613.             fmax v15.4s, v15.4s, v3.4s

  614.             b Write

  615. 

  616.     LoopDepthStartHalf4:

  617.         ld1 {v0.4s, v1.4s, v2.4s}, [x10], #48

  618.         ld1 {v3.4s, v4.4s}, [x14], #32

  619.         fmul v8.4s, v3.4s, v0.s[0]

  620.         fmul v10.4s, v3.4s, v0.s[1]

  621.         fmul v12.4s, v3.4s, v0.s[2]

  622.         fmul v14.4s, v3.4s, v0.s[3]

  623. 

  624.         subs x19, x19, #1

  625.         beq BiasHalf4

  626. 

  627.         LoopDepthHalf4:

  628.             ld1 {v0.4s, v1.4s, v2.4s}, [x10], #48

  629.             ld1 {v3.4s, v4.4s}, [x14], #32

  630.             fmla v8.4s, v3.4s, v0.s[0]

  631.             fmla v10.4s, v3.4s, v0.s[1]

  632.             fmla v12.4s, v3.4s, v0.s[2]

  633.             fmla v14.4s, v3.4s, v0.s[3]

  634. 

  635.             subs x19, x19, #1

  636.             bgt LoopDepthHalf4

  637. 

  638.         BiasHalf4:

  639.             cbz x3, ActivationHalf4

  640.             ld1 {v0.4s}, [x12], #16

  641.             ld1 {v1.4s}, [x12], #16

  642.             fadd v8.4s, v8.4s, v0.4s

  643.             fadd v10.4s, v10.4s, v0.4s

  644.             fadd v12.4s, v12.4s, v0.4s

  645.             fadd v14.4s, v14.4s, v0.4s

  646. 

  647.         ActivationHalf4:

  648.             cmp x4, #3

  649.             beq Relu6Half4

  650.             cmp x4, #1

  651.             beq ReluHalf4

  652.             b Write

  653. 

  654.         Relu6Half4:

  655.             mov w19, #6

  656.             dup v2.4s, w19

  657.             scvtf v2.4s, v2.4s

  658.             fmin v8.4s, v8.4s, v2.4s

  659.             fmin v10.4s, v10.4s, v2.4s

  660.             fmin v12.4s, v12.4s, v2.4s

  661.             fmin v14.4s, v14.4s, v2.4s

  662. 

  663.         ReluHalf4:

  664.             dup v3.4s, wzr

  665.             fmax v8.4s, v8.4s, v3.4s

  666.             fmax v10.4s, v10.4s, v3.4s

  667.             fmax v12.4s, v12.4s, v3.4s

  668.             fmax v14.4s, v14.4s, v3.4s

  669. 

  670.         Write:

  671.             cmp x9, #2

  672.             beq WriteWino

  673.             cbz x9, WriteC8

  674.             cmp x13, #1

  675.             beq Write1

  676.             cmp x13, #2

  677.             beq Write2

  678.             cmp x13, #3

  679.             beq Write3

  680.             cmp x13, #4

  681.             beq Write4

  682.             cmp x13, #5

  683.             beq Write5

  684.             cmp x13, #6

  685.             beq Write6

  686.             cmp x13, #7

  687.             beq Write7

  688.             b Write8

  689. 

  690.         Write1:

  691.             add x2, x2, #4

  692.             str s8, [x11]

  693.             cmp x6, #1

  694.             beq WriteEnd

  695.             add x11, x11, x8

  696.             str s10, [x11]

  697.             cmp x6, #2

  698.             beq WriteEnd

  699.             add x11, x11, x8

  700.             str s12, [x11]

  701.             cmp x6, #3

  702.             beq WriteEnd

  703.             add x11, x11, x8

  704.             str s14, [x11]

  705.             cmp x6, #4

  706.             beq WriteEnd

  707.             add x11, x11, x8

  708.             str s16, [x11]

  709.             cmp x6, #5

  710.             beq WriteEnd

  711.             add x11, x11, x8

  712.             str s18, [x11]

  713.             cmp x6, #6

  714.             beq WriteEnd

  715.             add x11, x11, x8

  716.             str s20, [x11]

  717.             cmp x6, #7

  718.             beq WriteEnd

  719.             add x11, x11, x8

  720.             str s22, [x11]

  721.             cmp x6, #8

  722.             beq WriteEnd

  723.             add x11, x11, x8

  724.             str s24, [x11]

  725.             cmp x6, #9

  726.             beq WriteEnd

  727.             add x11, x11, x8

  728.             str s26, [x11]

  729.             cmp x6, #10

  730.             beq WriteEnd

  731.             add x11, x11, x8

  732.             str s28, [x11]

  733.             cmp x6, #11

  734.             beq WriteEnd

  735.             add x11, x11, x8

  736.             str s30, [x11]

  737.             add x11, x11, x8

  738.             add x11, x11, #4

  739.             b WriteEnd

  740.         Write2:

  741.             add x2, x2, #8

  742.             str d8, [x11]

  743.             cmp x6, #1

  744.             beq WriteEnd

  745.             add x11, x11, x8

  746.             str d10, [x11]

  747.             cmp x6, #2

  748.             beq WriteEnd

  749.             add x11, x11, x8

  750.             str d12, [x11]

  751.             cmp x6, #3

  752.             beq WriteEnd

  753.             add x11, x11, x8

  754.             str d14, [x11]

  755.             cmp x6, #4

  756.             beq WriteEnd

  757.             add x11, x11, x8

  758.             str d16, [x11]

  759.             cmp x6, #5

  760.             beq WriteEnd

  761.             add x11, x11, x8

  762.             str d18, [x11]

  763.             cmp x6, #6

  764.             beq WriteEnd

  765.             add x11, x11, x8

  766.             str d20, [x11]

  767.             cmp x6, #7

  768.             beq WriteEnd

  769.             add x11, x11, x8

  770.             str d22, [x11]

  771.             cmp x6, #8

  772.             beq WriteEnd

  773.             add x11, x11, x8

  774.             str d24, [x11]

  775.             cmp x6, #9

  776.             beq WriteEnd

  777.             add x11, x11, x8

  778.             str d26, [x11]

  779.             cmp x6, #10

  780.             beq WriteEnd

  781.             add x11, x11, x8

  782.             str d28, [x11]

  783.             cmp x6, #11

  784.             beq WriteEnd

  785.             add x11, x11, x8

  786.             str d30, [x11]

  787.             add x11, x11, x8

  788.             add x11, x11, #8

  789.             b WriteEnd

  790.         Write3:

  791.             add x2, x2, #12

  792.             add x19, x11, #8

  793.             str d8, [x11]

  794.             st1 {v8.s}[2], [x19], x8

  795.             cmp x6, #1

  796.             beq WriteEnd

  797.             add x11, x11, x8

  798.             str d10, [x11]

  799.             st1 {v10.s}[2], [x19], x8

  800.             cmp x6, #2

  801.             beq WriteEnd

  802.             add x11, x11, x8

  803.             str d12, [x11]

  804.             st1 {v12.s}[2], [x19], x8

  805.             cmp x6, #3

  806.             beq WriteEnd

  807.             add x11, x11, x8

  808.             str d14, [x11]

  809.             st1 {v14.s}[2], [x19], x8

  810.             cmp x6, #4

  811.             beq WriteEnd

  812.             add x11, x11, x8

  813.             str d16, [x11]

  814.             st1 {v16.s}[2], [x19], x8

  815.             cmp x6, #5

  816.             beq WriteEnd

  817.             add x11, x11, x8

  818.             str d18, [x11]

  819.             st1 {v18.s}[2], [x19], x8

  820.             cmp x6, #6

  821.             beq WriteEnd

  822.             add x11, x11, x8

  823.             str d20, [x11]

  824.             st1 {v20.s}[2], [x19], x8

  825.             cmp x6, #7

  826.             beq WriteEnd

  827.             add x11, x11, x8

  828.             str d22, [x11]

  829.             st1 {v22.s}[2], [x19], x8

  830.             cmp x6, #8

  831.             beq WriteEnd

  832.             add x11, x11, x8

  833.             str d24, [x11]

  834.             st1 {v24.s}[2], [x19], x8

  835.             cmp x6, #9

  836.             beq WriteEnd

  837.             add x11, x11, x8

  838.             str d26, [x11]

  839.             st1 {v26.s}[2], [x19], x8

  840.             cmp x6, #10

  841.             beq WriteEnd

  842.             add x11, x11, x8

  843.             str d28, [x11]

  844.             st1 {v28.s}[2], [x19], x8

  845.             cmp x6, #11

  846.             beq WriteEnd

  847.             add x11, x11, x8

  848.             str d30, [x11]

  849.             st1 {v30.s}[2], [x19]

  850.             add x11, x11, x8

  851.             add x11, x11, #12

  852.             b WriteEnd

  853.         Write4:

  854.             add x2, x2, #16

  855.             st1 {v8.4s}, [x11], x8

  856.             cmp x6, #1

  857.             beq WriteEnd

  858.             st1 {v10.4s}, [x11], x8

  859.             cmp x6, #2

  860.             beq WriteEnd

  861.             st1 {v12.4s}, [x11], x8

  862.             cmp x6, #3

  863.             beq WriteEnd

  864.             st1 {v14.4s}, [x11], x8

  865.             cmp x6, #4

  866.             beq WriteEnd

  867.             st1 {v16.4s}, [x11], x8

  868.             cmp x6, #5

  869.             beq WriteEnd

  870.             st1 {v18.4s}, [x11], x8

  871.             cmp x6, #6

  872.             beq WriteEnd

  873.             st1 {v20.4s}, [x11], x8

  874.             cmp x6, #7

  875.             beq WriteEnd

  876.             st1 {v22.4s}, [x11], x8

  877.             cmp x6, #8

  878.             beq WriteEnd

  879.             st1 {v24.4s}, [x11], x8

  880.             cmp x6, #9

  881.             beq WriteEnd

  882.             st1 {v26.4s}, [x11], x8

  883.             cmp x6, #10

  884.             beq WriteEnd

  885.             st1 {v28.4s}, [x11], x8

  886.             cmp x6, #11

  887.             beq WriteEnd

  888.             st1 {v30.4s}, [x11], x8

  889.             add x11, x11, #16

  890.             b WriteEnd

  891.         Write5:

  892.             add x2, x2, #20

  893.             add x19, x11, #16

  894.             st1 {v8.4s}, [x11], x8

  895.             str s9, [x19]

  896.             cmp x6, #1

  897.             beq WriteEnd

  898.             add x19, x19, x8

  899.             st1 {v10.4s}, [x11], x8

  900.             str s11, [x19]

  901.             cmp x6, #2

  902.             beq WriteEnd

  903.             add x19, x19, x8

  904.             st1 {v12.4s}, [x11], x8

  905.             str s13, [x19]

  906.             cmp x6, #3

  907.             beq WriteEnd

  908.             add x19, x19, x8

  909.             st1 {v14.4s}, [x11], x8

  910.             str s15, [x19]

  911.             cmp x6, #4

  912.             beq WriteEnd

  913.             add x19, x19, x8

  914.             st1 {v16.4s}, [x11], x8

  915.             str s17, [x19]

  916.             cmp x6, #5

  917.             beq WriteEnd

  918.             add x19, x19, x8

  919.             st1 {v18.4s}, [x11], x8

  920.             str s19, [x19]

  921.             cmp x6, #6

  922.             beq WriteEnd

  923.             add x19, x19, x8

  924.             st1 {v20.4s}, [x11], x8

  925.             str s21, [x19]

  926.             cmp x6, #7

  927.             beq WriteEnd

  928.             add x19, x19, x8

  929.             st1 {v22.4s}, [x11], x8

  930.             str s23, [x19]

  931.             cmp x6, #8

  932.             beq WriteEnd

  933.             add x19, x19, x8

  934.             st1 {v24.4s}, [x11], x8

  935.             str s25, [x19]

  936.             cmp x6, #9

  937.             beq WriteEnd

  938.             add x19, x19, x8

  939.             st1 {v26.4s}, [x11], x8

  940.             str s27, [x19]

  941.             cmp x6, #10

  942.             beq WriteEnd

  943.             add x19, x19, x8

  944.             st1 {v28.4s}, [x11], x8

  945.             str s29, [x19]

  946.             cmp x6, #11

  947.             beq WriteEnd

  948.             add x19, x19, x8

  949.             st1 {v30.4s}, [x11], x8

  950.             str s31, [x19]

  951.             add x11, x11, #20

  952.             b WriteEnd

  953.         Write6:

  954.             add x2, x2, #24

  955.             add x19, x11, #16

  956.             st1 {v8.4s}, [x11], x8

  957.             str d9, [x19]

  958.             cmp x6, #1

  959.             beq WriteEnd

  960.             add x19, x19, x8

  961.             st1 {v10.4s}, [x11], x8

  962.             str d11, [x19]

  963.             cmp x6, #2

  964.             beq WriteEnd

  965.             add x19, x19, x8

  966.             st1 {v12.4s}, [x11], x8

  967.             str d13, [x19]

  968.             cmp x6, #3

  969.             beq WriteEnd

  970.             add x19, x19, x8

  971.             st1 {v14.4s}, [x11], x8

  972.             str d15, [x19]

  973.             cmp x6, #4

  974.             beq WriteEnd

  975.             add x19, x19, x8

  976.             st1 {v16.4s}, [x11], x8

  977.             str d17, [x19]

  978.             cmp x6, #5

  979.             beq WriteEnd

  980.             add x19, x19, x8

  981.             st1 {v18.4s}, [x11], x8

  982.             str d19, [x19]

  983.             cmp x6, #6

  984.             beq WriteEnd

  985.             add x19, x19, x8

  986.             st1 {v20.4s}, [x11], x8

  987.             str d21, [x19]

  988.             cmp x6, #7

  989.             beq WriteEnd

  990.             add x19, x19, x8

  991.             st1 {v22.4s}, [x11], x8

  992.             str d23, [x19]

  993.             cmp x6, #8

  994.             beq WriteEnd

  995.             add x19, x19, x8

  996.             st1 {v24.4s}, [x11], x8

  997.             str d25, [x19]

  998.             cmp x6, #9

  999.             beq WriteEnd

  1000.             add x19, x19, x8

  1001.             st1 {v26.4s}, [x11], x8

  1002.             str d27, [x19]

  1003.             cmp x6, #10

  1004.             beq WriteEnd

  1005.             add x19, x19, x8

  1006.             st1 {v28.4s}, [x11], x8

  1007.             str d29, [x19]

  1008.             cmp x6, #11

  1009.             beq WriteEnd

  1010.             add x19, x19, x8

  1011.             st1 {v30.4s}, [x11], x8

  1012.             str d31, [x19]

  1013.             add x11, x11, #24

  1014.             b WriteEnd

  1015.         Write7:

  1016.             add x2, x2, #28

  1017.             add x19, x11, #16

  1018.             add x20, x11, #24

  1019.             st1 {v8.4s}, [x11], x8

  1020.             str d9, [x19]

  1021.             st1 {v9.s}[2], [x20], x8

  1022.             cmp x6, #1

  1023.             beq WriteEnd

  1024.             add x19, x19, x8

  1025.             st1 {v10.4s}, [x11], x8

  1026.             str d11, [x19]

  1027.             st1 {v11.s}[2], [x20], x8

  1028.             cmp x6, #2

  1029.             beq WriteEnd

  1030.             add x19, x19, x8

  1031.             st1 {v12.4s}, [x11], x8

  1032.             str d13, [x19]

  1033.             st1 {v13.s}[2], [x20], x8

  1034.             cmp x6, #3

  1035.             beq WriteEnd

  1036.             add x19, x19, x8

  1037.             st1 {v14.4s}, [x11], x8

  1038.             str d15, [x19]

  1039.             st1 {v15.s}[2], [x20], x8

  1040.             cmp x6, #4

  1041.             beq WriteEnd

  1042.             add x19, x19, x8

  1043.             st1 {v16.4s}, [x11], x8

  1044.             str d17, [x19]

  1045.             st1 {v17.s}[2], [x20], x8

  1046.             cmp x6, #5

  1047.             beq WriteEnd

  1048.             add x19, x19, x8

  1049.             st1 {v18.4s}, [x11], x8

  1050.             str d19, [x19]

  1051.             st1 {v19.s}[2], [x20], x8

  1052.             cmp x6, #6

  1053.             beq WriteEnd

  1054.             add x19, x19, x8

  1055.             st1 {v20.4s}, [x11], x8

  1056.             str d21, [x19]

  1057.             st1 {v21.s}[2], [x20], x8

  1058.             cmp x6, #7

  1059.             beq WriteEnd

  1060.             add x19, x19, x8

  1061.             st1 {v22.4s}, [x11], x8

  1062.             str d23, [x19]

  1063.             st1 {v23.s}[2], [x20], x8

  1064.             cmp x6, #8

  1065.             beq WriteEnd

  1066.             add x19, x19, x8

  1067.             st1 {v24.4s}, [x11], x8

  1068.             str d25, [x19]

  1069.             st1 {v25.s}[2], [x20], x8

  1070.             cmp x6, #9

  1071.             beq WriteEnd

  1072.             add x19, x19, x8

  1073.             st1 {v26.4s}, [x11], x8

  1074.             str d27, [x19]

  1075.             st1 {v27.s}[2], [x20], x8

  1076.             cmp x6, #10

  1077.             beq WriteEnd

  1078.             add x19, x19, x8

  1079.             st1 {v28.4s}, [x11], x8

  1080.             str d29, [x19]

  1081.             st1 {v29.s}[2], [x20], x8

  1082.             cmp x6, #11

  1083.             beq WriteEnd

  1084.             add x19, x19, x8

  1085.             st1 {v30.4s}, [x11], x8

  1086.             str d31, [x19]

  1087.             add x19, x19, x8

  1088.             st1 {v31.s}[2], [x20], x8

  1089.             add x11, x11, #28

  1090.             b WriteEnd

  1091.         WriteC8:

  1092.             mov x19, x11

  1093.             st1 {v8.4s, v9.4s, v10.4s, v11.4s}, [x19], #64

  1094.             st1 {v12.4s, v13.4s, v14.4s, v15.4s}, [x19], #64

  1095.             st1 {v16.4s, v17.4s, v18.4s, v19.4s}, [x19], #64

  1096.             st1 {v20.4s, v21.4s, v22.4s, v23.4s}, [x19], #64

  1097.             st1 {v24.4s, v25.4s, v26.4s, v27.4s}, [x19], #64

  1098.             st1 {v28.4s, v29.4s, v30.4s, v31.4s}, [x19], #64

  1099.             add x11, x11, x16

  1100.             b WriteEnd

  1101.         WriteWino:

  1102.             add x2, x11, x16

  1103.             st1 {v8.4s, v9.4s}, [x11], x15

  1104.             st1 {v10.4s, v11.4s}, [x11], x15

  1105.             st1 {v12.4s, v13.4s}, [x11], x15

  1106.             st1 {v14.4s, v15.4s}, [x11], x15

  1107.             st1 {v16.4s, v17.4s}, [x11], x15

  1108.             st1 {v18.4s, v19.4s}, [x11], x15

  1109.             st1 {v20.4s, v21.4s}, [x11], x15

  1110.             st1 {v22.4s, v23.4s}, [x11], x15

  1111.             st1 {v24.4s, v25.4s}, [x11], x15

  1112.             st1 {v26.4s, v27.4s}, [x11], x15

  1113.             st1 {v28.4s, v29.4s}, [x11], x15

  1114.             st1 {v30.4s, v31.4s}, [x11], x15

  1115.             b WriteEnd

  1116.         Write8:

  1117.             add x2, x2, #32

  1118.             st1 {v8.4s, v9.4s}, [x11], x8

  1119.             cmp x6, #1

  1120.             beq WriteEnd

  1121.             st1 {v10.4s, v11.4s}, [x11], x8

  1122.             cmp x6, #2

  1123.             beq WriteEnd

  1124.             st1 {v12.4s, v13.4s}, [x11], x8

  1125.             cmp x6, #3

  1126.             beq WriteEnd

  1127.             st1 {v14.4s, v15.4s}, [x11], x8

  1128.             cmp x6, #4

  1129.             beq WriteEnd

  1130.             st1 {v16.4s, v17.4s}, [x11], x8

  1131.             cmp x6, #5

  1132.             beq WriteEnd

  1133.             st1 {v18.4s, v19.4s}, [x11], x8

  1134.             cmp x6, #6

  1135.             beq WriteEnd

  1136.             st1 {v20.4s, v21.4s}, [x11], x8

  1137.             cmp x6, #7

  1138.             beq WriteEnd

  1139.             st1 {v22.4s, v23.4s}, [x11], x8

  1140.             cmp x6, #8

  1141.             beq WriteEnd

  1142.             st1 {v24.4s, v25.4s}, [x11], x8

  1143.             cmp x6, #9

  1144.             beq WriteEnd

  1145.             st1 {v26.4s, v27.4s}, [x11], x8

  1146.             cmp x6, #10

  1147.             beq WriteEnd

  1148.             st1 {v28.4s, v29.4s}, [x11], x8

  1149.             cmp x6, #11

  1150.             beq WriteEnd

  1151.             st1 {v30.4s, v31.4s}, [x11], x8

  1152.             add x11, x11, #32

  1153. 

  1154.         WriteEnd:

  1155.             subs x13, x13, #8 // rhs col - 8

  1156.             ble LoopColEnd

  1157.             cmp x6, #4

  1158.             ble LoopCol4

  1159.             cmp x6, #8

  1160.             ble LoopCol8

  1161.             b LoopCol

  1162. 

  1163. LoopColEnd:

  1164.         add x0, x0, x17

  1165.         cbz x9, C8DstStep

  1166.         mov x18, #4

  1167.         mul x18, x18, x7

  1168.         sub x11, x11, x18

  1169.         mov x2, x11

  1170.         b NoDstStep

  1171.     C8DstStep:

  1172.         add x2, x2, #384

  1173.         mov x11, x2

  1174.     NoDstStep:

  1175.         subs x6, x6, #12

  1176.         bgt LoopRowStart

  1177. 

  1178.   sub sp, sp, #144

  1179.   ld1 {v8.4s, v9.4s, v10.4s, v11.4s}, [sp], #64

  1180.   ld1 {v12.4s, v13.4s, v14.4s, v15.4s}, [sp], #64

  1181.   ldp x19, x20, [sp], #16

  1182.   ret

  1183. #endif