zzy34407230
/
mindspore2022
Not watched
1
0
Fork 0
Code
Releases 22 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: b56fc0c2af
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'b56fc0c2af'
${ noResults }
mindspore2022/mindspore/lite/nnacl/assembly/opt/MatmulDpInt8Opt.S

1084 lines
33 kB
Raw Blame History 

  1. #ifdef __aarch64__

  2.     .text

  3.     .align 5

  4.     .global MatmulInt8DpOpt

  5. #ifndef __APPLE__

  6.     .type MatmulInt8DpOpt, %function

  7. #endif

  8. 

  9. //void MatmulInt8DpOpt(const int8_t *a, const int8_t *b, int8_t *dst, int row, int col, int deep4, const int *a_sums,

  10. //                     const int *bias, int act_min, int act_max, int out_zp, int32_t *multiplier, int32_t *left_shift,

  11. //                     int32_t *right_shift, size_t stride, size_t filter_peroc, int32_t *filter_zp)

  12. 

  13. // x0: a(left matrix ptr)

  14. // x1: b(right matrix ptr)

  15. // x2: out ptr

  16. // x3: row

  17. // x4: col

  18. // x5: deep4

  19. // x6: a_sums

  20. // x7: bias

  21. // w8: act_min

  22. // w9: act_max

  23. // w10: out_zp

  24. // x11: multiplier

  25. // x12: left_shift

  26. // x13: right_shift

  27. // x14: stride

  28. // x15: filter_peroc

  29. // x28: filter_zp

  30. 

  31. MatmulInt8DpOpt:

  32.   sub sp, sp, #208

  33.   st1 {v8.4s, v9.4s, v10.4s, v11.4s}, [sp], #64

  34.   st1 {v12.4s, v13.4s, v14.4s, v15.4s}, [sp], #64

  35.   stp x19, x20, [sp], #16

  36.   stp x21, x22, [sp], #16

  37.   stp x23, x24, [sp], #16

  38.   stp x25, x26, [sp], #16

  39.   stp x27, x28, [sp], #16

  40. 

  41.   ldr w8, [sp]

  42.   ldr w9, [sp, #8]

  43.   ldr w10, [sp, #16]

  44.   ldr x11, [sp, #24]

  45.   ldr x12, [sp, #32]

  46.   ldr x13, [sp, #40]

  47.   ldr x14, [sp, #48]

  48.   ldr x15, [sp, #56]

  49. 

  50.   mov x23, #4

  51.   mul x23, x23, x5 // lhs step

  52.   mov x24, #4

  53.   mul x24, x24, x14 // dst step

  54. 

  55. LoopRow:

  56.     mov x16, x1 // reload rhs ptr

  57.     mov x17, x4 // reload rhs col

  58.     mov x18, x7 // reload bias ptr

  59.     mov x25, x6 // reload input_sum ptr

  60.     mov x27, x2 // reload dst ptr

  61.     ldr x28, [sp, #64] // reload filter_zp

  62. 

  63.     LoopCol:

  64.         mov x19, x27 // reload dst ptr

  65.         mov x20, x0 // reload lhs ptr

  66.         mov x21, x5 // reload depth

  67. 

  68.         dup v16.4s, wzr

  69.         dup v17.4s, wzr

  70.         dup v18.4s, wzr

  71.         dup v19.4s, wzr

  72.         dup v20.4s, wzr

  73.         dup v21.4s, wzr

  74.         dup v22.4s, wzr

  75.         dup v23.4s, wzr

  76.         dup v24.4s, wzr

  77.         dup v25.4s, wzr

  78.         dup v26.4s, wzr

  79.         dup v27.4s, wzr

  80.         dup v28.4s, wzr

  81.         dup v29.4s, wzr

  82.         dup v30.4s, wzr

  83.         dup v31.4s, wzr

  84. 

  85.         cmp x17, #4

  86.         ble LoopDepthQuarter

  87.         cmp x17, #8

  88.         ble LoopDepthHalf

  89. 

  90.         LoopDepth:

  91.             ld1 {v0.16b}, [x20], #16

  92.             ld1 {v1.16b, v2.16b, v3.16b, v4.16b}, [x16], #64

  93.             sdot v16.4s, v1.16b, v0.4b[0]

  94.             sdot v17.4s, v2.16b, v0.4b[0]

  95.             sdot v18.4s, v3.16b, v0.4b[0]

  96.             sdot v19.4s, v4.16b, v0.4b[0]

  97.             sdot v20.4s, v1.16b, v0.4b[1]

  98.             sdot v21.4s, v2.16b, v0.4b[1]

  99.             sdot v22.4s, v3.16b, v0.4b[1]

  100.             sdot v23.4s, v4.16b, v0.4b[1]

  101.             sdot v24.4s, v1.16b, v0.4b[2]

  102.             sdot v25.4s, v2.16b, v0.4b[2]

  103.             sdot v26.4s, v3.16b, v0.4b[2]

  104.             sdot v27.4s, v4.16b, v0.4b[2]

  105.             sdot v28.4s, v1.16b, v0.4b[3]

  106.             sdot v29.4s, v2.16b, v0.4b[3]

  107.             sdot v30.4s, v3.16b, v0.4b[3]

  108.             sdot v31.4s, v4.16b, v0.4b[3]

  109. 

  110.             subs x21, x21, #4

  111.             bgt LoopDepth

  112. 

  113.         Bias:

  114.             cbz x7, NoReadBias

  115.             ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [x18], #64

  116.             add v16.4s, v16.4s, v0.4s

  117.             add v17.4s, v17.4s, v1.4s

  118.             add v18.4s, v18.4s, v2.4s

  119.             add v19.4s, v19.4s, v3.4s

  120.             add v20.4s, v20.4s, v0.4s

  121.             add v21.4s, v21.4s, v1.4s

  122.             add v22.4s, v22.4s, v2.4s

  123.             add v23.4s, v23.4s, v3.4s

  124.             add v24.4s, v24.4s, v0.4s

  125.             add v25.4s, v25.4s, v1.4s

  126.             add v26.4s, v26.4s, v2.4s

  127.             add v27.4s, v27.4s, v3.4s

  128.             add v28.4s, v28.4s, v0.4s

  129.             add v29.4s, v29.4s, v1.4s

  130.             add v30.4s, v30.4s, v2.4s

  131.             add v31.4s, v31.4s, v3.4s

  132. 

  133.         NoReadBias:

  134.             ld1r {v12.4s}, [x25], #4

  135.             ld1r {v13.4s}, [x25], #4

  136.             ld1r {v14.4s}, [x25], #4

  137.             ld1r {v15.4s}, [x25], #4

  138.             cbnz x15, PerChannelSum

  139. 

  140.         PerTensorSum:

  141.             sub v16.4s, v16.4s, v12.4s

  142.             sub v17.4s, v17.4s, v12.4s

  143.             sub v18.4s, v18.4s, v12.4s

  144.             sub v19.4s, v19.4s, v12.4s

  145.             sub v20.4s, v20.4s, v13.4s

  146.             sub v21.4s, v21.4s, v13.4s

  147.             sub v22.4s, v22.4s, v13.4s

  148.             sub v23.4s, v23.4s, v13.4s

  149.             sub v24.4s, v24.4s, v14.4s

  150.             sub v25.4s, v25.4s, v14.4s

  151.             sub v26.4s, v26.4s, v14.4s

  152.             sub v27.4s, v27.4s, v14.4s

  153.             sub v28.4s, v28.4s, v15.4s

  154.             sub v29.4s, v29.4s, v15.4s

  155.             sub v30.4s, v30.4s, v15.4s

  156.             sub v31.4s, v31.4s, v15.4s

  157. 

  158.             b PerTensor

  159. 

  160.         PerChannelSum:

  161.             ld1 {v8.4s, v9.4s, v10.4s, v11.4s}, [x28], #64

  162.             mul v0.4s, v8.4s, v12.4s

  163.             mul v1.4s, v9.4s, v12.4s

  164.             mul v2.4s, v10.4s, v12.4s

  165.             mul v3.4s, v11.4s, v12.4s

  166.             mul v4.4s, v8.4s, v13.4s

  167.             mul v5.4s, v9.4s, v13.4s

  168.             mul v6.4s, v10.4s, v13.4s

  169.             mul v7.4s, v11.4s, v13.4s

  170.             sub v16.4s, v16.4s, v0.4s

  171.             sub v17.4s, v17.4s, v1.4s

  172.             sub v18.4s, v18.4s, v2.4s

  173.             sub v19.4s, v19.4s, v3.4s

  174.             sub v20.4s, v20.4s, v4.4s

  175.             sub v21.4s, v21.4s, v5.4s

  176.             sub v22.4s, v22.4s, v6.4s

  177.             sub v23.4s, v23.4s, v7.4s

  178.             mul v0.4s, v8.4s, v14.4s

  179.             mul v1.4s, v9.4s, v14.4s

  180.             mul v2.4s, v10.4s, v14.4s

  181.             mul v3.4s, v11.4s, v14.4s

  182.             mul v4.4s, v8.4s, v15.4s

  183.             mul v5.4s, v9.4s, v15.4s

  184.             mul v6.4s, v10.4s, v15.4s

  185.             mul v7.4s, v11.4s, v15.4s

  186.             sub v24.4s, v24.4s, v0.4s

  187.             sub v25.4s, v25.4s, v1.4s

  188.             sub v26.4s, v26.4s, v2.4s

  189.             sub v27.4s, v27.4s, v3.4s

  190.             sub v28.4s, v28.4s, v4.4s

  191.             sub v29.4s, v29.4s, v5.4s

  192.             sub v30.4s, v30.4s, v6.4s

  193.             sub v31.4s, v31.4s, v7.4s

  194. 

  195.         PerTensor:

  196.             cbnz x15, PerChannel

  197.             ld1r {v0.4s}, [x12]

  198.             mov v1.16b, v0.16b

  199.             mov v2.16b, v0.16b

  200.             mov v3.16b, v0.16b

  201.             ld1r {v4.4s}, [x11]

  202.             mov v5.16b, v4.16b

  203.             mov v6.16b, v4.16b

  204.             mov v7.16b, v4.16b

  205.             ld1r {v8.4s}, [x13]

  206.             mov v9.16b, v8.16b

  207.             mov v10.16b, v8.16b

  208.             mov v11.16b, v8.16b

  209. 

  210.             b Quantization

  211. 

  212.         PerChannel:

  213.             ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [x12], #64

  214.             ld1 {v4.4s, v5.4s, v6.4s, v7.4s}, [x11], #64

  215.             ld1 {v8.4s, v9.4s, v10.4s, v11.4s}, [x13], #64

  216. 

  217.         Quantization:

  218.             sqshl v16.4s, v16.4s, v0.4s

  219.             sqshl v17.4s, v17.4s, v1.4s

  220.             sqshl v18.4s, v18.4s, v2.4s

  221.             sqshl v19.4s, v19.4s, v3.4s

  222.             sqshl v20.4s, v20.4s, v0.4s

  223.             sqshl v21.4s, v21.4s, v1.4s

  224.             sqshl v22.4s, v22.4s, v2.4s

  225.             sqshl v23.4s, v23.4s, v3.4s

  226.             sqshl v24.4s, v24.4s, v0.4s

  227.             sqshl v25.4s, v25.4s, v1.4s

  228.             sqshl v26.4s, v26.4s, v2.4s

  229.             sqshl v27.4s, v27.4s, v3.4s

  230.             sqshl v28.4s, v28.4s, v0.4s

  231.             sqshl v29.4s, v29.4s, v1.4s

  232.             sqshl v30.4s, v30.4s, v2.4s

  233.             sqshl v31.4s, v31.4s, v3.4s

  234. 

  235.             sqrdmulh v16.4s, v16.4s, v4.4s

  236.             sqrdmulh v17.4s, v17.4s, v5.4s

  237.             sqrdmulh v18.4s, v18.4s, v6.4s

  238.             sqrdmulh v19.4s, v19.4s, v7.4s

  239.             sqrdmulh v20.4s, v20.4s, v4.4s

  240.             sqrdmulh v21.4s, v21.4s, v5.4s

  241.             sqrdmulh v22.4s, v22.4s, v6.4s

  242.             sqrdmulh v23.4s, v23.4s, v7.4s

  243.             sqrdmulh v24.4s, v24.4s, v4.4s

  244.             sqrdmulh v25.4s, v25.4s, v5.4s

  245.             sqrdmulh v26.4s, v26.4s, v6.4s

  246.             sqrdmulh v27.4s, v27.4s, v7.4s

  247.             sqrdmulh v28.4s, v28.4s, v4.4s

  248.             sqrdmulh v29.4s, v29.4s, v5.4s

  249.             sqrdmulh v30.4s, v30.4s, v6.4s

  250.             sqrdmulh v31.4s, v31.4s, v7.4s

  251. 

  252.             and v0.16b, v8.16b, v16.16b

  253.             sshr v0.4s, v0.4s, #31

  254.             sqadd v16.4s, v16.4s, v0.4s

  255.             srshl v16.4s, v16.4s, v8.4s

  256.             and v1.16b, v9.16b, v17.16b

  257.             sshr v1.4s, v1.4s, #31

  258.             sqadd v17.4s, v17.4s, v1.4s

  259.             srshl v17.4s, v17.4s, v9.4s

  260.             and v2.16b, v10.16b, v18.16b

  261.             sshr v2.4s, v2.4s, #31

  262.             sqadd v18.4s, v18.4s, v2.4s

  263.             srshl v18.4s, v18.4s, v10.4s

  264.             and v3.16b, v11.16b, v19.16b

  265.             sshr v3.4s, v3.4s, #31

  266.             sqadd v19.4s, v19.4s, v3.4s

  267.             srshl v19.4s, v19.4s, v11.4s

  268. 

  269.             and v0.16b, v8.16b, v20.16b

  270.             sshr v0.4s, v0.4s, #31

  271.             sqadd v20.4s, v20.4s, v0.4s

  272.             srshl v20.4s, v20.4s, v8.4s

  273.             and v1.16b, v9.16b, v21.16b

  274.             sshr v1.4s, v1.4s, #31

  275.             sqadd v21.4s, v21.4s, v1.4s

  276.             srshl v21.4s, v21.4s, v9.4s

  277.             and v2.16b, v10.16b, v22.16b

  278.             sshr v2.4s, v2.4s, #31

  279.             sqadd v22.4s, v22.4s, v2.4s

  280.             srshl v22.4s, v22.4s, v10.4s

  281.             and v3.16b, v11.16b, v23.16b

  282.             sshr v3.4s, v3.4s, #31

  283.             sqadd v23.4s, v23.4s, v3.4s

  284.             srshl v23.4s, v23.4s, v11.4s

  285. 

  286.             and v0.16b, v8.16b, v24.16b

  287.             sshr v0.4s, v0.4s, #31

  288.             sqadd v24.4s, v24.4s, v0.4s

  289.             srshl v24.4s, v24.4s, v8.4s

  290.             and v1.16b, v9.16b, v25.16b

  291.             sshr v1.4s, v1.4s, #31

  292.             sqadd v25.4s, v25.4s, v1.4s

  293.             srshl v25.4s, v25.4s, v9.4s

  294.             and v2.16b, v10.16b, v26.16b

  295.             sshr v2.4s, v2.4s, #31

  296.             sqadd v26.4s, v26.4s, v2.4s

  297.             srshl v26.4s, v26.4s, v10.4s

  298.             and v3.16b, v11.16b, v27.16b

  299.             sshr v3.4s, v3.4s, #31

  300.             sqadd v27.4s, v27.4s, v3.4s

  301.             srshl v27.4s, v27.4s, v11.4s

  302. 

  303.             and v0.16b, v8.16b, v28.16b

  304.             sshr v0.4s, v0.4s, #31

  305.             sqadd v28.4s, v28.4s, v0.4s

  306.             srshl v28.4s, v28.4s, v8.4s

  307.             and v1.16b, v9.16b, v29.16b

  308.             sshr v1.4s, v1.4s, #31

  309.             sqadd v29.4s, v29.4s, v1.4s

  310.             srshl v29.4s, v29.4s, v9.4s

  311.             and v2.16b, v10.16b, v30.16b

  312.             sshr v2.4s, v2.4s, #31

  313.             sqadd v30.4s, v30.4s, v2.4s

  314.             srshl v30.4s, v30.4s, v10.4s

  315.             and v3.16b, v11.16b, v31.16b

  316.             sshr v3.4s, v3.4s, #31

  317.             sqadd v31.4s, v31.4s, v3.4s

  318.             srshl v31.4s, v31.4s, v11.4s

  319. 

  320.             // zp

  321.             dup v6.4s, w10

  322.             add v16.4s, v16.4s, v6.4s

  323.             add v17.4s, v17.4s, v6.4s

  324.             add v18.4s, v18.4s, v6.4s

  325.             add v19.4s, v19.4s, v6.4s

  326.             add v20.4s, v20.4s, v6.4s

  327.             add v21.4s, v21.4s, v6.4s

  328.             add v22.4s, v22.4s, v6.4s

  329.             add v23.4s, v23.4s, v6.4s

  330.             add v24.4s, v24.4s, v6.4s

  331.             add v25.4s, v25.4s, v6.4s

  332.             add v26.4s, v26.4s, v6.4s

  333.             add v27.4s, v27.4s, v6.4s

  334.             add v28.4s, v28.4s, v6.4s

  335.             add v29.4s, v29.4s, v6.4s

  336.             add v30.4s, v30.4s, v6.4s

  337.             add v31.4s, v31.4s, v6.4s

  338. 

  339.             // min

  340.             dup v0.4s, w8

  341.             smax v16.4s, v16.4s, v0.4s

  342.             smax v17.4s, v17.4s, v0.4s

  343.             smax v18.4s, v18.4s, v0.4s

  344.             smax v19.4s, v19.4s, v0.4s

  345.             smax v20.4s, v20.4s, v0.4s

  346.             smax v21.4s, v21.4s, v0.4s

  347.             smax v22.4s, v22.4s, v0.4s

  348.             smax v23.4s, v23.4s, v0.4s

  349.             smax v24.4s, v24.4s, v0.4s

  350.             smax v25.4s, v25.4s, v0.4s

  351.             smax v26.4s, v26.4s, v0.4s

  352.             smax v27.4s, v27.4s, v0.4s

  353.             smax v28.4s, v28.4s, v0.4s

  354.             smax v29.4s, v29.4s, v0.4s

  355.             smax v30.4s, v30.4s, v0.4s

  356.             smax v31.4s, v31.4s, v0.4s

  357. 

  358.             // max

  359.             dup v1.4s, w9

  360.             smin v16.4s, v16.4s, v1.4s

  361.             smin v17.4s, v17.4s, v1.4s

  362.             smin v18.4s, v18.4s, v1.4s

  363.             smin v19.4s, v19.4s, v1.4s

  364.             smin v20.4s, v20.4s, v1.4s

  365.             smin v21.4s, v21.4s, v1.4s

  366.             smin v22.4s, v22.4s, v1.4s

  367.             smin v23.4s, v23.4s, v1.4s

  368.             smin v24.4s, v24.4s, v1.4s

  369.             smin v25.4s, v25.4s, v1.4s

  370.             smin v26.4s, v26.4s, v1.4s

  371.             smin v27.4s, v27.4s, v1.4s

  372.             smin v28.4s, v28.4s, v1.4s

  373.             smin v29.4s, v29.4s, v1.4s

  374.             smin v30.4s, v30.4s, v1.4s

  375.             smin v31.4s, v31.4s, v1.4s

  376. 

  377.             sqxtn v16.4h, v16.4s

  378.             sqxtn2 v16.8h, v17.4s

  379.             sqxtn v0.8b, v16.8h

  380.             sqxtn v18.4h, v18.4s

  381.             sqxtn2 v18.8h, v19.4s

  382.             sqxtn2 v0.16b, v18.8h

  383. 

  384.             sqxtn v20.4h, v20.4s

  385.             sqxtn2 v20.8h, v21.4s

  386.             sqxtn v1.8b, v20.8h

  387.             sqxtn v22.4h, v22.4s

  388.             sqxtn2 v22.8h, v23.4s

  389.             sqxtn2 v1.16b, v22.8h

  390. 

  391.             sqxtn v24.4h, v24.4s

  392.             sqxtn2 v24.8h, v25.4s

  393.             sqxtn v2.8b, v24.8h

  394.             sqxtn v26.4h, v26.4s

  395.             sqxtn2 v26.8h, v27.4s

  396.             sqxtn2 v2.16b, v26.8h

  397. 

  398.             sqxtn v28.4h, v28.4s

  399.             sqxtn2 v28.8h, v29.4s

  400.             sqxtn v3.8b, v28.8h

  401.             sqxtn v30.4h, v30.4s

  402.             sqxtn2 v30.8h, v31.4s

  403.             sqxtn2 v3.16b, v30.8h

  404. 

  405.             b WriteStart

  406. 

  407.         LoopDepthHalf:

  408.             ld1 {v0.16b}, [x20], #16

  409.             ld1 {v1.16b, v2.16b}, [x16]

  410.             add x16, x16, #64

  411.             sdot v16.4s, v1.16b, v0.4b[0]

  412.             sdot v17.4s, v2.16b, v0.4b[0]

  413.             sdot v20.4s, v1.16b, v0.4b[1]

  414.             sdot v21.4s, v2.16b, v0.4b[1]

  415.             sdot v24.4s, v1.16b, v0.4b[2]

  416.             sdot v25.4s, v2.16b, v0.4b[2]

  417.             sdot v28.4s, v1.16b, v0.4b[3]

  418.             sdot v29.4s, v2.16b, v0.4b[3]

  419. 

  420.             subs x21, x21, #4

  421.             bgt LoopDepthHalf

  422. 

  423.         BiasHalf:

  424.             cbz x7, NoReadBiasHalf

  425.             ld1 {v0.4s, v1.4s}, [x18]

  426.             add x18, x18, #64

  427.             add v16.4s, v16.4s, v0.4s

  428.             add v17.4s, v17.4s, v1.4s

  429.             add v20.4s, v20.4s, v0.4s

  430.             add v21.4s, v21.4s, v1.4s

  431.             add v24.4s, v24.4s, v0.4s

  432.             add v25.4s, v25.4s, v1.4s

  433.             add v28.4s, v28.4s, v0.4s

  434.             add v29.4s, v29.4s, v1.4s

  435. 

  436.         NoReadBiasHalf:

  437.             ld1r {v12.4s}, [x25], #4

  438.             ld1r {v13.4s}, [x25], #4

  439.             ld1r {v14.4s}, [x25], #4

  440.             ld1r {v15.4s}, [x25], #4

  441.             cbnz x15, PerChannelSumHalf

  442. 

  443.         PerTensorSumHalf:

  444.             sub v16.4s, v16.4s, v12.4s

  445.             sub v17.4s, v17.4s, v12.4s

  446.             sub v20.4s, v20.4s, v13.4s

  447.             sub v21.4s, v21.4s, v13.4s

  448.             sub v24.4s, v24.4s, v14.4s

  449.             sub v25.4s, v25.4s, v14.4s

  450.             sub v28.4s, v28.4s, v15.4s

  451.             sub v29.4s, v29.4s, v15.4s

  452. 

  453.             b PerTensorHalf

  454. 

  455.         PerChannelSumHalf:

  456.             ld1 {v8.4s, v9.4s}, [x28]

  457.             add x28, x28, #64

  458.             mul v0.4s, v8.4s, v12.4s

  459.             mul v1.4s, v9.4s, v12.4s

  460.             mul v4.4s, v8.4s, v13.4s

  461.             mul v5.4s, v9.4s, v13.4s

  462.             sub v16.4s, v16.4s, v0.4s

  463.             sub v17.4s, v17.4s, v1.4s

  464.             sub v20.4s, v20.4s, v4.4s

  465.             sub v21.4s, v21.4s, v5.4s

  466.             mul v2.4s, v8.4s, v14.4s

  467.             mul v3.4s, v9.4s, v14.4s

  468.             mul v6.4s, v8.4s, v15.4s

  469.             mul v7.4s, v9.4s, v15.4s

  470.             sub v24.4s, v24.4s, v2.4s

  471.             sub v25.4s, v25.4s, v3.4s

  472.             sub v28.4s, v28.4s, v6.4s

  473.             sub v29.4s, v29.4s, v7.4s

  474. 

  475.         PerTensorHalf:

  476.             cbnz x15, PerChannelHalf

  477.             ld1r {v0.4s}, [x12]

  478.             mov v1.16b, v0.16b

  479.             ld1r {v4.4s}, [x11]

  480.             mov v5.16b, v4.16b

  481.             ld1r {v8.4s}, [x13]

  482.             mov v9.16b, v8.16b

  483. 

  484.             b QuantizationHalf

  485. 

  486.         PerChannelHalf:

  487.             ld1 {v0.4s, v1.4s}, [x12]

  488.             add x12, x12, #64

  489.             ld1 {v4.4s, v5.4s}, [x11]

  490.             add x11, x11, #64

  491.             ld1 {v8.4s, v9.4s}, [x13]

  492.             add x13, x13, #64

  493. 

  494.         QuantizationHalf:

  495.             sqshl v16.4s, v16.4s, v0.4s

  496.             sqshl v17.4s, v17.4s, v1.4s

  497.             sqshl v20.4s, v20.4s, v0.4s

  498.             sqshl v21.4s, v21.4s, v1.4s

  499.             sqshl v24.4s, v24.4s, v0.4s

  500.             sqshl v25.4s, v25.4s, v1.4s

  501.             sqshl v28.4s, v28.4s, v0.4s

  502.             sqshl v29.4s, v29.4s, v1.4s

  503. 

  504.             sqrdmulh v16.4s, v16.4s, v4.4s

  505.             sqrdmulh v17.4s, v17.4s, v5.4s

  506.             sqrdmulh v20.4s, v20.4s, v4.4s

  507.             sqrdmulh v21.4s, v21.4s, v5.4s

  508.             sqrdmulh v24.4s, v24.4s, v4.4s

  509.             sqrdmulh v25.4s, v25.4s, v5.4s

  510.             sqrdmulh v28.4s, v28.4s, v4.4s

  511.             sqrdmulh v29.4s, v29.4s, v5.4s

  512. 

  513.             and v0.16b, v8.16b, v16.16b

  514.             sshr v0.4s, v0.4s, #31

  515.             sqadd v16.4s, v16.4s, v0.4s

  516.             srshl v16.4s, v16.4s, v8.4s

  517.             and v1.16b, v9.16b, v17.16b

  518.             sshr v1.4s, v1.4s, #31

  519.             sqadd v17.4s, v17.4s, v1.4s

  520.             srshl v17.4s, v17.4s, v9.4s

  521. 

  522.             and v0.16b, v8.16b, v20.16b

  523.             sshr v0.4s, v0.4s, #31

  524.             sqadd v20.4s, v20.4s, v0.4s

  525.             srshl v20.4s, v20.4s, v8.4s

  526.             and v1.16b, v9.16b, v21.16b

  527.             sshr v1.4s, v1.4s, #31

  528.             sqadd v21.4s, v21.4s, v1.4s

  529.             srshl v21.4s, v21.4s, v9.4s

  530. 

  531.             and v0.16b, v8.16b, v24.16b

  532.             sshr v0.4s, v0.4s, #31

  533.             sqadd v24.4s, v24.4s, v0.4s

  534.             srshl v24.4s, v24.4s, v8.4s

  535.             and v1.16b, v9.16b, v25.16b

  536.             sshr v1.4s, v1.4s, #31

  537.             sqadd v25.4s, v25.4s, v1.4s

  538.             srshl v25.4s, v25.4s, v9.4s

  539. 

  540.             and v0.16b, v8.16b, v28.16b

  541.             sshr v0.4s, v0.4s, #31

  542.             sqadd v28.4s, v28.4s, v0.4s

  543.             srshl v28.4s, v28.4s, v8.4s

  544.             and v1.16b, v9.16b, v29.16b

  545.             sshr v1.4s, v1.4s, #31

  546.             sqadd v29.4s, v29.4s, v1.4s

  547.             srshl v29.4s, v29.4s, v9.4s

  548. 

  549.             // zp

  550.             dup v6.4s, w10

  551.             add v16.4s, v16.4s, v6.4s

  552.             add v17.4s, v17.4s, v6.4s

  553.             add v20.4s, v20.4s, v6.4s

  554.             add v21.4s, v21.4s, v6.4s

  555.             add v24.4s, v24.4s, v6.4s

  556.             add v25.4s, v25.4s, v6.4s

  557.             add v28.4s, v28.4s, v6.4s

  558.             add v29.4s, v29.4s, v6.4s

  559. 

  560.             // min

  561.             dup v0.4s, w8

  562.             smax v16.4s, v16.4s, v0.4s

  563.             smax v17.4s, v17.4s, v0.4s

  564.             smax v20.4s, v20.4s, v0.4s

  565.             smax v21.4s, v21.4s, v0.4s

  566.             smax v24.4s, v24.4s, v0.4s

  567.             smax v25.4s, v25.4s, v0.4s

  568.             smax v28.4s, v28.4s, v0.4s

  569.             smax v29.4s, v29.4s, v0.4s

  570. 

  571.             // max

  572.             dup v1.4s, w9

  573.             smin v16.4s, v16.4s, v1.4s

  574.             smin v17.4s, v17.4s, v1.4s

  575.             smin v20.4s, v20.4s, v1.4s

  576.             smin v21.4s, v21.4s, v1.4s

  577.             smin v24.4s, v24.4s, v1.4s

  578.             smin v25.4s, v25.4s, v1.4s

  579.             smin v28.4s, v28.4s, v1.4s

  580.             smin v29.4s, v29.4s, v1.4s

  581. 

  582.             sqxtn v16.4h, v16.4s

  583.             sqxtn2 v16.8h, v17.4s

  584.             sqxtn v0.8b, v16.8h

  585. 

  586.             sqxtn v20.4h, v20.4s

  587.             sqxtn2 v20.8h, v21.4s

  588.             sqxtn v1.8b, v20.8h

  589. 

  590.             sqxtn v24.4h, v24.4s

  591.             sqxtn2 v24.8h, v25.4s

  592.             sqxtn v2.8b, v24.8h

  593. 

  594.             sqxtn v28.4h, v28.4s

  595.             sqxtn2 v28.8h, v29.4s

  596.             sqxtn v3.8b, v28.8h

  597. 

  598.             b WriteStart

  599. 

  600.         LoopDepthQuarter:

  601.             ld1 {v0.16b}, [x20], #16

  602.             ld1 {v1.16b}, [x16]

  603.             add x16, x16, #64

  604.             sdot v16.4s, v1.16b, v0.4b[0]

  605.             sdot v20.4s, v1.16b, v0.4b[1]

  606.             sdot v24.4s, v1.16b, v0.4b[2]

  607.             sdot v28.4s, v1.16b, v0.4b[3]

  608. 

  609.             subs x21, x21, #4

  610.             bgt LoopDepthQuarter

  611. 

  612.         BiasQuarter:

  613.             cbz x7, NoReadBiasQuarter

  614.             ld1 {v0.4s}, [x18]

  615.             add x18, x18, #64

  616.             add v16.4s, v16.4s, v0.4s

  617.             add v20.4s, v20.4s, v0.4s

  618.             add v24.4s, v24.4s, v0.4s

  619.             add v28.4s, v28.4s, v0.4s

  620. 

  621.         NoReadBiasQuarter:

  622.             ld1r {v12.4s}, [x25], #4

  623.             ld1r {v13.4s}, [x25], #4

  624.             ld1r {v14.4s}, [x25], #4

  625.             ld1r {v15.4s}, [x25], #4

  626.             cbnz x15, PerChannelSumQuarter

  627. 

  628.         PerTensorSumQuarter:

  629.             sub v16.4s, v16.4s, v12.4s

  630.             sub v20.4s, v20.4s, v13.4s

  631.             sub v24.4s, v24.4s, v14.4s

  632.             sub v28.4s, v28.4s, v15.4s

  633. 

  634.             b PerTensorQuarter

  635. 

  636.         PerChannelSumQuarter:

  637.             ld1 {v8.4s}, [x28]

  638.             add x28, x28, #64

  639.             mul v0.4s, v8.4s, v12.4s

  640.             mul v4.4s, v8.4s, v13.4s

  641.             sub v16.4s, v16.4s, v0.4s

  642.             sub v20.4s, v20.4s, v4.4s

  643.             mul v2.4s, v8.4s, v14.4s

  644.             mul v6.4s, v8.4s, v15.4s

  645.             sub v24.4s, v24.4s, v2.4s

  646.             sub v28.4s, v28.4s, v6.4s

  647. 

  648.         PerTensorQuarter:

  649.             cbnz x15, PerChannelQuarter

  650.             ld1r {v0.4s}, [x12]

  651.             ld1r {v4.4s}, [x11]

  652.             ld1r {v8.4s}, [x13]

  653. 

  654.             b QuantizationHalf

  655. 

  656.         PerChannelQuarter:

  657.             ld1 {v0.4s}, [x12]

  658.             add x12, x12, #64

  659.             ld1 {v4.4s}, [x11]

  660.             add x11, x11, #64

  661.             ld1 {v8.4s}, [x13]

  662.             add x13, x13, #64

  663. 

  664.         QuantizationQuarter:

  665.             sqshl v16.4s, v16.4s, v0.4s

  666.             sqshl v20.4s, v20.4s, v0.4s

  667.             sqshl v24.4s, v24.4s, v0.4s

  668.             sqshl v28.4s, v28.4s, v0.4s

  669. 

  670.             sqrdmulh v16.4s, v16.4s, v4.4s

  671.             sqrdmulh v20.4s, v20.4s, v4.4s

  672.             sqrdmulh v24.4s, v24.4s, v4.4s

  673.             sqrdmulh v28.4s, v28.4s, v4.4s

  674. 

  675.             and v0.16b, v8.16b, v16.16b

  676.             sshr v0.4s, v0.4s, #31

  677.             sqadd v16.4s, v16.4s, v0.4s

  678.             srshl v16.4s, v16.4s, v8.4s

  679. 

  680.             and v0.16b, v8.16b, v20.16b

  681.             sshr v0.4s, v0.4s, #31

  682.             sqadd v20.4s, v20.4s, v0.4s

  683.             srshl v20.4s, v20.4s, v8.4s

  684. 

  685.             and v0.16b, v8.16b, v24.16b

  686.             sshr v0.4s, v0.4s, #31

  687.             sqadd v24.4s, v24.4s, v0.4s

  688.             srshl v24.4s, v24.4s, v8.4s

  689. 

  690.             and v0.16b, v8.16b, v28.16b

  691.             sshr v0.4s, v0.4s, #31

  692.             sqadd v28.4s, v28.4s, v0.4s

  693.             srshl v28.4s, v28.4s, v8.4s

  694. 

  695.             // zp

  696.             dup v6.4s, w10

  697.             add v16.4s, v16.4s, v6.4s

  698.             add v20.4s, v20.4s, v6.4s

  699.             add v24.4s, v24.4s, v6.4s

  700.             add v28.4s, v28.4s, v6.4s

  701. 

  702.             // min

  703.             dup v0.4s, w8

  704.             smax v16.4s, v16.4s, v0.4s

  705.             smax v20.4s, v20.4s, v0.4s

  706.             smax v24.4s, v24.4s, v0.4s

  707.             smax v28.4s, v28.4s, v0.4s

  708. 

  709.             // max

  710.             dup v1.4s, w9

  711.             smin v16.4s, v16.4s, v1.4s

  712.             smin v20.4s, v20.4s, v1.4s

  713.             smin v24.4s, v24.4s, v1.4s

  714.             smin v28.4s, v28.4s, v1.4s

  715. 

  716.             sqxtn v16.4h, v16.4s

  717.             sqxtn v0.8b, v16.8h

  718. 

  719.             sqxtn v20.4h, v20.4s

  720.             sqxtn v1.8b, v20.8h

  721. 

  722.             sqxtn v24.4h, v24.4s

  723.             sqxtn v2.8b, v24.8h

  724. 

  725.             sqxtn v28.4h, v28.4s

  726.             sqxtn v3.8b, v28.8h

  727. 

  728.             b WriteStart

  729. 

  730.         WriteStart:

  731.             cmp x17, #1

  732.             beq Write1

  733.             cmp x17, #2

  734.             beq Write2

  735.             cmp x17, #3

  736.             beq Write3

  737.             cmp x17, #4

  738.             beq Write4

  739.             cmp x17, #5

  740.             beq Write5

  741.             cmp x17, #6

  742.             beq Write6

  743.             cmp x17, #7

  744.             beq Write7

  745.             cmp x17, #8

  746.             beq Write8

  747.             cmp x17, #9

  748.             beq Write9

  749.             cmp x17, #10

  750.             beq Write10

  751.             cmp x17, #11

  752.             beq Write11

  753.             cmp x17, #12

  754.             beq Write12

  755.             cmp x17, #13

  756.             beq Write13

  757.             cmp x17, #14

  758.             beq Write14

  759.             cmp x17, #15

  760.             beq Write15

  761.             b Write16

  762. 

  763.         Write1:

  764.             add x27, x27, #1

  765.             st1 {v0.b}[0], [x19], x14

  766.             cmp x3, #1

  767.             beq WriteEnd

  768.             st1 {v1.b}[0], [x19], x14

  769.             cmp x3, #2

  770.             beq WriteEnd

  771.             st1 {v2.b}[0], [x19], x14

  772.             cmp x3, #3

  773.             beq WriteEnd

  774.             st1 {v3.b}[0], [x19], x14

  775.             b WriteEnd

  776.         Write2:

  777.             add x27, x27, #2

  778.             st1 {v0.h}[0], [x19], x14

  779.             cmp x3, #1

  780.             beq WriteEnd

  781.             st1 {v1.h}[0], [x19], x14

  782.             cmp x3, #2

  783.             beq WriteEnd

  784.             st1 {v2.h}[0], [x19], x14

  785.             cmp x3, #3

  786.             beq WriteEnd

  787.             st1 {v3.h}[0], [x19], x14

  788.             b WriteEnd

  789.         Write3:

  790.             add x27, x27, #3

  791.             add x22, x19, #2

  792.             st1 {v0.h}[0], [x19], x14

  793.             st1 {v0.b}[2], [x22], x14

  794.             cmp x3, #1

  795.             beq WriteEnd

  796.             st1 {v1.h}[0], [x19], x14

  797.             st1 {v1.b}[2], [x22], x14

  798.             cmp x3, #2

  799.             beq WriteEnd

  800.             st1 {v2.h}[0], [x19], x14

  801.             st1 {v2.b}[2], [x22], x14

  802.             cmp x3, #3

  803.             beq WriteEnd

  804.             st1 {v3.h}[0], [x19], x14

  805.             st1 {v3.b}[2], [x22], x14

  806.             b WriteEnd

  807.         Write4:

  808.             add x27, x27, #4

  809.             st1 {v0.s}[0], [x19], x14

  810.             cmp x3, #1

  811.             beq WriteEnd

  812.             st1 {v1.s}[0], [x19], x14

  813.             cmp x3, #2

  814.             beq WriteEnd

  815.             st1 {v2.s}[0], [x19], x14

  816.             cmp x3, #3

  817.             beq WriteEnd

  818.             st1 {v3.s}[0], [x19], x14

  819.             b WriteEnd

  820.         Write5:

  821.             add x27, x27, #5

  822.             add x22, x19, #4

  823.             st1 {v0.s}[0], [x19], x14

  824.             st1 {v0.b}[4], [x22], x14

  825.             cmp x3, #1

  826.             beq WriteEnd

  827.             st1 {v1.s}[0], [x19], x14

  828.             st1 {v1.b}[4], [x22], x14

  829.             cmp x3, #2

  830.             beq WriteEnd

  831.             st1 {v2.s}[0], [x19], x14

  832.             st1 {v2.b}[4], [x22], x14

  833.             cmp x3, #3

  834.             beq WriteEnd

  835.             st1 {v3.s}[0], [x19], x14

  836.             st1 {v3.b}[4], [x22], x14

  837.             b WriteEnd

  838.         Write6:

  839.             add x27, x27, #6

  840.             add x22, x19, #4

  841.             st1 {v0.s}[0], [x19], x14

  842.             st1 {v0.h}[2], [x22], x14

  843.             cmp x3, #1

  844.             beq WriteEnd

  845.             st1 {v1.s}[0], [x19], x14

  846.             st1 {v1.h}[2], [x22], x14

  847.             cmp x3, #2

  848.             beq WriteEnd

  849.             st1 {v2.s}[0], [x19], x14

  850.             st1 {v2.h}[2], [x22], x14

  851.             cmp x3, #3

  852.             beq WriteEnd

  853.             st1 {v3.s}[0], [x19], x14

  854.             st1 {v3.h}[2], [x22], x14

  855.             b WriteEnd

  856.         Write7:

  857.             add x27, x27, #7

  858.             add x22, x19, #4

  859.             add x26, x19, #6

  860.             st1 {v0.s}[0], [x19], x14

  861.             st1 {v0.h}[2], [x22], x14

  862.             st1 {v0.b}[6], [x26], x14

  863.             cmp x3, #1

  864.             beq WriteEnd

  865.             st1 {v1.s}[0], [x19], x14

  866.             st1 {v1.h}[2], [x22], x14

  867.             st1 {v1.b}[6], [x26], x14

  868.             cmp x3, #2

  869.             beq WriteEnd

  870.             st1 {v2.s}[0], [x19], x14

  871.             st1 {v2.h}[2], [x22], x14

  872.             st1 {v2.b}[6], [x26], x14

  873.             cmp x3, #3

  874.             beq WriteEnd

  875.             st1 {v3.s}[0], [x19], x14

  876.             st1 {v3.h}[2], [x22], x14

  877.             st1 {v3.b}[6], [x26], x14

  878.             b WriteEnd

  879.         Write8:

  880.             add x27, x27, #8

  881.             st1 {v0.8b}, [x19], x14

  882.             cmp x3, #1

  883.             beq WriteEnd

  884.             st1 {v1.8b}, [x19], x14

  885.             cmp x3, #2

  886.             beq WriteEnd

  887.             st1 {v2.8b}, [x19], x14

  888.             cmp x3, #3

  889.             beq WriteEnd

  890.             st1 {v3.8b}, [x19], x14

  891.             b WriteEnd

  892.         Write9:

  893.             add x27, x27, #9

  894.             add x22, x19, #8

  895.             st1 {v0.8b}, [x19], x14

  896.             st1 {v0.b}[8], [x22], x14

  897.             cmp x3, #1

  898.             beq WriteEnd

  899.             st1 {v1.8b}, [x19], x14

  900.             st1 {v1.b}[8], [x22], x14

  901.             cmp x3, #2

  902.             beq WriteEnd

  903.             st1 {v2.8b}, [x19], x14

  904.             st1 {v2.b}[8], [x22], x14

  905.             cmp x3, #3

  906.             beq WriteEnd

  907.             st1 {v3.8b}, [x19], x14

  908.             st1 {v3.b}[8], [x22], x14

  909.             b WriteEnd

  910.         Write10:

  911.             add x27, x27, #10

  912.             add x22, x19, #8

  913.             st1 {v0.8b}, [x19], x14

  914.             st1 {v0.h}[4], [x22], x14

  915.             cmp x3, #1

  916.             beq WriteEnd

  917.             st1 {v1.8b}, [x19], x14

  918.             st1 {v1.h}[4], [x22], x14

  919.             cmp x3, #2

  920.             beq WriteEnd

  921.             st1 {v2.8b}, [x19], x14

  922.             st1 {v2.h}[4], [x22], x14

  923.             cmp x3, #3

  924.             beq WriteEnd

  925.             st1 {v3.8b}, [x19], x14

  926.             st1 {v3.h}[4], [x22], x14

  927.             b WriteEnd

  928.         Write11:

  929.             add x27, x27, #11

  930.             add x22, x19, #8

  931.             add x26, x19, #10

  932.             st1 {v0.8b}, [x19], x14

  933.             st1 {v0.h}[4], [x22], x14

  934.             st1 {v0.b}[10], [x26], x14

  935.             cmp x3, #1

  936.             beq WriteEnd

  937.             st1 {v1.8b}, [x19], x14

  938.             st1 {v1.h}[4], [x22], x14

  939.             st1 {v1.b}[10], [x26], x14

  940.             cmp x3, #2

  941.             beq WriteEnd

  942.             st1 {v2.8b}, [x19], x14

  943.             st1 {v2.h}[4], [x22], x14

  944.             st1 {v2.b}[10], [x26], x14

  945.             cmp x3, #3

  946.             beq WriteEnd

  947.             st1 {v3.8b}, [x19], x14

  948.             st1 {v3.h}[4], [x22], x14

  949.             st1 {v3.b}[10], [x26], x14

  950.             b WriteEnd

  951.         Write12:

  952.             add x27, x27, #12

  953.             add x22, x19, #8

  954.             st1 {v0.8b}, [x19], x14

  955.             st1 {v0.s}[2], [x22], x14

  956.             cmp x3, #1

  957.             beq WriteEnd

  958.             st1 {v1.8b}, [x19], x14

  959.             st1 {v1.s}[2], [x22], x14

  960.             cmp x3, #2

  961.             beq WriteEnd

  962.             st1 {v2.8b}, [x19], x14

  963.             st1 {v2.s}[2], [x22], x14

  964.             cmp x3, #3

  965.             beq WriteEnd

  966.             st1 {v3.8b}, [x19], x14

  967.             st1 {v3.s}[2], [x22], x14

  968.             b WriteEnd

  969.         Write13:

  970.             add x27, x27, #13

  971.             add x22, x19, #8

  972.             add x26, x19, #12

  973.             st1 {v0.8b}, [x19], x14

  974.             st1 {v0.s}[2], [x22], x14

  975.             st1 {v0.b}[12], [x26], x14

  976.             cmp x3, #1

  977.             beq WriteEnd

  978.             st1 {v1.8b}, [x19], x14

  979.             st1 {v1.s}[2], [x22], x14

  980.             st1 {v1.b}[12], [x26], x14

  981.             cmp x3, #2

  982.             beq WriteEnd

  983.             st1 {v2.8b}, [x19], x14

  984.             st1 {v2.s}[2], [x22], x14

  985.             st1 {v2.b}[12], [x26], x14

  986.             cmp x3, #3

  987.             beq WriteEnd

  988.             st1 {v3.8b}, [x19], x14

  989.             st1 {v3.s}[2], [x22], x14

  990.             st1 {v3.b}[12], [x26], x14

  991.             b WriteEnd

  992.         Write14:

  993.             add x27, x27, #14

  994.             add x22, x19, #8

  995.             add x26, x19, #12

  996.             st1 {v0.8b}, [x19], x14

  997.             st1 {v0.s}[2], [x22], x14

  998.             st1 {v0.h}[6], [x26], x14

  999.             cmp x3, #1

  1000.             beq WriteEnd

  1001.             st1 {v1.8b}, [x19], x14

  1002.             st1 {v1.s}[2], [x22], x14

  1003.             st1 {v1.h}[6], [x26], x14

  1004.             cmp x3, #2

  1005.             beq WriteEnd

  1006.             st1 {v2.8b}, [x19], x14

  1007.             st1 {v2.s}[2], [x22], x14

  1008.             st1 {v2.h}[6], [x26], x14

  1009.             cmp x3, #3

  1010.             beq WriteEnd

  1011.             st1 {v3.8b}, [x19], x14

  1012.             st1 {v3.s}[2], [x22], x14

  1013.             st1 {v3.h}[6], [x26], x14

  1014.             b WriteEnd

  1015.         Write15:

  1016.             add x27, x27, #15

  1017.             add x22, x19, #8

  1018.             add x26, x19, #12

  1019.             add x21, x19, #14

  1020.             st1 {v0.8b}, [x19], x14

  1021.             st1 {v0.s}[2], [x22], x14

  1022.             st1 {v0.h}[6], [x26], x14

  1023.             st1 {v0.b}[14], [x21], x14

  1024.             cmp x3, #1

  1025.             beq WriteEnd

  1026.             st1 {v1.8b}, [x19], x14

  1027.             st1 {v1.s}[2], [x22], x14

  1028.             st1 {v1.h}[6], [x26], x14

  1029.             st1 {v1.b}[14], [x21], x14

  1030.             cmp x3, #2

  1031.             beq WriteEnd

  1032.             st1 {v2.8b}, [x19], x14

  1033.             st1 {v2.s}[2], [x22], x14

  1034.             st1 {v2.h}[6], [x26], x14

  1035.             st1 {v2.b}[14], [x21], x14

  1036.             cmp x3, #3

  1037.             beq WriteEnd

  1038.             st1 {v3.8b}, [x19], x14

  1039.             st1 {v3.s}[2], [x22], x14

  1040.             st1 {v3.h}[6], [x26], x14

  1041.             st1 {v3.b}[14], [x21], x14

  1042.             b WriteEnd

  1043.         Write16:

  1044.             add x27, x27, #16

  1045.             st1 {v0.16b}, [x19], x14

  1046.             cmp x3, #1

  1047.             beq WriteEnd

  1048.             st1 {v1.16b}, [x19], x14

  1049.             cmp x3, #2

  1050.             beq WriteEnd

  1051.             st1 {v2.16b}, [x19], x14

  1052.             cmp x3, #3

  1053.             beq WriteEnd

  1054.             st1 {v3.16b}, [x19], x14

  1055. 

  1056.     WriteEnd:

  1057.         subs x17, x17, #16

  1058.         ble LoopColEnd

  1059.         mov x25, x6

  1060.         b LoopCol

  1061. 

  1062. LoopColEnd:

  1063.     subs x3, x3, #4

  1064.     ble LoopRowEnd

  1065.     ldr x11, [sp, #24]

  1066.     ldr x12, [sp, #32]

  1067.     ldr x13, [sp, #40]

  1068.     add x6, x6, #16

  1069.     add x0, x0, x23

  1070.     add x2, x2, x24

  1071.     b LoopRow

  1072. 

  1073. LoopRowEnd:

  1074.   sub sp, sp, #208

  1075.   ld1 {v8.4s, v9.4s, v10.4s, v11.4s}, [sp], #64

  1076.   ld1 {v12.4s, v13.4s, v14.4s, v15.4s}, [sp], #64

  1077.   ldp x19, x20, [sp], #16

  1078.   ldp x21, x22, [sp], #16

  1079.   ldp x23, x24, [sp], #16

  1080.   ldp x25, x26, [sp], #16

  1081.   ldp x27, x28, [sp], #16

  1082.   ret

  1083. #endif