Debian Codegraph @

crypto_nPbatch/montgomery25519/arm64-neon-2x1-uma-9l/mladder.S

raw
1 #include "crypto_asm_hidden.h"
2
3 // 2-way Montgomery ladder, 9-limb implementation
4
5 .p2align 4
6 ASM_HIDDEN _CRYPTO_SHARED_NAMESPACE(mladder_2x1)
7 .globl _CRYPTO_SHARED_NAMESPACE(mladder_2x1)
8 ASM_HIDDEN CRYPTO_SHARED_NAMESPACE(mladder_2x1)
9 .globl CRYPTO_SHARED_NAMESPACE(mladder_2x1)
10
11 _CRYPTO_SHARED_NAMESPACE(mladder_2x1):
12 CRYPTO_SHARED_NAMESPACE(mladder_2x1):
13
14 sub sp, sp, #1488
15 stp x19, x20, [sp, #0]
16 stp x21, x22, [sp, #16]
17 stp x23, x24, [sp, #32]
18 stp x25, x26, [sp, #48]
19 stp x27, x28, [sp, #64]
20 stp x29, x30, [sp, #80]
21 stp d8, d9, [sp, #96]
22 stp d10, d11, [sp, #112]
23 stp d12, d13, [sp, #128]
24 stp d14, d15, [sp, #144]
25
26 mov x4, #1
27 mov w10, #1216
28
29 mov w13, #0x7fffff
30 mov w14, #0x1fffffff
31
32 movz x12, #0xdb42
33 movk x12, #0x0001, lsl 16
34 movk x12, #0xdb42, lsl 32
35 movk x12, #0x0001, lsl 48
36
37 movz x21, #0xffda
38 movk x21, #0x3fff, lsl 16
39 movk x21, #0xffda, lsl 32
40 movk x21, #0x3fff, lsl 48
41
42 movz x22, #0xfffe
43 movk x22, #0x3fff, lsl 16
44 movk x22, #0xfffe, lsl 32
45 movk x22, #0x3fff, lsl 48
46
47 movz x23, #0xfffe
48 movk x23, #0x00ff, lsl 16
49 movk x23, #0xfffe, lsl 32
50 movk x23, #0x00ff, lsl 48
51
52 // X1 ← XP,X3 ← XP
53 add x20, sp, #192
54
55 ldr q20, [x1, #0]
56 str q20, [x20, #0]
57 str q20, [x20, #432]
58
59 ldr q20, [x1, #16]
60 str q20, [x20, #16]
61 str q20, [x20, #448]
62
63 ldr q20, [x1, #32]
64 str q20, [x20, #32]
65 str q20, [x20, #464]
66
67 ldr q20, [x1, #48]
68 str q20, [x20, #48]
69 str q20, [x20, #480]
70
71 ldr q20, [x1, #64]
72 str q20, [x20, #64]
73 str q20, [x20, #496]
74
75 ldr q20, [x1, #80]
76 str q20, [x20, #80]
77 str q20, [x20, #512]
78
79 ldr q20, [x1, #96]
80 str q20, [x20, #96]
81 str q20, [x20, #528]
82
83 ldr q20, [x1, #112]
84 str q20, [x20, #112]
85 str q20, [x20, #544]
86
87 ldr q20, [x1, #128]
88 str q20, [x20, #128]
89 str q20, [x20, #560]
90
91 dup v20.2s, w4
92 dup v21.2s, wzr
93
94 // X2 ← 1
95 str q20, [x20, #144]
96 str q21, [x20, #160]
97 str q21, [x20, #176]
98 str q21, [x20, #192]
99 str q21, [x20, #208]
100 str q21, [x20, #224]
101 str q21, [x20, #240]
102 str q21, [x20, #256]
103 str q21, [x20, #272]
104
105 // Z2 ← 0
106 str q21, [x20, #288]
107 str q21, [x20, #304]
108 str q21, [x20, #320]
109 str q21, [x20, #336]
110 str q21, [x20, #352]
111 str q21, [x20, #368]
112 str q21, [x20, #384]
113 str q21, [x20, #400]
114 str q21, [x20, #416]
115
116 // Z3 ← 1
117 str q20, [x20, #576]
118 str q21, [x20, #592]
119 str q21, [x20, #608]
120 str q21, [x20, #624]
121 str q21, [x20, #640]
122 str q21, [x20, #656]
123 str q21, [x20, #672]
124 str q21, [x20, #688]
125 str q21, [x20, #704]
126
127 mov x5, #48
128 mov x6, #62
129 mov x15, x6
130 stp xzr, xzr, [sp, #160]
131 mov x3, x2
132
133 mov v19.d[0], x21
134
135 // Montgomery ladder loop
136 .L1:
137 add x3, x3, x5
138 ldr q30, [x3, #0]
139 str q30, [sp, #176]
140 mov x3, x2
141 .L2:
142
143 /*
144 * Montgomery ladder step
145 *
146 * T1 ← X2 + Z2
147 * T2 ← X2 - Z2
148 * T3 ← X3 + Z3
149 * T4 ← X3 - Z3
150 * Z3 ← T2 · T3
151 * X3 ← T1 · T4
152 *
153 * bit ← n[i]
154 * select ← bit ⊕ prevbit
155 * prevbit ← bit
156 * CSelect(T1,T3,select): if (select == 1) {T1 = T3}
157 * CSelect(T2,T4,select): if (select == 1) {T2 = T4}
158 *
159 * T2 ← T2^2
160 * T1 ← T1^2
161 * T3 ← X3 + Z3
162 * Z3 ← X3 - Z3
163 * Z3 ← Z3^2
164 * X3 ← T3^2
165 * T3 ← T1 - T2
166 * T4 ← ((A + 2)/4) · T3
167 * T4 ← T4 + T2
168 * X2 ← T1 · T2
169 * Z2 ← T3 · T4
170 * Z3 ← Z3 · X1
171 *
172 */
173
174 // X2
175 ldr q10, [x20, #144]
176 ldr q11, [x20, #160]
177 ldr q12, [x20, #176]
178 ldr q13, [x20, #192]
179 ldr q14, [x20, #208]
180 ldr q15, [x20, #224]
181 ldr q16, [x20, #240]
182 ldr q17, [x20, #256]
183 ldr q18, [x20, #272]
184
185 // Z2
186 ldr q0, [x20, #288]
187 ldr q1, [x20, #304]
188 ldr q2, [x20, #320]
189 ldr q3, [x20, #336]
190 ldr q4, [x20, #352]
191 ldr q5, [x20, #368]
192 ldr q6, [x20, #384]
193 ldr q7, [x20, #400]
194 ldr q8, [x20, #416]
195
196 // T1 ← X2 + Z2
197 add v20.2s, v0.2s, v10.2s
198 add v21.2s, v1.2s, v11.2s
199 add v22.2s, v2.2s, v12.2s
200 add v23.2s, v3.2s, v13.2s
201 add v24.2s, v4.2s, v14.2s
202 add v25.2s, v5.2s, v15.2s
203 add v26.2s, v6.2s, v16.2s
204 add v27.2s, v7.2s, v17.2s
205 add v28.2s, v8.2s, v18.2s
206
207 str q20, [x20, #720]
208 str q21, [x20, #736]
209 str q22, [x20, #752]
210 str q23, [x20, #768]
211 str q24, [x20, #784]
212 str q25, [x20, #800]
213 str q26, [x20, #816]
214 str q27, [x20, #832]
215 str q28, [x20, #848]
216
217 // T2 ← X2 - Z2
218 mov v30.d[0], x22
219 mov v31.d[0], x23
220 add v10.2s, v19.2s, v10.2s
221 add v11.2s, v30.2s, v11.2s
222 add v12.2s, v30.2s, v12.2s
223 add v13.2s, v30.2s, v13.2s
224 add v14.2s, v30.2s, v14.2s
225 add v15.2s, v30.2s, v15.2s
226 add v16.2s, v30.2s, v16.2s
227 add v17.2s, v30.2s, v17.2s
228 add v18.2s, v31.2s, v18.2s
229
230 sub v20.2s, v10.2s, v0.2s
231 sub v21.2s, v11.2s, v1.2s
232 sub v22.2s, v12.2s, v2.2s
233 sub v23.2s, v13.2s, v3.2s
234 sub v24.2s, v14.2s, v4.2s
235 sub v25.2s, v15.2s, v5.2s
236 sub v26.2s, v16.2s, v6.2s
237 sub v27.2s, v17.2s, v7.2s
238 sub v28.2s, v18.2s, v8.2s
239
240 str q20, [x20, #864]
241 str q21, [x20, #880]
242 str q22, [x20, #896]
243 str q23, [x20, #912]
244 str q24, [x20, #928]
245 str q25, [x20, #944]
246 str q26, [x20, #960]
247 str q27, [x20, #976]
248 str q28, [x20, #992]
249
250 // X3
251 ldr q10, [x20, #432]
252 ldr q11, [x20, #448]
253 ldr q12, [x20, #464]
254 ldr q13, [x20, #480]
255 ldr q14, [x20, #496]
256 ldr q15, [x20, #512]
257 ldr q16, [x20, #528]
258 ldr q17, [x20, #544]
259 ldr q18, [x20, #560]
260
261 // Z3
262 ldr q0, [x20, #576]
263 ldr q1, [x20, #592]
264 ldr q2, [x20, #608]
265 ldr q3, [x20, #624]
266 ldr q4, [x20, #640]
267 ldr q5, [x20, #656]
268 ldr q6, [x20, #672]
269 ldr q7, [x20, #688]
270 ldr q8, [x20, #704]
271
272 // T3 ← X3 + Z3
273 add v20.2s, v0.2s, v10.2s
274 add v21.2s, v1.2s, v11.2s
275 add v22.2s, v2.2s, v12.2s
276 add v23.2s, v3.2s, v13.2s
277 add v24.2s, v4.2s, v14.2s
278 add v25.2s, v5.2s, v15.2s
279 add v26.2s, v6.2s, v16.2s
280 add v27.2s, v7.2s, v17.2s
281 add v28.2s, v8.2s, v18.2s
282
283 str q20, [x20, #1008]
284 str q21, [x20, #1024]
285 str q22, [x20, #1040]
286 str q23, [x20, #1056]
287 str q24, [x20, #1072]
288 str q25, [x20, #1088]
289 str q26, [x20, #1104]
290 str q27, [x20, #1120]
291 str q28, [x20, #1136]
292
293 // T4 ← X3 - Z3
294 mov v30.d[0], x22
295 mov v31.d[0], x23
296 add v10.2s, v19.2s, v10.2s
297 add v11.2s, v30.2s, v11.2s
298 add v12.2s, v30.2s, v12.2s
299 add v13.2s, v30.2s, v13.2s
300 add v14.2s, v30.2s, v14.2s
301 add v15.2s, v30.2s, v15.2s
302 add v16.2s, v30.2s, v16.2s
303 add v17.2s, v30.2s, v17.2s
304 add v18.2s, v31.2s, v18.2s
305
306 sub v20.2s, v10.2s, v0.2s
307 sub v21.2s, v11.2s, v1.2s
308 sub v22.2s, v12.2s, v2.2s
309 sub v23.2s, v13.2s, v3.2s
310 sub v24.2s, v14.2s, v4.2s
311 sub v25.2s, v15.2s, v5.2s
312 sub v26.2s, v16.2s, v6.2s
313 sub v27.2s, v17.2s, v7.2s
314 sub v28.2s, v18.2s, v8.2s
315
316 str q20, [x20, #1152]
317 str q21, [x20, #1168]
318 str q22, [x20, #1184]
319 str q23, [x20, #1200]
320 str q24, [x20, #1216]
321 str q25, [x20, #1232]
322 str q26, [x20, #1248]
323 str q27, [x20, #1264]
324 str q28, [x20, #1280]
325
326 // X3 ← T1 · T4
327 ldr q10, [x20, #720]
328 ldr q11, [x20, #736]
329 ldr q12, [x20, #752]
330 ldr q13, [x20, #768]
331 ldr q14, [x20, #784]
332 ldr q15, [x20, #800]
333 ldr q16, [x20, #816]
334 ldr q17, [x20, #832]
335 ldr q18, [x20, #848]
336
337 dup v29.2d, x14
338 dup v30.2s, w10
339
340 umull v0.2d, v10.2s, v20.2s
341 umull v1.2d, v10.2s, v21.2s
342 umull v2.2d, v10.2s, v22.2s
343 umull v3.2d, v10.2s, v23.2s
344 umull v4.2d, v10.2s, v24.2s
345 umull v5.2d, v10.2s, v25.2s
346 umull v6.2d, v10.2s, v26.2s
347 umull v7.2d, v10.2s, v27.2s
348 umull v8.2d, v10.2s, v28.2s
349
350 umlal v1.2d, v11.2s, v20.2s
351 umlal v2.2d, v11.2s, v21.2s
352 umlal v3.2d, v11.2s, v22.2s
353 umlal v4.2d, v11.2s, v23.2s
354 umlal v5.2d, v11.2s, v24.2s
355 umlal v6.2d, v11.2s, v25.2s
356 umlal v7.2d, v11.2s, v26.2s
357 umlal v8.2d, v11.2s, v27.2s
358 umull v9.2d, v11.2s, v28.2s
359
360 umlal v2.2d, v12.2s, v20.2s
361 umlal v3.2d, v12.2s, v21.2s
362 umlal v4.2d, v12.2s, v22.2s
363 umlal v5.2d, v12.2s, v23.2s
364 umlal v6.2d, v12.2s, v24.2s
365 umlal v7.2d, v12.2s, v25.2s
366 umlal v8.2d, v12.2s, v26.2s
367 umlal v9.2d, v12.2s, v27.2s
368 umull v10.2d, v12.2s, v28.2s
369
370 umlal v3.2d, v13.2s, v20.2s
371 umlal v4.2d, v13.2s, v21.2s
372 umlal v5.2d, v13.2s, v22.2s
373 umlal v6.2d, v13.2s, v23.2s
374 umlal v7.2d, v13.2s, v24.2s
375 umlal v8.2d, v13.2s, v25.2s
376 umlal v9.2d, v13.2s, v26.2s
377 umlal v10.2d, v13.2s, v27.2s
378 umull v11.2d, v13.2s, v28.2s
379
380 umlal v4.2d, v14.2s, v20.2s
381 umlal v5.2d, v14.2s, v21.2s
382 umlal v6.2d, v14.2s, v22.2s
383 umlal v7.2d, v14.2s, v23.2s
384 umlal v8.2d, v14.2s, v24.2s
385 umlal v9.2d, v14.2s, v25.2s
386 umlal v10.2d, v14.2s, v26.2s
387 umlal v11.2d, v14.2s, v27.2s
388 umull v12.2d, v14.2s, v28.2s
389
390 umlal v5.2d, v15.2s, v20.2s
391 umlal v6.2d, v15.2s, v21.2s
392 umlal v7.2d, v15.2s, v22.2s
393 umlal v8.2d, v15.2s, v23.2s
394 umlal v9.2d, v15.2s, v24.2s
395 umlal v10.2d, v15.2s, v25.2s
396 umlal v11.2d, v15.2s, v26.2s
397 umlal v12.2d, v15.2s, v27.2s
398 umull v13.2d, v15.2s, v28.2s
399
400 umlal v6.2d, v16.2s, v20.2s
401 umlal v7.2d, v16.2s, v21.2s
402 umlal v8.2d, v16.2s, v22.2s
403 umlal v9.2d, v16.2s, v23.2s
404 umlal v10.2d, v16.2s, v24.2s
405 umlal v11.2d, v16.2s, v25.2s
406 umlal v12.2d, v16.2s, v26.2s
407 umlal v13.2d, v16.2s, v27.2s
408 umull v14.2d, v16.2s, v28.2s
409
410 umlal v7.2d, v17.2s, v20.2s
411 umlal v8.2d, v17.2s, v21.2s
412 umlal v9.2d, v17.2s, v22.2s
413 umlal v10.2d, v17.2s, v23.2s
414 umlal v11.2d, v17.2s, v24.2s
415 umlal v12.2d, v17.2s, v25.2s
416 umlal v13.2d, v17.2s, v26.2s
417 umlal v14.2d, v17.2s, v27.2s
418 umull v15.2d, v17.2s, v28.2s
419
420 umlal v8.2d, v18.2s, v20.2s
421 umlal v9.2d, v18.2s, v21.2s
422 umlal v10.2d, v18.2s, v22.2s
423 umlal v11.2d, v18.2s, v23.2s
424 umlal v12.2d, v18.2s, v24.2s
425 umlal v13.2d, v18.2s, v25.2s
426 umlal v14.2d, v18.2s, v26.2s
427 umlal v15.2d, v18.2s, v27.2s
428 umull v16.2d, v18.2s, v28.2s
429
430 usra v10.2d, v9.2d, #29
431 and v9.16b, v9.16b, v29.16b
432 xtn v9.2s, v9.2d
433 umull v9.2d, v9.2s, v30.2s
434 add v0.2d, v0.2d, v9.2d
435
436 usra v11.2d, v10.2d, #29
437 and v10.16b, v10.16b, v29.16b
438 xtn v10.2s, v10.2d
439 umull v10.2d, v10.2s, v30.2s
440 add v1.2d, v1.2d, v10.2d
441
442 usra v12.2d, v11.2d, #29
443 and v11.16b, v11.16b, v29.16b
444 xtn v11.2s, v11.2d
445 umull v11.2d, v11.2s, v30.2s
446 add v2.2d, v2.2d, v11.2d
447
448 usra v13.2d, v12.2d, #29
449 and v12.16b, v12.16b, v29.16b
450 xtn v12.2s, v12.2d
451 umull v12.2d, v12.2s, v30.2s
452 add v3.2d, v3.2d, v12.2d
453
454 usra v14.2d, v13.2d, #29
455 and v13.16b, v13.16b, v29.16b
456 xtn v13.2s, v13.2d
457 umull v13.2d, v13.2s, v30.2s
458 add v4.2d, v4.2d, v13.2d
459
460 usra v15.2d, v14.2d, #29
461 and v14.16b, v14.16b, v29.16b
462 xtn v14.2s, v14.2d
463 umull v14.2d, v14.2s, v30.2s
464 add v5.2d, v5.2d, v14.2d
465
466 usra v16.2d, v15.2d, #29
467 and v15.16b, v15.16b, v29.16b
468 xtn v15.2s, v15.2d
469 umull v15.2d, v15.2s, v30.2s
470 add v6.2d, v6.2d, v15.2d
471
472 ushr v9.2d, v16.2d, #29
473 and v16.16b, v16.16b, v29.16b
474 xtn v16.2s, v16.2d
475 umull v16.2d, v16.2s, v30.2s
476 add v7.2d, v7.2d, v16.2d
477
478 xtn v9.2s, v9.2d
479 umull v9.2d, v9.2s, v30.2s
480 add v8.2d, v8.2d, v9.2d
481
482 dup v30.2d, x13
483
484 usra v5.2d, v4.2d, #29
485 and v4.16b, v4.16b, v29.16b
486
487 usra v1.2d, v0.2d, #29
488 and v0.16b, v0.16b, v29.16b
489
490 usra v6.2d, v5.2d, #29
491 and v5.16b, v5.16b, v29.16b
492
493 usra v2.2d, v1.2d, #29
494 and v1.16b, v1.16b, v29.16b
495
496 usra v7.2d, v6.2d, #29
497 and v6.16b, v6.16b, v29.16b
498
499 usra v3.2d, v2.2d, #29
500 and v2.16b, v2.16b, v29.16b
501
502 usra v8.2d, v7.2d, #29
503 and v7.16b, v7.16b, v29.16b
504
505 usra v4.2d, v3.2d, #29
506 and v3.16b, v3.16b, v29.16b
507
508 bic v10.16b, v8.16b, v30.16b
509 usra v0.2d, v10.2d, #23
510 usra v0.2d, v10.2d, #22
511 usra v0.2d, v10.2d, #19
512 and v8.16b, v8.16b, v30.16b
513
514 usra v5.2d, v4.2d, #29
515 and v4.16b, v4.16b, v29.16b
516
517 usra v1.2d, v0.2d, #29
518 and v0.16b, v0.16b, v29.16b
519
520 xtn v10.2s, v0.2d
521 xtn v11.2s, v1.2d
522 xtn v12.2s, v2.2d
523 xtn v13.2s, v3.2d
524 xtn v14.2s, v4.2d
525 xtn v15.2s, v5.2d
526 xtn v16.2s, v6.2d
527 xtn v17.2s, v7.2d
528 xtn v18.2s, v8.2d
529
530 str q10, [x20, #432]
531 str q11, [x20, #448]
532 str q12, [x20, #464]
533 str q13, [x20, #480]
534 str q14, [x20, #496]
535 str q15, [x20, #512]
536 str q16, [x20, #528]
537 str q17, [x20, #544]
538 str q18, [x20, #560]
539
540 // Z3 ← T2 · T3
541 ldr q10, [x20, #864]
542 ldr q11, [x20, #880]
543 ldr q12, [x20, #896]
544 ldr q13, [x20, #912]
545 ldr q14, [x20, #928]
546 ldr q15, [x20, #944]
547 ldr q16, [x20, #960]
548 ldr q17, [x20, #976]
549 ldr q18, [x20, #992]
550
551 ldr q20, [x20, #1008]
552 ldr q21, [x20, #1024]
553 ldr q22, [x20, #1040]
554 ldr q23, [x20, #1056]
555 ldr q24, [x20, #1072]
556 ldr q25, [x20, #1088]
557 ldr q26, [x20, #1104]
558 ldr q27, [x20, #1120]
559 ldr q28, [x20, #1136]
560
561 dup v29.2d, x14
562 dup v30.2s, w10
563
564 umull v0.2d, v10.2s, v20.2s
565 umull v1.2d, v10.2s, v21.2s
566 umull v2.2d, v10.2s, v22.2s
567 umull v3.2d, v10.2s, v23.2s
568 umull v4.2d, v10.2s, v24.2s
569 umull v5.2d, v10.2s, v25.2s
570 umull v6.2d, v10.2s, v26.2s
571 umull v7.2d, v10.2s, v27.2s
572 umull v8.2d, v10.2s, v28.2s
573
574 umlal v1.2d, v11.2s, v20.2s
575 umlal v2.2d, v11.2s, v21.2s
576 umlal v3.2d, v11.2s, v22.2s
577 umlal v4.2d, v11.2s, v23.2s
578 umlal v5.2d, v11.2s, v24.2s
579 umlal v6.2d, v11.2s, v25.2s
580 umlal v7.2d, v11.2s, v26.2s
581 umlal v8.2d, v11.2s, v27.2s
582 umull v9.2d, v11.2s, v28.2s
583
584 umlal v2.2d, v12.2s, v20.2s
585 umlal v3.2d, v12.2s, v21.2s
586 umlal v4.2d, v12.2s, v22.2s
587 umlal v5.2d, v12.2s, v23.2s
588 umlal v6.2d, v12.2s, v24.2s
589 umlal v7.2d, v12.2s, v25.2s
590 umlal v8.2d, v12.2s, v26.2s
591 umlal v9.2d, v12.2s, v27.2s
592 umull v10.2d, v12.2s, v28.2s
593
594 umlal v3.2d, v13.2s, v20.2s
595 umlal v4.2d, v13.2s, v21.2s
596 umlal v5.2d, v13.2s, v22.2s
597 umlal v6.2d, v13.2s, v23.2s
598 umlal v7.2d, v13.2s, v24.2s
599 umlal v8.2d, v13.2s, v25.2s
600 umlal v9.2d, v13.2s, v26.2s
601 umlal v10.2d, v13.2s, v27.2s
602 umull v11.2d, v13.2s, v28.2s
603
604 umlal v4.2d, v14.2s, v20.2s
605 umlal v5.2d, v14.2s, v21.2s
606 umlal v6.2d, v14.2s, v22.2s
607 umlal v7.2d, v14.2s, v23.2s
608 umlal v8.2d, v14.2s, v24.2s
609 umlal v9.2d, v14.2s, v25.2s
610 umlal v10.2d, v14.2s, v26.2s
611 umlal v11.2d, v14.2s, v27.2s
612 umull v12.2d, v14.2s, v28.2s
613
614 umlal v5.2d, v15.2s, v20.2s
615 umlal v6.2d, v15.2s, v21.2s
616 umlal v7.2d, v15.2s, v22.2s
617 umlal v8.2d, v15.2s, v23.2s
618 umlal v9.2d, v15.2s, v24.2s
619 umlal v10.2d, v15.2s, v25.2s
620 umlal v11.2d, v15.2s, v26.2s
621 umlal v12.2d, v15.2s, v27.2s
622 umull v13.2d, v15.2s, v28.2s
623
624 umlal v6.2d, v16.2s, v20.2s
625 umlal v7.2d, v16.2s, v21.2s
626 umlal v8.2d, v16.2s, v22.2s
627 umlal v9.2d, v16.2s, v23.2s
628 umlal v10.2d, v16.2s, v24.2s
629 umlal v11.2d, v16.2s, v25.2s
630 umlal v12.2d, v16.2s, v26.2s
631 umlal v13.2d, v16.2s, v27.2s
632 umull v14.2d, v16.2s, v28.2s
633
634 umlal v7.2d, v17.2s, v20.2s
635 umlal v8.2d, v17.2s, v21.2s
636 umlal v9.2d, v17.2s, v22.2s
637 umlal v10.2d, v17.2s, v23.2s
638 umlal v11.2d, v17.2s, v24.2s
639 umlal v12.2d, v17.2s, v25.2s
640 umlal v13.2d, v17.2s, v26.2s
641 umlal v14.2d, v17.2s, v27.2s
642 umull v15.2d, v17.2s, v28.2s
643
644 umlal v8.2d, v18.2s, v20.2s
645 umlal v9.2d, v18.2s, v21.2s
646 umlal v10.2d, v18.2s, v22.2s
647 umlal v11.2d, v18.2s, v23.2s
648 umlal v12.2d, v18.2s, v24.2s
649 umlal v13.2d, v18.2s, v25.2s
650 umlal v14.2d, v18.2s, v26.2s
651 umlal v15.2d, v18.2s, v27.2s
652 umull v16.2d, v18.2s, v28.2s
653
654 usra v10.2d, v9.2d, #29
655 and v9.16b, v9.16b, v29.16b
656 xtn v9.2s, v9.2d
657 umull v9.2d, v9.2s, v30.2s
658 add v0.2d, v0.2d, v9.2d
659
660 usra v11.2d, v10.2d, #29
661 and v10.16b, v10.16b, v29.16b
662 xtn v10.2s, v10.2d
663 umull v10.2d, v10.2s, v30.2s
664 add v1.2d, v1.2d, v10.2d
665
666 usra v12.2d, v11.2d, #29
667 and v11.16b, v11.16b, v29.16b
668 xtn v11.2s, v11.2d
669 umull v11.2d, v11.2s, v30.2s
670 add v2.2d, v2.2d, v11.2d
671
672 usra v13.2d, v12.2d, #29
673 and v12.16b, v12.16b, v29.16b
674 xtn v12.2s, v12.2d
675 umull v12.2d, v12.2s, v30.2s
676 add v3.2d, v3.2d, v12.2d
677
678 usra v14.2d, v13.2d, #29
679 and v13.16b, v13.16b, v29.16b
680 xtn v13.2s, v13.2d
681 umull v13.2d, v13.2s, v30.2s
682 add v4.2d, v4.2d, v13.2d
683
684 usra v15.2d, v14.2d, #29
685 and v14.16b, v14.16b, v29.16b
686 xtn v14.2s, v14.2d
687 umull v14.2d, v14.2s, v30.2s
688 add v5.2d, v5.2d, v14.2d
689
690 usra v16.2d, v15.2d, #29
691 and v15.16b, v15.16b, v29.16b
692 xtn v15.2s, v15.2d
693 umull v15.2d, v15.2s, v30.2s
694 add v6.2d, v6.2d, v15.2d
695
696 ushr v9.2d, v16.2d, #29
697 and v16.16b, v16.16b, v29.16b
698 xtn v16.2s, v16.2d
699 umull v16.2d, v16.2s, v30.2s
700 add v7.2d, v7.2d, v16.2d
701
702 xtn v9.2s, v9.2d
703 umull v9.2d, v9.2s, v30.2s
704 add v8.2d, v8.2d, v9.2d
705
706 dup v30.2d, x13
707
708 usra v5.2d, v4.2d, #29
709 and v4.16b, v4.16b, v29.16b
710
711 usra v1.2d, v0.2d, #29
712 and v0.16b, v0.16b, v29.16b
713
714 usra v6.2d, v5.2d, #29
715 and v5.16b, v5.16b, v29.16b
716
717 usra v2.2d, v1.2d, #29
718 and v1.16b, v1.16b, v29.16b
719
720 usra v7.2d, v6.2d, #29
721 and v6.16b, v6.16b, v29.16b
722
723 usra v3.2d, v2.2d, #29
724 and v2.16b, v2.16b, v29.16b
725
726 usra v8.2d, v7.2d, #29
727 and v7.16b, v7.16b, v29.16b
728
729 usra v4.2d, v3.2d, #29
730 and v3.16b, v3.16b, v29.16b
731
732 bic v10.16b, v8.16b, v30.16b
733 usra v0.2d, v10.2d, #23
734 usra v0.2d, v10.2d, #22
735 usra v0.2d, v10.2d, #19
736 and v8.16b, v8.16b, v30.16b
737
738 usra v5.2d, v4.2d, #29
739 and v4.16b, v4.16b, v29.16b
740
741 usra v1.2d, v0.2d, #29
742 and v0.16b, v0.16b, v29.16b
743
744 xtn v10.2s, v0.2d
745 xtn v11.2s, v1.2d
746 xtn v12.2s, v2.2d
747 xtn v13.2s, v3.2d
748 xtn v14.2s, v4.2d
749 xtn v15.2s, v5.2d
750 xtn v16.2s, v6.2d
751 xtn v17.2s, v7.2d
752 xtn v18.2s, v8.2d
753
754 str q10, [x20, #576]
755 str q11, [x20, #592]
756 str q12, [x20, #608]
757 str q13, [x20, #624]
758 str q14, [x20, #640]
759 str q15, [x20, #656]
760 str q16, [x20, #672]
761 str q17, [x20, #688]
762 str q18, [x20, #704]
763
764 // conditional select
765 ldp x8, x9, [sp, #176]
766
767 lsr x8, x8, x15
768 lsr x9, x9, x15
769
770 mov v31.d[0], x8
771 mov v31.d[1], x9
772
773 xtn v31.2s, v31.2d
774
775 dup v30.2s, w4
776
777 and v29.16b, v31.16b, v30.16b
778
779 ldr q28, [sp, #160]
780 eor v31.16b, v29.16b, v28.16b
781 str q29, [sp, #160]
782
783 sub x15, x15, #1
784
785 dup v30.2s, wzr
786 sub v30.2s, v30.2s, v31.2s
787
788 // T1 = CSelect(T1,T3)
789 ldr q10, [x20, #720]
790 ldr q11, [x20, #736]
791 ldr q12, [x20, #752]
792 ldr q13, [x20, #768]
793 ldr q14, [x20, #784]
794 ldr q15, [x20, #800]
795 ldr q16, [x20, #816]
796 ldr q17, [x20, #832]
797 ldr q18, [x20, #848]
798
799 ldr q20, [x20, #1008]
800 ldr q21, [x20, #1024]
801 ldr q22, [x20, #1040]
802 ldr q23, [x20, #1056]
803 ldr q24, [x20, #1072]
804 ldr q25, [x20, #1088]
805 ldr q26, [x20, #1104]
806 ldr q27, [x20, #1120]
807 ldr q28, [x20, #1136]
808
809 eor v0.16b, v10.16b, v20.16b
810 eor v1.16b, v11.16b, v21.16b
811 eor v2.16b, v12.16b, v22.16b
812 eor v3.16b, v13.16b, v23.16b
813 eor v4.16b, v14.16b, v24.16b
814 eor v5.16b, v15.16b, v25.16b
815 eor v6.16b, v16.16b, v26.16b
816 eor v7.16b, v17.16b, v27.16b
817 eor v8.16b, v18.16b, v28.16b
818
819 and v0.16b, v0.16b, v30.16b
820 and v1.16b, v1.16b, v30.16b
821 and v2.16b, v2.16b, v30.16b
822 and v3.16b, v3.16b, v30.16b
823 and v4.16b, v4.16b, v30.16b
824 and v5.16b, v5.16b, v30.16b
825 and v6.16b, v6.16b, v30.16b
826 and v7.16b, v7.16b, v30.16b
827 and v8.16b, v8.16b, v30.16b
828
829 eor v0.16b, v0.16b, v10.16b
830 eor v1.16b, v1.16b, v11.16b
831 eor v2.16b, v2.16b, v12.16b
832 eor v3.16b, v3.16b, v13.16b
833 eor v4.16b, v4.16b, v14.16b
834 eor v5.16b, v5.16b, v15.16b
835 eor v6.16b, v6.16b, v16.16b
836 eor v7.16b, v7.16b, v17.16b
837 eor v8.16b, v8.16b, v18.16b
838
839 str q0, [x20, #720]
840 str q1, [x20, #736]
841 str q2, [x20, #752]
842 str q3, [x20, #768]
843 str q4, [x20, #784]
844 str q5, [x20, #800]
845 str q6, [x20, #816]
846 str q7, [x20, #832]
847 str q8, [x20, #848]
848
849 // T2 = CSelect(T2,T4)
850 ldr q10, [x20, #864]
851 ldr q11, [x20, #880]
852 ldr q12, [x20, #896]
853 ldr q13, [x20, #912]
854 ldr q14, [x20, #928]
855 ldr q15, [x20, #944]
856 ldr q16, [x20, #960]
857 ldr q17, [x20, #976]
858 ldr q18, [x20, #992]
859
860 ldr q20, [x20, #1152]
861 ldr q21, [x20, #1168]
862 ldr q22, [x20, #1184]
863 ldr q23, [x20, #1200]
864 ldr q24, [x20, #1216]
865 ldr q25, [x20, #1232]
866 ldr q26, [x20, #1248]
867 ldr q27, [x20, #1264]
868 ldr q28, [x20, #1280]
869
870 eor v0.16b, v10.16b, v20.16b
871 eor v1.16b, v11.16b, v21.16b
872 eor v2.16b, v12.16b, v22.16b
873 eor v3.16b, v13.16b, v23.16b
874 eor v4.16b, v14.16b, v24.16b
875 eor v5.16b, v15.16b, v25.16b
876 eor v6.16b, v16.16b, v26.16b
877 eor v7.16b, v17.16b, v27.16b
878 eor v8.16b, v18.16b, v28.16b
879
880 and v0.16b, v0.16b, v30.16b
881 and v1.16b, v1.16b, v30.16b
882 and v2.16b, v2.16b, v30.16b
883 and v3.16b, v3.16b, v30.16b
884 and v4.16b, v4.16b, v30.16b
885 and v5.16b, v5.16b, v30.16b
886 and v6.16b, v6.16b, v30.16b
887 and v7.16b, v7.16b, v30.16b
888 and v8.16b, v8.16b, v30.16b
889
890 eor v10.16b, v0.16b, v10.16b
891 eor v11.16b, v1.16b, v11.16b
892 eor v12.16b, v2.16b, v12.16b
893 eor v13.16b, v3.16b, v13.16b
894 eor v14.16b, v4.16b, v14.16b
895 eor v15.16b, v5.16b, v15.16b
896 eor v16.16b, v6.16b, v16.16b
897 eor v17.16b, v7.16b, v17.16b
898 eor v18.16b, v8.16b, v18.16b
899
900 // T2 ← T2^2
901 dup v29.2d, x14
902 dup v30.2s, w10
903
904 add v20.2d, v10.2d, v10.2d
905 add v21.2d, v11.2d, v11.2d
906 add v22.2d, v12.2d, v12.2d
907 add v23.2d, v13.2d, v13.2d
908 add v24.2d, v14.2d, v14.2d
909 add v25.2d, v15.2d, v15.2d
910 add v26.2d, v16.2d, v16.2d
911 add v27.2d, v17.2d, v17.2d
912
913 umull v0.2d, v10.2s, v10.2s
914 umull v1.2d, v20.2s, v11.2s
915 umull v2.2d, v20.2s, v12.2s
916 umull v3.2d, v20.2s, v13.2s
917 umull v4.2d, v20.2s, v14.2s
918 umull v5.2d, v20.2s, v15.2s
919 umull v6.2d, v20.2s, v16.2s
920 umull v7.2d, v20.2s, v17.2s
921 umull v8.2d, v20.2s, v18.2s
922
923 umlal v2.2d, v11.2s, v11.2s
924 umlal v3.2d, v21.2s, v12.2s
925 umlal v4.2d, v21.2s, v13.2s
926 umlal v5.2d, v21.2s, v14.2s
927 umlal v6.2d, v21.2s, v15.2s
928 umlal v7.2d, v21.2s, v16.2s
929 umlal v8.2d, v21.2s, v17.2s
930 umull v9.2d, v21.2s, v18.2s
931
932 umlal v4.2d, v12.2s, v12.2s
933 umlal v5.2d, v22.2s, v13.2s
934 umlal v6.2d, v22.2s, v14.2s
935 umlal v7.2d, v22.2s, v15.2s
936 umlal v8.2d, v22.2s, v16.2s
937 umlal v9.2d, v22.2s, v17.2s
938 umull v10.2d, v22.2s, v18.2s
939
940 umlal v6.2d, v13.2s, v13.2s
941 umlal v8.2d, v23.2s, v15.2s
942 umlal v9.2d, v23.2s, v16.2s
943 umlal v10.2d, v23.2s, v17.2s
944 umull v11.2d, v23.2s, v18.2s
945
946 umlal v8.2d, v14.2s, v14.2s
947 umlal v9.2d, v24.2s, v15.2s
948 umlal v10.2d, v24.2s, v16.2s
949 umlal v11.2d, v24.2s, v17.2s
950 umull v12.2d, v24.2s, v18.2s
951
952 umlal v10.2d, v15.2s, v15.2s
953 umlal v11.2d, v25.2s, v16.2s
954 umlal v12.2d, v25.2s, v17.2s
955 umull v13.2d, v25.2s, v18.2s
956
957 usra v10.2d, v9.2d, #29
958 and v9.16b, v9.16b, v29.16b
959 xtn v9.2s, v9.2d
960 umull v9.2d, v9.2s, v30.2s
961 add v0.2d, v0.2d, v9.2d
962
963 usra v11.2d, v10.2d, #29
964 and v10.16b, v10.16b, v29.16b
965 xtn v10.2s, v10.2d
966 umull v10.2d, v10.2s, v30.2s
967 add v1.2d, v1.2d, v10.2d
968
969 umlal v12.2d, v16.2s, v16.2s
970 umlal v13.2d, v26.2s, v17.2s
971 umull v24.2d, v26.2s, v18.2s
972
973 usra v12.2d, v11.2d, #29
974 and v11.16b, v11.16b, v29.16b
975 xtn v11.2s, v11.2d
976 umull v11.2d, v11.2s, v30.2s
977 add v2.2d, v2.2d, v11.2d
978
979 usra v13.2d, v12.2d, #29
980 and v12.16b, v12.16b, v29.16b
981 xtn v12.2s, v12.2d
982 umull v12.2d, v12.2s, v30.2s
983 add v3.2d, v3.2d, v12.2d
984
985 umlal v24.2d, v17.2s, v17.2s
986 umull v15.2d, v27.2s, v18.2s
987
988 usra v24.2d, v13.2d, #29
989 and v13.16b, v13.16b, v29.16b
990 xtn v13.2s, v13.2d
991 umull v13.2d, v13.2s, v30.2s
992 add v4.2d, v4.2d, v13.2d
993
994 usra v15.2d, v24.2d, #29
995 and v24.16b, v24.16b, v29.16b
996 xtn v24.2s, v24.2d
997 umull v24.2d, v24.2s, v30.2s
998 add v5.2d, v5.2d, v24.2d
999
1000 umull v16.2d, v18.2s, v18.2s
1001
1002 usra v16.2d, v15.2d, #29
1003 and v15.16b, v15.16b, v29.16b
1004 xtn v15.2s, v15.2d
1005 umull v15.2d, v15.2s, v30.2s
1006 add v6.2d, v6.2d, v15.2d
1007
1008 ushr v9.2d, v16.2d, #29
1009 and v16.16b, v16.16b, v29.16b
1010 xtn v16.2s, v16.2d
1011 umull v16.2d, v16.2s, v30.2s
1012 add v7.2d, v7.2d, v16.2d
1013
1014 xtn v9.2s, v9.2d
1015 umull v9.2d, v9.2s, v30.2s
1016 add v8.2d, v8.2d, v9.2d
1017
1018 dup v30.2d, x13
1019
1020 usra v5.2d, v4.2d, #29
1021 and v4.16b, v4.16b, v29.16b
1022
1023 usra v1.2d, v0.2d, #29
1024 and v0.16b, v0.16b, v29.16b
1025
1026 usra v6.2d, v5.2d, #29
1027 and v5.16b, v5.16b, v29.16b
1028
1029 usra v2.2d, v1.2d, #29
1030 and v1.16b, v1.16b, v29.16b
1031
1032 usra v7.2d, v6.2d, #29
1033 and v6.16b, v6.16b, v29.16b
1034
1035 usra v3.2d, v2.2d, #29
1036 and v2.16b, v2.16b, v29.16b
1037
1038 usra v8.2d, v7.2d, #29
1039 and v7.16b, v7.16b, v29.16b
1040
1041 umlal v7.2d, v23.2s, v14.2s
1042 usra v8.2d, v7.2d, #29
1043 and v7.16b, v7.16b, v29.16b
1044
1045 usra v4.2d, v3.2d, #29
1046 and v3.16b, v3.16b, v29.16b
1047
1048 bic v10.16b, v8.16b, v30.16b
1049 usra v0.2d, v10.2d, #23
1050 usra v0.2d, v10.2d, #22
1051 usra v0.2d, v10.2d, #19
1052 and v8.16b, v8.16b, v30.16b
1053
1054 usra v5.2d, v4.2d, #29
1055 and v4.16b, v4.16b, v29.16b
1056
1057 usra v1.2d, v0.2d, #29
1058 and v0.16b, v0.16b, v29.16b
1059
1060 xtn v10.2s, v0.2d
1061 xtn v11.2s, v1.2d
1062 xtn v12.2s, v2.2d
1063 xtn v13.2s, v3.2d
1064 xtn v14.2s, v4.2d
1065 xtn v15.2s, v5.2d
1066 xtn v16.2s, v6.2d
1067 xtn v17.2s, v7.2d
1068 xtn v18.2s, v8.2d
1069
1070 str q10, [x20, #864]
1071 str q11, [x20, #880]
1072 str q12, [x20, #896]
1073 str q13, [x20, #912]
1074 str q14, [x20, #928]
1075 str q15, [x20, #944]
1076 str q16, [x20, #960]
1077 str q17, [x20, #976]
1078 str q18, [x20, #992]
1079
1080 // T1 ← T1^2
1081 ldr q10, [x20, #720]
1082 ldr q11, [x20, #736]
1083 ldr q12, [x20, #752]
1084 ldr q13, [x20, #768]
1085 ldr q14, [x20, #784]
1086 ldr q15, [x20, #800]
1087 ldr q16, [x20, #816]
1088 ldr q17, [x20, #832]
1089 ldr q18, [x20, #848]
1090
1091 dup v29.2d, x14
1092 dup v30.2s, w10
1093
1094 add v20.2d, v10.2d, v10.2d
1095 add v21.2d, v11.2d, v11.2d
1096 add v22.2d, v12.2d, v12.2d
1097 add v23.2d, v13.2d, v13.2d
1098 add v24.2d, v14.2d, v14.2d
1099 add v25.2d, v15.2d, v15.2d
1100 add v26.2d, v16.2d, v16.2d
1101 add v27.2d, v17.2d, v17.2d
1102
1103 umull v0.2d, v10.2s, v10.2s
1104 umull v1.2d, v20.2s, v11.2s
1105 umull v2.2d, v20.2s, v12.2s
1106 umull v3.2d, v20.2s, v13.2s
1107 umull v4.2d, v20.2s, v14.2s
1108 umull v5.2d, v20.2s, v15.2s
1109 umull v6.2d, v20.2s, v16.2s
1110 umull v7.2d, v20.2s, v17.2s
1111 umull v8.2d, v20.2s, v18.2s
1112
1113 umlal v2.2d, v11.2s, v11.2s
1114 umlal v3.2d, v21.2s, v12.2s
1115 umlal v4.2d, v21.2s, v13.2s
1116 umlal v5.2d, v21.2s, v14.2s
1117 umlal v6.2d, v21.2s, v15.2s
1118 umlal v7.2d, v21.2s, v16.2s
1119 umlal v8.2d, v21.2s, v17.2s
1120 umull v9.2d, v21.2s, v18.2s
1121
1122 umlal v4.2d, v12.2s, v12.2s
1123 umlal v5.2d, v22.2s, v13.2s
1124 umlal v6.2d, v22.2s, v14.2s
1125 umlal v7.2d, v22.2s, v15.2s
1126 umlal v8.2d, v22.2s, v16.2s
1127 umlal v9.2d, v22.2s, v17.2s
1128 umull v10.2d, v22.2s, v18.2s
1129
1130 umlal v6.2d, v13.2s, v13.2s
1131 umlal v8.2d, v23.2s, v15.2s
1132 umlal v9.2d, v23.2s, v16.2s
1133 umlal v10.2d, v23.2s, v17.2s
1134 umull v11.2d, v23.2s, v18.2s
1135
1136 umlal v8.2d, v14.2s, v14.2s
1137 umlal v9.2d, v24.2s, v15.2s
1138 umlal v10.2d, v24.2s, v16.2s
1139 umlal v11.2d, v24.2s, v17.2s
1140 umull v12.2d, v24.2s, v18.2s
1141
1142 umlal v10.2d, v15.2s, v15.2s
1143 umlal v11.2d, v25.2s, v16.2s
1144 umlal v12.2d, v25.2s, v17.2s
1145 umull v13.2d, v25.2s, v18.2s
1146
1147 usra v10.2d, v9.2d, #29
1148 and v9.16b, v9.16b, v29.16b
1149 xtn v9.2s, v9.2d
1150 umull v9.2d, v9.2s, v30.2s
1151 add v0.2d, v0.2d, v9.2d
1152
1153 usra v11.2d, v10.2d, #29
1154 and v10.16b, v10.16b, v29.16b
1155 xtn v10.2s, v10.2d
1156 umull v10.2d, v10.2s, v30.2s
1157 add v1.2d, v1.2d, v10.2d
1158
1159 umlal v12.2d, v16.2s, v16.2s
1160 umlal v13.2d, v26.2s, v17.2s
1161 umull v24.2d, v26.2s, v18.2s
1162
1163 usra v12.2d, v11.2d, #29
1164 and v11.16b, v11.16b, v29.16b
1165 xtn v11.2s, v11.2d
1166 umull v11.2d, v11.2s, v30.2s
1167 add v2.2d, v2.2d, v11.2d
1168
1169 usra v13.2d, v12.2d, #29
1170 and v12.16b, v12.16b, v29.16b
1171 xtn v12.2s, v12.2d
1172 umull v12.2d, v12.2s, v30.2s
1173 add v3.2d, v3.2d, v12.2d
1174
1175 umlal v24.2d, v17.2s, v17.2s
1176 umull v15.2d, v27.2s, v18.2s
1177
1178 usra v24.2d, v13.2d, #29
1179 and v13.16b, v13.16b, v29.16b
1180 xtn v13.2s, v13.2d
1181 umull v13.2d, v13.2s, v30.2s
1182 add v4.2d, v4.2d, v13.2d
1183
1184 usra v15.2d, v24.2d, #29
1185 and v24.16b, v24.16b, v29.16b
1186 xtn v24.2s, v24.2d
1187 umull v24.2d, v24.2s, v30.2s
1188 add v5.2d, v5.2d, v24.2d
1189
1190 umull v16.2d, v18.2s, v18.2s
1191
1192 usra v16.2d, v15.2d, #29
1193 and v15.16b, v15.16b, v29.16b
1194 xtn v15.2s, v15.2d
1195 umull v15.2d, v15.2s, v30.2s
1196 add v6.2d, v6.2d, v15.2d
1197
1198 ushr v9.2d, v16.2d, #29
1199 and v16.16b, v16.16b, v29.16b
1200 xtn v16.2s, v16.2d
1201 umull v16.2d, v16.2s, v30.2s
1202 add v7.2d, v7.2d, v16.2d
1203
1204 xtn v9.2s, v9.2d
1205 umull v9.2d, v9.2s, v30.2s
1206 add v8.2d, v8.2d, v9.2d
1207
1208 dup v30.2d, x13
1209
1210 usra v5.2d, v4.2d, #29
1211 and v4.16b, v4.16b, v29.16b
1212
1213 usra v1.2d, v0.2d, #29
1214 and v0.16b, v0.16b, v29.16b
1215
1216 usra v6.2d, v5.2d, #29
1217 and v5.16b, v5.16b, v29.16b
1218
1219 usra v2.2d, v1.2d, #29
1220 and v1.16b, v1.16b, v29.16b
1221
1222 usra v7.2d, v6.2d, #29
1223 and v6.16b, v6.16b, v29.16b
1224
1225 usra v3.2d, v2.2d, #29
1226 and v2.16b, v2.16b, v29.16b
1227
1228 usra v8.2d, v7.2d, #29
1229 and v7.16b, v7.16b, v29.16b
1230
1231 umlal v7.2d, v23.2s, v14.2s
1232 usra v8.2d, v7.2d, #29
1233 and v7.16b, v7.16b, v29.16b
1234
1235 usra v4.2d, v3.2d, #29
1236 and v3.16b, v3.16b, v29.16b
1237
1238 bic v10.16b, v8.16b, v30.16b
1239 usra v0.2d, v10.2d, #23
1240 usra v0.2d, v10.2d, #22
1241 usra v0.2d, v10.2d, #19
1242 and v8.16b, v8.16b, v30.16b
1243
1244 usra v5.2d, v4.2d, #29
1245 and v4.16b, v4.16b, v29.16b
1246
1247 usra v1.2d, v0.2d, #29
1248 and v0.16b, v0.16b, v29.16b
1249
1250 xtn v10.2s, v0.2d
1251 xtn v11.2s, v1.2d
1252 xtn v12.2s, v2.2d
1253 xtn v13.2s, v3.2d
1254 xtn v14.2s, v4.2d
1255 xtn v15.2s, v5.2d
1256 xtn v16.2s, v6.2d
1257 xtn v17.2s, v7.2d
1258 xtn v18.2s, v8.2d
1259
1260 str q10, [x20, #720]
1261 str q11, [x20, #736]
1262 str q12, [x20, #752]
1263 str q13, [x20, #768]
1264 str q14, [x20, #784]
1265 str q15, [x20, #800]
1266 str q16, [x20, #816]
1267 str q17, [x20, #832]
1268 str q18, [x20, #848]
1269
1270 // X3
1271 ldr q0, [x20, #432]
1272 ldr q1, [x20, #448]
1273 ldr q2, [x20, #464]
1274 ldr q3, [x20, #480]
1275 ldr q4, [x20, #496]
1276 ldr q5, [x20, #512]
1277 ldr q6, [x20, #528]
1278 ldr q7, [x20, #544]
1279 ldr q8, [x20, #560]
1280
1281 // Z3
1282 ldr q10, [x20, #576]
1283 ldr q11, [x20, #592]
1284 ldr q12, [x20, #608]
1285 ldr q13, [x20, #624]
1286 ldr q14, [x20, #640]
1287 ldr q15, [x20, #656]
1288 ldr q16, [x20, #672]
1289 ldr q17, [x20, #688]
1290 ldr q18, [x20, #704]
1291
1292 // T3 ← X3 + Z3
1293 add v20.2s, v0.2s, v10.2s
1294 add v21.2s, v1.2s, v11.2s
1295 add v22.2s, v2.2s, v12.2s
1296 add v23.2s, v3.2s, v13.2s
1297 add v24.2s, v4.2s, v14.2s
1298 add v25.2s, v5.2s, v15.2s
1299 add v26.2s, v6.2s, v16.2s
1300 add v27.2s, v7.2s, v17.2s
1301 add v28.2s, v8.2s, v18.2s
1302
1303 str q20, [x20, #1008]
1304 str q21, [x20, #1024]
1305 str q22, [x20, #1040]
1306 str q23, [x20, #1056]
1307 str q24, [x20, #1072]
1308 str q25, [x20, #1088]
1309 str q26, [x20, #1104]
1310 str q27, [x20, #1120]
1311 str q28, [x20, #1136]
1312
1313 // Z3 ← X3 - Z3
1314 mov v30.d[0], x22
1315 mov v31.d[0], x23
1316 add v0.2s, v19.2s, v0.2s
1317 add v1.2s, v30.2s, v1.2s
1318 add v2.2s, v30.2s, v2.2s
1319 add v3.2s, v30.2s, v3.2s
1320 add v4.2s, v30.2s, v4.2s
1321 add v5.2s, v30.2s, v5.2s
1322 add v6.2s, v30.2s, v6.2s
1323 add v7.2s, v30.2s, v7.2s
1324 add v8.2s, v31.2s, v8.2s
1325
1326 sub v10.2s, v0.2s, v10.2s
1327 sub v11.2s, v1.2s, v11.2s
1328 sub v12.2s, v2.2s, v12.2s
1329 sub v13.2s, v3.2s, v13.2s
1330 sub v14.2s, v4.2s, v14.2s
1331 sub v15.2s, v5.2s, v15.2s
1332 sub v16.2s, v6.2s, v16.2s
1333 sub v17.2s, v7.2s, v17.2s
1334 sub v18.2s, v8.2s, v18.2s
1335
1336 // Z3 ← Z3^2
1337 dup v29.2d, x14
1338 dup v30.2s, w10
1339
1340 add v20.2d, v10.2d, v10.2d
1341 add v21.2d, v11.2d, v11.2d
1342 add v22.2d, v12.2d, v12.2d
1343 add v23.2d, v13.2d, v13.2d
1344 add v24.2d, v14.2d, v14.2d
1345 add v25.2d, v15.2d, v15.2d
1346 add v26.2d, v16.2d, v16.2d
1347 add v27.2d, v17.2d, v17.2d
1348
1349 umull v0.2d, v10.2s, v10.2s
1350 umull v1.2d, v20.2s, v11.2s
1351 umull v2.2d, v20.2s, v12.2s
1352 umull v3.2d, v20.2s, v13.2s
1353 umull v4.2d, v20.2s, v14.2s
1354 umull v5.2d, v20.2s, v15.2s
1355 umull v6.2d, v20.2s, v16.2s
1356 umull v7.2d, v20.2s, v17.2s
1357 umull v8.2d, v20.2s, v18.2s
1358
1359 umlal v2.2d, v11.2s, v11.2s
1360 umlal v3.2d, v21.2s, v12.2s
1361 umlal v4.2d, v21.2s, v13.2s
1362 umlal v5.2d, v21.2s, v14.2s
1363 umlal v6.2d, v21.2s, v15.2s
1364 umlal v7.2d, v21.2s, v16.2s
1365 umlal v8.2d, v21.2s, v17.2s
1366 umull v9.2d, v21.2s, v18.2s
1367
1368 umlal v4.2d, v12.2s, v12.2s
1369 umlal v5.2d, v22.2s, v13.2s
1370 umlal v6.2d, v22.2s, v14.2s
1371 umlal v7.2d, v22.2s, v15.2s
1372 umlal v8.2d, v22.2s, v16.2s
1373 umlal v9.2d, v22.2s, v17.2s
1374 umull v10.2d, v22.2s, v18.2s
1375
1376 umlal v6.2d, v13.2s, v13.2s
1377 umlal v8.2d, v23.2s, v15.2s
1378 umlal v9.2d, v23.2s, v16.2s
1379 umlal v10.2d, v23.2s, v17.2s
1380 umull v11.2d, v23.2s, v18.2s
1381
1382 umlal v8.2d, v14.2s, v14.2s
1383 umlal v9.2d, v24.2s, v15.2s
1384 umlal v10.2d, v24.2s, v16.2s
1385 umlal v11.2d, v24.2s, v17.2s
1386 umull v12.2d, v24.2s, v18.2s
1387
1388 umlal v10.2d, v15.2s, v15.2s
1389 umlal v11.2d, v25.2s, v16.2s
1390 umlal v12.2d, v25.2s, v17.2s
1391 umull v13.2d, v25.2s, v18.2s
1392
1393 usra v10.2d, v9.2d, #29
1394 and v9.16b, v9.16b, v29.16b
1395 xtn v9.2s, v9.2d
1396 umull v9.2d, v9.2s, v30.2s
1397 add v0.2d, v0.2d, v9.2d
1398
1399 usra v11.2d, v10.2d, #29
1400 and v10.16b, v10.16b, v29.16b
1401 xtn v10.2s, v10.2d
1402 umull v10.2d, v10.2s, v30.2s
1403 add v1.2d, v1.2d, v10.2d
1404
1405 umlal v12.2d, v16.2s, v16.2s
1406 umlal v13.2d, v26.2s, v17.2s
1407 umull v24.2d, v26.2s, v18.2s
1408
1409 usra v12.2d, v11.2d, #29
1410 and v11.16b, v11.16b, v29.16b
1411 xtn v11.2s, v11.2d
1412 umull v11.2d, v11.2s, v30.2s
1413 add v2.2d, v2.2d, v11.2d
1414
1415 usra v13.2d, v12.2d, #29
1416 and v12.16b, v12.16b, v29.16b
1417 xtn v12.2s, v12.2d
1418 umull v12.2d, v12.2s, v30.2s
1419 add v3.2d, v3.2d, v12.2d
1420
1421 umlal v24.2d, v17.2s, v17.2s
1422 umull v15.2d, v27.2s, v18.2s
1423
1424 usra v24.2d, v13.2d, #29
1425 and v13.16b, v13.16b, v29.16b
1426 xtn v13.2s, v13.2d
1427 umull v13.2d, v13.2s, v30.2s
1428 add v4.2d, v4.2d, v13.2d
1429
1430 usra v15.2d, v24.2d, #29
1431 and v24.16b, v24.16b, v29.16b
1432 xtn v24.2s, v24.2d
1433 umull v24.2d, v24.2s, v30.2s
1434 add v5.2d, v5.2d, v24.2d
1435
1436 umull v16.2d, v18.2s, v18.2s
1437
1438 usra v16.2d, v15.2d, #29
1439 and v15.16b, v15.16b, v29.16b
1440 xtn v15.2s, v15.2d
1441 umull v15.2d, v15.2s, v30.2s
1442 add v6.2d, v6.2d, v15.2d
1443
1444 ushr v9.2d, v16.2d, #29
1445 and v16.16b, v16.16b, v29.16b
1446 xtn v16.2s, v16.2d
1447 umull v16.2d, v16.2s, v30.2s
1448 add v7.2d, v7.2d, v16.2d
1449
1450 xtn v9.2s, v9.2d
1451 umull v9.2d, v9.2s, v30.2s
1452 add v8.2d, v8.2d, v9.2d
1453
1454 dup v30.2d, x13
1455
1456 usra v5.2d, v4.2d, #29
1457 and v4.16b, v4.16b, v29.16b
1458
1459 usra v1.2d, v0.2d, #29
1460 and v0.16b, v0.16b, v29.16b
1461
1462 usra v6.2d, v5.2d, #29
1463 and v5.16b, v5.16b, v29.16b
1464
1465 usra v2.2d, v1.2d, #29
1466 and v1.16b, v1.16b, v29.16b
1467
1468 usra v7.2d, v6.2d, #29
1469 and v6.16b, v6.16b, v29.16b
1470
1471 usra v3.2d, v2.2d, #29
1472 and v2.16b, v2.16b, v29.16b
1473
1474 usra v8.2d, v7.2d, #29
1475 and v7.16b, v7.16b, v29.16b
1476
1477 umlal v7.2d, v23.2s, v14.2s
1478 usra v8.2d, v7.2d, #29
1479 and v7.16b, v7.16b, v29.16b
1480
1481 usra v4.2d, v3.2d, #29
1482 and v3.16b, v3.16b, v29.16b
1483
1484 bic v10.16b, v8.16b, v30.16b
1485 usra v0.2d, v10.2d, #23
1486 usra v0.2d, v10.2d, #22
1487 usra v0.2d, v10.2d, #19
1488 and v8.16b, v8.16b, v30.16b
1489
1490 usra v5.2d, v4.2d, #29
1491 and v4.16b, v4.16b, v29.16b
1492
1493 usra v1.2d, v0.2d, #29
1494 and v0.16b, v0.16b, v29.16b
1495
1496 xtn v10.2s, v0.2d
1497 xtn v11.2s, v1.2d
1498 xtn v12.2s, v2.2d
1499 xtn v13.2s, v3.2d
1500 xtn v14.2s, v4.2d
1501 xtn v15.2s, v5.2d
1502 xtn v16.2s, v6.2d
1503 xtn v17.2s, v7.2d
1504 xtn v18.2s, v8.2d
1505
1506 str q10, [x20, #576]
1507 str q11, [x20, #592]
1508 str q12, [x20, #608]
1509 str q13, [x20, #624]
1510 str q14, [x20, #640]
1511 str q15, [x20, #656]
1512 str q16, [x20, #672]
1513 str q17, [x20, #688]
1514 str q18, [x20, #704]
1515
1516 // X3 ← T3^2
1517 ldr q10, [x20, #1008]
1518 ldr q11, [x20, #1024]
1519 ldr q12, [x20, #1040]
1520 ldr q13, [x20, #1056]
1521 ldr q14, [x20, #1072]
1522 ldr q15, [x20, #1088]
1523 ldr q16, [x20, #1104]
1524 ldr q17, [x20, #1120]
1525 ldr q18, [x20, #1136]
1526
1527 dup v29.2d, x14
1528 dup v30.2s, w10
1529
1530 add v20.2d, v10.2d, v10.2d
1531 add v21.2d, v11.2d, v11.2d
1532 add v22.2d, v12.2d, v12.2d
1533 add v23.2d, v13.2d, v13.2d
1534 add v24.2d, v14.2d, v14.2d
1535 add v25.2d, v15.2d, v15.2d
1536 add v26.2d, v16.2d, v16.2d
1537 add v27.2d, v17.2d, v17.2d
1538
1539 umull v0.2d, v10.2s, v10.2s
1540 umull v1.2d, v20.2s, v11.2s
1541 umull v2.2d, v20.2s, v12.2s
1542 umull v3.2d, v20.2s, v13.2s
1543 umull v4.2d, v20.2s, v14.2s
1544 umull v5.2d, v20.2s, v15.2s
1545 umull v6.2d, v20.2s, v16.2s
1546 umull v7.2d, v20.2s, v17.2s
1547 umull v8.2d, v20.2s, v18.2s
1548
1549 umlal v2.2d, v11.2s, v11.2s
1550 umlal v3.2d, v21.2s, v12.2s
1551 umlal v4.2d, v21.2s, v13.2s
1552 umlal v5.2d, v21.2s, v14.2s
1553 umlal v6.2d, v21.2s, v15.2s
1554 umlal v7.2d, v21.2s, v16.2s
1555 umlal v8.2d, v21.2s, v17.2s
1556 umull v9.2d, v21.2s, v18.2s
1557
1558 umlal v4.2d, v12.2s, v12.2s
1559 umlal v5.2d, v22.2s, v13.2s
1560 umlal v6.2d, v22.2s, v14.2s
1561 umlal v7.2d, v22.2s, v15.2s
1562 umlal v8.2d, v22.2s, v16.2s
1563 umlal v9.2d, v22.2s, v17.2s
1564 umull v10.2d, v22.2s, v18.2s
1565
1566 umlal v6.2d, v13.2s, v13.2s
1567 umlal v8.2d, v23.2s, v15.2s
1568 umlal v9.2d, v23.2s, v16.2s
1569 umlal v10.2d, v23.2s, v17.2s
1570 umull v11.2d, v23.2s, v18.2s
1571
1572 umlal v8.2d, v14.2s, v14.2s
1573 umlal v9.2d, v24.2s, v15.2s
1574 umlal v10.2d, v24.2s, v16.2s
1575 umlal v11.2d, v24.2s, v17.2s
1576 umull v12.2d, v24.2s, v18.2s
1577
1578 umlal v10.2d, v15.2s, v15.2s
1579 umlal v11.2d, v25.2s, v16.2s
1580 umlal v12.2d, v25.2s, v17.2s
1581 umull v13.2d, v25.2s, v18.2s
1582
1583 usra v10.2d, v9.2d, #29
1584 and v9.16b, v9.16b, v29.16b
1585 xtn v9.2s, v9.2d
1586 umull v9.2d, v9.2s, v30.2s
1587 add v0.2d, v0.2d, v9.2d
1588
1589 usra v11.2d, v10.2d, #29
1590 and v10.16b, v10.16b, v29.16b
1591 xtn v10.2s, v10.2d
1592 umull v10.2d, v10.2s, v30.2s
1593 add v1.2d, v1.2d, v10.2d
1594
1595 umlal v12.2d, v16.2s, v16.2s
1596 umlal v13.2d, v26.2s, v17.2s
1597 umull v24.2d, v26.2s, v18.2s
1598
1599 usra v12.2d, v11.2d, #29
1600 and v11.16b, v11.16b, v29.16b
1601 xtn v11.2s, v11.2d
1602 umull v11.2d, v11.2s, v30.2s
1603 add v2.2d, v2.2d, v11.2d
1604
1605 usra v13.2d, v12.2d, #29
1606 and v12.16b, v12.16b, v29.16b
1607 xtn v12.2s, v12.2d
1608 umull v12.2d, v12.2s, v30.2s
1609 add v3.2d, v3.2d, v12.2d
1610
1611 umlal v24.2d, v17.2s, v17.2s
1612 umull v15.2d, v27.2s, v18.2s
1613
1614 usra v24.2d, v13.2d, #29
1615 and v13.16b, v13.16b, v29.16b
1616 xtn v13.2s, v13.2d
1617 umull v13.2d, v13.2s, v30.2s
1618 add v4.2d, v4.2d, v13.2d
1619
1620 usra v15.2d, v24.2d, #29
1621 and v24.16b, v24.16b, v29.16b
1622 xtn v24.2s, v24.2d
1623 umull v24.2d, v24.2s, v30.2s
1624 add v5.2d, v5.2d, v24.2d
1625
1626 umull v16.2d, v18.2s, v18.2s
1627
1628 usra v16.2d, v15.2d, #29
1629 and v15.16b, v15.16b, v29.16b
1630 xtn v15.2s, v15.2d
1631 umull v15.2d, v15.2s, v30.2s
1632 add v6.2d, v6.2d, v15.2d
1633
1634 ushr v9.2d, v16.2d, #29
1635 and v16.16b, v16.16b, v29.16b
1636 xtn v16.2s, v16.2d
1637 umull v16.2d, v16.2s, v30.2s
1638 add v7.2d, v7.2d, v16.2d
1639
1640 xtn v9.2s, v9.2d
1641 umull v9.2d, v9.2s, v30.2s
1642 add v8.2d, v8.2d, v9.2d
1643
1644 dup v30.2d, x13
1645
1646 usra v5.2d, v4.2d, #29
1647 and v4.16b, v4.16b, v29.16b
1648
1649 usra v1.2d, v0.2d, #29
1650 and v0.16b, v0.16b, v29.16b
1651
1652 usra v6.2d, v5.2d, #29
1653 and v5.16b, v5.16b, v29.16b
1654
1655 usra v2.2d, v1.2d, #29
1656 and v1.16b, v1.16b, v29.16b
1657
1658 usra v7.2d, v6.2d, #29
1659 and v6.16b, v6.16b, v29.16b
1660
1661 usra v3.2d, v2.2d, #29
1662 and v2.16b, v2.16b, v29.16b
1663
1664 usra v8.2d, v7.2d, #29
1665 and v7.16b, v7.16b, v29.16b
1666
1667 umlal v7.2d, v23.2s, v14.2s
1668 usra v8.2d, v7.2d, #29
1669 and v7.16b, v7.16b, v29.16b
1670
1671 usra v4.2d, v3.2d, #29
1672 and v3.16b, v3.16b, v29.16b
1673
1674 bic v10.16b, v8.16b, v30.16b
1675 usra v0.2d, v10.2d, #23
1676 usra v0.2d, v10.2d, #22
1677 usra v0.2d, v10.2d, #19
1678 and v8.16b, v8.16b, v30.16b
1679
1680 usra v5.2d, v4.2d, #29
1681 and v4.16b, v4.16b, v29.16b
1682
1683 usra v1.2d, v0.2d, #29
1684 and v0.16b, v0.16b, v29.16b
1685
1686 xtn v10.2s, v0.2d
1687 xtn v11.2s, v1.2d
1688 xtn v12.2s, v2.2d
1689 xtn v13.2s, v3.2d
1690 xtn v14.2s, v4.2d
1691 xtn v15.2s, v5.2d
1692 xtn v16.2s, v6.2d
1693 xtn v17.2s, v7.2d
1694 xtn v18.2s, v8.2d
1695
1696 str q10, [x20, #432]
1697 str q11, [x20, #448]
1698 str q12, [x20, #464]
1699 str q13, [x20, #480]
1700 str q14, [x20, #496]
1701 str q15, [x20, #512]
1702 str q16, [x20, #528]
1703 str q17, [x20, #544]
1704 str q18, [x20, #560]
1705
1706 // T1
1707 ldr q10, [x20, #720]
1708 ldr q11, [x20, #736]
1709 ldr q12, [x20, #752]
1710 ldr q13, [x20, #768]
1711 ldr q14, [x20, #784]
1712 ldr q15, [x20, #800]
1713 ldr q16, [x20, #816]
1714 ldr q17, [x20, #832]
1715 ldr q18, [x20, #848]
1716
1717 // T2
1718 ldr q0, [x20, #864]
1719 ldr q1, [x20, #880]
1720 ldr q2, [x20, #896]
1721 ldr q3, [x20, #912]
1722 ldr q4, [x20, #928]
1723 ldr q5, [x20, #944]
1724 ldr q6, [x20, #960]
1725 ldr q7, [x20, #976]
1726 ldr q8, [x20, #992]
1727
1728 // T3 ← T1 - T2
1729 mov v30.d[0], x22
1730 mov v31.d[0], x23
1731 add v20.2s, v19.2s, v10.2s
1732 add v21.2s, v30.2s, v11.2s
1733 add v22.2s, v30.2s, v12.2s
1734 add v23.2s, v30.2s, v13.2s
1735 add v24.2s, v30.2s, v14.2s
1736 add v25.2s, v30.2s, v15.2s
1737 add v26.2s, v30.2s, v16.2s
1738 add v27.2s, v30.2s, v17.2s
1739 add v28.2s, v31.2s, v18.2s
1740
1741 sub v10.2s, v20.2s, v0.2s
1742 sub v11.2s, v21.2s, v1.2s
1743 sub v12.2s, v22.2s, v2.2s
1744 sub v13.2s, v23.2s, v3.2s
1745 sub v14.2s, v24.2s, v4.2s
1746 sub v15.2s, v25.2s, v5.2s
1747 sub v16.2s, v26.2s, v6.2s
1748 sub v17.2s, v27.2s, v7.2s
1749 sub v18.2s, v28.2s, v8.2s
1750
1751 str q10, [x20, #1008]
1752 str q11, [x20, #1024]
1753 str q12, [x20, #1040]
1754 str q13, [x20, #1056]
1755 str q14, [x20, #1072]
1756 str q15, [x20, #1088]
1757 str q16, [x20, #1104]
1758 str q17, [x20, #1120]
1759 str q18, [x20, #1136]
1760
1761 uxtl v0.2d, v0.2s
1762 uxtl v1.2d, v1.2s
1763 uxtl v2.2d, v2.2s
1764 uxtl v3.2d, v3.2s
1765 uxtl v4.2d, v4.2s
1766 uxtl v5.2d, v5.2s
1767 uxtl v6.2d, v6.2s
1768 uxtl v7.2d, v7.2s
1769 uxtl v8.2d, v8.2s
1770
1771 // T4 ← ((A + 2)/4) · T3 + T2
1772 mov v31.d[0], x12
1773
1774 umull v20.2d, v10.2s, v31.2s
1775 add v0.2d, v0.2d, v20.2d
1776 umull v21.2d, v11.2s, v31.2s
1777 add v1.2d, v1.2d, v21.2d
1778 umull v22.2d, v12.2s, v31.2s
1779 add v2.2d, v2.2d, v22.2d
1780 umull v23.2d, v13.2s, v31.2s
1781 add v3.2d, v3.2d, v23.2d
1782 umull v24.2d, v14.2s, v31.2s
1783 add v4.2d, v4.2d, v24.2d
1784 umull v25.2d, v15.2s, v31.2s
1785 add v5.2d, v5.2d, v25.2d
1786 umull v26.2d, v16.2s, v31.2s
1787 add v6.2d, v6.2d, v26.2d
1788 umull v27.2d, v17.2s, v31.2s
1789 add v7.2d, v7.2d, v27.2d
1790 umull v28.2d, v18.2s, v31.2s
1791 add v8.2d, v8.2d, v28.2d
1792
1793 dup v29.2d, x14
1794 dup v30.2d, x13
1795
1796 usra v5.2d, v4.2d, #29
1797 and v4.16b, v4.16b, v29.16b
1798
1799 usra v1.2d, v0.2d, #29
1800 and v0.16b, v0.16b, v29.16b
1801
1802 usra v6.2d, v5.2d, #29
1803 and v5.16b, v5.16b, v29.16b
1804
1805 usra v2.2d, v1.2d, #29
1806 and v1.16b, v1.16b, v29.16b
1807
1808 usra v7.2d, v6.2d, #29
1809 and v6.16b, v6.16b, v29.16b
1810
1811 usra v3.2d, v2.2d, #29
1812 and v2.16b, v2.16b, v29.16b
1813
1814 usra v8.2d, v7.2d, #29
1815 and v7.16b, v7.16b, v29.16b
1816
1817 usra v4.2d, v3.2d, #29
1818 and v3.16b, v3.16b, v29.16b
1819
1820 bic v10.16b, v8.16b, v30.16b
1821 usra v0.2d, v10.2d, #23
1822 usra v0.2d, v10.2d, #22
1823 usra v0.2d, v10.2d, #19
1824 and v8.16b, v8.16b, v30.16b
1825
1826 usra v5.2d, v4.2d, #29
1827 and v4.16b, v4.16b, v29.16b
1828
1829 usra v1.2d, v0.2d, #29
1830 and v0.16b, v0.16b, v29.16b
1831
1832 xtn v20.2s, v0.2d
1833 xtn v21.2s, v1.2d
1834 xtn v22.2s, v2.2d
1835 xtn v23.2s, v3.2d
1836 xtn v24.2s, v4.2d
1837 xtn v25.2s, v5.2d
1838 xtn v26.2s, v6.2d
1839 xtn v27.2s, v7.2d
1840 xtn v28.2s, v8.2d
1841
1842 // Z2 ← T3 · T4
1843 ldr q10, [x20, #1008]
1844 ldr q11, [x20, #1024]
1845 ldr q12, [x20, #1040]
1846 ldr q13, [x20, #1056]
1847 ldr q14, [x20, #1072]
1848 ldr q15, [x20, #1088]
1849 ldr q16, [x20, #1104]
1850 ldr q17, [x20, #1120]
1851 ldr q18, [x20, #1136]
1852
1853 dup v29.2d, x14
1854 dup v30.2s, w10
1855
1856 umull v0.2d, v10.2s, v20.2s
1857 umull v1.2d, v10.2s, v21.2s
1858 umull v2.2d, v10.2s, v22.2s
1859 umull v3.2d, v10.2s, v23.2s
1860 umull v4.2d, v10.2s, v24.2s
1861 umull v5.2d, v10.2s, v25.2s
1862 umull v6.2d, v10.2s, v26.2s
1863 umull v7.2d, v10.2s, v27.2s
1864 umull v8.2d, v10.2s, v28.2s
1865
1866 umlal v1.2d, v11.2s, v20.2s
1867 umlal v2.2d, v11.2s, v21.2s
1868 umlal v3.2d, v11.2s, v22.2s
1869 umlal v4.2d, v11.2s, v23.2s
1870 umlal v5.2d, v11.2s, v24.2s
1871 umlal v6.2d, v11.2s, v25.2s
1872 umlal v7.2d, v11.2s, v26.2s
1873 umlal v8.2d, v11.2s, v27.2s
1874 umull v9.2d, v11.2s, v28.2s
1875
1876 umlal v2.2d, v12.2s, v20.2s
1877 umlal v3.2d, v12.2s, v21.2s
1878 umlal v4.2d, v12.2s, v22.2s
1879 umlal v5.2d, v12.2s, v23.2s
1880 umlal v6.2d, v12.2s, v24.2s
1881 umlal v7.2d, v12.2s, v25.2s
1882 umlal v8.2d, v12.2s, v26.2s
1883 umlal v9.2d, v12.2s, v27.2s
1884 umull v10.2d, v12.2s, v28.2s
1885
1886 umlal v3.2d, v13.2s, v20.2s
1887 umlal v4.2d, v13.2s, v21.2s
1888 umlal v5.2d, v13.2s, v22.2s
1889 umlal v6.2d, v13.2s, v23.2s
1890 umlal v7.2d, v13.2s, v24.2s
1891 umlal v8.2d, v13.2s, v25.2s
1892 umlal v9.2d, v13.2s, v26.2s
1893 umlal v10.2d, v13.2s, v27.2s
1894 umull v11.2d, v13.2s, v28.2s
1895
1896 umlal v4.2d, v14.2s, v20.2s
1897 umlal v5.2d, v14.2s, v21.2s
1898 umlal v6.2d, v14.2s, v22.2s
1899 umlal v7.2d, v14.2s, v23.2s
1900 umlal v8.2d, v14.2s, v24.2s
1901 umlal v9.2d, v14.2s, v25.2s
1902 umlal v10.2d, v14.2s, v26.2s
1903 umlal v11.2d, v14.2s, v27.2s
1904 umull v12.2d, v14.2s, v28.2s
1905
1906 umlal v5.2d, v15.2s, v20.2s
1907 umlal v6.2d, v15.2s, v21.2s
1908 umlal v7.2d, v15.2s, v22.2s
1909 umlal v8.2d, v15.2s, v23.2s
1910 umlal v9.2d, v15.2s, v24.2s
1911 umlal v10.2d, v15.2s, v25.2s
1912 umlal v11.2d, v15.2s, v26.2s
1913 umlal v12.2d, v15.2s, v27.2s
1914 umull v13.2d, v15.2s, v28.2s
1915
1916 umlal v6.2d, v16.2s, v20.2s
1917 umlal v7.2d, v16.2s, v21.2s
1918 umlal v8.2d, v16.2s, v22.2s
1919 umlal v9.2d, v16.2s, v23.2s
1920 umlal v10.2d, v16.2s, v24.2s
1921 umlal v11.2d, v16.2s, v25.2s
1922 umlal v12.2d, v16.2s, v26.2s
1923 umlal v13.2d, v16.2s, v27.2s
1924 umull v14.2d, v16.2s, v28.2s
1925
1926 umlal v7.2d, v17.2s, v20.2s
1927 umlal v8.2d, v17.2s, v21.2s
1928 umlal v9.2d, v17.2s, v22.2s
1929 umlal v10.2d, v17.2s, v23.2s
1930 umlal v11.2d, v17.2s, v24.2s
1931 umlal v12.2d, v17.2s, v25.2s
1932 umlal v13.2d, v17.2s, v26.2s
1933 umlal v14.2d, v17.2s, v27.2s
1934 umull v15.2d, v17.2s, v28.2s
1935
1936 umlal v8.2d, v18.2s, v20.2s
1937 umlal v9.2d, v18.2s, v21.2s
1938 umlal v10.2d, v18.2s, v22.2s
1939 umlal v11.2d, v18.2s, v23.2s
1940 umlal v12.2d, v18.2s, v24.2s
1941 umlal v13.2d, v18.2s, v25.2s
1942 umlal v14.2d, v18.2s, v26.2s
1943 umlal v15.2d, v18.2s, v27.2s
1944 umull v16.2d, v18.2s, v28.2s
1945
1946 usra v10.2d, v9.2d, #29
1947 and v9.16b, v9.16b, v29.16b
1948 xtn v9.2s, v9.2d
1949 umull v9.2d, v9.2s, v30.2s
1950 add v0.2d, v0.2d, v9.2d
1951
1952 usra v11.2d, v10.2d, #29
1953 and v10.16b, v10.16b, v29.16b
1954 xtn v10.2s, v10.2d
1955 umull v10.2d, v10.2s, v30.2s
1956 add v1.2d, v1.2d, v10.2d
1957
1958 usra v12.2d, v11.2d, #29
1959 and v11.16b, v11.16b, v29.16b
1960 xtn v11.2s, v11.2d
1961 umull v11.2d, v11.2s, v30.2s
1962 add v2.2d, v2.2d, v11.2d
1963
1964 usra v13.2d, v12.2d, #29
1965 and v12.16b, v12.16b, v29.16b
1966 xtn v12.2s, v12.2d
1967 umull v12.2d, v12.2s, v30.2s
1968 add v3.2d, v3.2d, v12.2d
1969
1970 usra v14.2d, v13.2d, #29
1971 and v13.16b, v13.16b, v29.16b
1972 xtn v13.2s, v13.2d
1973 umull v13.2d, v13.2s, v30.2s
1974 add v4.2d, v4.2d, v13.2d
1975
1976 usra v15.2d, v14.2d, #29
1977 and v14.16b, v14.16b, v29.16b
1978 xtn v14.2s, v14.2d
1979 umull v14.2d, v14.2s, v30.2s
1980 add v5.2d, v5.2d, v14.2d
1981
1982 usra v16.2d, v15.2d, #29
1983 and v15.16b, v15.16b, v29.16b
1984 xtn v15.2s, v15.2d
1985 umull v15.2d, v15.2s, v30.2s
1986 add v6.2d, v6.2d, v15.2d
1987
1988 ushr v9.2d, v16.2d, #29
1989 and v16.16b, v16.16b, v29.16b
1990 xtn v16.2s, v16.2d
1991 umull v16.2d, v16.2s, v30.2s
1992 add v7.2d, v7.2d, v16.2d
1993
1994 xtn v9.2s, v9.2d
1995 umull v9.2d, v9.2s, v30.2s
1996 add v8.2d, v8.2d, v9.2d
1997
1998 dup v30.2d, x13
1999
2000 usra v5.2d, v4.2d, #29
2001 and v4.16b, v4.16b, v29.16b
2002
2003 usra v1.2d, v0.2d, #29
2004 and v0.16b, v0.16b, v29.16b
2005
2006 usra v6.2d, v5.2d, #29
2007 and v5.16b, v5.16b, v29.16b
2008
2009 usra v2.2d, v1.2d, #29
2010 and v1.16b, v1.16b, v29.16b
2011
2012 usra v7.2d, v6.2d, #29
2013 and v6.16b, v6.16b, v29.16b
2014
2015 usra v3.2d, v2.2d, #29
2016 and v2.16b, v2.16b, v29.16b
2017
2018 usra v8.2d, v7.2d, #29
2019 and v7.16b, v7.16b, v29.16b
2020
2021 usra v4.2d, v3.2d, #29
2022 and v3.16b, v3.16b, v29.16b
2023
2024 bic v10.16b, v8.16b, v30.16b
2025 usra v0.2d, v10.2d, #23
2026 usra v0.2d, v10.2d, #22
2027 usra v0.2d, v10.2d, #19
2028 and v8.16b, v8.16b, v30.16b
2029
2030 usra v5.2d, v4.2d, #29
2031 and v4.16b, v4.16b, v29.16b
2032
2033 usra v1.2d, v0.2d, #29
2034 and v0.16b, v0.16b, v29.16b
2035
2036 xtn v10.2s, v0.2d
2037 xtn v11.2s, v1.2d
2038 xtn v12.2s, v2.2d
2039 xtn v13.2s, v3.2d
2040 xtn v14.2s, v4.2d
2041 xtn v15.2s, v5.2d
2042 xtn v16.2s, v6.2d
2043 xtn v17.2s, v7.2d
2044 xtn v18.2s, v8.2d
2045
2046 str q10, [x20, #288]
2047 str q11, [x20, #304]
2048 str q12, [x20, #320]
2049 str q13, [x20, #336]
2050 str q14, [x20, #352]
2051 str q15, [x20, #368]
2052 str q16, [x20, #384]
2053 str q17, [x20, #400]
2054 str q18, [x20, #416]
2055
2056 // X2 ← T1 · T2
2057 ldr q10, [x20, #720]
2058 ldr q11, [x20, #736]
2059 ldr q12, [x20, #752]
2060 ldr q13, [x20, #768]
2061 ldr q14, [x20, #784]
2062 ldr q15, [x20, #800]
2063 ldr q16, [x20, #816]
2064 ldr q17, [x20, #832]
2065 ldr q18, [x20, #848]
2066
2067 ldr q20, [x20, #864]
2068 ldr q21, [x20, #880]
2069 ldr q22, [x20, #896]
2070 ldr q23, [x20, #912]
2071 ldr q24, [x20, #928]
2072 ldr q25, [x20, #944]
2073 ldr q26, [x20, #960]
2074 ldr q27, [x20, #976]
2075 ldr q28, [x20, #992]
2076
2077 dup v29.2d, x14
2078 dup v30.2s, w10
2079
2080 umull v0.2d, v10.2s, v20.2s
2081 umull v1.2d, v10.2s, v21.2s
2082 umull v2.2d, v10.2s, v22.2s
2083 umull v3.2d, v10.2s, v23.2s
2084 umull v4.2d, v10.2s, v24.2s
2085 umull v5.2d, v10.2s, v25.2s
2086 umull v6.2d, v10.2s, v26.2s
2087 umull v7.2d, v10.2s, v27.2s
2088 umull v8.2d, v10.2s, v28.2s
2089
2090 umlal v1.2d, v11.2s, v20.2s
2091 umlal v2.2d, v11.2s, v21.2s
2092 umlal v3.2d, v11.2s, v22.2s
2093 umlal v4.2d, v11.2s, v23.2s
2094 umlal v5.2d, v11.2s, v24.2s
2095 umlal v6.2d, v11.2s, v25.2s
2096 umlal v7.2d, v11.2s, v26.2s
2097 umlal v8.2d, v11.2s, v27.2s
2098 umull v9.2d, v11.2s, v28.2s
2099
2100 umlal v2.2d, v12.2s, v20.2s
2101 umlal v3.2d, v12.2s, v21.2s
2102 umlal v4.2d, v12.2s, v22.2s
2103 umlal v5.2d, v12.2s, v23.2s
2104 umlal v6.2d, v12.2s, v24.2s
2105 umlal v7.2d, v12.2s, v25.2s
2106 umlal v8.2d, v12.2s, v26.2s
2107 umlal v9.2d, v12.2s, v27.2s
2108 umull v10.2d, v12.2s, v28.2s
2109
2110 umlal v3.2d, v13.2s, v20.2s
2111 umlal v4.2d, v13.2s, v21.2s
2112 umlal v5.2d, v13.2s, v22.2s
2113 umlal v6.2d, v13.2s, v23.2s
2114 umlal v7.2d, v13.2s, v24.2s
2115 umlal v8.2d, v13.2s, v25.2s
2116 umlal v9.2d, v13.2s, v26.2s
2117 umlal v10.2d, v13.2s, v27.2s
2118 umull v11.2d, v13.2s, v28.2s
2119
2120 umlal v4.2d, v14.2s, v20.2s
2121 umlal v5.2d, v14.2s, v21.2s
2122 umlal v6.2d, v14.2s, v22.2s
2123 umlal v7.2d, v14.2s, v23.2s
2124 umlal v8.2d, v14.2s, v24.2s
2125 umlal v9.2d, v14.2s, v25.2s
2126 umlal v10.2d, v14.2s, v26.2s
2127 umlal v11.2d, v14.2s, v27.2s
2128 umull v12.2d, v14.2s, v28.2s
2129
2130 umlal v5.2d, v15.2s, v20.2s
2131 umlal v6.2d, v15.2s, v21.2s
2132 umlal v7.2d, v15.2s, v22.2s
2133 umlal v8.2d, v15.2s, v23.2s
2134 umlal v9.2d, v15.2s, v24.2s
2135 umlal v10.2d, v15.2s, v25.2s
2136 umlal v11.2d, v15.2s, v26.2s
2137 umlal v12.2d, v15.2s, v27.2s
2138 umull v13.2d, v15.2s, v28.2s
2139
2140 umlal v6.2d, v16.2s, v20.2s
2141 umlal v7.2d, v16.2s, v21.2s
2142 umlal v8.2d, v16.2s, v22.2s
2143 umlal v9.2d, v16.2s, v23.2s
2144 umlal v10.2d, v16.2s, v24.2s
2145 umlal v11.2d, v16.2s, v25.2s
2146 umlal v12.2d, v16.2s, v26.2s
2147 umlal v13.2d, v16.2s, v27.2s
2148 umull v14.2d, v16.2s, v28.2s
2149
2150 umlal v7.2d, v17.2s, v20.2s
2151 umlal v8.2d, v17.2s, v21.2s
2152 umlal v9.2d, v17.2s, v22.2s
2153 umlal v10.2d, v17.2s, v23.2s
2154 umlal v11.2d, v17.2s, v24.2s
2155 umlal v12.2d, v17.2s, v25.2s
2156 umlal v13.2d, v17.2s, v26.2s
2157 umlal v14.2d, v17.2s, v27.2s
2158 umull v15.2d, v17.2s, v28.2s
2159
2160 umlal v8.2d, v18.2s, v20.2s
2161 umlal v9.2d, v18.2s, v21.2s
2162 umlal v10.2d, v18.2s, v22.2s
2163 umlal v11.2d, v18.2s, v23.2s
2164 umlal v12.2d, v18.2s, v24.2s
2165 umlal v13.2d, v18.2s, v25.2s
2166 umlal v14.2d, v18.2s, v26.2s
2167 umlal v15.2d, v18.2s, v27.2s
2168 umull v16.2d, v18.2s, v28.2s
2169
2170 usra v10.2d, v9.2d, #29
2171 and v9.16b, v9.16b, v29.16b
2172 xtn v9.2s, v9.2d
2173 umull v9.2d, v9.2s, v30.2s
2174 add v0.2d, v0.2d, v9.2d
2175
2176 usra v11.2d, v10.2d, #29
2177 and v10.16b, v10.16b, v29.16b
2178 xtn v10.2s, v10.2d
2179 umull v10.2d, v10.2s, v30.2s
2180 add v1.2d, v1.2d, v10.2d
2181
2182 usra v12.2d, v11.2d, #29
2183 and v11.16b, v11.16b, v29.16b
2184 xtn v11.2s, v11.2d
2185 umull v11.2d, v11.2s, v30.2s
2186 add v2.2d, v2.2d, v11.2d
2187
2188 usra v13.2d, v12.2d, #29
2189 and v12.16b, v12.16b, v29.16b
2190 xtn v12.2s, v12.2d
2191 umull v12.2d, v12.2s, v30.2s
2192 add v3.2d, v3.2d, v12.2d
2193
2194 usra v14.2d, v13.2d, #29
2195 and v13.16b, v13.16b, v29.16b
2196 xtn v13.2s, v13.2d
2197 umull v13.2d, v13.2s, v30.2s
2198 add v4.2d, v4.2d, v13.2d
2199
2200 usra v15.2d, v14.2d, #29
2201 and v14.16b, v14.16b, v29.16b
2202 xtn v14.2s, v14.2d
2203 umull v14.2d, v14.2s, v30.2s
2204 add v5.2d, v5.2d, v14.2d
2205
2206 usra v16.2d, v15.2d, #29
2207 and v15.16b, v15.16b, v29.16b
2208 xtn v15.2s, v15.2d
2209 umull v15.2d, v15.2s, v30.2s
2210 add v6.2d, v6.2d, v15.2d
2211
2212 ushr v9.2d, v16.2d, #29
2213 and v16.16b, v16.16b, v29.16b
2214 xtn v16.2s, v16.2d
2215 umull v16.2d, v16.2s, v30.2s
2216 add v7.2d, v7.2d, v16.2d
2217
2218 xtn v9.2s, v9.2d
2219 umull v9.2d, v9.2s, v30.2s
2220 add v8.2d, v8.2d, v9.2d
2221
2222 dup v30.2d, x13
2223
2224 usra v5.2d, v4.2d, #29
2225 and v4.16b, v4.16b, v29.16b
2226
2227 usra v1.2d, v0.2d, #29
2228 and v0.16b, v0.16b, v29.16b
2229
2230 usra v6.2d, v5.2d, #29
2231 and v5.16b, v5.16b, v29.16b
2232
2233 usra v2.2d, v1.2d, #29
2234 and v1.16b, v1.16b, v29.16b
2235
2236 usra v7.2d, v6.2d, #29
2237 and v6.16b, v6.16b, v29.16b
2238
2239 usra v3.2d, v2.2d, #29
2240 and v2.16b, v2.16b, v29.16b
2241
2242 usra v8.2d, v7.2d, #29
2243 and v7.16b, v7.16b, v29.16b
2244
2245 usra v4.2d, v3.2d, #29
2246 and v3.16b, v3.16b, v29.16b
2247
2248 bic v10.16b, v8.16b, v30.16b
2249 usra v0.2d, v10.2d, #23
2250 usra v0.2d, v10.2d, #22
2251 usra v0.2d, v10.2d, #19
2252 and v8.16b, v8.16b, v30.16b
2253
2254 usra v5.2d, v4.2d, #29
2255 and v4.16b, v4.16b, v29.16b
2256
2257 usra v1.2d, v0.2d, #29
2258 and v0.16b, v0.16b, v29.16b
2259
2260 xtn v10.2s, v0.2d
2261 xtn v11.2s, v1.2d
2262 xtn v12.2s, v2.2d
2263 xtn v13.2s, v3.2d
2264 xtn v14.2s, v4.2d
2265 xtn v15.2s, v5.2d
2266 xtn v16.2s, v6.2d
2267 xtn v17.2s, v7.2d
2268 xtn v18.2s, v8.2d
2269
2270 str q10, [x20, #144]
2271 str q11, [x20, #160]
2272 str q12, [x20, #176]
2273 str q13, [x20, #192]
2274 str q14, [x20, #208]
2275 str q15, [x20, #224]
2276 str q16, [x20, #240]
2277 str q17, [x20, #256]
2278 str q18, [x20, #272]
2279
2280 // Z3 ← Z3 · X1
2281 ldr q10, [x20, #576]
2282 ldr q11, [x20, #592]
2283 ldr q12, [x20, #608]
2284 ldr q13, [x20, #624]
2285 ldr q14, [x20, #640]
2286 ldr q15, [x20, #656]
2287 ldr q16, [x20, #672]
2288 ldr q17, [x20, #688]
2289 ldr q18, [x20, #704]
2290
2291 ldr q20, [x20, #0]
2292 ldr q21, [x20, #16]
2293 ldr q22, [x20, #32]
2294 ldr q23, [x20, #48]
2295 ldr q24, [x20, #64]
2296 ldr q25, [x20, #80]
2297 ldr q26, [x20, #96]
2298 ldr q27, [x20, #112]
2299 ldr q28, [x20, #128]
2300
2301 dup v29.2d, x14
2302 dup v30.2s, w10
2303
2304 umull v0.2d, v10.2s, v20.2s
2305 umull v1.2d, v10.2s, v21.2s
2306 umull v2.2d, v10.2s, v22.2s
2307 umull v3.2d, v10.2s, v23.2s
2308 umull v4.2d, v10.2s, v24.2s
2309 umull v5.2d, v10.2s, v25.2s
2310 umull v6.2d, v10.2s, v26.2s
2311 umull v7.2d, v10.2s, v27.2s
2312 umull v8.2d, v10.2s, v28.2s
2313
2314 umlal v1.2d, v11.2s, v20.2s
2315 umlal v2.2d, v11.2s, v21.2s
2316 umlal v3.2d, v11.2s, v22.2s
2317 umlal v4.2d, v11.2s, v23.2s
2318 umlal v5.2d, v11.2s, v24.2s
2319 umlal v6.2d, v11.2s, v25.2s
2320 umlal v7.2d, v11.2s, v26.2s
2321 umlal v8.2d, v11.2s, v27.2s
2322 umull v9.2d, v11.2s, v28.2s
2323
2324 umlal v2.2d, v12.2s, v20.2s
2325 umlal v3.2d, v12.2s, v21.2s
2326 umlal v4.2d, v12.2s, v22.2s
2327 umlal v5.2d, v12.2s, v23.2s
2328 umlal v6.2d, v12.2s, v24.2s
2329 umlal v7.2d, v12.2s, v25.2s
2330 umlal v8.2d, v12.2s, v26.2s
2331 umlal v9.2d, v12.2s, v27.2s
2332 umull v10.2d, v12.2s, v28.2s
2333
2334 umlal v3.2d, v13.2s, v20.2s
2335 umlal v4.2d, v13.2s, v21.2s
2336 umlal v5.2d, v13.2s, v22.2s
2337 umlal v6.2d, v13.2s, v23.2s
2338 umlal v7.2d, v13.2s, v24.2s
2339 umlal v8.2d, v13.2s, v25.2s
2340 umlal v9.2d, v13.2s, v26.2s
2341 umlal v10.2d, v13.2s, v27.2s
2342 umull v11.2d, v13.2s, v28.2s
2343
2344 umlal v4.2d, v14.2s, v20.2s
2345 umlal v5.2d, v14.2s, v21.2s
2346 umlal v6.2d, v14.2s, v22.2s
2347 umlal v7.2d, v14.2s, v23.2s
2348 umlal v8.2d, v14.2s, v24.2s
2349 umlal v9.2d, v14.2s, v25.2s
2350 umlal v10.2d, v14.2s, v26.2s
2351 umlal v11.2d, v14.2s, v27.2s
2352 umull v12.2d, v14.2s, v28.2s
2353
2354 umlal v5.2d, v15.2s, v20.2s
2355 umlal v6.2d, v15.2s, v21.2s
2356 umlal v7.2d, v15.2s, v22.2s
2357 umlal v8.2d, v15.2s, v23.2s
2358 umlal v9.2d, v15.2s, v24.2s
2359 umlal v10.2d, v15.2s, v25.2s
2360 umlal v11.2d, v15.2s, v26.2s
2361 umlal v12.2d, v15.2s, v27.2s
2362 umull v13.2d, v15.2s, v28.2s
2363
2364 umlal v6.2d, v16.2s, v20.2s
2365 umlal v7.2d, v16.2s, v21.2s
2366 umlal v8.2d, v16.2s, v22.2s
2367 umlal v9.2d, v16.2s, v23.2s
2368 umlal v10.2d, v16.2s, v24.2s
2369 umlal v11.2d, v16.2s, v25.2s
2370 umlal v12.2d, v16.2s, v26.2s
2371 umlal v13.2d, v16.2s, v27.2s
2372 umull v14.2d, v16.2s, v28.2s
2373
2374 umlal v7.2d, v17.2s, v20.2s
2375 umlal v8.2d, v17.2s, v21.2s
2376 umlal v9.2d, v17.2s, v22.2s
2377 umlal v10.2d, v17.2s, v23.2s
2378 umlal v11.2d, v17.2s, v24.2s
2379 umlal v12.2d, v17.2s, v25.2s
2380 umlal v13.2d, v17.2s, v26.2s
2381 umlal v14.2d, v17.2s, v27.2s
2382 umull v15.2d, v17.2s, v28.2s
2383
2384 umlal v8.2d, v18.2s, v20.2s
2385 umlal v9.2d, v18.2s, v21.2s
2386 umlal v10.2d, v18.2s, v22.2s
2387 umlal v11.2d, v18.2s, v23.2s
2388 umlal v12.2d, v18.2s, v24.2s
2389 umlal v13.2d, v18.2s, v25.2s
2390 umlal v14.2d, v18.2s, v26.2s
2391 umlal v15.2d, v18.2s, v27.2s
2392 umull v16.2d, v18.2s, v28.2s
2393
2394 usra v10.2d, v9.2d, #29
2395 and v9.16b, v9.16b, v29.16b
2396 xtn v9.2s, v9.2d
2397 umull v9.2d, v9.2s, v30.2s
2398 add v0.2d, v0.2d, v9.2d
2399
2400 usra v11.2d, v10.2d, #29
2401 and v10.16b, v10.16b, v29.16b
2402 xtn v10.2s, v10.2d
2403 umull v10.2d, v10.2s, v30.2s
2404 add v1.2d, v1.2d, v10.2d
2405
2406 usra v12.2d, v11.2d, #29
2407 and v11.16b, v11.16b, v29.16b
2408 xtn v11.2s, v11.2d
2409 umull v11.2d, v11.2s, v30.2s
2410 add v2.2d, v2.2d, v11.2d
2411
2412 usra v13.2d, v12.2d, #29
2413 and v12.16b, v12.16b, v29.16b
2414 xtn v12.2s, v12.2d
2415 umull v12.2d, v12.2s, v30.2s
2416 add v3.2d, v3.2d, v12.2d
2417
2418 usra v14.2d, v13.2d, #29
2419 and v13.16b, v13.16b, v29.16b
2420 xtn v13.2s, v13.2d
2421 umull v13.2d, v13.2s, v30.2s
2422 add v4.2d, v4.2d, v13.2d
2423
2424 usra v15.2d, v14.2d, #29
2425 and v14.16b, v14.16b, v29.16b
2426 xtn v14.2s, v14.2d
2427 umull v14.2d, v14.2s, v30.2s
2428 add v5.2d, v5.2d, v14.2d
2429
2430 usra v16.2d, v15.2d, #29
2431 and v15.16b, v15.16b, v29.16b
2432 xtn v15.2s, v15.2d
2433 umull v15.2d, v15.2s, v30.2s
2434 add v6.2d, v6.2d, v15.2d
2435
2436 ushr v9.2d, v16.2d, #29
2437 and v16.16b, v16.16b, v29.16b
2438 xtn v16.2s, v16.2d
2439 umull v16.2d, v16.2s, v30.2s
2440 add v7.2d, v7.2d, v16.2d
2441
2442 xtn v9.2s, v9.2d
2443 umull v9.2d, v9.2s, v30.2s
2444 add v8.2d, v8.2d, v9.2d
2445
2446 dup v30.2d, x13
2447
2448 usra v5.2d, v4.2d, #29
2449 and v4.16b, v4.16b, v29.16b
2450
2451 usra v1.2d, v0.2d, #29
2452 and v0.16b, v0.16b, v29.16b
2453
2454 usra v6.2d, v5.2d, #29
2455 and v5.16b, v5.16b, v29.16b
2456
2457 usra v2.2d, v1.2d, #29
2458 and v1.16b, v1.16b, v29.16b
2459
2460 usra v7.2d, v6.2d, #29
2461 and v6.16b, v6.16b, v29.16b
2462
2463 usra v3.2d, v2.2d, #29
2464 and v2.16b, v2.16b, v29.16b
2465
2466 usra v8.2d, v7.2d, #29
2467 and v7.16b, v7.16b, v29.16b
2468
2469 usra v4.2d, v3.2d, #29
2470 and v3.16b, v3.16b, v29.16b
2471
2472 bic v10.16b, v8.16b, v30.16b
2473 usra v0.2d, v10.2d, #23
2474 usra v0.2d, v10.2d, #22
2475 usra v0.2d, v10.2d, #19
2476 and v8.16b, v8.16b, v30.16b
2477
2478 usra v5.2d, v4.2d, #29
2479 and v4.16b, v4.16b, v29.16b
2480
2481 usra v1.2d, v0.2d, #29
2482 and v0.16b, v0.16b, v29.16b
2483
2484 xtn v10.2s, v0.2d
2485 xtn v11.2s, v1.2d
2486 xtn v12.2s, v2.2d
2487 xtn v13.2s, v3.2d
2488 xtn v14.2s, v4.2d
2489 xtn v15.2s, v5.2d
2490 xtn v16.2s, v6.2d
2491 xtn v17.2s, v7.2d
2492 xtn v18.2s, v8.2d
2493
2494 str q10, [x20, #576]
2495 str q11, [x20, #592]
2496 str q12, [x20, #608]
2497 str q13, [x20, #624]
2498 str q14, [x20, #640]
2499 str q15, [x20, #656]
2500 str q16, [x20, #672]
2501 str q17, [x20, #688]
2502 str q18, [x20, #704]
2503
2504 sub x6, x6, #1
2505 cmp x6, #0
2506 bge .L2
2507
2508 mov x6, #63
2509 sub x5, x5, #16
2510 cmp x5, #0
2511 bge .L1
2512
2513 // reduce X2
2514 ldr q10, [x20, #144]
2515 ldr q11, [x20, #160]
2516 ldr q12, [x20, #176]
2517 ldr q13, [x20, #192]
2518 ldr q14, [x20, #208]
2519 ldr q15, [x20, #224]
2520 ldr q16, [x20, #240]
2521 ldr q17, [x20, #256]
2522 ldr q18, [x20, #272]
2523
2524 uxtl v0.2d, v10.2s
2525 uxtl v1.2d, v11.2s
2526 uxtl v2.2d, v12.2s
2527 uxtl v3.2d, v13.2s
2528 uxtl v4.2d, v14.2s
2529 uxtl v5.2d, v15.2s
2530 uxtl v6.2d, v16.2s
2531 uxtl v7.2d, v17.2s
2532 uxtl v8.2d, v18.2s
2533
2534 usra v1.2d, v0.2d, #29
2535 and v0.16b, v0.16b, v29.16b
2536
2537 usra v2.2d, v1.2d, #29
2538 and v1.16b, v1.16b, v29.16b
2539
2540 usra v3.2d, v2.2d, #29
2541 and v2.16b, v2.16b, v29.16b
2542
2543 usra v4.2d, v3.2d, #29
2544 and v3.16b, v3.16b, v29.16b
2545
2546 usra v5.2d, v4.2d, #29
2547 and v4.16b, v4.16b, v29.16b
2548
2549 usra v6.2d, v5.2d, #29
2550 and v5.16b, v5.16b, v29.16b
2551
2552 usra v7.2d, v6.2d, #29
2553 and v6.16b, v6.16b, v29.16b
2554
2555 usra v8.2d, v7.2d, #29
2556 and v7.16b, v7.16b, v29.16b
2557
2558 bic v10.16b, v8.16b, v30.16b
2559 usra v0.2d, v10.2d, #23
2560 usra v0.2d, v10.2d, #22
2561 usra v0.2d, v10.2d, #19
2562 and v8.16b, v8.16b, v30.16b
2563
2564 usra v1.2d, v0.2d, #29
2565 and v0.16b, v0.16b, v29.16b
2566
2567 // store X2
2568 str q0, [x0, #0]
2569 str q1, [x0, #16]
2570 str q2, [x0, #32]
2571 str q3, [x0, #48]
2572 str q4, [x0, #64]
2573 str q5, [x0, #80]
2574 str q6, [x0, #96]
2575 str q7, [x0, #112]
2576 str q8, [x0, #128]
2577
2578 // store Z2
2579 str q20, [x0, #144]
2580 str q21, [x0, #160]
2581 str q22, [x0, #176]
2582 str q23, [x0, #192]
2583 str q24, [x0, #208]
2584 str q25, [x0, #224]
2585 str q26, [x0, #240]
2586 str q27, [x0, #256]
2587 str q28, [x0, #272]
2588
2589 ldr q10, [x20, #288]
2590 ldr q11, [x20, #304]
2591 ldr q12, [x20, #320]
2592 ldr q13, [x20, #336]
2593 ldr q14, [x20, #352]
2594 ldr q15, [x20, #368]
2595 ldr q16, [x20, #384]
2596 ldr q17, [x20, #400]
2597 ldr q18, [x20, #416]
2598
2599 uxtl v0.2d, v10.2s
2600 uxtl v1.2d, v11.2s
2601 uxtl v2.2d, v12.2s
2602 uxtl v3.2d, v13.2s
2603 uxtl v4.2d, v14.2s
2604 uxtl v5.2d, v15.2s
2605 uxtl v6.2d, v16.2s
2606 uxtl v7.2d, v17.2s
2607 uxtl v8.2d, v18.2s
2608
2609 usra v1.2d, v0.2d, #29
2610 and v0.16b, v0.16b, v29.16b
2611
2612 usra v2.2d, v1.2d, #29
2613 and v1.16b, v1.16b, v29.16b
2614
2615 usra v3.2d, v2.2d, #29
2616 and v2.16b, v2.16b, v29.16b
2617
2618 usra v4.2d, v3.2d, #29
2619 and v3.16b, v3.16b, v29.16b
2620
2621 usra v5.2d, v4.2d, #29
2622 and v4.16b, v4.16b, v29.16b
2623
2624 usra v6.2d, v5.2d, #29
2625 and v5.16b, v5.16b, v29.16b
2626
2627 usra v7.2d, v6.2d, #29
2628 and v6.16b, v6.16b, v29.16b
2629
2630 usra v8.2d, v7.2d, #29
2631 and v7.16b, v7.16b, v29.16b
2632
2633 bic v10.16b, v8.16b, v30.16b
2634 usra v0.2d, v10.2d, #23
2635 usra v0.2d, v10.2d, #22
2636 usra v0.2d, v10.2d, #19
2637 and v8.16b, v8.16b, v30.16b
2638
2639 usra v1.2d, v0.2d, #29
2640 and v0.16b, v0.16b, v29.16b
2641
2642 // store Z2
2643 str q0, [x0, #144]
2644 str q1, [x0, #160]
2645 str q2, [x0, #176]
2646 str q3, [x0, #192]
2647 str q4, [x0, #208]
2648 str q5, [x0, #224]
2649 str q6, [x0, #240]
2650 str q7, [x0, #256]
2651 str q8, [x0, #272]
2652
2653 ldp d14, d15, [sp, #144]
2654 ldp d12, d13, [sp, #128]
2655 ldp d10, d11, [sp, #112]
2656 ldp d8, d9, [sp, #96]
2657 ldp x29, x30, [sp, #80]
2658 ldp x27, x28, [sp, #64]
2659 ldp x25, x26, [sp, #48]
2660 ldp x23, x24, [sp, #32]
2661 ldp x21, x22, [sp, #16]
2662 ldp x19, x20, [sp, #0]
2663 add sp, sp, #1488
2664
2665 ret
2666 .section .note.GNU-stack,"",@progbits