Debian Codegraph @

crypto_nPbatch/montgomery25519/arm64-neon-2x1-uma-10l/mladder.S

raw
1 #include "crypto_asm_hidden.h"
2
3 // 2-way Montgomery ladder, 10-limb implementation
4
5 .p2align 4
6 ASM_HIDDEN _CRYPTO_SHARED_NAMESPACE(mladder_2x1)
7 .globl _CRYPTO_SHARED_NAMESPACE(mladder_2x1)
8 ASM_HIDDEN CRYPTO_SHARED_NAMESPACE(mladder_2x1)
9 .globl CRYPTO_SHARED_NAMESPACE(mladder_2x1)
10
11 _CRYPTO_SHARED_NAMESPACE(mladder_2x1):
12 CRYPTO_SHARED_NAMESPACE(mladder_2x1):
13
14 sub sp, sp, #1632
15 stp x19, x20, [sp, #0]
16 stp x21, x22, [sp, #16]
17 stp x23, x24, [sp, #32]
18 stp x25, x26, [sp, #48]
19 stp x27, x28, [sp, #64]
20 stp x29, x30, [sp, #80]
21 stp d8, d9, [sp, #96]
22 stp d10, d11, [sp, #112]
23 stp d12, d13, [sp, #128]
24 stp d14, d15, [sp, #144]
25
26 mov x4, #1
27 mov w30, #19
28 mov w29, #0x03ffffff
29
30 movz x12, #0xdb42
31 movk x12, #0x0001, lsl 16
32 movk x12, #0xdb42, lsl 32
33 movk x12, #0x0001, lsl 48
34
35 movz x21, #0xffda
36 movk x21, #0x07ff, lsl 16
37 movk x21, #0xffda, lsl 32
38 movk x21, #0x07ff, lsl 48
39
40 movz x22, #0xfffe
41 movk x22, #0x03ff, lsl 16
42 movk x22, #0xfffe, lsl 32
43 movk x22, #0x03ff, lsl 48
44
45 movz x23, #0xfffe
46 movk x23, #0x07ff, lsl 16
47 movk x23, #0xfffe, lsl 32
48 movk x23, #0x07ff, lsl 48
49
50 // X1 ← XP,X3 ← XP
51
52 add x20, sp, #192
53
54 ldr q20, [x1, #0]
55 str q20, [x20, #0]
56 str q20, [x20, #480]
57
58 ldr q20, [x1, #16]
59 str q20, [x20, #16]
60 str q20, [x20, #496]
61
62 ldr q20, [x1, #32]
63 str q20, [x20, #32]
64 str q20, [x20, #512]
65
66 ldr q20, [x1, #48]
67 str q20, [x20, #48]
68 str q20, [x20, #528]
69
70 ldr q20, [x1, #64]
71 str q20, [x20, #64]
72 str q20, [x20, #544]
73
74 ldr q20, [x1, #80]
75 str q20, [x20, #80]
76 str q20, [x20, #560]
77
78 ldr q20, [x1, #96]
79 str q20, [x20, #96]
80 str q20, [x20, #576]
81
82 ldr q20, [x1, #112]
83 str q20, [x20, #112]
84 str q20, [x20, #592]
85
86 ldr q20, [x1, #128]
87 str q20, [x20, #128]
88 str q20, [x20, #608]
89
90 ldr q20, [x1, #144]
91 str q20, [x20, #144]
92 str q20, [x20, #624]
93
94 dup v20.2s, w4
95 dup v21.2s, wzr
96
97 // X2 ← 1
98 str q20, [x20, #160]
99 str q21, [x20, #176]
100 str q21, [x20, #192]
101 str q21, [x20, #208]
102 str q21, [x20, #224]
103 str q21, [x20, #240]
104 str q21, [x20, #256]
105 str q21, [x20, #272]
106 str q21, [x20, #288]
107 str q21, [x20, #304]
108
109 // Z2 ← 0
110 str q21, [x20, #320]
111 str q21, [x20, #336]
112 str q21, [x20, #352]
113 str q21, [x20, #368]
114 str q21, [x20, #384]
115 str q21, [x20, #400]
116 str q21, [x20, #416]
117 str q21, [x20, #432]
118 str q21, [x20, #448]
119 str q21, [x20, #464]
120
121 // Z3 ← 1
122 str q20, [x20, #640]
123 str q21, [x20, #656]
124 str q21, [x20, #672]
125 str q21, [x20, #688]
126 str q21, [x20, #704]
127 str q21, [x20, #720]
128 str q21, [x20, #736]
129 str q21, [x20, #752]
130 str q21, [x20, #768]
131 str q21, [x20, #784]
132
133 mov x5, #48
134 mov x6, #62
135 mov x15, x6
136 stp xzr, xzr, [sp, #160]
137 mov x3, x2
138
139 // Montgomery ladder loop
140 .L1:
141 add x3, x3, x5
142 ldr q30, [x3, #0]
143 str q30, [sp, #176]
144 mov x3, x2
145 .L2:
146
147 /*
148 * Montgomery ladder step
149 *
150 * T1 ← X2 + Z2
151 * T2 ← X2 - Z2
152 * T3 ← X3 + Z3
153 * T4 ← X3 - Z3
154 * Z3 ← T2 · T3
155 * X3 ← T1 · T4
156 *
157 * bit ← n[i]
158 * select ← bit ⊕ prevbit
159 * prevbit ← bit
160 * CSelect(T1,T3,select): if (select == 1) {T1 = T3}
161 * CSelect(T2,T4,select): if (select == 1) {T2 = T4}
162 *
163 * T2 ← T2^2
164 * T1 ← T1^2
165 * T3 ← X3 + Z3
166 * Z3 ← X3 - Z3
167 * Z3 ← Z3^2
168 * X3 ← T3^2
169 * T3 ← T1 - T2
170 * T4 ← ((A + 2)/4) · T3
171 * T4 ← T4 + T2
172 * X2 ← T1 · T2
173 * Z2 ← T3 · T4
174 * Z3 ← Z3 · X1
175 *
176 */
177
178 // X2
179 ldr q10, [x20, #160]
180 ldr q11, [x20, #176]
181 ldr q12, [x20, #192]
182 ldr q13, [x20, #208]
183 ldr q14, [x20, #224]
184 ldr q15, [x20, #240]
185 ldr q16, [x20, #256]
186 ldr q17, [x20, #272]
187 ldr q18, [x20, #288]
188 ldr q19, [x20, #304]
189
190 // Z2
191 ldr q0, [x20, #320]
192 ldr q1, [x20, #336]
193 ldr q2, [x20, #352]
194 ldr q3, [x20, #368]
195 ldr q4, [x20, #384]
196 ldr q5, [x20, #400]
197 ldr q6, [x20, #416]
198 ldr q7, [x20, #432]
199 ldr q8, [x20, #448]
200 ldr q9, [x20, #464]
201
202 // T1 ← X2 + Z2
203 add v20.2s, v0.2s, v10.2s
204 add v21.2s, v1.2s, v11.2s
205 add v22.2s, v2.2s, v12.2s
206 add v23.2s, v3.2s, v13.2s
207 add v24.2s, v4.2s, v14.2s
208 add v25.2s, v5.2s, v15.2s
209 add v26.2s, v6.2s, v16.2s
210 add v27.2s, v7.2s, v17.2s
211 add v28.2s, v8.2s, v18.2s
212 add v29.2s, v9.2s, v19.2s
213
214 str q20, [x20, #800]
215 str q21, [x20, #816]
216 str q22, [x20, #832]
217 str q23, [x20, #848]
218 str q24, [x20, #864]
219 str q25, [x20, #880]
220 str q26, [x20, #896]
221 str q27, [x20, #912]
222 str q28, [x20, #928]
223 str q29, [x20, #944]
224
225 // T2 ← X2 - Z2
226 mov v30.d[0], x21
227 add v10.2s, v30.2s, v10.2s
228 mov v30.d[0], x22
229 mov v31.d[0], x23
230 add v11.2s, v30.2s, v11.2s
231 add v12.2s, v31.2s, v12.2s
232 add v13.2s, v30.2s, v13.2s
233 add v14.2s, v31.2s, v14.2s
234 add v15.2s, v30.2s, v15.2s
235 add v16.2s, v31.2s, v16.2s
236 add v17.2s, v30.2s, v17.2s
237 add v18.2s, v31.2s, v18.2s
238 add v19.2s, v30.2s, v19.2s
239
240 sub v20.2s, v10.2s, v0.2s
241 sub v21.2s, v11.2s, v1.2s
242 sub v22.2s, v12.2s, v2.2s
243 sub v23.2s, v13.2s, v3.2s
244 sub v24.2s, v14.2s, v4.2s
245 sub v25.2s, v15.2s, v5.2s
246 sub v26.2s, v16.2s, v6.2s
247 sub v27.2s, v17.2s, v7.2s
248 sub v28.2s, v18.2s, v8.2s
249 sub v29.2s, v19.2s, v9.2s
250
251 str q20, [x20, #960]
252 str q21, [x20, #976]
253 str q22, [x20, #992]
254 str q23, [x20, #1008]
255 str q24, [x20, #1024]
256 str q25, [x20, #1040]
257 str q26, [x20, #1056]
258 str q27, [x20, #1072]
259 str q28, [x20, #1088]
260 str q29, [x20, #1104]
261
262 // X3
263 ldr q10, [x20, #480]
264 ldr q11, [x20, #496]
265 ldr q12, [x20, #512]
266 ldr q13, [x20, #528]
267 ldr q14, [x20, #544]
268 ldr q15, [x20, #560]
269 ldr q16, [x20, #576]
270 ldr q17, [x20, #592]
271 ldr q18, [x20, #608]
272 ldr q19, [x20, #624]
273
274 // Z3
275 ldr q0, [x20, #640]
276 ldr q1, [x20, #656]
277 ldr q2, [x20, #672]
278 ldr q3, [x20, #688]
279 ldr q4, [x20, #704]
280 ldr q5, [x20, #720]
281 ldr q6, [x20, #736]
282 ldr q7, [x20, #752]
283 ldr q8, [x20, #768]
284 ldr q9, [x20, #784]
285
286 // T3 ← X3 + Z3
287 add v20.2s, v0.2s, v10.2s
288 add v21.2s, v1.2s, v11.2s
289 add v22.2s, v2.2s, v12.2s
290 add v23.2s, v3.2s, v13.2s
291 add v24.2s, v4.2s, v14.2s
292 add v25.2s, v5.2s, v15.2s
293 add v26.2s, v6.2s, v16.2s
294 add v27.2s, v7.2s, v17.2s
295 add v28.2s, v8.2s, v18.2s
296 add v29.2s, v9.2s, v19.2s
297
298 str q20, [x20, #1120]
299 str q21, [x20, #1136]
300 str q22, [x20, #1152]
301 str q23, [x20, #1168]
302 str q24, [x20, #1184]
303 str q25, [x20, #1200]
304 str q26, [x20, #1216]
305 str q27, [x20, #1232]
306 str q28, [x20, #1248]
307 str q29, [x20, #1264]
308
309 // T4 ← X3 - Z3
310 mov v30.d[0], x21
311 add v10.2s, v30.2s, v10.2s
312 mov v30.d[0], x22
313 mov v31.d[0], x23
314 add v11.2s, v30.2s, v11.2s
315 add v12.2s, v31.2s, v12.2s
316 add v13.2s, v30.2s, v13.2s
317 add v14.2s, v31.2s, v14.2s
318 add v15.2s, v30.2s, v15.2s
319 add v16.2s, v31.2s, v16.2s
320 add v17.2s, v30.2s, v17.2s
321 add v18.2s, v31.2s, v18.2s
322 add v19.2s, v30.2s, v19.2s
323
324 sub v20.2s, v10.2s, v0.2s
325 sub v21.2s, v11.2s, v1.2s
326 sub v22.2s, v12.2s, v2.2s
327 sub v23.2s, v13.2s, v3.2s
328 sub v24.2s, v14.2s, v4.2s
329 sub v25.2s, v15.2s, v5.2s
330 sub v26.2s, v16.2s, v6.2s
331 sub v27.2s, v17.2s, v7.2s
332 sub v28.2s, v18.2s, v8.2s
333 sub v29.2s, v19.2s, v9.2s
334
335 str q20, [x20, #1280]
336 str q21, [x20, #1296]
337 str q22, [x20, #1312]
338 str q23, [x20, #1328]
339 str q24, [x20, #1344]
340 str q25, [x20, #1360]
341 str q26, [x20, #1376]
342 str q27, [x20, #1392]
343 str q28, [x20, #1408]
344 str q29, [x20, #1424]
345
346 // X3 ← T1 · T4
347 ldr q10, [x20, #800]
348 ldr q11, [x20, #816]
349 ldr q12, [x20, #832]
350 ldr q13, [x20, #848]
351 ldr q14, [x20, #864]
352 ldr q15, [x20, #880]
353 ldr q16, [x20, #896]
354 ldr q17, [x20, #912]
355 ldr q18, [x20, #928]
356 ldr q19, [x20, #944]
357
358 umull v0.2d, v10.2s, v20.2s
359
360 umull v1.2d, v10.2s, v21.2s
361 umlal v1.2d, v11.2s, v20.2s
362
363 umull v2.2d, v10.2s, v22.2s
364 umlal v2.2d, v12.2s, v20.2s
365
366 umull v3.2d, v10.2s, v23.2s
367 umlal v3.2d, v11.2s, v22.2s
368 umlal v3.2d, v12.2s, v21.2s
369 umlal v3.2d, v13.2s, v20.2s
370
371 umull v4.2d, v10.2s, v24.2s
372 umlal v4.2d, v12.2s, v22.2s
373 umlal v4.2d, v14.2s, v20.2s
374
375 umull v5.2d, v10.2s, v25.2s
376 umlal v5.2d, v11.2s, v24.2s
377 umlal v5.2d, v12.2s, v23.2s
378 umlal v5.2d, v13.2s, v22.2s
379 umlal v5.2d, v14.2s, v21.2s
380 umlal v5.2d, v15.2s, v20.2s
381
382 umull v6.2d, v10.2s, v26.2s
383 umlal v6.2d, v12.2s, v24.2s
384 umlal v6.2d, v14.2s, v22.2s
385 umlal v6.2d, v16.2s, v20.2s
386
387 umull v7.2d, v10.2s, v27.2s
388 umlal v7.2d, v11.2s, v26.2s
389 umlal v7.2d, v12.2s, v25.2s
390 umlal v7.2d, v13.2s, v24.2s
391 umlal v7.2d, v14.2s, v23.2s
392 umlal v7.2d, v15.2s, v22.2s
393 umlal v7.2d, v16.2s, v21.2s
394 umlal v7.2d, v17.2s, v20.2s
395
396 umull v8.2d, v10.2s, v28.2s
397 umlal v8.2d, v12.2s, v26.2s
398 umlal v8.2d, v14.2s, v24.2s
399 umlal v8.2d, v16.2s, v22.2s
400 umlal v8.2d, v18.2s, v20.2s
401
402 umull v9.2d, v10.2s, v29.2s
403 umlal v9.2d, v11.2s, v28.2s
404 umlal v9.2d, v12.2s, v27.2s
405 umlal v9.2d, v13.2s, v26.2s
406 umlal v9.2d, v14.2s, v25.2s
407 umlal v9.2d, v15.2s, v24.2s
408 umlal v9.2d, v16.2s, v23.2s
409 umlal v9.2d, v17.2s, v22.2s
410 umlal v9.2d, v18.2s, v21.2s
411 umlal v9.2d, v19.2s, v20.2s
412
413 dup v31.2s, w30
414
415 mul v22.2s, v22.2s, v31.2s
416 mul v24.2s, v24.2s, v31.2s
417 mul v26.2s, v26.2s, v31.2s
418 mul v28.2s, v28.2s, v31.2s
419
420 umlal v0.2d, v12.2s, v28.2s
421 umlal v0.2d, v14.2s, v26.2s
422 umlal v0.2d, v16.2s, v24.2s
423 umlal v0.2d, v18.2s, v22.2s
424
425 umlal v1.2d, v13.2s, v28.2s
426 umlal v1.2d, v15.2s, v26.2s
427 umlal v1.2d, v17.2s, v24.2s
428 umlal v1.2d, v19.2s, v22.2s
429
430 umlal v2.2d, v14.2s, v28.2s
431 umlal v2.2d, v16.2s, v26.2s
432 umlal v2.2d, v18.2s, v24.2s
433
434 umlal v3.2d, v15.2s, v28.2s
435 umlal v3.2d, v17.2s, v26.2s
436 umlal v3.2d, v19.2s, v24.2s
437
438 umlal v4.2d, v16.2s, v28.2s
439 umlal v4.2d, v18.2s, v26.2s
440
441 umlal v5.2d, v17.2s, v28.2s
442 umlal v5.2d, v19.2s, v26.2s
443
444 umlal v6.2d, v18.2s, v28.2s
445
446 umlal v7.2d, v19.2s, v28.2s
447
448 shl v11.2s, v11.2s, #1
449 shl v13.2s, v13.2s, #1
450 shl v15.2s, v15.2s, #1
451 shl v17.2s, v17.2s, #1
452 shl v19.2s, v19.2s, #1
453
454 umlal v2.2d, v11.2s, v21.2s
455
456 umlal v4.2d, v11.2s, v23.2s
457 umlal v4.2d, v13.2s, v21.2s
458
459 umlal v6.2d, v11.2s, v25.2s
460 umlal v6.2d, v13.2s, v23.2s
461 umlal v6.2d, v15.2s, v21.2s
462
463 umlal v8.2d, v11.2s, v27.2s
464 umlal v8.2d, v13.2s, v25.2s
465 umlal v8.2d, v15.2s, v23.2s
466 umlal v8.2d, v17.2s, v21.2s
467
468 mul v21.2s, v21.2s, v31.2s
469 mul v23.2s, v23.2s, v31.2s
470 mul v25.2s, v25.2s, v31.2s
471 mul v27.2s, v27.2s, v31.2s
472 mul v29.2s, v29.2s, v31.2s
473
474 umlal v0.2d, v11.2s, v29.2s
475 umlal v0.2d, v13.2s, v27.2s
476 umlal v0.2d, v15.2s, v25.2s
477 umlal v0.2d, v17.2s, v23.2s
478 umlal v0.2d, v19.2s, v21.2s
479
480 umlal v1.2d, v12.2s, v29.2s
481 umlal v1.2d, v14.2s, v27.2s
482 umlal v1.2d, v16.2s, v25.2s
483 umlal v1.2d, v18.2s, v23.2s
484
485 umlal v2.2d, v13.2s, v29.2s
486 umlal v2.2d, v15.2s, v27.2s
487 umlal v2.2d, v17.2s, v25.2s
488 umlal v2.2d, v19.2s, v23.2s
489
490 umlal v3.2d, v14.2s, v29.2s
491 umlal v3.2d, v16.2s, v27.2s
492 umlal v3.2d, v18.2s, v25.2s
493
494 umlal v4.2d, v15.2s, v29.2s
495 umlal v4.2d, v17.2s, v27.2s
496 umlal v4.2d, v19.2s, v25.2s
497
498 umlal v5.2d, v16.2s, v29.2s
499 umlal v5.2d, v18.2s, v27.2s
500
501 umlal v6.2d, v17.2s, v29.2s
502 umlal v6.2d, v19.2s, v27.2s
503
504 umlal v7.2d, v18.2s, v29.2s
505
506 umlal v8.2d, v19.2s, v29.2s
507
508 dup v30.2d, x29
509 ushr v25.2d, v30.2d, #1
510
511 usra v6.2d, v5.2d, #25
512 and v5.16b, v5.16b, v25.16b
513
514 usra v1.2d, v0.2d, #26
515 and v0.16b, v0.16b, v30.16b
516
517 usra v7.2d, v6.2d, #26
518 and v6.16b, v6.16b, v30.16b
519
520 usra v2.2d, v1.2d, #25
521 and v1.16b, v1.16b, v25.16b
522
523 usra v8.2d, v7.2d, #25
524 and v7.16b, v7.16b, v25.16b
525
526 usra v3.2d, v2.2d, #26
527 and v2.16b, v2.16b, v30.16b
528
529 usra v9.2d, v8.2d, #26
530 and v8.16b, v8.16b, v30.16b
531
532 usra v4.2d, v3.2d, #25
533 and v3.16b, v3.16b, v25.16b
534
535 bic v10.16b, v9.16b, v25.16b
536 usra v0.2d, v10.2d, #25
537 usra v0.2d, v10.2d, #24
538 usra v0.2d, v10.2d, #21
539 and v9.16b, v9.16b, v25.16b
540
541 usra v5.2d, v4.2d, #26
542 and v4.16b, v4.16b, v30.16b
543
544 usra v1.2d, v0.2d, #26
545 and v0.16b, v0.16b, v30.16b
546
547 usra v6.2d, v5.2d, #25
548 and v5.16b, v5.16b, v25.16b
549
550 xtn v10.2s, v0.2d
551 xtn v11.2s, v1.2d
552 xtn v12.2s, v2.2d
553 xtn v13.2s, v3.2d
554 xtn v14.2s, v4.2d
555 xtn v15.2s, v5.2d
556 xtn v16.2s, v6.2d
557 xtn v17.2s, v7.2d
558 xtn v18.2s, v8.2d
559 xtn v19.2s, v9.2d
560
561 str q10, [x20, #480]
562 str q11, [x20, #496]
563 str q12, [x20, #512]
564 str q13, [x20, #528]
565 str q14, [x20, #544]
566 str q15, [x20, #560]
567 str q16, [x20, #576]
568 str q17, [x20, #592]
569 str q18, [x20, #608]
570 str q19, [x20, #624]
571
572 // Z3 ← T2 · T3
573 ldr q10, [x20, #960]
574 ldr q11, [x20, #976]
575 ldr q12, [x20, #992]
576 ldr q13, [x20, #1008]
577 ldr q14, [x20, #1024]
578 ldr q15, [x20, #1040]
579 ldr q16, [x20, #1056]
580 ldr q17, [x20, #1072]
581 ldr q18, [x20, #1088]
582 ldr q19, [x20, #1104]
583
584 ldr q20, [x20, #1120]
585 ldr q21, [x20, #1136]
586 ldr q22, [x20, #1152]
587 ldr q23, [x20, #1168]
588 ldr q24, [x20, #1184]
589 ldr q25, [x20, #1200]
590 ldr q26, [x20, #1216]
591 ldr q27, [x20, #1232]
592 ldr q28, [x20, #1248]
593 ldr q29, [x20, #1264]
594
595 umull v0.2d, v10.2s, v20.2s
596
597 umull v1.2d, v10.2s, v21.2s
598 umlal v1.2d, v11.2s, v20.2s
599
600 umull v2.2d, v10.2s, v22.2s
601 umlal v2.2d, v12.2s, v20.2s
602
603 umull v3.2d, v10.2s, v23.2s
604 umlal v3.2d, v11.2s, v22.2s
605 umlal v3.2d, v12.2s, v21.2s
606 umlal v3.2d, v13.2s, v20.2s
607
608 umull v4.2d, v10.2s, v24.2s
609 umlal v4.2d, v12.2s, v22.2s
610 umlal v4.2d, v14.2s, v20.2s
611
612 umull v5.2d, v10.2s, v25.2s
613 umlal v5.2d, v11.2s, v24.2s
614 umlal v5.2d, v12.2s, v23.2s
615 umlal v5.2d, v13.2s, v22.2s
616 umlal v5.2d, v14.2s, v21.2s
617 umlal v5.2d, v15.2s, v20.2s
618
619 umull v6.2d, v10.2s, v26.2s
620 umlal v6.2d, v12.2s, v24.2s
621 umlal v6.2d, v14.2s, v22.2s
622 umlal v6.2d, v16.2s, v20.2s
623
624 umull v7.2d, v10.2s, v27.2s
625 umlal v7.2d, v11.2s, v26.2s
626 umlal v7.2d, v12.2s, v25.2s
627 umlal v7.2d, v13.2s, v24.2s
628 umlal v7.2d, v14.2s, v23.2s
629 umlal v7.2d, v15.2s, v22.2s
630 umlal v7.2d, v16.2s, v21.2s
631 umlal v7.2d, v17.2s, v20.2s
632
633 umull v8.2d, v10.2s, v28.2s
634 umlal v8.2d, v12.2s, v26.2s
635 umlal v8.2d, v14.2s, v24.2s
636 umlal v8.2d, v16.2s, v22.2s
637 umlal v8.2d, v18.2s, v20.2s
638
639 umull v9.2d, v10.2s, v29.2s
640 umlal v9.2d, v11.2s, v28.2s
641 umlal v9.2d, v12.2s, v27.2s
642 umlal v9.2d, v13.2s, v26.2s
643 umlal v9.2d, v14.2s, v25.2s
644 umlal v9.2d, v15.2s, v24.2s
645 umlal v9.2d, v16.2s, v23.2s
646 umlal v9.2d, v17.2s, v22.2s
647 umlal v9.2d, v18.2s, v21.2s
648 umlal v9.2d, v19.2s, v20.2s
649
650 dup v31.2s, w30
651
652 mul v22.2s, v22.2s, v31.2s
653 mul v24.2s, v24.2s, v31.2s
654 mul v26.2s, v26.2s, v31.2s
655 mul v28.2s, v28.2s, v31.2s
656
657 umlal v0.2d, v12.2s, v28.2s
658 umlal v0.2d, v14.2s, v26.2s
659 umlal v0.2d, v16.2s, v24.2s
660 umlal v0.2d, v18.2s, v22.2s
661
662 umlal v1.2d, v13.2s, v28.2s
663 umlal v1.2d, v15.2s, v26.2s
664 umlal v1.2d, v17.2s, v24.2s
665 umlal v1.2d, v19.2s, v22.2s
666
667 umlal v2.2d, v14.2s, v28.2s
668 umlal v2.2d, v16.2s, v26.2s
669 umlal v2.2d, v18.2s, v24.2s
670
671 umlal v3.2d, v15.2s, v28.2s
672 umlal v3.2d, v17.2s, v26.2s
673 umlal v3.2d, v19.2s, v24.2s
674
675 umlal v4.2d, v16.2s, v28.2s
676 umlal v4.2d, v18.2s, v26.2s
677
678 umlal v5.2d, v17.2s, v28.2s
679 umlal v5.2d, v19.2s, v26.2s
680
681 umlal v6.2d, v18.2s, v28.2s
682
683 umlal v7.2d, v19.2s, v28.2s
684
685 shl v11.2s, v11.2s, #1
686 shl v13.2s, v13.2s, #1
687 shl v15.2s, v15.2s, #1
688 shl v17.2s, v17.2s, #1
689 shl v19.2s, v19.2s, #1
690
691 umlal v2.2d, v11.2s, v21.2s
692
693 umlal v4.2d, v11.2s, v23.2s
694 umlal v4.2d, v13.2s, v21.2s
695
696 umlal v6.2d, v11.2s, v25.2s
697 umlal v6.2d, v13.2s, v23.2s
698 umlal v6.2d, v15.2s, v21.2s
699
700 umlal v8.2d, v11.2s, v27.2s
701 umlal v8.2d, v13.2s, v25.2s
702 umlal v8.2d, v15.2s, v23.2s
703 umlal v8.2d, v17.2s, v21.2s
704
705 mul v21.2s, v21.2s, v31.2s
706 mul v23.2s, v23.2s, v31.2s
707 mul v25.2s, v25.2s, v31.2s
708 mul v27.2s, v27.2s, v31.2s
709 mul v29.2s, v29.2s, v31.2s
710
711 umlal v0.2d, v11.2s, v29.2s
712 umlal v0.2d, v13.2s, v27.2s
713 umlal v0.2d, v15.2s, v25.2s
714 umlal v0.2d, v17.2s, v23.2s
715 umlal v0.2d, v19.2s, v21.2s
716
717 umlal v1.2d, v12.2s, v29.2s
718 umlal v1.2d, v14.2s, v27.2s
719 umlal v1.2d, v16.2s, v25.2s
720 umlal v1.2d, v18.2s, v23.2s
721
722 umlal v2.2d, v13.2s, v29.2s
723 umlal v2.2d, v15.2s, v27.2s
724 umlal v2.2d, v17.2s, v25.2s
725 umlal v2.2d, v19.2s, v23.2s
726
727 umlal v3.2d, v14.2s, v29.2s
728 umlal v3.2d, v16.2s, v27.2s
729 umlal v3.2d, v18.2s, v25.2s
730
731 umlal v4.2d, v15.2s, v29.2s
732 umlal v4.2d, v17.2s, v27.2s
733 umlal v4.2d, v19.2s, v25.2s
734
735 umlal v5.2d, v16.2s, v29.2s
736 umlal v5.2d, v18.2s, v27.2s
737
738 umlal v6.2d, v17.2s, v29.2s
739 umlal v6.2d, v19.2s, v27.2s
740
741 umlal v7.2d, v18.2s, v29.2s
742
743 umlal v8.2d, v19.2s, v29.2s
744
745 dup v30.2d, x29
746 ushr v25.2d, v30.2d, #1
747
748 usra v6.2d, v5.2d, #25
749 and v5.16b, v5.16b, v25.16b
750
751 usra v1.2d, v0.2d, #26
752 and v0.16b, v0.16b, v30.16b
753
754 usra v7.2d, v6.2d, #26
755 and v6.16b, v6.16b, v30.16b
756
757 usra v2.2d, v1.2d, #25
758 and v1.16b, v1.16b, v25.16b
759
760 usra v8.2d, v7.2d, #25
761 and v7.16b, v7.16b, v25.16b
762
763 usra v3.2d, v2.2d, #26
764 and v2.16b, v2.16b, v30.16b
765
766 usra v9.2d, v8.2d, #26
767 and v8.16b, v8.16b, v30.16b
768
769 usra v4.2d, v3.2d, #25
770 and v3.16b, v3.16b, v25.16b
771
772 bic v10.16b, v9.16b, v25.16b
773 usra v0.2d, v10.2d, #25
774 usra v0.2d, v10.2d, #24
775 usra v0.2d, v10.2d, #21
776 and v9.16b, v9.16b, v25.16b
777
778 usra v5.2d, v4.2d, #26
779 and v4.16b, v4.16b, v30.16b
780
781 usra v1.2d, v0.2d, #26
782 and v0.16b, v0.16b, v30.16b
783
784 usra v6.2d, v5.2d, #25
785 and v5.16b, v5.16b, v25.16b
786
787 xtn v10.2s, v0.2d
788 xtn v11.2s, v1.2d
789 xtn v12.2s, v2.2d
790 xtn v13.2s, v3.2d
791 xtn v14.2s, v4.2d
792 xtn v15.2s, v5.2d
793 xtn v16.2s, v6.2d
794 xtn v17.2s, v7.2d
795 xtn v18.2s, v8.2d
796 xtn v19.2s, v9.2d
797
798 str q10, [x20, #640]
799 str q11, [x20, #656]
800 str q12, [x20, #672]
801 str q13, [x20, #688]
802 str q14, [x20, #704]
803 str q15, [x20, #720]
804 str q16, [x20, #736]
805 str q17, [x20, #752]
806 str q18, [x20, #768]
807 str q19, [x20, #784]
808
809 // conditional select
810 ldp x8, x9, [sp, #176]
811
812 lsr x8, x8, x15
813 lsr x9, x9, x15
814
815 mov v31.d[0], x8
816 mov v31.d[1], x9
817
818 xtn v31.2s, v31.2d
819
820 dup v30.2s, w4
821
822 and v29.16b, v31.16b, v30.16b
823
824 ldr q28, [sp, #160]
825 eor v31.16b, v29.16b, v28.16b
826 str q29, [sp, #160]
827
828 sub x15, x15, #1
829
830 dup v30.2s, wzr
831 sub v30.2s, v30.2s, v31.2s
832
833 // T1 = CSelect(T1,T3)
834 ldr q10, [x20, #800]
835 ldr q11, [x20, #816]
836 ldr q12, [x20, #832]
837 ldr q13, [x20, #848]
838 ldr q14, [x20, #864]
839 ldr q15, [x20, #880]
840 ldr q16, [x20, #896]
841 ldr q17, [x20, #912]
842 ldr q18, [x20, #928]
843 ldr q19, [x20, #944]
844
845 ldr q20, [x20, #1120]
846 ldr q21, [x20, #1136]
847 ldr q22, [x20, #1152]
848 ldr q23, [x20, #1168]
849 ldr q24, [x20, #1184]
850 ldr q25, [x20, #1200]
851 ldr q26, [x20, #1216]
852 ldr q27, [x20, #1232]
853 ldr q28, [x20, #1248]
854 ldr q29, [x20, #1264]
855
856 eor v0.16b, v10.16b, v20.16b
857 eor v1.16b, v11.16b, v21.16b
858 eor v2.16b, v12.16b, v22.16b
859 eor v3.16b, v13.16b, v23.16b
860 eor v4.16b, v14.16b, v24.16b
861 eor v5.16b, v15.16b, v25.16b
862 eor v6.16b, v16.16b, v26.16b
863 eor v7.16b, v17.16b, v27.16b
864 eor v8.16b, v18.16b, v28.16b
865 eor v9.16b, v19.16b, v29.16b
866
867 and v0.16b, v0.16b, v30.16b
868 and v1.16b, v1.16b, v30.16b
869 and v2.16b, v2.16b, v30.16b
870 and v3.16b, v3.16b, v30.16b
871 and v4.16b, v4.16b, v30.16b
872 and v5.16b, v5.16b, v30.16b
873 and v6.16b, v6.16b, v30.16b
874 and v7.16b, v7.16b, v30.16b
875 and v8.16b, v8.16b, v30.16b
876 and v9.16b, v9.16b, v30.16b
877
878 eor v0.16b, v0.16b, v10.16b
879 eor v1.16b, v1.16b, v11.16b
880 eor v2.16b, v2.16b, v12.16b
881 eor v3.16b, v3.16b, v13.16b
882 eor v4.16b, v4.16b, v14.16b
883 eor v5.16b, v5.16b, v15.16b
884 eor v6.16b, v6.16b, v16.16b
885 eor v7.16b, v7.16b, v17.16b
886 eor v8.16b, v8.16b, v18.16b
887 eor v9.16b, v9.16b, v19.16b
888
889 str q0, [x20, #800]
890 str q1, [x20, #816]
891 str q2, [x20, #832]
892 str q3, [x20, #848]
893 str q4, [x20, #864]
894 str q5, [x20, #880]
895 str q6, [x20, #896]
896 str q7, [x20, #912]
897 str q8, [x20, #928]
898 str q9, [x20, #944]
899
900 // T2 = CSelect(T2,T4)
901 ldr q10, [x20, #960]
902 ldr q11, [x20, #976]
903 ldr q12, [x20, #992]
904 ldr q13, [x20, #1008]
905 ldr q14, [x20, #1024]
906 ldr q15, [x20, #1040]
907 ldr q16, [x20, #1056]
908 ldr q17, [x20, #1072]
909 ldr q18, [x20, #1088]
910 ldr q19, [x20, #1104]
911
912 ldr q20, [x20, #1280]
913 ldr q21, [x20, #1296]
914 ldr q22, [x20, #1312]
915 ldr q23, [x20, #1328]
916 ldr q24, [x20, #1344]
917 ldr q25, [x20, #1360]
918 ldr q26, [x20, #1376]
919 ldr q27, [x20, #1392]
920 ldr q28, [x20, #1408]
921 ldr q29, [x20, #1424]
922
923 eor v0.16b, v10.16b, v20.16b
924 eor v1.16b, v11.16b, v21.16b
925 eor v2.16b, v12.16b, v22.16b
926 eor v3.16b, v13.16b, v23.16b
927 eor v4.16b, v14.16b, v24.16b
928 eor v5.16b, v15.16b, v25.16b
929 eor v6.16b, v16.16b, v26.16b
930 eor v7.16b, v17.16b, v27.16b
931 eor v8.16b, v18.16b, v28.16b
932 eor v9.16b, v19.16b, v29.16b
933
934 and v0.16b, v0.16b, v30.16b
935 and v1.16b, v1.16b, v30.16b
936 and v2.16b, v2.16b, v30.16b
937 and v3.16b, v3.16b, v30.16b
938 and v4.16b, v4.16b, v30.16b
939 and v5.16b, v5.16b, v30.16b
940 and v6.16b, v6.16b, v30.16b
941 and v7.16b, v7.16b, v30.16b
942 and v8.16b, v8.16b, v30.16b
943 and v9.16b, v9.16b, v30.16b
944
945 eor v10.16b, v0.16b, v10.16b
946 eor v11.16b, v1.16b, v11.16b
947 eor v12.16b, v2.16b, v12.16b
948 eor v13.16b, v3.16b, v13.16b
949 eor v14.16b, v4.16b, v14.16b
950 eor v15.16b, v5.16b, v15.16b
951 eor v16.16b, v6.16b, v16.16b
952 eor v17.16b, v7.16b, v17.16b
953 eor v18.16b, v8.16b, v18.16b
954 eor v19.16b, v9.16b, v19.16b
955
956 // T2 ← T2^2
957 dup v31.2s, w30
958 shl v0.2d, v31.2d, #1
959
960 mul v20.2s, v16.2s, v31.2s
961 mul v21.2s, v18.2s, v31.2s
962 mul v22.2s, v15.2s, v0.2s
963 mul v23.2s, v17.2s, v0.2s
964 mul v24.2s, v19.2s, v0.2s
965
966 shl v25.2s, v10.2s, #1
967 shl v26.2s, v11.2s, #1
968 shl v27.2s, v12.2s, #1
969 shl v28.2s, v13.2s, #1
970 shl v29.2s, v14.2s, #1
971 shl v7.2s, v15.2s, #1
972 shl v8.2s, v16.2s, #1
973 shl v9.2s, v17.2s, #1
974
975 umull v0.2d, v10.2s, v10.2s
976 umlal v0.2d, v26.2s, v24.2s
977 umlal v0.2d, v27.2s, v21.2s
978 umlal v0.2d, v28.2s, v23.2s
979 umlal v0.2d, v29.2s, v20.2s
980 umlal v0.2d, v22.2s, v15.2s
981
982 umull v1.2d, v25.2s, v11.2s
983 umlal v1.2d, v24.2s, v12.2s
984 umlal v1.2d, v28.2s, v21.2s
985 umlal v1.2d, v23.2s, v14.2s
986 umlal v1.2d, v20.2s, v7.2s
987
988 umull v2.2d, v25.2s, v12.2s
989 umlal v2.2d, v26.2s, v11.2s
990 umlal v2.2d, v28.2s, v24.2s
991 umlal v2.2d, v29.2s, v21.2s
992 umlal v2.2d, v23.2s, v7.2s
993 umlal v2.2d, v20.2s, v16.2s
994
995 umull v3.2d, v25.2s, v13.2s
996 umlal v3.2d, v26.2s, v12.2s
997 umlal v3.2d, v24.2s, v14.2s
998 umlal v3.2d, v21.2s, v7.2s
999 umlal v3.2d, v23.2s, v16.2s
1000
1001 umull v4.2d, v25.2s, v14.2s
1002 umlal v4.2d, v26.2s, v28.2s
1003 umlal v4.2d, v12.2s, v12.2s
1004 umlal v4.2d, v24.2s, v7.2s
1005 umlal v4.2d, v8.2s, v21.2s
1006 umlal v4.2d, v23.2s, v17.2s
1007
1008 umull v5.2d, v25.2s, v15.2s
1009 umlal v5.2d, v26.2s, v14.2s
1010 umlal v5.2d, v27.2s, v13.2s
1011 umlal v5.2d, v24.2s, v16.2s
1012 umlal v5.2d, v21.2s, v9.2s
1013
1014 umull v6.2d, v25.2s, v16.2s
1015 umlal v6.2d, v26.2s, v7.2s
1016 umlal v6.2d, v27.2s, v14.2s
1017 umlal v6.2d, v28.2s, v13.2s
1018 umlal v6.2d, v24.2s, v9.2s
1019 umlal v6.2d, v21.2s, v18.2s
1020
1021 umull v8.2d, v25.2s, v18.2s
1022 umlal v8.2d, v26.2s, v9.2s
1023 umlal v8.2d, v27.2s, v16.2s
1024 umlal v8.2d, v28.2s, v7.2s
1025 umlal v8.2d, v14.2s, v14.2s
1026 umlal v8.2d, v24.2s, v19.2s
1027
1028 umull v7.2d, v25.2s, v17.2s
1029 umlal v7.2d, v26.2s, v16.2s
1030 umlal v7.2d, v27.2s, v15.2s
1031 umlal v7.2d, v28.2s, v14.2s
1032 umlal v7.2d, v24.2s, v18.2s
1033
1034 umull v9.2d, v25.2s, v19.2s
1035 umlal v9.2d, v26.2s, v18.2s
1036 umlal v9.2d, v27.2s, v17.2s
1037 umlal v9.2d, v28.2s, v16.2s
1038 umlal v9.2d, v29.2s, v15.2s
1039
1040 dup v30.2d, x29
1041 ushr v25.2d, v30.2d, #1
1042
1043 usra v6.2d, v5.2d, #25
1044 and v5.16b, v5.16b, v25.16b
1045
1046 usra v1.2d, v0.2d, #26
1047 and v0.16b, v0.16b, v30.16b
1048
1049 usra v7.2d, v6.2d, #26
1050 and v6.16b, v6.16b, v30.16b
1051
1052 usra v2.2d, v1.2d, #25
1053 and v1.16b, v1.16b, v25.16b
1054
1055 usra v8.2d, v7.2d, #25
1056 and v7.16b, v7.16b, v25.16b
1057
1058 usra v3.2d, v2.2d, #26
1059 and v2.16b, v2.16b, v30.16b
1060
1061 usra v9.2d, v8.2d, #26
1062 and v8.16b, v8.16b, v30.16b
1063
1064 usra v4.2d, v3.2d, #25
1065 and v3.16b, v3.16b, v25.16b
1066
1067 bic v10.16b, v9.16b, v25.16b
1068 usra v0.2d, v10.2d, #25
1069 usra v0.2d, v10.2d, #24
1070 usra v0.2d, v10.2d, #21
1071 and v9.16b, v9.16b, v25.16b
1072
1073 usra v5.2d, v4.2d, #26
1074 and v4.16b, v4.16b, v30.16b
1075
1076 usra v1.2d, v0.2d, #26
1077 and v0.16b, v0.16b, v30.16b
1078
1079 usra v6.2d, v5.2d, #25
1080 and v5.16b, v5.16b, v25.16b
1081
1082 xtn v10.2s, v0.2d
1083 xtn v11.2s, v1.2d
1084 xtn v12.2s, v2.2d
1085 xtn v13.2s, v3.2d
1086 xtn v14.2s, v4.2d
1087 xtn v15.2s, v5.2d
1088 xtn v16.2s, v6.2d
1089 xtn v17.2s, v7.2d
1090 xtn v18.2s, v8.2d
1091 xtn v19.2s, v9.2d
1092
1093 str q10, [x20, #960]
1094 str q11, [x20, #976]
1095 str q12, [x20, #992]
1096 str q13, [x20, #1008]
1097 str q14, [x20, #1024]
1098 str q15, [x20, #1040]
1099 str q16, [x20, #1056]
1100 str q17, [x20, #1072]
1101 str q18, [x20, #1088]
1102 str q19, [x20, #1104]
1103
1104 // T1 ← T1^2
1105 ldr q10, [x20, #800]
1106 ldr q11, [x20, #816]
1107 ldr q12, [x20, #832]
1108 ldr q13, [x20, #848]
1109 ldr q14, [x20, #864]
1110 ldr q15, [x20, #880]
1111 ldr q16, [x20, #896]
1112 ldr q17, [x20, #912]
1113 ldr q18, [x20, #928]
1114 ldr q19, [x20, #944]
1115
1116 dup v31.2s, w30
1117 shl v0.2d, v31.2d, #1
1118
1119 mul v20.2s, v16.2s, v31.2s
1120 mul v21.2s, v18.2s, v31.2s
1121 mul v22.2s, v15.2s, v0.2s
1122 mul v23.2s, v17.2s, v0.2s
1123 mul v24.2s, v19.2s, v0.2s
1124
1125 shl v25.2s, v10.2s, #1
1126 shl v26.2s, v11.2s, #1
1127 shl v27.2s, v12.2s, #1
1128 shl v28.2s, v13.2s, #1
1129 shl v29.2s, v14.2s, #1
1130 shl v7.2s, v15.2s, #1
1131 shl v8.2s, v16.2s, #1
1132 shl v9.2s, v17.2s, #1
1133
1134 umull v0.2d, v10.2s, v10.2s
1135 umlal v0.2d, v26.2s, v24.2s
1136 umlal v0.2d, v27.2s, v21.2s
1137 umlal v0.2d, v28.2s, v23.2s
1138 umlal v0.2d, v29.2s, v20.2s
1139 umlal v0.2d, v22.2s, v15.2s
1140
1141 umull v1.2d, v25.2s, v11.2s
1142 umlal v1.2d, v24.2s, v12.2s
1143 umlal v1.2d, v28.2s, v21.2s
1144 umlal v1.2d, v23.2s, v14.2s
1145 umlal v1.2d, v20.2s, v7.2s
1146
1147 umull v2.2d, v25.2s, v12.2s
1148 umlal v2.2d, v26.2s, v11.2s
1149 umlal v2.2d, v28.2s, v24.2s
1150 umlal v2.2d, v29.2s, v21.2s
1151 umlal v2.2d, v23.2s, v7.2s
1152 umlal v2.2d, v20.2s, v16.2s
1153
1154 umull v3.2d, v25.2s, v13.2s
1155 umlal v3.2d, v26.2s, v12.2s
1156 umlal v3.2d, v24.2s, v14.2s
1157 umlal v3.2d, v21.2s, v7.2s
1158 umlal v3.2d, v23.2s, v16.2s
1159
1160 umull v4.2d, v25.2s, v14.2s
1161 umlal v4.2d, v26.2s, v28.2s
1162 umlal v4.2d, v12.2s, v12.2s
1163 umlal v4.2d, v24.2s, v7.2s
1164 umlal v4.2d, v8.2s, v21.2s
1165 umlal v4.2d, v23.2s, v17.2s
1166
1167 umull v5.2d, v25.2s, v15.2s
1168 umlal v5.2d, v26.2s, v14.2s
1169 umlal v5.2d, v27.2s, v13.2s
1170 umlal v5.2d, v24.2s, v16.2s
1171 umlal v5.2d, v21.2s, v9.2s
1172
1173 umull v6.2d, v25.2s, v16.2s
1174 umlal v6.2d, v26.2s, v7.2s
1175 umlal v6.2d, v27.2s, v14.2s
1176 umlal v6.2d, v28.2s, v13.2s
1177 umlal v6.2d, v24.2s, v9.2s
1178 umlal v6.2d, v21.2s, v18.2s
1179
1180 umull v8.2d, v25.2s, v18.2s
1181 umlal v8.2d, v26.2s, v9.2s
1182 umlal v8.2d, v27.2s, v16.2s
1183 umlal v8.2d, v28.2s, v7.2s
1184 umlal v8.2d, v14.2s, v14.2s
1185 umlal v8.2d, v24.2s, v19.2s
1186
1187 umull v7.2d, v25.2s, v17.2s
1188 umlal v7.2d, v26.2s, v16.2s
1189 umlal v7.2d, v27.2s, v15.2s
1190 umlal v7.2d, v28.2s, v14.2s
1191 umlal v7.2d, v24.2s, v18.2s
1192
1193 umull v9.2d, v25.2s, v19.2s
1194 umlal v9.2d, v26.2s, v18.2s
1195 umlal v9.2d, v27.2s, v17.2s
1196 umlal v9.2d, v28.2s, v16.2s
1197 umlal v9.2d, v29.2s, v15.2s
1198
1199 dup v30.2d, x29
1200 ushr v25.2d, v30.2d, #1
1201
1202 usra v6.2d, v5.2d, #25
1203 and v5.16b, v5.16b, v25.16b
1204
1205 usra v1.2d, v0.2d, #26
1206 and v0.16b, v0.16b, v30.16b
1207
1208 usra v7.2d, v6.2d, #26
1209 and v6.16b, v6.16b, v30.16b
1210
1211 usra v2.2d, v1.2d, #25
1212 and v1.16b, v1.16b, v25.16b
1213
1214 usra v8.2d, v7.2d, #25
1215 and v7.16b, v7.16b, v25.16b
1216
1217 usra v3.2d, v2.2d, #26
1218 and v2.16b, v2.16b, v30.16b
1219
1220 usra v9.2d, v8.2d, #26
1221 and v8.16b, v8.16b, v30.16b
1222
1223 usra v4.2d, v3.2d, #25
1224 and v3.16b, v3.16b, v25.16b
1225
1226 bic v10.16b, v9.16b, v25.16b
1227 usra v0.2d, v10.2d, #25
1228 usra v0.2d, v10.2d, #24
1229 usra v0.2d, v10.2d, #21
1230 and v9.16b, v9.16b, v25.16b
1231
1232 usra v5.2d, v4.2d, #26
1233 and v4.16b, v4.16b, v30.16b
1234
1235 usra v1.2d, v0.2d, #26
1236 and v0.16b, v0.16b, v30.16b
1237
1238 usra v6.2d, v5.2d, #25
1239 and v5.16b, v5.16b, v25.16b
1240
1241 xtn v10.2s, v0.2d
1242 xtn v11.2s, v1.2d
1243 xtn v12.2s, v2.2d
1244 xtn v13.2s, v3.2d
1245 xtn v14.2s, v4.2d
1246 xtn v15.2s, v5.2d
1247 xtn v16.2s, v6.2d
1248 xtn v17.2s, v7.2d
1249 xtn v18.2s, v8.2d
1250 xtn v19.2s, v9.2d
1251
1252 str q10, [x20, #800]
1253 str q11, [x20, #816]
1254 str q12, [x20, #832]
1255 str q13, [x20, #848]
1256 str q14, [x20, #864]
1257 str q15, [x20, #880]
1258 str q16, [x20, #896]
1259 str q17, [x20, #912]
1260 str q18, [x20, #928]
1261 str q19, [x20, #944]
1262
1263 // X3
1264 ldr q0, [x20, #480]
1265 ldr q1, [x20, #496]
1266 ldr q2, [x20, #512]
1267 ldr q3, [x20, #528]
1268 ldr q4, [x20, #544]
1269 ldr q5, [x20, #560]
1270 ldr q6, [x20, #576]
1271 ldr q7, [x20, #592]
1272 ldr q8, [x20, #608]
1273 ldr q9, [x20, #624]
1274
1275 // Z3
1276 ldr q10, [x20, #640]
1277 ldr q11, [x20, #656]
1278 ldr q12, [x20, #672]
1279 ldr q13, [x20, #688]
1280 ldr q14, [x20, #704]
1281 ldr q15, [x20, #720]
1282 ldr q16, [x20, #736]
1283 ldr q17, [x20, #752]
1284 ldr q18, [x20, #768]
1285 ldr q19, [x20, #784]
1286
1287 // T3 ← X3 + Z3
1288 add v20.2s, v0.2s, v10.2s
1289 add v21.2s, v1.2s, v11.2s
1290 add v22.2s, v2.2s, v12.2s
1291 add v23.2s, v3.2s, v13.2s
1292 add v24.2s, v4.2s, v14.2s
1293 add v25.2s, v5.2s, v15.2s
1294 add v26.2s, v6.2s, v16.2s
1295 add v27.2s, v7.2s, v17.2s
1296 add v28.2s, v8.2s, v18.2s
1297 add v29.2s, v9.2s, v19.2s
1298
1299 str q20, [x20, #1120]
1300 str q21, [x20, #1136]
1301 str q22, [x20, #1152]
1302 str q23, [x20, #1168]
1303 str q24, [x20, #1184]
1304 str q25, [x20, #1200]
1305 str q26, [x20, #1216]
1306 str q27, [x20, #1232]
1307 str q28, [x20, #1248]
1308 str q29, [x20, #1264]
1309
1310 // Z3 ← X3 - Z3
1311 mov v30.d[0], x21
1312 add v0.2s, v30.2s, v0.2s
1313 mov v30.d[0], x22
1314 mov v31.d[0], x23
1315 add v1.2s, v30.2s, v1.2s
1316 add v2.2s, v31.2s, v2.2s
1317 add v3.2s, v30.2s, v3.2s
1318 add v4.2s, v31.2s, v4.2s
1319 add v5.2s, v30.2s, v5.2s
1320 add v6.2s, v31.2s, v6.2s
1321 add v7.2s, v30.2s, v7.2s
1322 add v8.2s, v31.2s, v8.2s
1323 add v9.2s, v30.2s, v9.2s
1324
1325 sub v10.2s, v0.2s, v10.2s
1326 sub v11.2s, v1.2s, v11.2s
1327 sub v12.2s, v2.2s, v12.2s
1328 sub v13.2s, v3.2s, v13.2s
1329 sub v14.2s, v4.2s, v14.2s
1330 sub v15.2s, v5.2s, v15.2s
1331 sub v16.2s, v6.2s, v16.2s
1332 sub v17.2s, v7.2s, v17.2s
1333 sub v18.2s, v8.2s, v18.2s
1334 sub v19.2s, v9.2s, v19.2s
1335
1336 // Z3 ← Z3^2
1337 dup v31.2s, w30
1338 shl v0.2d, v31.2d, #1
1339
1340 mul v20.2s, v16.2s, v31.2s
1341 mul v21.2s, v18.2s, v31.2s
1342 mul v22.2s, v15.2s, v0.2s
1343 mul v23.2s, v17.2s, v0.2s
1344 mul v24.2s, v19.2s, v0.2s
1345
1346 shl v25.2s, v10.2s, #1
1347 shl v26.2s, v11.2s, #1
1348 shl v27.2s, v12.2s, #1
1349 shl v28.2s, v13.2s, #1
1350 shl v29.2s, v14.2s, #1
1351 shl v7.2s, v15.2s, #1
1352 shl v8.2s, v16.2s, #1
1353 shl v9.2s, v17.2s, #1
1354
1355 umull v0.2d, v10.2s, v10.2s
1356 umlal v0.2d, v26.2s, v24.2s
1357 umlal v0.2d, v27.2s, v21.2s
1358 umlal v0.2d, v28.2s, v23.2s
1359 umlal v0.2d, v29.2s, v20.2s
1360 umlal v0.2d, v22.2s, v15.2s
1361
1362 umull v1.2d, v25.2s, v11.2s
1363 umlal v1.2d, v24.2s, v12.2s
1364 umlal v1.2d, v28.2s, v21.2s
1365 umlal v1.2d, v23.2s, v14.2s
1366 umlal v1.2d, v20.2s, v7.2s
1367
1368 umull v2.2d, v25.2s, v12.2s
1369 umlal v2.2d, v26.2s, v11.2s
1370 umlal v2.2d, v28.2s, v24.2s
1371 umlal v2.2d, v29.2s, v21.2s
1372 umlal v2.2d, v23.2s, v7.2s
1373 umlal v2.2d, v20.2s, v16.2s
1374
1375 umull v3.2d, v25.2s, v13.2s
1376 umlal v3.2d, v26.2s, v12.2s
1377 umlal v3.2d, v24.2s, v14.2s
1378 umlal v3.2d, v21.2s, v7.2s
1379 umlal v3.2d, v23.2s, v16.2s
1380
1381 umull v4.2d, v25.2s, v14.2s
1382 umlal v4.2d, v26.2s, v28.2s
1383 umlal v4.2d, v12.2s, v12.2s
1384 umlal v4.2d, v24.2s, v7.2s
1385 umlal v4.2d, v8.2s, v21.2s
1386 umlal v4.2d, v23.2s, v17.2s
1387
1388 umull v5.2d, v25.2s, v15.2s
1389 umlal v5.2d, v26.2s, v14.2s
1390 umlal v5.2d, v27.2s, v13.2s
1391 umlal v5.2d, v24.2s, v16.2s
1392 umlal v5.2d, v21.2s, v9.2s
1393
1394 umull v6.2d, v25.2s, v16.2s
1395 umlal v6.2d, v26.2s, v7.2s
1396 umlal v6.2d, v27.2s, v14.2s
1397 umlal v6.2d, v28.2s, v13.2s
1398 umlal v6.2d, v24.2s, v9.2s
1399 umlal v6.2d, v21.2s, v18.2s
1400
1401 umull v8.2d, v25.2s, v18.2s
1402 umlal v8.2d, v26.2s, v9.2s
1403 umlal v8.2d, v27.2s, v16.2s
1404 umlal v8.2d, v28.2s, v7.2s
1405 umlal v8.2d, v14.2s, v14.2s
1406 umlal v8.2d, v24.2s, v19.2s
1407
1408 umull v7.2d, v25.2s, v17.2s
1409 umlal v7.2d, v26.2s, v16.2s
1410 umlal v7.2d, v27.2s, v15.2s
1411 umlal v7.2d, v28.2s, v14.2s
1412 umlal v7.2d, v24.2s, v18.2s
1413
1414 umull v9.2d, v25.2s, v19.2s
1415 umlal v9.2d, v26.2s, v18.2s
1416 umlal v9.2d, v27.2s, v17.2s
1417 umlal v9.2d, v28.2s, v16.2s
1418 umlal v9.2d, v29.2s, v15.2s
1419
1420 dup v30.2d, x29
1421 ushr v25.2d, v30.2d, #1
1422
1423 usra v6.2d, v5.2d, #25
1424 and v5.16b, v5.16b, v25.16b
1425
1426 usra v1.2d, v0.2d, #26
1427 and v0.16b, v0.16b, v30.16b
1428
1429 usra v7.2d, v6.2d, #26
1430 and v6.16b, v6.16b, v30.16b
1431
1432 usra v2.2d, v1.2d, #25
1433 and v1.16b, v1.16b, v25.16b
1434
1435 usra v8.2d, v7.2d, #25
1436 and v7.16b, v7.16b, v25.16b
1437
1438 usra v3.2d, v2.2d, #26
1439 and v2.16b, v2.16b, v30.16b
1440
1441 usra v9.2d, v8.2d, #26
1442 and v8.16b, v8.16b, v30.16b
1443
1444 usra v4.2d, v3.2d, #25
1445 and v3.16b, v3.16b, v25.16b
1446
1447 bic v10.16b, v9.16b, v25.16b
1448 usra v0.2d, v10.2d, #25
1449 usra v0.2d, v10.2d, #24
1450 usra v0.2d, v10.2d, #21
1451 and v9.16b, v9.16b, v25.16b
1452
1453 usra v5.2d, v4.2d, #26
1454 and v4.16b, v4.16b, v30.16b
1455
1456 usra v1.2d, v0.2d, #26
1457 and v0.16b, v0.16b, v30.16b
1458
1459 usra v6.2d, v5.2d, #25
1460 and v5.16b, v5.16b, v25.16b
1461
1462 xtn v10.2s, v0.2d
1463 xtn v11.2s, v1.2d
1464 xtn v12.2s, v2.2d
1465 xtn v13.2s, v3.2d
1466 xtn v14.2s, v4.2d
1467 xtn v15.2s, v5.2d
1468 xtn v16.2s, v6.2d
1469 xtn v17.2s, v7.2d
1470 xtn v18.2s, v8.2d
1471 xtn v19.2s, v9.2d
1472
1473 str q10, [x20, #640]
1474 str q11, [x20, #656]
1475 str q12, [x20, #672]
1476 str q13, [x20, #688]
1477 str q14, [x20, #704]
1478 str q15, [x20, #720]
1479 str q16, [x20, #736]
1480 str q17, [x20, #752]
1481 str q18, [x20, #768]
1482 str q19, [x20, #784]
1483
1484 // X3 ← T3^2
1485 ldr q10, [x20, #1120]
1486 ldr q11, [x20, #1136]
1487 ldr q12, [x20, #1152]
1488 ldr q13, [x20, #1168]
1489 ldr q14, [x20, #1184]
1490 ldr q15, [x20, #1200]
1491 ldr q16, [x20, #1216]
1492 ldr q17, [x20, #1232]
1493 ldr q18, [x20, #1248]
1494 ldr q19, [x20, #1264]
1495
1496 dup v31.2s, w30
1497 shl v0.2d, v31.2d, #1
1498
1499 mul v20.2s, v16.2s, v31.2s
1500 mul v21.2s, v18.2s, v31.2s
1501 mul v22.2s, v15.2s, v0.2s
1502 mul v23.2s, v17.2s, v0.2s
1503 mul v24.2s, v19.2s, v0.2s
1504
1505 shl v25.2s, v10.2s, #1
1506 shl v26.2s, v11.2s, #1
1507 shl v27.2s, v12.2s, #1
1508 shl v28.2s, v13.2s, #1
1509 shl v29.2s, v14.2s, #1
1510 shl v7.2s, v15.2s, #1
1511 shl v8.2s, v16.2s, #1
1512 shl v9.2s, v17.2s, #1
1513
1514 umull v0.2d, v10.2s, v10.2s
1515 umlal v0.2d, v26.2s, v24.2s
1516 umlal v0.2d, v27.2s, v21.2s
1517 umlal v0.2d, v28.2s, v23.2s
1518 umlal v0.2d, v29.2s, v20.2s
1519 umlal v0.2d, v22.2s, v15.2s
1520
1521 umull v1.2d, v25.2s, v11.2s
1522 umlal v1.2d, v24.2s, v12.2s
1523 umlal v1.2d, v28.2s, v21.2s
1524 umlal v1.2d, v23.2s, v14.2s
1525 umlal v1.2d, v20.2s, v7.2s
1526
1527 umull v2.2d, v25.2s, v12.2s
1528 umlal v2.2d, v26.2s, v11.2s
1529 umlal v2.2d, v28.2s, v24.2s
1530 umlal v2.2d, v29.2s, v21.2s
1531 umlal v2.2d, v23.2s, v7.2s
1532 umlal v2.2d, v20.2s, v16.2s
1533
1534 umull v3.2d, v25.2s, v13.2s
1535 umlal v3.2d, v26.2s, v12.2s
1536 umlal v3.2d, v24.2s, v14.2s
1537 umlal v3.2d, v21.2s, v7.2s
1538 umlal v3.2d, v23.2s, v16.2s
1539
1540 umull v4.2d, v25.2s, v14.2s
1541 umlal v4.2d, v26.2s, v28.2s
1542 umlal v4.2d, v12.2s, v12.2s
1543 umlal v4.2d, v24.2s, v7.2s
1544 umlal v4.2d, v8.2s, v21.2s
1545 umlal v4.2d, v23.2s, v17.2s
1546
1547 umull v5.2d, v25.2s, v15.2s
1548 umlal v5.2d, v26.2s, v14.2s
1549 umlal v5.2d, v27.2s, v13.2s
1550 umlal v5.2d, v24.2s, v16.2s
1551 umlal v5.2d, v21.2s, v9.2s
1552
1553 umull v6.2d, v25.2s, v16.2s
1554 umlal v6.2d, v26.2s, v7.2s
1555 umlal v6.2d, v27.2s, v14.2s
1556 umlal v6.2d, v28.2s, v13.2s
1557 umlal v6.2d, v24.2s, v9.2s
1558 umlal v6.2d, v21.2s, v18.2s
1559
1560 umull v8.2d, v25.2s, v18.2s
1561 umlal v8.2d, v26.2s, v9.2s
1562 umlal v8.2d, v27.2s, v16.2s
1563 umlal v8.2d, v28.2s, v7.2s
1564 umlal v8.2d, v14.2s, v14.2s
1565 umlal v8.2d, v24.2s, v19.2s
1566
1567 umull v7.2d, v25.2s, v17.2s
1568 umlal v7.2d, v26.2s, v16.2s
1569 umlal v7.2d, v27.2s, v15.2s
1570 umlal v7.2d, v28.2s, v14.2s
1571 umlal v7.2d, v24.2s, v18.2s
1572
1573 umull v9.2d, v25.2s, v19.2s
1574 umlal v9.2d, v26.2s, v18.2s
1575 umlal v9.2d, v27.2s, v17.2s
1576 umlal v9.2d, v28.2s, v16.2s
1577 umlal v9.2d, v29.2s, v15.2s
1578
1579 dup v30.2d, x29
1580 ushr v25.2d, v30.2d, #1
1581
1582 usra v6.2d, v5.2d, #25
1583 and v5.16b, v5.16b, v25.16b
1584
1585 usra v1.2d, v0.2d, #26
1586 and v0.16b, v0.16b, v30.16b
1587
1588 usra v7.2d, v6.2d, #26
1589 and v6.16b, v6.16b, v30.16b
1590
1591 usra v2.2d, v1.2d, #25
1592 and v1.16b, v1.16b, v25.16b
1593
1594 usra v8.2d, v7.2d, #25
1595 and v7.16b, v7.16b, v25.16b
1596
1597 usra v3.2d, v2.2d, #26
1598 and v2.16b, v2.16b, v30.16b
1599
1600 usra v9.2d, v8.2d, #26
1601 and v8.16b, v8.16b, v30.16b
1602
1603 usra v4.2d, v3.2d, #25
1604 and v3.16b, v3.16b, v25.16b
1605
1606 bic v10.16b, v9.16b, v25.16b
1607 usra v0.2d, v10.2d, #25
1608 usra v0.2d, v10.2d, #24
1609 usra v0.2d, v10.2d, #21
1610 and v9.16b, v9.16b, v25.16b
1611
1612 usra v5.2d, v4.2d, #26
1613 and v4.16b, v4.16b, v30.16b
1614
1615 usra v1.2d, v0.2d, #26
1616 and v0.16b, v0.16b, v30.16b
1617
1618 usra v6.2d, v5.2d, #25
1619 and v5.16b, v5.16b, v25.16b
1620
1621 xtn v10.2s, v0.2d
1622 xtn v11.2s, v1.2d
1623 xtn v12.2s, v2.2d
1624 xtn v13.2s, v3.2d
1625 xtn v14.2s, v4.2d
1626 xtn v15.2s, v5.2d
1627 xtn v16.2s, v6.2d
1628 xtn v17.2s, v7.2d
1629 xtn v18.2s, v8.2d
1630 xtn v19.2s, v9.2d
1631
1632 str q10, [x20, #480]
1633 str q11, [x20, #496]
1634 str q12, [x20, #512]
1635 str q13, [x20, #528]
1636 str q14, [x20, #544]
1637 str q15, [x20, #560]
1638 str q16, [x20, #576]
1639 str q17, [x20, #592]
1640 str q18, [x20, #608]
1641 str q19, [x20, #624]
1642
1643 // T1
1644 ldr q10, [x20, #800]
1645 ldr q11, [x20, #816]
1646 ldr q12, [x20, #832]
1647 ldr q13, [x20, #848]
1648 ldr q14, [x20, #864]
1649 ldr q15, [x20, #880]
1650 ldr q16, [x20, #896]
1651 ldr q17, [x20, #912]
1652 ldr q18, [x20, #928]
1653 ldr q19, [x20, #944]
1654
1655 // T2
1656 ldr q0, [x20, #960]
1657 ldr q1, [x20, #976]
1658 ldr q2, [x20, #992]
1659 ldr q3, [x20, #1008]
1660 ldr q4, [x20, #1024]
1661 ldr q5, [x20, #1040]
1662 ldr q6, [x20, #1056]
1663 ldr q7, [x20, #1072]
1664 ldr q8, [x20, #1088]
1665 ldr q9, [x20, #1104]
1666
1667 // T3 ← T1 - T2
1668 mov v30.d[0], x21
1669 add v20.2s, v30.2s, v10.2s
1670 mov v30.d[0], x22
1671 mov v31.d[0], x23
1672 add v21.2s, v30.2s, v11.2s
1673 add v22.2s, v31.2s, v12.2s
1674 add v23.2s, v30.2s, v13.2s
1675 add v24.2s, v31.2s, v14.2s
1676 add v25.2s, v30.2s, v15.2s
1677 add v26.2s, v31.2s, v16.2s
1678 add v27.2s, v30.2s, v17.2s
1679 add v28.2s, v31.2s, v18.2s
1680 add v29.2s, v30.2s, v19.2s
1681
1682 sub v10.2s, v20.2s, v0.2s
1683 sub v11.2s, v21.2s, v1.2s
1684 sub v12.2s, v22.2s, v2.2s
1685 sub v13.2s, v23.2s, v3.2s
1686 sub v14.2s, v24.2s, v4.2s
1687 sub v15.2s, v25.2s, v5.2s
1688 sub v16.2s, v26.2s, v6.2s
1689 sub v17.2s, v27.2s, v7.2s
1690 sub v18.2s, v28.2s, v8.2s
1691 sub v19.2s, v29.2s, v9.2s
1692
1693 str q10, [x20, #1120]
1694 str q11, [x20, #1136]
1695 str q12, [x20, #1152]
1696 str q13, [x20, #1168]
1697 str q14, [x20, #1184]
1698 str q15, [x20, #1200]
1699 str q16, [x20, #1216]
1700 str q17, [x20, #1232]
1701 str q18, [x20, #1248]
1702 str q19, [x20, #1264]
1703
1704 uxtl v0.2d, v0.2s
1705 uxtl v1.2d, v1.2s
1706 uxtl v2.2d, v2.2s
1707 uxtl v3.2d, v3.2s
1708 uxtl v4.2d, v4.2s
1709 uxtl v5.2d, v5.2s
1710 uxtl v6.2d, v6.2s
1711 uxtl v7.2d, v7.2s
1712 uxtl v8.2d, v8.2s
1713 uxtl v9.2d, v9.2s
1714
1715 // T4 ← ((A + 2)/4) · T3 + T2
1716 mov v31.d[0], x12
1717
1718 umull v20.2d, v10.2s, v31.2s
1719 add v0.2d, v0.2d, v20.2d
1720 umull v21.2d, v11.2s, v31.2s
1721 add v1.2d, v1.2d, v21.2d
1722 umull v22.2d, v12.2s, v31.2s
1723 add v2.2d, v2.2d, v22.2d
1724 umull v23.2d, v13.2s, v31.2s
1725 add v3.2d, v3.2d, v23.2d
1726 umull v24.2d, v14.2s, v31.2s
1727 add v4.2d, v4.2d, v24.2d
1728 umull v25.2d, v15.2s, v31.2s
1729 add v5.2d, v5.2d, v25.2d
1730 umull v26.2d, v16.2s, v31.2s
1731 add v6.2d, v6.2d, v26.2d
1732 umull v27.2d, v17.2s, v31.2s
1733 add v7.2d, v7.2d, v27.2d
1734 umull v28.2d, v18.2s, v31.2s
1735 add v8.2d, v8.2d, v28.2d
1736 umull v29.2d, v19.2s, v31.2s
1737 add v9.2d, v9.2d, v29.2d
1738
1739 dup v30.2d, x29
1740 ushr v25.2d, v30.2d, #1
1741
1742 usra v6.2d, v5.2d, #25
1743 and v5.16b, v5.16b, v25.16b
1744
1745 usra v1.2d, v0.2d, #26
1746 and v0.16b, v0.16b, v30.16b
1747
1748 usra v7.2d, v6.2d, #26
1749 and v6.16b, v6.16b, v30.16b
1750
1751 usra v2.2d, v1.2d, #25
1752 and v1.16b, v1.16b, v25.16b
1753
1754 usra v8.2d, v7.2d, #25
1755 and v7.16b, v7.16b, v25.16b
1756
1757 usra v3.2d, v2.2d, #26
1758 and v2.16b, v2.16b, v30.16b
1759
1760 usra v9.2d, v8.2d, #26
1761 and v8.16b, v8.16b, v30.16b
1762
1763 usra v4.2d, v3.2d, #25
1764 and v3.16b, v3.16b, v25.16b
1765
1766 bic v10.16b, v9.16b, v25.16b
1767 usra v0.2d, v10.2d, #25
1768 usra v0.2d, v10.2d, #24
1769 usra v0.2d, v10.2d, #21
1770 and v9.16b, v9.16b, v25.16b
1771
1772 usra v5.2d, v4.2d, #26
1773 and v4.16b, v4.16b, v30.16b
1774
1775 usra v1.2d, v0.2d, #26
1776 and v0.16b, v0.16b, v30.16b
1777
1778 usra v6.2d, v5.2d, #25
1779 and v5.16b, v5.16b, v25.16b
1780
1781 xtn v20.2s, v0.2d
1782 xtn v21.2s, v1.2d
1783 xtn v22.2s, v2.2d
1784 xtn v23.2s, v3.2d
1785 xtn v24.2s, v4.2d
1786 xtn v25.2s, v5.2d
1787 xtn v26.2s, v6.2d
1788 xtn v27.2s, v7.2d
1789 xtn v28.2s, v8.2d
1790 xtn v29.2s, v9.2d
1791
1792 // Z2 ← T3 · T4
1793 ldr q10, [x20, #1120]
1794 ldr q11, [x20, #1136]
1795 ldr q12, [x20, #1152]
1796 ldr q13, [x20, #1168]
1797 ldr q14, [x20, #1184]
1798 ldr q15, [x20, #1200]
1799 ldr q16, [x20, #1216]
1800 ldr q17, [x20, #1232]
1801 ldr q18, [x20, #1248]
1802 ldr q19, [x20, #1264]
1803
1804 umull v0.2d, v10.2s, v20.2s
1805
1806 umull v1.2d, v10.2s, v21.2s
1807 umlal v1.2d, v11.2s, v20.2s
1808
1809 umull v2.2d, v10.2s, v22.2s
1810 umlal v2.2d, v12.2s, v20.2s
1811
1812 umull v3.2d, v10.2s, v23.2s
1813 umlal v3.2d, v11.2s, v22.2s
1814 umlal v3.2d, v12.2s, v21.2s
1815 umlal v3.2d, v13.2s, v20.2s
1816
1817 umull v4.2d, v10.2s, v24.2s
1818 umlal v4.2d, v12.2s, v22.2s
1819 umlal v4.2d, v14.2s, v20.2s
1820
1821 umull v5.2d, v10.2s, v25.2s
1822 umlal v5.2d, v11.2s, v24.2s
1823 umlal v5.2d, v12.2s, v23.2s
1824 umlal v5.2d, v13.2s, v22.2s
1825 umlal v5.2d, v14.2s, v21.2s
1826 umlal v5.2d, v15.2s, v20.2s
1827
1828 umull v6.2d, v10.2s, v26.2s
1829 umlal v6.2d, v12.2s, v24.2s
1830 umlal v6.2d, v14.2s, v22.2s
1831 umlal v6.2d, v16.2s, v20.2s
1832
1833 umull v7.2d, v10.2s, v27.2s
1834 umlal v7.2d, v11.2s, v26.2s
1835 umlal v7.2d, v12.2s, v25.2s
1836 umlal v7.2d, v13.2s, v24.2s
1837 umlal v7.2d, v14.2s, v23.2s
1838 umlal v7.2d, v15.2s, v22.2s
1839 umlal v7.2d, v16.2s, v21.2s
1840 umlal v7.2d, v17.2s, v20.2s
1841
1842 umull v8.2d, v10.2s, v28.2s
1843 umlal v8.2d, v12.2s, v26.2s
1844 umlal v8.2d, v14.2s, v24.2s
1845 umlal v8.2d, v16.2s, v22.2s
1846 umlal v8.2d, v18.2s, v20.2s
1847
1848 umull v9.2d, v10.2s, v29.2s
1849 umlal v9.2d, v11.2s, v28.2s
1850 umlal v9.2d, v12.2s, v27.2s
1851 umlal v9.2d, v13.2s, v26.2s
1852 umlal v9.2d, v14.2s, v25.2s
1853 umlal v9.2d, v15.2s, v24.2s
1854 umlal v9.2d, v16.2s, v23.2s
1855 umlal v9.2d, v17.2s, v22.2s
1856 umlal v9.2d, v18.2s, v21.2s
1857 umlal v9.2d, v19.2s, v20.2s
1858
1859 dup v31.2s, w30
1860
1861 mul v22.2s, v22.2s, v31.2s
1862 mul v24.2s, v24.2s, v31.2s
1863 mul v26.2s, v26.2s, v31.2s
1864 mul v28.2s, v28.2s, v31.2s
1865
1866 umlal v0.2d, v12.2s, v28.2s
1867 umlal v0.2d, v14.2s, v26.2s
1868 umlal v0.2d, v16.2s, v24.2s
1869 umlal v0.2d, v18.2s, v22.2s
1870
1871 umlal v1.2d, v13.2s, v28.2s
1872 umlal v1.2d, v15.2s, v26.2s
1873 umlal v1.2d, v17.2s, v24.2s
1874 umlal v1.2d, v19.2s, v22.2s
1875
1876 umlal v2.2d, v14.2s, v28.2s
1877 umlal v2.2d, v16.2s, v26.2s
1878 umlal v2.2d, v18.2s, v24.2s
1879
1880 umlal v3.2d, v15.2s, v28.2s
1881 umlal v3.2d, v17.2s, v26.2s
1882 umlal v3.2d, v19.2s, v24.2s
1883
1884 umlal v4.2d, v16.2s, v28.2s
1885 umlal v4.2d, v18.2s, v26.2s
1886
1887 umlal v5.2d, v17.2s, v28.2s
1888 umlal v5.2d, v19.2s, v26.2s
1889
1890 umlal v6.2d, v18.2s, v28.2s
1891
1892 umlal v7.2d, v19.2s, v28.2s
1893
1894 shl v11.2s, v11.2s, #1
1895 shl v13.2s, v13.2s, #1
1896 shl v15.2s, v15.2s, #1
1897 shl v17.2s, v17.2s, #1
1898 shl v19.2s, v19.2s, #1
1899
1900 umlal v2.2d, v11.2s, v21.2s
1901
1902 umlal v4.2d, v11.2s, v23.2s
1903 umlal v4.2d, v13.2s, v21.2s
1904
1905 umlal v6.2d, v11.2s, v25.2s
1906 umlal v6.2d, v13.2s, v23.2s
1907 umlal v6.2d, v15.2s, v21.2s
1908
1909 umlal v8.2d, v11.2s, v27.2s
1910 umlal v8.2d, v13.2s, v25.2s
1911 umlal v8.2d, v15.2s, v23.2s
1912 umlal v8.2d, v17.2s, v21.2s
1913
1914 mul v21.2s, v21.2s, v31.2s
1915 mul v23.2s, v23.2s, v31.2s
1916 mul v25.2s, v25.2s, v31.2s
1917 mul v27.2s, v27.2s, v31.2s
1918 mul v29.2s, v29.2s, v31.2s
1919
1920 umlal v0.2d, v11.2s, v29.2s
1921 umlal v0.2d, v13.2s, v27.2s
1922 umlal v0.2d, v15.2s, v25.2s
1923 umlal v0.2d, v17.2s, v23.2s
1924 umlal v0.2d, v19.2s, v21.2s
1925
1926 umlal v1.2d, v12.2s, v29.2s
1927 umlal v1.2d, v14.2s, v27.2s
1928 umlal v1.2d, v16.2s, v25.2s
1929 umlal v1.2d, v18.2s, v23.2s
1930
1931 umlal v2.2d, v13.2s, v29.2s
1932 umlal v2.2d, v15.2s, v27.2s
1933 umlal v2.2d, v17.2s, v25.2s
1934 umlal v2.2d, v19.2s, v23.2s
1935
1936 umlal v3.2d, v14.2s, v29.2s
1937 umlal v3.2d, v16.2s, v27.2s
1938 umlal v3.2d, v18.2s, v25.2s
1939
1940 umlal v4.2d, v15.2s, v29.2s
1941 umlal v4.2d, v17.2s, v27.2s
1942 umlal v4.2d, v19.2s, v25.2s
1943
1944 umlal v5.2d, v16.2s, v29.2s
1945 umlal v5.2d, v18.2s, v27.2s
1946
1947 umlal v6.2d, v17.2s, v29.2s
1948 umlal v6.2d, v19.2s, v27.2s
1949
1950 umlal v7.2d, v18.2s, v29.2s
1951
1952 umlal v8.2d, v19.2s, v29.2s
1953
1954 dup v30.2d, x29
1955 ushr v25.2d, v30.2d, #1
1956
1957 usra v6.2d, v5.2d, #25
1958 and v5.16b, v5.16b, v25.16b
1959
1960 usra v1.2d, v0.2d, #26
1961 and v0.16b, v0.16b, v30.16b
1962
1963 usra v7.2d, v6.2d, #26
1964 and v6.16b, v6.16b, v30.16b
1965
1966 usra v2.2d, v1.2d, #25
1967 and v1.16b, v1.16b, v25.16b
1968
1969 usra v8.2d, v7.2d, #25
1970 and v7.16b, v7.16b, v25.16b
1971
1972 usra v3.2d, v2.2d, #26
1973 and v2.16b, v2.16b, v30.16b
1974
1975 usra v9.2d, v8.2d, #26
1976 and v8.16b, v8.16b, v30.16b
1977
1978 usra v4.2d, v3.2d, #25
1979 and v3.16b, v3.16b, v25.16b
1980
1981 bic v10.16b, v9.16b, v25.16b
1982 usra v0.2d, v10.2d, #25
1983 usra v0.2d, v10.2d, #24
1984 usra v0.2d, v10.2d, #21
1985 and v9.16b, v9.16b, v25.16b
1986
1987 usra v5.2d, v4.2d, #26
1988 and v4.16b, v4.16b, v30.16b
1989
1990 usra v1.2d, v0.2d, #26
1991 and v0.16b, v0.16b, v30.16b
1992
1993 usra v6.2d, v5.2d, #25
1994 and v5.16b, v5.16b, v25.16b
1995
1996 xtn v10.2s, v0.2d
1997 xtn v11.2s, v1.2d
1998 xtn v12.2s, v2.2d
1999 xtn v13.2s, v3.2d
2000 xtn v14.2s, v4.2d
2001 xtn v15.2s, v5.2d
2002 xtn v16.2s, v6.2d
2003 xtn v17.2s, v7.2d
2004 xtn v18.2s, v8.2d
2005 xtn v19.2s, v9.2d
2006
2007 str q10, [x20, #320]
2008 str q11, [x20, #336]
2009 str q12, [x20, #352]
2010 str q13, [x20, #368]
2011 str q14, [x20, #384]
2012 str q15, [x20, #400]
2013 str q16, [x20, #416]
2014 str q17, [x20, #432]
2015 str q18, [x20, #448]
2016 str q19, [x20, #464]
2017
2018 // X2 ← T1 · T2
2019 ldr q10, [x20, #800]
2020 ldr q11, [x20, #816]
2021 ldr q12, [x20, #832]
2022 ldr q13, [x20, #848]
2023 ldr q14, [x20, #864]
2024 ldr q15, [x20, #880]
2025 ldr q16, [x20, #896]
2026 ldr q17, [x20, #912]
2027 ldr q18, [x20, #928]
2028 ldr q19, [x20, #944]
2029
2030 ldr q20, [x20, #960]
2031 ldr q21, [x20, #976]
2032 ldr q22, [x20, #992]
2033 ldr q23, [x20, #1008]
2034 ldr q24, [x20, #1024]
2035 ldr q25, [x20, #1040]
2036 ldr q26, [x20, #1056]
2037 ldr q27, [x20, #1072]
2038 ldr q28, [x20, #1088]
2039 ldr q29, [x20, #1104]
2040
2041 umull v0.2d, v10.2s, v20.2s
2042
2043 umull v1.2d, v10.2s, v21.2s
2044 umlal v1.2d, v11.2s, v20.2s
2045
2046 umull v2.2d, v10.2s, v22.2s
2047 umlal v2.2d, v12.2s, v20.2s
2048
2049 umull v3.2d, v10.2s, v23.2s
2050 umlal v3.2d, v11.2s, v22.2s
2051 umlal v3.2d, v12.2s, v21.2s
2052 umlal v3.2d, v13.2s, v20.2s
2053
2054 umull v4.2d, v10.2s, v24.2s
2055 umlal v4.2d, v12.2s, v22.2s
2056 umlal v4.2d, v14.2s, v20.2s
2057
2058 umull v5.2d, v10.2s, v25.2s
2059 umlal v5.2d, v11.2s, v24.2s
2060 umlal v5.2d, v12.2s, v23.2s
2061 umlal v5.2d, v13.2s, v22.2s
2062 umlal v5.2d, v14.2s, v21.2s
2063 umlal v5.2d, v15.2s, v20.2s
2064
2065 umull v6.2d, v10.2s, v26.2s
2066 umlal v6.2d, v12.2s, v24.2s
2067 umlal v6.2d, v14.2s, v22.2s
2068 umlal v6.2d, v16.2s, v20.2s
2069
2070 umull v7.2d, v10.2s, v27.2s
2071 umlal v7.2d, v11.2s, v26.2s
2072 umlal v7.2d, v12.2s, v25.2s
2073 umlal v7.2d, v13.2s, v24.2s
2074 umlal v7.2d, v14.2s, v23.2s
2075 umlal v7.2d, v15.2s, v22.2s
2076 umlal v7.2d, v16.2s, v21.2s
2077 umlal v7.2d, v17.2s, v20.2s
2078
2079 umull v8.2d, v10.2s, v28.2s
2080 umlal v8.2d, v12.2s, v26.2s
2081 umlal v8.2d, v14.2s, v24.2s
2082 umlal v8.2d, v16.2s, v22.2s
2083 umlal v8.2d, v18.2s, v20.2s
2084
2085 umull v9.2d, v10.2s, v29.2s
2086 umlal v9.2d, v11.2s, v28.2s
2087 umlal v9.2d, v12.2s, v27.2s
2088 umlal v9.2d, v13.2s, v26.2s
2089 umlal v9.2d, v14.2s, v25.2s
2090 umlal v9.2d, v15.2s, v24.2s
2091 umlal v9.2d, v16.2s, v23.2s
2092 umlal v9.2d, v17.2s, v22.2s
2093 umlal v9.2d, v18.2s, v21.2s
2094 umlal v9.2d, v19.2s, v20.2s
2095
2096 dup v31.2s, w30
2097
2098 mul v22.2s, v22.2s, v31.2s
2099 mul v24.2s, v24.2s, v31.2s
2100 mul v26.2s, v26.2s, v31.2s
2101 mul v28.2s, v28.2s, v31.2s
2102
2103 umlal v0.2d, v12.2s, v28.2s
2104 umlal v0.2d, v14.2s, v26.2s
2105 umlal v0.2d, v16.2s, v24.2s
2106 umlal v0.2d, v18.2s, v22.2s
2107
2108 umlal v1.2d, v13.2s, v28.2s
2109 umlal v1.2d, v15.2s, v26.2s
2110 umlal v1.2d, v17.2s, v24.2s
2111 umlal v1.2d, v19.2s, v22.2s
2112
2113 umlal v2.2d, v14.2s, v28.2s
2114 umlal v2.2d, v16.2s, v26.2s
2115 umlal v2.2d, v18.2s, v24.2s
2116
2117 umlal v3.2d, v15.2s, v28.2s
2118 umlal v3.2d, v17.2s, v26.2s
2119 umlal v3.2d, v19.2s, v24.2s
2120
2121 umlal v4.2d, v16.2s, v28.2s
2122 umlal v4.2d, v18.2s, v26.2s
2123
2124 umlal v5.2d, v17.2s, v28.2s
2125 umlal v5.2d, v19.2s, v26.2s
2126
2127 umlal v6.2d, v18.2s, v28.2s
2128
2129 umlal v7.2d, v19.2s, v28.2s
2130
2131 shl v11.2s, v11.2s, #1
2132 shl v13.2s, v13.2s, #1
2133 shl v15.2s, v15.2s, #1
2134 shl v17.2s, v17.2s, #1
2135 shl v19.2s, v19.2s, #1
2136
2137 umlal v2.2d, v11.2s, v21.2s
2138
2139 umlal v4.2d, v11.2s, v23.2s
2140 umlal v4.2d, v13.2s, v21.2s
2141
2142 umlal v6.2d, v11.2s, v25.2s
2143 umlal v6.2d, v13.2s, v23.2s
2144 umlal v6.2d, v15.2s, v21.2s
2145
2146 umlal v8.2d, v11.2s, v27.2s
2147 umlal v8.2d, v13.2s, v25.2s
2148 umlal v8.2d, v15.2s, v23.2s
2149 umlal v8.2d, v17.2s, v21.2s
2150
2151 mul v21.2s, v21.2s, v31.2s
2152 mul v23.2s, v23.2s, v31.2s
2153 mul v25.2s, v25.2s, v31.2s
2154 mul v27.2s, v27.2s, v31.2s
2155 mul v29.2s, v29.2s, v31.2s
2156
2157 umlal v0.2d, v11.2s, v29.2s
2158 umlal v0.2d, v13.2s, v27.2s
2159 umlal v0.2d, v15.2s, v25.2s
2160 umlal v0.2d, v17.2s, v23.2s
2161 umlal v0.2d, v19.2s, v21.2s
2162
2163 umlal v1.2d, v12.2s, v29.2s
2164 umlal v1.2d, v14.2s, v27.2s
2165 umlal v1.2d, v16.2s, v25.2s
2166 umlal v1.2d, v18.2s, v23.2s
2167
2168 umlal v2.2d, v13.2s, v29.2s
2169 umlal v2.2d, v15.2s, v27.2s
2170 umlal v2.2d, v17.2s, v25.2s
2171 umlal v2.2d, v19.2s, v23.2s
2172
2173 umlal v3.2d, v14.2s, v29.2s
2174 umlal v3.2d, v16.2s, v27.2s
2175 umlal v3.2d, v18.2s, v25.2s
2176
2177 umlal v4.2d, v15.2s, v29.2s
2178 umlal v4.2d, v17.2s, v27.2s
2179 umlal v4.2d, v19.2s, v25.2s
2180
2181 umlal v5.2d, v16.2s, v29.2s
2182 umlal v5.2d, v18.2s, v27.2s
2183
2184 umlal v6.2d, v17.2s, v29.2s
2185 umlal v6.2d, v19.2s, v27.2s
2186
2187 umlal v7.2d, v18.2s, v29.2s
2188
2189 umlal v8.2d, v19.2s, v29.2s
2190
2191 dup v30.2d, x29
2192 ushr v25.2d, v30.2d, #1
2193
2194 usra v6.2d, v5.2d, #25
2195 and v5.16b, v5.16b, v25.16b
2196
2197 usra v1.2d, v0.2d, #26
2198 and v0.16b, v0.16b, v30.16b
2199
2200 usra v7.2d, v6.2d, #26
2201 and v6.16b, v6.16b, v30.16b
2202
2203 usra v2.2d, v1.2d, #25
2204 and v1.16b, v1.16b, v25.16b
2205
2206 usra v8.2d, v7.2d, #25
2207 and v7.16b, v7.16b, v25.16b
2208
2209 usra v3.2d, v2.2d, #26
2210 and v2.16b, v2.16b, v30.16b
2211
2212 usra v9.2d, v8.2d, #26
2213 and v8.16b, v8.16b, v30.16b
2214
2215 usra v4.2d, v3.2d, #25
2216 and v3.16b, v3.16b, v25.16b
2217
2218 bic v10.16b, v9.16b, v25.16b
2219 usra v0.2d, v10.2d, #25
2220 usra v0.2d, v10.2d, #24
2221 usra v0.2d, v10.2d, #21
2222 and v9.16b, v9.16b, v25.16b
2223
2224 usra v5.2d, v4.2d, #26
2225 and v4.16b, v4.16b, v30.16b
2226
2227 usra v1.2d, v0.2d, #26
2228 and v0.16b, v0.16b, v30.16b
2229
2230 usra v6.2d, v5.2d, #25
2231 and v5.16b, v5.16b, v25.16b
2232
2233 xtn v10.2s, v0.2d
2234 xtn v11.2s, v1.2d
2235 xtn v12.2s, v2.2d
2236 xtn v13.2s, v3.2d
2237 xtn v14.2s, v4.2d
2238 xtn v15.2s, v5.2d
2239 xtn v16.2s, v6.2d
2240 xtn v17.2s, v7.2d
2241 xtn v18.2s, v8.2d
2242 xtn v19.2s, v9.2d
2243
2244 str q10, [x20, #160]
2245 str q11, [x20, #176]
2246 str q12, [x20, #192]
2247 str q13, [x20, #208]
2248 str q14, [x20, #224]
2249 str q15, [x20, #240]
2250 str q16, [x20, #256]
2251 str q17, [x20, #272]
2252 str q18, [x20, #288]
2253 str q19, [x20, #304]
2254
2255 // Z3 ← Z3 · X1
2256 ldr q10, [x20, #640]
2257 ldr q11, [x20, #656]
2258 ldr q12, [x20, #672]
2259 ldr q13, [x20, #688]
2260 ldr q14, [x20, #704]
2261 ldr q15, [x20, #720]
2262 ldr q16, [x20, #736]
2263 ldr q17, [x20, #752]
2264 ldr q18, [x20, #768]
2265 ldr q19, [x20, #784]
2266
2267 ldr q20, [x20, #0]
2268 ldr q21, [x20, #16]
2269 ldr q22, [x20, #32]
2270 ldr q23, [x20, #48]
2271 ldr q24, [x20, #64]
2272 ldr q25, [x20, #80]
2273 ldr q26, [x20, #96]
2274 ldr q27, [x20, #112]
2275 ldr q28, [x20, #128]
2276 ldr q29, [x20, #144]
2277
2278 umull v0.2d, v10.2s, v20.2s
2279
2280 umull v1.2d, v10.2s, v21.2s
2281 umlal v1.2d, v11.2s, v20.2s
2282
2283 umull v2.2d, v10.2s, v22.2s
2284 umlal v2.2d, v12.2s, v20.2s
2285
2286 umull v3.2d, v10.2s, v23.2s
2287 umlal v3.2d, v11.2s, v22.2s
2288 umlal v3.2d, v12.2s, v21.2s
2289 umlal v3.2d, v13.2s, v20.2s
2290
2291 umull v4.2d, v10.2s, v24.2s
2292 umlal v4.2d, v12.2s, v22.2s
2293 umlal v4.2d, v14.2s, v20.2s
2294
2295 umull v5.2d, v10.2s, v25.2s
2296 umlal v5.2d, v11.2s, v24.2s
2297 umlal v5.2d, v12.2s, v23.2s
2298 umlal v5.2d, v13.2s, v22.2s
2299 umlal v5.2d, v14.2s, v21.2s
2300 umlal v5.2d, v15.2s, v20.2s
2301
2302 umull v6.2d, v10.2s, v26.2s
2303 umlal v6.2d, v12.2s, v24.2s
2304 umlal v6.2d, v14.2s, v22.2s
2305 umlal v6.2d, v16.2s, v20.2s
2306
2307 umull v7.2d, v10.2s, v27.2s
2308 umlal v7.2d, v11.2s, v26.2s
2309 umlal v7.2d, v12.2s, v25.2s
2310 umlal v7.2d, v13.2s, v24.2s
2311 umlal v7.2d, v14.2s, v23.2s
2312 umlal v7.2d, v15.2s, v22.2s
2313 umlal v7.2d, v16.2s, v21.2s
2314 umlal v7.2d, v17.2s, v20.2s
2315
2316 umull v8.2d, v10.2s, v28.2s
2317 umlal v8.2d, v12.2s, v26.2s
2318 umlal v8.2d, v14.2s, v24.2s
2319 umlal v8.2d, v16.2s, v22.2s
2320 umlal v8.2d, v18.2s, v20.2s
2321
2322 umull v9.2d, v10.2s, v29.2s
2323 umlal v9.2d, v11.2s, v28.2s
2324 umlal v9.2d, v12.2s, v27.2s
2325 umlal v9.2d, v13.2s, v26.2s
2326 umlal v9.2d, v14.2s, v25.2s
2327 umlal v9.2d, v15.2s, v24.2s
2328 umlal v9.2d, v16.2s, v23.2s
2329 umlal v9.2d, v17.2s, v22.2s
2330 umlal v9.2d, v18.2s, v21.2s
2331 umlal v9.2d, v19.2s, v20.2s
2332
2333 dup v31.2s, w30
2334
2335 mul v22.2s, v22.2s, v31.2s
2336 mul v24.2s, v24.2s, v31.2s
2337 mul v26.2s, v26.2s, v31.2s
2338 mul v28.2s, v28.2s, v31.2s
2339
2340 umlal v0.2d, v12.2s, v28.2s
2341 umlal v0.2d, v14.2s, v26.2s
2342 umlal v0.2d, v16.2s, v24.2s
2343 umlal v0.2d, v18.2s, v22.2s
2344
2345 umlal v1.2d, v13.2s, v28.2s
2346 umlal v1.2d, v15.2s, v26.2s
2347 umlal v1.2d, v17.2s, v24.2s
2348 umlal v1.2d, v19.2s, v22.2s
2349
2350 umlal v2.2d, v14.2s, v28.2s
2351 umlal v2.2d, v16.2s, v26.2s
2352 umlal v2.2d, v18.2s, v24.2s
2353
2354 umlal v3.2d, v15.2s, v28.2s
2355 umlal v3.2d, v17.2s, v26.2s
2356 umlal v3.2d, v19.2s, v24.2s
2357
2358 umlal v4.2d, v16.2s, v28.2s
2359 umlal v4.2d, v18.2s, v26.2s
2360
2361 umlal v5.2d, v17.2s, v28.2s
2362 umlal v5.2d, v19.2s, v26.2s
2363
2364 umlal v6.2d, v18.2s, v28.2s
2365
2366 umlal v7.2d, v19.2s, v28.2s
2367
2368 shl v11.2s, v11.2s, #1
2369 shl v13.2s, v13.2s, #1
2370 shl v15.2s, v15.2s, #1
2371 shl v17.2s, v17.2s, #1
2372 shl v19.2s, v19.2s, #1
2373
2374 umlal v2.2d, v11.2s, v21.2s
2375
2376 umlal v4.2d, v11.2s, v23.2s
2377 umlal v4.2d, v13.2s, v21.2s
2378
2379 umlal v6.2d, v11.2s, v25.2s
2380 umlal v6.2d, v13.2s, v23.2s
2381 umlal v6.2d, v15.2s, v21.2s
2382
2383 umlal v8.2d, v11.2s, v27.2s
2384 umlal v8.2d, v13.2s, v25.2s
2385 umlal v8.2d, v15.2s, v23.2s
2386 umlal v8.2d, v17.2s, v21.2s
2387
2388 mul v21.2s, v21.2s, v31.2s
2389 mul v23.2s, v23.2s, v31.2s
2390 mul v25.2s, v25.2s, v31.2s
2391 mul v27.2s, v27.2s, v31.2s
2392 mul v29.2s, v29.2s, v31.2s
2393
2394 umlal v0.2d, v11.2s, v29.2s
2395 umlal v0.2d, v13.2s, v27.2s
2396 umlal v0.2d, v15.2s, v25.2s
2397 umlal v0.2d, v17.2s, v23.2s
2398 umlal v0.2d, v19.2s, v21.2s
2399
2400 umlal v1.2d, v12.2s, v29.2s
2401 umlal v1.2d, v14.2s, v27.2s
2402 umlal v1.2d, v16.2s, v25.2s
2403 umlal v1.2d, v18.2s, v23.2s
2404
2405 umlal v2.2d, v13.2s, v29.2s
2406 umlal v2.2d, v15.2s, v27.2s
2407 umlal v2.2d, v17.2s, v25.2s
2408 umlal v2.2d, v19.2s, v23.2s
2409
2410 umlal v3.2d, v14.2s, v29.2s
2411 umlal v3.2d, v16.2s, v27.2s
2412 umlal v3.2d, v18.2s, v25.2s
2413
2414 umlal v4.2d, v15.2s, v29.2s
2415 umlal v4.2d, v17.2s, v27.2s
2416 umlal v4.2d, v19.2s, v25.2s
2417
2418 umlal v5.2d, v16.2s, v29.2s
2419 umlal v5.2d, v18.2s, v27.2s
2420
2421 umlal v6.2d, v17.2s, v29.2s
2422 umlal v6.2d, v19.2s, v27.2s
2423
2424 umlal v7.2d, v18.2s, v29.2s
2425
2426 umlal v8.2d, v19.2s, v29.2s
2427
2428 dup v30.2d, x29
2429 ushr v25.2d, v30.2d, #1
2430
2431 usra v6.2d, v5.2d, #25
2432 and v5.16b, v5.16b, v25.16b
2433
2434 usra v1.2d, v0.2d, #26
2435 and v0.16b, v0.16b, v30.16b
2436
2437 usra v7.2d, v6.2d, #26
2438 and v6.16b, v6.16b, v30.16b
2439
2440 usra v2.2d, v1.2d, #25
2441 and v1.16b, v1.16b, v25.16b
2442
2443 usra v8.2d, v7.2d, #25
2444 and v7.16b, v7.16b, v25.16b
2445
2446 usra v3.2d, v2.2d, #26
2447 and v2.16b, v2.16b, v30.16b
2448
2449 usra v9.2d, v8.2d, #26
2450 and v8.16b, v8.16b, v30.16b
2451
2452 usra v4.2d, v3.2d, #25
2453 and v3.16b, v3.16b, v25.16b
2454
2455 bic v10.16b, v9.16b, v25.16b
2456 usra v0.2d, v10.2d, #25
2457 usra v0.2d, v10.2d, #24
2458 usra v0.2d, v10.2d, #21
2459 and v9.16b, v9.16b, v25.16b
2460
2461 usra v5.2d, v4.2d, #26
2462 and v4.16b, v4.16b, v30.16b
2463
2464 usra v1.2d, v0.2d, #26
2465 and v0.16b, v0.16b, v30.16b
2466
2467 usra v6.2d, v5.2d, #25
2468 and v5.16b, v5.16b, v25.16b
2469
2470 xtn v10.2s, v0.2d
2471 xtn v11.2s, v1.2d
2472 xtn v12.2s, v2.2d
2473 xtn v13.2s, v3.2d
2474 xtn v14.2s, v4.2d
2475 xtn v15.2s, v5.2d
2476 xtn v16.2s, v6.2d
2477 xtn v17.2s, v7.2d
2478 xtn v18.2s, v8.2d
2479 xtn v19.2s, v9.2d
2480
2481 str q10, [x20, #640]
2482 str q11, [x20, #656]
2483 str q12, [x20, #672]
2484 str q13, [x20, #688]
2485 str q14, [x20, #704]
2486 str q15, [x20, #720]
2487 str q16, [x20, #736]
2488 str q17, [x20, #752]
2489 str q18, [x20, #768]
2490 str q19, [x20, #784]
2491
2492 sub x6, x6, #1
2493 cmp x6, #0
2494 bge .L2
2495
2496 mov x6, #63
2497 sub x5, x5, #16
2498 cmp x5, #0
2499 bge .L1
2500
2501 // reduce X2
2502 ldr q10, [x20, #160]
2503 ldr q11, [x20, #176]
2504 ldr q12, [x20, #192]
2505 ldr q13, [x20, #208]
2506 ldr q14, [x20, #224]
2507 ldr q15, [x20, #240]
2508 ldr q16, [x20, #256]
2509 ldr q17, [x20, #272]
2510 ldr q18, [x20, #288]
2511 ldr q19, [x20, #304]
2512
2513 uxtl v0.2d, v10.2s
2514 uxtl v1.2d, v11.2s
2515 uxtl v2.2d, v12.2s
2516 uxtl v3.2d, v13.2s
2517 uxtl v4.2d, v14.2s
2518 uxtl v5.2d, v15.2s
2519 uxtl v6.2d, v16.2s
2520 uxtl v7.2d, v17.2s
2521 uxtl v8.2d, v18.2s
2522 uxtl v9.2d, v19.2s
2523
2524 usra v1.2d, v0.2d, #26
2525 and v0.16b, v0.16b, v30.16b
2526
2527 usra v2.2d, v1.2d, #25
2528 and v1.16b, v1.16b, v25.16b
2529
2530 usra v3.2d, v2.2d, #26
2531 and v2.16b, v2.16b, v30.16b
2532
2533 usra v4.2d, v3.2d, #25
2534 and v3.16b, v3.16b, v25.16b
2535
2536 usra v5.2d, v4.2d, #26
2537 and v4.16b, v4.16b, v30.16b
2538
2539 usra v6.2d, v5.2d, #25
2540 and v5.16b, v5.16b, v25.16b
2541
2542 usra v7.2d, v6.2d, #26
2543 and v6.16b, v6.16b, v30.16b
2544
2545 usra v8.2d, v7.2d, #25
2546 and v7.16b, v7.16b, v25.16b
2547
2548 usra v9.2d, v8.2d, #26
2549 and v8.16b, v8.16b, v30.16b
2550
2551 bic v10.16b, v9.16b, v25.16b
2552 usra v0.2d, v10.2d, #25
2553 usra v0.2d, v10.2d, #24
2554 usra v0.2d, v10.2d, #21
2555 and v9.16b, v9.16b, v25.16b
2556
2557 usra v1.2d, v0.2d, #26
2558 and v0.16b, v0.16b, v30.16b
2559
2560 // store X2
2561 str q0, [x0, #0]
2562 str q1, [x0, #16]
2563 str q2, [x0, #32]
2564 str q3, [x0, #48]
2565 str q4, [x0, #64]
2566 str q5, [x0, #80]
2567 str q6, [x0, #96]
2568 str q7, [x0, #112]
2569 str q8, [x0, #128]
2570 str q9, [x0, #144]
2571
2572 // reduce Z2
2573 ldr q10, [x20, #320]
2574 ldr q11, [x20, #336]
2575 ldr q12, [x20, #352]
2576 ldr q13, [x20, #368]
2577 ldr q14, [x20, #384]
2578 ldr q15, [x20, #400]
2579 ldr q16, [x20, #416]
2580 ldr q17, [x20, #432]
2581 ldr q18, [x20, #448]
2582 ldr q19, [x20, #464]
2583
2584 uxtl v0.2d, v10.2s
2585 uxtl v1.2d, v11.2s
2586 uxtl v2.2d, v12.2s
2587 uxtl v3.2d, v13.2s
2588 uxtl v4.2d, v14.2s
2589 uxtl v5.2d, v15.2s
2590 uxtl v6.2d, v16.2s
2591 uxtl v7.2d, v17.2s
2592 uxtl v8.2d, v18.2s
2593 uxtl v9.2d, v19.2s
2594
2595 usra v1.2d, v0.2d, #26
2596 and v0.16b, v0.16b, v30.16b
2597
2598 usra v2.2d, v1.2d, #25
2599 and v1.16b, v1.16b, v25.16b
2600
2601 usra v3.2d, v2.2d, #26
2602 and v2.16b, v2.16b, v30.16b
2603
2604 usra v4.2d, v3.2d, #25
2605 and v3.16b, v3.16b, v25.16b
2606
2607 usra v5.2d, v4.2d, #26
2608 and v4.16b, v4.16b, v30.16b
2609
2610 usra v6.2d, v5.2d, #25
2611 and v5.16b, v5.16b, v25.16b
2612
2613 usra v7.2d, v6.2d, #26
2614 and v6.16b, v6.16b, v30.16b
2615
2616 usra v8.2d, v7.2d, #25
2617 and v7.16b, v7.16b, v25.16b
2618
2619 usra v9.2d, v8.2d, #26
2620 and v8.16b, v8.16b, v30.16b
2621
2622 bic v10.16b, v9.16b, v25.16b
2623 usra v0.2d, v10.2d, #25
2624 usra v0.2d, v10.2d, #24
2625 usra v0.2d, v10.2d, #21
2626 and v9.16b, v9.16b, v25.16b
2627
2628 usra v1.2d, v0.2d, #26
2629 and v0.16b, v0.16b, v30.16b
2630
2631 // store Z2
2632 str q0, [x0, #160]
2633 str q1, [x0, #176]
2634 str q2, [x0, #192]
2635 str q3, [x0, #208]
2636 str q4, [x0, #224]
2637 str q5, [x0, #240]
2638 str q6, [x0, #256]
2639 str q7, [x0, #272]
2640 str q8, [x0, #288]
2641 str q9, [x0, #304]
2642
2643 ldp d14, d15, [sp, #144]
2644 ldp d12, d13, [sp, #128]
2645 ldp d10, d11, [sp, #112]
2646 ldp d8, d9, [sp, #96]
2647 ldp x29, x30, [sp, #80]
2648 ldp x27, x28, [sp, #64]
2649 ldp x25, x26, [sp, #48]
2650 ldp x23, x24, [sp, #32]
2651 ldp x21, x22, [sp, #16]
2652 ldp x19, x20, [sp, #0]
2653 add sp, sp, #1632
2654
2655 ret
2656 .section .note.GNU-stack,"",@progbits