Debian Codegraph @

win-x86_64/crypto/cipher_extra/chacha20_poly1305_x86_64-win.asm

raw
1 ; This file is generated from a similarly-named Perl script in the BoringSSL
2 ; source tree. Do not edit by hand.
3
4 %ifidn __OUTPUT_FORMAT__, win64
5 default rel
6 %define XMMWORD
7 %define YMMWORD
8 %define ZMMWORD
9
10 %ifdef BORINGSSL_PREFIX
11 %include "boringssl_prefix_symbols_nasm.inc"
12 %endif
13 section .text code align=64
14
15 EXTERN OPENSSL_ia32cap_P
16
17 chacha20_poly1305_constants:
18
19 section .rdata rdata align=8
20 ALIGN 64
21 $L$chacha20_consts:
22 DB 'e','x','p','a','n','d',' ','3','2','-','b','y','t','e',' ','k'
23 DB 'e','x','p','a','n','d',' ','3','2','-','b','y','t','e',' ','k'
24 $L$rol8:
25 DB 3,0,1,2,7,4,5,6,11,8,9,10,15,12,13,14
26 DB 3,0,1,2,7,4,5,6,11,8,9,10,15,12,13,14
27 $L$rol16:
28 DB 2,3,0,1,6,7,4,5,10,11,8,9,14,15,12,13
29 DB 2,3,0,1,6,7,4,5,10,11,8,9,14,15,12,13
30 $L$avx2_init:
31 DD 0,0,0,0
32 $L$sse_inc:
33 DD 1,0,0,0
34 $L$avx2_inc:
35 DD 2,0,0,0,2,0,0,0
36 $L$clamp:
37 DQ 0x0FFFFFFC0FFFFFFF,0x0FFFFFFC0FFFFFFC
38 DQ 0xFFFFFFFFFFFFFFFF,0xFFFFFFFFFFFFFFFF
39 ALIGN 16
40 $L$and_masks:
41 DB 0xff,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00
42 DB 0xff,0xff,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00
43 DB 0xff,0xff,0xff,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00
44 DB 0xff,0xff,0xff,0xff,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00
45 DB 0xff,0xff,0xff,0xff,0xff,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00
46 DB 0xff,0xff,0xff,0xff,0xff,0xff,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00
47 DB 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00
48 DB 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00
49 DB 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0x00,0x00,0x00,0x00,0x00,0x00,0x00
50 DB 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0x00,0x00,0x00,0x00,0x00,0x00
51 DB 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0x00,0x00,0x00,0x00,0x00
52 DB 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0x00,0x00,0x00,0x00
53 DB 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0x00,0x00,0x00
54 DB 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0x00,0x00
55 DB 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0x00
56 DB 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff
57 section .text
58
59
60
61 ALIGN 64
62 poly_hash_ad_internal:
63
64
65 xor r10,r10
66 xor r11,r11
67 xor r12,r12
68 cmp r8,13
69 jne NEAR $L$hash_ad_loop
70 $L$poly_fast_tls_ad:
71
72 mov r10,QWORD[rcx]
73 mov r11,QWORD[5+rcx]
74 shr r11,24
75 mov r12,1
76 mov rax,QWORD[((0+160+0))+rbp]
77 mov r15,rax
78 mul r10
79 mov r13,rax
80 mov r14,rdx
81 mov rax,QWORD[((0+160+0))+rbp]
82 mul r11
83 imul r15,r12
84 add r14,rax
85 adc r15,rdx
86 mov rax,QWORD[((8+160+0))+rbp]
87 mov r9,rax
88 mul r10
89 add r14,rax
90 adc rdx,0
91 mov r10,rdx
92 mov rax,QWORD[((8+160+0))+rbp]
93 mul r11
94 add r15,rax
95 adc rdx,0
96 imul r9,r12
97 add r15,r10
98 adc r9,rdx
99 mov r10,r13
100 mov r11,r14
101 mov r12,r15
102 and r12,3
103 mov r13,r15
104 and r13,-4
105 mov r14,r9
106 shrd r15,r9,2
107 shr r9,2
108 add r15,r13
109 adc r9,r14
110 add r10,r15
111 adc r11,r9
112 adc r12,0
113
114 DB 0F3h,0C3h ;repret
115 $L$hash_ad_loop:
116
117 cmp r8,16
118 jb NEAR $L$hash_ad_tail
119 add r10,QWORD[((0+0))+rcx]
120 adc r11,QWORD[((8+0))+rcx]
121 adc r12,1
122 mov rax,QWORD[((0+160+0))+rbp]
123 mov r15,rax
124 mul r10
125 mov r13,rax
126 mov r14,rdx
127 mov rax,QWORD[((0+160+0))+rbp]
128 mul r11
129 imul r15,r12
130 add r14,rax
131 adc r15,rdx
132 mov rax,QWORD[((8+160+0))+rbp]
133 mov r9,rax
134 mul r10
135 add r14,rax
136 adc rdx,0
137 mov r10,rdx
138 mov rax,QWORD[((8+160+0))+rbp]
139 mul r11
140 add r15,rax
141 adc rdx,0
142 imul r9,r12
143 add r15,r10
144 adc r9,rdx
145 mov r10,r13
146 mov r11,r14
147 mov r12,r15
148 and r12,3
149 mov r13,r15
150 and r13,-4
151 mov r14,r9
152 shrd r15,r9,2
153 shr r9,2
154 add r15,r13
155 adc r9,r14
156 add r10,r15
157 adc r11,r9
158 adc r12,0
159
160 lea rcx,[16+rcx]
161 sub r8,16
162 jmp NEAR $L$hash_ad_loop
163 $L$hash_ad_tail:
164 cmp r8,0
165 je NEAR $L$hash_ad_done
166
167 xor r13,r13
168 xor r14,r14
169 xor r15,r15
170 add rcx,r8
171 $L$hash_ad_tail_loop:
172 shld r14,r13,8
173 shl r13,8
174 movzx r15,BYTE[((-1))+rcx]
175 xor r13,r15
176 dec rcx
177 dec r8
178 jne NEAR $L$hash_ad_tail_loop
179
180 add r10,r13
181 adc r11,r14
182 adc r12,1
183 mov rax,QWORD[((0+160+0))+rbp]
184 mov r15,rax
185 mul r10
186 mov r13,rax
187 mov r14,rdx
188 mov rax,QWORD[((0+160+0))+rbp]
189 mul r11
190 imul r15,r12
191 add r14,rax
192 adc r15,rdx
193 mov rax,QWORD[((8+160+0))+rbp]
194 mov r9,rax
195 mul r10
196 add r14,rax
197 adc rdx,0
198 mov r10,rdx
199 mov rax,QWORD[((8+160+0))+rbp]
200 mul r11
201 add r15,rax
202 adc rdx,0
203 imul r9,r12
204 add r15,r10
205 adc r9,rdx
206 mov r10,r13
207 mov r11,r14
208 mov r12,r15
209 and r12,3
210 mov r13,r15
211 and r13,-4
212 mov r14,r9
213 shrd r15,r9,2
214 shr r9,2
215 add r15,r13
216 adc r9,r14
217 add r10,r15
218 adc r11,r9
219 adc r12,0
220
221
222 $L$hash_ad_done:
223 DB 0F3h,0C3h ;repret
224
225
226
227 global chacha20_poly1305_open
228
229 ALIGN 64
230 chacha20_poly1305_open:
231 mov QWORD[8+rsp],rdi ;WIN64 prologue
232 mov QWORD[16+rsp],rsi
233 mov rax,rsp
234 $L$SEH_begin_chacha20_poly1305_open:
235 mov rdi,rcx
236 mov rsi,rdx
237 mov rdx,r8
238 mov rcx,r9
239 mov r8,QWORD[40+rsp]
240 mov r9,QWORD[48+rsp]
241
242
243
244 push rbp
245
246 push rbx
247
248 push r12
249
250 push r13
251
252 push r14
253
254 push r15
255
256
257
258 push r9
259
260 sub rsp,288 + 160 + 32
261
262
263 lea rbp,[32+rsp]
264 and rbp,-32
265
266 movaps XMMWORD[(0+0)+rbp],xmm6
267 movaps XMMWORD[(16+0)+rbp],xmm7
268 movaps XMMWORD[(32+0)+rbp],xmm8
269 movaps XMMWORD[(48+0)+rbp],xmm9
270 movaps XMMWORD[(64+0)+rbp],xmm10
271 movaps XMMWORD[(80+0)+rbp],xmm11
272 movaps XMMWORD[(96+0)+rbp],xmm12
273 movaps XMMWORD[(112+0)+rbp],xmm13
274 movaps XMMWORD[(128+0)+rbp],xmm14
275 movaps XMMWORD[(144+0)+rbp],xmm15
276
277 mov rbx,rdx
278 mov QWORD[((0+160+32))+rbp],r8
279 mov QWORD[((8+160+32))+rbp],rbx
280
281 mov eax,DWORD[((OPENSSL_ia32cap_P+8))]
282 and eax,288
283 xor eax,288
284 jz NEAR chacha20_poly1305_open_avx2
285
286 cmp rbx,128
287 jbe NEAR $L$open_sse_128
288
289 movdqa xmm0,XMMWORD[$L$chacha20_consts]
290 movdqu xmm4,XMMWORD[r9]
291 movdqu xmm8,XMMWORD[16+r9]
292 movdqu xmm12,XMMWORD[32+r9]
293
294 movdqa xmm7,xmm12
295
296 movdqa XMMWORD[(160+48)+rbp],xmm4
297 movdqa XMMWORD[(160+64)+rbp],xmm8
298 movdqa XMMWORD[(160+96)+rbp],xmm12
299 mov r10,10
300 $L$open_sse_init_rounds:
301 paddd xmm0,xmm4
302 pxor xmm12,xmm0
303 pshufb xmm12,XMMWORD[$L$rol16]
304 paddd xmm8,xmm12
305 pxor xmm4,xmm8
306 movdqa xmm3,xmm4
307 pslld xmm3,12
308 psrld xmm4,20
309 pxor xmm4,xmm3
310 paddd xmm0,xmm4
311 pxor xmm12,xmm0
312 pshufb xmm12,XMMWORD[$L$rol8]
313 paddd xmm8,xmm12
314 pxor xmm4,xmm8
315 movdqa xmm3,xmm4
316 pslld xmm3,7
317 psrld xmm4,25
318 pxor xmm4,xmm3
319 DB 102,15,58,15,228,4
320 DB 102,69,15,58,15,192,8
321 DB 102,69,15,58,15,228,12
322 paddd xmm0,xmm4
323 pxor xmm12,xmm0
324 pshufb xmm12,XMMWORD[$L$rol16]
325 paddd xmm8,xmm12
326 pxor xmm4,xmm8
327 movdqa xmm3,xmm4
328 pslld xmm3,12
329 psrld xmm4,20
330 pxor xmm4,xmm3
331 paddd xmm0,xmm4
332 pxor xmm12,xmm0
333 pshufb xmm12,XMMWORD[$L$rol8]
334 paddd xmm8,xmm12
335 pxor xmm4,xmm8
336 movdqa xmm3,xmm4
337 pslld xmm3,7
338 psrld xmm4,25
339 pxor xmm4,xmm3
340 DB 102,15,58,15,228,12
341 DB 102,69,15,58,15,192,8
342 DB 102,69,15,58,15,228,4
343
344 dec r10
345 jne NEAR $L$open_sse_init_rounds
346
347 paddd xmm0,XMMWORD[$L$chacha20_consts]
348 paddd xmm4,XMMWORD[((160+48))+rbp]
349
350 pand xmm0,XMMWORD[$L$clamp]
351 movdqa XMMWORD[(160+0)+rbp],xmm0
352 movdqa XMMWORD[(160+16)+rbp],xmm4
353
354 mov r8,r8
355 call poly_hash_ad_internal
356 $L$open_sse_main_loop:
357 cmp rbx,16*16
358 jb NEAR $L$open_sse_tail
359
360 movdqa xmm0,XMMWORD[$L$chacha20_consts]
361 movdqa xmm4,XMMWORD[((160+48))+rbp]
362 movdqa xmm8,XMMWORD[((160+64))+rbp]
363 movdqa xmm1,xmm0
364 movdqa xmm5,xmm4
365 movdqa xmm9,xmm8
366 movdqa xmm2,xmm0
367 movdqa xmm6,xmm4
368 movdqa xmm10,xmm8
369 movdqa xmm3,xmm0
370 movdqa xmm7,xmm4
371 movdqa xmm11,xmm8
372 movdqa xmm15,XMMWORD[((160+96))+rbp]
373 paddd xmm15,XMMWORD[$L$sse_inc]
374 movdqa xmm14,xmm15
375 paddd xmm14,XMMWORD[$L$sse_inc]
376 movdqa xmm13,xmm14
377 paddd xmm13,XMMWORD[$L$sse_inc]
378 movdqa xmm12,xmm13
379 paddd xmm12,XMMWORD[$L$sse_inc]
380 movdqa XMMWORD[(160+96)+rbp],xmm12
381 movdqa XMMWORD[(160+112)+rbp],xmm13
382 movdqa XMMWORD[(160+128)+rbp],xmm14
383 movdqa XMMWORD[(160+144)+rbp],xmm15
384
385
386
387 mov rcx,4
388 mov r8,rsi
389 $L$open_sse_main_loop_rounds:
390 movdqa XMMWORD[(160+80)+rbp],xmm8
391 movdqa xmm8,XMMWORD[$L$rol16]
392 paddd xmm3,xmm7
393 paddd xmm2,xmm6
394 paddd xmm1,xmm5
395 paddd xmm0,xmm4
396 pxor xmm15,xmm3
397 pxor xmm14,xmm2
398 pxor xmm13,xmm1
399 pxor xmm12,xmm0
400 DB 102,69,15,56,0,248
401 DB 102,69,15,56,0,240
402 DB 102,69,15,56,0,232
403 DB 102,69,15,56,0,224
404 movdqa xmm8,XMMWORD[((160+80))+rbp]
405 paddd xmm11,xmm15
406 paddd xmm10,xmm14
407 paddd xmm9,xmm13
408 paddd xmm8,xmm12
409 pxor xmm7,xmm11
410 add r10,QWORD[((0+0))+r8]
411 adc r11,QWORD[((8+0))+r8]
412 adc r12,1
413
414 lea r8,[16+r8]
415 pxor xmm6,xmm10
416 pxor xmm5,xmm9
417 pxor xmm4,xmm8
418 movdqa XMMWORD[(160+80)+rbp],xmm8
419 movdqa xmm8,xmm7
420 psrld xmm8,20
421 pslld xmm7,32-20
422 pxor xmm7,xmm8
423 movdqa xmm8,xmm6
424 psrld xmm8,20
425 pslld xmm6,32-20
426 pxor xmm6,xmm8
427 movdqa xmm8,xmm5
428 psrld xmm8,20
429 pslld xmm5,32-20
430 pxor xmm5,xmm8
431 movdqa xmm8,xmm4
432 psrld xmm8,20
433 pslld xmm4,32-20
434 pxor xmm4,xmm8
435 mov rax,QWORD[((0+160+0))+rbp]
436 mov r15,rax
437 mul r10
438 mov r13,rax
439 mov r14,rdx
440 mov rax,QWORD[((0+160+0))+rbp]
441 mul r11
442 imul r15,r12
443 add r14,rax
444 adc r15,rdx
445 movdqa xmm8,XMMWORD[$L$rol8]
446 paddd xmm3,xmm7
447 paddd xmm2,xmm6
448 paddd xmm1,xmm5
449 paddd xmm0,xmm4
450 pxor xmm15,xmm3
451 pxor xmm14,xmm2
452 pxor xmm13,xmm1
453 pxor xmm12,xmm0
454 DB 102,69,15,56,0,248
455 DB 102,69,15,56,0,240
456 DB 102,69,15,56,0,232
457 DB 102,69,15,56,0,224
458 movdqa xmm8,XMMWORD[((160+80))+rbp]
459 paddd xmm11,xmm15
460 paddd xmm10,xmm14
461 paddd xmm9,xmm13
462 paddd xmm8,xmm12
463 pxor xmm7,xmm11
464 pxor xmm6,xmm10
465 mov rax,QWORD[((8+160+0))+rbp]
466 mov r9,rax
467 mul r10
468 add r14,rax
469 adc rdx,0
470 mov r10,rdx
471 mov rax,QWORD[((8+160+0))+rbp]
472 mul r11
473 add r15,rax
474 adc rdx,0
475 pxor xmm5,xmm9
476 pxor xmm4,xmm8
477 movdqa XMMWORD[(160+80)+rbp],xmm8
478 movdqa xmm8,xmm7
479 psrld xmm8,25
480 pslld xmm7,32-25
481 pxor xmm7,xmm8
482 movdqa xmm8,xmm6
483 psrld xmm8,25
484 pslld xmm6,32-25
485 pxor xmm6,xmm8
486 movdqa xmm8,xmm5
487 psrld xmm8,25
488 pslld xmm5,32-25
489 pxor xmm5,xmm8
490 movdqa xmm8,xmm4
491 psrld xmm8,25
492 pslld xmm4,32-25
493 pxor xmm4,xmm8
494 movdqa xmm8,XMMWORD[((160+80))+rbp]
495 imul r9,r12
496 add r15,r10
497 adc r9,rdx
498 DB 102,15,58,15,255,4
499 DB 102,69,15,58,15,219,8
500 DB 102,69,15,58,15,255,12
501 DB 102,15,58,15,246,4
502 DB 102,69,15,58,15,210,8
503 DB 102,69,15,58,15,246,12
504 DB 102,15,58,15,237,4
505 DB 102,69,15,58,15,201,8
506 DB 102,69,15,58,15,237,12
507 DB 102,15,58,15,228,4
508 DB 102,69,15,58,15,192,8
509 DB 102,69,15,58,15,228,12
510 movdqa XMMWORD[(160+80)+rbp],xmm8
511 movdqa xmm8,XMMWORD[$L$rol16]
512 paddd xmm3,xmm7
513 paddd xmm2,xmm6
514 paddd xmm1,xmm5
515 paddd xmm0,xmm4
516 pxor xmm15,xmm3
517 pxor xmm14,xmm2
518 mov r10,r13
519 mov r11,r14
520 mov r12,r15
521 and r12,3
522 mov r13,r15
523 and r13,-4
524 mov r14,r9
525 shrd r15,r9,2
526 shr r9,2
527 add r15,r13
528 adc r9,r14
529 add r10,r15
530 adc r11,r9
531 adc r12,0
532 pxor xmm13,xmm1
533 pxor xmm12,xmm0
534 DB 102,69,15,56,0,248
535 DB 102,69,15,56,0,240
536 DB 102,69,15,56,0,232
537 DB 102,69,15,56,0,224
538 movdqa xmm8,XMMWORD[((160+80))+rbp]
539 paddd xmm11,xmm15
540 paddd xmm10,xmm14
541 paddd xmm9,xmm13
542 paddd xmm8,xmm12
543 pxor xmm7,xmm11
544 pxor xmm6,xmm10
545 pxor xmm5,xmm9
546 pxor xmm4,xmm8
547 movdqa XMMWORD[(160+80)+rbp],xmm8
548 movdqa xmm8,xmm7
549 psrld xmm8,20
550 pslld xmm7,32-20
551 pxor xmm7,xmm8
552 movdqa xmm8,xmm6
553 psrld xmm8,20
554 pslld xmm6,32-20
555 pxor xmm6,xmm8
556 movdqa xmm8,xmm5
557 psrld xmm8,20
558 pslld xmm5,32-20
559 pxor xmm5,xmm8
560 movdqa xmm8,xmm4
561 psrld xmm8,20
562 pslld xmm4,32-20
563 pxor xmm4,xmm8
564 movdqa xmm8,XMMWORD[$L$rol8]
565 paddd xmm3,xmm7
566 paddd xmm2,xmm6
567 paddd xmm1,xmm5
568 paddd xmm0,xmm4
569 pxor xmm15,xmm3
570 pxor xmm14,xmm2
571 pxor xmm13,xmm1
572 pxor xmm12,xmm0
573 DB 102,69,15,56,0,248
574 DB 102,69,15,56,0,240
575 DB 102,69,15,56,0,232
576 DB 102,69,15,56,0,224
577 movdqa xmm8,XMMWORD[((160+80))+rbp]
578 paddd xmm11,xmm15
579 paddd xmm10,xmm14
580 paddd xmm9,xmm13
581 paddd xmm8,xmm12
582 pxor xmm7,xmm11
583 pxor xmm6,xmm10
584 pxor xmm5,xmm9
585 pxor xmm4,xmm8
586 movdqa XMMWORD[(160+80)+rbp],xmm8
587 movdqa xmm8,xmm7
588 psrld xmm8,25
589 pslld xmm7,32-25
590 pxor xmm7,xmm8
591 movdqa xmm8,xmm6
592 psrld xmm8,25
593 pslld xmm6,32-25
594 pxor xmm6,xmm8
595 movdqa xmm8,xmm5
596 psrld xmm8,25
597 pslld xmm5,32-25
598 pxor xmm5,xmm8
599 movdqa xmm8,xmm4
600 psrld xmm8,25
601 pslld xmm4,32-25
602 pxor xmm4,xmm8
603 movdqa xmm8,XMMWORD[((160+80))+rbp]
604 DB 102,15,58,15,255,12
605 DB 102,69,15,58,15,219,8
606 DB 102,69,15,58,15,255,4
607 DB 102,15,58,15,246,12
608 DB 102,69,15,58,15,210,8
609 DB 102,69,15,58,15,246,4
610 DB 102,15,58,15,237,12
611 DB 102,69,15,58,15,201,8
612 DB 102,69,15,58,15,237,4
613 DB 102,15,58,15,228,12
614 DB 102,69,15,58,15,192,8
615 DB 102,69,15,58,15,228,4
616
617 dec rcx
618 jge NEAR $L$open_sse_main_loop_rounds
619 add r10,QWORD[((0+0))+r8]
620 adc r11,QWORD[((8+0))+r8]
621 adc r12,1
622 mov rax,QWORD[((0+160+0))+rbp]
623 mov r15,rax
624 mul r10
625 mov r13,rax
626 mov r14,rdx
627 mov rax,QWORD[((0+160+0))+rbp]
628 mul r11
629 imul r15,r12
630 add r14,rax
631 adc r15,rdx
632 mov rax,QWORD[((8+160+0))+rbp]
633 mov r9,rax
634 mul r10
635 add r14,rax
636 adc rdx,0
637 mov r10,rdx
638 mov rax,QWORD[((8+160+0))+rbp]
639 mul r11
640 add r15,rax
641 adc rdx,0
642 imul r9,r12
643 add r15,r10
644 adc r9,rdx
645 mov r10,r13
646 mov r11,r14
647 mov r12,r15
648 and r12,3
649 mov r13,r15
650 and r13,-4
651 mov r14,r9
652 shrd r15,r9,2
653 shr r9,2
654 add r15,r13
655 adc r9,r14
656 add r10,r15
657 adc r11,r9
658 adc r12,0
659
660 lea r8,[16+r8]
661 cmp rcx,-6
662 jg NEAR $L$open_sse_main_loop_rounds
663 paddd xmm3,XMMWORD[$L$chacha20_consts]
664 paddd xmm7,XMMWORD[((160+48))+rbp]
665 paddd xmm11,XMMWORD[((160+64))+rbp]
666 paddd xmm15,XMMWORD[((160+144))+rbp]
667 paddd xmm2,XMMWORD[$L$chacha20_consts]
668 paddd xmm6,XMMWORD[((160+48))+rbp]
669 paddd xmm10,XMMWORD[((160+64))+rbp]
670 paddd xmm14,XMMWORD[((160+128))+rbp]
671 paddd xmm1,XMMWORD[$L$chacha20_consts]
672 paddd xmm5,XMMWORD[((160+48))+rbp]
673 paddd xmm9,XMMWORD[((160+64))+rbp]
674 paddd xmm13,XMMWORD[((160+112))+rbp]
675 paddd xmm0,XMMWORD[$L$chacha20_consts]
676 paddd xmm4,XMMWORD[((160+48))+rbp]
677 paddd xmm8,XMMWORD[((160+64))+rbp]
678 paddd xmm12,XMMWORD[((160+96))+rbp]
679 movdqa XMMWORD[(160+80)+rbp],xmm12
680 movdqu xmm12,XMMWORD[((0 + 0))+rsi]
681 pxor xmm12,xmm3
682 movdqu XMMWORD[(0 + 0)+rdi],xmm12
683 movdqu xmm12,XMMWORD[((16 + 0))+rsi]
684 pxor xmm12,xmm7
685 movdqu XMMWORD[(16 + 0)+rdi],xmm12
686 movdqu xmm12,XMMWORD[((32 + 0))+rsi]
687 pxor xmm12,xmm11
688 movdqu XMMWORD[(32 + 0)+rdi],xmm12
689 movdqu xmm12,XMMWORD[((48 + 0))+rsi]
690 pxor xmm12,xmm15
691 movdqu XMMWORD[(48 + 0)+rdi],xmm12
692 movdqu xmm3,XMMWORD[((0 + 64))+rsi]
693 movdqu xmm7,XMMWORD[((16 + 64))+rsi]
694 movdqu xmm11,XMMWORD[((32 + 64))+rsi]
695 movdqu xmm15,XMMWORD[((48 + 64))+rsi]
696 pxor xmm2,xmm3
697 pxor xmm6,xmm7
698 pxor xmm10,xmm11
699 pxor xmm15,xmm14
700 movdqu XMMWORD[(0 + 64)+rdi],xmm2
701 movdqu XMMWORD[(16 + 64)+rdi],xmm6
702 movdqu XMMWORD[(32 + 64)+rdi],xmm10
703 movdqu XMMWORD[(48 + 64)+rdi],xmm15
704 movdqu xmm3,XMMWORD[((0 + 128))+rsi]
705 movdqu xmm7,XMMWORD[((16 + 128))+rsi]
706 movdqu xmm11,XMMWORD[((32 + 128))+rsi]
707 movdqu xmm15,XMMWORD[((48 + 128))+rsi]
708 pxor xmm1,xmm3
709 pxor xmm5,xmm7
710 pxor xmm9,xmm11
711 pxor xmm15,xmm13
712 movdqu XMMWORD[(0 + 128)+rdi],xmm1
713 movdqu XMMWORD[(16 + 128)+rdi],xmm5
714 movdqu XMMWORD[(32 + 128)+rdi],xmm9
715 movdqu XMMWORD[(48 + 128)+rdi],xmm15
716 movdqu xmm3,XMMWORD[((0 + 192))+rsi]
717 movdqu xmm7,XMMWORD[((16 + 192))+rsi]
718 movdqu xmm11,XMMWORD[((32 + 192))+rsi]
719 movdqu xmm15,XMMWORD[((48 + 192))+rsi]
720 pxor xmm0,xmm3
721 pxor xmm4,xmm7
722 pxor xmm8,xmm11
723 pxor xmm15,XMMWORD[((160+80))+rbp]
724 movdqu XMMWORD[(0 + 192)+rdi],xmm0
725 movdqu XMMWORD[(16 + 192)+rdi],xmm4
726 movdqu XMMWORD[(32 + 192)+rdi],xmm8
727 movdqu XMMWORD[(48 + 192)+rdi],xmm15
728
729 lea rsi,[256+rsi]
730 lea rdi,[256+rdi]
731 sub rbx,16*16
732 jmp NEAR $L$open_sse_main_loop
733 $L$open_sse_tail:
734
735 test rbx,rbx
736 jz NEAR $L$open_sse_finalize
737 cmp rbx,12*16
738 ja NEAR $L$open_sse_tail_256
739 cmp rbx,8*16
740 ja NEAR $L$open_sse_tail_192
741 cmp rbx,4*16
742 ja NEAR $L$open_sse_tail_128
743 movdqa xmm0,XMMWORD[$L$chacha20_consts]
744 movdqa xmm4,XMMWORD[((160+48))+rbp]
745 movdqa xmm8,XMMWORD[((160+64))+rbp]
746 movdqa xmm12,XMMWORD[((160+96))+rbp]
747 paddd xmm12,XMMWORD[$L$sse_inc]
748 movdqa XMMWORD[(160+96)+rbp],xmm12
749
750 xor r8,r8
751 mov rcx,rbx
752 cmp rcx,16
753 jb NEAR $L$open_sse_tail_64_rounds
754 $L$open_sse_tail_64_rounds_and_x1hash:
755 add r10,QWORD[((0+0))+r8*1+rsi]
756 adc r11,QWORD[((8+0))+r8*1+rsi]
757 adc r12,1
758 mov rax,QWORD[((0+160+0))+rbp]
759 mov r15,rax
760 mul r10
761 mov r13,rax
762 mov r14,rdx
763 mov rax,QWORD[((0+160+0))+rbp]
764 mul r11
765 imul r15,r12
766 add r14,rax
767 adc r15,rdx
768 mov rax,QWORD[((8+160+0))+rbp]
769 mov r9,rax
770 mul r10
771 add r14,rax
772 adc rdx,0
773 mov r10,rdx
774 mov rax,QWORD[((8+160+0))+rbp]
775 mul r11
776 add r15,rax
777 adc rdx,0
778 imul r9,r12
779 add r15,r10
780 adc r9,rdx
781 mov r10,r13
782 mov r11,r14
783 mov r12,r15
784 and r12,3
785 mov r13,r15
786 and r13,-4
787 mov r14,r9
788 shrd r15,r9,2
789 shr r9,2
790 add r15,r13
791 adc r9,r14
792 add r10,r15
793 adc r11,r9
794 adc r12,0
795
796 sub rcx,16
797 $L$open_sse_tail_64_rounds:
798 add r8,16
799 paddd xmm0,xmm4
800 pxor xmm12,xmm0
801 pshufb xmm12,XMMWORD[$L$rol16]
802 paddd xmm8,xmm12
803 pxor xmm4,xmm8
804 movdqa xmm3,xmm4
805 pslld xmm3,12
806 psrld xmm4,20
807 pxor xmm4,xmm3
808 paddd xmm0,xmm4
809 pxor xmm12,xmm0
810 pshufb xmm12,XMMWORD[$L$rol8]
811 paddd xmm8,xmm12
812 pxor xmm4,xmm8
813 movdqa xmm3,xmm4
814 pslld xmm3,7
815 psrld xmm4,25
816 pxor xmm4,xmm3
817 DB 102,15,58,15,228,4
818 DB 102,69,15,58,15,192,8
819 DB 102,69,15,58,15,228,12
820 paddd xmm0,xmm4
821 pxor xmm12,xmm0
822 pshufb xmm12,XMMWORD[$L$rol16]
823 paddd xmm8,xmm12
824 pxor xmm4,xmm8
825 movdqa xmm3,xmm4
826 pslld xmm3,12
827 psrld xmm4,20
828 pxor xmm4,xmm3
829 paddd xmm0,xmm4
830 pxor xmm12,xmm0
831 pshufb xmm12,XMMWORD[$L$rol8]
832 paddd xmm8,xmm12
833 pxor xmm4,xmm8
834 movdqa xmm3,xmm4
835 pslld xmm3,7
836 psrld xmm4,25
837 pxor xmm4,xmm3
838 DB 102,15,58,15,228,12
839 DB 102,69,15,58,15,192,8
840 DB 102,69,15,58,15,228,4
841
842 cmp rcx,16
843 jae NEAR $L$open_sse_tail_64_rounds_and_x1hash
844 cmp r8,10*16
845 jne NEAR $L$open_sse_tail_64_rounds
846 paddd xmm0,XMMWORD[$L$chacha20_consts]
847 paddd xmm4,XMMWORD[((160+48))+rbp]
848 paddd xmm8,XMMWORD[((160+64))+rbp]
849 paddd xmm12,XMMWORD[((160+96))+rbp]
850
851 jmp NEAR $L$open_sse_tail_64_dec_loop
852
853 $L$open_sse_tail_128:
854 movdqa xmm0,XMMWORD[$L$chacha20_consts]
855 movdqa xmm4,XMMWORD[((160+48))+rbp]
856 movdqa xmm8,XMMWORD[((160+64))+rbp]
857 movdqa xmm1,xmm0
858 movdqa xmm5,xmm4
859 movdqa xmm9,xmm8
860 movdqa xmm13,XMMWORD[((160+96))+rbp]
861 paddd xmm13,XMMWORD[$L$sse_inc]
862 movdqa xmm12,xmm13
863 paddd xmm12,XMMWORD[$L$sse_inc]
864 movdqa XMMWORD[(160+96)+rbp],xmm12
865 movdqa XMMWORD[(160+112)+rbp],xmm13
866
867 mov rcx,rbx
868 and rcx,-16
869 xor r8,r8
870 $L$open_sse_tail_128_rounds_and_x1hash:
871 add r10,QWORD[((0+0))+r8*1+rsi]
872 adc r11,QWORD[((8+0))+r8*1+rsi]
873 adc r12,1
874 mov rax,QWORD[((0+160+0))+rbp]
875 mov r15,rax
876 mul r10
877 mov r13,rax
878 mov r14,rdx
879 mov rax,QWORD[((0+160+0))+rbp]
880 mul r11
881 imul r15,r12
882 add r14,rax
883 adc r15,rdx
884 mov rax,QWORD[((8+160+0))+rbp]
885 mov r9,rax
886 mul r10
887 add r14,rax
888 adc rdx,0
889 mov r10,rdx
890 mov rax,QWORD[((8+160+0))+rbp]
891 mul r11
892 add r15,rax
893 adc rdx,0
894 imul r9,r12
895 add r15,r10
896 adc r9,rdx
897 mov r10,r13
898 mov r11,r14
899 mov r12,r15
900 and r12,3
901 mov r13,r15
902 and r13,-4
903 mov r14,r9
904 shrd r15,r9,2
905 shr r9,2
906 add r15,r13
907 adc r9,r14
908 add r10,r15
909 adc r11,r9
910 adc r12,0
911
912 $L$open_sse_tail_128_rounds:
913 add r8,16
914 paddd xmm0,xmm4
915 pxor xmm12,xmm0
916 pshufb xmm12,XMMWORD[$L$rol16]
917 paddd xmm8,xmm12
918 pxor xmm4,xmm8
919 movdqa xmm3,xmm4
920 pslld xmm3,12
921 psrld xmm4,20
922 pxor xmm4,xmm3
923 paddd xmm0,xmm4
924 pxor xmm12,xmm0
925 pshufb xmm12,XMMWORD[$L$rol8]
926 paddd xmm8,xmm12
927 pxor xmm4,xmm8
928 movdqa xmm3,xmm4
929 pslld xmm3,7
930 psrld xmm4,25
931 pxor xmm4,xmm3
932 DB 102,15,58,15,228,4
933 DB 102,69,15,58,15,192,8
934 DB 102,69,15,58,15,228,12
935 paddd xmm1,xmm5
936 pxor xmm13,xmm1
937 pshufb xmm13,XMMWORD[$L$rol16]
938 paddd xmm9,xmm13
939 pxor xmm5,xmm9
940 movdqa xmm3,xmm5
941 pslld xmm3,12
942 psrld xmm5,20
943 pxor xmm5,xmm3
944 paddd xmm1,xmm5
945 pxor xmm13,xmm1
946 pshufb xmm13,XMMWORD[$L$rol8]
947 paddd xmm9,xmm13
948 pxor xmm5,xmm9
949 movdqa xmm3,xmm5
950 pslld xmm3,7
951 psrld xmm5,25
952 pxor xmm5,xmm3
953 DB 102,15,58,15,237,4
954 DB 102,69,15,58,15,201,8
955 DB 102,69,15,58,15,237,12
956 paddd xmm0,xmm4
957 pxor xmm12,xmm0
958 pshufb xmm12,XMMWORD[$L$rol16]
959 paddd xmm8,xmm12
960 pxor xmm4,xmm8
961 movdqa xmm3,xmm4
962 pslld xmm3,12
963 psrld xmm4,20
964 pxor xmm4,xmm3
965 paddd xmm0,xmm4
966 pxor xmm12,xmm0
967 pshufb xmm12,XMMWORD[$L$rol8]
968 paddd xmm8,xmm12
969 pxor xmm4,xmm8
970 movdqa xmm3,xmm4
971 pslld xmm3,7
972 psrld xmm4,25
973 pxor xmm4,xmm3
974 DB 102,15,58,15,228,12
975 DB 102,69,15,58,15,192,8
976 DB 102,69,15,58,15,228,4
977 paddd xmm1,xmm5
978 pxor xmm13,xmm1
979 pshufb xmm13,XMMWORD[$L$rol16]
980 paddd xmm9,xmm13
981 pxor xmm5,xmm9
982 movdqa xmm3,xmm5
983 pslld xmm3,12
984 psrld xmm5,20
985 pxor xmm5,xmm3
986 paddd xmm1,xmm5
987 pxor xmm13,xmm1
988 pshufb xmm13,XMMWORD[$L$rol8]
989 paddd xmm9,xmm13
990 pxor xmm5,xmm9
991 movdqa xmm3,xmm5
992 pslld xmm3,7
993 psrld xmm5,25
994 pxor xmm5,xmm3
995 DB 102,15,58,15,237,12
996 DB 102,69,15,58,15,201,8
997 DB 102,69,15,58,15,237,4
998
999 cmp r8,rcx
1000 jb NEAR $L$open_sse_tail_128_rounds_and_x1hash
1001 cmp r8,10*16
1002 jne NEAR $L$open_sse_tail_128_rounds
1003 paddd xmm1,XMMWORD[$L$chacha20_consts]
1004 paddd xmm5,XMMWORD[((160+48))+rbp]
1005 paddd xmm9,XMMWORD[((160+64))+rbp]
1006 paddd xmm13,XMMWORD[((160+112))+rbp]
1007 paddd xmm0,XMMWORD[$L$chacha20_consts]
1008 paddd xmm4,XMMWORD[((160+48))+rbp]
1009 paddd xmm8,XMMWORD[((160+64))+rbp]
1010 paddd xmm12,XMMWORD[((160+96))+rbp]
1011 movdqu xmm3,XMMWORD[((0 + 0))+rsi]
1012 movdqu xmm7,XMMWORD[((16 + 0))+rsi]
1013 movdqu xmm11,XMMWORD[((32 + 0))+rsi]
1014 movdqu xmm15,XMMWORD[((48 + 0))+rsi]
1015 pxor xmm1,xmm3
1016 pxor xmm5,xmm7
1017 pxor xmm9,xmm11
1018 pxor xmm15,xmm13
1019 movdqu XMMWORD[(0 + 0)+rdi],xmm1
1020 movdqu XMMWORD[(16 + 0)+rdi],xmm5
1021 movdqu XMMWORD[(32 + 0)+rdi],xmm9
1022 movdqu XMMWORD[(48 + 0)+rdi],xmm15
1023
1024 sub rbx,4*16
1025 lea rsi,[64+rsi]
1026 lea rdi,[64+rdi]
1027 jmp NEAR $L$open_sse_tail_64_dec_loop
1028
1029 $L$open_sse_tail_192:
1030 movdqa xmm0,XMMWORD[$L$chacha20_consts]
1031 movdqa xmm4,XMMWORD[((160+48))+rbp]
1032 movdqa xmm8,XMMWORD[((160+64))+rbp]
1033 movdqa xmm1,xmm0
1034 movdqa xmm5,xmm4
1035 movdqa xmm9,xmm8
1036 movdqa xmm2,xmm0
1037 movdqa xmm6,xmm4
1038 movdqa xmm10,xmm8
1039 movdqa xmm14,XMMWORD[((160+96))+rbp]
1040 paddd xmm14,XMMWORD[$L$sse_inc]
1041 movdqa xmm13,xmm14
1042 paddd xmm13,XMMWORD[$L$sse_inc]
1043 movdqa xmm12,xmm13
1044 paddd xmm12,XMMWORD[$L$sse_inc]
1045 movdqa XMMWORD[(160+96)+rbp],xmm12
1046 movdqa XMMWORD[(160+112)+rbp],xmm13
1047 movdqa XMMWORD[(160+128)+rbp],xmm14
1048
1049 mov rcx,rbx
1050 mov r8,10*16
1051 cmp rcx,10*16
1052 cmovg rcx,r8
1053 and rcx,-16
1054 xor r8,r8
1055 $L$open_sse_tail_192_rounds_and_x1hash:
1056 add r10,QWORD[((0+0))+r8*1+rsi]
1057 adc r11,QWORD[((8+0))+r8*1+rsi]
1058 adc r12,1
1059 mov rax,QWORD[((0+160+0))+rbp]
1060 mov r15,rax
1061 mul r10
1062 mov r13,rax
1063 mov r14,rdx
1064 mov rax,QWORD[((0+160+0))+rbp]
1065 mul r11
1066 imul r15,r12
1067 add r14,rax
1068 adc r15,rdx
1069 mov rax,QWORD[((8+160+0))+rbp]
1070 mov r9,rax
1071 mul r10
1072 add r14,rax
1073 adc rdx,0
1074 mov r10,rdx
1075 mov rax,QWORD[((8+160+0))+rbp]
1076 mul r11
1077 add r15,rax
1078 adc rdx,0
1079 imul r9,r12
1080 add r15,r10
1081 adc r9,rdx
1082 mov r10,r13
1083 mov r11,r14
1084 mov r12,r15
1085 and r12,3
1086 mov r13,r15
1087 and r13,-4
1088 mov r14,r9
1089 shrd r15,r9,2
1090 shr r9,2
1091 add r15,r13
1092 adc r9,r14
1093 add r10,r15
1094 adc r11,r9
1095 adc r12,0
1096
1097 $L$open_sse_tail_192_rounds:
1098 add r8,16
1099 paddd xmm0,xmm4
1100 pxor xmm12,xmm0
1101 pshufb xmm12,XMMWORD[$L$rol16]
1102 paddd xmm8,xmm12
1103 pxor xmm4,xmm8
1104 movdqa xmm3,xmm4
1105 pslld xmm3,12
1106 psrld xmm4,20
1107 pxor xmm4,xmm3
1108 paddd xmm0,xmm4
1109 pxor xmm12,xmm0
1110 pshufb xmm12,XMMWORD[$L$rol8]
1111 paddd xmm8,xmm12
1112 pxor xmm4,xmm8
1113 movdqa xmm3,xmm4
1114 pslld xmm3,7
1115 psrld xmm4,25
1116 pxor xmm4,xmm3
1117 DB 102,15,58,15,228,4
1118 DB 102,69,15,58,15,192,8
1119 DB 102,69,15,58,15,228,12
1120 paddd xmm1,xmm5
1121 pxor xmm13,xmm1
1122 pshufb xmm13,XMMWORD[$L$rol16]
1123 paddd xmm9,xmm13
1124 pxor xmm5,xmm9
1125 movdqa xmm3,xmm5
1126 pslld xmm3,12
1127 psrld xmm5,20
1128 pxor xmm5,xmm3
1129 paddd xmm1,xmm5
1130 pxor xmm13,xmm1
1131 pshufb xmm13,XMMWORD[$L$rol8]
1132 paddd xmm9,xmm13
1133 pxor xmm5,xmm9
1134 movdqa xmm3,xmm5
1135 pslld xmm3,7
1136 psrld xmm5,25
1137 pxor xmm5,xmm3
1138 DB 102,15,58,15,237,4
1139 DB 102,69,15,58,15,201,8
1140 DB 102,69,15,58,15,237,12
1141 paddd xmm2,xmm6
1142 pxor xmm14,xmm2
1143 pshufb xmm14,XMMWORD[$L$rol16]
1144 paddd xmm10,xmm14
1145 pxor xmm6,xmm10
1146 movdqa xmm3,xmm6
1147 pslld xmm3,12
1148 psrld xmm6,20
1149 pxor xmm6,xmm3
1150 paddd xmm2,xmm6
1151 pxor xmm14,xmm2
1152 pshufb xmm14,XMMWORD[$L$rol8]
1153 paddd xmm10,xmm14
1154 pxor xmm6,xmm10
1155 movdqa xmm3,xmm6
1156 pslld xmm3,7
1157 psrld xmm6,25
1158 pxor xmm6,xmm3
1159 DB 102,15,58,15,246,4
1160 DB 102,69,15,58,15,210,8
1161 DB 102,69,15,58,15,246,12
1162 paddd xmm0,xmm4
1163 pxor xmm12,xmm0
1164 pshufb xmm12,XMMWORD[$L$rol16]
1165 paddd xmm8,xmm12
1166 pxor xmm4,xmm8
1167 movdqa xmm3,xmm4
1168 pslld xmm3,12
1169 psrld xmm4,20
1170 pxor xmm4,xmm3
1171 paddd xmm0,xmm4
1172 pxor xmm12,xmm0
1173 pshufb xmm12,XMMWORD[$L$rol8]
1174 paddd xmm8,xmm12
1175 pxor xmm4,xmm8
1176 movdqa xmm3,xmm4
1177 pslld xmm3,7
1178 psrld xmm4,25
1179 pxor xmm4,xmm3
1180 DB 102,15,58,15,228,12
1181 DB 102,69,15,58,15,192,8
1182 DB 102,69,15,58,15,228,4
1183 paddd xmm1,xmm5
1184 pxor xmm13,xmm1
1185 pshufb xmm13,XMMWORD[$L$rol16]
1186 paddd xmm9,xmm13
1187 pxor xmm5,xmm9
1188 movdqa xmm3,xmm5
1189 pslld xmm3,12
1190 psrld xmm5,20
1191 pxor xmm5,xmm3
1192 paddd xmm1,xmm5
1193 pxor xmm13,xmm1
1194 pshufb xmm13,XMMWORD[$L$rol8]
1195 paddd xmm9,xmm13
1196 pxor xmm5,xmm9
1197 movdqa xmm3,xmm5
1198 pslld xmm3,7
1199 psrld xmm5,25
1200 pxor xmm5,xmm3
1201 DB 102,15,58,15,237,12
1202 DB 102,69,15,58,15,201,8
1203 DB 102,69,15,58,15,237,4
1204 paddd xmm2,xmm6
1205 pxor xmm14,xmm2
1206 pshufb xmm14,XMMWORD[$L$rol16]
1207 paddd xmm10,xmm14
1208 pxor xmm6,xmm10
1209 movdqa xmm3,xmm6
1210 pslld xmm3,12
1211 psrld xmm6,20
1212 pxor xmm6,xmm3
1213 paddd xmm2,xmm6
1214 pxor xmm14,xmm2
1215 pshufb xmm14,XMMWORD[$L$rol8]
1216 paddd xmm10,xmm14
1217 pxor xmm6,xmm10
1218 movdqa xmm3,xmm6
1219 pslld xmm3,7
1220 psrld xmm6,25
1221 pxor xmm6,xmm3
1222 DB 102,15,58,15,246,12
1223 DB 102,69,15,58,15,210,8
1224 DB 102,69,15,58,15,246,4
1225
1226 cmp r8,rcx
1227 jb NEAR $L$open_sse_tail_192_rounds_and_x1hash
1228 cmp r8,10*16
1229 jne NEAR $L$open_sse_tail_192_rounds
1230 cmp rbx,11*16
1231 jb NEAR $L$open_sse_tail_192_finish
1232 add r10,QWORD[((0+160))+rsi]
1233 adc r11,QWORD[((8+160))+rsi]
1234 adc r12,1
1235 mov rax,QWORD[((0+160+0))+rbp]
1236 mov r15,rax
1237 mul r10
1238 mov r13,rax
1239 mov r14,rdx
1240 mov rax,QWORD[((0+160+0))+rbp]
1241 mul r11
1242 imul r15,r12
1243 add r14,rax
1244 adc r15,rdx
1245 mov rax,QWORD[((8+160+0))+rbp]
1246 mov r9,rax
1247 mul r10
1248 add r14,rax
1249 adc rdx,0
1250 mov r10,rdx
1251 mov rax,QWORD[((8+160+0))+rbp]
1252 mul r11
1253 add r15,rax
1254 adc rdx,0
1255 imul r9,r12
1256 add r15,r10
1257 adc r9,rdx
1258 mov r10,r13
1259 mov r11,r14
1260 mov r12,r15
1261 and r12,3
1262 mov r13,r15
1263 and r13,-4
1264 mov r14,r9
1265 shrd r15,r9,2
1266 shr r9,2
1267 add r15,r13
1268 adc r9,r14
1269 add r10,r15
1270 adc r11,r9
1271 adc r12,0
1272
1273 cmp rbx,12*16
1274 jb NEAR $L$open_sse_tail_192_finish
1275 add r10,QWORD[((0+176))+rsi]
1276 adc r11,QWORD[((8+176))+rsi]
1277 adc r12,1
1278 mov rax,QWORD[((0+160+0))+rbp]
1279 mov r15,rax
1280 mul r10
1281 mov r13,rax
1282 mov r14,rdx
1283 mov rax,QWORD[((0+160+0))+rbp]
1284 mul r11
1285 imul r15,r12
1286 add r14,rax
1287 adc r15,rdx
1288 mov rax,QWORD[((8+160+0))+rbp]
1289 mov r9,rax
1290 mul r10
1291 add r14,rax
1292 adc rdx,0
1293 mov r10,rdx
1294 mov rax,QWORD[((8+160+0))+rbp]
1295 mul r11
1296 add r15,rax
1297 adc rdx,0
1298 imul r9,r12
1299 add r15,r10
1300 adc r9,rdx
1301 mov r10,r13
1302 mov r11,r14
1303 mov r12,r15
1304 and r12,3
1305 mov r13,r15
1306 and r13,-4
1307 mov r14,r9
1308 shrd r15,r9,2
1309 shr r9,2
1310 add r15,r13
1311 adc r9,r14
1312 add r10,r15
1313 adc r11,r9
1314 adc r12,0
1315
1316 $L$open_sse_tail_192_finish:
1317 paddd xmm2,XMMWORD[$L$chacha20_consts]
1318 paddd xmm6,XMMWORD[((160+48))+rbp]
1319 paddd xmm10,XMMWORD[((160+64))+rbp]
1320 paddd xmm14,XMMWORD[((160+128))+rbp]
1321 paddd xmm1,XMMWORD[$L$chacha20_consts]
1322 paddd xmm5,XMMWORD[((160+48))+rbp]
1323 paddd xmm9,XMMWORD[((160+64))+rbp]
1324 paddd xmm13,XMMWORD[((160+112))+rbp]
1325 paddd xmm0,XMMWORD[$L$chacha20_consts]
1326 paddd xmm4,XMMWORD[((160+48))+rbp]
1327 paddd xmm8,XMMWORD[((160+64))+rbp]
1328 paddd xmm12,XMMWORD[((160+96))+rbp]
1329 movdqu xmm3,XMMWORD[((0 + 0))+rsi]
1330 movdqu xmm7,XMMWORD[((16 + 0))+rsi]
1331 movdqu xmm11,XMMWORD[((32 + 0))+rsi]
1332 movdqu xmm15,XMMWORD[((48 + 0))+rsi]
1333 pxor xmm2,xmm3
1334 pxor xmm6,xmm7
1335 pxor xmm10,xmm11
1336 pxor xmm15,xmm14
1337 movdqu XMMWORD[(0 + 0)+rdi],xmm2
1338 movdqu XMMWORD[(16 + 0)+rdi],xmm6
1339 movdqu XMMWORD[(32 + 0)+rdi],xmm10
1340 movdqu XMMWORD[(48 + 0)+rdi],xmm15
1341 movdqu xmm3,XMMWORD[((0 + 64))+rsi]
1342 movdqu xmm7,XMMWORD[((16 + 64))+rsi]
1343 movdqu xmm11,XMMWORD[((32 + 64))+rsi]
1344 movdqu xmm15,XMMWORD[((48 + 64))+rsi]
1345 pxor xmm1,xmm3
1346 pxor xmm5,xmm7
1347 pxor xmm9,xmm11
1348 pxor xmm15,xmm13
1349 movdqu XMMWORD[(0 + 64)+rdi],xmm1
1350 movdqu XMMWORD[(16 + 64)+rdi],xmm5
1351 movdqu XMMWORD[(32 + 64)+rdi],xmm9
1352 movdqu XMMWORD[(48 + 64)+rdi],xmm15
1353
1354 sub rbx,8*16
1355 lea rsi,[128+rsi]
1356 lea rdi,[128+rdi]
1357 jmp NEAR $L$open_sse_tail_64_dec_loop
1358
1359 $L$open_sse_tail_256:
1360 movdqa xmm0,XMMWORD[$L$chacha20_consts]
1361 movdqa xmm4,XMMWORD[((160+48))+rbp]
1362 movdqa xmm8,XMMWORD[((160+64))+rbp]
1363 movdqa xmm1,xmm0
1364 movdqa xmm5,xmm4
1365 movdqa xmm9,xmm8
1366 movdqa xmm2,xmm0
1367 movdqa xmm6,xmm4
1368 movdqa xmm10,xmm8
1369 movdqa xmm3,xmm0
1370 movdqa xmm7,xmm4
1371 movdqa xmm11,xmm8
1372 movdqa xmm15,XMMWORD[((160+96))+rbp]
1373 paddd xmm15,XMMWORD[$L$sse_inc]
1374 movdqa xmm14,xmm15
1375 paddd xmm14,XMMWORD[$L$sse_inc]
1376 movdqa xmm13,xmm14
1377 paddd xmm13,XMMWORD[$L$sse_inc]
1378 movdqa xmm12,xmm13
1379 paddd xmm12,XMMWORD[$L$sse_inc]
1380 movdqa XMMWORD[(160+96)+rbp],xmm12
1381 movdqa XMMWORD[(160+112)+rbp],xmm13
1382 movdqa XMMWORD[(160+128)+rbp],xmm14
1383 movdqa XMMWORD[(160+144)+rbp],xmm15
1384
1385 xor r8,r8
1386 $L$open_sse_tail_256_rounds_and_x1hash:
1387 add r10,QWORD[((0+0))+r8*1+rsi]
1388 adc r11,QWORD[((8+0))+r8*1+rsi]
1389 adc r12,1
1390 movdqa XMMWORD[(160+80)+rbp],xmm11
1391 paddd xmm0,xmm4
1392 pxor xmm12,xmm0
1393 pshufb xmm12,XMMWORD[$L$rol16]
1394 paddd xmm8,xmm12
1395 pxor xmm4,xmm8
1396 movdqa xmm11,xmm4
1397 pslld xmm11,12
1398 psrld xmm4,20
1399 pxor xmm4,xmm11
1400 paddd xmm0,xmm4
1401 pxor xmm12,xmm0
1402 pshufb xmm12,XMMWORD[$L$rol8]
1403 paddd xmm8,xmm12
1404 pxor xmm4,xmm8
1405 movdqa xmm11,xmm4
1406 pslld xmm11,7
1407 psrld xmm4,25
1408 pxor xmm4,xmm11
1409 DB 102,15,58,15,228,4
1410 DB 102,69,15,58,15,192,8
1411 DB 102,69,15,58,15,228,12
1412 paddd xmm1,xmm5
1413 pxor xmm13,xmm1
1414 pshufb xmm13,XMMWORD[$L$rol16]
1415 paddd xmm9,xmm13
1416 pxor xmm5,xmm9
1417 movdqa xmm11,xmm5
1418 pslld xmm11,12
1419 psrld xmm5,20
1420 pxor xmm5,xmm11
1421 paddd xmm1,xmm5
1422 pxor xmm13,xmm1
1423 pshufb xmm13,XMMWORD[$L$rol8]
1424 paddd xmm9,xmm13
1425 pxor xmm5,xmm9
1426 movdqa xmm11,xmm5
1427 pslld xmm11,7
1428 psrld xmm5,25
1429 pxor xmm5,xmm11
1430 DB 102,15,58,15,237,4
1431 DB 102,69,15,58,15,201,8
1432 DB 102,69,15,58,15,237,12
1433 paddd xmm2,xmm6
1434 pxor xmm14,xmm2
1435 pshufb xmm14,XMMWORD[$L$rol16]
1436 paddd xmm10,xmm14
1437 pxor xmm6,xmm10
1438 movdqa xmm11,xmm6
1439 pslld xmm11,12
1440 psrld xmm6,20
1441 pxor xmm6,xmm11
1442 paddd xmm2,xmm6
1443 pxor xmm14,xmm2
1444 pshufb xmm14,XMMWORD[$L$rol8]
1445 paddd xmm10,xmm14
1446 pxor xmm6,xmm10
1447 movdqa xmm11,xmm6
1448 pslld xmm11,7
1449 psrld xmm6,25
1450 pxor xmm6,xmm11
1451 DB 102,15,58,15,246,4
1452 DB 102,69,15,58,15,210,8
1453 DB 102,69,15,58,15,246,12
1454 movdqa xmm11,XMMWORD[((160+80))+rbp]
1455 mov rax,QWORD[((0+160+0))+rbp]
1456 mov r15,rax
1457 mul r10
1458 mov r13,rax
1459 mov r14,rdx
1460 mov rax,QWORD[((0+160+0))+rbp]
1461 mul r11
1462 imul r15,r12
1463 add r14,rax
1464 adc r15,rdx
1465 movdqa XMMWORD[(160+80)+rbp],xmm9
1466 paddd xmm3,xmm7
1467 pxor xmm15,xmm3
1468 pshufb xmm15,XMMWORD[$L$rol16]
1469 paddd xmm11,xmm15
1470 pxor xmm7,xmm11
1471 movdqa xmm9,xmm7
1472 pslld xmm9,12
1473 psrld xmm7,20
1474 pxor xmm7,xmm9
1475 paddd xmm3,xmm7
1476 pxor xmm15,xmm3
1477 pshufb xmm15,XMMWORD[$L$rol8]
1478 paddd xmm11,xmm15
1479 pxor xmm7,xmm11
1480 movdqa xmm9,xmm7
1481 pslld xmm9,7
1482 psrld xmm7,25
1483 pxor xmm7,xmm9
1484 DB 102,15,58,15,255,4
1485 DB 102,69,15,58,15,219,8
1486 DB 102,69,15,58,15,255,12
1487 movdqa xmm9,XMMWORD[((160+80))+rbp]
1488 mov rax,QWORD[((8+160+0))+rbp]
1489 mov r9,rax
1490 mul r10
1491 add r14,rax
1492 adc rdx,0
1493 mov r10,rdx
1494 mov rax,QWORD[((8+160+0))+rbp]
1495 mul r11
1496 add r15,rax
1497 adc rdx,0
1498 movdqa XMMWORD[(160+80)+rbp],xmm11
1499 paddd xmm0,xmm4
1500 pxor xmm12,xmm0
1501 pshufb xmm12,XMMWORD[$L$rol16]
1502 paddd xmm8,xmm12
1503 pxor xmm4,xmm8
1504 movdqa xmm11,xmm4
1505 pslld xmm11,12
1506 psrld xmm4,20
1507 pxor xmm4,xmm11
1508 paddd xmm0,xmm4
1509 pxor xmm12,xmm0
1510 pshufb xmm12,XMMWORD[$L$rol8]
1511 paddd xmm8,xmm12
1512 pxor xmm4,xmm8
1513 movdqa xmm11,xmm4
1514 pslld xmm11,7
1515 psrld xmm4,25
1516 pxor xmm4,xmm11
1517 DB 102,15,58,15,228,12
1518 DB 102,69,15,58,15,192,8
1519 DB 102,69,15,58,15,228,4
1520 paddd xmm1,xmm5
1521 pxor xmm13,xmm1
1522 pshufb xmm13,XMMWORD[$L$rol16]
1523 paddd xmm9,xmm13
1524 pxor xmm5,xmm9
1525 movdqa xmm11,xmm5
1526 pslld xmm11,12
1527 psrld xmm5,20
1528 pxor xmm5,xmm11
1529 paddd xmm1,xmm5
1530 pxor xmm13,xmm1
1531 pshufb xmm13,XMMWORD[$L$rol8]
1532 paddd xmm9,xmm13
1533 pxor xmm5,xmm9
1534 movdqa xmm11,xmm5
1535 pslld xmm11,7
1536 psrld xmm5,25
1537 pxor xmm5,xmm11
1538 DB 102,15,58,15,237,12
1539 DB 102,69,15,58,15,201,8
1540 DB 102,69,15,58,15,237,4
1541 imul r9,r12
1542 add r15,r10
1543 adc r9,rdx
1544 paddd xmm2,xmm6
1545 pxor xmm14,xmm2
1546 pshufb xmm14,XMMWORD[$L$rol16]
1547 paddd xmm10,xmm14
1548 pxor xmm6,xmm10
1549 movdqa xmm11,xmm6
1550 pslld xmm11,12
1551 psrld xmm6,20
1552 pxor xmm6,xmm11
1553 paddd xmm2,xmm6
1554 pxor xmm14,xmm2
1555 pshufb xmm14,XMMWORD[$L$rol8]
1556 paddd xmm10,xmm14
1557 pxor xmm6,xmm10
1558 movdqa xmm11,xmm6
1559 pslld xmm11,7
1560 psrld xmm6,25
1561 pxor xmm6,xmm11
1562 DB 102,15,58,15,246,12
1563 DB 102,69,15,58,15,210,8
1564 DB 102,69,15,58,15,246,4
1565 movdqa xmm11,XMMWORD[((160+80))+rbp]
1566 mov r10,r13
1567 mov r11,r14
1568 mov r12,r15
1569 and r12,3
1570 mov r13,r15
1571 and r13,-4
1572 mov r14,r9
1573 shrd r15,r9,2
1574 shr r9,2
1575 add r15,r13
1576 adc r9,r14
1577 add r10,r15
1578 adc r11,r9
1579 adc r12,0
1580 movdqa XMMWORD[(160+80)+rbp],xmm9
1581 paddd xmm3,xmm7
1582 pxor xmm15,xmm3
1583 pshufb xmm15,XMMWORD[$L$rol16]
1584 paddd xmm11,xmm15
1585 pxor xmm7,xmm11
1586 movdqa xmm9,xmm7
1587 pslld xmm9,12
1588 psrld xmm7,20
1589 pxor xmm7,xmm9
1590 paddd xmm3,xmm7
1591 pxor xmm15,xmm3
1592 pshufb xmm15,XMMWORD[$L$rol8]
1593 paddd xmm11,xmm15
1594 pxor xmm7,xmm11
1595 movdqa xmm9,xmm7
1596 pslld xmm9,7
1597 psrld xmm7,25
1598 pxor xmm7,xmm9
1599 DB 102,15,58,15,255,12
1600 DB 102,69,15,58,15,219,8
1601 DB 102,69,15,58,15,255,4
1602 movdqa xmm9,XMMWORD[((160+80))+rbp]
1603
1604 add r8,16
1605 cmp r8,10*16
1606 jb NEAR $L$open_sse_tail_256_rounds_and_x1hash
1607
1608 mov rcx,rbx
1609 and rcx,-16
1610 $L$open_sse_tail_256_hash:
1611 add r10,QWORD[((0+0))+r8*1+rsi]
1612 adc r11,QWORD[((8+0))+r8*1+rsi]
1613 adc r12,1
1614 mov rax,QWORD[((0+160+0))+rbp]
1615 mov r15,rax
1616 mul r10
1617 mov r13,rax
1618 mov r14,rdx
1619 mov rax,QWORD[((0+160+0))+rbp]
1620 mul r11
1621 imul r15,r12
1622 add r14,rax
1623 adc r15,rdx
1624 mov rax,QWORD[((8+160+0))+rbp]
1625 mov r9,rax
1626 mul r10
1627 add r14,rax
1628 adc rdx,0
1629 mov r10,rdx
1630 mov rax,QWORD[((8+160+0))+rbp]
1631 mul r11
1632 add r15,rax
1633 adc rdx,0
1634 imul r9,r12
1635 add r15,r10
1636 adc r9,rdx
1637 mov r10,r13
1638 mov r11,r14
1639 mov r12,r15
1640 and r12,3
1641 mov r13,r15
1642 and r13,-4
1643 mov r14,r9
1644 shrd r15,r9,2
1645 shr r9,2
1646 add r15,r13
1647 adc r9,r14
1648 add r10,r15
1649 adc r11,r9
1650 adc r12,0
1651
1652 add r8,16
1653 cmp r8,rcx
1654 jb NEAR $L$open_sse_tail_256_hash
1655 paddd xmm3,XMMWORD[$L$chacha20_consts]
1656 paddd xmm7,XMMWORD[((160+48))+rbp]
1657 paddd xmm11,XMMWORD[((160+64))+rbp]
1658 paddd xmm15,XMMWORD[((160+144))+rbp]
1659 paddd xmm2,XMMWORD[$L$chacha20_consts]
1660 paddd xmm6,XMMWORD[((160+48))+rbp]
1661 paddd xmm10,XMMWORD[((160+64))+rbp]
1662 paddd xmm14,XMMWORD[((160+128))+rbp]
1663 paddd xmm1,XMMWORD[$L$chacha20_consts]
1664 paddd xmm5,XMMWORD[((160+48))+rbp]
1665 paddd xmm9,XMMWORD[((160+64))+rbp]
1666 paddd xmm13,XMMWORD[((160+112))+rbp]
1667 paddd xmm0,XMMWORD[$L$chacha20_consts]
1668 paddd xmm4,XMMWORD[((160+48))+rbp]
1669 paddd xmm8,XMMWORD[((160+64))+rbp]
1670 paddd xmm12,XMMWORD[((160+96))+rbp]
1671 movdqa XMMWORD[(160+80)+rbp],xmm12
1672 movdqu xmm12,XMMWORD[((0 + 0))+rsi]
1673 pxor xmm12,xmm3
1674 movdqu XMMWORD[(0 + 0)+rdi],xmm12
1675 movdqu xmm12,XMMWORD[((16 + 0))+rsi]
1676 pxor xmm12,xmm7
1677 movdqu XMMWORD[(16 + 0)+rdi],xmm12
1678 movdqu xmm12,XMMWORD[((32 + 0))+rsi]
1679 pxor xmm12,xmm11
1680 movdqu XMMWORD[(32 + 0)+rdi],xmm12
1681 movdqu xmm12,XMMWORD[((48 + 0))+rsi]
1682 pxor xmm12,xmm15
1683 movdqu XMMWORD[(48 + 0)+rdi],xmm12
1684 movdqu xmm3,XMMWORD[((0 + 64))+rsi]
1685 movdqu xmm7,XMMWORD[((16 + 64))+rsi]
1686 movdqu xmm11,XMMWORD[((32 + 64))+rsi]
1687 movdqu xmm15,XMMWORD[((48 + 64))+rsi]
1688 pxor xmm2,xmm3
1689 pxor xmm6,xmm7
1690 pxor xmm10,xmm11
1691 pxor xmm15,xmm14
1692 movdqu XMMWORD[(0 + 64)+rdi],xmm2
1693 movdqu XMMWORD[(16 + 64)+rdi],xmm6
1694 movdqu XMMWORD[(32 + 64)+rdi],xmm10
1695 movdqu XMMWORD[(48 + 64)+rdi],xmm15
1696 movdqu xmm3,XMMWORD[((0 + 128))+rsi]
1697 movdqu xmm7,XMMWORD[((16 + 128))+rsi]
1698 movdqu xmm11,XMMWORD[((32 + 128))+rsi]
1699 movdqu xmm15,XMMWORD[((48 + 128))+rsi]
1700 pxor xmm1,xmm3
1701 pxor xmm5,xmm7
1702 pxor xmm9,xmm11
1703 pxor xmm15,xmm13
1704 movdqu XMMWORD[(0 + 128)+rdi],xmm1
1705 movdqu XMMWORD[(16 + 128)+rdi],xmm5
1706 movdqu XMMWORD[(32 + 128)+rdi],xmm9
1707 movdqu XMMWORD[(48 + 128)+rdi],xmm15
1708
1709 movdqa xmm12,XMMWORD[((160+80))+rbp]
1710 sub rbx,12*16
1711 lea rsi,[192+rsi]
1712 lea rdi,[192+rdi]
1713
1714
1715 $L$open_sse_tail_64_dec_loop:
1716 cmp rbx,16
1717 jb NEAR $L$open_sse_tail_16_init
1718 sub rbx,16
1719 movdqu xmm3,XMMWORD[rsi]
1720 pxor xmm0,xmm3
1721 movdqu XMMWORD[rdi],xmm0
1722 lea rsi,[16+rsi]
1723 lea rdi,[16+rdi]
1724 movdqa xmm0,xmm4
1725 movdqa xmm4,xmm8
1726 movdqa xmm8,xmm12
1727 jmp NEAR $L$open_sse_tail_64_dec_loop
1728 $L$open_sse_tail_16_init:
1729 movdqa xmm1,xmm0
1730
1731
1732 $L$open_sse_tail_16:
1733 test rbx,rbx
1734 jz NEAR $L$open_sse_finalize
1735
1736
1737
1738 pxor xmm3,xmm3
1739 lea rsi,[((-1))+rbx*1+rsi]
1740 mov r8,rbx
1741 $L$open_sse_tail_16_compose:
1742 pslldq xmm3,1
1743 pinsrb xmm3,BYTE[rsi],0
1744 sub rsi,1
1745 sub r8,1
1746 jnz NEAR $L$open_sse_tail_16_compose
1747
1748 DB 102,73,15,126,221
1749 pextrq r14,xmm3,1
1750
1751 pxor xmm3,xmm1
1752
1753
1754 $L$open_sse_tail_16_extract:
1755 pextrb XMMWORD[rdi],xmm3,0
1756 psrldq xmm3,1
1757 add rdi,1
1758 sub rbx,1
1759 jne NEAR $L$open_sse_tail_16_extract
1760
1761 add r10,r13
1762 adc r11,r14
1763 adc r12,1
1764 mov rax,QWORD[((0+160+0))+rbp]
1765 mov r15,rax
1766 mul r10
1767 mov r13,rax
1768 mov r14,rdx
1769 mov rax,QWORD[((0+160+0))+rbp]
1770 mul r11
1771 imul r15,r12
1772 add r14,rax
1773 adc r15,rdx
1774 mov rax,QWORD[((8+160+0))+rbp]
1775 mov r9,rax
1776 mul r10
1777 add r14,rax
1778 adc rdx,0
1779 mov r10,rdx
1780 mov rax,QWORD[((8+160+0))+rbp]
1781 mul r11
1782 add r15,rax
1783 adc rdx,0
1784 imul r9,r12
1785 add r15,r10
1786 adc r9,rdx
1787 mov r10,r13
1788 mov r11,r14
1789 mov r12,r15
1790 and r12,3
1791 mov r13,r15
1792 and r13,-4
1793 mov r14,r9
1794 shrd r15,r9,2
1795 shr r9,2
1796 add r15,r13
1797 adc r9,r14
1798 add r10,r15
1799 adc r11,r9
1800 adc r12,0
1801
1802
1803 $L$open_sse_finalize:
1804 add r10,QWORD[((0+160+32))+rbp]
1805 adc r11,QWORD[((8+160+32))+rbp]
1806 adc r12,1
1807 mov rax,QWORD[((0+160+0))+rbp]
1808 mov r15,rax
1809 mul r10
1810 mov r13,rax
1811 mov r14,rdx
1812 mov rax,QWORD[((0+160+0))+rbp]
1813 mul r11
1814 imul r15,r12
1815 add r14,rax
1816 adc r15,rdx
1817 mov rax,QWORD[((8+160+0))+rbp]
1818 mov r9,rax
1819 mul r10
1820 add r14,rax
1821 adc rdx,0
1822 mov r10,rdx
1823 mov rax,QWORD[((8+160+0))+rbp]
1824 mul r11
1825 add r15,rax
1826 adc rdx,0
1827 imul r9,r12
1828 add r15,r10
1829 adc r9,rdx
1830 mov r10,r13
1831 mov r11,r14
1832 mov r12,r15
1833 and r12,3
1834 mov r13,r15
1835 and r13,-4
1836 mov r14,r9
1837 shrd r15,r9,2
1838 shr r9,2
1839 add r15,r13
1840 adc r9,r14
1841 add r10,r15
1842 adc r11,r9
1843 adc r12,0
1844
1845
1846 mov r13,r10
1847 mov r14,r11
1848 mov r15,r12
1849 sub r10,-5
1850 sbb r11,-1
1851 sbb r12,3
1852 cmovc r10,r13
1853 cmovc r11,r14
1854 cmovc r12,r15
1855
1856 add r10,QWORD[((0+160+16))+rbp]
1857 adc r11,QWORD[((8+160+16))+rbp]
1858
1859 movaps xmm6,XMMWORD[((0+0))+rbp]
1860 movaps xmm7,XMMWORD[((16+0))+rbp]
1861 movaps xmm8,XMMWORD[((32+0))+rbp]
1862 movaps xmm9,XMMWORD[((48+0))+rbp]
1863 movaps xmm10,XMMWORD[((64+0))+rbp]
1864 movaps xmm11,XMMWORD[((80+0))+rbp]
1865 movaps xmm12,XMMWORD[((96+0))+rbp]
1866 movaps xmm13,XMMWORD[((112+0))+rbp]
1867 movaps xmm14,XMMWORD[((128+0))+rbp]
1868 movaps xmm15,XMMWORD[((144+0))+rbp]
1869
1870
1871 add rsp,288 + 160 + 32
1872
1873
1874 pop r9
1875
1876 mov QWORD[r9],r10
1877 mov QWORD[8+r9],r11
1878 pop r15
1879
1880 pop r14
1881
1882 pop r13
1883
1884 pop r12
1885
1886 pop rbx
1887
1888 pop rbp
1889
1890 mov rdi,QWORD[8+rsp] ;WIN64 epilogue
1891 mov rsi,QWORD[16+rsp]
1892 DB 0F3h,0C3h ;repret
1893
1894 $L$open_sse_128:
1895
1896 movdqu xmm0,XMMWORD[$L$chacha20_consts]
1897 movdqa xmm1,xmm0
1898 movdqa xmm2,xmm0
1899 movdqu xmm4,XMMWORD[r9]
1900 movdqa xmm5,xmm4
1901 movdqa xmm6,xmm4
1902 movdqu xmm8,XMMWORD[16+r9]
1903 movdqa xmm9,xmm8
1904 movdqa xmm10,xmm8
1905 movdqu xmm12,XMMWORD[32+r9]
1906 movdqa xmm13,xmm12
1907 paddd xmm13,XMMWORD[$L$sse_inc]
1908 movdqa xmm14,xmm13
1909 paddd xmm14,XMMWORD[$L$sse_inc]
1910 movdqa xmm7,xmm4
1911 movdqa xmm11,xmm8
1912 movdqa xmm15,xmm13
1913 mov r10,10
1914
1915 $L$open_sse_128_rounds:
1916 paddd xmm0,xmm4
1917 pxor xmm12,xmm0
1918 pshufb xmm12,XMMWORD[$L$rol16]
1919 paddd xmm8,xmm12
1920 pxor xmm4,xmm8
1921 movdqa xmm3,xmm4
1922 pslld xmm3,12
1923 psrld xmm4,20
1924 pxor xmm4,xmm3
1925 paddd xmm0,xmm4
1926 pxor xmm12,xmm0
1927 pshufb xmm12,XMMWORD[$L$rol8]
1928 paddd xmm8,xmm12
1929 pxor xmm4,xmm8
1930 movdqa xmm3,xmm4
1931 pslld xmm3,7
1932 psrld xmm4,25
1933 pxor xmm4,xmm3
1934 DB 102,15,58,15,228,4
1935 DB 102,69,15,58,15,192,8
1936 DB 102,69,15,58,15,228,12
1937 paddd xmm1,xmm5
1938 pxor xmm13,xmm1
1939 pshufb xmm13,XMMWORD[$L$rol16]
1940 paddd xmm9,xmm13
1941 pxor xmm5,xmm9
1942 movdqa xmm3,xmm5
1943 pslld xmm3,12
1944 psrld xmm5,20
1945 pxor xmm5,xmm3
1946 paddd xmm1,xmm5
1947 pxor xmm13,xmm1
1948 pshufb xmm13,XMMWORD[$L$rol8]
1949 paddd xmm9,xmm13
1950 pxor xmm5,xmm9
1951 movdqa xmm3,xmm5
1952 pslld xmm3,7
1953 psrld xmm5,25
1954 pxor xmm5,xmm3
1955 DB 102,15,58,15,237,4
1956 DB 102,69,15,58,15,201,8
1957 DB 102,69,15,58,15,237,12
1958 paddd xmm2,xmm6
1959 pxor xmm14,xmm2
1960 pshufb xmm14,XMMWORD[$L$rol16]
1961 paddd xmm10,xmm14
1962 pxor xmm6,xmm10
1963 movdqa xmm3,xmm6
1964 pslld xmm3,12
1965 psrld xmm6,20
1966 pxor xmm6,xmm3
1967 paddd xmm2,xmm6
1968 pxor xmm14,xmm2
1969 pshufb xmm14,XMMWORD[$L$rol8]
1970 paddd xmm10,xmm14
1971 pxor xmm6,xmm10
1972 movdqa xmm3,xmm6
1973 pslld xmm3,7
1974 psrld xmm6,25
1975 pxor xmm6,xmm3
1976 DB 102,15,58,15,246,4
1977 DB 102,69,15,58,15,210,8
1978 DB 102,69,15,58,15,246,12
1979 paddd xmm0,xmm4
1980 pxor xmm12,xmm0
1981 pshufb xmm12,XMMWORD[$L$rol16]
1982 paddd xmm8,xmm12
1983 pxor xmm4,xmm8
1984 movdqa xmm3,xmm4
1985 pslld xmm3,12
1986 psrld xmm4,20
1987 pxor xmm4,xmm3
1988 paddd xmm0,xmm4
1989 pxor xmm12,xmm0
1990 pshufb xmm12,XMMWORD[$L$rol8]
1991 paddd xmm8,xmm12
1992 pxor xmm4,xmm8
1993 movdqa xmm3,xmm4
1994 pslld xmm3,7
1995 psrld xmm4,25
1996 pxor xmm4,xmm3
1997 DB 102,15,58,15,228,12
1998 DB 102,69,15,58,15,192,8
1999 DB 102,69,15,58,15,228,4
2000 paddd xmm1,xmm5
2001 pxor xmm13,xmm1
2002 pshufb xmm13,XMMWORD[$L$rol16]
2003 paddd xmm9,xmm13
2004 pxor xmm5,xmm9
2005 movdqa xmm3,xmm5
2006 pslld xmm3,12
2007 psrld xmm5,20
2008 pxor xmm5,xmm3
2009 paddd xmm1,xmm5
2010 pxor xmm13,xmm1
2011 pshufb xmm13,XMMWORD[$L$rol8]
2012 paddd xmm9,xmm13
2013 pxor xmm5,xmm9
2014 movdqa xmm3,xmm5
2015 pslld xmm3,7
2016 psrld xmm5,25
2017 pxor xmm5,xmm3
2018 DB 102,15,58,15,237,12
2019 DB 102,69,15,58,15,201,8
2020 DB 102,69,15,58,15,237,4
2021 paddd xmm2,xmm6
2022 pxor xmm14,xmm2
2023 pshufb xmm14,XMMWORD[$L$rol16]
2024 paddd xmm10,xmm14
2025 pxor xmm6,xmm10
2026 movdqa xmm3,xmm6
2027 pslld xmm3,12
2028 psrld xmm6,20
2029 pxor xmm6,xmm3
2030 paddd xmm2,xmm6
2031 pxor xmm14,xmm2
2032 pshufb xmm14,XMMWORD[$L$rol8]
2033 paddd xmm10,xmm14
2034 pxor xmm6,xmm10
2035 movdqa xmm3,xmm6
2036 pslld xmm3,7
2037 psrld xmm6,25
2038 pxor xmm6,xmm3
2039 DB 102,15,58,15,246,12
2040 DB 102,69,15,58,15,210,8
2041 DB 102,69,15,58,15,246,4
2042
2043 dec r10
2044 jnz NEAR $L$open_sse_128_rounds
2045 paddd xmm0,XMMWORD[$L$chacha20_consts]
2046 paddd xmm1,XMMWORD[$L$chacha20_consts]
2047 paddd xmm2,XMMWORD[$L$chacha20_consts]
2048 paddd xmm4,xmm7
2049 paddd xmm5,xmm7
2050 paddd xmm6,xmm7
2051 paddd xmm9,xmm11
2052 paddd xmm10,xmm11
2053 paddd xmm13,xmm15
2054 paddd xmm15,XMMWORD[$L$sse_inc]
2055 paddd xmm14,xmm15
2056
2057 pand xmm0,XMMWORD[$L$clamp]
2058 movdqa XMMWORD[(160+0)+rbp],xmm0
2059 movdqa XMMWORD[(160+16)+rbp],xmm4
2060
2061 mov r8,r8
2062 call poly_hash_ad_internal
2063 $L$open_sse_128_xor_hash:
2064 cmp rbx,16
2065 jb NEAR $L$open_sse_tail_16
2066 sub rbx,16
2067 add r10,QWORD[((0+0))+rsi]
2068 adc r11,QWORD[((8+0))+rsi]
2069 adc r12,1
2070
2071
2072 movdqu xmm3,XMMWORD[rsi]
2073 pxor xmm1,xmm3
2074 movdqu XMMWORD[rdi],xmm1
2075 lea rsi,[16+rsi]
2076 lea rdi,[16+rdi]
2077 mov rax,QWORD[((0+160+0))+rbp]
2078 mov r15,rax
2079 mul r10
2080 mov r13,rax
2081 mov r14,rdx
2082 mov rax,QWORD[((0+160+0))+rbp]
2083 mul r11
2084 imul r15,r12
2085 add r14,rax
2086 adc r15,rdx
2087 mov rax,QWORD[((8+160+0))+rbp]
2088 mov r9,rax
2089 mul r10
2090 add r14,rax
2091 adc rdx,0
2092 mov r10,rdx
2093 mov rax,QWORD[((8+160+0))+rbp]
2094 mul r11
2095 add r15,rax
2096 adc rdx,0
2097 imul r9,r12
2098 add r15,r10
2099 adc r9,rdx
2100 mov r10,r13
2101 mov r11,r14
2102 mov r12,r15
2103 and r12,3
2104 mov r13,r15
2105 and r13,-4
2106 mov r14,r9
2107 shrd r15,r9,2
2108 shr r9,2
2109 add r15,r13
2110 adc r9,r14
2111 add r10,r15
2112 adc r11,r9
2113 adc r12,0
2114
2115
2116 movdqa xmm1,xmm5
2117 movdqa xmm5,xmm9
2118 movdqa xmm9,xmm13
2119 movdqa xmm13,xmm2
2120 movdqa xmm2,xmm6
2121 movdqa xmm6,xmm10
2122 movdqa xmm10,xmm14
2123 jmp NEAR $L$open_sse_128_xor_hash
2124 $L$SEH_end_chacha20_poly1305_open:
2125
2126
2127
2128
2129
2130
2131
2132
2133 global chacha20_poly1305_seal
2134
2135 ALIGN 64
2136 chacha20_poly1305_seal:
2137 mov QWORD[8+rsp],rdi ;WIN64 prologue
2138 mov QWORD[16+rsp],rsi
2139 mov rax,rsp
2140 $L$SEH_begin_chacha20_poly1305_seal:
2141 mov rdi,rcx
2142 mov rsi,rdx
2143 mov rdx,r8
2144 mov rcx,r9
2145 mov r8,QWORD[40+rsp]
2146 mov r9,QWORD[48+rsp]
2147
2148
2149
2150 push rbp
2151
2152 push rbx
2153
2154 push r12
2155
2156 push r13
2157
2158 push r14
2159
2160 push r15
2161
2162
2163
2164 push r9
2165
2166 sub rsp,288 + 160 + 32
2167
2168 lea rbp,[32+rsp]
2169 and rbp,-32
2170
2171 movaps XMMWORD[(0+0)+rbp],xmm6
2172 movaps XMMWORD[(16+0)+rbp],xmm7
2173 movaps XMMWORD[(32+0)+rbp],xmm8
2174 movaps XMMWORD[(48+0)+rbp],xmm9
2175 movaps XMMWORD[(64+0)+rbp],xmm10
2176 movaps XMMWORD[(80+0)+rbp],xmm11
2177 movaps XMMWORD[(96+0)+rbp],xmm12
2178 movaps XMMWORD[(112+0)+rbp],xmm13
2179 movaps XMMWORD[(128+0)+rbp],xmm14
2180 movaps XMMWORD[(144+0)+rbp],xmm15
2181
2182 mov rbx,QWORD[56+r9]
2183 add rbx,rdx
2184 mov QWORD[((0+160+32))+rbp],r8
2185 mov QWORD[((8+160+32))+rbp],rbx
2186 mov rbx,rdx
2187
2188 mov eax,DWORD[((OPENSSL_ia32cap_P+8))]
2189 and eax,288
2190 xor eax,288
2191 jz NEAR chacha20_poly1305_seal_avx2
2192
2193 cmp rbx,128
2194 jbe NEAR $L$seal_sse_128
2195
2196 movdqa xmm0,XMMWORD[$L$chacha20_consts]
2197 movdqu xmm4,XMMWORD[r9]
2198 movdqu xmm8,XMMWORD[16+r9]
2199 movdqu xmm12,XMMWORD[32+r9]
2200
2201 movdqa xmm1,xmm0
2202 movdqa xmm2,xmm0
2203 movdqa xmm3,xmm0
2204 movdqa xmm5,xmm4
2205 movdqa xmm6,xmm4
2206 movdqa xmm7,xmm4
2207 movdqa xmm9,xmm8
2208 movdqa xmm10,xmm8
2209 movdqa xmm11,xmm8
2210 movdqa xmm15,xmm12
2211 paddd xmm12,XMMWORD[$L$sse_inc]
2212 movdqa xmm14,xmm12
2213 paddd xmm12,XMMWORD[$L$sse_inc]
2214 movdqa xmm13,xmm12
2215 paddd xmm12,XMMWORD[$L$sse_inc]
2216
2217 movdqa XMMWORD[(160+48)+rbp],xmm4
2218 movdqa XMMWORD[(160+64)+rbp],xmm8
2219 movdqa XMMWORD[(160+96)+rbp],xmm12
2220 movdqa XMMWORD[(160+112)+rbp],xmm13
2221 movdqa XMMWORD[(160+128)+rbp],xmm14
2222 movdqa XMMWORD[(160+144)+rbp],xmm15
2223 mov r10,10
2224 $L$seal_sse_init_rounds:
2225 movdqa XMMWORD[(160+80)+rbp],xmm8
2226 movdqa xmm8,XMMWORD[$L$rol16]
2227 paddd xmm3,xmm7
2228 paddd xmm2,xmm6
2229 paddd xmm1,xmm5
2230 paddd xmm0,xmm4
2231 pxor xmm15,xmm3
2232 pxor xmm14,xmm2
2233 pxor xmm13,xmm1
2234 pxor xmm12,xmm0
2235 DB 102,69,15,56,0,248
2236 DB 102,69,15,56,0,240
2237 DB 102,69,15,56,0,232
2238 DB 102,69,15,56,0,224
2239 movdqa xmm8,XMMWORD[((160+80))+rbp]
2240 paddd xmm11,xmm15
2241 paddd xmm10,xmm14
2242 paddd xmm9,xmm13
2243 paddd xmm8,xmm12
2244 pxor xmm7,xmm11
2245 pxor xmm6,xmm10
2246 pxor xmm5,xmm9
2247 pxor xmm4,xmm8
2248 movdqa XMMWORD[(160+80)+rbp],xmm8
2249 movdqa xmm8,xmm7
2250 psrld xmm8,20
2251 pslld xmm7,32-20
2252 pxor xmm7,xmm8
2253 movdqa xmm8,xmm6
2254 psrld xmm8,20
2255 pslld xmm6,32-20
2256 pxor xmm6,xmm8
2257 movdqa xmm8,xmm5
2258 psrld xmm8,20
2259 pslld xmm5,32-20
2260 pxor xmm5,xmm8
2261 movdqa xmm8,xmm4
2262 psrld xmm8,20
2263 pslld xmm4,32-20
2264 pxor xmm4,xmm8
2265 movdqa xmm8,XMMWORD[$L$rol8]
2266 paddd xmm3,xmm7
2267 paddd xmm2,xmm6
2268 paddd xmm1,xmm5
2269 paddd xmm0,xmm4
2270 pxor xmm15,xmm3
2271 pxor xmm14,xmm2
2272 pxor xmm13,xmm1
2273 pxor xmm12,xmm0
2274 DB 102,69,15,56,0,248
2275 DB 102,69,15,56,0,240
2276 DB 102,69,15,56,0,232
2277 DB 102,69,15,56,0,224
2278 movdqa xmm8,XMMWORD[((160+80))+rbp]
2279 paddd xmm11,xmm15
2280 paddd xmm10,xmm14
2281 paddd xmm9,xmm13
2282 paddd xmm8,xmm12
2283 pxor xmm7,xmm11
2284 pxor xmm6,xmm10
2285 pxor xmm5,xmm9
2286 pxor xmm4,xmm8
2287 movdqa XMMWORD[(160+80)+rbp],xmm8
2288 movdqa xmm8,xmm7
2289 psrld xmm8,25
2290 pslld xmm7,32-25
2291 pxor xmm7,xmm8
2292 movdqa xmm8,xmm6
2293 psrld xmm8,25
2294 pslld xmm6,32-25
2295 pxor xmm6,xmm8
2296 movdqa xmm8,xmm5
2297 psrld xmm8,25
2298 pslld xmm5,32-25
2299 pxor xmm5,xmm8
2300 movdqa xmm8,xmm4
2301 psrld xmm8,25
2302 pslld xmm4,32-25
2303 pxor xmm4,xmm8
2304 movdqa xmm8,XMMWORD[((160+80))+rbp]
2305 DB 102,15,58,15,255,4
2306 DB 102,69,15,58,15,219,8
2307 DB 102,69,15,58,15,255,12
2308 DB 102,15,58,15,246,4
2309 DB 102,69,15,58,15,210,8
2310 DB 102,69,15,58,15,246,12
2311 DB 102,15,58,15,237,4
2312 DB 102,69,15,58,15,201,8
2313 DB 102,69,15,58,15,237,12
2314 DB 102,15,58,15,228,4
2315 DB 102,69,15,58,15,192,8
2316 DB 102,69,15,58,15,228,12
2317 movdqa XMMWORD[(160+80)+rbp],xmm8
2318 movdqa xmm8,XMMWORD[$L$rol16]
2319 paddd xmm3,xmm7
2320 paddd xmm2,xmm6
2321 paddd xmm1,xmm5
2322 paddd xmm0,xmm4
2323 pxor xmm15,xmm3
2324 pxor xmm14,xmm2
2325 pxor xmm13,xmm1
2326 pxor xmm12,xmm0
2327 DB 102,69,15,56,0,248
2328 DB 102,69,15,56,0,240
2329 DB 102,69,15,56,0,232
2330 DB 102,69,15,56,0,224
2331 movdqa xmm8,XMMWORD[((160+80))+rbp]
2332 paddd xmm11,xmm15
2333 paddd xmm10,xmm14
2334 paddd xmm9,xmm13
2335 paddd xmm8,xmm12
2336 pxor xmm7,xmm11
2337 pxor xmm6,xmm10
2338 pxor xmm5,xmm9
2339 pxor xmm4,xmm8
2340 movdqa XMMWORD[(160+80)+rbp],xmm8
2341 movdqa xmm8,xmm7
2342 psrld xmm8,20
2343 pslld xmm7,32-20
2344 pxor xmm7,xmm8
2345 movdqa xmm8,xmm6
2346 psrld xmm8,20
2347 pslld xmm6,32-20
2348 pxor xmm6,xmm8
2349 movdqa xmm8,xmm5
2350 psrld xmm8,20
2351 pslld xmm5,32-20
2352 pxor xmm5,xmm8
2353 movdqa xmm8,xmm4
2354 psrld xmm8,20
2355 pslld xmm4,32-20
2356 pxor xmm4,xmm8
2357 movdqa xmm8,XMMWORD[$L$rol8]
2358 paddd xmm3,xmm7
2359 paddd xmm2,xmm6
2360 paddd xmm1,xmm5
2361 paddd xmm0,xmm4
2362 pxor xmm15,xmm3
2363 pxor xmm14,xmm2
2364 pxor xmm13,xmm1
2365 pxor xmm12,xmm0
2366 DB 102,69,15,56,0,248
2367 DB 102,69,15,56,0,240
2368 DB 102,69,15,56,0,232
2369 DB 102,69,15,56,0,224
2370 movdqa xmm8,XMMWORD[((160+80))+rbp]
2371 paddd xmm11,xmm15
2372 paddd xmm10,xmm14
2373 paddd xmm9,xmm13
2374 paddd xmm8,xmm12
2375 pxor xmm7,xmm11
2376 pxor xmm6,xmm10
2377 pxor xmm5,xmm9
2378 pxor xmm4,xmm8
2379 movdqa XMMWORD[(160+80)+rbp],xmm8
2380 movdqa xmm8,xmm7
2381 psrld xmm8,25
2382 pslld xmm7,32-25
2383 pxor xmm7,xmm8
2384 movdqa xmm8,xmm6
2385 psrld xmm8,25
2386 pslld xmm6,32-25
2387 pxor xmm6,xmm8
2388 movdqa xmm8,xmm5
2389 psrld xmm8,25
2390 pslld xmm5,32-25
2391 pxor xmm5,xmm8
2392 movdqa xmm8,xmm4
2393 psrld xmm8,25
2394 pslld xmm4,32-25
2395 pxor xmm4,xmm8
2396 movdqa xmm8,XMMWORD[((160+80))+rbp]
2397 DB 102,15,58,15,255,12
2398 DB 102,69,15,58,15,219,8
2399 DB 102,69,15,58,15,255,4
2400 DB 102,15,58,15,246,12
2401 DB 102,69,15,58,15,210,8
2402 DB 102,69,15,58,15,246,4
2403 DB 102,15,58,15,237,12
2404 DB 102,69,15,58,15,201,8
2405 DB 102,69,15,58,15,237,4
2406 DB 102,15,58,15,228,12
2407 DB 102,69,15,58,15,192,8
2408 DB 102,69,15,58,15,228,4
2409
2410 dec r10
2411 jnz NEAR $L$seal_sse_init_rounds
2412 paddd xmm3,XMMWORD[$L$chacha20_consts]
2413 paddd xmm7,XMMWORD[((160+48))+rbp]
2414 paddd xmm11,XMMWORD[((160+64))+rbp]
2415 paddd xmm15,XMMWORD[((160+144))+rbp]
2416 paddd xmm2,XMMWORD[$L$chacha20_consts]
2417 paddd xmm6,XMMWORD[((160+48))+rbp]
2418 paddd xmm10,XMMWORD[((160+64))+rbp]
2419 paddd xmm14,XMMWORD[((160+128))+rbp]
2420 paddd xmm1,XMMWORD[$L$chacha20_consts]
2421 paddd xmm5,XMMWORD[((160+48))+rbp]
2422 paddd xmm9,XMMWORD[((160+64))+rbp]
2423 paddd xmm13,XMMWORD[((160+112))+rbp]
2424 paddd xmm0,XMMWORD[$L$chacha20_consts]
2425 paddd xmm4,XMMWORD[((160+48))+rbp]
2426 paddd xmm8,XMMWORD[((160+64))+rbp]
2427 paddd xmm12,XMMWORD[((160+96))+rbp]
2428
2429
2430 pand xmm3,XMMWORD[$L$clamp]
2431 movdqa XMMWORD[(160+0)+rbp],xmm3
2432 movdqa XMMWORD[(160+16)+rbp],xmm7
2433
2434 mov r8,r8
2435 call poly_hash_ad_internal
2436 movdqu xmm3,XMMWORD[((0 + 0))+rsi]
2437 movdqu xmm7,XMMWORD[((16 + 0))+rsi]
2438 movdqu xmm11,XMMWORD[((32 + 0))+rsi]
2439 movdqu xmm15,XMMWORD[((48 + 0))+rsi]
2440 pxor xmm2,xmm3
2441 pxor xmm6,xmm7
2442 pxor xmm10,xmm11
2443 pxor xmm15,xmm14
2444 movdqu XMMWORD[(0 + 0)+rdi],xmm2
2445 movdqu XMMWORD[(16 + 0)+rdi],xmm6
2446 movdqu XMMWORD[(32 + 0)+rdi],xmm10
2447 movdqu XMMWORD[(48 + 0)+rdi],xmm15
2448 movdqu xmm3,XMMWORD[((0 + 64))+rsi]
2449 movdqu xmm7,XMMWORD[((16 + 64))+rsi]
2450 movdqu xmm11,XMMWORD[((32 + 64))+rsi]
2451 movdqu xmm15,XMMWORD[((48 + 64))+rsi]
2452 pxor xmm1,xmm3
2453 pxor xmm5,xmm7
2454 pxor xmm9,xmm11
2455 pxor xmm15,xmm13
2456 movdqu XMMWORD[(0 + 64)+rdi],xmm1
2457 movdqu XMMWORD[(16 + 64)+rdi],xmm5
2458 movdqu XMMWORD[(32 + 64)+rdi],xmm9
2459 movdqu XMMWORD[(48 + 64)+rdi],xmm15
2460
2461 cmp rbx,12*16
2462 ja NEAR $L$seal_sse_main_init
2463 mov rcx,8*16
2464 sub rbx,8*16
2465 lea rsi,[128+rsi]
2466 jmp NEAR $L$seal_sse_128_tail_hash
2467 $L$seal_sse_main_init:
2468 movdqu xmm3,XMMWORD[((0 + 128))+rsi]
2469 movdqu xmm7,XMMWORD[((16 + 128))+rsi]
2470 movdqu xmm11,XMMWORD[((32 + 128))+rsi]
2471 movdqu xmm15,XMMWORD[((48 + 128))+rsi]
2472 pxor xmm0,xmm3
2473 pxor xmm4,xmm7
2474 pxor xmm8,xmm11
2475 pxor xmm15,xmm12
2476 movdqu XMMWORD[(0 + 128)+rdi],xmm0
2477 movdqu XMMWORD[(16 + 128)+rdi],xmm4
2478 movdqu XMMWORD[(32 + 128)+rdi],xmm8
2479 movdqu XMMWORD[(48 + 128)+rdi],xmm15
2480
2481 mov rcx,12*16
2482 sub rbx,12*16
2483 lea rsi,[192+rsi]
2484 mov rcx,2
2485 mov r8,8
2486 cmp rbx,4*16
2487 jbe NEAR $L$seal_sse_tail_64
2488 cmp rbx,8*16
2489 jbe NEAR $L$seal_sse_tail_128
2490 cmp rbx,12*16
2491 jbe NEAR $L$seal_sse_tail_192
2492
2493 $L$seal_sse_main_loop:
2494 movdqa xmm0,XMMWORD[$L$chacha20_consts]
2495 movdqa xmm4,XMMWORD[((160+48))+rbp]
2496 movdqa xmm8,XMMWORD[((160+64))+rbp]
2497 movdqa xmm1,xmm0
2498 movdqa xmm5,xmm4
2499 movdqa xmm9,xmm8
2500 movdqa xmm2,xmm0
2501 movdqa xmm6,xmm4
2502 movdqa xmm10,xmm8
2503 movdqa xmm3,xmm0
2504 movdqa xmm7,xmm4
2505 movdqa xmm11,xmm8
2506 movdqa xmm15,XMMWORD[((160+96))+rbp]
2507 paddd xmm15,XMMWORD[$L$sse_inc]
2508 movdqa xmm14,xmm15
2509 paddd xmm14,XMMWORD[$L$sse_inc]
2510 movdqa xmm13,xmm14
2511 paddd xmm13,XMMWORD[$L$sse_inc]
2512 movdqa xmm12,xmm13
2513 paddd xmm12,XMMWORD[$L$sse_inc]
2514 movdqa XMMWORD[(160+96)+rbp],xmm12
2515 movdqa XMMWORD[(160+112)+rbp],xmm13
2516 movdqa XMMWORD[(160+128)+rbp],xmm14
2517 movdqa XMMWORD[(160+144)+rbp],xmm15
2518
2519 ALIGN 32
2520 $L$seal_sse_main_rounds:
2521 movdqa XMMWORD[(160+80)+rbp],xmm8
2522 movdqa xmm8,XMMWORD[$L$rol16]
2523 paddd xmm3,xmm7
2524 paddd xmm2,xmm6
2525 paddd xmm1,xmm5
2526 paddd xmm0,xmm4
2527 pxor xmm15,xmm3
2528 pxor xmm14,xmm2
2529 pxor xmm13,xmm1
2530 pxor xmm12,xmm0
2531 DB 102,69,15,56,0,248
2532 DB 102,69,15,56,0,240
2533 DB 102,69,15,56,0,232
2534 DB 102,69,15,56,0,224
2535 movdqa xmm8,XMMWORD[((160+80))+rbp]
2536 paddd xmm11,xmm15
2537 paddd xmm10,xmm14
2538 paddd xmm9,xmm13
2539 paddd xmm8,xmm12
2540 pxor xmm7,xmm11
2541 add r10,QWORD[((0+0))+rdi]
2542 adc r11,QWORD[((8+0))+rdi]
2543 adc r12,1
2544 pxor xmm6,xmm10
2545 pxor xmm5,xmm9
2546 pxor xmm4,xmm8
2547 movdqa XMMWORD[(160+80)+rbp],xmm8
2548 movdqa xmm8,xmm7
2549 psrld xmm8,20
2550 pslld xmm7,32-20
2551 pxor xmm7,xmm8
2552 movdqa xmm8,xmm6
2553 psrld xmm8,20
2554 pslld xmm6,32-20
2555 pxor xmm6,xmm8
2556 movdqa xmm8,xmm5
2557 psrld xmm8,20
2558 pslld xmm5,32-20
2559 pxor xmm5,xmm8
2560 movdqa xmm8,xmm4
2561 psrld xmm8,20
2562 pslld xmm4,32-20
2563 pxor xmm4,xmm8
2564 mov rax,QWORD[((0+160+0))+rbp]
2565 mov r15,rax
2566 mul r10
2567 mov r13,rax
2568 mov r14,rdx
2569 mov rax,QWORD[((0+160+0))+rbp]
2570 mul r11
2571 imul r15,r12
2572 add r14,rax
2573 adc r15,rdx
2574 movdqa xmm8,XMMWORD[$L$rol8]
2575 paddd xmm3,xmm7
2576 paddd xmm2,xmm6
2577 paddd xmm1,xmm5
2578 paddd xmm0,xmm4
2579 pxor xmm15,xmm3
2580 pxor xmm14,xmm2
2581 pxor xmm13,xmm1
2582 pxor xmm12,xmm0
2583 DB 102,69,15,56,0,248
2584 DB 102,69,15,56,0,240
2585 DB 102,69,15,56,0,232
2586 DB 102,69,15,56,0,224
2587 movdqa xmm8,XMMWORD[((160+80))+rbp]
2588 paddd xmm11,xmm15
2589 paddd xmm10,xmm14
2590 paddd xmm9,xmm13
2591 paddd xmm8,xmm12
2592 pxor xmm7,xmm11
2593 pxor xmm6,xmm10
2594 mov rax,QWORD[((8+160+0))+rbp]
2595 mov r9,rax
2596 mul r10
2597 add r14,rax
2598 adc rdx,0
2599 mov r10,rdx
2600 mov rax,QWORD[((8+160+0))+rbp]
2601 mul r11
2602 add r15,rax
2603 adc rdx,0
2604 pxor xmm5,xmm9
2605 pxor xmm4,xmm8
2606 movdqa XMMWORD[(160+80)+rbp],xmm8
2607 movdqa xmm8,xmm7
2608 psrld xmm8,25
2609 pslld xmm7,32-25
2610 pxor xmm7,xmm8
2611 movdqa xmm8,xmm6
2612 psrld xmm8,25
2613 pslld xmm6,32-25
2614 pxor xmm6,xmm8
2615 movdqa xmm8,xmm5
2616 psrld xmm8,25
2617 pslld xmm5,32-25
2618 pxor xmm5,xmm8
2619 movdqa xmm8,xmm4
2620 psrld xmm8,25
2621 pslld xmm4,32-25
2622 pxor xmm4,xmm8
2623 movdqa xmm8,XMMWORD[((160+80))+rbp]
2624 imul r9,r12
2625 add r15,r10
2626 adc r9,rdx
2627 DB 102,15,58,15,255,4
2628 DB 102,69,15,58,15,219,8
2629 DB 102,69,15,58,15,255,12
2630 DB 102,15,58,15,246,4
2631 DB 102,69,15,58,15,210,8
2632 DB 102,69,15,58,15,246,12
2633 DB 102,15,58,15,237,4
2634 DB 102,69,15,58,15,201,8
2635 DB 102,69,15,58,15,237,12
2636 DB 102,15,58,15,228,4
2637 DB 102,69,15,58,15,192,8
2638 DB 102,69,15,58,15,228,12
2639 movdqa XMMWORD[(160+80)+rbp],xmm8
2640 movdqa xmm8,XMMWORD[$L$rol16]
2641 paddd xmm3,xmm7
2642 paddd xmm2,xmm6
2643 paddd xmm1,xmm5
2644 paddd xmm0,xmm4
2645 pxor xmm15,xmm3
2646 pxor xmm14,xmm2
2647 mov r10,r13
2648 mov r11,r14
2649 mov r12,r15
2650 and r12,3
2651 mov r13,r15
2652 and r13,-4
2653 mov r14,r9
2654 shrd r15,r9,2
2655 shr r9,2
2656 add r15,r13
2657 adc r9,r14
2658 add r10,r15
2659 adc r11,r9
2660 adc r12,0
2661 pxor xmm13,xmm1
2662 pxor xmm12,xmm0
2663 DB 102,69,15,56,0,248
2664 DB 102,69,15,56,0,240
2665 DB 102,69,15,56,0,232
2666 DB 102,69,15,56,0,224
2667 movdqa xmm8,XMMWORD[((160+80))+rbp]
2668 paddd xmm11,xmm15
2669 paddd xmm10,xmm14
2670 paddd xmm9,xmm13
2671 paddd xmm8,xmm12
2672 pxor xmm7,xmm11
2673 pxor xmm6,xmm10
2674 pxor xmm5,xmm9
2675 pxor xmm4,xmm8
2676 movdqa XMMWORD[(160+80)+rbp],xmm8
2677 movdqa xmm8,xmm7
2678 psrld xmm8,20
2679 pslld xmm7,32-20
2680 pxor xmm7,xmm8
2681 movdqa xmm8,xmm6
2682 psrld xmm8,20
2683 pslld xmm6,32-20
2684 pxor xmm6,xmm8
2685 movdqa xmm8,xmm5
2686 psrld xmm8,20
2687 pslld xmm5,32-20
2688 pxor xmm5,xmm8
2689 movdqa xmm8,xmm4
2690 psrld xmm8,20
2691 pslld xmm4,32-20
2692 pxor xmm4,xmm8
2693 movdqa xmm8,XMMWORD[$L$rol8]
2694 paddd xmm3,xmm7
2695 paddd xmm2,xmm6
2696 paddd xmm1,xmm5
2697 paddd xmm0,xmm4
2698 pxor xmm15,xmm3
2699 pxor xmm14,xmm2
2700 pxor xmm13,xmm1
2701 pxor xmm12,xmm0
2702 DB 102,69,15,56,0,248
2703 DB 102,69,15,56,0,240
2704 DB 102,69,15,56,0,232
2705 DB 102,69,15,56,0,224
2706 movdqa xmm8,XMMWORD[((160+80))+rbp]
2707 paddd xmm11,xmm15
2708 paddd xmm10,xmm14
2709 paddd xmm9,xmm13
2710 paddd xmm8,xmm12
2711 pxor xmm7,xmm11
2712 pxor xmm6,xmm10
2713 pxor xmm5,xmm9
2714 pxor xmm4,xmm8
2715 movdqa XMMWORD[(160+80)+rbp],xmm8
2716 movdqa xmm8,xmm7
2717 psrld xmm8,25
2718 pslld xmm7,32-25
2719 pxor xmm7,xmm8
2720 movdqa xmm8,xmm6
2721 psrld xmm8,25
2722 pslld xmm6,32-25
2723 pxor xmm6,xmm8
2724 movdqa xmm8,xmm5
2725 psrld xmm8,25
2726 pslld xmm5,32-25
2727 pxor xmm5,xmm8
2728 movdqa xmm8,xmm4
2729 psrld xmm8,25
2730 pslld xmm4,32-25
2731 pxor xmm4,xmm8
2732 movdqa xmm8,XMMWORD[((160+80))+rbp]
2733 DB 102,15,58,15,255,12
2734 DB 102,69,15,58,15,219,8
2735 DB 102,69,15,58,15,255,4
2736 DB 102,15,58,15,246,12
2737 DB 102,69,15,58,15,210,8
2738 DB 102,69,15,58,15,246,4
2739 DB 102,15,58,15,237,12
2740 DB 102,69,15,58,15,201,8
2741 DB 102,69,15,58,15,237,4
2742 DB 102,15,58,15,228,12
2743 DB 102,69,15,58,15,192,8
2744 DB 102,69,15,58,15,228,4
2745
2746 lea rdi,[16+rdi]
2747 dec r8
2748 jge NEAR $L$seal_sse_main_rounds
2749 add r10,QWORD[((0+0))+rdi]
2750 adc r11,QWORD[((8+0))+rdi]
2751 adc r12,1
2752 mov rax,QWORD[((0+160+0))+rbp]
2753 mov r15,rax
2754 mul r10
2755 mov r13,rax
2756 mov r14,rdx
2757 mov rax,QWORD[((0+160+0))+rbp]
2758 mul r11
2759 imul r15,r12
2760 add r14,rax
2761 adc r15,rdx
2762 mov rax,QWORD[((8+160+0))+rbp]
2763 mov r9,rax
2764 mul r10
2765 add r14,rax
2766 adc rdx,0
2767 mov r10,rdx
2768 mov rax,QWORD[((8+160+0))+rbp]
2769 mul r11
2770 add r15,rax
2771 adc rdx,0
2772 imul r9,r12
2773 add r15,r10
2774 adc r9,rdx
2775 mov r10,r13
2776 mov r11,r14
2777 mov r12,r15
2778 and r12,3
2779 mov r13,r15
2780 and r13,-4
2781 mov r14,r9
2782 shrd r15,r9,2
2783 shr r9,2
2784 add r15,r13
2785 adc r9,r14
2786 add r10,r15
2787 adc r11,r9
2788 adc r12,0
2789
2790 lea rdi,[16+rdi]
2791 dec rcx
2792 jg NEAR $L$seal_sse_main_rounds
2793 paddd xmm3,XMMWORD[$L$chacha20_consts]
2794 paddd xmm7,XMMWORD[((160+48))+rbp]
2795 paddd xmm11,XMMWORD[((160+64))+rbp]
2796 paddd xmm15,XMMWORD[((160+144))+rbp]
2797 paddd xmm2,XMMWORD[$L$chacha20_consts]
2798 paddd xmm6,XMMWORD[((160+48))+rbp]
2799 paddd xmm10,XMMWORD[((160+64))+rbp]
2800 paddd xmm14,XMMWORD[((160+128))+rbp]
2801 paddd xmm1,XMMWORD[$L$chacha20_consts]
2802 paddd xmm5,XMMWORD[((160+48))+rbp]
2803 paddd xmm9,XMMWORD[((160+64))+rbp]
2804 paddd xmm13,XMMWORD[((160+112))+rbp]
2805 paddd xmm0,XMMWORD[$L$chacha20_consts]
2806 paddd xmm4,XMMWORD[((160+48))+rbp]
2807 paddd xmm8,XMMWORD[((160+64))+rbp]
2808 paddd xmm12,XMMWORD[((160+96))+rbp]
2809
2810 movdqa XMMWORD[(160+80)+rbp],xmm14
2811 movdqa XMMWORD[(160+80)+rbp],xmm14
2812 movdqu xmm14,XMMWORD[((0 + 0))+rsi]
2813 pxor xmm14,xmm3
2814 movdqu XMMWORD[(0 + 0)+rdi],xmm14
2815 movdqu xmm14,XMMWORD[((16 + 0))+rsi]
2816 pxor xmm14,xmm7
2817 movdqu XMMWORD[(16 + 0)+rdi],xmm14
2818 movdqu xmm14,XMMWORD[((32 + 0))+rsi]
2819 pxor xmm14,xmm11
2820 movdqu XMMWORD[(32 + 0)+rdi],xmm14
2821 movdqu xmm14,XMMWORD[((48 + 0))+rsi]
2822 pxor xmm14,xmm15
2823 movdqu XMMWORD[(48 + 0)+rdi],xmm14
2824
2825 movdqa xmm14,XMMWORD[((160+80))+rbp]
2826 movdqu xmm3,XMMWORD[((0 + 64))+rsi]
2827 movdqu xmm7,XMMWORD[((16 + 64))+rsi]
2828 movdqu xmm11,XMMWORD[((32 + 64))+rsi]
2829 movdqu xmm15,XMMWORD[((48 + 64))+rsi]
2830 pxor xmm2,xmm3
2831 pxor xmm6,xmm7
2832 pxor xmm10,xmm11
2833 pxor xmm15,xmm14
2834 movdqu XMMWORD[(0 + 64)+rdi],xmm2
2835 movdqu XMMWORD[(16 + 64)+rdi],xmm6
2836 movdqu XMMWORD[(32 + 64)+rdi],xmm10
2837 movdqu XMMWORD[(48 + 64)+rdi],xmm15
2838 movdqu xmm3,XMMWORD[((0 + 128))+rsi]
2839 movdqu xmm7,XMMWORD[((16 + 128))+rsi]
2840 movdqu xmm11,XMMWORD[((32 + 128))+rsi]
2841 movdqu xmm15,XMMWORD[((48 + 128))+rsi]
2842 pxor xmm1,xmm3
2843 pxor xmm5,xmm7
2844 pxor xmm9,xmm11
2845 pxor xmm15,xmm13
2846 movdqu XMMWORD[(0 + 128)+rdi],xmm1
2847 movdqu XMMWORD[(16 + 128)+rdi],xmm5
2848 movdqu XMMWORD[(32 + 128)+rdi],xmm9
2849 movdqu XMMWORD[(48 + 128)+rdi],xmm15
2850
2851 cmp rbx,16*16
2852 ja NEAR $L$seal_sse_main_loop_xor
2853
2854 mov rcx,12*16
2855 sub rbx,12*16
2856 lea rsi,[192+rsi]
2857 jmp NEAR $L$seal_sse_128_tail_hash
2858 $L$seal_sse_main_loop_xor:
2859 movdqu xmm3,XMMWORD[((0 + 192))+rsi]
2860 movdqu xmm7,XMMWORD[((16 + 192))+rsi]
2861 movdqu xmm11,XMMWORD[((32 + 192))+rsi]
2862 movdqu xmm15,XMMWORD[((48 + 192))+rsi]
2863 pxor xmm0,xmm3
2864 pxor xmm4,xmm7
2865 pxor xmm8,xmm11
2866 pxor xmm15,xmm12
2867 movdqu XMMWORD[(0 + 192)+rdi],xmm0
2868 movdqu XMMWORD[(16 + 192)+rdi],xmm4
2869 movdqu XMMWORD[(32 + 192)+rdi],xmm8
2870 movdqu XMMWORD[(48 + 192)+rdi],xmm15
2871
2872 lea rsi,[256+rsi]
2873 sub rbx,16*16
2874 mov rcx,6
2875 mov r8,4
2876 cmp rbx,12*16
2877 jg NEAR $L$seal_sse_main_loop
2878 mov rcx,rbx
2879 test rbx,rbx
2880 je NEAR $L$seal_sse_128_tail_hash
2881 mov rcx,6
2882 cmp rbx,8*16
2883 ja NEAR $L$seal_sse_tail_192
2884 cmp rbx,4*16
2885 ja NEAR $L$seal_sse_tail_128
2886
2887 $L$seal_sse_tail_64:
2888 movdqa xmm0,XMMWORD[$L$chacha20_consts]
2889 movdqa xmm4,XMMWORD[((160+48))+rbp]
2890 movdqa xmm8,XMMWORD[((160+64))+rbp]
2891 movdqa xmm12,XMMWORD[((160+96))+rbp]
2892 paddd xmm12,XMMWORD[$L$sse_inc]
2893 movdqa XMMWORD[(160+96)+rbp],xmm12
2894
2895 $L$seal_sse_tail_64_rounds_and_x2hash:
2896 add r10,QWORD[((0+0))+rdi]
2897 adc r11,QWORD[((8+0))+rdi]
2898 adc r12,1
2899 mov rax,QWORD[((0+160+0))+rbp]
2900 mov r15,rax
2901 mul r10
2902 mov r13,rax
2903 mov r14,rdx
2904 mov rax,QWORD[((0+160+0))+rbp]
2905 mul r11
2906 imul r15,r12
2907 add r14,rax
2908 adc r15,rdx
2909 mov rax,QWORD[((8+160+0))+rbp]
2910 mov r9,rax
2911 mul r10
2912 add r14,rax
2913 adc rdx,0
2914 mov r10,rdx
2915 mov rax,QWORD[((8+160+0))+rbp]
2916 mul r11
2917 add r15,rax
2918 adc rdx,0
2919 imul r9,r12
2920 add r15,r10
2921 adc r9,rdx
2922 mov r10,r13
2923 mov r11,r14
2924 mov r12,r15
2925 and r12,3
2926 mov r13,r15
2927 and r13,-4
2928 mov r14,r9
2929 shrd r15,r9,2
2930 shr r9,2
2931 add r15,r13
2932 adc r9,r14
2933 add r10,r15
2934 adc r11,r9
2935 adc r12,0
2936
2937 lea rdi,[16+rdi]
2938 $L$seal_sse_tail_64_rounds_and_x1hash:
2939 paddd xmm0,xmm4
2940 pxor xmm12,xmm0
2941 pshufb xmm12,XMMWORD[$L$rol16]
2942 paddd xmm8,xmm12
2943 pxor xmm4,xmm8
2944 movdqa xmm3,xmm4
2945 pslld xmm3,12
2946 psrld xmm4,20
2947 pxor xmm4,xmm3
2948 paddd xmm0,xmm4
2949 pxor xmm12,xmm0
2950 pshufb xmm12,XMMWORD[$L$rol8]
2951 paddd xmm8,xmm12
2952 pxor xmm4,xmm8
2953 movdqa xmm3,xmm4
2954 pslld xmm3,7
2955 psrld xmm4,25
2956 pxor xmm4,xmm3
2957 DB 102,15,58,15,228,4
2958 DB 102,69,15,58,15,192,8
2959 DB 102,69,15,58,15,228,12
2960 paddd xmm0,xmm4
2961 pxor xmm12,xmm0
2962 pshufb xmm12,XMMWORD[$L$rol16]
2963 paddd xmm8,xmm12
2964 pxor xmm4,xmm8
2965 movdqa xmm3,xmm4
2966 pslld xmm3,12
2967 psrld xmm4,20
2968 pxor xmm4,xmm3
2969 paddd xmm0,xmm4
2970 pxor xmm12,xmm0
2971 pshufb xmm12,XMMWORD[$L$rol8]
2972 paddd xmm8,xmm12
2973 pxor xmm4,xmm8
2974 movdqa xmm3,xmm4
2975 pslld xmm3,7
2976 psrld xmm4,25
2977 pxor xmm4,xmm3
2978 DB 102,15,58,15,228,12
2979 DB 102,69,15,58,15,192,8
2980 DB 102,69,15,58,15,228,4
2981 add r10,QWORD[((0+0))+rdi]
2982 adc r11,QWORD[((8+0))+rdi]
2983 adc r12,1
2984 mov rax,QWORD[((0+160+0))+rbp]
2985 mov r15,rax
2986 mul r10
2987 mov r13,rax
2988 mov r14,rdx
2989 mov rax,QWORD[((0+160+0))+rbp]
2990 mul r11
2991 imul r15,r12
2992 add r14,rax
2993 adc r15,rdx
2994 mov rax,QWORD[((8+160+0))+rbp]
2995 mov r9,rax
2996 mul r10
2997 add r14,rax
2998 adc rdx,0
2999 mov r10,rdx
3000 mov rax,QWORD[((8+160+0))+rbp]
3001 mul r11
3002 add r15,rax
3003 adc rdx,0
3004 imul r9,r12
3005 add r15,r10
3006 adc r9,rdx
3007 mov r10,r13
3008 mov r11,r14
3009 mov r12,r15
3010 and r12,3
3011 mov r13,r15
3012 and r13,-4
3013 mov r14,r9
3014 shrd r15,r9,2
3015 shr r9,2
3016 add r15,r13
3017 adc r9,r14
3018 add r10,r15
3019 adc r11,r9
3020 adc r12,0
3021
3022 lea rdi,[16+rdi]
3023 dec rcx
3024 jg NEAR $L$seal_sse_tail_64_rounds_and_x2hash
3025 dec r8
3026 jge NEAR $L$seal_sse_tail_64_rounds_and_x1hash
3027 paddd xmm0,XMMWORD[$L$chacha20_consts]
3028 paddd xmm4,XMMWORD[((160+48))+rbp]
3029 paddd xmm8,XMMWORD[((160+64))+rbp]
3030 paddd xmm12,XMMWORD[((160+96))+rbp]
3031
3032 jmp NEAR $L$seal_sse_128_tail_xor
3033
3034 $L$seal_sse_tail_128:
3035 movdqa xmm0,XMMWORD[$L$chacha20_consts]
3036 movdqa xmm4,XMMWORD[((160+48))+rbp]
3037 movdqa xmm8,XMMWORD[((160+64))+rbp]
3038 movdqa xmm1,xmm0
3039 movdqa xmm5,xmm4
3040 movdqa xmm9,xmm8
3041 movdqa xmm13,XMMWORD[((160+96))+rbp]
3042 paddd xmm13,XMMWORD[$L$sse_inc]
3043 movdqa xmm12,xmm13
3044 paddd xmm12,XMMWORD[$L$sse_inc]
3045 movdqa XMMWORD[(160+96)+rbp],xmm12
3046 movdqa XMMWORD[(160+112)+rbp],xmm13
3047
3048 $L$seal_sse_tail_128_rounds_and_x2hash:
3049 add r10,QWORD[((0+0))+rdi]
3050 adc r11,QWORD[((8+0))+rdi]
3051 adc r12,1
3052 mov rax,QWORD[((0+160+0))+rbp]
3053 mov r15,rax
3054 mul r10
3055 mov r13,rax
3056 mov r14,rdx
3057 mov rax,QWORD[((0+160+0))+rbp]
3058 mul r11
3059 imul r15,r12
3060 add r14,rax
3061 adc r15,rdx
3062 mov rax,QWORD[((8+160+0))+rbp]
3063 mov r9,rax
3064 mul r10
3065 add r14,rax
3066 adc rdx,0
3067 mov r10,rdx
3068 mov rax,QWORD[((8+160+0))+rbp]
3069 mul r11
3070 add r15,rax
3071 adc rdx,0
3072 imul r9,r12
3073 add r15,r10
3074 adc r9,rdx
3075 mov r10,r13
3076 mov r11,r14
3077 mov r12,r15
3078 and r12,3
3079 mov r13,r15
3080 and r13,-4
3081 mov r14,r9
3082 shrd r15,r9,2
3083 shr r9,2
3084 add r15,r13
3085 adc r9,r14
3086 add r10,r15
3087 adc r11,r9
3088 adc r12,0
3089
3090 lea rdi,[16+rdi]
3091 $L$seal_sse_tail_128_rounds_and_x1hash:
3092 paddd xmm0,xmm4
3093 pxor xmm12,xmm0
3094 pshufb xmm12,XMMWORD[$L$rol16]
3095 paddd xmm8,xmm12
3096 pxor xmm4,xmm8
3097 movdqa xmm3,xmm4
3098 pslld xmm3,12
3099 psrld xmm4,20
3100 pxor xmm4,xmm3
3101 paddd xmm0,xmm4
3102 pxor xmm12,xmm0
3103 pshufb xmm12,XMMWORD[$L$rol8]
3104 paddd xmm8,xmm12
3105 pxor xmm4,xmm8
3106 movdqa xmm3,xmm4
3107 pslld xmm3,7
3108 psrld xmm4,25
3109 pxor xmm4,xmm3
3110 DB 102,15,58,15,228,4
3111 DB 102,69,15,58,15,192,8
3112 DB 102,69,15,58,15,228,12
3113 paddd xmm1,xmm5
3114 pxor xmm13,xmm1
3115 pshufb xmm13,XMMWORD[$L$rol16]
3116 paddd xmm9,xmm13
3117 pxor xmm5,xmm9
3118 movdqa xmm3,xmm5
3119 pslld xmm3,12
3120 psrld xmm5,20
3121 pxor xmm5,xmm3
3122 paddd xmm1,xmm5
3123 pxor xmm13,xmm1
3124 pshufb xmm13,XMMWORD[$L$rol8]
3125 paddd xmm9,xmm13
3126 pxor xmm5,xmm9
3127 movdqa xmm3,xmm5
3128 pslld xmm3,7
3129 psrld xmm5,25
3130 pxor xmm5,xmm3
3131 DB 102,15,58,15,237,4
3132 DB 102,69,15,58,15,201,8
3133 DB 102,69,15,58,15,237,12
3134 add r10,QWORD[((0+0))+rdi]
3135 adc r11,QWORD[((8+0))+rdi]
3136 adc r12,1
3137 mov rax,QWORD[((0+160+0))+rbp]
3138 mov r15,rax
3139 mul r10
3140 mov r13,rax
3141 mov r14,rdx
3142 mov rax,QWORD[((0+160+0))+rbp]
3143 mul r11
3144 imul r15,r12
3145 add r14,rax
3146 adc r15,rdx
3147 mov rax,QWORD[((8+160+0))+rbp]
3148 mov r9,rax
3149 mul r10
3150 add r14,rax
3151 adc rdx,0
3152 mov r10,rdx
3153 mov rax,QWORD[((8+160+0))+rbp]
3154 mul r11
3155 add r15,rax
3156 adc rdx,0
3157 imul r9,r12
3158 add r15,r10
3159 adc r9,rdx
3160 mov r10,r13
3161 mov r11,r14
3162 mov r12,r15
3163 and r12,3
3164 mov r13,r15
3165 and r13,-4
3166 mov r14,r9
3167 shrd r15,r9,2
3168 shr r9,2
3169 add r15,r13
3170 adc r9,r14
3171 add r10,r15
3172 adc r11,r9
3173 adc r12,0
3174 paddd xmm0,xmm4
3175 pxor xmm12,xmm0
3176 pshufb xmm12,XMMWORD[$L$rol16]
3177 paddd xmm8,xmm12
3178 pxor xmm4,xmm8
3179 movdqa xmm3,xmm4
3180 pslld xmm3,12
3181 psrld xmm4,20
3182 pxor xmm4,xmm3
3183 paddd xmm0,xmm4
3184 pxor xmm12,xmm0
3185 pshufb xmm12,XMMWORD[$L$rol8]
3186 paddd xmm8,xmm12
3187 pxor xmm4,xmm8
3188 movdqa xmm3,xmm4
3189 pslld xmm3,7
3190 psrld xmm4,25
3191 pxor xmm4,xmm3
3192 DB 102,15,58,15,228,12
3193 DB 102,69,15,58,15,192,8
3194 DB 102,69,15,58,15,228,4
3195 paddd xmm1,xmm5
3196 pxor xmm13,xmm1
3197 pshufb xmm13,XMMWORD[$L$rol16]
3198 paddd xmm9,xmm13
3199 pxor xmm5,xmm9
3200 movdqa xmm3,xmm5
3201 pslld xmm3,12
3202 psrld xmm5,20
3203 pxor xmm5,xmm3
3204 paddd xmm1,xmm5
3205 pxor xmm13,xmm1
3206 pshufb xmm13,XMMWORD[$L$rol8]
3207 paddd xmm9,xmm13
3208 pxor xmm5,xmm9
3209 movdqa xmm3,xmm5
3210 pslld xmm3,7
3211 psrld xmm5,25
3212 pxor xmm5,xmm3
3213 DB 102,15,58,15,237,12
3214 DB 102,69,15,58,15,201,8
3215 DB 102,69,15,58,15,237,4
3216
3217 lea rdi,[16+rdi]
3218 dec rcx
3219 jg NEAR $L$seal_sse_tail_128_rounds_and_x2hash
3220 dec r8
3221 jge NEAR $L$seal_sse_tail_128_rounds_and_x1hash
3222 paddd xmm1,XMMWORD[$L$chacha20_consts]
3223 paddd xmm5,XMMWORD[((160+48))+rbp]
3224 paddd xmm9,XMMWORD[((160+64))+rbp]
3225 paddd xmm13,XMMWORD[((160+112))+rbp]
3226 paddd xmm0,XMMWORD[$L$chacha20_consts]
3227 paddd xmm4,XMMWORD[((160+48))+rbp]
3228 paddd xmm8,XMMWORD[((160+64))+rbp]
3229 paddd xmm12,XMMWORD[((160+96))+rbp]
3230 movdqu xmm3,XMMWORD[((0 + 0))+rsi]
3231 movdqu xmm7,XMMWORD[((16 + 0))+rsi]
3232 movdqu xmm11,XMMWORD[((32 + 0))+rsi]
3233 movdqu xmm15,XMMWORD[((48 + 0))+rsi]
3234 pxor xmm1,xmm3
3235 pxor xmm5,xmm7
3236 pxor xmm9,xmm11
3237 pxor xmm15,xmm13
3238 movdqu XMMWORD[(0 + 0)+rdi],xmm1
3239 movdqu XMMWORD[(16 + 0)+rdi],xmm5
3240 movdqu XMMWORD[(32 + 0)+rdi],xmm9
3241 movdqu XMMWORD[(48 + 0)+rdi],xmm15
3242
3243 mov rcx,4*16
3244 sub rbx,4*16
3245 lea rsi,[64+rsi]
3246 jmp NEAR $L$seal_sse_128_tail_hash
3247
3248 $L$seal_sse_tail_192:
3249 movdqa xmm0,XMMWORD[$L$chacha20_consts]
3250 movdqa xmm4,XMMWORD[((160+48))+rbp]
3251 movdqa xmm8,XMMWORD[((160+64))+rbp]
3252 movdqa xmm1,xmm0
3253 movdqa xmm5,xmm4
3254 movdqa xmm9,xmm8
3255 movdqa xmm2,xmm0
3256 movdqa xmm6,xmm4
3257 movdqa xmm10,xmm8
3258 movdqa xmm14,XMMWORD[((160+96))+rbp]
3259 paddd xmm14,XMMWORD[$L$sse_inc]
3260 movdqa xmm13,xmm14
3261 paddd xmm13,XMMWORD[$L$sse_inc]
3262 movdqa xmm12,xmm13
3263 paddd xmm12,XMMWORD[$L$sse_inc]
3264 movdqa XMMWORD[(160+96)+rbp],xmm12
3265 movdqa XMMWORD[(160+112)+rbp],xmm13
3266 movdqa XMMWORD[(160+128)+rbp],xmm14
3267
3268 $L$seal_sse_tail_192_rounds_and_x2hash:
3269 add r10,QWORD[((0+0))+rdi]
3270 adc r11,QWORD[((8+0))+rdi]
3271 adc r12,1
3272 mov rax,QWORD[((0+160+0))+rbp]
3273 mov r15,rax
3274 mul r10
3275 mov r13,rax
3276 mov r14,rdx
3277 mov rax,QWORD[((0+160+0))+rbp]
3278 mul r11
3279 imul r15,r12
3280 add r14,rax
3281 adc r15,rdx
3282 mov rax,QWORD[((8+160+0))+rbp]
3283 mov r9,rax
3284 mul r10
3285 add r14,rax
3286 adc rdx,0
3287 mov r10,rdx
3288 mov rax,QWORD[((8+160+0))+rbp]
3289 mul r11
3290 add r15,rax
3291 adc rdx,0
3292 imul r9,r12
3293 add r15,r10
3294 adc r9,rdx
3295 mov r10,r13
3296 mov r11,r14
3297 mov r12,r15
3298 and r12,3
3299 mov r13,r15
3300 and r13,-4
3301 mov r14,r9
3302 shrd r15,r9,2
3303 shr r9,2
3304 add r15,r13
3305 adc r9,r14
3306 add r10,r15
3307 adc r11,r9
3308 adc r12,0
3309
3310 lea rdi,[16+rdi]
3311 $L$seal_sse_tail_192_rounds_and_x1hash:
3312 paddd xmm0,xmm4
3313 pxor xmm12,xmm0
3314 pshufb xmm12,XMMWORD[$L$rol16]
3315 paddd xmm8,xmm12
3316 pxor xmm4,xmm8
3317 movdqa xmm3,xmm4
3318 pslld xmm3,12
3319 psrld xmm4,20
3320 pxor xmm4,xmm3
3321 paddd xmm0,xmm4
3322 pxor xmm12,xmm0
3323 pshufb xmm12,XMMWORD[$L$rol8]
3324 paddd xmm8,xmm12
3325 pxor xmm4,xmm8
3326 movdqa xmm3,xmm4
3327 pslld xmm3,7
3328 psrld xmm4,25
3329 pxor xmm4,xmm3
3330 DB 102,15,58,15,228,4
3331 DB 102,69,15,58,15,192,8
3332 DB 102,69,15,58,15,228,12
3333 paddd xmm1,xmm5
3334 pxor xmm13,xmm1
3335 pshufb xmm13,XMMWORD[$L$rol16]
3336 paddd xmm9,xmm13
3337 pxor xmm5,xmm9
3338 movdqa xmm3,xmm5
3339 pslld xmm3,12
3340 psrld xmm5,20
3341 pxor xmm5,xmm3
3342 paddd xmm1,xmm5
3343 pxor xmm13,xmm1
3344 pshufb xmm13,XMMWORD[$L$rol8]
3345 paddd xmm9,xmm13
3346 pxor xmm5,xmm9
3347 movdqa xmm3,xmm5
3348 pslld xmm3,7
3349 psrld xmm5,25
3350 pxor xmm5,xmm3
3351 DB 102,15,58,15,237,4
3352 DB 102,69,15,58,15,201,8
3353 DB 102,69,15,58,15,237,12
3354 paddd xmm2,xmm6
3355 pxor xmm14,xmm2
3356 pshufb xmm14,XMMWORD[$L$rol16]
3357 paddd xmm10,xmm14
3358 pxor xmm6,xmm10
3359 movdqa xmm3,xmm6
3360 pslld xmm3,12
3361 psrld xmm6,20
3362 pxor xmm6,xmm3
3363 paddd xmm2,xmm6
3364 pxor xmm14,xmm2
3365 pshufb xmm14,XMMWORD[$L$rol8]
3366 paddd xmm10,xmm14
3367 pxor xmm6,xmm10
3368 movdqa xmm3,xmm6
3369 pslld xmm3,7
3370 psrld xmm6,25
3371 pxor xmm6,xmm3
3372 DB 102,15,58,15,246,4
3373 DB 102,69,15,58,15,210,8
3374 DB 102,69,15,58,15,246,12
3375 add r10,QWORD[((0+0))+rdi]
3376 adc r11,QWORD[((8+0))+rdi]
3377 adc r12,1
3378 mov rax,QWORD[((0+160+0))+rbp]
3379 mov r15,rax
3380 mul r10
3381 mov r13,rax
3382 mov r14,rdx
3383 mov rax,QWORD[((0+160+0))+rbp]
3384 mul r11
3385 imul r15,r12
3386 add r14,rax
3387 adc r15,rdx
3388 mov rax,QWORD[((8+160+0))+rbp]
3389 mov r9,rax
3390 mul r10
3391 add r14,rax
3392 adc rdx,0
3393 mov r10,rdx
3394 mov rax,QWORD[((8+160+0))+rbp]
3395 mul r11
3396 add r15,rax
3397 adc rdx,0
3398 imul r9,r12
3399 add r15,r10
3400 adc r9,rdx
3401 mov r10,r13
3402 mov r11,r14
3403 mov r12,r15
3404 and r12,3
3405 mov r13,r15
3406 and r13,-4
3407 mov r14,r9
3408 shrd r15,r9,2
3409 shr r9,2
3410 add r15,r13
3411 adc r9,r14
3412 add r10,r15
3413 adc r11,r9
3414 adc r12,0
3415 paddd xmm0,xmm4
3416 pxor xmm12,xmm0
3417 pshufb xmm12,XMMWORD[$L$rol16]
3418 paddd xmm8,xmm12
3419 pxor xmm4,xmm8
3420 movdqa xmm3,xmm4
3421 pslld xmm3,12
3422 psrld xmm4,20
3423 pxor xmm4,xmm3
3424 paddd xmm0,xmm4
3425 pxor xmm12,xmm0
3426 pshufb xmm12,XMMWORD[$L$rol8]
3427 paddd xmm8,xmm12
3428 pxor xmm4,xmm8
3429 movdqa xmm3,xmm4
3430 pslld xmm3,7
3431 psrld xmm4,25
3432 pxor xmm4,xmm3
3433 DB 102,15,58,15,228,12
3434 DB 102,69,15,58,15,192,8
3435 DB 102,69,15,58,15,228,4
3436 paddd xmm1,xmm5
3437 pxor xmm13,xmm1
3438 pshufb xmm13,XMMWORD[$L$rol16]
3439 paddd xmm9,xmm13
3440 pxor xmm5,xmm9
3441 movdqa xmm3,xmm5
3442 pslld xmm3,12
3443 psrld xmm5,20
3444 pxor xmm5,xmm3
3445 paddd xmm1,xmm5
3446 pxor xmm13,xmm1
3447 pshufb xmm13,XMMWORD[$L$rol8]
3448 paddd xmm9,xmm13
3449 pxor xmm5,xmm9
3450 movdqa xmm3,xmm5
3451 pslld xmm3,7
3452 psrld xmm5,25
3453 pxor xmm5,xmm3
3454 DB 102,15,58,15,237,12
3455 DB 102,69,15,58,15,201,8
3456 DB 102,69,15,58,15,237,4
3457 paddd xmm2,xmm6
3458 pxor xmm14,xmm2
3459 pshufb xmm14,XMMWORD[$L$rol16]
3460 paddd xmm10,xmm14
3461 pxor xmm6,xmm10
3462 movdqa xmm3,xmm6
3463 pslld xmm3,12
3464 psrld xmm6,20
3465 pxor xmm6,xmm3
3466 paddd xmm2,xmm6
3467 pxor xmm14,xmm2
3468 pshufb xmm14,XMMWORD[$L$rol8]
3469 paddd xmm10,xmm14
3470 pxor xmm6,xmm10
3471 movdqa xmm3,xmm6
3472 pslld xmm3,7
3473 psrld xmm6,25
3474 pxor xmm6,xmm3
3475 DB 102,15,58,15,246,12
3476 DB 102,69,15,58,15,210,8
3477 DB 102,69,15,58,15,246,4
3478
3479 lea rdi,[16+rdi]
3480 dec rcx
3481 jg NEAR $L$seal_sse_tail_192_rounds_and_x2hash
3482 dec r8
3483 jge NEAR $L$seal_sse_tail_192_rounds_and_x1hash
3484 paddd xmm2,XMMWORD[$L$chacha20_consts]
3485 paddd xmm6,XMMWORD[((160+48))+rbp]
3486 paddd xmm10,XMMWORD[((160+64))+rbp]
3487 paddd xmm14,XMMWORD[((160+128))+rbp]
3488 paddd xmm1,XMMWORD[$L$chacha20_consts]
3489 paddd xmm5,XMMWORD[((160+48))+rbp]
3490 paddd xmm9,XMMWORD[((160+64))+rbp]
3491 paddd xmm13,XMMWORD[((160+112))+rbp]
3492 paddd xmm0,XMMWORD[$L$chacha20_consts]
3493 paddd xmm4,XMMWORD[((160+48))+rbp]
3494 paddd xmm8,XMMWORD[((160+64))+rbp]
3495 paddd xmm12,XMMWORD[((160+96))+rbp]
3496 movdqu xmm3,XMMWORD[((0 + 0))+rsi]
3497 movdqu xmm7,XMMWORD[((16 + 0))+rsi]
3498 movdqu xmm11,XMMWORD[((32 + 0))+rsi]
3499 movdqu xmm15,XMMWORD[((48 + 0))+rsi]
3500 pxor xmm2,xmm3
3501 pxor xmm6,xmm7
3502 pxor xmm10,xmm11
3503 pxor xmm15,xmm14
3504 movdqu XMMWORD[(0 + 0)+rdi],xmm2
3505 movdqu XMMWORD[(16 + 0)+rdi],xmm6
3506 movdqu XMMWORD[(32 + 0)+rdi],xmm10
3507 movdqu XMMWORD[(48 + 0)+rdi],xmm15
3508 movdqu xmm3,XMMWORD[((0 + 64))+rsi]
3509 movdqu xmm7,XMMWORD[((16 + 64))+rsi]
3510 movdqu xmm11,XMMWORD[((32 + 64))+rsi]
3511 movdqu xmm15,XMMWORD[((48 + 64))+rsi]
3512 pxor xmm1,xmm3
3513 pxor xmm5,xmm7
3514 pxor xmm9,xmm11
3515 pxor xmm15,xmm13
3516 movdqu XMMWORD[(0 + 64)+rdi],xmm1
3517 movdqu XMMWORD[(16 + 64)+rdi],xmm5
3518 movdqu XMMWORD[(32 + 64)+rdi],xmm9
3519 movdqu XMMWORD[(48 + 64)+rdi],xmm15
3520
3521 mov rcx,8*16
3522 sub rbx,8*16
3523 lea rsi,[128+rsi]
3524
3525 $L$seal_sse_128_tail_hash:
3526 cmp rcx,16
3527 jb NEAR $L$seal_sse_128_tail_xor
3528 add r10,QWORD[((0+0))+rdi]
3529 adc r11,QWORD[((8+0))+rdi]
3530 adc r12,1
3531 mov rax,QWORD[((0+160+0))+rbp]
3532 mov r15,rax
3533 mul r10
3534 mov r13,rax
3535 mov r14,rdx
3536 mov rax,QWORD[((0+160+0))+rbp]
3537 mul r11
3538 imul r15,r12
3539 add r14,rax
3540 adc r15,rdx
3541 mov rax,QWORD[((8+160+0))+rbp]
3542 mov r9,rax
3543 mul r10
3544 add r14,rax
3545 adc rdx,0
3546 mov r10,rdx
3547 mov rax,QWORD[((8+160+0))+rbp]
3548 mul r11
3549 add r15,rax
3550 adc rdx,0
3551 imul r9,r12
3552 add r15,r10
3553 adc r9,rdx
3554 mov r10,r13
3555 mov r11,r14
3556 mov r12,r15
3557 and r12,3
3558 mov r13,r15
3559 and r13,-4
3560 mov r14,r9
3561 shrd r15,r9,2
3562 shr r9,2
3563 add r15,r13
3564 adc r9,r14
3565 add r10,r15
3566 adc r11,r9
3567 adc r12,0
3568
3569 sub rcx,16
3570 lea rdi,[16+rdi]
3571 jmp NEAR $L$seal_sse_128_tail_hash
3572
3573 $L$seal_sse_128_tail_xor:
3574 cmp rbx,16
3575 jb NEAR $L$seal_sse_tail_16
3576 sub rbx,16
3577
3578 movdqu xmm3,XMMWORD[rsi]
3579 pxor xmm0,xmm3
3580 movdqu XMMWORD[rdi],xmm0
3581
3582 add r10,QWORD[rdi]
3583 adc r11,QWORD[8+rdi]
3584 adc r12,1
3585 lea rsi,[16+rsi]
3586 lea rdi,[16+rdi]
3587 mov rax,QWORD[((0+160+0))+rbp]
3588 mov r15,rax
3589 mul r10
3590 mov r13,rax
3591 mov r14,rdx
3592 mov rax,QWORD[((0+160+0))+rbp]
3593 mul r11
3594 imul r15,r12
3595 add r14,rax
3596 adc r15,rdx
3597 mov rax,QWORD[((8+160+0))+rbp]
3598 mov r9,rax
3599 mul r10
3600 add r14,rax
3601 adc rdx,0
3602 mov r10,rdx
3603 mov rax,QWORD[((8+160+0))+rbp]
3604 mul r11
3605 add r15,rax
3606 adc rdx,0
3607 imul r9,r12
3608 add r15,r10
3609 adc r9,rdx
3610 mov r10,r13
3611 mov r11,r14
3612 mov r12,r15
3613 and r12,3
3614 mov r13,r15
3615 and r13,-4
3616 mov r14,r9
3617 shrd r15,r9,2
3618 shr r9,2
3619 add r15,r13
3620 adc r9,r14
3621 add r10,r15
3622 adc r11,r9
3623 adc r12,0
3624
3625
3626 movdqa xmm0,xmm4
3627 movdqa xmm4,xmm8
3628 movdqa xmm8,xmm12
3629 movdqa xmm12,xmm1
3630 movdqa xmm1,xmm5
3631 movdqa xmm5,xmm9
3632 movdqa xmm9,xmm13
3633 jmp NEAR $L$seal_sse_128_tail_xor
3634
3635 $L$seal_sse_tail_16:
3636 test rbx,rbx
3637 jz NEAR $L$process_blocks_of_extra_in
3638
3639 mov r8,rbx
3640 mov rcx,rbx
3641 lea rsi,[((-1))+rbx*1+rsi]
3642 pxor xmm15,xmm15
3643 $L$seal_sse_tail_16_compose:
3644 pslldq xmm15,1
3645 pinsrb xmm15,BYTE[rsi],0
3646 lea rsi,[((-1))+rsi]
3647 dec rcx
3648 jne NEAR $L$seal_sse_tail_16_compose
3649
3650
3651 pxor xmm15,xmm0
3652
3653
3654 mov rcx,rbx
3655 movdqu xmm0,xmm15
3656 $L$seal_sse_tail_16_extract:
3657 pextrb XMMWORD[rdi],xmm0,0
3658 psrldq xmm0,1
3659 add rdi,1
3660 sub rcx,1
3661 jnz NEAR $L$seal_sse_tail_16_extract
3662
3663
3664
3665
3666
3667
3668
3669
3670 mov r9,QWORD[((288 + 160 + 32))+rsp]
3671 mov r14,QWORD[56+r9]
3672 mov r13,QWORD[48+r9]
3673 test r14,r14
3674 jz NEAR $L$process_partial_block
3675
3676 mov r15,16
3677 sub r15,rbx
3678 cmp r14,r15
3679
3680 jge NEAR $L$load_extra_in
3681 mov r15,r14
3682
3683 $L$load_extra_in:
3684
3685
3686 lea rsi,[((-1))+r15*1+r13]
3687
3688
3689 add r13,r15
3690 sub r14,r15
3691 mov QWORD[48+r9],r13
3692 mov QWORD[56+r9],r14
3693
3694
3695
3696 add r8,r15
3697
3698
3699 pxor xmm11,xmm11
3700 $L$load_extra_load_loop:
3701 pslldq xmm11,1
3702 pinsrb xmm11,BYTE[rsi],0
3703 lea rsi,[((-1))+rsi]
3704 sub r15,1
3705 jnz NEAR $L$load_extra_load_loop
3706
3707
3708
3709
3710 mov r15,rbx
3711
3712 $L$load_extra_shift_loop:
3713 pslldq xmm11,1
3714 sub r15,1
3715 jnz NEAR $L$load_extra_shift_loop
3716
3717
3718
3719
3720 lea r15,[$L$and_masks]
3721 shl rbx,4
3722 pand xmm15,XMMWORD[((-16))+rbx*1+r15]
3723
3724
3725 por xmm15,xmm11
3726
3727
3728
3729 DB 102,77,15,126,253
3730 pextrq r14,xmm15,1
3731 add r10,r13
3732 adc r11,r14
3733 adc r12,1
3734 mov rax,QWORD[((0+160+0))+rbp]
3735 mov r15,rax
3736 mul r10
3737 mov r13,rax
3738 mov r14,rdx
3739 mov rax,QWORD[((0+160+0))+rbp]
3740 mul r11
3741 imul r15,r12
3742 add r14,rax
3743 adc r15,rdx
3744 mov rax,QWORD[((8+160+0))+rbp]
3745 mov r9,rax
3746 mul r10
3747 add r14,rax
3748 adc rdx,0
3749 mov r10,rdx
3750 mov rax,QWORD[((8+160+0))+rbp]
3751 mul r11
3752 add r15,rax
3753 adc rdx,0
3754 imul r9,r12
3755 add r15,r10
3756 adc r9,rdx
3757 mov r10,r13
3758 mov r11,r14
3759 mov r12,r15
3760 and r12,3
3761 mov r13,r15
3762 and r13,-4
3763 mov r14,r9
3764 shrd r15,r9,2
3765 shr r9,2
3766 add r15,r13
3767 adc r9,r14
3768 add r10,r15
3769 adc r11,r9
3770 adc r12,0
3771
3772
3773 $L$process_blocks_of_extra_in:
3774
3775 mov r9,QWORD[((288+32+160 ))+rsp]
3776 mov rsi,QWORD[48+r9]
3777 mov r8,QWORD[56+r9]
3778 mov rcx,r8
3779 shr r8,4
3780
3781 $L$process_extra_hash_loop:
3782 jz NEAR process_extra_in_trailer
3783 add r10,QWORD[((0+0))+rsi]
3784 adc r11,QWORD[((8+0))+rsi]
3785 adc r12,1
3786 mov rax,QWORD[((0+160+0))+rbp]
3787 mov r15,rax
3788 mul r10
3789 mov r13,rax
3790 mov r14,rdx
3791 mov rax,QWORD[((0+160+0))+rbp]
3792 mul r11
3793 imul r15,r12
3794 add r14,rax
3795 adc r15,rdx
3796 mov rax,QWORD[((8+160+0))+rbp]
3797 mov r9,rax
3798 mul r10
3799 add r14,rax
3800 adc rdx,0
3801 mov r10,rdx
3802 mov rax,QWORD[((8+160+0))+rbp]
3803 mul r11
3804 add r15,rax
3805 adc rdx,0
3806 imul r9,r12
3807 add r15,r10
3808 adc r9,rdx
3809 mov r10,r13
3810 mov r11,r14
3811 mov r12,r15
3812 and r12,3
3813 mov r13,r15
3814 and r13,-4
3815 mov r14,r9
3816 shrd r15,r9,2
3817 shr r9,2
3818 add r15,r13
3819 adc r9,r14
3820 add r10,r15
3821 adc r11,r9
3822 adc r12,0
3823
3824 lea rsi,[16+rsi]
3825 sub r8,1
3826 jmp NEAR $L$process_extra_hash_loop
3827 process_extra_in_trailer:
3828 and rcx,15
3829 mov rbx,rcx
3830 jz NEAR $L$do_length_block
3831 lea rsi,[((-1))+rcx*1+rsi]
3832
3833 $L$process_extra_in_trailer_load:
3834 pslldq xmm15,1
3835 pinsrb xmm15,BYTE[rsi],0
3836 lea rsi,[((-1))+rsi]
3837 sub rcx,1
3838 jnz NEAR $L$process_extra_in_trailer_load
3839
3840 $L$process_partial_block:
3841
3842 lea r15,[$L$and_masks]
3843 shl rbx,4
3844 pand xmm15,XMMWORD[((-16))+rbx*1+r15]
3845 DB 102,77,15,126,253
3846 pextrq r14,xmm15,1
3847 add r10,r13
3848 adc r11,r14
3849 adc r12,1
3850 mov rax,QWORD[((0+160+0))+rbp]
3851 mov r15,rax
3852 mul r10
3853 mov r13,rax
3854 mov r14,rdx
3855 mov rax,QWORD[((0+160+0))+rbp]
3856 mul r11
3857 imul r15,r12
3858 add r14,rax
3859 adc r15,rdx
3860 mov rax,QWORD[((8+160+0))+rbp]
3861 mov r9,rax
3862 mul r10
3863 add r14,rax
3864 adc rdx,0
3865 mov r10,rdx
3866 mov rax,QWORD[((8+160+0))+rbp]
3867 mul r11
3868 add r15,rax
3869 adc rdx,0
3870 imul r9,r12
3871 add r15,r10
3872 adc r9,rdx
3873 mov r10,r13
3874 mov r11,r14
3875 mov r12,r15
3876 and r12,3
3877 mov r13,r15
3878 and r13,-4
3879 mov r14,r9
3880 shrd r15,r9,2
3881 shr r9,2
3882 add r15,r13
3883 adc r9,r14
3884 add r10,r15
3885 adc r11,r9
3886 adc r12,0
3887
3888
3889 $L$do_length_block:
3890 add r10,QWORD[((0+160+32))+rbp]
3891 adc r11,QWORD[((8+160+32))+rbp]
3892 adc r12,1
3893 mov rax,QWORD[((0+160+0))+rbp]
3894 mov r15,rax
3895 mul r10
3896 mov r13,rax
3897 mov r14,rdx
3898 mov rax,QWORD[((0+160+0))+rbp]
3899 mul r11
3900 imul r15,r12
3901 add r14,rax
3902 adc r15,rdx
3903 mov rax,QWORD[((8+160+0))+rbp]
3904 mov r9,rax
3905 mul r10
3906 add r14,rax
3907 adc rdx,0
3908 mov r10,rdx
3909 mov rax,QWORD[((8+160+0))+rbp]
3910 mul r11
3911 add r15,rax
3912 adc rdx,0
3913 imul r9,r12
3914 add r15,r10
3915 adc r9,rdx
3916 mov r10,r13
3917 mov r11,r14
3918 mov r12,r15
3919 and r12,3
3920 mov r13,r15
3921 and r13,-4
3922 mov r14,r9
3923 shrd r15,r9,2
3924 shr r9,2
3925 add r15,r13
3926 adc r9,r14
3927 add r10,r15
3928 adc r11,r9
3929 adc r12,0
3930
3931
3932 mov r13,r10
3933 mov r14,r11
3934 mov r15,r12
3935 sub r10,-5
3936 sbb r11,-1
3937 sbb r12,3
3938 cmovc r10,r13
3939 cmovc r11,r14
3940 cmovc r12,r15
3941
3942 add r10,QWORD[((0+160+16))+rbp]
3943 adc r11,QWORD[((8+160+16))+rbp]
3944
3945 movaps xmm6,XMMWORD[((0+0))+rbp]
3946 movaps xmm7,XMMWORD[((16+0))+rbp]
3947 movaps xmm8,XMMWORD[((32+0))+rbp]
3948 movaps xmm9,XMMWORD[((48+0))+rbp]
3949 movaps xmm10,XMMWORD[((64+0))+rbp]
3950 movaps xmm11,XMMWORD[((80+0))+rbp]
3951 movaps xmm12,XMMWORD[((96+0))+rbp]
3952 movaps xmm13,XMMWORD[((112+0))+rbp]
3953 movaps xmm14,XMMWORD[((128+0))+rbp]
3954 movaps xmm15,XMMWORD[((144+0))+rbp]
3955
3956
3957 add rsp,288 + 160 + 32
3958
3959
3960 pop r9
3961
3962 mov QWORD[r9],r10
3963 mov QWORD[8+r9],r11
3964 pop r15
3965
3966 pop r14
3967
3968 pop r13
3969
3970 pop r12
3971
3972 pop rbx
3973
3974 pop rbp
3975
3976 mov rdi,QWORD[8+rsp] ;WIN64 epilogue
3977 mov rsi,QWORD[16+rsp]
3978 DB 0F3h,0C3h ;repret
3979
3980 $L$seal_sse_128:
3981
3982 movdqu xmm0,XMMWORD[$L$chacha20_consts]
3983 movdqa xmm1,xmm0
3984 movdqa xmm2,xmm0
3985 movdqu xmm4,XMMWORD[r9]
3986 movdqa xmm5,xmm4
3987 movdqa xmm6,xmm4
3988 movdqu xmm8,XMMWORD[16+r9]
3989 movdqa xmm9,xmm8
3990 movdqa xmm10,xmm8
3991 movdqu xmm14,XMMWORD[32+r9]
3992 movdqa xmm12,xmm14
3993 paddd xmm12,XMMWORD[$L$sse_inc]
3994 movdqa xmm13,xmm12
3995 paddd xmm13,XMMWORD[$L$sse_inc]
3996 movdqa xmm7,xmm4
3997 movdqa xmm11,xmm8
3998 movdqa xmm15,xmm12
3999 mov r10,10
4000
4001 $L$seal_sse_128_rounds:
4002 paddd xmm0,xmm4
4003 pxor xmm12,xmm0
4004 pshufb xmm12,XMMWORD[$L$rol16]
4005 paddd xmm8,xmm12
4006 pxor xmm4,xmm8
4007 movdqa xmm3,xmm4
4008 pslld xmm3,12
4009 psrld xmm4,20
4010 pxor xmm4,xmm3
4011 paddd xmm0,xmm4
4012 pxor xmm12,xmm0
4013 pshufb xmm12,XMMWORD[$L$rol8]
4014 paddd xmm8,xmm12
4015 pxor xmm4,xmm8
4016 movdqa xmm3,xmm4
4017 pslld xmm3,7
4018 psrld xmm4,25
4019 pxor xmm4,xmm3
4020 DB 102,15,58,15,228,4
4021 DB 102,69,15,58,15,192,8
4022 DB 102,69,15,58,15,228,12
4023 paddd xmm1,xmm5
4024 pxor xmm13,xmm1
4025 pshufb xmm13,XMMWORD[$L$rol16]
4026 paddd xmm9,xmm13
4027 pxor xmm5,xmm9
4028 movdqa xmm3,xmm5
4029 pslld xmm3,12
4030 psrld xmm5,20
4031 pxor xmm5,xmm3
4032 paddd xmm1,xmm5
4033 pxor xmm13,xmm1
4034 pshufb xmm13,XMMWORD[$L$rol8]
4035 paddd xmm9,xmm13
4036 pxor xmm5,xmm9
4037 movdqa xmm3,xmm5
4038 pslld xmm3,7
4039 psrld xmm5,25
4040 pxor xmm5,xmm3
4041 DB 102,15,58,15,237,4
4042 DB 102,69,15,58,15,201,8
4043 DB 102,69,15,58,15,237,12
4044 paddd xmm2,xmm6
4045 pxor xmm14,xmm2
4046 pshufb xmm14,XMMWORD[$L$rol16]
4047 paddd xmm10,xmm14
4048 pxor xmm6,xmm10
4049 movdqa xmm3,xmm6
4050 pslld xmm3,12
4051 psrld xmm6,20
4052 pxor xmm6,xmm3
4053 paddd xmm2,xmm6
4054 pxor xmm14,xmm2
4055 pshufb xmm14,XMMWORD[$L$rol8]
4056 paddd xmm10,xmm14
4057 pxor xmm6,xmm10
4058 movdqa xmm3,xmm6
4059 pslld xmm3,7
4060 psrld xmm6,25
4061 pxor xmm6,xmm3
4062 DB 102,15,58,15,246,4
4063 DB 102,69,15,58,15,210,8
4064 DB 102,69,15,58,15,246,12
4065 paddd xmm0,xmm4
4066 pxor xmm12,xmm0
4067 pshufb xmm12,XMMWORD[$L$rol16]
4068 paddd xmm8,xmm12
4069 pxor xmm4,xmm8
4070 movdqa xmm3,xmm4
4071 pslld xmm3,12
4072 psrld xmm4,20
4073 pxor xmm4,xmm3
4074 paddd xmm0,xmm4
4075 pxor xmm12,xmm0
4076 pshufb xmm12,XMMWORD[$L$rol8]
4077 paddd xmm8,xmm12
4078 pxor xmm4,xmm8
4079 movdqa xmm3,xmm4
4080 pslld xmm3,7
4081 psrld xmm4,25
4082 pxor xmm4,xmm3
4083 DB 102,15,58,15,228,12
4084 DB 102,69,15,58,15,192,8
4085 DB 102,69,15,58,15,228,4
4086 paddd xmm1,xmm5
4087 pxor xmm13,xmm1
4088 pshufb xmm13,XMMWORD[$L$rol16]
4089 paddd xmm9,xmm13
4090 pxor xmm5,xmm9
4091 movdqa xmm3,xmm5
4092 pslld xmm3,12
4093 psrld xmm5,20
4094 pxor xmm5,xmm3
4095 paddd xmm1,xmm5
4096 pxor xmm13,xmm1
4097 pshufb xmm13,XMMWORD[$L$rol8]
4098 paddd xmm9,xmm13
4099 pxor xmm5,xmm9
4100 movdqa xmm3,xmm5
4101 pslld xmm3,7
4102 psrld xmm5,25
4103 pxor xmm5,xmm3
4104 DB 102,15,58,15,237,12
4105 DB 102,69,15,58,15,201,8
4106 DB 102,69,15,58,15,237,4
4107 paddd xmm2,xmm6
4108 pxor xmm14,xmm2
4109 pshufb xmm14,XMMWORD[$L$rol16]
4110 paddd xmm10,xmm14
4111 pxor xmm6,xmm10
4112 movdqa xmm3,xmm6
4113 pslld xmm3,12
4114 psrld xmm6,20
4115 pxor xmm6,xmm3
4116 paddd xmm2,xmm6
4117 pxor xmm14,xmm2
4118 pshufb xmm14,XMMWORD[$L$rol8]
4119 paddd xmm10,xmm14
4120 pxor xmm6,xmm10
4121 movdqa xmm3,xmm6
4122 pslld xmm3,7
4123 psrld xmm6,25
4124 pxor xmm6,xmm3
4125 DB 102,15,58,15,246,12
4126 DB 102,69,15,58,15,210,8
4127 DB 102,69,15,58,15,246,4
4128
4129 dec r10
4130 jnz NEAR $L$seal_sse_128_rounds
4131 paddd xmm0,XMMWORD[$L$chacha20_consts]
4132 paddd xmm1,XMMWORD[$L$chacha20_consts]
4133 paddd xmm2,XMMWORD[$L$chacha20_consts]
4134 paddd xmm4,xmm7
4135 paddd xmm5,xmm7
4136 paddd xmm6,xmm7
4137 paddd xmm8,xmm11
4138 paddd xmm9,xmm11
4139 paddd xmm12,xmm15
4140 paddd xmm15,XMMWORD[$L$sse_inc]
4141 paddd xmm13,xmm15
4142
4143 pand xmm2,XMMWORD[$L$clamp]
4144 movdqa XMMWORD[(160+0)+rbp],xmm2
4145 movdqa XMMWORD[(160+16)+rbp],xmm6
4146
4147 mov r8,r8
4148 call poly_hash_ad_internal
4149 jmp NEAR $L$seal_sse_128_tail_xor
4150 $L$SEH_end_chacha20_poly1305_seal:
4151
4152
4153
4154
4155 ALIGN 64
4156 chacha20_poly1305_open_avx2:
4157
4158
4159
4160
4161
4162
4163
4164
4165
4166
4167
4168
4169 vzeroupper
4170 vmovdqa ymm0,YMMWORD[$L$chacha20_consts]
4171 vbroadcasti128 ymm4,XMMWORD[r9]
4172 vbroadcasti128 ymm8,XMMWORD[16+r9]
4173 vbroadcasti128 ymm12,XMMWORD[32+r9]
4174 vpaddd ymm12,ymm12,YMMWORD[$L$avx2_init]
4175 cmp rbx,6*32
4176 jbe NEAR $L$open_avx2_192
4177 cmp rbx,10*32
4178 jbe NEAR $L$open_avx2_320
4179
4180 vmovdqa YMMWORD[(160+64)+rbp],ymm4
4181 vmovdqa YMMWORD[(160+96)+rbp],ymm8
4182 vmovdqa YMMWORD[(160+160)+rbp],ymm12
4183 mov r10,10
4184 $L$open_avx2_init_rounds:
4185 vpaddd ymm0,ymm0,ymm4
4186 vpxor ymm12,ymm12,ymm0
4187 vpshufb ymm12,ymm12,YMMWORD[$L$rol16]
4188 vpaddd ymm8,ymm8,ymm12
4189 vpxor ymm4,ymm4,ymm8
4190 vpsrld ymm3,ymm4,20
4191 vpslld ymm4,ymm4,12
4192 vpxor ymm4,ymm4,ymm3
4193 vpaddd ymm0,ymm0,ymm4
4194 vpxor ymm12,ymm12,ymm0
4195 vpshufb ymm12,ymm12,YMMWORD[$L$rol8]
4196 vpaddd ymm8,ymm8,ymm12
4197 vpxor ymm4,ymm4,ymm8
4198 vpslld ymm3,ymm4,7
4199 vpsrld ymm4,ymm4,25
4200 vpxor ymm4,ymm4,ymm3
4201 vpalignr ymm12,ymm12,ymm12,12
4202 vpalignr ymm8,ymm8,ymm8,8
4203 vpalignr ymm4,ymm4,ymm4,4
4204 vpaddd ymm0,ymm0,ymm4
4205 vpxor ymm12,ymm12,ymm0
4206 vpshufb ymm12,ymm12,YMMWORD[$L$rol16]
4207 vpaddd ymm8,ymm8,ymm12
4208 vpxor ymm4,ymm4,ymm8
4209 vpsrld ymm3,ymm4,20
4210 vpslld ymm4,ymm4,12
4211 vpxor ymm4,ymm4,ymm3
4212 vpaddd ymm0,ymm0,ymm4
4213 vpxor ymm12,ymm12,ymm0
4214 vpshufb ymm12,ymm12,YMMWORD[$L$rol8]
4215 vpaddd ymm8,ymm8,ymm12
4216 vpxor ymm4,ymm4,ymm8
4217 vpslld ymm3,ymm4,7
4218 vpsrld ymm4,ymm4,25
4219 vpxor ymm4,ymm4,ymm3
4220 vpalignr ymm12,ymm12,ymm12,4
4221 vpalignr ymm8,ymm8,ymm8,8
4222 vpalignr ymm4,ymm4,ymm4,12
4223
4224 dec r10
4225 jne NEAR $L$open_avx2_init_rounds
4226 vpaddd ymm0,ymm0,YMMWORD[$L$chacha20_consts]
4227 vpaddd ymm4,ymm4,YMMWORD[((160+64))+rbp]
4228 vpaddd ymm8,ymm8,YMMWORD[((160+96))+rbp]
4229 vpaddd ymm12,ymm12,YMMWORD[((160+160))+rbp]
4230
4231 vperm2i128 ymm3,ymm4,ymm0,0x02
4232
4233 vpand ymm3,ymm3,YMMWORD[$L$clamp]
4234 vmovdqa YMMWORD[(160+0)+rbp],ymm3
4235
4236 vperm2i128 ymm0,ymm4,ymm0,0x13
4237 vperm2i128 ymm4,ymm12,ymm8,0x13
4238
4239 mov r8,r8
4240 call poly_hash_ad_internal
4241
4242 xor rcx,rcx
4243 $L$open_avx2_init_hash:
4244 add r10,QWORD[((0+0))+rcx*1+rsi]
4245 adc r11,QWORD[((8+0))+rcx*1+rsi]
4246 adc r12,1
4247 mov rax,QWORD[((0+160+0))+rbp]
4248 mov r15,rax
4249 mul r10
4250 mov r13,rax
4251 mov r14,rdx
4252 mov rax,QWORD[((0+160+0))+rbp]
4253 mul r11
4254 imul r15,r12
4255 add r14,rax
4256 adc r15,rdx
4257 mov rax,QWORD[((8+160+0))+rbp]
4258 mov r9,rax
4259 mul r10
4260 add r14,rax
4261 adc rdx,0
4262 mov r10,rdx
4263 mov rax,QWORD[((8+160+0))+rbp]
4264 mul r11
4265 add r15,rax
4266 adc rdx,0
4267 imul r9,r12
4268 add r15,r10
4269 adc r9,rdx
4270 mov r10,r13
4271 mov r11,r14
4272 mov r12,r15
4273 and r12,3
4274 mov r13,r15
4275 and r13,-4
4276 mov r14,r9
4277 shrd r15,r9,2
4278 shr r9,2
4279 add r15,r13
4280 adc r9,r14
4281 add r10,r15
4282 adc r11,r9
4283 adc r12,0
4284
4285 add rcx,16
4286 cmp rcx,2*32
4287 jne NEAR $L$open_avx2_init_hash
4288
4289 vpxor ymm0,ymm0,YMMWORD[rsi]
4290 vpxor ymm4,ymm4,YMMWORD[32+rsi]
4291
4292 vmovdqu YMMWORD[rdi],ymm0
4293 vmovdqu YMMWORD[32+rdi],ymm4
4294 lea rsi,[64+rsi]
4295 lea rdi,[64+rdi]
4296 sub rbx,2*32
4297 $L$open_avx2_main_loop:
4298
4299 cmp rbx,16*32
4300 jb NEAR $L$open_avx2_main_loop_done
4301 vmovdqa ymm0,YMMWORD[$L$chacha20_consts]
4302 vmovdqa ymm4,YMMWORD[((160+64))+rbp]
4303 vmovdqa ymm8,YMMWORD[((160+96))+rbp]
4304 vmovdqa ymm1,ymm0
4305 vmovdqa ymm5,ymm4
4306 vmovdqa ymm9,ymm8
4307 vmovdqa ymm2,ymm0
4308 vmovdqa ymm6,ymm4
4309 vmovdqa ymm10,ymm8
4310 vmovdqa ymm3,ymm0
4311 vmovdqa ymm7,ymm4
4312 vmovdqa ymm11,ymm8
4313 vmovdqa ymm12,YMMWORD[$L$avx2_inc]
4314 vpaddd ymm15,ymm12,YMMWORD[((160+160))+rbp]
4315 vpaddd ymm14,ymm12,ymm15
4316 vpaddd ymm13,ymm12,ymm14
4317 vpaddd ymm12,ymm12,ymm13
4318 vmovdqa YMMWORD[(160+256)+rbp],ymm15
4319 vmovdqa YMMWORD[(160+224)+rbp],ymm14
4320 vmovdqa YMMWORD[(160+192)+rbp],ymm13
4321 vmovdqa YMMWORD[(160+160)+rbp],ymm12
4322
4323 xor rcx,rcx
4324 $L$open_avx2_main_loop_rounds:
4325 add r10,QWORD[((0+0))+rcx*1+rsi]
4326 adc r11,QWORD[((8+0))+rcx*1+rsi]
4327 adc r12,1
4328 vmovdqa YMMWORD[(160+128)+rbp],ymm8
4329 vmovdqa ymm8,YMMWORD[$L$rol16]
4330 vpaddd ymm3,ymm3,ymm7
4331 vpaddd ymm2,ymm2,ymm6
4332 vpaddd ymm1,ymm1,ymm5
4333 vpaddd ymm0,ymm0,ymm4
4334 vpxor ymm15,ymm15,ymm3
4335 vpxor ymm14,ymm14,ymm2
4336 vpxor ymm13,ymm13,ymm1
4337 vpxor ymm12,ymm12,ymm0
4338 mov rdx,QWORD[((0+160+0))+rbp]
4339 mov r15,rdx
4340 mulx r14,r13,r10
4341 mulx rdx,rax,r11
4342 imul r15,r12
4343 add r14,rax
4344 adc r15,rdx
4345 vpshufb ymm15,ymm15,ymm8
4346 vpshufb ymm14,ymm14,ymm8
4347 vpshufb ymm13,ymm13,ymm8
4348 vpshufb ymm12,ymm12,ymm8
4349 vpaddd ymm11,ymm11,ymm15
4350 vpaddd ymm10,ymm10,ymm14
4351 vpaddd ymm9,ymm9,ymm13
4352 vpaddd ymm8,ymm12,YMMWORD[((160+128))+rbp]
4353 vpxor ymm7,ymm7,ymm11
4354 mov rdx,QWORD[((8+160+0))+rbp]
4355 mulx rax,r10,r10
4356 add r14,r10
4357 mulx r9,r11,r11
4358 adc r15,r11
4359 adc r9,0
4360 imul rdx,r12
4361 vpxor ymm6,ymm6,ymm10
4362 vpxor ymm5,ymm5,ymm9
4363 vpxor ymm4,ymm4,ymm8
4364 vmovdqa YMMWORD[(160+128)+rbp],ymm8
4365 vpsrld ymm8,ymm7,20
4366 vpslld ymm7,ymm7,32-20
4367 vpxor ymm7,ymm7,ymm8
4368 vpsrld ymm8,ymm6,20
4369 vpslld ymm6,ymm6,32-20
4370 vpxor ymm6,ymm6,ymm8
4371 vpsrld ymm8,ymm5,20
4372 vpslld ymm5,ymm5,32-20
4373 add r15,rax
4374 adc r9,rdx
4375 vpxor ymm5,ymm5,ymm8
4376 vpsrld ymm8,ymm4,20
4377 vpslld ymm4,ymm4,32-20
4378 vpxor ymm4,ymm4,ymm8
4379 vmovdqa ymm8,YMMWORD[$L$rol8]
4380 vpaddd ymm3,ymm3,ymm7
4381 vpaddd ymm2,ymm2,ymm6
4382 vpaddd ymm1,ymm1,ymm5
4383 vpaddd ymm0,ymm0,ymm4
4384 vpxor ymm15,ymm15,ymm3
4385 mov r10,r13
4386 mov r11,r14
4387 mov r12,r15
4388 and r12,3
4389 mov r13,r15
4390 and r13,-4
4391 mov r14,r9
4392 shrd r15,r9,2
4393 shr r9,2
4394 add r15,r13
4395 adc r9,r14
4396 add r10,r15
4397 adc r11,r9
4398 adc r12,0
4399 vpxor ymm14,ymm14,ymm2
4400 vpxor ymm13,ymm13,ymm1
4401 vpxor ymm12,ymm12,ymm0
4402 vpshufb ymm15,ymm15,ymm8
4403 vpshufb ymm14,ymm14,ymm8
4404 vpshufb ymm13,ymm13,ymm8
4405 vpshufb ymm12,ymm12,ymm8
4406 vpaddd ymm11,ymm11,ymm15
4407 vpaddd ymm10,ymm10,ymm14
4408 add r10,QWORD[((0+16))+rcx*1+rsi]
4409 adc r11,QWORD[((8+16))+rcx*1+rsi]
4410 adc r12,1
4411 vpaddd ymm9,ymm9,ymm13
4412 vpaddd ymm8,ymm12,YMMWORD[((160+128))+rbp]
4413 vpxor ymm7,ymm7,ymm11
4414 vpxor ymm6,ymm6,ymm10
4415 vpxor ymm5,ymm5,ymm9
4416 vpxor ymm4,ymm4,ymm8
4417 vmovdqa YMMWORD[(160+128)+rbp],ymm8
4418 vpsrld ymm8,ymm7,25
4419 mov rdx,QWORD[((0+160+0))+rbp]
4420 mov r15,rdx
4421 mulx r14,r13,r10
4422 mulx rdx,rax,r11
4423 imul r15,r12
4424 add r14,rax
4425 adc r15,rdx
4426 vpslld ymm7,ymm7,32-25
4427 vpxor ymm7,ymm7,ymm8
4428 vpsrld ymm8,ymm6,25
4429 vpslld ymm6,ymm6,32-25
4430 vpxor ymm6,ymm6,ymm8
4431 vpsrld ymm8,ymm5,25
4432 vpslld ymm5,ymm5,32-25
4433 vpxor ymm5,ymm5,ymm8
4434 vpsrld ymm8,ymm4,25
4435 vpslld ymm4,ymm4,32-25
4436 vpxor ymm4,ymm4,ymm8
4437 vmovdqa ymm8,YMMWORD[((160+128))+rbp]
4438 vpalignr ymm7,ymm7,ymm7,4
4439 vpalignr ymm11,ymm11,ymm11,8
4440 vpalignr ymm15,ymm15,ymm15,12
4441 vpalignr ymm6,ymm6,ymm6,4
4442 vpalignr ymm10,ymm10,ymm10,8
4443 vpalignr ymm14,ymm14,ymm14,12
4444 mov rdx,QWORD[((8+160+0))+rbp]
4445 mulx rax,r10,r10
4446 add r14,r10
4447 mulx r9,r11,r11
4448 adc r15,r11
4449 adc r9,0
4450 imul rdx,r12
4451 vpalignr ymm5,ymm5,ymm5,4
4452 vpalignr ymm9,ymm9,ymm9,8
4453 vpalignr ymm13,ymm13,ymm13,12
4454 vpalignr ymm4,ymm4,ymm4,4
4455 vpalignr ymm8,ymm8,ymm8,8
4456 vpalignr ymm12,ymm12,ymm12,12
4457 vmovdqa YMMWORD[(160+128)+rbp],ymm8
4458 vmovdqa ymm8,YMMWORD[$L$rol16]
4459 vpaddd ymm3,ymm3,ymm7
4460 vpaddd ymm2,ymm2,ymm6
4461 vpaddd ymm1,ymm1,ymm5
4462 vpaddd ymm0,ymm0,ymm4
4463 vpxor ymm15,ymm15,ymm3
4464 vpxor ymm14,ymm14,ymm2
4465 vpxor ymm13,ymm13,ymm1
4466 vpxor ymm12,ymm12,ymm0
4467 vpshufb ymm15,ymm15,ymm8
4468 vpshufb ymm14,ymm14,ymm8
4469 add r15,rax
4470 adc r9,rdx
4471 vpshufb ymm13,ymm13,ymm8
4472 vpshufb ymm12,ymm12,ymm8
4473 vpaddd ymm11,ymm11,ymm15
4474 vpaddd ymm10,ymm10,ymm14
4475 vpaddd ymm9,ymm9,ymm13
4476 vpaddd ymm8,ymm12,YMMWORD[((160+128))+rbp]
4477 vpxor ymm7,ymm7,ymm11
4478 vpxor ymm6,ymm6,ymm10
4479 vpxor ymm5,ymm5,ymm9
4480 mov r10,r13
4481 mov r11,r14
4482 mov r12,r15
4483 and r12,3
4484 mov r13,r15
4485 and r13,-4
4486 mov r14,r9
4487 shrd r15,r9,2
4488 shr r9,2
4489 add r15,r13
4490 adc r9,r14
4491 add r10,r15
4492 adc r11,r9
4493 adc r12,0
4494 vpxor ymm4,ymm4,ymm8
4495 vmovdqa YMMWORD[(160+128)+rbp],ymm8
4496 vpsrld ymm8,ymm7,20
4497 vpslld ymm7,ymm7,32-20
4498 vpxor ymm7,ymm7,ymm8
4499 vpsrld ymm8,ymm6,20
4500 vpslld ymm6,ymm6,32-20
4501 vpxor ymm6,ymm6,ymm8
4502 add r10,QWORD[((0+32))+rcx*1+rsi]
4503 adc r11,QWORD[((8+32))+rcx*1+rsi]
4504 adc r12,1
4505
4506 lea rcx,[48+rcx]
4507 vpsrld ymm8,ymm5,20
4508 vpslld ymm5,ymm5,32-20
4509 vpxor ymm5,ymm5,ymm8
4510 vpsrld ymm8,ymm4,20
4511 vpslld ymm4,ymm4,32-20
4512 vpxor ymm4,ymm4,ymm8
4513 vmovdqa ymm8,YMMWORD[$L$rol8]
4514 vpaddd ymm3,ymm3,ymm7
4515 vpaddd ymm2,ymm2,ymm6
4516 vpaddd ymm1,ymm1,ymm5
4517 vpaddd ymm0,ymm0,ymm4
4518 vpxor ymm15,ymm15,ymm3
4519 vpxor ymm14,ymm14,ymm2
4520 vpxor ymm13,ymm13,ymm1
4521 vpxor ymm12,ymm12,ymm0
4522 vpshufb ymm15,ymm15,ymm8
4523 vpshufb ymm14,ymm14,ymm8
4524 vpshufb ymm13,ymm13,ymm8
4525 mov rdx,QWORD[((0+160+0))+rbp]
4526 mov r15,rdx
4527 mulx r14,r13,r10
4528 mulx rdx,rax,r11
4529 imul r15,r12
4530 add r14,rax
4531 adc r15,rdx
4532 vpshufb ymm12,ymm12,ymm8
4533 vpaddd ymm11,ymm11,ymm15
4534 vpaddd ymm10,ymm10,ymm14
4535 vpaddd ymm9,ymm9,ymm13
4536 vpaddd ymm8,ymm12,YMMWORD[((160+128))+rbp]
4537 vpxor ymm7,ymm7,ymm11
4538 vpxor ymm6,ymm6,ymm10
4539 vpxor ymm5,ymm5,ymm9
4540 mov rdx,QWORD[((8+160+0))+rbp]
4541 mulx rax,r10,r10
4542 add r14,r10
4543 mulx r9,r11,r11
4544 adc r15,r11
4545 adc r9,0
4546 imul rdx,r12
4547 vpxor ymm4,ymm4,ymm8
4548 vmovdqa YMMWORD[(160+128)+rbp],ymm8
4549 vpsrld ymm8,ymm7,25
4550 vpslld ymm7,ymm7,32-25
4551 vpxor ymm7,ymm7,ymm8
4552 vpsrld ymm8,ymm6,25
4553 vpslld ymm6,ymm6,32-25
4554 vpxor ymm6,ymm6,ymm8
4555 add r15,rax
4556 adc r9,rdx
4557 vpsrld ymm8,ymm5,25
4558 vpslld ymm5,ymm5,32-25
4559 vpxor ymm5,ymm5,ymm8
4560 vpsrld ymm8,ymm4,25
4561 vpslld ymm4,ymm4,32-25
4562 vpxor ymm4,ymm4,ymm8
4563 vmovdqa ymm8,YMMWORD[((160+128))+rbp]
4564 vpalignr ymm7,ymm7,ymm7,12
4565 vpalignr ymm11,ymm11,ymm11,8
4566 vpalignr ymm15,ymm15,ymm15,4
4567 vpalignr ymm6,ymm6,ymm6,12
4568 vpalignr ymm10,ymm10,ymm10,8
4569 vpalignr ymm14,ymm14,ymm14,4
4570 vpalignr ymm5,ymm5,ymm5,12
4571 vpalignr ymm9,ymm9,ymm9,8
4572 vpalignr ymm13,ymm13,ymm13,4
4573 vpalignr ymm4,ymm4,ymm4,12
4574 vpalignr ymm8,ymm8,ymm8,8
4575 mov r10,r13
4576 mov r11,r14
4577 mov r12,r15
4578 and r12,3
4579 mov r13,r15
4580 and r13,-4
4581 mov r14,r9
4582 shrd r15,r9,2
4583 shr r9,2
4584 add r15,r13
4585 adc r9,r14
4586 add r10,r15
4587 adc r11,r9
4588 adc r12,0
4589 vpalignr ymm12,ymm12,ymm12,4
4590
4591 cmp rcx,10*6*8
4592 jne NEAR $L$open_avx2_main_loop_rounds
4593 vpaddd ymm3,ymm3,YMMWORD[$L$chacha20_consts]
4594 vpaddd ymm7,ymm7,YMMWORD[((160+64))+rbp]
4595 vpaddd ymm11,ymm11,YMMWORD[((160+96))+rbp]
4596 vpaddd ymm15,ymm15,YMMWORD[((160+256))+rbp]
4597 vpaddd ymm2,ymm2,YMMWORD[$L$chacha20_consts]
4598 vpaddd ymm6,ymm6,YMMWORD[((160+64))+rbp]
4599 vpaddd ymm10,ymm10,YMMWORD[((160+96))+rbp]
4600 vpaddd ymm14,ymm14,YMMWORD[((160+224))+rbp]
4601 vpaddd ymm1,ymm1,YMMWORD[$L$chacha20_consts]
4602 vpaddd ymm5,ymm5,YMMWORD[((160+64))+rbp]
4603 vpaddd ymm9,ymm9,YMMWORD[((160+96))+rbp]
4604 vpaddd ymm13,ymm13,YMMWORD[((160+192))+rbp]
4605 vpaddd ymm0,ymm0,YMMWORD[$L$chacha20_consts]
4606 vpaddd ymm4,ymm4,YMMWORD[((160+64))+rbp]
4607 vpaddd ymm8,ymm8,YMMWORD[((160+96))+rbp]
4608 vpaddd ymm12,ymm12,YMMWORD[((160+160))+rbp]
4609
4610 vmovdqa YMMWORD[(160+128)+rbp],ymm0
4611 add r10,QWORD[((0+480))+rsi]
4612 adc r11,QWORD[((8+480))+rsi]
4613 adc r12,1
4614 vperm2i128 ymm0,ymm7,ymm3,0x02
4615 vperm2i128 ymm7,ymm7,ymm3,0x13
4616 vperm2i128 ymm3,ymm15,ymm11,0x02
4617 vperm2i128 ymm11,ymm15,ymm11,0x13
4618 vpxor ymm0,ymm0,YMMWORD[((0+0))+rsi]
4619 vpxor ymm3,ymm3,YMMWORD[((32+0))+rsi]
4620 vpxor ymm7,ymm7,YMMWORD[((64+0))+rsi]
4621 vpxor ymm11,ymm11,YMMWORD[((96+0))+rsi]
4622 vmovdqu YMMWORD[(0+0)+rdi],ymm0
4623 vmovdqu YMMWORD[(32+0)+rdi],ymm3
4624 vmovdqu YMMWORD[(64+0)+rdi],ymm7
4625 vmovdqu YMMWORD[(96+0)+rdi],ymm11
4626
4627 vmovdqa ymm0,YMMWORD[((160+128))+rbp]
4628 mov rax,QWORD[((0+160+0))+rbp]
4629 mov r15,rax
4630 mul r10
4631 mov r13,rax
4632 mov r14,rdx
4633 mov rax,QWORD[((0+160+0))+rbp]
4634 mul r11
4635 imul r15,r12
4636 add r14,rax
4637 adc r15,rdx
4638 mov rax,QWORD[((8+160+0))+rbp]
4639 mov r9,rax
4640 mul r10
4641 add r14,rax
4642 adc rdx,0
4643 mov r10,rdx
4644 mov rax,QWORD[((8+160+0))+rbp]
4645 mul r11
4646 add r15,rax
4647 adc rdx,0
4648 imul r9,r12
4649 add r15,r10
4650 adc r9,rdx
4651 mov r10,r13
4652 mov r11,r14
4653 mov r12,r15
4654 and r12,3
4655 mov r13,r15
4656 and r13,-4
4657 mov r14,r9
4658 shrd r15,r9,2
4659 shr r9,2
4660 add r15,r13
4661 adc r9,r14
4662 add r10,r15
4663 adc r11,r9
4664 adc r12,0
4665 vperm2i128 ymm3,ymm6,ymm2,0x02
4666 vperm2i128 ymm6,ymm6,ymm2,0x13
4667 vperm2i128 ymm2,ymm14,ymm10,0x02
4668 vperm2i128 ymm10,ymm14,ymm10,0x13
4669 vpxor ymm3,ymm3,YMMWORD[((0+128))+rsi]
4670 vpxor ymm2,ymm2,YMMWORD[((32+128))+rsi]
4671 vpxor ymm6,ymm6,YMMWORD[((64+128))+rsi]
4672 vpxor ymm10,ymm10,YMMWORD[((96+128))+rsi]
4673 vmovdqu YMMWORD[(0+128)+rdi],ymm3
4674 vmovdqu YMMWORD[(32+128)+rdi],ymm2
4675 vmovdqu YMMWORD[(64+128)+rdi],ymm6
4676 vmovdqu YMMWORD[(96+128)+rdi],ymm10
4677 add r10,QWORD[((0+480+16))+rsi]
4678 adc r11,QWORD[((8+480+16))+rsi]
4679 adc r12,1
4680 vperm2i128 ymm3,ymm5,ymm1,0x02
4681 vperm2i128 ymm5,ymm5,ymm1,0x13
4682 vperm2i128 ymm1,ymm13,ymm9,0x02
4683 vperm2i128 ymm9,ymm13,ymm9,0x13
4684 vpxor ymm3,ymm3,YMMWORD[((0+256))+rsi]
4685 vpxor ymm1,ymm1,YMMWORD[((32+256))+rsi]
4686 vpxor ymm5,ymm5,YMMWORD[((64+256))+rsi]
4687 vpxor ymm9,ymm9,YMMWORD[((96+256))+rsi]
4688 vmovdqu YMMWORD[(0+256)+rdi],ymm3
4689 vmovdqu YMMWORD[(32+256)+rdi],ymm1
4690 vmovdqu YMMWORD[(64+256)+rdi],ymm5
4691 vmovdqu YMMWORD[(96+256)+rdi],ymm9
4692 mov rax,QWORD[((0+160+0))+rbp]
4693 mov r15,rax
4694 mul r10
4695 mov r13,rax
4696 mov r14,rdx
4697 mov rax,QWORD[((0+160+0))+rbp]
4698 mul r11
4699 imul r15,r12
4700 add r14,rax
4701 adc r15,rdx
4702 mov rax,QWORD[((8+160+0))+rbp]
4703 mov r9,rax
4704 mul r10
4705 add r14,rax
4706 adc rdx,0
4707 mov r10,rdx
4708 mov rax,QWORD[((8+160+0))+rbp]
4709 mul r11
4710 add r15,rax
4711 adc rdx,0
4712 imul r9,r12
4713 add r15,r10
4714 adc r9,rdx
4715 mov r10,r13
4716 mov r11,r14
4717 mov r12,r15
4718 and r12,3
4719 mov r13,r15
4720 and r13,-4
4721 mov r14,r9
4722 shrd r15,r9,2
4723 shr r9,2
4724 add r15,r13
4725 adc r9,r14
4726 add r10,r15
4727 adc r11,r9
4728 adc r12,0
4729 vperm2i128 ymm3,ymm4,ymm0,0x02
4730 vperm2i128 ymm4,ymm4,ymm0,0x13
4731 vperm2i128 ymm0,ymm12,ymm8,0x02
4732 vperm2i128 ymm8,ymm12,ymm8,0x13
4733 vpxor ymm3,ymm3,YMMWORD[((0+384))+rsi]
4734 vpxor ymm0,ymm0,YMMWORD[((32+384))+rsi]
4735 vpxor ymm4,ymm4,YMMWORD[((64+384))+rsi]
4736 vpxor ymm8,ymm8,YMMWORD[((96+384))+rsi]
4737 vmovdqu YMMWORD[(0+384)+rdi],ymm3
4738 vmovdqu YMMWORD[(32+384)+rdi],ymm0
4739 vmovdqu YMMWORD[(64+384)+rdi],ymm4
4740 vmovdqu YMMWORD[(96+384)+rdi],ymm8
4741
4742 lea rsi,[512+rsi]
4743 lea rdi,[512+rdi]
4744 sub rbx,16*32
4745 jmp NEAR $L$open_avx2_main_loop
4746 $L$open_avx2_main_loop_done:
4747 test rbx,rbx
4748 vzeroupper
4749 je NEAR $L$open_sse_finalize
4750
4751 cmp rbx,12*32
4752 ja NEAR $L$open_avx2_tail_512
4753 cmp rbx,8*32
4754 ja NEAR $L$open_avx2_tail_384
4755 cmp rbx,4*32
4756 ja NEAR $L$open_avx2_tail_256
4757 vmovdqa ymm0,YMMWORD[$L$chacha20_consts]
4758 vmovdqa ymm4,YMMWORD[((160+64))+rbp]
4759 vmovdqa ymm8,YMMWORD[((160+96))+rbp]
4760 vmovdqa ymm12,YMMWORD[$L$avx2_inc]
4761 vpaddd ymm12,ymm12,YMMWORD[((160+160))+rbp]
4762 vmovdqa YMMWORD[(160+160)+rbp],ymm12
4763
4764 xor r8,r8
4765 mov rcx,rbx
4766 and rcx,-16
4767 test rcx,rcx
4768 je NEAR $L$open_avx2_tail_128_rounds
4769 $L$open_avx2_tail_128_rounds_and_x1hash:
4770 add r10,QWORD[((0+0))+r8*1+rsi]
4771 adc r11,QWORD[((8+0))+r8*1+rsi]
4772 adc r12,1
4773 mov rax,QWORD[((0+160+0))+rbp]
4774 mov r15,rax
4775 mul r10
4776 mov r13,rax
4777 mov r14,rdx
4778 mov rax,QWORD[((0+160+0))+rbp]
4779 mul r11
4780 imul r15,r12
4781 add r14,rax
4782 adc r15,rdx
4783 mov rax,QWORD[((8+160+0))+rbp]
4784 mov r9,rax
4785 mul r10
4786 add r14,rax
4787 adc rdx,0
4788 mov r10,rdx
4789 mov rax,QWORD[((8+160+0))+rbp]
4790 mul r11
4791 add r15,rax
4792 adc rdx,0
4793 imul r9,r12
4794 add r15,r10
4795 adc r9,rdx
4796 mov r10,r13
4797 mov r11,r14
4798 mov r12,r15
4799 and r12,3
4800 mov r13,r15
4801 and r13,-4
4802 mov r14,r9
4803 shrd r15,r9,2
4804 shr r9,2
4805 add r15,r13
4806 adc r9,r14
4807 add r10,r15
4808 adc r11,r9
4809 adc r12,0
4810
4811 $L$open_avx2_tail_128_rounds:
4812 add r8,16
4813 vpaddd ymm0,ymm0,ymm4
4814 vpxor ymm12,ymm12,ymm0
4815 vpshufb ymm12,ymm12,YMMWORD[$L$rol16]
4816 vpaddd ymm8,ymm8,ymm12
4817 vpxor ymm4,ymm4,ymm8
4818 vpsrld ymm3,ymm4,20
4819 vpslld ymm4,ymm4,12
4820 vpxor ymm4,ymm4,ymm3
4821 vpaddd ymm0,ymm0,ymm4
4822 vpxor ymm12,ymm12,ymm0
4823 vpshufb ymm12,ymm12,YMMWORD[$L$rol8]
4824 vpaddd ymm8,ymm8,ymm12
4825 vpxor ymm4,ymm4,ymm8
4826 vpslld ymm3,ymm4,7
4827 vpsrld ymm4,ymm4,25
4828 vpxor ymm4,ymm4,ymm3
4829 vpalignr ymm12,ymm12,ymm12,12
4830 vpalignr ymm8,ymm8,ymm8,8
4831 vpalignr ymm4,ymm4,ymm4,4
4832 vpaddd ymm0,ymm0,ymm4
4833 vpxor ymm12,ymm12,ymm0
4834 vpshufb ymm12,ymm12,YMMWORD[$L$rol16]
4835 vpaddd ymm8,ymm8,ymm12
4836 vpxor ymm4,ymm4,ymm8
4837 vpsrld ymm3,ymm4,20
4838 vpslld ymm4,ymm4,12
4839 vpxor ymm4,ymm4,ymm3
4840 vpaddd ymm0,ymm0,ymm4
4841 vpxor ymm12,ymm12,ymm0
4842 vpshufb ymm12,ymm12,YMMWORD[$L$rol8]
4843 vpaddd ymm8,ymm8,ymm12
4844 vpxor ymm4,ymm4,ymm8
4845 vpslld ymm3,ymm4,7
4846 vpsrld ymm4,ymm4,25
4847 vpxor ymm4,ymm4,ymm3
4848 vpalignr ymm12,ymm12,ymm12,4
4849 vpalignr ymm8,ymm8,ymm8,8
4850 vpalignr ymm4,ymm4,ymm4,12
4851
4852 cmp r8,rcx
4853 jb NEAR $L$open_avx2_tail_128_rounds_and_x1hash
4854 cmp r8,160
4855 jne NEAR $L$open_avx2_tail_128_rounds
4856 vpaddd ymm0,ymm0,YMMWORD[$L$chacha20_consts]
4857 vpaddd ymm4,ymm4,YMMWORD[((160+64))+rbp]
4858 vpaddd ymm8,ymm8,YMMWORD[((160+96))+rbp]
4859 vpaddd ymm12,ymm12,YMMWORD[((160+160))+rbp]
4860 vperm2i128 ymm3,ymm4,ymm0,0x13
4861 vperm2i128 ymm0,ymm4,ymm0,0x02
4862 vperm2i128 ymm4,ymm12,ymm8,0x02
4863 vperm2i128 ymm12,ymm12,ymm8,0x13
4864 vmovdqa ymm8,ymm3
4865
4866 jmp NEAR $L$open_avx2_tail_128_xor
4867
4868 $L$open_avx2_tail_256:
4869 vmovdqa ymm0,YMMWORD[$L$chacha20_consts]
4870 vmovdqa ymm4,YMMWORD[((160+64))+rbp]
4871 vmovdqa ymm8,YMMWORD[((160+96))+rbp]
4872 vmovdqa ymm1,ymm0
4873 vmovdqa ymm5,ymm4
4874 vmovdqa ymm9,ymm8
4875 vmovdqa ymm12,YMMWORD[$L$avx2_inc]
4876 vpaddd ymm13,ymm12,YMMWORD[((160+160))+rbp]
4877 vpaddd ymm12,ymm12,ymm13
4878 vmovdqa YMMWORD[(160+160)+rbp],ymm12
4879 vmovdqa YMMWORD[(160+192)+rbp],ymm13
4880
4881 mov QWORD[((160+128))+rbp],rbx
4882 mov rcx,rbx
4883 sub rcx,4*32
4884 shr rcx,4
4885 mov r8,10
4886 cmp rcx,10
4887 cmovg rcx,r8
4888 mov rbx,rsi
4889 xor r8,r8
4890 $L$open_avx2_tail_256_rounds_and_x1hash:
4891 add r10,QWORD[((0+0))+rbx]
4892 adc r11,QWORD[((8+0))+rbx]
4893 adc r12,1
4894 mov rdx,QWORD[((0+160+0))+rbp]
4895 mov r15,rdx
4896 mulx r14,r13,r10
4897 mulx rdx,rax,r11
4898 imul r15,r12
4899 add r14,rax
4900 adc r15,rdx
4901 mov rdx,QWORD[((8+160+0))+rbp]
4902 mulx rax,r10,r10
4903 add r14,r10
4904 mulx r9,r11,r11
4905 adc r15,r11
4906 adc r9,0
4907 imul rdx,r12
4908 add r15,rax
4909 adc r9,rdx
4910 mov r10,r13
4911 mov r11,r14
4912 mov r12,r15
4913 and r12,3
4914 mov r13,r15
4915 and r13,-4
4916 mov r14,r9
4917 shrd r15,r9,2
4918 shr r9,2
4919 add r15,r13
4920 adc r9,r14
4921 add r10,r15
4922 adc r11,r9
4923 adc r12,0
4924
4925 lea rbx,[16+rbx]
4926 $L$open_avx2_tail_256_rounds:
4927 vpaddd ymm0,ymm0,ymm4
4928 vpxor ymm12,ymm12,ymm0
4929 vpshufb ymm12,ymm12,YMMWORD[$L$rol16]
4930 vpaddd ymm8,ymm8,ymm12
4931 vpxor ymm4,ymm4,ymm8
4932 vpsrld ymm3,ymm4,20
4933 vpslld ymm4,ymm4,12
4934 vpxor ymm4,ymm4,ymm3
4935 vpaddd ymm0,ymm0,ymm4
4936 vpxor ymm12,ymm12,ymm0
4937 vpshufb ymm12,ymm12,YMMWORD[$L$rol8]
4938 vpaddd ymm8,ymm8,ymm12
4939 vpxor ymm4,ymm4,ymm8
4940 vpslld ymm3,ymm4,7
4941 vpsrld ymm4,ymm4,25
4942 vpxor ymm4,ymm4,ymm3
4943 vpalignr ymm12,ymm12,ymm12,12
4944 vpalignr ymm8,ymm8,ymm8,8
4945 vpalignr ymm4,ymm4,ymm4,4
4946 vpaddd ymm1,ymm1,ymm5
4947 vpxor ymm13,ymm13,ymm1
4948 vpshufb ymm13,ymm13,YMMWORD[$L$rol16]
4949 vpaddd ymm9,ymm9,ymm13
4950 vpxor ymm5,ymm5,ymm9
4951 vpsrld ymm3,ymm5,20
4952 vpslld ymm5,ymm5,12
4953 vpxor ymm5,ymm5,ymm3
4954 vpaddd ymm1,ymm1,ymm5
4955 vpxor ymm13,ymm13,ymm1
4956 vpshufb ymm13,ymm13,YMMWORD[$L$rol8]
4957 vpaddd ymm9,ymm9,ymm13
4958 vpxor ymm5,ymm5,ymm9
4959 vpslld ymm3,ymm5,7
4960 vpsrld ymm5,ymm5,25
4961 vpxor ymm5,ymm5,ymm3
4962 vpalignr ymm13,ymm13,ymm13,12
4963 vpalignr ymm9,ymm9,ymm9,8
4964 vpalignr ymm5,ymm5,ymm5,4
4965
4966 inc r8
4967 vpaddd ymm0,ymm0,ymm4
4968 vpxor ymm12,ymm12,ymm0
4969 vpshufb ymm12,ymm12,YMMWORD[$L$rol16]
4970 vpaddd ymm8,ymm8,ymm12
4971 vpxor ymm4,ymm4,ymm8
4972 vpsrld ymm3,ymm4,20
4973 vpslld ymm4,ymm4,12
4974 vpxor ymm4,ymm4,ymm3
4975 vpaddd ymm0,ymm0,ymm4
4976 vpxor ymm12,ymm12,ymm0
4977 vpshufb ymm12,ymm12,YMMWORD[$L$rol8]
4978 vpaddd ymm8,ymm8,ymm12
4979 vpxor ymm4,ymm4,ymm8
4980 vpslld ymm3,ymm4,7
4981 vpsrld ymm4,ymm4,25
4982 vpxor ymm4,ymm4,ymm3
4983 vpalignr ymm12,ymm12,ymm12,4
4984 vpalignr ymm8,ymm8,ymm8,8
4985 vpalignr ymm4,ymm4,ymm4,12
4986 vpaddd ymm1,ymm1,ymm5
4987 vpxor ymm13,ymm13,ymm1
4988 vpshufb ymm13,ymm13,YMMWORD[$L$rol16]
4989 vpaddd ymm9,ymm9,ymm13
4990 vpxor ymm5,ymm5,ymm9
4991 vpsrld ymm3,ymm5,20
4992 vpslld ymm5,ymm5,12
4993 vpxor ymm5,ymm5,ymm3
4994 vpaddd ymm1,ymm1,ymm5
4995 vpxor ymm13,ymm13,ymm1
4996 vpshufb ymm13,ymm13,YMMWORD[$L$rol8]
4997 vpaddd ymm9,ymm9,ymm13
4998 vpxor ymm5,ymm5,ymm9
4999 vpslld ymm3,ymm5,7
5000 vpsrld ymm5,ymm5,25
5001 vpxor ymm5,ymm5,ymm3
5002 vpalignr ymm13,ymm13,ymm13,4
5003 vpalignr ymm9,ymm9,ymm9,8
5004 vpalignr ymm5,ymm5,ymm5,12
5005 vpaddd ymm2,ymm2,ymm6
5006 vpxor ymm14,ymm14,ymm2
5007 vpshufb ymm14,ymm14,YMMWORD[$L$rol16]
5008 vpaddd ymm10,ymm10,ymm14
5009 vpxor ymm6,ymm6,ymm10
5010 vpsrld ymm3,ymm6,20
5011 vpslld ymm6,ymm6,12
5012 vpxor ymm6,ymm6,ymm3
5013 vpaddd ymm2,ymm2,ymm6
5014 vpxor ymm14,ymm14,ymm2
5015 vpshufb ymm14,ymm14,YMMWORD[$L$rol8]
5016 vpaddd ymm10,ymm10,ymm14
5017 vpxor ymm6,ymm6,ymm10
5018 vpslld ymm3,ymm6,7
5019 vpsrld ymm6,ymm6,25
5020 vpxor ymm6,ymm6,ymm3
5021 vpalignr ymm14,ymm14,ymm14,4
5022 vpalignr ymm10,ymm10,ymm10,8
5023 vpalignr ymm6,ymm6,ymm6,12
5024
5025 cmp r8,rcx
5026 jb NEAR $L$open_avx2_tail_256_rounds_and_x1hash
5027 cmp r8,10
5028 jne NEAR $L$open_avx2_tail_256_rounds
5029 mov r8,rbx
5030 sub rbx,rsi
5031 mov rcx,rbx
5032 mov rbx,QWORD[((160+128))+rbp]
5033 $L$open_avx2_tail_256_hash:
5034 add rcx,16
5035 cmp rcx,rbx
5036 jg NEAR $L$open_avx2_tail_256_done
5037 add r10,QWORD[((0+0))+r8]
5038 adc r11,QWORD[((8+0))+r8]
5039 adc r12,1
5040 mov rdx,QWORD[((0+160+0))+rbp]
5041 mov r15,rdx
5042 mulx r14,r13,r10
5043 mulx rdx,rax,r11
5044 imul r15,r12
5045 add r14,rax
5046 adc r15,rdx
5047 mov rdx,QWORD[((8+160+0))+rbp]
5048 mulx rax,r10,r10
5049 add r14,r10
5050 mulx r9,r11,r11
5051 adc r15,r11
5052 adc r9,0
5053 imul rdx,r12
5054 add r15,rax
5055 adc r9,rdx
5056 mov r10,r13
5057 mov r11,r14
5058 mov r12,r15
5059 and r12,3
5060 mov r13,r15
5061 and r13,-4
5062 mov r14,r9
5063 shrd r15,r9,2
5064 shr r9,2
5065 add r15,r13
5066 adc r9,r14
5067 add r10,r15
5068 adc r11,r9
5069 adc r12,0
5070
5071 lea r8,[16+r8]
5072 jmp NEAR $L$open_avx2_tail_256_hash
5073 $L$open_avx2_tail_256_done:
5074 vpaddd ymm1,ymm1,YMMWORD[$L$chacha20_consts]
5075 vpaddd ymm5,ymm5,YMMWORD[((160+64))+rbp]
5076 vpaddd ymm9,ymm9,YMMWORD[((160+96))+rbp]
5077 vpaddd ymm13,ymm13,YMMWORD[((160+192))+rbp]
5078 vpaddd ymm0,ymm0,YMMWORD[$L$chacha20_consts]
5079 vpaddd ymm4,ymm4,YMMWORD[((160+64))+rbp]
5080 vpaddd ymm8,ymm8,YMMWORD[((160+96))+rbp]
5081 vpaddd ymm12,ymm12,YMMWORD[((160+160))+rbp]
5082 vperm2i128 ymm3,ymm5,ymm1,0x02
5083 vperm2i128 ymm5,ymm5,ymm1,0x13
5084 vperm2i128 ymm1,ymm13,ymm9,0x02
5085 vperm2i128 ymm9,ymm13,ymm9,0x13
5086 vpxor ymm3,ymm3,YMMWORD[((0+0))+rsi]
5087 vpxor ymm1,ymm1,YMMWORD[((32+0))+rsi]
5088 vpxor ymm5,ymm5,YMMWORD[((64+0))+rsi]
5089 vpxor ymm9,ymm9,YMMWORD[((96+0))+rsi]
5090 vmovdqu YMMWORD[(0+0)+rdi],ymm3
5091 vmovdqu YMMWORD[(32+0)+rdi],ymm1
5092 vmovdqu YMMWORD[(64+0)+rdi],ymm5
5093 vmovdqu YMMWORD[(96+0)+rdi],ymm9
5094 vperm2i128 ymm3,ymm4,ymm0,0x13
5095 vperm2i128 ymm0,ymm4,ymm0,0x02
5096 vperm2i128 ymm4,ymm12,ymm8,0x02
5097 vperm2i128 ymm12,ymm12,ymm8,0x13
5098 vmovdqa ymm8,ymm3
5099
5100 lea rsi,[128+rsi]
5101 lea rdi,[128+rdi]
5102 sub rbx,4*32
5103 jmp NEAR $L$open_avx2_tail_128_xor
5104
5105 $L$open_avx2_tail_384:
5106 vmovdqa ymm0,YMMWORD[$L$chacha20_consts]
5107 vmovdqa ymm4,YMMWORD[((160+64))+rbp]
5108 vmovdqa ymm8,YMMWORD[((160+96))+rbp]
5109 vmovdqa ymm1,ymm0
5110 vmovdqa ymm5,ymm4
5111 vmovdqa ymm9,ymm8
5112 vmovdqa ymm2,ymm0
5113 vmovdqa ymm6,ymm4
5114 vmovdqa ymm10,ymm8
5115 vmovdqa ymm12,YMMWORD[$L$avx2_inc]
5116 vpaddd ymm14,ymm12,YMMWORD[((160+160))+rbp]
5117 vpaddd ymm13,ymm12,ymm14
5118 vpaddd ymm12,ymm12,ymm13
5119 vmovdqa YMMWORD[(160+160)+rbp],ymm12
5120 vmovdqa YMMWORD[(160+192)+rbp],ymm13
5121 vmovdqa YMMWORD[(160+224)+rbp],ymm14
5122
5123 mov QWORD[((160+128))+rbp],rbx
5124 mov rcx,rbx
5125 sub rcx,8*32
5126 shr rcx,4
5127 add rcx,6
5128 mov r8,10
5129 cmp rcx,10
5130 cmovg rcx,r8
5131 mov rbx,rsi
5132 xor r8,r8
5133 $L$open_avx2_tail_384_rounds_and_x2hash:
5134 add r10,QWORD[((0+0))+rbx]
5135 adc r11,QWORD[((8+0))+rbx]
5136 adc r12,1
5137 mov rdx,QWORD[((0+160+0))+rbp]
5138 mov r15,rdx
5139 mulx r14,r13,r10
5140 mulx rdx,rax,r11
5141 imul r15,r12
5142 add r14,rax
5143 adc r15,rdx
5144 mov rdx,QWORD[((8+160+0))+rbp]
5145 mulx rax,r10,r10
5146 add r14,r10
5147 mulx r9,r11,r11
5148 adc r15,r11
5149 adc r9,0
5150 imul rdx,r12
5151 add r15,rax
5152 adc r9,rdx
5153 mov r10,r13
5154 mov r11,r14
5155 mov r12,r15
5156 and r12,3
5157 mov r13,r15
5158 and r13,-4
5159 mov r14,r9
5160 shrd r15,r9,2
5161 shr r9,2
5162 add r15,r13
5163 adc r9,r14
5164 add r10,r15
5165 adc r11,r9
5166 adc r12,0
5167
5168 lea rbx,[16+rbx]
5169 $L$open_avx2_tail_384_rounds_and_x1hash:
5170 vpaddd ymm2,ymm2,ymm6
5171 vpxor ymm14,ymm14,ymm2
5172 vpshufb ymm14,ymm14,YMMWORD[$L$rol16]
5173 vpaddd ymm10,ymm10,ymm14
5174 vpxor ymm6,ymm6,ymm10
5175 vpsrld ymm3,ymm6,20
5176 vpslld ymm6,ymm6,12
5177 vpxor ymm6,ymm6,ymm3
5178 vpaddd ymm2,ymm2,ymm6
5179 vpxor ymm14,ymm14,ymm2
5180 vpshufb ymm14,ymm14,YMMWORD[$L$rol8]
5181 vpaddd ymm10,ymm10,ymm14
5182 vpxor ymm6,ymm6,ymm10
5183 vpslld ymm3,ymm6,7
5184 vpsrld ymm6,ymm6,25
5185 vpxor ymm6,ymm6,ymm3
5186 vpalignr ymm14,ymm14,ymm14,12
5187 vpalignr ymm10,ymm10,ymm10,8
5188 vpalignr ymm6,ymm6,ymm6,4
5189 vpaddd ymm1,ymm1,ymm5
5190 vpxor ymm13,ymm13,ymm1
5191 vpshufb ymm13,ymm13,YMMWORD[$L$rol16]
5192 vpaddd ymm9,ymm9,ymm13
5193 vpxor ymm5,ymm5,ymm9
5194 vpsrld ymm3,ymm5,20
5195 vpslld ymm5,ymm5,12
5196 vpxor ymm5,ymm5,ymm3
5197 vpaddd ymm1,ymm1,ymm5
5198 vpxor ymm13,ymm13,ymm1
5199 vpshufb ymm13,ymm13,YMMWORD[$L$rol8]
5200 vpaddd ymm9,ymm9,ymm13
5201 vpxor ymm5,ymm5,ymm9
5202 vpslld ymm3,ymm5,7
5203 vpsrld ymm5,ymm5,25
5204 vpxor ymm5,ymm5,ymm3
5205 vpalignr ymm13,ymm13,ymm13,12
5206 vpalignr ymm9,ymm9,ymm9,8
5207 vpalignr ymm5,ymm5,ymm5,4
5208 vpaddd ymm0,ymm0,ymm4
5209 vpxor ymm12,ymm12,ymm0
5210 vpshufb ymm12,ymm12,YMMWORD[$L$rol16]
5211 vpaddd ymm8,ymm8,ymm12
5212 vpxor ymm4,ymm4,ymm8
5213 vpsrld ymm3,ymm4,20
5214 vpslld ymm4,ymm4,12
5215 vpxor ymm4,ymm4,ymm3
5216 vpaddd ymm0,ymm0,ymm4
5217 vpxor ymm12,ymm12,ymm0
5218 vpshufb ymm12,ymm12,YMMWORD[$L$rol8]
5219 vpaddd ymm8,ymm8,ymm12
5220 vpxor ymm4,ymm4,ymm8
5221 vpslld ymm3,ymm4,7
5222 vpsrld ymm4,ymm4,25
5223 vpxor ymm4,ymm4,ymm3
5224 vpalignr ymm12,ymm12,ymm12,12
5225 vpalignr ymm8,ymm8,ymm8,8
5226 vpalignr ymm4,ymm4,ymm4,4
5227 add r10,QWORD[((0+0))+rbx]
5228 adc r11,QWORD[((8+0))+rbx]
5229 adc r12,1
5230 mov rax,QWORD[((0+160+0))+rbp]
5231 mov r15,rax
5232 mul r10
5233 mov r13,rax
5234 mov r14,rdx
5235 mov rax,QWORD[((0+160+0))+rbp]
5236 mul r11
5237 imul r15,r12
5238 add r14,rax
5239 adc r15,rdx
5240 mov rax,QWORD[((8+160+0))+rbp]
5241 mov r9,rax
5242 mul r10
5243 add r14,rax
5244 adc rdx,0
5245 mov r10,rdx
5246 mov rax,QWORD[((8+160+0))+rbp]
5247 mul r11
5248 add r15,rax
5249 adc rdx,0
5250 imul r9,r12
5251 add r15,r10
5252 adc r9,rdx
5253 mov r10,r13
5254 mov r11,r14
5255 mov r12,r15
5256 and r12,3
5257 mov r13,r15
5258 and r13,-4
5259 mov r14,r9
5260 shrd r15,r9,2
5261 shr r9,2
5262 add r15,r13
5263 adc r9,r14
5264 add r10,r15
5265 adc r11,r9
5266 adc r12,0
5267
5268 lea rbx,[16+rbx]
5269 inc r8
5270 vpaddd ymm2,ymm2,ymm6
5271 vpxor ymm14,ymm14,ymm2
5272 vpshufb ymm14,ymm14,YMMWORD[$L$rol16]
5273 vpaddd ymm10,ymm10,ymm14
5274 vpxor ymm6,ymm6,ymm10
5275 vpsrld ymm3,ymm6,20
5276 vpslld ymm6,ymm6,12
5277 vpxor ymm6,ymm6,ymm3
5278 vpaddd ymm2,ymm2,ymm6
5279 vpxor ymm14,ymm14,ymm2
5280 vpshufb ymm14,ymm14,YMMWORD[$L$rol8]
5281 vpaddd ymm10,ymm10,ymm14
5282 vpxor ymm6,ymm6,ymm10
5283 vpslld ymm3,ymm6,7
5284 vpsrld ymm6,ymm6,25
5285 vpxor ymm6,ymm6,ymm3
5286 vpalignr ymm14,ymm14,ymm14,4
5287 vpalignr ymm10,ymm10,ymm10,8
5288 vpalignr ymm6,ymm6,ymm6,12
5289 vpaddd ymm1,ymm1,ymm5
5290 vpxor ymm13,ymm13,ymm1
5291 vpshufb ymm13,ymm13,YMMWORD[$L$rol16]
5292 vpaddd ymm9,ymm9,ymm13
5293 vpxor ymm5,ymm5,ymm9
5294 vpsrld ymm3,ymm5,20
5295 vpslld ymm5,ymm5,12
5296 vpxor ymm5,ymm5,ymm3
5297 vpaddd ymm1,ymm1,ymm5
5298 vpxor ymm13,ymm13,ymm1
5299 vpshufb ymm13,ymm13,YMMWORD[$L$rol8]
5300 vpaddd ymm9,ymm9,ymm13
5301 vpxor ymm5,ymm5,ymm9
5302 vpslld ymm3,ymm5,7
5303 vpsrld ymm5,ymm5,25
5304 vpxor ymm5,ymm5,ymm3
5305 vpalignr ymm13,ymm13,ymm13,4
5306 vpalignr ymm9,ymm9,ymm9,8
5307 vpalignr ymm5,ymm5,ymm5,12
5308 vpaddd ymm0,ymm0,ymm4
5309 vpxor ymm12,ymm12,ymm0
5310 vpshufb ymm12,ymm12,YMMWORD[$L$rol16]
5311 vpaddd ymm8,ymm8,ymm12
5312 vpxor ymm4,ymm4,ymm8
5313 vpsrld ymm3,ymm4,20
5314 vpslld ymm4,ymm4,12
5315 vpxor ymm4,ymm4,ymm3
5316 vpaddd ymm0,ymm0,ymm4
5317 vpxor ymm12,ymm12,ymm0
5318 vpshufb ymm12,ymm12,YMMWORD[$L$rol8]
5319 vpaddd ymm8,ymm8,ymm12
5320 vpxor ymm4,ymm4,ymm8
5321 vpslld ymm3,ymm4,7
5322 vpsrld ymm4,ymm4,25
5323 vpxor ymm4,ymm4,ymm3
5324 vpalignr ymm12,ymm12,ymm12,4
5325 vpalignr ymm8,ymm8,ymm8,8
5326 vpalignr ymm4,ymm4,ymm4,12
5327
5328 cmp r8,rcx
5329 jb NEAR $L$open_avx2_tail_384_rounds_and_x2hash
5330 cmp r8,10
5331 jne NEAR $L$open_avx2_tail_384_rounds_and_x1hash
5332 mov r8,rbx
5333 sub rbx,rsi
5334 mov rcx,rbx
5335 mov rbx,QWORD[((160+128))+rbp]
5336 $L$open_avx2_384_tail_hash:
5337 add rcx,16
5338 cmp rcx,rbx
5339 jg NEAR $L$open_avx2_384_tail_done
5340 add r10,QWORD[((0+0))+r8]
5341 adc r11,QWORD[((8+0))+r8]
5342 adc r12,1
5343 mov rdx,QWORD[((0+160+0))+rbp]
5344 mov r15,rdx
5345 mulx r14,r13,r10
5346 mulx rdx,rax,r11
5347 imul r15,r12
5348 add r14,rax
5349 adc r15,rdx
5350 mov rdx,QWORD[((8+160+0))+rbp]
5351 mulx rax,r10,r10
5352 add r14,r10
5353 mulx r9,r11,r11
5354 adc r15,r11
5355 adc r9,0
5356 imul rdx,r12
5357 add r15,rax
5358 adc r9,rdx
5359 mov r10,r13
5360 mov r11,r14
5361 mov r12,r15
5362 and r12,3
5363 mov r13,r15
5364 and r13,-4
5365 mov r14,r9
5366 shrd r15,r9,2
5367 shr r9,2
5368 add r15,r13
5369 adc r9,r14
5370 add r10,r15
5371 adc r11,r9
5372 adc r12,0
5373
5374 lea r8,[16+r8]
5375 jmp NEAR $L$open_avx2_384_tail_hash
5376 $L$open_avx2_384_tail_done:
5377 vpaddd ymm2,ymm2,YMMWORD[$L$chacha20_consts]
5378 vpaddd ymm6,ymm6,YMMWORD[((160+64))+rbp]
5379 vpaddd ymm10,ymm10,YMMWORD[((160+96))+rbp]
5380 vpaddd ymm14,ymm14,YMMWORD[((160+224))+rbp]
5381 vpaddd ymm1,ymm1,YMMWORD[$L$chacha20_consts]
5382 vpaddd ymm5,ymm5,YMMWORD[((160+64))+rbp]
5383 vpaddd ymm9,ymm9,YMMWORD[((160+96))+rbp]
5384 vpaddd ymm13,ymm13,YMMWORD[((160+192))+rbp]
5385 vpaddd ymm0,ymm0,YMMWORD[$L$chacha20_consts]
5386 vpaddd ymm4,ymm4,YMMWORD[((160+64))+rbp]
5387 vpaddd ymm8,ymm8,YMMWORD[((160+96))+rbp]
5388 vpaddd ymm12,ymm12,YMMWORD[((160+160))+rbp]
5389 vperm2i128 ymm3,ymm6,ymm2,0x02
5390 vperm2i128 ymm6,ymm6,ymm2,0x13
5391 vperm2i128 ymm2,ymm14,ymm10,0x02
5392 vperm2i128 ymm10,ymm14,ymm10,0x13
5393 vpxor ymm3,ymm3,YMMWORD[((0+0))+rsi]
5394 vpxor ymm2,ymm2,YMMWORD[((32+0))+rsi]
5395 vpxor ymm6,ymm6,YMMWORD[((64+0))+rsi]
5396 vpxor ymm10,ymm10,YMMWORD[((96+0))+rsi]
5397 vmovdqu YMMWORD[(0+0)+rdi],ymm3
5398 vmovdqu YMMWORD[(32+0)+rdi],ymm2
5399 vmovdqu YMMWORD[(64+0)+rdi],ymm6
5400 vmovdqu YMMWORD[(96+0)+rdi],ymm10
5401 vperm2i128 ymm3,ymm5,ymm1,0x02
5402 vperm2i128 ymm5,ymm5,ymm1,0x13
5403 vperm2i128 ymm1,ymm13,ymm9,0x02
5404 vperm2i128 ymm9,ymm13,ymm9,0x13
5405 vpxor ymm3,ymm3,YMMWORD[((0+128))+rsi]
5406 vpxor ymm1,ymm1,YMMWORD[((32+128))+rsi]
5407 vpxor ymm5,ymm5,YMMWORD[((64+128))+rsi]
5408 vpxor ymm9,ymm9,YMMWORD[((96+128))+rsi]
5409 vmovdqu YMMWORD[(0+128)+rdi],ymm3
5410 vmovdqu YMMWORD[(32+128)+rdi],ymm1
5411 vmovdqu YMMWORD[(64+128)+rdi],ymm5
5412 vmovdqu YMMWORD[(96+128)+rdi],ymm9
5413 vperm2i128 ymm3,ymm4,ymm0,0x13
5414 vperm2i128 ymm0,ymm4,ymm0,0x02
5415 vperm2i128 ymm4,ymm12,ymm8,0x02
5416 vperm2i128 ymm12,ymm12,ymm8,0x13
5417 vmovdqa ymm8,ymm3
5418
5419 lea rsi,[256+rsi]
5420 lea rdi,[256+rdi]
5421 sub rbx,8*32
5422 jmp NEAR $L$open_avx2_tail_128_xor
5423
5424 $L$open_avx2_tail_512:
5425 vmovdqa ymm0,YMMWORD[$L$chacha20_consts]
5426 vmovdqa ymm4,YMMWORD[((160+64))+rbp]
5427 vmovdqa ymm8,YMMWORD[((160+96))+rbp]
5428 vmovdqa ymm1,ymm0
5429 vmovdqa ymm5,ymm4
5430 vmovdqa ymm9,ymm8
5431 vmovdqa ymm2,ymm0
5432 vmovdqa ymm6,ymm4
5433 vmovdqa ymm10,ymm8
5434 vmovdqa ymm3,ymm0
5435 vmovdqa ymm7,ymm4
5436 vmovdqa ymm11,ymm8
5437 vmovdqa ymm12,YMMWORD[$L$avx2_inc]
5438 vpaddd ymm15,ymm12,YMMWORD[((160+160))+rbp]
5439 vpaddd ymm14,ymm12,ymm15
5440 vpaddd ymm13,ymm12,ymm14
5441 vpaddd ymm12,ymm12,ymm13
5442 vmovdqa YMMWORD[(160+256)+rbp],ymm15
5443 vmovdqa YMMWORD[(160+224)+rbp],ymm14
5444 vmovdqa YMMWORD[(160+192)+rbp],ymm13
5445 vmovdqa YMMWORD[(160+160)+rbp],ymm12
5446
5447 xor rcx,rcx
5448 mov r8,rsi
5449 $L$open_avx2_tail_512_rounds_and_x2hash:
5450 add r10,QWORD[((0+0))+r8]
5451 adc r11,QWORD[((8+0))+r8]
5452 adc r12,1
5453 mov rax,QWORD[((0+160+0))+rbp]
5454 mov r15,rax
5455 mul r10
5456 mov r13,rax
5457 mov r14,rdx
5458 mov rax,QWORD[((0+160+0))+rbp]
5459 mul r11
5460 imul r15,r12
5461 add r14,rax
5462 adc r15,rdx
5463 mov rax,QWORD[((8+160+0))+rbp]
5464 mov r9,rax
5465 mul r10
5466 add r14,rax
5467 adc rdx,0
5468 mov r10,rdx
5469 mov rax,QWORD[((8+160+0))+rbp]
5470 mul r11
5471 add r15,rax
5472 adc rdx,0
5473 imul r9,r12
5474 add r15,r10
5475 adc r9,rdx
5476 mov r10,r13
5477 mov r11,r14
5478 mov r12,r15
5479 and r12,3
5480 mov r13,r15
5481 and r13,-4
5482 mov r14,r9
5483 shrd r15,r9,2
5484 shr r9,2
5485 add r15,r13
5486 adc r9,r14
5487 add r10,r15
5488 adc r11,r9
5489 adc r12,0
5490
5491 lea r8,[16+r8]
5492 $L$open_avx2_tail_512_rounds_and_x1hash:
5493 vmovdqa YMMWORD[(160+128)+rbp],ymm8
5494 vmovdqa ymm8,YMMWORD[$L$rol16]
5495 vpaddd ymm3,ymm3,ymm7
5496 vpaddd ymm2,ymm2,ymm6
5497 vpaddd ymm1,ymm1,ymm5
5498 vpaddd ymm0,ymm0,ymm4
5499 vpxor ymm15,ymm15,ymm3
5500 vpxor ymm14,ymm14,ymm2
5501 vpxor ymm13,ymm13,ymm1
5502 vpxor ymm12,ymm12,ymm0
5503 vpshufb ymm15,ymm15,ymm8
5504 vpshufb ymm14,ymm14,ymm8
5505 vpshufb ymm13,ymm13,ymm8
5506 vpshufb ymm12,ymm12,ymm8
5507 vpaddd ymm11,ymm11,ymm15
5508 vpaddd ymm10,ymm10,ymm14
5509 vpaddd ymm9,ymm9,ymm13
5510 vpaddd ymm8,ymm12,YMMWORD[((160+128))+rbp]
5511 vpxor ymm7,ymm7,ymm11
5512 vpxor ymm6,ymm6,ymm10
5513 vpxor ymm5,ymm5,ymm9
5514 vpxor ymm4,ymm4,ymm8
5515 vmovdqa YMMWORD[(160+128)+rbp],ymm8
5516 vpsrld ymm8,ymm7,20
5517 vpslld ymm7,ymm7,32-20
5518 vpxor ymm7,ymm7,ymm8
5519 vpsrld ymm8,ymm6,20
5520 vpslld ymm6,ymm6,32-20
5521 vpxor ymm6,ymm6,ymm8
5522 vpsrld ymm8,ymm5,20
5523 vpslld ymm5,ymm5,32-20
5524 vpxor ymm5,ymm5,ymm8
5525 vpsrld ymm8,ymm4,20
5526 vpslld ymm4,ymm4,32-20
5527 vpxor ymm4,ymm4,ymm8
5528 vmovdqa ymm8,YMMWORD[$L$rol8]
5529 vpaddd ymm3,ymm3,ymm7
5530 add r10,QWORD[((0+0))+r8]
5531 adc r11,QWORD[((8+0))+r8]
5532 adc r12,1
5533 mov rdx,QWORD[((0+160+0))+rbp]
5534 mov r15,rdx
5535 mulx r14,r13,r10
5536 mulx rdx,rax,r11
5537 imul r15,r12
5538 add r14,rax
5539 adc r15,rdx
5540 mov rdx,QWORD[((8+160+0))+rbp]
5541 mulx rax,r10,r10
5542 add r14,r10
5543 mulx r9,r11,r11
5544 adc r15,r11
5545 adc r9,0
5546 imul rdx,r12
5547 add r15,rax
5548 adc r9,rdx
5549 mov r10,r13
5550 mov r11,r14
5551 mov r12,r15
5552 and r12,3
5553 mov r13,r15
5554 and r13,-4
5555 mov r14,r9
5556 shrd r15,r9,2
5557 shr r9,2
5558 add r15,r13
5559 adc r9,r14
5560 add r10,r15
5561 adc r11,r9
5562 adc r12,0
5563 vpaddd ymm2,ymm2,ymm6
5564 vpaddd ymm1,ymm1,ymm5
5565 vpaddd ymm0,ymm0,ymm4
5566 vpxor ymm15,ymm15,ymm3
5567 vpxor ymm14,ymm14,ymm2
5568 vpxor ymm13,ymm13,ymm1
5569 vpxor ymm12,ymm12,ymm0
5570 vpshufb ymm15,ymm15,ymm8
5571 vpshufb ymm14,ymm14,ymm8
5572 vpshufb ymm13,ymm13,ymm8
5573 vpshufb ymm12,ymm12,ymm8
5574 vpaddd ymm11,ymm11,ymm15
5575 vpaddd ymm10,ymm10,ymm14
5576 vpaddd ymm9,ymm9,ymm13
5577 vpaddd ymm8,ymm12,YMMWORD[((160+128))+rbp]
5578 vpxor ymm7,ymm7,ymm11
5579 vpxor ymm6,ymm6,ymm10
5580 vpxor ymm5,ymm5,ymm9
5581 vpxor ymm4,ymm4,ymm8
5582 vmovdqa YMMWORD[(160+128)+rbp],ymm8
5583 vpsrld ymm8,ymm7,25
5584 vpslld ymm7,ymm7,32-25
5585 vpxor ymm7,ymm7,ymm8
5586 vpsrld ymm8,ymm6,25
5587 vpslld ymm6,ymm6,32-25
5588 vpxor ymm6,ymm6,ymm8
5589 vpsrld ymm8,ymm5,25
5590 vpslld ymm5,ymm5,32-25
5591 vpxor ymm5,ymm5,ymm8
5592 vpsrld ymm8,ymm4,25
5593 vpslld ymm4,ymm4,32-25
5594 vpxor ymm4,ymm4,ymm8
5595 vmovdqa ymm8,YMMWORD[((160+128))+rbp]
5596 vpalignr ymm7,ymm7,ymm7,4
5597 vpalignr ymm11,ymm11,ymm11,8
5598 vpalignr ymm15,ymm15,ymm15,12
5599 vpalignr ymm6,ymm6,ymm6,4
5600 vpalignr ymm10,ymm10,ymm10,8
5601 vpalignr ymm14,ymm14,ymm14,12
5602 vpalignr ymm5,ymm5,ymm5,4
5603 vpalignr ymm9,ymm9,ymm9,8
5604 vpalignr ymm13,ymm13,ymm13,12
5605 vpalignr ymm4,ymm4,ymm4,4
5606 vpalignr ymm8,ymm8,ymm8,8
5607 vpalignr ymm12,ymm12,ymm12,12
5608 vmovdqa YMMWORD[(160+128)+rbp],ymm8
5609 vmovdqa ymm8,YMMWORD[$L$rol16]
5610 vpaddd ymm3,ymm3,ymm7
5611 add r10,QWORD[((0+16))+r8]
5612 adc r11,QWORD[((8+16))+r8]
5613 adc r12,1
5614 mov rdx,QWORD[((0+160+0))+rbp]
5615 mov r15,rdx
5616 mulx r14,r13,r10
5617 mulx rdx,rax,r11
5618 imul r15,r12
5619 add r14,rax
5620 adc r15,rdx
5621 mov rdx,QWORD[((8+160+0))+rbp]
5622 mulx rax,r10,r10
5623 add r14,r10
5624 mulx r9,r11,r11
5625 adc r15,r11
5626 adc r9,0
5627 imul rdx,r12
5628 add r15,rax
5629 adc r9,rdx
5630 mov r10,r13
5631 mov r11,r14
5632 mov r12,r15
5633 and r12,3
5634 mov r13,r15
5635 and r13,-4
5636 mov r14,r9
5637 shrd r15,r9,2
5638 shr r9,2
5639 add r15,r13
5640 adc r9,r14
5641 add r10,r15
5642 adc r11,r9
5643 adc r12,0
5644
5645 lea r8,[32+r8]
5646 vpaddd ymm2,ymm2,ymm6
5647 vpaddd ymm1,ymm1,ymm5
5648 vpaddd ymm0,ymm0,ymm4
5649 vpxor ymm15,ymm15,ymm3
5650 vpxor ymm14,ymm14,ymm2
5651 vpxor ymm13,ymm13,ymm1
5652 vpxor ymm12,ymm12,ymm0
5653 vpshufb ymm15,ymm15,ymm8
5654 vpshufb ymm14,ymm14,ymm8
5655 vpshufb ymm13,ymm13,ymm8
5656 vpshufb ymm12,ymm12,ymm8
5657 vpaddd ymm11,ymm11,ymm15
5658 vpaddd ymm10,ymm10,ymm14
5659 vpaddd ymm9,ymm9,ymm13
5660 vpaddd ymm8,ymm12,YMMWORD[((160+128))+rbp]
5661 vpxor ymm7,ymm7,ymm11
5662 vpxor ymm6,ymm6,ymm10
5663 vpxor ymm5,ymm5,ymm9
5664 vpxor ymm4,ymm4,ymm8
5665 vmovdqa YMMWORD[(160+128)+rbp],ymm8
5666 vpsrld ymm8,ymm7,20
5667 vpslld ymm7,ymm7,32-20
5668 vpxor ymm7,ymm7,ymm8
5669 vpsrld ymm8,ymm6,20
5670 vpslld ymm6,ymm6,32-20
5671 vpxor ymm6,ymm6,ymm8
5672 vpsrld ymm8,ymm5,20
5673 vpslld ymm5,ymm5,32-20
5674 vpxor ymm5,ymm5,ymm8
5675 vpsrld ymm8,ymm4,20
5676 vpslld ymm4,ymm4,32-20
5677 vpxor ymm4,ymm4,ymm8
5678 vmovdqa ymm8,YMMWORD[$L$rol8]
5679 vpaddd ymm3,ymm3,ymm7
5680 vpaddd ymm2,ymm2,ymm6
5681 vpaddd ymm1,ymm1,ymm5
5682 vpaddd ymm0,ymm0,ymm4
5683 vpxor ymm15,ymm15,ymm3
5684 vpxor ymm14,ymm14,ymm2
5685 vpxor ymm13,ymm13,ymm1
5686 vpxor ymm12,ymm12,ymm0
5687 vpshufb ymm15,ymm15,ymm8
5688 vpshufb ymm14,ymm14,ymm8
5689 vpshufb ymm13,ymm13,ymm8
5690 vpshufb ymm12,ymm12,ymm8
5691 vpaddd ymm11,ymm11,ymm15
5692 vpaddd ymm10,ymm10,ymm14
5693 vpaddd ymm9,ymm9,ymm13
5694 vpaddd ymm8,ymm12,YMMWORD[((160+128))+rbp]
5695 vpxor ymm7,ymm7,ymm11
5696 vpxor ymm6,ymm6,ymm10
5697 vpxor ymm5,ymm5,ymm9
5698 vpxor ymm4,ymm4,ymm8
5699 vmovdqa YMMWORD[(160+128)+rbp],ymm8
5700 vpsrld ymm8,ymm7,25
5701 vpslld ymm7,ymm7,32-25
5702 vpxor ymm7,ymm7,ymm8
5703 vpsrld ymm8,ymm6,25
5704 vpslld ymm6,ymm6,32-25
5705 vpxor ymm6,ymm6,ymm8
5706 vpsrld ymm8,ymm5,25
5707 vpslld ymm5,ymm5,32-25
5708 vpxor ymm5,ymm5,ymm8
5709 vpsrld ymm8,ymm4,25
5710 vpslld ymm4,ymm4,32-25
5711 vpxor ymm4,ymm4,ymm8
5712 vmovdqa ymm8,YMMWORD[((160+128))+rbp]
5713 vpalignr ymm7,ymm7,ymm7,12
5714 vpalignr ymm11,ymm11,ymm11,8
5715 vpalignr ymm15,ymm15,ymm15,4
5716 vpalignr ymm6,ymm6,ymm6,12
5717 vpalignr ymm10,ymm10,ymm10,8
5718 vpalignr ymm14,ymm14,ymm14,4
5719 vpalignr ymm5,ymm5,ymm5,12
5720 vpalignr ymm9,ymm9,ymm9,8
5721 vpalignr ymm13,ymm13,ymm13,4
5722 vpalignr ymm4,ymm4,ymm4,12
5723 vpalignr ymm8,ymm8,ymm8,8
5724 vpalignr ymm12,ymm12,ymm12,4
5725
5726 inc rcx
5727 cmp rcx,4
5728 jl NEAR $L$open_avx2_tail_512_rounds_and_x2hash
5729 cmp rcx,10
5730 jne NEAR $L$open_avx2_tail_512_rounds_and_x1hash
5731 mov rcx,rbx
5732 sub rcx,12*32
5733 and rcx,-16
5734 $L$open_avx2_tail_512_hash:
5735 test rcx,rcx
5736 je NEAR $L$open_avx2_tail_512_done
5737 add r10,QWORD[((0+0))+r8]
5738 adc r11,QWORD[((8+0))+r8]
5739 adc r12,1
5740 mov rdx,QWORD[((0+160+0))+rbp]
5741 mov r15,rdx
5742 mulx r14,r13,r10
5743 mulx rdx,rax,r11
5744 imul r15,r12
5745 add r14,rax
5746 adc r15,rdx
5747 mov rdx,QWORD[((8+160+0))+rbp]
5748 mulx rax,r10,r10
5749 add r14,r10
5750 mulx r9,r11,r11
5751 adc r15,r11
5752 adc r9,0
5753 imul rdx,r12
5754 add r15,rax
5755 adc r9,rdx
5756 mov r10,r13
5757 mov r11,r14
5758 mov r12,r15
5759 and r12,3
5760 mov r13,r15
5761 and r13,-4
5762 mov r14,r9
5763 shrd r15,r9,2
5764 shr r9,2
5765 add r15,r13
5766 adc r9,r14
5767 add r10,r15
5768 adc r11,r9
5769 adc r12,0
5770
5771 lea r8,[16+r8]
5772 sub rcx,2*8
5773 jmp NEAR $L$open_avx2_tail_512_hash
5774 $L$open_avx2_tail_512_done:
5775 vpaddd ymm3,ymm3,YMMWORD[$L$chacha20_consts]
5776 vpaddd ymm7,ymm7,YMMWORD[((160+64))+rbp]
5777 vpaddd ymm11,ymm11,YMMWORD[((160+96))+rbp]
5778 vpaddd ymm15,ymm15,YMMWORD[((160+256))+rbp]
5779 vpaddd ymm2,ymm2,YMMWORD[$L$chacha20_consts]
5780 vpaddd ymm6,ymm6,YMMWORD[((160+64))+rbp]
5781 vpaddd ymm10,ymm10,YMMWORD[((160+96))+rbp]
5782 vpaddd ymm14,ymm14,YMMWORD[((160+224))+rbp]
5783 vpaddd ymm1,ymm1,YMMWORD[$L$chacha20_consts]
5784 vpaddd ymm5,ymm5,YMMWORD[((160+64))+rbp]
5785 vpaddd ymm9,ymm9,YMMWORD[((160+96))+rbp]
5786 vpaddd ymm13,ymm13,YMMWORD[((160+192))+rbp]
5787 vpaddd ymm0,ymm0,YMMWORD[$L$chacha20_consts]
5788 vpaddd ymm4,ymm4,YMMWORD[((160+64))+rbp]
5789 vpaddd ymm8,ymm8,YMMWORD[((160+96))+rbp]
5790 vpaddd ymm12,ymm12,YMMWORD[((160+160))+rbp]
5791
5792 vmovdqa YMMWORD[(160+128)+rbp],ymm0
5793 vperm2i128 ymm0,ymm7,ymm3,0x02
5794 vperm2i128 ymm7,ymm7,ymm3,0x13
5795 vperm2i128 ymm3,ymm15,ymm11,0x02
5796 vperm2i128 ymm11,ymm15,ymm11,0x13
5797 vpxor ymm0,ymm0,YMMWORD[((0+0))+rsi]
5798 vpxor ymm3,ymm3,YMMWORD[((32+0))+rsi]
5799 vpxor ymm7,ymm7,YMMWORD[((64+0))+rsi]
5800 vpxor ymm11,ymm11,YMMWORD[((96+0))+rsi]
5801 vmovdqu YMMWORD[(0+0)+rdi],ymm0
5802 vmovdqu YMMWORD[(32+0)+rdi],ymm3
5803 vmovdqu YMMWORD[(64+0)+rdi],ymm7
5804 vmovdqu YMMWORD[(96+0)+rdi],ymm11
5805
5806 vmovdqa ymm0,YMMWORD[((160+128))+rbp]
5807 vperm2i128 ymm3,ymm6,ymm2,0x02
5808 vperm2i128 ymm6,ymm6,ymm2,0x13
5809 vperm2i128 ymm2,ymm14,ymm10,0x02
5810 vperm2i128 ymm10,ymm14,ymm10,0x13
5811 vpxor ymm3,ymm3,YMMWORD[((0+128))+rsi]
5812 vpxor ymm2,ymm2,YMMWORD[((32+128))+rsi]
5813 vpxor ymm6,ymm6,YMMWORD[((64+128))+rsi]
5814 vpxor ymm10,ymm10,YMMWORD[((96+128))+rsi]
5815 vmovdqu YMMWORD[(0+128)+rdi],ymm3
5816 vmovdqu YMMWORD[(32+128)+rdi],ymm2
5817 vmovdqu YMMWORD[(64+128)+rdi],ymm6
5818 vmovdqu YMMWORD[(96+128)+rdi],ymm10
5819 vperm2i128 ymm3,ymm5,ymm1,0x02
5820 vperm2i128 ymm5,ymm5,ymm1,0x13
5821 vperm2i128 ymm1,ymm13,ymm9,0x02
5822 vperm2i128 ymm9,ymm13,ymm9,0x13
5823 vpxor ymm3,ymm3,YMMWORD[((0+256))+rsi]
5824 vpxor ymm1,ymm1,YMMWORD[((32+256))+rsi]
5825 vpxor ymm5,ymm5,YMMWORD[((64+256))+rsi]
5826 vpxor ymm9,ymm9,YMMWORD[((96+256))+rsi]
5827 vmovdqu YMMWORD[(0+256)+rdi],ymm3
5828 vmovdqu YMMWORD[(32+256)+rdi],ymm1
5829 vmovdqu YMMWORD[(64+256)+rdi],ymm5
5830 vmovdqu YMMWORD[(96+256)+rdi],ymm9
5831 vperm2i128 ymm3,ymm4,ymm0,0x13
5832 vperm2i128 ymm0,ymm4,ymm0,0x02
5833 vperm2i128 ymm4,ymm12,ymm8,0x02
5834 vperm2i128 ymm12,ymm12,ymm8,0x13
5835 vmovdqa ymm8,ymm3
5836
5837 lea rsi,[384+rsi]
5838 lea rdi,[384+rdi]
5839 sub rbx,12*32
5840 $L$open_avx2_tail_128_xor:
5841 cmp rbx,32
5842 jb NEAR $L$open_avx2_tail_32_xor
5843 sub rbx,32
5844 vpxor ymm0,ymm0,YMMWORD[rsi]
5845 vmovdqu YMMWORD[rdi],ymm0
5846 lea rsi,[32+rsi]
5847 lea rdi,[32+rdi]
5848 vmovdqa ymm0,ymm4
5849 vmovdqa ymm4,ymm8
5850 vmovdqa ymm8,ymm12
5851 jmp NEAR $L$open_avx2_tail_128_xor
5852 $L$open_avx2_tail_32_xor:
5853 cmp rbx,16
5854 vmovdqa xmm1,xmm0
5855 jb NEAR $L$open_avx2_exit
5856 sub rbx,16
5857
5858 vpxor xmm1,xmm0,XMMWORD[rsi]
5859 vmovdqu XMMWORD[rdi],xmm1
5860 lea rsi,[16+rsi]
5861 lea rdi,[16+rdi]
5862 vperm2i128 ymm0,ymm0,ymm0,0x11
5863 vmovdqa xmm1,xmm0
5864 $L$open_avx2_exit:
5865 vzeroupper
5866 jmp NEAR $L$open_sse_tail_16
5867
5868 $L$open_avx2_192:
5869 vmovdqa ymm1,ymm0
5870 vmovdqa ymm2,ymm0
5871 vmovdqa ymm5,ymm4
5872 vmovdqa ymm6,ymm4
5873 vmovdqa ymm9,ymm8
5874 vmovdqa ymm10,ymm8
5875 vpaddd ymm13,ymm12,YMMWORD[$L$avx2_inc]
5876 vmovdqa ymm11,ymm12
5877 vmovdqa ymm15,ymm13
5878 mov r10,10
5879 $L$open_avx2_192_rounds:
5880 vpaddd ymm0,ymm0,ymm4
5881 vpxor ymm12,ymm12,ymm0
5882 vpshufb ymm12,ymm12,YMMWORD[$L$rol16]
5883 vpaddd ymm8,ymm8,ymm12
5884 vpxor ymm4,ymm4,ymm8
5885 vpsrld ymm3,ymm4,20
5886 vpslld ymm4,ymm4,12
5887 vpxor ymm4,ymm4,ymm3
5888 vpaddd ymm0,ymm0,ymm4
5889 vpxor ymm12,ymm12,ymm0
5890 vpshufb ymm12,ymm12,YMMWORD[$L$rol8]
5891 vpaddd ymm8,ymm8,ymm12
5892 vpxor ymm4,ymm4,ymm8
5893 vpslld ymm3,ymm4,7
5894 vpsrld ymm4,ymm4,25
5895 vpxor ymm4,ymm4,ymm3
5896 vpalignr ymm12,ymm12,ymm12,12
5897 vpalignr ymm8,ymm8,ymm8,8
5898 vpalignr ymm4,ymm4,ymm4,4
5899 vpaddd ymm1,ymm1,ymm5
5900 vpxor ymm13,ymm13,ymm1
5901 vpshufb ymm13,ymm13,YMMWORD[$L$rol16]
5902 vpaddd ymm9,ymm9,ymm13
5903 vpxor ymm5,ymm5,ymm9
5904 vpsrld ymm3,ymm5,20
5905 vpslld ymm5,ymm5,12
5906 vpxor ymm5,ymm5,ymm3
5907 vpaddd ymm1,ymm1,ymm5
5908 vpxor ymm13,ymm13,ymm1
5909 vpshufb ymm13,ymm13,YMMWORD[$L$rol8]
5910 vpaddd ymm9,ymm9,ymm13
5911 vpxor ymm5,ymm5,ymm9
5912 vpslld ymm3,ymm5,7
5913 vpsrld ymm5,ymm5,25
5914 vpxor ymm5,ymm5,ymm3
5915 vpalignr ymm13,ymm13,ymm13,12
5916 vpalignr ymm9,ymm9,ymm9,8
5917 vpalignr ymm5,ymm5,ymm5,4
5918 vpaddd ymm0,ymm0,ymm4
5919 vpxor ymm12,ymm12,ymm0
5920 vpshufb ymm12,ymm12,YMMWORD[$L$rol16]
5921 vpaddd ymm8,ymm8,ymm12
5922 vpxor ymm4,ymm4,ymm8
5923 vpsrld ymm3,ymm4,20
5924 vpslld ymm4,ymm4,12
5925 vpxor ymm4,ymm4,ymm3
5926 vpaddd ymm0,ymm0,ymm4
5927 vpxor ymm12,ymm12,ymm0
5928 vpshufb ymm12,ymm12,YMMWORD[$L$rol8]
5929 vpaddd ymm8,ymm8,ymm12
5930 vpxor ymm4,ymm4,ymm8
5931 vpslld ymm3,ymm4,7
5932 vpsrld ymm4,ymm4,25
5933 vpxor ymm4,ymm4,ymm3
5934 vpalignr ymm12,ymm12,ymm12,4
5935 vpalignr ymm8,ymm8,ymm8,8
5936 vpalignr ymm4,ymm4,ymm4,12
5937 vpaddd ymm1,ymm1,ymm5
5938 vpxor ymm13,ymm13,ymm1
5939 vpshufb ymm13,ymm13,YMMWORD[$L$rol16]
5940 vpaddd ymm9,ymm9,ymm13
5941 vpxor ymm5,ymm5,ymm9
5942 vpsrld ymm3,ymm5,20
5943 vpslld ymm5,ymm5,12
5944 vpxor ymm5,ymm5,ymm3
5945 vpaddd ymm1,ymm1,ymm5
5946 vpxor ymm13,ymm13,ymm1
5947 vpshufb ymm13,ymm13,YMMWORD[$L$rol8]
5948 vpaddd ymm9,ymm9,ymm13
5949 vpxor ymm5,ymm5,ymm9
5950 vpslld ymm3,ymm5,7
5951 vpsrld ymm5,ymm5,25
5952 vpxor ymm5,ymm5,ymm3
5953 vpalignr ymm13,ymm13,ymm13,4
5954 vpalignr ymm9,ymm9,ymm9,8
5955 vpalignr ymm5,ymm5,ymm5,12
5956
5957 dec r10
5958 jne NEAR $L$open_avx2_192_rounds
5959 vpaddd ymm0,ymm0,ymm2
5960 vpaddd ymm1,ymm1,ymm2
5961 vpaddd ymm4,ymm4,ymm6
5962 vpaddd ymm5,ymm5,ymm6
5963 vpaddd ymm8,ymm8,ymm10
5964 vpaddd ymm9,ymm9,ymm10
5965 vpaddd ymm12,ymm12,ymm11
5966 vpaddd ymm13,ymm13,ymm15
5967 vperm2i128 ymm3,ymm4,ymm0,0x02
5968
5969 vpand ymm3,ymm3,YMMWORD[$L$clamp]
5970 vmovdqa YMMWORD[(160+0)+rbp],ymm3
5971
5972 vperm2i128 ymm0,ymm4,ymm0,0x13
5973 vperm2i128 ymm4,ymm12,ymm8,0x13
5974 vperm2i128 ymm8,ymm5,ymm1,0x02
5975 vperm2i128 ymm12,ymm13,ymm9,0x02
5976 vperm2i128 ymm1,ymm5,ymm1,0x13
5977 vperm2i128 ymm5,ymm13,ymm9,0x13
5978 $L$open_avx2_short:
5979 mov r8,r8
5980 call poly_hash_ad_internal
5981 $L$open_avx2_short_hash_and_xor_loop:
5982 cmp rbx,32
5983 jb NEAR $L$open_avx2_short_tail_32
5984 sub rbx,32
5985 add r10,QWORD[((0+0))+rsi]
5986 adc r11,QWORD[((8+0))+rsi]
5987 adc r12,1
5988 mov rax,QWORD[((0+160+0))+rbp]
5989 mov r15,rax
5990 mul r10
5991 mov r13,rax
5992 mov r14,rdx
5993 mov rax,QWORD[((0+160+0))+rbp]
5994 mul r11
5995 imul r15,r12
5996 add r14,rax
5997 adc r15,rdx
5998 mov rax,QWORD[((8+160+0))+rbp]
5999 mov r9,rax
6000 mul r10
6001 add r14,rax
6002 adc rdx,0
6003 mov r10,rdx
6004 mov rax,QWORD[((8+160+0))+rbp]
6005 mul r11
6006 add r15,rax
6007 adc rdx,0
6008 imul r9,r12
6009 add r15,r10
6010 adc r9,rdx
6011 mov r10,r13
6012 mov r11,r14
6013 mov r12,r15
6014 and r12,3
6015 mov r13,r15
6016 and r13,-4
6017 mov r14,r9
6018 shrd r15,r9,2
6019 shr r9,2
6020 add r15,r13
6021 adc r9,r14
6022 add r10,r15
6023 adc r11,r9
6024 adc r12,0
6025 add r10,QWORD[((0+16))+rsi]
6026 adc r11,QWORD[((8+16))+rsi]
6027 adc r12,1
6028 mov rax,QWORD[((0+160+0))+rbp]
6029 mov r15,rax
6030 mul r10
6031 mov r13,rax
6032 mov r14,rdx
6033 mov rax,QWORD[((0+160+0))+rbp]
6034 mul r11
6035 imul r15,r12
6036 add r14,rax
6037 adc r15,rdx
6038 mov rax,QWORD[((8+160+0))+rbp]
6039 mov r9,rax
6040 mul r10
6041 add r14,rax
6042 adc rdx,0
6043 mov r10,rdx
6044 mov rax,QWORD[((8+160+0))+rbp]
6045 mul r11
6046 add r15,rax
6047 adc rdx,0
6048 imul r9,r12
6049 add r15,r10
6050 adc r9,rdx
6051 mov r10,r13
6052 mov r11,r14
6053 mov r12,r15
6054 and r12,3
6055 mov r13,r15
6056 and r13,-4
6057 mov r14,r9
6058 shrd r15,r9,2
6059 shr r9,2
6060 add r15,r13
6061 adc r9,r14
6062 add r10,r15
6063 adc r11,r9
6064 adc r12,0
6065
6066
6067 vpxor ymm0,ymm0,YMMWORD[rsi]
6068 vmovdqu YMMWORD[rdi],ymm0
6069 lea rsi,[32+rsi]
6070 lea rdi,[32+rdi]
6071
6072 vmovdqa ymm0,ymm4
6073 vmovdqa ymm4,ymm8
6074 vmovdqa ymm8,ymm12
6075 vmovdqa ymm12,ymm1
6076 vmovdqa ymm1,ymm5
6077 vmovdqa ymm5,ymm9
6078 vmovdqa ymm9,ymm13
6079 vmovdqa ymm13,ymm2
6080 vmovdqa ymm2,ymm6
6081 jmp NEAR $L$open_avx2_short_hash_and_xor_loop
6082 $L$open_avx2_short_tail_32:
6083 cmp rbx,16
6084 vmovdqa xmm1,xmm0
6085 jb NEAR $L$open_avx2_short_tail_32_exit
6086 sub rbx,16
6087 add r10,QWORD[((0+0))+rsi]
6088 adc r11,QWORD[((8+0))+rsi]
6089 adc r12,1
6090 mov rax,QWORD[((0+160+0))+rbp]
6091 mov r15,rax
6092 mul r10
6093 mov r13,rax
6094 mov r14,rdx
6095 mov rax,QWORD[((0+160+0))+rbp]
6096 mul r11
6097 imul r15,r12
6098 add r14,rax
6099 adc r15,rdx
6100 mov rax,QWORD[((8+160+0))+rbp]
6101 mov r9,rax
6102 mul r10
6103 add r14,rax
6104 adc rdx,0
6105 mov r10,rdx
6106 mov rax,QWORD[((8+160+0))+rbp]
6107 mul r11
6108 add r15,rax
6109 adc rdx,0
6110 imul r9,r12
6111 add r15,r10
6112 adc r9,rdx
6113 mov r10,r13
6114 mov r11,r14
6115 mov r12,r15
6116 and r12,3
6117 mov r13,r15
6118 and r13,-4
6119 mov r14,r9
6120 shrd r15,r9,2
6121 shr r9,2
6122 add r15,r13
6123 adc r9,r14
6124 add r10,r15
6125 adc r11,r9
6126 adc r12,0
6127
6128 vpxor xmm3,xmm0,XMMWORD[rsi]
6129 vmovdqu XMMWORD[rdi],xmm3
6130 lea rsi,[16+rsi]
6131 lea rdi,[16+rdi]
6132 vextracti128 xmm1,ymm0,1
6133 $L$open_avx2_short_tail_32_exit:
6134 vzeroupper
6135 jmp NEAR $L$open_sse_tail_16
6136
6137 $L$open_avx2_320:
6138 vmovdqa ymm1,ymm0
6139 vmovdqa ymm2,ymm0
6140 vmovdqa ymm5,ymm4
6141 vmovdqa ymm6,ymm4
6142 vmovdqa ymm9,ymm8
6143 vmovdqa ymm10,ymm8
6144 vpaddd ymm13,ymm12,YMMWORD[$L$avx2_inc]
6145 vpaddd ymm14,ymm13,YMMWORD[$L$avx2_inc]
6146 vmovdqa ymm7,ymm4
6147 vmovdqa ymm11,ymm8
6148 vmovdqa YMMWORD[(160+160)+rbp],ymm12
6149 vmovdqa YMMWORD[(160+192)+rbp],ymm13
6150 vmovdqa YMMWORD[(160+224)+rbp],ymm14
6151 mov r10,10
6152 $L$open_avx2_320_rounds:
6153 vpaddd ymm0,ymm0,ymm4
6154 vpxor ymm12,ymm12,ymm0
6155 vpshufb ymm12,ymm12,YMMWORD[$L$rol16]
6156 vpaddd ymm8,ymm8,ymm12
6157 vpxor ymm4,ymm4,ymm8
6158 vpsrld ymm3,ymm4,20
6159 vpslld ymm4,ymm4,12
6160 vpxor ymm4,ymm4,ymm3
6161 vpaddd ymm0,ymm0,ymm4
6162 vpxor ymm12,ymm12,ymm0
6163 vpshufb ymm12,ymm12,YMMWORD[$L$rol8]
6164 vpaddd ymm8,ymm8,ymm12
6165 vpxor ymm4,ymm4,ymm8
6166 vpslld ymm3,ymm4,7
6167 vpsrld ymm4,ymm4,25
6168 vpxor ymm4,ymm4,ymm3
6169 vpalignr ymm12,ymm12,ymm12,12
6170 vpalignr ymm8,ymm8,ymm8,8
6171 vpalignr ymm4,ymm4,ymm4,4
6172 vpaddd ymm1,ymm1,ymm5
6173 vpxor ymm13,ymm13,ymm1
6174 vpshufb ymm13,ymm13,YMMWORD[$L$rol16]
6175 vpaddd ymm9,ymm9,ymm13
6176 vpxor ymm5,ymm5,ymm9
6177 vpsrld ymm3,ymm5,20
6178 vpslld ymm5,ymm5,12
6179 vpxor ymm5,ymm5,ymm3
6180 vpaddd ymm1,ymm1,ymm5
6181 vpxor ymm13,ymm13,ymm1
6182 vpshufb ymm13,ymm13,YMMWORD[$L$rol8]
6183 vpaddd ymm9,ymm9,ymm13
6184 vpxor ymm5,ymm5,ymm9
6185 vpslld ymm3,ymm5,7
6186 vpsrld ymm5,ymm5,25
6187 vpxor ymm5,ymm5,ymm3
6188 vpalignr ymm13,ymm13,ymm13,12
6189 vpalignr ymm9,ymm9,ymm9,8
6190 vpalignr ymm5,ymm5,ymm5,4
6191 vpaddd ymm2,ymm2,ymm6
6192 vpxor ymm14,ymm14,ymm2
6193 vpshufb ymm14,ymm14,YMMWORD[$L$rol16]
6194 vpaddd ymm10,ymm10,ymm14
6195 vpxor ymm6,ymm6,ymm10
6196 vpsrld ymm3,ymm6,20
6197 vpslld ymm6,ymm6,12
6198 vpxor ymm6,ymm6,ymm3
6199 vpaddd ymm2,ymm2,ymm6
6200 vpxor ymm14,ymm14,ymm2
6201 vpshufb ymm14,ymm14,YMMWORD[$L$rol8]
6202 vpaddd ymm10,ymm10,ymm14
6203 vpxor ymm6,ymm6,ymm10
6204 vpslld ymm3,ymm6,7
6205 vpsrld ymm6,ymm6,25
6206 vpxor ymm6,ymm6,ymm3
6207 vpalignr ymm14,ymm14,ymm14,12
6208 vpalignr ymm10,ymm10,ymm10,8
6209 vpalignr ymm6,ymm6,ymm6,4
6210 vpaddd ymm0,ymm0,ymm4
6211 vpxor ymm12,ymm12,ymm0
6212 vpshufb ymm12,ymm12,YMMWORD[$L$rol16]
6213 vpaddd ymm8,ymm8,ymm12
6214 vpxor ymm4,ymm4,ymm8
6215 vpsrld ymm3,ymm4,20
6216 vpslld ymm4,ymm4,12
6217 vpxor ymm4,ymm4,ymm3
6218 vpaddd ymm0,ymm0,ymm4
6219 vpxor ymm12,ymm12,ymm0
6220 vpshufb ymm12,ymm12,YMMWORD[$L$rol8]
6221 vpaddd ymm8,ymm8,ymm12
6222 vpxor ymm4,ymm4,ymm8
6223 vpslld ymm3,ymm4,7
6224 vpsrld ymm4,ymm4,25
6225 vpxor ymm4,ymm4,ymm3
6226 vpalignr ymm12,ymm12,ymm12,4
6227 vpalignr ymm8,ymm8,ymm8,8
6228 vpalignr ymm4,ymm4,ymm4,12
6229 vpaddd ymm1,ymm1,ymm5
6230 vpxor ymm13,ymm13,ymm1
6231 vpshufb ymm13,ymm13,YMMWORD[$L$rol16]
6232 vpaddd ymm9,ymm9,ymm13
6233 vpxor ymm5,ymm5,ymm9
6234 vpsrld ymm3,ymm5,20
6235 vpslld ymm5,ymm5,12
6236 vpxor ymm5,ymm5,ymm3
6237 vpaddd ymm1,ymm1,ymm5
6238 vpxor ymm13,ymm13,ymm1
6239 vpshufb ymm13,ymm13,YMMWORD[$L$rol8]
6240 vpaddd ymm9,ymm9,ymm13
6241 vpxor ymm5,ymm5,ymm9
6242 vpslld ymm3,ymm5,7
6243 vpsrld ymm5,ymm5,25
6244 vpxor ymm5,ymm5,ymm3
6245 vpalignr ymm13,ymm13,ymm13,4
6246 vpalignr ymm9,ymm9,ymm9,8
6247 vpalignr ymm5,ymm5,ymm5,12
6248 vpaddd ymm2,ymm2,ymm6
6249 vpxor ymm14,ymm14,ymm2
6250 vpshufb ymm14,ymm14,YMMWORD[$L$rol16]
6251 vpaddd ymm10,ymm10,ymm14
6252 vpxor ymm6,ymm6,ymm10
6253 vpsrld ymm3,ymm6,20
6254 vpslld ymm6,ymm6,12
6255 vpxor ymm6,ymm6,ymm3
6256 vpaddd ymm2,ymm2,ymm6
6257 vpxor ymm14,ymm14,ymm2
6258 vpshufb ymm14,ymm14,YMMWORD[$L$rol8]
6259 vpaddd ymm10,ymm10,ymm14
6260 vpxor ymm6,ymm6,ymm10
6261 vpslld ymm3,ymm6,7
6262 vpsrld ymm6,ymm6,25
6263 vpxor ymm6,ymm6,ymm3
6264 vpalignr ymm14,ymm14,ymm14,4
6265 vpalignr ymm10,ymm10,ymm10,8
6266 vpalignr ymm6,ymm6,ymm6,12
6267
6268 dec r10
6269 jne NEAR $L$open_avx2_320_rounds
6270 vpaddd ymm0,ymm0,YMMWORD[$L$chacha20_consts]
6271 vpaddd ymm1,ymm1,YMMWORD[$L$chacha20_consts]
6272 vpaddd ymm2,ymm2,YMMWORD[$L$chacha20_consts]
6273 vpaddd ymm4,ymm4,ymm7
6274 vpaddd ymm5,ymm5,ymm7
6275 vpaddd ymm6,ymm6,ymm7
6276 vpaddd ymm8,ymm8,ymm11
6277 vpaddd ymm9,ymm9,ymm11
6278 vpaddd ymm10,ymm10,ymm11
6279 vpaddd ymm12,ymm12,YMMWORD[((160+160))+rbp]
6280 vpaddd ymm13,ymm13,YMMWORD[((160+192))+rbp]
6281 vpaddd ymm14,ymm14,YMMWORD[((160+224))+rbp]
6282 vperm2i128 ymm3,ymm4,ymm0,0x02
6283
6284 vpand ymm3,ymm3,YMMWORD[$L$clamp]
6285 vmovdqa YMMWORD[(160+0)+rbp],ymm3
6286
6287 vperm2i128 ymm0,ymm4,ymm0,0x13
6288 vperm2i128 ymm4,ymm12,ymm8,0x13
6289 vperm2i128 ymm8,ymm5,ymm1,0x02
6290 vperm2i128 ymm12,ymm13,ymm9,0x02
6291 vperm2i128 ymm1,ymm5,ymm1,0x13
6292 vperm2i128 ymm5,ymm13,ymm9,0x13
6293 vperm2i128 ymm9,ymm6,ymm2,0x02
6294 vperm2i128 ymm13,ymm14,ymm10,0x02
6295 vperm2i128 ymm2,ymm6,ymm2,0x13
6296 vperm2i128 ymm6,ymm14,ymm10,0x13
6297 jmp NEAR $L$open_avx2_short
6298
6299
6300
6301
6302
6303 ALIGN 64
6304 chacha20_poly1305_seal_avx2:
6305
6306
6307
6308
6309
6310
6311
6312
6313
6314
6315
6316
6317 vzeroupper
6318 vmovdqa ymm0,YMMWORD[$L$chacha20_consts]
6319 vbroadcasti128 ymm4,XMMWORD[r9]
6320 vbroadcasti128 ymm8,XMMWORD[16+r9]
6321 vbroadcasti128 ymm12,XMMWORD[32+r9]
6322 vpaddd ymm12,ymm12,YMMWORD[$L$avx2_init]
6323 cmp rbx,6*32
6324 jbe NEAR $L$seal_avx2_192
6325 cmp rbx,10*32
6326 jbe NEAR $L$seal_avx2_320
6327 vmovdqa ymm1,ymm0
6328 vmovdqa ymm2,ymm0
6329 vmovdqa ymm3,ymm0
6330 vmovdqa ymm5,ymm4
6331 vmovdqa ymm6,ymm4
6332 vmovdqa ymm7,ymm4
6333 vmovdqa YMMWORD[(160+64)+rbp],ymm4
6334 vmovdqa ymm9,ymm8
6335 vmovdqa ymm10,ymm8
6336 vmovdqa ymm11,ymm8
6337 vmovdqa YMMWORD[(160+96)+rbp],ymm8
6338 vmovdqa ymm15,ymm12
6339 vpaddd ymm14,ymm15,YMMWORD[$L$avx2_inc]
6340 vpaddd ymm13,ymm14,YMMWORD[$L$avx2_inc]
6341 vpaddd ymm12,ymm13,YMMWORD[$L$avx2_inc]
6342 vmovdqa YMMWORD[(160+160)+rbp],ymm12
6343 vmovdqa YMMWORD[(160+192)+rbp],ymm13
6344 vmovdqa YMMWORD[(160+224)+rbp],ymm14
6345 vmovdqa YMMWORD[(160+256)+rbp],ymm15
6346 mov r10,10
6347 $L$seal_avx2_init_rounds:
6348 vmovdqa YMMWORD[(160+128)+rbp],ymm8
6349 vmovdqa ymm8,YMMWORD[$L$rol16]
6350 vpaddd ymm3,ymm3,ymm7
6351 vpaddd ymm2,ymm2,ymm6
6352 vpaddd ymm1,ymm1,ymm5
6353 vpaddd ymm0,ymm0,ymm4
6354 vpxor ymm15,ymm15,ymm3
6355 vpxor ymm14,ymm14,ymm2
6356 vpxor ymm13,ymm13,ymm1
6357 vpxor ymm12,ymm12,ymm0
6358 vpshufb ymm15,ymm15,ymm8
6359 vpshufb ymm14,ymm14,ymm8
6360 vpshufb ymm13,ymm13,ymm8
6361 vpshufb ymm12,ymm12,ymm8
6362 vpaddd ymm11,ymm11,ymm15
6363 vpaddd ymm10,ymm10,ymm14
6364 vpaddd ymm9,ymm9,ymm13
6365 vpaddd ymm8,ymm12,YMMWORD[((160+128))+rbp]
6366 vpxor ymm7,ymm7,ymm11
6367 vpxor ymm6,ymm6,ymm10
6368 vpxor ymm5,ymm5,ymm9
6369 vpxor ymm4,ymm4,ymm8
6370 vmovdqa YMMWORD[(160+128)+rbp],ymm8
6371 vpsrld ymm8,ymm7,20
6372 vpslld ymm7,ymm7,32-20
6373 vpxor ymm7,ymm7,ymm8
6374 vpsrld ymm8,ymm6,20
6375 vpslld ymm6,ymm6,32-20
6376 vpxor ymm6,ymm6,ymm8
6377 vpsrld ymm8,ymm5,20
6378 vpslld ymm5,ymm5,32-20
6379 vpxor ymm5,ymm5,ymm8
6380 vpsrld ymm8,ymm4,20
6381 vpslld ymm4,ymm4,32-20
6382 vpxor ymm4,ymm4,ymm8
6383 vmovdqa ymm8,YMMWORD[$L$rol8]
6384 vpaddd ymm3,ymm3,ymm7
6385 vpaddd ymm2,ymm2,ymm6
6386 vpaddd ymm1,ymm1,ymm5
6387 vpaddd ymm0,ymm0,ymm4
6388 vpxor ymm15,ymm15,ymm3
6389 vpxor ymm14,ymm14,ymm2
6390 vpxor ymm13,ymm13,ymm1
6391 vpxor ymm12,ymm12,ymm0
6392 vpshufb ymm15,ymm15,ymm8
6393 vpshufb ymm14,ymm14,ymm8
6394 vpshufb ymm13,ymm13,ymm8
6395 vpshufb ymm12,ymm12,ymm8
6396 vpaddd ymm11,ymm11,ymm15
6397 vpaddd ymm10,ymm10,ymm14
6398 vpaddd ymm9,ymm9,ymm13
6399 vpaddd ymm8,ymm12,YMMWORD[((160+128))+rbp]
6400 vpxor ymm7,ymm7,ymm11
6401 vpxor ymm6,ymm6,ymm10
6402 vpxor ymm5,ymm5,ymm9
6403 vpxor ymm4,ymm4,ymm8
6404 vmovdqa YMMWORD[(160+128)+rbp],ymm8
6405 vpsrld ymm8,ymm7,25
6406 vpslld ymm7,ymm7,32-25
6407 vpxor ymm7,ymm7,ymm8
6408 vpsrld ymm8,ymm6,25
6409 vpslld ymm6,ymm6,32-25
6410 vpxor ymm6,ymm6,ymm8
6411 vpsrld ymm8,ymm5,25
6412 vpslld ymm5,ymm5,32-25
6413 vpxor ymm5,ymm5,ymm8
6414 vpsrld ymm8,ymm4,25
6415 vpslld ymm4,ymm4,32-25
6416 vpxor ymm4,ymm4,ymm8
6417 vmovdqa ymm8,YMMWORD[((160+128))+rbp]
6418 vpalignr ymm7,ymm7,ymm7,4
6419 vpalignr ymm11,ymm11,ymm11,8
6420 vpalignr ymm15,ymm15,ymm15,12
6421 vpalignr ymm6,ymm6,ymm6,4
6422 vpalignr ymm10,ymm10,ymm10,8
6423 vpalignr ymm14,ymm14,ymm14,12
6424 vpalignr ymm5,ymm5,ymm5,4
6425 vpalignr ymm9,ymm9,ymm9,8
6426 vpalignr ymm13,ymm13,ymm13,12
6427 vpalignr ymm4,ymm4,ymm4,4
6428 vpalignr ymm8,ymm8,ymm8,8
6429 vpalignr ymm12,ymm12,ymm12,12
6430 vmovdqa YMMWORD[(160+128)+rbp],ymm8
6431 vmovdqa ymm8,YMMWORD[$L$rol16]
6432 vpaddd ymm3,ymm3,ymm7
6433 vpaddd ymm2,ymm2,ymm6
6434 vpaddd ymm1,ymm1,ymm5
6435 vpaddd ymm0,ymm0,ymm4
6436 vpxor ymm15,ymm15,ymm3
6437 vpxor ymm14,ymm14,ymm2
6438 vpxor ymm13,ymm13,ymm1
6439 vpxor ymm12,ymm12,ymm0
6440 vpshufb ymm15,ymm15,ymm8
6441 vpshufb ymm14,ymm14,ymm8
6442 vpshufb ymm13,ymm13,ymm8
6443 vpshufb ymm12,ymm12,ymm8
6444 vpaddd ymm11,ymm11,ymm15
6445 vpaddd ymm10,ymm10,ymm14
6446 vpaddd ymm9,ymm9,ymm13
6447 vpaddd ymm8,ymm12,YMMWORD[((160+128))+rbp]
6448 vpxor ymm7,ymm7,ymm11
6449 vpxor ymm6,ymm6,ymm10
6450 vpxor ymm5,ymm5,ymm9
6451 vpxor ymm4,ymm4,ymm8
6452 vmovdqa YMMWORD[(160+128)+rbp],ymm8
6453 vpsrld ymm8,ymm7,20
6454 vpslld ymm7,ymm7,32-20
6455 vpxor ymm7,ymm7,ymm8
6456 vpsrld ymm8,ymm6,20
6457 vpslld ymm6,ymm6,32-20
6458 vpxor ymm6,ymm6,ymm8
6459 vpsrld ymm8,ymm5,20
6460 vpslld ymm5,ymm5,32-20
6461 vpxor ymm5,ymm5,ymm8
6462 vpsrld ymm8,ymm4,20
6463 vpslld ymm4,ymm4,32-20
6464 vpxor ymm4,ymm4,ymm8
6465 vmovdqa ymm8,YMMWORD[$L$rol8]
6466 vpaddd ymm3,ymm3,ymm7
6467 vpaddd ymm2,ymm2,ymm6
6468 vpaddd ymm1,ymm1,ymm5
6469 vpaddd ymm0,ymm0,ymm4
6470 vpxor ymm15,ymm15,ymm3
6471 vpxor ymm14,ymm14,ymm2
6472 vpxor ymm13,ymm13,ymm1
6473 vpxor ymm12,ymm12,ymm0
6474 vpshufb ymm15,ymm15,ymm8
6475 vpshufb ymm14,ymm14,ymm8
6476 vpshufb ymm13,ymm13,ymm8
6477 vpshufb ymm12,ymm12,ymm8
6478 vpaddd ymm11,ymm11,ymm15
6479 vpaddd ymm10,ymm10,ymm14
6480 vpaddd ymm9,ymm9,ymm13
6481 vpaddd ymm8,ymm12,YMMWORD[((160+128))+rbp]
6482 vpxor ymm7,ymm7,ymm11
6483 vpxor ymm6,ymm6,ymm10
6484 vpxor ymm5,ymm5,ymm9
6485 vpxor ymm4,ymm4,ymm8
6486 vmovdqa YMMWORD[(160+128)+rbp],ymm8
6487 vpsrld ymm8,ymm7,25
6488 vpslld ymm7,ymm7,32-25
6489 vpxor ymm7,ymm7,ymm8
6490 vpsrld ymm8,ymm6,25
6491 vpslld ymm6,ymm6,32-25
6492 vpxor ymm6,ymm6,ymm8
6493 vpsrld ymm8,ymm5,25
6494 vpslld ymm5,ymm5,32-25
6495 vpxor ymm5,ymm5,ymm8
6496 vpsrld ymm8,ymm4,25
6497 vpslld ymm4,ymm4,32-25
6498 vpxor ymm4,ymm4,ymm8
6499 vmovdqa ymm8,YMMWORD[((160+128))+rbp]
6500 vpalignr ymm7,ymm7,ymm7,12
6501 vpalignr ymm11,ymm11,ymm11,8
6502 vpalignr ymm15,ymm15,ymm15,4
6503 vpalignr ymm6,ymm6,ymm6,12
6504 vpalignr ymm10,ymm10,ymm10,8
6505 vpalignr ymm14,ymm14,ymm14,4
6506 vpalignr ymm5,ymm5,ymm5,12
6507 vpalignr ymm9,ymm9,ymm9,8
6508 vpalignr ymm13,ymm13,ymm13,4
6509 vpalignr ymm4,ymm4,ymm4,12
6510 vpalignr ymm8,ymm8,ymm8,8
6511 vpalignr ymm12,ymm12,ymm12,4
6512
6513 dec r10
6514 jnz NEAR $L$seal_avx2_init_rounds
6515 vpaddd ymm3,ymm3,YMMWORD[$L$chacha20_consts]
6516 vpaddd ymm7,ymm7,YMMWORD[((160+64))+rbp]
6517 vpaddd ymm11,ymm11,YMMWORD[((160+96))+rbp]
6518 vpaddd ymm15,ymm15,YMMWORD[((160+256))+rbp]
6519 vpaddd ymm2,ymm2,YMMWORD[$L$chacha20_consts]
6520 vpaddd ymm6,ymm6,YMMWORD[((160+64))+rbp]
6521 vpaddd ymm10,ymm10,YMMWORD[((160+96))+rbp]
6522 vpaddd ymm14,ymm14,YMMWORD[((160+224))+rbp]
6523 vpaddd ymm1,ymm1,YMMWORD[$L$chacha20_consts]
6524 vpaddd ymm5,ymm5,YMMWORD[((160+64))+rbp]
6525 vpaddd ymm9,ymm9,YMMWORD[((160+96))+rbp]
6526 vpaddd ymm13,ymm13,YMMWORD[((160+192))+rbp]
6527 vpaddd ymm0,ymm0,YMMWORD[$L$chacha20_consts]
6528 vpaddd ymm4,ymm4,YMMWORD[((160+64))+rbp]
6529 vpaddd ymm8,ymm8,YMMWORD[((160+96))+rbp]
6530 vpaddd ymm12,ymm12,YMMWORD[((160+160))+rbp]
6531
6532 vperm2i128 ymm11,ymm15,ymm11,0x13
6533 vperm2i128 ymm15,ymm7,ymm3,0x02
6534 vperm2i128 ymm3,ymm7,ymm3,0x13
6535 vpand ymm15,ymm15,YMMWORD[$L$clamp]
6536 vmovdqa YMMWORD[(160+0)+rbp],ymm15
6537 mov r8,r8
6538 call poly_hash_ad_internal
6539
6540 vpxor ymm3,ymm3,YMMWORD[rsi]
6541 vpxor ymm11,ymm11,YMMWORD[32+rsi]
6542 vmovdqu YMMWORD[rdi],ymm3
6543 vmovdqu YMMWORD[32+rdi],ymm11
6544 vperm2i128 ymm15,ymm6,ymm2,0x02
6545 vperm2i128 ymm6,ymm6,ymm2,0x13
6546 vperm2i128 ymm2,ymm14,ymm10,0x02
6547 vperm2i128 ymm10,ymm14,ymm10,0x13
6548 vpxor ymm15,ymm15,YMMWORD[((0+64))+rsi]
6549 vpxor ymm2,ymm2,YMMWORD[((32+64))+rsi]
6550 vpxor ymm6,ymm6,YMMWORD[((64+64))+rsi]
6551 vpxor ymm10,ymm10,YMMWORD[((96+64))+rsi]
6552 vmovdqu YMMWORD[(0+64)+rdi],ymm15
6553 vmovdqu YMMWORD[(32+64)+rdi],ymm2
6554 vmovdqu YMMWORD[(64+64)+rdi],ymm6
6555 vmovdqu YMMWORD[(96+64)+rdi],ymm10
6556 vperm2i128 ymm15,ymm5,ymm1,0x02
6557 vperm2i128 ymm5,ymm5,ymm1,0x13
6558 vperm2i128 ymm1,ymm13,ymm9,0x02
6559 vperm2i128 ymm9,ymm13,ymm9,0x13
6560 vpxor ymm15,ymm15,YMMWORD[((0+192))+rsi]
6561 vpxor ymm1,ymm1,YMMWORD[((32+192))+rsi]
6562 vpxor ymm5,ymm5,YMMWORD[((64+192))+rsi]
6563 vpxor ymm9,ymm9,YMMWORD[((96+192))+rsi]
6564 vmovdqu YMMWORD[(0+192)+rdi],ymm15
6565 vmovdqu YMMWORD[(32+192)+rdi],ymm1
6566 vmovdqu YMMWORD[(64+192)+rdi],ymm5
6567 vmovdqu YMMWORD[(96+192)+rdi],ymm9
6568 vperm2i128 ymm15,ymm4,ymm0,0x13
6569 vperm2i128 ymm0,ymm4,ymm0,0x02
6570 vperm2i128 ymm4,ymm12,ymm8,0x02
6571 vperm2i128 ymm12,ymm12,ymm8,0x13
6572 vmovdqa ymm8,ymm15
6573
6574 lea rsi,[320+rsi]
6575 sub rbx,10*32
6576 mov rcx,10*32
6577 cmp rbx,4*32
6578 jbe NEAR $L$seal_avx2_short_hash_remainder
6579 vpxor ymm0,ymm0,YMMWORD[rsi]
6580 vpxor ymm4,ymm4,YMMWORD[32+rsi]
6581 vpxor ymm8,ymm8,YMMWORD[64+rsi]
6582 vpxor ymm12,ymm12,YMMWORD[96+rsi]
6583 vmovdqu YMMWORD[320+rdi],ymm0
6584 vmovdqu YMMWORD[352+rdi],ymm4
6585 vmovdqu YMMWORD[384+rdi],ymm8
6586 vmovdqu YMMWORD[416+rdi],ymm12
6587 lea rsi,[128+rsi]
6588 sub rbx,4*32
6589 mov rcx,8
6590 mov r8,2
6591 cmp rbx,4*32
6592 jbe NEAR $L$seal_avx2_tail_128
6593 cmp rbx,8*32
6594 jbe NEAR $L$seal_avx2_tail_256
6595 cmp rbx,12*32
6596 jbe NEAR $L$seal_avx2_tail_384
6597 cmp rbx,16*32
6598 jbe NEAR $L$seal_avx2_tail_512
6599 vmovdqa ymm0,YMMWORD[$L$chacha20_consts]
6600 vmovdqa ymm4,YMMWORD[((160+64))+rbp]
6601 vmovdqa ymm8,YMMWORD[((160+96))+rbp]
6602 vmovdqa ymm1,ymm0
6603 vmovdqa ymm5,ymm4
6604 vmovdqa ymm9,ymm8
6605 vmovdqa ymm2,ymm0
6606 vmovdqa ymm6,ymm4
6607 vmovdqa ymm10,ymm8
6608 vmovdqa ymm3,ymm0
6609 vmovdqa ymm7,ymm4
6610 vmovdqa ymm11,ymm8
6611 vmovdqa ymm12,YMMWORD[$L$avx2_inc]
6612 vpaddd ymm15,ymm12,YMMWORD[((160+160))+rbp]
6613 vpaddd ymm14,ymm12,ymm15
6614 vpaddd ymm13,ymm12,ymm14
6615 vpaddd ymm12,ymm12,ymm13
6616 vmovdqa YMMWORD[(160+256)+rbp],ymm15
6617 vmovdqa YMMWORD[(160+224)+rbp],ymm14
6618 vmovdqa YMMWORD[(160+192)+rbp],ymm13
6619 vmovdqa YMMWORD[(160+160)+rbp],ymm12
6620 vmovdqa YMMWORD[(160+128)+rbp],ymm8
6621 vmovdqa ymm8,YMMWORD[$L$rol16]
6622 vpaddd ymm3,ymm3,ymm7
6623 vpaddd ymm2,ymm2,ymm6
6624 vpaddd ymm1,ymm1,ymm5
6625 vpaddd ymm0,ymm0,ymm4
6626 vpxor ymm15,ymm15,ymm3
6627 vpxor ymm14,ymm14,ymm2
6628 vpxor ymm13,ymm13,ymm1
6629 vpxor ymm12,ymm12,ymm0
6630 vpshufb ymm15,ymm15,ymm8
6631 vpshufb ymm14,ymm14,ymm8
6632 vpshufb ymm13,ymm13,ymm8
6633 vpshufb ymm12,ymm12,ymm8
6634 vpaddd ymm11,ymm11,ymm15
6635 vpaddd ymm10,ymm10,ymm14
6636 vpaddd ymm9,ymm9,ymm13
6637 vpaddd ymm8,ymm12,YMMWORD[((160+128))+rbp]
6638 vpxor ymm7,ymm7,ymm11
6639 vpxor ymm6,ymm6,ymm10
6640 vpxor ymm5,ymm5,ymm9
6641 vpxor ymm4,ymm4,ymm8
6642 vmovdqa YMMWORD[(160+128)+rbp],ymm8
6643 vpsrld ymm8,ymm7,20
6644 vpslld ymm7,ymm7,32-20
6645 vpxor ymm7,ymm7,ymm8
6646 vpsrld ymm8,ymm6,20
6647 vpslld ymm6,ymm6,32-20
6648 vpxor ymm6,ymm6,ymm8
6649 vpsrld ymm8,ymm5,20
6650 vpslld ymm5,ymm5,32-20
6651 vpxor ymm5,ymm5,ymm8
6652 vpsrld ymm8,ymm4,20
6653 vpslld ymm4,ymm4,32-20
6654 vpxor ymm4,ymm4,ymm8
6655 vmovdqa ymm8,YMMWORD[$L$rol8]
6656 vpaddd ymm3,ymm3,ymm7
6657 vpaddd ymm2,ymm2,ymm6
6658 vpaddd ymm1,ymm1,ymm5
6659 vpaddd ymm0,ymm0,ymm4
6660 vpxor ymm15,ymm15,ymm3
6661 vpxor ymm14,ymm14,ymm2
6662 vpxor ymm13,ymm13,ymm1
6663 vpxor ymm12,ymm12,ymm0
6664 vpshufb ymm15,ymm15,ymm8
6665 vpshufb ymm14,ymm14,ymm8
6666 vpshufb ymm13,ymm13,ymm8
6667 vpshufb ymm12,ymm12,ymm8
6668 vpaddd ymm11,ymm11,ymm15
6669 vpaddd ymm10,ymm10,ymm14
6670 vpaddd ymm9,ymm9,ymm13
6671 vpaddd ymm8,ymm12,YMMWORD[((160+128))+rbp]
6672 vpxor ymm7,ymm7,ymm11
6673 vpxor ymm6,ymm6,ymm10
6674 vpxor ymm5,ymm5,ymm9
6675 vpxor ymm4,ymm4,ymm8
6676 vmovdqa YMMWORD[(160+128)+rbp],ymm8
6677 vpsrld ymm8,ymm7,25
6678 vpslld ymm7,ymm7,32-25
6679 vpxor ymm7,ymm7,ymm8
6680 vpsrld ymm8,ymm6,25
6681 vpslld ymm6,ymm6,32-25
6682 vpxor ymm6,ymm6,ymm8
6683 vpsrld ymm8,ymm5,25
6684 vpslld ymm5,ymm5,32-25
6685 vpxor ymm5,ymm5,ymm8
6686 vpsrld ymm8,ymm4,25
6687 vpslld ymm4,ymm4,32-25
6688 vpxor ymm4,ymm4,ymm8
6689 vmovdqa ymm8,YMMWORD[((160+128))+rbp]
6690 vpalignr ymm7,ymm7,ymm7,4
6691 vpalignr ymm11,ymm11,ymm11,8
6692 vpalignr ymm15,ymm15,ymm15,12
6693 vpalignr ymm6,ymm6,ymm6,4
6694 vpalignr ymm10,ymm10,ymm10,8
6695 vpalignr ymm14,ymm14,ymm14,12
6696 vpalignr ymm5,ymm5,ymm5,4
6697 vpalignr ymm9,ymm9,ymm9,8
6698 vpalignr ymm13,ymm13,ymm13,12
6699 vpalignr ymm4,ymm4,ymm4,4
6700 vpalignr ymm8,ymm8,ymm8,8
6701 vpalignr ymm12,ymm12,ymm12,12
6702 vmovdqa YMMWORD[(160+128)+rbp],ymm8
6703 vmovdqa ymm8,YMMWORD[$L$rol16]
6704 vpaddd ymm3,ymm3,ymm7
6705 vpaddd ymm2,ymm2,ymm6
6706 vpaddd ymm1,ymm1,ymm5
6707 vpaddd ymm0,ymm0,ymm4
6708 vpxor ymm15,ymm15,ymm3
6709 vpxor ymm14,ymm14,ymm2
6710 vpxor ymm13,ymm13,ymm1
6711 vpxor ymm12,ymm12,ymm0
6712 vpshufb ymm15,ymm15,ymm8
6713 vpshufb ymm14,ymm14,ymm8
6714 vpshufb ymm13,ymm13,ymm8
6715 vpshufb ymm12,ymm12,ymm8
6716 vpaddd ymm11,ymm11,ymm15
6717 vpaddd ymm10,ymm10,ymm14
6718 vpaddd ymm9,ymm9,ymm13
6719 vpaddd ymm8,ymm12,YMMWORD[((160+128))+rbp]
6720 vpxor ymm7,ymm7,ymm11
6721 vpxor ymm6,ymm6,ymm10
6722 vpxor ymm5,ymm5,ymm9
6723 vpxor ymm4,ymm4,ymm8
6724 vmovdqa YMMWORD[(160+128)+rbp],ymm8
6725 vpsrld ymm8,ymm7,20
6726 vpslld ymm7,ymm7,32-20
6727 vpxor ymm7,ymm7,ymm8
6728 vpsrld ymm8,ymm6,20
6729 vpslld ymm6,ymm6,32-20
6730 vpxor ymm6,ymm6,ymm8
6731 vpsrld ymm8,ymm5,20
6732 vpslld ymm5,ymm5,32-20
6733 vpxor ymm5,ymm5,ymm8
6734 vpsrld ymm8,ymm4,20
6735 vpslld ymm4,ymm4,32-20
6736 vpxor ymm4,ymm4,ymm8
6737 vmovdqa ymm8,YMMWORD[$L$rol8]
6738 vpaddd ymm3,ymm3,ymm7
6739 vpaddd ymm2,ymm2,ymm6
6740 vpaddd ymm1,ymm1,ymm5
6741 vpaddd ymm0,ymm0,ymm4
6742 vpxor ymm15,ymm15,ymm3
6743 vpxor ymm14,ymm14,ymm2
6744 vpxor ymm13,ymm13,ymm1
6745 vpxor ymm12,ymm12,ymm0
6746 vpshufb ymm15,ymm15,ymm8
6747 vpshufb ymm14,ymm14,ymm8
6748 vpshufb ymm13,ymm13,ymm8
6749 vpshufb ymm12,ymm12,ymm8
6750 vpaddd ymm11,ymm11,ymm15
6751 vpaddd ymm10,ymm10,ymm14
6752 vpaddd ymm9,ymm9,ymm13
6753 vpaddd ymm8,ymm12,YMMWORD[((160+128))+rbp]
6754 vpxor ymm7,ymm7,ymm11
6755 vpxor ymm6,ymm6,ymm10
6756 vpxor ymm5,ymm5,ymm9
6757 vpxor ymm4,ymm4,ymm8
6758 vmovdqa YMMWORD[(160+128)+rbp],ymm8
6759 vpsrld ymm8,ymm7,25
6760 vpslld ymm7,ymm7,32-25
6761 vpxor ymm7,ymm7,ymm8
6762 vpsrld ymm8,ymm6,25
6763 vpslld ymm6,ymm6,32-25
6764 vpxor ymm6,ymm6,ymm8
6765 vpsrld ymm8,ymm5,25
6766 vpslld ymm5,ymm5,32-25
6767 vpxor ymm5,ymm5,ymm8
6768 vpsrld ymm8,ymm4,25
6769 vpslld ymm4,ymm4,32-25
6770 vpxor ymm4,ymm4,ymm8
6771 vmovdqa ymm8,YMMWORD[((160+128))+rbp]
6772 vpalignr ymm7,ymm7,ymm7,12
6773 vpalignr ymm11,ymm11,ymm11,8
6774 vpalignr ymm15,ymm15,ymm15,4
6775 vpalignr ymm6,ymm6,ymm6,12
6776 vpalignr ymm10,ymm10,ymm10,8
6777 vpalignr ymm14,ymm14,ymm14,4
6778 vpalignr ymm5,ymm5,ymm5,12
6779 vpalignr ymm9,ymm9,ymm9,8
6780 vpalignr ymm13,ymm13,ymm13,4
6781 vpalignr ymm4,ymm4,ymm4,12
6782 vpalignr ymm8,ymm8,ymm8,8
6783 vpalignr ymm12,ymm12,ymm12,4
6784 vmovdqa YMMWORD[(160+128)+rbp],ymm8
6785 vmovdqa ymm8,YMMWORD[$L$rol16]
6786 vpaddd ymm3,ymm3,ymm7
6787 vpaddd ymm2,ymm2,ymm6
6788 vpaddd ymm1,ymm1,ymm5
6789 vpaddd ymm0,ymm0,ymm4
6790 vpxor ymm15,ymm15,ymm3
6791 vpxor ymm14,ymm14,ymm2
6792 vpxor ymm13,ymm13,ymm1
6793 vpxor ymm12,ymm12,ymm0
6794 vpshufb ymm15,ymm15,ymm8
6795 vpshufb ymm14,ymm14,ymm8
6796 vpshufb ymm13,ymm13,ymm8
6797 vpshufb ymm12,ymm12,ymm8
6798 vpaddd ymm11,ymm11,ymm15
6799 vpaddd ymm10,ymm10,ymm14
6800 vpaddd ymm9,ymm9,ymm13
6801 vpaddd ymm8,ymm12,YMMWORD[((160+128))+rbp]
6802 vpxor ymm7,ymm7,ymm11
6803 vpxor ymm6,ymm6,ymm10
6804 vpxor ymm5,ymm5,ymm9
6805 vpxor ymm4,ymm4,ymm8
6806 vmovdqa YMMWORD[(160+128)+rbp],ymm8
6807 vpsrld ymm8,ymm7,20
6808 vpslld ymm7,ymm7,32-20
6809 vpxor ymm7,ymm7,ymm8
6810 vpsrld ymm8,ymm6,20
6811 vpslld ymm6,ymm6,32-20
6812 vpxor ymm6,ymm6,ymm8
6813 vpsrld ymm8,ymm5,20
6814 vpslld ymm5,ymm5,32-20
6815 vpxor ymm5,ymm5,ymm8
6816 vpsrld ymm8,ymm4,20
6817 vpslld ymm4,ymm4,32-20
6818 vpxor ymm4,ymm4,ymm8
6819 vmovdqa ymm8,YMMWORD[$L$rol8]
6820 vpaddd ymm3,ymm3,ymm7
6821 vpaddd ymm2,ymm2,ymm6
6822 vpaddd ymm1,ymm1,ymm5
6823 vpaddd ymm0,ymm0,ymm4
6824 vpxor ymm15,ymm15,ymm3
6825
6826 sub rdi,16
6827 mov rcx,9
6828 jmp NEAR $L$seal_avx2_main_loop_rounds_entry
6829 ALIGN 32
6830 $L$seal_avx2_main_loop:
6831 vmovdqa ymm0,YMMWORD[$L$chacha20_consts]
6832 vmovdqa ymm4,YMMWORD[((160+64))+rbp]
6833 vmovdqa ymm8,YMMWORD[((160+96))+rbp]
6834 vmovdqa ymm1,ymm0
6835 vmovdqa ymm5,ymm4
6836 vmovdqa ymm9,ymm8
6837 vmovdqa ymm2,ymm0
6838 vmovdqa ymm6,ymm4
6839 vmovdqa ymm10,ymm8
6840 vmovdqa ymm3,ymm0
6841 vmovdqa ymm7,ymm4
6842 vmovdqa ymm11,ymm8
6843 vmovdqa ymm12,YMMWORD[$L$avx2_inc]
6844 vpaddd ymm15,ymm12,YMMWORD[((160+160))+rbp]
6845 vpaddd ymm14,ymm12,ymm15
6846 vpaddd ymm13,ymm12,ymm14
6847 vpaddd ymm12,ymm12,ymm13
6848 vmovdqa YMMWORD[(160+256)+rbp],ymm15
6849 vmovdqa YMMWORD[(160+224)+rbp],ymm14
6850 vmovdqa YMMWORD[(160+192)+rbp],ymm13
6851 vmovdqa YMMWORD[(160+160)+rbp],ymm12
6852
6853 mov rcx,10
6854 ALIGN 32
6855 $L$seal_avx2_main_loop_rounds:
6856 add r10,QWORD[((0+0))+rdi]
6857 adc r11,QWORD[((8+0))+rdi]
6858 adc r12,1
6859 vmovdqa YMMWORD[(160+128)+rbp],ymm8
6860 vmovdqa ymm8,YMMWORD[$L$rol16]
6861 vpaddd ymm3,ymm3,ymm7
6862 vpaddd ymm2,ymm2,ymm6
6863 vpaddd ymm1,ymm1,ymm5
6864 vpaddd ymm0,ymm0,ymm4
6865 vpxor ymm15,ymm15,ymm3
6866 vpxor ymm14,ymm14,ymm2
6867 vpxor ymm13,ymm13,ymm1
6868 vpxor ymm12,ymm12,ymm0
6869 mov rdx,QWORD[((0+160+0))+rbp]
6870 mov r15,rdx
6871 mulx r14,r13,r10
6872 mulx rdx,rax,r11
6873 imul r15,r12
6874 add r14,rax
6875 adc r15,rdx
6876 vpshufb ymm15,ymm15,ymm8
6877 vpshufb ymm14,ymm14,ymm8
6878 vpshufb ymm13,ymm13,ymm8
6879 vpshufb ymm12,ymm12,ymm8
6880 vpaddd ymm11,ymm11,ymm15
6881 vpaddd ymm10,ymm10,ymm14
6882 vpaddd ymm9,ymm9,ymm13
6883 vpaddd ymm8,ymm12,YMMWORD[((160+128))+rbp]
6884 vpxor ymm7,ymm7,ymm11
6885 mov rdx,QWORD[((8+160+0))+rbp]
6886 mulx rax,r10,r10
6887 add r14,r10
6888 mulx r9,r11,r11
6889 adc r15,r11
6890 adc r9,0
6891 imul rdx,r12
6892 vpxor ymm6,ymm6,ymm10
6893 vpxor ymm5,ymm5,ymm9
6894 vpxor ymm4,ymm4,ymm8
6895 vmovdqa YMMWORD[(160+128)+rbp],ymm8
6896 vpsrld ymm8,ymm7,20
6897 vpslld ymm7,ymm7,32-20
6898 vpxor ymm7,ymm7,ymm8
6899 vpsrld ymm8,ymm6,20
6900 vpslld ymm6,ymm6,32-20
6901 vpxor ymm6,ymm6,ymm8
6902 vpsrld ymm8,ymm5,20
6903 vpslld ymm5,ymm5,32-20
6904 add r15,rax
6905 adc r9,rdx
6906 vpxor ymm5,ymm5,ymm8
6907 vpsrld ymm8,ymm4,20
6908 vpslld ymm4,ymm4,32-20
6909 vpxor ymm4,ymm4,ymm8
6910 vmovdqa ymm8,YMMWORD[$L$rol8]
6911 vpaddd ymm3,ymm3,ymm7
6912 vpaddd ymm2,ymm2,ymm6
6913 vpaddd ymm1,ymm1,ymm5
6914 vpaddd ymm0,ymm0,ymm4
6915 vpxor ymm15,ymm15,ymm3
6916 mov r10,r13
6917 mov r11,r14
6918 mov r12,r15
6919 and r12,3
6920 mov r13,r15
6921 and r13,-4
6922 mov r14,r9
6923 shrd r15,r9,2
6924 shr r9,2
6925 add r15,r13
6926 adc r9,r14
6927 add r10,r15
6928 adc r11,r9
6929 adc r12,0
6930
6931 $L$seal_avx2_main_loop_rounds_entry:
6932 vpxor ymm14,ymm14,ymm2
6933 vpxor ymm13,ymm13,ymm1
6934 vpxor ymm12,ymm12,ymm0
6935 vpshufb ymm15,ymm15,ymm8
6936 vpshufb ymm14,ymm14,ymm8
6937 vpshufb ymm13,ymm13,ymm8
6938 vpshufb ymm12,ymm12,ymm8
6939 vpaddd ymm11,ymm11,ymm15
6940 vpaddd ymm10,ymm10,ymm14
6941 add r10,QWORD[((0+16))+rdi]
6942 adc r11,QWORD[((8+16))+rdi]
6943 adc r12,1
6944 vpaddd ymm9,ymm9,ymm13
6945 vpaddd ymm8,ymm12,YMMWORD[((160+128))+rbp]
6946 vpxor ymm7,ymm7,ymm11
6947 vpxor ymm6,ymm6,ymm10
6948 vpxor ymm5,ymm5,ymm9
6949 vpxor ymm4,ymm4,ymm8
6950 vmovdqa YMMWORD[(160+128)+rbp],ymm8
6951 vpsrld ymm8,ymm7,25
6952 mov rdx,QWORD[((0+160+0))+rbp]
6953 mov r15,rdx
6954 mulx r14,r13,r10
6955 mulx rdx,rax,r11
6956 imul r15,r12
6957 add r14,rax
6958 adc r15,rdx
6959 vpslld ymm7,ymm7,32-25
6960 vpxor ymm7,ymm7,ymm8
6961 vpsrld ymm8,ymm6,25
6962 vpslld ymm6,ymm6,32-25
6963 vpxor ymm6,ymm6,ymm8
6964 vpsrld ymm8,ymm5,25
6965 vpslld ymm5,ymm5,32-25
6966 vpxor ymm5,ymm5,ymm8
6967 vpsrld ymm8,ymm4,25
6968 vpslld ymm4,ymm4,32-25
6969 vpxor ymm4,ymm4,ymm8
6970 vmovdqa ymm8,YMMWORD[((160+128))+rbp]
6971 vpalignr ymm7,ymm7,ymm7,4
6972 vpalignr ymm11,ymm11,ymm11,8
6973 vpalignr ymm15,ymm15,ymm15,12
6974 vpalignr ymm6,ymm6,ymm6,4
6975 vpalignr ymm10,ymm10,ymm10,8
6976 vpalignr ymm14,ymm14,ymm14,12
6977 mov rdx,QWORD[((8+160+0))+rbp]
6978 mulx rax,r10,r10
6979 add r14,r10
6980 mulx r9,r11,r11
6981 adc r15,r11
6982 adc r9,0
6983 imul rdx,r12
6984 vpalignr ymm5,ymm5,ymm5,4
6985 vpalignr ymm9,ymm9,ymm9,8
6986 vpalignr ymm13,ymm13,ymm13,12
6987 vpalignr ymm4,ymm4,ymm4,4
6988 vpalignr ymm8,ymm8,ymm8,8
6989 vpalignr ymm12,ymm12,ymm12,12
6990 vmovdqa YMMWORD[(160+128)+rbp],ymm8
6991 vmovdqa ymm8,YMMWORD[$L$rol16]
6992 vpaddd ymm3,ymm3,ymm7
6993 vpaddd ymm2,ymm2,ymm6
6994 vpaddd ymm1,ymm1,ymm5
6995 vpaddd ymm0,ymm0,ymm4
6996 vpxor ymm15,ymm15,ymm3
6997 vpxor ymm14,ymm14,ymm2
6998 vpxor ymm13,ymm13,ymm1
6999 vpxor ymm12,ymm12,ymm0
7000 vpshufb ymm15,ymm15,ymm8
7001 vpshufb ymm14,ymm14,ymm8
7002 add r15,rax
7003 adc r9,rdx
7004 vpshufb ymm13,ymm13,ymm8
7005 vpshufb ymm12,ymm12,ymm8
7006 vpaddd ymm11,ymm11,ymm15
7007 vpaddd ymm10,ymm10,ymm14
7008 vpaddd ymm9,ymm9,ymm13
7009 vpaddd ymm8,ymm12,YMMWORD[((160+128))+rbp]
7010 vpxor ymm7,ymm7,ymm11
7011 vpxor ymm6,ymm6,ymm10
7012 vpxor ymm5,ymm5,ymm9
7013 mov r10,r13
7014 mov r11,r14
7015 mov r12,r15
7016 and r12,3
7017 mov r13,r15
7018 and r13,-4
7019 mov r14,r9
7020 shrd r15,r9,2
7021 shr r9,2
7022 add r15,r13
7023 adc r9,r14
7024 add r10,r15
7025 adc r11,r9
7026 adc r12,0
7027 vpxor ymm4,ymm4,ymm8
7028 vmovdqa YMMWORD[(160+128)+rbp],ymm8
7029 vpsrld ymm8,ymm7,20
7030 vpslld ymm7,ymm7,32-20
7031 vpxor ymm7,ymm7,ymm8
7032 vpsrld ymm8,ymm6,20
7033 vpslld ymm6,ymm6,32-20
7034 vpxor ymm6,ymm6,ymm8
7035 add r10,QWORD[((0+32))+rdi]
7036 adc r11,QWORD[((8+32))+rdi]
7037 adc r12,1
7038
7039 lea rdi,[48+rdi]
7040 vpsrld ymm8,ymm5,20
7041 vpslld ymm5,ymm5,32-20
7042 vpxor ymm5,ymm5,ymm8
7043 vpsrld ymm8,ymm4,20
7044 vpslld ymm4,ymm4,32-20
7045 vpxor ymm4,ymm4,ymm8
7046 vmovdqa ymm8,YMMWORD[$L$rol8]
7047 vpaddd ymm3,ymm3,ymm7
7048 vpaddd ymm2,ymm2,ymm6
7049 vpaddd ymm1,ymm1,ymm5
7050 vpaddd ymm0,ymm0,ymm4
7051 vpxor ymm15,ymm15,ymm3
7052 vpxor ymm14,ymm14,ymm2
7053 vpxor ymm13,ymm13,ymm1
7054 vpxor ymm12,ymm12,ymm0
7055 vpshufb ymm15,ymm15,ymm8
7056 vpshufb ymm14,ymm14,ymm8
7057 vpshufb ymm13,ymm13,ymm8
7058 mov rdx,QWORD[((0+160+0))+rbp]
7059 mov r15,rdx
7060 mulx r14,r13,r10
7061 mulx rdx,rax,r11
7062 imul r15,r12
7063 add r14,rax
7064 adc r15,rdx
7065 vpshufb ymm12,ymm12,ymm8
7066 vpaddd ymm11,ymm11,ymm15
7067 vpaddd ymm10,ymm10,ymm14
7068 vpaddd ymm9,ymm9,ymm13
7069 vpaddd ymm8,ymm12,YMMWORD[((160+128))+rbp]
7070 vpxor ymm7,ymm7,ymm11
7071 vpxor ymm6,ymm6,ymm10
7072 vpxor ymm5,ymm5,ymm9
7073 mov rdx,QWORD[((8+160+0))+rbp]
7074 mulx rax,r10,r10
7075 add r14,r10
7076 mulx r9,r11,r11
7077 adc r15,r11
7078 adc r9,0
7079 imul rdx,r12
7080 vpxor ymm4,ymm4,ymm8
7081 vmovdqa YMMWORD[(160+128)+rbp],ymm8
7082 vpsrld ymm8,ymm7,25
7083 vpslld ymm7,ymm7,32-25
7084 vpxor ymm7,ymm7,ymm8
7085 vpsrld ymm8,ymm6,25
7086 vpslld ymm6,ymm6,32-25
7087 vpxor ymm6,ymm6,ymm8
7088 add r15,rax
7089 adc r9,rdx
7090 vpsrld ymm8,ymm5,25
7091 vpslld ymm5,ymm5,32-25
7092 vpxor ymm5,ymm5,ymm8
7093 vpsrld ymm8,ymm4,25
7094 vpslld ymm4,ymm4,32-25
7095 vpxor ymm4,ymm4,ymm8
7096 vmovdqa ymm8,YMMWORD[((160+128))+rbp]
7097 vpalignr ymm7,ymm7,ymm7,12
7098 vpalignr ymm11,ymm11,ymm11,8
7099 vpalignr ymm15,ymm15,ymm15,4
7100 vpalignr ymm6,ymm6,ymm6,12
7101 vpalignr ymm10,ymm10,ymm10,8
7102 vpalignr ymm14,ymm14,ymm14,4
7103 vpalignr ymm5,ymm5,ymm5,12
7104 vpalignr ymm9,ymm9,ymm9,8
7105 vpalignr ymm13,ymm13,ymm13,4
7106 vpalignr ymm4,ymm4,ymm4,12
7107 vpalignr ymm8,ymm8,ymm8,8
7108 mov r10,r13
7109 mov r11,r14
7110 mov r12,r15
7111 and r12,3
7112 mov r13,r15
7113 and r13,-4
7114 mov r14,r9
7115 shrd r15,r9,2
7116 shr r9,2
7117 add r15,r13
7118 adc r9,r14
7119 add r10,r15
7120 adc r11,r9
7121 adc r12,0
7122 vpalignr ymm12,ymm12,ymm12,4
7123
7124 dec rcx
7125 jne NEAR $L$seal_avx2_main_loop_rounds
7126 vpaddd ymm3,ymm3,YMMWORD[$L$chacha20_consts]
7127 vpaddd ymm7,ymm7,YMMWORD[((160+64))+rbp]
7128 vpaddd ymm11,ymm11,YMMWORD[((160+96))+rbp]
7129 vpaddd ymm15,ymm15,YMMWORD[((160+256))+rbp]
7130 vpaddd ymm2,ymm2,YMMWORD[$L$chacha20_consts]
7131 vpaddd ymm6,ymm6,YMMWORD[((160+64))+rbp]
7132 vpaddd ymm10,ymm10,YMMWORD[((160+96))+rbp]
7133 vpaddd ymm14,ymm14,YMMWORD[((160+224))+rbp]
7134 vpaddd ymm1,ymm1,YMMWORD[$L$chacha20_consts]
7135 vpaddd ymm5,ymm5,YMMWORD[((160+64))+rbp]
7136 vpaddd ymm9,ymm9,YMMWORD[((160+96))+rbp]
7137 vpaddd ymm13,ymm13,YMMWORD[((160+192))+rbp]
7138 vpaddd ymm0,ymm0,YMMWORD[$L$chacha20_consts]
7139 vpaddd ymm4,ymm4,YMMWORD[((160+64))+rbp]
7140 vpaddd ymm8,ymm8,YMMWORD[((160+96))+rbp]
7141 vpaddd ymm12,ymm12,YMMWORD[((160+160))+rbp]
7142
7143 vmovdqa YMMWORD[(160+128)+rbp],ymm0
7144 add r10,QWORD[((0+0))+rdi]
7145 adc r11,QWORD[((8+0))+rdi]
7146 adc r12,1
7147 mov rdx,QWORD[((0+160+0))+rbp]
7148 mov r15,rdx
7149 mulx r14,r13,r10
7150 mulx rdx,rax,r11
7151 imul r15,r12
7152 add r14,rax
7153 adc r15,rdx
7154 mov rdx,QWORD[((8+160+0))+rbp]
7155 mulx rax,r10,r10
7156 add r14,r10
7157 mulx r9,r11,r11
7158 adc r15,r11
7159 adc r9,0
7160 imul rdx,r12
7161 add r15,rax
7162 adc r9,rdx
7163 mov r10,r13
7164 mov r11,r14
7165 mov r12,r15
7166 and r12,3
7167 mov r13,r15
7168 and r13,-4
7169 mov r14,r9
7170 shrd r15,r9,2
7171 shr r9,2
7172 add r15,r13
7173 adc r9,r14
7174 add r10,r15
7175 adc r11,r9
7176 adc r12,0
7177 add r10,QWORD[((0+16))+rdi]
7178 adc r11,QWORD[((8+16))+rdi]
7179 adc r12,1
7180 mov rdx,QWORD[((0+160+0))+rbp]
7181 mov r15,rdx
7182 mulx r14,r13,r10
7183 mulx rdx,rax,r11
7184 imul r15,r12
7185 add r14,rax
7186 adc r15,rdx
7187 mov rdx,QWORD[((8+160+0))+rbp]
7188 mulx rax,r10,r10
7189 add r14,r10
7190 mulx r9,r11,r11
7191 adc r15,r11
7192 adc r9,0
7193 imul rdx,r12
7194 add r15,rax
7195 adc r9,rdx
7196 mov r10,r13
7197 mov r11,r14
7198 mov r12,r15
7199 and r12,3
7200 mov r13,r15
7201 and r13,-4
7202 mov r14,r9
7203 shrd r15,r9,2
7204 shr r9,2
7205 add r15,r13
7206 adc r9,r14
7207 add r10,r15
7208 adc r11,r9
7209 adc r12,0
7210
7211 lea rdi,[32+rdi]
7212 vperm2i128 ymm0,ymm7,ymm3,0x02
7213 vperm2i128 ymm7,ymm7,ymm3,0x13
7214 vperm2i128 ymm3,ymm15,ymm11,0x02
7215 vperm2i128 ymm11,ymm15,ymm11,0x13
7216 vpxor ymm0,ymm0,YMMWORD[((0+0))+rsi]
7217 vpxor ymm3,ymm3,YMMWORD[((32+0))+rsi]
7218 vpxor ymm7,ymm7,YMMWORD[((64+0))+rsi]
7219 vpxor ymm11,ymm11,YMMWORD[((96+0))+rsi]
7220 vmovdqu YMMWORD[(0+0)+rdi],ymm0
7221 vmovdqu YMMWORD[(32+0)+rdi],ymm3
7222 vmovdqu YMMWORD[(64+0)+rdi],ymm7
7223 vmovdqu YMMWORD[(96+0)+rdi],ymm11
7224
7225 vmovdqa ymm0,YMMWORD[((160+128))+rbp]
7226 vperm2i128 ymm3,ymm6,ymm2,0x02
7227 vperm2i128 ymm6,ymm6,ymm2,0x13
7228 vperm2i128 ymm2,ymm14,ymm10,0x02
7229 vperm2i128 ymm10,ymm14,ymm10,0x13
7230 vpxor ymm3,ymm3,YMMWORD[((0+128))+rsi]
7231 vpxor ymm2,ymm2,YMMWORD[((32+128))+rsi]
7232 vpxor ymm6,ymm6,YMMWORD[((64+128))+rsi]
7233 vpxor ymm10,ymm10,YMMWORD[((96+128))+rsi]
7234 vmovdqu YMMWORD[(0+128)+rdi],ymm3
7235 vmovdqu YMMWORD[(32+128)+rdi],ymm2
7236 vmovdqu YMMWORD[(64+128)+rdi],ymm6
7237 vmovdqu YMMWORD[(96+128)+rdi],ymm10
7238 vperm2i128 ymm3,ymm5,ymm1,0x02
7239 vperm2i128 ymm5,ymm5,ymm1,0x13
7240 vperm2i128 ymm1,ymm13,ymm9,0x02
7241 vperm2i128 ymm9,ymm13,ymm9,0x13
7242 vpxor ymm3,ymm3,YMMWORD[((0+256))+rsi]
7243 vpxor ymm1,ymm1,YMMWORD[((32+256))+rsi]
7244 vpxor ymm5,ymm5,YMMWORD[((64+256))+rsi]
7245 vpxor ymm9,ymm9,YMMWORD[((96+256))+rsi]
7246 vmovdqu YMMWORD[(0+256)+rdi],ymm3
7247 vmovdqu YMMWORD[(32+256)+rdi],ymm1
7248 vmovdqu YMMWORD[(64+256)+rdi],ymm5
7249 vmovdqu YMMWORD[(96+256)+rdi],ymm9
7250 vperm2i128 ymm3,ymm4,ymm0,0x02
7251 vperm2i128 ymm4,ymm4,ymm0,0x13
7252 vperm2i128 ymm0,ymm12,ymm8,0x02
7253 vperm2i128 ymm8,ymm12,ymm8,0x13
7254 vpxor ymm3,ymm3,YMMWORD[((0+384))+rsi]
7255 vpxor ymm0,ymm0,YMMWORD[((32+384))+rsi]
7256 vpxor ymm4,ymm4,YMMWORD[((64+384))+rsi]
7257 vpxor ymm8,ymm8,YMMWORD[((96+384))+rsi]
7258 vmovdqu YMMWORD[(0+384)+rdi],ymm3
7259 vmovdqu YMMWORD[(32+384)+rdi],ymm0
7260 vmovdqu YMMWORD[(64+384)+rdi],ymm4
7261 vmovdqu YMMWORD[(96+384)+rdi],ymm8
7262
7263 lea rsi,[512+rsi]
7264 sub rbx,16*32
7265 cmp rbx,16*32
7266 jg NEAR $L$seal_avx2_main_loop
7267
7268 add r10,QWORD[((0+0))+rdi]
7269 adc r11,QWORD[((8+0))+rdi]
7270 adc r12,1
7271 mov rdx,QWORD[((0+160+0))+rbp]
7272 mov r15,rdx
7273 mulx r14,r13,r10
7274 mulx rdx,rax,r11
7275 imul r15,r12
7276 add r14,rax
7277 adc r15,rdx
7278 mov rdx,QWORD[((8+160+0))+rbp]
7279 mulx rax,r10,r10
7280 add r14,r10
7281 mulx r9,r11,r11
7282 adc r15,r11
7283 adc r9,0
7284 imul rdx,r12
7285 add r15,rax
7286 adc r9,rdx
7287 mov r10,r13
7288 mov r11,r14
7289 mov r12,r15
7290 and r12,3
7291 mov r13,r15
7292 and r13,-4
7293 mov r14,r9
7294 shrd r15,r9,2
7295 shr r9,2
7296 add r15,r13
7297 adc r9,r14
7298 add r10,r15
7299 adc r11,r9
7300 adc r12,0
7301 add r10,QWORD[((0+16))+rdi]
7302 adc r11,QWORD[((8+16))+rdi]
7303 adc r12,1
7304 mov rdx,QWORD[((0+160+0))+rbp]
7305 mov r15,rdx
7306 mulx r14,r13,r10
7307 mulx rdx,rax,r11
7308 imul r15,r12
7309 add r14,rax
7310 adc r15,rdx
7311 mov rdx,QWORD[((8+160+0))+rbp]
7312 mulx rax,r10,r10
7313 add r14,r10
7314 mulx r9,r11,r11
7315 adc r15,r11
7316 adc r9,0
7317 imul rdx,r12
7318 add r15,rax
7319 adc r9,rdx
7320 mov r10,r13
7321 mov r11,r14
7322 mov r12,r15
7323 and r12,3
7324 mov r13,r15
7325 and r13,-4
7326 mov r14,r9
7327 shrd r15,r9,2
7328 shr r9,2
7329 add r15,r13
7330 adc r9,r14
7331 add r10,r15
7332 adc r11,r9
7333 adc r12,0
7334
7335 lea rdi,[32+rdi]
7336 mov rcx,10
7337 xor r8,r8
7338
7339 cmp rbx,12*32
7340 ja NEAR $L$seal_avx2_tail_512
7341 cmp rbx,8*32
7342 ja NEAR $L$seal_avx2_tail_384
7343 cmp rbx,4*32
7344 ja NEAR $L$seal_avx2_tail_256
7345
7346 $L$seal_avx2_tail_128:
7347 vmovdqa ymm0,YMMWORD[$L$chacha20_consts]
7348 vmovdqa ymm4,YMMWORD[((160+64))+rbp]
7349 vmovdqa ymm8,YMMWORD[((160+96))+rbp]
7350 vmovdqa ymm12,YMMWORD[$L$avx2_inc]
7351 vpaddd ymm12,ymm12,YMMWORD[((160+160))+rbp]
7352 vmovdqa YMMWORD[(160+160)+rbp],ymm12
7353
7354 $L$seal_avx2_tail_128_rounds_and_3xhash:
7355 add r10,QWORD[((0+0))+rdi]
7356 adc r11,QWORD[((8+0))+rdi]
7357 adc r12,1
7358 mov rdx,QWORD[((0+160+0))+rbp]
7359 mov r15,rdx
7360 mulx r14,r13,r10
7361 mulx rdx,rax,r11
7362 imul r15,r12
7363 add r14,rax
7364 adc r15,rdx
7365 mov rdx,QWORD[((8+160+0))+rbp]
7366 mulx rax,r10,r10
7367 add r14,r10
7368 mulx r9,r11,r11
7369 adc r15,r11
7370 adc r9,0
7371 imul rdx,r12
7372 add r15,rax
7373 adc r9,rdx
7374 mov r10,r13
7375 mov r11,r14
7376 mov r12,r15
7377 and r12,3
7378 mov r13,r15
7379 and r13,-4
7380 mov r14,r9
7381 shrd r15,r9,2
7382 shr r9,2
7383 add r15,r13
7384 adc r9,r14
7385 add r10,r15
7386 adc r11,r9
7387 adc r12,0
7388
7389 lea rdi,[16+rdi]
7390 $L$seal_avx2_tail_128_rounds_and_2xhash:
7391 vpaddd ymm0,ymm0,ymm4
7392 vpxor ymm12,ymm12,ymm0
7393 vpshufb ymm12,ymm12,YMMWORD[$L$rol16]
7394 vpaddd ymm8,ymm8,ymm12
7395 vpxor ymm4,ymm4,ymm8
7396 vpsrld ymm3,ymm4,20
7397 vpslld ymm4,ymm4,12
7398 vpxor ymm4,ymm4,ymm3
7399 vpaddd ymm0,ymm0,ymm4
7400 vpxor ymm12,ymm12,ymm0
7401 vpshufb ymm12,ymm12,YMMWORD[$L$rol8]
7402 vpaddd ymm8,ymm8,ymm12
7403 vpxor ymm4,ymm4,ymm8
7404 vpslld ymm3,ymm4,7
7405 vpsrld ymm4,ymm4,25
7406 vpxor ymm4,ymm4,ymm3
7407 vpalignr ymm12,ymm12,ymm12,12
7408 vpalignr ymm8,ymm8,ymm8,8
7409 vpalignr ymm4,ymm4,ymm4,4
7410 add r10,QWORD[((0+0))+rdi]
7411 adc r11,QWORD[((8+0))+rdi]
7412 adc r12,1
7413 mov rdx,QWORD[((0+160+0))+rbp]
7414 mov r15,rdx
7415 mulx r14,r13,r10
7416 mulx rdx,rax,r11
7417 imul r15,r12
7418 add r14,rax
7419 adc r15,rdx
7420 mov rdx,QWORD[((8+160+0))+rbp]
7421 mulx rax,r10,r10
7422 add r14,r10
7423 mulx r9,r11,r11
7424 adc r15,r11
7425 adc r9,0
7426 imul rdx,r12
7427 add r15,rax
7428 adc r9,rdx
7429 mov r10,r13
7430 mov r11,r14
7431 mov r12,r15
7432 and r12,3
7433 mov r13,r15
7434 and r13,-4
7435 mov r14,r9
7436 shrd r15,r9,2
7437 shr r9,2
7438 add r15,r13
7439 adc r9,r14
7440 add r10,r15
7441 adc r11,r9
7442 adc r12,0
7443 vpaddd ymm0,ymm0,ymm4
7444 vpxor ymm12,ymm12,ymm0
7445 vpshufb ymm12,ymm12,YMMWORD[$L$rol16]
7446 vpaddd ymm8,ymm8,ymm12
7447 vpxor ymm4,ymm4,ymm8
7448 vpsrld ymm3,ymm4,20
7449 vpslld ymm4,ymm4,12
7450 vpxor ymm4,ymm4,ymm3
7451 vpaddd ymm0,ymm0,ymm4
7452 vpxor ymm12,ymm12,ymm0
7453 vpshufb ymm12,ymm12,YMMWORD[$L$rol8]
7454 vpaddd ymm8,ymm8,ymm12
7455 vpxor ymm4,ymm4,ymm8
7456 vpslld ymm3,ymm4,7
7457 vpsrld ymm4,ymm4,25
7458 vpxor ymm4,ymm4,ymm3
7459 vpalignr ymm12,ymm12,ymm12,4
7460 vpalignr ymm8,ymm8,ymm8,8
7461 vpalignr ymm4,ymm4,ymm4,12
7462 add r10,QWORD[((0+16))+rdi]
7463 adc r11,QWORD[((8+16))+rdi]
7464 adc r12,1
7465 mov rdx,QWORD[((0+160+0))+rbp]
7466 mov r15,rdx
7467 mulx r14,r13,r10
7468 mulx rdx,rax,r11
7469 imul r15,r12
7470 add r14,rax
7471 adc r15,rdx
7472 mov rdx,QWORD[((8+160+0))+rbp]
7473 mulx rax,r10,r10
7474 add r14,r10
7475 mulx r9,r11,r11
7476 adc r15,r11
7477 adc r9,0
7478 imul rdx,r12
7479 add r15,rax
7480 adc r9,rdx
7481 mov r10,r13
7482 mov r11,r14
7483 mov r12,r15
7484 and r12,3
7485 mov r13,r15
7486 and r13,-4
7487 mov r14,r9
7488 shrd r15,r9,2
7489 shr r9,2
7490 add r15,r13
7491 adc r9,r14
7492 add r10,r15
7493 adc r11,r9
7494 adc r12,0
7495
7496 lea rdi,[32+rdi]
7497 dec rcx
7498 jg NEAR $L$seal_avx2_tail_128_rounds_and_3xhash
7499 dec r8
7500 jge NEAR $L$seal_avx2_tail_128_rounds_and_2xhash
7501 vpaddd ymm0,ymm0,YMMWORD[$L$chacha20_consts]
7502 vpaddd ymm4,ymm4,YMMWORD[((160+64))+rbp]
7503 vpaddd ymm8,ymm8,YMMWORD[((160+96))+rbp]
7504 vpaddd ymm12,ymm12,YMMWORD[((160+160))+rbp]
7505 vperm2i128 ymm3,ymm4,ymm0,0x13
7506 vperm2i128 ymm0,ymm4,ymm0,0x02
7507 vperm2i128 ymm4,ymm12,ymm8,0x02
7508 vperm2i128 ymm12,ymm12,ymm8,0x13
7509 vmovdqa ymm8,ymm3
7510
7511 jmp NEAR $L$seal_avx2_short_loop
7512
7513 $L$seal_avx2_tail_256:
7514 vmovdqa ymm0,YMMWORD[$L$chacha20_consts]
7515 vmovdqa ymm4,YMMWORD[((160+64))+rbp]
7516 vmovdqa ymm8,YMMWORD[((160+96))+rbp]
7517 vmovdqa ymm1,ymm0
7518 vmovdqa ymm5,ymm4
7519 vmovdqa ymm9,ymm8
7520 vmovdqa ymm12,YMMWORD[$L$avx2_inc]
7521 vpaddd ymm13,ymm12,YMMWORD[((160+160))+rbp]
7522 vpaddd ymm12,ymm12,ymm13
7523 vmovdqa YMMWORD[(160+160)+rbp],ymm12
7524 vmovdqa YMMWORD[(160+192)+rbp],ymm13
7525
7526 $L$seal_avx2_tail_256_rounds_and_3xhash:
7527 add r10,QWORD[((0+0))+rdi]
7528 adc r11,QWORD[((8+0))+rdi]
7529 adc r12,1
7530 mov rax,QWORD[((0+160+0))+rbp]
7531 mov r15,rax
7532 mul r10
7533 mov r13,rax
7534 mov r14,rdx
7535 mov rax,QWORD[((0+160+0))+rbp]
7536 mul r11
7537 imul r15,r12
7538 add r14,rax
7539 adc r15,rdx
7540 mov rax,QWORD[((8+160+0))+rbp]
7541 mov r9,rax
7542 mul r10
7543 add r14,rax
7544 adc rdx,0
7545 mov r10,rdx
7546 mov rax,QWORD[((8+160+0))+rbp]
7547 mul r11
7548 add r15,rax
7549 adc rdx,0
7550 imul r9,r12
7551 add r15,r10
7552 adc r9,rdx
7553 mov r10,r13
7554 mov r11,r14
7555 mov r12,r15
7556 and r12,3
7557 mov r13,r15
7558 and r13,-4
7559 mov r14,r9
7560 shrd r15,r9,2
7561 shr r9,2
7562 add r15,r13
7563 adc r9,r14
7564 add r10,r15
7565 adc r11,r9
7566 adc r12,0
7567
7568 lea rdi,[16+rdi]
7569 $L$seal_avx2_tail_256_rounds_and_2xhash:
7570 vpaddd ymm0,ymm0,ymm4
7571 vpxor ymm12,ymm12,ymm0
7572 vpshufb ymm12,ymm12,YMMWORD[$L$rol16]
7573 vpaddd ymm8,ymm8,ymm12
7574 vpxor ymm4,ymm4,ymm8
7575 vpsrld ymm3,ymm4,20
7576 vpslld ymm4,ymm4,12
7577 vpxor ymm4,ymm4,ymm3
7578 vpaddd ymm0,ymm0,ymm4
7579 vpxor ymm12,ymm12,ymm0
7580 vpshufb ymm12,ymm12,YMMWORD[$L$rol8]
7581 vpaddd ymm8,ymm8,ymm12
7582 vpxor ymm4,ymm4,ymm8
7583 vpslld ymm3,ymm4,7
7584 vpsrld ymm4,ymm4,25
7585 vpxor ymm4,ymm4,ymm3
7586 vpalignr ymm12,ymm12,ymm12,12
7587 vpalignr ymm8,ymm8,ymm8,8
7588 vpalignr ymm4,ymm4,ymm4,4
7589 vpaddd ymm1,ymm1,ymm5
7590 vpxor ymm13,ymm13,ymm1
7591 vpshufb ymm13,ymm13,YMMWORD[$L$rol16]
7592 vpaddd ymm9,ymm9,ymm13
7593 vpxor ymm5,ymm5,ymm9
7594 vpsrld ymm3,ymm5,20
7595 vpslld ymm5,ymm5,12
7596 vpxor ymm5,ymm5,ymm3
7597 vpaddd ymm1,ymm1,ymm5
7598 vpxor ymm13,ymm13,ymm1
7599 vpshufb ymm13,ymm13,YMMWORD[$L$rol8]
7600 vpaddd ymm9,ymm9,ymm13
7601 vpxor ymm5,ymm5,ymm9
7602 vpslld ymm3,ymm5,7
7603 vpsrld ymm5,ymm5,25
7604 vpxor ymm5,ymm5,ymm3
7605 vpalignr ymm13,ymm13,ymm13,12
7606 vpalignr ymm9,ymm9,ymm9,8
7607 vpalignr ymm5,ymm5,ymm5,4
7608 add r10,QWORD[((0+0))+rdi]
7609 adc r11,QWORD[((8+0))+rdi]
7610 adc r12,1
7611 mov rax,QWORD[((0+160+0))+rbp]
7612 mov r15,rax
7613 mul r10
7614 mov r13,rax
7615 mov r14,rdx
7616 mov rax,QWORD[((0+160+0))+rbp]
7617 mul r11
7618 imul r15,r12
7619 add r14,rax
7620 adc r15,rdx
7621 mov rax,QWORD[((8+160+0))+rbp]
7622 mov r9,rax
7623 mul r10
7624 add r14,rax
7625 adc rdx,0
7626 mov r10,rdx
7627 mov rax,QWORD[((8+160+0))+rbp]
7628 mul r11
7629 add r15,rax
7630 adc rdx,0
7631 imul r9,r12
7632 add r15,r10
7633 adc r9,rdx
7634 mov r10,r13
7635 mov r11,r14
7636 mov r12,r15
7637 and r12,3
7638 mov r13,r15
7639 and r13,-4
7640 mov r14,r9
7641 shrd r15,r9,2
7642 shr r9,2
7643 add r15,r13
7644 adc r9,r14
7645 add r10,r15
7646 adc r11,r9
7647 adc r12,0
7648 vpaddd ymm0,ymm0,ymm4
7649 vpxor ymm12,ymm12,ymm0
7650 vpshufb ymm12,ymm12,YMMWORD[$L$rol16]
7651 vpaddd ymm8,ymm8,ymm12
7652 vpxor ymm4,ymm4,ymm8
7653 vpsrld ymm3,ymm4,20
7654 vpslld ymm4,ymm4,12
7655 vpxor ymm4,ymm4,ymm3
7656 vpaddd ymm0,ymm0,ymm4
7657 vpxor ymm12,ymm12,ymm0
7658 vpshufb ymm12,ymm12,YMMWORD[$L$rol8]
7659 vpaddd ymm8,ymm8,ymm12
7660 vpxor ymm4,ymm4,ymm8
7661 vpslld ymm3,ymm4,7
7662 vpsrld ymm4,ymm4,25
7663 vpxor ymm4,ymm4,ymm3
7664 vpalignr ymm12,ymm12,ymm12,4
7665 vpalignr ymm8,ymm8,ymm8,8
7666 vpalignr ymm4,ymm4,ymm4,12
7667 vpaddd ymm1,ymm1,ymm5
7668 vpxor ymm13,ymm13,ymm1
7669 vpshufb ymm13,ymm13,YMMWORD[$L$rol16]
7670 vpaddd ymm9,ymm9,ymm13
7671 vpxor ymm5,ymm5,ymm9
7672 vpsrld ymm3,ymm5,20
7673 vpslld ymm5,ymm5,12
7674 vpxor ymm5,ymm5,ymm3
7675 vpaddd ymm1,ymm1,ymm5
7676 vpxor ymm13,ymm13,ymm1
7677 vpshufb ymm13,ymm13,YMMWORD[$L$rol8]
7678 vpaddd ymm9,ymm9,ymm13
7679 vpxor ymm5,ymm5,ymm9
7680 vpslld ymm3,ymm5,7
7681 vpsrld ymm5,ymm5,25
7682 vpxor ymm5,ymm5,ymm3
7683 vpalignr ymm13,ymm13,ymm13,4
7684 vpalignr ymm9,ymm9,ymm9,8
7685 vpalignr ymm5,ymm5,ymm5,12
7686 add r10,QWORD[((0+16))+rdi]
7687 adc r11,QWORD[((8+16))+rdi]
7688 adc r12,1
7689 mov rax,QWORD[((0+160+0))+rbp]
7690 mov r15,rax
7691 mul r10
7692 mov r13,rax
7693 mov r14,rdx
7694 mov rax,QWORD[((0+160+0))+rbp]
7695 mul r11
7696 imul r15,r12
7697 add r14,rax
7698 adc r15,rdx
7699 mov rax,QWORD[((8+160+0))+rbp]
7700 mov r9,rax
7701 mul r10
7702 add r14,rax
7703 adc rdx,0
7704 mov r10,rdx
7705 mov rax,QWORD[((8+160+0))+rbp]
7706 mul r11
7707 add r15,rax
7708 adc rdx,0
7709 imul r9,r12
7710 add r15,r10
7711 adc r9,rdx
7712 mov r10,r13
7713 mov r11,r14
7714 mov r12,r15
7715 and r12,3
7716 mov r13,r15
7717 and r13,-4
7718 mov r14,r9
7719 shrd r15,r9,2
7720 shr r9,2
7721 add r15,r13
7722 adc r9,r14
7723 add r10,r15
7724 adc r11,r9
7725 adc r12,0
7726
7727 lea rdi,[32+rdi]
7728 dec rcx
7729 jg NEAR $L$seal_avx2_tail_256_rounds_and_3xhash
7730 dec r8
7731 jge NEAR $L$seal_avx2_tail_256_rounds_and_2xhash
7732 vpaddd ymm1,ymm1,YMMWORD[$L$chacha20_consts]
7733 vpaddd ymm5,ymm5,YMMWORD[((160+64))+rbp]
7734 vpaddd ymm9,ymm9,YMMWORD[((160+96))+rbp]
7735 vpaddd ymm13,ymm13,YMMWORD[((160+192))+rbp]
7736 vpaddd ymm0,ymm0,YMMWORD[$L$chacha20_consts]
7737 vpaddd ymm4,ymm4,YMMWORD[((160+64))+rbp]
7738 vpaddd ymm8,ymm8,YMMWORD[((160+96))+rbp]
7739 vpaddd ymm12,ymm12,YMMWORD[((160+160))+rbp]
7740 vperm2i128 ymm3,ymm5,ymm1,0x02
7741 vperm2i128 ymm5,ymm5,ymm1,0x13
7742 vperm2i128 ymm1,ymm13,ymm9,0x02
7743 vperm2i128 ymm9,ymm13,ymm9,0x13
7744 vpxor ymm3,ymm3,YMMWORD[((0+0))+rsi]
7745 vpxor ymm1,ymm1,YMMWORD[((32+0))+rsi]
7746 vpxor ymm5,ymm5,YMMWORD[((64+0))+rsi]
7747 vpxor ymm9,ymm9,YMMWORD[((96+0))+rsi]
7748 vmovdqu YMMWORD[(0+0)+rdi],ymm3
7749 vmovdqu YMMWORD[(32+0)+rdi],ymm1
7750 vmovdqu YMMWORD[(64+0)+rdi],ymm5
7751 vmovdqu YMMWORD[(96+0)+rdi],ymm9
7752 vperm2i128 ymm3,ymm4,ymm0,0x13
7753 vperm2i128 ymm0,ymm4,ymm0,0x02
7754 vperm2i128 ymm4,ymm12,ymm8,0x02
7755 vperm2i128 ymm12,ymm12,ymm8,0x13
7756 vmovdqa ymm8,ymm3
7757
7758 mov rcx,4*32
7759 lea rsi,[128+rsi]
7760 sub rbx,4*32
7761 jmp NEAR $L$seal_avx2_short_hash_remainder
7762
7763 $L$seal_avx2_tail_384:
7764 vmovdqa ymm0,YMMWORD[$L$chacha20_consts]
7765 vmovdqa ymm4,YMMWORD[((160+64))+rbp]
7766 vmovdqa ymm8,YMMWORD[((160+96))+rbp]
7767 vmovdqa ymm1,ymm0
7768 vmovdqa ymm5,ymm4
7769 vmovdqa ymm9,ymm8
7770 vmovdqa ymm2,ymm0
7771 vmovdqa ymm6,ymm4
7772 vmovdqa ymm10,ymm8
7773 vmovdqa ymm12,YMMWORD[$L$avx2_inc]
7774 vpaddd ymm14,ymm12,YMMWORD[((160+160))+rbp]
7775 vpaddd ymm13,ymm12,ymm14
7776 vpaddd ymm12,ymm12,ymm13
7777 vmovdqa YMMWORD[(160+160)+rbp],ymm12
7778 vmovdqa YMMWORD[(160+192)+rbp],ymm13
7779 vmovdqa YMMWORD[(160+224)+rbp],ymm14
7780
7781 $L$seal_avx2_tail_384_rounds_and_3xhash:
7782 add r10,QWORD[((0+0))+rdi]
7783 adc r11,QWORD[((8+0))+rdi]
7784 adc r12,1
7785 mov rax,QWORD[((0+160+0))+rbp]
7786 mov r15,rax
7787 mul r10
7788 mov r13,rax
7789 mov r14,rdx
7790 mov rax,QWORD[((0+160+0))+rbp]
7791 mul r11
7792 imul r15,r12
7793 add r14,rax
7794 adc r15,rdx
7795 mov rax,QWORD[((8+160+0))+rbp]
7796 mov r9,rax
7797 mul r10
7798 add r14,rax
7799 adc rdx,0
7800 mov r10,rdx
7801 mov rax,QWORD[((8+160+0))+rbp]
7802 mul r11
7803 add r15,rax
7804 adc rdx,0
7805 imul r9,r12
7806 add r15,r10
7807 adc r9,rdx
7808 mov r10,r13
7809 mov r11,r14
7810 mov r12,r15
7811 and r12,3
7812 mov r13,r15
7813 and r13,-4
7814 mov r14,r9
7815 shrd r15,r9,2
7816 shr r9,2
7817 add r15,r13
7818 adc r9,r14
7819 add r10,r15
7820 adc r11,r9
7821 adc r12,0
7822
7823 lea rdi,[16+rdi]
7824 $L$seal_avx2_tail_384_rounds_and_2xhash:
7825 vpaddd ymm0,ymm0,ymm4
7826 vpxor ymm12,ymm12,ymm0
7827 vpshufb ymm12,ymm12,YMMWORD[$L$rol16]
7828 vpaddd ymm8,ymm8,ymm12
7829 vpxor ymm4,ymm4,ymm8
7830 vpsrld ymm3,ymm4,20
7831 vpslld ymm4,ymm4,12
7832 vpxor ymm4,ymm4,ymm3
7833 vpaddd ymm0,ymm0,ymm4
7834 vpxor ymm12,ymm12,ymm0
7835 vpshufb ymm12,ymm12,YMMWORD[$L$rol8]
7836 vpaddd ymm8,ymm8,ymm12
7837 vpxor ymm4,ymm4,ymm8
7838 vpslld ymm3,ymm4,7
7839 vpsrld ymm4,ymm4,25
7840 vpxor ymm4,ymm4,ymm3
7841 vpalignr ymm12,ymm12,ymm12,12
7842 vpalignr ymm8,ymm8,ymm8,8
7843 vpalignr ymm4,ymm4,ymm4,4
7844 vpaddd ymm1,ymm1,ymm5
7845 vpxor ymm13,ymm13,ymm1
7846 vpshufb ymm13,ymm13,YMMWORD[$L$rol16]
7847 vpaddd ymm9,ymm9,ymm13
7848 vpxor ymm5,ymm5,ymm9
7849 vpsrld ymm3,ymm5,20
7850 vpslld ymm5,ymm5,12
7851 vpxor ymm5,ymm5,ymm3
7852 vpaddd ymm1,ymm1,ymm5
7853 vpxor ymm13,ymm13,ymm1
7854 vpshufb ymm13,ymm13,YMMWORD[$L$rol8]
7855 vpaddd ymm9,ymm9,ymm13
7856 vpxor ymm5,ymm5,ymm9
7857 vpslld ymm3,ymm5,7
7858 vpsrld ymm5,ymm5,25
7859 vpxor ymm5,ymm5,ymm3
7860 vpalignr ymm13,ymm13,ymm13,12
7861 vpalignr ymm9,ymm9,ymm9,8
7862 vpalignr ymm5,ymm5,ymm5,4
7863 add r10,QWORD[((0+0))+rdi]
7864 adc r11,QWORD[((8+0))+rdi]
7865 adc r12,1
7866 mov rax,QWORD[((0+160+0))+rbp]
7867 mov r15,rax
7868 mul r10
7869 mov r13,rax
7870 mov r14,rdx
7871 mov rax,QWORD[((0+160+0))+rbp]
7872 mul r11
7873 imul r15,r12
7874 add r14,rax
7875 adc r15,rdx
7876 mov rax,QWORD[((8+160+0))+rbp]
7877 mov r9,rax
7878 mul r10
7879 add r14,rax
7880 adc rdx,0
7881 mov r10,rdx
7882 mov rax,QWORD[((8+160+0))+rbp]
7883 mul r11
7884 add r15,rax
7885 adc rdx,0
7886 imul r9,r12
7887 add r15,r10
7888 adc r9,rdx
7889 mov r10,r13
7890 mov r11,r14
7891 mov r12,r15
7892 and r12,3
7893 mov r13,r15
7894 and r13,-4
7895 mov r14,r9
7896 shrd r15,r9,2
7897 shr r9,2
7898 add r15,r13
7899 adc r9,r14
7900 add r10,r15
7901 adc r11,r9
7902 adc r12,0
7903 vpaddd ymm2,ymm2,ymm6
7904 vpxor ymm14,ymm14,ymm2
7905 vpshufb ymm14,ymm14,YMMWORD[$L$rol16]
7906 vpaddd ymm10,ymm10,ymm14
7907 vpxor ymm6,ymm6,ymm10
7908 vpsrld ymm3,ymm6,20
7909 vpslld ymm6,ymm6,12
7910 vpxor ymm6,ymm6,ymm3
7911 vpaddd ymm2,ymm2,ymm6
7912 vpxor ymm14,ymm14,ymm2
7913 vpshufb ymm14,ymm14,YMMWORD[$L$rol8]
7914 vpaddd ymm10,ymm10,ymm14
7915 vpxor ymm6,ymm6,ymm10
7916 vpslld ymm3,ymm6,7
7917 vpsrld ymm6,ymm6,25
7918 vpxor ymm6,ymm6,ymm3
7919 vpalignr ymm14,ymm14,ymm14,12
7920 vpalignr ymm10,ymm10,ymm10,8
7921 vpalignr ymm6,ymm6,ymm6,4
7922 vpaddd ymm0,ymm0,ymm4
7923 vpxor ymm12,ymm12,ymm0
7924 vpshufb ymm12,ymm12,YMMWORD[$L$rol16]
7925 vpaddd ymm8,ymm8,ymm12
7926 vpxor ymm4,ymm4,ymm8
7927 vpsrld ymm3,ymm4,20
7928 vpslld ymm4,ymm4,12
7929 vpxor ymm4,ymm4,ymm3
7930 vpaddd ymm0,ymm0,ymm4
7931 vpxor ymm12,ymm12,ymm0
7932 vpshufb ymm12,ymm12,YMMWORD[$L$rol8]
7933 vpaddd ymm8,ymm8,ymm12
7934 vpxor ymm4,ymm4,ymm8
7935 vpslld ymm3,ymm4,7
7936 vpsrld ymm4,ymm4,25
7937 vpxor ymm4,ymm4,ymm3
7938 vpalignr ymm12,ymm12,ymm12,4
7939 vpalignr ymm8,ymm8,ymm8,8
7940 vpalignr ymm4,ymm4,ymm4,12
7941 add r10,QWORD[((0+16))+rdi]
7942 adc r11,QWORD[((8+16))+rdi]
7943 adc r12,1
7944 mov rax,QWORD[((0+160+0))+rbp]
7945 mov r15,rax
7946 mul r10
7947 mov r13,rax
7948 mov r14,rdx
7949 mov rax,QWORD[((0+160+0))+rbp]
7950 mul r11
7951 imul r15,r12
7952 add r14,rax
7953 adc r15,rdx
7954 mov rax,QWORD[((8+160+0))+rbp]
7955 mov r9,rax
7956 mul r10
7957 add r14,rax
7958 adc rdx,0
7959 mov r10,rdx
7960 mov rax,QWORD[((8+160+0))+rbp]
7961 mul r11
7962 add r15,rax
7963 adc rdx,0
7964 imul r9,r12
7965 add r15,r10
7966 adc r9,rdx
7967 mov r10,r13
7968 mov r11,r14
7969 mov r12,r15
7970 and r12,3
7971 mov r13,r15
7972 and r13,-4
7973 mov r14,r9
7974 shrd r15,r9,2
7975 shr r9,2
7976 add r15,r13
7977 adc r9,r14
7978 add r10,r15
7979 adc r11,r9
7980 adc r12,0
7981 vpaddd ymm1,ymm1,ymm5
7982 vpxor ymm13,ymm13,ymm1
7983 vpshufb ymm13,ymm13,YMMWORD[$L$rol16]
7984 vpaddd ymm9,ymm9,ymm13
7985 vpxor ymm5,ymm5,ymm9
7986 vpsrld ymm3,ymm5,20
7987 vpslld ymm5,ymm5,12
7988 vpxor ymm5,ymm5,ymm3
7989 vpaddd ymm1,ymm1,ymm5
7990 vpxor ymm13,ymm13,ymm1
7991 vpshufb ymm13,ymm13,YMMWORD[$L$rol8]
7992 vpaddd ymm9,ymm9,ymm13
7993 vpxor ymm5,ymm5,ymm9
7994 vpslld ymm3,ymm5,7
7995 vpsrld ymm5,ymm5,25
7996 vpxor ymm5,ymm5,ymm3
7997 vpalignr ymm13,ymm13,ymm13,4
7998 vpalignr ymm9,ymm9,ymm9,8
7999 vpalignr ymm5,ymm5,ymm5,12
8000 vpaddd ymm2,ymm2,ymm6
8001 vpxor ymm14,ymm14,ymm2
8002 vpshufb ymm14,ymm14,YMMWORD[$L$rol16]
8003 vpaddd ymm10,ymm10,ymm14
8004 vpxor ymm6,ymm6,ymm10
8005 vpsrld ymm3,ymm6,20
8006 vpslld ymm6,ymm6,12
8007 vpxor ymm6,ymm6,ymm3
8008 vpaddd ymm2,ymm2,ymm6
8009 vpxor ymm14,ymm14,ymm2
8010 vpshufb ymm14,ymm14,YMMWORD[$L$rol8]
8011 vpaddd ymm10,ymm10,ymm14
8012 vpxor ymm6,ymm6,ymm10
8013 vpslld ymm3,ymm6,7
8014 vpsrld ymm6,ymm6,25
8015 vpxor ymm6,ymm6,ymm3
8016 vpalignr ymm14,ymm14,ymm14,4
8017 vpalignr ymm10,ymm10,ymm10,8
8018 vpalignr ymm6,ymm6,ymm6,12
8019
8020 lea rdi,[32+rdi]
8021 dec rcx
8022 jg NEAR $L$seal_avx2_tail_384_rounds_and_3xhash
8023 dec r8
8024 jge NEAR $L$seal_avx2_tail_384_rounds_and_2xhash
8025 vpaddd ymm2,ymm2,YMMWORD[$L$chacha20_consts]
8026 vpaddd ymm6,ymm6,YMMWORD[((160+64))+rbp]
8027 vpaddd ymm10,ymm10,YMMWORD[((160+96))+rbp]
8028 vpaddd ymm14,ymm14,YMMWORD[((160+224))+rbp]
8029 vpaddd ymm1,ymm1,YMMWORD[$L$chacha20_consts]
8030 vpaddd ymm5,ymm5,YMMWORD[((160+64))+rbp]
8031 vpaddd ymm9,ymm9,YMMWORD[((160+96))+rbp]
8032 vpaddd ymm13,ymm13,YMMWORD[((160+192))+rbp]
8033 vpaddd ymm0,ymm0,YMMWORD[$L$chacha20_consts]
8034 vpaddd ymm4,ymm4,YMMWORD[((160+64))+rbp]
8035 vpaddd ymm8,ymm8,YMMWORD[((160+96))+rbp]
8036 vpaddd ymm12,ymm12,YMMWORD[((160+160))+rbp]
8037 vperm2i128 ymm3,ymm6,ymm2,0x02
8038 vperm2i128 ymm6,ymm6,ymm2,0x13
8039 vperm2i128 ymm2,ymm14,ymm10,0x02
8040 vperm2i128 ymm10,ymm14,ymm10,0x13
8041 vpxor ymm3,ymm3,YMMWORD[((0+0))+rsi]
8042 vpxor ymm2,ymm2,YMMWORD[((32+0))+rsi]
8043 vpxor ymm6,ymm6,YMMWORD[((64+0))+rsi]
8044 vpxor ymm10,ymm10,YMMWORD[((96+0))+rsi]
8045 vmovdqu YMMWORD[(0+0)+rdi],ymm3
8046 vmovdqu YMMWORD[(32+0)+rdi],ymm2
8047 vmovdqu YMMWORD[(64+0)+rdi],ymm6
8048 vmovdqu YMMWORD[(96+0)+rdi],ymm10
8049 vperm2i128 ymm3,ymm5,ymm1,0x02
8050 vperm2i128 ymm5,ymm5,ymm1,0x13
8051 vperm2i128 ymm1,ymm13,ymm9,0x02
8052 vperm2i128 ymm9,ymm13,ymm9,0x13
8053 vpxor ymm3,ymm3,YMMWORD[((0+128))+rsi]
8054 vpxor ymm1,ymm1,YMMWORD[((32+128))+rsi]
8055 vpxor ymm5,ymm5,YMMWORD[((64+128))+rsi]
8056 vpxor ymm9,ymm9,YMMWORD[((96+128))+rsi]
8057 vmovdqu YMMWORD[(0+128)+rdi],ymm3
8058 vmovdqu YMMWORD[(32+128)+rdi],ymm1
8059 vmovdqu YMMWORD[(64+128)+rdi],ymm5
8060 vmovdqu YMMWORD[(96+128)+rdi],ymm9
8061 vperm2i128 ymm3,ymm4,ymm0,0x13
8062 vperm2i128 ymm0,ymm4,ymm0,0x02
8063 vperm2i128 ymm4,ymm12,ymm8,0x02
8064 vperm2i128 ymm12,ymm12,ymm8,0x13
8065 vmovdqa ymm8,ymm3
8066
8067 mov rcx,8*32
8068 lea rsi,[256+rsi]
8069 sub rbx,8*32
8070 jmp NEAR $L$seal_avx2_short_hash_remainder
8071
8072 $L$seal_avx2_tail_512:
8073 vmovdqa ymm0,YMMWORD[$L$chacha20_consts]
8074 vmovdqa ymm4,YMMWORD[((160+64))+rbp]
8075 vmovdqa ymm8,YMMWORD[((160+96))+rbp]
8076 vmovdqa ymm1,ymm0
8077 vmovdqa ymm5,ymm4
8078 vmovdqa ymm9,ymm8
8079 vmovdqa ymm2,ymm0
8080 vmovdqa ymm6,ymm4
8081 vmovdqa ymm10,ymm8
8082 vmovdqa ymm3,ymm0
8083 vmovdqa ymm7,ymm4
8084 vmovdqa ymm11,ymm8
8085 vmovdqa ymm12,YMMWORD[$L$avx2_inc]
8086 vpaddd ymm15,ymm12,YMMWORD[((160+160))+rbp]
8087 vpaddd ymm14,ymm12,ymm15
8088 vpaddd ymm13,ymm12,ymm14
8089 vpaddd ymm12,ymm12,ymm13
8090 vmovdqa YMMWORD[(160+256)+rbp],ymm15
8091 vmovdqa YMMWORD[(160+224)+rbp],ymm14
8092 vmovdqa YMMWORD[(160+192)+rbp],ymm13
8093 vmovdqa YMMWORD[(160+160)+rbp],ymm12
8094
8095 $L$seal_avx2_tail_512_rounds_and_3xhash:
8096 add r10,QWORD[((0+0))+rdi]
8097 adc r11,QWORD[((8+0))+rdi]
8098 adc r12,1
8099 mov rdx,QWORD[((0+160+0))+rbp]
8100 mov r15,rdx
8101 mulx r14,r13,r10
8102 mulx rdx,rax,r11
8103 imul r15,r12
8104 add r14,rax
8105 adc r15,rdx
8106 mov rdx,QWORD[((8+160+0))+rbp]
8107 mulx rax,r10,r10
8108 add r14,r10
8109 mulx r9,r11,r11
8110 adc r15,r11
8111 adc r9,0
8112 imul rdx,r12
8113 add r15,rax
8114 adc r9,rdx
8115 mov r10,r13
8116 mov r11,r14
8117 mov r12,r15
8118 and r12,3
8119 mov r13,r15
8120 and r13,-4
8121 mov r14,r9
8122 shrd r15,r9,2
8123 shr r9,2
8124 add r15,r13
8125 adc r9,r14
8126 add r10,r15
8127 adc r11,r9
8128 adc r12,0
8129
8130 lea rdi,[16+rdi]
8131 $L$seal_avx2_tail_512_rounds_and_2xhash:
8132 vmovdqa YMMWORD[(160+128)+rbp],ymm8
8133 vmovdqa ymm8,YMMWORD[$L$rol16]
8134 vpaddd ymm3,ymm3,ymm7
8135 vpaddd ymm2,ymm2,ymm6
8136 vpaddd ymm1,ymm1,ymm5
8137 vpaddd ymm0,ymm0,ymm4
8138 vpxor ymm15,ymm15,ymm3
8139 vpxor ymm14,ymm14,ymm2
8140 vpxor ymm13,ymm13,ymm1
8141 vpxor ymm12,ymm12,ymm0
8142 vpshufb ymm15,ymm15,ymm8
8143 vpshufb ymm14,ymm14,ymm8
8144 vpshufb ymm13,ymm13,ymm8
8145 vpshufb ymm12,ymm12,ymm8
8146 vpaddd ymm11,ymm11,ymm15
8147 vpaddd ymm10,ymm10,ymm14
8148 vpaddd ymm9,ymm9,ymm13
8149 vpaddd ymm8,ymm12,YMMWORD[((160+128))+rbp]
8150 vpxor ymm7,ymm7,ymm11
8151 vpxor ymm6,ymm6,ymm10
8152 add r10,QWORD[((0+0))+rdi]
8153 adc r11,QWORD[((8+0))+rdi]
8154 adc r12,1
8155 vpxor ymm5,ymm5,ymm9
8156 vpxor ymm4,ymm4,ymm8
8157 vmovdqa YMMWORD[(160+128)+rbp],ymm8
8158 vpsrld ymm8,ymm7,20
8159 vpslld ymm7,ymm7,32-20
8160 vpxor ymm7,ymm7,ymm8
8161 vpsrld ymm8,ymm6,20
8162 vpslld ymm6,ymm6,32-20
8163 vpxor ymm6,ymm6,ymm8
8164 vpsrld ymm8,ymm5,20
8165 vpslld ymm5,ymm5,32-20
8166 vpxor ymm5,ymm5,ymm8
8167 vpsrld ymm8,ymm4,20
8168 vpslld ymm4,ymm4,32-20
8169 vpxor ymm4,ymm4,ymm8
8170 vmovdqa ymm8,YMMWORD[$L$rol8]
8171 vpaddd ymm3,ymm3,ymm7
8172 vpaddd ymm2,ymm2,ymm6
8173 vpaddd ymm1,ymm1,ymm5
8174 vpaddd ymm0,ymm0,ymm4
8175 mov rdx,QWORD[((0+160+0))+rbp]
8176 mov r15,rdx
8177 mulx r14,r13,r10
8178 mulx rdx,rax,r11
8179 imul r15,r12
8180 add r14,rax
8181 adc r15,rdx
8182 vpxor ymm15,ymm15,ymm3
8183 vpxor ymm14,ymm14,ymm2
8184 vpxor ymm13,ymm13,ymm1
8185 vpxor ymm12,ymm12,ymm0
8186 vpshufb ymm15,ymm15,ymm8
8187 vpshufb ymm14,ymm14,ymm8
8188 vpshufb ymm13,ymm13,ymm8
8189 vpshufb ymm12,ymm12,ymm8
8190 vpaddd ymm11,ymm11,ymm15
8191 vpaddd ymm10,ymm10,ymm14
8192 vpaddd ymm9,ymm9,ymm13
8193 vpaddd ymm8,ymm12,YMMWORD[((160+128))+rbp]
8194 vpxor ymm7,ymm7,ymm11
8195 vpxor ymm6,ymm6,ymm10
8196 vpxor ymm5,ymm5,ymm9
8197 vpxor ymm4,ymm4,ymm8
8198 vmovdqa YMMWORD[(160+128)+rbp],ymm8
8199 vpsrld ymm8,ymm7,25
8200 vpslld ymm7,ymm7,32-25
8201 vpxor ymm7,ymm7,ymm8
8202 mov rdx,QWORD[((8+160+0))+rbp]
8203 mulx rax,r10,r10
8204 add r14,r10
8205 mulx r9,r11,r11
8206 adc r15,r11
8207 adc r9,0
8208 imul rdx,r12
8209 vpsrld ymm8,ymm6,25
8210 vpslld ymm6,ymm6,32-25
8211 vpxor ymm6,ymm6,ymm8
8212 vpsrld ymm8,ymm5,25
8213 vpslld ymm5,ymm5,32-25
8214 vpxor ymm5,ymm5,ymm8
8215 vpsrld ymm8,ymm4,25
8216 vpslld ymm4,ymm4,32-25
8217 vpxor ymm4,ymm4,ymm8
8218 vmovdqa ymm8,YMMWORD[((160+128))+rbp]
8219 vpalignr ymm7,ymm7,ymm7,4
8220 vpalignr ymm11,ymm11,ymm11,8
8221 vpalignr ymm15,ymm15,ymm15,12
8222 vpalignr ymm6,ymm6,ymm6,4
8223 vpalignr ymm10,ymm10,ymm10,8
8224 vpalignr ymm14,ymm14,ymm14,12
8225 vpalignr ymm5,ymm5,ymm5,4
8226 vpalignr ymm9,ymm9,ymm9,8
8227 vpalignr ymm13,ymm13,ymm13,12
8228 vpalignr ymm4,ymm4,ymm4,4
8229 add r15,rax
8230 adc r9,rdx
8231 vpalignr ymm8,ymm8,ymm8,8
8232 vpalignr ymm12,ymm12,ymm12,12
8233 vmovdqa YMMWORD[(160+128)+rbp],ymm8
8234 vmovdqa ymm8,YMMWORD[$L$rol16]
8235 vpaddd ymm3,ymm3,ymm7
8236 vpaddd ymm2,ymm2,ymm6
8237 vpaddd ymm1,ymm1,ymm5
8238 vpaddd ymm0,ymm0,ymm4
8239 vpxor ymm15,ymm15,ymm3
8240 vpxor ymm14,ymm14,ymm2
8241 vpxor ymm13,ymm13,ymm1
8242 vpxor ymm12,ymm12,ymm0
8243 vpshufb ymm15,ymm15,ymm8
8244 vpshufb ymm14,ymm14,ymm8
8245 vpshufb ymm13,ymm13,ymm8
8246 vpshufb ymm12,ymm12,ymm8
8247 vpaddd ymm11,ymm11,ymm15
8248 vpaddd ymm10,ymm10,ymm14
8249 vpaddd ymm9,ymm9,ymm13
8250 vpaddd ymm8,ymm12,YMMWORD[((160+128))+rbp]
8251 mov r10,r13
8252 mov r11,r14
8253 mov r12,r15
8254 and r12,3
8255 mov r13,r15
8256 and r13,-4
8257 mov r14,r9
8258 shrd r15,r9,2
8259 shr r9,2
8260 add r15,r13
8261 adc r9,r14
8262 add r10,r15
8263 adc r11,r9
8264 adc r12,0
8265 vpxor ymm7,ymm7,ymm11
8266 vpxor ymm6,ymm6,ymm10
8267 vpxor ymm5,ymm5,ymm9
8268 vpxor ymm4,ymm4,ymm8
8269 vmovdqa YMMWORD[(160+128)+rbp],ymm8
8270 vpsrld ymm8,ymm7,20
8271 vpslld ymm7,ymm7,32-20
8272 vpxor ymm7,ymm7,ymm8
8273 vpsrld ymm8,ymm6,20
8274 vpslld ymm6,ymm6,32-20
8275 vpxor ymm6,ymm6,ymm8
8276 vpsrld ymm8,ymm5,20
8277 vpslld ymm5,ymm5,32-20
8278 vpxor ymm5,ymm5,ymm8
8279 vpsrld ymm8,ymm4,20
8280 vpslld ymm4,ymm4,32-20
8281 vpxor ymm4,ymm4,ymm8
8282 vmovdqa ymm8,YMMWORD[$L$rol8]
8283 vpaddd ymm3,ymm3,ymm7
8284 vpaddd ymm2,ymm2,ymm6
8285 add r10,QWORD[((0+16))+rdi]
8286 adc r11,QWORD[((8+16))+rdi]
8287 adc r12,1
8288 vpaddd ymm1,ymm1,ymm5
8289 vpaddd ymm0,ymm0,ymm4
8290 vpxor ymm15,ymm15,ymm3
8291 vpxor ymm14,ymm14,ymm2
8292 vpxor ymm13,ymm13,ymm1
8293 vpxor ymm12,ymm12,ymm0
8294 vpshufb ymm15,ymm15,ymm8
8295 vpshufb ymm14,ymm14,ymm8
8296 vpshufb ymm13,ymm13,ymm8
8297 vpshufb ymm12,ymm12,ymm8
8298 vpaddd ymm11,ymm11,ymm15
8299 vpaddd ymm10,ymm10,ymm14
8300 vpaddd ymm9,ymm9,ymm13
8301 vpaddd ymm8,ymm12,YMMWORD[((160+128))+rbp]
8302 vpxor ymm7,ymm7,ymm11
8303 vpxor ymm6,ymm6,ymm10
8304 vpxor ymm5,ymm5,ymm9
8305 vpxor ymm4,ymm4,ymm8
8306 vmovdqa YMMWORD[(160+128)+rbp],ymm8
8307 vpsrld ymm8,ymm7,25
8308 mov rdx,QWORD[((0+160+0))+rbp]
8309 mov r15,rdx
8310 mulx r14,r13,r10
8311 mulx rdx,rax,r11
8312 imul r15,r12
8313 add r14,rax
8314 adc r15,rdx
8315 vpslld ymm7,ymm7,32-25
8316 vpxor ymm7,ymm7,ymm8
8317 vpsrld ymm8,ymm6,25
8318 vpslld ymm6,ymm6,32-25
8319 vpxor ymm6,ymm6,ymm8
8320 vpsrld ymm8,ymm5,25
8321 vpslld ymm5,ymm5,32-25
8322 vpxor ymm5,ymm5,ymm8
8323 vpsrld ymm8,ymm4,25
8324 vpslld ymm4,ymm4,32-25
8325 vpxor ymm4,ymm4,ymm8
8326 vmovdqa ymm8,YMMWORD[((160+128))+rbp]
8327 vpalignr ymm7,ymm7,ymm7,12
8328 vpalignr ymm11,ymm11,ymm11,8
8329 vpalignr ymm15,ymm15,ymm15,4
8330 vpalignr ymm6,ymm6,ymm6,12
8331 vpalignr ymm10,ymm10,ymm10,8
8332 vpalignr ymm14,ymm14,ymm14,4
8333 vpalignr ymm5,ymm5,ymm5,12
8334 vpalignr ymm9,ymm9,ymm9,8
8335 mov rdx,QWORD[((8+160+0))+rbp]
8336 mulx rax,r10,r10
8337 add r14,r10
8338 mulx r9,r11,r11
8339 adc r15,r11
8340 adc r9,0
8341 imul rdx,r12
8342 vpalignr ymm13,ymm13,ymm13,4
8343 vpalignr ymm4,ymm4,ymm4,12
8344 vpalignr ymm8,ymm8,ymm8,8
8345 vpalignr ymm12,ymm12,ymm12,4
8346
8347
8348
8349
8350
8351
8352
8353
8354
8355
8356
8357
8358
8359
8360
8361
8362 add r15,rax
8363 adc r9,rdx
8364
8365
8366
8367
8368
8369
8370
8371
8372
8373
8374
8375
8376
8377
8378
8379
8380
8381
8382
8383
8384 mov r10,r13
8385 mov r11,r14
8386 mov r12,r15
8387 and r12,3
8388 mov r13,r15
8389 and r13,-4
8390 mov r14,r9
8391 shrd r15,r9,2
8392 shr r9,2
8393 add r15,r13
8394 adc r9,r14
8395 add r10,r15
8396 adc r11,r9
8397 adc r12,0
8398
8399 lea rdi,[32+rdi]
8400 dec rcx
8401 jg NEAR $L$seal_avx2_tail_512_rounds_and_3xhash
8402 dec r8
8403 jge NEAR $L$seal_avx2_tail_512_rounds_and_2xhash
8404 vpaddd ymm3,ymm3,YMMWORD[$L$chacha20_consts]
8405 vpaddd ymm7,ymm7,YMMWORD[((160+64))+rbp]
8406 vpaddd ymm11,ymm11,YMMWORD[((160+96))+rbp]
8407 vpaddd ymm15,ymm15,YMMWORD[((160+256))+rbp]
8408 vpaddd ymm2,ymm2,YMMWORD[$L$chacha20_consts]
8409 vpaddd ymm6,ymm6,YMMWORD[((160+64))+rbp]
8410 vpaddd ymm10,ymm10,YMMWORD[((160+96))+rbp]
8411 vpaddd ymm14,ymm14,YMMWORD[((160+224))+rbp]
8412 vpaddd ymm1,ymm1,YMMWORD[$L$chacha20_consts]
8413 vpaddd ymm5,ymm5,YMMWORD[((160+64))+rbp]
8414 vpaddd ymm9,ymm9,YMMWORD[((160+96))+rbp]
8415 vpaddd ymm13,ymm13,YMMWORD[((160+192))+rbp]
8416 vpaddd ymm0,ymm0,YMMWORD[$L$chacha20_consts]
8417 vpaddd ymm4,ymm4,YMMWORD[((160+64))+rbp]
8418 vpaddd ymm8,ymm8,YMMWORD[((160+96))+rbp]
8419 vpaddd ymm12,ymm12,YMMWORD[((160+160))+rbp]
8420
8421 vmovdqa YMMWORD[(160+128)+rbp],ymm0
8422 vperm2i128 ymm0,ymm7,ymm3,0x02
8423 vperm2i128 ymm7,ymm7,ymm3,0x13
8424 vperm2i128 ymm3,ymm15,ymm11,0x02
8425 vperm2i128 ymm11,ymm15,ymm11,0x13
8426 vpxor ymm0,ymm0,YMMWORD[((0+0))+rsi]
8427 vpxor ymm3,ymm3,YMMWORD[((32+0))+rsi]
8428 vpxor ymm7,ymm7,YMMWORD[((64+0))+rsi]
8429 vpxor ymm11,ymm11,YMMWORD[((96+0))+rsi]
8430 vmovdqu YMMWORD[(0+0)+rdi],ymm0
8431 vmovdqu YMMWORD[(32+0)+rdi],ymm3
8432 vmovdqu YMMWORD[(64+0)+rdi],ymm7
8433 vmovdqu YMMWORD[(96+0)+rdi],ymm11
8434
8435 vmovdqa ymm0,YMMWORD[((160+128))+rbp]
8436 vperm2i128 ymm3,ymm6,ymm2,0x02
8437 vperm2i128 ymm6,ymm6,ymm2,0x13
8438 vperm2i128 ymm2,ymm14,ymm10,0x02
8439 vperm2i128 ymm10,ymm14,ymm10,0x13
8440 vpxor ymm3,ymm3,YMMWORD[((0+128))+rsi]
8441 vpxor ymm2,ymm2,YMMWORD[((32+128))+rsi]
8442 vpxor ymm6,ymm6,YMMWORD[((64+128))+rsi]
8443 vpxor ymm10,ymm10,YMMWORD[((96+128))+rsi]
8444 vmovdqu YMMWORD[(0+128)+rdi],ymm3
8445 vmovdqu YMMWORD[(32+128)+rdi],ymm2
8446 vmovdqu YMMWORD[(64+128)+rdi],ymm6
8447 vmovdqu YMMWORD[(96+128)+rdi],ymm10
8448 vperm2i128 ymm3,ymm5,ymm1,0x02
8449 vperm2i128 ymm5,ymm5,ymm1,0x13
8450 vperm2i128 ymm1,ymm13,ymm9,0x02
8451 vperm2i128 ymm9,ymm13,ymm9,0x13
8452 vpxor ymm3,ymm3,YMMWORD[((0+256))+rsi]
8453 vpxor ymm1,ymm1,YMMWORD[((32+256))+rsi]
8454 vpxor ymm5,ymm5,YMMWORD[((64+256))+rsi]
8455 vpxor ymm9,ymm9,YMMWORD[((96+256))+rsi]
8456 vmovdqu YMMWORD[(0+256)+rdi],ymm3
8457 vmovdqu YMMWORD[(32+256)+rdi],ymm1
8458 vmovdqu YMMWORD[(64+256)+rdi],ymm5
8459 vmovdqu YMMWORD[(96+256)+rdi],ymm9
8460 vperm2i128 ymm3,ymm4,ymm0,0x13
8461 vperm2i128 ymm0,ymm4,ymm0,0x02
8462 vperm2i128 ymm4,ymm12,ymm8,0x02
8463 vperm2i128 ymm12,ymm12,ymm8,0x13
8464 vmovdqa ymm8,ymm3
8465
8466 mov rcx,12*32
8467 lea rsi,[384+rsi]
8468 sub rbx,12*32
8469 jmp NEAR $L$seal_avx2_short_hash_remainder
8470
8471 $L$seal_avx2_320:
8472 vmovdqa ymm1,ymm0
8473 vmovdqa ymm2,ymm0
8474 vmovdqa ymm5,ymm4
8475 vmovdqa ymm6,ymm4
8476 vmovdqa ymm9,ymm8
8477 vmovdqa ymm10,ymm8
8478 vpaddd ymm13,ymm12,YMMWORD[$L$avx2_inc]
8479 vpaddd ymm14,ymm13,YMMWORD[$L$avx2_inc]
8480 vmovdqa ymm7,ymm4
8481 vmovdqa ymm11,ymm8
8482 vmovdqa YMMWORD[(160+160)+rbp],ymm12
8483 vmovdqa YMMWORD[(160+192)+rbp],ymm13
8484 vmovdqa YMMWORD[(160+224)+rbp],ymm14
8485 mov r10,10
8486 $L$seal_avx2_320_rounds:
8487 vpaddd ymm0,ymm0,ymm4
8488 vpxor ymm12,ymm12,ymm0
8489 vpshufb ymm12,ymm12,YMMWORD[$L$rol16]
8490 vpaddd ymm8,ymm8,ymm12
8491 vpxor ymm4,ymm4,ymm8
8492 vpsrld ymm3,ymm4,20
8493 vpslld ymm4,ymm4,12
8494 vpxor ymm4,ymm4,ymm3
8495 vpaddd ymm0,ymm0,ymm4
8496 vpxor ymm12,ymm12,ymm0
8497 vpshufb ymm12,ymm12,YMMWORD[$L$rol8]
8498 vpaddd ymm8,ymm8,ymm12
8499 vpxor ymm4,ymm4,ymm8
8500 vpslld ymm3,ymm4,7
8501 vpsrld ymm4,ymm4,25
8502 vpxor ymm4,ymm4,ymm3
8503 vpalignr ymm12,ymm12,ymm12,12
8504 vpalignr ymm8,ymm8,ymm8,8
8505 vpalignr ymm4,ymm4,ymm4,4
8506 vpaddd ymm1,ymm1,ymm5
8507 vpxor ymm13,ymm13,ymm1
8508 vpshufb ymm13,ymm13,YMMWORD[$L$rol16]
8509 vpaddd ymm9,ymm9,ymm13
8510 vpxor ymm5,ymm5,ymm9
8511 vpsrld ymm3,ymm5,20
8512 vpslld ymm5,ymm5,12
8513 vpxor ymm5,ymm5,ymm3
8514 vpaddd ymm1,ymm1,ymm5
8515 vpxor ymm13,ymm13,ymm1
8516 vpshufb ymm13,ymm13,YMMWORD[$L$rol8]
8517 vpaddd ymm9,ymm9,ymm13
8518 vpxor ymm5,ymm5,ymm9
8519 vpslld ymm3,ymm5,7
8520 vpsrld ymm5,ymm5,25
8521 vpxor ymm5,ymm5,ymm3
8522 vpalignr ymm13,ymm13,ymm13,12
8523 vpalignr ymm9,ymm9,ymm9,8
8524 vpalignr ymm5,ymm5,ymm5,4
8525 vpaddd ymm2,ymm2,ymm6
8526 vpxor ymm14,ymm14,ymm2
8527 vpshufb ymm14,ymm14,YMMWORD[$L$rol16]
8528 vpaddd ymm10,ymm10,ymm14
8529 vpxor ymm6,ymm6,ymm10
8530 vpsrld ymm3,ymm6,20
8531 vpslld ymm6,ymm6,12
8532 vpxor ymm6,ymm6,ymm3
8533 vpaddd ymm2,ymm2,ymm6
8534 vpxor ymm14,ymm14,ymm2
8535 vpshufb ymm14,ymm14,YMMWORD[$L$rol8]
8536 vpaddd ymm10,ymm10,ymm14
8537 vpxor ymm6,ymm6,ymm10
8538 vpslld ymm3,ymm6,7
8539 vpsrld ymm6,ymm6,25
8540 vpxor ymm6,ymm6,ymm3
8541 vpalignr ymm14,ymm14,ymm14,12
8542 vpalignr ymm10,ymm10,ymm10,8
8543 vpalignr ymm6,ymm6,ymm6,4
8544 vpaddd ymm0,ymm0,ymm4
8545 vpxor ymm12,ymm12,ymm0
8546 vpshufb ymm12,ymm12,YMMWORD[$L$rol16]
8547 vpaddd ymm8,ymm8,ymm12
8548 vpxor ymm4,ymm4,ymm8
8549 vpsrld ymm3,ymm4,20
8550 vpslld ymm4,ymm4,12
8551 vpxor ymm4,ymm4,ymm3
8552 vpaddd ymm0,ymm0,ymm4
8553 vpxor ymm12,ymm12,ymm0
8554 vpshufb ymm12,ymm12,YMMWORD[$L$rol8]
8555 vpaddd ymm8,ymm8,ymm12
8556 vpxor ymm4,ymm4,ymm8
8557 vpslld ymm3,ymm4,7
8558 vpsrld ymm4,ymm4,25
8559 vpxor ymm4,ymm4,ymm3
8560 vpalignr ymm12,ymm12,ymm12,4
8561 vpalignr ymm8,ymm8,ymm8,8
8562 vpalignr ymm4,ymm4,ymm4,12
8563 vpaddd ymm1,ymm1,ymm5
8564 vpxor ymm13,ymm13,ymm1
8565 vpshufb ymm13,ymm13,YMMWORD[$L$rol16]
8566 vpaddd ymm9,ymm9,ymm13
8567 vpxor ymm5,ymm5,ymm9
8568 vpsrld ymm3,ymm5,20
8569 vpslld ymm5,ymm5,12
8570 vpxor ymm5,ymm5,ymm3
8571 vpaddd ymm1,ymm1,ymm5
8572 vpxor ymm13,ymm13,ymm1
8573 vpshufb ymm13,ymm13,YMMWORD[$L$rol8]
8574 vpaddd ymm9,ymm9,ymm13
8575 vpxor ymm5,ymm5,ymm9
8576 vpslld ymm3,ymm5,7
8577 vpsrld ymm5,ymm5,25
8578 vpxor ymm5,ymm5,ymm3
8579 vpalignr ymm13,ymm13,ymm13,4
8580 vpalignr ymm9,ymm9,ymm9,8
8581 vpalignr ymm5,ymm5,ymm5,12
8582 vpaddd ymm2,ymm2,ymm6
8583 vpxor ymm14,ymm14,ymm2
8584 vpshufb ymm14,ymm14,YMMWORD[$L$rol16]
8585 vpaddd ymm10,ymm10,ymm14
8586 vpxor ymm6,ymm6,ymm10
8587 vpsrld ymm3,ymm6,20
8588 vpslld ymm6,ymm6,12
8589 vpxor ymm6,ymm6,ymm3
8590 vpaddd ymm2,ymm2,ymm6
8591 vpxor ymm14,ymm14,ymm2
8592 vpshufb ymm14,ymm14,YMMWORD[$L$rol8]
8593 vpaddd ymm10,ymm10,ymm14
8594 vpxor ymm6,ymm6,ymm10
8595 vpslld ymm3,ymm6,7
8596 vpsrld ymm6,ymm6,25
8597 vpxor ymm6,ymm6,ymm3
8598 vpalignr ymm14,ymm14,ymm14,4
8599 vpalignr ymm10,ymm10,ymm10,8
8600 vpalignr ymm6,ymm6,ymm6,12
8601
8602 dec r10
8603 jne NEAR $L$seal_avx2_320_rounds
8604 vpaddd ymm0,ymm0,YMMWORD[$L$chacha20_consts]
8605 vpaddd ymm1,ymm1,YMMWORD[$L$chacha20_consts]
8606 vpaddd ymm2,ymm2,YMMWORD[$L$chacha20_consts]
8607 vpaddd ymm4,ymm4,ymm7
8608 vpaddd ymm5,ymm5,ymm7
8609 vpaddd ymm6,ymm6,ymm7
8610 vpaddd ymm8,ymm8,ymm11
8611 vpaddd ymm9,ymm9,ymm11
8612 vpaddd ymm10,ymm10,ymm11
8613 vpaddd ymm12,ymm12,YMMWORD[((160+160))+rbp]
8614 vpaddd ymm13,ymm13,YMMWORD[((160+192))+rbp]
8615 vpaddd ymm14,ymm14,YMMWORD[((160+224))+rbp]
8616 vperm2i128 ymm3,ymm4,ymm0,0x02
8617
8618 vpand ymm3,ymm3,YMMWORD[$L$clamp]
8619 vmovdqa YMMWORD[(160+0)+rbp],ymm3
8620
8621 vperm2i128 ymm0,ymm4,ymm0,0x13
8622 vperm2i128 ymm4,ymm12,ymm8,0x13
8623 vperm2i128 ymm8,ymm5,ymm1,0x02
8624 vperm2i128 ymm12,ymm13,ymm9,0x02
8625 vperm2i128 ymm1,ymm5,ymm1,0x13
8626 vperm2i128 ymm5,ymm13,ymm9,0x13
8627 vperm2i128 ymm9,ymm6,ymm2,0x02
8628 vperm2i128 ymm13,ymm14,ymm10,0x02
8629 vperm2i128 ymm2,ymm6,ymm2,0x13
8630 vperm2i128 ymm6,ymm14,ymm10,0x13
8631 jmp NEAR $L$seal_avx2_short
8632
8633 $L$seal_avx2_192:
8634 vmovdqa ymm1,ymm0
8635 vmovdqa ymm2,ymm0
8636 vmovdqa ymm5,ymm4
8637 vmovdqa ymm6,ymm4
8638 vmovdqa ymm9,ymm8
8639 vmovdqa ymm10,ymm8
8640 vpaddd ymm13,ymm12,YMMWORD[$L$avx2_inc]
8641 vmovdqa ymm11,ymm12
8642 vmovdqa ymm15,ymm13
8643 mov r10,10
8644 $L$seal_avx2_192_rounds:
8645 vpaddd ymm0,ymm0,ymm4
8646 vpxor ymm12,ymm12,ymm0
8647 vpshufb ymm12,ymm12,YMMWORD[$L$rol16]
8648 vpaddd ymm8,ymm8,ymm12
8649 vpxor ymm4,ymm4,ymm8
8650 vpsrld ymm3,ymm4,20
8651 vpslld ymm4,ymm4,12
8652 vpxor ymm4,ymm4,ymm3
8653 vpaddd ymm0,ymm0,ymm4
8654 vpxor ymm12,ymm12,ymm0
8655 vpshufb ymm12,ymm12,YMMWORD[$L$rol8]
8656 vpaddd ymm8,ymm8,ymm12
8657 vpxor ymm4,ymm4,ymm8
8658 vpslld ymm3,ymm4,7
8659 vpsrld ymm4,ymm4,25
8660 vpxor ymm4,ymm4,ymm3
8661 vpalignr ymm12,ymm12,ymm12,12
8662 vpalignr ymm8,ymm8,ymm8,8
8663 vpalignr ymm4,ymm4,ymm4,4
8664 vpaddd ymm1,ymm1,ymm5
8665 vpxor ymm13,ymm13,ymm1
8666 vpshufb ymm13,ymm13,YMMWORD[$L$rol16]
8667 vpaddd ymm9,ymm9,ymm13
8668 vpxor ymm5,ymm5,ymm9
8669 vpsrld ymm3,ymm5,20
8670 vpslld ymm5,ymm5,12
8671 vpxor ymm5,ymm5,ymm3
8672 vpaddd ymm1,ymm1,ymm5
8673 vpxor ymm13,ymm13,ymm1
8674 vpshufb ymm13,ymm13,YMMWORD[$L$rol8]
8675 vpaddd ymm9,ymm9,ymm13
8676 vpxor ymm5,ymm5,ymm9
8677 vpslld ymm3,ymm5,7
8678 vpsrld ymm5,ymm5,25
8679 vpxor ymm5,ymm5,ymm3
8680 vpalignr ymm13,ymm13,ymm13,12
8681 vpalignr ymm9,ymm9,ymm9,8
8682 vpalignr ymm5,ymm5,ymm5,4
8683 vpaddd ymm0,ymm0,ymm4
8684 vpxor ymm12,ymm12,ymm0
8685 vpshufb ymm12,ymm12,YMMWORD[$L$rol16]
8686 vpaddd ymm8,ymm8,ymm12
8687 vpxor ymm4,ymm4,ymm8
8688 vpsrld ymm3,ymm4,20
8689 vpslld ymm4,ymm4,12
8690 vpxor ymm4,ymm4,ymm3
8691 vpaddd ymm0,ymm0,ymm4
8692 vpxor ymm12,ymm12,ymm0
8693 vpshufb ymm12,ymm12,YMMWORD[$L$rol8]
8694 vpaddd ymm8,ymm8,ymm12
8695 vpxor ymm4,ymm4,ymm8
8696 vpslld ymm3,ymm4,7
8697 vpsrld ymm4,ymm4,25
8698 vpxor ymm4,ymm4,ymm3
8699 vpalignr ymm12,ymm12,ymm12,4
8700 vpalignr ymm8,ymm8,ymm8,8
8701 vpalignr ymm4,ymm4,ymm4,12
8702 vpaddd ymm1,ymm1,ymm5
8703 vpxor ymm13,ymm13,ymm1
8704 vpshufb ymm13,ymm13,YMMWORD[$L$rol16]
8705 vpaddd ymm9,ymm9,ymm13
8706 vpxor ymm5,ymm5,ymm9
8707 vpsrld ymm3,ymm5,20
8708 vpslld ymm5,ymm5,12
8709 vpxor ymm5,ymm5,ymm3
8710 vpaddd ymm1,ymm1,ymm5
8711 vpxor ymm13,ymm13,ymm1
8712 vpshufb ymm13,ymm13,YMMWORD[$L$rol8]
8713 vpaddd ymm9,ymm9,ymm13
8714 vpxor ymm5,ymm5,ymm9
8715 vpslld ymm3,ymm5,7
8716 vpsrld ymm5,ymm5,25
8717 vpxor ymm5,ymm5,ymm3
8718 vpalignr ymm13,ymm13,ymm13,4
8719 vpalignr ymm9,ymm9,ymm9,8
8720 vpalignr ymm5,ymm5,ymm5,12
8721
8722 dec r10
8723 jne NEAR $L$seal_avx2_192_rounds
8724 vpaddd ymm0,ymm0,ymm2
8725 vpaddd ymm1,ymm1,ymm2
8726 vpaddd ymm4,ymm4,ymm6
8727 vpaddd ymm5,ymm5,ymm6
8728 vpaddd ymm8,ymm8,ymm10
8729 vpaddd ymm9,ymm9,ymm10
8730 vpaddd ymm12,ymm12,ymm11
8731 vpaddd ymm13,ymm13,ymm15
8732 vperm2i128 ymm3,ymm4,ymm0,0x02
8733
8734 vpand ymm3,ymm3,YMMWORD[$L$clamp]
8735 vmovdqa YMMWORD[(160+0)+rbp],ymm3
8736
8737 vperm2i128 ymm0,ymm4,ymm0,0x13
8738 vperm2i128 ymm4,ymm12,ymm8,0x13
8739 vperm2i128 ymm8,ymm5,ymm1,0x02
8740 vperm2i128 ymm12,ymm13,ymm9,0x02
8741 vperm2i128 ymm1,ymm5,ymm1,0x13
8742 vperm2i128 ymm5,ymm13,ymm9,0x13
8743 $L$seal_avx2_short:
8744 mov r8,r8
8745 call poly_hash_ad_internal
8746 xor rcx,rcx
8747 $L$seal_avx2_short_hash_remainder:
8748 cmp rcx,16
8749 jb NEAR $L$seal_avx2_short_loop
8750 add r10,QWORD[((0+0))+rdi]
8751 adc r11,QWORD[((8+0))+rdi]
8752 adc r12,1
8753 mov rax,QWORD[((0+160+0))+rbp]
8754 mov r15,rax
8755 mul r10
8756 mov r13,rax
8757 mov r14,rdx
8758 mov rax,QWORD[((0+160+0))+rbp]
8759 mul r11
8760 imul r15,r12
8761 add r14,rax
8762 adc r15,rdx
8763 mov rax,QWORD[((8+160+0))+rbp]
8764 mov r9,rax
8765 mul r10
8766 add r14,rax
8767 adc rdx,0
8768 mov r10,rdx
8769 mov rax,QWORD[((8+160+0))+rbp]
8770 mul r11
8771 add r15,rax
8772 adc rdx,0
8773 imul r9,r12
8774 add r15,r10
8775 adc r9,rdx
8776 mov r10,r13
8777 mov r11,r14
8778 mov r12,r15
8779 and r12,3
8780 mov r13,r15
8781 and r13,-4
8782 mov r14,r9
8783 shrd r15,r9,2
8784 shr r9,2
8785 add r15,r13
8786 adc r9,r14
8787 add r10,r15
8788 adc r11,r9
8789 adc r12,0
8790
8791 sub rcx,16
8792 add rdi,16
8793 jmp NEAR $L$seal_avx2_short_hash_remainder
8794 $L$seal_avx2_short_loop:
8795 cmp rbx,32
8796 jb NEAR $L$seal_avx2_short_tail
8797 sub rbx,32
8798
8799 vpxor ymm0,ymm0,YMMWORD[rsi]
8800 vmovdqu YMMWORD[rdi],ymm0
8801 lea rsi,[32+rsi]
8802
8803 add r10,QWORD[((0+0))+rdi]
8804 adc r11,QWORD[((8+0))+rdi]
8805 adc r12,1
8806 mov rax,QWORD[((0+160+0))+rbp]
8807 mov r15,rax
8808 mul r10
8809 mov r13,rax
8810 mov r14,rdx
8811 mov rax,QWORD[((0+160+0))+rbp]
8812 mul r11
8813 imul r15,r12
8814 add r14,rax
8815 adc r15,rdx
8816 mov rax,QWORD[((8+160+0))+rbp]
8817 mov r9,rax
8818 mul r10
8819 add r14,rax
8820 adc rdx,0
8821 mov r10,rdx
8822 mov rax,QWORD[((8+160+0))+rbp]
8823 mul r11
8824 add r15,rax
8825 adc rdx,0
8826 imul r9,r12
8827 add r15,r10
8828 adc r9,rdx
8829 mov r10,r13
8830 mov r11,r14
8831 mov r12,r15
8832 and r12,3
8833 mov r13,r15
8834 and r13,-4
8835 mov r14,r9
8836 shrd r15,r9,2
8837 shr r9,2
8838 add r15,r13
8839 adc r9,r14
8840 add r10,r15
8841 adc r11,r9
8842 adc r12,0
8843 add r10,QWORD[((0+16))+rdi]
8844 adc r11,QWORD[((8+16))+rdi]
8845 adc r12,1
8846 mov rax,QWORD[((0+160+0))+rbp]
8847 mov r15,rax
8848 mul r10
8849 mov r13,rax
8850 mov r14,rdx
8851 mov rax,QWORD[((0+160+0))+rbp]
8852 mul r11
8853 imul r15,r12
8854 add r14,rax
8855 adc r15,rdx
8856 mov rax,QWORD[((8+160+0))+rbp]
8857 mov r9,rax
8858 mul r10
8859 add r14,rax
8860 adc rdx,0
8861 mov r10,rdx
8862 mov rax,QWORD[((8+160+0))+rbp]
8863 mul r11
8864 add r15,rax
8865 adc rdx,0
8866 imul r9,r12
8867 add r15,r10
8868 adc r9,rdx
8869 mov r10,r13
8870 mov r11,r14
8871 mov r12,r15
8872 and r12,3
8873 mov r13,r15
8874 and r13,-4
8875 mov r14,r9
8876 shrd r15,r9,2
8877 shr r9,2
8878 add r15,r13
8879 adc r9,r14
8880 add r10,r15
8881 adc r11,r9
8882 adc r12,0
8883
8884 lea rdi,[32+rdi]
8885
8886 vmovdqa ymm0,ymm4
8887 vmovdqa ymm4,ymm8
8888 vmovdqa ymm8,ymm12
8889 vmovdqa ymm12,ymm1
8890 vmovdqa ymm1,ymm5
8891 vmovdqa ymm5,ymm9
8892 vmovdqa ymm9,ymm13
8893 vmovdqa ymm13,ymm2
8894 vmovdqa ymm2,ymm6
8895 jmp NEAR $L$seal_avx2_short_loop
8896 $L$seal_avx2_short_tail:
8897 cmp rbx,16
8898 jb NEAR $L$seal_avx2_exit
8899 sub rbx,16
8900 vpxor xmm3,xmm0,XMMWORD[rsi]
8901 vmovdqu XMMWORD[rdi],xmm3
8902 lea rsi,[16+rsi]
8903 add r10,QWORD[((0+0))+rdi]
8904 adc r11,QWORD[((8+0))+rdi]
8905 adc r12,1
8906 mov rax,QWORD[((0+160+0))+rbp]
8907 mov r15,rax
8908 mul r10
8909 mov r13,rax
8910 mov r14,rdx
8911 mov rax,QWORD[((0+160+0))+rbp]
8912 mul r11
8913 imul r15,r12
8914 add r14,rax
8915 adc r15,rdx
8916 mov rax,QWORD[((8+160+0))+rbp]
8917 mov r9,rax
8918 mul r10
8919 add r14,rax
8920 adc rdx,0
8921 mov r10,rdx
8922 mov rax,QWORD[((8+160+0))+rbp]
8923 mul r11
8924 add r15,rax
8925 adc rdx,0
8926 imul r9,r12
8927 add r15,r10
8928 adc r9,rdx
8929 mov r10,r13
8930 mov r11,r14
8931 mov r12,r15
8932 and r12,3
8933 mov r13,r15
8934 and r13,-4
8935 mov r14,r9
8936 shrd r15,r9,2
8937 shr r9,2
8938 add r15,r13
8939 adc r9,r14
8940 add r10,r15
8941 adc r11,r9
8942 adc r12,0
8943
8944 lea rdi,[16+rdi]
8945 vextracti128 xmm0,ymm0,1
8946 $L$seal_avx2_exit:
8947 vzeroupper
8948 jmp NEAR $L$seal_sse_tail_16
8949
8950
8951 %else
8952 ; Work around https://bugzilla.nasm.us/show_bug.cgi?id=3392738
8953 ret
8954 %endif