Debian Codegraph @

external/boringssl/crypto/poly1305/asm/poly1305-x86.pl

raw
1 #!/usr/bin/env perl
2 #
3 # ====================================================================
4 # Written by Andy Polyakov <[email protected]> for the OpenSSL
5 # project. The module is, however, dual licensed under OpenSSL and
6 # CRYPTOGAMS licenses depending on where you obtain it. For further
7 # details see http://www.openssl.org/~appro/cryptogams/.
8 # ====================================================================
9 #
10 # This module implements Poly1305 hash for x86.
11 #
12 # April 2015
13 #
14 # Numbers are cycles per processed byte with poly1305_blocks alone,
15 # measured with rdtsc at fixed clock frequency.
16 #
17 # IALU/gcc-3.4(*) SSE2(**) AVX2
18 # Pentium 15.7/+80% -
19 # PIII 6.21/+90% -
20 # P4 19.8/+40% 3.24
21 # Core 2 4.85/+90% 1.80
22 # Westmere 4.58/+100% 1.43
23 # Sandy Bridge 3.90/+100% 1.36
24 # Haswell 3.88/+70% 1.18 0.72
25 # Silvermont 11.0/+40% 4.80
26 # VIA Nano 6.71/+90% 2.47
27 # Sledgehammer 3.51/+180% 4.27
28 # Bulldozer 4.53/+140% 1.31
29 #
30 # (*) gcc 4.8 for some reason generated worse code;
31 # (**) besides SSE2 there are floating-point and AVX options; FP
32 # is deemed unnecessary, because pre-SSE2 processor are too
33 # old to care about, while it's not the fastest option on
34 # SSE2-capable ones; AVX is omitted, because it doesn't give
35 # a lot of improvement, 5-10% depending on processor;
36
37 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
38 push(@INC,"${dir}","${dir}../../perlasm");
39 require "x86asm.pl";
40
41 &asm_init($ARGV[0],"poly1305-x86.pl",$ARGV[$#ARGV] eq "386");
42
43 $sse2=$avx=0;
44 for (@ARGV) { $sse2=1 if (/-DOPENSSL_IA32_SSE2/); }
45
46 if ($sse2) {
47 &static_label("const_sse2");
48 &static_label("enter_blocks");
49 &static_label("enter_emit");
50 &external_label("OPENSSL_ia32cap_P");
51
52 # This may be set to 2, but valgrind can't do AVX2 on 32-bit. Without a
53 # way to verify test coverage, keep it disabled.
54 $avx = 0;
55 }
56
57 ########################################################################
58 # Layout of opaque area is following.
59 #
60 # unsigned __int32 h[5]; # current hash value base 2^32
61 # unsigned __int32 pad; # is_base2_26 in vector context
62 # unsigned __int32 r[4]; # key value base 2^32
63
64 &align(64);
65 &function_begin("poly1305_init");
66 &mov ("edi",&wparam(0)); # context
67 &mov ("esi",&wparam(1)); # key
68 &mov ("ebp",&wparam(2)); # function table
69
70 &xor ("eax","eax");
71 &mov (&DWP(4*0,"edi"),"eax"); # zero hash value
72 &mov (&DWP(4*1,"edi"),"eax");
73 &mov (&DWP(4*2,"edi"),"eax");
74 &mov (&DWP(4*3,"edi"),"eax");
75 &mov (&DWP(4*4,"edi"),"eax");
76 &mov (&DWP(4*5,"edi"),"eax"); # is_base2_26
77
78 &cmp ("esi",0);
79 &je (&label("nokey"));
80
81 if ($sse2) {
82 &call (&label("pic_point"));
83 &set_label("pic_point");
84 &blindpop("ebx");
85
86 &lea ("eax",&DWP("poly1305_blocks-".&label("pic_point"),"ebx"));
87 &lea ("edx",&DWP("poly1305_emit-".&label("pic_point"),"ebx"));
88
89 &picmeup("edi","OPENSSL_ia32cap_P","ebx",&label("pic_point"));
90 &mov ("ecx",&DWP(0,"edi"));
91 &and ("ecx",1<<26|1<<24);
92 &cmp ("ecx",1<<26|1<<24); # SSE2 and XMM?
93 &jne (&label("no_sse2"));
94
95 &lea ("eax",&DWP("_poly1305_blocks_sse2-".&label("pic_point"),"ebx"));
96 &lea ("edx",&DWP("_poly1305_emit_sse2-".&label("pic_point"),"ebx"));
97
98 if ($avx>1) {
99 &mov ("ecx",&DWP(8,"edi"));
100 &test ("ecx",1<<5); # AVX2?
101 &jz (&label("no_sse2"));
102
103 &lea ("eax",&DWP("_poly1305_blocks_avx2-".&label("pic_point"),"ebx"));
104 }
105 &set_label("no_sse2");
106 &mov ("edi",&wparam(0)); # reload context
107 &mov (&DWP(0,"ebp"),"eax"); # fill function table
108 &mov (&DWP(4,"ebp"),"edx");
109 }
110
111 &mov ("eax",&DWP(4*0,"esi")); # load input key
112 &mov ("ebx",&DWP(4*1,"esi"));
113 &mov ("ecx",&DWP(4*2,"esi"));
114 &mov ("edx",&DWP(4*3,"esi"));
115 &and ("eax",0x0fffffff);
116 &and ("ebx",0x0ffffffc);
117 &and ("ecx",0x0ffffffc);
118 &and ("edx",0x0ffffffc);
119 &mov (&DWP(4*6,"edi"),"eax");
120 &mov (&DWP(4*7,"edi"),"ebx");
121 &mov (&DWP(4*8,"edi"),"ecx");
122 &mov (&DWP(4*9,"edi"),"edx");
123
124 &mov ("eax",$sse2);
125 &set_label("nokey");
126 &function_end("poly1305_init");
127
128 ($h0,$h1,$h2,$h3,$h4,
129 $d0,$d1,$d2,$d3,
130 $r0,$r1,$r2,$r3,
131 $s1,$s2,$s3)=map(4*$_,(0..15));
132
133 &function_begin("poly1305_blocks");
134 &mov ("edi",&wparam(0)); # ctx
135 &mov ("esi",&wparam(1)); # inp
136 &mov ("ecx",&wparam(2)); # len
137 &set_label("enter_blocks");
138 &and ("ecx",-15);
139 &jz (&label("nodata"));
140
141 &stack_push(16);
142 &mov ("eax",&DWP(4*6,"edi")); # r0
143 &mov ("ebx",&DWP(4*7,"edi")); # r1
144 &lea ("ebp",&DWP(0,"esi","ecx")); # end of input
145 &mov ("ecx",&DWP(4*8,"edi")); # r2
146 &mov ("edx",&DWP(4*9,"edi")); # r3
147
148 &mov (&wparam(2),"ebp");
149 &mov ("ebp","esi");
150
151 &mov (&DWP($r0,"esp"),"eax"); # r0
152 &mov ("eax","ebx");
153 &shr ("eax",2);
154 &mov (&DWP($r1,"esp"),"ebx"); # r1
155 &add ("eax","ebx"); # s1
156 &mov ("ebx","ecx");
157 &shr ("ebx",2);
158 &mov (&DWP($r2,"esp"),"ecx"); # r2
159 &add ("ebx","ecx"); # s2
160 &mov ("ecx","edx");
161 &shr ("ecx",2);
162 &mov (&DWP($r3,"esp"),"edx"); # r3
163 &add ("ecx","edx"); # s3
164 &mov (&DWP($s1,"esp"),"eax"); # s1
165 &mov (&DWP($s2,"esp"),"ebx"); # s2
166 &mov (&DWP($s3,"esp"),"ecx"); # s3
167
168 &mov ("eax",&DWP(4*0,"edi")); # load hash value
169 &mov ("ebx",&DWP(4*1,"edi"));
170 &mov ("ecx",&DWP(4*2,"edi"));
171 &mov ("esi",&DWP(4*3,"edi"));
172 &mov ("edi",&DWP(4*4,"edi"));
173 &jmp (&label("loop"));
174
175 &set_label("loop",32);
176 &add ("eax",&DWP(4*0,"ebp")); # accumulate input
177 &adc ("ebx",&DWP(4*1,"ebp"));
178 &adc ("ecx",&DWP(4*2,"ebp"));
179 &adc ("esi",&DWP(4*3,"ebp"));
180 &lea ("ebp",&DWP(4*4,"ebp"));
181 &adc ("edi",&wparam(3)); # padbit
182
183 &mov (&DWP($h0,"esp"),"eax"); # put aside hash[+inp]
184 &mov (&DWP($h3,"esp"),"esi");
185
186 &mul (&DWP($r0,"esp")); # h0*r0
187 &mov (&DWP($h4,"esp"),"edi");
188 &mov ("edi","eax");
189 &mov ("eax","ebx"); # h1
190 &mov ("esi","edx");
191 &mul (&DWP($s3,"esp")); # h1*s3
192 &add ("edi","eax");
193 &mov ("eax","ecx"); # h2
194 &adc ("esi","edx");
195 &mul (&DWP($s2,"esp")); # h2*s2
196 &add ("edi","eax");
197 &mov ("eax",&DWP($h3,"esp"));
198 &adc ("esi","edx");
199 &mul (&DWP($s1,"esp")); # h3*s1
200 &add ("edi","eax");
201 &mov ("eax",&DWP($h0,"esp"));
202 &adc ("esi","edx");
203
204 &mul (&DWP($r1,"esp")); # h0*r1
205 &mov (&DWP($d0,"esp"),"edi");
206 &xor ("edi","edi");
207 &add ("esi","eax");
208 &mov ("eax","ebx"); # h1
209 &adc ("edi","edx");
210 &mul (&DWP($r0,"esp")); # h1*r0
211 &add ("esi","eax");
212 &mov ("eax","ecx"); # h2
213 &adc ("edi","edx");
214 &mul (&DWP($s3,"esp")); # h2*s3
215 &add ("esi","eax");
216 &mov ("eax",&DWP($h3,"esp"));
217 &adc ("edi","edx");
218 &mul (&DWP($s2,"esp")); # h3*s2
219 &add ("esi","eax");
220 &mov ("eax",&DWP($h4,"esp"));
221 &adc ("edi","edx");
222 &imul ("eax",&DWP($s1,"esp")); # h4*s1
223 &add ("esi","eax");
224 &mov ("eax",&DWP($h0,"esp"));
225 &adc ("edi",0);
226
227 &mul (&DWP($r2,"esp")); # h0*r2
228 &mov (&DWP($d1,"esp"),"esi");
229 &xor ("esi","esi");
230 &add ("edi","eax");
231 &mov ("eax","ebx"); # h1
232 &adc ("esi","edx");
233 &mul (&DWP($r1,"esp")); # h1*r1
234 &add ("edi","eax");
235 &mov ("eax","ecx"); # h2
236 &adc ("esi","edx");
237 &mul (&DWP($r0,"esp")); # h2*r0
238 &add ("edi","eax");
239 &mov ("eax",&DWP($h3,"esp"));
240 &adc ("esi","edx");
241 &mul (&DWP($s3,"esp")); # h3*s3
242 &add ("edi","eax");
243 &mov ("eax",&DWP($h4,"esp"));
244 &adc ("esi","edx");
245 &imul ("eax",&DWP($s2,"esp")); # h4*s2
246 &add ("edi","eax");
247 &mov ("eax",&DWP($h0,"esp"));
248 &adc ("esi",0);
249
250 &mul (&DWP($r3,"esp")); # h0*r3
251 &mov (&DWP($d2,"esp"),"edi");
252 &xor ("edi","edi");
253 &add ("esi","eax");
254 &mov ("eax","ebx"); # h1
255 &adc ("edi","edx");
256 &mul (&DWP($r2,"esp")); # h1*r2
257 &add ("esi","eax");
258 &mov ("eax","ecx"); # h2
259 &adc ("edi","edx");
260 &mul (&DWP($r1,"esp")); # h2*r1
261 &add ("esi","eax");
262 &mov ("eax",&DWP($h3,"esp"));
263 &adc ("edi","edx");
264 &mul (&DWP($r0,"esp")); # h3*r0
265 &add ("esi","eax");
266 &mov ("ecx",&DWP($h4,"esp"));
267 &adc ("edi","edx");
268
269 &mov ("edx","ecx");
270 &imul ("ecx",&DWP($s3,"esp")); # h4*s3
271 &add ("esi","ecx");
272 &mov ("eax",&DWP($d0,"esp"));
273 &adc ("edi",0);
274
275 &imul ("edx",&DWP($r0,"esp")); # h4*r0
276 &add ("edx","edi");
277
278 &mov ("ebx",&DWP($d1,"esp"));
279 &mov ("ecx",&DWP($d2,"esp"));
280
281 &mov ("edi","edx"); # last reduction step
282 &shr ("edx",2);
283 &and ("edi",3);
284 &lea ("edx",&DWP(0,"edx","edx",4)); # *5
285 &add ("eax","edx");
286 &adc ("ebx",0);
287 &adc ("ecx",0);
288 &adc ("esi",0);
289
290 &cmp ("ebp",&wparam(2)); # done yet?
291 &jne (&label("loop"));
292
293 &mov ("edx",&wparam(0)); # ctx
294 &stack_pop(16);
295 &mov (&DWP(4*0,"edx"),"eax"); # store hash value
296 &mov (&DWP(4*1,"edx"),"ebx");
297 &mov (&DWP(4*2,"edx"),"ecx");
298 &mov (&DWP(4*3,"edx"),"esi");
299 &mov (&DWP(4*4,"edx"),"edi");
300 &set_label("nodata");
301 &function_end("poly1305_blocks");
302
303 &function_begin("poly1305_emit");
304 &mov ("ebp",&wparam(0)); # context
305 &set_label("enter_emit");
306 &mov ("edi",&wparam(1)); # output
307 &mov ("eax",&DWP(4*0,"ebp")); # load hash value
308 &mov ("ebx",&DWP(4*1,"ebp"));
309 &mov ("ecx",&DWP(4*2,"ebp"));
310 &mov ("edx",&DWP(4*3,"ebp"));
311 &mov ("esi",&DWP(4*4,"ebp"));
312
313 &add ("eax",5); # compare to modulus
314 &adc ("ebx",0);
315 &adc ("ecx",0);
316 &adc ("edx",0);
317 &adc ("esi",0);
318 &shr ("esi",2); # did it carry/borrow?
319 &neg ("esi"); # do we choose hash-modulus?
320
321 &and ("eax","esi");
322 &and ("ebx","esi");
323 &and ("ecx","esi");
324 &and ("edx","esi");
325 &mov (&DWP(4*0,"edi"),"eax");
326 &mov (&DWP(4*1,"edi"),"ebx");
327 &mov (&DWP(4*2,"edi"),"ecx");
328 &mov (&DWP(4*3,"edi"),"edx");
329
330 &not ("esi"); # or original hash value?
331 &mov ("eax",&DWP(4*0,"ebp"));
332 &mov ("ebx",&DWP(4*1,"ebp"));
333 &mov ("ecx",&DWP(4*2,"ebp"));
334 &mov ("edx",&DWP(4*3,"ebp"));
335 &mov ("ebp",&wparam(2));
336 &and ("eax","esi");
337 &and ("ebx","esi");
338 &and ("ecx","esi");
339 &and ("edx","esi");
340 &or ("eax",&DWP(4*0,"edi"));
341 &or ("ebx",&DWP(4*1,"edi"));
342 &or ("ecx",&DWP(4*2,"edi"));
343 &or ("edx",&DWP(4*3,"edi"));
344
345 &add ("eax",&DWP(4*0,"ebp")); # accumulate key
346 &adc ("ebx",&DWP(4*1,"ebp"));
347 &adc ("ecx",&DWP(4*2,"ebp"));
348 &adc ("edx",&DWP(4*3,"ebp"));
349
350 &mov (&DWP(4*0,"edi"),"eax");
351 &mov (&DWP(4*1,"edi"),"ebx");
352 &mov (&DWP(4*2,"edi"),"ecx");
353 &mov (&DWP(4*3,"edi"),"edx");
354 &function_end("poly1305_emit");
355
356 if ($sse2) {
357 ########################################################################
358 # Layout of opaque area is following.
359 #
360 # unsigned __int32 h[5]; # current hash value base 2^26
361 # unsigned __int32 is_base2_26;
362 # unsigned __int32 r[4]; # key value base 2^32
363 # unsigned __int32 pad[2];
364 # struct { unsigned __int32 r^4, r^3, r^2, r^1; } r[9];
365 #
366 # where r^n are base 2^26 digits of degrees of multiplier key. There are
367 # 5 digits, but last four are interleaved with multiples of 5, totalling
368 # in 9 elements: r0, r1, 5*r1, r2, 5*r2, r3, 5*r3, r4, 5*r4.
369
370 my ($D0,$D1,$D2,$D3,$D4,$T0,$T1,$T2)=map("xmm$_",(0..7));
371 my $MASK=$T2; # borrow and keep in mind
372
373 &align (32);
374 &function_begin_B("_poly1305_init_sse2");
375 &movdqu ($D4,&QWP(4*6,"edi")); # key base 2^32
376 &lea ("edi",&DWP(16*3,"edi")); # size optimization
377 &mov ("ebp","esp");
378 &sub ("esp",16*(9+5));
379 &and ("esp",-16);
380
381 #&pand ($D4,&QWP(96,"ebx")); # magic mask
382 &movq ($MASK,&QWP(64,"ebx"));
383
384 &movdqa ($D0,$D4);
385 &movdqa ($D1,$D4);
386 &movdqa ($D2,$D4);
387
388 &pand ($D0,$MASK); # -> base 2^26
389 &psrlq ($D1,26);
390 &psrldq ($D2,6);
391 &pand ($D1,$MASK);
392 &movdqa ($D3,$D2);
393 &psrlq ($D2,4)
394 &psrlq ($D3,30);
395 &pand ($D2,$MASK);
396 &pand ($D3,$MASK);
397 &psrldq ($D4,13);
398
399 &lea ("edx",&DWP(16*9,"esp")); # size optimization
400 &mov ("ecx",2);
401 &set_label("square");
402 &movdqa (&QWP(16*0,"esp"),$D0);
403 &movdqa (&QWP(16*1,"esp"),$D1);
404 &movdqa (&QWP(16*2,"esp"),$D2);
405 &movdqa (&QWP(16*3,"esp"),$D3);
406 &movdqa (&QWP(16*4,"esp"),$D4);
407
408 &movdqa ($T1,$D1);
409 &movdqa ($T0,$D2);
410 &pslld ($T1,2);
411 &pslld ($T0,2);
412 &paddd ($T1,$D1); # *5
413 &paddd ($T0,$D2); # *5
414 &movdqa (&QWP(16*5,"esp"),$T1);
415 &movdqa (&QWP(16*6,"esp"),$T0);
416 &movdqa ($T1,$D3);
417 &movdqa ($T0,$D4);
418 &pslld ($T1,2);
419 &pslld ($T0,2);
420 &paddd ($T1,$D3); # *5
421 &paddd ($T0,$D4); # *5
422 &movdqa (&QWP(16*7,"esp"),$T1);
423 &movdqa (&QWP(16*8,"esp"),$T0);
424
425 &pshufd ($T1,$D0,0b01000100);
426 &movdqa ($T0,$D1);
427 &pshufd ($D1,$D1,0b01000100);
428 &pshufd ($D2,$D2,0b01000100);
429 &pshufd ($D3,$D3,0b01000100);
430 &pshufd ($D4,$D4,0b01000100);
431 &movdqa (&QWP(16*0,"edx"),$T1);
432 &movdqa (&QWP(16*1,"edx"),$D1);
433 &movdqa (&QWP(16*2,"edx"),$D2);
434 &movdqa (&QWP(16*3,"edx"),$D3);
435 &movdqa (&QWP(16*4,"edx"),$D4);
436
437 ################################################################
438 # d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4
439 # d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4
440 # d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4
441 # d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4
442 # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
443
444 &pmuludq ($D4,$D0); # h4*r0
445 &pmuludq ($D3,$D0); # h3*r0
446 &pmuludq ($D2,$D0); # h2*r0
447 &pmuludq ($D1,$D0); # h1*r0
448 &pmuludq ($D0,$T1); # h0*r0
449
450 sub pmuladd {
451 my $load = shift;
452 my $base = shift; $base = "esp" if (!defined($base));
453
454 ################################################################
455 # As for choice to "rotate" $T0-$T2 in order to move paddq
456 # past next multiplication. While it makes code harder to read
457 # and doesn't have significant effect on most processors, it
458 # makes a lot of difference on Atom, up to 30% improvement.
459
460 &movdqa ($T1,$T0);
461 &pmuludq ($T0,&QWP(16*3,$base)); # r1*h3
462 &movdqa ($T2,$T1);
463 &pmuludq ($T1,&QWP(16*2,$base)); # r1*h2
464 &paddq ($D4,$T0);
465 &movdqa ($T0,$T2);
466 &pmuludq ($T2,&QWP(16*1,$base)); # r1*h1
467 &paddq ($D3,$T1);
468 &$load ($T1,5); # s1
469 &pmuludq ($T0,&QWP(16*0,$base)); # r1*h0
470 &paddq ($D2,$T2);
471 &pmuludq ($T1,&QWP(16*4,$base)); # s1*h4
472 &$load ($T2,2); # r2^n
473 &paddq ($D1,$T0);
474
475 &movdqa ($T0,$T2);
476 &pmuludq ($T2,&QWP(16*2,$base)); # r2*h2
477 &paddq ($D0,$T1);
478 &movdqa ($T1,$T0);
479 &pmuludq ($T0,&QWP(16*1,$base)); # r2*h1
480 &paddq ($D4,$T2);
481 &$load ($T2,6); # s2^n
482 &pmuludq ($T1,&QWP(16*0,$base)); # r2*h0
483 &paddq ($D3,$T0);
484 &movdqa ($T0,$T2);
485 &pmuludq ($T2,&QWP(16*4,$base)); # s2*h4
486 &paddq ($D2,$T1);
487 &pmuludq ($T0,&QWP(16*3,$base)); # s2*h3
488 &$load ($T1,3); # r3^n
489 &paddq ($D1,$T2);
490
491 &movdqa ($T2,$T1);
492 &pmuludq ($T1,&QWP(16*1,$base)); # r3*h1
493 &paddq ($D0,$T0);
494 &$load ($T0,7); # s3^n
495 &pmuludq ($T2,&QWP(16*0,$base)); # r3*h0
496 &paddq ($D4,$T1);
497 &movdqa ($T1,$T0);
498 &pmuludq ($T0,&QWP(16*4,$base)); # s3*h4
499 &paddq ($D3,$T2);
500 &movdqa ($T2,$T1);
501 &pmuludq ($T1,&QWP(16*3,$base)); # s3*h3
502 &paddq ($D2,$T0);
503 &pmuludq ($T2,&QWP(16*2,$base)); # s3*h2
504 &$load ($T0,4); # r4^n
505 &paddq ($D1,$T1);
506
507 &$load ($T1,8); # s4^n
508 &pmuludq ($T0,&QWP(16*0,$base)); # r4*h0
509 &paddq ($D0,$T2);
510 &movdqa ($T2,$T1);
511 &pmuludq ($T1,&QWP(16*4,$base)); # s4*h4
512 &paddq ($D4,$T0);
513 &movdqa ($T0,$T2);
514 &pmuludq ($T2,&QWP(16*1,$base)); # s4*h1
515 &paddq ($D3,$T1);
516 &movdqa ($T1,$T0);
517 &pmuludq ($T0,&QWP(16*2,$base)); # s4*h2
518 &paddq ($D0,$T2);
519 &pmuludq ($T1,&QWP(16*3,$base)); # s4*h3
520 &movdqa ($MASK,&QWP(64,"ebx"));
521 &paddq ($D1,$T0);
522 &paddq ($D2,$T1);
523 }
524 &pmuladd (sub { my ($reg,$i)=@_;
525 &movdqa ($reg,&QWP(16*$i,"esp"));
526 },"edx");
527
528 sub lazy_reduction {
529 my $extra = shift;
530 my $paddx = defined($extra) ? paddq : paddd;
531
532 ################################################################
533 # lazy reduction as discussed in "NEON crypto" by D.J. Bernstein
534 # and P. Schwabe
535
536 &movdqa ($T0,$D3);
537 &pand ($D3,$MASK);
538 &psrlq ($T0,26);
539 &$extra () if (defined($extra));
540 &paddq ($T0,$D4); # h3 -> h4
541 &movdqa ($T1,$D0);
542 &pand ($D0,$MASK);
543 &psrlq ($T1,26);
544 &movdqa ($D4,$T0);
545 &paddq ($T1,$D1); # h0 -> h1
546 &psrlq ($T0,26);
547 &pand ($D4,$MASK);
548 &movdqa ($D1,$T1);
549 &psrlq ($T1,26);
550 &paddd ($D0,$T0); # favour paddd when
551 # possible, because
552 # paddq is "broken"
553 # on Atom
554 &psllq ($T0,2);
555 &paddq ($T1,$D2); # h1 -> h2
556 &$paddx ($T0,$D0); # h4 -> h0
557 &pand ($D1,$MASK);
558 &movdqa ($D2,$T1);
559 &psrlq ($T1,26);
560 &pand ($D2,$MASK);
561 &paddd ($T1,$D3); # h2 -> h3
562 &movdqa ($D0,$T0);
563 &psrlq ($T0,26);
564 &movdqa ($D3,$T1);
565 &psrlq ($T1,26);
566 &pand ($D0,$MASK);
567 &paddd ($D1,$T0); # h0 -> h1
568 &pand ($D3,$MASK);
569 &paddd ($D4,$T1); # h3 -> h4
570 }
571 &lazy_reduction ();
572
573 &dec ("ecx");
574 &jz (&label("square_break"));
575
576 &punpcklqdq ($D0,&QWP(16*0,"esp")); # 0:r^1:0:r^2
577 &punpcklqdq ($D1,&QWP(16*1,"esp"));
578 &punpcklqdq ($D2,&QWP(16*2,"esp"));
579 &punpcklqdq ($D3,&QWP(16*3,"esp"));
580 &punpcklqdq ($D4,&QWP(16*4,"esp"));
581 &jmp (&label("square"));
582
583 &set_label("square_break");
584 &psllq ($D0,32); # -> r^3:0:r^4:0
585 &psllq ($D1,32);
586 &psllq ($D2,32);
587 &psllq ($D3,32);
588 &psllq ($D4,32);
589 &por ($D0,&QWP(16*0,"esp")); # r^3:r^1:r^4:r^2
590 &por ($D1,&QWP(16*1,"esp"));
591 &por ($D2,&QWP(16*2,"esp"));
592 &por ($D3,&QWP(16*3,"esp"));
593 &por ($D4,&QWP(16*4,"esp"));
594
595 &pshufd ($D0,$D0,0b10001101); # -> r^1:r^2:r^3:r^4
596 &pshufd ($D1,$D1,0b10001101);
597 &pshufd ($D2,$D2,0b10001101);
598 &pshufd ($D3,$D3,0b10001101);
599 &pshufd ($D4,$D4,0b10001101);
600
601 &movdqu (&QWP(16*0,"edi"),$D0); # save the table
602 &movdqu (&QWP(16*1,"edi"),$D1);
603 &movdqu (&QWP(16*2,"edi"),$D2);
604 &movdqu (&QWP(16*3,"edi"),$D3);
605 &movdqu (&QWP(16*4,"edi"),$D4);
606
607 &movdqa ($T1,$D1);
608 &movdqa ($T0,$D2);
609 &pslld ($T1,2);
610 &pslld ($T0,2);
611 &paddd ($T1,$D1); # *5
612 &paddd ($T0,$D2); # *5
613 &movdqu (&QWP(16*5,"edi"),$T1);
614 &movdqu (&QWP(16*6,"edi"),$T0);
615 &movdqa ($T1,$D3);
616 &movdqa ($T0,$D4);
617 &pslld ($T1,2);
618 &pslld ($T0,2);
619 &paddd ($T1,$D3); # *5
620 &paddd ($T0,$D4); # *5
621 &movdqu (&QWP(16*7,"edi"),$T1);
622 &movdqu (&QWP(16*8,"edi"),$T0);
623
624 &mov ("esp","ebp");
625 &lea ("edi",&DWP(-16*3,"edi")); # size de-optimization
626 &ret ();
627 &function_end_B("_poly1305_init_sse2");
628
629 &align (32);
630 &function_begin("_poly1305_blocks_sse2");
631 &mov ("edi",&wparam(0)); # ctx
632 &mov ("esi",&wparam(1)); # inp
633 &mov ("ecx",&wparam(2)); # len
634
635 &mov ("eax",&DWP(4*5,"edi")); # is_base2_26
636 &and ("ecx",-16);
637 &jz (&label("nodata"));
638 &cmp ("ecx",64);
639 &jae (&label("enter_sse2"));
640 &test ("eax","eax"); # is_base2_26?
641 &jz (&label("enter_blocks"));
642
643 &set_label("enter_sse2",16);
644 &call (&label("pic_point"));
645 &set_label("pic_point");
646 &blindpop("ebx");
647 &lea ("ebx",&DWP(&label("const_sse2")."-".&label("pic_point"),"ebx"));
648
649 &test ("eax","eax"); # is_base2_26?
650 &jnz (&label("base2_26"));
651
652 &call ("_poly1305_init_sse2");
653
654 ################################################# base 2^32 -> base 2^26
655 &mov ("eax",&DWP(0,"edi"));
656 &mov ("ecx",&DWP(3,"edi"));
657 &mov ("edx",&DWP(6,"edi"));
658 &mov ("esi",&DWP(9,"edi"));
659 &mov ("ebp",&DWP(13,"edi"));
660 &mov (&DWP(4*5,"edi"),1); # is_base2_26
661
662 &shr ("ecx",2);
663 &and ("eax",0x3ffffff);
664 &shr ("edx",4);
665 &and ("ecx",0x3ffffff);
666 &shr ("esi",6);
667 &and ("edx",0x3ffffff);
668
669 &movd ($D0,"eax");
670 &movd ($D1,"ecx");
671 &movd ($D2,"edx");
672 &movd ($D3,"esi");
673 &movd ($D4,"ebp");
674
675 &mov ("esi",&wparam(1)); # [reload] inp
676 &mov ("ecx",&wparam(2)); # [reload] len
677 &jmp (&label("base2_32"));
678
679 &set_label("base2_26",16);
680 &movd ($D0,&DWP(4*0,"edi")); # load hash value
681 &movd ($D1,&DWP(4*1,"edi"));
682 &movd ($D2,&DWP(4*2,"edi"));
683 &movd ($D3,&DWP(4*3,"edi"));
684 &movd ($D4,&DWP(4*4,"edi"));
685 &movdqa ($MASK,&QWP(64,"ebx"));
686
687 &set_label("base2_32");
688 &mov ("eax",&wparam(3)); # padbit
689 &mov ("ebp","esp");
690
691 &sub ("esp",16*(5+5+5+9+9));
692 &and ("esp",-16);
693
694 &lea ("edi",&DWP(16*3,"edi")); # size optimization
695 &shl ("eax",24); # padbit
696
697 &test ("ecx",31);
698 &jz (&label("even"));
699
700 ################################################################
701 # process single block, with SSE2, because it's still faster
702 # even though half of result is discarded
703
704 &movdqu ($T1,&QWP(0,"esi")); # input
705 &lea ("esi",&DWP(16,"esi"));
706
707 &movdqa ($T0,$T1); # -> base 2^26 ...
708 &pand ($T1,$MASK);
709 &paddd ($D0,$T1); # ... and accumuate
710
711 &movdqa ($T1,$T0);
712 &psrlq ($T0,26);
713 &psrldq ($T1,6);
714 &pand ($T0,$MASK);
715 &paddd ($D1,$T0);
716
717 &movdqa ($T0,$T1);
718 &psrlq ($T1,4);
719 &pand ($T1,$MASK);
720 &paddd ($D2,$T1);
721
722 &movdqa ($T1,$T0);
723 &psrlq ($T0,30);
724 &pand ($T0,$MASK);
725 &psrldq ($T1,7);
726 &paddd ($D3,$T0);
727
728 &movd ($T0,"eax"); # padbit
729 &paddd ($D4,$T1);
730 &movd ($T1,&DWP(16*0+12,"edi")); # r0
731 &paddd ($D4,$T0);
732
733 &movdqa (&QWP(16*0,"esp"),$D0);
734 &movdqa (&QWP(16*1,"esp"),$D1);
735 &movdqa (&QWP(16*2,"esp"),$D2);
736 &movdqa (&QWP(16*3,"esp"),$D3);
737 &movdqa (&QWP(16*4,"esp"),$D4);
738
739 ################################################################
740 # d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4
741 # d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4
742 # d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4
743 # d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4
744 # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
745
746 &pmuludq ($D0,$T1); # h4*r0
747 &pmuludq ($D1,$T1); # h3*r0
748 &pmuludq ($D2,$T1); # h2*r0
749 &movd ($T0,&DWP(16*1+12,"edi")); # r1
750 &pmuludq ($D3,$T1); # h1*r0
751 &pmuludq ($D4,$T1); # h0*r0
752
753 &pmuladd (sub { my ($reg,$i)=@_;
754 &movd ($reg,&DWP(16*$i+12,"edi"));
755 });
756
757 &lazy_reduction ();
758
759 &sub ("ecx",16);
760 &jz (&label("done"));
761
762 &set_label("even");
763 &lea ("edx",&DWP(16*(5+5+5+9),"esp"));# size optimization
764 &lea ("eax",&DWP(-16*2,"esi"));
765 &sub ("ecx",64);
766
767 ################################################################
768 # expand and copy pre-calculated table to stack
769
770 &movdqu ($T0,&QWP(16*0,"edi")); # r^1:r^2:r^3:r^4
771 &pshufd ($T1,$T0,0b01000100); # duplicate r^3:r^4
772 &cmovb ("esi","eax");
773 &pshufd ($T0,$T0,0b11101110); # duplicate r^1:r^2
774 &movdqa (&QWP(16*0,"edx"),$T1);
775 &lea ("eax",&DWP(16*10,"esp"));
776 &movdqu ($T1,&QWP(16*1,"edi"));
777 &movdqa (&QWP(16*(0-9),"edx"),$T0);
778 &pshufd ($T0,$T1,0b01000100);
779 &pshufd ($T1,$T1,0b11101110);
780 &movdqa (&QWP(16*1,"edx"),$T0);
781 &movdqu ($T0,&QWP(16*2,"edi"));
782 &movdqa (&QWP(16*(1-9),"edx"),$T1);
783 &pshufd ($T1,$T0,0b01000100);
784 &pshufd ($T0,$T0,0b11101110);
785 &movdqa (&QWP(16*2,"edx"),$T1);
786 &movdqu ($T1,&QWP(16*3,"edi"));
787 &movdqa (&QWP(16*(2-9),"edx"),$T0);
788 &pshufd ($T0,$T1,0b01000100);
789 &pshufd ($T1,$T1,0b11101110);
790 &movdqa (&QWP(16*3,"edx"),$T0);
791 &movdqu ($T0,&QWP(16*4,"edi"));
792 &movdqa (&QWP(16*(3-9),"edx"),$T1);
793 &pshufd ($T1,$T0,0b01000100);
794 &pshufd ($T0,$T0,0b11101110);
795 &movdqa (&QWP(16*4,"edx"),$T1);
796 &movdqu ($T1,&QWP(16*5,"edi"));
797 &movdqa (&QWP(16*(4-9),"edx"),$T0);
798 &pshufd ($T0,$T1,0b01000100);
799 &pshufd ($T1,$T1,0b11101110);
800 &movdqa (&QWP(16*5,"edx"),$T0);
801 &movdqu ($T0,&QWP(16*6,"edi"));
802 &movdqa (&QWP(16*(5-9),"edx"),$T1);
803 &pshufd ($T1,$T0,0b01000100);
804 &pshufd ($T0,$T0,0b11101110);
805 &movdqa (&QWP(16*6,"edx"),$T1);
806 &movdqu ($T1,&QWP(16*7,"edi"));
807 &movdqa (&QWP(16*(6-9),"edx"),$T0);
808 &pshufd ($T0,$T1,0b01000100);
809 &pshufd ($T1,$T1,0b11101110);
810 &movdqa (&QWP(16*7,"edx"),$T0);
811 &movdqu ($T0,&QWP(16*8,"edi"));
812 &movdqa (&QWP(16*(7-9),"edx"),$T1);
813 &pshufd ($T1,$T0,0b01000100);
814 &pshufd ($T0,$T0,0b11101110);
815 &movdqa (&QWP(16*8,"edx"),$T1);
816 &movdqa (&QWP(16*(8-9),"edx"),$T0);
817
818 sub load_input {
819 my ($inpbase,$offbase)=@_;
820
821 &movdqu ($T0,&QWP($inpbase+0,"esi")); # load input
822 &movdqu ($T1,&QWP($inpbase+16,"esi"));
823 &lea ("esi",&DWP(16*2,"esi"));
824
825 &movdqa (&QWP($offbase+16*2,"esp"),$D2);
826 &movdqa (&QWP($offbase+16*3,"esp"),$D3);
827 &movdqa (&QWP($offbase+16*4,"esp"),$D4);
828
829 &movdqa ($D2,$T0); # splat input
830 &movdqa ($D3,$T1);
831 &psrldq ($D2,6);
832 &psrldq ($D3,6);
833 &movdqa ($D4,$T0);
834 &punpcklqdq ($D2,$D3); # 2:3
835 &punpckhqdq ($D4,$T1); # 4
836 &punpcklqdq ($T0,$T1); # 0:1
837
838 &movdqa ($D3,$D2);
839 &psrlq ($D2,4);
840 &psrlq ($D3,30);
841 &movdqa ($T1,$T0);
842 &psrlq ($D4,40); # 4
843 &psrlq ($T1,26);
844 &pand ($T0,$MASK); # 0
845 &pand ($T1,$MASK); # 1
846 &pand ($D2,$MASK); # 2
847 &pand ($D3,$MASK); # 3
848 &por ($D4,&QWP(0,"ebx")); # padbit, yes, always
849
850 &movdqa (&QWP($offbase+16*0,"esp"),$D0) if ($offbase);
851 &movdqa (&QWP($offbase+16*1,"esp"),$D1) if ($offbase);
852 }
853 &load_input (16*2,16*5);
854
855 &jbe (&label("skip_loop"));
856 &jmp (&label("loop"));
857
858 &set_label("loop",32);
859 ################################################################
860 # ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2
861 # ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r
862 # \___________________/
863 # ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2+inp[8])*r^2
864 # ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^4+inp[7]*r^2+inp[9])*r
865 # \___________________/ \____________________/
866 ################################################################
867
868 &movdqa ($T2,&QWP(16*(0-9),"edx")); # r0^2
869 &movdqa (&QWP(16*1,"eax"),$T1);
870 &movdqa (&QWP(16*2,"eax"),$D2);
871 &movdqa (&QWP(16*3,"eax"),$D3);
872 &movdqa (&QWP(16*4,"eax"),$D4);
873
874 ################################################################
875 # d4 = h4*r0 + h0*r4 + h1*r3 + h2*r2 + h3*r1
876 # d3 = h3*r0 + h0*r3 + h1*r2 + h2*r1 + h4*5*r4
877 # d2 = h2*r0 + h0*r2 + h1*r1 + h3*5*r4 + h4*5*r3
878 # d1 = h1*r0 + h0*r1 + h2*5*r4 + h3*5*r3 + h4*5*r2
879 # d0 = h0*r0 + h1*5*r4 + h2*5*r3 + h3*5*r2 + h4*5*r1
880
881 &movdqa ($D1,$T0);
882 &pmuludq ($T0,$T2); # h0*r0
883 &movdqa ($D0,$T1);
884 &pmuludq ($T1,$T2); # h1*r0
885 &pmuludq ($D2,$T2); # h2*r0
886 &pmuludq ($D3,$T2); # h3*r0
887 &pmuludq ($D4,$T2); # h4*r0
888
889 sub pmuladd_alt {
890 my $addr = shift;
891
892 &pmuludq ($D0,&$addr(8)); # h1*s4
893 &movdqa ($T2,$D1);
894 &pmuludq ($D1,&$addr(1)); # h0*r1
895 &paddq ($D0,$T0);
896 &movdqa ($T0,$T2);
897 &pmuludq ($T2,&$addr(2)); # h0*r2
898 &paddq ($D1,$T1);
899 &movdqa ($T1,$T0);
900 &pmuludq ($T0,&$addr(3)); # h0*r3
901 &paddq ($D2,$T2);
902 &movdqa ($T2,&QWP(16*1,"eax")); # pull h1
903 &pmuludq ($T1,&$addr(4)); # h0*r4
904 &paddq ($D3,$T0);
905
906 &movdqa ($T0,$T2);
907 &pmuludq ($T2,&$addr(1)); # h1*r1
908 &paddq ($D4,$T1);
909 &movdqa ($T1,$T0);
910 &pmuludq ($T0,&$addr(2)); # h1*r2
911 &paddq ($D2,$T2);
912 &movdqa ($T2,&QWP(16*2,"eax")); # pull h2
913 &pmuludq ($T1,&$addr(3)); # h1*r3
914 &paddq ($D3,$T0);
915 &movdqa ($T0,$T2);
916 &pmuludq ($T2,&$addr(7)); # h2*s3
917 &paddq ($D4,$T1);
918 &movdqa ($T1,$T0);
919 &pmuludq ($T0,&$addr(8)); # h2*s4
920 &paddq ($D0,$T2);
921
922 &movdqa ($T2,$T1);
923 &pmuludq ($T1,&$addr(1)); # h2*r1
924 &paddq ($D1,$T0);
925 &movdqa ($T0,&QWP(16*3,"eax")); # pull h3
926 &pmuludq ($T2,&$addr(2)); # h2*r2
927 &paddq ($D3,$T1);
928 &movdqa ($T1,$T0);
929 &pmuludq ($T0,&$addr(6)); # h3*s2
930 &paddq ($D4,$T2);
931 &movdqa ($T2,$T1);
932 &pmuludq ($T1,&$addr(7)); # h3*s3
933 &paddq ($D0,$T0);
934 &movdqa ($T0,$T2);
935 &pmuludq ($T2,&$addr(8)); # h3*s4
936 &paddq ($D1,$T1);
937
938 &movdqa ($T1,&QWP(16*4,"eax")); # pull h4
939 &pmuludq ($T0,&$addr(1)); # h3*r1
940 &paddq ($D2,$T2);
941 &movdqa ($T2,$T1);
942 &pmuludq ($T1,&$addr(8)); # h4*s4
943 &paddq ($D4,$T0);
944 &movdqa ($T0,$T2);
945 &pmuludq ($T2,&$addr(5)); # h4*s1
946 &paddq ($D3,$T1);
947 &movdqa ($T1,$T0);
948 &pmuludq ($T0,&$addr(6)); # h4*s2
949 &paddq ($D0,$T2);
950 &movdqa ($MASK,&QWP(64,"ebx"));
951 &pmuludq ($T1,&$addr(7)); # h4*s3
952 &paddq ($D1,$T0);
953 &paddq ($D2,$T1);
954 }
955 &pmuladd_alt (sub { my $i=shift; &QWP(16*($i-9),"edx"); });
956
957 &load_input (-16*2,0);
958 &lea ("eax",&DWP(-16*2,"esi"));
959 &sub ("ecx",64);
960
961 &paddd ($T0,&QWP(16*(5+0),"esp")); # add hash value
962 &paddd ($T1,&QWP(16*(5+1),"esp"));
963 &paddd ($D2,&QWP(16*(5+2),"esp"));
964 &paddd ($D3,&QWP(16*(5+3),"esp"));
965 &paddd ($D4,&QWP(16*(5+4),"esp"));
966
967 &cmovb ("esi","eax");
968 &lea ("eax",&DWP(16*10,"esp"));
969
970 &movdqa ($T2,&QWP(16*0,"edx")); # r0^4
971 &movdqa (&QWP(16*1,"esp"),$D1);
972 &movdqa (&QWP(16*1,"eax"),$T1);
973 &movdqa (&QWP(16*2,"eax"),$D2);
974 &movdqa (&QWP(16*3,"eax"),$D3);
975 &movdqa (&QWP(16*4,"eax"),$D4);
976
977 ################################################################
978 # d4 += h4*r0 + h0*r4 + h1*r3 + h2*r2 + h3*r1
979 # d3 += h3*r0 + h0*r3 + h1*r2 + h2*r1 + h4*5*r4
980 # d2 += h2*r0 + h0*r2 + h1*r1 + h3*5*r4 + h4*5*r3
981 # d1 += h1*r0 + h0*r1 + h2*5*r4 + h3*5*r3 + h4*5*r2
982 # d0 += h0*r0 + h1*5*r4 + h2*5*r3 + h3*5*r2 + h4*5*r1
983
984 &movdqa ($D1,$T0);
985 &pmuludq ($T0,$T2); # h0*r0
986 &paddq ($T0,$D0);
987 &movdqa ($D0,$T1);
988 &pmuludq ($T1,$T2); # h1*r0
989 &pmuludq ($D2,$T2); # h2*r0
990 &pmuludq ($D3,$T2); # h3*r0
991 &pmuludq ($D4,$T2); # h4*r0
992
993 &paddq ($T1,&QWP(16*1,"esp"));
994 &paddq ($D2,&QWP(16*2,"esp"));
995 &paddq ($D3,&QWP(16*3,"esp"));
996 &paddq ($D4,&QWP(16*4,"esp"));
997
998 &pmuladd_alt (sub { my $i=shift; &QWP(16*$i,"edx"); });
999
1000 &lazy_reduction ();
1001
1002 &load_input (16*2,16*5);
1003
1004 &ja (&label("loop"));
1005
1006 &set_label("skip_loop");
1007 ################################################################
1008 # multiply (inp[0:1]+hash) or inp[2:3] by r^2:r^1
1009
1010 &pshufd ($T2,&QWP(16*(0-9),"edx"),0x10);# r0^n
1011 &add ("ecx",32);
1012 &jnz (&label("long_tail"));
1013
1014 &paddd ($T0,$D0); # add hash value
1015 &paddd ($T1,$D1);
1016 &paddd ($D2,&QWP(16*7,"esp"));
1017 &paddd ($D3,&QWP(16*8,"esp"));
1018 &paddd ($D4,&QWP(16*9,"esp"));
1019
1020 &set_label("long_tail");
1021
1022 &movdqa (&QWP(16*0,"eax"),$T0);
1023 &movdqa (&QWP(16*1,"eax"),$T1);
1024 &movdqa (&QWP(16*2,"eax"),$D2);
1025 &movdqa (&QWP(16*3,"eax"),$D3);
1026 &movdqa (&QWP(16*4,"eax"),$D4);
1027
1028 ################################################################
1029 # d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4
1030 # d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4
1031 # d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4
1032 # d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4
1033 # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
1034
1035 &pmuludq ($T0,$T2); # h0*r0
1036 &pmuludq ($T1,$T2); # h1*r0
1037 &pmuludq ($D2,$T2); # h2*r0
1038 &movdqa ($D0,$T0);
1039 &pshufd ($T0,&QWP(16*(1-9),"edx"),0x10);# r1^n
1040 &pmuludq ($D3,$T2); # h3*r0
1041 &movdqa ($D1,$T1);
1042 &pmuludq ($D4,$T2); # h4*r0
1043
1044 &pmuladd (sub { my ($reg,$i)=@_;
1045 &pshufd ($reg,&QWP(16*($i-9),"edx"),0x10);
1046 },"eax");
1047
1048 &jz (&label("short_tail"));
1049
1050 &load_input (-16*2,0);
1051
1052 &pshufd ($T2,&QWP(16*0,"edx"),0x10); # r0^n
1053 &paddd ($T0,&QWP(16*5,"esp")); # add hash value
1054 &paddd ($T1,&QWP(16*6,"esp"));
1055 &paddd ($D2,&QWP(16*7,"esp"));
1056 &paddd ($D3,&QWP(16*8,"esp"));
1057 &paddd ($D4,&QWP(16*9,"esp"));
1058
1059 ################################################################
1060 # multiply inp[0:1] by r^4:r^3 and accumulate
1061
1062 &movdqa (&QWP(16*0,"esp"),$T0);
1063 &pmuludq ($T0,$T2); # h0*r0
1064 &movdqa (&QWP(16*1,"esp"),$T1);
1065 &pmuludq ($T1,$T2); # h1*r0
1066 &paddq ($D0,$T0);
1067 &movdqa ($T0,$D2);
1068 &pmuludq ($D2,$T2); # h2*r0
1069 &paddq ($D1,$T1);
1070 &movdqa ($T1,$D3);
1071 &pmuludq ($D3,$T2); # h3*r0
1072 &paddq ($D2,&QWP(16*2,"esp"));
1073 &movdqa (&QWP(16*2,"esp"),$T0);
1074 &pshufd ($T0,&QWP(16*1,"edx"),0x10); # r1^n
1075 &paddq ($D3,&QWP(16*3,"esp"));
1076 &movdqa (&QWP(16*3,"esp"),$T1);
1077 &movdqa ($T1,$D4);
1078 &pmuludq ($D4,$T2); # h4*r0
1079 &paddq ($D4,&QWP(16*4,"esp"));
1080 &movdqa (&QWP(16*4,"esp"),$T1);
1081
1082 &pmuladd (sub { my ($reg,$i)=@_;
1083 &pshufd ($reg,&QWP(16*$i,"edx"),0x10);
1084 });
1085
1086 &set_label("short_tail");
1087
1088 ################################################################
1089 # horizontal addition
1090
1091 &pshufd ($T1,$D4,0b01001110);
1092 &pshufd ($T0,$D3,0b01001110);
1093 &paddq ($D4,$T1);
1094 &paddq ($D3,$T0);
1095 &pshufd ($T1,$D0,0b01001110);
1096 &pshufd ($T0,$D1,0b01001110);
1097 &paddq ($D0,$T1);
1098 &paddq ($D1,$T0);
1099 &pshufd ($T1,$D2,0b01001110);
1100 #&paddq ($D2,$T1);
1101
1102 &lazy_reduction (sub { &paddq ($D2,$T1) });
1103
1104 &set_label("done");
1105 &movd (&DWP(-16*3+4*0,"edi"),$D0); # store hash value
1106 &movd (&DWP(-16*3+4*1,"edi"),$D1);
1107 &movd (&DWP(-16*3+4*2,"edi"),$D2);
1108 &movd (&DWP(-16*3+4*3,"edi"),$D3);
1109 &movd (&DWP(-16*3+4*4,"edi"),$D4);
1110 &mov ("esp","ebp");
1111 &set_label("nodata");
1112 &function_end("_poly1305_blocks_sse2");
1113
1114 &align (32);
1115 &function_begin("_poly1305_emit_sse2");
1116 &mov ("ebp",&wparam(0)); # context
1117
1118 &cmp (&DWP(4*5,"ebp"),0); # is_base2_26?
1119 &je (&label("enter_emit"));
1120
1121 &mov ("eax",&DWP(4*0,"ebp")); # load hash value
1122 &mov ("edi",&DWP(4*1,"ebp"));
1123 &mov ("ecx",&DWP(4*2,"ebp"));
1124 &mov ("edx",&DWP(4*3,"ebp"));
1125 &mov ("esi",&DWP(4*4,"ebp"));
1126
1127 &mov ("ebx","edi"); # base 2^26 -> base 2^32
1128 &shl ("edi",26);
1129 &shr ("ebx",6);
1130 &add ("eax","edi");
1131 &mov ("edi","ecx");
1132 &adc ("ebx",0);
1133
1134 &shl ("edi",20);
1135 &shr ("ecx",12);
1136 &add ("ebx","edi");
1137 &mov ("edi","edx");
1138 &adc ("ecx",0);
1139
1140 &shl ("edi",14);
1141 &shr ("edx",18);
1142 &add ("ecx","edi");
1143 &mov ("edi","esi");
1144 &adc ("edx",0);
1145
1146 &shl ("edi",8);
1147 &shr ("esi",24);
1148 &add ("edx","edi");
1149 &adc ("esi",0); # can be partially reduced
1150
1151 &mov ("edi","esi"); # final reduction
1152 &and ("esi",3);
1153 &shr ("edi",2);
1154 &lea ("ebp",&DWP(0,"edi","edi",4)); # *5
1155 &mov ("edi",&wparam(1)); # output
1156 add ("eax","ebp");
1157 &mov ("ebp",&wparam(2)); # key
1158 adc ("ebx",0);
1159 adc ("ecx",0);
1160 adc ("edx",0);
1161
1162 &movd ($D0,"eax"); # offload original hash value
1163 &add ("eax",5); # compare to modulus
1164 &movd ($D1,"ebx");
1165 &adc ("ebx",0);
1166 &movd ($D2,"ecx");
1167 &adc ("ecx",0);
1168 &movd ($D3,"edx");
1169 &adc ("edx",0);
1170 &adc ("esi",0);
1171 &shr ("esi",2); # did it carry/borrow?
1172
1173 &neg ("esi"); # do we choose (hash-modulus) ...
1174 &and ("eax","esi");
1175 &and ("ebx","esi");
1176 &and ("ecx","esi");
1177 &and ("edx","esi");
1178 &mov (&DWP(4*0,"edi"),"eax");
1179 &movd ("eax",$D0);
1180 &mov (&DWP(4*1,"edi"),"ebx");
1181 &movd ("ebx",$D1);
1182 &mov (&DWP(4*2,"edi"),"ecx");
1183 &movd ("ecx",$D2);
1184 &mov (&DWP(4*3,"edi"),"edx");
1185 &movd ("edx",$D3);
1186
1187 &not ("esi"); # ... or original hash value?
1188 &and ("eax","esi");
1189 &and ("ebx","esi");
1190 &or ("eax",&DWP(4*0,"edi"));
1191 &and ("ecx","esi");
1192 &or ("ebx",&DWP(4*1,"edi"));
1193 &and ("edx","esi");
1194 &or ("ecx",&DWP(4*2,"edi"));
1195 &or ("edx",&DWP(4*3,"edi"));
1196
1197 &add ("eax",&DWP(4*0,"ebp")); # accumulate key
1198 &adc ("ebx",&DWP(4*1,"ebp"));
1199 &mov (&DWP(4*0,"edi"),"eax");
1200 &adc ("ecx",&DWP(4*2,"ebp"));
1201 &mov (&DWP(4*1,"edi"),"ebx");
1202 &adc ("edx",&DWP(4*3,"ebp"));
1203 &mov (&DWP(4*2,"edi"),"ecx");
1204 &mov (&DWP(4*3,"edi"),"edx");
1205 &function_end("_poly1305_emit_sse2");
1206
1207 if ($avx>1) {
1208 ########################################################################
1209 # Note that poly1305_init_avx2 operates on %xmm, I could have used
1210 # poly1305_init_sse2...
1211
1212 &align (32);
1213 &function_begin_B("_poly1305_init_avx2");
1214 &vmovdqu ($D4,&QWP(4*6,"edi")); # key base 2^32
1215 &lea ("edi",&DWP(16*3,"edi")); # size optimization
1216 &mov ("ebp","esp");
1217 &sub ("esp",16*(9+5));
1218 &and ("esp",-16);
1219
1220 #&vpand ($D4,$D4,&QWP(96,"ebx")); # magic mask
1221 &vmovdqa ($MASK,&QWP(64,"ebx"));
1222
1223 &vpand ($D0,$D4,$MASK); # -> base 2^26
1224 &vpsrlq ($D1,$D4,26);
1225 &vpsrldq ($D3,$D4,6);
1226 &vpand ($D1,$D1,$MASK);
1227 &vpsrlq ($D2,$D3,4)
1228 &vpsrlq ($D3,$D3,30);
1229 &vpand ($D2,$D2,$MASK);
1230 &vpand ($D3,$D3,$MASK);
1231 &vpsrldq ($D4,$D4,13);
1232
1233 &lea ("edx",&DWP(16*9,"esp")); # size optimization
1234 &mov ("ecx",2);
1235 &set_label("square");
1236 &vmovdqa (&QWP(16*0,"esp"),$D0);
1237 &vmovdqa (&QWP(16*1,"esp"),$D1);
1238 &vmovdqa (&QWP(16*2,"esp"),$D2);
1239 &vmovdqa (&QWP(16*3,"esp"),$D3);
1240 &vmovdqa (&QWP(16*4,"esp"),$D4);
1241
1242 &vpslld ($T1,$D1,2);
1243 &vpslld ($T0,$D2,2);
1244 &vpaddd ($T1,$T1,$D1); # *5
1245 &vpaddd ($T0,$T0,$D2); # *5
1246 &vmovdqa (&QWP(16*5,"esp"),$T1);
1247 &vmovdqa (&QWP(16*6,"esp"),$T0);
1248 &vpslld ($T1,$D3,2);
1249 &vpslld ($T0,$D4,2);
1250 &vpaddd ($T1,$T1,$D3); # *5
1251 &vpaddd ($T0,$T0,$D4); # *5
1252 &vmovdqa (&QWP(16*7,"esp"),$T1);
1253 &vmovdqa (&QWP(16*8,"esp"),$T0);
1254
1255 &vpshufd ($T0,$D0,0b01000100);
1256 &vmovdqa ($T1,$D1);
1257 &vpshufd ($D1,$D1,0b01000100);
1258 &vpshufd ($D2,$D2,0b01000100);
1259 &vpshufd ($D3,$D3,0b01000100);
1260 &vpshufd ($D4,$D4,0b01000100);
1261 &vmovdqa (&QWP(16*0,"edx"),$T0);
1262 &vmovdqa (&QWP(16*1,"edx"),$D1);
1263 &vmovdqa (&QWP(16*2,"edx"),$D2);
1264 &vmovdqa (&QWP(16*3,"edx"),$D3);
1265 &vmovdqa (&QWP(16*4,"edx"),$D4);
1266
1267 ################################################################
1268 # d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4
1269 # d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4
1270 # d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4
1271 # d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4
1272 # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
1273
1274 &vpmuludq ($D4,$D4,$D0); # h4*r0
1275 &vpmuludq ($D3,$D3,$D0); # h3*r0
1276 &vpmuludq ($D2,$D2,$D0); # h2*r0
1277 &vpmuludq ($D1,$D1,$D0); # h1*r0
1278 &vpmuludq ($D0,$T0,$D0); # h0*r0
1279
1280 &vpmuludq ($T0,$T1,&QWP(16*3,"edx")); # r1*h3
1281 &vpaddq ($D4,$D4,$T0);
1282 &vpmuludq ($T2,$T1,&QWP(16*2,"edx")); # r1*h2
1283 &vpaddq ($D3,$D3,$T2);
1284 &vpmuludq ($T0,$T1,&QWP(16*1,"edx")); # r1*h1
1285 &vpaddq ($D2,$D2,$T0);
1286 &vmovdqa ($T2,&QWP(16*5,"esp")); # s1
1287 &vpmuludq ($T1,$T1,&QWP(16*0,"edx")); # r1*h0
1288 &vpaddq ($D1,$D1,$T1);
1289 &vmovdqa ($T0,&QWP(16*2,"esp")); # r2
1290 &vpmuludq ($T2,$T2,&QWP(16*4,"edx")); # s1*h4
1291 &vpaddq ($D0,$D0,$T2);
1292
1293 &vpmuludq ($T1,$T0,&QWP(16*2,"edx")); # r2*h2
1294 &vpaddq ($D4,$D4,$T1);
1295 &vpmuludq ($T2,$T0,&QWP(16*1,"edx")); # r2*h1
1296 &vpaddq ($D3,$D3,$T2);
1297 &vmovdqa ($T1,&QWP(16*6,"esp")); # s2
1298 &vpmuludq ($T0,$T0,&QWP(16*0,"edx")); # r2*h0
1299 &vpaddq ($D2,$D2,$T0);
1300 &vpmuludq ($T2,$T1,&QWP(16*4,"edx")); # s2*h4
1301 &vpaddq ($D1,$D1,$T2);
1302 &vmovdqa ($T0,&QWP(16*3,"esp")); # r3
1303 &vpmuludq ($T1,$T1,&QWP(16*3,"edx")); # s2*h3
1304 &vpaddq ($D0,$D0,$T1);
1305
1306 &vpmuludq ($T2,$T0,&QWP(16*1,"edx")); # r3*h1
1307 &vpaddq ($D4,$D4,$T2);
1308 &vmovdqa ($T1,&QWP(16*7,"esp")); # s3
1309 &vpmuludq ($T0,$T0,&QWP(16*0,"edx")); # r3*h0
1310 &vpaddq ($D3,$D3,$T0);
1311 &vpmuludq ($T2,$T1,&QWP(16*4,"edx")); # s3*h4
1312 &vpaddq ($D2,$D2,$T2);
1313 &vpmuludq ($T0,$T1,&QWP(16*3,"edx")); # s3*h3
1314 &vpaddq ($D1,$D1,$T0);
1315 &vmovdqa ($T2,&QWP(16*4,"esp")); # r4
1316 &vpmuludq ($T1,$T1,&QWP(16*2,"edx")); # s3*h2
1317 &vpaddq ($D0,$D0,$T1);
1318
1319 &vmovdqa ($T0,&QWP(16*8,"esp")); # s4
1320 &vpmuludq ($T2,$T2,&QWP(16*0,"edx")); # r4*h0
1321 &vpaddq ($D4,$D4,$T2);
1322 &vpmuludq ($T1,$T0,&QWP(16*4,"edx")); # s4*h4
1323 &vpaddq ($D3,$D3,$T1);
1324 &vpmuludq ($T2,$T0,&QWP(16*1,"edx")); # s4*h1
1325 &vpaddq ($D0,$D0,$T2);
1326 &vpmuludq ($T1,$T0,&QWP(16*2,"edx")); # s4*h2
1327 &vpaddq ($D1,$D1,$T1);
1328 &vmovdqa ($MASK,&QWP(64,"ebx"));
1329 &vpmuludq ($T0,$T0,&QWP(16*3,"edx")); # s4*h3
1330 &vpaddq ($D2,$D2,$T0);
1331
1332 ################################################################
1333 # lazy reduction
1334 &vpsrlq ($T0,$D3,26);
1335 &vpand ($D3,$D3,$MASK);
1336 &vpsrlq ($T1,$D0,26);
1337 &vpand ($D0,$D0,$MASK);
1338 &vpaddq ($D4,$D4,$T0); # h3 -> h4
1339 &vpaddq ($D1,$D1,$T1); # h0 -> h1
1340 &vpsrlq ($T0,$D4,26);
1341 &vpand ($D4,$D4,$MASK);
1342 &vpsrlq ($T1,$D1,26);
1343 &vpand ($D1,$D1,$MASK);
1344 &vpaddq ($D2,$D2,$T1); # h1 -> h2
1345 &vpaddd ($D0,$D0,$T0);
1346 &vpsllq ($T0,$T0,2);
1347 &vpsrlq ($T1,$D2,26);
1348 &vpand ($D2,$D2,$MASK);
1349 &vpaddd ($D0,$D0,$T0); # h4 -> h0
1350 &vpaddd ($D3,$D3,$T1); # h2 -> h3
1351 &vpsrlq ($T1,$D3,26);
1352 &vpsrlq ($T0,$D0,26);
1353 &vpand ($D0,$D0,$MASK);
1354 &vpand ($D3,$D3,$MASK);
1355 &vpaddd ($D1,$D1,$T0); # h0 -> h1
1356 &vpaddd ($D4,$D4,$T1); # h3 -> h4
1357
1358 &dec ("ecx");
1359 &jz (&label("square_break"));
1360
1361 &vpunpcklqdq ($D0,$D0,&QWP(16*0,"esp")); # 0:r^1:0:r^2
1362 &vpunpcklqdq ($D1,$D1,&QWP(16*1,"esp"));
1363 &vpunpcklqdq ($D2,$D2,&QWP(16*2,"esp"));
1364 &vpunpcklqdq ($D3,$D3,&QWP(16*3,"esp"));
1365 &vpunpcklqdq ($D4,$D4,&QWP(16*4,"esp"));
1366 &jmp (&label("square"));
1367
1368 &set_label("square_break");
1369 &vpsllq ($D0,$D0,32); # -> r^3:0:r^4:0
1370 &vpsllq ($D1,$D1,32);
1371 &vpsllq ($D2,$D2,32);
1372 &vpsllq ($D3,$D3,32);
1373 &vpsllq ($D4,$D4,32);
1374 &vpor ($D0,$D0,&QWP(16*0,"esp")); # r^3:r^1:r^4:r^2
1375 &vpor ($D1,$D1,&QWP(16*1,"esp"));
1376 &vpor ($D2,$D2,&QWP(16*2,"esp"));
1377 &vpor ($D3,$D3,&QWP(16*3,"esp"));
1378 &vpor ($D4,$D4,&QWP(16*4,"esp"));
1379
1380 &vpshufd ($D0,$D0,0b10001101); # -> r^1:r^2:r^3:r^4
1381 &vpshufd ($D1,$D1,0b10001101);
1382 &vpshufd ($D2,$D2,0b10001101);
1383 &vpshufd ($D3,$D3,0b10001101);
1384 &vpshufd ($D4,$D4,0b10001101);
1385
1386 &vmovdqu (&QWP(16*0,"edi"),$D0); # save the table
1387 &vmovdqu (&QWP(16*1,"edi"),$D1);
1388 &vmovdqu (&QWP(16*2,"edi"),$D2);
1389 &vmovdqu (&QWP(16*3,"edi"),$D3);
1390 &vmovdqu (&QWP(16*4,"edi"),$D4);
1391
1392 &vpslld ($T1,$D1,2);
1393 &vpslld ($T0,$D2,2);
1394 &vpaddd ($T1,$T1,$D1); # *5
1395 &vpaddd ($T0,$T0,$D2); # *5
1396 &vmovdqu (&QWP(16*5,"edi"),$T1);
1397 &vmovdqu (&QWP(16*6,"edi"),$T0);
1398 &vpslld ($T1,$D3,2);
1399 &vpslld ($T0,$D4,2);
1400 &vpaddd ($T1,$T1,$D3); # *5
1401 &vpaddd ($T0,$T0,$D4); # *5
1402 &vmovdqu (&QWP(16*7,"edi"),$T1);
1403 &vmovdqu (&QWP(16*8,"edi"),$T0);
1404
1405 &mov ("esp","ebp");
1406 &lea ("edi",&DWP(-16*3,"edi")); # size de-optimization
1407 &ret ();
1408 &function_end_B("_poly1305_init_avx2");
1409
1410 ########################################################################
1411 # now it's time to switch to %ymm
1412
1413 my ($D0,$D1,$D2,$D3,$D4,$T0,$T1,$T2)=map("ymm$_",(0..7));
1414 my $MASK=$T2;
1415
1416 sub X { my $reg=shift; $reg=~s/^ymm/xmm/; $reg; }
1417
1418 &align (32);
1419 &function_begin("_poly1305_blocks_avx2");
1420 &mov ("edi",&wparam(0)); # ctx
1421 &mov ("esi",&wparam(1)); # inp
1422 &mov ("ecx",&wparam(2)); # len
1423
1424 &mov ("eax",&DWP(4*5,"edi")); # is_base2_26
1425 &and ("ecx",-16);
1426 &jz (&label("nodata"));
1427 &cmp ("ecx",64);
1428 &jae (&label("enter_avx2"));
1429 &test ("eax","eax"); # is_base2_26?
1430 &jz (&label("enter_blocks"));
1431
1432 &set_label("enter_avx2");
1433 &vzeroupper ();
1434
1435 &call (&label("pic_point"));
1436 &set_label("pic_point");
1437 &blindpop("ebx");
1438 &lea ("ebx",&DWP(&label("const_sse2")."-".&label("pic_point"),"ebx"));
1439
1440 &test ("eax","eax"); # is_base2_26?
1441 &jnz (&label("base2_26"));
1442
1443 &call ("_poly1305_init_avx2");
1444
1445 ################################################# base 2^32 -> base 2^26
1446 &mov ("eax",&DWP(0,"edi"));
1447 &mov ("ecx",&DWP(3,"edi"));
1448 &mov ("edx",&DWP(6,"edi"));
1449 &mov ("esi",&DWP(9,"edi"));
1450 &mov ("ebp",&DWP(13,"edi"));
1451
1452 &shr ("ecx",2);
1453 &and ("eax",0x3ffffff);
1454 &shr ("edx",4);
1455 &and ("ecx",0x3ffffff);
1456 &shr ("esi",6);
1457 &and ("edx",0x3ffffff);
1458
1459 &mov (&DWP(4*0,"edi"),"eax");
1460 &mov (&DWP(4*1,"edi"),"ecx");
1461 &mov (&DWP(4*2,"edi"),"edx");
1462 &mov (&DWP(4*3,"edi"),"esi");
1463 &mov (&DWP(4*4,"edi"),"ebp");
1464 &mov (&DWP(4*5,"edi"),1); # is_base2_26
1465
1466 &mov ("esi",&wparam(1)); # [reload] inp
1467 &mov ("ecx",&wparam(2)); # [reload] len
1468
1469 &set_label("base2_26");
1470 &mov ("eax",&wparam(3)); # padbit
1471 &mov ("ebp","esp");
1472
1473 &sub ("esp",32*(5+9));
1474 &and ("esp",-512); # ensure that frame
1475 # doesn't cross page
1476 # boundary, which is
1477 # essential for
1478 # misaligned 32-byte
1479 # loads
1480
1481 ################################################################
1482 # expand and copy pre-calculated table to stack
1483
1484 &vmovdqu (&X($D0),&QWP(16*(3+0),"edi"));
1485 &lea ("edx",&DWP(32*5+128,"esp")); # +128 size optimization
1486 &vmovdqu (&X($D1),&QWP(16*(3+1),"edi"));
1487 &vmovdqu (&X($D2),&QWP(16*(3+2),"edi"));
1488 &vmovdqu (&X($D3),&QWP(16*(3+3),"edi"));
1489 &vmovdqu (&X($D4),&QWP(16*(3+4),"edi"));
1490 &lea ("edi",&DWP(16*3,"edi")); # size optimization
1491 &vpermq ($D0,$D0,0b01000000); # 00001234 -> 12343434
1492 &vpermq ($D1,$D1,0b01000000);
1493 &vpermq ($D2,$D2,0b01000000);
1494 &vpermq ($D3,$D3,0b01000000);
1495 &vpermq ($D4,$D4,0b01000000);
1496 &vpshufd ($D0,$D0,0b11001000); # 12343434 -> 14243444
1497 &vpshufd ($D1,$D1,0b11001000);
1498 &vpshufd ($D2,$D2,0b11001000);
1499 &vpshufd ($D3,$D3,0b11001000);
1500 &vpshufd ($D4,$D4,0b11001000);
1501 &vmovdqa (&QWP(32*0-128,"edx"),$D0);
1502 &vmovdqu (&X($D0),&QWP(16*5,"edi"));
1503 &vmovdqa (&QWP(32*1-128,"edx"),$D1);
1504 &vmovdqu (&X($D1),&QWP(16*6,"edi"));
1505 &vmovdqa (&QWP(32*2-128,"edx"),$D2);
1506 &vmovdqu (&X($D2),&QWP(16*7,"edi"));
1507 &vmovdqa (&QWP(32*3-128,"edx"),$D3);
1508 &vmovdqu (&X($D3),&QWP(16*8,"edi"));
1509 &vmovdqa (&QWP(32*4-128,"edx"),$D4);
1510 &vpermq ($D0,$D0,0b01000000);
1511 &vpermq ($D1,$D1,0b01000000);
1512 &vpermq ($D2,$D2,0b01000000);
1513 &vpermq ($D3,$D3,0b01000000);
1514 &vpshufd ($D0,$D0,0b11001000);
1515 &vpshufd ($D1,$D1,0b11001000);
1516 &vpshufd ($D2,$D2,0b11001000);
1517 &vpshufd ($D3,$D3,0b11001000);
1518 &vmovdqa (&QWP(32*5-128,"edx"),$D0);
1519 &vmovd (&X($D0),&DWP(-16*3+4*0,"edi"));# load hash value
1520 &vmovdqa (&QWP(32*6-128,"edx"),$D1);
1521 &vmovd (&X($D1),&DWP(-16*3+4*1,"edi"));
1522 &vmovdqa (&QWP(32*7-128,"edx"),$D2);
1523 &vmovd (&X($D2),&DWP(-16*3+4*2,"edi"));
1524 &vmovdqa (&QWP(32*8-128,"edx"),$D3);
1525 &vmovd (&X($D3),&DWP(-16*3+4*3,"edi"));
1526 &vmovd (&X($D4),&DWP(-16*3+4*4,"edi"));
1527 &vmovdqa ($MASK,&QWP(64,"ebx"));
1528 &neg ("eax"); # padbit
1529
1530 &test ("ecx",63);
1531 &jz (&label("even"));
1532
1533 &mov ("edx","ecx");
1534 &and ("ecx",-64);
1535 &and ("edx",63);
1536
1537 &vmovdqu (&X($T0),&QWP(16*0,"esi"));
1538 &cmp ("edx",32);
1539 &jb (&label("one"));
1540
1541 &vmovdqu (&X($T1),&QWP(16*1,"esi"));
1542 &je (&label("two"));
1543
1544 &vinserti128 ($T0,$T0,&QWP(16*2,"esi"),1);
1545 &lea ("esi",&DWP(16*3,"esi"));
1546 &lea ("ebx",&DWP(8,"ebx")); # three padbits
1547 &lea ("edx",&DWP(32*5+128+8,"esp")); # --:r^1:r^2:r^3 (*)
1548 &jmp (&label("tail"));
1549
1550 &set_label("two");
1551 &lea ("esi",&DWP(16*2,"esi"));
1552 &lea ("ebx",&DWP(16,"ebx")); # two padbits
1553 &lea ("edx",&DWP(32*5+128+16,"esp"));# --:--:r^1:r^2 (*)
1554 &jmp (&label("tail"));
1555
1556 &set_label("one");
1557 &lea ("esi",&DWP(16*1,"esi"));
1558 &vpxor ($T1,$T1,$T1);
1559 &lea ("ebx",&DWP(32,"ebx","eax",8)); # one or no padbits
1560 &lea ("edx",&DWP(32*5+128+24,"esp"));# --:--:--:r^1 (*)
1561 &jmp (&label("tail"));
1562
1563 # (*) spots marked with '--' are data from next table entry, but they
1564 # are multiplied by 0 and therefore rendered insignificant
1565
1566 &set_label("even",32);
1567 &vmovdqu (&X($T0),&QWP(16*0,"esi")); # load input
1568 &vmovdqu (&X($T1),&QWP(16*1,"esi"));
1569 &vinserti128 ($T0,$T0,&QWP(16*2,"esi"),1);
1570 &vinserti128 ($T1,$T1,&QWP(16*3,"esi"),1);
1571 &lea ("esi",&DWP(16*4,"esi"));
1572 &sub ("ecx",64);
1573 &jz (&label("tail"));
1574
1575 &set_label("loop");
1576 ################################################################
1577 # ((inp[0]*r^4+r[4])*r^4+r[8])*r^4
1578 # ((inp[1]*r^4+r[5])*r^4+r[9])*r^3
1579 # ((inp[2]*r^4+r[6])*r^4+r[10])*r^2
1580 # ((inp[3]*r^4+r[7])*r^4+r[11])*r^1
1581 # \________/ \_______/
1582 ################################################################
1583
1584 sub vsplat_input {
1585 &vmovdqa (&QWP(32*2,"esp"),$D2);
1586 &vpsrldq ($D2,$T0,6); # splat input
1587 &vmovdqa (&QWP(32*0,"esp"),$D0);
1588 &vpsrldq ($D0,$T1,6);
1589 &vmovdqa (&QWP(32*1,"esp"),$D1);
1590 &vpunpckhqdq ($D1,$T0,$T1); # 4
1591 &vpunpcklqdq ($T0,$T0,$T1); # 0:1
1592 &vpunpcklqdq ($D2,$D2,$D0); # 2:3
1593
1594 &vpsrlq ($D0,$D2,30);
1595 &vpsrlq ($D2,$D2,4);
1596 &vpsrlq ($T1,$T0,26);
1597 &vpsrlq ($D1,$D1,40); # 4
1598 &vpand ($D2,$D2,$MASK); # 2
1599 &vpand ($T0,$T0,$MASK); # 0
1600 &vpand ($T1,$T1,$MASK); # 1
1601 &vpand ($D0,$D0,$MASK); # 3 (*)
1602 &vpor ($D1,$D1,&QWP(0,"ebx")); # padbit, yes, always
1603
1604 # (*) note that output is counterintuitive, inp[3:4] is
1605 # returned in $D1-2, while $D3-4 are preserved;
1606 }
1607 &vsplat_input ();
1608
1609 sub vpmuladd {
1610 my $addr = shift;
1611
1612 &vpaddq ($D2,$D2,&QWP(32*2,"esp")); # add hash value
1613 &vpaddq ($T0,$T0,&QWP(32*0,"esp"));
1614 &vpaddq ($T1,$T1,&QWP(32*1,"esp"));
1615 &vpaddq ($D0,$D0,$D3);
1616 &vpaddq ($D1,$D1,$D4);
1617
1618 ################################################################
1619 # d3 = h2*r1 + h0*r3 + h1*r2 + h3*r0 + h4*5*r4
1620 # d4 = h2*r2 + h0*r4 + h1*r3 + h3*r1 + h4*r0
1621 # d0 = h2*5*r3 + h0*r0 + h1*5*r4 + h3*5*r2 + h4*5*r1
1622 # d1 = h2*5*r4 + h0*r1 + h1*r0 + h3*5*r3 + h4*5*r2
1623 # d2 = h2*r0 + h0*r2 + h1*r1 + h3*5*r4 + h4*5*r3
1624
1625 &vpmuludq ($D3,$D2,&$addr(1)); # d3 = h2*r1
1626 &vmovdqa (QWP(32*1,"esp"),$T1);
1627 &vpmuludq ($D4,$D2,&$addr(2)); # d4 = h2*r2
1628 &vmovdqa (QWP(32*3,"esp"),$D0);
1629 &vpmuludq ($D0,$D2,&$addr(7)); # d0 = h2*s3
1630 &vmovdqa (QWP(32*4,"esp"),$D1);
1631 &vpmuludq ($D1,$D2,&$addr(8)); # d1 = h2*s4
1632 &vpmuludq ($D2,$D2,&$addr(0)); # d2 = h2*r0
1633
1634 &vpmuludq ($T2,$T0,&$addr(3)); # h0*r3
1635 &vpaddq ($D3,$D3,$T2); # d3 += h0*r3
1636 &vpmuludq ($T1,$T0,&$addr(4)); # h0*r4
1637 &vpaddq ($D4,$D4,$T1); # d4 + h0*r4
1638 &vpmuludq ($T2,$T0,&$addr(0)); # h0*r0
1639 &vpaddq ($D0,$D0,$T2); # d0 + h0*r0
1640 &vmovdqa ($T2,&QWP(32*1,"esp")); # h1
1641 &vpmuludq ($T1,$T0,&$addr(1)); # h0*r1
1642 &vpaddq ($D1,$D1,$T1); # d1 += h0*r1
1643 &vpmuludq ($T0,$T0,&$addr(2)); # h0*r2
1644 &vpaddq ($D2,$D2,$T0); # d2 += h0*r2
1645
1646 &vpmuludq ($T1,$T2,&$addr(2)); # h1*r2
1647 &vpaddq ($D3,$D3,$T1); # d3 += h1*r2
1648 &vpmuludq ($T0,$T2,&$addr(3)); # h1*r3
1649 &vpaddq ($D4,$D4,$T0); # d4 += h1*r3
1650 &vpmuludq ($T1,$T2,&$addr(8)); # h1*s4
1651 &vpaddq ($D0,$D0,$T1); # d0 += h1*s4
1652 &vmovdqa ($T1,&QWP(32*3,"esp")); # h3
1653 &vpmuludq ($T0,$T2,&$addr(0)); # h1*r0
1654 &vpaddq ($D1,$D1,$T0); # d1 += h1*r0
1655 &vpmuludq ($T2,$T2,&$addr(1)); # h1*r1
1656 &vpaddq ($D2,$D2,$T2); # d2 += h1*r1
1657
1658 &vpmuludq ($T0,$T1,&$addr(0)); # h3*r0
1659 &vpaddq ($D3,$D3,$T0); # d3 += h3*r0
1660 &vpmuludq ($T2,$T1,&$addr(1)); # h3*r1
1661 &vpaddq ($D4,$D4,$T2); # d4 += h3*r1
1662 &vpmuludq ($T0,$T1,&$addr(6)); # h3*s2
1663 &vpaddq ($D0,$D0,$T0); # d0 += h3*s2
1664 &vmovdqa ($T0,&QWP(32*4,"esp")); # h4
1665 &vpmuludq ($T2,$T1,&$addr(7)); # h3*s3
1666 &vpaddq ($D1,$D1,$T2); # d1+= h3*s3
1667 &vpmuludq ($T1,$T1,&$addr(8)); # h3*s4
1668 &vpaddq ($D2,$D2,$T1); # d2 += h3*s4
1669
1670 &vpmuludq ($T2,$T0,&$addr(8)); # h4*s4
1671 &vpaddq ($D3,$D3,$T2); # d3 += h4*s4
1672 &vpmuludq ($T1,$T0,&$addr(5)); # h4*s1
1673 &vpaddq ($D0,$D0,$T1); # d0 += h4*s1
1674 &vpmuludq ($T2,$T0,&$addr(0)); # h4*r0
1675 &vpaddq ($D4,$D4,$T2); # d4 += h4*r0
1676 &vmovdqa ($MASK,&QWP(64,"ebx"));
1677 &vpmuludq ($T1,$T0,&$addr(6)); # h4*s2
1678 &vpaddq ($D1,$D1,$T1); # d1 += h4*s2
1679 &vpmuludq ($T0,$T0,&$addr(7)); # h4*s3
1680 &vpaddq ($D2,$D2,$T0); # d2 += h4*s3
1681 }
1682 &vpmuladd (sub { my $i=shift; &QWP(32*$i-128,"edx"); });
1683
1684 sub vlazy_reduction {
1685 ################################################################
1686 # lazy reduction
1687
1688 &vpsrlq ($T0,$D3,26);
1689 &vpand ($D3,$D3,$MASK);
1690 &vpsrlq ($T1,$D0,26);
1691 &vpand ($D0,$D0,$MASK);
1692 &vpaddq ($D4,$D4,$T0); # h3 -> h4
1693 &vpaddq ($D1,$D1,$T1); # h0 -> h1
1694 &vpsrlq ($T0,$D4,26);
1695 &vpand ($D4,$D4,$MASK);
1696 &vpsrlq ($T1,$D1,26);
1697 &vpand ($D1,$D1,$MASK);
1698 &vpaddq ($D2,$D2,$T1); # h1 -> h2
1699 &vpaddq ($D0,$D0,$T0);
1700 &vpsllq ($T0,$T0,2);
1701 &vpsrlq ($T1,$D2,26);
1702 &vpand ($D2,$D2,$MASK);
1703 &vpaddq ($D0,$D0,$T0); # h4 -> h0
1704 &vpaddq ($D3,$D3,$T1); # h2 -> h3
1705 &vpsrlq ($T1,$D3,26);
1706 &vpsrlq ($T0,$D0,26);
1707 &vpand ($D0,$D0,$MASK);
1708 &vpand ($D3,$D3,$MASK);
1709 &vpaddq ($D1,$D1,$T0); # h0 -> h1
1710 &vpaddq ($D4,$D4,$T1); # h3 -> h4
1711 }
1712 &vlazy_reduction();
1713
1714 &vmovdqu (&X($T0),&QWP(16*0,"esi")); # load input
1715 &vmovdqu (&X($T1),&QWP(16*1,"esi"));
1716 &vinserti128 ($T0,$T0,&QWP(16*2,"esi"),1);
1717 &vinserti128 ($T1,$T1,&QWP(16*3,"esi"),1);
1718 &lea ("esi",&DWP(16*4,"esi"));
1719 &sub ("ecx",64);
1720 &jnz (&label("loop"));
1721
1722 &set_label("tail");
1723 &vsplat_input ();
1724 &and ("ebx",-64); # restore pointer
1725
1726 &vpmuladd (sub { my $i=shift; &QWP(4+32*$i-128,"edx"); });
1727
1728 ################################################################
1729 # horizontal addition
1730
1731 &vpsrldq ($T0,$D4,8);
1732 &vpsrldq ($T1,$D3,8);
1733 &vpaddq ($D4,$D4,$T0);
1734 &vpsrldq ($T0,$D0,8);
1735 &vpaddq ($D3,$D3,$T1);
1736 &vpsrldq ($T1,$D1,8);
1737 &vpaddq ($D0,$D0,$T0);
1738 &vpsrldq ($T0,$D2,8);
1739 &vpaddq ($D1,$D1,$T1);
1740 &vpermq ($T1,$D4,2); # keep folding
1741 &vpaddq ($D2,$D2,$T0);
1742 &vpermq ($T0,$D3,2);
1743 &vpaddq ($D4,$D4,$T1);
1744 &vpermq ($T1,$D0,2);
1745 &vpaddq ($D3,$D3,$T0);
1746 &vpermq ($T0,$D1,2);
1747 &vpaddq ($D0,$D0,$T1);
1748 &vpermq ($T1,$D2,2);
1749 &vpaddq ($D1,$D1,$T0);
1750 &vpaddq ($D2,$D2,$T1);
1751
1752 &vlazy_reduction();
1753
1754 &cmp ("ecx",0);
1755 &je (&label("done"));
1756
1757 ################################################################
1758 # clear all but single word
1759
1760 &vpshufd (&X($D0),&X($D0),0b11111100);
1761 &lea ("edx",&DWP(32*5+128,"esp")); # restore pointer
1762 &vpshufd (&X($D1),&X($D1),0b11111100);
1763 &vpshufd (&X($D2),&X($D2),0b11111100);
1764 &vpshufd (&X($D3),&X($D3),0b11111100);
1765 &vpshufd (&X($D4),&X($D4),0b11111100);
1766 &jmp (&label("even"));
1767
1768 &set_label("done",16);
1769 &vmovd (&DWP(-16*3+4*0,"edi"),&X($D0));# store hash value
1770 &vmovd (&DWP(-16*3+4*1,"edi"),&X($D1));
1771 &vmovd (&DWP(-16*3+4*2,"edi"),&X($D2));
1772 &vmovd (&DWP(-16*3+4*3,"edi"),&X($D3));
1773 &vmovd (&DWP(-16*3+4*4,"edi"),&X($D4));
1774 &vzeroupper ();
1775 &mov ("esp","ebp");
1776 &set_label("nodata");
1777 &function_end("_poly1305_blocks_avx2");
1778 }
1779 &set_label("const_sse2",64);
1780 &data_word(1<<24,0, 1<<24,0, 1<<24,0, 1<<24,0);
1781 &data_word(0,0, 0,0, 0,0, 0,0);
1782 &data_word(0x03ffffff,0,0x03ffffff,0, 0x03ffffff,0, 0x03ffffff,0);
1783 &data_word(0x0fffffff,0x0ffffffc,0x0ffffffc,0x0ffffffc);
1784 }
1785 &asciz ("Poly1305 for x86, CRYPTOGAMS by <appro\@openssl.org>");
1786 &align (4);
1787
1788 &asm_finish();