Debian Codegraph @

external/boringssl/crypto/chacha/asm/chacha-x86.pl

raw
1 #!/usr/bin/env perl
2 #
3 # ====================================================================
4 # Written by Andy Polyakov <[email protected]> for the OpenSSL
5 # project. The module is, however, dual licensed under OpenSSL and
6 # CRYPTOGAMS licenses depending on where you obtain it. For further
7 # details see http://www.openssl.org/~appro/cryptogams/.
8 # ====================================================================
9 #
10 # January 2015
11 #
12 # ChaCha20 for x86.
13 #
14 # Performance in cycles per byte out of large buffer.
15 #
16 # 1xIALU/gcc 4xSSSE3
17 # Pentium 17.5/+80%
18 # PIII 14.2/+60%
19 # P4 18.6/+84%
20 # Core2 9.56/+89% 4.83
21 # Westmere 9.50/+45% 3.35
22 # Sandy Bridge 10.5/+47% 3.20
23 # Haswell 8.15/+50% 2.83
24 # Silvermont 17.4/+36% 8.35
25 # Sledgehammer 10.2/+54%
26 # Bulldozer 13.4/+50% 4.38(*)
27 #
28 # (*) Bulldozer actually executes 4xXOP code path that delivers 3.55;
29 #
30 # Modified from upstream OpenSSL to remove the XOP code.
31
32 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
33 push(@INC,"${dir}","${dir}../../perlasm");
34 require "x86asm.pl";
35
36 &asm_init($ARGV[0],"chacha-x86.pl",$ARGV[$#ARGV] eq "386");
37
38 $xmm=$ymm=0;
39 for (@ARGV) { $xmm=1 if (/-DOPENSSL_IA32_SSE2/); }
40
41 $ymm=$xmm;
42
43 $a="eax";
44 ($b,$b_)=("ebx","ebp");
45 ($c,$c_)=("ecx","esi");
46 ($d,$d_)=("edx","edi");
47
48 sub QUARTERROUND {
49 my ($ai,$bi,$ci,$di,$i)=@_;
50 my ($an,$bn,$cn,$dn)=map(($_&~3)+(($_+1)&3),($ai,$bi,$ci,$di)); # next
51 my ($ap,$bp,$cp,$dp)=map(($_&~3)+(($_-1)&3),($ai,$bi,$ci,$di)); # previous
52
53 # a b c d
54 #
55 # 0 4 8 12 < even round
56 # 1 5 9 13
57 # 2 6 10 14
58 # 3 7 11 15
59 # 0 5 10 15 < odd round
60 # 1 6 11 12
61 # 2 7 8 13
62 # 3 4 9 14
63
64 if ($i==0) {
65 my $j=4;
66 ($ap,$bp,$cp,$dp)=map(($_&~3)+(($_-$j--)&3),($ap,$bp,$cp,$dp));
67 } elsif ($i==3) {
68 my $j=0;
69 ($an,$bn,$cn,$dn)=map(($_&~3)+(($_+$j++)&3),($an,$bn,$cn,$dn));
70 } elsif ($i==4) {
71 my $j=4;
72 ($ap,$bp,$cp,$dp)=map(($_&~3)+(($_+$j--)&3),($ap,$bp,$cp,$dp));
73 } elsif ($i==7) {
74 my $j=0;
75 ($an,$bn,$cn,$dn)=map(($_&~3)+(($_-$j++)&3),($an,$bn,$cn,$dn));
76 }
77
78 #&add ($a,$b); # see elsewhere
79 &xor ($d,$a);
80 &mov (&DWP(4*$cp,"esp"),$c_) if ($ai>0 && $ai<3);
81 &rol ($d,16);
82 &mov (&DWP(4*$bp,"esp"),$b_) if ($i!=0);
83 &add ($c,$d);
84 &mov ($c_,&DWP(4*$cn,"esp")) if ($ai>0 && $ai<3);
85 &xor ($b,$c);
86 &mov ($d_,&DWP(4*$dn,"esp")) if ($di!=$dn);
87 &rol ($b,12);
88 &mov ($b_,&DWP(4*$bn,"esp")) if ($i<7);
89 &mov ($b_,&DWP(128,"esp")) if ($i==7); # loop counter
90 &add ($a,$b);
91 &xor ($d,$a);
92 &mov (&DWP(4*$ai,"esp"),$a);
93 &rol ($d,8);
94 &mov ($a,&DWP(4*$an,"esp"));
95 &add ($c,$d);
96 &mov (&DWP(4*$di,"esp"),$d) if ($di!=$dn);
97 &mov ($d_,$d) if ($di==$dn);
98 &xor ($b,$c);
99 &add ($a,$b_) if ($i<7); # elsewhere
100 &rol ($b,7);
101
102 ($b,$b_)=($b_,$b);
103 ($c,$c_)=($c_,$c);
104 ($d,$d_)=($d_,$d);
105 }
106
107 &static_label("ssse3_shortcut");
108 &static_label("ssse3_data");
109 &static_label("pic_point");
110
111 &function_begin("ChaCha20_ctr32");
112 &xor ("eax","eax");
113 &cmp ("eax",&wparam(2)); # len==0?
114 &je (&label("no_data"));
115 if ($xmm) {
116 &call (&label("pic_point"));
117 &set_label("pic_point");
118 &blindpop("eax");
119 &picmeup("ebp","OPENSSL_ia32cap_P","eax",&label("pic_point"));
120 &test (&DWP(0,"ebp"),1<<24); # test FXSR bit
121 &jz (&label("x86"));
122 &test (&DWP(4,"ebp"),1<<9); # test SSSE3 bit
123 &jz (&label("x86"));
124 &jmp (&label("ssse3_shortcut"));
125 &set_label("x86");
126 }
127 &mov ("esi",&wparam(3)); # key
128 &mov ("edi",&wparam(4)); # counter and nonce
129
130 &stack_push(33);
131
132 &mov ("eax",&DWP(4*0,"esi")); # copy key
133 &mov ("ebx",&DWP(4*1,"esi"));
134 &mov ("ecx",&DWP(4*2,"esi"));
135 &mov ("edx",&DWP(4*3,"esi"));
136 &mov (&DWP(64+4*4,"esp"),"eax");
137 &mov (&DWP(64+4*5,"esp"),"ebx");
138 &mov (&DWP(64+4*6,"esp"),"ecx");
139 &mov (&DWP(64+4*7,"esp"),"edx");
140 &mov ("eax",&DWP(4*4,"esi"));
141 &mov ("ebx",&DWP(4*5,"esi"));
142 &mov ("ecx",&DWP(4*6,"esi"));
143 &mov ("edx",&DWP(4*7,"esi"));
144 &mov (&DWP(64+4*8,"esp"),"eax");
145 &mov (&DWP(64+4*9,"esp"),"ebx");
146 &mov (&DWP(64+4*10,"esp"),"ecx");
147 &mov (&DWP(64+4*11,"esp"),"edx");
148 &mov ("eax",&DWP(4*0,"edi")); # copy counter and nonce
149 &mov ("ebx",&DWP(4*1,"edi"));
150 &mov ("ecx",&DWP(4*2,"edi"));
151 &mov ("edx",&DWP(4*3,"edi"));
152 &sub ("eax",1);
153 &mov (&DWP(64+4*12,"esp"),"eax");
154 &mov (&DWP(64+4*13,"esp"),"ebx");
155 &mov (&DWP(64+4*14,"esp"),"ecx");
156 &mov (&DWP(64+4*15,"esp"),"edx");
157 &jmp (&label("entry"));
158
159 &set_label("outer_loop",16);
160 &mov (&wparam(1),$b); # save input
161 &mov (&wparam(0),$a); # save output
162 &mov (&wparam(2),$c); # save len
163 &set_label("entry");
164 &mov ($a,0x61707865);
165 &mov (&DWP(4*1,"esp"),0x3320646e);
166 &mov (&DWP(4*2,"esp"),0x79622d32);
167 &mov (&DWP(4*3,"esp"),0x6b206574);
168
169 &mov ($b, &DWP(64+4*5,"esp")); # copy key material
170 &mov ($b_,&DWP(64+4*6,"esp"));
171 &mov ($c, &DWP(64+4*10,"esp"));
172 &mov ($c_,&DWP(64+4*11,"esp"));
173 &mov ($d, &DWP(64+4*13,"esp"));
174 &mov ($d_,&DWP(64+4*14,"esp"));
175 &mov (&DWP(4*5,"esp"),$b);
176 &mov (&DWP(4*6,"esp"),$b_);
177 &mov (&DWP(4*10,"esp"),$c);
178 &mov (&DWP(4*11,"esp"),$c_);
179 &mov (&DWP(4*13,"esp"),$d);
180 &mov (&DWP(4*14,"esp"),$d_);
181
182 &mov ($b, &DWP(64+4*7,"esp"));
183 &mov ($d_,&DWP(64+4*15,"esp"));
184 &mov ($d, &DWP(64+4*12,"esp"));
185 &mov ($b_,&DWP(64+4*4,"esp"));
186 &mov ($c, &DWP(64+4*8,"esp"));
187 &mov ($c_,&DWP(64+4*9,"esp"));
188 &add ($d,1); # counter value
189 &mov (&DWP(4*7,"esp"),$b);
190 &mov (&DWP(4*15,"esp"),$d_);
191 &mov (&DWP(64+4*12,"esp"),$d); # save counter value
192
193 &mov ($b,10); # loop counter
194 &jmp (&label("loop"));
195
196 &set_label("loop",16);
197 &add ($a,$b_); # elsewhere
198 &mov (&DWP(128,"esp"),$b); # save loop counter
199 &mov ($b,$b_);
200 &QUARTERROUND(0, 4, 8, 12, 0);
201 &QUARTERROUND(1, 5, 9, 13, 1);
202 &QUARTERROUND(2, 6,10, 14, 2);
203 &QUARTERROUND(3, 7,11, 15, 3);
204 &QUARTERROUND(0, 5,10, 15, 4);
205 &QUARTERROUND(1, 6,11, 12, 5);
206 &QUARTERROUND(2, 7, 8, 13, 6);
207 &QUARTERROUND(3, 4, 9, 14, 7);
208 &dec ($b);
209 &jnz (&label("loop"));
210
211 &mov ($b,&wparam(2)); # load len
212
213 &add ($a,0x61707865); # accumulate key material
214 &add ($b_,&DWP(64+4*4,"esp"));
215 &add ($c, &DWP(64+4*8,"esp"));
216 &add ($c_,&DWP(64+4*9,"esp"));
217
218 &cmp ($b,64);
219 &jb (&label("tail"));
220
221 &mov ($b,&wparam(1)); # load input pointer
222 &add ($d, &DWP(64+4*12,"esp"));
223 &add ($d_,&DWP(64+4*14,"esp"));
224
225 &xor ($a, &DWP(4*0,$b)); # xor with input
226 &xor ($b_,&DWP(4*4,$b));
227 &mov (&DWP(4*0,"esp"),$a);
228 &mov ($a,&wparam(0)); # load output pointer
229 &xor ($c, &DWP(4*8,$b));
230 &xor ($c_,&DWP(4*9,$b));
231 &xor ($d, &DWP(4*12,$b));
232 &xor ($d_,&DWP(4*14,$b));
233 &mov (&DWP(4*4,$a),$b_); # write output
234 &mov (&DWP(4*8,$a),$c);
235 &mov (&DWP(4*9,$a),$c_);
236 &mov (&DWP(4*12,$a),$d);
237 &mov (&DWP(4*14,$a),$d_);
238
239 &mov ($b_,&DWP(4*1,"esp"));
240 &mov ($c, &DWP(4*2,"esp"));
241 &mov ($c_,&DWP(4*3,"esp"));
242 &mov ($d, &DWP(4*5,"esp"));
243 &mov ($d_,&DWP(4*6,"esp"));
244 &add ($b_,0x3320646e); # accumulate key material
245 &add ($c, 0x79622d32);
246 &add ($c_,0x6b206574);
247 &add ($d, &DWP(64+4*5,"esp"));
248 &add ($d_,&DWP(64+4*6,"esp"));
249 &xor ($b_,&DWP(4*1,$b));
250 &xor ($c, &DWP(4*2,$b));
251 &xor ($c_,&DWP(4*3,$b));
252 &xor ($d, &DWP(4*5,$b));
253 &xor ($d_,&DWP(4*6,$b));
254 &mov (&DWP(4*1,$a),$b_);
255 &mov (&DWP(4*2,$a),$c);
256 &mov (&DWP(4*3,$a),$c_);
257 &mov (&DWP(4*5,$a),$d);
258 &mov (&DWP(4*6,$a),$d_);
259
260 &mov ($b_,&DWP(4*7,"esp"));
261 &mov ($c, &DWP(4*10,"esp"));
262 &mov ($c_,&DWP(4*11,"esp"));
263 &mov ($d, &DWP(4*13,"esp"));
264 &mov ($d_,&DWP(4*15,"esp"));
265 &add ($b_,&DWP(64+4*7,"esp"));
266 &add ($c, &DWP(64+4*10,"esp"));
267 &add ($c_,&DWP(64+4*11,"esp"));
268 &add ($d, &DWP(64+4*13,"esp"));
269 &add ($d_,&DWP(64+4*15,"esp"));
270 &xor ($b_,&DWP(4*7,$b));
271 &xor ($c, &DWP(4*10,$b));
272 &xor ($c_,&DWP(4*11,$b));
273 &xor ($d, &DWP(4*13,$b));
274 &xor ($d_,&DWP(4*15,$b));
275 &lea ($b,&DWP(4*16,$b));
276 &mov (&DWP(4*7,$a),$b_);
277 &mov ($b_,&DWP(4*0,"esp"));
278 &mov (&DWP(4*10,$a),$c);
279 &mov ($c,&wparam(2)); # len
280 &mov (&DWP(4*11,$a),$c_);
281 &mov (&DWP(4*13,$a),$d);
282 &mov (&DWP(4*15,$a),$d_);
283 &mov (&DWP(4*0,$a),$b_);
284 &lea ($a,&DWP(4*16,$a));
285 &sub ($c,64);
286 &jnz (&label("outer_loop"));
287
288 &jmp (&label("done"));
289
290 &set_label("tail");
291 &add ($d, &DWP(64+4*12,"esp"));
292 &add ($d_,&DWP(64+4*14,"esp"));
293 &mov (&DWP(4*0,"esp"),$a);
294 &mov (&DWP(4*4,"esp"),$b_);
295 &mov (&DWP(4*8,"esp"),$c);
296 &mov (&DWP(4*9,"esp"),$c_);
297 &mov (&DWP(4*12,"esp"),$d);
298 &mov (&DWP(4*14,"esp"),$d_);
299
300 &mov ($b_,&DWP(4*1,"esp"));
301 &mov ($c, &DWP(4*2,"esp"));
302 &mov ($c_,&DWP(4*3,"esp"));
303 &mov ($d, &DWP(4*5,"esp"));
304 &mov ($d_,&DWP(4*6,"esp"));
305 &add ($b_,0x3320646e); # accumulate key material
306 &add ($c, 0x79622d32);
307 &add ($c_,0x6b206574);
308 &add ($d, &DWP(64+4*5,"esp"));
309 &add ($d_,&DWP(64+4*6,"esp"));
310 &mov (&DWP(4*1,"esp"),$b_);
311 &mov (&DWP(4*2,"esp"),$c);
312 &mov (&DWP(4*3,"esp"),$c_);
313 &mov (&DWP(4*5,"esp"),$d);
314 &mov (&DWP(4*6,"esp"),$d_);
315
316 &mov ($b_,&DWP(4*7,"esp"));
317 &mov ($c, &DWP(4*10,"esp"));
318 &mov ($c_,&DWP(4*11,"esp"));
319 &mov ($d, &DWP(4*13,"esp"));
320 &mov ($d_,&DWP(4*15,"esp"));
321 &add ($b_,&DWP(64+4*7,"esp"));
322 &add ($c, &DWP(64+4*10,"esp"));
323 &add ($c_,&DWP(64+4*11,"esp"));
324 &add ($d, &DWP(64+4*13,"esp"));
325 &add ($d_,&DWP(64+4*15,"esp"));
326 &mov (&DWP(4*7,"esp"),$b_);
327 &mov ($b_,&wparam(1)); # load input
328 &mov (&DWP(4*10,"esp"),$c);
329 &mov ($c,&wparam(0)); # load output
330 &mov (&DWP(4*11,"esp"),$c_);
331 &xor ($c_,$c_);
332 &mov (&DWP(4*13,"esp"),$d);
333 &mov (&DWP(4*15,"esp"),$d_);
334
335 &xor ("eax","eax");
336 &xor ("edx","edx");
337 &set_label("tail_loop");
338 &movb ("al",&BP(0,$c_,$b_));
339 &movb ("dl",&BP(0,"esp",$c_));
340 &lea ($c_,&DWP(1,$c_));
341 &xor ("al","dl");
342 &mov (&BP(-1,$c,$c_),"al");
343 &dec ($b);
344 &jnz (&label("tail_loop"));
345
346 &set_label("done");
347 &stack_pop(33);
348 &set_label("no_data");
349 &function_end("ChaCha20_ctr32");
350
351 if ($xmm) {
352 my ($xa,$xa_,$xb,$xb_,$xc,$xc_,$xd,$xd_)=map("xmm$_",(0..7));
353 my ($out,$inp,$len)=("edi","esi","ecx");
354
355 sub QUARTERROUND_SSSE3 {
356 my ($ai,$bi,$ci,$di,$i)=@_;
357 my ($an,$bn,$cn,$dn)=map(($_&~3)+(($_+1)&3),($ai,$bi,$ci,$di)); # next
358 my ($ap,$bp,$cp,$dp)=map(($_&~3)+(($_-1)&3),($ai,$bi,$ci,$di)); # previous
359
360 # a b c d
361 #
362 # 0 4 8 12 < even round
363 # 1 5 9 13
364 # 2 6 10 14
365 # 3 7 11 15
366 # 0 5 10 15 < odd round
367 # 1 6 11 12
368 # 2 7 8 13
369 # 3 4 9 14
370
371 if ($i==0) {
372 my $j=4;
373 ($ap,$bp,$cp,$dp)=map(($_&~3)+(($_-$j--)&3),($ap,$bp,$cp,$dp));
374 } elsif ($i==3) {
375 my $j=0;
376 ($an,$bn,$cn,$dn)=map(($_&~3)+(($_+$j++)&3),($an,$bn,$cn,$dn));
377 } elsif ($i==4) {
378 my $j=4;
379 ($ap,$bp,$cp,$dp)=map(($_&~3)+(($_+$j--)&3),($ap,$bp,$cp,$dp));
380 } elsif ($i==7) {
381 my $j=0;
382 ($an,$bn,$cn,$dn)=map(($_&~3)+(($_-$j++)&3),($an,$bn,$cn,$dn));
383 }
384
385 #&paddd ($xa,$xb); # see elsewhere
386 #&pxor ($xd,$xa); # see elsewhere
387 &movdqa(&QWP(16*$cp-128,"ebx"),$xc_) if ($ai>0 && $ai<3);
388 &pshufb ($xd,&QWP(0,"eax")); # rot16
389 &movdqa(&QWP(16*$bp-128,"ebx"),$xb_) if ($i!=0);
390 &paddd ($xc,$xd);
391 &movdqa($xc_,&QWP(16*$cn-128,"ebx")) if ($ai>0 && $ai<3);
392 &pxor ($xb,$xc);
393 &movdqa($xb_,&QWP(16*$bn-128,"ebx")) if ($i<7);
394 &movdqa ($xa_,$xb); # borrow as temporary
395 &pslld ($xb,12);
396 &psrld ($xa_,20);
397 &por ($xb,$xa_);
398 &movdqa($xa_,&QWP(16*$an-128,"ebx"));
399 &paddd ($xa,$xb);
400 &movdqa($xd_,&QWP(16*$dn-128,"ebx")) if ($di!=$dn);
401 &pxor ($xd,$xa);
402 &movdqa (&QWP(16*$ai-128,"ebx"),$xa);
403 &pshufb ($xd,&QWP(16,"eax")); # rot8
404 &paddd ($xc,$xd);
405 &movdqa (&QWP(16*$di-128,"ebx"),$xd) if ($di!=$dn);
406 &movdqa ($xd_,$xd) if ($di==$dn);
407 &pxor ($xb,$xc);
408 &paddd ($xa_,$xb_) if ($i<7); # elsewhere
409 &movdqa ($xa,$xb); # borrow as temporary
410 &pslld ($xb,7);
411 &psrld ($xa,25);
412 &pxor ($xd_,$xa_) if ($i<7); # elsewhere
413 &por ($xb,$xa);
414
415 ($xa,$xa_)=($xa_,$xa);
416 ($xb,$xb_)=($xb_,$xb);
417 ($xc,$xc_)=($xc_,$xc);
418 ($xd,$xd_)=($xd_,$xd);
419 }
420
421 &function_begin("ChaCha20_ssse3");
422 &set_label("ssse3_shortcut");
423 &mov ($out,&wparam(0));
424 &mov ($inp,&wparam(1));
425 &mov ($len,&wparam(2));
426 &mov ("edx",&wparam(3)); # key
427 &mov ("ebx",&wparam(4)); # counter and nonce
428
429 &mov ("ebp","esp");
430 &stack_push (131);
431 &and ("esp",-64);
432 &mov (&DWP(512,"esp"),"ebp");
433
434 &lea ("eax",&DWP(&label("ssse3_data")."-".
435 &label("pic_point"),"eax"));
436 &movdqu ("xmm3",&QWP(0,"ebx")); # counter and nonce
437
438 &cmp ($len,64*4);
439 &jb (&label("1x"));
440
441 &mov (&DWP(512+4,"esp"),"edx"); # offload pointers
442 &mov (&DWP(512+8,"esp"),"ebx");
443 &sub ($len,64*4); # bias len
444 &lea ("ebp",&DWP(256+128,"esp")); # size optimization
445
446 &movdqu ("xmm7",&QWP(0,"edx")); # key
447 &pshufd ("xmm0","xmm3",0x00);
448 &pshufd ("xmm1","xmm3",0x55);
449 &pshufd ("xmm2","xmm3",0xaa);
450 &pshufd ("xmm3","xmm3",0xff);
451 &paddd ("xmm0",&QWP(16*3,"eax")); # fix counters
452 &pshufd ("xmm4","xmm7",0x00);
453 &pshufd ("xmm5","xmm7",0x55);
454 &psubd ("xmm0",&QWP(16*4,"eax"));
455 &pshufd ("xmm6","xmm7",0xaa);
456 &pshufd ("xmm7","xmm7",0xff);
457 &movdqa (&QWP(16*12-128,"ebp"),"xmm0");
458 &movdqa (&QWP(16*13-128,"ebp"),"xmm1");
459 &movdqa (&QWP(16*14-128,"ebp"),"xmm2");
460 &movdqa (&QWP(16*15-128,"ebp"),"xmm3");
461 &movdqu ("xmm3",&QWP(16,"edx")); # key
462 &movdqa (&QWP(16*4-128,"ebp"),"xmm4");
463 &movdqa (&QWP(16*5-128,"ebp"),"xmm5");
464 &movdqa (&QWP(16*6-128,"ebp"),"xmm6");
465 &movdqa (&QWP(16*7-128,"ebp"),"xmm7");
466 &movdqa ("xmm7",&QWP(16*2,"eax")); # sigma
467 &lea ("ebx",&DWP(128,"esp")); # size optimization
468
469 &pshufd ("xmm0","xmm3",0x00);
470 &pshufd ("xmm1","xmm3",0x55);
471 &pshufd ("xmm2","xmm3",0xaa);
472 &pshufd ("xmm3","xmm3",0xff);
473 &pshufd ("xmm4","xmm7",0x00);
474 &pshufd ("xmm5","xmm7",0x55);
475 &pshufd ("xmm6","xmm7",0xaa);
476 &pshufd ("xmm7","xmm7",0xff);
477 &movdqa (&QWP(16*8-128,"ebp"),"xmm0");
478 &movdqa (&QWP(16*9-128,"ebp"),"xmm1");
479 &movdqa (&QWP(16*10-128,"ebp"),"xmm2");
480 &movdqa (&QWP(16*11-128,"ebp"),"xmm3");
481 &movdqa (&QWP(16*0-128,"ebp"),"xmm4");
482 &movdqa (&QWP(16*1-128,"ebp"),"xmm5");
483 &movdqa (&QWP(16*2-128,"ebp"),"xmm6");
484 &movdqa (&QWP(16*3-128,"ebp"),"xmm7");
485
486 &lea ($inp,&DWP(128,$inp)); # size optimization
487 &lea ($out,&DWP(128,$out)); # size optimization
488 &jmp (&label("outer_loop"));
489
490 &set_label("outer_loop",16);
491 #&movdqa ("xmm0",&QWP(16*0-128,"ebp")); # copy key material
492 &movdqa ("xmm1",&QWP(16*1-128,"ebp"));
493 &movdqa ("xmm2",&QWP(16*2-128,"ebp"));
494 &movdqa ("xmm3",&QWP(16*3-128,"ebp"));
495 #&movdqa ("xmm4",&QWP(16*4-128,"ebp"));
496 &movdqa ("xmm5",&QWP(16*5-128,"ebp"));
497 &movdqa ("xmm6",&QWP(16*6-128,"ebp"));
498 &movdqa ("xmm7",&QWP(16*7-128,"ebp"));
499 #&movdqa (&QWP(16*0-128,"ebx"),"xmm0");
500 &movdqa (&QWP(16*1-128,"ebx"),"xmm1");
501 &movdqa (&QWP(16*2-128,"ebx"),"xmm2");
502 &movdqa (&QWP(16*3-128,"ebx"),"xmm3");
503 #&movdqa (&QWP(16*4-128,"ebx"),"xmm4");
504 &movdqa (&QWP(16*5-128,"ebx"),"xmm5");
505 &movdqa (&QWP(16*6-128,"ebx"),"xmm6");
506 &movdqa (&QWP(16*7-128,"ebx"),"xmm7");
507 #&movdqa ("xmm0",&QWP(16*8-128,"ebp"));
508 #&movdqa ("xmm1",&QWP(16*9-128,"ebp"));
509 &movdqa ("xmm2",&QWP(16*10-128,"ebp"));
510 &movdqa ("xmm3",&QWP(16*11-128,"ebp"));
511 &movdqa ("xmm4",&QWP(16*12-128,"ebp"));
512 &movdqa ("xmm5",&QWP(16*13-128,"ebp"));
513 &movdqa ("xmm6",&QWP(16*14-128,"ebp"));
514 &movdqa ("xmm7",&QWP(16*15-128,"ebp"));
515 &paddd ("xmm4",&QWP(16*4,"eax")); # counter value
516 #&movdqa (&QWP(16*8-128,"ebx"),"xmm0");
517 #&movdqa (&QWP(16*9-128,"ebx"),"xmm1");
518 &movdqa (&QWP(16*10-128,"ebx"),"xmm2");
519 &movdqa (&QWP(16*11-128,"ebx"),"xmm3");
520 &movdqa (&QWP(16*12-128,"ebx"),"xmm4");
521 &movdqa (&QWP(16*13-128,"ebx"),"xmm5");
522 &movdqa (&QWP(16*14-128,"ebx"),"xmm6");
523 &movdqa (&QWP(16*15-128,"ebx"),"xmm7");
524 &movdqa (&QWP(16*12-128,"ebp"),"xmm4"); # save counter value
525
526 &movdqa ($xa, &QWP(16*0-128,"ebp"));
527 &movdqa ($xd, "xmm4");
528 &movdqa ($xb_,&QWP(16*4-128,"ebp"));
529 &movdqa ($xc, &QWP(16*8-128,"ebp"));
530 &movdqa ($xc_,&QWP(16*9-128,"ebp"));
531
532 &mov ("edx",10); # loop counter
533 &nop ();
534
535 &set_label("loop",16);
536 &paddd ($xa,$xb_); # elsewhere
537 &movdqa ($xb,$xb_);
538 &pxor ($xd,$xa); # elsewhere
539 &QUARTERROUND_SSSE3(0, 4, 8, 12, 0);
540 &QUARTERROUND_SSSE3(1, 5, 9, 13, 1);
541 &QUARTERROUND_SSSE3(2, 6,10, 14, 2);
542 &QUARTERROUND_SSSE3(3, 7,11, 15, 3);
543 &QUARTERROUND_SSSE3(0, 5,10, 15, 4);
544 &QUARTERROUND_SSSE3(1, 6,11, 12, 5);
545 &QUARTERROUND_SSSE3(2, 7, 8, 13, 6);
546 &QUARTERROUND_SSSE3(3, 4, 9, 14, 7);
547 &dec ("edx");
548 &jnz (&label("loop"));
549
550 &movdqa (&QWP(16*4-128,"ebx"),$xb_);
551 &movdqa (&QWP(16*8-128,"ebx"),$xc);
552 &movdqa (&QWP(16*9-128,"ebx"),$xc_);
553 &movdqa (&QWP(16*12-128,"ebx"),$xd);
554 &movdqa (&QWP(16*14-128,"ebx"),$xd_);
555
556 my ($xa0,$xa1,$xa2,$xa3,$xt0,$xt1,$xt2,$xt3)=map("xmm$_",(0..7));
557
558 #&movdqa ($xa0,&QWP(16*0-128,"ebx")); # it's there
559 &movdqa ($xa1,&QWP(16*1-128,"ebx"));
560 &movdqa ($xa2,&QWP(16*2-128,"ebx"));
561 &movdqa ($xa3,&QWP(16*3-128,"ebx"));
562
563 for($i=0;$i<256;$i+=64) {
564 &paddd ($xa0,&QWP($i+16*0-128,"ebp")); # accumulate key material
565 &paddd ($xa1,&QWP($i+16*1-128,"ebp"));
566 &paddd ($xa2,&QWP($i+16*2-128,"ebp"));
567 &paddd ($xa3,&QWP($i+16*3-128,"ebp"));
568
569 &movdqa ($xt2,$xa0); # "de-interlace" data
570 &punpckldq ($xa0,$xa1);
571 &movdqa ($xt3,$xa2);
572 &punpckldq ($xa2,$xa3);
573 &punpckhdq ($xt2,$xa1);
574 &punpckhdq ($xt3,$xa3);
575 &movdqa ($xa1,$xa0);
576 &punpcklqdq ($xa0,$xa2); # "a0"
577 &movdqa ($xa3,$xt2);
578 &punpcklqdq ($xt2,$xt3); # "a2"
579 &punpckhqdq ($xa1,$xa2); # "a1"
580 &punpckhqdq ($xa3,$xt3); # "a3"
581
582 #($xa2,$xt2)=($xt2,$xa2);
583
584 &movdqu ($xt0,&QWP(64*0-128,$inp)); # load input
585 &movdqu ($xt1,&QWP(64*1-128,$inp));
586 &movdqu ($xa2,&QWP(64*2-128,$inp));
587 &movdqu ($xt3,&QWP(64*3-128,$inp));
588 &lea ($inp,&QWP($i<192?16:(64*4-16*3),$inp));
589 &pxor ($xt0,$xa0);
590 &movdqa ($xa0,&QWP($i+16*4-128,"ebx")) if ($i<192);
591 &pxor ($xt1,$xa1);
592 &movdqa ($xa1,&QWP($i+16*5-128,"ebx")) if ($i<192);
593 &pxor ($xt2,$xa2);
594 &movdqa ($xa2,&QWP($i+16*6-128,"ebx")) if ($i<192);
595 &pxor ($xt3,$xa3);
596 &movdqa ($xa3,&QWP($i+16*7-128,"ebx")) if ($i<192);
597 &movdqu (&QWP(64*0-128,$out),$xt0); # store output
598 &movdqu (&QWP(64*1-128,$out),$xt1);
599 &movdqu (&QWP(64*2-128,$out),$xt2);
600 &movdqu (&QWP(64*3-128,$out),$xt3);
601 &lea ($out,&QWP($i<192?16:(64*4-16*3),$out));
602 }
603 &sub ($len,64*4);
604 &jnc (&label("outer_loop"));
605
606 &add ($len,64*4);
607 &jz (&label("done"));
608
609 &mov ("ebx",&DWP(512+8,"esp")); # restore pointers
610 &lea ($inp,&DWP(-128,$inp));
611 &mov ("edx",&DWP(512+4,"esp"));
612 &lea ($out,&DWP(-128,$out));
613
614 &movd ("xmm2",&DWP(16*12-128,"ebp")); # counter value
615 &movdqu ("xmm3",&QWP(0,"ebx"));
616 &paddd ("xmm2",&QWP(16*6,"eax")); # +four
617 &pand ("xmm3",&QWP(16*7,"eax"));
618 &por ("xmm3","xmm2"); # counter value
619 {
620 my ($a,$b,$c,$d,$t,$t1,$rot16,$rot24)=map("xmm$_",(0..7));
621
622 sub SSSE3ROUND { # critical path is 20 "SIMD ticks" per round
623 &paddd ($a,$b);
624 &pxor ($d,$a);
625 &pshufb ($d,$rot16);
626
627 &paddd ($c,$d);
628 &pxor ($b,$c);
629 &movdqa ($t,$b);
630 &psrld ($b,20);
631 &pslld ($t,12);
632 &por ($b,$t);
633
634 &paddd ($a,$b);
635 &pxor ($d,$a);
636 &pshufb ($d,$rot24);
637
638 &paddd ($c,$d);
639 &pxor ($b,$c);
640 &movdqa ($t,$b);
641 &psrld ($b,25);
642 &pslld ($t,7);
643 &por ($b,$t);
644 }
645
646 &set_label("1x");
647 &movdqa ($a,&QWP(16*2,"eax")); # sigma
648 &movdqu ($b,&QWP(0,"edx"));
649 &movdqu ($c,&QWP(16,"edx"));
650 #&movdqu ($d,&QWP(0,"ebx")); # already loaded
651 &movdqa ($rot16,&QWP(0,"eax"));
652 &movdqa ($rot24,&QWP(16,"eax"));
653 &mov (&DWP(16*3,"esp"),"ebp");
654
655 &movdqa (&QWP(16*0,"esp"),$a);
656 &movdqa (&QWP(16*1,"esp"),$b);
657 &movdqa (&QWP(16*2,"esp"),$c);
658 &movdqa (&QWP(16*3,"esp"),$d);
659 &mov ("edx",10);
660 &jmp (&label("loop1x"));
661
662 &set_label("outer1x",16);
663 &movdqa ($d,&QWP(16*5,"eax")); # one
664 &movdqa ($a,&QWP(16*0,"esp"));
665 &movdqa ($b,&QWP(16*1,"esp"));
666 &movdqa ($c,&QWP(16*2,"esp"));
667 &paddd ($d,&QWP(16*3,"esp"));
668 &mov ("edx",10);
669 &movdqa (&QWP(16*3,"esp"),$d);
670 &jmp (&label("loop1x"));
671
672 &set_label("loop1x",16);
673 &SSSE3ROUND();
674 &pshufd ($c,$c,0b01001110);
675 &pshufd ($b,$b,0b00111001);
676 &pshufd ($d,$d,0b10010011);
677 &nop ();
678
679 &SSSE3ROUND();
680 &pshufd ($c,$c,0b01001110);
681 &pshufd ($b,$b,0b10010011);
682 &pshufd ($d,$d,0b00111001);
683
684 &dec ("edx");
685 &jnz (&label("loop1x"));
686
687 &paddd ($a,&QWP(16*0,"esp"));
688 &paddd ($b,&QWP(16*1,"esp"));
689 &paddd ($c,&QWP(16*2,"esp"));
690 &paddd ($d,&QWP(16*3,"esp"));
691
692 &cmp ($len,64);
693 &jb (&label("tail"));
694
695 &movdqu ($t,&QWP(16*0,$inp));
696 &movdqu ($t1,&QWP(16*1,$inp));
697 &pxor ($a,$t); # xor with input
698 &movdqu ($t,&QWP(16*2,$inp));
699 &pxor ($b,$t1);
700 &movdqu ($t1,&QWP(16*3,$inp));
701 &pxor ($c,$t);
702 &pxor ($d,$t1);
703 &lea ($inp,&DWP(16*4,$inp)); # inp+=64
704
705 &movdqu (&QWP(16*0,$out),$a); # write output
706 &movdqu (&QWP(16*1,$out),$b);
707 &movdqu (&QWP(16*2,$out),$c);
708 &movdqu (&QWP(16*3,$out),$d);
709 &lea ($out,&DWP(16*4,$out)); # inp+=64
710
711 &sub ($len,64);
712 &jnz (&label("outer1x"));
713
714 &jmp (&label("done"));
715
716 &set_label("tail");
717 &movdqa (&QWP(16*0,"esp"),$a);
718 &movdqa (&QWP(16*1,"esp"),$b);
719 &movdqa (&QWP(16*2,"esp"),$c);
720 &movdqa (&QWP(16*3,"esp"),$d);
721
722 &xor ("eax","eax");
723 &xor ("edx","edx");
724 &xor ("ebp","ebp");
725
726 &set_label("tail_loop");
727 &movb ("al",&BP(0,"esp","ebp"));
728 &movb ("dl",&BP(0,$inp,"ebp"));
729 &lea ("ebp",&DWP(1,"ebp"));
730 &xor ("al","dl");
731 &movb (&BP(-1,$out,"ebp"),"al");
732 &dec ($len);
733 &jnz (&label("tail_loop"));
734 }
735 &set_label("done");
736 &mov ("esp",&DWP(512,"esp"));
737 &function_end("ChaCha20_ssse3");
738
739 &align (64);
740 &set_label("ssse3_data");
741 &data_byte(0x2,0x3,0x0,0x1, 0x6,0x7,0x4,0x5, 0xa,0xb,0x8,0x9, 0xe,0xf,0xc,0xd);
742 &data_byte(0x3,0x0,0x1,0x2, 0x7,0x4,0x5,0x6, 0xb,0x8,0x9,0xa, 0xf,0xc,0xd,0xe);
743 &data_word(0x61707865,0x3320646e,0x79622d32,0x6b206574);
744 &data_word(0,1,2,3);
745 &data_word(4,4,4,4);
746 &data_word(1,0,0,0);
747 &data_word(4,0,0,0);
748 &data_word(0,-1,-1,-1);
749 &align (64);
750 }
751 &asciz ("ChaCha20 for x86, CRYPTOGAMS by <appro\@openssl.org>");
752
753 &asm_finish();