Debian Codegraph @

external/boringssl/crypto/sha/asm/sha512-x86_64.pl

raw
1 #!/usr/bin/env perl
2 #
3 # ====================================================================
4 # Written by Andy Polyakov <[email protected]> for the OpenSSL
5 # project. Rights for redistribution and usage in source and binary
6 # forms are granted according to the OpenSSL license.
7 # ====================================================================
8 #
9 # sha256/512_block procedure for x86_64.
10 #
11 # 40% improvement over compiler-generated code on Opteron. On EM64T
12 # sha256 was observed to run >80% faster and sha512 - >40%. No magical
13 # tricks, just straight implementation... I really wonder why gcc
14 # [being armed with inline assembler] fails to generate as fast code.
15 # The only thing which is cool about this module is that it's very
16 # same instruction sequence used for both SHA-256 and SHA-512. In
17 # former case the instructions operate on 32-bit operands, while in
18 # latter - on 64-bit ones. All I had to do is to get one flavor right,
19 # the other one passed the test right away:-)
20 #
21 # sha256_block runs in ~1005 cycles on Opteron, which gives you
22 # asymptotic performance of 64*1000/1005=63.7MBps times CPU clock
23 # frequency in GHz. sha512_block runs in ~1275 cycles, which results
24 # in 128*1000/1275=100MBps per GHz. Is there room for improvement?
25 # Well, if you compare it to IA-64 implementation, which maintains
26 # X[16] in register bank[!], tends to 4 instructions per CPU clock
27 # cycle and runs in 1003 cycles, 1275 is very good result for 3-way
28 # issue Opteron pipeline and X[16] maintained in memory. So that *if*
29 # there is a way to improve it, *then* the only way would be to try to
30 # offload X[16] updates to SSE unit, but that would require "deeper"
31 # loop unroll, which in turn would naturally cause size blow-up, not
32 # to mention increased complexity! And once again, only *if* it's
33 # actually possible to noticeably improve overall ILP, instruction
34 # level parallelism, on a given CPU implementation in this case.
35 #
36 # Special note on Intel EM64T. While Opteron CPU exhibits perfect
37 # perfromance ratio of 1.5 between 64- and 32-bit flavors [see above],
38 # [currently available] EM64T CPUs apparently are far from it. On the
39 # contrary, 64-bit version, sha512_block, is ~30% *slower* than 32-bit
40 # sha256_block:-( This is presumably because 64-bit shifts/rotates
41 # apparently are not atomic instructions, but implemented in microcode.
42 #
43 # May 2012.
44 #
45 # Optimization including one of Pavel Semjanov's ideas, alternative
46 # Maj, resulted in >=5% improvement on most CPUs, +20% SHA256 and
47 # unfortunately -2% SHA512 on P4 [which nobody should care about
48 # that much].
49 #
50 # June 2012.
51 #
52 # Add SIMD code paths, see below for improvement coefficients. SSSE3
53 # code path was not attempted for SHA512, because improvement is not
54 # estimated to be high enough, noticeably less than 9%, to justify
55 # the effort, not on pre-AVX processors. [Obviously with exclusion
56 # for VIA Nano, but it has SHA512 instruction that is faster and
57 # should be used instead.] For reference, corresponding estimated
58 # upper limit for improvement for SSSE3 SHA256 is 28%. The fact that
59 # higher coefficients are observed on VIA Nano and Bulldozer has more
60 # to do with specifics of their architecture [which is topic for
61 # separate discussion].
62 #
63 # November 2012.
64 #
65 # Add AVX2 code path. Two consecutive input blocks are loaded to
66 # 256-bit %ymm registers, with data from first block to least
67 # significant 128-bit halves and data from second to most significant.
68 # The data is then processed with same SIMD instruction sequence as
69 # for AVX, but with %ymm as operands. Side effect is increased stack
70 # frame, 448 additional bytes in SHA256 and 1152 in SHA512, and 1.2KB
71 # code size increase.
72 #
73 # March 2014.
74 #
75 # Add support for Intel SHA Extensions.
76
77 ######################################################################
78 # Current performance in cycles per processed byte (less is better):
79 #
80 # SHA256 SSSE3 AVX/XOP(*) SHA512 AVX/XOP(*)
81 #
82 # AMD K8 14.9 - - 9.57 -
83 # P4 17.3 - - 30.8 -
84 # Core 2 15.6 13.8(+13%) - 9.97 -
85 # Westmere 14.8 12.3(+19%) - 9.58 -
86 # Sandy Bridge 17.4 14.2(+23%) 11.6(+50%(**)) 11.2 8.10(+38%(**))
87 # Ivy Bridge 12.6 10.5(+20%) 10.3(+22%) 8.17 7.22(+13%)
88 # Haswell 12.2 9.28(+31%) 7.80(+56%) 7.66 5.40(+42%)
89 # Bulldozer 21.1 13.6(+54%) 13.6(+54%(***)) 13.5 8.58(+57%)
90 # VIA Nano 23.0 16.5(+39%) - 14.7 -
91 # Atom 23.0 18.9(+22%) - 14.7 -
92 # Silvermont 27.4 20.6(+33%) - 17.5 -
93 #
94 # (*) whichever best applicable;
95 # (**) switch from ror to shrd stands for fair share of improvement;
96 # (***) execution time is fully determined by remaining integer-only
97 # part, body_00_15; reducing the amount of SIMD instructions
98 # below certain limit makes no difference/sense; to conserve
99 # space SHA256 XOP code path is therefore omitted;
100
101 $flavour = shift;
102 $output = shift;
103 if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }
104
105 $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
106
107 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
108 ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
109 ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
110 die "can't locate x86_64-xlate.pl";
111
112 # In upstream, this is controlled by shelling out to the compiler to check
113 # versions, but BoringSSL is intended to be used with pre-generated perlasm
114 # output, so this isn't useful anyway.
115 #
116 # TODO(davidben): Enable AVX2 code after testing by setting $avx to 2. Is it
117 # necessary to disable AVX2 code when SHA Extensions code is disabled? Upstream
118 # did not tie them together until after $shaext was added.
119 $avx = 1;
120
121 # TODO(davidben): Consider enabling the Intel SHA Extensions code once it's
122 # been tested.
123 $shaext=0; ### set to zero if compiling for 1.0.1
124 $avx=1 if (!$shaext && $avx);
125
126 open OUT,"| \"$^X\" $xlate $flavour";
127 *STDOUT=*OUT;
128
129 if ($output =~ /512/) {
130 $func="sha512_block_data_order";
131 $TABLE="K512";
132 $SZ=8;
133 @ROT=($A,$B,$C,$D,$E,$F,$G,$H)=("%rax","%rbx","%rcx","%rdx",
134 "%r8", "%r9", "%r10","%r11");
135 ($T1,$a0,$a1,$a2,$a3)=("%r12","%r13","%r14","%r15","%rdi");
136 @Sigma0=(28,34,39);
137 @Sigma1=(14,18,41);
138 @sigma0=(1, 8, 7);
139 @sigma1=(19,61, 6);
140 $rounds=80;
141 } else {
142 $func="sha256_block_data_order";
143 $TABLE="K256";
144 $SZ=4;
145 @ROT=($A,$B,$C,$D,$E,$F,$G,$H)=("%eax","%ebx","%ecx","%edx",
146 "%r8d","%r9d","%r10d","%r11d");
147 ($T1,$a0,$a1,$a2,$a3)=("%r12d","%r13d","%r14d","%r15d","%edi");
148 @Sigma0=( 2,13,22);
149 @Sigma1=( 6,11,25);
150 @sigma0=( 7,18, 3);
151 @sigma1=(17,19,10);
152 $rounds=64;
153 }
154
155 $ctx="%rdi"; # 1st arg, zapped by $a3
156 $inp="%rsi"; # 2nd arg
157 $Tbl="%rbp";
158
159 $_ctx="16*$SZ+0*8(%rsp)";
160 $_inp="16*$SZ+1*8(%rsp)";
161 $_end="16*$SZ+2*8(%rsp)";
162 $_rsp="16*$SZ+3*8(%rsp)";
163 $framesz="16*$SZ+4*8";
164
165
166 sub ROUND_00_15()
167 { my ($i,$a,$b,$c,$d,$e,$f,$g,$h) = @_;
168 my $STRIDE=$SZ;
169 $STRIDE += 16 if ($i%(16/$SZ)==(16/$SZ-1));
170
171 $code.=<<___;
172 ror \$`$Sigma1[2]-$Sigma1[1]`,$a0
173 mov $f,$a2
174
175 xor $e,$a0
176 ror \$`$Sigma0[2]-$Sigma0[1]`,$a1
177 xor $g,$a2 # f^g
178
179 mov $T1,`$SZ*($i&0xf)`(%rsp)
180 xor $a,$a1
181 and $e,$a2 # (f^g)&e
182
183 ror \$`$Sigma1[1]-$Sigma1[0]`,$a0
184 add $h,$T1 # T1+=h
185 xor $g,$a2 # Ch(e,f,g)=((f^g)&e)^g
186
187 ror \$`$Sigma0[1]-$Sigma0[0]`,$a1
188 xor $e,$a0
189 add $a2,$T1 # T1+=Ch(e,f,g)
190
191 mov $a,$a2
192 add ($Tbl),$T1 # T1+=K[round]
193 xor $a,$a1
194
195 xor $b,$a2 # a^b, b^c in next round
196 ror \$$Sigma1[0],$a0 # Sigma1(e)
197 mov $b,$h
198
199 and $a2,$a3
200 ror \$$Sigma0[0],$a1 # Sigma0(a)
201 add $a0,$T1 # T1+=Sigma1(e)
202
203 xor $a3,$h # h=Maj(a,b,c)=Ch(a^b,c,b)
204 add $T1,$d # d+=T1
205 add $T1,$h # h+=T1
206
207 lea $STRIDE($Tbl),$Tbl # round++
208 ___
209 $code.=<<___ if ($i<15);
210 add $a1,$h # h+=Sigma0(a)
211 ___
212 ($a2,$a3) = ($a3,$a2);
213 }
214
215 sub ROUND_16_XX()
216 { my ($i,$a,$b,$c,$d,$e,$f,$g,$h) = @_;
217
218 $code.=<<___;
219 mov `$SZ*(($i+1)&0xf)`(%rsp),$a0
220 mov `$SZ*(($i+14)&0xf)`(%rsp),$a2
221
222 mov $a0,$T1
223 ror \$`$sigma0[1]-$sigma0[0]`,$a0
224 add $a1,$a # modulo-scheduled h+=Sigma0(a)
225 mov $a2,$a1
226 ror \$`$sigma1[1]-$sigma1[0]`,$a2
227
228 xor $T1,$a0
229 shr \$$sigma0[2],$T1
230 ror \$$sigma0[0],$a0
231 xor $a1,$a2
232 shr \$$sigma1[2],$a1
233
234 ror \$$sigma1[0],$a2
235 xor $a0,$T1 # sigma0(X[(i+1)&0xf])
236 xor $a1,$a2 # sigma1(X[(i+14)&0xf])
237 add `$SZ*(($i+9)&0xf)`(%rsp),$T1
238
239 add `$SZ*($i&0xf)`(%rsp),$T1
240 mov $e,$a0
241 add $a2,$T1
242 mov $a,$a1
243 ___
244 &ROUND_00_15(@_);
245 }
246
247 $code=<<___;
248 .text
249
250 .extern OPENSSL_ia32cap_P
251 .globl $func
252 .type $func,\@function,3
253 .align 16
254 $func:
255 ___
256 $code.=<<___ if ($SZ==4 || $avx);
257 lea OPENSSL_ia32cap_P(%rip),%r11
258 mov 0(%r11),%r9d
259 mov 4(%r11),%r10d
260 mov 8(%r11),%r11d
261 ___
262 $code.=<<___ if ($SZ==4 && $shaext);
263 test \$`1<<29`,%r11d # check for SHA
264 jnz _shaext_shortcut
265 ___
266 $code.=<<___ if ($avx && $SZ==8);
267 test \$`1<<11`,%r10d # check for XOP
268 jnz .Lxop_shortcut
269 ___
270 $code.=<<___ if ($avx>1);
271 and \$`1<<8|1<<5|1<<3`,%r11d # check for BMI2+AVX2+BMI1
272 cmp \$`1<<8|1<<5|1<<3`,%r11d
273 je .Lavx2_shortcut
274 ___
275 $code.=<<___ if ($avx);
276 and \$`1<<30`,%r9d # mask "Intel CPU" bit
277 and \$`1<<28|1<<9`,%r10d # mask AVX and SSSE3 bits
278 or %r9d,%r10d
279 cmp \$`1<<28|1<<9|1<<30`,%r10d
280 je .Lavx_shortcut
281 ___
282 $code.=<<___ if ($SZ==4);
283 test \$`1<<9`,%r10d
284 jnz .Lssse3_shortcut
285 ___
286 $code.=<<___;
287 push %rbx
288 push %rbp
289 push %r12
290 push %r13
291 push %r14
292 push %r15
293 mov %rsp,%r11 # copy %rsp
294 shl \$4,%rdx # num*16
295 sub \$$framesz,%rsp
296 lea ($inp,%rdx,$SZ),%rdx # inp+num*16*$SZ
297 and \$-64,%rsp # align stack frame
298 mov $ctx,$_ctx # save ctx, 1st arg
299 mov $inp,$_inp # save inp, 2nd arh
300 mov %rdx,$_end # save end pointer, "3rd" arg
301 mov %r11,$_rsp # save copy of %rsp
302 .Lprologue:
303
304 mov $SZ*0($ctx),$A
305 mov $SZ*1($ctx),$B
306 mov $SZ*2($ctx),$C
307 mov $SZ*3($ctx),$D
308 mov $SZ*4($ctx),$E
309 mov $SZ*5($ctx),$F
310 mov $SZ*6($ctx),$G
311 mov $SZ*7($ctx),$H
312 jmp .Lloop
313
314 .align 16
315 .Lloop:
316 mov $B,$a3
317 lea $TABLE(%rip),$Tbl
318 xor $C,$a3 # magic
319 ___
320 for($i=0;$i<16;$i++) {
321 $code.=" mov $SZ*$i($inp),$T1\n";
322 $code.=" mov @ROT[4],$a0\n";
323 $code.=" mov @ROT[0],$a1\n";
324 $code.=" bswap $T1\n";
325 &ROUND_00_15($i,@ROT);
326 unshift(@ROT,pop(@ROT));
327 }
328 $code.=<<___;
329 jmp .Lrounds_16_xx
330 .align 16
331 .Lrounds_16_xx:
332 ___
333 for(;$i<32;$i++) {
334 &ROUND_16_XX($i,@ROT);
335 unshift(@ROT,pop(@ROT));
336 }
337
338 $code.=<<___;
339 cmpb \$0,`$SZ-1`($Tbl)
340 jnz .Lrounds_16_xx
341
342 mov $_ctx,$ctx
343 add $a1,$A # modulo-scheduled h+=Sigma0(a)
344 lea 16*$SZ($inp),$inp
345
346 add $SZ*0($ctx),$A
347 add $SZ*1($ctx),$B
348 add $SZ*2($ctx),$C
349 add $SZ*3($ctx),$D
350 add $SZ*4($ctx),$E
351 add $SZ*5($ctx),$F
352 add $SZ*6($ctx),$G
353 add $SZ*7($ctx),$H
354
355 cmp $_end,$inp
356
357 mov $A,$SZ*0($ctx)
358 mov $B,$SZ*1($ctx)
359 mov $C,$SZ*2($ctx)
360 mov $D,$SZ*3($ctx)
361 mov $E,$SZ*4($ctx)
362 mov $F,$SZ*5($ctx)
363 mov $G,$SZ*6($ctx)
364 mov $H,$SZ*7($ctx)
365 jb .Lloop
366
367 mov $_rsp,%rsi
368 mov (%rsi),%r15
369 mov 8(%rsi),%r14
370 mov 16(%rsi),%r13
371 mov 24(%rsi),%r12
372 mov 32(%rsi),%rbp
373 mov 40(%rsi),%rbx
374 lea 48(%rsi),%rsp
375 .Lepilogue:
376 ret
377 .size $func,.-$func
378 ___
379
380 if ($SZ==4) {
381 $code.=<<___;
382 .align 64
383 .type $TABLE,\@object
384 $TABLE:
385 .long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
386 .long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
387 .long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5
388 .long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5
389 .long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3
390 .long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3
391 .long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174
392 .long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174
393 .long 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc
394 .long 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc
395 .long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da
396 .long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da
397 .long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7
398 .long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7
399 .long 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967
400 .long 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967
401 .long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13
402 .long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13
403 .long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85
404 .long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85
405 .long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3
406 .long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3
407 .long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070
408 .long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070
409 .long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5
410 .long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5
411 .long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3
412 .long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3
413 .long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
414 .long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
415 .long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
416 .long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
417
418 .long 0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f
419 .long 0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f
420 .long 0x03020100,0x0b0a0908,0xffffffff,0xffffffff
421 .long 0x03020100,0x0b0a0908,0xffffffff,0xffffffff
422 .long 0xffffffff,0xffffffff,0x03020100,0x0b0a0908
423 .long 0xffffffff,0xffffffff,0x03020100,0x0b0a0908
424 .asciz "SHA256 block transform for x86_64, CRYPTOGAMS by <appro\@openssl.org>"
425 ___
426 } else {
427 $code.=<<___;
428 .align 64
429 .type $TABLE,\@object
430 $TABLE:
431 .quad 0x428a2f98d728ae22,0x7137449123ef65cd
432 .quad 0x428a2f98d728ae22,0x7137449123ef65cd
433 .quad 0xb5c0fbcfec4d3b2f,0xe9b5dba58189dbbc
434 .quad 0xb5c0fbcfec4d3b2f,0xe9b5dba58189dbbc
435 .quad 0x3956c25bf348b538,0x59f111f1b605d019
436 .quad 0x3956c25bf348b538,0x59f111f1b605d019
437 .quad 0x923f82a4af194f9b,0xab1c5ed5da6d8118
438 .quad 0x923f82a4af194f9b,0xab1c5ed5da6d8118
439 .quad 0xd807aa98a3030242,0x12835b0145706fbe
440 .quad 0xd807aa98a3030242,0x12835b0145706fbe
441 .quad 0x243185be4ee4b28c,0x550c7dc3d5ffb4e2
442 .quad 0x243185be4ee4b28c,0x550c7dc3d5ffb4e2
443 .quad 0x72be5d74f27b896f,0x80deb1fe3b1696b1
444 .quad 0x72be5d74f27b896f,0x80deb1fe3b1696b1
445 .quad 0x9bdc06a725c71235,0xc19bf174cf692694
446 .quad 0x9bdc06a725c71235,0xc19bf174cf692694
447 .quad 0xe49b69c19ef14ad2,0xefbe4786384f25e3
448 .quad 0xe49b69c19ef14ad2,0xefbe4786384f25e3
449 .quad 0x0fc19dc68b8cd5b5,0x240ca1cc77ac9c65
450 .quad 0x0fc19dc68b8cd5b5,0x240ca1cc77ac9c65
451 .quad 0x2de92c6f592b0275,0x4a7484aa6ea6e483
452 .quad 0x2de92c6f592b0275,0x4a7484aa6ea6e483
453 .quad 0x5cb0a9dcbd41fbd4,0x76f988da831153b5
454 .quad 0x5cb0a9dcbd41fbd4,0x76f988da831153b5
455 .quad 0x983e5152ee66dfab,0xa831c66d2db43210
456 .quad 0x983e5152ee66dfab,0xa831c66d2db43210
457 .quad 0xb00327c898fb213f,0xbf597fc7beef0ee4
458 .quad 0xb00327c898fb213f,0xbf597fc7beef0ee4
459 .quad 0xc6e00bf33da88fc2,0xd5a79147930aa725
460 .quad 0xc6e00bf33da88fc2,0xd5a79147930aa725
461 .quad 0x06ca6351e003826f,0x142929670a0e6e70
462 .quad 0x06ca6351e003826f,0x142929670a0e6e70
463 .quad 0x27b70a8546d22ffc,0x2e1b21385c26c926
464 .quad 0x27b70a8546d22ffc,0x2e1b21385c26c926
465 .quad 0x4d2c6dfc5ac42aed,0x53380d139d95b3df
466 .quad 0x4d2c6dfc5ac42aed,0x53380d139d95b3df
467 .quad 0x650a73548baf63de,0x766a0abb3c77b2a8
468 .quad 0x650a73548baf63de,0x766a0abb3c77b2a8
469 .quad 0x81c2c92e47edaee6,0x92722c851482353b
470 .quad 0x81c2c92e47edaee6,0x92722c851482353b
471 .quad 0xa2bfe8a14cf10364,0xa81a664bbc423001
472 .quad 0xa2bfe8a14cf10364,0xa81a664bbc423001
473 .quad 0xc24b8b70d0f89791,0xc76c51a30654be30
474 .quad 0xc24b8b70d0f89791,0xc76c51a30654be30
475 .quad 0xd192e819d6ef5218,0xd69906245565a910
476 .quad 0xd192e819d6ef5218,0xd69906245565a910
477 .quad 0xf40e35855771202a,0x106aa07032bbd1b8
478 .quad 0xf40e35855771202a,0x106aa07032bbd1b8
479 .quad 0x19a4c116b8d2d0c8,0x1e376c085141ab53
480 .quad 0x19a4c116b8d2d0c8,0x1e376c085141ab53
481 .quad 0x2748774cdf8eeb99,0x34b0bcb5e19b48a8
482 .quad 0x2748774cdf8eeb99,0x34b0bcb5e19b48a8
483 .quad 0x391c0cb3c5c95a63,0x4ed8aa4ae3418acb
484 .quad 0x391c0cb3c5c95a63,0x4ed8aa4ae3418acb
485 .quad 0x5b9cca4f7763e373,0x682e6ff3d6b2b8a3
486 .quad 0x5b9cca4f7763e373,0x682e6ff3d6b2b8a3
487 .quad 0x748f82ee5defb2fc,0x78a5636f43172f60
488 .quad 0x748f82ee5defb2fc,0x78a5636f43172f60
489 .quad 0x84c87814a1f0ab72,0x8cc702081a6439ec
490 .quad 0x84c87814a1f0ab72,0x8cc702081a6439ec
491 .quad 0x90befffa23631e28,0xa4506cebde82bde9
492 .quad 0x90befffa23631e28,0xa4506cebde82bde9
493 .quad 0xbef9a3f7b2c67915,0xc67178f2e372532b
494 .quad 0xbef9a3f7b2c67915,0xc67178f2e372532b
495 .quad 0xca273eceea26619c,0xd186b8c721c0c207
496 .quad 0xca273eceea26619c,0xd186b8c721c0c207
497 .quad 0xeada7dd6cde0eb1e,0xf57d4f7fee6ed178
498 .quad 0xeada7dd6cde0eb1e,0xf57d4f7fee6ed178
499 .quad 0x06f067aa72176fba,0x0a637dc5a2c898a6
500 .quad 0x06f067aa72176fba,0x0a637dc5a2c898a6
501 .quad 0x113f9804bef90dae,0x1b710b35131c471b
502 .quad 0x113f9804bef90dae,0x1b710b35131c471b
503 .quad 0x28db77f523047d84,0x32caab7b40c72493
504 .quad 0x28db77f523047d84,0x32caab7b40c72493
505 .quad 0x3c9ebe0a15c9bebc,0x431d67c49c100d4c
506 .quad 0x3c9ebe0a15c9bebc,0x431d67c49c100d4c
507 .quad 0x4cc5d4becb3e42b6,0x597f299cfc657e2a
508 .quad 0x4cc5d4becb3e42b6,0x597f299cfc657e2a
509 .quad 0x5fcb6fab3ad6faec,0x6c44198c4a475817
510 .quad 0x5fcb6fab3ad6faec,0x6c44198c4a475817
511
512 .quad 0x0001020304050607,0x08090a0b0c0d0e0f
513 .quad 0x0001020304050607,0x08090a0b0c0d0e0f
514 .asciz "SHA512 block transform for x86_64, CRYPTOGAMS by <appro\@openssl.org>"
515 ___
516 }
517
518 ######################################################################
519 # SIMD code paths
520 #
521 if ($SZ==4 && $shaext) {{{
522 ######################################################################
523 # Intel SHA Extensions implementation of SHA256 update function.
524 #
525 my ($ctx,$inp,$num,$Tbl)=("%rdi","%rsi","%rdx","%rcx");
526
527 my ($Wi,$ABEF,$CDGH,$TMP,$BSWAP,$ABEF_SAVE,$CDGH_SAVE)=map("%xmm$_",(0..2,7..10));
528 my @MSG=map("%xmm$_",(3..6));
529
530 $code.=<<___;
531 .type sha256_block_data_order_shaext,\@function,3
532 .align 64
533 sha256_block_data_order_shaext:
534 _shaext_shortcut:
535 ___
536 $code.=<<___ if ($win64);
537 lea `-8-5*16`(%rsp),%rsp
538 movaps %xmm6,-8-5*16(%rax)
539 movaps %xmm7,-8-4*16(%rax)
540 movaps %xmm8,-8-3*16(%rax)
541 movaps %xmm9,-8-2*16(%rax)
542 movaps %xmm10,-8-1*16(%rax)
543 .Lprologue_shaext:
544 ___
545 $code.=<<___;
546 lea K256+0x80(%rip),$Tbl
547 movdqu ($ctx),$ABEF # DCBA
548 movdqu 16($ctx),$CDGH # HGFE
549 movdqa 0x200-0x80($Tbl),$TMP # byte swap mask
550
551 pshufd \$0x1b,$ABEF,$Wi # ABCD
552 pshufd \$0xb1,$ABEF,$ABEF # CDAB
553 pshufd \$0x1b,$CDGH,$CDGH # EFGH
554 movdqa $TMP,$BSWAP # offload
555 palignr \$8,$CDGH,$ABEF # ABEF
556 punpcklqdq $Wi,$CDGH # CDGH
557 jmp .Loop_shaext
558
559 .align 16
560 .Loop_shaext:
561 movdqu ($inp),@MSG[0]
562 movdqu 0x10($inp),@MSG[1]
563 movdqu 0x20($inp),@MSG[2]
564 pshufb $TMP,@MSG[0]
565 movdqu 0x30($inp),@MSG[3]
566
567 movdqa 0*32-0x80($Tbl),$Wi
568 paddd @MSG[0],$Wi
569 pshufb $TMP,@MSG[1]
570 movdqa $CDGH,$CDGH_SAVE # offload
571 sha256rnds2 $ABEF,$CDGH # 0-3
572 pshufd \$0x0e,$Wi,$Wi
573 nop
574 movdqa $ABEF,$ABEF_SAVE # offload
575 sha256rnds2 $CDGH,$ABEF
576
577 movdqa 1*32-0x80($Tbl),$Wi
578 paddd @MSG[1],$Wi
579 pshufb $TMP,@MSG[2]
580 sha256rnds2 $ABEF,$CDGH # 4-7
581 pshufd \$0x0e,$Wi,$Wi
582 lea 0x40($inp),$inp
583 sha256msg1 @MSG[1],@MSG[0]
584 sha256rnds2 $CDGH,$ABEF
585
586 movdqa 2*32-0x80($Tbl),$Wi
587 paddd @MSG[2],$Wi
588 pshufb $TMP,@MSG[3]
589 sha256rnds2 $ABEF,$CDGH # 8-11
590 pshufd \$0x0e,$Wi,$Wi
591 movdqa @MSG[3],$TMP
592 palignr \$4,@MSG[2],$TMP
593 nop
594 paddd $TMP,@MSG[0]
595 sha256msg1 @MSG[2],@MSG[1]
596 sha256rnds2 $CDGH,$ABEF
597
598 movdqa 3*32-0x80($Tbl),$Wi
599 paddd @MSG[3],$Wi
600 sha256msg2 @MSG[3],@MSG[0]
601 sha256rnds2 $ABEF,$CDGH # 12-15
602 pshufd \$0x0e,$Wi,$Wi
603 movdqa @MSG[0],$TMP
604 palignr \$4,@MSG[3],$TMP
605 nop
606 paddd $TMP,@MSG[1]
607 sha256msg1 @MSG[3],@MSG[2]
608 sha256rnds2 $CDGH,$ABEF
609 ___
610 for($i=4;$i<16-3;$i++) {
611 $code.=<<___;
612 movdqa $i*32-0x80($Tbl),$Wi
613 paddd @MSG[0],$Wi
614 sha256msg2 @MSG[0],@MSG[1]
615 sha256rnds2 $ABEF,$CDGH # 16-19...
616 pshufd \$0x0e,$Wi,$Wi
617 movdqa @MSG[1],$TMP
618 palignr \$4,@MSG[0],$TMP
619 nop
620 paddd $TMP,@MSG[2]
621 sha256msg1 @MSG[0],@MSG[3]
622 sha256rnds2 $CDGH,$ABEF
623 ___
624 push(@MSG,shift(@MSG));
625 }
626 $code.=<<___;
627 movdqa 13*32-0x80($Tbl),$Wi
628 paddd @MSG[0],$Wi
629 sha256msg2 @MSG[0],@MSG[1]
630 sha256rnds2 $ABEF,$CDGH # 52-55
631 pshufd \$0x0e,$Wi,$Wi
632 movdqa @MSG[1],$TMP
633 palignr \$4,@MSG[0],$TMP
634 sha256rnds2 $CDGH,$ABEF
635 paddd $TMP,@MSG[2]
636
637 movdqa 14*32-0x80($Tbl),$Wi
638 paddd @MSG[1],$Wi
639 sha256rnds2 $ABEF,$CDGH # 56-59
640 pshufd \$0x0e,$Wi,$Wi
641 sha256msg2 @MSG[1],@MSG[2]
642 movdqa $BSWAP,$TMP
643 sha256rnds2 $CDGH,$ABEF
644
645 movdqa 15*32-0x80($Tbl),$Wi
646 paddd @MSG[2],$Wi
647 nop
648 sha256rnds2 $ABEF,$CDGH # 60-63
649 pshufd \$0x0e,$Wi,$Wi
650 dec $num
651 nop
652 sha256rnds2 $CDGH,$ABEF
653
654 paddd $CDGH_SAVE,$CDGH
655 paddd $ABEF_SAVE,$ABEF
656 jnz .Loop_shaext
657
658 pshufd \$0xb1,$CDGH,$CDGH # DCHG
659 pshufd \$0x1b,$ABEF,$TMP # FEBA
660 pshufd \$0xb1,$ABEF,$ABEF # BAFE
661 punpckhqdq $CDGH,$ABEF # DCBA
662 palignr \$8,$TMP,$CDGH # HGFE
663
664 movdqu $ABEF,($ctx)
665 movdqu $CDGH,16($ctx)
666 ___
667 $code.=<<___ if ($win64);
668 movaps -8-5*16(%rax),%xmm6
669 movaps -8-4*16(%rax),%xmm7
670 movaps -8-3*16(%rax),%xmm8
671 movaps -8-2*16(%rax),%xmm9
672 movaps -8-1*16(%rax),%xmm10
673 mov %rax,%rsp
674 .Lepilogue_shaext:
675 ___
676 $code.=<<___;
677 ret
678 .size sha256_block_data_order_shaext,.-sha256_block_data_order_shaext
679 ___
680 }}}
681 {{{
682
683 my $a4=$T1;
684 my ($a,$b,$c,$d,$e,$f,$g,$h);
685
686 sub AUTOLOAD() # thunk [simplified] 32-bit style perlasm
687 { my $opcode = $AUTOLOAD; $opcode =~ s/.*:://;
688 my $arg = pop;
689 $arg = "\$$arg" if ($arg*1 eq $arg);
690 $code .= "\t$opcode\t".join(',',$arg,reverse @_)."\n";
691 }
692
693 sub body_00_15 () {
694 (
695 '($a,$b,$c,$d,$e,$f,$g,$h)=@ROT;'.
696
697 '&ror ($a0,$Sigma1[2]-$Sigma1[1])',
698 '&mov ($a,$a1)',
699 '&mov ($a4,$f)',
700
701 '&ror ($a1,$Sigma0[2]-$Sigma0[1])',
702 '&xor ($a0,$e)',
703 '&xor ($a4,$g)', # f^g
704
705 '&ror ($a0,$Sigma1[1]-$Sigma1[0])',
706 '&xor ($a1,$a)',
707 '&and ($a4,$e)', # (f^g)&e
708
709 '&xor ($a0,$e)',
710 '&add ($h,$SZ*($i&15)."(%rsp)")', # h+=X[i]+K[i]
711 '&mov ($a2,$a)',
712
713 '&xor ($a4,$g)', # Ch(e,f,g)=((f^g)&e)^g
714 '&ror ($a1,$Sigma0[1]-$Sigma0[0])',
715 '&xor ($a2,$b)', # a^b, b^c in next round
716
717 '&add ($h,$a4)', # h+=Ch(e,f,g)
718 '&ror ($a0,$Sigma1[0])', # Sigma1(e)
719 '&and ($a3,$a2)', # (b^c)&(a^b)
720
721 '&xor ($a1,$a)',
722 '&add ($h,$a0)', # h+=Sigma1(e)
723 '&xor ($a3,$b)', # Maj(a,b,c)=Ch(a^b,c,b)
724
725 '&ror ($a1,$Sigma0[0])', # Sigma0(a)
726 '&add ($d,$h)', # d+=h
727 '&add ($h,$a3)', # h+=Maj(a,b,c)
728
729 '&mov ($a0,$d)',
730 '&add ($a1,$h);'. # h+=Sigma0(a)
731 '($a2,$a3) = ($a3,$a2); unshift(@ROT,pop(@ROT)); $i++;'
732 );
733 }
734
735 ######################################################################
736 # SSSE3 code path
737 #
738 if ($SZ==4) { # SHA256 only
739 my @X = map("%xmm$_",(0..3));
740 my ($t0,$t1,$t2,$t3, $t4,$t5) = map("%xmm$_",(4..9));
741
742 $code.=<<___;
743 .type ${func}_ssse3,\@function,3
744 .align 64
745 ${func}_ssse3:
746 .Lssse3_shortcut:
747 push %rbx
748 push %rbp
749 push %r12
750 push %r13
751 push %r14
752 push %r15
753 mov %rsp,%r11 # copy %rsp
754 shl \$4,%rdx # num*16
755 sub \$`$framesz+$win64*16*4`,%rsp
756 lea ($inp,%rdx,$SZ),%rdx # inp+num*16*$SZ
757 and \$-64,%rsp # align stack frame
758 mov $ctx,$_ctx # save ctx, 1st arg
759 mov $inp,$_inp # save inp, 2nd arh
760 mov %rdx,$_end # save end pointer, "3rd" arg
761 mov %r11,$_rsp # save copy of %rsp
762 ___
763 $code.=<<___ if ($win64);
764 movaps %xmm6,16*$SZ+32(%rsp)
765 movaps %xmm7,16*$SZ+48(%rsp)
766 movaps %xmm8,16*$SZ+64(%rsp)
767 movaps %xmm9,16*$SZ+80(%rsp)
768 ___
769 $code.=<<___;
770 .Lprologue_ssse3:
771
772 mov $SZ*0($ctx),$A
773 mov $SZ*1($ctx),$B
774 mov $SZ*2($ctx),$C
775 mov $SZ*3($ctx),$D
776 mov $SZ*4($ctx),$E
777 mov $SZ*5($ctx),$F
778 mov $SZ*6($ctx),$G
779 mov $SZ*7($ctx),$H
780 ___
781
782 $code.=<<___;
783 #movdqa $TABLE+`$SZ*2*$rounds`+32(%rip),$t4
784 #movdqa $TABLE+`$SZ*2*$rounds`+64(%rip),$t5
785 jmp .Lloop_ssse3
786 .align 16
787 .Lloop_ssse3:
788 movdqa $TABLE+`$SZ*2*$rounds`(%rip),$t3
789 movdqu 0x00($inp),@X[0]
790 movdqu 0x10($inp),@X[1]
791 movdqu 0x20($inp),@X[2]
792 pshufb $t3,@X[0]
793 movdqu 0x30($inp),@X[3]
794 lea $TABLE(%rip),$Tbl
795 pshufb $t3,@X[1]
796 movdqa 0x00($Tbl),$t0
797 movdqa 0x20($Tbl),$t1
798 pshufb $t3,@X[2]
799 paddd @X[0],$t0
800 movdqa 0x40($Tbl),$t2
801 pshufb $t3,@X[3]
802 movdqa 0x60($Tbl),$t3
803 paddd @X[1],$t1
804 paddd @X[2],$t2
805 paddd @X[3],$t3
806 movdqa $t0,0x00(%rsp)
807 mov $A,$a1
808 movdqa $t1,0x10(%rsp)
809 mov $B,$a3
810 movdqa $t2,0x20(%rsp)
811 xor $C,$a3 # magic
812 movdqa $t3,0x30(%rsp)
813 mov $E,$a0
814 jmp .Lssse3_00_47
815
816 .align 16
817 .Lssse3_00_47:
818 sub \$`-16*2*$SZ`,$Tbl # size optimization
819 ___
820 sub Xupdate_256_SSSE3 () {
821 (
822 '&movdqa ($t0,@X[1]);',
823 '&movdqa ($t3,@X[3])',
824 '&palignr ($t0,@X[0],$SZ)', # X[1..4]
825 '&palignr ($t3,@X[2],$SZ);', # X[9..12]
826 '&movdqa ($t1,$t0)',
827 '&movdqa ($t2,$t0);',
828 '&psrld ($t0,$sigma0[2])',
829 '&paddd (@X[0],$t3);', # X[0..3] += X[9..12]
830 '&psrld ($t2,$sigma0[0])',
831 '&pshufd ($t3,@X[3],0b11111010)',# X[14..15]
832 '&pslld ($t1,8*$SZ-$sigma0[1]);'.
833 '&pxor ($t0,$t2)',
834 '&psrld ($t2,$sigma0[1]-$sigma0[0]);'.
835 '&pxor ($t0,$t1)',
836 '&pslld ($t1,$sigma0[1]-$sigma0[0]);'.
837 '&pxor ($t0,$t2);',
838 '&movdqa ($t2,$t3)',
839 '&pxor ($t0,$t1);', # sigma0(X[1..4])
840 '&psrld ($t3,$sigma1[2])',
841 '&paddd (@X[0],$t0);', # X[0..3] += sigma0(X[1..4])
842 '&psrlq ($t2,$sigma1[0])',
843 '&pxor ($t3,$t2);',
844 '&psrlq ($t2,$sigma1[1]-$sigma1[0])',
845 '&pxor ($t3,$t2)',
846 '&pshufb ($t3,$t4)', # sigma1(X[14..15])
847 '&paddd (@X[0],$t3)', # X[0..1] += sigma1(X[14..15])
848 '&pshufd ($t3,@X[0],0b01010000)',# X[16..17]
849 '&movdqa ($t2,$t3);',
850 '&psrld ($t3,$sigma1[2])',
851 '&psrlq ($t2,$sigma1[0])',
852 '&pxor ($t3,$t2);',
853 '&psrlq ($t2,$sigma1[1]-$sigma1[0])',
854 '&pxor ($t3,$t2);',
855 '&movdqa ($t2,16*2*$j."($Tbl)")',
856 '&pshufb ($t3,$t5)',
857 '&paddd (@X[0],$t3)' # X[2..3] += sigma1(X[16..17])
858 );
859 }
860
861 sub SSSE3_256_00_47 () {
862 my $j = shift;
863 my $body = shift;
864 my @X = @_;
865 my @insns = (&$body,&$body,&$body,&$body); # 104 instructions
866
867 if (0) {
868 foreach (Xupdate_256_SSSE3()) { # 36 instructions
869 eval;
870 eval(shift(@insns));
871 eval(shift(@insns));
872 eval(shift(@insns));
873 }
874 } else { # squeeze extra 4% on Westmere and 19% on Atom
875 eval(shift(@insns)); #@
876 &movdqa ($t0,@X[1]);
877 eval(shift(@insns));
878 eval(shift(@insns));
879 &movdqa ($t3,@X[3]);
880 eval(shift(@insns)); #@
881 eval(shift(@insns));
882 eval(shift(@insns));
883 eval(shift(@insns)); #@
884 eval(shift(@insns));
885 &palignr ($t0,@X[0],$SZ); # X[1..4]
886 eval(shift(@insns));
887 eval(shift(@insns));
888 &palignr ($t3,@X[2],$SZ); # X[9..12]
889 eval(shift(@insns));
890 eval(shift(@insns));
891 eval(shift(@insns));
892 eval(shift(@insns)); #@
893 &movdqa ($t1,$t0);
894 eval(shift(@insns));
895 eval(shift(@insns));
896 &movdqa ($t2,$t0);
897 eval(shift(@insns)); #@
898 eval(shift(@insns));
899 &psrld ($t0,$sigma0[2]);
900 eval(shift(@insns));
901 eval(shift(@insns));
902 eval(shift(@insns));
903 &paddd (@X[0],$t3); # X[0..3] += X[9..12]
904 eval(shift(@insns)); #@
905 eval(shift(@insns));
906 &psrld ($t2,$sigma0[0]);
907 eval(shift(@insns));
908 eval(shift(@insns));
909 &pshufd ($t3,@X[3],0b11111010); # X[4..15]
910 eval(shift(@insns));
911 eval(shift(@insns)); #@
912 &pslld ($t1,8*$SZ-$sigma0[1]);
913 eval(shift(@insns));
914 eval(shift(@insns));
915 &pxor ($t0,$t2);
916 eval(shift(@insns)); #@
917 eval(shift(@insns));
918 eval(shift(@insns));
919 eval(shift(@insns)); #@
920 &psrld ($t2,$sigma0[1]-$sigma0[0]);
921 eval(shift(@insns));
922 &pxor ($t0,$t1);
923 eval(shift(@insns));
924 eval(shift(@insns));
925 &pslld ($t1,$sigma0[1]-$sigma0[0]);
926 eval(shift(@insns));
927 eval(shift(@insns));
928 &pxor ($t0,$t2);
929 eval(shift(@insns));
930 eval(shift(@insns)); #@
931 &movdqa ($t2,$t3);
932 eval(shift(@insns));
933 eval(shift(@insns));
934 &pxor ($t0,$t1); # sigma0(X[1..4])
935 eval(shift(@insns)); #@
936 eval(shift(@insns));
937 eval(shift(@insns));
938 &psrld ($t3,$sigma1[2]);
939 eval(shift(@insns));
940 eval(shift(@insns));
941 &paddd (@X[0],$t0); # X[0..3] += sigma0(X[1..4])
942 eval(shift(@insns)); #@
943 eval(shift(@insns));
944 &psrlq ($t2,$sigma1[0]);
945 eval(shift(@insns));
946 eval(shift(@insns));
947 eval(shift(@insns));
948 &pxor ($t3,$t2);
949 eval(shift(@insns)); #@
950 eval(shift(@insns));
951 eval(shift(@insns));
952 eval(shift(@insns)); #@
953 &psrlq ($t2,$sigma1[1]-$sigma1[0]);
954 eval(shift(@insns));
955 eval(shift(@insns));
956 &pxor ($t3,$t2);
957 eval(shift(@insns)); #@
958 eval(shift(@insns));
959 eval(shift(@insns));
960 #&pshufb ($t3,$t4); # sigma1(X[14..15])
961 &pshufd ($t3,$t3,0b10000000);
962 eval(shift(@insns));
963 eval(shift(@insns));
964 eval(shift(@insns));
965 &psrldq ($t3,8);
966 eval(shift(@insns));
967 eval(shift(@insns)); #@
968 eval(shift(@insns));
969 eval(shift(@insns));
970 eval(shift(@insns)); #@
971 &paddd (@X[0],$t3); # X[0..1] += sigma1(X[14..15])
972 eval(shift(@insns));
973 eval(shift(@insns));
974 eval(shift(@insns));
975 &pshufd ($t3,@X[0],0b01010000); # X[16..17]
976 eval(shift(@insns));
977 eval(shift(@insns)); #@
978 eval(shift(@insns));
979 &movdqa ($t2,$t3);
980 eval(shift(@insns));
981 eval(shift(@insns));
982 &psrld ($t3,$sigma1[2]);
983 eval(shift(@insns));
984 eval(shift(@insns)); #@
985 &psrlq ($t2,$sigma1[0]);
986 eval(shift(@insns));
987 eval(shift(@insns));
988 &pxor ($t3,$t2);
989 eval(shift(@insns)); #@
990 eval(shift(@insns));
991 eval(shift(@insns));
992 eval(shift(@insns)); #@
993 eval(shift(@insns));
994 &psrlq ($t2,$sigma1[1]-$sigma1[0]);
995 eval(shift(@insns));
996 eval(shift(@insns));
997 eval(shift(@insns));
998 &pxor ($t3,$t2);
999 eval(shift(@insns));
1000 eval(shift(@insns));
1001 eval(shift(@insns)); #@
1002 #&pshufb ($t3,$t5);
1003 &pshufd ($t3,$t3,0b00001000);
1004 eval(shift(@insns));
1005 eval(shift(@insns));
1006 &movdqa ($t2,16*2*$j."($Tbl)");
1007 eval(shift(@insns)); #@
1008 eval(shift(@insns));
1009 &pslldq ($t3,8);
1010 eval(shift(@insns));
1011 eval(shift(@insns));
1012 eval(shift(@insns));
1013 &paddd (@X[0],$t3); # X[2..3] += sigma1(X[16..17])
1014 eval(shift(@insns)); #@
1015 eval(shift(@insns));
1016 eval(shift(@insns));
1017 }
1018 &paddd ($t2,@X[0]);
1019 foreach (@insns) { eval; } # remaining instructions
1020 &movdqa (16*$j."(%rsp)",$t2);
1021 }
1022
1023 for ($i=0,$j=0; $j<4; $j++) {
1024 &SSSE3_256_00_47($j,\&body_00_15,@X);
1025 push(@X,shift(@X)); # rotate(@X)
1026 }
1027 &cmpb ($SZ-1+16*2*$SZ."($Tbl)",0);
1028 &jne (".Lssse3_00_47");
1029
1030 for ($i=0; $i<16; ) {
1031 foreach(body_00_15()) { eval; }
1032 }
1033 $code.=<<___;
1034 mov $_ctx,$ctx
1035 mov $a1,$A
1036
1037 add $SZ*0($ctx),$A
1038 lea 16*$SZ($inp),$inp
1039 add $SZ*1($ctx),$B
1040 add $SZ*2($ctx),$C
1041 add $SZ*3($ctx),$D
1042 add $SZ*4($ctx),$E
1043 add $SZ*5($ctx),$F
1044 add $SZ*6($ctx),$G
1045 add $SZ*7($ctx),$H
1046
1047 cmp $_end,$inp
1048
1049 mov $A,$SZ*0($ctx)
1050 mov $B,$SZ*1($ctx)
1051 mov $C,$SZ*2($ctx)
1052 mov $D,$SZ*3($ctx)
1053 mov $E,$SZ*4($ctx)
1054 mov $F,$SZ*5($ctx)
1055 mov $G,$SZ*6($ctx)
1056 mov $H,$SZ*7($ctx)
1057 jb .Lloop_ssse3
1058
1059 mov $_rsp,%rsi
1060 ___
1061 $code.=<<___ if ($win64);
1062 movaps 16*$SZ+32(%rsp),%xmm6
1063 movaps 16*$SZ+48(%rsp),%xmm7
1064 movaps 16*$SZ+64(%rsp),%xmm8
1065 movaps 16*$SZ+80(%rsp),%xmm9
1066 ___
1067 $code.=<<___;
1068 mov (%rsi),%r15
1069 mov 8(%rsi),%r14
1070 mov 16(%rsi),%r13
1071 mov 24(%rsi),%r12
1072 mov 32(%rsi),%rbp
1073 mov 40(%rsi),%rbx
1074 lea 48(%rsi),%rsp
1075 .Lepilogue_ssse3:
1076 ret
1077 .size ${func}_ssse3,.-${func}_ssse3
1078 ___
1079 }
1080
1081 if ($avx) {{
1082 ######################################################################
1083 # XOP code path
1084 #
1085 if ($SZ==8) { # SHA512 only
1086 $code.=<<___;
1087 .type ${func}_xop,\@function,3
1088 .align 64
1089 ${func}_xop:
1090 .Lxop_shortcut:
1091 push %rbx
1092 push %rbp
1093 push %r12
1094 push %r13
1095 push %r14
1096 push %r15
1097 mov %rsp,%r11 # copy %rsp
1098 shl \$4,%rdx # num*16
1099 sub \$`$framesz+$win64*16*($SZ==4?4:6)`,%rsp
1100 lea ($inp,%rdx,$SZ),%rdx # inp+num*16*$SZ
1101 and \$-64,%rsp # align stack frame
1102 mov $ctx,$_ctx # save ctx, 1st arg
1103 mov $inp,$_inp # save inp, 2nd arh
1104 mov %rdx,$_end # save end pointer, "3rd" arg
1105 mov %r11,$_rsp # save copy of %rsp
1106 ___
1107 $code.=<<___ if ($win64);
1108 movaps %xmm6,16*$SZ+32(%rsp)
1109 movaps %xmm7,16*$SZ+48(%rsp)
1110 movaps %xmm8,16*$SZ+64(%rsp)
1111 movaps %xmm9,16*$SZ+80(%rsp)
1112 ___
1113 $code.=<<___ if ($win64 && $SZ>4);
1114 movaps %xmm10,16*$SZ+96(%rsp)
1115 movaps %xmm11,16*$SZ+112(%rsp)
1116 ___
1117 $code.=<<___;
1118 .Lprologue_xop:
1119
1120 vzeroupper
1121 mov $SZ*0($ctx),$A
1122 mov $SZ*1($ctx),$B
1123 mov $SZ*2($ctx),$C
1124 mov $SZ*3($ctx),$D
1125 mov $SZ*4($ctx),$E
1126 mov $SZ*5($ctx),$F
1127 mov $SZ*6($ctx),$G
1128 mov $SZ*7($ctx),$H
1129 jmp .Lloop_xop
1130 ___
1131 if ($SZ==4) { # SHA256
1132 my @X = map("%xmm$_",(0..3));
1133 my ($t0,$t1,$t2,$t3) = map("%xmm$_",(4..7));
1134
1135 $code.=<<___;
1136 .align 16
1137 .Lloop_xop:
1138 vmovdqa $TABLE+`$SZ*2*$rounds`(%rip),$t3
1139 vmovdqu 0x00($inp),@X[0]
1140 vmovdqu 0x10($inp),@X[1]
1141 vmovdqu 0x20($inp),@X[2]
1142 vmovdqu 0x30($inp),@X[3]
1143 vpshufb $t3,@X[0],@X[0]
1144 lea $TABLE(%rip),$Tbl
1145 vpshufb $t3,@X[1],@X[1]
1146 vpshufb $t3,@X[2],@X[2]
1147 vpaddd 0x00($Tbl),@X[0],$t0
1148 vpshufb $t3,@X[3],@X[3]
1149 vpaddd 0x20($Tbl),@X[1],$t1
1150 vpaddd 0x40($Tbl),@X[2],$t2
1151 vpaddd 0x60($Tbl),@X[3],$t3
1152 vmovdqa $t0,0x00(%rsp)
1153 mov $A,$a1
1154 vmovdqa $t1,0x10(%rsp)
1155 mov $B,$a3
1156 vmovdqa $t2,0x20(%rsp)
1157 xor $C,$a3 # magic
1158 vmovdqa $t3,0x30(%rsp)
1159 mov $E,$a0
1160 jmp .Lxop_00_47
1161
1162 .align 16
1163 .Lxop_00_47:
1164 sub \$`-16*2*$SZ`,$Tbl # size optimization
1165 ___
1166 sub XOP_256_00_47 () {
1167 my $j = shift;
1168 my $body = shift;
1169 my @X = @_;
1170 my @insns = (&$body,&$body,&$body,&$body); # 104 instructions
1171
1172 &vpalignr ($t0,@X[1],@X[0],$SZ); # X[1..4]
1173 eval(shift(@insns));
1174 eval(shift(@insns));
1175 &vpalignr ($t3,@X[3],@X[2],$SZ); # X[9..12]
1176 eval(shift(@insns));
1177 eval(shift(@insns));
1178 &vprotd ($t1,$t0,8*$SZ-$sigma0[1]);
1179 eval(shift(@insns));
1180 eval(shift(@insns));
1181 &vpsrld ($t0,$t0,$sigma0[2]);
1182 eval(shift(@insns));
1183 eval(shift(@insns));
1184 &vpaddd (@X[0],@X[0],$t3); # X[0..3] += X[9..12]
1185 eval(shift(@insns));
1186 eval(shift(@insns));
1187 eval(shift(@insns));
1188 eval(shift(@insns));
1189 &vprotd ($t2,$t1,$sigma0[1]-$sigma0[0]);
1190 eval(shift(@insns));
1191 eval(shift(@insns));
1192 &vpxor ($t0,$t0,$t1);
1193 eval(shift(@insns));
1194 eval(shift(@insns));
1195 eval(shift(@insns));
1196 eval(shift(@insns));
1197 &vprotd ($t3,@X[3],8*$SZ-$sigma1[1]);
1198 eval(shift(@insns));
1199 eval(shift(@insns));
1200 &vpxor ($t0,$t0,$t2); # sigma0(X[1..4])
1201 eval(shift(@insns));
1202 eval(shift(@insns));
1203 &vpsrld ($t2,@X[3],$sigma1[2]);
1204 eval(shift(@insns));
1205 eval(shift(@insns));
1206 &vpaddd (@X[0],@X[0],$t0); # X[0..3] += sigma0(X[1..4])
1207 eval(shift(@insns));
1208 eval(shift(@insns));
1209 &vprotd ($t1,$t3,$sigma1[1]-$sigma1[0]);
1210 eval(shift(@insns));
1211 eval(shift(@insns));
1212 &vpxor ($t3,$t3,$t2);
1213 eval(shift(@insns));
1214 eval(shift(@insns));
1215 eval(shift(@insns));
1216 eval(shift(@insns));
1217 &vpxor ($t3,$t3,$t1); # sigma1(X[14..15])
1218 eval(shift(@insns));
1219 eval(shift(@insns));
1220 eval(shift(@insns));
1221 eval(shift(@insns));
1222 &vpsrldq ($t3,$t3,8);
1223 eval(shift(@insns));
1224 eval(shift(@insns));
1225 eval(shift(@insns));
1226 eval(shift(@insns));
1227 &vpaddd (@X[0],@X[0],$t3); # X[0..1] += sigma1(X[14..15])
1228 eval(shift(@insns));
1229 eval(shift(@insns));
1230 eval(shift(@insns));
1231 eval(shift(@insns));
1232 &vprotd ($t3,@X[0],8*$SZ-$sigma1[1]);
1233 eval(shift(@insns));
1234 eval(shift(@insns));
1235 &vpsrld ($t2,@X[0],$sigma1[2]);
1236 eval(shift(@insns));
1237 eval(shift(@insns));
1238 &vprotd ($t1,$t3,$sigma1[1]-$sigma1[0]);
1239 eval(shift(@insns));
1240 eval(shift(@insns));
1241 &vpxor ($t3,$t3,$t2);
1242 eval(shift(@insns));
1243 eval(shift(@insns));
1244 eval(shift(@insns));
1245 eval(shift(@insns));
1246 &vpxor ($t3,$t3,$t1); # sigma1(X[16..17])
1247 eval(shift(@insns));
1248 eval(shift(@insns));
1249 eval(shift(@insns));
1250 eval(shift(@insns));
1251 &vpslldq ($t3,$t3,8); # 22 instructions
1252 eval(shift(@insns));
1253 eval(shift(@insns));
1254 eval(shift(@insns));
1255 eval(shift(@insns));
1256 &vpaddd (@X[0],@X[0],$t3); # X[2..3] += sigma1(X[16..17])
1257 eval(shift(@insns));
1258 eval(shift(@insns));
1259 eval(shift(@insns));
1260 eval(shift(@insns));
1261 &vpaddd ($t2,@X[0],16*2*$j."($Tbl)");
1262 foreach (@insns) { eval; } # remaining instructions
1263 &vmovdqa (16*$j."(%rsp)",$t2);
1264 }
1265
1266 for ($i=0,$j=0; $j<4; $j++) {
1267 &XOP_256_00_47($j,\&body_00_15,@X);
1268 push(@X,shift(@X)); # rotate(@X)
1269 }
1270 &cmpb ($SZ-1+16*2*$SZ."($Tbl)",0);
1271 &jne (".Lxop_00_47");
1272
1273 for ($i=0; $i<16; ) {
1274 foreach(body_00_15()) { eval; }
1275 }
1276
1277 } else { # SHA512
1278 my @X = map("%xmm$_",(0..7));
1279 my ($t0,$t1,$t2,$t3) = map("%xmm$_",(8..11));
1280
1281 $code.=<<___;
1282 .align 16
1283 .Lloop_xop:
1284 vmovdqa $TABLE+`$SZ*2*$rounds`(%rip),$t3
1285 vmovdqu 0x00($inp),@X[0]
1286 lea $TABLE+0x80(%rip),$Tbl # size optimization
1287 vmovdqu 0x10($inp),@X[1]
1288 vmovdqu 0x20($inp),@X[2]
1289 vpshufb $t3,@X[0],@X[0]
1290 vmovdqu 0x30($inp),@X[3]
1291 vpshufb $t3,@X[1],@X[1]
1292 vmovdqu 0x40($inp),@X[4]
1293 vpshufb $t3,@X[2],@X[2]
1294 vmovdqu 0x50($inp),@X[5]
1295 vpshufb $t3,@X[3],@X[3]
1296 vmovdqu 0x60($inp),@X[6]
1297 vpshufb $t3,@X[4],@X[4]
1298 vmovdqu 0x70($inp),@X[7]
1299 vpshufb $t3,@X[5],@X[5]
1300 vpaddq -0x80($Tbl),@X[0],$t0
1301 vpshufb $t3,@X[6],@X[6]
1302 vpaddq -0x60($Tbl),@X[1],$t1
1303 vpshufb $t3,@X[7],@X[7]
1304 vpaddq -0x40($Tbl),@X[2],$t2
1305 vpaddq -0x20($Tbl),@X[3],$t3
1306 vmovdqa $t0,0x00(%rsp)
1307 vpaddq 0x00($Tbl),@X[4],$t0
1308 vmovdqa $t1,0x10(%rsp)
1309 vpaddq 0x20($Tbl),@X[5],$t1
1310 vmovdqa $t2,0x20(%rsp)
1311 vpaddq 0x40($Tbl),@X[6],$t2
1312 vmovdqa $t3,0x30(%rsp)
1313 vpaddq 0x60($Tbl),@X[7],$t3
1314 vmovdqa $t0,0x40(%rsp)
1315 mov $A,$a1
1316 vmovdqa $t1,0x50(%rsp)
1317 mov $B,$a3
1318 vmovdqa $t2,0x60(%rsp)
1319 xor $C,$a3 # magic
1320 vmovdqa $t3,0x70(%rsp)
1321 mov $E,$a0
1322 jmp .Lxop_00_47
1323
1324 .align 16
1325 .Lxop_00_47:
1326 add \$`16*2*$SZ`,$Tbl
1327 ___
1328 sub XOP_512_00_47 () {
1329 my $j = shift;
1330 my $body = shift;
1331 my @X = @_;
1332 my @insns = (&$body,&$body); # 52 instructions
1333
1334 &vpalignr ($t0,@X[1],@X[0],$SZ); # X[1..2]
1335 eval(shift(@insns));
1336 eval(shift(@insns));
1337 &vpalignr ($t3,@X[5],@X[4],$SZ); # X[9..10]
1338 eval(shift(@insns));
1339 eval(shift(@insns));
1340 &vprotq ($t1,$t0,8*$SZ-$sigma0[1]);
1341 eval(shift(@insns));
1342 eval(shift(@insns));
1343 &vpsrlq ($t0,$t0,$sigma0[2]);
1344 eval(shift(@insns));
1345 eval(shift(@insns));
1346 &vpaddq (@X[0],@X[0],$t3); # X[0..1] += X[9..10]
1347 eval(shift(@insns));
1348 eval(shift(@insns));
1349 eval(shift(@insns));
1350 eval(shift(@insns));
1351 &vprotq ($t2,$t1,$sigma0[1]-$sigma0[0]);
1352 eval(shift(@insns));
1353 eval(shift(@insns));
1354 &vpxor ($t0,$t0,$t1);
1355 eval(shift(@insns));
1356 eval(shift(@insns));
1357 eval(shift(@insns));
1358 eval(shift(@insns));
1359 &vprotq ($t3,@X[7],8*$SZ-$sigma1[1]);
1360 eval(shift(@insns));
1361 eval(shift(@insns));
1362 &vpxor ($t0,$t0,$t2); # sigma0(X[1..2])
1363 eval(shift(@insns));
1364 eval(shift(@insns));
1365 &vpsrlq ($t2,@X[7],$sigma1[2]);
1366 eval(shift(@insns));
1367 eval(shift(@insns));
1368 &vpaddq (@X[0],@X[0],$t0); # X[0..1] += sigma0(X[1..2])
1369 eval(shift(@insns));
1370 eval(shift(@insns));
1371 &vprotq ($t1,$t3,$sigma1[1]-$sigma1[0]);
1372 eval(shift(@insns));
1373 eval(shift(@insns));
1374 &vpxor ($t3,$t3,$t2);
1375 eval(shift(@insns));
1376 eval(shift(@insns));
1377 eval(shift(@insns));
1378 eval(shift(@insns));
1379 &vpxor ($t3,$t3,$t1); # sigma1(X[14..15])
1380 eval(shift(@insns));
1381 eval(shift(@insns));
1382 eval(shift(@insns));
1383 eval(shift(@insns));
1384 &vpaddq (@X[0],@X[0],$t3); # X[0..1] += sigma1(X[14..15])
1385 eval(shift(@insns));
1386 eval(shift(@insns));
1387 eval(shift(@insns));
1388 eval(shift(@insns));
1389 &vpaddq ($t2,@X[0],16*2*$j-0x80."($Tbl)");
1390 foreach (@insns) { eval; } # remaining instructions
1391 &vmovdqa (16*$j."(%rsp)",$t2);
1392 }
1393
1394 for ($i=0,$j=0; $j<8; $j++) {
1395 &XOP_512_00_47($j,\&body_00_15,@X);
1396 push(@X,shift(@X)); # rotate(@X)
1397 }
1398 &cmpb ($SZ-1+16*2*$SZ-0x80."($Tbl)",0);
1399 &jne (".Lxop_00_47");
1400
1401 for ($i=0; $i<16; ) {
1402 foreach(body_00_15()) { eval; }
1403 }
1404 }
1405 $code.=<<___;
1406 mov $_ctx,$ctx
1407 mov $a1,$A
1408
1409 add $SZ*0($ctx),$A
1410 lea 16*$SZ($inp),$inp
1411 add $SZ*1($ctx),$B
1412 add $SZ*2($ctx),$C
1413 add $SZ*3($ctx),$D
1414 add $SZ*4($ctx),$E
1415 add $SZ*5($ctx),$F
1416 add $SZ*6($ctx),$G
1417 add $SZ*7($ctx),$H
1418
1419 cmp $_end,$inp
1420
1421 mov $A,$SZ*0($ctx)
1422 mov $B,$SZ*1($ctx)
1423 mov $C,$SZ*2($ctx)
1424 mov $D,$SZ*3($ctx)
1425 mov $E,$SZ*4($ctx)
1426 mov $F,$SZ*5($ctx)
1427 mov $G,$SZ*6($ctx)
1428 mov $H,$SZ*7($ctx)
1429 jb .Lloop_xop
1430
1431 mov $_rsp,%rsi
1432 vzeroupper
1433 ___
1434 $code.=<<___ if ($win64);
1435 movaps 16*$SZ+32(%rsp),%xmm6
1436 movaps 16*$SZ+48(%rsp),%xmm7
1437 movaps 16*$SZ+64(%rsp),%xmm8
1438 movaps 16*$SZ+80(%rsp),%xmm9
1439 ___
1440 $code.=<<___ if ($win64 && $SZ>4);
1441 movaps 16*$SZ+96(%rsp),%xmm10
1442 movaps 16*$SZ+112(%rsp),%xmm11
1443 ___
1444 $code.=<<___;
1445 mov (%rsi),%r15
1446 mov 8(%rsi),%r14
1447 mov 16(%rsi),%r13
1448 mov 24(%rsi),%r12
1449 mov 32(%rsi),%rbp
1450 mov 40(%rsi),%rbx
1451 lea 48(%rsi),%rsp
1452 .Lepilogue_xop:
1453 ret
1454 .size ${func}_xop,.-${func}_xop
1455 ___
1456 }
1457 ######################################################################
1458 # AVX+shrd code path
1459 #
1460 local *ror = sub { &shrd(@_[0],@_) };
1461
1462 $code.=<<___;
1463 .type ${func}_avx,\@function,3
1464 .align 64
1465 ${func}_avx:
1466 .Lavx_shortcut:
1467 push %rbx
1468 push %rbp
1469 push %r12
1470 push %r13
1471 push %r14
1472 push %r15
1473 mov %rsp,%r11 # copy %rsp
1474 shl \$4,%rdx # num*16
1475 sub \$`$framesz+$win64*16*($SZ==4?4:6)`,%rsp
1476 lea ($inp,%rdx,$SZ),%rdx # inp+num*16*$SZ
1477 and \$-64,%rsp # align stack frame
1478 mov $ctx,$_ctx # save ctx, 1st arg
1479 mov $inp,$_inp # save inp, 2nd arh
1480 mov %rdx,$_end # save end pointer, "3rd" arg
1481 mov %r11,$_rsp # save copy of %rsp
1482 ___
1483 $code.=<<___ if ($win64);
1484 movaps %xmm6,16*$SZ+32(%rsp)
1485 movaps %xmm7,16*$SZ+48(%rsp)
1486 movaps %xmm8,16*$SZ+64(%rsp)
1487 movaps %xmm9,16*$SZ+80(%rsp)
1488 ___
1489 $code.=<<___ if ($win64 && $SZ>4);
1490 movaps %xmm10,16*$SZ+96(%rsp)
1491 movaps %xmm11,16*$SZ+112(%rsp)
1492 ___
1493 $code.=<<___;
1494 .Lprologue_avx:
1495
1496 vzeroupper
1497 mov $SZ*0($ctx),$A
1498 mov $SZ*1($ctx),$B
1499 mov $SZ*2($ctx),$C
1500 mov $SZ*3($ctx),$D
1501 mov $SZ*4($ctx),$E
1502 mov $SZ*5($ctx),$F
1503 mov $SZ*6($ctx),$G
1504 mov $SZ*7($ctx),$H
1505 ___
1506 if ($SZ==4) { # SHA256
1507 my @X = map("%xmm$_",(0..3));
1508 my ($t0,$t1,$t2,$t3, $t4,$t5) = map("%xmm$_",(4..9));
1509
1510 $code.=<<___;
1511 vmovdqa $TABLE+`$SZ*2*$rounds`+32(%rip),$t4
1512 vmovdqa $TABLE+`$SZ*2*$rounds`+64(%rip),$t5
1513 jmp .Lloop_avx
1514 .align 16
1515 .Lloop_avx:
1516 vmovdqa $TABLE+`$SZ*2*$rounds`(%rip),$t3
1517 vmovdqu 0x00($inp),@X[0]
1518 vmovdqu 0x10($inp),@X[1]
1519 vmovdqu 0x20($inp),@X[2]
1520 vmovdqu 0x30($inp),@X[3]
1521 vpshufb $t3,@X[0],@X[0]
1522 lea $TABLE(%rip),$Tbl
1523 vpshufb $t3,@X[1],@X[1]
1524 vpshufb $t3,@X[2],@X[2]
1525 vpaddd 0x00($Tbl),@X[0],$t0
1526 vpshufb $t3,@X[3],@X[3]
1527 vpaddd 0x20($Tbl),@X[1],$t1
1528 vpaddd 0x40($Tbl),@X[2],$t2
1529 vpaddd 0x60($Tbl),@X[3],$t3
1530 vmovdqa $t0,0x00(%rsp)
1531 mov $A,$a1
1532 vmovdqa $t1,0x10(%rsp)
1533 mov $B,$a3
1534 vmovdqa $t2,0x20(%rsp)
1535 xor $C,$a3 # magic
1536 vmovdqa $t3,0x30(%rsp)
1537 mov $E,$a0
1538 jmp .Lavx_00_47
1539
1540 .align 16
1541 .Lavx_00_47:
1542 sub \$`-16*2*$SZ`,$Tbl # size optimization
1543 ___
1544 sub Xupdate_256_AVX () {
1545 (
1546 '&vpalignr ($t0,@X[1],@X[0],$SZ)', # X[1..4]
1547 '&vpalignr ($t3,@X[3],@X[2],$SZ)', # X[9..12]
1548 '&vpsrld ($t2,$t0,$sigma0[0]);',
1549 '&vpaddd (@X[0],@X[0],$t3)', # X[0..3] += X[9..12]
1550 '&vpsrld ($t3,$t0,$sigma0[2])',
1551 '&vpslld ($t1,$t0,8*$SZ-$sigma0[1]);',
1552 '&vpxor ($t0,$t3,$t2)',
1553 '&vpshufd ($t3,@X[3],0b11111010)',# X[14..15]
1554 '&vpsrld ($t2,$t2,$sigma0[1]-$sigma0[0]);',
1555 '&vpxor ($t0,$t0,$t1)',
1556 '&vpslld ($t1,$t1,$sigma0[1]-$sigma0[0]);',
1557 '&vpxor ($t0,$t0,$t2)',
1558 '&vpsrld ($t2,$t3,$sigma1[2]);',
1559 '&vpxor ($t0,$t0,$t1)', # sigma0(X[1..4])
1560 '&vpsrlq ($t3,$t3,$sigma1[0]);',
1561 '&vpaddd (@X[0],@X[0],$t0)', # X[0..3] += sigma0(X[1..4])
1562 '&vpxor ($t2,$t2,$t3);',
1563 '&vpsrlq ($t3,$t3,$sigma1[1]-$sigma1[0])',
1564 '&vpxor ($t2,$t2,$t3)',
1565 '&vpshufb ($t2,$t2,$t4)', # sigma1(X[14..15])
1566 '&vpaddd (@X[0],@X[0],$t2)', # X[0..1] += sigma1(X[14..15])
1567 '&vpshufd ($t3,@X[0],0b01010000)',# X[16..17]
1568 '&vpsrld ($t2,$t3,$sigma1[2])',
1569 '&vpsrlq ($t3,$t3,$sigma1[0])',
1570 '&vpxor ($t2,$t2,$t3);',
1571 '&vpsrlq ($t3,$t3,$sigma1[1]-$sigma1[0])',
1572 '&vpxor ($t2,$t2,$t3)',
1573 '&vpshufb ($t2,$t2,$t5)',
1574 '&vpaddd (@X[0],@X[0],$t2)' # X[2..3] += sigma1(X[16..17])
1575 );
1576 }
1577
1578 sub AVX_256_00_47 () {
1579 my $j = shift;
1580 my $body = shift;
1581 my @X = @_;
1582 my @insns = (&$body,&$body,&$body,&$body); # 104 instructions
1583
1584 foreach (Xupdate_256_AVX()) { # 29 instructions
1585 eval;
1586 eval(shift(@insns));
1587 eval(shift(@insns));
1588 eval(shift(@insns));
1589 }
1590 &vpaddd ($t2,@X[0],16*2*$j."($Tbl)");
1591 foreach (@insns) { eval; } # remaining instructions
1592 &vmovdqa (16*$j."(%rsp)",$t2);
1593 }
1594
1595 for ($i=0,$j=0; $j<4; $j++) {
1596 &AVX_256_00_47($j,\&body_00_15,@X);
1597 push(@X,shift(@X)); # rotate(@X)
1598 }
1599 &cmpb ($SZ-1+16*2*$SZ."($Tbl)",0);
1600 &jne (".Lavx_00_47");
1601
1602 for ($i=0; $i<16; ) {
1603 foreach(body_00_15()) { eval; }
1604 }
1605
1606 } else { # SHA512
1607 my @X = map("%xmm$_",(0..7));
1608 my ($t0,$t1,$t2,$t3) = map("%xmm$_",(8..11));
1609
1610 $code.=<<___;
1611 jmp .Lloop_avx
1612 .align 16
1613 .Lloop_avx:
1614 vmovdqa $TABLE+`$SZ*2*$rounds`(%rip),$t3
1615 vmovdqu 0x00($inp),@X[0]
1616 lea $TABLE+0x80(%rip),$Tbl # size optimization
1617 vmovdqu 0x10($inp),@X[1]
1618 vmovdqu 0x20($inp),@X[2]
1619 vpshufb $t3,@X[0],@X[0]
1620 vmovdqu 0x30($inp),@X[3]
1621 vpshufb $t3,@X[1],@X[1]
1622 vmovdqu 0x40($inp),@X[4]
1623 vpshufb $t3,@X[2],@X[2]
1624 vmovdqu 0x50($inp),@X[5]
1625 vpshufb $t3,@X[3],@X[3]
1626 vmovdqu 0x60($inp),@X[6]
1627 vpshufb $t3,@X[4],@X[4]
1628 vmovdqu 0x70($inp),@X[7]
1629 vpshufb $t3,@X[5],@X[5]
1630 vpaddq -0x80($Tbl),@X[0],$t0
1631 vpshufb $t3,@X[6],@X[6]
1632 vpaddq -0x60($Tbl),@X[1],$t1
1633 vpshufb $t3,@X[7],@X[7]
1634 vpaddq -0x40($Tbl),@X[2],$t2
1635 vpaddq -0x20($Tbl),@X[3],$t3
1636 vmovdqa $t0,0x00(%rsp)
1637 vpaddq 0x00($Tbl),@X[4],$t0
1638 vmovdqa $t1,0x10(%rsp)
1639 vpaddq 0x20($Tbl),@X[5],$t1
1640 vmovdqa $t2,0x20(%rsp)
1641 vpaddq 0x40($Tbl),@X[6],$t2
1642 vmovdqa $t3,0x30(%rsp)
1643 vpaddq 0x60($Tbl),@X[7],$t3
1644 vmovdqa $t0,0x40(%rsp)
1645 mov $A,$a1
1646 vmovdqa $t1,0x50(%rsp)
1647 mov $B,$a3
1648 vmovdqa $t2,0x60(%rsp)
1649 xor $C,$a3 # magic
1650 vmovdqa $t3,0x70(%rsp)
1651 mov $E,$a0
1652 jmp .Lavx_00_47
1653
1654 .align 16
1655 .Lavx_00_47:
1656 add \$`16*2*$SZ`,$Tbl
1657 ___
1658 sub Xupdate_512_AVX () {
1659 (
1660 '&vpalignr ($t0,@X[1],@X[0],$SZ)', # X[1..2]
1661 '&vpalignr ($t3,@X[5],@X[4],$SZ)', # X[9..10]
1662 '&vpsrlq ($t2,$t0,$sigma0[0])',
1663 '&vpaddq (@X[0],@X[0],$t3);', # X[0..1] += X[9..10]
1664 '&vpsrlq ($t3,$t0,$sigma0[2])',
1665 '&vpsllq ($t1,$t0,8*$SZ-$sigma0[1]);',
1666 '&vpxor ($t0,$t3,$t2)',
1667 '&vpsrlq ($t2,$t2,$sigma0[1]-$sigma0[0]);',
1668 '&vpxor ($t0,$t0,$t1)',
1669 '&vpsllq ($t1,$t1,$sigma0[1]-$sigma0[0]);',
1670 '&vpxor ($t0,$t0,$t2)',
1671 '&vpsrlq ($t3,@X[7],$sigma1[2]);',
1672 '&vpxor ($t0,$t0,$t1)', # sigma0(X[1..2])
1673 '&vpsllq ($t2,@X[7],8*$SZ-$sigma1[1]);',
1674 '&vpaddq (@X[0],@X[0],$t0)', # X[0..1] += sigma0(X[1..2])
1675 '&vpsrlq ($t1,@X[7],$sigma1[0]);',
1676 '&vpxor ($t3,$t3,$t2)',
1677 '&vpsllq ($t2,$t2,$sigma1[1]-$sigma1[0]);',
1678 '&vpxor ($t3,$t3,$t1)',
1679 '&vpsrlq ($t1,$t1,$sigma1[1]-$sigma1[0]);',
1680 '&vpxor ($t3,$t3,$t2)',
1681 '&vpxor ($t3,$t3,$t1)', # sigma1(X[14..15])
1682 '&vpaddq (@X[0],@X[0],$t3)', # X[0..1] += sigma1(X[14..15])
1683 );
1684 }
1685
1686 sub AVX_512_00_47 () {
1687 my $j = shift;
1688 my $body = shift;
1689 my @X = @_;
1690 my @insns = (&$body,&$body); # 52 instructions
1691
1692 foreach (Xupdate_512_AVX()) { # 23 instructions
1693 eval;
1694 eval(shift(@insns));
1695 eval(shift(@insns));
1696 }
1697 &vpaddq ($t2,@X[0],16*2*$j-0x80."($Tbl)");
1698 foreach (@insns) { eval; } # remaining instructions
1699 &vmovdqa (16*$j."(%rsp)",$t2);
1700 }
1701
1702 for ($i=0,$j=0; $j<8; $j++) {
1703 &AVX_512_00_47($j,\&body_00_15,@X);
1704 push(@X,shift(@X)); # rotate(@X)
1705 }
1706 &cmpb ($SZ-1+16*2*$SZ-0x80."($Tbl)",0);
1707 &jne (".Lavx_00_47");
1708
1709 for ($i=0; $i<16; ) {
1710 foreach(body_00_15()) { eval; }
1711 }
1712 }
1713 $code.=<<___;
1714 mov $_ctx,$ctx
1715 mov $a1,$A
1716
1717 add $SZ*0($ctx),$A
1718 lea 16*$SZ($inp),$inp
1719 add $SZ*1($ctx),$B
1720 add $SZ*2($ctx),$C
1721 add $SZ*3($ctx),$D
1722 add $SZ*4($ctx),$E
1723 add $SZ*5($ctx),$F
1724 add $SZ*6($ctx),$G
1725 add $SZ*7($ctx),$H
1726
1727 cmp $_end,$inp
1728
1729 mov $A,$SZ*0($ctx)
1730 mov $B,$SZ*1($ctx)
1731 mov $C,$SZ*2($ctx)
1732 mov $D,$SZ*3($ctx)
1733 mov $E,$SZ*4($ctx)
1734 mov $F,$SZ*5($ctx)
1735 mov $G,$SZ*6($ctx)
1736 mov $H,$SZ*7($ctx)
1737 jb .Lloop_avx
1738
1739 mov $_rsp,%rsi
1740 vzeroupper
1741 ___
1742 $code.=<<___ if ($win64);
1743 movaps 16*$SZ+32(%rsp),%xmm6
1744 movaps 16*$SZ+48(%rsp),%xmm7
1745 movaps 16*$SZ+64(%rsp),%xmm8
1746 movaps 16*$SZ+80(%rsp),%xmm9
1747 ___
1748 $code.=<<___ if ($win64 && $SZ>4);
1749 movaps 16*$SZ+96(%rsp),%xmm10
1750 movaps 16*$SZ+112(%rsp),%xmm11
1751 ___
1752 $code.=<<___;
1753 mov (%rsi),%r15
1754 mov 8(%rsi),%r14
1755 mov 16(%rsi),%r13
1756 mov 24(%rsi),%r12
1757 mov 32(%rsi),%rbp
1758 mov 40(%rsi),%rbx
1759 lea 48(%rsi),%rsp
1760 .Lepilogue_avx:
1761 ret
1762 .size ${func}_avx,.-${func}_avx
1763 ___
1764
1765 if ($avx>1) {{
1766 ######################################################################
1767 # AVX2+BMI code path
1768 #
1769 my $a5=$SZ==4?"%esi":"%rsi"; # zap $inp
1770 my $PUSH8=8*2*$SZ;
1771 use integer;
1772
1773 sub bodyx_00_15 () {
1774 # at start $a1 should be zero, $a3 - $b^$c and $a4 copy of $f
1775 (
1776 '($a,$b,$c,$d,$e,$f,$g,$h)=@ROT;'.
1777
1778 '&add ($h,(32*($i/(16/$SZ))+$SZ*($i%(16/$SZ)))%$PUSH8.$base)', # h+=X[i]+K[i]
1779 '&and ($a4,$e)', # f&e
1780 '&rorx ($a0,$e,$Sigma1[2])',
1781 '&rorx ($a2,$e,$Sigma1[1])',
1782
1783 '&lea ($a,"($a,$a1)")', # h+=Sigma0(a) from the past
1784 '&lea ($h,"($h,$a4)")',
1785 '&andn ($a4,$e,$g)', # ~e&g
1786 '&xor ($a0,$a2)',
1787
1788 '&rorx ($a1,$e,$Sigma1[0])',
1789 '&lea ($h,"($h,$a4)")', # h+=Ch(e,f,g)=(e&f)+(~e&g)
1790 '&xor ($a0,$a1)', # Sigma1(e)
1791 '&mov ($a2,$a)',
1792
1793 '&rorx ($a4,$a,$Sigma0[2])',
1794 '&lea ($h,"($h,$a0)")', # h+=Sigma1(e)
1795 '&xor ($a2,$b)', # a^b, b^c in next round
1796 '&rorx ($a1,$a,$Sigma0[1])',
1797
1798 '&rorx ($a0,$a,$Sigma0[0])',
1799 '&lea ($d,"($d,$h)")', # d+=h
1800 '&and ($a3,$a2)', # (b^c)&(a^b)
1801 '&xor ($a1,$a4)',
1802
1803 '&xor ($a3,$b)', # Maj(a,b,c)=Ch(a^b,c,b)
1804 '&xor ($a1,$a0)', # Sigma0(a)
1805 '&lea ($h,"($h,$a3)");'. # h+=Maj(a,b,c)
1806 '&mov ($a4,$e)', # copy of f in future
1807
1808 '($a2,$a3) = ($a3,$a2); unshift(@ROT,pop(@ROT)); $i++;'
1809 );
1810 # and at the finish one has to $a+=$a1
1811 }
1812
1813 $code.=<<___;
1814 .type ${func}_avx2,\@function,3
1815 .align 64
1816 ${func}_avx2:
1817 .Lavx2_shortcut:
1818 push %rbx
1819 push %rbp
1820 push %r12
1821 push %r13
1822 push %r14
1823 push %r15
1824 mov %rsp,%r11 # copy %rsp
1825 sub \$`2*$SZ*$rounds+4*8+$win64*16*($SZ==4?4:6)`,%rsp
1826 shl \$4,%rdx # num*16
1827 and \$-256*$SZ,%rsp # align stack frame
1828 lea ($inp,%rdx,$SZ),%rdx # inp+num*16*$SZ
1829 add \$`2*$SZ*($rounds-8)`,%rsp
1830 mov $ctx,$_ctx # save ctx, 1st arg
1831 mov $inp,$_inp # save inp, 2nd arh
1832 mov %rdx,$_end # save end pointer, "3rd" arg
1833 mov %r11,$_rsp # save copy of %rsp
1834 ___
1835 $code.=<<___ if ($win64);
1836 movaps %xmm6,16*$SZ+32(%rsp)
1837 movaps %xmm7,16*$SZ+48(%rsp)
1838 movaps %xmm8,16*$SZ+64(%rsp)
1839 movaps %xmm9,16*$SZ+80(%rsp)
1840 ___
1841 $code.=<<___ if ($win64 && $SZ>4);
1842 movaps %xmm10,16*$SZ+96(%rsp)
1843 movaps %xmm11,16*$SZ+112(%rsp)
1844 ___
1845 $code.=<<___;
1846 .Lprologue_avx2:
1847
1848 vzeroupper
1849 sub \$-16*$SZ,$inp # inp++, size optimization
1850 mov $SZ*0($ctx),$A
1851 mov $inp,%r12 # borrow $T1
1852 mov $SZ*1($ctx),$B
1853 cmp %rdx,$inp # $_end
1854 mov $SZ*2($ctx),$C
1855 cmove %rsp,%r12 # next block or random data
1856 mov $SZ*3($ctx),$D
1857 mov $SZ*4($ctx),$E
1858 mov $SZ*5($ctx),$F
1859 mov $SZ*6($ctx),$G
1860 mov $SZ*7($ctx),$H
1861 ___
1862 if ($SZ==4) { # SHA256
1863 my @X = map("%ymm$_",(0..3));
1864 my ($t0,$t1,$t2,$t3, $t4,$t5) = map("%ymm$_",(4..9));
1865
1866 $code.=<<___;
1867 vmovdqa $TABLE+`$SZ*2*$rounds`+32(%rip),$t4
1868 vmovdqa $TABLE+`$SZ*2*$rounds`+64(%rip),$t5
1869 jmp .Loop_avx2
1870 .align 16
1871 .Loop_avx2:
1872 vmovdqa $TABLE+`$SZ*2*$rounds`(%rip),$t3
1873 vmovdqu -16*$SZ+0($inp),%xmm0
1874 vmovdqu -16*$SZ+16($inp),%xmm1
1875 vmovdqu -16*$SZ+32($inp),%xmm2
1876 vmovdqu -16*$SZ+48($inp),%xmm3
1877 #mov $inp,$_inp # offload $inp
1878 vinserti128 \$1,(%r12),@X[0],@X[0]
1879 vinserti128 \$1,16(%r12),@X[1],@X[1]
1880 vpshufb $t3,@X[0],@X[0]
1881 vinserti128 \$1,32(%r12),@X[2],@X[2]
1882 vpshufb $t3,@X[1],@X[1]
1883 vinserti128 \$1,48(%r12),@X[3],@X[3]
1884
1885 lea $TABLE(%rip),$Tbl
1886 vpshufb $t3,@X[2],@X[2]
1887 vpaddd 0x00($Tbl),@X[0],$t0
1888 vpshufb $t3,@X[3],@X[3]
1889 vpaddd 0x20($Tbl),@X[1],$t1
1890 vpaddd 0x40($Tbl),@X[2],$t2
1891 vpaddd 0x60($Tbl),@X[3],$t3
1892 vmovdqa $t0,0x00(%rsp)
1893 xor $a1,$a1
1894 vmovdqa $t1,0x20(%rsp)
1895 lea -$PUSH8(%rsp),%rsp
1896 mov $B,$a3
1897 vmovdqa $t2,0x00(%rsp)
1898 xor $C,$a3 # magic
1899 vmovdqa $t3,0x20(%rsp)
1900 mov $F,$a4
1901 sub \$-16*2*$SZ,$Tbl # size optimization
1902 jmp .Lavx2_00_47
1903
1904 .align 16
1905 .Lavx2_00_47:
1906 ___
1907
1908 sub AVX2_256_00_47 () {
1909 my $j = shift;
1910 my $body = shift;
1911 my @X = @_;
1912 my @insns = (&$body,&$body,&$body,&$body); # 96 instructions
1913 my $base = "+2*$PUSH8(%rsp)";
1914
1915 &lea ("%rsp","-$PUSH8(%rsp)") if (($j%2)==0);
1916 foreach (Xupdate_256_AVX()) { # 29 instructions
1917 eval;
1918 eval(shift(@insns));
1919 eval(shift(@insns));
1920 eval(shift(@insns));
1921 }
1922 &vpaddd ($t2,@X[0],16*2*$j."($Tbl)");
1923 foreach (@insns) { eval; } # remaining instructions
1924 &vmovdqa ((32*$j)%$PUSH8."(%rsp)",$t2);
1925 }
1926
1927 for ($i=0,$j=0; $j<4; $j++) {
1928 &AVX2_256_00_47($j,\&bodyx_00_15,@X);
1929 push(@X,shift(@X)); # rotate(@X)
1930 }
1931 &lea ($Tbl,16*2*$SZ."($Tbl)");
1932 &cmpb (($SZ-1)."($Tbl)",0);
1933 &jne (".Lavx2_00_47");
1934
1935 for ($i=0; $i<16; ) {
1936 my $base=$i<8?"+$PUSH8(%rsp)":"(%rsp)";
1937 foreach(bodyx_00_15()) { eval; }
1938 }
1939 } else { # SHA512
1940 my @X = map("%ymm$_",(0..7));
1941 my ($t0,$t1,$t2,$t3) = map("%ymm$_",(8..11));
1942
1943 $code.=<<___;
1944 jmp .Loop_avx2
1945 .align 16
1946 .Loop_avx2:
1947 vmovdqu -16*$SZ($inp),%xmm0
1948 vmovdqu -16*$SZ+16($inp),%xmm1
1949 vmovdqu -16*$SZ+32($inp),%xmm2
1950 lea $TABLE+0x80(%rip),$Tbl # size optimization
1951 vmovdqu -16*$SZ+48($inp),%xmm3
1952 vmovdqu -16*$SZ+64($inp),%xmm4
1953 vmovdqu -16*$SZ+80($inp),%xmm5
1954 vmovdqu -16*$SZ+96($inp),%xmm6
1955 vmovdqu -16*$SZ+112($inp),%xmm7
1956 #mov $inp,$_inp # offload $inp
1957 vmovdqa `$SZ*2*$rounds-0x80`($Tbl),$t2
1958 vinserti128 \$1,(%r12),@X[0],@X[0]
1959 vinserti128 \$1,16(%r12),@X[1],@X[1]
1960 vpshufb $t2,@X[0],@X[0]
1961 vinserti128 \$1,32(%r12),@X[2],@X[2]
1962 vpshufb $t2,@X[1],@X[1]
1963 vinserti128 \$1,48(%r12),@X[3],@X[3]
1964 vpshufb $t2,@X[2],@X[2]
1965 vinserti128 \$1,64(%r12),@X[4],@X[4]
1966 vpshufb $t2,@X[3],@X[3]
1967 vinserti128 \$1,80(%r12),@X[5],@X[5]
1968 vpshufb $t2,@X[4],@X[4]
1969 vinserti128 \$1,96(%r12),@X[6],@X[6]
1970 vpshufb $t2,@X[5],@X[5]
1971 vinserti128 \$1,112(%r12),@X[7],@X[7]
1972
1973 vpaddq -0x80($Tbl),@X[0],$t0
1974 vpshufb $t2,@X[6],@X[6]
1975 vpaddq -0x60($Tbl),@X[1],$t1
1976 vpshufb $t2,@X[7],@X[7]
1977 vpaddq -0x40($Tbl),@X[2],$t2
1978 vpaddq -0x20($Tbl),@X[3],$t3
1979 vmovdqa $t0,0x00(%rsp)
1980 vpaddq 0x00($Tbl),@X[4],$t0
1981 vmovdqa $t1,0x20(%rsp)
1982 vpaddq 0x20($Tbl),@X[5],$t1
1983 vmovdqa $t2,0x40(%rsp)
1984 vpaddq 0x40($Tbl),@X[6],$t2
1985 vmovdqa $t3,0x60(%rsp)
1986 lea -$PUSH8(%rsp),%rsp
1987 vpaddq 0x60($Tbl),@X[7],$t3
1988 vmovdqa $t0,0x00(%rsp)
1989 xor $a1,$a1
1990 vmovdqa $t1,0x20(%rsp)
1991 mov $B,$a3
1992 vmovdqa $t2,0x40(%rsp)
1993 xor $C,$a3 # magic
1994 vmovdqa $t3,0x60(%rsp)
1995 mov $F,$a4
1996 add \$16*2*$SZ,$Tbl
1997 jmp .Lavx2_00_47
1998
1999 .align 16
2000 .Lavx2_00_47:
2001 ___
2002
2003 sub AVX2_512_00_47 () {
2004 my $j = shift;
2005 my $body = shift;
2006 my @X = @_;
2007 my @insns = (&$body,&$body); # 48 instructions
2008 my $base = "+2*$PUSH8(%rsp)";
2009
2010 &lea ("%rsp","-$PUSH8(%rsp)") if (($j%4)==0);
2011 foreach (Xupdate_512_AVX()) { # 23 instructions
2012 eval;
2013 if ($_ !~ /\;$/) {
2014 eval(shift(@insns));
2015 eval(shift(@insns));
2016 eval(shift(@insns));
2017 }
2018 }
2019 &vpaddq ($t2,@X[0],16*2*$j-0x80."($Tbl)");
2020 foreach (@insns) { eval; } # remaining instructions
2021 &vmovdqa ((32*$j)%$PUSH8."(%rsp)",$t2);
2022 }
2023
2024 for ($i=0,$j=0; $j<8; $j++) {
2025 &AVX2_512_00_47($j,\&bodyx_00_15,@X);
2026 push(@X,shift(@X)); # rotate(@X)
2027 }
2028 &lea ($Tbl,16*2*$SZ."($Tbl)");
2029 &cmpb (($SZ-1-0x80)."($Tbl)",0);
2030 &jne (".Lavx2_00_47");
2031
2032 for ($i=0; $i<16; ) {
2033 my $base=$i<8?"+$PUSH8(%rsp)":"(%rsp)";
2034 foreach(bodyx_00_15()) { eval; }
2035 }
2036 }
2037 $code.=<<___;
2038 mov `2*$SZ*$rounds`(%rsp),$ctx # $_ctx
2039 add $a1,$A
2040 #mov `2*$SZ*$rounds+8`(%rsp),$inp # $_inp
2041 lea `2*$SZ*($rounds-8)`(%rsp),$Tbl
2042
2043 add $SZ*0($ctx),$A
2044 add $SZ*1($ctx),$B
2045 add $SZ*2($ctx),$C
2046 add $SZ*3($ctx),$D
2047 add $SZ*4($ctx),$E
2048 add $SZ*5($ctx),$F
2049 add $SZ*6($ctx),$G
2050 add $SZ*7($ctx),$H
2051
2052 mov $A,$SZ*0($ctx)
2053 mov $B,$SZ*1($ctx)
2054 mov $C,$SZ*2($ctx)
2055 mov $D,$SZ*3($ctx)
2056 mov $E,$SZ*4($ctx)
2057 mov $F,$SZ*5($ctx)
2058 mov $G,$SZ*6($ctx)
2059 mov $H,$SZ*7($ctx)
2060
2061 cmp `$PUSH8+2*8`($Tbl),$inp # $_end
2062 je .Ldone_avx2
2063
2064 xor $a1,$a1
2065 mov $B,$a3
2066 xor $C,$a3 # magic
2067 mov $F,$a4
2068 jmp .Lower_avx2
2069 .align 16
2070 .Lower_avx2:
2071 ___
2072 for ($i=0; $i<8; ) {
2073 my $base="+16($Tbl)";
2074 foreach(bodyx_00_15()) { eval; }
2075 }
2076 $code.=<<___;
2077 lea -$PUSH8($Tbl),$Tbl
2078 cmp %rsp,$Tbl
2079 jae .Lower_avx2
2080
2081 mov `2*$SZ*$rounds`(%rsp),$ctx # $_ctx
2082 add $a1,$A
2083 #mov `2*$SZ*$rounds+8`(%rsp),$inp # $_inp
2084 lea `2*$SZ*($rounds-8)`(%rsp),%rsp
2085
2086 add $SZ*0($ctx),$A
2087 add $SZ*1($ctx),$B
2088 add $SZ*2($ctx),$C
2089 add $SZ*3($ctx),$D
2090 add $SZ*4($ctx),$E
2091 add $SZ*5($ctx),$F
2092 lea `2*16*$SZ`($inp),$inp # inp+=2
2093 add $SZ*6($ctx),$G
2094 mov $inp,%r12
2095 add $SZ*7($ctx),$H
2096 cmp $_end,$inp
2097
2098 mov $A,$SZ*0($ctx)
2099 cmove %rsp,%r12 # next block or stale data
2100 mov $B,$SZ*1($ctx)
2101 mov $C,$SZ*2($ctx)
2102 mov $D,$SZ*3($ctx)
2103 mov $E,$SZ*4($ctx)
2104 mov $F,$SZ*5($ctx)
2105 mov $G,$SZ*6($ctx)
2106 mov $H,$SZ*7($ctx)
2107
2108 jbe .Loop_avx2
2109 lea (%rsp),$Tbl
2110
2111 .Ldone_avx2:
2112 lea ($Tbl),%rsp
2113 mov $_rsp,%rsi
2114 vzeroupper
2115 ___
2116 $code.=<<___ if ($win64);
2117 movaps 16*$SZ+32(%rsp),%xmm6
2118 movaps 16*$SZ+48(%rsp),%xmm7
2119 movaps 16*$SZ+64(%rsp),%xmm8
2120 movaps 16*$SZ+80(%rsp),%xmm9
2121 ___
2122 $code.=<<___ if ($win64 && $SZ>4);
2123 movaps 16*$SZ+96(%rsp),%xmm10
2124 movaps 16*$SZ+112(%rsp),%xmm11
2125 ___
2126 $code.=<<___;
2127 mov (%rsi),%r15
2128 mov 8(%rsi),%r14
2129 mov 16(%rsi),%r13
2130 mov 24(%rsi),%r12
2131 mov 32(%rsi),%rbp
2132 mov 40(%rsi),%rbx
2133 lea 48(%rsi),%rsp
2134 .Lepilogue_avx2:
2135 ret
2136 .size ${func}_avx2,.-${func}_avx2
2137 ___
2138 }}
2139 }}}}}
2140
2141 # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame,
2142 # CONTEXT *context,DISPATCHER_CONTEXT *disp)
2143 if ($win64) {
2144 $rec="%rcx";
2145 $frame="%rdx";
2146 $context="%r8";
2147 $disp="%r9";
2148
2149 $code.=<<___;
2150 .extern __imp_RtlVirtualUnwind
2151 .type se_handler,\@abi-omnipotent
2152 .align 16
2153 se_handler:
2154 push %rsi
2155 push %rdi
2156 push %rbx
2157 push %rbp
2158 push %r12
2159 push %r13
2160 push %r14
2161 push %r15
2162 pushfq
2163 sub \$64,%rsp
2164
2165 mov 120($context),%rax # pull context->Rax
2166 mov 248($context),%rbx # pull context->Rip
2167
2168 mov 8($disp),%rsi # disp->ImageBase
2169 mov 56($disp),%r11 # disp->HanderlData
2170
2171 mov 0(%r11),%r10d # HandlerData[0]
2172 lea (%rsi,%r10),%r10 # prologue label
2173 cmp %r10,%rbx # context->Rip<prologue label
2174 jb .Lin_prologue
2175
2176 mov 152($context),%rax # pull context->Rsp
2177
2178 mov 4(%r11),%r10d # HandlerData[1]
2179 lea (%rsi,%r10),%r10 # epilogue label
2180 cmp %r10,%rbx # context->Rip>=epilogue label
2181 jae .Lin_prologue
2182 ___
2183 $code.=<<___ if ($avx>1);
2184 lea .Lavx2_shortcut(%rip),%r10
2185 cmp %r10,%rbx # context->Rip<avx2_shortcut
2186 jb .Lnot_in_avx2
2187
2188 and \$-256*$SZ,%rax
2189 add \$`2*$SZ*($rounds-8)`,%rax
2190 .Lnot_in_avx2:
2191 ___
2192 $code.=<<___;
2193 mov %rax,%rsi # put aside Rsp
2194 mov 16*$SZ+3*8(%rax),%rax # pull $_rsp
2195 lea 48(%rax),%rax
2196
2197 mov -8(%rax),%rbx
2198 mov -16(%rax),%rbp
2199 mov -24(%rax),%r12
2200 mov -32(%rax),%r13
2201 mov -40(%rax),%r14
2202 mov -48(%rax),%r15
2203 mov %rbx,144($context) # restore context->Rbx
2204 mov %rbp,160($context) # restore context->Rbp
2205 mov %r12,216($context) # restore context->R12
2206 mov %r13,224($context) # restore context->R13
2207 mov %r14,232($context) # restore context->R14
2208 mov %r15,240($context) # restore context->R15
2209
2210 lea .Lepilogue(%rip),%r10
2211 cmp %r10,%rbx
2212 jb .Lin_prologue # non-AVX code
2213
2214 lea 16*$SZ+4*8(%rsi),%rsi # Xmm6- save area
2215 lea 512($context),%rdi # &context.Xmm6
2216 mov \$`$SZ==4?8:12`,%ecx
2217 .long 0xa548f3fc # cld; rep movsq
2218
2219 .Lin_prologue:
2220 mov 8(%rax),%rdi
2221 mov 16(%rax),%rsi
2222 mov %rax,152($context) # restore context->Rsp
2223 mov %rsi,168($context) # restore context->Rsi
2224 mov %rdi,176($context) # restore context->Rdi
2225
2226 mov 40($disp),%rdi # disp->ContextRecord
2227 mov $context,%rsi # context
2228 mov \$154,%ecx # sizeof(CONTEXT)
2229 .long 0xa548f3fc # cld; rep movsq
2230
2231 mov $disp,%rsi
2232 xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER
2233 mov 8(%rsi),%rdx # arg2, disp->ImageBase
2234 mov 0(%rsi),%r8 # arg3, disp->ControlPc
2235 mov 16(%rsi),%r9 # arg4, disp->FunctionEntry
2236 mov 40(%rsi),%r10 # disp->ContextRecord
2237 lea 56(%rsi),%r11 # &disp->HandlerData
2238 lea 24(%rsi),%r12 # &disp->EstablisherFrame
2239 mov %r10,32(%rsp) # arg5
2240 mov %r11,40(%rsp) # arg6
2241 mov %r12,48(%rsp) # arg7
2242 mov %rcx,56(%rsp) # arg8, (NULL)
2243 call *__imp_RtlVirtualUnwind(%rip)
2244
2245 mov \$1,%eax # ExceptionContinueSearch
2246 add \$64,%rsp
2247 popfq
2248 pop %r15
2249 pop %r14
2250 pop %r13
2251 pop %r12
2252 pop %rbp
2253 pop %rbx
2254 pop %rdi
2255 pop %rsi
2256 ret
2257 .size se_handler,.-se_handler
2258 ___
2259
2260 $code.=<<___ if ($SZ==4 && $shaext);
2261 .type shaext_handler,\@abi-omnipotent
2262 .align 16
2263 shaext_handler:
2264 push %rsi
2265 push %rdi
2266 push %rbx
2267 push %rbp
2268 push %r12
2269 push %r13
2270 push %r14
2271 push %r15
2272 pushfq
2273 sub \$64,%rsp
2274
2275 mov 120($context),%rax # pull context->Rax
2276 mov 248($context),%rbx # pull context->Rip
2277
2278 lea .Lprologue_shaext(%rip),%r10
2279 cmp %r10,%rbx # context->Rip<.Lprologue
2280 jb .Lin_prologue
2281
2282 lea .Lepilogue_shaext(%rip),%r10
2283 cmp %r10,%rbx # context->Rip>=.Lepilogue
2284 jae .Lin_prologue
2285
2286 lea -8-5*16(%rax),%rsi
2287 lea 512($context),%rdi # &context.Xmm6
2288 mov \$10,%ecx
2289 .long 0xa548f3fc # cld; rep movsq
2290
2291 jmp .Lin_prologue
2292 .size shaext_handler,.-shaext_handler
2293 ___
2294
2295 $code.=<<___;
2296 .section .pdata
2297 .align 4
2298 .rva .LSEH_begin_$func
2299 .rva .LSEH_end_$func
2300 .rva .LSEH_info_$func
2301 ___
2302 $code.=<<___ if ($SZ==4 && $shaext);
2303 .rva .LSEH_begin_${func}_shaext
2304 .rva .LSEH_end_${func}_shaext
2305 .rva .LSEH_info_${func}_shaext
2306 ___
2307 $code.=<<___ if ($SZ==4);
2308 .rva .LSEH_begin_${func}_ssse3
2309 .rva .LSEH_end_${func}_ssse3
2310 .rva .LSEH_info_${func}_ssse3
2311 ___
2312 $code.=<<___ if ($avx && $SZ==8);
2313 .rva .LSEH_begin_${func}_xop
2314 .rva .LSEH_end_${func}_xop
2315 .rva .LSEH_info_${func}_xop
2316 ___
2317 $code.=<<___ if ($avx);
2318 .rva .LSEH_begin_${func}_avx
2319 .rva .LSEH_end_${func}_avx
2320 .rva .LSEH_info_${func}_avx
2321 ___
2322 $code.=<<___ if ($avx>1);
2323 .rva .LSEH_begin_${func}_avx2
2324 .rva .LSEH_end_${func}_avx2
2325 .rva .LSEH_info_${func}_avx2
2326 ___
2327 $code.=<<___;
2328 .section .xdata
2329 .align 8
2330 .LSEH_info_$func:
2331 .byte 9,0,0,0
2332 .rva se_handler
2333 .rva .Lprologue,.Lepilogue # HandlerData[]
2334 ___
2335 $code.=<<___ if ($SZ==4 && $shaext);
2336 .LSEH_info_${func}_shaext:
2337 .byte 9,0,0,0
2338 .rva shaext_handler
2339 ___
2340 $code.=<<___ if ($SZ==4);
2341 .LSEH_info_${func}_ssse3:
2342 .byte 9,0,0,0
2343 .rva se_handler
2344 .rva .Lprologue_ssse3,.Lepilogue_ssse3 # HandlerData[]
2345 ___
2346 $code.=<<___ if ($avx && $SZ==8);
2347 .LSEH_info_${func}_xop:
2348 .byte 9,0,0,0
2349 .rva se_handler
2350 .rva .Lprologue_xop,.Lepilogue_xop # HandlerData[]
2351 ___
2352 $code.=<<___ if ($avx);
2353 .LSEH_info_${func}_avx:
2354 .byte 9,0,0,0
2355 .rva se_handler
2356 .rva .Lprologue_avx,.Lepilogue_avx # HandlerData[]
2357 ___
2358 $code.=<<___ if ($avx>1);
2359 .LSEH_info_${func}_avx2:
2360 .byte 9,0,0,0
2361 .rva se_handler
2362 .rva .Lprologue_avx2,.Lepilogue_avx2 # HandlerData[]
2363 ___
2364 }
2365
2366 sub sha256op38 {
2367 my $instr = shift;
2368 my %opcodelet = (
2369 "sha256rnds2" => 0xcb,
2370 "sha256msg1" => 0xcc,
2371 "sha256msg2" => 0xcd );
2372
2373 if (defined($opcodelet{$instr}) && @_[0] =~ /%xmm([0-7]),\s*%xmm([0-7])/) {
2374 my @opcode=(0x0f,0x38);
2375 push @opcode,$opcodelet{$instr};
2376 push @opcode,0xc0|($1&7)|(($2&7)<<3); # ModR/M
2377 return ".byte\t".join(',',@opcode);
2378 } else {
2379 return $instr."\t".@_[0];
2380 }
2381 }
2382
2383 foreach (split("\n",$code)) {
2384 s/\`([^\`]*)\`/eval $1/geo;
2385
2386 s/\b(sha256[^\s]*)\s+(.*)/sha256op38($1,$2)/geo;
2387
2388 print $_,"\n";
2389 }
2390 close STDOUT;