Debian Codegraph @

external/boringssl/crypto/bn/asm/rsaz-x86_64.pl

raw
1 #!/usr/bin/env perl
2
3 ##############################################################################
4 # #
5 # Copyright (c) 2012, Intel Corporation #
6 # #
7 # All rights reserved. #
8 # #
9 # Redistribution and use in source and binary forms, with or without #
10 # modification, are permitted provided that the following conditions are #
11 # met: #
12 # #
13 # * Redistributions of source code must retain the above copyright #
14 # notice, this list of conditions and the following disclaimer. #
15 # #
16 # * Redistributions in binary form must reproduce the above copyright #
17 # notice, this list of conditions and the following disclaimer in the #
18 # documentation and/or other materials provided with the #
19 # distribution. #
20 # #
21 # * Neither the name of the Intel Corporation nor the names of its #
22 # contributors may be used to endorse or promote products derived from #
23 # this software without specific prior written permission. #
24 # #
25 # #
26 # THIS SOFTWARE IS PROVIDED BY INTEL CORPORATION ""AS IS"" AND ANY #
27 # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE #
28 # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR #
29 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL INTEL CORPORATION OR #
30 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, #
31 # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, #
32 # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR #
33 # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF #
34 # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING #
35 # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS #
36 # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #
37 # #
38 ##############################################################################
39 # Developers and authors: #
40 # Shay Gueron (1, 2), and Vlad Krasnov (1) #
41 # (1) Intel Architecture Group, Microprocessor and Chipset Development, #
42 # Israel Development Center, Haifa, Israel #
43 # (2) University of Haifa #
44 ##############################################################################
45 # Reference: #
46 # [1] S. Gueron, "Efficient Software Implementations of Modular #
47 # Exponentiation", http://eprint.iacr.org/2011/239 #
48 # [2] S. Gueron, V. Krasnov. "Speeding up Big-Numbers Squaring". #
49 # IEEE Proceedings of 9th International Conference on Information #
50 # Technology: New Generations (ITNG 2012), 821-823 (2012). #
51 # [3] S. Gueron, Efficient Software Implementations of Modular Exponentiation#
52 # Journal of Cryptographic Engineering 2:31-43 (2012). #
53 # [4] S. Gueron, V. Krasnov: "[PATCH] Efficient and side channel analysis #
54 # resistant 512-bit and 1024-bit modular exponentiation for optimizing #
55 # RSA1024 and RSA2048 on x86_64 platforms", #
56 # http://rt.openssl.org/Ticket/Display.html?id=2582&user=guest&pass=guest#
57 ##############################################################################
58
59 # While original submission covers 512- and 1024-bit exponentiation,
60 # this module is limited to 512-bit version only (and as such
61 # accelerates RSA1024 sign). This is because improvement for longer
62 # keys is not high enough to justify the effort, highest measured
63 # was ~5% on Westmere. [This is relative to OpenSSL 1.0.2, upcoming
64 # for the moment of this writing!] Nor does this module implement
65 # "monolithic" complete exponentiation jumbo-subroutine, but adheres
66 # to more modular mixture of C and assembly. And it's optimized even
67 # for processors other than Intel Core family (see table below for
68 # improvement coefficients).
69 # <[email protected]>
70 #
71 # RSA1024 sign/sec this/original |this/rsax(*) this/fips(*)
72 # ----------------+---------------------------
73 # Opteron +13% |+5% +20%
74 # Bulldozer -0% |-1% +10%
75 # P4 +11% |+7% +8%
76 # Westmere +5% |+14% +17%
77 # Sandy Bridge +2% |+12% +29%
78 # Ivy Bridge +1% |+11% +35%
79 # Haswell(**) -0% |+12% +39%
80 # Atom +13% |+11% +4%
81 # VIA Nano +70% |+9% +25%
82 #
83 # (*) rsax engine and fips numbers are presented for reference
84 # purposes;
85 # (**) MULX was attempted, but found to give only marginal improvement;
86
87 $flavour = shift;
88 $output = shift;
89 if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }
90
91 $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
92
93 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
94 ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
95 ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
96 die "can't locate x86_64-xlate.pl";
97
98 open OUT,"| \"$^X\" $xlate $flavour $output";
99 *STDOUT=*OUT;
100
101 # In upstream, this is controlled by shelling out to the compiler to check
102 # versions, but BoringSSL is intended to be used with pre-generated perlasm
103 # output, so this isn't useful anyway.
104 #
105 # TODO(davidben): Enable this after testing. $addx goes up to 1.
106 $addx = 0;
107
108 ($out, $inp, $mod) = ("%rdi", "%rsi", "%rbp"); # common internal API
109 {
110 my ($out,$inp,$mod,$n0,$times) = ("%rdi","%rsi","%rdx","%rcx","%r8d");
111
112 $code.=<<___;
113 .text
114
115 .extern OPENSSL_ia32cap_P
116
117 .globl rsaz_512_sqr
118 .type rsaz_512_sqr,\@function,5
119 .align 32
120 rsaz_512_sqr: # 25-29% faster than rsaz_512_mul
121 push %rbx
122 push %rbp
123 push %r12
124 push %r13
125 push %r14
126 push %r15
127
128 subq \$128+24, %rsp
129 .Lsqr_body:
130 movq $mod, %rbp # common argument
131 movq ($inp), %rdx
132 movq 8($inp), %rax
133 movq $n0, 128(%rsp)
134 ___
135 $code.=<<___ if ($addx);
136 movl \$0x80100,%r11d
137 andl OPENSSL_ia32cap_P+8(%rip),%r11d
138 cmpl \$0x80100,%r11d # check for MULX and ADO/CX
139 je .Loop_sqrx
140 ___
141 $code.=<<___;
142 jmp .Loop_sqr
143
144 .align 32
145 .Loop_sqr:
146 movl $times,128+8(%rsp)
147 #first iteration
148 movq %rdx, %rbx
149 mulq %rdx
150 movq %rax, %r8
151 movq 16($inp), %rax
152 movq %rdx, %r9
153
154 mulq %rbx
155 addq %rax, %r9
156 movq 24($inp), %rax
157 movq %rdx, %r10
158 adcq \$0, %r10
159
160 mulq %rbx
161 addq %rax, %r10
162 movq 32($inp), %rax
163 movq %rdx, %r11
164 adcq \$0, %r11
165
166 mulq %rbx
167 addq %rax, %r11
168 movq 40($inp), %rax
169 movq %rdx, %r12
170 adcq \$0, %r12
171
172 mulq %rbx
173 addq %rax, %r12
174 movq 48($inp), %rax
175 movq %rdx, %r13
176 adcq \$0, %r13
177
178 mulq %rbx
179 addq %rax, %r13
180 movq 56($inp), %rax
181 movq %rdx, %r14
182 adcq \$0, %r14
183
184 mulq %rbx
185 addq %rax, %r14
186 movq %rbx, %rax
187 movq %rdx, %r15
188 adcq \$0, %r15
189
190 addq %r8, %r8 #shlq \$1, %r8
191 movq %r9, %rcx
192 adcq %r9, %r9 #shld \$1, %r8, %r9
193
194 mulq %rax
195 movq %rax, (%rsp)
196 addq %rdx, %r8
197 adcq \$0, %r9
198
199 movq %r8, 8(%rsp)
200 shrq \$63, %rcx
201
202 #second iteration
203 movq 8($inp), %r8
204 movq 16($inp), %rax
205 mulq %r8
206 addq %rax, %r10
207 movq 24($inp), %rax
208 movq %rdx, %rbx
209 adcq \$0, %rbx
210
211 mulq %r8
212 addq %rax, %r11
213 movq 32($inp), %rax
214 adcq \$0, %rdx
215 addq %rbx, %r11
216 movq %rdx, %rbx
217 adcq \$0, %rbx
218
219 mulq %r8
220 addq %rax, %r12
221 movq 40($inp), %rax
222 adcq \$0, %rdx
223 addq %rbx, %r12
224 movq %rdx, %rbx
225 adcq \$0, %rbx
226
227 mulq %r8
228 addq %rax, %r13
229 movq 48($inp), %rax
230 adcq \$0, %rdx
231 addq %rbx, %r13
232 movq %rdx, %rbx
233 adcq \$0, %rbx
234
235 mulq %r8
236 addq %rax, %r14
237 movq 56($inp), %rax
238 adcq \$0, %rdx
239 addq %rbx, %r14
240 movq %rdx, %rbx
241 adcq \$0, %rbx
242
243 mulq %r8
244 addq %rax, %r15
245 movq %r8, %rax
246 adcq \$0, %rdx
247 addq %rbx, %r15
248 movq %rdx, %r8
249 movq %r10, %rdx
250 adcq \$0, %r8
251
252 add %rdx, %rdx
253 lea (%rcx,%r10,2), %r10 #shld \$1, %rcx, %r10
254 movq %r11, %rbx
255 adcq %r11, %r11 #shld \$1, %r10, %r11
256
257 mulq %rax
258 addq %rax, %r9
259 adcq %rdx, %r10
260 adcq \$0, %r11
261
262 movq %r9, 16(%rsp)
263 movq %r10, 24(%rsp)
264 shrq \$63, %rbx
265
266 #third iteration
267 movq 16($inp), %r9
268 movq 24($inp), %rax
269 mulq %r9
270 addq %rax, %r12
271 movq 32($inp), %rax
272 movq %rdx, %rcx
273 adcq \$0, %rcx
274
275 mulq %r9
276 addq %rax, %r13
277 movq 40($inp), %rax
278 adcq \$0, %rdx
279 addq %rcx, %r13
280 movq %rdx, %rcx
281 adcq \$0, %rcx
282
283 mulq %r9
284 addq %rax, %r14
285 movq 48($inp), %rax
286 adcq \$0, %rdx
287 addq %rcx, %r14
288 movq %rdx, %rcx
289 adcq \$0, %rcx
290
291 mulq %r9
292 movq %r12, %r10
293 lea (%rbx,%r12,2), %r12 #shld \$1, %rbx, %r12
294 addq %rax, %r15
295 movq 56($inp), %rax
296 adcq \$0, %rdx
297 addq %rcx, %r15
298 movq %rdx, %rcx
299 adcq \$0, %rcx
300
301 mulq %r9
302 shrq \$63, %r10
303 addq %rax, %r8
304 movq %r9, %rax
305 adcq \$0, %rdx
306 addq %rcx, %r8
307 movq %rdx, %r9
308 adcq \$0, %r9
309
310 movq %r13, %rcx
311 leaq (%r10,%r13,2), %r13 #shld \$1, %r12, %r13
312
313 mulq %rax
314 addq %rax, %r11
315 adcq %rdx, %r12
316 adcq \$0, %r13
317
318 movq %r11, 32(%rsp)
319 movq %r12, 40(%rsp)
320 shrq \$63, %rcx
321
322 #fourth iteration
323 movq 24($inp), %r10
324 movq 32($inp), %rax
325 mulq %r10
326 addq %rax, %r14
327 movq 40($inp), %rax
328 movq %rdx, %rbx
329 adcq \$0, %rbx
330
331 mulq %r10
332 addq %rax, %r15
333 movq 48($inp), %rax
334 adcq \$0, %rdx
335 addq %rbx, %r15
336 movq %rdx, %rbx
337 adcq \$0, %rbx
338
339 mulq %r10
340 movq %r14, %r12
341 leaq (%rcx,%r14,2), %r14 #shld \$1, %rcx, %r14
342 addq %rax, %r8
343 movq 56($inp), %rax
344 adcq \$0, %rdx
345 addq %rbx, %r8
346 movq %rdx, %rbx
347 adcq \$0, %rbx
348
349 mulq %r10
350 shrq \$63, %r12
351 addq %rax, %r9
352 movq %r10, %rax
353 adcq \$0, %rdx
354 addq %rbx, %r9
355 movq %rdx, %r10
356 adcq \$0, %r10
357
358 movq %r15, %rbx
359 leaq (%r12,%r15,2),%r15 #shld \$1, %r14, %r15
360
361 mulq %rax
362 addq %rax, %r13
363 adcq %rdx, %r14
364 adcq \$0, %r15
365
366 movq %r13, 48(%rsp)
367 movq %r14, 56(%rsp)
368 shrq \$63, %rbx
369
370 #fifth iteration
371 movq 32($inp), %r11
372 movq 40($inp), %rax
373 mulq %r11
374 addq %rax, %r8
375 movq 48($inp), %rax
376 movq %rdx, %rcx
377 adcq \$0, %rcx
378
379 mulq %r11
380 addq %rax, %r9
381 movq 56($inp), %rax
382 adcq \$0, %rdx
383 movq %r8, %r12
384 leaq (%rbx,%r8,2), %r8 #shld \$1, %rbx, %r8
385 addq %rcx, %r9
386 movq %rdx, %rcx
387 adcq \$0, %rcx
388
389 mulq %r11
390 shrq \$63, %r12
391 addq %rax, %r10
392 movq %r11, %rax
393 adcq \$0, %rdx
394 addq %rcx, %r10
395 movq %rdx, %r11
396 adcq \$0, %r11
397
398 movq %r9, %rcx
399 leaq (%r12,%r9,2), %r9 #shld \$1, %r8, %r9
400
401 mulq %rax
402 addq %rax, %r15
403 adcq %rdx, %r8
404 adcq \$0, %r9
405
406 movq %r15, 64(%rsp)
407 movq %r8, 72(%rsp)
408 shrq \$63, %rcx
409
410 #sixth iteration
411 movq 40($inp), %r12
412 movq 48($inp), %rax
413 mulq %r12
414 addq %rax, %r10
415 movq 56($inp), %rax
416 movq %rdx, %rbx
417 adcq \$0, %rbx
418
419 mulq %r12
420 addq %rax, %r11
421 movq %r12, %rax
422 movq %r10, %r15
423 leaq (%rcx,%r10,2), %r10 #shld \$1, %rcx, %r10
424 adcq \$0, %rdx
425 shrq \$63, %r15
426 addq %rbx, %r11
427 movq %rdx, %r12
428 adcq \$0, %r12
429
430 movq %r11, %rbx
431 leaq (%r15,%r11,2), %r11 #shld \$1, %r10, %r11
432
433 mulq %rax
434 addq %rax, %r9
435 adcq %rdx, %r10
436 adcq \$0, %r11
437
438 movq %r9, 80(%rsp)
439 movq %r10, 88(%rsp)
440
441 #seventh iteration
442 movq 48($inp), %r13
443 movq 56($inp), %rax
444 mulq %r13
445 addq %rax, %r12
446 movq %r13, %rax
447 movq %rdx, %r13
448 adcq \$0, %r13
449
450 xorq %r14, %r14
451 shlq \$1, %rbx
452 adcq %r12, %r12 #shld \$1, %rbx, %r12
453 adcq %r13, %r13 #shld \$1, %r12, %r13
454 adcq %r14, %r14 #shld \$1, %r13, %r14
455
456 mulq %rax
457 addq %rax, %r11
458 adcq %rdx, %r12
459 adcq \$0, %r13
460
461 movq %r11, 96(%rsp)
462 movq %r12, 104(%rsp)
463
464 #eighth iteration
465 movq 56($inp), %rax
466 mulq %rax
467 addq %rax, %r13
468 adcq \$0, %rdx
469
470 addq %rdx, %r14
471
472 movq %r13, 112(%rsp)
473 movq %r14, 120(%rsp)
474
475 movq (%rsp), %r8
476 movq 8(%rsp), %r9
477 movq 16(%rsp), %r10
478 movq 24(%rsp), %r11
479 movq 32(%rsp), %r12
480 movq 40(%rsp), %r13
481 movq 48(%rsp), %r14
482 movq 56(%rsp), %r15
483
484 call __rsaz_512_reduce
485
486 addq 64(%rsp), %r8
487 adcq 72(%rsp), %r9
488 adcq 80(%rsp), %r10
489 adcq 88(%rsp), %r11
490 adcq 96(%rsp), %r12
491 adcq 104(%rsp), %r13
492 adcq 112(%rsp), %r14
493 adcq 120(%rsp), %r15
494 sbbq %rcx, %rcx
495
496 call __rsaz_512_subtract
497
498 movq %r8, %rdx
499 movq %r9, %rax
500 movl 128+8(%rsp), $times
501 movq $out, $inp
502
503 decl $times
504 jnz .Loop_sqr
505 ___
506 if ($addx) {
507 $code.=<<___;
508 jmp .Lsqr_tail
509
510 .align 32
511 .Loop_sqrx:
512 movl $times,128+8(%rsp)
513 movq $out, %xmm0 # off-load
514 movq %rbp, %xmm1 # off-load
515 #first iteration
516 mulx %rax, %r8, %r9
517
518 mulx 16($inp), %rcx, %r10
519 xor %rbp, %rbp # cf=0, of=0
520
521 mulx 24($inp), %rax, %r11
522 adcx %rcx, %r9
523
524 mulx 32($inp), %rcx, %r12
525 adcx %rax, %r10
526
527 mulx 40($inp), %rax, %r13
528 adcx %rcx, %r11
529
530 .byte 0xc4,0x62,0xf3,0xf6,0xb6,0x30,0x00,0x00,0x00 # mulx 48($inp), %rcx, %r14
531 adcx %rax, %r12
532 adcx %rcx, %r13
533
534 .byte 0xc4,0x62,0xfb,0xf6,0xbe,0x38,0x00,0x00,0x00 # mulx 56($inp), %rax, %r15
535 adcx %rax, %r14
536 adcx %rbp, %r15 # %rbp is 0
537
538 mov %r9, %rcx
539 shld \$1, %r8, %r9
540 shl \$1, %r8
541
542 xor %ebp, %ebp
543 mulx %rdx, %rax, %rdx
544 adcx %rdx, %r8
545 mov 8($inp), %rdx
546 adcx %rbp, %r9
547
548 mov %rax, (%rsp)
549 mov %r8, 8(%rsp)
550
551 #second iteration
552 mulx 16($inp), %rax, %rbx
553 adox %rax, %r10
554 adcx %rbx, %r11
555
556 .byte 0xc4,0x62,0xc3,0xf6,0x86,0x18,0x00,0x00,0x00 # mulx 24($inp), $out, %r8
557 adox $out, %r11
558 adcx %r8, %r12
559
560 mulx 32($inp), %rax, %rbx
561 adox %rax, %r12
562 adcx %rbx, %r13
563
564 mulx 40($inp), $out, %r8
565 adox $out, %r13
566 adcx %r8, %r14
567
568 .byte 0xc4,0xe2,0xfb,0xf6,0x9e,0x30,0x00,0x00,0x00 # mulx 48($inp), %rax, %rbx
569 adox %rax, %r14
570 adcx %rbx, %r15
571
572 .byte 0xc4,0x62,0xc3,0xf6,0x86,0x38,0x00,0x00,0x00 # mulx 56($inp), $out, %r8
573 adox $out, %r15
574 adcx %rbp, %r8
575 adox %rbp, %r8
576
577 mov %r11, %rbx
578 shld \$1, %r10, %r11
579 shld \$1, %rcx, %r10
580
581 xor %ebp,%ebp
582 mulx %rdx, %rax, %rcx
583 mov 16($inp), %rdx
584 adcx %rax, %r9
585 adcx %rcx, %r10
586 adcx %rbp, %r11
587
588 mov %r9, 16(%rsp)
589 .byte 0x4c,0x89,0x94,0x24,0x18,0x00,0x00,0x00 # mov %r10, 24(%rsp)
590
591 #third iteration
592 .byte 0xc4,0x62,0xc3,0xf6,0x8e,0x18,0x00,0x00,0x00 # mulx 24($inp), $out, %r9
593 adox $out, %r12
594 adcx %r9, %r13
595
596 mulx 32($inp), %rax, %rcx
597 adox %rax, %r13
598 adcx %rcx, %r14
599
600 mulx 40($inp), $out, %r9
601 adox $out, %r14
602 adcx %r9, %r15
603
604 .byte 0xc4,0xe2,0xfb,0xf6,0x8e,0x30,0x00,0x00,0x00 # mulx 48($inp), %rax, %rcx
605 adox %rax, %r15
606 adcx %rcx, %r8
607
608 .byte 0xc4,0x62,0xc3,0xf6,0x8e,0x38,0x00,0x00,0x00 # mulx 56($inp), $out, %r9
609 adox $out, %r8
610 adcx %rbp, %r9
611 adox %rbp, %r9
612
613 mov %r13, %rcx
614 shld \$1, %r12, %r13
615 shld \$1, %rbx, %r12
616
617 xor %ebp, %ebp
618 mulx %rdx, %rax, %rdx
619 adcx %rax, %r11
620 adcx %rdx, %r12
621 mov 24($inp), %rdx
622 adcx %rbp, %r13
623
624 mov %r11, 32(%rsp)
625 .byte 0x4c,0x89,0xa4,0x24,0x28,0x00,0x00,0x00 # mov %r12, 40(%rsp)
626
627 #fourth iteration
628 .byte 0xc4,0xe2,0xfb,0xf6,0x9e,0x20,0x00,0x00,0x00 # mulx 32($inp), %rax, %rbx
629 adox %rax, %r14
630 adcx %rbx, %r15
631
632 mulx 40($inp), $out, %r10
633 adox $out, %r15
634 adcx %r10, %r8
635
636 mulx 48($inp), %rax, %rbx
637 adox %rax, %r8
638 adcx %rbx, %r9
639
640 mulx 56($inp), $out, %r10
641 adox $out, %r9
642 adcx %rbp, %r10
643 adox %rbp, %r10
644
645 .byte 0x66
646 mov %r15, %rbx
647 shld \$1, %r14, %r15
648 shld \$1, %rcx, %r14
649
650 xor %ebp, %ebp
651 mulx %rdx, %rax, %rdx
652 adcx %rax, %r13
653 adcx %rdx, %r14
654 mov 32($inp), %rdx
655 adcx %rbp, %r15
656
657 mov %r13, 48(%rsp)
658 mov %r14, 56(%rsp)
659
660 #fifth iteration
661 .byte 0xc4,0x62,0xc3,0xf6,0x9e,0x28,0x00,0x00,0x00 # mulx 40($inp), $out, %r11
662 adox $out, %r8
663 adcx %r11, %r9
664
665 mulx 48($inp), %rax, %rcx
666 adox %rax, %r9
667 adcx %rcx, %r10
668
669 mulx 56($inp), $out, %r11
670 adox $out, %r10
671 adcx %rbp, %r11
672 adox %rbp, %r11
673
674 mov %r9, %rcx
675 shld \$1, %r8, %r9
676 shld \$1, %rbx, %r8
677
678 xor %ebp, %ebp
679 mulx %rdx, %rax, %rdx
680 adcx %rax, %r15
681 adcx %rdx, %r8
682 mov 40($inp), %rdx
683 adcx %rbp, %r9
684
685 mov %r15, 64(%rsp)
686 mov %r8, 72(%rsp)
687
688 #sixth iteration
689 .byte 0xc4,0xe2,0xfb,0xf6,0x9e,0x30,0x00,0x00,0x00 # mulx 48($inp), %rax, %rbx
690 adox %rax, %r10
691 adcx %rbx, %r11
692
693 .byte 0xc4,0x62,0xc3,0xf6,0xa6,0x38,0x00,0x00,0x00 # mulx 56($inp), $out, %r12
694 adox $out, %r11
695 adcx %rbp, %r12
696 adox %rbp, %r12
697
698 mov %r11, %rbx
699 shld \$1, %r10, %r11
700 shld \$1, %rcx, %r10
701
702 xor %ebp, %ebp
703 mulx %rdx, %rax, %rdx
704 adcx %rax, %r9
705 adcx %rdx, %r10
706 mov 48($inp), %rdx
707 adcx %rbp, %r11
708
709 mov %r9, 80(%rsp)
710 mov %r10, 88(%rsp)
711
712 #seventh iteration
713 .byte 0xc4,0x62,0xfb,0xf6,0xae,0x38,0x00,0x00,0x00 # mulx 56($inp), %rax, %r13
714 adox %rax, %r12
715 adox %rbp, %r13
716
717 xor %r14, %r14
718 shld \$1, %r13, %r14
719 shld \$1, %r12, %r13
720 shld \$1, %rbx, %r12
721
722 xor %ebp, %ebp
723 mulx %rdx, %rax, %rdx
724 adcx %rax, %r11
725 adcx %rdx, %r12
726 mov 56($inp), %rdx
727 adcx %rbp, %r13
728
729 .byte 0x4c,0x89,0x9c,0x24,0x60,0x00,0x00,0x00 # mov %r11, 96(%rsp)
730 .byte 0x4c,0x89,0xa4,0x24,0x68,0x00,0x00,0x00 # mov %r12, 104(%rsp)
731
732 #eighth iteration
733 mulx %rdx, %rax, %rdx
734 adox %rax, %r13
735 adox %rbp, %rdx
736
737 .byte 0x66
738 add %rdx, %r14
739
740 movq %r13, 112(%rsp)
741 movq %r14, 120(%rsp)
742 movq %xmm0, $out
743 movq %xmm1, %rbp
744
745 movq 128(%rsp), %rdx # pull $n0
746 movq (%rsp), %r8
747 movq 8(%rsp), %r9
748 movq 16(%rsp), %r10
749 movq 24(%rsp), %r11
750 movq 32(%rsp), %r12
751 movq 40(%rsp), %r13
752 movq 48(%rsp), %r14
753 movq 56(%rsp), %r15
754
755 call __rsaz_512_reducex
756
757 addq 64(%rsp), %r8
758 adcq 72(%rsp), %r9
759 adcq 80(%rsp), %r10
760 adcq 88(%rsp), %r11
761 adcq 96(%rsp), %r12
762 adcq 104(%rsp), %r13
763 adcq 112(%rsp), %r14
764 adcq 120(%rsp), %r15
765 sbbq %rcx, %rcx
766
767 call __rsaz_512_subtract
768
769 movq %r8, %rdx
770 movq %r9, %rax
771 movl 128+8(%rsp), $times
772 movq $out, $inp
773
774 decl $times
775 jnz .Loop_sqrx
776
777 .Lsqr_tail:
778 ___
779 }
780 $code.=<<___;
781
782 leaq 128+24+48(%rsp), %rax
783 movq -48(%rax), %r15
784 movq -40(%rax), %r14
785 movq -32(%rax), %r13
786 movq -24(%rax), %r12
787 movq -16(%rax), %rbp
788 movq -8(%rax), %rbx
789 leaq (%rax), %rsp
790 .Lsqr_epilogue:
791 ret
792 .size rsaz_512_sqr,.-rsaz_512_sqr
793 ___
794 }
795 {
796 my ($out,$ap,$bp,$mod,$n0) = ("%rdi","%rsi","%rdx","%rcx","%r8");
797 $code.=<<___;
798 .globl rsaz_512_mul
799 .type rsaz_512_mul,\@function,5
800 .align 32
801 rsaz_512_mul:
802 push %rbx
803 push %rbp
804 push %r12
805 push %r13
806 push %r14
807 push %r15
808
809 subq \$128+24, %rsp
810 .Lmul_body:
811 movq $out, %xmm0 # off-load arguments
812 movq $mod, %xmm1
813 movq $n0, 128(%rsp)
814 ___
815 $code.=<<___ if ($addx);
816 movl \$0x80100,%r11d
817 andl OPENSSL_ia32cap_P+8(%rip),%r11d
818 cmpl \$0x80100,%r11d # check for MULX and ADO/CX
819 je .Lmulx
820 ___
821 $code.=<<___;
822 movq ($bp), %rbx # pass b[0]
823 movq $bp, %rbp # pass argument
824 call __rsaz_512_mul
825
826 movq %xmm0, $out
827 movq %xmm1, %rbp
828
829 movq (%rsp), %r8
830 movq 8(%rsp), %r9
831 movq 16(%rsp), %r10
832 movq 24(%rsp), %r11
833 movq 32(%rsp), %r12
834 movq 40(%rsp), %r13
835 movq 48(%rsp), %r14
836 movq 56(%rsp), %r15
837
838 call __rsaz_512_reduce
839 ___
840 $code.=<<___ if ($addx);
841 jmp .Lmul_tail
842
843 .align 32
844 .Lmulx:
845 movq $bp, %rbp # pass argument
846 movq ($bp), %rdx # pass b[0]
847 call __rsaz_512_mulx
848
849 movq %xmm0, $out
850 movq %xmm1, %rbp
851
852 movq 128(%rsp), %rdx # pull $n0
853 movq (%rsp), %r8
854 movq 8(%rsp), %r9
855 movq 16(%rsp), %r10
856 movq 24(%rsp), %r11
857 movq 32(%rsp), %r12
858 movq 40(%rsp), %r13
859 movq 48(%rsp), %r14
860 movq 56(%rsp), %r15
861
862 call __rsaz_512_reducex
863 .Lmul_tail:
864 ___
865 $code.=<<___;
866 addq 64(%rsp), %r8
867 adcq 72(%rsp), %r9
868 adcq 80(%rsp), %r10
869 adcq 88(%rsp), %r11
870 adcq 96(%rsp), %r12
871 adcq 104(%rsp), %r13
872 adcq 112(%rsp), %r14
873 adcq 120(%rsp), %r15
874 sbbq %rcx, %rcx
875
876 call __rsaz_512_subtract
877
878 leaq 128+24+48(%rsp), %rax
879 movq -48(%rax), %r15
880 movq -40(%rax), %r14
881 movq -32(%rax), %r13
882 movq -24(%rax), %r12
883 movq -16(%rax), %rbp
884 movq -8(%rax), %rbx
885 leaq (%rax), %rsp
886 .Lmul_epilogue:
887 ret
888 .size rsaz_512_mul,.-rsaz_512_mul
889 ___
890 }
891 {
892 my ($out,$ap,$bp,$mod,$n0,$pwr) = ("%rdi","%rsi","%rdx","%rcx","%r8","%r9d");
893 $code.=<<___;
894 .globl rsaz_512_mul_gather4
895 .type rsaz_512_mul_gather4,\@function,6
896 .align 32
897 rsaz_512_mul_gather4:
898 push %rbx
899 push %rbp
900 push %r12
901 push %r13
902 push %r14
903 push %r15
904
905 subq \$`128+24+($win64?0xb0:0)`, %rsp
906 ___
907 $code.=<<___ if ($win64);
908 movaps %xmm6,0xa0(%rsp)
909 movaps %xmm7,0xb0(%rsp)
910 movaps %xmm8,0xc0(%rsp)
911 movaps %xmm9,0xd0(%rsp)
912 movaps %xmm10,0xe0(%rsp)
913 movaps %xmm11,0xf0(%rsp)
914 movaps %xmm12,0x100(%rsp)
915 movaps %xmm13,0x110(%rsp)
916 movaps %xmm14,0x120(%rsp)
917 movaps %xmm15,0x130(%rsp)
918 ___
919 $code.=<<___;
920 .Lmul_gather4_body:
921 movd $pwr,%xmm8
922 movdqa .Linc+16(%rip),%xmm1 # 00000002000000020000000200000002
923 movdqa .Linc(%rip),%xmm0 # 00000001000000010000000000000000
924
925 pshufd \$0,%xmm8,%xmm8 # broadcast $power
926 movdqa %xmm1,%xmm7
927 movdqa %xmm1,%xmm2
928 ___
929 ########################################################################
930 # calculate mask by comparing 0..15 to $power
931 #
932 for($i=0;$i<4;$i++) {
933 $code.=<<___;
934 paddd %xmm`$i`,%xmm`$i+1`
935 pcmpeqd %xmm8,%xmm`$i`
936 movdqa %xmm7,%xmm`$i+3`
937 ___
938 }
939 for(;$i<7;$i++) {
940 $code.=<<___;
941 paddd %xmm`$i`,%xmm`$i+1`
942 pcmpeqd %xmm8,%xmm`$i`
943 ___
944 }
945 $code.=<<___;
946 pcmpeqd %xmm8,%xmm7
947
948 movdqa 16*0($bp),%xmm8
949 movdqa 16*1($bp),%xmm9
950 movdqa 16*2($bp),%xmm10
951 movdqa 16*3($bp),%xmm11
952 pand %xmm0,%xmm8
953 movdqa 16*4($bp),%xmm12
954 pand %xmm1,%xmm9
955 movdqa 16*5($bp),%xmm13
956 pand %xmm2,%xmm10
957 movdqa 16*6($bp),%xmm14
958 pand %xmm3,%xmm11
959 movdqa 16*7($bp),%xmm15
960 leaq 128($bp), %rbp
961 pand %xmm4,%xmm12
962 pand %xmm5,%xmm13
963 pand %xmm6,%xmm14
964 pand %xmm7,%xmm15
965 por %xmm10,%xmm8
966 por %xmm11,%xmm9
967 por %xmm12,%xmm8
968 por %xmm13,%xmm9
969 por %xmm14,%xmm8
970 por %xmm15,%xmm9
971
972 por %xmm9,%xmm8
973 pshufd \$0x4e,%xmm8,%xmm9
974 por %xmm9,%xmm8
975 ___
976 $code.=<<___ if ($addx);
977 movl \$0x80100,%r11d
978 andl OPENSSL_ia32cap_P+8(%rip),%r11d
979 cmpl \$0x80100,%r11d # check for MULX and ADO/CX
980 je .Lmulx_gather
981 ___
982 $code.=<<___;
983 movq %xmm8,%rbx
984
985 movq $n0, 128(%rsp) # off-load arguments
986 movq $out, 128+8(%rsp)
987 movq $mod, 128+16(%rsp)
988
989 movq ($ap), %rax
990 movq 8($ap), %rcx
991 mulq %rbx # 0 iteration
992 movq %rax, (%rsp)
993 movq %rcx, %rax
994 movq %rdx, %r8
995
996 mulq %rbx
997 addq %rax, %r8
998 movq 16($ap), %rax
999 movq %rdx, %r9
1000 adcq \$0, %r9
1001
1002 mulq %rbx
1003 addq %rax, %r9
1004 movq 24($ap), %rax
1005 movq %rdx, %r10
1006 adcq \$0, %r10
1007
1008 mulq %rbx
1009 addq %rax, %r10
1010 movq 32($ap), %rax
1011 movq %rdx, %r11
1012 adcq \$0, %r11
1013
1014 mulq %rbx
1015 addq %rax, %r11
1016 movq 40($ap), %rax
1017 movq %rdx, %r12
1018 adcq \$0, %r12
1019
1020 mulq %rbx
1021 addq %rax, %r12
1022 movq 48($ap), %rax
1023 movq %rdx, %r13
1024 adcq \$0, %r13
1025
1026 mulq %rbx
1027 addq %rax, %r13
1028 movq 56($ap), %rax
1029 movq %rdx, %r14
1030 adcq \$0, %r14
1031
1032 mulq %rbx
1033 addq %rax, %r14
1034 movq ($ap), %rax
1035 movq %rdx, %r15
1036 adcq \$0, %r15
1037
1038 leaq 8(%rsp), %rdi
1039 movl \$7, %ecx
1040 jmp .Loop_mul_gather
1041
1042 .align 32
1043 .Loop_mul_gather:
1044 movdqa 16*0(%rbp),%xmm8
1045 movdqa 16*1(%rbp),%xmm9
1046 movdqa 16*2(%rbp),%xmm10
1047 movdqa 16*3(%rbp),%xmm11
1048 pand %xmm0,%xmm8
1049 movdqa 16*4(%rbp),%xmm12
1050 pand %xmm1,%xmm9
1051 movdqa 16*5(%rbp),%xmm13
1052 pand %xmm2,%xmm10
1053 movdqa 16*6(%rbp),%xmm14
1054 pand %xmm3,%xmm11
1055 movdqa 16*7(%rbp),%xmm15
1056 leaq 128(%rbp), %rbp
1057 pand %xmm4,%xmm12
1058 pand %xmm5,%xmm13
1059 pand %xmm6,%xmm14
1060 pand %xmm7,%xmm15
1061 por %xmm10,%xmm8
1062 por %xmm11,%xmm9
1063 por %xmm12,%xmm8
1064 por %xmm13,%xmm9
1065 por %xmm14,%xmm8
1066 por %xmm15,%xmm9
1067
1068 por %xmm9,%xmm8
1069 pshufd \$0x4e,%xmm8,%xmm9
1070 por %xmm9,%xmm8
1071 movq %xmm8,%rbx
1072
1073 mulq %rbx
1074 addq %rax, %r8
1075 movq 8($ap), %rax
1076 movq %r8, (%rdi)
1077 movq %rdx, %r8
1078 adcq \$0, %r8
1079
1080 mulq %rbx
1081 addq %rax, %r9
1082 movq 16($ap), %rax
1083 adcq \$0, %rdx
1084 addq %r9, %r8
1085 movq %rdx, %r9
1086 adcq \$0, %r9
1087
1088 mulq %rbx
1089 addq %rax, %r10
1090 movq 24($ap), %rax
1091 adcq \$0, %rdx
1092 addq %r10, %r9
1093 movq %rdx, %r10
1094 adcq \$0, %r10
1095
1096 mulq %rbx
1097 addq %rax, %r11
1098 movq 32($ap), %rax
1099 adcq \$0, %rdx
1100 addq %r11, %r10
1101 movq %rdx, %r11
1102 adcq \$0, %r11
1103
1104 mulq %rbx
1105 addq %rax, %r12
1106 movq 40($ap), %rax
1107 adcq \$0, %rdx
1108 addq %r12, %r11
1109 movq %rdx, %r12
1110 adcq \$0, %r12
1111
1112 mulq %rbx
1113 addq %rax, %r13
1114 movq 48($ap), %rax
1115 adcq \$0, %rdx
1116 addq %r13, %r12
1117 movq %rdx, %r13
1118 adcq \$0, %r13
1119
1120 mulq %rbx
1121 addq %rax, %r14
1122 movq 56($ap), %rax
1123 adcq \$0, %rdx
1124 addq %r14, %r13
1125 movq %rdx, %r14
1126 adcq \$0, %r14
1127
1128 mulq %rbx
1129 addq %rax, %r15
1130 movq ($ap), %rax
1131 adcq \$0, %rdx
1132 addq %r15, %r14
1133 movq %rdx, %r15
1134 adcq \$0, %r15
1135
1136 leaq 8(%rdi), %rdi
1137
1138 decl %ecx
1139 jnz .Loop_mul_gather
1140
1141 movq %r8, (%rdi)
1142 movq %r9, 8(%rdi)
1143 movq %r10, 16(%rdi)
1144 movq %r11, 24(%rdi)
1145 movq %r12, 32(%rdi)
1146 movq %r13, 40(%rdi)
1147 movq %r14, 48(%rdi)
1148 movq %r15, 56(%rdi)
1149
1150 movq 128+8(%rsp), $out
1151 movq 128+16(%rsp), %rbp
1152
1153 movq (%rsp), %r8
1154 movq 8(%rsp), %r9
1155 movq 16(%rsp), %r10
1156 movq 24(%rsp), %r11
1157 movq 32(%rsp), %r12
1158 movq 40(%rsp), %r13
1159 movq 48(%rsp), %r14
1160 movq 56(%rsp), %r15
1161
1162 call __rsaz_512_reduce
1163 ___
1164 $code.=<<___ if ($addx);
1165 jmp .Lmul_gather_tail
1166
1167 .align 32
1168 .Lmulx_gather:
1169 movq %xmm8,%rdx
1170
1171 mov $n0, 128(%rsp) # off-load arguments
1172 mov $out, 128+8(%rsp)
1173 mov $mod, 128+16(%rsp)
1174
1175 mulx ($ap), %rbx, %r8 # 0 iteration
1176 mov %rbx, (%rsp)
1177 xor %edi, %edi # cf=0, of=0
1178
1179 mulx 8($ap), %rax, %r9
1180
1181 mulx 16($ap), %rbx, %r10
1182 adcx %rax, %r8
1183
1184 mulx 24($ap), %rax, %r11
1185 adcx %rbx, %r9
1186
1187 mulx 32($ap), %rbx, %r12
1188 adcx %rax, %r10
1189
1190 mulx 40($ap), %rax, %r13
1191 adcx %rbx, %r11
1192
1193 mulx 48($ap), %rbx, %r14
1194 adcx %rax, %r12
1195
1196 mulx 56($ap), %rax, %r15
1197 adcx %rbx, %r13
1198 adcx %rax, %r14
1199 .byte 0x67
1200 mov %r8, %rbx
1201 adcx %rdi, %r15 # %rdi is 0
1202
1203 mov \$-7, %rcx
1204 jmp .Loop_mulx_gather
1205
1206 .align 32
1207 .Loop_mulx_gather:
1208 movdqa 16*0(%rbp),%xmm8
1209 movdqa 16*1(%rbp),%xmm9
1210 movdqa 16*2(%rbp),%xmm10
1211 movdqa 16*3(%rbp),%xmm11
1212 pand %xmm0,%xmm8
1213 movdqa 16*4(%rbp),%xmm12
1214 pand %xmm1,%xmm9
1215 movdqa 16*5(%rbp),%xmm13
1216 pand %xmm2,%xmm10
1217 movdqa 16*6(%rbp),%xmm14
1218 pand %xmm3,%xmm11
1219 movdqa 16*7(%rbp),%xmm15
1220 leaq 128(%rbp), %rbp
1221 pand %xmm4,%xmm12
1222 pand %xmm5,%xmm13
1223 pand %xmm6,%xmm14
1224 pand %xmm7,%xmm15
1225 por %xmm10,%xmm8
1226 por %xmm11,%xmm9
1227 por %xmm12,%xmm8
1228 por %xmm13,%xmm9
1229 por %xmm14,%xmm8
1230 por %xmm15,%xmm9
1231
1232 por %xmm9,%xmm8
1233 pshufd \$0x4e,%xmm8,%xmm9
1234 por %xmm9,%xmm8
1235 movq %xmm8,%rdx
1236
1237 .byte 0xc4,0x62,0xfb,0xf6,0x86,0x00,0x00,0x00,0x00 # mulx ($ap), %rax, %r8
1238 adcx %rax, %rbx
1239 adox %r9, %r8
1240
1241 mulx 8($ap), %rax, %r9
1242 adcx %rax, %r8
1243 adox %r10, %r9
1244
1245 mulx 16($ap), %rax, %r10
1246 adcx %rax, %r9
1247 adox %r11, %r10
1248
1249 .byte 0xc4,0x62,0xfb,0xf6,0x9e,0x18,0x00,0x00,0x00 # mulx 24($ap), %rax, %r11
1250 adcx %rax, %r10
1251 adox %r12, %r11
1252
1253 mulx 32($ap), %rax, %r12
1254 adcx %rax, %r11
1255 adox %r13, %r12
1256
1257 mulx 40($ap), %rax, %r13
1258 adcx %rax, %r12
1259 adox %r14, %r13
1260
1261 .byte 0xc4,0x62,0xfb,0xf6,0xb6,0x30,0x00,0x00,0x00 # mulx 48($ap), %rax, %r14
1262 adcx %rax, %r13
1263 .byte 0x67
1264 adox %r15, %r14
1265
1266 mulx 56($ap), %rax, %r15
1267 mov %rbx, 64(%rsp,%rcx,8)
1268 adcx %rax, %r14
1269 adox %rdi, %r15
1270 mov %r8, %rbx
1271 adcx %rdi, %r15 # cf=0
1272
1273 inc %rcx # of=0
1274 jnz .Loop_mulx_gather
1275
1276 mov %r8, 64(%rsp)
1277 mov %r9, 64+8(%rsp)
1278 mov %r10, 64+16(%rsp)
1279 mov %r11, 64+24(%rsp)
1280 mov %r12, 64+32(%rsp)
1281 mov %r13, 64+40(%rsp)
1282 mov %r14, 64+48(%rsp)
1283 mov %r15, 64+56(%rsp)
1284
1285 mov 128(%rsp), %rdx # pull arguments
1286 mov 128+8(%rsp), $out
1287 mov 128+16(%rsp), %rbp
1288
1289 mov (%rsp), %r8
1290 mov 8(%rsp), %r9
1291 mov 16(%rsp), %r10
1292 mov 24(%rsp), %r11
1293 mov 32(%rsp), %r12
1294 mov 40(%rsp), %r13
1295 mov 48(%rsp), %r14
1296 mov 56(%rsp), %r15
1297
1298 call __rsaz_512_reducex
1299
1300 .Lmul_gather_tail:
1301 ___
1302 $code.=<<___;
1303 addq 64(%rsp), %r8
1304 adcq 72(%rsp), %r9
1305 adcq 80(%rsp), %r10
1306 adcq 88(%rsp), %r11
1307 adcq 96(%rsp), %r12
1308 adcq 104(%rsp), %r13
1309 adcq 112(%rsp), %r14
1310 adcq 120(%rsp), %r15
1311 sbbq %rcx, %rcx
1312
1313 call __rsaz_512_subtract
1314
1315 leaq 128+24+48(%rsp), %rax
1316 ___
1317 $code.=<<___ if ($win64);
1318 movaps 0xa0-0xc8(%rax),%xmm6
1319 movaps 0xb0-0xc8(%rax),%xmm7
1320 movaps 0xc0-0xc8(%rax),%xmm8
1321 movaps 0xd0-0xc8(%rax),%xmm9
1322 movaps 0xe0-0xc8(%rax),%xmm10
1323 movaps 0xf0-0xc8(%rax),%xmm11
1324 movaps 0x100-0xc8(%rax),%xmm12
1325 movaps 0x110-0xc8(%rax),%xmm13
1326 movaps 0x120-0xc8(%rax),%xmm14
1327 movaps 0x130-0xc8(%rax),%xmm15
1328 lea 0xb0(%rax),%rax
1329 ___
1330 $code.=<<___;
1331 movq -48(%rax), %r15
1332 movq -40(%rax), %r14
1333 movq -32(%rax), %r13
1334 movq -24(%rax), %r12
1335 movq -16(%rax), %rbp
1336 movq -8(%rax), %rbx
1337 leaq (%rax), %rsp
1338 .Lmul_gather4_epilogue:
1339 ret
1340 .size rsaz_512_mul_gather4,.-rsaz_512_mul_gather4
1341 ___
1342 }
1343 {
1344 my ($out,$ap,$mod,$n0,$tbl,$pwr) = ("%rdi","%rsi","%rdx","%rcx","%r8","%r9d");
1345 $code.=<<___;
1346 .globl rsaz_512_mul_scatter4
1347 .type rsaz_512_mul_scatter4,\@function,6
1348 .align 32
1349 rsaz_512_mul_scatter4:
1350 push %rbx
1351 push %rbp
1352 push %r12
1353 push %r13
1354 push %r14
1355 push %r15
1356
1357 mov $pwr, $pwr
1358 subq \$128+24, %rsp
1359 .Lmul_scatter4_body:
1360 leaq ($tbl,$pwr,8), $tbl
1361 movq $out, %xmm0 # off-load arguments
1362 movq $mod, %xmm1
1363 movq $tbl, %xmm2
1364 movq $n0, 128(%rsp)
1365
1366 movq $out, %rbp
1367 ___
1368 $code.=<<___ if ($addx);
1369 movl \$0x80100,%r11d
1370 andl OPENSSL_ia32cap_P+8(%rip),%r11d
1371 cmpl \$0x80100,%r11d # check for MULX and ADO/CX
1372 je .Lmulx_scatter
1373 ___
1374 $code.=<<___;
1375 movq ($out),%rbx # pass b[0]
1376 call __rsaz_512_mul
1377
1378 movq %xmm0, $out
1379 movq %xmm1, %rbp
1380
1381 movq (%rsp), %r8
1382 movq 8(%rsp), %r9
1383 movq 16(%rsp), %r10
1384 movq 24(%rsp), %r11
1385 movq 32(%rsp), %r12
1386 movq 40(%rsp), %r13
1387 movq 48(%rsp), %r14
1388 movq 56(%rsp), %r15
1389
1390 call __rsaz_512_reduce
1391 ___
1392 $code.=<<___ if ($addx);
1393 jmp .Lmul_scatter_tail
1394
1395 .align 32
1396 .Lmulx_scatter:
1397 movq ($out), %rdx # pass b[0]
1398 call __rsaz_512_mulx
1399
1400 movq %xmm0, $out
1401 movq %xmm1, %rbp
1402
1403 movq 128(%rsp), %rdx # pull $n0
1404 movq (%rsp), %r8
1405 movq 8(%rsp), %r9
1406 movq 16(%rsp), %r10
1407 movq 24(%rsp), %r11
1408 movq 32(%rsp), %r12
1409 movq 40(%rsp), %r13
1410 movq 48(%rsp), %r14
1411 movq 56(%rsp), %r15
1412
1413 call __rsaz_512_reducex
1414
1415 .Lmul_scatter_tail:
1416 ___
1417 $code.=<<___;
1418 addq 64(%rsp), %r8
1419 adcq 72(%rsp), %r9
1420 adcq 80(%rsp), %r10
1421 adcq 88(%rsp), %r11
1422 adcq 96(%rsp), %r12
1423 adcq 104(%rsp), %r13
1424 adcq 112(%rsp), %r14
1425 adcq 120(%rsp), %r15
1426 movq %xmm2, $inp
1427 sbbq %rcx, %rcx
1428
1429 call __rsaz_512_subtract
1430
1431 movq %r8, 128*0($inp) # scatter
1432 movq %r9, 128*1($inp)
1433 movq %r10, 128*2($inp)
1434 movq %r11, 128*3($inp)
1435 movq %r12, 128*4($inp)
1436 movq %r13, 128*5($inp)
1437 movq %r14, 128*6($inp)
1438 movq %r15, 128*7($inp)
1439
1440 leaq 128+24+48(%rsp), %rax
1441 movq -48(%rax), %r15
1442 movq -40(%rax), %r14
1443 movq -32(%rax), %r13
1444 movq -24(%rax), %r12
1445 movq -16(%rax), %rbp
1446 movq -8(%rax), %rbx
1447 leaq (%rax), %rsp
1448 .Lmul_scatter4_epilogue:
1449 ret
1450 .size rsaz_512_mul_scatter4,.-rsaz_512_mul_scatter4
1451 ___
1452 }
1453 {
1454 my ($out,$inp,$mod,$n0) = ("%rdi","%rsi","%rdx","%rcx");
1455 $code.=<<___;
1456 .globl rsaz_512_mul_by_one
1457 .type rsaz_512_mul_by_one,\@function,4
1458 .align 32
1459 rsaz_512_mul_by_one:
1460 push %rbx
1461 push %rbp
1462 push %r12
1463 push %r13
1464 push %r14
1465 push %r15
1466
1467 subq \$128+24, %rsp
1468 .Lmul_by_one_body:
1469 ___
1470 $code.=<<___ if ($addx);
1471 movl OPENSSL_ia32cap_P+8(%rip),%eax
1472 ___
1473 $code.=<<___;
1474 movq $mod, %rbp # reassign argument
1475 movq $n0, 128(%rsp)
1476
1477 movq ($inp), %r8
1478 pxor %xmm0, %xmm0
1479 movq 8($inp), %r9
1480 movq 16($inp), %r10
1481 movq 24($inp), %r11
1482 movq 32($inp), %r12
1483 movq 40($inp), %r13
1484 movq 48($inp), %r14
1485 movq 56($inp), %r15
1486
1487 movdqa %xmm0, (%rsp)
1488 movdqa %xmm0, 16(%rsp)
1489 movdqa %xmm0, 32(%rsp)
1490 movdqa %xmm0, 48(%rsp)
1491 movdqa %xmm0, 64(%rsp)
1492 movdqa %xmm0, 80(%rsp)
1493 movdqa %xmm0, 96(%rsp)
1494 ___
1495 $code.=<<___ if ($addx);
1496 andl \$0x80100,%eax
1497 cmpl \$0x80100,%eax # check for MULX and ADO/CX
1498 je .Lby_one_callx
1499 ___
1500 $code.=<<___;
1501 call __rsaz_512_reduce
1502 ___
1503 $code.=<<___ if ($addx);
1504 jmp .Lby_one_tail
1505 .align 32
1506 .Lby_one_callx:
1507 movq 128(%rsp), %rdx # pull $n0
1508 call __rsaz_512_reducex
1509 .Lby_one_tail:
1510 ___
1511 $code.=<<___;
1512 movq %r8, ($out)
1513 movq %r9, 8($out)
1514 movq %r10, 16($out)
1515 movq %r11, 24($out)
1516 movq %r12, 32($out)
1517 movq %r13, 40($out)
1518 movq %r14, 48($out)
1519 movq %r15, 56($out)
1520
1521 leaq 128+24+48(%rsp), %rax
1522 movq -48(%rax), %r15
1523 movq -40(%rax), %r14
1524 movq -32(%rax), %r13
1525 movq -24(%rax), %r12
1526 movq -16(%rax), %rbp
1527 movq -8(%rax), %rbx
1528 leaq (%rax), %rsp
1529 .Lmul_by_one_epilogue:
1530 ret
1531 .size rsaz_512_mul_by_one,.-rsaz_512_mul_by_one
1532 ___
1533 }
1534 { # __rsaz_512_reduce
1535 #
1536 # input: %r8-%r15, %rbp - mod, 128(%rsp) - n0
1537 # output: %r8-%r15
1538 # clobbers: everything except %rbp and %rdi
1539 $code.=<<___;
1540 .type __rsaz_512_reduce,\@abi-omnipotent
1541 .align 32
1542 __rsaz_512_reduce:
1543 movq %r8, %rbx
1544 imulq 128+8(%rsp), %rbx
1545 movq 0(%rbp), %rax
1546 movl \$8, %ecx
1547 jmp .Lreduction_loop
1548
1549 .align 32
1550 .Lreduction_loop:
1551 mulq %rbx
1552 movq 8(%rbp), %rax
1553 negq %r8
1554 movq %rdx, %r8
1555 adcq \$0, %r8
1556
1557 mulq %rbx
1558 addq %rax, %r9
1559 movq 16(%rbp), %rax
1560 adcq \$0, %rdx
1561 addq %r9, %r8
1562 movq %rdx, %r9
1563 adcq \$0, %r9
1564
1565 mulq %rbx
1566 addq %rax, %r10
1567 movq 24(%rbp), %rax
1568 adcq \$0, %rdx
1569 addq %r10, %r9
1570 movq %rdx, %r10
1571 adcq \$0, %r10
1572
1573 mulq %rbx
1574 addq %rax, %r11
1575 movq 32(%rbp), %rax
1576 adcq \$0, %rdx
1577 addq %r11, %r10
1578 movq 128+8(%rsp), %rsi
1579 #movq %rdx, %r11
1580 #adcq \$0, %r11
1581 adcq \$0, %rdx
1582 movq %rdx, %r11
1583
1584 mulq %rbx
1585 addq %rax, %r12
1586 movq 40(%rbp), %rax
1587 adcq \$0, %rdx
1588 imulq %r8, %rsi
1589 addq %r12, %r11
1590 movq %rdx, %r12
1591 adcq \$0, %r12
1592
1593 mulq %rbx
1594 addq %rax, %r13
1595 movq 48(%rbp), %rax
1596 adcq \$0, %rdx
1597 addq %r13, %r12
1598 movq %rdx, %r13
1599 adcq \$0, %r13
1600
1601 mulq %rbx
1602 addq %rax, %r14
1603 movq 56(%rbp), %rax
1604 adcq \$0, %rdx
1605 addq %r14, %r13
1606 movq %rdx, %r14
1607 adcq \$0, %r14
1608
1609 mulq %rbx
1610 movq %rsi, %rbx
1611 addq %rax, %r15
1612 movq 0(%rbp), %rax
1613 adcq \$0, %rdx
1614 addq %r15, %r14
1615 movq %rdx, %r15
1616 adcq \$0, %r15
1617
1618 decl %ecx
1619 jne .Lreduction_loop
1620
1621 ret
1622 .size __rsaz_512_reduce,.-__rsaz_512_reduce
1623 ___
1624 }
1625 if ($addx) {
1626 # __rsaz_512_reducex
1627 #
1628 # input: %r8-%r15, %rbp - mod, 128(%rsp) - n0
1629 # output: %r8-%r15
1630 # clobbers: everything except %rbp and %rdi
1631 $code.=<<___;
1632 .type __rsaz_512_reducex,\@abi-omnipotent
1633 .align 32
1634 __rsaz_512_reducex:
1635 #movq 128+8(%rsp), %rdx # pull $n0
1636 imulq %r8, %rdx
1637 xorq %rsi, %rsi # cf=0,of=0
1638 movl \$8, %ecx
1639 jmp .Lreduction_loopx
1640
1641 .align 32
1642 .Lreduction_loopx:
1643 mov %r8, %rbx
1644 mulx 0(%rbp), %rax, %r8
1645 adcx %rbx, %rax
1646 adox %r9, %r8
1647
1648 mulx 8(%rbp), %rax, %r9
1649 adcx %rax, %r8
1650 adox %r10, %r9
1651
1652 mulx 16(%rbp), %rbx, %r10
1653 adcx %rbx, %r9
1654 adox %r11, %r10
1655
1656 mulx 24(%rbp), %rbx, %r11
1657 adcx %rbx, %r10
1658 adox %r12, %r11
1659
1660 .byte 0xc4,0x62,0xe3,0xf6,0xa5,0x20,0x00,0x00,0x00 # mulx 32(%rbp), %rbx, %r12
1661 mov %rdx, %rax
1662 mov %r8, %rdx
1663 adcx %rbx, %r11
1664 adox %r13, %r12
1665
1666 mulx 128+8(%rsp), %rbx, %rdx
1667 mov %rax, %rdx
1668
1669 mulx 40(%rbp), %rax, %r13
1670 adcx %rax, %r12
1671 adox %r14, %r13
1672
1673 .byte 0xc4,0x62,0xfb,0xf6,0xb5,0x30,0x00,0x00,0x00 # mulx 48(%rbp), %rax, %r14
1674 adcx %rax, %r13
1675 adox %r15, %r14
1676
1677 mulx 56(%rbp), %rax, %r15
1678 mov %rbx, %rdx
1679 adcx %rax, %r14
1680 adox %rsi, %r15 # %rsi is 0
1681 adcx %rsi, %r15 # cf=0
1682
1683 decl %ecx # of=0
1684 jne .Lreduction_loopx
1685
1686 ret
1687 .size __rsaz_512_reducex,.-__rsaz_512_reducex
1688 ___
1689 }
1690 { # __rsaz_512_subtract
1691 # input: %r8-%r15, %rdi - $out, %rbp - $mod, %rcx - mask
1692 # output:
1693 # clobbers: everything but %rdi, %rsi and %rbp
1694 $code.=<<___;
1695 .type __rsaz_512_subtract,\@abi-omnipotent
1696 .align 32
1697 __rsaz_512_subtract:
1698 movq %r8, ($out)
1699 movq %r9, 8($out)
1700 movq %r10, 16($out)
1701 movq %r11, 24($out)
1702 movq %r12, 32($out)
1703 movq %r13, 40($out)
1704 movq %r14, 48($out)
1705 movq %r15, 56($out)
1706
1707 movq 0($mod), %r8
1708 movq 8($mod), %r9
1709 negq %r8
1710 notq %r9
1711 andq %rcx, %r8
1712 movq 16($mod), %r10
1713 andq %rcx, %r9
1714 notq %r10
1715 movq 24($mod), %r11
1716 andq %rcx, %r10
1717 notq %r11
1718 movq 32($mod), %r12
1719 andq %rcx, %r11
1720 notq %r12
1721 movq 40($mod), %r13
1722 andq %rcx, %r12
1723 notq %r13
1724 movq 48($mod), %r14
1725 andq %rcx, %r13
1726 notq %r14
1727 movq 56($mod), %r15
1728 andq %rcx, %r14
1729 notq %r15
1730 andq %rcx, %r15
1731
1732 addq ($out), %r8
1733 adcq 8($out), %r9
1734 adcq 16($out), %r10
1735 adcq 24($out), %r11
1736 adcq 32($out), %r12
1737 adcq 40($out), %r13
1738 adcq 48($out), %r14
1739 adcq 56($out), %r15
1740
1741 movq %r8, ($out)
1742 movq %r9, 8($out)
1743 movq %r10, 16($out)
1744 movq %r11, 24($out)
1745 movq %r12, 32($out)
1746 movq %r13, 40($out)
1747 movq %r14, 48($out)
1748 movq %r15, 56($out)
1749
1750 ret
1751 .size __rsaz_512_subtract,.-__rsaz_512_subtract
1752 ___
1753 }
1754 { # __rsaz_512_mul
1755 #
1756 # input: %rsi - ap, %rbp - bp
1757 # ouput:
1758 # clobbers: everything
1759 my ($ap,$bp) = ("%rsi","%rbp");
1760 $code.=<<___;
1761 .type __rsaz_512_mul,\@abi-omnipotent
1762 .align 32
1763 __rsaz_512_mul:
1764 leaq 8(%rsp), %rdi
1765
1766 movq ($ap), %rax
1767 mulq %rbx
1768 movq %rax, (%rdi)
1769 movq 8($ap), %rax
1770 movq %rdx, %r8
1771
1772 mulq %rbx
1773 addq %rax, %r8
1774 movq 16($ap), %rax
1775 movq %rdx, %r9
1776 adcq \$0, %r9
1777
1778 mulq %rbx
1779 addq %rax, %r9
1780 movq 24($ap), %rax
1781 movq %rdx, %r10
1782 adcq \$0, %r10
1783
1784 mulq %rbx
1785 addq %rax, %r10
1786 movq 32($ap), %rax
1787 movq %rdx, %r11
1788 adcq \$0, %r11
1789
1790 mulq %rbx
1791 addq %rax, %r11
1792 movq 40($ap), %rax
1793 movq %rdx, %r12
1794 adcq \$0, %r12
1795
1796 mulq %rbx
1797 addq %rax, %r12
1798 movq 48($ap), %rax
1799 movq %rdx, %r13
1800 adcq \$0, %r13
1801
1802 mulq %rbx
1803 addq %rax, %r13
1804 movq 56($ap), %rax
1805 movq %rdx, %r14
1806 adcq \$0, %r14
1807
1808 mulq %rbx
1809 addq %rax, %r14
1810 movq ($ap), %rax
1811 movq %rdx, %r15
1812 adcq \$0, %r15
1813
1814 leaq 8($bp), $bp
1815 leaq 8(%rdi), %rdi
1816
1817 movl \$7, %ecx
1818 jmp .Loop_mul
1819
1820 .align 32
1821 .Loop_mul:
1822 movq ($bp), %rbx
1823 mulq %rbx
1824 addq %rax, %r8
1825 movq 8($ap), %rax
1826 movq %r8, (%rdi)
1827 movq %rdx, %r8
1828 adcq \$0, %r8
1829
1830 mulq %rbx
1831 addq %rax, %r9
1832 movq 16($ap), %rax
1833 adcq \$0, %rdx
1834 addq %r9, %r8
1835 movq %rdx, %r9
1836 adcq \$0, %r9
1837
1838 mulq %rbx
1839 addq %rax, %r10
1840 movq 24($ap), %rax
1841 adcq \$0, %rdx
1842 addq %r10, %r9
1843 movq %rdx, %r10
1844 adcq \$0, %r10
1845
1846 mulq %rbx
1847 addq %rax, %r11
1848 movq 32($ap), %rax
1849 adcq \$0, %rdx
1850 addq %r11, %r10
1851 movq %rdx, %r11
1852 adcq \$0, %r11
1853
1854 mulq %rbx
1855 addq %rax, %r12
1856 movq 40($ap), %rax
1857 adcq \$0, %rdx
1858 addq %r12, %r11
1859 movq %rdx, %r12
1860 adcq \$0, %r12
1861
1862 mulq %rbx
1863 addq %rax, %r13
1864 movq 48($ap), %rax
1865 adcq \$0, %rdx
1866 addq %r13, %r12
1867 movq %rdx, %r13
1868 adcq \$0, %r13
1869
1870 mulq %rbx
1871 addq %rax, %r14
1872 movq 56($ap), %rax
1873 adcq \$0, %rdx
1874 addq %r14, %r13
1875 movq %rdx, %r14
1876 leaq 8($bp), $bp
1877 adcq \$0, %r14
1878
1879 mulq %rbx
1880 addq %rax, %r15
1881 movq ($ap), %rax
1882 adcq \$0, %rdx
1883 addq %r15, %r14
1884 movq %rdx, %r15
1885 adcq \$0, %r15
1886
1887 leaq 8(%rdi), %rdi
1888
1889 decl %ecx
1890 jnz .Loop_mul
1891
1892 movq %r8, (%rdi)
1893 movq %r9, 8(%rdi)
1894 movq %r10, 16(%rdi)
1895 movq %r11, 24(%rdi)
1896 movq %r12, 32(%rdi)
1897 movq %r13, 40(%rdi)
1898 movq %r14, 48(%rdi)
1899 movq %r15, 56(%rdi)
1900
1901 ret
1902 .size __rsaz_512_mul,.-__rsaz_512_mul
1903 ___
1904 }
1905 if ($addx) {
1906 # __rsaz_512_mulx
1907 #
1908 # input: %rsi - ap, %rbp - bp
1909 # ouput:
1910 # clobbers: everything
1911 my ($ap,$bp,$zero) = ("%rsi","%rbp","%rdi");
1912 $code.=<<___;
1913 .type __rsaz_512_mulx,\@abi-omnipotent
1914 .align 32
1915 __rsaz_512_mulx:
1916 mulx ($ap), %rbx, %r8 # initial %rdx preloaded by caller
1917 mov \$-6, %rcx
1918
1919 mulx 8($ap), %rax, %r9
1920 movq %rbx, 8(%rsp)
1921
1922 mulx 16($ap), %rbx, %r10
1923 adc %rax, %r8
1924
1925 mulx 24($ap), %rax, %r11
1926 adc %rbx, %r9
1927
1928 mulx 32($ap), %rbx, %r12
1929 adc %rax, %r10
1930
1931 mulx 40($ap), %rax, %r13
1932 adc %rbx, %r11
1933
1934 mulx 48($ap), %rbx, %r14
1935 adc %rax, %r12
1936
1937 mulx 56($ap), %rax, %r15
1938 mov 8($bp), %rdx
1939 adc %rbx, %r13
1940 adc %rax, %r14
1941 adc \$0, %r15
1942
1943 xor $zero, $zero # cf=0,of=0
1944 jmp .Loop_mulx
1945
1946 .align 32
1947 .Loop_mulx:
1948 movq %r8, %rbx
1949 mulx ($ap), %rax, %r8
1950 adcx %rax, %rbx
1951 adox %r9, %r8
1952
1953 mulx 8($ap), %rax, %r9
1954 adcx %rax, %r8
1955 adox %r10, %r9
1956
1957 mulx 16($ap), %rax, %r10
1958 adcx %rax, %r9
1959 adox %r11, %r10
1960
1961 mulx 24($ap), %rax, %r11
1962 adcx %rax, %r10
1963 adox %r12, %r11
1964
1965 .byte 0x3e,0xc4,0x62,0xfb,0xf6,0xa6,0x20,0x00,0x00,0x00 # mulx 32($ap), %rax, %r12
1966 adcx %rax, %r11
1967 adox %r13, %r12
1968
1969 mulx 40($ap), %rax, %r13
1970 adcx %rax, %r12
1971 adox %r14, %r13
1972
1973 mulx 48($ap), %rax, %r14
1974 adcx %rax, %r13
1975 adox %r15, %r14
1976
1977 mulx 56($ap), %rax, %r15
1978 movq 64($bp,%rcx,8), %rdx
1979 movq %rbx, 8+64-8(%rsp,%rcx,8)
1980 adcx %rax, %r14
1981 adox $zero, %r15
1982 adcx $zero, %r15 # cf=0
1983
1984 inc %rcx # of=0
1985 jnz .Loop_mulx
1986
1987 movq %r8, %rbx
1988 mulx ($ap), %rax, %r8
1989 adcx %rax, %rbx
1990 adox %r9, %r8
1991
1992 .byte 0xc4,0x62,0xfb,0xf6,0x8e,0x08,0x00,0x00,0x00 # mulx 8($ap), %rax, %r9
1993 adcx %rax, %r8
1994 adox %r10, %r9
1995
1996 .byte 0xc4,0x62,0xfb,0xf6,0x96,0x10,0x00,0x00,0x00 # mulx 16($ap), %rax, %r10
1997 adcx %rax, %r9
1998 adox %r11, %r10
1999
2000 mulx 24($ap), %rax, %r11
2001 adcx %rax, %r10
2002 adox %r12, %r11
2003
2004 mulx 32($ap), %rax, %r12
2005 adcx %rax, %r11
2006 adox %r13, %r12
2007
2008 mulx 40($ap), %rax, %r13
2009 adcx %rax, %r12
2010 adox %r14, %r13
2011
2012 .byte 0xc4,0x62,0xfb,0xf6,0xb6,0x30,0x00,0x00,0x00 # mulx 48($ap), %rax, %r14
2013 adcx %rax, %r13
2014 adox %r15, %r14
2015
2016 .byte 0xc4,0x62,0xfb,0xf6,0xbe,0x38,0x00,0x00,0x00 # mulx 56($ap), %rax, %r15
2017 adcx %rax, %r14
2018 adox $zero, %r15
2019 adcx $zero, %r15
2020
2021 mov %rbx, 8+64-8(%rsp)
2022 mov %r8, 8+64(%rsp)
2023 mov %r9, 8+64+8(%rsp)
2024 mov %r10, 8+64+16(%rsp)
2025 mov %r11, 8+64+24(%rsp)
2026 mov %r12, 8+64+32(%rsp)
2027 mov %r13, 8+64+40(%rsp)
2028 mov %r14, 8+64+48(%rsp)
2029 mov %r15, 8+64+56(%rsp)
2030
2031 ret
2032 .size __rsaz_512_mulx,.-__rsaz_512_mulx
2033 ___
2034 }
2035 {
2036 my ($out,$inp,$power)= $win64 ? ("%rcx","%rdx","%r8d") : ("%rdi","%rsi","%edx");
2037 $code.=<<___;
2038 .globl rsaz_512_scatter4
2039 .type rsaz_512_scatter4,\@abi-omnipotent
2040 .align 16
2041 rsaz_512_scatter4:
2042 leaq ($out,$power,8), $out
2043 movl \$8, %r9d
2044 jmp .Loop_scatter
2045 .align 16
2046 .Loop_scatter:
2047 movq ($inp), %rax
2048 leaq 8($inp), $inp
2049 movq %rax, ($out)
2050 leaq 128($out), $out
2051 decl %r9d
2052 jnz .Loop_scatter
2053 ret
2054 .size rsaz_512_scatter4,.-rsaz_512_scatter4
2055
2056 .globl rsaz_512_gather4
2057 .type rsaz_512_gather4,\@abi-omnipotent
2058 .align 16
2059 rsaz_512_gather4:
2060 ___
2061 $code.=<<___ if ($win64);
2062 .LSEH_begin_rsaz_512_gather4:
2063 .byte 0x48,0x81,0xec,0xa8,0x00,0x00,0x00 # sub $0xa8,%rsp
2064 .byte 0x0f,0x29,0x34,0x24 # movaps %xmm6,(%rsp)
2065 .byte 0x0f,0x29,0x7c,0x24,0x10 # movaps %xmm7,0x10(%rsp)
2066 .byte 0x44,0x0f,0x29,0x44,0x24,0x20 # movaps %xmm8,0x20(%rsp)
2067 .byte 0x44,0x0f,0x29,0x4c,0x24,0x30 # movaps %xmm9,0x30(%rsp)
2068 .byte 0x44,0x0f,0x29,0x54,0x24,0x40 # movaps %xmm10,0x40(%rsp)
2069 .byte 0x44,0x0f,0x29,0x5c,0x24,0x50 # movaps %xmm11,0x50(%rsp)
2070 .byte 0x44,0x0f,0x29,0x64,0x24,0x60 # movaps %xmm12,0x60(%rsp)
2071 .byte 0x44,0x0f,0x29,0x6c,0x24,0x70 # movaps %xmm13,0x70(%rsp)
2072 .byte 0x44,0x0f,0x29,0xb4,0x24,0x80,0,0,0 # movaps %xmm14,0x80(%rsp)
2073 .byte 0x44,0x0f,0x29,0xbc,0x24,0x90,0,0,0 # movaps %xmm15,0x90(%rsp)
2074 ___
2075 $code.=<<___;
2076 movd $power,%xmm8
2077 movdqa .Linc+16(%rip),%xmm1 # 00000002000000020000000200000002
2078 movdqa .Linc(%rip),%xmm0 # 00000001000000010000000000000000
2079
2080 pshufd \$0,%xmm8,%xmm8 # broadcast $power
2081 movdqa %xmm1,%xmm7
2082 movdqa %xmm1,%xmm2
2083 ___
2084 ########################################################################
2085 # calculate mask by comparing 0..15 to $power
2086 #
2087 for($i=0;$i<4;$i++) {
2088 $code.=<<___;
2089 paddd %xmm`$i`,%xmm`$i+1`
2090 pcmpeqd %xmm8,%xmm`$i`
2091 movdqa %xmm7,%xmm`$i+3`
2092 ___
2093 }
2094 for(;$i<7;$i++) {
2095 $code.=<<___;
2096 paddd %xmm`$i`,%xmm`$i+1`
2097 pcmpeqd %xmm8,%xmm`$i`
2098 ___
2099 }
2100 $code.=<<___;
2101 pcmpeqd %xmm8,%xmm7
2102 movl \$8, %r9d
2103 jmp .Loop_gather
2104 .align 16
2105 .Loop_gather:
2106 movdqa 16*0($inp),%xmm8
2107 movdqa 16*1($inp),%xmm9
2108 movdqa 16*2($inp),%xmm10
2109 movdqa 16*3($inp),%xmm11
2110 pand %xmm0,%xmm8
2111 movdqa 16*4($inp),%xmm12
2112 pand %xmm1,%xmm9
2113 movdqa 16*5($inp),%xmm13
2114 pand %xmm2,%xmm10
2115 movdqa 16*6($inp),%xmm14
2116 pand %xmm3,%xmm11
2117 movdqa 16*7($inp),%xmm15
2118 leaq 128($inp), $inp
2119 pand %xmm4,%xmm12
2120 pand %xmm5,%xmm13
2121 pand %xmm6,%xmm14
2122 pand %xmm7,%xmm15
2123 por %xmm10,%xmm8
2124 por %xmm11,%xmm9
2125 por %xmm12,%xmm8
2126 por %xmm13,%xmm9
2127 por %xmm14,%xmm8
2128 por %xmm15,%xmm9
2129
2130 por %xmm9,%xmm8
2131 pshufd \$0x4e,%xmm8,%xmm9
2132 por %xmm9,%xmm8
2133 movq %xmm8,($out)
2134 leaq 8($out), $out
2135 decl %r9d
2136 jnz .Loop_gather
2137 ___
2138 $code.=<<___ if ($win64);
2139 movaps 0x00(%rsp),%xmm6
2140 movaps 0x10(%rsp),%xmm7
2141 movaps 0x20(%rsp),%xmm8
2142 movaps 0x30(%rsp),%xmm9
2143 movaps 0x40(%rsp),%xmm10
2144 movaps 0x50(%rsp),%xmm11
2145 movaps 0x60(%rsp),%xmm12
2146 movaps 0x70(%rsp),%xmm13
2147 movaps 0x80(%rsp),%xmm14
2148 movaps 0x90(%rsp),%xmm15
2149 add \$0xa8,%rsp
2150 ___
2151 $code.=<<___;
2152 ret
2153 .LSEH_end_rsaz_512_gather4:
2154 .size rsaz_512_gather4,.-rsaz_512_gather4
2155
2156 .align 64
2157 .Linc:
2158 .long 0,0, 1,1
2159 .long 2,2, 2,2
2160 ___
2161 }
2162
2163 # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame,
2164 # CONTEXT *context,DISPATCHER_CONTEXT *disp)
2165 if ($win64) {
2166 $rec="%rcx";
2167 $frame="%rdx";
2168 $context="%r8";
2169 $disp="%r9";
2170
2171 $code.=<<___;
2172 .extern __imp_RtlVirtualUnwind
2173 .type se_handler,\@abi-omnipotent
2174 .align 16
2175 se_handler:
2176 push %rsi
2177 push %rdi
2178 push %rbx
2179 push %rbp
2180 push %r12
2181 push %r13
2182 push %r14
2183 push %r15
2184 pushfq
2185 sub \$64,%rsp
2186
2187 mov 120($context),%rax # pull context->Rax
2188 mov 248($context),%rbx # pull context->Rip
2189
2190 mov 8($disp),%rsi # disp->ImageBase
2191 mov 56($disp),%r11 # disp->HandlerData
2192
2193 mov 0(%r11),%r10d # HandlerData[0]
2194 lea (%rsi,%r10),%r10 # end of prologue label
2195 cmp %r10,%rbx # context->Rip<end of prologue label
2196 jb .Lcommon_seh_tail
2197
2198 mov 152($context),%rax # pull context->Rsp
2199
2200 mov 4(%r11),%r10d # HandlerData[1]
2201 lea (%rsi,%r10),%r10 # epilogue label
2202 cmp %r10,%rbx # context->Rip>=epilogue label
2203 jae .Lcommon_seh_tail
2204
2205 lea 128+24+48(%rax),%rax
2206
2207 lea .Lmul_gather4_epilogue(%rip),%rbx
2208 cmp %r10,%rbx
2209 jne .Lse_not_in_mul_gather4
2210
2211 lea 0xb0(%rax),%rax
2212
2213 lea -48-0xa8(%rax),%rsi
2214 lea 512($context),%rdi
2215 mov \$20,%ecx
2216 .long 0xa548f3fc # cld; rep movsq
2217
2218 .Lse_not_in_mul_gather4:
2219 mov -8(%rax),%rbx
2220 mov -16(%rax),%rbp
2221 mov -24(%rax),%r12
2222 mov -32(%rax),%r13
2223 mov -40(%rax),%r14
2224 mov -48(%rax),%r15
2225 mov %rbx,144($context) # restore context->Rbx
2226 mov %rbp,160($context) # restore context->Rbp
2227 mov %r12,216($context) # restore context->R12
2228 mov %r13,224($context) # restore context->R13
2229 mov %r14,232($context) # restore context->R14
2230 mov %r15,240($context) # restore context->R15
2231
2232 .Lcommon_seh_tail:
2233 mov 8(%rax),%rdi
2234 mov 16(%rax),%rsi
2235 mov %rax,152($context) # restore context->Rsp
2236 mov %rsi,168($context) # restore context->Rsi
2237 mov %rdi,176($context) # restore context->Rdi
2238
2239 mov 40($disp),%rdi # disp->ContextRecord
2240 mov $context,%rsi # context
2241 mov \$154,%ecx # sizeof(CONTEXT)
2242 .long 0xa548f3fc # cld; rep movsq
2243
2244 mov $disp,%rsi
2245 xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER
2246 mov 8(%rsi),%rdx # arg2, disp->ImageBase
2247 mov 0(%rsi),%r8 # arg3, disp->ControlPc
2248 mov 16(%rsi),%r9 # arg4, disp->FunctionEntry
2249 mov 40(%rsi),%r10 # disp->ContextRecord
2250 lea 56(%rsi),%r11 # &disp->HandlerData
2251 lea 24(%rsi),%r12 # &disp->EstablisherFrame
2252 mov %r10,32(%rsp) # arg5
2253 mov %r11,40(%rsp) # arg6
2254 mov %r12,48(%rsp) # arg7
2255 mov %rcx,56(%rsp) # arg8, (NULL)
2256 call *__imp_RtlVirtualUnwind(%rip)
2257
2258 mov \$1,%eax # ExceptionContinueSearch
2259 add \$64,%rsp
2260 popfq
2261 pop %r15
2262 pop %r14
2263 pop %r13
2264 pop %r12
2265 pop %rbp
2266 pop %rbx
2267 pop %rdi
2268 pop %rsi
2269 ret
2270 .size se_handler,.-se_handler
2271
2272 .section .pdata
2273 .align 4
2274 .rva .LSEH_begin_rsaz_512_sqr
2275 .rva .LSEH_end_rsaz_512_sqr
2276 .rva .LSEH_info_rsaz_512_sqr
2277
2278 .rva .LSEH_begin_rsaz_512_mul
2279 .rva .LSEH_end_rsaz_512_mul
2280 .rva .LSEH_info_rsaz_512_mul
2281
2282 .rva .LSEH_begin_rsaz_512_mul_gather4
2283 .rva .LSEH_end_rsaz_512_mul_gather4
2284 .rva .LSEH_info_rsaz_512_mul_gather4
2285
2286 .rva .LSEH_begin_rsaz_512_mul_scatter4
2287 .rva .LSEH_end_rsaz_512_mul_scatter4
2288 .rva .LSEH_info_rsaz_512_mul_scatter4
2289
2290 .rva .LSEH_begin_rsaz_512_mul_by_one
2291 .rva .LSEH_end_rsaz_512_mul_by_one
2292 .rva .LSEH_info_rsaz_512_mul_by_one
2293
2294 .rva .LSEH_begin_rsaz_512_gather4
2295 .rva .LSEH_end_rsaz_512_gather4
2296 .rva .LSEH_info_rsaz_512_gather4
2297
2298 .section .xdata
2299 .align 8
2300 .LSEH_info_rsaz_512_sqr:
2301 .byte 9,0,0,0
2302 .rva se_handler
2303 .rva .Lsqr_body,.Lsqr_epilogue # HandlerData[]
2304 .LSEH_info_rsaz_512_mul:
2305 .byte 9,0,0,0
2306 .rva se_handler
2307 .rva .Lmul_body,.Lmul_epilogue # HandlerData[]
2308 .LSEH_info_rsaz_512_mul_gather4:
2309 .byte 9,0,0,0
2310 .rva se_handler
2311 .rva .Lmul_gather4_body,.Lmul_gather4_epilogue # HandlerData[]
2312 .LSEH_info_rsaz_512_mul_scatter4:
2313 .byte 9,0,0,0
2314 .rva se_handler
2315 .rva .Lmul_scatter4_body,.Lmul_scatter4_epilogue # HandlerData[]
2316 .LSEH_info_rsaz_512_mul_by_one:
2317 .byte 9,0,0,0
2318 .rva se_handler
2319 .rva .Lmul_by_one_body,.Lmul_by_one_epilogue # HandlerData[]
2320 .LSEH_info_rsaz_512_gather4:
2321 .byte 0x01,0x46,0x16,0x00
2322 .byte 0x46,0xf8,0x09,0x00 # vmovaps 0x90(rsp),xmm15
2323 .byte 0x3d,0xe8,0x08,0x00 # vmovaps 0x80(rsp),xmm14
2324 .byte 0x34,0xd8,0x07,0x00 # vmovaps 0x70(rsp),xmm13
2325 .byte 0x2e,0xc8,0x06,0x00 # vmovaps 0x60(rsp),xmm12
2326 .byte 0x28,0xb8,0x05,0x00 # vmovaps 0x50(rsp),xmm11
2327 .byte 0x22,0xa8,0x04,0x00 # vmovaps 0x40(rsp),xmm10
2328 .byte 0x1c,0x98,0x03,0x00 # vmovaps 0x30(rsp),xmm9
2329 .byte 0x16,0x88,0x02,0x00 # vmovaps 0x20(rsp),xmm8
2330 .byte 0x10,0x78,0x01,0x00 # vmovaps 0x10(rsp),xmm7
2331 .byte 0x0b,0x68,0x00,0x00 # vmovaps 0x00(rsp),xmm6
2332 .byte 0x07,0x01,0x15,0x00 # sub rsp,0xa8
2333 ___
2334 }
2335
2336 $code =~ s/\`([^\`]*)\`/eval $1/gem;
2337 print $code;
2338 close STDOUT;