Debian Codegraph @

external/boringssl/crypto/ec/asm/p256-x86_64-asm.pl

raw
1 #!/usr/bin/env perl
2
3 # Copyright (c) 2014, Intel Corporation.
4 #
5 # Permission to use, copy, modify, and/or distribute this software for any
6 # purpose with or without fee is hereby granted, provided that the above
7 # copyright notice and this permission notice appear in all copies.
8 #
9 # THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
10 # WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
11 # MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
12 # SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
13 # WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
14 # OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
15 # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
16
17 # Developers and authors:
18 # Shay Gueron (1, 2), and Vlad Krasnov (1)
19 # (1) Intel Corporation, Israel Development Center
20 # (2) University of Haifa
21
22 # Reference:
23 # S.Gueron and V.Krasnov, "Fast Prime Field Elliptic Curve Cryptography with
24 # 256 Bit Primes"
25
26 # Further optimization by <[email protected]>:
27 #
28 # this/original
29 # Opteron +12-49%
30 # Bulldozer +14-45%
31 # P4 +18-46%
32 # Westmere +12-34%
33 # Sandy Bridge +9-35%
34 # Ivy Bridge +9-35%
35 # Haswell +8-37%
36 # Broadwell +18-58%
37 # Atom +15-50%
38 # VIA Nano +43-160%
39 #
40 # Ranges denote minimum and maximum improvement coefficients depending
41 # on benchmark.
42
43 $flavour = shift;
44 $output = shift;
45 if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }
46
47 $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
48
49 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
50 ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
51 ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
52 die "can't locate x86_64-xlate.pl";
53
54 open OUT,"| \"$^X\" $xlate $flavour $output";
55 *STDOUT=*OUT;
56
57 # TODO: enable these after testing. $avx goes to two and $addx to one.
58 $avx=0;
59 $addx=0;
60
61 $code.=<<___;
62 .text
63 .extern OPENSSL_ia32cap_P
64
65 # The polynomial
66 .align 64
67 .Lpoly:
68 .quad 0xffffffffffffffff, 0x00000000ffffffff, 0x0000000000000000, 0xffffffff00000001
69
70 .LOne:
71 .long 1,1,1,1,1,1,1,1
72 .LTwo:
73 .long 2,2,2,2,2,2,2,2
74 .LThree:
75 .long 3,3,3,3,3,3,3,3
76 .LONE_mont:
77 .quad 0x0000000000000001, 0xffffffff00000000, 0xffffffffffffffff, 0x00000000fffffffe
78 ___
79
80 {
81 ################################################################################
82 # void ecp_nistz256_mul_by_2(uint64_t res[4], uint64_t a[4]);
83
84 my ($a0,$a1,$a2,$a3)=map("%r$_",(8..11));
85 my ($t0,$t1,$t2,$t3,$t4)=("%rax","%rdx","%rcx","%r12","%r13");
86 my ($r_ptr,$a_ptr,$b_ptr)=("%rdi","%rsi","%rdx");
87
88 $code.=<<___;
89
90 .type ecp_nistz256_mul_by_2,\@function,2
91 .align 64
92 ecp_nistz256_mul_by_2:
93 push %r12
94 push %r13
95
96 mov 8*0($a_ptr), $a0
97 mov 8*1($a_ptr), $a1
98 add $a0, $a0 # a0:a3+a0:a3
99 mov 8*2($a_ptr), $a2
100 adc $a1, $a1
101 mov 8*3($a_ptr), $a3
102 lea .Lpoly(%rip), $a_ptr
103 mov $a0, $t0
104 adc $a2, $a2
105 adc $a3, $a3
106 mov $a1, $t1
107 sbb $t4, $t4
108
109 sub 8*0($a_ptr), $a0
110 mov $a2, $t2
111 sbb 8*1($a_ptr), $a1
112 sbb 8*2($a_ptr), $a2
113 mov $a3, $t3
114 sbb 8*3($a_ptr), $a3
115 test $t4, $t4
116
117 cmovz $t0, $a0
118 cmovz $t1, $a1
119 mov $a0, 8*0($r_ptr)
120 cmovz $t2, $a2
121 mov $a1, 8*1($r_ptr)
122 cmovz $t3, $a3
123 mov $a2, 8*2($r_ptr)
124 mov $a3, 8*3($r_ptr)
125
126 pop %r13
127 pop %r12
128 ret
129 .size ecp_nistz256_mul_by_2,.-ecp_nistz256_mul_by_2
130
131 ################################################################################
132 # void ecp_nistz256_neg(uint64_t res[4], uint64_t a[4]);
133 .globl ecp_nistz256_neg
134 .type ecp_nistz256_neg,\@function,2
135 .align 32
136 ecp_nistz256_neg:
137 push %r12
138 push %r13
139
140 xor $a0, $a0
141 xor $a1, $a1
142 xor $a2, $a2
143 xor $a3, $a3
144 xor $t4, $t4
145
146 sub 8*0($a_ptr), $a0
147 sbb 8*1($a_ptr), $a1
148 sbb 8*2($a_ptr), $a2
149 mov $a0, $t0
150 sbb 8*3($a_ptr), $a3
151 lea .Lpoly(%rip), $a_ptr
152 mov $a1, $t1
153 sbb \$0, $t4
154
155 add 8*0($a_ptr), $a0
156 mov $a2, $t2
157 adc 8*1($a_ptr), $a1
158 adc 8*2($a_ptr), $a2
159 mov $a3, $t3
160 adc 8*3($a_ptr), $a3
161 test $t4, $t4
162
163 cmovz $t0, $a0
164 cmovz $t1, $a1
165 mov $a0, 8*0($r_ptr)
166 cmovz $t2, $a2
167 mov $a1, 8*1($r_ptr)
168 cmovz $t3, $a3
169 mov $a2, 8*2($r_ptr)
170 mov $a3, 8*3($r_ptr)
171
172 pop %r13
173 pop %r12
174 ret
175 .size ecp_nistz256_neg,.-ecp_nistz256_neg
176 ___
177 }
178 {
179 my ($r_ptr,$a_ptr,$b_org,$b_ptr)=("%rdi","%rsi","%rdx","%rbx");
180 my ($acc0,$acc1,$acc2,$acc3,$acc4,$acc5,$acc6,$acc7)=map("%r$_",(8..15));
181 my ($t0,$t1,$t2,$t3,$t4)=("%rcx","%rbp","%rbx","%rdx","%rax");
182 my ($poly1,$poly3)=($acc6,$acc7);
183
184 $code.=<<___;
185 ################################################################################
186 # void ecp_nistz256_mul_mont(
187 # uint64_t res[4],
188 # uint64_t a[4],
189 # uint64_t b[4]);
190
191 .globl ecp_nistz256_mul_mont
192 .type ecp_nistz256_mul_mont,\@function,3
193 .align 32
194 ecp_nistz256_mul_mont:
195 ___
196 $code.=<<___ if ($addx);
197 mov \$0x80100, %ecx
198 and OPENSSL_ia32cap_P+8(%rip), %ecx
199 ___
200 $code.=<<___;
201 .Lmul_mont:
202 push %rbp
203 push %rbx
204 push %r12
205 push %r13
206 push %r14
207 push %r15
208 ___
209 $code.=<<___ if ($addx);
210 cmp \$0x80100, %ecx
211 je .Lmul_montx
212 ___
213 $code.=<<___;
214 mov $b_org, $b_ptr
215 mov 8*0($b_org), %rax
216 mov 8*0($a_ptr), $acc1
217 mov 8*1($a_ptr), $acc2
218 mov 8*2($a_ptr), $acc3
219 mov 8*3($a_ptr), $acc4
220
221 call __ecp_nistz256_mul_montq
222 ___
223 $code.=<<___ if ($addx);
224 jmp .Lmul_mont_done
225
226 .align 32
227 .Lmul_montx:
228 mov $b_org, $b_ptr
229 mov 8*0($b_org), %rdx
230 mov 8*0($a_ptr), $acc1
231 mov 8*1($a_ptr), $acc2
232 mov 8*2($a_ptr), $acc3
233 mov 8*3($a_ptr), $acc4
234 lea -128($a_ptr), $a_ptr # control u-op density
235
236 call __ecp_nistz256_mul_montx
237 ___
238 $code.=<<___;
239 .Lmul_mont_done:
240 pop %r15
241 pop %r14
242 pop %r13
243 pop %r12
244 pop %rbx
245 pop %rbp
246 ret
247 .size ecp_nistz256_mul_mont,.-ecp_nistz256_mul_mont
248
249 .type __ecp_nistz256_mul_montq,\@abi-omnipotent
250 .align 32
251 __ecp_nistz256_mul_montq:
252 ########################################################################
253 # Multiply a by b[0]
254 mov %rax, $t1
255 mulq $acc1
256 mov .Lpoly+8*1(%rip),$poly1
257 mov %rax, $acc0
258 mov $t1, %rax
259 mov %rdx, $acc1
260
261 mulq $acc2
262 mov .Lpoly+8*3(%rip),$poly3
263 add %rax, $acc1
264 mov $t1, %rax
265 adc \$0, %rdx
266 mov %rdx, $acc2
267
268 mulq $acc3
269 add %rax, $acc2
270 mov $t1, %rax
271 adc \$0, %rdx
272 mov %rdx, $acc3
273
274 mulq $acc4
275 add %rax, $acc3
276 mov $acc0, %rax
277 adc \$0, %rdx
278 xor $acc5, $acc5
279 mov %rdx, $acc4
280
281 ########################################################################
282 # First reduction step
283 # Basically now we want to multiply acc[0] by p256,
284 # and add the result to the acc.
285 # Due to the special form of p256 we do some optimizations
286 #
287 # acc[0] x p256[0..1] = acc[0] x 2^96 - acc[0]
288 # then we add acc[0] and get acc[0] x 2^96
289
290 mov $acc0, $t1
291 shl \$32, $acc0
292 mulq $poly3
293 shr \$32, $t1
294 add $acc0, $acc1 # +=acc[0]<<96
295 adc $t1, $acc2
296 adc %rax, $acc3
297 mov 8*1($b_ptr), %rax
298 adc %rdx, $acc4
299 adc \$0, $acc5
300 xor $acc0, $acc0
301
302 ########################################################################
303 # Multiply by b[1]
304 mov %rax, $t1
305 mulq 8*0($a_ptr)
306 add %rax, $acc1
307 mov $t1, %rax
308 adc \$0, %rdx
309 mov %rdx, $t0
310
311 mulq 8*1($a_ptr)
312 add $t0, $acc2
313 adc \$0, %rdx
314 add %rax, $acc2
315 mov $t1, %rax
316 adc \$0, %rdx
317 mov %rdx, $t0
318
319 mulq 8*2($a_ptr)
320 add $t0, $acc3
321 adc \$0, %rdx
322 add %rax, $acc3
323 mov $t1, %rax
324 adc \$0, %rdx
325 mov %rdx, $t0
326
327 mulq 8*3($a_ptr)
328 add $t0, $acc4
329 adc \$0, %rdx
330 add %rax, $acc4
331 mov $acc1, %rax
332 adc %rdx, $acc5
333 adc \$0, $acc0
334
335 ########################################################################
336 # Second reduction step
337 mov $acc1, $t1
338 shl \$32, $acc1
339 mulq $poly3
340 shr \$32, $t1
341 add $acc1, $acc2
342 adc $t1, $acc3
343 adc %rax, $acc4
344 mov 8*2($b_ptr), %rax
345 adc %rdx, $acc5
346 adc \$0, $acc0
347 xor $acc1, $acc1
348
349 ########################################################################
350 # Multiply by b[2]
351 mov %rax, $t1
352 mulq 8*0($a_ptr)
353 add %rax, $acc2
354 mov $t1, %rax
355 adc \$0, %rdx
356 mov %rdx, $t0
357
358 mulq 8*1($a_ptr)
359 add $t0, $acc3
360 adc \$0, %rdx
361 add %rax, $acc3
362 mov $t1, %rax
363 adc \$0, %rdx
364 mov %rdx, $t0
365
366 mulq 8*2($a_ptr)
367 add $t0, $acc4
368 adc \$0, %rdx
369 add %rax, $acc4
370 mov $t1, %rax
371 adc \$0, %rdx
372 mov %rdx, $t0
373
374 mulq 8*3($a_ptr)
375 add $t0, $acc5
376 adc \$0, %rdx
377 add %rax, $acc5
378 mov $acc2, %rax
379 adc %rdx, $acc0
380 adc \$0, $acc1
381
382 ########################################################################
383 # Third reduction step
384 mov $acc2, $t1
385 shl \$32, $acc2
386 mulq $poly3
387 shr \$32, $t1
388 add $acc2, $acc3
389 adc $t1, $acc4
390 adc %rax, $acc5
391 mov 8*3($b_ptr), %rax
392 adc %rdx, $acc0
393 adc \$0, $acc1
394 xor $acc2, $acc2
395
396 ########################################################################
397 # Multiply by b[3]
398 mov %rax, $t1
399 mulq 8*0($a_ptr)
400 add %rax, $acc3
401 mov $t1, %rax
402 adc \$0, %rdx
403 mov %rdx, $t0
404
405 mulq 8*1($a_ptr)
406 add $t0, $acc4
407 adc \$0, %rdx
408 add %rax, $acc4
409 mov $t1, %rax
410 adc \$0, %rdx
411 mov %rdx, $t0
412
413 mulq 8*2($a_ptr)
414 add $t0, $acc5
415 adc \$0, %rdx
416 add %rax, $acc5
417 mov $t1, %rax
418 adc \$0, %rdx
419 mov %rdx, $t0
420
421 mulq 8*3($a_ptr)
422 add $t0, $acc0
423 adc \$0, %rdx
424 add %rax, $acc0
425 mov $acc3, %rax
426 adc %rdx, $acc1
427 adc \$0, $acc2
428
429 ########################################################################
430 # Final reduction step
431 mov $acc3, $t1
432 shl \$32, $acc3
433 mulq $poly3
434 shr \$32, $t1
435 add $acc3, $acc4
436 adc $t1, $acc5
437 mov $acc4, $t0
438 adc %rax, $acc0
439 adc %rdx, $acc1
440 mov $acc5, $t1
441 adc \$0, $acc2
442
443 ########################################################################
444 # Branch-less conditional subtraction of P
445 sub \$-1, $acc4 # .Lpoly[0]
446 mov $acc0, $t2
447 sbb $poly1, $acc5 # .Lpoly[1]
448 sbb \$0, $acc0 # .Lpoly[2]
449 mov $acc1, $t3
450 sbb $poly3, $acc1 # .Lpoly[3]
451 sbb \$0, $acc2
452
453 cmovc $t0, $acc4
454 cmovc $t1, $acc5
455 mov $acc4, 8*0($r_ptr)
456 cmovc $t2, $acc0
457 mov $acc5, 8*1($r_ptr)
458 cmovc $t3, $acc1
459 mov $acc0, 8*2($r_ptr)
460 mov $acc1, 8*3($r_ptr)
461
462 ret
463 .size __ecp_nistz256_mul_montq,.-__ecp_nistz256_mul_montq
464
465 ################################################################################
466 # void ecp_nistz256_sqr_mont(
467 # uint64_t res[4],
468 # uint64_t a[4]);
469
470 # we optimize the square according to S.Gueron and V.Krasnov,
471 # "Speeding up Big-Number Squaring"
472 .globl ecp_nistz256_sqr_mont
473 .type ecp_nistz256_sqr_mont,\@function,2
474 .align 32
475 ecp_nistz256_sqr_mont:
476 ___
477 $code.=<<___ if ($addx);
478 mov \$0x80100, %ecx
479 and OPENSSL_ia32cap_P+8(%rip), %ecx
480 ___
481 $code.=<<___;
482 push %rbp
483 push %rbx
484 push %r12
485 push %r13
486 push %r14
487 push %r15
488 ___
489 $code.=<<___ if ($addx);
490 cmp \$0x80100, %ecx
491 je .Lsqr_montx
492 ___
493 $code.=<<___;
494 mov 8*0($a_ptr), %rax
495 mov 8*1($a_ptr), $acc6
496 mov 8*2($a_ptr), $acc7
497 mov 8*3($a_ptr), $acc0
498
499 call __ecp_nistz256_sqr_montq
500 ___
501 $code.=<<___ if ($addx);
502 jmp .Lsqr_mont_done
503
504 .align 32
505 .Lsqr_montx:
506 mov 8*0($a_ptr), %rdx
507 mov 8*1($a_ptr), $acc6
508 mov 8*2($a_ptr), $acc7
509 mov 8*3($a_ptr), $acc0
510 lea -128($a_ptr), $a_ptr # control u-op density
511
512 call __ecp_nistz256_sqr_montx
513 ___
514 $code.=<<___;
515 .Lsqr_mont_done:
516 pop %r15
517 pop %r14
518 pop %r13
519 pop %r12
520 pop %rbx
521 pop %rbp
522 ret
523 .size ecp_nistz256_sqr_mont,.-ecp_nistz256_sqr_mont
524
525 .type __ecp_nistz256_sqr_montq,\@abi-omnipotent
526 .align 32
527 __ecp_nistz256_sqr_montq:
528 mov %rax, $acc5
529 mulq $acc6 # a[1]*a[0]
530 mov %rax, $acc1
531 mov $acc7, %rax
532 mov %rdx, $acc2
533
534 mulq $acc5 # a[0]*a[2]
535 add %rax, $acc2
536 mov $acc0, %rax
537 adc \$0, %rdx
538 mov %rdx, $acc3
539
540 mulq $acc5 # a[0]*a[3]
541 add %rax, $acc3
542 mov $acc7, %rax
543 adc \$0, %rdx
544 mov %rdx, $acc4
545
546 #################################
547 mulq $acc6 # a[1]*a[2]
548 add %rax, $acc3
549 mov $acc0, %rax
550 adc \$0, %rdx
551 mov %rdx, $t1
552
553 mulq $acc6 # a[1]*a[3]
554 add %rax, $acc4
555 mov $acc0, %rax
556 adc \$0, %rdx
557 add $t1, $acc4
558 mov %rdx, $acc5
559 adc \$0, $acc5
560
561 #################################
562 mulq $acc7 # a[2]*a[3]
563 xor $acc7, $acc7
564 add %rax, $acc5
565 mov 8*0($a_ptr), %rax
566 mov %rdx, $acc6
567 adc \$0, $acc6
568
569 add $acc1, $acc1 # acc1:6<<1
570 adc $acc2, $acc2
571 adc $acc3, $acc3
572 adc $acc4, $acc4
573 adc $acc5, $acc5
574 adc $acc6, $acc6
575 adc \$0, $acc7
576
577 mulq %rax
578 mov %rax, $acc0
579 mov 8*1($a_ptr), %rax
580 mov %rdx, $t0
581
582 mulq %rax
583 add $t0, $acc1
584 adc %rax, $acc2
585 mov 8*2($a_ptr), %rax
586 adc \$0, %rdx
587 mov %rdx, $t0
588
589 mulq %rax
590 add $t0, $acc3
591 adc %rax, $acc4
592 mov 8*3($a_ptr), %rax
593 adc \$0, %rdx
594 mov %rdx, $t0
595
596 mulq %rax
597 add $t0, $acc5
598 adc %rax, $acc6
599 mov $acc0, %rax
600 adc %rdx, $acc7
601
602 mov .Lpoly+8*1(%rip), $a_ptr
603 mov .Lpoly+8*3(%rip), $t1
604
605 ##########################################
606 # Now the reduction
607 # First iteration
608 mov $acc0, $t0
609 shl \$32, $acc0
610 mulq $t1
611 shr \$32, $t0
612 add $acc0, $acc1 # +=acc[0]<<96
613 adc $t0, $acc2
614 adc %rax, $acc3
615 mov $acc1, %rax
616 adc \$0, %rdx
617
618 ##########################################
619 # Second iteration
620 mov $acc1, $t0
621 shl \$32, $acc1
622 mov %rdx, $acc0
623 mulq $t1
624 shr \$32, $t0
625 add $acc1, $acc2
626 adc $t0, $acc3
627 adc %rax, $acc0
628 mov $acc2, %rax
629 adc \$0, %rdx
630
631 ##########################################
632 # Third iteration
633 mov $acc2, $t0
634 shl \$32, $acc2
635 mov %rdx, $acc1
636 mulq $t1
637 shr \$32, $t0
638 add $acc2, $acc3
639 adc $t0, $acc0
640 adc %rax, $acc1
641 mov $acc3, %rax
642 adc \$0, %rdx
643
644 ###########################################
645 # Last iteration
646 mov $acc3, $t0
647 shl \$32, $acc3
648 mov %rdx, $acc2
649 mulq $t1
650 shr \$32, $t0
651 add $acc3, $acc0
652 adc $t0, $acc1
653 adc %rax, $acc2
654 adc \$0, %rdx
655 xor $acc3, $acc3
656
657 ############################################
658 # Add the rest of the acc
659 add $acc0, $acc4
660 adc $acc1, $acc5
661 mov $acc4, $acc0
662 adc $acc2, $acc6
663 adc %rdx, $acc7
664 mov $acc5, $acc1
665 adc \$0, $acc3
666
667 sub \$-1, $acc4 # .Lpoly[0]
668 mov $acc6, $acc2
669 sbb $a_ptr, $acc5 # .Lpoly[1]
670 sbb \$0, $acc6 # .Lpoly[2]
671 mov $acc7, $t0
672 sbb $t1, $acc7 # .Lpoly[3]
673 sbb \$0, $acc3
674
675 cmovc $acc0, $acc4
676 cmovc $acc1, $acc5
677 mov $acc4, 8*0($r_ptr)
678 cmovc $acc2, $acc6
679 mov $acc5, 8*1($r_ptr)
680 cmovc $t0, $acc7
681 mov $acc6, 8*2($r_ptr)
682 mov $acc7, 8*3($r_ptr)
683
684 ret
685 .size __ecp_nistz256_sqr_montq,.-__ecp_nistz256_sqr_montq
686 ___
687
688 if ($addx) {
689 $code.=<<___;
690 .type __ecp_nistz256_mul_montx,\@abi-omnipotent
691 .align 32
692 __ecp_nistz256_mul_montx:
693 ########################################################################
694 # Multiply by b[0]
695 mulx $acc1, $acc0, $acc1
696 mulx $acc2, $t0, $acc2
697 mov \$32, $poly1
698 xor $acc5, $acc5 # cf=0
699 mulx $acc3, $t1, $acc3
700 mov .Lpoly+8*3(%rip), $poly3
701 adc $t0, $acc1
702 mulx $acc4, $t0, $acc4
703 mov $acc0, %rdx
704 adc $t1, $acc2
705 shlx $poly1,$acc0,$t1
706 adc $t0, $acc3
707 shrx $poly1,$acc0,$t0
708 adc \$0, $acc4
709
710 ########################################################################
711 # First reduction step
712 add $t1, $acc1
713 adc $t0, $acc2
714
715 mulx $poly3, $t0, $t1
716 mov 8*1($b_ptr), %rdx
717 adc $t0, $acc3
718 adc $t1, $acc4
719 adc \$0, $acc5
720 xor $acc0, $acc0 # $acc0=0,cf=0,of=0
721
722 ########################################################################
723 # Multiply by b[1]
724 mulx 8*0+128($a_ptr), $t0, $t1
725 adcx $t0, $acc1
726 adox $t1, $acc2
727
728 mulx 8*1+128($a_ptr), $t0, $t1
729 adcx $t0, $acc2
730 adox $t1, $acc3
731
732 mulx 8*2+128($a_ptr), $t0, $t1
733 adcx $t0, $acc3
734 adox $t1, $acc4
735
736 mulx 8*3+128($a_ptr), $t0, $t1
737 mov $acc1, %rdx
738 adcx $t0, $acc4
739 shlx $poly1, $acc1, $t0
740 adox $t1, $acc5
741 shrx $poly1, $acc1, $t1
742
743 adcx $acc0, $acc5
744 adox $acc0, $acc0
745 adc \$0, $acc0
746
747 ########################################################################
748 # Second reduction step
749 add $t0, $acc2
750 adc $t1, $acc3
751
752 mulx $poly3, $t0, $t1
753 mov 8*2($b_ptr), %rdx
754 adc $t0, $acc4
755 adc $t1, $acc5
756 adc \$0, $acc0
757 xor $acc1 ,$acc1 # $acc1=0,cf=0,of=0
758
759 ########################################################################
760 # Multiply by b[2]
761 mulx 8*0+128($a_ptr), $t0, $t1
762 adcx $t0, $acc2
763 adox $t1, $acc3
764
765 mulx 8*1+128($a_ptr), $t0, $t1
766 adcx $t0, $acc3
767 adox $t1, $acc4
768
769 mulx 8*2+128($a_ptr), $t0, $t1
770 adcx $t0, $acc4
771 adox $t1, $acc5
772
773 mulx 8*3+128($a_ptr), $t0, $t1
774 mov $acc2, %rdx
775 adcx $t0, $acc5
776 shlx $poly1, $acc2, $t0
777 adox $t1, $acc0
778 shrx $poly1, $acc2, $t1
779
780 adcx $acc1, $acc0
781 adox $acc1, $acc1
782 adc \$0, $acc1
783
784 ########################################################################
785 # Third reduction step
786 add $t0, $acc3
787 adc $t1, $acc4
788
789 mulx $poly3, $t0, $t1
790 mov 8*3($b_ptr), %rdx
791 adc $t0, $acc5
792 adc $t1, $acc0
793 adc \$0, $acc1
794 xor $acc2, $acc2 # $acc2=0,cf=0,of=0
795
796 ########################################################################
797 # Multiply by b[3]
798 mulx 8*0+128($a_ptr), $t0, $t1
799 adcx $t0, $acc3
800 adox $t1, $acc4
801
802 mulx 8*1+128($a_ptr), $t0, $t1
803 adcx $t0, $acc4
804 adox $t1, $acc5
805
806 mulx 8*2+128($a_ptr), $t0, $t1
807 adcx $t0, $acc5
808 adox $t1, $acc0
809
810 mulx 8*3+128($a_ptr), $t0, $t1
811 mov $acc3, %rdx
812 adcx $t0, $acc0
813 shlx $poly1, $acc3, $t0
814 adox $t1, $acc1
815 shrx $poly1, $acc3, $t1
816
817 adcx $acc2, $acc1
818 adox $acc2, $acc2
819 adc \$0, $acc2
820
821 ########################################################################
822 # Fourth reduction step
823 add $t0, $acc4
824 adc $t1, $acc5
825
826 mulx $poly3, $t0, $t1
827 mov $acc4, $t2
828 mov .Lpoly+8*1(%rip), $poly1
829 adc $t0, $acc0
830 mov $acc5, $t3
831 adc $t1, $acc1
832 adc \$0, $acc2
833
834 ########################################################################
835 # Branch-less conditional subtraction of P
836 xor %eax, %eax
837 mov $acc0, $t0
838 sbb \$-1, $acc4 # .Lpoly[0]
839 sbb $poly1, $acc5 # .Lpoly[1]
840 sbb \$0, $acc0 # .Lpoly[2]
841 mov $acc1, $t1
842 sbb $poly3, $acc1 # .Lpoly[3]
843 sbb \$0, $acc2
844
845 cmovc $t2, $acc4
846 cmovc $t3, $acc5
847 mov $acc4, 8*0($r_ptr)
848 cmovc $t0, $acc0
849 mov $acc5, 8*1($r_ptr)
850 cmovc $t1, $acc1
851 mov $acc0, 8*2($r_ptr)
852 mov $acc1, 8*3($r_ptr)
853
854 ret
855 .size __ecp_nistz256_mul_montx,.-__ecp_nistz256_mul_montx
856
857 .type __ecp_nistz256_sqr_montx,\@abi-omnipotent
858 .align 32
859 __ecp_nistz256_sqr_montx:
860 mulx $acc6, $acc1, $acc2 # a[0]*a[1]
861 mulx $acc7, $t0, $acc3 # a[0]*a[2]
862 xor %eax, %eax
863 adc $t0, $acc2
864 mulx $acc0, $t1, $acc4 # a[0]*a[3]
865 mov $acc6, %rdx
866 adc $t1, $acc3
867 adc \$0, $acc4
868 xor $acc5, $acc5 # $acc5=0,cf=0,of=0
869
870 #################################
871 mulx $acc7, $t0, $t1 # a[1]*a[2]
872 adcx $t0, $acc3
873 adox $t1, $acc4
874
875 mulx $acc0, $t0, $t1 # a[1]*a[3]
876 mov $acc7, %rdx
877 adcx $t0, $acc4
878 adox $t1, $acc5
879 adc \$0, $acc5
880
881 #################################
882 mulx $acc0, $t0, $acc6 # a[2]*a[3]
883 mov 8*0+128($a_ptr), %rdx
884 xor $acc7, $acc7 # $acc7=0,cf=0,of=0
885 adcx $acc1, $acc1 # acc1:6<<1
886 adox $t0, $acc5
887 adcx $acc2, $acc2
888 adox $acc7, $acc6 # of=0
889
890 mulx %rdx, $acc0, $t1
891 mov 8*1+128($a_ptr), %rdx
892 adcx $acc3, $acc3
893 adox $t1, $acc1
894 adcx $acc4, $acc4
895 mulx %rdx, $t0, $t4
896 mov 8*2+128($a_ptr), %rdx
897 adcx $acc5, $acc5
898 adox $t0, $acc2
899 adcx $acc6, $acc6
900 .byte 0x67
901 mulx %rdx, $t0, $t1
902 mov 8*3+128($a_ptr), %rdx
903 adox $t4, $acc3
904 adcx $acc7, $acc7
905 adox $t0, $acc4
906 mov \$32, $a_ptr
907 adox $t1, $acc5
908 .byte 0x67,0x67
909 mulx %rdx, $t0, $t4
910 mov $acc0, %rdx
911 adox $t0, $acc6
912 shlx $a_ptr, $acc0, $t0
913 adox $t4, $acc7
914 shrx $a_ptr, $acc0, $t4
915 mov .Lpoly+8*3(%rip), $t1
916
917 # reduction step 1
918 add $t0, $acc1
919 adc $t4, $acc2
920
921 mulx $t1, $t0, $acc0
922 mov $acc1, %rdx
923 adc $t0, $acc3
924 shlx $a_ptr, $acc1, $t0
925 adc \$0, $acc0
926 shrx $a_ptr, $acc1, $t4
927
928 # reduction step 2
929 add $t0, $acc2
930 adc $t4, $acc3
931
932 mulx $t1, $t0, $acc1
933 mov $acc2, %rdx
934 adc $t0, $acc0
935 shlx $a_ptr, $acc2, $t0
936 adc \$0, $acc1
937 shrx $a_ptr, $acc2, $t4
938
939 # reduction step 3
940 add $t0, $acc3
941 adc $t4, $acc0
942
943 mulx $t1, $t0, $acc2
944 mov $acc3, %rdx
945 adc $t0, $acc1
946 shlx $a_ptr, $acc3, $t0
947 adc \$0, $acc2
948 shrx $a_ptr, $acc3, $t4
949
950 # reduction step 4
951 add $t0, $acc0
952 adc $t4, $acc1
953
954 mulx $t1, $t0, $acc3
955 adc $t0, $acc2
956 adc \$0, $acc3
957
958 xor $t3, $t3 # cf=0
959 adc $acc0, $acc4 # accumulate upper half
960 mov .Lpoly+8*1(%rip), $a_ptr
961 adc $acc1, $acc5
962 mov $acc4, $acc0
963 adc $acc2, $acc6
964 adc $acc3, $acc7
965 mov $acc5, $acc1
966 adc \$0, $t3
967
968 xor %eax, %eax # cf=0
969 sbb \$-1, $acc4 # .Lpoly[0]
970 mov $acc6, $acc2
971 sbb $a_ptr, $acc5 # .Lpoly[1]
972 sbb \$0, $acc6 # .Lpoly[2]
973 mov $acc7, $acc3
974 sbb $t1, $acc7 # .Lpoly[3]
975 sbb \$0, $t3
976
977 cmovc $acc0, $acc4
978 cmovc $acc1, $acc5
979 mov $acc4, 8*0($r_ptr)
980 cmovc $acc2, $acc6
981 mov $acc5, 8*1($r_ptr)
982 cmovc $acc3, $acc7
983 mov $acc6, 8*2($r_ptr)
984 mov $acc7, 8*3($r_ptr)
985
986 ret
987 .size __ecp_nistz256_sqr_montx,.-__ecp_nistz256_sqr_montx
988 ___
989 }
990 }
991 {
992 my ($r_ptr,$in_ptr)=("%rdi","%rsi");
993 my ($acc0,$acc1,$acc2,$acc3)=map("%r$_",(8..11));
994 my ($t0,$t1,$t2)=("%rcx","%r12","%r13");
995
996 $code.=<<___;
997 ################################################################################
998 # void ecp_nistz256_from_mont(
999 # uint64_t res[4],
1000 # uint64_t in[4]);
1001 # This one performs Montgomery multiplication by 1, so we only need the reduction
1002
1003 .globl ecp_nistz256_from_mont
1004 .type ecp_nistz256_from_mont,\@function,2
1005 .align 32
1006 ecp_nistz256_from_mont:
1007 push %r12
1008 push %r13
1009
1010 mov 8*0($in_ptr), %rax
1011 mov .Lpoly+8*3(%rip), $t2
1012 mov 8*1($in_ptr), $acc1
1013 mov 8*2($in_ptr), $acc2
1014 mov 8*3($in_ptr), $acc3
1015 mov %rax, $acc0
1016 mov .Lpoly+8*1(%rip), $t1
1017
1018 #########################################
1019 # First iteration
1020 mov %rax, $t0
1021 shl \$32, $acc0
1022 mulq $t2
1023 shr \$32, $t0
1024 add $acc0, $acc1
1025 adc $t0, $acc2
1026 adc %rax, $acc3
1027 mov $acc1, %rax
1028 adc \$0, %rdx
1029
1030 #########################################
1031 # Second iteration
1032 mov $acc1, $t0
1033 shl \$32, $acc1
1034 mov %rdx, $acc0
1035 mulq $t2
1036 shr \$32, $t0
1037 add $acc1, $acc2
1038 adc $t0, $acc3
1039 adc %rax, $acc0
1040 mov $acc2, %rax
1041 adc \$0, %rdx
1042
1043 ##########################################
1044 # Third iteration
1045 mov $acc2, $t0
1046 shl \$32, $acc2
1047 mov %rdx, $acc1
1048 mulq $t2
1049 shr \$32, $t0
1050 add $acc2, $acc3
1051 adc $t0, $acc0
1052 adc %rax, $acc1
1053 mov $acc3, %rax
1054 adc \$0, %rdx
1055
1056 ###########################################
1057 # Last iteration
1058 mov $acc3, $t0
1059 shl \$32, $acc3
1060 mov %rdx, $acc2
1061 mulq $t2
1062 shr \$32, $t0
1063 add $acc3, $acc0
1064 adc $t0, $acc1
1065 mov $acc0, $t0
1066 adc %rax, $acc2
1067 mov $acc1, $in_ptr
1068 adc \$0, %rdx
1069
1070 sub \$-1, $acc0
1071 mov $acc2, %rax
1072 sbb $t1, $acc1
1073 sbb \$0, $acc2
1074 mov %rdx, $acc3
1075 sbb $t2, %rdx
1076 sbb $t2, $t2
1077
1078 cmovnz $t0, $acc0
1079 cmovnz $in_ptr, $acc1
1080 mov $acc0, 8*0($r_ptr)
1081 cmovnz %rax, $acc2
1082 mov $acc1, 8*1($r_ptr)
1083 cmovz %rdx, $acc3
1084 mov $acc2, 8*2($r_ptr)
1085 mov $acc3, 8*3($r_ptr)
1086
1087 pop %r13
1088 pop %r12
1089 ret
1090 .size ecp_nistz256_from_mont,.-ecp_nistz256_from_mont
1091 ___
1092 }
1093 {
1094 my ($val,$in_t,$index)=$win64?("%rcx","%rdx","%r8d"):("%rdi","%rsi","%edx");
1095 my ($ONE,$INDEX,$Ra,$Rb,$Rc,$Rd,$Re,$Rf)=map("%xmm$_",(0..7));
1096 my ($M0,$T0a,$T0b,$T0c,$T0d,$T0e,$T0f,$TMP0)=map("%xmm$_",(8..15));
1097 my ($M1,$T2a,$T2b,$TMP2,$M2,$T2a,$T2b,$TMP2)=map("%xmm$_",(8..15));
1098
1099 $code.=<<___;
1100 ################################################################################
1101 # void ecp_nistz256_select_w5(uint64_t *val, uint64_t *in_t, int index);
1102 .globl ecp_nistz256_select_w5
1103 .type ecp_nistz256_select_w5,\@abi-omnipotent
1104 .align 32
1105 ecp_nistz256_select_w5:
1106 ___
1107 $code.=<<___ if ($avx>1);
1108 mov OPENSSL_ia32cap_P+8(%rip), %eax
1109 test \$`1<<5`, %eax
1110 jnz .Lavx2_select_w5
1111 ___
1112 $code.=<<___ if ($win64);
1113 lea -0x88(%rsp), %rax
1114 .LSEH_begin_ecp_nistz256_select_w5:
1115 .byte 0x48,0x8d,0x60,0xe0 #lea -0x20(%rax), %rsp
1116 .byte 0x0f,0x29,0x70,0xe0 #movaps %xmm6, -0x20(%rax)
1117 .byte 0x0f,0x29,0x78,0xf0 #movaps %xmm7, -0x10(%rax)
1118 .byte 0x44,0x0f,0x29,0x00 #movaps %xmm8, 0(%rax)
1119 .byte 0x44,0x0f,0x29,0x48,0x10 #movaps %xmm9, 0x10(%rax)
1120 .byte 0x44,0x0f,0x29,0x50,0x20 #movaps %xmm10, 0x20(%rax)
1121 .byte 0x44,0x0f,0x29,0x58,0x30 #movaps %xmm11, 0x30(%rax)
1122 .byte 0x44,0x0f,0x29,0x60,0x40 #movaps %xmm12, 0x40(%rax)
1123 .byte 0x44,0x0f,0x29,0x68,0x50 #movaps %xmm13, 0x50(%rax)
1124 .byte 0x44,0x0f,0x29,0x70,0x60 #movaps %xmm14, 0x60(%rax)
1125 .byte 0x44,0x0f,0x29,0x78,0x70 #movaps %xmm15, 0x70(%rax)
1126 ___
1127 $code.=<<___;
1128 movdqa .LOne(%rip), $ONE
1129 movd $index, $INDEX
1130
1131 pxor $Ra, $Ra
1132 pxor $Rb, $Rb
1133 pxor $Rc, $Rc
1134 pxor $Rd, $Rd
1135 pxor $Re, $Re
1136 pxor $Rf, $Rf
1137
1138 movdqa $ONE, $M0
1139 pshufd \$0, $INDEX, $INDEX
1140
1141 mov \$16, %rax
1142 .Lselect_loop_sse_w5:
1143
1144 movdqa $M0, $TMP0
1145 paddd $ONE, $M0
1146 pcmpeqd $INDEX, $TMP0
1147
1148 movdqa 16*0($in_t), $T0a
1149 movdqa 16*1($in_t), $T0b
1150 movdqa 16*2($in_t), $T0c
1151 movdqa 16*3($in_t), $T0d
1152 movdqa 16*4($in_t), $T0e
1153 movdqa 16*5($in_t), $T0f
1154 lea 16*6($in_t), $in_t
1155
1156 pand $TMP0, $T0a
1157 pand $TMP0, $T0b
1158 por $T0a, $Ra
1159 pand $TMP0, $T0c
1160 por $T0b, $Rb
1161 pand $TMP0, $T0d
1162 por $T0c, $Rc
1163 pand $TMP0, $T0e
1164 por $T0d, $Rd
1165 pand $TMP0, $T0f
1166 por $T0e, $Re
1167 por $T0f, $Rf
1168
1169 dec %rax
1170 jnz .Lselect_loop_sse_w5
1171
1172 movdqu $Ra, 16*0($val)
1173 movdqu $Rb, 16*1($val)
1174 movdqu $Rc, 16*2($val)
1175 movdqu $Rd, 16*3($val)
1176 movdqu $Re, 16*4($val)
1177 movdqu $Rf, 16*5($val)
1178 ___
1179 $code.=<<___ if ($win64);
1180 movaps (%rsp), %xmm6
1181 movaps 0x10(%rsp), %xmm7
1182 movaps 0x20(%rsp), %xmm8
1183 movaps 0x30(%rsp), %xmm9
1184 movaps 0x40(%rsp), %xmm10
1185 movaps 0x50(%rsp), %xmm11
1186 movaps 0x60(%rsp), %xmm12
1187 movaps 0x70(%rsp), %xmm13
1188 movaps 0x80(%rsp), %xmm14
1189 movaps 0x90(%rsp), %xmm15
1190 lea 0xa8(%rsp), %rsp
1191 .LSEH_end_ecp_nistz256_select_w5:
1192 ___
1193 $code.=<<___;
1194 ret
1195 .size ecp_nistz256_select_w5,.-ecp_nistz256_select_w5
1196
1197 ################################################################################
1198 # void ecp_nistz256_select_w7(uint64_t *val, uint64_t *in_t, int index);
1199 .globl ecp_nistz256_select_w7
1200 .type ecp_nistz256_select_w7,\@abi-omnipotent
1201 .align 32
1202 ecp_nistz256_select_w7:
1203 ___
1204 $code.=<<___ if ($avx>1);
1205 mov OPENSSL_ia32cap_P+8(%rip), %eax
1206 test \$`1<<5`, %eax
1207 jnz .Lavx2_select_w7
1208 ___
1209 $code.=<<___ if ($win64);
1210 lea -0x88(%rsp), %rax
1211 .LSEH_begin_ecp_nistz256_select_w7:
1212 .byte 0x48,0x8d,0x60,0xe0 #lea -0x20(%rax), %rsp
1213 .byte 0x0f,0x29,0x70,0xe0 #movaps %xmm6, -0x20(%rax)
1214 .byte 0x0f,0x29,0x78,0xf0 #movaps %xmm7, -0x10(%rax)
1215 .byte 0x44,0x0f,0x29,0x00 #movaps %xmm8, 0(%rax)
1216 .byte 0x44,0x0f,0x29,0x48,0x10 #movaps %xmm9, 0x10(%rax)
1217 .byte 0x44,0x0f,0x29,0x50,0x20 #movaps %xmm10, 0x20(%rax)
1218 .byte 0x44,0x0f,0x29,0x58,0x30 #movaps %xmm11, 0x30(%rax)
1219 .byte 0x44,0x0f,0x29,0x60,0x40 #movaps %xmm12, 0x40(%rax)
1220 .byte 0x44,0x0f,0x29,0x68,0x50 #movaps %xmm13, 0x50(%rax)
1221 .byte 0x44,0x0f,0x29,0x70,0x60 #movaps %xmm14, 0x60(%rax)
1222 .byte 0x44,0x0f,0x29,0x78,0x70 #movaps %xmm15, 0x70(%rax)
1223 ___
1224 $code.=<<___;
1225 movdqa .LOne(%rip), $M0
1226 movd $index, $INDEX
1227
1228 pxor $Ra, $Ra
1229 pxor $Rb, $Rb
1230 pxor $Rc, $Rc
1231 pxor $Rd, $Rd
1232
1233 movdqa $M0, $ONE
1234 pshufd \$0, $INDEX, $INDEX
1235 mov \$64, %rax
1236
1237 .Lselect_loop_sse_w7:
1238 movdqa $M0, $TMP0
1239 paddd $ONE, $M0
1240 movdqa 16*0($in_t), $T0a
1241 movdqa 16*1($in_t), $T0b
1242 pcmpeqd $INDEX, $TMP0
1243 movdqa 16*2($in_t), $T0c
1244 movdqa 16*3($in_t), $T0d
1245 lea 16*4($in_t), $in_t
1246
1247 pand $TMP0, $T0a
1248 pand $TMP0, $T0b
1249 por $T0a, $Ra
1250 pand $TMP0, $T0c
1251 por $T0b, $Rb
1252 pand $TMP0, $T0d
1253 por $T0c, $Rc
1254 prefetcht0 255($in_t)
1255 por $T0d, $Rd
1256
1257 dec %rax
1258 jnz .Lselect_loop_sse_w7
1259
1260 movdqu $Ra, 16*0($val)
1261 movdqu $Rb, 16*1($val)
1262 movdqu $Rc, 16*2($val)
1263 movdqu $Rd, 16*3($val)
1264 ___
1265 $code.=<<___ if ($win64);
1266 movaps (%rsp), %xmm6
1267 movaps 0x10(%rsp), %xmm7
1268 movaps 0x20(%rsp), %xmm8
1269 movaps 0x30(%rsp), %xmm9
1270 movaps 0x40(%rsp), %xmm10
1271 movaps 0x50(%rsp), %xmm11
1272 movaps 0x60(%rsp), %xmm12
1273 movaps 0x70(%rsp), %xmm13
1274 movaps 0x80(%rsp), %xmm14
1275 movaps 0x90(%rsp), %xmm15
1276 lea 0xa8(%rsp), %rsp
1277 .LSEH_end_ecp_nistz256_select_w7:
1278 ___
1279 $code.=<<___;
1280 ret
1281 .size ecp_nistz256_select_w7,.-ecp_nistz256_select_w7
1282 ___
1283 }
1284 if ($avx>1) {
1285 my ($val,$in_t,$index)=$win64?("%rcx","%rdx","%r8d"):("%rdi","%rsi","%edx");
1286 my ($TWO,$INDEX,$Ra,$Rb,$Rc)=map("%ymm$_",(0..4));
1287 my ($M0,$T0a,$T0b,$T0c,$TMP0)=map("%ymm$_",(5..9));
1288 my ($M1,$T1a,$T1b,$T1c,$TMP1)=map("%ymm$_",(10..14));
1289
1290 $code.=<<___;
1291 ################################################################################
1292 # void ecp_nistz256_avx2_select_w5(uint64_t *val, uint64_t *in_t, int index);
1293 .type ecp_nistz256_avx2_select_w5,\@abi-omnipotent
1294 .align 32
1295 ecp_nistz256_avx2_select_w5:
1296 .Lavx2_select_w5:
1297 vzeroupper
1298 ___
1299 $code.=<<___ if ($win64);
1300 lea -0x88(%rsp), %rax
1301 .LSEH_begin_ecp_nistz256_avx2_select_w5:
1302 .byte 0x48,0x8d,0x60,0xe0 #lea -0x20(%rax), %rsp
1303 .byte 0xc5,0xf8,0x29,0x70,0xe0 #vmovaps %xmm6, -0x20(%rax)
1304 .byte 0xc5,0xf8,0x29,0x78,0xf0 #vmovaps %xmm7, -0x10(%rax)
1305 .byte 0xc5,0x78,0x29,0x40,0x00 #vmovaps %xmm8, 8(%rax)
1306 .byte 0xc5,0x78,0x29,0x48,0x10 #vmovaps %xmm9, 0x10(%rax)
1307 .byte 0xc5,0x78,0x29,0x50,0x20 #vmovaps %xmm10, 0x20(%rax)
1308 .byte 0xc5,0x78,0x29,0x58,0x30 #vmovaps %xmm11, 0x30(%rax)
1309 .byte 0xc5,0x78,0x29,0x60,0x40 #vmovaps %xmm12, 0x40(%rax)
1310 .byte 0xc5,0x78,0x29,0x68,0x50 #vmovaps %xmm13, 0x50(%rax)
1311 .byte 0xc5,0x78,0x29,0x70,0x60 #vmovaps %xmm14, 0x60(%rax)
1312 .byte 0xc5,0x78,0x29,0x78,0x70 #vmovaps %xmm15, 0x70(%rax)
1313 ___
1314 $code.=<<___;
1315 vmovdqa .LTwo(%rip), $TWO
1316
1317 vpxor $Ra, $Ra, $Ra
1318 vpxor $Rb, $Rb, $Rb
1319 vpxor $Rc, $Rc, $Rc
1320
1321 vmovdqa .LOne(%rip), $M0
1322 vmovdqa .LTwo(%rip), $M1
1323
1324 vmovd $index, %xmm1
1325 vpermd $INDEX, $Ra, $INDEX
1326
1327 mov \$8, %rax
1328 .Lselect_loop_avx2_w5:
1329
1330 vmovdqa 32*0($in_t), $T0a
1331 vmovdqa 32*1($in_t), $T0b
1332 vmovdqa 32*2($in_t), $T0c
1333
1334 vmovdqa 32*3($in_t), $T1a
1335 vmovdqa 32*4($in_t), $T1b
1336 vmovdqa 32*5($in_t), $T1c
1337
1338 vpcmpeqd $INDEX, $M0, $TMP0
1339 vpcmpeqd $INDEX, $M1, $TMP1
1340
1341 vpaddd $TWO, $M0, $M0
1342 vpaddd $TWO, $M1, $M1
1343 lea 32*6($in_t), $in_t
1344
1345 vpand $TMP0, $T0a, $T0a
1346 vpand $TMP0, $T0b, $T0b
1347 vpand $TMP0, $T0c, $T0c
1348 vpand $TMP1, $T1a, $T1a
1349 vpand $TMP1, $T1b, $T1b
1350 vpand $TMP1, $T1c, $T1c
1351
1352 vpxor $T0a, $Ra, $Ra
1353 vpxor $T0b, $Rb, $Rb
1354 vpxor $T0c, $Rc, $Rc
1355 vpxor $T1a, $Ra, $Ra
1356 vpxor $T1b, $Rb, $Rb
1357 vpxor $T1c, $Rc, $Rc
1358
1359 dec %rax
1360 jnz .Lselect_loop_avx2_w5
1361
1362 vmovdqu $Ra, 32*0($val)
1363 vmovdqu $Rb, 32*1($val)
1364 vmovdqu $Rc, 32*2($val)
1365 vzeroupper
1366 ___
1367 $code.=<<___ if ($win64);
1368 movaps (%rsp), %xmm6
1369 movaps 0x10(%rsp), %xmm7
1370 movaps 0x20(%rsp), %xmm8
1371 movaps 0x30(%rsp), %xmm9
1372 movaps 0x40(%rsp), %xmm10
1373 movaps 0x50(%rsp), %xmm11
1374 movaps 0x60(%rsp), %xmm12
1375 movaps 0x70(%rsp), %xmm13
1376 movaps 0x80(%rsp), %xmm14
1377 movaps 0x90(%rsp), %xmm15
1378 lea 0xa8(%rsp), %rsp
1379 .LSEH_end_ecp_nistz256_avx2_select_w5:
1380 ___
1381 $code.=<<___;
1382 ret
1383 .size ecp_nistz256_avx2_select_w5,.-ecp_nistz256_avx2_select_w5
1384 ___
1385 }
1386 if ($avx>1) {
1387 my ($val,$in_t,$index)=$win64?("%rcx","%rdx","%r8d"):("%rdi","%rsi","%edx");
1388 my ($THREE,$INDEX,$Ra,$Rb)=map("%ymm$_",(0..3));
1389 my ($M0,$T0a,$T0b,$TMP0)=map("%ymm$_",(4..7));
1390 my ($M1,$T1a,$T1b,$TMP1)=map("%ymm$_",(8..11));
1391 my ($M2,$T2a,$T2b,$TMP2)=map("%ymm$_",(12..15));
1392
1393 $code.=<<___;
1394
1395 ################################################################################
1396 # void ecp_nistz256_avx2_select_w7(uint64_t *val, uint64_t *in_t, int index);
1397 .globl ecp_nistz256_avx2_select_w7
1398 .type ecp_nistz256_avx2_select_w7,\@abi-omnipotent
1399 .align 32
1400 ecp_nistz256_avx2_select_w7:
1401 .Lavx2_select_w7:
1402 vzeroupper
1403 ___
1404 $code.=<<___ if ($win64);
1405 lea -0x88(%rsp), %rax
1406 .LSEH_begin_ecp_nistz256_avx2_select_w7:
1407 .byte 0x48,0x8d,0x60,0xe0 #lea -0x20(%rax), %rsp
1408 .byte 0xc5,0xf8,0x29,0x70,0xe0 #vmovaps %xmm6, -0x20(%rax)
1409 .byte 0xc5,0xf8,0x29,0x78,0xf0 #vmovaps %xmm7, -0x10(%rax)
1410 .byte 0xc5,0x78,0x29,0x40,0x00 #vmovaps %xmm8, 8(%rax)
1411 .byte 0xc5,0x78,0x29,0x48,0x10 #vmovaps %xmm9, 0x10(%rax)
1412 .byte 0xc5,0x78,0x29,0x50,0x20 #vmovaps %xmm10, 0x20(%rax)
1413 .byte 0xc5,0x78,0x29,0x58,0x30 #vmovaps %xmm11, 0x30(%rax)
1414 .byte 0xc5,0x78,0x29,0x60,0x40 #vmovaps %xmm12, 0x40(%rax)
1415 .byte 0xc5,0x78,0x29,0x68,0x50 #vmovaps %xmm13, 0x50(%rax)
1416 .byte 0xc5,0x78,0x29,0x70,0x60 #vmovaps %xmm14, 0x60(%rax)
1417 .byte 0xc5,0x78,0x29,0x78,0x70 #vmovaps %xmm15, 0x70(%rax)
1418 ___
1419 $code.=<<___;
1420 vmovdqa .LThree(%rip), $THREE
1421
1422 vpxor $Ra, $Ra, $Ra
1423 vpxor $Rb, $Rb, $Rb
1424
1425 vmovdqa .LOne(%rip), $M0
1426 vmovdqa .LTwo(%rip), $M1
1427 vmovdqa .LThree(%rip), $M2
1428
1429 vmovd $index, %xmm1
1430 vpermd $INDEX, $Ra, $INDEX
1431 # Skip index = 0, because it is implicitly the point at infinity
1432
1433 mov \$21, %rax
1434 .Lselect_loop_avx2_w7:
1435
1436 vmovdqa 32*0($in_t), $T0a
1437 vmovdqa 32*1($in_t), $T0b
1438
1439 vmovdqa 32*2($in_t), $T1a
1440 vmovdqa 32*3($in_t), $T1b
1441
1442 vmovdqa 32*4($in_t), $T2a
1443 vmovdqa 32*5($in_t), $T2b
1444
1445 vpcmpeqd $INDEX, $M0, $TMP0
1446 vpcmpeqd $INDEX, $M1, $TMP1
1447 vpcmpeqd $INDEX, $M2, $TMP2
1448
1449 vpaddd $THREE, $M0, $M0
1450 vpaddd $THREE, $M1, $M1
1451 vpaddd $THREE, $M2, $M2
1452 lea 32*6($in_t), $in_t
1453
1454 vpand $TMP0, $T0a, $T0a
1455 vpand $TMP0, $T0b, $T0b
1456 vpand $TMP1, $T1a, $T1a
1457 vpand $TMP1, $T1b, $T1b
1458 vpand $TMP2, $T2a, $T2a
1459 vpand $TMP2, $T2b, $T2b
1460
1461 vpxor $T0a, $Ra, $Ra
1462 vpxor $T0b, $Rb, $Rb
1463 vpxor $T1a, $Ra, $Ra
1464 vpxor $T1b, $Rb, $Rb
1465 vpxor $T2a, $Ra, $Ra
1466 vpxor $T2b, $Rb, $Rb
1467
1468 dec %rax
1469 jnz .Lselect_loop_avx2_w7
1470
1471
1472 vmovdqa 32*0($in_t), $T0a
1473 vmovdqa 32*1($in_t), $T0b
1474
1475 vpcmpeqd $INDEX, $M0, $TMP0
1476
1477 vpand $TMP0, $T0a, $T0a
1478 vpand $TMP0, $T0b, $T0b
1479
1480 vpxor $T0a, $Ra, $Ra
1481 vpxor $T0b, $Rb, $Rb
1482
1483 vmovdqu $Ra, 32*0($val)
1484 vmovdqu $Rb, 32*1($val)
1485 vzeroupper
1486 ___
1487 $code.=<<___ if ($win64);
1488 movaps (%rsp), %xmm6
1489 movaps 0x10(%rsp), %xmm7
1490 movaps 0x20(%rsp), %xmm8
1491 movaps 0x30(%rsp), %xmm9
1492 movaps 0x40(%rsp), %xmm10
1493 movaps 0x50(%rsp), %xmm11
1494 movaps 0x60(%rsp), %xmm12
1495 movaps 0x70(%rsp), %xmm13
1496 movaps 0x80(%rsp), %xmm14
1497 movaps 0x90(%rsp), %xmm15
1498 lea 0xa8(%rsp), %rsp
1499 .LSEH_end_ecp_nistz256_avx2_select_w7:
1500 ___
1501 $code.=<<___;
1502 ret
1503 .size ecp_nistz256_avx2_select_w7,.-ecp_nistz256_avx2_select_w7
1504 ___
1505 } else {
1506 $code.=<<___;
1507 .globl ecp_nistz256_avx2_select_w7
1508 .type ecp_nistz256_avx2_select_w7,\@function,3
1509 .align 32
1510 ecp_nistz256_avx2_select_w7:
1511 .byte 0x0f,0x0b # ud2
1512 ret
1513 .size ecp_nistz256_avx2_select_w7,.-ecp_nistz256_avx2_select_w7
1514 ___
1515 }
1516 {{{
1517 ########################################################################
1518 # This block implements higher level point_double, point_add and
1519 # point_add_affine. The key to performance in this case is to allow
1520 # out-of-order execution logic to overlap computations from next step
1521 # with tail processing from current step. By using tailored calling
1522 # sequence we minimize inter-step overhead to give processor better
1523 # shot at overlapping operations...
1524 #
1525 # You will notice that input data is copied to stack. Trouble is that
1526 # there are no registers to spare for holding original pointers and
1527 # reloading them, pointers, would create undesired dependencies on
1528 # effective addresses calculation paths. In other words it's too done
1529 # to favour out-of-order execution logic.
1530 # <[email protected]>
1531
1532 my ($r_ptr,$a_ptr,$b_org,$b_ptr)=("%rdi","%rsi","%rdx","%rbx");
1533 my ($acc0,$acc1,$acc2,$acc3,$acc4,$acc5,$acc6,$acc7)=map("%r$_",(8..15));
1534 my ($t0,$t1,$t2,$t3,$t4)=("%rax","%rbp","%rcx",$acc4,$acc4);
1535 my ($poly1,$poly3)=($acc6,$acc7);
1536
1537 sub load_for_mul () {
1538 my ($a,$b,$src0) = @_;
1539 my $bias = $src0 eq "%rax" ? 0 : -128;
1540
1541 " mov $b, $src0
1542 lea $b, $b_ptr
1543 mov 8*0+$a, $acc1
1544 mov 8*1+$a, $acc2
1545 lea $bias+$a, $a_ptr
1546 mov 8*2+$a, $acc3
1547 mov 8*3+$a, $acc4"
1548 }
1549
1550 sub load_for_sqr () {
1551 my ($a,$src0) = @_;
1552 my $bias = $src0 eq "%rax" ? 0 : -128;
1553
1554 " mov 8*0+$a, $src0
1555 mov 8*1+$a, $acc6
1556 lea $bias+$a, $a_ptr
1557 mov 8*2+$a, $acc7
1558 mov 8*3+$a, $acc0"
1559 }
1560
1561 {
1562 ########################################################################
1563 # operate in 4-5-0-1 "name space" that matches multiplication output
1564 #
1565 my ($a0,$a1,$a2,$a3,$t3,$t4)=($acc4,$acc5,$acc0,$acc1,$acc2,$acc3);
1566
1567 $code.=<<___;
1568 .type __ecp_nistz256_add_toq,\@abi-omnipotent
1569 .align 32
1570 __ecp_nistz256_add_toq:
1571 add 8*0($b_ptr), $a0
1572 adc 8*1($b_ptr), $a1
1573 mov $a0, $t0
1574 adc 8*2($b_ptr), $a2
1575 adc 8*3($b_ptr), $a3
1576 mov $a1, $t1
1577 sbb $t4, $t4
1578
1579 sub \$-1, $a0
1580 mov $a2, $t2
1581 sbb $poly1, $a1
1582 sbb \$0, $a2
1583 mov $a3, $t3
1584 sbb $poly3, $a3
1585 test $t4, $t4
1586
1587 cmovz $t0, $a0
1588 cmovz $t1, $a1
1589 mov $a0, 8*0($r_ptr)
1590 cmovz $t2, $a2
1591 mov $a1, 8*1($r_ptr)
1592 cmovz $t3, $a3
1593 mov $a2, 8*2($r_ptr)
1594 mov $a3, 8*3($r_ptr)
1595
1596 ret
1597 .size __ecp_nistz256_add_toq,.-__ecp_nistz256_add_toq
1598
1599 .type __ecp_nistz256_sub_fromq,\@abi-omnipotent
1600 .align 32
1601 __ecp_nistz256_sub_fromq:
1602 sub 8*0($b_ptr), $a0
1603 sbb 8*1($b_ptr), $a1
1604 mov $a0, $t0
1605 sbb 8*2($b_ptr), $a2
1606 sbb 8*3($b_ptr), $a3
1607 mov $a1, $t1
1608 sbb $t4, $t4
1609
1610 add \$-1, $a0
1611 mov $a2, $t2
1612 adc $poly1, $a1
1613 adc \$0, $a2
1614 mov $a3, $t3
1615 adc $poly3, $a3
1616 test $t4, $t4
1617
1618 cmovz $t0, $a0
1619 cmovz $t1, $a1
1620 mov $a0, 8*0($r_ptr)
1621 cmovz $t2, $a2
1622 mov $a1, 8*1($r_ptr)
1623 cmovz $t3, $a3
1624 mov $a2, 8*2($r_ptr)
1625 mov $a3, 8*3($r_ptr)
1626
1627 ret
1628 .size __ecp_nistz256_sub_fromq,.-__ecp_nistz256_sub_fromq
1629
1630 .type __ecp_nistz256_subq,\@abi-omnipotent
1631 .align 32
1632 __ecp_nistz256_subq:
1633 sub $a0, $t0
1634 sbb $a1, $t1
1635 mov $t0, $a0
1636 sbb $a2, $t2
1637 sbb $a3, $t3
1638 mov $t1, $a1
1639 sbb $t4, $t4
1640
1641 add \$-1, $t0
1642 mov $t2, $a2
1643 adc $poly1, $t1
1644 adc \$0, $t2
1645 mov $t3, $a3
1646 adc $poly3, $t3
1647 test $t4, $t4
1648
1649 cmovnz $t0, $a0
1650 cmovnz $t1, $a1
1651 cmovnz $t2, $a2
1652 cmovnz $t3, $a3
1653
1654 ret
1655 .size __ecp_nistz256_subq,.-__ecp_nistz256_subq
1656
1657 .type __ecp_nistz256_mul_by_2q,\@abi-omnipotent
1658 .align 32
1659 __ecp_nistz256_mul_by_2q:
1660 add $a0, $a0 # a0:a3+a0:a3
1661 adc $a1, $a1
1662 mov $a0, $t0
1663 adc $a2, $a2
1664 adc $a3, $a3
1665 mov $a1, $t1
1666 sbb $t4, $t4
1667
1668 sub \$-1, $a0
1669 mov $a2, $t2
1670 sbb $poly1, $a1
1671 sbb \$0, $a2
1672 mov $a3, $t3
1673 sbb $poly3, $a3
1674 test $t4, $t4
1675
1676 cmovz $t0, $a0
1677 cmovz $t1, $a1
1678 mov $a0, 8*0($r_ptr)
1679 cmovz $t2, $a2
1680 mov $a1, 8*1($r_ptr)
1681 cmovz $t3, $a3
1682 mov $a2, 8*2($r_ptr)
1683 mov $a3, 8*3($r_ptr)
1684
1685 ret
1686 .size __ecp_nistz256_mul_by_2q,.-__ecp_nistz256_mul_by_2q
1687 ___
1688 }
1689 sub gen_double () {
1690 my $x = shift;
1691 my ($src0,$sfx,$bias);
1692 my ($S,$M,$Zsqr,$in_x,$tmp0)=map(32*$_,(0..4));
1693
1694 if ($x ne "x") {
1695 $src0 = "%rax";
1696 $sfx = "";
1697 $bias = 0;
1698
1699 $code.=<<___;
1700 .globl ecp_nistz256_point_double
1701 .type ecp_nistz256_point_double,\@function,2
1702 .align 32
1703 ecp_nistz256_point_double:
1704 ___
1705 $code.=<<___ if ($addx);
1706 mov \$0x80100, %ecx
1707 and OPENSSL_ia32cap_P+8(%rip), %ecx
1708 cmp \$0x80100, %ecx
1709 je .Lpoint_doublex
1710 ___
1711 } else {
1712 $src0 = "%rdx";
1713 $sfx = "x";
1714 $bias = 128;
1715
1716 $code.=<<___;
1717 .type ecp_nistz256_point_doublex,\@function,2
1718 .align 32
1719 ecp_nistz256_point_doublex:
1720 .Lpoint_doublex:
1721 ___
1722 }
1723 $code.=<<___;
1724 push %rbp
1725 push %rbx
1726 push %r12
1727 push %r13
1728 push %r14
1729 push %r15
1730 sub \$32*5+8, %rsp
1731
1732 .Lpoint_double_shortcut$x:
1733 movdqu 0x00($a_ptr), %xmm0 # copy *(P256_POINT *)$a_ptr.x
1734 mov $a_ptr, $b_ptr # backup copy
1735 movdqu 0x10($a_ptr), %xmm1
1736 mov 0x20+8*0($a_ptr), $acc4 # load in_y in "5-4-0-1" order
1737 mov 0x20+8*1($a_ptr), $acc5
1738 mov 0x20+8*2($a_ptr), $acc0
1739 mov 0x20+8*3($a_ptr), $acc1
1740 mov .Lpoly+8*1(%rip), $poly1
1741 mov .Lpoly+8*3(%rip), $poly3
1742 movdqa %xmm0, $in_x(%rsp)
1743 movdqa %xmm1, $in_x+0x10(%rsp)
1744 lea 0x20($r_ptr), $acc2
1745 lea 0x40($r_ptr), $acc3
1746 movq $r_ptr, %xmm0
1747 movq $acc2, %xmm1
1748 movq $acc3, %xmm2
1749
1750 lea $S(%rsp), $r_ptr
1751 call __ecp_nistz256_mul_by_2$x # p256_mul_by_2(S, in_y);
1752
1753 mov 0x40+8*0($a_ptr), $src0
1754 mov 0x40+8*1($a_ptr), $acc6
1755 mov 0x40+8*2($a_ptr), $acc7
1756 mov 0x40+8*3($a_ptr), $acc0
1757 lea 0x40-$bias($a_ptr), $a_ptr
1758 lea $Zsqr(%rsp), $r_ptr
1759 call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Zsqr, in_z);
1760
1761 `&load_for_sqr("$S(%rsp)", "$src0")`
1762 lea $S(%rsp), $r_ptr
1763 call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(S, S);
1764
1765 mov 0x20($b_ptr), $src0 # $b_ptr is still valid
1766 mov 0x40+8*0($b_ptr), $acc1
1767 mov 0x40+8*1($b_ptr), $acc2
1768 mov 0x40+8*2($b_ptr), $acc3
1769 mov 0x40+8*3($b_ptr), $acc4
1770 lea 0x40-$bias($b_ptr), $a_ptr
1771 lea 0x20($b_ptr), $b_ptr
1772 movq %xmm2, $r_ptr
1773 call __ecp_nistz256_mul_mont$x # p256_mul_mont(res_z, in_z, in_y);
1774 call __ecp_nistz256_mul_by_2$x # p256_mul_by_2(res_z, res_z);
1775
1776 mov $in_x+8*0(%rsp), $acc4 # "5-4-0-1" order
1777 mov $in_x+8*1(%rsp), $acc5
1778 lea $Zsqr(%rsp), $b_ptr
1779 mov $in_x+8*2(%rsp), $acc0
1780 mov $in_x+8*3(%rsp), $acc1
1781 lea $M(%rsp), $r_ptr
1782 call __ecp_nistz256_add_to$x # p256_add(M, in_x, Zsqr);
1783
1784 mov $in_x+8*0(%rsp), $acc4 # "5-4-0-1" order
1785 mov $in_x+8*1(%rsp), $acc5
1786 lea $Zsqr(%rsp), $b_ptr
1787 mov $in_x+8*2(%rsp), $acc0
1788 mov $in_x+8*3(%rsp), $acc1
1789 lea $Zsqr(%rsp), $r_ptr
1790 call __ecp_nistz256_sub_from$x # p256_sub(Zsqr, in_x, Zsqr);
1791
1792 `&load_for_sqr("$S(%rsp)", "$src0")`
1793 movq %xmm1, $r_ptr
1794 call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(res_y, S);
1795 ___
1796 {
1797 ######## ecp_nistz256_div_by_2(res_y, res_y); ##########################
1798 # operate in 4-5-6-7 "name space" that matches squaring output
1799 #
1800 my ($poly1,$poly3)=($a_ptr,$t1);
1801 my ($a0,$a1,$a2,$a3,$t3,$t4,$t1)=($acc4,$acc5,$acc6,$acc7,$acc0,$acc1,$acc2);
1802
1803 $code.=<<___;
1804 xor $t4, $t4
1805 mov $a0, $t0
1806 add \$-1, $a0
1807 mov $a1, $t1
1808 adc $poly1, $a1
1809 mov $a2, $t2
1810 adc \$0, $a2
1811 mov $a3, $t3
1812 adc $poly3, $a3
1813 adc \$0, $t4
1814 xor $a_ptr, $a_ptr # borrow $a_ptr
1815 test \$1, $t0
1816
1817 cmovz $t0, $a0
1818 cmovz $t1, $a1
1819 cmovz $t2, $a2
1820 cmovz $t3, $a3
1821 cmovz $a_ptr, $t4
1822
1823 mov $a1, $t0 # a0:a3>>1
1824 shr \$1, $a0
1825 shl \$63, $t0
1826 mov $a2, $t1
1827 shr \$1, $a1
1828 or $t0, $a0
1829 shl \$63, $t1
1830 mov $a3, $t2
1831 shr \$1, $a2
1832 or $t1, $a1
1833 shl \$63, $t2
1834 mov $a0, 8*0($r_ptr)
1835 shr \$1, $a3
1836 mov $a1, 8*1($r_ptr)
1837 shl \$63, $t4
1838 or $t2, $a2
1839 or $t4, $a3
1840 mov $a2, 8*2($r_ptr)
1841 mov $a3, 8*3($r_ptr)
1842 ___
1843 }
1844 $code.=<<___;
1845 `&load_for_mul("$M(%rsp)", "$Zsqr(%rsp)", "$src0")`
1846 lea $M(%rsp), $r_ptr
1847 call __ecp_nistz256_mul_mont$x # p256_mul_mont(M, M, Zsqr);
1848
1849 lea $tmp0(%rsp), $r_ptr
1850 call __ecp_nistz256_mul_by_2$x
1851
1852 lea $M(%rsp), $b_ptr
1853 lea $M(%rsp), $r_ptr
1854 call __ecp_nistz256_add_to$x # p256_mul_by_3(M, M);
1855
1856 `&load_for_mul("$S(%rsp)", "$in_x(%rsp)", "$src0")`
1857 lea $S(%rsp), $r_ptr
1858 call __ecp_nistz256_mul_mont$x # p256_mul_mont(S, S, in_x);
1859
1860 lea $tmp0(%rsp), $r_ptr
1861 call __ecp_nistz256_mul_by_2$x # p256_mul_by_2(tmp0, S);
1862
1863 `&load_for_sqr("$M(%rsp)", "$src0")`
1864 movq %xmm0, $r_ptr
1865 call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(res_x, M);
1866
1867 lea $tmp0(%rsp), $b_ptr
1868 mov $acc6, $acc0 # harmonize sqr output and sub input
1869 mov $acc7, $acc1
1870 mov $a_ptr, $poly1
1871 mov $t1, $poly3
1872 call __ecp_nistz256_sub_from$x # p256_sub(res_x, res_x, tmp0);
1873
1874 mov $S+8*0(%rsp), $t0
1875 mov $S+8*1(%rsp), $t1
1876 mov $S+8*2(%rsp), $t2
1877 mov $S+8*3(%rsp), $acc2 # "4-5-0-1" order
1878 lea $S(%rsp), $r_ptr
1879 call __ecp_nistz256_sub$x # p256_sub(S, S, res_x);
1880
1881 mov $M(%rsp), $src0
1882 lea $M(%rsp), $b_ptr
1883 mov $acc4, $acc6 # harmonize sub output and mul input
1884 xor %ecx, %ecx
1885 mov $acc4, $S+8*0(%rsp) # have to save:-(
1886 mov $acc5, $acc2
1887 mov $acc5, $S+8*1(%rsp)
1888 cmovz $acc0, $acc3
1889 mov $acc0, $S+8*2(%rsp)
1890 lea $S-$bias(%rsp), $a_ptr
1891 cmovz $acc1, $acc4
1892 mov $acc1, $S+8*3(%rsp)
1893 mov $acc6, $acc1
1894 lea $S(%rsp), $r_ptr
1895 call __ecp_nistz256_mul_mont$x # p256_mul_mont(S, S, M);
1896
1897 movq %xmm1, $b_ptr
1898 movq %xmm1, $r_ptr
1899 call __ecp_nistz256_sub_from$x # p256_sub(res_y, S, res_y);
1900
1901 add \$32*5+8, %rsp
1902 pop %r15
1903 pop %r14
1904 pop %r13
1905 pop %r12
1906 pop %rbx
1907 pop %rbp
1908 ret
1909 .size ecp_nistz256_point_double$sfx,.-ecp_nistz256_point_double$sfx
1910 ___
1911 }
1912 &gen_double("q");
1913
1914 sub gen_add () {
1915 my $x = shift;
1916 my ($src0,$sfx,$bias);
1917 my ($H,$Hsqr,$R,$Rsqr,$Hcub,
1918 $U1,$U2,$S1,$S2,
1919 $res_x,$res_y,$res_z,
1920 $in1_x,$in1_y,$in1_z,
1921 $in2_x,$in2_y,$in2_z)=map(32*$_,(0..17));
1922 my ($Z1sqr, $Z2sqr) = ($Hsqr, $Rsqr);
1923
1924 if ($x ne "x") {
1925 $src0 = "%rax";
1926 $sfx = "";
1927 $bias = 0;
1928
1929 $code.=<<___;
1930 .globl ecp_nistz256_point_add
1931 .type ecp_nistz256_point_add,\@function,3
1932 .align 32
1933 ecp_nistz256_point_add:
1934 ___
1935 $code.=<<___ if ($addx);
1936 mov \$0x80100, %ecx
1937 and OPENSSL_ia32cap_P+8(%rip), %ecx
1938 cmp \$0x80100, %ecx
1939 je .Lpoint_addx
1940 ___
1941 } else {
1942 $src0 = "%rdx";
1943 $sfx = "x";
1944 $bias = 128;
1945
1946 $code.=<<___;
1947 .type ecp_nistz256_point_addx,\@function,3
1948 .align 32
1949 ecp_nistz256_point_addx:
1950 .Lpoint_addx:
1951 ___
1952 }
1953 $code.=<<___;
1954 push %rbp
1955 push %rbx
1956 push %r12
1957 push %r13
1958 push %r14
1959 push %r15
1960 sub \$32*18+8, %rsp
1961
1962 movdqu 0x00($a_ptr), %xmm0 # copy *(P256_POINT *)$a_ptr
1963 movdqu 0x10($a_ptr), %xmm1
1964 movdqu 0x20($a_ptr), %xmm2
1965 movdqu 0x30($a_ptr), %xmm3
1966 movdqu 0x40($a_ptr), %xmm4
1967 movdqu 0x50($a_ptr), %xmm5
1968 mov $a_ptr, $b_ptr # reassign
1969 mov $b_org, $a_ptr # reassign
1970 movdqa %xmm0, $in1_x(%rsp)
1971 movdqa %xmm1, $in1_x+0x10(%rsp)
1972 por %xmm0, %xmm1
1973 movdqa %xmm2, $in1_y(%rsp)
1974 movdqa %xmm3, $in1_y+0x10(%rsp)
1975 por %xmm2, %xmm3
1976 movdqa %xmm4, $in1_z(%rsp)
1977 movdqa %xmm5, $in1_z+0x10(%rsp)
1978 por %xmm1, %xmm3
1979
1980 movdqu 0x00($a_ptr), %xmm0 # copy *(P256_POINT *)$b_ptr
1981 pshufd \$0xb1, %xmm3, %xmm5
1982 movdqu 0x10($a_ptr), %xmm1
1983 movdqu 0x20($a_ptr), %xmm2
1984 por %xmm3, %xmm5
1985 movdqu 0x30($a_ptr), %xmm3
1986 mov 0x40+8*0($a_ptr), $src0 # load original in2_z
1987 mov 0x40+8*1($a_ptr), $acc6
1988 mov 0x40+8*2($a_ptr), $acc7
1989 mov 0x40+8*3($a_ptr), $acc0
1990 movdqa %xmm0, $in2_x(%rsp)
1991 pshufd \$0x1e, %xmm5, %xmm4
1992 movdqa %xmm1, $in2_x+0x10(%rsp)
1993 por %xmm0, %xmm1
1994 movq $r_ptr, %xmm0 # save $r_ptr
1995 movdqa %xmm2, $in2_y(%rsp)
1996 movdqa %xmm3, $in2_y+0x10(%rsp)
1997 por %xmm2, %xmm3
1998 por %xmm4, %xmm5
1999 pxor %xmm4, %xmm4
2000 por %xmm1, %xmm3
2001
2002 lea 0x40-$bias($a_ptr), $a_ptr # $a_ptr is still valid
2003 mov $src0, $in2_z+8*0(%rsp) # make in2_z copy
2004 mov $acc6, $in2_z+8*1(%rsp)
2005 mov $acc7, $in2_z+8*2(%rsp)
2006 mov $acc0, $in2_z+8*3(%rsp)
2007 lea $Z2sqr(%rsp), $r_ptr # Z2^2
2008 call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Z2sqr, in2_z);
2009
2010 pcmpeqd %xmm4, %xmm5
2011 pshufd \$0xb1, %xmm3, %xmm4
2012 por %xmm3, %xmm4
2013 pshufd \$0, %xmm5, %xmm5 # in1infty
2014 pshufd \$0x1e, %xmm4, %xmm3
2015 por %xmm3, %xmm4
2016 pxor %xmm3, %xmm3
2017 pcmpeqd %xmm3, %xmm4
2018 pshufd \$0, %xmm4, %xmm4 # in2infty
2019 mov 0x40+8*0($b_ptr), $src0 # load original in1_z
2020 mov 0x40+8*1($b_ptr), $acc6
2021 mov 0x40+8*2($b_ptr), $acc7
2022 mov 0x40+8*3($b_ptr), $acc0
2023 movq $b_ptr, %xmm1
2024
2025 lea 0x40-$bias($b_ptr), $a_ptr
2026 lea $Z1sqr(%rsp), $r_ptr # Z1^2
2027 call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Z1sqr, in1_z);
2028
2029 `&load_for_mul("$Z2sqr(%rsp)", "$in2_z(%rsp)", "$src0")`
2030 lea $S1(%rsp), $r_ptr # S1 = Z2^3
2031 call __ecp_nistz256_mul_mont$x # p256_mul_mont(S1, Z2sqr, in2_z);
2032
2033 `&load_for_mul("$Z1sqr(%rsp)", "$in1_z(%rsp)", "$src0")`
2034 lea $S2(%rsp), $r_ptr # S2 = Z1^3
2035 call __ecp_nistz256_mul_mont$x # p256_mul_mont(S2, Z1sqr, in1_z);
2036
2037 `&load_for_mul("$S1(%rsp)", "$in1_y(%rsp)", "$src0")`
2038 lea $S1(%rsp), $r_ptr # S1 = Y1*Z2^3
2039 call __ecp_nistz256_mul_mont$x # p256_mul_mont(S1, S1, in1_y);
2040
2041 `&load_for_mul("$S2(%rsp)", "$in2_y(%rsp)", "$src0")`
2042 lea $S2(%rsp), $r_ptr # S2 = Y2*Z1^3
2043 call __ecp_nistz256_mul_mont$x # p256_mul_mont(S2, S2, in2_y);
2044
2045 lea $S1(%rsp), $b_ptr
2046 lea $R(%rsp), $r_ptr # R = S2 - S1
2047 call __ecp_nistz256_sub_from$x # p256_sub(R, S2, S1);
2048
2049 or $acc5, $acc4 # see if result is zero
2050 movdqa %xmm4, %xmm2
2051 or $acc0, $acc4
2052 or $acc1, $acc4
2053 por %xmm5, %xmm2 # in1infty || in2infty
2054 movq $acc4, %xmm3
2055
2056 `&load_for_mul("$Z2sqr(%rsp)", "$in1_x(%rsp)", "$src0")`
2057 lea $U1(%rsp), $r_ptr # U1 = X1*Z2^2
2058 call __ecp_nistz256_mul_mont$x # p256_mul_mont(U1, in1_x, Z2sqr);
2059
2060 `&load_for_mul("$Z1sqr(%rsp)", "$in2_x(%rsp)", "$src0")`
2061 lea $U2(%rsp), $r_ptr # U2 = X2*Z1^2
2062 call __ecp_nistz256_mul_mont$x # p256_mul_mont(U2, in2_x, Z1sqr);
2063
2064 lea $U1(%rsp), $b_ptr
2065 lea $H(%rsp), $r_ptr # H = U2 - U1
2066 call __ecp_nistz256_sub_from$x # p256_sub(H, U2, U1);
2067
2068 or $acc5, $acc4 # see if result is zero
2069 or $acc0, $acc4
2070 or $acc1, $acc4
2071
2072 .byte 0x3e # predict taken
2073 jnz .Ladd_proceed$x # is_equal(U1,U2)?
2074 movq %xmm2, $acc0
2075 movq %xmm3, $acc1
2076 test $acc0, $acc0
2077 jnz .Ladd_proceed$x # (in1infty || in2infty)?
2078 test $acc1, $acc1
2079 jz .Ladd_double$x # is_equal(S1,S2)?
2080
2081 movq %xmm0, $r_ptr # restore $r_ptr
2082 pxor %xmm0, %xmm0
2083 movdqu %xmm0, 0x00($r_ptr)
2084 movdqu %xmm0, 0x10($r_ptr)
2085 movdqu %xmm0, 0x20($r_ptr)
2086 movdqu %xmm0, 0x30($r_ptr)
2087 movdqu %xmm0, 0x40($r_ptr)
2088 movdqu %xmm0, 0x50($r_ptr)
2089 jmp .Ladd_done$x
2090
2091 .align 32
2092 .Ladd_double$x:
2093 movq %xmm1, $a_ptr # restore $a_ptr
2094 movq %xmm0, $r_ptr # restore $r_ptr
2095 add \$`32*(18-5)`, %rsp # difference in frame sizes
2096 jmp .Lpoint_double_shortcut$x
2097
2098 .align 32
2099 .Ladd_proceed$x:
2100 `&load_for_sqr("$R(%rsp)", "$src0")`
2101 lea $Rsqr(%rsp), $r_ptr # R^2
2102 call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Rsqr, R);
2103
2104 `&load_for_mul("$H(%rsp)", "$in1_z(%rsp)", "$src0")`
2105 lea $res_z(%rsp), $r_ptr # Z3 = H*Z1*Z2
2106 call __ecp_nistz256_mul_mont$x # p256_mul_mont(res_z, H, in1_z);
2107
2108 `&load_for_sqr("$H(%rsp)", "$src0")`
2109 lea $Hsqr(%rsp), $r_ptr # H^2
2110 call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Hsqr, H);
2111
2112 `&load_for_mul("$res_z(%rsp)", "$in2_z(%rsp)", "$src0")`
2113 lea $res_z(%rsp), $r_ptr # Z3 = H*Z1*Z2
2114 call __ecp_nistz256_mul_mont$x # p256_mul_mont(res_z, res_z, in2_z);
2115
2116 `&load_for_mul("$Hsqr(%rsp)", "$H(%rsp)", "$src0")`
2117 lea $Hcub(%rsp), $r_ptr # H^3
2118 call __ecp_nistz256_mul_mont$x # p256_mul_mont(Hcub, Hsqr, H);
2119
2120 `&load_for_mul("$Hsqr(%rsp)", "$U1(%rsp)", "$src0")`
2121 lea $U2(%rsp), $r_ptr # U1*H^2
2122 call __ecp_nistz256_mul_mont$x # p256_mul_mont(U2, U1, Hsqr);
2123 ___
2124 {
2125 #######################################################################
2126 # operate in 4-5-0-1 "name space" that matches multiplication output
2127 #
2128 my ($acc0,$acc1,$acc2,$acc3,$t3,$t4)=($acc4,$acc5,$acc0,$acc1,$acc2,$acc3);
2129 my ($poly1, $poly3)=($acc6,$acc7);
2130
2131 $code.=<<___;
2132 #lea $U2(%rsp), $a_ptr
2133 #lea $Hsqr(%rsp), $r_ptr # 2*U1*H^2
2134 #call __ecp_nistz256_mul_by_2 # ecp_nistz256_mul_by_2(Hsqr, U2);
2135
2136 add $acc0, $acc0 # a0:a3+a0:a3
2137 lea $Rsqr(%rsp), $a_ptr
2138 adc $acc1, $acc1
2139 mov $acc0, $t0
2140 adc $acc2, $acc2
2141 adc $acc3, $acc3
2142 mov $acc1, $t1
2143 sbb $t4, $t4
2144
2145 sub \$-1, $acc0
2146 mov $acc2, $t2
2147 sbb $poly1, $acc1
2148 sbb \$0, $acc2
2149 mov $acc3, $t3
2150 sbb $poly3, $acc3
2151 test $t4, $t4
2152
2153 cmovz $t0, $acc0
2154 mov 8*0($a_ptr), $t0
2155 cmovz $t1, $acc1
2156 mov 8*1($a_ptr), $t1
2157 cmovz $t2, $acc2
2158 mov 8*2($a_ptr), $t2
2159 cmovz $t3, $acc3
2160 mov 8*3($a_ptr), $t3
2161
2162 call __ecp_nistz256_sub$x # p256_sub(res_x, Rsqr, Hsqr);
2163
2164 lea $Hcub(%rsp), $b_ptr
2165 lea $res_x(%rsp), $r_ptr
2166 call __ecp_nistz256_sub_from$x # p256_sub(res_x, res_x, Hcub);
2167
2168 mov $U2+8*0(%rsp), $t0
2169 mov $U2+8*1(%rsp), $t1
2170 mov $U2+8*2(%rsp), $t2
2171 mov $U2+8*3(%rsp), $t3
2172 lea $res_y(%rsp), $r_ptr
2173
2174 call __ecp_nistz256_sub$x # p256_sub(res_y, U2, res_x);
2175
2176 mov $acc0, 8*0($r_ptr) # save the result, as
2177 mov $acc1, 8*1($r_ptr) # __ecp_nistz256_sub doesn't
2178 mov $acc2, 8*2($r_ptr)
2179 mov $acc3, 8*3($r_ptr)
2180 ___
2181 }
2182 $code.=<<___;
2183 `&load_for_mul("$S1(%rsp)", "$Hcub(%rsp)", "$src0")`
2184 lea $S2(%rsp), $r_ptr
2185 call __ecp_nistz256_mul_mont$x # p256_mul_mont(S2, S1, Hcub);
2186
2187 `&load_for_mul("$R(%rsp)", "$res_y(%rsp)", "$src0")`
2188 lea $res_y(%rsp), $r_ptr
2189 call __ecp_nistz256_mul_mont$x # p256_mul_mont(res_y, R, res_y);
2190
2191 lea $S2(%rsp), $b_ptr
2192 lea $res_y(%rsp), $r_ptr
2193 call __ecp_nistz256_sub_from$x # p256_sub(res_y, res_y, S2);
2194
2195 movq %xmm0, $r_ptr # restore $r_ptr
2196
2197 movdqa %xmm5, %xmm0 # copy_conditional(res_z, in2_z, in1infty);
2198 movdqa %xmm5, %xmm1
2199 pandn $res_z(%rsp), %xmm0
2200 movdqa %xmm5, %xmm2
2201 pandn $res_z+0x10(%rsp), %xmm1
2202 movdqa %xmm5, %xmm3
2203 pand $in2_z(%rsp), %xmm2
2204 pand $in2_z+0x10(%rsp), %xmm3
2205 por %xmm0, %xmm2
2206 por %xmm1, %xmm3
2207
2208 movdqa %xmm4, %xmm0 # copy_conditional(res_z, in1_z, in2infty);
2209 movdqa %xmm4, %xmm1
2210 pandn %xmm2, %xmm0
2211 movdqa %xmm4, %xmm2
2212 pandn %xmm3, %xmm1
2213 movdqa %xmm4, %xmm3
2214 pand $in1_z(%rsp), %xmm2
2215 pand $in1_z+0x10(%rsp), %xmm3
2216 por %xmm0, %xmm2
2217 por %xmm1, %xmm3
2218 movdqu %xmm2, 0x40($r_ptr)
2219 movdqu %xmm3, 0x50($r_ptr)
2220
2221 movdqa %xmm5, %xmm0 # copy_conditional(res_x, in2_x, in1infty);
2222 movdqa %xmm5, %xmm1
2223 pandn $res_x(%rsp), %xmm0
2224 movdqa %xmm5, %xmm2
2225 pandn $res_x+0x10(%rsp), %xmm1
2226 movdqa %xmm5, %xmm3
2227 pand $in2_x(%rsp), %xmm2
2228 pand $in2_x+0x10(%rsp), %xmm3
2229 por %xmm0, %xmm2
2230 por %xmm1, %xmm3
2231
2232 movdqa %xmm4, %xmm0 # copy_conditional(res_x, in1_x, in2infty);
2233 movdqa %xmm4, %xmm1
2234 pandn %xmm2, %xmm0
2235 movdqa %xmm4, %xmm2
2236 pandn %xmm3, %xmm1
2237 movdqa %xmm4, %xmm3
2238 pand $in1_x(%rsp), %xmm2
2239 pand $in1_x+0x10(%rsp), %xmm3
2240 por %xmm0, %xmm2
2241 por %xmm1, %xmm3
2242 movdqu %xmm2, 0x00($r_ptr)
2243 movdqu %xmm3, 0x10($r_ptr)
2244
2245 movdqa %xmm5, %xmm0 # copy_conditional(res_y, in2_y, in1infty);
2246 movdqa %xmm5, %xmm1
2247 pandn $res_y(%rsp), %xmm0
2248 movdqa %xmm5, %xmm2
2249 pandn $res_y+0x10(%rsp), %xmm1
2250 movdqa %xmm5, %xmm3
2251 pand $in2_y(%rsp), %xmm2
2252 pand $in2_y+0x10(%rsp), %xmm3
2253 por %xmm0, %xmm2
2254 por %xmm1, %xmm3
2255
2256 movdqa %xmm4, %xmm0 # copy_conditional(res_y, in1_y, in2infty);
2257 movdqa %xmm4, %xmm1
2258 pandn %xmm2, %xmm0
2259 movdqa %xmm4, %xmm2
2260 pandn %xmm3, %xmm1
2261 movdqa %xmm4, %xmm3
2262 pand $in1_y(%rsp), %xmm2
2263 pand $in1_y+0x10(%rsp), %xmm3
2264 por %xmm0, %xmm2
2265 por %xmm1, %xmm3
2266 movdqu %xmm2, 0x20($r_ptr)
2267 movdqu %xmm3, 0x30($r_ptr)
2268
2269 .Ladd_done$x:
2270 add \$32*18+8, %rsp
2271 pop %r15
2272 pop %r14
2273 pop %r13
2274 pop %r12
2275 pop %rbx
2276 pop %rbp
2277 ret
2278 .size ecp_nistz256_point_add$sfx,.-ecp_nistz256_point_add$sfx
2279 ___
2280 }
2281 &gen_add("q");
2282
2283 sub gen_add_affine () {
2284 my $x = shift;
2285 my ($src0,$sfx,$bias);
2286 my ($U2,$S2,$H,$R,$Hsqr,$Hcub,$Rsqr,
2287 $res_x,$res_y,$res_z,
2288 $in1_x,$in1_y,$in1_z,
2289 $in2_x,$in2_y)=map(32*$_,(0..14));
2290 my $Z1sqr = $S2;
2291
2292 if ($x ne "x") {
2293 $src0 = "%rax";
2294 $sfx = "";
2295 $bias = 0;
2296
2297 $code.=<<___;
2298 .globl ecp_nistz256_point_add_affine
2299 .type ecp_nistz256_point_add_affine,\@function,3
2300 .align 32
2301 ecp_nistz256_point_add_affine:
2302 ___
2303 $code.=<<___ if ($addx);
2304 mov \$0x80100, %ecx
2305 and OPENSSL_ia32cap_P+8(%rip), %ecx
2306 cmp \$0x80100, %ecx
2307 je .Lpoint_add_affinex
2308 ___
2309 } else {
2310 $src0 = "%rdx";
2311 $sfx = "x";
2312 $bias = 128;
2313
2314 $code.=<<___;
2315 .type ecp_nistz256_point_add_affinex,\@function,3
2316 .align 32
2317 ecp_nistz256_point_add_affinex:
2318 .Lpoint_add_affinex:
2319 ___
2320 }
2321 $code.=<<___;
2322 push %rbp
2323 push %rbx
2324 push %r12
2325 push %r13
2326 push %r14
2327 push %r15
2328 sub \$32*15+8, %rsp
2329
2330 movdqu 0x00($a_ptr), %xmm0 # copy *(P256_POINT *)$a_ptr
2331 mov $b_org, $b_ptr # reassign
2332 movdqu 0x10($a_ptr), %xmm1
2333 movdqu 0x20($a_ptr), %xmm2
2334 movdqu 0x30($a_ptr), %xmm3
2335 movdqu 0x40($a_ptr), %xmm4
2336 movdqu 0x50($a_ptr), %xmm5
2337 mov 0x40+8*0($a_ptr), $src0 # load original in1_z
2338 mov 0x40+8*1($a_ptr), $acc6
2339 mov 0x40+8*2($a_ptr), $acc7
2340 mov 0x40+8*3($a_ptr), $acc0
2341 movdqa %xmm0, $in1_x(%rsp)
2342 movdqa %xmm1, $in1_x+0x10(%rsp)
2343 por %xmm0, %xmm1
2344 movdqa %xmm2, $in1_y(%rsp)
2345 movdqa %xmm3, $in1_y+0x10(%rsp)
2346 por %xmm2, %xmm3
2347 movdqa %xmm4, $in1_z(%rsp)
2348 movdqa %xmm5, $in1_z+0x10(%rsp)
2349 por %xmm1, %xmm3
2350
2351 movdqu 0x00($b_ptr), %xmm0 # copy *(P256_POINT_AFFINE *)$b_ptr
2352 pshufd \$0xb1, %xmm3, %xmm5
2353 movdqu 0x10($b_ptr), %xmm1
2354 movdqu 0x20($b_ptr), %xmm2
2355 por %xmm3, %xmm5
2356 movdqu 0x30($b_ptr), %xmm3
2357 movdqa %xmm0, $in2_x(%rsp)
2358 pshufd \$0x1e, %xmm5, %xmm4
2359 movdqa %xmm1, $in2_x+0x10(%rsp)
2360 por %xmm0, %xmm1
2361 movq $r_ptr, %xmm0 # save $r_ptr
2362 movdqa %xmm2, $in2_y(%rsp)
2363 movdqa %xmm3, $in2_y+0x10(%rsp)
2364 por %xmm2, %xmm3
2365 por %xmm4, %xmm5
2366 pxor %xmm4, %xmm4
2367 por %xmm1, %xmm3
2368
2369 lea 0x40-$bias($a_ptr), $a_ptr # $a_ptr is still valid
2370 lea $Z1sqr(%rsp), $r_ptr # Z1^2
2371 call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Z1sqr, in1_z);
2372
2373 pcmpeqd %xmm4, %xmm5
2374 pshufd \$0xb1, %xmm3, %xmm4
2375 mov 0x00($b_ptr), $src0 # $b_ptr is still valid
2376 #lea 0x00($b_ptr), $b_ptr
2377 mov $acc4, $acc1 # harmonize sqr output and mul input
2378 por %xmm3, %xmm4
2379 pshufd \$0, %xmm5, %xmm5 # in1infty
2380 pshufd \$0x1e, %xmm4, %xmm3
2381 mov $acc5, $acc2
2382 por %xmm3, %xmm4
2383 pxor %xmm3, %xmm3
2384 mov $acc6, $acc3
2385 pcmpeqd %xmm3, %xmm4
2386 pshufd \$0, %xmm4, %xmm4 # in2infty
2387
2388 lea $Z1sqr-$bias(%rsp), $a_ptr
2389 mov $acc7, $acc4
2390 lea $U2(%rsp), $r_ptr # U2 = X2*Z1^2
2391 call __ecp_nistz256_mul_mont$x # p256_mul_mont(U2, Z1sqr, in2_x);
2392
2393 lea $in1_x(%rsp), $b_ptr
2394 lea $H(%rsp), $r_ptr # H = U2 - U1
2395 call __ecp_nistz256_sub_from$x # p256_sub(H, U2, in1_x);
2396
2397 `&load_for_mul("$Z1sqr(%rsp)", "$in1_z(%rsp)", "$src0")`
2398 lea $S2(%rsp), $r_ptr # S2 = Z1^3
2399 call __ecp_nistz256_mul_mont$x # p256_mul_mont(S2, Z1sqr, in1_z);
2400
2401 `&load_for_mul("$H(%rsp)", "$in1_z(%rsp)", "$src0")`
2402 lea $res_z(%rsp), $r_ptr # Z3 = H*Z1*Z2
2403 call __ecp_nistz256_mul_mont$x # p256_mul_mont(res_z, H, in1_z);
2404
2405 `&load_for_mul("$S2(%rsp)", "$in2_y(%rsp)", "$src0")`
2406 lea $S2(%rsp), $r_ptr # S2 = Y2*Z1^3
2407 call __ecp_nistz256_mul_mont$x # p256_mul_mont(S2, S2, in2_y);
2408
2409 lea $in1_y(%rsp), $b_ptr
2410 lea $R(%rsp), $r_ptr # R = S2 - S1
2411 call __ecp_nistz256_sub_from$x # p256_sub(R, S2, in1_y);
2412
2413 `&load_for_sqr("$H(%rsp)", "$src0")`
2414 lea $Hsqr(%rsp), $r_ptr # H^2
2415 call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Hsqr, H);
2416
2417 `&load_for_sqr("$R(%rsp)", "$src0")`
2418 lea $Rsqr(%rsp), $r_ptr # R^2
2419 call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Rsqr, R);
2420
2421 `&load_for_mul("$H(%rsp)", "$Hsqr(%rsp)", "$src0")`
2422 lea $Hcub(%rsp), $r_ptr # H^3
2423 call __ecp_nistz256_mul_mont$x # p256_mul_mont(Hcub, Hsqr, H);
2424
2425 `&load_for_mul("$Hsqr(%rsp)", "$in1_x(%rsp)", "$src0")`
2426 lea $U2(%rsp), $r_ptr # U1*H^2
2427 call __ecp_nistz256_mul_mont$x # p256_mul_mont(U2, in1_x, Hsqr);
2428 ___
2429 {
2430 #######################################################################
2431 # operate in 4-5-0-1 "name space" that matches multiplication output
2432 #
2433 my ($acc0,$acc1,$acc2,$acc3,$t3,$t4)=($acc4,$acc5,$acc0,$acc1,$acc2,$acc3);
2434 my ($poly1, $poly3)=($acc6,$acc7);
2435
2436 $code.=<<___;
2437 #lea $U2(%rsp), $a_ptr
2438 #lea $Hsqr(%rsp), $r_ptr # 2*U1*H^2
2439 #call __ecp_nistz256_mul_by_2 # ecp_nistz256_mul_by_2(Hsqr, U2);
2440
2441 add $acc0, $acc0 # a0:a3+a0:a3
2442 lea $Rsqr(%rsp), $a_ptr
2443 adc $acc1, $acc1
2444 mov $acc0, $t0
2445 adc $acc2, $acc2
2446 adc $acc3, $acc3
2447 mov $acc1, $t1
2448 sbb $t4, $t4
2449
2450 sub \$-1, $acc0
2451 mov $acc2, $t2
2452 sbb $poly1, $acc1
2453 sbb \$0, $acc2
2454 mov $acc3, $t3
2455 sbb $poly3, $acc3
2456 test $t4, $t4
2457
2458 cmovz $t0, $acc0
2459 mov 8*0($a_ptr), $t0
2460 cmovz $t1, $acc1
2461 mov 8*1($a_ptr), $t1
2462 cmovz $t2, $acc2
2463 mov 8*2($a_ptr), $t2
2464 cmovz $t3, $acc3
2465 mov 8*3($a_ptr), $t3
2466
2467 call __ecp_nistz256_sub$x # p256_sub(res_x, Rsqr, Hsqr);
2468
2469 lea $Hcub(%rsp), $b_ptr
2470 lea $res_x(%rsp), $r_ptr
2471 call __ecp_nistz256_sub_from$x # p256_sub(res_x, res_x, Hcub);
2472
2473 mov $U2+8*0(%rsp), $t0
2474 mov $U2+8*1(%rsp), $t1
2475 mov $U2+8*2(%rsp), $t2
2476 mov $U2+8*3(%rsp), $t3
2477 lea $H(%rsp), $r_ptr
2478
2479 call __ecp_nistz256_sub$x # p256_sub(H, U2, res_x);
2480
2481 mov $acc0, 8*0($r_ptr) # save the result, as
2482 mov $acc1, 8*1($r_ptr) # __ecp_nistz256_sub doesn't
2483 mov $acc2, 8*2($r_ptr)
2484 mov $acc3, 8*3($r_ptr)
2485 ___
2486 }
2487 $code.=<<___;
2488 `&load_for_mul("$Hcub(%rsp)", "$in1_y(%rsp)", "$src0")`
2489 lea $S2(%rsp), $r_ptr
2490 call __ecp_nistz256_mul_mont$x # p256_mul_mont(S2, Hcub, in1_y);
2491
2492 `&load_for_mul("$H(%rsp)", "$R(%rsp)", "$src0")`
2493 lea $H(%rsp), $r_ptr
2494 call __ecp_nistz256_mul_mont$x # p256_mul_mont(H, H, R);
2495
2496 lea $S2(%rsp), $b_ptr
2497 lea $res_y(%rsp), $r_ptr
2498 call __ecp_nistz256_sub_from$x # p256_sub(res_y, H, S2);
2499
2500 movq %xmm0, $r_ptr # restore $r_ptr
2501
2502 movdqa %xmm5, %xmm0 # copy_conditional(res_z, ONE, in1infty);
2503 movdqa %xmm5, %xmm1
2504 pandn $res_z(%rsp), %xmm0
2505 movdqa %xmm5, %xmm2
2506 pandn $res_z+0x10(%rsp), %xmm1
2507 movdqa %xmm5, %xmm3
2508 pand .LONE_mont(%rip), %xmm2
2509 pand .LONE_mont+0x10(%rip), %xmm3
2510 por %xmm0, %xmm2
2511 por %xmm1, %xmm3
2512
2513 movdqa %xmm4, %xmm0 # copy_conditional(res_z, in1_z, in2infty);
2514 movdqa %xmm4, %xmm1
2515 pandn %xmm2, %xmm0
2516 movdqa %xmm4, %xmm2
2517 pandn %xmm3, %xmm1
2518 movdqa %xmm4, %xmm3
2519 pand $in1_z(%rsp), %xmm2
2520 pand $in1_z+0x10(%rsp), %xmm3
2521 por %xmm0, %xmm2
2522 por %xmm1, %xmm3
2523 movdqu %xmm2, 0x40($r_ptr)
2524 movdqu %xmm3, 0x50($r_ptr)
2525
2526 movdqa %xmm5, %xmm0 # copy_conditional(res_x, in2_x, in1infty);
2527 movdqa %xmm5, %xmm1
2528 pandn $res_x(%rsp), %xmm0
2529 movdqa %xmm5, %xmm2
2530 pandn $res_x+0x10(%rsp), %xmm1
2531 movdqa %xmm5, %xmm3
2532 pand $in2_x(%rsp), %xmm2
2533 pand $in2_x+0x10(%rsp), %xmm3
2534 por %xmm0, %xmm2
2535 por %xmm1, %xmm3
2536
2537 movdqa %xmm4, %xmm0 # copy_conditional(res_x, in1_x, in2infty);
2538 movdqa %xmm4, %xmm1
2539 pandn %xmm2, %xmm0
2540 movdqa %xmm4, %xmm2
2541 pandn %xmm3, %xmm1
2542 movdqa %xmm4, %xmm3
2543 pand $in1_x(%rsp), %xmm2
2544 pand $in1_x+0x10(%rsp), %xmm3
2545 por %xmm0, %xmm2
2546 por %xmm1, %xmm3
2547 movdqu %xmm2, 0x00($r_ptr)
2548 movdqu %xmm3, 0x10($r_ptr)
2549
2550 movdqa %xmm5, %xmm0 # copy_conditional(res_y, in2_y, in1infty);
2551 movdqa %xmm5, %xmm1
2552 pandn $res_y(%rsp), %xmm0
2553 movdqa %xmm5, %xmm2
2554 pandn $res_y+0x10(%rsp), %xmm1
2555 movdqa %xmm5, %xmm3
2556 pand $in2_y(%rsp), %xmm2
2557 pand $in2_y+0x10(%rsp), %xmm3
2558 por %xmm0, %xmm2
2559 por %xmm1, %xmm3
2560
2561 movdqa %xmm4, %xmm0 # copy_conditional(res_y, in1_y, in2infty);
2562 movdqa %xmm4, %xmm1
2563 pandn %xmm2, %xmm0
2564 movdqa %xmm4, %xmm2
2565 pandn %xmm3, %xmm1
2566 movdqa %xmm4, %xmm3
2567 pand $in1_y(%rsp), %xmm2
2568 pand $in1_y+0x10(%rsp), %xmm3
2569 por %xmm0, %xmm2
2570 por %xmm1, %xmm3
2571 movdqu %xmm2, 0x20($r_ptr)
2572 movdqu %xmm3, 0x30($r_ptr)
2573
2574 add \$32*15+8, %rsp
2575 pop %r15
2576 pop %r14
2577 pop %r13
2578 pop %r12
2579 pop %rbx
2580 pop %rbp
2581 ret
2582 .size ecp_nistz256_point_add_affine$sfx,.-ecp_nistz256_point_add_affine$sfx
2583 ___
2584 }
2585 &gen_add_affine("q");
2586
2587 ########################################################################
2588 # AD*X magic
2589 #
2590 if ($addx) { {
2591 ########################################################################
2592 # operate in 4-5-0-1 "name space" that matches multiplication output
2593 #
2594 my ($a0,$a1,$a2,$a3,$t3,$t4)=($acc4,$acc5,$acc0,$acc1,$acc2,$acc3);
2595
2596 $code.=<<___;
2597 .type __ecp_nistz256_add_tox,\@abi-omnipotent
2598 .align 32
2599 __ecp_nistz256_add_tox:
2600 xor $t4, $t4
2601 adc 8*0($b_ptr), $a0
2602 adc 8*1($b_ptr), $a1
2603 mov $a0, $t0
2604 adc 8*2($b_ptr), $a2
2605 adc 8*3($b_ptr), $a3
2606 mov $a1, $t1
2607 adc \$0, $t4
2608
2609 xor $t3, $t3
2610 sbb \$-1, $a0
2611 mov $a2, $t2
2612 sbb $poly1, $a1
2613 sbb \$0, $a2
2614 mov $a3, $t3
2615 sbb $poly3, $a3
2616
2617 bt \$0, $t4
2618 cmovnc $t0, $a0
2619 cmovnc $t1, $a1
2620 mov $a0, 8*0($r_ptr)
2621 cmovnc $t2, $a2
2622 mov $a1, 8*1($r_ptr)
2623 cmovnc $t3, $a3
2624 mov $a2, 8*2($r_ptr)
2625 mov $a3, 8*3($r_ptr)
2626
2627 ret
2628 .size __ecp_nistz256_add_tox,.-__ecp_nistz256_add_tox
2629
2630 .type __ecp_nistz256_sub_fromx,\@abi-omnipotent
2631 .align 32
2632 __ecp_nistz256_sub_fromx:
2633 xor $t4, $t4
2634 sbb 8*0($b_ptr), $a0
2635 sbb 8*1($b_ptr), $a1
2636 mov $a0, $t0
2637 sbb 8*2($b_ptr), $a2
2638 sbb 8*3($b_ptr), $a3
2639 mov $a1, $t1
2640 sbb \$0, $t4
2641
2642 xor $t3, $t3
2643 adc \$-1, $a0
2644 mov $a2, $t2
2645 adc $poly1, $a1
2646 adc \$0, $a2
2647 mov $a3, $t3
2648 adc $poly3, $a3
2649
2650 bt \$0, $t4
2651 cmovnc $t0, $a0
2652 cmovnc $t1, $a1
2653 mov $a0, 8*0($r_ptr)
2654 cmovnc $t2, $a2
2655 mov $a1, 8*1($r_ptr)
2656 cmovnc $t3, $a3
2657 mov $a2, 8*2($r_ptr)
2658 mov $a3, 8*3($r_ptr)
2659
2660 ret
2661 .size __ecp_nistz256_sub_fromx,.-__ecp_nistz256_sub_fromx
2662
2663 .type __ecp_nistz256_subx,\@abi-omnipotent
2664 .align 32
2665 __ecp_nistz256_subx:
2666 xor $t4, $t4
2667 sbb $a0, $t0
2668 sbb $a1, $t1
2669 mov $t0, $a0
2670 sbb $a2, $t2
2671 sbb $a3, $t3
2672 mov $t1, $a1
2673 sbb \$0, $t4
2674
2675 xor $a3 ,$a3
2676 adc \$-1, $t0
2677 mov $t2, $a2
2678 adc $poly1, $t1
2679 adc \$0, $t2
2680 mov $t3, $a3
2681 adc $poly3, $t3
2682
2683 bt \$0, $t4
2684 cmovc $t0, $a0
2685 cmovc $t1, $a1
2686 cmovc $t2, $a2
2687 cmovc $t3, $a3
2688
2689 ret
2690 .size __ecp_nistz256_subx,.-__ecp_nistz256_subx
2691
2692 .type __ecp_nistz256_mul_by_2x,\@abi-omnipotent
2693 .align 32
2694 __ecp_nistz256_mul_by_2x:
2695 xor $t4, $t4
2696 adc $a0, $a0 # a0:a3+a0:a3
2697 adc $a1, $a1
2698 mov $a0, $t0
2699 adc $a2, $a2
2700 adc $a3, $a3
2701 mov $a1, $t1
2702 adc \$0, $t4
2703
2704 xor $t3, $t3
2705 sbb \$-1, $a0
2706 mov $a2, $t2
2707 sbb $poly1, $a1
2708 sbb \$0, $a2
2709 mov $a3, $t3
2710 sbb $poly3, $a3
2711
2712 bt \$0, $t4
2713 cmovnc $t0, $a0
2714 cmovnc $t1, $a1
2715 mov $a0, 8*0($r_ptr)
2716 cmovnc $t2, $a2
2717 mov $a1, 8*1($r_ptr)
2718 cmovnc $t3, $a3
2719 mov $a2, 8*2($r_ptr)
2720 mov $a3, 8*3($r_ptr)
2721
2722 ret
2723 .size __ecp_nistz256_mul_by_2x,.-__ecp_nistz256_mul_by_2x
2724 ___
2725 }
2726 &gen_double("x");
2727 &gen_add("x");
2728 &gen_add_affine("x");
2729 }
2730 }}}
2731
2732 $code =~ s/\`([^\`]*)\`/eval $1/gem;
2733 print $code;
2734 close STDOUT;