| 1 |
#!/usr/bin/env perl |
| 2 |
|
| 3 |
# ==================================================================== |
| 4 |
# Written by Andy Polyakov <[email protected]> for the OpenSSL |
| 5 |
# project. The module is, however, dual licensed under OpenSSL and |
| 6 |
# CRYPTOGAMS licenses depending on where you obtain it. For further |
| 7 |
# details see http://www.openssl.org/~appro/cryptogams/. |
| 8 |
# ==================================================================== |
| 9 |
|
| 10 |
# October 2005. |
| 11 |
# |
| 12 |
# Montgomery multiplication routine for x86_64. While it gives modest |
| 13 |
# 9% improvement of rsa4096 sign on Opteron, rsa512 sign runs more |
| 14 |
# than twice, >2x, as fast. Most common rsa1024 sign is improved by |
| 15 |
# respectful 50%. It remains to be seen if loop unrolling and |
| 16 |
# dedicated squaring routine can provide further improvement... |
| 17 |
|
| 18 |
# July 2011. |
| 19 |
# |
| 20 |
# Add dedicated squaring procedure. Performance improvement varies |
| 21 |
# from platform to platform, but in average it's ~5%/15%/25%/33% |
| 22 |
# for 512-/1024-/2048-/4096-bit RSA *sign* benchmarks respectively. |
| 23 |
|
| 24 |
# August 2011. |
| 25 |
# |
| 26 |
# Unroll and modulo-schedule inner loops in such manner that they |
| 27 |
# are "fallen through" for input lengths of 8, which is critical for |
| 28 |
# 1024-bit RSA *sign*. Average performance improvement in comparison |
| 29 |
# to *initial* version of this module from 2005 is ~0%/30%/40%/45% |
| 30 |
# for 512-/1024-/2048-/4096-bit RSA *sign* benchmarks respectively. |
| 31 |
|
| 32 |
# June 2013. |
| 33 |
# |
| 34 |
# Optimize reduction in squaring procedure and improve 1024+-bit RSA |
| 35 |
# sign performance by 10-16% on Intel Sandy Bridge and later |
| 36 |
# (virtually same on non-Intel processors). |
| 37 |
|
| 38 |
# August 2013. |
| 39 |
# |
| 40 |
# Add MULX/ADOX/ADCX code path. |
| 41 |
|
| 42 |
$flavour = shift; |
| 43 |
$output = shift; |
| 44 |
if ($flavour =~ /\./) { $output = $flavour; undef $flavour; } |
| 45 |
|
| 46 |
$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); |
| 47 |
|
| 48 |
$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; |
| 49 |
( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or |
| 50 |
( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or |
| 51 |
die "can't locate x86_64-xlate.pl"; |
| 52 |
|
| 53 |
open OUT,"| \"$^X\" $xlate $flavour $output"; |
| 54 |
*STDOUT=*OUT; |
| 55 |
|
| 56 |
# In upstream, this is controlled by shelling out to the compiler to check |
| 57 |
# versions, but BoringSSL is intended to be used with pre-generated perlasm |
| 58 |
# output, so this isn't useful anyway. |
| 59 |
# |
| 60 |
# TODO(davidben): Enable this option after testing. $addx goes up to 1. |
| 61 |
$addx = 0; |
| 62 |
|
| 63 |
# int bn_mul_mont( |
| 64 |
$rp="%rdi"; # BN_ULONG *rp, |
| 65 |
$ap="%rsi"; # const BN_ULONG *ap, |
| 66 |
$bp="%rdx"; # const BN_ULONG *bp, |
| 67 |
$np="%rcx"; # const BN_ULONG *np, |
| 68 |
$n0="%r8"; # const BN_ULONG *n0, |
| 69 |
$num="%r9"; # int num); |
| 70 |
$lo0="%r10"; |
| 71 |
$hi0="%r11"; |
| 72 |
$hi1="%r13"; |
| 73 |
$i="%r14"; |
| 74 |
$j="%r15"; |
| 75 |
$m0="%rbx"; |
| 76 |
$m1="%rbp"; |
| 77 |
|
| 78 |
$code=<<___; |
| 79 |
.text |
| 80 |
|
| 81 |
.extern OPENSSL_ia32cap_P |
| 82 |
|
| 83 |
.globl bn_mul_mont |
| 84 |
.type bn_mul_mont,\@function,6 |
| 85 |
.align 16 |
| 86 |
bn_mul_mont: |
| 87 |
test \$3,${num}d |
| 88 |
jnz .Lmul_enter |
| 89 |
cmp \$8,${num}d |
| 90 |
jb .Lmul_enter |
| 91 |
___ |
| 92 |
$code.=<<___ if ($addx); |
| 93 |
mov OPENSSL_ia32cap_P+8(%rip),%r11d |
| 94 |
___ |
| 95 |
$code.=<<___; |
| 96 |
cmp $ap,$bp |
| 97 |
jne .Lmul4x_enter |
| 98 |
test \$7,${num}d |
| 99 |
jz .Lsqr8x_enter |
| 100 |
jmp .Lmul4x_enter |
| 101 |
|
| 102 |
.align 16 |
| 103 |
.Lmul_enter: |
| 104 |
push %rbx |
| 105 |
push %rbp |
| 106 |
push %r12 |
| 107 |
push %r13 |
| 108 |
push %r14 |
| 109 |
push %r15 |
| 110 |
|
| 111 |
mov ${num}d,${num}d |
| 112 |
lea 2($num),%r10 |
| 113 |
mov %rsp,%r11 |
| 114 |
neg %r10 |
| 115 |
lea (%rsp,%r10,8),%rsp # tp=alloca(8*(num+2)) |
| 116 |
and \$-1024,%rsp # minimize TLB usage |
| 117 |
|
| 118 |
mov %r11,8(%rsp,$num,8) # tp[num+1]=%rsp |
| 119 |
.Lmul_body: |
| 120 |
mov $bp,%r12 # reassign $bp |
| 121 |
___ |
| 122 |
$bp="%r12"; |
| 123 |
$code.=<<___; |
| 124 |
mov ($n0),$n0 # pull n0[0] value |
| 125 |
mov ($bp),$m0 # m0=bp[0] |
| 126 |
mov ($ap),%rax |
| 127 |
|
| 128 |
xor $i,$i # i=0 |
| 129 |
xor $j,$j # j=0 |
| 130 |
|
| 131 |
mov $n0,$m1 |
| 132 |
mulq $m0 # ap[0]*bp[0] |
| 133 |
mov %rax,$lo0 |
| 134 |
mov ($np),%rax |
| 135 |
|
| 136 |
imulq $lo0,$m1 # "tp[0]"*n0 |
| 137 |
mov %rdx,$hi0 |
| 138 |
|
| 139 |
mulq $m1 # np[0]*m1 |
| 140 |
add %rax,$lo0 # discarded |
| 141 |
mov 8($ap),%rax |
| 142 |
adc \$0,%rdx |
| 143 |
mov %rdx,$hi1 |
| 144 |
|
| 145 |
lea 1($j),$j # j++ |
| 146 |
jmp .L1st_enter |
| 147 |
|
| 148 |
.align 16 |
| 149 |
.L1st: |
| 150 |
add %rax,$hi1 |
| 151 |
mov ($ap,$j,8),%rax |
| 152 |
adc \$0,%rdx |
| 153 |
add $hi0,$hi1 # np[j]*m1+ap[j]*bp[0] |
| 154 |
mov $lo0,$hi0 |
| 155 |
adc \$0,%rdx |
| 156 |
mov $hi1,-16(%rsp,$j,8) # tp[j-1] |
| 157 |
mov %rdx,$hi1 |
| 158 |
|
| 159 |
.L1st_enter: |
| 160 |
mulq $m0 # ap[j]*bp[0] |
| 161 |
add %rax,$hi0 |
| 162 |
mov ($np,$j,8),%rax |
| 163 |
adc \$0,%rdx |
| 164 |
lea 1($j),$j # j++ |
| 165 |
mov %rdx,$lo0 |
| 166 |
|
| 167 |
mulq $m1 # np[j]*m1 |
| 168 |
cmp $num,$j |
| 169 |
jne .L1st |
| 170 |
|
| 171 |
add %rax,$hi1 |
| 172 |
mov ($ap),%rax # ap[0] |
| 173 |
adc \$0,%rdx |
| 174 |
add $hi0,$hi1 # np[j]*m1+ap[j]*bp[0] |
| 175 |
adc \$0,%rdx |
| 176 |
mov $hi1,-16(%rsp,$j,8) # tp[j-1] |
| 177 |
mov %rdx,$hi1 |
| 178 |
mov $lo0,$hi0 |
| 179 |
|
| 180 |
xor %rdx,%rdx |
| 181 |
add $hi0,$hi1 |
| 182 |
adc \$0,%rdx |
| 183 |
mov $hi1,-8(%rsp,$num,8) |
| 184 |
mov %rdx,(%rsp,$num,8) # store upmost overflow bit |
| 185 |
|
| 186 |
lea 1($i),$i # i++ |
| 187 |
jmp .Louter |
| 188 |
.align 16 |
| 189 |
.Louter: |
| 190 |
mov ($bp,$i,8),$m0 # m0=bp[i] |
| 191 |
xor $j,$j # j=0 |
| 192 |
mov $n0,$m1 |
| 193 |
mov (%rsp),$lo0 |
| 194 |
mulq $m0 # ap[0]*bp[i] |
| 195 |
add %rax,$lo0 # ap[0]*bp[i]+tp[0] |
| 196 |
mov ($np),%rax |
| 197 |
adc \$0,%rdx |
| 198 |
|
| 199 |
imulq $lo0,$m1 # tp[0]*n0 |
| 200 |
mov %rdx,$hi0 |
| 201 |
|
| 202 |
mulq $m1 # np[0]*m1 |
| 203 |
add %rax,$lo0 # discarded |
| 204 |
mov 8($ap),%rax |
| 205 |
adc \$0,%rdx |
| 206 |
mov 8(%rsp),$lo0 # tp[1] |
| 207 |
mov %rdx,$hi1 |
| 208 |
|
| 209 |
lea 1($j),$j # j++ |
| 210 |
jmp .Linner_enter |
| 211 |
|
| 212 |
.align 16 |
| 213 |
.Linner: |
| 214 |
add %rax,$hi1 |
| 215 |
mov ($ap,$j,8),%rax |
| 216 |
adc \$0,%rdx |
| 217 |
add $lo0,$hi1 # np[j]*m1+ap[j]*bp[i]+tp[j] |
| 218 |
mov (%rsp,$j,8),$lo0 |
| 219 |
adc \$0,%rdx |
| 220 |
mov $hi1,-16(%rsp,$j,8) # tp[j-1] |
| 221 |
mov %rdx,$hi1 |
| 222 |
|
| 223 |
.Linner_enter: |
| 224 |
mulq $m0 # ap[j]*bp[i] |
| 225 |
add %rax,$hi0 |
| 226 |
mov ($np,$j,8),%rax |
| 227 |
adc \$0,%rdx |
| 228 |
add $hi0,$lo0 # ap[j]*bp[i]+tp[j] |
| 229 |
mov %rdx,$hi0 |
| 230 |
adc \$0,$hi0 |
| 231 |
lea 1($j),$j # j++ |
| 232 |
|
| 233 |
mulq $m1 # np[j]*m1 |
| 234 |
cmp $num,$j |
| 235 |
jne .Linner |
| 236 |
|
| 237 |
add %rax,$hi1 |
| 238 |
mov ($ap),%rax # ap[0] |
| 239 |
adc \$0,%rdx |
| 240 |
add $lo0,$hi1 # np[j]*m1+ap[j]*bp[i]+tp[j] |
| 241 |
mov (%rsp,$j,8),$lo0 |
| 242 |
adc \$0,%rdx |
| 243 |
mov $hi1,-16(%rsp,$j,8) # tp[j-1] |
| 244 |
mov %rdx,$hi1 |
| 245 |
|
| 246 |
xor %rdx,%rdx |
| 247 |
add $hi0,$hi1 |
| 248 |
adc \$0,%rdx |
| 249 |
add $lo0,$hi1 # pull upmost overflow bit |
| 250 |
adc \$0,%rdx |
| 251 |
mov $hi1,-8(%rsp,$num,8) |
| 252 |
mov %rdx,(%rsp,$num,8) # store upmost overflow bit |
| 253 |
|
| 254 |
lea 1($i),$i # i++ |
| 255 |
cmp $num,$i |
| 256 |
jb .Louter |
| 257 |
|
| 258 |
xor $i,$i # i=0 and clear CF! |
| 259 |
mov (%rsp),%rax # tp[0] |
| 260 |
lea (%rsp),$ap # borrow ap for tp |
| 261 |
mov $num,$j # j=num |
| 262 |
jmp .Lsub |
| 263 |
.align 16 |
| 264 |
.Lsub: sbb ($np,$i,8),%rax |
| 265 |
mov %rax,($rp,$i,8) # rp[i]=tp[i]-np[i] |
| 266 |
mov 8($ap,$i,8),%rax # tp[i+1] |
| 267 |
lea 1($i),$i # i++ |
| 268 |
dec $j # doesn't affect CF! |
| 269 |
jnz .Lsub |
| 270 |
|
| 271 |
sbb \$0,%rax # handle upmost overflow bit |
| 272 |
xor $i,$i |
| 273 |
mov $num,$j # j=num |
| 274 |
.align 16 |
| 275 |
.Lcopy: # copy or in-place refresh |
| 276 |
mov (%rsp,$i,8),$ap |
| 277 |
mov ($rp,$i,8),$np |
| 278 |
xor $np,$ap # conditional select: |
| 279 |
and %rax,$ap # ((ap ^ np) & %rax) ^ np |
| 280 |
xor $np,$ap # ap = borrow?tp:rp |
| 281 |
mov $i,(%rsp,$i,8) # zap temporary vector |
| 282 |
mov $ap,($rp,$i,8) # rp[i]=tp[i] |
| 283 |
lea 1($i),$i |
| 284 |
sub \$1,$j |
| 285 |
jnz .Lcopy |
| 286 |
|
| 287 |
mov 8(%rsp,$num,8),%rsi # restore %rsp |
| 288 |
mov \$1,%rax |
| 289 |
mov (%rsi),%r15 |
| 290 |
mov 8(%rsi),%r14 |
| 291 |
mov 16(%rsi),%r13 |
| 292 |
mov 24(%rsi),%r12 |
| 293 |
mov 32(%rsi),%rbp |
| 294 |
mov 40(%rsi),%rbx |
| 295 |
lea 48(%rsi),%rsp |
| 296 |
.Lmul_epilogue: |
| 297 |
ret |
| 298 |
.size bn_mul_mont,.-bn_mul_mont |
| 299 |
___ |
| 300 |
{{{ |
| 301 |
my @A=("%r10","%r11"); |
| 302 |
my @N=("%r13","%rdi"); |
| 303 |
$code.=<<___; |
| 304 |
.type bn_mul4x_mont,\@function,6 |
| 305 |
.align 16 |
| 306 |
bn_mul4x_mont: |
| 307 |
.Lmul4x_enter: |
| 308 |
___ |
| 309 |
$code.=<<___ if ($addx); |
| 310 |
and \$0x80100,%r11d |
| 311 |
cmp \$0x80100,%r11d |
| 312 |
je .Lmulx4x_enter |
| 313 |
___ |
| 314 |
$code.=<<___; |
| 315 |
push %rbx |
| 316 |
push %rbp |
| 317 |
push %r12 |
| 318 |
push %r13 |
| 319 |
push %r14 |
| 320 |
push %r15 |
| 321 |
|
| 322 |
mov ${num}d,${num}d |
| 323 |
lea 4($num),%r10 |
| 324 |
mov %rsp,%r11 |
| 325 |
neg %r10 |
| 326 |
lea (%rsp,%r10,8),%rsp # tp=alloca(8*(num+4)) |
| 327 |
and \$-1024,%rsp # minimize TLB usage |
| 328 |
|
| 329 |
mov %r11,8(%rsp,$num,8) # tp[num+1]=%rsp |
| 330 |
.Lmul4x_body: |
| 331 |
mov $rp,16(%rsp,$num,8) # tp[num+2]=$rp |
| 332 |
mov %rdx,%r12 # reassign $bp |
| 333 |
___ |
| 334 |
$bp="%r12"; |
| 335 |
$code.=<<___; |
| 336 |
mov ($n0),$n0 # pull n0[0] value |
| 337 |
mov ($bp),$m0 # m0=bp[0] |
| 338 |
mov ($ap),%rax |
| 339 |
|
| 340 |
xor $i,$i # i=0 |
| 341 |
xor $j,$j # j=0 |
| 342 |
|
| 343 |
mov $n0,$m1 |
| 344 |
mulq $m0 # ap[0]*bp[0] |
| 345 |
mov %rax,$A[0] |
| 346 |
mov ($np),%rax |
| 347 |
|
| 348 |
imulq $A[0],$m1 # "tp[0]"*n0 |
| 349 |
mov %rdx,$A[1] |
| 350 |
|
| 351 |
mulq $m1 # np[0]*m1 |
| 352 |
add %rax,$A[0] # discarded |
| 353 |
mov 8($ap),%rax |
| 354 |
adc \$0,%rdx |
| 355 |
mov %rdx,$N[1] |
| 356 |
|
| 357 |
mulq $m0 |
| 358 |
add %rax,$A[1] |
| 359 |
mov 8($np),%rax |
| 360 |
adc \$0,%rdx |
| 361 |
mov %rdx,$A[0] |
| 362 |
|
| 363 |
mulq $m1 |
| 364 |
add %rax,$N[1] |
| 365 |
mov 16($ap),%rax |
| 366 |
adc \$0,%rdx |
| 367 |
add $A[1],$N[1] |
| 368 |
lea 4($j),$j # j++ |
| 369 |
adc \$0,%rdx |
| 370 |
mov $N[1],(%rsp) |
| 371 |
mov %rdx,$N[0] |
| 372 |
jmp .L1st4x |
| 373 |
.align 16 |
| 374 |
.L1st4x: |
| 375 |
mulq $m0 # ap[j]*bp[0] |
| 376 |
add %rax,$A[0] |
| 377 |
mov -16($np,$j,8),%rax |
| 378 |
adc \$0,%rdx |
| 379 |
mov %rdx,$A[1] |
| 380 |
|
| 381 |
mulq $m1 # np[j]*m1 |
| 382 |
add %rax,$N[0] |
| 383 |
mov -8($ap,$j,8),%rax |
| 384 |
adc \$0,%rdx |
| 385 |
add $A[0],$N[0] # np[j]*m1+ap[j]*bp[0] |
| 386 |
adc \$0,%rdx |
| 387 |
mov $N[0],-24(%rsp,$j,8) # tp[j-1] |
| 388 |
mov %rdx,$N[1] |
| 389 |
|
| 390 |
mulq $m0 # ap[j]*bp[0] |
| 391 |
add %rax,$A[1] |
| 392 |
mov -8($np,$j,8),%rax |
| 393 |
adc \$0,%rdx |
| 394 |
mov %rdx,$A[0] |
| 395 |
|
| 396 |
mulq $m1 # np[j]*m1 |
| 397 |
add %rax,$N[1] |
| 398 |
mov ($ap,$j,8),%rax |
| 399 |
adc \$0,%rdx |
| 400 |
add $A[1],$N[1] # np[j]*m1+ap[j]*bp[0] |
| 401 |
adc \$0,%rdx |
| 402 |
mov $N[1],-16(%rsp,$j,8) # tp[j-1] |
| 403 |
mov %rdx,$N[0] |
| 404 |
|
| 405 |
mulq $m0 # ap[j]*bp[0] |
| 406 |
add %rax,$A[0] |
| 407 |
mov ($np,$j,8),%rax |
| 408 |
adc \$0,%rdx |
| 409 |
mov %rdx,$A[1] |
| 410 |
|
| 411 |
mulq $m1 # np[j]*m1 |
| 412 |
add %rax,$N[0] |
| 413 |
mov 8($ap,$j,8),%rax |
| 414 |
adc \$0,%rdx |
| 415 |
add $A[0],$N[0] # np[j]*m1+ap[j]*bp[0] |
| 416 |
adc \$0,%rdx |
| 417 |
mov $N[0],-8(%rsp,$j,8) # tp[j-1] |
| 418 |
mov %rdx,$N[1] |
| 419 |
|
| 420 |
mulq $m0 # ap[j]*bp[0] |
| 421 |
add %rax,$A[1] |
| 422 |
mov 8($np,$j,8),%rax |
| 423 |
adc \$0,%rdx |
| 424 |
lea 4($j),$j # j++ |
| 425 |
mov %rdx,$A[0] |
| 426 |
|
| 427 |
mulq $m1 # np[j]*m1 |
| 428 |
add %rax,$N[1] |
| 429 |
mov -16($ap,$j,8),%rax |
| 430 |
adc \$0,%rdx |
| 431 |
add $A[1],$N[1] # np[j]*m1+ap[j]*bp[0] |
| 432 |
adc \$0,%rdx |
| 433 |
mov $N[1],-32(%rsp,$j,8) # tp[j-1] |
| 434 |
mov %rdx,$N[0] |
| 435 |
cmp $num,$j |
| 436 |
jb .L1st4x |
| 437 |
|
| 438 |
mulq $m0 # ap[j]*bp[0] |
| 439 |
add %rax,$A[0] |
| 440 |
mov -16($np,$j,8),%rax |
| 441 |
adc \$0,%rdx |
| 442 |
mov %rdx,$A[1] |
| 443 |
|
| 444 |
mulq $m1 # np[j]*m1 |
| 445 |
add %rax,$N[0] |
| 446 |
mov -8($ap,$j,8),%rax |
| 447 |
adc \$0,%rdx |
| 448 |
add $A[0],$N[0] # np[j]*m1+ap[j]*bp[0] |
| 449 |
adc \$0,%rdx |
| 450 |
mov $N[0],-24(%rsp,$j,8) # tp[j-1] |
| 451 |
mov %rdx,$N[1] |
| 452 |
|
| 453 |
mulq $m0 # ap[j]*bp[0] |
| 454 |
add %rax,$A[1] |
| 455 |
mov -8($np,$j,8),%rax |
| 456 |
adc \$0,%rdx |
| 457 |
mov %rdx,$A[0] |
| 458 |
|
| 459 |
mulq $m1 # np[j]*m1 |
| 460 |
add %rax,$N[1] |
| 461 |
mov ($ap),%rax # ap[0] |
| 462 |
adc \$0,%rdx |
| 463 |
add $A[1],$N[1] # np[j]*m1+ap[j]*bp[0] |
| 464 |
adc \$0,%rdx |
| 465 |
mov $N[1],-16(%rsp,$j,8) # tp[j-1] |
| 466 |
mov %rdx,$N[0] |
| 467 |
|
| 468 |
xor $N[1],$N[1] |
| 469 |
add $A[0],$N[0] |
| 470 |
adc \$0,$N[1] |
| 471 |
mov $N[0],-8(%rsp,$j,8) |
| 472 |
mov $N[1],(%rsp,$j,8) # store upmost overflow bit |
| 473 |
|
| 474 |
lea 1($i),$i # i++ |
| 475 |
.align 4 |
| 476 |
.Louter4x: |
| 477 |
mov ($bp,$i,8),$m0 # m0=bp[i] |
| 478 |
xor $j,$j # j=0 |
| 479 |
mov (%rsp),$A[0] |
| 480 |
mov $n0,$m1 |
| 481 |
mulq $m0 # ap[0]*bp[i] |
| 482 |
add %rax,$A[0] # ap[0]*bp[i]+tp[0] |
| 483 |
mov ($np),%rax |
| 484 |
adc \$0,%rdx |
| 485 |
|
| 486 |
imulq $A[0],$m1 # tp[0]*n0 |
| 487 |
mov %rdx,$A[1] |
| 488 |
|
| 489 |
mulq $m1 # np[0]*m1 |
| 490 |
add %rax,$A[0] # "$N[0]", discarded |
| 491 |
mov 8($ap),%rax |
| 492 |
adc \$0,%rdx |
| 493 |
mov %rdx,$N[1] |
| 494 |
|
| 495 |
mulq $m0 # ap[j]*bp[i] |
| 496 |
add %rax,$A[1] |
| 497 |
mov 8($np),%rax |
| 498 |
adc \$0,%rdx |
| 499 |
add 8(%rsp),$A[1] # +tp[1] |
| 500 |
adc \$0,%rdx |
| 501 |
mov %rdx,$A[0] |
| 502 |
|
| 503 |
mulq $m1 # np[j]*m1 |
| 504 |
add %rax,$N[1] |
| 505 |
mov 16($ap),%rax |
| 506 |
adc \$0,%rdx |
| 507 |
add $A[1],$N[1] # np[j]*m1+ap[j]*bp[i]+tp[j] |
| 508 |
lea 4($j),$j # j+=2 |
| 509 |
adc \$0,%rdx |
| 510 |
mov $N[1],(%rsp) # tp[j-1] |
| 511 |
mov %rdx,$N[0] |
| 512 |
jmp .Linner4x |
| 513 |
.align 16 |
| 514 |
.Linner4x: |
| 515 |
mulq $m0 # ap[j]*bp[i] |
| 516 |
add %rax,$A[0] |
| 517 |
mov -16($np,$j,8),%rax |
| 518 |
adc \$0,%rdx |
| 519 |
add -16(%rsp,$j,8),$A[0] # ap[j]*bp[i]+tp[j] |
| 520 |
adc \$0,%rdx |
| 521 |
mov %rdx,$A[1] |
| 522 |
|
| 523 |
mulq $m1 # np[j]*m1 |
| 524 |
add %rax,$N[0] |
| 525 |
mov -8($ap,$j,8),%rax |
| 526 |
adc \$0,%rdx |
| 527 |
add $A[0],$N[0] |
| 528 |
adc \$0,%rdx |
| 529 |
mov $N[0],-24(%rsp,$j,8) # tp[j-1] |
| 530 |
mov %rdx,$N[1] |
| 531 |
|
| 532 |
mulq $m0 # ap[j]*bp[i] |
| 533 |
add %rax,$A[1] |
| 534 |
mov -8($np,$j,8),%rax |
| 535 |
adc \$0,%rdx |
| 536 |
add -8(%rsp,$j,8),$A[1] |
| 537 |
adc \$0,%rdx |
| 538 |
mov %rdx,$A[0] |
| 539 |
|
| 540 |
mulq $m1 # np[j]*m1 |
| 541 |
add %rax,$N[1] |
| 542 |
mov ($ap,$j,8),%rax |
| 543 |
adc \$0,%rdx |
| 544 |
add $A[1],$N[1] |
| 545 |
adc \$0,%rdx |
| 546 |
mov $N[1],-16(%rsp,$j,8) # tp[j-1] |
| 547 |
mov %rdx,$N[0] |
| 548 |
|
| 549 |
mulq $m0 # ap[j]*bp[i] |
| 550 |
add %rax,$A[0] |
| 551 |
mov ($np,$j,8),%rax |
| 552 |
adc \$0,%rdx |
| 553 |
add (%rsp,$j,8),$A[0] # ap[j]*bp[i]+tp[j] |
| 554 |
adc \$0,%rdx |
| 555 |
mov %rdx,$A[1] |
| 556 |
|
| 557 |
mulq $m1 # np[j]*m1 |
| 558 |
add %rax,$N[0] |
| 559 |
mov 8($ap,$j,8),%rax |
| 560 |
adc \$0,%rdx |
| 561 |
add $A[0],$N[0] |
| 562 |
adc \$0,%rdx |
| 563 |
mov $N[0],-8(%rsp,$j,8) # tp[j-1] |
| 564 |
mov %rdx,$N[1] |
| 565 |
|
| 566 |
mulq $m0 # ap[j]*bp[i] |
| 567 |
add %rax,$A[1] |
| 568 |
mov 8($np,$j,8),%rax |
| 569 |
adc \$0,%rdx |
| 570 |
add 8(%rsp,$j,8),$A[1] |
| 571 |
adc \$0,%rdx |
| 572 |
lea 4($j),$j # j++ |
| 573 |
mov %rdx,$A[0] |
| 574 |
|
| 575 |
mulq $m1 # np[j]*m1 |
| 576 |
add %rax,$N[1] |
| 577 |
mov -16($ap,$j,8),%rax |
| 578 |
adc \$0,%rdx |
| 579 |
add $A[1],$N[1] |
| 580 |
adc \$0,%rdx |
| 581 |
mov $N[1],-32(%rsp,$j,8) # tp[j-1] |
| 582 |
mov %rdx,$N[0] |
| 583 |
cmp $num,$j |
| 584 |
jb .Linner4x |
| 585 |
|
| 586 |
mulq $m0 # ap[j]*bp[i] |
| 587 |
add %rax,$A[0] |
| 588 |
mov -16($np,$j,8),%rax |
| 589 |
adc \$0,%rdx |
| 590 |
add -16(%rsp,$j,8),$A[0] # ap[j]*bp[i]+tp[j] |
| 591 |
adc \$0,%rdx |
| 592 |
mov %rdx,$A[1] |
| 593 |
|
| 594 |
mulq $m1 # np[j]*m1 |
| 595 |
add %rax,$N[0] |
| 596 |
mov -8($ap,$j,8),%rax |
| 597 |
adc \$0,%rdx |
| 598 |
add $A[0],$N[0] |
| 599 |
adc \$0,%rdx |
| 600 |
mov $N[0],-24(%rsp,$j,8) # tp[j-1] |
| 601 |
mov %rdx,$N[1] |
| 602 |
|
| 603 |
mulq $m0 # ap[j]*bp[i] |
| 604 |
add %rax,$A[1] |
| 605 |
mov -8($np,$j,8),%rax |
| 606 |
adc \$0,%rdx |
| 607 |
add -8(%rsp,$j,8),$A[1] |
| 608 |
adc \$0,%rdx |
| 609 |
lea 1($i),$i # i++ |
| 610 |
mov %rdx,$A[0] |
| 611 |
|
| 612 |
mulq $m1 # np[j]*m1 |
| 613 |
add %rax,$N[1] |
| 614 |
mov ($ap),%rax # ap[0] |
| 615 |
adc \$0,%rdx |
| 616 |
add $A[1],$N[1] |
| 617 |
adc \$0,%rdx |
| 618 |
mov $N[1],-16(%rsp,$j,8) # tp[j-1] |
| 619 |
mov %rdx,$N[0] |
| 620 |
|
| 621 |
xor $N[1],$N[1] |
| 622 |
add $A[0],$N[0] |
| 623 |
adc \$0,$N[1] |
| 624 |
add (%rsp,$num,8),$N[0] # pull upmost overflow bit |
| 625 |
adc \$0,$N[1] |
| 626 |
mov $N[0],-8(%rsp,$j,8) |
| 627 |
mov $N[1],(%rsp,$j,8) # store upmost overflow bit |
| 628 |
|
| 629 |
cmp $num,$i |
| 630 |
jb .Louter4x |
| 631 |
___ |
| 632 |
{ |
| 633 |
my @ri=("%rax","%rdx",$m0,$m1); |
| 634 |
$code.=<<___; |
| 635 |
mov 16(%rsp,$num,8),$rp # restore $rp |
| 636 |
mov 0(%rsp),@ri[0] # tp[0] |
| 637 |
mov 8(%rsp),@ri[1] # tp[1] |
| 638 |
shr \$2,$num # num/=4 |
| 639 |
lea (%rsp),$ap # borrow ap for tp |
| 640 |
xor $i,$i # i=0 and clear CF! |
| 641 |
|
| 642 |
sub 0($np),@ri[0] |
| 643 |
mov 16($ap),@ri[2] # tp[2] |
| 644 |
mov 24($ap),@ri[3] # tp[3] |
| 645 |
sbb 8($np),@ri[1] |
| 646 |
lea -1($num),$j # j=num/4-1 |
| 647 |
jmp .Lsub4x |
| 648 |
.align 16 |
| 649 |
.Lsub4x: |
| 650 |
mov @ri[0],0($rp,$i,8) # rp[i]=tp[i]-np[i] |
| 651 |
mov @ri[1],8($rp,$i,8) # rp[i]=tp[i]-np[i] |
| 652 |
sbb 16($np,$i,8),@ri[2] |
| 653 |
mov 32($ap,$i,8),@ri[0] # tp[i+1] |
| 654 |
mov 40($ap,$i,8),@ri[1] |
| 655 |
sbb 24($np,$i,8),@ri[3] |
| 656 |
mov @ri[2],16($rp,$i,8) # rp[i]=tp[i]-np[i] |
| 657 |
mov @ri[3],24($rp,$i,8) # rp[i]=tp[i]-np[i] |
| 658 |
sbb 32($np,$i,8),@ri[0] |
| 659 |
mov 48($ap,$i,8),@ri[2] |
| 660 |
mov 56($ap,$i,8),@ri[3] |
| 661 |
sbb 40($np,$i,8),@ri[1] |
| 662 |
lea 4($i),$i # i++ |
| 663 |
dec $j # doesnn't affect CF! |
| 664 |
jnz .Lsub4x |
| 665 |
|
| 666 |
mov @ri[0],0($rp,$i,8) # rp[i]=tp[i]-np[i] |
| 667 |
mov 32($ap,$i,8),@ri[0] # load overflow bit |
| 668 |
sbb 16($np,$i,8),@ri[2] |
| 669 |
mov @ri[1],8($rp,$i,8) # rp[i]=tp[i]-np[i] |
| 670 |
sbb 24($np,$i,8),@ri[3] |
| 671 |
mov @ri[2],16($rp,$i,8) # rp[i]=tp[i]-np[i] |
| 672 |
|
| 673 |
sbb \$0,@ri[0] # handle upmost overflow bit |
| 674 |
mov @ri[0],%xmm0 |
| 675 |
punpcklqdq %xmm0,%xmm0 # extend mask to 128 bits |
| 676 |
mov @ri[3],24($rp,$i,8) # rp[i]=tp[i]-np[i] |
| 677 |
xor $i,$i # i=0 |
| 678 |
|
| 679 |
mov $num,$j |
| 680 |
pxor %xmm5,%xmm5 |
| 681 |
jmp .Lcopy4x |
| 682 |
.align 16 |
| 683 |
.Lcopy4x: # copy or in-place refresh |
| 684 |
movdqu (%rsp,$i),%xmm2 |
| 685 |
movdqu 16(%rsp,$i),%xmm4 |
| 686 |
movdqu ($rp,$i),%xmm1 |
| 687 |
movdqu 16($rp,$i),%xmm3 |
| 688 |
pxor %xmm1,%xmm2 # conditional select |
| 689 |
pxor %xmm3,%xmm4 |
| 690 |
pand %xmm0,%xmm2 |
| 691 |
pand %xmm0,%xmm4 |
| 692 |
pxor %xmm1,%xmm2 |
| 693 |
pxor %xmm3,%xmm4 |
| 694 |
movdqu %xmm2,($rp,$i) |
| 695 |
movdqu %xmm4,16($rp,$i) |
| 696 |
movdqa %xmm5,(%rsp,$i) # zap temporary vectors |
| 697 |
movdqa %xmm5,16(%rsp,$i) |
| 698 |
|
| 699 |
lea 32($i),$i |
| 700 |
dec $j |
| 701 |
jnz .Lcopy4x |
| 702 |
|
| 703 |
shl \$2,$num |
| 704 |
___ |
| 705 |
} |
| 706 |
$code.=<<___; |
| 707 |
mov 8(%rsp,$num,8),%rsi # restore %rsp |
| 708 |
mov \$1,%rax |
| 709 |
mov (%rsi),%r15 |
| 710 |
mov 8(%rsi),%r14 |
| 711 |
mov 16(%rsi),%r13 |
| 712 |
mov 24(%rsi),%r12 |
| 713 |
mov 32(%rsi),%rbp |
| 714 |
mov 40(%rsi),%rbx |
| 715 |
lea 48(%rsi),%rsp |
| 716 |
.Lmul4x_epilogue: |
| 717 |
ret |
| 718 |
.size bn_mul4x_mont,.-bn_mul4x_mont |
| 719 |
___ |
| 720 |
}}} |
| 721 |
{{{ |
| 722 |
###################################################################### |
| 723 |
# void bn_sqr8x_mont( |
| 724 |
my $rptr="%rdi"; # const BN_ULONG *rptr, |
| 725 |
my $aptr="%rsi"; # const BN_ULONG *aptr, |
| 726 |
my $bptr="%rdx"; # not used |
| 727 |
my $nptr="%rcx"; # const BN_ULONG *nptr, |
| 728 |
my $n0 ="%r8"; # const BN_ULONG *n0); |
| 729 |
my $num ="%r9"; # int num, has to be divisible by 8 |
| 730 |
|
| 731 |
my ($i,$j,$tptr)=("%rbp","%rcx",$rptr); |
| 732 |
my @A0=("%r10","%r11"); |
| 733 |
my @A1=("%r12","%r13"); |
| 734 |
my ($a0,$a1,$ai)=("%r14","%r15","%rbx"); |
| 735 |
|
| 736 |
$code.=<<___ if ($addx); |
| 737 |
.extern bn_sqrx8x_internal # see x86_64-mont5 module |
| 738 |
___ |
| 739 |
$code.=<<___; |
| 740 |
.extern bn_sqr8x_internal # see x86_64-mont5 module |
| 741 |
|
| 742 |
.type bn_sqr8x_mont,\@function,6 |
| 743 |
.align 32 |
| 744 |
bn_sqr8x_mont: |
| 745 |
.Lsqr8x_enter: |
| 746 |
mov %rsp,%rax |
| 747 |
push %rbx |
| 748 |
push %rbp |
| 749 |
push %r12 |
| 750 |
push %r13 |
| 751 |
push %r14 |
| 752 |
push %r15 |
| 753 |
|
| 754 |
mov ${num}d,%r10d |
| 755 |
shl \$3,${num}d # convert $num to bytes |
| 756 |
shl \$3+2,%r10 # 4*$num |
| 757 |
neg $num |
| 758 |
|
| 759 |
############################################################## |
| 760 |
# ensure that stack frame doesn't alias with $aptr modulo |
| 761 |
# 4096. this is done to allow memory disambiguation logic |
| 762 |
# do its job. |
| 763 |
# |
| 764 |
lea -64(%rsp,$num,2),%r11 |
| 765 |
mov ($n0),$n0 # *n0 |
| 766 |
sub $aptr,%r11 |
| 767 |
and \$4095,%r11 |
| 768 |
cmp %r11,%r10 |
| 769 |
jb .Lsqr8x_sp_alt |
| 770 |
sub %r11,%rsp # align with $aptr |
| 771 |
lea -64(%rsp,$num,2),%rsp # alloca(frame+2*$num) |
| 772 |
jmp .Lsqr8x_sp_done |
| 773 |
|
| 774 |
.align 32 |
| 775 |
.Lsqr8x_sp_alt: |
| 776 |
lea 4096-64(,$num,2),%r10 # 4096-frame-2*$num |
| 777 |
lea -64(%rsp,$num,2),%rsp # alloca(frame+2*$num) |
| 778 |
sub %r10,%r11 |
| 779 |
mov \$0,%r10 |
| 780 |
cmovc %r10,%r11 |
| 781 |
sub %r11,%rsp |
| 782 |
.Lsqr8x_sp_done: |
| 783 |
and \$-64,%rsp |
| 784 |
mov $num,%r10 |
| 785 |
neg $num |
| 786 |
|
| 787 |
mov $n0, 32(%rsp) |
| 788 |
mov %rax, 40(%rsp) # save original %rsp |
| 789 |
.Lsqr8x_body: |
| 790 |
|
| 791 |
movq $nptr, %xmm2 # save pointer to modulus |
| 792 |
pxor %xmm0,%xmm0 |
| 793 |
movq $rptr,%xmm1 # save $rptr |
| 794 |
movq %r10, %xmm3 # -$num |
| 795 |
___ |
| 796 |
$code.=<<___ if ($addx); |
| 797 |
mov OPENSSL_ia32cap_P+8(%rip),%eax |
| 798 |
and \$0x80100,%eax |
| 799 |
cmp \$0x80100,%eax |
| 800 |
jne .Lsqr8x_nox |
| 801 |
|
| 802 |
call bn_sqrx8x_internal # see x86_64-mont5 module |
| 803 |
# %rax top-most carry |
| 804 |
# %rbp nptr |
| 805 |
# %rcx -8*num |
| 806 |
# %r8 end of tp[2*num] |
| 807 |
lea (%r8,%rcx),%rbx |
| 808 |
mov %rcx,$num |
| 809 |
mov %rcx,%rdx |
| 810 |
movq %xmm1,$rptr |
| 811 |
sar \$3+2,%rcx # %cf=0 |
| 812 |
jmp .Lsqr8x_sub |
| 813 |
|
| 814 |
.align 32 |
| 815 |
.Lsqr8x_nox: |
| 816 |
___ |
| 817 |
$code.=<<___; |
| 818 |
call bn_sqr8x_internal # see x86_64-mont5 module |
| 819 |
# %rax top-most carry |
| 820 |
# %rbp nptr |
| 821 |
# %r8 -8*num |
| 822 |
# %rdi end of tp[2*num] |
| 823 |
lea (%rdi,$num),%rbx |
| 824 |
mov $num,%rcx |
| 825 |
mov $num,%rdx |
| 826 |
movq %xmm1,$rptr |
| 827 |
sar \$3+2,%rcx # %cf=0 |
| 828 |
jmp .Lsqr8x_sub |
| 829 |
|
| 830 |
.align 32 |
| 831 |
.Lsqr8x_sub: |
| 832 |
mov 8*0(%rbx),%r12 |
| 833 |
mov 8*1(%rbx),%r13 |
| 834 |
mov 8*2(%rbx),%r14 |
| 835 |
mov 8*3(%rbx),%r15 |
| 836 |
lea 8*4(%rbx),%rbx |
| 837 |
sbb 8*0(%rbp),%r12 |
| 838 |
sbb 8*1(%rbp),%r13 |
| 839 |
sbb 8*2(%rbp),%r14 |
| 840 |
sbb 8*3(%rbp),%r15 |
| 841 |
lea 8*4(%rbp),%rbp |
| 842 |
mov %r12,8*0($rptr) |
| 843 |
mov %r13,8*1($rptr) |
| 844 |
mov %r14,8*2($rptr) |
| 845 |
mov %r15,8*3($rptr) |
| 846 |
lea 8*4($rptr),$rptr |
| 847 |
inc %rcx # preserves %cf |
| 848 |
jnz .Lsqr8x_sub |
| 849 |
|
| 850 |
sbb \$0,%rax # top-most carry |
| 851 |
lea (%rbx,$num),%rbx # rewind |
| 852 |
lea ($rptr,$num),$rptr # rewind |
| 853 |
|
| 854 |
movq %rax,%xmm1 |
| 855 |
pxor %xmm0,%xmm0 |
| 856 |
pshufd \$0,%xmm1,%xmm1 |
| 857 |
mov 40(%rsp),%rsi # restore %rsp |
| 858 |
jmp .Lsqr8x_cond_copy |
| 859 |
|
| 860 |
.align 32 |
| 861 |
.Lsqr8x_cond_copy: |
| 862 |
movdqa 16*0(%rbx),%xmm2 |
| 863 |
movdqa 16*1(%rbx),%xmm3 |
| 864 |
lea 16*2(%rbx),%rbx |
| 865 |
movdqu 16*0($rptr),%xmm4 |
| 866 |
movdqu 16*1($rptr),%xmm5 |
| 867 |
lea 16*2($rptr),$rptr |
| 868 |
movdqa %xmm0,-16*2(%rbx) # zero tp |
| 869 |
movdqa %xmm0,-16*1(%rbx) |
| 870 |
movdqa %xmm0,-16*2(%rbx,%rdx) |
| 871 |
movdqa %xmm0,-16*1(%rbx,%rdx) |
| 872 |
pcmpeqd %xmm1,%xmm0 |
| 873 |
pand %xmm1,%xmm2 |
| 874 |
pand %xmm1,%xmm3 |
| 875 |
pand %xmm0,%xmm4 |
| 876 |
pand %xmm0,%xmm5 |
| 877 |
pxor %xmm0,%xmm0 |
| 878 |
por %xmm2,%xmm4 |
| 879 |
por %xmm3,%xmm5 |
| 880 |
movdqu %xmm4,-16*2($rptr) |
| 881 |
movdqu %xmm5,-16*1($rptr) |
| 882 |
add \$32,$num |
| 883 |
jnz .Lsqr8x_cond_copy |
| 884 |
|
| 885 |
mov \$1,%rax |
| 886 |
mov -48(%rsi),%r15 |
| 887 |
mov -40(%rsi),%r14 |
| 888 |
mov -32(%rsi),%r13 |
| 889 |
mov -24(%rsi),%r12 |
| 890 |
mov -16(%rsi),%rbp |
| 891 |
mov -8(%rsi),%rbx |
| 892 |
lea (%rsi),%rsp |
| 893 |
.Lsqr8x_epilogue: |
| 894 |
ret |
| 895 |
.size bn_sqr8x_mont,.-bn_sqr8x_mont |
| 896 |
___ |
| 897 |
}}} |
| 898 |
|
| 899 |
if ($addx) {{{ |
| 900 |
my $bp="%rdx"; # original value |
| 901 |
|
| 902 |
$code.=<<___; |
| 903 |
.type bn_mulx4x_mont,\@function,6 |
| 904 |
.align 32 |
| 905 |
bn_mulx4x_mont: |
| 906 |
.Lmulx4x_enter: |
| 907 |
mov %rsp,%rax |
| 908 |
push %rbx |
| 909 |
push %rbp |
| 910 |
push %r12 |
| 911 |
push %r13 |
| 912 |
push %r14 |
| 913 |
push %r15 |
| 914 |
|
| 915 |
shl \$3,${num}d # convert $num to bytes |
| 916 |
.byte 0x67 |
| 917 |
xor %r10,%r10 |
| 918 |
sub $num,%r10 # -$num |
| 919 |
mov ($n0),$n0 # *n0 |
| 920 |
lea -72(%rsp,%r10),%rsp # alloca(frame+$num+8) |
| 921 |
lea ($bp,$num),%r10 |
| 922 |
and \$-128,%rsp |
| 923 |
############################################################## |
| 924 |
# Stack layout |
| 925 |
# +0 num |
| 926 |
# +8 off-loaded &b[i] |
| 927 |
# +16 end of b[num] |
| 928 |
# +24 saved n0 |
| 929 |
# +32 saved rp |
| 930 |
# +40 saved %rsp |
| 931 |
# +48 inner counter |
| 932 |
# +56 |
| 933 |
# +64 tmp[num+1] |
| 934 |
# |
| 935 |
mov $num,0(%rsp) # save $num |
| 936 |
shr \$5,$num |
| 937 |
mov %r10,16(%rsp) # end of b[num] |
| 938 |
sub \$1,$num |
| 939 |
mov $n0, 24(%rsp) # save *n0 |
| 940 |
mov $rp, 32(%rsp) # save $rp |
| 941 |
mov %rax,40(%rsp) # save original %rsp |
| 942 |
mov $num,48(%rsp) # inner counter |
| 943 |
jmp .Lmulx4x_body |
| 944 |
|
| 945 |
.align 32 |
| 946 |
.Lmulx4x_body: |
| 947 |
___ |
| 948 |
my ($aptr, $bptr, $nptr, $tptr, $mi, $bi, $zero, $num)= |
| 949 |
("%rsi","%rdi","%rcx","%rbx","%r8","%r9","%rbp","%rax"); |
| 950 |
my $rptr=$bptr; |
| 951 |
$code.=<<___; |
| 952 |
lea 8($bp),$bptr |
| 953 |
mov ($bp),%rdx # b[0], $bp==%rdx actually |
| 954 |
lea 64+32(%rsp),$tptr |
| 955 |
mov %rdx,$bi |
| 956 |
|
| 957 |
mulx 0*8($aptr),$mi,%rax # a[0]*b[0] |
| 958 |
mulx 1*8($aptr),%r11,%r14 # a[1]*b[0] |
| 959 |
add %rax,%r11 |
| 960 |
mov $bptr,8(%rsp) # off-load &b[i] |
| 961 |
mulx 2*8($aptr),%r12,%r13 # ... |
| 962 |
adc %r14,%r12 |
| 963 |
adc \$0,%r13 |
| 964 |
|
| 965 |
mov $mi,$bptr # borrow $bptr |
| 966 |
imulq 24(%rsp),$mi # "t[0]"*n0 |
| 967 |
xor $zero,$zero # cf=0, of=0 |
| 968 |
|
| 969 |
mulx 3*8($aptr),%rax,%r14 |
| 970 |
mov $mi,%rdx |
| 971 |
lea 4*8($aptr),$aptr |
| 972 |
adcx %rax,%r13 |
| 973 |
adcx $zero,%r14 # cf=0 |
| 974 |
|
| 975 |
mulx 0*8($nptr),%rax,%r10 |
| 976 |
adcx %rax,$bptr # discarded |
| 977 |
adox %r11,%r10 |
| 978 |
mulx 1*8($nptr),%rax,%r11 |
| 979 |
adcx %rax,%r10 |
| 980 |
adox %r12,%r11 |
| 981 |
.byte 0xc4,0x62,0xfb,0xf6,0xa1,0x10,0x00,0x00,0x00 # mulx 2*8($nptr),%rax,%r12 |
| 982 |
mov 48(%rsp),$bptr # counter value |
| 983 |
mov %r10,-4*8($tptr) |
| 984 |
adcx %rax,%r11 |
| 985 |
adox %r13,%r12 |
| 986 |
mulx 3*8($nptr),%rax,%r15 |
| 987 |
mov $bi,%rdx |
| 988 |
mov %r11,-3*8($tptr) |
| 989 |
adcx %rax,%r12 |
| 990 |
adox $zero,%r15 # of=0 |
| 991 |
lea 4*8($nptr),$nptr |
| 992 |
mov %r12,-2*8($tptr) |
| 993 |
|
| 994 |
jmp .Lmulx4x_1st |
| 995 |
|
| 996 |
.align 32 |
| 997 |
.Lmulx4x_1st: |
| 998 |
adcx $zero,%r15 # cf=0, modulo-scheduled |
| 999 |
mulx 0*8($aptr),%r10,%rax # a[4]*b[0] |
| 1000 |
adcx %r14,%r10 |
| 1001 |
mulx 1*8($aptr),%r11,%r14 # a[5]*b[0] |
| 1002 |
adcx %rax,%r11 |
| 1003 |
mulx 2*8($aptr),%r12,%rax # ... |
| 1004 |
adcx %r14,%r12 |
| 1005 |
mulx 3*8($aptr),%r13,%r14 |
| 1006 |
.byte 0x67,0x67 |
| 1007 |
mov $mi,%rdx |
| 1008 |
adcx %rax,%r13 |
| 1009 |
adcx $zero,%r14 # cf=0 |
| 1010 |
lea 4*8($aptr),$aptr |
| 1011 |
lea 4*8($tptr),$tptr |
| 1012 |
|
| 1013 |
adox %r15,%r10 |
| 1014 |
mulx 0*8($nptr),%rax,%r15 |
| 1015 |
adcx %rax,%r10 |
| 1016 |
adox %r15,%r11 |
| 1017 |
mulx 1*8($nptr),%rax,%r15 |
| 1018 |
adcx %rax,%r11 |
| 1019 |
adox %r15,%r12 |
| 1020 |
mulx 2*8($nptr),%rax,%r15 |
| 1021 |
mov %r10,-5*8($tptr) |
| 1022 |
adcx %rax,%r12 |
| 1023 |
mov %r11,-4*8($tptr) |
| 1024 |
adox %r15,%r13 |
| 1025 |
mulx 3*8($nptr),%rax,%r15 |
| 1026 |
mov $bi,%rdx |
| 1027 |
mov %r12,-3*8($tptr) |
| 1028 |
adcx %rax,%r13 |
| 1029 |
adox $zero,%r15 |
| 1030 |
lea 4*8($nptr),$nptr |
| 1031 |
mov %r13,-2*8($tptr) |
| 1032 |
|
| 1033 |
dec $bptr # of=0, pass cf |
| 1034 |
jnz .Lmulx4x_1st |
| 1035 |
|
| 1036 |
mov 0(%rsp),$num # load num |
| 1037 |
mov 8(%rsp),$bptr # re-load &b[i] |
| 1038 |
adc $zero,%r15 # modulo-scheduled |
| 1039 |
add %r15,%r14 |
| 1040 |
sbb %r15,%r15 # top-most carry |
| 1041 |
mov %r14,-1*8($tptr) |
| 1042 |
jmp .Lmulx4x_outer |
| 1043 |
|
| 1044 |
.align 32 |
| 1045 |
.Lmulx4x_outer: |
| 1046 |
mov ($bptr),%rdx # b[i] |
| 1047 |
lea 8($bptr),$bptr # b++ |
| 1048 |
sub $num,$aptr # rewind $aptr |
| 1049 |
mov %r15,($tptr) # save top-most carry |
| 1050 |
lea 64+4*8(%rsp),$tptr |
| 1051 |
sub $num,$nptr # rewind $nptr |
| 1052 |
|
| 1053 |
mulx 0*8($aptr),$mi,%r11 # a[0]*b[i] |
| 1054 |
xor %ebp,%ebp # xor $zero,$zero # cf=0, of=0 |
| 1055 |
mov %rdx,$bi |
| 1056 |
mulx 1*8($aptr),%r14,%r12 # a[1]*b[i] |
| 1057 |
adox -4*8($tptr),$mi |
| 1058 |
adcx %r14,%r11 |
| 1059 |
mulx 2*8($aptr),%r15,%r13 # ... |
| 1060 |
adox -3*8($tptr),%r11 |
| 1061 |
adcx %r15,%r12 |
| 1062 |
adox $zero,%r12 |
| 1063 |
adcx $zero,%r13 |
| 1064 |
|
| 1065 |
mov $bptr,8(%rsp) # off-load &b[i] |
| 1066 |
.byte 0x67 |
| 1067 |
mov $mi,%r15 |
| 1068 |
imulq 24(%rsp),$mi # "t[0]"*n0 |
| 1069 |
xor %ebp,%ebp # xor $zero,$zero # cf=0, of=0 |
| 1070 |
|
| 1071 |
mulx 3*8($aptr),%rax,%r14 |
| 1072 |
mov $mi,%rdx |
| 1073 |
adox -2*8($tptr),%r12 |
| 1074 |
adcx %rax,%r13 |
| 1075 |
adox -1*8($tptr),%r13 |
| 1076 |
adcx $zero,%r14 |
| 1077 |
lea 4*8($aptr),$aptr |
| 1078 |
adox $zero,%r14 |
| 1079 |
|
| 1080 |
mulx 0*8($nptr),%rax,%r10 |
| 1081 |
adcx %rax,%r15 # discarded |
| 1082 |
adox %r11,%r10 |
| 1083 |
mulx 1*8($nptr),%rax,%r11 |
| 1084 |
adcx %rax,%r10 |
| 1085 |
adox %r12,%r11 |
| 1086 |
mulx 2*8($nptr),%rax,%r12 |
| 1087 |
mov %r10,-4*8($tptr) |
| 1088 |
adcx %rax,%r11 |
| 1089 |
adox %r13,%r12 |
| 1090 |
mulx 3*8($nptr),%rax,%r15 |
| 1091 |
mov $bi,%rdx |
| 1092 |
mov %r11,-3*8($tptr) |
| 1093 |
lea 4*8($nptr),$nptr |
| 1094 |
adcx %rax,%r12 |
| 1095 |
adox $zero,%r15 # of=0 |
| 1096 |
mov 48(%rsp),$bptr # counter value |
| 1097 |
mov %r12,-2*8($tptr) |
| 1098 |
|
| 1099 |
jmp .Lmulx4x_inner |
| 1100 |
|
| 1101 |
.align 32 |
| 1102 |
.Lmulx4x_inner: |
| 1103 |
mulx 0*8($aptr),%r10,%rax # a[4]*b[i] |
| 1104 |
adcx $zero,%r15 # cf=0, modulo-scheduled |
| 1105 |
adox %r14,%r10 |
| 1106 |
mulx 1*8($aptr),%r11,%r14 # a[5]*b[i] |
| 1107 |
adcx 0*8($tptr),%r10 |
| 1108 |
adox %rax,%r11 |
| 1109 |
mulx 2*8($aptr),%r12,%rax # ... |
| 1110 |
adcx 1*8($tptr),%r11 |
| 1111 |
adox %r14,%r12 |
| 1112 |
mulx 3*8($aptr),%r13,%r14 |
| 1113 |
mov $mi,%rdx |
| 1114 |
adcx 2*8($tptr),%r12 |
| 1115 |
adox %rax,%r13 |
| 1116 |
adcx 3*8($tptr),%r13 |
| 1117 |
adox $zero,%r14 # of=0 |
| 1118 |
lea 4*8($aptr),$aptr |
| 1119 |
lea 4*8($tptr),$tptr |
| 1120 |
adcx $zero,%r14 # cf=0 |
| 1121 |
|
| 1122 |
adox %r15,%r10 |
| 1123 |
mulx 0*8($nptr),%rax,%r15 |
| 1124 |
adcx %rax,%r10 |
| 1125 |
adox %r15,%r11 |
| 1126 |
mulx 1*8($nptr),%rax,%r15 |
| 1127 |
adcx %rax,%r11 |
| 1128 |
adox %r15,%r12 |
| 1129 |
mulx 2*8($nptr),%rax,%r15 |
| 1130 |
mov %r10,-5*8($tptr) |
| 1131 |
adcx %rax,%r12 |
| 1132 |
adox %r15,%r13 |
| 1133 |
mulx 3*8($nptr),%rax,%r15 |
| 1134 |
mov $bi,%rdx |
| 1135 |
mov %r11,-4*8($tptr) |
| 1136 |
mov %r12,-3*8($tptr) |
| 1137 |
adcx %rax,%r13 |
| 1138 |
adox $zero,%r15 |
| 1139 |
lea 4*8($nptr),$nptr |
| 1140 |
mov %r13,-2*8($tptr) |
| 1141 |
|
| 1142 |
dec $bptr # of=0, pass cf |
| 1143 |
jnz .Lmulx4x_inner |
| 1144 |
|
| 1145 |
mov 0(%rsp),$num # load num |
| 1146 |
mov 8(%rsp),$bptr # re-load &b[i] |
| 1147 |
adc $zero,%r15 # modulo-scheduled |
| 1148 |
sub 0*8($tptr),$zero # pull top-most carry |
| 1149 |
adc %r15,%r14 |
| 1150 |
sbb %r15,%r15 # top-most carry |
| 1151 |
mov %r14,-1*8($tptr) |
| 1152 |
|
| 1153 |
cmp 16(%rsp),$bptr |
| 1154 |
jne .Lmulx4x_outer |
| 1155 |
|
| 1156 |
lea 64(%rsp),$tptr |
| 1157 |
sub $num,$nptr # rewind $nptr |
| 1158 |
neg %r15 |
| 1159 |
mov $num,%rdx |
| 1160 |
shr \$3+2,$num # %cf=0 |
| 1161 |
mov 32(%rsp),$rptr # restore rp |
| 1162 |
jmp .Lmulx4x_sub |
| 1163 |
|
| 1164 |
.align 32 |
| 1165 |
.Lmulx4x_sub: |
| 1166 |
mov 8*0($tptr),%r11 |
| 1167 |
mov 8*1($tptr),%r12 |
| 1168 |
mov 8*2($tptr),%r13 |
| 1169 |
mov 8*3($tptr),%r14 |
| 1170 |
lea 8*4($tptr),$tptr |
| 1171 |
sbb 8*0($nptr),%r11 |
| 1172 |
sbb 8*1($nptr),%r12 |
| 1173 |
sbb 8*2($nptr),%r13 |
| 1174 |
sbb 8*3($nptr),%r14 |
| 1175 |
lea 8*4($nptr),$nptr |
| 1176 |
mov %r11,8*0($rptr) |
| 1177 |
mov %r12,8*1($rptr) |
| 1178 |
mov %r13,8*2($rptr) |
| 1179 |
mov %r14,8*3($rptr) |
| 1180 |
lea 8*4($rptr),$rptr |
| 1181 |
dec $num # preserves %cf |
| 1182 |
jnz .Lmulx4x_sub |
| 1183 |
|
| 1184 |
sbb \$0,%r15 # top-most carry |
| 1185 |
lea 64(%rsp),$tptr |
| 1186 |
sub %rdx,$rptr # rewind |
| 1187 |
|
| 1188 |
movq %r15,%xmm1 |
| 1189 |
pxor %xmm0,%xmm0 |
| 1190 |
pshufd \$0,%xmm1,%xmm1 |
| 1191 |
mov 40(%rsp),%rsi # restore %rsp |
| 1192 |
jmp .Lmulx4x_cond_copy |
| 1193 |
|
| 1194 |
.align 32 |
| 1195 |
.Lmulx4x_cond_copy: |
| 1196 |
movdqa 16*0($tptr),%xmm2 |
| 1197 |
movdqa 16*1($tptr),%xmm3 |
| 1198 |
lea 16*2($tptr),$tptr |
| 1199 |
movdqu 16*0($rptr),%xmm4 |
| 1200 |
movdqu 16*1($rptr),%xmm5 |
| 1201 |
lea 16*2($rptr),$rptr |
| 1202 |
movdqa %xmm0,-16*2($tptr) # zero tp |
| 1203 |
movdqa %xmm0,-16*1($tptr) |
| 1204 |
pcmpeqd %xmm1,%xmm0 |
| 1205 |
pand %xmm1,%xmm2 |
| 1206 |
pand %xmm1,%xmm3 |
| 1207 |
pand %xmm0,%xmm4 |
| 1208 |
pand %xmm0,%xmm5 |
| 1209 |
pxor %xmm0,%xmm0 |
| 1210 |
por %xmm2,%xmm4 |
| 1211 |
por %xmm3,%xmm5 |
| 1212 |
movdqu %xmm4,-16*2($rptr) |
| 1213 |
movdqu %xmm5,-16*1($rptr) |
| 1214 |
sub \$32,%rdx |
| 1215 |
jnz .Lmulx4x_cond_copy |
| 1216 |
|
| 1217 |
mov %rdx,($tptr) |
| 1218 |
|
| 1219 |
mov \$1,%rax |
| 1220 |
mov -48(%rsi),%r15 |
| 1221 |
mov -40(%rsi),%r14 |
| 1222 |
mov -32(%rsi),%r13 |
| 1223 |
mov -24(%rsi),%r12 |
| 1224 |
mov -16(%rsi),%rbp |
| 1225 |
mov -8(%rsi),%rbx |
| 1226 |
lea (%rsi),%rsp |
| 1227 |
.Lmulx4x_epilogue: |
| 1228 |
ret |
| 1229 |
.size bn_mulx4x_mont,.-bn_mulx4x_mont |
| 1230 |
___ |
| 1231 |
}}} |
| 1232 |
$code.=<<___; |
| 1233 |
.asciz "Montgomery Multiplication for x86_64, CRYPTOGAMS by <appro\@openssl.org>" |
| 1234 |
.align 16 |
| 1235 |
___ |
| 1236 |
|
| 1237 |
# EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame, |
| 1238 |
# CONTEXT *context,DISPATCHER_CONTEXT *disp) |
| 1239 |
if ($win64) { |
| 1240 |
$rec="%rcx"; |
| 1241 |
$frame="%rdx"; |
| 1242 |
$context="%r8"; |
| 1243 |
$disp="%r9"; |
| 1244 |
|
| 1245 |
$code.=<<___; |
| 1246 |
.extern __imp_RtlVirtualUnwind |
| 1247 |
.type mul_handler,\@abi-omnipotent |
| 1248 |
.align 16 |
| 1249 |
mul_handler: |
| 1250 |
push %rsi |
| 1251 |
push %rdi |
| 1252 |
push %rbx |
| 1253 |
push %rbp |
| 1254 |
push %r12 |
| 1255 |
push %r13 |
| 1256 |
push %r14 |
| 1257 |
push %r15 |
| 1258 |
pushfq |
| 1259 |
sub \$64,%rsp |
| 1260 |
|
| 1261 |
mov 120($context),%rax # pull context->Rax |
| 1262 |
mov 248($context),%rbx # pull context->Rip |
| 1263 |
|
| 1264 |
mov 8($disp),%rsi # disp->ImageBase |
| 1265 |
mov 56($disp),%r11 # disp->HandlerData |
| 1266 |
|
| 1267 |
mov 0(%r11),%r10d # HandlerData[0] |
| 1268 |
lea (%rsi,%r10),%r10 # end of prologue label |
| 1269 |
cmp %r10,%rbx # context->Rip<end of prologue label |
| 1270 |
jb .Lcommon_seh_tail |
| 1271 |
|
| 1272 |
mov 152($context),%rax # pull context->Rsp |
| 1273 |
|
| 1274 |
mov 4(%r11),%r10d # HandlerData[1] |
| 1275 |
lea (%rsi,%r10),%r10 # epilogue label |
| 1276 |
cmp %r10,%rbx # context->Rip>=epilogue label |
| 1277 |
jae .Lcommon_seh_tail |
| 1278 |
|
| 1279 |
mov 192($context),%r10 # pull $num |
| 1280 |
mov 8(%rax,%r10,8),%rax # pull saved stack pointer |
| 1281 |
lea 48(%rax),%rax |
| 1282 |
|
| 1283 |
mov -8(%rax),%rbx |
| 1284 |
mov -16(%rax),%rbp |
| 1285 |
mov -24(%rax),%r12 |
| 1286 |
mov -32(%rax),%r13 |
| 1287 |
mov -40(%rax),%r14 |
| 1288 |
mov -48(%rax),%r15 |
| 1289 |
mov %rbx,144($context) # restore context->Rbx |
| 1290 |
mov %rbp,160($context) # restore context->Rbp |
| 1291 |
mov %r12,216($context) # restore context->R12 |
| 1292 |
mov %r13,224($context) # restore context->R13 |
| 1293 |
mov %r14,232($context) # restore context->R14 |
| 1294 |
mov %r15,240($context) # restore context->R15 |
| 1295 |
|
| 1296 |
jmp .Lcommon_seh_tail |
| 1297 |
.size mul_handler,.-mul_handler |
| 1298 |
|
| 1299 |
.type sqr_handler,\@abi-omnipotent |
| 1300 |
.align 16 |
| 1301 |
sqr_handler: |
| 1302 |
push %rsi |
| 1303 |
push %rdi |
| 1304 |
push %rbx |
| 1305 |
push %rbp |
| 1306 |
push %r12 |
| 1307 |
push %r13 |
| 1308 |
push %r14 |
| 1309 |
push %r15 |
| 1310 |
pushfq |
| 1311 |
sub \$64,%rsp |
| 1312 |
|
| 1313 |
mov 120($context),%rax # pull context->Rax |
| 1314 |
mov 248($context),%rbx # pull context->Rip |
| 1315 |
|
| 1316 |
mov 8($disp),%rsi # disp->ImageBase |
| 1317 |
mov 56($disp),%r11 # disp->HandlerData |
| 1318 |
|
| 1319 |
mov 0(%r11),%r10d # HandlerData[0] |
| 1320 |
lea (%rsi,%r10),%r10 # end of prologue label |
| 1321 |
cmp %r10,%rbx # context->Rip<.Lsqr_body |
| 1322 |
jb .Lcommon_seh_tail |
| 1323 |
|
| 1324 |
mov 152($context),%rax # pull context->Rsp |
| 1325 |
|
| 1326 |
mov 4(%r11),%r10d # HandlerData[1] |
| 1327 |
lea (%rsi,%r10),%r10 # epilogue label |
| 1328 |
cmp %r10,%rbx # context->Rip>=.Lsqr_epilogue |
| 1329 |
jae .Lcommon_seh_tail |
| 1330 |
|
| 1331 |
mov 40(%rax),%rax # pull saved stack pointer |
| 1332 |
|
| 1333 |
mov -8(%rax),%rbx |
| 1334 |
mov -16(%rax),%rbp |
| 1335 |
mov -24(%rax),%r12 |
| 1336 |
mov -32(%rax),%r13 |
| 1337 |
mov -40(%rax),%r14 |
| 1338 |
mov -48(%rax),%r15 |
| 1339 |
mov %rbx,144($context) # restore context->Rbx |
| 1340 |
mov %rbp,160($context) # restore context->Rbp |
| 1341 |
mov %r12,216($context) # restore context->R12 |
| 1342 |
mov %r13,224($context) # restore context->R13 |
| 1343 |
mov %r14,232($context) # restore context->R14 |
| 1344 |
mov %r15,240($context) # restore context->R15 |
| 1345 |
|
| 1346 |
.Lcommon_seh_tail: |
| 1347 |
mov 8(%rax),%rdi |
| 1348 |
mov 16(%rax),%rsi |
| 1349 |
mov %rax,152($context) # restore context->Rsp |
| 1350 |
mov %rsi,168($context) # restore context->Rsi |
| 1351 |
mov %rdi,176($context) # restore context->Rdi |
| 1352 |
|
| 1353 |
mov 40($disp),%rdi # disp->ContextRecord |
| 1354 |
mov $context,%rsi # context |
| 1355 |
mov \$154,%ecx # sizeof(CONTEXT) |
| 1356 |
.long 0xa548f3fc # cld; rep movsq |
| 1357 |
|
| 1358 |
mov $disp,%rsi |
| 1359 |
xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER |
| 1360 |
mov 8(%rsi),%rdx # arg2, disp->ImageBase |
| 1361 |
mov 0(%rsi),%r8 # arg3, disp->ControlPc |
| 1362 |
mov 16(%rsi),%r9 # arg4, disp->FunctionEntry |
| 1363 |
mov 40(%rsi),%r10 # disp->ContextRecord |
| 1364 |
lea 56(%rsi),%r11 # &disp->HandlerData |
| 1365 |
lea 24(%rsi),%r12 # &disp->EstablisherFrame |
| 1366 |
mov %r10,32(%rsp) # arg5 |
| 1367 |
mov %r11,40(%rsp) # arg6 |
| 1368 |
mov %r12,48(%rsp) # arg7 |
| 1369 |
mov %rcx,56(%rsp) # arg8, (NULL) |
| 1370 |
call *__imp_RtlVirtualUnwind(%rip) |
| 1371 |
|
| 1372 |
mov \$1,%eax # ExceptionContinueSearch |
| 1373 |
add \$64,%rsp |
| 1374 |
popfq |
| 1375 |
pop %r15 |
| 1376 |
pop %r14 |
| 1377 |
pop %r13 |
| 1378 |
pop %r12 |
| 1379 |
pop %rbp |
| 1380 |
pop %rbx |
| 1381 |
pop %rdi |
| 1382 |
pop %rsi |
| 1383 |
ret |
| 1384 |
.size sqr_handler,.-sqr_handler |
| 1385 |
|
| 1386 |
.section .pdata |
| 1387 |
.align 4 |
| 1388 |
.rva .LSEH_begin_bn_mul_mont |
| 1389 |
.rva .LSEH_end_bn_mul_mont |
| 1390 |
.rva .LSEH_info_bn_mul_mont |
| 1391 |
|
| 1392 |
.rva .LSEH_begin_bn_mul4x_mont |
| 1393 |
.rva .LSEH_end_bn_mul4x_mont |
| 1394 |
.rva .LSEH_info_bn_mul4x_mont |
| 1395 |
|
| 1396 |
.rva .LSEH_begin_bn_sqr8x_mont |
| 1397 |
.rva .LSEH_end_bn_sqr8x_mont |
| 1398 |
.rva .LSEH_info_bn_sqr8x_mont |
| 1399 |
___ |
| 1400 |
$code.=<<___ if ($addx); |
| 1401 |
.rva .LSEH_begin_bn_mulx4x_mont |
| 1402 |
.rva .LSEH_end_bn_mulx4x_mont |
| 1403 |
.rva .LSEH_info_bn_mulx4x_mont |
| 1404 |
___ |
| 1405 |
$code.=<<___; |
| 1406 |
.section .xdata |
| 1407 |
.align 8 |
| 1408 |
.LSEH_info_bn_mul_mont: |
| 1409 |
.byte 9,0,0,0 |
| 1410 |
.rva mul_handler |
| 1411 |
.rva .Lmul_body,.Lmul_epilogue # HandlerData[] |
| 1412 |
.LSEH_info_bn_mul4x_mont: |
| 1413 |
.byte 9,0,0,0 |
| 1414 |
.rva mul_handler |
| 1415 |
.rva .Lmul4x_body,.Lmul4x_epilogue # HandlerData[] |
| 1416 |
.LSEH_info_bn_sqr8x_mont: |
| 1417 |
.byte 9,0,0,0 |
| 1418 |
.rva sqr_handler |
| 1419 |
.rva .Lsqr8x_body,.Lsqr8x_epilogue # HandlerData[] |
| 1420 |
___ |
| 1421 |
$code.=<<___ if ($addx); |
| 1422 |
.LSEH_info_bn_mulx4x_mont: |
| 1423 |
.byte 9,0,0,0 |
| 1424 |
.rva sqr_handler |
| 1425 |
.rva .Lmulx4x_body,.Lmulx4x_epilogue # HandlerData[] |
| 1426 |
___ |
| 1427 |
} |
| 1428 |
|
| 1429 |
print $code; |
| 1430 |
close STDOUT; |