| 1 |
#!/usr/bin/env perl |
| 2 |
# |
| 3 |
# ==================================================================== |
| 4 |
# Written by Andy Polyakov <[email protected]> for the OpenSSL |
| 5 |
# project. The module is, however, dual licensed under OpenSSL and |
| 6 |
# CRYPTOGAMS licenses depending on where you obtain it. For further |
| 7 |
# details see http://www.openssl.org/~appro/cryptogams/. |
| 8 |
# ==================================================================== |
| 9 |
# |
| 10 |
# This module implements Poly1305 hash for ARMv8. |
| 11 |
# |
| 12 |
# June 2015 |
| 13 |
# |
| 14 |
# Numbers are cycles per processed byte with poly1305_blocks alone. |
| 15 |
# |
| 16 |
# IALU/gcc-4.9 NEON |
| 17 |
# |
| 18 |
# Apple A7 1.86/+5% 0.72 |
| 19 |
# Cortex-A53 2.63/+58% 1.47 |
| 20 |
# Cortex-A57 2.70/+7% 1.14 |
| 21 |
# Denver 1.39/+50% 1.18(*) |
| 22 |
# X-Gene 2.00/+68% 2.19 |
| 23 |
# |
| 24 |
# (*) estimate based on resources availability is less than 1.0, |
| 25 |
# i.e. measured result is worse than expected, presumably binary |
| 26 |
# translator is not almighty; |
| 27 |
|
| 28 |
$flavour=shift; |
| 29 |
$output=shift; |
| 30 |
|
| 31 |
$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; |
| 32 |
( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or |
| 33 |
( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or |
| 34 |
die "can't locate arm-xlate.pl"; |
| 35 |
|
| 36 |
open OUT,"| \"$^X\" $xlate $flavour $output"; |
| 37 |
*STDOUT=*OUT; |
| 38 |
|
| 39 |
my ($ctx,$inp,$len,$padbit) = map("x$_",(0..3)); |
| 40 |
my ($mac,$nonce)=($inp,$len); |
| 41 |
|
| 42 |
my ($h0,$h1,$h2,$r0,$r1,$s1,$t0,$t1,$d0,$d1,$d2) = map("x$_",(4..14)); |
| 43 |
|
| 44 |
$code.=<<___; |
| 45 |
#include <openssl/arm_arch.h> |
| 46 |
|
| 47 |
.text |
| 48 |
|
| 49 |
// forward "declarations" are required for Apple |
| 50 |
.extern OPENSSL_armcap_P |
| 51 |
.globl poly1305_blocks |
| 52 |
.globl poly1305_emit |
| 53 |
|
| 54 |
.globl poly1305_init |
| 55 |
.type poly1305_init,%function |
| 56 |
.align 5 |
| 57 |
poly1305_init: |
| 58 |
cmp $inp,xzr |
| 59 |
stp xzr,xzr,[$ctx] // zero hash value |
| 60 |
stp xzr,xzr,[$ctx,#16] // [along with is_base2_26] |
| 61 |
|
| 62 |
csel x0,xzr,x0,eq |
| 63 |
b.eq .Lno_key |
| 64 |
|
| 65 |
#ifdef __ILP32__ |
| 66 |
ldrsw $t1,.LOPENSSL_armcap_P |
| 67 |
#else |
| 68 |
ldr $t1,.LOPENSSL_armcap_P |
| 69 |
#endif |
| 70 |
adr $t0,.LOPENSSL_armcap_P |
| 71 |
|
| 72 |
ldp $r0,$r1,[$inp] // load key |
| 73 |
mov $s1,#0xfffffffc0fffffff |
| 74 |
movk $s1,#0x0fff,lsl#48 |
| 75 |
ldr w17,[$t0,$t1] |
| 76 |
#ifdef __ARMEB__ |
| 77 |
rev $r0,$r0 // flip bytes |
| 78 |
rev $r1,$r1 |
| 79 |
#endif |
| 80 |
and $r0,$r0,$s1 // &=0ffffffc0fffffff |
| 81 |
and $s1,$s1,#-4 |
| 82 |
and $r1,$r1,$s1 // &=0ffffffc0ffffffc |
| 83 |
stp $r0,$r1,[$ctx,#32] // save key value |
| 84 |
|
| 85 |
tst w17,#ARMV7_NEON |
| 86 |
|
| 87 |
adr $d0,poly1305_blocks |
| 88 |
adr $r0,poly1305_blocks_neon |
| 89 |
adr $d1,poly1305_emit |
| 90 |
adr $r1,poly1305_emit_neon |
| 91 |
|
| 92 |
csel $d0,$d0,$r0,eq |
| 93 |
csel $d1,$d1,$r1,eq |
| 94 |
|
| 95 |
stp $d0,$d1,[$len] |
| 96 |
|
| 97 |
mov x0,#1 |
| 98 |
.Lno_key: |
| 99 |
ret |
| 100 |
.size poly1305_init,.-poly1305_init |
| 101 |
|
| 102 |
.type poly1305_blocks,%function |
| 103 |
.align 5 |
| 104 |
poly1305_blocks: |
| 105 |
ands $len,$len,#-16 |
| 106 |
b.eq .Lno_data |
| 107 |
|
| 108 |
ldp $h0,$h1,[$ctx] // load hash value |
| 109 |
ldp $r0,$r1,[$ctx,#32] // load key value |
| 110 |
ldr $h2,[$ctx,#16] |
| 111 |
add $s1,$r1,$r1,lsr#2 // s1 = r1 + (r1 >> 2) |
| 112 |
b .Loop |
| 113 |
|
| 114 |
.align 5 |
| 115 |
.Loop: |
| 116 |
ldp $t0,$t1,[$inp],#16 // load input |
| 117 |
sub $len,$len,#16 |
| 118 |
#ifdef __ARMEB__ |
| 119 |
rev $t0,$t0 |
| 120 |
rev $t1,$t1 |
| 121 |
#endif |
| 122 |
adds $h0,$h0,$t0 // accumulate input |
| 123 |
adcs $h1,$h1,$t1 |
| 124 |
|
| 125 |
mul $d0,$h0,$r0 // h0*r0 |
| 126 |
adc $h2,$h2,$padbit |
| 127 |
umulh $d1,$h0,$r0 |
| 128 |
|
| 129 |
mul $t0,$h1,$s1 // h1*5*r1 |
| 130 |
umulh $t1,$h1,$s1 |
| 131 |
|
| 132 |
adds $d0,$d0,$t0 |
| 133 |
mul $t0,$h0,$r1 // h0*r1 |
| 134 |
adc $d1,$d1,$t1 |
| 135 |
umulh $d2,$h0,$r1 |
| 136 |
|
| 137 |
adds $d1,$d1,$t0 |
| 138 |
mul $t0,$h1,$r0 // h1*r0 |
| 139 |
adc $d2,$d2,xzr |
| 140 |
umulh $t1,$h1,$r0 |
| 141 |
|
| 142 |
adds $d1,$d1,$t0 |
| 143 |
mul $t0,$h2,$s1 // h2*5*r1 |
| 144 |
adc $d2,$d2,$t1 |
| 145 |
mul $t1,$h2,$r0 // h2*r0 |
| 146 |
|
| 147 |
adds $d1,$d1,$t0 |
| 148 |
adc $d2,$d2,$t1 |
| 149 |
|
| 150 |
and $t0,$d2,#-4 // final reduction |
| 151 |
and $h2,$d2,#3 |
| 152 |
add $t0,$t0,$d2,lsr#2 |
| 153 |
adds $h0,$d0,$t0 |
| 154 |
adc $h1,$d1,xzr |
| 155 |
|
| 156 |
cbnz $len,.Loop |
| 157 |
|
| 158 |
stp $h0,$h1,[$ctx] // store hash value |
| 159 |
str $h2,[$ctx,#16] |
| 160 |
|
| 161 |
.Lno_data: |
| 162 |
ret |
| 163 |
.size poly1305_blocks,.-poly1305_blocks |
| 164 |
|
| 165 |
.type poly1305_emit,%function |
| 166 |
.align 5 |
| 167 |
poly1305_emit: |
| 168 |
ldp $h0,$h1,[$ctx] // load hash base 2^64 |
| 169 |
ldr $h2,[$ctx,#16] |
| 170 |
ldp $t0,$t1,[$nonce] // load nonce |
| 171 |
|
| 172 |
adds $d0,$h0,#5 // compare to modulus |
| 173 |
adcs $d1,$h1,xzr |
| 174 |
adc $d2,$h2,xzr |
| 175 |
|
| 176 |
tst $d2,#-4 // see if it's carried/borrowed |
| 177 |
|
| 178 |
csel $h0,$h0,$d0,eq |
| 179 |
csel $h1,$h1,$d1,eq |
| 180 |
|
| 181 |
#ifdef __ARMEB__ |
| 182 |
ror $t0,$t0,#32 // flip nonce words |
| 183 |
ror $t1,$t1,#32 |
| 184 |
#endif |
| 185 |
adds $h0,$h0,$t0 // accumulate nonce |
| 186 |
adc $h1,$h1,$t1 |
| 187 |
#ifdef __ARMEB__ |
| 188 |
rev $h0,$h0 // flip output bytes |
| 189 |
rev $h1,$h1 |
| 190 |
#endif |
| 191 |
stp $h0,$h1,[$mac] // write result |
| 192 |
|
| 193 |
ret |
| 194 |
.size poly1305_emit,.-poly1305_emit |
| 195 |
___ |
| 196 |
my ($R0,$R1,$S1,$R2,$S2,$R3,$S3,$R4,$S4) = map("v$_.4s",(0..8)); |
| 197 |
my ($IN01_0,$IN01_1,$IN01_2,$IN01_3,$IN01_4) = map("v$_.2s",(9..13)); |
| 198 |
my ($IN23_0,$IN23_1,$IN23_2,$IN23_3,$IN23_4) = map("v$_.2s",(14..18)); |
| 199 |
my ($ACC0,$ACC1,$ACC2,$ACC3,$ACC4) = map("v$_.2d",(19..23)); |
| 200 |
my ($H0,$H1,$H2,$H3,$H4) = map("v$_.2s",(24..28)); |
| 201 |
my ($T0,$T1,$MASK) = map("v$_",(29..31)); |
| 202 |
|
| 203 |
my ($in2,$zeros)=("x16","x17"); |
| 204 |
my $is_base2_26 = $zeros; # borrow |
| 205 |
|
| 206 |
$code.=<<___; |
| 207 |
.type poly1305_mult,%function |
| 208 |
.align 5 |
| 209 |
poly1305_mult: |
| 210 |
mul $d0,$h0,$r0 // h0*r0 |
| 211 |
umulh $d1,$h0,$r0 |
| 212 |
|
| 213 |
mul $t0,$h1,$s1 // h1*5*r1 |
| 214 |
umulh $t1,$h1,$s1 |
| 215 |
|
| 216 |
adds $d0,$d0,$t0 |
| 217 |
mul $t0,$h0,$r1 // h0*r1 |
| 218 |
adc $d1,$d1,$t1 |
| 219 |
umulh $d2,$h0,$r1 |
| 220 |
|
| 221 |
adds $d1,$d1,$t0 |
| 222 |
mul $t0,$h1,$r0 // h1*r0 |
| 223 |
adc $d2,$d2,xzr |
| 224 |
umulh $t1,$h1,$r0 |
| 225 |
|
| 226 |
adds $d1,$d1,$t0 |
| 227 |
mul $t0,$h2,$s1 // h2*5*r1 |
| 228 |
adc $d2,$d2,$t1 |
| 229 |
mul $t1,$h2,$r0 // h2*r0 |
| 230 |
|
| 231 |
adds $d1,$d1,$t0 |
| 232 |
adc $d2,$d2,$t1 |
| 233 |
|
| 234 |
and $t0,$d2,#-4 // final reduction |
| 235 |
and $h2,$d2,#3 |
| 236 |
add $t0,$t0,$d2,lsr#2 |
| 237 |
adds $h0,$d0,$t0 |
| 238 |
adc $h1,$d1,xzr |
| 239 |
|
| 240 |
ret |
| 241 |
.size poly1305_mult,.-poly1305_mult |
| 242 |
|
| 243 |
.type poly1305_splat,%function |
| 244 |
.align 5 |
| 245 |
poly1305_splat: |
| 246 |
and x12,$h0,#0x03ffffff // base 2^64 -> base 2^26 |
| 247 |
ubfx x13,$h0,#26,#26 |
| 248 |
extr x14,$h1,$h0,#52 |
| 249 |
and x14,x14,#0x03ffffff |
| 250 |
ubfx x15,$h1,#14,#26 |
| 251 |
extr x16,$h2,$h1,#40 |
| 252 |
|
| 253 |
str w12,[$ctx,#16*0] // r0 |
| 254 |
add w12,w13,w13,lsl#2 // r1*5 |
| 255 |
str w13,[$ctx,#16*1] // r1 |
| 256 |
add w13,w14,w14,lsl#2 // r2*5 |
| 257 |
str w12,[$ctx,#16*2] // s1 |
| 258 |
str w14,[$ctx,#16*3] // r2 |
| 259 |
add w14,w15,w15,lsl#2 // r3*5 |
| 260 |
str w13,[$ctx,#16*4] // s2 |
| 261 |
str w15,[$ctx,#16*5] // r3 |
| 262 |
add w15,w16,w16,lsl#2 // r4*5 |
| 263 |
str w14,[$ctx,#16*6] // s3 |
| 264 |
str w16,[$ctx,#16*7] // r4 |
| 265 |
str w15,[$ctx,#16*8] // s4 |
| 266 |
|
| 267 |
ret |
| 268 |
.size poly1305_splat,.-poly1305_splat |
| 269 |
|
| 270 |
.type poly1305_blocks_neon,%function |
| 271 |
.align 5 |
| 272 |
poly1305_blocks_neon: |
| 273 |
ldr $is_base2_26,[$ctx,#24] |
| 274 |
cmp $len,#128 |
| 275 |
b.hs .Lblocks_neon |
| 276 |
cbz $is_base2_26,poly1305_blocks |
| 277 |
|
| 278 |
.Lblocks_neon: |
| 279 |
stp x29,x30,[sp,#-80]! |
| 280 |
add x29,sp,#0 |
| 281 |
|
| 282 |
ands $len,$len,#-16 |
| 283 |
b.eq .Lno_data_neon |
| 284 |
|
| 285 |
cbz $is_base2_26,.Lbase2_64_neon |
| 286 |
|
| 287 |
ldp w10,w11,[$ctx] // load hash value base 2^26 |
| 288 |
ldp w12,w13,[$ctx,#8] |
| 289 |
ldr w14,[$ctx,#16] |
| 290 |
|
| 291 |
tst $len,#31 |
| 292 |
b.eq .Leven_neon |
| 293 |
|
| 294 |
ldp $r0,$r1,[$ctx,#32] // load key value |
| 295 |
|
| 296 |
add $h0,x10,x11,lsl#26 // base 2^26 -> base 2^64 |
| 297 |
lsr $h1,x12,#12 |
| 298 |
adds $h0,$h0,x12,lsl#52 |
| 299 |
add $h1,$h1,x13,lsl#14 |
| 300 |
adc $h1,$h1,xzr |
| 301 |
lsr $h2,x14,#24 |
| 302 |
adds $h1,$h1,x14,lsl#40 |
| 303 |
adc $d2,$h2,xzr // can be partially reduced... |
| 304 |
|
| 305 |
ldp $d0,$d1,[$inp],#16 // load input |
| 306 |
sub $len,$len,#16 |
| 307 |
add $s1,$r1,$r1,lsr#2 // s1 = r1 + (r1 >> 2) |
| 308 |
|
| 309 |
and $t0,$d2,#-4 // ... so reduce |
| 310 |
and $h2,$d2,#3 |
| 311 |
add $t0,$t0,$d2,lsr#2 |
| 312 |
adds $h0,$h0,$t0 |
| 313 |
adc $h1,$h1,xzr |
| 314 |
|
| 315 |
#ifdef __ARMEB__ |
| 316 |
rev $d0,$d0 |
| 317 |
rev $d1,$d1 |
| 318 |
#endif |
| 319 |
adds $h0,$h0,$d0 // accumulate input |
| 320 |
adcs $h1,$h1,$d1 |
| 321 |
adc $h2,$h2,$padbit |
| 322 |
|
| 323 |
bl poly1305_mult |
| 324 |
ldr x30,[sp,#8] |
| 325 |
|
| 326 |
cbz $padbit,.Lstore_base2_64_neon |
| 327 |
|
| 328 |
and x10,$h0,#0x03ffffff // base 2^64 -> base 2^26 |
| 329 |
ubfx x11,$h0,#26,#26 |
| 330 |
extr x12,$h1,$h0,#52 |
| 331 |
and x12,x12,#0x03ffffff |
| 332 |
ubfx x13,$h1,#14,#26 |
| 333 |
extr x14,$h2,$h1,#40 |
| 334 |
|
| 335 |
cbnz $len,.Leven_neon |
| 336 |
|
| 337 |
stp w10,w11,[$ctx] // store hash value base 2^26 |
| 338 |
stp w12,w13,[$ctx,#8] |
| 339 |
str w14,[$ctx,#16] |
| 340 |
b .Lno_data_neon |
| 341 |
|
| 342 |
.align 4 |
| 343 |
.Lstore_base2_64_neon: |
| 344 |
stp $h0,$h1,[$ctx] // store hash value base 2^64 |
| 345 |
stp $h2,xzr,[$ctx,#16] // note that is_base2_26 is zeroed |
| 346 |
b .Lno_data_neon |
| 347 |
|
| 348 |
.align 4 |
| 349 |
.Lbase2_64_neon: |
| 350 |
ldp $r0,$r1,[$ctx,#32] // load key value |
| 351 |
|
| 352 |
ldp $h0,$h1,[$ctx] // load hash value base 2^64 |
| 353 |
ldr $h2,[$ctx,#16] |
| 354 |
|
| 355 |
tst $len,#31 |
| 356 |
b.eq .Linit_neon |
| 357 |
|
| 358 |
ldp $d0,$d1,[$inp],#16 // load input |
| 359 |
sub $len,$len,#16 |
| 360 |
add $s1,$r1,$r1,lsr#2 // s1 = r1 + (r1 >> 2) |
| 361 |
#ifdef __ARMEB__ |
| 362 |
rev $d0,$d0 |
| 363 |
rev $d1,$d1 |
| 364 |
#endif |
| 365 |
adds $h0,$h0,$d0 // accumulate input |
| 366 |
adcs $h1,$h1,$d1 |
| 367 |
adc $h2,$h2,$padbit |
| 368 |
|
| 369 |
bl poly1305_mult |
| 370 |
|
| 371 |
.Linit_neon: |
| 372 |
and x10,$h0,#0x03ffffff // base 2^64 -> base 2^26 |
| 373 |
ubfx x11,$h0,#26,#26 |
| 374 |
extr x12,$h1,$h0,#52 |
| 375 |
and x12,x12,#0x03ffffff |
| 376 |
ubfx x13,$h1,#14,#26 |
| 377 |
extr x14,$h2,$h1,#40 |
| 378 |
|
| 379 |
stp d8,d9,[sp,#16] // meet ABI requirements |
| 380 |
stp d10,d11,[sp,#32] |
| 381 |
stp d12,d13,[sp,#48] |
| 382 |
stp d14,d15,[sp,#64] |
| 383 |
|
| 384 |
fmov ${H0},x10 |
| 385 |
fmov ${H1},x11 |
| 386 |
fmov ${H2},x12 |
| 387 |
fmov ${H3},x13 |
| 388 |
fmov ${H4},x14 |
| 389 |
|
| 390 |
////////////////////////////////// initialize r^n table |
| 391 |
mov $h0,$r0 // r^1 |
| 392 |
add $s1,$r1,$r1,lsr#2 // s1 = r1 + (r1 >> 2) |
| 393 |
mov $h1,$r1 |
| 394 |
mov $h2,xzr |
| 395 |
add $ctx,$ctx,#48+12 |
| 396 |
bl poly1305_splat |
| 397 |
|
| 398 |
bl poly1305_mult // r^2 |
| 399 |
sub $ctx,$ctx,#4 |
| 400 |
bl poly1305_splat |
| 401 |
|
| 402 |
bl poly1305_mult // r^3 |
| 403 |
sub $ctx,$ctx,#4 |
| 404 |
bl poly1305_splat |
| 405 |
|
| 406 |
bl poly1305_mult // r^4 |
| 407 |
sub $ctx,$ctx,#4 |
| 408 |
bl poly1305_splat |
| 409 |
ldr x30,[sp,#8] |
| 410 |
|
| 411 |
add $in2,$inp,#32 |
| 412 |
adr $zeros,.Lzeros |
| 413 |
subs $len,$len,#64 |
| 414 |
csel $in2,$zeros,$in2,lo |
| 415 |
|
| 416 |
mov x4,#1 |
| 417 |
str x4,[$ctx,#-24] // set is_base2_26 |
| 418 |
sub $ctx,$ctx,#48 // restore original $ctx |
| 419 |
b .Ldo_neon |
| 420 |
|
| 421 |
.align 4 |
| 422 |
.Leven_neon: |
| 423 |
add $in2,$inp,#32 |
| 424 |
adr $zeros,.Lzeros |
| 425 |
subs $len,$len,#64 |
| 426 |
csel $in2,$zeros,$in2,lo |
| 427 |
|
| 428 |
stp d8,d9,[sp,#16] // meet ABI requirements |
| 429 |
stp d10,d11,[sp,#32] |
| 430 |
stp d12,d13,[sp,#48] |
| 431 |
stp d14,d15,[sp,#64] |
| 432 |
|
| 433 |
fmov ${H0},x10 |
| 434 |
fmov ${H1},x11 |
| 435 |
fmov ${H2},x12 |
| 436 |
fmov ${H3},x13 |
| 437 |
fmov ${H4},x14 |
| 438 |
|
| 439 |
.Ldo_neon: |
| 440 |
ldp x8,x12,[$in2],#16 // inp[2:3] (or zero) |
| 441 |
ldp x9,x13,[$in2],#48 |
| 442 |
|
| 443 |
lsl $padbit,$padbit,#24 |
| 444 |
add x15,$ctx,#48 |
| 445 |
|
| 446 |
#ifdef __ARMEB__ |
| 447 |
rev x8,x8 |
| 448 |
rev x12,x12 |
| 449 |
rev x9,x9 |
| 450 |
rev x13,x13 |
| 451 |
#endif |
| 452 |
and x4,x8,#0x03ffffff // base 2^64 -> base 2^26 |
| 453 |
and x5,x9,#0x03ffffff |
| 454 |
ubfx x6,x8,#26,#26 |
| 455 |
ubfx x7,x9,#26,#26 |
| 456 |
add x4,x4,x5,lsl#32 // bfi x4,x5,#32,#32 |
| 457 |
extr x8,x12,x8,#52 |
| 458 |
extr x9,x13,x9,#52 |
| 459 |
add x6,x6,x7,lsl#32 // bfi x6,x7,#32,#32 |
| 460 |
fmov $IN23_0,x4 |
| 461 |
and x8,x8,#0x03ffffff |
| 462 |
and x9,x9,#0x03ffffff |
| 463 |
ubfx x10,x12,#14,#26 |
| 464 |
ubfx x11,x13,#14,#26 |
| 465 |
add x12,$padbit,x12,lsr#40 |
| 466 |
add x13,$padbit,x13,lsr#40 |
| 467 |
add x8,x8,x9,lsl#32 // bfi x8,x9,#32,#32 |
| 468 |
fmov $IN23_1,x6 |
| 469 |
add x10,x10,x11,lsl#32 // bfi x10,x11,#32,#32 |
| 470 |
add x12,x12,x13,lsl#32 // bfi x12,x13,#32,#32 |
| 471 |
fmov $IN23_2,x8 |
| 472 |
fmov $IN23_3,x10 |
| 473 |
fmov $IN23_4,x12 |
| 474 |
|
| 475 |
ldp x8,x12,[$inp],#16 // inp[0:1] |
| 476 |
ldp x9,x13,[$inp],#48 |
| 477 |
|
| 478 |
ld1 {$R0,$R1,$S1,$R2},[x15],#64 |
| 479 |
ld1 {$S2,$R3,$S3,$R4},[x15],#64 |
| 480 |
ld1 {$S4},[x15] |
| 481 |
|
| 482 |
#ifdef __ARMEB__ |
| 483 |
rev x8,x8 |
| 484 |
rev x12,x12 |
| 485 |
rev x9,x9 |
| 486 |
rev x13,x13 |
| 487 |
#endif |
| 488 |
and x4,x8,#0x03ffffff // base 2^64 -> base 2^26 |
| 489 |
and x5,x9,#0x03ffffff |
| 490 |
ubfx x6,x8,#26,#26 |
| 491 |
ubfx x7,x9,#26,#26 |
| 492 |
add x4,x4,x5,lsl#32 // bfi x4,x5,#32,#32 |
| 493 |
extr x8,x12,x8,#52 |
| 494 |
extr x9,x13,x9,#52 |
| 495 |
add x6,x6,x7,lsl#32 // bfi x6,x7,#32,#32 |
| 496 |
fmov $IN01_0,x4 |
| 497 |
and x8,x8,#0x03ffffff |
| 498 |
and x9,x9,#0x03ffffff |
| 499 |
ubfx x10,x12,#14,#26 |
| 500 |
ubfx x11,x13,#14,#26 |
| 501 |
add x12,$padbit,x12,lsr#40 |
| 502 |
add x13,$padbit,x13,lsr#40 |
| 503 |
add x8,x8,x9,lsl#32 // bfi x8,x9,#32,#32 |
| 504 |
fmov $IN01_1,x6 |
| 505 |
add x10,x10,x11,lsl#32 // bfi x10,x11,#32,#32 |
| 506 |
add x12,x12,x13,lsl#32 // bfi x12,x13,#32,#32 |
| 507 |
fmov $IN01_2,x8 |
| 508 |
fmov $IN01_3,x10 |
| 509 |
fmov $IN01_4,x12 |
| 510 |
|
| 511 |
b.ls .Lskip_loop |
| 512 |
|
| 513 |
.align 4 |
| 514 |
.Loop_neon: |
| 515 |
//////////////////////////////////////////////////////////////// |
| 516 |
// ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2 |
| 517 |
// ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r |
| 518 |
// \___________________/ |
| 519 |
// ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2+inp[8])*r^2 |
| 520 |
// ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^4+inp[7]*r^2+inp[9])*r |
| 521 |
// \___________________/ \____________________/ |
| 522 |
// |
| 523 |
// Note that we start with inp[2:3]*r^2. This is because it |
| 524 |
// doesn't depend on reduction in previous iteration. |
| 525 |
//////////////////////////////////////////////////////////////// |
| 526 |
// d4 = h0*r4 + h1*r3 + h2*r2 + h3*r1 + h4*r0 |
| 527 |
// d3 = h0*r3 + h1*r2 + h2*r1 + h3*r0 + h4*5*r4 |
| 528 |
// d2 = h0*r2 + h1*r1 + h2*r0 + h3*5*r4 + h4*5*r3 |
| 529 |
// d1 = h0*r1 + h1*r0 + h2*5*r4 + h3*5*r3 + h4*5*r2 |
| 530 |
// d0 = h0*r0 + h1*5*r4 + h2*5*r3 + h3*5*r2 + h4*5*r1 |
| 531 |
|
| 532 |
subs $len,$len,#64 |
| 533 |
umull $ACC4,$IN23_0,${R4}[2] |
| 534 |
csel $in2,$zeros,$in2,lo |
| 535 |
umull $ACC3,$IN23_0,${R3}[2] |
| 536 |
umull $ACC2,$IN23_0,${R2}[2] |
| 537 |
ldp x8,x12,[$in2],#16 // inp[2:3] (or zero) |
| 538 |
umull $ACC1,$IN23_0,${R1}[2] |
| 539 |
ldp x9,x13,[$in2],#48 |
| 540 |
umull $ACC0,$IN23_0,${R0}[2] |
| 541 |
#ifdef __ARMEB__ |
| 542 |
rev x8,x8 |
| 543 |
rev x12,x12 |
| 544 |
rev x9,x9 |
| 545 |
rev x13,x13 |
| 546 |
#endif |
| 547 |
|
| 548 |
umlal $ACC4,$IN23_1,${R3}[2] |
| 549 |
and x4,x8,#0x03ffffff // base 2^64 -> base 2^26 |
| 550 |
umlal $ACC3,$IN23_1,${R2}[2] |
| 551 |
and x5,x9,#0x03ffffff |
| 552 |
umlal $ACC2,$IN23_1,${R1}[2] |
| 553 |
ubfx x6,x8,#26,#26 |
| 554 |
umlal $ACC1,$IN23_1,${R0}[2] |
| 555 |
ubfx x7,x9,#26,#26 |
| 556 |
umlal $ACC0,$IN23_1,${S4}[2] |
| 557 |
add x4,x4,x5,lsl#32 // bfi x4,x5,#32,#32 |
| 558 |
|
| 559 |
umlal $ACC4,$IN23_2,${R2}[2] |
| 560 |
extr x8,x12,x8,#52 |
| 561 |
umlal $ACC3,$IN23_2,${R1}[2] |
| 562 |
extr x9,x13,x9,#52 |
| 563 |
umlal $ACC2,$IN23_2,${R0}[2] |
| 564 |
add x6,x6,x7,lsl#32 // bfi x6,x7,#32,#32 |
| 565 |
umlal $ACC1,$IN23_2,${S4}[2] |
| 566 |
fmov $IN23_0,x4 |
| 567 |
umlal $ACC0,$IN23_2,${S3}[2] |
| 568 |
and x8,x8,#0x03ffffff |
| 569 |
|
| 570 |
umlal $ACC4,$IN23_3,${R1}[2] |
| 571 |
and x9,x9,#0x03ffffff |
| 572 |
umlal $ACC3,$IN23_3,${R0}[2] |
| 573 |
ubfx x10,x12,#14,#26 |
| 574 |
umlal $ACC2,$IN23_3,${S4}[2] |
| 575 |
ubfx x11,x13,#14,#26 |
| 576 |
umlal $ACC1,$IN23_3,${S3}[2] |
| 577 |
add x8,x8,x9,lsl#32 // bfi x8,x9,#32,#32 |
| 578 |
umlal $ACC0,$IN23_3,${S2}[2] |
| 579 |
fmov $IN23_1,x6 |
| 580 |
|
| 581 |
add $IN01_2,$IN01_2,$H2 |
| 582 |
add x12,$padbit,x12,lsr#40 |
| 583 |
umlal $ACC4,$IN23_4,${R0}[2] |
| 584 |
add x13,$padbit,x13,lsr#40 |
| 585 |
umlal $ACC3,$IN23_4,${S4}[2] |
| 586 |
add x10,x10,x11,lsl#32 // bfi x10,x11,#32,#32 |
| 587 |
umlal $ACC2,$IN23_4,${S3}[2] |
| 588 |
add x12,x12,x13,lsl#32 // bfi x12,x13,#32,#32 |
| 589 |
umlal $ACC1,$IN23_4,${S2}[2] |
| 590 |
fmov $IN23_2,x8 |
| 591 |
umlal $ACC0,$IN23_4,${S1}[2] |
| 592 |
fmov $IN23_3,x10 |
| 593 |
|
| 594 |
//////////////////////////////////////////////////////////////// |
| 595 |
// (hash+inp[0:1])*r^4 and accumulate |
| 596 |
|
| 597 |
add $IN01_0,$IN01_0,$H0 |
| 598 |
fmov $IN23_4,x12 |
| 599 |
umlal $ACC3,$IN01_2,${R1}[0] |
| 600 |
ldp x8,x12,[$inp],#16 // inp[0:1] |
| 601 |
umlal $ACC0,$IN01_2,${S3}[0] |
| 602 |
ldp x9,x13,[$inp],#48 |
| 603 |
umlal $ACC4,$IN01_2,${R2}[0] |
| 604 |
umlal $ACC1,$IN01_2,${S4}[0] |
| 605 |
umlal $ACC2,$IN01_2,${R0}[0] |
| 606 |
#ifdef __ARMEB__ |
| 607 |
rev x8,x8 |
| 608 |
rev x12,x12 |
| 609 |
rev x9,x9 |
| 610 |
rev x13,x13 |
| 611 |
#endif |
| 612 |
|
| 613 |
add $IN01_1,$IN01_1,$H1 |
| 614 |
umlal $ACC3,$IN01_0,${R3}[0] |
| 615 |
umlal $ACC4,$IN01_0,${R4}[0] |
| 616 |
and x4,x8,#0x03ffffff // base 2^64 -> base 2^26 |
| 617 |
umlal $ACC2,$IN01_0,${R2}[0] |
| 618 |
and x5,x9,#0x03ffffff |
| 619 |
umlal $ACC0,$IN01_0,${R0}[0] |
| 620 |
ubfx x6,x8,#26,#26 |
| 621 |
umlal $ACC1,$IN01_0,${R1}[0] |
| 622 |
ubfx x7,x9,#26,#26 |
| 623 |
|
| 624 |
add $IN01_3,$IN01_3,$H3 |
| 625 |
add x4,x4,x5,lsl#32 // bfi x4,x5,#32,#32 |
| 626 |
umlal $ACC3,$IN01_1,${R2}[0] |
| 627 |
extr x8,x12,x8,#52 |
| 628 |
umlal $ACC4,$IN01_1,${R3}[0] |
| 629 |
extr x9,x13,x9,#52 |
| 630 |
umlal $ACC0,$IN01_1,${S4}[0] |
| 631 |
add x6,x6,x7,lsl#32 // bfi x6,x7,#32,#32 |
| 632 |
umlal $ACC2,$IN01_1,${R1}[0] |
| 633 |
fmov $IN01_0,x4 |
| 634 |
umlal $ACC1,$IN01_1,${R0}[0] |
| 635 |
and x8,x8,#0x03ffffff |
| 636 |
|
| 637 |
add $IN01_4,$IN01_4,$H4 |
| 638 |
and x9,x9,#0x03ffffff |
| 639 |
umlal $ACC3,$IN01_3,${R0}[0] |
| 640 |
ubfx x10,x12,#14,#26 |
| 641 |
umlal $ACC0,$IN01_3,${S2}[0] |
| 642 |
ubfx x11,x13,#14,#26 |
| 643 |
umlal $ACC4,$IN01_3,${R1}[0] |
| 644 |
add x8,x8,x9,lsl#32 // bfi x8,x9,#32,#32 |
| 645 |
umlal $ACC1,$IN01_3,${S3}[0] |
| 646 |
fmov $IN01_1,x6 |
| 647 |
umlal $ACC2,$IN01_3,${S4}[0] |
| 648 |
add x12,$padbit,x12,lsr#40 |
| 649 |
|
| 650 |
umlal $ACC3,$IN01_4,${S4}[0] |
| 651 |
add x13,$padbit,x13,lsr#40 |
| 652 |
umlal $ACC0,$IN01_4,${S1}[0] |
| 653 |
add x10,x10,x11,lsl#32 // bfi x10,x11,#32,#32 |
| 654 |
umlal $ACC4,$IN01_4,${R0}[0] |
| 655 |
add x12,x12,x13,lsl#32 // bfi x12,x13,#32,#32 |
| 656 |
umlal $ACC1,$IN01_4,${S2}[0] |
| 657 |
fmov $IN01_2,x8 |
| 658 |
umlal $ACC2,$IN01_4,${S3}[0] |
| 659 |
fmov $IN01_3,x10 |
| 660 |
|
| 661 |
///////////////////////////////////////////////////////////////// |
| 662 |
// lazy reduction as discussed in "NEON crypto" by D.J. Bernstein |
| 663 |
// and P. Schwabe |
| 664 |
|
| 665 |
ushr $T0.2d,$ACC3,#26 |
| 666 |
fmov $IN01_4,x12 |
| 667 |
xtn $H3,$ACC3 |
| 668 |
ushr $T1.2d,$ACC0,#26 |
| 669 |
xtn $H0,$ACC0 |
| 670 |
add $ACC4,$ACC4,$T0.2d // h3 -> h4 |
| 671 |
bic $H3,#0xfc,lsl#24 // &=0x03ffffff |
| 672 |
add $ACC1,$ACC1,$T1.2d // h0 -> h1 |
| 673 |
bic $H0,#0xfc,lsl#24 |
| 674 |
|
| 675 |
shrn $T0.2s,$ACC4,#26 |
| 676 |
xtn $H4,$ACC4 |
| 677 |
ushr $T1.2d,$ACC1,#26 |
| 678 |
xtn $H1,$ACC1 |
| 679 |
add $ACC2,$ACC2,$T1.2d // h1 -> h2 |
| 680 |
bic $H4,#0xfc,lsl#24 |
| 681 |
bic $H1,#0xfc,lsl#24 |
| 682 |
|
| 683 |
add $H0,$H0,$T0.2s |
| 684 |
shl $T0.2s,$T0.2s,#2 |
| 685 |
shrn $T1.2s,$ACC2,#26 |
| 686 |
xtn $H2,$ACC2 |
| 687 |
add $H0,$H0,$T0.2s // h4 -> h0 |
| 688 |
add $H3,$H3,$T1.2s // h2 -> h3 |
| 689 |
bic $H2,#0xfc,lsl#24 |
| 690 |
|
| 691 |
ushr $T0.2s,$H0,#26 |
| 692 |
bic $H0,#0xfc,lsl#24 |
| 693 |
ushr $T1.2s,$H3,#26 |
| 694 |
bic $H3,#0xfc,lsl#24 |
| 695 |
add $H1,$H1,$T0.2s // h0 -> h1 |
| 696 |
add $H4,$H4,$T1.2s // h3 -> h4 |
| 697 |
|
| 698 |
b.hi .Loop_neon |
| 699 |
|
| 700 |
.Lskip_loop: |
| 701 |
dup $IN23_2,${IN23_2}[0] |
| 702 |
movi $MASK.2d,#-1 |
| 703 |
add $IN01_2,$IN01_2,$H2 |
| 704 |
ushr $MASK.2d,$MASK.2d,#38 |
| 705 |
|
| 706 |
//////////////////////////////////////////////////////////////// |
| 707 |
// multiply (inp[0:1]+hash) or inp[2:3] by r^2:r^1 |
| 708 |
|
| 709 |
adds $len,$len,#32 |
| 710 |
b.ne .Long_tail |
| 711 |
|
| 712 |
dup $IN23_2,${IN01_2}[0] |
| 713 |
add $IN23_0,$IN01_0,$H0 |
| 714 |
add $IN23_3,$IN01_3,$H3 |
| 715 |
add $IN23_1,$IN01_1,$H1 |
| 716 |
add $IN23_4,$IN01_4,$H4 |
| 717 |
|
| 718 |
.Long_tail: |
| 719 |
dup $IN23_0,${IN23_0}[0] |
| 720 |
umull2 $ACC0,$IN23_2,${S3} |
| 721 |
umull2 $ACC3,$IN23_2,${R1} |
| 722 |
umull2 $ACC4,$IN23_2,${R2} |
| 723 |
umull2 $ACC2,$IN23_2,${R0} |
| 724 |
umull2 $ACC1,$IN23_2,${S4} |
| 725 |
|
| 726 |
dup $IN23_1,${IN23_1}[0] |
| 727 |
umlal2 $ACC0,$IN23_0,${R0} |
| 728 |
umlal2 $ACC2,$IN23_0,${R2} |
| 729 |
umlal2 $ACC3,$IN23_0,${R3} |
| 730 |
umlal2 $ACC4,$IN23_0,${R4} |
| 731 |
umlal2 $ACC1,$IN23_0,${R1} |
| 732 |
|
| 733 |
dup $IN23_3,${IN23_3}[0] |
| 734 |
umlal2 $ACC0,$IN23_1,${S4} |
| 735 |
umlal2 $ACC3,$IN23_1,${R2} |
| 736 |
umlal2 $ACC2,$IN23_1,${R1} |
| 737 |
umlal2 $ACC4,$IN23_1,${R3} |
| 738 |
umlal2 $ACC1,$IN23_1,${R0} |
| 739 |
|
| 740 |
dup $IN23_4,${IN23_4}[0] |
| 741 |
umlal2 $ACC3,$IN23_3,${R0} |
| 742 |
umlal2 $ACC4,$IN23_3,${R1} |
| 743 |
umlal2 $ACC0,$IN23_3,${S2} |
| 744 |
umlal2 $ACC1,$IN23_3,${S3} |
| 745 |
umlal2 $ACC2,$IN23_3,${S4} |
| 746 |
|
| 747 |
umlal2 $ACC3,$IN23_4,${S4} |
| 748 |
umlal2 $ACC0,$IN23_4,${S1} |
| 749 |
umlal2 $ACC4,$IN23_4,${R0} |
| 750 |
umlal2 $ACC1,$IN23_4,${S2} |
| 751 |
umlal2 $ACC2,$IN23_4,${S3} |
| 752 |
|
| 753 |
b.eq .Lshort_tail |
| 754 |
|
| 755 |
//////////////////////////////////////////////////////////////// |
| 756 |
// (hash+inp[0:1])*r^4:r^3 and accumulate |
| 757 |
|
| 758 |
add $IN01_0,$IN01_0,$H0 |
| 759 |
umlal $ACC3,$IN01_2,${R1} |
| 760 |
umlal $ACC0,$IN01_2,${S3} |
| 761 |
umlal $ACC4,$IN01_2,${R2} |
| 762 |
umlal $ACC1,$IN01_2,${S4} |
| 763 |
umlal $ACC2,$IN01_2,${R0} |
| 764 |
|
| 765 |
add $IN01_1,$IN01_1,$H1 |
| 766 |
umlal $ACC3,$IN01_0,${R3} |
| 767 |
umlal $ACC0,$IN01_0,${R0} |
| 768 |
umlal $ACC4,$IN01_0,${R4} |
| 769 |
umlal $ACC1,$IN01_0,${R1} |
| 770 |
umlal $ACC2,$IN01_0,${R2} |
| 771 |
|
| 772 |
add $IN01_3,$IN01_3,$H3 |
| 773 |
umlal $ACC3,$IN01_1,${R2} |
| 774 |
umlal $ACC0,$IN01_1,${S4} |
| 775 |
umlal $ACC4,$IN01_1,${R3} |
| 776 |
umlal $ACC1,$IN01_1,${R0} |
| 777 |
umlal $ACC2,$IN01_1,${R1} |
| 778 |
|
| 779 |
add $IN01_4,$IN01_4,$H4 |
| 780 |
umlal $ACC3,$IN01_3,${R0} |
| 781 |
umlal $ACC0,$IN01_3,${S2} |
| 782 |
umlal $ACC4,$IN01_3,${R1} |
| 783 |
umlal $ACC1,$IN01_3,${S3} |
| 784 |
umlal $ACC2,$IN01_3,${S4} |
| 785 |
|
| 786 |
umlal $ACC3,$IN01_4,${S4} |
| 787 |
umlal $ACC0,$IN01_4,${S1} |
| 788 |
umlal $ACC4,$IN01_4,${R0} |
| 789 |
umlal $ACC1,$IN01_4,${S2} |
| 790 |
umlal $ACC2,$IN01_4,${S3} |
| 791 |
|
| 792 |
.Lshort_tail: |
| 793 |
//////////////////////////////////////////////////////////////// |
| 794 |
// horizontal add |
| 795 |
|
| 796 |
addp $ACC3,$ACC3,$ACC3 |
| 797 |
ldp d8,d9,[sp,#16] // meet ABI requirements |
| 798 |
addp $ACC0,$ACC0,$ACC0 |
| 799 |
ldp d10,d11,[sp,#32] |
| 800 |
addp $ACC4,$ACC4,$ACC4 |
| 801 |
ldp d12,d13,[sp,#48] |
| 802 |
addp $ACC1,$ACC1,$ACC1 |
| 803 |
ldp d14,d15,[sp,#64] |
| 804 |
addp $ACC2,$ACC2,$ACC2 |
| 805 |
|
| 806 |
//////////////////////////////////////////////////////////////// |
| 807 |
// lazy reduction, but without narrowing |
| 808 |
|
| 809 |
ushr $T0.2d,$ACC3,#26 |
| 810 |
and $ACC3,$ACC3,$MASK.2d |
| 811 |
ushr $T1.2d,$ACC0,#26 |
| 812 |
and $ACC0,$ACC0,$MASK.2d |
| 813 |
|
| 814 |
add $ACC4,$ACC4,$T0.2d // h3 -> h4 |
| 815 |
add $ACC1,$ACC1,$T1.2d // h0 -> h1 |
| 816 |
|
| 817 |
ushr $T0.2d,$ACC4,#26 |
| 818 |
and $ACC4,$ACC4,$MASK.2d |
| 819 |
ushr $T1.2d,$ACC1,#26 |
| 820 |
and $ACC1,$ACC1,$MASK.2d |
| 821 |
add $ACC2,$ACC2,$T1.2d // h1 -> h2 |
| 822 |
|
| 823 |
add $ACC0,$ACC0,$T0.2d |
| 824 |
shl $T0.2d,$T0.2d,#2 |
| 825 |
ushr $T1.2d,$ACC2,#26 |
| 826 |
and $ACC2,$ACC2,$MASK.2d |
| 827 |
add $ACC0,$ACC0,$T0.2d // h4 -> h0 |
| 828 |
add $ACC3,$ACC3,$T1.2d // h2 -> h3 |
| 829 |
|
| 830 |
ushr $T0.2d,$ACC0,#26 |
| 831 |
and $ACC0,$ACC0,$MASK.2d |
| 832 |
ushr $T1.2d,$ACC3,#26 |
| 833 |
and $ACC3,$ACC3,$MASK.2d |
| 834 |
add $ACC1,$ACC1,$T0.2d // h0 -> h1 |
| 835 |
add $ACC4,$ACC4,$T1.2d // h3 -> h4 |
| 836 |
|
| 837 |
//////////////////////////////////////////////////////////////// |
| 838 |
// write the result, can be partially reduced |
| 839 |
|
| 840 |
st4 {$ACC0,$ACC1,$ACC2,$ACC3}[0],[$ctx],#16 |
| 841 |
st1 {$ACC4}[0],[$ctx] |
| 842 |
|
| 843 |
.Lno_data_neon: |
| 844 |
ldr x29,[sp],#80 |
| 845 |
ret |
| 846 |
.size poly1305_blocks_neon,.-poly1305_blocks_neon |
| 847 |
|
| 848 |
.type poly1305_emit_neon,%function |
| 849 |
.align 5 |
| 850 |
poly1305_emit_neon: |
| 851 |
ldr $is_base2_26,[$ctx,#24] |
| 852 |
cbz $is_base2_26,poly1305_emit |
| 853 |
|
| 854 |
ldp w10,w11,[$ctx] // load hash value base 2^26 |
| 855 |
ldp w12,w13,[$ctx,#8] |
| 856 |
ldr w14,[$ctx,#16] |
| 857 |
|
| 858 |
add $h0,x10,x11,lsl#26 // base 2^26 -> base 2^64 |
| 859 |
lsr $h1,x12,#12 |
| 860 |
adds $h0,$h0,x12,lsl#52 |
| 861 |
add $h1,$h1,x13,lsl#14 |
| 862 |
adc $h1,$h1,xzr |
| 863 |
lsr $h2,x14,#24 |
| 864 |
adds $h1,$h1,x14,lsl#40 |
| 865 |
adc $h2,$h2,xzr // can be partially reduced... |
| 866 |
|
| 867 |
ldp $t0,$t1,[$nonce] // load nonce |
| 868 |
|
| 869 |
and $d0,$h2,#-4 // ... so reduce |
| 870 |
add $d0,$d0,$h2,lsr#2 |
| 871 |
and $h2,$h2,#3 |
| 872 |
adds $h0,$h0,$d0 |
| 873 |
adc $h1,$h1,xzr |
| 874 |
|
| 875 |
adds $d0,$h0,#5 // compare to modulus |
| 876 |
adcs $d1,$h1,xzr |
| 877 |
adc $d2,$h2,xzr |
| 878 |
|
| 879 |
tst $d2,#-4 // see if it's carried/borrowed |
| 880 |
|
| 881 |
csel $h0,$h0,$d0,eq |
| 882 |
csel $h1,$h1,$d1,eq |
| 883 |
|
| 884 |
#ifdef __ARMEB__ |
| 885 |
ror $t0,$t0,#32 // flip nonce words |
| 886 |
ror $t1,$t1,#32 |
| 887 |
#endif |
| 888 |
adds $h0,$h0,$t0 // accumulate nonce |
| 889 |
adc $h1,$h1,$t1 |
| 890 |
#ifdef __ARMEB__ |
| 891 |
rev $h0,$h0 // flip output bytes |
| 892 |
rev $h1,$h1 |
| 893 |
#endif |
| 894 |
stp $h0,$h1,[$mac] // write result |
| 895 |
|
| 896 |
ret |
| 897 |
.size poly1305_emit_neon,.-poly1305_emit_neon |
| 898 |
|
| 899 |
.align 5 |
| 900 |
.Lzeros: |
| 901 |
.long 0,0,0,0,0,0,0,0 |
| 902 |
.LOPENSSL_armcap_P: |
| 903 |
#ifdef __ILP32__ |
| 904 |
.long OPENSSL_armcap_P-. |
| 905 |
#else |
| 906 |
.quad OPENSSL_armcap_P-. |
| 907 |
#endif |
| 908 |
.asciz "Poly1305 for ARMv8, CRYPTOGAMS by <appro\@openssl.org>" |
| 909 |
.align 2 |
| 910 |
___ |
| 911 |
|
| 912 |
foreach (split("\n",$code)) { |
| 913 |
s/\b(shrn\s+v[0-9]+)\.[24]d/$1.2s/ or |
| 914 |
s/\b(fmov\s+)v([0-9]+)[^,]*,\s*x([0-9]+)/$1d$2,x$3/ or |
| 915 |
(m/\bdup\b/ and (s/\.[24]s/.2d/g or 1)) or |
| 916 |
(m/\b(eor|and)/ and (s/\.[248][sdh]/.16b/g or 1)) or |
| 917 |
(m/\bum(ul|la)l\b/ and (s/\.4s/.2s/g or 1)) or |
| 918 |
(m/\bum(ul|la)l2\b/ and (s/\.2s/.4s/g or 1)) or |
| 919 |
(m/\bst[1-4]\s+{[^}]+}\[/ and (s/\.[24]d/.s/g or 1)); |
| 920 |
|
| 921 |
s/\.[124]([sd])\[/.$1\[/; |
| 922 |
|
| 923 |
print $_,"\n"; |
| 924 |
} |
| 925 |
close STDOUT; |