| 1 |
#!/usr/bin/env perl |
| 2 |
|
| 3 |
# ==================================================================== |
| 4 |
# Written by Andy Polyakov <[email protected]> for the OpenSSL |
| 5 |
# project. The module is, however, dual licensed under OpenSSL and |
| 6 |
# CRYPTOGAMS licenses depending on where you obtain it. For further |
| 7 |
# details see http://www.openssl.org/~appro/cryptogams/. |
| 8 |
# |
| 9 |
# Specific modes and adaptation for Linux kernel by Ard Biesheuvel |
| 10 |
# <[email protected]>. Permission to use under GPL terms is |
| 11 |
# granted. |
| 12 |
# ==================================================================== |
| 13 |
|
| 14 |
# Bit-sliced AES for ARM NEON |
| 15 |
# |
| 16 |
# February 2012. |
| 17 |
# |
| 18 |
# This implementation is direct adaptation of bsaes-x86_64 module for |
| 19 |
# ARM NEON. Except that this module is endian-neutral [in sense that |
| 20 |
# it can be compiled for either endianness] by courtesy of vld1.8's |
| 21 |
# neutrality. Initial version doesn't implement interface to OpenSSL, |
| 22 |
# only low-level primitives and unsupported entry points, just enough |
| 23 |
# to collect performance results, which for Cortex-A8 core are: |
| 24 |
# |
| 25 |
# encrypt 19.5 cycles per byte processed with 128-bit key |
| 26 |
# decrypt 22.1 cycles per byte processed with 128-bit key |
| 27 |
# key conv. 440 cycles per 128-bit key/0.18 of 8x block |
| 28 |
# |
| 29 |
# Snapdragon S4 encrypts byte in 17.6 cycles and decrypts in 19.7, |
| 30 |
# which is [much] worse than anticipated (for further details see |
| 31 |
# http://www.openssl.org/~appro/Snapdragon-S4.html). |
| 32 |
# |
| 33 |
# Cortex-A15 manages in 14.2/16.1 cycles [when integer-only code |
| 34 |
# manages in 20.0 cycles]. |
| 35 |
# |
| 36 |
# When comparing to x86_64 results keep in mind that NEON unit is |
| 37 |
# [mostly] single-issue and thus can't [fully] benefit from |
| 38 |
# instruction-level parallelism. And when comparing to aes-armv4 |
| 39 |
# results keep in mind key schedule conversion overhead (see |
| 40 |
# bsaes-x86_64.pl for further details)... |
| 41 |
# |
| 42 |
# <[email protected]> |
| 43 |
|
| 44 |
# April-August 2013 |
| 45 |
# |
| 46 |
# Add CBC, CTR and XTS subroutines, adapt for kernel use. |
| 47 |
# |
| 48 |
# <[email protected]> |
| 49 |
|
| 50 |
$flavour = shift; |
| 51 |
if ($flavour=~/^\w[\w\-]*\.\w+$/) { $output=$flavour; undef $flavour; } |
| 52 |
else { while (($output=shift) && ($output!~/^\w[\w\-]*\.\w+$/)) {} } |
| 53 |
|
| 54 |
if ($flavour && $flavour ne "void") { |
| 55 |
$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; |
| 56 |
( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or |
| 57 |
( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or |
| 58 |
die "can't locate arm-xlate.pl"; |
| 59 |
|
| 60 |
open STDOUT,"| \"$^X\" $xlate $flavour $output"; |
| 61 |
} else { |
| 62 |
open STDOUT,">$output"; |
| 63 |
} |
| 64 |
|
| 65 |
my ($inp,$out,$len,$key)=("r0","r1","r2","r3"); |
| 66 |
my @XMM=map("q$_",(0..15)); |
| 67 |
|
| 68 |
{ |
| 69 |
my ($key,$rounds,$const)=("r4","r5","r6"); |
| 70 |
|
| 71 |
sub Dlo() { shift=~m|q([1]?[0-9])|?"d".($1*2):""; } |
| 72 |
sub Dhi() { shift=~m|q([1]?[0-9])|?"d".($1*2+1):""; } |
| 73 |
|
| 74 |
sub Sbox { |
| 75 |
# input in lsb > [b0, b1, b2, b3, b4, b5, b6, b7] < msb |
| 76 |
# output in lsb > [b0, b1, b4, b6, b3, b7, b2, b5] < msb |
| 77 |
my @b=@_[0..7]; |
| 78 |
my @t=@_[8..11]; |
| 79 |
my @s=@_[12..15]; |
| 80 |
&InBasisChange (@b); |
| 81 |
&Inv_GF256 (@b[6,5,0,3,7,1,4,2],@t,@s); |
| 82 |
&OutBasisChange (@b[7,1,4,2,6,5,0,3]); |
| 83 |
} |
| 84 |
|
| 85 |
sub InBasisChange { |
| 86 |
# input in lsb > [b0, b1, b2, b3, b4, b5, b6, b7] < msb |
| 87 |
# output in lsb > [b6, b5, b0, b3, b7, b1, b4, b2] < msb |
| 88 |
my @b=@_[0..7]; |
| 89 |
$code.=<<___; |
| 90 |
veor @b[2], @b[2], @b[1] |
| 91 |
veor @b[5], @b[5], @b[6] |
| 92 |
veor @b[3], @b[3], @b[0] |
| 93 |
veor @b[6], @b[6], @b[2] |
| 94 |
veor @b[5], @b[5], @b[0] |
| 95 |
|
| 96 |
veor @b[6], @b[6], @b[3] |
| 97 |
veor @b[3], @b[3], @b[7] |
| 98 |
veor @b[7], @b[7], @b[5] |
| 99 |
veor @b[3], @b[3], @b[4] |
| 100 |
veor @b[4], @b[4], @b[5] |
| 101 |
|
| 102 |
veor @b[2], @b[2], @b[7] |
| 103 |
veor @b[3], @b[3], @b[1] |
| 104 |
veor @b[1], @b[1], @b[5] |
| 105 |
___ |
| 106 |
} |
| 107 |
|
| 108 |
sub OutBasisChange { |
| 109 |
# input in lsb > [b0, b1, b2, b3, b4, b5, b6, b7] < msb |
| 110 |
# output in lsb > [b6, b1, b2, b4, b7, b0, b3, b5] < msb |
| 111 |
my @b=@_[0..7]; |
| 112 |
$code.=<<___; |
| 113 |
veor @b[0], @b[0], @b[6] |
| 114 |
veor @b[1], @b[1], @b[4] |
| 115 |
veor @b[4], @b[4], @b[6] |
| 116 |
veor @b[2], @b[2], @b[0] |
| 117 |
veor @b[6], @b[6], @b[1] |
| 118 |
|
| 119 |
veor @b[1], @b[1], @b[5] |
| 120 |
veor @b[5], @b[5], @b[3] |
| 121 |
veor @b[3], @b[3], @b[7] |
| 122 |
veor @b[7], @b[7], @b[5] |
| 123 |
veor @b[2], @b[2], @b[5] |
| 124 |
|
| 125 |
veor @b[4], @b[4], @b[7] |
| 126 |
___ |
| 127 |
} |
| 128 |
|
| 129 |
sub InvSbox { |
| 130 |
# input in lsb > [b0, b1, b2, b3, b4, b5, b6, b7] < msb |
| 131 |
# output in lsb > [b0, b1, b6, b4, b2, b7, b3, b5] < msb |
| 132 |
my @b=@_[0..7]; |
| 133 |
my @t=@_[8..11]; |
| 134 |
my @s=@_[12..15]; |
| 135 |
&InvInBasisChange (@b); |
| 136 |
&Inv_GF256 (@b[5,1,2,6,3,7,0,4],@t,@s); |
| 137 |
&InvOutBasisChange (@b[3,7,0,4,5,1,2,6]); |
| 138 |
} |
| 139 |
|
| 140 |
sub InvInBasisChange { # OutBasisChange in reverse (with twist) |
| 141 |
my @b=@_[5,1,2,6,3,7,0,4]; |
| 142 |
$code.=<<___ |
| 143 |
veor @b[1], @b[1], @b[7] |
| 144 |
veor @b[4], @b[4], @b[7] |
| 145 |
|
| 146 |
veor @b[7], @b[7], @b[5] |
| 147 |
veor @b[1], @b[1], @b[3] |
| 148 |
veor @b[2], @b[2], @b[5] |
| 149 |
veor @b[3], @b[3], @b[7] |
| 150 |
|
| 151 |
veor @b[6], @b[6], @b[1] |
| 152 |
veor @b[2], @b[2], @b[0] |
| 153 |
veor @b[5], @b[5], @b[3] |
| 154 |
veor @b[4], @b[4], @b[6] |
| 155 |
veor @b[0], @b[0], @b[6] |
| 156 |
veor @b[1], @b[1], @b[4] |
| 157 |
___ |
| 158 |
} |
| 159 |
|
| 160 |
sub InvOutBasisChange { # InBasisChange in reverse |
| 161 |
my @b=@_[2,5,7,3,6,1,0,4]; |
| 162 |
$code.=<<___; |
| 163 |
veor @b[1], @b[1], @b[5] |
| 164 |
veor @b[2], @b[2], @b[7] |
| 165 |
|
| 166 |
veor @b[3], @b[3], @b[1] |
| 167 |
veor @b[4], @b[4], @b[5] |
| 168 |
veor @b[7], @b[7], @b[5] |
| 169 |
veor @b[3], @b[3], @b[4] |
| 170 |
veor @b[5], @b[5], @b[0] |
| 171 |
veor @b[3], @b[3], @b[7] |
| 172 |
veor @b[6], @b[6], @b[2] |
| 173 |
veor @b[2], @b[2], @b[1] |
| 174 |
veor @b[6], @b[6], @b[3] |
| 175 |
|
| 176 |
veor @b[3], @b[3], @b[0] |
| 177 |
veor @b[5], @b[5], @b[6] |
| 178 |
___ |
| 179 |
} |
| 180 |
|
| 181 |
sub Mul_GF4 { |
| 182 |
#;************************************************************* |
| 183 |
#;* Mul_GF4: Input x0-x1,y0-y1 Output x0-x1 Temp t0 (8) * |
| 184 |
#;************************************************************* |
| 185 |
my ($x0,$x1,$y0,$y1,$t0,$t1)=@_; |
| 186 |
$code.=<<___; |
| 187 |
veor $t0, $y0, $y1 |
| 188 |
vand $t0, $t0, $x0 |
| 189 |
veor $x0, $x0, $x1 |
| 190 |
vand $t1, $x1, $y0 |
| 191 |
vand $x0, $x0, $y1 |
| 192 |
veor $x1, $t1, $t0 |
| 193 |
veor $x0, $x0, $t1 |
| 194 |
___ |
| 195 |
} |
| 196 |
|
| 197 |
sub Mul_GF4_N { # not used, see next subroutine |
| 198 |
# multiply and scale by N |
| 199 |
my ($x0,$x1,$y0,$y1,$t0)=@_; |
| 200 |
$code.=<<___; |
| 201 |
veor $t0, $y0, $y1 |
| 202 |
vand $t0, $t0, $x0 |
| 203 |
veor $x0, $x0, $x1 |
| 204 |
vand $x1, $x1, $y0 |
| 205 |
vand $x0, $x0, $y1 |
| 206 |
veor $x1, $x1, $x0 |
| 207 |
veor $x0, $x0, $t0 |
| 208 |
___ |
| 209 |
} |
| 210 |
|
| 211 |
sub Mul_GF4_N_GF4 { |
| 212 |
# interleaved Mul_GF4_N and Mul_GF4 |
| 213 |
my ($x0,$x1,$y0,$y1,$t0, |
| 214 |
$x2,$x3,$y2,$y3,$t1)=@_; |
| 215 |
$code.=<<___; |
| 216 |
veor $t0, $y0, $y1 |
| 217 |
veor $t1, $y2, $y3 |
| 218 |
vand $t0, $t0, $x0 |
| 219 |
vand $t1, $t1, $x2 |
| 220 |
veor $x0, $x0, $x1 |
| 221 |
veor $x2, $x2, $x3 |
| 222 |
vand $x1, $x1, $y0 |
| 223 |
vand $x3, $x3, $y2 |
| 224 |
vand $x0, $x0, $y1 |
| 225 |
vand $x2, $x2, $y3 |
| 226 |
veor $x1, $x1, $x0 |
| 227 |
veor $x2, $x2, $x3 |
| 228 |
veor $x0, $x0, $t0 |
| 229 |
veor $x3, $x3, $t1 |
| 230 |
___ |
| 231 |
} |
| 232 |
sub Mul_GF16_2 { |
| 233 |
my @x=@_[0..7]; |
| 234 |
my @y=@_[8..11]; |
| 235 |
my @t=@_[12..15]; |
| 236 |
$code.=<<___; |
| 237 |
veor @t[0], @x[0], @x[2] |
| 238 |
veor @t[1], @x[1], @x[3] |
| 239 |
___ |
| 240 |
&Mul_GF4 (@x[0], @x[1], @y[0], @y[1], @t[2..3]); |
| 241 |
$code.=<<___; |
| 242 |
veor @y[0], @y[0], @y[2] |
| 243 |
veor @y[1], @y[1], @y[3] |
| 244 |
___ |
| 245 |
Mul_GF4_N_GF4 (@t[0], @t[1], @y[0], @y[1], @t[3], |
| 246 |
@x[2], @x[3], @y[2], @y[3], @t[2]); |
| 247 |
$code.=<<___; |
| 248 |
veor @x[0], @x[0], @t[0] |
| 249 |
veor @x[2], @x[2], @t[0] |
| 250 |
veor @x[1], @x[1], @t[1] |
| 251 |
veor @x[3], @x[3], @t[1] |
| 252 |
|
| 253 |
veor @t[0], @x[4], @x[6] |
| 254 |
veor @t[1], @x[5], @x[7] |
| 255 |
___ |
| 256 |
&Mul_GF4_N_GF4 (@t[0], @t[1], @y[0], @y[1], @t[3], |
| 257 |
@x[6], @x[7], @y[2], @y[3], @t[2]); |
| 258 |
$code.=<<___; |
| 259 |
veor @y[0], @y[0], @y[2] |
| 260 |
veor @y[1], @y[1], @y[3] |
| 261 |
___ |
| 262 |
&Mul_GF4 (@x[4], @x[5], @y[0], @y[1], @t[2..3]); |
| 263 |
$code.=<<___; |
| 264 |
veor @x[4], @x[4], @t[0] |
| 265 |
veor @x[6], @x[6], @t[0] |
| 266 |
veor @x[5], @x[5], @t[1] |
| 267 |
veor @x[7], @x[7], @t[1] |
| 268 |
___ |
| 269 |
} |
| 270 |
sub Inv_GF256 { |
| 271 |
#;******************************************************************** |
| 272 |
#;* Inv_GF256: Input x0-x7 Output x0-x7 Temp t0-t3,s0-s3 (144) * |
| 273 |
#;******************************************************************** |
| 274 |
my @x=@_[0..7]; |
| 275 |
my @t=@_[8..11]; |
| 276 |
my @s=@_[12..15]; |
| 277 |
# direct optimizations from hardware |
| 278 |
$code.=<<___; |
| 279 |
veor @t[3], @x[4], @x[6] |
| 280 |
veor @t[2], @x[5], @x[7] |
| 281 |
veor @t[1], @x[1], @x[3] |
| 282 |
veor @s[1], @x[7], @x[6] |
| 283 |
vmov @t[0], @t[2] |
| 284 |
veor @s[0], @x[0], @x[2] |
| 285 |
|
| 286 |
vorr @t[2], @t[2], @t[1] |
| 287 |
veor @s[3], @t[3], @t[0] |
| 288 |
vand @s[2], @t[3], @s[0] |
| 289 |
vorr @t[3], @t[3], @s[0] |
| 290 |
veor @s[0], @s[0], @t[1] |
| 291 |
vand @t[0], @t[0], @t[1] |
| 292 |
veor @t[1], @x[3], @x[2] |
| 293 |
vand @s[3], @s[3], @s[0] |
| 294 |
vand @s[1], @s[1], @t[1] |
| 295 |
veor @t[1], @x[4], @x[5] |
| 296 |
veor @s[0], @x[1], @x[0] |
| 297 |
veor @t[3], @t[3], @s[1] |
| 298 |
veor @t[2], @t[2], @s[1] |
| 299 |
vand @s[1], @t[1], @s[0] |
| 300 |
vorr @t[1], @t[1], @s[0] |
| 301 |
veor @t[3], @t[3], @s[3] |
| 302 |
veor @t[0], @t[0], @s[1] |
| 303 |
veor @t[2], @t[2], @s[2] |
| 304 |
veor @t[1], @t[1], @s[3] |
| 305 |
veor @t[0], @t[0], @s[2] |
| 306 |
vand @s[0], @x[7], @x[3] |
| 307 |
veor @t[1], @t[1], @s[2] |
| 308 |
vand @s[1], @x[6], @x[2] |
| 309 |
vand @s[2], @x[5], @x[1] |
| 310 |
vorr @s[3], @x[4], @x[0] |
| 311 |
veor @t[3], @t[3], @s[0] |
| 312 |
veor @t[1], @t[1], @s[2] |
| 313 |
veor @t[0], @t[0], @s[3] |
| 314 |
veor @t[2], @t[2], @s[1] |
| 315 |
|
| 316 |
@ Inv_GF16 \t0, \t1, \t2, \t3, \s0, \s1, \s2, \s3 |
| 317 |
|
| 318 |
@ new smaller inversion |
| 319 |
|
| 320 |
vand @s[2], @t[3], @t[1] |
| 321 |
vmov @s[0], @t[0] |
| 322 |
|
| 323 |
veor @s[1], @t[2], @s[2] |
| 324 |
veor @s[3], @t[0], @s[2] |
| 325 |
veor @s[2], @t[0], @s[2] @ @s[2]=@s[3] |
| 326 |
|
| 327 |
vbsl @s[1], @t[1], @t[0] |
| 328 |
vbsl @s[3], @t[3], @t[2] |
| 329 |
veor @t[3], @t[3], @t[2] |
| 330 |
|
| 331 |
vbsl @s[0], @s[1], @s[2] |
| 332 |
vbsl @t[0], @s[2], @s[1] |
| 333 |
|
| 334 |
vand @s[2], @s[0], @s[3] |
| 335 |
veor @t[1], @t[1], @t[0] |
| 336 |
|
| 337 |
veor @s[2], @s[2], @t[3] |
| 338 |
___ |
| 339 |
# output in s3, s2, s1, t1 |
| 340 |
|
| 341 |
# Mul_GF16_2 \x0, \x1, \x2, \x3, \x4, \x5, \x6, \x7, \t2, \t3, \t0, \t1, \s0, \s1, \s2, \s3 |
| 342 |
|
| 343 |
# Mul_GF16_2 \x0, \x1, \x2, \x3, \x4, \x5, \x6, \x7, \s3, \s2, \s1, \t1, \s0, \t0, \t2, \t3 |
| 344 |
&Mul_GF16_2(@x,@s[3,2,1],@t[1],@s[0],@t[0,2,3]); |
| 345 |
|
| 346 |
### output msb > [x3,x2,x1,x0,x7,x6,x5,x4] < lsb |
| 347 |
} |
| 348 |
|
| 349 |
# AES linear components |
| 350 |
|
| 351 |
sub ShiftRows { |
| 352 |
my @x=@_[0..7]; |
| 353 |
my @t=@_[8..11]; |
| 354 |
my $mask=pop; |
| 355 |
$code.=<<___; |
| 356 |
vldmia $key!, {@t[0]-@t[3]} |
| 357 |
veor @t[0], @t[0], @x[0] |
| 358 |
veor @t[1], @t[1], @x[1] |
| 359 |
vtbl.8 `&Dlo(@x[0])`, {@t[0]}, `&Dlo($mask)` |
| 360 |
vtbl.8 `&Dhi(@x[0])`, {@t[0]}, `&Dhi($mask)` |
| 361 |
vldmia $key!, {@t[0]} |
| 362 |
veor @t[2], @t[2], @x[2] |
| 363 |
vtbl.8 `&Dlo(@x[1])`, {@t[1]}, `&Dlo($mask)` |
| 364 |
vtbl.8 `&Dhi(@x[1])`, {@t[1]}, `&Dhi($mask)` |
| 365 |
vldmia $key!, {@t[1]} |
| 366 |
veor @t[3], @t[3], @x[3] |
| 367 |
vtbl.8 `&Dlo(@x[2])`, {@t[2]}, `&Dlo($mask)` |
| 368 |
vtbl.8 `&Dhi(@x[2])`, {@t[2]}, `&Dhi($mask)` |
| 369 |
vldmia $key!, {@t[2]} |
| 370 |
vtbl.8 `&Dlo(@x[3])`, {@t[3]}, `&Dlo($mask)` |
| 371 |
vtbl.8 `&Dhi(@x[3])`, {@t[3]}, `&Dhi($mask)` |
| 372 |
vldmia $key!, {@t[3]} |
| 373 |
veor @t[0], @t[0], @x[4] |
| 374 |
veor @t[1], @t[1], @x[5] |
| 375 |
vtbl.8 `&Dlo(@x[4])`, {@t[0]}, `&Dlo($mask)` |
| 376 |
vtbl.8 `&Dhi(@x[4])`, {@t[0]}, `&Dhi($mask)` |
| 377 |
veor @t[2], @t[2], @x[6] |
| 378 |
vtbl.8 `&Dlo(@x[5])`, {@t[1]}, `&Dlo($mask)` |
| 379 |
vtbl.8 `&Dhi(@x[5])`, {@t[1]}, `&Dhi($mask)` |
| 380 |
veor @t[3], @t[3], @x[7] |
| 381 |
vtbl.8 `&Dlo(@x[6])`, {@t[2]}, `&Dlo($mask)` |
| 382 |
vtbl.8 `&Dhi(@x[6])`, {@t[2]}, `&Dhi($mask)` |
| 383 |
vtbl.8 `&Dlo(@x[7])`, {@t[3]}, `&Dlo($mask)` |
| 384 |
vtbl.8 `&Dhi(@x[7])`, {@t[3]}, `&Dhi($mask)` |
| 385 |
___ |
| 386 |
} |
| 387 |
|
| 388 |
sub MixColumns { |
| 389 |
# modified to emit output in order suitable for feeding back to aesenc[last] |
| 390 |
my @x=@_[0..7]; |
| 391 |
my @t=@_[8..15]; |
| 392 |
my $inv=@_[16]; # optional |
| 393 |
$code.=<<___; |
| 394 |
vext.8 @t[0], @x[0], @x[0], #12 @ x0 <<< 32 |
| 395 |
vext.8 @t[1], @x[1], @x[1], #12 |
| 396 |
veor @x[0], @x[0], @t[0] @ x0 ^ (x0 <<< 32) |
| 397 |
vext.8 @t[2], @x[2], @x[2], #12 |
| 398 |
veor @x[1], @x[1], @t[1] |
| 399 |
vext.8 @t[3], @x[3], @x[3], #12 |
| 400 |
veor @x[2], @x[2], @t[2] |
| 401 |
vext.8 @t[4], @x[4], @x[4], #12 |
| 402 |
veor @x[3], @x[3], @t[3] |
| 403 |
vext.8 @t[5], @x[5], @x[5], #12 |
| 404 |
veor @x[4], @x[4], @t[4] |
| 405 |
vext.8 @t[6], @x[6], @x[6], #12 |
| 406 |
veor @x[5], @x[5], @t[5] |
| 407 |
vext.8 @t[7], @x[7], @x[7], #12 |
| 408 |
veor @x[6], @x[6], @t[6] |
| 409 |
|
| 410 |
veor @t[1], @t[1], @x[0] |
| 411 |
veor @x[7], @x[7], @t[7] |
| 412 |
vext.8 @x[0], @x[0], @x[0], #8 @ (x0 ^ (x0 <<< 32)) <<< 64) |
| 413 |
veor @t[2], @t[2], @x[1] |
| 414 |
veor @t[0], @t[0], @x[7] |
| 415 |
veor @t[1], @t[1], @x[7] |
| 416 |
vext.8 @x[1], @x[1], @x[1], #8 |
| 417 |
veor @t[5], @t[5], @x[4] |
| 418 |
veor @x[0], @x[0], @t[0] |
| 419 |
veor @t[6], @t[6], @x[5] |
| 420 |
veor @x[1], @x[1], @t[1] |
| 421 |
vext.8 @t[0], @x[4], @x[4], #8 |
| 422 |
veor @t[4], @t[4], @x[3] |
| 423 |
vext.8 @t[1], @x[5], @x[5], #8 |
| 424 |
veor @t[7], @t[7], @x[6] |
| 425 |
vext.8 @x[4], @x[3], @x[3], #8 |
| 426 |
veor @t[3], @t[3], @x[2] |
| 427 |
vext.8 @x[5], @x[7], @x[7], #8 |
| 428 |
veor @t[4], @t[4], @x[7] |
| 429 |
vext.8 @x[3], @x[6], @x[6], #8 |
| 430 |
veor @t[3], @t[3], @x[7] |
| 431 |
vext.8 @x[6], @x[2], @x[2], #8 |
| 432 |
veor @x[7], @t[1], @t[5] |
| 433 |
___ |
| 434 |
$code.=<<___ if (!$inv); |
| 435 |
veor @x[2], @t[0], @t[4] |
| 436 |
veor @x[4], @x[4], @t[3] |
| 437 |
veor @x[5], @x[5], @t[7] |
| 438 |
veor @x[3], @x[3], @t[6] |
| 439 |
@ vmov @x[2], @t[0] |
| 440 |
veor @x[6], @x[6], @t[2] |
| 441 |
@ vmov @x[7], @t[1] |
| 442 |
___ |
| 443 |
$code.=<<___ if ($inv); |
| 444 |
veor @t[3], @t[3], @x[4] |
| 445 |
veor @x[5], @x[5], @t[7] |
| 446 |
veor @x[2], @x[3], @t[6] |
| 447 |
veor @x[3], @t[0], @t[4] |
| 448 |
veor @x[4], @x[6], @t[2] |
| 449 |
vmov @x[6], @t[3] |
| 450 |
@ vmov @x[7], @t[1] |
| 451 |
___ |
| 452 |
} |
| 453 |
|
| 454 |
sub InvMixColumns_orig { |
| 455 |
my @x=@_[0..7]; |
| 456 |
my @t=@_[8..15]; |
| 457 |
|
| 458 |
$code.=<<___; |
| 459 |
@ multiplication by 0x0e |
| 460 |
vext.8 @t[7], @x[7], @x[7], #12 |
| 461 |
vmov @t[2], @x[2] |
| 462 |
veor @x[2], @x[2], @x[5] @ 2 5 |
| 463 |
veor @x[7], @x[7], @x[5] @ 7 5 |
| 464 |
vext.8 @t[0], @x[0], @x[0], #12 |
| 465 |
vmov @t[5], @x[5] |
| 466 |
veor @x[5], @x[5], @x[0] @ 5 0 [1] |
| 467 |
veor @x[0], @x[0], @x[1] @ 0 1 |
| 468 |
vext.8 @t[1], @x[1], @x[1], #12 |
| 469 |
veor @x[1], @x[1], @x[2] @ 1 25 |
| 470 |
veor @x[0], @x[0], @x[6] @ 01 6 [2] |
| 471 |
vext.8 @t[3], @x[3], @x[3], #12 |
| 472 |
veor @x[1], @x[1], @x[3] @ 125 3 [4] |
| 473 |
veor @x[2], @x[2], @x[0] @ 25 016 [3] |
| 474 |
veor @x[3], @x[3], @x[7] @ 3 75 |
| 475 |
veor @x[7], @x[7], @x[6] @ 75 6 [0] |
| 476 |
vext.8 @t[6], @x[6], @x[6], #12 |
| 477 |
vmov @t[4], @x[4] |
| 478 |
veor @x[6], @x[6], @x[4] @ 6 4 |
| 479 |
veor @x[4], @x[4], @x[3] @ 4 375 [6] |
| 480 |
veor @x[3], @x[3], @x[7] @ 375 756=36 |
| 481 |
veor @x[6], @x[6], @t[5] @ 64 5 [7] |
| 482 |
veor @x[3], @x[3], @t[2] @ 36 2 |
| 483 |
vext.8 @t[5], @t[5], @t[5], #12 |
| 484 |
veor @x[3], @x[3], @t[4] @ 362 4 [5] |
| 485 |
___ |
| 486 |
my @y = @x[7,5,0,2,1,3,4,6]; |
| 487 |
$code.=<<___; |
| 488 |
@ multiplication by 0x0b |
| 489 |
veor @y[1], @y[1], @y[0] |
| 490 |
veor @y[0], @y[0], @t[0] |
| 491 |
vext.8 @t[2], @t[2], @t[2], #12 |
| 492 |
veor @y[1], @y[1], @t[1] |
| 493 |
veor @y[0], @y[0], @t[5] |
| 494 |
vext.8 @t[4], @t[4], @t[4], #12 |
| 495 |
veor @y[1], @y[1], @t[6] |
| 496 |
veor @y[0], @y[0], @t[7] |
| 497 |
veor @t[7], @t[7], @t[6] @ clobber t[7] |
| 498 |
|
| 499 |
veor @y[3], @y[3], @t[0] |
| 500 |
veor @y[1], @y[1], @y[0] |
| 501 |
vext.8 @t[0], @t[0], @t[0], #12 |
| 502 |
veor @y[2], @y[2], @t[1] |
| 503 |
veor @y[4], @y[4], @t[1] |
| 504 |
vext.8 @t[1], @t[1], @t[1], #12 |
| 505 |
veor @y[2], @y[2], @t[2] |
| 506 |
veor @y[3], @y[3], @t[2] |
| 507 |
veor @y[5], @y[5], @t[2] |
| 508 |
veor @y[2], @y[2], @t[7] |
| 509 |
vext.8 @t[2], @t[2], @t[2], #12 |
| 510 |
veor @y[3], @y[3], @t[3] |
| 511 |
veor @y[6], @y[6], @t[3] |
| 512 |
veor @y[4], @y[4], @t[3] |
| 513 |
veor @y[7], @y[7], @t[4] |
| 514 |
vext.8 @t[3], @t[3], @t[3], #12 |
| 515 |
veor @y[5], @y[5], @t[4] |
| 516 |
veor @y[7], @y[7], @t[7] |
| 517 |
veor @t[7], @t[7], @t[5] @ clobber t[7] even more |
| 518 |
veor @y[3], @y[3], @t[5] |
| 519 |
veor @y[4], @y[4], @t[4] |
| 520 |
|
| 521 |
veor @y[5], @y[5], @t[7] |
| 522 |
vext.8 @t[4], @t[4], @t[4], #12 |
| 523 |
veor @y[6], @y[6], @t[7] |
| 524 |
veor @y[4], @y[4], @t[7] |
| 525 |
|
| 526 |
veor @t[7], @t[7], @t[5] |
| 527 |
vext.8 @t[5], @t[5], @t[5], #12 |
| 528 |
|
| 529 |
@ multiplication by 0x0d |
| 530 |
veor @y[4], @y[4], @y[7] |
| 531 |
veor @t[7], @t[7], @t[6] @ restore t[7] |
| 532 |
veor @y[7], @y[7], @t[4] |
| 533 |
vext.8 @t[6], @t[6], @t[6], #12 |
| 534 |
veor @y[2], @y[2], @t[0] |
| 535 |
veor @y[7], @y[7], @t[5] |
| 536 |
vext.8 @t[7], @t[7], @t[7], #12 |
| 537 |
veor @y[2], @y[2], @t[2] |
| 538 |
|
| 539 |
veor @y[3], @y[3], @y[1] |
| 540 |
veor @y[1], @y[1], @t[1] |
| 541 |
veor @y[0], @y[0], @t[0] |
| 542 |
veor @y[3], @y[3], @t[0] |
| 543 |
veor @y[1], @y[1], @t[5] |
| 544 |
veor @y[0], @y[0], @t[5] |
| 545 |
vext.8 @t[0], @t[0], @t[0], #12 |
| 546 |
veor @y[1], @y[1], @t[7] |
| 547 |
veor @y[0], @y[0], @t[6] |
| 548 |
veor @y[3], @y[3], @y[1] |
| 549 |
veor @y[4], @y[4], @t[1] |
| 550 |
vext.8 @t[1], @t[1], @t[1], #12 |
| 551 |
|
| 552 |
veor @y[7], @y[7], @t[7] |
| 553 |
veor @y[4], @y[4], @t[2] |
| 554 |
veor @y[5], @y[5], @t[2] |
| 555 |
veor @y[2], @y[2], @t[6] |
| 556 |
veor @t[6], @t[6], @t[3] @ clobber t[6] |
| 557 |
vext.8 @t[2], @t[2], @t[2], #12 |
| 558 |
veor @y[4], @y[4], @y[7] |
| 559 |
veor @y[3], @y[3], @t[6] |
| 560 |
|
| 561 |
veor @y[6], @y[6], @t[6] |
| 562 |
veor @y[5], @y[5], @t[5] |
| 563 |
vext.8 @t[5], @t[5], @t[5], #12 |
| 564 |
veor @y[6], @y[6], @t[4] |
| 565 |
vext.8 @t[4], @t[4], @t[4], #12 |
| 566 |
veor @y[5], @y[5], @t[6] |
| 567 |
veor @y[6], @y[6], @t[7] |
| 568 |
vext.8 @t[7], @t[7], @t[7], #12 |
| 569 |
veor @t[6], @t[6], @t[3] @ restore t[6] |
| 570 |
vext.8 @t[3], @t[3], @t[3], #12 |
| 571 |
|
| 572 |
@ multiplication by 0x09 |
| 573 |
veor @y[4], @y[4], @y[1] |
| 574 |
veor @t[1], @t[1], @y[1] @ t[1]=y[1] |
| 575 |
veor @t[0], @t[0], @t[5] @ clobber t[0] |
| 576 |
vext.8 @t[6], @t[6], @t[6], #12 |
| 577 |
veor @t[1], @t[1], @t[5] |
| 578 |
veor @y[3], @y[3], @t[0] |
| 579 |
veor @t[0], @t[0], @y[0] @ t[0]=y[0] |
| 580 |
veor @t[1], @t[1], @t[6] |
| 581 |
veor @t[6], @t[6], @t[7] @ clobber t[6] |
| 582 |
veor @y[4], @y[4], @t[1] |
| 583 |
veor @y[7], @y[7], @t[4] |
| 584 |
veor @y[6], @y[6], @t[3] |
| 585 |
veor @y[5], @y[5], @t[2] |
| 586 |
veor @t[4], @t[4], @y[4] @ t[4]=y[4] |
| 587 |
veor @t[3], @t[3], @y[3] @ t[3]=y[3] |
| 588 |
veor @t[5], @t[5], @y[5] @ t[5]=y[5] |
| 589 |
veor @t[2], @t[2], @y[2] @ t[2]=y[2] |
| 590 |
veor @t[3], @t[3], @t[7] |
| 591 |
veor @XMM[5], @t[5], @t[6] |
| 592 |
veor @XMM[6], @t[6], @y[6] @ t[6]=y[6] |
| 593 |
veor @XMM[2], @t[2], @t[6] |
| 594 |
veor @XMM[7], @t[7], @y[7] @ t[7]=y[7] |
| 595 |
|
| 596 |
vmov @XMM[0], @t[0] |
| 597 |
vmov @XMM[1], @t[1] |
| 598 |
@ vmov @XMM[2], @t[2] |
| 599 |
vmov @XMM[3], @t[3] |
| 600 |
vmov @XMM[4], @t[4] |
| 601 |
@ vmov @XMM[5], @t[5] |
| 602 |
@ vmov @XMM[6], @t[6] |
| 603 |
@ vmov @XMM[7], @t[7] |
| 604 |
___ |
| 605 |
} |
| 606 |
|
| 607 |
sub InvMixColumns { |
| 608 |
my @x=@_[0..7]; |
| 609 |
my @t=@_[8..15]; |
| 610 |
|
| 611 |
# Thanks to Jussi Kivilinna for providing pointer to |
| 612 |
# |
| 613 |
# | 0e 0b 0d 09 | | 02 03 01 01 | | 05 00 04 00 | |
| 614 |
# | 09 0e 0b 0d | = | 01 02 03 01 | x | 00 05 00 04 | |
| 615 |
# | 0d 09 0e 0b | | 01 01 02 03 | | 04 00 05 00 | |
| 616 |
# | 0b 0d 09 0e | | 03 01 01 02 | | 00 04 00 05 | |
| 617 |
|
| 618 |
$code.=<<___; |
| 619 |
@ multiplication by 0x05-0x00-0x04-0x00 |
| 620 |
vext.8 @t[0], @x[0], @x[0], #8 |
| 621 |
vext.8 @t[6], @x[6], @x[6], #8 |
| 622 |
vext.8 @t[7], @x[7], @x[7], #8 |
| 623 |
veor @t[0], @t[0], @x[0] |
| 624 |
vext.8 @t[1], @x[1], @x[1], #8 |
| 625 |
veor @t[6], @t[6], @x[6] |
| 626 |
vext.8 @t[2], @x[2], @x[2], #8 |
| 627 |
veor @t[7], @t[7], @x[7] |
| 628 |
vext.8 @t[3], @x[3], @x[3], #8 |
| 629 |
veor @t[1], @t[1], @x[1] |
| 630 |
vext.8 @t[4], @x[4], @x[4], #8 |
| 631 |
veor @t[2], @t[2], @x[2] |
| 632 |
vext.8 @t[5], @x[5], @x[5], #8 |
| 633 |
veor @t[3], @t[3], @x[3] |
| 634 |
veor @t[4], @t[4], @x[4] |
| 635 |
veor @t[5], @t[5], @x[5] |
| 636 |
|
| 637 |
veor @x[0], @x[0], @t[6] |
| 638 |
veor @x[1], @x[1], @t[6] |
| 639 |
veor @x[2], @x[2], @t[0] |
| 640 |
veor @x[4], @x[4], @t[2] |
| 641 |
veor @x[3], @x[3], @t[1] |
| 642 |
veor @x[1], @x[1], @t[7] |
| 643 |
veor @x[2], @x[2], @t[7] |
| 644 |
veor @x[4], @x[4], @t[6] |
| 645 |
veor @x[5], @x[5], @t[3] |
| 646 |
veor @x[3], @x[3], @t[6] |
| 647 |
veor @x[6], @x[6], @t[4] |
| 648 |
veor @x[4], @x[4], @t[7] |
| 649 |
veor @x[5], @x[5], @t[7] |
| 650 |
veor @x[7], @x[7], @t[5] |
| 651 |
___ |
| 652 |
&MixColumns (@x,@t,1); # flipped 2<->3 and 4<->6 |
| 653 |
} |
| 654 |
|
| 655 |
sub swapmove { |
| 656 |
my ($a,$b,$n,$mask,$t)=@_; |
| 657 |
$code.=<<___; |
| 658 |
vshr.u64 $t, $b, #$n |
| 659 |
veor $t, $t, $a |
| 660 |
vand $t, $t, $mask |
| 661 |
veor $a, $a, $t |
| 662 |
vshl.u64 $t, $t, #$n |
| 663 |
veor $b, $b, $t |
| 664 |
___ |
| 665 |
} |
| 666 |
sub swapmove2x { |
| 667 |
my ($a0,$b0,$a1,$b1,$n,$mask,$t0,$t1)=@_; |
| 668 |
$code.=<<___; |
| 669 |
vshr.u64 $t0, $b0, #$n |
| 670 |
vshr.u64 $t1, $b1, #$n |
| 671 |
veor $t0, $t0, $a0 |
| 672 |
veor $t1, $t1, $a1 |
| 673 |
vand $t0, $t0, $mask |
| 674 |
vand $t1, $t1, $mask |
| 675 |
veor $a0, $a0, $t0 |
| 676 |
vshl.u64 $t0, $t0, #$n |
| 677 |
veor $a1, $a1, $t1 |
| 678 |
vshl.u64 $t1, $t1, #$n |
| 679 |
veor $b0, $b0, $t0 |
| 680 |
veor $b1, $b1, $t1 |
| 681 |
___ |
| 682 |
} |
| 683 |
|
| 684 |
sub bitslice { |
| 685 |
my @x=reverse(@_[0..7]); |
| 686 |
my ($t0,$t1,$t2,$t3)=@_[8..11]; |
| 687 |
$code.=<<___; |
| 688 |
vmov.i8 $t0,#0x55 @ compose .LBS0 |
| 689 |
vmov.i8 $t1,#0x33 @ compose .LBS1 |
| 690 |
___ |
| 691 |
&swapmove2x(@x[0,1,2,3],1,$t0,$t2,$t3); |
| 692 |
&swapmove2x(@x[4,5,6,7],1,$t0,$t2,$t3); |
| 693 |
$code.=<<___; |
| 694 |
vmov.i8 $t0,#0x0f @ compose .LBS2 |
| 695 |
___ |
| 696 |
&swapmove2x(@x[0,2,1,3],2,$t1,$t2,$t3); |
| 697 |
&swapmove2x(@x[4,6,5,7],2,$t1,$t2,$t3); |
| 698 |
|
| 699 |
&swapmove2x(@x[0,4,1,5],4,$t0,$t2,$t3); |
| 700 |
&swapmove2x(@x[2,6,3,7],4,$t0,$t2,$t3); |
| 701 |
} |
| 702 |
|
| 703 |
$code.=<<___; |
| 704 |
#ifndef __KERNEL__ |
| 705 |
# include <openssl/arm_arch.h> |
| 706 |
|
| 707 |
# define VFP_ABI_PUSH vstmdb sp!,{d8-d15} |
| 708 |
# define VFP_ABI_POP vldmia sp!,{d8-d15} |
| 709 |
# define VFP_ABI_FRAME 0x40 |
| 710 |
#else |
| 711 |
# define VFP_ABI_PUSH |
| 712 |
# define VFP_ABI_POP |
| 713 |
# define VFP_ABI_FRAME 0 |
| 714 |
# define BSAES_ASM_EXTENDED_KEY |
| 715 |
# define XTS_CHAIN_TWEAK |
| 716 |
# define __ARM_ARCH__ __LINUX_ARM_ARCH__ |
| 717 |
# define __ARM_MAX_ARCH__ 7 |
| 718 |
#endif |
| 719 |
|
| 720 |
#ifdef __thumb__ |
| 721 |
# define adrl adr |
| 722 |
#endif |
| 723 |
|
| 724 |
#if __ARM_MAX_ARCH__>=7 |
| 725 |
.arch armv7-a |
| 726 |
.fpu neon |
| 727 |
|
| 728 |
.text |
| 729 |
.syntax unified @ ARMv7-capable assembler is expected to handle this |
| 730 |
#if defined(__thumb2__) && !defined(__APPLE__) |
| 731 |
.thumb |
| 732 |
#else |
| 733 |
.code 32 |
| 734 |
#endif |
| 735 |
|
| 736 |
.type _bsaes_decrypt8,%function |
| 737 |
.align 4 |
| 738 |
_bsaes_decrypt8: |
| 739 |
adr $const,_bsaes_decrypt8 |
| 740 |
vldmia $key!, {@XMM[9]} @ round 0 key |
| 741 |
#ifdef __APPLE__ |
| 742 |
adr $const,.LM0ISR |
| 743 |
#else |
| 744 |
add $const,$const,#.LM0ISR-_bsaes_decrypt8 |
| 745 |
#endif |
| 746 |
|
| 747 |
vldmia $const!, {@XMM[8]} @ .LM0ISR |
| 748 |
veor @XMM[10], @XMM[0], @XMM[9] @ xor with round0 key |
| 749 |
veor @XMM[11], @XMM[1], @XMM[9] |
| 750 |
vtbl.8 `&Dlo(@XMM[0])`, {@XMM[10]}, `&Dlo(@XMM[8])` |
| 751 |
vtbl.8 `&Dhi(@XMM[0])`, {@XMM[10]}, `&Dhi(@XMM[8])` |
| 752 |
veor @XMM[12], @XMM[2], @XMM[9] |
| 753 |
vtbl.8 `&Dlo(@XMM[1])`, {@XMM[11]}, `&Dlo(@XMM[8])` |
| 754 |
vtbl.8 `&Dhi(@XMM[1])`, {@XMM[11]}, `&Dhi(@XMM[8])` |
| 755 |
veor @XMM[13], @XMM[3], @XMM[9] |
| 756 |
vtbl.8 `&Dlo(@XMM[2])`, {@XMM[12]}, `&Dlo(@XMM[8])` |
| 757 |
vtbl.8 `&Dhi(@XMM[2])`, {@XMM[12]}, `&Dhi(@XMM[8])` |
| 758 |
veor @XMM[14], @XMM[4], @XMM[9] |
| 759 |
vtbl.8 `&Dlo(@XMM[3])`, {@XMM[13]}, `&Dlo(@XMM[8])` |
| 760 |
vtbl.8 `&Dhi(@XMM[3])`, {@XMM[13]}, `&Dhi(@XMM[8])` |
| 761 |
veor @XMM[15], @XMM[5], @XMM[9] |
| 762 |
vtbl.8 `&Dlo(@XMM[4])`, {@XMM[14]}, `&Dlo(@XMM[8])` |
| 763 |
vtbl.8 `&Dhi(@XMM[4])`, {@XMM[14]}, `&Dhi(@XMM[8])` |
| 764 |
veor @XMM[10], @XMM[6], @XMM[9] |
| 765 |
vtbl.8 `&Dlo(@XMM[5])`, {@XMM[15]}, `&Dlo(@XMM[8])` |
| 766 |
vtbl.8 `&Dhi(@XMM[5])`, {@XMM[15]}, `&Dhi(@XMM[8])` |
| 767 |
veor @XMM[11], @XMM[7], @XMM[9] |
| 768 |
vtbl.8 `&Dlo(@XMM[6])`, {@XMM[10]}, `&Dlo(@XMM[8])` |
| 769 |
vtbl.8 `&Dhi(@XMM[6])`, {@XMM[10]}, `&Dhi(@XMM[8])` |
| 770 |
vtbl.8 `&Dlo(@XMM[7])`, {@XMM[11]}, `&Dlo(@XMM[8])` |
| 771 |
vtbl.8 `&Dhi(@XMM[7])`, {@XMM[11]}, `&Dhi(@XMM[8])` |
| 772 |
___ |
| 773 |
&bitslice (@XMM[0..7, 8..11]); |
| 774 |
$code.=<<___; |
| 775 |
sub $rounds,$rounds,#1 |
| 776 |
b .Ldec_sbox |
| 777 |
.align 4 |
| 778 |
.Ldec_loop: |
| 779 |
___ |
| 780 |
&ShiftRows (@XMM[0..7, 8..12]); |
| 781 |
$code.=".Ldec_sbox:\n"; |
| 782 |
&InvSbox (@XMM[0..7, 8..15]); |
| 783 |
$code.=<<___; |
| 784 |
subs $rounds,$rounds,#1 |
| 785 |
bcc .Ldec_done |
| 786 |
___ |
| 787 |
&InvMixColumns (@XMM[0,1,6,4,2,7,3,5, 8..15]); |
| 788 |
$code.=<<___; |
| 789 |
vldmia $const, {@XMM[12]} @ .LISR |
| 790 |
ite eq @ Thumb2 thing, sanity check in ARM |
| 791 |
addeq $const,$const,#0x10 |
| 792 |
bne .Ldec_loop |
| 793 |
vldmia $const, {@XMM[12]} @ .LISRM0 |
| 794 |
b .Ldec_loop |
| 795 |
.align 4 |
| 796 |
.Ldec_done: |
| 797 |
___ |
| 798 |
&bitslice (@XMM[0,1,6,4,2,7,3,5, 8..11]); |
| 799 |
$code.=<<___; |
| 800 |
vldmia $key, {@XMM[8]} @ last round key |
| 801 |
veor @XMM[6], @XMM[6], @XMM[8] |
| 802 |
veor @XMM[4], @XMM[4], @XMM[8] |
| 803 |
veor @XMM[2], @XMM[2], @XMM[8] |
| 804 |
veor @XMM[7], @XMM[7], @XMM[8] |
| 805 |
veor @XMM[3], @XMM[3], @XMM[8] |
| 806 |
veor @XMM[5], @XMM[5], @XMM[8] |
| 807 |
veor @XMM[0], @XMM[0], @XMM[8] |
| 808 |
veor @XMM[1], @XMM[1], @XMM[8] |
| 809 |
bx lr |
| 810 |
.size _bsaes_decrypt8,.-_bsaes_decrypt8 |
| 811 |
|
| 812 |
.type _bsaes_const,%object |
| 813 |
.align 6 |
| 814 |
_bsaes_const: |
| 815 |
.LM0ISR: @ InvShiftRows constants |
| 816 |
.quad 0x0a0e0206070b0f03, 0x0004080c0d010509 |
| 817 |
.LISR: |
| 818 |
.quad 0x0504070602010003, 0x0f0e0d0c080b0a09 |
| 819 |
.LISRM0: |
| 820 |
.quad 0x01040b0e0205080f, 0x0306090c00070a0d |
| 821 |
.LM0SR: @ ShiftRows constants |
| 822 |
.quad 0x0a0e02060f03070b, 0x0004080c05090d01 |
| 823 |
.LSR: |
| 824 |
.quad 0x0504070600030201, 0x0f0e0d0c0a09080b |
| 825 |
.LSRM0: |
| 826 |
.quad 0x0304090e00050a0f, 0x01060b0c0207080d |
| 827 |
.LM0: |
| 828 |
.quad 0x02060a0e03070b0f, 0x0004080c0105090d |
| 829 |
.LREVM0SR: |
| 830 |
.quad 0x090d01050c000408, 0x03070b0f060a0e02 |
| 831 |
.asciz "Bit-sliced AES for NEON, CRYPTOGAMS by <appro\@openssl.org>" |
| 832 |
.align 6 |
| 833 |
.size _bsaes_const,.-_bsaes_const |
| 834 |
|
| 835 |
.type _bsaes_encrypt8,%function |
| 836 |
.align 4 |
| 837 |
_bsaes_encrypt8: |
| 838 |
adr $const,_bsaes_encrypt8 |
| 839 |
vldmia $key!, {@XMM[9]} @ round 0 key |
| 840 |
#ifdef __APPLE__ |
| 841 |
adr $const,.LM0SR |
| 842 |
#else |
| 843 |
sub $const,$const,#_bsaes_encrypt8-.LM0SR |
| 844 |
#endif |
| 845 |
|
| 846 |
vldmia $const!, {@XMM[8]} @ .LM0SR |
| 847 |
_bsaes_encrypt8_alt: |
| 848 |
veor @XMM[10], @XMM[0], @XMM[9] @ xor with round0 key |
| 849 |
veor @XMM[11], @XMM[1], @XMM[9] |
| 850 |
vtbl.8 `&Dlo(@XMM[0])`, {@XMM[10]}, `&Dlo(@XMM[8])` |
| 851 |
vtbl.8 `&Dhi(@XMM[0])`, {@XMM[10]}, `&Dhi(@XMM[8])` |
| 852 |
veor @XMM[12], @XMM[2], @XMM[9] |
| 853 |
vtbl.8 `&Dlo(@XMM[1])`, {@XMM[11]}, `&Dlo(@XMM[8])` |
| 854 |
vtbl.8 `&Dhi(@XMM[1])`, {@XMM[11]}, `&Dhi(@XMM[8])` |
| 855 |
veor @XMM[13], @XMM[3], @XMM[9] |
| 856 |
vtbl.8 `&Dlo(@XMM[2])`, {@XMM[12]}, `&Dlo(@XMM[8])` |
| 857 |
vtbl.8 `&Dhi(@XMM[2])`, {@XMM[12]}, `&Dhi(@XMM[8])` |
| 858 |
veor @XMM[14], @XMM[4], @XMM[9] |
| 859 |
vtbl.8 `&Dlo(@XMM[3])`, {@XMM[13]}, `&Dlo(@XMM[8])` |
| 860 |
vtbl.8 `&Dhi(@XMM[3])`, {@XMM[13]}, `&Dhi(@XMM[8])` |
| 861 |
veor @XMM[15], @XMM[5], @XMM[9] |
| 862 |
vtbl.8 `&Dlo(@XMM[4])`, {@XMM[14]}, `&Dlo(@XMM[8])` |
| 863 |
vtbl.8 `&Dhi(@XMM[4])`, {@XMM[14]}, `&Dhi(@XMM[8])` |
| 864 |
veor @XMM[10], @XMM[6], @XMM[9] |
| 865 |
vtbl.8 `&Dlo(@XMM[5])`, {@XMM[15]}, `&Dlo(@XMM[8])` |
| 866 |
vtbl.8 `&Dhi(@XMM[5])`, {@XMM[15]}, `&Dhi(@XMM[8])` |
| 867 |
veor @XMM[11], @XMM[7], @XMM[9] |
| 868 |
vtbl.8 `&Dlo(@XMM[6])`, {@XMM[10]}, `&Dlo(@XMM[8])` |
| 869 |
vtbl.8 `&Dhi(@XMM[6])`, {@XMM[10]}, `&Dhi(@XMM[8])` |
| 870 |
vtbl.8 `&Dlo(@XMM[7])`, {@XMM[11]}, `&Dlo(@XMM[8])` |
| 871 |
vtbl.8 `&Dhi(@XMM[7])`, {@XMM[11]}, `&Dhi(@XMM[8])` |
| 872 |
_bsaes_encrypt8_bitslice: |
| 873 |
___ |
| 874 |
&bitslice (@XMM[0..7, 8..11]); |
| 875 |
$code.=<<___; |
| 876 |
sub $rounds,$rounds,#1 |
| 877 |
b .Lenc_sbox |
| 878 |
.align 4 |
| 879 |
.Lenc_loop: |
| 880 |
___ |
| 881 |
&ShiftRows (@XMM[0..7, 8..12]); |
| 882 |
$code.=".Lenc_sbox:\n"; |
| 883 |
&Sbox (@XMM[0..7, 8..15]); |
| 884 |
$code.=<<___; |
| 885 |
subs $rounds,$rounds,#1 |
| 886 |
bcc .Lenc_done |
| 887 |
___ |
| 888 |
&MixColumns (@XMM[0,1,4,6,3,7,2,5, 8..15]); |
| 889 |
$code.=<<___; |
| 890 |
vldmia $const, {@XMM[12]} @ .LSR |
| 891 |
ite eq @ Thumb2 thing, samity check in ARM |
| 892 |
addeq $const,$const,#0x10 |
| 893 |
bne .Lenc_loop |
| 894 |
vldmia $const, {@XMM[12]} @ .LSRM0 |
| 895 |
b .Lenc_loop |
| 896 |
.align 4 |
| 897 |
.Lenc_done: |
| 898 |
___ |
| 899 |
# output in lsb > [t0, t1, t4, t6, t3, t7, t2, t5] < msb |
| 900 |
&bitslice (@XMM[0,1,4,6,3,7,2,5, 8..11]); |
| 901 |
$code.=<<___; |
| 902 |
vldmia $key, {@XMM[8]} @ last round key |
| 903 |
veor @XMM[4], @XMM[4], @XMM[8] |
| 904 |
veor @XMM[6], @XMM[6], @XMM[8] |
| 905 |
veor @XMM[3], @XMM[3], @XMM[8] |
| 906 |
veor @XMM[7], @XMM[7], @XMM[8] |
| 907 |
veor @XMM[2], @XMM[2], @XMM[8] |
| 908 |
veor @XMM[5], @XMM[5], @XMM[8] |
| 909 |
veor @XMM[0], @XMM[0], @XMM[8] |
| 910 |
veor @XMM[1], @XMM[1], @XMM[8] |
| 911 |
bx lr |
| 912 |
.size _bsaes_encrypt8,.-_bsaes_encrypt8 |
| 913 |
___ |
| 914 |
} |
| 915 |
{ |
| 916 |
my ($out,$inp,$rounds,$const)=("r12","r4","r5","r6"); |
| 917 |
|
| 918 |
sub bitslice_key { |
| 919 |
my @x=reverse(@_[0..7]); |
| 920 |
my ($bs0,$bs1,$bs2,$t2,$t3)=@_[8..12]; |
| 921 |
|
| 922 |
&swapmove (@x[0,1],1,$bs0,$t2,$t3); |
| 923 |
$code.=<<___; |
| 924 |
@ &swapmove(@x[2,3],1,$t0,$t2,$t3); |
| 925 |
vmov @x[2], @x[0] |
| 926 |
vmov @x[3], @x[1] |
| 927 |
___ |
| 928 |
#&swapmove2x(@x[4,5,6,7],1,$t0,$t2,$t3); |
| 929 |
|
| 930 |
&swapmove2x (@x[0,2,1,3],2,$bs1,$t2,$t3); |
| 931 |
$code.=<<___; |
| 932 |
@ &swapmove2x(@x[4,6,5,7],2,$t1,$t2,$t3); |
| 933 |
vmov @x[4], @x[0] |
| 934 |
vmov @x[6], @x[2] |
| 935 |
vmov @x[5], @x[1] |
| 936 |
vmov @x[7], @x[3] |
| 937 |
___ |
| 938 |
&swapmove2x (@x[0,4,1,5],4,$bs2,$t2,$t3); |
| 939 |
&swapmove2x (@x[2,6,3,7],4,$bs2,$t2,$t3); |
| 940 |
} |
| 941 |
|
| 942 |
$code.=<<___; |
| 943 |
.type _bsaes_key_convert,%function |
| 944 |
.align 4 |
| 945 |
_bsaes_key_convert: |
| 946 |
adr $const,_bsaes_key_convert |
| 947 |
vld1.8 {@XMM[7]}, [$inp]! @ load round 0 key |
| 948 |
#ifdef __APPLE__ |
| 949 |
adr $const,.LM0 |
| 950 |
#else |
| 951 |
sub $const,$const,#_bsaes_key_convert-.LM0 |
| 952 |
#endif |
| 953 |
vld1.8 {@XMM[15]}, [$inp]! @ load round 1 key |
| 954 |
|
| 955 |
vmov.i8 @XMM[8], #0x01 @ bit masks |
| 956 |
vmov.i8 @XMM[9], #0x02 |
| 957 |
vmov.i8 @XMM[10], #0x04 |
| 958 |
vmov.i8 @XMM[11], #0x08 |
| 959 |
vmov.i8 @XMM[12], #0x10 |
| 960 |
vmov.i8 @XMM[13], #0x20 |
| 961 |
vldmia $const, {@XMM[14]} @ .LM0 |
| 962 |
|
| 963 |
#ifdef __ARMEL__ |
| 964 |
vrev32.8 @XMM[7], @XMM[7] |
| 965 |
vrev32.8 @XMM[15], @XMM[15] |
| 966 |
#endif |
| 967 |
sub $rounds,$rounds,#1 |
| 968 |
vstmia $out!, {@XMM[7]} @ save round 0 key |
| 969 |
b .Lkey_loop |
| 970 |
|
| 971 |
.align 4 |
| 972 |
.Lkey_loop: |
| 973 |
vtbl.8 `&Dlo(@XMM[7])`,{@XMM[15]},`&Dlo(@XMM[14])` |
| 974 |
vtbl.8 `&Dhi(@XMM[7])`,{@XMM[15]},`&Dhi(@XMM[14])` |
| 975 |
vmov.i8 @XMM[6], #0x40 |
| 976 |
vmov.i8 @XMM[15], #0x80 |
| 977 |
|
| 978 |
vtst.8 @XMM[0], @XMM[7], @XMM[8] |
| 979 |
vtst.8 @XMM[1], @XMM[7], @XMM[9] |
| 980 |
vtst.8 @XMM[2], @XMM[7], @XMM[10] |
| 981 |
vtst.8 @XMM[3], @XMM[7], @XMM[11] |
| 982 |
vtst.8 @XMM[4], @XMM[7], @XMM[12] |
| 983 |
vtst.8 @XMM[5], @XMM[7], @XMM[13] |
| 984 |
vtst.8 @XMM[6], @XMM[7], @XMM[6] |
| 985 |
vtst.8 @XMM[7], @XMM[7], @XMM[15] |
| 986 |
vld1.8 {@XMM[15]}, [$inp]! @ load next round key |
| 987 |
vmvn @XMM[0], @XMM[0] @ "pnot" |
| 988 |
vmvn @XMM[1], @XMM[1] |
| 989 |
vmvn @XMM[5], @XMM[5] |
| 990 |
vmvn @XMM[6], @XMM[6] |
| 991 |
#ifdef __ARMEL__ |
| 992 |
vrev32.8 @XMM[15], @XMM[15] |
| 993 |
#endif |
| 994 |
subs $rounds,$rounds,#1 |
| 995 |
vstmia $out!,{@XMM[0]-@XMM[7]} @ write bit-sliced round key |
| 996 |
bne .Lkey_loop |
| 997 |
|
| 998 |
vmov.i8 @XMM[7],#0x63 @ compose .L63 |
| 999 |
@ don't save last round key |
| 1000 |
bx lr |
| 1001 |
.size _bsaes_key_convert,.-_bsaes_key_convert |
| 1002 |
___ |
| 1003 |
} |
| 1004 |
|
| 1005 |
if (0) { # following four functions are unsupported interface |
| 1006 |
# used for benchmarking... |
| 1007 |
$code.=<<___; |
| 1008 |
.globl bsaes_enc_key_convert |
| 1009 |
.type bsaes_enc_key_convert,%function |
| 1010 |
.align 4 |
| 1011 |
bsaes_enc_key_convert: |
| 1012 |
stmdb sp!,{r4-r6,lr} |
| 1013 |
vstmdb sp!,{d8-d15} @ ABI specification says so |
| 1014 |
|
| 1015 |
ldr r5,[$inp,#240] @ pass rounds |
| 1016 |
mov r4,$inp @ pass key |
| 1017 |
mov r12,$out @ pass key schedule |
| 1018 |
bl _bsaes_key_convert |
| 1019 |
veor @XMM[7],@XMM[7],@XMM[15] @ fix up last round key |
| 1020 |
vstmia r12, {@XMM[7]} @ save last round key |
| 1021 |
|
| 1022 |
vldmia sp!,{d8-d15} |
| 1023 |
ldmia sp!,{r4-r6,pc} |
| 1024 |
.size bsaes_enc_key_convert,.-bsaes_enc_key_convert |
| 1025 |
|
| 1026 |
.globl bsaes_encrypt_128 |
| 1027 |
.type bsaes_encrypt_128,%function |
| 1028 |
.align 4 |
| 1029 |
bsaes_encrypt_128: |
| 1030 |
stmdb sp!,{r4-r6,lr} |
| 1031 |
vstmdb sp!,{d8-d15} @ ABI specification says so |
| 1032 |
.Lenc128_loop: |
| 1033 |
vld1.8 {@XMM[0]-@XMM[1]}, [$inp]! @ load input |
| 1034 |
vld1.8 {@XMM[2]-@XMM[3]}, [$inp]! |
| 1035 |
mov r4,$key @ pass the key |
| 1036 |
vld1.8 {@XMM[4]-@XMM[5]}, [$inp]! |
| 1037 |
mov r5,#10 @ pass rounds |
| 1038 |
vld1.8 {@XMM[6]-@XMM[7]}, [$inp]! |
| 1039 |
|
| 1040 |
bl _bsaes_encrypt8 |
| 1041 |
|
| 1042 |
vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output |
| 1043 |
vst1.8 {@XMM[4]}, [$out]! |
| 1044 |
vst1.8 {@XMM[6]}, [$out]! |
| 1045 |
vst1.8 {@XMM[3]}, [$out]! |
| 1046 |
vst1.8 {@XMM[7]}, [$out]! |
| 1047 |
vst1.8 {@XMM[2]}, [$out]! |
| 1048 |
subs $len,$len,#0x80 |
| 1049 |
vst1.8 {@XMM[5]}, [$out]! |
| 1050 |
bhi .Lenc128_loop |
| 1051 |
|
| 1052 |
vldmia sp!,{d8-d15} |
| 1053 |
ldmia sp!,{r4-r6,pc} |
| 1054 |
.size bsaes_encrypt_128,.-bsaes_encrypt_128 |
| 1055 |
|
| 1056 |
.globl bsaes_dec_key_convert |
| 1057 |
.type bsaes_dec_key_convert,%function |
| 1058 |
.align 4 |
| 1059 |
bsaes_dec_key_convert: |
| 1060 |
stmdb sp!,{r4-r6,lr} |
| 1061 |
vstmdb sp!,{d8-d15} @ ABI specification says so |
| 1062 |
|
| 1063 |
ldr r5,[$inp,#240] @ pass rounds |
| 1064 |
mov r4,$inp @ pass key |
| 1065 |
mov r12,$out @ pass key schedule |
| 1066 |
bl _bsaes_key_convert |
| 1067 |
vldmia $out, {@XMM[6]} |
| 1068 |
vstmia r12, {@XMM[15]} @ save last round key |
| 1069 |
veor @XMM[7], @XMM[7], @XMM[6] @ fix up round 0 key |
| 1070 |
vstmia $out, {@XMM[7]} |
| 1071 |
|
| 1072 |
vldmia sp!,{d8-d15} |
| 1073 |
ldmia sp!,{r4-r6,pc} |
| 1074 |
.size bsaes_dec_key_convert,.-bsaes_dec_key_convert |
| 1075 |
|
| 1076 |
.globl bsaes_decrypt_128 |
| 1077 |
.type bsaes_decrypt_128,%function |
| 1078 |
.align 4 |
| 1079 |
bsaes_decrypt_128: |
| 1080 |
stmdb sp!,{r4-r6,lr} |
| 1081 |
vstmdb sp!,{d8-d15} @ ABI specification says so |
| 1082 |
.Ldec128_loop: |
| 1083 |
vld1.8 {@XMM[0]-@XMM[1]}, [$inp]! @ load input |
| 1084 |
vld1.8 {@XMM[2]-@XMM[3]}, [$inp]! |
| 1085 |
mov r4,$key @ pass the key |
| 1086 |
vld1.8 {@XMM[4]-@XMM[5]}, [$inp]! |
| 1087 |
mov r5,#10 @ pass rounds |
| 1088 |
vld1.8 {@XMM[6]-@XMM[7]}, [$inp]! |
| 1089 |
|
| 1090 |
bl _bsaes_decrypt8 |
| 1091 |
|
| 1092 |
vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output |
| 1093 |
vst1.8 {@XMM[6]}, [$out]! |
| 1094 |
vst1.8 {@XMM[4]}, [$out]! |
| 1095 |
vst1.8 {@XMM[2]}, [$out]! |
| 1096 |
vst1.8 {@XMM[7]}, [$out]! |
| 1097 |
vst1.8 {@XMM[3]}, [$out]! |
| 1098 |
subs $len,$len,#0x80 |
| 1099 |
vst1.8 {@XMM[5]}, [$out]! |
| 1100 |
bhi .Ldec128_loop |
| 1101 |
|
| 1102 |
vldmia sp!,{d8-d15} |
| 1103 |
ldmia sp!,{r4-r6,pc} |
| 1104 |
.size bsaes_decrypt_128,.-bsaes_decrypt_128 |
| 1105 |
___ |
| 1106 |
} |
| 1107 |
{ |
| 1108 |
my ($inp,$out,$len,$key, $ivp,$fp,$rounds)=map("r$_",(0..3,8..10)); |
| 1109 |
my ($keysched)=("sp"); |
| 1110 |
|
| 1111 |
$code.=<<___; |
| 1112 |
.extern AES_cbc_encrypt |
| 1113 |
.extern AES_decrypt |
| 1114 |
|
| 1115 |
.global bsaes_cbc_encrypt |
| 1116 |
.type bsaes_cbc_encrypt,%function |
| 1117 |
.align 5 |
| 1118 |
bsaes_cbc_encrypt: |
| 1119 |
#ifndef __KERNEL__ |
| 1120 |
cmp $len, #128 |
| 1121 |
#ifndef __thumb__ |
| 1122 |
blo AES_cbc_encrypt |
| 1123 |
#else |
| 1124 |
bhs 1f |
| 1125 |
b AES_cbc_encrypt |
| 1126 |
1: |
| 1127 |
#endif |
| 1128 |
#endif |
| 1129 |
|
| 1130 |
@ it is up to the caller to make sure we are called with enc == 0 |
| 1131 |
|
| 1132 |
mov ip, sp |
| 1133 |
stmdb sp!, {r4-r10, lr} |
| 1134 |
VFP_ABI_PUSH |
| 1135 |
ldr $ivp, [ip] @ IV is 1st arg on the stack |
| 1136 |
mov $len, $len, lsr#4 @ len in 16 byte blocks |
| 1137 |
sub sp, #0x10 @ scratch space to carry over the IV |
| 1138 |
mov $fp, sp @ save sp |
| 1139 |
|
| 1140 |
ldr $rounds, [$key, #240] @ get # of rounds |
| 1141 |
#ifndef BSAES_ASM_EXTENDED_KEY |
| 1142 |
@ allocate the key schedule on the stack |
| 1143 |
sub r12, sp, $rounds, lsl#7 @ 128 bytes per inner round key |
| 1144 |
add r12, #`128-32` @ sifze of bit-slices key schedule |
| 1145 |
|
| 1146 |
@ populate the key schedule |
| 1147 |
mov r4, $key @ pass key |
| 1148 |
mov r5, $rounds @ pass # of rounds |
| 1149 |
mov sp, r12 @ sp is $keysched |
| 1150 |
bl _bsaes_key_convert |
| 1151 |
vldmia $keysched, {@XMM[6]} |
| 1152 |
vstmia r12, {@XMM[15]} @ save last round key |
| 1153 |
veor @XMM[7], @XMM[7], @XMM[6] @ fix up round 0 key |
| 1154 |
vstmia $keysched, {@XMM[7]} |
| 1155 |
#else |
| 1156 |
ldr r12, [$key, #244] |
| 1157 |
eors r12, #1 |
| 1158 |
beq 0f |
| 1159 |
|
| 1160 |
@ populate the key schedule |
| 1161 |
str r12, [$key, #244] |
| 1162 |
mov r4, $key @ pass key |
| 1163 |
mov r5, $rounds @ pass # of rounds |
| 1164 |
add r12, $key, #248 @ pass key schedule |
| 1165 |
bl _bsaes_key_convert |
| 1166 |
add r4, $key, #248 |
| 1167 |
vldmia r4, {@XMM[6]} |
| 1168 |
vstmia r12, {@XMM[15]} @ save last round key |
| 1169 |
veor @XMM[7], @XMM[7], @XMM[6] @ fix up round 0 key |
| 1170 |
vstmia r4, {@XMM[7]} |
| 1171 |
|
| 1172 |
.align 2 |
| 1173 |
0: |
| 1174 |
#endif |
| 1175 |
|
| 1176 |
vld1.8 {@XMM[15]}, [$ivp] @ load IV |
| 1177 |
b .Lcbc_dec_loop |
| 1178 |
|
| 1179 |
.align 4 |
| 1180 |
.Lcbc_dec_loop: |
| 1181 |
subs $len, $len, #0x8 |
| 1182 |
bmi .Lcbc_dec_loop_finish |
| 1183 |
|
| 1184 |
vld1.8 {@XMM[0]-@XMM[1]}, [$inp]! @ load input |
| 1185 |
vld1.8 {@XMM[2]-@XMM[3]}, [$inp]! |
| 1186 |
#ifndef BSAES_ASM_EXTENDED_KEY |
| 1187 |
mov r4, $keysched @ pass the key |
| 1188 |
#else |
| 1189 |
add r4, $key, #248 |
| 1190 |
#endif |
| 1191 |
vld1.8 {@XMM[4]-@XMM[5]}, [$inp]! |
| 1192 |
mov r5, $rounds |
| 1193 |
vld1.8 {@XMM[6]-@XMM[7]}, [$inp] |
| 1194 |
sub $inp, $inp, #0x60 |
| 1195 |
vstmia $fp, {@XMM[15]} @ put aside IV |
| 1196 |
|
| 1197 |
bl _bsaes_decrypt8 |
| 1198 |
|
| 1199 |
vldmia $fp, {@XMM[14]} @ reload IV |
| 1200 |
vld1.8 {@XMM[8]-@XMM[9]}, [$inp]! @ reload input |
| 1201 |
veor @XMM[0], @XMM[0], @XMM[14] @ ^= IV |
| 1202 |
vld1.8 {@XMM[10]-@XMM[11]}, [$inp]! |
| 1203 |
veor @XMM[1], @XMM[1], @XMM[8] |
| 1204 |
veor @XMM[6], @XMM[6], @XMM[9] |
| 1205 |
vld1.8 {@XMM[12]-@XMM[13]}, [$inp]! |
| 1206 |
veor @XMM[4], @XMM[4], @XMM[10] |
| 1207 |
veor @XMM[2], @XMM[2], @XMM[11] |
| 1208 |
vld1.8 {@XMM[14]-@XMM[15]}, [$inp]! |
| 1209 |
veor @XMM[7], @XMM[7], @XMM[12] |
| 1210 |
vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output |
| 1211 |
veor @XMM[3], @XMM[3], @XMM[13] |
| 1212 |
vst1.8 {@XMM[6]}, [$out]! |
| 1213 |
veor @XMM[5], @XMM[5], @XMM[14] |
| 1214 |
vst1.8 {@XMM[4]}, [$out]! |
| 1215 |
vst1.8 {@XMM[2]}, [$out]! |
| 1216 |
vst1.8 {@XMM[7]}, [$out]! |
| 1217 |
vst1.8 {@XMM[3]}, [$out]! |
| 1218 |
vst1.8 {@XMM[5]}, [$out]! |
| 1219 |
|
| 1220 |
b .Lcbc_dec_loop |
| 1221 |
|
| 1222 |
.Lcbc_dec_loop_finish: |
| 1223 |
adds $len, $len, #8 |
| 1224 |
beq .Lcbc_dec_done |
| 1225 |
|
| 1226 |
vld1.8 {@XMM[0]}, [$inp]! @ load input |
| 1227 |
cmp $len, #2 |
| 1228 |
blo .Lcbc_dec_one |
| 1229 |
vld1.8 {@XMM[1]}, [$inp]! |
| 1230 |
#ifndef BSAES_ASM_EXTENDED_KEY |
| 1231 |
mov r4, $keysched @ pass the key |
| 1232 |
#else |
| 1233 |
add r4, $key, #248 |
| 1234 |
#endif |
| 1235 |
mov r5, $rounds |
| 1236 |
vstmia $fp, {@XMM[15]} @ put aside IV |
| 1237 |
beq .Lcbc_dec_two |
| 1238 |
vld1.8 {@XMM[2]}, [$inp]! |
| 1239 |
cmp $len, #4 |
| 1240 |
blo .Lcbc_dec_three |
| 1241 |
vld1.8 {@XMM[3]}, [$inp]! |
| 1242 |
beq .Lcbc_dec_four |
| 1243 |
vld1.8 {@XMM[4]}, [$inp]! |
| 1244 |
cmp $len, #6 |
| 1245 |
blo .Lcbc_dec_five |
| 1246 |
vld1.8 {@XMM[5]}, [$inp]! |
| 1247 |
beq .Lcbc_dec_six |
| 1248 |
vld1.8 {@XMM[6]}, [$inp]! |
| 1249 |
sub $inp, $inp, #0x70 |
| 1250 |
|
| 1251 |
bl _bsaes_decrypt8 |
| 1252 |
|
| 1253 |
vldmia $fp, {@XMM[14]} @ reload IV |
| 1254 |
vld1.8 {@XMM[8]-@XMM[9]}, [$inp]! @ reload input |
| 1255 |
veor @XMM[0], @XMM[0], @XMM[14] @ ^= IV |
| 1256 |
vld1.8 {@XMM[10]-@XMM[11]}, [$inp]! |
| 1257 |
veor @XMM[1], @XMM[1], @XMM[8] |
| 1258 |
veor @XMM[6], @XMM[6], @XMM[9] |
| 1259 |
vld1.8 {@XMM[12]-@XMM[13]}, [$inp]! |
| 1260 |
veor @XMM[4], @XMM[4], @XMM[10] |
| 1261 |
veor @XMM[2], @XMM[2], @XMM[11] |
| 1262 |
vld1.8 {@XMM[15]}, [$inp]! |
| 1263 |
veor @XMM[7], @XMM[7], @XMM[12] |
| 1264 |
vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output |
| 1265 |
veor @XMM[3], @XMM[3], @XMM[13] |
| 1266 |
vst1.8 {@XMM[6]}, [$out]! |
| 1267 |
vst1.8 {@XMM[4]}, [$out]! |
| 1268 |
vst1.8 {@XMM[2]}, [$out]! |
| 1269 |
vst1.8 {@XMM[7]}, [$out]! |
| 1270 |
vst1.8 {@XMM[3]}, [$out]! |
| 1271 |
b .Lcbc_dec_done |
| 1272 |
.align 4 |
| 1273 |
.Lcbc_dec_six: |
| 1274 |
sub $inp, $inp, #0x60 |
| 1275 |
bl _bsaes_decrypt8 |
| 1276 |
vldmia $fp,{@XMM[14]} @ reload IV |
| 1277 |
vld1.8 {@XMM[8]-@XMM[9]}, [$inp]! @ reload input |
| 1278 |
veor @XMM[0], @XMM[0], @XMM[14] @ ^= IV |
| 1279 |
vld1.8 {@XMM[10]-@XMM[11]}, [$inp]! |
| 1280 |
veor @XMM[1], @XMM[1], @XMM[8] |
| 1281 |
veor @XMM[6], @XMM[6], @XMM[9] |
| 1282 |
vld1.8 {@XMM[12]}, [$inp]! |
| 1283 |
veor @XMM[4], @XMM[4], @XMM[10] |
| 1284 |
veor @XMM[2], @XMM[2], @XMM[11] |
| 1285 |
vld1.8 {@XMM[15]}, [$inp]! |
| 1286 |
veor @XMM[7], @XMM[7], @XMM[12] |
| 1287 |
vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output |
| 1288 |
vst1.8 {@XMM[6]}, [$out]! |
| 1289 |
vst1.8 {@XMM[4]}, [$out]! |
| 1290 |
vst1.8 {@XMM[2]}, [$out]! |
| 1291 |
vst1.8 {@XMM[7]}, [$out]! |
| 1292 |
b .Lcbc_dec_done |
| 1293 |
.align 4 |
| 1294 |
.Lcbc_dec_five: |
| 1295 |
sub $inp, $inp, #0x50 |
| 1296 |
bl _bsaes_decrypt8 |
| 1297 |
vldmia $fp, {@XMM[14]} @ reload IV |
| 1298 |
vld1.8 {@XMM[8]-@XMM[9]}, [$inp]! @ reload input |
| 1299 |
veor @XMM[0], @XMM[0], @XMM[14] @ ^= IV |
| 1300 |
vld1.8 {@XMM[10]-@XMM[11]}, [$inp]! |
| 1301 |
veor @XMM[1], @XMM[1], @XMM[8] |
| 1302 |
veor @XMM[6], @XMM[6], @XMM[9] |
| 1303 |
vld1.8 {@XMM[15]}, [$inp]! |
| 1304 |
veor @XMM[4], @XMM[4], @XMM[10] |
| 1305 |
vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output |
| 1306 |
veor @XMM[2], @XMM[2], @XMM[11] |
| 1307 |
vst1.8 {@XMM[6]}, [$out]! |
| 1308 |
vst1.8 {@XMM[4]}, [$out]! |
| 1309 |
vst1.8 {@XMM[2]}, [$out]! |
| 1310 |
b .Lcbc_dec_done |
| 1311 |
.align 4 |
| 1312 |
.Lcbc_dec_four: |
| 1313 |
sub $inp, $inp, #0x40 |
| 1314 |
bl _bsaes_decrypt8 |
| 1315 |
vldmia $fp, {@XMM[14]} @ reload IV |
| 1316 |
vld1.8 {@XMM[8]-@XMM[9]}, [$inp]! @ reload input |
| 1317 |
veor @XMM[0], @XMM[0], @XMM[14] @ ^= IV |
| 1318 |
vld1.8 {@XMM[10]}, [$inp]! |
| 1319 |
veor @XMM[1], @XMM[1], @XMM[8] |
| 1320 |
veor @XMM[6], @XMM[6], @XMM[9] |
| 1321 |
vld1.8 {@XMM[15]}, [$inp]! |
| 1322 |
veor @XMM[4], @XMM[4], @XMM[10] |
| 1323 |
vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output |
| 1324 |
vst1.8 {@XMM[6]}, [$out]! |
| 1325 |
vst1.8 {@XMM[4]}, [$out]! |
| 1326 |
b .Lcbc_dec_done |
| 1327 |
.align 4 |
| 1328 |
.Lcbc_dec_three: |
| 1329 |
sub $inp, $inp, #0x30 |
| 1330 |
bl _bsaes_decrypt8 |
| 1331 |
vldmia $fp, {@XMM[14]} @ reload IV |
| 1332 |
vld1.8 {@XMM[8]-@XMM[9]}, [$inp]! @ reload input |
| 1333 |
veor @XMM[0], @XMM[0], @XMM[14] @ ^= IV |
| 1334 |
vld1.8 {@XMM[15]}, [$inp]! |
| 1335 |
veor @XMM[1], @XMM[1], @XMM[8] |
| 1336 |
veor @XMM[6], @XMM[6], @XMM[9] |
| 1337 |
vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output |
| 1338 |
vst1.8 {@XMM[6]}, [$out]! |
| 1339 |
b .Lcbc_dec_done |
| 1340 |
.align 4 |
| 1341 |
.Lcbc_dec_two: |
| 1342 |
sub $inp, $inp, #0x20 |
| 1343 |
bl _bsaes_decrypt8 |
| 1344 |
vldmia $fp, {@XMM[14]} @ reload IV |
| 1345 |
vld1.8 {@XMM[8]}, [$inp]! @ reload input |
| 1346 |
veor @XMM[0], @XMM[0], @XMM[14] @ ^= IV |
| 1347 |
vld1.8 {@XMM[15]}, [$inp]! @ reload input |
| 1348 |
veor @XMM[1], @XMM[1], @XMM[8] |
| 1349 |
vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output |
| 1350 |
b .Lcbc_dec_done |
| 1351 |
.align 4 |
| 1352 |
.Lcbc_dec_one: |
| 1353 |
sub $inp, $inp, #0x10 |
| 1354 |
mov $rounds, $out @ save original out pointer |
| 1355 |
mov $out, $fp @ use the iv scratch space as out buffer |
| 1356 |
mov r2, $key |
| 1357 |
vmov @XMM[4],@XMM[15] @ just in case ensure that IV |
| 1358 |
vmov @XMM[5],@XMM[0] @ and input are preserved |
| 1359 |
bl AES_decrypt |
| 1360 |
vld1.8 {@XMM[0]}, [$fp,:64] @ load result |
| 1361 |
veor @XMM[0], @XMM[0], @XMM[4] @ ^= IV |
| 1362 |
vmov @XMM[15], @XMM[5] @ @XMM[5] holds input |
| 1363 |
vst1.8 {@XMM[0]}, [$rounds] @ write output |
| 1364 |
|
| 1365 |
.Lcbc_dec_done: |
| 1366 |
#ifndef BSAES_ASM_EXTENDED_KEY |
| 1367 |
vmov.i32 q0, #0 |
| 1368 |
vmov.i32 q1, #0 |
| 1369 |
.Lcbc_dec_bzero: @ wipe key schedule [if any] |
| 1370 |
vstmia $keysched!, {q0-q1} |
| 1371 |
cmp $keysched, $fp |
| 1372 |
bne .Lcbc_dec_bzero |
| 1373 |
#endif |
| 1374 |
|
| 1375 |
mov sp, $fp |
| 1376 |
add sp, #0x10 @ add sp,$fp,#0x10 is no good for thumb |
| 1377 |
vst1.8 {@XMM[15]}, [$ivp] @ return IV |
| 1378 |
VFP_ABI_POP |
| 1379 |
ldmia sp!, {r4-r10, pc} |
| 1380 |
.size bsaes_cbc_encrypt,.-bsaes_cbc_encrypt |
| 1381 |
___ |
| 1382 |
} |
| 1383 |
{ |
| 1384 |
my ($inp,$out,$len,$key, $ctr,$fp,$rounds)=(map("r$_",(0..3,8..10))); |
| 1385 |
my $const = "r6"; # shared with _bsaes_encrypt8_alt |
| 1386 |
my $keysched = "sp"; |
| 1387 |
|
| 1388 |
$code.=<<___; |
| 1389 |
.extern AES_encrypt |
| 1390 |
.global bsaes_ctr32_encrypt_blocks |
| 1391 |
.type bsaes_ctr32_encrypt_blocks,%function |
| 1392 |
.align 5 |
| 1393 |
bsaes_ctr32_encrypt_blocks: |
| 1394 |
cmp $len, #8 @ use plain AES for |
| 1395 |
blo .Lctr_enc_short @ small sizes |
| 1396 |
|
| 1397 |
mov ip, sp |
| 1398 |
stmdb sp!, {r4-r10, lr} |
| 1399 |
VFP_ABI_PUSH |
| 1400 |
ldr $ctr, [ip] @ ctr is 1st arg on the stack |
| 1401 |
sub sp, sp, #0x10 @ scratch space to carry over the ctr |
| 1402 |
mov $fp, sp @ save sp |
| 1403 |
|
| 1404 |
ldr $rounds, [$key, #240] @ get # of rounds |
| 1405 |
#ifndef BSAES_ASM_EXTENDED_KEY |
| 1406 |
@ allocate the key schedule on the stack |
| 1407 |
sub r12, sp, $rounds, lsl#7 @ 128 bytes per inner round key |
| 1408 |
add r12, #`128-32` @ size of bit-sliced key schedule |
| 1409 |
|
| 1410 |
@ populate the key schedule |
| 1411 |
mov r4, $key @ pass key |
| 1412 |
mov r5, $rounds @ pass # of rounds |
| 1413 |
mov sp, r12 @ sp is $keysched |
| 1414 |
bl _bsaes_key_convert |
| 1415 |
veor @XMM[7],@XMM[7],@XMM[15] @ fix up last round key |
| 1416 |
vstmia r12, {@XMM[7]} @ save last round key |
| 1417 |
|
| 1418 |
vld1.8 {@XMM[0]}, [$ctr] @ load counter |
| 1419 |
#ifdef __APPLE__ |
| 1420 |
mov $ctr, #:lower16:(.LREVM0SR-.LM0) |
| 1421 |
add $ctr, $const, $ctr |
| 1422 |
#else |
| 1423 |
add $ctr, $const, #.LREVM0SR-.LM0 @ borrow $ctr |
| 1424 |
#endif |
| 1425 |
vldmia $keysched, {@XMM[4]} @ load round0 key |
| 1426 |
#else |
| 1427 |
ldr r12, [$key, #244] |
| 1428 |
eors r12, #1 |
| 1429 |
beq 0f |
| 1430 |
|
| 1431 |
@ populate the key schedule |
| 1432 |
str r12, [$key, #244] |
| 1433 |
mov r4, $key @ pass key |
| 1434 |
mov r5, $rounds @ pass # of rounds |
| 1435 |
add r12, $key, #248 @ pass key schedule |
| 1436 |
bl _bsaes_key_convert |
| 1437 |
veor @XMM[7],@XMM[7],@XMM[15] @ fix up last round key |
| 1438 |
vstmia r12, {@XMM[7]} @ save last round key |
| 1439 |
|
| 1440 |
.align 2 |
| 1441 |
0: add r12, $key, #248 |
| 1442 |
vld1.8 {@XMM[0]}, [$ctr] @ load counter |
| 1443 |
adrl $ctr, .LREVM0SR @ borrow $ctr |
| 1444 |
vldmia r12, {@XMM[4]} @ load round0 key |
| 1445 |
sub sp, #0x10 @ place for adjusted round0 key |
| 1446 |
#endif |
| 1447 |
|
| 1448 |
vmov.i32 @XMM[8],#1 @ compose 1<<96 |
| 1449 |
veor @XMM[9],@XMM[9],@XMM[9] |
| 1450 |
vrev32.8 @XMM[0],@XMM[0] |
| 1451 |
vext.8 @XMM[8],@XMM[9],@XMM[8],#4 |
| 1452 |
vrev32.8 @XMM[4],@XMM[4] |
| 1453 |
vadd.u32 @XMM[9],@XMM[8],@XMM[8] @ compose 2<<96 |
| 1454 |
vstmia $keysched, {@XMM[4]} @ save adjusted round0 key |
| 1455 |
b .Lctr_enc_loop |
| 1456 |
|
| 1457 |
.align 4 |
| 1458 |
.Lctr_enc_loop: |
| 1459 |
vadd.u32 @XMM[10], @XMM[8], @XMM[9] @ compose 3<<96 |
| 1460 |
vadd.u32 @XMM[1], @XMM[0], @XMM[8] @ +1 |
| 1461 |
vadd.u32 @XMM[2], @XMM[0], @XMM[9] @ +2 |
| 1462 |
vadd.u32 @XMM[3], @XMM[0], @XMM[10] @ +3 |
| 1463 |
vadd.u32 @XMM[4], @XMM[1], @XMM[10] |
| 1464 |
vadd.u32 @XMM[5], @XMM[2], @XMM[10] |
| 1465 |
vadd.u32 @XMM[6], @XMM[3], @XMM[10] |
| 1466 |
vadd.u32 @XMM[7], @XMM[4], @XMM[10] |
| 1467 |
vadd.u32 @XMM[10], @XMM[5], @XMM[10] @ next counter |
| 1468 |
|
| 1469 |
@ Borrow prologue from _bsaes_encrypt8 to use the opportunity |
| 1470 |
@ to flip byte order in 32-bit counter |
| 1471 |
|
| 1472 |
vldmia $keysched, {@XMM[9]} @ load round0 key |
| 1473 |
#ifndef BSAES_ASM_EXTENDED_KEY |
| 1474 |
add r4, $keysched, #0x10 @ pass next round key |
| 1475 |
#else |
| 1476 |
add r4, $key, #`248+16` |
| 1477 |
#endif |
| 1478 |
vldmia $ctr, {@XMM[8]} @ .LREVM0SR |
| 1479 |
mov r5, $rounds @ pass rounds |
| 1480 |
vstmia $fp, {@XMM[10]} @ save next counter |
| 1481 |
#ifdef __APPLE__ |
| 1482 |
mov $const, #:lower16:(.LREVM0SR-.LSR) |
| 1483 |
sub $const, $ctr, $const |
| 1484 |
#else |
| 1485 |
sub $const, $ctr, #.LREVM0SR-.LSR @ pass constants |
| 1486 |
#endif |
| 1487 |
|
| 1488 |
bl _bsaes_encrypt8_alt |
| 1489 |
|
| 1490 |
subs $len, $len, #8 |
| 1491 |
blo .Lctr_enc_loop_done |
| 1492 |
|
| 1493 |
vld1.8 {@XMM[8]-@XMM[9]}, [$inp]! @ load input |
| 1494 |
vld1.8 {@XMM[10]-@XMM[11]}, [$inp]! |
| 1495 |
veor @XMM[0], @XMM[8] |
| 1496 |
veor @XMM[1], @XMM[9] |
| 1497 |
vld1.8 {@XMM[12]-@XMM[13]}, [$inp]! |
| 1498 |
veor @XMM[4], @XMM[10] |
| 1499 |
veor @XMM[6], @XMM[11] |
| 1500 |
vld1.8 {@XMM[14]-@XMM[15]}, [$inp]! |
| 1501 |
veor @XMM[3], @XMM[12] |
| 1502 |
vst1.8 {@XMM[0]-@XMM[1]}, [$out]! @ write output |
| 1503 |
veor @XMM[7], @XMM[13] |
| 1504 |
veor @XMM[2], @XMM[14] |
| 1505 |
vst1.8 {@XMM[4]}, [$out]! |
| 1506 |
veor @XMM[5], @XMM[15] |
| 1507 |
vst1.8 {@XMM[6]}, [$out]! |
| 1508 |
vmov.i32 @XMM[8], #1 @ compose 1<<96 |
| 1509 |
vst1.8 {@XMM[3]}, [$out]! |
| 1510 |
veor @XMM[9], @XMM[9], @XMM[9] |
| 1511 |
vst1.8 {@XMM[7]}, [$out]! |
| 1512 |
vext.8 @XMM[8], @XMM[9], @XMM[8], #4 |
| 1513 |
vst1.8 {@XMM[2]}, [$out]! |
| 1514 |
vadd.u32 @XMM[9],@XMM[8],@XMM[8] @ compose 2<<96 |
| 1515 |
vst1.8 {@XMM[5]}, [$out]! |
| 1516 |
vldmia $fp, {@XMM[0]} @ load counter |
| 1517 |
|
| 1518 |
bne .Lctr_enc_loop |
| 1519 |
b .Lctr_enc_done |
| 1520 |
|
| 1521 |
.align 4 |
| 1522 |
.Lctr_enc_loop_done: |
| 1523 |
add $len, $len, #8 |
| 1524 |
vld1.8 {@XMM[8]}, [$inp]! @ load input |
| 1525 |
veor @XMM[0], @XMM[8] |
| 1526 |
vst1.8 {@XMM[0]}, [$out]! @ write output |
| 1527 |
cmp $len, #2 |
| 1528 |
blo .Lctr_enc_done |
| 1529 |
vld1.8 {@XMM[9]}, [$inp]! |
| 1530 |
veor @XMM[1], @XMM[9] |
| 1531 |
vst1.8 {@XMM[1]}, [$out]! |
| 1532 |
beq .Lctr_enc_done |
| 1533 |
vld1.8 {@XMM[10]}, [$inp]! |
| 1534 |
veor @XMM[4], @XMM[10] |
| 1535 |
vst1.8 {@XMM[4]}, [$out]! |
| 1536 |
cmp $len, #4 |
| 1537 |
blo .Lctr_enc_done |
| 1538 |
vld1.8 {@XMM[11]}, [$inp]! |
| 1539 |
veor @XMM[6], @XMM[11] |
| 1540 |
vst1.8 {@XMM[6]}, [$out]! |
| 1541 |
beq .Lctr_enc_done |
| 1542 |
vld1.8 {@XMM[12]}, [$inp]! |
| 1543 |
veor @XMM[3], @XMM[12] |
| 1544 |
vst1.8 {@XMM[3]}, [$out]! |
| 1545 |
cmp $len, #6 |
| 1546 |
blo .Lctr_enc_done |
| 1547 |
vld1.8 {@XMM[13]}, [$inp]! |
| 1548 |
veor @XMM[7], @XMM[13] |
| 1549 |
vst1.8 {@XMM[7]}, [$out]! |
| 1550 |
beq .Lctr_enc_done |
| 1551 |
vld1.8 {@XMM[14]}, [$inp] |
| 1552 |
veor @XMM[2], @XMM[14] |
| 1553 |
vst1.8 {@XMM[2]}, [$out]! |
| 1554 |
|
| 1555 |
.Lctr_enc_done: |
| 1556 |
vmov.i32 q0, #0 |
| 1557 |
vmov.i32 q1, #0 |
| 1558 |
#ifndef BSAES_ASM_EXTENDED_KEY |
| 1559 |
.Lctr_enc_bzero: @ wipe key schedule [if any] |
| 1560 |
vstmia $keysched!, {q0-q1} |
| 1561 |
cmp $keysched, $fp |
| 1562 |
bne .Lctr_enc_bzero |
| 1563 |
#else |
| 1564 |
vstmia $keysched, {q0-q1} |
| 1565 |
#endif |
| 1566 |
|
| 1567 |
mov sp, $fp |
| 1568 |
add sp, #0x10 @ add sp,$fp,#0x10 is no good for thumb |
| 1569 |
VFP_ABI_POP |
| 1570 |
ldmia sp!, {r4-r10, pc} @ return |
| 1571 |
|
| 1572 |
.align 4 |
| 1573 |
.Lctr_enc_short: |
| 1574 |
ldr ip, [sp] @ ctr pointer is passed on stack |
| 1575 |
stmdb sp!, {r4-r8, lr} |
| 1576 |
|
| 1577 |
mov r4, $inp @ copy arguments |
| 1578 |
mov r5, $out |
| 1579 |
mov r6, $len |
| 1580 |
mov r7, $key |
| 1581 |
ldr r8, [ip, #12] @ load counter LSW |
| 1582 |
vld1.8 {@XMM[1]}, [ip] @ load whole counter value |
| 1583 |
#ifdef __ARMEL__ |
| 1584 |
rev r8, r8 |
| 1585 |
#endif |
| 1586 |
sub sp, sp, #0x10 |
| 1587 |
vst1.8 {@XMM[1]}, [sp] @ copy counter value |
| 1588 |
sub sp, sp, #0x10 |
| 1589 |
|
| 1590 |
.Lctr_enc_short_loop: |
| 1591 |
add r0, sp, #0x10 @ input counter value |
| 1592 |
mov r1, sp @ output on the stack |
| 1593 |
mov r2, r7 @ key |
| 1594 |
|
| 1595 |
bl AES_encrypt |
| 1596 |
|
| 1597 |
vld1.8 {@XMM[0]}, [r4]! @ load input |
| 1598 |
vld1.8 {@XMM[1]}, [sp] @ load encrypted counter |
| 1599 |
add r8, r8, #1 |
| 1600 |
#ifdef __ARMEL__ |
| 1601 |
rev r0, r8 |
| 1602 |
str r0, [sp, #0x1c] @ next counter value |
| 1603 |
#else |
| 1604 |
str r8, [sp, #0x1c] @ next counter value |
| 1605 |
#endif |
| 1606 |
veor @XMM[0],@XMM[0],@XMM[1] |
| 1607 |
vst1.8 {@XMM[0]}, [r5]! @ store output |
| 1608 |
subs r6, r6, #1 |
| 1609 |
bne .Lctr_enc_short_loop |
| 1610 |
|
| 1611 |
vmov.i32 q0, #0 |
| 1612 |
vmov.i32 q1, #0 |
| 1613 |
vstmia sp!, {q0-q1} |
| 1614 |
|
| 1615 |
ldmia sp!, {r4-r8, pc} |
| 1616 |
.size bsaes_ctr32_encrypt_blocks,.-bsaes_ctr32_encrypt_blocks |
| 1617 |
___ |
| 1618 |
} |
| 1619 |
{ |
| 1620 |
###################################################################### |
| 1621 |
# void bsaes_xts_[en|de]crypt(const char *inp,char *out,size_t len, |
| 1622 |
# const AES_KEY *key1, const AES_KEY *key2, |
| 1623 |
# const unsigned char iv[16]); |
| 1624 |
# |
| 1625 |
my ($inp,$out,$len,$key,$rounds,$magic,$fp)=(map("r$_",(7..10,1..3))); |
| 1626 |
my $const="r6"; # returned by _bsaes_key_convert |
| 1627 |
my $twmask=@XMM[5]; |
| 1628 |
my @T=@XMM[6..7]; |
| 1629 |
|
| 1630 |
$code.=<<___; |
| 1631 |
.globl bsaes_xts_encrypt |
| 1632 |
.type bsaes_xts_encrypt,%function |
| 1633 |
.align 4 |
| 1634 |
bsaes_xts_encrypt: |
| 1635 |
mov ip, sp |
| 1636 |
stmdb sp!, {r4-r10, lr} @ 0x20 |
| 1637 |
VFP_ABI_PUSH |
| 1638 |
mov r6, sp @ future $fp |
| 1639 |
|
| 1640 |
mov $inp, r0 |
| 1641 |
mov $out, r1 |
| 1642 |
mov $len, r2 |
| 1643 |
mov $key, r3 |
| 1644 |
|
| 1645 |
sub r0, sp, #0x10 @ 0x10 |
| 1646 |
bic r0, #0xf @ align at 16 bytes |
| 1647 |
mov sp, r0 |
| 1648 |
|
| 1649 |
#ifdef XTS_CHAIN_TWEAK |
| 1650 |
ldr r0, [ip] @ pointer to input tweak |
| 1651 |
#else |
| 1652 |
@ generate initial tweak |
| 1653 |
ldr r0, [ip, #4] @ iv[] |
| 1654 |
mov r1, sp |
| 1655 |
ldr r2, [ip, #0] @ key2 |
| 1656 |
bl AES_encrypt |
| 1657 |
mov r0,sp @ pointer to initial tweak |
| 1658 |
#endif |
| 1659 |
|
| 1660 |
ldr $rounds, [$key, #240] @ get # of rounds |
| 1661 |
mov $fp, r6 |
| 1662 |
#ifndef BSAES_ASM_EXTENDED_KEY |
| 1663 |
@ allocate the key schedule on the stack |
| 1664 |
sub r12, sp, $rounds, lsl#7 @ 128 bytes per inner round key |
| 1665 |
@ add r12, #`128-32` @ size of bit-sliced key schedule |
| 1666 |
sub r12, #`32+16` @ place for tweak[9] |
| 1667 |
|
| 1668 |
@ populate the key schedule |
| 1669 |
mov r4, $key @ pass key |
| 1670 |
mov r5, $rounds @ pass # of rounds |
| 1671 |
mov sp, r12 |
| 1672 |
add r12, #0x90 @ pass key schedule |
| 1673 |
bl _bsaes_key_convert |
| 1674 |
veor @XMM[7], @XMM[7], @XMM[15] @ fix up last round key |
| 1675 |
vstmia r12, {@XMM[7]} @ save last round key |
| 1676 |
#else |
| 1677 |
ldr r12, [$key, #244] |
| 1678 |
eors r12, #1 |
| 1679 |
beq 0f |
| 1680 |
|
| 1681 |
str r12, [$key, #244] |
| 1682 |
mov r4, $key @ pass key |
| 1683 |
mov r5, $rounds @ pass # of rounds |
| 1684 |
add r12, $key, #248 @ pass key schedule |
| 1685 |
bl _bsaes_key_convert |
| 1686 |
veor @XMM[7], @XMM[7], @XMM[15] @ fix up last round key |
| 1687 |
vstmia r12, {@XMM[7]} |
| 1688 |
|
| 1689 |
.align 2 |
| 1690 |
0: sub sp, #0x90 @ place for tweak[9] |
| 1691 |
#endif |
| 1692 |
|
| 1693 |
vld1.8 {@XMM[8]}, [r0] @ initial tweak |
| 1694 |
adr $magic, .Lxts_magic |
| 1695 |
|
| 1696 |
subs $len, #0x80 |
| 1697 |
blo .Lxts_enc_short |
| 1698 |
b .Lxts_enc_loop |
| 1699 |
|
| 1700 |
.align 4 |
| 1701 |
.Lxts_enc_loop: |
| 1702 |
vldmia $magic, {$twmask} @ load XTS magic |
| 1703 |
vshr.s64 @T[0], @XMM[8], #63 |
| 1704 |
mov r0, sp |
| 1705 |
vand @T[0], @T[0], $twmask |
| 1706 |
___ |
| 1707 |
for($i=9;$i<16;$i++) { |
| 1708 |
$code.=<<___; |
| 1709 |
vadd.u64 @XMM[$i], @XMM[$i-1], @XMM[$i-1] |
| 1710 |
vst1.64 {@XMM[$i-1]}, [r0,:128]! |
| 1711 |
vswp `&Dhi("@T[0]")`,`&Dlo("@T[0]")` |
| 1712 |
vshr.s64 @T[1], @XMM[$i], #63 |
| 1713 |
veor @XMM[$i], @XMM[$i], @T[0] |
| 1714 |
vand @T[1], @T[1], $twmask |
| 1715 |
___ |
| 1716 |
@T=reverse(@T); |
| 1717 |
|
| 1718 |
$code.=<<___ if ($i>=10); |
| 1719 |
vld1.8 {@XMM[$i-10]}, [$inp]! |
| 1720 |
___ |
| 1721 |
$code.=<<___ if ($i>=11); |
| 1722 |
veor @XMM[$i-11], @XMM[$i-11], @XMM[$i-3] |
| 1723 |
___ |
| 1724 |
} |
| 1725 |
$code.=<<___; |
| 1726 |
vadd.u64 @XMM[8], @XMM[15], @XMM[15] |
| 1727 |
vst1.64 {@XMM[15]}, [r0,:128]! |
| 1728 |
vswp `&Dhi("@T[0]")`,`&Dlo("@T[0]")` |
| 1729 |
veor @XMM[8], @XMM[8], @T[0] |
| 1730 |
vst1.64 {@XMM[8]}, [r0,:128] @ next round tweak |
| 1731 |
|
| 1732 |
vld1.8 {@XMM[6]-@XMM[7]}, [$inp]! |
| 1733 |
veor @XMM[5], @XMM[5], @XMM[13] |
| 1734 |
#ifndef BSAES_ASM_EXTENDED_KEY |
| 1735 |
add r4, sp, #0x90 @ pass key schedule |
| 1736 |
#else |
| 1737 |
add r4, $key, #248 @ pass key schedule |
| 1738 |
#endif |
| 1739 |
veor @XMM[6], @XMM[6], @XMM[14] |
| 1740 |
mov r5, $rounds @ pass rounds |
| 1741 |
veor @XMM[7], @XMM[7], @XMM[15] |
| 1742 |
mov r0, sp |
| 1743 |
|
| 1744 |
bl _bsaes_encrypt8 |
| 1745 |
|
| 1746 |
vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]! |
| 1747 |
vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]! |
| 1748 |
veor @XMM[0], @XMM[0], @XMM[ 8] |
| 1749 |
vld1.64 {@XMM[12]-@XMM[13]}, [r0,:128]! |
| 1750 |
veor @XMM[1], @XMM[1], @XMM[ 9] |
| 1751 |
veor @XMM[8], @XMM[4], @XMM[10] |
| 1752 |
vst1.8 {@XMM[0]-@XMM[1]}, [$out]! |
| 1753 |
veor @XMM[9], @XMM[6], @XMM[11] |
| 1754 |
vld1.64 {@XMM[14]-@XMM[15]}, [r0,:128]! |
| 1755 |
veor @XMM[10], @XMM[3], @XMM[12] |
| 1756 |
vst1.8 {@XMM[8]-@XMM[9]}, [$out]! |
| 1757 |
veor @XMM[11], @XMM[7], @XMM[13] |
| 1758 |
veor @XMM[12], @XMM[2], @XMM[14] |
| 1759 |
vst1.8 {@XMM[10]-@XMM[11]}, [$out]! |
| 1760 |
veor @XMM[13], @XMM[5], @XMM[15] |
| 1761 |
vst1.8 {@XMM[12]-@XMM[13]}, [$out]! |
| 1762 |
|
| 1763 |
vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak |
| 1764 |
|
| 1765 |
subs $len, #0x80 |
| 1766 |
bpl .Lxts_enc_loop |
| 1767 |
|
| 1768 |
.Lxts_enc_short: |
| 1769 |
adds $len, #0x70 |
| 1770 |
bmi .Lxts_enc_done |
| 1771 |
|
| 1772 |
vldmia $magic, {$twmask} @ load XTS magic |
| 1773 |
vshr.s64 @T[0], @XMM[8], #63 |
| 1774 |
mov r0, sp |
| 1775 |
vand @T[0], @T[0], $twmask |
| 1776 |
___ |
| 1777 |
for($i=9;$i<16;$i++) { |
| 1778 |
$code.=<<___; |
| 1779 |
vadd.u64 @XMM[$i], @XMM[$i-1], @XMM[$i-1] |
| 1780 |
vst1.64 {@XMM[$i-1]}, [r0,:128]! |
| 1781 |
vswp `&Dhi("@T[0]")`,`&Dlo("@T[0]")` |
| 1782 |
vshr.s64 @T[1], @XMM[$i], #63 |
| 1783 |
veor @XMM[$i], @XMM[$i], @T[0] |
| 1784 |
vand @T[1], @T[1], $twmask |
| 1785 |
___ |
| 1786 |
@T=reverse(@T); |
| 1787 |
|
| 1788 |
$code.=<<___ if ($i>=10); |
| 1789 |
vld1.8 {@XMM[$i-10]}, [$inp]! |
| 1790 |
subs $len, #0x10 |
| 1791 |
bmi .Lxts_enc_`$i-9` |
| 1792 |
___ |
| 1793 |
$code.=<<___ if ($i>=11); |
| 1794 |
veor @XMM[$i-11], @XMM[$i-11], @XMM[$i-3] |
| 1795 |
___ |
| 1796 |
} |
| 1797 |
$code.=<<___; |
| 1798 |
sub $len, #0x10 |
| 1799 |
vst1.64 {@XMM[15]}, [r0,:128] @ next round tweak |
| 1800 |
|
| 1801 |
vld1.8 {@XMM[6]}, [$inp]! |
| 1802 |
veor @XMM[5], @XMM[5], @XMM[13] |
| 1803 |
#ifndef BSAES_ASM_EXTENDED_KEY |
| 1804 |
add r4, sp, #0x90 @ pass key schedule |
| 1805 |
#else |
| 1806 |
add r4, $key, #248 @ pass key schedule |
| 1807 |
#endif |
| 1808 |
veor @XMM[6], @XMM[6], @XMM[14] |
| 1809 |
mov r5, $rounds @ pass rounds |
| 1810 |
mov r0, sp |
| 1811 |
|
| 1812 |
bl _bsaes_encrypt8 |
| 1813 |
|
| 1814 |
vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]! |
| 1815 |
vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]! |
| 1816 |
veor @XMM[0], @XMM[0], @XMM[ 8] |
| 1817 |
vld1.64 {@XMM[12]-@XMM[13]}, [r0,:128]! |
| 1818 |
veor @XMM[1], @XMM[1], @XMM[ 9] |
| 1819 |
veor @XMM[8], @XMM[4], @XMM[10] |
| 1820 |
vst1.8 {@XMM[0]-@XMM[1]}, [$out]! |
| 1821 |
veor @XMM[9], @XMM[6], @XMM[11] |
| 1822 |
vld1.64 {@XMM[14]}, [r0,:128]! |
| 1823 |
veor @XMM[10], @XMM[3], @XMM[12] |
| 1824 |
vst1.8 {@XMM[8]-@XMM[9]}, [$out]! |
| 1825 |
veor @XMM[11], @XMM[7], @XMM[13] |
| 1826 |
veor @XMM[12], @XMM[2], @XMM[14] |
| 1827 |
vst1.8 {@XMM[10]-@XMM[11]}, [$out]! |
| 1828 |
vst1.8 {@XMM[12]}, [$out]! |
| 1829 |
|
| 1830 |
vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak |
| 1831 |
b .Lxts_enc_done |
| 1832 |
.align 4 |
| 1833 |
.Lxts_enc_6: |
| 1834 |
vst1.64 {@XMM[14]}, [r0,:128] @ next round tweak |
| 1835 |
|
| 1836 |
veor @XMM[4], @XMM[4], @XMM[12] |
| 1837 |
#ifndef BSAES_ASM_EXTENDED_KEY |
| 1838 |
add r4, sp, #0x90 @ pass key schedule |
| 1839 |
#else |
| 1840 |
add r4, $key, #248 @ pass key schedule |
| 1841 |
#endif |
| 1842 |
veor @XMM[5], @XMM[5], @XMM[13] |
| 1843 |
mov r5, $rounds @ pass rounds |
| 1844 |
mov r0, sp |
| 1845 |
|
| 1846 |
bl _bsaes_encrypt8 |
| 1847 |
|
| 1848 |
vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]! |
| 1849 |
vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]! |
| 1850 |
veor @XMM[0], @XMM[0], @XMM[ 8] |
| 1851 |
vld1.64 {@XMM[12]-@XMM[13]}, [r0,:128]! |
| 1852 |
veor @XMM[1], @XMM[1], @XMM[ 9] |
| 1853 |
veor @XMM[8], @XMM[4], @XMM[10] |
| 1854 |
vst1.8 {@XMM[0]-@XMM[1]}, [$out]! |
| 1855 |
veor @XMM[9], @XMM[6], @XMM[11] |
| 1856 |
veor @XMM[10], @XMM[3], @XMM[12] |
| 1857 |
vst1.8 {@XMM[8]-@XMM[9]}, [$out]! |
| 1858 |
veor @XMM[11], @XMM[7], @XMM[13] |
| 1859 |
vst1.8 {@XMM[10]-@XMM[11]}, [$out]! |
| 1860 |
|
| 1861 |
vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak |
| 1862 |
b .Lxts_enc_done |
| 1863 |
|
| 1864 |
@ put this in range for both ARM and Thumb mode adr instructions |
| 1865 |
.align 5 |
| 1866 |
.Lxts_magic: |
| 1867 |
.quad 1, 0x87 |
| 1868 |
|
| 1869 |
.align 5 |
| 1870 |
.Lxts_enc_5: |
| 1871 |
vst1.64 {@XMM[13]}, [r0,:128] @ next round tweak |
| 1872 |
|
| 1873 |
veor @XMM[3], @XMM[3], @XMM[11] |
| 1874 |
#ifndef BSAES_ASM_EXTENDED_KEY |
| 1875 |
add r4, sp, #0x90 @ pass key schedule |
| 1876 |
#else |
| 1877 |
add r4, $key, #248 @ pass key schedule |
| 1878 |
#endif |
| 1879 |
veor @XMM[4], @XMM[4], @XMM[12] |
| 1880 |
mov r5, $rounds @ pass rounds |
| 1881 |
mov r0, sp |
| 1882 |
|
| 1883 |
bl _bsaes_encrypt8 |
| 1884 |
|
| 1885 |
vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]! |
| 1886 |
vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]! |
| 1887 |
veor @XMM[0], @XMM[0], @XMM[ 8] |
| 1888 |
vld1.64 {@XMM[12]}, [r0,:128]! |
| 1889 |
veor @XMM[1], @XMM[1], @XMM[ 9] |
| 1890 |
veor @XMM[8], @XMM[4], @XMM[10] |
| 1891 |
vst1.8 {@XMM[0]-@XMM[1]}, [$out]! |
| 1892 |
veor @XMM[9], @XMM[6], @XMM[11] |
| 1893 |
veor @XMM[10], @XMM[3], @XMM[12] |
| 1894 |
vst1.8 {@XMM[8]-@XMM[9]}, [$out]! |
| 1895 |
vst1.8 {@XMM[10]}, [$out]! |
| 1896 |
|
| 1897 |
vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak |
| 1898 |
b .Lxts_enc_done |
| 1899 |
.align 4 |
| 1900 |
.Lxts_enc_4: |
| 1901 |
vst1.64 {@XMM[12]}, [r0,:128] @ next round tweak |
| 1902 |
|
| 1903 |
veor @XMM[2], @XMM[2], @XMM[10] |
| 1904 |
#ifndef BSAES_ASM_EXTENDED_KEY |
| 1905 |
add r4, sp, #0x90 @ pass key schedule |
| 1906 |
#else |
| 1907 |
add r4, $key, #248 @ pass key schedule |
| 1908 |
#endif |
| 1909 |
veor @XMM[3], @XMM[3], @XMM[11] |
| 1910 |
mov r5, $rounds @ pass rounds |
| 1911 |
mov r0, sp |
| 1912 |
|
| 1913 |
bl _bsaes_encrypt8 |
| 1914 |
|
| 1915 |
vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]! |
| 1916 |
vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]! |
| 1917 |
veor @XMM[0], @XMM[0], @XMM[ 8] |
| 1918 |
veor @XMM[1], @XMM[1], @XMM[ 9] |
| 1919 |
veor @XMM[8], @XMM[4], @XMM[10] |
| 1920 |
vst1.8 {@XMM[0]-@XMM[1]}, [$out]! |
| 1921 |
veor @XMM[9], @XMM[6], @XMM[11] |
| 1922 |
vst1.8 {@XMM[8]-@XMM[9]}, [$out]! |
| 1923 |
|
| 1924 |
vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak |
| 1925 |
b .Lxts_enc_done |
| 1926 |
.align 4 |
| 1927 |
.Lxts_enc_3: |
| 1928 |
vst1.64 {@XMM[11]}, [r0,:128] @ next round tweak |
| 1929 |
|
| 1930 |
veor @XMM[1], @XMM[1], @XMM[9] |
| 1931 |
#ifndef BSAES_ASM_EXTENDED_KEY |
| 1932 |
add r4, sp, #0x90 @ pass key schedule |
| 1933 |
#else |
| 1934 |
add r4, $key, #248 @ pass key schedule |
| 1935 |
#endif |
| 1936 |
veor @XMM[2], @XMM[2], @XMM[10] |
| 1937 |
mov r5, $rounds @ pass rounds |
| 1938 |
mov r0, sp |
| 1939 |
|
| 1940 |
bl _bsaes_encrypt8 |
| 1941 |
|
| 1942 |
vld1.64 {@XMM[8]-@XMM[9]}, [r0,:128]! |
| 1943 |
vld1.64 {@XMM[10]}, [r0,:128]! |
| 1944 |
veor @XMM[0], @XMM[0], @XMM[ 8] |
| 1945 |
veor @XMM[1], @XMM[1], @XMM[ 9] |
| 1946 |
veor @XMM[8], @XMM[4], @XMM[10] |
| 1947 |
vst1.8 {@XMM[0]-@XMM[1]}, [$out]! |
| 1948 |
vst1.8 {@XMM[8]}, [$out]! |
| 1949 |
|
| 1950 |
vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak |
| 1951 |
b .Lxts_enc_done |
| 1952 |
.align 4 |
| 1953 |
.Lxts_enc_2: |
| 1954 |
vst1.64 {@XMM[10]}, [r0,:128] @ next round tweak |
| 1955 |
|
| 1956 |
veor @XMM[0], @XMM[0], @XMM[8] |
| 1957 |
#ifndef BSAES_ASM_EXTENDED_KEY |
| 1958 |
add r4, sp, #0x90 @ pass key schedule |
| 1959 |
#else |
| 1960 |
add r4, $key, #248 @ pass key schedule |
| 1961 |
#endif |
| 1962 |
veor @XMM[1], @XMM[1], @XMM[9] |
| 1963 |
mov r5, $rounds @ pass rounds |
| 1964 |
mov r0, sp |
| 1965 |
|
| 1966 |
bl _bsaes_encrypt8 |
| 1967 |
|
| 1968 |
vld1.64 {@XMM[8]-@XMM[9]}, [r0,:128]! |
| 1969 |
veor @XMM[0], @XMM[0], @XMM[ 8] |
| 1970 |
veor @XMM[1], @XMM[1], @XMM[ 9] |
| 1971 |
vst1.8 {@XMM[0]-@XMM[1]}, [$out]! |
| 1972 |
|
| 1973 |
vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak |
| 1974 |
b .Lxts_enc_done |
| 1975 |
.align 4 |
| 1976 |
.Lxts_enc_1: |
| 1977 |
mov r0, sp |
| 1978 |
veor @XMM[0], @XMM[8] |
| 1979 |
mov r1, sp |
| 1980 |
vst1.8 {@XMM[0]}, [sp,:128] |
| 1981 |
mov r2, $key |
| 1982 |
mov r4, $fp @ preserve fp |
| 1983 |
|
| 1984 |
bl AES_encrypt |
| 1985 |
|
| 1986 |
vld1.8 {@XMM[0]}, [sp,:128] |
| 1987 |
veor @XMM[0], @XMM[0], @XMM[8] |
| 1988 |
vst1.8 {@XMM[0]}, [$out]! |
| 1989 |
mov $fp, r4 |
| 1990 |
|
| 1991 |
vmov @XMM[8], @XMM[9] @ next round tweak |
| 1992 |
|
| 1993 |
.Lxts_enc_done: |
| 1994 |
#ifndef XTS_CHAIN_TWEAK |
| 1995 |
adds $len, #0x10 |
| 1996 |
beq .Lxts_enc_ret |
| 1997 |
sub r6, $out, #0x10 |
| 1998 |
|
| 1999 |
.Lxts_enc_steal: |
| 2000 |
ldrb r0, [$inp], #1 |
| 2001 |
ldrb r1, [$out, #-0x10] |
| 2002 |
strb r0, [$out, #-0x10] |
| 2003 |
strb r1, [$out], #1 |
| 2004 |
|
| 2005 |
subs $len, #1 |
| 2006 |
bhi .Lxts_enc_steal |
| 2007 |
|
| 2008 |
vld1.8 {@XMM[0]}, [r6] |
| 2009 |
mov r0, sp |
| 2010 |
veor @XMM[0], @XMM[0], @XMM[8] |
| 2011 |
mov r1, sp |
| 2012 |
vst1.8 {@XMM[0]}, [sp,:128] |
| 2013 |
mov r2, $key |
| 2014 |
mov r4, $fp @ preserve fp |
| 2015 |
|
| 2016 |
bl AES_encrypt |
| 2017 |
|
| 2018 |
vld1.8 {@XMM[0]}, [sp,:128] |
| 2019 |
veor @XMM[0], @XMM[0], @XMM[8] |
| 2020 |
vst1.8 {@XMM[0]}, [r6] |
| 2021 |
mov $fp, r4 |
| 2022 |
#endif |
| 2023 |
|
| 2024 |
.Lxts_enc_ret: |
| 2025 |
bic r0, $fp, #0xf |
| 2026 |
vmov.i32 q0, #0 |
| 2027 |
vmov.i32 q1, #0 |
| 2028 |
#ifdef XTS_CHAIN_TWEAK |
| 2029 |
ldr r1, [$fp, #0x20+VFP_ABI_FRAME] @ chain tweak |
| 2030 |
#endif |
| 2031 |
.Lxts_enc_bzero: @ wipe key schedule [if any] |
| 2032 |
vstmia sp!, {q0-q1} |
| 2033 |
cmp sp, r0 |
| 2034 |
bne .Lxts_enc_bzero |
| 2035 |
|
| 2036 |
mov sp, $fp |
| 2037 |
#ifdef XTS_CHAIN_TWEAK |
| 2038 |
vst1.8 {@XMM[8]}, [r1] |
| 2039 |
#endif |
| 2040 |
VFP_ABI_POP |
| 2041 |
ldmia sp!, {r4-r10, pc} @ return |
| 2042 |
|
| 2043 |
.size bsaes_xts_encrypt,.-bsaes_xts_encrypt |
| 2044 |
|
| 2045 |
.globl bsaes_xts_decrypt |
| 2046 |
.type bsaes_xts_decrypt,%function |
| 2047 |
.align 4 |
| 2048 |
bsaes_xts_decrypt: |
| 2049 |
mov ip, sp |
| 2050 |
stmdb sp!, {r4-r10, lr} @ 0x20 |
| 2051 |
VFP_ABI_PUSH |
| 2052 |
mov r6, sp @ future $fp |
| 2053 |
|
| 2054 |
mov $inp, r0 |
| 2055 |
mov $out, r1 |
| 2056 |
mov $len, r2 |
| 2057 |
mov $key, r3 |
| 2058 |
|
| 2059 |
sub r0, sp, #0x10 @ 0x10 |
| 2060 |
bic r0, #0xf @ align at 16 bytes |
| 2061 |
mov sp, r0 |
| 2062 |
|
| 2063 |
#ifdef XTS_CHAIN_TWEAK |
| 2064 |
ldr r0, [ip] @ pointer to input tweak |
| 2065 |
#else |
| 2066 |
@ generate initial tweak |
| 2067 |
ldr r0, [ip, #4] @ iv[] |
| 2068 |
mov r1, sp |
| 2069 |
ldr r2, [ip, #0] @ key2 |
| 2070 |
bl AES_encrypt |
| 2071 |
mov r0, sp @ pointer to initial tweak |
| 2072 |
#endif |
| 2073 |
|
| 2074 |
ldr $rounds, [$key, #240] @ get # of rounds |
| 2075 |
mov $fp, r6 |
| 2076 |
#ifndef BSAES_ASM_EXTENDED_KEY |
| 2077 |
@ allocate the key schedule on the stack |
| 2078 |
sub r12, sp, $rounds, lsl#7 @ 128 bytes per inner round key |
| 2079 |
@ add r12, #`128-32` @ size of bit-sliced key schedule |
| 2080 |
sub r12, #`32+16` @ place for tweak[9] |
| 2081 |
|
| 2082 |
@ populate the key schedule |
| 2083 |
mov r4, $key @ pass key |
| 2084 |
mov r5, $rounds @ pass # of rounds |
| 2085 |
mov sp, r12 |
| 2086 |
add r12, #0x90 @ pass key schedule |
| 2087 |
bl _bsaes_key_convert |
| 2088 |
add r4, sp, #0x90 |
| 2089 |
vldmia r4, {@XMM[6]} |
| 2090 |
vstmia r12, {@XMM[15]} @ save last round key |
| 2091 |
veor @XMM[7], @XMM[7], @XMM[6] @ fix up round 0 key |
| 2092 |
vstmia r4, {@XMM[7]} |
| 2093 |
#else |
| 2094 |
ldr r12, [$key, #244] |
| 2095 |
eors r12, #1 |
| 2096 |
beq 0f |
| 2097 |
|
| 2098 |
str r12, [$key, #244] |
| 2099 |
mov r4, $key @ pass key |
| 2100 |
mov r5, $rounds @ pass # of rounds |
| 2101 |
add r12, $key, #248 @ pass key schedule |
| 2102 |
bl _bsaes_key_convert |
| 2103 |
add r4, $key, #248 |
| 2104 |
vldmia r4, {@XMM[6]} |
| 2105 |
vstmia r12, {@XMM[15]} @ save last round key |
| 2106 |
veor @XMM[7], @XMM[7], @XMM[6] @ fix up round 0 key |
| 2107 |
vstmia r4, {@XMM[7]} |
| 2108 |
|
| 2109 |
.align 2 |
| 2110 |
0: sub sp, #0x90 @ place for tweak[9] |
| 2111 |
#endif |
| 2112 |
vld1.8 {@XMM[8]}, [r0] @ initial tweak |
| 2113 |
adr $magic, .Lxts_magic |
| 2114 |
|
| 2115 |
#ifndef XTS_CHAIN_TWEAK |
| 2116 |
tst $len, #0xf @ if not multiple of 16 |
| 2117 |
it ne @ Thumb2 thing, sanity check in ARM |
| 2118 |
subne $len, #0x10 @ subtract another 16 bytes |
| 2119 |
#endif |
| 2120 |
subs $len, #0x80 |
| 2121 |
|
| 2122 |
blo .Lxts_dec_short |
| 2123 |
b .Lxts_dec_loop |
| 2124 |
|
| 2125 |
.align 4 |
| 2126 |
.Lxts_dec_loop: |
| 2127 |
vldmia $magic, {$twmask} @ load XTS magic |
| 2128 |
vshr.s64 @T[0], @XMM[8], #63 |
| 2129 |
mov r0, sp |
| 2130 |
vand @T[0], @T[0], $twmask |
| 2131 |
___ |
| 2132 |
for($i=9;$i<16;$i++) { |
| 2133 |
$code.=<<___; |
| 2134 |
vadd.u64 @XMM[$i], @XMM[$i-1], @XMM[$i-1] |
| 2135 |
vst1.64 {@XMM[$i-1]}, [r0,:128]! |
| 2136 |
vswp `&Dhi("@T[0]")`,`&Dlo("@T[0]")` |
| 2137 |
vshr.s64 @T[1], @XMM[$i], #63 |
| 2138 |
veor @XMM[$i], @XMM[$i], @T[0] |
| 2139 |
vand @T[1], @T[1], $twmask |
| 2140 |
___ |
| 2141 |
@T=reverse(@T); |
| 2142 |
|
| 2143 |
$code.=<<___ if ($i>=10); |
| 2144 |
vld1.8 {@XMM[$i-10]}, [$inp]! |
| 2145 |
___ |
| 2146 |
$code.=<<___ if ($i>=11); |
| 2147 |
veor @XMM[$i-11], @XMM[$i-11], @XMM[$i-3] |
| 2148 |
___ |
| 2149 |
} |
| 2150 |
$code.=<<___; |
| 2151 |
vadd.u64 @XMM[8], @XMM[15], @XMM[15] |
| 2152 |
vst1.64 {@XMM[15]}, [r0,:128]! |
| 2153 |
vswp `&Dhi("@T[0]")`,`&Dlo("@T[0]")` |
| 2154 |
veor @XMM[8], @XMM[8], @T[0] |
| 2155 |
vst1.64 {@XMM[8]}, [r0,:128] @ next round tweak |
| 2156 |
|
| 2157 |
vld1.8 {@XMM[6]-@XMM[7]}, [$inp]! |
| 2158 |
veor @XMM[5], @XMM[5], @XMM[13] |
| 2159 |
#ifndef BSAES_ASM_EXTENDED_KEY |
| 2160 |
add r4, sp, #0x90 @ pass key schedule |
| 2161 |
#else |
| 2162 |
add r4, $key, #248 @ pass key schedule |
| 2163 |
#endif |
| 2164 |
veor @XMM[6], @XMM[6], @XMM[14] |
| 2165 |
mov r5, $rounds @ pass rounds |
| 2166 |
veor @XMM[7], @XMM[7], @XMM[15] |
| 2167 |
mov r0, sp |
| 2168 |
|
| 2169 |
bl _bsaes_decrypt8 |
| 2170 |
|
| 2171 |
vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]! |
| 2172 |
vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]! |
| 2173 |
veor @XMM[0], @XMM[0], @XMM[ 8] |
| 2174 |
vld1.64 {@XMM[12]-@XMM[13]}, [r0,:128]! |
| 2175 |
veor @XMM[1], @XMM[1], @XMM[ 9] |
| 2176 |
veor @XMM[8], @XMM[6], @XMM[10] |
| 2177 |
vst1.8 {@XMM[0]-@XMM[1]}, [$out]! |
| 2178 |
veor @XMM[9], @XMM[4], @XMM[11] |
| 2179 |
vld1.64 {@XMM[14]-@XMM[15]}, [r0,:128]! |
| 2180 |
veor @XMM[10], @XMM[2], @XMM[12] |
| 2181 |
vst1.8 {@XMM[8]-@XMM[9]}, [$out]! |
| 2182 |
veor @XMM[11], @XMM[7], @XMM[13] |
| 2183 |
veor @XMM[12], @XMM[3], @XMM[14] |
| 2184 |
vst1.8 {@XMM[10]-@XMM[11]}, [$out]! |
| 2185 |
veor @XMM[13], @XMM[5], @XMM[15] |
| 2186 |
vst1.8 {@XMM[12]-@XMM[13]}, [$out]! |
| 2187 |
|
| 2188 |
vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak |
| 2189 |
|
| 2190 |
subs $len, #0x80 |
| 2191 |
bpl .Lxts_dec_loop |
| 2192 |
|
| 2193 |
.Lxts_dec_short: |
| 2194 |
adds $len, #0x70 |
| 2195 |
bmi .Lxts_dec_done |
| 2196 |
|
| 2197 |
vldmia $magic, {$twmask} @ load XTS magic |
| 2198 |
vshr.s64 @T[0], @XMM[8], #63 |
| 2199 |
mov r0, sp |
| 2200 |
vand @T[0], @T[0], $twmask |
| 2201 |
___ |
| 2202 |
for($i=9;$i<16;$i++) { |
| 2203 |
$code.=<<___; |
| 2204 |
vadd.u64 @XMM[$i], @XMM[$i-1], @XMM[$i-1] |
| 2205 |
vst1.64 {@XMM[$i-1]}, [r0,:128]! |
| 2206 |
vswp `&Dhi("@T[0]")`,`&Dlo("@T[0]")` |
| 2207 |
vshr.s64 @T[1], @XMM[$i], #63 |
| 2208 |
veor @XMM[$i], @XMM[$i], @T[0] |
| 2209 |
vand @T[1], @T[1], $twmask |
| 2210 |
___ |
| 2211 |
@T=reverse(@T); |
| 2212 |
|
| 2213 |
$code.=<<___ if ($i>=10); |
| 2214 |
vld1.8 {@XMM[$i-10]}, [$inp]! |
| 2215 |
subs $len, #0x10 |
| 2216 |
bmi .Lxts_dec_`$i-9` |
| 2217 |
___ |
| 2218 |
$code.=<<___ if ($i>=11); |
| 2219 |
veor @XMM[$i-11], @XMM[$i-11], @XMM[$i-3] |
| 2220 |
___ |
| 2221 |
} |
| 2222 |
$code.=<<___; |
| 2223 |
sub $len, #0x10 |
| 2224 |
vst1.64 {@XMM[15]}, [r0,:128] @ next round tweak |
| 2225 |
|
| 2226 |
vld1.8 {@XMM[6]}, [$inp]! |
| 2227 |
veor @XMM[5], @XMM[5], @XMM[13] |
| 2228 |
#ifndef BSAES_ASM_EXTENDED_KEY |
| 2229 |
add r4, sp, #0x90 @ pass key schedule |
| 2230 |
#else |
| 2231 |
add r4, $key, #248 @ pass key schedule |
| 2232 |
#endif |
| 2233 |
veor @XMM[6], @XMM[6], @XMM[14] |
| 2234 |
mov r5, $rounds @ pass rounds |
| 2235 |
mov r0, sp |
| 2236 |
|
| 2237 |
bl _bsaes_decrypt8 |
| 2238 |
|
| 2239 |
vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]! |
| 2240 |
vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]! |
| 2241 |
veor @XMM[0], @XMM[0], @XMM[ 8] |
| 2242 |
vld1.64 {@XMM[12]-@XMM[13]}, [r0,:128]! |
| 2243 |
veor @XMM[1], @XMM[1], @XMM[ 9] |
| 2244 |
veor @XMM[8], @XMM[6], @XMM[10] |
| 2245 |
vst1.8 {@XMM[0]-@XMM[1]}, [$out]! |
| 2246 |
veor @XMM[9], @XMM[4], @XMM[11] |
| 2247 |
vld1.64 {@XMM[14]}, [r0,:128]! |
| 2248 |
veor @XMM[10], @XMM[2], @XMM[12] |
| 2249 |
vst1.8 {@XMM[8]-@XMM[9]}, [$out]! |
| 2250 |
veor @XMM[11], @XMM[7], @XMM[13] |
| 2251 |
veor @XMM[12], @XMM[3], @XMM[14] |
| 2252 |
vst1.8 {@XMM[10]-@XMM[11]}, [$out]! |
| 2253 |
vst1.8 {@XMM[12]}, [$out]! |
| 2254 |
|
| 2255 |
vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak |
| 2256 |
b .Lxts_dec_done |
| 2257 |
.align 4 |
| 2258 |
.Lxts_dec_6: |
| 2259 |
vst1.64 {@XMM[14]}, [r0,:128] @ next round tweak |
| 2260 |
|
| 2261 |
veor @XMM[4], @XMM[4], @XMM[12] |
| 2262 |
#ifndef BSAES_ASM_EXTENDED_KEY |
| 2263 |
add r4, sp, #0x90 @ pass key schedule |
| 2264 |
#else |
| 2265 |
add r4, $key, #248 @ pass key schedule |
| 2266 |
#endif |
| 2267 |
veor @XMM[5], @XMM[5], @XMM[13] |
| 2268 |
mov r5, $rounds @ pass rounds |
| 2269 |
mov r0, sp |
| 2270 |
|
| 2271 |
bl _bsaes_decrypt8 |
| 2272 |
|
| 2273 |
vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]! |
| 2274 |
vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]! |
| 2275 |
veor @XMM[0], @XMM[0], @XMM[ 8] |
| 2276 |
vld1.64 {@XMM[12]-@XMM[13]}, [r0,:128]! |
| 2277 |
veor @XMM[1], @XMM[1], @XMM[ 9] |
| 2278 |
veor @XMM[8], @XMM[6], @XMM[10] |
| 2279 |
vst1.8 {@XMM[0]-@XMM[1]}, [$out]! |
| 2280 |
veor @XMM[9], @XMM[4], @XMM[11] |
| 2281 |
veor @XMM[10], @XMM[2], @XMM[12] |
| 2282 |
vst1.8 {@XMM[8]-@XMM[9]}, [$out]! |
| 2283 |
veor @XMM[11], @XMM[7], @XMM[13] |
| 2284 |
vst1.8 {@XMM[10]-@XMM[11]}, [$out]! |
| 2285 |
|
| 2286 |
vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak |
| 2287 |
b .Lxts_dec_done |
| 2288 |
.align 4 |
| 2289 |
.Lxts_dec_5: |
| 2290 |
vst1.64 {@XMM[13]}, [r0,:128] @ next round tweak |
| 2291 |
|
| 2292 |
veor @XMM[3], @XMM[3], @XMM[11] |
| 2293 |
#ifndef BSAES_ASM_EXTENDED_KEY |
| 2294 |
add r4, sp, #0x90 @ pass key schedule |
| 2295 |
#else |
| 2296 |
add r4, $key, #248 @ pass key schedule |
| 2297 |
#endif |
| 2298 |
veor @XMM[4], @XMM[4], @XMM[12] |
| 2299 |
mov r5, $rounds @ pass rounds |
| 2300 |
mov r0, sp |
| 2301 |
|
| 2302 |
bl _bsaes_decrypt8 |
| 2303 |
|
| 2304 |
vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]! |
| 2305 |
vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]! |
| 2306 |
veor @XMM[0], @XMM[0], @XMM[ 8] |
| 2307 |
vld1.64 {@XMM[12]}, [r0,:128]! |
| 2308 |
veor @XMM[1], @XMM[1], @XMM[ 9] |
| 2309 |
veor @XMM[8], @XMM[6], @XMM[10] |
| 2310 |
vst1.8 {@XMM[0]-@XMM[1]}, [$out]! |
| 2311 |
veor @XMM[9], @XMM[4], @XMM[11] |
| 2312 |
veor @XMM[10], @XMM[2], @XMM[12] |
| 2313 |
vst1.8 {@XMM[8]-@XMM[9]}, [$out]! |
| 2314 |
vst1.8 {@XMM[10]}, [$out]! |
| 2315 |
|
| 2316 |
vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak |
| 2317 |
b .Lxts_dec_done |
| 2318 |
.align 4 |
| 2319 |
.Lxts_dec_4: |
| 2320 |
vst1.64 {@XMM[12]}, [r0,:128] @ next round tweak |
| 2321 |
|
| 2322 |
veor @XMM[2], @XMM[2], @XMM[10] |
| 2323 |
#ifndef BSAES_ASM_EXTENDED_KEY |
| 2324 |
add r4, sp, #0x90 @ pass key schedule |
| 2325 |
#else |
| 2326 |
add r4, $key, #248 @ pass key schedule |
| 2327 |
#endif |
| 2328 |
veor @XMM[3], @XMM[3], @XMM[11] |
| 2329 |
mov r5, $rounds @ pass rounds |
| 2330 |
mov r0, sp |
| 2331 |
|
| 2332 |
bl _bsaes_decrypt8 |
| 2333 |
|
| 2334 |
vld1.64 {@XMM[ 8]-@XMM[ 9]}, [r0,:128]! |
| 2335 |
vld1.64 {@XMM[10]-@XMM[11]}, [r0,:128]! |
| 2336 |
veor @XMM[0], @XMM[0], @XMM[ 8] |
| 2337 |
veor @XMM[1], @XMM[1], @XMM[ 9] |
| 2338 |
veor @XMM[8], @XMM[6], @XMM[10] |
| 2339 |
vst1.8 {@XMM[0]-@XMM[1]}, [$out]! |
| 2340 |
veor @XMM[9], @XMM[4], @XMM[11] |
| 2341 |
vst1.8 {@XMM[8]-@XMM[9]}, [$out]! |
| 2342 |
|
| 2343 |
vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak |
| 2344 |
b .Lxts_dec_done |
| 2345 |
.align 4 |
| 2346 |
.Lxts_dec_3: |
| 2347 |
vst1.64 {@XMM[11]}, [r0,:128] @ next round tweak |
| 2348 |
|
| 2349 |
veor @XMM[1], @XMM[1], @XMM[9] |
| 2350 |
#ifndef BSAES_ASM_EXTENDED_KEY |
| 2351 |
add r4, sp, #0x90 @ pass key schedule |
| 2352 |
#else |
| 2353 |
add r4, $key, #248 @ pass key schedule |
| 2354 |
#endif |
| 2355 |
veor @XMM[2], @XMM[2], @XMM[10] |
| 2356 |
mov r5, $rounds @ pass rounds |
| 2357 |
mov r0, sp |
| 2358 |
|
| 2359 |
bl _bsaes_decrypt8 |
| 2360 |
|
| 2361 |
vld1.64 {@XMM[8]-@XMM[9]}, [r0,:128]! |
| 2362 |
vld1.64 {@XMM[10]}, [r0,:128]! |
| 2363 |
veor @XMM[0], @XMM[0], @XMM[ 8] |
| 2364 |
veor @XMM[1], @XMM[1], @XMM[ 9] |
| 2365 |
veor @XMM[8], @XMM[6], @XMM[10] |
| 2366 |
vst1.8 {@XMM[0]-@XMM[1]}, [$out]! |
| 2367 |
vst1.8 {@XMM[8]}, [$out]! |
| 2368 |
|
| 2369 |
vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak |
| 2370 |
b .Lxts_dec_done |
| 2371 |
.align 4 |
| 2372 |
.Lxts_dec_2: |
| 2373 |
vst1.64 {@XMM[10]}, [r0,:128] @ next round tweak |
| 2374 |
|
| 2375 |
veor @XMM[0], @XMM[0], @XMM[8] |
| 2376 |
#ifndef BSAES_ASM_EXTENDED_KEY |
| 2377 |
add r4, sp, #0x90 @ pass key schedule |
| 2378 |
#else |
| 2379 |
add r4, $key, #248 @ pass key schedule |
| 2380 |
#endif |
| 2381 |
veor @XMM[1], @XMM[1], @XMM[9] |
| 2382 |
mov r5, $rounds @ pass rounds |
| 2383 |
mov r0, sp |
| 2384 |
|
| 2385 |
bl _bsaes_decrypt8 |
| 2386 |
|
| 2387 |
vld1.64 {@XMM[8]-@XMM[9]}, [r0,:128]! |
| 2388 |
veor @XMM[0], @XMM[0], @XMM[ 8] |
| 2389 |
veor @XMM[1], @XMM[1], @XMM[ 9] |
| 2390 |
vst1.8 {@XMM[0]-@XMM[1]}, [$out]! |
| 2391 |
|
| 2392 |
vld1.64 {@XMM[8]}, [r0,:128] @ next round tweak |
| 2393 |
b .Lxts_dec_done |
| 2394 |
.align 4 |
| 2395 |
.Lxts_dec_1: |
| 2396 |
mov r0, sp |
| 2397 |
veor @XMM[0], @XMM[8] |
| 2398 |
mov r1, sp |
| 2399 |
vst1.8 {@XMM[0]}, [sp,:128] |
| 2400 |
mov r2, $key |
| 2401 |
mov r4, $fp @ preserve fp |
| 2402 |
mov r5, $magic @ preserve magic |
| 2403 |
|
| 2404 |
bl AES_decrypt |
| 2405 |
|
| 2406 |
vld1.8 {@XMM[0]}, [sp,:128] |
| 2407 |
veor @XMM[0], @XMM[0], @XMM[8] |
| 2408 |
vst1.8 {@XMM[0]}, [$out]! |
| 2409 |
mov $fp, r4 |
| 2410 |
mov $magic, r5 |
| 2411 |
|
| 2412 |
vmov @XMM[8], @XMM[9] @ next round tweak |
| 2413 |
|
| 2414 |
.Lxts_dec_done: |
| 2415 |
#ifndef XTS_CHAIN_TWEAK |
| 2416 |
adds $len, #0x10 |
| 2417 |
beq .Lxts_dec_ret |
| 2418 |
|
| 2419 |
@ calculate one round of extra tweak for the stolen ciphertext |
| 2420 |
vldmia $magic, {$twmask} |
| 2421 |
vshr.s64 @XMM[6], @XMM[8], #63 |
| 2422 |
vand @XMM[6], @XMM[6], $twmask |
| 2423 |
vadd.u64 @XMM[9], @XMM[8], @XMM[8] |
| 2424 |
vswp `&Dhi("@XMM[6]")`,`&Dlo("@XMM[6]")` |
| 2425 |
veor @XMM[9], @XMM[9], @XMM[6] |
| 2426 |
|
| 2427 |
@ perform the final decryption with the last tweak value |
| 2428 |
vld1.8 {@XMM[0]}, [$inp]! |
| 2429 |
mov r0, sp |
| 2430 |
veor @XMM[0], @XMM[0], @XMM[9] |
| 2431 |
mov r1, sp |
| 2432 |
vst1.8 {@XMM[0]}, [sp,:128] |
| 2433 |
mov r2, $key |
| 2434 |
mov r4, $fp @ preserve fp |
| 2435 |
|
| 2436 |
bl AES_decrypt |
| 2437 |
|
| 2438 |
vld1.8 {@XMM[0]}, [sp,:128] |
| 2439 |
veor @XMM[0], @XMM[0], @XMM[9] |
| 2440 |
vst1.8 {@XMM[0]}, [$out] |
| 2441 |
|
| 2442 |
mov r6, $out |
| 2443 |
.Lxts_dec_steal: |
| 2444 |
ldrb r1, [$out] |
| 2445 |
ldrb r0, [$inp], #1 |
| 2446 |
strb r1, [$out, #0x10] |
| 2447 |
strb r0, [$out], #1 |
| 2448 |
|
| 2449 |
subs $len, #1 |
| 2450 |
bhi .Lxts_dec_steal |
| 2451 |
|
| 2452 |
vld1.8 {@XMM[0]}, [r6] |
| 2453 |
mov r0, sp |
| 2454 |
veor @XMM[0], @XMM[8] |
| 2455 |
mov r1, sp |
| 2456 |
vst1.8 {@XMM[0]}, [sp,:128] |
| 2457 |
mov r2, $key |
| 2458 |
|
| 2459 |
bl AES_decrypt |
| 2460 |
|
| 2461 |
vld1.8 {@XMM[0]}, [sp,:128] |
| 2462 |
veor @XMM[0], @XMM[0], @XMM[8] |
| 2463 |
vst1.8 {@XMM[0]}, [r6] |
| 2464 |
mov $fp, r4 |
| 2465 |
#endif |
| 2466 |
|
| 2467 |
.Lxts_dec_ret: |
| 2468 |
bic r0, $fp, #0xf |
| 2469 |
vmov.i32 q0, #0 |
| 2470 |
vmov.i32 q1, #0 |
| 2471 |
#ifdef XTS_CHAIN_TWEAK |
| 2472 |
ldr r1, [$fp, #0x20+VFP_ABI_FRAME] @ chain tweak |
| 2473 |
#endif |
| 2474 |
.Lxts_dec_bzero: @ wipe key schedule [if any] |
| 2475 |
vstmia sp!, {q0-q1} |
| 2476 |
cmp sp, r0 |
| 2477 |
bne .Lxts_dec_bzero |
| 2478 |
|
| 2479 |
mov sp, $fp |
| 2480 |
#ifdef XTS_CHAIN_TWEAK |
| 2481 |
vst1.8 {@XMM[8]}, [r1] |
| 2482 |
#endif |
| 2483 |
VFP_ABI_POP |
| 2484 |
ldmia sp!, {r4-r10, pc} @ return |
| 2485 |
|
| 2486 |
.size bsaes_xts_decrypt,.-bsaes_xts_decrypt |
| 2487 |
___ |
| 2488 |
} |
| 2489 |
$code.=<<___; |
| 2490 |
#endif |
| 2491 |
___ |
| 2492 |
|
| 2493 |
$code =~ s/\`([^\`]*)\`/eval($1)/gem; |
| 2494 |
|
| 2495 |
open SELF,$0; |
| 2496 |
while(<SELF>) { |
| 2497 |
next if (/^#!/); |
| 2498 |
last if (!s/^#/@/ and !/^$/); |
| 2499 |
print; |
| 2500 |
} |
| 2501 |
close SELF; |
| 2502 |
|
| 2503 |
print $code; |
| 2504 |
|
| 2505 |
close STDOUT; |