Debian Codegraph @

external/boringssl/crypto/sha/asm/sha512-armv4.pl

raw
1 #!/usr/bin/env perl
2
3 # ====================================================================
4 # Written by Andy Polyakov <[email protected]> for the OpenSSL
5 # project. The module is, however, dual licensed under OpenSSL and
6 # CRYPTOGAMS licenses depending on where you obtain it. For further
7 # details see http://www.openssl.org/~appro/cryptogams/.
8 #
9 # Permission to use under GPL terms is granted.
10 # ====================================================================
11
12 # SHA512 block procedure for ARMv4. September 2007.
13
14 # This code is ~4.5 (four and a half) times faster than code generated
15 # by gcc 3.4 and it spends ~72 clock cycles per byte [on single-issue
16 # Xscale PXA250 core].
17 #
18 # July 2010.
19 #
20 # Rescheduling for dual-issue pipeline resulted in 6% improvement on
21 # Cortex A8 core and ~40 cycles per processed byte.
22
23 # February 2011.
24 #
25 # Profiler-assisted and platform-specific optimization resulted in 7%
26 # improvement on Coxtex A8 core and ~38 cycles per byte.
27
28 # March 2011.
29 #
30 # Add NEON implementation. On Cortex A8 it was measured to process
31 # one byte in 23.3 cycles or ~60% faster than integer-only code.
32
33 # August 2012.
34 #
35 # Improve NEON performance by 12% on Snapdragon S4. In absolute
36 # terms it's 22.6 cycles per byte, which is disappointing result.
37 # Technical writers asserted that 3-way S4 pipeline can sustain
38 # multiple NEON instructions per cycle, but dual NEON issue could
39 # not be observed, see http://www.openssl.org/~appro/Snapdragon-S4.html
40 # for further details. On side note Cortex-A15 processes one byte in
41 # 16 cycles.
42
43 # Byte order [in]dependence. =========================================
44 #
45 # Originally caller was expected to maintain specific *dword* order in
46 # h[0-7], namely with most significant dword at *lower* address, which
47 # was reflected in below two parameters as 0 and 4. Now caller is
48 # expected to maintain native byte order for whole 64-bit values.
49 $hi="HI";
50 $lo="LO";
51 # ====================================================================
52
53 $flavour = shift;
54 if ($flavour=~/^\w[\w\-]*\.\w+$/) { $output=$flavour; undef $flavour; }
55 else { while (($output=shift) && ($output!~/^\w[\w\-]*\.\w+$/)) {} }
56
57 if ($flavour && $flavour ne "void") {
58 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
59 ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
60 ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
61 die "can't locate arm-xlate.pl";
62
63 open STDOUT,"| \"$^X\" $xlate $flavour $output";
64 } else {
65 open STDOUT,">$output";
66 }
67
68 $ctx="r0"; # parameter block
69 $inp="r1";
70 $len="r2";
71
72 $Tlo="r3";
73 $Thi="r4";
74 $Alo="r5";
75 $Ahi="r6";
76 $Elo="r7";
77 $Ehi="r8";
78 $t0="r9";
79 $t1="r10";
80 $t2="r11";
81 $t3="r12";
82 ############ r13 is stack pointer
83 $Ktbl="r14";
84 ############ r15 is program counter
85
86 $Aoff=8*0;
87 $Boff=8*1;
88 $Coff=8*2;
89 $Doff=8*3;
90 $Eoff=8*4;
91 $Foff=8*5;
92 $Goff=8*6;
93 $Hoff=8*7;
94 $Xoff=8*8;
95
96 sub BODY_00_15() {
97 my $magic = shift;
98 $code.=<<___;
99 @ Sigma1(x) (ROTR((x),14) ^ ROTR((x),18) ^ ROTR((x),41))
100 @ LO lo>>14^hi<<18 ^ lo>>18^hi<<14 ^ hi>>9^lo<<23
101 @ HI hi>>14^lo<<18 ^ hi>>18^lo<<14 ^ lo>>9^hi<<23
102 mov $t0,$Elo,lsr#14
103 str $Tlo,[sp,#$Xoff+0]
104 mov $t1,$Ehi,lsr#14
105 str $Thi,[sp,#$Xoff+4]
106 eor $t0,$t0,$Ehi,lsl#18
107 ldr $t2,[sp,#$Hoff+0] @ h.lo
108 eor $t1,$t1,$Elo,lsl#18
109 ldr $t3,[sp,#$Hoff+4] @ h.hi
110 eor $t0,$t0,$Elo,lsr#18
111 eor $t1,$t1,$Ehi,lsr#18
112 eor $t0,$t0,$Ehi,lsl#14
113 eor $t1,$t1,$Elo,lsl#14
114 eor $t0,$t0,$Ehi,lsr#9
115 eor $t1,$t1,$Elo,lsr#9
116 eor $t0,$t0,$Elo,lsl#23
117 eor $t1,$t1,$Ehi,lsl#23 @ Sigma1(e)
118 adds $Tlo,$Tlo,$t0
119 ldr $t0,[sp,#$Foff+0] @ f.lo
120 adc $Thi,$Thi,$t1 @ T += Sigma1(e)
121 ldr $t1,[sp,#$Foff+4] @ f.hi
122 adds $Tlo,$Tlo,$t2
123 ldr $t2,[sp,#$Goff+0] @ g.lo
124 adc $Thi,$Thi,$t3 @ T += h
125 ldr $t3,[sp,#$Goff+4] @ g.hi
126
127 eor $t0,$t0,$t2
128 str $Elo,[sp,#$Eoff+0]
129 eor $t1,$t1,$t3
130 str $Ehi,[sp,#$Eoff+4]
131 and $t0,$t0,$Elo
132 str $Alo,[sp,#$Aoff+0]
133 and $t1,$t1,$Ehi
134 str $Ahi,[sp,#$Aoff+4]
135 eor $t0,$t0,$t2
136 ldr $t2,[$Ktbl,#$lo] @ K[i].lo
137 eor $t1,$t1,$t3 @ Ch(e,f,g)
138 ldr $t3,[$Ktbl,#$hi] @ K[i].hi
139
140 adds $Tlo,$Tlo,$t0
141 ldr $Elo,[sp,#$Doff+0] @ d.lo
142 adc $Thi,$Thi,$t1 @ T += Ch(e,f,g)
143 ldr $Ehi,[sp,#$Doff+4] @ d.hi
144 adds $Tlo,$Tlo,$t2
145 and $t0,$t2,#0xff
146 adc $Thi,$Thi,$t3 @ T += K[i]
147 adds $Elo,$Elo,$Tlo
148 ldr $t2,[sp,#$Boff+0] @ b.lo
149 adc $Ehi,$Ehi,$Thi @ d += T
150 teq $t0,#$magic
151
152 ldr $t3,[sp,#$Coff+0] @ c.lo
153 #if __ARM_ARCH__>=7
154 it eq @ Thumb2 thing, sanity check in ARM
155 #endif
156 orreq $Ktbl,$Ktbl,#1
157 @ Sigma0(x) (ROTR((x),28) ^ ROTR((x),34) ^ ROTR((x),39))
158 @ LO lo>>28^hi<<4 ^ hi>>2^lo<<30 ^ hi>>7^lo<<25
159 @ HI hi>>28^lo<<4 ^ lo>>2^hi<<30 ^ lo>>7^hi<<25
160 mov $t0,$Alo,lsr#28
161 mov $t1,$Ahi,lsr#28
162 eor $t0,$t0,$Ahi,lsl#4
163 eor $t1,$t1,$Alo,lsl#4
164 eor $t0,$t0,$Ahi,lsr#2
165 eor $t1,$t1,$Alo,lsr#2
166 eor $t0,$t0,$Alo,lsl#30
167 eor $t1,$t1,$Ahi,lsl#30
168 eor $t0,$t0,$Ahi,lsr#7
169 eor $t1,$t1,$Alo,lsr#7
170 eor $t0,$t0,$Alo,lsl#25
171 eor $t1,$t1,$Ahi,lsl#25 @ Sigma0(a)
172 adds $Tlo,$Tlo,$t0
173 and $t0,$Alo,$t2
174 adc $Thi,$Thi,$t1 @ T += Sigma0(a)
175
176 ldr $t1,[sp,#$Boff+4] @ b.hi
177 orr $Alo,$Alo,$t2
178 ldr $t2,[sp,#$Coff+4] @ c.hi
179 and $Alo,$Alo,$t3
180 and $t3,$Ahi,$t1
181 orr $Ahi,$Ahi,$t1
182 orr $Alo,$Alo,$t0 @ Maj(a,b,c).lo
183 and $Ahi,$Ahi,$t2
184 adds $Alo,$Alo,$Tlo
185 orr $Ahi,$Ahi,$t3 @ Maj(a,b,c).hi
186 sub sp,sp,#8
187 adc $Ahi,$Ahi,$Thi @ h += T
188 tst $Ktbl,#1
189 add $Ktbl,$Ktbl,#8
190 ___
191 }
192 $code=<<___;
193 #ifndef __KERNEL__
194 # include <openssl/arm_arch.h>
195 # define VFP_ABI_PUSH vstmdb sp!,{d8-d15}
196 # define VFP_ABI_POP vldmia sp!,{d8-d15}
197 #else
198 # define __ARM_ARCH__ __LINUX_ARM_ARCH__
199 # define __ARM_MAX_ARCH__ 7
200 # define VFP_ABI_PUSH
201 # define VFP_ABI_POP
202 #endif
203
204 #ifdef __ARMEL__
205 # define LO 0
206 # define HI 4
207 # define WORD64(hi0,lo0,hi1,lo1) .word lo0,hi0, lo1,hi1
208 #else
209 # define HI 0
210 # define LO 4
211 # define WORD64(hi0,lo0,hi1,lo1) .word hi0,lo0, hi1,lo1
212 #endif
213
214 .text
215 #if __ARM_ARCH__<7 || defined(__APPLE__)
216 .code 32
217 #else
218 .syntax unified
219 # ifdef __thumb2__
220 # define adrl adr
221 .thumb
222 # else
223 .code 32
224 # endif
225 #endif
226
227 .type K512,%object
228 .align 5
229 K512:
230 WORD64(0x428a2f98,0xd728ae22, 0x71374491,0x23ef65cd)
231 WORD64(0xb5c0fbcf,0xec4d3b2f, 0xe9b5dba5,0x8189dbbc)
232 WORD64(0x3956c25b,0xf348b538, 0x59f111f1,0xb605d019)
233 WORD64(0x923f82a4,0xaf194f9b, 0xab1c5ed5,0xda6d8118)
234 WORD64(0xd807aa98,0xa3030242, 0x12835b01,0x45706fbe)
235 WORD64(0x243185be,0x4ee4b28c, 0x550c7dc3,0xd5ffb4e2)
236 WORD64(0x72be5d74,0xf27b896f, 0x80deb1fe,0x3b1696b1)
237 WORD64(0x9bdc06a7,0x25c71235, 0xc19bf174,0xcf692694)
238 WORD64(0xe49b69c1,0x9ef14ad2, 0xefbe4786,0x384f25e3)
239 WORD64(0x0fc19dc6,0x8b8cd5b5, 0x240ca1cc,0x77ac9c65)
240 WORD64(0x2de92c6f,0x592b0275, 0x4a7484aa,0x6ea6e483)
241 WORD64(0x5cb0a9dc,0xbd41fbd4, 0x76f988da,0x831153b5)
242 WORD64(0x983e5152,0xee66dfab, 0xa831c66d,0x2db43210)
243 WORD64(0xb00327c8,0x98fb213f, 0xbf597fc7,0xbeef0ee4)
244 WORD64(0xc6e00bf3,0x3da88fc2, 0xd5a79147,0x930aa725)
245 WORD64(0x06ca6351,0xe003826f, 0x14292967,0x0a0e6e70)
246 WORD64(0x27b70a85,0x46d22ffc, 0x2e1b2138,0x5c26c926)
247 WORD64(0x4d2c6dfc,0x5ac42aed, 0x53380d13,0x9d95b3df)
248 WORD64(0x650a7354,0x8baf63de, 0x766a0abb,0x3c77b2a8)
249 WORD64(0x81c2c92e,0x47edaee6, 0x92722c85,0x1482353b)
250 WORD64(0xa2bfe8a1,0x4cf10364, 0xa81a664b,0xbc423001)
251 WORD64(0xc24b8b70,0xd0f89791, 0xc76c51a3,0x0654be30)
252 WORD64(0xd192e819,0xd6ef5218, 0xd6990624,0x5565a910)
253 WORD64(0xf40e3585,0x5771202a, 0x106aa070,0x32bbd1b8)
254 WORD64(0x19a4c116,0xb8d2d0c8, 0x1e376c08,0x5141ab53)
255 WORD64(0x2748774c,0xdf8eeb99, 0x34b0bcb5,0xe19b48a8)
256 WORD64(0x391c0cb3,0xc5c95a63, 0x4ed8aa4a,0xe3418acb)
257 WORD64(0x5b9cca4f,0x7763e373, 0x682e6ff3,0xd6b2b8a3)
258 WORD64(0x748f82ee,0x5defb2fc, 0x78a5636f,0x43172f60)
259 WORD64(0x84c87814,0xa1f0ab72, 0x8cc70208,0x1a6439ec)
260 WORD64(0x90befffa,0x23631e28, 0xa4506ceb,0xde82bde9)
261 WORD64(0xbef9a3f7,0xb2c67915, 0xc67178f2,0xe372532b)
262 WORD64(0xca273ece,0xea26619c, 0xd186b8c7,0x21c0c207)
263 WORD64(0xeada7dd6,0xcde0eb1e, 0xf57d4f7f,0xee6ed178)
264 WORD64(0x06f067aa,0x72176fba, 0x0a637dc5,0xa2c898a6)
265 WORD64(0x113f9804,0xbef90dae, 0x1b710b35,0x131c471b)
266 WORD64(0x28db77f5,0x23047d84, 0x32caab7b,0x40c72493)
267 WORD64(0x3c9ebe0a,0x15c9bebc, 0x431d67c4,0x9c100d4c)
268 WORD64(0x4cc5d4be,0xcb3e42b6, 0x597f299c,0xfc657e2a)
269 WORD64(0x5fcb6fab,0x3ad6faec, 0x6c44198c,0x4a475817)
270 .size K512,.-K512
271 #if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
272 .LOPENSSL_armcap:
273 .word OPENSSL_armcap_P-.Lsha512_block_data_order
274 .skip 32-4
275 #else
276 .skip 32
277 #endif
278
279 .global sha512_block_data_order
280 .type sha512_block_data_order,%function
281 sha512_block_data_order:
282 .Lsha512_block_data_order:
283 #if __ARM_ARCH__<7
284 sub r3,pc,#8 @ sha512_block_data_order
285 #else
286 adr r3,sha512_block_data_order
287 #endif
288 #if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
289 ldr r12,.LOPENSSL_armcap
290 ldr r12,[r3,r12] @ OPENSSL_armcap_P
291 #ifdef __APPLE__
292 ldr r12,[r12]
293 #endif
294 tst r12,#ARMV7_NEON
295 bne .LNEON
296 #endif
297 add $len,$inp,$len,lsl#7 @ len to point at the end of inp
298 stmdb sp!,{r4-r12,lr}
299 sub $Ktbl,r3,#672 @ K512
300 sub sp,sp,#9*8
301
302 ldr $Elo,[$ctx,#$Eoff+$lo]
303 ldr $Ehi,[$ctx,#$Eoff+$hi]
304 ldr $t0, [$ctx,#$Goff+$lo]
305 ldr $t1, [$ctx,#$Goff+$hi]
306 ldr $t2, [$ctx,#$Hoff+$lo]
307 ldr $t3, [$ctx,#$Hoff+$hi]
308 .Loop:
309 str $t0, [sp,#$Goff+0]
310 str $t1, [sp,#$Goff+4]
311 str $t2, [sp,#$Hoff+0]
312 str $t3, [sp,#$Hoff+4]
313 ldr $Alo,[$ctx,#$Aoff+$lo]
314 ldr $Ahi,[$ctx,#$Aoff+$hi]
315 ldr $Tlo,[$ctx,#$Boff+$lo]
316 ldr $Thi,[$ctx,#$Boff+$hi]
317 ldr $t0, [$ctx,#$Coff+$lo]
318 ldr $t1, [$ctx,#$Coff+$hi]
319 ldr $t2, [$ctx,#$Doff+$lo]
320 ldr $t3, [$ctx,#$Doff+$hi]
321 str $Tlo,[sp,#$Boff+0]
322 str $Thi,[sp,#$Boff+4]
323 str $t0, [sp,#$Coff+0]
324 str $t1, [sp,#$Coff+4]
325 str $t2, [sp,#$Doff+0]
326 str $t3, [sp,#$Doff+4]
327 ldr $Tlo,[$ctx,#$Foff+$lo]
328 ldr $Thi,[$ctx,#$Foff+$hi]
329 str $Tlo,[sp,#$Foff+0]
330 str $Thi,[sp,#$Foff+4]
331
332 .L00_15:
333 #if __ARM_ARCH__<7
334 ldrb $Tlo,[$inp,#7]
335 ldrb $t0, [$inp,#6]
336 ldrb $t1, [$inp,#5]
337 ldrb $t2, [$inp,#4]
338 ldrb $Thi,[$inp,#3]
339 ldrb $t3, [$inp,#2]
340 orr $Tlo,$Tlo,$t0,lsl#8
341 ldrb $t0, [$inp,#1]
342 orr $Tlo,$Tlo,$t1,lsl#16
343 ldrb $t1, [$inp],#8
344 orr $Tlo,$Tlo,$t2,lsl#24
345 orr $Thi,$Thi,$t3,lsl#8
346 orr $Thi,$Thi,$t0,lsl#16
347 orr $Thi,$Thi,$t1,lsl#24
348 #else
349 ldr $Tlo,[$inp,#4]
350 ldr $Thi,[$inp],#8
351 #ifdef __ARMEL__
352 rev $Tlo,$Tlo
353 rev $Thi,$Thi
354 #endif
355 #endif
356 ___
357 &BODY_00_15(0x94);
358 $code.=<<___;
359 tst $Ktbl,#1
360 beq .L00_15
361 ldr $t0,[sp,#`$Xoff+8*(16-1)`+0]
362 ldr $t1,[sp,#`$Xoff+8*(16-1)`+4]
363 bic $Ktbl,$Ktbl,#1
364 .L16_79:
365 @ sigma0(x) (ROTR((x),1) ^ ROTR((x),8) ^ ((x)>>7))
366 @ LO lo>>1^hi<<31 ^ lo>>8^hi<<24 ^ lo>>7^hi<<25
367 @ HI hi>>1^lo<<31 ^ hi>>8^lo<<24 ^ hi>>7
368 mov $Tlo,$t0,lsr#1
369 ldr $t2,[sp,#`$Xoff+8*(16-14)`+0]
370 mov $Thi,$t1,lsr#1
371 ldr $t3,[sp,#`$Xoff+8*(16-14)`+4]
372 eor $Tlo,$Tlo,$t1,lsl#31
373 eor $Thi,$Thi,$t0,lsl#31
374 eor $Tlo,$Tlo,$t0,lsr#8
375 eor $Thi,$Thi,$t1,lsr#8
376 eor $Tlo,$Tlo,$t1,lsl#24
377 eor $Thi,$Thi,$t0,lsl#24
378 eor $Tlo,$Tlo,$t0,lsr#7
379 eor $Thi,$Thi,$t1,lsr#7
380 eor $Tlo,$Tlo,$t1,lsl#25
381
382 @ sigma1(x) (ROTR((x),19) ^ ROTR((x),61) ^ ((x)>>6))
383 @ LO lo>>19^hi<<13 ^ hi>>29^lo<<3 ^ lo>>6^hi<<26
384 @ HI hi>>19^lo<<13 ^ lo>>29^hi<<3 ^ hi>>6
385 mov $t0,$t2,lsr#19
386 mov $t1,$t3,lsr#19
387 eor $t0,$t0,$t3,lsl#13
388 eor $t1,$t1,$t2,lsl#13
389 eor $t0,$t0,$t3,lsr#29
390 eor $t1,$t1,$t2,lsr#29
391 eor $t0,$t0,$t2,lsl#3
392 eor $t1,$t1,$t3,lsl#3
393 eor $t0,$t0,$t2,lsr#6
394 eor $t1,$t1,$t3,lsr#6
395 ldr $t2,[sp,#`$Xoff+8*(16-9)`+0]
396 eor $t0,$t0,$t3,lsl#26
397
398 ldr $t3,[sp,#`$Xoff+8*(16-9)`+4]
399 adds $Tlo,$Tlo,$t0
400 ldr $t0,[sp,#`$Xoff+8*16`+0]
401 adc $Thi,$Thi,$t1
402
403 ldr $t1,[sp,#`$Xoff+8*16`+4]
404 adds $Tlo,$Tlo,$t2
405 adc $Thi,$Thi,$t3
406 adds $Tlo,$Tlo,$t0
407 adc $Thi,$Thi,$t1
408 ___
409 &BODY_00_15(0x17);
410 $code.=<<___;
411 #if __ARM_ARCH__>=7
412 ittt eq @ Thumb2 thing, sanity check in ARM
413 #endif
414 ldreq $t0,[sp,#`$Xoff+8*(16-1)`+0]
415 ldreq $t1,[sp,#`$Xoff+8*(16-1)`+4]
416 beq .L16_79
417 bic $Ktbl,$Ktbl,#1
418
419 ldr $Tlo,[sp,#$Boff+0]
420 ldr $Thi,[sp,#$Boff+4]
421 ldr $t0, [$ctx,#$Aoff+$lo]
422 ldr $t1, [$ctx,#$Aoff+$hi]
423 ldr $t2, [$ctx,#$Boff+$lo]
424 ldr $t3, [$ctx,#$Boff+$hi]
425 adds $t0,$Alo,$t0
426 str $t0, [$ctx,#$Aoff+$lo]
427 adc $t1,$Ahi,$t1
428 str $t1, [$ctx,#$Aoff+$hi]
429 adds $t2,$Tlo,$t2
430 str $t2, [$ctx,#$Boff+$lo]
431 adc $t3,$Thi,$t3
432 str $t3, [$ctx,#$Boff+$hi]
433
434 ldr $Alo,[sp,#$Coff+0]
435 ldr $Ahi,[sp,#$Coff+4]
436 ldr $Tlo,[sp,#$Doff+0]
437 ldr $Thi,[sp,#$Doff+4]
438 ldr $t0, [$ctx,#$Coff+$lo]
439 ldr $t1, [$ctx,#$Coff+$hi]
440 ldr $t2, [$ctx,#$Doff+$lo]
441 ldr $t3, [$ctx,#$Doff+$hi]
442 adds $t0,$Alo,$t0
443 str $t0, [$ctx,#$Coff+$lo]
444 adc $t1,$Ahi,$t1
445 str $t1, [$ctx,#$Coff+$hi]
446 adds $t2,$Tlo,$t2
447 str $t2, [$ctx,#$Doff+$lo]
448 adc $t3,$Thi,$t3
449 str $t3, [$ctx,#$Doff+$hi]
450
451 ldr $Tlo,[sp,#$Foff+0]
452 ldr $Thi,[sp,#$Foff+4]
453 ldr $t0, [$ctx,#$Eoff+$lo]
454 ldr $t1, [$ctx,#$Eoff+$hi]
455 ldr $t2, [$ctx,#$Foff+$lo]
456 ldr $t3, [$ctx,#$Foff+$hi]
457 adds $Elo,$Elo,$t0
458 str $Elo,[$ctx,#$Eoff+$lo]
459 adc $Ehi,$Ehi,$t1
460 str $Ehi,[$ctx,#$Eoff+$hi]
461 adds $t2,$Tlo,$t2
462 str $t2, [$ctx,#$Foff+$lo]
463 adc $t3,$Thi,$t3
464 str $t3, [$ctx,#$Foff+$hi]
465
466 ldr $Alo,[sp,#$Goff+0]
467 ldr $Ahi,[sp,#$Goff+4]
468 ldr $Tlo,[sp,#$Hoff+0]
469 ldr $Thi,[sp,#$Hoff+4]
470 ldr $t0, [$ctx,#$Goff+$lo]
471 ldr $t1, [$ctx,#$Goff+$hi]
472 ldr $t2, [$ctx,#$Hoff+$lo]
473 ldr $t3, [$ctx,#$Hoff+$hi]
474 adds $t0,$Alo,$t0
475 str $t0, [$ctx,#$Goff+$lo]
476 adc $t1,$Ahi,$t1
477 str $t1, [$ctx,#$Goff+$hi]
478 adds $t2,$Tlo,$t2
479 str $t2, [$ctx,#$Hoff+$lo]
480 adc $t3,$Thi,$t3
481 str $t3, [$ctx,#$Hoff+$hi]
482
483 add sp,sp,#640
484 sub $Ktbl,$Ktbl,#640
485
486 teq $inp,$len
487 bne .Loop
488
489 add sp,sp,#8*9 @ destroy frame
490 #if __ARM_ARCH__>=5
491 ldmia sp!,{r4-r12,pc}
492 #else
493 ldmia sp!,{r4-r12,lr}
494 tst lr,#1
495 moveq pc,lr @ be binary compatible with V4, yet
496 bx lr @ interoperable with Thumb ISA:-)
497 #endif
498 .size sha512_block_data_order,.-sha512_block_data_order
499 ___
500
501 {
502 my @Sigma0=(28,34,39);
503 my @Sigma1=(14,18,41);
504 my @sigma0=(1, 8, 7);
505 my @sigma1=(19,61,6);
506
507 my $Ktbl="r3";
508 my $cnt="r12"; # volatile register known as ip, intra-procedure-call scratch
509
510 my @X=map("d$_",(0..15));
511 my @V=($A,$B,$C,$D,$E,$F,$G,$H)=map("d$_",(16..23));
512
513 sub NEON_00_15() {
514 my $i=shift;
515 my ($a,$b,$c,$d,$e,$f,$g,$h)=@_;
516 my ($t0,$t1,$t2,$T1,$K,$Ch,$Maj)=map("d$_",(24..31)); # temps
517
518 $code.=<<___ if ($i<16 || $i&1);
519 vshr.u64 $t0,$e,#@Sigma1[0] @ $i
520 #if $i<16
521 vld1.64 {@X[$i%16]},[$inp]! @ handles unaligned
522 #endif
523 vshr.u64 $t1,$e,#@Sigma1[1]
524 #if $i>0
525 vadd.i64 $a,$Maj @ h+=Maj from the past
526 #endif
527 vshr.u64 $t2,$e,#@Sigma1[2]
528 ___
529 $code.=<<___;
530 vld1.64 {$K},[$Ktbl,:64]! @ K[i++]
531 vsli.64 $t0,$e,#`64-@Sigma1[0]`
532 vsli.64 $t1,$e,#`64-@Sigma1[1]`
533 vmov $Ch,$e
534 vsli.64 $t2,$e,#`64-@Sigma1[2]`
535 #if $i<16 && defined(__ARMEL__)
536 vrev64.8 @X[$i],@X[$i]
537 #endif
538 veor $t1,$t0
539 vbsl $Ch,$f,$g @ Ch(e,f,g)
540 vshr.u64 $t0,$a,#@Sigma0[0]
541 veor $t2,$t1 @ Sigma1(e)
542 vadd.i64 $T1,$Ch,$h
543 vshr.u64 $t1,$a,#@Sigma0[1]
544 vsli.64 $t0,$a,#`64-@Sigma0[0]`
545 vadd.i64 $T1,$t2
546 vshr.u64 $t2,$a,#@Sigma0[2]
547 vadd.i64 $K,@X[$i%16]
548 vsli.64 $t1,$a,#`64-@Sigma0[1]`
549 veor $Maj,$a,$b
550 vsli.64 $t2,$a,#`64-@Sigma0[2]`
551 veor $h,$t0,$t1
552 vadd.i64 $T1,$K
553 vbsl $Maj,$c,$b @ Maj(a,b,c)
554 veor $h,$t2 @ Sigma0(a)
555 vadd.i64 $d,$T1
556 vadd.i64 $Maj,$T1
557 @ vadd.i64 $h,$Maj
558 ___
559 }
560
561 sub NEON_16_79() {
562 my $i=shift;
563
564 if ($i&1) { &NEON_00_15($i,@_); return; }
565
566 # 2x-vectorized, therefore runs every 2nd round
567 my @X=map("q$_",(0..7)); # view @X as 128-bit vector
568 my ($t0,$t1,$s0,$s1) = map("q$_",(12..15)); # temps
569 my ($d0,$d1,$d2) = map("d$_",(24..26)); # temps from NEON_00_15
570 my $e=@_[4]; # $e from NEON_00_15
571 $i /= 2;
572 $code.=<<___;
573 vshr.u64 $t0,@X[($i+7)%8],#@sigma1[0]
574 vshr.u64 $t1,@X[($i+7)%8],#@sigma1[1]
575 vadd.i64 @_[0],d30 @ h+=Maj from the past
576 vshr.u64 $s1,@X[($i+7)%8],#@sigma1[2]
577 vsli.64 $t0,@X[($i+7)%8],#`64-@sigma1[0]`
578 vext.8 $s0,@X[$i%8],@X[($i+1)%8],#8 @ X[i+1]
579 vsli.64 $t1,@X[($i+7)%8],#`64-@sigma1[1]`
580 veor $s1,$t0
581 vshr.u64 $t0,$s0,#@sigma0[0]
582 veor $s1,$t1 @ sigma1(X[i+14])
583 vshr.u64 $t1,$s0,#@sigma0[1]
584 vadd.i64 @X[$i%8],$s1
585 vshr.u64 $s1,$s0,#@sigma0[2]
586 vsli.64 $t0,$s0,#`64-@sigma0[0]`
587 vsli.64 $t1,$s0,#`64-@sigma0[1]`
588 vext.8 $s0,@X[($i+4)%8],@X[($i+5)%8],#8 @ X[i+9]
589 veor $s1,$t0
590 vshr.u64 $d0,$e,#@Sigma1[0] @ from NEON_00_15
591 vadd.i64 @X[$i%8],$s0
592 vshr.u64 $d1,$e,#@Sigma1[1] @ from NEON_00_15
593 veor $s1,$t1 @ sigma0(X[i+1])
594 vshr.u64 $d2,$e,#@Sigma1[2] @ from NEON_00_15
595 vadd.i64 @X[$i%8],$s1
596 ___
597 &NEON_00_15(2*$i,@_);
598 }
599
600 $code.=<<___;
601 #if __ARM_MAX_ARCH__>=7
602 .arch armv7-a
603 .fpu neon
604
605 .global sha512_block_data_order_neon
606 .type sha512_block_data_order_neon,%function
607 .align 4
608 sha512_block_data_order_neon:
609 .LNEON:
610 dmb @ errata #451034 on early Cortex A8
611 add $len,$inp,$len,lsl#7 @ len to point at the end of inp
612 adr $Ktbl,K512
613 VFP_ABI_PUSH
614 vldmia $ctx,{$A-$H} @ load context
615 .Loop_neon:
616 ___
617 for($i=0;$i<16;$i++) { &NEON_00_15($i,@V); unshift(@V,pop(@V)); }
618 $code.=<<___;
619 mov $cnt,#4
620 .L16_79_neon:
621 subs $cnt,#1
622 ___
623 for(;$i<32;$i++) { &NEON_16_79($i,@V); unshift(@V,pop(@V)); }
624 $code.=<<___;
625 bne .L16_79_neon
626
627 vadd.i64 $A,d30 @ h+=Maj from the past
628 vldmia $ctx,{d24-d31} @ load context to temp
629 vadd.i64 q8,q12 @ vectorized accumulate
630 vadd.i64 q9,q13
631 vadd.i64 q10,q14
632 vadd.i64 q11,q15
633 vstmia $ctx,{$A-$H} @ save context
634 teq $inp,$len
635 sub $Ktbl,#640 @ rewind K512
636 bne .Loop_neon
637
638 VFP_ABI_POP
639 ret @ bx lr
640 .size sha512_block_data_order_neon,.-sha512_block_data_order_neon
641 #endif
642 ___
643 }
644 $code.=<<___;
645 .asciz "SHA512 block transform for ARMv4/NEON, CRYPTOGAMS by <appro\@openssl.org>"
646 .align 2
647 #if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__)
648 .comm OPENSSL_armcap_P,4,4
649 .hidden OPENSSL_armcap_P
650 #endif
651 ___
652
653 $code =~ s/\`([^\`]*)\`/eval $1/gem;
654 $code =~ s/\bbx\s+lr\b/.word\t0xe12fff1e/gm; # make it possible to compile with -march=armv4
655 $code =~ s/\bret\b/bx lr/gm;
656
657 open SELF,$0;
658 while(<SELF>) {
659 next if (/^#!/);
660 last if (!s/^#/@/ and !/^$/);
661 print;
662 }
663 close SELF;
664
665 print $code;
666 close STDOUT; # enforce flush