Debian Codegraph @

apple-aarch64/crypto/fipsmodule/p256-armv8-asm-apple.S

raw
1 // This file is generated from a similarly-named Perl script in the BoringSSL
2 // source tree. Do not edit by hand.
3
4 #if !defined(__has_feature)
5 #define __has_feature(x) 0
6 #endif
7 #if __has_feature(memory_sanitizer) && !defined(OPENSSL_NO_ASM)
8 #define OPENSSL_NO_ASM
9 #endif
10
11 #if !defined(OPENSSL_NO_ASM) && defined(__AARCH64EL__) && defined(__APPLE__)
12 #if defined(BORINGSSL_PREFIX)
13 #include <boringssl_prefix_symbols_asm.h>
14 #endif
15 #include "openssl/arm_arch.h"
16
17 .section __TEXT,__const
18 .align 5
19 Lpoly:
20 .quad 0xffffffffffffffff,0x00000000ffffffff,0x0000000000000000,0xffffffff00000001
21 LRR: // 2^512 mod P precomputed for NIST P256 polynomial
22 .quad 0x0000000000000003,0xfffffffbffffffff,0xfffffffffffffffe,0x00000004fffffffd
23 Lone_mont:
24 .quad 0x0000000000000001,0xffffffff00000000,0xffffffffffffffff,0x00000000fffffffe
25 Lone:
26 .quad 1,0,0,0
27 Lord:
28 .quad 0xf3b9cac2fc632551,0xbce6faada7179e84,0xffffffffffffffff,0xffffffff00000000
29 LordK:
30 .quad 0xccd1c8aaee00bc4f
31 .byte 69,67,80,95,78,73,83,84,90,50,53,54,32,102,111,114,32,65,82,77,118,56,44,32,67,82,89,80,84,79,71,65,77,83,32,98,121,32,60,97,112,112,114,111,64,111,112,101,110,115,115,108,46,111,114,103,62,0
32 .align 2
33 .text
34
35 // void ecp_nistz256_mul_mont(BN_ULONG x0[4],const BN_ULONG x1[4],
36 // const BN_ULONG x2[4]);
37 .globl _ecp_nistz256_mul_mont
38 .private_extern _ecp_nistz256_mul_mont
39
40 .align 4
41 _ecp_nistz256_mul_mont:
42 AARCH64_SIGN_LINK_REGISTER
43 stp x29,x30,[sp,#-32]!
44 add x29,sp,#0
45 stp x19,x20,[sp,#16]
46
47 ldr x3,[x2] // bp[0]
48 ldp x4,x5,[x1]
49 ldp x6,x7,[x1,#16]
50 adrp x13,Lpoly@PAGE
51 add x13,x13,Lpoly@PAGEOFF
52 ldr x12,[x13,#8]
53 ldr x13,[x13,#24]
54
55 bl __ecp_nistz256_mul_mont
56
57 ldp x19,x20,[sp,#16]
58 ldp x29,x30,[sp],#32
59 AARCH64_VALIDATE_LINK_REGISTER
60 ret
61
62
63 // void ecp_nistz256_sqr_mont(BN_ULONG x0[4],const BN_ULONG x1[4]);
64 .globl _ecp_nistz256_sqr_mont
65 .private_extern _ecp_nistz256_sqr_mont
66
67 .align 4
68 _ecp_nistz256_sqr_mont:
69 AARCH64_SIGN_LINK_REGISTER
70 stp x29,x30,[sp,#-32]!
71 add x29,sp,#0
72 stp x19,x20,[sp,#16]
73
74 ldp x4,x5,[x1]
75 ldp x6,x7,[x1,#16]
76 adrp x13,Lpoly@PAGE
77 add x13,x13,Lpoly@PAGEOFF
78 ldr x12,[x13,#8]
79 ldr x13,[x13,#24]
80
81 bl __ecp_nistz256_sqr_mont
82
83 ldp x19,x20,[sp,#16]
84 ldp x29,x30,[sp],#32
85 AARCH64_VALIDATE_LINK_REGISTER
86 ret
87
88
89 // void ecp_nistz256_div_by_2(BN_ULONG x0[4],const BN_ULONG x1[4]);
90 .globl _ecp_nistz256_div_by_2
91 .private_extern _ecp_nistz256_div_by_2
92
93 .align 4
94 _ecp_nistz256_div_by_2:
95 AARCH64_SIGN_LINK_REGISTER
96 stp x29,x30,[sp,#-16]!
97 add x29,sp,#0
98
99 ldp x14,x15,[x1]
100 ldp x16,x17,[x1,#16]
101 adrp x13,Lpoly@PAGE
102 add x13,x13,Lpoly@PAGEOFF
103 ldr x12,[x13,#8]
104 ldr x13,[x13,#24]
105
106 bl __ecp_nistz256_div_by_2
107
108 ldp x29,x30,[sp],#16
109 AARCH64_VALIDATE_LINK_REGISTER
110 ret
111
112
113 // void ecp_nistz256_mul_by_2(BN_ULONG x0[4],const BN_ULONG x1[4]);
114 .globl _ecp_nistz256_mul_by_2
115 .private_extern _ecp_nistz256_mul_by_2
116
117 .align 4
118 _ecp_nistz256_mul_by_2:
119 AARCH64_SIGN_LINK_REGISTER
120 stp x29,x30,[sp,#-16]!
121 add x29,sp,#0
122
123 ldp x14,x15,[x1]
124 ldp x16,x17,[x1,#16]
125 adrp x13,Lpoly@PAGE
126 add x13,x13,Lpoly@PAGEOFF
127 ldr x12,[x13,#8]
128 ldr x13,[x13,#24]
129 mov x8,x14
130 mov x9,x15
131 mov x10,x16
132 mov x11,x17
133
134 bl __ecp_nistz256_add_to // ret = a+a // 2*a
135
136 ldp x29,x30,[sp],#16
137 AARCH64_VALIDATE_LINK_REGISTER
138 ret
139
140
141 // void ecp_nistz256_mul_by_3(BN_ULONG x0[4],const BN_ULONG x1[4]);
142 .globl _ecp_nistz256_mul_by_3
143 .private_extern _ecp_nistz256_mul_by_3
144
145 .align 4
146 _ecp_nistz256_mul_by_3:
147 AARCH64_SIGN_LINK_REGISTER
148 stp x29,x30,[sp,#-16]!
149 add x29,sp,#0
150
151 ldp x14,x15,[x1]
152 ldp x16,x17,[x1,#16]
153 adrp x13,Lpoly@PAGE
154 add x13,x13,Lpoly@PAGEOFF
155 ldr x12,[x13,#8]
156 ldr x13,[x13,#24]
157 mov x8,x14
158 mov x9,x15
159 mov x10,x16
160 mov x11,x17
161 mov x4,x14
162 mov x5,x15
163 mov x6,x16
164 mov x7,x17
165
166 bl __ecp_nistz256_add_to // ret = a+a // 2*a
167
168 mov x8,x4
169 mov x9,x5
170 mov x10,x6
171 mov x11,x7
172
173 bl __ecp_nistz256_add_to // ret += a // 2*a+a=3*a
174
175 ldp x29,x30,[sp],#16
176 AARCH64_VALIDATE_LINK_REGISTER
177 ret
178
179
180 // void ecp_nistz256_sub(BN_ULONG x0[4],const BN_ULONG x1[4],
181 // const BN_ULONG x2[4]);
182 .globl _ecp_nistz256_sub
183 .private_extern _ecp_nistz256_sub
184
185 .align 4
186 _ecp_nistz256_sub:
187 AARCH64_SIGN_LINK_REGISTER
188 stp x29,x30,[sp,#-16]!
189 add x29,sp,#0
190
191 ldp x14,x15,[x1]
192 ldp x16,x17,[x1,#16]
193 adrp x13,Lpoly@PAGE
194 add x13,x13,Lpoly@PAGEOFF
195 ldr x12,[x13,#8]
196 ldr x13,[x13,#24]
197
198 bl __ecp_nistz256_sub_from
199
200 ldp x29,x30,[sp],#16
201 AARCH64_VALIDATE_LINK_REGISTER
202 ret
203
204
205 // void ecp_nistz256_neg(BN_ULONG x0[4],const BN_ULONG x1[4]);
206 .globl _ecp_nistz256_neg
207 .private_extern _ecp_nistz256_neg
208
209 .align 4
210 _ecp_nistz256_neg:
211 AARCH64_SIGN_LINK_REGISTER
212 stp x29,x30,[sp,#-16]!
213 add x29,sp,#0
214
215 mov x2,x1
216 mov x14,xzr // a = 0
217 mov x15,xzr
218 mov x16,xzr
219 mov x17,xzr
220 adrp x13,Lpoly@PAGE
221 add x13,x13,Lpoly@PAGEOFF
222 ldr x12,[x13,#8]
223 ldr x13,[x13,#24]
224
225 bl __ecp_nistz256_sub_from
226
227 ldp x29,x30,[sp],#16
228 AARCH64_VALIDATE_LINK_REGISTER
229 ret
230
231
232 // note that __ecp_nistz256_mul_mont expects a[0-3] input pre-loaded
233 // to x4-x7 and b[0] - to x3
234
235 .align 4
236 __ecp_nistz256_mul_mont:
237 mul x14,x4,x3 // a[0]*b[0]
238 umulh x8,x4,x3
239
240 mul x15,x5,x3 // a[1]*b[0]
241 umulh x9,x5,x3
242
243 mul x16,x6,x3 // a[2]*b[0]
244 umulh x10,x6,x3
245
246 mul x17,x7,x3 // a[3]*b[0]
247 umulh x11,x7,x3
248 ldr x3,[x2,#8] // b[1]
249
250 adds x15,x15,x8 // accumulate high parts of multiplication
251 lsl x8,x14,#32
252 adcs x16,x16,x9
253 lsr x9,x14,#32
254 adcs x17,x17,x10
255 adc x19,xzr,x11
256 mov x20,xzr
257 subs x10,x14,x8 // "*0xffff0001"
258 sbc x11,x14,x9
259 adds x14,x15,x8 // +=acc[0]<<96 and omit acc[0]
260 mul x8,x4,x3 // lo(a[0]*b[i])
261 adcs x15,x16,x9
262 mul x9,x5,x3 // lo(a[1]*b[i])
263 adcs x16,x17,x10 // +=acc[0]*0xffff0001
264 mul x10,x6,x3 // lo(a[2]*b[i])
265 adcs x17,x19,x11
266 mul x11,x7,x3 // lo(a[3]*b[i])
267 adc x19,x20,xzr
268
269 adds x14,x14,x8 // accumulate low parts of multiplication
270 umulh x8,x4,x3 // hi(a[0]*b[i])
271 adcs x15,x15,x9
272 umulh x9,x5,x3 // hi(a[1]*b[i])
273 adcs x16,x16,x10
274 umulh x10,x6,x3 // hi(a[2]*b[i])
275 adcs x17,x17,x11
276 umulh x11,x7,x3 // hi(a[3]*b[i])
277 adc x19,x19,xzr
278 ldr x3,[x2,#8*(1+1)] // b[1+1]
279 adds x15,x15,x8 // accumulate high parts of multiplication
280 lsl x8,x14,#32
281 adcs x16,x16,x9
282 lsr x9,x14,#32
283 adcs x17,x17,x10
284 adcs x19,x19,x11
285 adc x20,xzr,xzr
286 subs x10,x14,x8 // "*0xffff0001"
287 sbc x11,x14,x9
288 adds x14,x15,x8 // +=acc[0]<<96 and omit acc[0]
289 mul x8,x4,x3 // lo(a[0]*b[i])
290 adcs x15,x16,x9
291 mul x9,x5,x3 // lo(a[1]*b[i])
292 adcs x16,x17,x10 // +=acc[0]*0xffff0001
293 mul x10,x6,x3 // lo(a[2]*b[i])
294 adcs x17,x19,x11
295 mul x11,x7,x3 // lo(a[3]*b[i])
296 adc x19,x20,xzr
297
298 adds x14,x14,x8 // accumulate low parts of multiplication
299 umulh x8,x4,x3 // hi(a[0]*b[i])
300 adcs x15,x15,x9
301 umulh x9,x5,x3 // hi(a[1]*b[i])
302 adcs x16,x16,x10
303 umulh x10,x6,x3 // hi(a[2]*b[i])
304 adcs x17,x17,x11
305 umulh x11,x7,x3 // hi(a[3]*b[i])
306 adc x19,x19,xzr
307 ldr x3,[x2,#8*(2+1)] // b[2+1]
308 adds x15,x15,x8 // accumulate high parts of multiplication
309 lsl x8,x14,#32
310 adcs x16,x16,x9
311 lsr x9,x14,#32
312 adcs x17,x17,x10
313 adcs x19,x19,x11
314 adc x20,xzr,xzr
315 subs x10,x14,x8 // "*0xffff0001"
316 sbc x11,x14,x9
317 adds x14,x15,x8 // +=acc[0]<<96 and omit acc[0]
318 mul x8,x4,x3 // lo(a[0]*b[i])
319 adcs x15,x16,x9
320 mul x9,x5,x3 // lo(a[1]*b[i])
321 adcs x16,x17,x10 // +=acc[0]*0xffff0001
322 mul x10,x6,x3 // lo(a[2]*b[i])
323 adcs x17,x19,x11
324 mul x11,x7,x3 // lo(a[3]*b[i])
325 adc x19,x20,xzr
326
327 adds x14,x14,x8 // accumulate low parts of multiplication
328 umulh x8,x4,x3 // hi(a[0]*b[i])
329 adcs x15,x15,x9
330 umulh x9,x5,x3 // hi(a[1]*b[i])
331 adcs x16,x16,x10
332 umulh x10,x6,x3 // hi(a[2]*b[i])
333 adcs x17,x17,x11
334 umulh x11,x7,x3 // hi(a[3]*b[i])
335 adc x19,x19,xzr
336 adds x15,x15,x8 // accumulate high parts of multiplication
337 lsl x8,x14,#32
338 adcs x16,x16,x9
339 lsr x9,x14,#32
340 adcs x17,x17,x10
341 adcs x19,x19,x11
342 adc x20,xzr,xzr
343 // last reduction
344 subs x10,x14,x8 // "*0xffff0001"
345 sbc x11,x14,x9
346 adds x14,x15,x8 // +=acc[0]<<96 and omit acc[0]
347 adcs x15,x16,x9
348 adcs x16,x17,x10 // +=acc[0]*0xffff0001
349 adcs x17,x19,x11
350 adc x19,x20,xzr
351
352 adds x8,x14,#1 // subs x8,x14,#-1 // tmp = ret-modulus
353 sbcs x9,x15,x12
354 sbcs x10,x16,xzr
355 sbcs x11,x17,x13
356 sbcs xzr,x19,xzr // did it borrow?
357
358 csel x14,x14,x8,lo // ret = borrow ? ret : ret-modulus
359 csel x15,x15,x9,lo
360 csel x16,x16,x10,lo
361 stp x14,x15,[x0]
362 csel x17,x17,x11,lo
363 stp x16,x17,[x0,#16]
364
365 ret
366
367
368 // note that __ecp_nistz256_sqr_mont expects a[0-3] input pre-loaded
369 // to x4-x7
370
371 .align 4
372 __ecp_nistz256_sqr_mont:
373 // | | | | | |a1*a0| |
374 // | | | | |a2*a0| | |
375 // | |a3*a2|a3*a0| | | |
376 // | | | |a2*a1| | | |
377 // | | |a3*a1| | | | |
378 // *| | | | | | | | 2|
379 // +|a3*a3|a2*a2|a1*a1|a0*a0|
380 // |--+--+--+--+--+--+--+--|
381 // |A7|A6|A5|A4|A3|A2|A1|A0|, where Ax is , i.e. follow
382 //
383 // "can't overflow" below mark carrying into high part of
384 // multiplication result, which can't overflow, because it
385 // can never be all ones.
386
387 mul x15,x5,x4 // a[1]*a[0]
388 umulh x9,x5,x4
389 mul x16,x6,x4 // a[2]*a[0]
390 umulh x10,x6,x4
391 mul x17,x7,x4 // a[3]*a[0]
392 umulh x19,x7,x4
393
394 adds x16,x16,x9 // accumulate high parts of multiplication
395 mul x8,x6,x5 // a[2]*a[1]
396 umulh x9,x6,x5
397 adcs x17,x17,x10
398 mul x10,x7,x5 // a[3]*a[1]
399 umulh x11,x7,x5
400 adc x19,x19,xzr // can't overflow
401
402 mul x20,x7,x6 // a[3]*a[2]
403 umulh x1,x7,x6
404
405 adds x9,x9,x10 // accumulate high parts of multiplication
406 mul x14,x4,x4 // a[0]*a[0]
407 adc x10,x11,xzr // can't overflow
408
409 adds x17,x17,x8 // accumulate low parts of multiplication
410 umulh x4,x4,x4
411 adcs x19,x19,x9
412 mul x9,x5,x5 // a[1]*a[1]
413 adcs x20,x20,x10
414 umulh x5,x5,x5
415 adc x1,x1,xzr // can't overflow
416
417 adds x15,x15,x15 // acc[1-6]*=2
418 mul x10,x6,x6 // a[2]*a[2]
419 adcs x16,x16,x16
420 umulh x6,x6,x6
421 adcs x17,x17,x17
422 mul x11,x7,x7 // a[3]*a[3]
423 adcs x19,x19,x19
424 umulh x7,x7,x7
425 adcs x20,x20,x20
426 adcs x1,x1,x1
427 adc x2,xzr,xzr
428
429 adds x15,x15,x4 // +a[i]*a[i]
430 adcs x16,x16,x9
431 adcs x17,x17,x5
432 adcs x19,x19,x10
433 adcs x20,x20,x6
434 lsl x8,x14,#32
435 adcs x1,x1,x11
436 lsr x9,x14,#32
437 adc x2,x2,x7
438 subs x10,x14,x8 // "*0xffff0001"
439 sbc x11,x14,x9
440 adds x14,x15,x8 // +=acc[0]<<96 and omit acc[0]
441 adcs x15,x16,x9
442 lsl x8,x14,#32
443 adcs x16,x17,x10 // +=acc[0]*0xffff0001
444 lsr x9,x14,#32
445 adc x17,x11,xzr // can't overflow
446 subs x10,x14,x8 // "*0xffff0001"
447 sbc x11,x14,x9
448 adds x14,x15,x8 // +=acc[0]<<96 and omit acc[0]
449 adcs x15,x16,x9
450 lsl x8,x14,#32
451 adcs x16,x17,x10 // +=acc[0]*0xffff0001
452 lsr x9,x14,#32
453 adc x17,x11,xzr // can't overflow
454 subs x10,x14,x8 // "*0xffff0001"
455 sbc x11,x14,x9
456 adds x14,x15,x8 // +=acc[0]<<96 and omit acc[0]
457 adcs x15,x16,x9
458 lsl x8,x14,#32
459 adcs x16,x17,x10 // +=acc[0]*0xffff0001
460 lsr x9,x14,#32
461 adc x17,x11,xzr // can't overflow
462 subs x10,x14,x8 // "*0xffff0001"
463 sbc x11,x14,x9
464 adds x14,x15,x8 // +=acc[0]<<96 and omit acc[0]
465 adcs x15,x16,x9
466 adcs x16,x17,x10 // +=acc[0]*0xffff0001
467 adc x17,x11,xzr // can't overflow
468
469 adds x14,x14,x19 // accumulate upper half
470 adcs x15,x15,x20
471 adcs x16,x16,x1
472 adcs x17,x17,x2
473 adc x19,xzr,xzr
474
475 adds x8,x14,#1 // subs x8,x14,#-1 // tmp = ret-modulus
476 sbcs x9,x15,x12
477 sbcs x10,x16,xzr
478 sbcs x11,x17,x13
479 sbcs xzr,x19,xzr // did it borrow?
480
481 csel x14,x14,x8,lo // ret = borrow ? ret : ret-modulus
482 csel x15,x15,x9,lo
483 csel x16,x16,x10,lo
484 stp x14,x15,[x0]
485 csel x17,x17,x11,lo
486 stp x16,x17,[x0,#16]
487
488 ret
489
490
491 // Note that __ecp_nistz256_add_to expects both input vectors pre-loaded to
492 // x4-x7 and x8-x11. This is done because it's used in multiple
493 // contexts, e.g. in multiplication by 2 and 3...
494
495 .align 4
496 __ecp_nistz256_add_to:
497 adds x14,x14,x8 // ret = a+b
498 adcs x15,x15,x9
499 adcs x16,x16,x10
500 adcs x17,x17,x11
501 adc x1,xzr,xzr // zap x1
502
503 adds x8,x14,#1 // subs x8,x4,#-1 // tmp = ret-modulus
504 sbcs x9,x15,x12
505 sbcs x10,x16,xzr
506 sbcs x11,x17,x13
507 sbcs xzr,x1,xzr // did subtraction borrow?
508
509 csel x14,x14,x8,lo // ret = borrow ? ret : ret-modulus
510 csel x15,x15,x9,lo
511 csel x16,x16,x10,lo
512 stp x14,x15,[x0]
513 csel x17,x17,x11,lo
514 stp x16,x17,[x0,#16]
515
516 ret
517
518
519
520 .align 4
521 __ecp_nistz256_sub_from:
522 ldp x8,x9,[x2]
523 ldp x10,x11,[x2,#16]
524 subs x14,x14,x8 // ret = a-b
525 sbcs x15,x15,x9
526 sbcs x16,x16,x10
527 sbcs x17,x17,x11
528 sbc x1,xzr,xzr // zap x1
529
530 subs x8,x14,#1 // adds x8,x4,#-1 // tmp = ret+modulus
531 adcs x9,x15,x12
532 adcs x10,x16,xzr
533 adc x11,x17,x13
534 cmp x1,xzr // did subtraction borrow?
535
536 csel x14,x14,x8,eq // ret = borrow ? ret+modulus : ret
537 csel x15,x15,x9,eq
538 csel x16,x16,x10,eq
539 stp x14,x15,[x0]
540 csel x17,x17,x11,eq
541 stp x16,x17,[x0,#16]
542
543 ret
544
545
546
547 .align 4
548 __ecp_nistz256_sub_morf:
549 ldp x8,x9,[x2]
550 ldp x10,x11,[x2,#16]
551 subs x14,x8,x14 // ret = b-a
552 sbcs x15,x9,x15
553 sbcs x16,x10,x16
554 sbcs x17,x11,x17
555 sbc x1,xzr,xzr // zap x1
556
557 subs x8,x14,#1 // adds x8,x4,#-1 // tmp = ret+modulus
558 adcs x9,x15,x12
559 adcs x10,x16,xzr
560 adc x11,x17,x13
561 cmp x1,xzr // did subtraction borrow?
562
563 csel x14,x14,x8,eq // ret = borrow ? ret+modulus : ret
564 csel x15,x15,x9,eq
565 csel x16,x16,x10,eq
566 stp x14,x15,[x0]
567 csel x17,x17,x11,eq
568 stp x16,x17,[x0,#16]
569
570 ret
571
572
573
574 .align 4
575 __ecp_nistz256_div_by_2:
576 subs x8,x14,#1 // adds x8,x4,#-1 // tmp = a+modulus
577 adcs x9,x15,x12
578 adcs x10,x16,xzr
579 adcs x11,x17,x13
580 adc x1,xzr,xzr // zap x1
581 tst x14,#1 // is a even?
582
583 csel x14,x14,x8,eq // ret = even ? a : a+modulus
584 csel x15,x15,x9,eq
585 csel x16,x16,x10,eq
586 csel x17,x17,x11,eq
587 csel x1,xzr,x1,eq
588
589 lsr x14,x14,#1 // ret >>= 1
590 orr x14,x14,x15,lsl#63
591 lsr x15,x15,#1
592 orr x15,x15,x16,lsl#63
593 lsr x16,x16,#1
594 orr x16,x16,x17,lsl#63
595 lsr x17,x17,#1
596 stp x14,x15,[x0]
597 orr x17,x17,x1,lsl#63
598 stp x16,x17,[x0,#16]
599
600 ret
601
602 .globl _ecp_nistz256_point_double
603 .private_extern _ecp_nistz256_point_double
604
605 .align 5
606 _ecp_nistz256_point_double:
607 AARCH64_SIGN_LINK_REGISTER
608 stp x29,x30,[sp,#-96]!
609 add x29,sp,#0
610 stp x19,x20,[sp,#16]
611 stp x21,x22,[sp,#32]
612 sub sp,sp,#32*4
613
614 Ldouble_shortcut:
615 ldp x14,x15,[x1,#32]
616 mov x21,x0
617 ldp x16,x17,[x1,#48]
618 mov x22,x1
619 adrp x13,Lpoly@PAGE
620 add x13,x13,Lpoly@PAGEOFF
621 ldr x12,[x13,#8]
622 mov x8,x14
623 ldr x13,[x13,#24]
624 mov x9,x15
625 ldp x4,x5,[x22,#64] // forward load for p256_sqr_mont
626 mov x10,x16
627 mov x11,x17
628 ldp x6,x7,[x22,#64+16]
629 add x0,sp,#0
630 bl __ecp_nistz256_add_to // p256_mul_by_2(S, in_y);
631
632 add x0,sp,#64
633 bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Zsqr, in_z);
634
635 ldp x8,x9,[x22]
636 ldp x10,x11,[x22,#16]
637 mov x4,x14 // put Zsqr aside for p256_sub
638 mov x5,x15
639 mov x6,x16
640 mov x7,x17
641 add x0,sp,#32
642 bl __ecp_nistz256_add_to // p256_add(M, Zsqr, in_x);
643
644 add x2,x22,#0
645 mov x14,x4 // restore Zsqr
646 mov x15,x5
647 ldp x4,x5,[sp,#0] // forward load for p256_sqr_mont
648 mov x16,x6
649 mov x17,x7
650 ldp x6,x7,[sp,#0+16]
651 add x0,sp,#64
652 bl __ecp_nistz256_sub_morf // p256_sub(Zsqr, in_x, Zsqr);
653
654 add x0,sp,#0
655 bl __ecp_nistz256_sqr_mont // p256_sqr_mont(S, S);
656
657 ldr x3,[x22,#32]
658 ldp x4,x5,[x22,#64]
659 ldp x6,x7,[x22,#64+16]
660 add x2,x22,#32
661 add x0,sp,#96
662 bl __ecp_nistz256_mul_mont // p256_mul_mont(tmp0, in_z, in_y);
663
664 mov x8,x14
665 mov x9,x15
666 ldp x4,x5,[sp,#0] // forward load for p256_sqr_mont
667 mov x10,x16
668 mov x11,x17
669 ldp x6,x7,[sp,#0+16]
670 add x0,x21,#64
671 bl __ecp_nistz256_add_to // p256_mul_by_2(res_z, tmp0);
672
673 add x0,sp,#96
674 bl __ecp_nistz256_sqr_mont // p256_sqr_mont(tmp0, S);
675
676 ldr x3,[sp,#64] // forward load for p256_mul_mont
677 ldp x4,x5,[sp,#32]
678 ldp x6,x7,[sp,#32+16]
679 add x0,x21,#32
680 bl __ecp_nistz256_div_by_2 // p256_div_by_2(res_y, tmp0);
681
682 add x2,sp,#64
683 add x0,sp,#32
684 bl __ecp_nistz256_mul_mont // p256_mul_mont(M, M, Zsqr);
685
686 mov x8,x14 // duplicate M
687 mov x9,x15
688 mov x10,x16
689 mov x11,x17
690 mov x4,x14 // put M aside
691 mov x5,x15
692 mov x6,x16
693 mov x7,x17
694 add x0,sp,#32
695 bl __ecp_nistz256_add_to
696 mov x8,x4 // restore M
697 mov x9,x5
698 ldr x3,[x22] // forward load for p256_mul_mont
699 mov x10,x6
700 ldp x4,x5,[sp,#0]
701 mov x11,x7
702 ldp x6,x7,[sp,#0+16]
703 bl __ecp_nistz256_add_to // p256_mul_by_3(M, M);
704
705 add x2,x22,#0
706 add x0,sp,#0
707 bl __ecp_nistz256_mul_mont // p256_mul_mont(S, S, in_x);
708
709 mov x8,x14
710 mov x9,x15
711 ldp x4,x5,[sp,#32] // forward load for p256_sqr_mont
712 mov x10,x16
713 mov x11,x17
714 ldp x6,x7,[sp,#32+16]
715 add x0,sp,#96
716 bl __ecp_nistz256_add_to // p256_mul_by_2(tmp0, S);
717
718 add x0,x21,#0
719 bl __ecp_nistz256_sqr_mont // p256_sqr_mont(res_x, M);
720
721 add x2,sp,#96
722 bl __ecp_nistz256_sub_from // p256_sub(res_x, res_x, tmp0);
723
724 add x2,sp,#0
725 add x0,sp,#0
726 bl __ecp_nistz256_sub_morf // p256_sub(S, S, res_x);
727
728 ldr x3,[sp,#32]
729 mov x4,x14 // copy S
730 mov x5,x15
731 mov x6,x16
732 mov x7,x17
733 add x2,sp,#32
734 bl __ecp_nistz256_mul_mont // p256_mul_mont(S, S, M);
735
736 add x2,x21,#32
737 add x0,x21,#32
738 bl __ecp_nistz256_sub_from // p256_sub(res_y, S, res_y);
739
740 add sp,x29,#0 // destroy frame
741 ldp x19,x20,[x29,#16]
742 ldp x21,x22,[x29,#32]
743 ldp x29,x30,[sp],#96
744 AARCH64_VALIDATE_LINK_REGISTER
745 ret
746
747 .globl _ecp_nistz256_point_add
748 .private_extern _ecp_nistz256_point_add
749
750 .align 5
751 _ecp_nistz256_point_add:
752 AARCH64_SIGN_LINK_REGISTER
753 stp x29,x30,[sp,#-96]!
754 add x29,sp,#0
755 stp x19,x20,[sp,#16]
756 stp x21,x22,[sp,#32]
757 stp x23,x24,[sp,#48]
758 stp x25,x26,[sp,#64]
759 stp x27,x28,[sp,#80]
760 sub sp,sp,#32*12
761
762 ldp x4,x5,[x2,#64] // in2_z
763 ldp x6,x7,[x2,#64+16]
764 mov x21,x0
765 mov x22,x1
766 mov x23,x2
767 adrp x13,Lpoly@PAGE
768 add x13,x13,Lpoly@PAGEOFF
769 ldr x12,[x13,#8]
770 ldr x13,[x13,#24]
771 orr x8,x4,x5
772 orr x10,x6,x7
773 orr x25,x8,x10
774 cmp x25,#0
775 csetm x25,ne // ~in2infty
776 add x0,sp,#192
777 bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Z2sqr, in2_z);
778
779 ldp x4,x5,[x22,#64] // in1_z
780 ldp x6,x7,[x22,#64+16]
781 orr x8,x4,x5
782 orr x10,x6,x7
783 orr x24,x8,x10
784 cmp x24,#0
785 csetm x24,ne // ~in1infty
786 add x0,sp,#128
787 bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Z1sqr, in1_z);
788
789 ldr x3,[x23,#64]
790 ldp x4,x5,[sp,#192]
791 ldp x6,x7,[sp,#192+16]
792 add x2,x23,#64
793 add x0,sp,#320
794 bl __ecp_nistz256_mul_mont // p256_mul_mont(S1, Z2sqr, in2_z);
795
796 ldr x3,[x22,#64]
797 ldp x4,x5,[sp,#128]
798 ldp x6,x7,[sp,#128+16]
799 add x2,x22,#64
800 add x0,sp,#352
801 bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, Z1sqr, in1_z);
802
803 ldr x3,[x22,#32]
804 ldp x4,x5,[sp,#320]
805 ldp x6,x7,[sp,#320+16]
806 add x2,x22,#32
807 add x0,sp,#320
808 bl __ecp_nistz256_mul_mont // p256_mul_mont(S1, S1, in1_y);
809
810 ldr x3,[x23,#32]
811 ldp x4,x5,[sp,#352]
812 ldp x6,x7,[sp,#352+16]
813 add x2,x23,#32
814 add x0,sp,#352
815 bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, S2, in2_y);
816
817 add x2,sp,#320
818 ldr x3,[sp,#192] // forward load for p256_mul_mont
819 ldp x4,x5,[x22]
820 ldp x6,x7,[x22,#16]
821 add x0,sp,#160
822 bl __ecp_nistz256_sub_from // p256_sub(R, S2, S1);
823
824 orr x14,x14,x15 // see if result is zero
825 orr x16,x16,x17
826 orr x26,x14,x16 // ~is_equal(S1,S2)
827
828 add x2,sp,#192
829 add x0,sp,#256
830 bl __ecp_nistz256_mul_mont // p256_mul_mont(U1, in1_x, Z2sqr);
831
832 ldr x3,[sp,#128]
833 ldp x4,x5,[x23]
834 ldp x6,x7,[x23,#16]
835 add x2,sp,#128
836 add x0,sp,#288
837 bl __ecp_nistz256_mul_mont // p256_mul_mont(U2, in2_x, Z1sqr);
838
839 add x2,sp,#256
840 ldp x4,x5,[sp,#160] // forward load for p256_sqr_mont
841 ldp x6,x7,[sp,#160+16]
842 add x0,sp,#96
843 bl __ecp_nistz256_sub_from // p256_sub(H, U2, U1);
844
845 orr x14,x14,x15 // see if result is zero
846 orr x16,x16,x17
847 orr x14,x14,x16 // ~is_equal(U1,U2)
848
849 mvn x27,x24 // -1/0 -> 0/-1
850 mvn x28,x25 // -1/0 -> 0/-1
851 orr x14,x14,x27
852 orr x14,x14,x28
853 orr x14,x14,x26
854 cbnz x14,Ladd_proceed // if(~is_equal(U1,U2) | in1infty | in2infty | ~is_equal(S1,S2))
855
856 Ladd_double:
857 mov x1,x22
858 mov x0,x21
859 ldp x23,x24,[x29,#48]
860 ldp x25,x26,[x29,#64]
861 ldp x27,x28,[x29,#80]
862 add sp,sp,#256 // #256 is from #32*(12-4). difference in stack frames
863 b Ldouble_shortcut
864
865 .align 4
866 Ladd_proceed:
867 add x0,sp,#192
868 bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Rsqr, R);
869
870 ldr x3,[x22,#64]
871 ldp x4,x5,[sp,#96]
872 ldp x6,x7,[sp,#96+16]
873 add x2,x22,#64
874 add x0,sp,#64
875 bl __ecp_nistz256_mul_mont // p256_mul_mont(res_z, H, in1_z);
876
877 ldp x4,x5,[sp,#96]
878 ldp x6,x7,[sp,#96+16]
879 add x0,sp,#128
880 bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Hsqr, H);
881
882 ldr x3,[x23,#64]
883 ldp x4,x5,[sp,#64]
884 ldp x6,x7,[sp,#64+16]
885 add x2,x23,#64
886 add x0,sp,#64
887 bl __ecp_nistz256_mul_mont // p256_mul_mont(res_z, res_z, in2_z);
888
889 ldr x3,[sp,#96]
890 ldp x4,x5,[sp,#128]
891 ldp x6,x7,[sp,#128+16]
892 add x2,sp,#96
893 add x0,sp,#224
894 bl __ecp_nistz256_mul_mont // p256_mul_mont(Hcub, Hsqr, H);
895
896 ldr x3,[sp,#128]
897 ldp x4,x5,[sp,#256]
898 ldp x6,x7,[sp,#256+16]
899 add x2,sp,#128
900 add x0,sp,#288
901 bl __ecp_nistz256_mul_mont // p256_mul_mont(U2, U1, Hsqr);
902
903 mov x8,x14
904 mov x9,x15
905 mov x10,x16
906 mov x11,x17
907 add x0,sp,#128
908 bl __ecp_nistz256_add_to // p256_mul_by_2(Hsqr, U2);
909
910 add x2,sp,#192
911 add x0,sp,#0
912 bl __ecp_nistz256_sub_morf // p256_sub(res_x, Rsqr, Hsqr);
913
914 add x2,sp,#224
915 bl __ecp_nistz256_sub_from // p256_sub(res_x, res_x, Hcub);
916
917 add x2,sp,#288
918 ldr x3,[sp,#224] // forward load for p256_mul_mont
919 ldp x4,x5,[sp,#320]
920 ldp x6,x7,[sp,#320+16]
921 add x0,sp,#32
922 bl __ecp_nistz256_sub_morf // p256_sub(res_y, U2, res_x);
923
924 add x2,sp,#224
925 add x0,sp,#352
926 bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, S1, Hcub);
927
928 ldr x3,[sp,#160]
929 ldp x4,x5,[sp,#32]
930 ldp x6,x7,[sp,#32+16]
931 add x2,sp,#160
932 add x0,sp,#32
933 bl __ecp_nistz256_mul_mont // p256_mul_mont(res_y, res_y, R);
934
935 add x2,sp,#352
936 bl __ecp_nistz256_sub_from // p256_sub(res_y, res_y, S2);
937
938 ldp x4,x5,[sp,#0] // res
939 ldp x6,x7,[sp,#0+16]
940 ldp x8,x9,[x23] // in2
941 ldp x10,x11,[x23,#16]
942 ldp x14,x15,[x22,#0] // in1
943 cmp x24,#0 // ~, remember?
944 ldp x16,x17,[x22,#0+16]
945 csel x8,x4,x8,ne
946 csel x9,x5,x9,ne
947 ldp x4,x5,[sp,#0+0+32] // res
948 csel x10,x6,x10,ne
949 csel x11,x7,x11,ne
950 cmp x25,#0 // ~, remember?
951 ldp x6,x7,[sp,#0+0+48]
952 csel x14,x8,x14,ne
953 csel x15,x9,x15,ne
954 ldp x8,x9,[x23,#0+32] // in2
955 csel x16,x10,x16,ne
956 csel x17,x11,x17,ne
957 ldp x10,x11,[x23,#0+48]
958 stp x14,x15,[x21,#0]
959 stp x16,x17,[x21,#0+16]
960 ldp x14,x15,[x22,#32] // in1
961 cmp x24,#0 // ~, remember?
962 ldp x16,x17,[x22,#32+16]
963 csel x8,x4,x8,ne
964 csel x9,x5,x9,ne
965 ldp x4,x5,[sp,#0+32+32] // res
966 csel x10,x6,x10,ne
967 csel x11,x7,x11,ne
968 cmp x25,#0 // ~, remember?
969 ldp x6,x7,[sp,#0+32+48]
970 csel x14,x8,x14,ne
971 csel x15,x9,x15,ne
972 ldp x8,x9,[x23,#32+32] // in2
973 csel x16,x10,x16,ne
974 csel x17,x11,x17,ne
975 ldp x10,x11,[x23,#32+48]
976 stp x14,x15,[x21,#32]
977 stp x16,x17,[x21,#32+16]
978 ldp x14,x15,[x22,#64] // in1
979 cmp x24,#0 // ~, remember?
980 ldp x16,x17,[x22,#64+16]
981 csel x8,x4,x8,ne
982 csel x9,x5,x9,ne
983 csel x10,x6,x10,ne
984 csel x11,x7,x11,ne
985 cmp x25,#0 // ~, remember?
986 csel x14,x8,x14,ne
987 csel x15,x9,x15,ne
988 csel x16,x10,x16,ne
989 csel x17,x11,x17,ne
990 stp x14,x15,[x21,#64]
991 stp x16,x17,[x21,#64+16]
992
993 Ladd_done:
994 add sp,x29,#0 // destroy frame
995 ldp x19,x20,[x29,#16]
996 ldp x21,x22,[x29,#32]
997 ldp x23,x24,[x29,#48]
998 ldp x25,x26,[x29,#64]
999 ldp x27,x28,[x29,#80]
1000 ldp x29,x30,[sp],#96
1001 AARCH64_VALIDATE_LINK_REGISTER
1002 ret
1003
1004 .globl _ecp_nistz256_point_add_affine
1005 .private_extern _ecp_nistz256_point_add_affine
1006
1007 .align 5
1008 _ecp_nistz256_point_add_affine:
1009 AARCH64_SIGN_LINK_REGISTER
1010 stp x29,x30,[sp,#-80]!
1011 add x29,sp,#0
1012 stp x19,x20,[sp,#16]
1013 stp x21,x22,[sp,#32]
1014 stp x23,x24,[sp,#48]
1015 stp x25,x26,[sp,#64]
1016 sub sp,sp,#32*10
1017
1018 mov x21,x0
1019 mov x22,x1
1020 mov x23,x2
1021 adrp x13,Lpoly@PAGE
1022 add x13,x13,Lpoly@PAGEOFF
1023 ldr x12,[x13,#8]
1024 ldr x13,[x13,#24]
1025
1026 ldp x4,x5,[x1,#64] // in1_z
1027 ldp x6,x7,[x1,#64+16]
1028 orr x8,x4,x5
1029 orr x10,x6,x7
1030 orr x24,x8,x10
1031 cmp x24,#0
1032 csetm x24,ne // ~in1infty
1033
1034 ldp x14,x15,[x2] // in2_x
1035 ldp x16,x17,[x2,#16]
1036 ldp x8,x9,[x2,#32] // in2_y
1037 ldp x10,x11,[x2,#48]
1038 orr x14,x14,x15
1039 orr x16,x16,x17
1040 orr x8,x8,x9
1041 orr x10,x10,x11
1042 orr x14,x14,x16
1043 orr x8,x8,x10
1044 orr x25,x14,x8
1045 cmp x25,#0
1046 csetm x25,ne // ~in2infty
1047
1048 add x0,sp,#128
1049 bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Z1sqr, in1_z);
1050
1051 mov x4,x14
1052 mov x5,x15
1053 mov x6,x16
1054 mov x7,x17
1055 ldr x3,[x23]
1056 add x2,x23,#0
1057 add x0,sp,#96
1058 bl __ecp_nistz256_mul_mont // p256_mul_mont(U2, Z1sqr, in2_x);
1059
1060 add x2,x22,#0
1061 ldr x3,[x22,#64] // forward load for p256_mul_mont
1062 ldp x4,x5,[sp,#128]
1063 ldp x6,x7,[sp,#128+16]
1064 add x0,sp,#160
1065 bl __ecp_nistz256_sub_from // p256_sub(H, U2, in1_x);
1066
1067 add x2,x22,#64
1068 add x0,sp,#128
1069 bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, Z1sqr, in1_z);
1070
1071 ldr x3,[x22,#64]
1072 ldp x4,x5,[sp,#160]
1073 ldp x6,x7,[sp,#160+16]
1074 add x2,x22,#64
1075 add x0,sp,#64
1076 bl __ecp_nistz256_mul_mont // p256_mul_mont(res_z, H, in1_z);
1077
1078 ldr x3,[x23,#32]
1079 ldp x4,x5,[sp,#128]
1080 ldp x6,x7,[sp,#128+16]
1081 add x2,x23,#32
1082 add x0,sp,#128
1083 bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, S2, in2_y);
1084
1085 add x2,x22,#32
1086 ldp x4,x5,[sp,#160] // forward load for p256_sqr_mont
1087 ldp x6,x7,[sp,#160+16]
1088 add x0,sp,#192
1089 bl __ecp_nistz256_sub_from // p256_sub(R, S2, in1_y);
1090
1091 add x0,sp,#224
1092 bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Hsqr, H);
1093
1094 ldp x4,x5,[sp,#192]
1095 ldp x6,x7,[sp,#192+16]
1096 add x0,sp,#288
1097 bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Rsqr, R);
1098
1099 ldr x3,[sp,#160]
1100 ldp x4,x5,[sp,#224]
1101 ldp x6,x7,[sp,#224+16]
1102 add x2,sp,#160
1103 add x0,sp,#256
1104 bl __ecp_nistz256_mul_mont // p256_mul_mont(Hcub, Hsqr, H);
1105
1106 ldr x3,[x22]
1107 ldp x4,x5,[sp,#224]
1108 ldp x6,x7,[sp,#224+16]
1109 add x2,x22,#0
1110 add x0,sp,#96
1111 bl __ecp_nistz256_mul_mont // p256_mul_mont(U2, in1_x, Hsqr);
1112
1113 mov x8,x14
1114 mov x9,x15
1115 mov x10,x16
1116 mov x11,x17
1117 add x0,sp,#224
1118 bl __ecp_nistz256_add_to // p256_mul_by_2(Hsqr, U2);
1119
1120 add x2,sp,#288
1121 add x0,sp,#0
1122 bl __ecp_nistz256_sub_morf // p256_sub(res_x, Rsqr, Hsqr);
1123
1124 add x2,sp,#256
1125 bl __ecp_nistz256_sub_from // p256_sub(res_x, res_x, Hcub);
1126
1127 add x2,sp,#96
1128 ldr x3,[x22,#32] // forward load for p256_mul_mont
1129 ldp x4,x5,[sp,#256]
1130 ldp x6,x7,[sp,#256+16]
1131 add x0,sp,#32
1132 bl __ecp_nistz256_sub_morf // p256_sub(res_y, U2, res_x);
1133
1134 add x2,x22,#32
1135 add x0,sp,#128
1136 bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, in1_y, Hcub);
1137
1138 ldr x3,[sp,#192]
1139 ldp x4,x5,[sp,#32]
1140 ldp x6,x7,[sp,#32+16]
1141 add x2,sp,#192
1142 add x0,sp,#32
1143 bl __ecp_nistz256_mul_mont // p256_mul_mont(res_y, res_y, R);
1144
1145 add x2,sp,#128
1146 bl __ecp_nistz256_sub_from // p256_sub(res_y, res_y, S2);
1147
1148 ldp x4,x5,[sp,#0] // res
1149 ldp x6,x7,[sp,#0+16]
1150 ldp x8,x9,[x23] // in2
1151 ldp x10,x11,[x23,#16]
1152 ldp x14,x15,[x22,#0] // in1
1153 cmp x24,#0 // ~, remember?
1154 ldp x16,x17,[x22,#0+16]
1155 csel x8,x4,x8,ne
1156 csel x9,x5,x9,ne
1157 ldp x4,x5,[sp,#0+0+32] // res
1158 csel x10,x6,x10,ne
1159 csel x11,x7,x11,ne
1160 cmp x25,#0 // ~, remember?
1161 ldp x6,x7,[sp,#0+0+48]
1162 csel x14,x8,x14,ne
1163 csel x15,x9,x15,ne
1164 ldp x8,x9,[x23,#0+32] // in2
1165 csel x16,x10,x16,ne
1166 csel x17,x11,x17,ne
1167 ldp x10,x11,[x23,#0+48]
1168 stp x14,x15,[x21,#0]
1169 stp x16,x17,[x21,#0+16]
1170 adrp x23,Lone_mont@PAGE-64
1171 add x23,x23,Lone_mont@PAGEOFF-64
1172 ldp x14,x15,[x22,#32] // in1
1173 cmp x24,#0 // ~, remember?
1174 ldp x16,x17,[x22,#32+16]
1175 csel x8,x4,x8,ne
1176 csel x9,x5,x9,ne
1177 ldp x4,x5,[sp,#0+32+32] // res
1178 csel x10,x6,x10,ne
1179 csel x11,x7,x11,ne
1180 cmp x25,#0 // ~, remember?
1181 ldp x6,x7,[sp,#0+32+48]
1182 csel x14,x8,x14,ne
1183 csel x15,x9,x15,ne
1184 ldp x8,x9,[x23,#32+32] // in2
1185 csel x16,x10,x16,ne
1186 csel x17,x11,x17,ne
1187 ldp x10,x11,[x23,#32+48]
1188 stp x14,x15,[x21,#32]
1189 stp x16,x17,[x21,#32+16]
1190 ldp x14,x15,[x22,#64] // in1
1191 cmp x24,#0 // ~, remember?
1192 ldp x16,x17,[x22,#64+16]
1193 csel x8,x4,x8,ne
1194 csel x9,x5,x9,ne
1195 csel x10,x6,x10,ne
1196 csel x11,x7,x11,ne
1197 cmp x25,#0 // ~, remember?
1198 csel x14,x8,x14,ne
1199 csel x15,x9,x15,ne
1200 csel x16,x10,x16,ne
1201 csel x17,x11,x17,ne
1202 stp x14,x15,[x21,#64]
1203 stp x16,x17,[x21,#64+16]
1204
1205 add sp,x29,#0 // destroy frame
1206 ldp x19,x20,[x29,#16]
1207 ldp x21,x22,[x29,#32]
1208 ldp x23,x24,[x29,#48]
1209 ldp x25,x26,[x29,#64]
1210 ldp x29,x30,[sp],#80
1211 AARCH64_VALIDATE_LINK_REGISTER
1212 ret
1213
1214 ////////////////////////////////////////////////////////////////////////
1215 // void ecp_nistz256_ord_mul_mont(uint64_t res[4], uint64_t a[4],
1216 // uint64_t b[4]);
1217 .globl _ecp_nistz256_ord_mul_mont
1218 .private_extern _ecp_nistz256_ord_mul_mont
1219
1220 .align 4
1221 _ecp_nistz256_ord_mul_mont:
1222 AARCH64_VALID_CALL_TARGET
1223 // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later.
1224 stp x29,x30,[sp,#-64]!
1225 add x29,sp,#0
1226 stp x19,x20,[sp,#16]
1227 stp x21,x22,[sp,#32]
1228 stp x23,x24,[sp,#48]
1229
1230 adrp x23,Lord@PAGE
1231 add x23,x23,Lord@PAGEOFF
1232 ldr x3,[x2] // bp[0]
1233 ldp x4,x5,[x1]
1234 ldp x6,x7,[x1,#16]
1235
1236 ldp x12,x13,[x23,#0]
1237 ldp x21,x22,[x23,#16]
1238 ldr x23,[x23,#32]
1239
1240 mul x14,x4,x3 // a[0]*b[0]
1241 umulh x8,x4,x3
1242
1243 mul x15,x5,x3 // a[1]*b[0]
1244 umulh x9,x5,x3
1245
1246 mul x16,x6,x3 // a[2]*b[0]
1247 umulh x10,x6,x3
1248
1249 mul x17,x7,x3 // a[3]*b[0]
1250 umulh x19,x7,x3
1251
1252 mul x24,x14,x23
1253
1254 adds x15,x15,x8 // accumulate high parts of multiplication
1255 adcs x16,x16,x9
1256 adcs x17,x17,x10
1257 adc x19,x19,xzr
1258 mov x20,xzr
1259 ldr x3,[x2,#8*1] // b[i]
1260
1261 lsl x8,x24,#32
1262 subs x16,x16,x24
1263 lsr x9,x24,#32
1264 sbcs x17,x17,x8
1265 sbcs x19,x19,x9
1266 sbc x20,x20,xzr
1267
1268 subs xzr,x14,#1
1269 umulh x9,x12,x24
1270 mul x10,x13,x24
1271 umulh x11,x13,x24
1272
1273 adcs x10,x10,x9
1274 mul x8,x4,x3
1275 adc x11,x11,xzr
1276 mul x9,x5,x3
1277
1278 adds x14,x15,x10
1279 mul x10,x6,x3
1280 adcs x15,x16,x11
1281 mul x11,x7,x3
1282 adcs x16,x17,x24
1283 adcs x17,x19,x24
1284 adc x19,x20,xzr
1285
1286 adds x14,x14,x8 // accumulate low parts
1287 umulh x8,x4,x3
1288 adcs x15,x15,x9
1289 umulh x9,x5,x3
1290 adcs x16,x16,x10
1291 umulh x10,x6,x3
1292 adcs x17,x17,x11
1293 umulh x11,x7,x3
1294 adc x19,x19,xzr
1295 mul x24,x14,x23
1296 adds x15,x15,x8 // accumulate high parts
1297 adcs x16,x16,x9
1298 adcs x17,x17,x10
1299 adcs x19,x19,x11
1300 adc x20,xzr,xzr
1301 ldr x3,[x2,#8*2] // b[i]
1302
1303 lsl x8,x24,#32
1304 subs x16,x16,x24
1305 lsr x9,x24,#32
1306 sbcs x17,x17,x8
1307 sbcs x19,x19,x9
1308 sbc x20,x20,xzr
1309
1310 subs xzr,x14,#1
1311 umulh x9,x12,x24
1312 mul x10,x13,x24
1313 umulh x11,x13,x24
1314
1315 adcs x10,x10,x9
1316 mul x8,x4,x3
1317 adc x11,x11,xzr
1318 mul x9,x5,x3
1319
1320 adds x14,x15,x10
1321 mul x10,x6,x3
1322 adcs x15,x16,x11
1323 mul x11,x7,x3
1324 adcs x16,x17,x24
1325 adcs x17,x19,x24
1326 adc x19,x20,xzr
1327
1328 adds x14,x14,x8 // accumulate low parts
1329 umulh x8,x4,x3
1330 adcs x15,x15,x9
1331 umulh x9,x5,x3
1332 adcs x16,x16,x10
1333 umulh x10,x6,x3
1334 adcs x17,x17,x11
1335 umulh x11,x7,x3
1336 adc x19,x19,xzr
1337 mul x24,x14,x23
1338 adds x15,x15,x8 // accumulate high parts
1339 adcs x16,x16,x9
1340 adcs x17,x17,x10
1341 adcs x19,x19,x11
1342 adc x20,xzr,xzr
1343 ldr x3,[x2,#8*3] // b[i]
1344
1345 lsl x8,x24,#32
1346 subs x16,x16,x24
1347 lsr x9,x24,#32
1348 sbcs x17,x17,x8
1349 sbcs x19,x19,x9
1350 sbc x20,x20,xzr
1351
1352 subs xzr,x14,#1
1353 umulh x9,x12,x24
1354 mul x10,x13,x24
1355 umulh x11,x13,x24
1356
1357 adcs x10,x10,x9
1358 mul x8,x4,x3
1359 adc x11,x11,xzr
1360 mul x9,x5,x3
1361
1362 adds x14,x15,x10
1363 mul x10,x6,x3
1364 adcs x15,x16,x11
1365 mul x11,x7,x3
1366 adcs x16,x17,x24
1367 adcs x17,x19,x24
1368 adc x19,x20,xzr
1369
1370 adds x14,x14,x8 // accumulate low parts
1371 umulh x8,x4,x3
1372 adcs x15,x15,x9
1373 umulh x9,x5,x3
1374 adcs x16,x16,x10
1375 umulh x10,x6,x3
1376 adcs x17,x17,x11
1377 umulh x11,x7,x3
1378 adc x19,x19,xzr
1379 mul x24,x14,x23
1380 adds x15,x15,x8 // accumulate high parts
1381 adcs x16,x16,x9
1382 adcs x17,x17,x10
1383 adcs x19,x19,x11
1384 adc x20,xzr,xzr
1385 lsl x8,x24,#32 // last reduction
1386 subs x16,x16,x24
1387 lsr x9,x24,#32
1388 sbcs x17,x17,x8
1389 sbcs x19,x19,x9
1390 sbc x20,x20,xzr
1391
1392 subs xzr,x14,#1
1393 umulh x9,x12,x24
1394 mul x10,x13,x24
1395 umulh x11,x13,x24
1396
1397 adcs x10,x10,x9
1398 adc x11,x11,xzr
1399
1400 adds x14,x15,x10
1401 adcs x15,x16,x11
1402 adcs x16,x17,x24
1403 adcs x17,x19,x24
1404 adc x19,x20,xzr
1405
1406 subs x8,x14,x12 // ret -= modulus
1407 sbcs x9,x15,x13
1408 sbcs x10,x16,x21
1409 sbcs x11,x17,x22
1410 sbcs xzr,x19,xzr
1411
1412 csel x14,x14,x8,lo // ret = borrow ? ret : ret-modulus
1413 csel x15,x15,x9,lo
1414 csel x16,x16,x10,lo
1415 stp x14,x15,[x0]
1416 csel x17,x17,x11,lo
1417 stp x16,x17,[x0,#16]
1418
1419 ldp x19,x20,[sp,#16]
1420 ldp x21,x22,[sp,#32]
1421 ldp x23,x24,[sp,#48]
1422 ldr x29,[sp],#64
1423 ret
1424
1425
1426 ////////////////////////////////////////////////////////////////////////
1427 // void ecp_nistz256_ord_sqr_mont(uint64_t res[4], uint64_t a[4],
1428 // uint64_t rep);
1429 .globl _ecp_nistz256_ord_sqr_mont
1430 .private_extern _ecp_nistz256_ord_sqr_mont
1431
1432 .align 4
1433 _ecp_nistz256_ord_sqr_mont:
1434 AARCH64_VALID_CALL_TARGET
1435 // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later.
1436 stp x29,x30,[sp,#-64]!
1437 add x29,sp,#0
1438 stp x19,x20,[sp,#16]
1439 stp x21,x22,[sp,#32]
1440 stp x23,x24,[sp,#48]
1441
1442 adrp x23,Lord@PAGE
1443 add x23,x23,Lord@PAGEOFF
1444 ldp x4,x5,[x1]
1445 ldp x6,x7,[x1,#16]
1446
1447 ldp x12,x13,[x23,#0]
1448 ldp x21,x22,[x23,#16]
1449 ldr x23,[x23,#32]
1450 b Loop_ord_sqr
1451
1452 .align 4
1453 Loop_ord_sqr:
1454 sub x2,x2,#1
1455 ////////////////////////////////////////////////////////////////
1456 // | | | | | |a1*a0| |
1457 // | | | | |a2*a0| | |
1458 // | |a3*a2|a3*a0| | | |
1459 // | | | |a2*a1| | | |
1460 // | | |a3*a1| | | | |
1461 // *| | | | | | | | 2|
1462 // +|a3*a3|a2*a2|a1*a1|a0*a0|
1463 // |--+--+--+--+--+--+--+--|
1464 // |A7|A6|A5|A4|A3|A2|A1|A0|, where Ax is , i.e. follow
1465 //
1466 // "can't overflow" below mark carrying into high part of
1467 // multiplication result, which can't overflow, because it
1468 // can never be all ones.
1469
1470 mul x15,x5,x4 // a[1]*a[0]
1471 umulh x9,x5,x4
1472 mul x16,x6,x4 // a[2]*a[0]
1473 umulh x10,x6,x4
1474 mul x17,x7,x4 // a[3]*a[0]
1475 umulh x19,x7,x4
1476
1477 adds x16,x16,x9 // accumulate high parts of multiplication
1478 mul x8,x6,x5 // a[2]*a[1]
1479 umulh x9,x6,x5
1480 adcs x17,x17,x10
1481 mul x10,x7,x5 // a[3]*a[1]
1482 umulh x11,x7,x5
1483 adc x19,x19,xzr // can't overflow
1484
1485 mul x20,x7,x6 // a[3]*a[2]
1486 umulh x1,x7,x6
1487
1488 adds x9,x9,x10 // accumulate high parts of multiplication
1489 mul x14,x4,x4 // a[0]*a[0]
1490 adc x10,x11,xzr // can't overflow
1491
1492 adds x17,x17,x8 // accumulate low parts of multiplication
1493 umulh x4,x4,x4
1494 adcs x19,x19,x9
1495 mul x9,x5,x5 // a[1]*a[1]
1496 adcs x20,x20,x10
1497 umulh x5,x5,x5
1498 adc x1,x1,xzr // can't overflow
1499
1500 adds x15,x15,x15 // acc[1-6]*=2
1501 mul x10,x6,x6 // a[2]*a[2]
1502 adcs x16,x16,x16
1503 umulh x6,x6,x6
1504 adcs x17,x17,x17
1505 mul x11,x7,x7 // a[3]*a[3]
1506 adcs x19,x19,x19
1507 umulh x7,x7,x7
1508 adcs x20,x20,x20
1509 adcs x1,x1,x1
1510 adc x3,xzr,xzr
1511
1512 adds x15,x15,x4 // +a[i]*a[i]
1513 mul x24,x14,x23
1514 adcs x16,x16,x9
1515 adcs x17,x17,x5
1516 adcs x19,x19,x10
1517 adcs x20,x20,x6
1518 adcs x1,x1,x11
1519 adc x3,x3,x7
1520 subs xzr,x14,#1
1521 umulh x9,x12,x24
1522 mul x10,x13,x24
1523 umulh x11,x13,x24
1524
1525 adcs x10,x10,x9
1526 adc x11,x11,xzr
1527
1528 adds x14,x15,x10
1529 adcs x15,x16,x11
1530 adcs x16,x17,x24
1531 adc x17,xzr,x24 // can't overflow
1532 mul x11,x14,x23
1533 lsl x8,x24,#32
1534 subs x15,x15,x24
1535 lsr x9,x24,#32
1536 sbcs x16,x16,x8
1537 sbc x17,x17,x9 // can't borrow
1538 subs xzr,x14,#1
1539 umulh x9,x12,x11
1540 mul x10,x13,x11
1541 umulh x24,x13,x11
1542
1543 adcs x10,x10,x9
1544 adc x24,x24,xzr
1545
1546 adds x14,x15,x10
1547 adcs x15,x16,x24
1548 adcs x16,x17,x11
1549 adc x17,xzr,x11 // can't overflow
1550 mul x24,x14,x23
1551 lsl x8,x11,#32
1552 subs x15,x15,x11
1553 lsr x9,x11,#32
1554 sbcs x16,x16,x8
1555 sbc x17,x17,x9 // can't borrow
1556 subs xzr,x14,#1
1557 umulh x9,x12,x24
1558 mul x10,x13,x24
1559 umulh x11,x13,x24
1560
1561 adcs x10,x10,x9
1562 adc x11,x11,xzr
1563
1564 adds x14,x15,x10
1565 adcs x15,x16,x11
1566 adcs x16,x17,x24
1567 adc x17,xzr,x24 // can't overflow
1568 mul x11,x14,x23
1569 lsl x8,x24,#32
1570 subs x15,x15,x24
1571 lsr x9,x24,#32
1572 sbcs x16,x16,x8
1573 sbc x17,x17,x9 // can't borrow
1574 subs xzr,x14,#1
1575 umulh x9,x12,x11
1576 mul x10,x13,x11
1577 umulh x24,x13,x11
1578
1579 adcs x10,x10,x9
1580 adc x24,x24,xzr
1581
1582 adds x14,x15,x10
1583 adcs x15,x16,x24
1584 adcs x16,x17,x11
1585 adc x17,xzr,x11 // can't overflow
1586 lsl x8,x11,#32
1587 subs x15,x15,x11
1588 lsr x9,x11,#32
1589 sbcs x16,x16,x8
1590 sbc x17,x17,x9 // can't borrow
1591 adds x14,x14,x19 // accumulate upper half
1592 adcs x15,x15,x20
1593 adcs x16,x16,x1
1594 adcs x17,x17,x3
1595 adc x19,xzr,xzr
1596
1597 subs x8,x14,x12 // ret -= modulus
1598 sbcs x9,x15,x13
1599 sbcs x10,x16,x21
1600 sbcs x11,x17,x22
1601 sbcs xzr,x19,xzr
1602
1603 csel x4,x14,x8,lo // ret = borrow ? ret : ret-modulus
1604 csel x5,x15,x9,lo
1605 csel x6,x16,x10,lo
1606 csel x7,x17,x11,lo
1607
1608 cbnz x2,Loop_ord_sqr
1609
1610 stp x4,x5,[x0]
1611 stp x6,x7,[x0,#16]
1612
1613 ldp x19,x20,[sp,#16]
1614 ldp x21,x22,[sp,#32]
1615 ldp x23,x24,[sp,#48]
1616 ldr x29,[sp],#64
1617 ret
1618
1619 ////////////////////////////////////////////////////////////////////////
1620 // void ecp_nistz256_select_w5(uint64_t *val, uint64_t *in_t, int index);
1621 .globl _ecp_nistz256_select_w5
1622 .private_extern _ecp_nistz256_select_w5
1623
1624 .align 4
1625 _ecp_nistz256_select_w5:
1626 AARCH64_VALID_CALL_TARGET
1627
1628 // x10 := x0
1629 // w9 := 0; loop counter and incremented internal index
1630 mov x10, x0
1631 mov w9, #0
1632
1633 // [v16-v21] := 0
1634 movi v16.16b, #0
1635 movi v17.16b, #0
1636 movi v18.16b, #0
1637 movi v19.16b, #0
1638 movi v20.16b, #0
1639 movi v21.16b, #0
1640
1641 Lselect_w5_loop:
1642 // Loop 16 times.
1643
1644 // Increment index (loop counter); tested at the end of the loop
1645 add w9, w9, #1
1646
1647 // [v22-v27] := Load a (3*256-bit = 6*128-bit) table entry starting at x1
1648 // and advance x1 to point to the next entry
1649 ld1 {v22.2d, v23.2d, v24.2d, v25.2d}, [x1],#64
1650
1651 // x11 := (w9 == w2)? All 1s : All 0s
1652 cmp w9, w2
1653 csetm x11, eq
1654
1655 // continue loading ...
1656 ld1 {v26.2d, v27.2d}, [x1],#32
1657
1658 // duplicate mask_64 into Mask (all 0s or all 1s)
1659 dup v3.2d, x11
1660
1661 // [v16-v19] := (Mask == all 1s)? [v22-v25] : [v16-v19]
1662 // i.e., values in output registers will remain the same if w9 != w2
1663 bit v16.16b, v22.16b, v3.16b
1664 bit v17.16b, v23.16b, v3.16b
1665
1666 bit v18.16b, v24.16b, v3.16b
1667 bit v19.16b, v25.16b, v3.16b
1668
1669 bit v20.16b, v26.16b, v3.16b
1670 bit v21.16b, v27.16b, v3.16b
1671
1672 // If bit #4 is not 0 (i.e. idx_ctr < 16) loop back
1673 tbz w9, #4, Lselect_w5_loop
1674
1675 // Write [v16-v21] to memory at the output pointer
1676 st1 {v16.2d, v17.2d, v18.2d, v19.2d}, [x10],#64
1677 st1 {v20.2d, v21.2d}, [x10]
1678
1679 ret
1680
1681
1682
1683 ////////////////////////////////////////////////////////////////////////
1684 // void ecp_nistz256_select_w7(uint64_t *val, uint64_t *in_t, int index);
1685 .globl _ecp_nistz256_select_w7
1686 .private_extern _ecp_nistz256_select_w7
1687
1688 .align 4
1689 _ecp_nistz256_select_w7:
1690 AARCH64_VALID_CALL_TARGET
1691
1692 // w9 := 0; loop counter and incremented internal index
1693 mov w9, #0
1694
1695 // [v16-v21] := 0
1696 movi v16.16b, #0
1697 movi v17.16b, #0
1698 movi v18.16b, #0
1699 movi v19.16b, #0
1700
1701 Lselect_w7_loop:
1702 // Loop 64 times.
1703
1704 // Increment index (loop counter); tested at the end of the loop
1705 add w9, w9, #1
1706
1707 // [v22-v25] := Load a (2*256-bit = 4*128-bit) table entry starting at x1
1708 // and advance x1 to point to the next entry
1709 ld1 {v22.2d, v23.2d, v24.2d, v25.2d}, [x1],#64
1710
1711 // x11 := (w9 == w2)? All 1s : All 0s
1712 cmp w9, w2
1713 csetm x11, eq
1714
1715 // duplicate mask_64 into Mask (all 0s or all 1s)
1716 dup v3.2d, x11
1717
1718 // [v16-v19] := (Mask == all 1s)? [v22-v25] : [v16-v19]
1719 // i.e., values in output registers will remain the same if w9 != w2
1720 bit v16.16b, v22.16b, v3.16b
1721 bit v17.16b, v23.16b, v3.16b
1722
1723 bit v18.16b, v24.16b, v3.16b
1724 bit v19.16b, v25.16b, v3.16b
1725
1726 // If bit #6 is not 0 (i.e. idx_ctr < 64) loop back
1727 tbz w9, #6, Lselect_w7_loop
1728
1729 // Write [v16-v19] to memory at the output pointer
1730 st1 {v16.2d, v17.2d, v18.2d, v19.2d}, [x0]
1731
1732 ret
1733
1734 #endif // !OPENSSL_NO_ASM && defined(__AARCH64EL__) && defined(__APPLE__)
1735 #if defined(__ELF__)
1736 // See https://www.airs.com/blog/archives/518.
1737 .section .note.GNU-stack,"",%progbits
1738 #endif