Debian Codegraph @

apple-aarch64/crypto/fipsmodule/aesv8-gcm-armv8-apple.S

raw
1 // This file is generated from a similarly-named Perl script in the BoringSSL
2 // source tree. Do not edit by hand.
3
4 #if !defined(__has_feature)
5 #define __has_feature(x) 0
6 #endif
7 #if __has_feature(memory_sanitizer) && !defined(OPENSSL_NO_ASM)
8 #define OPENSSL_NO_ASM
9 #endif
10
11 #if !defined(OPENSSL_NO_ASM) && defined(__AARCH64EL__) && defined(__APPLE__)
12 #if defined(BORINGSSL_PREFIX)
13 #include <boringssl_prefix_symbols_asm.h>
14 #endif
15 #include <openssl/arm_arch.h>
16 #if __ARM_MAX_ARCH__ >= 8
17
18
19 .text
20 .globl _aes_gcm_enc_kernel
21 .private_extern _aes_gcm_enc_kernel
22
23 .align 4
24 _aes_gcm_enc_kernel:
25 AARCH64_SIGN_LINK_REGISTER
26 stp x29, x30, [sp, #-128]!
27 mov x29, sp
28 stp x19, x20, [sp, #16]
29 mov x16, x4
30 mov x8, x5
31 stp x21, x22, [sp, #32]
32 stp x23, x24, [sp, #48]
33 stp d8, d9, [sp, #64]
34 stp d10, d11, [sp, #80]
35 stp d12, d13, [sp, #96]
36 stp d14, d15, [sp, #112]
37 ldr w17, [x8, #240]
38 add x19, x8, x17, lsl #4 // borrow input_l1 for last key
39 ldp x13, x14, [x19] // load round N keys
40 ldr q31, [x19, #-16] // load round N-1 keys
41 add x4, x0, x1, lsr #3 // end_input_ptr
42 lsr x5, x1, #3 // byte_len
43 mov x15, x5
44 ldp x10, x11, [x16] // ctr96_b64, ctr96_t32
45 ld1 { v0.16b}, [x16] // special case vector load initial counter so we can start first AES block as quickly as possible
46 sub x5, x5, #1 // byte_len - 1
47 ldr q18, [x8, #0] // load rk0
48 and x5, x5, #0xffffffffffffffc0 // number of bytes to be processed in main loop (at least 1 byte must be handled by tail)
49 ldr q25, [x8, #112] // load rk7
50 add x5, x5, x0
51 lsr x12, x11, #32
52 fmov d2, x10 // CTR block 2
53 orr w11, w11, w11
54 rev w12, w12 // rev_ctr32
55 fmov d1, x10 // CTR block 1
56 aese v0.16b, v18.16b
57 aesmc v0.16b, v0.16b // AES block 0 - round 0
58 add w12, w12, #1 // increment rev_ctr32
59 rev w9, w12 // CTR block 1
60 fmov d3, x10 // CTR block 3
61 orr x9, x11, x9, lsl #32 // CTR block 1
62 add w12, w12, #1 // CTR block 1
63 ldr q19, [x8, #16] // load rk1
64 fmov v1.d[1], x9 // CTR block 1
65 rev w9, w12 // CTR block 2
66 add w12, w12, #1 // CTR block 2
67 orr x9, x11, x9, lsl #32 // CTR block 2
68 ldr q20, [x8, #32] // load rk2
69 fmov v2.d[1], x9 // CTR block 2
70 rev w9, w12 // CTR block 3
71 aese v0.16b, v19.16b
72 aesmc v0.16b, v0.16b // AES block 0 - round 1
73 orr x9, x11, x9, lsl #32 // CTR block 3
74 fmov v3.d[1], x9 // CTR block 3
75 aese v1.16b, v18.16b
76 aesmc v1.16b, v1.16b // AES block 1 - round 0
77 ldr q21, [x8, #48] // load rk3
78 aese v0.16b, v20.16b
79 aesmc v0.16b, v0.16b // AES block 0 - round 2
80 ldr q24, [x8, #96] // load rk6
81 aese v2.16b, v18.16b
82 aesmc v2.16b, v2.16b // AES block 2 - round 0
83 ldr q23, [x8, #80] // load rk5
84 aese v1.16b, v19.16b
85 aesmc v1.16b, v1.16b // AES block 1 - round 1
86 ldr q14, [x3, #80] // load h3l | h3h
87 ext v14.16b, v14.16b, v14.16b, #8
88 aese v3.16b, v18.16b
89 aesmc v3.16b, v3.16b // AES block 3 - round 0
90 aese v2.16b, v19.16b
91 aesmc v2.16b, v2.16b // AES block 2 - round 1
92 ldr q22, [x8, #64] // load rk4
93 aese v1.16b, v20.16b
94 aesmc v1.16b, v1.16b // AES block 1 - round 2
95 ldr q13, [x3, #64] // load h2l | h2h
96 ext v13.16b, v13.16b, v13.16b, #8
97 aese v3.16b, v19.16b
98 aesmc v3.16b, v3.16b // AES block 3 - round 1
99 ldr q30, [x8, #192] // load rk12
100 aese v2.16b, v20.16b
101 aesmc v2.16b, v2.16b // AES block 2 - round 2
102 ldr q15, [x3, #112] // load h4l | h4h
103 ext v15.16b, v15.16b, v15.16b, #8
104 aese v1.16b, v21.16b
105 aesmc v1.16b, v1.16b // AES block 1 - round 3
106 ldr q29, [x8, #176] // load rk11
107 aese v3.16b, v20.16b
108 aesmc v3.16b, v3.16b // AES block 3 - round 2
109 ldr q26, [x8, #128] // load rk8
110 aese v2.16b, v21.16b
111 aesmc v2.16b, v2.16b // AES block 2 - round 3
112 add w12, w12, #1 // CTR block 3
113 aese v0.16b, v21.16b
114 aesmc v0.16b, v0.16b // AES block 0 - round 3
115 aese v3.16b, v21.16b
116 aesmc v3.16b, v3.16b // AES block 3 - round 3
117 ld1 { v11.16b}, [x3]
118 ext v11.16b, v11.16b, v11.16b, #8
119 rev64 v11.16b, v11.16b
120 aese v2.16b, v22.16b
121 aesmc v2.16b, v2.16b // AES block 2 - round 4
122 aese v0.16b, v22.16b
123 aesmc v0.16b, v0.16b // AES block 0 - round 4
124 aese v1.16b, v22.16b
125 aesmc v1.16b, v1.16b // AES block 1 - round 4
126 aese v3.16b, v22.16b
127 aesmc v3.16b, v3.16b // AES block 3 - round 4
128 cmp x17, #12 // setup flags for AES-128/192/256 check
129 aese v0.16b, v23.16b
130 aesmc v0.16b, v0.16b // AES block 0 - round 5
131 aese v1.16b, v23.16b
132 aesmc v1.16b, v1.16b // AES block 1 - round 5
133 aese v3.16b, v23.16b
134 aesmc v3.16b, v3.16b // AES block 3 - round 5
135 aese v2.16b, v23.16b
136 aesmc v2.16b, v2.16b // AES block 2 - round 5
137 aese v1.16b, v24.16b
138 aesmc v1.16b, v1.16b // AES block 1 - round 6
139 trn2 v17.2d, v14.2d, v15.2d // h4l | h3l
140 aese v3.16b, v24.16b
141 aesmc v3.16b, v3.16b // AES block 3 - round 6
142 ldr q27, [x8, #144] // load rk9
143 aese v0.16b, v24.16b
144 aesmc v0.16b, v0.16b // AES block 0 - round 6
145 ldr q12, [x3, #32] // load h1l | h1h
146 ext v12.16b, v12.16b, v12.16b, #8
147 aese v2.16b, v24.16b
148 aesmc v2.16b, v2.16b // AES block 2 - round 6
149 ldr q28, [x8, #160] // load rk10
150 aese v1.16b, v25.16b
151 aesmc v1.16b, v1.16b // AES block 1 - round 7
152 trn1 v9.2d, v14.2d, v15.2d // h4h | h3h
153 aese v0.16b, v25.16b
154 aesmc v0.16b, v0.16b // AES block 0 - round 7
155 aese v2.16b, v25.16b
156 aesmc v2.16b, v2.16b // AES block 2 - round 7
157 aese v3.16b, v25.16b
158 aesmc v3.16b, v3.16b // AES block 3 - round 7
159 trn2 v16.2d, v12.2d, v13.2d // h2l | h1l
160 aese v1.16b, v26.16b
161 aesmc v1.16b, v1.16b // AES block 1 - round 8
162 aese v2.16b, v26.16b
163 aesmc v2.16b, v2.16b // AES block 2 - round 8
164 aese v3.16b, v26.16b
165 aesmc v3.16b, v3.16b // AES block 3 - round 8
166 aese v0.16b, v26.16b
167 aesmc v0.16b, v0.16b // AES block 0 - round 8
168 b.lt Lenc_finish_first_blocks // branch if AES-128
169
170 aese v1.16b, v27.16b
171 aesmc v1.16b, v1.16b // AES block 1 - round 9
172 aese v2.16b, v27.16b
173 aesmc v2.16b, v2.16b // AES block 2 - round 9
174 aese v3.16b, v27.16b
175 aesmc v3.16b, v3.16b // AES block 3 - round 9
176 aese v0.16b, v27.16b
177 aesmc v0.16b, v0.16b // AES block 0 - round 9
178 aese v1.16b, v28.16b
179 aesmc v1.16b, v1.16b // AES block 1 - round 10
180 aese v2.16b, v28.16b
181 aesmc v2.16b, v2.16b // AES block 2 - round 10
182 aese v3.16b, v28.16b
183 aesmc v3.16b, v3.16b // AES block 3 - round 10
184 aese v0.16b, v28.16b
185 aesmc v0.16b, v0.16b // AES block 0 - round 10
186 b.eq Lenc_finish_first_blocks // branch if AES-192
187
188 aese v1.16b, v29.16b
189 aesmc v1.16b, v1.16b // AES block 1 - round 11
190 aese v2.16b, v29.16b
191 aesmc v2.16b, v2.16b // AES block 2 - round 11
192 aese v0.16b, v29.16b
193 aesmc v0.16b, v0.16b // AES block 0 - round 11
194 aese v3.16b, v29.16b
195 aesmc v3.16b, v3.16b // AES block 3 - round 11
196 aese v1.16b, v30.16b
197 aesmc v1.16b, v1.16b // AES block 1 - round 12
198 aese v2.16b, v30.16b
199 aesmc v2.16b, v2.16b // AES block 2 - round 12
200 aese v0.16b, v30.16b
201 aesmc v0.16b, v0.16b // AES block 0 - round 12
202 aese v3.16b, v30.16b
203 aesmc v3.16b, v3.16b // AES block 3 - round 12
204
205 Lenc_finish_first_blocks:
206 cmp x0, x5 // check if we have <= 4 blocks
207 eor v17.16b, v17.16b, v9.16b // h4k | h3k
208 aese v2.16b, v31.16b // AES block 2 - round N-1
209 trn1 v8.2d, v12.2d, v13.2d // h2h | h1h
210 aese v1.16b, v31.16b // AES block 1 - round N-1
211 aese v0.16b, v31.16b // AES block 0 - round N-1
212 aese v3.16b, v31.16b // AES block 3 - round N-1
213 eor v16.16b, v16.16b, v8.16b // h2k | h1k
214 b.ge Lenc_tail // handle tail
215
216 ldp x19, x20, [x0, #16] // AES block 1 - load plaintext
217 rev w9, w12 // CTR block 4
218 ldp x6, x7, [x0, #0] // AES block 0 - load plaintext
219 ldp x23, x24, [x0, #48] // AES block 3 - load plaintext
220 ldp x21, x22, [x0, #32] // AES block 2 - load plaintext
221 add x0, x0, #64 // AES input_ptr update
222 eor x19, x19, x13 // AES block 1 - round N low
223 eor x20, x20, x14 // AES block 1 - round N high
224 fmov d5, x19 // AES block 1 - mov low
225 eor x6, x6, x13 // AES block 0 - round N low
226 eor x7, x7, x14 // AES block 0 - round N high
227 eor x24, x24, x14 // AES block 3 - round N high
228 fmov d4, x6 // AES block 0 - mov low
229 cmp x0, x5 // check if we have <= 8 blocks
230 fmov v4.d[1], x7 // AES block 0 - mov high
231 eor x23, x23, x13 // AES block 3 - round N low
232 eor x21, x21, x13 // AES block 2 - round N low
233 fmov v5.d[1], x20 // AES block 1 - mov high
234 fmov d6, x21 // AES block 2 - mov low
235 add w12, w12, #1 // CTR block 4
236 orr x9, x11, x9, lsl #32 // CTR block 4
237 fmov d7, x23 // AES block 3 - mov low
238 eor x22, x22, x14 // AES block 2 - round N high
239 fmov v6.d[1], x22 // AES block 2 - mov high
240 eor v4.16b, v4.16b, v0.16b // AES block 0 - result
241 fmov d0, x10 // CTR block 4
242 fmov v0.d[1], x9 // CTR block 4
243 rev w9, w12 // CTR block 5
244 add w12, w12, #1 // CTR block 5
245 eor v5.16b, v5.16b, v1.16b // AES block 1 - result
246 fmov d1, x10 // CTR block 5
247 orr x9, x11, x9, lsl #32 // CTR block 5
248 fmov v1.d[1], x9 // CTR block 5
249 rev w9, w12 // CTR block 6
250 st1 { v4.16b}, [x2], #16 // AES block 0 - store result
251 fmov v7.d[1], x24 // AES block 3 - mov high
252 orr x9, x11, x9, lsl #32 // CTR block 6
253 eor v6.16b, v6.16b, v2.16b // AES block 2 - result
254 st1 { v5.16b}, [x2], #16 // AES block 1 - store result
255 add w12, w12, #1 // CTR block 6
256 fmov d2, x10 // CTR block 6
257 fmov v2.d[1], x9 // CTR block 6
258 st1 { v6.16b}, [x2], #16 // AES block 2 - store result
259 rev w9, w12 // CTR block 7
260 orr x9, x11, x9, lsl #32 // CTR block 7
261 eor v7.16b, v7.16b, v3.16b // AES block 3 - result
262 st1 { v7.16b}, [x2], #16 // AES block 3 - store result
263 b.ge Lenc_prepretail // do prepretail
264
265 Lenc_main_loop: // main loop start
266 aese v0.16b, v18.16b
267 aesmc v0.16b, v0.16b // AES block 4k+4 - round 0
268 rev64 v4.16b, v4.16b // GHASH block 4k (only t0 is free)
269 aese v1.16b, v18.16b
270 aesmc v1.16b, v1.16b // AES block 4k+5 - round 0
271 fmov d3, x10 // CTR block 4k+3
272 aese v2.16b, v18.16b
273 aesmc v2.16b, v2.16b // AES block 4k+6 - round 0
274 ext v11.16b, v11.16b, v11.16b, #8 // PRE 0
275 aese v0.16b, v19.16b
276 aesmc v0.16b, v0.16b // AES block 4k+4 - round 1
277 fmov v3.d[1], x9 // CTR block 4k+3
278 aese v1.16b, v19.16b
279 aesmc v1.16b, v1.16b // AES block 4k+5 - round 1
280 ldp x23, x24, [x0, #48] // AES block 4k+7 - load plaintext
281 aese v2.16b, v19.16b
282 aesmc v2.16b, v2.16b // AES block 4k+6 - round 1
283 ldp x21, x22, [x0, #32] // AES block 4k+6 - load plaintext
284 aese v0.16b, v20.16b
285 aesmc v0.16b, v0.16b // AES block 4k+4 - round 2
286 eor v4.16b, v4.16b, v11.16b // PRE 1
287 aese v1.16b, v20.16b
288 aesmc v1.16b, v1.16b // AES block 4k+5 - round 2
289 aese v3.16b, v18.16b
290 aesmc v3.16b, v3.16b // AES block 4k+7 - round 0
291 eor x23, x23, x13 // AES block 4k+7 - round N low
292 aese v0.16b, v21.16b
293 aesmc v0.16b, v0.16b // AES block 4k+4 - round 3
294 mov d10, v17.d[1] // GHASH block 4k - mid
295 pmull2 v9.1q, v4.2d, v15.2d // GHASH block 4k - high
296 eor x22, x22, x14 // AES block 4k+6 - round N high
297 mov d8, v4.d[1] // GHASH block 4k - mid
298 aese v3.16b, v19.16b
299 aesmc v3.16b, v3.16b // AES block 4k+7 - round 1
300 rev64 v5.16b, v5.16b // GHASH block 4k+1 (t0 and t1 free)
301 aese v0.16b, v22.16b
302 aesmc v0.16b, v0.16b // AES block 4k+4 - round 4
303 pmull v11.1q, v4.1d, v15.1d // GHASH block 4k - low
304 eor v8.8b, v8.8b, v4.8b // GHASH block 4k - mid
305 aese v2.16b, v20.16b
306 aesmc v2.16b, v2.16b // AES block 4k+6 - round 2
307 aese v0.16b, v23.16b
308 aesmc v0.16b, v0.16b // AES block 4k+4 - round 5
309 rev64 v7.16b, v7.16b // GHASH block 4k+3 (t0, t1, t2 and t3 free)
310 pmull2 v4.1q, v5.2d, v14.2d // GHASH block 4k+1 - high
311 pmull v10.1q, v8.1d, v10.1d // GHASH block 4k - mid
312 rev64 v6.16b, v6.16b // GHASH block 4k+2 (t0, t1, and t2 free)
313 pmull v8.1q, v5.1d, v14.1d // GHASH block 4k+1 - low
314 eor v9.16b, v9.16b, v4.16b // GHASH block 4k+1 - high
315 mov d4, v5.d[1] // GHASH block 4k+1 - mid
316 aese v1.16b, v21.16b
317 aesmc v1.16b, v1.16b // AES block 4k+5 - round 3
318 aese v3.16b, v20.16b
319 aesmc v3.16b, v3.16b // AES block 4k+7 - round 2
320 eor v11.16b, v11.16b, v8.16b // GHASH block 4k+1 - low
321 aese v2.16b, v21.16b
322 aesmc v2.16b, v2.16b // AES block 4k+6 - round 3
323 aese v1.16b, v22.16b
324 aesmc v1.16b, v1.16b // AES block 4k+5 - round 4
325 mov d8, v6.d[1] // GHASH block 4k+2 - mid
326 aese v3.16b, v21.16b
327 aesmc v3.16b, v3.16b // AES block 4k+7 - round 3
328 eor v4.8b, v4.8b, v5.8b // GHASH block 4k+1 - mid
329 aese v2.16b, v22.16b
330 aesmc v2.16b, v2.16b // AES block 4k+6 - round 4
331 aese v0.16b, v24.16b
332 aesmc v0.16b, v0.16b // AES block 4k+4 - round 6
333 eor v8.8b, v8.8b, v6.8b // GHASH block 4k+2 - mid
334 aese v3.16b, v22.16b
335 aesmc v3.16b, v3.16b // AES block 4k+7 - round 4
336 pmull v4.1q, v4.1d, v17.1d // GHASH block 4k+1 - mid
337 aese v0.16b, v25.16b
338 aesmc v0.16b, v0.16b // AES block 4k+4 - round 7
339 aese v3.16b, v23.16b
340 aesmc v3.16b, v3.16b // AES block 4k+7 - round 5
341 ins v8.d[1], v8.d[0] // GHASH block 4k+2 - mid
342 aese v1.16b, v23.16b
343 aesmc v1.16b, v1.16b // AES block 4k+5 - round 5
344 aese v0.16b, v26.16b
345 aesmc v0.16b, v0.16b // AES block 4k+4 - round 8
346 aese v2.16b, v23.16b
347 aesmc v2.16b, v2.16b // AES block 4k+6 - round 5
348 aese v1.16b, v24.16b
349 aesmc v1.16b, v1.16b // AES block 4k+5 - round 6
350 eor v10.16b, v10.16b, v4.16b // GHASH block 4k+1 - mid
351 pmull2 v4.1q, v6.2d, v13.2d // GHASH block 4k+2 - high
352 pmull v5.1q, v6.1d, v13.1d // GHASH block 4k+2 - low
353 aese v1.16b, v25.16b
354 aesmc v1.16b, v1.16b // AES block 4k+5 - round 7
355 pmull v6.1q, v7.1d, v12.1d // GHASH block 4k+3 - low
356 eor v9.16b, v9.16b, v4.16b // GHASH block 4k+2 - high
357 aese v3.16b, v24.16b
358 aesmc v3.16b, v3.16b // AES block 4k+7 - round 6
359 ldp x19, x20, [x0, #16] // AES block 4k+5 - load plaintext
360 aese v1.16b, v26.16b
361 aesmc v1.16b, v1.16b // AES block 4k+5 - round 8
362 mov d4, v7.d[1] // GHASH block 4k+3 - mid
363 aese v2.16b, v24.16b
364 aesmc v2.16b, v2.16b // AES block 4k+6 - round 6
365 eor v11.16b, v11.16b, v5.16b // GHASH block 4k+2 - low
366 pmull2 v8.1q, v8.2d, v16.2d // GHASH block 4k+2 - mid
367 pmull2 v5.1q, v7.2d, v12.2d // GHASH block 4k+3 - high
368 eor v4.8b, v4.8b, v7.8b // GHASH block 4k+3 - mid
369 aese v2.16b, v25.16b
370 aesmc v2.16b, v2.16b // AES block 4k+6 - round 7
371 eor x19, x19, x13 // AES block 4k+5 - round N low
372 aese v2.16b, v26.16b
373 aesmc v2.16b, v2.16b // AES block 4k+6 - round 8
374 eor v10.16b, v10.16b, v8.16b // GHASH block 4k+2 - mid
375 aese v3.16b, v25.16b
376 aesmc v3.16b, v3.16b // AES block 4k+7 - round 7
377 eor x21, x21, x13 // AES block 4k+6 - round N low
378 aese v3.16b, v26.16b
379 aesmc v3.16b, v3.16b // AES block 4k+7 - round 8
380 movi v8.8b, #0xc2
381 pmull v4.1q, v4.1d, v16.1d // GHASH block 4k+3 - mid
382 eor v9.16b, v9.16b, v5.16b // GHASH block 4k+3 - high
383 cmp x17, #12 // setup flags for AES-128/192/256 check
384 fmov d5, x19 // AES block 4k+5 - mov low
385 ldp x6, x7, [x0, #0] // AES block 4k+4 - load plaintext
386 b.lt Lenc_main_loop_continue // branch if AES-128
387
388 aese v1.16b, v27.16b
389 aesmc v1.16b, v1.16b // AES block 4k+5 - round 9
390 aese v0.16b, v27.16b
391 aesmc v0.16b, v0.16b // AES block 4k+4 - round 9
392 aese v2.16b, v27.16b
393 aesmc v2.16b, v2.16b // AES block 4k+6 - round 9
394 aese v3.16b, v27.16b
395 aesmc v3.16b, v3.16b // AES block 4k+7 - round 9
396 aese v0.16b, v28.16b
397 aesmc v0.16b, v0.16b // AES block 4k+4 - round 10
398 aese v1.16b, v28.16b
399 aesmc v1.16b, v1.16b // AES block 4k+5 - round 10
400 aese v2.16b, v28.16b
401 aesmc v2.16b, v2.16b // AES block 4k+6 - round 10
402 aese v3.16b, v28.16b
403 aesmc v3.16b, v3.16b // AES block 4k+7 - round 10
404 b.eq Lenc_main_loop_continue // branch if AES-192
405
406 aese v0.16b, v29.16b
407 aesmc v0.16b, v0.16b // AES block 4k+4 - round 11
408 aese v1.16b, v29.16b
409 aesmc v1.16b, v1.16b // AES block 4k+5 - round 11
410 aese v2.16b, v29.16b
411 aesmc v2.16b, v2.16b // AES block 4k+6 - round 11
412 aese v3.16b, v29.16b
413 aesmc v3.16b, v3.16b // AES block 4k+7 - round 11
414 aese v1.16b, v30.16b
415 aesmc v1.16b, v1.16b // AES block 4k+5 - round 12
416 aese v0.16b, v30.16b
417 aesmc v0.16b, v0.16b // AES block 4k+4 - round 12
418 aese v2.16b, v30.16b
419 aesmc v2.16b, v2.16b // AES block 4k+6 - round 12
420 aese v3.16b, v30.16b
421 aesmc v3.16b, v3.16b // AES block 4k+7 - round 12
422
423 Lenc_main_loop_continue:
424 shl d8, d8, #56 // mod_constant
425 eor v11.16b, v11.16b, v6.16b // GHASH block 4k+3 - low
426 eor v10.16b, v10.16b, v4.16b // GHASH block 4k+3 - mid
427 add w12, w12, #1 // CTR block 4k+3
428 eor v4.16b, v11.16b, v9.16b // MODULO - karatsuba tidy up
429 add x0, x0, #64 // AES input_ptr update
430 pmull v7.1q, v9.1d, v8.1d // MODULO - top 64b align with mid
431 rev w9, w12 // CTR block 4k+8
432 ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment
433 eor x6, x6, x13 // AES block 4k+4 - round N low
434 eor v10.16b, v10.16b, v4.16b // MODULO - karatsuba tidy up
435 eor x7, x7, x14 // AES block 4k+4 - round N high
436 fmov d4, x6 // AES block 4k+4 - mov low
437 orr x9, x11, x9, lsl #32 // CTR block 4k+8
438 eor v7.16b, v9.16b, v7.16b // MODULO - fold into mid
439 eor x20, x20, x14 // AES block 4k+5 - round N high
440 eor x24, x24, x14 // AES block 4k+7 - round N high
441 add w12, w12, #1 // CTR block 4k+8
442 aese v0.16b, v31.16b // AES block 4k+4 - round N-1
443 fmov v4.d[1], x7 // AES block 4k+4 - mov high
444 eor v10.16b, v10.16b, v7.16b // MODULO - fold into mid
445 fmov d7, x23 // AES block 4k+7 - mov low
446 aese v1.16b, v31.16b // AES block 4k+5 - round N-1
447 fmov v5.d[1], x20 // AES block 4k+5 - mov high
448 fmov d6, x21 // AES block 4k+6 - mov low
449 cmp x0, x5 // LOOP CONTROL
450 fmov v6.d[1], x22 // AES block 4k+6 - mov high
451 pmull v9.1q, v10.1d, v8.1d // MODULO - mid 64b align with low
452 eor v4.16b, v4.16b, v0.16b // AES block 4k+4 - result
453 fmov d0, x10 // CTR block 4k+8
454 fmov v0.d[1], x9 // CTR block 4k+8
455 rev w9, w12 // CTR block 4k+9
456 add w12, w12, #1 // CTR block 4k+9
457 eor v5.16b, v5.16b, v1.16b // AES block 4k+5 - result
458 fmov d1, x10 // CTR block 4k+9
459 orr x9, x11, x9, lsl #32 // CTR block 4k+9
460 fmov v1.d[1], x9 // CTR block 4k+9
461 aese v2.16b, v31.16b // AES block 4k+6 - round N-1
462 rev w9, w12 // CTR block 4k+10
463 st1 { v4.16b}, [x2], #16 // AES block 4k+4 - store result
464 orr x9, x11, x9, lsl #32 // CTR block 4k+10
465 eor v11.16b, v11.16b, v9.16b // MODULO - fold into low
466 fmov v7.d[1], x24 // AES block 4k+7 - mov high
467 ext v10.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment
468 st1 { v5.16b}, [x2], #16 // AES block 4k+5 - store result
469 add w12, w12, #1 // CTR block 4k+10
470 aese v3.16b, v31.16b // AES block 4k+7 - round N-1
471 eor v6.16b, v6.16b, v2.16b // AES block 4k+6 - result
472 fmov d2, x10 // CTR block 4k+10
473 st1 { v6.16b}, [x2], #16 // AES block 4k+6 - store result
474 fmov v2.d[1], x9 // CTR block 4k+10
475 rev w9, w12 // CTR block 4k+11
476 eor v11.16b, v11.16b, v10.16b // MODULO - fold into low
477 orr x9, x11, x9, lsl #32 // CTR block 4k+11
478 eor v7.16b, v7.16b, v3.16b // AES block 4k+7 - result
479 st1 { v7.16b}, [x2], #16 // AES block 4k+7 - store result
480 b.lt Lenc_main_loop
481
482 Lenc_prepretail: // PREPRETAIL
483 aese v1.16b, v18.16b
484 aesmc v1.16b, v1.16b // AES block 4k+5 - round 0
485 rev64 v6.16b, v6.16b // GHASH block 4k+2 (t0, t1, and t2 free)
486 aese v2.16b, v18.16b
487 aesmc v2.16b, v2.16b // AES block 4k+6 - round 0
488 fmov d3, x10 // CTR block 4k+3
489 aese v0.16b, v18.16b
490 aesmc v0.16b, v0.16b // AES block 4k+4 - round 0
491 rev64 v4.16b, v4.16b // GHASH block 4k (only t0 is free)
492 fmov v3.d[1], x9 // CTR block 4k+3
493 ext v11.16b, v11.16b, v11.16b, #8 // PRE 0
494 aese v2.16b, v19.16b
495 aesmc v2.16b, v2.16b // AES block 4k+6 - round 1
496 aese v0.16b, v19.16b
497 aesmc v0.16b, v0.16b // AES block 4k+4 - round 1
498 eor v4.16b, v4.16b, v11.16b // PRE 1
499 rev64 v5.16b, v5.16b // GHASH block 4k+1 (t0 and t1 free)
500 aese v2.16b, v20.16b
501 aesmc v2.16b, v2.16b // AES block 4k+6 - round 2
502 aese v3.16b, v18.16b
503 aesmc v3.16b, v3.16b // AES block 4k+7 - round 0
504 mov d10, v17.d[1] // GHASH block 4k - mid
505 aese v1.16b, v19.16b
506 aesmc v1.16b, v1.16b // AES block 4k+5 - round 1
507 pmull v11.1q, v4.1d, v15.1d // GHASH block 4k - low
508 mov d8, v4.d[1] // GHASH block 4k - mid
509 pmull2 v9.1q, v4.2d, v15.2d // GHASH block 4k - high
510 aese v2.16b, v21.16b
511 aesmc v2.16b, v2.16b // AES block 4k+6 - round 3
512 aese v1.16b, v20.16b
513 aesmc v1.16b, v1.16b // AES block 4k+5 - round 2
514 eor v8.8b, v8.8b, v4.8b // GHASH block 4k - mid
515 aese v0.16b, v20.16b
516 aesmc v0.16b, v0.16b // AES block 4k+4 - round 2
517 aese v3.16b, v19.16b
518 aesmc v3.16b, v3.16b // AES block 4k+7 - round 1
519 aese v1.16b, v21.16b
520 aesmc v1.16b, v1.16b // AES block 4k+5 - round 3
521 pmull v10.1q, v8.1d, v10.1d // GHASH block 4k - mid
522 pmull2 v4.1q, v5.2d, v14.2d // GHASH block 4k+1 - high
523 pmull v8.1q, v5.1d, v14.1d // GHASH block 4k+1 - low
524 aese v3.16b, v20.16b
525 aesmc v3.16b, v3.16b // AES block 4k+7 - round 2
526 eor v9.16b, v9.16b, v4.16b // GHASH block 4k+1 - high
527 mov d4, v5.d[1] // GHASH block 4k+1 - mid
528 aese v0.16b, v21.16b
529 aesmc v0.16b, v0.16b // AES block 4k+4 - round 3
530 eor v11.16b, v11.16b, v8.16b // GHASH block 4k+1 - low
531 aese v3.16b, v21.16b
532 aesmc v3.16b, v3.16b // AES block 4k+7 - round 3
533 eor v4.8b, v4.8b, v5.8b // GHASH block 4k+1 - mid
534 mov d8, v6.d[1] // GHASH block 4k+2 - mid
535 aese v0.16b, v22.16b
536 aesmc v0.16b, v0.16b // AES block 4k+4 - round 4
537 rev64 v7.16b, v7.16b // GHASH block 4k+3 (t0, t1, t2 and t3 free)
538 aese v3.16b, v22.16b
539 aesmc v3.16b, v3.16b // AES block 4k+7 - round 4
540 pmull v4.1q, v4.1d, v17.1d // GHASH block 4k+1 - mid
541 eor v8.8b, v8.8b, v6.8b // GHASH block 4k+2 - mid
542 add w12, w12, #1 // CTR block 4k+3
543 pmull v5.1q, v6.1d, v13.1d // GHASH block 4k+2 - low
544 aese v3.16b, v23.16b
545 aesmc v3.16b, v3.16b // AES block 4k+7 - round 5
546 aese v2.16b, v22.16b
547 aesmc v2.16b, v2.16b // AES block 4k+6 - round 4
548 eor v10.16b, v10.16b, v4.16b // GHASH block 4k+1 - mid
549 pmull2 v4.1q, v6.2d, v13.2d // GHASH block 4k+2 - high
550 eor v11.16b, v11.16b, v5.16b // GHASH block 4k+2 - low
551 ins v8.d[1], v8.d[0] // GHASH block 4k+2 - mid
552 aese v2.16b, v23.16b
553 aesmc v2.16b, v2.16b // AES block 4k+6 - round 5
554 eor v9.16b, v9.16b, v4.16b // GHASH block 4k+2 - high
555 mov d4, v7.d[1] // GHASH block 4k+3 - mid
556 aese v1.16b, v22.16b
557 aesmc v1.16b, v1.16b // AES block 4k+5 - round 4
558 pmull2 v8.1q, v8.2d, v16.2d // GHASH block 4k+2 - mid
559 eor v4.8b, v4.8b, v7.8b // GHASH block 4k+3 - mid
560 pmull2 v5.1q, v7.2d, v12.2d // GHASH block 4k+3 - high
561 aese v1.16b, v23.16b
562 aesmc v1.16b, v1.16b // AES block 4k+5 - round 5
563 pmull v4.1q, v4.1d, v16.1d // GHASH block 4k+3 - mid
564 eor v10.16b, v10.16b, v8.16b // GHASH block 4k+2 - mid
565 aese v0.16b, v23.16b
566 aesmc v0.16b, v0.16b // AES block 4k+4 - round 5
567 aese v1.16b, v24.16b
568 aesmc v1.16b, v1.16b // AES block 4k+5 - round 6
569 aese v2.16b, v24.16b
570 aesmc v2.16b, v2.16b // AES block 4k+6 - round 6
571 aese v0.16b, v24.16b
572 aesmc v0.16b, v0.16b // AES block 4k+4 - round 6
573 movi v8.8b, #0xc2
574 aese v3.16b, v24.16b
575 aesmc v3.16b, v3.16b // AES block 4k+7 - round 6
576 aese v1.16b, v25.16b
577 aesmc v1.16b, v1.16b // AES block 4k+5 - round 7
578 eor v9.16b, v9.16b, v5.16b // GHASH block 4k+3 - high
579 aese v0.16b, v25.16b
580 aesmc v0.16b, v0.16b // AES block 4k+4 - round 7
581 aese v3.16b, v25.16b
582 aesmc v3.16b, v3.16b // AES block 4k+7 - round 7
583 shl d8, d8, #56 // mod_constant
584 aese v1.16b, v26.16b
585 aesmc v1.16b, v1.16b // AES block 4k+5 - round 8
586 eor v10.16b, v10.16b, v4.16b // GHASH block 4k+3 - mid
587 pmull v6.1q, v7.1d, v12.1d // GHASH block 4k+3 - low
588 aese v3.16b, v26.16b
589 aesmc v3.16b, v3.16b // AES block 4k+7 - round 8
590 cmp x17, #12 // setup flags for AES-128/192/256 check
591 aese v0.16b, v26.16b
592 aesmc v0.16b, v0.16b // AES block 4k+4 - round 8
593 eor v11.16b, v11.16b, v6.16b // GHASH block 4k+3 - low
594 aese v2.16b, v25.16b
595 aesmc v2.16b, v2.16b // AES block 4k+6 - round 7
596 eor v10.16b, v10.16b, v9.16b // karatsuba tidy up
597 aese v2.16b, v26.16b
598 aesmc v2.16b, v2.16b // AES block 4k+6 - round 8
599 pmull v4.1q, v9.1d, v8.1d
600 ext v9.16b, v9.16b, v9.16b, #8
601 eor v10.16b, v10.16b, v11.16b
602 b.lt Lenc_finish_prepretail // branch if AES-128
603
604 aese v1.16b, v27.16b
605 aesmc v1.16b, v1.16b // AES block 4k+5 - round 9
606 aese v3.16b, v27.16b
607 aesmc v3.16b, v3.16b // AES block 4k+7 - round 9
608 aese v0.16b, v27.16b
609 aesmc v0.16b, v0.16b // AES block 4k+4 - round 9
610 aese v2.16b, v27.16b
611 aesmc v2.16b, v2.16b // AES block 4k+6 - round 9
612 aese v3.16b, v28.16b
613 aesmc v3.16b, v3.16b // AES block 4k+7 - round 10
614 aese v1.16b, v28.16b
615 aesmc v1.16b, v1.16b // AES block 4k+5 - round 10
616 aese v0.16b, v28.16b
617 aesmc v0.16b, v0.16b // AES block 4k+4 - round 10
618 aese v2.16b, v28.16b
619 aesmc v2.16b, v2.16b // AES block 4k+6 - round 10
620 b.eq Lenc_finish_prepretail // branch if AES-192
621
622 aese v1.16b, v29.16b
623 aesmc v1.16b, v1.16b // AES block 4k+5 - round 11
624 aese v0.16b, v29.16b
625 aesmc v0.16b, v0.16b // AES block 4k+4 - round 11
626 aese v3.16b, v29.16b
627 aesmc v3.16b, v3.16b // AES block 4k+7 - round 11
628 aese v2.16b, v29.16b
629 aesmc v2.16b, v2.16b // AES block 4k+6 - round 11
630 aese v1.16b, v30.16b
631 aesmc v1.16b, v1.16b // AES block 4k+5 - round 12
632 aese v0.16b, v30.16b
633 aesmc v0.16b, v0.16b // AES block 4k+4 - round 12
634 aese v3.16b, v30.16b
635 aesmc v3.16b, v3.16b // AES block 4k+7 - round 12
636 aese v2.16b, v30.16b
637 aesmc v2.16b, v2.16b // AES block 4k+6 - round 12
638
639 Lenc_finish_prepretail:
640 eor v10.16b, v10.16b, v4.16b
641 eor v10.16b, v10.16b, v9.16b
642 pmull v4.1q, v10.1d, v8.1d
643 ext v10.16b, v10.16b, v10.16b, #8
644 aese v1.16b, v31.16b // AES block 4k+5 - round N-1
645 eor v11.16b, v11.16b, v4.16b
646 aese v3.16b, v31.16b // AES block 4k+7 - round N-1
647 aese v0.16b, v31.16b // AES block 4k+4 - round N-1
648 aese v2.16b, v31.16b // AES block 4k+6 - round N-1
649 eor v11.16b, v11.16b, v10.16b
650
651 Lenc_tail: // TAIL
652 ext v8.16b, v11.16b, v11.16b, #8 // prepare final partial tag
653 sub x5, x4, x0 // main_end_input_ptr is number of bytes left to process
654 ldp x6, x7, [x0], #16 // AES block 4k+4 - load plaintext
655 eor x6, x6, x13 // AES block 4k+4 - round N low
656 eor x7, x7, x14 // AES block 4k+4 - round N high
657 cmp x5, #48
658 fmov d4, x6 // AES block 4k+4 - mov low
659 fmov v4.d[1], x7 // AES block 4k+4 - mov high
660 eor v5.16b, v4.16b, v0.16b // AES block 4k+4 - result
661 b.gt Lenc_blocks_more_than_3
662 cmp x5, #32
663 mov v3.16b, v2.16b
664 movi v11.8b, #0
665 movi v9.8b, #0
666 sub w12, w12, #1
667 mov v2.16b, v1.16b
668 movi v10.8b, #0
669 b.gt Lenc_blocks_more_than_2
670 mov v3.16b, v1.16b
671 sub w12, w12, #1
672 cmp x5, #16
673 b.gt Lenc_blocks_more_than_1
674 sub w12, w12, #1
675 b Lenc_blocks_less_than_1
676 Lenc_blocks_more_than_3: // blocks left > 3
677 st1 { v5.16b}, [x2], #16 // AES final-3 block - store result
678 ldp x6, x7, [x0], #16 // AES final-2 block - load input low & high
679 rev64 v4.16b, v5.16b // GHASH final-3 block
680 eor x6, x6, x13 // AES final-2 block - round N low
681 eor v4.16b, v4.16b, v8.16b // feed in partial tag
682 eor x7, x7, x14 // AES final-2 block - round N high
683 mov d22, v4.d[1] // GHASH final-3 block - mid
684 fmov d5, x6 // AES final-2 block - mov low
685 fmov v5.d[1], x7 // AES final-2 block - mov high
686 eor v22.8b, v22.8b, v4.8b // GHASH final-3 block - mid
687 movi v8.8b, #0 // suppress further partial tag feed in
688 mov d10, v17.d[1] // GHASH final-3 block - mid
689 pmull v11.1q, v4.1d, v15.1d // GHASH final-3 block - low
690 pmull2 v9.1q, v4.2d, v15.2d // GHASH final-3 block - high
691 pmull v10.1q, v22.1d, v10.1d // GHASH final-3 block - mid
692 eor v5.16b, v5.16b, v1.16b // AES final-2 block - result
693 Lenc_blocks_more_than_2: // blocks left > 2
694 st1 { v5.16b}, [x2], #16 // AES final-2 block - store result
695 ldp x6, x7, [x0], #16 // AES final-1 block - load input low & high
696 rev64 v4.16b, v5.16b // GHASH final-2 block
697 eor x6, x6, x13 // AES final-1 block - round N low
698 eor v4.16b, v4.16b, v8.16b // feed in partial tag
699 fmov d5, x6 // AES final-1 block - mov low
700 eor x7, x7, x14 // AES final-1 block - round N high
701 fmov v5.d[1], x7 // AES final-1 block - mov high
702 movi v8.8b, #0 // suppress further partial tag feed in
703 pmull2 v20.1q, v4.2d, v14.2d // GHASH final-2 block - high
704 mov d22, v4.d[1] // GHASH final-2 block - mid
705 pmull v21.1q, v4.1d, v14.1d // GHASH final-2 block - low
706 eor v22.8b, v22.8b, v4.8b // GHASH final-2 block - mid
707 eor v5.16b, v5.16b, v2.16b // AES final-1 block - result
708 eor v9.16b, v9.16b, v20.16b // GHASH final-2 block - high
709 pmull v22.1q, v22.1d, v17.1d // GHASH final-2 block - mid
710 eor v11.16b, v11.16b, v21.16b // GHASH final-2 block - low
711 eor v10.16b, v10.16b, v22.16b // GHASH final-2 block - mid
712 Lenc_blocks_more_than_1: // blocks left > 1
713 st1 { v5.16b}, [x2], #16 // AES final-1 block - store result
714 rev64 v4.16b, v5.16b // GHASH final-1 block
715 ldp x6, x7, [x0], #16 // AES final block - load input low & high
716 eor v4.16b, v4.16b, v8.16b // feed in partial tag
717 movi v8.8b, #0 // suppress further partial tag feed in
718 eor x6, x6, x13 // AES final block - round N low
719 mov d22, v4.d[1] // GHASH final-1 block - mid
720 pmull2 v20.1q, v4.2d, v13.2d // GHASH final-1 block - high
721 eor x7, x7, x14 // AES final block - round N high
722 eor v22.8b, v22.8b, v4.8b // GHASH final-1 block - mid
723 eor v9.16b, v9.16b, v20.16b // GHASH final-1 block - high
724 ins v22.d[1], v22.d[0] // GHASH final-1 block - mid
725 fmov d5, x6 // AES final block - mov low
726 fmov v5.d[1], x7 // AES final block - mov high
727 pmull2 v22.1q, v22.2d, v16.2d // GHASH final-1 block - mid
728 pmull v21.1q, v4.1d, v13.1d // GHASH final-1 block - low
729 eor v5.16b, v5.16b, v3.16b // AES final block - result
730 eor v10.16b, v10.16b, v22.16b // GHASH final-1 block - mid
731 eor v11.16b, v11.16b, v21.16b // GHASH final-1 block - low
732 Lenc_blocks_less_than_1: // blocks left <= 1
733 and x1, x1, #127 // bit_length %= 128
734 mvn x13, xzr // rkN_l = 0xffffffffffffffff
735 sub x1, x1, #128 // bit_length -= 128
736 neg x1, x1 // bit_length = 128 - #bits in input (in range [1,128])
737 ld1 { v18.16b}, [x2] // load existing bytes where the possibly partial last block is to be stored
738 mvn x14, xzr // rkN_h = 0xffffffffffffffff
739 and x1, x1, #127 // bit_length %= 128
740 lsr x14, x14, x1 // rkN_h is mask for top 64b of last block
741 cmp x1, #64
742 csel x6, x13, x14, lt
743 csel x7, x14, xzr, lt
744 fmov d0, x6 // ctr0b is mask for last block
745 fmov v0.d[1], x7
746 and v5.16b, v5.16b, v0.16b // possibly partial last block has zeroes in highest bits
747 rev64 v4.16b, v5.16b // GHASH final block
748 eor v4.16b, v4.16b, v8.16b // feed in partial tag
749 bif v5.16b, v18.16b, v0.16b // insert existing bytes in top end of result before storing
750 pmull2 v20.1q, v4.2d, v12.2d // GHASH final block - high
751 mov d8, v4.d[1] // GHASH final block - mid
752 rev w9, w12
753 pmull v21.1q, v4.1d, v12.1d // GHASH final block - low
754 eor v9.16b, v9.16b, v20.16b // GHASH final block - high
755 eor v8.8b, v8.8b, v4.8b // GHASH final block - mid
756 pmull v8.1q, v8.1d, v16.1d // GHASH final block - mid
757 eor v11.16b, v11.16b, v21.16b // GHASH final block - low
758 eor v10.16b, v10.16b, v8.16b // GHASH final block - mid
759 movi v8.8b, #0xc2
760 eor v4.16b, v11.16b, v9.16b // MODULO - karatsuba tidy up
761 shl d8, d8, #56 // mod_constant
762 eor v10.16b, v10.16b, v4.16b // MODULO - karatsuba tidy up
763 pmull v7.1q, v9.1d, v8.1d // MODULO - top 64b align with mid
764 ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment
765 eor v10.16b, v10.16b, v7.16b // MODULO - fold into mid
766 eor v10.16b, v10.16b, v9.16b // MODULO - fold into mid
767 pmull v9.1q, v10.1d, v8.1d // MODULO - mid 64b align with low
768 ext v10.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment
769 str w9, [x16, #12] // store the updated counter
770 st1 { v5.16b}, [x2] // store all 16B
771 eor v11.16b, v11.16b, v9.16b // MODULO - fold into low
772 eor v11.16b, v11.16b, v10.16b // MODULO - fold into low
773 ext v11.16b, v11.16b, v11.16b, #8
774 rev64 v11.16b, v11.16b
775 mov x0, x15
776 st1 { v11.16b }, [x3]
777 ldp x19, x20, [sp, #16]
778 ldp x21, x22, [sp, #32]
779 ldp x23, x24, [sp, #48]
780 ldp d8, d9, [sp, #64]
781 ldp d10, d11, [sp, #80]
782 ldp d12, d13, [sp, #96]
783 ldp d14, d15, [sp, #112]
784 ldp x29, x30, [sp], #128
785 AARCH64_VALIDATE_LINK_REGISTER
786 ret
787
788 .globl _aes_gcm_dec_kernel
789 .private_extern _aes_gcm_dec_kernel
790
791 .align 4
792 _aes_gcm_dec_kernel:
793 AARCH64_SIGN_LINK_REGISTER
794 stp x29, x30, [sp, #-128]!
795 mov x29, sp
796 stp x19, x20, [sp, #16]
797 mov x16, x4
798 mov x8, x5
799 stp x21, x22, [sp, #32]
800 stp x23, x24, [sp, #48]
801 stp d8, d9, [sp, #64]
802 stp d10, d11, [sp, #80]
803 stp d12, d13, [sp, #96]
804 stp d14, d15, [sp, #112]
805 ldr w17, [x8, #240]
806 add x19, x8, x17, lsl #4 // borrow input_l1 for last key
807 ldp x13, x14, [x19] // load round N keys
808 ldr q31, [x19, #-16] // load round N-1 keys
809 lsr x5, x1, #3 // byte_len
810 mov x15, x5
811 ldp x10, x11, [x16] // ctr96_b64, ctr96_t32
812 ldr q26, [x8, #128] // load rk8
813 sub x5, x5, #1 // byte_len - 1
814 ldr q25, [x8, #112] // load rk7
815 and x5, x5, #0xffffffffffffffc0 // number of bytes to be processed in main loop (at least 1 byte must be handled by tail)
816 add x4, x0, x1, lsr #3 // end_input_ptr
817 ldr q24, [x8, #96] // load rk6
818 lsr x12, x11, #32
819 ldr q23, [x8, #80] // load rk5
820 orr w11, w11, w11
821 ldr q21, [x8, #48] // load rk3
822 add x5, x5, x0
823 rev w12, w12 // rev_ctr32
824 add w12, w12, #1 // increment rev_ctr32
825 fmov d3, x10 // CTR block 3
826 rev w9, w12 // CTR block 1
827 add w12, w12, #1 // CTR block 1
828 fmov d1, x10 // CTR block 1
829 orr x9, x11, x9, lsl #32 // CTR block 1
830 ld1 { v0.16b}, [x16] // special case vector load initial counter so we can start first AES block as quickly as possible
831 fmov v1.d[1], x9 // CTR block 1
832 rev w9, w12 // CTR block 2
833 add w12, w12, #1 // CTR block 2
834 fmov d2, x10 // CTR block 2
835 orr x9, x11, x9, lsl #32 // CTR block 2
836 fmov v2.d[1], x9 // CTR block 2
837 rev w9, w12 // CTR block 3
838 orr x9, x11, x9, lsl #32 // CTR block 3
839 ldr q18, [x8, #0] // load rk0
840 fmov v3.d[1], x9 // CTR block 3
841 add w12, w12, #1 // CTR block 3
842 ldr q22, [x8, #64] // load rk4
843 ldr q19, [x8, #16] // load rk1
844 aese v0.16b, v18.16b
845 aesmc v0.16b, v0.16b // AES block 0 - round 0
846 ldr q14, [x3, #80] // load h3l | h3h
847 ext v14.16b, v14.16b, v14.16b, #8
848 aese v3.16b, v18.16b
849 aesmc v3.16b, v3.16b // AES block 3 - round 0
850 ldr q15, [x3, #112] // load h4l | h4h
851 ext v15.16b, v15.16b, v15.16b, #8
852 aese v1.16b, v18.16b
853 aesmc v1.16b, v1.16b // AES block 1 - round 0
854 ldr q13, [x3, #64] // load h2l | h2h
855 ext v13.16b, v13.16b, v13.16b, #8
856 aese v2.16b, v18.16b
857 aesmc v2.16b, v2.16b // AES block 2 - round 0
858 ldr q20, [x8, #32] // load rk2
859 aese v0.16b, v19.16b
860 aesmc v0.16b, v0.16b // AES block 0 - round 1
861 aese v1.16b, v19.16b
862 aesmc v1.16b, v1.16b // AES block 1 - round 1
863 ld1 { v11.16b}, [x3]
864 ext v11.16b, v11.16b, v11.16b, #8
865 rev64 v11.16b, v11.16b
866 aese v2.16b, v19.16b
867 aesmc v2.16b, v2.16b // AES block 2 - round 1
868 ldr q27, [x8, #144] // load rk9
869 aese v3.16b, v19.16b
870 aesmc v3.16b, v3.16b // AES block 3 - round 1
871 ldr q30, [x8, #192] // load rk12
872 aese v0.16b, v20.16b
873 aesmc v0.16b, v0.16b // AES block 0 - round 2
874 ldr q12, [x3, #32] // load h1l | h1h
875 ext v12.16b, v12.16b, v12.16b, #8
876 aese v2.16b, v20.16b
877 aesmc v2.16b, v2.16b // AES block 2 - round 2
878 ldr q28, [x8, #160] // load rk10
879 aese v3.16b, v20.16b
880 aesmc v3.16b, v3.16b // AES block 3 - round 2
881 aese v0.16b, v21.16b
882 aesmc v0.16b, v0.16b // AES block 0 - round 3
883 aese v1.16b, v20.16b
884 aesmc v1.16b, v1.16b // AES block 1 - round 2
885 aese v3.16b, v21.16b
886 aesmc v3.16b, v3.16b // AES block 3 - round 3
887 aese v0.16b, v22.16b
888 aesmc v0.16b, v0.16b // AES block 0 - round 4
889 aese v2.16b, v21.16b
890 aesmc v2.16b, v2.16b // AES block 2 - round 3
891 aese v1.16b, v21.16b
892 aesmc v1.16b, v1.16b // AES block 1 - round 3
893 aese v3.16b, v22.16b
894 aesmc v3.16b, v3.16b // AES block 3 - round 4
895 aese v2.16b, v22.16b
896 aesmc v2.16b, v2.16b // AES block 2 - round 4
897 aese v1.16b, v22.16b
898 aesmc v1.16b, v1.16b // AES block 1 - round 4
899 aese v3.16b, v23.16b
900 aesmc v3.16b, v3.16b // AES block 3 - round 5
901 aese v0.16b, v23.16b
902 aesmc v0.16b, v0.16b // AES block 0 - round 5
903 aese v1.16b, v23.16b
904 aesmc v1.16b, v1.16b // AES block 1 - round 5
905 aese v2.16b, v23.16b
906 aesmc v2.16b, v2.16b // AES block 2 - round 5
907 aese v0.16b, v24.16b
908 aesmc v0.16b, v0.16b // AES block 0 - round 6
909 aese v3.16b, v24.16b
910 aesmc v3.16b, v3.16b // AES block 3 - round 6
911 cmp x17, #12 // setup flags for AES-128/192/256 check
912 aese v1.16b, v24.16b
913 aesmc v1.16b, v1.16b // AES block 1 - round 6
914 aese v2.16b, v24.16b
915 aesmc v2.16b, v2.16b // AES block 2 - round 6
916 aese v0.16b, v25.16b
917 aesmc v0.16b, v0.16b // AES block 0 - round 7
918 aese v1.16b, v25.16b
919 aesmc v1.16b, v1.16b // AES block 1 - round 7
920 aese v3.16b, v25.16b
921 aesmc v3.16b, v3.16b // AES block 3 - round 7
922 aese v0.16b, v26.16b
923 aesmc v0.16b, v0.16b // AES block 0 - round 8
924 aese v2.16b, v25.16b
925 aesmc v2.16b, v2.16b // AES block 2 - round 7
926 aese v3.16b, v26.16b
927 aesmc v3.16b, v3.16b // AES block 3 - round 8
928 aese v1.16b, v26.16b
929 aesmc v1.16b, v1.16b // AES block 1 - round 8
930 ldr q29, [x8, #176] // load rk11
931 aese v2.16b, v26.16b
932 aesmc v2.16b, v2.16b // AES block 2 - round 8
933 b.lt Ldec_finish_first_blocks // branch if AES-128
934
935 aese v0.16b, v27.16b
936 aesmc v0.16b, v0.16b // AES block 0 - round 9
937 aese v1.16b, v27.16b
938 aesmc v1.16b, v1.16b // AES block 1 - round 9
939 aese v3.16b, v27.16b
940 aesmc v3.16b, v3.16b // AES block 3 - round 9
941 aese v2.16b, v27.16b
942 aesmc v2.16b, v2.16b // AES block 2 - round 9
943 aese v0.16b, v28.16b
944 aesmc v0.16b, v0.16b // AES block 0 - round 10
945 aese v1.16b, v28.16b
946 aesmc v1.16b, v1.16b // AES block 1 - round 10
947 aese v3.16b, v28.16b
948 aesmc v3.16b, v3.16b // AES block 3 - round 10
949 aese v2.16b, v28.16b
950 aesmc v2.16b, v2.16b // AES block 2 - round 10
951 b.eq Ldec_finish_first_blocks // branch if AES-192
952
953 aese v0.16b, v29.16b
954 aesmc v0.16b, v0.16b // AES block 0 - round 11
955 aese v3.16b, v29.16b
956 aesmc v3.16b, v3.16b // AES block 3 - round 11
957 aese v1.16b, v29.16b
958 aesmc v1.16b, v1.16b // AES block 1 - round 11
959 aese v2.16b, v29.16b
960 aesmc v2.16b, v2.16b // AES block 2 - round 11
961 aese v1.16b, v30.16b
962 aesmc v1.16b, v1.16b // AES block 1 - round 12
963 aese v0.16b, v30.16b
964 aesmc v0.16b, v0.16b // AES block 0 - round 12
965 aese v2.16b, v30.16b
966 aesmc v2.16b, v2.16b // AES block 2 - round 12
967 aese v3.16b, v30.16b
968 aesmc v3.16b, v3.16b // AES block 3 - round 12
969
970 Ldec_finish_first_blocks:
971 cmp x0, x5 // check if we have <= 4 blocks
972 trn1 v9.2d, v14.2d, v15.2d // h4h | h3h
973 trn2 v17.2d, v14.2d, v15.2d // h4l | h3l
974 trn1 v8.2d, v12.2d, v13.2d // h2h | h1h
975 trn2 v16.2d, v12.2d, v13.2d // h2l | h1l
976 eor v17.16b, v17.16b, v9.16b // h4k | h3k
977 aese v1.16b, v31.16b // AES block 1 - round N-1
978 aese v2.16b, v31.16b // AES block 2 - round N-1
979 eor v16.16b, v16.16b, v8.16b // h2k | h1k
980 aese v3.16b, v31.16b // AES block 3 - round N-1
981 aese v0.16b, v31.16b // AES block 0 - round N-1
982 b.ge Ldec_tail // handle tail
983
984 ldr q4, [x0, #0] // AES block 0 - load ciphertext
985 ldr q5, [x0, #16] // AES block 1 - load ciphertext
986 rev w9, w12 // CTR block 4
987 eor v0.16b, v4.16b, v0.16b // AES block 0 - result
988 eor v1.16b, v5.16b, v1.16b // AES block 1 - result
989 rev64 v5.16b, v5.16b // GHASH block 1
990 ldr q7, [x0, #48] // AES block 3 - load ciphertext
991 mov x7, v0.d[1] // AES block 0 - mov high
992 mov x6, v0.d[0] // AES block 0 - mov low
993 rev64 v4.16b, v4.16b // GHASH block 0
994 add w12, w12, #1 // CTR block 4
995 fmov d0, x10 // CTR block 4
996 orr x9, x11, x9, lsl #32 // CTR block 4
997 fmov v0.d[1], x9 // CTR block 4
998 rev w9, w12 // CTR block 5
999 add w12, w12, #1 // CTR block 5
1000 mov x19, v1.d[0] // AES block 1 - mov low
1001 orr x9, x11, x9, lsl #32 // CTR block 5
1002 mov x20, v1.d[1] // AES block 1 - mov high
1003 eor x7, x7, x14 // AES block 0 - round N high
1004 eor x6, x6, x13 // AES block 0 - round N low
1005 stp x6, x7, [x2], #16 // AES block 0 - store result
1006 fmov d1, x10 // CTR block 5
1007 ldr q6, [x0, #32] // AES block 2 - load ciphertext
1008 add x0, x0, #64 // AES input_ptr update
1009 fmov v1.d[1], x9 // CTR block 5
1010 rev w9, w12 // CTR block 6
1011 add w12, w12, #1 // CTR block 6
1012 eor x19, x19, x13 // AES block 1 - round N low
1013 orr x9, x11, x9, lsl #32 // CTR block 6
1014 eor x20, x20, x14 // AES block 1 - round N high
1015 stp x19, x20, [x2], #16 // AES block 1 - store result
1016 eor v2.16b, v6.16b, v2.16b // AES block 2 - result
1017 cmp x0, x5 // check if we have <= 8 blocks
1018 b.ge Ldec_prepretail // do prepretail
1019
1020 Ldec_main_loop: // main loop start
1021 mov x21, v2.d[0] // AES block 4k+2 - mov low
1022 ext v11.16b, v11.16b, v11.16b, #8 // PRE 0
1023 eor v3.16b, v7.16b, v3.16b // AES block 4k+3 - result
1024 aese v0.16b, v18.16b
1025 aesmc v0.16b, v0.16b // AES block 4k+4 - round 0
1026 mov x22, v2.d[1] // AES block 4k+2 - mov high
1027 aese v1.16b, v18.16b
1028 aesmc v1.16b, v1.16b // AES block 4k+5 - round 0
1029 fmov d2, x10 // CTR block 4k+6
1030 fmov v2.d[1], x9 // CTR block 4k+6
1031 eor v4.16b, v4.16b, v11.16b // PRE 1
1032 rev w9, w12 // CTR block 4k+7
1033 aese v0.16b, v19.16b
1034 aesmc v0.16b, v0.16b // AES block 4k+4 - round 1
1035 mov x24, v3.d[1] // AES block 4k+3 - mov high
1036 aese v1.16b, v19.16b
1037 aesmc v1.16b, v1.16b // AES block 4k+5 - round 1
1038 mov x23, v3.d[0] // AES block 4k+3 - mov low
1039 pmull2 v9.1q, v4.2d, v15.2d // GHASH block 4k - high
1040 mov d8, v4.d[1] // GHASH block 4k - mid
1041 fmov d3, x10 // CTR block 4k+7
1042 aese v0.16b, v20.16b
1043 aesmc v0.16b, v0.16b // AES block 4k+4 - round 2
1044 orr x9, x11, x9, lsl #32 // CTR block 4k+7
1045 aese v2.16b, v18.16b
1046 aesmc v2.16b, v2.16b // AES block 4k+6 - round 0
1047 fmov v3.d[1], x9 // CTR block 4k+7
1048 aese v1.16b, v20.16b
1049 aesmc v1.16b, v1.16b // AES block 4k+5 - round 2
1050 eor v8.8b, v8.8b, v4.8b // GHASH block 4k - mid
1051 aese v0.16b, v21.16b
1052 aesmc v0.16b, v0.16b // AES block 4k+4 - round 3
1053 eor x22, x22, x14 // AES block 4k+2 - round N high
1054 aese v2.16b, v19.16b
1055 aesmc v2.16b, v2.16b // AES block 4k+6 - round 1
1056 mov d10, v17.d[1] // GHASH block 4k - mid
1057 aese v1.16b, v21.16b
1058 aesmc v1.16b, v1.16b // AES block 4k+5 - round 3
1059 rev64 v6.16b, v6.16b // GHASH block 4k+2
1060 aese v3.16b, v18.16b
1061 aesmc v3.16b, v3.16b // AES block 4k+7 - round 0
1062 eor x21, x21, x13 // AES block 4k+2 - round N low
1063 aese v2.16b, v20.16b
1064 aesmc v2.16b, v2.16b // AES block 4k+6 - round 2
1065 stp x21, x22, [x2], #16 // AES block 4k+2 - store result
1066 pmull v11.1q, v4.1d, v15.1d // GHASH block 4k - low
1067 pmull2 v4.1q, v5.2d, v14.2d // GHASH block 4k+1 - high
1068 aese v2.16b, v21.16b
1069 aesmc v2.16b, v2.16b // AES block 4k+6 - round 3
1070 rev64 v7.16b, v7.16b // GHASH block 4k+3
1071 pmull v10.1q, v8.1d, v10.1d // GHASH block 4k - mid
1072 eor x23, x23, x13 // AES block 4k+3 - round N low
1073 pmull v8.1q, v5.1d, v14.1d // GHASH block 4k+1 - low
1074 eor x24, x24, x14 // AES block 4k+3 - round N high
1075 eor v9.16b, v9.16b, v4.16b // GHASH block 4k+1 - high
1076 aese v2.16b, v22.16b
1077 aesmc v2.16b, v2.16b // AES block 4k+6 - round 4
1078 aese v3.16b, v19.16b
1079 aesmc v3.16b, v3.16b // AES block 4k+7 - round 1
1080 mov d4, v5.d[1] // GHASH block 4k+1 - mid
1081 aese v0.16b, v22.16b
1082 aesmc v0.16b, v0.16b // AES block 4k+4 - round 4
1083 eor v11.16b, v11.16b, v8.16b // GHASH block 4k+1 - low
1084 aese v2.16b, v23.16b
1085 aesmc v2.16b, v2.16b // AES block 4k+6 - round 5
1086 add w12, w12, #1 // CTR block 4k+7
1087 aese v3.16b, v20.16b
1088 aesmc v3.16b, v3.16b // AES block 4k+7 - round 2
1089 mov d8, v6.d[1] // GHASH block 4k+2 - mid
1090 aese v1.16b, v22.16b
1091 aesmc v1.16b, v1.16b // AES block 4k+5 - round 4
1092 eor v4.8b, v4.8b, v5.8b // GHASH block 4k+1 - mid
1093 pmull v5.1q, v6.1d, v13.1d // GHASH block 4k+2 - low
1094 aese v3.16b, v21.16b
1095 aesmc v3.16b, v3.16b // AES block 4k+7 - round 3
1096 eor v8.8b, v8.8b, v6.8b // GHASH block 4k+2 - mid
1097 aese v1.16b, v23.16b
1098 aesmc v1.16b, v1.16b // AES block 4k+5 - round 5
1099 aese v0.16b, v23.16b
1100 aesmc v0.16b, v0.16b // AES block 4k+4 - round 5
1101 eor v11.16b, v11.16b, v5.16b // GHASH block 4k+2 - low
1102 pmull v4.1q, v4.1d, v17.1d // GHASH block 4k+1 - mid
1103 rev w9, w12 // CTR block 4k+8
1104 aese v1.16b, v24.16b
1105 aesmc v1.16b, v1.16b // AES block 4k+5 - round 6
1106 ins v8.d[1], v8.d[0] // GHASH block 4k+2 - mid
1107 aese v0.16b, v24.16b
1108 aesmc v0.16b, v0.16b // AES block 4k+4 - round 6
1109 add w12, w12, #1 // CTR block 4k+8
1110 aese v3.16b, v22.16b
1111 aesmc v3.16b, v3.16b // AES block 4k+7 - round 4
1112 aese v1.16b, v25.16b
1113 aesmc v1.16b, v1.16b // AES block 4k+5 - round 7
1114 eor v10.16b, v10.16b, v4.16b // GHASH block 4k+1 - mid
1115 aese v0.16b, v25.16b
1116 aesmc v0.16b, v0.16b // AES block 4k+4 - round 7
1117 pmull2 v4.1q, v6.2d, v13.2d // GHASH block 4k+2 - high
1118 mov d6, v7.d[1] // GHASH block 4k+3 - mid
1119 aese v3.16b, v23.16b
1120 aesmc v3.16b, v3.16b // AES block 4k+7 - round 5
1121 pmull2 v8.1q, v8.2d, v16.2d // GHASH block 4k+2 - mid
1122 aese v0.16b, v26.16b
1123 aesmc v0.16b, v0.16b // AES block 4k+4 - round 8
1124 eor v9.16b, v9.16b, v4.16b // GHASH block 4k+2 - high
1125 aese v3.16b, v24.16b
1126 aesmc v3.16b, v3.16b // AES block 4k+7 - round 6
1127 pmull v4.1q, v7.1d, v12.1d // GHASH block 4k+3 - low
1128 orr x9, x11, x9, lsl #32 // CTR block 4k+8
1129 eor v10.16b, v10.16b, v8.16b // GHASH block 4k+2 - mid
1130 pmull2 v5.1q, v7.2d, v12.2d // GHASH block 4k+3 - high
1131 cmp x17, #12 // setup flags for AES-128/192/256 check
1132 eor v6.8b, v6.8b, v7.8b // GHASH block 4k+3 - mid
1133 aese v1.16b, v26.16b
1134 aesmc v1.16b, v1.16b // AES block 4k+5 - round 8
1135 aese v2.16b, v24.16b
1136 aesmc v2.16b, v2.16b // AES block 4k+6 - round 6
1137 eor v9.16b, v9.16b, v5.16b // GHASH block 4k+3 - high
1138 pmull v6.1q, v6.1d, v16.1d // GHASH block 4k+3 - mid
1139 movi v8.8b, #0xc2
1140 aese v2.16b, v25.16b
1141 aesmc v2.16b, v2.16b // AES block 4k+6 - round 7
1142 eor v11.16b, v11.16b, v4.16b // GHASH block 4k+3 - low
1143 aese v3.16b, v25.16b
1144 aesmc v3.16b, v3.16b // AES block 4k+7 - round 7
1145 shl d8, d8, #56 // mod_constant
1146 aese v2.16b, v26.16b
1147 aesmc v2.16b, v2.16b // AES block 4k+6 - round 8
1148 eor v10.16b, v10.16b, v6.16b // GHASH block 4k+3 - mid
1149 aese v3.16b, v26.16b
1150 aesmc v3.16b, v3.16b // AES block 4k+7 - round 8
1151 b.lt Ldec_main_loop_continue // branch if AES-128
1152
1153 aese v0.16b, v27.16b
1154 aesmc v0.16b, v0.16b // AES block 4k+4 - round 9
1155 aese v2.16b, v27.16b
1156 aesmc v2.16b, v2.16b // AES block 4k+6 - round 9
1157 aese v1.16b, v27.16b
1158 aesmc v1.16b, v1.16b // AES block 4k+5 - round 9
1159 aese v3.16b, v27.16b
1160 aesmc v3.16b, v3.16b // AES block 4k+7 - round 9
1161 aese v0.16b, v28.16b
1162 aesmc v0.16b, v0.16b // AES block 4k+4 - round 10
1163 aese v1.16b, v28.16b
1164 aesmc v1.16b, v1.16b // AES block 4k+5 - round 10
1165 aese v2.16b, v28.16b
1166 aesmc v2.16b, v2.16b // AES block 4k+6 - round 10
1167 aese v3.16b, v28.16b
1168 aesmc v3.16b, v3.16b // AES block 4k+7 - round 10
1169 b.eq Ldec_main_loop_continue // branch if AES-192
1170
1171 aese v0.16b, v29.16b
1172 aesmc v0.16b, v0.16b // AES block 4k+4 - round 11
1173 aese v1.16b, v29.16b
1174 aesmc v1.16b, v1.16b // AES block 4k+5 - round 11
1175 aese v2.16b, v29.16b
1176 aesmc v2.16b, v2.16b // AES block 4k+6 - round 11
1177 aese v3.16b, v29.16b
1178 aesmc v3.16b, v3.16b // AES block 4k+7 - round 11
1179 aese v0.16b, v30.16b
1180 aesmc v0.16b, v0.16b // AES block 4k+4 - round 12
1181 aese v1.16b, v30.16b
1182 aesmc v1.16b, v1.16b // AES block 4k+5 - round 12
1183 aese v2.16b, v30.16b
1184 aesmc v2.16b, v2.16b // AES block 4k+6 - round 12
1185 aese v3.16b, v30.16b
1186 aesmc v3.16b, v3.16b // AES block 4k+7 - round 12
1187
1188 Ldec_main_loop_continue:
1189 pmull v7.1q, v9.1d, v8.1d // MODULO - top 64b align with mid
1190 eor v6.16b, v11.16b, v9.16b // MODULO - karatsuba tidy up
1191 ldr q4, [x0, #0] // AES block 4k+4 - load ciphertext
1192 aese v0.16b, v31.16b // AES block 4k+4 - round N-1
1193 ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment
1194 eor v10.16b, v10.16b, v6.16b // MODULO - karatsuba tidy up
1195 ldr q5, [x0, #16] // AES block 4k+5 - load ciphertext
1196 eor v0.16b, v4.16b, v0.16b // AES block 4k+4 - result
1197 stp x23, x24, [x2], #16 // AES block 4k+3 - store result
1198 eor v10.16b, v10.16b, v7.16b // MODULO - fold into mid
1199 ldr q7, [x0, #48] // AES block 4k+7 - load ciphertext
1200 ldr q6, [x0, #32] // AES block 4k+6 - load ciphertext
1201 mov x7, v0.d[1] // AES block 4k+4 - mov high
1202 eor v10.16b, v10.16b, v9.16b // MODULO - fold into mid
1203 aese v1.16b, v31.16b // AES block 4k+5 - round N-1
1204 add x0, x0, #64 // AES input_ptr update
1205 mov x6, v0.d[0] // AES block 4k+4 - mov low
1206 fmov d0, x10 // CTR block 4k+8
1207 fmov v0.d[1], x9 // CTR block 4k+8
1208 pmull v8.1q, v10.1d, v8.1d // MODULO - mid 64b align with low
1209 eor v1.16b, v5.16b, v1.16b // AES block 4k+5 - result
1210 rev w9, w12 // CTR block 4k+9
1211 aese v2.16b, v31.16b // AES block 4k+6 - round N-1
1212 orr x9, x11, x9, lsl #32 // CTR block 4k+9
1213 cmp x0, x5 // LOOP CONTROL
1214 add w12, w12, #1 // CTR block 4k+9
1215 eor x6, x6, x13 // AES block 4k+4 - round N low
1216 eor x7, x7, x14 // AES block 4k+4 - round N high
1217 mov x20, v1.d[1] // AES block 4k+5 - mov high
1218 eor v2.16b, v6.16b, v2.16b // AES block 4k+6 - result
1219 eor v11.16b, v11.16b, v8.16b // MODULO - fold into low
1220 mov x19, v1.d[0] // AES block 4k+5 - mov low
1221 fmov d1, x10 // CTR block 4k+9
1222 ext v10.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment
1223 fmov v1.d[1], x9 // CTR block 4k+9
1224 rev w9, w12 // CTR block 4k+10
1225 add w12, w12, #1 // CTR block 4k+10
1226 aese v3.16b, v31.16b // AES block 4k+7 - round N-1
1227 orr x9, x11, x9, lsl #32 // CTR block 4k+10
1228 rev64 v5.16b, v5.16b // GHASH block 4k+5
1229 eor x20, x20, x14 // AES block 4k+5 - round N high
1230 stp x6, x7, [x2], #16 // AES block 4k+4 - store result
1231 eor x19, x19, x13 // AES block 4k+5 - round N low
1232 stp x19, x20, [x2], #16 // AES block 4k+5 - store result
1233 rev64 v4.16b, v4.16b // GHASH block 4k+4
1234 eor v11.16b, v11.16b, v10.16b // MODULO - fold into low
1235 b.lt Ldec_main_loop
1236
1237 Ldec_prepretail: // PREPRETAIL
1238 ext v11.16b, v11.16b, v11.16b, #8 // PRE 0
1239 mov x21, v2.d[0] // AES block 4k+2 - mov low
1240 eor v3.16b, v7.16b, v3.16b // AES block 4k+3 - result
1241 aese v0.16b, v18.16b
1242 aesmc v0.16b, v0.16b // AES block 4k+4 - round 0
1243 mov x22, v2.d[1] // AES block 4k+2 - mov high
1244 aese v1.16b, v18.16b
1245 aesmc v1.16b, v1.16b // AES block 4k+5 - round 0
1246 fmov d2, x10 // CTR block 4k+6
1247 fmov v2.d[1], x9 // CTR block 4k+6
1248 rev w9, w12 // CTR block 4k+7
1249 eor v4.16b, v4.16b, v11.16b // PRE 1
1250 rev64 v6.16b, v6.16b // GHASH block 4k+2
1251 orr x9, x11, x9, lsl #32 // CTR block 4k+7
1252 mov x23, v3.d[0] // AES block 4k+3 - mov low
1253 aese v1.16b, v19.16b
1254 aesmc v1.16b, v1.16b // AES block 4k+5 - round 1
1255 mov x24, v3.d[1] // AES block 4k+3 - mov high
1256 pmull v11.1q, v4.1d, v15.1d // GHASH block 4k - low
1257 mov d8, v4.d[1] // GHASH block 4k - mid
1258 fmov d3, x10 // CTR block 4k+7
1259 pmull2 v9.1q, v4.2d, v15.2d // GHASH block 4k - high
1260 fmov v3.d[1], x9 // CTR block 4k+7
1261 aese v2.16b, v18.16b
1262 aesmc v2.16b, v2.16b // AES block 4k+6 - round 0
1263 mov d10, v17.d[1] // GHASH block 4k - mid
1264 aese v0.16b, v19.16b
1265 aesmc v0.16b, v0.16b // AES block 4k+4 - round 1
1266 eor v8.8b, v8.8b, v4.8b // GHASH block 4k - mid
1267 pmull2 v4.1q, v5.2d, v14.2d // GHASH block 4k+1 - high
1268 aese v2.16b, v19.16b
1269 aesmc v2.16b, v2.16b // AES block 4k+6 - round 1
1270 rev64 v7.16b, v7.16b // GHASH block 4k+3
1271 aese v3.16b, v18.16b
1272 aesmc v3.16b, v3.16b // AES block 4k+7 - round 0
1273 pmull v10.1q, v8.1d, v10.1d // GHASH block 4k - mid
1274 eor v9.16b, v9.16b, v4.16b // GHASH block 4k+1 - high
1275 pmull v8.1q, v5.1d, v14.1d // GHASH block 4k+1 - low
1276 aese v3.16b, v19.16b
1277 aesmc v3.16b, v3.16b // AES block 4k+7 - round 1
1278 mov d4, v5.d[1] // GHASH block 4k+1 - mid
1279 aese v0.16b, v20.16b
1280 aesmc v0.16b, v0.16b // AES block 4k+4 - round 2
1281 aese v1.16b, v20.16b
1282 aesmc v1.16b, v1.16b // AES block 4k+5 - round 2
1283 eor v11.16b, v11.16b, v8.16b // GHASH block 4k+1 - low
1284 aese v2.16b, v20.16b
1285 aesmc v2.16b, v2.16b // AES block 4k+6 - round 2
1286 aese v0.16b, v21.16b
1287 aesmc v0.16b, v0.16b // AES block 4k+4 - round 3
1288 mov d8, v6.d[1] // GHASH block 4k+2 - mid
1289 aese v3.16b, v20.16b
1290 aesmc v3.16b, v3.16b // AES block 4k+7 - round 2
1291 eor v4.8b, v4.8b, v5.8b // GHASH block 4k+1 - mid
1292 pmull v5.1q, v6.1d, v13.1d // GHASH block 4k+2 - low
1293 aese v0.16b, v22.16b
1294 aesmc v0.16b, v0.16b // AES block 4k+4 - round 4
1295 aese v3.16b, v21.16b
1296 aesmc v3.16b, v3.16b // AES block 4k+7 - round 3
1297 eor v8.8b, v8.8b, v6.8b // GHASH block 4k+2 - mid
1298 pmull v4.1q, v4.1d, v17.1d // GHASH block 4k+1 - mid
1299 aese v0.16b, v23.16b
1300 aesmc v0.16b, v0.16b // AES block 4k+4 - round 5
1301 eor v11.16b, v11.16b, v5.16b // GHASH block 4k+2 - low
1302 aese v3.16b, v22.16b
1303 aesmc v3.16b, v3.16b // AES block 4k+7 - round 4
1304 pmull2 v5.1q, v7.2d, v12.2d // GHASH block 4k+3 - high
1305 eor v10.16b, v10.16b, v4.16b // GHASH block 4k+1 - mid
1306 pmull2 v4.1q, v6.2d, v13.2d // GHASH block 4k+2 - high
1307 aese v3.16b, v23.16b
1308 aesmc v3.16b, v3.16b // AES block 4k+7 - round 5
1309 ins v8.d[1], v8.d[0] // GHASH block 4k+2 - mid
1310 aese v2.16b, v21.16b
1311 aesmc v2.16b, v2.16b // AES block 4k+6 - round 3
1312 aese v1.16b, v21.16b
1313 aesmc v1.16b, v1.16b // AES block 4k+5 - round 3
1314 eor v9.16b, v9.16b, v4.16b // GHASH block 4k+2 - high
1315 pmull v4.1q, v7.1d, v12.1d // GHASH block 4k+3 - low
1316 aese v2.16b, v22.16b
1317 aesmc v2.16b, v2.16b // AES block 4k+6 - round 4
1318 mov d6, v7.d[1] // GHASH block 4k+3 - mid
1319 aese v1.16b, v22.16b
1320 aesmc v1.16b, v1.16b // AES block 4k+5 - round 4
1321 pmull2 v8.1q, v8.2d, v16.2d // GHASH block 4k+2 - mid
1322 aese v2.16b, v23.16b
1323 aesmc v2.16b, v2.16b // AES block 4k+6 - round 5
1324 eor v6.8b, v6.8b, v7.8b // GHASH block 4k+3 - mid
1325 aese v1.16b, v23.16b
1326 aesmc v1.16b, v1.16b // AES block 4k+5 - round 5
1327 aese v3.16b, v24.16b
1328 aesmc v3.16b, v3.16b // AES block 4k+7 - round 6
1329 eor v10.16b, v10.16b, v8.16b // GHASH block 4k+2 - mid
1330 aese v2.16b, v24.16b
1331 aesmc v2.16b, v2.16b // AES block 4k+6 - round 6
1332 aese v0.16b, v24.16b
1333 aesmc v0.16b, v0.16b // AES block 4k+4 - round 6
1334 movi v8.8b, #0xc2
1335 aese v1.16b, v24.16b
1336 aesmc v1.16b, v1.16b // AES block 4k+5 - round 6
1337 eor v11.16b, v11.16b, v4.16b // GHASH block 4k+3 - low
1338 pmull v6.1q, v6.1d, v16.1d // GHASH block 4k+3 - mid
1339 aese v3.16b, v25.16b
1340 aesmc v3.16b, v3.16b // AES block 4k+7 - round 7
1341 cmp x17, #12 // setup flags for AES-128/192/256 check
1342 eor v9.16b, v9.16b, v5.16b // GHASH block 4k+3 - high
1343 aese v1.16b, v25.16b
1344 aesmc v1.16b, v1.16b // AES block 4k+5 - round 7
1345 aese v0.16b, v25.16b
1346 aesmc v0.16b, v0.16b // AES block 4k+4 - round 7
1347 eor v10.16b, v10.16b, v6.16b // GHASH block 4k+3 - mid
1348 aese v3.16b, v26.16b
1349 aesmc v3.16b, v3.16b // AES block 4k+7 - round 8
1350 aese v2.16b, v25.16b
1351 aesmc v2.16b, v2.16b // AES block 4k+6 - round 7
1352 eor v6.16b, v11.16b, v9.16b // MODULO - karatsuba tidy up
1353 aese v1.16b, v26.16b
1354 aesmc v1.16b, v1.16b // AES block 4k+5 - round 8
1355 aese v0.16b, v26.16b
1356 aesmc v0.16b, v0.16b // AES block 4k+4 - round 8
1357 shl d8, d8, #56 // mod_constant
1358 aese v2.16b, v26.16b
1359 aesmc v2.16b, v2.16b // AES block 4k+6 - round 8
1360 b.lt Ldec_finish_prepretail // branch if AES-128
1361
1362 aese v1.16b, v27.16b
1363 aesmc v1.16b, v1.16b // AES block 4k+5 - round 9
1364 aese v2.16b, v27.16b
1365 aesmc v2.16b, v2.16b // AES block 4k+6 - round 9
1366 aese v3.16b, v27.16b
1367 aesmc v3.16b, v3.16b // AES block 4k+7 - round 9
1368 aese v0.16b, v27.16b
1369 aesmc v0.16b, v0.16b // AES block 4k+4 - round 9
1370 aese v2.16b, v28.16b
1371 aesmc v2.16b, v2.16b // AES block 4k+6 - round 10
1372 aese v3.16b, v28.16b
1373 aesmc v3.16b, v3.16b // AES block 4k+7 - round 10
1374 aese v0.16b, v28.16b
1375 aesmc v0.16b, v0.16b // AES block 4k+4 - round 10
1376 aese v1.16b, v28.16b
1377 aesmc v1.16b, v1.16b // AES block 4k+5 - round 10
1378 b.eq Ldec_finish_prepretail // branch if AES-192
1379
1380 aese v2.16b, v29.16b
1381 aesmc v2.16b, v2.16b // AES block 4k+6 - round 11
1382 aese v0.16b, v29.16b
1383 aesmc v0.16b, v0.16b // AES block 4k+4 - round 11
1384 aese v1.16b, v29.16b
1385 aesmc v1.16b, v1.16b // AES block 4k+5 - round 11
1386 aese v2.16b, v30.16b
1387 aesmc v2.16b, v2.16b // AES block 4k+6 - round 12
1388 aese v3.16b, v29.16b
1389 aesmc v3.16b, v3.16b // AES block 4k+7 - round 11
1390 aese v1.16b, v30.16b
1391 aesmc v1.16b, v1.16b // AES block 4k+5 - round 12
1392 aese v0.16b, v30.16b
1393 aesmc v0.16b, v0.16b // AES block 4k+4 - round 12
1394 aese v3.16b, v30.16b
1395 aesmc v3.16b, v3.16b // AES block 4k+7 - round 12
1396
1397 Ldec_finish_prepretail:
1398 eor v10.16b, v10.16b, v6.16b // MODULO - karatsuba tidy up
1399 pmull v7.1q, v9.1d, v8.1d // MODULO - top 64b align with mid
1400 ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment
1401 eor v10.16b, v10.16b, v7.16b // MODULO - fold into mid
1402 eor x22, x22, x14 // AES block 4k+2 - round N high
1403 eor x23, x23, x13 // AES block 4k+3 - round N low
1404 eor v10.16b, v10.16b, v9.16b // MODULO - fold into mid
1405 add w12, w12, #1 // CTR block 4k+7
1406 eor x21, x21, x13 // AES block 4k+2 - round N low
1407 pmull v8.1q, v10.1d, v8.1d // MODULO - mid 64b align with low
1408 eor x24, x24, x14 // AES block 4k+3 - round N high
1409 stp x21, x22, [x2], #16 // AES block 4k+2 - store result
1410 ext v10.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment
1411 stp x23, x24, [x2], #16 // AES block 4k+3 - store result
1412
1413 eor v11.16b, v11.16b, v8.16b // MODULO - fold into low
1414 aese v1.16b, v31.16b // AES block 4k+5 - round N-1
1415 aese v0.16b, v31.16b // AES block 4k+4 - round N-1
1416 aese v3.16b, v31.16b // AES block 4k+7 - round N-1
1417 aese v2.16b, v31.16b // AES block 4k+6 - round N-1
1418 eor v11.16b, v11.16b, v10.16b // MODULO - fold into low
1419
1420 Ldec_tail: // TAIL
1421 sub x5, x4, x0 // main_end_input_ptr is number of bytes left to process
1422 ld1 { v5.16b}, [x0], #16 // AES block 4k+4 - load ciphertext
1423 eor v0.16b, v5.16b, v0.16b // AES block 4k+4 - result
1424 mov x6, v0.d[0] // AES block 4k+4 - mov low
1425 mov x7, v0.d[1] // AES block 4k+4 - mov high
1426 ext v8.16b, v11.16b, v11.16b, #8 // prepare final partial tag
1427 cmp x5, #48
1428 eor x6, x6, x13 // AES block 4k+4 - round N low
1429 eor x7, x7, x14 // AES block 4k+4 - round N high
1430 b.gt Ldec_blocks_more_than_3
1431 sub w12, w12, #1
1432 mov v3.16b, v2.16b
1433 movi v10.8b, #0
1434 movi v11.8b, #0
1435 cmp x5, #32
1436 movi v9.8b, #0
1437 mov v2.16b, v1.16b
1438 b.gt Ldec_blocks_more_than_2
1439 sub w12, w12, #1
1440 mov v3.16b, v1.16b
1441 cmp x5, #16
1442 b.gt Ldec_blocks_more_than_1
1443 sub w12, w12, #1
1444 b Ldec_blocks_less_than_1
1445 Ldec_blocks_more_than_3: // blocks left > 3
1446 rev64 v4.16b, v5.16b // GHASH final-3 block
1447 ld1 { v5.16b}, [x0], #16 // AES final-2 block - load ciphertext
1448 stp x6, x7, [x2], #16 // AES final-3 block - store result
1449 mov d10, v17.d[1] // GHASH final-3 block - mid
1450 eor v4.16b, v4.16b, v8.16b // feed in partial tag
1451 eor v0.16b, v5.16b, v1.16b // AES final-2 block - result
1452 mov d22, v4.d[1] // GHASH final-3 block - mid
1453 mov x6, v0.d[0] // AES final-2 block - mov low
1454 mov x7, v0.d[1] // AES final-2 block - mov high
1455 eor v22.8b, v22.8b, v4.8b // GHASH final-3 block - mid
1456 movi v8.8b, #0 // suppress further partial tag feed in
1457 pmull2 v9.1q, v4.2d, v15.2d // GHASH final-3 block - high
1458 pmull v10.1q, v22.1d, v10.1d // GHASH final-3 block - mid
1459 eor x6, x6, x13 // AES final-2 block - round N low
1460 pmull v11.1q, v4.1d, v15.1d // GHASH final-3 block - low
1461 eor x7, x7, x14 // AES final-2 block - round N high
1462 Ldec_blocks_more_than_2: // blocks left > 2
1463 rev64 v4.16b, v5.16b // GHASH final-2 block
1464 ld1 { v5.16b}, [x0], #16 // AES final-1 block - load ciphertext
1465 eor v4.16b, v4.16b, v8.16b // feed in partial tag
1466 stp x6, x7, [x2], #16 // AES final-2 block - store result
1467 eor v0.16b, v5.16b, v2.16b // AES final-1 block - result
1468 mov d22, v4.d[1] // GHASH final-2 block - mid
1469 pmull v21.1q, v4.1d, v14.1d // GHASH final-2 block - low
1470 pmull2 v20.1q, v4.2d, v14.2d // GHASH final-2 block - high
1471 eor v22.8b, v22.8b, v4.8b // GHASH final-2 block - mid
1472 mov x6, v0.d[0] // AES final-1 block - mov low
1473 mov x7, v0.d[1] // AES final-1 block - mov high
1474 eor v11.16b, v11.16b, v21.16b // GHASH final-2 block - low
1475 movi v8.8b, #0 // suppress further partial tag feed in
1476 pmull v22.1q, v22.1d, v17.1d // GHASH final-2 block - mid
1477 eor v9.16b, v9.16b, v20.16b // GHASH final-2 block - high
1478 eor x6, x6, x13 // AES final-1 block - round N low
1479 eor v10.16b, v10.16b, v22.16b // GHASH final-2 block - mid
1480 eor x7, x7, x14 // AES final-1 block - round N high
1481 Ldec_blocks_more_than_1: // blocks left > 1
1482 stp x6, x7, [x2], #16 // AES final-1 block - store result
1483 rev64 v4.16b, v5.16b // GHASH final-1 block
1484 ld1 { v5.16b}, [x0], #16 // AES final block - load ciphertext
1485 eor v4.16b, v4.16b, v8.16b // feed in partial tag
1486 movi v8.8b, #0 // suppress further partial tag feed in
1487 mov d22, v4.d[1] // GHASH final-1 block - mid
1488 eor v0.16b, v5.16b, v3.16b // AES final block - result
1489 pmull2 v20.1q, v4.2d, v13.2d // GHASH final-1 block - high
1490 eor v22.8b, v22.8b, v4.8b // GHASH final-1 block - mid
1491 pmull v21.1q, v4.1d, v13.1d // GHASH final-1 block - low
1492 mov x6, v0.d[0] // AES final block - mov low
1493 ins v22.d[1], v22.d[0] // GHASH final-1 block - mid
1494 mov x7, v0.d[1] // AES final block - mov high
1495 pmull2 v22.1q, v22.2d, v16.2d // GHASH final-1 block - mid
1496 eor x6, x6, x13 // AES final block - round N low
1497 eor v11.16b, v11.16b, v21.16b // GHASH final-1 block - low
1498 eor v9.16b, v9.16b, v20.16b // GHASH final-1 block - high
1499 eor v10.16b, v10.16b, v22.16b // GHASH final-1 block - mid
1500 eor x7, x7, x14 // AES final block - round N high
1501 Ldec_blocks_less_than_1: // blocks left <= 1
1502 and x1, x1, #127 // bit_length %= 128
1503 mvn x14, xzr // rkN_h = 0xffffffffffffffff
1504 sub x1, x1, #128 // bit_length -= 128
1505 mvn x13, xzr // rkN_l = 0xffffffffffffffff
1506 ldp x4, x5, [x2] // load existing bytes we need to not overwrite
1507 neg x1, x1 // bit_length = 128 - #bits in input (in range [1,128])
1508 and x1, x1, #127 // bit_length %= 128
1509 lsr x14, x14, x1 // rkN_h is mask for top 64b of last block
1510 cmp x1, #64
1511 csel x9, x13, x14, lt
1512 csel x10, x14, xzr, lt
1513 fmov d0, x9 // ctr0b is mask for last block
1514 and x6, x6, x9
1515 mov v0.d[1], x10
1516 bic x4, x4, x9 // mask out low existing bytes
1517 rev w9, w12
1518 bic x5, x5, x10 // mask out high existing bytes
1519 orr x6, x6, x4
1520 and x7, x7, x10
1521 orr x7, x7, x5
1522 and v5.16b, v5.16b, v0.16b // possibly partial last block has zeroes in highest bits
1523 rev64 v4.16b, v5.16b // GHASH final block
1524 eor v4.16b, v4.16b, v8.16b // feed in partial tag
1525 pmull v21.1q, v4.1d, v12.1d // GHASH final block - low
1526 mov d8, v4.d[1] // GHASH final block - mid
1527 eor v8.8b, v8.8b, v4.8b // GHASH final block - mid
1528 pmull2 v20.1q, v4.2d, v12.2d // GHASH final block - high
1529 pmull v8.1q, v8.1d, v16.1d // GHASH final block - mid
1530 eor v9.16b, v9.16b, v20.16b // GHASH final block - high
1531 eor v11.16b, v11.16b, v21.16b // GHASH final block - low
1532 eor v10.16b, v10.16b, v8.16b // GHASH final block - mid
1533 movi v8.8b, #0xc2
1534 eor v6.16b, v11.16b, v9.16b // MODULO - karatsuba tidy up
1535 shl d8, d8, #56 // mod_constant
1536 eor v10.16b, v10.16b, v6.16b // MODULO - karatsuba tidy up
1537 pmull v7.1q, v9.1d, v8.1d // MODULO - top 64b align with mid
1538 ext v9.16b, v9.16b, v9.16b, #8 // MODULO - other top alignment
1539 eor v10.16b, v10.16b, v7.16b // MODULO - fold into mid
1540 eor v10.16b, v10.16b, v9.16b // MODULO - fold into mid
1541 pmull v8.1q, v10.1d, v8.1d // MODULO - mid 64b align with low
1542 ext v10.16b, v10.16b, v10.16b, #8 // MODULO - other mid alignment
1543 eor v11.16b, v11.16b, v8.16b // MODULO - fold into low
1544 stp x6, x7, [x2]
1545 str w9, [x16, #12] // store the updated counter
1546 eor v11.16b, v11.16b, v10.16b // MODULO - fold into low
1547 ext v11.16b, v11.16b, v11.16b, #8
1548 rev64 v11.16b, v11.16b
1549 mov x0, x15
1550 st1 { v11.16b }, [x3]
1551 ldp x19, x20, [sp, #16]
1552 ldp x21, x22, [sp, #32]
1553 ldp x23, x24, [sp, #48]
1554 ldp d8, d9, [sp, #64]
1555 ldp d10, d11, [sp, #80]
1556 ldp d12, d13, [sp, #96]
1557 ldp d14, d15, [sp, #112]
1558 ldp x29, x30, [sp], #128
1559 AARCH64_VALIDATE_LINK_REGISTER
1560 ret
1561
1562 #endif
1563 #endif // !OPENSSL_NO_ASM && defined(__AARCH64EL__) && defined(__APPLE__)
1564 #if defined(__ELF__)
1565 // See https://www.airs.com/blog/archives/518.
1566 .section .note.GNU-stack,"",%progbits
1567 #endif