Debian Codegraph @

external/boringssl/crypto/modes/gcm.c

raw
1 /* ====================================================================
2 * Copyright (c) 2008 The OpenSSL Project. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 *
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 *
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in
13 * the documentation and/or other materials provided with the
14 * distribution.
15 *
16 * 3. All advertising materials mentioning features or use of this
17 * software must display the following acknowledgment:
18 * "This product includes software developed by the OpenSSL Project
19 * for use in the OpenSSL Toolkit. (http://www.openssl.org/)"
20 *
21 * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
22 * endorse or promote products derived from this software without
23 * prior written permission. For written permission, please contact
24 * [email protected].
25 *
26 * 5. Products derived from this software may not be called "OpenSSL"
27 * nor may "OpenSSL" appear in their names without prior written
28 * permission of the OpenSSL Project.
29 *
30 * 6. Redistributions of any form whatsoever must retain the following
31 * acknowledgment:
32 * "This product includes software developed by the OpenSSL Project
33 * for use in the OpenSSL Toolkit (http://www.openssl.org/)"
34 *
35 * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
36 * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
37 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
38 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR
39 * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
40 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
41 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
42 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
43 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
44 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
45 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
46 * OF THE POSSIBILITY OF SUCH DAMAGE.
47 * ==================================================================== */
48
49 #include <openssl/base.h>
50
51 #include <assert.h>
52 #include <string.h>
53
54 #include <openssl/mem.h>
55 #include <openssl/cpu.h>
56
57 #include "internal.h"
58 #include "../internal.h"
59
60
61 #if !defined(OPENSSL_NO_ASM) && \
62 (defined(OPENSSL_X86) || defined(OPENSSL_X86_64) || \
63 defined(OPENSSL_ARM) || defined(OPENSSL_AARCH64))
64 #define GHASH_ASM
65 #endif
66
67 #if defined(BSWAP4) && STRICT_ALIGNMENT == 1
68 /* redefine, because alignment is ensured */
69 #undef GETU32
70 #define GETU32(p) BSWAP4(*(const uint32_t *)(p))
71 #undef PUTU32
72 #define PUTU32(p, v) *(uint32_t *)(p) = BSWAP4(v)
73 #endif
74
75 #define PACK(s) ((size_t)(s) << (sizeof(size_t) * 8 - 16))
76 #define REDUCE1BIT(V) \
77 do { \
78 if (sizeof(size_t) == 8) { \
79 uint64_t T = UINT64_C(0xe100000000000000) & (0 - (V.lo & 1)); \
80 V.lo = (V.hi << 63) | (V.lo >> 1); \
81 V.hi = (V.hi >> 1) ^ T; \
82 } else { \
83 uint32_t T = 0xe1000000U & (0 - (uint32_t)(V.lo & 1)); \
84 V.lo = (V.hi << 63) | (V.lo >> 1); \
85 V.hi = (V.hi >> 1) ^ ((uint64_t)T << 32); \
86 } \
87 } while (0)
88
89 // kSizeTWithoutLower4Bits is a mask that can be used to zero the lower four
90 // bits of a |size_t|.
91 static const size_t kSizeTWithoutLower4Bits = (size_t) -16;
92
93 static void gcm_init_4bit(u128 Htable[16], uint64_t H[2]) {
94 u128 V;
95
96 Htable[0].hi = 0;
97 Htable[0].lo = 0;
98 V.hi = H[0];
99 V.lo = H[1];
100
101 Htable[8] = V;
102 REDUCE1BIT(V);
103 Htable[4] = V;
104 REDUCE1BIT(V);
105 Htable[2] = V;
106 REDUCE1BIT(V);
107 Htable[1] = V;
108 Htable[3].hi = V.hi ^ Htable[2].hi, Htable[3].lo = V.lo ^ Htable[2].lo;
109 V = Htable[4];
110 Htable[5].hi = V.hi ^ Htable[1].hi, Htable[5].lo = V.lo ^ Htable[1].lo;
111 Htable[6].hi = V.hi ^ Htable[2].hi, Htable[6].lo = V.lo ^ Htable[2].lo;
112 Htable[7].hi = V.hi ^ Htable[3].hi, Htable[7].lo = V.lo ^ Htable[3].lo;
113 V = Htable[8];
114 Htable[9].hi = V.hi ^ Htable[1].hi, Htable[9].lo = V.lo ^ Htable[1].lo;
115 Htable[10].hi = V.hi ^ Htable[2].hi, Htable[10].lo = V.lo ^ Htable[2].lo;
116 Htable[11].hi = V.hi ^ Htable[3].hi, Htable[11].lo = V.lo ^ Htable[3].lo;
117 Htable[12].hi = V.hi ^ Htable[4].hi, Htable[12].lo = V.lo ^ Htable[4].lo;
118 Htable[13].hi = V.hi ^ Htable[5].hi, Htable[13].lo = V.lo ^ Htable[5].lo;
119 Htable[14].hi = V.hi ^ Htable[6].hi, Htable[14].lo = V.lo ^ Htable[6].lo;
120 Htable[15].hi = V.hi ^ Htable[7].hi, Htable[15].lo = V.lo ^ Htable[7].lo;
121
122 #if defined(GHASH_ASM) && defined(OPENSSL_ARM)
123 /* ARM assembler expects specific dword order in Htable. */
124 {
125 int j;
126 const union {
127 long one;
128 char little;
129 } is_endian = {1};
130
131 if (is_endian.little) {
132 for (j = 0; j < 16; ++j) {
133 V = Htable[j];
134 Htable[j].hi = V.lo;
135 Htable[j].lo = V.hi;
136 }
137 } else {
138 for (j = 0; j < 16; ++j) {
139 V = Htable[j];
140 Htable[j].hi = V.lo << 32 | V.lo >> 32;
141 Htable[j].lo = V.hi << 32 | V.hi >> 32;
142 }
143 }
144 }
145 #endif
146 }
147
148 #if !defined(GHASH_ASM) || defined(OPENSSL_AARCH64)
149 static const size_t rem_4bit[16] = {
150 PACK(0x0000), PACK(0x1C20), PACK(0x3840), PACK(0x2460),
151 PACK(0x7080), PACK(0x6CA0), PACK(0x48C0), PACK(0x54E0),
152 PACK(0xE100), PACK(0xFD20), PACK(0xD940), PACK(0xC560),
153 PACK(0x9180), PACK(0x8DA0), PACK(0xA9C0), PACK(0xB5E0)};
154
155 static void gcm_gmult_4bit(uint64_t Xi[2], const u128 Htable[16]) {
156 u128 Z;
157 int cnt = 15;
158 size_t rem, nlo, nhi;
159 const union {
160 long one;
161 char little;
162 } is_endian = {1};
163
164 nlo = ((const uint8_t *)Xi)[15];
165 nhi = nlo >> 4;
166 nlo &= 0xf;
167
168 Z.hi = Htable[nlo].hi;
169 Z.lo = Htable[nlo].lo;
170
171 while (1) {
172 rem = (size_t)Z.lo & 0xf;
173 Z.lo = (Z.hi << 60) | (Z.lo >> 4);
174 Z.hi = (Z.hi >> 4);
175 if (sizeof(size_t) == 8) {
176 Z.hi ^= rem_4bit[rem];
177 } else {
178 Z.hi ^= (uint64_t)rem_4bit[rem] << 32;
179 }
180
181 Z.hi ^= Htable[nhi].hi;
182 Z.lo ^= Htable[nhi].lo;
183
184 if (--cnt < 0) {
185 break;
186 }
187
188 nlo = ((const uint8_t *)Xi)[cnt];
189 nhi = nlo >> 4;
190 nlo &= 0xf;
191
192 rem = (size_t)Z.lo & 0xf;
193 Z.lo = (Z.hi << 60) | (Z.lo >> 4);
194 Z.hi = (Z.hi >> 4);
195 if (sizeof(size_t) == 8) {
196 Z.hi ^= rem_4bit[rem];
197 } else {
198 Z.hi ^= (uint64_t)rem_4bit[rem] << 32;
199 }
200
201 Z.hi ^= Htable[nlo].hi;
202 Z.lo ^= Htable[nlo].lo;
203 }
204
205 if (is_endian.little) {
206 #ifdef BSWAP8
207 Xi[0] = BSWAP8(Z.hi);
208 Xi[1] = BSWAP8(Z.lo);
209 #else
210 uint8_t *p = (uint8_t *)Xi;
211 uint32_t v;
212 v = (uint32_t)(Z.hi >> 32);
213 PUTU32(p, v);
214 v = (uint32_t)(Z.hi);
215 PUTU32(p + 4, v);
216 v = (uint32_t)(Z.lo >> 32);
217 PUTU32(p + 8, v);
218 v = (uint32_t)(Z.lo);
219 PUTU32(p + 12, v);
220 #endif
221 } else {
222 Xi[0] = Z.hi;
223 Xi[1] = Z.lo;
224 }
225 }
226
227 /* Streamed gcm_mult_4bit, see CRYPTO_gcm128_[en|de]crypt for
228 * details... Compiler-generated code doesn't seem to give any
229 * performance improvement, at least not on x86[_64]. It's here
230 * mostly as reference and a placeholder for possible future
231 * non-trivial optimization[s]... */
232 static void gcm_ghash_4bit(uint64_t Xi[2], const u128 Htable[16], const uint8_t *inp,
233 size_t len) {
234 u128 Z;
235 int cnt;
236 size_t rem, nlo, nhi;
237 const union {
238 long one;
239 char little;
240 } is_endian = {1};
241
242 do {
243 cnt = 15;
244 nlo = ((const uint8_t *)Xi)[15];
245 nlo ^= inp[15];
246 nhi = nlo >> 4;
247 nlo &= 0xf;
248
249 Z.hi = Htable[nlo].hi;
250 Z.lo = Htable[nlo].lo;
251
252 while (1) {
253 rem = (size_t)Z.lo & 0xf;
254 Z.lo = (Z.hi << 60) | (Z.lo >> 4);
255 Z.hi = (Z.hi >> 4);
256 if (sizeof(size_t) == 8) {
257 Z.hi ^= rem_4bit[rem];
258 } else {
259 Z.hi ^= (uint64_t)rem_4bit[rem] << 32;
260 }
261
262 Z.hi ^= Htable[nhi].hi;
263 Z.lo ^= Htable[nhi].lo;
264
265 if (--cnt < 0) {
266 break;
267 }
268
269 nlo = ((const uint8_t *)Xi)[cnt];
270 nlo ^= inp[cnt];
271 nhi = nlo >> 4;
272 nlo &= 0xf;
273
274 rem = (size_t)Z.lo & 0xf;
275 Z.lo = (Z.hi << 60) | (Z.lo >> 4);
276 Z.hi = (Z.hi >> 4);
277 if (sizeof(size_t) == 8) {
278 Z.hi ^= rem_4bit[rem];
279 } else {
280 Z.hi ^= (uint64_t)rem_4bit[rem] << 32;
281 }
282
283 Z.hi ^= Htable[nlo].hi;
284 Z.lo ^= Htable[nlo].lo;
285 }
286
287 if (is_endian.little) {
288 #ifdef BSWAP8
289 Xi[0] = BSWAP8(Z.hi);
290 Xi[1] = BSWAP8(Z.lo);
291 #else
292 uint8_t *p = (uint8_t *)Xi;
293 uint32_t v;
294 v = (uint32_t)(Z.hi >> 32);
295 PUTU32(p, v);
296 v = (uint32_t)(Z.hi);
297 PUTU32(p + 4, v);
298 v = (uint32_t)(Z.lo >> 32);
299 PUTU32(p + 8, v);
300 v = (uint32_t)(Z.lo);
301 PUTU32(p + 12, v);
302 #endif
303 } else {
304 Xi[0] = Z.hi;
305 Xi[1] = Z.lo;
306 }
307 } while (inp += 16, len -= 16);
308 }
309 #else /* GHASH_ASM */
310 void gcm_gmult_4bit(uint64_t Xi[2], const u128 Htable[16]);
311 void gcm_ghash_4bit(uint64_t Xi[2], const u128 Htable[16], const uint8_t *inp,
312 size_t len);
313 #endif
314
315 #define GCM_MUL(ctx, Xi) gcm_gmult_4bit(ctx->Xi.u, ctx->Htable)
316 #if defined(GHASH_ASM)
317 #define GHASH(ctx, in, len) gcm_ghash_4bit((ctx)->Xi.u, (ctx)->Htable, in, len)
318 /* GHASH_CHUNK is "stride parameter" missioned to mitigate cache
319 * trashing effect. In other words idea is to hash data while it's
320 * still in L1 cache after encryption pass... */
321 #define GHASH_CHUNK (3 * 1024)
322 #endif
323
324
325 #if defined(GHASH_ASM)
326 #if defined(OPENSSL_X86) || defined(OPENSSL_X86_64)
327 #define GHASH_ASM_X86_OR_64
328 #define GCM_FUNCREF_4BIT
329 void gcm_init_clmul(u128 Htable[16], const uint64_t Xi[2]);
330 void gcm_gmult_clmul(uint64_t Xi[2], const u128 Htable[16]);
331 void gcm_ghash_clmul(uint64_t Xi[2], const u128 Htable[16], const uint8_t *inp,
332 size_t len);
333
334 #if defined(OPENSSL_X86)
335 #define gcm_init_avx gcm_init_clmul
336 #define gcm_gmult_avx gcm_gmult_clmul
337 #define gcm_ghash_avx gcm_ghash_clmul
338 #else
339 void gcm_init_avx(u128 Htable[16], const uint64_t Xi[2]);
340 void gcm_gmult_avx(uint64_t Xi[2], const u128 Htable[16]);
341 void gcm_ghash_avx(uint64_t Xi[2], const u128 Htable[16], const uint8_t *in,
342 size_t len);
343 #define AESNI_GCM
344 static int aesni_gcm_enabled(GCM128_CONTEXT *ctx, ctr128_f stream) {
345 return stream == aesni_ctr32_encrypt_blocks &&
346 ctx->ghash == gcm_ghash_avx;
347 }
348
349 size_t aesni_gcm_encrypt(const uint8_t *in, uint8_t *out, size_t len,
350 const void *key, uint8_t ivec[16], uint64_t *Xi);
351 size_t aesni_gcm_decrypt(const uint8_t *in, uint8_t *out, size_t len,
352 const void *key, uint8_t ivec[16], uint64_t *Xi);
353 #endif
354
355 #if defined(OPENSSL_X86)
356 #define GHASH_ASM_X86
357 void gcm_gmult_4bit_mmx(uint64_t Xi[2], const u128 Htable[16]);
358 void gcm_ghash_4bit_mmx(uint64_t Xi[2], const u128 Htable[16], const uint8_t *inp,
359 size_t len);
360
361 void gcm_gmult_4bit_x86(uint64_t Xi[2], const u128 Htable[16]);
362 void gcm_ghash_4bit_x86(uint64_t Xi[2], const u128 Htable[16], const uint8_t *inp,
363 size_t len);
364 #endif
365 #elif defined(OPENSSL_ARM) || defined(OPENSSL_AARCH64)
366 #include <openssl/arm_arch.h>
367 #if __ARM_ARCH__ >= 7
368 #define GHASH_ASM_ARM
369 #define GCM_FUNCREF_4BIT
370
371 static int pmull_capable(void) {
372 return CRYPTO_is_ARMv8_PMULL_capable();
373 }
374
375 void gcm_init_v8(u128 Htable[16], const uint64_t Xi[2]);
376 void gcm_gmult_v8(uint64_t Xi[2], const u128 Htable[16]);
377 void gcm_ghash_v8(uint64_t Xi[2], const u128 Htable[16], const uint8_t *inp,
378 size_t len);
379
380 #if defined(OPENSSL_ARM)
381 /* 32-bit ARM also has support for doing GCM with NEON instructions. */
382 static int neon_capable(void) {
383 return CRYPTO_is_NEON_capable();
384 }
385
386 void gcm_init_neon(u128 Htable[16], const uint64_t Xi[2]);
387 void gcm_gmult_neon(uint64_t Xi[2], const u128 Htable[16]);
388 void gcm_ghash_neon(uint64_t Xi[2], const u128 Htable[16], const uint8_t *inp,
389 size_t len);
390 #else
391 /* AArch64 only has the ARMv8 versions of functions. */
392 static int neon_capable(void) {
393 return 0;
394 }
395 static void gcm_init_neon(u128 Htable[16], const uint64_t Xi[2]) {
396 abort();
397 }
398 static void gcm_gmult_neon(uint64_t Xi[2], const u128 Htable[16]) {
399 abort();
400 }
401 static void gcm_ghash_neon(uint64_t Xi[2], const u128 Htable[16],
402 const uint8_t *inp, size_t len) {
403 abort();
404 }
405 #endif
406
407 #endif
408 #endif
409 #endif
410
411 #ifdef GCM_FUNCREF_4BIT
412 #undef GCM_MUL
413 #define GCM_MUL(ctx, Xi) (*gcm_gmult_p)(ctx->Xi.u, ctx->Htable)
414 #ifdef GHASH
415 #undef GHASH
416 #define GHASH(ctx, in, len) (*gcm_ghash_p)(ctx->Xi.u, ctx->Htable, in, len)
417 #endif
418 #endif
419
420 void CRYPTO_gcm128_init(GCM128_CONTEXT *ctx, const void *key,
421 block128_f block) {
422 const union {
423 long one;
424 char little;
425 } is_endian = {1};
426
427 memset(ctx, 0, sizeof(*ctx));
428 ctx->block = block;
429
430 (*block)(ctx->H.c, ctx->H.c, key);
431
432 if (is_endian.little) {
433 /* H is stored in host byte order */
434 #ifdef BSWAP8
435 ctx->H.u[0] = BSWAP8(ctx->H.u[0]);
436 ctx->H.u[1] = BSWAP8(ctx->H.u[1]);
437 #else
438 uint8_t *p = ctx->H.c;
439 uint64_t hi, lo;
440 hi = (uint64_t)GETU32(p) << 32 | GETU32(p + 4);
441 lo = (uint64_t)GETU32(p + 8) << 32 | GETU32(p + 12);
442 ctx->H.u[0] = hi;
443 ctx->H.u[1] = lo;
444 #endif
445 }
446
447 #if defined(GHASH_ASM_X86_OR_64)
448 if (crypto_gcm_clmul_enabled()) {
449 if (((OPENSSL_ia32cap_P[1] >> 22) & 0x41) == 0x41) { /* AVX+MOVBE */
450 gcm_init_avx(ctx->Htable, ctx->H.u);
451 ctx->gmult = gcm_gmult_avx;
452 ctx->ghash = gcm_ghash_avx;
453 } else {
454 gcm_init_clmul(ctx->Htable, ctx->H.u);
455 ctx->gmult = gcm_gmult_clmul;
456 ctx->ghash = gcm_ghash_clmul;
457 }
458 return;
459 }
460 gcm_init_4bit(ctx->Htable, ctx->H.u);
461 #if defined(GHASH_ASM_X86) /* x86 only */
462 if (OPENSSL_ia32cap_P[0] & (1 << 25)) { /* check SSE bit */
463 ctx->gmult = gcm_gmult_4bit_mmx;
464 ctx->ghash = gcm_ghash_4bit_mmx;
465 } else {
466 ctx->gmult = gcm_gmult_4bit_x86;
467 ctx->ghash = gcm_ghash_4bit_x86;
468 }
469 #else
470 ctx->gmult = gcm_gmult_4bit;
471 ctx->ghash = gcm_ghash_4bit;
472 #endif
473 #elif defined(GHASH_ASM_ARM)
474 if (pmull_capable()) {
475 gcm_init_v8(ctx->Htable, ctx->H.u);
476 ctx->gmult = gcm_gmult_v8;
477 ctx->ghash = gcm_ghash_v8;
478 } else if (neon_capable()) {
479 gcm_init_neon(ctx->Htable,ctx->H.u);
480 ctx->gmult = gcm_gmult_neon;
481 ctx->ghash = gcm_ghash_neon;
482 } else {
483 gcm_init_4bit(ctx->Htable, ctx->H.u);
484 ctx->gmult = gcm_gmult_4bit;
485 ctx->ghash = gcm_ghash_4bit;
486 }
487 #else
488 gcm_init_4bit(ctx->Htable, ctx->H.u);
489 ctx->gmult = gcm_gmult_4bit;
490 ctx->ghash = gcm_ghash_4bit;
491 #endif
492 }
493
494 void CRYPTO_gcm128_setiv(GCM128_CONTEXT *ctx, const void *key,
495 const uint8_t *iv, size_t len) {
496 const union {
497 long one;
498 char little;
499 } is_endian = {1};
500 unsigned int ctr;
501 #ifdef GCM_FUNCREF_4BIT
502 void (*gcm_gmult_p)(uint64_t Xi[2], const u128 Htable[16]) = ctx->gmult;
503 #endif
504
505 ctx->Yi.u[0] = 0;
506 ctx->Yi.u[1] = 0;
507 ctx->Xi.u[0] = 0;
508 ctx->Xi.u[1] = 0;
509 ctx->len.u[0] = 0; /* AAD length */
510 ctx->len.u[1] = 0; /* message length */
511 ctx->ares = 0;
512 ctx->mres = 0;
513
514 if (len == 12) {
515 memcpy(ctx->Yi.c, iv, 12);
516 ctx->Yi.c[15] = 1;
517 ctr = 1;
518 } else {
519 size_t i;
520 uint64_t len0 = len;
521
522 while (len >= 16) {
523 for (i = 0; i < 16; ++i) {
524 ctx->Yi.c[i] ^= iv[i];
525 }
526 GCM_MUL(ctx, Yi);
527 iv += 16;
528 len -= 16;
529 }
530 if (len) {
531 for (i = 0; i < len; ++i) {
532 ctx->Yi.c[i] ^= iv[i];
533 }
534 GCM_MUL(ctx, Yi);
535 }
536 len0 <<= 3;
537 if (is_endian.little) {
538 #ifdef BSWAP8
539 ctx->Yi.u[1] ^= BSWAP8(len0);
540 #else
541 ctx->Yi.c[8] ^= (uint8_t)(len0 >> 56);
542 ctx->Yi.c[9] ^= (uint8_t)(len0 >> 48);
543 ctx->Yi.c[10] ^= (uint8_t)(len0 >> 40);
544 ctx->Yi.c[11] ^= (uint8_t)(len0 >> 32);
545 ctx->Yi.c[12] ^= (uint8_t)(len0 >> 24);
546 ctx->Yi.c[13] ^= (uint8_t)(len0 >> 16);
547 ctx->Yi.c[14] ^= (uint8_t)(len0 >> 8);
548 ctx->Yi.c[15] ^= (uint8_t)(len0);
549 #endif
550 } else {
551 ctx->Yi.u[1] ^= len0;
552 }
553
554 GCM_MUL(ctx, Yi);
555
556 if (is_endian.little) {
557 ctr = GETU32(ctx->Yi.c + 12);
558 } else {
559 ctr = ctx->Yi.d[3];
560 }
561 }
562
563 (*ctx->block)(ctx->Yi.c, ctx->EK0.c, key);
564 ++ctr;
565 if (is_endian.little) {
566 PUTU32(ctx->Yi.c + 12, ctr);
567 } else {
568 ctx->Yi.d[3] = ctr;
569 }
570 }
571
572 int CRYPTO_gcm128_aad(GCM128_CONTEXT *ctx, const uint8_t *aad, size_t len) {
573 size_t i;
574 unsigned int n;
575 uint64_t alen = ctx->len.u[0];
576 #ifdef GCM_FUNCREF_4BIT
577 void (*gcm_gmult_p)(uint64_t Xi[2], const u128 Htable[16]) = ctx->gmult;
578 #ifdef GHASH
579 void (*gcm_ghash_p)(uint64_t Xi[2], const u128 Htable[16], const uint8_t *inp,
580 size_t len) = ctx->ghash;
581 #endif
582 #endif
583
584 if (ctx->len.u[1]) {
585 return 0;
586 }
587
588 alen += len;
589 if (alen > (UINT64_C(1) << 61) || (sizeof(len) == 8 && alen < len)) {
590 return 0;
591 }
592 ctx->len.u[0] = alen;
593
594 n = ctx->ares;
595 if (n) {
596 while (n && len) {
597 ctx->Xi.c[n] ^= *(aad++);
598 --len;
599 n = (n + 1) % 16;
600 }
601 if (n == 0) {
602 GCM_MUL(ctx, Xi);
603 } else {
604 ctx->ares = n;
605 return 1;
606 }
607 }
608
609 #ifdef GHASH
610 i = len & kSizeTWithoutLower4Bits;
611 if (i != 0) {
612 GHASH(ctx, aad, i);
613 aad += i;
614 len -= i;
615 }
616 #else
617 while (len >= 16) {
618 for (i = 0; i < 16; ++i) {
619 ctx->Xi.c[i] ^= aad[i];
620 }
621 GCM_MUL(ctx, Xi);
622 aad += 16;
623 len -= 16;
624 }
625 #endif
626 if (len) {
627 n = (unsigned int)len;
628 for (i = 0; i < len; ++i) {
629 ctx->Xi.c[i] ^= aad[i];
630 }
631 }
632
633 ctx->ares = n;
634 return 1;
635 }
636
637 int CRYPTO_gcm128_encrypt(GCM128_CONTEXT *ctx, const void *key,
638 const unsigned char *in, unsigned char *out,
639 size_t len) {
640 const union {
641 long one;
642 char little;
643 } is_endian = {1};
644 unsigned int n, ctr;
645 size_t i;
646 uint64_t mlen = ctx->len.u[1];
647 block128_f block = ctx->block;
648 #ifdef GCM_FUNCREF_4BIT
649 void (*gcm_gmult_p)(uint64_t Xi[2], const u128 Htable[16]) = ctx->gmult;
650 #ifdef GHASH
651 void (*gcm_ghash_p)(uint64_t Xi[2], const u128 Htable[16], const uint8_t *inp,
652 size_t len) = ctx->ghash;
653 #endif
654 #endif
655
656 mlen += len;
657 if (mlen > ((UINT64_C(1) << 36) - 32) ||
658 (sizeof(len) == 8 && mlen < len)) {
659 return 0;
660 }
661 ctx->len.u[1] = mlen;
662
663 if (ctx->ares) {
664 /* First call to encrypt finalizes GHASH(AAD) */
665 GCM_MUL(ctx, Xi);
666 ctx->ares = 0;
667 }
668
669 if (is_endian.little) {
670 ctr = GETU32(ctx->Yi.c + 12);
671 } else {
672 ctr = ctx->Yi.d[3];
673 }
674
675 n = ctx->mres;
676 if (n) {
677 while (n && len) {
678 ctx->Xi.c[n] ^= *(out++) = *(in++) ^ ctx->EKi.c[n];
679 --len;
680 n = (n + 1) % 16;
681 }
682 if (n == 0) {
683 GCM_MUL(ctx, Xi);
684 } else {
685 ctx->mres = n;
686 return 1;
687 }
688 }
689 if (STRICT_ALIGNMENT && ((size_t)in | (size_t)out) % sizeof(size_t) != 0) {
690 for (i = 0; i < len; ++i) {
691 if (n == 0) {
692 (*block)(ctx->Yi.c, ctx->EKi.c, key);
693 ++ctr;
694 if (is_endian.little) {
695 PUTU32(ctx->Yi.c + 12, ctr);
696 } else {
697 ctx->Yi.d[3] = ctr;
698 }
699 }
700 ctx->Xi.c[n] ^= out[i] = in[i] ^ ctx->EKi.c[n];
701 n = (n + 1) % 16;
702 if (n == 0) {
703 GCM_MUL(ctx, Xi);
704 }
705 }
706
707 ctx->mres = n;
708 return 1;
709 }
710 #if defined(GHASH) && defined(GHASH_CHUNK)
711 while (len >= GHASH_CHUNK) {
712 size_t j = GHASH_CHUNK;
713
714 while (j) {
715 size_t *out_t = (size_t *)out;
716 const size_t *in_t = (const size_t *)in;
717
718 (*block)(ctx->Yi.c, ctx->EKi.c, key);
719 ++ctr;
720 if (is_endian.little) {
721 PUTU32(ctx->Yi.c + 12, ctr);
722 } else {
723 ctx->Yi.d[3] = ctr;
724 }
725 for (i = 0; i < 16 / sizeof(size_t); ++i) {
726 out_t[i] = in_t[i] ^ ctx->EKi.t[i];
727 }
728 out += 16;
729 in += 16;
730 j -= 16;
731 }
732 GHASH(ctx, out - GHASH_CHUNK, GHASH_CHUNK);
733 len -= GHASH_CHUNK;
734 }
735 if ((i = (len & (size_t) - 16))) {
736 size_t j = i;
737
738 while (len >= 16) {
739 size_t *out_t = (size_t *)out;
740 const size_t *in_t = (const size_t *)in;
741
742 (*block)(ctx->Yi.c, ctx->EKi.c, key);
743 ++ctr;
744 if (is_endian.little) {
745 PUTU32(ctx->Yi.c + 12, ctr);
746 } else {
747 ctx->Yi.d[3] = ctr;
748 }
749 for (i = 0; i < 16 / sizeof(size_t); ++i) {
750 out_t[i] = in_t[i] ^ ctx->EKi.t[i];
751 }
752 out += 16;
753 in += 16;
754 len -= 16;
755 }
756 GHASH(ctx, out - j, j);
757 }
758 #else
759 while (len >= 16) {
760 size_t *out_t = (size_t *)out;
761 const size_t *in_t = (const size_t *)in;
762
763 (*block)(ctx->Yi.c, ctx->EKi.c, key);
764 ++ctr;
765 if (is_endian.little) {
766 PUTU32(ctx->Yi.c + 12, ctr);
767 } else {
768 ctx->Yi.d[3] = ctr;
769 }
770 for (i = 0; i < 16 / sizeof(size_t); ++i) {
771 ctx->Xi.t[i] ^= out_t[i] = in_t[i] ^ ctx->EKi.t[i];
772 }
773 GCM_MUL(ctx, Xi);
774 out += 16;
775 in += 16;
776 len -= 16;
777 }
778 #endif
779 if (len) {
780 (*block)(ctx->Yi.c, ctx->EKi.c, key);
781 ++ctr;
782 if (is_endian.little) {
783 PUTU32(ctx->Yi.c + 12, ctr);
784 } else {
785 ctx->Yi.d[3] = ctr;
786 }
787 while (len--) {
788 ctx->Xi.c[n] ^= out[n] = in[n] ^ ctx->EKi.c[n];
789 ++n;
790 }
791 }
792
793 ctx->mres = n;
794 return 1;
795 }
796
797 int CRYPTO_gcm128_decrypt(GCM128_CONTEXT *ctx, const void *key,
798 const unsigned char *in, unsigned char *out,
799 size_t len) {
800 const union {
801 long one;
802 char little;
803 } is_endian = {1};
804 unsigned int n, ctr;
805 size_t i;
806 uint64_t mlen = ctx->len.u[1];
807 block128_f block = ctx->block;
808 #ifdef GCM_FUNCREF_4BIT
809 void (*gcm_gmult_p)(uint64_t Xi[2], const u128 Htable[16]) = ctx->gmult;
810 #ifdef GHASH
811 void (*gcm_ghash_p)(uint64_t Xi[2], const u128 Htable[16], const uint8_t *inp,
812 size_t len) = ctx->ghash;
813 #endif
814 #endif
815
816 mlen += len;
817 if (mlen > ((UINT64_C(1) << 36) - 32) ||
818 (sizeof(len) == 8 && mlen < len)) {
819 return 0;
820 }
821 ctx->len.u[1] = mlen;
822
823 if (ctx->ares) {
824 /* First call to decrypt finalizes GHASH(AAD) */
825 GCM_MUL(ctx, Xi);
826 ctx->ares = 0;
827 }
828
829 if (is_endian.little) {
830 ctr = GETU32(ctx->Yi.c + 12);
831 } else {
832 ctr = ctx->Yi.d[3];
833 }
834
835 n = ctx->mres;
836 if (n) {
837 while (n && len) {
838 uint8_t c = *(in++);
839 *(out++) = c ^ ctx->EKi.c[n];
840 ctx->Xi.c[n] ^= c;
841 --len;
842 n = (n + 1) % 16;
843 }
844 if (n == 0) {
845 GCM_MUL(ctx, Xi);
846 } else {
847 ctx->mres = n;
848 return 1;
849 }
850 }
851 if (STRICT_ALIGNMENT && ((size_t)in | (size_t)out) % sizeof(size_t) != 0) {
852 for (i = 0; i < len; ++i) {
853 uint8_t c;
854 if (n == 0) {
855 (*block)(ctx->Yi.c, ctx->EKi.c, key);
856 ++ctr;
857 if (is_endian.little) {
858 PUTU32(ctx->Yi.c + 12, ctr);
859 } else {
860 ctx->Yi.d[3] = ctr;
861 }
862 }
863 c = in[i];
864 out[i] = c ^ ctx->EKi.c[n];
865 ctx->Xi.c[n] ^= c;
866 n = (n + 1) % 16;
867 if (n == 0) {
868 GCM_MUL(ctx, Xi);
869 }
870 }
871
872 ctx->mres = n;
873 return 1;
874 }
875 #if defined(GHASH) && defined(GHASH_CHUNK)
876 while (len >= GHASH_CHUNK) {
877 size_t j = GHASH_CHUNK;
878
879 GHASH(ctx, in, GHASH_CHUNK);
880 while (j) {
881 size_t *out_t = (size_t *)out;
882 const size_t *in_t = (const size_t *)in;
883
884 (*block)(ctx->Yi.c, ctx->EKi.c, key);
885 ++ctr;
886 if (is_endian.little) {
887 PUTU32(ctx->Yi.c + 12, ctr);
888 } else {
889 ctx->Yi.d[3] = ctr;
890 }
891 for (i = 0; i < 16 / sizeof(size_t); ++i) {
892 out_t[i] = in_t[i] ^ ctx->EKi.t[i];
893 }
894 out += 16;
895 in += 16;
896 j -= 16;
897 }
898 len -= GHASH_CHUNK;
899 }
900 i = len & kSizeTWithoutLower4Bits;
901 if (i != 0) {
902 GHASH(ctx, in, i);
903 while (len >= 16) {
904 size_t *out_t = (size_t *)out;
905 const size_t *in_t = (const size_t *)in;
906
907 (*block)(ctx->Yi.c, ctx->EKi.c, key);
908 ++ctr;
909 if (is_endian.little) {
910 PUTU32(ctx->Yi.c + 12, ctr);
911 } else {
912 ctx->Yi.d[3] = ctr;
913 }
914 for (i = 0; i < 16 / sizeof(size_t); ++i) {
915 out_t[i] = in_t[i] ^ ctx->EKi.t[i];
916 }
917 out += 16;
918 in += 16;
919 len -= 16;
920 }
921 }
922 #else
923 while (len >= 16) {
924 size_t *out_t = (size_t *)out;
925 const size_t *in_t = (const size_t *)in;
926
927 (*block)(ctx->Yi.c, ctx->EKi.c, key);
928 ++ctr;
929 if (is_endian.little) {
930 PUTU32(ctx->Yi.c + 12, ctr);
931 } else {
932 ctx->Yi.d[3] = ctr;
933 }
934 for (i = 0; i < 16 / sizeof(size_t); ++i) {
935 size_t c = in_t[i];
936 out_t[i] = c ^ ctx->EKi.t[i];
937 ctx->Xi.t[i] ^= c;
938 }
939 GCM_MUL(ctx, Xi);
940 out += 16;
941 in += 16;
942 len -= 16;
943 }
944 #endif
945 if (len) {
946 (*block)(ctx->Yi.c, ctx->EKi.c, key);
947 ++ctr;
948 if (is_endian.little) {
949 PUTU32(ctx->Yi.c + 12, ctr);
950 } else {
951 ctx->Yi.d[3] = ctr;
952 }
953 while (len--) {
954 uint8_t c = in[n];
955 ctx->Xi.c[n] ^= c;
956 out[n] = c ^ ctx->EKi.c[n];
957 ++n;
958 }
959 }
960
961 ctx->mres = n;
962 return 1;
963 }
964
965 int CRYPTO_gcm128_encrypt_ctr32(GCM128_CONTEXT *ctx, const void *key,
966 const uint8_t *in, uint8_t *out, size_t len,
967 ctr128_f stream) {
968 const union {
969 long one;
970 char little;
971 } is_endian = {1};
972 unsigned int n, ctr;
973 uint64_t mlen = ctx->len.u[1];
974 #ifdef GCM_FUNCREF_4BIT
975 void (*gcm_gmult_p)(uint64_t Xi[2], const u128 Htable[16]) = ctx->gmult;
976 #ifdef GHASH
977 void (*gcm_ghash_p)(uint64_t Xi[2], const u128 Htable[16], const uint8_t *inp,
978 size_t len) = ctx->ghash;
979 #endif
980 #endif
981
982 mlen += len;
983 if (mlen > ((UINT64_C(1) << 36) - 32) ||
984 (sizeof(len) == 8 && mlen < len)) {
985 return 0;
986 }
987 ctx->len.u[1] = mlen;
988
989 if (ctx->ares) {
990 /* First call to encrypt finalizes GHASH(AAD) */
991 GCM_MUL(ctx, Xi);
992 ctx->ares = 0;
993 }
994
995 n = ctx->mres;
996 if (n) {
997 while (n && len) {
998 ctx->Xi.c[n] ^= *(out++) = *(in++) ^ ctx->EKi.c[n];
999 --len;
1000 n = (n + 1) % 16;
1001 }
1002 if (n == 0) {
1003 GCM_MUL(ctx, Xi);
1004 } else {
1005 ctx->mres = n;
1006 return 1;
1007 }
1008 }
1009
1010 #if defined(AESNI_GCM)
1011 if (aesni_gcm_enabled(ctx, stream)) {
1012 /* |aesni_gcm_encrypt| may not process all the input given to it. It may
1013 * not process *any* of its input if it is deemed too small. */
1014 size_t bulk = aesni_gcm_encrypt(in, out, len, key, ctx->Yi.c, ctx->Xi.u);
1015 in += bulk;
1016 out += bulk;
1017 len -= bulk;
1018 }
1019 #endif
1020
1021 if (is_endian.little) {
1022 ctr = GETU32(ctx->Yi.c + 12);
1023 } else {
1024 ctr = ctx->Yi.d[3];
1025 }
1026
1027 #if defined(GHASH)
1028 while (len >= GHASH_CHUNK) {
1029 (*stream)(in, out, GHASH_CHUNK / 16, key, ctx->Yi.c);
1030 ctr += GHASH_CHUNK / 16;
1031 if (is_endian.little) {
1032 PUTU32(ctx->Yi.c + 12, ctr);
1033 } else {
1034 ctx->Yi.d[3] = ctr;
1035 }
1036 GHASH(ctx, out, GHASH_CHUNK);
1037 out += GHASH_CHUNK;
1038 in += GHASH_CHUNK;
1039 len -= GHASH_CHUNK;
1040 }
1041 #endif
1042 size_t i = len & kSizeTWithoutLower4Bits;
1043 if (i != 0) {
1044 size_t j = i / 16;
1045
1046 (*stream)(in, out, j, key, ctx->Yi.c);
1047 ctr += (unsigned int)j;
1048 if (is_endian.little) {
1049 PUTU32(ctx->Yi.c + 12, ctr);
1050 } else {
1051 ctx->Yi.d[3] = ctr;
1052 }
1053 in += i;
1054 len -= i;
1055 #if defined(GHASH)
1056 GHASH(ctx, out, i);
1057 out += i;
1058 #else
1059 while (j--) {
1060 for (i = 0; i < 16; ++i) {
1061 ctx->Xi.c[i] ^= out[i];
1062 }
1063 GCM_MUL(ctx, Xi);
1064 out += 16;
1065 }
1066 #endif
1067 }
1068 if (len) {
1069 (*ctx->block)(ctx->Yi.c, ctx->EKi.c, key);
1070 ++ctr;
1071 if (is_endian.little) {
1072 PUTU32(ctx->Yi.c + 12, ctr);
1073 } else {
1074 ctx->Yi.d[3] = ctr;
1075 }
1076 while (len--) {
1077 ctx->Xi.c[n] ^= out[n] = in[n] ^ ctx->EKi.c[n];
1078 ++n;
1079 }
1080 }
1081
1082 ctx->mres = n;
1083 return 1;
1084 }
1085
1086 int CRYPTO_gcm128_decrypt_ctr32(GCM128_CONTEXT *ctx, const void *key,
1087 const uint8_t *in, uint8_t *out, size_t len,
1088 ctr128_f stream) {
1089 const union {
1090 long one;
1091 char little;
1092 } is_endian = {1};
1093 unsigned int n, ctr;
1094 uint64_t mlen = ctx->len.u[1];
1095 #ifdef GCM_FUNCREF_4BIT
1096 void (*gcm_gmult_p)(uint64_t Xi[2], const u128 Htable[16]) = ctx->gmult;
1097 #ifdef GHASH
1098 void (*gcm_ghash_p)(uint64_t Xi[2], const u128 Htable[16], const uint8_t *inp,
1099 size_t len) = ctx->ghash;
1100 #endif
1101 #endif
1102
1103 mlen += len;
1104 if (mlen > ((UINT64_C(1) << 36) - 32) ||
1105 (sizeof(len) == 8 && mlen < len)) {
1106 return 0;
1107 }
1108 ctx->len.u[1] = mlen;
1109
1110 if (ctx->ares) {
1111 /* First call to decrypt finalizes GHASH(AAD) */
1112 GCM_MUL(ctx, Xi);
1113 ctx->ares = 0;
1114 }
1115
1116 n = ctx->mres;
1117 if (n) {
1118 while (n && len) {
1119 uint8_t c = *(in++);
1120 *(out++) = c ^ ctx->EKi.c[n];
1121 ctx->Xi.c[n] ^= c;
1122 --len;
1123 n = (n + 1) % 16;
1124 }
1125 if (n == 0) {
1126 GCM_MUL(ctx, Xi);
1127 } else {
1128 ctx->mres = n;
1129 return 1;
1130 }
1131 }
1132
1133 #if defined(AESNI_GCM)
1134 if (aesni_gcm_enabled(ctx, stream)) {
1135 /* |aesni_gcm_decrypt| may not process all the input given to it. It may
1136 * not process *any* of its input if it is deemed too small. */
1137 size_t bulk = aesni_gcm_decrypt(in, out, len, key, ctx->Yi.c, ctx->Xi.u);
1138 in += bulk;
1139 out += bulk;
1140 len -= bulk;
1141 }
1142 #endif
1143
1144 if (is_endian.little) {
1145 ctr = GETU32(ctx->Yi.c + 12);
1146 } else {
1147 ctr = ctx->Yi.d[3];
1148 }
1149
1150 #if defined(GHASH)
1151 while (len >= GHASH_CHUNK) {
1152 GHASH(ctx, in, GHASH_CHUNK);
1153 (*stream)(in, out, GHASH_CHUNK / 16, key, ctx->Yi.c);
1154 ctr += GHASH_CHUNK / 16;
1155 if (is_endian.little) {
1156 PUTU32(ctx->Yi.c + 12, ctr);
1157 } else {
1158 ctx->Yi.d[3] = ctr;
1159 }
1160 out += GHASH_CHUNK;
1161 in += GHASH_CHUNK;
1162 len -= GHASH_CHUNK;
1163 }
1164 #endif
1165 size_t i = len & kSizeTWithoutLower4Bits;
1166 if (i != 0) {
1167 size_t j = i / 16;
1168
1169 #if defined(GHASH)
1170 GHASH(ctx, in, i);
1171 #else
1172 while (j--) {
1173 size_t k;
1174 for (k = 0; k < 16; ++k) {
1175 ctx->Xi.c[k] ^= in[k];
1176 }
1177 GCM_MUL(ctx, Xi);
1178 in += 16;
1179 }
1180 j = i / 16;
1181 in -= i;
1182 #endif
1183 (*stream)(in, out, j, key, ctx->Yi.c);
1184 ctr += (unsigned int)j;
1185 if (is_endian.little) {
1186 PUTU32(ctx->Yi.c + 12, ctr);
1187 } else {
1188 ctx->Yi.d[3] = ctr;
1189 }
1190 out += i;
1191 in += i;
1192 len -= i;
1193 }
1194 if (len) {
1195 (*ctx->block)(ctx->Yi.c, ctx->EKi.c, key);
1196 ++ctr;
1197 if (is_endian.little) {
1198 PUTU32(ctx->Yi.c + 12, ctr);
1199 } else {
1200 ctx->Yi.d[3] = ctr;
1201 }
1202 while (len--) {
1203 uint8_t c = in[n];
1204 ctx->Xi.c[n] ^= c;
1205 out[n] = c ^ ctx->EKi.c[n];
1206 ++n;
1207 }
1208 }
1209
1210 ctx->mres = n;
1211 return 1;
1212 }
1213
1214 int CRYPTO_gcm128_finish(GCM128_CONTEXT *ctx, const uint8_t *tag, size_t len) {
1215 const union {
1216 long one;
1217 char little;
1218 } is_endian = {1};
1219 uint64_t alen = ctx->len.u[0] << 3;
1220 uint64_t clen = ctx->len.u[1] << 3;
1221 #ifdef GCM_FUNCREF_4BIT
1222 void (*gcm_gmult_p)(uint64_t Xi[2], const u128 Htable[16]) = ctx->gmult;
1223 #endif
1224
1225 if (ctx->mres || ctx->ares) {
1226 GCM_MUL(ctx, Xi);
1227 }
1228
1229 if (is_endian.little) {
1230 #ifdef BSWAP8
1231 alen = BSWAP8(alen);
1232 clen = BSWAP8(clen);
1233 #else
1234 uint8_t *p = ctx->len.c;
1235
1236 ctx->len.u[0] = alen;
1237 ctx->len.u[1] = clen;
1238
1239 alen = (uint64_t)GETU32(p) << 32 | GETU32(p + 4);
1240 clen = (uint64_t)GETU32(p + 8) << 32 | GETU32(p + 12);
1241 #endif
1242 }
1243
1244 ctx->Xi.u[0] ^= alen;
1245 ctx->Xi.u[1] ^= clen;
1246 GCM_MUL(ctx, Xi);
1247
1248 ctx->Xi.u[0] ^= ctx->EK0.u[0];
1249 ctx->Xi.u[1] ^= ctx->EK0.u[1];
1250
1251 if (tag && len <= sizeof(ctx->Xi)) {
1252 return CRYPTO_memcmp(ctx->Xi.c, tag, len) == 0;
1253 } else {
1254 return 0;
1255 }
1256 }
1257
1258 void CRYPTO_gcm128_tag(GCM128_CONTEXT *ctx, unsigned char *tag, size_t len) {
1259 CRYPTO_gcm128_finish(ctx, NULL, 0);
1260 memcpy(tag, ctx->Xi.c, len <= sizeof(ctx->Xi.c) ? len : sizeof(ctx->Xi.c));
1261 }
1262
1263 #if defined(OPENSSL_X86) || defined(OPENSSL_X86_64)
1264 int crypto_gcm_clmul_enabled(void) {
1265 #ifdef GHASH_ASM
1266 return OPENSSL_ia32cap_P[0] & (1 << 24) && /* check FXSR bit */
1267 OPENSSL_ia32cap_P[1] & (1 << 1); /* check PCLMULQDQ bit */
1268 #else
1269 return 0;
1270 #endif
1271 }
1272 #endif