| 1 |
// Implementation adapted from mbedtls. |
| 2 |
|
| 3 |
use core::arch::{aarch64::*, asm}; |
| 4 |
|
| 5 |
use crate::consts::K64; |
| 6 |
|
| 7 |
cpufeatures::new!(sha3_hwcap, "sha3"); |
| 8 |
|
| 9 |
pub fn compress(state: &mut [u64; 8], blocks: &[[u8; 128]]) { |
| 10 |
// TODO: Replace with https://github.com/rust-lang/rfcs/pull/2725 |
| 11 |
// after stabilization |
| 12 |
if sha3_hwcap::get() { |
| 13 |
unsafe { sha512_compress(state, blocks) } |
| 14 |
} else { |
| 15 |
super::soft::compress(state, blocks); |
| 16 |
} |
| 17 |
} |
| 18 |
|
| 19 |
#[target_feature(enable = "sha3")] |
| 20 |
unsafe fn sha512_compress(state: &mut [u64; 8], blocks: &[[u8; 128]]) { |
| 21 |
// SAFETY: Requires the sha3 feature. |
| 22 |
|
| 23 |
// Load state into vectors. |
| 24 |
let mut ab = vld1q_u64(state[0..2].as_ptr()); |
| 25 |
let mut cd = vld1q_u64(state[2..4].as_ptr()); |
| 26 |
let mut ef = vld1q_u64(state[4..6].as_ptr()); |
| 27 |
let mut gh = vld1q_u64(state[6..8].as_ptr()); |
| 28 |
|
| 29 |
// Iterate through the message blocks. |
| 30 |
for block in blocks { |
| 31 |
// Keep original state values. |
| 32 |
let ab_orig = ab; |
| 33 |
let cd_orig = cd; |
| 34 |
let ef_orig = ef; |
| 35 |
let gh_orig = gh; |
| 36 |
|
| 37 |
// Load the message block into vectors, assuming little endianness. |
| 38 |
let mut s0 = vreinterpretq_u64_u8(vrev64q_u8(vld1q_u8(block[0..16].as_ptr()))); |
| 39 |
let mut s1 = vreinterpretq_u64_u8(vrev64q_u8(vld1q_u8(block[16..32].as_ptr()))); |
| 40 |
let mut s2 = vreinterpretq_u64_u8(vrev64q_u8(vld1q_u8(block[32..48].as_ptr()))); |
| 41 |
let mut s3 = vreinterpretq_u64_u8(vrev64q_u8(vld1q_u8(block[48..64].as_ptr()))); |
| 42 |
let mut s4 = vreinterpretq_u64_u8(vrev64q_u8(vld1q_u8(block[64..80].as_ptr()))); |
| 43 |
let mut s5 = vreinterpretq_u64_u8(vrev64q_u8(vld1q_u8(block[80..96].as_ptr()))); |
| 44 |
let mut s6 = vreinterpretq_u64_u8(vrev64q_u8(vld1q_u8(block[96..112].as_ptr()))); |
| 45 |
let mut s7 = vreinterpretq_u64_u8(vrev64q_u8(vld1q_u8(block[112..128].as_ptr()))); |
| 46 |
|
| 47 |
// Rounds 0 and 1 |
| 48 |
let mut initial_sum = vaddq_u64(s0, vld1q_u64(&K64[0])); |
| 49 |
let mut sum = vaddq_u64(vextq_u64(initial_sum, initial_sum, 1), gh); |
| 50 |
let mut intermed = vsha512hq_u64(sum, vextq_u64(ef, gh, 1), vextq_u64(cd, ef, 1)); |
| 51 |
gh = vsha512h2q_u64(intermed, cd, ab); |
| 52 |
cd = vaddq_u64(cd, intermed); |
| 53 |
|
| 54 |
// Rounds 2 and 3 |
| 55 |
initial_sum = vaddq_u64(s1, vld1q_u64(&K64[2])); |
| 56 |
sum = vaddq_u64(vextq_u64(initial_sum, initial_sum, 1), ef); |
| 57 |
intermed = vsha512hq_u64(sum, vextq_u64(cd, ef, 1), vextq_u64(ab, cd, 1)); |
| 58 |
ef = vsha512h2q_u64(intermed, ab, gh); |
| 59 |
ab = vaddq_u64(ab, intermed); |
| 60 |
|
| 61 |
// Rounds 4 and 5 |
| 62 |
initial_sum = vaddq_u64(s2, vld1q_u64(&K64[4])); |
| 63 |
sum = vaddq_u64(vextq_u64(initial_sum, initial_sum, 1), cd); |
| 64 |
intermed = vsha512hq_u64(sum, vextq_u64(ab, cd, 1), vextq_u64(gh, ab, 1)); |
| 65 |
cd = vsha512h2q_u64(intermed, gh, ef); |
| 66 |
gh = vaddq_u64(gh, intermed); |
| 67 |
|
| 68 |
// Rounds 6 and 7 |
| 69 |
initial_sum = vaddq_u64(s3, vld1q_u64(&K64[6])); |
| 70 |
sum = vaddq_u64(vextq_u64(initial_sum, initial_sum, 1), ab); |
| 71 |
intermed = vsha512hq_u64(sum, vextq_u64(gh, ab, 1), vextq_u64(ef, gh, 1)); |
| 72 |
ab = vsha512h2q_u64(intermed, ef, cd); |
| 73 |
ef = vaddq_u64(ef, intermed); |
| 74 |
|
| 75 |
// Rounds 8 and 9 |
| 76 |
initial_sum = vaddq_u64(s4, vld1q_u64(&K64[8])); |
| 77 |
sum = vaddq_u64(vextq_u64(initial_sum, initial_sum, 1), gh); |
| 78 |
intermed = vsha512hq_u64(sum, vextq_u64(ef, gh, 1), vextq_u64(cd, ef, 1)); |
| 79 |
gh = vsha512h2q_u64(intermed, cd, ab); |
| 80 |
cd = vaddq_u64(cd, intermed); |
| 81 |
|
| 82 |
// Rounds 10 and 11 |
| 83 |
initial_sum = vaddq_u64(s5, vld1q_u64(&K64[10])); |
| 84 |
sum = vaddq_u64(vextq_u64(initial_sum, initial_sum, 1), ef); |
| 85 |
intermed = vsha512hq_u64(sum, vextq_u64(cd, ef, 1), vextq_u64(ab, cd, 1)); |
| 86 |
ef = vsha512h2q_u64(intermed, ab, gh); |
| 87 |
ab = vaddq_u64(ab, intermed); |
| 88 |
|
| 89 |
// Rounds 12 and 13 |
| 90 |
initial_sum = vaddq_u64(s6, vld1q_u64(&K64[12])); |
| 91 |
sum = vaddq_u64(vextq_u64(initial_sum, initial_sum, 1), cd); |
| 92 |
intermed = vsha512hq_u64(sum, vextq_u64(ab, cd, 1), vextq_u64(gh, ab, 1)); |
| 93 |
cd = vsha512h2q_u64(intermed, gh, ef); |
| 94 |
gh = vaddq_u64(gh, intermed); |
| 95 |
|
| 96 |
// Rounds 14 and 15 |
| 97 |
initial_sum = vaddq_u64(s7, vld1q_u64(&K64[14])); |
| 98 |
sum = vaddq_u64(vextq_u64(initial_sum, initial_sum, 1), ab); |
| 99 |
intermed = vsha512hq_u64(sum, vextq_u64(gh, ab, 1), vextq_u64(ef, gh, 1)); |
| 100 |
ab = vsha512h2q_u64(intermed, ef, cd); |
| 101 |
ef = vaddq_u64(ef, intermed); |
| 102 |
|
| 103 |
for t in (16..80).step_by(16) { |
| 104 |
// Rounds t and t + 1 |
| 105 |
s0 = vsha512su1q_u64(vsha512su0q_u64(s0, s1), s7, vextq_u64(s4, s5, 1)); |
| 106 |
initial_sum = vaddq_u64(s0, vld1q_u64(&K64[t])); |
| 107 |
sum = vaddq_u64(vextq_u64(initial_sum, initial_sum, 1), gh); |
| 108 |
intermed = vsha512hq_u64(sum, vextq_u64(ef, gh, 1), vextq_u64(cd, ef, 1)); |
| 109 |
gh = vsha512h2q_u64(intermed, cd, ab); |
| 110 |
cd = vaddq_u64(cd, intermed); |
| 111 |
|
| 112 |
// Rounds t + 2 and t + 3 |
| 113 |
s1 = vsha512su1q_u64(vsha512su0q_u64(s1, s2), s0, vextq_u64(s5, s6, 1)); |
| 114 |
initial_sum = vaddq_u64(s1, vld1q_u64(&K64[t + 2])); |
| 115 |
sum = vaddq_u64(vextq_u64(initial_sum, initial_sum, 1), ef); |
| 116 |
intermed = vsha512hq_u64(sum, vextq_u64(cd, ef, 1), vextq_u64(ab, cd, 1)); |
| 117 |
ef = vsha512h2q_u64(intermed, ab, gh); |
| 118 |
ab = vaddq_u64(ab, intermed); |
| 119 |
|
| 120 |
// Rounds t + 4 and t + 5 |
| 121 |
s2 = vsha512su1q_u64(vsha512su0q_u64(s2, s3), s1, vextq_u64(s6, s7, 1)); |
| 122 |
initial_sum = vaddq_u64(s2, vld1q_u64(&K64[t + 4])); |
| 123 |
sum = vaddq_u64(vextq_u64(initial_sum, initial_sum, 1), cd); |
| 124 |
intermed = vsha512hq_u64(sum, vextq_u64(ab, cd, 1), vextq_u64(gh, ab, 1)); |
| 125 |
cd = vsha512h2q_u64(intermed, gh, ef); |
| 126 |
gh = vaddq_u64(gh, intermed); |
| 127 |
|
| 128 |
// Rounds t + 6 and t + 7 |
| 129 |
s3 = vsha512su1q_u64(vsha512su0q_u64(s3, s4), s2, vextq_u64(s7, s0, 1)); |
| 130 |
initial_sum = vaddq_u64(s3, vld1q_u64(&K64[t + 6])); |
| 131 |
sum = vaddq_u64(vextq_u64(initial_sum, initial_sum, 1), ab); |
| 132 |
intermed = vsha512hq_u64(sum, vextq_u64(gh, ab, 1), vextq_u64(ef, gh, 1)); |
| 133 |
ab = vsha512h2q_u64(intermed, ef, cd); |
| 134 |
ef = vaddq_u64(ef, intermed); |
| 135 |
|
| 136 |
// Rounds t + 8 and t + 9 |
| 137 |
s4 = vsha512su1q_u64(vsha512su0q_u64(s4, s5), s3, vextq_u64(s0, s1, 1)); |
| 138 |
initial_sum = vaddq_u64(s4, vld1q_u64(&K64[t + 8])); |
| 139 |
sum = vaddq_u64(vextq_u64(initial_sum, initial_sum, 1), gh); |
| 140 |
intermed = vsha512hq_u64(sum, vextq_u64(ef, gh, 1), vextq_u64(cd, ef, 1)); |
| 141 |
gh = vsha512h2q_u64(intermed, cd, ab); |
| 142 |
cd = vaddq_u64(cd, intermed); |
| 143 |
|
| 144 |
// Rounds t + 10 and t + 11 |
| 145 |
s5 = vsha512su1q_u64(vsha512su0q_u64(s5, s6), s4, vextq_u64(s1, s2, 1)); |
| 146 |
initial_sum = vaddq_u64(s5, vld1q_u64(&K64[t + 10])); |
| 147 |
sum = vaddq_u64(vextq_u64(initial_sum, initial_sum, 1), ef); |
| 148 |
intermed = vsha512hq_u64(sum, vextq_u64(cd, ef, 1), vextq_u64(ab, cd, 1)); |
| 149 |
ef = vsha512h2q_u64(intermed, ab, gh); |
| 150 |
ab = vaddq_u64(ab, intermed); |
| 151 |
|
| 152 |
// Rounds t + 12 and t + 13 |
| 153 |
s6 = vsha512su1q_u64(vsha512su0q_u64(s6, s7), s5, vextq_u64(s2, s3, 1)); |
| 154 |
initial_sum = vaddq_u64(s6, vld1q_u64(&K64[t + 12])); |
| 155 |
sum = vaddq_u64(vextq_u64(initial_sum, initial_sum, 1), cd); |
| 156 |
intermed = vsha512hq_u64(sum, vextq_u64(ab, cd, 1), vextq_u64(gh, ab, 1)); |
| 157 |
cd = vsha512h2q_u64(intermed, gh, ef); |
| 158 |
gh = vaddq_u64(gh, intermed); |
| 159 |
|
| 160 |
// Rounds t + 14 and t + 15 |
| 161 |
s7 = vsha512su1q_u64(vsha512su0q_u64(s7, s0), s6, vextq_u64(s3, s4, 1)); |
| 162 |
initial_sum = vaddq_u64(s7, vld1q_u64(&K64[t + 14])); |
| 163 |
sum = vaddq_u64(vextq_u64(initial_sum, initial_sum, 1), ab); |
| 164 |
intermed = vsha512hq_u64(sum, vextq_u64(gh, ab, 1), vextq_u64(ef, gh, 1)); |
| 165 |
ab = vsha512h2q_u64(intermed, ef, cd); |
| 166 |
ef = vaddq_u64(ef, intermed); |
| 167 |
} |
| 168 |
|
| 169 |
// Add the block-specific state to the original state. |
| 170 |
ab = vaddq_u64(ab, ab_orig); |
| 171 |
cd = vaddq_u64(cd, cd_orig); |
| 172 |
ef = vaddq_u64(ef, ef_orig); |
| 173 |
gh = vaddq_u64(gh, gh_orig); |
| 174 |
} |
| 175 |
|
| 176 |
// Store vectors into state. |
| 177 |
vst1q_u64(state[0..2].as_mut_ptr(), ab); |
| 178 |
vst1q_u64(state[2..4].as_mut_ptr(), cd); |
| 179 |
vst1q_u64(state[4..6].as_mut_ptr(), ef); |
| 180 |
vst1q_u64(state[6..8].as_mut_ptr(), gh); |
| 181 |
} |
| 182 |
|
| 183 |
// TODO remove these polyfills once SHA3 intrinsics land |
| 184 |
|
| 185 |
#[inline(always)] |
| 186 |
unsafe fn vsha512hq_u64( |
| 187 |
mut hash_ed: uint64x2_t, |
| 188 |
hash_gf: uint64x2_t, |
| 189 |
kwh_kwh2: uint64x2_t, |
| 190 |
) -> uint64x2_t { |
| 191 |
asm!( |
| 192 |
"SHA512H {:q}, {:q}, {:v}.2D", |
| 193 |
inout(vreg) hash_ed, in(vreg) hash_gf, in(vreg) kwh_kwh2, |
| 194 |
options(pure, nomem, nostack, preserves_flags) |
| 195 |
); |
| 196 |
hash_ed |
| 197 |
} |
| 198 |
|
| 199 |
#[inline(always)] |
| 200 |
unsafe fn vsha512h2q_u64( |
| 201 |
mut sum_ab: uint64x2_t, |
| 202 |
hash_c_: uint64x2_t, |
| 203 |
hash_ab: uint64x2_t, |
| 204 |
) -> uint64x2_t { |
| 205 |
asm!( |
| 206 |
"SHA512H2 {:q}, {:q}, {:v}.2D", |
| 207 |
inout(vreg) sum_ab, in(vreg) hash_c_, in(vreg) hash_ab, |
| 208 |
options(pure, nomem, nostack, preserves_flags) |
| 209 |
); |
| 210 |
sum_ab |
| 211 |
} |
| 212 |
|
| 213 |
#[inline(always)] |
| 214 |
unsafe fn vsha512su0q_u64(mut w0_1: uint64x2_t, w2_: uint64x2_t) -> uint64x2_t { |
| 215 |
asm!( |
| 216 |
"SHA512SU0 {:v}.2D, {:v}.2D", |
| 217 |
inout(vreg) w0_1, in(vreg) w2_, |
| 218 |
options(pure, nomem, nostack, preserves_flags) |
| 219 |
); |
| 220 |
w0_1 |
| 221 |
} |
| 222 |
|
| 223 |
#[inline(always)] |
| 224 |
unsafe fn vsha512su1q_u64( |
| 225 |
mut s01_s02: uint64x2_t, |
| 226 |
w14_15: uint64x2_t, |
| 227 |
w9_10: uint64x2_t, |
| 228 |
) -> uint64x2_t { |
| 229 |
asm!( |
| 230 |
"SHA512SU1 {:v}.2D, {:v}.2D, {:v}.2D", |
| 231 |
inout(vreg) s01_s02, in(vreg) w14_15, in(vreg) w9_10, |
| 232 |
options(pure, nomem, nostack, preserves_flags) |
| 233 |
); |
| 234 |
s01_s02 |
| 235 |
} |