| 1 |
//! SHA-256 `aarch64` backend. |
| 2 |
|
| 3 |
// Implementation adapted from mbedtls. |
| 4 |
|
| 5 |
// TODO: stdarch intrinsics: RustCrypto/hashes#257 |
| 6 |
|
| 7 |
use core::arch::{aarch64::*, asm}; |
| 8 |
|
| 9 |
use crate::consts::K32; |
| 10 |
|
| 11 |
cpufeatures::new!(sha2_hwcap, "sha2"); |
| 12 |
|
| 13 |
pub fn compress(state: &mut [u32; 8], blocks: &[[u8; 64]]) { |
| 14 |
// TODO: Replace with https://github.com/rust-lang/rfcs/pull/2725 |
| 15 |
// after stabilization |
| 16 |
if sha2_hwcap::get() { |
| 17 |
unsafe { sha256_compress(state, blocks) } |
| 18 |
} else { |
| 19 |
super::soft::compress(state, blocks); |
| 20 |
} |
| 21 |
} |
| 22 |
|
| 23 |
#[target_feature(enable = "sha2")] |
| 24 |
unsafe fn sha256_compress(state: &mut [u32; 8], blocks: &[[u8; 64]]) { |
| 25 |
// SAFETY: Requires the sha2 feature. |
| 26 |
|
| 27 |
// Load state into vectors. |
| 28 |
let mut abcd = vld1q_u32(state[0..4].as_ptr()); |
| 29 |
let mut efgh = vld1q_u32(state[4..8].as_ptr()); |
| 30 |
|
| 31 |
// Iterate through the message blocks. |
| 32 |
for block in blocks { |
| 33 |
// Keep original state values. |
| 34 |
let abcd_orig = abcd; |
| 35 |
let efgh_orig = efgh; |
| 36 |
|
| 37 |
// Load the message block into vectors, assuming little endianness. |
| 38 |
let mut s0 = vreinterpretq_u32_u8(vrev32q_u8(vld1q_u8(block[0..16].as_ptr()))); |
| 39 |
let mut s1 = vreinterpretq_u32_u8(vrev32q_u8(vld1q_u8(block[16..32].as_ptr()))); |
| 40 |
let mut s2 = vreinterpretq_u32_u8(vrev32q_u8(vld1q_u8(block[32..48].as_ptr()))); |
| 41 |
let mut s3 = vreinterpretq_u32_u8(vrev32q_u8(vld1q_u8(block[48..64].as_ptr()))); |
| 42 |
|
| 43 |
// Rounds 0 to 3 |
| 44 |
let mut tmp = vaddq_u32(s0, vld1q_u32(&K32[0])); |
| 45 |
let mut abcd_prev = abcd; |
| 46 |
abcd = vsha256hq_u32(abcd_prev, efgh, tmp); |
| 47 |
efgh = vsha256h2q_u32(efgh, abcd_prev, tmp); |
| 48 |
|
| 49 |
// Rounds 4 to 7 |
| 50 |
tmp = vaddq_u32(s1, vld1q_u32(&K32[4])); |
| 51 |
abcd_prev = abcd; |
| 52 |
abcd = vsha256hq_u32(abcd_prev, efgh, tmp); |
| 53 |
efgh = vsha256h2q_u32(efgh, abcd_prev, tmp); |
| 54 |
|
| 55 |
// Rounds 8 to 11 |
| 56 |
tmp = vaddq_u32(s2, vld1q_u32(&K32[8])); |
| 57 |
abcd_prev = abcd; |
| 58 |
abcd = vsha256hq_u32(abcd_prev, efgh, tmp); |
| 59 |
efgh = vsha256h2q_u32(efgh, abcd_prev, tmp); |
| 60 |
|
| 61 |
// Rounds 12 to 15 |
| 62 |
tmp = vaddq_u32(s3, vld1q_u32(&K32[12])); |
| 63 |
abcd_prev = abcd; |
| 64 |
abcd = vsha256hq_u32(abcd_prev, efgh, tmp); |
| 65 |
efgh = vsha256h2q_u32(efgh, abcd_prev, tmp); |
| 66 |
|
| 67 |
for t in (16..64).step_by(16) { |
| 68 |
// Rounds t to t + 3 |
| 69 |
s0 = vsha256su1q_u32(vsha256su0q_u32(s0, s1), s2, s3); |
| 70 |
tmp = vaddq_u32(s0, vld1q_u32(&K32[t])); |
| 71 |
abcd_prev = abcd; |
| 72 |
abcd = vsha256hq_u32(abcd_prev, efgh, tmp); |
| 73 |
efgh = vsha256h2q_u32(efgh, abcd_prev, tmp); |
| 74 |
|
| 75 |
// Rounds t + 4 to t + 7 |
| 76 |
s1 = vsha256su1q_u32(vsha256su0q_u32(s1, s2), s3, s0); |
| 77 |
tmp = vaddq_u32(s1, vld1q_u32(&K32[t + 4])); |
| 78 |
abcd_prev = abcd; |
| 79 |
abcd = vsha256hq_u32(abcd_prev, efgh, tmp); |
| 80 |
efgh = vsha256h2q_u32(efgh, abcd_prev, tmp); |
| 81 |
|
| 82 |
// Rounds t + 8 to t + 11 |
| 83 |
s2 = vsha256su1q_u32(vsha256su0q_u32(s2, s3), s0, s1); |
| 84 |
tmp = vaddq_u32(s2, vld1q_u32(&K32[t + 8])); |
| 85 |
abcd_prev = abcd; |
| 86 |
abcd = vsha256hq_u32(abcd_prev, efgh, tmp); |
| 87 |
efgh = vsha256h2q_u32(efgh, abcd_prev, tmp); |
| 88 |
|
| 89 |
// Rounds t + 12 to t + 15 |
| 90 |
s3 = vsha256su1q_u32(vsha256su0q_u32(s3, s0), s1, s2); |
| 91 |
tmp = vaddq_u32(s3, vld1q_u32(&K32[t + 12])); |
| 92 |
abcd_prev = abcd; |
| 93 |
abcd = vsha256hq_u32(abcd_prev, efgh, tmp); |
| 94 |
efgh = vsha256h2q_u32(efgh, abcd_prev, tmp); |
| 95 |
} |
| 96 |
|
| 97 |
// Add the block-specific state to the original state. |
| 98 |
abcd = vaddq_u32(abcd, abcd_orig); |
| 99 |
efgh = vaddq_u32(efgh, efgh_orig); |
| 100 |
} |
| 101 |
|
| 102 |
// Store vectors into state. |
| 103 |
vst1q_u32(state[0..4].as_mut_ptr(), abcd); |
| 104 |
vst1q_u32(state[4..8].as_mut_ptr(), efgh); |
| 105 |
} |
| 106 |
|
| 107 |
// TODO remove these polyfills once SHA2 intrinsics land |
| 108 |
|
| 109 |
#[inline(always)] |
| 110 |
unsafe fn vsha256hq_u32( |
| 111 |
mut hash_efgh: uint32x4_t, |
| 112 |
hash_abcd: uint32x4_t, |
| 113 |
wk: uint32x4_t, |
| 114 |
) -> uint32x4_t { |
| 115 |
asm!( |
| 116 |
"SHA256H {:q}, {:q}, {:v}.4S", |
| 117 |
inout(vreg) hash_efgh, in(vreg) hash_abcd, in(vreg) wk, |
| 118 |
options(pure, nomem, nostack, preserves_flags) |
| 119 |
); |
| 120 |
hash_efgh |
| 121 |
} |
| 122 |
|
| 123 |
#[inline(always)] |
| 124 |
unsafe fn vsha256h2q_u32( |
| 125 |
mut hash_efgh: uint32x4_t, |
| 126 |
hash_abcd: uint32x4_t, |
| 127 |
wk: uint32x4_t, |
| 128 |
) -> uint32x4_t { |
| 129 |
asm!( |
| 130 |
"SHA256H2 {:q}, {:q}, {:v}.4S", |
| 131 |
inout(vreg) hash_efgh, in(vreg) hash_abcd, in(vreg) wk, |
| 132 |
options(pure, nomem, nostack, preserves_flags) |
| 133 |
); |
| 134 |
hash_efgh |
| 135 |
} |
| 136 |
|
| 137 |
#[inline(always)] |
| 138 |
unsafe fn vsha256su0q_u32(mut w0_3: uint32x4_t, w4_7: uint32x4_t) -> uint32x4_t { |
| 139 |
asm!( |
| 140 |
"SHA256SU0 {:v}.4S, {:v}.4S", |
| 141 |
inout(vreg) w0_3, in(vreg) w4_7, |
| 142 |
options(pure, nomem, nostack, preserves_flags) |
| 143 |
); |
| 144 |
w0_3 |
| 145 |
} |
| 146 |
|
| 147 |
#[inline(always)] |
| 148 |
unsafe fn vsha256su1q_u32( |
| 149 |
mut tw0_3: uint32x4_t, |
| 150 |
w8_11: uint32x4_t, |
| 151 |
w12_15: uint32x4_t, |
| 152 |
) -> uint32x4_t { |
| 153 |
asm!( |
| 154 |
"SHA256SU1 {:v}.4S, {:v}.4S, {:v}.4S", |
| 155 |
inout(vreg) tw0_3, in(vreg) w8_11, in(vreg) w12_15, |
| 156 |
options(pure, nomem, nostack, preserves_flags) |
| 157 |
); |
| 158 |
tw0_3 |
| 159 |
} |