Debian Codegraph @

.cargo/vendor/sha2/src/sha512/aarch64.rs

raw
1 // Implementation adapted from mbedtls.
2
3 use core::arch::{aarch64::*, asm};
4
5 use crate::consts::K64;
6
7 cpufeatures::new!(sha3_hwcap, "sha3");
8
9 pub fn compress(state: &mut [u64; 8], blocks: &[[u8; 128]]) {
10 // TODO: Replace with https://github.com/rust-lang/rfcs/pull/2725
11 // after stabilization
12 if sha3_hwcap::get() {
13 unsafe { sha512_compress(state, blocks) }
14 } else {
15 super::soft::compress(state, blocks);
16 }
17 }
18
19 #[target_feature(enable = "sha3")]
20 unsafe fn sha512_compress(state: &mut [u64; 8], blocks: &[[u8; 128]]) {
21 // SAFETY: Requires the sha3 feature.
22
23 // Load state into vectors.
24 let mut ab = vld1q_u64(state[0..2].as_ptr());
25 let mut cd = vld1q_u64(state[2..4].as_ptr());
26 let mut ef = vld1q_u64(state[4..6].as_ptr());
27 let mut gh = vld1q_u64(state[6..8].as_ptr());
28
29 // Iterate through the message blocks.
30 for block in blocks {
31 // Keep original state values.
32 let ab_orig = ab;
33 let cd_orig = cd;
34 let ef_orig = ef;
35 let gh_orig = gh;
36
37 // Load the message block into vectors, assuming little endianness.
38 let mut s0 = vreinterpretq_u64_u8(vrev64q_u8(vld1q_u8(block[0..16].as_ptr())));
39 let mut s1 = vreinterpretq_u64_u8(vrev64q_u8(vld1q_u8(block[16..32].as_ptr())));
40 let mut s2 = vreinterpretq_u64_u8(vrev64q_u8(vld1q_u8(block[32..48].as_ptr())));
41 let mut s3 = vreinterpretq_u64_u8(vrev64q_u8(vld1q_u8(block[48..64].as_ptr())));
42 let mut s4 = vreinterpretq_u64_u8(vrev64q_u8(vld1q_u8(block[64..80].as_ptr())));
43 let mut s5 = vreinterpretq_u64_u8(vrev64q_u8(vld1q_u8(block[80..96].as_ptr())));
44 let mut s6 = vreinterpretq_u64_u8(vrev64q_u8(vld1q_u8(block[96..112].as_ptr())));
45 let mut s7 = vreinterpretq_u64_u8(vrev64q_u8(vld1q_u8(block[112..128].as_ptr())));
46
47 // Rounds 0 and 1
48 let mut initial_sum = vaddq_u64(s0, vld1q_u64(&K64[0]));
49 let mut sum = vaddq_u64(vextq_u64(initial_sum, initial_sum, 1), gh);
50 let mut intermed = vsha512hq_u64(sum, vextq_u64(ef, gh, 1), vextq_u64(cd, ef, 1));
51 gh = vsha512h2q_u64(intermed, cd, ab);
52 cd = vaddq_u64(cd, intermed);
53
54 // Rounds 2 and 3
55 initial_sum = vaddq_u64(s1, vld1q_u64(&K64[2]));
56 sum = vaddq_u64(vextq_u64(initial_sum, initial_sum, 1), ef);
57 intermed = vsha512hq_u64(sum, vextq_u64(cd, ef, 1), vextq_u64(ab, cd, 1));
58 ef = vsha512h2q_u64(intermed, ab, gh);
59 ab = vaddq_u64(ab, intermed);
60
61 // Rounds 4 and 5
62 initial_sum = vaddq_u64(s2, vld1q_u64(&K64[4]));
63 sum = vaddq_u64(vextq_u64(initial_sum, initial_sum, 1), cd);
64 intermed = vsha512hq_u64(sum, vextq_u64(ab, cd, 1), vextq_u64(gh, ab, 1));
65 cd = vsha512h2q_u64(intermed, gh, ef);
66 gh = vaddq_u64(gh, intermed);
67
68 // Rounds 6 and 7
69 initial_sum = vaddq_u64(s3, vld1q_u64(&K64[6]));
70 sum = vaddq_u64(vextq_u64(initial_sum, initial_sum, 1), ab);
71 intermed = vsha512hq_u64(sum, vextq_u64(gh, ab, 1), vextq_u64(ef, gh, 1));
72 ab = vsha512h2q_u64(intermed, ef, cd);
73 ef = vaddq_u64(ef, intermed);
74
75 // Rounds 8 and 9
76 initial_sum = vaddq_u64(s4, vld1q_u64(&K64[8]));
77 sum = vaddq_u64(vextq_u64(initial_sum, initial_sum, 1), gh);
78 intermed = vsha512hq_u64(sum, vextq_u64(ef, gh, 1), vextq_u64(cd, ef, 1));
79 gh = vsha512h2q_u64(intermed, cd, ab);
80 cd = vaddq_u64(cd, intermed);
81
82 // Rounds 10 and 11
83 initial_sum = vaddq_u64(s5, vld1q_u64(&K64[10]));
84 sum = vaddq_u64(vextq_u64(initial_sum, initial_sum, 1), ef);
85 intermed = vsha512hq_u64(sum, vextq_u64(cd, ef, 1), vextq_u64(ab, cd, 1));
86 ef = vsha512h2q_u64(intermed, ab, gh);
87 ab = vaddq_u64(ab, intermed);
88
89 // Rounds 12 and 13
90 initial_sum = vaddq_u64(s6, vld1q_u64(&K64[12]));
91 sum = vaddq_u64(vextq_u64(initial_sum, initial_sum, 1), cd);
92 intermed = vsha512hq_u64(sum, vextq_u64(ab, cd, 1), vextq_u64(gh, ab, 1));
93 cd = vsha512h2q_u64(intermed, gh, ef);
94 gh = vaddq_u64(gh, intermed);
95
96 // Rounds 14 and 15
97 initial_sum = vaddq_u64(s7, vld1q_u64(&K64[14]));
98 sum = vaddq_u64(vextq_u64(initial_sum, initial_sum, 1), ab);
99 intermed = vsha512hq_u64(sum, vextq_u64(gh, ab, 1), vextq_u64(ef, gh, 1));
100 ab = vsha512h2q_u64(intermed, ef, cd);
101 ef = vaddq_u64(ef, intermed);
102
103 for t in (16..80).step_by(16) {
104 // Rounds t and t + 1
105 s0 = vsha512su1q_u64(vsha512su0q_u64(s0, s1), s7, vextq_u64(s4, s5, 1));
106 initial_sum = vaddq_u64(s0, vld1q_u64(&K64[t]));
107 sum = vaddq_u64(vextq_u64(initial_sum, initial_sum, 1), gh);
108 intermed = vsha512hq_u64(sum, vextq_u64(ef, gh, 1), vextq_u64(cd, ef, 1));
109 gh = vsha512h2q_u64(intermed, cd, ab);
110 cd = vaddq_u64(cd, intermed);
111
112 // Rounds t + 2 and t + 3
113 s1 = vsha512su1q_u64(vsha512su0q_u64(s1, s2), s0, vextq_u64(s5, s6, 1));
114 initial_sum = vaddq_u64(s1, vld1q_u64(&K64[t + 2]));
115 sum = vaddq_u64(vextq_u64(initial_sum, initial_sum, 1), ef);
116 intermed = vsha512hq_u64(sum, vextq_u64(cd, ef, 1), vextq_u64(ab, cd, 1));
117 ef = vsha512h2q_u64(intermed, ab, gh);
118 ab = vaddq_u64(ab, intermed);
119
120 // Rounds t + 4 and t + 5
121 s2 = vsha512su1q_u64(vsha512su0q_u64(s2, s3), s1, vextq_u64(s6, s7, 1));
122 initial_sum = vaddq_u64(s2, vld1q_u64(&K64[t + 4]));
123 sum = vaddq_u64(vextq_u64(initial_sum, initial_sum, 1), cd);
124 intermed = vsha512hq_u64(sum, vextq_u64(ab, cd, 1), vextq_u64(gh, ab, 1));
125 cd = vsha512h2q_u64(intermed, gh, ef);
126 gh = vaddq_u64(gh, intermed);
127
128 // Rounds t + 6 and t + 7
129 s3 = vsha512su1q_u64(vsha512su0q_u64(s3, s4), s2, vextq_u64(s7, s0, 1));
130 initial_sum = vaddq_u64(s3, vld1q_u64(&K64[t + 6]));
131 sum = vaddq_u64(vextq_u64(initial_sum, initial_sum, 1), ab);
132 intermed = vsha512hq_u64(sum, vextq_u64(gh, ab, 1), vextq_u64(ef, gh, 1));
133 ab = vsha512h2q_u64(intermed, ef, cd);
134 ef = vaddq_u64(ef, intermed);
135
136 // Rounds t + 8 and t + 9
137 s4 = vsha512su1q_u64(vsha512su0q_u64(s4, s5), s3, vextq_u64(s0, s1, 1));
138 initial_sum = vaddq_u64(s4, vld1q_u64(&K64[t + 8]));
139 sum = vaddq_u64(vextq_u64(initial_sum, initial_sum, 1), gh);
140 intermed = vsha512hq_u64(sum, vextq_u64(ef, gh, 1), vextq_u64(cd, ef, 1));
141 gh = vsha512h2q_u64(intermed, cd, ab);
142 cd = vaddq_u64(cd, intermed);
143
144 // Rounds t + 10 and t + 11
145 s5 = vsha512su1q_u64(vsha512su0q_u64(s5, s6), s4, vextq_u64(s1, s2, 1));
146 initial_sum = vaddq_u64(s5, vld1q_u64(&K64[t + 10]));
147 sum = vaddq_u64(vextq_u64(initial_sum, initial_sum, 1), ef);
148 intermed = vsha512hq_u64(sum, vextq_u64(cd, ef, 1), vextq_u64(ab, cd, 1));
149 ef = vsha512h2q_u64(intermed, ab, gh);
150 ab = vaddq_u64(ab, intermed);
151
152 // Rounds t + 12 and t + 13
153 s6 = vsha512su1q_u64(vsha512su0q_u64(s6, s7), s5, vextq_u64(s2, s3, 1));
154 initial_sum = vaddq_u64(s6, vld1q_u64(&K64[t + 12]));
155 sum = vaddq_u64(vextq_u64(initial_sum, initial_sum, 1), cd);
156 intermed = vsha512hq_u64(sum, vextq_u64(ab, cd, 1), vextq_u64(gh, ab, 1));
157 cd = vsha512h2q_u64(intermed, gh, ef);
158 gh = vaddq_u64(gh, intermed);
159
160 // Rounds t + 14 and t + 15
161 s7 = vsha512su1q_u64(vsha512su0q_u64(s7, s0), s6, vextq_u64(s3, s4, 1));
162 initial_sum = vaddq_u64(s7, vld1q_u64(&K64[t + 14]));
163 sum = vaddq_u64(vextq_u64(initial_sum, initial_sum, 1), ab);
164 intermed = vsha512hq_u64(sum, vextq_u64(gh, ab, 1), vextq_u64(ef, gh, 1));
165 ab = vsha512h2q_u64(intermed, ef, cd);
166 ef = vaddq_u64(ef, intermed);
167 }
168
169 // Add the block-specific state to the original state.
170 ab = vaddq_u64(ab, ab_orig);
171 cd = vaddq_u64(cd, cd_orig);
172 ef = vaddq_u64(ef, ef_orig);
173 gh = vaddq_u64(gh, gh_orig);
174 }
175
176 // Store vectors into state.
177 vst1q_u64(state[0..2].as_mut_ptr(), ab);
178 vst1q_u64(state[2..4].as_mut_ptr(), cd);
179 vst1q_u64(state[4..6].as_mut_ptr(), ef);
180 vst1q_u64(state[6..8].as_mut_ptr(), gh);
181 }
182
183 // TODO remove these polyfills once SHA3 intrinsics land
184
185 #[inline(always)]
186 unsafe fn vsha512hq_u64(
187 mut hash_ed: uint64x2_t,
188 hash_gf: uint64x2_t,
189 kwh_kwh2: uint64x2_t,
190 ) -> uint64x2_t {
191 asm!(
192 "SHA512H {:q}, {:q}, {:v}.2D",
193 inout(vreg) hash_ed, in(vreg) hash_gf, in(vreg) kwh_kwh2,
194 options(pure, nomem, nostack, preserves_flags)
195 );
196 hash_ed
197 }
198
199 #[inline(always)]
200 unsafe fn vsha512h2q_u64(
201 mut sum_ab: uint64x2_t,
202 hash_c_: uint64x2_t,
203 hash_ab: uint64x2_t,
204 ) -> uint64x2_t {
205 asm!(
206 "SHA512H2 {:q}, {:q}, {:v}.2D",
207 inout(vreg) sum_ab, in(vreg) hash_c_, in(vreg) hash_ab,
208 options(pure, nomem, nostack, preserves_flags)
209 );
210 sum_ab
211 }
212
213 #[inline(always)]
214 unsafe fn vsha512su0q_u64(mut w0_1: uint64x2_t, w2_: uint64x2_t) -> uint64x2_t {
215 asm!(
216 "SHA512SU0 {:v}.2D, {:v}.2D",
217 inout(vreg) w0_1, in(vreg) w2_,
218 options(pure, nomem, nostack, preserves_flags)
219 );
220 w0_1
221 }
222
223 #[inline(always)]
224 unsafe fn vsha512su1q_u64(
225 mut s01_s02: uint64x2_t,
226 w14_15: uint64x2_t,
227 w9_10: uint64x2_t,
228 ) -> uint64x2_t {
229 asm!(
230 "SHA512SU1 {:v}.2D, {:v}.2D, {:v}.2D",
231 inout(vreg) s01_s02, in(vreg) w14_15, in(vreg) w9_10,
232 options(pure, nomem, nostack, preserves_flags)
233 );
234 s01_s02
235 }