Debian Codegraph @

Modules/_hacl/libintvector.h

raw
1 #ifndef __Vec_Intrin_H
2 #define __Vec_Intrin_H
3
4 #include <sys/types.h>
5
6 #ifndef HACL_INTRINSICS_SHIMMED
7
8 /* We include config.h here to ensure that the various feature-flags are
9 * properly brought into scope. Users can either run the configure script, or
10 * write a config.h themselves and put it under version control. */
11 #if defined(__has_include)
12 #if __has_include("config.h")
13 #include "config.h"
14 #endif
15 #endif
16
17 /* # DEBUGGING:
18 * ============
19 * It is possible to debug the current definitions by using libintvector_debug.h
20 * See the include at the bottom of the file. */
21
22 #define Lib_IntVector_Intrinsics_bit_mask64(x) -((x) & 1)
23
24 #if defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86)
25
26 #if defined(HACL_CAN_COMPILE_VEC128)
27
28 #include <emmintrin.h>
29 #include <tmmintrin.h>
30 #include <smmintrin.h>
31
32 typedef __m128i Lib_IntVector_Intrinsics_vec128;
33
34 #define Lib_IntVector_Intrinsics_ni_aes_enc(x0, x1) \
35 (_mm_aesenc_si128(x0, x1))
36
37 #define Lib_IntVector_Intrinsics_ni_aes_enc_last(x0, x1) \
38 (_mm_aesenclast_si128(x0, x1))
39
40 #define Lib_IntVector_Intrinsics_ni_aes_keygen_assist(x0, x1) \
41 (_mm_aeskeygenassist_si128(x0, x1))
42
43 #define Lib_IntVector_Intrinsics_ni_clmul(x0, x1, x2) \
44 (_mm_clmulepi64_si128(x0, x1, x2))
45
46
47 #define Lib_IntVector_Intrinsics_vec128_xor(x0, x1) \
48 (_mm_xor_si128(x0, x1))
49
50 #define Lib_IntVector_Intrinsics_vec128_eq64(x0, x1) \
51 (_mm_cmpeq_epi64(x0, x1))
52
53 #define Lib_IntVector_Intrinsics_vec128_eq32(x0, x1) \
54 (_mm_cmpeq_epi32(x0, x1))
55
56 #define Lib_IntVector_Intrinsics_vec128_gt64(x0, x1) \
57 (_mm_cmpgt_epi64(x0, x1))
58
59 #define Lib_IntVector_Intrinsics_vec128_gt32(x0, x1) \
60 (_mm_cmpgt_epi32(x0, x1))
61
62 #define Lib_IntVector_Intrinsics_vec128_or(x0, x1) \
63 (_mm_or_si128(x0, x1))
64
65 #define Lib_IntVector_Intrinsics_vec128_and(x0, x1) \
66 (_mm_and_si128(x0, x1))
67
68 #define Lib_IntVector_Intrinsics_vec128_lognot(x0) \
69 (_mm_xor_si128(x0, _mm_set1_epi32(-1)))
70
71
72 #define Lib_IntVector_Intrinsics_vec128_shift_left(x0, x1) \
73 (_mm_slli_si128(x0, (x1)/8))
74
75 #define Lib_IntVector_Intrinsics_vec128_shift_right(x0, x1) \
76 (_mm_srli_si128(x0, (x1)/8))
77
78 #define Lib_IntVector_Intrinsics_vec128_shift_left64(x0, x1) \
79 (_mm_slli_epi64(x0, x1))
80
81 #define Lib_IntVector_Intrinsics_vec128_shift_right64(x0, x1) \
82 (_mm_srli_epi64(x0, x1))
83
84 #define Lib_IntVector_Intrinsics_vec128_shift_left32(x0, x1) \
85 (_mm_slli_epi32(x0, x1))
86
87 #define Lib_IntVector_Intrinsics_vec128_shift_right32(x0, x1) \
88 (_mm_srli_epi32(x0, x1))
89
90 #define Lib_IntVector_Intrinsics_vec128_rotate_left32_8(x0) \
91 (_mm_shuffle_epi8(x0, _mm_set_epi8(14,13,12,15,10,9,8,11,6,5,4,7,2,1,0,3)))
92
93 #define Lib_IntVector_Intrinsics_vec128_rotate_left32_16(x0) \
94 (_mm_shuffle_epi8(x0, _mm_set_epi8(13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2)))
95
96 #define Lib_IntVector_Intrinsics_vec128_rotate_left32_24(x0) \
97 (_mm_shuffle_epi8(x0, _mm_set_epi8(12,15,14,13,8,11,10,9,4,7,6,5,0,3,2,1)))
98
99 #define Lib_IntVector_Intrinsics_vec128_rotate_left32(x0,x1) \
100 (((x1) == 8? Lib_IntVector_Intrinsics_vec128_rotate_left32_8(x0) : \
101 ((x1) == 16? Lib_IntVector_Intrinsics_vec128_rotate_left32_16(x0) : \
102 ((x1) == 24? Lib_IntVector_Intrinsics_vec128_rotate_left32_24(x0) : \
103 _mm_xor_si128(_mm_slli_epi32(x0,x1),_mm_srli_epi32(x0,32-(x1)))))))
104
105 #define Lib_IntVector_Intrinsics_vec128_rotate_right32(x0,x1) \
106 (Lib_IntVector_Intrinsics_vec128_rotate_left32(x0,32-(x1)))
107
108 #define Lib_IntVector_Intrinsics_vec128_shuffle32(x0, x1, x2, x3, x4) \
109 (_mm_shuffle_epi32(x0, _MM_SHUFFLE(x4,x3,x2,x1)))
110
111 #define Lib_IntVector_Intrinsics_vec128_shuffle64(x0, x1, x2) \
112 (_mm_shuffle_epi32(x0, _MM_SHUFFLE(2*x1+1,2*x1,2*x2+1,2*x2)))
113
114 #define Lib_IntVector_Intrinsics_vec128_rotate_right_lanes32(x0, x1) \
115 (_mm_shuffle_epi32(x0, _MM_SHUFFLE((x1+3)%4,(x1+2)%4,(x1+1)%4,x1%4)))
116
117 #define Lib_IntVector_Intrinsics_vec128_rotate_right_lanes64(x0, x1) \
118 (_mm_shuffle_epi32(x0, _MM_SHUFFLE((2*x1+3)%4,(2*x1+2)%4,(2*x1+1)%4,(2*x1)%4)))
119
120 #define Lib_IntVector_Intrinsics_vec128_load32_le(x0) \
121 (_mm_loadu_si128((__m128i*)(x0)))
122
123 #define Lib_IntVector_Intrinsics_vec128_load64_le(x0) \
124 (_mm_loadu_si128((__m128i*)(x0)))
125
126 #define Lib_IntVector_Intrinsics_vec128_store32_le(x0, x1) \
127 (_mm_storeu_si128((__m128i*)(x0), x1))
128
129 #define Lib_IntVector_Intrinsics_vec128_store64_le(x0, x1) \
130 (_mm_storeu_si128((__m128i*)(x0), x1))
131
132 #define Lib_IntVector_Intrinsics_vec128_load_be(x0) \
133 (_mm_shuffle_epi8(_mm_loadu_si128((__m128i*)(x0)), _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)))
134
135 #define Lib_IntVector_Intrinsics_vec128_load32_be(x0) \
136 (_mm_shuffle_epi8(_mm_loadu_si128((__m128i*)(x0)), _mm_set_epi8(12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3)))
137
138 #define Lib_IntVector_Intrinsics_vec128_load64_be(x0) \
139 (_mm_shuffle_epi8(_mm_loadu_si128((__m128i*)(x0)), _mm_set_epi8(8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7)))
140
141 #define Lib_IntVector_Intrinsics_vec128_store_be(x0, x1) \
142 (_mm_storeu_si128((__m128i*)(x0), _mm_shuffle_epi8(x1, _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15))))
143
144
145 #define Lib_IntVector_Intrinsics_vec128_store32_be(x0, x1) \
146 (_mm_storeu_si128((__m128i*)(x0), _mm_shuffle_epi8(x1, _mm_set_epi8(12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3))))
147
148 #define Lib_IntVector_Intrinsics_vec128_store64_be(x0, x1) \
149 (_mm_storeu_si128((__m128i*)(x0), _mm_shuffle_epi8(x1, _mm_set_epi8(8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7))))
150
151
152
153 #define Lib_IntVector_Intrinsics_vec128_insert8(x0, x1, x2) \
154 (_mm_insert_epi8(x0, x1, x2))
155
156 #define Lib_IntVector_Intrinsics_vec128_insert32(x0, x1, x2) \
157 (_mm_insert_epi32(x0, x1, x2))
158
159 #define Lib_IntVector_Intrinsics_vec128_insert64(x0, x1, x2) \
160 (_mm_insert_epi64(x0, x1, x2))
161
162 #define Lib_IntVector_Intrinsics_vec128_extract8(x0, x1) \
163 (_mm_extract_epi8(x0, x1))
164
165 #define Lib_IntVector_Intrinsics_vec128_extract32(x0, x1) \
166 (_mm_extract_epi32(x0, x1))
167
168 #define Lib_IntVector_Intrinsics_vec128_extract64(x0, x1) \
169 (_mm_extract_epi64(x0, x1))
170
171 #define Lib_IntVector_Intrinsics_vec128_zero \
172 (_mm_setzero_si128())
173
174
175 #define Lib_IntVector_Intrinsics_vec128_add64(x0, x1) \
176 (_mm_add_epi64(x0, x1))
177
178 #define Lib_IntVector_Intrinsics_vec128_sub64(x0, x1) \
179 (_mm_sub_epi64(x0, x1))
180
181 #define Lib_IntVector_Intrinsics_vec128_mul64(x0, x1) \
182 (_mm_mul_epu32(x0, x1))
183
184 #define Lib_IntVector_Intrinsics_vec128_smul64(x0, x1) \
185 (_mm_mul_epu32(x0, _mm_set1_epi64x(x1)))
186
187 #define Lib_IntVector_Intrinsics_vec128_add32(x0, x1) \
188 (_mm_add_epi32(x0, x1))
189
190 #define Lib_IntVector_Intrinsics_vec128_sub32(x0, x1) \
191 (_mm_sub_epi32(x0, x1))
192
193 #define Lib_IntVector_Intrinsics_vec128_mul32(x0, x1) \
194 (_mm_mullo_epi32(x0, x1))
195
196 #define Lib_IntVector_Intrinsics_vec128_smul32(x0, x1) \
197 (_mm_mullo_epi32(x0, _mm_set1_epi32(x1)))
198
199 #define Lib_IntVector_Intrinsics_vec128_load128(x) \
200 ((__m128i)x)
201
202 #define Lib_IntVector_Intrinsics_vec128_load64(x) \
203 (_mm_set1_epi64x(x)) /* hi lo */
204
205 #define Lib_IntVector_Intrinsics_vec128_load64s(x0, x1) \
206 (_mm_set_epi64x(x1, x0)) /* hi lo */
207
208 #define Lib_IntVector_Intrinsics_vec128_load32(x) \
209 (_mm_set1_epi32(x))
210
211 #define Lib_IntVector_Intrinsics_vec128_load32s(x0, x1, x2, x3) \
212 (_mm_set_epi32(x3, x2, x1, x0)) /* hi lo */
213
214 #define Lib_IntVector_Intrinsics_vec128_interleave_low32(x1, x2) \
215 (_mm_unpacklo_epi32(x1, x2))
216
217 #define Lib_IntVector_Intrinsics_vec128_interleave_high32(x1, x2) \
218 (_mm_unpackhi_epi32(x1, x2))
219
220 #define Lib_IntVector_Intrinsics_vec128_interleave_low64(x1, x2) \
221 (_mm_unpacklo_epi64(x1, x2))
222
223 #define Lib_IntVector_Intrinsics_vec128_interleave_high64(x1, x2) \
224 (_mm_unpackhi_epi64(x1, x2))
225
226 #endif /* HACL_CAN_COMPILE_VEC128 */
227
228 #if defined(HACL_CAN_COMPILE_VEC256)
229
230 #include <immintrin.h>
231
232 typedef __m256i Lib_IntVector_Intrinsics_vec256;
233
234
235 #define Lib_IntVector_Intrinsics_vec256_eq64(x0, x1) \
236 (_mm256_cmpeq_epi64(x0, x1))
237
238 #define Lib_IntVector_Intrinsics_vec256_eq32(x0, x1) \
239 (_mm256_cmpeq_epi32(x0, x1))
240
241 #define Lib_IntVector_Intrinsics_vec256_gt64(x0, x1) \
242 (_mm256_cmpgt_epi64(x0, x1))
243
244 #define Lib_IntVector_Intrinsics_vec256_gt32(x0, x1) \
245 (_mm256_cmpgt_epi32(x0, x1))
246
247 #define Lib_IntVector_Intrinsics_vec256_xor(x0, x1) \
248 (_mm256_xor_si256(x0, x1))
249
250 #define Lib_IntVector_Intrinsics_vec256_or(x0, x1) \
251 (_mm256_or_si256(x0, x1))
252
253 #define Lib_IntVector_Intrinsics_vec256_and(x0, x1) \
254 (_mm256_and_si256(x0, x1))
255
256 #define Lib_IntVector_Intrinsics_vec256_lognot(x0) \
257 (_mm256_xor_si256(x0, _mm256_set1_epi32(-1)))
258
259 #define Lib_IntVector_Intrinsics_vec256_shift_left(x0, x1) \
260 (_mm256_slli_si256(x0, (x1)/8))
261
262 #define Lib_IntVector_Intrinsics_vec256_shift_right(x0, x1) \
263 (_mm256_srli_si256(x0, (x1)/8))
264
265 #define Lib_IntVector_Intrinsics_vec256_shift_left64(x0, x1) \
266 (_mm256_slli_epi64(x0, x1))
267
268 #define Lib_IntVector_Intrinsics_vec256_shift_right64(x0, x1) \
269 (_mm256_srli_epi64(x0, x1))
270
271 #define Lib_IntVector_Intrinsics_vec256_shift_left32(x0, x1) \
272 (_mm256_slli_epi32(x0, x1))
273
274 #define Lib_IntVector_Intrinsics_vec256_shift_right32(x0, x1) \
275 (_mm256_srli_epi32(x0, x1))
276
277 #define Lib_IntVector_Intrinsics_vec256_rotate_left32_8(x0) \
278 (_mm256_shuffle_epi8(x0, _mm256_set_epi8(14,13,12,15,10,9,8,11,6,5,4,7,2,1,0,3,14,13,12,15,10,9,8,11,6,5,4,7,2,1,0,3)))
279
280 #define Lib_IntVector_Intrinsics_vec256_rotate_left32_16(x0) \
281 (_mm256_shuffle_epi8(x0, _mm256_set_epi8(13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2)))
282
283 #define Lib_IntVector_Intrinsics_vec256_rotate_left32_24(x0) \
284 (_mm256_shuffle_epi8(x0, _mm256_set_epi8(12,15,14,13,8,11,10,9,4,7,6,5,0,3,2,1,12,15,14,13,8,11,10,9,4,7,6,5,0,3,2,1)))
285
286 #define Lib_IntVector_Intrinsics_vec256_rotate_left32(x0,x1) \
287 ((x1 == 8? Lib_IntVector_Intrinsics_vec256_rotate_left32_8(x0) : \
288 (x1 == 16? Lib_IntVector_Intrinsics_vec256_rotate_left32_16(x0) : \
289 (x1 == 24? Lib_IntVector_Intrinsics_vec256_rotate_left32_24(x0) : \
290 _mm256_or_si256(_mm256_slli_epi32(x0,x1),_mm256_srli_epi32(x0,32-(x1)))))))
291
292 #define Lib_IntVector_Intrinsics_vec256_rotate_right32(x0,x1) \
293 (Lib_IntVector_Intrinsics_vec256_rotate_left32(x0,32-(x1)))
294
295 #define Lib_IntVector_Intrinsics_vec256_rotate_right64_8(x0) \
296 (_mm256_shuffle_epi8(x0, _mm256_set_epi8(8,15,14,13,12,11,10,9,0,7,6,5,4,3,2,1,8,15,14,13,12,11,10,9,0,7,6,5,4,3,2,1)))
297
298 #define Lib_IntVector_Intrinsics_vec256_rotate_right64_16(x0) \
299 (_mm256_shuffle_epi8(x0, _mm256_set_epi8(9,8,15,14,13,12,11,10,1,0,7,6,5,4,3,2,9,8,15,14,13,12,11,10,1,0,7,6,5,4,3,2)))
300
301 #define Lib_IntVector_Intrinsics_vec256_rotate_right64_24(x0) \
302 (_mm256_shuffle_epi8(x0, _mm256_set_epi8(10,9,8,15,14,13,12,11,2,1,0,7,6,5,4,3,10,9,8,15,14,13,12,11,2,1,0,7,6,5,4,3)))
303
304 #define Lib_IntVector_Intrinsics_vec256_rotate_right64_32(x0) \
305 (_mm256_shuffle_epi8(x0, _mm256_set_epi8(11,10,9,8,15,14,13,12,3,2,1,0,7,6,5,4,11,10,9,8,15,14,13,12,3,2,1,0,7,6,5,4)))
306
307 #define Lib_IntVector_Intrinsics_vec256_rotate_right64_40(x0) \
308 (_mm256_shuffle_epi8(x0, _mm256_set_epi8(12,11,10,9,8,15,14,13,4,3,2,1,0,7,6,5,12,11,10,9,8,15,14,13,4,3,2,1,0,7,6,5)))
309
310 #define Lib_IntVector_Intrinsics_vec256_rotate_right64_48(x0) \
311 (_mm256_shuffle_epi8(x0, _mm256_set_epi8(13,12,11,10,9,8,15,14,5,4,3,2,1,0,7,6,13,12,11,10,9,8,15,14,5,4,3,2,1,0,7,6)))
312
313 #define Lib_IntVector_Intrinsics_vec256_rotate_right64_56(x0) \
314 (_mm256_shuffle_epi8(x0, _mm256_set_epi8(14,13,12,11,10,9,8,15,6,5,4,3,2,1,0,7,14,13,12,11,10,9,8,15,6,5,4,3,2,1,0,7)))
315
316 #define Lib_IntVector_Intrinsics_vec256_rotate_right64(x0,x1) \
317 ((x1 == 8? Lib_IntVector_Intrinsics_vec256_rotate_right64_8(x0) : \
318 (x1 == 16? Lib_IntVector_Intrinsics_vec256_rotate_right64_16(x0) : \
319 (x1 == 24? Lib_IntVector_Intrinsics_vec256_rotate_right64_24(x0) : \
320 (x1 == 32? Lib_IntVector_Intrinsics_vec256_rotate_right64_32(x0) : \
321 (x1 == 40? Lib_IntVector_Intrinsics_vec256_rotate_right64_40(x0) : \
322 (x1 == 48? Lib_IntVector_Intrinsics_vec256_rotate_right64_48(x0) : \
323 (x1 == 56? Lib_IntVector_Intrinsics_vec256_rotate_right64_56(x0) : \
324 _mm256_xor_si256(_mm256_srli_epi64((x0),(x1)),_mm256_slli_epi64((x0),(64-(x1))))))))))))
325
326 #define Lib_IntVector_Intrinsics_vec256_rotate_left64(x0,x1) \
327 (Lib_IntVector_Intrinsics_vec256_rotate_right64(x0,64-(x1)))
328
329 #define Lib_IntVector_Intrinsics_vec256_shuffle64(x0, x1, x2, x3, x4) \
330 (_mm256_permute4x64_epi64(x0, _MM_SHUFFLE(x4,x3,x2,x1)))
331
332 #define Lib_IntVector_Intrinsics_vec256_shuffle32(x0, x1, x2, x3, x4, x5, x6, x7, x8) \
333 (_mm256_permutevar8x32_epi32(x0, _mm256_set_epi32(x8,x7,x6,x5,x4,x3,x2,x1)))
334
335 #define Lib_IntVector_Intrinsics_vec256_rotate_right_lanes32(x0, x1) \
336 (_mm256_permutevar8x32_epi32(x0, _mm256_set_epi32((x1+7)%8,(x1+6)%8,(x1+5)%8,(x1+4)%8,(x1+3%8),(x1+2)%8,(x1+1)%8,x1%8)))
337
338 #define Lib_IntVector_Intrinsics_vec256_rotate_right_lanes64(x0, x1) \
339 (_mm256_permute4x64_epi64(x0, _MM_SHUFFLE((x1+3)%4,(x1+2)%4,(x1+1)%4,x1%4)))
340
341 #define Lib_IntVector_Intrinsics_vec256_load32_le(x0) \
342 (_mm256_loadu_si256((__m256i*)(x0)))
343
344 #define Lib_IntVector_Intrinsics_vec256_load64_le(x0) \
345 (_mm256_loadu_si256((__m256i*)(x0)))
346
347 #define Lib_IntVector_Intrinsics_vec256_load32_be(x0) \
348 (_mm256_shuffle_epi8(_mm256_loadu_si256((__m256i*)(x0)), _mm256_set_epi8(12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3, 12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3)))
349
350 #define Lib_IntVector_Intrinsics_vec256_load64_be(x0) \
351 (_mm256_shuffle_epi8(_mm256_loadu_si256((__m256i*)(x0)), _mm256_set_epi8(8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7)))
352
353
354 #define Lib_IntVector_Intrinsics_vec256_store32_le(x0, x1) \
355 (_mm256_storeu_si256((__m256i*)(x0), x1))
356
357 #define Lib_IntVector_Intrinsics_vec256_store64_le(x0, x1) \
358 (_mm256_storeu_si256((__m256i*)(x0), x1))
359
360 #define Lib_IntVector_Intrinsics_vec256_store32_be(x0, x1) \
361 (_mm256_storeu_si256((__m256i*)(x0), _mm256_shuffle_epi8(x1, _mm256_set_epi8(12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3, 12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3))))
362
363 #define Lib_IntVector_Intrinsics_vec256_store64_be(x0, x1) \
364 (_mm256_storeu_si256((__m256i*)(x0), _mm256_shuffle_epi8(x1, _mm256_set_epi8(8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7))))
365
366
367 #define Lib_IntVector_Intrinsics_vec256_insert8(x0, x1, x2) \
368 (_mm256_insert_epi8(x0, x1, x2))
369
370 #define Lib_IntVector_Intrinsics_vec256_insert32(x0, x1, x2) \
371 (_mm256_insert_epi32(x0, x1, x2))
372
373 #define Lib_IntVector_Intrinsics_vec256_insert64(x0, x1, x2) \
374 (_mm256_insert_epi64(x0, x1, x2))
375
376 #define Lib_IntVector_Intrinsics_vec256_extract8(x0, x1) \
377 (_mm256_extract_epi8(x0, x1))
378
379 #define Lib_IntVector_Intrinsics_vec256_extract32(x0, x1) \
380 (_mm256_extract_epi32(x0, x1))
381
382 #define Lib_IntVector_Intrinsics_vec256_extract64(x0, x1) \
383 (_mm256_extract_epi64(x0, x1))
384
385 #define Lib_IntVector_Intrinsics_vec256_zero \
386 (_mm256_setzero_si256())
387
388 #define Lib_IntVector_Intrinsics_vec256_add64(x0, x1) \
389 (_mm256_add_epi64(x0, x1))
390
391 #define Lib_IntVector_Intrinsics_vec256_sub64(x0, x1) \
392 (_mm256_sub_epi64(x0, x1))
393
394 #define Lib_IntVector_Intrinsics_vec256_mul64(x0, x1) \
395 (_mm256_mul_epu32(x0, x1))
396
397 #define Lib_IntVector_Intrinsics_vec256_smul64(x0, x1) \
398 (_mm256_mul_epu32(x0, _mm256_set1_epi64x(x1)))
399
400
401 #define Lib_IntVector_Intrinsics_vec256_add32(x0, x1) \
402 (_mm256_add_epi32(x0, x1))
403
404 #define Lib_IntVector_Intrinsics_vec256_sub32(x0, x1) \
405 (_mm256_sub_epi32(x0, x1))
406
407 #define Lib_IntVector_Intrinsics_vec256_mul32(x0, x1) \
408 (_mm256_mullo_epi32(x0, x1))
409
410 #define Lib_IntVector_Intrinsics_vec256_smul32(x0, x1) \
411 (_mm256_mullo_epi32(x0, _mm256_set1_epi32(x1)))
412
413
414 #define Lib_IntVector_Intrinsics_vec256_load64(x1) \
415 (_mm256_set1_epi64x(x1)) /* hi lo */
416
417 #define Lib_IntVector_Intrinsics_vec256_load64s(x0, x1, x2, x3) \
418 (_mm256_set_epi64x(x3,x2,x1,x0)) /* hi lo */
419
420 #define Lib_IntVector_Intrinsics_vec256_load32(x) \
421 (_mm256_set1_epi32(x))
422
423 #define Lib_IntVector_Intrinsics_vec256_load32s(x0,x1,x2,x3,x4, x5, x6, x7) \
424 (_mm256_set_epi32(x7, x6, x5, x4, x3, x2, x1, x0)) /* hi lo */
425
426 #define Lib_IntVector_Intrinsics_vec256_load128(x) \
427 (_mm256_set_m128i((__m128i)x))
428
429 #define Lib_IntVector_Intrinsics_vec256_load128s(x0,x1) \
430 (_mm256_set_m128i((__m128i)x1,(__m128i)x0))
431
432 #define Lib_IntVector_Intrinsics_vec256_interleave_low32(x1, x2) \
433 (_mm256_unpacklo_epi32(x1, x2))
434
435 #define Lib_IntVector_Intrinsics_vec256_interleave_high32(x1, x2) \
436 (_mm256_unpackhi_epi32(x1, x2))
437
438 #define Lib_IntVector_Intrinsics_vec256_interleave_low64(x1, x2) \
439 (_mm256_unpacklo_epi64(x1, x2))
440
441 #define Lib_IntVector_Intrinsics_vec256_interleave_high64(x1, x2) \
442 (_mm256_unpackhi_epi64(x1, x2))
443
444 #define Lib_IntVector_Intrinsics_vec256_interleave_low128(x1, x2) \
445 (_mm256_permute2x128_si256(x1, x2, 0x20))
446
447 #define Lib_IntVector_Intrinsics_vec256_interleave_high128(x1, x2) \
448 (_mm256_permute2x128_si256(x1, x2, 0x31))
449
450 #endif /* HACL_CAN_COMPILE_VEC256 */
451
452 #elif (defined(__aarch64__) || defined(_M_ARM64) || defined(__arm__) || defined(_M_ARM)) \
453 && !defined(__ARM_32BIT_STATE)
454
455 #if defined(HACL_CAN_COMPILE_VEC128)
456
457 #include <arm_neon.h>
458
459 typedef uint32x4_t Lib_IntVector_Intrinsics_vec128;
460
461 #define Lib_IntVector_Intrinsics_vec128_xor(x0, x1) \
462 (veorq_u32(x0,x1))
463
464 #define Lib_IntVector_Intrinsics_vec128_eq64(x0, x1) \
465 (vceqq_u32(x0,x1))
466
467 #define Lib_IntVector_Intrinsics_vec128_eq32(x0, x1) \
468 (vceqq_u32(x0,x1))
469
470 #define Lib_IntVector_Intrinsics_vec128_gt32(x0, x1) \
471 (vcgtq_u32(x0, x1))
472
473 #define high32(x0) \
474 (vmovn_u64(vshrq_n_u64(vreinterpretq_u64_u32(x0),32)))
475
476 #define low32(x0) \
477 (vmovn_u64(vreinterpretq_u64_u32(x0)))
478
479 #define Lib_IntVector_Intrinsics_vec128_gt64(x0, x1) \
480 (vreinterpretq_u32_u64(vmovl_u32(vorr_u32(vcgt_u32(high32(x0),high32(x1)),vand_u32(vceq_u32(high32(x0),high32(x1)),vcgt_u32(low32(x0),low32(x1)))))))
481
482 #define Lib_IntVector_Intrinsics_vec128_or(x0, x1) \
483 (vorrq_u32(x0, x1))
484
485 #define Lib_IntVector_Intrinsics_vec128_and(x0, x1) \
486 (vandq_u32(x0, x1))
487
488 #define Lib_IntVector_Intrinsics_vec128_lognot(x0) \
489 (vmvnq_u32(x0))
490
491
492 #define Lib_IntVector_Intrinsics_vec128_shift_left(x0, x1) \
493 (vextq_u32(x0, vdupq_n_u8(0), 16-(x1)/8))
494
495 #define Lib_IntVector_Intrinsics_vec128_shift_right(x0, x1) \
496 (vextq_u32(x0, vdupq_n_u8(0), (x1)/8))
497
498 #define Lib_IntVector_Intrinsics_vec128_shift_left64(x0, x1) \
499 (vreinterpretq_u32_u64(vshlq_n_u64(vreinterpretq_u64_u32(x0), x1)))
500
501 #define Lib_IntVector_Intrinsics_vec128_shift_right64(x0, x1) \
502 (vreinterpretq_u32_u64(vshrq_n_u64(vreinterpretq_u64_u32(x0), x1)))
503
504 #define Lib_IntVector_Intrinsics_vec128_shift_left32(x0, x1) \
505 (vshlq_n_u32(x0, x1))
506
507 #define Lib_IntVector_Intrinsics_vec128_shift_right32(x0, x1) \
508 (vshrq_n_u32(x0, x1))
509
510 #define Lib_IntVector_Intrinsics_vec128_rotate_left32_16(x1) \
511 (vreinterpretq_u32_u16(vrev32q_u16(vreinterpretq_u16_u32(x1))))
512
513 #define Lib_IntVector_Intrinsics_vec128_rotate_left32(x0,x1) \
514 (((x1) == 16? Lib_IntVector_Intrinsics_vec128_rotate_left32_16(x0) : \
515 vsriq_n_u32(vshlq_n_u32((x0),(x1)),(x0),32-(x1))))
516
517 #define Lib_IntVector_Intrinsics_vec128_rotate_right32_16(x1) \
518 (vreinterpretq_u32_u16(vrev32q_u16(vreinterpretq_u16_u32(x1))))
519
520 #define Lib_IntVector_Intrinsics_vec128_rotate_right32(x0,x1) \
521 (((x1) == 16? Lib_IntVector_Intrinsics_vec128_rotate_right32_16(x0) : \
522 vsriq_n_u32(vshlq_n_u32((x0),32-(x1)),(x0),(x1))))
523
524 #define Lib_IntVector_Intrinsics_vec128_rotate_right_lanes32(x0, x1) \
525 (vextq_u32(x0,x0,x1))
526
527 #define Lib_IntVector_Intrinsics_vec128_rotate_right_lanes64(x0, x1) \
528 (vextq_u64(x0,x0,x1))
529
530
531 /*
532 #define Lib_IntVector_Intrinsics_vec128_shuffle32(x0, x1, x2, x3, x4) \
533 (_mm_shuffle_epi32(x0, _MM_SHUFFLE(x1,x2,x3,x4)))
534
535 #define Lib_IntVector_Intrinsics_vec128_shuffle64(x0, x1, x2) \
536 (_mm_shuffle_epi32(x0, _MM_SHUFFLE(2*x1+1,2*x1,2*x2+1,2*x2)))
537 */
538
539 #define Lib_IntVector_Intrinsics_vec128_load32_le(x0) \
540 (vld1q_u32((const uint32_t*) (x0)))
541
542 #define Lib_IntVector_Intrinsics_vec128_load64_le(x0) \
543 (vld1q_u32((const uint32_t*) (x0)))
544
545 #define Lib_IntVector_Intrinsics_vec128_store32_le(x0, x1) \
546 (vst1q_u32((uint32_t*)(x0),(x1)))
547
548 #define Lib_IntVector_Intrinsics_vec128_store64_le(x0, x1) \
549 (vst1q_u32((uint32_t*)(x0),(x1)))
550
551 /*
552 #define Lib_IntVector_Intrinsics_vec128_load_be(x0) \
553 ( Lib_IntVector_Intrinsics_vec128 l = vrev64q_u8(vld1q_u32((uint32_t*)(x0)));
554
555 */
556
557 #define Lib_IntVector_Intrinsics_vec128_load32_be(x0) \
558 (vreinterpretq_u32_u8(vrev32q_u8(vreinterpretq_u8_u32(vld1q_u32((const uint32_t*)(x0))))))
559
560 #define Lib_IntVector_Intrinsics_vec128_load64_be(x0) \
561 (vreinterpretq_u32_u8(vrev64q_u8(vreinterpretq_u8_u32(vld1q_u32((const uint32_t*)(x0))))))
562
563 /*
564 #define Lib_IntVector_Intrinsics_vec128_store_be(x0, x1) \
565 (_mm_storeu_si128((__m128i*)(x0), _mm_shuffle_epi8(x1, _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15))))
566 */
567
568 #define Lib_IntVector_Intrinsics_vec128_store32_be(x0, x1) \
569 (vst1q_u32((uint32_t*)(x0),(vreinterpretq_u32_u8(vrev32q_u8(vreinterpretq_u8_u32(x1))))))
570
571 #define Lib_IntVector_Intrinsics_vec128_store64_be(x0, x1) \
572 (vst1q_u32((uint32_t*)(x0),(vreinterpretq_u32_u8(vrev64q_u8(vreinterpretq_u8_u32(x1))))))
573
574 #define Lib_IntVector_Intrinsics_vec128_insert8(x0, x1, x2) \
575 (vsetq_lane_u8(x1,x0,x2))
576
577 #define Lib_IntVector_Intrinsics_vec128_insert32(x0, x1, x2) \
578 (vsetq_lane_u32(x1,x0,x2))
579
580 #define Lib_IntVector_Intrinsics_vec128_insert64(x0, x1, x2) \
581 (vreinterpretq_u32_u64(vsetq_lane_u64(x1,vreinterpretq_u64_u32(x0),x2)))
582
583 #define Lib_IntVector_Intrinsics_vec128_extract8(x0, x1) \
584 (vgetq_lane_u8(x0,x1))
585
586 #define Lib_IntVector_Intrinsics_vec128_extract32(x0, x1) \
587 (vgetq_lane_u32(x0,x1))
588
589 #define Lib_IntVector_Intrinsics_vec128_extract64(x0, x1) \
590 (vgetq_lane_u64(vreinterpretq_u64_u32(x0),x1))
591
592 #define Lib_IntVector_Intrinsics_vec128_zero \
593 (vdupq_n_u32(0))
594
595 #define Lib_IntVector_Intrinsics_vec128_add64(x0, x1) \
596 (vreinterpretq_u32_u64(vaddq_u64(vreinterpretq_u64_u32(x0), vreinterpretq_u64_u32(x1))))
597
598 #define Lib_IntVector_Intrinsics_vec128_sub64(x0, x1) \
599 (vreinterpretq_u32_u64(vsubq_u64(vreinterpretq_u64_u32(x0), vreinterpretq_u64_u32(x1))))
600
601 #define Lib_IntVector_Intrinsics_vec128_mul64(x0, x1) \
602 (vreinterpretq_u32_u64(vmull_u32(vmovn_u64(vreinterpretq_u64_u32(x0)), vmovn_u64(vreinterpretq_u64_u32(x1)))))
603
604 #define Lib_IntVector_Intrinsics_vec128_smul64(x0, x1) \
605 (vreinterpretq_u32_u64(vmull_n_u32(vmovn_u64(vreinterpretq_u64_u32(x0)), (uint32_t)x1)))
606
607 #define Lib_IntVector_Intrinsics_vec128_add32(x0, x1) \
608 (vaddq_u32(x0, x1))
609
610 #define Lib_IntVector_Intrinsics_vec128_sub32(x0, x1) \
611 (vsubq_u32(x0, x1))
612
613 #define Lib_IntVector_Intrinsics_vec128_mul32(x0, x1) \
614 (vmulq_lane_u32(x0, x1))
615
616 #define Lib_IntVector_Intrinsics_vec128_smul32(x0, x1) \
617 (vmulq_lane_u32(x0, vdupq_n_u32(x1)))
618
619 #define Lib_IntVector_Intrinsics_vec128_load128(x) \
620 ((uint32x4_t)(x))
621
622 #define Lib_IntVector_Intrinsics_vec128_load64(x) \
623 (vreinterpretq_u32_u64(vdupq_n_u64(x))) /* hi lo */
624
625 #define Lib_IntVector_Intrinsics_vec128_load32(x) \
626 (vdupq_n_u32(x)) /* hi lo */
627
628 static inline Lib_IntVector_Intrinsics_vec128 Lib_IntVector_Intrinsics_vec128_load64s(uint64_t x1, uint64_t x2){
629 const uint64_t a[2] = {x1,x2};
630 return vreinterpretq_u32_u64(vld1q_u64(a));
631 }
632
633 static inline Lib_IntVector_Intrinsics_vec128 Lib_IntVector_Intrinsics_vec128_load32s(uint32_t x1, uint32_t x2, uint32_t x3, uint32_t x4){
634 const uint32_t a[4] = {x1,x2,x3,x4};
635 return vld1q_u32(a);
636 }
637
638 #define Lib_IntVector_Intrinsics_vec128_interleave_low32(x1, x2) \
639 (vzip1q_u32(x1,x2))
640
641 #define Lib_IntVector_Intrinsics_vec128_interleave_high32(x1, x2) \
642 (vzip2q_u32(x1,x2))
643
644 #define Lib_IntVector_Intrinsics_vec128_interleave_low64(x1,x2) \
645 (vreinterpretq_u32_u64(vzip1q_u64(vreinterpretq_u64_u32(x1),vreinterpretq_u64_u32(x2))))
646
647 #define Lib_IntVector_Intrinsics_vec128_interleave_high64(x1,x2) \
648 (vreinterpretq_u32_u64(vzip2q_u64(vreinterpretq_u64_u32(x1),vreinterpretq_u64_u32(x2))))
649
650 #endif /* HACL_CAN_COMPILE_VEC128 */
651
652 /* IBM z architecture */
653 #elif defined(__s390x__) /* this flag is for GCC only */
654
655 #if defined(HACL_CAN_COMPILE_VEC128)
656
657 #include <stdint.h>
658 #include <vecintrin.h>
659
660 /* The main vector 128 type
661 * We can't use uint8_t, uint32_t, uint64_t... instead of unsigned char,
662 * unsigned int, unsigned long long: the compiler complains that the parameter
663 * combination is invalid. */
664 typedef unsigned char vector128_8 __attribute__ ((vector_size(16)));
665 typedef unsigned int vector128_32 __attribute__ ((vector_size(16)));
666 typedef unsigned long long vector128_64 __attribute__ ((vector_size(16)));
667
668 typedef vector128_8 Lib_IntVector_Intrinsics_vec128;
669 typedef vector128_8 vector128;
670
671 #define Lib_IntVector_Intrinsics_vec128_load32_le(x) \
672 (vector128) ((vector128_32) vec_revb(*((vector128_32*) (const uint8_t*)(x))))
673
674 #define Lib_IntVector_Intrinsics_vec128_load32_be(x) \
675 (vector128) (*((vector128_32*) (const uint8_t*)(x)))
676
677 #define Lib_IntVector_Intrinsics_vec128_load64_le(x) \
678 (vector128) ((vector128_64) vec_revb(*((vector128_64*) (const uint8_t*)(x))))
679
680 static inline
681 void Lib_IntVector_Intrinsics_vec128_store32_le(const uint8_t *x0, vector128 x1) {
682 *((vector128_32*)x0) = vec_revb((vector128_32) x1);
683 }
684
685 static inline
686 void Lib_IntVector_Intrinsics_vec128_store32_be(const uint8_t *x0, vector128 x1) {
687 *((vector128_32*)x0) = (vector128_32) x1;
688 }
689
690 static inline
691 void Lib_IntVector_Intrinsics_vec128_store64_le(const uint8_t *x0, vector128 x1) {
692 *((vector128_64*)x0) = vec_revb((vector128_64) x1);
693 }
694
695 #define Lib_IntVector_Intrinsics_vec128_add32(x0,x1) \
696 ((vector128)((vector128_32)(((vector128_32)(x0)) + ((vector128_32)(x1)))))
697
698 #define Lib_IntVector_Intrinsics_vec128_add64(x0, x1) \
699 ((vector128)((vector128_64)(((vector128_64)(x0)) + ((vector128_64)(x1)))))
700
701 #define Lib_IntVector_Intrinsics_vec128_and(x0, x1) \
702 ((vector128)(vec_and((vector128)(x0),(vector128)(x1))))
703
704 #define Lib_IntVector_Intrinsics_vec128_eq32(x0, x1) \
705 ((vector128)(vec_cmpeq(((vector128_32)(x0)),((vector128_32)(x1)))))
706
707 #define Lib_IntVector_Intrinsics_vec128_eq64(x0, x1) \
708 ((vector128)(vec_cmpeq(((vector128_64)(x0)),((vector128_64)(x1)))))
709
710 #define Lib_IntVector_Intrinsics_vec128_extract32(x0, x1) \
711 ((unsigned int)(vec_extract((vector128_32)(x0), x1)))
712
713 #define Lib_IntVector_Intrinsics_vec128_extract64(x0, x1) \
714 ((unsigned long long)(vec_extract((vector128_64)(x0), x1)))
715
716 #define Lib_IntVector_Intrinsics_vec128_gt32(x0, x1) \
717 ((vector128)((vector128_32)(((vector128_32)(x0)) > ((vector128_32)(x1)))))
718
719 #define Lib_IntVector_Intrinsics_vec128_gt64(x0, x1) \
720 ((vector128)((vector128_64)(((vector128_64)(x0)) > ((vector128_64)(x1)))))
721
722 #define Lib_IntVector_Intrinsics_vec128_insert32(x0, x1, x2) \
723 ((vector128)((vector128_32)vec_insert((unsigned int)(x1), (vector128_32)(x0), x2)))
724
725 #define Lib_IntVector_Intrinsics_vec128_insert64(x0, x1, x2) \
726 ((vector128)((vector128_64)vec_insert((unsigned long long)(x1), (vector128_64)(x0), x2)))
727
728 #define Lib_IntVector_Intrinsics_vec128_interleave_high32(x0, x1) \
729 ((vector128)((vector128_32)vec_mergel((vector128_32)(x0), (vector128_32)(x1))))
730
731 #define Lib_IntVector_Intrinsics_vec128_interleave_high64(x0, x1) \
732 ((vector128)((vector128_64)vec_mergel((vector128_64)(x0), (vector128_64)(x1))))
733
734 #define Lib_IntVector_Intrinsics_vec128_interleave_low32(x0, x1) \
735 ((vector128)((vector128_32)vec_mergeh((vector128_32)(x0), (vector128_32)(x1))))
736
737 #define Lib_IntVector_Intrinsics_vec128_interleave_low64(x0, x1) \
738 ((vector128)((vector128_64)vec_mergeh((vector128_64)(x0), (vector128_64)(x1))))
739
740 #define Lib_IntVector_Intrinsics_vec128_load32(x) \
741 ((vector128)((vector128_32){(unsigned int)(x), (unsigned int)(x), \
742 (unsigned int)(x), (unsigned int)(x)}))
743
744 #define Lib_IntVector_Intrinsics_vec128_load32s(x0, x1, x2, x3) \
745 ((vector128)((vector128_32){(unsigned int)(x0),(unsigned int)(x1),(unsigned int)(x2),(unsigned int)(x3)}))
746
747 #define Lib_IntVector_Intrinsics_vec128_load64(x) \
748 ((vector128)((vector128_64)vec_load_pair((unsigned long long)(x),(unsigned long long)(x))))
749
750 #define Lib_IntVector_Intrinsics_vec128_lognot(x0) \
751 ((vector128)(vec_xor((vector128)(x0), (vector128)vec_splat_u32(-1))))
752
753 #define Lib_IntVector_Intrinsics_vec128_mul64(x0, x1) \
754 ((vector128)(vec_mulo((vector128_32)(x0), \
755 (vector128_32)(x1))))
756
757 #define Lib_IntVector_Intrinsics_vec128_or(x0, x1) \
758 ((vector128)(vec_or((vector128)(x0),(vector128)(x1))))
759
760 #define Lib_IntVector_Intrinsics_vec128_rotate_left32(x0, x1) \
761 ((vector128)(vec_rli((vector128_32)(x0), (unsigned long)(x1))))
762
763 #define Lib_IntVector_Intrinsics_vec128_rotate_right32(x0, x1) \
764 (Lib_IntVector_Intrinsics_vec128_rotate_left32(x0,(uint32_t)(32-(x1))))
765
766 #define Lib_IntVector_Intrinsics_vec128_rotate_right_lanes32(x0, x1) \
767 ((vector128)(vec_sld((vector128)(x0), (vector128)(x0), (x1%4)*4)))
768
769 #define Lib_IntVector_Intrinsics_vec128_shift_left64(x0, x1) \
770 (((vector128)((vector128_64)vec_rli((vector128_64)(x0), (unsigned long)(x1)))) & \
771 ((vector128)((vector128_64){0xffffffffffffffff << (x1), 0xffffffffffffffff << (x1)})))
772
773 #define Lib_IntVector_Intrinsics_vec128_shift_right64(x0, x1) \
774 (((vector128)((vector128_64)vec_rli((vector128_64)(x0), (unsigned long)(64-(x1))))) & \
775 ((vector128)((vector128_64){0xffffffffffffffff >> (x1), 0xffffffffffffffff >> (x1)})))
776
777 #define Lib_IntVector_Intrinsics_vec128_shift_right32(x0, x1) \
778 (((vector128)((vector128_32)vec_rli((vector128_32)(x0), (unsigned int)(32-(x1))))) & \
779 ((vector128)((vector128_32){0xffffffff >> (x1), 0xffffffff >> (x1), \
780 0xffffffff >> (x1), 0xffffffff >> (x1)})))
781
782 /* Doesn't work with vec_splat_u64 */
783 #define Lib_IntVector_Intrinsics_vec128_smul64(x0, x1) \
784 ((vector128)(Lib_IntVector_Intrinsics_vec128_mul64(x0,((vector128_64){(unsigned long long)(x1),(unsigned long long)(x1)}))))
785
786 #define Lib_IntVector_Intrinsics_vec128_sub64(x0, x1) \
787 ((vector128)((vector128_64)(x0) - (vector128_64)(x1)))
788
789 static inline
790 vector128 Lib_IntVector_Intrinsics_vec128_xor(vector128 x0, vector128 x1) {
791 return ((vector128)(vec_xor((vector128)(x0), (vector128)(x1))));
792 }
793
794
795 #define Lib_IntVector_Intrinsics_vec128_zero \
796 ((vector128){})
797
798 #endif /* HACL_CAN_COMPILE_VEC128 */
799
800 #elif defined(__powerpc64__) // PowerPC 64 - this flag is for GCC only
801
802 #if defined(HACL_CAN_COMPILE_VEC128)
803
804 #include <altivec.h>
805 #include <string.h> // for memcpy
806 #include <stdint.h>
807
808 // The main vector 128 type
809 // We can't use uint8_t, uint32_t, uint64_t... instead of unsigned char,
810 // unsigned int, unsigned long long: the compiler complains that the parameter
811 // combination is invalid.
812 typedef vector unsigned char vector128_8;
813 typedef vector unsigned int vector128_32;
814 typedef vector unsigned long long vector128_64;
815
816 typedef vector128_8 Lib_IntVector_Intrinsics_vec128;
817 typedef vector128_8 vector128;
818
819 #define Lib_IntVector_Intrinsics_vec128_load32_le(x) \
820 ((vector128)((vector128_32)(vec_xl(0, (const unsigned int*) ((const uint8_t*)(x))))))
821
822 #define Lib_IntVector_Intrinsics_vec128_load64_le(x) \
823 ((vector128)((vector128_64)(vec_xl(0, (const unsigned long long*) ((const uint8_t*)(x))))))
824
825 #define Lib_IntVector_Intrinsics_vec128_store32_le(x0, x1) \
826 (vec_xst((vector128_32)(x1), 0, (unsigned int*) ((uint8_t*)(x0))))
827
828 #define Lib_IntVector_Intrinsics_vec128_store64_le(x0, x1) \
829 (vec_xst((vector128_64)(x1), 0, (unsigned long long*) ((uint8_t*)(x0))))
830
831 #define Lib_IntVector_Intrinsics_vec128_add32(x0,x1) \
832 ((vector128)((vector128_32)(((vector128_32)(x0)) + ((vector128_32)(x1)))))
833
834 #define Lib_IntVector_Intrinsics_vec128_add64(x0, x1) \
835 ((vector128)((vector128_64)(((vector128_64)(x0)) + ((vector128_64)(x1)))))
836
837 #define Lib_IntVector_Intrinsics_vec128_and(x0, x1) \
838 ((vector128)(vec_and((vector128)(x0),(vector128)(x1))))
839
840 #define Lib_IntVector_Intrinsics_vec128_eq32(x0, x1) \
841 ((vector128)(vec_cmpeq(((vector128_32)(x0)),((vector128_32)(x1)))))
842
843 #define Lib_IntVector_Intrinsics_vec128_eq64(x0, x1) \
844 ((vector128)(vec_cmpeq(((vector128_64)(x0)),((vector128_64)(x1)))))
845
846 #define Lib_IntVector_Intrinsics_vec128_extract32(x0, x1) \
847 ((unsigned int)(vec_extract((vector128_32)(x0), x1)))
848
849 #define Lib_IntVector_Intrinsics_vec128_extract64(x0, x1) \
850 ((unsigned long long)(vec_extract((vector128_64)(x0), x1)))
851
852 #define Lib_IntVector_Intrinsics_vec128_gt32(x0, x1) \
853 ((vector128)((vector128_32)(((vector128_32)(x0)) > ((vector128_32)(x1)))))
854
855 #define Lib_IntVector_Intrinsics_vec128_gt64(x0, x1) \
856 ((vector128)((vector128_64)(((vector128_64)(x0)) > ((vector128_64)(x1)))))
857
858 #define Lib_IntVector_Intrinsics_vec128_insert32(x0, x1, x2) \
859 ((vector128)((vector128_32)vec_insert((unsigned int)(x1), (vector128_32)(x0), x2)))
860
861 #define Lib_IntVector_Intrinsics_vec128_insert64(x0, x1, x2) \
862 ((vector128)((vector128_64)vec_insert((unsigned long long)(x1), (vector128_64)(x0), x2)))
863
864 #define Lib_IntVector_Intrinsics_vec128_interleave_high32(x0, x1) \
865 ((vector128)((vector128_32)vec_mergel((vector128_32)(x0), (vector128_32)(x1))))
866
867 #define Lib_IntVector_Intrinsics_vec128_interleave_high64(x0, x1) \
868 ((vector128)((vector128_64)vec_mergel((vector128_64)(x0), (vector128_64)(x1))))
869
870 #define Lib_IntVector_Intrinsics_vec128_interleave_low32(x0, x1) \
871 ((vector128)((vector128_32)vec_mergeh((vector128_32)(x0), (vector128_32)(x1))))
872
873 #define Lib_IntVector_Intrinsics_vec128_interleave_low64(x0, x1) \
874 ((vector128)((vector128_64)vec_mergeh((vector128_64)(x0), (vector128_64)(x1))))
875
876 #define Lib_IntVector_Intrinsics_vec128_load32(x) \
877 ((vector128)((vector128_32){(unsigned int)(x), (unsigned int)(x), \
878 (unsigned int)(x), (unsigned int)(x)}))
879
880 #define Lib_IntVector_Intrinsics_vec128_load32s(x0, x1, x2, x3) \
881 ((vector128)((vector128_32){(unsigned int)(x0),(unsigned int)(x1),(unsigned int)(x2),(unsigned int)(x3)}))
882
883 #define Lib_IntVector_Intrinsics_vec128_load64(x) \
884 ((vector128)((vector128_64){(unsigned long long)(x),(unsigned long long)(x)}))
885
886 #define Lib_IntVector_Intrinsics_vec128_lognot(x0) \
887 ((vector128)(vec_xor((vector128)(x0), (vector128)vec_splat_u32(-1))))
888
889 #define Lib_IntVector_Intrinsics_vec128_mul64(x0, x1) \
890 ((vector128)(vec_mule((vector128_32)(x0), \
891 (vector128_32)(x1))))
892
893 #define Lib_IntVector_Intrinsics_vec128_or(x0, x1) \
894 ((vector128)(vec_or((vector128)(x0),(vector128)(x1))))
895
896 #define Lib_IntVector_Intrinsics_vec128_rotate_left32(x0, x1) \
897 ((vector128)(vec_rl((vector128_32)(x0), (vector128_32){(unsigned int)(x1),(unsigned int)(x1),(unsigned int)(x1),(unsigned int)(x1)})))
898
899 #define Lib_IntVector_Intrinsics_vec128_rotate_right32(x0, x1) \
900 (Lib_IntVector_Intrinsics_vec128_rotate_left32(x0,(uint32_t)(32-(x1))))
901
902 #define Lib_IntVector_Intrinsics_vec128_rotate_right_lanes32(x0, x1) \
903 ((vector128)(vec_sld((vector128)(x0), (vector128)(x0), ((4-(x1))%4)*4)))
904
905 #define Lib_IntVector_Intrinsics_vec128_shift_left64(x0, x1) \
906 ((vector128)((vector128_64)vec_sl((vector128_64)(x0), (vector128_64){(unsigned long)(x1),(unsigned long)(x1)})))
907
908 #define Lib_IntVector_Intrinsics_vec128_shift_right64(x0, x1) \
909 ((vector128)((vector128_64)vec_sr((vector128_64)(x0), (vector128_64){(unsigned long)(x1),(unsigned long)(x1)})))
910
911 // Doesn't work with vec_splat_u64
912 #define Lib_IntVector_Intrinsics_vec128_smul64(x0, x1) \
913 ((vector128)(Lib_IntVector_Intrinsics_vec128_mul64(x0,((vector128_64){(unsigned long long)(x1),(unsigned long long)(x1)}))))
914
915 #define Lib_IntVector_Intrinsics_vec128_sub64(x0, x1) \
916 ((vector128)((vector128_64)(x0) - (vector128_64)(x1)))
917
918 #define Lib_IntVector_Intrinsics_vec128_xor(x0, x1) \
919 ((vector128)(vec_xor((vector128)(x0), (vector128)(x1))))
920
921 #define Lib_IntVector_Intrinsics_vec128_zero \
922 ((vector128){})
923
924 #endif /* HACL_CAN_COMPILE_VEC128 */
925
926 #endif // PowerPC64
927
928 // DEBUGGING:
929 // If libintvector_debug.h exists, use it to debug the current implementations.
930 // Note that some flags must be enabled for the debugging to be effective:
931 // see libintvector_debug.h for more details.
932 #if defined(__has_include)
933 #if __has_include("libintvector_debug.h")
934 #include "libintvector_debug.h"
935 #endif
936 #endif
937
938 #endif // HACL_INTRINSICS_SHIMMED
939
940 #endif // __Vec_Intrin_H