| 1 |
#ifndef __Vec_Intrin_H |
| 2 |
#define __Vec_Intrin_H |
| 3 |
|
| 4 |
#include <sys/types.h> |
| 5 |
|
| 6 |
#ifndef HACL_INTRINSICS_SHIMMED |
| 7 |
|
| 8 |
/* We include config.h here to ensure that the various feature-flags are |
| 9 |
* properly brought into scope. Users can either run the configure script, or |
| 10 |
* write a config.h themselves and put it under version control. */ |
| 11 |
#if defined(__has_include) |
| 12 |
#if __has_include("config.h") |
| 13 |
#include "config.h" |
| 14 |
#endif |
| 15 |
#endif |
| 16 |
|
| 17 |
/* # DEBUGGING: |
| 18 |
* ============ |
| 19 |
* It is possible to debug the current definitions by using libintvector_debug.h |
| 20 |
* See the include at the bottom of the file. */ |
| 21 |
|
| 22 |
#define Lib_IntVector_Intrinsics_bit_mask64(x) -((x) & 1) |
| 23 |
|
| 24 |
#if defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86) |
| 25 |
|
| 26 |
#if defined(HACL_CAN_COMPILE_VEC128) |
| 27 |
|
| 28 |
#include <emmintrin.h> |
| 29 |
#include <tmmintrin.h> |
| 30 |
#include <smmintrin.h> |
| 31 |
|
| 32 |
typedef __m128i Lib_IntVector_Intrinsics_vec128; |
| 33 |
|
| 34 |
#define Lib_IntVector_Intrinsics_ni_aes_enc(x0, x1) \ |
| 35 |
(_mm_aesenc_si128(x0, x1)) |
| 36 |
|
| 37 |
#define Lib_IntVector_Intrinsics_ni_aes_enc_last(x0, x1) \ |
| 38 |
(_mm_aesenclast_si128(x0, x1)) |
| 39 |
|
| 40 |
#define Lib_IntVector_Intrinsics_ni_aes_keygen_assist(x0, x1) \ |
| 41 |
(_mm_aeskeygenassist_si128(x0, x1)) |
| 42 |
|
| 43 |
#define Lib_IntVector_Intrinsics_ni_clmul(x0, x1, x2) \ |
| 44 |
(_mm_clmulepi64_si128(x0, x1, x2)) |
| 45 |
|
| 46 |
|
| 47 |
#define Lib_IntVector_Intrinsics_vec128_xor(x0, x1) \ |
| 48 |
(_mm_xor_si128(x0, x1)) |
| 49 |
|
| 50 |
#define Lib_IntVector_Intrinsics_vec128_eq64(x0, x1) \ |
| 51 |
(_mm_cmpeq_epi64(x0, x1)) |
| 52 |
|
| 53 |
#define Lib_IntVector_Intrinsics_vec128_eq32(x0, x1) \ |
| 54 |
(_mm_cmpeq_epi32(x0, x1)) |
| 55 |
|
| 56 |
#define Lib_IntVector_Intrinsics_vec128_gt64(x0, x1) \ |
| 57 |
(_mm_cmpgt_epi64(x0, x1)) |
| 58 |
|
| 59 |
#define Lib_IntVector_Intrinsics_vec128_gt32(x0, x1) \ |
| 60 |
(_mm_cmpgt_epi32(x0, x1)) |
| 61 |
|
| 62 |
#define Lib_IntVector_Intrinsics_vec128_or(x0, x1) \ |
| 63 |
(_mm_or_si128(x0, x1)) |
| 64 |
|
| 65 |
#define Lib_IntVector_Intrinsics_vec128_and(x0, x1) \ |
| 66 |
(_mm_and_si128(x0, x1)) |
| 67 |
|
| 68 |
#define Lib_IntVector_Intrinsics_vec128_lognot(x0) \ |
| 69 |
(_mm_xor_si128(x0, _mm_set1_epi32(-1))) |
| 70 |
|
| 71 |
|
| 72 |
#define Lib_IntVector_Intrinsics_vec128_shift_left(x0, x1) \ |
| 73 |
(_mm_slli_si128(x0, (x1)/8)) |
| 74 |
|
| 75 |
#define Lib_IntVector_Intrinsics_vec128_shift_right(x0, x1) \ |
| 76 |
(_mm_srli_si128(x0, (x1)/8)) |
| 77 |
|
| 78 |
#define Lib_IntVector_Intrinsics_vec128_shift_left64(x0, x1) \ |
| 79 |
(_mm_slli_epi64(x0, x1)) |
| 80 |
|
| 81 |
#define Lib_IntVector_Intrinsics_vec128_shift_right64(x0, x1) \ |
| 82 |
(_mm_srli_epi64(x0, x1)) |
| 83 |
|
| 84 |
#define Lib_IntVector_Intrinsics_vec128_shift_left32(x0, x1) \ |
| 85 |
(_mm_slli_epi32(x0, x1)) |
| 86 |
|
| 87 |
#define Lib_IntVector_Intrinsics_vec128_shift_right32(x0, x1) \ |
| 88 |
(_mm_srli_epi32(x0, x1)) |
| 89 |
|
| 90 |
#define Lib_IntVector_Intrinsics_vec128_rotate_left32_8(x0) \ |
| 91 |
(_mm_shuffle_epi8(x0, _mm_set_epi8(14,13,12,15,10,9,8,11,6,5,4,7,2,1,0,3))) |
| 92 |
|
| 93 |
#define Lib_IntVector_Intrinsics_vec128_rotate_left32_16(x0) \ |
| 94 |
(_mm_shuffle_epi8(x0, _mm_set_epi8(13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2))) |
| 95 |
|
| 96 |
#define Lib_IntVector_Intrinsics_vec128_rotate_left32_24(x0) \ |
| 97 |
(_mm_shuffle_epi8(x0, _mm_set_epi8(12,15,14,13,8,11,10,9,4,7,6,5,0,3,2,1))) |
| 98 |
|
| 99 |
#define Lib_IntVector_Intrinsics_vec128_rotate_left32(x0,x1) \ |
| 100 |
(((x1) == 8? Lib_IntVector_Intrinsics_vec128_rotate_left32_8(x0) : \ |
| 101 |
((x1) == 16? Lib_IntVector_Intrinsics_vec128_rotate_left32_16(x0) : \ |
| 102 |
((x1) == 24? Lib_IntVector_Intrinsics_vec128_rotate_left32_24(x0) : \ |
| 103 |
_mm_xor_si128(_mm_slli_epi32(x0,x1),_mm_srli_epi32(x0,32-(x1))))))) |
| 104 |
|
| 105 |
#define Lib_IntVector_Intrinsics_vec128_rotate_right32(x0,x1) \ |
| 106 |
(Lib_IntVector_Intrinsics_vec128_rotate_left32(x0,32-(x1))) |
| 107 |
|
| 108 |
#define Lib_IntVector_Intrinsics_vec128_shuffle32(x0, x1, x2, x3, x4) \ |
| 109 |
(_mm_shuffle_epi32(x0, _MM_SHUFFLE(x4,x3,x2,x1))) |
| 110 |
|
| 111 |
#define Lib_IntVector_Intrinsics_vec128_shuffle64(x0, x1, x2) \ |
| 112 |
(_mm_shuffle_epi32(x0, _MM_SHUFFLE(2*x1+1,2*x1,2*x2+1,2*x2))) |
| 113 |
|
| 114 |
#define Lib_IntVector_Intrinsics_vec128_rotate_right_lanes32(x0, x1) \ |
| 115 |
(_mm_shuffle_epi32(x0, _MM_SHUFFLE((x1+3)%4,(x1+2)%4,(x1+1)%4,x1%4))) |
| 116 |
|
| 117 |
#define Lib_IntVector_Intrinsics_vec128_rotate_right_lanes64(x0, x1) \ |
| 118 |
(_mm_shuffle_epi32(x0, _MM_SHUFFLE((2*x1+3)%4,(2*x1+2)%4,(2*x1+1)%4,(2*x1)%4))) |
| 119 |
|
| 120 |
#define Lib_IntVector_Intrinsics_vec128_load32_le(x0) \ |
| 121 |
(_mm_loadu_si128((__m128i*)(x0))) |
| 122 |
|
| 123 |
#define Lib_IntVector_Intrinsics_vec128_load64_le(x0) \ |
| 124 |
(_mm_loadu_si128((__m128i*)(x0))) |
| 125 |
|
| 126 |
#define Lib_IntVector_Intrinsics_vec128_store32_le(x0, x1) \ |
| 127 |
(_mm_storeu_si128((__m128i*)(x0), x1)) |
| 128 |
|
| 129 |
#define Lib_IntVector_Intrinsics_vec128_store64_le(x0, x1) \ |
| 130 |
(_mm_storeu_si128((__m128i*)(x0), x1)) |
| 131 |
|
| 132 |
#define Lib_IntVector_Intrinsics_vec128_load_be(x0) \ |
| 133 |
(_mm_shuffle_epi8(_mm_loadu_si128((__m128i*)(x0)), _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15))) |
| 134 |
|
| 135 |
#define Lib_IntVector_Intrinsics_vec128_load32_be(x0) \ |
| 136 |
(_mm_shuffle_epi8(_mm_loadu_si128((__m128i*)(x0)), _mm_set_epi8(12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3))) |
| 137 |
|
| 138 |
#define Lib_IntVector_Intrinsics_vec128_load64_be(x0) \ |
| 139 |
(_mm_shuffle_epi8(_mm_loadu_si128((__m128i*)(x0)), _mm_set_epi8(8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7))) |
| 140 |
|
| 141 |
#define Lib_IntVector_Intrinsics_vec128_store_be(x0, x1) \ |
| 142 |
(_mm_storeu_si128((__m128i*)(x0), _mm_shuffle_epi8(x1, _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)))) |
| 143 |
|
| 144 |
|
| 145 |
#define Lib_IntVector_Intrinsics_vec128_store32_be(x0, x1) \ |
| 146 |
(_mm_storeu_si128((__m128i*)(x0), _mm_shuffle_epi8(x1, _mm_set_epi8(12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3)))) |
| 147 |
|
| 148 |
#define Lib_IntVector_Intrinsics_vec128_store64_be(x0, x1) \ |
| 149 |
(_mm_storeu_si128((__m128i*)(x0), _mm_shuffle_epi8(x1, _mm_set_epi8(8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7)))) |
| 150 |
|
| 151 |
|
| 152 |
|
| 153 |
#define Lib_IntVector_Intrinsics_vec128_insert8(x0, x1, x2) \ |
| 154 |
(_mm_insert_epi8(x0, x1, x2)) |
| 155 |
|
| 156 |
#define Lib_IntVector_Intrinsics_vec128_insert32(x0, x1, x2) \ |
| 157 |
(_mm_insert_epi32(x0, x1, x2)) |
| 158 |
|
| 159 |
#define Lib_IntVector_Intrinsics_vec128_insert64(x0, x1, x2) \ |
| 160 |
(_mm_insert_epi64(x0, x1, x2)) |
| 161 |
|
| 162 |
#define Lib_IntVector_Intrinsics_vec128_extract8(x0, x1) \ |
| 163 |
(_mm_extract_epi8(x0, x1)) |
| 164 |
|
| 165 |
#define Lib_IntVector_Intrinsics_vec128_extract32(x0, x1) \ |
| 166 |
(_mm_extract_epi32(x0, x1)) |
| 167 |
|
| 168 |
#define Lib_IntVector_Intrinsics_vec128_extract64(x0, x1) \ |
| 169 |
(_mm_extract_epi64(x0, x1)) |
| 170 |
|
| 171 |
#define Lib_IntVector_Intrinsics_vec128_zero \ |
| 172 |
(_mm_setzero_si128()) |
| 173 |
|
| 174 |
|
| 175 |
#define Lib_IntVector_Intrinsics_vec128_add64(x0, x1) \ |
| 176 |
(_mm_add_epi64(x0, x1)) |
| 177 |
|
| 178 |
#define Lib_IntVector_Intrinsics_vec128_sub64(x0, x1) \ |
| 179 |
(_mm_sub_epi64(x0, x1)) |
| 180 |
|
| 181 |
#define Lib_IntVector_Intrinsics_vec128_mul64(x0, x1) \ |
| 182 |
(_mm_mul_epu32(x0, x1)) |
| 183 |
|
| 184 |
#define Lib_IntVector_Intrinsics_vec128_smul64(x0, x1) \ |
| 185 |
(_mm_mul_epu32(x0, _mm_set1_epi64x(x1))) |
| 186 |
|
| 187 |
#define Lib_IntVector_Intrinsics_vec128_add32(x0, x1) \ |
| 188 |
(_mm_add_epi32(x0, x1)) |
| 189 |
|
| 190 |
#define Lib_IntVector_Intrinsics_vec128_sub32(x0, x1) \ |
| 191 |
(_mm_sub_epi32(x0, x1)) |
| 192 |
|
| 193 |
#define Lib_IntVector_Intrinsics_vec128_mul32(x0, x1) \ |
| 194 |
(_mm_mullo_epi32(x0, x1)) |
| 195 |
|
| 196 |
#define Lib_IntVector_Intrinsics_vec128_smul32(x0, x1) \ |
| 197 |
(_mm_mullo_epi32(x0, _mm_set1_epi32(x1))) |
| 198 |
|
| 199 |
#define Lib_IntVector_Intrinsics_vec128_load128(x) \ |
| 200 |
((__m128i)x) |
| 201 |
|
| 202 |
#define Lib_IntVector_Intrinsics_vec128_load64(x) \ |
| 203 |
(_mm_set1_epi64x(x)) /* hi lo */ |
| 204 |
|
| 205 |
#define Lib_IntVector_Intrinsics_vec128_load64s(x0, x1) \ |
| 206 |
(_mm_set_epi64x(x1, x0)) /* hi lo */ |
| 207 |
|
| 208 |
#define Lib_IntVector_Intrinsics_vec128_load32(x) \ |
| 209 |
(_mm_set1_epi32(x)) |
| 210 |
|
| 211 |
#define Lib_IntVector_Intrinsics_vec128_load32s(x0, x1, x2, x3) \ |
| 212 |
(_mm_set_epi32(x3, x2, x1, x0)) /* hi lo */ |
| 213 |
|
| 214 |
#define Lib_IntVector_Intrinsics_vec128_interleave_low32(x1, x2) \ |
| 215 |
(_mm_unpacklo_epi32(x1, x2)) |
| 216 |
|
| 217 |
#define Lib_IntVector_Intrinsics_vec128_interleave_high32(x1, x2) \ |
| 218 |
(_mm_unpackhi_epi32(x1, x2)) |
| 219 |
|
| 220 |
#define Lib_IntVector_Intrinsics_vec128_interleave_low64(x1, x2) \ |
| 221 |
(_mm_unpacklo_epi64(x1, x2)) |
| 222 |
|
| 223 |
#define Lib_IntVector_Intrinsics_vec128_interleave_high64(x1, x2) \ |
| 224 |
(_mm_unpackhi_epi64(x1, x2)) |
| 225 |
|
| 226 |
#endif /* HACL_CAN_COMPILE_VEC128 */ |
| 227 |
|
| 228 |
#if defined(HACL_CAN_COMPILE_VEC256) |
| 229 |
|
| 230 |
#include <immintrin.h> |
| 231 |
|
| 232 |
typedef __m256i Lib_IntVector_Intrinsics_vec256; |
| 233 |
|
| 234 |
|
| 235 |
#define Lib_IntVector_Intrinsics_vec256_eq64(x0, x1) \ |
| 236 |
(_mm256_cmpeq_epi64(x0, x1)) |
| 237 |
|
| 238 |
#define Lib_IntVector_Intrinsics_vec256_eq32(x0, x1) \ |
| 239 |
(_mm256_cmpeq_epi32(x0, x1)) |
| 240 |
|
| 241 |
#define Lib_IntVector_Intrinsics_vec256_gt64(x0, x1) \ |
| 242 |
(_mm256_cmpgt_epi64(x0, x1)) |
| 243 |
|
| 244 |
#define Lib_IntVector_Intrinsics_vec256_gt32(x0, x1) \ |
| 245 |
(_mm256_cmpgt_epi32(x0, x1)) |
| 246 |
|
| 247 |
#define Lib_IntVector_Intrinsics_vec256_xor(x0, x1) \ |
| 248 |
(_mm256_xor_si256(x0, x1)) |
| 249 |
|
| 250 |
#define Lib_IntVector_Intrinsics_vec256_or(x0, x1) \ |
| 251 |
(_mm256_or_si256(x0, x1)) |
| 252 |
|
| 253 |
#define Lib_IntVector_Intrinsics_vec256_and(x0, x1) \ |
| 254 |
(_mm256_and_si256(x0, x1)) |
| 255 |
|
| 256 |
#define Lib_IntVector_Intrinsics_vec256_lognot(x0) \ |
| 257 |
(_mm256_xor_si256(x0, _mm256_set1_epi32(-1))) |
| 258 |
|
| 259 |
#define Lib_IntVector_Intrinsics_vec256_shift_left(x0, x1) \ |
| 260 |
(_mm256_slli_si256(x0, (x1)/8)) |
| 261 |
|
| 262 |
#define Lib_IntVector_Intrinsics_vec256_shift_right(x0, x1) \ |
| 263 |
(_mm256_srli_si256(x0, (x1)/8)) |
| 264 |
|
| 265 |
#define Lib_IntVector_Intrinsics_vec256_shift_left64(x0, x1) \ |
| 266 |
(_mm256_slli_epi64(x0, x1)) |
| 267 |
|
| 268 |
#define Lib_IntVector_Intrinsics_vec256_shift_right64(x0, x1) \ |
| 269 |
(_mm256_srli_epi64(x0, x1)) |
| 270 |
|
| 271 |
#define Lib_IntVector_Intrinsics_vec256_shift_left32(x0, x1) \ |
| 272 |
(_mm256_slli_epi32(x0, x1)) |
| 273 |
|
| 274 |
#define Lib_IntVector_Intrinsics_vec256_shift_right32(x0, x1) \ |
| 275 |
(_mm256_srli_epi32(x0, x1)) |
| 276 |
|
| 277 |
#define Lib_IntVector_Intrinsics_vec256_rotate_left32_8(x0) \ |
| 278 |
(_mm256_shuffle_epi8(x0, _mm256_set_epi8(14,13,12,15,10,9,8,11,6,5,4,7,2,1,0,3,14,13,12,15,10,9,8,11,6,5,4,7,2,1,0,3))) |
| 279 |
|
| 280 |
#define Lib_IntVector_Intrinsics_vec256_rotate_left32_16(x0) \ |
| 281 |
(_mm256_shuffle_epi8(x0, _mm256_set_epi8(13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2))) |
| 282 |
|
| 283 |
#define Lib_IntVector_Intrinsics_vec256_rotate_left32_24(x0) \ |
| 284 |
(_mm256_shuffle_epi8(x0, _mm256_set_epi8(12,15,14,13,8,11,10,9,4,7,6,5,0,3,2,1,12,15,14,13,8,11,10,9,4,7,6,5,0,3,2,1))) |
| 285 |
|
| 286 |
#define Lib_IntVector_Intrinsics_vec256_rotate_left32(x0,x1) \ |
| 287 |
((x1 == 8? Lib_IntVector_Intrinsics_vec256_rotate_left32_8(x0) : \ |
| 288 |
(x1 == 16? Lib_IntVector_Intrinsics_vec256_rotate_left32_16(x0) : \ |
| 289 |
(x1 == 24? Lib_IntVector_Intrinsics_vec256_rotate_left32_24(x0) : \ |
| 290 |
_mm256_or_si256(_mm256_slli_epi32(x0,x1),_mm256_srli_epi32(x0,32-(x1))))))) |
| 291 |
|
| 292 |
#define Lib_IntVector_Intrinsics_vec256_rotate_right32(x0,x1) \ |
| 293 |
(Lib_IntVector_Intrinsics_vec256_rotate_left32(x0,32-(x1))) |
| 294 |
|
| 295 |
#define Lib_IntVector_Intrinsics_vec256_rotate_right64_8(x0) \ |
| 296 |
(_mm256_shuffle_epi8(x0, _mm256_set_epi8(8,15,14,13,12,11,10,9,0,7,6,5,4,3,2,1,8,15,14,13,12,11,10,9,0,7,6,5,4,3,2,1))) |
| 297 |
|
| 298 |
#define Lib_IntVector_Intrinsics_vec256_rotate_right64_16(x0) \ |
| 299 |
(_mm256_shuffle_epi8(x0, _mm256_set_epi8(9,8,15,14,13,12,11,10,1,0,7,6,5,4,3,2,9,8,15,14,13,12,11,10,1,0,7,6,5,4,3,2))) |
| 300 |
|
| 301 |
#define Lib_IntVector_Intrinsics_vec256_rotate_right64_24(x0) \ |
| 302 |
(_mm256_shuffle_epi8(x0, _mm256_set_epi8(10,9,8,15,14,13,12,11,2,1,0,7,6,5,4,3,10,9,8,15,14,13,12,11,2,1,0,7,6,5,4,3))) |
| 303 |
|
| 304 |
#define Lib_IntVector_Intrinsics_vec256_rotate_right64_32(x0) \ |
| 305 |
(_mm256_shuffle_epi8(x0, _mm256_set_epi8(11,10,9,8,15,14,13,12,3,2,1,0,7,6,5,4,11,10,9,8,15,14,13,12,3,2,1,0,7,6,5,4))) |
| 306 |
|
| 307 |
#define Lib_IntVector_Intrinsics_vec256_rotate_right64_40(x0) \ |
| 308 |
(_mm256_shuffle_epi8(x0, _mm256_set_epi8(12,11,10,9,8,15,14,13,4,3,2,1,0,7,6,5,12,11,10,9,8,15,14,13,4,3,2,1,0,7,6,5))) |
| 309 |
|
| 310 |
#define Lib_IntVector_Intrinsics_vec256_rotate_right64_48(x0) \ |
| 311 |
(_mm256_shuffle_epi8(x0, _mm256_set_epi8(13,12,11,10,9,8,15,14,5,4,3,2,1,0,7,6,13,12,11,10,9,8,15,14,5,4,3,2,1,0,7,6))) |
| 312 |
|
| 313 |
#define Lib_IntVector_Intrinsics_vec256_rotate_right64_56(x0) \ |
| 314 |
(_mm256_shuffle_epi8(x0, _mm256_set_epi8(14,13,12,11,10,9,8,15,6,5,4,3,2,1,0,7,14,13,12,11,10,9,8,15,6,5,4,3,2,1,0,7))) |
| 315 |
|
| 316 |
#define Lib_IntVector_Intrinsics_vec256_rotate_right64(x0,x1) \ |
| 317 |
((x1 == 8? Lib_IntVector_Intrinsics_vec256_rotate_right64_8(x0) : \ |
| 318 |
(x1 == 16? Lib_IntVector_Intrinsics_vec256_rotate_right64_16(x0) : \ |
| 319 |
(x1 == 24? Lib_IntVector_Intrinsics_vec256_rotate_right64_24(x0) : \ |
| 320 |
(x1 == 32? Lib_IntVector_Intrinsics_vec256_rotate_right64_32(x0) : \ |
| 321 |
(x1 == 40? Lib_IntVector_Intrinsics_vec256_rotate_right64_40(x0) : \ |
| 322 |
(x1 == 48? Lib_IntVector_Intrinsics_vec256_rotate_right64_48(x0) : \ |
| 323 |
(x1 == 56? Lib_IntVector_Intrinsics_vec256_rotate_right64_56(x0) : \ |
| 324 |
_mm256_xor_si256(_mm256_srli_epi64((x0),(x1)),_mm256_slli_epi64((x0),(64-(x1)))))))))))) |
| 325 |
|
| 326 |
#define Lib_IntVector_Intrinsics_vec256_rotate_left64(x0,x1) \ |
| 327 |
(Lib_IntVector_Intrinsics_vec256_rotate_right64(x0,64-(x1))) |
| 328 |
|
| 329 |
#define Lib_IntVector_Intrinsics_vec256_shuffle64(x0, x1, x2, x3, x4) \ |
| 330 |
(_mm256_permute4x64_epi64(x0, _MM_SHUFFLE(x4,x3,x2,x1))) |
| 331 |
|
| 332 |
#define Lib_IntVector_Intrinsics_vec256_shuffle32(x0, x1, x2, x3, x4, x5, x6, x7, x8) \ |
| 333 |
(_mm256_permutevar8x32_epi32(x0, _mm256_set_epi32(x8,x7,x6,x5,x4,x3,x2,x1))) |
| 334 |
|
| 335 |
#define Lib_IntVector_Intrinsics_vec256_rotate_right_lanes32(x0, x1) \ |
| 336 |
(_mm256_permutevar8x32_epi32(x0, _mm256_set_epi32((x1+7)%8,(x1+6)%8,(x1+5)%8,(x1+4)%8,(x1+3%8),(x1+2)%8,(x1+1)%8,x1%8))) |
| 337 |
|
| 338 |
#define Lib_IntVector_Intrinsics_vec256_rotate_right_lanes64(x0, x1) \ |
| 339 |
(_mm256_permute4x64_epi64(x0, _MM_SHUFFLE((x1+3)%4,(x1+2)%4,(x1+1)%4,x1%4))) |
| 340 |
|
| 341 |
#define Lib_IntVector_Intrinsics_vec256_load32_le(x0) \ |
| 342 |
(_mm256_loadu_si256((__m256i*)(x0))) |
| 343 |
|
| 344 |
#define Lib_IntVector_Intrinsics_vec256_load64_le(x0) \ |
| 345 |
(_mm256_loadu_si256((__m256i*)(x0))) |
| 346 |
|
| 347 |
#define Lib_IntVector_Intrinsics_vec256_load32_be(x0) \ |
| 348 |
(_mm256_shuffle_epi8(_mm256_loadu_si256((__m256i*)(x0)), _mm256_set_epi8(12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3, 12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3))) |
| 349 |
|
| 350 |
#define Lib_IntVector_Intrinsics_vec256_load64_be(x0) \ |
| 351 |
(_mm256_shuffle_epi8(_mm256_loadu_si256((__m256i*)(x0)), _mm256_set_epi8(8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7))) |
| 352 |
|
| 353 |
|
| 354 |
#define Lib_IntVector_Intrinsics_vec256_store32_le(x0, x1) \ |
| 355 |
(_mm256_storeu_si256((__m256i*)(x0), x1)) |
| 356 |
|
| 357 |
#define Lib_IntVector_Intrinsics_vec256_store64_le(x0, x1) \ |
| 358 |
(_mm256_storeu_si256((__m256i*)(x0), x1)) |
| 359 |
|
| 360 |
#define Lib_IntVector_Intrinsics_vec256_store32_be(x0, x1) \ |
| 361 |
(_mm256_storeu_si256((__m256i*)(x0), _mm256_shuffle_epi8(x1, _mm256_set_epi8(12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3, 12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3)))) |
| 362 |
|
| 363 |
#define Lib_IntVector_Intrinsics_vec256_store64_be(x0, x1) \ |
| 364 |
(_mm256_storeu_si256((__m256i*)(x0), _mm256_shuffle_epi8(x1, _mm256_set_epi8(8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7)))) |
| 365 |
|
| 366 |
|
| 367 |
#define Lib_IntVector_Intrinsics_vec256_insert8(x0, x1, x2) \ |
| 368 |
(_mm256_insert_epi8(x0, x1, x2)) |
| 369 |
|
| 370 |
#define Lib_IntVector_Intrinsics_vec256_insert32(x0, x1, x2) \ |
| 371 |
(_mm256_insert_epi32(x0, x1, x2)) |
| 372 |
|
| 373 |
#define Lib_IntVector_Intrinsics_vec256_insert64(x0, x1, x2) \ |
| 374 |
(_mm256_insert_epi64(x0, x1, x2)) |
| 375 |
|
| 376 |
#define Lib_IntVector_Intrinsics_vec256_extract8(x0, x1) \ |
| 377 |
(_mm256_extract_epi8(x0, x1)) |
| 378 |
|
| 379 |
#define Lib_IntVector_Intrinsics_vec256_extract32(x0, x1) \ |
| 380 |
(_mm256_extract_epi32(x0, x1)) |
| 381 |
|
| 382 |
#define Lib_IntVector_Intrinsics_vec256_extract64(x0, x1) \ |
| 383 |
(_mm256_extract_epi64(x0, x1)) |
| 384 |
|
| 385 |
#define Lib_IntVector_Intrinsics_vec256_zero \ |
| 386 |
(_mm256_setzero_si256()) |
| 387 |
|
| 388 |
#define Lib_IntVector_Intrinsics_vec256_add64(x0, x1) \ |
| 389 |
(_mm256_add_epi64(x0, x1)) |
| 390 |
|
| 391 |
#define Lib_IntVector_Intrinsics_vec256_sub64(x0, x1) \ |
| 392 |
(_mm256_sub_epi64(x0, x1)) |
| 393 |
|
| 394 |
#define Lib_IntVector_Intrinsics_vec256_mul64(x0, x1) \ |
| 395 |
(_mm256_mul_epu32(x0, x1)) |
| 396 |
|
| 397 |
#define Lib_IntVector_Intrinsics_vec256_smul64(x0, x1) \ |
| 398 |
(_mm256_mul_epu32(x0, _mm256_set1_epi64x(x1))) |
| 399 |
|
| 400 |
|
| 401 |
#define Lib_IntVector_Intrinsics_vec256_add32(x0, x1) \ |
| 402 |
(_mm256_add_epi32(x0, x1)) |
| 403 |
|
| 404 |
#define Lib_IntVector_Intrinsics_vec256_sub32(x0, x1) \ |
| 405 |
(_mm256_sub_epi32(x0, x1)) |
| 406 |
|
| 407 |
#define Lib_IntVector_Intrinsics_vec256_mul32(x0, x1) \ |
| 408 |
(_mm256_mullo_epi32(x0, x1)) |
| 409 |
|
| 410 |
#define Lib_IntVector_Intrinsics_vec256_smul32(x0, x1) \ |
| 411 |
(_mm256_mullo_epi32(x0, _mm256_set1_epi32(x1))) |
| 412 |
|
| 413 |
|
| 414 |
#define Lib_IntVector_Intrinsics_vec256_load64(x1) \ |
| 415 |
(_mm256_set1_epi64x(x1)) /* hi lo */ |
| 416 |
|
| 417 |
#define Lib_IntVector_Intrinsics_vec256_load64s(x0, x1, x2, x3) \ |
| 418 |
(_mm256_set_epi64x(x3,x2,x1,x0)) /* hi lo */ |
| 419 |
|
| 420 |
#define Lib_IntVector_Intrinsics_vec256_load32(x) \ |
| 421 |
(_mm256_set1_epi32(x)) |
| 422 |
|
| 423 |
#define Lib_IntVector_Intrinsics_vec256_load32s(x0,x1,x2,x3,x4, x5, x6, x7) \ |
| 424 |
(_mm256_set_epi32(x7, x6, x5, x4, x3, x2, x1, x0)) /* hi lo */ |
| 425 |
|
| 426 |
#define Lib_IntVector_Intrinsics_vec256_load128(x) \ |
| 427 |
(_mm256_set_m128i((__m128i)x)) |
| 428 |
|
| 429 |
#define Lib_IntVector_Intrinsics_vec256_load128s(x0,x1) \ |
| 430 |
(_mm256_set_m128i((__m128i)x1,(__m128i)x0)) |
| 431 |
|
| 432 |
#define Lib_IntVector_Intrinsics_vec256_interleave_low32(x1, x2) \ |
| 433 |
(_mm256_unpacklo_epi32(x1, x2)) |
| 434 |
|
| 435 |
#define Lib_IntVector_Intrinsics_vec256_interleave_high32(x1, x2) \ |
| 436 |
(_mm256_unpackhi_epi32(x1, x2)) |
| 437 |
|
| 438 |
#define Lib_IntVector_Intrinsics_vec256_interleave_low64(x1, x2) \ |
| 439 |
(_mm256_unpacklo_epi64(x1, x2)) |
| 440 |
|
| 441 |
#define Lib_IntVector_Intrinsics_vec256_interleave_high64(x1, x2) \ |
| 442 |
(_mm256_unpackhi_epi64(x1, x2)) |
| 443 |
|
| 444 |
#define Lib_IntVector_Intrinsics_vec256_interleave_low128(x1, x2) \ |
| 445 |
(_mm256_permute2x128_si256(x1, x2, 0x20)) |
| 446 |
|
| 447 |
#define Lib_IntVector_Intrinsics_vec256_interleave_high128(x1, x2) \ |
| 448 |
(_mm256_permute2x128_si256(x1, x2, 0x31)) |
| 449 |
|
| 450 |
#endif /* HACL_CAN_COMPILE_VEC256 */ |
| 451 |
|
| 452 |
#elif (defined(__aarch64__) || defined(_M_ARM64) || defined(__arm__) || defined(_M_ARM)) \ |
| 453 |
&& !defined(__ARM_32BIT_STATE) |
| 454 |
|
| 455 |
#if defined(HACL_CAN_COMPILE_VEC128) |
| 456 |
|
| 457 |
#include <arm_neon.h> |
| 458 |
|
| 459 |
typedef uint32x4_t Lib_IntVector_Intrinsics_vec128; |
| 460 |
|
| 461 |
#define Lib_IntVector_Intrinsics_vec128_xor(x0, x1) \ |
| 462 |
(veorq_u32(x0,x1)) |
| 463 |
|
| 464 |
#define Lib_IntVector_Intrinsics_vec128_eq64(x0, x1) \ |
| 465 |
(vceqq_u32(x0,x1)) |
| 466 |
|
| 467 |
#define Lib_IntVector_Intrinsics_vec128_eq32(x0, x1) \ |
| 468 |
(vceqq_u32(x0,x1)) |
| 469 |
|
| 470 |
#define Lib_IntVector_Intrinsics_vec128_gt32(x0, x1) \ |
| 471 |
(vcgtq_u32(x0, x1)) |
| 472 |
|
| 473 |
#define high32(x0) \ |
| 474 |
(vmovn_u64(vshrq_n_u64(vreinterpretq_u64_u32(x0),32))) |
| 475 |
|
| 476 |
#define low32(x0) \ |
| 477 |
(vmovn_u64(vreinterpretq_u64_u32(x0))) |
| 478 |
|
| 479 |
#define Lib_IntVector_Intrinsics_vec128_gt64(x0, x1) \ |
| 480 |
(vreinterpretq_u32_u64(vmovl_u32(vorr_u32(vcgt_u32(high32(x0),high32(x1)),vand_u32(vceq_u32(high32(x0),high32(x1)),vcgt_u32(low32(x0),low32(x1))))))) |
| 481 |
|
| 482 |
#define Lib_IntVector_Intrinsics_vec128_or(x0, x1) \ |
| 483 |
(vorrq_u32(x0, x1)) |
| 484 |
|
| 485 |
#define Lib_IntVector_Intrinsics_vec128_and(x0, x1) \ |
| 486 |
(vandq_u32(x0, x1)) |
| 487 |
|
| 488 |
#define Lib_IntVector_Intrinsics_vec128_lognot(x0) \ |
| 489 |
(vmvnq_u32(x0)) |
| 490 |
|
| 491 |
|
| 492 |
#define Lib_IntVector_Intrinsics_vec128_shift_left(x0, x1) \ |
| 493 |
(vextq_u32(x0, vdupq_n_u8(0), 16-(x1)/8)) |
| 494 |
|
| 495 |
#define Lib_IntVector_Intrinsics_vec128_shift_right(x0, x1) \ |
| 496 |
(vextq_u32(x0, vdupq_n_u8(0), (x1)/8)) |
| 497 |
|
| 498 |
#define Lib_IntVector_Intrinsics_vec128_shift_left64(x0, x1) \ |
| 499 |
(vreinterpretq_u32_u64(vshlq_n_u64(vreinterpretq_u64_u32(x0), x1))) |
| 500 |
|
| 501 |
#define Lib_IntVector_Intrinsics_vec128_shift_right64(x0, x1) \ |
| 502 |
(vreinterpretq_u32_u64(vshrq_n_u64(vreinterpretq_u64_u32(x0), x1))) |
| 503 |
|
| 504 |
#define Lib_IntVector_Intrinsics_vec128_shift_left32(x0, x1) \ |
| 505 |
(vshlq_n_u32(x0, x1)) |
| 506 |
|
| 507 |
#define Lib_IntVector_Intrinsics_vec128_shift_right32(x0, x1) \ |
| 508 |
(vshrq_n_u32(x0, x1)) |
| 509 |
|
| 510 |
#define Lib_IntVector_Intrinsics_vec128_rotate_left32_16(x1) \ |
| 511 |
(vreinterpretq_u32_u16(vrev32q_u16(vreinterpretq_u16_u32(x1)))) |
| 512 |
|
| 513 |
#define Lib_IntVector_Intrinsics_vec128_rotate_left32(x0,x1) \ |
| 514 |
(((x1) == 16? Lib_IntVector_Intrinsics_vec128_rotate_left32_16(x0) : \ |
| 515 |
vsriq_n_u32(vshlq_n_u32((x0),(x1)),(x0),32-(x1)))) |
| 516 |
|
| 517 |
#define Lib_IntVector_Intrinsics_vec128_rotate_right32_16(x1) \ |
| 518 |
(vreinterpretq_u32_u16(vrev32q_u16(vreinterpretq_u16_u32(x1)))) |
| 519 |
|
| 520 |
#define Lib_IntVector_Intrinsics_vec128_rotate_right32(x0,x1) \ |
| 521 |
(((x1) == 16? Lib_IntVector_Intrinsics_vec128_rotate_right32_16(x0) : \ |
| 522 |
vsriq_n_u32(vshlq_n_u32((x0),32-(x1)),(x0),(x1)))) |
| 523 |
|
| 524 |
#define Lib_IntVector_Intrinsics_vec128_rotate_right_lanes32(x0, x1) \ |
| 525 |
(vextq_u32(x0,x0,x1)) |
| 526 |
|
| 527 |
#define Lib_IntVector_Intrinsics_vec128_rotate_right_lanes64(x0, x1) \ |
| 528 |
(vextq_u64(x0,x0,x1)) |
| 529 |
|
| 530 |
|
| 531 |
/* |
| 532 |
#define Lib_IntVector_Intrinsics_vec128_shuffle32(x0, x1, x2, x3, x4) \ |
| 533 |
(_mm_shuffle_epi32(x0, _MM_SHUFFLE(x1,x2,x3,x4))) |
| 534 |
|
| 535 |
#define Lib_IntVector_Intrinsics_vec128_shuffle64(x0, x1, x2) \ |
| 536 |
(_mm_shuffle_epi32(x0, _MM_SHUFFLE(2*x1+1,2*x1,2*x2+1,2*x2))) |
| 537 |
*/ |
| 538 |
|
| 539 |
#define Lib_IntVector_Intrinsics_vec128_load32_le(x0) \ |
| 540 |
(vld1q_u32((const uint32_t*) (x0))) |
| 541 |
|
| 542 |
#define Lib_IntVector_Intrinsics_vec128_load64_le(x0) \ |
| 543 |
(vld1q_u32((const uint32_t*) (x0))) |
| 544 |
|
| 545 |
#define Lib_IntVector_Intrinsics_vec128_store32_le(x0, x1) \ |
| 546 |
(vst1q_u32((uint32_t*)(x0),(x1))) |
| 547 |
|
| 548 |
#define Lib_IntVector_Intrinsics_vec128_store64_le(x0, x1) \ |
| 549 |
(vst1q_u32((uint32_t*)(x0),(x1))) |
| 550 |
|
| 551 |
/* |
| 552 |
#define Lib_IntVector_Intrinsics_vec128_load_be(x0) \ |
| 553 |
( Lib_IntVector_Intrinsics_vec128 l = vrev64q_u8(vld1q_u32((uint32_t*)(x0))); |
| 554 |
|
| 555 |
*/ |
| 556 |
|
| 557 |
#define Lib_IntVector_Intrinsics_vec128_load32_be(x0) \ |
| 558 |
(vreinterpretq_u32_u8(vrev32q_u8(vreinterpretq_u8_u32(vld1q_u32((const uint32_t*)(x0)))))) |
| 559 |
|
| 560 |
#define Lib_IntVector_Intrinsics_vec128_load64_be(x0) \ |
| 561 |
(vreinterpretq_u32_u8(vrev64q_u8(vreinterpretq_u8_u32(vld1q_u32((const uint32_t*)(x0)))))) |
| 562 |
|
| 563 |
/* |
| 564 |
#define Lib_IntVector_Intrinsics_vec128_store_be(x0, x1) \ |
| 565 |
(_mm_storeu_si128((__m128i*)(x0), _mm_shuffle_epi8(x1, _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)))) |
| 566 |
*/ |
| 567 |
|
| 568 |
#define Lib_IntVector_Intrinsics_vec128_store32_be(x0, x1) \ |
| 569 |
(vst1q_u32((uint32_t*)(x0),(vreinterpretq_u32_u8(vrev32q_u8(vreinterpretq_u8_u32(x1)))))) |
| 570 |
|
| 571 |
#define Lib_IntVector_Intrinsics_vec128_store64_be(x0, x1) \ |
| 572 |
(vst1q_u32((uint32_t*)(x0),(vreinterpretq_u32_u8(vrev64q_u8(vreinterpretq_u8_u32(x1)))))) |
| 573 |
|
| 574 |
#define Lib_IntVector_Intrinsics_vec128_insert8(x0, x1, x2) \ |
| 575 |
(vsetq_lane_u8(x1,x0,x2)) |
| 576 |
|
| 577 |
#define Lib_IntVector_Intrinsics_vec128_insert32(x0, x1, x2) \ |
| 578 |
(vsetq_lane_u32(x1,x0,x2)) |
| 579 |
|
| 580 |
#define Lib_IntVector_Intrinsics_vec128_insert64(x0, x1, x2) \ |
| 581 |
(vreinterpretq_u32_u64(vsetq_lane_u64(x1,vreinterpretq_u64_u32(x0),x2))) |
| 582 |
|
| 583 |
#define Lib_IntVector_Intrinsics_vec128_extract8(x0, x1) \ |
| 584 |
(vgetq_lane_u8(x0,x1)) |
| 585 |
|
| 586 |
#define Lib_IntVector_Intrinsics_vec128_extract32(x0, x1) \ |
| 587 |
(vgetq_lane_u32(x0,x1)) |
| 588 |
|
| 589 |
#define Lib_IntVector_Intrinsics_vec128_extract64(x0, x1) \ |
| 590 |
(vgetq_lane_u64(vreinterpretq_u64_u32(x0),x1)) |
| 591 |
|
| 592 |
#define Lib_IntVector_Intrinsics_vec128_zero \ |
| 593 |
(vdupq_n_u32(0)) |
| 594 |
|
| 595 |
#define Lib_IntVector_Intrinsics_vec128_add64(x0, x1) \ |
| 596 |
(vreinterpretq_u32_u64(vaddq_u64(vreinterpretq_u64_u32(x0), vreinterpretq_u64_u32(x1)))) |
| 597 |
|
| 598 |
#define Lib_IntVector_Intrinsics_vec128_sub64(x0, x1) \ |
| 599 |
(vreinterpretq_u32_u64(vsubq_u64(vreinterpretq_u64_u32(x0), vreinterpretq_u64_u32(x1)))) |
| 600 |
|
| 601 |
#define Lib_IntVector_Intrinsics_vec128_mul64(x0, x1) \ |
| 602 |
(vreinterpretq_u32_u64(vmull_u32(vmovn_u64(vreinterpretq_u64_u32(x0)), vmovn_u64(vreinterpretq_u64_u32(x1))))) |
| 603 |
|
| 604 |
#define Lib_IntVector_Intrinsics_vec128_smul64(x0, x1) \ |
| 605 |
(vreinterpretq_u32_u64(vmull_n_u32(vmovn_u64(vreinterpretq_u64_u32(x0)), (uint32_t)x1))) |
| 606 |
|
| 607 |
#define Lib_IntVector_Intrinsics_vec128_add32(x0, x1) \ |
| 608 |
(vaddq_u32(x0, x1)) |
| 609 |
|
| 610 |
#define Lib_IntVector_Intrinsics_vec128_sub32(x0, x1) \ |
| 611 |
(vsubq_u32(x0, x1)) |
| 612 |
|
| 613 |
#define Lib_IntVector_Intrinsics_vec128_mul32(x0, x1) \ |
| 614 |
(vmulq_lane_u32(x0, x1)) |
| 615 |
|
| 616 |
#define Lib_IntVector_Intrinsics_vec128_smul32(x0, x1) \ |
| 617 |
(vmulq_lane_u32(x0, vdupq_n_u32(x1))) |
| 618 |
|
| 619 |
#define Lib_IntVector_Intrinsics_vec128_load128(x) \ |
| 620 |
((uint32x4_t)(x)) |
| 621 |
|
| 622 |
#define Lib_IntVector_Intrinsics_vec128_load64(x) \ |
| 623 |
(vreinterpretq_u32_u64(vdupq_n_u64(x))) /* hi lo */ |
| 624 |
|
| 625 |
#define Lib_IntVector_Intrinsics_vec128_load32(x) \ |
| 626 |
(vdupq_n_u32(x)) /* hi lo */ |
| 627 |
|
| 628 |
static inline Lib_IntVector_Intrinsics_vec128 Lib_IntVector_Intrinsics_vec128_load64s(uint64_t x1, uint64_t x2){ |
| 629 |
const uint64_t a[2] = {x1,x2}; |
| 630 |
return vreinterpretq_u32_u64(vld1q_u64(a)); |
| 631 |
} |
| 632 |
|
| 633 |
static inline Lib_IntVector_Intrinsics_vec128 Lib_IntVector_Intrinsics_vec128_load32s(uint32_t x1, uint32_t x2, uint32_t x3, uint32_t x4){ |
| 634 |
const uint32_t a[4] = {x1,x2,x3,x4}; |
| 635 |
return vld1q_u32(a); |
| 636 |
} |
| 637 |
|
| 638 |
#define Lib_IntVector_Intrinsics_vec128_interleave_low32(x1, x2) \ |
| 639 |
(vzip1q_u32(x1,x2)) |
| 640 |
|
| 641 |
#define Lib_IntVector_Intrinsics_vec128_interleave_high32(x1, x2) \ |
| 642 |
(vzip2q_u32(x1,x2)) |
| 643 |
|
| 644 |
#define Lib_IntVector_Intrinsics_vec128_interleave_low64(x1,x2) \ |
| 645 |
(vreinterpretq_u32_u64(vzip1q_u64(vreinterpretq_u64_u32(x1),vreinterpretq_u64_u32(x2)))) |
| 646 |
|
| 647 |
#define Lib_IntVector_Intrinsics_vec128_interleave_high64(x1,x2) \ |
| 648 |
(vreinterpretq_u32_u64(vzip2q_u64(vreinterpretq_u64_u32(x1),vreinterpretq_u64_u32(x2)))) |
| 649 |
|
| 650 |
#endif /* HACL_CAN_COMPILE_VEC128 */ |
| 651 |
|
| 652 |
/* IBM z architecture */ |
| 653 |
#elif defined(__s390x__) /* this flag is for GCC only */ |
| 654 |
|
| 655 |
#if defined(HACL_CAN_COMPILE_VEC128) |
| 656 |
|
| 657 |
#include <stdint.h> |
| 658 |
#include <vecintrin.h> |
| 659 |
|
| 660 |
/* The main vector 128 type |
| 661 |
* We can't use uint8_t, uint32_t, uint64_t... instead of unsigned char, |
| 662 |
* unsigned int, unsigned long long: the compiler complains that the parameter |
| 663 |
* combination is invalid. */ |
| 664 |
typedef unsigned char vector128_8 __attribute__ ((vector_size(16))); |
| 665 |
typedef unsigned int vector128_32 __attribute__ ((vector_size(16))); |
| 666 |
typedef unsigned long long vector128_64 __attribute__ ((vector_size(16))); |
| 667 |
|
| 668 |
typedef vector128_8 Lib_IntVector_Intrinsics_vec128; |
| 669 |
typedef vector128_8 vector128; |
| 670 |
|
| 671 |
#define Lib_IntVector_Intrinsics_vec128_load32_le(x) \ |
| 672 |
(vector128) ((vector128_32) vec_revb(*((vector128_32*) (const uint8_t*)(x)))) |
| 673 |
|
| 674 |
#define Lib_IntVector_Intrinsics_vec128_load32_be(x) \ |
| 675 |
(vector128) (*((vector128_32*) (const uint8_t*)(x))) |
| 676 |
|
| 677 |
#define Lib_IntVector_Intrinsics_vec128_load64_le(x) \ |
| 678 |
(vector128) ((vector128_64) vec_revb(*((vector128_64*) (const uint8_t*)(x)))) |
| 679 |
|
| 680 |
static inline |
| 681 |
void Lib_IntVector_Intrinsics_vec128_store32_le(const uint8_t *x0, vector128 x1) { |
| 682 |
*((vector128_32*)x0) = vec_revb((vector128_32) x1); |
| 683 |
} |
| 684 |
|
| 685 |
static inline |
| 686 |
void Lib_IntVector_Intrinsics_vec128_store32_be(const uint8_t *x0, vector128 x1) { |
| 687 |
*((vector128_32*)x0) = (vector128_32) x1; |
| 688 |
} |
| 689 |
|
| 690 |
static inline |
| 691 |
void Lib_IntVector_Intrinsics_vec128_store64_le(const uint8_t *x0, vector128 x1) { |
| 692 |
*((vector128_64*)x0) = vec_revb((vector128_64) x1); |
| 693 |
} |
| 694 |
|
| 695 |
#define Lib_IntVector_Intrinsics_vec128_add32(x0,x1) \ |
| 696 |
((vector128)((vector128_32)(((vector128_32)(x0)) + ((vector128_32)(x1))))) |
| 697 |
|
| 698 |
#define Lib_IntVector_Intrinsics_vec128_add64(x0, x1) \ |
| 699 |
((vector128)((vector128_64)(((vector128_64)(x0)) + ((vector128_64)(x1))))) |
| 700 |
|
| 701 |
#define Lib_IntVector_Intrinsics_vec128_and(x0, x1) \ |
| 702 |
((vector128)(vec_and((vector128)(x0),(vector128)(x1)))) |
| 703 |
|
| 704 |
#define Lib_IntVector_Intrinsics_vec128_eq32(x0, x1) \ |
| 705 |
((vector128)(vec_cmpeq(((vector128_32)(x0)),((vector128_32)(x1))))) |
| 706 |
|
| 707 |
#define Lib_IntVector_Intrinsics_vec128_eq64(x0, x1) \ |
| 708 |
((vector128)(vec_cmpeq(((vector128_64)(x0)),((vector128_64)(x1))))) |
| 709 |
|
| 710 |
#define Lib_IntVector_Intrinsics_vec128_extract32(x0, x1) \ |
| 711 |
((unsigned int)(vec_extract((vector128_32)(x0), x1))) |
| 712 |
|
| 713 |
#define Lib_IntVector_Intrinsics_vec128_extract64(x0, x1) \ |
| 714 |
((unsigned long long)(vec_extract((vector128_64)(x0), x1))) |
| 715 |
|
| 716 |
#define Lib_IntVector_Intrinsics_vec128_gt32(x0, x1) \ |
| 717 |
((vector128)((vector128_32)(((vector128_32)(x0)) > ((vector128_32)(x1))))) |
| 718 |
|
| 719 |
#define Lib_IntVector_Intrinsics_vec128_gt64(x0, x1) \ |
| 720 |
((vector128)((vector128_64)(((vector128_64)(x0)) > ((vector128_64)(x1))))) |
| 721 |
|
| 722 |
#define Lib_IntVector_Intrinsics_vec128_insert32(x0, x1, x2) \ |
| 723 |
((vector128)((vector128_32)vec_insert((unsigned int)(x1), (vector128_32)(x0), x2))) |
| 724 |
|
| 725 |
#define Lib_IntVector_Intrinsics_vec128_insert64(x0, x1, x2) \ |
| 726 |
((vector128)((vector128_64)vec_insert((unsigned long long)(x1), (vector128_64)(x0), x2))) |
| 727 |
|
| 728 |
#define Lib_IntVector_Intrinsics_vec128_interleave_high32(x0, x1) \ |
| 729 |
((vector128)((vector128_32)vec_mergel((vector128_32)(x0), (vector128_32)(x1)))) |
| 730 |
|
| 731 |
#define Lib_IntVector_Intrinsics_vec128_interleave_high64(x0, x1) \ |
| 732 |
((vector128)((vector128_64)vec_mergel((vector128_64)(x0), (vector128_64)(x1)))) |
| 733 |
|
| 734 |
#define Lib_IntVector_Intrinsics_vec128_interleave_low32(x0, x1) \ |
| 735 |
((vector128)((vector128_32)vec_mergeh((vector128_32)(x0), (vector128_32)(x1)))) |
| 736 |
|
| 737 |
#define Lib_IntVector_Intrinsics_vec128_interleave_low64(x0, x1) \ |
| 738 |
((vector128)((vector128_64)vec_mergeh((vector128_64)(x0), (vector128_64)(x1)))) |
| 739 |
|
| 740 |
#define Lib_IntVector_Intrinsics_vec128_load32(x) \ |
| 741 |
((vector128)((vector128_32){(unsigned int)(x), (unsigned int)(x), \ |
| 742 |
(unsigned int)(x), (unsigned int)(x)})) |
| 743 |
|
| 744 |
#define Lib_IntVector_Intrinsics_vec128_load32s(x0, x1, x2, x3) \ |
| 745 |
((vector128)((vector128_32){(unsigned int)(x0),(unsigned int)(x1),(unsigned int)(x2),(unsigned int)(x3)})) |
| 746 |
|
| 747 |
#define Lib_IntVector_Intrinsics_vec128_load64(x) \ |
| 748 |
((vector128)((vector128_64)vec_load_pair((unsigned long long)(x),(unsigned long long)(x)))) |
| 749 |
|
| 750 |
#define Lib_IntVector_Intrinsics_vec128_lognot(x0) \ |
| 751 |
((vector128)(vec_xor((vector128)(x0), (vector128)vec_splat_u32(-1)))) |
| 752 |
|
| 753 |
#define Lib_IntVector_Intrinsics_vec128_mul64(x0, x1) \ |
| 754 |
((vector128)(vec_mulo((vector128_32)(x0), \ |
| 755 |
(vector128_32)(x1)))) |
| 756 |
|
| 757 |
#define Lib_IntVector_Intrinsics_vec128_or(x0, x1) \ |
| 758 |
((vector128)(vec_or((vector128)(x0),(vector128)(x1)))) |
| 759 |
|
| 760 |
#define Lib_IntVector_Intrinsics_vec128_rotate_left32(x0, x1) \ |
| 761 |
((vector128)(vec_rli((vector128_32)(x0), (unsigned long)(x1)))) |
| 762 |
|
| 763 |
#define Lib_IntVector_Intrinsics_vec128_rotate_right32(x0, x1) \ |
| 764 |
(Lib_IntVector_Intrinsics_vec128_rotate_left32(x0,(uint32_t)(32-(x1)))) |
| 765 |
|
| 766 |
#define Lib_IntVector_Intrinsics_vec128_rotate_right_lanes32(x0, x1) \ |
| 767 |
((vector128)(vec_sld((vector128)(x0), (vector128)(x0), (x1%4)*4))) |
| 768 |
|
| 769 |
#define Lib_IntVector_Intrinsics_vec128_shift_left64(x0, x1) \ |
| 770 |
(((vector128)((vector128_64)vec_rli((vector128_64)(x0), (unsigned long)(x1)))) & \ |
| 771 |
((vector128)((vector128_64){0xffffffffffffffff << (x1), 0xffffffffffffffff << (x1)}))) |
| 772 |
|
| 773 |
#define Lib_IntVector_Intrinsics_vec128_shift_right64(x0, x1) \ |
| 774 |
(((vector128)((vector128_64)vec_rli((vector128_64)(x0), (unsigned long)(64-(x1))))) & \ |
| 775 |
((vector128)((vector128_64){0xffffffffffffffff >> (x1), 0xffffffffffffffff >> (x1)}))) |
| 776 |
|
| 777 |
#define Lib_IntVector_Intrinsics_vec128_shift_right32(x0, x1) \ |
| 778 |
(((vector128)((vector128_32)vec_rli((vector128_32)(x0), (unsigned int)(32-(x1))))) & \ |
| 779 |
((vector128)((vector128_32){0xffffffff >> (x1), 0xffffffff >> (x1), \ |
| 780 |
0xffffffff >> (x1), 0xffffffff >> (x1)}))) |
| 781 |
|
| 782 |
/* Doesn't work with vec_splat_u64 */ |
| 783 |
#define Lib_IntVector_Intrinsics_vec128_smul64(x0, x1) \ |
| 784 |
((vector128)(Lib_IntVector_Intrinsics_vec128_mul64(x0,((vector128_64){(unsigned long long)(x1),(unsigned long long)(x1)})))) |
| 785 |
|
| 786 |
#define Lib_IntVector_Intrinsics_vec128_sub64(x0, x1) \ |
| 787 |
((vector128)((vector128_64)(x0) - (vector128_64)(x1))) |
| 788 |
|
| 789 |
static inline |
| 790 |
vector128 Lib_IntVector_Intrinsics_vec128_xor(vector128 x0, vector128 x1) { |
| 791 |
return ((vector128)(vec_xor((vector128)(x0), (vector128)(x1)))); |
| 792 |
} |
| 793 |
|
| 794 |
|
| 795 |
#define Lib_IntVector_Intrinsics_vec128_zero \ |
| 796 |
((vector128){}) |
| 797 |
|
| 798 |
#endif /* HACL_CAN_COMPILE_VEC128 */ |
| 799 |
|
| 800 |
#elif defined(__powerpc64__) // PowerPC 64 - this flag is for GCC only |
| 801 |
|
| 802 |
#if defined(HACL_CAN_COMPILE_VEC128) |
| 803 |
|
| 804 |
#include <altivec.h> |
| 805 |
#include <string.h> // for memcpy |
| 806 |
#include <stdint.h> |
| 807 |
|
| 808 |
// The main vector 128 type |
| 809 |
// We can't use uint8_t, uint32_t, uint64_t... instead of unsigned char, |
| 810 |
// unsigned int, unsigned long long: the compiler complains that the parameter |
| 811 |
// combination is invalid. |
| 812 |
typedef vector unsigned char vector128_8; |
| 813 |
typedef vector unsigned int vector128_32; |
| 814 |
typedef vector unsigned long long vector128_64; |
| 815 |
|
| 816 |
typedef vector128_8 Lib_IntVector_Intrinsics_vec128; |
| 817 |
typedef vector128_8 vector128; |
| 818 |
|
| 819 |
#define Lib_IntVector_Intrinsics_vec128_load32_le(x) \ |
| 820 |
((vector128)((vector128_32)(vec_xl(0, (const unsigned int*) ((const uint8_t*)(x)))))) |
| 821 |
|
| 822 |
#define Lib_IntVector_Intrinsics_vec128_load64_le(x) \ |
| 823 |
((vector128)((vector128_64)(vec_xl(0, (const unsigned long long*) ((const uint8_t*)(x)))))) |
| 824 |
|
| 825 |
#define Lib_IntVector_Intrinsics_vec128_store32_le(x0, x1) \ |
| 826 |
(vec_xst((vector128_32)(x1), 0, (unsigned int*) ((uint8_t*)(x0)))) |
| 827 |
|
| 828 |
#define Lib_IntVector_Intrinsics_vec128_store64_le(x0, x1) \ |
| 829 |
(vec_xst((vector128_64)(x1), 0, (unsigned long long*) ((uint8_t*)(x0)))) |
| 830 |
|
| 831 |
#define Lib_IntVector_Intrinsics_vec128_add32(x0,x1) \ |
| 832 |
((vector128)((vector128_32)(((vector128_32)(x0)) + ((vector128_32)(x1))))) |
| 833 |
|
| 834 |
#define Lib_IntVector_Intrinsics_vec128_add64(x0, x1) \ |
| 835 |
((vector128)((vector128_64)(((vector128_64)(x0)) + ((vector128_64)(x1))))) |
| 836 |
|
| 837 |
#define Lib_IntVector_Intrinsics_vec128_and(x0, x1) \ |
| 838 |
((vector128)(vec_and((vector128)(x0),(vector128)(x1)))) |
| 839 |
|
| 840 |
#define Lib_IntVector_Intrinsics_vec128_eq32(x0, x1) \ |
| 841 |
((vector128)(vec_cmpeq(((vector128_32)(x0)),((vector128_32)(x1))))) |
| 842 |
|
| 843 |
#define Lib_IntVector_Intrinsics_vec128_eq64(x0, x1) \ |
| 844 |
((vector128)(vec_cmpeq(((vector128_64)(x0)),((vector128_64)(x1))))) |
| 845 |
|
| 846 |
#define Lib_IntVector_Intrinsics_vec128_extract32(x0, x1) \ |
| 847 |
((unsigned int)(vec_extract((vector128_32)(x0), x1))) |
| 848 |
|
| 849 |
#define Lib_IntVector_Intrinsics_vec128_extract64(x0, x1) \ |
| 850 |
((unsigned long long)(vec_extract((vector128_64)(x0), x1))) |
| 851 |
|
| 852 |
#define Lib_IntVector_Intrinsics_vec128_gt32(x0, x1) \ |
| 853 |
((vector128)((vector128_32)(((vector128_32)(x0)) > ((vector128_32)(x1))))) |
| 854 |
|
| 855 |
#define Lib_IntVector_Intrinsics_vec128_gt64(x0, x1) \ |
| 856 |
((vector128)((vector128_64)(((vector128_64)(x0)) > ((vector128_64)(x1))))) |
| 857 |
|
| 858 |
#define Lib_IntVector_Intrinsics_vec128_insert32(x0, x1, x2) \ |
| 859 |
((vector128)((vector128_32)vec_insert((unsigned int)(x1), (vector128_32)(x0), x2))) |
| 860 |
|
| 861 |
#define Lib_IntVector_Intrinsics_vec128_insert64(x0, x1, x2) \ |
| 862 |
((vector128)((vector128_64)vec_insert((unsigned long long)(x1), (vector128_64)(x0), x2))) |
| 863 |
|
| 864 |
#define Lib_IntVector_Intrinsics_vec128_interleave_high32(x0, x1) \ |
| 865 |
((vector128)((vector128_32)vec_mergel((vector128_32)(x0), (vector128_32)(x1)))) |
| 866 |
|
| 867 |
#define Lib_IntVector_Intrinsics_vec128_interleave_high64(x0, x1) \ |
| 868 |
((vector128)((vector128_64)vec_mergel((vector128_64)(x0), (vector128_64)(x1)))) |
| 869 |
|
| 870 |
#define Lib_IntVector_Intrinsics_vec128_interleave_low32(x0, x1) \ |
| 871 |
((vector128)((vector128_32)vec_mergeh((vector128_32)(x0), (vector128_32)(x1)))) |
| 872 |
|
| 873 |
#define Lib_IntVector_Intrinsics_vec128_interleave_low64(x0, x1) \ |
| 874 |
((vector128)((vector128_64)vec_mergeh((vector128_64)(x0), (vector128_64)(x1)))) |
| 875 |
|
| 876 |
#define Lib_IntVector_Intrinsics_vec128_load32(x) \ |
| 877 |
((vector128)((vector128_32){(unsigned int)(x), (unsigned int)(x), \ |
| 878 |
(unsigned int)(x), (unsigned int)(x)})) |
| 879 |
|
| 880 |
#define Lib_IntVector_Intrinsics_vec128_load32s(x0, x1, x2, x3) \ |
| 881 |
((vector128)((vector128_32){(unsigned int)(x0),(unsigned int)(x1),(unsigned int)(x2),(unsigned int)(x3)})) |
| 882 |
|
| 883 |
#define Lib_IntVector_Intrinsics_vec128_load64(x) \ |
| 884 |
((vector128)((vector128_64){(unsigned long long)(x),(unsigned long long)(x)})) |
| 885 |
|
| 886 |
#define Lib_IntVector_Intrinsics_vec128_lognot(x0) \ |
| 887 |
((vector128)(vec_xor((vector128)(x0), (vector128)vec_splat_u32(-1)))) |
| 888 |
|
| 889 |
#define Lib_IntVector_Intrinsics_vec128_mul64(x0, x1) \ |
| 890 |
((vector128)(vec_mule((vector128_32)(x0), \ |
| 891 |
(vector128_32)(x1)))) |
| 892 |
|
| 893 |
#define Lib_IntVector_Intrinsics_vec128_or(x0, x1) \ |
| 894 |
((vector128)(vec_or((vector128)(x0),(vector128)(x1)))) |
| 895 |
|
| 896 |
#define Lib_IntVector_Intrinsics_vec128_rotate_left32(x0, x1) \ |
| 897 |
((vector128)(vec_rl((vector128_32)(x0), (vector128_32){(unsigned int)(x1),(unsigned int)(x1),(unsigned int)(x1),(unsigned int)(x1)}))) |
| 898 |
|
| 899 |
#define Lib_IntVector_Intrinsics_vec128_rotate_right32(x0, x1) \ |
| 900 |
(Lib_IntVector_Intrinsics_vec128_rotate_left32(x0,(uint32_t)(32-(x1)))) |
| 901 |
|
| 902 |
#define Lib_IntVector_Intrinsics_vec128_rotate_right_lanes32(x0, x1) \ |
| 903 |
((vector128)(vec_sld((vector128)(x0), (vector128)(x0), ((4-(x1))%4)*4))) |
| 904 |
|
| 905 |
#define Lib_IntVector_Intrinsics_vec128_shift_left64(x0, x1) \ |
| 906 |
((vector128)((vector128_64)vec_sl((vector128_64)(x0), (vector128_64){(unsigned long)(x1),(unsigned long)(x1)}))) |
| 907 |
|
| 908 |
#define Lib_IntVector_Intrinsics_vec128_shift_right64(x0, x1) \ |
| 909 |
((vector128)((vector128_64)vec_sr((vector128_64)(x0), (vector128_64){(unsigned long)(x1),(unsigned long)(x1)}))) |
| 910 |
|
| 911 |
// Doesn't work with vec_splat_u64 |
| 912 |
#define Lib_IntVector_Intrinsics_vec128_smul64(x0, x1) \ |
| 913 |
((vector128)(Lib_IntVector_Intrinsics_vec128_mul64(x0,((vector128_64){(unsigned long long)(x1),(unsigned long long)(x1)})))) |
| 914 |
|
| 915 |
#define Lib_IntVector_Intrinsics_vec128_sub64(x0, x1) \ |
| 916 |
((vector128)((vector128_64)(x0) - (vector128_64)(x1))) |
| 917 |
|
| 918 |
#define Lib_IntVector_Intrinsics_vec128_xor(x0, x1) \ |
| 919 |
((vector128)(vec_xor((vector128)(x0), (vector128)(x1)))) |
| 920 |
|
| 921 |
#define Lib_IntVector_Intrinsics_vec128_zero \ |
| 922 |
((vector128){}) |
| 923 |
|
| 924 |
#endif /* HACL_CAN_COMPILE_VEC128 */ |
| 925 |
|
| 926 |
#endif // PowerPC64 |
| 927 |
|
| 928 |
// DEBUGGING: |
| 929 |
// If libintvector_debug.h exists, use it to debug the current implementations. |
| 930 |
// Note that some flags must be enabled for the debugging to be effective: |
| 931 |
// see libintvector_debug.h for more details. |
| 932 |
#if defined(__has_include) |
| 933 |
#if __has_include("libintvector_debug.h") |
| 934 |
#include "libintvector_debug.h" |
| 935 |
#endif |
| 936 |
#endif |
| 937 |
|
| 938 |
#endif // HACL_INTRINSICS_SHIMMED |
| 939 |
|
| 940 |
#endif // __Vec_Intrin_H |