Debian Codegraph @

.cargo/vendor/encoding_rs/src/simd_funcs.rs

raw
1 // Copyright Mozilla Foundation. See the COPYRIGHT
2 // file at the top-level directory of this distribution.
3 //
4 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
5 // https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
6 // <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
7 // option. This file may not be copied, modified, or distributed
8 // except according to those terms.
9
10 use any_all_workaround::all_mask16x8;
11 use any_all_workaround::all_mask8x16;
12 use any_all_workaround::any_mask16x8;
13 use any_all_workaround::any_mask8x16;
14 use core::simd::cmp::SimdPartialEq;
15 use core::simd::cmp::SimdPartialOrd;
16 use core::simd::mask16x8;
17 use core::simd::mask8x16;
18 use core::simd::simd_swizzle;
19 use core::simd::u16x8;
20 use core::simd::u8x16;
21 use core::simd::ToBytes;
22
23 // TODO: Migrate unaligned access to stdlib code if/when the RFC
24 // https://github.com/rust-lang/rfcs/pull/1725 is implemented.
25
26 /// Safety invariant: ptr must be valid for an unaligned read of 16 bytes
27 #[inline(always)]
28 pub unsafe fn load16_unaligned(ptr: *const u8) -> u8x16 {
29 let mut simd = ::core::mem::MaybeUninit::<u8x16>::uninit();
30 ::core::ptr::copy_nonoverlapping(ptr, simd.as_mut_ptr() as *mut u8, 16);
31 // Safety: copied 16 bytes of initialized memory into this, it is now initialized
32 simd.assume_init()
33 }
34
35 /// Safety invariant: ptr must be valid for an aligned-for-u8x16 read of 16 bytes
36 #[allow(dead_code)]
37 #[inline(always)]
38 pub unsafe fn load16_aligned(ptr: *const u8) -> u8x16 {
39 *(ptr as *const u8x16)
40 }
41
42 /// Safety invariant: ptr must be valid for an unaligned store of 16 bytes
43 #[inline(always)]
44 pub unsafe fn store16_unaligned(ptr: *mut u8, s: u8x16) {
45 ::core::ptr::copy_nonoverlapping(&s as *const u8x16 as *const u8, ptr, 16);
46 }
47
48 /// Safety invariant: ptr must be valid for an aligned-for-u8x16 store of 16 bytes
49 #[allow(dead_code)]
50 #[inline(always)]
51 pub unsafe fn store16_aligned(ptr: *mut u8, s: u8x16) {
52 *(ptr as *mut u8x16) = s;
53 }
54
55 /// Safety invariant: ptr must be valid for an unaligned read of 16 bytes
56 #[inline(always)]
57 pub unsafe fn load8_unaligned(ptr: *const u16) -> u16x8 {
58 let mut simd = ::core::mem::MaybeUninit::<u16x8>::uninit();
59 ::core::ptr::copy_nonoverlapping(ptr as *const u8, simd.as_mut_ptr() as *mut u8, 16);
60 // Safety: copied 16 bytes of initialized memory into this, it is now initialized
61 simd.assume_init()
62 }
63
64 /// Safety invariant: ptr must be valid for an aligned-for-u16x8 read of 16 bytes
65 #[allow(dead_code)]
66 #[inline(always)]
67 pub unsafe fn load8_aligned(ptr: *const u16) -> u16x8 {
68 *(ptr as *const u16x8)
69 }
70
71 /// Safety invariant: ptr must be valid for an unaligned store of 16 bytes
72 #[inline(always)]
73 pub unsafe fn store8_unaligned(ptr: *mut u16, s: u16x8) {
74 ::core::ptr::copy_nonoverlapping(&s as *const u16x8 as *const u8, ptr as *mut u8, 16);
75 }
76
77 /// Safety invariant: ptr must be valid for an aligned-for-u16x8 store of 16 bytes
78 #[allow(dead_code)]
79 #[inline(always)]
80 pub unsafe fn store8_aligned(ptr: *mut u16, s: u16x8) {
81 *(ptr as *mut u16x8) = s;
82 }
83
84 cfg_if! {
85 if #[cfg(all(target_feature = "sse2", target_arch = "x86_64"))] {
86 use core::arch::x86_64::__m128i;
87 use core::arch::x86_64::_mm_movemask_epi8;
88 use core::arch::x86_64::_mm_packus_epi16;
89 } else if #[cfg(all(target_feature = "sse2", target_arch = "x86"))] {
90 use core::arch::x86::__m128i;
91 use core::arch::x86::_mm_movemask_epi8;
92 use core::arch::x86::_mm_packus_epi16;
93 } else if #[cfg(target_arch = "aarch64")]{
94 use core::arch::aarch64::vmaxvq_u8;
95 use core::arch::aarch64::vmaxvq_u16;
96 } else {
97
98 }
99 }
100
101 // #[inline(always)]
102 // fn simd_byte_swap_u8(s: u8x16) -> u8x16 {
103 // unsafe {
104 // shuffle!(s, s, [1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14])
105 // }
106 // }
107
108 // #[inline(always)]
109 // pub fn simd_byte_swap(s: u16x8) -> u16x8 {
110 // to_u16_lanes(simd_byte_swap_u8(to_u8_lanes(s)))
111 // }
112
113 #[inline(always)]
114 pub fn simd_byte_swap(s: u16x8) -> u16x8 {
115 let left = s << 8;
116 let right = s >> 8;
117 left | right
118 }
119
120 #[inline(always)]
121 pub fn to_u16_lanes(s: u8x16) -> u16x8 {
122 u16x8::from_ne_bytes(s)
123 }
124
125 cfg_if! {
126 if #[cfg(target_feature = "sse2")] {
127
128 // Expose low-level mask instead of higher-level conclusion,
129 // because the non-ASCII case would perform less well otherwise.
130 // Safety-usable invariant: This returned value is whether each high bit is set
131 #[inline(always)]
132 pub fn mask_ascii(s: u8x16) -> i32 {
133 unsafe {
134 _mm_movemask_epi8(s.into())
135 }
136 }
137
138 } else {
139
140 }
141 }
142
143 cfg_if! {
144 if #[cfg(target_feature = "sse2")] {
145 #[inline(always)]
146 pub fn simd_is_ascii(s: u8x16) -> bool {
147 unsafe {
148 // Safety: We have cfg()d the correct platform
149 _mm_movemask_epi8(s.into()) == 0
150 }
151 }
152 } else if #[cfg(target_arch = "aarch64")]{
153 #[inline(always)]
154 pub fn simd_is_ascii(s: u8x16) -> bool {
155 unsafe {
156 // Safety: We have cfg()d the correct platform
157 vmaxvq_u8(s.into()) < 0x80
158 }
159 }
160 } else {
161 #[inline(always)]
162 pub fn simd_is_ascii(s: u8x16) -> bool {
163 // This optimizes better on ARM than
164 // the lt formulation.
165 let highest_ascii = u8x16::splat(0x7F);
166 !any_mask8x16(s.simd_gt(highest_ascii))
167 }
168 }
169 }
170
171 cfg_if! {
172 if #[cfg(target_feature = "sse2")] {
173 #[inline(always)]
174 pub fn simd_is_str_latin1(s: u8x16) -> bool {
175 if simd_is_ascii(s) {
176 return true;
177 }
178 let above_str_latin1 = u8x16::splat(0xC4);
179 s.simd_lt(above_str_latin1).all()
180 }
181 } else if #[cfg(target_arch = "aarch64")]{
182 #[inline(always)]
183 pub fn simd_is_str_latin1(s: u8x16) -> bool {
184 unsafe {
185 // Safety: We have cfg()d the correct platform
186 vmaxvq_u8(s.into()) < 0xC4
187 }
188 }
189 } else {
190 #[inline(always)]
191 pub fn simd_is_str_latin1(s: u8x16) -> bool {
192 let above_str_latin1 = u8x16::splat(0xC4);
193 all_mask8x16(s.simd_lt(above_str_latin1))
194 }
195 }
196 }
197
198 cfg_if! {
199 if #[cfg(target_arch = "aarch64")]{
200 #[inline(always)]
201 pub fn simd_is_basic_latin(s: u16x8) -> bool {
202 unsafe {
203 // Safety: We have cfg()d the correct platform
204 vmaxvq_u16(s.into()) < 0x80
205 }
206 }
207
208 #[inline(always)]
209 pub fn simd_is_latin1(s: u16x8) -> bool {
210 unsafe {
211 // Safety: We have cfg()d the correct platform
212 vmaxvq_u16(s.into()) < 0x100
213 }
214 }
215 } else {
216 #[inline(always)]
217 pub fn simd_is_basic_latin(s: u16x8) -> bool {
218 let above_ascii = u16x8::splat(0x80);
219 all_mask16x8(s.simd_lt(above_ascii))
220 }
221
222 #[inline(always)]
223 pub fn simd_is_latin1(s: u16x8) -> bool {
224 // For some reason, on SSE2 this formulation
225 // seems faster in this case while the above
226 // function is better the other way round...
227 let highest_latin1 = u16x8::splat(0xFF);
228 !any_mask16x8(s.simd_gt(highest_latin1))
229 }
230 }
231 }
232
233 #[inline(always)]
234 pub fn contains_surrogates(s: u16x8) -> bool {
235 let mask = u16x8::splat(0xF800);
236 let surrogate_bits = u16x8::splat(0xD800);
237 any_mask16x8((s & mask).simd_eq(surrogate_bits))
238 }
239
240 cfg_if! {
241 if #[cfg(target_arch = "aarch64")]{
242 macro_rules! aarch64_return_false_if_below_hebrew {
243 ($s:ident) => ({
244 unsafe {
245 // Safety: We have cfg()d the correct platform
246 if vmaxvq_u16($s.into()) < 0x0590 {
247 return false;
248 }
249 }
250 })
251 }
252
253 macro_rules! non_aarch64_return_false_if_all {
254 ($s:ident) => ()
255 }
256 } else {
257 macro_rules! aarch64_return_false_if_below_hebrew {
258 ($s:ident) => ()
259 }
260
261 macro_rules! non_aarch64_return_false_if_all {
262 ($s:ident) => ({
263 if all_mask16x8($s) {
264 return false;
265 }
266 })
267 }
268 }
269 }
270
271 macro_rules! in_range16x8 {
272 ($s:ident, $start:expr, $end:expr) => {{
273 // SIMD sub is wrapping
274 ($s - u16x8::splat($start)).simd_lt(u16x8::splat($end - $start))
275 }};
276 }
277
278 #[inline(always)]
279 pub fn is_u16x8_bidi(s: u16x8) -> bool {
280 // We try to first quickly refute the RTLness of the vector. If that
281 // fails, we do the real RTL check, so in that case we end up wasting
282 // the work for the up-front quick checks. Even the quick-check is
283 // two-fold in order to return `false` ASAP if everything is below
284 // Hebrew.
285
286 aarch64_return_false_if_below_hebrew!(s);
287
288 let below_hebrew = s.simd_lt(u16x8::splat(0x0590));
289
290 non_aarch64_return_false_if_all!(below_hebrew);
291
292 if all_mask16x8(
293 below_hebrew | in_range16x8!(s, 0x0900, 0x200F) | in_range16x8!(s, 0x2068, 0xD802),
294 ) {
295 return false;
296 }
297
298 // Quick refutation failed. Let's do the full check.
299
300 any_mask16x8(
301 (in_range16x8!(s, 0x0590, 0x0900)
302 | in_range16x8!(s, 0xFB1D, 0xFE00)
303 | in_range16x8!(s, 0xFE70, 0xFEFF)
304 | in_range16x8!(s, 0xD802, 0xD804)
305 | in_range16x8!(s, 0xD83A, 0xD83C)
306 | s.simd_eq(u16x8::splat(0x200F))
307 | s.simd_eq(u16x8::splat(0x202B))
308 | s.simd_eq(u16x8::splat(0x202E))
309 | s.simd_eq(u16x8::splat(0x2067))),
310 )
311 }
312
313 #[inline(always)]
314 pub fn simd_unpack(s: u8x16) -> (u16x8, u16x8) {
315 let first: u8x16 = simd_swizzle!(
316 s,
317 u8x16::splat(0),
318 [0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23]
319 );
320 let second: u8x16 = simd_swizzle!(
321 s,
322 u8x16::splat(0),
323 [8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31]
324 );
325 (u16x8::from_ne_bytes(first), u16x8::from_ne_bytes(second))
326 }
327
328 cfg_if! {
329 if #[cfg(target_feature = "sse2")] {
330 #[inline(always)]
331 pub fn simd_pack(a: u16x8, b: u16x8) -> u8x16 {
332 unsafe {
333 // Safety: We have cfg()d the correct platform
334 _mm_packus_epi16(a.into(), b.into()).into()
335 }
336 }
337 } else {
338 #[inline(always)]
339 pub fn simd_pack(a: u16x8, b: u16x8) -> u8x16 {
340 let first: u8x16 = a.to_ne_bytes();
341 let second: u8x16 = b.to_ne_bytes();
342 simd_swizzle!(
343 first,
344 second,
345 [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30]
346 )
347 }
348 }
349 }
350
351 #[cfg(test)]
352 mod tests {
353 use super::*;
354 use alloc::vec::Vec;
355
356 #[test]
357 fn test_unpack() {
358 let ascii: [u8; 16] = [
359 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x70, 0x71, 0x72, 0x73, 0x74,
360 0x75, 0x76,
361 ];
362 let basic_latin: [u16; 16] = [
363 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x70, 0x71, 0x72, 0x73, 0x74,
364 0x75, 0x76,
365 ];
366 let simd = unsafe { load16_unaligned(ascii.as_ptr()) };
367 let mut vec = Vec::with_capacity(16);
368 vec.resize(16, 0u16);
369 let (first, second) = simd_unpack(simd);
370 let ptr = vec.as_mut_ptr();
371 unsafe {
372 store8_unaligned(ptr, first);
373 store8_unaligned(ptr.add(8), second);
374 }
375 assert_eq!(&vec[..], &basic_latin[..]);
376 }
377
378 #[test]
379 fn test_simd_is_basic_latin_success() {
380 let ascii: [u8; 16] = [
381 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x70, 0x71, 0x72, 0x73, 0x74,
382 0x75, 0x76,
383 ];
384 let basic_latin: [u16; 16] = [
385 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x70, 0x71, 0x72, 0x73, 0x74,
386 0x75, 0x76,
387 ];
388 let first = unsafe { load8_unaligned(basic_latin.as_ptr()) };
389 let second = unsafe { load8_unaligned(basic_latin.as_ptr().add(8)) };
390 let mut vec = Vec::with_capacity(16);
391 vec.resize(16, 0u8);
392 let ptr = vec.as_mut_ptr();
393 assert!(simd_is_basic_latin(first | second));
394 unsafe {
395 store16_unaligned(ptr, simd_pack(first, second));
396 }
397 assert_eq!(&vec[..], &ascii[..]);
398 }
399
400 #[test]
401 fn test_simd_is_basic_latin_c0() {
402 let input: [u16; 16] = [
403 0x61, 0x62, 0x63, 0x81, 0x65, 0x66, 0x67, 0x68, 0x69, 0x70, 0x71, 0x72, 0x73, 0x74,
404 0x75, 0x76,
405 ];
406 let first = unsafe { load8_unaligned(input.as_ptr()) };
407 let second = unsafe { load8_unaligned(input.as_ptr().add(8)) };
408 assert!(!simd_is_basic_latin(first | second));
409 }
410
411 #[test]
412 fn test_simd_is_basic_latin_0fff() {
413 let input: [u16; 16] = [
414 0x61, 0x62, 0x63, 0x0FFF, 0x65, 0x66, 0x67, 0x68, 0x69, 0x70, 0x71, 0x72, 0x73, 0x74,
415 0x75, 0x76,
416 ];
417 let first = unsafe { load8_unaligned(input.as_ptr()) };
418 let second = unsafe { load8_unaligned(input.as_ptr().add(8)) };
419 assert!(!simd_is_basic_latin(first | second));
420 }
421
422 #[test]
423 fn test_simd_is_basic_latin_ffff() {
424 let input: [u16; 16] = [
425 0x61, 0x62, 0x63, 0xFFFF, 0x65, 0x66, 0x67, 0x68, 0x69, 0x70, 0x71, 0x72, 0x73, 0x74,
426 0x75, 0x76,
427 ];
428 let first = unsafe { load8_unaligned(input.as_ptr()) };
429 let second = unsafe { load8_unaligned(input.as_ptr().add(8)) };
430 assert!(!simd_is_basic_latin(first | second));
431 }
432
433 #[test]
434 fn test_simd_is_ascii_success() {
435 let ascii: [u8; 16] = [
436 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x70, 0x71, 0x72, 0x73, 0x74,
437 0x75, 0x76,
438 ];
439 let simd = unsafe { load16_unaligned(ascii.as_ptr()) };
440 assert!(simd_is_ascii(simd));
441 }
442
443 #[test]
444 fn test_simd_is_ascii_failure() {
445 let input: [u8; 16] = [
446 0x61, 0x62, 0x63, 0x64, 0x81, 0x66, 0x67, 0x68, 0x69, 0x70, 0x71, 0x72, 0x73, 0x74,
447 0x75, 0x76,
448 ];
449 let simd = unsafe { load16_unaligned(input.as_ptr()) };
450 assert!(!simd_is_ascii(simd));
451 }
452
453 #[cfg(target_feature = "sse2")]
454 #[test]
455 fn test_check_ascii() {
456 let input: [u8; 16] = [
457 0x61, 0x62, 0x63, 0x64, 0x81, 0x66, 0x67, 0x68, 0x69, 0x70, 0x71, 0x72, 0x73, 0x74,
458 0x75, 0x76,
459 ];
460 let simd = unsafe { load16_unaligned(input.as_ptr()) };
461 let mask = mask_ascii(simd);
462 assert_ne!(mask, 0);
463 assert_eq!(mask.trailing_zeros(), 4);
464 }
465
466 #[test]
467 fn test_alu() {
468 let input: [u8; 16] = [
469 0x61, 0x62, 0x63, 0x64, 0x81, 0x66, 0x67, 0x68, 0x69, 0x70, 0x71, 0x72, 0x73, 0x74,
470 0x75, 0x76,
471 ];
472 let mut alu = 0u64;
473 unsafe {
474 ::core::ptr::copy_nonoverlapping(input.as_ptr(), &mut alu as *mut u64 as *mut u8, 8);
475 }
476 let masked = alu & 0x8080808080808080;
477 assert_eq!(masked.trailing_zeros(), 39);
478 }
479 }