Debian Codegraph @

utf-8-validate/deps/is_utf8/src/is_utf8.cpp

raw
1 #ifndef IS_UTF8_H
2 #define IS_UTF8_H
3 #include <cstring>
4
5 #ifndef IS_UTF8_COMPILER_CHECK_H
6 #define IS_UTF8_COMPILER_CHECK_H
7
8 #ifndef __cplusplus
9 #error we require a C++ compiler
10 #endif
11
12 #ifndef IS_UTF8_CPLUSPLUS
13 #if defined(_MSVC_LANG) && !defined(__clang__)
14 #define IS_UTF8_CPLUSPLUS (_MSC_VER == 1900 ? 201103L : _MSVC_LANG)
15 #else
16 #define IS_UTF8_CPLUSPLUS __cplusplus
17 #endif
18 #endif
19
20 // C++ 17
21 #if !defined(IS_UTF8_CPLUSPLUS17) && (IS_UTF8_CPLUSPLUS >= 201703L)
22 #define IS_UTF8_CPLUSPLUS17 1
23 #endif
24
25 // C++ 14
26 #if !defined(IS_UTF8_CPLUSPLUS14) && (IS_UTF8_CPLUSPLUS >= 201402L)
27 #define IS_UTF8_CPLUSPLUS14 1
28 #endif
29
30 // C++ 11
31 #if !defined(IS_UTF8_CPLUSPLUS11) && (IS_UTF8_CPLUSPLUS >= 201103L)
32 #define IS_UTF8_CPLUSPLUS11 1
33 #endif
34
35 #ifndef IS_UTF8_CPLUSPLUS11
36 #error we require a compiler compliant with the C++11 standard
37 #endif
38
39 #endif // IS_UTF8_COMPILER_CHECK_H
40
41 #ifndef IS_UTF8_COMMON_DEFS_H
42 #define IS_UTF8_COMMON_DEFS_H
43
44 #include <cassert>
45
46 #ifndef IS_UTF8_PORTABILITY_H
47 #define IS_UTF8_PORTABILITY_H
48
49 #include <cassert>
50 #include <cfloat>
51 #include <cstddef>
52 #include <cstdint>
53 #include <cstdlib>
54 #ifndef _WIN32
55 // strcasecmp, strncasecmp
56 #include <strings.h>
57 #endif
58
59 #ifdef _MSC_VER
60 #define IS_UTF8_VISUAL_STUDIO 1
61 /**
62 * We want to differentiate carefully between
63 * clang under visual studio and regular visual
64 * studio.
65 *
66 * Under clang for Windows, we enable:
67 * * target pragmas so that part and only part of the
68 * code gets compiled for advanced instructions.
69 *
70 */
71 #ifdef __clang__
72 // clang under visual studio
73 #define IS_UTF8_CLANG_VISUAL_STUDIO 1
74 #else
75 // just regular visual studio (best guess)
76 #define IS_UTF8_REGULAR_VISUAL_STUDIO 1
77 #endif // __clang__
78 #endif // _MSC_VER
79
80 #ifdef IS_UTF8_REGULAR_VISUAL_STUDIO
81 // https://en.wikipedia.org/wiki/C_alternative_tokens
82 // This header should have no effect, except maybe
83 // under Visual Studio.
84 #include <iso646.h>
85 #endif
86
87 #if defined(__x86_64__) || defined(_M_AMD64)
88 #define IS_UTF8_IS_X86_64 1
89 #elif defined(__aarch64__) || defined(_M_ARM64)
90 #define IS_UTF8_IS_ARM64 1
91 #elif defined(__PPC64__) || defined(_M_PPC64)
92 //#define IS_UTF8_IS_PPC64 1
93 #pragma message("The library does yet support SIMD acceleration under\
94 POWER processors.")
95 #else
96 #define IS_UTF8_IS_32BITS 1
97
98 // We do not support 32-bit platforms, but it can be
99 // handy to identify them.
100 #if defined(_M_IX86) || defined(__i386__)
101 #define IS_UTF8_IS_X86_32BITS 1
102 #elif defined(__arm__) || defined(_M_ARM)
103 #define IS_UTF8_IS_ARM_32BITS 1
104 #elif defined(__PPC__) || defined(_M_PPC)
105 #define IS_UTF8_IS_PPC_32BITS 1
106 #endif
107
108 #endif // defined(__x86_64__) || defined(_M_AMD64)
109
110 // this is almost standard?
111 #define IS_UTF8_STRINGIFY_IMPLEMENTATION_(a) #a
112 #define IS_UTF8_STRINGIFY(a) IS_UTF8_STRINGIFY_IMPLEMENTATION_(a)
113
114 // We are going to use runtime dispatch.
115 #ifdef IS_UTF8_IS_X86_64
116 #ifdef __clang__
117 // clang does not have GCC push pop
118 // warning: clang attribute push can't be used within a namespace in clang up
119 // til 8.0 so IS_UTF8_TARGET_REGION and IS_UTF8_UNTARGET_REGION must be
120 // *outside* of a namespace.
121 #define IS_UTF8_TARGET_REGION(T) \
122 _Pragma(IS_UTF8_STRINGIFY( \
123 clang attribute push(__attribute__((target(T))), apply_to = function)))
124 #define IS_UTF8_UNTARGET_REGION _Pragma("clang attribute pop")
125 #elif defined(__GNUC__)
126 // GCC is easier
127 #define IS_UTF8_TARGET_REGION(T) \
128 _Pragma("GCC push_options") _Pragma(IS_UTF8_STRINGIFY(GCC target(T)))
129 #define IS_UTF8_UNTARGET_REGION _Pragma("GCC pop_options")
130 #endif // clang then gcc
131
132 #endif // x86
133
134 // Default target region macros don't do anything.
135 #ifndef IS_UTF8_TARGET_REGION
136 #define IS_UTF8_TARGET_REGION(T)
137 #define IS_UTF8_UNTARGET_REGION
138 #endif
139
140 #if defined(__GNUC__) && !defined(__clang__)
141 #if __GNUC__ >= 11
142 #define IS_UTF8_GCC11ORMORE 1
143 #endif // __GNUC__ >= 11
144 #endif // defined(__GNUC__) && !defined(__clang__)
145
146 #endif // IS_UTF8_PORTABILITY_H
147 #ifndef IS_UTF8_AVX512_H_
148 #define IS_UTF8_AVX512_H_
149
150 /*
151 It's possible to override AVX512 settings with cmake DCMAKE_CXX_FLAGS.
152
153 All preprocessor directives has form `IS_UTF8_HAS_AVX512{feature}`,
154 where a feature is a code name for extensions.
155
156 Please see the listing below to find which are supported.
157 */
158
159 #ifndef IS_UTF8_HAS_AVX512F
160 #if defined(__AVX512F__) && __AVX512F__ == 1
161 #define IS_UTF8_HAS_AVX512F 1
162 #endif
163 #endif
164
165 #ifndef IS_UTF8_HAS_AVX512DQ
166 #if defined(__AVX512DQ__) && __AVX512DQ__ == 1
167 #define IS_UTF8_HAS_AVX512DQ 1
168 #endif
169 #endif
170
171 #ifndef IS_UTF8_HAS_AVX512IFMA
172 #if defined(__AVX512IFMA__) && __AVX512IFMA__ == 1
173 #define IS_UTF8_HAS_AVX512IFMA 1
174 #endif
175 #endif
176
177 #ifndef IS_UTF8_HAS_AVX512CD
178 #if defined(__AVX512CD__) && __AVX512CD__ == 1
179 #define IS_UTF8_HAS_AVX512CD 1
180 #endif
181 #endif
182
183 #ifndef IS_UTF8_HAS_AVX512BW
184 #if defined(__AVX512BW__) && __AVX512BW__ == 1
185 #define IS_UTF8_HAS_AVX512BW 1
186 #endif
187 #endif
188
189 #ifndef IS_UTF8_HAS_AVX512VL
190 #if defined(__AVX512VL__) && __AVX512VL__ == 1
191 #define IS_UTF8_HAS_AVX512VL 1
192 #endif
193 #endif
194
195 #ifndef IS_UTF8_HAS_AVX512VBMI
196 #if defined(__AVX512VBMI__) && __AVX512VBMI__ == 1
197 #define IS_UTF8_HAS_AVX512VBMI 1
198 #endif
199 #endif
200
201 #ifndef IS_UTF8_HAS_AVX512VBMI2
202 #if defined(__AVX512VBMI2__) && __AVX512VBMI2__ == 1
203 #define IS_UTF8_HAS_AVX512VBMI2 1
204 #endif
205 #endif
206
207 #ifndef IS_UTF8_HAS_AVX512VNNI
208 #if defined(__AVX512VNNI__) && __AVX512VNNI__ == 1
209 #define IS_UTF8_HAS_AVX512VNNI 1
210 #endif
211 #endif
212
213 #ifndef IS_UTF8_HAS_AVX512BITALG
214 #if defined(__AVX512BITALG__) && __AVX512BITALG__ == 1
215 #define IS_UTF8_HAS_AVX512BITALG 1
216 #endif
217 #endif
218
219 #ifndef IS_UTF8_HAS_AVX512VPOPCNTDQ
220 #if defined(__AVX512VPOPCNTDQ__) && __AVX512VPOPCNTDQ__ == 1
221 #define IS_UTF8_HAS_AVX512VPOPCNTDQ 1
222 #endif
223 #endif
224
225 #endif // IS_UTF8_AVX512_H_
226
227 #if defined(__GNUC__)
228 // Marks a block with a name so that MCA analysis can see it.
229 #define IS_UTF8_BEGIN_DEBUG_BLOCK(name) \
230 __asm volatile("# LLVM-MCA-BEGIN " #name);
231 #define IS_UTF8_END_DEBUG_BLOCK(name) __asm volatile("# LLVM-MCA-END " #name);
232 #define IS_UTF8_DEBUG_BLOCK(name, block) \
233 BEGIN_DEBUG_BLOCK(name); \
234 block; \
235 END_DEBUG_BLOCK(name);
236 #else
237 #define IS_UTF8_BEGIN_DEBUG_BLOCK(name)
238 #define IS_UTF8_END_DEBUG_BLOCK(name)
239 #define IS_UTF8_DEBUG_BLOCK(name, block)
240 #endif
241
242 // Align to N-byte boundary
243 #define IS_UTF8_ROUNDUP_N(a, n) (((a) + ((n)-1)) & ~((n)-1))
244 #define IS_UTF8_ROUNDDOWN_N(a, n) ((a) & ~((n)-1))
245
246 #define IS_UTF8_ISALIGNED_N(ptr, n) (((uintptr_t)(ptr) & ((n)-1)) == 0)
247
248 #if defined(IS_UTF8_REGULAR_VISUAL_STUDIO)
249
250 #define is_utf8_really_inline __forceinline
251 #define is_utf8_never_inline __declspec(noinline)
252
253 #define is_utf8_unused
254 #define is_utf8_warn_unused
255
256 #ifndef is_utf8_likely
257 #define is_utf8_likely(x) x
258 #endif
259 #ifndef is_utf8_unlikely
260 #define is_utf8_unlikely(x) x
261 #endif
262
263 #define IS_UTF8_PUSH_DISABLE_WARNINGS __pragma(warning(push))
264 #define IS_UTF8_PUSH_DISABLE_ALL_WARNINGS __pragma(warning(push, 0))
265 #define IS_UTF8_DISABLE_VS_WARNING(WARNING_NUMBER) \
266 __pragma(warning(disable : WARNING_NUMBER))
267 // Get rid of Intellisense-only warnings (Code Analysis)
268 // Though __has_include is C++17, it is supported in Visual Studio 2017 or
269 // better (_MSC_VER>=1910).
270 #ifdef __has_include
271 #if __has_include(<CppCoreCheck\Warnings.h>)
272 #include <CppCoreCheck\Warnings.h>
273 #define IS_UTF8_DISABLE_UNDESIRED_WARNINGS \
274 IS_UTF8_DISABLE_VS_WARNING(ALL_CPPCORECHECK_WARNINGS)
275 #endif
276 #endif
277
278 #ifndef IS_UTF8_DISABLE_UNDESIRED_WARNINGS
279 #define IS_UTF8_DISABLE_UNDESIRED_WARNINGS
280 #endif
281
282 #define IS_UTF8_DISABLE_DEPRECATED_WARNING IS_UTF8_DISABLE_VS_WARNING(4996)
283 #define IS_UTF8_DISABLE_STRICT_OVERFLOW_WARNING
284 #define IS_UTF8_POP_DISABLE_WARNINGS __pragma(warning(pop))
285
286 #else // IS_UTF8_REGULAR_VISUAL_STUDIO
287
288 #define is_utf8_really_inline inline __attribute__((always_inline))
289 #define is_utf8_never_inline inline __attribute__((noinline))
290
291 #define is_utf8_unused __attribute__((unused))
292 #define is_utf8_warn_unused __attribute__((warn_unused_result))
293
294 #ifndef is_utf8_likely
295 #define is_utf8_likely(x) __builtin_expect(!!(x), 1)
296 #endif
297 #ifndef is_utf8_unlikely
298 #define is_utf8_unlikely(x) __builtin_expect(!!(x), 0)
299 #endif
300
301 #define IS_UTF8_PUSH_DISABLE_WARNINGS _Pragma("GCC diagnostic push")
302 // gcc doesn't seem to disable all warnings with all and extra, add warnings
303 // here as necessary
304 #define IS_UTF8_PUSH_DISABLE_ALL_WARNINGS \
305 IS_UTF8_PUSH_DISABLE_WARNINGS \
306 IS_UTF8_DISABLE_GCC_WARNING(-Weffc++) \
307 IS_UTF8_DISABLE_GCC_WARNING(-Wall) \
308 IS_UTF8_DISABLE_GCC_WARNING(-Wconversion) \
309 IS_UTF8_DISABLE_GCC_WARNING(-Wextra) \
310 IS_UTF8_DISABLE_GCC_WARNING(-Wattributes) \
311 IS_UTF8_DISABLE_GCC_WARNING(-Wimplicit-fallthrough) \
312 IS_UTF8_DISABLE_GCC_WARNING(-Wnon-virtual-dtor) \
313 IS_UTF8_DISABLE_GCC_WARNING(-Wreturn-type) \
314 IS_UTF8_DISABLE_GCC_WARNING(-Wshadow) \
315 IS_UTF8_DISABLE_GCC_WARNING(-Wunused-parameter) \
316 IS_UTF8_DISABLE_GCC_WARNING(-Wunused-variable)
317 #define IS_UTF8_PRAGMA(P) _Pragma(#P)
318 #define IS_UTF8_DISABLE_GCC_WARNING(WARNING) \
319 IS_UTF8_PRAGMA(GCC diagnostic ignored #WARNING)
320 #if defined(IS_UTF8_CLANG_VISUAL_STUDIO)
321 #define IS_UTF8_DISABLE_UNDESIRED_WARNINGS \
322 IS_UTF8_DISABLE_GCC_WARNING(-Wmicrosoft-include)
323 #else
324 #define IS_UTF8_DISABLE_UNDESIRED_WARNINGS
325 #endif
326 #define IS_UTF8_DISABLE_DEPRECATED_WARNING \
327 IS_UTF8_DISABLE_GCC_WARNING(-Wdeprecated-declarations)
328 #define IS_UTF8_DISABLE_STRICT_OVERFLOW_WARNING \
329 IS_UTF8_DISABLE_GCC_WARNING(-Wstrict-overflow)
330 #define IS_UTF8_POP_DISABLE_WARNINGS _Pragma("GCC diagnostic pop")
331
332 #endif // MSC_VER
333
334 #if defined(IS_UTF8_VISUAL_STUDIO)
335 /**
336 * It does not matter here whether you are using
337 * the regular visual studio or clang under visual
338 * studio.
339 */
340 #if IS_UTF8_USING_LIBRARY
341 #define IS_UTF8_DLLIMPORTEXPORT __declspec(dllimport)
342 #else
343 #define IS_UTF8_DLLIMPORTEXPORT __declspec(dllexport)
344 #endif
345 #else
346 #define IS_UTF8_DLLIMPORTEXPORT
347 #endif
348
349 /// If EXPR is an error, returns it.
350 #define IS_UTF8_TRY(EXPR) \
351 { \
352 auto _err = (EXPR); \
353 if (_err) { \
354 return _err; \
355 } \
356 }
357
358 #endif // IS_UTF8_COMMON_DEFS_H
359 #include <string>
360
361 namespace is_utf8_internals {
362
363 enum encoding_type {
364 UTF8 = 1, // BOM 0xef 0xbb 0xbf
365 UTF16_LE = 2, // BOM 0xff 0xfe
366 UTF16_BE = 4, // BOM 0xfe 0xff
367 UTF32_LE = 8, // BOM 0xff 0xfe 0x00 0x00
368 UTF32_BE = 16, // BOM 0x00 0x00 0xfe 0xff
369
370 unspecified = 0
371 };
372
373 enum endianness { LITTLE, BIG };
374
375 std::string to_string(encoding_type bom);
376
377 // Note that BOM for UTF8 is discouraged.
378 namespace BOM {
379
380 /**
381 * Checks for a BOM. If not, returns unspecified
382 * @param input the string to process
383 * @param length the length of the string in words
384 * @return the corresponding encoding
385 */
386
387 encoding_type check_bom(const uint8_t *byte, size_t length);
388 encoding_type check_bom(const char *byte, size_t length);
389 /**
390 * Returns the size, in bytes, of the BOM for a given encoding type.
391 * Note that UTF8 BOM are discouraged.
392 * @param bom the encoding type
393 * @return the size in bytes of the corresponding BOM
394 */
395 size_t bom_byte_size(encoding_type bom);
396
397 } // namespace BOM
398 } // namespace is_utf8_internals
399 #ifndef ERROR_H
400 #define ERROR_H
401 namespace is_utf8_internals {
402
403 enum error_code {
404 SUCCESS = 0,
405 HEADER_BITS, // Any byte must have fewer than 5 header bits.
406 TOO_SHORT, // The leading byte must be followed by N-1 continuation bytes,
407 // where N is the UTF-8 character length This is also the error
408 // when the input is truncated.
409 TOO_LONG, // The leading byte must not be a continuation byte.
410 OVERLONG, // The decoded character must be above U+7F for two-byte characters,
411 // U+7FF for three-byte characters, and U+FFFF for four-byte
412 // characters.
413 TOO_LARGE, // The decoded character must be less than or equal to U+10FFFF OR
414 // less than or equal than U+7F for ASCII.
415 SURROGATE, // The decoded character must be not be in U+D800...DFFF (UTF-8 or
416 // UTF-32) OR a high surrogate must be followed by a low surrogate
417 // and a low surrogate must be preceded by a high surrogate
418 // (UTF-16)
419 OTHER // Not related to validation/transcoding.
420 };
421
422 struct result {
423 error_code error;
424 size_t
425 count; // In case of error, indicates the position of the error. In case
426 // of success, indicates the number of words validated/written.
427
428 is_utf8_really_inline result();
429
430 is_utf8_really_inline result(error_code, size_t);
431 };
432
433 } // namespace is_utf8_internals
434 #endif
435
436 IS_UTF8_PUSH_DISABLE_WARNINGS
437 IS_UTF8_DISABLE_UNDESIRED_WARNINGS
438
439 #ifndef IS_UTF8_IMPLEMENTATION_H
440 #define IS_UTF8_IMPLEMENTATION_H
441 #include <string>
442 #if !defined(IS_UTF8_NO_THREADS)
443 #include <atomic>
444 #endif
445 #include <tuple>
446 #include <vector>
447 /* From
448 https://github.com/endorno/pytorch/blob/master/torch/lib/TH/generic/simd/simd.h
449 Highly modified.
450
451 Copyright (c) 2016- Facebook, Inc (Adam Paszke)
452 Copyright (c) 2014- Facebook, Inc (Soumith Chintala)
453 Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert)
454 Copyright (c) 2012-2014 Deepmind Technologies (Koray Kavukcuoglu)
455 Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu)
456 Copyright (c) 2011-2013 NYU (Clement Farabet)
457 Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou,
458 Iain Melvin, Jason Weston) Copyright (c) 2006 Idiap Research Institute
459 (Samy Bengio) Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert,
460 Samy Bengio, Johnny Mariethoz)
461
462 All rights reserved.
463
464 Redistribution and use in source and binary forms, with or without
465 modification, are permitted provided that the following conditions are met:
466
467 1. Redistributions of source code must retain the above copyright
468 notice, this list of conditions and the following disclaimer.
469
470 2. Redistributions in binary form must reproduce the above copyright
471 notice, this list of conditions and the following disclaimer in the
472 documentation and/or other materials provided with the distribution.
473
474 3. Neither the names of Facebook, Deepmind Technologies, NYU, NEC Laboratories
475 America and IDIAP Research Institute nor the names of its contributors may be
476 used to endorse or promote products derived from this software without
477 specific prior written permission.
478
479 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
480 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
481 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
482 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
483 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
484 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
485 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
486 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
487 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
488 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
489 POSSIBILITY OF SUCH DAMAGE.
490 */
491
492 #ifndef IS_UTF8_INTERNAL_ISADETECTION_H
493 #define IS_UTF8_INTERNAL_ISADETECTION_H
494
495 #include <cstdint>
496 #include <cstdlib>
497 #if defined(_MSC_VER)
498 #include <intrin.h>
499 #elif defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID)
500 #include <cpuid.h>
501 #endif
502
503 namespace is_utf8_internals {
504 namespace internal {
505
506 enum instruction_set {
507 DEFAULT = 0x0,
508 NEON = 0x1,
509 AVX2 = 0x4,
510 SSE42 = 0x8,
511 PCLMULQDQ = 0x10,
512 BMI1 = 0x20,
513 BMI2 = 0x40,
514 ALTIVEC = 0x80,
515 AVX512F = 0x100,
516 AVX512DQ = 0x200,
517 AVX512IFMA = 0x400,
518 AVX512PF = 0x800,
519 AVX512ER = 0x1000,
520 AVX512CD = 0x2000,
521 AVX512BW = 0x4000,
522 AVX512VL = 0x8000,
523 AVX512VBMI2 = 0x10000
524 };
525
526 #if defined(__PPC64__)
527
528 static inline uint32_t detect_supported_architectures() {
529 return instruction_set::ALTIVEC;
530 }
531
532 #elif defined(__aarch64__) || defined(_M_ARM64)
533
534 static inline uint32_t detect_supported_architectures() {
535 return instruction_set::NEON;
536 }
537
538 #elif defined(__x86_64__) || defined(_M_AMD64) // x64
539
540 namespace {
541 namespace cpuid_bit {
542 // Can be found on Intel ISA Reference for CPUID
543
544 // EAX = 0x01
545 constexpr uint32_t pclmulqdq = uint32_t(1)
546 << 1; ///< @private bit 1 of ECX for EAX=0x1
547 constexpr uint32_t sse42 = uint32_t(1)
548 << 20; ///< @private bit 20 of ECX for EAX=0x1
549
550 // EAX = 0x7f (Structured Extended Feature Flags), ECX = 0x00 (Sub-leaf)
551 // See: "Table 3-8. Information Returned by CPUID Instruction"
552 namespace ebx {
553 constexpr uint32_t bmi1 = uint32_t(1) << 3;
554 constexpr uint32_t avx2 = uint32_t(1) << 5;
555 constexpr uint32_t bmi2 = uint32_t(1) << 8;
556 constexpr uint32_t avx512f = uint32_t(1) << 16;
557 constexpr uint32_t avx512dq = uint32_t(1) << 17;
558 constexpr uint32_t avx512cd = uint32_t(1) << 28;
559 constexpr uint32_t avx512bw = uint32_t(1) << 30;
560 constexpr uint32_t avx512vl = uint32_t(1) << 31;
561 } // namespace ebx
562
563 namespace ecx {
564 constexpr uint32_t avx512vbmi2 = uint32_t(1) << 6;
565 } // namespace ecx
566 } // namespace cpuid_bit
567 } // namespace
568
569 static inline void cpuid(uint32_t *eax, uint32_t *ebx, uint32_t *ecx,
570 uint32_t *edx) {
571 #if defined(_MSC_VER)
572 int cpu_info[4];
573 __cpuid(cpu_info, *eax);
574 *eax = cpu_info[0];
575 *ebx = cpu_info[1];
576 *ecx = cpu_info[2];
577 *edx = cpu_info[3];
578 #elif defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID)
579 uint32_t level = *eax;
580 __get_cpuid(level, eax, ebx, ecx, edx);
581 #else
582 uint32_t a = *eax, b, c = *ecx, d;
583 asm volatile("cpuid\n\t" : "+a"(a), "=b"(b), "+c"(c), "=d"(d));
584 *eax = a;
585 *ebx = b;
586 *ecx = c;
587 *edx = d;
588 #endif
589 }
590
591 static inline uint32_t detect_supported_architectures() {
592 uint32_t eax;
593 uint32_t ebx = 0;
594 uint32_t ecx = 0;
595 uint32_t edx = 0;
596 uint32_t host_isa = 0x0;
597
598 // EBX for EAX=0x1
599 eax = 0x1;
600 cpuid(&eax, &ebx, &ecx, &edx);
601
602 if (ecx & cpuid_bit::sse42) {
603 host_isa |= instruction_set::SSE42;
604 }
605
606 if (ecx & cpuid_bit::pclmulqdq) {
607 host_isa |= instruction_set::PCLMULQDQ;
608 }
609
610 // ECX for EAX=0x7
611 eax = 0x7;
612 ecx = 0x0; // Sub-leaf = 0
613 cpuid(&eax, &ebx, &ecx, &edx);
614 if (ebx & cpuid_bit::ebx::avx2) {
615 host_isa |= instruction_set::AVX2;
616 }
617 if (ebx & cpuid_bit::ebx::bmi1) {
618 host_isa |= instruction_set::BMI1;
619 }
620 if (ebx & cpuid_bit::ebx::bmi2) {
621 host_isa |= instruction_set::BMI2;
622 }
623 if (ebx & cpuid_bit::ebx::avx512f) {
624 host_isa |= instruction_set::AVX512F;
625 }
626 if (ebx & cpuid_bit::ebx::avx512bw) {
627 host_isa |= instruction_set::AVX512BW;
628 }
629 if (ebx & cpuid_bit::ebx::avx512cd) {
630 host_isa |= instruction_set::AVX512CD;
631 }
632 if (ebx & cpuid_bit::ebx::avx512dq) {
633 host_isa |= instruction_set::AVX512DQ;
634 }
635 if (ebx & cpuid_bit::ebx::avx512vl) {
636 host_isa |= instruction_set::AVX512VL;
637 }
638 if (ecx & cpuid_bit::ecx::avx512vbmi2) {
639 host_isa |= instruction_set::AVX512VBMI2;
640 }
641 return host_isa;
642 }
643 #else // fallback
644
645 static inline uint32_t detect_supported_architectures() {
646 return instruction_set::DEFAULT;
647 }
648
649 #endif // end SIMD extension detection code
650
651 } // namespace internal
652 } // namespace is_utf8_internals
653
654 #endif // IS_UTF8_INTERNAL_ISADETECTION_H
655
656 namespace is_utf8_internals {
657
658 /**
659 * Validate the UTF-8 string. This function may be best when you expect
660 * the input to be almost always valid. Otherwise, consider using
661 * validate_utf8_with_errors.
662 *
663 * Overridden by each implementation.
664 *
665 * @param buf the UTF-8 string to validate.
666 * @param len the length of the string in bytes.
667 * @return true if and only if the string is valid UTF-8.
668 */
669 bool validate_utf8(const char *buf, size_t len) noexcept;
670
671 class implementation {
672 public:
673 virtual const std::string &name() const { return _name; }
674
675 virtual const std::string &description() const { return _description; }
676
677 bool supported_by_runtime_system() const;
678
679 virtual uint32_t required_instruction_sets() const {
680 return _required_instruction_sets;
681 }
682
683 /**
684 * Validate the UTF-8 string.
685 *
686 * Overridden by each implementation.
687 *
688 * @param buf the UTF-8 string to validate.
689 * @param len the length of the string in bytes.
690 * @return true if and only if the string is valid UTF-8.
691 */
692 is_utf8_warn_unused virtual bool validate_utf8(const char *buf,
693 size_t len) const noexcept = 0;
694
695 protected:
696 /** @private Construct an implementation with the given name and description.
697 * For subclasses. */
698 is_utf8_really_inline implementation(std::string name,
699 std::string description,
700 uint32_t required_instruction_sets)
701 : _name(name), _description(description),
702 _required_instruction_sets(required_instruction_sets) {}
703 virtual ~implementation() = default;
704
705 private:
706 /**
707 * The name of this implementation.
708 */
709 const std::string _name;
710
711 /**
712 * The description of this implementation.
713 */
714 const std::string _description;
715
716 /**
717 * Instruction sets required for this implementation.
718 */
719 const uint32_t _required_instruction_sets;
720 };
721
722 /** @private */
723 namespace internal {
724
725 class available_implementation_list {
726 public:
727 /** Get the list of available implementations */
728 is_utf8_really_inline available_implementation_list() {}
729 /** Number of implementations */
730 size_t size() const noexcept;
731 /** STL const begin() iterator */
732 const implementation *const *begin() const noexcept;
733 /** STL const end() iterator */
734 const implementation *const *end() const noexcept;
735
736 /**
737 * Get the implementation with the given name.
738 *
739 * Case sensitive.
740 *
741 * const implementation *impl =
742 * is_utf8_internals::available_implementations["westmere"]; if (!impl) {
743 * exit(1); } if (!imp->supported_by_runtime_system()) { exit(1); }
744 * is_utf8_internals::active_implementation = impl;
745 *
746 * @param name the implementation to find, e.g. "westmere", "haswell", "arm64"
747 * @return the implementation, or nullptr if the parse failed.
748 */
749 const implementation *operator[](const std::string &name) const noexcept {
750 for (const implementation *impl : *this) {
751 if (impl->name() == name) {
752 return impl;
753 }
754 }
755 return nullptr;
756 }
757
758 /**
759 * Detect the most advanced implementation supported by the current host.
760 *
761 * This is used to initialize the implementation on startup.
762 *
763 * const implementation *impl =
764 * is_utf8_internals::available_implementation::detect_best_supported();
765 * is_utf8_internals::active_implementation = impl;
766 *
767 * @return the most advanced supported implementation for the current host, or
768 * an implementation that returns UNSUPPORTED_ARCHITECTURE if there is no
769 * supported implementation. Will never return nullptr.
770 */
771 const implementation *detect_best_supported() const noexcept;
772 };
773
774 template <typename T> class atomic_ptr {
775 public:
776 atomic_ptr(T *_ptr) : ptr{_ptr} {}
777
778 #if defined(IS_UTF8_NO_THREADS)
779 operator const T *() const { return ptr; }
780 const T &operator*() const { return *ptr; }
781 const T *operator->() const { return ptr; }
782
783 operator T *() { return ptr; }
784 T &operator*() { return *ptr; }
785 T *operator->() { return ptr; }
786 atomic_ptr &operator=(T *_ptr) {
787 ptr = _ptr;
788 return *this;
789 }
790
791 #else
792 operator const T *() const { return ptr.load(); }
793 const T &operator*() const { return *ptr; }
794 const T *operator->() const { return ptr.load(); }
795
796 operator T *() { return ptr.load(); }
797 T &operator*() { return *ptr; }
798 T *operator->() { return ptr.load(); }
799 atomic_ptr &operator=(T *_ptr) {
800 ptr = _ptr;
801 return *this;
802 }
803
804 #endif
805
806 private:
807 #if defined(IS_UTF8_NO_THREADS)
808 T *ptr;
809 #else
810 std::atomic<T *> ptr;
811 #endif
812 };
813
814 } // namespace internal
815
816 /**
817 * The list of available implementations compiled into simdutf.
818 */
819 extern IS_UTF8_DLLIMPORTEXPORT const internal::available_implementation_list& get_available_implementations();
820
821 /**
822 * The active implementation.
823 *
824 * Automatically initialized on first use to the most advanced implementation
825 * supported by this hardware.
826 */
827 extern IS_UTF8_DLLIMPORTEXPORT internal::atomic_ptr<const implementation>& get_active_implementation();
828
829 } // namespace is_utf8_internals
830
831 #endif // IS_UTF8_IMPLEMENTATION_H
832
833 // Implementation-internal files (must be included before the implementations
834 // themselves, to keep amalgamation working--otherwise, the first time a file is
835 // included, it might be put inside the #ifdef
836 // IS_UTF8_IMPLEMENTATION_ARM64/FALLBACK/etc., which means the other
837 // implementations can't compile unless that implementation is turned on).
838
839 IS_UTF8_POP_DISABLE_WARNINGS
840
841 #endif // IS_UTF8_H
842
843 #include <climits>
844 #include <initializer_list>
845
846 // Useful for debugging purposes
847 namespace is_utf8_internals {
848 namespace {
849
850 template <typename T> std::string toBinaryString(T b) {
851 std::string binary = "";
852 T mask = T(1) << (sizeof(T) * CHAR_BIT - 1);
853 while (mask > 0) {
854 binary += ((b & mask) == 0) ? '0' : '1';
855 mask >>= 1;
856 }
857 return binary;
858 }
859 } // namespace
860 } // namespace is_utf8_internals
861
862 // Implementations
863 // The best choice should always come first!
864
865 #ifndef IS_UTF8_ARM64_H
866 #define IS_UTF8_ARM64_H
867
868 #ifdef IS_UTF8_FALLBACK_H
869 #error "arm64.h must be included before fallback.h"
870 #endif
871
872 #ifndef IS_UTF8_IMPLEMENTATION_ARM64
873 #define IS_UTF8_IMPLEMENTATION_ARM64 (IS_UTF8_IS_ARM64)
874 #endif
875
876 #if IS_UTF8_IMPLEMENTATION_ARM64 &&IS_UTF8_IS_ARM64
877 #define IS_UTF8_CAN_ALWAYS_RUN_ARM64 1
878 #else
879 #define IS_UTF8_CAN_ALWAYS_RUN_ARM64 0
880 #endif
881
882
883 #if IS_UTF8_IMPLEMENTATION_ARM64
884
885 namespace is_utf8_internals {
886 /**
887 * Implementation for NEON (ARMv8).
888 */
889 namespace arm64 {} // namespace arm64
890 } // namespace is_utf8_internals
891
892 #ifndef IS_UTF8_ARM64_IMPLEMENTATION_H
893 #define IS_UTF8_ARM64_IMPLEMENTATION_H
894
895 namespace is_utf8_internals {
896 namespace arm64 {
897
898 class implementation final : public is_utf8_internals::implementation {
899 public:
900 is_utf8_really_inline implementation()
901 : is_utf8_internals::implementation("arm64", "ARM NEON",
902 internal::instruction_set::NEON) {}
903 is_utf8_warn_unused bool validate_utf8(const char *buf,
904 size_t len) const noexcept final;
905 };
906
907 } // namespace arm64
908 } // namespace is_utf8_internals
909
910 #endif // IS_UTF8_ARM64_IMPLEMENTATION_H
911
912 // redefining IS_UTF8_IMPLEMENTATION to "arm64"
913 // #define IS_UTF8_IMPLEMENTATION arm64
914
915 // Declarations
916
917 #ifndef IS_UTF8_ARM64_INTRINSICS_H
918 #define IS_UTF8_ARM64_INTRINSICS_H
919
920 // This should be the correct header whether
921 // you use visual studio or other compilers.
922 #include <arm_neon.h>
923
924 #endif // IS_UTF8_ARM64_INTRINSICS_H
925
926 #ifndef IS_UTF8_ARM64_BITMANIPULATION_H
927 #define IS_UTF8_ARM64_BITMANIPULATION_H
928
929 namespace is_utf8_internals {
930 namespace arm64 {
931 namespace {} // unnamed namespace
932 } // namespace arm64
933 } // namespace is_utf8_internals
934
935 #endif // IS_UTF8_ARM64_BITMANIPULATION_H
936
937 #ifndef IS_UTF8_ARM64_SIMD_H
938 #define IS_UTF8_ARM64_SIMD_H
939
940 #include <type_traits>
941
942 namespace is_utf8_internals {
943 namespace arm64 {
944 namespace {
945 namespace simd {
946
947 #ifdef IS_UTF8_REGULAR_VISUAL_STUDIO
948 namespace {
949 // Start of private section with Visual Studio workaround
950
951 /**
952 * make_uint8x16_t initializes a SIMD register (uint8x16_t).
953 * This is needed because, incredibly, the syntax uint8x16_t x = {1,2,3...}
954 * is not recognized under Visual Studio! This is a workaround.
955 * Using a std::initializer_list<uint8_t> as a parameter resulted in
956 * inefficient code. With the current approach, if the parameters are
957 * compile-time constants,
958 * GNU GCC compiles it to ldr, the same as uint8x16_t x = {1,2,3...}.
959 * You should not use this function except for compile-time constants:
960 * it is not efficient.
961 */
962 is_utf8_really_inline uint8x16_t make_uint8x16_t(
963 uint8_t x1, uint8_t x2, uint8_t x3, uint8_t x4, uint8_t x5, uint8_t x6,
964 uint8_t x7, uint8_t x8, uint8_t x9, uint8_t x10, uint8_t x11, uint8_t x12,
965 uint8_t x13, uint8_t x14, uint8_t x15, uint8_t x16) {
966 // Doing a load like so end ups generating worse code.
967 // uint8_t array[16] = {x1, x2, x3, x4, x5, x6, x7, x8,
968 // x9, x10,x11,x12,x13,x14,x15,x16};
969 // return vld1q_u8(array);
970 uint8x16_t x{};
971 // incredibly, Visual Studio does not allow x[0] = x1
972 x = vsetq_lane_u8(x1, x, 0);
973 x = vsetq_lane_u8(x2, x, 1);
974 x = vsetq_lane_u8(x3, x, 2);
975 x = vsetq_lane_u8(x4, x, 3);
976 x = vsetq_lane_u8(x5, x, 4);
977 x = vsetq_lane_u8(x6, x, 5);
978 x = vsetq_lane_u8(x7, x, 6);
979 x = vsetq_lane_u8(x8, x, 7);
980 x = vsetq_lane_u8(x9, x, 8);
981 x = vsetq_lane_u8(x10, x, 9);
982 x = vsetq_lane_u8(x11, x, 10);
983 x = vsetq_lane_u8(x12, x, 11);
984 x = vsetq_lane_u8(x13, x, 12);
985 x = vsetq_lane_u8(x14, x, 13);
986 x = vsetq_lane_u8(x15, x, 14);
987 x = vsetq_lane_u8(x16, x, 15);
988 return x;
989 }
990
991 // We have to do the same work for make_int8x16_t
992 is_utf8_really_inline int8x16_t
993 make_int8x16_t(int8_t x1, int8_t x2, int8_t x3, int8_t x4, int8_t x5, int8_t x6,
994 int8_t x7, int8_t x8, int8_t x9, int8_t x10, int8_t x11,
995 int8_t x12, int8_t x13, int8_t x14, int8_t x15, int8_t x16) {
996 // Doing a load like so end ups generating worse code.
997 // int8_t array[16] = {x1, x2, x3, x4, x5, x6, x7, x8,
998 // x9, x10,x11,x12,x13,x14,x15,x16};
999 // return vld1q_s8(array);
1000 int8x16_t x{};
1001 // incredibly, Visual Studio does not allow x[0] = x1
1002 x = vsetq_lane_s8(x1, x, 0);
1003 x = vsetq_lane_s8(x2, x, 1);
1004 x = vsetq_lane_s8(x3, x, 2);
1005 x = vsetq_lane_s8(x4, x, 3);
1006 x = vsetq_lane_s8(x5, x, 4);
1007 x = vsetq_lane_s8(x6, x, 5);
1008 x = vsetq_lane_s8(x7, x, 6);
1009 x = vsetq_lane_s8(x8, x, 7);
1010 x = vsetq_lane_s8(x9, x, 8);
1011 x = vsetq_lane_s8(x10, x, 9);
1012 x = vsetq_lane_s8(x11, x, 10);
1013 x = vsetq_lane_s8(x12, x, 11);
1014 x = vsetq_lane_s8(x13, x, 12);
1015 x = vsetq_lane_s8(x14, x, 13);
1016 x = vsetq_lane_s8(x15, x, 14);
1017 x = vsetq_lane_s8(x16, x, 15);
1018 return x;
1019 }
1020
1021 is_utf8_really_inline uint8x8_t make_uint8x8_t(uint8_t x1, uint8_t x2,
1022 uint8_t x3, uint8_t x4,
1023 uint8_t x5, uint8_t x6,
1024 uint8_t x7, uint8_t x8) {
1025 uint8x8_t x{};
1026 x = vset_lane_u8(x1, x, 0);
1027 x = vset_lane_u8(x2, x, 1);
1028 x = vset_lane_u8(x3, x, 2);
1029 x = vset_lane_u8(x4, x, 3);
1030 x = vset_lane_u8(x5, x, 4);
1031 x = vset_lane_u8(x6, x, 5);
1032 x = vset_lane_u8(x7, x, 6);
1033 x = vset_lane_u8(x8, x, 7);
1034 return x;
1035 }
1036
1037 is_utf8_really_inline uint16x8_t make_uint16x8_t(uint16_t x1, uint16_t x2,
1038 uint16_t x3, uint16_t x4,
1039 uint16_t x5, uint16_t x6,
1040 uint16_t x7, uint16_t x8) {
1041 uint16x8_t x{};
1042 x = vsetq_lane_u16(x1, x, 0);
1043 x = vsetq_lane_u16(x2, x, 1);
1044 x = vsetq_lane_u16(x3, x, 2);
1045 x = vsetq_lane_u16(x4, x, 3);
1046 x = vsetq_lane_u16(x5, x, 4);
1047 x = vsetq_lane_u16(x6, x, 5);
1048 x = vsetq_lane_u16(x7, x, 6);
1049 x = vsetq_lane_u16(x8, x, 7);
1050 ;
1051 return x;
1052 }
1053
1054 is_utf8_really_inline int16x8_t make_int16x8_t(int16_t x1, int16_t x2,
1055 int16_t x3, int16_t x4,
1056 int16_t x5, int16_t x6,
1057 int16_t x7, int16_t x8) {
1058 uint16x8_t x{};
1059 x = vsetq_lane_s16(x1, x, 0);
1060 x = vsetq_lane_s16(x2, x, 1);
1061 x = vsetq_lane_s16(x3, x, 2);
1062 x = vsetq_lane_s16(x4, x, 3);
1063 x = vsetq_lane_s16(x5, x, 4);
1064 x = vsetq_lane_s16(x6, x, 5);
1065 x = vsetq_lane_s16(x7, x, 6);
1066 x = vsetq_lane_s16(x8, x, 7);
1067 ;
1068 return x;
1069 }
1070
1071 // End of private section with Visual Studio workaround
1072 } // namespace
1073 #endif // IS_UTF8_REGULAR_VISUAL_STUDIO
1074
1075 template <typename T> struct simd8;
1076
1077 //
1078 // Base class of simd8<uint8_t> and simd8<bool>, both of which use uint8x16_t
1079 // internally.
1080 //
1081 template <typename T, typename Mask = simd8<bool>> struct base_u8 {
1082 uint8x16_t value;
1083 static const int SIZE = sizeof(value);
1084
1085 // Conversion from/to SIMD register
1086 is_utf8_really_inline base_u8(const uint8x16_t _value) : value(_value) {}
1087 is_utf8_really_inline operator const uint8x16_t &() const {
1088 return this->value;
1089 }
1090 is_utf8_really_inline operator uint8x16_t &() { return this->value; }
1091 is_utf8_really_inline T first() const { return vgetq_lane_u8(*this, 0); }
1092 is_utf8_really_inline T last() const { return vgetq_lane_u8(*this, 15); }
1093
1094 // Bit operations
1095 is_utf8_really_inline simd8<T> operator|(const simd8<T> other) const {
1096 return vorrq_u8(*this, other);
1097 }
1098 is_utf8_really_inline simd8<T> operator&(const simd8<T> other) const {
1099 return vandq_u8(*this, other);
1100 }
1101 is_utf8_really_inline simd8<T> operator^(const simd8<T> other) const {
1102 return veorq_u8(*this, other);
1103 }
1104 is_utf8_really_inline simd8<T> bit_andnot(const simd8<T> other) const {
1105 return vbicq_u8(*this, other);
1106 }
1107 is_utf8_really_inline simd8<T> operator~() const { return *this ^ 0xFFu; }
1108 is_utf8_really_inline simd8<T> &operator|=(const simd8<T> other) {
1109 auto this_cast = static_cast<simd8<T> *>(this);
1110 *this_cast = *this_cast | other;
1111 return *this_cast;
1112 }
1113 is_utf8_really_inline simd8<T> &operator&=(const simd8<T> other) {
1114 auto this_cast = static_cast<simd8<T> *>(this);
1115 *this_cast = *this_cast & other;
1116 return *this_cast;
1117 }
1118 is_utf8_really_inline simd8<T> &operator^=(const simd8<T> other) {
1119 auto this_cast = static_cast<simd8<T> *>(this);
1120 *this_cast = *this_cast ^ other;
1121 return *this_cast;
1122 }
1123
1124 friend is_utf8_really_inline Mask operator==(const simd8<T> lhs,
1125 const simd8<T> rhs) {
1126 return vceqq_u8(lhs, rhs);
1127 }
1128
1129 template <int N = 1>
1130 is_utf8_really_inline simd8<T> prev(const simd8<T> prev_chunk) const {
1131 return vextq_u8(prev_chunk, *this, 16 - N);
1132 }
1133 };
1134
1135 // SIMD byte mask type (returned by things like eq and gt)
1136 template <> struct simd8<bool> : base_u8<bool> {
1137 typedef uint16_t bitmask_t;
1138 typedef uint32_t bitmask2_t;
1139
1140 static is_utf8_really_inline simd8<bool> splat(bool _value) {
1141 return vmovq_n_u8(uint8_t(-(!!_value)));
1142 }
1143
1144 is_utf8_really_inline simd8(const uint8x16_t _value)
1145 : base_u8<bool>(_value) {}
1146 // False constructor
1147 is_utf8_really_inline simd8() : simd8(vdupq_n_u8(0)) {}
1148 // Splat constructor
1149 is_utf8_really_inline simd8(bool _value) : simd8(splat(_value)) {}
1150 is_utf8_really_inline void store(uint8_t dst[16]) const {
1151 return vst1q_u8(dst, *this);
1152 }
1153
1154 // We return uint32_t instead of uint16_t because that seems to be more
1155 // efficient for most purposes (cutting it down to uint16_t costs performance
1156 // in some compilers).
1157 is_utf8_really_inline uint32_t to_bitmask() const {
1158 #ifdef IS_UTF8_REGULAR_VISUAL_STUDIO
1159 const uint8x16_t bit_mask =
1160 make_uint8x16_t(0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80, 0x01,
1161 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80);
1162 #else
1163 const uint8x16_t bit_mask = {0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80,
1164 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80};
1165 #endif
1166 auto minput = *this & bit_mask;
1167 uint8x16_t tmp = vpaddq_u8(minput, minput);
1168 tmp = vpaddq_u8(tmp, tmp);
1169 tmp = vpaddq_u8(tmp, tmp);
1170 return vgetq_lane_u16(vreinterpretq_u16_u8(tmp), 0);
1171 }
1172
1173 // Returns 4-bit out of each byte, alternating between the high 4 bits and low
1174 // bits result it is 64 bit. This method is expected to be faster than none()
1175 // and is equivalent when the vector register is the result of a comparison,
1176 // with byte values 0xff and 0x00.
1177 is_utf8_really_inline uint64_t to_bitmask64() const {
1178 return vget_lane_u64(
1179 vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(*this), 4)), 0);
1180 }
1181
1182 is_utf8_really_inline bool any() const { return vmaxvq_u8(*this) != 0; }
1183 is_utf8_really_inline bool none() const { return vmaxvq_u8(*this) == 0; }
1184 is_utf8_really_inline bool all() const { return vminvq_u8(*this) == 0xFF; }
1185 };
1186
1187 // Unsigned bytes
1188 template <> struct simd8<uint8_t> : base_u8<uint8_t> {
1189 static is_utf8_really_inline simd8<uint8_t> splat(uint8_t _value) {
1190 return vmovq_n_u8(_value);
1191 }
1192 static is_utf8_really_inline simd8<uint8_t> zero() { return vdupq_n_u8(0); }
1193 static is_utf8_really_inline simd8<uint8_t> load(const uint8_t *values) {
1194 return vld1q_u8(values);
1195 }
1196 is_utf8_really_inline simd8(const uint8x16_t _value)
1197 : base_u8<uint8_t>(_value) {}
1198 // Zero constructor
1199 is_utf8_really_inline simd8() : simd8(zero()) {}
1200 // Array constructor
1201 is_utf8_really_inline simd8(const uint8_t values[16]) : simd8(load(values)) {}
1202 // Splat constructor
1203 is_utf8_really_inline simd8(uint8_t _value) : simd8(splat(_value)) {}
1204 // Member-by-member initialization
1205 #ifdef IS_UTF8_REGULAR_VISUAL_STUDIO
1206 is_utf8_really_inline
1207 simd8(uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5,
1208 uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, uint8_t v10,
1209 uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15)
1210 : simd8(make_uint8x16_t(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11,
1211 v12, v13, v14, v15)) {}
1212 #else
1213 is_utf8_really_inline
1214 simd8(uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5,
1215 uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, uint8_t v10,
1216 uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15)
1217 : simd8(uint8x16_t{v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12,
1218 v13, v14, v15}) {}
1219 #endif
1220
1221 // Repeat 16 values as many times as necessary (usually for lookup tables)
1222 is_utf8_really_inline static simd8<uint8_t>
1223 repeat_16(uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4,
1224 uint8_t v5, uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9,
1225 uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14,
1226 uint8_t v15) {
1227 return simd8<uint8_t>(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12,
1228 v13, v14, v15);
1229 }
1230
1231 // Store to array
1232 is_utf8_really_inline void store(uint8_t dst[16]) const {
1233 return vst1q_u8(dst, *this);
1234 }
1235
1236 // Saturated math
1237 is_utf8_really_inline simd8<uint8_t>
1238 saturating_add(const simd8<uint8_t> other) const {
1239 return vqaddq_u8(*this, other);
1240 }
1241 is_utf8_really_inline simd8<uint8_t>
1242 saturating_sub(const simd8<uint8_t> other) const {
1243 return vqsubq_u8(*this, other);
1244 }
1245
1246 // Addition/subtraction are the same for signed and unsigned
1247 is_utf8_really_inline simd8<uint8_t>
1248 operator+(const simd8<uint8_t> other) const {
1249 return vaddq_u8(*this, other);
1250 }
1251 is_utf8_really_inline simd8<uint8_t>
1252 operator-(const simd8<uint8_t> other) const {
1253 return vsubq_u8(*this, other);
1254 }
1255 is_utf8_really_inline simd8<uint8_t> &operator+=(const simd8<uint8_t> other) {
1256 *this = *this + other;
1257 return *this;
1258 }
1259 is_utf8_really_inline simd8<uint8_t> &operator-=(const simd8<uint8_t> other) {
1260 *this = *this - other;
1261 return *this;
1262 }
1263
1264 // Order-specific operations
1265 is_utf8_really_inline uint8_t max_val() const { return vmaxvq_u8(*this); }
1266 is_utf8_really_inline uint8_t min_val() const { return vminvq_u8(*this); }
1267 is_utf8_really_inline simd8<uint8_t>
1268 max_val(const simd8<uint8_t> other) const {
1269 return vmaxq_u8(*this, other);
1270 }
1271 is_utf8_really_inline simd8<uint8_t>
1272 min_val(const simd8<uint8_t> other) const {
1273 return vminq_u8(*this, other);
1274 }
1275 is_utf8_really_inline simd8<bool>
1276 operator<=(const simd8<uint8_t> other) const {
1277 return vcleq_u8(*this, other);
1278 }
1279 is_utf8_really_inline simd8<bool>
1280 operator>=(const simd8<uint8_t> other) const {
1281 return vcgeq_u8(*this, other);
1282 }
1283 is_utf8_really_inline simd8<bool>
1284 operator<(const simd8<uint8_t> other) const {
1285 return vcltq_u8(*this, other);
1286 }
1287 is_utf8_really_inline simd8<bool>
1288 operator>(const simd8<uint8_t> other) const {
1289 return vcgtq_u8(*this, other);
1290 }
1291 // Same as >, but instead of guaranteeing all 1's == true, false = 0 and true
1292 // = nonzero. For ARM, returns all 1's.
1293 is_utf8_really_inline simd8<uint8_t>
1294 gt_bits(const simd8<uint8_t> other) const {
1295 return simd8<uint8_t>(*this > other);
1296 }
1297 // Same as <, but instead of guaranteeing all 1's == true, false = 0 and true
1298 // = nonzero. For ARM, returns all 1's.
1299 is_utf8_really_inline simd8<uint8_t>
1300 lt_bits(const simd8<uint8_t> other) const {
1301 return simd8<uint8_t>(*this < other);
1302 }
1303
1304 // Bit-specific operations
1305 is_utf8_really_inline simd8<bool> any_bits_set(simd8<uint8_t> bits) const {
1306 return vtstq_u8(*this, bits);
1307 }
1308 is_utf8_really_inline bool is_ascii() const {
1309 return this->max_val() < 0b10000000u;
1310 }
1311
1312 is_utf8_really_inline bool any_bits_set_anywhere() const {
1313 return this->max_val() != 0;
1314 }
1315 is_utf8_really_inline bool any_bits_set_anywhere(simd8<uint8_t> bits) const {
1316 return (*this & bits).any_bits_set_anywhere();
1317 }
1318 template <int N> is_utf8_really_inline simd8<uint8_t> shr() const {
1319 return vshrq_n_u8(*this, N);
1320 }
1321 template <int N> is_utf8_really_inline simd8<uint8_t> shl() const {
1322 return vshlq_n_u8(*this, N);
1323 }
1324
1325 // Perform a lookup assuming the value is between 0 and 16 (undefined behavior
1326 // for out of range values)
1327 template <typename L>
1328 is_utf8_really_inline simd8<L> lookup_16(simd8<L> lookup_table) const {
1329 return lookup_table.apply_lookup_16_to(*this);
1330 }
1331
1332 template <typename L>
1333 is_utf8_really_inline simd8<L>
1334 lookup_16(L replace0, L replace1, L replace2, L replace3, L replace4,
1335 L replace5, L replace6, L replace7, L replace8, L replace9,
1336 L replace10, L replace11, L replace12, L replace13, L replace14,
1337 L replace15) const {
1338 return lookup_16(simd8<L>::repeat_16(
1339 replace0, replace1, replace2, replace3, replace4, replace5, replace6,
1340 replace7, replace8, replace9, replace10, replace11, replace12,
1341 replace13, replace14, replace15));
1342 }
1343
1344 template <typename T>
1345 is_utf8_really_inline simd8<uint8_t>
1346 apply_lookup_16_to(const simd8<T> original) const {
1347 return vqtbl1q_u8(*this, simd8<uint8_t>(original));
1348 }
1349 };
1350
1351 // Signed bytes
1352 template <> struct simd8<int8_t> {
1353 int8x16_t value;
1354
1355 static is_utf8_really_inline simd8<int8_t> splat(int8_t _value) {
1356 return vmovq_n_s8(_value);
1357 }
1358 static is_utf8_really_inline simd8<int8_t> zero() { return vdupq_n_s8(0); }
1359 static is_utf8_really_inline simd8<int8_t> load(const int8_t values[16]) {
1360 return vld1q_s8(values);
1361 }
1362 template <endianness big_endian>
1363 is_utf8_really_inline void store_ascii_as_utf16(char16_t *p) const {
1364 uint16x8_t first = vmovl_u8(vget_low_u8(vreinterpretq_u8_s8(this->value)));
1365 uint16x8_t second = vmovl_high_u8(vreinterpretq_u8_s8(this->value));
1366 if (big_endian) {
1367 #ifdef IS_UTF8_REGULAR_VISUAL_STUDIO
1368 const uint8x16_t swap =
1369 make_uint8x16_t(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14);
1370 #else
1371 const uint8x16_t swap = {1, 0, 3, 2, 5, 4, 7, 6,
1372 9, 8, 11, 10, 13, 12, 15, 14};
1373 #endif
1374 first =
1375 vreinterpretq_u16_u8(vqtbl1q_u8(vreinterpretq_u8_u16(first), swap));
1376 second =
1377 vreinterpretq_u16_u8(vqtbl1q_u8(vreinterpretq_u8_u16(second), swap));
1378 }
1379 vst1q_u16(reinterpret_cast<uint16_t *>(p), first);
1380 vst1q_u16(reinterpret_cast<uint16_t *>(p + 8), second);
1381 }
1382 is_utf8_really_inline void store_ascii_as_utf32(char32_t *p) const {
1383 vst1q_u32(reinterpret_cast<uint32_t *>(p),
1384 vmovl_u16(vget_low_u16(
1385 vmovl_u8(vget_low_u8(vreinterpretq_u8_s8(this->value))))));
1386 vst1q_u32(reinterpret_cast<uint32_t *>(p + 4),
1387 vmovl_high_u16(
1388 vmovl_u8(vget_low_u8(vreinterpretq_u8_s8(this->value)))));
1389 vst1q_u32(reinterpret_cast<uint32_t *>(p + 8),
1390 vmovl_u16(vget_low_u16(
1391 vmovl_high_u8(vreinterpretq_u8_s8(this->value)))));
1392 vst1q_u32(reinterpret_cast<uint32_t *>(p + 12),
1393 vmovl_high_u16(vmovl_high_u8(vreinterpretq_u8_s8(this->value))));
1394 }
1395 // Conversion from/to SIMD register
1396 is_utf8_really_inline simd8(const int8x16_t _value) : value{_value} {}
1397 is_utf8_really_inline operator const int8x16_t &() const {
1398 return this->value;
1399 }
1400 is_utf8_really_inline operator const uint8x16_t() const {
1401 return vreinterpretq_u8_s8(this->value);
1402 }
1403 is_utf8_really_inline operator int8x16_t &() { return this->value; }
1404
1405 // Zero constructor
1406 is_utf8_really_inline simd8() : simd8(zero()) {}
1407 // Splat constructor
1408 is_utf8_really_inline simd8(int8_t _value) : simd8(splat(_value)) {}
1409 // Array constructor
1410 is_utf8_really_inline simd8(const int8_t *values) : simd8(load(values)) {}
1411 // Member-by-member initialization
1412 #ifdef IS_UTF8_REGULAR_VISUAL_STUDIO
1413 is_utf8_really_inline simd8(int8_t v0, int8_t v1, int8_t v2, int8_t v3,
1414 int8_t v4, int8_t v5, int8_t v6, int8_t v7,
1415 int8_t v8, int8_t v9, int8_t v10, int8_t v11,
1416 int8_t v12, int8_t v13, int8_t v14, int8_t v15)
1417 : simd8(make_int8x16_t(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11,
1418 v12, v13, v14, v15)) {}
1419 #else
1420 is_utf8_really_inline simd8(int8_t v0, int8_t v1, int8_t v2, int8_t v3,
1421 int8_t v4, int8_t v5, int8_t v6, int8_t v7,
1422 int8_t v8, int8_t v9, int8_t v10, int8_t v11,
1423 int8_t v12, int8_t v13, int8_t v14, int8_t v15)
1424 : simd8(int8x16_t{v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12,
1425 v13, v14, v15}) {}
1426 #endif
1427 // Repeat 16 values as many times as necessary (usually for lookup tables)
1428 is_utf8_really_inline static simd8<int8_t>
1429 repeat_16(int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5,
1430 int8_t v6, int8_t v7, int8_t v8, int8_t v9, int8_t v10, int8_t v11,
1431 int8_t v12, int8_t v13, int8_t v14, int8_t v15) {
1432 return simd8<int8_t>(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12,
1433 v13, v14, v15);
1434 }
1435
1436 // Store to array
1437 is_utf8_really_inline void store(int8_t dst[16]) const {
1438 return vst1q_s8(dst, value);
1439 }
1440 // Explicit conversion to/from unsigned
1441 //
1442 // Under Visual Studio/ARM64 uint8x16_t and int8x16_t are apparently the same
1443 // type. In theory, we could check this occurrence with std::same_as and
1444 // std::enabled_if but it is C++14 and relatively ugly and hard to read.
1445 #ifndef IS_UTF8_REGULAR_VISUAL_STUDIO
1446 is_utf8_really_inline explicit simd8(const uint8x16_t other)
1447 : simd8(vreinterpretq_s8_u8(other)) {}
1448 #endif
1449 is_utf8_really_inline operator simd8<uint8_t>() const {
1450 return vreinterpretq_u8_s8(this->value);
1451 }
1452
1453 is_utf8_really_inline simd8<int8_t>
1454 operator|(const simd8<int8_t> other) const {
1455 return vorrq_s8(value, other.value);
1456 }
1457 is_utf8_really_inline simd8<int8_t>
1458 operator&(const simd8<int8_t> other) const {
1459 return vandq_s8(value, other.value);
1460 }
1461 is_utf8_really_inline simd8<int8_t>
1462 operator^(const simd8<int8_t> other) const {
1463 return veorq_s8(value, other.value);
1464 }
1465 is_utf8_really_inline simd8<int8_t>
1466 bit_andnot(const simd8<int8_t> other) const {
1467 return vbicq_s8(value, other.value);
1468 }
1469
1470 // Math
1471 is_utf8_really_inline simd8<int8_t>
1472 operator+(const simd8<int8_t> other) const {
1473 return vaddq_s8(value, other.value);
1474 }
1475 is_utf8_really_inline simd8<int8_t>
1476 operator-(const simd8<int8_t> other) const {
1477 return vsubq_s8(value, other.value);
1478 }
1479 is_utf8_really_inline simd8<int8_t> &operator+=(const simd8<int8_t> other) {
1480 *this = *this + other;
1481 return *this;
1482 }
1483 is_utf8_really_inline simd8<int8_t> &operator-=(const simd8<int8_t> other) {
1484 *this = *this - other;
1485 return *this;
1486 }
1487
1488 is_utf8_really_inline int8_t max_val() const { return vmaxvq_s8(value); }
1489 is_utf8_really_inline int8_t min_val() const { return vminvq_s8(value); }
1490 is_utf8_really_inline bool is_ascii() const { return this->min_val() >= 0; }
1491
1492 // Order-sensitive comparisons
1493 is_utf8_really_inline simd8<int8_t> max_val(const simd8<int8_t> other) const {
1494 return vmaxq_s8(value, other.value);
1495 }
1496 is_utf8_really_inline simd8<int8_t> min_val(const simd8<int8_t> other) const {
1497 return vminq_s8(value, other.value);
1498 }
1499 is_utf8_really_inline simd8<bool> operator>(const simd8<int8_t> other) const {
1500 return vcgtq_s8(value, other.value);
1501 }
1502 is_utf8_really_inline simd8<bool> operator<(const simd8<int8_t> other) const {
1503 return vcltq_s8(value, other.value);
1504 }
1505 is_utf8_really_inline simd8<bool>
1506 operator==(const simd8<int8_t> other) const {
1507 return vceqq_s8(value, other.value);
1508 }
1509
1510 template <int N = 1>
1511 is_utf8_really_inline simd8<int8_t>
1512 prev(const simd8<int8_t> prev_chunk) const {
1513 return vextq_s8(prev_chunk, *this, 16 - N);
1514 }
1515
1516 // Perform a lookup assuming no value is larger than 16
1517 template <typename L>
1518 is_utf8_really_inline simd8<L> lookup_16(simd8<L> lookup_table) const {
1519 return lookup_table.apply_lookup_16_to(*this);
1520 }
1521 template <typename L>
1522 is_utf8_really_inline simd8<L>
1523 lookup_16(L replace0, L replace1, L replace2, L replace3, L replace4,
1524 L replace5, L replace6, L replace7, L replace8, L replace9,
1525 L replace10, L replace11, L replace12, L replace13, L replace14,
1526 L replace15) const {
1527 return lookup_16(simd8<L>::repeat_16(
1528 replace0, replace1, replace2, replace3, replace4, replace5, replace6,
1529 replace7, replace8, replace9, replace10, replace11, replace12,
1530 replace13, replace14, replace15));
1531 }
1532
1533 template <typename T>
1534 is_utf8_really_inline simd8<int8_t>
1535 apply_lookup_16_to(const simd8<T> original) {
1536 return vqtbl1q_s8(*this, simd8<uint8_t>(original));
1537 }
1538 };
1539
1540 template <typename T> struct simd8x64 {
1541 static constexpr int NUM_CHUNKS = 64 / sizeof(simd8<T>);
1542 static_assert(NUM_CHUNKS == 4,
1543 "ARM kernel should use four registers per 64-byte block.");
1544 simd8<T> chunks[NUM_CHUNKS];
1545
1546 simd8x64(const simd8x64<T> &o) = delete; // no copy allowed
1547 simd8x64<T> &
1548 operator=(const simd8<T> other) = delete; // no assignment allowed
1549 simd8x64() = delete; // no default constructor allowed
1550
1551 is_utf8_really_inline simd8x64(const simd8<T> chunk0, const simd8<T> chunk1,
1552 const simd8<T> chunk2, const simd8<T> chunk3)
1553 : chunks{chunk0, chunk1, chunk2, chunk3} {}
1554 is_utf8_really_inline simd8x64(const T *ptr)
1555 : chunks{simd8<T>::load(ptr),
1556 simd8<T>::load(ptr + sizeof(simd8<T>) / sizeof(T)),
1557 simd8<T>::load(ptr + 2 * sizeof(simd8<T>) / sizeof(T)),
1558 simd8<T>::load(ptr + 3 * sizeof(simd8<T>) / sizeof(T))} {}
1559
1560 is_utf8_really_inline void store(T *ptr) const {
1561 this->chunks[0].store(ptr + sizeof(simd8<T>) * 0 / sizeof(T));
1562 this->chunks[1].store(ptr + sizeof(simd8<T>) * 1 / sizeof(T));
1563 this->chunks[2].store(ptr + sizeof(simd8<T>) * 2 / sizeof(T));
1564 this->chunks[3].store(ptr + sizeof(simd8<T>) * 3 / sizeof(T));
1565 }
1566
1567 is_utf8_really_inline simd8x64<T> &operator|=(const simd8x64<T> &other) {
1568 this->chunks[0] |= other.chunks[0];
1569 this->chunks[1] |= other.chunks[1];
1570 this->chunks[2] |= other.chunks[2];
1571 this->chunks[3] |= other.chunks[3];
1572 return *this;
1573 }
1574
1575 is_utf8_really_inline simd8<T> reduce_or() const {
1576 return (this->chunks[0] | this->chunks[1]) |
1577 (this->chunks[2] | this->chunks[3]);
1578 }
1579
1580 is_utf8_really_inline bool is_ascii() const { return reduce_or().is_ascii(); }
1581
1582 template <endianness endian>
1583 is_utf8_really_inline void store_ascii_as_utf16(char16_t *ptr) const {
1584 this->chunks[0].template store_ascii_as_utf16<endian>(ptr +
1585 sizeof(simd8<T>) * 0);
1586 this->chunks[1].template store_ascii_as_utf16<endian>(ptr +
1587 sizeof(simd8<T>) * 1);
1588 this->chunks[2].template store_ascii_as_utf16<endian>(ptr +
1589 sizeof(simd8<T>) * 2);
1590 this->chunks[3].template store_ascii_as_utf16<endian>(ptr +
1591 sizeof(simd8<T>) * 3);
1592 }
1593
1594 is_utf8_really_inline void store_ascii_as_utf32(char32_t *ptr) const {
1595 this->chunks[0].store_ascii_as_utf32(ptr + sizeof(simd8<T>) * 0);
1596 this->chunks[1].store_ascii_as_utf32(ptr + sizeof(simd8<T>) * 1);
1597 this->chunks[2].store_ascii_as_utf32(ptr + sizeof(simd8<T>) * 2);
1598 this->chunks[3].store_ascii_as_utf32(ptr + sizeof(simd8<T>) * 3);
1599 }
1600
1601 is_utf8_really_inline uint64_t to_bitmask() const {
1602 #ifdef IS_UTF8_REGULAR_VISUAL_STUDIO
1603 const uint8x16_t bit_mask =
1604 make_uint8x16_t(0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80, 0x01,
1605 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80);
1606 #else
1607 const uint8x16_t bit_mask = {0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80,
1608 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80};
1609 #endif
1610 // Add each of the elements next to each other, successively, to stuff each
1611 // 8 byte mask into one.
1612 uint8x16_t sum0 =
1613 vpaddq_u8(vandq_u8(uint8x16_t(this->chunks[0]), bit_mask),
1614 vandq_u8(uint8x16_t(this->chunks[1]), bit_mask));
1615 uint8x16_t sum1 =
1616 vpaddq_u8(vandq_u8(uint8x16_t(this->chunks[2]), bit_mask),
1617 vandq_u8(uint8x16_t(this->chunks[3]), bit_mask));
1618 sum0 = vpaddq_u8(sum0, sum1);
1619 sum0 = vpaddq_u8(sum0, sum0);
1620 return vgetq_lane_u64(vreinterpretq_u64_u8(sum0), 0);
1621 }
1622
1623 is_utf8_really_inline uint64_t eq(const T m) const {
1624 const simd8<T> mask = simd8<T>::splat(m);
1625 return simd8x64<bool>(this->chunks[0] == mask, this->chunks[1] == mask,
1626 this->chunks[2] == mask, this->chunks[3] == mask)
1627 .to_bitmask();
1628 }
1629
1630 is_utf8_really_inline uint64_t lteq(const T m) const {
1631 const simd8<T> mask = simd8<T>::splat(m);
1632 return simd8x64<bool>(this->chunks[0] <= mask, this->chunks[1] <= mask,
1633 this->chunks[2] <= mask, this->chunks[3] <= mask)
1634 .to_bitmask();
1635 }
1636
1637 is_utf8_really_inline uint64_t in_range(const T low, const T high) const {
1638 const simd8<T> mask_low = simd8<T>::splat(low);
1639 const simd8<T> mask_high = simd8<T>::splat(high);
1640
1641 return simd8x64<bool>(
1642 (this->chunks[0] <= mask_high) & (this->chunks[0] >= mask_low),
1643 (this->chunks[1] <= mask_high) & (this->chunks[1] >= mask_low),
1644 (this->chunks[2] <= mask_high) & (this->chunks[2] >= mask_low),
1645 (this->chunks[3] <= mask_high) & (this->chunks[3] >= mask_low))
1646 .to_bitmask();
1647 }
1648 is_utf8_really_inline uint64_t not_in_range(const T low, const T high) const {
1649 const simd8<T> mask_low = simd8<T>::splat(low);
1650 const simd8<T> mask_high = simd8<T>::splat(high);
1651 return simd8x64<bool>(
1652 (this->chunks[0] > mask_high) | (this->chunks[0] < mask_low),
1653 (this->chunks[1] > mask_high) | (this->chunks[1] < mask_low),
1654 (this->chunks[2] > mask_high) | (this->chunks[2] < mask_low),
1655 (this->chunks[3] > mask_high) | (this->chunks[3] < mask_low))
1656 .to_bitmask();
1657 }
1658 is_utf8_really_inline uint64_t lt(const T m) const {
1659 const simd8<T> mask = simd8<T>::splat(m);
1660 return simd8x64<bool>(this->chunks[0] < mask, this->chunks[1] < mask,
1661 this->chunks[2] < mask, this->chunks[3] < mask)
1662 .to_bitmask();
1663 }
1664 is_utf8_really_inline uint64_t gt(const T m) const {
1665 const simd8<T> mask = simd8<T>::splat(m);
1666 return simd8x64<bool>(this->chunks[0] > mask, this->chunks[1] > mask,
1667 this->chunks[2] > mask, this->chunks[3] > mask)
1668 .to_bitmask();
1669 }
1670 is_utf8_really_inline uint64_t gteq(const T m) const {
1671 const simd8<T> mask = simd8<T>::splat(m);
1672 return simd8x64<bool>(this->chunks[0] >= mask, this->chunks[1] >= mask,
1673 this->chunks[2] >= mask, this->chunks[3] >= mask)
1674 .to_bitmask();
1675 }
1676 is_utf8_really_inline uint64_t gteq_unsigned(const uint8_t m) const {
1677 const simd8<uint8_t> mask = simd8<uint8_t>::splat(m);
1678 return simd8x64<bool>(simd8<uint8_t>(uint8x16_t(this->chunks[0])) >= mask,
1679 simd8<uint8_t>(uint8x16_t(this->chunks[1])) >= mask,
1680 simd8<uint8_t>(uint8x16_t(this->chunks[2])) >= mask,
1681 simd8<uint8_t>(uint8x16_t(this->chunks[3])) >= mask)
1682 .to_bitmask();
1683 }
1684 }; // struct simd8x64<T>
1685
1686 template <typename T> struct simd16;
1687
1688 template <typename T, typename Mask = simd16<bool>> struct base_u16 {
1689 uint16x8_t value;
1690 static const int SIZE = sizeof(value);
1691
1692 // Conversion from/to SIMD register
1693 is_utf8_really_inline base_u16() = default;
1694 is_utf8_really_inline base_u16(const uint16x8_t _value) : value(_value) {}
1695 is_utf8_really_inline operator const uint16x8_t &() const {
1696 return this->value;
1697 }
1698 is_utf8_really_inline operator uint16x8_t &() { return this->value; }
1699 // Bit operations
1700 is_utf8_really_inline simd16<T> operator|(const simd16<T> other) const {
1701 return vorrq_u16(*this, other);
1702 }
1703 is_utf8_really_inline simd16<T> operator&(const simd16<T> other) const {
1704 return vandq_u16(*this, other);
1705 }
1706 is_utf8_really_inline simd16<T> operator^(const simd16<T> other) const {
1707 return veorq_u16(*this, other);
1708 }
1709 is_utf8_really_inline simd16<T> bit_andnot(const simd16<T> other) const {
1710 return vbicq_u16(*this, other);
1711 }
1712 is_utf8_really_inline simd16<T> operator~() const { return *this ^ 0xFFu; }
1713 is_utf8_really_inline simd16<T> &operator|=(const simd16<T> other) {
1714 auto this_cast = static_cast<simd16<T> *>(this);
1715 *this_cast = *this_cast | other;
1716 return *this_cast;
1717 }
1718 is_utf8_really_inline simd16<T> &operator&=(const simd16<T> other) {
1719 auto this_cast = static_cast<simd16<T> *>(this);
1720 *this_cast = *this_cast & other;
1721 return *this_cast;
1722 }
1723 is_utf8_really_inline simd16<T> &operator^=(const simd16<T> other) {
1724 auto this_cast = static_cast<simd16<T> *>(this);
1725 *this_cast = *this_cast ^ other;
1726 return *this_cast;
1727 }
1728
1729 is_utf8_really_inline Mask operator==(const simd16<T> other) const {
1730 return vceqq_u16(*this, other);
1731 }
1732
1733 template <int N = 1>
1734 is_utf8_really_inline simd16<T> prev(const simd16<T> prev_chunk) const {
1735 return vextq_u18(prev_chunk, *this, 8 - N);
1736 }
1737 };
1738
1739 template <typename T, typename Mask = simd16<bool>>
1740 struct base16 : base_u16<T> {
1741 typedef uint16_t bitmask_t;
1742 typedef uint32_t bitmask2_t;
1743
1744 is_utf8_really_inline base16() : base_u16<T>() {}
1745 is_utf8_really_inline base16(const uint16x8_t _value) : base_u16<T>(_value) {}
1746 template <typename Pointer>
1747 is_utf8_really_inline base16(const Pointer *ptr) : base16(vld1q_u16(ptr)) {}
1748
1749 is_utf8_really_inline Mask operator==(const simd16<T> other) const {
1750 return vceqq_u16(*this, other);
1751 }
1752
1753 static const int SIZE = sizeof(base_u16<T>::value);
1754
1755 template <int N = 1>
1756 is_utf8_really_inline simd16<T> prev(const simd16<T> prev_chunk) const {
1757 return vextq_u18(prev_chunk, *this, 8 - N);
1758 }
1759 };
1760
1761 // SIMD byte mask type (returned by things like eq and gt)
1762 template <> struct simd16<bool> : base16<bool> {
1763 static is_utf8_really_inline simd16<bool> splat(bool _value) {
1764 return vmovq_n_u16(uint16_t(-(!!_value)));
1765 }
1766
1767 is_utf8_really_inline simd16<bool>() : base16() {}
1768 is_utf8_really_inline simd16<bool>(const uint16x8_t _value)
1769 : base16<bool>(_value) {}
1770 // Splat constructor
1771 is_utf8_really_inline simd16<bool>(bool _value)
1772 : base16<bool>(splat(_value)) {}
1773 };
1774
1775 template <typename T> struct base16_numeric : base16<T> {
1776 static is_utf8_really_inline simd16<T> splat(T _value) {
1777 return vmovq_n_u16(_value);
1778 }
1779 static is_utf8_really_inline simd16<T> zero() { return vdupq_n_u16(0); }
1780 static is_utf8_really_inline simd16<T> load(const T values[8]) {
1781 return vld1q_u16(reinterpret_cast<const uint16_t *>(values));
1782 }
1783
1784 is_utf8_really_inline base16_numeric() : base16<T>() {}
1785 is_utf8_really_inline base16_numeric(const uint16x8_t _value)
1786 : base16<T>(_value) {}
1787
1788 // Store to array
1789 is_utf8_really_inline void store(T dst[8]) const {
1790 return vst1q_u16(dst, *this);
1791 }
1792
1793 // Override to distinguish from bool version
1794 is_utf8_really_inline simd16<T> operator~() const { return *this ^ 0xFFu; }
1795
1796 // Addition/subtraction are the same for signed and unsigned
1797 is_utf8_really_inline simd16<T> operator+(const simd16<T> other) const {
1798 return vaddq_u8(*this, other);
1799 }
1800 is_utf8_really_inline simd16<T> operator-(const simd16<T> other) const {
1801 return vsubq_u8(*this, other);
1802 }
1803 is_utf8_really_inline simd16<T> &operator+=(const simd16<T> other) {
1804 *this = *this + other;
1805 return *static_cast<simd16<T> *>(this);
1806 }
1807 is_utf8_really_inline simd16<T> &operator-=(const simd16<T> other) {
1808 *this = *this - other;
1809 return *static_cast<simd16<T> *>(this);
1810 }
1811 };
1812
1813 // Signed words
1814 template <> struct simd16<int16_t> : base16_numeric<int16_t> {
1815 is_utf8_really_inline simd16() : base16_numeric<int16_t>() {}
1816 #ifndef IS_UTF8_REGULAR_VISUAL_STUDIO
1817 is_utf8_really_inline simd16(const uint16x8_t _value)
1818 : base16_numeric<int16_t>(_value) {}
1819 #endif
1820 is_utf8_really_inline simd16(const int16x8_t _value)
1821 : base16_numeric<int16_t>(vreinterpretq_u16_s16(_value)) {}
1822
1823 // Splat constructor
1824 is_utf8_really_inline simd16(int16_t _value) : simd16(splat(_value)) {}
1825 // Array constructor
1826 is_utf8_really_inline simd16(const int16_t *values) : simd16(load(values)) {}
1827 is_utf8_really_inline simd16(const char16_t *values)
1828 : simd16(load(reinterpret_cast<const int16_t *>(values))) {}
1829 is_utf8_really_inline operator simd16<uint16_t>() const;
1830 is_utf8_really_inline operator const uint16x8_t &() const {
1831 return this->value;
1832 }
1833 is_utf8_really_inline operator const int16x8_t() const {
1834 return vreinterpretq_s16_u16(this->value);
1835 }
1836
1837 is_utf8_really_inline int16_t max_val() const {
1838 return vmaxvq_s16(vreinterpretq_s16_u16(this->value));
1839 }
1840 is_utf8_really_inline int16_t min_val() const {
1841 return vminvq_s16(vreinterpretq_s16_u16(this->value));
1842 }
1843 // Order-sensitive comparisons
1844 is_utf8_really_inline simd16<int16_t>
1845 max_val(const simd16<int16_t> other) const {
1846 return vmaxq_s16(vreinterpretq_s16_u16(this->value),
1847 vreinterpretq_s16_u16(other.value));
1848 }
1849 is_utf8_really_inline simd16<int16_t>
1850 min_val(const simd16<int16_t> other) const {
1851 return vmaxq_s16(vreinterpretq_s16_u16(this->value),
1852 vreinterpretq_s16_u16(other.value));
1853 }
1854 is_utf8_really_inline simd16<bool>
1855 operator>(const simd16<int16_t> other) const {
1856 return vcgtq_s16(vreinterpretq_s16_u16(this->value),
1857 vreinterpretq_s16_u16(other.value));
1858 }
1859 is_utf8_really_inline simd16<bool>
1860 operator<(const simd16<int16_t> other) const {
1861 return vcltq_s16(vreinterpretq_s16_u16(this->value),
1862 vreinterpretq_s16_u16(other.value));
1863 }
1864 };
1865
1866 // Unsigned words
1867 template <> struct simd16<uint16_t> : base16_numeric<uint16_t> {
1868 is_utf8_really_inline simd16() : base16_numeric<uint16_t>() {}
1869 is_utf8_really_inline simd16(const uint16x8_t _value)
1870 : base16_numeric<uint16_t>(_value) {}
1871
1872 // Splat constructor
1873 is_utf8_really_inline simd16(uint16_t _value) : simd16(splat(_value)) {}
1874 // Array constructor
1875 is_utf8_really_inline simd16(const uint16_t *values) : simd16(load(values)) {}
1876 is_utf8_really_inline simd16(const char16_t *values)
1877 : simd16(load(reinterpret_cast<const uint16_t *>(values))) {}
1878
1879 is_utf8_really_inline int16_t max_val() const { return vmaxvq_u16(*this); }
1880 is_utf8_really_inline int16_t min_val() const { return vminvq_u16(*this); }
1881 // Saturated math
1882 is_utf8_really_inline simd16<uint16_t>
1883 saturating_add(const simd16<uint16_t> other) const {
1884 return vqaddq_u16(*this, other);
1885 }
1886 is_utf8_really_inline simd16<uint16_t>
1887 saturating_sub(const simd16<uint16_t> other) const {
1888 return vqsubq_u16(*this, other);
1889 }
1890
1891 // Order-specific operations
1892 is_utf8_really_inline simd16<uint16_t>
1893 max_val(const simd16<uint16_t> other) const {
1894 return vmaxq_u16(*this, other);
1895 }
1896 is_utf8_really_inline simd16<uint16_t>
1897 min_val(const simd16<uint16_t> other) const {
1898 return vminq_u16(*this, other);
1899 }
1900 // Same as >, but only guarantees true is nonzero (< guarantees true = -1)
1901 is_utf8_really_inline simd16<uint16_t>
1902 gt_bits(const simd16<uint16_t> other) const {
1903 return this->saturating_sub(other);
1904 }
1905 // Same as <, but only guarantees true is nonzero (< guarantees true = -1)
1906 is_utf8_really_inline simd16<uint16_t>
1907 lt_bits(const simd16<uint16_t> other) const {
1908 return other.saturating_sub(*this);
1909 }
1910 is_utf8_really_inline simd16<bool>
1911 operator<=(const simd16<uint16_t> other) const {
1912 return vcleq_u16(*this, other);
1913 }
1914 is_utf8_really_inline simd16<bool>
1915 operator>=(const simd16<uint16_t> other) const {
1916 return vcgeq_u16(*this, other);
1917 }
1918 is_utf8_really_inline simd16<bool>
1919 operator>(const simd16<uint16_t> other) const {
1920 return vcgtq_u16(*this, other);
1921 }
1922 is_utf8_really_inline simd16<bool>
1923 operator<(const simd16<uint16_t> other) const {
1924 return vcltq_u16(*this, other);
1925 }
1926
1927 // Bit-specific operations
1928 is_utf8_really_inline simd16<bool> bits_not_set() const {
1929 return *this == uint16_t(0);
1930 }
1931 template <int N> is_utf8_really_inline simd16<uint16_t> shr() const {
1932 return simd16<uint16_t>(vshrq_n_u16(*this, N));
1933 }
1934 template <int N> is_utf8_really_inline simd16<uint16_t> shl() const {
1935 return simd16<uint16_t>(vshlq_n_u16(*this, N));
1936 }
1937
1938 // logical operations
1939 is_utf8_really_inline simd16<uint16_t>
1940 operator|(const simd16<uint16_t> other) const {
1941 return vorrq_u16(*this, other);
1942 }
1943 is_utf8_really_inline simd16<uint16_t>
1944 operator&(const simd16<uint16_t> other) const {
1945 return vandq_u16(*this, other);
1946 }
1947 is_utf8_really_inline simd16<uint16_t>
1948 operator^(const simd16<uint16_t> other) const {
1949 return veorq_u16(*this, other);
1950 }
1951
1952 // Pack with the unsigned saturation two uint16_t words into single uint8_t
1953 // vector
1954 static is_utf8_really_inline simd8<uint8_t> pack(const simd16<uint16_t> &v0,
1955 const simd16<uint16_t> &v1) {
1956 return vqmovn_high_u16(vqmovn_u16(v0), v1);
1957 }
1958
1959 // Change the endianness
1960 is_utf8_really_inline simd16<uint16_t> swap_bytes() const {
1961 #ifdef IS_UTF8_REGULAR_VISUAL_STUDIO
1962 const uint8x16_t swap =
1963 make_uint8x16_t(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14);
1964 #else
1965 const uint8x16_t swap = {1, 0, 3, 2, 5, 4, 7, 6,
1966 9, 8, 11, 10, 13, 12, 15, 14};
1967 #endif
1968 return vreinterpretq_u16_u8(vqtbl1q_u8(vreinterpretq_u8_u16(*this), swap));
1969 }
1970 };
1971 is_utf8_really_inline simd16<int16_t>::operator simd16<uint16_t>() const {
1972 return this->value;
1973 }
1974
1975 template <typename T> struct simd16x32 {
1976 static constexpr int NUM_CHUNKS = 64 / sizeof(simd16<T>);
1977 static_assert(NUM_CHUNKS == 4,
1978 "ARM kernel should use four registers per 64-byte block.");
1979 simd16<T> chunks[NUM_CHUNKS];
1980
1981 simd16x32(const simd16x32<T> &o) = delete; // no copy allowed
1982 simd16x32<T> &
1983 operator=(const simd16<T> other) = delete; // no assignment allowed
1984 simd16x32() = delete; // no default constructor allowed
1985
1986 is_utf8_really_inline
1987 simd16x32(const simd16<T> chunk0, const simd16<T> chunk1,
1988 const simd16<T> chunk2, const simd16<T> chunk3)
1989 : chunks{chunk0, chunk1, chunk2, chunk3} {}
1990 is_utf8_really_inline simd16x32(const T *ptr)
1991 : chunks{simd16<T>::load(ptr),
1992 simd16<T>::load(ptr + sizeof(simd16<T>) / sizeof(T)),
1993 simd16<T>::load(ptr + 2 * sizeof(simd16<T>) / sizeof(T)),
1994 simd16<T>::load(ptr + 3 * sizeof(simd16<T>) / sizeof(T))} {}
1995
1996 is_utf8_really_inline void store(T *ptr) const {
1997 this->chunks[0].store(ptr + sizeof(simd16<T>) * 0 / sizeof(T));
1998 this->chunks[1].store(ptr + sizeof(simd16<T>) * 1 / sizeof(T));
1999 this->chunks[2].store(ptr + sizeof(simd16<T>) * 2 / sizeof(T));
2000 this->chunks[3].store(ptr + sizeof(simd16<T>) * 3 / sizeof(T));
2001 }
2002
2003 is_utf8_really_inline simd16<T> reduce_or() const {
2004 return (this->chunks[0] | this->chunks[1]) |
2005 (this->chunks[2] | this->chunks[3]);
2006 }
2007
2008 is_utf8_really_inline bool is_ascii() const { return reduce_or().is_ascii(); }
2009
2010 is_utf8_really_inline void store_ascii_as_utf16(char16_t *ptr) const {
2011 this->chunks[0].store_ascii_as_utf16(ptr + sizeof(simd16<T>) * 0);
2012 this->chunks[1].store_ascii_as_utf16(ptr + sizeof(simd16<T>) * 1);
2013 this->chunks[2].store_ascii_as_utf16(ptr + sizeof(simd16<T>) * 2);
2014 this->chunks[3].store_ascii_as_utf16(ptr + sizeof(simd16<T>) * 3);
2015 }
2016
2017 is_utf8_really_inline uint64_t to_bitmask() const {
2018 #ifdef IS_UTF8_REGULAR_VISUAL_STUDIO
2019 const uint8x16_t bit_mask =
2020 make_uint8x16_t(0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80, 0x01,
2021 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80);
2022 #else
2023 const uint8x16_t bit_mask = {0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80,
2024 0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80};
2025 #endif
2026 // Add each of the elements next to each other, successively, to stuff each
2027 // 8 byte mask into one.
2028 uint8x16_t sum0 = vpaddq_u8(
2029 vreinterpretq_u8_u16(this->chunks[0] & vreinterpretq_u16_u8(bit_mask)),
2030 vreinterpretq_u8_u16(this->chunks[1] & vreinterpretq_u16_u8(bit_mask)));
2031 uint8x16_t sum1 = vpaddq_u8(
2032 vreinterpretq_u8_u16(this->chunks[2] & vreinterpretq_u16_u8(bit_mask)),
2033 vreinterpretq_u8_u16(this->chunks[3] & vreinterpretq_u16_u8(bit_mask)));
2034 sum0 = vpaddq_u8(sum0, sum1);
2035 sum0 = vpaddq_u8(sum0, sum0);
2036 return vgetq_lane_u64(vreinterpretq_u64_u8(sum0), 0);
2037 }
2038
2039 is_utf8_really_inline void swap_bytes() {
2040 this->chunks[0] = this->chunks[0].swap_bytes();
2041 this->chunks[1] = this->chunks[1].swap_bytes();
2042 this->chunks[2] = this->chunks[2].swap_bytes();
2043 this->chunks[3] = this->chunks[3].swap_bytes();
2044 }
2045
2046 is_utf8_really_inline uint64_t eq(const T m) const {
2047 const simd16<T> mask = simd16<T>::splat(m);
2048 return simd16x32<bool>(this->chunks[0] == mask, this->chunks[1] == mask,
2049 this->chunks[2] == mask, this->chunks[3] == mask)
2050 .to_bitmask();
2051 }
2052
2053 is_utf8_really_inline uint64_t lteq(const T m) const {
2054 const simd16<T> mask = simd16<T>::splat(m);
2055 return simd16x32<bool>(this->chunks[0] <= mask, this->chunks[1] <= mask,
2056 this->chunks[2] <= mask, this->chunks[3] <= mask)
2057 .to_bitmask();
2058 }
2059
2060 is_utf8_really_inline uint64_t in_range(const T low, const T high) const {
2061 const simd16<T> mask_low = simd16<T>::splat(low);
2062 const simd16<T> mask_high = simd16<T>::splat(high);
2063
2064 return simd16x32<bool>(
2065 (this->chunks[0] <= mask_high) & (this->chunks[0] >= mask_low),
2066 (this->chunks[1] <= mask_high) & (this->chunks[1] >= mask_low),
2067 (this->chunks[2] <= mask_high) & (this->chunks[2] >= mask_low),
2068 (this->chunks[3] <= mask_high) & (this->chunks[3] >= mask_low))
2069 .to_bitmask();
2070 }
2071 is_utf8_really_inline uint64_t not_in_range(const T low, const T high) const {
2072 const simd16<T> mask_low = simd16<T>::splat(low);
2073 const simd16<T> mask_high = simd16<T>::splat(high);
2074 return simd16x32<bool>(
2075 (this->chunks[0] > mask_high) | (this->chunks[0] < mask_low),
2076 (this->chunks[1] > mask_high) | (this->chunks[1] < mask_low),
2077 (this->chunks[2] > mask_high) | (this->chunks[2] < mask_low),
2078 (this->chunks[3] > mask_high) | (this->chunks[3] < mask_low))
2079 .to_bitmask();
2080 }
2081 is_utf8_really_inline uint64_t lt(const T m) const {
2082 const simd16<T> mask = simd16<T>::splat(m);
2083 return simd16x32<bool>(this->chunks[0] < mask, this->chunks[1] < mask,
2084 this->chunks[2] < mask, this->chunks[3] < mask)
2085 .to_bitmask();
2086 }
2087
2088 }; // struct simd16x32<T>
2089 template <>
2090 is_utf8_really_inline uint64_t simd16x32<uint16_t>::not_in_range(
2091 const uint16_t low, const uint16_t high) const {
2092 const simd16<uint16_t> mask_low = simd16<uint16_t>::splat(low);
2093 const simd16<uint16_t> mask_high = simd16<uint16_t>::splat(high);
2094 simd16x32<uint16_t> x(simd16<uint16_t>((this->chunks[0] > mask_high) |
2095 (this->chunks[0] < mask_low)),
2096 simd16<uint16_t>((this->chunks[1] > mask_high) |
2097 (this->chunks[1] < mask_low)),
2098 simd16<uint16_t>((this->chunks[2] > mask_high) |
2099 (this->chunks[2] < mask_low)),
2100 simd16<uint16_t>((this->chunks[3] > mask_high) |
2101 (this->chunks[3] < mask_low)));
2102 return x.to_bitmask();
2103 }
2104
2105 } // namespace simd
2106 } // unnamed namespace
2107 } // namespace arm64
2108 } // namespace is_utf8_internals
2109
2110 #endif // IS_UTF8_ARM64_SIMD_H
2111
2112 #endif // IS_UTF8_IMPLEMENTATION_ARM64
2113
2114 #endif // IS_UTF8_ARM64_H
2115
2116 #ifndef IS_UTF8_ICELAKE_H
2117 #define IS_UTF8_ICELAKE_H
2118
2119 #ifdef __has_include
2120 // How do we detect that a compiler supports vbmi2?
2121 // For sure if the following header is found, we are ok?
2122 #if __has_include(<avx512vbmi2intrin.h>)
2123 #define IS_UTF8_COMPILER_SUPPORTS_VBMI2 1
2124 #endif
2125 #endif
2126
2127 #ifdef _MSC_VER
2128 #if _MSC_VER >= 1920
2129 // Visual Studio 2019 and up support VBMI2 under x64 even if the header
2130 // avx512vbmi2intrin.h is not found.
2131 #define IS_UTF8_COMPILER_SUPPORTS_VBMI2 1
2132 #endif
2133 #endif
2134
2135 // We allow icelake on x64 as long as the compiler is known to support VBMI2.
2136 #ifndef IS_UTF8_IMPLEMENTATION_ICELAKE
2137 #define IS_UTF8_IMPLEMENTATION_ICELAKE \
2138 ((IS_UTF8_IS_X86_64) && (IS_UTF8_COMPILER_SUPPORTS_VBMI2))
2139 #endif
2140
2141 // To see why (__BMI__) && (__PCLMUL__) && (__LZCNT__) are not part of this
2142 // next line, see https://github.com/simdutf/simdutf/issues/1247
2143 #define IS_UTF8_CAN_ALWAYS_RUN_ICELAKE \
2144 ((IS_UTF8_IMPLEMENTATION_ICELAKE) && (IS_UTF8_IS_X86_64) && (__AVX2__) && \
2145 (IS_UTF8_HAS_AVX512F && IS_UTF8_HAS_AVX512DQ && IS_UTF8_HAS_AVX512VL && \
2146 IS_UTF8_HAS_AVX512VBMI2) && \
2147 (!IS_UTF8_IS_32BITS))
2148
2149 #if IS_UTF8_IMPLEMENTATION_ICELAKE
2150 #if IS_UTF8_CAN_ALWAYS_RUN_ICELAKE
2151 #define IS_UTF8_TARGET_ICELAKE
2152 #define IS_UTF8_UNTARGET_ICELAKE
2153 #else
2154 #define IS_UTF8_TARGET_ICELAKE \
2155 IS_UTF8_TARGET_REGION("avx512f,avx512dq,avx512cd,avx512bw,avx512vbmi," \
2156 "avx512vbmi2,avx512vl,avx2,bmi,bmi2,pclmul,lzcnt")
2157 #define IS_UTF8_UNTARGET_ICELAKE IS_UTF8_UNTARGET_REGION
2158 #endif
2159
2160 namespace is_utf8_internals {
2161 namespace icelake {} // namespace icelake
2162 } // namespace is_utf8_internals
2163
2164 //
2165 // These two need to be included outside IS_UTF8_TARGET_REGION
2166 //
2167
2168 #ifndef IS_UTF8_ICELAKE_INTRINSICS_H
2169 #define IS_UTF8_ICELAKE_INTRINSICS_H
2170
2171 #ifdef IS_UTF8_VISUAL_STUDIO
2172 // under clang within visual studio, this will include <x86intrin.h>
2173 #include <immintrin.h>
2174 #include <intrin.h> // visual studio or clang
2175 #else
2176
2177 #if IS_UTF8_GCC11ORMORE
2178 // We should not get warnings while including <x86intrin.h> yet we do
2179 // under some versions of GCC.
2180 // If the x86intrin.h header has uninitialized values that are problematic,
2181 // it is a GCC issue, we want to ignore these warnings.
2182 IS_UTF8_DISABLE_GCC_WARNING(-Wuninitialized)
2183 #endif
2184
2185 #include <x86intrin.h> // elsewhere
2186
2187 #if IS_UTF8_GCC11ORMORE
2188 // cancels the suppression of the -Wuninitialized
2189 IS_UTF8_POP_DISABLE_WARNINGS
2190 #endif
2191
2192 #ifndef _tzcnt_u64
2193 #define _tzcnt_u64(x) __tzcnt_u64(x)
2194 #endif // _tzcnt_u64
2195 #endif // IS_UTF8_VISUAL_STUDIO
2196
2197 #ifdef IS_UTF8_CLANG_VISUAL_STUDIO
2198
2199 #include <avx2intrin.h>
2200 #include <avxintrin.h>
2201 #include <bmi2intrin.h> // for _pext_u64, _pdep_u64
2202 #include <bmiintrin.h> // for _blsr_u64
2203 #include <immintrin.h> // for most things (AVX2, AVX512, _popcnt64)
2204 #include <lzcntintrin.h> // for __lzcnt64
2205 #include <smmintrin.h>
2206 #include <tmmintrin.h>
2207 #include <wmmintrin.h> // for _mm_clmulepi64_si128
2208 // Important: we need the AVX-512 headers:
2209 #include <avx512bwintrin.h>
2210 #include <avx512cdintrin.h>
2211 #include <avx512dqintrin.h>
2212 #include <avx512fintrin.h>
2213 #include <avx512vbmi2intrin.h>
2214 #include <avx512vbmiintrin.h>
2215 #include <avx512vlbwintrin.h>
2216 #include <avx512vlintrin.h>
2217 // unfortunately, we may not get _blsr_u64, but, thankfully, clang
2218 // has it as a macro.
2219 #ifndef _blsr_u64
2220 // we roll our own
2221 #define _blsr_u64(n) ((n - 1) & n)
2222 #endif // _blsr_u64
2223 #endif // IS_UTF8_CLANG_VISUAL_STUDIO
2224
2225 #if defined(__GNUC__) && !defined(__clang__)
2226
2227 #if __GNUC__ == 8
2228 #define IS_UTF8_GCC8 1
2229 #elif __GNUC__ == 9
2230 #define IS_UTF8_GCC9 1
2231 #endif // __GNUC__ == 8 || __GNUC__ == 9
2232
2233 #endif // defined(__GNUC__) && !defined(__clang__)
2234
2235 #if IS_UTF8_GCC8
2236 #pragma GCC push_options
2237 #pragma GCC target("avx512f")
2238 /**
2239 * GCC 8 fails to provide _mm512_set_epi8. We roll our own.
2240 */
2241 inline __m512i
2242 _mm512_set_epi8(uint8_t a0, uint8_t a1, uint8_t a2, uint8_t a3, uint8_t a4,
2243 uint8_t a5, uint8_t a6, uint8_t a7, uint8_t a8, uint8_t a9,
2244 uint8_t a10, uint8_t a11, uint8_t a12, uint8_t a13, uint8_t a14,
2245 uint8_t a15, uint8_t a16, uint8_t a17, uint8_t a18, uint8_t a19,
2246 uint8_t a20, uint8_t a21, uint8_t a22, uint8_t a23, uint8_t a24,
2247 uint8_t a25, uint8_t a26, uint8_t a27, uint8_t a28, uint8_t a29,
2248 uint8_t a30, uint8_t a31, uint8_t a32, uint8_t a33, uint8_t a34,
2249 uint8_t a35, uint8_t a36, uint8_t a37, uint8_t a38, uint8_t a39,
2250 uint8_t a40, uint8_t a41, uint8_t a42, uint8_t a43, uint8_t a44,
2251 uint8_t a45, uint8_t a46, uint8_t a47, uint8_t a48, uint8_t a49,
2252 uint8_t a50, uint8_t a51, uint8_t a52, uint8_t a53, uint8_t a54,
2253 uint8_t a55, uint8_t a56, uint8_t a57, uint8_t a58, uint8_t a59,
2254 uint8_t a60, uint8_t a61, uint8_t a62, uint8_t a63) {
2255 return _mm512_set_epi64(
2256 uint64_t(a7) + (uint64_t(a6) << 8) + (uint64_t(a5) << 16) +
2257 (uint64_t(a4) << 24) + (uint64_t(a3) << 32) + (uint64_t(a2) << 40) +
2258 (uint64_t(a1) << 48) + (uint64_t(a0) << 56),
2259 uint64_t(a15) + (uint64_t(a14) << 8) + (uint64_t(a13) << 16) +
2260 (uint64_t(a12) << 24) + (uint64_t(a11) << 32) +
2261 (uint64_t(a10) << 40) + (uint64_t(a9) << 48) + (uint64_t(a8) << 56),
2262 uint64_t(a23) + (uint64_t(a22) << 8) + (uint64_t(a21) << 16) +
2263 (uint64_t(a20) << 24) + (uint64_t(a19) << 32) +
2264 (uint64_t(a18) << 40) + (uint64_t(a17) << 48) + (uint64_t(a16) << 56),
2265 uint64_t(a31) + (uint64_t(a30) << 8) + (uint64_t(a29) << 16) +
2266 (uint64_t(a28) << 24) + (uint64_t(a27) << 32) +
2267 (uint64_t(a26) << 40) + (uint64_t(a25) << 48) + (uint64_t(a24) << 56),
2268 uint64_t(a39) + (uint64_t(a38) << 8) + (uint64_t(a37) << 16) +
2269 (uint64_t(a36) << 24) + (uint64_t(a35) << 32) +
2270 (uint64_t(a34) << 40) + (uint64_t(a33) << 48) + (uint64_t(a32) << 56),
2271 uint64_t(a47) + (uint64_t(a46) << 8) + (uint64_t(a45) << 16) +
2272 (uint64_t(a44) << 24) + (uint64_t(a43) << 32) +
2273 (uint64_t(a42) << 40) + (uint64_t(a41) << 48) + (uint64_t(a40) << 56),
2274 uint64_t(a55) + (uint64_t(a54) << 8) + (uint64_t(a53) << 16) +
2275 (uint64_t(a52) << 24) + (uint64_t(a51) << 32) +
2276 (uint64_t(a50) << 40) + (uint64_t(a49) << 48) + (uint64_t(a48) << 56),
2277 uint64_t(a63) + (uint64_t(a62) << 8) + (uint64_t(a61) << 16) +
2278 (uint64_t(a60) << 24) + (uint64_t(a59) << 32) +
2279 (uint64_t(a58) << 40) + (uint64_t(a57) << 48) +
2280 (uint64_t(a56) << 56));
2281 }
2282 #pragma GCC pop_options
2283 #endif // IS_UTF8_GCC8
2284
2285 #endif // IS_UTF8_HASWELL_INTRINSICS_H
2286
2287 #ifndef IS_UTF8_ICELAKE_IMPLEMENTATION_H
2288 #define IS_UTF8_ICELAKE_IMPLEMENTATION_H
2289
2290 namespace is_utf8_internals {
2291 namespace icelake {
2292
2293 class implementation final : public is_utf8_internals::implementation {
2294 public:
2295 is_utf8_really_inline implementation()
2296 : is_utf8_internals::implementation(
2297 "icelake",
2298 "Intel AVX512 (AVX-512BW, AVX-512CD, AVX-512VL, AVX-512VBMI2 "
2299 "extensions)",
2300 internal::instruction_set::AVX2 |
2301 internal::instruction_set::PCLMULQDQ |
2302 internal::instruction_set::BMI1 |
2303 internal::instruction_set::BMI2 |
2304 internal::instruction_set::AVX512BW |
2305 internal::instruction_set::AVX512CD |
2306 internal::instruction_set::AVX512VL |
2307 internal::instruction_set::AVX512VBMI2) {}
2308 is_utf8_warn_unused bool validate_utf8(const char *buf,
2309 size_t len) const noexcept final;
2310 };
2311
2312 } // namespace icelake
2313 } // namespace is_utf8_internals
2314
2315 #endif // IS_UTF8_ICELAKE_IMPLEMENTATION_H
2316
2317 #if IS_UTF8_GCC11ORMORE // workaround for
2318 // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=105593
2319 IS_UTF8_POP_DISABLE_WARNINGS
2320 #endif // end of workaround
2321
2322 #endif // IS_UTF8_IMPLEMENTATION_ICELAKE
2323 #endif // IS_UTF8_ICELAKE_H
2324
2325 #ifndef IS_UTF8_HASWELL_H
2326 #define IS_UTF8_HASWELL_H
2327
2328 #ifdef IS_UTF8_WESTMERE_H
2329 #error "haswell.h must be included before westmere.h"
2330 #endif
2331 #ifdef IS_UTF8_FALLBACK_H
2332 #error "haswell.h must be included before fallback.h"
2333 #endif
2334
2335 // Default Haswell to on if this is x86-64. Even if we're not compiled for it,
2336 // it could be selected at runtime.
2337 #ifndef IS_UTF8_IMPLEMENTATION_HASWELL
2338 //
2339 // You do not want to restrict it like so: IS_UTF8_IS_X86_64 && __AVX2__
2340 // because we want to rely on *runtime dispatch*.
2341 //
2342 #if IS_UTF8_CAN_ALWAYS_RUN_ICELAKE
2343 #define IS_UTF8_IMPLEMENTATION_HASWELL 0
2344 #else
2345 #define IS_UTF8_IMPLEMENTATION_HASWELL (IS_UTF8_IS_X86_64)
2346 #endif
2347
2348 #endif
2349 // To see why (__BMI__) && (__PCLMUL__) && (__LZCNT__) are not part of this
2350 // next line, see https://github.com/simdutf/simdutf/issues/1247
2351 #if ((IS_UTF8_IMPLEMENTATION_HASWELL) && (IS_UTF8_IS_X86_64) && (__AVX2__))
2352 #define IS_UTF8_CAN_ALWAYS_RUN_HASWELL 1
2353 #else
2354 #define IS_UTF8_CAN_ALWAYS_RUN_HASWELL 0
2355 #endif
2356
2357 #if IS_UTF8_IMPLEMENTATION_HASWELL
2358
2359 #define IS_UTF8_TARGET_HASWELL IS_UTF8_TARGET_REGION("avx2,bmi,pclmul,lzcnt")
2360
2361 namespace is_utf8_internals {
2362 /**
2363 * Implementation for Haswell (Intel AVX2).
2364 */
2365 namespace haswell {} // namespace haswell
2366 } // namespace is_utf8_internals
2367
2368 //
2369 // These two need to be included outside IS_UTF8_TARGET_REGION
2370 //
2371
2372 #ifndef IS_UTF8_HASWELL_IMPLEMENTATION_H
2373 #define IS_UTF8_HASWELL_IMPLEMENTATION_H
2374
2375 // The constructor may be executed on any host, so we take care not to use
2376 // IS_UTF8_TARGET_REGION
2377 namespace is_utf8_internals {
2378 namespace haswell {
2379
2380 class implementation final : public is_utf8_internals::implementation {
2381 public:
2382 is_utf8_really_inline implementation()
2383 : is_utf8_internals::implementation(
2384 "haswell", "Intel/AMD AVX2",
2385 internal::instruction_set::AVX2 |
2386 internal::instruction_set::PCLMULQDQ |
2387 internal::instruction_set::BMI1 |
2388 internal::instruction_set::BMI2) {}
2389 is_utf8_warn_unused bool validate_utf8(const char *buf,
2390 size_t len) const noexcept final;
2391 };
2392
2393 } // namespace haswell
2394 } // namespace is_utf8_internals
2395
2396 #endif // IS_UTF8_HASWELL_IMPLEMENTATION_H
2397
2398 #ifndef IS_UTF8_HASWELL_INTRINSICS_H
2399 #define IS_UTF8_HASWELL_INTRINSICS_H
2400
2401 #ifdef IS_UTF8_VISUAL_STUDIO
2402 // under clang within visual studio, this will include <x86intrin.h>
2403 #include <intrin.h> // visual studio or clang
2404 #else
2405
2406 #if IS_UTF8_GCC11ORMORE
2407 // We should not get warnings while including <x86intrin.h> yet we do
2408 // under some versions of GCC.
2409 // If the x86intrin.h header has uninitialized values that are problematic,
2410 // it is a GCC issue, we want to ignore these warnings.
2411 IS_UTF8_DISABLE_GCC_WARNING(-Wuninitialized)
2412 #endif
2413
2414 #include <x86intrin.h> // elsewhere
2415
2416 #if IS_UTF8_GCC11ORMORE
2417 // cancels the suppression of the -Wuninitialized
2418 IS_UTF8_POP_DISABLE_WARNINGS
2419 #endif
2420
2421 #endif // IS_UTF8_VISUAL_STUDIO
2422
2423 #ifdef IS_UTF8_CLANG_VISUAL_STUDIO
2424
2425 #include <avx2intrin.h>
2426 #include <avxintrin.h>
2427 #include <bmiintrin.h> // for _blsr_u64
2428 #include <immintrin.h> // for most things (AVX2, AVX512, _popcnt64)
2429 #include <lzcntintrin.h> // for __lzcnt64
2430 #include <smmintrin.h>
2431 #include <tmmintrin.h>
2432 #include <wmmintrin.h> // for _mm_clmulepi64_si128
2433 // unfortunately, we may not get _blsr_u64, but, thankfully, clang
2434 // has it as a macro.
2435 #ifndef _blsr_u64
2436 // we roll our own
2437 #define _blsr_u64(n) ((n - 1) & n)
2438 #endif // _blsr_u64
2439 #endif // IS_UTF8_CLANG_VISUAL_STUDIO
2440
2441 #endif // IS_UTF8_HASWELL_INTRINSICS_H
2442
2443 //
2444 // The rest need to be inside the region
2445 //
2446
2447 // redefining IS_UTF8_IMPLEMENTATION to "haswell"
2448 // #define IS_UTF8_IMPLEMENTATION haswell
2449 IS_UTF8_TARGET_HASWELL
2450
2451 #if IS_UTF8_GCC11ORMORE // workaround for
2452 // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=105593
2453 IS_UTF8_DISABLE_GCC_WARNING(-Wmaybe-uninitialized)
2454 #endif // end of workaround
2455
2456 // Declarations
2457
2458 #ifndef IS_UTF8_HASWELL_SIMD_H
2459 #define IS_UTF8_HASWELL_SIMD_H
2460
2461 namespace is_utf8_internals {
2462 namespace haswell {
2463 namespace {
2464 namespace simd {
2465
2466 // Forward-declared so they can be used by splat and friends.
2467 template <typename Child> struct base {
2468 __m256i value;
2469
2470 // Zero constructor
2471 is_utf8_really_inline base() : value{__m256i()} {}
2472
2473 // Conversion from SIMD register
2474 is_utf8_really_inline base(const __m256i _value) : value(_value) {}
2475 // Conversion to SIMD register
2476 is_utf8_really_inline operator const __m256i &() const { return this->value; }
2477 is_utf8_really_inline operator __m256i &() { return this->value; }
2478 template <endianness big_endian>
2479 is_utf8_really_inline void store_ascii_as_utf16(char16_t *ptr) const {
2480 __m256i first = _mm256_cvtepu8_epi16(_mm256_castsi256_si128(*this));
2481 __m256i second = _mm256_cvtepu8_epi16(_mm256_extractf128_si256(*this, 1));
2482 if (big_endian) {
2483 const __m256i swap = _mm256_setr_epi8(
2484 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14, 17, 16, 19, 18,
2485 21, 20, 23, 22, 25, 24, 27, 26, 29, 28, 31, 30);
2486 first = _mm256_shuffle_epi8(first, swap);
2487 second = _mm256_shuffle_epi8(second, swap);
2488 }
2489 _mm256_storeu_si256(reinterpret_cast<__m256i *>(ptr), first);
2490 _mm256_storeu_si256(reinterpret_cast<__m256i *>(ptr + 16), second);
2491 }
2492 is_utf8_really_inline void store_ascii_as_utf32(char32_t *ptr) const {
2493 _mm256_storeu_si256(reinterpret_cast<__m256i *>(ptr),
2494 _mm256_cvtepu8_epi32(_mm256_castsi256_si128(*this)));
2495 _mm256_storeu_si256(reinterpret_cast<__m256i *>(ptr + 8),
2496 _mm256_cvtepu8_epi32(_mm256_castsi256_si128(
2497 _mm256_srli_si256(*this, 8))));
2498 _mm256_storeu_si256(
2499 reinterpret_cast<__m256i *>(ptr + 16),
2500 _mm256_cvtepu8_epi32(_mm256_extractf128_si256(*this, 1)));
2501 _mm256_storeu_si256(reinterpret_cast<__m256i *>(ptr + 24),
2502 _mm256_cvtepu8_epi32(_mm_srli_si128(
2503 _mm256_extractf128_si256(*this, 1), 8)));
2504 }
2505 // Bit operations
2506 is_utf8_really_inline Child operator|(const Child other) const {
2507 return _mm256_or_si256(*this, other);
2508 }
2509 is_utf8_really_inline Child operator&(const Child other) const {
2510 return _mm256_and_si256(*this, other);
2511 }
2512 is_utf8_really_inline Child operator^(const Child other) const {
2513 return _mm256_xor_si256(*this, other);
2514 }
2515 is_utf8_really_inline Child bit_andnot(const Child other) const {
2516 return _mm256_andnot_si256(other, *this);
2517 }
2518 is_utf8_really_inline Child &operator|=(const Child other) {
2519 auto this_cast = static_cast<Child *>(this);
2520 *this_cast = *this_cast | other;
2521 return *this_cast;
2522 }
2523 is_utf8_really_inline Child &operator&=(const Child other) {
2524 auto this_cast = static_cast<Child *>(this);
2525 *this_cast = *this_cast & other;
2526 return *this_cast;
2527 }
2528 is_utf8_really_inline Child &operator^=(const Child other) {
2529 auto this_cast = static_cast<Child *>(this);
2530 *this_cast = *this_cast ^ other;
2531 return *this_cast;
2532 }
2533 };
2534
2535 // Forward-declared so they can be used by splat and friends.
2536 template <typename T> struct simd8;
2537
2538 template <typename T, typename Mask = simd8<bool>>
2539 struct base8 : base<simd8<T>> {
2540 typedef uint32_t bitmask_t;
2541 typedef uint64_t bitmask2_t;
2542
2543 is_utf8_really_inline base8() : base<simd8<T>>() {}
2544 is_utf8_really_inline base8(const __m256i _value) : base<simd8<T>>(_value) {}
2545 is_utf8_really_inline T first() const {
2546 return _mm256_extract_epi8(*this, 0);
2547 }
2548 is_utf8_really_inline T last() const {
2549 return _mm256_extract_epi8(*this, 31);
2550 }
2551 friend is_utf8_really_inline Mask operator==(const simd8<T> lhs,
2552 const simd8<T> rhs) {
2553 return _mm256_cmpeq_epi8(lhs, rhs);
2554 }
2555
2556 static const int SIZE = sizeof(base<T>::value);
2557
2558 template <int N = 1>
2559 is_utf8_really_inline simd8<T> prev(const simd8<T> prev_chunk) const {
2560 return _mm256_alignr_epi8(
2561 *this, _mm256_permute2x128_si256(prev_chunk, *this, 0x21), 16 - N);
2562 }
2563 };
2564
2565 // SIMD byte mask type (returned by things like eq and gt)
2566 template <> struct simd8<bool> : base8<bool> {
2567 static is_utf8_really_inline simd8<bool> splat(bool _value) {
2568 return _mm256_set1_epi8(uint8_t(-(!!_value)));
2569 }
2570
2571 is_utf8_really_inline simd8() : base8() {}
2572 is_utf8_really_inline simd8(const __m256i _value)
2573 : base8<bool>(_value) {}
2574 // Splat constructor
2575 is_utf8_really_inline simd8(bool _value) : base8<bool>(splat(_value)) {}
2576
2577 is_utf8_really_inline uint32_t to_bitmask() const {
2578 return uint32_t(_mm256_movemask_epi8(*this));
2579 }
2580 is_utf8_really_inline bool any() const {
2581 return !_mm256_testz_si256(*this, *this);
2582 }
2583 is_utf8_really_inline bool none() const {
2584 return _mm256_testz_si256(*this, *this);
2585 }
2586 is_utf8_really_inline bool all() const {
2587 return static_cast<uint32_t>(_mm256_movemask_epi8(*this)) == 0xFFFFFFFF;
2588 }
2589 is_utf8_really_inline simd8<bool> operator~() const { return *this ^ true; }
2590 };
2591
2592 template <typename T> struct base8_numeric : base8<T> {
2593 static is_utf8_really_inline simd8<T> splat(T _value) {
2594 return _mm256_set1_epi8(_value);
2595 }
2596 static is_utf8_really_inline simd8<T> zero() {
2597 return _mm256_setzero_si256();
2598 }
2599 static is_utf8_really_inline simd8<T> load(const T values[32]) {
2600 return _mm256_loadu_si256(reinterpret_cast<const __m256i *>(values));
2601 }
2602 // Repeat 16 values as many times as necessary (usually for lookup tables)
2603 static is_utf8_really_inline simd8<T> repeat_16(T v0, T v1, T v2, T v3, T v4,
2604 T v5, T v6, T v7, T v8, T v9,
2605 T v10, T v11, T v12, T v13,
2606 T v14, T v15) {
2607 return simd8<T>(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13,
2608 v14, v15, v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11,
2609 v12, v13, v14, v15);
2610 }
2611
2612 is_utf8_really_inline base8_numeric() : base8<T>() {}
2613 is_utf8_really_inline base8_numeric(const __m256i _value)
2614 : base8<T>(_value) {}
2615
2616 // Store to array
2617 is_utf8_really_inline void store(T dst[32]) const {
2618 return _mm256_storeu_si256(reinterpret_cast<__m256i *>(dst), *this);
2619 }
2620
2621 // Addition/subtraction are the same for signed and unsigned
2622 is_utf8_really_inline simd8<T> operator+(const simd8<T> other) const {
2623 return _mm256_add_epi8(*this, other);
2624 }
2625 is_utf8_really_inline simd8<T> operator-(const simd8<T> other) const {
2626 return _mm256_sub_epi8(*this, other);
2627 }
2628 is_utf8_really_inline simd8<T> &operator+=(const simd8<T> other) {
2629 *this = *this + other;
2630 return *static_cast<simd8<T> *>(this);
2631 }
2632 is_utf8_really_inline simd8<T> &operator-=(const simd8<T> other) {
2633 *this = *this - other;
2634 return *static_cast<simd8<T> *>(this);
2635 }
2636
2637 // Override to distinguish from bool version
2638 is_utf8_really_inline simd8<T> operator~() const { return *this ^ 0xFFu; }
2639
2640 // Perform a lookup assuming the value is between 0 and 16 (undefined behavior
2641 // for out of range values)
2642 template <typename L>
2643 is_utf8_really_inline simd8<L> lookup_16(simd8<L> lookup_table) const {
2644 return _mm256_shuffle_epi8(lookup_table, *this);
2645 }
2646
2647 template <typename L>
2648 is_utf8_really_inline simd8<L>
2649 lookup_16(L replace0, L replace1, L replace2, L replace3, L replace4,
2650 L replace5, L replace6, L replace7, L replace8, L replace9,
2651 L replace10, L replace11, L replace12, L replace13, L replace14,
2652 L replace15) const {
2653 return lookup_16(simd8<L>::repeat_16(
2654 replace0, replace1, replace2, replace3, replace4, replace5, replace6,
2655 replace7, replace8, replace9, replace10, replace11, replace12,
2656 replace13, replace14, replace15));
2657 }
2658 };
2659
2660 // Signed bytes
2661 template <> struct simd8<int8_t> : base8_numeric<int8_t> {
2662 is_utf8_really_inline simd8() : base8_numeric<int8_t>() {}
2663 is_utf8_really_inline simd8(const __m256i _value)
2664 : base8_numeric<int8_t>(_value) {}
2665
2666 // Splat constructor
2667 is_utf8_really_inline simd8(int8_t _value) : simd8(splat(_value)) {}
2668 // Array constructor
2669 is_utf8_really_inline simd8(const int8_t values[32]) : simd8(load(values)) {}
2670 is_utf8_really_inline operator simd8<uint8_t>() const;
2671 // Member-by-member initialization
2672 is_utf8_really_inline
2673 simd8(int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5,
2674 int8_t v6, int8_t v7, int8_t v8, int8_t v9, int8_t v10, int8_t v11,
2675 int8_t v12, int8_t v13, int8_t v14, int8_t v15, int8_t v16, int8_t v17,
2676 int8_t v18, int8_t v19, int8_t v20, int8_t v21, int8_t v22, int8_t v23,
2677 int8_t v24, int8_t v25, int8_t v26, int8_t v27, int8_t v28, int8_t v29,
2678 int8_t v30, int8_t v31)
2679 : simd8(_mm256_setr_epi8(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11,
2680 v12, v13, v14, v15, v16, v17, v18, v19, v20, v21,
2681 v22, v23, v24, v25, v26, v27, v28, v29, v30,
2682 v31)) {}
2683 // Repeat 16 values as many times as necessary (usually for lookup tables)
2684 is_utf8_really_inline static simd8<int8_t>
2685 repeat_16(int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5,
2686 int8_t v6, int8_t v7, int8_t v8, int8_t v9, int8_t v10, int8_t v11,
2687 int8_t v12, int8_t v13, int8_t v14, int8_t v15) {
2688 return simd8<int8_t>(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12,
2689 v13, v14, v15, v0, v1, v2, v3, v4, v5, v6, v7, v8, v9,
2690 v10, v11, v12, v13, v14, v15);
2691 }
2692 is_utf8_really_inline bool is_ascii() const {
2693 return _mm256_movemask_epi8(*this) == 0;
2694 }
2695 // Order-sensitive comparisons
2696 is_utf8_really_inline simd8<int8_t> max_val(const simd8<int8_t> other) const {
2697 return _mm256_max_epi8(*this, other);
2698 }
2699 is_utf8_really_inline simd8<int8_t> min_val(const simd8<int8_t> other) const {
2700 return _mm256_min_epi8(*this, other);
2701 }
2702 is_utf8_really_inline simd8<bool> operator>(const simd8<int8_t> other) const {
2703 return _mm256_cmpgt_epi8(*this, other);
2704 }
2705 is_utf8_really_inline simd8<bool> operator<(const simd8<int8_t> other) const {
2706 return _mm256_cmpgt_epi8(other, *this);
2707 }
2708 };
2709
2710 // Unsigned bytes
2711 template <> struct simd8<uint8_t> : base8_numeric<uint8_t> {
2712 is_utf8_really_inline simd8() : base8_numeric<uint8_t>() {}
2713 is_utf8_really_inline simd8(const __m256i _value)
2714 : base8_numeric<uint8_t>(_value) {}
2715 // Splat constructor
2716 is_utf8_really_inline simd8(uint8_t _value) : simd8(splat(_value)) {}
2717 // Array constructor
2718 is_utf8_really_inline simd8(const uint8_t values[32]) : simd8(load(values)) {}
2719 // Member-by-member initialization
2720 is_utf8_really_inline
2721 simd8(uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5,
2722 uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, uint8_t v10,
2723 uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15,
2724 uint8_t v16, uint8_t v17, uint8_t v18, uint8_t v19, uint8_t v20,
2725 uint8_t v21, uint8_t v22, uint8_t v23, uint8_t v24, uint8_t v25,
2726 uint8_t v26, uint8_t v27, uint8_t v28, uint8_t v29, uint8_t v30,
2727 uint8_t v31)
2728 : simd8(_mm256_setr_epi8(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11,
2729 v12, v13, v14, v15, v16, v17, v18, v19, v20, v21,
2730 v22, v23, v24, v25, v26, v27, v28, v29, v30,
2731 v31)) {}
2732 // Repeat 16 values as many times as necessary (usually for lookup tables)
2733 is_utf8_really_inline static simd8<uint8_t>
2734 repeat_16(uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4,
2735 uint8_t v5, uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9,
2736 uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14,
2737 uint8_t v15) {
2738 return simd8<uint8_t>(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12,
2739 v13, v14, v15, v0, v1, v2, v3, v4, v5, v6, v7, v8, v9,
2740 v10, v11, v12, v13, v14, v15);
2741 }
2742
2743 // Saturated math
2744 is_utf8_really_inline simd8<uint8_t>
2745 saturating_add(const simd8<uint8_t> other) const {
2746 return _mm256_adds_epu8(*this, other);
2747 }
2748 is_utf8_really_inline simd8<uint8_t>
2749 saturating_sub(const simd8<uint8_t> other) const {
2750 return _mm256_subs_epu8(*this, other);
2751 }
2752
2753 // Order-specific operations
2754 is_utf8_really_inline simd8<uint8_t>
2755 max_val(const simd8<uint8_t> other) const {
2756 return _mm256_max_epu8(*this, other);
2757 }
2758 is_utf8_really_inline simd8<uint8_t>
2759 min_val(const simd8<uint8_t> other) const {
2760 return _mm256_min_epu8(other, *this);
2761 }
2762 // Same as >, but only guarantees true is nonzero (< guarantees true = -1)
2763 is_utf8_really_inline simd8<uint8_t>
2764 gt_bits(const simd8<uint8_t> other) const {
2765 return this->saturating_sub(other);
2766 }
2767 // Same as <, but only guarantees true is nonzero (< guarantees true = -1)
2768 is_utf8_really_inline simd8<uint8_t>
2769 lt_bits(const simd8<uint8_t> other) const {
2770 return other.saturating_sub(*this);
2771 }
2772 is_utf8_really_inline simd8<bool>
2773 operator<=(const simd8<uint8_t> other) const {
2774 return other.max_val(*this) == other;
2775 }
2776 is_utf8_really_inline simd8<bool>
2777 operator>=(const simd8<uint8_t> other) const {
2778 return other.min_val(*this) == other;
2779 }
2780 is_utf8_really_inline simd8<bool>
2781 operator>(const simd8<uint8_t> other) const {
2782 return this->gt_bits(other).any_bits_set();
2783 }
2784 is_utf8_really_inline simd8<bool>
2785 operator<(const simd8<uint8_t> other) const {
2786 return this->lt_bits(other).any_bits_set();
2787 }
2788
2789 // Bit-specific operations
2790 is_utf8_really_inline simd8<bool> bits_not_set() const {
2791 return *this == uint8_t(0);
2792 }
2793 is_utf8_really_inline simd8<bool> bits_not_set(simd8<uint8_t> bits) const {
2794 return (*this & bits).bits_not_set();
2795 }
2796 is_utf8_really_inline simd8<bool> any_bits_set() const {
2797 return ~this->bits_not_set();
2798 }
2799 is_utf8_really_inline simd8<bool> any_bits_set(simd8<uint8_t> bits) const {
2800 return ~this->bits_not_set(bits);
2801 }
2802 is_utf8_really_inline bool is_ascii() const {
2803 return _mm256_movemask_epi8(*this) == 0;
2804 }
2805 is_utf8_really_inline bool bits_not_set_anywhere() const {
2806 return _mm256_testz_si256(*this, *this);
2807 }
2808 is_utf8_really_inline bool any_bits_set_anywhere() const {
2809 return !bits_not_set_anywhere();
2810 }
2811 is_utf8_really_inline bool bits_not_set_anywhere(simd8<uint8_t> bits) const {
2812 return _mm256_testz_si256(*this, bits);
2813 }
2814 is_utf8_really_inline bool any_bits_set_anywhere(simd8<uint8_t> bits) const {
2815 return !bits_not_set_anywhere(bits);
2816 }
2817 template <int N> is_utf8_really_inline simd8<uint8_t> shr() const {
2818 return simd8<uint8_t>(_mm256_srli_epi16(*this, N)) & uint8_t(0xFFu >> N);
2819 }
2820 template <int N> is_utf8_really_inline simd8<uint8_t> shl() const {
2821 return simd8<uint8_t>(_mm256_slli_epi16(*this, N)) & uint8_t(0xFFu << N);
2822 }
2823 // Get one of the bits and make a bitmask out of it.
2824 // e.g. value.get_bit<7>() gets the high bit
2825 template <int N> is_utf8_really_inline int get_bit() const {
2826 return _mm256_movemask_epi8(_mm256_slli_epi16(*this, 7 - N));
2827 }
2828 };
2829 is_utf8_really_inline simd8<int8_t>::operator simd8<uint8_t>() const {
2830 return this->value;
2831 }
2832
2833 template <typename T> struct simd8x64 {
2834 static constexpr int NUM_CHUNKS = 64 / sizeof(simd8<T>);
2835 static_assert(NUM_CHUNKS == 2,
2836 "Haswell kernel should use two registers per 64-byte block.");
2837 simd8<T> chunks[NUM_CHUNKS];
2838
2839 simd8x64(const simd8x64<T> &o) = delete; // no copy allowed
2840 simd8x64<T> &
2841 operator=(const simd8<T> other) = delete; // no assignment allowed
2842 simd8x64() = delete; // no default constructor allowed
2843
2844 is_utf8_really_inline simd8x64(const simd8<T> chunk0, const simd8<T> chunk1)
2845 : chunks{chunk0, chunk1} {}
2846 is_utf8_really_inline simd8x64(const T *ptr)
2847 : chunks{simd8<T>::load(ptr),
2848 simd8<T>::load(ptr + sizeof(simd8<T>) / sizeof(T))} {}
2849
2850 is_utf8_really_inline void store(T *ptr) const {
2851 this->chunks[0].store(ptr + sizeof(simd8<T>) * 0 / sizeof(T));
2852 this->chunks[1].store(ptr + sizeof(simd8<T>) * 1 / sizeof(T));
2853 }
2854
2855 is_utf8_really_inline uint64_t to_bitmask() const {
2856 uint64_t r_lo = uint32_t(this->chunks[0].to_bitmask());
2857 uint64_t r_hi = this->chunks[1].to_bitmask();
2858 return r_lo | (r_hi << 32);
2859 }
2860
2861 is_utf8_really_inline simd8x64<T> &operator|=(const simd8x64<T> &other) {
2862 this->chunks[0] |= other.chunks[0];
2863 this->chunks[1] |= other.chunks[1];
2864 return *this;
2865 }
2866
2867 is_utf8_really_inline simd8<T> reduce_or() const {
2868 return this->chunks[0] | this->chunks[1];
2869 }
2870
2871 is_utf8_really_inline bool is_ascii() const {
2872 return this->reduce_or().is_ascii();
2873 }
2874
2875 template <endianness endian>
2876 is_utf8_really_inline void store_ascii_as_utf16(char16_t *ptr) const {
2877 this->chunks[0].template store_ascii_as_utf16<endian>(ptr +
2878 sizeof(simd8<T>) * 0);
2879 this->chunks[1].template store_ascii_as_utf16<endian>(ptr +
2880 sizeof(simd8<T>) * 1);
2881 }
2882
2883 is_utf8_really_inline void store_ascii_as_utf32(char32_t *ptr) const {
2884 this->chunks[0].store_ascii_as_utf32(ptr + sizeof(simd8<T>) * 0);
2885 this->chunks[1].store_ascii_as_utf32(ptr + sizeof(simd8<T>) * 1);
2886 }
2887
2888 is_utf8_really_inline simd8x64<T> bit_or(const T m) const {
2889 const simd8<T> mask = simd8<T>::splat(m);
2890 return simd8x64<T>(this->chunks[0] | mask, this->chunks[1] | mask);
2891 }
2892
2893 is_utf8_really_inline uint64_t eq(const T m) const {
2894 const simd8<T> mask = simd8<T>::splat(m);
2895 return simd8x64<bool>(this->chunks[0] == mask, this->chunks[1] == mask)
2896 .to_bitmask();
2897 }
2898
2899 is_utf8_really_inline uint64_t eq(const simd8x64<uint8_t> &other) const {
2900 return simd8x64<bool>(this->chunks[0] == other.chunks[0],
2901 this->chunks[1] == other.chunks[1])
2902 .to_bitmask();
2903 }
2904
2905 is_utf8_really_inline uint64_t lteq(const T m) const {
2906 const simd8<T> mask = simd8<T>::splat(m);
2907 return simd8x64<bool>(this->chunks[0] <= mask, this->chunks[1] <= mask)
2908 .to_bitmask();
2909 }
2910
2911 is_utf8_really_inline uint64_t in_range(const T low, const T high) const {
2912 const simd8<T> mask_low = simd8<T>::splat(low);
2913 const simd8<T> mask_high = simd8<T>::splat(high);
2914
2915 return simd8x64<bool>(
2916 (this->chunks[0] <= mask_high) & (this->chunks[0] >= mask_low),
2917 (this->chunks[1] <= mask_high) & (this->chunks[1] >= mask_low),
2918 (this->chunks[2] <= mask_high) & (this->chunks[2] >= mask_low),
2919 (this->chunks[3] <= mask_high) & (this->chunks[3] >= mask_low))
2920 .to_bitmask();
2921 }
2922 is_utf8_really_inline uint64_t not_in_range(const T low, const T high) const {
2923 const simd8<T> mask_low = simd8<T>::splat(low);
2924 const simd8<T> mask_high = simd8<T>::splat(high);
2925 return simd8x64<bool>(
2926 (this->chunks[0] > mask_high) | (this->chunks[0] < mask_low),
2927 (this->chunks[1] > mask_high) | (this->chunks[1] < mask_low))
2928 .to_bitmask();
2929 }
2930 is_utf8_really_inline uint64_t lt(const T m) const {
2931 const simd8<T> mask = simd8<T>::splat(m);
2932 return simd8x64<bool>(this->chunks[0] < mask, this->chunks[1] < mask)
2933 .to_bitmask();
2934 }
2935
2936 is_utf8_really_inline uint64_t gt(const T m) const {
2937 const simd8<T> mask = simd8<T>::splat(m);
2938 return simd8x64<bool>(this->chunks[0] > mask, this->chunks[1] > mask)
2939 .to_bitmask();
2940 }
2941 is_utf8_really_inline uint64_t gteq(const T m) const {
2942 const simd8<T> mask = simd8<T>::splat(m);
2943 return simd8x64<bool>(this->chunks[0] >= mask, this->chunks[1] >= mask)
2944 .to_bitmask();
2945 }
2946 is_utf8_really_inline uint64_t gteq_unsigned(const uint8_t m) const {
2947 const simd8<uint8_t> mask = simd8<uint8_t>::splat(m);
2948 return simd8x64<bool>((simd8<uint8_t>(__m256i(this->chunks[0])) >= mask),
2949 (simd8<uint8_t>(__m256i(this->chunks[1])) >= mask))
2950 .to_bitmask();
2951 }
2952 }; // struct simd8x64<T>
2953
2954 #ifdef __GNUC__
2955 #if __GNUC__ < 8
2956 #define _mm256_set_m128i(xmm1, xmm2) \
2957 _mm256_permute2f128_si256(_mm256_castsi128_si256(xmm1), \
2958 _mm256_castsi128_si256(xmm2), 2)
2959 #define _mm256_setr_m128i(xmm2, xmm1) \
2960 _mm256_permute2f128_si256(_mm256_castsi128_si256(xmm1), \
2961 _mm256_castsi128_si256(xmm2), 2)
2962 #endif
2963 #endif
2964
2965 template <typename T> struct simd16;
2966
2967 template <typename T, typename Mask = simd16<bool>>
2968 struct base16 : base<simd16<T>> {
2969 using bitmask_type = uint32_t;
2970
2971 is_utf8_really_inline base16() : base<simd16<T>>() {}
2972 is_utf8_really_inline base16(const __m256i _value)
2973 : base<simd16<T>>(_value) {}
2974 template <typename Pointer>
2975 is_utf8_really_inline base16(const Pointer *ptr)
2976 : base16(_mm256_loadu_si256(reinterpret_cast<const __m256i *>(ptr))) {}
2977
2978 friend is_utf8_really_inline Mask operator==(const simd16<T> lhs,
2979 const simd16<T> rhs) {
2980 return _mm256_cmpeq_epi16(lhs, rhs);
2981 }
2982
2983 /// the size of vector in bytes
2984 static const int SIZE = sizeof(base<simd16<T>>::value);
2985
2986 /// the number of elements of type T a vector can hold
2987 static const int ELEMENTS = SIZE / sizeof(T);
2988
2989 template <int N = 1>
2990 is_utf8_really_inline simd16<T> prev(const simd16<T> prev_chunk) const {
2991 return _mm256_alignr_epi8(*this, prev_chunk, 16 - N);
2992 }
2993 };
2994
2995 // SIMD byte mask type (returned by things like eq and gt)
2996 template <> struct simd16<bool> : base16<bool> {
2997 static is_utf8_really_inline simd16<bool> splat(bool _value) {
2998 return _mm256_set1_epi16(uint16_t(-(!!_value)));
2999 }
3000
3001 is_utf8_really_inline simd16() : base16() {}
3002 is_utf8_really_inline simd16(const __m256i _value)
3003 : base16<bool>(_value) {}
3004 // Splat constructor
3005 is_utf8_really_inline simd16(bool _value)
3006 : base16<bool>(splat(_value)) {}
3007
3008 is_utf8_really_inline bitmask_type to_bitmask() const {
3009 return _mm256_movemask_epi8(*this);
3010 }
3011 is_utf8_really_inline bool any() const {
3012 return !_mm256_testz_si256(*this, *this);
3013 }
3014 is_utf8_really_inline simd16<bool> operator~() const { return *this ^ true; }
3015 };
3016
3017 template <typename T> struct base16_numeric : base16<T> {
3018 static is_utf8_really_inline simd16<T> splat(T _value) {
3019 return _mm256_set1_epi16(_value);
3020 }
3021 static is_utf8_really_inline simd16<T> zero() {
3022 return _mm256_setzero_si256();
3023 }
3024 static is_utf8_really_inline simd16<T> load(const T values[8]) {
3025 return _mm256_loadu_si256(reinterpret_cast<const __m256i *>(values));
3026 }
3027
3028 is_utf8_really_inline base16_numeric() : base16<T>() {}
3029 is_utf8_really_inline base16_numeric(const __m256i _value)
3030 : base16<T>(_value) {}
3031
3032 // Store to array
3033 is_utf8_really_inline void store(T dst[8]) const {
3034 return _mm256_storeu_si256(reinterpret_cast<__m256i *>(dst), *this);
3035 }
3036
3037 // Override to distinguish from bool version
3038 is_utf8_really_inline simd16<T> operator~() const { return *this ^ 0xFFFFu; }
3039
3040 // Addition/subtraction are the same for signed and unsigned
3041 is_utf8_really_inline simd16<T> operator+(const simd16<T> other) const {
3042 return _mm256_add_epi16(*this, other);
3043 }
3044 is_utf8_really_inline simd16<T> operator-(const simd16<T> other) const {
3045 return _mm256_sub_epi16(*this, other);
3046 }
3047 is_utf8_really_inline simd16<T> &operator+=(const simd16<T> other) {
3048 *this = *this + other;
3049 return *static_cast<simd16<T> *>(this);
3050 }
3051 is_utf8_really_inline simd16<T> &operator-=(const simd16<T> other) {
3052 *this = *this - other;
3053 return *static_cast<simd16<T> *>(this);
3054 }
3055 };
3056
3057 // Signed words
3058 template <> struct simd16<int16_t> : base16_numeric<int16_t> {
3059 is_utf8_really_inline simd16() : base16_numeric<int16_t>() {}
3060 is_utf8_really_inline simd16(const __m256i _value)
3061 : base16_numeric<int16_t>(_value) {}
3062 // Splat constructor
3063 is_utf8_really_inline simd16(int16_t _value) : simd16(splat(_value)) {}
3064 // Array constructor
3065 is_utf8_really_inline simd16(const int16_t *values) : simd16(load(values)) {}
3066 is_utf8_really_inline simd16(const char16_t *values)
3067 : simd16(load(reinterpret_cast<const int16_t *>(values))) {}
3068 // Order-sensitive comparisons
3069 is_utf8_really_inline simd16<int16_t>
3070 max_val(const simd16<int16_t> other) const {
3071 return _mm256_max_epi16(*this, other);
3072 }
3073 is_utf8_really_inline simd16<int16_t>
3074 min_val(const simd16<int16_t> other) const {
3075 return _mm256_min_epi16(*this, other);
3076 }
3077 is_utf8_really_inline simd16<bool>
3078 operator>(const simd16<int16_t> other) const {
3079 return _mm256_cmpgt_epi16(*this, other);
3080 }
3081 is_utf8_really_inline simd16<bool>
3082 operator<(const simd16<int16_t> other) const {
3083 return _mm256_cmpgt_epi16(other, *this);
3084 }
3085 };
3086
3087 // Unsigned words
3088 template <> struct simd16<uint16_t> : base16_numeric<uint16_t> {
3089 is_utf8_really_inline simd16() : base16_numeric<uint16_t>() {}
3090 is_utf8_really_inline simd16(const __m256i _value)
3091 : base16_numeric<uint16_t>(_value) {}
3092
3093 // Splat constructor
3094 is_utf8_really_inline simd16(uint16_t _value) : simd16(splat(_value)) {}
3095 // Array constructor
3096 is_utf8_really_inline simd16(const uint16_t *values) : simd16(load(values)) {}
3097 is_utf8_really_inline simd16(const char16_t *values)
3098 : simd16(load(reinterpret_cast<const uint16_t *>(values))) {}
3099
3100 // Saturated math
3101 is_utf8_really_inline simd16<uint16_t>
3102 saturating_add(const simd16<uint16_t> other) const {
3103 return _mm256_adds_epu16(*this, other);
3104 }
3105 is_utf8_really_inline simd16<uint16_t>
3106 saturating_sub(const simd16<uint16_t> other) const {
3107 return _mm256_subs_epu16(*this, other);
3108 }
3109
3110 // Order-specific operations
3111 is_utf8_really_inline simd16<uint16_t>
3112 max_val(const simd16<uint16_t> other) const {
3113 return _mm256_max_epu16(*this, other);
3114 }
3115 is_utf8_really_inline simd16<uint16_t>
3116 min_val(const simd16<uint16_t> other) const {
3117 return _mm256_min_epu16(*this, other);
3118 }
3119 // Same as >, but only guarantees true is nonzero (< guarantees true = -1)
3120 is_utf8_really_inline simd16<uint16_t>
3121 gt_bits(const simd16<uint16_t> other) const {
3122 return this->saturating_sub(other);
3123 }
3124 // Same as <, but only guarantees true is nonzero (< guarantees true = -1)
3125 is_utf8_really_inline simd16<uint16_t>
3126 lt_bits(const simd16<uint16_t> other) const {
3127 return other.saturating_sub(*this);
3128 }
3129 is_utf8_really_inline simd16<bool>
3130 operator<=(const simd16<uint16_t> other) const {
3131 return other.max_val(*this) == other;
3132 }
3133 is_utf8_really_inline simd16<bool>
3134 operator>=(const simd16<uint16_t> other) const {
3135 return other.min_val(*this) == other;
3136 }
3137 is_utf8_really_inline simd16<bool>
3138 operator>(const simd16<uint16_t> other) const {
3139 return this->gt_bits(other).any_bits_set();
3140 }
3141 is_utf8_really_inline simd16<bool>
3142 operator<(const simd16<uint16_t> other) const {
3143 return this->gt_bits(other).any_bits_set();
3144 }
3145
3146 // Bit-specific operations
3147 is_utf8_really_inline simd16<bool> bits_not_set() const {
3148 return *this == uint16_t(0);
3149 }
3150 is_utf8_really_inline simd16<bool> bits_not_set(simd16<uint16_t> bits) const {
3151 return (*this & bits).bits_not_set();
3152 }
3153 is_utf8_really_inline simd16<bool> any_bits_set() const {
3154 return ~this->bits_not_set();
3155 }
3156 is_utf8_really_inline simd16<bool> any_bits_set(simd16<uint16_t> bits) const {
3157 return ~this->bits_not_set(bits);
3158 }
3159
3160 is_utf8_really_inline bool bits_not_set_anywhere() const {
3161 return _mm256_testz_si256(*this, *this);
3162 }
3163 is_utf8_really_inline bool any_bits_set_anywhere() const {
3164 return !bits_not_set_anywhere();
3165 }
3166 is_utf8_really_inline bool
3167 bits_not_set_anywhere(simd16<uint16_t> bits) const {
3168 return _mm256_testz_si256(*this, bits);
3169 }
3170 is_utf8_really_inline bool
3171 any_bits_set_anywhere(simd16<uint16_t> bits) const {
3172 return !bits_not_set_anywhere(bits);
3173 }
3174 template <int N> is_utf8_really_inline simd16<uint16_t> shr() const {
3175 return simd16<uint16_t>(_mm256_srli_epi16(*this, N));
3176 }
3177 template <int N> is_utf8_really_inline simd16<uint16_t> shl() const {
3178 return simd16<uint16_t>(_mm256_slli_epi16(*this, N));
3179 }
3180 // Get one of the bits and make a bitmask out of it.
3181 // e.g. value.get_bit<7>() gets the high bit
3182 template <int N> is_utf8_really_inline int get_bit() const {
3183 return _mm256_movemask_epi8(_mm256_slli_epi16(*this, 15 - N));
3184 }
3185
3186 // Change the endianness
3187 is_utf8_really_inline simd16<uint16_t> swap_bytes() const {
3188 const __m256i swap = _mm256_setr_epi8(
3189 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14, 17, 16, 19, 18,
3190 21, 20, 23, 22, 25, 24, 27, 26, 29, 28, 31, 30);
3191 return _mm256_shuffle_epi8(*this, swap);
3192 }
3193
3194 // Pack with the unsigned saturation two uint16_t words into single uint8_t
3195 // vector
3196 static is_utf8_really_inline simd8<uint8_t> pack(const simd16<uint16_t> &v0,
3197 const simd16<uint16_t> &v1) {
3198 // Note: the AVX2 variant of pack operates on 128-bit lanes, thus
3199 // we have to shuffle lanes in order to produce bytes in the
3200 // correct order.
3201
3202 // get the 0th lanes
3203 const __m128i lo_0 = _mm256_extracti128_si256(v0, 0);
3204 const __m128i lo_1 = _mm256_extracti128_si256(v1, 0);
3205
3206 // get the 1st lanes
3207 const __m128i hi_0 = _mm256_extracti128_si256(v0, 1);
3208 const __m128i hi_1 = _mm256_extracti128_si256(v1, 1);
3209
3210 // build new vectors (shuffle lanes)
3211 const __m256i t0 = _mm256_set_m128i(lo_1, lo_0);
3212 const __m256i t1 = _mm256_set_m128i(hi_1, hi_0);
3213
3214 // pack words in linear order from v0 and v1
3215 return _mm256_packus_epi16(t0, t1);
3216 }
3217 };
3218
3219 template <typename T> struct simd16x32 {
3220 static constexpr int NUM_CHUNKS = 64 / sizeof(simd16<T>);
3221 static_assert(NUM_CHUNKS == 2,
3222 "Haswell kernel should use two registers per 64-byte block.");
3223 simd16<T> chunks[NUM_CHUNKS];
3224
3225 simd16x32(const simd16x32<T> &o) = delete; // no copy allowed
3226 simd16x32<T> &
3227 operator=(const simd16<T> other) = delete; // no assignment allowed
3228 simd16x32() = delete; // no default constructor allowed
3229
3230 is_utf8_really_inline simd16x32(const simd16<T> chunk0,
3231 const simd16<T> chunk1)
3232 : chunks{chunk0, chunk1} {}
3233 is_utf8_really_inline simd16x32(const T *ptr)
3234 : chunks{simd16<T>::load(ptr),
3235 simd16<T>::load(ptr + sizeof(simd16<T>) / sizeof(T))} {}
3236
3237 is_utf8_really_inline void store(T *ptr) const {
3238 this->chunks[0].store(ptr + sizeof(simd16<T>) * 0 / sizeof(T));
3239 this->chunks[1].store(ptr + sizeof(simd16<T>) * 1 / sizeof(T));
3240 }
3241
3242 is_utf8_really_inline uint64_t to_bitmask() const {
3243 uint64_t r_lo = uint32_t(this->chunks[0].to_bitmask());
3244 uint64_t r_hi = this->chunks[1].to_bitmask();
3245 return r_lo | (r_hi << 32);
3246 }
3247
3248 is_utf8_really_inline simd16<T> reduce_or() const {
3249 return this->chunks[0] | this->chunks[1];
3250 }
3251
3252 is_utf8_really_inline bool is_ascii() const {
3253 return this->reduce_or().is_ascii();
3254 }
3255
3256 is_utf8_really_inline void store_ascii_as_utf16(char16_t *ptr) const {
3257 this->chunks[0].store_ascii_as_utf16(ptr + sizeof(simd16<T>) * 0);
3258 this->chunks[1].store_ascii_as_utf16(ptr + sizeof(simd16<T>));
3259 }
3260
3261 is_utf8_really_inline simd16x32<T> bit_or(const T m) const {
3262 const simd16<T> mask = simd16<T>::splat(m);
3263 return simd16x32<T>(this->chunks[0] | mask, this->chunks[1] | mask);
3264 }
3265
3266 is_utf8_really_inline void swap_bytes() {
3267 this->chunks[0] = this->chunks[0].swap_bytes();
3268 this->chunks[1] = this->chunks[1].swap_bytes();
3269 }
3270
3271 is_utf8_really_inline uint64_t eq(const T m) const {
3272 const simd16<T> mask = simd16<T>::splat(m);
3273 return simd16x32<bool>(this->chunks[0] == mask, this->chunks[1] == mask)
3274 .to_bitmask();
3275 }
3276
3277 is_utf8_really_inline uint64_t eq(const simd16x32<uint16_t> &other) const {
3278 return simd16x32<bool>(this->chunks[0] == other.chunks[0],
3279 this->chunks[1] == other.chunks[1])
3280 .to_bitmask();
3281 }
3282
3283 is_utf8_really_inline uint64_t lteq(const T m) const {
3284 const simd16<T> mask = simd16<T>::splat(m);
3285 return simd16x32<bool>(this->chunks[0] <= mask, this->chunks[1] <= mask)
3286 .to_bitmask();
3287 }
3288
3289 is_utf8_really_inline uint64_t in_range(const T low, const T high) const {
3290 const simd16<T> mask_low = simd16<T>::splat(low);
3291 const simd16<T> mask_high = simd16<T>::splat(high);
3292
3293 return simd16x32<bool>(
3294 (this->chunks[0] <= mask_high) & (this->chunks[0] >= mask_low),
3295 (this->chunks[1] <= mask_high) & (this->chunks[1] >= mask_low),
3296 (this->chunks[2] <= mask_high) & (this->chunks[2] >= mask_low),
3297 (this->chunks[3] <= mask_high) & (this->chunks[3] >= mask_low))
3298 .to_bitmask();
3299 }
3300 is_utf8_really_inline uint64_t not_in_range(const T low, const T high) const {
3301 const simd16<T> mask_low = simd16<T>::splat(static_cast<T>(low - 1));
3302 const simd16<T> mask_high = simd16<T>::splat(static_cast<T>(high + 1));
3303 return simd16x32<bool>(
3304 (this->chunks[0] >= mask_high) | (this->chunks[0] <= mask_low),
3305 (this->chunks[1] >= mask_high) | (this->chunks[1] <= mask_low))
3306 .to_bitmask();
3307 }
3308 is_utf8_really_inline uint64_t lt(const T m) const {
3309 const simd16<T> mask = simd16<T>::splat(m);
3310 return simd16x32<bool>(this->chunks[0] < mask, this->chunks[1] < mask)
3311 .to_bitmask();
3312 }
3313 }; // struct simd16x32<T>
3314
3315 } // namespace simd
3316
3317 } // unnamed namespace
3318 } // namespace haswell
3319 } // namespace is_utf8_internals
3320
3321 #endif // IS_UTF8_HASWELL_SIMD_H
3322
3323 IS_UTF8_UNTARGET_REGION
3324
3325 #if IS_UTF8_GCC11ORMORE // workaround for
3326 // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=105593
3327 #pragma GCC diagnostic pop
3328 #endif // end of workaround
3329
3330 #endif // IS_UTF8_IMPLEMENTATION_HASWELL
3331 #endif // IS_UTF8_HASWELL_COMMON_H
3332
3333 #ifndef IS_UTF8_WESTMERE_H
3334 #define IS_UTF8_WESTMERE_H
3335
3336 #ifdef IS_UTF8_FALLBACK_H
3337 #error "westmere.h must be included before fallback.h"
3338 #endif
3339
3340 // Default Westmere to on if this is x86-64, unless we'll always select Haswell.
3341 #ifndef IS_UTF8_IMPLEMENTATION_WESTMERE
3342 //
3343 // You do not want to set it to (IS_UTF8_IS_X86_64 && !IS_UTF8_REQUIRES_HASWELL)
3344 // because you want to rely on runtime dispatch!
3345 //
3346 #if IS_UTF8_CAN_ALWAYS_RUN_ICELAKE || IS_UTF8_CAN_ALWAYS_RUN_HASWELL
3347 #define IS_UTF8_IMPLEMENTATION_WESTMERE 0
3348 #else
3349 #define IS_UTF8_IMPLEMENTATION_WESTMERE (IS_UTF8_IS_X86_64)
3350 #endif
3351
3352 #endif
3353
3354 #if IS_UTF8_IMPLEMENTATION_WESTMERE && IS_UTF8_IS_X86_64 && __SSE4_2__ && __PCLMUL__
3355 #define IS_UTF8_CAN_ALWAYS_RUN_WESTMERE 1
3356 #else
3357 #define IS_UTF8_CAN_ALWAYS_RUN_WESTMERE 0
3358 #endif
3359
3360 #if IS_UTF8_IMPLEMENTATION_WESTMERE
3361
3362 #define IS_UTF8_TARGET_WESTMERE IS_UTF8_TARGET_REGION("sse4.2,pclmul")
3363
3364 namespace is_utf8_internals {
3365 /**
3366 * Implementation for Westmere (Intel SSE4.2).
3367 */
3368 namespace westmere {} // namespace westmere
3369 } // namespace is_utf8_internals
3370
3371 //
3372 // These two need to be included outside IS_UTF8_TARGET_REGION
3373 //
3374
3375 #ifndef IS_UTF8_WESTMERE_IMPLEMENTATION_H
3376 #define IS_UTF8_WESTMERE_IMPLEMENTATION_H
3377
3378 // The constructor may be executed on any host, so we take care not to use
3379 // IS_UTF8_TARGET_REGION
3380 namespace is_utf8_internals {
3381 namespace westmere {
3382
3383 class implementation final : public is_utf8_internals::implementation {
3384 public:
3385 is_utf8_really_inline implementation()
3386 : is_utf8_internals::implementation(
3387 "westmere", "Intel/AMD SSE4.2",
3388 internal::instruction_set::SSE42 |
3389 internal::instruction_set::PCLMULQDQ) {}
3390 is_utf8_warn_unused bool validate_utf8(const char *buf,
3391 size_t len) const noexcept final;
3392 };
3393
3394 } // namespace westmere
3395 } // namespace is_utf8_internals
3396
3397 #endif // IS_UTF8_WESTMERE_IMPLEMENTATION_H
3398
3399 #ifndef IS_UTF8_WESTMERE_INTRINSICS_H
3400 #define IS_UTF8_WESTMERE_INTRINSICS_H
3401
3402 #ifdef IS_UTF8_VISUAL_STUDIO
3403 // under clang within visual studio, this will include <x86intrin.h>
3404 #include <intrin.h> // visual studio or clang
3405 #else
3406
3407 #if IS_UTF8_GCC11ORMORE
3408 // We should not get warnings while including <x86intrin.h> yet we do
3409 // under some versions of GCC.
3410 // If the x86intrin.h header has uninitialized values that are problematic,
3411 // it is a GCC issue, we want to ignore these warnings.
3412 IS_UTF8_DISABLE_GCC_WARNING(-Wuninitialized)
3413 #endif
3414
3415 #include <x86intrin.h> // elsewhere
3416
3417 #if IS_UTF8_GCC11ORMORE
3418 // cancels the suppression of the -Wuninitialized
3419 IS_UTF8_POP_DISABLE_WARNINGS
3420 #endif
3421
3422 #endif // IS_UTF8_VISUAL_STUDIO
3423
3424 #ifdef IS_UTF8_CLANG_VISUAL_STUDIO
3425 /**
3426 * You are not supposed, normally, to include these
3427 * headers directly. Instead you should either include intrin.h
3428 * or x86intrin.h. However, when compiling with clang
3429 * under Windows (i.e., when _MSC_VER is set), these headers
3430 * only get included *if* the corresponding features are detected
3431 * from macros:
3432 */
3433 #include <smmintrin.h> // for _mm_alignr_epi8
3434 #include <wmmintrin.h> // for _mm_clmulepi64_si128
3435 #endif
3436
3437 #endif // IS_UTF8_WESTMERE_INTRINSICS_H
3438
3439 //
3440 // The rest need to be inside the region
3441 //
3442
3443 // redefining IS_UTF8_IMPLEMENTATION to "westmere"
3444 // #define IS_UTF8_IMPLEMENTATION westmere
3445 IS_UTF8_TARGET_WESTMERE
3446
3447 // Declarations
3448
3449 #ifndef IS_UTF8_WESTMERE_SIMD_H
3450 #define IS_UTF8_WESTMERE_SIMD_H
3451
3452 namespace is_utf8_internals {
3453 namespace westmere {
3454 namespace {
3455 namespace simd {
3456
3457 template <typename Child> struct base {
3458 __m128i value;
3459
3460 // Zero constructor
3461 is_utf8_really_inline base() : value{__m128i()} {}
3462
3463 // Conversion from SIMD register
3464 is_utf8_really_inline base(const __m128i _value) : value(_value) {}
3465 // Conversion to SIMD register
3466 is_utf8_really_inline operator const __m128i &() const { return this->value; }
3467 is_utf8_really_inline operator __m128i &() { return this->value; }
3468 template <endianness big_endian>
3469 is_utf8_really_inline void store_ascii_as_utf16(char16_t *p) const {
3470 __m128i first = _mm_cvtepu8_epi16(*this);
3471 __m128i second = _mm_cvtepu8_epi16(_mm_srli_si128(*this, 8));
3472 if (big_endian) {
3473 const __m128i swap =
3474 _mm_setr_epi8(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14);
3475 first = _mm_shuffle_epi8(first, swap);
3476 second = _mm_shuffle_epi8(second, swap);
3477 }
3478 _mm_storeu_si128(reinterpret_cast<__m128i *>(p), first);
3479 _mm_storeu_si128(reinterpret_cast<__m128i *>(p + 8), second);
3480 }
3481 is_utf8_really_inline void store_ascii_as_utf32(char32_t *p) const {
3482 _mm_storeu_si128(reinterpret_cast<__m128i *>(p), _mm_cvtepu8_epi32(*this));
3483 _mm_storeu_si128(reinterpret_cast<__m128i *>(p + 4),
3484 _mm_cvtepu8_epi32(_mm_srli_si128(*this, 4)));
3485 _mm_storeu_si128(reinterpret_cast<__m128i *>(p + 8),
3486 _mm_cvtepu8_epi32(_mm_srli_si128(*this, 8)));
3487 _mm_storeu_si128(reinterpret_cast<__m128i *>(p + 12),
3488 _mm_cvtepu8_epi32(_mm_srli_si128(*this, 12)));
3489 }
3490 // Bit operations
3491 is_utf8_really_inline Child operator|(const Child other) const {
3492 return _mm_or_si128(*this, other);
3493 }
3494 is_utf8_really_inline Child operator&(const Child other) const {
3495 return _mm_and_si128(*this, other);
3496 }
3497 is_utf8_really_inline Child operator^(const Child other) const {
3498 return _mm_xor_si128(*this, other);
3499 }
3500 is_utf8_really_inline Child bit_andnot(const Child other) const {
3501 return _mm_andnot_si128(other, *this);
3502 }
3503 is_utf8_really_inline Child &operator|=(const Child other) {
3504 auto this_cast = static_cast<Child *>(this);
3505 *this_cast = *this_cast | other;
3506 return *this_cast;
3507 }
3508 is_utf8_really_inline Child &operator&=(const Child other) {
3509 auto this_cast = static_cast<Child *>(this);
3510 *this_cast = *this_cast & other;
3511 return *this_cast;
3512 }
3513 is_utf8_really_inline Child &operator^=(const Child other) {
3514 auto this_cast = static_cast<Child *>(this);
3515 *this_cast = *this_cast ^ other;
3516 return *this_cast;
3517 }
3518 };
3519
3520 // Forward-declared so they can be used by splat and friends.
3521 template <typename T> struct simd8;
3522
3523 template <typename T, typename Mask = simd8<bool>>
3524 struct base8 : base<simd8<T>> {
3525 typedef uint16_t bitmask_t;
3526 typedef uint32_t bitmask2_t;
3527
3528 is_utf8_really_inline T first() const { return _mm_extract_epi8(*this, 0); }
3529 is_utf8_really_inline T last() const { return _mm_extract_epi8(*this, 15); }
3530 is_utf8_really_inline base8() : base<simd8<T>>() {}
3531 is_utf8_really_inline base8(const __m128i _value) : base<simd8<T>>(_value) {}
3532
3533 friend is_utf8_really_inline Mask operator==(const simd8<T> lhs,
3534 const simd8<T> rhs) {
3535 return _mm_cmpeq_epi8(lhs, rhs);
3536 }
3537
3538 static const int SIZE = sizeof(base<simd8<T>>::value);
3539
3540 template <int N = 1>
3541 is_utf8_really_inline simd8<T> prev(const simd8<T> prev_chunk) const {
3542 return _mm_alignr_epi8(*this, prev_chunk, 16 - N);
3543 }
3544 };
3545
3546 // SIMD byte mask type (returned by things like eq and gt)
3547 template <> struct simd8<bool> : base8<bool> {
3548 static is_utf8_really_inline simd8<bool> splat(bool _value) {
3549 return _mm_set1_epi8(uint8_t(-(!!_value)));
3550 }
3551
3552 is_utf8_really_inline simd8() : base8() {}
3553 is_utf8_really_inline simd8(const __m128i _value)
3554 : base8<bool>(_value) {}
3555 // Splat constructor
3556 is_utf8_really_inline simd8(bool _value) : base8<bool>(splat(_value)) {}
3557
3558 is_utf8_really_inline int to_bitmask() const {
3559 return _mm_movemask_epi8(*this);
3560 }
3561 is_utf8_really_inline bool any() const {
3562 return !_mm_testz_si128(*this, *this);
3563 }
3564 is_utf8_really_inline bool none() const {
3565 return _mm_testz_si128(*this, *this);
3566 }
3567 is_utf8_really_inline bool all() const {
3568 return _mm_movemask_epi8(*this) == 0xFFFF;
3569 }
3570 is_utf8_really_inline simd8<bool> operator~() const { return *this ^ true; }
3571 };
3572
3573 template <typename T> struct base8_numeric : base8<T> {
3574 static is_utf8_really_inline simd8<T> splat(T _value) {
3575 return _mm_set1_epi8(_value);
3576 }
3577 static is_utf8_really_inline simd8<T> zero() { return _mm_setzero_si128(); }
3578 static is_utf8_really_inline simd8<T> load(const T values[16]) {
3579 return _mm_loadu_si128(reinterpret_cast<const __m128i *>(values));
3580 }
3581 // Repeat 16 values as many times as necessary (usually for lookup tables)
3582 static is_utf8_really_inline simd8<T> repeat_16(T v0, T v1, T v2, T v3, T v4,
3583 T v5, T v6, T v7, T v8, T v9,
3584 T v10, T v11, T v12, T v13,
3585 T v14, T v15) {
3586 return simd8<T>(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13,
3587 v14, v15);
3588 }
3589
3590 is_utf8_really_inline base8_numeric() : base8<T>() {}
3591 is_utf8_really_inline base8_numeric(const __m128i _value)
3592 : base8<T>(_value) {}
3593
3594 // Store to array
3595 is_utf8_really_inline void store(T dst[16]) const {
3596 return _mm_storeu_si128(reinterpret_cast<__m128i *>(dst), *this);
3597 }
3598
3599 // Override to distinguish from bool version
3600 is_utf8_really_inline simd8<T> operator~() const { return *this ^ 0xFFu; }
3601
3602 // Addition/subtraction are the same for signed and unsigned
3603 is_utf8_really_inline simd8<T> operator+(const simd8<T> other) const {
3604 return _mm_add_epi8(*this, other);
3605 }
3606 is_utf8_really_inline simd8<T> operator-(const simd8<T> other) const {
3607 return _mm_sub_epi8(*this, other);
3608 }
3609 is_utf8_really_inline simd8<T> &operator+=(const simd8<T> other) {
3610 *this = *this + other;
3611 return *static_cast<simd8<T> *>(this);
3612 }
3613 is_utf8_really_inline simd8<T> &operator-=(const simd8<T> other) {
3614 *this = *this - other;
3615 return *static_cast<simd8<T> *>(this);
3616 }
3617
3618 // Perform a lookup assuming the value is between 0 and 16 (undefined behavior
3619 // for out of range values)
3620 template <typename L>
3621 is_utf8_really_inline simd8<L> lookup_16(simd8<L> lookup_table) const {
3622 return _mm_shuffle_epi8(lookup_table, *this);
3623 }
3624
3625 template <typename L>
3626 is_utf8_really_inline simd8<L>
3627 lookup_16(L replace0, L replace1, L replace2, L replace3, L replace4,
3628 L replace5, L replace6, L replace7, L replace8, L replace9,
3629 L replace10, L replace11, L replace12, L replace13, L replace14,
3630 L replace15) const {
3631 return lookup_16(simd8<L>::repeat_16(
3632 replace0, replace1, replace2, replace3, replace4, replace5, replace6,
3633 replace7, replace8, replace9, replace10, replace11, replace12,
3634 replace13, replace14, replace15));
3635 }
3636 };
3637
3638 // Signed bytes
3639 template <> struct simd8<int8_t> : base8_numeric<int8_t> {
3640 is_utf8_really_inline simd8() : base8_numeric<int8_t>() {}
3641 is_utf8_really_inline simd8(const __m128i _value)
3642 : base8_numeric<int8_t>(_value) {}
3643 // Splat constructor
3644 is_utf8_really_inline simd8(int8_t _value) : simd8(splat(_value)) {}
3645 // Array constructor
3646 is_utf8_really_inline simd8(const int8_t *values) : simd8(load(values)) {}
3647 // Member-by-member initialization
3648 is_utf8_really_inline simd8(int8_t v0, int8_t v1, int8_t v2, int8_t v3,
3649 int8_t v4, int8_t v5, int8_t v6, int8_t v7,
3650 int8_t v8, int8_t v9, int8_t v10, int8_t v11,
3651 int8_t v12, int8_t v13, int8_t v14, int8_t v15)
3652 : simd8(_mm_setr_epi8(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11,
3653 v12, v13, v14, v15)) {}
3654 // Repeat 16 values as many times as necessary (usually for lookup tables)
3655 is_utf8_really_inline static simd8<int8_t>
3656 repeat_16(int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5,
3657 int8_t v6, int8_t v7, int8_t v8, int8_t v9, int8_t v10, int8_t v11,
3658 int8_t v12, int8_t v13, int8_t v14, int8_t v15) {
3659 return simd8<int8_t>(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12,
3660 v13, v14, v15);
3661 }
3662 is_utf8_really_inline operator simd8<uint8_t>() const;
3663 is_utf8_really_inline bool is_ascii() const {
3664 return _mm_movemask_epi8(*this) == 0;
3665 }
3666
3667 // Order-sensitive comparisons
3668 is_utf8_really_inline simd8<int8_t> max_val(const simd8<int8_t> other) const {
3669 return _mm_max_epi8(*this, other);
3670 }
3671 is_utf8_really_inline simd8<int8_t> min_val(const simd8<int8_t> other) const {
3672 return _mm_min_epi8(*this, other);
3673 }
3674 is_utf8_really_inline simd8<bool> operator>(const simd8<int8_t> other) const {
3675 return _mm_cmpgt_epi8(*this, other);
3676 }
3677 is_utf8_really_inline simd8<bool> operator<(const simd8<int8_t> other) const {
3678 return _mm_cmpgt_epi8(other, *this);
3679 }
3680 };
3681
3682 // Unsigned bytes
3683 template <> struct simd8<uint8_t> : base8_numeric<uint8_t> {
3684 is_utf8_really_inline simd8() : base8_numeric<uint8_t>() {}
3685 is_utf8_really_inline simd8(const __m128i _value)
3686 : base8_numeric<uint8_t>(_value) {}
3687
3688 // Splat constructor
3689 is_utf8_really_inline simd8(uint8_t _value) : simd8(splat(_value)) {}
3690 // Array constructor
3691 is_utf8_really_inline simd8(const uint8_t *values) : simd8(load(values)) {}
3692 // Member-by-member initialization
3693 is_utf8_really_inline
3694 simd8(uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5,
3695 uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, uint8_t v10,
3696 uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15)
3697 : simd8(_mm_setr_epi8(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11,
3698 v12, v13, v14, v15)) {}
3699 // Repeat 16 values as many times as necessary (usually for lookup tables)
3700 is_utf8_really_inline static simd8<uint8_t>
3701 repeat_16(uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4,
3702 uint8_t v5, uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9,
3703 uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14,
3704 uint8_t v15) {
3705 return simd8<uint8_t>(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12,
3706 v13, v14, v15);
3707 }
3708
3709 // Saturated math
3710 is_utf8_really_inline simd8<uint8_t>
3711 saturating_add(const simd8<uint8_t> other) const {
3712 return _mm_adds_epu8(*this, other);
3713 }
3714 is_utf8_really_inline simd8<uint8_t>
3715 saturating_sub(const simd8<uint8_t> other) const {
3716 return _mm_subs_epu8(*this, other);
3717 }
3718
3719 // Order-specific operations
3720 is_utf8_really_inline simd8<uint8_t>
3721 max_val(const simd8<uint8_t> other) const {
3722 return _mm_max_epu8(*this, other);
3723 }
3724 is_utf8_really_inline simd8<uint8_t>
3725 min_val(const simd8<uint8_t> other) const {
3726 return _mm_min_epu8(*this, other);
3727 }
3728 // Same as >, but only guarantees true is nonzero (< guarantees true = -1)
3729 is_utf8_really_inline simd8<uint8_t>
3730 gt_bits(const simd8<uint8_t> other) const {
3731 return this->saturating_sub(other);
3732 }
3733 // Same as <, but only guarantees true is nonzero (< guarantees true = -1)
3734 is_utf8_really_inline simd8<uint8_t>
3735 lt_bits(const simd8<uint8_t> other) const {
3736 return other.saturating_sub(*this);
3737 }
3738 is_utf8_really_inline simd8<bool>
3739 operator<=(const simd8<uint8_t> other) const {
3740 return other.max_val(*this) == other;
3741 }
3742 is_utf8_really_inline simd8<bool>
3743 operator>=(const simd8<uint8_t> other) const {
3744 return other.min_val(*this) == other;
3745 }
3746 is_utf8_really_inline simd8<bool>
3747 operator>(const simd8<uint8_t> other) const {
3748 return this->gt_bits(other).any_bits_set();
3749 }
3750 is_utf8_really_inline simd8<bool>
3751 operator<(const simd8<uint8_t> other) const {
3752 return this->gt_bits(other).any_bits_set();
3753 }
3754
3755 // Bit-specific operations
3756 is_utf8_really_inline simd8<bool> bits_not_set() const {
3757 return *this == uint8_t(0);
3758 }
3759 is_utf8_really_inline simd8<bool> bits_not_set(simd8<uint8_t> bits) const {
3760 return (*this & bits).bits_not_set();
3761 }
3762 is_utf8_really_inline simd8<bool> any_bits_set() const {
3763 return ~this->bits_not_set();
3764 }
3765 is_utf8_really_inline simd8<bool> any_bits_set(simd8<uint8_t> bits) const {
3766 return ~this->bits_not_set(bits);
3767 }
3768 is_utf8_really_inline bool is_ascii() const {
3769 return _mm_movemask_epi8(*this) == 0;
3770 }
3771
3772 is_utf8_really_inline bool bits_not_set_anywhere() const {
3773 return _mm_testz_si128(*this, *this);
3774 }
3775 is_utf8_really_inline bool any_bits_set_anywhere() const {
3776 return !bits_not_set_anywhere();
3777 }
3778 is_utf8_really_inline bool bits_not_set_anywhere(simd8<uint8_t> bits) const {
3779 return _mm_testz_si128(*this, bits);
3780 }
3781 is_utf8_really_inline bool any_bits_set_anywhere(simd8<uint8_t> bits) const {
3782 return !bits_not_set_anywhere(bits);
3783 }
3784 template <int N> is_utf8_really_inline simd8<uint8_t> shr() const {
3785 return simd8<uint8_t>(_mm_srli_epi16(*this, N)) & uint8_t(0xFFu >> N);
3786 }
3787 template <int N> is_utf8_really_inline simd8<uint8_t> shl() const {
3788 return simd8<uint8_t>(_mm_slli_epi16(*this, N)) & uint8_t(0xFFu << N);
3789 }
3790 // Get one of the bits and make a bitmask out of it.
3791 // e.g. value.get_bit<7>() gets the high bit
3792 template <int N> is_utf8_really_inline int get_bit() const {
3793 return _mm_movemask_epi8(_mm_slli_epi16(*this, 7 - N));
3794 }
3795 };
3796 is_utf8_really_inline simd8<int8_t>::operator simd8<uint8_t>() const {
3797 return this->value;
3798 }
3799
3800 // Unsigned bytes
3801 template <> struct simd8<uint16_t> : base<uint16_t> {
3802 static is_utf8_really_inline simd8<uint16_t> splat(uint16_t _value) {
3803 return _mm_set1_epi16(_value);
3804 }
3805 static is_utf8_really_inline simd8<uint16_t> load(const uint16_t values[8]) {
3806 return _mm_loadu_si128(reinterpret_cast<const __m128i *>(values));
3807 }
3808
3809 is_utf8_really_inline simd8() : base<uint16_t>() {}
3810 is_utf8_really_inline simd8(const __m128i _value) : base<uint16_t>(_value) {}
3811 // Splat constructor
3812 is_utf8_really_inline simd8(uint16_t _value) : simd8(splat(_value)) {}
3813 // Array constructor
3814 is_utf8_really_inline simd8(const uint16_t *values) : simd8(load(values)) {}
3815 // Member-by-member initialization
3816 is_utf8_really_inline simd8(uint16_t v0, uint16_t v1, uint16_t v2,
3817 uint16_t v3, uint16_t v4, uint16_t v5,
3818 uint16_t v6, uint16_t v7)
3819 : simd8(_mm_setr_epi16(v0, v1, v2, v3, v4, v5, v6, v7)) {}
3820
3821 // Saturated math
3822 is_utf8_really_inline simd8<uint16_t>
3823 saturating_add(const simd8<uint16_t> other) const {
3824 return _mm_adds_epu16(*this, other);
3825 }
3826 is_utf8_really_inline simd8<uint16_t>
3827 saturating_sub(const simd8<uint16_t> other) const {
3828 return _mm_subs_epu16(*this, other);
3829 }
3830
3831 // Order-specific operations
3832 is_utf8_really_inline simd8<uint16_t>
3833 max_val(const simd8<uint16_t> other) const {
3834 return _mm_max_epu16(*this, other);
3835 }
3836 is_utf8_really_inline simd8<uint16_t>
3837 min_val(const simd8<uint16_t> other) const {
3838 return _mm_min_epu16(*this, other);
3839 }
3840 // Same as >, but only guarantees true is nonzero (< guarantees true = -1)
3841 is_utf8_really_inline simd8<uint16_t>
3842 gt_bits(const simd8<uint16_t> other) const {
3843 return this->saturating_sub(other);
3844 }
3845 // Same as <, but only guarantees true is nonzero (< guarantees true = -1)
3846 is_utf8_really_inline simd8<uint16_t>
3847 lt_bits(const simd8<uint16_t> other) const {
3848 return other.saturating_sub(*this);
3849 }
3850 is_utf8_really_inline simd8<bool>
3851 operator<=(const simd8<uint16_t> other) const {
3852 return other.max_val(*this) == other;
3853 }
3854 is_utf8_really_inline simd8<bool>
3855 operator>=(const simd8<uint16_t> other) const {
3856 return other.min_val(*this) == other;
3857 }
3858 is_utf8_really_inline simd8<bool>
3859 operator==(const simd8<uint16_t> other) const {
3860 return _mm_cmpeq_epi16(*this, other);
3861 }
3862 is_utf8_really_inline simd8<bool>
3863 operator&(const simd8<uint16_t> other) const {
3864 return _mm_and_si128(*this, other);
3865 }
3866 is_utf8_really_inline simd8<bool>
3867 operator|(const simd8<uint16_t> other) const {
3868 return _mm_or_si128(*this, other);
3869 }
3870
3871 // Bit-specific operations
3872 is_utf8_really_inline simd8<bool> bits_not_set() const {
3873 return *this == uint16_t(0);
3874 }
3875 is_utf8_really_inline simd8<bool> any_bits_set() const {
3876 return ~this->bits_not_set();
3877 }
3878
3879 is_utf8_really_inline bool bits_not_set_anywhere() const {
3880 return _mm_testz_si128(*this, *this);
3881 }
3882 is_utf8_really_inline bool any_bits_set_anywhere() const {
3883 return !bits_not_set_anywhere();
3884 }
3885 is_utf8_really_inline bool bits_not_set_anywhere(simd8<uint16_t> bits) const {
3886 return _mm_testz_si128(*this, bits);
3887 }
3888 is_utf8_really_inline bool any_bits_set_anywhere(simd8<uint16_t> bits) const {
3889 return !bits_not_set_anywhere(bits);
3890 }
3891 };
3892 template <typename T> struct simd8x64 {
3893 static constexpr int NUM_CHUNKS = 64 / sizeof(simd8<T>);
3894 static_assert(NUM_CHUNKS == 4,
3895 "Westmere kernel should use four registers per 64-byte block.");
3896 simd8<T> chunks[NUM_CHUNKS];
3897
3898 simd8x64(const simd8x64<T> &o) = delete; // no copy allowed
3899 simd8x64<T> &
3900 operator=(const simd8<T> other) = delete; // no assignment allowed
3901 simd8x64() = delete; // no default constructor allowed
3902
3903 is_utf8_really_inline simd8x64(const simd8<T> chunk0, const simd8<T> chunk1,
3904 const simd8<T> chunk2, const simd8<T> chunk3)
3905 : chunks{chunk0, chunk1, chunk2, chunk3} {}
3906 is_utf8_really_inline simd8x64(const T *ptr)
3907 : chunks{simd8<T>::load(ptr),
3908 simd8<T>::load(ptr + sizeof(simd8<T>) / sizeof(T)),
3909 simd8<T>::load(ptr + 2 * sizeof(simd8<T>) / sizeof(T)),
3910 simd8<T>::load(ptr + 3 * sizeof(simd8<T>) / sizeof(T))} {}
3911
3912 is_utf8_really_inline void store(T *ptr) const {
3913 this->chunks[0].store(ptr + sizeof(simd8<T>) * 0 / sizeof(T));
3914 this->chunks[1].store(ptr + sizeof(simd8<T>) * 1 / sizeof(T));
3915 this->chunks[2].store(ptr + sizeof(simd8<T>) * 2 / sizeof(T));
3916 this->chunks[3].store(ptr + sizeof(simd8<T>) * 3 / sizeof(T));
3917 }
3918
3919 is_utf8_really_inline simd8x64<T> &operator|=(const simd8x64<T> &other) {
3920 this->chunks[0] |= other.chunks[0];
3921 this->chunks[1] |= other.chunks[1];
3922 this->chunks[2] |= other.chunks[2];
3923 this->chunks[3] |= other.chunks[3];
3924 return *this;
3925 }
3926
3927 is_utf8_really_inline simd8<T> reduce_or() const {
3928 return (this->chunks[0] | this->chunks[1]) |
3929 (this->chunks[2] | this->chunks[3]);
3930 }
3931
3932 is_utf8_really_inline bool is_ascii() const {
3933 return this->reduce_or().is_ascii();
3934 }
3935
3936 template <endianness endian>
3937 is_utf8_really_inline void store_ascii_as_utf16(char16_t *ptr) const {
3938 this->chunks[0].template store_ascii_as_utf16<endian>(ptr +
3939 sizeof(simd8<T>) * 0);
3940 this->chunks[1].template store_ascii_as_utf16<endian>(ptr +
3941 sizeof(simd8<T>) * 1);
3942 this->chunks[2].template store_ascii_as_utf16<endian>(ptr +
3943 sizeof(simd8<T>) * 2);
3944 this->chunks[3].template store_ascii_as_utf16<endian>(ptr +
3945 sizeof(simd8<T>) * 3);
3946 }
3947
3948 is_utf8_really_inline void store_ascii_as_utf32(char32_t *ptr) const {
3949 this->chunks[0].store_ascii_as_utf32(ptr + sizeof(simd8<T>) * 0);
3950 this->chunks[1].store_ascii_as_utf32(ptr + sizeof(simd8<T>) * 1);
3951 this->chunks[2].store_ascii_as_utf32(ptr + sizeof(simd8<T>) * 2);
3952 this->chunks[3].store_ascii_as_utf32(ptr + sizeof(simd8<T>) * 3);
3953 }
3954
3955 is_utf8_really_inline uint64_t to_bitmask() const {
3956 uint64_t r0 = uint32_t(this->chunks[0].to_bitmask());
3957 uint64_t r1 = this->chunks[1].to_bitmask();
3958 uint64_t r2 = this->chunks[2].to_bitmask();
3959 uint64_t r3 = this->chunks[3].to_bitmask();
3960 return r0 | (r1 << 16) | (r2 << 32) | (r3 << 48);
3961 }
3962
3963 is_utf8_really_inline uint64_t eq(const T m) const {
3964 const simd8<T> mask = simd8<T>::splat(m);
3965 return simd8x64<bool>(this->chunks[0] == mask, this->chunks[1] == mask,
3966 this->chunks[2] == mask, this->chunks[3] == mask)
3967 .to_bitmask();
3968 }
3969
3970 is_utf8_really_inline uint64_t eq(const simd8x64<uint8_t> &other) const {
3971 return simd8x64<bool>(this->chunks[0] == other.chunks[0],
3972 this->chunks[1] == other.chunks[1],
3973 this->chunks[2] == other.chunks[2],
3974 this->chunks[3] == other.chunks[3])
3975 .to_bitmask();
3976 }
3977
3978 is_utf8_really_inline uint64_t lteq(const T m) const {
3979 const simd8<T> mask = simd8<T>::splat(m);
3980 return simd8x64<bool>(this->chunks[0] <= mask, this->chunks[1] <= mask,
3981 this->chunks[2] <= mask, this->chunks[3] <= mask)
3982 .to_bitmask();
3983 }
3984
3985 is_utf8_really_inline uint64_t in_range(const T low, const T high) const {
3986 const simd8<T> mask_low = simd8<T>::splat(low);
3987 const simd8<T> mask_high = simd8<T>::splat(high);
3988
3989 return simd8x64<bool>(
3990 (this->chunks[0] <= mask_high) & (this->chunks[0] >= mask_low),
3991 (this->chunks[1] <= mask_high) & (this->chunks[1] >= mask_low),
3992 (this->chunks[2] <= mask_high) & (this->chunks[2] >= mask_low),
3993 (this->chunks[3] <= mask_high) & (this->chunks[3] >= mask_low))
3994 .to_bitmask();
3995 }
3996 is_utf8_really_inline uint64_t not_in_range(const T low, const T high) const {
3997 const simd8<T> mask_low = simd8<T>::splat(low - 1);
3998 const simd8<T> mask_high = simd8<T>::splat(high + 1);
3999 return simd8x64<bool>(
4000 (this->chunks[0] >= mask_high) | (this->chunks[0] <= mask_low),
4001 (this->chunks[1] >= mask_high) | (this->chunks[1] <= mask_low),
4002 (this->chunks[2] >= mask_high) | (this->chunks[2] <= mask_low),
4003 (this->chunks[3] >= mask_high) | (this->chunks[3] <= mask_low))
4004 .to_bitmask();
4005 }
4006 is_utf8_really_inline uint64_t lt(const T m) const {
4007 const simd8<T> mask = simd8<T>::splat(m);
4008 return simd8x64<bool>(this->chunks[0] < mask, this->chunks[1] < mask,
4009 this->chunks[2] < mask, this->chunks[3] < mask)
4010 .to_bitmask();
4011 }
4012
4013 is_utf8_really_inline uint64_t gt(const T m) const {
4014 const simd8<T> mask = simd8<T>::splat(m);
4015 return simd8x64<bool>(this->chunks[0] > mask, this->chunks[1] > mask,
4016 this->chunks[2] > mask, this->chunks[3] > mask)
4017 .to_bitmask();
4018 }
4019 is_utf8_really_inline uint64_t gteq(const T m) const {
4020 const simd8<T> mask = simd8<T>::splat(m);
4021 return simd8x64<bool>(this->chunks[0] >= mask, this->chunks[1] >= mask,
4022 this->chunks[2] >= mask, this->chunks[3] >= mask)
4023 .to_bitmask();
4024 }
4025 is_utf8_really_inline uint64_t gteq_unsigned(const uint8_t m) const {
4026 const simd8<uint8_t> mask = simd8<uint8_t>::splat(m);
4027 return simd8x64<bool>(simd8<uint8_t>(__m128i(this->chunks[0])) >= mask,
4028 simd8<uint8_t>(__m128i(this->chunks[1])) >= mask,
4029 simd8<uint8_t>(__m128i(this->chunks[2])) >= mask,
4030 simd8<uint8_t>(__m128i(this->chunks[3])) >= mask)
4031 .to_bitmask();
4032 }
4033 }; // struct simd8x64<T>
4034
4035 template <typename T> struct simd16;
4036
4037 template <typename T, typename Mask = simd16<bool>>
4038 struct base16 : base<simd16<T>> {
4039 typedef uint16_t bitmask_t;
4040 typedef uint32_t bitmask2_t;
4041
4042 is_utf8_really_inline base16() : base<simd16<T>>() {}
4043 is_utf8_really_inline base16(const __m128i _value)
4044 : base<simd16<T>>(_value) {}
4045 template <typename Pointer>
4046 is_utf8_really_inline base16(const Pointer *ptr)
4047 : base16(_mm_loadu_si128(reinterpret_cast<const __m128i *>(ptr))) {}
4048
4049 friend is_utf8_really_inline Mask operator==(const simd16<T> lhs,
4050 const simd16<T> rhs) {
4051 return _mm_cmpeq_epi16(lhs, rhs);
4052 }
4053
4054 static const int SIZE = sizeof(base<simd16<T>>::value);
4055
4056 template <int N = 1>
4057 is_utf8_really_inline simd16<T> prev(const simd16<T> prev_chunk) const {
4058 return _mm_alignr_epi8(*this, prev_chunk, 16 - N);
4059 }
4060 };
4061
4062 // SIMD byte mask type (returned by things like eq and gt)
4063 template <> struct simd16<bool> : base16<bool> {
4064 static is_utf8_really_inline simd16<bool> splat(bool _value) {
4065 return _mm_set1_epi16(uint16_t(-(!!_value)));
4066 }
4067
4068 is_utf8_really_inline simd16() : base16() {}
4069 is_utf8_really_inline simd16(const __m128i _value)
4070 : base16<bool>(_value) {}
4071 // Splat constructor
4072 is_utf8_really_inline simd16(bool _value)
4073 : base16<bool>(splat(_value)) {}
4074
4075 is_utf8_really_inline int to_bitmask() const {
4076 return _mm_movemask_epi8(*this);
4077 }
4078 is_utf8_really_inline bool any() const {
4079 return !_mm_testz_si128(*this, *this);
4080 }
4081 is_utf8_really_inline simd16<bool> operator~() const { return *this ^ true; }
4082 };
4083
4084 template <typename T> struct base16_numeric : base16<T> {
4085 static is_utf8_really_inline simd16<T> splat(T _value) {
4086 return _mm_set1_epi16(_value);
4087 }
4088 static is_utf8_really_inline simd16<T> zero() { return _mm_setzero_si128(); }
4089 static is_utf8_really_inline simd16<T> load(const T values[8]) {
4090 return _mm_loadu_si128(reinterpret_cast<const __m128i *>(values));
4091 }
4092
4093 is_utf8_really_inline base16_numeric() : base16<T>() {}
4094 is_utf8_really_inline base16_numeric(const __m128i _value)
4095 : base16<T>(_value) {}
4096
4097 // Store to array
4098 is_utf8_really_inline void store(T dst[8]) const {
4099 return _mm_storeu_si128(reinterpret_cast<__m128i *>(dst), *this);
4100 }
4101
4102 // Override to distinguish from bool version
4103 is_utf8_really_inline simd16<T> operator~() const { return *this ^ 0xFFu; }
4104
4105 // Addition/subtraction are the same for signed and unsigned
4106 is_utf8_really_inline simd16<T> operator+(const simd16<T> other) const {
4107 return _mm_add_epi16(*this, other);
4108 }
4109 is_utf8_really_inline simd16<T> operator-(const simd16<T> other) const {
4110 return _mm_sub_epi16(*this, other);
4111 }
4112 is_utf8_really_inline simd16<T> &operator+=(const simd16<T> other) {
4113 *this = *this + other;
4114 return *static_cast<simd16<T> *>(this);
4115 }
4116 is_utf8_really_inline simd16<T> &operator-=(const simd16<T> other) {
4117 *this = *this - other;
4118 return *static_cast<simd16<T> *>(this);
4119 }
4120 };
4121
4122 // Signed words
4123 template <> struct simd16<int16_t> : base16_numeric<int16_t> {
4124 is_utf8_really_inline simd16() : base16_numeric<int16_t>() {}
4125 is_utf8_really_inline simd16(const __m128i _value)
4126 : base16_numeric<int16_t>(_value) {}
4127 // Splat constructor
4128 is_utf8_really_inline simd16(int16_t _value) : simd16(splat(_value)) {}
4129 // Array constructor
4130 is_utf8_really_inline simd16(const int16_t *values) : simd16(load(values)) {}
4131 is_utf8_really_inline simd16(const char16_t *values)
4132 : simd16(load(reinterpret_cast<const int16_t *>(values))) {}
4133 // Member-by-member initialization
4134 is_utf8_really_inline simd16(int16_t v0, int16_t v1, int16_t v2, int16_t v3,
4135 int16_t v4, int16_t v5, int16_t v6, int16_t v7)
4136 : simd16(_mm_setr_epi16(v0, v1, v2, v3, v4, v5, v6, v7)) {}
4137 is_utf8_really_inline operator simd16<uint16_t>() const;
4138
4139 // Order-sensitive comparisons
4140 is_utf8_really_inline simd16<int16_t>
4141 max_val(const simd16<int16_t> other) const {
4142 return _mm_max_epi16(*this, other);
4143 }
4144 is_utf8_really_inline simd16<int16_t>
4145 min_val(const simd16<int16_t> other) const {
4146 return _mm_min_epi16(*this, other);
4147 }
4148 is_utf8_really_inline simd16<bool>
4149 operator>(const simd16<int16_t> other) const {
4150 return _mm_cmpgt_epi16(*this, other);
4151 }
4152 is_utf8_really_inline simd16<bool>
4153 operator<(const simd16<int16_t> other) const {
4154 return _mm_cmpgt_epi16(other, *this);
4155 }
4156 };
4157
4158 // Unsigned words
4159 template <> struct simd16<uint16_t> : base16_numeric<uint16_t> {
4160 is_utf8_really_inline simd16() : base16_numeric<uint16_t>() {}
4161 is_utf8_really_inline simd16(const __m128i _value)
4162 : base16_numeric<uint16_t>(_value) {}
4163
4164 // Splat constructor
4165 is_utf8_really_inline simd16(uint16_t _value) : simd16(splat(_value)) {}
4166 // Array constructor
4167 is_utf8_really_inline simd16(const uint16_t *values) : simd16(load(values)) {}
4168 is_utf8_really_inline simd16(const char16_t *values)
4169 : simd16(load(reinterpret_cast<const uint16_t *>(values))) {}
4170 // Member-by-member initialization
4171 is_utf8_really_inline simd16(uint16_t v0, uint16_t v1, uint16_t v2,
4172 uint16_t v3, uint16_t v4, uint16_t v5,
4173 uint16_t v6, uint16_t v7)
4174 : simd16(_mm_setr_epi16(v0, v1, v2, v3, v4, v5, v6, v7)) {}
4175 // Repeat 16 values as many times as necessary (usually for lookup tables)
4176 is_utf8_really_inline static simd16<uint16_t>
4177 repeat_16(uint16_t v0, uint16_t v1, uint16_t v2, uint16_t v3, uint16_t v4,
4178 uint16_t v5, uint16_t v6, uint16_t v7) {
4179 return simd16<uint16_t>(v0, v1, v2, v3, v4, v5, v6, v7);
4180 }
4181
4182 // Saturated math
4183 is_utf8_really_inline simd16<uint16_t>
4184 saturating_add(const simd16<uint16_t> other) const {
4185 return _mm_adds_epu16(*this, other);
4186 }
4187 is_utf8_really_inline simd16<uint16_t>
4188 saturating_sub(const simd16<uint16_t> other) const {
4189 return _mm_subs_epu16(*this, other);
4190 }
4191
4192 // Order-specific operations
4193 is_utf8_really_inline simd16<uint16_t>
4194 max_val(const simd16<uint16_t> other) const {
4195 return _mm_max_epu16(*this, other);
4196 }
4197 is_utf8_really_inline simd16<uint16_t>
4198 min_val(const simd16<uint16_t> other) const {
4199 return _mm_min_epu16(*this, other);
4200 }
4201 // Same as >, but only guarantees true is nonzero (< guarantees true = -1)
4202 is_utf8_really_inline simd16<uint16_t>
4203 gt_bits(const simd16<uint16_t> other) const {
4204 return this->saturating_sub(other);
4205 }
4206 // Same as <, but only guarantees true is nonzero (< guarantees true = -1)
4207 is_utf8_really_inline simd16<uint16_t>
4208 lt_bits(const simd16<uint16_t> other) const {
4209 return other.saturating_sub(*this);
4210 }
4211 is_utf8_really_inline simd16<bool>
4212 operator<=(const simd16<uint16_t> other) const {
4213 return other.max_val(*this) == other;
4214 }
4215 is_utf8_really_inline simd16<bool>
4216 operator>=(const simd16<uint16_t> other) const {
4217 return other.min_val(*this) == other;
4218 }
4219 is_utf8_really_inline simd16<bool>
4220 operator>(const simd16<uint16_t> other) const {
4221 return this->gt_bits(other).any_bits_set();
4222 }
4223 is_utf8_really_inline simd16<bool>
4224 operator<(const simd16<uint16_t> other) const {
4225 return this->gt_bits(other).any_bits_set();
4226 }
4227
4228 // Bit-specific operations
4229 is_utf8_really_inline simd16<bool> bits_not_set() const {
4230 return *this == uint16_t(0);
4231 }
4232 is_utf8_really_inline simd16<bool> bits_not_set(simd16<uint16_t> bits) const {
4233 return (*this & bits).bits_not_set();
4234 }
4235 is_utf8_really_inline simd16<bool> any_bits_set() const {
4236 return ~this->bits_not_set();
4237 }
4238 is_utf8_really_inline simd16<bool> any_bits_set(simd16<uint16_t> bits) const {
4239 return ~this->bits_not_set(bits);
4240 }
4241
4242 is_utf8_really_inline bool bits_not_set_anywhere() const {
4243 return _mm_testz_si128(*this, *this);
4244 }
4245 is_utf8_really_inline bool any_bits_set_anywhere() const {
4246 return !bits_not_set_anywhere();
4247 }
4248 is_utf8_really_inline bool
4249 bits_not_set_anywhere(simd16<uint16_t> bits) const {
4250 return _mm_testz_si128(*this, bits);
4251 }
4252 is_utf8_really_inline bool
4253 any_bits_set_anywhere(simd16<uint16_t> bits) const {
4254 return !bits_not_set_anywhere(bits);
4255 }
4256 template <int N> is_utf8_really_inline simd16<uint16_t> shr() const {
4257 return simd16<uint16_t>(_mm_srli_epi16(*this, N));
4258 }
4259 template <int N> is_utf8_really_inline simd16<uint16_t> shl() const {
4260 return simd16<uint16_t>(_mm_slli_epi16(*this, N));
4261 }
4262 // Get one of the bits and make a bitmask out of it.
4263 // e.g. value.get_bit<7>() gets the high bit
4264 template <int N> is_utf8_really_inline int get_bit() const {
4265 return _mm_movemask_epi8(_mm_slli_epi16(*this, 7 - N));
4266 }
4267
4268 // Change the endianness
4269 is_utf8_really_inline simd16<uint16_t> swap_bytes() const {
4270 const __m128i swap =
4271 _mm_setr_epi8(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14);
4272 return _mm_shuffle_epi8(*this, swap);
4273 }
4274
4275 // Pack with the unsigned saturation two uint16_t words into single uint8_t
4276 // vector
4277 static is_utf8_really_inline simd8<uint8_t> pack(const simd16<uint16_t> &v0,
4278 const simd16<uint16_t> &v1) {
4279 return _mm_packus_epi16(v0, v1);
4280 }
4281 };
4282 is_utf8_really_inline simd16<int16_t>::operator simd16<uint16_t>() const {
4283 return this->value;
4284 }
4285
4286 template <typename T> struct simd16x32 {
4287 static constexpr int NUM_CHUNKS = 64 / sizeof(simd16<T>);
4288 static_assert(NUM_CHUNKS == 4,
4289 "Westmere kernel should use four registers per 64-byte block.");
4290 simd16<T> chunks[NUM_CHUNKS];
4291
4292 simd16x32(const simd16x32<T> &o) = delete; // no copy allowed
4293 simd16x32<T> &
4294 operator=(const simd16<T> other) = delete; // no assignment allowed
4295 simd16x32() = delete; // no default constructor allowed
4296
4297 is_utf8_really_inline
4298 simd16x32(const simd16<T> chunk0, const simd16<T> chunk1,
4299 const simd16<T> chunk2, const simd16<T> chunk3)
4300 : chunks{chunk0, chunk1, chunk2, chunk3} {}
4301 is_utf8_really_inline simd16x32(const T *ptr)
4302 : chunks{simd16<T>::load(ptr),
4303 simd16<T>::load(ptr + sizeof(simd16<T>) / sizeof(T)),
4304 simd16<T>::load(ptr + 2 * sizeof(simd16<T>) / sizeof(T)),
4305 simd16<T>::load(ptr + 3 * sizeof(simd16<T>) / sizeof(T))} {}
4306
4307 is_utf8_really_inline void store(T *ptr) const {
4308 this->chunks[0].store(ptr + sizeof(simd16<T>) * 0 / sizeof(T));
4309 this->chunks[1].store(ptr + sizeof(simd16<T>) * 1 / sizeof(T));
4310 this->chunks[2].store(ptr + sizeof(simd16<T>) * 2 / sizeof(T));
4311 this->chunks[3].store(ptr + sizeof(simd16<T>) * 3 / sizeof(T));
4312 }
4313
4314 is_utf8_really_inline simd16<T> reduce_or() const {
4315 return (this->chunks[0] | this->chunks[1]) |
4316 (this->chunks[2] | this->chunks[3]);
4317 }
4318
4319 is_utf8_really_inline bool is_ascii() const {
4320 return this->reduce_or().is_ascii();
4321 }
4322
4323 is_utf8_really_inline void store_ascii_as_utf16(char16_t *ptr) const {
4324 this->chunks[0].store_ascii_as_utf16(ptr + sizeof(simd16<T>) * 0);
4325 this->chunks[1].store_ascii_as_utf16(ptr + sizeof(simd16<T>) * 1);
4326 this->chunks[2].store_ascii_as_utf16(ptr + sizeof(simd16<T>) * 2);
4327 this->chunks[3].store_ascii_as_utf16(ptr + sizeof(simd16<T>) * 3);
4328 }
4329
4330 is_utf8_really_inline uint64_t to_bitmask() const {
4331 uint64_t r0 = uint32_t(this->chunks[0].to_bitmask());
4332 uint64_t r1 = this->chunks[1].to_bitmask();
4333 uint64_t r2 = this->chunks[2].to_bitmask();
4334 uint64_t r3 = this->chunks[3].to_bitmask();
4335 return r0 | (r1 << 16) | (r2 << 32) | (r3 << 48);
4336 }
4337
4338 is_utf8_really_inline void swap_bytes() {
4339 this->chunks[0] = this->chunks[0].swap_bytes();
4340 this->chunks[1] = this->chunks[1].swap_bytes();
4341 this->chunks[2] = this->chunks[2].swap_bytes();
4342 this->chunks[3] = this->chunks[3].swap_bytes();
4343 }
4344
4345 is_utf8_really_inline uint64_t eq(const T m) const {
4346 const simd16<T> mask = simd16<T>::splat(m);
4347 return simd16x32<bool>(this->chunks[0] == mask, this->chunks[1] == mask,
4348 this->chunks[2] == mask, this->chunks[3] == mask)
4349 .to_bitmask();
4350 }
4351
4352 is_utf8_really_inline uint64_t eq(const simd16x32<uint16_t> &other) const {
4353 return simd16x32<bool>(this->chunks[0] == other.chunks[0],
4354 this->chunks[1] == other.chunks[1],
4355 this->chunks[2] == other.chunks[2],
4356 this->chunks[3] == other.chunks[3])
4357 .to_bitmask();
4358 }
4359
4360 is_utf8_really_inline uint64_t lteq(const T m) const {
4361 const simd16<T> mask = simd16<T>::splat(m);
4362 return simd16x32<bool>(this->chunks[0] <= mask, this->chunks[1] <= mask,
4363 this->chunks[2] <= mask, this->chunks[3] <= mask)
4364 .to_bitmask();
4365 }
4366
4367 is_utf8_really_inline uint64_t in_range(const T low, const T high) const {
4368 const simd16<T> mask_low = simd16<T>::splat(low);
4369 const simd16<T> mask_high = simd16<T>::splat(high);
4370
4371 return simd16x32<bool>(
4372 (this->chunks[0] <= mask_high) & (this->chunks[0] >= mask_low),
4373 (this->chunks[1] <= mask_high) & (this->chunks[1] >= mask_low),
4374 (this->chunks[2] <= mask_high) & (this->chunks[2] >= mask_low),
4375 (this->chunks[3] <= mask_high) & (this->chunks[3] >= mask_low))
4376 .to_bitmask();
4377 }
4378 is_utf8_really_inline uint64_t not_in_range(const T low, const T high) const {
4379 const simd16<T> mask_low = simd16<T>::splat(static_cast<T>(low - 1));
4380 const simd16<T> mask_high = simd16<T>::splat(static_cast<T>(high + 1));
4381 return simd16x32<bool>(
4382 (this->chunks[0] >= mask_high) | (this->chunks[0] <= mask_low),
4383 (this->chunks[1] >= mask_high) | (this->chunks[1] <= mask_low),
4384 (this->chunks[2] >= mask_high) | (this->chunks[2] <= mask_low),
4385 (this->chunks[3] >= mask_high) | (this->chunks[3] <= mask_low))
4386 .to_bitmask();
4387 }
4388 is_utf8_really_inline uint64_t lt(const T m) const {
4389 const simd16<T> mask = simd16<T>::splat(m);
4390 return simd16x32<bool>(this->chunks[0] < mask, this->chunks[1] < mask,
4391 this->chunks[2] < mask, this->chunks[3] < mask)
4392 .to_bitmask();
4393 }
4394 }; // struct simd16x32<T>
4395
4396 } // namespace simd
4397 } // unnamed namespace
4398 } // namespace westmere
4399 } // namespace is_utf8_internals
4400
4401 #endif // IS_UTF8_WESTMERE_SIMD_INPUT_H
4402
4403 IS_UTF8_UNTARGET_REGION
4404
4405 #endif // IS_UTF8_IMPLEMENTATION_WESTMERE
4406 #endif // IS_UTF8_WESTMERE_COMMON_H
4407
4408 #ifndef IS_UTF8_FALLBACK_H
4409 #define IS_UTF8_FALLBACK_H
4410
4411 // Note that fallback.h is always imported last.
4412
4413 // Default Fallback to on unless a builtin implementation has already been
4414 // selected.
4415 #ifndef IS_UTF8_IMPLEMENTATION_FALLBACK
4416 #if IS_UTF8_CAN_ALWAYS_RUN_ARM64 || IS_UTF8_CAN_ALWAYS_RUN_ICELAKE || \
4417 IS_UTF8_CAN_ALWAYS_RUN_HASWELL || IS_UTF8_CAN_ALWAYS_RUN_WESTMERE || \
4418 IS_UTF8_CAN_ALWAYS_RUN_PPC64
4419 #define IS_UTF8_IMPLEMENTATION_FALLBACK 0
4420 #else
4421 #define IS_UTF8_IMPLEMENTATION_FALLBACK 1
4422 #endif
4423 #endif
4424
4425 #if IS_UTF8_IMPLEMENTATION_FALLBACK
4426 #define IS_UTF8_CAN_ALWAYS_RUN_FALLBACK 1
4427 #else
4428 #define IS_UTF8_CAN_ALWAYS_RUN_FALLBACK 0
4429 #endif
4430
4431 #if IS_UTF8_IMPLEMENTATION_FALLBACK
4432
4433 namespace is_utf8_internals {
4434 /**
4435 * Fallback implementation (runs on any machine).
4436 */
4437 namespace fallback {} // namespace fallback
4438 } // namespace is_utf8_internals
4439
4440 #ifndef IS_UTF8_FALLBACK_IMPLEMENTATION_H
4441 #define IS_UTF8_FALLBACK_IMPLEMENTATION_H
4442
4443 namespace is_utf8_internals {
4444 namespace fallback {
4445
4446 class implementation final : public is_utf8_internals::implementation {
4447 public:
4448 is_utf8_really_inline implementation()
4449 : is_utf8_internals::implementation(
4450 "fallback", "Generic fallback implementation", 0) {}
4451 is_utf8_warn_unused bool validate_utf8(const char *buf,
4452 size_t len) const noexcept final;
4453 };
4454
4455 } // namespace fallback
4456 } // namespace is_utf8_internals
4457
4458 #endif // IS_UTF8_FALLBACK_IMPLEMENTATION_H
4459
4460 // redefining IS_UTF8_IMPLEMENTATION to "fallback"
4461 // #define IS_UTF8_IMPLEMENTATION fallback
4462
4463 // Declarations
4464
4465 #ifndef IS_UTF8_FALLBACK_BITMANIPULATION_H
4466 #define IS_UTF8_FALLBACK_BITMANIPULATION_H
4467
4468 #include <limits>
4469
4470 namespace is_utf8_internals {
4471 namespace fallback {
4472 namespace {
4473
4474 #if defined(_MSC_VER) && !defined(_M_ARM64) && !defined(_M_X64)
4475 static inline unsigned char _BitScanForward64(unsigned long *ret, uint64_t x) {
4476 unsigned long x0 = (unsigned long)x, top, bottom;
4477 _BitScanForward(&top, (unsigned long)(x >> 32));
4478 _BitScanForward(&bottom, x0);
4479 *ret = x0 ? bottom : 32 + top;
4480 return x != 0;
4481 }
4482 static unsigned char _BitScanReverse64(unsigned long *ret, uint64_t x) {
4483 unsigned long x1 = (unsigned long)(x >> 32), top, bottom;
4484 _BitScanReverse(&top, x1);
4485 _BitScanReverse(&bottom, (unsigned long)x);
4486 *ret = x1 ? top + 32 : bottom;
4487 return x != 0;
4488 }
4489 #endif
4490
4491 } // unnamed namespace
4492 } // namespace fallback
4493 } // namespace is_utf8_internals
4494
4495 #endif // IS_UTF8_FALLBACK_BITMANIPULATION_H
4496
4497 #endif // IS_UTF8_IMPLEMENTATION_FALLBACK
4498 #endif // IS_UTF8_FALLBACK_H
4499
4500 namespace is_utf8_internals {
4501 bool implementation::supported_by_runtime_system() const {
4502 uint32_t required_instruction_sets = this->required_instruction_sets();
4503 uint32_t supported_instruction_sets =
4504 internal::detect_supported_architectures();
4505 return ((supported_instruction_sets & required_instruction_sets) ==
4506 required_instruction_sets);
4507 }
4508
4509 namespace internal {
4510
4511 // Static array of known implementations. We're hoping these get baked into the
4512 // executable without requiring a static initializer.
4513
4514 #if IS_UTF8_IMPLEMENTATION_ICELAKE
4515 const icelake::implementation icelake_singleton{};
4516 #endif
4517 #if IS_UTF8_IMPLEMENTATION_HASWELL
4518 const haswell::implementation haswell_singleton{};
4519 #endif
4520 #if IS_UTF8_IMPLEMENTATION_WESTMERE
4521 const westmere::implementation westmere_singleton{};
4522 #endif
4523 #if IS_UTF8_IMPLEMENTATION_ARM64
4524 const arm64::implementation arm64_singleton{};
4525 #endif
4526 #if IS_UTF8_IMPLEMENTATION_PPC64
4527 const ppc64::implementation ppc64_singleton{};
4528 #endif
4529 #if IS_UTF8_IMPLEMENTATION_FALLBACK
4530 const fallback::implementation fallback_singleton{};
4531 #endif
4532
4533 /**
4534 * @private Detects best supported implementation on first use, and sets it
4535 */
4536 class detect_best_supported_implementation_on_first_use final
4537 : public implementation {
4538 public:
4539 const std::string &name() const noexcept final { return set_best()->name(); }
4540 const std::string &description() const noexcept final {
4541 return set_best()->description();
4542 }
4543 uint32_t required_instruction_sets() const noexcept final {
4544 return set_best()->required_instruction_sets();
4545 }
4546
4547 is_utf8_warn_unused bool
4548 validate_utf8(const char *buf, size_t len) const noexcept final override {
4549 return set_best()->validate_utf8(buf, len);
4550 }
4551
4552 is_utf8_really_inline
4553 detect_best_supported_implementation_on_first_use() noexcept
4554 : implementation("best_supported_detector",
4555 "Detects the best supported implementation and sets it",
4556 0) {}
4557
4558 private:
4559 const implementation *set_best() const noexcept;
4560 };
4561
4562 const detect_best_supported_implementation_on_first_use
4563 detect_best_supported_implementation_on_first_use_singleton;
4564
4565 const std::initializer_list<const implementation *>
4566 available_implementation_pointers {
4567 #if IS_UTF8_IMPLEMENTATION_ICELAKE
4568 &icelake_singleton,
4569 #endif
4570 #if IS_UTF8_IMPLEMENTATION_HASWELL
4571 &haswell_singleton,
4572 #endif
4573 #if IS_UTF8_IMPLEMENTATION_WESTMERE
4574 &westmere_singleton,
4575 #endif
4576 #if IS_UTF8_IMPLEMENTATION_ARM64
4577 &arm64_singleton,
4578 #endif
4579 #if IS_UTF8_IMPLEMENTATION_PPC64
4580 &ppc64_singleton,
4581 #endif
4582 #if IS_UTF8_IMPLEMENTATION_FALLBACK
4583 &fallback_singleton,
4584 #endif
4585 }; // available_implementation_pointers
4586
4587 // So we can return UNSUPPORTED_ARCHITECTURE from the parser when there is no
4588 // support
4589 class unsupported_implementation final : public implementation {
4590 public:
4591 is_utf8_warn_unused bool validate_utf8(const char *,
4592 size_t) const noexcept final override {
4593 return false; // Just refuse to validate. Given that we have a fallback
4594 // implementation
4595 // it seems unlikely that unsupported_implementation will ever be used. If
4596 // it is used, then it will flag all strings as invalid. The alternative is
4597 // to return an error_code from which the user has to figure out whether the
4598 // string is valid UTF-8... which seems like a lot of work just to handle
4599 // the very unlikely case that we have an unsupported implementation. And,
4600 // when it does happen (that we have an unsupported implementation), what
4601 // are the chances that the programmer has a fallback? Given that *we*
4602 // provide the fallback, it implies that the programmer would need a
4603 // fallback for our fallback.
4604 }
4605
4606 unsupported_implementation()
4607 : implementation("unsupported",
4608 "Unsupported CPU (no detected SIMD instructions)", 0) {}
4609 };
4610
4611 const unsupported_implementation unsupported_singleton{};
4612
4613 size_t available_implementation_list::size() const noexcept {
4614 return internal::available_implementation_pointers.size();
4615 }
4616 const implementation *const *
4617 available_implementation_list::begin() const noexcept {
4618 return internal::available_implementation_pointers.begin();
4619 }
4620 const implementation *const *
4621 available_implementation_list::end() const noexcept {
4622 return internal::available_implementation_pointers.end();
4623 }
4624 const implementation *
4625 available_implementation_list::detect_best_supported() const noexcept {
4626 // They are prelisted in priority order, so we just go down the list
4627 uint32_t supported_instruction_sets =
4628 internal::detect_supported_architectures();
4629 for (const implementation *impl :
4630 internal::available_implementation_pointers) {
4631 uint32_t required_instruction_sets = impl->required_instruction_sets();
4632 if ((supported_instruction_sets & required_instruction_sets) ==
4633 required_instruction_sets) {
4634 return impl;
4635 }
4636 }
4637 return &unsupported_singleton; // this should never happen?
4638 }
4639
4640 const implementation *
4641 detect_best_supported_implementation_on_first_use::set_best() const noexcept {
4642 IS_UTF8_PUSH_DISABLE_WARNINGS
4643 IS_UTF8_DISABLE_DEPRECATED_WARNING // Disable CRT_SECURE warning on MSVC:
4644 // manually verified this is safe
4645 char *force_implementation_name = getenv("IS_UTF8_FORCE_IMPLEMENTATION");
4646 IS_UTF8_POP_DISABLE_WARNINGS
4647
4648 if (force_implementation_name) {
4649 auto force_implementation =
4650 get_available_implementations()[force_implementation_name];
4651 if (force_implementation) {
4652 return get_active_implementation() = force_implementation;
4653 } else {
4654 // Note: abort() and stderr usage within the library is forbidden.
4655 return get_active_implementation() = &unsupported_singleton;
4656 }
4657 }
4658 return get_active_implementation() =
4659 get_available_implementations().detect_best_supported();
4660 }
4661
4662 } // namespace internal
4663
4664 IS_UTF8_DLLIMPORTEXPORT const internal::available_implementation_list& get_available_implementations() {
4665 static const internal::available_implementation_list available_implementations{};
4666 return available_implementations;
4667 }
4668
4669 IS_UTF8_DLLIMPORTEXPORT internal::atomic_ptr<const implementation>& get_active_implementation() {
4670 static const internal::detect_best_supported_implementation_on_first_use detect_best_supported_implementation_on_first_use_singleton;
4671 static internal::atomic_ptr<const implementation> active_implementation{&detect_best_supported_implementation_on_first_use_singleton};
4672 return active_implementation;
4673 }
4674
4675
4676 is_utf8_warn_unused bool validate_utf8(const char *buf, size_t len) noexcept {
4677 return get_active_implementation()->validate_utf8(buf, len);
4678 }
4679
4680 const implementation *builtin_implementation() {
4681 static const implementation *builtin_impl =
4682 get_available_implementations()[IS_UTF8_STRINGIFY(
4683 IS_UTF8_BUILTIN_IMPLEMENTATION)];
4684 return builtin_impl;
4685 }
4686
4687 } // namespace is_utf8_internals
4688
4689 // The scalar routines should be included once.
4690
4691 #ifndef IS_UTF8_UTF8_H
4692 #define IS_UTF8_UTF8_H
4693
4694 namespace is_utf8_internals {
4695 namespace scalar {
4696 namespace {
4697 namespace utf8 {
4698 #if IS_UTF8_IS_ARM64
4699 // not needed
4700 #else
4701 // credit: based on code from Google Fuchsia (Apache Licensed)
4702 inline is_utf8_warn_unused bool validate(const char *buf, size_t len) noexcept {
4703 const uint8_t *data = reinterpret_cast<const uint8_t *>(buf);
4704 uint64_t pos = 0;
4705 uint32_t code_point = 0;
4706 while (pos < len) {
4707 // check of the next 8 bytes are ascii.
4708 uint64_t next_pos = pos + 16;
4709 if (next_pos <=
4710 len) { // if it is safe to read 8 more bytes, check that they are ascii
4711 uint64_t v1;
4712 std::memcpy(&v1, data + pos, sizeof(uint64_t));
4713 uint64_t v2;
4714 std::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t));
4715 uint64_t v{v1 | v2};
4716 if ((v & 0x8080808080808080) == 0) {
4717 pos = next_pos;
4718 continue;
4719 }
4720 }
4721 unsigned char byte = data[pos];
4722
4723 while (byte < 0b10000000) {
4724 if (++pos == len) {
4725 return true;
4726 }
4727 byte = data[pos];
4728 }
4729
4730 if ((byte & 0b11100000) == 0b11000000) {
4731 next_pos = pos + 2;
4732 if (next_pos > len) {
4733 return false;
4734 }
4735 if ((data[pos + 1] & 0b11000000) != 0b10000000) {
4736 return false;
4737 }
4738 // range check
4739 code_point = (byte & 0b00011111) << 6 | (data[pos + 1] & 0b00111111);
4740 if ((code_point < 0x80) || (0x7ff < code_point)) {
4741 return false;
4742 }
4743 } else if ((byte & 0b11110000) == 0b11100000) {
4744 next_pos = pos + 3;
4745 if (next_pos > len) {
4746 return false;
4747 }
4748 if ((data[pos + 1] & 0b11000000) != 0b10000000) {
4749 return false;
4750 }
4751 if ((data[pos + 2] & 0b11000000) != 0b10000000) {
4752 return false;
4753 }
4754 // range check
4755 code_point = (byte & 0b00001111) << 12 |
4756 (data[pos + 1] & 0b00111111) << 6 |
4757 (data[pos + 2] & 0b00111111);
4758 if ((code_point < 0x800) || (0xffff < code_point) ||
4759 (0xd7ff < code_point && code_point < 0xe000)) {
4760 return false;
4761 }
4762 } else if ((byte & 0b11111000) == 0b11110000) { // 0b11110000
4763 next_pos = pos + 4;
4764 if (next_pos > len) {
4765 return false;
4766 }
4767 if ((data[pos + 1] & 0b11000000) != 0b10000000) {
4768 return false;
4769 }
4770 if ((data[pos + 2] & 0b11000000) != 0b10000000) {
4771 return false;
4772 }
4773 if ((data[pos + 3] & 0b11000000) != 0b10000000) {
4774 return false;
4775 }
4776 // range check
4777 code_point =
4778 (byte & 0b00000111) << 18 | (data[pos + 1] & 0b00111111) << 12 |
4779 (data[pos + 2] & 0b00111111) << 6 | (data[pos + 3] & 0b00111111);
4780 if (code_point <= 0xffff || 0x10ffff < code_point) {
4781 return false;
4782 }
4783 } else {
4784 // we may have a continuation
4785 return false;
4786 }
4787 pos = next_pos;
4788 }
4789 return true;
4790 }
4791 #endif
4792 } // namespace utf8
4793 } // unnamed namespace
4794 } // namespace scalar
4795 } // namespace is_utf8_internals
4796
4797 #endif
4798
4799 IS_UTF8_PUSH_DISABLE_WARNINGS
4800 IS_UTF8_DISABLE_UNDESIRED_WARNINGS
4801
4802 #if IS_UTF8_IMPLEMENTATION_ARM64
4803
4804 // redefining IS_UTF8_IMPLEMENTATION to "arm64"
4805 // #define IS_UTF8_IMPLEMENTATION arm64
4806
4807 namespace is_utf8_internals {
4808 namespace arm64 {
4809 namespace {
4810 #ifndef IS_UTF8_ARM64_H
4811 #error "arm64.h must be included"
4812 #endif
4813 using namespace simd;
4814
4815 is_utf8_really_inline bool is_ascii(const simd8x64<uint8_t> &input) {
4816 simd8<uint8_t> bits = input.reduce_or();
4817 return bits.max_val() < 0b10000000u;
4818 }
4819
4820 is_utf8_unused is_utf8_really_inline simd8<bool>
4821 must_be_continuation(const simd8<uint8_t> prev1, const simd8<uint8_t> prev2,
4822 const simd8<uint8_t> prev3) {
4823 simd8<bool> is_second_byte = prev1 >= uint8_t(0b11000000u);
4824 simd8<bool> is_third_byte = prev2 >= uint8_t(0b11100000u);
4825 simd8<bool> is_fourth_byte = prev3 >= uint8_t(0b11110000u);
4826 // Use ^ instead of | for is_*_byte, because ^ is commutative, and the caller
4827 // is using ^ as well. This will work fine because we only have to report
4828 // errors for cases with 0-1 lead bytes. Multiple lead bytes implies 2
4829 // overlapping multibyte characters, and if that happens, there is guaranteed
4830 // to be at least *one* lead byte that is part of only 1 other multibyte
4831 // character. The error will be detected there.
4832 return is_second_byte ^ is_third_byte ^ is_fourth_byte;
4833 }
4834
4835 is_utf8_really_inline simd8<bool>
4836 must_be_2_3_continuation(const simd8<uint8_t> prev2,
4837 const simd8<uint8_t> prev3) {
4838 simd8<bool> is_third_byte = prev2 >= uint8_t(0b11100000u);
4839 simd8<bool> is_fourth_byte = prev3 >= uint8_t(0b11110000u);
4840 return is_third_byte ^ is_fourth_byte;
4841 }
4842
4843 } // unnamed namespace
4844 } // namespace arm64
4845 } // namespace is_utf8_internals
4846
4847 namespace is_utf8_internals {
4848 namespace arm64 {
4849 namespace {
4850
4851 // Walks through a buffer in block-sized increments, loading the last part with
4852 // spaces
4853 template <size_t STEP_SIZE> struct buf_block_reader {
4854 public:
4855 is_utf8_really_inline buf_block_reader(const uint8_t *_buf, size_t _len);
4856 is_utf8_really_inline size_t block_index();
4857 is_utf8_really_inline bool has_full_block() const;
4858 is_utf8_really_inline const uint8_t *full_block() const;
4859 /**
4860 * Get the last block, padded with spaces.
4861 *
4862 * There will always be a last block, with at least 1 byte, unless len == 0
4863 * (in which case this function fills the buffer with spaces and returns 0. In
4864 * particular, if len == STEP_SIZE there will be 0 full_blocks and 1 remainder
4865 * block with STEP_SIZE bytes and no spaces for padding.
4866 *
4867 * @return the number of effective characters in the last block.
4868 */
4869 is_utf8_really_inline size_t get_remainder(uint8_t *dst) const;
4870 is_utf8_really_inline void advance();
4871
4872 private:
4873 const uint8_t *buf;
4874 const size_t len;
4875 const size_t lenminusstep;
4876 size_t idx;
4877 };
4878
4879 // Routines to print masks and text for debugging bitmask operations
4880 is_utf8_unused static char *format_input_text_64(const uint8_t *text) {
4881 static char *buf =
4882 reinterpret_cast<char *>(malloc(sizeof(simd8x64<uint8_t>) + 1));
4883 for (size_t i = 0; i < sizeof(simd8x64<uint8_t>); i++) {
4884 buf[i] = int8_t(text[i]) < ' ' ? '_' : int8_t(text[i]);
4885 }
4886 buf[sizeof(simd8x64<uint8_t>)] = '\0';
4887 return buf;
4888 }
4889
4890 // Routines to print masks and text for debugging bitmask operations
4891 is_utf8_unused static char *format_input_text(const simd8x64<uint8_t> &in) {
4892 static char *buf =
4893 reinterpret_cast<char *>(malloc(sizeof(simd8x64<uint8_t>) + 1));
4894 in.store(reinterpret_cast<uint8_t *>(buf));
4895 for (size_t i = 0; i < sizeof(simd8x64<uint8_t>); i++) {
4896 if (buf[i] < ' ') {
4897 buf[i] = '_';
4898 }
4899 }
4900 buf[sizeof(simd8x64<uint8_t>)] = '\0';
4901 return buf;
4902 }
4903
4904 is_utf8_unused static char *format_mask(uint64_t mask) {
4905 static char *buf = reinterpret_cast<char *>(malloc(64 + 1));
4906 for (size_t i = 0; i < 64; i++) {
4907 buf[i] = (mask & (size_t(1) << i)) ? 'X' : ' ';
4908 }
4909 buf[64] = '\0';
4910 return buf;
4911 }
4912
4913 template <size_t STEP_SIZE>
4914 is_utf8_really_inline
4915 buf_block_reader<STEP_SIZE>::buf_block_reader(const uint8_t *_buf, size_t _len)
4916 : buf{_buf}, len{_len},
4917 lenminusstep{len < STEP_SIZE ? 0 : len - STEP_SIZE}, idx{0} {}
4918
4919 template <size_t STEP_SIZE>
4920 is_utf8_really_inline size_t buf_block_reader<STEP_SIZE>::block_index() {
4921 return idx;
4922 }
4923
4924 template <size_t STEP_SIZE>
4925 is_utf8_really_inline bool buf_block_reader<STEP_SIZE>::has_full_block() const {
4926 return idx < lenminusstep;
4927 }
4928
4929 template <size_t STEP_SIZE>
4930 is_utf8_really_inline const uint8_t *
4931 buf_block_reader<STEP_SIZE>::full_block() const {
4932 return &buf[idx];
4933 }
4934
4935 template <size_t STEP_SIZE>
4936 is_utf8_really_inline size_t
4937 buf_block_reader<STEP_SIZE>::get_remainder(uint8_t *dst) const {
4938 if (len == idx) {
4939 return 0;
4940 } // memcpy(dst, null, 0) will trigger an error with some sanitizers
4941 std::memset(dst, 0x20,
4942 STEP_SIZE); // std::memset STEP_SIZE because it's more efficient
4943 // to write out 8 or 16 bytes at once.
4944 std::memcpy(dst, buf + idx, len - idx);
4945 return len - idx;
4946 }
4947
4948 template <size_t STEP_SIZE>
4949 is_utf8_really_inline void buf_block_reader<STEP_SIZE>::advance() {
4950 idx += STEP_SIZE;
4951 }
4952
4953 } // unnamed namespace
4954 } // namespace arm64
4955 } // namespace is_utf8_internals
4956
4957 namespace is_utf8_internals {
4958 namespace arm64 {
4959 namespace {
4960 namespace utf8_validation {
4961
4962 using namespace simd;
4963
4964 is_utf8_really_inline simd8<uint8_t>
4965 check_special_cases(const simd8<uint8_t> input, const simd8<uint8_t> prev1) {
4966 // Bit 0 = Too Short (lead byte/ASCII followed by lead byte/ASCII)
4967 // Bit 1 = Too Long (ASCII followed by continuation)
4968 // Bit 2 = Overlong 3-byte
4969 // Bit 4 = Surrogate
4970 // Bit 5 = Overlong 2-byte
4971 // Bit 7 = Two Continuations
4972 constexpr const uint8_t TOO_SHORT = 1 << 0; // 11______ 0_______
4973 // 11______ 11______
4974 constexpr const uint8_t TOO_LONG = 1 << 1; // 0_______ 10______
4975 constexpr const uint8_t OVERLONG_3 = 1 << 2; // 11100000 100_____
4976 constexpr const uint8_t SURROGATE = 1 << 4; // 11101101 101_____
4977 constexpr const uint8_t OVERLONG_2 = 1 << 5; // 1100000_ 10______
4978 constexpr const uint8_t TWO_CONTS = 1 << 7; // 10______ 10______
4979 constexpr const uint8_t TOO_LARGE = 1 << 3; // 11110100 1001____
4980 // 11110100 101_____
4981 // 11110101 1001____
4982 // 11110101 101_____
4983 // 1111011_ 1001____
4984 // 1111011_ 101_____
4985 // 11111___ 1001____
4986 // 11111___ 101_____
4987 constexpr const uint8_t TOO_LARGE_1000 = 1 << 6;
4988 // 11110101 1000____
4989 // 1111011_ 1000____
4990 // 11111___ 1000____
4991 constexpr const uint8_t OVERLONG_4 = 1 << 6; // 11110000 1000____
4992
4993 const simd8<uint8_t> byte_1_high = prev1.shr<4>().lookup_16<uint8_t>(
4994 // 0_______ ________ <ASCII in byte 1>
4995 TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG,
4996 TOO_LONG,
4997 // 10______ ________ <continuation in byte 1>
4998 TWO_CONTS, TWO_CONTS, TWO_CONTS, TWO_CONTS,
4999 // 1100____ ________ <two byte lead in byte 1>
5000 TOO_SHORT | OVERLONG_2,
5001 // 1101____ ________ <two byte lead in byte 1>
5002 TOO_SHORT,
5003 // 1110____ ________ <three byte lead in byte 1>
5004 TOO_SHORT | OVERLONG_3 | SURROGATE,
5005 // 1111____ ________ <four+ byte lead in byte 1>
5006 TOO_SHORT | TOO_LARGE | TOO_LARGE_1000 | OVERLONG_4);
5007 constexpr const uint8_t CARRY =
5008 TOO_SHORT | TOO_LONG | TWO_CONTS; // These all have ____ in byte 1 .
5009 const simd8<uint8_t> byte_1_low =
5010 (prev1 & 0x0F)
5011 .lookup_16<uint8_t>(
5012 // ____0000 ________
5013 CARRY | OVERLONG_3 | OVERLONG_2 | OVERLONG_4,
5014 // ____0001 ________
5015 CARRY | OVERLONG_2,
5016 // ____001_ ________
5017 CARRY, CARRY,
5018
5019 // ____0100 ________
5020 CARRY | TOO_LARGE,
5021 // ____0101 ________
5022 CARRY | TOO_LARGE | TOO_LARGE_1000,
5023 // ____011_ ________
5024 CARRY | TOO_LARGE | TOO_LARGE_1000,
5025 CARRY | TOO_LARGE | TOO_LARGE_1000,
5026
5027 // ____1___ ________
5028 CARRY | TOO_LARGE | TOO_LARGE_1000,
5029 CARRY | TOO_LARGE | TOO_LARGE_1000,
5030 CARRY | TOO_LARGE | TOO_LARGE_1000,
5031 CARRY | TOO_LARGE | TOO_LARGE_1000,
5032 CARRY | TOO_LARGE | TOO_LARGE_1000,
5033 // ____1101 ________
5034 CARRY | TOO_LARGE | TOO_LARGE_1000 | SURROGATE,
5035 CARRY | TOO_LARGE | TOO_LARGE_1000,
5036 CARRY | TOO_LARGE | TOO_LARGE_1000);
5037 const simd8<uint8_t> byte_2_high = input.shr<4>().lookup_16<uint8_t>(
5038 // ________ 0_______ <ASCII in byte 2>
5039 TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT,
5040 TOO_SHORT, TOO_SHORT,
5041
5042 // ________ 1000____
5043 TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE_1000 |
5044 OVERLONG_4,
5045 // ________ 1001____
5046 TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE,
5047 // ________ 101_____
5048 TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE,
5049 TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE,
5050
5051 // ________ 11______
5052 TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT);
5053 return (byte_1_high & byte_1_low & byte_2_high);
5054 }
5055 is_utf8_really_inline simd8<uint8_t>
5056 check_multibyte_lengths(const simd8<uint8_t> input,
5057 const simd8<uint8_t> prev_input,
5058 const simd8<uint8_t> sc) {
5059 simd8<uint8_t> prev2 = input.prev<2>(prev_input);
5060 simd8<uint8_t> prev3 = input.prev<3>(prev_input);
5061 simd8<uint8_t> must23 =
5062 simd8<uint8_t>(must_be_2_3_continuation(prev2, prev3));
5063 simd8<uint8_t> must23_80 = must23 & uint8_t(0x80);
5064 return must23_80 ^ sc;
5065 }
5066
5067 //
5068 // Return nonzero if there are incomplete multibyte characters at the end of the
5069 // block: e.g. if there is a 4-byte character, but it's 3 bytes from the end.
5070 //
5071 is_utf8_really_inline simd8<uint8_t> is_incomplete(const simd8<uint8_t> input) {
5072 // If the previous input's last 3 bytes match this, they're too short (they
5073 // ended at EOF):
5074 // ... 1111____ 111_____ 11______
5075 static const uint8_t max_array[32] = {255,
5076 255,
5077 255,
5078 255,
5079 255,
5080 255,
5081 255,
5082 255,
5083 255,
5084 255,
5085 255,
5086 255,
5087 255,
5088 255,
5089 255,
5090 255,
5091 255,
5092 255,
5093 255,
5094 255,
5095 255,
5096 255,
5097 255,
5098 255,
5099 255,
5100 255,
5101 255,
5102 255,
5103 255,
5104 0b11110000u - 1,
5105 0b11100000u - 1,
5106 0b11000000u - 1};
5107 const simd8<uint8_t> max_value(
5108 &max_array[sizeof(max_array) - sizeof(simd8<uint8_t>)]);
5109 return input.gt_bits(max_value);
5110 }
5111
5112 struct utf8_checker {
5113 // If this is nonzero, there has been a UTF-8 error.
5114 simd8<uint8_t> error;
5115 // The last input we received
5116 simd8<uint8_t> prev_input_block;
5117 // Whether the last input we received was incomplete (used for ASCII fast
5118 // path)
5119 simd8<uint8_t> prev_incomplete;
5120
5121 //
5122 // Check whether the current bytes are valid UTF-8.
5123 //
5124 is_utf8_really_inline void check_utf8_bytes(const simd8<uint8_t> input,
5125 const simd8<uint8_t> prev_input) {
5126 // Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+
5127 // lead bytes (2, 3, 4-byte leads become large positive numbers instead of
5128 // small negative numbers)
5129 simd8<uint8_t> prev1 = input.prev<1>(prev_input);
5130 simd8<uint8_t> sc = check_special_cases(input, prev1);
5131 this->error |= check_multibyte_lengths(input, prev_input, sc);
5132 }
5133
5134 // The only problem that can happen at EOF is that a multibyte character is
5135 // too short or a byte value too large in the last bytes: check_special_cases
5136 // only checks for bytes too large in the first of two bytes.
5137 is_utf8_really_inline void check_eof() {
5138 // If the previous block had incomplete UTF-8 characters at the end, an
5139 // ASCII block can't possibly finish them.
5140 this->error |= this->prev_incomplete;
5141 }
5142
5143 is_utf8_really_inline void check_next_input(const simd8x64<uint8_t> &input) {
5144 if (is_utf8_likely(is_ascii(input))) {
5145 this->error |= this->prev_incomplete;
5146 } else {
5147 // you might think that a for-loop would work, but under Visual Studio, it
5148 // is not good enough.
5149 static_assert((simd8x64<uint8_t>::NUM_CHUNKS == 2) ||
5150 (simd8x64<uint8_t>::NUM_CHUNKS == 4),
5151 "We support either two or four chunks per 64-byte block.");
5152 if (simd8x64<uint8_t>::NUM_CHUNKS == 2) {
5153 this->check_utf8_bytes(input.chunks[0], this->prev_input_block);
5154 this->check_utf8_bytes(input.chunks[1], input.chunks[0]);
5155 } else if (simd8x64<uint8_t>::NUM_CHUNKS == 4) {
5156 this->check_utf8_bytes(input.chunks[0], this->prev_input_block);
5157 this->check_utf8_bytes(input.chunks[1], input.chunks[0]);
5158 this->check_utf8_bytes(input.chunks[2], input.chunks[1]);
5159 this->check_utf8_bytes(input.chunks[3], input.chunks[2]);
5160 }
5161 this->prev_incomplete =
5162 is_incomplete(input.chunks[simd8x64<uint8_t>::NUM_CHUNKS - 1]);
5163 this->prev_input_block = input.chunks[simd8x64<uint8_t>::NUM_CHUNKS - 1];
5164 }
5165 }
5166
5167 // do not forget to call check_eof!
5168 is_utf8_really_inline bool errors() const {
5169 return this->error.any_bits_set_anywhere();
5170 }
5171
5172 }; // struct utf8_checker
5173 } // namespace utf8_validation
5174
5175 using utf8_validation::utf8_checker;
5176
5177 } // unnamed namespace
5178 } // namespace arm64
5179 } // namespace is_utf8_internals
5180
5181 namespace is_utf8_internals {
5182 namespace arm64 {
5183 namespace {
5184 namespace utf8_validation {
5185
5186 /**
5187 * Validates that the string is actual UTF-8.
5188 */
5189 template <class checker>
5190 bool generic_validate_utf8(const uint8_t *input, size_t length) {
5191 checker c{};
5192 buf_block_reader<64> reader(input, length);
5193 while (reader.has_full_block()) {
5194 simd::simd8x64<uint8_t> in(reader.full_block());
5195 c.check_next_input(in);
5196 reader.advance();
5197 }
5198 uint8_t block[64]{};
5199 reader.get_remainder(block);
5200 simd::simd8x64<uint8_t> in(block);
5201 c.check_next_input(in);
5202 reader.advance();
5203 c.check_eof();
5204 return !c.errors();
5205 }
5206
5207 bool generic_validate_utf8(const char *input, size_t length) {
5208 return generic_validate_utf8<utf8_checker>(
5209 reinterpret_cast<const uint8_t *>(input), length);
5210 }
5211
5212 } // namespace utf8_validation
5213 } // unnamed namespace
5214 } // namespace arm64
5215 } // namespace is_utf8_internals
5216
5217 //
5218 // Implementation-specific overrides
5219 //
5220 namespace is_utf8_internals {
5221 namespace arm64 {
5222
5223 is_utf8_warn_unused bool
5224 implementation::validate_utf8(const char *buf, size_t len) const noexcept {
5225 return arm64::utf8_validation::generic_validate_utf8(buf, len);
5226 }
5227
5228 } // namespace arm64
5229 } // namespace is_utf8_internals
5230
5231 #endif
5232 #if IS_UTF8_IMPLEMENTATION_FALLBACK
5233
5234 // redefining IS_UTF8_IMPLEMENTATION to "fallback"
5235 // #define IS_UTF8_IMPLEMENTATION fallback
5236
5237 namespace is_utf8_internals {
5238 namespace fallback {
5239
5240 is_utf8_warn_unused bool
5241 implementation::validate_utf8(const char *buf, size_t len) const noexcept {
5242 return scalar::utf8::validate(buf, len);
5243 }
5244
5245 } // namespace fallback
5246 } // namespace is_utf8_internals
5247
5248 #endif
5249 #if IS_UTF8_IMPLEMENTATION_ICELAKE
5250
5251 // redefining IS_UTF8_IMPLEMENTATION to "icelake"
5252 // #define IS_UTF8_IMPLEMENTATION icelake
5253 IS_UTF8_TARGET_ICELAKE
5254
5255 #if IS_UTF8_GCC11ORMORE // workaround for
5256 // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=105593
5257 IS_UTF8_DISABLE_GCC_WARNING(-Wmaybe-uninitialized)
5258 #endif // end of workaround
5259
5260 namespace is_utf8_internals {
5261 namespace icelake {
5262 namespace {
5263 #ifndef IS_UTF8_ICELAKE_H
5264 #error "icelake.h must be included"
5265 #endif
5266
5267 /**
5268 * Store the last N bytes of previous followed by 512-N bytes from input.
5269 */
5270 template <int N> __m512i prev(__m512i input, __m512i previous) {
5271 static_assert(N <= 32, "N must be no larger than 32");
5272 const __m512i movemask =
5273 _mm512_setr_epi32(28, 29, 30, 31, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11);
5274 const __m512i rotated = _mm512_permutex2var_epi32(input, movemask, previous);
5275 #if IS_UTF8_GCC8 || IS_UTF8_GCC9
5276 constexpr int shift = 16 - N; // workaround for GCC8,9
5277 return _mm512_alignr_epi8(input, rotated, shift);
5278 #else
5279 return _mm512_alignr_epi8(input, rotated, 16 - N);
5280 #endif // IS_UTF8_GCC8 || IS_UTF8_GCC9
5281 }
5282
5283 is_utf8_really_inline __m512i check_special_cases(__m512i input,
5284 const __m512i prev1) {
5285 __m512i mask1 = _mm512_setr_epi64(0x0202020202020202, 0x4915012180808080,
5286 0x0202020202020202, 0x4915012180808080,
5287 0x0202020202020202, 0x4915012180808080,
5288 0x0202020202020202, 0x4915012180808080);
5289 const __m512i v_0f = _mm512_set1_epi8(0x0f);
5290 __m512i index1 = _mm512_and_si512(_mm512_srli_epi16(prev1, 4), v_0f);
5291
5292 __m512i byte_1_high = _mm512_shuffle_epi8(mask1, index1);
5293 __m512i mask2 = _mm512_setr_epi64(0xcbcbcb8b8383a3e7, 0xcbcbdbcbcbcbcbcb,
5294 0xcbcbcb8b8383a3e7, 0xcbcbdbcbcbcbcbcb,
5295 0xcbcbcb8b8383a3e7, 0xcbcbdbcbcbcbcbcb,
5296 0xcbcbcb8b8383a3e7, 0xcbcbdbcbcbcbcbcb);
5297 __m512i index2 = _mm512_and_si512(prev1, v_0f);
5298
5299 __m512i byte_1_low = _mm512_shuffle_epi8(mask2, index2);
5300 __m512i mask3 =
5301 _mm512_setr_epi64(0x101010101010101, 0x1010101babaaee6, 0x101010101010101,
5302 0x1010101babaaee6, 0x101010101010101, 0x1010101babaaee6,
5303 0x101010101010101, 0x1010101babaaee6);
5304 __m512i index3 = _mm512_and_si512(_mm512_srli_epi16(input, 4), v_0f);
5305 __m512i byte_2_high = _mm512_shuffle_epi8(mask3, index3);
5306 return _mm512_ternarylogic_epi64(byte_1_high, byte_1_low, byte_2_high, 128);
5307 }
5308
5309 is_utf8_really_inline __m512i check_multibyte_lengths(const __m512i input,
5310 const __m512i prev_input,
5311 const __m512i sc) {
5312 __m512i prev2 = prev<2>(input, prev_input);
5313 __m512i prev3 = prev<3>(input, prev_input);
5314 __m512i is_third_byte = _mm512_subs_epu8(
5315 prev2, _mm512_set1_epi8(char(0b11011111))); // Only 111_____ will be > 0
5316 __m512i is_fourth_byte = _mm512_subs_epu8(
5317 prev3, _mm512_set1_epi8(char(0b11101111))); // Only 1111____ will be > 0
5318 __m512i is_third_or_fourth_byte =
5319 _mm512_or_si512(is_third_byte, is_fourth_byte);
5320 const __m512i v_7f = _mm512_set1_epi8(char(0x7f));
5321 is_third_or_fourth_byte = _mm512_adds_epu8(v_7f, is_third_or_fourth_byte);
5322 // We want to compute (is_third_or_fourth_byte AND v80) XOR sc.
5323 const __m512i v_80 = _mm512_set1_epi8(char(0x80));
5324 return _mm512_ternarylogic_epi32(is_third_or_fourth_byte, v_80, sc,
5325 0b1101010);
5326 //__m512i is_third_or_fourth_byte_mask =
5327 //_mm512_and_si512(is_third_or_fourth_byte, v_80); return
5328 // _mm512_xor_si512(is_third_or_fourth_byte_mask, sc);
5329 }
5330 //
5331 // Return nonzero if there are incomplete multibyte characters at the end of the
5332 // block: e.g. if there is a 4-byte character, but it's 3 bytes from the end.
5333 //
5334 is_utf8_really_inline __m512i is_incomplete(const __m512i input) {
5335 // If the previous input's last 3 bytes match this, they're too short (they
5336 // ended at EOF):
5337 // ... 1111____ 111_____ 11______
5338 __m512i max_value = _mm512_setr_epi64(0xffffffffffffffff, 0xffffffffffffffff,
5339 0xffffffffffffffff, 0xffffffffffffffff,
5340 0xffffffffffffffff, 0xffffffffffffffff,
5341 0xffffffffffffffff, 0xbfdfefffffffffff);
5342 return _mm512_subs_epu8(input, max_value);
5343 }
5344
5345 struct avx512_utf8_checker {
5346 // If this is nonzero, there has been a UTF-8 error.
5347 __m512i error{};
5348
5349 // The last input we received
5350 __m512i prev_input_block{};
5351 // Whether the last input we received was incomplete (used for ASCII fast
5352 // path)
5353 __m512i prev_incomplete{};
5354
5355 //
5356 // Check whether the current bytes are valid UTF-8.
5357 //
5358 is_utf8_really_inline void check_utf8_bytes(const __m512i input,
5359 const __m512i prev_input) {
5360 // Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+
5361 // lead bytes (2, 3, 4-byte leads become large positive numbers instead of
5362 // small negative numbers)
5363 __m512i prev1 = prev<1>(input, prev_input);
5364 __m512i sc = check_special_cases(input, prev1);
5365 this->error = _mm512_or_si512(
5366 check_multibyte_lengths(input, prev_input, sc), this->error);
5367 }
5368
5369 // The only problem that can happen at EOF is that a multibyte character is
5370 // too short or a byte value too large in the last bytes: check_special_cases
5371 // only checks for bytes too large in the first of two bytes.
5372 is_utf8_really_inline void check_eof() {
5373 // If the previous block had incomplete UTF-8 characters at the end, an
5374 // ASCII block can't possibly finish them.
5375 this->error = _mm512_or_si512(this->error, this->prev_incomplete);
5376 }
5377
5378 // returns true if ASCII.
5379 is_utf8_really_inline bool check_next_input(const __m512i input) {
5380 const __m512i v_80 = _mm512_set1_epi8(char(0x80));
5381 const __mmask64 ascii = _mm512_test_epi8_mask(input, v_80);
5382 if (ascii == 0) {
5383 this->error = _mm512_or_si512(this->error, this->prev_incomplete);
5384 return true;
5385 } else {
5386 this->check_utf8_bytes(input, this->prev_input_block);
5387 this->prev_incomplete = is_incomplete(input);
5388 this->prev_input_block = input;
5389 return false;
5390 }
5391 }
5392 // do not forget to call check_eof!
5393 is_utf8_really_inline bool errors() const {
5394 return _mm512_test_epi8_mask(this->error, this->error) != 0;
5395 }
5396
5397 }; // struct avx512_utf8_checker
5398
5399 } // namespace
5400 } // namespace icelake
5401 } // namespace is_utf8_internals
5402
5403 namespace is_utf8_internals {
5404 namespace icelake {
5405
5406 is_utf8_warn_unused bool
5407 implementation::validate_utf8(const char *buf, size_t len) const noexcept {
5408 avx512_utf8_checker checker{};
5409 const char *ptr = buf;
5410 const char *end = ptr + len;
5411 for (; ptr + 64 <= end; ptr += 64) {
5412 const __m512i utf8 = _mm512_loadu_si512((const __m512i *)ptr);
5413 checker.check_next_input(utf8);
5414 }
5415 {
5416 const __m512i utf8 = _mm512_maskz_loadu_epi8((1ULL << (end - ptr)) - 1,
5417 (const __m512i *)ptr);
5418 checker.check_next_input(utf8);
5419 }
5420 checker.check_eof();
5421 return !checker.errors();
5422 }
5423
5424 } // namespace icelake
5425 } // namespace is_utf8_internals
5426
5427 IS_UTF8_UNTARGET_REGION
5428
5429 #if IS_UTF8_GCC11ORMORE // workaround for
5430 // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=105593
5431 IS_UTF8_POP_DISABLE_WARNINGS
5432 #endif // end of workaround
5433
5434 #endif
5435 #if IS_UTF8_IMPLEMENTATION_HASWELL
5436
5437 // redefining IS_UTF8_IMPLEMENTATION to "haswell"
5438 // #define IS_UTF8_IMPLEMENTATION haswell
5439 IS_UTF8_TARGET_HASWELL
5440
5441 #if IS_UTF8_GCC11ORMORE // workaround for
5442 // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=105593
5443 IS_UTF8_DISABLE_GCC_WARNING(-Wmaybe-uninitialized)
5444 #endif // end of workaround
5445
5446 namespace is_utf8_internals {
5447 namespace haswell {
5448 namespace {
5449 #ifndef IS_UTF8_HASWELL_H
5450 #error "haswell.h must be included"
5451 #endif
5452 using namespace simd;
5453
5454 is_utf8_really_inline bool is_ascii(const simd8x64<uint8_t> &input) {
5455 return input.reduce_or().is_ascii();
5456 }
5457
5458 is_utf8_unused is_utf8_really_inline simd8<bool>
5459 must_be_continuation(const simd8<uint8_t> prev1, const simd8<uint8_t> prev2,
5460 const simd8<uint8_t> prev3) {
5461 simd8<uint8_t> is_second_byte =
5462 prev1.saturating_sub(0b11000000u - 1); // Only 11______ will be > 0
5463 simd8<uint8_t> is_third_byte =
5464 prev2.saturating_sub(0b11100000u - 1); // Only 111_____ will be > 0
5465 simd8<uint8_t> is_fourth_byte =
5466 prev3.saturating_sub(0b11110000u - 1); // Only 1111____ will be > 0
5467 // Caller requires a bool (all 1's). All values resulting from the subtraction
5468 // will be <= 64, so signed comparison is fine.
5469 return simd8<int8_t>(is_second_byte | is_third_byte | is_fourth_byte) >
5470 int8_t(0);
5471 }
5472
5473 is_utf8_really_inline simd8<bool>
5474 must_be_2_3_continuation(const simd8<uint8_t> prev2,
5475 const simd8<uint8_t> prev3) {
5476 simd8<uint8_t> is_third_byte =
5477 prev2.saturating_sub(0b11100000u - 1); // Only 111_____ will be > 0
5478 simd8<uint8_t> is_fourth_byte =
5479 prev3.saturating_sub(0b11110000u - 1); // Only 1111____ will be > 0
5480 // Caller requires a bool (all 1's). All values resulting from the subtraction
5481 // will be <= 64, so signed comparison is fine.
5482 return simd8<int8_t>(is_third_byte | is_fourth_byte) > int8_t(0);
5483 }
5484
5485 } // unnamed namespace
5486 } // namespace haswell
5487 } // namespace is_utf8_internals
5488
5489 namespace is_utf8_internals {
5490 namespace haswell {
5491 namespace {
5492
5493 // Walks through a buffer in block-sized increments, loading the last part with
5494 // spaces
5495 template <size_t STEP_SIZE> struct buf_block_reader {
5496 public:
5497 is_utf8_really_inline buf_block_reader(const uint8_t *_buf, size_t _len);
5498 is_utf8_really_inline size_t block_index();
5499 is_utf8_really_inline bool has_full_block() const;
5500 is_utf8_really_inline const uint8_t *full_block() const;
5501 /**
5502 * Get the last block, padded with spaces.
5503 *
5504 * There will always be a last block, with at least 1 byte, unless len == 0
5505 * (in which case this function fills the buffer with spaces and returns 0. In
5506 * particular, if len == STEP_SIZE there will be 0 full_blocks and 1 remainder
5507 * block with STEP_SIZE bytes and no spaces for padding.
5508 *
5509 * @return the number of effective characters in the last block.
5510 */
5511 is_utf8_really_inline size_t get_remainder(uint8_t *dst) const;
5512 is_utf8_really_inline void advance();
5513
5514 private:
5515 const uint8_t *buf;
5516 const size_t len;
5517 const size_t lenminusstep;
5518 size_t idx;
5519 };
5520
5521 // Routines to print masks and text for debugging bitmask operations
5522 is_utf8_unused static char *format_input_text_64(const uint8_t *text) {
5523 static char *buf =
5524 reinterpret_cast<char *>(malloc(sizeof(simd8x64<uint8_t>) + 1));
5525 for (size_t i = 0; i < sizeof(simd8x64<uint8_t>); i++) {
5526 buf[i] = int8_t(text[i]) < ' ' ? '_' : int8_t(text[i]);
5527 }
5528 buf[sizeof(simd8x64<uint8_t>)] = '\0';
5529 return buf;
5530 }
5531
5532 // Routines to print masks and text for debugging bitmask operations
5533 is_utf8_unused static char *format_input_text(const simd8x64<uint8_t> &in) {
5534 static char *buf =
5535 reinterpret_cast<char *>(malloc(sizeof(simd8x64<uint8_t>) + 1));
5536 in.store(reinterpret_cast<uint8_t *>(buf));
5537 for (size_t i = 0; i < sizeof(simd8x64<uint8_t>); i++) {
5538 if (buf[i] < ' ') {
5539 buf[i] = '_';
5540 }
5541 }
5542 buf[sizeof(simd8x64<uint8_t>)] = '\0';
5543 return buf;
5544 }
5545
5546 is_utf8_unused static char *format_mask(uint64_t mask) {
5547 static char *buf = reinterpret_cast<char *>(malloc(64 + 1));
5548 for (size_t i = 0; i < 64; i++) {
5549 buf[i] = (mask & (size_t(1) << i)) ? 'X' : ' ';
5550 }
5551 buf[64] = '\0';
5552 return buf;
5553 }
5554
5555 template <size_t STEP_SIZE>
5556 is_utf8_really_inline
5557 buf_block_reader<STEP_SIZE>::buf_block_reader(const uint8_t *_buf, size_t _len)
5558 : buf{_buf}, len{_len},
5559 lenminusstep{len < STEP_SIZE ? 0 : len - STEP_SIZE}, idx{0} {}
5560
5561 template <size_t STEP_SIZE>
5562 is_utf8_really_inline size_t buf_block_reader<STEP_SIZE>::block_index() {
5563 return idx;
5564 }
5565
5566 template <size_t STEP_SIZE>
5567 is_utf8_really_inline bool buf_block_reader<STEP_SIZE>::has_full_block() const {
5568 return idx < lenminusstep;
5569 }
5570
5571 template <size_t STEP_SIZE>
5572 is_utf8_really_inline const uint8_t *
5573 buf_block_reader<STEP_SIZE>::full_block() const {
5574 return &buf[idx];
5575 }
5576
5577 template <size_t STEP_SIZE>
5578 is_utf8_really_inline size_t
5579 buf_block_reader<STEP_SIZE>::get_remainder(uint8_t *dst) const {
5580 if (len == idx) {
5581 return 0;
5582 } // memcpy(dst, null, 0) will trigger an error with some sanitizers
5583 std::memset(dst, 0x20,
5584 STEP_SIZE); // std::memset STEP_SIZE because it's more efficient
5585 // to write out 8 or 16 bytes at once.
5586 std::memcpy(dst, buf + idx, len - idx);
5587 return len - idx;
5588 }
5589
5590 template <size_t STEP_SIZE>
5591 is_utf8_really_inline void buf_block_reader<STEP_SIZE>::advance() {
5592 idx += STEP_SIZE;
5593 }
5594
5595 } // unnamed namespace
5596 } // namespace haswell
5597 } // namespace is_utf8_internals
5598
5599 namespace is_utf8_internals {
5600 namespace haswell {
5601 namespace {
5602 namespace utf8_validation {
5603
5604 using namespace simd;
5605
5606 is_utf8_really_inline simd8<uint8_t>
5607 check_special_cases(const simd8<uint8_t> input, const simd8<uint8_t> prev1) {
5608 // Bit 0 = Too Short (lead byte/ASCII followed by lead byte/ASCII)
5609 // Bit 1 = Too Long (ASCII followed by continuation)
5610 // Bit 2 = Overlong 3-byte
5611 // Bit 4 = Surrogate
5612 // Bit 5 = Overlong 2-byte
5613 // Bit 7 = Two Continuations
5614 constexpr const uint8_t TOO_SHORT = 1 << 0; // 11______ 0_______
5615 // 11______ 11______
5616 constexpr const uint8_t TOO_LONG = 1 << 1; // 0_______ 10______
5617 constexpr const uint8_t OVERLONG_3 = 1 << 2; // 11100000 100_____
5618 constexpr const uint8_t SURROGATE = 1 << 4; // 11101101 101_____
5619 constexpr const uint8_t OVERLONG_2 = 1 << 5; // 1100000_ 10______
5620 constexpr const uint8_t TWO_CONTS = 1 << 7; // 10______ 10______
5621 constexpr const uint8_t TOO_LARGE = 1 << 3; // 11110100 1001____
5622 // 11110100 101_____
5623 // 11110101 1001____
5624 // 11110101 101_____
5625 // 1111011_ 1001____
5626 // 1111011_ 101_____
5627 // 11111___ 1001____
5628 // 11111___ 101_____
5629 constexpr const uint8_t TOO_LARGE_1000 = 1 << 6;
5630 // 11110101 1000____
5631 // 1111011_ 1000____
5632 // 11111___ 1000____
5633 constexpr const uint8_t OVERLONG_4 = 1 << 6; // 11110000 1000____
5634
5635 const simd8<uint8_t> byte_1_high = prev1.shr<4>().lookup_16<uint8_t>(
5636 // 0_______ ________ <ASCII in byte 1>
5637 TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG,
5638 TOO_LONG,
5639 // 10______ ________ <continuation in byte 1>
5640 TWO_CONTS, TWO_CONTS, TWO_CONTS, TWO_CONTS,
5641 // 1100____ ________ <two byte lead in byte 1>
5642 TOO_SHORT | OVERLONG_2,
5643 // 1101____ ________ <two byte lead in byte 1>
5644 TOO_SHORT,
5645 // 1110____ ________ <three byte lead in byte 1>
5646 TOO_SHORT | OVERLONG_3 | SURROGATE,
5647 // 1111____ ________ <four+ byte lead in byte 1>
5648 TOO_SHORT | TOO_LARGE | TOO_LARGE_1000 | OVERLONG_4);
5649 constexpr const uint8_t CARRY =
5650 TOO_SHORT | TOO_LONG | TWO_CONTS; // These all have ____ in byte 1 .
5651 const simd8<uint8_t> byte_1_low =
5652 (prev1 & 0x0F)
5653 .lookup_16<uint8_t>(
5654 // ____0000 ________
5655 CARRY | OVERLONG_3 | OVERLONG_2 | OVERLONG_4,
5656 // ____0001 ________
5657 CARRY | OVERLONG_2,
5658 // ____001_ ________
5659 CARRY, CARRY,
5660
5661 // ____0100 ________
5662 CARRY | TOO_LARGE,
5663 // ____0101 ________
5664 CARRY | TOO_LARGE | TOO_LARGE_1000,
5665 // ____011_ ________
5666 CARRY | TOO_LARGE | TOO_LARGE_1000,
5667 CARRY | TOO_LARGE | TOO_LARGE_1000,
5668
5669 // ____1___ ________
5670 CARRY | TOO_LARGE | TOO_LARGE_1000,
5671 CARRY | TOO_LARGE | TOO_LARGE_1000,
5672 CARRY | TOO_LARGE | TOO_LARGE_1000,
5673 CARRY | TOO_LARGE | TOO_LARGE_1000,
5674 CARRY | TOO_LARGE | TOO_LARGE_1000,
5675 // ____1101 ________
5676 CARRY | TOO_LARGE | TOO_LARGE_1000 | SURROGATE,
5677 CARRY | TOO_LARGE | TOO_LARGE_1000,
5678 CARRY | TOO_LARGE | TOO_LARGE_1000);
5679 const simd8<uint8_t> byte_2_high = input.shr<4>().lookup_16<uint8_t>(
5680 // ________ 0_______ <ASCII in byte 2>
5681 TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT,
5682 TOO_SHORT, TOO_SHORT,
5683
5684 // ________ 1000____
5685 TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE_1000 |
5686 OVERLONG_4,
5687 // ________ 1001____
5688 TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE,
5689 // ________ 101_____
5690 TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE,
5691 TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE,
5692
5693 // ________ 11______
5694 TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT);
5695 return (byte_1_high & byte_1_low & byte_2_high);
5696 }
5697 is_utf8_really_inline simd8<uint8_t>
5698 check_multibyte_lengths(const simd8<uint8_t> input,
5699 const simd8<uint8_t> prev_input,
5700 const simd8<uint8_t> sc) {
5701 simd8<uint8_t> prev2 = input.prev<2>(prev_input);
5702 simd8<uint8_t> prev3 = input.prev<3>(prev_input);
5703 simd8<uint8_t> must23 =
5704 simd8<uint8_t>(must_be_2_3_continuation(prev2, prev3));
5705 simd8<uint8_t> must23_80 = must23 & uint8_t(0x80);
5706 return must23_80 ^ sc;
5707 }
5708
5709 //
5710 // Return nonzero if there are incomplete multibyte characters at the end of the
5711 // block: e.g. if there is a 4-byte character, but it's 3 bytes from the end.
5712 //
5713 is_utf8_really_inline simd8<uint8_t> is_incomplete(const simd8<uint8_t> input) {
5714 // If the previous input's last 3 bytes match this, they're too short (they
5715 // ended at EOF):
5716 // ... 1111____ 111_____ 11______
5717 static const uint8_t max_array[32] = {255,
5718 255,
5719 255,
5720 255,
5721 255,
5722 255,
5723 255,
5724 255,
5725 255,
5726 255,
5727 255,
5728 255,
5729 255,
5730 255,
5731 255,
5732 255,
5733 255,
5734 255,
5735 255,
5736 255,
5737 255,
5738 255,
5739 255,
5740 255,
5741 255,
5742 255,
5743 255,
5744 255,
5745 255,
5746 0b11110000u - 1,
5747 0b11100000u - 1,
5748 0b11000000u - 1};
5749 const simd8<uint8_t> max_value(
5750 &max_array[sizeof(max_array) - sizeof(simd8<uint8_t>)]);
5751 return input.gt_bits(max_value);
5752 }
5753
5754 struct utf8_checker {
5755 // If this is nonzero, there has been a UTF-8 error.
5756 simd8<uint8_t> error;
5757 // The last input we received
5758 simd8<uint8_t> prev_input_block;
5759 // Whether the last input we received was incomplete (used for ASCII fast
5760 // path)
5761 simd8<uint8_t> prev_incomplete;
5762
5763 //
5764 // Check whether the current bytes are valid UTF-8.
5765 //
5766 is_utf8_really_inline void check_utf8_bytes(const simd8<uint8_t> input,
5767 const simd8<uint8_t> prev_input) {
5768 // Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+
5769 // lead bytes (2, 3, 4-byte leads become large positive numbers instead of
5770 // small negative numbers)
5771 simd8<uint8_t> prev1 = input.prev<1>(prev_input);
5772 simd8<uint8_t> sc = check_special_cases(input, prev1);
5773 this->error |= check_multibyte_lengths(input, prev_input, sc);
5774 }
5775
5776 // The only problem that can happen at EOF is that a multibyte character is
5777 // too short or a byte value too large in the last bytes: check_special_cases
5778 // only checks for bytes too large in the first of two bytes.
5779 is_utf8_really_inline void check_eof() {
5780 // If the previous block had incomplete UTF-8 characters at the end, an
5781 // ASCII block can't possibly finish them.
5782 this->error |= this->prev_incomplete;
5783 }
5784
5785 is_utf8_really_inline void check_next_input(const simd8x64<uint8_t> &input) {
5786 if (is_utf8_likely(is_ascii(input))) {
5787 this->error |= this->prev_incomplete;
5788 } else {
5789 // you might think that a for-loop would work, but under Visual Studio, it
5790 // is not good enough.
5791 static_assert((simd8x64<uint8_t>::NUM_CHUNKS == 2) ||
5792 (simd8x64<uint8_t>::NUM_CHUNKS == 4),
5793 "We support either two or four chunks per 64-byte block.");
5794 if (simd8x64<uint8_t>::NUM_CHUNKS == 2) {
5795 this->check_utf8_bytes(input.chunks[0], this->prev_input_block);
5796 this->check_utf8_bytes(input.chunks[1], input.chunks[0]);
5797 } else if (simd8x64<uint8_t>::NUM_CHUNKS == 4) {
5798 this->check_utf8_bytes(input.chunks[0], this->prev_input_block);
5799 this->check_utf8_bytes(input.chunks[1], input.chunks[0]);
5800 this->check_utf8_bytes(input.chunks[2], input.chunks[1]);
5801 this->check_utf8_bytes(input.chunks[3], input.chunks[2]);
5802 }
5803 this->prev_incomplete =
5804 is_incomplete(input.chunks[simd8x64<uint8_t>::NUM_CHUNKS - 1]);
5805 this->prev_input_block = input.chunks[simd8x64<uint8_t>::NUM_CHUNKS - 1];
5806 }
5807 }
5808
5809 // do not forget to call check_eof!
5810 is_utf8_really_inline bool errors() const {
5811 return this->error.any_bits_set_anywhere();
5812 }
5813
5814 }; // struct utf8_checker
5815 } // namespace utf8_validation
5816
5817 using utf8_validation::utf8_checker;
5818
5819 } // unnamed namespace
5820 } // namespace haswell
5821 } // namespace is_utf8_internals
5822
5823 namespace is_utf8_internals {
5824 namespace haswell {
5825 namespace {
5826 namespace utf8_validation {
5827
5828 /**
5829 * Validates that the string is actual UTF-8.
5830 */
5831 template <class checker>
5832 bool generic_validate_utf8(const uint8_t *input, size_t length) {
5833 checker c{};
5834 buf_block_reader<64> reader(input, length);
5835 while (reader.has_full_block()) {
5836 simd::simd8x64<uint8_t> in(reader.full_block());
5837 c.check_next_input(in);
5838 reader.advance();
5839 }
5840 uint8_t block[64]{};
5841 reader.get_remainder(block);
5842 simd::simd8x64<uint8_t> in(block);
5843 c.check_next_input(in);
5844 reader.advance();
5845 c.check_eof();
5846 return !c.errors();
5847 }
5848
5849 bool generic_validate_utf8(const char *input, size_t length) {
5850 return generic_validate_utf8<utf8_checker>(
5851 reinterpret_cast<const uint8_t *>(input), length);
5852 }
5853
5854 } // namespace utf8_validation
5855 } // unnamed namespace
5856 } // namespace haswell
5857 } // namespace is_utf8_internals
5858
5859 namespace is_utf8_internals {
5860 namespace haswell {
5861
5862 is_utf8_warn_unused bool
5863 implementation::validate_utf8(const char *buf, size_t len) const noexcept {
5864 return haswell::utf8_validation::generic_validate_utf8(buf, len);
5865 }
5866
5867 } // namespace haswell
5868 } // namespace is_utf8_internals
5869
5870 IS_UTF8_UNTARGET_REGION
5871
5872 #if IS_UTF8_GCC11ORMORE // workaround for
5873 // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=105593
5874 #pragma GCC diagnostic pop
5875 #endif // end of workaround
5876
5877 #endif
5878 #if IS_UTF8_IMPLEMENTATION_PPC64
5879
5880 // redefining IS_UTF8_IMPLEMENTATION to "ppc64"
5881 // #define IS_UTF8_IMPLEMENTATION ppc64
5882
5883 namespace is_utf8_internals {
5884 namespace ppc64 {
5885 namespace {
5886 #ifndef IS_UTF8_PPC64_H
5887 #error "ppc64.h must be included"
5888 #endif
5889 using namespace simd;
5890
5891 is_utf8_really_inline bool is_ascii(const simd8x64<uint8_t> &input) {
5892 // careful: 0x80 is not ascii.
5893 return input.reduce_or().saturating_sub(0b01111111u).bits_not_set_anywhere();
5894 }
5895
5896 is_utf8_unused is_utf8_really_inline simd8<bool>
5897 must_be_continuation(const simd8<uint8_t> prev1, const simd8<uint8_t> prev2,
5898 const simd8<uint8_t> prev3) {
5899 simd8<uint8_t> is_second_byte =
5900 prev1.saturating_sub(0b11000000u - 1); // Only 11______ will be > 0
5901 simd8<uint8_t> is_third_byte =
5902 prev2.saturating_sub(0b11100000u - 1); // Only 111_____ will be > 0
5903 simd8<uint8_t> is_fourth_byte =
5904 prev3.saturating_sub(0b11110000u - 1); // Only 1111____ will be > 0
5905 // Caller requires a bool (all 1's). All values resulting from the subtraction
5906 // will be <= 64, so signed comparison is fine.
5907 return simd8<int8_t>(is_second_byte | is_third_byte | is_fourth_byte) >
5908 int8_t(0);
5909 }
5910
5911 is_utf8_really_inline simd8<bool>
5912 must_be_2_3_continuation(const simd8<uint8_t> prev2,
5913 const simd8<uint8_t> prev3) {
5914 simd8<uint8_t> is_third_byte =
5915 prev2.saturating_sub(0b11100000u - 1); // Only 111_____ will be > 0
5916 simd8<uint8_t> is_fourth_byte =
5917 prev3.saturating_sub(0b11110000u - 1); // Only 1111____ will be > 0
5918 // Caller requires a bool (all 1's). All values resulting from the subtraction
5919 // will be <= 64, so signed comparison is fine.
5920 return simd8<int8_t>(is_third_byte | is_fourth_byte) > int8_t(0);
5921 }
5922
5923 } // unnamed namespace
5924 } // namespace ppc64
5925 } // namespace is_utf8_internals
5926
5927 namespace is_utf8_internals {
5928 namespace ppc64 {
5929 namespace {
5930
5931 // Walks through a buffer in block-sized increments, loading the last part with
5932 // spaces
5933 template <size_t STEP_SIZE> struct buf_block_reader {
5934 public:
5935 is_utf8_really_inline buf_block_reader(const uint8_t *_buf, size_t _len);
5936 is_utf8_really_inline size_t block_index();
5937 is_utf8_really_inline bool has_full_block() const;
5938 is_utf8_really_inline const uint8_t *full_block() const;
5939 /**
5940 * Get the last block, padded with spaces.
5941 *
5942 * There will always be a last block, with at least 1 byte, unless len == 0
5943 * (in which case this function fills the buffer with spaces and returns 0. In
5944 * particular, if len == STEP_SIZE there will be 0 full_blocks and 1 remainder
5945 * block with STEP_SIZE bytes and no spaces for padding.
5946 *
5947 * @return the number of effective characters in the last block.
5948 */
5949 is_utf8_really_inline size_t get_remainder(uint8_t *dst) const;
5950 is_utf8_really_inline void advance();
5951
5952 private:
5953 const uint8_t *buf;
5954 const size_t len;
5955 const size_t lenminusstep;
5956 size_t idx;
5957 };
5958
5959 // Routines to print masks and text for debugging bitmask operations
5960 is_utf8_unused static char *format_input_text_64(const uint8_t *text) {
5961 static char *buf =
5962 reinterpret_cast<char *>(malloc(sizeof(simd8x64<uint8_t>) + 1));
5963 for (size_t i = 0; i < sizeof(simd8x64<uint8_t>); i++) {
5964 buf[i] = int8_t(text[i]) < ' ' ? '_' : int8_t(text[i]);
5965 }
5966 buf[sizeof(simd8x64<uint8_t>)] = '\0';
5967 return buf;
5968 }
5969
5970 // Routines to print masks and text for debugging bitmask operations
5971 is_utf8_unused static char *format_input_text(const simd8x64<uint8_t> &in) {
5972 static char *buf =
5973 reinterpret_cast<char *>(malloc(sizeof(simd8x64<uint8_t>) + 1));
5974 in.store(reinterpret_cast<uint8_t *>(buf));
5975 for (size_t i = 0; i < sizeof(simd8x64<uint8_t>); i++) {
5976 if (buf[i] < ' ') {
5977 buf[i] = '_';
5978 }
5979 }
5980 buf[sizeof(simd8x64<uint8_t>)] = '\0';
5981 return buf;
5982 }
5983
5984 is_utf8_unused static char *format_mask(uint64_t mask) {
5985 static char *buf = reinterpret_cast<char *>(malloc(64 + 1));
5986 for (size_t i = 0; i < 64; i++) {
5987 buf[i] = (mask & (size_t(1) << i)) ? 'X' : ' ';
5988 }
5989 buf[64] = '\0';
5990 return buf;
5991 }
5992
5993 template <size_t STEP_SIZE>
5994 is_utf8_really_inline
5995 buf_block_reader<STEP_SIZE>::buf_block_reader(const uint8_t *_buf, size_t _len)
5996 : buf{_buf}, len{_len},
5997 lenminusstep{len < STEP_SIZE ? 0 : len - STEP_SIZE}, idx{0} {}
5998
5999 template <size_t STEP_SIZE>
6000 is_utf8_really_inline size_t buf_block_reader<STEP_SIZE>::block_index() {
6001 return idx;
6002 }
6003
6004 template <size_t STEP_SIZE>
6005 is_utf8_really_inline bool buf_block_reader<STEP_SIZE>::has_full_block() const {
6006 return idx < lenminusstep;
6007 }
6008
6009 template <size_t STEP_SIZE>
6010 is_utf8_really_inline const uint8_t *
6011 buf_block_reader<STEP_SIZE>::full_block() const {
6012 return &buf[idx];
6013 }
6014
6015 template <size_t STEP_SIZE>
6016 is_utf8_really_inline size_t
6017 buf_block_reader<STEP_SIZE>::get_remainder(uint8_t *dst) const {
6018 if (len == idx) {
6019 return 0;
6020 } // memcpy(dst, null, 0) will trigger an error with some sanitizers
6021 std::memset(dst, 0x20,
6022 STEP_SIZE); // std::memset STEP_SIZE because it's more efficient
6023 // to write out 8 or 16 bytes at once.
6024 std::memcpy(dst, buf + idx, len - idx);
6025 return len - idx;
6026 }
6027
6028 template <size_t STEP_SIZE>
6029 is_utf8_really_inline void buf_block_reader<STEP_SIZE>::advance() {
6030 idx += STEP_SIZE;
6031 }
6032
6033 } // unnamed namespace
6034 } // namespace ppc64
6035 } // namespace is_utf8_internals
6036
6037 namespace is_utf8_internals {
6038 namespace ppc64 {
6039 namespace {
6040 namespace utf8_validation {
6041
6042 using namespace simd;
6043
6044 is_utf8_really_inline simd8<uint8_t>
6045 check_special_cases(const simd8<uint8_t> input, const simd8<uint8_t> prev1) {
6046 // Bit 0 = Too Short (lead byte/ASCII followed by lead byte/ASCII)
6047 // Bit 1 = Too Long (ASCII followed by continuation)
6048 // Bit 2 = Overlong 3-byte
6049 // Bit 4 = Surrogate
6050 // Bit 5 = Overlong 2-byte
6051 // Bit 7 = Two Continuations
6052 constexpr const uint8_t TOO_SHORT = 1 << 0; // 11______ 0_______
6053 // 11______ 11______
6054 constexpr const uint8_t TOO_LONG = 1 << 1; // 0_______ 10______
6055 constexpr const uint8_t OVERLONG_3 = 1 << 2; // 11100000 100_____
6056 constexpr const uint8_t SURROGATE = 1 << 4; // 11101101 101_____
6057 constexpr const uint8_t OVERLONG_2 = 1 << 5; // 1100000_ 10______
6058 constexpr const uint8_t TWO_CONTS = 1 << 7; // 10______ 10______
6059 constexpr const uint8_t TOO_LARGE = 1 << 3; // 11110100 1001____
6060 // 11110100 101_____
6061 // 11110101 1001____
6062 // 11110101 101_____
6063 // 1111011_ 1001____
6064 // 1111011_ 101_____
6065 // 11111___ 1001____
6066 // 11111___ 101_____
6067 constexpr const uint8_t TOO_LARGE_1000 = 1 << 6;
6068 // 11110101 1000____
6069 // 1111011_ 1000____
6070 // 11111___ 1000____
6071 constexpr const uint8_t OVERLONG_4 = 1 << 6; // 11110000 1000____
6072
6073 const simd8<uint8_t> byte_1_high = prev1.shr<4>().lookup_16<uint8_t>(
6074 // 0_______ ________ <ASCII in byte 1>
6075 TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG,
6076 TOO_LONG,
6077 // 10______ ________ <continuation in byte 1>
6078 TWO_CONTS, TWO_CONTS, TWO_CONTS, TWO_CONTS,
6079 // 1100____ ________ <two byte lead in byte 1>
6080 TOO_SHORT | OVERLONG_2,
6081 // 1101____ ________ <two byte lead in byte 1>
6082 TOO_SHORT,
6083 // 1110____ ________ <three byte lead in byte 1>
6084 TOO_SHORT | OVERLONG_3 | SURROGATE,
6085 // 1111____ ________ <four+ byte lead in byte 1>
6086 TOO_SHORT | TOO_LARGE | TOO_LARGE_1000 | OVERLONG_4);
6087 constexpr const uint8_t CARRY =
6088 TOO_SHORT | TOO_LONG | TWO_CONTS; // These all have ____ in byte 1 .
6089 const simd8<uint8_t> byte_1_low =
6090 (prev1 & 0x0F)
6091 .lookup_16<uint8_t>(
6092 // ____0000 ________
6093 CARRY | OVERLONG_3 | OVERLONG_2 | OVERLONG_4,
6094 // ____0001 ________
6095 CARRY | OVERLONG_2,
6096 // ____001_ ________
6097 CARRY, CARRY,
6098
6099 // ____0100 ________
6100 CARRY | TOO_LARGE,
6101 // ____0101 ________
6102 CARRY | TOO_LARGE | TOO_LARGE_1000,
6103 // ____011_ ________
6104 CARRY | TOO_LARGE | TOO_LARGE_1000,
6105 CARRY | TOO_LARGE | TOO_LARGE_1000,
6106
6107 // ____1___ ________
6108 CARRY | TOO_LARGE | TOO_LARGE_1000,
6109 CARRY | TOO_LARGE | TOO_LARGE_1000,
6110 CARRY | TOO_LARGE | TOO_LARGE_1000,
6111 CARRY | TOO_LARGE | TOO_LARGE_1000,
6112 CARRY | TOO_LARGE | TOO_LARGE_1000,
6113 // ____1101 ________
6114 CARRY | TOO_LARGE | TOO_LARGE_1000 | SURROGATE,
6115 CARRY | TOO_LARGE | TOO_LARGE_1000,
6116 CARRY | TOO_LARGE | TOO_LARGE_1000);
6117 const simd8<uint8_t> byte_2_high = input.shr<4>().lookup_16<uint8_t>(
6118 // ________ 0_______ <ASCII in byte 2>
6119 TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT,
6120 TOO_SHORT, TOO_SHORT,
6121
6122 // ________ 1000____
6123 TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE_1000 |
6124 OVERLONG_4,
6125 // ________ 1001____
6126 TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE,
6127 // ________ 101_____
6128 TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE,
6129 TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE,
6130
6131 // ________ 11______
6132 TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT);
6133 return (byte_1_high & byte_1_low & byte_2_high);
6134 }
6135 is_utf8_really_inline simd8<uint8_t>
6136 check_multibyte_lengths(const simd8<uint8_t> input,
6137 const simd8<uint8_t> prev_input,
6138 const simd8<uint8_t> sc) {
6139 simd8<uint8_t> prev2 = input.prev<2>(prev_input);
6140 simd8<uint8_t> prev3 = input.prev<3>(prev_input);
6141 simd8<uint8_t> must23 =
6142 simd8<uint8_t>(must_be_2_3_continuation(prev2, prev3));
6143 simd8<uint8_t> must23_80 = must23 & uint8_t(0x80);
6144 return must23_80 ^ sc;
6145 }
6146
6147 //
6148 // Return nonzero if there are incomplete multibyte characters at the end of the
6149 // block: e.g. if there is a 4-byte character, but it's 3 bytes from the end.
6150 //
6151 is_utf8_really_inline simd8<uint8_t> is_incomplete(const simd8<uint8_t> input) {
6152 // If the previous input's last 3 bytes match this, they're too short (they
6153 // ended at EOF):
6154 // ... 1111____ 111_____ 11______
6155 static const uint8_t max_array[32] = {255,
6156 255,
6157 255,
6158 255,
6159 255,
6160 255,
6161 255,
6162 255,
6163 255,
6164 255,
6165 255,
6166 255,
6167 255,
6168 255,
6169 255,
6170 255,
6171 255,
6172 255,
6173 255,
6174 255,
6175 255,
6176 255,
6177 255,
6178 255,
6179 255,
6180 255,
6181 255,
6182 255,
6183 255,
6184 0b11110000u - 1,
6185 0b11100000u - 1,
6186 0b11000000u - 1};
6187 const simd8<uint8_t> max_value(
6188 &max_array[sizeof(max_array) - sizeof(simd8<uint8_t>)]);
6189 return input.gt_bits(max_value);
6190 }
6191
6192 struct utf8_checker {
6193 // If this is nonzero, there has been a UTF-8 error.
6194 simd8<uint8_t> error;
6195 // The last input we received
6196 simd8<uint8_t> prev_input_block;
6197 // Whether the last input we received was incomplete (used for ASCII fast
6198 // path)
6199 simd8<uint8_t> prev_incomplete;
6200
6201 //
6202 // Check whether the current bytes are valid UTF-8.
6203 //
6204 is_utf8_really_inline void check_utf8_bytes(const simd8<uint8_t> input,
6205 const simd8<uint8_t> prev_input) {
6206 // Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+
6207 // lead bytes (2, 3, 4-byte leads become large positive numbers instead of
6208 // small negative numbers)
6209 simd8<uint8_t> prev1 = input.prev<1>(prev_input);
6210 simd8<uint8_t> sc = check_special_cases(input, prev1);
6211 this->error |= check_multibyte_lengths(input, prev_input, sc);
6212 }
6213
6214 // The only problem that can happen at EOF is that a multibyte character is
6215 // too short or a byte value too large in the last bytes: check_special_cases
6216 // only checks for bytes too large in the first of two bytes.
6217 is_utf8_really_inline void check_eof() {
6218 // If the previous block had incomplete UTF-8 characters at the end, an
6219 // ASCII block can't possibly finish them.
6220 this->error |= this->prev_incomplete;
6221 }
6222
6223 is_utf8_really_inline void check_next_input(const simd8x64<uint8_t> &input) {
6224 if (is_utf8_likely(is_ascii(input))) {
6225 this->error |= this->prev_incomplete;
6226 } else {
6227 // you might think that a for-loop would work, but under Visual Studio, it
6228 // is not good enough.
6229 static_assert((simd8x64<uint8_t>::NUM_CHUNKS == 2) ||
6230 (simd8x64<uint8_t>::NUM_CHUNKS == 4),
6231 "We support either two or four chunks per 64-byte block.");
6232 if (simd8x64<uint8_t>::NUM_CHUNKS == 2) {
6233 this->check_utf8_bytes(input.chunks[0], this->prev_input_block);
6234 this->check_utf8_bytes(input.chunks[1], input.chunks[0]);
6235 } else if (simd8x64<uint8_t>::NUM_CHUNKS == 4) {
6236 this->check_utf8_bytes(input.chunks[0], this->prev_input_block);
6237 this->check_utf8_bytes(input.chunks[1], input.chunks[0]);
6238 this->check_utf8_bytes(input.chunks[2], input.chunks[1]);
6239 this->check_utf8_bytes(input.chunks[3], input.chunks[2]);
6240 }
6241 this->prev_incomplete =
6242 is_incomplete(input.chunks[simd8x64<uint8_t>::NUM_CHUNKS - 1]);
6243 this->prev_input_block = input.chunks[simd8x64<uint8_t>::NUM_CHUNKS - 1];
6244 }
6245 }
6246
6247 // do not forget to call check_eof!
6248 is_utf8_really_inline bool errors() const {
6249 return this->error.any_bits_set_anywhere();
6250 }
6251
6252 }; // struct utf8_checker
6253 } // namespace utf8_validation
6254
6255 using utf8_validation::utf8_checker;
6256
6257 } // unnamed namespace
6258 } // namespace ppc64
6259 } // namespace is_utf8_internals
6260
6261 namespace is_utf8_internals {
6262 namespace ppc64 {
6263 namespace {
6264 namespace utf8_validation {
6265
6266 /**
6267 * Validates that the string is actual UTF-8.
6268 */
6269 template <class checker>
6270 bool generic_validate_utf8(const uint8_t *input, size_t length) {
6271 checker c{};
6272 buf_block_reader<64> reader(input, length);
6273 while (reader.has_full_block()) {
6274 simd::simd8x64<uint8_t> in(reader.full_block());
6275 c.check_next_input(in);
6276 reader.advance();
6277 }
6278 uint8_t block[64]{};
6279 reader.get_remainder(block);
6280 simd::simd8x64<uint8_t> in(block);
6281 c.check_next_input(in);
6282 reader.advance();
6283 c.check_eof();
6284 return !c.errors();
6285 }
6286
6287 bool generic_validate_utf8(const char *input, size_t length) {
6288 return generic_validate_utf8<utf8_checker>(
6289 reinterpret_cast<const uint8_t *>(input), length);
6290 }
6291
6292 } // namespace utf8_validation
6293 } // unnamed namespace
6294 } // namespace ppc64
6295 } // namespace is_utf8_internals
6296
6297 //
6298 // Implementation-specific overrides
6299 //
6300 namespace is_utf8_internals {
6301 namespace ppc64 {
6302
6303 is_utf8_warn_unused bool
6304 implementation::validate_utf8(const char *buf, size_t len) const noexcept {
6305 return ppc64::utf8_validation::generic_validate_utf8(buf, len);
6306 }
6307
6308 } // namespace ppc64
6309 } // namespace is_utf8_internals
6310
6311 #endif
6312 #if IS_UTF8_IMPLEMENTATION_WESTMERE
6313
6314 // redefining IS_UTF8_IMPLEMENTATION to "westmere"
6315 // #define IS_UTF8_IMPLEMENTATION westmere
6316 IS_UTF8_TARGET_WESTMERE
6317
6318 namespace is_utf8_internals {
6319 namespace westmere {
6320 namespace {
6321 #ifndef IS_UTF8_WESTMERE_H
6322 #error "westmere.h must be included"
6323 #endif
6324 using namespace simd;
6325
6326 is_utf8_really_inline bool is_ascii(const simd8x64<uint8_t> &input) {
6327 return input.reduce_or().is_ascii();
6328 }
6329
6330 is_utf8_unused is_utf8_really_inline simd8<bool>
6331 must_be_continuation(const simd8<uint8_t> prev1, const simd8<uint8_t> prev2,
6332 const simd8<uint8_t> prev3) {
6333 simd8<uint8_t> is_second_byte =
6334 prev1.saturating_sub(0b11000000u - 1); // Only 11______ will be > 0
6335 simd8<uint8_t> is_third_byte =
6336 prev2.saturating_sub(0b11100000u - 1); // Only 111_____ will be > 0
6337 simd8<uint8_t> is_fourth_byte =
6338 prev3.saturating_sub(0b11110000u - 1); // Only 1111____ will be > 0
6339 // Caller requires a bool (all 1's). All values resulting from the subtraction
6340 // will be <= 64, so signed comparison is fine.
6341 return simd8<int8_t>(is_second_byte | is_third_byte | is_fourth_byte) >
6342 int8_t(0);
6343 }
6344
6345 is_utf8_really_inline simd8<bool>
6346 must_be_2_3_continuation(const simd8<uint8_t> prev2,
6347 const simd8<uint8_t> prev3) {
6348 simd8<uint8_t> is_third_byte =
6349 prev2.saturating_sub(0b11100000u - 1); // Only 111_____ will be > 0
6350 simd8<uint8_t> is_fourth_byte =
6351 prev3.saturating_sub(0b11110000u - 1); // Only 1111____ will be > 0
6352 // Caller requires a bool (all 1's). All values resulting from the subtraction
6353 // will be <= 64, so signed comparison is fine.
6354 return simd8<int8_t>(is_third_byte | is_fourth_byte) > int8_t(0);
6355 }
6356
6357 } // unnamed namespace
6358 } // namespace westmere
6359 } // namespace is_utf8_internals
6360
6361 namespace is_utf8_internals {
6362 namespace westmere {
6363 namespace {
6364
6365 // Walks through a buffer in block-sized increments, loading the last part with
6366 // spaces
6367 template <size_t STEP_SIZE> struct buf_block_reader {
6368 public:
6369 is_utf8_really_inline buf_block_reader(const uint8_t *_buf, size_t _len);
6370 is_utf8_really_inline size_t block_index();
6371 is_utf8_really_inline bool has_full_block() const;
6372 is_utf8_really_inline const uint8_t *full_block() const;
6373 /**
6374 * Get the last block, padded with spaces.
6375 *
6376 * There will always be a last block, with at least 1 byte, unless len == 0
6377 * (in which case this function fills the buffer with spaces and returns 0. In
6378 * particular, if len == STEP_SIZE there will be 0 full_blocks and 1 remainder
6379 * block with STEP_SIZE bytes and no spaces for padding.
6380 *
6381 * @return the number of effective characters in the last block.
6382 */
6383 is_utf8_really_inline size_t get_remainder(uint8_t *dst) const;
6384 is_utf8_really_inline void advance();
6385
6386 private:
6387 const uint8_t *buf;
6388 const size_t len;
6389 const size_t lenminusstep;
6390 size_t idx;
6391 };
6392
6393 // Routines to print masks and text for debugging bitmask operations
6394 is_utf8_unused static char *format_input_text_64(const uint8_t *text) {
6395 static char *buf =
6396 reinterpret_cast<char *>(malloc(sizeof(simd8x64<uint8_t>) + 1));
6397 for (size_t i = 0; i < sizeof(simd8x64<uint8_t>); i++) {
6398 buf[i] = int8_t(text[i]) < ' ' ? '_' : int8_t(text[i]);
6399 }
6400 buf[sizeof(simd8x64<uint8_t>)] = '\0';
6401 return buf;
6402 }
6403
6404 // Routines to print masks and text for debugging bitmask operations
6405 is_utf8_unused static char *format_input_text(const simd8x64<uint8_t> &in) {
6406 static char *buf =
6407 reinterpret_cast<char *>(malloc(sizeof(simd8x64<uint8_t>) + 1));
6408 in.store(reinterpret_cast<uint8_t *>(buf));
6409 for (size_t i = 0; i < sizeof(simd8x64<uint8_t>); i++) {
6410 if (buf[i] < ' ') {
6411 buf[i] = '_';
6412 }
6413 }
6414 buf[sizeof(simd8x64<uint8_t>)] = '\0';
6415 return buf;
6416 }
6417
6418 is_utf8_unused static char *format_mask(uint64_t mask) {
6419 static char *buf = reinterpret_cast<char *>(malloc(64 + 1));
6420 for (size_t i = 0; i < 64; i++) {
6421 buf[i] = (mask & (size_t(1) << i)) ? 'X' : ' ';
6422 }
6423 buf[64] = '\0';
6424 return buf;
6425 }
6426
6427 template <size_t STEP_SIZE>
6428 is_utf8_really_inline
6429 buf_block_reader<STEP_SIZE>::buf_block_reader(const uint8_t *_buf, size_t _len)
6430 : buf{_buf}, len{_len},
6431 lenminusstep{len < STEP_SIZE ? 0 : len - STEP_SIZE}, idx{0} {}
6432
6433 template <size_t STEP_SIZE>
6434 is_utf8_really_inline size_t buf_block_reader<STEP_SIZE>::block_index() {
6435 return idx;
6436 }
6437
6438 template <size_t STEP_SIZE>
6439 is_utf8_really_inline bool buf_block_reader<STEP_SIZE>::has_full_block() const {
6440 return idx < lenminusstep;
6441 }
6442
6443 template <size_t STEP_SIZE>
6444 is_utf8_really_inline const uint8_t *
6445 buf_block_reader<STEP_SIZE>::full_block() const {
6446 return &buf[idx];
6447 }
6448
6449 template <size_t STEP_SIZE>
6450 is_utf8_really_inline size_t
6451 buf_block_reader<STEP_SIZE>::get_remainder(uint8_t *dst) const {
6452 if (len == idx) {
6453 return 0;
6454 } // memcpy(dst, null, 0) will trigger an error with some sanitizers
6455 std::memset(dst, 0x20,
6456 STEP_SIZE); // std::memset STEP_SIZE because it's more efficient
6457 // to write out 8 or 16 bytes at once.
6458 std::memcpy(dst, buf + idx, len - idx);
6459 return len - idx;
6460 }
6461
6462 template <size_t STEP_SIZE>
6463 is_utf8_really_inline void buf_block_reader<STEP_SIZE>::advance() {
6464 idx += STEP_SIZE;
6465 }
6466
6467 } // unnamed namespace
6468 } // namespace westmere
6469 } // namespace is_utf8_internals
6470
6471 namespace is_utf8_internals {
6472 namespace westmere {
6473 namespace {
6474 namespace utf8_validation {
6475
6476 using namespace simd;
6477
6478 is_utf8_really_inline simd8<uint8_t>
6479 check_special_cases(const simd8<uint8_t> input, const simd8<uint8_t> prev1) {
6480 // Bit 0 = Too Short (lead byte/ASCII followed by lead byte/ASCII)
6481 // Bit 1 = Too Long (ASCII followed by continuation)
6482 // Bit 2 = Overlong 3-byte
6483 // Bit 4 = Surrogate
6484 // Bit 5 = Overlong 2-byte
6485 // Bit 7 = Two Continuations
6486 constexpr const uint8_t TOO_SHORT = 1 << 0; // 11______ 0_______
6487 // 11______ 11______
6488 constexpr const uint8_t TOO_LONG = 1 << 1; // 0_______ 10______
6489 constexpr const uint8_t OVERLONG_3 = 1 << 2; // 11100000 100_____
6490 constexpr const uint8_t SURROGATE = 1 << 4; // 11101101 101_____
6491 constexpr const uint8_t OVERLONG_2 = 1 << 5; // 1100000_ 10______
6492 constexpr const uint8_t TWO_CONTS = 1 << 7; // 10______ 10______
6493 constexpr const uint8_t TOO_LARGE = 1 << 3; // 11110100 1001____
6494 // 11110100 101_____
6495 // 11110101 1001____
6496 // 11110101 101_____
6497 // 1111011_ 1001____
6498 // 1111011_ 101_____
6499 // 11111___ 1001____
6500 // 11111___ 101_____
6501 constexpr const uint8_t TOO_LARGE_1000 = 1 << 6;
6502 // 11110101 1000____
6503 // 1111011_ 1000____
6504 // 11111___ 1000____
6505 constexpr const uint8_t OVERLONG_4 = 1 << 6; // 11110000 1000____
6506
6507 const simd8<uint8_t> byte_1_high = prev1.shr<4>().lookup_16<uint8_t>(
6508 // 0_______ ________ <ASCII in byte 1>
6509 TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG,
6510 TOO_LONG,
6511 // 10______ ________ <continuation in byte 1>
6512 TWO_CONTS, TWO_CONTS, TWO_CONTS, TWO_CONTS,
6513 // 1100____ ________ <two byte lead in byte 1>
6514 TOO_SHORT | OVERLONG_2,
6515 // 1101____ ________ <two byte lead in byte 1>
6516 TOO_SHORT,
6517 // 1110____ ________ <three byte lead in byte 1>
6518 TOO_SHORT | OVERLONG_3 | SURROGATE,
6519 // 1111____ ________ <four+ byte lead in byte 1>
6520 TOO_SHORT | TOO_LARGE | TOO_LARGE_1000 | OVERLONG_4);
6521 constexpr const uint8_t CARRY =
6522 TOO_SHORT | TOO_LONG | TWO_CONTS; // These all have ____ in byte 1 .
6523 const simd8<uint8_t> byte_1_low =
6524 (prev1 & 0x0F)
6525 .lookup_16<uint8_t>(
6526 // ____0000 ________
6527 CARRY | OVERLONG_3 | OVERLONG_2 | OVERLONG_4,
6528 // ____0001 ________
6529 CARRY | OVERLONG_2,
6530 // ____001_ ________
6531 CARRY, CARRY,
6532
6533 // ____0100 ________
6534 CARRY | TOO_LARGE,
6535 // ____0101 ________
6536 CARRY | TOO_LARGE | TOO_LARGE_1000,
6537 // ____011_ ________
6538 CARRY | TOO_LARGE | TOO_LARGE_1000,
6539 CARRY | TOO_LARGE | TOO_LARGE_1000,
6540
6541 // ____1___ ________
6542 CARRY | TOO_LARGE | TOO_LARGE_1000,
6543 CARRY | TOO_LARGE | TOO_LARGE_1000,
6544 CARRY | TOO_LARGE | TOO_LARGE_1000,
6545 CARRY | TOO_LARGE | TOO_LARGE_1000,
6546 CARRY | TOO_LARGE | TOO_LARGE_1000,
6547 // ____1101 ________
6548 CARRY | TOO_LARGE | TOO_LARGE_1000 | SURROGATE,
6549 CARRY | TOO_LARGE | TOO_LARGE_1000,
6550 CARRY | TOO_LARGE | TOO_LARGE_1000);
6551 const simd8<uint8_t> byte_2_high = input.shr<4>().lookup_16<uint8_t>(
6552 // ________ 0_______ <ASCII in byte 2>
6553 TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT,
6554 TOO_SHORT, TOO_SHORT,
6555
6556 // ________ 1000____
6557 TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE_1000 |
6558 OVERLONG_4,
6559 // ________ 1001____
6560 TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE,
6561 // ________ 101_____
6562 TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE,
6563 TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE,
6564
6565 // ________ 11______
6566 TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT);
6567 return (byte_1_high & byte_1_low & byte_2_high);
6568 }
6569 is_utf8_really_inline simd8<uint8_t>
6570 check_multibyte_lengths(const simd8<uint8_t> input,
6571 const simd8<uint8_t> prev_input,
6572 const simd8<uint8_t> sc) {
6573 simd8<uint8_t> prev2 = input.prev<2>(prev_input);
6574 simd8<uint8_t> prev3 = input.prev<3>(prev_input);
6575 simd8<uint8_t> must23 =
6576 simd8<uint8_t>(must_be_2_3_continuation(prev2, prev3));
6577 simd8<uint8_t> must23_80 = must23 & uint8_t(0x80);
6578 return must23_80 ^ sc;
6579 }
6580
6581 //
6582 // Return nonzero if there are incomplete multibyte characters at the end of the
6583 // block: e.g. if there is a 4-byte character, but it's 3 bytes from the end.
6584 //
6585 is_utf8_really_inline simd8<uint8_t> is_incomplete(const simd8<uint8_t> input) {
6586 // If the previous input's last 3 bytes match this, they're too short (they
6587 // ended at EOF):
6588 // ... 1111____ 111_____ 11______
6589 static const uint8_t max_array[32] = {255,
6590 255,
6591 255,
6592 255,
6593 255,
6594 255,
6595 255,
6596 255,
6597 255,
6598 255,
6599 255,
6600 255,
6601 255,
6602 255,
6603 255,
6604 255,
6605 255,
6606 255,
6607 255,
6608 255,
6609 255,
6610 255,
6611 255,
6612 255,
6613 255,
6614 255,
6615 255,
6616 255,
6617 255,
6618 0b11110000u - 1,
6619 0b11100000u - 1,
6620 0b11000000u - 1};
6621 const simd8<uint8_t> max_value(
6622 &max_array[sizeof(max_array) - sizeof(simd8<uint8_t>)]);
6623 return input.gt_bits(max_value);
6624 }
6625
6626 struct utf8_checker {
6627 // If this is nonzero, there has been a UTF-8 error.
6628 simd8<uint8_t> error;
6629 // The last input we received
6630 simd8<uint8_t> prev_input_block;
6631 // Whether the last input we received was incomplete (used for ASCII fast
6632 // path)
6633 simd8<uint8_t> prev_incomplete;
6634
6635 //
6636 // Check whether the current bytes are valid UTF-8.
6637 //
6638 is_utf8_really_inline void check_utf8_bytes(const simd8<uint8_t> input,
6639 const simd8<uint8_t> prev_input) {
6640 // Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+
6641 // lead bytes (2, 3, 4-byte leads become large positive numbers instead of
6642 // small negative numbers)
6643 simd8<uint8_t> prev1 = input.prev<1>(prev_input);
6644 simd8<uint8_t> sc = check_special_cases(input, prev1);
6645 this->error |= check_multibyte_lengths(input, prev_input, sc);
6646 }
6647
6648 // The only problem that can happen at EOF is that a multibyte character is
6649 // too short or a byte value too large in the last bytes: check_special_cases
6650 // only checks for bytes too large in the first of two bytes.
6651 is_utf8_really_inline void check_eof() {
6652 // If the previous block had incomplete UTF-8 characters at the end, an
6653 // ASCII block can't possibly finish them.
6654 this->error |= this->prev_incomplete;
6655 }
6656
6657 is_utf8_really_inline void check_next_input(const simd8x64<uint8_t> &input) {
6658 if (is_utf8_likely(is_ascii(input))) {
6659 this->error |= this->prev_incomplete;
6660 } else {
6661 // you might think that a for-loop would work, but under Visual Studio, it
6662 // is not good enough.
6663 static_assert((simd8x64<uint8_t>::NUM_CHUNKS == 2) ||
6664 (simd8x64<uint8_t>::NUM_CHUNKS == 4),
6665 "We support either two or four chunks per 64-byte block.");
6666 if (simd8x64<uint8_t>::NUM_CHUNKS == 2) {
6667 this->check_utf8_bytes(input.chunks[0], this->prev_input_block);
6668 this->check_utf8_bytes(input.chunks[1], input.chunks[0]);
6669 } else if (simd8x64<uint8_t>::NUM_CHUNKS == 4) {
6670 this->check_utf8_bytes(input.chunks[0], this->prev_input_block);
6671 this->check_utf8_bytes(input.chunks[1], input.chunks[0]);
6672 this->check_utf8_bytes(input.chunks[2], input.chunks[1]);
6673 this->check_utf8_bytes(input.chunks[3], input.chunks[2]);
6674 }
6675 this->prev_incomplete =
6676 is_incomplete(input.chunks[simd8x64<uint8_t>::NUM_CHUNKS - 1]);
6677 this->prev_input_block = input.chunks[simd8x64<uint8_t>::NUM_CHUNKS - 1];
6678 }
6679 }
6680
6681 // do not forget to call check_eof!
6682 is_utf8_really_inline bool errors() const {
6683 return this->error.any_bits_set_anywhere();
6684 }
6685
6686 }; // struct utf8_checker
6687 } // namespace utf8_validation
6688
6689 using utf8_validation::utf8_checker;
6690
6691 } // unnamed namespace
6692 } // namespace westmere
6693 } // namespace is_utf8_internals
6694
6695 namespace is_utf8_internals {
6696 namespace westmere {
6697 namespace {
6698 namespace utf8_validation {
6699
6700 /**
6701 * Validates that the string is actual UTF-8.
6702 */
6703 template <class checker>
6704 bool generic_validate_utf8(const uint8_t *input, size_t length) {
6705 checker c{};
6706 buf_block_reader<64> reader(input, length);
6707 while (reader.has_full_block()) {
6708 simd::simd8x64<uint8_t> in(reader.full_block());
6709 c.check_next_input(in);
6710 reader.advance();
6711 }
6712 uint8_t block[64]{};
6713 reader.get_remainder(block);
6714 simd::simd8x64<uint8_t> in(block);
6715 c.check_next_input(in);
6716 reader.advance();
6717 c.check_eof();
6718 return !c.errors();
6719 }
6720
6721 bool generic_validate_utf8(const char *input, size_t length) {
6722 return generic_validate_utf8<utf8_checker>(
6723 reinterpret_cast<const uint8_t *>(input), length);
6724 }
6725
6726 } // namespace utf8_validation
6727 } // unnamed namespace
6728 } // namespace westmere
6729 } // namespace is_utf8_internals
6730 //
6731 // Implementation-specific overrides
6732 //
6733
6734 namespace is_utf8_internals {
6735 namespace westmere {
6736
6737 is_utf8_warn_unused bool
6738 implementation::validate_utf8(const char *buf, size_t len) const noexcept {
6739 return westmere::utf8_validation::generic_validate_utf8(buf, len);
6740 }
6741
6742 } // namespace westmere
6743 } // namespace is_utf8_internals
6744
6745 IS_UTF8_UNTARGET_REGION
6746
6747 #endif
6748
6749 IS_UTF8_POP_DISABLE_WARNINGS
6750
6751 extern "C" {
6752 bool is_utf8(const char *src, size_t len) {
6753 return is_utf8_internals::validate_utf8(src, len);
6754 }
6755 }