| 1 |
/* Finding the optimal width of unicode characters in a buffer */ |
| 2 |
|
| 3 |
/* find_max_char for one-byte will work for bytes objects as well. */ |
| 4 |
#if !STRINGLIB_IS_UNICODE && STRINGLIB_SIZEOF_CHAR > 1 |
| 5 |
# error "find_max_char.h is specific to Unicode" |
| 6 |
#endif |
| 7 |
|
| 8 |
/* Mask to quickly check whether a C 'size_t' contains a |
| 9 |
non-ASCII, UTF8-encoded char. */ |
| 10 |
#if (SIZEOF_SIZE_T == 8) |
| 11 |
# define UCS1_ASCII_CHAR_MASK 0x8080808080808080ULL |
| 12 |
#elif (SIZEOF_SIZE_T == 4) |
| 13 |
# define UCS1_ASCII_CHAR_MASK 0x80808080U |
| 14 |
#else |
| 15 |
# error C 'size_t' size should be either 4 or 8! |
| 16 |
#endif |
| 17 |
|
| 18 |
#if STRINGLIB_SIZEOF_CHAR == 1 |
| 19 |
|
| 20 |
Py_LOCAL_INLINE(Py_UCS4) |
| 21 |
STRINGLIB(find_max_char)(const STRINGLIB_CHAR *begin, const STRINGLIB_CHAR *end) |
| 22 |
{ |
| 23 |
const unsigned char *p = (const unsigned char *) begin; |
| 24 |
const unsigned char *_end = (const unsigned char *)end; |
| 25 |
|
| 26 |
while (p < _end) { |
| 27 |
if (_Py_IS_ALIGNED(p, ALIGNOF_SIZE_T)) { |
| 28 |
/* Help register allocation */ |
| 29 |
const unsigned char *_p = p; |
| 30 |
while (_p + SIZEOF_SIZE_T <= _end) { |
| 31 |
size_t value = *(const size_t *) _p; |
| 32 |
if (value & UCS1_ASCII_CHAR_MASK) |
| 33 |
return 255; |
| 34 |
_p += SIZEOF_SIZE_T; |
| 35 |
} |
| 36 |
p = _p; |
| 37 |
if (p == _end) |
| 38 |
break; |
| 39 |
} |
| 40 |
if (*p++ & 0x80) |
| 41 |
return 255; |
| 42 |
} |
| 43 |
return 127; |
| 44 |
} |
| 45 |
|
| 46 |
#undef ASCII_CHAR_MASK |
| 47 |
|
| 48 |
#else /* STRINGLIB_SIZEOF_CHAR == 1 */ |
| 49 |
|
| 50 |
#define MASK_ASCII 0xFFFFFF80 |
| 51 |
#define MASK_UCS1 0xFFFFFF00 |
| 52 |
#define MASK_UCS2 0xFFFF0000 |
| 53 |
|
| 54 |
#define MAX_CHAR_ASCII 0x7f |
| 55 |
#define MAX_CHAR_UCS1 0xff |
| 56 |
#define MAX_CHAR_UCS2 0xffff |
| 57 |
#define MAX_CHAR_UCS4 0x10ffff |
| 58 |
|
| 59 |
Py_LOCAL_INLINE(Py_UCS4) |
| 60 |
STRINGLIB(find_max_char)(const STRINGLIB_CHAR *begin, const STRINGLIB_CHAR *end) |
| 61 |
{ |
| 62 |
#if STRINGLIB_SIZEOF_CHAR == 2 |
| 63 |
const Py_UCS4 mask_limit = MASK_UCS1; |
| 64 |
const Py_UCS4 max_char_limit = MAX_CHAR_UCS2; |
| 65 |
#elif STRINGLIB_SIZEOF_CHAR == 4 |
| 66 |
const Py_UCS4 mask_limit = MASK_UCS2; |
| 67 |
const Py_UCS4 max_char_limit = MAX_CHAR_UCS4; |
| 68 |
#else |
| 69 |
#error Invalid STRINGLIB_SIZEOF_CHAR (must be 1, 2 or 4) |
| 70 |
#endif |
| 71 |
Py_UCS4 mask; |
| 72 |
Py_ssize_t n = end - begin; |
| 73 |
const STRINGLIB_CHAR *p = begin; |
| 74 |
const STRINGLIB_CHAR *unrolled_end = begin + _Py_SIZE_ROUND_DOWN(n, 4); |
| 75 |
Py_UCS4 max_char; |
| 76 |
|
| 77 |
max_char = MAX_CHAR_ASCII; |
| 78 |
mask = MASK_ASCII; |
| 79 |
while (p < unrolled_end) { |
| 80 |
STRINGLIB_CHAR bits = p[0] | p[1] | p[2] | p[3]; |
| 81 |
if (bits & mask) { |
| 82 |
if (mask == mask_limit) { |
| 83 |
/* Limit reached */ |
| 84 |
return max_char_limit; |
| 85 |
} |
| 86 |
if (mask == MASK_ASCII) { |
| 87 |
max_char = MAX_CHAR_UCS1; |
| 88 |
mask = MASK_UCS1; |
| 89 |
} |
| 90 |
else { |
| 91 |
/* mask can't be MASK_UCS2 because of mask_limit above */ |
| 92 |
assert(mask == MASK_UCS1); |
| 93 |
max_char = MAX_CHAR_UCS2; |
| 94 |
mask = MASK_UCS2; |
| 95 |
} |
| 96 |
/* We check the new mask on the same chars in the next iteration */ |
| 97 |
continue; |
| 98 |
} |
| 99 |
p += 4; |
| 100 |
} |
| 101 |
while (p < end) { |
| 102 |
if (p[0] & mask) { |
| 103 |
if (mask == mask_limit) { |
| 104 |
/* Limit reached */ |
| 105 |
return max_char_limit; |
| 106 |
} |
| 107 |
if (mask == MASK_ASCII) { |
| 108 |
max_char = MAX_CHAR_UCS1; |
| 109 |
mask = MASK_UCS1; |
| 110 |
} |
| 111 |
else { |
| 112 |
/* mask can't be MASK_UCS2 because of mask_limit above */ |
| 113 |
assert(mask == MASK_UCS1); |
| 114 |
max_char = MAX_CHAR_UCS2; |
| 115 |
mask = MASK_UCS2; |
| 116 |
} |
| 117 |
/* We check the new mask on the same chars in the next iteration */ |
| 118 |
continue; |
| 119 |
} |
| 120 |
p++; |
| 121 |
} |
| 122 |
return max_char; |
| 123 |
} |
| 124 |
|
| 125 |
#undef MASK_ASCII |
| 126 |
#undef MASK_UCS1 |
| 127 |
#undef MASK_UCS2 |
| 128 |
#undef MAX_CHAR_ASCII |
| 129 |
#undef MAX_CHAR_UCS1 |
| 130 |
#undef MAX_CHAR_UCS2 |
| 131 |
#undef MAX_CHAR_UCS4 |
| 132 |
|
| 133 |
#endif /* STRINGLIB_SIZEOF_CHAR == 1 */ |
| 134 |
|