| 1 |
/* |
| 2 |
* Process 2x16 bytes in each iteration. |
| 3 |
* Comments removed for brevity. See range-neon.c for details. |
| 4 |
*/ |
| 5 |
#ifdef __aarch64__ |
| 6 |
|
| 7 |
#include <stdio.h> |
| 8 |
#include <stdint.h> |
| 9 |
#include <arm_neon.h> |
| 10 |
|
| 11 |
int utf8_naive(const unsigned char *data, int len); |
| 12 |
|
| 13 |
static const uint8_t _first_len_tbl[] = { |
| 14 |
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 3, |
| 15 |
}; |
| 16 |
|
| 17 |
static const uint8_t _first_range_tbl[] = { |
| 18 |
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 8, 8, |
| 19 |
}; |
| 20 |
|
| 21 |
static const uint8_t _range_min_tbl[] = { |
| 22 |
0x00, 0x80, 0x80, 0x80, 0xA0, 0x80, 0x90, 0x80, |
| 23 |
0xC2, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, |
| 24 |
}; |
| 25 |
static const uint8_t _range_max_tbl[] = { |
| 26 |
0x7F, 0xBF, 0xBF, 0xBF, 0xBF, 0x9F, 0xBF, 0x8F, |
| 27 |
0xF4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, |
| 28 |
}; |
| 29 |
|
| 30 |
static const uint8_t _range_adjust_tbl[] = { |
| 31 |
2, 3, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, |
| 32 |
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, |
| 33 |
}; |
| 34 |
|
| 35 |
/* Return 0 on success, -1 on error */ |
| 36 |
int utf8_range2(const unsigned char *data, int len) |
| 37 |
{ |
| 38 |
if (len >= 32) { |
| 39 |
uint8x16_t prev_input = vdupq_n_u8(0); |
| 40 |
uint8x16_t prev_first_len = vdupq_n_u8(0); |
| 41 |
|
| 42 |
const uint8x16_t first_len_tbl = vld1q_u8(_first_len_tbl); |
| 43 |
const uint8x16_t first_range_tbl = vld1q_u8(_first_range_tbl); |
| 44 |
const uint8x16_t range_min_tbl = vld1q_u8(_range_min_tbl); |
| 45 |
const uint8x16_t range_max_tbl = vld1q_u8(_range_max_tbl); |
| 46 |
const uint8x16x2_t range_adjust_tbl = vld2q_u8(_range_adjust_tbl); |
| 47 |
|
| 48 |
const uint8x16_t const_1 = vdupq_n_u8(1); |
| 49 |
const uint8x16_t const_2 = vdupq_n_u8(2); |
| 50 |
const uint8x16_t const_e0 = vdupq_n_u8(0xE0); |
| 51 |
|
| 52 |
uint8x16_t error1 = vdupq_n_u8(0); |
| 53 |
uint8x16_t error2 = vdupq_n_u8(0); |
| 54 |
uint8x16_t error3 = vdupq_n_u8(0); |
| 55 |
uint8x16_t error4 = vdupq_n_u8(0); |
| 56 |
|
| 57 |
while (len >= 32) { |
| 58 |
/******************* two blocks interleaved **********************/ |
| 59 |
|
| 60 |
#if defined(__GNUC__) && !defined(__clang__) && (__GNUC__ < 8) |
| 61 |
/* gcc doesn't support vldq1_u8_x2 until version 8 */ |
| 62 |
const uint8x16_t input_a = vld1q_u8(data); |
| 63 |
const uint8x16_t input_b = vld1q_u8(data + 16); |
| 64 |
#else |
| 65 |
/* Forces a double load on Clang */ |
| 66 |
const uint8x16x2_t input_pair = vld1q_u8_x2(data); |
| 67 |
const uint8x16_t input_a = input_pair.val[0]; |
| 68 |
const uint8x16_t input_b = input_pair.val[1]; |
| 69 |
#endif |
| 70 |
|
| 71 |
const uint8x16_t high_nibbles_a = vshrq_n_u8(input_a, 4); |
| 72 |
const uint8x16_t high_nibbles_b = vshrq_n_u8(input_b, 4); |
| 73 |
|
| 74 |
const uint8x16_t first_len_a = |
| 75 |
vqtbl1q_u8(first_len_tbl, high_nibbles_a); |
| 76 |
const uint8x16_t first_len_b = |
| 77 |
vqtbl1q_u8(first_len_tbl, high_nibbles_b); |
| 78 |
|
| 79 |
uint8x16_t range_a = vqtbl1q_u8(first_range_tbl, high_nibbles_a); |
| 80 |
uint8x16_t range_b = vqtbl1q_u8(first_range_tbl, high_nibbles_b); |
| 81 |
|
| 82 |
range_a = |
| 83 |
vorrq_u8(range_a, vextq_u8(prev_first_len, first_len_a, 15)); |
| 84 |
range_b = |
| 85 |
vorrq_u8(range_b, vextq_u8(first_len_a, first_len_b, 15)); |
| 86 |
|
| 87 |
uint8x16_t tmp1_a, tmp2_a, tmp1_b, tmp2_b; |
| 88 |
tmp1_a = vextq_u8(prev_first_len, first_len_a, 14); |
| 89 |
tmp1_a = vqsubq_u8(tmp1_a, const_1); |
| 90 |
range_a = vorrq_u8(range_a, tmp1_a); |
| 91 |
|
| 92 |
tmp1_b = vextq_u8(first_len_a, first_len_b, 14); |
| 93 |
tmp1_b = vqsubq_u8(tmp1_b, const_1); |
| 94 |
range_b = vorrq_u8(range_b, tmp1_b); |
| 95 |
|
| 96 |
tmp2_a = vextq_u8(prev_first_len, first_len_a, 13); |
| 97 |
tmp2_a = vqsubq_u8(tmp2_a, const_2); |
| 98 |
range_a = vorrq_u8(range_a, tmp2_a); |
| 99 |
|
| 100 |
tmp2_b = vextq_u8(first_len_a, first_len_b, 13); |
| 101 |
tmp2_b = vqsubq_u8(tmp2_b, const_2); |
| 102 |
range_b = vorrq_u8(range_b, tmp2_b); |
| 103 |
|
| 104 |
uint8x16_t shift1_a = vextq_u8(prev_input, input_a, 15); |
| 105 |
uint8x16_t pos_a = vsubq_u8(shift1_a, const_e0); |
| 106 |
range_a = vaddq_u8(range_a, vqtbl2q_u8(range_adjust_tbl, pos_a)); |
| 107 |
|
| 108 |
uint8x16_t shift1_b = vextq_u8(input_a, input_b, 15); |
| 109 |
uint8x16_t pos_b = vsubq_u8(shift1_b, const_e0); |
| 110 |
range_b = vaddq_u8(range_b, vqtbl2q_u8(range_adjust_tbl, pos_b)); |
| 111 |
|
| 112 |
uint8x16_t minv_a = vqtbl1q_u8(range_min_tbl, range_a); |
| 113 |
uint8x16_t maxv_a = vqtbl1q_u8(range_max_tbl, range_a); |
| 114 |
|
| 115 |
uint8x16_t minv_b = vqtbl1q_u8(range_min_tbl, range_b); |
| 116 |
uint8x16_t maxv_b = vqtbl1q_u8(range_max_tbl, range_b); |
| 117 |
|
| 118 |
error1 = vorrq_u8(error1, vcltq_u8(input_a, minv_a)); |
| 119 |
error2 = vorrq_u8(error2, vcgtq_u8(input_a, maxv_a)); |
| 120 |
|
| 121 |
error3 = vorrq_u8(error3, vcltq_u8(input_b, minv_b)); |
| 122 |
error4 = vorrq_u8(error4, vcgtq_u8(input_b, maxv_b)); |
| 123 |
|
| 124 |
/************************ next iteration *************************/ |
| 125 |
prev_input = input_b; |
| 126 |
prev_first_len = first_len_b; |
| 127 |
|
| 128 |
data += 32; |
| 129 |
len -= 32; |
| 130 |
} |
| 131 |
error1 = vorrq_u8(error1, error2); |
| 132 |
error1 = vorrq_u8(error1, error3); |
| 133 |
error1 = vorrq_u8(error1, error4); |
| 134 |
|
| 135 |
if (vmaxvq_u8(error1)) |
| 136 |
return -1; |
| 137 |
|
| 138 |
uint32_t token4; |
| 139 |
vst1q_lane_u32(&token4, vreinterpretq_u32_u8(prev_input), 3); |
| 140 |
|
| 141 |
const int8_t *token = (const int8_t *)&token4; |
| 142 |
int lookahead = 0; |
| 143 |
if (token[3] > (int8_t)0xBF) |
| 144 |
lookahead = 1; |
| 145 |
else if (token[2] > (int8_t)0xBF) |
| 146 |
lookahead = 2; |
| 147 |
else if (token[1] > (int8_t)0xBF) |
| 148 |
lookahead = 3; |
| 149 |
|
| 150 |
data -= lookahead; |
| 151 |
len += lookahead; |
| 152 |
} |
| 153 |
|
| 154 |
return utf8_naive(data, len); |
| 155 |
} |
| 156 |
|
| 157 |
#endif |