Debian Codegraph @

.cargo/vendor/regex-automata/src/util/utf8.rs

raw
1 /*!
2 Utilities for dealing with UTF-8.
3
4 This module provides some UTF-8 related helper routines, including an
5 incremental decoder.
6 */
7
8 /// Returns true if and only if the given byte is considered a word character.
9 /// This only applies to ASCII.
10 ///
11 /// This was copied from regex-syntax so that we can use it to determine the
12 /// starting DFA state while searching without depending on regex-syntax. The
13 /// definition is never going to change, so there's no maintenance/bit-rot
14 /// hazard here.
15 #[cfg_attr(feature = "perf-inline", inline(always))]
16 pub(crate) fn is_word_byte(b: u8) -> bool {
17 const fn mkwordset() -> [bool; 256] {
18 // FIXME: Use as_usize() once const functions in traits are stable.
19 let mut set = [false; 256];
20 set[b'_' as usize] = true;
21
22 let mut byte = b'0';
23 while byte <= b'9' {
24 set[byte as usize] = true;
25 byte += 1;
26 }
27 byte = b'A';
28 while byte <= b'Z' {
29 set[byte as usize] = true;
30 byte += 1;
31 }
32 byte = b'a';
33 while byte <= b'z' {
34 set[byte as usize] = true;
35 byte += 1;
36 }
37 set
38 }
39 const WORD: [bool; 256] = mkwordset();
40 WORD[b as usize]
41 }
42
43 /// Decodes the next UTF-8 encoded codepoint from the given byte slice.
44 ///
45 /// If no valid encoding of a codepoint exists at the beginning of the given
46 /// byte slice, then the first byte is returned instead.
47 ///
48 /// This returns `None` if and only if `bytes` is empty.
49 ///
50 /// This never panics.
51 ///
52 /// *WARNING*: This is not designed for performance. If you're looking for a
53 /// fast UTF-8 decoder, this is not it. If you feel like you need one in this
54 /// crate, then please file an issue and discuss your use case.
55 #[cfg_attr(feature = "perf-inline", inline(always))]
56 pub(crate) fn decode(bytes: &[u8]) -> Option<Result<char, u8>> {
57 if bytes.is_empty() {
58 return None;
59 }
60 let len = match len(bytes[0]) {
61 None => return Some(Err(bytes[0])),
62 Some(len) if len > bytes.len() => return Some(Err(bytes[0])),
63 Some(1) => return Some(Ok(char::from(bytes[0]))),
64 Some(len) => len,
65 };
66 match core::str::from_utf8(&bytes[..len]) {
67 Ok(s) => Some(Ok(s.chars().next().unwrap())),
68 Err(_) => Some(Err(bytes[0])),
69 }
70 }
71
72 /// Decodes the last UTF-8 encoded codepoint from the given byte slice.
73 ///
74 /// If no valid encoding of a codepoint exists at the end of the given byte
75 /// slice, then the last byte is returned instead.
76 ///
77 /// This returns `None` if and only if `bytes` is empty.
78 #[cfg_attr(feature = "perf-inline", inline(always))]
79 pub(crate) fn decode_last(bytes: &[u8]) -> Option<Result<char, u8>> {
80 if bytes.is_empty() {
81 return None;
82 }
83 let mut start = bytes.len() - 1;
84 let limit = bytes.len().saturating_sub(4);
85 while start > limit && !is_leading_or_invalid_byte(bytes[start]) {
86 start -= 1;
87 }
88 match decode(&bytes[start..]) {
89 None => None,
90 Some(Ok(ch)) => Some(Ok(ch)),
91 Some(Err(_)) => Some(Err(bytes[bytes.len() - 1])),
92 }
93 }
94
95 /// Given a UTF-8 leading byte, this returns the total number of code units
96 /// in the following encoded codepoint.
97 ///
98 /// If the given byte is not a valid UTF-8 leading byte, then this returns
99 /// `None`.
100 #[cfg_attr(feature = "perf-inline", inline(always))]
101 fn len(byte: u8) -> Option<usize> {
102 if byte <= 0x7F {
103 return Some(1);
104 } else if byte & 0b1100_0000 == 0b1000_0000 {
105 return None;
106 } else if byte <= 0b1101_1111 {
107 Some(2)
108 } else if byte <= 0b1110_1111 {
109 Some(3)
110 } else if byte <= 0b1111_0111 {
111 Some(4)
112 } else {
113 None
114 }
115 }
116
117 /// Returns true if and only if the given offset in the given bytes falls on a
118 /// valid UTF-8 encoded codepoint boundary.
119 ///
120 /// If `bytes` is not valid UTF-8, then the behavior of this routine is
121 /// unspecified.
122 #[cfg_attr(feature = "perf-inline", inline(always))]
123 pub(crate) fn is_boundary(bytes: &[u8], i: usize) -> bool {
124 match bytes.get(i) {
125 // The position at the end of the bytes always represents an empty
126 // string, which is a valid boundary. But anything after that doesn't
127 // make much sense to call valid a boundary.
128 None => i == bytes.len(),
129 // Other than ASCII (where the most significant bit is never set),
130 // valid starting bytes always have their most significant two bits
131 // set, where as continuation bytes never have their second most
132 // significant bit set. Therefore, this only returns true when bytes[i]
133 // corresponds to a byte that begins a valid UTF-8 encoding of a
134 // Unicode scalar value.
135 Some(&b) => b <= 0b0111_1111 || b >= 0b1100_0000,
136 }
137 }
138
139 /// Returns true if and only if the given byte is either a valid leading UTF-8
140 /// byte, or is otherwise an invalid byte that can never appear anywhere in a
141 /// valid UTF-8 sequence.
142 #[cfg_attr(feature = "perf-inline", inline(always))]
143 fn is_leading_or_invalid_byte(b: u8) -> bool {
144 // In the ASCII case, the most significant bit is never set. The leading
145 // byte of a 2/3/4-byte sequence always has the top two most significant
146 // bits set. For bytes that can never appear anywhere in valid UTF-8, this
147 // also returns true, since every such byte has its two most significant
148 // bits set:
149 //
150 // \xC0 :: 11000000
151 // \xC1 :: 11000001
152 // \xF5 :: 11110101
153 // \xF6 :: 11110110
154 // \xF7 :: 11110111
155 // \xF8 :: 11111000
156 // \xF9 :: 11111001
157 // \xFA :: 11111010
158 // \xFB :: 11111011
159 // \xFC :: 11111100
160 // \xFD :: 11111101
161 // \xFE :: 11111110
162 // \xFF :: 11111111
163 (b & 0b1100_0000) != 0b1000_0000
164 }
165
166 /*
167 /// Returns the smallest possible index of the next valid UTF-8 sequence
168 /// starting after `i`.
169 ///
170 /// For all inputs, including invalid UTF-8 and any value of `i`, the return
171 /// value is guaranteed to be greater than `i`. (If there is no value greater
172 /// than `i` that fits in `usize`, then this panics.)
173 ///
174 /// Generally speaking, this should only be called on `text` when it is
175 /// permitted to assume that it is valid UTF-8 and where either `i >=
176 /// text.len()` or where `text[i]` is a leading byte of a UTF-8 sequence.
177 ///
178 /// NOTE: This method was used in a previous conception of iterators where we
179 /// specifically tried to skip over empty matches that split a codepoint by
180 /// simply requiring that our next search begin at the beginning of codepoint.
181 /// But we ended up changing that technique to always advance by 1 byte and
182 /// then filter out matches that split a codepoint after-the-fact. Thus, we no
183 /// longer use this method. But I've kept it around in case we want to switch
184 /// back to this approach. Its guarantees are a little subtle, so I'd prefer
185 /// not to rebuild it from whole cloth.
186 pub(crate) fn next(text: &[u8], i: usize) -> usize {
187 let b = match text.get(i) {
188 None => return i.checked_add(1).unwrap(),
189 Some(&b) => b,
190 };
191 // For cases where we see an invalid UTF-8 byte, there isn't much we can do
192 // other than just start at the next byte.
193 let inc = len(b).unwrap_or(1);
194 i.checked_add(inc).unwrap()
195 }
196 */