Debian Codegraph @

.cargo/vendor/regex/testdata/utf8.toml

raw
1 # These test the UTF-8 modes expose by regex-automata. Namely, when utf8 is
2 # true, then we promise that the haystack is valid UTF-8. (Otherwise behavior
3 # is unspecified.) This also corresponds to building the regex engine with the
4 # following two guarantees:
5 #
6 # 1) For any non-empty match reported, its span is guaranteed to correspond to
7 # valid UTF-8.
8 # 2) All empty or zero-width matches reported must never split a UTF-8
9 # encoded codepoint. If the haystack has invalid UTF-8, then this results in
10 # unspecified behavior.
11 #
12 # The (2) is in particular what we focus our testing on since (1) is generally
13 # guaranteed by regex-syntax's AST-to-HIR translator and is well tested there.
14 # The thing with (2) is that it can't be described in the HIR, so the regex
15 # engines have to handle that case. Thus, we test it here.
16 #
17 # Note that it is possible to build a regex that has property (1) but not
18 # (2), and vice versa. This is done by building the HIR with 'utf8=true' but
19 # building the Thompson NFA with 'utf8=false'. We don't test that here because
20 # the harness doesn't expose a way to enable or disable UTF-8 mode with that
21 # granularity. Instead, those combinations are lightly tested via doc examples.
22 # That's not to say that (1) without (2) is uncommon. Indeed, ripgrep uses it
23 # because it cannot guarantee that its haystack is valid UTF-8.
24
25 # This tests that an empty regex doesn't split a codepoint.
26 [[test]]
27 name = "empty-utf8yes"
28 regex = ''
29 haystack = '☃'
30 matches = [[0, 0], [3, 3]]
31 unicode = true
32 utf8 = true
33
34 # Tests the overlapping case of the above.
35 [[test]]
36 name = "empty-utf8yes-overlapping"
37 regex = ''
38 haystack = '☃'
39 matches = [[0, 0], [3, 3]]
40 unicode = true
41 utf8 = true
42 match-kind = "all"
43 search-kind = "overlapping"
44
45 # This tests that an empty regex DOES split a codepoint when utf=false.
46 [[test]]
47 name = "empty-utf8no"
48 regex = ''
49 haystack = '☃'
50 matches = [[0, 0], [1, 1], [2, 2], [3, 3]]
51 unicode = true
52 utf8 = false
53
54 # Tests the overlapping case of the above.
55 [[test]]
56 name = "empty-utf8no-overlapping"
57 regex = ''
58 haystack = '☃'
59 matches = [[0, 0], [1, 1], [2, 2], [3, 3]]
60 unicode = true
61 utf8 = false
62 match-kind = "all"
63 search-kind = "overlapping"
64
65 # This tests that an empty regex doesn't split a codepoint, even if we give
66 # it bounds entirely within the codepoint.
67 #
68 # This is one of the trickier cases and is what motivated the current UTF-8
69 # mode design. In particular, at one point, this test failed the 'is_match'
70 # variant of the test but not 'find'. This is because the 'is_match' code path
71 # is specifically optimized for "was a match found" rather than "where is the
72 # match." In the former case, you don't really care about the empty-vs-non-empty
73 # matches, and thus, the codepoint splitting filtering logic wasn't getting
74 # applied. (In multiple ways across multiple regex engines.) In this way, you
75 # can wind up with a situation where 'is_match' says "yes," but 'find' says,
76 # "I didn't find anything." Which is... not great.
77 #
78 # I could have decided to say that providing boundaries that themselves split
79 # a codepoint would have unspecified behavior. But I couldn't quite convince
80 # myself that such boundaries were the only way to get an inconsistency between
81 # 'is_match' and 'find'.
82 #
83 # Note that I also tried to come up with a test like this that fails without
84 # using `bounds`. Specifically, a test where 'is_match' and 'find' disagree.
85 # But I couldn't do it, and I'm tempted to conclude it is impossible. The
86 # fundamental problem is that you need to simultaneously produce an empty match
87 # that splits a codepoint while *not* matching before or after the codepoint.
88 [[test]]
89 name = "empty-utf8yes-bounds"
90 regex = ''
91 haystack = '𝛃'
92 bounds = [1, 3]
93 matches = []
94 unicode = true
95 utf8 = true
96
97 # Tests the overlapping case of the above.
98 [[test]]
99 name = "empty-utf8yes-bounds-overlapping"
100 regex = ''
101 haystack = '𝛃'
102 bounds = [1, 3]
103 matches = []
104 unicode = true
105 utf8 = true
106 match-kind = "all"
107 search-kind = "overlapping"
108
109 # This tests that an empty regex splits a codepoint when the bounds are
110 # entirely within the codepoint.
111 [[test]]
112 name = "empty-utf8no-bounds"
113 regex = ''
114 haystack = '𝛃'
115 bounds = [1, 3]
116 matches = [[1, 1], [2, 2], [3, 3]]
117 unicode = true
118 utf8 = false
119
120 # Tests the overlapping case of the above.
121 [[test]]
122 name = "empty-utf8no-bounds-overlapping"
123 regex = ''
124 haystack = '𝛃'
125 bounds = [1, 3]
126 matches = [[1, 1], [2, 2], [3, 3]]
127 unicode = true
128 utf8 = false
129 match-kind = "all"
130 search-kind = "overlapping"
131
132 # In this test, we anchor the search. Since the start position is also a UTF-8
133 # boundary, we get a match.
134 [[test]]
135 name = "empty-utf8yes-anchored"
136 regex = ''
137 haystack = '𝛃'
138 matches = [[0, 0]]
139 anchored = true
140 unicode = true
141 utf8 = true
142
143 # Tests the overlapping case of the above.
144 [[test]]
145 name = "empty-utf8yes-anchored-overlapping"
146 regex = ''
147 haystack = '𝛃'
148 matches = [[0, 0]]
149 anchored = true
150 unicode = true
151 utf8 = true
152 match-kind = "all"
153 search-kind = "overlapping"
154
155 # Same as above, except with UTF-8 mode disabled. It almost doesn't change the
156 # result, except for the fact that since this is an anchored search and we
157 # always find all matches, the test harness will keep reporting matches until
158 # none are found. Because it's anchored, matches will be reported so long as
159 # they are directly adjacent. Since with UTF-8 mode the next anchored search
160 # after the match at [0, 0] fails, iteration stops (and doesn't find the last
161 # match at [4, 4]).
162 [[test]]
163 name = "empty-utf8no-anchored"
164 regex = ''
165 haystack = '𝛃'
166 matches = [[0, 0], [1, 1], [2, 2], [3, 3], [4, 4]]
167 anchored = true
168 unicode = true
169 utf8 = false
170
171 # Tests the overlapping case of the above.
172 #
173 # Note that overlapping anchored searches are a little weird, and it's not
174 # totally clear what their semantics ought to be. For now, we just test the
175 # current behavior of our test shim that implements overlapping search. (This
176 # is one of the reasons why we don't really expose regex-level overlapping
177 # searches.)
178 [[test]]
179 name = "empty-utf8no-anchored-overlapping"
180 regex = ''
181 haystack = '𝛃'
182 matches = [[0, 0]]
183 anchored = true
184 unicode = true
185 utf8 = false
186 match-kind = "all"
187 search-kind = "overlapping"
188
189 # In this test, we anchor the search, but also set bounds. The bounds start the
190 # search in the middle of a codepoint, so there should never be a match.
191 [[test]]
192 name = "empty-utf8yes-anchored-bounds"
193 regex = ''
194 haystack = '𝛃'
195 matches = []
196 bounds = [1, 3]
197 anchored = true
198 unicode = true
199 utf8 = true
200
201 # Tests the overlapping case of the above.
202 [[test]]
203 name = "empty-utf8yes-anchored-bounds-overlapping"
204 regex = ''
205 haystack = '𝛃'
206 matches = []
207 bounds = [1, 3]
208 anchored = true
209 unicode = true
210 utf8 = true
211 match-kind = "all"
212 search-kind = "overlapping"
213
214 # Same as above, except with UTF-8 mode disabled. Without UTF-8 mode enabled,
215 # matching within a codepoint is allowed. And remember, as in the anchored test
216 # above with UTF-8 mode disabled, iteration will report all adjacent matches.
217 # The matches at [0, 0] and [4, 4] are not included because of the bounds of
218 # the search.
219 [[test]]
220 name = "empty-utf8no-anchored-bounds"
221 regex = ''
222 haystack = '𝛃'
223 bounds = [1, 3]
224 matches = [[1, 1], [2, 2], [3, 3]]
225 anchored = true
226 unicode = true
227 utf8 = false
228
229 # Tests the overlapping case of the above.
230 #
231 # Note that overlapping anchored searches are a little weird, and it's not
232 # totally clear what their semantics ought to be. For now, we just test the
233 # current behavior of our test shim that implements overlapping search. (This
234 # is one of the reasons why we don't really expose regex-level overlapping
235 # searches.)
236 [[test]]
237 name = "empty-utf8no-anchored-bounds-overlapping"
238 regex = ''
239 haystack = '𝛃'
240 bounds = [1, 3]
241 matches = [[1, 1]]
242 anchored = true
243 unicode = true
244 utf8 = false
245 match-kind = "all"
246 search-kind = "overlapping"
247
248 # This tests that we find the match at the end of the string when the bounds
249 # exclude the first match.
250 [[test]]
251 name = "empty-utf8yes-startbound"
252 regex = ''
253 haystack = '𝛃'
254 bounds = [1, 4]
255 matches = [[4, 4]]
256 unicode = true
257 utf8 = true
258
259 # Tests the overlapping case of the above.
260 [[test]]
261 name = "empty-utf8yes-startbound-overlapping"
262 regex = ''
263 haystack = '𝛃'
264 bounds = [1, 4]
265 matches = [[4, 4]]
266 unicode = true
267 utf8 = true
268 match-kind = "all"
269 search-kind = "overlapping"
270
271 # Same as above, except since UTF-8 mode is disabled, we also find the matches
272 # inbetween that split the codepoint.
273 [[test]]
274 name = "empty-utf8no-startbound"
275 regex = ''
276 haystack = '𝛃'
277 bounds = [1, 4]
278 matches = [[1, 1], [2, 2], [3, 3], [4, 4]]
279 unicode = true
280 utf8 = false
281
282 # Tests the overlapping case of the above.
283 [[test]]
284 name = "empty-utf8no-startbound-overlapping"
285 regex = ''
286 haystack = '𝛃'
287 bounds = [1, 4]
288 matches = [[1, 1], [2, 2], [3, 3], [4, 4]]
289 unicode = true
290 utf8 = false
291 match-kind = "all"
292 search-kind = "overlapping"
293
294 # This tests that we don't find any matches in an anchored search, even when
295 # the bounds include a match (at the end).
296 [[test]]
297 name = "empty-utf8yes-anchored-startbound"
298 regex = ''
299 haystack = '𝛃'
300 bounds = [1, 4]
301 matches = []
302 anchored = true
303 unicode = true
304 utf8 = true
305
306 # Tests the overlapping case of the above.
307 [[test]]
308 name = "empty-utf8yes-anchored-startbound-overlapping"
309 regex = ''
310 haystack = '𝛃'
311 bounds = [1, 4]
312 matches = []
313 anchored = true
314 unicode = true
315 utf8 = true
316 match-kind = "all"
317 search-kind = "overlapping"
318
319 # Same as above, except since UTF-8 mode is disabled, we also find the matches
320 # inbetween that split the codepoint. Even though this is an anchored search,
321 # since the matches are adjacent, we find all of them.
322 [[test]]
323 name = "empty-utf8no-anchored-startbound"
324 regex = ''
325 haystack = '𝛃'
326 bounds = [1, 4]
327 matches = [[1, 1], [2, 2], [3, 3], [4, 4]]
328 anchored = true
329 unicode = true
330 utf8 = false
331
332 # Tests the overlapping case of the above.
333 #
334 # Note that overlapping anchored searches are a little weird, and it's not
335 # totally clear what their semantics ought to be. For now, we just test the
336 # current behavior of our test shim that implements overlapping search. (This
337 # is one of the reasons why we don't really expose regex-level overlapping
338 # searches.)
339 [[test]]
340 name = "empty-utf8no-anchored-startbound-overlapping"
341 regex = ''
342 haystack = '𝛃'
343 bounds = [1, 4]
344 matches = [[1, 1]]
345 anchored = true
346 unicode = true
347 utf8 = false
348 match-kind = "all"
349 search-kind = "overlapping"
350
351 # This tests that we find the match at the end of the haystack in UTF-8 mode
352 # when our bounds only include the empty string at the end of the haystack.
353 [[test]]
354 name = "empty-utf8yes-anchored-endbound"
355 regex = ''
356 haystack = '𝛃'
357 bounds = [4, 4]
358 matches = [[4, 4]]
359 anchored = true
360 unicode = true
361 utf8 = true
362
363 # Tests the overlapping case of the above.
364 [[test]]
365 name = "empty-utf8yes-anchored-endbound-overlapping"
366 regex = ''
367 haystack = '𝛃'
368 bounds = [4, 4]
369 matches = [[4, 4]]
370 anchored = true
371 unicode = true
372 utf8 = true
373 match-kind = "all"
374 search-kind = "overlapping"
375
376 # Same as above, but with UTF-8 mode disabled. Results remain the same since
377 # the only possible match does not split a codepoint.
378 [[test]]
379 name = "empty-utf8no-anchored-endbound"
380 regex = ''
381 haystack = '𝛃'
382 bounds = [4, 4]
383 matches = [[4, 4]]
384 anchored = true
385 unicode = true
386 utf8 = false
387
388 # Tests the overlapping case of the above.
389 [[test]]
390 name = "empty-utf8no-anchored-endbound-overlapping"
391 regex = ''
392 haystack = '𝛃'
393 bounds = [4, 4]
394 matches = [[4, 4]]
395 anchored = true
396 unicode = true
397 utf8 = false
398 match-kind = "all"
399 search-kind = "overlapping"