Debian Codegraph @

strings/tokenize.ha

raw
1 // SPDX-License-Identifier: MPL-2.0
2 // (c) Hare authors <https://harelang.org>
3
4 use bytes;
5 use types;
6
7 export type tokenizer = bytes::tokenizer;
8
9 // Tokenizes a string, returning an iterator that yields substrings separated by
10 // one or more delimiters, such that the string will be split along any of the
11 // characters found in "delim". If the string begins with or ends with a
12 // delimiter, an empty string is returned respectively as the first and last
13 // call to [[next_token]].
14 //
15 // Each character of the delimiter string must be an ASCII character (see
16 // [[ascii::valid]]).
17 //
18 // The input string and delimiter string are borrowed from the caller for the
19 // lifetime of the tokenizer.
20 //
21 // The caller must ensure that at least one delimiter is provided and that the
22 // length of the input string is less than [[types::I64_MAX]].
23 //
24 // const tok = strings::tokenize("Hello world!\tMy name is Harriet.", " \t");
25 // assert(next_token(&tok) as str == "Hello");
26 // assert(next_token(&tok) as str == "world!");
27 // assert(next_token(&tok) as str == "My");
28 // assert(next_token(&tok) as str == "name");
29 // assert(next_token(&tok) as str == "is");
30 // assert(next_token(&tok) as str == "Harriet");
31 // assert(next_token(&tok) is done);
32 export fn tokenize(s: str, delim: str) tokenizer = {
33 const in = toutf8(s);
34 const delim = toutf8(delim);
35 for (let ch .. delim) {
36 assert(ch & 0x80 == 0, "strings::tokenize cannot tokenize on non-ASCII delimiters");
37 };
38 return bytes::tokenize(in, delim...);
39 };
40
41 // Like [[tokenize]], but tokenizes the string in reverse, such that the first
42 // call to [[next_token]] returns the last token and the last call returns the
43 // first token.
44 export fn rtokenize(s: str, delim: str) tokenizer = {
45 const in = toutf8(s);
46 const delim = toutf8(delim);
47 for (let ch .. delim) {
48 assert(ch & 0x80 == 0, "strings::tokenize cannot tokenize on non-ASCII delimiters");
49 };
50 return bytes::rtokenize(in, delim...);
51 };
52
53 // Returns the next token from a [[tokenizer]] and advances the cursor.
54 export fn next_token(s: *tokenizer) (str | done) = {
55 let s = s: *bytes::tokenizer;
56 match (bytes::next_token(s)) {
57 case let b: []u8 =>
58 return fromutf8_unsafe(b);
59 case done => return done;
60 };
61 };
62
63 // Returns the next token from a [[tokenizer]] without advancing the cursor.
64 export fn peek_token(s: *tokenizer) (str | done) = {
65 let s = s: *bytes::tokenizer;
66 return match (bytes::peek_token(s)) {
67 case let b: []u8 =>
68 yield fromutf8_unsafe(b);
69 case done =>
70 return done;
71 };
72 };
73
74 // Returns the remainder of the input string from a [[tokenizer]] ahead of the
75 // token cursor.
76 export fn remaining_tokens(s: *tokenizer) str = {
77 let s = s: *bytes::tokenizer;
78 return fromutf8_unsafe(bytes::remaining_tokens(s));
79 };
80
81 fn tokenize_test(
82 in: str,
83 delim: str,
84 tokens: []str,
85 iters: size = types::SIZE_MAX,
86 ) tokenizer = {
87 const tok = tokenize(in, delim);
88 let n = 0z;
89 for (const want .. tokens) {
90 if (n >= iters) {
91 return tok;
92 };
93 n += 1;
94
95 const p = peek_token(&tok) as str;
96 const n = next_token(&tok) as str;
97 assert(p == n);
98 assert(n == want);
99 };
100
101 if (n >= iters) {
102 return tok;
103 };
104
105 assert(peek_token(&tok) is done);
106 assert(next_token(&tok) is done);
107 return tok;
108 };
109
110 @test fn tokenize() void = {
111 // simple case
112 tokenize_test(
113 "Hello world! My name is Harriet.", " ",
114 [
115 "Hello",
116 "world!",
117 "My",
118 "name",
119 "is",
120 "Harriet.",
121 ]);
122
123 // multiple delimiters
124 tokenize_test(
125 "/dev/sda1\t/ ext4 rw,relatime\t0 0", " \t",
126 [
127 "/dev/sda1",
128 "/",
129 "ext4",
130 "rw,relatime",
131 "0",
132 "0",
133 ]);
134
135 // consecutive delimiters
136 tokenize_test(
137 "hello world", " ",
138 [
139 "hello",
140 "",
141 "",
142 "",
143 "world",
144 ]);
145
146 // leading delimiters
147 tokenize_test(
148 " hello world ", " ",
149 [
150 "",
151 "hello",
152 "world",
153 "",
154 ]);
155
156 // remaining_tokens
157 const tok = tokenize_test(
158 "Hello world! My name is Harriet.", " ",
159 [
160 "Hello",
161 "world!",
162 ], 2);
163 assert(remaining_tokens(&tok) == "My name is Harriet.");
164 };
165
166 // Splits a string into tokens delimited by 'delim', starting at the beginning
167 // of the string, and returning a slice of up to N tokens. The caller must free
168 // this slice. The strings within the slice are borrowed from 'in'.
169 //
170 // The caller must ensure that 'delim' is not an empty string. The caller must
171 // free the result after use.
172 export fn splitn(in: str, delim: str, n: size) ([]str | nomem) = {
173 let toks: []str = [];
174 let ok = false;
175 defer if (!ok) freeall(toks);
176 let tok = tokenize(in, delim);
177 for (let i = 0z; i < n - 1z; i += 1) {
178 match (next_token(&tok)) {
179 case let s: str =>
180 append(toks, s)?;
181 case done =>
182 ok = true;
183 return toks;
184 };
185 };
186 match(peek_token(&tok)) {
187 case done => void;
188 case let s: str =>
189 append(toks, remaining_tokens(&tok))?;
190 };
191 ok = true;
192 return toks;
193 };
194
195 // Splits a string into tokens delimited by 'delim', starting at the end
196 // of the string, and returning a slice of up to N tokens. The caller must free
197 // this slice. The strings within the slice are borrowed from 'in'.
198 //
199 // The caller must ensure that 'delim' is not an empty string.
200 export fn rsplitn(in: str, delim: str, n: size) ([]str | nomem) = {
201 let toks: []str = [];
202 let tok = rtokenize(in, delim);
203 let ok = false;
204 defer if (!ok) freeall(toks);
205 for (let i = 0z; i < n - 1z; i += 1) {
206 match (next_token(&tok)) {
207 case let s: str =>
208 append(toks, s)?;
209 case done =>
210 ok = true;
211 return toks;
212 };
213 };
214 match(peek_token(&tok)) {
215 case done => void;
216 case let s: str =>
217 append(toks, remaining_tokens(&tok))?;
218 };
219
220 for (let i = 0z; i < len(toks) / 2; i += 1) {
221 const tmp = toks[i];
222 toks[i] = toks[len(toks) - i - 1];
223 toks[len(toks) - i - 1] = tmp;
224 };
225
226 ok = true;
227 return toks;
228 };
229
230 // Splits a string into tokens delimited by any number of ASCII characters in
231 // 'delim' (see [[tokenize]] for details).
232 //
233 // strings::split("hello world", " ");
234 // // ["hello", "world"]
235 // strings::split("hello;world,foo:bar", ",:;");
236 // // ["hello", "world", "foo", "bar"]
237 //
238 // The caller must free the returned slice. The strings within the slice are
239 // borrowed from 'in'.
240 //
241 // The caller must ensure that 'delim' is not an empty string.
242 export fn split(in: str, delim: str) ([]str | nomem) =
243 splitn(in, delim, types::SIZE_MAX);
244
245 @test fn split() void = {
246 const expected = ["Hello,", "my", "name", "is Drew"];
247 const actual = splitn("Hello, my name is Drew", " ", 4z)!;
248 assert(len(expected) == len(actual));
249 for (let i = 0z; i < len(expected); i += 1) {
250 assert(expected[i] == actual[i]);
251 };
252 free(actual);
253
254 const expected2 = ["Hello,", "my", "name", "is", "Drew"];
255 const actual2 = split("Hello, my name is Drew", " ")!;
256 assert(len(expected2) == len(actual2));
257 for (let i = 0z; i < len(expected2); i += 1) {
258 assert(expected2[i] == actual2[i]);
259 };
260 free(actual2);
261
262 const expected3 = ["one"];
263 const actual3 = splitn("one", "=", 2z)!;
264 assert(len(expected3) == len(actual3));
265 for (let i = 0z; i < len(expected3); i += 1) {
266 assert(expected3[i] == actual3[i]);
267 };
268 free(actual3);
269
270 const expected4 = ["Hello, my", "name", "is", "Drew"];
271 const actual4 = rsplitn("Hello, my name is Drew", " ", 4z)!;
272 assert(len(expected4) == len(actual4));
273 for (let i = 0z; i < len(expected4); i += 1) {
274 assert(expected4[i] == actual4[i]);
275 };
276 free(actual4);
277 };
278
279 // Returns a string "cut" along the first instance of a delimiter, returning
280 // everything up to the delimiter, and everything after the delimiter, in a
281 // tuple.
282 //
283 // strings::cut("hello=world=foobar", "=") // ("hello", "world=foobar")
284 // strings::cut("hello world", "=") // ("hello world", "")
285 //
286 // The return value is borrowed from the 'in' parameter. The caller must ensure
287 // that 'delim' is not an empty string.
288 export fn cut(in: str, delim: str) (str, str) = {
289 let c = bytes::cut(toutf8(in), toutf8(delim));
290 return (fromutf8_unsafe(c.0), fromutf8_unsafe(c.1));
291 };
292
293 // Returns a string "cut" along the last instance of a delimiter, returning
294 // everything up to the delimiter, and everything after the delimiter, in a
295 // tuple.
296 //
297 // strings::rcut("hello=world=foobar", "=") // ("hello=world", "foobar")
298 // strings::rcut("hello world", "=") // ("hello world", "")
299 //
300 // The return value is borrowed from the 'in' parameter. The caller must ensure
301 // that 'delim' is not an empty string.
302 export fn rcut(in: str, delim: str) (str, str) = {
303 let c = bytes::rcut(toutf8(in), toutf8(delim));
304 return (fromutf8_unsafe(c.0), fromutf8_unsafe(c.1));
305 };
306
307 @test fn cut() void = {
308 const sample = cut("hello=world", "=");
309 assert(sample.0 == "hello" && sample.1 == "world");
310 const sample = cut("hello=world=foobar", "=");
311 assert(sample.0 == "hello" && sample.1 == "world=foobar");
312 const sample = cut("hello world", "=");
313 assert(sample.0 == "hello world" && sample.1 == "");
314 const sample = cut("", "=");
315 assert(sample.0 == "" && sample.1 == "");
316
317 const sample = rcut("hello=world=foobar", "=");
318 assert(sample.0 == "hello=world" && sample.1 == "foobar");
319 };