Debian Codegraph @

bufio/scanner.ha

raw
1 // SPDX-License-Identifier: MPL-2.0
2 // (c) Hare authors <https://harelang.org>
3
4 use bytes;
5 use encoding::utf8;
6 use errors;
7 use io;
8 use strings;
9 use types;
10
11 def BUFSZ: size = 4096;
12
13 const scanner_vtable = io::vtable {
14 reader = &scan_read,
15 seeker = &scan_seek,
16 ...
17 };
18
19 export type scanner = struct {
20 stream: io::stream,
21 src: io::handle,
22 buffer: []u8,
23 // Index of start of pending bytes in buffer
24 start: size,
25 // Sub-slice with pending bytes in buffer
26 pending: []u8,
27 // User-confirmed maximum size of read buffer
28 maxread: size,
29 // Change some scanning behaviors
30 opts: scan_options,
31 };
32
33 // Options which fine-tune the behavior of a [[scanner]].
34 export type scan_options = enum uint {
35 DEFAULT = EOF_DISCARD,
36 // Upon encountering EOF, all bytes or characters between the
37 // final token and EOF are discarded and EOF is returned
38 // immediately.
39 //
40 // This option is recommended for use-cases where the user is
41 // scanning over a file or buffer which may contain partial
42 // content, and the user wishes to consume as many tokens as
43 // possible and assume that additional data may follow EOF
44 // before a new delimiter is written.
45 //
46 // This is the default behavior. Note that on Unix, text files
47 // are always terminated with a new line, and [[scan_line]] will
48 // enumerate all well-formed lines in a file with this flag --
49 // however, when scanning ill-formed text files which include
50 // text following the final line feed, this additional text will
51 // be discarded.
52 EOF_DISCARD = 0,
53 // Upon encountering EOF, all bytes or characters between the
54 // final token and EOF are treated as a token and returned to
55 // the caller before returning EOF.
56 //
57 // This is recommended for use-cases where EOF is effectively
58 // considered an additional delimiter between tokens, or where
59 // the remainder of the file following the final delimiter is
60 // meaningful.
61 EOF_GREEDY = 1 << 0,
62 };
63
64 // Creates a new [[scanner]] which will allocate and maintain a read buffer for
65 // efficient reading of a handle. The scanner will read ahead only up to maxread
66 // bytes, which defaults to [[types::SIZE_MAX]] if no limit is required. The
67 // user must free resources associated with the scanner using [[finish]] after
68 // use.
69 //
70 // Reads from the scanner will return [[errors::overflow]] if maxread is
71 // reached.
72 export fn newscanner(
73 src: io::handle,
74 maxread: size = types::SIZE_MAX,
75 opts: scan_options = scan_options::DEFAULT,
76 ) scanner = {
77 return scanner {
78 stream = &scanner_vtable,
79 src = src,
80 buffer = [],
81 maxread = maxread,
82 start = 0,
83 pending = [],
84 opts = opts,
85 };
86 };
87
88 // Creates a new [[scanner]] using a user-provided buffer. The scanner will
89 // return [[errors::overflow]] if the buffer length is reached, but will not
90 // perform any allocations. The user should not call [[finish]] after use unless
91 // they wish to free the underlying buffer through bufio.
92 export fn newscanner_static(
93 src: io::handle,
94 buffer: []u8,
95 opts: scan_options = scan_options::DEFAULT,
96 ) scanner = {
97 return scanner {
98 stream = &scanner_vtable,
99 src = src,
100 buffer = buffer,
101 maxread = len(buffer),
102 start = 0,
103 pending = [],
104 opts = opts,
105 };
106 };
107
108 // Frees resources associated with a [[scanner]]. Does not close the underlying
109 // I/O handle.
110 export fn finish(scan: *scanner) void = {
111 free(scan.buffer);
112 };
113
114 fn scan_read(s: *io::stream, buf: []u8) (size | io::EOF | io::error) = {
115 let scan = s: *scanner;
116
117 if (len(scan.pending) == 0) {
118 match (scan_readahead(scan)?) {
119 case io::EOF =>
120 return io::EOF;
121 case size => void;
122 };
123 };
124
125 const n = if (len(buf) > len(scan.pending)) len(scan.pending) else len(buf);
126 buf[..n] = scan_consume(scan, n)[..];
127 return n;
128 };
129
130 fn scan_seek(
131 s: *io::stream,
132 off: io::off,
133 w: io::whence,
134 ) (io::off | io::error) = {
135 let scan = s: *scanner;
136
137 if (w == io::whence::CUR) {
138 // Fast path for io::tell
139 if (off == 0) {
140 let new_off = io::tell(scan.src)?;
141 new_off -= len(scan.pending): io::off;
142 return new_off;
143 };
144
145 // Adjust offset to account for buffered data
146 off -= len(scan.pending): io::off;
147 };
148
149 const new_off = io::seek(scan.src, off, w)?;
150
151 // Discard read-ahead buffer
152 scan.start = 0;
153 scan.pending = scan.buffer[..0];
154 return new_off;
155 };
156
157 // Fills up the scanner buffer with data from the underlying I/O handle. If no
158 // space remains in the read buffer, it is expanded by BUFSZ (up to maxread).
159 // Then, one read from the underlying I/O handle is performed and scan.pending
160 // is updated accordingly. Returns the number of bytes which had been available
161 // prior to the call.
162 fn scan_readahead(scan: *scanner) (size | io::EOF | io::error) = {
163 let start = scan.start;
164 const pending = len(scan.pending);
165
166 if (start + pending == len(scan.buffer)) {
167 if (start > 0) {
168 // Shift buffer to the left to free space at the end
169 scan.buffer[..len(scan.buffer) - start] = scan.buffer[start..];
170 scan.pending = scan.buffer[..pending];
171 start = 0;
172 scan.start = 0;
173 } else {
174 // Buffer is full, expand it
175 let readahead = pending + BUFSZ;
176 if (readahead > scan.maxread) {
177 readahead = scan.maxread;
178 };
179 if (pending >= readahead) {
180 return errors::overflow;
181 };
182 append(scan.buffer, [0...], readahead)?;
183 };
184 };
185
186 match (io::read(scan.src, scan.buffer[start + pending..])?) {
187 case let z: size =>
188 scan.pending = scan.buffer[start..start + pending + z];
189 return pending;
190 case io::EOF =>
191 return io::EOF;
192 };
193 };
194
195 // Consumes N bytes from the buffer.
196 fn scan_consume(scan: *scanner, n: size) []u8 = {
197 assert(len(scan.pending) >= n);
198 scan.start += n;
199 defer scan.pending = scan.pending[n..];
200 return scan.pending[..n];
201 };
202
203 // Reads one byte from a [[scanner]].
204 export fn scan_byte(scan: *scanner) (u8 | io::EOF | io::error) = {
205 if (len(scan.pending) == 0) {
206 match (scan_readahead(scan)?) {
207 case io::EOF =>
208 return io::EOF;
209 case size => void;
210 };
211 };
212
213 return scan_consume(scan, 1)[0];
214 };
215
216 // Reads the next token from a [[scanner]], delimited by delim. The delimiter is
217 // read from the source handle but not included in the returned slice. The
218 // return value is borrowed from the internal scanner buffer, which is
219 // invalidated during subsequent operations which use this scanner.
220 export fn scan_bytes(
221 scan: *scanner,
222 delim: (u8 | []u8),
223 ) ([]u8 | io::EOF | io::error) = {
224 let i = 0z;
225 for (true) {
226 match (bytes::index(scan.pending[i..], delim)) {
227 case let ix: size =>
228 i += ix;
229 break;
230 case void => void;
231 };
232
233 match (scan_readahead(scan)?) {
234 case io::EOF =>
235 if (scan.opts == scan_options::EOF_DISCARD) {
236 return io::EOF;
237 };
238 if (len(scan.pending) == 0) {
239 return io::EOF;
240 };
241 return scan_consume(scan, len(scan.pending));
242 case let prevpending: size =>
243 // No need to re-index the earlier part of the buffer
244 i = prevpending;
245 };
246 };
247
248 const ndelim = match (delim) {
249 case u8 =>
250 yield 1z;
251 case let u: []u8 =>
252 yield len(u);
253 };
254 const nconsume = i + ndelim;
255 return scan_consume(scan, nconsume)[..i];
256 };
257
258 // Reads one rune from a [[scanner]].
259 export fn scan_rune(
260 scan: *scanner,
261 ) (rune | io::EOF | io::error | utf8::invalid) = {
262 if (len(scan.pending) < 4) {
263 match (scan_readahead(scan)?) {
264 case io::EOF =>
265 if (len(scan.pending) == 0) {
266 return io::EOF;
267 };
268 case size => void;
269 };
270 };
271 const sz = utf8::utf8sz(scan.pending[0])?;
272 if (len(scan.pending) < sz) {
273 return utf8::invalid;
274 };
275 const buf = scan_consume(scan, sz);
276 const dec = utf8::decode(buf[..sz]);
277 match (utf8::next(&dec)?) {
278 case let r: rune =>
279 return r;
280 case done =>
281 return io::EOF;
282 case utf8::more =>
283 return utf8::invalid;
284 };
285 };
286
287 // Scans a string of text from a [[scanner]] up to some delimiter. The delimiter
288 // is read from the source handle but not included in the returned string. The
289 // return value is borrowed from the internal scanner buffer, which is
290 // invalidated during subsequent operations which use this scanner.
291 export fn scan_string(
292 scan: *scanner,
293 delim: str,
294 ) (const str | io::EOF | io::error | utf8::invalid) = {
295 const token = match (scan_bytes(scan, strings::toutf8(delim))?) {
296 case let token: []u8 =>
297 yield token;
298 case io::EOF =>
299 return io::EOF;
300 };
301 return strings::fromutf8(token)?;
302 };
303
304 // Scans the next line of text from a [[scanner]]. The return value is borrowed
305 // from the internal scanner buffer, which is invalidated during subsequent
306 // operations which use this scanner.
307 export fn scan_line(
308 scan: *scanner,
309 ) (const str | io::EOF | io::error | utf8::invalid) = {
310 return scan_string(scan, "\n");
311 };
312
313 // Returns the internal scanner buffer, which contains all bytes read ahead by
314 // the scanner up to this point.
315 export fn scan_buffer(scan: *scanner) []u8 = {
316 return scan.pending[..];
317 };
318
319 fn scan_unread(scan: *scanner, buf: []u8) void = {
320 if (len(buf) == 0) {
321 return;
322 };
323 if (len(buf) <= scan.start) {
324 const pending_end = scan.start + len(scan.pending);
325 scan.buffer[scan.start - len(buf)..scan.start] = buf;
326 scan.start -= len(buf);
327 scan.pending = scan.buffer[scan.start..pending_end];
328 } else {
329 assert(len(buf) <= len(scan.buffer) - len(scan.pending),
330 "Attempted to unread more data than buffer has available");
331 // Shift buffer to the right to free space at the beginning
332 scan.buffer[len(buf)..len(buf) + len(scan.pending)] =
333 scan.buffer[scan.start..scan.start + len(scan.pending)];
334 scan.buffer[..len(buf)] = buf;
335 scan.pending = scan.buffer[..len(scan.pending) + len(buf)];
336 scan.start = 0;
337 };
338 };
339
340 // Reads a single byte from an [[io::handle]].
341 export fn read_byte(h: io::handle) (u8 | io::EOF | io::error) = {
342 let buf: [1]u8 = [0...];
343
344 match (io::readall(h, buf)?) {
345 case size =>
346 return buf[0];
347 case io::EOF =>
348 return io::EOF;
349 };
350 };
351
352 // Reads a slice of bytes until the delimiter. Delimiter is not included but
353 // it is read from the handle. The return value must be freed by the caller.
354 export fn read_tok(h: io::handle, delim: u8...) ([]u8 | io::EOF | io::error) = {
355 let buf: []u8 = [];
356 let ok = false;
357 defer if (!ok) free(buf);
358
359 for (true) {
360 match (read_byte(h)?) {
361 case let res: u8 =>
362 if (bytes::contains(delim, res)) {
363 break;
364 };
365 append(buf, res)?;
366 case io::EOF =>
367 if (len(buf) == 0) {
368 return io::EOF;
369 };
370 break;
371 };
372 };
373
374 ok = true;
375 return buf;
376 };
377
378 // Reads a slice of bytes until a newline character (\n, 0x0A). Newline itself
379 // is not included but it is read from the handle. The return value must be
380 // freed by the caller.
381 export fn read_line(h: io::handle) ([]u8 | io::EOF | io::error) =
382 read_tok(h, '\n');
383
384 // Reads a rune from a UTF-8 stream.
385 export fn read_rune(
386 h: io::handle,
387 ) (rune | utf8::invalid | io::EOF | io::error) = {
388 let b: [4]u8 = [0...];
389 match (io::readall(h, b[..1])?) {
390 case let n: size => void;
391 case io::EOF =>
392 return io::EOF;
393 };
394
395 const sz = utf8::utf8sz(b[0])?;
396
397 if (sz == 1) {
398 return b[0]: rune;
399 };
400
401 match (io::readall(h, b[1..sz])) {
402 case let n: size => void;
403 case io::EOF =>
404 return io::EOF;
405 case let err: io::error =>
406 return if (err is io::underread) utf8::invalid else err;
407 };
408
409 let dec = utf8::decode(b[..sz]);
410 match (utf8::next(&dec)?) {
411 case let r: rune =>
412 return r;
413 case done =>
414 return io::EOF;
415 case utf8::more =>
416 return utf8::invalid;
417 };
418 };