| 1 |
// SPDX-License-Identifier: MPL-2.0 |
| 2 |
// (c) Hare authors <https://harelang.org> |
| 3 |
|
| 4 |
use bytes; |
| 5 |
use encoding::utf8; |
| 6 |
use errors; |
| 7 |
use io; |
| 8 |
use strings; |
| 9 |
use types; |
| 10 |
|
| 11 |
def BUFSZ: size = 4096; |
| 12 |
|
| 13 |
const scanner_vtable = io::vtable { |
| 14 |
reader = &scan_read, |
| 15 |
seeker = &scan_seek, |
| 16 |
... |
| 17 |
}; |
| 18 |
|
| 19 |
export type scanner = struct { |
| 20 |
stream: io::stream, |
| 21 |
src: io::handle, |
| 22 |
buffer: []u8, |
| 23 |
// Index of start of pending bytes in buffer |
| 24 |
start: size, |
| 25 |
// Sub-slice with pending bytes in buffer |
| 26 |
pending: []u8, |
| 27 |
// User-confirmed maximum size of read buffer |
| 28 |
maxread: size, |
| 29 |
// Change some scanning behaviors |
| 30 |
opts: scan_options, |
| 31 |
}; |
| 32 |
|
| 33 |
// Options which fine-tune the behavior of a [[scanner]]. |
| 34 |
export type scan_options = enum uint { |
| 35 |
DEFAULT = EOF_DISCARD, |
| 36 |
// Upon encountering EOF, all bytes or characters between the |
| 37 |
// final token and EOF are discarded and EOF is returned |
| 38 |
// immediately. |
| 39 |
// |
| 40 |
// This option is recommended for use-cases where the user is |
| 41 |
// scanning over a file or buffer which may contain partial |
| 42 |
// content, and the user wishes to consume as many tokens as |
| 43 |
// possible and assume that additional data may follow EOF |
| 44 |
// before a new delimiter is written. |
| 45 |
// |
| 46 |
// This is the default behavior. Note that on Unix, text files |
| 47 |
// are always terminated with a new line, and [[scan_line]] will |
| 48 |
// enumerate all well-formed lines in a file with this flag -- |
| 49 |
// however, when scanning ill-formed text files which include |
| 50 |
// text following the final line feed, this additional text will |
| 51 |
// be discarded. |
| 52 |
EOF_DISCARD = 0, |
| 53 |
// Upon encountering EOF, all bytes or characters between the |
| 54 |
// final token and EOF are treated as a token and returned to |
| 55 |
// the caller before returning EOF. |
| 56 |
// |
| 57 |
// This is recommended for use-cases where EOF is effectively |
| 58 |
// considered an additional delimiter between tokens, or where |
| 59 |
// the remainder of the file following the final delimiter is |
| 60 |
// meaningful. |
| 61 |
EOF_GREEDY = 1 << 0, |
| 62 |
}; |
| 63 |
|
| 64 |
// Creates a new [[scanner]] which will allocate and maintain a read buffer for |
| 65 |
// efficient reading of a handle. The scanner will read ahead only up to maxread |
| 66 |
// bytes, which defaults to [[types::SIZE_MAX]] if no limit is required. The |
| 67 |
// user must free resources associated with the scanner using [[finish]] after |
| 68 |
// use. |
| 69 |
// |
| 70 |
// Reads from the scanner will return [[errors::overflow]] if maxread is |
| 71 |
// reached. |
| 72 |
export fn newscanner( |
| 73 |
src: io::handle, |
| 74 |
maxread: size = types::SIZE_MAX, |
| 75 |
opts: scan_options = scan_options::DEFAULT, |
| 76 |
) scanner = { |
| 77 |
return scanner { |
| 78 |
stream = &scanner_vtable, |
| 79 |
src = src, |
| 80 |
buffer = [], |
| 81 |
maxread = maxread, |
| 82 |
start = 0, |
| 83 |
pending = [], |
| 84 |
opts = opts, |
| 85 |
}; |
| 86 |
}; |
| 87 |
|
| 88 |
// Creates a new [[scanner]] using a user-provided buffer. The scanner will |
| 89 |
// return [[errors::overflow]] if the buffer length is reached, but will not |
| 90 |
// perform any allocations. The user should not call [[finish]] after use unless |
| 91 |
// they wish to free the underlying buffer through bufio. |
| 92 |
export fn newscanner_static( |
| 93 |
src: io::handle, |
| 94 |
buffer: []u8, |
| 95 |
opts: scan_options = scan_options::DEFAULT, |
| 96 |
) scanner = { |
| 97 |
return scanner { |
| 98 |
stream = &scanner_vtable, |
| 99 |
src = src, |
| 100 |
buffer = buffer, |
| 101 |
maxread = len(buffer), |
| 102 |
start = 0, |
| 103 |
pending = [], |
| 104 |
opts = opts, |
| 105 |
}; |
| 106 |
}; |
| 107 |
|
| 108 |
// Frees resources associated with a [[scanner]]. Does not close the underlying |
| 109 |
// I/O handle. |
| 110 |
export fn finish(scan: *scanner) void = { |
| 111 |
free(scan.buffer); |
| 112 |
}; |
| 113 |
|
| 114 |
fn scan_read(s: *io::stream, buf: []u8) (size | io::EOF | io::error) = { |
| 115 |
let scan = s: *scanner; |
| 116 |
|
| 117 |
if (len(scan.pending) == 0) { |
| 118 |
match (scan_readahead(scan)?) { |
| 119 |
case io::EOF => |
| 120 |
return io::EOF; |
| 121 |
case size => void; |
| 122 |
}; |
| 123 |
}; |
| 124 |
|
| 125 |
const n = if (len(buf) > len(scan.pending)) len(scan.pending) else len(buf); |
| 126 |
buf[..n] = scan_consume(scan, n)[..]; |
| 127 |
return n; |
| 128 |
}; |
| 129 |
|
| 130 |
fn scan_seek( |
| 131 |
s: *io::stream, |
| 132 |
off: io::off, |
| 133 |
w: io::whence, |
| 134 |
) (io::off | io::error) = { |
| 135 |
let scan = s: *scanner; |
| 136 |
|
| 137 |
if (w == io::whence::CUR) { |
| 138 |
// Fast path for io::tell |
| 139 |
if (off == 0) { |
| 140 |
let new_off = io::tell(scan.src)?; |
| 141 |
new_off -= len(scan.pending): io::off; |
| 142 |
return new_off; |
| 143 |
}; |
| 144 |
|
| 145 |
// Adjust offset to account for buffered data |
| 146 |
off -= len(scan.pending): io::off; |
| 147 |
}; |
| 148 |
|
| 149 |
const new_off = io::seek(scan.src, off, w)?; |
| 150 |
|
| 151 |
// Discard read-ahead buffer |
| 152 |
scan.start = 0; |
| 153 |
scan.pending = scan.buffer[..0]; |
| 154 |
return new_off; |
| 155 |
}; |
| 156 |
|
| 157 |
// Fills up the scanner buffer with data from the underlying I/O handle. If no |
| 158 |
// space remains in the read buffer, it is expanded by BUFSZ (up to maxread). |
| 159 |
// Then, one read from the underlying I/O handle is performed and scan.pending |
| 160 |
// is updated accordingly. Returns the number of bytes which had been available |
| 161 |
// prior to the call. |
| 162 |
fn scan_readahead(scan: *scanner) (size | io::EOF | io::error) = { |
| 163 |
let start = scan.start; |
| 164 |
const pending = len(scan.pending); |
| 165 |
|
| 166 |
if (start + pending == len(scan.buffer)) { |
| 167 |
if (start > 0) { |
| 168 |
// Shift buffer to the left to free space at the end |
| 169 |
scan.buffer[..len(scan.buffer) - start] = scan.buffer[start..]; |
| 170 |
scan.pending = scan.buffer[..pending]; |
| 171 |
start = 0; |
| 172 |
scan.start = 0; |
| 173 |
} else { |
| 174 |
// Buffer is full, expand it |
| 175 |
let readahead = pending + BUFSZ; |
| 176 |
if (readahead > scan.maxread) { |
| 177 |
readahead = scan.maxread; |
| 178 |
}; |
| 179 |
if (pending >= readahead) { |
| 180 |
return errors::overflow; |
| 181 |
}; |
| 182 |
append(scan.buffer, [0...], readahead)?; |
| 183 |
}; |
| 184 |
}; |
| 185 |
|
| 186 |
match (io::read(scan.src, scan.buffer[start + pending..])?) { |
| 187 |
case let z: size => |
| 188 |
scan.pending = scan.buffer[start..start + pending + z]; |
| 189 |
return pending; |
| 190 |
case io::EOF => |
| 191 |
return io::EOF; |
| 192 |
}; |
| 193 |
}; |
| 194 |
|
| 195 |
// Consumes N bytes from the buffer. |
| 196 |
fn scan_consume(scan: *scanner, n: size) []u8 = { |
| 197 |
assert(len(scan.pending) >= n); |
| 198 |
scan.start += n; |
| 199 |
defer scan.pending = scan.pending[n..]; |
| 200 |
return scan.pending[..n]; |
| 201 |
}; |
| 202 |
|
| 203 |
// Reads one byte from a [[scanner]]. |
| 204 |
export fn scan_byte(scan: *scanner) (u8 | io::EOF | io::error) = { |
| 205 |
if (len(scan.pending) == 0) { |
| 206 |
match (scan_readahead(scan)?) { |
| 207 |
case io::EOF => |
| 208 |
return io::EOF; |
| 209 |
case size => void; |
| 210 |
}; |
| 211 |
}; |
| 212 |
|
| 213 |
return scan_consume(scan, 1)[0]; |
| 214 |
}; |
| 215 |
|
| 216 |
// Reads the next token from a [[scanner]], delimited by delim. The delimiter is |
| 217 |
// read from the source handle but not included in the returned slice. The |
| 218 |
// return value is borrowed from the internal scanner buffer, which is |
| 219 |
// invalidated during subsequent operations which use this scanner. |
| 220 |
export fn scan_bytes( |
| 221 |
scan: *scanner, |
| 222 |
delim: (u8 | []u8), |
| 223 |
) ([]u8 | io::EOF | io::error) = { |
| 224 |
let i = 0z; |
| 225 |
for (true) { |
| 226 |
match (bytes::index(scan.pending[i..], delim)) { |
| 227 |
case let ix: size => |
| 228 |
i += ix; |
| 229 |
break; |
| 230 |
case void => void; |
| 231 |
}; |
| 232 |
|
| 233 |
match (scan_readahead(scan)?) { |
| 234 |
case io::EOF => |
| 235 |
if (scan.opts == scan_options::EOF_DISCARD) { |
| 236 |
return io::EOF; |
| 237 |
}; |
| 238 |
if (len(scan.pending) == 0) { |
| 239 |
return io::EOF; |
| 240 |
}; |
| 241 |
return scan_consume(scan, len(scan.pending)); |
| 242 |
case let prevpending: size => |
| 243 |
// No need to re-index the earlier part of the buffer |
| 244 |
i = prevpending; |
| 245 |
}; |
| 246 |
}; |
| 247 |
|
| 248 |
const ndelim = match (delim) { |
| 249 |
case u8 => |
| 250 |
yield 1z; |
| 251 |
case let u: []u8 => |
| 252 |
yield len(u); |
| 253 |
}; |
| 254 |
const nconsume = i + ndelim; |
| 255 |
return scan_consume(scan, nconsume)[..i]; |
| 256 |
}; |
| 257 |
|
| 258 |
// Reads one rune from a [[scanner]]. |
| 259 |
export fn scan_rune( |
| 260 |
scan: *scanner, |
| 261 |
) (rune | io::EOF | io::error | utf8::invalid) = { |
| 262 |
if (len(scan.pending) < 4) { |
| 263 |
match (scan_readahead(scan)?) { |
| 264 |
case io::EOF => |
| 265 |
if (len(scan.pending) == 0) { |
| 266 |
return io::EOF; |
| 267 |
}; |
| 268 |
case size => void; |
| 269 |
}; |
| 270 |
}; |
| 271 |
const sz = utf8::utf8sz(scan.pending[0])?; |
| 272 |
if (len(scan.pending) < sz) { |
| 273 |
return utf8::invalid; |
| 274 |
}; |
| 275 |
const buf = scan_consume(scan, sz); |
| 276 |
const dec = utf8::decode(buf[..sz]); |
| 277 |
match (utf8::next(&dec)?) { |
| 278 |
case let r: rune => |
| 279 |
return r; |
| 280 |
case done => |
| 281 |
return io::EOF; |
| 282 |
case utf8::more => |
| 283 |
return utf8::invalid; |
| 284 |
}; |
| 285 |
}; |
| 286 |
|
| 287 |
// Scans a string of text from a [[scanner]] up to some delimiter. The delimiter |
| 288 |
// is read from the source handle but not included in the returned string. The |
| 289 |
// return value is borrowed from the internal scanner buffer, which is |
| 290 |
// invalidated during subsequent operations which use this scanner. |
| 291 |
export fn scan_string( |
| 292 |
scan: *scanner, |
| 293 |
delim: str, |
| 294 |
) (const str | io::EOF | io::error | utf8::invalid) = { |
| 295 |
const token = match (scan_bytes(scan, strings::toutf8(delim))?) { |
| 296 |
case let token: []u8 => |
| 297 |
yield token; |
| 298 |
case io::EOF => |
| 299 |
return io::EOF; |
| 300 |
}; |
| 301 |
return strings::fromutf8(token)?; |
| 302 |
}; |
| 303 |
|
| 304 |
// Scans the next line of text from a [[scanner]]. The return value is borrowed |
| 305 |
// from the internal scanner buffer, which is invalidated during subsequent |
| 306 |
// operations which use this scanner. |
| 307 |
export fn scan_line( |
| 308 |
scan: *scanner, |
| 309 |
) (const str | io::EOF | io::error | utf8::invalid) = { |
| 310 |
return scan_string(scan, "\n"); |
| 311 |
}; |
| 312 |
|
| 313 |
// Returns the internal scanner buffer, which contains all bytes read ahead by |
| 314 |
// the scanner up to this point. |
| 315 |
export fn scan_buffer(scan: *scanner) []u8 = { |
| 316 |
return scan.pending[..]; |
| 317 |
}; |
| 318 |
|
| 319 |
fn scan_unread(scan: *scanner, buf: []u8) void = { |
| 320 |
if (len(buf) == 0) { |
| 321 |
return; |
| 322 |
}; |
| 323 |
if (len(buf) <= scan.start) { |
| 324 |
const pending_end = scan.start + len(scan.pending); |
| 325 |
scan.buffer[scan.start - len(buf)..scan.start] = buf; |
| 326 |
scan.start -= len(buf); |
| 327 |
scan.pending = scan.buffer[scan.start..pending_end]; |
| 328 |
} else { |
| 329 |
assert(len(buf) <= len(scan.buffer) - len(scan.pending), |
| 330 |
"Attempted to unread more data than buffer has available"); |
| 331 |
// Shift buffer to the right to free space at the beginning |
| 332 |
scan.buffer[len(buf)..len(buf) + len(scan.pending)] = |
| 333 |
scan.buffer[scan.start..scan.start + len(scan.pending)]; |
| 334 |
scan.buffer[..len(buf)] = buf; |
| 335 |
scan.pending = scan.buffer[..len(scan.pending) + len(buf)]; |
| 336 |
scan.start = 0; |
| 337 |
}; |
| 338 |
}; |
| 339 |
|
| 340 |
// Reads a single byte from an [[io::handle]]. |
| 341 |
export fn read_byte(h: io::handle) (u8 | io::EOF | io::error) = { |
| 342 |
let buf: [1]u8 = [0...]; |
| 343 |
|
| 344 |
match (io::readall(h, buf)?) { |
| 345 |
case size => |
| 346 |
return buf[0]; |
| 347 |
case io::EOF => |
| 348 |
return io::EOF; |
| 349 |
}; |
| 350 |
}; |
| 351 |
|
| 352 |
// Reads a slice of bytes until the delimiter. Delimiter is not included but |
| 353 |
// it is read from the handle. The return value must be freed by the caller. |
| 354 |
export fn read_tok(h: io::handle, delim: u8...) ([]u8 | io::EOF | io::error) = { |
| 355 |
let buf: []u8 = []; |
| 356 |
let ok = false; |
| 357 |
defer if (!ok) free(buf); |
| 358 |
|
| 359 |
for (true) { |
| 360 |
match (read_byte(h)?) { |
| 361 |
case let res: u8 => |
| 362 |
if (bytes::contains(delim, res)) { |
| 363 |
break; |
| 364 |
}; |
| 365 |
append(buf, res)?; |
| 366 |
case io::EOF => |
| 367 |
if (len(buf) == 0) { |
| 368 |
return io::EOF; |
| 369 |
}; |
| 370 |
break; |
| 371 |
}; |
| 372 |
}; |
| 373 |
|
| 374 |
ok = true; |
| 375 |
return buf; |
| 376 |
}; |
| 377 |
|
| 378 |
// Reads a slice of bytes until a newline character (\n, 0x0A). Newline itself |
| 379 |
// is not included but it is read from the handle. The return value must be |
| 380 |
// freed by the caller. |
| 381 |
export fn read_line(h: io::handle) ([]u8 | io::EOF | io::error) = |
| 382 |
read_tok(h, '\n'); |
| 383 |
|
| 384 |
// Reads a rune from a UTF-8 stream. |
| 385 |
export fn read_rune( |
| 386 |
h: io::handle, |
| 387 |
) (rune | utf8::invalid | io::EOF | io::error) = { |
| 388 |
let b: [4]u8 = [0...]; |
| 389 |
match (io::readall(h, b[..1])?) { |
| 390 |
case let n: size => void; |
| 391 |
case io::EOF => |
| 392 |
return io::EOF; |
| 393 |
}; |
| 394 |
|
| 395 |
const sz = utf8::utf8sz(b[0])?; |
| 396 |
|
| 397 |
if (sz == 1) { |
| 398 |
return b[0]: rune; |
| 399 |
}; |
| 400 |
|
| 401 |
match (io::readall(h, b[1..sz])) { |
| 402 |
case let n: size => void; |
| 403 |
case io::EOF => |
| 404 |
return io::EOF; |
| 405 |
case let err: io::error => |
| 406 |
return if (err is io::underread) utf8::invalid else err; |
| 407 |
}; |
| 408 |
|
| 409 |
let dec = utf8::decode(b[..sz]); |
| 410 |
match (utf8::next(&dec)?) { |
| 411 |
case let r: rune => |
| 412 |
return r; |
| 413 |
case done => |
| 414 |
return io::EOF; |
| 415 |
case utf8::more => |
| 416 |
return utf8::invalid; |
| 417 |
}; |
| 418 |
}; |