Debian Codegraph @

tests/inputs/parser_2.civet

raw
1 civet coffeeComment
2 { fail } from node:assert
3 { tokens, type Token } from ./tokens.civet
4 /*
5 type * as Ast from ./astTypes.civet
6 assert from ./assert.civet
7 */
8
9 /// TokenType ::= keyof typeof tokens
10 /// tokenEntries := Object.entries(tokens) as [TokenType, Token][]
11
12 class TokenStream <: Iterable<[string, TokenType, readonly [number, number]]>
13 #sourceLocation = [1, 1] as tuple
14 @(#program: string)
15
16 :iterator() ###
17 :outer while #program# ###
18 for [type, token] of tokenEntries
19 length := token.matchLength #program
20 /// if length > 0
21 // chunk := #program[..<length] // comment
22 /// #program |>= &[length<=..]
23 /// yield [chunk, type, #sourceLocation] as tuple
24 /// linesInChunk := chunk.split '\n'
25 // if linesInChunk# > 1 // comment
26 #sourceLocation.0 += linesInChunk# - 1
27 #sourceLocation.1 = 1
28 #sourceLocation.1 += linesInChunk.-1#
29 continue outer
30 throw new SyntaxError
31 `Unrecognized token starting with '${#program.0}' at input:${#sourceLocation.join ':'}`
32
33 #function collectUntil<T>(iter: Iterator<T>, pred: (arg: T) => boolean)
34 # loop
35 # next := iter.next()
36 # break if next.done or pred next.value
37 # yield next.value