| 1 |
civet coffeeComment |
| 2 |
{ fail } from node:assert |
| 3 |
{ tokens, type Token } from ./tokens.civet |
| 4 |
/* |
| 5 |
type * as Ast from ./astTypes.civet |
| 6 |
assert from ./assert.civet |
| 7 |
*/ |
| 8 |
|
| 9 |
/// TokenType ::= keyof typeof tokens |
| 10 |
/// tokenEntries := Object.entries(tokens) as [TokenType, Token][] |
| 11 |
|
| 12 |
class TokenStream <: Iterable<[string, TokenType, readonly [number, number]]> |
| 13 |
#sourceLocation = [1, 1] as tuple |
| 14 |
@(#program: string) |
| 15 |
|
| 16 |
:iterator() ### |
| 17 |
:outer while #program# ### |
| 18 |
for [type, token] of tokenEntries |
| 19 |
length := token.matchLength #program |
| 20 |
/// if length > 0 |
| 21 |
// chunk := #program[..<length] // comment |
| 22 |
/// #program |>= &[length<=..] |
| 23 |
/// yield [chunk, type, #sourceLocation] as tuple |
| 24 |
/// linesInChunk := chunk.split '\n' |
| 25 |
// if linesInChunk# > 1 // comment |
| 26 |
#sourceLocation.0 += linesInChunk# - 1 |
| 27 |
#sourceLocation.1 = 1 |
| 28 |
#sourceLocation.1 += linesInChunk.-1# |
| 29 |
continue outer |
| 30 |
throw new SyntaxError |
| 31 |
`Unrecognized token starting with '${#program.0}' at input:${#sourceLocation.join ':'}` |
| 32 |
|
| 33 |
#function collectUntil<T>(iter: Iterator<T>, pred: (arg: T) => boolean) |
| 34 |
# loop |
| 35 |
# next := iter.next() |
| 36 |
# break if next.done or pred next.value |
| 37 |
# yield next.value |