| 1 |
{ fail } from node:assert |
| 2 |
{ tokens, type Token } from ./tokens.civet |
| 3 |
/* |
| 4 |
type * as Ast from ./astTypes.civet |
| 5 |
assert from ./assert.civet |
| 6 |
*/ |
| 7 |
|
| 8 |
/// TokenType ::= keyof typeof tokens |
| 9 |
/// tokenEntries := Object.entries(tokens) as [TokenType, Token][] |
| 10 |
|
| 11 |
class TokenStream <: Iterable<[string, TokenType, readonly [number, number]]> |
| 12 |
#sourceLocation = [1, 1] as tuple |
| 13 |
@(#program: string) |
| 14 |
|
| 15 |
:iterator() ### |
| 16 |
:outer while #program# ### |
| 17 |
for [type, token] of tokenEntries |
| 18 |
length := token.matchLength #program |
| 19 |
/// if length > 0 |
| 20 |
// chunk := #program[..<length] // comment |
| 21 |
/// #program |>= &[length<=..] |
| 22 |
/// yield [chunk, type, #sourceLocation] as tuple |
| 23 |
/// linesInChunk := chunk.split '\n' |
| 24 |
// if linesInChunk# > 1 // comment |
| 25 |
#sourceLocation.0 += linesInChunk# - 1 |
| 26 |
#sourceLocation.1 = 1 |
| 27 |
#sourceLocation.1 += linesInChunk.-1# |
| 28 |
continue outer |
| 29 |
throw new SyntaxError |
| 30 |
`Unrecognized token starting with '${#program.0}' at input:${#sourceLocation.join ':'}` |
| 31 |
|
| 32 |
function collectUntil<T>(iter: Iterator<T>, pred: (arg: T) => boolean) |
| 33 |
loop |
| 34 |
next := iter.next() |
| 35 |
break if next.done or pred next.value |
| 36 |
yield next.value |
| 37 |
|
| 38 |
processExpression := (expr: string, line: number, column: number) => |
| 39 |
processSplits := (parts: string[]): Ast.NumberSyntaxTree => |
| 40 |
if parts# % 2 is 0 |
| 41 |
throw new SyntaxError `Incomplete expression: '${parts.join ''}' (near input:${line}:${column})` |
| 42 |
if parts# > 2 |
| 43 |
type := switch parts.-2 |
| 44 |
'+' |
| 45 |
'addition' as const |
| 46 |
'_' |
| 47 |
'subtraction' as const |
| 48 |
else |
| 49 |
throw new SyntaxError |
| 50 |
`Missing operator in expression containing '${parts[-3...].join ''}' (near input:${line}:${column})` |
| 51 |
{ |
| 52 |
type |
| 53 |
value: |
| 54 |
. processSplits parts[...-2] |
| 55 |
. processSplits [parts.-1] |
| 56 |
} |
| 57 |
else |
| 58 |
part := parts.0 |
| 59 |
switch part |
| 60 |
'+', '_' |
| 61 |
throw new SyntaxError `Unexpected operator with no operands (near input:${line}:${column})` |
| 62 |
/\p{Letter}+/v |
| 63 |
type: 'variable', value: part |
| 64 |
/[0-9]+/ |
| 65 |
type: 'literal', value: Number part |
| 66 |
else |
| 67 |
fail(); |
| 68 |
|
| 69 |
splitsAndEmpty := expr.split /(\+|_|\p{Letter}+|[0-9]+)/gv |
| 70 |
splits := splitsAndEmpty.flatMap (el, i) => |
| 71 |
if i % 2 is 0 |
| 72 |
assert => el is '' |
| 73 |
[] |
| 74 |
else |
| 75 |
[el] |
| 76 |
return processSplits splits |