| 1 |
import lexer as lx |
| 2 |
|
| 3 |
Token = lx.Token |
| 4 |
|
| 5 |
|
| 6 |
class PLexer: |
| 7 |
def __init__(self, src: str, filename: str): |
| 8 |
self.src = src |
| 9 |
self.filename = filename |
| 10 |
self.tokens = list(lx.tokenize(self.src, filename=filename)) |
| 11 |
self.pos = 0 |
| 12 |
|
| 13 |
def getpos(self) -> int: |
| 14 |
# Current position |
| 15 |
return self.pos |
| 16 |
|
| 17 |
def eof(self) -> bool: |
| 18 |
# Are we at EOF? |
| 19 |
return self.pos >= len(self.tokens) |
| 20 |
|
| 21 |
def setpos(self, pos: int) -> None: |
| 22 |
# Reset position |
| 23 |
assert 0 <= pos <= len(self.tokens), (pos, len(self.tokens)) |
| 24 |
self.pos = pos |
| 25 |
|
| 26 |
def backup(self) -> None: |
| 27 |
# Back up position by 1 |
| 28 |
assert self.pos > 0 |
| 29 |
self.pos -= 1 |
| 30 |
|
| 31 |
def next(self, raw: bool = False) -> Token | None: |
| 32 |
# Return next token and advance position; None if at EOF |
| 33 |
# TODO: Return synthetic EOF token instead of None? |
| 34 |
while self.pos < len(self.tokens): |
| 35 |
tok = self.tokens[self.pos] |
| 36 |
self.pos += 1 |
| 37 |
if raw or tok.kind != "COMMENT": |
| 38 |
return tok |
| 39 |
return None |
| 40 |
|
| 41 |
def peek(self, raw: bool = False) -> Token | None: |
| 42 |
# Return next token without advancing position |
| 43 |
tok = self.next(raw=raw) |
| 44 |
self.backup() |
| 45 |
return tok |
| 46 |
|
| 47 |
def maybe(self, kind: str, raw: bool = False) -> Token | None: |
| 48 |
# Return next token without advancing position if kind matches |
| 49 |
tok = self.peek(raw=raw) |
| 50 |
if tok and tok.kind == kind: |
| 51 |
return tok |
| 52 |
return None |
| 53 |
|
| 54 |
def expect(self, kind: str) -> Token | None: |
| 55 |
# Return next token and advance position if kind matches |
| 56 |
tkn = self.next() |
| 57 |
if tkn is not None: |
| 58 |
if tkn.kind == kind: |
| 59 |
return tkn |
| 60 |
self.backup() |
| 61 |
return None |
| 62 |
|
| 63 |
def require(self, kind: str) -> Token: |
| 64 |
# Return next token and advance position, requiring kind to match |
| 65 |
tkn = self.next() |
| 66 |
if tkn is not None and tkn.kind == kind: |
| 67 |
return tkn |
| 68 |
raise self.make_syntax_error( |
| 69 |
f"Expected {kind!r} but got {tkn and tkn.text!r}", tkn |
| 70 |
) |
| 71 |
|
| 72 |
def consume_to(self, end: str) -> list[Token]: |
| 73 |
res: list[Token] = [] |
| 74 |
parens = 0 |
| 75 |
while tkn := self.next(raw=True): |
| 76 |
res.append(tkn) |
| 77 |
if tkn.kind == end and parens == 0: |
| 78 |
return res |
| 79 |
if tkn.kind == "LPAREN": |
| 80 |
parens += 1 |
| 81 |
if tkn.kind == "RPAREN": |
| 82 |
parens -= 1 |
| 83 |
raise self.make_syntax_error( |
| 84 |
f"Expected {end!r} but reached EOF", tkn) |
| 85 |
|
| 86 |
def extract_line(self, lineno: int) -> str: |
| 87 |
# Return source line `lineno` (1-based) |
| 88 |
lines = self.src.splitlines() |
| 89 |
if lineno > len(lines): |
| 90 |
return "" |
| 91 |
return lines[lineno - 1] |
| 92 |
|
| 93 |
def make_syntax_error(self, message: str, tkn: Token | None = None) -> SyntaxError: |
| 94 |
# Construct a SyntaxError instance from message and token |
| 95 |
if tkn is None: |
| 96 |
tkn = self.peek() |
| 97 |
if tkn is None: |
| 98 |
tkn = self.tokens[-1] |
| 99 |
return lx.make_syntax_error( |
| 100 |
message, self.filename, tkn.line, tkn.column, self.extract_line(tkn.line) |
| 101 |
) |
| 102 |
|
| 103 |
|
| 104 |
if __name__ == "__main__": |
| 105 |
import sys |
| 106 |
|
| 107 |
if sys.argv[1:]: |
| 108 |
filename = sys.argv[1] |
| 109 |
if filename == "-c" and sys.argv[2:]: |
| 110 |
src = sys.argv[2] |
| 111 |
filename = "<string>" |
| 112 |
else: |
| 113 |
with open(filename) as f: |
| 114 |
src = f.read() |
| 115 |
else: |
| 116 |
filename = "<default>" |
| 117 |
src = "if (x) { x.foo; // comment\n}" |
| 118 |
p = PLexer(src, filename) |
| 119 |
while not p.eof(): |
| 120 |
tok = p.next(raw=True) |
| 121 |
assert tok |
| 122 |
left = repr(tok) |
| 123 |
right = lx.to_text([tok]).rstrip() |
| 124 |
print(f"{left:40.40} {right}") |