Debian Codegraph @

Tools/cases_generator/plexer.py

raw
1 import lexer as lx
2
3 Token = lx.Token
4
5
6 class PLexer:
7 def __init__(self, src: str, filename: str):
8 self.src = src
9 self.filename = filename
10 self.tokens = list(lx.tokenize(self.src, filename=filename))
11 self.pos = 0
12
13 def getpos(self) -> int:
14 # Current position
15 return self.pos
16
17 def eof(self) -> bool:
18 # Are we at EOF?
19 return self.pos >= len(self.tokens)
20
21 def setpos(self, pos: int) -> None:
22 # Reset position
23 assert 0 <= pos <= len(self.tokens), (pos, len(self.tokens))
24 self.pos = pos
25
26 def backup(self) -> None:
27 # Back up position by 1
28 assert self.pos > 0
29 self.pos -= 1
30
31 def next(self, raw: bool = False) -> Token | None:
32 # Return next token and advance position; None if at EOF
33 # TODO: Return synthetic EOF token instead of None?
34 while self.pos < len(self.tokens):
35 tok = self.tokens[self.pos]
36 self.pos += 1
37 if raw or tok.kind != "COMMENT":
38 return tok
39 return None
40
41 def peek(self, raw: bool = False) -> Token | None:
42 # Return next token without advancing position
43 tok = self.next(raw=raw)
44 self.backup()
45 return tok
46
47 def maybe(self, kind: str, raw: bool = False) -> Token | None:
48 # Return next token without advancing position if kind matches
49 tok = self.peek(raw=raw)
50 if tok and tok.kind == kind:
51 return tok
52 return None
53
54 def expect(self, kind: str) -> Token | None:
55 # Return next token and advance position if kind matches
56 tkn = self.next()
57 if tkn is not None:
58 if tkn.kind == kind:
59 return tkn
60 self.backup()
61 return None
62
63 def require(self, kind: str) -> Token:
64 # Return next token and advance position, requiring kind to match
65 tkn = self.next()
66 if tkn is not None and tkn.kind == kind:
67 return tkn
68 raise self.make_syntax_error(
69 f"Expected {kind!r} but got {tkn and tkn.text!r}", tkn
70 )
71
72 def consume_to(self, end: str) -> list[Token]:
73 res: list[Token] = []
74 parens = 0
75 while tkn := self.next(raw=True):
76 res.append(tkn)
77 if tkn.kind == end and parens == 0:
78 return res
79 if tkn.kind == "LPAREN":
80 parens += 1
81 if tkn.kind == "RPAREN":
82 parens -= 1
83 raise self.make_syntax_error(
84 f"Expected {end!r} but reached EOF", tkn)
85
86 def extract_line(self, lineno: int) -> str:
87 # Return source line `lineno` (1-based)
88 lines = self.src.splitlines()
89 if lineno > len(lines):
90 return ""
91 return lines[lineno - 1]
92
93 def make_syntax_error(self, message: str, tkn: Token | None = None) -> SyntaxError:
94 # Construct a SyntaxError instance from message and token
95 if tkn is None:
96 tkn = self.peek()
97 if tkn is None:
98 tkn = self.tokens[-1]
99 return lx.make_syntax_error(
100 message, self.filename, tkn.line, tkn.column, self.extract_line(tkn.line)
101 )
102
103
104 if __name__ == "__main__":
105 import sys
106
107 if sys.argv[1:]:
108 filename = sys.argv[1]
109 if filename == "-c" and sys.argv[2:]:
110 src = sys.argv[2]
111 filename = "<string>"
112 else:
113 with open(filename) as f:
114 src = f.read()
115 else:
116 filename = "<default>"
117 src = "if (x) { x.foo; // comment\n}"
118 p = PLexer(src, filename)
119 while not p.eof():
120 tok = p.next(raw=True)
121 assert tok
122 left = repr(tok)
123 right = lx.to_text([tok]).rstrip()
124 print(f"{left:40.40} {right}")