Debian Codegraph @

Tools/peg_generator/pegen/tokenizer.py

raw
1 import token
2 import tokenize
3 from collections.abc import Iterator
4
5 Mark = int # NewType('Mark', int)
6
7 exact_token_types = token.EXACT_TOKEN_TYPES
8
9
10 def shorttok(tok: tokenize.TokenInfo) -> str:
11 formatted = (
12 f"{tok.start[0]}.{tok.start[1]}: "
13 f"{token.tok_name[tok.type]}:{tok.string!r}"
14 )
15 return f"{formatted:<25.25}"
16
17
18 class Tokenizer:
19 """Caching wrapper for the tokenize module.
20
21 This is pretty tied to Python's syntax.
22 """
23
24 _tokens: list[tokenize.TokenInfo]
25
26 def __init__(
27 self, tokengen: Iterator[tokenize.TokenInfo], *, path: str = "", verbose: bool = False
28 ):
29 self._tokengen = tokengen
30 self._tokens = []
31 self._index = 0
32 self._verbose = verbose
33 self._lines: dict[int, str] = {}
34 self._path = path
35 if verbose:
36 self.report(False, False)
37
38 def getnext(self) -> tokenize.TokenInfo:
39 """Return the next token and updates the index."""
40 cached = not self._index == len(self._tokens)
41 tok = self.peek()
42 self._index += 1
43 if self._verbose:
44 self.report(cached, False)
45 return tok
46
47 def peek(self) -> tokenize.TokenInfo:
48 """Return the next token *without* updating the index."""
49 while self._index == len(self._tokens):
50 tok = next(self._tokengen)
51 if tok.type in (tokenize.NL, tokenize.COMMENT):
52 continue
53 if tok.type == token.ERRORTOKEN and tok.string.isspace():
54 continue
55 if (
56 tok.type == token.NEWLINE
57 and self._tokens
58 and self._tokens[-1].type == token.NEWLINE
59 ):
60 continue
61 self._tokens.append(tok)
62 if not self._path:
63 self._lines[tok.start[0]] = tok.line
64 return self._tokens[self._index]
65
66 def diagnose(self) -> tokenize.TokenInfo:
67 if not self._tokens:
68 self.getnext()
69 return self._tokens[-1]
70
71 def get_last_non_whitespace_token(self) -> tokenize.TokenInfo:
72 for tok in reversed(self._tokens[: self._index]):
73 if tok.type != tokenize.ENDMARKER and (
74 tok.type < tokenize.NEWLINE or tok.type > tokenize.DEDENT
75 ):
76 break
77 return tok
78
79 def get_lines(self, line_numbers: list[int]) -> list[str]:
80 """Retrieve source lines corresponding to line numbers."""
81 if self._lines:
82 lines = self._lines
83 else:
84 n = len(line_numbers)
85 lines = {}
86 count = 0
87 seen = 0
88 with open(self._path) as f:
89 for l in f:
90 count += 1
91 if count in line_numbers:
92 seen += 1
93 lines[count] = l
94 if seen == n:
95 break
96
97 return [lines[n] for n in line_numbers]
98
99 def mark(self) -> Mark:
100 return self._index
101
102 def reset(self, index: Mark) -> None:
103 if index == self._index:
104 return
105 assert 0 <= index <= len(self._tokens), (index, len(self._tokens))
106 old_index = self._index
107 self._index = index
108 if self._verbose:
109 self.report(True, index < old_index)
110
111 def report(self, cached: bool, back: bool) -> None:
112 if back:
113 fill = "-" * self._index + "-"
114 elif cached:
115 fill = "-" * self._index + ">"
116 else:
117 fill = "-" * self._index + "*"
118 if self._index == 0:
119 print(f"{fill} (Bof)")
120 else:
121 tok = self._tokens[self._index - 1]
122 print(f"{fill} {shorttok(tok)}")