| 1 |
import token |
| 2 |
import tokenize |
| 3 |
from collections.abc import Iterator |
| 4 |
|
| 5 |
Mark = int # NewType('Mark', int) |
| 6 |
|
| 7 |
exact_token_types = token.EXACT_TOKEN_TYPES |
| 8 |
|
| 9 |
|
| 10 |
def shorttok(tok: tokenize.TokenInfo) -> str: |
| 11 |
formatted = ( |
| 12 |
f"{tok.start[0]}.{tok.start[1]}: " |
| 13 |
f"{token.tok_name[tok.type]}:{tok.string!r}" |
| 14 |
) |
| 15 |
return f"{formatted:<25.25}" |
| 16 |
|
| 17 |
|
| 18 |
class Tokenizer: |
| 19 |
"""Caching wrapper for the tokenize module. |
| 20 |
|
| 21 |
This is pretty tied to Python's syntax. |
| 22 |
""" |
| 23 |
|
| 24 |
_tokens: list[tokenize.TokenInfo] |
| 25 |
|
| 26 |
def __init__( |
| 27 |
self, tokengen: Iterator[tokenize.TokenInfo], *, path: str = "", verbose: bool = False |
| 28 |
): |
| 29 |
self._tokengen = tokengen |
| 30 |
self._tokens = [] |
| 31 |
self._index = 0 |
| 32 |
self._verbose = verbose |
| 33 |
self._lines: dict[int, str] = {} |
| 34 |
self._path = path |
| 35 |
if verbose: |
| 36 |
self.report(False, False) |
| 37 |
|
| 38 |
def getnext(self) -> tokenize.TokenInfo: |
| 39 |
"""Return the next token and updates the index.""" |
| 40 |
cached = not self._index == len(self._tokens) |
| 41 |
tok = self.peek() |
| 42 |
self._index += 1 |
| 43 |
if self._verbose: |
| 44 |
self.report(cached, False) |
| 45 |
return tok |
| 46 |
|
| 47 |
def peek(self) -> tokenize.TokenInfo: |
| 48 |
"""Return the next token *without* updating the index.""" |
| 49 |
while self._index == len(self._tokens): |
| 50 |
tok = next(self._tokengen) |
| 51 |
if tok.type in (tokenize.NL, tokenize.COMMENT): |
| 52 |
continue |
| 53 |
if tok.type == token.ERRORTOKEN and tok.string.isspace(): |
| 54 |
continue |
| 55 |
if ( |
| 56 |
tok.type == token.NEWLINE |
| 57 |
and self._tokens |
| 58 |
and self._tokens[-1].type == token.NEWLINE |
| 59 |
): |
| 60 |
continue |
| 61 |
self._tokens.append(tok) |
| 62 |
if not self._path: |
| 63 |
self._lines[tok.start[0]] = tok.line |
| 64 |
return self._tokens[self._index] |
| 65 |
|
| 66 |
def diagnose(self) -> tokenize.TokenInfo: |
| 67 |
if not self._tokens: |
| 68 |
self.getnext() |
| 69 |
return self._tokens[-1] |
| 70 |
|
| 71 |
def get_last_non_whitespace_token(self) -> tokenize.TokenInfo: |
| 72 |
for tok in reversed(self._tokens[: self._index]): |
| 73 |
if tok.type != tokenize.ENDMARKER and ( |
| 74 |
tok.type < tokenize.NEWLINE or tok.type > tokenize.DEDENT |
| 75 |
): |
| 76 |
break |
| 77 |
return tok |
| 78 |
|
| 79 |
def get_lines(self, line_numbers: list[int]) -> list[str]: |
| 80 |
"""Retrieve source lines corresponding to line numbers.""" |
| 81 |
if self._lines: |
| 82 |
lines = self._lines |
| 83 |
else: |
| 84 |
n = len(line_numbers) |
| 85 |
lines = {} |
| 86 |
count = 0 |
| 87 |
seen = 0 |
| 88 |
with open(self._path) as f: |
| 89 |
for l in f: |
| 90 |
count += 1 |
| 91 |
if count in line_numbers: |
| 92 |
seen += 1 |
| 93 |
lines[count] = l |
| 94 |
if seen == n: |
| 95 |
break |
| 96 |
|
| 97 |
return [lines[n] for n in line_numbers] |
| 98 |
|
| 99 |
def mark(self) -> Mark: |
| 100 |
return self._index |
| 101 |
|
| 102 |
def reset(self, index: Mark) -> None: |
| 103 |
if index == self._index: |
| 104 |
return |
| 105 |
assert 0 <= index <= len(self._tokens), (index, len(self._tokens)) |
| 106 |
old_index = self._index |
| 107 |
self._index = index |
| 108 |
if self._verbose: |
| 109 |
self.report(True, index < old_index) |
| 110 |
|
| 111 |
def report(self, cached: bool, back: bool) -> None: |
| 112 |
if back: |
| 113 |
fill = "-" * self._index + "-" |
| 114 |
elif cached: |
| 115 |
fill = "-" * self._index + ">" |
| 116 |
else: |
| 117 |
fill = "-" * self._index + "*" |
| 118 |
if self._index == 0: |
| 119 |
print(f"{fill} (Bof)") |
| 120 |
else: |
| 121 |
tok = self._tokens[self._index - 1] |
| 122 |
print(f"{fill} {shorttok(tok)}") |