| 1 |
from __future__ import annotations |
| 2 |
import builtins |
| 3 |
import functools |
| 4 |
import keyword |
| 5 |
import re |
| 6 |
import token as T |
| 7 |
import tokenize |
| 8 |
import unicodedata |
| 9 |
import _colorize |
| 10 |
|
| 11 |
from collections import deque |
| 12 |
from io import StringIO |
| 13 |
from tokenize import TokenInfo as TI |
| 14 |
from typing import Iterable, Iterator, Match, NamedTuple, Self |
| 15 |
|
| 16 |
from .types import CharBuffer, CharWidths |
| 17 |
from .trace import trace |
| 18 |
|
| 19 |
ANSI_ESCAPE_SEQUENCE = re.compile(r"\x1b\[[ -@]*[A-~]") |
| 20 |
ZERO_WIDTH_BRACKET = re.compile(r"\x01.*?\x02") |
| 21 |
ZERO_WIDTH_TRANS = str.maketrans({"\x01": "", "\x02": ""}) |
| 22 |
IDENTIFIERS_AFTER = {"def", "class"} |
| 23 |
KEYWORD_CONSTANTS = {"True", "False", "None"} |
| 24 |
BUILTINS = {str(name) for name in dir(builtins) if not name.startswith('_')} |
| 25 |
|
| 26 |
|
| 27 |
def THEME(**kwargs): |
| 28 |
# Not cached: the user can modify the theme inside the interactive session. |
| 29 |
return _colorize.get_theme(**kwargs).syntax |
| 30 |
|
| 31 |
|
| 32 |
class Span(NamedTuple): |
| 33 |
"""Span indexing that's inclusive on both ends.""" |
| 34 |
|
| 35 |
start: int |
| 36 |
end: int |
| 37 |
|
| 38 |
@classmethod |
| 39 |
def from_re(cls, m: Match[str], group: int | str) -> Self: |
| 40 |
re_span = m.span(group) |
| 41 |
return cls(re_span[0], re_span[1] - 1) |
| 42 |
|
| 43 |
@classmethod |
| 44 |
def from_token(cls, token: TI, line_len: list[int]) -> Self: |
| 45 |
end_offset = -1 |
| 46 |
if (token.type in {T.FSTRING_MIDDLE, T.TSTRING_MIDDLE} |
| 47 |
and token.string.endswith(("{", "}"))): |
| 48 |
# gh-134158: a visible trailing brace comes from a double brace in input |
| 49 |
end_offset += 1 |
| 50 |
|
| 51 |
return cls( |
| 52 |
line_len[token.start[0] - 1] + token.start[1], |
| 53 |
line_len[token.end[0] - 1] + token.end[1] + end_offset, |
| 54 |
) |
| 55 |
|
| 56 |
|
| 57 |
class ColorSpan(NamedTuple): |
| 58 |
span: Span |
| 59 |
tag: str |
| 60 |
|
| 61 |
|
| 62 |
@functools.cache |
| 63 |
def str_width(c: str) -> int: |
| 64 |
if ord(c) < 128: |
| 65 |
return 1 |
| 66 |
# gh-139246 for zero-width joiner and combining characters |
| 67 |
if unicodedata.combining(c): |
| 68 |
return 0 |
| 69 |
category = unicodedata.category(c) |
| 70 |
if category == "Cf" and c != "\u00ad": |
| 71 |
return 0 |
| 72 |
w = unicodedata.east_asian_width(c) |
| 73 |
if w in ("N", "Na", "H", "A"): |
| 74 |
return 1 |
| 75 |
return 2 |
| 76 |
|
| 77 |
|
| 78 |
def wlen(s: str) -> int: |
| 79 |
if len(s) == 1 and s != "\x1a": |
| 80 |
return str_width(s) |
| 81 |
length = sum(str_width(i) for i in s) |
| 82 |
# remove lengths of any escape sequences |
| 83 |
sequence = ANSI_ESCAPE_SEQUENCE.findall(s) |
| 84 |
ctrl_z_cnt = s.count("\x1a") |
| 85 |
return length - sum(len(i) for i in sequence) + ctrl_z_cnt |
| 86 |
|
| 87 |
|
| 88 |
def unbracket(s: str, including_content: bool = False) -> str: |
| 89 |
r"""Return `s` with \001 and \002 characters removed. |
| 90 |
|
| 91 |
If `including_content` is True, content between \001 and \002 is also |
| 92 |
stripped. |
| 93 |
""" |
| 94 |
if including_content: |
| 95 |
return ZERO_WIDTH_BRACKET.sub("", s) |
| 96 |
return s.translate(ZERO_WIDTH_TRANS) |
| 97 |
|
| 98 |
|
| 99 |
def gen_colors(buffer: str) -> Iterator[ColorSpan]: |
| 100 |
"""Returns a list of index spans to color using the given color tag. |
| 101 |
|
| 102 |
The input `buffer` should be a valid start of a Python code block, i.e. |
| 103 |
it cannot be a block starting in the middle of a multiline string. |
| 104 |
""" |
| 105 |
sio = StringIO(buffer) |
| 106 |
line_lengths = [0] + [len(line) for line in sio.readlines()] |
| 107 |
# make line_lengths cumulative |
| 108 |
for i in range(1, len(line_lengths)): |
| 109 |
line_lengths[i] += line_lengths[i-1] |
| 110 |
|
| 111 |
sio.seek(0) |
| 112 |
gen = tokenize.generate_tokens(sio.readline) |
| 113 |
last_emitted: ColorSpan | None = None |
| 114 |
try: |
| 115 |
for color in gen_colors_from_token_stream(gen, line_lengths): |
| 116 |
yield color |
| 117 |
last_emitted = color |
| 118 |
except SyntaxError: |
| 119 |
return |
| 120 |
except tokenize.TokenError as te: |
| 121 |
yield from recover_unterminated_string( |
| 122 |
te, line_lengths, last_emitted, buffer |
| 123 |
) |
| 124 |
|
| 125 |
|
| 126 |
def recover_unterminated_string( |
| 127 |
exc: tokenize.TokenError, |
| 128 |
line_lengths: list[int], |
| 129 |
last_emitted: ColorSpan | None, |
| 130 |
buffer: str, |
| 131 |
) -> Iterator[ColorSpan]: |
| 132 |
msg, loc = exc.args |
| 133 |
if loc is None: |
| 134 |
return |
| 135 |
|
| 136 |
line_no, column = loc |
| 137 |
|
| 138 |
if msg.startswith( |
| 139 |
( |
| 140 |
"unterminated string literal", |
| 141 |
"unterminated f-string literal", |
| 142 |
"unterminated t-string literal", |
| 143 |
"EOF in multi-line string", |
| 144 |
"unterminated triple-quoted f-string literal", |
| 145 |
"unterminated triple-quoted t-string literal", |
| 146 |
) |
| 147 |
): |
| 148 |
start = line_lengths[line_no - 1] + column - 1 |
| 149 |
end = line_lengths[-1] - 1 |
| 150 |
|
| 151 |
# in case FSTRING_START was already emitted |
| 152 |
if last_emitted and start <= last_emitted.span.start: |
| 153 |
trace("before last emitted = {s}", s=start) |
| 154 |
start = last_emitted.span.end + 1 |
| 155 |
|
| 156 |
span = Span(start, end) |
| 157 |
trace("yielding span {a} -> {b}", a=span.start, b=span.end) |
| 158 |
yield ColorSpan(span, "string") |
| 159 |
else: |
| 160 |
trace( |
| 161 |
"unhandled token error({buffer}) = {te}", |
| 162 |
buffer=repr(buffer), |
| 163 |
te=str(exc), |
| 164 |
) |
| 165 |
|
| 166 |
|
| 167 |
def gen_colors_from_token_stream( |
| 168 |
token_generator: Iterator[TI], |
| 169 |
line_lengths: list[int], |
| 170 |
) -> Iterator[ColorSpan]: |
| 171 |
token_window = prev_next_window(token_generator) |
| 172 |
|
| 173 |
is_def_name = False |
| 174 |
bracket_level = 0 |
| 175 |
for prev_token, token, next_token in token_window: |
| 176 |
assert token is not None |
| 177 |
if token.start == token.end: |
| 178 |
continue |
| 179 |
|
| 180 |
match token.type: |
| 181 |
case ( |
| 182 |
T.STRING |
| 183 |
| T.FSTRING_START | T.FSTRING_MIDDLE | T.FSTRING_END |
| 184 |
| T.TSTRING_START | T.TSTRING_MIDDLE | T.TSTRING_END |
| 185 |
): |
| 186 |
span = Span.from_token(token, line_lengths) |
| 187 |
yield ColorSpan(span, "string") |
| 188 |
case T.COMMENT: |
| 189 |
span = Span.from_token(token, line_lengths) |
| 190 |
yield ColorSpan(span, "comment") |
| 191 |
case T.NUMBER: |
| 192 |
span = Span.from_token(token, line_lengths) |
| 193 |
yield ColorSpan(span, "number") |
| 194 |
case T.OP: |
| 195 |
if token.string in "([{": |
| 196 |
bracket_level += 1 |
| 197 |
elif token.string in ")]}": |
| 198 |
bracket_level -= 1 |
| 199 |
span = Span.from_token(token, line_lengths) |
| 200 |
yield ColorSpan(span, "op") |
| 201 |
case T.NAME: |
| 202 |
if is_def_name: |
| 203 |
is_def_name = False |
| 204 |
span = Span.from_token(token, line_lengths) |
| 205 |
yield ColorSpan(span, "definition") |
| 206 |
elif keyword.iskeyword(token.string): |
| 207 |
span_cls = "keyword" |
| 208 |
if token.string in KEYWORD_CONSTANTS: |
| 209 |
span_cls = "keyword_constant" |
| 210 |
span = Span.from_token(token, line_lengths) |
| 211 |
yield ColorSpan(span, span_cls) |
| 212 |
if token.string in IDENTIFIERS_AFTER: |
| 213 |
is_def_name = True |
| 214 |
elif ( |
| 215 |
keyword.issoftkeyword(token.string) |
| 216 |
and bracket_level == 0 |
| 217 |
and is_soft_keyword_used(prev_token, token, next_token) |
| 218 |
): |
| 219 |
span = Span.from_token(token, line_lengths) |
| 220 |
yield ColorSpan(span, "soft_keyword") |
| 221 |
elif ( |
| 222 |
token.string in BUILTINS |
| 223 |
and not (prev_token and prev_token.exact_type == T.DOT) |
| 224 |
): |
| 225 |
span = Span.from_token(token, line_lengths) |
| 226 |
yield ColorSpan(span, "builtin") |
| 227 |
|
| 228 |
|
| 229 |
keyword_first_sets_match = {"False", "None", "True", "await", "lambda", "not"} |
| 230 |
keyword_first_sets_case = {"False", "None", "True"} |
| 231 |
|
| 232 |
|
| 233 |
def is_soft_keyword_used(*tokens: TI | None) -> bool: |
| 234 |
"""Returns True if the current token is a keyword in this context. |
| 235 |
|
| 236 |
For the `*tokens` to match anything, they have to be a three-tuple of |
| 237 |
(previous, current, next). |
| 238 |
""" |
| 239 |
trace("is_soft_keyword_used{t}", t=tokens) |
| 240 |
match tokens: |
| 241 |
case ( |
| 242 |
None | TI(T.NEWLINE) | TI(T.INDENT) | TI(string=":"), |
| 243 |
TI(string="match"), |
| 244 |
TI(T.NUMBER | T.STRING | T.FSTRING_START | T.TSTRING_START) |
| 245 |
| TI(T.OP, string="(" | "*" | "[" | "{" | "~" | "...") |
| 246 |
): |
| 247 |
return True |
| 248 |
case ( |
| 249 |
None | TI(T.NEWLINE) | TI(T.INDENT) | TI(string=":"), |
| 250 |
TI(string="match"), |
| 251 |
TI(T.NAME, string=s) |
| 252 |
): |
| 253 |
if keyword.iskeyword(s): |
| 254 |
return s in keyword_first_sets_match |
| 255 |
return True |
| 256 |
case ( |
| 257 |
None | TI(T.NEWLINE) | TI(T.INDENT) | TI(T.DEDENT) | TI(string=":"), |
| 258 |
TI(string="case"), |
| 259 |
TI(T.NUMBER | T.STRING | T.FSTRING_START | T.TSTRING_START) |
| 260 |
| TI(T.OP, string="(" | "*" | "-" | "[" | "{") |
| 261 |
): |
| 262 |
return True |
| 263 |
case ( |
| 264 |
None | TI(T.NEWLINE) | TI(T.INDENT) | TI(T.DEDENT) | TI(string=":"), |
| 265 |
TI(string="case"), |
| 266 |
TI(T.NAME, string=s) |
| 267 |
): |
| 268 |
if keyword.iskeyword(s): |
| 269 |
return s in keyword_first_sets_case |
| 270 |
return True |
| 271 |
case (TI(string="case"), TI(string="_"), TI(string=":")): |
| 272 |
return True |
| 273 |
case ( |
| 274 |
None | TI(T.NEWLINE) | TI(T.INDENT) | TI(T.DEDENT) | TI(string=":"), |
| 275 |
TI(string="type"), |
| 276 |
TI(T.NAME, string=s) |
| 277 |
): |
| 278 |
return not keyword.iskeyword(s) |
| 279 |
case _: |
| 280 |
return False |
| 281 |
|
| 282 |
|
| 283 |
def disp_str( |
| 284 |
buffer: str, |
| 285 |
colors: list[ColorSpan] | None = None, |
| 286 |
start_index: int = 0, |
| 287 |
force_color: bool = False, |
| 288 |
) -> tuple[CharBuffer, CharWidths]: |
| 289 |
r"""Decompose the input buffer into a printable variant with applied colors. |
| 290 |
|
| 291 |
Returns a tuple of two lists: |
| 292 |
- the first list is the input buffer, character by character, with color |
| 293 |
escape codes added (while those codes contain multiple ASCII characters, |
| 294 |
each code is considered atomic *and is attached for the corresponding |
| 295 |
visible character*); |
| 296 |
- the second list is the visible width of each character in the input |
| 297 |
buffer. |
| 298 |
|
| 299 |
Note on colors: |
| 300 |
- The `colors` list, if provided, is partially consumed within. We're using |
| 301 |
a list and not a generator since we need to hold onto the current |
| 302 |
unfinished span between calls to disp_str in case of multiline strings. |
| 303 |
- The `colors` list is computed from the start of the input block. `buffer` |
| 304 |
is only a subset of that input block, a single line within. This is why |
| 305 |
we need `start_index` to inform us which position is the start of `buffer` |
| 306 |
actually within user input. This allows us to match color spans correctly. |
| 307 |
|
| 308 |
Examples: |
| 309 |
>>> utils.disp_str("a = 9") |
| 310 |
(['a', ' ', '=', ' ', '9'], [1, 1, 1, 1, 1]) |
| 311 |
|
| 312 |
>>> line = "while 1:" |
| 313 |
>>> colors = list(utils.gen_colors(line)) |
| 314 |
>>> utils.disp_str(line, colors=colors) |
| 315 |
(['\x1b[1;34mw', 'h', 'i', 'l', 'e\x1b[0m', ' ', '1', ':'], [1, 1, 1, 1, 1, 1, 1, 1]) |
| 316 |
|
| 317 |
""" |
| 318 |
chars: CharBuffer = [] |
| 319 |
char_widths: CharWidths = [] |
| 320 |
|
| 321 |
if not buffer: |
| 322 |
return chars, char_widths |
| 323 |
|
| 324 |
while colors and colors[0].span.end < start_index: |
| 325 |
# move past irrelevant spans |
| 326 |
colors.pop(0) |
| 327 |
|
| 328 |
theme = THEME(force_color=force_color) |
| 329 |
pre_color = "" |
| 330 |
post_color = "" |
| 331 |
if colors and colors[0].span.start < start_index: |
| 332 |
# looks like we're continuing a previous color (e.g. a multiline str) |
| 333 |
pre_color = theme[colors[0].tag] |
| 334 |
|
| 335 |
for i, c in enumerate(buffer, start_index): |
| 336 |
if colors and colors[0].span.start == i: # new color starts now |
| 337 |
pre_color = theme[colors[0].tag] |
| 338 |
|
| 339 |
if c == "\x1a": # CTRL-Z on Windows |
| 340 |
chars.append(c) |
| 341 |
char_widths.append(2) |
| 342 |
elif ord(c) < 128: |
| 343 |
chars.append(c) |
| 344 |
char_widths.append(1) |
| 345 |
elif unicodedata.category(c).startswith("C"): |
| 346 |
c = r"\u%04x" % ord(c) |
| 347 |
chars.append(c) |
| 348 |
char_widths.append(len(c)) |
| 349 |
else: |
| 350 |
chars.append(c) |
| 351 |
char_widths.append(str_width(c)) |
| 352 |
|
| 353 |
if colors and colors[0].span.end == i: # current color ends now |
| 354 |
post_color = theme.reset |
| 355 |
colors.pop(0) |
| 356 |
|
| 357 |
chars[-1] = pre_color + chars[-1] + post_color |
| 358 |
pre_color = "" |
| 359 |
post_color = "" |
| 360 |
|
| 361 |
if colors and colors[0].span.start < i and colors[0].span.end > i: |
| 362 |
# even though the current color should be continued, reset it for now. |
| 363 |
# the next call to `disp_str()` will revive it. |
| 364 |
chars[-1] += theme.reset |
| 365 |
|
| 366 |
return chars, char_widths |
| 367 |
|
| 368 |
|
| 369 |
def prev_next_window[T]( |
| 370 |
iterable: Iterable[T] |
| 371 |
) -> Iterator[tuple[T | None, ...]]: |
| 372 |
"""Generates three-tuples of (previous, current, next) items. |
| 373 |
|
| 374 |
On the first iteration previous is None. On the last iteration next |
| 375 |
is None. In case of exception next is None and the exception is re-raised |
| 376 |
on a subsequent next() call. |
| 377 |
|
| 378 |
Inspired by `sliding_window` from `itertools` recipes. |
| 379 |
""" |
| 380 |
|
| 381 |
iterator = iter(iterable) |
| 382 |
window = deque((None, next(iterator)), maxlen=3) |
| 383 |
try: |
| 384 |
for x in iterator: |
| 385 |
window.append(x) |
| 386 |
yield tuple(window) |
| 387 |
except Exception: |
| 388 |
raise |
| 389 |
finally: |
| 390 |
window.append(None) |
| 391 |
yield tuple(window) |