Debian Codegraph @

Lib/_pyrepl/utils.py

raw
1 from __future__ import annotations
2 import builtins
3 import functools
4 import keyword
5 import re
6 import token as T
7 import tokenize
8 import unicodedata
9 import _colorize
10
11 from collections import deque
12 from io import StringIO
13 from tokenize import TokenInfo as TI
14 from typing import Iterable, Iterator, Match, NamedTuple, Self
15
16 from .types import CharBuffer, CharWidths
17 from .trace import trace
18
19 ANSI_ESCAPE_SEQUENCE = re.compile(r"\x1b\[[ -@]*[A-~]")
20 ZERO_WIDTH_BRACKET = re.compile(r"\x01.*?\x02")
21 ZERO_WIDTH_TRANS = str.maketrans({"\x01": "", "\x02": ""})
22 IDENTIFIERS_AFTER = {"def", "class"}
23 KEYWORD_CONSTANTS = {"True", "False", "None"}
24 BUILTINS = {str(name) for name in dir(builtins) if not name.startswith('_')}
25
26
27 def THEME(**kwargs):
28 # Not cached: the user can modify the theme inside the interactive session.
29 return _colorize.get_theme(**kwargs).syntax
30
31
32 class Span(NamedTuple):
33 """Span indexing that's inclusive on both ends."""
34
35 start: int
36 end: int
37
38 @classmethod
39 def from_re(cls, m: Match[str], group: int | str) -> Self:
40 re_span = m.span(group)
41 return cls(re_span[0], re_span[1] - 1)
42
43 @classmethod
44 def from_token(cls, token: TI, line_len: list[int]) -> Self:
45 end_offset = -1
46 if (token.type in {T.FSTRING_MIDDLE, T.TSTRING_MIDDLE}
47 and token.string.endswith(("{", "}"))):
48 # gh-134158: a visible trailing brace comes from a double brace in input
49 end_offset += 1
50
51 return cls(
52 line_len[token.start[0] - 1] + token.start[1],
53 line_len[token.end[0] - 1] + token.end[1] + end_offset,
54 )
55
56
57 class ColorSpan(NamedTuple):
58 span: Span
59 tag: str
60
61
62 @functools.cache
63 def str_width(c: str) -> int:
64 if ord(c) < 128:
65 return 1
66 # gh-139246 for zero-width joiner and combining characters
67 if unicodedata.combining(c):
68 return 0
69 category = unicodedata.category(c)
70 if category == "Cf" and c != "\u00ad":
71 return 0
72 w = unicodedata.east_asian_width(c)
73 if w in ("N", "Na", "H", "A"):
74 return 1
75 return 2
76
77
78 def wlen(s: str) -> int:
79 if len(s) == 1 and s != "\x1a":
80 return str_width(s)
81 length = sum(str_width(i) for i in s)
82 # remove lengths of any escape sequences
83 sequence = ANSI_ESCAPE_SEQUENCE.findall(s)
84 ctrl_z_cnt = s.count("\x1a")
85 return length - sum(len(i) for i in sequence) + ctrl_z_cnt
86
87
88 def unbracket(s: str, including_content: bool = False) -> str:
89 r"""Return `s` with \001 and \002 characters removed.
90
91 If `including_content` is True, content between \001 and \002 is also
92 stripped.
93 """
94 if including_content:
95 return ZERO_WIDTH_BRACKET.sub("", s)
96 return s.translate(ZERO_WIDTH_TRANS)
97
98
99 def gen_colors(buffer: str) -> Iterator[ColorSpan]:
100 """Returns a list of index spans to color using the given color tag.
101
102 The input `buffer` should be a valid start of a Python code block, i.e.
103 it cannot be a block starting in the middle of a multiline string.
104 """
105 sio = StringIO(buffer)
106 line_lengths = [0] + [len(line) for line in sio.readlines()]
107 # make line_lengths cumulative
108 for i in range(1, len(line_lengths)):
109 line_lengths[i] += line_lengths[i-1]
110
111 sio.seek(0)
112 gen = tokenize.generate_tokens(sio.readline)
113 last_emitted: ColorSpan | None = None
114 try:
115 for color in gen_colors_from_token_stream(gen, line_lengths):
116 yield color
117 last_emitted = color
118 except SyntaxError:
119 return
120 except tokenize.TokenError as te:
121 yield from recover_unterminated_string(
122 te, line_lengths, last_emitted, buffer
123 )
124
125
126 def recover_unterminated_string(
127 exc: tokenize.TokenError,
128 line_lengths: list[int],
129 last_emitted: ColorSpan | None,
130 buffer: str,
131 ) -> Iterator[ColorSpan]:
132 msg, loc = exc.args
133 if loc is None:
134 return
135
136 line_no, column = loc
137
138 if msg.startswith(
139 (
140 "unterminated string literal",
141 "unterminated f-string literal",
142 "unterminated t-string literal",
143 "EOF in multi-line string",
144 "unterminated triple-quoted f-string literal",
145 "unterminated triple-quoted t-string literal",
146 )
147 ):
148 start = line_lengths[line_no - 1] + column - 1
149 end = line_lengths[-1] - 1
150
151 # in case FSTRING_START was already emitted
152 if last_emitted and start <= last_emitted.span.start:
153 trace("before last emitted = {s}", s=start)
154 start = last_emitted.span.end + 1
155
156 span = Span(start, end)
157 trace("yielding span {a} -> {b}", a=span.start, b=span.end)
158 yield ColorSpan(span, "string")
159 else:
160 trace(
161 "unhandled token error({buffer}) = {te}",
162 buffer=repr(buffer),
163 te=str(exc),
164 )
165
166
167 def gen_colors_from_token_stream(
168 token_generator: Iterator[TI],
169 line_lengths: list[int],
170 ) -> Iterator[ColorSpan]:
171 token_window = prev_next_window(token_generator)
172
173 is_def_name = False
174 bracket_level = 0
175 for prev_token, token, next_token in token_window:
176 assert token is not None
177 if token.start == token.end:
178 continue
179
180 match token.type:
181 case (
182 T.STRING
183 | T.FSTRING_START | T.FSTRING_MIDDLE | T.FSTRING_END
184 | T.TSTRING_START | T.TSTRING_MIDDLE | T.TSTRING_END
185 ):
186 span = Span.from_token(token, line_lengths)
187 yield ColorSpan(span, "string")
188 case T.COMMENT:
189 span = Span.from_token(token, line_lengths)
190 yield ColorSpan(span, "comment")
191 case T.NUMBER:
192 span = Span.from_token(token, line_lengths)
193 yield ColorSpan(span, "number")
194 case T.OP:
195 if token.string in "([{":
196 bracket_level += 1
197 elif token.string in ")]}":
198 bracket_level -= 1
199 span = Span.from_token(token, line_lengths)
200 yield ColorSpan(span, "op")
201 case T.NAME:
202 if is_def_name:
203 is_def_name = False
204 span = Span.from_token(token, line_lengths)
205 yield ColorSpan(span, "definition")
206 elif keyword.iskeyword(token.string):
207 span_cls = "keyword"
208 if token.string in KEYWORD_CONSTANTS:
209 span_cls = "keyword_constant"
210 span = Span.from_token(token, line_lengths)
211 yield ColorSpan(span, span_cls)
212 if token.string in IDENTIFIERS_AFTER:
213 is_def_name = True
214 elif (
215 keyword.issoftkeyword(token.string)
216 and bracket_level == 0
217 and is_soft_keyword_used(prev_token, token, next_token)
218 ):
219 span = Span.from_token(token, line_lengths)
220 yield ColorSpan(span, "soft_keyword")
221 elif (
222 token.string in BUILTINS
223 and not (prev_token and prev_token.exact_type == T.DOT)
224 ):
225 span = Span.from_token(token, line_lengths)
226 yield ColorSpan(span, "builtin")
227
228
229 keyword_first_sets_match = {"False", "None", "True", "await", "lambda", "not"}
230 keyword_first_sets_case = {"False", "None", "True"}
231
232
233 def is_soft_keyword_used(*tokens: TI | None) -> bool:
234 """Returns True if the current token is a keyword in this context.
235
236 For the `*tokens` to match anything, they have to be a three-tuple of
237 (previous, current, next).
238 """
239 trace("is_soft_keyword_used{t}", t=tokens)
240 match tokens:
241 case (
242 None | TI(T.NEWLINE) | TI(T.INDENT) | TI(string=":"),
243 TI(string="match"),
244 TI(T.NUMBER | T.STRING | T.FSTRING_START | T.TSTRING_START)
245 | TI(T.OP, string="(" | "*" | "[" | "{" | "~" | "...")
246 ):
247 return True
248 case (
249 None | TI(T.NEWLINE) | TI(T.INDENT) | TI(string=":"),
250 TI(string="match"),
251 TI(T.NAME, string=s)
252 ):
253 if keyword.iskeyword(s):
254 return s in keyword_first_sets_match
255 return True
256 case (
257 None | TI(T.NEWLINE) | TI(T.INDENT) | TI(T.DEDENT) | TI(string=":"),
258 TI(string="case"),
259 TI(T.NUMBER | T.STRING | T.FSTRING_START | T.TSTRING_START)
260 | TI(T.OP, string="(" | "*" | "-" | "[" | "{")
261 ):
262 return True
263 case (
264 None | TI(T.NEWLINE) | TI(T.INDENT) | TI(T.DEDENT) | TI(string=":"),
265 TI(string="case"),
266 TI(T.NAME, string=s)
267 ):
268 if keyword.iskeyword(s):
269 return s in keyword_first_sets_case
270 return True
271 case (TI(string="case"), TI(string="_"), TI(string=":")):
272 return True
273 case (
274 None | TI(T.NEWLINE) | TI(T.INDENT) | TI(T.DEDENT) | TI(string=":"),
275 TI(string="type"),
276 TI(T.NAME, string=s)
277 ):
278 return not keyword.iskeyword(s)
279 case _:
280 return False
281
282
283 def disp_str(
284 buffer: str,
285 colors: list[ColorSpan] | None = None,
286 start_index: int = 0,
287 force_color: bool = False,
288 ) -> tuple[CharBuffer, CharWidths]:
289 r"""Decompose the input buffer into a printable variant with applied colors.
290
291 Returns a tuple of two lists:
292 - the first list is the input buffer, character by character, with color
293 escape codes added (while those codes contain multiple ASCII characters,
294 each code is considered atomic *and is attached for the corresponding
295 visible character*);
296 - the second list is the visible width of each character in the input
297 buffer.
298
299 Note on colors:
300 - The `colors` list, if provided, is partially consumed within. We're using
301 a list and not a generator since we need to hold onto the current
302 unfinished span between calls to disp_str in case of multiline strings.
303 - The `colors` list is computed from the start of the input block. `buffer`
304 is only a subset of that input block, a single line within. This is why
305 we need `start_index` to inform us which position is the start of `buffer`
306 actually within user input. This allows us to match color spans correctly.
307
308 Examples:
309 >>> utils.disp_str("a = 9")
310 (['a', ' ', '=', ' ', '9'], [1, 1, 1, 1, 1])
311
312 >>> line = "while 1:"
313 >>> colors = list(utils.gen_colors(line))
314 >>> utils.disp_str(line, colors=colors)
315 (['\x1b[1;34mw', 'h', 'i', 'l', 'e\x1b[0m', ' ', '1', ':'], [1, 1, 1, 1, 1, 1, 1, 1])
316
317 """
318 chars: CharBuffer = []
319 char_widths: CharWidths = []
320
321 if not buffer:
322 return chars, char_widths
323
324 while colors and colors[0].span.end < start_index:
325 # move past irrelevant spans
326 colors.pop(0)
327
328 theme = THEME(force_color=force_color)
329 pre_color = ""
330 post_color = ""
331 if colors and colors[0].span.start < start_index:
332 # looks like we're continuing a previous color (e.g. a multiline str)
333 pre_color = theme[colors[0].tag]
334
335 for i, c in enumerate(buffer, start_index):
336 if colors and colors[0].span.start == i: # new color starts now
337 pre_color = theme[colors[0].tag]
338
339 if c == "\x1a": # CTRL-Z on Windows
340 chars.append(c)
341 char_widths.append(2)
342 elif ord(c) < 128:
343 chars.append(c)
344 char_widths.append(1)
345 elif unicodedata.category(c).startswith("C"):
346 c = r"\u%04x" % ord(c)
347 chars.append(c)
348 char_widths.append(len(c))
349 else:
350 chars.append(c)
351 char_widths.append(str_width(c))
352
353 if colors and colors[0].span.end == i: # current color ends now
354 post_color = theme.reset
355 colors.pop(0)
356
357 chars[-1] = pre_color + chars[-1] + post_color
358 pre_color = ""
359 post_color = ""
360
361 if colors and colors[0].span.start < i and colors[0].span.end > i:
362 # even though the current color should be continued, reset it for now.
363 # the next call to `disp_str()` will revive it.
364 chars[-1] += theme.reset
365
366 return chars, char_widths
367
368
369 def prev_next_window[T](
370 iterable: Iterable[T]
371 ) -> Iterator[tuple[T | None, ...]]:
372 """Generates three-tuples of (previous, current, next) items.
373
374 On the first iteration previous is None. On the last iteration next
375 is None. In case of exception next is None and the exception is re-raised
376 on a subsequent next() call.
377
378 Inspired by `sliding_window` from `itertools` recipes.
379 """
380
381 iterator = iter(iterable)
382 window = deque((None, next(iterator)), maxlen=3)
383 try:
384 for x in iterator:
385 window.append(x)
386 yield tuple(window)
387 except Exception:
388 raise
389 finally:
390 window.append(None)
391 yield tuple(window)