Debian Codegraph @

Lib/test/test_tokenize.py

raw
1 import contextlib
2 import itertools
3 import os
4 import re
5 import string
6 import tempfile
7 import token
8 import tokenize
9 import unittest
10 from io import BytesIO, StringIO
11 from textwrap import dedent
12 from unittest import TestCase, mock
13 from test import support
14 from test.support import os_helper
15 from test.support.script_helper import run_test_script, make_script, run_python_until_end
16 from test.support.numbers import (
17 VALID_UNDERSCORE_LITERALS,
18 INVALID_UNDERSCORE_LITERALS,
19 )
20
21
22 # Converts a source string into a list of textual representation
23 # of the tokens such as:
24 # ` NAME 'if' (1, 0) (1, 2)`
25 # to make writing tests easier.
26 def stringify_tokens_from_source(token_generator, source_string):
27 result = []
28 num_lines = len(source_string.splitlines())
29 missing_trailing_nl = source_string[-1] not in '\r\n'
30
31 for type, token, start, end, line in token_generator:
32 if type == tokenize.ENDMARKER:
33 break
34 # Ignore the new line on the last line if the input lacks one
35 if missing_trailing_nl and type == tokenize.NEWLINE and end[0] == num_lines:
36 continue
37 type = tokenize.tok_name[type]
38 result.append(f" {type:10} {token!r:13} {start} {end}")
39
40 return result
41
42 class TokenizeTest(TestCase):
43 # Tests for the tokenize module.
44
45 # The tests can be really simple. Given a small fragment of source
46 # code, print out a table with tokens. The ENDMARKER, ENCODING and
47 # final NEWLINE are omitted for brevity.
48
49 def check_tokenize(self, s, expected):
50 # Format the tokens in s in a table format.
51 # The ENDMARKER and final NEWLINE are omitted.
52 f = BytesIO(s.encode('utf-8'))
53 result = stringify_tokens_from_source(tokenize.tokenize(f.readline), s)
54 self.assertEqual(result,
55 [" ENCODING 'utf-8' (0, 0) (0, 0)"] +
56 expected.rstrip().splitlines())
57
58 def test_invalid_readline(self):
59 def gen():
60 yield "sdfosdg"
61 yield "sdfosdg"
62 with self.assertRaises(TypeError):
63 list(tokenize.tokenize(gen().__next__))
64
65 def gen():
66 yield b"sdfosdg"
67 yield b"sdfosdg"
68 with self.assertRaises(TypeError):
69 list(tokenize.generate_tokens(gen().__next__))
70
71 def gen():
72 yield "sdfosdg"
73 1/0
74 with self.assertRaises(ZeroDivisionError):
75 list(tokenize.generate_tokens(gen().__next__))
76
77 def test_implicit_newline(self):
78 # Make sure that the tokenizer puts in an implicit NEWLINE
79 # when the input lacks a trailing new line.
80 f = BytesIO("x".encode('utf-8'))
81 tokens = list(tokenize.tokenize(f.readline))
82 self.assertEqual(tokens[-2].type, tokenize.NEWLINE)
83 self.assertEqual(tokens[-1].type, tokenize.ENDMARKER)
84
85 def test_basic(self):
86 self.check_tokenize("1 + 1", """\
87 NUMBER '1' (1, 0) (1, 1)
88 OP '+' (1, 2) (1, 3)
89 NUMBER '1' (1, 4) (1, 5)
90 """)
91 self.check_tokenize("if False:\n"
92 " # NL\n"
93 " \n"
94 " True = False # NEWLINE\n", """\
95 NAME 'if' (1, 0) (1, 2)
96 NAME 'False' (1, 3) (1, 8)
97 OP ':' (1, 8) (1, 9)
98 NEWLINE '\\n' (1, 9) (1, 10)
99 COMMENT '# NL' (2, 4) (2, 8)
100 NL '\\n' (2, 8) (2, 9)
101 NL '\\n' (3, 4) (3, 5)
102 INDENT ' ' (4, 0) (4, 4)
103 NAME 'True' (4, 4) (4, 8)
104 OP '=' (4, 9) (4, 10)
105 NAME 'False' (4, 11) (4, 16)
106 COMMENT '# NEWLINE' (4, 17) (4, 26)
107 NEWLINE '\\n' (4, 26) (4, 27)
108 DEDENT '' (5, 0) (5, 0)
109 """)
110
111 self.check_tokenize("if True:\r\n # NL\r\n foo='bar'\r\n\r\n", """\
112 NAME 'if' (1, 0) (1, 2)
113 NAME 'True' (1, 3) (1, 7)
114 OP ':' (1, 7) (1, 8)
115 NEWLINE '\\r\\n' (1, 8) (1, 10)
116 COMMENT '# NL' (2, 4) (2, 8)
117 NL '\\r\\n' (2, 8) (2, 10)
118 INDENT ' ' (3, 0) (3, 4)
119 NAME 'foo' (3, 4) (3, 7)
120 OP '=' (3, 7) (3, 8)
121 STRING "\'bar\'" (3, 8) (3, 13)
122 NEWLINE '\\r\\n' (3, 13) (3, 15)
123 NL '\\r\\n' (4, 0) (4, 2)
124 DEDENT '' (5, 0) (5, 0)
125 """)
126
127 self.check_tokenize("x = 1 + \\\r\n1\r\n", """\
128 NAME 'x' (1, 0) (1, 1)
129 OP '=' (1, 2) (1, 3)
130 NUMBER '1' (1, 4) (1, 5)
131 OP '+' (1, 6) (1, 7)
132 NUMBER '1' (2, 0) (2, 1)
133 NEWLINE '\\r\\n' (2, 1) (2, 3)
134 """)
135
136 indent_error_file = b"""\
137 def k(x):
138 x += 2
139 x += 5
140 """
141 readline = BytesIO(indent_error_file).readline
142 with self.assertRaisesRegex(IndentationError,
143 "unindent does not match any "
144 "outer indentation level") as e:
145 for tok in tokenize.tokenize(readline):
146 pass
147 self.assertEqual(e.exception.lineno, 3)
148 self.assertEqual(e.exception.filename, '<string>')
149 self.assertEqual(e.exception.end_lineno, None)
150 self.assertEqual(e.exception.end_offset, None)
151 self.assertEqual(
152 e.exception.msg,
153 'unindent does not match any outer indentation level')
154 self.assertEqual(e.exception.offset, 9)
155 self.assertEqual(e.exception.text, ' x += 5')
156
157 def test_int(self):
158 # Ordinary integers and binary operators
159 self.check_tokenize("0xff <= 255", """\
160 NUMBER '0xff' (1, 0) (1, 4)
161 OP '<=' (1, 5) (1, 7)
162 NUMBER '255' (1, 8) (1, 11)
163 """)
164 self.check_tokenize("0b10 <= 255", """\
165 NUMBER '0b10' (1, 0) (1, 4)
166 OP '<=' (1, 5) (1, 7)
167 NUMBER '255' (1, 8) (1, 11)
168 """)
169 self.check_tokenize("0o123 <= 0O123", """\
170 NUMBER '0o123' (1, 0) (1, 5)
171 OP '<=' (1, 6) (1, 8)
172 NUMBER '0O123' (1, 9) (1, 14)
173 """)
174 self.check_tokenize("1234567 > ~0x15", """\
175 NUMBER '1234567' (1, 0) (1, 7)
176 OP '>' (1, 8) (1, 9)
177 OP '~' (1, 10) (1, 11)
178 NUMBER '0x15' (1, 11) (1, 15)
179 """)
180 self.check_tokenize("2134568 != 1231515", """\
181 NUMBER '2134568' (1, 0) (1, 7)
182 OP '!=' (1, 8) (1, 10)
183 NUMBER '1231515' (1, 11) (1, 18)
184 """)
185 self.check_tokenize("(-124561-1) & 200000000", """\
186 OP '(' (1, 0) (1, 1)
187 OP '-' (1, 1) (1, 2)
188 NUMBER '124561' (1, 2) (1, 8)
189 OP '-' (1, 8) (1, 9)
190 NUMBER '1' (1, 9) (1, 10)
191 OP ')' (1, 10) (1, 11)
192 OP '&' (1, 12) (1, 13)
193 NUMBER '200000000' (1, 14) (1, 23)
194 """)
195 self.check_tokenize("0xdeadbeef != -1", """\
196 NUMBER '0xdeadbeef' (1, 0) (1, 10)
197 OP '!=' (1, 11) (1, 13)
198 OP '-' (1, 14) (1, 15)
199 NUMBER '1' (1, 15) (1, 16)
200 """)
201 self.check_tokenize("0xdeadc0de & 12345", """\
202 NUMBER '0xdeadc0de' (1, 0) (1, 10)
203 OP '&' (1, 11) (1, 12)
204 NUMBER '12345' (1, 13) (1, 18)
205 """)
206 self.check_tokenize("0xFF & 0x15 | 1234", """\
207 NUMBER '0xFF' (1, 0) (1, 4)
208 OP '&' (1, 5) (1, 6)
209 NUMBER '0x15' (1, 7) (1, 11)
210 OP '|' (1, 12) (1, 13)
211 NUMBER '1234' (1, 14) (1, 18)
212 """)
213
214 def test_long(self):
215 # Long integers
216 self.check_tokenize("x = 0", """\
217 NAME 'x' (1, 0) (1, 1)
218 OP '=' (1, 2) (1, 3)
219 NUMBER '0' (1, 4) (1, 5)
220 """)
221 self.check_tokenize("x = 0xfffffffffff", """\
222 NAME 'x' (1, 0) (1, 1)
223 OP '=' (1, 2) (1, 3)
224 NUMBER '0xfffffffffff' (1, 4) (1, 17)
225 """)
226 self.check_tokenize("x = 123141242151251616110", """\
227 NAME 'x' (1, 0) (1, 1)
228 OP '=' (1, 2) (1, 3)
229 NUMBER '123141242151251616110' (1, 4) (1, 25)
230 """)
231 self.check_tokenize("x = -15921590215012591", """\
232 NAME 'x' (1, 0) (1, 1)
233 OP '=' (1, 2) (1, 3)
234 OP '-' (1, 4) (1, 5)
235 NUMBER '15921590215012591' (1, 5) (1, 22)
236 """)
237
238 def test_float(self):
239 # Floating-point numbers
240 self.check_tokenize("x = 3.14159", """\
241 NAME 'x' (1, 0) (1, 1)
242 OP '=' (1, 2) (1, 3)
243 NUMBER '3.14159' (1, 4) (1, 11)
244 """)
245 self.check_tokenize("x = 314159.", """\
246 NAME 'x' (1, 0) (1, 1)
247 OP '=' (1, 2) (1, 3)
248 NUMBER '314159.' (1, 4) (1, 11)
249 """)
250 self.check_tokenize("x = .314159", """\
251 NAME 'x' (1, 0) (1, 1)
252 OP '=' (1, 2) (1, 3)
253 NUMBER '.314159' (1, 4) (1, 11)
254 """)
255 self.check_tokenize("x = 3e14159", """\
256 NAME 'x' (1, 0) (1, 1)
257 OP '=' (1, 2) (1, 3)
258 NUMBER '3e14159' (1, 4) (1, 11)
259 """)
260 self.check_tokenize("x = 3E123", """\
261 NAME 'x' (1, 0) (1, 1)
262 OP '=' (1, 2) (1, 3)
263 NUMBER '3E123' (1, 4) (1, 9)
264 """)
265 self.check_tokenize("x+y = 3e-1230", """\
266 NAME 'x' (1, 0) (1, 1)
267 OP '+' (1, 1) (1, 2)
268 NAME 'y' (1, 2) (1, 3)
269 OP '=' (1, 4) (1, 5)
270 NUMBER '3e-1230' (1, 6) (1, 13)
271 """)
272 self.check_tokenize("x = 3.14e159", """\
273 NAME 'x' (1, 0) (1, 1)
274 OP '=' (1, 2) (1, 3)
275 NUMBER '3.14e159' (1, 4) (1, 12)
276 """)
277
278 def test_underscore_literals(self):
279 def number_token(s):
280 f = BytesIO(s.encode('utf-8'))
281 for toktype, token, start, end, line in tokenize.tokenize(f.readline):
282 if toktype == tokenize.NUMBER:
283 return token
284 return 'invalid token'
285 for lit in VALID_UNDERSCORE_LITERALS:
286 if '(' in lit:
287 # this won't work with compound complex inputs
288 continue
289 self.assertEqual(number_token(lit), lit)
290 # Valid cases with extra underscores in the tokenize module
291 # See gh-105549 for context
292 extra_valid_cases = {"0_7", "09_99"}
293 for lit in INVALID_UNDERSCORE_LITERALS:
294 if lit in extra_valid_cases:
295 continue
296 try:
297 number_token(lit)
298 except tokenize.TokenError:
299 continue
300 self.assertNotEqual(number_token(lit), lit)
301
302 def test_string(self):
303 # String literals
304 self.check_tokenize("x = ''; y = \"\"", """\
305 NAME 'x' (1, 0) (1, 1)
306 OP '=' (1, 2) (1, 3)
307 STRING "''" (1, 4) (1, 6)
308 OP ';' (1, 6) (1, 7)
309 NAME 'y' (1, 8) (1, 9)
310 OP '=' (1, 10) (1, 11)
311 STRING '""' (1, 12) (1, 14)
312 """)
313 self.check_tokenize("x = '\"'; y = \"'\"", """\
314 NAME 'x' (1, 0) (1, 1)
315 OP '=' (1, 2) (1, 3)
316 STRING '\\'"\\'' (1, 4) (1, 7)
317 OP ';' (1, 7) (1, 8)
318 NAME 'y' (1, 9) (1, 10)
319 OP '=' (1, 11) (1, 12)
320 STRING '"\\'"' (1, 13) (1, 16)
321 """)
322 self.check_tokenize("x = \"doesn't \"shrink\", does it\"", """\
323 NAME 'x' (1, 0) (1, 1)
324 OP '=' (1, 2) (1, 3)
325 STRING '"doesn\\'t "' (1, 4) (1, 14)
326 NAME 'shrink' (1, 14) (1, 20)
327 STRING '", does it"' (1, 20) (1, 31)
328 """)
329 self.check_tokenize("x = 'abc' + 'ABC'", """\
330 NAME 'x' (1, 0) (1, 1)
331 OP '=' (1, 2) (1, 3)
332 STRING "'abc'" (1, 4) (1, 9)
333 OP '+' (1, 10) (1, 11)
334 STRING "'ABC'" (1, 12) (1, 17)
335 """)
336 self.check_tokenize('y = "ABC" + "ABC"', """\
337 NAME 'y' (1, 0) (1, 1)
338 OP '=' (1, 2) (1, 3)
339 STRING '"ABC"' (1, 4) (1, 9)
340 OP '+' (1, 10) (1, 11)
341 STRING '"ABC"' (1, 12) (1, 17)
342 """)
343 self.check_tokenize("x = r'abc' + r'ABC' + R'ABC' + R'ABC'", """\
344 NAME 'x' (1, 0) (1, 1)
345 OP '=' (1, 2) (1, 3)
346 STRING "r'abc'" (1, 4) (1, 10)
347 OP '+' (1, 11) (1, 12)
348 STRING "r'ABC'" (1, 13) (1, 19)
349 OP '+' (1, 20) (1, 21)
350 STRING "R'ABC'" (1, 22) (1, 28)
351 OP '+' (1, 29) (1, 30)
352 STRING "R'ABC'" (1, 31) (1, 37)
353 """)
354 self.check_tokenize('y = r"abc" + r"ABC" + R"ABC" + R"ABC"', """\
355 NAME 'y' (1, 0) (1, 1)
356 OP '=' (1, 2) (1, 3)
357 STRING 'r"abc"' (1, 4) (1, 10)
358 OP '+' (1, 11) (1, 12)
359 STRING 'r"ABC"' (1, 13) (1, 19)
360 OP '+' (1, 20) (1, 21)
361 STRING 'R"ABC"' (1, 22) (1, 28)
362 OP '+' (1, 29) (1, 30)
363 STRING 'R"ABC"' (1, 31) (1, 37)
364 """)
365
366 self.check_tokenize("u'abc' + U'abc'", """\
367 STRING "u'abc'" (1, 0) (1, 6)
368 OP '+' (1, 7) (1, 8)
369 STRING "U'abc'" (1, 9) (1, 15)
370 """)
371 self.check_tokenize('u"abc" + U"abc"', """\
372 STRING 'u"abc"' (1, 0) (1, 6)
373 OP '+' (1, 7) (1, 8)
374 STRING 'U"abc"' (1, 9) (1, 15)
375 """)
376
377 self.check_tokenize("b'abc' + B'abc'", """\
378 STRING "b'abc'" (1, 0) (1, 6)
379 OP '+' (1, 7) (1, 8)
380 STRING "B'abc'" (1, 9) (1, 15)
381 """)
382 self.check_tokenize('b"abc" + B"abc"', """\
383 STRING 'b"abc"' (1, 0) (1, 6)
384 OP '+' (1, 7) (1, 8)
385 STRING 'B"abc"' (1, 9) (1, 15)
386 """)
387 self.check_tokenize("br'abc' + bR'abc' + Br'abc' + BR'abc'", """\
388 STRING "br'abc'" (1, 0) (1, 7)
389 OP '+' (1, 8) (1, 9)
390 STRING "bR'abc'" (1, 10) (1, 17)
391 OP '+' (1, 18) (1, 19)
392 STRING "Br'abc'" (1, 20) (1, 27)
393 OP '+' (1, 28) (1, 29)
394 STRING "BR'abc'" (1, 30) (1, 37)
395 """)
396 self.check_tokenize('br"abc" + bR"abc" + Br"abc" + BR"abc"', """\
397 STRING 'br"abc"' (1, 0) (1, 7)
398 OP '+' (1, 8) (1, 9)
399 STRING 'bR"abc"' (1, 10) (1, 17)
400 OP '+' (1, 18) (1, 19)
401 STRING 'Br"abc"' (1, 20) (1, 27)
402 OP '+' (1, 28) (1, 29)
403 STRING 'BR"abc"' (1, 30) (1, 37)
404 """)
405 self.check_tokenize("rb'abc' + rB'abc' + Rb'abc' + RB'abc'", """\
406 STRING "rb'abc'" (1, 0) (1, 7)
407 OP '+' (1, 8) (1, 9)
408 STRING "rB'abc'" (1, 10) (1, 17)
409 OP '+' (1, 18) (1, 19)
410 STRING "Rb'abc'" (1, 20) (1, 27)
411 OP '+' (1, 28) (1, 29)
412 STRING "RB'abc'" (1, 30) (1, 37)
413 """)
414 self.check_tokenize('rb"abc" + rB"abc" + Rb"abc" + RB"abc"', """\
415 STRING 'rb"abc"' (1, 0) (1, 7)
416 OP '+' (1, 8) (1, 9)
417 STRING 'rB"abc"' (1, 10) (1, 17)
418 OP '+' (1, 18) (1, 19)
419 STRING 'Rb"abc"' (1, 20) (1, 27)
420 OP '+' (1, 28) (1, 29)
421 STRING 'RB"abc"' (1, 30) (1, 37)
422 """)
423 # Check 0, 1, and 2 character string prefixes.
424 self.check_tokenize(r'"a\
425 de\
426 fg"', """\
427 STRING '"a\\\\\\nde\\\\\\nfg"\' (1, 0) (3, 3)
428 """)
429 self.check_tokenize(r'u"a\
430 de"', """\
431 STRING 'u"a\\\\\\nde"\' (1, 0) (2, 3)
432 """)
433 self.check_tokenize(r'rb"a\
434 d"', """\
435 STRING 'rb"a\\\\\\nd"\' (1, 0) (2, 2)
436 """)
437 self.check_tokenize(r'"""a\
438 b"""', """\
439 STRING '\"\""a\\\\\\nb\"\""' (1, 0) (2, 4)
440 """)
441 self.check_tokenize(r'u"""a\
442 b"""', """\
443 STRING 'u\"\""a\\\\\\nb\"\""' (1, 0) (2, 4)
444 """)
445 self.check_tokenize(r'rb"""a\
446 b\
447 c"""', """\
448 STRING 'rb"\""a\\\\\\nb\\\\\\nc"\""' (1, 0) (3, 4)
449 """)
450 self.check_tokenize('f"abc"', """\
451 FSTRING_START 'f"' (1, 0) (1, 2)
452 FSTRING_MIDDLE 'abc' (1, 2) (1, 5)
453 FSTRING_END '"' (1, 5) (1, 6)
454 """)
455 self.check_tokenize('fR"a{b}c"', """\
456 FSTRING_START 'fR"' (1, 0) (1, 3)
457 FSTRING_MIDDLE 'a' (1, 3) (1, 4)
458 OP '{' (1, 4) (1, 5)
459 NAME 'b' (1, 5) (1, 6)
460 OP '}' (1, 6) (1, 7)
461 FSTRING_MIDDLE 'c' (1, 7) (1, 8)
462 FSTRING_END '"' (1, 8) (1, 9)
463 """)
464 self.check_tokenize('fR"a{{{b!r}}}c"', """\
465 FSTRING_START 'fR"' (1, 0) (1, 3)
466 FSTRING_MIDDLE 'a{' (1, 3) (1, 5)
467 OP '{' (1, 6) (1, 7)
468 NAME 'b' (1, 7) (1, 8)
469 OP '!' (1, 8) (1, 9)
470 NAME 'r' (1, 9) (1, 10)
471 OP '}' (1, 10) (1, 11)
472 FSTRING_MIDDLE '}' (1, 11) (1, 12)
473 FSTRING_MIDDLE 'c' (1, 13) (1, 14)
474 FSTRING_END '"' (1, 14) (1, 15)
475 """)
476 self.check_tokenize('f"{{{1+1}}}"', """\
477 FSTRING_START 'f"' (1, 0) (1, 2)
478 FSTRING_MIDDLE '{' (1, 2) (1, 3)
479 OP '{' (1, 4) (1, 5)
480 NUMBER '1' (1, 5) (1, 6)
481 OP '+' (1, 6) (1, 7)
482 NUMBER '1' (1, 7) (1, 8)
483 OP '}' (1, 8) (1, 9)
484 FSTRING_MIDDLE '}' (1, 9) (1, 10)
485 FSTRING_END '"' (1, 11) (1, 12)
486 """)
487 self.check_tokenize('f"""{f\'\'\'{f\'{f"{1+1}"}\'}\'\'\'}"""', """\
488 FSTRING_START 'f\"""' (1, 0) (1, 4)
489 OP '{' (1, 4) (1, 5)
490 FSTRING_START "f'''" (1, 5) (1, 9)
491 OP '{' (1, 9) (1, 10)
492 FSTRING_START "f'" (1, 10) (1, 12)
493 OP '{' (1, 12) (1, 13)
494 FSTRING_START 'f"' (1, 13) (1, 15)
495 OP '{' (1, 15) (1, 16)
496 NUMBER '1' (1, 16) (1, 17)
497 OP '+' (1, 17) (1, 18)
498 NUMBER '1' (1, 18) (1, 19)
499 OP '}' (1, 19) (1, 20)
500 FSTRING_END '"' (1, 20) (1, 21)
501 OP '}' (1, 21) (1, 22)
502 FSTRING_END "'" (1, 22) (1, 23)
503 OP '}' (1, 23) (1, 24)
504 FSTRING_END "'''" (1, 24) (1, 27)
505 OP '}' (1, 27) (1, 28)
506 FSTRING_END '\"""' (1, 28) (1, 31)
507 """)
508 self.check_tokenize('f""" x\nstr(data, encoding={invalid!r})\n"""', """\
509 FSTRING_START 'f\"""' (1, 0) (1, 4)
510 FSTRING_MIDDLE ' x\\nstr(data, encoding=' (1, 4) (2, 19)
511 OP '{' (2, 19) (2, 20)
512 NAME 'invalid' (2, 20) (2, 27)
513 OP '!' (2, 27) (2, 28)
514 NAME 'r' (2, 28) (2, 29)
515 OP '}' (2, 29) (2, 30)
516 FSTRING_MIDDLE ')\\n' (2, 30) (3, 0)
517 FSTRING_END '\"""' (3, 0) (3, 3)
518 """)
519 self.check_tokenize('f"""123456789\nsomething{None}bad"""', """\
520 FSTRING_START 'f\"""' (1, 0) (1, 4)
521 FSTRING_MIDDLE '123456789\\nsomething' (1, 4) (2, 9)
522 OP '{' (2, 9) (2, 10)
523 NAME 'None' (2, 10) (2, 14)
524 OP '}' (2, 14) (2, 15)
525 FSTRING_MIDDLE 'bad' (2, 15) (2, 18)
526 FSTRING_END '\"""' (2, 18) (2, 21)
527 """)
528 self.check_tokenize('f"""abc"""', """\
529 FSTRING_START 'f\"""' (1, 0) (1, 4)
530 FSTRING_MIDDLE 'abc' (1, 4) (1, 7)
531 FSTRING_END '\"""' (1, 7) (1, 10)
532 """)
533 self.check_tokenize(r'f"abc\
534 def"', """\
535 FSTRING_START 'f"' (1, 0) (1, 2)
536 FSTRING_MIDDLE 'abc\\\\\\ndef' (1, 2) (2, 3)
537 FSTRING_END '"' (2, 3) (2, 4)
538 """)
539 self.check_tokenize(r'Rf"abc\
540 def"', """\
541 FSTRING_START 'Rf"' (1, 0) (1, 3)
542 FSTRING_MIDDLE 'abc\\\\\\ndef' (1, 3) (2, 3)
543 FSTRING_END '"' (2, 3) (2, 4)
544 """)
545 self.check_tokenize("f'some words {a+b:.3f} more words {c+d=} final words'", """\
546 FSTRING_START "f'" (1, 0) (1, 2)
547 FSTRING_MIDDLE 'some words ' (1, 2) (1, 13)
548 OP '{' (1, 13) (1, 14)
549 NAME 'a' (1, 14) (1, 15)
550 OP '+' (1, 15) (1, 16)
551 NAME 'b' (1, 16) (1, 17)
552 OP ':' (1, 17) (1, 18)
553 FSTRING_MIDDLE '.3f' (1, 18) (1, 21)
554 OP '}' (1, 21) (1, 22)
555 FSTRING_MIDDLE ' more words ' (1, 22) (1, 34)
556 OP '{' (1, 34) (1, 35)
557 NAME 'c' (1, 35) (1, 36)
558 OP '+' (1, 36) (1, 37)
559 NAME 'd' (1, 37) (1, 38)
560 OP '=' (1, 38) (1, 39)
561 OP '}' (1, 39) (1, 40)
562 FSTRING_MIDDLE ' final words' (1, 40) (1, 52)
563 FSTRING_END "'" (1, 52) (1, 53)
564 """)
565 self.check_tokenize("""\
566 f'''{
567 3
568 =}'''""", """\
569 FSTRING_START "f'''" (1, 0) (1, 4)
570 OP '{' (1, 4) (1, 5)
571 NL '\\n' (1, 5) (1, 6)
572 NUMBER '3' (2, 0) (2, 1)
573 NL '\\n' (2, 1) (2, 2)
574 OP '=' (3, 0) (3, 1)
575 OP '}' (3, 1) (3, 2)
576 FSTRING_END "'''" (3, 2) (3, 5)
577 """)
578 self.check_tokenize("""\
579 f'''__{
580 x:a
581 }__'''""", """\
582 FSTRING_START "f'''" (1, 0) (1, 4)
583 FSTRING_MIDDLE '__' (1, 4) (1, 6)
584 OP '{' (1, 6) (1, 7)
585 NL '\\n' (1, 7) (1, 8)
586 NAME 'x' (2, 4) (2, 5)
587 OP ':' (2, 5) (2, 6)
588 FSTRING_MIDDLE 'a\\n' (2, 6) (3, 0)
589 OP '}' (3, 0) (3, 1)
590 FSTRING_MIDDLE '__' (3, 1) (3, 3)
591 FSTRING_END "'''" (3, 3) (3, 6)
592 """)
593 self.check_tokenize("""\
594 f'''__{
595 x:a
596 b
597 c
598 d
599 }__'''""", """\
600 FSTRING_START "f'''" (1, 0) (1, 4)
601 FSTRING_MIDDLE '__' (1, 4) (1, 6)
602 OP '{' (1, 6) (1, 7)
603 NL '\\n' (1, 7) (1, 8)
604 NAME 'x' (2, 4) (2, 5)
605 OP ':' (2, 5) (2, 6)
606 FSTRING_MIDDLE 'a\\n b\\n c\\n d\\n' (2, 6) (6, 0)
607 OP '}' (6, 0) (6, 1)
608 FSTRING_MIDDLE '__' (6, 1) (6, 3)
609 FSTRING_END "'''" (6, 3) (6, 6)
610 """)
611
612 self.check_tokenize("""\
613 '''Autorzy, którzy tą jednostkę mają wpisani jako AKTUALNA -- czyli
614 aktualni pracownicy, obecni pracownicy'''
615 """, """\
616 INDENT ' ' (1, 0) (1, 4)
617 STRING "'''Autorzy, którzy tą jednostkę mają wpisani jako AKTUALNA -- czyli\\n aktualni pracownicy, obecni pracownicy'''" (1, 4) (2, 45)
618 NEWLINE '\\n' (2, 45) (2, 46)
619 DEDENT '' (3, 0) (3, 0)
620 """)
621
622 def test_function(self):
623 self.check_tokenize("def d22(a, b, c=2, d=2, *k): pass", """\
624 NAME 'def' (1, 0) (1, 3)
625 NAME 'd22' (1, 4) (1, 7)
626 OP '(' (1, 7) (1, 8)
627 NAME 'a' (1, 8) (1, 9)
628 OP ',' (1, 9) (1, 10)
629 NAME 'b' (1, 11) (1, 12)
630 OP ',' (1, 12) (1, 13)
631 NAME 'c' (1, 14) (1, 15)
632 OP '=' (1, 15) (1, 16)
633 NUMBER '2' (1, 16) (1, 17)
634 OP ',' (1, 17) (1, 18)
635 NAME 'd' (1, 19) (1, 20)
636 OP '=' (1, 20) (1, 21)
637 NUMBER '2' (1, 21) (1, 22)
638 OP ',' (1, 22) (1, 23)
639 OP '*' (1, 24) (1, 25)
640 NAME 'k' (1, 25) (1, 26)
641 OP ')' (1, 26) (1, 27)
642 OP ':' (1, 27) (1, 28)
643 NAME 'pass' (1, 29) (1, 33)
644 """)
645 self.check_tokenize("def d01v_(a=1, *k, **w): pass", """\
646 NAME 'def' (1, 0) (1, 3)
647 NAME 'd01v_' (1, 4) (1, 9)
648 OP '(' (1, 9) (1, 10)
649 NAME 'a' (1, 10) (1, 11)
650 OP '=' (1, 11) (1, 12)
651 NUMBER '1' (1, 12) (1, 13)
652 OP ',' (1, 13) (1, 14)
653 OP '*' (1, 15) (1, 16)
654 NAME 'k' (1, 16) (1, 17)
655 OP ',' (1, 17) (1, 18)
656 OP '**' (1, 19) (1, 21)
657 NAME 'w' (1, 21) (1, 22)
658 OP ')' (1, 22) (1, 23)
659 OP ':' (1, 23) (1, 24)
660 NAME 'pass' (1, 25) (1, 29)
661 """)
662 self.check_tokenize("def d23(a: str, b: int=3) -> int: pass", """\
663 NAME 'def' (1, 0) (1, 3)
664 NAME 'd23' (1, 4) (1, 7)
665 OP '(' (1, 7) (1, 8)
666 NAME 'a' (1, 8) (1, 9)
667 OP ':' (1, 9) (1, 10)
668 NAME 'str' (1, 11) (1, 14)
669 OP ',' (1, 14) (1, 15)
670 NAME 'b' (1, 16) (1, 17)
671 OP ':' (1, 17) (1, 18)
672 NAME 'int' (1, 19) (1, 22)
673 OP '=' (1, 22) (1, 23)
674 NUMBER '3' (1, 23) (1, 24)
675 OP ')' (1, 24) (1, 25)
676 OP '->' (1, 26) (1, 28)
677 NAME 'int' (1, 29) (1, 32)
678 OP ':' (1, 32) (1, 33)
679 NAME 'pass' (1, 34) (1, 38)
680 """)
681
682 def test_comparison(self):
683 # Comparison
684 self.check_tokenize("if 1 < 1 > 1 == 1 >= 5 <= 0x15 <= 0x12 != "
685 "1 and 5 in 1 not in 1 is 1 or 5 is not 1: pass", """\
686 NAME 'if' (1, 0) (1, 2)
687 NUMBER '1' (1, 3) (1, 4)
688 OP '<' (1, 5) (1, 6)
689 NUMBER '1' (1, 7) (1, 8)
690 OP '>' (1, 9) (1, 10)
691 NUMBER '1' (1, 11) (1, 12)
692 OP '==' (1, 13) (1, 15)
693 NUMBER '1' (1, 16) (1, 17)
694 OP '>=' (1, 18) (1, 20)
695 NUMBER '5' (1, 21) (1, 22)
696 OP '<=' (1, 23) (1, 25)
697 NUMBER '0x15' (1, 26) (1, 30)
698 OP '<=' (1, 31) (1, 33)
699 NUMBER '0x12' (1, 34) (1, 38)
700 OP '!=' (1, 39) (1, 41)
701 NUMBER '1' (1, 42) (1, 43)
702 NAME 'and' (1, 44) (1, 47)
703 NUMBER '5' (1, 48) (1, 49)
704 NAME 'in' (1, 50) (1, 52)
705 NUMBER '1' (1, 53) (1, 54)
706 NAME 'not' (1, 55) (1, 58)
707 NAME 'in' (1, 59) (1, 61)
708 NUMBER '1' (1, 62) (1, 63)
709 NAME 'is' (1, 64) (1, 66)
710 NUMBER '1' (1, 67) (1, 68)
711 NAME 'or' (1, 69) (1, 71)
712 NUMBER '5' (1, 72) (1, 73)
713 NAME 'is' (1, 74) (1, 76)
714 NAME 'not' (1, 77) (1, 80)
715 NUMBER '1' (1, 81) (1, 82)
716 OP ':' (1, 82) (1, 83)
717 NAME 'pass' (1, 84) (1, 88)
718 """)
719
720 def test_shift(self):
721 # Shift
722 self.check_tokenize("x = 1 << 1 >> 5", """\
723 NAME 'x' (1, 0) (1, 1)
724 OP '=' (1, 2) (1, 3)
725 NUMBER '1' (1, 4) (1, 5)
726 OP '<<' (1, 6) (1, 8)
727 NUMBER '1' (1, 9) (1, 10)
728 OP '>>' (1, 11) (1, 13)
729 NUMBER '5' (1, 14) (1, 15)
730 """)
731
732 def test_additive(self):
733 # Additive
734 self.check_tokenize("x = 1 - y + 15 - 1 + 0x124 + z + a[5]", """\
735 NAME 'x' (1, 0) (1, 1)
736 OP '=' (1, 2) (1, 3)
737 NUMBER '1' (1, 4) (1, 5)
738 OP '-' (1, 6) (1, 7)
739 NAME 'y' (1, 8) (1, 9)
740 OP '+' (1, 10) (1, 11)
741 NUMBER '15' (1, 12) (1, 14)
742 OP '-' (1, 15) (1, 16)
743 NUMBER '1' (1, 17) (1, 18)
744 OP '+' (1, 19) (1, 20)
745 NUMBER '0x124' (1, 21) (1, 26)
746 OP '+' (1, 27) (1, 28)
747 NAME 'z' (1, 29) (1, 30)
748 OP '+' (1, 31) (1, 32)
749 NAME 'a' (1, 33) (1, 34)
750 OP '[' (1, 34) (1, 35)
751 NUMBER '5' (1, 35) (1, 36)
752 OP ']' (1, 36) (1, 37)
753 """)
754
755 def test_multiplicative(self):
756 # Multiplicative
757 self.check_tokenize("x = 1//1*1/5*12%0x12@42", """\
758 NAME 'x' (1, 0) (1, 1)
759 OP '=' (1, 2) (1, 3)
760 NUMBER '1' (1, 4) (1, 5)
761 OP '//' (1, 5) (1, 7)
762 NUMBER '1' (1, 7) (1, 8)
763 OP '*' (1, 8) (1, 9)
764 NUMBER '1' (1, 9) (1, 10)
765 OP '/' (1, 10) (1, 11)
766 NUMBER '5' (1, 11) (1, 12)
767 OP '*' (1, 12) (1, 13)
768 NUMBER '12' (1, 13) (1, 15)
769 OP '%' (1, 15) (1, 16)
770 NUMBER '0x12' (1, 16) (1, 20)
771 OP '@' (1, 20) (1, 21)
772 NUMBER '42' (1, 21) (1, 23)
773 """)
774
775 def test_unary(self):
776 # Unary
777 self.check_tokenize("~1 ^ 1 & 1 |1 ^ -1", """\
778 OP '~' (1, 0) (1, 1)
779 NUMBER '1' (1, 1) (1, 2)
780 OP '^' (1, 3) (1, 4)
781 NUMBER '1' (1, 5) (1, 6)
782 OP '&' (1, 7) (1, 8)
783 NUMBER '1' (1, 9) (1, 10)
784 OP '|' (1, 11) (1, 12)
785 NUMBER '1' (1, 12) (1, 13)
786 OP '^' (1, 14) (1, 15)
787 OP '-' (1, 16) (1, 17)
788 NUMBER '1' (1, 17) (1, 18)
789 """)
790 self.check_tokenize("-1*1/1+1*1//1 - ---1**1", """\
791 OP '-' (1, 0) (1, 1)
792 NUMBER '1' (1, 1) (1, 2)
793 OP '*' (1, 2) (1, 3)
794 NUMBER '1' (1, 3) (1, 4)
795 OP '/' (1, 4) (1, 5)
796 NUMBER '1' (1, 5) (1, 6)
797 OP '+' (1, 6) (1, 7)
798 NUMBER '1' (1, 7) (1, 8)
799 OP '*' (1, 8) (1, 9)
800 NUMBER '1' (1, 9) (1, 10)
801 OP '//' (1, 10) (1, 12)
802 NUMBER '1' (1, 12) (1, 13)
803 OP '-' (1, 14) (1, 15)
804 OP '-' (1, 16) (1, 17)
805 OP '-' (1, 17) (1, 18)
806 OP '-' (1, 18) (1, 19)
807 NUMBER '1' (1, 19) (1, 20)
808 OP '**' (1, 20) (1, 22)
809 NUMBER '1' (1, 22) (1, 23)
810 """)
811
812 def test_selector(self):
813 # Selector
814 self.check_tokenize("import sys, time\nx = sys.modules['time'].time()", """\
815 NAME 'import' (1, 0) (1, 6)
816 NAME 'sys' (1, 7) (1, 10)
817 OP ',' (1, 10) (1, 11)
818 NAME 'time' (1, 12) (1, 16)
819 NEWLINE '\\n' (1, 16) (1, 17)
820 NAME 'x' (2, 0) (2, 1)
821 OP '=' (2, 2) (2, 3)
822 NAME 'sys' (2, 4) (2, 7)
823 OP '.' (2, 7) (2, 8)
824 NAME 'modules' (2, 8) (2, 15)
825 OP '[' (2, 15) (2, 16)
826 STRING "'time'" (2, 16) (2, 22)
827 OP ']' (2, 22) (2, 23)
828 OP '.' (2, 23) (2, 24)
829 NAME 'time' (2, 24) (2, 28)
830 OP '(' (2, 28) (2, 29)
831 OP ')' (2, 29) (2, 30)
832 """)
833
834 def test_method(self):
835 # Methods
836 self.check_tokenize("@staticmethod\ndef foo(x,y): pass", """\
837 OP '@' (1, 0) (1, 1)
838 NAME 'staticmethod' (1, 1) (1, 13)
839 NEWLINE '\\n' (1, 13) (1, 14)
840 NAME 'def' (2, 0) (2, 3)
841 NAME 'foo' (2, 4) (2, 7)
842 OP '(' (2, 7) (2, 8)
843 NAME 'x' (2, 8) (2, 9)
844 OP ',' (2, 9) (2, 10)
845 NAME 'y' (2, 10) (2, 11)
846 OP ')' (2, 11) (2, 12)
847 OP ':' (2, 12) (2, 13)
848 NAME 'pass' (2, 14) (2, 18)
849 """)
850
851 def test_tabs(self):
852 # Evil tabs
853 self.check_tokenize("def f():\n"
854 "\tif x\n"
855 " \tpass", """\
856 NAME 'def' (1, 0) (1, 3)
857 NAME 'f' (1, 4) (1, 5)
858 OP '(' (1, 5) (1, 6)
859 OP ')' (1, 6) (1, 7)
860 OP ':' (1, 7) (1, 8)
861 NEWLINE '\\n' (1, 8) (1, 9)
862 INDENT '\\t' (2, 0) (2, 1)
863 NAME 'if' (2, 1) (2, 3)
864 NAME 'x' (2, 4) (2, 5)
865 NEWLINE '\\n' (2, 5) (2, 6)
866 INDENT ' \\t' (3, 0) (3, 9)
867 NAME 'pass' (3, 9) (3, 13)
868 DEDENT '' (4, 0) (4, 0)
869 DEDENT '' (4, 0) (4, 0)
870 """)
871
872 def test_non_ascii_identifiers(self):
873 # Non-ascii identifiers
874 self.check_tokenize("Örter = 'places'\ngrün = 'green'", """\
875 NAME 'Örter' (1, 0) (1, 5)
876 OP '=' (1, 6) (1, 7)
877 STRING "'places'" (1, 8) (1, 16)
878 NEWLINE '\\n' (1, 16) (1, 17)
879 NAME 'grün' (2, 0) (2, 4)
880 OP '=' (2, 5) (2, 6)
881 STRING "'green'" (2, 7) (2, 14)
882 """)
883
884 def test_unicode(self):
885 # Legacy unicode literals:
886 self.check_tokenize("Örter = u'places'\ngrün = U'green'", """\
887 NAME 'Örter' (1, 0) (1, 5)
888 OP '=' (1, 6) (1, 7)
889 STRING "u'places'" (1, 8) (1, 17)
890 NEWLINE '\\n' (1, 17) (1, 18)
891 NAME 'grün' (2, 0) (2, 4)
892 OP '=' (2, 5) (2, 6)
893 STRING "U'green'" (2, 7) (2, 15)
894 """)
895
896 def test_async(self):
897 # Async/await extension:
898 self.check_tokenize("async = 1", """\
899 NAME 'async' (1, 0) (1, 5)
900 OP '=' (1, 6) (1, 7)
901 NUMBER '1' (1, 8) (1, 9)
902 """)
903
904 self.check_tokenize("a = (async = 1)", """\
905 NAME 'a' (1, 0) (1, 1)
906 OP '=' (1, 2) (1, 3)
907 OP '(' (1, 4) (1, 5)
908 NAME 'async' (1, 5) (1, 10)
909 OP '=' (1, 11) (1, 12)
910 NUMBER '1' (1, 13) (1, 14)
911 OP ')' (1, 14) (1, 15)
912 """)
913
914 self.check_tokenize("async()", """\
915 NAME 'async' (1, 0) (1, 5)
916 OP '(' (1, 5) (1, 6)
917 OP ')' (1, 6) (1, 7)
918 """)
919
920 self.check_tokenize("class async(Bar):pass", """\
921 NAME 'class' (1, 0) (1, 5)
922 NAME 'async' (1, 6) (1, 11)
923 OP '(' (1, 11) (1, 12)
924 NAME 'Bar' (1, 12) (1, 15)
925 OP ')' (1, 15) (1, 16)
926 OP ':' (1, 16) (1, 17)
927 NAME 'pass' (1, 17) (1, 21)
928 """)
929
930 self.check_tokenize("class async:pass", """\
931 NAME 'class' (1, 0) (1, 5)
932 NAME 'async' (1, 6) (1, 11)
933 OP ':' (1, 11) (1, 12)
934 NAME 'pass' (1, 12) (1, 16)
935 """)
936
937 self.check_tokenize("await = 1", """\
938 NAME 'await' (1, 0) (1, 5)
939 OP '=' (1, 6) (1, 7)
940 NUMBER '1' (1, 8) (1, 9)
941 """)
942
943 self.check_tokenize("foo.async", """\
944 NAME 'foo' (1, 0) (1, 3)
945 OP '.' (1, 3) (1, 4)
946 NAME 'async' (1, 4) (1, 9)
947 """)
948
949 self.check_tokenize("async for a in b: pass", """\
950 NAME 'async' (1, 0) (1, 5)
951 NAME 'for' (1, 6) (1, 9)
952 NAME 'a' (1, 10) (1, 11)
953 NAME 'in' (1, 12) (1, 14)
954 NAME 'b' (1, 15) (1, 16)
955 OP ':' (1, 16) (1, 17)
956 NAME 'pass' (1, 18) (1, 22)
957 """)
958
959 self.check_tokenize("async with a as b: pass", """\
960 NAME 'async' (1, 0) (1, 5)
961 NAME 'with' (1, 6) (1, 10)
962 NAME 'a' (1, 11) (1, 12)
963 NAME 'as' (1, 13) (1, 15)
964 NAME 'b' (1, 16) (1, 17)
965 OP ':' (1, 17) (1, 18)
966 NAME 'pass' (1, 19) (1, 23)
967 """)
968
969 self.check_tokenize("async.foo", """\
970 NAME 'async' (1, 0) (1, 5)
971 OP '.' (1, 5) (1, 6)
972 NAME 'foo' (1, 6) (1, 9)
973 """)
974
975 self.check_tokenize("async", """\
976 NAME 'async' (1, 0) (1, 5)
977 """)
978
979 self.check_tokenize("async\n#comment\nawait", """\
980 NAME 'async' (1, 0) (1, 5)
981 NEWLINE '\\n' (1, 5) (1, 6)
982 COMMENT '#comment' (2, 0) (2, 8)
983 NL '\\n' (2, 8) (2, 9)
984 NAME 'await' (3, 0) (3, 5)
985 """)
986
987 self.check_tokenize("async\n...\nawait", """\
988 NAME 'async' (1, 0) (1, 5)
989 NEWLINE '\\n' (1, 5) (1, 6)
990 OP '...' (2, 0) (2, 3)
991 NEWLINE '\\n' (2, 3) (2, 4)
992 NAME 'await' (3, 0) (3, 5)
993 """)
994
995 self.check_tokenize("async\nawait", """\
996 NAME 'async' (1, 0) (1, 5)
997 NEWLINE '\\n' (1, 5) (1, 6)
998 NAME 'await' (2, 0) (2, 5)
999 """)
1000
1001 self.check_tokenize("foo.async + 1", """\
1002 NAME 'foo' (1, 0) (1, 3)
1003 OP '.' (1, 3) (1, 4)
1004 NAME 'async' (1, 4) (1, 9)
1005 OP '+' (1, 10) (1, 11)
1006 NUMBER '1' (1, 12) (1, 13)
1007 """)
1008
1009 self.check_tokenize("async def foo(): pass", """\
1010 NAME 'async' (1, 0) (1, 5)
1011 NAME 'def' (1, 6) (1, 9)
1012 NAME 'foo' (1, 10) (1, 13)
1013 OP '(' (1, 13) (1, 14)
1014 OP ')' (1, 14) (1, 15)
1015 OP ':' (1, 15) (1, 16)
1016 NAME 'pass' (1, 17) (1, 21)
1017 """)
1018
1019 self.check_tokenize('''\
1020 async def foo():
1021 def foo(await):
1022 await = 1
1023 if 1:
1024 await
1025 async += 1
1026 ''', """\
1027 NAME 'async' (1, 0) (1, 5)
1028 NAME 'def' (1, 6) (1, 9)
1029 NAME 'foo' (1, 10) (1, 13)
1030 OP '(' (1, 13) (1, 14)
1031 OP ')' (1, 14) (1, 15)
1032 OP ':' (1, 15) (1, 16)
1033 NEWLINE '\\n' (1, 16) (1, 17)
1034 INDENT ' ' (2, 0) (2, 2)
1035 NAME 'def' (2, 2) (2, 5)
1036 NAME 'foo' (2, 6) (2, 9)
1037 OP '(' (2, 9) (2, 10)
1038 NAME 'await' (2, 10) (2, 15)
1039 OP ')' (2, 15) (2, 16)
1040 OP ':' (2, 16) (2, 17)
1041 NEWLINE '\\n' (2, 17) (2, 18)
1042 INDENT ' ' (3, 0) (3, 4)
1043 NAME 'await' (3, 4) (3, 9)
1044 OP '=' (3, 10) (3, 11)
1045 NUMBER '1' (3, 12) (3, 13)
1046 NEWLINE '\\n' (3, 13) (3, 14)
1047 DEDENT '' (4, 2) (4, 2)
1048 NAME 'if' (4, 2) (4, 4)
1049 NUMBER '1' (4, 5) (4, 6)
1050 OP ':' (4, 6) (4, 7)
1051 NEWLINE '\\n' (4, 7) (4, 8)
1052 INDENT ' ' (5, 0) (5, 4)
1053 NAME 'await' (5, 4) (5, 9)
1054 NEWLINE '\\n' (5, 9) (5, 10)
1055 DEDENT '' (6, 0) (6, 0)
1056 DEDENT '' (6, 0) (6, 0)
1057 NAME 'async' (6, 0) (6, 5)
1058 OP '+=' (6, 6) (6, 8)
1059 NUMBER '1' (6, 9) (6, 10)
1060 NEWLINE '\\n' (6, 10) (6, 11)
1061 """)
1062
1063 self.check_tokenize('''\
1064 async def foo():
1065 async for i in 1: pass''', """\
1066 NAME 'async' (1, 0) (1, 5)
1067 NAME 'def' (1, 6) (1, 9)
1068 NAME 'foo' (1, 10) (1, 13)
1069 OP '(' (1, 13) (1, 14)
1070 OP ')' (1, 14) (1, 15)
1071 OP ':' (1, 15) (1, 16)
1072 NEWLINE '\\n' (1, 16) (1, 17)
1073 INDENT ' ' (2, 0) (2, 2)
1074 NAME 'async' (2, 2) (2, 7)
1075 NAME 'for' (2, 8) (2, 11)
1076 NAME 'i' (2, 12) (2, 13)
1077 NAME 'in' (2, 14) (2, 16)
1078 NUMBER '1' (2, 17) (2, 18)
1079 OP ':' (2, 18) (2, 19)
1080 NAME 'pass' (2, 20) (2, 24)
1081 DEDENT '' (3, 0) (3, 0)
1082 """)
1083
1084 self.check_tokenize('''async def foo(async): await''', """\
1085 NAME 'async' (1, 0) (1, 5)
1086 NAME 'def' (1, 6) (1, 9)
1087 NAME 'foo' (1, 10) (1, 13)
1088 OP '(' (1, 13) (1, 14)
1089 NAME 'async' (1, 14) (1, 19)
1090 OP ')' (1, 19) (1, 20)
1091 OP ':' (1, 20) (1, 21)
1092 NAME 'await' (1, 22) (1, 27)
1093 """)
1094
1095 self.check_tokenize('''\
1096 def f():
1097
1098 def baz(): pass
1099 async def bar(): pass
1100
1101 await = 2''', """\
1102 NAME 'def' (1, 0) (1, 3)
1103 NAME 'f' (1, 4) (1, 5)
1104 OP '(' (1, 5) (1, 6)
1105 OP ')' (1, 6) (1, 7)
1106 OP ':' (1, 7) (1, 8)
1107 NEWLINE '\\n' (1, 8) (1, 9)
1108 NL '\\n' (2, 0) (2, 1)
1109 INDENT ' ' (3, 0) (3, 2)
1110 NAME 'def' (3, 2) (3, 5)
1111 NAME 'baz' (3, 6) (3, 9)
1112 OP '(' (3, 9) (3, 10)
1113 OP ')' (3, 10) (3, 11)
1114 OP ':' (3, 11) (3, 12)
1115 NAME 'pass' (3, 13) (3, 17)
1116 NEWLINE '\\n' (3, 17) (3, 18)
1117 NAME 'async' (4, 2) (4, 7)
1118 NAME 'def' (4, 8) (4, 11)
1119 NAME 'bar' (4, 12) (4, 15)
1120 OP '(' (4, 15) (4, 16)
1121 OP ')' (4, 16) (4, 17)
1122 OP ':' (4, 17) (4, 18)
1123 NAME 'pass' (4, 19) (4, 23)
1124 NEWLINE '\\n' (4, 23) (4, 24)
1125 NL '\\n' (5, 0) (5, 1)
1126 NAME 'await' (6, 2) (6, 7)
1127 OP '=' (6, 8) (6, 9)
1128 NUMBER '2' (6, 10) (6, 11)
1129 DEDENT '' (7, 0) (7, 0)
1130 """)
1131
1132 self.check_tokenize('''\
1133 async def f():
1134
1135 def baz(): pass
1136 async def bar(): pass
1137
1138 await = 2''', """\
1139 NAME 'async' (1, 0) (1, 5)
1140 NAME 'def' (1, 6) (1, 9)
1141 NAME 'f' (1, 10) (1, 11)
1142 OP '(' (1, 11) (1, 12)
1143 OP ')' (1, 12) (1, 13)
1144 OP ':' (1, 13) (1, 14)
1145 NEWLINE '\\n' (1, 14) (1, 15)
1146 NL '\\n' (2, 0) (2, 1)
1147 INDENT ' ' (3, 0) (3, 2)
1148 NAME 'def' (3, 2) (3, 5)
1149 NAME 'baz' (3, 6) (3, 9)
1150 OP '(' (3, 9) (3, 10)
1151 OP ')' (3, 10) (3, 11)
1152 OP ':' (3, 11) (3, 12)
1153 NAME 'pass' (3, 13) (3, 17)
1154 NEWLINE '\\n' (3, 17) (3, 18)
1155 NAME 'async' (4, 2) (4, 7)
1156 NAME 'def' (4, 8) (4, 11)
1157 NAME 'bar' (4, 12) (4, 15)
1158 OP '(' (4, 15) (4, 16)
1159 OP ')' (4, 16) (4, 17)
1160 OP ':' (4, 17) (4, 18)
1161 NAME 'pass' (4, 19) (4, 23)
1162 NEWLINE '\\n' (4, 23) (4, 24)
1163 NL '\\n' (5, 0) (5, 1)
1164 NAME 'await' (6, 2) (6, 7)
1165 OP '=' (6, 8) (6, 9)
1166 NUMBER '2' (6, 10) (6, 11)
1167 DEDENT '' (7, 0) (7, 0)
1168 """)
1169
1170 def test_newline_after_parenthesized_block_with_comment(self):
1171 self.check_tokenize('''\
1172 [
1173 # A comment here
1174 1
1175 ]
1176 ''', """\
1177 OP '[' (1, 0) (1, 1)
1178 NL '\\n' (1, 1) (1, 2)
1179 COMMENT '# A comment here' (2, 4) (2, 20)
1180 NL '\\n' (2, 20) (2, 21)
1181 NUMBER '1' (3, 4) (3, 5)
1182 NL '\\n' (3, 5) (3, 6)
1183 OP ']' (4, 0) (4, 1)
1184 NEWLINE '\\n' (4, 1) (4, 2)
1185 """)
1186
1187 def test_closing_parenthesis_from_different_line(self):
1188 self.check_tokenize("); x", """\
1189 OP ')' (1, 0) (1, 1)
1190 OP ';' (1, 1) (1, 2)
1191 NAME 'x' (1, 3) (1, 4)
1192 """)
1193
1194 def test_multiline_non_ascii_fstring(self):
1195 self.check_tokenize("""\
1196 a = f'''
1197 Autorzy, którzy tą jednostkę mają wpisani jako AKTUALNA -- czyli'''""", """\
1198 NAME 'a' (1, 0) (1, 1)
1199 OP '=' (1, 2) (1, 3)
1200 FSTRING_START "f\'\'\'" (1, 4) (1, 8)
1201 FSTRING_MIDDLE '\\n Autorzy, którzy tą jednostkę mają wpisani jako AKTUALNA -- czyli' (1, 8) (2, 68)
1202 FSTRING_END "\'\'\'" (2, 68) (2, 71)
1203 """)
1204
1205 def test_multiline_non_ascii_fstring_with_expr(self):
1206 self.check_tokenize("""\
1207 f'''
1208 🔗 This is a test {test_arg1}🔗
1209 🔗'''""", """\
1210 FSTRING_START "f\'\'\'" (1, 0) (1, 4)
1211 FSTRING_MIDDLE '\\n 🔗 This is a test ' (1, 4) (2, 21)
1212 OP '{' (2, 21) (2, 22)
1213 NAME 'test_arg1' (2, 22) (2, 31)
1214 OP '}' (2, 31) (2, 32)
1215 FSTRING_MIDDLE '🔗\\n🔗' (2, 32) (3, 1)
1216 FSTRING_END "\'\'\'" (3, 1) (3, 4)
1217 """)
1218
1219 # gh-139516, the '\n' is explicit to ensure no trailing whitespace which would invalidate the test
1220 self.check_tokenize('''f"{f(a=lambda: 'à'\n)}"''', """\
1221 FSTRING_START \'f"\' (1, 0) (1, 2)
1222 OP '{' (1, 2) (1, 3)
1223 NAME 'f' (1, 3) (1, 4)
1224 OP '(' (1, 4) (1, 5)
1225 NAME 'a' (1, 5) (1, 6)
1226 OP '=' (1, 6) (1, 7)
1227 NAME 'lambda' (1, 7) (1, 13)
1228 OP ':' (1, 13) (1, 14)
1229 STRING "\'à\'" (1, 15) (1, 18)
1230 NL '\\n' (1, 18) (1, 19)
1231 OP ')' (2, 0) (2, 1)
1232 OP '}' (2, 1) (2, 2)
1233 FSTRING_END \'"\' (2, 2) (2, 3)
1234 """)
1235
1236 class GenerateTokensTest(TokenizeTest):
1237 def check_tokenize(self, s, expected):
1238 # Format the tokens in s in a table format.
1239 # The ENDMARKER and final NEWLINE are omitted.
1240 f = StringIO(s)
1241 result = stringify_tokens_from_source(tokenize.generate_tokens(f.readline), s)
1242 self.assertEqual(result, expected.rstrip().splitlines())
1243
1244
1245 def decistmt(s):
1246 result = []
1247 g = tokenize.tokenize(BytesIO(s.encode('utf-8')).readline) # tokenize the string
1248 for toknum, tokval, _, _, _ in g:
1249 if toknum == tokenize.NUMBER and '.' in tokval: # replace NUMBER tokens
1250 result.extend([
1251 (tokenize.NAME, 'Decimal'),
1252 (tokenize.OP, '('),
1253 (tokenize.STRING, repr(tokval)),
1254 (tokenize.OP, ')')
1255 ])
1256 else:
1257 result.append((toknum, tokval))
1258 return tokenize.untokenize(result).decode('utf-8').strip()
1259
1260 class TestMisc(TestCase):
1261
1262 def test_decistmt(self):
1263 # Substitute Decimals for floats in a string of statements.
1264 # This is an example from the docs.
1265
1266 from decimal import Decimal
1267 s = '+21.3e-5*-.1234/81.7'
1268 self.assertEqual(decistmt(s),
1269 "+Decimal ('21.3e-5')*-Decimal ('.1234')/Decimal ('81.7')")
1270
1271 # The format of the exponent is inherited from the platform C library.
1272 # Known cases are "e-007" (Windows) and "e-07" (not Windows). Since
1273 # we're only showing 11 digits, and the 12th isn't close to 5, the
1274 # rest of the output should be platform-independent.
1275 self.assertRegex(repr(eval(s)), '-3.2171603427[0-9]*e-0+7')
1276
1277 # Output from calculations with Decimal should be identical across all
1278 # platforms.
1279 self.assertEqual(eval(decistmt(s)),
1280 Decimal('-3.217160342717258261933904529E-7'))
1281
1282 def test___all__(self):
1283 expected = token.__all__ + [
1284 "TokenInfo", "TokenError", "generate_tokens",
1285 "detect_encoding", "untokenize", "open", "tokenize",
1286 ]
1287 self.assertCountEqual(tokenize.__all__, expected)
1288
1289
1290 class TestTokenizerAdheresToPep0263(TestCase):
1291 """
1292 Test that tokenizer adheres to the coding behaviour stipulated in PEP 0263.
1293 """
1294
1295 def _testFile(self, filename):
1296 path = os.path.join(os.path.dirname(__file__), 'tokenizedata', filename)
1297 with open(path, 'rb') as f:
1298 TestRoundtrip.check_roundtrip(self, f)
1299
1300 def test_utf8_coding_cookie_and_no_utf8_bom(self):
1301 f = 'tokenize_tests-utf8-coding-cookie-and-no-utf8-bom-sig.txt'
1302 self._testFile(f)
1303
1304 def test_latin1_coding_cookie_and_utf8_bom(self):
1305 """
1306 As per PEP 0263, if a file starts with a utf-8 BOM signature, the only
1307 allowed encoding for the comment is 'utf-8'. The text file used in
1308 this test starts with a BOM signature, but specifies latin1 as the
1309 coding, so verify that a SyntaxError is raised, which matches the
1310 behaviour of the interpreter when it encounters a similar condition.
1311 """
1312 f = 'tokenize_tests-latin1-coding-cookie-and-utf8-bom-sig.txt'
1313 self.assertRaises(SyntaxError, self._testFile, f)
1314
1315 def test_no_coding_cookie_and_utf8_bom(self):
1316 f = 'tokenize_tests-no-coding-cookie-and-utf8-bom-sig-only.txt'
1317 self._testFile(f)
1318
1319 def test_utf8_coding_cookie_and_utf8_bom(self):
1320 f = 'tokenize_tests-utf8-coding-cookie-and-utf8-bom-sig.txt'
1321 self._testFile(f)
1322
1323 def test_bad_coding_cookie(self):
1324 self.assertRaises(SyntaxError, self._testFile, 'bad_coding.py')
1325 self.assertRaises(SyntaxError, self._testFile, 'bad_coding2.py')
1326
1327
1328 class Test_Tokenize(TestCase):
1329
1330 def test__tokenize_decodes_with_specified_encoding(self):
1331 literal = '"ЉЊЈЁЂ"'
1332 line = literal.encode('utf-8')
1333 first = False
1334 def readline():
1335 nonlocal first
1336 if not first:
1337 first = True
1338 yield line
1339 else:
1340 yield b''
1341
1342 # skip the initial encoding token and the end tokens
1343 tokens = list(tokenize._generate_tokens_from_c_tokenizer(readline().__next__,
1344 encoding='utf-8',
1345 extra_tokens=True))[:-2]
1346 expected_tokens = [tokenize.TokenInfo(3, '"ЉЊЈЁЂ"', (1, 0), (1, 7), '"ЉЊЈЁЂ"')]
1347 self.assertEqual(tokens, expected_tokens,
1348 "bytes not decoded with encoding")
1349
1350
1351 class TestDetectEncoding(TestCase):
1352
1353 def get_readline(self, lines):
1354 index = 0
1355 def readline():
1356 nonlocal index
1357 if index == len(lines):
1358 raise StopIteration
1359 line = lines[index]
1360 index += 1
1361 return line
1362 return readline
1363
1364 def test_no_bom_no_encoding_cookie(self):
1365 lines = (
1366 b'#!/home/\xc3\xa4/bin/python\n',
1367 b'# something \xe2\x82\xac\n',
1368 b'print(something)\n',
1369 b'do_something(else)\n'
1370 )
1371 encoding, consumed_lines = tokenize.detect_encoding(self.get_readline(lines))
1372 self.assertEqual(encoding, 'utf-8')
1373 self.assertEqual(consumed_lines, list(lines[:2]))
1374
1375 def test_no_bom_no_encoding_cookie_first_line_error(self):
1376 lines = (
1377 b'#!/home/\xa4/bin/python\n\n',
1378 b'print(something)\n',
1379 b'do_something(else)\n'
1380 )
1381 with self.assertRaises(SyntaxError):
1382 tokenize.detect_encoding(self.get_readline(lines))
1383
1384 def test_no_bom_no_encoding_cookie_second_line_error(self):
1385 lines = (
1386 b'#!/usr/bin/python\n',
1387 b'# something \xe2\n',
1388 b'print(something)\n',
1389 b'do_something(else)\n'
1390 )
1391 with self.assertRaises(SyntaxError):
1392 tokenize.detect_encoding(self.get_readline(lines))
1393
1394 def test_bom_no_cookie(self):
1395 lines = (
1396 b'\xef\xbb\xbf#!/home/\xc3\xa4/bin/python\n',
1397 b'print(something)\n',
1398 b'do_something(else)\n'
1399 )
1400 encoding, consumed_lines = tokenize.detect_encoding(self.get_readline(lines))
1401 self.assertEqual(encoding, 'utf-8-sig')
1402 self.assertEqual(consumed_lines,
1403 [b'#!/home/\xc3\xa4/bin/python\n', b'print(something)\n'])
1404
1405 def test_bom_no_cookie_first_line_error(self):
1406 lines = (
1407 b'\xef\xbb\xbf#!/home/\xa4/bin/python\n',
1408 b'print(something)\n',
1409 b'do_something(else)\n'
1410 )
1411 with self.assertRaises(SyntaxError):
1412 tokenize.detect_encoding(self.get_readline(lines))
1413
1414 def test_bom_no_cookie_second_line_error(self):
1415 lines = (
1416 b'\xef\xbb\xbf#!/usr/bin/python\n',
1417 b'# something \xe2\n',
1418 b'print(something)\n',
1419 b'do_something(else)\n'
1420 )
1421 with self.assertRaises(SyntaxError):
1422 tokenize.detect_encoding(self.get_readline(lines))
1423
1424 def test_cookie_first_line_no_bom(self):
1425 lines = (
1426 b'# -*- coding: latin-1 -*-\n',
1427 b'print(something)\n',
1428 b'do_something(else)\n'
1429 )
1430 encoding, consumed_lines = tokenize.detect_encoding(self.get_readline(lines))
1431 self.assertEqual(encoding, 'iso-8859-1')
1432 self.assertEqual(consumed_lines, [b'# -*- coding: latin-1 -*-\n'])
1433
1434 def test_matched_bom_and_cookie_first_line(self):
1435 lines = (
1436 b'\xef\xbb\xbf# coding=utf-8\n',
1437 b'print(something)\n',
1438 b'do_something(else)\n'
1439 )
1440 encoding, consumed_lines = tokenize.detect_encoding(self.get_readline(lines))
1441 self.assertEqual(encoding, 'utf-8-sig')
1442 self.assertEqual(consumed_lines, [b'# coding=utf-8\n'])
1443
1444 def test_mismatched_bom_and_cookie_first_line_raises_syntaxerror(self):
1445 lines = (
1446 b'\xef\xbb\xbf# vim: set fileencoding=ascii :\n',
1447 b'print(something)\n',
1448 b'do_something(else)\n'
1449 )
1450 readline = self.get_readline(lines)
1451 self.assertRaises(SyntaxError, tokenize.detect_encoding, readline)
1452
1453 def test_cookie_second_line_no_bom(self):
1454 lines = (
1455 b'#! something\n',
1456 b'# vim: set fileencoding=ascii :\n',
1457 b'print(something)\n',
1458 b'do_something(else)\n'
1459 )
1460 encoding, consumed_lines = tokenize.detect_encoding(self.get_readline(lines))
1461 self.assertEqual(encoding, 'ascii')
1462 expected = [b'#! something\n', b'# vim: set fileencoding=ascii :\n']
1463 self.assertEqual(consumed_lines, expected)
1464
1465 def test_matched_bom_and_cookie_second_line(self):
1466 lines = (
1467 b'\xef\xbb\xbf#! something\n',
1468 b'f# coding=utf-8\n',
1469 b'print(something)\n',
1470 b'do_something(else)\n'
1471 )
1472 encoding, consumed_lines = tokenize.detect_encoding(self.get_readline(lines))
1473 self.assertEqual(encoding, 'utf-8-sig')
1474 self.assertEqual(consumed_lines,
1475 [b'#! something\n', b'f# coding=utf-8\n'])
1476
1477 def test_mismatched_bom_and_cookie_second_line_raises_syntaxerror(self):
1478 lines = (
1479 b'\xef\xbb\xbf#! something\n',
1480 b'# vim: set fileencoding=ascii :\n',
1481 b'print(something)\n',
1482 b'do_something(else)\n'
1483 )
1484 readline = self.get_readline(lines)
1485 self.assertRaises(SyntaxError, tokenize.detect_encoding, readline)
1486
1487 def test_cookie_second_line_noncommented_first_line(self):
1488 lines = (
1489 b"print('\xc2\xa3')\n",
1490 b'# vim: set fileencoding=iso8859-15 :\n',
1491 b"print('\xe2\x82\xac')\n"
1492 )
1493 encoding, consumed_lines = tokenize.detect_encoding(self.get_readline(lines))
1494 self.assertEqual(encoding, 'utf-8')
1495 expected = [b"print('\xc2\xa3')\n"]
1496 self.assertEqual(consumed_lines, expected)
1497
1498 def test_first_non_utf8_coding_line(self):
1499 lines = (
1500 b'#coding:iso-8859-15 \xa4\n',
1501 b'print(something)\n'
1502 )
1503 encoding, consumed_lines = tokenize.detect_encoding(self.get_readline(lines))
1504 self.assertEqual(encoding, 'iso-8859-15')
1505 self.assertEqual(consumed_lines, list(lines[:1]))
1506
1507 def test_first_utf8_coding_line_error(self):
1508 lines = (
1509 b'#coding:ascii \xc3\xa4\n',
1510 b'print(something)\n'
1511 )
1512 with self.assertRaises(SyntaxError):
1513 tokenize.detect_encoding(self.get_readline(lines))
1514
1515 def test_second_non_utf8_coding_line(self):
1516 lines = (
1517 b'#!/usr/bin/python\n',
1518 b'#coding:iso-8859-15 \xa4\n',
1519 b'print(something)\n'
1520 )
1521 encoding, consumed_lines = tokenize.detect_encoding(self.get_readline(lines))
1522 self.assertEqual(encoding, 'iso-8859-15')
1523 self.assertEqual(consumed_lines, list(lines[:2]))
1524
1525 def test_second_utf8_coding_line_error(self):
1526 lines = (
1527 b'#!/usr/bin/python\n',
1528 b'#coding:ascii \xc3\xa4\n',
1529 b'print(something)\n'
1530 )
1531 with self.assertRaises(SyntaxError):
1532 tokenize.detect_encoding(self.get_readline(lines))
1533
1534 def test_non_utf8_shebang(self):
1535 lines = (
1536 b'#!/home/\xa4/bin/python\n',
1537 b'#coding:iso-8859-15\n',
1538 b'print(something)\n'
1539 )
1540 encoding, consumed_lines = tokenize.detect_encoding(self.get_readline(lines))
1541 self.assertEqual(encoding, 'iso-8859-15')
1542 self.assertEqual(consumed_lines, list(lines[:2]))
1543
1544 def test_utf8_shebang_error(self):
1545 lines = (
1546 b'#!/home/\xc3\xa4/bin/python\n',
1547 b'#coding:ascii\n',
1548 b'print(something)\n'
1549 )
1550 with self.assertRaises(SyntaxError):
1551 tokenize.detect_encoding(self.get_readline(lines))
1552
1553 def test_cookie_second_line_empty_first_line(self):
1554 lines = (
1555 b'\n',
1556 b'# vim: set fileencoding=iso8859-15 :\n',
1557 b"print('\xe2\x82\xac')\n"
1558 )
1559 encoding, consumed_lines = tokenize.detect_encoding(self.get_readline(lines))
1560 self.assertEqual(encoding, 'iso8859-15')
1561 expected = [b'\n', b'# vim: set fileencoding=iso8859-15 :\n']
1562 self.assertEqual(consumed_lines, expected)
1563
1564 def test_cookie_third_line(self):
1565 lines = (
1566 b'#!/home/\xc3\xa4/bin/python\n',
1567 b'# something\n',
1568 b'# vim: set fileencoding=ascii :\n',
1569 b'print(something)\n',
1570 b'do_something(else)\n'
1571 )
1572 encoding, consumed_lines = tokenize.detect_encoding(self.get_readline(lines))
1573 self.assertEqual(encoding, 'utf-8')
1574 self.assertEqual(consumed_lines, list(lines[:2]))
1575
1576 def test_double_coding_line(self):
1577 # If the first line matches the second line is ignored.
1578 lines = (
1579 b'#coding:iso8859-15\n',
1580 b'#coding:latin1\n',
1581 b'print(something)\n'
1582 )
1583 encoding, consumed_lines = tokenize.detect_encoding(self.get_readline(lines))
1584 self.assertEqual(encoding, 'iso8859-15')
1585 self.assertEqual(consumed_lines, list(lines[:1]))
1586
1587 def test_double_coding_same_line(self):
1588 lines = (
1589 b'#coding:iso8859-15 coding:latin1\n',
1590 b'print(something)\n'
1591 )
1592 encoding, consumed_lines = tokenize.detect_encoding(self.get_readline(lines))
1593 self.assertEqual(encoding, 'iso8859-15')
1594 self.assertEqual(consumed_lines, list(lines[:1]))
1595
1596 def test_double_coding_utf8(self):
1597 lines = (
1598 b'#coding:utf-8\n',
1599 b'#coding:latin1\n',
1600 b'print(something)\n'
1601 )
1602 encoding, consumed_lines = tokenize.detect_encoding(self.get_readline(lines))
1603 self.assertEqual(encoding, 'utf-8')
1604 self.assertEqual(consumed_lines, list(lines[:1]))
1605
1606 def test_nul_in_first_coding_line(self):
1607 lines = (
1608 b'#coding:iso8859-15\x00\n',
1609 b'\n',
1610 b'\n',
1611 b'print(something)\n'
1612 )
1613 with self.assertRaisesRegex(SyntaxError,
1614 "source code cannot contain null bytes"):
1615 tokenize.detect_encoding(self.get_readline(lines))
1616
1617 def test_nul_in_second_coding_line(self):
1618 lines = (
1619 b'#!/usr/bin/python\n',
1620 b'#coding:iso8859-15\x00\n',
1621 b'\n',
1622 b'print(something)\n'
1623 )
1624 with self.assertRaisesRegex(SyntaxError,
1625 "source code cannot contain null bytes"):
1626 tokenize.detect_encoding(self.get_readline(lines))
1627
1628 def test_latin1_normalization(self):
1629 # See get_normal_name() in Parser/tokenizer/helpers.c.
1630 encodings = ("latin-1", "iso-8859-1", "iso-latin-1", "latin-1-unix",
1631 "iso-8859-1-unix", "iso-latin-1-mac")
1632 for encoding in encodings:
1633 for rep in ("-", "_"):
1634 enc = encoding.replace("-", rep)
1635 lines = (b"#!/usr/bin/python\n",
1636 b"# coding: " + enc.encode("ascii") + b"\n",
1637 b"print(things)\n",
1638 b"do_something += 4\n")
1639 rl = self.get_readline(lines)
1640 found, consumed_lines = tokenize.detect_encoding(rl)
1641 self.assertEqual(found, "iso-8859-1")
1642
1643 def test_syntaxerror_latin1(self):
1644 # Issue 14629: need to raise TokenError if the first
1645 # line(s) have non-UTF-8 characters
1646 lines = (
1647 b'print("\xdf")', # Latin-1: LATIN SMALL LETTER SHARP S
1648 )
1649 readline = self.get_readline(lines)
1650 self.assertRaises(SyntaxError, tokenize.detect_encoding, readline)
1651
1652 def test_utf8_normalization(self):
1653 # See get_normal_name() in Parser/tokenizer/helpers.c.
1654 encodings = ("utf-8", "utf-8-mac", "utf-8-unix")
1655 for encoding in encodings:
1656 for rep in ("-", "_"):
1657 enc = encoding.replace("-", rep)
1658 lines = (b"#!/usr/bin/python\n",
1659 b"# coding: " + enc.encode("ascii") + b"\n",
1660 b"1 + 3\n")
1661 rl = self.get_readline(lines)
1662 found, consumed_lines = tokenize.detect_encoding(rl)
1663 self.assertEqual(found, "utf-8")
1664
1665 def test_short_files(self):
1666 readline = self.get_readline((b'print(something)\n',))
1667 encoding, consumed_lines = tokenize.detect_encoding(readline)
1668 self.assertEqual(encoding, 'utf-8')
1669 self.assertEqual(consumed_lines, [b'print(something)\n'])
1670
1671 encoding, consumed_lines = tokenize.detect_encoding(self.get_readline(()))
1672 self.assertEqual(encoding, 'utf-8')
1673 self.assertEqual(consumed_lines, [])
1674
1675 readline = self.get_readline((b'\xef\xbb\xbfprint(something)\n',))
1676 encoding, consumed_lines = tokenize.detect_encoding(readline)
1677 self.assertEqual(encoding, 'utf-8-sig')
1678 self.assertEqual(consumed_lines, [b'print(something)\n'])
1679
1680 readline = self.get_readline((b'\xef\xbb\xbf',))
1681 encoding, consumed_lines = tokenize.detect_encoding(readline)
1682 self.assertEqual(encoding, 'utf-8-sig')
1683 self.assertEqual(consumed_lines, [])
1684
1685 readline = self.get_readline((b'# coding: bad\n',))
1686 self.assertRaises(SyntaxError, tokenize.detect_encoding, readline)
1687
1688 def test_false_encoding(self):
1689 # Issue 18873: "Encoding" detected in non-comment lines
1690 readline = self.get_readline((b'print("#coding=fake")',))
1691 encoding, consumed_lines = tokenize.detect_encoding(readline)
1692 self.assertEqual(encoding, 'utf-8')
1693 self.assertEqual(consumed_lines, [b'print("#coding=fake")'])
1694
1695 @support.thread_unsafe
1696 def test_open(self):
1697 filename = os_helper.TESTFN + '.py'
1698 self.addCleanup(os_helper.unlink, filename)
1699
1700 # test coding cookie
1701 for encoding in ('iso-8859-15', 'utf-8'):
1702 with open(filename, 'w', encoding=encoding) as fp:
1703 print("# coding: %s" % encoding, file=fp)
1704 print("print('euro:\u20ac')", file=fp)
1705 with tokenize.open(filename) as fp:
1706 self.assertEqual(fp.encoding, encoding)
1707 self.assertEqual(fp.mode, 'r')
1708
1709 # test BOM (no coding cookie)
1710 with open(filename, 'w', encoding='utf-8-sig') as fp:
1711 print("print('euro:\u20ac')", file=fp)
1712 with tokenize.open(filename) as fp:
1713 self.assertEqual(fp.encoding, 'utf-8-sig')
1714 self.assertEqual(fp.mode, 'r')
1715
1716 def test_filename_in_exception(self):
1717 # When possible, include the file name in the exception.
1718 path = 'some_file_path'
1719 lines = (
1720 b'print("\xdf")', # Latin-1: LATIN SMALL LETTER SHARP S
1721 )
1722 class Bunk:
1723 def __init__(self, lines, path):
1724 self.name = path
1725 self._lines = lines
1726 self._index = 0
1727
1728 def readline(self):
1729 if self._index == len(lines):
1730 raise StopIteration
1731 line = lines[self._index]
1732 self._index += 1
1733 return line
1734
1735 with self.assertRaises(SyntaxError):
1736 ins = Bunk(lines, path)
1737 # Make sure lacking a name isn't an issue.
1738 del ins.name
1739 tokenize.detect_encoding(ins.readline)
1740 with self.assertRaisesRegex(SyntaxError, '.*{}'.format(path)):
1741 ins = Bunk(lines, path)
1742 tokenize.detect_encoding(ins.readline)
1743
1744 def test_open_error(self):
1745 # Issue #23840: open() must close the binary file on error
1746 m = BytesIO(b'#coding:xxx')
1747 with mock.patch('tokenize._builtin_open', return_value=m):
1748 self.assertRaises(SyntaxError, tokenize.open, 'foobar')
1749 self.assertTrue(m.closed)
1750
1751
1752 class TestTokenize(TestCase):
1753
1754 def test_tokenize(self):
1755 import tokenize as tokenize_module
1756 encoding = "utf-8"
1757 encoding_used = None
1758 def mock_detect_encoding(readline):
1759 return encoding, [b'first', b'second']
1760
1761 def mock__tokenize(readline, encoding, **kwargs):
1762 nonlocal encoding_used
1763 encoding_used = encoding
1764 out = []
1765 while True:
1766 try:
1767 next_line = readline()
1768 except StopIteration:
1769 return out
1770 if next_line:
1771 out.append(next_line)
1772 continue
1773 return out
1774
1775 counter = 0
1776 def mock_readline():
1777 nonlocal counter
1778 counter += 1
1779 if counter == 5:
1780 return b''
1781 return str(counter).encode()
1782
1783 orig_detect_encoding = tokenize_module.detect_encoding
1784 orig_c_token = tokenize_module._generate_tokens_from_c_tokenizer
1785 tokenize_module.detect_encoding = mock_detect_encoding
1786 tokenize_module._generate_tokens_from_c_tokenizer = mock__tokenize
1787 try:
1788 results = tokenize.tokenize(mock_readline)
1789 self.assertEqual(list(results)[1:],
1790 [b'first', b'second', b'1', b'2', b'3', b'4'])
1791 finally:
1792 tokenize_module.detect_encoding = orig_detect_encoding
1793 tokenize_module._generate_tokens_from_c_tokenizer = orig_c_token
1794
1795 self.assertEqual(encoding_used, encoding)
1796
1797 def test_oneline_defs(self):
1798 buf = []
1799 for i in range(500):
1800 buf.append('def i{i}(): return {i}'.format(i=i))
1801 buf.append('OK')
1802 buf = '\n'.join(buf)
1803
1804 # Test that 500 consequent, one-line defs is OK
1805 toks = list(tokenize.tokenize(BytesIO(buf.encode('utf-8')).readline))
1806 self.assertEqual(toks[-3].string, 'OK') # [-1] is always ENDMARKER
1807 # [-2] is always NEWLINE
1808
1809 def assertExactTypeEqual(self, opstr, *optypes):
1810 tokens = list(tokenize.tokenize(BytesIO(opstr.encode('utf-8')).readline))
1811 num_optypes = len(optypes)
1812 self.assertEqual(len(tokens), 3 + num_optypes)
1813 self.assertEqual(tokenize.tok_name[tokens[0].exact_type],
1814 tokenize.tok_name[tokenize.ENCODING])
1815 for i in range(num_optypes):
1816 self.assertEqual(tokenize.tok_name[tokens[i + 1].exact_type],
1817 tokenize.tok_name[optypes[i]])
1818 self.assertEqual(tokenize.tok_name[tokens[1 + num_optypes].exact_type],
1819 tokenize.tok_name[token.NEWLINE])
1820 self.assertEqual(tokenize.tok_name[tokens[2 + num_optypes].exact_type],
1821 tokenize.tok_name[token.ENDMARKER])
1822
1823 def test_exact_type(self):
1824 self.assertExactTypeEqual('()', token.LPAR, token.RPAR)
1825 self.assertExactTypeEqual('[]', token.LSQB, token.RSQB)
1826 self.assertExactTypeEqual(':', token.COLON)
1827 self.assertExactTypeEqual(',', token.COMMA)
1828 self.assertExactTypeEqual(';', token.SEMI)
1829 self.assertExactTypeEqual('+', token.PLUS)
1830 self.assertExactTypeEqual('-', token.MINUS)
1831 self.assertExactTypeEqual('*', token.STAR)
1832 self.assertExactTypeEqual('/', token.SLASH)
1833 self.assertExactTypeEqual('|', token.VBAR)
1834 self.assertExactTypeEqual('&', token.AMPER)
1835 self.assertExactTypeEqual('<', token.LESS)
1836 self.assertExactTypeEqual('>', token.GREATER)
1837 self.assertExactTypeEqual('=', token.EQUAL)
1838 self.assertExactTypeEqual('.', token.DOT)
1839 self.assertExactTypeEqual('%', token.PERCENT)
1840 self.assertExactTypeEqual('{}', token.LBRACE, token.RBRACE)
1841 self.assertExactTypeEqual('==', token.EQEQUAL)
1842 self.assertExactTypeEqual('!=', token.NOTEQUAL)
1843 self.assertExactTypeEqual('<=', token.LESSEQUAL)
1844 self.assertExactTypeEqual('>=', token.GREATEREQUAL)
1845 self.assertExactTypeEqual('~', token.TILDE)
1846 self.assertExactTypeEqual('^', token.CIRCUMFLEX)
1847 self.assertExactTypeEqual('<<', token.LEFTSHIFT)
1848 self.assertExactTypeEqual('>>', token.RIGHTSHIFT)
1849 self.assertExactTypeEqual('**', token.DOUBLESTAR)
1850 self.assertExactTypeEqual('+=', token.PLUSEQUAL)
1851 self.assertExactTypeEqual('-=', token.MINEQUAL)
1852 self.assertExactTypeEqual('*=', token.STAREQUAL)
1853 self.assertExactTypeEqual('/=', token.SLASHEQUAL)
1854 self.assertExactTypeEqual('%=', token.PERCENTEQUAL)
1855 self.assertExactTypeEqual('&=', token.AMPEREQUAL)
1856 self.assertExactTypeEqual('|=', token.VBAREQUAL)
1857 self.assertExactTypeEqual('^=', token.CIRCUMFLEXEQUAL)
1858 self.assertExactTypeEqual('^=', token.CIRCUMFLEXEQUAL)
1859 self.assertExactTypeEqual('<<=', token.LEFTSHIFTEQUAL)
1860 self.assertExactTypeEqual('>>=', token.RIGHTSHIFTEQUAL)
1861 self.assertExactTypeEqual('**=', token.DOUBLESTAREQUAL)
1862 self.assertExactTypeEqual('//', token.DOUBLESLASH)
1863 self.assertExactTypeEqual('//=', token.DOUBLESLASHEQUAL)
1864 self.assertExactTypeEqual(':=', token.COLONEQUAL)
1865 self.assertExactTypeEqual('...', token.ELLIPSIS)
1866 self.assertExactTypeEqual('->', token.RARROW)
1867 self.assertExactTypeEqual('@', token.AT)
1868 self.assertExactTypeEqual('@=', token.ATEQUAL)
1869
1870 self.assertExactTypeEqual('a**2+b**2==c**2',
1871 tokenize.NAME, token.DOUBLESTAR, tokenize.NUMBER,
1872 token.PLUS,
1873 tokenize.NAME, token.DOUBLESTAR, tokenize.NUMBER,
1874 token.EQEQUAL,
1875 tokenize.NAME, token.DOUBLESTAR, tokenize.NUMBER)
1876 self.assertExactTypeEqual('{1, 2, 3}',
1877 token.LBRACE,
1878 token.NUMBER, token.COMMA,
1879 token.NUMBER, token.COMMA,
1880 token.NUMBER,
1881 token.RBRACE)
1882 self.assertExactTypeEqual('^(x & 0x1)',
1883 token.CIRCUMFLEX,
1884 token.LPAR,
1885 token.NAME, token.AMPER, token.NUMBER,
1886 token.RPAR)
1887
1888 def test_pathological_trailing_whitespace(self):
1889 # See http://bugs.python.org/issue16152
1890 self.assertExactTypeEqual('@ ', token.AT)
1891
1892 def test_comment_at_the_end_of_the_source_without_newline(self):
1893 # See http://bugs.python.org/issue44667
1894 source = 'b = 1\n\n#test'
1895 expected_tokens = [
1896 tokenize.TokenInfo(type=token.ENCODING, string='utf-8', start=(0, 0), end=(0, 0), line=''),
1897 tokenize.TokenInfo(type=token.NAME, string='b', start=(1, 0), end=(1, 1), line='b = 1\n'),
1898 tokenize.TokenInfo(type=token.OP, string='=', start=(1, 2), end=(1, 3), line='b = 1\n'),
1899 tokenize.TokenInfo(type=token.NUMBER, string='1', start=(1, 4), end=(1, 5), line='b = 1\n'),
1900 tokenize.TokenInfo(type=token.NEWLINE, string='\n', start=(1, 5), end=(1, 6), line='b = 1\n'),
1901 tokenize.TokenInfo(type=token.NL, string='\n', start=(2, 0), end=(2, 1), line='\n'),
1902 tokenize.TokenInfo(type=token.COMMENT, string='#test', start=(3, 0), end=(3, 5), line='#test'),
1903 tokenize.TokenInfo(type=token.NL, string='', start=(3, 5), end=(3, 6), line='#test'),
1904 tokenize.TokenInfo(type=token.ENDMARKER, string='', start=(4, 0), end=(4, 0), line='')
1905 ]
1906
1907 tokens = list(tokenize.tokenize(BytesIO(source.encode('utf-8')).readline))
1908 self.assertEqual(tokens, expected_tokens)
1909
1910 def test_newline_and_space_at_the_end_of_the_source_without_newline(self):
1911 # See https://github.com/python/cpython/issues/105435
1912 source = 'a\n '
1913 expected_tokens = [
1914 tokenize.TokenInfo(token.ENCODING, string='utf-8', start=(0, 0), end=(0, 0), line=''),
1915 tokenize.TokenInfo(token.NAME, string='a', start=(1, 0), end=(1, 1), line='a\n'),
1916 tokenize.TokenInfo(token.NEWLINE, string='\n', start=(1, 1), end=(1, 2), line='a\n'),
1917 tokenize.TokenInfo(token.NL, string='', start=(2, 1), end=(2, 2), line=' '),
1918 tokenize.TokenInfo(token.ENDMARKER, string='', start=(3, 0), end=(3, 0), line='')
1919 ]
1920
1921 tokens = list(tokenize.tokenize(BytesIO(source.encode('utf-8')).readline))
1922 self.assertEqual(tokens, expected_tokens)
1923
1924 def test_invalid_character_in_fstring_middle(self):
1925 # See gh-103824
1926 script = b'''F"""
1927 \xe5"""'''
1928
1929 with os_helper.temp_dir() as temp_dir:
1930 filename = os.path.join(temp_dir, "script.py")
1931 with open(filename, 'wb') as file:
1932 file.write(script)
1933 rs, _ = run_python_until_end(filename)
1934 self.assertIn(b"SyntaxError", rs.err)
1935
1936
1937 class UntokenizeTest(TestCase):
1938
1939 def test_bad_input_order(self):
1940 # raise if previous row
1941 u = tokenize.Untokenizer()
1942 u.prev_row = 2
1943 u.prev_col = 2
1944 with self.assertRaises(ValueError) as cm:
1945 u.add_whitespace((1,3))
1946 self.assertEqual(cm.exception.args[0],
1947 'start (1,3) precedes previous end (2,2)')
1948 # raise if previous column in row
1949 self.assertRaises(ValueError, u.add_whitespace, (2,1))
1950
1951 def test_backslash_continuation(self):
1952 # The problem is that <whitespace>\<newline> leaves no token
1953 u = tokenize.Untokenizer()
1954 u.prev_row = 1
1955 u.prev_col = 1
1956 u.tokens = []
1957 u.add_whitespace((2, 0))
1958 self.assertEqual(u.tokens, ['\\\n'])
1959 u.prev_row = 2
1960 u.add_whitespace((4, 4))
1961 self.assertEqual(u.tokens, ['\\\n', '\\\n\\\n', ' '])
1962 TestRoundtrip.check_roundtrip(self, 'a\n b\n c\n \\\n c\n')
1963
1964 def test_iter_compat(self):
1965 u = tokenize.Untokenizer()
1966 token = (tokenize.NAME, 'Hello')
1967 tokens = [(tokenize.ENCODING, 'utf-8'), token]
1968 u.compat(token, iter([]))
1969 self.assertEqual(u.tokens, ["Hello "])
1970 u = tokenize.Untokenizer()
1971 self.assertEqual(u.untokenize(iter([token])), 'Hello ')
1972 u = tokenize.Untokenizer()
1973 self.assertEqual(u.untokenize(iter(tokens)), 'Hello ')
1974 self.assertEqual(u.encoding, 'utf-8')
1975 self.assertEqual(tokenize.untokenize(iter(tokens)), b'Hello ')
1976
1977
1978 def contains_ambiguous_backslash(source):
1979 """Return `True` if the source contains a backslash on a
1980 line by itself. For example:
1981
1982 a = (1
1983 \\
1984 )
1985
1986 Code like this cannot be untokenized exactly. This is because
1987 the tokenizer does not produce any tokens for the line containing
1988 the backslash and so there is no way to know its indent.
1989 """
1990 pattern = re.compile(br'\n\s*\\\r?\n')
1991 return pattern.search(source) is not None
1992
1993
1994 class TestRoundtrip(TestCase):
1995
1996 def check_roundtrip(self, f):
1997 """
1998 Test roundtrip for `untokenize`. `f` is an open file or a string.
1999 The source code in f is tokenized to both 5- and 2-tuples.
2000 Both sequences are converted back to source code via
2001 tokenize.untokenize(), and the latter tokenized again to 2-tuples.
2002 The test fails if the 3 pair tokenizations do not match.
2003
2004 If the source code can be untokenized unambiguously, the
2005 untokenized code must match the original code exactly.
2006
2007 When untokenize bugs are fixed, untokenize with 5-tuples should
2008 reproduce code that does not contain a backslash continuation
2009 following spaces. A proper test should test this.
2010 """
2011 # Get source code and original tokenizations
2012 if isinstance(f, str):
2013 code = f.encode('utf-8')
2014 else:
2015 code = f.read()
2016 readline = iter(code.splitlines(keepends=True)).__next__
2017 tokens5 = list(tokenize.tokenize(readline))
2018 tokens2 = [tok[:2] for tok in tokens5]
2019 # Reproduce tokens2 from pairs
2020 bytes_from2 = tokenize.untokenize(tokens2)
2021 readline2 = iter(bytes_from2.splitlines(keepends=True)).__next__
2022 tokens2_from2 = [tok[:2] for tok in tokenize.tokenize(readline2)]
2023 self.assertEqual(tokens2_from2, tokens2)
2024 # Reproduce tokens2 from 5-tuples
2025 bytes_from5 = tokenize.untokenize(tokens5)
2026 readline5 = iter(bytes_from5.splitlines(keepends=True)).__next__
2027 tokens2_from5 = [tok[:2] for tok in tokenize.tokenize(readline5)]
2028 self.assertEqual(tokens2_from5, tokens2)
2029
2030 if not contains_ambiguous_backslash(code):
2031 # The BOM does not produce a token so there is no way to preserve it.
2032 code_without_bom = code.removeprefix(b'\xef\xbb\xbf')
2033 readline = iter(code_without_bom.splitlines(keepends=True)).__next__
2034 untokenized_code = tokenize.untokenize(tokenize.tokenize(readline))
2035 self.assertEqual(code_without_bom, untokenized_code)
2036
2037 def check_line_extraction(self, f):
2038 if isinstance(f, str):
2039 code = f.encode('utf-8')
2040 else:
2041 code = f.read()
2042 readline = iter(code.splitlines(keepends=True)).__next__
2043 for tok in tokenize.tokenize(readline):
2044 if tok.type in {tokenize.ENCODING, tokenize.ENDMARKER}:
2045 continue
2046 self.assertEqual(tok.string, tok.line[tok.start[1]: tok.end[1]])
2047
2048 def test_roundtrip(self):
2049 # There are some standard formatting practices that are easy to get right.
2050
2051 self.check_roundtrip("if x == 1:\n"
2052 " print(x)\n")
2053 self.check_roundtrip("# This is a comment\n"
2054 "# This also\n")
2055
2056 # Some people use different formatting conventions, which makes
2057 # untokenize a little trickier. Note that this test involves trailing
2058 # whitespace after the colon. Note that we use hex escapes to make the
2059 # two trailing blanks apparent in the expected output.
2060
2061 self.check_roundtrip("if x == 1 : \n"
2062 " print(x)\n")
2063 fn = support.findfile("tokenize_tests.txt", subdir="tokenizedata")
2064 with open(fn, 'rb') as f:
2065 self.check_roundtrip(f)
2066 self.check_roundtrip("if x == 1:\n"
2067 " # A comment by itself.\n"
2068 " print(x) # Comment here, too.\n"
2069 " # Another comment.\n"
2070 "after_if = True\n")
2071 self.check_roundtrip("if (x # The comments need to go in the right place\n"
2072 " == 1):\n"
2073 " print('x==1')\n")
2074 self.check_roundtrip("class Test: # A comment here\n"
2075 " # A comment with weird indent\n"
2076 " after_com = 5\n"
2077 " def x(m): return m*5 # a one liner\n"
2078 " def y(m): # A whitespace after the colon\n"
2079 " return y*4 # 3-space indent\n")
2080
2081 # Some error-handling code
2082 self.check_roundtrip("try: import somemodule\n"
2083 "except ImportError: # comment\n"
2084 " print('Can not import' # comment2\n)"
2085 "else: print('Loaded')\n")
2086
2087 self.check_roundtrip("f'\\N{EXCLAMATION MARK}'")
2088 self.check_roundtrip(r"f'\\N{SNAKE}'")
2089 self.check_roundtrip(r"f'\\N{{SNAKE}}'")
2090 self.check_roundtrip(r"f'\N{SNAKE}'")
2091 self.check_roundtrip(r"f'\\\N{SNAKE}'")
2092 self.check_roundtrip(r"f'\\\\\N{SNAKE}'")
2093 self.check_roundtrip(r"f'\\\\\\\N{SNAKE}'")
2094
2095 self.check_roundtrip(r"f'\\N{1}'")
2096 self.check_roundtrip(r"f'\\\\N{2}'")
2097 self.check_roundtrip(r"f'\\\\\\N{3}'")
2098 self.check_roundtrip(r"f'\\\\\\\\N{4}'")
2099
2100 self.check_roundtrip(r"f'\\N{{'")
2101 self.check_roundtrip(r"f'\\\\N{{'")
2102 self.check_roundtrip(r"f'\\\\\\N{{'")
2103 self.check_roundtrip(r"f'\\\\\\\\N{{'")
2104
2105 self.check_roundtrip(r"f'\n{{foo}}'")
2106 self.check_roundtrip(r"f'\\n{{foo}}'")
2107 self.check_roundtrip(r"f'\\\n{{foo}}'")
2108 self.check_roundtrip(r"f'\\\\n{{foo}}'")
2109
2110 self.check_roundtrip(r"f'\t{{foo}}'")
2111 self.check_roundtrip(r"f'\\t{{foo}}'")
2112 self.check_roundtrip(r"f'\\\t{{foo}}'")
2113 self.check_roundtrip(r"f'\\\\t{{foo}}'")
2114
2115 self.check_roundtrip(r"rf'\t{{foo}}'")
2116 self.check_roundtrip(r"rf'\\t{{foo}}'")
2117 self.check_roundtrip(r"rf'\\\t{{foo}}'")
2118 self.check_roundtrip(r"rf'\\\\t{{foo}}'")
2119
2120 self.check_roundtrip(r"rf'\{{foo}}'")
2121 self.check_roundtrip(r"f'\\{{foo}}'")
2122 self.check_roundtrip(r"rf'\\\{{foo}}'")
2123 self.check_roundtrip(r"f'\\\\{{foo}}'")
2124 cases = [
2125 """
2126 if 1:
2127 "foo"
2128 "bar"
2129 """,
2130 """
2131 if 1:
2132 ("foo"
2133 "bar")
2134 """,
2135 """
2136 if 1:
2137 "foo"
2138 "bar"
2139 """ ]
2140 for case in cases:
2141 self.check_roundtrip(case)
2142
2143 self.check_roundtrip(r"t'{ {}}'")
2144 self.check_roundtrip(r"t'{f'{ {}}'}{ {}}'")
2145 self.check_roundtrip(r"f'{t'{ {}}'}{ {}}'")
2146
2147
2148 def test_continuation(self):
2149 # Balancing continuation
2150 self.check_roundtrip("a = (3,4, \n"
2151 "5,6)\n"
2152 "y = [3, 4,\n"
2153 "5]\n"
2154 "z = {'a': 5,\n"
2155 "'b':15, 'c':True}\n"
2156 "x = len(y) + 5 - a[\n"
2157 "3] - a[2]\n"
2158 "+ len(z) - z[\n"
2159 "'b']\n")
2160
2161 def test_backslash_continuation(self):
2162 # Backslash means line continuation, except for comments
2163 self.check_roundtrip("x=1+\\\n"
2164 "1\n"
2165 "# This is a comment\\\n"
2166 "# This also\n")
2167 self.check_roundtrip("# Comment \\\n"
2168 "x = 0")
2169
2170 def test_string_concatenation(self):
2171 # Two string literals on the same line
2172 self.check_roundtrip("'' ''")
2173
2174 def test_random_files(self):
2175 # Test roundtrip on random python modules.
2176 # pass the '-ucpu' option to process the full directory.
2177
2178 import glob, random
2179 tempdir = os.path.dirname(__file__) or os.curdir
2180 testfiles = glob.glob(os.path.join(glob.escape(tempdir), "test*.py"))
2181
2182 if not support.is_resource_enabled("cpu"):
2183 testfiles = random.sample(testfiles, 10)
2184
2185 for testfile in testfiles:
2186 if support.verbose >= 2:
2187 print('tokenize', testfile)
2188 with open(testfile, 'rb') as f:
2189 with self.subTest(file=testfile):
2190 self.check_roundtrip(f)
2191 self.check_line_extraction(f)
2192
2193
2194 def roundtrip(self, code):
2195 if isinstance(code, str):
2196 code = code.encode('utf-8')
2197 return tokenize.untokenize(tokenize.tokenize(BytesIO(code).readline)).decode('utf-8')
2198
2199 def test_indentation_semantics_retained(self):
2200 """
2201 Ensure that although whitespace might be mutated in a roundtrip,
2202 the semantic meaning of the indentation remains consistent.
2203 """
2204 code = "if False:\n\tx=3\n\tx=3\n"
2205 codelines = self.roundtrip(code).split('\n')
2206 self.assertEqual(codelines[1], codelines[2])
2207 self.check_roundtrip(code)
2208
2209
2210 class InvalidPythonTests(TestCase):
2211 def test_number_followed_by_name(self):
2212 # See issue #gh-105549
2213 source = "2sin(x)"
2214 expected_tokens = [
2215 tokenize.TokenInfo(type=token.NUMBER, string='2', start=(1, 0), end=(1, 1), line='2sin(x)'),
2216 tokenize.TokenInfo(type=token.NAME, string='sin', start=(1, 1), end=(1, 4), line='2sin(x)'),
2217 tokenize.TokenInfo(type=token.OP, string='(', start=(1, 4), end=(1, 5), line='2sin(x)'),
2218 tokenize.TokenInfo(type=token.NAME, string='x', start=(1, 5), end=(1, 6), line='2sin(x)'),
2219 tokenize.TokenInfo(type=token.OP, string=')', start=(1, 6), end=(1, 7), line='2sin(x)'),
2220 tokenize.TokenInfo(type=token.NEWLINE, string='', start=(1, 7), end=(1, 8), line='2sin(x)'),
2221 tokenize.TokenInfo(type=token.ENDMARKER, string='', start=(2, 0), end=(2, 0), line='')
2222 ]
2223
2224 tokens = list(tokenize.generate_tokens(StringIO(source).readline))
2225 self.assertEqual(tokens, expected_tokens)
2226
2227 def test_number_starting_with_zero(self):
2228 source = "01234"
2229 expected_tokens = [
2230 tokenize.TokenInfo(type=token.NUMBER, string='01234', start=(1, 0), end=(1, 5), line='01234'),
2231 tokenize.TokenInfo(type=token.NEWLINE, string='', start=(1, 5), end=(1, 6), line='01234'),
2232 tokenize.TokenInfo(type=token.ENDMARKER, string='', start=(2, 0), end=(2, 0), line='')
2233 ]
2234
2235 tokens = list(tokenize.generate_tokens(StringIO(source).readline))
2236 self.assertEqual(tokens, expected_tokens)
2237
2238 class CTokenizeTest(TestCase):
2239 def check_tokenize(self, s, expected):
2240 # Format the tokens in s in a table format.
2241 # The ENDMARKER and final NEWLINE are omitted.
2242 f = StringIO(s)
2243 with self.subTest(source=s):
2244 result = stringify_tokens_from_source(
2245 tokenize._generate_tokens_from_c_tokenizer(f.readline), s
2246 )
2247 self.assertEqual(result, expected.rstrip().splitlines())
2248
2249 def test_encoding(self):
2250 def readline(encoding):
2251 yield "1+1".encode(encoding)
2252
2253 expected = [
2254 tokenize.TokenInfo(type=tokenize.NUMBER, string='1', start=(1, 0), end=(1, 1), line='1+1'),
2255 tokenize.TokenInfo(type=tokenize.OP, string='+', start=(1, 1), end=(1, 2), line='1+1'),
2256 tokenize.TokenInfo(type=tokenize.NUMBER, string='1', start=(1, 2), end=(1, 3), line='1+1'),
2257 tokenize.TokenInfo(type=tokenize.NEWLINE, string='', start=(1, 3), end=(1, 4), line='1+1'),
2258 tokenize.TokenInfo(type=tokenize.ENDMARKER, string='', start=(2, 0), end=(2, 0), line='')
2259 ]
2260 for encoding in ["utf-8", "latin-1", "utf-16"]:
2261 with self.subTest(encoding=encoding):
2262 tokens = list(tokenize._generate_tokens_from_c_tokenizer(
2263 readline(encoding).__next__,
2264 extra_tokens=True,
2265 encoding=encoding,
2266 ))
2267 self.assertEqual(tokens, expected)
2268
2269 def test_int(self):
2270
2271 self.check_tokenize('0xff <= 255', """\
2272 NUMBER '0xff' (1, 0) (1, 4)
2273 LESSEQUAL '<=' (1, 5) (1, 7)
2274 NUMBER '255' (1, 8) (1, 11)
2275 """)
2276
2277 self.check_tokenize('0b10 <= 255', """\
2278 NUMBER '0b10' (1, 0) (1, 4)
2279 LESSEQUAL '<=' (1, 5) (1, 7)
2280 NUMBER '255' (1, 8) (1, 11)
2281 """)
2282
2283 self.check_tokenize('0o123 <= 0O123', """\
2284 NUMBER '0o123' (1, 0) (1, 5)
2285 LESSEQUAL '<=' (1, 6) (1, 8)
2286 NUMBER '0O123' (1, 9) (1, 14)
2287 """)
2288
2289 self.check_tokenize('1234567 > ~0x15', """\
2290 NUMBER '1234567' (1, 0) (1, 7)
2291 GREATER '>' (1, 8) (1, 9)
2292 TILDE '~' (1, 10) (1, 11)
2293 NUMBER '0x15' (1, 11) (1, 15)
2294 """)
2295
2296 self.check_tokenize('2134568 != 1231515', """\
2297 NUMBER '2134568' (1, 0) (1, 7)
2298 NOTEQUAL '!=' (1, 8) (1, 10)
2299 NUMBER '1231515' (1, 11) (1, 18)
2300 """)
2301
2302 self.check_tokenize('(-124561-1) & 200000000', """\
2303 LPAR '(' (1, 0) (1, 1)
2304 MINUS '-' (1, 1) (1, 2)
2305 NUMBER '124561' (1, 2) (1, 8)
2306 MINUS '-' (1, 8) (1, 9)
2307 NUMBER '1' (1, 9) (1, 10)
2308 RPAR ')' (1, 10) (1, 11)
2309 AMPER '&' (1, 12) (1, 13)
2310 NUMBER '200000000' (1, 14) (1, 23)
2311 """)
2312
2313 self.check_tokenize('0xdeadbeef != -1', """\
2314 NUMBER '0xdeadbeef' (1, 0) (1, 10)
2315 NOTEQUAL '!=' (1, 11) (1, 13)
2316 MINUS '-' (1, 14) (1, 15)
2317 NUMBER '1' (1, 15) (1, 16)
2318 """)
2319
2320 self.check_tokenize('0xdeadc0de & 12345', """\
2321 NUMBER '0xdeadc0de' (1, 0) (1, 10)
2322 AMPER '&' (1, 11) (1, 12)
2323 NUMBER '12345' (1, 13) (1, 18)
2324 """)
2325
2326 self.check_tokenize('0xFF & 0x15 | 1234', """\
2327 NUMBER '0xFF' (1, 0) (1, 4)
2328 AMPER '&' (1, 5) (1, 6)
2329 NUMBER '0x15' (1, 7) (1, 11)
2330 VBAR '|' (1, 12) (1, 13)
2331 NUMBER '1234' (1, 14) (1, 18)
2332 """)
2333
2334 def test_float(self):
2335
2336 self.check_tokenize('x = 3.14159', """\
2337 NAME 'x' (1, 0) (1, 1)
2338 EQUAL '=' (1, 2) (1, 3)
2339 NUMBER '3.14159' (1, 4) (1, 11)
2340 """)
2341
2342 self.check_tokenize('x = 314159.', """\
2343 NAME 'x' (1, 0) (1, 1)
2344 EQUAL '=' (1, 2) (1, 3)
2345 NUMBER '314159.' (1, 4) (1, 11)
2346 """)
2347
2348 self.check_tokenize('x = .314159', """\
2349 NAME 'x' (1, 0) (1, 1)
2350 EQUAL '=' (1, 2) (1, 3)
2351 NUMBER '.314159' (1, 4) (1, 11)
2352 """)
2353
2354 self.check_tokenize('x = 3e14159', """\
2355 NAME 'x' (1, 0) (1, 1)
2356 EQUAL '=' (1, 2) (1, 3)
2357 NUMBER '3e14159' (1, 4) (1, 11)
2358 """)
2359
2360 self.check_tokenize('x = 3E123', """\
2361 NAME 'x' (1, 0) (1, 1)
2362 EQUAL '=' (1, 2) (1, 3)
2363 NUMBER '3E123' (1, 4) (1, 9)
2364 """)
2365
2366 self.check_tokenize('x+y = 3e-1230', """\
2367 NAME 'x' (1, 0) (1, 1)
2368 PLUS '+' (1, 1) (1, 2)
2369 NAME 'y' (1, 2) (1, 3)
2370 EQUAL '=' (1, 4) (1, 5)
2371 NUMBER '3e-1230' (1, 6) (1, 13)
2372 """)
2373
2374 self.check_tokenize('x = 3.14e159', """\
2375 NAME 'x' (1, 0) (1, 1)
2376 EQUAL '=' (1, 2) (1, 3)
2377 NUMBER '3.14e159' (1, 4) (1, 12)
2378 """)
2379
2380 def test_string(self):
2381
2382 self.check_tokenize('x = \'\'; y = ""', """\
2383 NAME 'x' (1, 0) (1, 1)
2384 EQUAL '=' (1, 2) (1, 3)
2385 STRING "''" (1, 4) (1, 6)
2386 SEMI ';' (1, 6) (1, 7)
2387 NAME 'y' (1, 8) (1, 9)
2388 EQUAL '=' (1, 10) (1, 11)
2389 STRING '""' (1, 12) (1, 14)
2390 """)
2391
2392 self.check_tokenize('x = \'"\'; y = "\'"', """\
2393 NAME 'x' (1, 0) (1, 1)
2394 EQUAL '=' (1, 2) (1, 3)
2395 STRING '\\'"\\'' (1, 4) (1, 7)
2396 SEMI ';' (1, 7) (1, 8)
2397 NAME 'y' (1, 9) (1, 10)
2398 EQUAL '=' (1, 11) (1, 12)
2399 STRING '"\\'"' (1, 13) (1, 16)
2400 """)
2401
2402 self.check_tokenize('x = "doesn\'t "shrink", does it"', """\
2403 NAME 'x' (1, 0) (1, 1)
2404 EQUAL '=' (1, 2) (1, 3)
2405 STRING '"doesn\\'t "' (1, 4) (1, 14)
2406 NAME 'shrink' (1, 14) (1, 20)
2407 STRING '", does it"' (1, 20) (1, 31)
2408 """)
2409
2410 self.check_tokenize("x = 'abc' + 'ABC'", """\
2411 NAME 'x' (1, 0) (1, 1)
2412 EQUAL '=' (1, 2) (1, 3)
2413 STRING "'abc'" (1, 4) (1, 9)
2414 PLUS '+' (1, 10) (1, 11)
2415 STRING "'ABC'" (1, 12) (1, 17)
2416 """)
2417
2418 self.check_tokenize('y = "ABC" + "ABC"', """\
2419 NAME 'y' (1, 0) (1, 1)
2420 EQUAL '=' (1, 2) (1, 3)
2421 STRING '"ABC"' (1, 4) (1, 9)
2422 PLUS '+' (1, 10) (1, 11)
2423 STRING '"ABC"' (1, 12) (1, 17)
2424 """)
2425
2426 self.check_tokenize("x = r'abc' + r'ABC' + R'ABC' + R'ABC'", """\
2427 NAME 'x' (1, 0) (1, 1)
2428 EQUAL '=' (1, 2) (1, 3)
2429 STRING "r'abc'" (1, 4) (1, 10)
2430 PLUS '+' (1, 11) (1, 12)
2431 STRING "r'ABC'" (1, 13) (1, 19)
2432 PLUS '+' (1, 20) (1, 21)
2433 STRING "R'ABC'" (1, 22) (1, 28)
2434 PLUS '+' (1, 29) (1, 30)
2435 STRING "R'ABC'" (1, 31) (1, 37)
2436 """)
2437
2438 self.check_tokenize('y = r"abc" + r"ABC" + R"ABC" + R"ABC"', """\
2439 NAME 'y' (1, 0) (1, 1)
2440 EQUAL '=' (1, 2) (1, 3)
2441 STRING 'r"abc"' (1, 4) (1, 10)
2442 PLUS '+' (1, 11) (1, 12)
2443 STRING 'r"ABC"' (1, 13) (1, 19)
2444 PLUS '+' (1, 20) (1, 21)
2445 STRING 'R"ABC"' (1, 22) (1, 28)
2446 PLUS '+' (1, 29) (1, 30)
2447 STRING 'R"ABC"' (1, 31) (1, 37)
2448 """)
2449
2450 self.check_tokenize("u'abc' + U'abc'", """\
2451 STRING "u'abc'" (1, 0) (1, 6)
2452 PLUS '+' (1, 7) (1, 8)
2453 STRING "U'abc'" (1, 9) (1, 15)
2454 """)
2455
2456 self.check_tokenize('u"abc" + U"abc"', """\
2457 STRING 'u"abc"' (1, 0) (1, 6)
2458 PLUS '+' (1, 7) (1, 8)
2459 STRING 'U"abc"' (1, 9) (1, 15)
2460 """)
2461
2462 self.check_tokenize("b'abc' + B'abc'", """\
2463 STRING "b'abc'" (1, 0) (1, 6)
2464 PLUS '+' (1, 7) (1, 8)
2465 STRING "B'abc'" (1, 9) (1, 15)
2466 """)
2467
2468 self.check_tokenize('b"abc" + B"abc"', """\
2469 STRING 'b"abc"' (1, 0) (1, 6)
2470 PLUS '+' (1, 7) (1, 8)
2471 STRING 'B"abc"' (1, 9) (1, 15)
2472 """)
2473
2474 self.check_tokenize("br'abc' + bR'abc' + Br'abc' + BR'abc'", """\
2475 STRING "br'abc'" (1, 0) (1, 7)
2476 PLUS '+' (1, 8) (1, 9)
2477 STRING "bR'abc'" (1, 10) (1, 17)
2478 PLUS '+' (1, 18) (1, 19)
2479 STRING "Br'abc'" (1, 20) (1, 27)
2480 PLUS '+' (1, 28) (1, 29)
2481 STRING "BR'abc'" (1, 30) (1, 37)
2482 """)
2483
2484 self.check_tokenize('br"abc" + bR"abc" + Br"abc" + BR"abc"', """\
2485 STRING 'br"abc"' (1, 0) (1, 7)
2486 PLUS '+' (1, 8) (1, 9)
2487 STRING 'bR"abc"' (1, 10) (1, 17)
2488 PLUS '+' (1, 18) (1, 19)
2489 STRING 'Br"abc"' (1, 20) (1, 27)
2490 PLUS '+' (1, 28) (1, 29)
2491 STRING 'BR"abc"' (1, 30) (1, 37)
2492 """)
2493
2494 self.check_tokenize("rb'abc' + rB'abc' + Rb'abc' + RB'abc'", """\
2495 STRING "rb'abc'" (1, 0) (1, 7)
2496 PLUS '+' (1, 8) (1, 9)
2497 STRING "rB'abc'" (1, 10) (1, 17)
2498 PLUS '+' (1, 18) (1, 19)
2499 STRING "Rb'abc'" (1, 20) (1, 27)
2500 PLUS '+' (1, 28) (1, 29)
2501 STRING "RB'abc'" (1, 30) (1, 37)
2502 """)
2503
2504 self.check_tokenize('rb"abc" + rB"abc" + Rb"abc" + RB"abc"', """\
2505 STRING 'rb"abc"' (1, 0) (1, 7)
2506 PLUS '+' (1, 8) (1, 9)
2507 STRING 'rB"abc"' (1, 10) (1, 17)
2508 PLUS '+' (1, 18) (1, 19)
2509 STRING 'Rb"abc"' (1, 20) (1, 27)
2510 PLUS '+' (1, 28) (1, 29)
2511 STRING 'RB"abc"' (1, 30) (1, 37)
2512 """)
2513
2514 self.check_tokenize('"a\\\nde\\\nfg"', """\
2515 STRING '"a\\\\\\nde\\\\\\nfg"\' (1, 0) (3, 3)
2516 """)
2517
2518 self.check_tokenize('u"a\\\nde"', """\
2519 STRING 'u"a\\\\\\nde"\' (1, 0) (2, 3)
2520 """)
2521
2522 self.check_tokenize('rb"a\\\nd"', """\
2523 STRING 'rb"a\\\\\\nd"\' (1, 0) (2, 2)
2524 """)
2525
2526 self.check_tokenize(r'"""a\
2527 b"""', """\
2528 STRING '\"\""a\\\\\\nb\"\""' (1, 0) (2, 4)
2529 """)
2530 self.check_tokenize(r'u"""a\
2531 b"""', """\
2532 STRING 'u\"\""a\\\\\\nb\"\""' (1, 0) (2, 4)
2533 """)
2534 self.check_tokenize(r'rb"""a\
2535 b\
2536 c"""', """\
2537 STRING 'rb"\""a\\\\\\nb\\\\\\nc"\""' (1, 0) (3, 4)
2538 """)
2539
2540 self.check_tokenize(r'"hola\\\r\ndfgf"', """\
2541 STRING \'"hola\\\\\\\\\\\\r\\\\ndfgf"\' (1, 0) (1, 16)
2542 """)
2543
2544 self.check_tokenize('f"abc"', """\
2545 FSTRING_START 'f"' (1, 0) (1, 2)
2546 FSTRING_MIDDLE 'abc' (1, 2) (1, 5)
2547 FSTRING_END '"' (1, 5) (1, 6)
2548 """)
2549
2550 self.check_tokenize('fR"a{b}c"', """\
2551 FSTRING_START 'fR"' (1, 0) (1, 3)
2552 FSTRING_MIDDLE 'a' (1, 3) (1, 4)
2553 LBRACE '{' (1, 4) (1, 5)
2554 NAME 'b' (1, 5) (1, 6)
2555 RBRACE '}' (1, 6) (1, 7)
2556 FSTRING_MIDDLE 'c' (1, 7) (1, 8)
2557 FSTRING_END '"' (1, 8) (1, 9)
2558 """)
2559
2560 self.check_tokenize('f"""abc"""', """\
2561 FSTRING_START 'f\"""' (1, 0) (1, 4)
2562 FSTRING_MIDDLE 'abc' (1, 4) (1, 7)
2563 FSTRING_END '\"""' (1, 7) (1, 10)
2564 """)
2565
2566 self.check_tokenize(r'f"abc\
2567 def"', """\
2568 FSTRING_START \'f"\' (1, 0) (1, 2)
2569 FSTRING_MIDDLE 'abc\\\\\\ndef' (1, 2) (2, 3)
2570 FSTRING_END '"' (2, 3) (2, 4)
2571 """)
2572
2573 self.check_tokenize('''\
2574 f"{
2575 a}"''', """\
2576 FSTRING_START 'f"' (1, 0) (1, 2)
2577 LBRACE '{' (1, 2) (1, 3)
2578 NAME 'a' (2, 0) (2, 1)
2579 RBRACE '}' (2, 1) (2, 2)
2580 FSTRING_END '"' (2, 2) (2, 3)
2581 """)
2582
2583 self.check_tokenize(r'Rf"abc\
2584 def"', """\
2585 FSTRING_START 'Rf"' (1, 0) (1, 3)
2586 FSTRING_MIDDLE 'abc\\\\\\ndef' (1, 3) (2, 3)
2587 FSTRING_END '"' (2, 3) (2, 4)
2588 """)
2589
2590 self.check_tokenize(r'f"hola\\\r\ndfgf"', """\
2591 FSTRING_START \'f"\' (1, 0) (1, 2)
2592 FSTRING_MIDDLE 'hola\\\\\\\\\\\\r\\\\ndfgf' (1, 2) (1, 16)
2593 FSTRING_END \'"\' (1, 16) (1, 17)
2594 """)
2595
2596 self.check_tokenize("""\
2597 f'''__{
2598 x:a
2599 }__'''""", """\
2600 FSTRING_START "f'''" (1, 0) (1, 4)
2601 FSTRING_MIDDLE '__' (1, 4) (1, 6)
2602 LBRACE '{' (1, 6) (1, 7)
2603 NAME 'x' (2, 4) (2, 5)
2604 COLON ':' (2, 5) (2, 6)
2605 FSTRING_MIDDLE 'a\\n' (2, 6) (3, 0)
2606 RBRACE '}' (3, 0) (3, 1)
2607 FSTRING_MIDDLE '__' (3, 1) (3, 3)
2608 FSTRING_END "'''" (3, 3) (3, 6)
2609 """)
2610
2611 self.check_tokenize("""\
2612 f'''__{
2613 x:a
2614 b
2615 c
2616 d
2617 }__'''""", """\
2618 FSTRING_START "f'''" (1, 0) (1, 4)
2619 FSTRING_MIDDLE '__' (1, 4) (1, 6)
2620 LBRACE '{' (1, 6) (1, 7)
2621 NAME 'x' (2, 4) (2, 5)
2622 COLON ':' (2, 5) (2, 6)
2623 FSTRING_MIDDLE 'a\\n b\\n c\\n d\\n' (2, 6) (6, 0)
2624 RBRACE '}' (6, 0) (6, 1)
2625 FSTRING_MIDDLE '__' (6, 1) (6, 3)
2626 FSTRING_END "'''" (6, 3) (6, 6)
2627 """)
2628
2629 def test_function(self):
2630
2631 self.check_tokenize('def d22(a, b, c=2, d=2, *k): pass', """\
2632 NAME 'def' (1, 0) (1, 3)
2633 NAME 'd22' (1, 4) (1, 7)
2634 LPAR '(' (1, 7) (1, 8)
2635 NAME 'a' (1, 8) (1, 9)
2636 COMMA ',' (1, 9) (1, 10)
2637 NAME 'b' (1, 11) (1, 12)
2638 COMMA ',' (1, 12) (1, 13)
2639 NAME 'c' (1, 14) (1, 15)
2640 EQUAL '=' (1, 15) (1, 16)
2641 NUMBER '2' (1, 16) (1, 17)
2642 COMMA ',' (1, 17) (1, 18)
2643 NAME 'd' (1, 19) (1, 20)
2644 EQUAL '=' (1, 20) (1, 21)
2645 NUMBER '2' (1, 21) (1, 22)
2646 COMMA ',' (1, 22) (1, 23)
2647 STAR '*' (1, 24) (1, 25)
2648 NAME 'k' (1, 25) (1, 26)
2649 RPAR ')' (1, 26) (1, 27)
2650 COLON ':' (1, 27) (1, 28)
2651 NAME 'pass' (1, 29) (1, 33)
2652 """)
2653
2654 self.check_tokenize('def d01v_(a=1, *k, **w): pass', """\
2655 NAME 'def' (1, 0) (1, 3)
2656 NAME 'd01v_' (1, 4) (1, 9)
2657 LPAR '(' (1, 9) (1, 10)
2658 NAME 'a' (1, 10) (1, 11)
2659 EQUAL '=' (1, 11) (1, 12)
2660 NUMBER '1' (1, 12) (1, 13)
2661 COMMA ',' (1, 13) (1, 14)
2662 STAR '*' (1, 15) (1, 16)
2663 NAME 'k' (1, 16) (1, 17)
2664 COMMA ',' (1, 17) (1, 18)
2665 DOUBLESTAR '**' (1, 19) (1, 21)
2666 NAME 'w' (1, 21) (1, 22)
2667 RPAR ')' (1, 22) (1, 23)
2668 COLON ':' (1, 23) (1, 24)
2669 NAME 'pass' (1, 25) (1, 29)
2670 """)
2671
2672 self.check_tokenize('def d23(a: str, b: int=3) -> int: pass', """\
2673 NAME 'def' (1, 0) (1, 3)
2674 NAME 'd23' (1, 4) (1, 7)
2675 LPAR '(' (1, 7) (1, 8)
2676 NAME 'a' (1, 8) (1, 9)
2677 COLON ':' (1, 9) (1, 10)
2678 NAME 'str' (1, 11) (1, 14)
2679 COMMA ',' (1, 14) (1, 15)
2680 NAME 'b' (1, 16) (1, 17)
2681 COLON ':' (1, 17) (1, 18)
2682 NAME 'int' (1, 19) (1, 22)
2683 EQUAL '=' (1, 22) (1, 23)
2684 NUMBER '3' (1, 23) (1, 24)
2685 RPAR ')' (1, 24) (1, 25)
2686 RARROW '->' (1, 26) (1, 28)
2687 NAME 'int' (1, 29) (1, 32)
2688 COLON ':' (1, 32) (1, 33)
2689 NAME 'pass' (1, 34) (1, 38)
2690 """)
2691
2692 def test_comparison(self):
2693
2694 self.check_tokenize("if 1 < 1 > 1 == 1 >= 5 <= 0x15 <= 0x12 != "
2695 "1 and 5 in 1 not in 1 is 1 or 5 is not 1: pass", """\
2696 NAME 'if' (1, 0) (1, 2)
2697 NUMBER '1' (1, 3) (1, 4)
2698 LESS '<' (1, 5) (1, 6)
2699 NUMBER '1' (1, 7) (1, 8)
2700 GREATER '>' (1, 9) (1, 10)
2701 NUMBER '1' (1, 11) (1, 12)
2702 EQEQUAL '==' (1, 13) (1, 15)
2703 NUMBER '1' (1, 16) (1, 17)
2704 GREATEREQUAL '>=' (1, 18) (1, 20)
2705 NUMBER '5' (1, 21) (1, 22)
2706 LESSEQUAL '<=' (1, 23) (1, 25)
2707 NUMBER '0x15' (1, 26) (1, 30)
2708 LESSEQUAL '<=' (1, 31) (1, 33)
2709 NUMBER '0x12' (1, 34) (1, 38)
2710 NOTEQUAL '!=' (1, 39) (1, 41)
2711 NUMBER '1' (1, 42) (1, 43)
2712 NAME 'and' (1, 44) (1, 47)
2713 NUMBER '5' (1, 48) (1, 49)
2714 NAME 'in' (1, 50) (1, 52)
2715 NUMBER '1' (1, 53) (1, 54)
2716 NAME 'not' (1, 55) (1, 58)
2717 NAME 'in' (1, 59) (1, 61)
2718 NUMBER '1' (1, 62) (1, 63)
2719 NAME 'is' (1, 64) (1, 66)
2720 NUMBER '1' (1, 67) (1, 68)
2721 NAME 'or' (1, 69) (1, 71)
2722 NUMBER '5' (1, 72) (1, 73)
2723 NAME 'is' (1, 74) (1, 76)
2724 NAME 'not' (1, 77) (1, 80)
2725 NUMBER '1' (1, 81) (1, 82)
2726 COLON ':' (1, 82) (1, 83)
2727 NAME 'pass' (1, 84) (1, 88)
2728 """)
2729
2730 def test_additive(self):
2731
2732 self.check_tokenize('x = 1 - y + 15 - 1 + 0x124 + z + a[5]', """\
2733 NAME 'x' (1, 0) (1, 1)
2734 EQUAL '=' (1, 2) (1, 3)
2735 NUMBER '1' (1, 4) (1, 5)
2736 MINUS '-' (1, 6) (1, 7)
2737 NAME 'y' (1, 8) (1, 9)
2738 PLUS '+' (1, 10) (1, 11)
2739 NUMBER '15' (1, 12) (1, 14)
2740 MINUS '-' (1, 15) (1, 16)
2741 NUMBER '1' (1, 17) (1, 18)
2742 PLUS '+' (1, 19) (1, 20)
2743 NUMBER '0x124' (1, 21) (1, 26)
2744 PLUS '+' (1, 27) (1, 28)
2745 NAME 'z' (1, 29) (1, 30)
2746 PLUS '+' (1, 31) (1, 32)
2747 NAME 'a' (1, 33) (1, 34)
2748 LSQB '[' (1, 34) (1, 35)
2749 NUMBER '5' (1, 35) (1, 36)
2750 RSQB ']' (1, 36) (1, 37)
2751 """)
2752
2753 def test_multiplicative(self):
2754
2755 self.check_tokenize('x = 1//1*1/5*12%0x12@42', """\
2756 NAME 'x' (1, 0) (1, 1)
2757 EQUAL '=' (1, 2) (1, 3)
2758 NUMBER '1' (1, 4) (1, 5)
2759 DOUBLESLASH '//' (1, 5) (1, 7)
2760 NUMBER '1' (1, 7) (1, 8)
2761 STAR '*' (1, 8) (1, 9)
2762 NUMBER '1' (1, 9) (1, 10)
2763 SLASH '/' (1, 10) (1, 11)
2764 NUMBER '5' (1, 11) (1, 12)
2765 STAR '*' (1, 12) (1, 13)
2766 NUMBER '12' (1, 13) (1, 15)
2767 PERCENT '%' (1, 15) (1, 16)
2768 NUMBER '0x12' (1, 16) (1, 20)
2769 AT '@' (1, 20) (1, 21)
2770 NUMBER '42' (1, 21) (1, 23)
2771 """)
2772
2773 def test_unary(self):
2774
2775 self.check_tokenize('~1 ^ 1 & 1 |1 ^ -1', """\
2776 TILDE '~' (1, 0) (1, 1)
2777 NUMBER '1' (1, 1) (1, 2)
2778 CIRCUMFLEX '^' (1, 3) (1, 4)
2779 NUMBER '1' (1, 5) (1, 6)
2780 AMPER '&' (1, 7) (1, 8)
2781 NUMBER '1' (1, 9) (1, 10)
2782 VBAR '|' (1, 11) (1, 12)
2783 NUMBER '1' (1, 12) (1, 13)
2784 CIRCUMFLEX '^' (1, 14) (1, 15)
2785 MINUS '-' (1, 16) (1, 17)
2786 NUMBER '1' (1, 17) (1, 18)
2787 """)
2788
2789 self.check_tokenize('-1*1/1+1*1//1 - ---1**1', """\
2790 MINUS '-' (1, 0) (1, 1)
2791 NUMBER '1' (1, 1) (1, 2)
2792 STAR '*' (1, 2) (1, 3)
2793 NUMBER '1' (1, 3) (1, 4)
2794 SLASH '/' (1, 4) (1, 5)
2795 NUMBER '1' (1, 5) (1, 6)
2796 PLUS '+' (1, 6) (1, 7)
2797 NUMBER '1' (1, 7) (1, 8)
2798 STAR '*' (1, 8) (1, 9)
2799 NUMBER '1' (1, 9) (1, 10)
2800 DOUBLESLASH '//' (1, 10) (1, 12)
2801 NUMBER '1' (1, 12) (1, 13)
2802 MINUS '-' (1, 14) (1, 15)
2803 MINUS '-' (1, 16) (1, 17)
2804 MINUS '-' (1, 17) (1, 18)
2805 MINUS '-' (1, 18) (1, 19)
2806 NUMBER '1' (1, 19) (1, 20)
2807 DOUBLESTAR '**' (1, 20) (1, 22)
2808 NUMBER '1' (1, 22) (1, 23)
2809 """)
2810
2811 def test_selector(self):
2812
2813 self.check_tokenize("import sys, time\nx = sys.modules['time'].time()", """\
2814 NAME 'import' (1, 0) (1, 6)
2815 NAME 'sys' (1, 7) (1, 10)
2816 COMMA ',' (1, 10) (1, 11)
2817 NAME 'time' (1, 12) (1, 16)
2818 NEWLINE '' (1, 16) (1, 16)
2819 NAME 'x' (2, 0) (2, 1)
2820 EQUAL '=' (2, 2) (2, 3)
2821 NAME 'sys' (2, 4) (2, 7)
2822 DOT '.' (2, 7) (2, 8)
2823 NAME 'modules' (2, 8) (2, 15)
2824 LSQB '[' (2, 15) (2, 16)
2825 STRING "'time'" (2, 16) (2, 22)
2826 RSQB ']' (2, 22) (2, 23)
2827 DOT '.' (2, 23) (2, 24)
2828 NAME 'time' (2, 24) (2, 28)
2829 LPAR '(' (2, 28) (2, 29)
2830 RPAR ')' (2, 29) (2, 30)
2831 """)
2832
2833 def test_method(self):
2834
2835 self.check_tokenize('@staticmethod\ndef foo(x,y): pass', """\
2836 AT '@' (1, 0) (1, 1)
2837 NAME 'staticmethod' (1, 1) (1, 13)
2838 NEWLINE '' (1, 13) (1, 13)
2839 NAME 'def' (2, 0) (2, 3)
2840 NAME 'foo' (2, 4) (2, 7)
2841 LPAR '(' (2, 7) (2, 8)
2842 NAME 'x' (2, 8) (2, 9)
2843 COMMA ',' (2, 9) (2, 10)
2844 NAME 'y' (2, 10) (2, 11)
2845 RPAR ')' (2, 11) (2, 12)
2846 COLON ':' (2, 12) (2, 13)
2847 NAME 'pass' (2, 14) (2, 18)
2848 """)
2849
2850 def test_tabs(self):
2851
2852 self.check_tokenize('@staticmethod\ndef foo(x,y): pass', """\
2853 AT '@' (1, 0) (1, 1)
2854 NAME 'staticmethod' (1, 1) (1, 13)
2855 NEWLINE '' (1, 13) (1, 13)
2856 NAME 'def' (2, 0) (2, 3)
2857 NAME 'foo' (2, 4) (2, 7)
2858 LPAR '(' (2, 7) (2, 8)
2859 NAME 'x' (2, 8) (2, 9)
2860 COMMA ',' (2, 9) (2, 10)
2861 NAME 'y' (2, 10) (2, 11)
2862 RPAR ')' (2, 11) (2, 12)
2863 COLON ':' (2, 12) (2, 13)
2864 NAME 'pass' (2, 14) (2, 18)
2865 """)
2866
2867 def test_async(self):
2868
2869 self.check_tokenize('async = 1', """\
2870 NAME 'async' (1, 0) (1, 5)
2871 EQUAL '=' (1, 6) (1, 7)
2872 NUMBER '1' (1, 8) (1, 9)
2873 """)
2874
2875 self.check_tokenize('a = (async = 1)', """\
2876 NAME 'a' (1, 0) (1, 1)
2877 EQUAL '=' (1, 2) (1, 3)
2878 LPAR '(' (1, 4) (1, 5)
2879 NAME 'async' (1, 5) (1, 10)
2880 EQUAL '=' (1, 11) (1, 12)
2881 NUMBER '1' (1, 13) (1, 14)
2882 RPAR ')' (1, 14) (1, 15)
2883 """)
2884
2885 self.check_tokenize('async()', """\
2886 NAME 'async' (1, 0) (1, 5)
2887 LPAR '(' (1, 5) (1, 6)
2888 RPAR ')' (1, 6) (1, 7)
2889 """)
2890
2891 self.check_tokenize('class async(Bar):pass', """\
2892 NAME 'class' (1, 0) (1, 5)
2893 NAME 'async' (1, 6) (1, 11)
2894 LPAR '(' (1, 11) (1, 12)
2895 NAME 'Bar' (1, 12) (1, 15)
2896 RPAR ')' (1, 15) (1, 16)
2897 COLON ':' (1, 16) (1, 17)
2898 NAME 'pass' (1, 17) (1, 21)
2899 """)
2900
2901 self.check_tokenize('class async:pass', """\
2902 NAME 'class' (1, 0) (1, 5)
2903 NAME 'async' (1, 6) (1, 11)
2904 COLON ':' (1, 11) (1, 12)
2905 NAME 'pass' (1, 12) (1, 16)
2906 """)
2907
2908 self.check_tokenize('await = 1', """\
2909 NAME 'await' (1, 0) (1, 5)
2910 EQUAL '=' (1, 6) (1, 7)
2911 NUMBER '1' (1, 8) (1, 9)
2912 """)
2913
2914 self.check_tokenize('foo.async', """\
2915 NAME 'foo' (1, 0) (1, 3)
2916 DOT '.' (1, 3) (1, 4)
2917 NAME 'async' (1, 4) (1, 9)
2918 """)
2919
2920 self.check_tokenize('async for a in b: pass', """\
2921 NAME 'async' (1, 0) (1, 5)
2922 NAME 'for' (1, 6) (1, 9)
2923 NAME 'a' (1, 10) (1, 11)
2924 NAME 'in' (1, 12) (1, 14)
2925 NAME 'b' (1, 15) (1, 16)
2926 COLON ':' (1, 16) (1, 17)
2927 NAME 'pass' (1, 18) (1, 22)
2928 """)
2929
2930 self.check_tokenize('async with a as b: pass', """\
2931 NAME 'async' (1, 0) (1, 5)
2932 NAME 'with' (1, 6) (1, 10)
2933 NAME 'a' (1, 11) (1, 12)
2934 NAME 'as' (1, 13) (1, 15)
2935 NAME 'b' (1, 16) (1, 17)
2936 COLON ':' (1, 17) (1, 18)
2937 NAME 'pass' (1, 19) (1, 23)
2938 """)
2939
2940 self.check_tokenize('async.foo', """\
2941 NAME 'async' (1, 0) (1, 5)
2942 DOT '.' (1, 5) (1, 6)
2943 NAME 'foo' (1, 6) (1, 9)
2944 """)
2945
2946 self.check_tokenize('async', """\
2947 NAME 'async' (1, 0) (1, 5)
2948 """)
2949
2950 self.check_tokenize('async\n#comment\nawait', """\
2951 NAME 'async' (1, 0) (1, 5)
2952 NEWLINE '' (1, 5) (1, 5)
2953 NAME 'await' (3, 0) (3, 5)
2954 """)
2955
2956 self.check_tokenize('async\n...\nawait', """\
2957 NAME 'async' (1, 0) (1, 5)
2958 NEWLINE '' (1, 5) (1, 5)
2959 ELLIPSIS '...' (2, 0) (2, 3)
2960 NEWLINE '' (2, 3) (2, 3)
2961 NAME 'await' (3, 0) (3, 5)
2962 """)
2963
2964 self.check_tokenize('async\nawait', """\
2965 NAME 'async' (1, 0) (1, 5)
2966 NEWLINE '' (1, 5) (1, 5)
2967 NAME 'await' (2, 0) (2, 5)
2968 """)
2969
2970 self.check_tokenize('foo.async + 1', """\
2971 NAME 'foo' (1, 0) (1, 3)
2972 DOT '.' (1, 3) (1, 4)
2973 NAME 'async' (1, 4) (1, 9)
2974 PLUS '+' (1, 10) (1, 11)
2975 NUMBER '1' (1, 12) (1, 13)
2976 """)
2977
2978 self.check_tokenize('async def foo(): pass', """\
2979 NAME 'async' (1, 0) (1, 5)
2980 NAME 'def' (1, 6) (1, 9)
2981 NAME 'foo' (1, 10) (1, 13)
2982 LPAR '(' (1, 13) (1, 14)
2983 RPAR ')' (1, 14) (1, 15)
2984 COLON ':' (1, 15) (1, 16)
2985 NAME 'pass' (1, 17) (1, 21)
2986 """)
2987
2988 self.check_tokenize('''\
2989 async def foo():
2990 def foo(await):
2991 await = 1
2992 if 1:
2993 await
2994 async += 1
2995 ''', """\
2996 NAME 'async' (1, 0) (1, 5)
2997 NAME 'def' (1, 6) (1, 9)
2998 NAME 'foo' (1, 10) (1, 13)
2999 LPAR '(' (1, 13) (1, 14)
3000 RPAR ')' (1, 14) (1, 15)
3001 COLON ':' (1, 15) (1, 16)
3002 NEWLINE '' (1, 16) (1, 16)
3003 INDENT '' (2, -1) (2, -1)
3004 NAME 'def' (2, 2) (2, 5)
3005 NAME 'foo' (2, 6) (2, 9)
3006 LPAR '(' (2, 9) (2, 10)
3007 NAME 'await' (2, 10) (2, 15)
3008 RPAR ')' (2, 15) (2, 16)
3009 COLON ':' (2, 16) (2, 17)
3010 NEWLINE '' (2, 17) (2, 17)
3011 INDENT '' (3, -1) (3, -1)
3012 NAME 'await' (3, 4) (3, 9)
3013 EQUAL '=' (3, 10) (3, 11)
3014 NUMBER '1' (3, 12) (3, 13)
3015 NEWLINE '' (3, 13) (3, 13)
3016 DEDENT '' (4, -1) (4, -1)
3017 NAME 'if' (4, 2) (4, 4)
3018 NUMBER '1' (4, 5) (4, 6)
3019 COLON ':' (4, 6) (4, 7)
3020 NEWLINE '' (4, 7) (4, 7)
3021 INDENT '' (5, -1) (5, -1)
3022 NAME 'await' (5, 4) (5, 9)
3023 NEWLINE '' (5, 9) (5, 9)
3024 DEDENT '' (6, -1) (6, -1)
3025 DEDENT '' (6, -1) (6, -1)
3026 NAME 'async' (6, 0) (6, 5)
3027 PLUSEQUAL '+=' (6, 6) (6, 8)
3028 NUMBER '1' (6, 9) (6, 10)
3029 NEWLINE '' (6, 10) (6, 10)
3030 """)
3031
3032 self.check_tokenize('async def foo():\n async for i in 1: pass', """\
3033 NAME 'async' (1, 0) (1, 5)
3034 NAME 'def' (1, 6) (1, 9)
3035 NAME 'foo' (1, 10) (1, 13)
3036 LPAR '(' (1, 13) (1, 14)
3037 RPAR ')' (1, 14) (1, 15)
3038 COLON ':' (1, 15) (1, 16)
3039 NEWLINE '' (1, 16) (1, 16)
3040 INDENT '' (2, -1) (2, -1)
3041 NAME 'async' (2, 2) (2, 7)
3042 NAME 'for' (2, 8) (2, 11)
3043 NAME 'i' (2, 12) (2, 13)
3044 NAME 'in' (2, 14) (2, 16)
3045 NUMBER '1' (2, 17) (2, 18)
3046 COLON ':' (2, 18) (2, 19)
3047 NAME 'pass' (2, 20) (2, 24)
3048 DEDENT '' (2, -1) (2, -1)
3049 """)
3050
3051 self.check_tokenize('async def foo(async): await', """\
3052 NAME 'async' (1, 0) (1, 5)
3053 NAME 'def' (1, 6) (1, 9)
3054 NAME 'foo' (1, 10) (1, 13)
3055 LPAR '(' (1, 13) (1, 14)
3056 NAME 'async' (1, 14) (1, 19)
3057 RPAR ')' (1, 19) (1, 20)
3058 COLON ':' (1, 20) (1, 21)
3059 NAME 'await' (1, 22) (1, 27)
3060 """)
3061
3062 self.check_tokenize('''\
3063 def f():
3064
3065 def baz(): pass
3066 async def bar(): pass
3067
3068 await = 2''', """\
3069 NAME 'def' (1, 0) (1, 3)
3070 NAME 'f' (1, 4) (1, 5)
3071 LPAR '(' (1, 5) (1, 6)
3072 RPAR ')' (1, 6) (1, 7)
3073 COLON ':' (1, 7) (1, 8)
3074 NEWLINE '' (1, 8) (1, 8)
3075 INDENT '' (3, -1) (3, -1)
3076 NAME 'def' (3, 2) (3, 5)
3077 NAME 'baz' (3, 6) (3, 9)
3078 LPAR '(' (3, 9) (3, 10)
3079 RPAR ')' (3, 10) (3, 11)
3080 COLON ':' (3, 11) (3, 12)
3081 NAME 'pass' (3, 13) (3, 17)
3082 NEWLINE '' (3, 17) (3, 17)
3083 NAME 'async' (4, 2) (4, 7)
3084 NAME 'def' (4, 8) (4, 11)
3085 NAME 'bar' (4, 12) (4, 15)
3086 LPAR '(' (4, 15) (4, 16)
3087 RPAR ')' (4, 16) (4, 17)
3088 COLON ':' (4, 17) (4, 18)
3089 NAME 'pass' (4, 19) (4, 23)
3090 NEWLINE '' (4, 23) (4, 23)
3091 NAME 'await' (6, 2) (6, 7)
3092 EQUAL '=' (6, 8) (6, 9)
3093 NUMBER '2' (6, 10) (6, 11)
3094 DEDENT '' (6, -1) (6, -1)
3095 """)
3096
3097 self.check_tokenize('''\
3098 async def f():
3099
3100 def baz(): pass
3101 async def bar(): pass
3102
3103 await = 2''', """\
3104 NAME 'async' (1, 0) (1, 5)
3105 NAME 'def' (1, 6) (1, 9)
3106 NAME 'f' (1, 10) (1, 11)
3107 LPAR '(' (1, 11) (1, 12)
3108 RPAR ')' (1, 12) (1, 13)
3109 COLON ':' (1, 13) (1, 14)
3110 NEWLINE '' (1, 14) (1, 14)
3111 INDENT '' (3, -1) (3, -1)
3112 NAME 'def' (3, 2) (3, 5)
3113 NAME 'baz' (3, 6) (3, 9)
3114 LPAR '(' (3, 9) (3, 10)
3115 RPAR ')' (3, 10) (3, 11)
3116 COLON ':' (3, 11) (3, 12)
3117 NAME 'pass' (3, 13) (3, 17)
3118 NEWLINE '' (3, 17) (3, 17)
3119 NAME 'async' (4, 2) (4, 7)
3120 NAME 'def' (4, 8) (4, 11)
3121 NAME 'bar' (4, 12) (4, 15)
3122 LPAR '(' (4, 15) (4, 16)
3123 RPAR ')' (4, 16) (4, 17)
3124 COLON ':' (4, 17) (4, 18)
3125 NAME 'pass' (4, 19) (4, 23)
3126 NEWLINE '' (4, 23) (4, 23)
3127 NAME 'await' (6, 2) (6, 7)
3128 EQUAL '=' (6, 8) (6, 9)
3129 NUMBER '2' (6, 10) (6, 11)
3130 DEDENT '' (6, -1) (6, -1)
3131 """)
3132
3133 def test_unicode(self):
3134
3135 self.check_tokenize("Örter = u'places'\ngrün = U'green'", """\
3136 NAME 'Örter' (1, 0) (1, 5)
3137 EQUAL '=' (1, 6) (1, 7)
3138 STRING "u'places'" (1, 8) (1, 17)
3139 NEWLINE '' (1, 17) (1, 17)
3140 NAME 'grün' (2, 0) (2, 4)
3141 EQUAL '=' (2, 5) (2, 6)
3142 STRING "U'green'" (2, 7) (2, 15)
3143 """)
3144
3145 def test_invalid_syntax(self):
3146 def get_tokens(string):
3147 the_string = StringIO(string)
3148 return list(tokenize._generate_tokens_from_c_tokenizer(the_string.readline))
3149
3150 for case in [
3151 "(1+2]",
3152 "(1+2}",
3153 "{1+2]",
3154 "1_",
3155 "1.2_",
3156 "1e2_",
3157 "1e+",
3158
3159 "\xa0",
3160 "€",
3161 "0b12",
3162 "0b1_2",
3163 "0b2",
3164 "0b1_",
3165 "0b",
3166 "0o18",
3167 "0o1_8",
3168 "0o8",
3169 "0o1_",
3170 "0o",
3171 "0x1_",
3172 "0x",
3173 "1_",
3174 "012",
3175 "1.2_",
3176 "1e2_",
3177 "1e+",
3178 "'sdfsdf",
3179 "'''sdfsdf''",
3180 "("*1000+"a"+")"*1000,
3181 "]",
3182 """\
3183 f'__{
3184 x:d
3185 }__'""",
3186 " a\n\x00",
3187 ]:
3188 with self.subTest(case=case):
3189 self.assertRaises(tokenize.TokenError, get_tokens, case)
3190
3191 @support.skip_wasi_stack_overflow()
3192 def test_max_indent(self):
3193 MAXINDENT = 100
3194
3195 def generate_source(indents):
3196 source = ''.join((' ' * x) + 'if True:\n' for x in range(indents))
3197 source += ' ' * indents + 'pass\n'
3198 return source
3199
3200 valid = generate_source(MAXINDENT - 1)
3201 the_input = StringIO(valid)
3202 tokens = list(tokenize._generate_tokens_from_c_tokenizer(the_input.readline))
3203 self.assertEqual(tokens[-2].type, tokenize.DEDENT)
3204 self.assertEqual(tokens[-1].type, tokenize.ENDMARKER)
3205 compile(valid, "<string>", "exec")
3206
3207 invalid = generate_source(MAXINDENT)
3208 the_input = StringIO(invalid)
3209 self.assertRaises(IndentationError, lambda: list(tokenize._generate_tokens_from_c_tokenizer(the_input.readline)))
3210 self.assertRaises(
3211 IndentationError, compile, invalid, "<string>", "exec"
3212 )
3213
3214 def test_continuation_lines_indentation(self):
3215 def get_tokens(string):
3216 the_string = StringIO(string)
3217 return [(kind, string) for (kind, string, *_)
3218 in tokenize._generate_tokens_from_c_tokenizer(the_string.readline)]
3219
3220 code = dedent("""
3221 def fib(n):
3222 \\
3223 '''Print a Fibonacci series up to n.'''
3224 \\
3225 a, b = 0, 1
3226 """)
3227
3228 self.check_tokenize(code, """\
3229 NAME 'def' (2, 0) (2, 3)
3230 NAME 'fib' (2, 4) (2, 7)
3231 LPAR '(' (2, 7) (2, 8)
3232 NAME 'n' (2, 8) (2, 9)
3233 RPAR ')' (2, 9) (2, 10)
3234 COLON ':' (2, 10) (2, 11)
3235 NEWLINE '' (2, 11) (2, 11)
3236 INDENT '' (4, -1) (4, -1)
3237 STRING "'''Print a Fibonacci series up to n.'''" (4, 0) (4, 39)
3238 NEWLINE '' (4, 39) (4, 39)
3239 NAME 'a' (6, 0) (6, 1)
3240 COMMA ',' (6, 1) (6, 2)
3241 NAME 'b' (6, 3) (6, 4)
3242 EQUAL '=' (6, 5) (6, 6)
3243 NUMBER '0' (6, 7) (6, 8)
3244 COMMA ',' (6, 8) (6, 9)
3245 NUMBER '1' (6, 10) (6, 11)
3246 NEWLINE '' (6, 11) (6, 11)
3247 DEDENT '' (6, -1) (6, -1)
3248 """)
3249
3250 code_no_cont = dedent("""
3251 def fib(n):
3252 '''Print a Fibonacci series up to n.'''
3253 a, b = 0, 1
3254 """)
3255
3256 self.assertEqual(get_tokens(code), get_tokens(code_no_cont))
3257
3258 code = dedent("""
3259 pass
3260 \\
3261
3262 pass
3263 """)
3264
3265 self.check_tokenize(code, """\
3266 NAME 'pass' (2, 0) (2, 4)
3267 NEWLINE '' (2, 4) (2, 4)
3268 NAME 'pass' (5, 0) (5, 4)
3269 NEWLINE '' (5, 4) (5, 4)
3270 """)
3271
3272 code_no_cont = dedent("""
3273 pass
3274 pass
3275 """)
3276
3277 self.assertEqual(get_tokens(code), get_tokens(code_no_cont))
3278
3279 code = dedent("""
3280 if x:
3281 y = 1
3282 \\
3283 \\
3284 \\
3285 \\
3286 foo = 1
3287 """)
3288
3289 self.check_tokenize(code, """\
3290 NAME 'if' (2, 0) (2, 2)
3291 NAME 'x' (2, 3) (2, 4)
3292 COLON ':' (2, 4) (2, 5)
3293 NEWLINE '' (2, 5) (2, 5)
3294 INDENT '' (3, -1) (3, -1)
3295 NAME 'y' (3, 4) (3, 5)
3296 EQUAL '=' (3, 6) (3, 7)
3297 NUMBER '1' (3, 8) (3, 9)
3298 NEWLINE '' (3, 9) (3, 9)
3299 NAME 'foo' (8, 4) (8, 7)
3300 EQUAL '=' (8, 8) (8, 9)
3301 NUMBER '1' (8, 10) (8, 11)
3302 NEWLINE '' (8, 11) (8, 11)
3303 DEDENT '' (8, -1) (8, -1)
3304 """)
3305
3306 code_no_cont = dedent("""
3307 if x:
3308 y = 1
3309 foo = 1
3310 """)
3311
3312 self.assertEqual(get_tokens(code), get_tokens(code_no_cont))
3313
3314
3315 class CTokenizerBufferTests(unittest.TestCase):
3316 def test_newline_at_the_end_of_buffer(self):
3317 # See issue 99581: Make sure that if we need to add a new line at the
3318 # end of the buffer, we have enough space in the buffer, specially when
3319 # the current line is as long as the buffer space available.
3320 test_script = f"""\
3321 #coding: latin-1
3322 #{"a"*10000}
3323 #{"a"*10002}"""
3324 with os_helper.temp_dir() as temp_dir:
3325 file_name = make_script(temp_dir, 'foo', test_script)
3326 run_test_script(file_name)
3327
3328
3329 class CommandLineTest(unittest.TestCase):
3330 def setUp(self):
3331 self.filename = tempfile.mktemp()
3332 self.addCleanup(os_helper.unlink, self.filename)
3333
3334 @staticmethod
3335 def text_normalize(string):
3336 """Dedent *string* and strip it from its surrounding whitespaces.
3337
3338 This method is used by the other utility functions so that any
3339 string to write or to match against can be freely indented.
3340 """
3341 return re.sub(r'\s+', ' ', string).strip()
3342
3343 def set_source(self, content):
3344 with open(self.filename, 'w') as fp:
3345 fp.write(content)
3346
3347 def invoke_tokenize(self, *flags):
3348 output = StringIO()
3349 with contextlib.redirect_stdout(output):
3350 tokenize._main(args=[*flags, self.filename])
3351 return self.text_normalize(output.getvalue())
3352
3353 def check_output(self, source, expect, *flags):
3354 with self.subTest(source=source, flags=flags):
3355 self.set_source(source)
3356 res = self.invoke_tokenize(*flags)
3357 expect = self.text_normalize(expect)
3358 self.assertListEqual(res.splitlines(), expect.splitlines())
3359
3360 def test_invocation(self):
3361 # test various combinations of parameters
3362 base_flags = ('-e', '--exact')
3363
3364 self.set_source('''
3365 def f():
3366 print(x)
3367 return None
3368 ''')
3369
3370 for flag in base_flags:
3371 with self.subTest(args=flag):
3372 _ = self.invoke_tokenize(flag)
3373
3374 with self.assertRaises(SystemExit):
3375 # suppress argparse error message
3376 with contextlib.redirect_stderr(StringIO()):
3377 _ = self.invoke_tokenize('--unknown')
3378
3379 def test_without_flag(self):
3380 # test 'python -m tokenize source.py'
3381 source = 'a = 1'
3382 expect = '''
3383 0,0-0,0: ENCODING 'utf-8'
3384 1,0-1,1: NAME 'a'
3385 1,2-1,3: OP '='
3386 1,4-1,5: NUMBER '1'
3387 1,5-1,6: NEWLINE ''
3388 2,0-2,0: ENDMARKER ''
3389 '''
3390 self.check_output(source, expect)
3391
3392 def test_exact_flag(self):
3393 # test 'python -m tokenize -e/--exact source.py'
3394 source = 'a = 1'
3395 expect = '''
3396 0,0-0,0: ENCODING 'utf-8'
3397 1,0-1,1: NAME 'a'
3398 1,2-1,3: EQUAL '='
3399 1,4-1,5: NUMBER '1'
3400 1,5-1,6: NEWLINE ''
3401 2,0-2,0: ENDMARKER ''
3402 '''
3403 for flag in ['-e', '--exact']:
3404 self.check_output(source, expect, flag)
3405
3406
3407 class StringPrefixTest(unittest.TestCase):
3408 @staticmethod
3409 def determine_valid_prefixes():
3410 # Try all lengths until we find a length that has zero valid
3411 # prefixes. This will miss the case where for example there
3412 # are no valid 3 character prefixes, but there are valid 4
3413 # character prefixes. That seems unlikely.
3414
3415 single_char_valid_prefixes = set()
3416
3417 # Find all of the single character string prefixes. Just get
3418 # the lowercase version, we'll deal with combinations of upper
3419 # and lower case later. I'm using this logic just in case
3420 # some uppercase-only prefix is added.
3421 for letter in itertools.chain(string.ascii_lowercase, string.ascii_uppercase):
3422 try:
3423 eval(f'{letter}""')
3424 single_char_valid_prefixes.add(letter.lower())
3425 except SyntaxError:
3426 pass
3427
3428 # This logic assumes that all combinations of valid prefixes only use
3429 # the characters that are valid single character prefixes. That seems
3430 # like a valid assumption, but if it ever changes this will need
3431 # adjusting.
3432 valid_prefixes = set()
3433 for length in itertools.count():
3434 num_at_this_length = 0
3435 for prefix in (
3436 "".join(l)
3437 for l in itertools.combinations(single_char_valid_prefixes, length)
3438 ):
3439 for t in itertools.permutations(prefix):
3440 for u in itertools.product(*[(c, c.upper()) for c in t]):
3441 p = "".join(u)
3442 if p == "not":
3443 # 'not' can never be a string prefix,
3444 # because it's a valid expression: not ""
3445 continue
3446 try:
3447 eval(f'{p}""')
3448
3449 # No syntax error, so p is a valid string
3450 # prefix.
3451
3452 valid_prefixes.add(p)
3453 num_at_this_length += 1
3454 except SyntaxError:
3455 pass
3456 if num_at_this_length == 0:
3457 return valid_prefixes
3458
3459
3460 def test_prefixes(self):
3461 # Get the list of defined string prefixes. I don't see an
3462 # obvious documented way of doing this, but probably the best
3463 # thing is to split apart tokenize.StringPrefix.
3464
3465 # Make sure StringPrefix begins and ends in parens. We're
3466 # assuming it's of the form "(a|b|ab)", if a, b, and cd are
3467 # valid string prefixes.
3468 self.assertEqual(tokenize.StringPrefix[0], '(')
3469 self.assertEqual(tokenize.StringPrefix[-1], ')')
3470
3471 # Then split apart everything else by '|'.
3472 defined_prefixes = set(tokenize.StringPrefix[1:-1].split('|'))
3473
3474 # Now compute the actual allowed string prefixes and compare
3475 # to what is defined in the tokenize module.
3476 self.assertEqual(defined_prefixes, self.determine_valid_prefixes())
3477
3478
3479 if __name__ == "__main__":
3480 unittest.main()