Debian Codegraph @

Lib/test/test_source_encoding.py

raw
1 # -*- coding: utf-8 -*-
2
3 import unittest
4 from test import support
5 from test.support import script_helper
6 from test.support.os_helper import TESTFN, unlink, rmtree
7 from test.support.import_helper import unload
8 import importlib
9 import os
10 import sys
11 import subprocess
12 import tempfile
13
14 class MiscSourceEncodingTest(unittest.TestCase):
15
16 def test_import_encoded_module(self):
17 from test.encoded_modules import test_strings
18 # Make sure we're actually testing something
19 self.assertGreaterEqual(len(test_strings), 1)
20 for modname, encoding, teststr in test_strings:
21 mod = importlib.import_module('test.encoded_modules.'
22 'module_' + modname)
23 self.assertEqual(teststr, mod.test)
24
25 def test_compilestring(self):
26 # see #1882
27 c = compile(b"\n# coding: utf-8\nu = '\xc3\xb3'\n", "dummy", "exec")
28 d = {}
29 exec(c, d)
30 self.assertEqual(d['u'], '\xf3')
31
32 def test_issue2301(self):
33 try:
34 compile(b"# coding: cp932\nprint '\x94\x4e'", "dummy", "exec")
35 except SyntaxError as v:
36 self.assertEqual(v.text.rstrip('\n'), "print '\u5e74'")
37 else:
38 self.fail()
39
40 def test_issue4626(self):
41 c = compile("# coding=latin-1\n\u00c6 = '\u00c6'", "dummy", "exec")
42 d = {}
43 exec(c, d)
44 self.assertEqual(d['\xc6'], '\xc6')
45
46 def test_issue3297(self):
47 c = compile("a, b = '\U0001010F', '\\U0001010F'", "dummy", "exec")
48 d = {}
49 exec(c, d)
50 self.assertEqual(d['a'], d['b'])
51 self.assertEqual(len(d['a']), len(d['b']))
52 self.assertEqual(ascii(d['a']), ascii(d['b']))
53
54 def test_issue7820(self):
55 # Ensure that check_bom() restores all bytes in the right order if
56 # check_bom() fails in pydebug mode: a buffer starts with the first
57 # byte of a valid BOM, but next bytes are different
58
59 # one byte in common with the UTF-16-LE BOM
60 self.assertRaises(SyntaxError, eval, b'\xff\x20')
61
62 # one byte in common with the UTF-8 BOM
63 self.assertRaises(SyntaxError, eval, b'\xef\x20')
64
65 # two bytes in common with the UTF-8 BOM
66 self.assertRaises(SyntaxError, eval, b'\xef\xbb\x20')
67
68 def test_truncated_utf8_at_eof(self):
69 # Regression test for https://issues.oss-fuzz.com/issues/451112368
70 # Truncated multi-byte UTF-8 sequences at end of input caused an
71 # out-of-bounds read in Parser/tokenizer/helpers.c:valid_utf8().
72 truncated = [
73 b'\xc2', # 2-byte lead, missing 1 continuation
74 b'\xdf', # 2-byte lead, missing 1 continuation
75 b'\xe0', # 3-byte lead, missing 2 continuations
76 b'\xe0\xa0', # 3-byte lead, missing 1 continuation
77 b'\xf0\x90', # 4-byte lead, missing 2 continuations
78 b'\xf0\x90\x80', # 4-byte lead, missing 1 continuation
79 b'\xf3', # 4-byte lead, missing 3 (the oss-fuzz reproducer)
80 ]
81 for seq in truncated:
82 with self.subTest(seq=seq):
83 self.assertRaises(SyntaxError, compile, seq, '<test>', 'exec')
84
85 @support.requires_subprocess()
86 def test_20731(self):
87 sub = subprocess.Popen([sys.executable,
88 os.path.join(os.path.dirname(__file__),
89 'tokenizedata',
90 'coding20731.py')],
91 stderr=subprocess.PIPE)
92 err = sub.communicate()[1]
93 self.assertEqual(sub.returncode, 0)
94 self.assertNotIn(b'SyntaxError', err)
95
96 def test_error_message(self):
97 compile(b'# -*- coding: iso-8859-15 -*-\n', 'dummy', 'exec')
98 compile(b'\xef\xbb\xbf\n', 'dummy', 'exec')
99 compile(b'\xef\xbb\xbf# -*- coding: utf-8 -*-\n', 'dummy', 'exec')
100 with self.assertRaisesRegex(SyntaxError, 'fake'):
101 compile(b'# -*- coding: fake -*-\n', 'dummy', 'exec')
102 with self.assertRaisesRegex(SyntaxError, 'iso-8859-15'):
103 compile(b'\xef\xbb\xbf# -*- coding: iso-8859-15 -*-\n',
104 'dummy', 'exec')
105 with self.assertRaisesRegex(SyntaxError, 'BOM'):
106 compile(b'\xef\xbb\xbf# -*- coding: iso-8859-15 -*-\n',
107 'dummy', 'exec')
108 with self.assertRaisesRegex(SyntaxError, 'fake'):
109 compile(b'\xef\xbb\xbf# -*- coding: fake -*-\n', 'dummy', 'exec')
110 with self.assertRaisesRegex(SyntaxError, 'BOM'):
111 compile(b'\xef\xbb\xbf# -*- coding: fake -*-\n', 'dummy', 'exec')
112
113 def test_bad_coding(self):
114 module_name = 'bad_coding'
115 self.verify_bad_module(module_name)
116
117 def test_bad_coding2(self):
118 module_name = 'bad_coding2'
119 self.verify_bad_module(module_name)
120
121 def verify_bad_module(self, module_name):
122 self.assertRaises(SyntaxError, __import__, 'test.tokenizedata.' + module_name)
123
124 path = os.path.dirname(__file__)
125 filename = os.path.join(path, 'tokenizedata', module_name + '.py')
126 with open(filename, "rb") as fp:
127 bytes = fp.read()
128 self.assertRaises(SyntaxError, compile, bytes, filename, 'exec')
129
130 def test_exec_valid_coding(self):
131 d = {}
132 exec(b'# coding: cp949\na = "\xaa\xa7"\n', d)
133 self.assertEqual(d['a'], '\u3047')
134
135 def test_file_parse(self):
136 # issue1134: all encodings outside latin-1 and utf-8 fail on
137 # multiline strings and long lines (>512 columns)
138 unload(TESTFN)
139 filename = TESTFN + ".py"
140 f = open(filename, "w", encoding="cp1252")
141 sys.path.insert(0, os.curdir)
142 try:
143 with f:
144 f.write("# -*- coding: cp1252 -*-\n")
145 f.write("'''A short string\n")
146 f.write("'''\n")
147 f.write("'A very long string %s'\n" % ("X" * 1000))
148
149 importlib.invalidate_caches()
150 __import__(TESTFN)
151 finally:
152 del sys.path[0]
153 unlink(filename)
154 unlink(filename + "c")
155 unlink(filename + "o")
156 unload(TESTFN)
157 rmtree('__pycache__')
158
159 def test_error_from_string(self):
160 # See http://bugs.python.org/issue6289
161 input = "# coding: ascii\n\N{SNOWMAN}".encode('utf-8')
162 with self.assertRaises(SyntaxError) as c:
163 compile(input, "<string>", "exec")
164 expected = "'ascii' codec can't decode byte 0xe2 in position 16: " \
165 "ordinal not in range(128)"
166 self.assertStartsWith(c.exception.args[0], expected)
167
168 def test_file_parse_error_multiline(self):
169 # gh96611:
170 with open(TESTFN, "wb") as fd:
171 fd.write(b'print("""\n\xb1""")\n')
172
173 try:
174 retcode, stdout, stderr = script_helper.assert_python_failure(TESTFN)
175
176 self.assertGreater(retcode, 0)
177 self.assertIn(b"Non-UTF-8 code starting with '\\xb1'", stderr)
178 finally:
179 os.unlink(TESTFN)
180
181 def test_tokenizer_fstring_warning_in_first_line(self):
182 source = "0b1and 2"
183 with open(TESTFN, "w") as fd:
184 fd.write("{}".format(source))
185 try:
186 retcode, stdout, stderr = script_helper.assert_python_ok(TESTFN)
187 self.assertIn(b"SyntaxWarning: invalid binary litera", stderr)
188 self.assertEqual(stderr.count(source.encode()), 1)
189 finally:
190 os.unlink(TESTFN)
191
192
193 BUFSIZ = 2**13
194
195 class AbstractSourceEncodingTest:
196
197 def test_default_coding(self):
198 src = (b'print(ascii("\xc3\xa4"))\n')
199 self.check_script_output(src, br"'\xe4'")
200
201 def test_first_coding_line(self):
202 src = (b'#coding:iso8859-15\n'
203 b'print(ascii("\xc3\xa4"))\n')
204 self.check_script_output(src, br"'\xc3\u20ac'")
205
206 def test_second_coding_line(self):
207 src = (b'#!/usr/bin/python\n'
208 b'#coding:iso8859-15\n'
209 b'print(ascii("\xc3\xa4"))\n')
210 self.check_script_output(src, br"'\xc3\u20ac'")
211
212 def test_second_coding_line_empty_first_line(self):
213 src = (b'\n'
214 b'#coding:iso8859-15\n'
215 b'print(ascii("\xc3\xa4"))\n')
216 self.check_script_output(src, br"'\xc3\u20ac'")
217
218 def test_third_coding_line(self):
219 # Only first two lines are tested for a magic comment.
220 src = (b'#!/usr/bin/python\n'
221 b'#\n'
222 b'#coding:iso8859-15\n'
223 b'print(ascii("\xc3\xa4"))\n')
224 self.check_script_output(src, br"'\xe4'")
225
226 def test_double_coding_line(self):
227 # If the first line matches the second line is ignored.
228 src = (b'#coding:iso8859-15\n'
229 b'#coding:latin1\n'
230 b'print(ascii("\xc3\xa4"))\n')
231 self.check_script_output(src, br"'\xc3\u20ac'")
232
233 def test_double_coding_same_line(self):
234 src = (b'#coding:iso8859-15 coding:latin1\n'
235 b'print(ascii("\xc3\xa4"))\n')
236 self.check_script_output(src, br"'\xc3\u20ac'")
237
238 def test_double_coding_utf8(self):
239 src = (b'#coding:utf-8\n'
240 b'#coding:latin1\n'
241 b'print(ascii("\xc3\xa4"))\n')
242 self.check_script_output(src, br"'\xe4'")
243
244 def test_long_first_coding_line(self):
245 src = (b'#' + b' '*BUFSIZ + b'coding:iso8859-15\n'
246 b'print(ascii("\xc3\xa4"))\n')
247 self.check_script_output(src, br"'\xc3\u20ac'")
248
249 def test_long_second_coding_line(self):
250 src = (b'#!/usr/bin/python\n'
251 b'#' + b' '*BUFSIZ + b'coding:iso8859-15\n'
252 b'print(ascii("\xc3\xa4"))\n')
253 self.check_script_output(src, br"'\xc3\u20ac'")
254
255 def test_long_coding_line(self):
256 src = (b'#coding:iso-8859-15' + b' '*BUFSIZ + b'\n'
257 b'print(ascii("\xc3\xa4"))\n')
258 self.check_script_output(src, br"'\xc3\u20ac'")
259
260 def test_long_coding_name(self):
261 src = (b'#coding:iso-8859-1-' + b'x'*BUFSIZ + b'\n'
262 b'print(ascii("\xc3\xa4"))\n')
263 self.check_script_output(src, br"'\xc3\xa4'")
264
265 def test_long_first_utf8_line(self):
266 src = b'#' + b'\xc3\xa4'*(BUFSIZ//2) + b'\n'
267 self.check_script_output(src, b'')
268 src = b'# ' + b'\xc3\xa4'*(BUFSIZ//2) + b'\n'
269 self.check_script_output(src, b'')
270
271 def test_long_second_utf8_line(self):
272 src = b'\n#' + b'\xc3\xa4'*(BUFSIZ//2) + b'\n'
273 self.check_script_output(src, b'')
274 src = b'\n# ' + b'\xc3\xa4'*(BUFSIZ//2) + b'\n'
275 self.check_script_output(src, b'')
276
277 def test_first_non_utf8_coding_line(self):
278 src = (b'#coding:iso-8859-15 \xa4\n'
279 b'print(ascii("\xc3\xa4"))\n')
280 self.check_script_output(src, br"'\xc3\u20ac'")
281
282 def test_second_non_utf8_coding_line(self):
283 src = (b'#!/usr/bin/python\n'
284 b'#coding:iso-8859-15 \xa4\n'
285 b'print(ascii("\xc3\xa4"))\n')
286 self.check_script_output(src, br"'\xc3\u20ac'")
287
288 def test_first_utf8_coding_line_error(self):
289 src = (b'#coding:ascii \xc3\xa4\n'
290 b'raise RuntimeError\n')
291 self.check_script_error(src, br"(\(unicode error\) )?'ascii' codec can't decode byte")
292
293 def test_second_utf8_coding_line_error(self):
294 src = (b'#!/usr/bin/python\n'
295 b'#coding:ascii \xc3\xa4\n'
296 b'raise RuntimeError\n')
297 self.check_script_error(src, br"(\(unicode error\) )?'ascii' codec can't decode byte")
298
299 def test_utf8_bom(self):
300 src = (b'\xef\xbb\xbfprint(ascii("\xc3\xa4"))\n')
301 self.check_script_output(src, br"'\xe4'")
302
303 def test_utf8_bom_utf8_comments(self):
304 src = (b'\xef\xbb\xbf#\xc3\xa4\n'
305 b'#\xc3\xa4\n'
306 b'print(ascii("\xc3\xa4"))\n')
307 self.check_script_output(src, br"'\xe4'")
308
309 def test_utf8_bom_and_utf8_coding_line(self):
310 src = (b'\xef\xbb\xbf#coding:utf-8\n'
311 b'print(ascii("\xc3\xa4"))\n')
312 self.check_script_output(src, br"'\xe4'")
313
314 def test_utf8_bom_and_non_utf8_first_coding_line(self):
315 src = (b'\xef\xbb\xbf#coding:iso-8859-15\n'
316 b'raise RuntimeError\n')
317 self.check_script_error(src,
318 br"encoding problem: iso-8859-15 with BOM",
319 lineno=1)
320
321 def test_utf8_bom_and_non_utf8_second_coding_line(self):
322 src = (b'\xef\xbb\xbf#first\n'
323 b'#coding:iso-8859-15\n'
324 b'raise RuntimeError\n')
325 self.check_script_error(src,
326 br"encoding problem: iso-8859-15 with BOM",
327 lineno=2)
328
329 def test_non_utf8_shebang(self):
330 src = (b'#!/home/\xa4/bin/python\n'
331 b'#coding:iso-8859-15\n'
332 b'print(ascii("\xc3\xa4"))\n')
333 self.check_script_output(src, br"'\xc3\u20ac'")
334
335 def test_utf8_shebang_error(self):
336 src = (b'#!/home/\xc3\xa4/bin/python\n'
337 b'#coding:ascii\n'
338 b'raise RuntimeError\n')
339 self.check_script_error(src, br"(\(unicode error\) )?'ascii' codec can't decode byte")
340
341 def test_non_utf8_shebang_error(self):
342 src = (b'#!/home/\xa4/bin/python\n'
343 b'raise RuntimeError\n')
344 self.check_script_error(src, br"Non-UTF-8 code starting with .* on line 1",
345 lineno=1)
346
347 def test_non_utf8_second_line_error(self):
348 src = (b'#first\n'
349 b'#second\xa4\n'
350 b'raise RuntimeError\n')
351 self.check_script_error(src,
352 br"Non-UTF-8 code starting with .* on line 2",
353 lineno=2)
354
355 def test_non_utf8_third_line_error(self):
356 src = (b'#first\n'
357 b'#second\n'
358 b'#third\xa4\n'
359 b'raise RuntimeError\n')
360 self.check_script_error(src,
361 br"Non-UTF-8 code starting with .* on line 3",
362 lineno=3)
363
364 def test_utf8_bom_non_utf8_third_line_error(self):
365 src = (b'\xef\xbb\xbf#first\n'
366 b'#second\n'
367 b'#third\xa4\n'
368 b'raise RuntimeError\n')
369 self.check_script_error(src,
370 br"Non-UTF-8 code starting with .* on line 3|"
371 br"'utf-8' codec can't decode byte",
372 lineno=3)
373
374 def test_utf_8_non_utf8_third_line_error(self):
375 src = (b'#coding: utf-8\n'
376 b'#second\n'
377 b'#third\xa4\n'
378 b'raise RuntimeError\n')
379 self.check_script_error(src,
380 br"Non-UTF-8 code starting with .* on line 3|"
381 br"'utf-8' codec can't decode byte",
382 lineno=3)
383
384 def test_utf8_non_utf8_third_line_error(self):
385 src = (b'#coding: utf8\n'
386 b'#second\n'
387 b'#third\xa4\n'
388 b'raise RuntimeError\n')
389 self.check_script_error(src,
390 br"'utf-8' codec can't decode byte|"
391 br"encoding problem: utf8")
392
393 def test_crlf(self):
394 src = (b'print(ascii("""\r\n"""))\n')
395 self.check_script_output(src, br"'\n'")
396
397 def test_crcrlf(self):
398 src = (b'print(ascii("""\r\r\n"""))\n')
399 self.check_script_output(src, br"'\n\n'")
400
401 def test_crcrcrlf(self):
402 src = (b'print(ascii("""\r\r\r\n"""))\n')
403 self.check_script_output(src, br"'\n\n\n'")
404
405 def test_crcrcrlf2(self):
406 src = (b'#coding:iso-8859-1\n'
407 b'print(ascii("""\r\r\r\n"""))\n')
408 self.check_script_output(src, br"'\n\n\n'")
409
410 def test_nul_in_first_coding_line(self):
411 src = (b'#coding:iso8859-15\x00\n'
412 b'\n'
413 b'\n'
414 b'raise RuntimeError\n')
415 self.check_script_error(src, br"source code (string )?cannot contain null bytes")
416
417 def test_nul_in_second_coding_line(self):
418 src = (b'#!/usr/bin/python\n'
419 b'#coding:iso8859-15\x00\n'
420 b'\n'
421 b'raise RuntimeError\n')
422 self.check_script_error(src, br"source code (string )?cannot contain null bytes")
423
424
425 class UTF8ValidatorTest(unittest.TestCase):
426 @unittest.skipIf(not sys.platform.startswith("linux"),
427 "Too slow to run on non-Linux platforms")
428 @support.requires_resource('cpu')
429 def test_invalid_utf8(self):
430 # This is a port of test_utf8_decode_invalid_sequences in
431 # test_unicode.py to exercise the separate utf8 validator in
432 # Parser/tokenizer/helpers.c used when reading source files.
433
434 # That file is written using low-level C file I/O, so the only way to
435 # test it is to write actual files to disk.
436
437 # Each example is put inside a string at the top of the file so
438 # it's an otherwise valid Python source file. Put some newlines
439 # beforehand so we can assert that the error is reported on the
440 # correct line.
441 template = b'\n\n\n"%s"\n'
442
443 fn = TESTFN
444 self.addCleanup(unlink, fn)
445
446 def check(content):
447 with open(fn, 'wb') as fp:
448 fp.write(template % content)
449 rc, stdout, stderr = script_helper.assert_python_failure(fn)
450 # We want to assert that the python subprocess failed gracefully,
451 # not via a signal.
452 self.assertGreaterEqual(rc, 1)
453 self.assertIn(b"Non-UTF-8 code starting with", stderr)
454 self.assertIn(b"on line 4", stderr)
455
456 # continuation bytes in a sequence of 2, 3, or 4 bytes
457 continuation_bytes = [bytes([x]) for x in range(0x80, 0xC0)]
458 # start bytes of a 2-byte sequence equivalent to code points < 0x7F
459 invalid_2B_seq_start_bytes = [bytes([x]) for x in range(0xC0, 0xC2)]
460 # start bytes of a 4-byte sequence equivalent to code points > 0x10FFFF
461 invalid_4B_seq_start_bytes = [bytes([x]) for x in range(0xF5, 0xF8)]
462 invalid_start_bytes = (
463 continuation_bytes + invalid_2B_seq_start_bytes +
464 invalid_4B_seq_start_bytes + [bytes([x]) for x in range(0xF7, 0x100)]
465 )
466
467 for byte in invalid_start_bytes:
468 check(byte)
469
470 for sb in invalid_2B_seq_start_bytes:
471 for cb in continuation_bytes:
472 check(sb + cb)
473
474 for sb in invalid_4B_seq_start_bytes:
475 for cb1 in continuation_bytes[:3]:
476 for cb3 in continuation_bytes[:3]:
477 check(sb+cb1+b'\x80'+cb3)
478
479 for cb in [bytes([x]) for x in range(0x80, 0xA0)]:
480 check(b'\xE0'+cb+b'\x80')
481 check(b'\xE0'+cb+b'\xBF')
482 # surrogates
483 for cb in [bytes([x]) for x in range(0xA0, 0xC0)]:
484 check(b'\xED'+cb+b'\x80')
485 check(b'\xED'+cb+b'\xBF')
486 for cb in [bytes([x]) for x in range(0x80, 0x90)]:
487 check(b'\xF0'+cb+b'\x80\x80')
488 check(b'\xF0'+cb+b'\xBF\xBF')
489 for cb in [bytes([x]) for x in range(0x90, 0xC0)]:
490 check(b'\xF4'+cb+b'\x80\x80')
491 check(b'\xF4'+cb+b'\xBF\xBF')
492
493
494 @support.force_not_colorized_test_class
495 class BytesSourceEncodingTest(AbstractSourceEncodingTest, unittest.TestCase):
496
497 def check_script_output(self, src, expected):
498 with support.captured_stdout() as stdout:
499 exec(src)
500 out = stdout.getvalue().encode('latin1')
501 self.assertEqual(out.rstrip(), expected)
502
503 def check_script_error(self, src, expected, lineno=...):
504 with self.assertRaises(SyntaxError) as cm:
505 exec(src)
506 exc = cm.exception
507 self.assertRegex(str(exc), expected.decode())
508 if lineno is not ...:
509 self.assertEqual(exc.lineno, lineno)
510 line = src.splitlines()[lineno-1].decode(errors='replace')
511 if lineno == 1:
512 line = line.removeprefix('\ufeff')
513 self.assertEqual(line, exc.text)
514
515
516 @support.force_not_colorized_test_class
517 class FileSourceEncodingTest(AbstractSourceEncodingTest, unittest.TestCase):
518
519 def check_script_output(self, src, expected):
520 with tempfile.TemporaryDirectory() as tmpd:
521 fn = os.path.join(tmpd, 'test.py')
522 with open(fn, 'wb') as fp:
523 fp.write(src)
524 res = script_helper.assert_python_ok(fn)
525 self.assertEqual(res.out.rstrip(), expected)
526
527 def check_script_error(self, src, expected, lineno=...):
528 with tempfile.TemporaryDirectory() as tmpd:
529 fn = os.path.join(tmpd, 'test.py')
530 with open(fn, 'wb') as fp:
531 fp.write(src)
532 res = script_helper.assert_python_failure(fn)
533 err = res.err.rstrip()
534 self.assertRegex(err.splitlines()[-1], b'SyntaxError: ' + expected)
535 if lineno is not ...:
536 self.assertIn(f', line {lineno}\n'.encode(),
537 err.replace(os.linesep.encode(), b'\n'))
538 line = src.splitlines()[lineno-1].decode(errors='replace')
539 if lineno == 1:
540 line = line.removeprefix('\ufeff')
541 line = line.encode(sys.__stderr__.encoding, sys.__stderr__.errors)
542 self.assertIn(line, err)
543
544
545
546 if __name__ == "__main__":
547 unittest.main()