Debian Codegraph @

Lib/test/test_peg_generator/test_c_parser.py

raw
1 import contextlib
2 import subprocess
3 import sysconfig
4 import textwrap
5 import unittest
6 import os
7 import shutil
8 import tempfile
9 from pathlib import Path
10
11 from test import test_tools
12 from test import support
13 from test.support import os_helper, import_helper
14 from test.support.script_helper import assert_python_ok
15
16 if support.check_cflags_pgo():
17 raise unittest.SkipTest("peg_generator test disabled under PGO build")
18
19 test_tools.skip_if_missing("peg_generator")
20 with test_tools.imports_under_tool("peg_generator"):
21 from pegen.grammar_parser import GeneratedParser as GrammarParser
22 from pegen.testutil import (
23 parse_string,
24 generate_parser_c_extension,
25 generate_c_parser_source,
26 )
27
28
29 TEST_TEMPLATE = """
30 tmp_dir = {extension_path!r}
31
32 import ast
33 import traceback
34 import sys
35 import unittest
36
37 from test import test_tools
38 with test_tools.imports_under_tool("peg_generator"):
39 from pegen.ast_dump import ast_dump
40
41 sys.path.insert(0, tmp_dir)
42 import parse
43
44 class Tests(unittest.TestCase):
45
46 def check_input_strings_for_grammar(
47 self,
48 valid_cases = (),
49 invalid_cases = (),
50 ):
51 if valid_cases:
52 for case in valid_cases:
53 parse.parse_string(case, mode=0)
54
55 if invalid_cases:
56 for case in invalid_cases:
57 with self.assertRaises(SyntaxError):
58 parse.parse_string(case, mode=0)
59
60 def verify_ast_generation(self, stmt):
61 expected_ast = ast.parse(stmt)
62 actual_ast = parse.parse_string(stmt, mode=1)
63 self.assertEqual(ast_dump(expected_ast), ast_dump(actual_ast))
64
65 def test_parse(self):
66 {test_source}
67
68 unittest.main()
69 """
70
71
72 @support.requires_subprocess()
73 class TestCParser(unittest.TestCase):
74
75 _has_run = False
76
77 @classmethod
78 def setUpClass(cls):
79 if cls._has_run:
80 # Since gh-104798 (Use setuptools in peg-generator and reenable
81 # tests), this test case has been producing ref leaks. Initial
82 # debugging points to bug(s) in setuptools and/or importlib.
83 # See gh-105063 for more info.
84 raise unittest.SkipTest("gh-105063: can not rerun because of ref. leaks")
85 cls._has_run = True
86
87 # When running under regtest, a separate tempdir is used
88 # as the current directory and watched for left-overs.
89 # Reusing that as the base for temporary directories
90 # ensures everything is cleaned up properly and
91 # cleans up afterwards if not (with warnings).
92 cls.tmp_base = os.getcwd()
93 if os.path.samefile(cls.tmp_base, os_helper.SAVEDCWD):
94 cls.tmp_base = None
95 # Create a directory for the reuseable static library part of
96 # the pegen extension build process. This greatly reduces the
97 # runtime overhead of spawning compiler processes.
98 cls.library_dir = tempfile.mkdtemp(dir=cls.tmp_base)
99 cls.addClassCleanup(shutil.rmtree, cls.library_dir)
100
101 with contextlib.ExitStack() as stack:
102 python_exe = stack.enter_context(support.setup_venv_with_pip_setuptools("venv"))
103 sitepackages = subprocess.check_output(
104 [python_exe, "-c", "import sysconfig; print(sysconfig.get_path('platlib'))"],
105 text=True,
106 ).strip()
107 stack.enter_context(import_helper.DirsOnSysPath(sitepackages))
108 cls.addClassCleanup(stack.pop_all().close)
109
110 @support.requires_venv_with_pip()
111 def setUp(self):
112 self._backup_config_vars = dict(sysconfig._CONFIG_VARS)
113 cmd = support.missing_compiler_executable()
114 if cmd is not None:
115 self.skipTest("The %r command is not found" % cmd)
116 self.old_cwd = os.getcwd()
117 self.tmp_path = tempfile.mkdtemp(dir=self.tmp_base)
118 self.enterContext(os_helper.change_cwd(self.tmp_path))
119
120 def tearDown(self):
121 os.chdir(self.old_cwd)
122 shutil.rmtree(self.tmp_path)
123 sysconfig._CONFIG_VARS.clear()
124 sysconfig._CONFIG_VARS.update(self._backup_config_vars)
125
126 def build_extension(self, grammar_source):
127 grammar = parse_string(grammar_source, GrammarParser)
128 # Because setUp() already changes the current directory to the
129 # temporary path, use a relative path here to prevent excessive
130 # path lengths when compiling.
131 generate_parser_c_extension(grammar, Path('.'), library_dir=self.library_dir)
132
133 def run_test(self, grammar_source, test_source):
134 self.build_extension(grammar_source)
135 test_source = textwrap.indent(textwrap.dedent(test_source), 8 * " ")
136 assert_python_ok(
137 "-c",
138 TEST_TEMPLATE.format(extension_path=self.tmp_path, test_source=test_source),
139 )
140
141 def test_c_parser(self) -> None:
142 grammar_source = """
143 start[mod_ty]: a[asdl_stmt_seq*]=stmt* $ { _PyAST_Module(a, NULL, p->arena) }
144 stmt[stmt_ty]: a=expr_stmt { a }
145 expr_stmt[stmt_ty]: a=expression NEWLINE { _PyAST_Expr(a, EXTRA) }
146 expression[expr_ty]: ( l=expression '+' r=term { _PyAST_BinOp(l, Add, r, EXTRA) }
147 | l=expression '-' r=term { _PyAST_BinOp(l, Sub, r, EXTRA) }
148 | t=term { t }
149 )
150 term[expr_ty]: ( l=term '*' r=factor { _PyAST_BinOp(l, Mult, r, EXTRA) }
151 | l=term '/' r=factor { _PyAST_BinOp(l, Div, r, EXTRA) }
152 | f=factor { f }
153 )
154 factor[expr_ty]: ('(' e=expression ')' { e }
155 | a=atom { a }
156 )
157 atom[expr_ty]: ( n=NAME { n }
158 | n=NUMBER { n }
159 | s=STRING { s }
160 )
161 """
162 test_source = """
163 expressions = [
164 "4+5",
165 "4-5",
166 "4*5",
167 "1+4*5",
168 "1+4/5",
169 "(1+1) + (1+1)",
170 "(1+1) - (1+1)",
171 "(1+1) * (1+1)",
172 "(1+1) / (1+1)",
173 ]
174
175 for expr in expressions:
176 the_ast = parse.parse_string(expr, mode=1)
177 expected_ast = ast.parse(expr)
178 self.assertEqual(ast_dump(the_ast), ast_dump(expected_ast))
179 """
180 self.run_test(grammar_source, test_source)
181
182 def test_lookahead(self) -> None:
183 grammar_source = """
184 start: NAME &NAME expr NEWLINE? ENDMARKER
185 expr: NAME | NUMBER
186 """
187 test_source = """
188 valid_cases = ["foo bar"]
189 invalid_cases = ["foo 34"]
190 self.check_input_strings_for_grammar(valid_cases, invalid_cases)
191 """
192 self.run_test(grammar_source, test_source)
193
194 def test_negative_lookahead(self) -> None:
195 grammar_source = """
196 start: NAME !NAME expr NEWLINE? ENDMARKER
197 expr: NAME | NUMBER
198 """
199 test_source = """
200 valid_cases = ["foo 34"]
201 invalid_cases = ["foo bar"]
202 self.check_input_strings_for_grammar(valid_cases, invalid_cases)
203 """
204 self.run_test(grammar_source, test_source)
205
206 def test_cut(self) -> None:
207 grammar_source = """
208 start: X ~ Y Z | X Q S
209 X: 'x'
210 Y: 'y'
211 Z: 'z'
212 Q: 'q'
213 S: 's'
214 """
215 test_source = """
216 valid_cases = ["x y z"]
217 invalid_cases = ["x q s"]
218 self.check_input_strings_for_grammar(valid_cases, invalid_cases)
219 """
220 self.run_test(grammar_source, test_source)
221
222 def test_gather(self) -> None:
223 grammar_source = """
224 start: ';'.pass_stmt+ NEWLINE
225 pass_stmt: 'pass'
226 """
227 test_source = """
228 valid_cases = ["pass", "pass; pass"]
229 invalid_cases = ["pass;", "pass; pass;"]
230 self.check_input_strings_for_grammar(valid_cases, invalid_cases)
231 """
232 self.run_test(grammar_source, test_source)
233
234 def test_left_recursion(self) -> None:
235 grammar_source = """
236 start: expr NEWLINE
237 expr: ('-' term | expr '+' term | term)
238 term: NUMBER
239 """
240 test_source = """
241 valid_cases = ["-34", "34", "34 + 12", "1 + 1 + 2 + 3"]
242 self.check_input_strings_for_grammar(valid_cases)
243 """
244 self.run_test(grammar_source, test_source)
245
246 def test_advanced_left_recursive(self) -> None:
247 grammar_source = """
248 start: NUMBER | sign start
249 sign: ['-']
250 """
251 test_source = """
252 valid_cases = ["23", "-34"]
253 self.check_input_strings_for_grammar(valid_cases)
254 """
255 self.run_test(grammar_source, test_source)
256
257 def test_mutually_left_recursive(self) -> None:
258 grammar_source = """
259 start: foo 'E'
260 foo: bar 'A' | 'B'
261 bar: foo 'C' | 'D'
262 """
263 test_source = """
264 valid_cases = ["B E", "D A C A E"]
265 self.check_input_strings_for_grammar(valid_cases)
266 """
267 self.run_test(grammar_source, test_source)
268
269 def test_nasty_mutually_left_recursive(self) -> None:
270 grammar_source = """
271 start: target '='
272 target: maybe '+' | NAME
273 maybe: maybe '-' | target
274 """
275 test_source = """
276 valid_cases = ["x ="]
277 invalid_cases = ["x - + ="]
278 self.check_input_strings_for_grammar(valid_cases, invalid_cases)
279 """
280 self.run_test(grammar_source, test_source)
281
282 def test_return_stmt_noexpr_action(self) -> None:
283 grammar_source = """
284 start[mod_ty]: a=[statements] ENDMARKER { _PyAST_Module(a, NULL, p->arena) }
285 statements[asdl_stmt_seq*]: a[asdl_stmt_seq*]=statement+ { a }
286 statement[stmt_ty]: simple_stmt
287 simple_stmt[stmt_ty]: small_stmt
288 small_stmt[stmt_ty]: return_stmt
289 return_stmt[stmt_ty]: a='return' NEWLINE { _PyAST_Return(NULL, EXTRA) }
290 """
291 test_source = """
292 stmt = "return"
293 self.verify_ast_generation(stmt)
294 """
295 self.run_test(grammar_source, test_source)
296
297 def test_gather_action_ast(self) -> None:
298 grammar_source = """
299 start[mod_ty]: a[asdl_stmt_seq*]=';'.pass_stmt+ NEWLINE ENDMARKER { _PyAST_Module(a, NULL, p->arena) }
300 pass_stmt[stmt_ty]: a='pass' { _PyAST_Pass(EXTRA)}
301 """
302 test_source = """
303 stmt = "pass; pass"
304 self.verify_ast_generation(stmt)
305 """
306 self.run_test(grammar_source, test_source)
307
308 def test_pass_stmt_action(self) -> None:
309 grammar_source = """
310 start[mod_ty]: a=[statements] ENDMARKER { _PyAST_Module(a, NULL, p->arena) }
311 statements[asdl_stmt_seq*]: a[asdl_stmt_seq*]=statement+ { a }
312 statement[stmt_ty]: simple_stmt
313 simple_stmt[stmt_ty]: small_stmt
314 small_stmt[stmt_ty]: pass_stmt
315 pass_stmt[stmt_ty]: a='pass' NEWLINE { _PyAST_Pass(EXTRA) }
316 """
317 test_source = """
318 stmt = "pass"
319 self.verify_ast_generation(stmt)
320 """
321 self.run_test(grammar_source, test_source)
322
323 def test_if_stmt_action(self) -> None:
324 grammar_source = """
325 start[mod_ty]: a=[statements] ENDMARKER { _PyAST_Module(a, NULL, p->arena) }
326 statements[asdl_stmt_seq*]: a=statement+ { (asdl_stmt_seq*)_PyPegen_seq_flatten(p, a) }
327 statement[asdl_stmt_seq*]: a=compound_stmt { (asdl_stmt_seq*)_PyPegen_singleton_seq(p, a) } | simple_stmt
328
329 simple_stmt[asdl_stmt_seq*]: a=small_stmt b=further_small_stmt* [';'] NEWLINE {
330 (asdl_stmt_seq*)_PyPegen_seq_insert_in_front(p, a, b) }
331 further_small_stmt[stmt_ty]: ';' a=small_stmt { a }
332
333 block: simple_stmt | NEWLINE INDENT a=statements DEDENT { a }
334
335 compound_stmt: if_stmt
336
337 if_stmt: 'if' a=full_expression ':' b=block { _PyAST_If(a, b, NULL, EXTRA) }
338
339 small_stmt[stmt_ty]: pass_stmt
340
341 pass_stmt[stmt_ty]: a='pass' { _PyAST_Pass(EXTRA) }
342
343 full_expression: NAME
344 """
345 test_source = """
346 stmt = "pass"
347 self.verify_ast_generation(stmt)
348 """
349 self.run_test(grammar_source, test_source)
350
351 def test_same_name_different_types(self) -> None:
352 grammar_source = """
353 start[mod_ty]: a[asdl_stmt_seq*]=import_from+ NEWLINE ENDMARKER { _PyAST_Module(a, NULL, p->arena)}
354 import_from[stmt_ty]: ( a='from' !'import' c=simple_name 'import' d=import_as_names_from {
355 _PyAST_ImportFrom(c->v.Name.id, d, 0, EXTRA) }
356 | a='from' '.' 'import' c=import_as_names_from {
357 _PyAST_ImportFrom(NULL, c, 1, EXTRA) }
358 )
359 simple_name[expr_ty]: NAME
360 import_as_names_from[asdl_alias_seq*]: a[asdl_alias_seq*]=','.import_as_name_from+ { a }
361 import_as_name_from[alias_ty]: a=NAME 'as' b=NAME { _PyAST_alias(((expr_ty) a)->v.Name.id, ((expr_ty) b)->v.Name.id, EXTRA) }
362 """
363 test_source = """
364 for stmt in ("from a import b as c", "from . import a as b"):
365 expected_ast = ast.parse(stmt)
366 actual_ast = parse.parse_string(stmt, mode=1)
367 self.assertEqual(ast_dump(expected_ast), ast_dump(actual_ast))
368 """
369 self.run_test(grammar_source, test_source)
370
371 def test_with_stmt_with_paren(self) -> None:
372 grammar_source = """
373 start[mod_ty]: a=[statements] ENDMARKER { _PyAST_Module(a, NULL, p->arena) }
374 statements[asdl_stmt_seq*]: a=statement+ { (asdl_stmt_seq*)_PyPegen_seq_flatten(p, a) }
375 statement[asdl_stmt_seq*]: a=compound_stmt { (asdl_stmt_seq*)_PyPegen_singleton_seq(p, a) }
376 compound_stmt[stmt_ty]: with_stmt
377 with_stmt[stmt_ty]: (
378 a='with' '(' b[asdl_withitem_seq*]=','.with_item+ ')' ':' c=block {
379 _PyAST_With(b, (asdl_stmt_seq*) _PyPegen_singleton_seq(p, c), NULL, EXTRA) }
380 )
381 with_item[withitem_ty]: (
382 e=NAME o=['as' t=NAME { t }] { _PyAST_withitem(e, _PyPegen_set_expr_context(p, o, Store), p->arena) }
383 )
384 block[stmt_ty]: a=pass_stmt NEWLINE { a } | NEWLINE INDENT a=pass_stmt DEDENT { a }
385 pass_stmt[stmt_ty]: a='pass' { _PyAST_Pass(EXTRA) }
386 """
387 test_source = """
388 stmt = "with (\\n a as b,\\n c as d\\n): pass"
389 the_ast = parse.parse_string(stmt, mode=1)
390 self.assertStartsWith(ast_dump(the_ast),
391 "Module(body=[With(items=[withitem(context_expr=Name(id='a', ctx=Load()), optional_vars=Name(id='b', ctx=Store())), "
392 "withitem(context_expr=Name(id='c', ctx=Load()), optional_vars=Name(id='d', ctx=Store()))]"
393 )
394 """
395 self.run_test(grammar_source, test_source)
396
397 def test_ternary_operator(self) -> None:
398 grammar_source = """
399 start[mod_ty]: a=expr ENDMARKER { _PyAST_Module(a, NULL, p->arena) }
400 expr[asdl_stmt_seq*]: a=listcomp NEWLINE { (asdl_stmt_seq*)_PyPegen_singleton_seq(p, _PyAST_Expr(a, EXTRA)) }
401 listcomp[expr_ty]: (
402 a='[' b=NAME c=for_if_clauses d=']' { _PyAST_ListComp(b, c, EXTRA) }
403 )
404 for_if_clauses[asdl_comprehension_seq*]: (
405 a[asdl_comprehension_seq*]=(y=['async'] 'for' a=NAME 'in' b=NAME c[asdl_expr_seq*]=('if' z=NAME { z })*
406 { _PyAST_comprehension(_PyAST_Name(((expr_ty) a)->v.Name.id, Store, EXTRA), b, c, (y == NULL) ? 0 : 1, p->arena) })+ { a }
407 )
408 """
409 test_source = """
410 stmt = "[i for i in a if b]"
411 self.verify_ast_generation(stmt)
412 """
413 self.run_test(grammar_source, test_source)
414
415 def test_syntax_error_for_string(self) -> None:
416 grammar_source = """
417 start: expr+ NEWLINE? ENDMARKER
418 expr: NAME
419 """
420 test_source = r"""
421 for text in ("a b 42 b a", "\u540d \u540d 42 \u540d \u540d"):
422 try:
423 parse.parse_string(text, mode=0)
424 except SyntaxError as e:
425 tb = traceback.format_exc()
426 self.assertTrue('File "<string>", line 1' in tb)
427 self.assertTrue(f"SyntaxError: invalid syntax" in tb)
428 """
429 self.run_test(grammar_source, test_source)
430
431 def test_headers_and_trailer(self) -> None:
432 grammar_source = """
433 @header 'SOME HEADER'
434 @subheader 'SOME SUBHEADER'
435 @trailer 'SOME TRAILER'
436 start: expr+ NEWLINE? ENDMARKER
437 expr: x=NAME
438 """
439 grammar = parse_string(grammar_source, GrammarParser)
440 parser_source = generate_c_parser_source(grammar)
441
442 self.assertTrue("SOME HEADER" in parser_source)
443 self.assertTrue("SOME SUBHEADER" in parser_source)
444 self.assertTrue("SOME TRAILER" in parser_source)
445
446 def test_error_in_rules(self) -> None:
447 grammar_source = """
448 start: expr+ NEWLINE? ENDMARKER
449 expr: NAME {PyTuple_New(-1)}
450 """
451 # PyTuple_New raises SystemError if an invalid argument was passed.
452 test_source = """
453 with self.assertRaises(SystemError):
454 parse.parse_string("a", mode=0)
455 """
456 self.run_test(grammar_source, test_source)
457
458 def test_no_soft_keywords(self) -> None:
459 grammar_source = """
460 start: expr+ NEWLINE? ENDMARKER
461 expr: 'foo'
462 """
463 grammar = parse_string(grammar_source, GrammarParser)
464 parser_source = generate_c_parser_source(grammar)
465 assert "expect_soft_keyword" not in parser_source
466
467 def test_soft_keywords(self) -> None:
468 grammar_source = """
469 start: expr+ NEWLINE? ENDMARKER
470 expr: "foo"
471 """
472 grammar = parse_string(grammar_source, GrammarParser)
473 parser_source = generate_c_parser_source(grammar)
474 assert "expect_soft_keyword" in parser_source
475
476 def test_soft_keywords_parse(self) -> None:
477 grammar_source = """
478 start: "if" expr '+' expr NEWLINE
479 expr: NAME
480 """
481 test_source = """
482 valid_cases = ["if if + if"]
483 invalid_cases = ["if if"]
484 self.check_input_strings_for_grammar(valid_cases, invalid_cases)
485 """
486 self.run_test(grammar_source, test_source)
487
488 def test_soft_keywords_lookahead(self) -> None:
489 grammar_source = """
490 start: &"if" "if" expr '+' expr NEWLINE
491 expr: NAME
492 """
493 test_source = """
494 valid_cases = ["if if + if"]
495 invalid_cases = ["if if"]
496 self.check_input_strings_for_grammar(valid_cases, invalid_cases)
497 """
498 self.run_test(grammar_source, test_source)
499
500 def test_forced(self) -> None:
501 grammar_source = """
502 start: NAME &&':' | NAME
503 """
504 test_source = """
505 self.assertEqual(parse.parse_string("number :", mode=0), None)
506 with self.assertRaises(SyntaxError) as e:
507 parse.parse_string("a", mode=0)
508 self.assertIn("expected ':'", str(e.exception))
509 """
510 self.run_test(grammar_source, test_source)
511
512 def test_forced_with_group(self) -> None:
513 grammar_source = """
514 start: NAME &&(':' | ';') | NAME
515 """
516 test_source = """
517 self.assertEqual(parse.parse_string("number :", mode=0), None)
518 self.assertEqual(parse.parse_string("number ;", mode=0), None)
519 with self.assertRaises(SyntaxError) as e:
520 parse.parse_string("a", mode=0)
521 self.assertIn("expected (':' | ';')", e.exception.args[0])
522 """
523 self.run_test(grammar_source, test_source)