Debian Codegraph @

Tools/peg_generator/pegen/build.py

raw
1 import itertools
2 import logging
3 import os
4 import pathlib
5 import sys
6 import sysconfig
7 import tempfile
8 import tokenize
9 from typing import IO, Any
10
11 from pegen.c_generator import CParserGenerator
12 from pegen.grammar import Grammar
13 from pegen.grammar_parser import GeneratedParser as GrammarParser
14 from pegen.parser import Parser
15 from pegen.parser_generator import ParserGenerator
16 from pegen.python_generator import PythonParserGenerator
17 from pegen.tokenizer import Tokenizer
18
19 MOD_DIR = pathlib.Path(__file__).resolve().parent
20
21 TokenDefinitions = tuple[dict[int, str], dict[str, int], set[str]]
22 Incomplete = Any # TODO: install `types-setuptools` and remove this alias
23
24
25 def get_extra_flags(compiler_flags: str, compiler_py_flags_nodist: str) -> list[str]:
26 flags = sysconfig.get_config_var(compiler_flags)
27 py_flags_nodist = sysconfig.get_config_var(compiler_py_flags_nodist)
28 if flags is None or py_flags_nodist is None:
29 return []
30 return f"{flags} {py_flags_nodist}".split()
31
32
33 def fixup_build_ext(cmd: Incomplete) -> None:
34 """Function needed to make build_ext tests pass.
35
36 When Python was built with --enable-shared on Unix, -L. is not enough to
37 find libpython<blah>.so, because regrtest runs in a tempdir, not in the
38 source directory where the .so lives.
39
40 When Python was built with in debug mode on Windows, build_ext commands
41 need their debug attribute set, and it is not done automatically for
42 some reason.
43
44 This function handles both of these things. Example use:
45
46 cmd = build_ext(dist)
47 support.fixup_build_ext(cmd)
48 cmd.ensure_finalized()
49
50 Unlike most other Unix platforms, Mac OS X embeds absolute paths
51 to shared libraries into executables, so the fixup is not needed there.
52
53 Taken from distutils (was part of the CPython stdlib until Python 3.11)
54 """
55 if os.name == "nt":
56 cmd.debug = sys.executable.endswith("_d.exe")
57 elif sysconfig.get_config_var("Py_ENABLE_SHARED"):
58 # To further add to the shared builds fun on Unix, we can't just add
59 # library_dirs to the Extension() instance because that doesn't get
60 # plumbed through to the final compiler command.
61 runshared = sysconfig.get_config_var("RUNSHARED")
62 if runshared is None:
63 cmd.library_dirs = ["."]
64 else:
65 if sys.platform == "darwin":
66 cmd.library_dirs = []
67 else:
68 name, equals, value = runshared.partition("=")
69 cmd.library_dirs = [d for d in value.split(os.pathsep) if d]
70
71
72 def compile_c_extension(
73 generated_source_path: str,
74 build_dir: str | None = None,
75 verbose: bool = False,
76 keep_asserts: bool = True,
77 disable_optimization: bool = False,
78 library_dir: str | None = None,
79 ) -> pathlib.Path:
80 """Compile the generated source for a parser generator into an extension module.
81
82 The extension module will be generated in the same directory as the provided path
83 for the generated source, with the same basename (in addition to extension module
84 metadata). For example, for the source mydir/parser.c the generated extension
85 in a darwin system with python 3.8 will be mydir/parser.cpython-38-darwin.so.
86
87 If *build_dir* is provided, that path will be used as the temporary build directory
88 of distutils (this is useful in case you want to use a temporary directory).
89
90 If *library_dir* is provided, that path will be used as the directory for a
91 static library of the common parser sources (this is useful in case you are
92 creating multiple extensions).
93 """
94 import setuptools.command.build_ext
95 import setuptools.logging
96 from setuptools import Distribution, Extension
97 from setuptools._distutils.ccompiler import new_compiler
98 from setuptools._distutils.sysconfig import customize_compiler
99 from setuptools.modified import newer_group
100
101 if verbose:
102 setuptools.logging.set_threshold(logging.DEBUG)
103
104 source_file_path = pathlib.Path(generated_source_path)
105 extension_name = source_file_path.stem
106 extra_compile_args = get_extra_flags("CFLAGS", "PY_CFLAGS_NODIST")
107 extra_compile_args.append("-DPy_BUILD_CORE_MODULE")
108 # Define _Py_TEST_PEGEN to not call PyAST_Validate() in Parser/pegen.c
109 extra_compile_args.append("-D_Py_TEST_PEGEN")
110 if sys.platform == "win32" and sysconfig.get_config_var("Py_GIL_DISABLED"):
111 extra_compile_args.append("-DPy_GIL_DISABLED")
112 extra_link_args = get_extra_flags("LDFLAGS", "PY_LDFLAGS_NODIST")
113 if keep_asserts:
114 extra_compile_args.append("-UNDEBUG")
115 if disable_optimization:
116 if sys.platform == "win32":
117 extra_compile_args.append("/Od")
118 extra_link_args.append("/LTCG:OFF")
119 else:
120 extra_compile_args.append("-O0")
121 if sysconfig.get_config_var("GNULD") == "yes":
122 extra_link_args.append("-fno-lto")
123
124 common_sources = [
125 str(MOD_DIR.parent.parent.parent / "Python" / "Python-ast.c"),
126 str(MOD_DIR.parent.parent.parent / "Python" / "asdl.c"),
127 str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "lexer.c"),
128 str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "state.c"),
129 str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "buffer.c"),
130 str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer" / "string_tokenizer.c"),
131 str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer" / "file_tokenizer.c"),
132 str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer" / "utf8_tokenizer.c"),
133 str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer" / "readline_tokenizer.c"),
134 str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer" / "helpers.c"),
135 str(MOD_DIR.parent.parent.parent / "Parser" / "pegen.c"),
136 str(MOD_DIR.parent.parent.parent / "Parser" / "pegen_errors.c"),
137 str(MOD_DIR.parent.parent.parent / "Parser" / "action_helpers.c"),
138 str(MOD_DIR.parent.parent.parent / "Parser" / "string_parser.c"),
139 str(MOD_DIR.parent / "peg_extension" / "peg_extension.c"),
140 ]
141 include_dirs = [
142 str(MOD_DIR.parent.parent.parent / "Include" / "internal"),
143 str(MOD_DIR.parent.parent.parent / "Include" / "internal" / "mimalloc"),
144 str(MOD_DIR.parent.parent.parent / "Parser"),
145 str(MOD_DIR.parent.parent.parent / "Parser" / "lexer"),
146 str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer"),
147 ]
148 if sys.platform == "win32":
149 # HACK: The location of pyconfig.h has moved within our build, and
150 # setuptools hasn't updated for it yet. So add the path manually for now
151 include_dirs.append(pathlib.Path(sysconfig.get_config_h_filename()).parent)
152 extension = Extension(
153 extension_name,
154 sources=[generated_source_path],
155 extra_compile_args=extra_compile_args,
156 extra_link_args=extra_link_args,
157 )
158 dist = Distribution({"name": extension_name, "ext_modules": [extension]})
159 cmd = dist.get_command_obj("build_ext")
160 assert isinstance(cmd, setuptools.command.build_ext.build_ext)
161 fixup_build_ext(cmd)
162 cmd.build_lib = str(source_file_path.parent)
163 cmd.include_dirs = include_dirs
164 if build_dir:
165 cmd.build_temp = build_dir
166 cmd.ensure_finalized()
167
168 compiler = new_compiler()
169 customize_compiler(compiler)
170 compiler.set_include_dirs(cmd.include_dirs)
171 compiler.set_library_dirs(cmd.library_dirs)
172 # build static lib
173 if library_dir:
174 library_filename = compiler.library_filename(extension_name, output_dir=library_dir)
175 if newer_group(common_sources, library_filename, "newer"):
176 if sys.platform == "win32":
177 assert compiler.static_lib_format
178 pdb = compiler.static_lib_format % (extension_name, ".pdb")
179 compile_opts = [f"/Fd{library_dir}\\{pdb}"]
180 compile_opts.extend(extra_compile_args)
181 else:
182 compile_opts = extra_compile_args
183 objects = compiler.compile(
184 common_sources,
185 output_dir=library_dir,
186 debug=cmd.debug,
187 extra_postargs=compile_opts,
188 )
189 compiler.create_static_lib(
190 objects, extension_name, output_dir=library_dir, debug=cmd.debug
191 )
192 if sys.platform == "win32":
193 compiler.add_library_dir(library_dir)
194 extension.libraries = [extension_name]
195 elif sys.platform == "darwin":
196 compiler.set_link_objects(
197 [
198 "-Wl,-force_load",
199 library_filename,
200 ]
201 )
202 else:
203 compiler.set_link_objects(
204 [
205 "-Wl,--whole-archive",
206 library_filename,
207 "-Wl,--no-whole-archive",
208 ]
209 )
210 else:
211 extension.sources[0:0] = common_sources
212
213 # Compile the source code to object files.
214 ext_path = cmd.get_ext_fullpath(extension_name)
215 if newer_group(extension.sources, ext_path, "newer"):
216 objects = compiler.compile(
217 extension.sources,
218 output_dir=cmd.build_temp,
219 debug=cmd.debug,
220 extra_postargs=extra_compile_args,
221 )
222 else:
223 objects = compiler.object_filenames(extension.sources, output_dir=cmd.build_temp)
224 # The cmd.get_libraries() call needs a valid compiler attribute or we will
225 # get an incorrect library name on the free-threaded Windows build.
226 cmd.compiler = compiler
227 # Now link the object files together into a "shared object"
228 compiler.link_shared_object(
229 objects,
230 ext_path,
231 libraries=cmd.get_libraries(extension),
232 extra_postargs=extra_link_args,
233 export_symbols=cmd.get_export_symbols(extension), # type: ignore[no-untyped-call]
234 debug=cmd.debug,
235 build_temp=cmd.build_temp,
236 )
237
238 return pathlib.Path(ext_path)
239
240
241 def build_parser(
242 grammar_file: str, verbose_tokenizer: bool = False, verbose_parser: bool = False
243 ) -> tuple[Grammar, Parser, Tokenizer]:
244 with open(grammar_file) as file:
245 tokenizer = Tokenizer(tokenize.generate_tokens(file.readline), verbose=verbose_tokenizer)
246 parser = GrammarParser(tokenizer, verbose=verbose_parser)
247 grammar = parser.start()
248
249 if not grammar:
250 raise parser.make_syntax_error(grammar_file)
251
252 return grammar, parser, tokenizer
253
254
255 def generate_token_definitions(tokens: IO[str]) -> TokenDefinitions:
256 all_tokens = {}
257 exact_tokens = {}
258 non_exact_tokens = set()
259 numbers = itertools.count(0)
260
261 for line in tokens:
262 line = line.strip()
263
264 if not line or line.startswith("#"):
265 continue
266
267 pieces = line.split()
268 index = next(numbers)
269
270 if len(pieces) == 1:
271 (token,) = pieces
272 non_exact_tokens.add(token)
273 all_tokens[index] = token
274 elif len(pieces) == 2:
275 token, op = pieces
276 exact_tokens[op.strip("'")] = index
277 all_tokens[index] = token
278 else:
279 raise ValueError(f"Unexpected line found in Tokens file: {line}")
280
281 return all_tokens, exact_tokens, non_exact_tokens
282
283
284 def build_c_generator(
285 grammar: Grammar,
286 grammar_file: str,
287 tokens_file: str,
288 output_file: str,
289 compile_extension: bool = False,
290 verbose_c_extension: bool = False,
291 keep_asserts_in_extension: bool = True,
292 skip_actions: bool = False,
293 ) -> ParserGenerator:
294 with open(tokens_file) as tok_file:
295 all_tokens, exact_tok, non_exact_tok = generate_token_definitions(tok_file)
296 with open(output_file, "w") as file:
297 gen: ParserGenerator = CParserGenerator(
298 grammar, all_tokens, exact_tok, non_exact_tok, file, skip_actions=skip_actions
299 )
300 gen.generate(grammar_file)
301
302 if compile_extension:
303 with tempfile.TemporaryDirectory() as build_dir:
304 compile_c_extension(
305 output_file,
306 build_dir=build_dir,
307 verbose=verbose_c_extension,
308 keep_asserts=keep_asserts_in_extension,
309 )
310 return gen
311
312
313 def build_python_generator(
314 grammar: Grammar,
315 grammar_file: str,
316 output_file: str,
317 skip_actions: bool = False,
318 ) -> ParserGenerator:
319 with open(output_file, "w") as file:
320 gen: ParserGenerator = PythonParserGenerator(grammar, file) # TODO: skip_actions
321 gen.generate(grammar_file)
322 return gen
323
324
325 def build_c_parser_and_generator(
326 grammar_file: str,
327 tokens_file: str,
328 output_file: str,
329 compile_extension: bool = False,
330 verbose_tokenizer: bool = False,
331 verbose_parser: bool = False,
332 verbose_c_extension: bool = False,
333 keep_asserts_in_extension: bool = True,
334 skip_actions: bool = False,
335 ) -> tuple[Grammar, Parser, Tokenizer, ParserGenerator]:
336 """Generate rules, C parser, tokenizer, parser generator for a given grammar
337
338 Args:
339 grammar_file (string): Path for the grammar file
340 tokens_file (string): Path for the tokens file
341 output_file (string): Path for the output file
342 compile_extension (bool, optional): Whether to compile the C extension.
343 Defaults to False.
344 verbose_tokenizer (bool, optional): Whether to display additional output
345 when generating the tokenizer. Defaults to False.
346 verbose_parser (bool, optional): Whether to display additional output
347 when generating the parser. Defaults to False.
348 verbose_c_extension (bool, optional): Whether to display additional
349 output when compiling the C extension . Defaults to False.
350 keep_asserts_in_extension (bool, optional): Whether to keep the assert statements
351 when compiling the extension module. Defaults to True.
352 skip_actions (bool, optional): Whether to pretend no rule has any actions.
353 """
354 grammar, parser, tokenizer = build_parser(grammar_file, verbose_tokenizer, verbose_parser)
355 gen = build_c_generator(
356 grammar,
357 grammar_file,
358 tokens_file,
359 output_file,
360 compile_extension,
361 verbose_c_extension,
362 keep_asserts_in_extension,
363 skip_actions=skip_actions,
364 )
365
366 return grammar, parser, tokenizer, gen
367
368
369 def build_python_parser_and_generator(
370 grammar_file: str,
371 output_file: str,
372 verbose_tokenizer: bool = False,
373 verbose_parser: bool = False,
374 skip_actions: bool = False,
375 ) -> tuple[Grammar, Parser, Tokenizer, ParserGenerator]:
376 """Generate rules, python parser, tokenizer, parser generator for a given grammar
377
378 Args:
379 grammar_file (string): Path for the grammar file
380 output_file (string): Path for the output file
381 verbose_tokenizer (bool, optional): Whether to display additional output
382 when generating the tokenizer. Defaults to False.
383 verbose_parser (bool, optional): Whether to display additional output
384 when generating the parser. Defaults to False.
385 skip_actions (bool, optional): Whether to pretend no rule has any actions.
386 """
387 grammar, parser, tokenizer = build_parser(grammar_file, verbose_tokenizer, verbose_parser)
388 gen = build_python_generator(
389 grammar,
390 grammar_file,
391 output_file,
392 skip_actions=skip_actions,
393 )
394 return grammar, parser, tokenizer, gen