Debian Codegraph @

Tools/peg_generator/pegen/build.py

raw
1 import itertools
2 import logging
3 import os
4 import pathlib
5 import sys
6 import sysconfig
7 import tempfile
8 import tokenize
9 from typing import IO, Any
10
11 from pegen.c_generator import CParserGenerator
12 from pegen.grammar import Grammar
13 from pegen.grammar_parser import GeneratedParser as GrammarParser
14 from pegen.parser import Parser
15 from pegen.parser_generator import ParserGenerator
16 from pegen.python_generator import PythonParserGenerator
17 from pegen.tokenizer import Tokenizer
18
19 MOD_DIR = pathlib.Path(__file__).resolve().parent
20
21 TokenDefinitions = tuple[dict[int, str], dict[str, int], set[str]]
22 Incomplete = Any # TODO: install `types-setuptools` and remove this alias
23
24
25 def get_extra_flags(compiler_flags: str, compiler_py_flags_nodist: str) -> list[str]:
26 flags = sysconfig.get_config_var(compiler_flags)
27 py_flags_nodist = sysconfig.get_config_var(compiler_py_flags_nodist)
28 if flags is None or py_flags_nodist is None:
29 return []
30 return f"{flags} {py_flags_nodist}".split()
31
32
33 def fixup_build_ext(cmd: Incomplete) -> None:
34 """Function needed to make build_ext tests pass.
35
36 When Python was built with --enable-shared on Unix, -L. is not enough to
37 find libpython<blah>.so, because regrtest runs in a tempdir, not in the
38 source directory where the .so lives.
39
40 When Python was built with in debug mode on Windows, build_ext commands
41 need their debug attribute set, and it is not done automatically for
42 some reason.
43
44 This function handles both of these things. Example use:
45
46 cmd = build_ext(dist)
47 support.fixup_build_ext(cmd)
48 cmd.ensure_finalized()
49
50 Unlike most other Unix platforms, Mac OS X embeds absolute paths
51 to shared libraries into executables, so the fixup is not needed there.
52
53 Taken from distutils (was part of the CPython stdlib until Python 3.11)
54 """
55 if os.name == "nt":
56 cmd.debug = sys.executable.endswith("_d.exe")
57 elif sysconfig.get_config_var("Py_ENABLE_SHARED"):
58 # To further add to the shared builds fun on Unix, we can't just add
59 # library_dirs to the Extension() instance because that doesn't get
60 # plumbed through to the final compiler command.
61 runshared = sysconfig.get_config_var("RUNSHARED")
62 if runshared is None:
63 cmd.library_dirs = ["."]
64 else:
65 if sys.platform == "darwin":
66 cmd.library_dirs = []
67 else:
68 name, equals, value = runshared.partition("=")
69 cmd.library_dirs = [d for d in value.split(os.pathsep) if d]
70
71
72 def compile_c_extension(
73 generated_source_path: str,
74 build_dir: str | None = None,
75 verbose: bool = False,
76 keep_asserts: bool = True,
77 disable_optimization: bool = False,
78 library_dir: str | None = None,
79 ) -> pathlib.Path:
80 """Compile the generated source for a parser generator into an extension module.
81
82 The extension module will be generated in the same directory as the provided path
83 for the generated source, with the same basename (in addition to extension module
84 metadata). For example, for the source mydir/parser.c the generated extension
85 in a darwin system with python 3.8 will be mydir/parser.cpython-38-darwin.so.
86
87 If *build_dir* is provided, that path will be used as the temporary build directory
88 of distutils (this is useful in case you want to use a temporary directory).
89
90 If *library_dir* is provided, that path will be used as the directory for a
91 static library of the common parser sources (this is useful in case you are
92 creating multiple extensions).
93 """
94 import setuptools.command.build_ext
95 import setuptools.logging
96 from setuptools import Distribution, Extension
97 try:
98 from setuptools.modified import newer_group
99 except ImportError:
100 from setuptools._distutils.dep_util import newer_group
101 from setuptools._distutils.ccompiler import new_compiler
102 from setuptools._distutils.sysconfig import customize_compiler
103
104 if verbose:
105 setuptools.logging.set_threshold(logging.DEBUG)
106
107 source_file_path = pathlib.Path(generated_source_path)
108 extension_name = source_file_path.stem
109 extra_compile_args = get_extra_flags("CFLAGS", "PY_CFLAGS_NODIST")
110 extra_compile_args.append("-DPy_BUILD_CORE_MODULE")
111 # Define _Py_TEST_PEGEN to not call PyAST_Validate() in Parser/pegen.c
112 extra_compile_args.append("-D_Py_TEST_PEGEN")
113 extra_link_args = get_extra_flags("LDFLAGS", "PY_LDFLAGS_NODIST")
114 if keep_asserts:
115 extra_compile_args.append("-UNDEBUG")
116 if disable_optimization:
117 if sys.platform == "win32":
118 extra_compile_args.append("/Od")
119 extra_link_args.append("/LTCG:OFF")
120 else:
121 extra_compile_args.append("-O0")
122 if sysconfig.get_config_var("GNULD") == "yes":
123 extra_link_args.append("-fno-lto")
124
125 common_sources = [
126 str(MOD_DIR.parent.parent.parent / "Python" / "Python-ast.c"),
127 str(MOD_DIR.parent.parent.parent / "Python" / "asdl.c"),
128 str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "lexer.c"),
129 str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "state.c"),
130 str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "buffer.c"),
131 str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer" / "string_tokenizer.c"),
132 str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer" / "file_tokenizer.c"),
133 str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer" / "utf8_tokenizer.c"),
134 str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer" / "readline_tokenizer.c"),
135 str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer" / "helpers.c"),
136 str(MOD_DIR.parent.parent.parent / "Parser" / "pegen.c"),
137 str(MOD_DIR.parent.parent.parent / "Parser" / "pegen_errors.c"),
138 str(MOD_DIR.parent.parent.parent / "Parser" / "action_helpers.c"),
139 str(MOD_DIR.parent.parent.parent / "Parser" / "string_parser.c"),
140 str(MOD_DIR.parent / "peg_extension" / "peg_extension.c"),
141 ]
142 include_dirs = [
143 str(MOD_DIR.parent.parent.parent / "Include" / "internal"),
144 str(MOD_DIR.parent.parent.parent / "Include" / "internal" / "mimalloc"),
145 str(MOD_DIR.parent.parent.parent / "Parser"),
146 str(MOD_DIR.parent.parent.parent / "Parser" / "lexer"),
147 str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer"),
148 ]
149 if sys.platform == "win32":
150 # HACK: The location of pyconfig.h has moved within our build, and
151 # setuptools hasn't updated for it yet. So add the path manually for now
152 include_dirs.append(pathlib.Path(sysconfig.get_config_h_filename()).parent)
153 extension = Extension(
154 extension_name,
155 sources=[generated_source_path],
156 extra_compile_args=extra_compile_args,
157 extra_link_args=extra_link_args,
158 )
159 dist = Distribution({"name": extension_name, "ext_modules": [extension]})
160 cmd = dist.get_command_obj("build_ext")
161 assert isinstance(cmd, setuptools.command.build_ext.build_ext)
162 fixup_build_ext(cmd)
163 cmd.build_lib = str(source_file_path.parent)
164 cmd.include_dirs = include_dirs
165 if build_dir:
166 cmd.build_temp = build_dir
167 cmd.ensure_finalized()
168
169 compiler = new_compiler()
170 customize_compiler(compiler)
171 compiler.set_include_dirs(cmd.include_dirs)
172 compiler.set_library_dirs(cmd.library_dirs)
173 # build static lib
174 if library_dir:
175 library_filename = compiler.library_filename(extension_name, output_dir=library_dir)
176 if newer_group(common_sources, library_filename, "newer"):
177 if sys.platform == "win32":
178 assert compiler.static_lib_format
179 pdb = compiler.static_lib_format % (extension_name, ".pdb")
180 compile_opts = [f"/Fd{library_dir}\\{pdb}"]
181 compile_opts.extend(extra_compile_args)
182 else:
183 compile_opts = extra_compile_args
184 objects = compiler.compile(
185 common_sources,
186 output_dir=library_dir,
187 debug=cmd.debug,
188 extra_postargs=compile_opts,
189 )
190 compiler.create_static_lib(
191 objects, extension_name, output_dir=library_dir, debug=cmd.debug
192 )
193 if sys.platform == "win32":
194 compiler.add_library_dir(library_dir)
195 extension.libraries = [extension_name]
196 elif sys.platform == "darwin":
197 compiler.set_link_objects(
198 [
199 "-Wl,-force_load",
200 library_filename,
201 ]
202 )
203 else:
204 compiler.set_link_objects(
205 [
206 "-Wl,--whole-archive",
207 library_filename,
208 "-Wl,--no-whole-archive",
209 ]
210 )
211 else:
212 extension.sources[0:0] = common_sources
213
214 # Compile the source code to object files.
215 ext_path = cmd.get_ext_fullpath(extension_name)
216 if newer_group(extension.sources, ext_path, "newer"):
217 objects = compiler.compile(
218 extension.sources,
219 output_dir=cmd.build_temp,
220 debug=cmd.debug,
221 extra_postargs=extra_compile_args,
222 )
223 else:
224 objects = compiler.object_filenames(extension.sources, output_dir=cmd.build_temp)
225 # The cmd.get_libraries() call needs a valid compiler attribute or we will
226 # get an incorrect library name on the free-threaded Windows build.
227 cmd.compiler = compiler
228 # Now link the object files together into a "shared object"
229 compiler.link_shared_object(
230 objects,
231 ext_path,
232 libraries=cmd.get_libraries(extension),
233 extra_postargs=extra_link_args,
234 export_symbols=cmd.get_export_symbols(extension), # type: ignore[no-untyped-call]
235 debug=cmd.debug,
236 build_temp=cmd.build_temp,
237 )
238
239 return pathlib.Path(ext_path)
240
241
242 def build_parser(
243 grammar_file: str, verbose_tokenizer: bool = False, verbose_parser: bool = False
244 ) -> tuple[Grammar, Parser, Tokenizer]:
245 with open(grammar_file) as file:
246 tokenizer = Tokenizer(tokenize.generate_tokens(file.readline), verbose=verbose_tokenizer)
247 parser = GrammarParser(tokenizer, verbose=verbose_parser)
248 grammar = parser.start()
249
250 if not grammar:
251 raise parser.make_syntax_error(grammar_file)
252
253 return grammar, parser, tokenizer
254
255
256 def generate_token_definitions(tokens: IO[str]) -> TokenDefinitions:
257 all_tokens = {}
258 exact_tokens = {}
259 non_exact_tokens = set()
260 numbers = itertools.count(0)
261
262 for line in tokens:
263 line = line.strip()
264
265 if not line or line.startswith("#"):
266 continue
267
268 pieces = line.split()
269 index = next(numbers)
270
271 if len(pieces) == 1:
272 (token,) = pieces
273 non_exact_tokens.add(token)
274 all_tokens[index] = token
275 elif len(pieces) == 2:
276 token, op = pieces
277 exact_tokens[op.strip("'")] = index
278 all_tokens[index] = token
279 else:
280 raise ValueError(f"Unexpected line found in Tokens file: {line}")
281
282 return all_tokens, exact_tokens, non_exact_tokens
283
284
285 def build_c_generator(
286 grammar: Grammar,
287 grammar_file: str,
288 tokens_file: str,
289 output_file: str,
290 compile_extension: bool = False,
291 verbose_c_extension: bool = False,
292 keep_asserts_in_extension: bool = True,
293 skip_actions: bool = False,
294 ) -> ParserGenerator:
295 with open(tokens_file) as tok_file:
296 all_tokens, exact_tok, non_exact_tok = generate_token_definitions(tok_file)
297 with open(output_file, "w") as file:
298 gen: ParserGenerator = CParserGenerator(
299 grammar, all_tokens, exact_tok, non_exact_tok, file, skip_actions=skip_actions
300 )
301 gen.generate(grammar_file)
302
303 if compile_extension:
304 with tempfile.TemporaryDirectory() as build_dir:
305 compile_c_extension(
306 output_file,
307 build_dir=build_dir,
308 verbose=verbose_c_extension,
309 keep_asserts=keep_asserts_in_extension,
310 )
311 return gen
312
313
314 def build_python_generator(
315 grammar: Grammar,
316 grammar_file: str,
317 output_file: str,
318 skip_actions: bool = False,
319 ) -> ParserGenerator:
320 with open(output_file, "w") as file:
321 gen: ParserGenerator = PythonParserGenerator(grammar, file) # TODO: skip_actions
322 gen.generate(grammar_file)
323 return gen
324
325
326 def build_c_parser_and_generator(
327 grammar_file: str,
328 tokens_file: str,
329 output_file: str,
330 compile_extension: bool = False,
331 verbose_tokenizer: bool = False,
332 verbose_parser: bool = False,
333 verbose_c_extension: bool = False,
334 keep_asserts_in_extension: bool = True,
335 skip_actions: bool = False,
336 ) -> tuple[Grammar, Parser, Tokenizer, ParserGenerator]:
337 """Generate rules, C parser, tokenizer, parser generator for a given grammar
338
339 Args:
340 grammar_file (string): Path for the grammar file
341 tokens_file (string): Path for the tokens file
342 output_file (string): Path for the output file
343 compile_extension (bool, optional): Whether to compile the C extension.
344 Defaults to False.
345 verbose_tokenizer (bool, optional): Whether to display additional output
346 when generating the tokenizer. Defaults to False.
347 verbose_parser (bool, optional): Whether to display additional output
348 when generating the parser. Defaults to False.
349 verbose_c_extension (bool, optional): Whether to display additional
350 output when compiling the C extension . Defaults to False.
351 keep_asserts_in_extension (bool, optional): Whether to keep the assert statements
352 when compiling the extension module. Defaults to True.
353 skip_actions (bool, optional): Whether to pretend no rule has any actions.
354 """
355 grammar, parser, tokenizer = build_parser(grammar_file, verbose_tokenizer, verbose_parser)
356 gen = build_c_generator(
357 grammar,
358 grammar_file,
359 tokens_file,
360 output_file,
361 compile_extension,
362 verbose_c_extension,
363 keep_asserts_in_extension,
364 skip_actions=skip_actions,
365 )
366
367 return grammar, parser, tokenizer, gen
368
369
370 def build_python_parser_and_generator(
371 grammar_file: str,
372 output_file: str,
373 verbose_tokenizer: bool = False,
374 verbose_parser: bool = False,
375 skip_actions: bool = False,
376 ) -> tuple[Grammar, Parser, Tokenizer, ParserGenerator]:
377 """Generate rules, python parser, tokenizer, parser generator for a given grammar
378
379 Args:
380 grammar_file (string): Path for the grammar file
381 output_file (string): Path for the output file
382 verbose_tokenizer (bool, optional): Whether to display additional output
383 when generating the tokenizer. Defaults to False.
384 verbose_parser (bool, optional): Whether to display additional output
385 when generating the parser. Defaults to False.
386 skip_actions (bool, optional): Whether to pretend no rule has any actions.
387 """
388 grammar, parser, tokenizer = build_parser(grammar_file, verbose_tokenizer, verbose_parser)
389 gen = build_python_generator(
390 grammar,
391 grammar_file,
392 output_file,
393 skip_actions=skip_actions,
394 )
395 return grammar, parser, tokenizer, gen