Debian Codegraph @

Tools/c-analyzer/cpython/_parser.py

raw
1 import os.path
2
3 from c_parser.preprocessor import (
4 get_preprocessor as _get_preprocessor,
5 )
6 from c_parser import (
7 parse_file as _parse_file,
8 parse_files as _parse_files,
9 )
10 from . import REPO_ROOT
11
12
13 GLOB_ALL = '**/*'
14
15
16 def _abs(relfile):
17 return os.path.join(REPO_ROOT, relfile)
18
19
20 def format_conf_lines(lines):
21 """Format conf entries for use in a TSV table."""
22 return [_abs(entry) for entry in lines]
23
24
25 def format_tsv_lines(lines):
26 """Format entries for use in a TSV table."""
27 lines = ((_abs(first), *rest) for first, *rest in lines)
28 lines = map('\t'.join, lines)
29 return list(map(str.rstrip, lines))
30
31
32 '''
33 @begin=sh@
34 ./python ../c-parser/cpython.py
35 --exclude '+../c-parser/EXCLUDED'
36 --macros '+../c-parser/MACROS'
37 --incldirs '+../c-parser/INCL_DIRS'
38 --same './Include/cpython/'
39 Include/*.h
40 Include/internal/*.h
41 Modules/**/*.c
42 Objects/**/*.c
43 Parser/**/*.c
44 Python/**/*.c
45 @end=sh@
46 '''
47
48 # XXX Handle these.
49 EXCLUDED = format_conf_lines([
50 # macOS
51 'Modules/_scproxy.c', # SystemConfiguration/SystemConfiguration.h
52
53 # Windows
54 'Modules/_winapi.c', # windows.h
55 'Modules/expat/winconfig.h',
56 'Modules/overlapped.c', # winsock.h
57 'Python/dynload_win.c', # windows.h
58 'Python/thread_nt.h',
59
60 # other OS-dependent
61 'Python/dynload_aix.c', # sys/ldr.h
62 'Python/dynload_dl.c', # dl.h
63 'Python/dynload_hpux.c', # dl.h
64 'Python/emscripten_signal.c',
65 'Python/emscripten_syscalls.c',
66 'Python/emscripten_trampoline_inner.c',
67 'Python/thread_pthread.h',
68 'Python/thread_pthread_stubs.h',
69
70 # only huge constants (safe but parsing is slow)
71 'Modules/_ssl_data_*.h',
72 'Modules/cjkcodecs/mappings_*.h',
73 'Modules/unicodedata_db.h',
74 'Modules/unicodename_db.h',
75 'Objects/unicodetype_db.h',
76
77 # generated
78 'Python/deepfreeze/*.c',
79 'Python/frozen_modules/*.h',
80 'Python/generated_cases.c.h',
81 'Python/executor_cases.c.h',
82 'Python/optimizer_cases.c.h',
83 # XXX: Throws errors if PY_VERSION_HEX is not mocked out
84 'Modules/clinic/_testclinic_depr.c.h',
85
86 # not actually source
87 'Python/bytecodes.c',
88 'Python/optimizer_bytecodes.c',
89
90 # mimalloc
91 'Objects/mimalloc/*.c',
92 'Include/internal/mimalloc/*.h',
93 'Include/internal/mimalloc/mimalloc/*.h',
94 ])
95
96 # XXX Fix the parser.
97 EXCLUDED += format_conf_lines([
98 # The tool should be able to parse these...
99
100 # The problem with xmlparse.c is that something
101 # has gone wrong where # we handle "maybe inline actual"
102 # in Tools/c-analyzer/c_parser/parser/_global.py.
103 'Modules/expat/internal.h',
104 'Modules/expat/xmlparse.c',
105 ])
106
107 INCL_DIRS = format_tsv_lines([
108 # (glob, dirname)
109
110 ('*', '.'),
111 ('*', './Include'),
112 ('*', './Include/internal'),
113 ('*', './Include/internal/mimalloc'),
114
115 ('Modules/_decimal/**/*.c', 'Modules/_decimal/libmpdec'),
116 ('Modules/_elementtree.c', 'Modules/expat'),
117 ('Modules/_hacl/*.c', 'Modules/_hacl/include'),
118 ('Modules/_hacl/*.c', 'Modules/_hacl/'),
119 ('Modules/_hacl/*.h', 'Modules/_hacl/include'),
120 ('Modules/_hacl/*.h', 'Modules/_hacl/'),
121 ('Modules/md5module.c', 'Modules/_hacl/include'),
122 ('Modules/sha1module.c', 'Modules/_hacl/include'),
123 ('Modules/sha2module.c', 'Modules/_hacl/include'),
124 ('Modules/sha3module.c', 'Modules/_hacl/include'),
125 ('Modules/blake2module.c', 'Modules/_hacl/include'),
126 ('Modules/hmacmodule.c', 'Modules/_hacl/include'),
127 ('Objects/stringlib/*.h', 'Objects'),
128
129 # possible system-installed headers, just in case
130 ('Modules/_tkinter.c', '/usr/include/tcl8.6'),
131 ('Modules/_uuidmodule.c', '/usr/include/uuid'),
132 ('Modules/tkappinit.c', '/usr/include/tcl'),
133
134 ])
135
136 INCLUDES = format_tsv_lines([
137 # (glob, include)
138
139 ('**/*.h', 'Python.h'),
140 ('Include/**/*.h', 'object.h'),
141
142 # for Py_HAVE_CONDVAR
143 ('Include/internal/pycore_gil.h', 'pycore_condvar.h'),
144 ('Python/thread_pthread.h', 'pycore_condvar.h'),
145
146 # other
147
148 ('Objects/stringlib/join.h', 'stringlib/stringdefs.h'),
149 ('Objects/stringlib/ctype.h', 'stringlib/stringdefs.h'),
150 ('Objects/stringlib/transmogrify.h', 'stringlib/stringdefs.h'),
151 # ('Objects/stringlib/fastsearch.h', 'stringlib/stringdefs.h'),
152 # ('Objects/stringlib/count.h', 'stringlib/stringdefs.h'),
153 # ('Objects/stringlib/find.h', 'stringlib/stringdefs.h'),
154 # ('Objects/stringlib/partition.h', 'stringlib/stringdefs.h'),
155 # ('Objects/stringlib/split.h', 'stringlib/stringdefs.h'),
156 ('Objects/stringlib/fastsearch.h', 'stringlib/ucs1lib.h'),
157 ('Objects/stringlib/count.h', 'stringlib/ucs1lib.h'),
158 ('Objects/stringlib/find.h', 'stringlib/ucs1lib.h'),
159 ('Objects/stringlib/partition.h', 'stringlib/ucs1lib.h'),
160 ('Objects/stringlib/split.h', 'stringlib/ucs1lib.h'),
161 ('Objects/stringlib/find_max_char.h', 'Objects/stringlib/ucs1lib.h'),
162 ('Objects/stringlib/count.h', 'Objects/stringlib/fastsearch.h'),
163 ('Objects/stringlib/find.h', 'Objects/stringlib/fastsearch.h'),
164 ('Objects/stringlib/partition.h', 'Objects/stringlib/fastsearch.h'),
165 ('Objects/stringlib/replace.h', 'Objects/stringlib/fastsearch.h'),
166 ('Objects/stringlib/repr.h', 'Objects/stringlib/fastsearch.h'),
167 ('Objects/stringlib/split.h', 'Objects/stringlib/fastsearch.h'),
168 ])
169
170 MACROS = format_tsv_lines([
171 # (glob, name, value)
172
173 ('Include/internal/*.h', 'Py_BUILD_CORE', '1'),
174 ('Python/**/*.c', 'Py_BUILD_CORE', '1'),
175 ('Python/**/*.h', 'Py_BUILD_CORE', '1'),
176 ('Parser/**/*.c', 'Py_BUILD_CORE', '1'),
177 ('Parser/**/*.h', 'Py_BUILD_CORE', '1'),
178 ('Objects/**/*.c', 'Py_BUILD_CORE', '1'),
179 ('Objects/**/*.h', 'Py_BUILD_CORE', '1'),
180
181 ('Modules/_asynciomodule.c', 'Py_BUILD_CORE', '1'),
182 ('Modules/_codecsmodule.c', 'Py_BUILD_CORE', '1'),
183 ('Modules/_collectionsmodule.c', 'Py_BUILD_CORE', '1'),
184 ('Modules/_ctypes/_ctypes.c', 'Py_BUILD_CORE', '1'),
185 ('Modules/_ctypes/cfield.c', 'Py_BUILD_CORE', '1'),
186 ('Modules/_cursesmodule.c', 'Py_BUILD_CORE', '1'),
187 ('Modules/_datetimemodule.c', 'Py_BUILD_CORE', '1'),
188 ('Modules/_functoolsmodule.c', 'Py_BUILD_CORE', '1'),
189 ('Modules/_heapqmodule.c', 'Py_BUILD_CORE', '1'),
190 ('Modules/_io/*.c', 'Py_BUILD_CORE', '1'),
191 ('Modules/_io/*.h', 'Py_BUILD_CORE', '1'),
192 ('Modules/_localemodule.c', 'Py_BUILD_CORE', '1'),
193 ('Modules/_operator.c', 'Py_BUILD_CORE', '1'),
194 ('Modules/_posixsubprocess.c', 'Py_BUILD_CORE', '1'),
195 ('Modules/_sre/sre.c', 'Py_BUILD_CORE', '1'),
196 ('Modules/_threadmodule.c', 'Py_BUILD_CORE', '1'),
197 ('Modules/_tracemalloc.c', 'Py_BUILD_CORE', '1'),
198 ('Modules/_weakref.c', 'Py_BUILD_CORE', '1'),
199 ('Modules/_zoneinfo.c', 'Py_BUILD_CORE', '1'),
200 ('Modules/atexitmodule.c', 'Py_BUILD_CORE', '1'),
201 ('Modules/cmathmodule.c', 'Py_BUILD_CORE', '1'),
202 ('Modules/faulthandler.c', 'Py_BUILD_CORE', '1'),
203 ('Modules/gcmodule.c', 'Py_BUILD_CORE', '1'),
204 ('Modules/getpath.c', 'Py_BUILD_CORE', '1'),
205 ('Modules/getpath_noop.c', 'Py_BUILD_CORE', '1'),
206 ('Modules/itertoolsmodule.c', 'Py_BUILD_CORE', '1'),
207 ('Modules/main.c', 'Py_BUILD_CORE', '1'),
208 ('Modules/mathmodule.c', 'Py_BUILD_CORE', '1'),
209 ('Modules/posixmodule.c', 'Py_BUILD_CORE', '1'),
210 ('Modules/sha256module.c', 'Py_BUILD_CORE', '1'),
211 ('Modules/sha512module.c', 'Py_BUILD_CORE', '1'),
212 ('Modules/signalmodule.c', 'Py_BUILD_CORE', '1'),
213 ('Modules/symtablemodule.c', 'Py_BUILD_CORE', '1'),
214 ('Modules/timemodule.c', 'Py_BUILD_CORE', '1'),
215 ('Modules/unicodedata.c', 'Py_BUILD_CORE', '1'),
216
217 ('Modules/_json.c', 'Py_BUILD_CORE_BUILTIN', '1'),
218 ('Modules/_pickle.c', 'Py_BUILD_CORE_BUILTIN', '1'),
219 ('Modules/_testinternalcapi.c', 'Py_BUILD_CORE_BUILTIN', '1'),
220
221 ('Include/cpython/abstract.h', 'Py_CPYTHON_ABSTRACTOBJECT_H', '1'),
222 ('Include/cpython/bytearrayobject.h', 'Py_CPYTHON_BYTEARRAYOBJECT_H', '1'),
223 ('Include/cpython/bytesobject.h', 'Py_CPYTHON_BYTESOBJECT_H', '1'),
224 ('Include/cpython/ceval.h', 'Py_CPYTHON_CEVAL_H', '1'),
225 ('Include/cpython/code.h', 'Py_CPYTHON_CODE_H', '1'),
226 ('Include/cpython/dictobject.h', 'Py_CPYTHON_DICTOBJECT_H', '1'),
227 ('Include/cpython/fileobject.h', 'Py_CPYTHON_FILEOBJECT_H', '1'),
228 ('Include/cpython/fileutils.h', 'Py_CPYTHON_FILEUTILS_H', '1'),
229 ('Include/cpython/frameobject.h', 'Py_CPYTHON_FRAMEOBJECT_H', '1'),
230 ('Include/cpython/import.h', 'Py_CPYTHON_IMPORT_H', '1'),
231 ('Include/cpython/interpreteridobject.h', 'Py_CPYTHON_INTERPRETERIDOBJECT_H', '1'),
232 ('Include/cpython/listobject.h', 'Py_CPYTHON_LISTOBJECT_H', '1'),
233 ('Include/cpython/methodobject.h', 'Py_CPYTHON_METHODOBJECT_H', '1'),
234 ('Include/cpython/object.h', 'Py_CPYTHON_OBJECT_H', '1'),
235 ('Include/cpython/objimpl.h', 'Py_CPYTHON_OBJIMPL_H', '1'),
236 ('Include/cpython/pyerrors.h', 'Py_CPYTHON_ERRORS_H', '1'),
237 ('Include/cpython/pylifecycle.h', 'Py_CPYTHON_PYLIFECYCLE_H', '1'),
238 ('Include/cpython/pymem.h', 'Py_CPYTHON_PYMEM_H', '1'),
239 ('Include/cpython/pystate.h', 'Py_CPYTHON_PYSTATE_H', '1'),
240 ('Include/cpython/sysmodule.h', 'Py_CPYTHON_SYSMODULE_H', '1'),
241 ('Include/cpython/traceback.h', 'Py_CPYTHON_TRACEBACK_H', '1'),
242 ('Include/cpython/tupleobject.h', 'Py_CPYTHON_TUPLEOBJECT_H', '1'),
243 ('Include/cpython/unicodeobject.h', 'Py_CPYTHON_UNICODEOBJECT_H', '1'),
244
245 # implied include of <unistd.h>
246 ('Include/**/*.h', '_POSIX_THREADS', '1'),
247 ('Include/**/*.h', 'HAVE_PTHREAD_H', '1'),
248
249 # from pyconfig.h
250 ('Include/cpython/pthread_stubs.h', 'HAVE_PTHREAD_STUBS', '1'),
251 ('Python/thread_pthread_stubs.h', 'HAVE_PTHREAD_STUBS', '1'),
252
253 # from Objects/bytesobject.c
254 ('Objects/stringlib/partition.h', 'STRINGLIB_GET_EMPTY()', 'bytes_get_empty()'),
255 ('Objects/stringlib/join.h', 'STRINGLIB_MUTABLE', '0'),
256 ('Objects/stringlib/partition.h', 'STRINGLIB_MUTABLE', '0'),
257 ('Objects/stringlib/split.h', 'STRINGLIB_MUTABLE', '0'),
258 ('Objects/stringlib/transmogrify.h', 'STRINGLIB_MUTABLE', '0'),
259
260 # from Makefile
261 ('Modules/getpath.c', 'PYTHONPATH', '1'),
262 ('Modules/getpath.c', 'PREFIX', '...'),
263 ('Modules/getpath.c', 'EXEC_PREFIX', '...'),
264 ('Modules/getpath.c', 'VERSION', '...'),
265 ('Modules/getpath.c', 'VPATH', '...'),
266 ('Modules/getpath.c', 'PLATLIBDIR', '...'),
267 # ('Modules/_dbmmodule.c', 'USE_GDBM_COMPAT', '1'),
268 ('Modules/_dbmmodule.c', 'USE_NDBM', '1'),
269 # ('Modules/_dbmmodule.c', 'USE_BERKDB', '1'),
270
271 # See: setup.py
272 ('Modules/_decimal/**/*.c', 'CONFIG_64', '1'),
273 ('Modules/_decimal/**/*.c', 'ASM', '1'),
274 ('Modules/expat/xmlparse.c', 'HAVE_EXPAT_CONFIG_H', '1'),
275 ('Modules/expat/xmlparse.c', 'XML_POOR_ENTROPY', '1'),
276 ('Modules/_dbmmodule.c', 'HAVE_GDBM_DASH_NDBM_H', '1'),
277
278 # others
279 ('Modules/_sre/sre_lib.h', 'LOCAL(type)', 'static inline type'),
280 ('Modules/_sre/sre_lib.h', 'SRE(F)', 'sre_ucs2_##F'),
281 ('Objects/stringlib/codecs.h', 'STRINGLIB_IS_UNICODE', '1'),
282 ('Include/internal/pycore_crossinterp_data_registry.h', 'Py_CORE_CROSSINTERP_DATA_REGISTRY_H', '1'),
283 ])
284
285 # -pthread
286 # -Wno-unused-result
287 # -Wsign-compare
288 # -g
289 # -Og
290 # -Wall
291 # -std=c99
292 # -Wextra
293 # -Wno-unused-result -Wno-unused-parameter
294 # -Wno-missing-field-initializers
295 # -Werror=implicit-function-declaration
296
297 SAME = {
298 _abs('Include/*.h'): [_abs('Include/cpython/')],
299 _abs('Python/ceval.c'): ['Python/generated_cases.c.h'],
300 }
301
302 MAX_SIZES = {
303 # GLOB: (MAXTEXT, MAXLINES),
304 # default: (10_000, 200)
305 # First match wins.
306 _abs('Modules/_ctypes/ctypes.h'): (5_000, 500),
307 _abs('Modules/_datetimemodule.c'): (20_000, 300),
308 _abs('Modules/_hacl/*.c'): (200_000, 500),
309 _abs('Modules/posixmodule.c'): (20_000, 500),
310 _abs('Modules/termios.c'): (10_000, 800),
311 _abs('Modules/_testcapimodule.c'): (20_000, 400),
312 _abs('Modules/expat/expat.h'): (10_000, 400),
313 _abs('Objects/stringlib/unicode_format.h'): (10_000, 400),
314 _abs('Objects/typeobject.c'): (35_000, 200),
315 _abs('Python/compile.c'): (20_000, 500),
316 _abs('Python/optimizer.c'): (100_000, 5_000),
317 _abs('Python/parking_lot.c'): (40_000, 1000),
318 _abs('Python/pylifecycle.c'): (500_000, 5000),
319 _abs('Python/pystate.c'): (500_000, 5000),
320 _abs('Python/initconfig.c'): (50_000, 500),
321
322 # Generated files:
323 _abs('Include/internal/pycore_opcode.h'): (10_000, 1000),
324 _abs('Include/internal/pycore_global_strings.h'): (5_000, 1000),
325 _abs('Include/internal/pycore_runtime_init_generated.h'): (5_000, 1000),
326 _abs('Python/deepfreeze/*.c'): (20_000, 500),
327 _abs('Python/frozen_modules/*.h'): (20_000, 500),
328 _abs('Python/opcode_targets.h'): (10_000, 500),
329 _abs('Python/stdlib_module_names.h'): (5_000, 500),
330
331 # These large files are currently ignored (see above).
332 _abs('Modules/_ssl_data_31.h'): (80_000, 10_000),
333 _abs('Modules/_ssl_data_300.h'): (80_000, 10_000),
334 _abs('Modules/_ssl_data_111.h'): (80_000, 10_000),
335 _abs('Modules/cjkcodecs/mappings_*.h'): (160_000, 2_000),
336 _abs('Modules/clinic/_testclinic.c.h'): (120_000, 5_000),
337 _abs('Modules/unicodedata_db.h'): (180_000, 3_000),
338 _abs('Modules/unicodename_db.h'): (1_200_000, 15_000),
339 _abs('Objects/unicodetype_db.h'): (240_000, 3_000),
340
341 # Catch-alls:
342 _abs('Include/**/*.h'): (5_000, 500),
343 }
344
345
346 def get_preprocessor(*,
347 file_macros=None,
348 file_includes=None,
349 file_incldirs=None,
350 file_same=None,
351 **kwargs
352 ):
353 macros = tuple(MACROS)
354 if file_macros:
355 macros += tuple(file_macros)
356 includes = tuple(INCLUDES)
357 if file_includes:
358 includes += tuple(file_includes)
359 incldirs = tuple(INCL_DIRS)
360 if file_incldirs:
361 incldirs += tuple(file_incldirs)
362 samefiles = dict(SAME)
363 if file_same:
364 samefiles.update(file_same)
365 return _get_preprocessor(
366 file_macros=macros,
367 file_includes=includes,
368 file_incldirs=incldirs,
369 file_same=samefiles,
370 **kwargs
371 )
372
373
374 def parse_file(filename, *,
375 match_kind=None,
376 ignore_exc=None,
377 log_err=None,
378 ):
379 get_file_preprocessor = get_preprocessor(
380 ignore_exc=ignore_exc,
381 log_err=log_err,
382 )
383 yield from _parse_file(
384 filename,
385 match_kind=match_kind,
386 get_file_preprocessor=get_file_preprocessor,
387 file_maxsizes=MAX_SIZES,
388 )
389
390
391 def parse_files(filenames=None, *,
392 match_kind=None,
393 ignore_exc=None,
394 log_err=None,
395 get_file_preprocessor=None,
396 **file_kwargs
397 ):
398 if get_file_preprocessor is None:
399 get_file_preprocessor = get_preprocessor(
400 ignore_exc=ignore_exc,
401 log_err=log_err,
402 )
403 yield from _parse_files(
404 filenames,
405 match_kind=match_kind,
406 get_file_preprocessor=get_file_preprocessor,
407 file_maxsizes=MAX_SIZES,
408 **file_kwargs
409 )