Debian Codegraph @

Tools/c-analyzer/c_parser/preprocessor/gcc.py

raw
1 import os.path
2 import re
3
4 from . import common as _common
5
6 # The following C files must not built with Py_BUILD_CORE.
7 FILES_WITHOUT_INTERNAL_CAPI = frozenset((
8 # Modules/
9 '_testcapimodule.c',
10 '_testlimitedcapi.c',
11 '_testclinic_limited.c',
12 'xxlimited.c',
13 'xxlimited_35.c',
14 ))
15
16 # C files in the fhe following directories must not be built with
17 # Py_BUILD_CORE.
18 DIRS_WITHOUT_INTERNAL_CAPI = frozenset((
19 '_testcapi', # Modules/_testcapi/
20 '_testlimitedcapi', # Modules/_testlimitedcapi/
21 ))
22
23 TOOL = 'gcc'
24
25 META_FILES = {
26 '<built-in>',
27 '<command-line>',
28 }
29
30 # https://gcc.gnu.org/onlinedocs/cpp/Preprocessor-Output.html
31 # flags:
32 # 1 start of a new file
33 # 2 returning to a file (after including another)
34 # 3 following text comes from a system header file
35 # 4 following text treated wrapped in implicit extern "C" block
36 LINE_MARKER_RE = re.compile(r'^# (\d+) "([^"]+)"((?: [1234])*)$')
37 PREPROC_DIRECTIVE_RE = re.compile(r'^\s*#\s*(\w+)\b.*')
38 COMPILER_DIRECTIVE_RE = re.compile(r'''
39 ^
40 (.*?) # <before>
41 (__\w+__) # <directive>
42 \s*
43 [(] [(]
44 (
45 [^()]*
46 (?:
47 [(]
48 [^()]*
49 [)]
50 [^()]*
51 )*
52 ) # <args>
53 ( [)] [)] ) # <closed>
54 ''', re.VERBOSE)
55
56 POST_ARGS = (
57 '-pthread',
58 '-std=c99',
59 #'-g',
60 #'-Og',
61 #'-Wno-unused-result',
62 #'-Wsign-compare',
63 #'-Wall',
64 #'-Wextra',
65 '-E',
66 )
67
68
69 def preprocess(filename,
70 incldirs=None,
71 includes=None,
72 macros=None,
73 samefiles=None,
74 cwd=None,
75 ):
76 if not cwd or not os.path.isabs(cwd):
77 cwd = os.path.abspath(cwd or '.')
78 filename = _normpath(filename, cwd)
79
80 postargs = POST_ARGS
81 basename = os.path.basename(filename)
82 dirname = os.path.basename(os.path.dirname(filename))
83 if (basename not in FILES_WITHOUT_INTERNAL_CAPI
84 and dirname not in DIRS_WITHOUT_INTERNAL_CAPI):
85 postargs += ('-DPy_BUILD_CORE=1',)
86
87 text = _common.preprocess(
88 TOOL,
89 filename,
90 incldirs=incldirs,
91 includes=includes,
92 macros=macros,
93 #preargs=PRE_ARGS,
94 postargs=postargs,
95 executable=['gcc'],
96 compiler='unix',
97 cwd=cwd,
98 )
99 return _iter_lines(text, filename, samefiles, cwd)
100
101
102 def _iter_lines(text, reqfile, samefiles, cwd, raw=False):
103 lines = iter(text.splitlines())
104
105 # The first line is special.
106 # The next two lines are consistent.
107 firstlines = [
108 f'# 0 "{reqfile}"',
109 '# 0 "<built-in>"',
110 '# 0 "<command-line>"',
111 ]
112 if text.startswith('# 1 '):
113 # Some preprocessors emit a lineno of 1 for line-less entries.
114 firstlines = [l.replace('# 0 ', '# 1 ') for l in firstlines]
115 for expected in firstlines:
116 line = next(lines)
117 if line != expected:
118 raise NotImplementedError((line, expected))
119
120 # Do all the CLI-provided includes.
121 filter_reqfile = (lambda f: _filter_reqfile(f, reqfile, samefiles))
122 make_info = (lambda lno: _common.FileInfo(reqfile, lno))
123 last = None
124 for line in lines:
125 assert last != reqfile, (last,)
126 lno, included, flags = _parse_marker_line(line, reqfile)
127 if not included:
128 raise NotImplementedError((line,))
129 if included == reqfile:
130 # This will be the last one.
131 assert not flags, (line, flags)
132 else:
133 assert 1 in flags, (line, flags)
134 yield from _iter_top_include_lines(
135 lines,
136 _normpath(included, cwd),
137 cwd,
138 filter_reqfile,
139 make_info,
140 raw,
141 )
142 last = included
143 # The last one is always the requested file.
144 assert included == reqfile, (line,)
145
146
147 def _iter_top_include_lines(lines, topfile, cwd,
148 filter_reqfile, make_info,
149 raw):
150 partial = 0 # depth
151 files = [topfile]
152 # We start at 1 in case there are source lines (including blank ones)
153 # before the first marker line. Also, we already verified in
154 # _parse_marker_line() that the preprocessor reported lno as 1.
155 lno = 1
156 for line in lines:
157 if line == '# 0 "<command-line>" 2' or line == '# 1 "<command-line>" 2':
158 # We're done with this top-level include.
159 return
160
161 _lno, included, flags = _parse_marker_line(line)
162 if included:
163 lno = _lno
164 included = _normpath(included, cwd)
165 # We hit a marker line.
166 if 1 in flags:
167 # We're entering a file.
168 # XXX Cycles are unexpected?
169 #assert included not in files, (line, files)
170 files.append(included)
171 elif 2 in flags:
172 # We're returning to a file.
173 assert files and included in files, (line, files)
174 assert included != files[-1], (line, files)
175 while files[-1] != included:
176 files.pop()
177 # XXX How can a file return to line 1?
178 #assert lno > 1, (line, lno)
179 else:
180 if included == files[-1]:
181 # It's the next line from the file.
182 assert lno > 1, (line, lno)
183 else:
184 # We ran into a user-added #LINE directive,
185 # which we promptly ignore.
186 pass
187 elif not files:
188 raise NotImplementedError((line,))
189 elif filter_reqfile(files[-1]):
190 assert lno is not None, (line, files[-1])
191 if (m := PREPROC_DIRECTIVE_RE.match(line)):
192 name, = m.groups()
193 if name != 'pragma':
194 raise Exception(line)
195 else:
196 line = re.sub(r'__inline__', 'inline', line)
197 if not raw:
198 line, partial = _strip_directives(line, partial=partial)
199 yield _common.SourceLine(
200 make_info(lno),
201 'source',
202 line or '',
203 None,
204 )
205 lno += 1
206
207
208 def _parse_marker_line(line, reqfile=None):
209 m = LINE_MARKER_RE.match(line)
210 if not m:
211 return None, None, None
212 lno, origfile, flags = m.groups()
213 lno = int(lno)
214 assert origfile not in META_FILES, (line,)
215 assert lno > 0, (line, lno)
216 flags = set(int(f) for f in flags.split()) if flags else ()
217
218 if 1 in flags:
219 # We're entering a file.
220 assert lno == 1, (line, lno)
221 assert 2 not in flags, (line,)
222 elif 2 in flags:
223 # We're returning to a file.
224 #assert lno > 1, (line, lno)
225 pass
226 elif reqfile and origfile == reqfile:
227 # We're starting the requested file.
228 assert lno == 1, (line, lno)
229 assert not flags, (line, flags)
230 else:
231 # It's the next line from the file.
232 assert lno > 1, (line, lno)
233 return lno, origfile, flags
234
235
236 def _strip_directives(line, partial=0):
237 # We assume there are no string literals with parens in directive bodies.
238 while partial > 0:
239 if not (m := re.match(r'[^{}]*([()])', line)):
240 return None, partial
241 delim, = m.groups()
242 partial += 1 if delim == '(' else -1 # opened/closed
243 line = line[m.end():]
244
245 line = re.sub(r'__extension__', '', line)
246 line = re.sub(r'__thread\b', '_Thread_local', line)
247
248 while (m := COMPILER_DIRECTIVE_RE.match(line)):
249 before, _, _, closed = m.groups()
250 if closed:
251 line = f'{before} {line[m.end():]}'
252 else:
253 after, partial = _strip_directives(line[m.end():], 2)
254 line = f'{before} {after or ""}'
255 if partial:
256 break
257
258 return line, partial
259
260
261 def _filter_reqfile(current, reqfile, samefiles):
262 if current == reqfile:
263 return True
264 if current == '<stdin>':
265 return True
266 if current in samefiles:
267 return True
268 return False
269
270
271 def _normpath(filename, cwd):
272 assert cwd
273 return os.path.normpath(os.path.join(cwd, filename))