Debian Codegraph @

Tools/c-analyzer/cpython/_builtin_types.py

raw
1 from collections import namedtuple
2 import os.path
3 import re
4 import textwrap
5
6 from c_common import tables
7 from . import REPO_ROOT
8 from ._files import iter_header_files, iter_filenames
9
10
11 CAPI_PREFIX = os.path.join('Include', '')
12 INTERNAL_PREFIX = os.path.join('Include', 'internal', '')
13
14 REGEX = re.compile(textwrap.dedent(rf'''
15 (?:
16 ^
17 (?:
18 (?:
19 (?:
20 (?:
21 (?:
22 ( static ) # <static>
23 \s+
24 |
25 ( extern ) # <extern>
26 \s+
27 )?
28 PyTypeObject \s+
29 )
30 |
31 (?:
32 ( PyAPI_DATA ) # <capi>
33 \s* [(] \s* PyTypeObject \s* [)] \s*
34 )
35 )
36 (\w+) # <name>
37 \s*
38 (?:
39 (?:
40 ( = \s* {{ ) # <def>
41 $
42 )
43 |
44 ( ; ) # <decl>
45 )
46 )
47 |
48 (?:
49 # These are specific to Objects/exceptions.c:
50 (?:
51 SimpleExtendsException
52 |
53 MiddlingExtendsException
54 |
55 ComplexExtendsException
56 )
57 \( \w+ \s* , \s*
58 ( \w+ ) # <excname>
59 \s* ,
60 )
61 )
62 )
63 '''), re.VERBOSE)
64
65
66 def _parse_line(line):
67 m = re.match(REGEX, line)
68 if not m:
69 return None
70 (static, extern, capi,
71 name,
72 def_, decl,
73 excname,
74 ) = m.groups()
75 if def_:
76 isdecl = False
77 if extern or capi:
78 raise NotImplementedError(line)
79 kind = 'static' if static else None
80 elif excname:
81 name = f'_PyExc_{excname}'
82 isdecl = False
83 kind = 'static'
84 else:
85 isdecl = True
86 if static:
87 kind = 'static'
88 elif extern:
89 kind = 'extern'
90 elif capi:
91 kind = 'capi'
92 else:
93 kind = None
94 return name, isdecl, kind
95
96
97 class BuiltinTypeDecl(namedtuple('BuiltinTypeDecl', 'file lno name kind')):
98
99 KINDS = {
100 'static',
101 'extern',
102 'capi',
103 'forward',
104 }
105
106 @classmethod
107 def from_line(cls, line, filename, lno):
108 # This is similar to ._capi.CAPIItem.from_line().
109 parsed = _parse_line(line)
110 if not parsed:
111 return None
112 name, isdecl, kind = parsed
113 if not isdecl:
114 return None
115 return cls.from_parsed(name, kind, filename, lno)
116
117 @classmethod
118 def from_parsed(cls, name, kind, filename, lno):
119 if not kind:
120 kind = 'forward'
121 return cls.from_values(filename, lno, name, kind)
122
123 @classmethod
124 def from_values(cls, filename, lno, name, kind):
125 if kind not in cls.KINDS:
126 raise ValueError(f'unsupported kind {kind!r}')
127 self = cls(filename, lno, name, kind)
128 if self.kind not in ('extern', 'capi') and self.api:
129 raise NotImplementedError(self)
130 elif self.kind == 'capi' and not self.api:
131 raise NotImplementedError(self)
132 return self
133
134 @property
135 def relfile(self):
136 return self.file[len(REPO_ROOT) + 1:]
137
138 @property
139 def api(self):
140 return self.relfile.startswith(CAPI_PREFIX)
141
142 @property
143 def internal(self):
144 return self.relfile.startswith(INTERNAL_PREFIX)
145
146 @property
147 def private(self):
148 if not self.name.startswith('_'):
149 return False
150 return self.api and not self.internal
151
152 @property
153 def public(self):
154 if self.kind != 'capi':
155 return False
156 return not self.internal and not self.private
157
158
159 class BuiltinTypeInfo(namedtuple('BuiltinTypeInfo', 'file lno name static decl')):
160
161 @classmethod
162 def from_line(cls, line, filename, lno, *, decls=None):
163 parsed = _parse_line(line)
164 if not parsed:
165 return None
166 name, isdecl, kind = parsed
167 if isdecl:
168 return None
169 return cls.from_parsed(name, kind, filename, lno, decls=decls)
170
171 @classmethod
172 def from_parsed(cls, name, kind, filename, lno, *, decls=None):
173 if not kind:
174 static = False
175 elif kind == 'static':
176 static = True
177 else:
178 raise NotImplementedError((filename, line, kind))
179 decl = decls.get(name) if decls else None
180 return cls(filename, lno, name, static, decl)
181
182 @property
183 def relfile(self):
184 return self.file[len(REPO_ROOT) + 1:]
185
186 @property
187 def exported(self):
188 return not self.static
189
190 @property
191 def api(self):
192 if not self.decl:
193 return False
194 return self.decl.api
195
196 @property
197 def internal(self):
198 if not self.decl:
199 return False
200 return self.decl.internal
201
202 @property
203 def private(self):
204 if not self.decl:
205 return False
206 return self.decl.private
207
208 @property
209 def public(self):
210 if not self.decl:
211 return False
212 return self.decl.public
213
214 @property
215 def inmodule(self):
216 return self.relfile.startswith('Modules' + os.path.sep)
217
218 def render_rowvalues(self, kinds):
219 row = {
220 'name': self.name,
221 **{k: '' for k in kinds},
222 'filename': f'{self.relfile}:{self.lno}',
223 }
224 if self.static:
225 kind = 'static'
226 else:
227 if self.internal:
228 kind = 'internal'
229 elif self.private:
230 kind = 'private'
231 elif self.public:
232 kind = 'public'
233 else:
234 kind = 'global'
235 row['kind'] = kind
236 row[kind] = kind
237 return row
238
239
240 def _ensure_decl(decl, decls):
241 prev = decls.get(decl.name)
242 if prev:
243 if decl.kind == 'forward':
244 return None
245 if prev.kind != 'forward':
246 if decl.kind == prev.kind and decl.file == prev.file:
247 assert decl.lno != prev.lno, (decl, prev)
248 return None
249 raise NotImplementedError(f'duplicate {decl} (was {prev}')
250 decls[decl.name] = decl
251
252
253 def iter_builtin_types(filenames=None):
254 decls = {}
255 seen = set()
256 for filename in iter_header_files():
257 seen.add(filename)
258 with open(filename) as infile:
259 for lno, line in enumerate(infile, 1):
260 decl = BuiltinTypeDecl.from_line(line, filename, lno)
261 if not decl:
262 continue
263 _ensure_decl(decl, decls)
264 srcfiles = []
265 for filename in iter_filenames():
266 if filename.endswith('.c'):
267 srcfiles.append(filename)
268 continue
269 if filename in seen:
270 continue
271 with open(filename) as infile:
272 for lno, line in enumerate(infile, 1):
273 decl = BuiltinTypeDecl.from_line(line, filename, lno)
274 if not decl:
275 continue
276 _ensure_decl(decl, decls)
277
278 for filename in srcfiles:
279 with open(filename) as infile:
280 localdecls = {}
281 for lno, line in enumerate(infile, 1):
282 parsed = _parse_line(line)
283 if not parsed:
284 continue
285 name, isdecl, kind = parsed
286 if isdecl:
287 decl = BuiltinTypeDecl.from_parsed(name, kind, filename, lno)
288 if not decl:
289 raise NotImplementedError((filename, line))
290 _ensure_decl(decl, localdecls)
291 else:
292 builtin = BuiltinTypeInfo.from_parsed(
293 name, kind, filename, lno,
294 decls=decls if name in decls else localdecls)
295 if not builtin:
296 raise NotImplementedError((filename, line))
297 yield builtin
298
299
300 def resolve_matcher(showmodules=False):
301 def match(info, *, log=None):
302 if not info.inmodule:
303 return True
304 if log is not None:
305 log(f'ignored {info.name!r}')
306 return False
307 return match
308
309
310 ##################################
311 # CLI rendering
312
313 def resolve_format(fmt):
314 if not fmt:
315 return 'table'
316 elif isinstance(fmt, str) and fmt in _FORMATS:
317 return fmt
318 else:
319 raise NotImplementedError(fmt)
320
321
322 def get_renderer(fmt):
323 fmt = resolve_format(fmt)
324 if isinstance(fmt, str):
325 try:
326 return _FORMATS[fmt]
327 except KeyError:
328 raise ValueError(f'unsupported format {fmt!r}')
329 else:
330 raise NotImplementedError(fmt)
331
332
333 def render_table(types):
334 types = sorted(types, key=(lambda t: t.name))
335 colspecs = tables.resolve_columns(
336 'name:<33 static:^ global:^ internal:^ private:^ public:^ filename:<30')
337 header, div, rowfmt = tables.build_table(colspecs)
338 leader = ' ' * sum(c.width+2 for c in colspecs[:3]) + ' '
339 yield leader + f'{"API":^29}'
340 yield leader + '-' * 29
341 yield header
342 yield div
343 kinds = [c[0] for c in colspecs[1:-1]]
344 counts = {k: 0 for k in kinds}
345 base = {k: '' for k in kinds}
346 for t in types:
347 row = t.render_rowvalues(kinds)
348 kind = row['kind']
349 yield rowfmt.format(**row)
350 counts[kind] += 1
351 yield ''
352 yield f'total: {sum(counts.values()):>3}'
353 for kind in kinds:
354 yield f' {kind:>10}: {counts[kind]:>3}'
355
356
357 def render_repr(types):
358 for t in types:
359 yield repr(t)
360
361
362 _FORMATS = {
363 'table': render_table,
364 'repr': render_repr,
365 }