Debian Codegraph @

Tools/c-analyzer/cpython/_analyzer.py

raw
1 import os.path
2
3 from c_common.clsutil import classonly
4 from c_parser.info import (
5 KIND,
6 Declaration,
7 TypeDeclaration,
8 Member,
9 FIXED_TYPE,
10 )
11 from c_parser.match import (
12 is_pots,
13 is_funcptr,
14 )
15 from c_analyzer.match import (
16 is_system_type,
17 is_process_global,
18 is_fixed_type,
19 is_immutable,
20 )
21 import c_analyzer as _c_analyzer
22 import c_analyzer.info as _info
23 import c_analyzer.datafiles as _datafiles
24 from . import _parser, REPO_ROOT
25
26
27 _DATA_DIR = os.path.dirname(__file__)
28 KNOWN_FILE = os.path.join(_DATA_DIR, 'known.tsv')
29 IGNORED_FILE = os.path.join(_DATA_DIR, 'ignored.tsv')
30 NEED_FIX_FILE = os.path.join(_DATA_DIR, 'globals-to-fix.tsv')
31 KNOWN_IN_DOT_C = {
32 'struct _odictobject': False,
33 'PyTupleObject': False,
34 'struct _typeobject': False,
35 'struct _arena': True, # ???
36 'struct _frame': False,
37 'struct _ts': True, # ???
38 'struct PyCodeObject': False,
39 'struct _is': True, # ???
40 'PyWideStringList': True, # ???
41 # recursive
42 'struct _dictkeysobject': False,
43 }
44 # These are loaded from the respective .tsv files upon first use.
45 _KNOWN = {
46 # {(file, ID) | ID => info | bool}
47 #'PyWideStringList': True,
48 }
49 #_KNOWN = {(Struct(None, typeid.partition(' ')[-1], None)
50 # if typeid.startswith('struct ')
51 # else TypeDef(None, typeid, None)
52 # ): ([], {'unsupported': None if supported else True})
53 # for typeid, supported in _KNOWN_IN_DOT_C.items()}
54 _IGNORED = {
55 # {ID => reason}
56 }
57
58 # XXX We should be handling these through known.tsv.
59 _OTHER_SUPPORTED_TYPES = {
60 # Holds tuple of strings, which we statically initialize:
61 '_PyArg_Parser',
62 # Uses of these should be const, but we don't worry about it.
63 'PyModuleDef',
64 'PyModuleDef_Slot[]',
65 'PyType_Spec',
66 'PyType_Slot[]',
67 'PyMethodDef',
68 'PyMethodDef[]',
69 'PyMemberDef[]',
70 'PyGetSetDef[]',
71 'PyNumberMethods',
72 'PySequenceMethods',
73 'PyMappingMethods',
74 'PyAsyncMethods',
75 'PyBufferProcs',
76 'PyStructSequence_Field[]',
77 'PyStructSequence_Desc',
78 }
79
80 # XXX We should normalize all cases to a single name,
81 # e.g. "kwlist" (currently the most common).
82 _KWLIST_VARIANTS = [
83 ('*', 'kwlist'),
84 ('*', 'keywords'),
85 ('*', 'kwargs'),
86 ('Modules/_csv.c', 'dialect_kws'),
87 ('Modules/_datetimemodule.c', 'date_kws'),
88 ('Modules/_datetimemodule.c', 'datetime_kws'),
89 ('Modules/_datetimemodule.c', 'time_kws'),
90 ('Modules/_datetimemodule.c', 'timezone_kws'),
91 ('Modules/_lzmamodule.c', 'optnames'),
92 ('Modules/_lzmamodule.c', 'arg_names'),
93 ('Modules/cjkcodecs/multibytecodec.c', 'incnewkwarglist'),
94 ('Modules/cjkcodecs/multibytecodec.c', 'streamkwarglist'),
95 ('Modules/socketmodule.c', 'kwnames'),
96 ]
97
98 KINDS = frozenset((*KIND.TYPES, KIND.VARIABLE))
99
100
101 def read_known():
102 if not _KNOWN:
103 # Cache a copy the first time.
104 extracols = None # XXX
105 #extracols = ['unsupported']
106 known = _datafiles.read_known(KNOWN_FILE, extracols, REPO_ROOT)
107 # For now we ignore known.values() (i.e. "extra").
108 types, _ = _datafiles.analyze_known(
109 known,
110 analyze_resolved=analyze_resolved,
111 )
112 _KNOWN.update(types)
113 return _KNOWN.copy()
114
115
116 def write_known():
117 raise NotImplementedError
118 datafiles.write_known(decls, IGNORED_FILE, ['unsupported'], relroot=REPO_ROOT)
119
120
121 def read_ignored():
122 if not _IGNORED:
123 _IGNORED.update(_datafiles.read_ignored(IGNORED_FILE, relroot=REPO_ROOT))
124 _IGNORED.update(_datafiles.read_ignored(NEED_FIX_FILE, relroot=REPO_ROOT))
125 return dict(_IGNORED)
126
127
128 def write_ignored():
129 raise NotImplementedError
130 _datafiles.write_ignored(variables, IGNORED_FILE, relroot=REPO_ROOT)
131
132
133 def analyze(filenames, *,
134 skip_objects=False,
135 **kwargs
136 ):
137 if skip_objects:
138 # XXX Set up a filter.
139 raise NotImplementedError
140
141 known = read_known()
142
143 decls = iter_decls(filenames)
144 results = _c_analyzer.analyze_decls(
145 decls,
146 known,
147 analyze_resolved=analyze_resolved,
148 )
149 analysis = Analysis.from_results(results)
150
151 return analysis
152
153
154 def iter_decls(filenames, **kwargs):
155 decls = _c_analyzer.iter_decls(
156 filenames,
157 # We ignore functions (and statements).
158 kinds=KINDS,
159 parse_files=_parser.parse_files,
160 **kwargs
161 )
162 for decl in decls:
163 if not decl.data:
164 # Ignore forward declarations.
165 continue
166 yield decl
167
168
169 def analyze_resolved(resolved, decl, types, knowntypes, extra=None):
170 if decl.kind not in KINDS:
171 # Skip it!
172 return None
173
174 typedeps = resolved
175 if typedeps is _info.UNKNOWN:
176 if decl.kind in (KIND.STRUCT, KIND.UNION):
177 typedeps = [typedeps] * len(decl.members)
178 else:
179 typedeps = [typedeps]
180 #assert isinstance(typedeps, (list, TypeDeclaration)), typedeps
181
182 if extra is None:
183 extra = {}
184 elif 'unsupported' in extra:
185 raise NotImplementedError((decl, extra))
186
187 unsupported = _check_unsupported(decl, typedeps, types, knowntypes)
188 extra['unsupported'] = unsupported
189
190 return typedeps, extra
191
192
193 def _check_unsupported(decl, typedeps, types, knowntypes):
194 if typedeps is None:
195 raise NotImplementedError(decl)
196
197 if decl.kind in (KIND.STRUCT, KIND.UNION):
198 return _check_members(decl, typedeps, types, knowntypes)
199 elif decl.kind is KIND.ENUM:
200 if typedeps:
201 raise NotImplementedError((decl, typedeps))
202 return None
203 else:
204 return _check_typedep(decl, typedeps, types, knowntypes)
205
206
207 def _check_members(decl, typedeps, types, knowntypes):
208 if isinstance(typedeps, TypeDeclaration):
209 raise NotImplementedError((decl, typedeps))
210
211 #members = decl.members or () # A forward decl has no members.
212 members = decl.members
213 if not members:
214 # A forward decl has no members, but that shouldn't surface here..
215 raise NotImplementedError(decl)
216 if len(members) != len(typedeps):
217 raise NotImplementedError((decl, typedeps))
218
219 unsupported = []
220 for member, typedecl in zip(members, typedeps):
221 checked = _check_typedep(member, typedecl, types, knowntypes)
222 unsupported.append(checked)
223 if any(None if v is FIXED_TYPE else v for v in unsupported):
224 return unsupported
225 elif FIXED_TYPE in unsupported:
226 return FIXED_TYPE
227 else:
228 return None
229
230
231 def _check_typedep(decl, typedecl, types, knowntypes):
232 if not isinstance(typedecl, TypeDeclaration):
233 if hasattr(type(typedecl), '__len__'):
234 if len(typedecl) == 1:
235 typedecl, = typedecl
236 if typedecl is None:
237 # XXX Fail?
238 return 'typespec (missing)'
239 elif typedecl is _info.UNKNOWN:
240 if _has_other_supported_type(decl):
241 return None
242 # XXX Is this right?
243 return 'typespec (unknown)'
244 elif not isinstance(typedecl, TypeDeclaration):
245 raise NotImplementedError((decl, typedecl))
246
247 if isinstance(decl, Member):
248 return _check_vartype(decl, typedecl, types, knowntypes)
249 elif not isinstance(decl, Declaration):
250 raise NotImplementedError(decl)
251 elif decl.kind is KIND.TYPEDEF:
252 return _check_vartype(decl, typedecl, types, knowntypes)
253 elif decl.kind is KIND.VARIABLE:
254 if not is_process_global(decl):
255 return None
256 if _is_kwlist(decl):
257 return None
258 if _has_other_supported_type(decl):
259 return None
260 checked = _check_vartype(decl, typedecl, types, knowntypes)
261 return 'mutable' if checked is FIXED_TYPE else checked
262 else:
263 raise NotImplementedError(decl)
264
265
266 def _is_kwlist(decl):
267 # keywords for PyArg_ParseTupleAndKeywords()
268 # "static char *name[]" -> "static const char * const name[]"
269 # XXX These should be made const.
270 for relpath, name in _KWLIST_VARIANTS:
271 if decl.name == name:
272 if relpath == '*':
273 break
274 assert os.path.isabs(decl.file.filename)
275 relpath = os.path.normpath(relpath)
276 if decl.file.filename.endswith(os.path.sep + relpath):
277 break
278 else:
279 return False
280 vartype = ''.join(str(decl.vartype).split())
281 return vartype == 'char*[]'
282
283 def _is_local_static_mutex(decl):
284 if not hasattr(decl, "vartype"):
285 return False
286
287 if not hasattr(decl, "parent") or decl.parent is None:
288 # We only want to allow local variables
289 return False
290
291 vartype = decl.vartype
292 return (vartype.typespec == 'PyMutex') and (decl.storage == 'static')
293
294 def _has_other_supported_type(decl):
295 if hasattr(decl, 'file') and decl.file.filename.endswith('.c.h'):
296 assert 'clinic' in decl.file.filename, (decl,)
297 if decl.name == '_kwtuple':
298 return True
299 if _is_local_static_mutex(decl):
300 # GH-127081: Local static mutexes are used to
301 # wrap libc functions that aren't thread safe
302 return True
303 vartype = str(decl.vartype).split()
304 if vartype[0] == 'struct':
305 vartype = vartype[1:]
306 vartype = ''.join(vartype)
307 return vartype in _OTHER_SUPPORTED_TYPES
308
309
310 def _check_vartype(decl, typedecl, types, knowntypes):
311 """Return failure reason."""
312 checked = _check_typespec(decl, typedecl, types, knowntypes)
313 if checked:
314 return checked
315 if is_immutable(decl.vartype):
316 return None
317 if is_fixed_type(decl.vartype):
318 return FIXED_TYPE
319 return 'mutable'
320
321
322 def _check_typespec(decl, typedecl, types, knowntypes):
323 typespec = decl.vartype.typespec
324 if typedecl is not None:
325 found = types.get(typedecl)
326 if found is None:
327 found = knowntypes.get(typedecl)
328
329 if found is not None:
330 _, extra = found
331 if extra is None:
332 # XXX Under what circumstances does this happen?
333 extra = {}
334 unsupported = extra.get('unsupported')
335 if unsupported is FIXED_TYPE:
336 unsupported = None
337 return 'typespec' if unsupported else None
338 # Fall back to default known types.
339 if is_pots(typespec):
340 return None
341 elif is_system_type(typespec):
342 return None
343 elif is_funcptr(decl.vartype):
344 return None
345 return 'typespec'
346
347
348 class Analyzed(_info.Analyzed):
349
350 @classonly
351 def is_target(cls, raw):
352 if not super().is_target(raw):
353 return False
354 if raw.kind not in KINDS:
355 return False
356 return True
357
358 #@classonly
359 #def _parse_raw_result(cls, result, extra):
360 # typedecl, extra = super()._parse_raw_result(result, extra)
361 # if typedecl is None:
362 # return None, extra
363 # raise NotImplementedError
364
365 def __init__(self, item, typedecl=None, *, unsupported=None, **extra):
366 if 'unsupported' in extra:
367 raise NotImplementedError((item, typedecl, unsupported, extra))
368 if not unsupported:
369 unsupported = None
370 elif isinstance(unsupported, (str, TypeDeclaration)):
371 unsupported = (unsupported,)
372 elif unsupported is not FIXED_TYPE:
373 unsupported = tuple(unsupported)
374 self.unsupported = unsupported
375 extra['unsupported'] = self.unsupported # ...for __repr__(), etc.
376 if self.unsupported is None:
377 #self.supported = None
378 self.supported = True
379 elif self.unsupported is FIXED_TYPE:
380 if item.kind is KIND.VARIABLE:
381 raise NotImplementedError(item, typedecl, unsupported)
382 self.supported = True
383 else:
384 self.supported = not self.unsupported
385 super().__init__(item, typedecl, **extra)
386
387 def render(self, fmt='line', *, itemonly=False):
388 if fmt == 'raw':
389 yield repr(self)
390 return
391 rendered = super().render(fmt, itemonly=itemonly)
392 # XXX ???
393 #if itemonly:
394 # yield from rendered
395 supported = self.supported
396 if fmt in ('line', 'brief'):
397 rendered, = rendered
398 parts = [
399 '+' if supported else '-' if supported is False else '',
400 rendered,
401 ]
402 yield '\t'.join(parts)
403 elif fmt == 'summary':
404 raise NotImplementedError(fmt)
405 elif fmt == 'full':
406 yield from rendered
407 if supported:
408 yield f'\tsupported:\t{supported}'
409 else:
410 raise NotImplementedError(fmt)
411
412
413 class Analysis(_info.Analysis):
414 _item_class = Analyzed
415
416 @classonly
417 def build_item(cls, info, result=None):
418 if not isinstance(info, Declaration) or info.kind not in KINDS:
419 raise NotImplementedError((info, result))
420 return super().build_item(info, result)
421
422
423 def check_globals(analysis):
424 # yield (data, failure)
425 ignored = read_ignored()
426 for item in analysis:
427 if item.kind != KIND.VARIABLE:
428 continue
429 if item.supported:
430 continue
431 if item.id in ignored:
432 continue
433 reason = item.unsupported
434 if not reason:
435 reason = '???'
436 elif not isinstance(reason, str):
437 if len(reason) == 1:
438 reason, = reason
439 reason = f'({reason})'
440 yield item, f'not supported {reason:20}\t{item.storage or ""} {item.vartype}'