Debian Codegraph @

Tools/c-analyzer/c_analyzer/__main__.py

raw
1 import io
2 import logging
3 import os
4 import os.path
5 import re
6 import sys
7
8 from c_common import fsutil
9 from c_common.logging import VERBOSITY, Printer
10 from c_common.scriptutil import (
11 add_verbosity_cli,
12 add_traceback_cli,
13 add_sepval_cli,
14 add_progress_cli,
15 add_files_cli,
16 add_commands_cli,
17 process_args_by_key,
18 configure_logger,
19 get_prog,
20 filter_filenames,
21 )
22 from c_parser.info import KIND
23 from .match import filter_forward
24 from . import (
25 analyze as _analyze,
26 datafiles as _datafiles,
27 check_all as _check_all,
28 )
29
30
31 KINDS = [
32 KIND.TYPEDEF,
33 KIND.STRUCT,
34 KIND.UNION,
35 KIND.ENUM,
36 KIND.FUNCTION,
37 KIND.VARIABLE,
38 KIND.STATEMENT,
39 ]
40
41 logger = logging.getLogger(__name__)
42
43
44 #######################################
45 # table helpers
46
47 TABLE_SECTIONS = {
48 'types': (
49 ['kind', 'name', 'data', 'file'],
50 KIND.is_type_decl,
51 (lambda v: (v.kind.value, v.filename or '', v.name)),
52 ),
53 'typedefs': 'types',
54 'structs': 'types',
55 'unions': 'types',
56 'enums': 'types',
57 'functions': (
58 ['name', 'data', 'file'],
59 (lambda kind: kind is KIND.FUNCTION),
60 (lambda v: (v.filename or '', v.name)),
61 ),
62 'variables': (
63 ['name', 'parent', 'data', 'file'],
64 (lambda kind: kind is KIND.VARIABLE),
65 (lambda v: (v.filename or '', str(v.parent) if v.parent else '', v.name)),
66 ),
67 'statements': (
68 ['file', 'parent', 'data'],
69 (lambda kind: kind is KIND.STATEMENT),
70 (lambda v: (v.filename or '', str(v.parent) if v.parent else '', v.name)),
71 ),
72 KIND.TYPEDEF: 'typedefs',
73 KIND.STRUCT: 'structs',
74 KIND.UNION: 'unions',
75 KIND.ENUM: 'enums',
76 KIND.FUNCTION: 'functions',
77 KIND.VARIABLE: 'variables',
78 KIND.STATEMENT: 'statements',
79 }
80
81
82 def _render_table(items, columns, relroot=None):
83 # XXX improve this
84 header = '\t'.join(columns)
85 div = '--------------------'
86 yield header
87 yield div
88 total = 0
89 for item in items:
90 rowdata = item.render_rowdata(columns)
91 row = [rowdata[c] for c in columns]
92 if relroot and 'file' in columns:
93 index = columns.index('file')
94 row[index] = os.path.relpath(row[index], relroot)
95 yield '\t'.join(row)
96 total += 1
97 yield div
98 yield f'total: {total}'
99
100
101 def build_section(name, groupitems, *, relroot=None):
102 info = TABLE_SECTIONS[name]
103 while type(info) is not tuple:
104 if name in KINDS:
105 name = info
106 info = TABLE_SECTIONS[info]
107
108 columns, match_kind, sortkey = info
109 items = (v for v in groupitems if match_kind(v.kind))
110 items = sorted(items, key=sortkey)
111 def render():
112 yield ''
113 yield f'{name}:'
114 yield ''
115 for line in _render_table(items, columns, relroot):
116 yield line
117 return items, render
118
119
120 #######################################
121 # the checks
122
123 CHECKS = {
124 #'globals': _check_globals,
125 }
126
127
128 def add_checks_cli(parser, checks=None, *, add_flags=None):
129 default = False
130 if not checks:
131 checks = list(CHECKS)
132 default = True
133 elif isinstance(checks, str):
134 checks = [checks]
135 if (add_flags is None and len(checks) > 1) or default:
136 add_flags = True
137
138 process_checks = add_sepval_cli(parser, '--check', 'checks', checks)
139 if add_flags:
140 for check in checks:
141 parser.add_argument(f'--{check}', dest='checks',
142 action='append_const', const=check)
143 return [
144 process_checks,
145 ]
146
147
148 def _get_check_handlers(fmt, printer, verbosity=VERBOSITY):
149 div = None
150 def handle_after():
151 pass
152 if not fmt:
153 div = ''
154 def handle_failure(failure, data):
155 data = repr(data)
156 if verbosity >= 3:
157 logger.info(f'failure: {failure}')
158 logger.info(f'data: {data}')
159 else:
160 logger.warn(f'failure: {failure} (data: {data})')
161 elif fmt == 'raw':
162 def handle_failure(failure, data):
163 print(f'{failure!r} {data!r}')
164 elif fmt == 'brief':
165 def handle_failure(failure, data):
166 parent = data.parent or ''
167 funcname = parent if isinstance(parent, str) else parent.name
168 name = f'({funcname}).{data.name}' if funcname else data.name
169 failure = failure.split('\t')[0]
170 print(f'{data.filename}:{name} - {failure}')
171 elif fmt == 'summary':
172 def handle_failure(failure, data):
173 print(_fmt_one_summary(data, failure))
174 elif fmt == 'full':
175 div = ''
176 def handle_failure(failure, data):
177 name = data.shortkey if data.kind is KIND.VARIABLE else data.name
178 parent = data.parent or ''
179 funcname = parent if isinstance(parent, str) else parent.name
180 known = 'yes' if data.is_known else '*** NO ***'
181 print(f'{data.kind.value} {name!r} failed ({failure})')
182 print(f' file: {data.filename}')
183 print(f' func: {funcname or "-"}')
184 print(f' name: {data.name}')
185 print(f' data: ...')
186 print(f' type unknown: {known}')
187 else:
188 if fmt in FORMATS:
189 raise NotImplementedError(fmt)
190 raise ValueError(f'unsupported fmt {fmt!r}')
191 return handle_failure, handle_after, div
192
193
194 #######################################
195 # the formats
196
197 def fmt_raw(analysis):
198 for item in analysis:
199 yield from item.render('raw')
200
201
202 def fmt_brief(analysis):
203 # XXX Support sorting.
204 items = sorted(analysis)
205 for kind in KINDS:
206 if kind is KIND.STATEMENT:
207 continue
208 for item in items:
209 if item.kind is not kind:
210 continue
211 yield from item.render('brief')
212 yield f' total: {len(items)}'
213
214
215 def fmt_summary(analysis):
216 # XXX Support sorting and grouping.
217 items = list(analysis)
218 total = len(items)
219
220 def section(name):
221 _, render = build_section(name, items)
222 yield from render()
223
224 yield from section('types')
225 yield from section('functions')
226 yield from section('variables')
227 yield from section('statements')
228
229 yield ''
230 # yield f'grand total: {len(supported) + len(unsupported)}'
231 yield f'grand total: {total}'
232
233
234 def _fmt_one_summary(item, extra=None):
235 parent = item.parent or ''
236 funcname = parent if isinstance(parent, str) else parent.name
237 if extra:
238 return f'{item.filename:35}\t{funcname or "-":35}\t{item.name:40}\t{extra}'
239 else:
240 return f'{item.filename:35}\t{funcname or "-":35}\t{item.name}'
241
242
243 def fmt_full(analysis):
244 # XXX Support sorting.
245 items = sorted(analysis, key=lambda v: v.key)
246 yield ''
247 for item in items:
248 yield from item.render('full')
249 yield ''
250 yield f'total: {len(items)}'
251
252
253 FORMATS = {
254 'raw': fmt_raw,
255 'brief': fmt_brief,
256 'summary': fmt_summary,
257 'full': fmt_full,
258 }
259
260
261 def add_output_cli(parser, *, default='summary'):
262 parser.add_argument('--format', dest='fmt', default=default, choices=tuple(FORMATS))
263
264 def process_args(args, *, argv=None):
265 pass
266 return process_args
267
268
269 #######################################
270 # the commands
271
272 def _cli_check(parser, checks=None, **kwargs):
273 if isinstance(checks, str):
274 checks = [checks]
275 if checks is False:
276 process_checks = None
277 elif checks is None:
278 process_checks = add_checks_cli(parser)
279 elif len(checks) == 1 and type(checks) is not dict and re.match(r'^<.*>$', checks[0]):
280 check = checks[0][1:-1]
281 def process_checks(args, *, argv=None):
282 args.checks = [check]
283 else:
284 process_checks = add_checks_cli(parser, checks=checks)
285 process_progress = add_progress_cli(parser)
286 process_output = add_output_cli(parser, default=None)
287 process_files = add_files_cli(parser, **kwargs)
288 return [
289 process_checks,
290 process_progress,
291 process_output,
292 process_files,
293 ]
294
295
296 def cmd_check(filenames, *,
297 checks=None,
298 ignored=None,
299 fmt=None,
300 failfast=False,
301 iter_filenames=None,
302 relroot=fsutil.USE_CWD,
303 track_progress=None,
304 verbosity=VERBOSITY,
305 _analyze=_analyze,
306 _CHECKS=CHECKS,
307 **kwargs
308 ):
309 if not checks:
310 checks = _CHECKS
311 elif isinstance(checks, str):
312 checks = [checks]
313 checks = [_CHECKS[c] if isinstance(c, str) else c
314 for c in checks]
315 printer = Printer(verbosity)
316 (handle_failure, handle_after, div
317 ) = _get_check_handlers(fmt, printer, verbosity)
318
319 filenames, relroot = fsutil.fix_filenames(filenames, relroot=relroot)
320 filenames = filter_filenames(filenames, iter_filenames, relroot)
321 if track_progress:
322 filenames = track_progress(filenames)
323
324 logger.info('analyzing files...')
325 analyzed = _analyze(filenames, **kwargs)
326 analyzed.fix_filenames(relroot, normalize=False)
327 decls = filter_forward(analyzed, markpublic=True)
328
329 logger.info('checking analysis results...')
330 failed = []
331 for data, failure in _check_all(decls, checks, failfast=failfast):
332 if data is None:
333 printer.info('stopping after one failure')
334 break
335 if div is not None and len(failed) > 0:
336 printer.info(div)
337 failed.append(data)
338 handle_failure(failure, data)
339 handle_after()
340
341 printer.info('-------------------------')
342 logger.info(f'total failures: {len(failed)}')
343 logger.info('done checking')
344
345 if fmt == 'summary':
346 print('Categorized by storage:')
347 print()
348 from .match import group_by_storage
349 grouped = group_by_storage(failed, ignore_non_match=False)
350 for group, decls in grouped.items():
351 print()
352 print(group)
353 for decl in decls:
354 print(' ', _fmt_one_summary(decl))
355 print(f'subtotal: {len(decls)}')
356
357 if len(failed) > 0:
358 sys.exit(len(failed))
359
360
361 def _cli_analyze(parser, **kwargs):
362 process_progress = add_progress_cli(parser)
363 process_output = add_output_cli(parser)
364 process_files = add_files_cli(parser, **kwargs)
365 return [
366 process_progress,
367 process_output,
368 process_files,
369 ]
370
371
372 # XXX Support filtering by kind.
373 def cmd_analyze(filenames, *,
374 fmt=None,
375 iter_filenames=None,
376 relroot=fsutil.USE_CWD,
377 track_progress=None,
378 verbosity=None,
379 _analyze=_analyze,
380 formats=FORMATS,
381 **kwargs
382 ):
383 verbosity = verbosity if verbosity is not None else 3
384
385 try:
386 do_fmt = formats[fmt]
387 except KeyError:
388 raise ValueError(f'unsupported fmt {fmt!r}')
389
390 filenames, relroot = fsutil.fix_filenames(filenames, relroot=relroot)
391 filenames = filter_filenames(filenames, iter_filenames, relroot)
392 if track_progress:
393 filenames = track_progress(filenames)
394
395 logger.info('analyzing files...')
396 analyzed = _analyze(filenames, **kwargs)
397 analyzed.fix_filenames(relroot, normalize=False)
398 decls = filter_forward(analyzed, markpublic=True)
399
400 for line in do_fmt(decls):
401 print(line)
402
403
404 def _cli_data(parser, filenames=None, known=None):
405 ArgumentParser = type(parser)
406 common = ArgumentParser(add_help=False)
407 # These flags will get processed by the top-level parse_args().
408 add_verbosity_cli(common)
409 add_traceback_cli(common)
410
411 subs = parser.add_subparsers(dest='datacmd')
412
413 sub = subs.add_parser('show', parents=[common])
414 if known is None:
415 sub.add_argument('--known', required=True)
416 if filenames is None:
417 sub.add_argument('filenames', metavar='FILE', nargs='+')
418
419 sub = subs.add_parser('dump', parents=[common])
420 if known is None:
421 sub.add_argument('--known')
422 sub.add_argument('--show', action='store_true')
423 process_progress = add_progress_cli(sub)
424
425 sub = subs.add_parser('check', parents=[common])
426 if known is None:
427 sub.add_argument('--known', required=True)
428
429 def process_args(args, *, argv):
430 if args.datacmd == 'dump':
431 process_progress(args, argv)
432 return process_args
433
434
435 def cmd_data(datacmd, filenames, known=None, *,
436 _analyze=_analyze,
437 formats=FORMATS,
438 extracolumns=None,
439 relroot=fsutil.USE_CWD,
440 track_progress=None,
441 **kwargs
442 ):
443 kwargs.pop('verbosity', None)
444 usestdout = kwargs.pop('show', None)
445 if datacmd == 'show':
446 do_fmt = formats['summary']
447 if isinstance(known, str):
448 known, _ = _datafiles.get_known(known, extracolumns, relroot)
449 for line in do_fmt(known):
450 print(line)
451 elif datacmd == 'dump':
452 filenames, relroot = fsutil.fix_filenames(filenames, relroot=relroot)
453 if track_progress:
454 filenames = track_progress(filenames)
455 analyzed = _analyze(filenames, **kwargs)
456 analyzed.fix_filenames(relroot, normalize=False)
457 if known is None or usestdout:
458 outfile = io.StringIO()
459 _datafiles.write_known(analyzed, outfile, extracolumns,
460 relroot=relroot)
461 print(outfile.getvalue())
462 else:
463 _datafiles.write_known(analyzed, known, extracolumns,
464 relroot=relroot)
465 elif datacmd == 'check':
466 raise NotImplementedError(datacmd)
467 else:
468 raise ValueError(f'unsupported data command {datacmd!r}')
469
470
471 COMMANDS = {
472 'check': (
473 'analyze and fail if the given C source/header files have any problems',
474 [_cli_check],
475 cmd_check,
476 ),
477 'analyze': (
478 'report on the state of the given C source/header files',
479 [_cli_analyze],
480 cmd_analyze,
481 ),
482 'data': (
483 'check/manage local data (e.g. known types, ignored vars, caches)',
484 [_cli_data],
485 cmd_data,
486 ),
487 }
488
489
490 #######################################
491 # the script
492
493 def parse_args(argv=sys.argv[1:], prog=sys.argv[0], *, subset=None):
494 import argparse
495 parser = argparse.ArgumentParser(
496 prog=prog or get_prog(),
497 )
498
499 processors = add_commands_cli(
500 parser,
501 commands={k: v[1] for k, v in COMMANDS.items()},
502 commonspecs=[
503 add_verbosity_cli,
504 add_traceback_cli,
505 ],
506 subset=subset,
507 )
508
509 args = parser.parse_args(argv)
510 ns = vars(args)
511
512 cmd = ns.pop('cmd')
513
514 verbosity, traceback_cm = process_args_by_key(
515 args,
516 argv,
517 processors[cmd],
518 ['verbosity', 'traceback_cm'],
519 )
520 # "verbosity" is sent to the commands, so we put it back.
521 args.verbosity = verbosity
522
523 return cmd, ns, verbosity, traceback_cm
524
525
526 def main(cmd, cmd_kwargs):
527 try:
528 run_cmd = COMMANDS[cmd][0]
529 except KeyError:
530 raise ValueError(f'unsupported cmd {cmd!r}')
531 run_cmd(**cmd_kwargs)
532
533
534 if __name__ == '__main__':
535 cmd, cmd_kwargs, verbosity, traceback_cm = parse_args()
536 configure_logger(verbosity)
537 with traceback_cm:
538 main(cmd, cmd_kwargs)