Debian Codegraph @

llvm-project/clang/utils/analyzer/SATest.py

raw
1 #!/usr/bin/env python
2
3 import argparse
4 import sys
5 import os
6
7 from subprocess import call
8
9 SCRIPTS_DIR = os.path.dirname(os.path.realpath(__file__))
10 PROJECTS_DIR = os.path.join(SCRIPTS_DIR, "projects")
11 DEFAULT_LLVM_DIR = os.path.realpath(os.path.join(SCRIPTS_DIR,
12 os.path.pardir,
13 os.path.pardir,
14 os.path.pardir))
15
16
17 def add(parser, args):
18 import SATestAdd
19 from ProjectMap import ProjectInfo
20
21 if args.source == "git" and (args.origin == "" or args.commit == ""):
22 parser.error(
23 "Please provide both --origin and --commit if source is 'git'")
24
25 if args.source != "git" and (args.origin != "" or args.commit != ""):
26 parser.error("Options --origin and --commit don't make sense when "
27 "source is not 'git'")
28
29 project = ProjectInfo(args.name[0], args.mode, args.source, args.origin,
30 args.commit)
31
32 SATestAdd.add_new_project(project)
33
34
35 def build(parser, args):
36 import SATestBuild
37
38 SATestBuild.VERBOSE = args.verbose
39
40 projects = get_projects(parser, args)
41 tester = SATestBuild.RegressionTester(args.jobs,
42 projects,
43 args.override_compiler,
44 args.extra_analyzer_config,
45 args.extra_checkers,
46 args.regenerate,
47 args.strictness)
48 tests_passed = tester.test_all()
49
50 if not tests_passed:
51 sys.stderr.write("ERROR: Tests failed.\n")
52 sys.exit(42)
53
54
55 def compare(parser, args):
56 import CmpRuns
57
58 choices = [CmpRuns.HistogramType.RELATIVE.value,
59 CmpRuns.HistogramType.LOG_RELATIVE.value,
60 CmpRuns.HistogramType.ABSOLUTE.value]
61
62 if args.histogram is not None and args.histogram not in choices:
63 parser.error("Incorrect histogram type, available choices are {}"
64 .format(choices))
65
66 dir_old = CmpRuns.ResultsDirectory(args.old[0], args.root_old)
67 dir_new = CmpRuns.ResultsDirectory(args.new[0], args.root_new)
68
69 CmpRuns.dump_scan_build_results_diff(dir_old, dir_new,
70 show_stats=args.show_stats,
71 stats_only=args.stats_only,
72 histogram=args.histogram,
73 verbose_log=args.verbose_log)
74
75
76 def update(parser, args):
77 import SATestUpdateDiffs
78 from ProjectMap import ProjectMap
79
80 project_map = ProjectMap()
81 for project in project_map.projects:
82 SATestUpdateDiffs.update_reference_results(project, args.git)
83
84
85 def benchmark(parser, args):
86 from SATestBenchmark import Benchmark
87
88 projects = get_projects(parser, args)
89 benchmark = Benchmark(projects, args.iterations, args.output)
90 benchmark.run()
91
92
93 def benchmark_compare(parser, args):
94 import SATestBenchmark
95 SATestBenchmark.compare(args.old, args.new, args.output)
96
97
98 def get_projects(parser, args):
99 from ProjectMap import ProjectMap, Size
100
101 project_map = ProjectMap()
102 projects = project_map.projects
103
104 def filter_projects(projects, predicate, force=False):
105 return [project.with_fields(enabled=(force or project.enabled) and
106 predicate(project))
107 for project in projects]
108
109 if args.projects:
110 projects_arg = args.projects.split(",")
111 available_projects = [project.name
112 for project in projects]
113
114 # validate that given projects are present in the project map file
115 for manual_project in projects_arg:
116 if manual_project not in available_projects:
117 parser.error("Project '{project}' is not found in "
118 "the project map file. Available projects are "
119 "{all}.".format(project=manual_project,
120 all=available_projects))
121
122 projects = filter_projects(projects, lambda project:
123 project.name in projects_arg,
124 force=True)
125
126 try:
127 max_size = Size.from_str(args.max_size)
128 except ValueError as e:
129 parser.error("{}".format(e))
130
131 projects = filter_projects(projects, lambda project:
132 project.size <= max_size)
133
134 return projects
135
136
137 def docker(parser, args):
138 if len(args.rest) > 0:
139 if args.rest[0] != "--":
140 parser.error("REST arguments should start with '--'")
141 args.rest = args.rest[1:]
142
143 if args.build_image:
144 docker_build_image()
145 elif args.shell:
146 docker_shell(args)
147 else:
148 sys.exit(docker_run(args, ' '.join(args.rest)))
149
150
151 def docker_build_image():
152 sys.exit(call("docker build --tag satest-image {}".format(SCRIPTS_DIR),
153 shell=True))
154
155
156 def docker_shell(args):
157 try:
158 # First we need to start the docker container in a waiting mode,
159 # so it doesn't do anything, but most importantly keeps working
160 # while the shell session is in progress.
161 docker_run(args, "--wait", "--detach")
162 # Since the docker container is running, we can actually connect to it
163 call("docker exec -it satest bash", shell=True)
164
165 except KeyboardInterrupt:
166 pass
167
168 finally:
169 docker_cleanup()
170
171
172 def docker_run(args, command, docker_args=""):
173 try:
174 return call("docker run --rm --name satest "
175 "-v {llvm}:/llvm-project "
176 "-v {build}:/build "
177 "-v {clang}:/analyzer "
178 "-v {scripts}:/scripts "
179 "-v {projects}:/projects "
180 "{docker_args} "
181 "satest-image:latest {command}"
182 .format(llvm=args.llvm_project_dir,
183 build=args.build_dir,
184 clang=args.clang_dir,
185 scripts=SCRIPTS_DIR,
186 projects=PROJECTS_DIR,
187 docker_args=docker_args,
188 command=command),
189 shell=True)
190
191 except KeyboardInterrupt:
192 docker_cleanup()
193
194
195 def docker_cleanup():
196 print("Please wait for docker to clean up")
197 call("docker stop satest", shell=True)
198
199
200 def main():
201 parser = argparse.ArgumentParser()
202 subparsers = parser.add_subparsers()
203
204 # add subcommand
205 add_parser = subparsers.add_parser(
206 "add",
207 help="Add a new project for the analyzer testing.")
208 # TODO: Add an option not to build.
209 # TODO: Set the path to the Repository directory.
210 add_parser.add_argument("name", nargs=1, help="Name of the new project")
211 add_parser.add_argument("--mode", action="store", default=1, type=int,
212 choices=[0, 1, 2],
213 help="Build mode: 0 for single file project, "
214 "1 for scan_build, "
215 "2 for single file c++11 project")
216 add_parser.add_argument("--source", action="store", default="script",
217 choices=["script", "git", "zip"],
218 help="Source type of the new project: "
219 "'git' for getting from git "
220 "(please provide --origin and --commit), "
221 "'zip' for unpacking source from a zip file, "
222 "'script' for downloading source by running "
223 "a custom script")
224 add_parser.add_argument("--origin", action="store", default="",
225 help="Origin link for a git repository")
226 add_parser.add_argument("--commit", action="store", default="",
227 help="Git hash for a commit to checkout")
228 add_parser.set_defaults(func=add)
229
230 # build subcommand
231 build_parser = subparsers.add_parser(
232 "build",
233 help="Build projects from the project map and compare results with "
234 "the reference.")
235 build_parser.add_argument("--strictness", dest="strictness",
236 type=int, default=0,
237 help="0 to fail on runtime errors, 1 to fail "
238 "when the number of found bugs are different "
239 "from the reference, 2 to fail on any "
240 "difference from the reference. Default is 0.")
241 build_parser.add_argument("-r", dest="regenerate", action="store_true",
242 default=False,
243 help="Regenerate reference output.")
244 build_parser.add_argument("--override-compiler", action="store_true",
245 default=False, help="Call scan-build with "
246 "--override-compiler option.")
247 build_parser.add_argument("-j", "--jobs", dest="jobs",
248 type=int, default=0,
249 help="Number of projects to test concurrently")
250 build_parser.add_argument("--extra-analyzer-config",
251 dest="extra_analyzer_config", type=str,
252 default="",
253 help="Arguments passed to to -analyzer-config")
254 build_parser.add_argument("--extra-checkers",
255 dest="extra_checkers", type=str,
256 default="",
257 help="Extra checkers to enable")
258 build_parser.add_argument("--projects", action="store", default="",
259 help="Comma-separated list of projects to test")
260 build_parser.add_argument("--max-size", action="store", default=None,
261 help="Maximum size for the projects to test")
262 build_parser.add_argument("-v", "--verbose", action="count", default=0)
263 build_parser.set_defaults(func=build)
264
265 # compare subcommand
266 cmp_parser = subparsers.add_parser(
267 "compare",
268 help="Comparing two static analyzer runs in terms of "
269 "reported warnings and execution time statistics.")
270 cmp_parser.add_argument("--root-old", dest="root_old",
271 help="Prefix to ignore on source files for "
272 "OLD directory",
273 action="store", type=str, default="")
274 cmp_parser.add_argument("--root-new", dest="root_new",
275 help="Prefix to ignore on source files for "
276 "NEW directory",
277 action="store", type=str, default="")
278 cmp_parser.add_argument("--verbose-log", dest="verbose_log",
279 help="Write additional information to LOG "
280 "[default=None]",
281 action="store", type=str, default=None,
282 metavar="LOG")
283 cmp_parser.add_argument("--stats-only", action="store_true",
284 dest="stats_only", default=False,
285 help="Only show statistics on reports")
286 cmp_parser.add_argument("--show-stats", action="store_true",
287 dest="show_stats", default=False,
288 help="Show change in statistics")
289 cmp_parser.add_argument("--histogram", action="store", default=None,
290 help="Show histogram of paths differences. "
291 "Requires matplotlib")
292 cmp_parser.add_argument("old", nargs=1, help="Directory with old results")
293 cmp_parser.add_argument("new", nargs=1, help="Directory with new results")
294 cmp_parser.set_defaults(func=compare)
295
296 # update subcommand
297 upd_parser = subparsers.add_parser(
298 "update",
299 help="Update static analyzer reference results based on the previous "
300 "run of SATest build. Assumes that SATest build was just run.")
301 upd_parser.add_argument("--git", action="store_true",
302 help="Stage updated results using git.")
303 upd_parser.set_defaults(func=update)
304
305 # docker subcommand
306 dock_parser = subparsers.add_parser(
307 "docker",
308 help="Run regression system in the docker.")
309
310 dock_parser.add_argument("--build-image", action="store_true",
311 help="Build docker image for running tests.")
312 dock_parser.add_argument("--shell", action="store_true",
313 help="Start a shell on docker.")
314 dock_parser.add_argument("--llvm-project-dir", action="store",
315 default=DEFAULT_LLVM_DIR,
316 help="Path to LLVM source code. Defaults "
317 "to the repo where this script is located. ")
318 dock_parser.add_argument("--build-dir", action="store", default="",
319 help="Path to a directory where docker should "
320 "build LLVM code.")
321 dock_parser.add_argument("--clang-dir", action="store", default="",
322 help="Path to find/install LLVM installation.")
323 dock_parser.add_argument("rest", nargs=argparse.REMAINDER, default=[],
324 help="Additional args that will be forwarded "
325 "to the docker's entrypoint.")
326 dock_parser.set_defaults(func=docker)
327
328 # benchmark subcommand
329 bench_parser = subparsers.add_parser(
330 "benchmark",
331 help="Run benchmarks by building a set of projects multiple times.")
332
333 bench_parser.add_argument("-i", "--iterations", action="store",
334 type=int, default=20,
335 help="Number of iterations for building each "
336 "project.")
337 bench_parser.add_argument("-o", "--output", action="store",
338 default="benchmark.csv",
339 help="Output csv file for the benchmark results")
340 bench_parser.add_argument("--projects", action="store", default="",
341 help="Comma-separated list of projects to test")
342 bench_parser.add_argument("--max-size", action="store", default=None,
343 help="Maximum size for the projects to test")
344 bench_parser.set_defaults(func=benchmark)
345
346 bench_subparsers = bench_parser.add_subparsers()
347 bench_compare_parser = bench_subparsers.add_parser(
348 "compare",
349 help="Compare benchmark runs.")
350 bench_compare_parser.add_argument("--old", action="store", required=True,
351 help="Benchmark reference results to "
352 "compare agains.")
353 bench_compare_parser.add_argument("--new", action="store", required=True,
354 help="New benchmark results to check.")
355 bench_compare_parser.add_argument("-o", "--output",
356 action="store", required=True,
357 help="Output file for plots.")
358 bench_compare_parser.set_defaults(func=benchmark_compare)
359
360 args = parser.parse_args()
361 args.func(parser, args)
362
363
364 if __name__ == "__main__":
365 main()