Debian Codegraph @

share/analysis-scripts/summary.py

raw
1 #!/usr/bin/env python3
2 # -*- coding: utf-8 -*-
3 ##########################################################################
4 # #
5 # This file is part of Frama-C. #
6 # #
7 # Copyright (C) 2007-2022 #
8 # CEA (Commissariat à l'énergie atomique et aux énergies #
9 # alternatives) #
10 # #
11 # you can redistribute it and/or modify it under the terms of the GNU #
12 # Lesser General Public License as published by the Free Software #
13 # Foundation, version 2.1. #
14 # #
15 # It is distributed in the hope that it will be useful, #
16 # but WITHOUT ANY WARRANTY; without even the implied warranty of #
17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
18 # GNU Lesser General Public License for more details. #
19 # #
20 # See the GNU Lesser General Public License version 2.1 #
21 # for more details (enclosed in the file licenses/LGPLv2.1). #
22 # #
23 ##########################################################################
24
25 import re
26 import sys
27 import subprocess
28 import time
29 import os
30 import signal
31 import argparse
32 import uuid
33
34 import frama_c_results
35 import results_display
36 import benchmark_database
37
38
39 class OperationException(Exception):
40 pass
41
42
43 def build_make_environment(framac):
44 if framac is None:
45 env = {**os.environ}
46 args = []
47 else:
48 env = { **os.environ, "PATH" : f"{framac}/bin:{os.environ['PATH']}" }
49 args = [
50 f"FRAMAC_BIN={framac}/bin",
51 f"FRAMAC={framac}/bin/frama-c"
52 ]
53 return env, args
54
55
56 def list_targets(dir):
57 if not os.path.isdir(dir):
58 raise OperationException(f"target is not a directory: {dir}")
59
60 env, args = build_make_environment(framac)
61 res = subprocess.run(
62 ["make", "--directory", dir, "--quiet", "display-targets"] + args,
63 env=env,
64 stdout=subprocess.PIPE,
65 encoding="ascii",
66 )
67 targets = res.stdout.split()
68 res = []
69 for target in targets:
70 if target.endswith(".eva") or target.endswith(".parse"):
71 res += [f"{dir}/{target}"]
72 else:
73 res += list_targets(target)
74 return res
75
76
77 def clone_frama_c(clonedir, hash):
78 print("Cloning Frama-C", hash, "...")
79 res = subprocess.run(
80 ["./scripts/clone.sh", "--clone-dir", clonedir, hash],
81 stdout=subprocess.PIPE,
82 encoding="ascii",
83 )
84 if res.returncode != 0:
85 raise OperationException("Cannot clone repository. Try to manually"
86 "remove the broken clone in " + clonedir)
87 return res.stdout.strip() + "/build"
88
89 def run_make(framac, benchmark_tag=None):
90 args = ["make", "--keep-going", "all"]
91 env, var_args = build_make_environment(framac)
92 if benchmark_tag is None:
93 args += ["-j", str(os.cpu_count())]
94 else:
95 args += ["BENCHMARK=" + benchmark_tag]
96 return subprocess.Popen(
97 args + var_args,
98 env=env,
99 stdout=subprocess.DEVNULL,
100 stderr=subprocess.PIPE,
101 preexec_fn=os.setsid,
102 )
103
104
105 def terminate_process(process):
106 if process is None:
107 return b""
108 else:
109 try:
110 os.killpg(os.getpgid(process.pid), signal.SIGTERM)
111 pass
112 except ProcessLookupError:
113 pass
114 output, errors = process.communicate()
115 return errors
116
117
118 def smart_rename(target):
119 target = re.sub("^\./", "", target)
120 target = re.sub("main\.eva$", "", target)
121 target = re.sub("\.eva$", "", target)
122 target = re.sub("\.frama-c/", "", target)
123 target = re.sub("qds/frama-c", "qds", target)
124 return target
125
126
127 def is_running(target):
128 return os.path.isfile(target + "/running")
129
130
131 def poll_results(targets, benchmark_tag):
132 results = []
133 for target in targets:
134 filename = target + "/stats.txt"
135 result = frama_c_results.read(filename)
136 result["target"] = target
137 result["target_name"] = smart_rename(target)
138 result["is_running"] = is_running(target)
139 result["up_to_date"] = benchmark_tag is None or benchmark_tag == result["benchmark_tag"]
140 results.append(result)
141 return results
142
143
144 def run_analyses(display, database, framac, benchmark_tag):
145 results = []
146 targets = list_targets(".")
147 process = run_make(framac, benchmark_tag)
148 errors = b""
149 next_poll = time.time()
150
151 def update():
152 nonlocal display, database, targets, benchmark_tag, results
153 results = poll_results(targets, benchmark_tag)
154 if not database is None:
155 database.update(results)
156 display.needs_update = True
157
158 try:
159 while process.poll() is None:
160 if time.time() >= next_poll:
161 update()
162 next_poll = time.time() + 2.0
163 display.process_inputs()
164 if display.needs_update:
165 display.print_table(results)
166 time.sleep(0.05)
167 update()
168 except (KeyboardInterrupt, results_display.UserExitRequest):
169 print("Analyzes interrupted by user.")
170 except Exception as e:
171 # terminate_process below is somehow blocking the exception printing
172 errors += bytearray(str(e), "ascii")
173 raise e
174 finally:
175 errors += terminate_process(process)
176 return results, errors
177
178
179 parser = argparse.ArgumentParser(
180 description="Run analyses and summarize the results. Must be run in a "
181 "directory with a Makefile having two rules: 'all', a target "
182 "that runs the analysis, and 'display-targets', the target that "
183 "lists the built results.")
184 parser.add_argument("rev", nargs="?", metavar="REVISION",
185 help="a Frama-C revision to use for analyses (default: use the "
186 "default configuration for Frama-C)")
187 parser.add_argument("-b", "--benchmark",
188 action="store_true",
189 help="sets benchmark mode: do not run analyses in parallel and rerun all "
190 "analyses")
191 parser.add_argument("-v", "--vs",
192 action="store", metavar="REVISION", default="master",
193 help="a revision to compare the results to")
194 parser.add_argument("-c", "--comment",
195 action="store", metavar="COMMENT",
196 help="when benchmarking, add this comment inside the database")
197 parser.add_argument("-p", "--repository-path",
198 action="store", metavar="PATH",
199 help="don't clone Frama-C, use this git repository instead")
200
201
202 errors = b""
203
204 try:
205 args = parser.parse_args()
206
207 if args.repository_path is None:
208 if args.rev is None:
209 gitdir = None
210 framac = None
211 else:
212 clonedir = "./frama-c-clones"
213 gitdir = clonedir + "/frama-c.git"
214 framac = clone_frama_c(clonedir, args.rev)
215 else:
216 framac = os.path.abspath(args.repository_path)
217 gitdir = framac
218
219 if args.benchmark:
220 benchmark_tag = str(uuid.uuid1())
221 print("Running benchmarks with benchmark tag", benchmark_tag, "...")
222 else:
223 benchmark_tag = None
224 print("Running analyses ...")
225
226 benchmark_comment = args.comment
227
228 if gitdir is None:
229 database = None
230 else:
231 database = benchmark_database.Database(benchmark_tag, benchmark_comment,
232 gitdir, args.rev, args.vs)
233
234 results,errors = results_display.wrapper(run_analyses, database, framac,
235 benchmark_tag, curses=True)
236
237 print("Results:\n")
238 results_display.PlainDisplay().print_table(results)
239
240 except OperationException as e:
241 errors += bytearray(str(e), "ascii")
242
243 sys.stderr.buffer.write(errors + b"\n")