Debian Codegraph @

share/analysis-scripts/benchmark_database.py

raw
1 ##########################################################################
2 # #
3 # This file is part of Frama-C. #
4 # #
5 # Copyright (C) 2007-2022 #
6 # CEA (Commissariat à l'énergie atomique et aux énergies #
7 # alternatives) #
8 # #
9 # you can redistribute it and/or modify it under the terms of the GNU #
10 # Lesser General Public License as published by the Free Software #
11 # Foundation, version 2.1. #
12 # #
13 # It is distributed in the hope that it will be useful, #
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of #
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
16 # GNU Lesser General Public License for more details. #
17 # #
18 # See the GNU Lesser General Public License version 2.1 #
19 # for more details (enclosed in the file licenses/LGPLv2.1). #
20 # #
21 ##########################################################################
22
23 import os
24 import csv
25 import sqlite3
26
27 import git_utils
28
29
30 def dict_factory(cursor, row):
31 d = {}
32 for idx, col in enumerate(cursor.description):
33 d[col[0]] = row[idx]
34 return d
35
36
37 class Database:
38 inserted_targets = {}
39
40 def __init__(self, benchmark_tag, benchmark_comment, gitdir, analyzer_rev, reference_rev):
41 self.benchmark_tag = benchmark_tag
42 self.benchmark_comment = benchmark_comment
43 if analyzer_rev is None:
44 self.analyzer_hash = None
45 self.analyzer = None
46 else:
47 self.analyzer_hash = git_utils.rev_parse(gitdir, analyzer_rev)
48 self.analyzer = git_utils.name_rev(gitdir, analyzer_rev)
49 self.reference_hash = git_utils.rev_parse(gitdir, reference_rev)
50 self.connection = sqlite3.connect("benchmark-results.db")
51 self.connection.row_factory = dict_factory
52 self.setup_rdb()
53 self.reference_results = self.query_rdb(self.reference_hash)
54
55 def update(self, results):
56 if not self.benchmark_tag is None:
57 for result in results:
58 inserted = result["target"] in self.inserted_targets
59 if result["up_to_date"] and not inserted:
60 self.insert(result)
61 self.inserted_targets[result["target"]] = True
62
63 for result in results:
64 if result["target"] in self.reference_results:
65 ref = self.reference_results[result["target"]]
66
67 def compute_diff(column, ratio):
68 nonlocal result, ref
69 try:
70 if ratio:
71 diff = result[column] / ref[column] - 1.0
72 else:
73 diff = result[column] - ref[column]
74 except TypeError:
75 diff = None
76 result["diff_" + column] = diff
77
78 compute_diff("alarms", False)
79 compute_diff("warnings", False)
80 compute_diff("user_time", True)
81 compute_diff("memory", True)
82 compute_diff("coverage", False)
83 else:
84 result["diff_alarms"] = None
85 result["diff_warnings"] = None
86 result["diff_user_time"] = None
87 result["diff_memory"] = None
88 result["diff_coverage"] = None
89
90 def insert_csv(self, result):
91 filename = "benchmark-results.csv"
92 file_already_exists = os.path.isfile(filename)
93 fieldnames = [
94 "benchmark_tag",
95 "timestamp",
96 "analyzer",
97 "analyzer_hash",
98 "target",
99 "target_hash",
100 "user_time",
101 "memory",
102 "alarms",
103 "warnings",
104 "sem_reach_fun",
105 "syn_reach_fun",
106 "total_fun",
107 "sem_reach_stmt",
108 "syn_reach_stmt",
109 "cmd_args",
110 "benchmark_comment",
111 ]
112 with open(filename, "a", newline="") as file:
113 writer = csv.DictWriter(
114 file, fieldnames=fieldnames, extrasaction="ignore", delimiter="\t", quotechar='"'
115 )
116 if not file_already_exists:
117 writer.writeheader()
118 writer.writerow(result)
119
120 def insert(self, result):
121 completed_result = {
122 **result,
123 "benchmark_tag": self.benchmark_tag,
124 "benchmark_comment": self.benchmark_comment,
125 "target_hash": git_utils.current_rev(result["target"]),
126 "analyzer": self.analyzer,
127 "analyzer_hash": self.analyzer_hash,
128 }
129 self.insert_csv(completed_result)
130 self.insert_rdb(completed_result)
131
132 def setup_rdb(self):
133 cursor = self.connection.cursor()
134 cursor.execute(
135 "CREATE TABLE IF NOT EXISTS benchmark_results ("
136 "benchmark_tag TEXT NOT NULL,"
137 "timestamp TEXT NOT NULL,"
138 "analyzer TEXT NOT NULL,"
139 "analyzer_hash TEXT NOT NULL,"
140 "target TEXT NOT NULL,"
141 "target_hash TEXT NOT NULL,"
142 "user_time REAL NOT NULL,"
143 "memory INTEGER NOT NULL,"
144 "alarms INTEGER NOT NULL,"
145 "warnings INTEGER NOT NULL,"
146 "sem_reach_fun INTEGER NOT NULL,"
147 "syn_reach_fun INTEGER NOT NULL,"
148 "total_fun INTEGER NOT NULL,"
149 "sem_reach_stmt INTEGER NOT NULL,"
150 "syn_reach_stmt INTEGER NOT NULL,"
151 "cmd_args TEXT NOT NULL,"
152 "benchmark_comment TEXT);"
153 )
154 self.connection.commit()
155
156 def insert_rdb(self, result):
157 cursor = self.connection.cursor()
158 cursor.execute(
159 "INSERT INTO benchmark_results("
160 "benchmark_tag, timestamp, "
161 "analyzer, analyzer_hash, target, target_hash, "
162 "user_time, memory, alarms, warnings, "
163 "sem_reach_fun, syn_reach_fun, total_fun, "
164 "sem_reach_stmt, syn_reach_stmt, "
165 "cmd_args, benchmark_comment) "
166 "VALUES("
167 "DATETIME('now','localtime'), "
168 ":benchmark_tag, :analyzer, "
169 ":analyzer_hash, :target, :target_hash, "
170 ":user_time, :memory, :alarms, :warnings, "
171 ":sem_reach_fun, :syn_reach_fun, :total_fun, "
172 ":sem_reach_stmt, :syn_reach_stmt, "
173 ":cmd_args, :benchmark_comment)",
174 result,
175 )
176 self.connection.commit()
177
178 def query_rdb(self, analyzer_hash):
179 cursor = self.connection.cursor()
180 cursor.execute(
181 "SELECT "
182 "target, "
183 "avg(user_time) as user_time, avg(memory) as memory, "
184 "min(alarms) as alarms, min(warnings) as warnings, "
185 "max(sem_reach_stmt) as sem_reach_stmt, "
186 "max(syn_reach_stmt) as syn_reach_stmt "
187 "FROM benchmark_results "
188 "WHERE analyzer_hash=? "
189 "GROUP BY target",
190 (analyzer_hash,),
191 )
192 results = {}
193 for r in cursor.fetchall():
194 r["coverage"] = r["sem_reach_stmt"] / r["syn_reach_stmt"]
195 results[r["target"]] = r
196 return results