Debian Codegraph @

llvm-project/clang/utils/analyzer/SATestBenchmark.py

raw
1 """
2 Static Analyzer qualification infrastructure.
3
4 This source file contains all the functionality related to benchmarking
5 the analyzer on a set projects. Right now, this includes measuring
6 execution time and peak memory usage. Benchmark runs analysis on every
7 project multiple times to get a better picture about the distribution
8 of measured values.
9
10 Additionally, this file includes a comparison routine for two benchmarking
11 results that plots the result together on one chart.
12 """
13
14 import SATestUtils as utils
15 from SATestBuild import ProjectTester, stdout, TestInfo
16 from ProjectMap import ProjectInfo
17
18 import pandas as pd
19 from typing import List, Tuple
20
21
22 INDEX_COLUMN = "index"
23
24
25 def _save(data: pd.DataFrame, file_path: str):
26 data.to_csv(file_path, index_label=INDEX_COLUMN)
27
28
29 def _load(file_path: str) -> pd.DataFrame:
30 return pd.read_csv(file_path, index_col=INDEX_COLUMN)
31
32
33 class Benchmark:
34 """
35 Becnhmark class encapsulates one functionality: it runs the analysis
36 multiple times for the given set of projects and stores results in the
37 specified file.
38 """
39 def __init__(self, projects: List[ProjectInfo], iterations: int,
40 output_path: str):
41 self.projects = projects
42 self.iterations = iterations
43 self.out = output_path
44
45 def run(self):
46 results = [self._benchmark_project(project)
47 for project in self.projects]
48
49 data = pd.concat(results, ignore_index=True)
50 _save(data, self.out)
51
52 def _benchmark_project(self, project: ProjectInfo) -> pd.DataFrame:
53 if not project.enabled:
54 stdout(f" \n\n--- Skipping disabled project {project.name}\n")
55 return
56
57 stdout(f" \n\n--- Benchmarking project {project.name}\n")
58
59 test_info = TestInfo(project)
60 tester = ProjectTester(test_info, silent=True)
61 project_dir = tester.get_project_dir()
62 output_dir = tester.get_output_dir()
63
64 raw_data = []
65
66 for i in range(self.iterations):
67 stdout(f"Iteration #{i + 1}")
68 time, mem = tester.build(project_dir, output_dir)
69 raw_data.append({"time": time, "memory": mem,
70 "iteration": i, "project": project.name})
71 stdout(f"time: {utils.time_to_str(time)}, "
72 f"peak memory: {utils.memory_to_str(mem)}")
73
74 return pd.DataFrame(raw_data)
75
76
77 def compare(old_path: str, new_path: str, plot_file: str):
78 """
79 Compare two benchmarking results stored as .csv files
80 and produce a plot in the specified file.
81 """
82 old = _load(old_path)
83 new = _load(new_path)
84
85 old_projects = set(old["project"])
86 new_projects = set(new["project"])
87 common_projects = old_projects & new_projects
88
89 # Leave only rows for projects common to both dataframes.
90 old = old[old["project"].isin(common_projects)]
91 new = new[new["project"].isin(common_projects)]
92
93 old, new = _normalize(old, new)
94
95 # Seaborn prefers all the data to be in one dataframe.
96 old["kind"] = "old"
97 new["kind"] = "new"
98 data = pd.concat([old, new], ignore_index=True)
99
100 # TODO: compare data in old and new dataframes using statistical tests
101 # to check if they belong to the same distribution
102 _plot(data, plot_file)
103
104
105 def _normalize(old: pd.DataFrame,
106 new: pd.DataFrame) -> Tuple[pd.DataFrame, pd.DataFrame]:
107 # This creates a dataframe with all numerical data averaged.
108 means = old.groupby("project").mean()
109 return _normalize_impl(old, means), _normalize_impl(new, means)
110
111
112 def _normalize_impl(data: pd.DataFrame, means: pd.DataFrame):
113 # Right now 'means' has one row corresponding to one project,
114 # while 'data' has N rows for each project (one for each iteration).
115 #
116 # In order for us to work easier with this data, we duplicate
117 # 'means' data to match the size of the 'data' dataframe.
118 #
119 # All the columns from 'data' will maintain their names, while
120 # new columns coming from 'means' will have "_mean" suffix.
121 joined_data = data.merge(means, on="project", suffixes=("", "_mean"))
122 _normalize_key(joined_data, "time")
123 _normalize_key(joined_data, "memory")
124 return joined_data
125
126
127 def _normalize_key(data: pd.DataFrame, key: str):
128 norm_key = _normalized_name(key)
129 mean_key = f"{key}_mean"
130 data[norm_key] = data[key] / data[mean_key]
131
132
133 def _normalized_name(name: str) -> str:
134 return f"normalized {name}"
135
136
137 def _plot(data: pd.DataFrame, plot_file: str):
138 import matplotlib
139 import seaborn as sns
140 from matplotlib import pyplot as plt
141
142 sns.set_style("whitegrid")
143 # We want to have time and memory charts one above the other.
144 figure, (ax1, ax2) = plt.subplots(2, 1, figsize=(8, 6))
145
146 def _subplot(key: str, ax: matplotlib.axes.Axes):
147 sns.boxplot(x="project", y=_normalized_name(key), hue="kind",
148 data=data, palette=sns.color_palette("BrBG", 2), ax=ax)
149
150 _subplot("time", ax1)
151 # No need to have xlabels on both top and bottom charts.
152 ax1.set_xlabel("")
153
154 _subplot("memory", ax2)
155 # The legend on the top chart is enough.
156 ax2.get_legend().remove()
157
158 figure.savefig(plot_file)