Debian Codegraph @

scripts/performance/summarize

raw
1 #!/usr/bin/env python
2 """
3 Summarizes results of benchmarking.
4
5 Usage
6 =====
7
8 Run this script with::
9
10 ./summarize performance.csv
11
12
13 And that should output::
14
15 +------------------------+----------+----------------------+
16 | Metric over 1 run(s) | Mean | Standard Deviation |
17 +========================+==========+======================+
18 | Total Time (seconds) | 1.200 | 0.0 |
19 +------------------------+----------+----------------------+
20 | Maximum Memory | 42.3 MiB | 0 Bytes |
21 +------------------------+----------+----------------------+
22 | Maximum CPU (percent) | 88.1 | 0.0 |
23 +------------------------+----------+----------------------+
24 | Average Memory | 33.9 MiB | 0 Bytes |
25 +------------------------+----------+----------------------+
26 | Average CPU (percent) | 30.5 | 0.0 |
27 +------------------------+----------+----------------------+
28
29
30 The script can also be ran with multiple files:
31
32 ./summarize performance.csv performance-2.csv
33
34 And will have a similar output:
35
36 +------------------------+----------+----------------------+
37 | Metric over 2 run(s) | Mean | Standard Deviation |
38 +========================+==========+======================+
39 | Total Time (seconds) | 1.155 | 0.0449999570847 |
40 +------------------------+----------+----------------------+
41 | Maximum Memory | 42.5 MiB | 110.0 KiB |
42 +------------------------+----------+----------------------+
43 | Maximum CPU (percent) | 94.5 | 6.45 |
44 +------------------------+----------+----------------------+
45 | Average Memory | 35.6 MiB | 1.7 MiB |
46 +------------------------+----------+----------------------+
47 | Average CPU (percent) | 27.5 | 3.03068181818 |
48 +------------------------+----------+----------------------+
49
50
51 You can also specify the ``--output-format json`` option to print the
52 summary as JSON instead of a pretty printed table::
53
54 {
55 "total_time": 72.76999998092651,
56 "std_dev_average_memory": 0.0,
57 "std_dev_total_time": 0.0,
58 "average_memory": 56884518.57534247,
59 "std_dev_average_cpu": 0.0,
60 "std_dev_max_memory": 0.0,
61 "average_cpu": 61.19315068493151,
62 "max_memory": 58331136.0
63 }
64
65 """
66
67 import argparse
68 import csv
69 import json
70 from math import sqrt
71
72 from tabulate import tabulate
73
74
75 def human_readable_size(value):
76 """Converts integer values in bytes to human readable values"""
77 hummanize_suffixes = ('KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB')
78 base = 1024
79 bytes_int = float(value)
80
81 if bytes_int == 1:
82 return '1 Byte'
83 elif bytes_int < base:
84 return f'{bytes_int} Bytes'
85
86 for i, suffix in enumerate(hummanize_suffixes):
87 unit = base ** (i + 2)
88 if round((bytes_int / unit) * base) < base:
89 return f'{(base * bytes_int / unit):.1f} {suffix}'
90
91
92 class Summarizer:
93 DATA_INDEX_IN_ROW = {'time': 0, 'memory': 1, 'cpu': 2}
94
95 def __init__(self):
96 self.total_files = 0
97 self._num_rows = 0
98 self._start_time = None
99 self._end_time = None
100 self._totals = {
101 'time': [],
102 'average_memory': [],
103 'average_cpu': [],
104 'max_memory': [],
105 'max_cpu': [],
106 }
107 self._averages = {
108 'memory': 0.0,
109 'cpu': 0.0,
110 }
111 self._maximums = {'memory': 0.0, 'cpu': 0.0}
112
113 @property
114 def total_time(self):
115 return self._average_across_all_files('time')
116
117 @property
118 def max_cpu(self):
119 return self._average_across_all_files('max_cpu')
120
121 @property
122 def max_memory(self):
123 return self._average_across_all_files('max_memory')
124
125 @property
126 def average_cpu(self):
127 return self._average_across_all_files('average_cpu')
128
129 @property
130 def average_memory(self):
131 return self._average_across_all_files('average_memory')
132
133 @property
134 def std_dev_total_time(self):
135 return self._standard_deviation_across_all_files('time')
136
137 @property
138 def std_dev_max_cpu(self):
139 return self._standard_deviation_across_all_files('max_cpu')
140
141 @property
142 def std_dev_max_memory(self):
143 return self._standard_deviation_across_all_files('max_memory')
144
145 @property
146 def std_dev_average_cpu(self):
147 return self._standard_deviation_across_all_files('average_cpu')
148
149 @property
150 def std_dev_average_memory(self):
151 return self._standard_deviation_across_all_files('average_memory')
152
153 def _average_across_all_files(self, name):
154 return sum(self._totals[name]) / len(self._totals[name])
155
156 def _standard_deviation_across_all_files(self, name):
157 mean = self._average_across_all_files(name)
158 differences = [total - mean for total in self._totals[name]]
159 sq_differences = [difference**2 for difference in differences]
160 return sqrt(sum(sq_differences) / len(self._totals[name]))
161
162 def summarize_as_table(self):
163 """Formats the processed data as pretty printed table.
164
165 :return: str of formatted table
166 """
167 h = human_readable_size
168 table = [
169 [
170 'Total Time (seconds)',
171 f'{self.total_time:.3f}',
172 self.std_dev_total_time,
173 ],
174 ['Maximum Memory', h(self.max_memory), h(self.std_dev_max_memory)],
175 [
176 'Maximum CPU (percent)',
177 f'{self.max_cpu:.1f}',
178 self.std_dev_max_cpu,
179 ],
180 [
181 'Average Memory',
182 h(self.average_memory),
183 h(self.std_dev_average_memory),
184 ],
185 [
186 'Average CPU (percent)',
187 f'{self.average_cpu:.1f}',
188 self.std_dev_average_cpu,
189 ],
190 ]
191 return tabulate(
192 table,
193 headers=[
194 f'Metric over {self.total_files} run(s)',
195 'Mean',
196 'Standard Deviation',
197 ],
198 tablefmt="grid",
199 )
200
201 def summarize_as_json(self):
202 """Return JSON summary of processed data.
203
204 :return: str of formatted JSON
205 """
206 return json.dumps(
207 {
208 'total_time': self.total_time,
209 'std_dev_total_time': self.std_dev_total_time,
210 'max_memory': self.max_memory,
211 'std_dev_max_memory': self.std_dev_max_memory,
212 'average_memory': self.average_memory,
213 'std_dev_average_memory': self.std_dev_average_memory,
214 'std_dev_max_cpu': self.std_dev_max_cpu,
215 'max_cpu': self.max_cpu,
216 'average_cpu': self.average_cpu,
217 'std_dev_average_cpu': self.std_dev_average_cpu,
218 },
219 indent=2,
220 )
221
222 def process(self, args):
223 """Processes the data from the CSV file"""
224 for benchmark_file in args.benchmark_files:
225 self.process_individual_file(benchmark_file)
226 self.total_files += 1
227
228 def process_individual_file(self, benchmark_file):
229 with open(benchmark_file) as f:
230 reader = csv.reader(f)
231 # Process each row from the CSV file
232 row = None
233 for row in reader:
234 self._validate_row(row, benchmark_file)
235 self.process_data_row(row)
236 self._validate_row(row, benchmark_file)
237 self._end_time = self._get_time(row)
238 self._finalize_processed_data_for_file()
239
240 def _validate_row(self, row, filename):
241 if not row:
242 raise RuntimeError(
243 f'Row: {row} could not be processed. The CSV file ({filename}) may be '
244 'empty.'
245 )
246
247 def process_data_row(self, row):
248 # If the row is the first row collect the start time.
249 if self._num_rows == 0:
250 self._start_time = self._get_time(row)
251 self._num_rows += 1
252 self.process_data_point(row, 'memory')
253 self.process_data_point(row, 'cpu')
254
255 def process_data_point(self, row, name):
256 # Determine where in the CSV row the requested data is located.
257 index = self.DATA_INDEX_IN_ROW[name]
258 # Get the data point.
259 data_point = float(row[index])
260 self._add_to_average(name, data_point)
261 self._account_for_maximum(name, data_point)
262
263 def _finalize_processed_data_for_file(self):
264 # Add numbers to the total, which keeps track of data over
265 # all files provided.
266 self._totals['time'].append(self._end_time - self._start_time)
267 self._totals['max_cpu'].append(self._maximums['cpu'])
268 self._totals['max_memory'].append(self._maximums['memory'])
269 self._totals['average_cpu'].append(
270 self._averages['cpu'] / self._num_rows
271 )
272 self._totals['average_memory'].append(
273 self._averages['memory'] / self._num_rows
274 )
275
276 # Reset some of the data needed to be tracked for each specific
277 # file.
278 self._num_rows = 0
279 self._maximums = self._maximums.fromkeys(self._maximums, 0.0)
280 self._averages = self._averages.fromkeys(self._averages, 0.0)
281
282 def _get_time(self, row):
283 return float(row[self.DATA_INDEX_IN_ROW['time']])
284
285 def _add_to_average(self, name, data_point):
286 self._averages[name] += data_point
287
288 def _account_for_maximum(self, name, data_point):
289 if data_point > self._maximums[name]:
290 self._maximums[name] = data_point
291
292
293 def main():
294 parser = argparse.ArgumentParser(usage=__doc__)
295 parser.add_argument(
296 'benchmark_files',
297 nargs='+',
298 help=(
299 'The CSV output file from the benchmark script. If you provide'
300 'more than one of these files, it will give you the average '
301 'across all of the files for each metric.'
302 ),
303 )
304 parser.add_argument(
305 '-f',
306 '--output-format',
307 default='table',
308 choices=['table', 'json'],
309 help=(
310 'Specify what output format to use for displaying results. '
311 'By default, a pretty printed table is used, but you can also '
312 'specify "json" to display pretty printed JSON.'
313 ),
314 )
315 args = parser.parse_args()
316 summarizer = Summarizer()
317 summarizer.process(args)
318 if args.output_format == 'table':
319 result = summarizer.summarize_as_table()
320 else:
321 result = summarizer.summarize_as_json()
322 print(result)
323
324
325 if __name__ == '__main__':
326 main()