| 1 |
#!/usr/bin/env python |
| 2 |
""" |
| 3 |
Summarizes results of benchmarking. |
| 4 |
|
| 5 |
Usage |
| 6 |
===== |
| 7 |
|
| 8 |
Run this script with:: |
| 9 |
|
| 10 |
./summarize performance.csv |
| 11 |
|
| 12 |
|
| 13 |
And that should output:: |
| 14 |
|
| 15 |
+------------------------+----------+----------------------+ |
| 16 |
| Metric over 1 run(s) | Mean | Standard Deviation | |
| 17 |
+========================+==========+======================+ |
| 18 |
| Total Time (seconds) | 1.200 | 0.0 | |
| 19 |
+------------------------+----------+----------------------+ |
| 20 |
| Maximum Memory | 42.3 MiB | 0 Bytes | |
| 21 |
+------------------------+----------+----------------------+ |
| 22 |
| Maximum CPU (percent) | 88.1 | 0.0 | |
| 23 |
+------------------------+----------+----------------------+ |
| 24 |
| Average Memory | 33.9 MiB | 0 Bytes | |
| 25 |
+------------------------+----------+----------------------+ |
| 26 |
| Average CPU (percent) | 30.5 | 0.0 | |
| 27 |
+------------------------+----------+----------------------+ |
| 28 |
|
| 29 |
|
| 30 |
The script can also be ran with multiple files: |
| 31 |
|
| 32 |
./summarize performance.csv performance-2.csv |
| 33 |
|
| 34 |
And will have a similar output: |
| 35 |
|
| 36 |
+------------------------+----------+----------------------+ |
| 37 |
| Metric over 2 run(s) | Mean | Standard Deviation | |
| 38 |
+========================+==========+======================+ |
| 39 |
| Total Time (seconds) | 1.155 | 0.0449999570847 | |
| 40 |
+------------------------+----------+----------------------+ |
| 41 |
| Maximum Memory | 42.5 MiB | 110.0 KiB | |
| 42 |
+------------------------+----------+----------------------+ |
| 43 |
| Maximum CPU (percent) | 94.5 | 6.45 | |
| 44 |
+------------------------+----------+----------------------+ |
| 45 |
| Average Memory | 35.6 MiB | 1.7 MiB | |
| 46 |
+------------------------+----------+----------------------+ |
| 47 |
| Average CPU (percent) | 27.5 | 3.03068181818 | |
| 48 |
+------------------------+----------+----------------------+ |
| 49 |
|
| 50 |
|
| 51 |
You can also specify the ``--output-format json`` option to print the |
| 52 |
summary as JSON instead of a pretty printed table:: |
| 53 |
|
| 54 |
{ |
| 55 |
"total_time": 72.76999998092651, |
| 56 |
"std_dev_average_memory": 0.0, |
| 57 |
"std_dev_total_time": 0.0, |
| 58 |
"average_memory": 56884518.57534247, |
| 59 |
"std_dev_average_cpu": 0.0, |
| 60 |
"std_dev_max_memory": 0.0, |
| 61 |
"average_cpu": 61.19315068493151, |
| 62 |
"max_memory": 58331136.0 |
| 63 |
} |
| 64 |
|
| 65 |
""" |
| 66 |
|
| 67 |
import argparse |
| 68 |
import csv |
| 69 |
import json |
| 70 |
from math import sqrt |
| 71 |
|
| 72 |
from tabulate import tabulate |
| 73 |
|
| 74 |
|
| 75 |
def human_readable_size(value): |
| 76 |
"""Converts integer values in bytes to human readable values""" |
| 77 |
hummanize_suffixes = ('KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB') |
| 78 |
base = 1024 |
| 79 |
bytes_int = float(value) |
| 80 |
|
| 81 |
if bytes_int == 1: |
| 82 |
return '1 Byte' |
| 83 |
elif bytes_int < base: |
| 84 |
return f'{bytes_int} Bytes' |
| 85 |
|
| 86 |
for i, suffix in enumerate(hummanize_suffixes): |
| 87 |
unit = base ** (i + 2) |
| 88 |
if round((bytes_int / unit) * base) < base: |
| 89 |
return f'{(base * bytes_int / unit):.1f} {suffix}' |
| 90 |
|
| 91 |
|
| 92 |
class Summarizer: |
| 93 |
DATA_INDEX_IN_ROW = {'time': 0, 'memory': 1, 'cpu': 2} |
| 94 |
|
| 95 |
def __init__(self): |
| 96 |
self.total_files = 0 |
| 97 |
self._num_rows = 0 |
| 98 |
self._start_time = None |
| 99 |
self._end_time = None |
| 100 |
self._totals = { |
| 101 |
'time': [], |
| 102 |
'average_memory': [], |
| 103 |
'average_cpu': [], |
| 104 |
'max_memory': [], |
| 105 |
'max_cpu': [], |
| 106 |
} |
| 107 |
self._averages = { |
| 108 |
'memory': 0.0, |
| 109 |
'cpu': 0.0, |
| 110 |
} |
| 111 |
self._maximums = {'memory': 0.0, 'cpu': 0.0} |
| 112 |
|
| 113 |
@property |
| 114 |
def total_time(self): |
| 115 |
return self._average_across_all_files('time') |
| 116 |
|
| 117 |
@property |
| 118 |
def max_cpu(self): |
| 119 |
return self._average_across_all_files('max_cpu') |
| 120 |
|
| 121 |
@property |
| 122 |
def max_memory(self): |
| 123 |
return self._average_across_all_files('max_memory') |
| 124 |
|
| 125 |
@property |
| 126 |
def average_cpu(self): |
| 127 |
return self._average_across_all_files('average_cpu') |
| 128 |
|
| 129 |
@property |
| 130 |
def average_memory(self): |
| 131 |
return self._average_across_all_files('average_memory') |
| 132 |
|
| 133 |
@property |
| 134 |
def std_dev_total_time(self): |
| 135 |
return self._standard_deviation_across_all_files('time') |
| 136 |
|
| 137 |
@property |
| 138 |
def std_dev_max_cpu(self): |
| 139 |
return self._standard_deviation_across_all_files('max_cpu') |
| 140 |
|
| 141 |
@property |
| 142 |
def std_dev_max_memory(self): |
| 143 |
return self._standard_deviation_across_all_files('max_memory') |
| 144 |
|
| 145 |
@property |
| 146 |
def std_dev_average_cpu(self): |
| 147 |
return self._standard_deviation_across_all_files('average_cpu') |
| 148 |
|
| 149 |
@property |
| 150 |
def std_dev_average_memory(self): |
| 151 |
return self._standard_deviation_across_all_files('average_memory') |
| 152 |
|
| 153 |
def _average_across_all_files(self, name): |
| 154 |
return sum(self._totals[name]) / len(self._totals[name]) |
| 155 |
|
| 156 |
def _standard_deviation_across_all_files(self, name): |
| 157 |
mean = self._average_across_all_files(name) |
| 158 |
differences = [total - mean for total in self._totals[name]] |
| 159 |
sq_differences = [difference**2 for difference in differences] |
| 160 |
return sqrt(sum(sq_differences) / len(self._totals[name])) |
| 161 |
|
| 162 |
def summarize_as_table(self): |
| 163 |
"""Formats the processed data as pretty printed table. |
| 164 |
|
| 165 |
:return: str of formatted table |
| 166 |
""" |
| 167 |
h = human_readable_size |
| 168 |
table = [ |
| 169 |
[ |
| 170 |
'Total Time (seconds)', |
| 171 |
f'{self.total_time:.3f}', |
| 172 |
self.std_dev_total_time, |
| 173 |
], |
| 174 |
['Maximum Memory', h(self.max_memory), h(self.std_dev_max_memory)], |
| 175 |
[ |
| 176 |
'Maximum CPU (percent)', |
| 177 |
f'{self.max_cpu:.1f}', |
| 178 |
self.std_dev_max_cpu, |
| 179 |
], |
| 180 |
[ |
| 181 |
'Average Memory', |
| 182 |
h(self.average_memory), |
| 183 |
h(self.std_dev_average_memory), |
| 184 |
], |
| 185 |
[ |
| 186 |
'Average CPU (percent)', |
| 187 |
f'{self.average_cpu:.1f}', |
| 188 |
self.std_dev_average_cpu, |
| 189 |
], |
| 190 |
] |
| 191 |
return tabulate( |
| 192 |
table, |
| 193 |
headers=[ |
| 194 |
f'Metric over {self.total_files} run(s)', |
| 195 |
'Mean', |
| 196 |
'Standard Deviation', |
| 197 |
], |
| 198 |
tablefmt="grid", |
| 199 |
) |
| 200 |
|
| 201 |
def summarize_as_json(self): |
| 202 |
"""Return JSON summary of processed data. |
| 203 |
|
| 204 |
:return: str of formatted JSON |
| 205 |
""" |
| 206 |
return json.dumps( |
| 207 |
{ |
| 208 |
'total_time': self.total_time, |
| 209 |
'std_dev_total_time': self.std_dev_total_time, |
| 210 |
'max_memory': self.max_memory, |
| 211 |
'std_dev_max_memory': self.std_dev_max_memory, |
| 212 |
'average_memory': self.average_memory, |
| 213 |
'std_dev_average_memory': self.std_dev_average_memory, |
| 214 |
'std_dev_max_cpu': self.std_dev_max_cpu, |
| 215 |
'max_cpu': self.max_cpu, |
| 216 |
'average_cpu': self.average_cpu, |
| 217 |
'std_dev_average_cpu': self.std_dev_average_cpu, |
| 218 |
}, |
| 219 |
indent=2, |
| 220 |
) |
| 221 |
|
| 222 |
def process(self, args): |
| 223 |
"""Processes the data from the CSV file""" |
| 224 |
for benchmark_file in args.benchmark_files: |
| 225 |
self.process_individual_file(benchmark_file) |
| 226 |
self.total_files += 1 |
| 227 |
|
| 228 |
def process_individual_file(self, benchmark_file): |
| 229 |
with open(benchmark_file) as f: |
| 230 |
reader = csv.reader(f) |
| 231 |
# Process each row from the CSV file |
| 232 |
row = None |
| 233 |
for row in reader: |
| 234 |
self._validate_row(row, benchmark_file) |
| 235 |
self.process_data_row(row) |
| 236 |
self._validate_row(row, benchmark_file) |
| 237 |
self._end_time = self._get_time(row) |
| 238 |
self._finalize_processed_data_for_file() |
| 239 |
|
| 240 |
def _validate_row(self, row, filename): |
| 241 |
if not row: |
| 242 |
raise RuntimeError( |
| 243 |
f'Row: {row} could not be processed. The CSV file ({filename}) may be ' |
| 244 |
'empty.' |
| 245 |
) |
| 246 |
|
| 247 |
def process_data_row(self, row): |
| 248 |
# If the row is the first row collect the start time. |
| 249 |
if self._num_rows == 0: |
| 250 |
self._start_time = self._get_time(row) |
| 251 |
self._num_rows += 1 |
| 252 |
self.process_data_point(row, 'memory') |
| 253 |
self.process_data_point(row, 'cpu') |
| 254 |
|
| 255 |
def process_data_point(self, row, name): |
| 256 |
# Determine where in the CSV row the requested data is located. |
| 257 |
index = self.DATA_INDEX_IN_ROW[name] |
| 258 |
# Get the data point. |
| 259 |
data_point = float(row[index]) |
| 260 |
self._add_to_average(name, data_point) |
| 261 |
self._account_for_maximum(name, data_point) |
| 262 |
|
| 263 |
def _finalize_processed_data_for_file(self): |
| 264 |
# Add numbers to the total, which keeps track of data over |
| 265 |
# all files provided. |
| 266 |
self._totals['time'].append(self._end_time - self._start_time) |
| 267 |
self._totals['max_cpu'].append(self._maximums['cpu']) |
| 268 |
self._totals['max_memory'].append(self._maximums['memory']) |
| 269 |
self._totals['average_cpu'].append( |
| 270 |
self._averages['cpu'] / self._num_rows |
| 271 |
) |
| 272 |
self._totals['average_memory'].append( |
| 273 |
self._averages['memory'] / self._num_rows |
| 274 |
) |
| 275 |
|
| 276 |
# Reset some of the data needed to be tracked for each specific |
| 277 |
# file. |
| 278 |
self._num_rows = 0 |
| 279 |
self._maximums = self._maximums.fromkeys(self._maximums, 0.0) |
| 280 |
self._averages = self._averages.fromkeys(self._averages, 0.0) |
| 281 |
|
| 282 |
def _get_time(self, row): |
| 283 |
return float(row[self.DATA_INDEX_IN_ROW['time']]) |
| 284 |
|
| 285 |
def _add_to_average(self, name, data_point): |
| 286 |
self._averages[name] += data_point |
| 287 |
|
| 288 |
def _account_for_maximum(self, name, data_point): |
| 289 |
if data_point > self._maximums[name]: |
| 290 |
self._maximums[name] = data_point |
| 291 |
|
| 292 |
|
| 293 |
def main(): |
| 294 |
parser = argparse.ArgumentParser(usage=__doc__) |
| 295 |
parser.add_argument( |
| 296 |
'benchmark_files', |
| 297 |
nargs='+', |
| 298 |
help=( |
| 299 |
'The CSV output file from the benchmark script. If you provide' |
| 300 |
'more than one of these files, it will give you the average ' |
| 301 |
'across all of the files for each metric.' |
| 302 |
), |
| 303 |
) |
| 304 |
parser.add_argument( |
| 305 |
'-f', |
| 306 |
'--output-format', |
| 307 |
default='table', |
| 308 |
choices=['table', 'json'], |
| 309 |
help=( |
| 310 |
'Specify what output format to use for displaying results. ' |
| 311 |
'By default, a pretty printed table is used, but you can also ' |
| 312 |
'specify "json" to display pretty printed JSON.' |
| 313 |
), |
| 314 |
) |
| 315 |
args = parser.parse_args() |
| 316 |
summarizer = Summarizer() |
| 317 |
summarizer.process(args) |
| 318 |
if args.output_format == 'table': |
| 319 |
result = summarizer.summarize_as_table() |
| 320 |
else: |
| 321 |
result = summarizer.summarize_as_json() |
| 322 |
print(result) |
| 323 |
|
| 324 |
|
| 325 |
if __name__ == '__main__': |
| 326 |
main() |