Debian Codegraph @

src/c-blosc2/bench/aggregate_multiple.py

raw
1 from pathlib import Path
2 import re
3 import os
4 import subprocess
5 from typing import Union
6
7 import numpy as np
8 import argparse
9
10
11 def run_command(command: str, return_output: bool = False) -> Union[tuple[list[float], str], tuple[list[float]]]:
12 bench_folder = Path(__file__).parent.parent / "build" / "bench"
13 if os.name == "nt":
14 bench_folder /= "Release"
15 res = subprocess.run(f"{bench_folder}/{command}", shell=True, capture_output=True)
16 assert res.returncode == 0, f"\nstdout: {res.stdout.decode('utf-8')}\nstderr: {res.stderr.decode('utf-8')}"
17
18 out = res.stdout.decode("utf-8")
19 numbers = []
20 for number in re.findall(r"-?\d+(?:\.\d+)?", out):
21 numbers.append(float(number))
22 assert len(numbers) > 0, f"Could not find any numbers in the output:\n{out}"
23
24 if return_output:
25 return numbers, out
26 else:
27 return numbers
28
29
30 class ReplaceNumbers:
31 def __init__(self, numbers: np.ndarray):
32 self.numbers = numbers
33 self.index = 0
34
35 def __call__(self, match: re.Match) -> str:
36 assert match.group() == "NUMBER"
37 number = self.numbers[self.index]
38 self.index += 1
39
40 if number.is_integer():
41 return str(int(number))
42 else:
43 return f"{number:.3f}"
44
45
46 if __name__ == "__main__":
47 parser = argparse.ArgumentParser(
48 description=(
49 "Repeat multiple runs of a benchmark and aggregate all numbers in the output (the outputs from all runs must "
50 "have the same format)."
51 ),
52 formatter_class=argparse.ArgumentDefaultsHelpFormatter,
53 )
54 parser.add_argument(
55 "--command",
56 required=True,
57 type=str,
58 help="The command to execute (e.g. sframe_bench 1000 insert 1000 io_mmap).",
59 )
60 parser.add_argument(
61 "--warmups",
62 default=3,
63 required=False,
64 type=int,
65 help="Number of warmup runs.",
66 )
67 parser.add_argument(
68 "--runs",
69 default=30,
70 required=False,
71 type=int,
72 help="Number of runs to perform and aggregate the output from.",
73 )
74 args = parser.parse_args()
75
76 for i in range(args.warmups):
77 print(f"Warmup {i+1}/{args.warmups}", end="\r")
78 run_command(args.command)
79 print()
80
81 # Collect numbers from outputs of multiple runs
82 example_output = None
83 numbers = []
84 for i in range(args.runs):
85 print(f"Run {i+1}/{args.runs}", end="\r")
86
87 if example_output is None:
88 values, text = run_command(args.command, return_output=True)
89 example_output = text
90 numbers.append(values)
91 else:
92 numbers.append(run_command(args.command))
93 print()
94
95 numbers = np.asarray(numbers)
96 numbers = np.min(numbers, axis=0)
97
98 assert example_output is not None, "No output was collected."
99 example_output = re.sub(r"-?\d+(?:\.\d+)?", "NUMBER", example_output)
100 aggregated_output = re.sub("NUMBER", ReplaceNumbers(numbers), example_output)
101 print(aggregated_output)