Debian Codegraph @

third-party/benchmark/tools/compare.py

raw
1 #!/usr/bin/env python3
2
3 # type: ignore
4
5 """
6 compare.py - versatile benchmark output compare tool
7 """
8
9 import argparse
10 import json
11 import os
12 import sys
13 import unittest
14 from argparse import ArgumentParser
15
16 import gbench
17 from gbench import report, util
18
19
20 def check_inputs(in1, in2, flags):
21 """
22 Perform checking on the user provided inputs and diagnose any abnormalities
23 """
24 in1_kind, in1_err = util.classify_input_file(in1)
25 in2_kind, in2_err = util.classify_input_file(in2)
26 output_file = util.find_benchmark_flag("--benchmark_out=", flags)
27 output_type = util.find_benchmark_flag("--benchmark_out_format=", flags)
28 if (
29 in1_kind == util.IT_Executable
30 and in2_kind == util.IT_Executable
31 and output_file
32 ):
33 print(
34 (
35 "WARNING: '--benchmark_out=%s' will be passed to both "
36 "benchmarks causing it to be overwritten"
37 )
38 % output_file
39 )
40 if in1_kind == util.IT_JSON and in2_kind == util.IT_JSON:
41 # When both sides are JSON the only supported flag is
42 # --benchmark_filter=
43 for flag in util.remove_benchmark_flags("--benchmark_filter=", flags):
44 print(
45 "WARNING: passing %s has no effect since both "
46 "inputs are JSON" % flag
47 )
48 if output_type is not None and output_type != "json":
49 print(
50 (
51 "ERROR: passing '--benchmark_out_format=%s' to 'compare.py`"
52 " is not supported."
53 )
54 % output_type
55 )
56 sys.exit(1)
57
58
59 def create_parser():
60 parser = ArgumentParser(
61 description="versatile benchmark output compare tool"
62 )
63
64 parser.add_argument(
65 "-a",
66 "--display_aggregates_only",
67 dest="display_aggregates_only",
68 action="store_true",
69 help="If there are repetitions, by default, we display everything - the"
70 " actual runs, and the aggregates computed. Sometimes, it is "
71 "desirable to only view the aggregates. E.g. when there are a lot "
72 "of repetitions. Do note that only the display is affected. "
73 "Internally, all the actual runs are still used, e.g. for U test.",
74 )
75
76 parser.add_argument(
77 "--no-color",
78 dest="color",
79 default=True,
80 action="store_false",
81 help="Do not use colors in the terminal output",
82 )
83
84 parser.add_argument(
85 "-d",
86 "--dump_to_json",
87 dest="dump_to_json",
88 help="Additionally, dump benchmark comparison output to this file in JSON format.",
89 )
90
91 utest = parser.add_argument_group()
92 utest.add_argument(
93 "--no-utest",
94 dest="utest",
95 default=True,
96 action="store_false",
97 help="The tool can do a two-tailed Mann-Whitney U test with the null hypothesis that it is equally likely that a randomly selected value from one sample will be less than or greater than a randomly selected value from a second sample.\nWARNING: requires **LARGE** (no less than {}) number of repetitions to be meaningful!\nThe test is being done by default, if at least {} repetitions were done.\nThis option can disable the U Test.".format(
98 report.UTEST_OPTIMAL_REPETITIONS, report.UTEST_MIN_REPETITIONS
99 ),
100 )
101 alpha_default = 0.05
102 utest.add_argument(
103 "--alpha",
104 dest="utest_alpha",
105 default=alpha_default,
106 type=float,
107 help=(
108 "significance level alpha. if the calculated p-value is below this value, then the result is said to be statistically significant and the null hypothesis is rejected.\n(default: %0.4f)"
109 )
110 % alpha_default,
111 )
112
113 subparsers = parser.add_subparsers(
114 help="This tool has multiple modes of operation:", dest="mode"
115 )
116
117 parser_a = subparsers.add_parser(
118 "benchmarks",
119 help="The most simple use-case, compare all the output of these two benchmarks",
120 )
121 baseline = parser_a.add_argument_group("baseline", "The benchmark baseline")
122 baseline.add_argument(
123 "test_baseline",
124 metavar="test_baseline",
125 type=argparse.FileType("r"),
126 nargs=1,
127 help="A benchmark executable or JSON output file",
128 )
129 contender = parser_a.add_argument_group(
130 "contender", "The benchmark that will be compared against the baseline"
131 )
132 contender.add_argument(
133 "test_contender",
134 metavar="test_contender",
135 type=argparse.FileType("r"),
136 nargs=1,
137 help="A benchmark executable or JSON output file",
138 )
139 parser_a.add_argument(
140 "benchmark_options",
141 metavar="benchmark_options",
142 nargs=argparse.REMAINDER,
143 help="Arguments to pass when running benchmark executables",
144 )
145
146 parser_b = subparsers.add_parser(
147 "filters", help="Compare filter one with the filter two of benchmark"
148 )
149 baseline = parser_b.add_argument_group("baseline", "The benchmark baseline")
150 baseline.add_argument(
151 "test",
152 metavar="test",
153 type=argparse.FileType("r"),
154 nargs=1,
155 help="A benchmark executable or JSON output file",
156 )
157 baseline.add_argument(
158 "filter_baseline",
159 metavar="filter_baseline",
160 type=str,
161 nargs=1,
162 help="The first filter, that will be used as baseline",
163 )
164 contender = parser_b.add_argument_group(
165 "contender", "The benchmark that will be compared against the baseline"
166 )
167 contender.add_argument(
168 "filter_contender",
169 metavar="filter_contender",
170 type=str,
171 nargs=1,
172 help="The second filter, that will be compared against the baseline",
173 )
174 parser_b.add_argument(
175 "benchmark_options",
176 metavar="benchmark_options",
177 nargs=argparse.REMAINDER,
178 help="Arguments to pass when running benchmark executables",
179 )
180
181 parser_c = subparsers.add_parser(
182 "benchmarksfiltered",
183 help="Compare filter one of first benchmark with filter two of the second benchmark",
184 )
185 baseline = parser_c.add_argument_group("baseline", "The benchmark baseline")
186 baseline.add_argument(
187 "test_baseline",
188 metavar="test_baseline",
189 type=argparse.FileType("r"),
190 nargs=1,
191 help="A benchmark executable or JSON output file",
192 )
193 baseline.add_argument(
194 "filter_baseline",
195 metavar="filter_baseline",
196 type=str,
197 nargs=1,
198 help="The first filter, that will be used as baseline",
199 )
200 contender = parser_c.add_argument_group(
201 "contender", "The benchmark that will be compared against the baseline"
202 )
203 contender.add_argument(
204 "test_contender",
205 metavar="test_contender",
206 type=argparse.FileType("r"),
207 nargs=1,
208 help="The second benchmark executable or JSON output file, that will be compared against the baseline",
209 )
210 contender.add_argument(
211 "filter_contender",
212 metavar="filter_contender",
213 type=str,
214 nargs=1,
215 help="The second filter, that will be compared against the baseline",
216 )
217 parser_c.add_argument(
218 "benchmark_options",
219 metavar="benchmark_options",
220 nargs=argparse.REMAINDER,
221 help="Arguments to pass when running benchmark executables",
222 )
223
224 return parser
225
226
227 def main():
228 # Parse the command line flags
229 parser = create_parser()
230 args, unknown_args = parser.parse_known_args()
231 if args.mode is None:
232 parser.print_help()
233 exit(1)
234 assert not unknown_args
235 benchmark_options = args.benchmark_options
236
237 if args.mode == "benchmarks":
238 test_baseline = args.test_baseline[0].name
239 test_contender = args.test_contender[0].name
240 filter_baseline = ""
241 filter_contender = ""
242
243 # NOTE: if test_baseline == test_contender, you are analyzing the stdev
244
245 description = "Comparing %s to %s" % (test_baseline, test_contender)
246 elif args.mode == "filters":
247 test_baseline = args.test[0].name
248 test_contender = args.test[0].name
249 filter_baseline = args.filter_baseline[0]
250 filter_contender = args.filter_contender[0]
251
252 # NOTE: if filter_baseline == filter_contender, you are analyzing the
253 # stdev
254
255 description = "Comparing %s to %s (from %s)" % (
256 filter_baseline,
257 filter_contender,
258 args.test[0].name,
259 )
260 elif args.mode == "benchmarksfiltered":
261 test_baseline = args.test_baseline[0].name
262 test_contender = args.test_contender[0].name
263 filter_baseline = args.filter_baseline[0]
264 filter_contender = args.filter_contender[0]
265
266 # NOTE: if test_baseline == test_contender and
267 # filter_baseline == filter_contender, you are analyzing the stdev
268
269 description = "Comparing %s (from %s) to %s (from %s)" % (
270 filter_baseline,
271 test_baseline,
272 filter_contender,
273 test_contender,
274 )
275 else:
276 # should never happen
277 print("Unrecognized mode of operation: '%s'" % args.mode)
278 parser.print_help()
279 exit(1)
280
281 check_inputs(test_baseline, test_contender, benchmark_options)
282
283 if args.display_aggregates_only:
284 benchmark_options += ["--benchmark_display_aggregates_only=true"]
285
286 options_baseline = []
287 options_contender = []
288
289 if filter_baseline and filter_contender:
290 options_baseline = ["--benchmark_filter=%s" % filter_baseline]
291 options_contender = ["--benchmark_filter=%s" % filter_contender]
292
293 # Run the benchmarks and report the results
294 json1 = json1_orig = gbench.util.sort_benchmark_results(
295 gbench.util.run_or_load_benchmark(
296 test_baseline, benchmark_options + options_baseline
297 )
298 )
299 json2 = json2_orig = gbench.util.sort_benchmark_results(
300 gbench.util.run_or_load_benchmark(
301 test_contender, benchmark_options + options_contender
302 )
303 )
304
305 # Now, filter the benchmarks so that the difference report can work
306 if filter_baseline and filter_contender:
307 replacement = "[%s vs. %s]" % (filter_baseline, filter_contender)
308 json1 = gbench.report.filter_benchmark(
309 json1_orig, filter_baseline, replacement
310 )
311 json2 = gbench.report.filter_benchmark(
312 json2_orig, filter_contender, replacement
313 )
314
315 diff_report = gbench.report.get_difference_report(json1, json2, args.utest)
316 output_lines = gbench.report.print_difference_report(
317 diff_report,
318 args.display_aggregates_only,
319 args.utest,
320 args.utest_alpha,
321 args.color,
322 )
323 print(description)
324 for ln in output_lines:
325 print(ln)
326
327 # Optionally, diff and output to JSON
328 if args.dump_to_json is not None:
329 with open(args.dump_to_json, "w") as f_json:
330 json.dump(diff_report, f_json, indent=1)
331
332
333 class TestParser(unittest.TestCase):
334 def setUp(self):
335 self.parser = create_parser()
336 testInputs = os.path.join(
337 os.path.dirname(os.path.realpath(__file__)), "gbench", "Inputs"
338 )
339 self.testInput0 = os.path.join(testInputs, "test1_run1.json")
340 self.testInput1 = os.path.join(testInputs, "test1_run2.json")
341
342 def test_benchmarks_basic(self):
343 parsed = self.parser.parse_args(
344 ["benchmarks", self.testInput0, self.testInput1]
345 )
346 self.assertFalse(parsed.display_aggregates_only)
347 self.assertTrue(parsed.utest)
348 self.assertEqual(parsed.mode, "benchmarks")
349 self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
350 self.assertEqual(parsed.test_contender[0].name, self.testInput1)
351 self.assertFalse(parsed.benchmark_options)
352
353 def test_benchmarks_basic_without_utest(self):
354 parsed = self.parser.parse_args(
355 ["--no-utest", "benchmarks", self.testInput0, self.testInput1]
356 )
357 self.assertFalse(parsed.display_aggregates_only)
358 self.assertFalse(parsed.utest)
359 self.assertEqual(parsed.utest_alpha, 0.05)
360 self.assertEqual(parsed.mode, "benchmarks")
361 self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
362 self.assertEqual(parsed.test_contender[0].name, self.testInput1)
363 self.assertFalse(parsed.benchmark_options)
364
365 def test_benchmarks_basic_display_aggregates_only(self):
366 parsed = self.parser.parse_args(
367 ["-a", "benchmarks", self.testInput0, self.testInput1]
368 )
369 self.assertTrue(parsed.display_aggregates_only)
370 self.assertTrue(parsed.utest)
371 self.assertEqual(parsed.mode, "benchmarks")
372 self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
373 self.assertEqual(parsed.test_contender[0].name, self.testInput1)
374 self.assertFalse(parsed.benchmark_options)
375
376 def test_benchmarks_basic_with_utest_alpha(self):
377 parsed = self.parser.parse_args(
378 ["--alpha=0.314", "benchmarks", self.testInput0, self.testInput1]
379 )
380 self.assertFalse(parsed.display_aggregates_only)
381 self.assertTrue(parsed.utest)
382 self.assertEqual(parsed.utest_alpha, 0.314)
383 self.assertEqual(parsed.mode, "benchmarks")
384 self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
385 self.assertEqual(parsed.test_contender[0].name, self.testInput1)
386 self.assertFalse(parsed.benchmark_options)
387
388 def test_benchmarks_basic_without_utest_with_utest_alpha(self):
389 parsed = self.parser.parse_args(
390 [
391 "--no-utest",
392 "--alpha=0.314",
393 "benchmarks",
394 self.testInput0,
395 self.testInput1,
396 ]
397 )
398 self.assertFalse(parsed.display_aggregates_only)
399 self.assertFalse(parsed.utest)
400 self.assertEqual(parsed.utest_alpha, 0.314)
401 self.assertEqual(parsed.mode, "benchmarks")
402 self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
403 self.assertEqual(parsed.test_contender[0].name, self.testInput1)
404 self.assertFalse(parsed.benchmark_options)
405
406 def test_benchmarks_with_remainder(self):
407 parsed = self.parser.parse_args(
408 ["benchmarks", self.testInput0, self.testInput1, "d"]
409 )
410 self.assertFalse(parsed.display_aggregates_only)
411 self.assertTrue(parsed.utest)
412 self.assertEqual(parsed.mode, "benchmarks")
413 self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
414 self.assertEqual(parsed.test_contender[0].name, self.testInput1)
415 self.assertEqual(parsed.benchmark_options, ["d"])
416
417 def test_benchmarks_with_remainder_after_doubleminus(self):
418 parsed = self.parser.parse_args(
419 ["benchmarks", self.testInput0, self.testInput1, "--", "e"]
420 )
421 self.assertFalse(parsed.display_aggregates_only)
422 self.assertTrue(parsed.utest)
423 self.assertEqual(parsed.mode, "benchmarks")
424 self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
425 self.assertEqual(parsed.test_contender[0].name, self.testInput1)
426 self.assertEqual(parsed.benchmark_options, ["e"])
427
428 def test_filters_basic(self):
429 parsed = self.parser.parse_args(["filters", self.testInput0, "c", "d"])
430 self.assertFalse(parsed.display_aggregates_only)
431 self.assertTrue(parsed.utest)
432 self.assertEqual(parsed.mode, "filters")
433 self.assertEqual(parsed.test[0].name, self.testInput0)
434 self.assertEqual(parsed.filter_baseline[0], "c")
435 self.assertEqual(parsed.filter_contender[0], "d")
436 self.assertFalse(parsed.benchmark_options)
437
438 def test_filters_with_remainder(self):
439 parsed = self.parser.parse_args(
440 ["filters", self.testInput0, "c", "d", "e"]
441 )
442 self.assertFalse(parsed.display_aggregates_only)
443 self.assertTrue(parsed.utest)
444 self.assertEqual(parsed.mode, "filters")
445 self.assertEqual(parsed.test[0].name, self.testInput0)
446 self.assertEqual(parsed.filter_baseline[0], "c")
447 self.assertEqual(parsed.filter_contender[0], "d")
448 self.assertEqual(parsed.benchmark_options, ["e"])
449
450 def test_filters_with_remainder_after_doubleminus(self):
451 parsed = self.parser.parse_args(
452 ["filters", self.testInput0, "c", "d", "--", "f"]
453 )
454 self.assertFalse(parsed.display_aggregates_only)
455 self.assertTrue(parsed.utest)
456 self.assertEqual(parsed.mode, "filters")
457 self.assertEqual(parsed.test[0].name, self.testInput0)
458 self.assertEqual(parsed.filter_baseline[0], "c")
459 self.assertEqual(parsed.filter_contender[0], "d")
460 self.assertEqual(parsed.benchmark_options, ["f"])
461
462 def test_benchmarksfiltered_basic(self):
463 parsed = self.parser.parse_args(
464 ["benchmarksfiltered", self.testInput0, "c", self.testInput1, "e"]
465 )
466 self.assertFalse(parsed.display_aggregates_only)
467 self.assertTrue(parsed.utest)
468 self.assertEqual(parsed.mode, "benchmarksfiltered")
469 self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
470 self.assertEqual(parsed.filter_baseline[0], "c")
471 self.assertEqual(parsed.test_contender[0].name, self.testInput1)
472 self.assertEqual(parsed.filter_contender[0], "e")
473 self.assertFalse(parsed.benchmark_options)
474
475 def test_benchmarksfiltered_with_remainder(self):
476 parsed = self.parser.parse_args(
477 [
478 "benchmarksfiltered",
479 self.testInput0,
480 "c",
481 self.testInput1,
482 "e",
483 "f",
484 ]
485 )
486 self.assertFalse(parsed.display_aggregates_only)
487 self.assertTrue(parsed.utest)
488 self.assertEqual(parsed.mode, "benchmarksfiltered")
489 self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
490 self.assertEqual(parsed.filter_baseline[0], "c")
491 self.assertEqual(parsed.test_contender[0].name, self.testInput1)
492 self.assertEqual(parsed.filter_contender[0], "e")
493 self.assertEqual(parsed.benchmark_options[0], "f")
494
495 def test_benchmarksfiltered_with_remainder_after_doubleminus(self):
496 parsed = self.parser.parse_args(
497 [
498 "benchmarksfiltered",
499 self.testInput0,
500 "c",
501 self.testInput1,
502 "e",
503 "--",
504 "g",
505 ]
506 )
507 self.assertFalse(parsed.display_aggregates_only)
508 self.assertTrue(parsed.utest)
509 self.assertEqual(parsed.mode, "benchmarksfiltered")
510 self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
511 self.assertEqual(parsed.filter_baseline[0], "c")
512 self.assertEqual(parsed.test_contender[0].name, self.testInput1)
513 self.assertEqual(parsed.filter_contender[0], "e")
514 self.assertEqual(parsed.benchmark_options[0], "g")
515
516
517 if __name__ == "__main__":
518 # unittest.main()
519 main()
520
521 # vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4
522 # kate: tab-width: 4; replace-tabs on; indent-width 4; tab-indents: off;
523 # kate: indent-mode python; remove-trailing-spaces modified;