Debian Codegraph @

tests/TestScripts/testSharding.py

raw
1 #!/usr/bin/env python3
2
3 # Copyright Catch2 Authors
4 # Distributed under the Boost Software License, Version 1.0.
5 # (See accompanying file LICENSE.txt or copy at
6 # https://www.boost.org/LICENSE_1_0.txt)
7
8 # SPDX-License-Identifier: BSL-1.0
9
10 """
11 This test script verifies that sharding tests does change which tests are run.
12 This is done by running the binary multiple times, once to list all the tests,
13 once per shard to list the tests for that shard, and once again per shard to
14 execute the tests. The sharded lists are compared to the full list to ensure
15 none are skipped, duplicated, and that the order remains the same.
16 """
17
18 import random
19 import subprocess
20 import sys
21 import xml.etree.ElementTree as ET
22
23 from collections import namedtuple
24
25 from typing import List, Dict
26
27 seed = random.randint(0, 2 ** 32 - 1)
28 number_of_shards = 5
29
30 def make_base_commandline(self_test_exe):
31 return [
32 self_test_exe,
33 '--reporter', 'xml',
34 '--order', 'rand',
35 '--rng-seed', str(seed),
36 "[generators]~[benchmarks]~[.]"
37 ]
38
39
40 def list_tests(self_test_exe: str, extra_args: List[str] = None):
41 cmd = make_base_commandline(self_test_exe) + ['--list-tests']
42 if extra_args:
43 cmd.extend(extra_args)
44
45 try:
46 ret = subprocess.run(cmd,
47 stdout = subprocess.PIPE,
48 stderr = subprocess.PIPE,
49 timeout = 10,
50 check = True,
51 universal_newlines = True)
52 except subprocess.CalledProcessError as ex:
53 print('Could not list tests:\n{}'.format(ex.stderr))
54
55 if ret.stderr:
56 raise RuntimeError("Unexpected error output:\n" + ret.stderr)
57
58 root = ET.fromstring(ret.stdout)
59 result = [elem.text for elem in root.findall('./TestCase/Name')]
60
61 if len(result) < 2:
62 raise RuntimeError("Unexpectedly few tests listed (got {})".format(
63 len(result)))
64
65
66 return result
67
68
69 def execute_tests(self_test_exe: str, extra_args: List[str] = None):
70 cmd = make_base_commandline(self_test_exe)
71 if extra_args:
72 cmd.extend(extra_args)
73
74 try:
75 ret = subprocess.run(cmd,
76 stdout = subprocess.PIPE,
77 stderr = subprocess.PIPE,
78 timeout = 10,
79 check = True,
80 universal_newlines = True)
81 except subprocess.CalledProcessError as ex:
82 print('Could not list tests:\n{}'.format(ex.stderr))
83
84 if ret.stderr:
85 raise RuntimeError("Unexpected error output:\n" + process.stderr)
86
87 root = ET.fromstring(ret.stdout)
88 result = [elem.attrib["name"] for elem in root.findall('./TestCase')]
89
90 if len(result) < 2:
91 raise RuntimeError("Unexpectedly few tests listed (got {})".format(
92 len(result)))
93
94 return result
95
96
97 def test_sharded_listing(self_test_exe: str) -> Dict[int, List[str]]:
98 """
99 Asks the test binary for list of all tests, and also for lists of
100 tests from shards.
101
102 The combination of shards is then checked whether it corresponds to
103 the full list of all tests.
104
105 Returns the dictionary of shard-index => listed tests for later use.
106 """
107 all_tests = list_tests(self_test_exe)
108 big_shard_tests = list_tests(self_test_exe, ['--shard-count', '1', '--shard-index', '0'])
109
110 assert all_tests == big_shard_tests, (
111 "No-sharding test list does not match the listing of big shard:\nNo shard:\n{}\n\nWith shard:\n{}\n".format(
112 '\n'.join(all_tests),
113 '\n'.join(big_shard_tests)
114 )
115 )
116
117 shard_listings = dict()
118 for shard_idx in range(number_of_shards):
119 shard_listings[shard_idx] = list_tests(self_test_exe, ['--shard-count', str(number_of_shards), '--shard-index', str(shard_idx)])
120
121 shard_sizes = [len(v) for v in shard_listings.values()]
122 assert len(all_tests) == sum(shard_sizes)
123
124 # Check that the shards have roughly the right sizes (e.g. we don't
125 # have all tests in single shard and the others are empty)
126 differences = [abs(x1 - x2) for x1, x2 in zip(shard_sizes, shard_sizes[1:])]
127 assert all(diff <= 1 for diff in differences), "A shard has weird size: {}".format(shard_sizes)
128
129 combined_shards = [inner for outer in shard_listings.values() for inner in outer]
130 assert all_tests == combined_shards, (
131 "All tests and combined shards disagree.\nNo shard:\n{}\n\nCombined:\n{}\n\n".format(
132 '\n'.join(all_tests),
133 '\n'.join(combined_shards)
134 )
135 )
136 shard_listings[-1] = all_tests
137
138 return shard_listings
139
140
141 def test_sharded_execution(self_test_exe: str, listings: Dict[int, List[str]]):
142 """
143 Runs the test binary and checks that the executed tests match the
144 previously listed tests.
145
146 Also does this for various shard indices, and that the combination
147 of all shards matches the full run/listing.
148 """
149 all_tests = execute_tests(self_test_exe)
150 big_shard_tests = execute_tests(self_test_exe, ['--shard-count', '1', '--shard-index', '0'])
151 assert all_tests == big_shard_tests
152
153 assert listings[-1] == all_tests
154
155 for shard_idx in range(number_of_shards):
156 assert listings[shard_idx] == execute_tests(self_test_exe, ['--shard-count', str(number_of_shards), '--shard-index', str(shard_idx)])
157
158
159 def main():
160 self_test_exe, = sys.argv[1:]
161 listings = test_sharded_listing(self_test_exe)
162 test_sharded_execution(self_test_exe, listings)
163
164 if __name__ == '__main__':
165 sys.exit(main())