| 1 |
#!/usr/bin/env python3 |
| 2 |
|
| 3 |
# Copyright Catch2 Authors |
| 4 |
# Distributed under the Boost Software License, Version 1.0. |
| 5 |
# (See accompanying file LICENSE.txt or copy at |
| 6 |
# https://www.boost.org/LICENSE_1_0.txt) |
| 7 |
|
| 8 |
# SPDX-License-Identifier: BSL-1.0 |
| 9 |
|
| 10 |
""" |
| 11 |
This test script verifies that sharding tests does change which tests are run. |
| 12 |
This is done by running the binary multiple times, once to list all the tests, |
| 13 |
once per shard to list the tests for that shard, and once again per shard to |
| 14 |
execute the tests. The sharded lists are compared to the full list to ensure |
| 15 |
none are skipped, duplicated, and that the order remains the same. |
| 16 |
""" |
| 17 |
|
| 18 |
import random |
| 19 |
import subprocess |
| 20 |
import sys |
| 21 |
import xml.etree.ElementTree as ET |
| 22 |
|
| 23 |
from collections import namedtuple |
| 24 |
|
| 25 |
from typing import List, Dict |
| 26 |
|
| 27 |
seed = random.randint(0, 2 ** 32 - 1) |
| 28 |
number_of_shards = 5 |
| 29 |
|
| 30 |
def make_base_commandline(self_test_exe): |
| 31 |
return [ |
| 32 |
self_test_exe, |
| 33 |
'--reporter', 'xml', |
| 34 |
'--order', 'rand', |
| 35 |
'--rng-seed', str(seed), |
| 36 |
"[generators]~[benchmarks]~[.]" |
| 37 |
] |
| 38 |
|
| 39 |
|
| 40 |
def list_tests(self_test_exe: str, extra_args: List[str] = None): |
| 41 |
cmd = make_base_commandline(self_test_exe) + ['--list-tests'] |
| 42 |
if extra_args: |
| 43 |
cmd.extend(extra_args) |
| 44 |
|
| 45 |
try: |
| 46 |
ret = subprocess.run(cmd, |
| 47 |
stdout = subprocess.PIPE, |
| 48 |
stderr = subprocess.PIPE, |
| 49 |
timeout = 10, |
| 50 |
check = True, |
| 51 |
universal_newlines = True) |
| 52 |
except subprocess.CalledProcessError as ex: |
| 53 |
print('Could not list tests:\n{}'.format(ex.stderr)) |
| 54 |
|
| 55 |
if ret.stderr: |
| 56 |
raise RuntimeError("Unexpected error output:\n" + ret.stderr) |
| 57 |
|
| 58 |
root = ET.fromstring(ret.stdout) |
| 59 |
result = [elem.text for elem in root.findall('./TestCase/Name')] |
| 60 |
|
| 61 |
if len(result) < 2: |
| 62 |
raise RuntimeError("Unexpectedly few tests listed (got {})".format( |
| 63 |
len(result))) |
| 64 |
|
| 65 |
|
| 66 |
return result |
| 67 |
|
| 68 |
|
| 69 |
def execute_tests(self_test_exe: str, extra_args: List[str] = None): |
| 70 |
cmd = make_base_commandline(self_test_exe) |
| 71 |
if extra_args: |
| 72 |
cmd.extend(extra_args) |
| 73 |
|
| 74 |
try: |
| 75 |
ret = subprocess.run(cmd, |
| 76 |
stdout = subprocess.PIPE, |
| 77 |
stderr = subprocess.PIPE, |
| 78 |
timeout = 10, |
| 79 |
check = True, |
| 80 |
universal_newlines = True) |
| 81 |
except subprocess.CalledProcessError as ex: |
| 82 |
print('Could not list tests:\n{}'.format(ex.stderr)) |
| 83 |
|
| 84 |
if ret.stderr: |
| 85 |
raise RuntimeError("Unexpected error output:\n" + process.stderr) |
| 86 |
|
| 87 |
root = ET.fromstring(ret.stdout) |
| 88 |
result = [elem.attrib["name"] for elem in root.findall('./TestCase')] |
| 89 |
|
| 90 |
if len(result) < 2: |
| 91 |
raise RuntimeError("Unexpectedly few tests listed (got {})".format( |
| 92 |
len(result))) |
| 93 |
|
| 94 |
return result |
| 95 |
|
| 96 |
|
| 97 |
def test_sharded_listing(self_test_exe: str) -> Dict[int, List[str]]: |
| 98 |
""" |
| 99 |
Asks the test binary for list of all tests, and also for lists of |
| 100 |
tests from shards. |
| 101 |
|
| 102 |
The combination of shards is then checked whether it corresponds to |
| 103 |
the full list of all tests. |
| 104 |
|
| 105 |
Returns the dictionary of shard-index => listed tests for later use. |
| 106 |
""" |
| 107 |
all_tests = list_tests(self_test_exe) |
| 108 |
big_shard_tests = list_tests(self_test_exe, ['--shard-count', '1', '--shard-index', '0']) |
| 109 |
|
| 110 |
assert all_tests == big_shard_tests, ( |
| 111 |
"No-sharding test list does not match the listing of big shard:\nNo shard:\n{}\n\nWith shard:\n{}\n".format( |
| 112 |
'\n'.join(all_tests), |
| 113 |
'\n'.join(big_shard_tests) |
| 114 |
) |
| 115 |
) |
| 116 |
|
| 117 |
shard_listings = dict() |
| 118 |
for shard_idx in range(number_of_shards): |
| 119 |
shard_listings[shard_idx] = list_tests(self_test_exe, ['--shard-count', str(number_of_shards), '--shard-index', str(shard_idx)]) |
| 120 |
|
| 121 |
shard_sizes = [len(v) for v in shard_listings.values()] |
| 122 |
assert len(all_tests) == sum(shard_sizes) |
| 123 |
|
| 124 |
# Check that the shards have roughly the right sizes (e.g. we don't |
| 125 |
# have all tests in single shard and the others are empty) |
| 126 |
differences = [abs(x1 - x2) for x1, x2 in zip(shard_sizes, shard_sizes[1:])] |
| 127 |
assert all(diff <= 1 for diff in differences), "A shard has weird size: {}".format(shard_sizes) |
| 128 |
|
| 129 |
combined_shards = [inner for outer in shard_listings.values() for inner in outer] |
| 130 |
assert all_tests == combined_shards, ( |
| 131 |
"All tests and combined shards disagree.\nNo shard:\n{}\n\nCombined:\n{}\n\n".format( |
| 132 |
'\n'.join(all_tests), |
| 133 |
'\n'.join(combined_shards) |
| 134 |
) |
| 135 |
) |
| 136 |
shard_listings[-1] = all_tests |
| 137 |
|
| 138 |
return shard_listings |
| 139 |
|
| 140 |
|
| 141 |
def test_sharded_execution(self_test_exe: str, listings: Dict[int, List[str]]): |
| 142 |
""" |
| 143 |
Runs the test binary and checks that the executed tests match the |
| 144 |
previously listed tests. |
| 145 |
|
| 146 |
Also does this for various shard indices, and that the combination |
| 147 |
of all shards matches the full run/listing. |
| 148 |
""" |
| 149 |
all_tests = execute_tests(self_test_exe) |
| 150 |
big_shard_tests = execute_tests(self_test_exe, ['--shard-count', '1', '--shard-index', '0']) |
| 151 |
assert all_tests == big_shard_tests |
| 152 |
|
| 153 |
assert listings[-1] == all_tests |
| 154 |
|
| 155 |
for shard_idx in range(number_of_shards): |
| 156 |
assert listings[shard_idx] == execute_tests(self_test_exe, ['--shard-count', str(number_of_shards), '--shard-index', str(shard_idx)]) |
| 157 |
|
| 158 |
|
| 159 |
def main(): |
| 160 |
self_test_exe, = sys.argv[1:] |
| 161 |
listings = test_sharded_listing(self_test_exe) |
| 162 |
test_sharded_execution(self_test_exe, listings) |
| 163 |
|
| 164 |
if __name__ == '__main__': |
| 165 |
sys.exit(main()) |