Debian Codegraph @

tests/TestScripts/testRandomOrder.py

raw
1 #!/usr/bin/env python3
2
3 # Copyright Catch2 Authors
4 # Distributed under the Boost Software License, Version 1.0.
5 # (See accompanying file LICENSE.txt or copy at
6 # https://www.boost.org/LICENSE_1_0.txt)
7
8 # SPDX-License-Identifier: BSL-1.0
9
10 """
11 This test script verifies that the random ordering of tests inside
12 Catch2 is invariant in regards to subsetting. This is done by running
13 the binary 3 times, once with all tests selected, and twice with smaller
14 subsets of tests selected, and verifying that the selected tests are in
15 the same relative order.
16 """
17
18 import subprocess
19 import sys
20 import random
21 import xml.etree.ElementTree as ET
22
23 def none_to_empty_str(e):
24 if e is None:
25 return ""
26 assert type(e) is str
27 return e
28
29 def list_tests(self_test_exe, tags, rng_seed):
30 cmd = [self_test_exe, '--reporter', 'xml', '--list-tests', '--order', 'rand',
31 '--rng-seed', str(rng_seed)]
32 tags_arg = ','.join('[{}]~[.]'.format(t) for t in tags)
33 if tags_arg:
34 cmd.append(tags_arg)
35 process = subprocess.Popen(
36 cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
37 stdout, stderr = process.communicate()
38 if stderr:
39 raise RuntimeError("Unexpected error output:\n" + process.stderr)
40
41 root = ET.fromstring(stdout)
42 result = [(none_to_empty_str(tc.find('Name').text),
43 none_to_empty_str(tc.find('Tags').text),
44 none_to_empty_str(tc.find('ClassName').text)) for tc in root.findall('./TestCase')]
45
46 if len(result) < 2:
47 raise RuntimeError("Unexpectedly few tests listed (got {})".format(
48 len(result)))
49 return result
50
51 def check_is_sublist_of(shorter, longer):
52 assert len(shorter) < len(longer)
53 assert len(set(longer)) == len(longer)
54
55 indexes_in_longer = {s: i for i, s in enumerate(longer)}
56 for s1, s2 in zip(shorter, shorter[1:]):
57 assert indexes_in_longer[s1] < indexes_in_longer[s2], (
58 '{} comes before {} in longer list.\n'
59 'Longer: {}\nShorter: {}'.format(s2, s1, longer, shorter))
60
61 def main():
62 self_test_exe, = sys.argv[1:]
63
64 # We want a random seed for the test, but want to avoid 0,
65 # because it has special meaning
66 seed = random.randint(1, 2 ** 32 - 1)
67
68 list_one_tag = list_tests(self_test_exe, ['generators'], seed)
69 list_two_tags = list_tests(self_test_exe, ['generators', 'matchers'], seed)
70 list_all = list_tests(self_test_exe, [], seed)
71
72 # First, verify that restricting to a subset yields the same order
73 check_is_sublist_of(list_two_tags, list_all)
74 check_is_sublist_of(list_one_tag, list_two_tags)
75
76 if __name__ == '__main__':
77 sys.exit(main())