| 1 |
#!/usr/bin/env python3 |
| 2 |
|
| 3 |
# Copyright Catch2 Authors |
| 4 |
# Distributed under the Boost Software License, Version 1.0. |
| 5 |
# (See accompanying file LICENSE.txt or copy at |
| 6 |
# https://www.boost.org/LICENSE_1_0.txt) |
| 7 |
|
| 8 |
# SPDX-License-Identifier: BSL-1.0 |
| 9 |
|
| 10 |
""" |
| 11 |
This test script verifies that the random ordering of tests inside |
| 12 |
Catch2 is invariant in regards to subsetting. This is done by running |
| 13 |
the binary 3 times, once with all tests selected, and twice with smaller |
| 14 |
subsets of tests selected, and verifying that the selected tests are in |
| 15 |
the same relative order. |
| 16 |
""" |
| 17 |
|
| 18 |
import subprocess |
| 19 |
import sys |
| 20 |
import random |
| 21 |
import xml.etree.ElementTree as ET |
| 22 |
|
| 23 |
def none_to_empty_str(e): |
| 24 |
if e is None: |
| 25 |
return "" |
| 26 |
assert type(e) is str |
| 27 |
return e |
| 28 |
|
| 29 |
def list_tests(self_test_exe, tags, rng_seed): |
| 30 |
cmd = [self_test_exe, '--reporter', 'xml', '--list-tests', '--order', 'rand', |
| 31 |
'--rng-seed', str(rng_seed)] |
| 32 |
tags_arg = ','.join('[{}]~[.]'.format(t) for t in tags) |
| 33 |
if tags_arg: |
| 34 |
cmd.append(tags_arg) |
| 35 |
process = subprocess.Popen( |
| 36 |
cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) |
| 37 |
stdout, stderr = process.communicate() |
| 38 |
if stderr: |
| 39 |
raise RuntimeError("Unexpected error output:\n" + process.stderr) |
| 40 |
|
| 41 |
root = ET.fromstring(stdout) |
| 42 |
result = [(none_to_empty_str(tc.find('Name').text), |
| 43 |
none_to_empty_str(tc.find('Tags').text), |
| 44 |
none_to_empty_str(tc.find('ClassName').text)) for tc in root.findall('./TestCase')] |
| 45 |
|
| 46 |
if len(result) < 2: |
| 47 |
raise RuntimeError("Unexpectedly few tests listed (got {})".format( |
| 48 |
len(result))) |
| 49 |
return result |
| 50 |
|
| 51 |
def check_is_sublist_of(shorter, longer): |
| 52 |
assert len(shorter) < len(longer) |
| 53 |
assert len(set(longer)) == len(longer) |
| 54 |
|
| 55 |
indexes_in_longer = {s: i for i, s in enumerate(longer)} |
| 56 |
for s1, s2 in zip(shorter, shorter[1:]): |
| 57 |
assert indexes_in_longer[s1] < indexes_in_longer[s2], ( |
| 58 |
'{} comes before {} in longer list.\n' |
| 59 |
'Longer: {}\nShorter: {}'.format(s2, s1, longer, shorter)) |
| 60 |
|
| 61 |
def main(): |
| 62 |
self_test_exe, = sys.argv[1:] |
| 63 |
|
| 64 |
# We want a random seed for the test, but want to avoid 0, |
| 65 |
# because it has special meaning |
| 66 |
seed = random.randint(1, 2 ** 32 - 1) |
| 67 |
|
| 68 |
list_one_tag = list_tests(self_test_exe, ['generators'], seed) |
| 69 |
list_two_tags = list_tests(self_test_exe, ['generators', 'matchers'], seed) |
| 70 |
list_all = list_tests(self_test_exe, [], seed) |
| 71 |
|
| 72 |
# First, verify that restricting to a subset yields the same order |
| 73 |
check_is_sublist_of(list_two_tags, list_all) |
| 74 |
check_is_sublist_of(list_one_tag, list_two_tags) |
| 75 |
|
| 76 |
if __name__ == '__main__': |
| 77 |
sys.exit(main()) |