| 1 |
#!/usr/bin/env python3 |
| 2 |
|
| 3 |
# Copyright Catch2 Authors |
| 4 |
# Distributed under the Boost Software License, Version 1.0. |
| 5 |
# (See accompanying file LICENSE.txt or copy at |
| 6 |
# https://www.boost.org/LICENSE_1_0.txt) |
| 7 |
|
| 8 |
# SPDX-License-Identifier: BSL-1.0 |
| 9 |
|
| 10 |
import os |
| 11 |
import subprocess |
| 12 |
import sys |
| 13 |
import re |
| 14 |
import json |
| 15 |
|
| 16 |
cmake_version_regex = re.compile('cmake version (\d+)\.(\d+)\.(\d+)') |
| 17 |
|
| 18 |
def get_cmake_version(): |
| 19 |
result = subprocess.run(['cmake', '--version'], |
| 20 |
capture_output = True, |
| 21 |
check = True, |
| 22 |
text = True) |
| 23 |
version_match = cmake_version_regex.match(result.stdout) |
| 24 |
if not version_match: |
| 25 |
print('Could not find cmake version in output') |
| 26 |
print(f"output: '{result.stdout}'") |
| 27 |
exit(4) |
| 28 |
return (int(version_match.group(1)), |
| 29 |
int(version_match.group(2)), |
| 30 |
int(version_match.group(3))) |
| 31 |
|
| 32 |
def build_project(sources_dir, output_base_path, catch2_path): |
| 33 |
build_dir = os.path.join(output_base_path, 'ctest-registration-test') |
| 34 |
config_cmd = ['cmake', |
| 35 |
'-B', build_dir, |
| 36 |
'-S', sources_dir, |
| 37 |
f'-DCATCH2_PATH={catch2_path}', |
| 38 |
'-DCMAKE_BUILD_TYPE=Debug'] |
| 39 |
|
| 40 |
build_cmd = ['cmake', |
| 41 |
'--build', build_dir, |
| 42 |
'--config', 'Debug'] |
| 43 |
|
| 44 |
try: |
| 45 |
subprocess.run(config_cmd, |
| 46 |
capture_output = True, |
| 47 |
check = True, |
| 48 |
text = True) |
| 49 |
subprocess.run(build_cmd, |
| 50 |
capture_output = True, |
| 51 |
check = True, |
| 52 |
text = True) |
| 53 |
except subprocess.CalledProcessError as err: |
| 54 |
print('Error when building the test project') |
| 55 |
print(f'cmd: {err.cmd}') |
| 56 |
print(f'stderr: {err.stderr}') |
| 57 |
print(f'stdout: {err.stdout}') |
| 58 |
exit(3) |
| 59 |
|
| 60 |
return build_dir |
| 61 |
|
| 62 |
|
| 63 |
|
| 64 |
def get_test_names(build_path): |
| 65 |
# For now we assume that Windows builds are done using MSBuild under |
| 66 |
# Debug configuration. This means that we need to add "Debug" folder |
| 67 |
# to the path when constructing it. On Linux, we don't add anything. |
| 68 |
config_path = "Debug" if os.name == 'nt' else "" |
| 69 |
full_path = os.path.join(build_path, config_path, 'tests') |
| 70 |
|
| 71 |
|
| 72 |
cmd = [full_path, '--reporter', 'xml', '--list-tests'] |
| 73 |
result = subprocess.run(cmd, |
| 74 |
capture_output = True, |
| 75 |
check = True, |
| 76 |
text = True) |
| 77 |
|
| 78 |
import xml.etree.ElementTree as ET |
| 79 |
root = ET.fromstring(result.stdout) |
| 80 |
return [tc.text for tc in root.findall('TestCase/Name')] |
| 81 |
|
| 82 |
def get_ctest_listing(build_path): |
| 83 |
old_path = os.getcwd() |
| 84 |
os.chdir(build_path) |
| 85 |
|
| 86 |
cmd = ['ctest', '-C', 'debug', '--show-only=json-v1'] |
| 87 |
result = subprocess.run(cmd, |
| 88 |
capture_output = True, |
| 89 |
check = True, |
| 90 |
text = True) |
| 91 |
os.chdir(old_path) |
| 92 |
return result.stdout |
| 93 |
|
| 94 |
def extract_tests_from_ctest(ctest_output): |
| 95 |
ctest_response = json.loads(ctest_output) |
| 96 |
tests = ctest_response['tests'] |
| 97 |
test_names = [] |
| 98 |
for test in tests: |
| 99 |
test_command = test['command'] |
| 100 |
# First part of the command is the binary, second is the filter. |
| 101 |
# If there are less, registration has failed. If there are more, |
| 102 |
# registration has changed and the script needs updating. |
| 103 |
assert len(test_command) == 2 |
| 104 |
test_names.append(test_command[1]) |
| 105 |
test_name = test_command[1] |
| 106 |
|
| 107 |
return test_names |
| 108 |
|
| 109 |
def check_DL_PATHS(ctest_output): |
| 110 |
ctest_response = json.loads(ctest_output) |
| 111 |
tests = ctest_response['tests'] |
| 112 |
for test in tests: |
| 113 |
properties = test['properties'] |
| 114 |
for property in properties: |
| 115 |
if property['name'] == 'ENVIRONMENT_MODIFICATION': |
| 116 |
assert len(property['value']) == 2, f"The test provides 2 arguments to DL_PATHS, but instead found {len(property['value'])}" |
| 117 |
|
| 118 |
def escape_catch2_test_name(name): |
| 119 |
for char in ('\\', ',', '[', ']'): |
| 120 |
name = name.replace(char, f"\\{char}") |
| 121 |
return name |
| 122 |
|
| 123 |
if __name__ == '__main__': |
| 124 |
if len(sys.argv) != 3: |
| 125 |
print(f'Usage: {sys.argv[0]} path-to-catch2-cml output-path') |
| 126 |
exit(2) |
| 127 |
catch2_path = sys.argv[1] |
| 128 |
output_base_path = sys.argv[2] |
| 129 |
sources_dir = os.path.dirname(os.path.abspath(sys.argv[0])) |
| 130 |
|
| 131 |
build_path = build_project(sources_dir, output_base_path, catch2_path) |
| 132 |
|
| 133 |
catch_test_names = [escape_catch2_test_name(name) for name in get_test_names(build_path)] |
| 134 |
ctest_output = get_ctest_listing(build_path) |
| 135 |
ctest_test_names = extract_tests_from_ctest(ctest_output) |
| 136 |
|
| 137 |
mismatched = 0 |
| 138 |
for catch_test in catch_test_names: |
| 139 |
if catch_test not in ctest_test_names: |
| 140 |
print(f"Catch2 test '{catch_test}' not found in CTest") |
| 141 |
mismatched += 1 |
| 142 |
for ctest_test in ctest_test_names: |
| 143 |
if ctest_test not in catch_test_names: |
| 144 |
print(f"CTest test '{ctest_test}' not found in Catch2") |
| 145 |
mismatched += 1 |
| 146 |
|
| 147 |
if mismatched: |
| 148 |
print(f"Found {mismatched} mismatched tests catch test names and ctest test commands!") |
| 149 |
exit(1) |
| 150 |
|
| 151 |
cmake_version = get_cmake_version() |
| 152 |
if cmake_version >= (3, 27): |
| 153 |
check_DL_PATHS(ctest_output) |