Debian Codegraph @

tests/TestScripts/DiscoverTests/VerifyRegistration.py

raw
1 #!/usr/bin/env python3
2
3 # Copyright Catch2 Authors
4 # Distributed under the Boost Software License, Version 1.0.
5 # (See accompanying file LICENSE.txt or copy at
6 # https://www.boost.org/LICENSE_1_0.txt)
7
8 # SPDX-License-Identifier: BSL-1.0
9
10 import os
11 import subprocess
12 import sys
13 import re
14 import json
15
16 cmake_version_regex = re.compile('cmake version (\d+)\.(\d+)\.(\d+)')
17
18 def get_cmake_version():
19 result = subprocess.run(['cmake', '--version'],
20 capture_output = True,
21 check = True,
22 text = True)
23 version_match = cmake_version_regex.match(result.stdout)
24 if not version_match:
25 print('Could not find cmake version in output')
26 print(f"output: '{result.stdout}'")
27 exit(4)
28 return (int(version_match.group(1)),
29 int(version_match.group(2)),
30 int(version_match.group(3)))
31
32 def build_project(sources_dir, output_base_path, catch2_path):
33 build_dir = os.path.join(output_base_path, 'ctest-registration-test')
34 config_cmd = ['cmake',
35 '-B', build_dir,
36 '-S', sources_dir,
37 f'-DCATCH2_PATH={catch2_path}',
38 '-DCMAKE_BUILD_TYPE=Debug']
39
40 build_cmd = ['cmake',
41 '--build', build_dir,
42 '--config', 'Debug']
43
44 try:
45 subprocess.run(config_cmd,
46 capture_output = True,
47 check = True,
48 text = True)
49 subprocess.run(build_cmd,
50 capture_output = True,
51 check = True,
52 text = True)
53 except subprocess.CalledProcessError as err:
54 print('Error when building the test project')
55 print(f'cmd: {err.cmd}')
56 print(f'stderr: {err.stderr}')
57 print(f'stdout: {err.stdout}')
58 exit(3)
59
60 return build_dir
61
62
63
64 def get_test_names(build_path):
65 # For now we assume that Windows builds are done using MSBuild under
66 # Debug configuration. This means that we need to add "Debug" folder
67 # to the path when constructing it. On Linux, we don't add anything.
68 config_path = "Debug" if os.name == 'nt' else ""
69 full_path = os.path.join(build_path, config_path, 'tests')
70
71
72 cmd = [full_path, '--reporter', 'xml', '--list-tests']
73 result = subprocess.run(cmd,
74 capture_output = True,
75 check = True,
76 text = True)
77
78 import xml.etree.ElementTree as ET
79 root = ET.fromstring(result.stdout)
80 return [tc.text for tc in root.findall('TestCase/Name')]
81
82 def get_ctest_listing(build_path):
83 old_path = os.getcwd()
84 os.chdir(build_path)
85
86 cmd = ['ctest', '-C', 'debug', '--show-only=json-v1']
87 result = subprocess.run(cmd,
88 capture_output = True,
89 check = True,
90 text = True)
91 os.chdir(old_path)
92 return result.stdout
93
94 def extract_tests_from_ctest(ctest_output):
95 ctest_response = json.loads(ctest_output)
96 tests = ctest_response['tests']
97 test_names = []
98 for test in tests:
99 test_command = test['command']
100 # First part of the command is the binary, second is the filter.
101 # If there are less, registration has failed. If there are more,
102 # registration has changed and the script needs updating.
103 assert len(test_command) == 2
104 test_names.append(test_command[1])
105 test_name = test_command[1]
106
107 return test_names
108
109 def check_DL_PATHS(ctest_output):
110 ctest_response = json.loads(ctest_output)
111 tests = ctest_response['tests']
112 for test in tests:
113 properties = test['properties']
114 for property in properties:
115 if property['name'] == 'ENVIRONMENT_MODIFICATION':
116 assert len(property['value']) == 2, f"The test provides 2 arguments to DL_PATHS, but instead found {len(property['value'])}"
117
118 def escape_catch2_test_name(name):
119 for char in ('\\', ',', '[', ']'):
120 name = name.replace(char, f"\\{char}")
121 return name
122
123 if __name__ == '__main__':
124 if len(sys.argv) != 3:
125 print(f'Usage: {sys.argv[0]} path-to-catch2-cml output-path')
126 exit(2)
127 catch2_path = sys.argv[1]
128 output_base_path = sys.argv[2]
129 sources_dir = os.path.dirname(os.path.abspath(sys.argv[0]))
130
131 build_path = build_project(sources_dir, output_base_path, catch2_path)
132
133 catch_test_names = [escape_catch2_test_name(name) for name in get_test_names(build_path)]
134 ctest_output = get_ctest_listing(build_path)
135 ctest_test_names = extract_tests_from_ctest(ctest_output)
136
137 mismatched = 0
138 for catch_test in catch_test_names:
139 if catch_test not in ctest_test_names:
140 print(f"Catch2 test '{catch_test}' not found in CTest")
141 mismatched += 1
142 for ctest_test in ctest_test_names:
143 if ctest_test not in catch_test_names:
144 print(f"CTest test '{ctest_test}' not found in Catch2")
145 mismatched += 1
146
147 if mismatched:
148 print(f"Found {mismatched} mismatched tests catch test names and ctest test commands!")
149 exit(1)
150
151 cmake_version = get_cmake_version()
152 if cmake_version >= (3, 27):
153 check_DL_PATHS(ctest_output)