Debian Codegraph @

tests/TestScripts/testBazelReporter.py

raw
1 #!/usr/bin/env python3
2
3 # Copyright Catch2 Authors
4 # Distributed under the Boost Software License, Version 1.0.
5 # (See accompanying file LICENSE.txt or copy at
6 # https://www.boost.org/LICENSE_1_0.txt)
7
8 # SPDX-License-Identifier: BSL-1.0
9
10 import os
11 import re
12 import sys
13 import xml.etree.ElementTree as ET
14 import subprocess
15
16 """
17 Test that Catch2 recognizes `XML_OUTPUT_FILE` env variable and creates
18 a junit reporter that writes to the provided path.
19
20 Requires 2 arguments, path to Catch2 binary configured with
21 `CATCH_CONFIG_BAZEL_SUPPORT`, and the output directory for the output file.
22 """
23 if len(sys.argv) != 3:
24 print("Wrong number of arguments: {}".format(len(sys.argv)))
25 print("Usage: {} test-bin-path output-dir".format(sys.argv[0]))
26 exit(1)
27
28
29 bin_path = os.path.abspath(sys.argv[1])
30 output_dir = os.path.abspath(sys.argv[2])
31 xml_out_path = os.path.join(output_dir, '{}.xml'.format(os.path.basename(bin_path)))
32
33 # Ensure no file exists from previous test runs
34 if os.path.isfile(xml_out_path):
35 os.remove(xml_out_path)
36
37 print('bin path:', bin_path)
38 print('xml out path:', xml_out_path)
39
40 env = os.environ.copy()
41 env["XML_OUTPUT_FILE"] = xml_out_path
42 test_passing = True
43
44 try:
45 ret = subprocess.run(
46 bin_path,
47 stdout=subprocess.PIPE,
48 stderr=subprocess.PIPE,
49 check=True,
50 universal_newlines=True,
51 env=env
52 )
53 stdout = ret.stdout
54 except subprocess.SubprocessError as ex:
55 if ex.returncode == 42:
56 # The test cases are allowed to fail.
57 test_passing = False
58 stdout = ex.stdout
59 else:
60 print('Could not run "{}"'.format(bin_path))
61 print("Return code: {}".format(ex.returncode))
62 print("stdout: {}".format(ex.stdout))
63 print("stderr: {}".format(ex.stderr))
64 raise
65
66 # Check for valid XML output
67 try:
68 tree = ET.parse(xml_out_path)
69 except ET.ParseError as ex:
70 print("Invalid XML: '{}'".format(ex))
71 raise
72 except FileNotFoundError as ex:
73 print("Could not find '{}'".format(xml_out_path))
74 raise
75
76 bin_name = os.path.basename(bin_path)
77 # Check for matching testsuite
78 if not tree.find('.//testsuite[@name="{}"]'.format(bin_name)):
79 print("Could not find '{}' testsuite".format(bin_name))
80 exit(2)
81
82 # Check that we haven't disabled the default reporter
83 summary_test_cases = re.findall(r'test cases: \d* \| \d* passed \| \d* failed', stdout)
84 if len(summary_test_cases) == 0:
85 print("Could not find test summary in {}".format(stdout))
86 exit(2)
87
88 total, passed, failed = [int(s) for s in summary_test_cases[0].split() if s.isdigit()]
89
90 if failed == 0 and not test_passing:
91 print("Expected at least 1 test failure!")
92 exit(2)
93
94 if len(tree.findall('.//testcase')) != total:
95 print("Unexpected number of test cases!")
96 exit(2)
97
98 if len(tree.findall('.//failure')) != failed:
99 print("Unexpected number of test failures!")
100 exit(2)
101
102 if (passed + failed) != total:
103 print("Something has gone very wrong, ({} + {}) != {}".format(passed, failed, total))
104 exit(2)