| 1 |
#!/usr/bin/env python |
| 2 |
# |
| 3 |
# Copyright 2008, Google Inc. |
| 4 |
# All rights reserved. |
| 5 |
# |
| 6 |
# Redistribution and use in source and binary forms, with or without |
| 7 |
# modification, are permitted provided that the following conditions are |
| 8 |
# met: |
| 9 |
# |
| 10 |
# * Redistributions of source code must retain the above copyright |
| 11 |
# notice, this list of conditions and the following disclaimer. |
| 12 |
# * Redistributions in binary form must reproduce the above |
| 13 |
# copyright notice, this list of conditions and the following disclaimer |
| 14 |
# in the documentation and/or other materials provided with the |
| 15 |
# distribution. |
| 16 |
# * Neither the name of Google Inc. nor the names of its |
| 17 |
# contributors may be used to endorse or promote products derived from |
| 18 |
# this software without specific prior written permission. |
| 19 |
# |
| 20 |
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| 21 |
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| 22 |
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| 23 |
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| 24 |
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 25 |
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| 26 |
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 27 |
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 28 |
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 29 |
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 30 |
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 31 |
|
| 32 |
r"""Tests the text output of Google C++ Testing and Mocking Framework. |
| 33 |
|
| 34 |
To update the golden file: |
| 35 |
googletest_output_test.py --build_dir=BUILD/DIR --gengolden |
| 36 |
where BUILD/DIR contains the built googletest-output-test_ file. |
| 37 |
googletest_output_test.py --gengolden |
| 38 |
googletest_output_test.py |
| 39 |
""" |
| 40 |
|
| 41 |
import difflib |
| 42 |
import os |
| 43 |
import re |
| 44 |
import sys |
| 45 |
from googletest.test import gtest_test_utils |
| 46 |
|
| 47 |
|
| 48 |
# The flag for generating the golden file |
| 49 |
GENGOLDEN_FLAG = '--gengolden' |
| 50 |
CATCH_EXCEPTIONS_ENV_VAR_NAME = 'GTEST_CATCH_EXCEPTIONS' |
| 51 |
|
| 52 |
# The flag indicating stacktraces are not supported |
| 53 |
NO_STACKTRACE_SUPPORT_FLAG = '--no_stacktrace_support' |
| 54 |
|
| 55 |
IS_LINUX = os.name == 'posix' and os.uname()[0] == 'Linux' |
| 56 |
IS_WINDOWS = os.name == 'nt' |
| 57 |
|
| 58 |
GOLDEN_NAME = 'googletest-output-test-golden-lin.txt' |
| 59 |
|
| 60 |
PROGRAM_PATH = gtest_test_utils.GetTestExecutablePath('googletest-output-test_') |
| 61 |
|
| 62 |
# At least one command we exercise must not have the |
| 63 |
# 'internal_skip_environment_and_ad_hoc_tests' argument. |
| 64 |
COMMAND_LIST_TESTS = ({}, [PROGRAM_PATH, '--gtest_list_tests']) |
| 65 |
COMMAND_WITH_COLOR = ({}, [PROGRAM_PATH, '--gtest_color=yes']) |
| 66 |
COMMAND_WITH_TIME = ({}, [PROGRAM_PATH, |
| 67 |
'--gtest_print_time', |
| 68 |
'internal_skip_environment_and_ad_hoc_tests', |
| 69 |
'--gtest_filter=FatalFailureTest.*:LoggingTest.*']) |
| 70 |
COMMAND_WITH_DISABLED = ( |
| 71 |
{}, [PROGRAM_PATH, |
| 72 |
'--gtest_also_run_disabled_tests', |
| 73 |
'internal_skip_environment_and_ad_hoc_tests', |
| 74 |
'--gtest_filter=*DISABLED_*']) |
| 75 |
COMMAND_WITH_SHARDING = ( |
| 76 |
{'GTEST_SHARD_INDEX': '1', 'GTEST_TOTAL_SHARDS': '2'}, |
| 77 |
[PROGRAM_PATH, |
| 78 |
'internal_skip_environment_and_ad_hoc_tests', |
| 79 |
'--gtest_filter=PassingTest.*']) |
| 80 |
|
| 81 |
GOLDEN_PATH = os.path.join(gtest_test_utils.GetSourceDir(), GOLDEN_NAME) |
| 82 |
|
| 83 |
|
| 84 |
def ToUnixLineEnding(s): |
| 85 |
"""Changes all Windows/Mac line endings in s to UNIX line endings.""" |
| 86 |
|
| 87 |
return s.replace('\r\n', '\n').replace('\r', '\n') |
| 88 |
|
| 89 |
|
| 90 |
def RemoveLocations(test_output): |
| 91 |
"""Removes all file location info from a Google Test program's output. |
| 92 |
|
| 93 |
Args: |
| 94 |
test_output: the output of a Google Test program. |
| 95 |
|
| 96 |
Returns: |
| 97 |
output with all file location info (in the form of |
| 98 |
'DIRECTORY/FILE_NAME:LINE_NUMBER: 'or |
| 99 |
'DIRECTORY\\FILE_NAME(LINE_NUMBER): ') replaced by |
| 100 |
'FILE_NAME:#: '. |
| 101 |
""" |
| 102 |
|
| 103 |
return re.sub(r'.*[/\\]((googletest-output-test_|gtest).cc)(\:\d+|\(\d+\))\: ', |
| 104 |
r'\1:#: ', test_output) |
| 105 |
|
| 106 |
|
| 107 |
def RemoveStackTraceDetails(output): |
| 108 |
"""Removes all stack traces from a Google Test program's output.""" |
| 109 |
|
| 110 |
# *? means "find the shortest string that matches". |
| 111 |
return re.sub(r'Stack trace:(.|\n)*?\n\n', |
| 112 |
'Stack trace: (omitted)\n\n', output) |
| 113 |
|
| 114 |
|
| 115 |
def RemoveStackTraces(output): |
| 116 |
"""Removes all traces of stack traces from a Google Test program's output.""" |
| 117 |
|
| 118 |
# *? means "find the shortest string that matches". |
| 119 |
return re.sub(r'Stack trace:(.|\n)*?\n\n', '', output) |
| 120 |
|
| 121 |
|
| 122 |
def RemoveTime(output): |
| 123 |
"""Removes all time information from a Google Test program's output.""" |
| 124 |
|
| 125 |
return re.sub(r'\(\d+ ms', '(? ms', output) |
| 126 |
|
| 127 |
|
| 128 |
def RemoveTypeInfoDetails(test_output): |
| 129 |
"""Removes compiler-specific type info from Google Test program's output. |
| 130 |
|
| 131 |
Args: |
| 132 |
test_output: the output of a Google Test program. |
| 133 |
|
| 134 |
Returns: |
| 135 |
output with type information normalized to canonical form. |
| 136 |
""" |
| 137 |
|
| 138 |
# some compilers output the name of type 'unsigned int' as 'unsigned' |
| 139 |
return re.sub(r'unsigned int', 'unsigned', test_output) |
| 140 |
|
| 141 |
|
| 142 |
def NormalizeToCurrentPlatform(test_output): |
| 143 |
"""Normalizes platform specific output details for easier comparison.""" |
| 144 |
|
| 145 |
if IS_WINDOWS: |
| 146 |
# Removes the color information that is not present on Windows. |
| 147 |
test_output = re.sub('\x1b\\[(0;3\d)?m', '', test_output) |
| 148 |
# Changes failure message headers into the Windows format. |
| 149 |
test_output = re.sub(r': Failure\n', r': error: ', test_output) |
| 150 |
# Changes file(line_number) to file:line_number. |
| 151 |
test_output = re.sub(r'((\w|\.)+)\((\d+)\):', r'\1:\3:', test_output) |
| 152 |
|
| 153 |
return test_output |
| 154 |
|
| 155 |
|
| 156 |
def RemoveTestCounts(output): |
| 157 |
"""Removes test counts from a Google Test program's output.""" |
| 158 |
|
| 159 |
output = re.sub(r'\d+ tests?, listed below', |
| 160 |
'? tests, listed below', output) |
| 161 |
output = re.sub(r'\d+ FAILED TESTS', |
| 162 |
'? FAILED TESTS', output) |
| 163 |
output = re.sub(r'\d+ tests? from \d+ test cases?', |
| 164 |
'? tests from ? test cases', output) |
| 165 |
output = re.sub(r'\d+ tests? from ([a-zA-Z_])', |
| 166 |
r'? tests from \1', output) |
| 167 |
return re.sub(r'\d+ tests?\.', '? tests.', output) |
| 168 |
|
| 169 |
|
| 170 |
def RemoveMatchingTests(test_output, pattern): |
| 171 |
"""Removes output of specified tests from a Google Test program's output. |
| 172 |
|
| 173 |
This function strips not only the beginning and the end of a test but also |
| 174 |
all output in between. |
| 175 |
|
| 176 |
Args: |
| 177 |
test_output: A string containing the test output. |
| 178 |
pattern: A regex string that matches names of test cases or |
| 179 |
tests to remove. |
| 180 |
|
| 181 |
Returns: |
| 182 |
Contents of test_output with tests whose names match pattern removed. |
| 183 |
""" |
| 184 |
|
| 185 |
test_output = re.sub( |
| 186 |
r'.*\[ RUN \] .*%s(.|\n)*?\[( FAILED | OK )\] .*%s.*\n' % ( |
| 187 |
pattern, pattern), |
| 188 |
'', |
| 189 |
test_output) |
| 190 |
return re.sub(r'.*%s.*\n' % pattern, '', test_output) |
| 191 |
|
| 192 |
|
| 193 |
def NormalizeOutput(output): |
| 194 |
"""Normalizes output (the output of googletest-output-test_.exe).""" |
| 195 |
|
| 196 |
output = ToUnixLineEnding(output) |
| 197 |
output = RemoveLocations(output) |
| 198 |
output = RemoveStackTraceDetails(output) |
| 199 |
output = RemoveTime(output) |
| 200 |
return output |
| 201 |
|
| 202 |
|
| 203 |
def GetShellCommandOutput(env_cmd): |
| 204 |
"""Runs a command in a sub-process, and returns its output in a string. |
| 205 |
|
| 206 |
Args: |
| 207 |
env_cmd: The shell command. A 2-tuple where element 0 is a dict of extra |
| 208 |
environment variables to set, and element 1 is a string with |
| 209 |
the command and any flags. |
| 210 |
|
| 211 |
Returns: |
| 212 |
A string with the command's combined standard and diagnostic output. |
| 213 |
""" |
| 214 |
|
| 215 |
# Spawns cmd in a sub-process, and gets its standard I/O file objects. |
| 216 |
# Set and save the environment properly. |
| 217 |
environ = os.environ.copy() |
| 218 |
environ.update(env_cmd[0]) |
| 219 |
p = gtest_test_utils.Subprocess(env_cmd[1], env=environ) |
| 220 |
|
| 221 |
return p.output |
| 222 |
|
| 223 |
|
| 224 |
def GetCommandOutput(env_cmd): |
| 225 |
"""Runs a command and returns its output with all file location |
| 226 |
info stripped off. |
| 227 |
|
| 228 |
Args: |
| 229 |
env_cmd: The shell command. A 2-tuple where element 0 is a dict of extra |
| 230 |
environment variables to set, and element 1 is a string with |
| 231 |
the command and any flags. |
| 232 |
""" |
| 233 |
|
| 234 |
# Disables exception pop-ups on Windows. |
| 235 |
environ, cmdline = env_cmd |
| 236 |
environ = dict(environ) # Ensures we are modifying a copy. |
| 237 |
environ[CATCH_EXCEPTIONS_ENV_VAR_NAME] = '1' |
| 238 |
return NormalizeOutput(GetShellCommandOutput((environ, cmdline))) |
| 239 |
|
| 240 |
|
| 241 |
def GetOutputOfAllCommands(): |
| 242 |
"""Returns concatenated output from several representative commands.""" |
| 243 |
|
| 244 |
return (GetCommandOutput(COMMAND_WITH_COLOR) + |
| 245 |
GetCommandOutput(COMMAND_WITH_TIME) + |
| 246 |
GetCommandOutput(COMMAND_WITH_DISABLED) + |
| 247 |
GetCommandOutput(COMMAND_WITH_SHARDING)) |
| 248 |
|
| 249 |
|
| 250 |
test_list = GetShellCommandOutput(COMMAND_LIST_TESTS) |
| 251 |
SUPPORTS_DEATH_TESTS = 'DeathTest' in test_list |
| 252 |
SUPPORTS_TYPED_TESTS = 'TypedTest' in test_list |
| 253 |
SUPPORTS_THREADS = 'ExpectFailureWithThreadsTest' in test_list |
| 254 |
SUPPORTS_STACK_TRACES = NO_STACKTRACE_SUPPORT_FLAG not in sys.argv |
| 255 |
|
| 256 |
CAN_GENERATE_GOLDEN_FILE = (SUPPORTS_DEATH_TESTS and |
| 257 |
SUPPORTS_TYPED_TESTS and |
| 258 |
SUPPORTS_THREADS and |
| 259 |
SUPPORTS_STACK_TRACES) |
| 260 |
|
| 261 |
class GTestOutputTest(gtest_test_utils.TestCase): |
| 262 |
def RemoveUnsupportedTests(self, test_output): |
| 263 |
if not SUPPORTS_DEATH_TESTS: |
| 264 |
test_output = RemoveMatchingTests(test_output, 'DeathTest') |
| 265 |
if not SUPPORTS_TYPED_TESTS: |
| 266 |
test_output = RemoveMatchingTests(test_output, 'TypedTest') |
| 267 |
test_output = RemoveMatchingTests(test_output, 'TypedDeathTest') |
| 268 |
test_output = RemoveMatchingTests(test_output, 'TypeParamDeathTest') |
| 269 |
if not SUPPORTS_THREADS: |
| 270 |
test_output = RemoveMatchingTests(test_output, |
| 271 |
'ExpectFailureWithThreadsTest') |
| 272 |
test_output = RemoveMatchingTests(test_output, |
| 273 |
'ScopedFakeTestPartResultReporterTest') |
| 274 |
test_output = RemoveMatchingTests(test_output, |
| 275 |
'WorksConcurrently') |
| 276 |
if not SUPPORTS_STACK_TRACES: |
| 277 |
test_output = RemoveStackTraces(test_output) |
| 278 |
|
| 279 |
return test_output |
| 280 |
|
| 281 |
def testOutput(self): |
| 282 |
output = GetOutputOfAllCommands() |
| 283 |
|
| 284 |
golden_file = open(GOLDEN_PATH, 'rb') |
| 285 |
# A mis-configured source control system can cause \r appear in EOL |
| 286 |
# sequences when we read the golden file irrespective of an operating |
| 287 |
# system used. Therefore, we need to strip those \r's from newlines |
| 288 |
# unconditionally. |
| 289 |
golden = ToUnixLineEnding(golden_file.read().decode()) |
| 290 |
golden_file.close() |
| 291 |
|
| 292 |
# We want the test to pass regardless of certain features being |
| 293 |
# supported or not. |
| 294 |
|
| 295 |
# We still have to remove type name specifics in all cases. |
| 296 |
normalized_actual = RemoveTypeInfoDetails(output) |
| 297 |
normalized_golden = RemoveTypeInfoDetails(golden) |
| 298 |
|
| 299 |
if CAN_GENERATE_GOLDEN_FILE: |
| 300 |
self.assertEqual(normalized_golden, normalized_actual, |
| 301 |
'\n'.join(difflib.unified_diff( |
| 302 |
normalized_golden.split('\n'), |
| 303 |
normalized_actual.split('\n'), |
| 304 |
'golden', 'actual'))) |
| 305 |
else: |
| 306 |
normalized_actual = NormalizeToCurrentPlatform( |
| 307 |
RemoveTestCounts(normalized_actual)) |
| 308 |
normalized_golden = NormalizeToCurrentPlatform( |
| 309 |
RemoveTestCounts(self.RemoveUnsupportedTests(normalized_golden))) |
| 310 |
|
| 311 |
# This code is very handy when debugging golden file differences: |
| 312 |
if os.getenv('DEBUG_GTEST_OUTPUT_TEST'): |
| 313 |
open(os.path.join( |
| 314 |
gtest_test_utils.GetSourceDir(), |
| 315 |
'_googletest-output-test_normalized_actual.txt'), 'wb').write( |
| 316 |
normalized_actual) |
| 317 |
open(os.path.join( |
| 318 |
gtest_test_utils.GetSourceDir(), |
| 319 |
'_googletest-output-test_normalized_golden.txt'), 'wb').write( |
| 320 |
normalized_golden) |
| 321 |
|
| 322 |
self.assertEqual(normalized_golden, normalized_actual) |
| 323 |
|
| 324 |
|
| 325 |
if __name__ == '__main__': |
| 326 |
if NO_STACKTRACE_SUPPORT_FLAG in sys.argv: |
| 327 |
# unittest.main() can't handle unknown flags |
| 328 |
sys.argv.remove(NO_STACKTRACE_SUPPORT_FLAG) |
| 329 |
|
| 330 |
if GENGOLDEN_FLAG in sys.argv: |
| 331 |
if CAN_GENERATE_GOLDEN_FILE: |
| 332 |
output = GetOutputOfAllCommands() |
| 333 |
golden_file = open(GOLDEN_PATH, 'wb') |
| 334 |
golden_file.write(output.encode()) |
| 335 |
golden_file.close() |
| 336 |
else: |
| 337 |
message = ( |
| 338 |
"""Unable to write a golden file when compiled in an environment |
| 339 |
that does not support all the required features (death tests, |
| 340 |
typed tests, stack traces, and multiple threads). |
| 341 |
Please build this test and generate the golden file using Blaze on Linux.""") |
| 342 |
|
| 343 |
sys.stderr.write(message) |
| 344 |
sys.exit(1) |
| 345 |
else: |
| 346 |
gtest_test_utils.Main() |