| 1 |
#!/usr/bin/env python |
| 2 |
# |
| 3 |
# Copyright 2009 Google Inc. All Rights Reserved. |
| 4 |
# |
| 5 |
# Redistribution and use in source and binary forms, with or without |
| 6 |
# modification, are permitted provided that the following conditions are |
| 7 |
# met: |
| 8 |
# |
| 9 |
# * Redistributions of source code must retain the above copyright |
| 10 |
# notice, this list of conditions and the following disclaimer. |
| 11 |
# * Redistributions in binary form must reproduce the above |
| 12 |
# copyright notice, this list of conditions and the following disclaimer |
| 13 |
# in the documentation and/or other materials provided with the |
| 14 |
# distribution. |
| 15 |
# * Neither the name of Google Inc. nor the names of its |
| 16 |
# contributors may be used to endorse or promote products derived from |
| 17 |
# this software without specific prior written permission. |
| 18 |
# |
| 19 |
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| 20 |
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| 21 |
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| 22 |
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| 23 |
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 24 |
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| 25 |
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 26 |
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 27 |
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 28 |
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 29 |
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 30 |
|
| 31 |
"""Verifies that test shuffling works.""" |
| 32 |
|
| 33 |
import os |
| 34 |
from googletest.test import gtest_test_utils |
| 35 |
|
| 36 |
# Command to run the googletest-shuffle-test_ program. |
| 37 |
COMMAND = gtest_test_utils.GetTestExecutablePath('googletest-shuffle-test_') |
| 38 |
|
| 39 |
# The environment variables for test sharding. |
| 40 |
TOTAL_SHARDS_ENV_VAR = 'GTEST_TOTAL_SHARDS' |
| 41 |
SHARD_INDEX_ENV_VAR = 'GTEST_SHARD_INDEX' |
| 42 |
|
| 43 |
TEST_FILTER = 'A*.A:A*.B:C*' |
| 44 |
|
| 45 |
ALL_TESTS = [] |
| 46 |
ACTIVE_TESTS = [] |
| 47 |
FILTERED_TESTS = [] |
| 48 |
SHARDED_TESTS = [] |
| 49 |
|
| 50 |
SHUFFLED_ALL_TESTS = [] |
| 51 |
SHUFFLED_ACTIVE_TESTS = [] |
| 52 |
SHUFFLED_FILTERED_TESTS = [] |
| 53 |
SHUFFLED_SHARDED_TESTS = [] |
| 54 |
|
| 55 |
|
| 56 |
def AlsoRunDisabledTestsFlag(): |
| 57 |
return '--gtest_also_run_disabled_tests' |
| 58 |
|
| 59 |
|
| 60 |
def FilterFlag(test_filter): |
| 61 |
return '--gtest_filter=%s' % (test_filter,) |
| 62 |
|
| 63 |
|
| 64 |
def RepeatFlag(n): |
| 65 |
return '--gtest_repeat=%s' % (n,) |
| 66 |
|
| 67 |
|
| 68 |
def ShuffleFlag(): |
| 69 |
return '--gtest_shuffle' |
| 70 |
|
| 71 |
|
| 72 |
def RandomSeedFlag(n): |
| 73 |
return '--gtest_random_seed=%s' % (n,) |
| 74 |
|
| 75 |
|
| 76 |
def RunAndReturnOutput(extra_env, args): |
| 77 |
"""Runs the test program and returns its output.""" |
| 78 |
|
| 79 |
environ_copy = os.environ.copy() |
| 80 |
environ_copy.update(extra_env) |
| 81 |
|
| 82 |
return gtest_test_utils.Subprocess([COMMAND] + args, env=environ_copy).output |
| 83 |
|
| 84 |
|
| 85 |
def GetTestsForAllIterations(extra_env, args): |
| 86 |
"""Runs the test program and returns a list of test lists. |
| 87 |
|
| 88 |
Args: |
| 89 |
extra_env: a map from environment variables to their values |
| 90 |
args: command line flags to pass to googletest-shuffle-test_ |
| 91 |
|
| 92 |
Returns: |
| 93 |
A list where the i-th element is the list of tests run in the i-th |
| 94 |
test iteration. |
| 95 |
""" |
| 96 |
|
| 97 |
test_iterations = [] |
| 98 |
for line in RunAndReturnOutput(extra_env, args).split('\n'): |
| 99 |
if line.startswith('----'): |
| 100 |
tests = [] |
| 101 |
test_iterations.append(tests) |
| 102 |
elif line.strip(): |
| 103 |
tests.append(line.strip()) # 'TestCaseName.TestName' |
| 104 |
|
| 105 |
return test_iterations |
| 106 |
|
| 107 |
|
| 108 |
def GetTestCases(tests): |
| 109 |
"""Returns a list of test cases in the given full test names. |
| 110 |
|
| 111 |
Args: |
| 112 |
tests: a list of full test names |
| 113 |
|
| 114 |
Returns: |
| 115 |
A list of test cases from 'tests', in their original order. |
| 116 |
Consecutive duplicates are removed. |
| 117 |
""" |
| 118 |
|
| 119 |
test_cases = [] |
| 120 |
for test in tests: |
| 121 |
test_case = test.split('.')[0] |
| 122 |
if not test_case in test_cases: |
| 123 |
test_cases.append(test_case) |
| 124 |
|
| 125 |
return test_cases |
| 126 |
|
| 127 |
|
| 128 |
def CalculateTestLists(): |
| 129 |
"""Calculates the list of tests run under different flags.""" |
| 130 |
|
| 131 |
if not ALL_TESTS: |
| 132 |
ALL_TESTS.extend( |
| 133 |
GetTestsForAllIterations({}, [AlsoRunDisabledTestsFlag()])[0]) |
| 134 |
|
| 135 |
if not ACTIVE_TESTS: |
| 136 |
ACTIVE_TESTS.extend(GetTestsForAllIterations({}, [])[0]) |
| 137 |
|
| 138 |
if not FILTERED_TESTS: |
| 139 |
FILTERED_TESTS.extend( |
| 140 |
GetTestsForAllIterations({}, [FilterFlag(TEST_FILTER)])[0]) |
| 141 |
|
| 142 |
if not SHARDED_TESTS: |
| 143 |
SHARDED_TESTS.extend( |
| 144 |
GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3', |
| 145 |
SHARD_INDEX_ENV_VAR: '1'}, |
| 146 |
[])[0]) |
| 147 |
|
| 148 |
if not SHUFFLED_ALL_TESTS: |
| 149 |
SHUFFLED_ALL_TESTS.extend(GetTestsForAllIterations( |
| 150 |
{}, [AlsoRunDisabledTestsFlag(), ShuffleFlag(), RandomSeedFlag(1)])[0]) |
| 151 |
|
| 152 |
if not SHUFFLED_ACTIVE_TESTS: |
| 153 |
SHUFFLED_ACTIVE_TESTS.extend(GetTestsForAllIterations( |
| 154 |
{}, [ShuffleFlag(), RandomSeedFlag(1)])[0]) |
| 155 |
|
| 156 |
if not SHUFFLED_FILTERED_TESTS: |
| 157 |
SHUFFLED_FILTERED_TESTS.extend(GetTestsForAllIterations( |
| 158 |
{}, [ShuffleFlag(), RandomSeedFlag(1), FilterFlag(TEST_FILTER)])[0]) |
| 159 |
|
| 160 |
if not SHUFFLED_SHARDED_TESTS: |
| 161 |
SHUFFLED_SHARDED_TESTS.extend( |
| 162 |
GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3', |
| 163 |
SHARD_INDEX_ENV_VAR: '1'}, |
| 164 |
[ShuffleFlag(), RandomSeedFlag(1)])[0]) |
| 165 |
|
| 166 |
|
| 167 |
class GTestShuffleUnitTest(gtest_test_utils.TestCase): |
| 168 |
"""Tests test shuffling.""" |
| 169 |
|
| 170 |
def setUp(self): |
| 171 |
CalculateTestLists() |
| 172 |
|
| 173 |
def testShufflePreservesNumberOfTests(self): |
| 174 |
self.assertEqual(len(ALL_TESTS), len(SHUFFLED_ALL_TESTS)) |
| 175 |
self.assertEqual(len(ACTIVE_TESTS), len(SHUFFLED_ACTIVE_TESTS)) |
| 176 |
self.assertEqual(len(FILTERED_TESTS), len(SHUFFLED_FILTERED_TESTS)) |
| 177 |
self.assertEqual(len(SHARDED_TESTS), len(SHUFFLED_SHARDED_TESTS)) |
| 178 |
|
| 179 |
def testShuffleChangesTestOrder(self): |
| 180 |
self.assert_(SHUFFLED_ALL_TESTS != ALL_TESTS, SHUFFLED_ALL_TESTS) |
| 181 |
self.assert_(SHUFFLED_ACTIVE_TESTS != ACTIVE_TESTS, SHUFFLED_ACTIVE_TESTS) |
| 182 |
self.assert_(SHUFFLED_FILTERED_TESTS != FILTERED_TESTS, |
| 183 |
SHUFFLED_FILTERED_TESTS) |
| 184 |
self.assert_(SHUFFLED_SHARDED_TESTS != SHARDED_TESTS, |
| 185 |
SHUFFLED_SHARDED_TESTS) |
| 186 |
|
| 187 |
def testShuffleChangesTestCaseOrder(self): |
| 188 |
self.assert_(GetTestCases(SHUFFLED_ALL_TESTS) != GetTestCases(ALL_TESTS), |
| 189 |
GetTestCases(SHUFFLED_ALL_TESTS)) |
| 190 |
self.assert_( |
| 191 |
GetTestCases(SHUFFLED_ACTIVE_TESTS) != GetTestCases(ACTIVE_TESTS), |
| 192 |
GetTestCases(SHUFFLED_ACTIVE_TESTS)) |
| 193 |
self.assert_( |
| 194 |
GetTestCases(SHUFFLED_FILTERED_TESTS) != GetTestCases(FILTERED_TESTS), |
| 195 |
GetTestCases(SHUFFLED_FILTERED_TESTS)) |
| 196 |
self.assert_( |
| 197 |
GetTestCases(SHUFFLED_SHARDED_TESTS) != GetTestCases(SHARDED_TESTS), |
| 198 |
GetTestCases(SHUFFLED_SHARDED_TESTS)) |
| 199 |
|
| 200 |
def testShuffleDoesNotRepeatTest(self): |
| 201 |
for test in SHUFFLED_ALL_TESTS: |
| 202 |
self.assertEqual(1, SHUFFLED_ALL_TESTS.count(test), |
| 203 |
'%s appears more than once' % (test,)) |
| 204 |
for test in SHUFFLED_ACTIVE_TESTS: |
| 205 |
self.assertEqual(1, SHUFFLED_ACTIVE_TESTS.count(test), |
| 206 |
'%s appears more than once' % (test,)) |
| 207 |
for test in SHUFFLED_FILTERED_TESTS: |
| 208 |
self.assertEqual(1, SHUFFLED_FILTERED_TESTS.count(test), |
| 209 |
'%s appears more than once' % (test,)) |
| 210 |
for test in SHUFFLED_SHARDED_TESTS: |
| 211 |
self.assertEqual(1, SHUFFLED_SHARDED_TESTS.count(test), |
| 212 |
'%s appears more than once' % (test,)) |
| 213 |
|
| 214 |
def testShuffleDoesNotCreateNewTest(self): |
| 215 |
for test in SHUFFLED_ALL_TESTS: |
| 216 |
self.assert_(test in ALL_TESTS, '%s is an invalid test' % (test,)) |
| 217 |
for test in SHUFFLED_ACTIVE_TESTS: |
| 218 |
self.assert_(test in ACTIVE_TESTS, '%s is an invalid test' % (test,)) |
| 219 |
for test in SHUFFLED_FILTERED_TESTS: |
| 220 |
self.assert_(test in FILTERED_TESTS, '%s is an invalid test' % (test,)) |
| 221 |
for test in SHUFFLED_SHARDED_TESTS: |
| 222 |
self.assert_(test in SHARDED_TESTS, '%s is an invalid test' % (test,)) |
| 223 |
|
| 224 |
def testShuffleIncludesAllTests(self): |
| 225 |
for test in ALL_TESTS: |
| 226 |
self.assert_(test in SHUFFLED_ALL_TESTS, '%s is missing' % (test,)) |
| 227 |
for test in ACTIVE_TESTS: |
| 228 |
self.assert_(test in SHUFFLED_ACTIVE_TESTS, '%s is missing' % (test,)) |
| 229 |
for test in FILTERED_TESTS: |
| 230 |
self.assert_(test in SHUFFLED_FILTERED_TESTS, '%s is missing' % (test,)) |
| 231 |
for test in SHARDED_TESTS: |
| 232 |
self.assert_(test in SHUFFLED_SHARDED_TESTS, '%s is missing' % (test,)) |
| 233 |
|
| 234 |
def testShuffleLeavesDeathTestsAtFront(self): |
| 235 |
non_death_test_found = False |
| 236 |
for test in SHUFFLED_ACTIVE_TESTS: |
| 237 |
if 'DeathTest.' in test: |
| 238 |
self.assert_(not non_death_test_found, |
| 239 |
'%s appears after a non-death test' % (test,)) |
| 240 |
else: |
| 241 |
non_death_test_found = True |
| 242 |
|
| 243 |
def _VerifyTestCasesDoNotInterleave(self, tests): |
| 244 |
test_cases = [] |
| 245 |
for test in tests: |
| 246 |
[test_case, _] = test.split('.') |
| 247 |
if test_cases and test_cases[-1] != test_case: |
| 248 |
test_cases.append(test_case) |
| 249 |
self.assertEqual(1, test_cases.count(test_case), |
| 250 |
'Test case %s is not grouped together in %s' % |
| 251 |
(test_case, tests)) |
| 252 |
|
| 253 |
def testShuffleDoesNotInterleaveTestCases(self): |
| 254 |
self._VerifyTestCasesDoNotInterleave(SHUFFLED_ALL_TESTS) |
| 255 |
self._VerifyTestCasesDoNotInterleave(SHUFFLED_ACTIVE_TESTS) |
| 256 |
self._VerifyTestCasesDoNotInterleave(SHUFFLED_FILTERED_TESTS) |
| 257 |
self._VerifyTestCasesDoNotInterleave(SHUFFLED_SHARDED_TESTS) |
| 258 |
|
| 259 |
def testShuffleRestoresOrderAfterEachIteration(self): |
| 260 |
# Get the test lists in all 3 iterations, using random seed 1, 2, |
| 261 |
# and 3 respectively. Google Test picks a different seed in each |
| 262 |
# iteration, and this test depends on the current implementation |
| 263 |
# picking successive numbers. This dependency is not ideal, but |
| 264 |
# makes the test much easier to write. |
| 265 |
[tests_in_iteration1, tests_in_iteration2, tests_in_iteration3] = ( |
| 266 |
GetTestsForAllIterations( |
| 267 |
{}, [ShuffleFlag(), RandomSeedFlag(1), RepeatFlag(3)])) |
| 268 |
|
| 269 |
# Make sure running the tests with random seed 1 gets the same |
| 270 |
# order as in iteration 1 above. |
| 271 |
[tests_with_seed1] = GetTestsForAllIterations( |
| 272 |
{}, [ShuffleFlag(), RandomSeedFlag(1)]) |
| 273 |
self.assertEqual(tests_in_iteration1, tests_with_seed1) |
| 274 |
|
| 275 |
# Make sure running the tests with random seed 2 gets the same |
| 276 |
# order as in iteration 2 above. Success means that Google Test |
| 277 |
# correctly restores the test order before re-shuffling at the |
| 278 |
# beginning of iteration 2. |
| 279 |
[tests_with_seed2] = GetTestsForAllIterations( |
| 280 |
{}, [ShuffleFlag(), RandomSeedFlag(2)]) |
| 281 |
self.assertEqual(tests_in_iteration2, tests_with_seed2) |
| 282 |
|
| 283 |
# Make sure running the tests with random seed 3 gets the same |
| 284 |
# order as in iteration 3 above. Success means that Google Test |
| 285 |
# correctly restores the test order before re-shuffling at the |
| 286 |
# beginning of iteration 3. |
| 287 |
[tests_with_seed3] = GetTestsForAllIterations( |
| 288 |
{}, [ShuffleFlag(), RandomSeedFlag(3)]) |
| 289 |
self.assertEqual(tests_in_iteration3, tests_with_seed3) |
| 290 |
|
| 291 |
def testShuffleGeneratesNewOrderInEachIteration(self): |
| 292 |
[tests_in_iteration1, tests_in_iteration2, tests_in_iteration3] = ( |
| 293 |
GetTestsForAllIterations( |
| 294 |
{}, [ShuffleFlag(), RandomSeedFlag(1), RepeatFlag(3)])) |
| 295 |
|
| 296 |
self.assert_(tests_in_iteration1 != tests_in_iteration2, |
| 297 |
tests_in_iteration1) |
| 298 |
self.assert_(tests_in_iteration1 != tests_in_iteration3, |
| 299 |
tests_in_iteration1) |
| 300 |
self.assert_(tests_in_iteration2 != tests_in_iteration3, |
| 301 |
tests_in_iteration2) |
| 302 |
|
| 303 |
def testShuffleShardedTestsPreservesPartition(self): |
| 304 |
# If we run M tests on N shards, the same M tests should be run in |
| 305 |
# total, regardless of the random seeds used by the shards. |
| 306 |
[tests1] = GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3', |
| 307 |
SHARD_INDEX_ENV_VAR: '0'}, |
| 308 |
[ShuffleFlag(), RandomSeedFlag(1)]) |
| 309 |
[tests2] = GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3', |
| 310 |
SHARD_INDEX_ENV_VAR: '1'}, |
| 311 |
[ShuffleFlag(), RandomSeedFlag(20)]) |
| 312 |
[tests3] = GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3', |
| 313 |
SHARD_INDEX_ENV_VAR: '2'}, |
| 314 |
[ShuffleFlag(), RandomSeedFlag(25)]) |
| 315 |
sorted_sharded_tests = tests1 + tests2 + tests3 |
| 316 |
sorted_sharded_tests.sort() |
| 317 |
sorted_active_tests = [] |
| 318 |
sorted_active_tests.extend(ACTIVE_TESTS) |
| 319 |
sorted_active_tests.sort() |
| 320 |
self.assertEqual(sorted_active_tests, sorted_sharded_tests) |
| 321 |
|
| 322 |
if __name__ == '__main__': |
| 323 |
gtest_test_utils.Main() |