| 1 |
# Check the behavior of the ALLOW_RETRIES keyword. |
| 2 |
|
| 3 |
# This test uses a file that's stable across retries of the test to fail and |
| 4 |
# only succeed the fourth time it is retried. |
| 5 |
# |
| 6 |
# RUN: rm -f %t.counter |
| 7 |
# RUN: %{lit} %{inputs}/allow-retries/succeeds-within-limit.py -Dcounter=%t.counter -Dpython=%{python} | FileCheck --check-prefix=CHECK-TEST1 %s |
| 8 |
# CHECK-TEST1: Passed With Retry: 1 |
| 9 |
|
| 10 |
# Test that a per-file ALLOW_RETRIES overwrites the config-wide test_retry_attempts property, if any. |
| 11 |
# |
| 12 |
# RUN: rm -f %t.counter |
| 13 |
# RUN: %{lit} %{inputs}/allow-retries/succeeds-within-limit.py -Dtest_retry_attempts=2 -Dcounter=%t.counter -Dpython=%{python} | FileCheck --check-prefix=CHECK-TEST2 %s |
| 14 |
# CHECK-TEST2: Passed With Retry: 1 |
| 15 |
|
| 16 |
# This test does not succeed within the allowed retry limit |
| 17 |
# |
| 18 |
# Check that the execution trace isn't corrupt due to reprocessing the script |
| 19 |
# multiple times (e.g., '%dbg(...)' processing used to accumulate across |
| 20 |
# retries). |
| 21 |
# |
| 22 |
# RUN: not %{lit} %{inputs}/allow-retries/does-not-succeed-within-limit.py -v |\ |
| 23 |
# RUN: FileCheck --check-prefix=CHECK-TEST3 -match-full-lines %s |
| 24 |
# |
| 25 |
# CHECK-TEST3: FAIL: allow-retries :: does-not-succeed-within-limit.py (1 of 1, 4 of 4 attempts) |
| 26 |
# CHECK-TEST3-NEXT: {{\**}} TEST 'allow-retries :: does-not-succeed-within-limit.py' FAILED {{\**}} |
| 27 |
# CHECK-TEST3-NEXT: Exit Code: 1 |
| 28 |
# CHECK-TEST3-EMPTY: |
| 29 |
# CHECK-TEST3-NEXT: Command Output (stdout): |
| 30 |
# CHECK-TEST3-NEXT: -- |
| 31 |
# CHECK-TEST3-NEXT: # {{RUN}}: at line 3 |
| 32 |
# CHECK-TEST3-NEXT: false |
| 33 |
# CHECK-TEST3-NEXT: # executed command: false |
| 34 |
# CHECK-TEST3-NEXT: # note: command had no output on stdout or stderr |
| 35 |
# CHECK-TEST3-NEXT: # error: command failed with exit status: 1 |
| 36 |
# CHECK-TEST3-EMPTY: |
| 37 |
# CHECK-TEST3-NEXT: -- |
| 38 |
# CHECK-TEST3: Failed Tests (1): |
| 39 |
# CHECK-TEST3: allow-retries :: does-not-succeed-within-limit.py |
| 40 |
|
| 41 |
# This test should be UNRESOLVED since it has more than one ALLOW_RETRIES |
| 42 |
# lines, and that is not allowed. |
| 43 |
# |
| 44 |
# RUN: not %{lit} %{inputs}/allow-retries/more-than-one-allow-retries-lines.py | FileCheck --check-prefix=CHECK-TEST4 %s |
| 45 |
# CHECK-TEST4: Unresolved Tests (1): |
| 46 |
# CHECK-TEST4: allow-retries :: more-than-one-allow-retries-lines.py |
| 47 |
|
| 48 |
# This test does not provide a valid integer to the ALLOW_RETRIES keyword. |
| 49 |
# It should be unresolved. |
| 50 |
# |
| 51 |
# RUN: not %{lit} %{inputs}/allow-retries/not-a-valid-integer.py | FileCheck --check-prefix=CHECK-TEST5 %s |
| 52 |
# CHECK-TEST5: Unresolved Tests (1): |
| 53 |
# CHECK-TEST5: allow-retries :: not-a-valid-integer.py |
| 54 |
|
| 55 |
# This test checks that the config-wide test_retry_attempts property is used |
| 56 |
# when no ALLOW_RETRIES keyword is present. |
| 57 |
# |
| 58 |
# RUN: rm -f %t.counter |
| 59 |
# RUN: %{lit} %{inputs}/test_retry_attempts/test.py -Dcounter=%t.counter -Dpython=%{python} | FileCheck --check-prefix=CHECK-TEST6 %s |
| 60 |
# CHECK-TEST6: Passed With Retry: 1 |
| 61 |
|
| 62 |
# This test checks that --per-test-coverage doesn't accumulate inserted |
| 63 |
# LLVM_PROFILE_FILE= commands across retries. |
| 64 |
# |
| 65 |
# RUN: rm -f %t.counter |
| 66 |
# RUN: %{lit} -a %{inputs}/test_retry_attempts/test.py --per-test-coverage\ |
| 67 |
# RUN: -Dcounter=%t.counter -Dpython=%{python} | \ |
| 68 |
# RUN: FileCheck --check-prefix=CHECK-TEST7 %s |
| 69 |
# CHECK-TEST7: Command Output (stdout): |
| 70 |
# CHECK-TEST7: # executed command: export LLVM_PROFILE_FILE= |
| 71 |
# CHECK-TEST7-NOT: # executed command: export LLVM_PROFILE_FILE= |
| 72 |
# CHECK-TEST7: Passed With Retry: 1 |
| 73 |
|
| 74 |
# This test only passes on the 4th try. Here we check that a test can be re-run when: |
| 75 |
# * The "--max-retries-per-test" is specified high enough (7). |
| 76 |
# * No ALLOW_RETRIES keyword is used in the test script. |
| 77 |
# * No config.test_retry_attempts is adjusted in the test suite config file. |
| 78 |
# RUN: rm -f %t.counter |
| 79 |
# RUN: %{lit} %{inputs}/max-retries-per-test/no-allow-retries-no-test_retry_attempts/test.py \ |
| 80 |
# RUN: --max-retries-per-test=7 \ |
| 81 |
# RUN: -Dcounter=%t.counter \ |
| 82 |
# RUN: -Dpython=%{python} \ |
| 83 |
# RUN: | FileCheck --check-prefix=CHECK-TEST8 %s |
| 84 |
# CHECK-TEST8: FLAKYPASS: no-allow-retries-no-test_retry_attempts :: test.py (1 of 1, 4 of 8 attempts) |
| 85 |
# CHECK-TEST8: Passed With Retry: 1 |
| 86 |
|
| 87 |
# This test only passes on the 4th try. Here we check that a test can be re-run when: |
| 88 |
# * The "--max-retries-per-test" is specified too low (2). |
| 89 |
# * ALLOW_RETRIES is specified high enough (8) |
| 90 |
# * No config.test_retry_attempts is adjusted in the test suite config file. |
| 91 |
# RUN: rm -f %t.counter |
| 92 |
# RUN: %{lit} %{inputs}/max-retries-per-test/allow-retries-no-test_retry_attempts/test.py \ |
| 93 |
# RUN: --max-retries-per-test=2 \ |
| 94 |
# RUN: -Dcounter=%t.counter \ |
| 95 |
# RUN: -Dpython=%{python} \ |
| 96 |
# RUN: | FileCheck --check-prefix=CHECK-TEST9 %s |
| 97 |
# CHECK-TEST9: FLAKYPASS: allow-retries-no-test_retry_attempts :: test.py (1 of 1, 4 of 9 attempts) |
| 98 |
# CHECK-TEST9: Passed With Retry: 1 |
| 99 |
|
| 100 |
# This test only passes on the 4th try. Here we check that a test can be re-run when: |
| 101 |
# * The "--max-retries-per-test" is specified too low (2). |
| 102 |
# * No ALLOW_RETRIES keyword is used in the test script. |
| 103 |
# * config.test_retry_attempts is set high enough (9). |
| 104 |
# RUN: rm -f %t.counter |
| 105 |
# RUN: %{lit} %{inputs}/max-retries-per-test/no-allow-retries-test_retry_attempts/test.py \ |
| 106 |
# RUN: --max-retries-per-test=2 \ |
| 107 |
# RUN: -Dcounter=%t.counter \ |
| 108 |
# RUN: -Dpython=%{python} \ |
| 109 |
# RUN: | FileCheck --check-prefix=CHECK-TEST10 %s |
| 110 |
# CHECK-TEST10: FLAKYPASS: no-allow-retries-test_retry_attempts :: test.py (1 of 1, 4 of 10 attempts) |
| 111 |
# CHECK-TEST10: Passed With Retry: 1 |
| 112 |
|
| 113 |
# This test only passes on the 4th try. Here we check that a test can be re-run when: |
| 114 |
# * The "--max-retries-per-test" is specified too low (1). |
| 115 |
# * ALLOW_RETRIES keyword is set high enough (10) |
| 116 |
# * config.test_retry_attempts is set too low (2). |
| 117 |
# RUN: rm -f %t.counter |
| 118 |
# RUN: %{lit} %{inputs}/max-retries-per-test/allow-retries-test_retry_attempts/test.py \ |
| 119 |
# RUN: --max-retries-per-test=1 \ |
| 120 |
# RUN: -Dcounter=%t.counter \ |
| 121 |
# RUN: -Dpython=%{python} \ |
| 122 |
# RUN: | FileCheck --check-prefix=CHECK-TEST11 %s |
| 123 |
# CHECK-TEST11: FLAKYPASS: allow-retries-test_retry_attempts :: test.py (1 of 1, 4 of 11 attempts) |
| 124 |
# CHECK-TEST11: Passed With Retry: 1 |