Debian Codegraph @

llvm/utils/lit/tests/allow-retries.py

raw
1 # Check the behavior of the ALLOW_RETRIES keyword.
2
3 # This test uses a file that's stable across retries of the test to fail and
4 # only succeed the fourth time it is retried.
5 #
6 # RUN: rm -f %t.counter
7 # RUN: %{lit} %{inputs}/allow-retries/succeeds-within-limit.py -Dcounter=%t.counter -Dpython=%{python} | FileCheck --check-prefix=CHECK-TEST1 %s
8 # CHECK-TEST1: Passed With Retry: 1
9
10 # Test that a per-file ALLOW_RETRIES overwrites the config-wide test_retry_attempts property, if any.
11 #
12 # RUN: rm -f %t.counter
13 # RUN: %{lit} %{inputs}/allow-retries/succeeds-within-limit.py -Dtest_retry_attempts=2 -Dcounter=%t.counter -Dpython=%{python} | FileCheck --check-prefix=CHECK-TEST2 %s
14 # CHECK-TEST2: Passed With Retry: 1
15
16 # This test does not succeed within the allowed retry limit
17 #
18 # Check that the execution trace isn't corrupt due to reprocessing the script
19 # multiple times (e.g., '%dbg(...)' processing used to accumulate across
20 # retries).
21 #
22 # RUN: not %{lit} %{inputs}/allow-retries/does-not-succeed-within-limit.py -v |\
23 # RUN: FileCheck --check-prefix=CHECK-TEST3 -match-full-lines %s
24 #
25 # CHECK-TEST3: FAIL: allow-retries :: does-not-succeed-within-limit.py (1 of 1, 4 of 4 attempts)
26 # CHECK-TEST3-NEXT: {{\**}} TEST 'allow-retries :: does-not-succeed-within-limit.py' FAILED {{\**}}
27 # CHECK-TEST3-NEXT: Exit Code: 1
28 # CHECK-TEST3-EMPTY:
29 # CHECK-TEST3-NEXT: Command Output (stdout):
30 # CHECK-TEST3-NEXT: --
31 # CHECK-TEST3-NEXT: # {{RUN}}: at line 3
32 # CHECK-TEST3-NEXT: false
33 # CHECK-TEST3-NEXT: # executed command: false
34 # CHECK-TEST3-NEXT: # note: command had no output on stdout or stderr
35 # CHECK-TEST3-NEXT: # error: command failed with exit status: 1
36 # CHECK-TEST3-EMPTY:
37 # CHECK-TEST3-NEXT: --
38 # CHECK-TEST3: Failed Tests (1):
39 # CHECK-TEST3: allow-retries :: does-not-succeed-within-limit.py
40
41 # This test should be UNRESOLVED since it has more than one ALLOW_RETRIES
42 # lines, and that is not allowed.
43 #
44 # RUN: not %{lit} %{inputs}/allow-retries/more-than-one-allow-retries-lines.py | FileCheck --check-prefix=CHECK-TEST4 %s
45 # CHECK-TEST4: Unresolved Tests (1):
46 # CHECK-TEST4: allow-retries :: more-than-one-allow-retries-lines.py
47
48 # This test does not provide a valid integer to the ALLOW_RETRIES keyword.
49 # It should be unresolved.
50 #
51 # RUN: not %{lit} %{inputs}/allow-retries/not-a-valid-integer.py | FileCheck --check-prefix=CHECK-TEST5 %s
52 # CHECK-TEST5: Unresolved Tests (1):
53 # CHECK-TEST5: allow-retries :: not-a-valid-integer.py
54
55 # This test checks that the config-wide test_retry_attempts property is used
56 # when no ALLOW_RETRIES keyword is present.
57 #
58 # RUN: rm -f %t.counter
59 # RUN: %{lit} %{inputs}/test_retry_attempts/test.py -Dcounter=%t.counter -Dpython=%{python} | FileCheck --check-prefix=CHECK-TEST6 %s
60 # CHECK-TEST6: Passed With Retry: 1
61
62 # This test checks that --per-test-coverage doesn't accumulate inserted
63 # LLVM_PROFILE_FILE= commands across retries.
64 #
65 # RUN: rm -f %t.counter
66 # RUN: %{lit} -a %{inputs}/test_retry_attempts/test.py --per-test-coverage\
67 # RUN: -Dcounter=%t.counter -Dpython=%{python} | \
68 # RUN: FileCheck --check-prefix=CHECK-TEST7 %s
69 # CHECK-TEST7: Command Output (stdout):
70 # CHECK-TEST7: # executed command: export LLVM_PROFILE_FILE=
71 # CHECK-TEST7-NOT: # executed command: export LLVM_PROFILE_FILE=
72 # CHECK-TEST7: Passed With Retry: 1
73
74 # This test only passes on the 4th try. Here we check that a test can be re-run when:
75 # * The "--max-retries-per-test" is specified high enough (7).
76 # * No ALLOW_RETRIES keyword is used in the test script.
77 # * No config.test_retry_attempts is adjusted in the test suite config file.
78 # RUN: rm -f %t.counter
79 # RUN: %{lit} %{inputs}/max-retries-per-test/no-allow-retries-no-test_retry_attempts/test.py \
80 # RUN: --max-retries-per-test=7 \
81 # RUN: -Dcounter=%t.counter \
82 # RUN: -Dpython=%{python} \
83 # RUN: | FileCheck --check-prefix=CHECK-TEST8 %s
84 # CHECK-TEST8: FLAKYPASS: no-allow-retries-no-test_retry_attempts :: test.py (1 of 1, 4 of 8 attempts)
85 # CHECK-TEST8: Passed With Retry: 1
86
87 # This test only passes on the 4th try. Here we check that a test can be re-run when:
88 # * The "--max-retries-per-test" is specified too low (2).
89 # * ALLOW_RETRIES is specified high enough (8)
90 # * No config.test_retry_attempts is adjusted in the test suite config file.
91 # RUN: rm -f %t.counter
92 # RUN: %{lit} %{inputs}/max-retries-per-test/allow-retries-no-test_retry_attempts/test.py \
93 # RUN: --max-retries-per-test=2 \
94 # RUN: -Dcounter=%t.counter \
95 # RUN: -Dpython=%{python} \
96 # RUN: | FileCheck --check-prefix=CHECK-TEST9 %s
97 # CHECK-TEST9: FLAKYPASS: allow-retries-no-test_retry_attempts :: test.py (1 of 1, 4 of 9 attempts)
98 # CHECK-TEST9: Passed With Retry: 1
99
100 # This test only passes on the 4th try. Here we check that a test can be re-run when:
101 # * The "--max-retries-per-test" is specified too low (2).
102 # * No ALLOW_RETRIES keyword is used in the test script.
103 # * config.test_retry_attempts is set high enough (9).
104 # RUN: rm -f %t.counter
105 # RUN: %{lit} %{inputs}/max-retries-per-test/no-allow-retries-test_retry_attempts/test.py \
106 # RUN: --max-retries-per-test=2 \
107 # RUN: -Dcounter=%t.counter \
108 # RUN: -Dpython=%{python} \
109 # RUN: | FileCheck --check-prefix=CHECK-TEST10 %s
110 # CHECK-TEST10: FLAKYPASS: no-allow-retries-test_retry_attempts :: test.py (1 of 1, 4 of 10 attempts)
111 # CHECK-TEST10: Passed With Retry: 1
112
113 # This test only passes on the 4th try. Here we check that a test can be re-run when:
114 # * The "--max-retries-per-test" is specified too low (1).
115 # * ALLOW_RETRIES keyword is set high enough (10)
116 # * config.test_retry_attempts is set too low (2).
117 # RUN: rm -f %t.counter
118 # RUN: %{lit} %{inputs}/max-retries-per-test/allow-retries-test_retry_attempts/test.py \
119 # RUN: --max-retries-per-test=1 \
120 # RUN: -Dcounter=%t.counter \
121 # RUN: -Dpython=%{python} \
122 # RUN: | FileCheck --check-prefix=CHECK-TEST11 %s
123 # CHECK-TEST11: FLAKYPASS: allow-retries-test_retry_attempts :: test.py (1 of 1, 4 of 11 attempts)
124 # CHECK-TEST11: Passed With Retry: 1