| 1 |
# Check the basic discovery process, including a sub-suite. |
| 2 |
# |
| 3 |
# RUN: %{lit} %{inputs}/discovery \ |
| 4 |
# RUN: --debug --show-tests --show-suites \ |
| 5 |
# RUN: -v > %t.out 2> %t.err |
| 6 |
# RUN: FileCheck --check-prefix=CHECK-BASIC-OUT < %t.out %s |
| 7 |
# RUN: FileCheck --check-prefix=CHECK-BASIC-ERR < %t.err %s |
| 8 |
# |
| 9 |
# CHECK-BASIC-ERR: loading suite config '{{.*(/|\\\\)discovery(/|\\\\)lit.cfg}}' |
| 10 |
# CHECK-BASIC-ERR-DAG: loading suite config '{{.*(/|\\\\)discovery(/|\\\\)subsuite(/|\\\\)lit.cfg}}' |
| 11 |
# CHECK-BASIC-ERR-DAG: loading local config '{{.*(/|\\\\)discovery(/|\\\\)subdir(/|\\\\)lit.local.cfg}}' |
| 12 |
# |
| 13 |
# CHECK-BASIC-OUT: -- Test Suites -- |
| 14 |
# CHECK-BASIC-OUT: sub-suite - 2 tests |
| 15 |
# CHECK-BASIC-OUT: Source Root: {{.*[/\\]discovery[/\\]subsuite$}} |
| 16 |
# CHECK-BASIC-OUT: Exec Root : {{.*[/\\]discovery[/\\]subsuite$}} |
| 17 |
# CHECK-BASIC-OUT: top-level-suite - 3 tests |
| 18 |
# CHECK-BASIC-OUT: Source Root: {{.*[/\\]discovery$}} |
| 19 |
# CHECK-BASIC-OUT: Exec Root : {{.*[/\\]discovery$}} |
| 20 |
# CHECK-BASIC-OUT: Available Features: feature1 feature2 |
| 21 |
# CHECK-BASIC-OUT: Available Substitutions: %key1 => value1 |
| 22 |
# CHECK-BASIC-OUT: %key2 => value2 |
| 23 |
# |
| 24 |
# CHECK-BASIC-OUT: -- Available Tests -- |
| 25 |
# CHECK-BASIC-OUT: sub-suite :: test-one |
| 26 |
# CHECK-BASIC-OUT: sub-suite :: test-two |
| 27 |
# CHECK-BASIC-OUT: top-level-suite :: subdir/test-three |
| 28 |
# CHECK-BASIC-OUT: top-level-suite :: test-one |
| 29 |
# CHECK-BASIC-OUT: top-level-suite :: test-two |
| 30 |
|
| 31 |
# RUN: %{lit} %{inputs}/discovery \ |
| 32 |
# RUN: -v > %t.out 2> %t.err |
| 33 |
# RUN: FileCheck --check-prefix=CHECK-PERCENTAGES-OUT < %/t.out %s |
| 34 |
# |
| 35 |
# CHECK-PERCENTAGES-OUT: Total Discovered Tests: {{[0-9]*}} |
| 36 |
# CHECK-PERCENTAGES-OUT: Passed: {{[0-9]*}} {{\([0-9]*\.[0-9]*%\)}} |
| 37 |
|
| 38 |
# Check discovery when providing the special builtin 'config_map' |
| 39 |
# RUN: %{python} %{inputs}/config-map-discovery/driver.py \ |
| 40 |
# RUN: %{inputs}/config-map-discovery/main-config/lit.cfg \ |
| 41 |
# RUN: %{inputs}/config-map-discovery/lit.alt.cfg \ |
| 42 |
# RUN: --workers=1 --debug --show-tests --show-suites > %t.out 2> %t.err |
| 43 |
# RUN: FileCheck --check-prefix=CHECK-CONFIG-MAP-OUT < %t.out %s |
| 44 |
# RUN: FileCheck --check-prefix=CHECK-CONFIG-MAP-ERR < %t.err %s |
| 45 |
|
| 46 |
# CHECK-CONFIG-MAP-OUT-NOT: ERROR: lit.cfg invoked |
| 47 |
# CHECK-CONFIG-MAP-OUT: -- Test Suites -- |
| 48 |
# CHECK-CONFIG-MAP-OUT: config-map - 2 tests |
| 49 |
# CHECK-CONFIG-MAP-OUT: Source Root: {{.*[/\\]config-map-discovery[/\\]tests}} |
| 50 |
# CHECK-CONFIG-MAP-OUT: Exec Root : {{.*[/\\]tests[/\\]Inputs[/\\]config-map-discovery}} |
| 51 |
# CHECK-CONFIG-MAP-OUT: -- Available Tests -- |
| 52 |
# CHECK-CONFIG-MAP-OUT-NOT: invalid-test.txt |
| 53 |
# CHECK-CONFIG-MAP-OUT: config-map :: test1.txt |
| 54 |
# CHECK-CONFIG-MAP-OUT: config-map :: test2.txt |
| 55 |
|
| 56 |
# CHECK-CONFIG-MAP-ERR: loading suite config '{{.*}}lit.alt.cfg' |
| 57 |
# CHECK-CONFIG-MAP-ERR: loaded config '{{.*}}lit.alt.cfg' |
| 58 |
# CHECK-CONFIG-MAP-ERR: resolved input '{{.*(/|\\\\)config-map-discovery(/|\\\\)main-config}}' to 'config-map'::() |
| 59 |
|
| 60 |
|
| 61 |
# Check discovery when tests are named directly. |
| 62 |
# |
| 63 |
# RUN: %{lit} \ |
| 64 |
# RUN: %{inputs}/discovery/subdir/test-three.py \ |
| 65 |
# RUN: %{inputs}/discovery/subsuite/test-one.txt \ |
| 66 |
# RUN: --show-tests --show-suites -v > %t.out |
| 67 |
# RUN: FileCheck --check-prefix=CHECK-DIRECT-TEST < %t.out %s |
| 68 |
# |
| 69 |
# CHECK-DIRECT-TEST: -- Available Tests -- |
| 70 |
# CHECK-DIRECT-TEST: sub-suite :: test-one |
| 71 |
# CHECK-DIRECT-TEST: top-level-suite :: subdir/test-three |
| 72 |
|
| 73 |
# Check discovery when config files end in .py |
| 74 |
# RUN: %{lit} %{inputs}/py-config-discovery \ |
| 75 |
# RUN: --debug --show-tests --show-suites \ |
| 76 |
# RUN: -v > %t.out 2> %t.err |
| 77 |
# RUN: FileCheck --check-prefix=CHECK-PYCONFIG-OUT < %t.out %s |
| 78 |
# RUN: FileCheck --check-prefix=CHECK-PYCONFIG-ERR < %t.err %s |
| 79 |
# |
| 80 |
# CHECK-PYCONFIG-ERR: loading suite config '{{.*(/|\\\\)py-config-discovery(/|\\\\)lit.site.cfg.py}}' |
| 81 |
# CHECK-PYCONFIG-ERR: load_config from '{{.*(/|\\\\)discovery(/|\\\\)lit.cfg}}' |
| 82 |
# CHECK-PYCONFIG-ERR: loaded config '{{.*(/|\\\\)discovery(/|\\\\)lit.cfg}}' |
| 83 |
# CHECK-PYCONFIG-ERR: loaded config '{{.*(/|\\\\)py-config-discovery(/|\\\\)lit.site.cfg.py}}' |
| 84 |
# CHECK-PYCONFIG-ERR-DAG: loading suite config '{{.*(/|\\\\)discovery(/|\\\\)subsuite(/|\\\\)lit.cfg}}' |
| 85 |
# CHECK-PYCONFIG-ERR-DAG: loading local config '{{.*(/|\\\\)discovery(/|\\\\)subdir(/|\\\\)lit.local.cfg}}' |
| 86 |
# |
| 87 |
# CHECK-PYCONFIG-OUT: -- Test Suites -- |
| 88 |
# CHECK-PYCONFIG-OUT: sub-suite - 2 tests |
| 89 |
# CHECK-PYCONFIG-OUT: Source Root: {{.*[/\\]discovery[/\\]subsuite$}} |
| 90 |
# CHECK-PYCONFIG-OUT: Exec Root : {{.*[/\\]discovery[/\\]subsuite$}} |
| 91 |
# CHECK-PYCONFIG-OUT: top-level-suite - 3 tests |
| 92 |
# CHECK-PYCONFIG-OUT: Source Root: {{.*[/\\]discovery$}} |
| 93 |
# CHECK-PYCONFIG-OUT: Exec Root : {{.*[/\\]py-config-discovery$}} |
| 94 |
# |
| 95 |
# CHECK-PYCONFIG-OUT: -- Available Tests -- |
| 96 |
# CHECK-PYCONFIG-OUT: sub-suite :: test-one |
| 97 |
# CHECK-PYCONFIG-OUT: sub-suite :: test-two |
| 98 |
# CHECK-PYCONFIG-OUT: top-level-suite :: subdir/test-three |
| 99 |
# CHECK-PYCONFIG-OUT: top-level-suite :: test-one |
| 100 |
# CHECK-PYCONFIG-OUT: top-level-suite :: test-two |
| 101 |
|
| 102 |
# Check discovery when using an exec path. |
| 103 |
# |
| 104 |
# RUN: %{lit} %{inputs}/exec-discovery \ |
| 105 |
# RUN: --debug --show-tests --show-suites \ |
| 106 |
# RUN: -v > %t.out 2> %t.err |
| 107 |
# RUN: FileCheck --check-prefix=CHECK-ASEXEC-OUT < %t.out %s |
| 108 |
# RUN: FileCheck --check-prefix=CHECK-ASEXEC-ERR < %t.err %s |
| 109 |
# |
| 110 |
# CHECK-ASEXEC-ERR: loading suite config '{{.*(/|\\\\)exec-discovery(/|\\\\)lit.site.cfg}}' |
| 111 |
# CHECK-ASEXEC-ERR: load_config from '{{.*(/|\\\\)discovery(/|\\\\)lit.cfg}}' |
| 112 |
# CHECK-ASEXEC-ERR: loaded config '{{.*(/|\\\\)discovery(/|\\\\)lit.cfg}}' |
| 113 |
# CHECK-ASEXEC-ERR: loaded config '{{.*(/|\\\\)exec-discovery(/|\\\\)lit.site.cfg}}' |
| 114 |
# CHECK-ASEXEC-ERR-DAG: loading suite config '{{.*(/|\\\\)discovery(/|\\\\)subsuite(/|\\\\)lit.cfg}}' |
| 115 |
# CHECK-ASEXEC-ERR-DAG: loading local config '{{.*(/|\\\\)discovery(/|\\\\)subdir(/|\\\\)lit.local.cfg}}' |
| 116 |
# |
| 117 |
# CHECK-ASEXEC-OUT: -- Test Suites -- |
| 118 |
# CHECK-ASEXEC-OUT: sub-suite - 2 tests |
| 119 |
# CHECK-ASEXEC-OUT: Source Root: {{.*[/\\]discovery[/\\]subsuite$}} |
| 120 |
# CHECK-ASEXEC-OUT: Exec Root : {{.*[/\\]discovery[/\\]subsuite$}} |
| 121 |
# CHECK-ASEXEC-OUT: top-level-suite - 3 tests |
| 122 |
# CHECK-ASEXEC-OUT: Source Root: {{.*[/\\]discovery$}} |
| 123 |
# CHECK-ASEXEC-OUT: Exec Root : {{.*[/\\]exec-discovery$}} |
| 124 |
# |
| 125 |
# CHECK-ASEXEC-OUT: -- Available Tests -- |
| 126 |
# CHECK-ASEXEC-OUT: sub-suite :: test-one |
| 127 |
# CHECK-ASEXEC-OUT: sub-suite :: test-two |
| 128 |
# CHECK-ASEXEC-OUT: top-level-suite :: subdir/test-three |
| 129 |
# CHECK-ASEXEC-OUT: top-level-suite :: test-one |
| 130 |
# CHECK-ASEXEC-OUT: top-level-suite :: test-two |
| 131 |
|
| 132 |
# Check discovery when tests are named directly. |
| 133 |
# |
| 134 |
# FIXME: Note that using a path into a subsuite doesn't work correctly here. |
| 135 |
# |
| 136 |
# RUN: %{lit} \ |
| 137 |
# RUN: %{inputs}/exec-discovery/subdir/test-three.py \ |
| 138 |
# RUN: --show-tests --show-suites -v > %t.out |
| 139 |
# RUN: FileCheck --check-prefix=CHECK-ASEXEC-DIRECT-TEST < %t.out %s |
| 140 |
# |
| 141 |
# CHECK-ASEXEC-DIRECT-TEST: -- Available Tests -- |
| 142 |
# CHECK-ASEXEC-DIRECT-TEST: top-level-suite :: subdir/test-three |
| 143 |
|
| 144 |
# Check that an error is emitted when the directly named test does not satisfy |
| 145 |
# the test config's requirements. |
| 146 |
# |
| 147 |
# RUN: not %{lit} \ |
| 148 |
# RUN: %{inputs}/discovery/test.not-txt 2>%t.err |
| 149 |
# RUN: FileCheck --check-prefix=CHECK-ERROR-INPUT-CONTAINED-NO-TESTS < %t.err %s |
| 150 |
# |
| 151 |
# CHECK-ERROR-INPUT-CONTAINED-NO-TESTS: warning: input 'Inputs/discovery/test.not-txt' contained no tests |
| 152 |
# CHECK-ERROR-INPUT-CONTAINED-NO-TESTS: error: did not discover any tests for provided path(s) |
| 153 |
|
| 154 |
# Check that a standalone test with no suffixes set is run without any errors. |
| 155 |
# |
| 156 |
# RUN: %{lit} %{inputs}/standalone-tests/true.txt > %t.out |
| 157 |
# RUN: FileCheck --check-prefix=CHECK-STANDALONE < %t.out %s |
| 158 |
# |
| 159 |
# CHECK-STANDALONE: PASS: Standalone tests :: true.txt |
| 160 |
|
| 161 |
# Check that an error is produced if suffixes variable is set for a suite with |
| 162 |
# standalone tests. |
| 163 |
# |
| 164 |
# RUN: not %{lit} %{inputs}/standalone-tests-with-suffixes 2> %t.err |
| 165 |
# RUN: FileCheck --check-prefixes=CHECK-STANDALONE-SUFFIXES,CHECK-STANDALONE-DISCOVERY < %t.err %s |
| 166 |
# |
| 167 |
# CHECK-STANDALONE-SUFFIXES: standalone_tests set {{.*}} but suffixes |
| 168 |
|
| 169 |
# Check that an error is produced if excludes variable is set for a suite with |
| 170 |
# standalone tests. |
| 171 |
# |
| 172 |
# RUN: not %{lit} %{inputs}/standalone-tests-with-excludes 2> %t.err |
| 173 |
# RUN: FileCheck --check-prefixes=CHECK-STANDALONE-EXCLUDES,CHECK-STANDALONE-DISCOVERY < %t.err %s |
| 174 |
# |
| 175 |
# CHECK-STANDALONE-EXCLUDES: standalone_tests set {{.*}} but {{.*}} excludes |
| 176 |
|
| 177 |
# Check that no discovery is done for testsuite with standalone tests. |
| 178 |
# |
| 179 |
# RUN: not %{lit} %{inputs}/standalone-tests 2>%t.err |
| 180 |
# RUN: FileCheck --check-prefix=CHECK-STANDALONE-DISCOVERY < %t.err %s |
| 181 |
# |
| 182 |
# CHECK-STANDALONE-DISCOVERY: error: did not discover any tests for provided path(s) |
| 183 |
|
| 184 |
# Check that a single file path can result in multiple tests being discovered if |
| 185 |
# the test format implements those semantics. |
| 186 |
# |
| 187 |
# RUN: %{lit} %{inputs}/discovery-getTestsForPath/x.test > %t.out |
| 188 |
# RUN: FileCheck --check-prefix=CHECK-getTestsForPath < %t.out %s |
| 189 |
# |
| 190 |
# CHECK-getTestsForPath: PASS: discovery-getTestsForPath-suite :: {{.+}}one.test |
| 191 |
# CHECK-getTestsForPath: PASS: discovery-getTestsForPath-suite :: {{.+}}two.test |
| 192 |
|
| 193 |
# Check that we don't recurse infinitely when loading an site specific test |
| 194 |
# suite located inside the test source root. |
| 195 |
# |
| 196 |
# RUN: %{lit} \ |
| 197 |
# RUN: %{inputs}/exec-discovery-in-tree/obj/ \ |
| 198 |
# RUN: --show-tests --show-suites -v > %t.out |
| 199 |
# RUN: FileCheck --check-prefix=CHECK-ASEXEC-INTREE < %t.out %s |
| 200 |
# |
| 201 |
# Try it again after cd'ing into the test suite using a short relative path. |
| 202 |
# |
| 203 |
# RUN: cd %{inputs}/exec-discovery-in-tree/obj/ |
| 204 |
# RUN: %{lit} . \ |
| 205 |
# RUN: --show-tests --show-suites -v > %t.out |
| 206 |
# RUN: FileCheck --check-prefix=CHECK-ASEXEC-INTREE < %t.out %s |
| 207 |
# |
| 208 |
# CHECK-ASEXEC-INTREE: exec-discovery-in-tree-suite - 1 tests |
| 209 |
# CHECK-ASEXEC-INTREE-NEXT: Source Root: {{.*[/\\]exec-discovery-in-tree$}} |
| 210 |
# CHECK-ASEXEC-INTREE-NEXT: Exec Root : {{.*[/\\]exec-discovery-in-tree[/\\]obj$}} |
| 211 |
# CHECK-ASEXEC-INTREE-NEXT: Available Features: |
| 212 |
# CHECK-ASEXEC-INTREE-NEXT: Available Substitutions: |
| 213 |
# CHECK-ASEXEC-INTREE-NEXT: -- Available Tests -- |
| 214 |
# CHECK-ASEXEC-INTREE-NEXT: exec-discovery-in-tree-suite :: test-one |