| 1 |
# @HEADER |
| 2 |
# ***************************************************************************** |
| 3 |
# TriBITS: Tribal Build, Integrate, and Test System |
| 4 |
# |
| 5 |
# Copyright 2013-2016 NTESS and the TriBITS contributors. |
| 6 |
# SPDX-License-Identifier: BSD-3-Clause |
| 7 |
# ***************************************************************************** |
| 8 |
# @HEADER |
| 9 |
|
| 10 |
|
| 11 |
include("${CMAKE_CURRENT_LIST_DIR}/../common/TribitsCMakePolicies.cmake" NO_POLICY_SCOPE) |
| 12 |
include("${CMAKE_CURRENT_LIST_DIR}/TribitsAddTestHelpers.cmake") |
| 13 |
|
| 14 |
|
| 15 |
# @FUNCTION: tribits_add_test() |
| 16 |
# |
| 17 |
# Add a test or a set of tests for a single executable or command using CTest |
| 18 |
# ``add_test()``. |
| 19 |
# |
| 20 |
# Usage:: |
| 21 |
# |
| 22 |
# tribits_add_test( |
| 23 |
# <exeRootName> [NOEXEPREFIX] [NOEXESUFFIX] |
| 24 |
# [NAME <testName> | NAME_POSTFIX <testNamePostfix>] |
| 25 |
# [DIRECTORY <directory>] |
| 26 |
# [ADD_DIR_TO_NAME] |
| 27 |
# [RUN_SERIAL] |
| 28 |
# [ARGS "<arg0> <arg1> ..." "<arg2> <arg3> ..." ... |
| 29 |
# | POSTFIX_AND_ARGS_0 <postfix0> <arg0> <arg1> ... |
| 30 |
# POSTFIX_AND_ARGS_1 ... ] |
| 31 |
# [COMM [serial] [mpi]] |
| 32 |
# [NUM_MPI_PROCS <numMpiProcs>] |
| 33 |
# [NUM_TOTAL_CORES_USED <numTotalCoresUsed>] |
| 34 |
# [CATEGORIES <category0> <category1> ...] |
| 35 |
# [HOST <host0> <host1> ...] |
| 36 |
# [XHOST <host0> <host1> ...] |
| 37 |
# [HOSTTYPE <hosttype0> <hosttype1> ...] |
| 38 |
# [XHOSTTYPE <hosttype0> <hosttype1> ...] |
| 39 |
# [EXCLUDE_IF_NOT_TRUE <varname0> <varname1> ...] |
| 40 |
# [DISABLED <messageWhyDisabled>] |
| 41 |
# [STANDARD_PASS_OUTPUT |
| 42 |
# | PASS_REGULAR_EXPRESSION "<regex0>" "<regex1>" ...] |
| 43 |
# [FAIL_REGULAR_EXPRESSION "<regex0>" "<regex1>" ...] |
| 44 |
# [WILL_FAIL] |
| 45 |
# [ENVIRONMENT <var0>=<value0> <var1>=<value1> ...] |
| 46 |
# [TIMEOUT <maxSeconds>] |
| 47 |
# [LIST_SEPARATOR <sep>] |
| 48 |
# [ADDED_TESTS_NAMES_OUT <testsNames>] |
| 49 |
# ) |
| 50 |
# |
| 51 |
# The tests are only added if tests are enabled for the package |
| 52 |
# (i.e. `${PACKAGE_NAME}_ENABLE_TESTS`_ ``= ON``). (NOTE: A more efficient |
| 53 |
# way to optionally enable tests or examples is to put them in a ``test/`` or |
| 54 |
# ``example/`` subdir and then include that subdir with |
| 55 |
# `tribits_add_test_directories()`_ or `tribits_add_example_directories()`_, |
| 56 |
# respectively.) |
| 57 |
# |
| 58 |
# *Sections:* |
| 59 |
# |
| 60 |
# * `Formal Arguments (tribits_add_test())`_ |
| 61 |
# * `Determining the Executable or Command to Run (tribits_add_test())`_ |
| 62 |
# * `Determining the Full Test Name (tribits_add_test())`_ |
| 63 |
# * `Adding Multiple Tests (tribits_add_test())`_ |
| 64 |
# * `Determining Pass/Fail (tribits_add_test())`_ |
| 65 |
# * `Setting additional test properties (tribits_add_test())`_ |
| 66 |
# * `Running multiple tests at the same time (tribits_add_test())`_ |
| 67 |
# * `Setting timeouts for tests (tribits_add_test())`_ |
| 68 |
# * `Debugging and Examining Test Generation (tribits_add_test())`_ |
| 69 |
# * `Disabling Tests Externally (tribits_add_test())`_ |
| 70 |
# * `Adding extra commandline arguments externally (tribits_add_test())`_ |
| 71 |
# |
| 72 |
# .. _Formal Arguments (tribits_add_test()): |
| 73 |
# |
| 74 |
# **Formal Arguments (tribits_add_test())** |
| 75 |
# |
| 76 |
# ``<exeRootName>`` |
| 77 |
# |
| 78 |
# The name of the executable or path to the executable to run for the test |
| 79 |
# (see `Determining the Executable or Command to Run |
| 80 |
# (tribits_add_test())`_). This name is also the default root name for |
| 81 |
# the test (see `Determining the Full Test Name (tribits_add_test())`_). |
| 82 |
# |
| 83 |
# ``NOEXEPREFIX`` |
| 84 |
# |
| 85 |
# If specified, then the prefix ``${PACKAGE_NAME}_`` is assumed **not** to |
| 86 |
# be prepended to ``<exeRootName>`` (see `Determining the Executable or |
| 87 |
# Command to Run (tribits_add_test())`_). |
| 88 |
# |
| 89 |
# ``NOEXESUFFIX`` |
| 90 |
# |
| 91 |
# If specified, then the postfix |
| 92 |
# ``${${PROJECT_NAME}_CMAKE_EXECUTABLE_SUFFIX}`` is assumed **not** to be |
| 93 |
# post-pended to ``<exeRootName>`` (except on Windows platforms, see |
| 94 |
# `Determining the Executable or Command to Run (tribits_add_test())`_). |
| 95 |
# |
| 96 |
# ``NAME <testRootName>`` |
| 97 |
# |
| 98 |
# If specified, gives the root name of the test. If not specified, then |
| 99 |
# ``<testRootName>`` is taken to be ``<exeRootName>``. The actual test |
| 100 |
# name passed to ``add_test()`` will always be prefixed as |
| 101 |
# ``${PACKAGE_NAME}_<testRootName>``. The main purpose of this argument |
| 102 |
# is to allow multiple tests to be defined for the same executable. CTest |
| 103 |
# requires all test names to be globally unique in a single project. See |
| 104 |
# `Determining the Full Test Name (tribits_add_test())`_. |
| 105 |
# |
| 106 |
# ``NAME_POSTFIX <testNamePostfix>`` |
| 107 |
# |
| 108 |
# If specified, gives a postfix that will be added to the standard test |
| 109 |
# name based on ``<exeRootName>`` (appended as ``_<NAME_POSTFIX>``). If |
| 110 |
# the ``NAME <testRootName>`` argument is given, this argument is ignored. |
| 111 |
# See `Determining the Full Test Name (tribits_add_test())`_. |
| 112 |
# |
| 113 |
# ``DIRECTORY <dir>`` |
| 114 |
# |
| 115 |
# If specified, then the executable is assumed to be in the directory |
| 116 |
# given by ``<dir>``. The directory ``<dir>`` can either be a relative or |
| 117 |
# absolute path. If not specified, the executable is assumed to be in the |
| 118 |
# current binary directory ``${CMAKE_CURRENT_BINARY_DIR}``. See |
| 119 |
# `Determining the Executable or Command to Run (tribits_add_test())`_. |
| 120 |
# |
| 121 |
# ``ADD_DIR_TO_NAME`` |
| 122 |
# |
| 123 |
# If specified, then the directory name that this test resides in will be |
| 124 |
# added into the name of the test after the package name is added and |
| 125 |
# before the root test name (see `Determining the Full Test Name |
| 126 |
# (tribits_add_test())`_). The directory name will have the package's |
| 127 |
# base directory stripped off so only the unique part of the test |
| 128 |
# directory will be used. All directory separators ``"/"`` will be |
| 129 |
# changed into underscores ``"_"``. |
| 130 |
# |
| 131 |
# ``RUN_SERIAL`` |
| 132 |
# |
| 133 |
# If specified, then no other tests will be allowed to run while this test |
| 134 |
# is running. This is useful for devices (like CUDA GPUs) that require |
| 135 |
# exclusive access for processes/threads. This just sets the CTest test |
| 136 |
# property ``RUN_SERIAL`` using the built-in CMake function |
| 137 |
# ``set_tests_properties()``. Also, the addition of the ``RUN_SERIAL`` |
| 138 |
# test property can be triggered by (the user) setting the global cache |
| 139 |
# variable ``<fullTestName>_SET_RUN_SERIAL=ON``. NOTE: If ``RUN_SERIAL`` |
| 140 |
# is passed in but ``<fullTestName>_SET_RUN_SERIAL=OFF`` (or any value |
| 141 |
# evaluating to ``FALSE``), then the ``RUN_SERIAL`` test property will |
| 142 |
# **NOT** be set on the added test(s). |
| 143 |
# |
| 144 |
# ``ARGS "<arg0> <arg1> ..." "<arg2> <arg3> ..." ...`` |
| 145 |
# |
| 146 |
# If specified, then a set of arguments can be passed in quotes. If |
| 147 |
# multiple groups of arguments are passed in different quoted clusters of |
| 148 |
# arguments then a different test will be added for each set of arguments. |
| 149 |
# In this way, many different tests can be added for a single executable |
| 150 |
# in a single call to this function. Each of these separate tests will be |
| 151 |
# named ``<fullTestName>_xy`` where ``xy`` = ``00``, ``01``, ``02``, and so |
| 152 |
# on. **WARNING:** When defining multiple tests it is preferred to use the |
| 153 |
# ``POSTFIX_AND_ARGS_<IDX>`` form instead. **WARNING:** Multiple |
| 154 |
# arguments passed to a single test invocation must be quoted or multiple |
| 155 |
# tests taking single arguments will be created instead! See `Adding |
| 156 |
# Multiple Tests (tribits_add_test())`_ for more details and examples. |
| 157 |
# |
| 158 |
# ``POSTFIX_AND_ARGS_<IDX> <postfix> <arg0> <arg1> ...`` |
| 159 |
# |
| 160 |
# If specified, gives a sequence of sets of test postfix names and |
| 161 |
# arguments lists for different tests (up to ``POSTFIX_AND_ARGS_19``). |
| 162 |
# For example, a set of three different tests with argument lists can be |
| 163 |
# specified as:: |
| 164 |
# |
| 165 |
# POSTIFX_AND_ARGS_0 postfix0 --arg1 --arg2=dummy |
| 166 |
# POSTIFX_AND_ARGS_1 postfix1 --arg2=fly |
| 167 |
# POSTIFX_AND_ARGS_2 postfix2 --arg2=bags |
| 168 |
# |
| 169 |
# This will create three different test cases with the postfix names |
| 170 |
# ``postfix0``, ``postfix1``, and ``postfix2``. The indexes must be |
| 171 |
# consecutive starting a ``0`` and going up to (currently) ``19``. The |
| 172 |
# main advantages of using these arguments instead of just ``ARGS`` are |
| 173 |
# that one can give a meaningful name to each test case and one can |
| 174 |
# specify multiple arguments without having to quote them and one can |
| 175 |
# allow long argument lists to span multiple lines. See `Adding Multiple |
| 176 |
# Tests (tribits_add_test())`_ for more details and examples. |
| 177 |
# |
| 178 |
# Note that one of the `<postfix>` arguments can be empty, in which case |
| 179 |
# the base test name is not appended so:: |
| 180 |
# |
| 181 |
# POSTIFX_AND_ARGS_0 "" --arg1 --arg2=dummy |
| 182 |
# |
| 183 |
# would create one test without appending the test name. |
| 184 |
# |
| 185 |
# ``COMM [serial] [mpi]`` |
| 186 |
# |
| 187 |
# If specified, determines if the test will be added in serial and/or MPI |
| 188 |
# mode. If the ``COMM`` argument is missing, the test will be added in |
| 189 |
# both serial and MPI builds of the code. That is if ``COMM mpi`` is |
| 190 |
# passed in, then the test will **not** be added if |
| 191 |
# ``TPL_ENABLE_MPI=OFF``. Likewise, if ``COMM serial`` is passed in, then |
| 192 |
# the test will **not** be added if ``TPL_ENABLE_MPI=ON``. If ``COMM |
| 193 |
# serial mpi`` or ``COMM mpi serial`` is passed in, then the value of |
| 194 |
# ``TPL_ENABLE_MPI`` does not determine if the test is added or not. |
| 195 |
# |
| 196 |
# ``NUM_MPI_PROCS <numMpiProcs>`` |
| 197 |
# |
| 198 |
# If specified, gives the number of MPI processes used to run the test |
| 199 |
# with the MPI exec program ``${MPI_EXEC}``. If ``<numMpiProcs>`` is |
| 200 |
# greater than ``${MPI_EXEC_MAX_NUMPROCS}`` then the test will be |
| 201 |
# excluded. If not specified, then the default number of processes for an |
| 202 |
# MPI build (i.e. ``TPL_ENABLE_MPI=ON``) will be |
| 203 |
# ``${MPI_EXEC_DEFAULT_NUMPROCS}``. For serial builds |
| 204 |
# (i.e. ``TPL_ENABLE_MPI=OFF``), passing in a value ``<numMpiProcs>`` > |
| 205 |
# ``1`` will cause the test to not be added. The value ``<numMpiProcs>`` |
| 206 |
# will also be set as the built-in test property ``PROCESSORS`` if |
| 207 |
# ``NUM_TOTAL_CORES_USED`` is not specified. |
| 208 |
# |
| 209 |
# ``NUM_TOTAL_CORES_USED <numTotalCoresUsed>`` |
| 210 |
# |
| 211 |
# If specified, gives the total number of processes or cores that is |
| 212 |
# reported to CTest as the built-in CTest ``PROCESSORS`` property. If |
| 213 |
# this is not specified, then ``PROCESSORS`` is specified by the argument |
| 214 |
# ``NUM_MPI_PROCS <numMpiProcs>``. This argument is used for test |
| 215 |
# scripts/executables that use more cores than MPI processes |
| 216 |
# (i.e. ``<numMpiProcs>``) and its only purpose is to inform CTest and |
| 217 |
# TriBITS of the maximum number of processes or cores that are used by the |
| 218 |
# underlying test executable/script. When specified, if |
| 219 |
# ``<numTotalCoresUsed>`` is greater than ``${MPI_EXEC_MAX_NUMPROCS}``, |
| 220 |
# then the test will not be added. Otherwise, the CTest property |
| 221 |
# ``PROCESSORS`` is set to ``<numTotalCoresUsed>`` so that CTest knows how |
| 222 |
# to best schedule the test w.r.t. other tests on a given number of |
| 223 |
# available processes. See `Running multiple tests at the same time |
| 224 |
# (tribits_add_test())`_. |
| 225 |
# |
| 226 |
# ``CATEGORIES <category0> <category1> ...`` |
| 227 |
# |
| 228 |
# If specified, gives the specific categories of the test. Valid test |
| 229 |
# categories include ``BASIC``, ``CONTINUOUS``, ``NIGHTLY``, ``HEAVY`` |
| 230 |
# and ``PERFORMANCE``. If not specified, the default category is |
| 231 |
# ``BASIC``. When the test category does not match |
| 232 |
# ``${PROJECT_NAME}_TEST_CATEGORIES``, then the test is **not** added. |
| 233 |
# When ``CATEGORIES`` contains ``BASIC`` it will match |
| 234 |
# ``${PROJECT_NAME}_TEST_CATEGORIES`` equal to ``CONTINUOUS``, |
| 235 |
# ``NIGHTLY``, and ``HEAVY``. When ``CATEGORIES`` contains |
| 236 |
# ``CONTINUOUS`` it will match ``${PROJECT_NAME}_TEST_CATEGORIES`` equal |
| 237 |
# to ``CONTINUOUS``, ``NIGHTLY``, and ``HEAVY``. When ``CATEGORIES`` |
| 238 |
# contains ``NIGHTLY`` it will match ``${PROJECT_NAME}_TEST_CATEGORIES`` |
| 239 |
# equal to ``NIGHTLY`` and ``HEAVY``. When ``CATEGORIES`` contains |
| 240 |
# ``PERFORMANCE`` it will match |
| 241 |
# ``${PROJECT_NAME}_TEST_CATEGORIES=PERFORMANCE`` only. |
| 242 |
# |
| 243 |
# ``HOST <host0> <host1> ...`` |
| 244 |
# |
| 245 |
# If specified, gives a list of hostnames where the test will be included. |
| 246 |
# The current hostname is determined by the built-in CMake command |
| 247 |
# ``site_name(${PROJECT_NAME}_HOSTNAME)``. On Linux/Unix systems, this is |
| 248 |
# typically the value returned by ``uname -n``. If this list is given, |
| 249 |
# the value of ``${${PROJECT_NAME}_HOSTNAME}`` must equal one of the |
| 250 |
# listed host names ``<hosti>`` or test will **not** be added. The value |
| 251 |
# of ``${PROJECT_NAME}_HOSTNAME`` gets printed out in the TriBITS cmake |
| 252 |
# output under the section ``Probing the environment`` (see `Full |
| 253 |
# Processing of TriBITS Project Files`_). |
| 254 |
# |
| 255 |
# ``XHOST <host0> <host1> ...`` |
| 256 |
# |
| 257 |
# If specified, gives a list of hostnames (see ``HOST`` argument) on which |
| 258 |
# the test will **not** be added. This check is performed after the check |
| 259 |
# for the hostnames in the ``HOST`` list if it should exist. Therefore, |
| 260 |
# this exclusion list overrides the ``HOST`` inclusion list. |
| 261 |
# |
| 262 |
# ``HOSTTYPE <hosttype0> <hosttype1> ...`` |
| 263 |
# |
| 264 |
# If specified, gives the names of the host system type (given by the |
| 265 |
# built-in CMake cache variable ``CMAKE_HOST_SYSTEM_NAME`` which is |
| 266 |
# printed in the TriBITS cmake configure output in the section ``Probing |
| 267 |
# the environment``) for which the test is allowed to be added. If |
| 268 |
# ``HOSTTYPE`` is specified and ``CMAKE_HOST_SYSTEM_NAME`` is not equal to |
| 269 |
# one of the values of ``<hosttypei>``, then the test will **not** be |
| 270 |
# added. Typical host system type names include ``Linux``, ``Darwin``, |
| 271 |
# ``Windows``, etc. |
| 272 |
# |
| 273 |
# ``XHOSTTYPE <hosttype0> <hosttype1> ...`` |
| 274 |
# |
| 275 |
# If specified, gives the names of the host system type (see the |
| 276 |
# ``HOSTTYPE`` argument above) for which **not** to include the test on. |
| 277 |
# This check is performed after the check for the host system names in the |
| 278 |
# ``HOSTTYPE`` list if it should exist. Therefore, this exclusion list |
| 279 |
# overrides the ``HOSTTYPE`` inclusion list. |
| 280 |
# |
| 281 |
# ``EXCLUDE_IF_NOT_TRUE <varname0> <varname1> ...`` |
| 282 |
# |
| 283 |
# If specified, gives the names of CMake variables that must evaluate to |
| 284 |
# true, or the test will not be added. |
| 285 |
# |
| 286 |
# ``DISABLED <messageWhyDisabled>`` |
| 287 |
# |
| 288 |
# If ``<messageWhyDisabled>`` is non-empty and does not evaluate to FALSE, |
| 289 |
# then the test will be added by ``add_test()`` (so CTest will see it) but |
| 290 |
# the ctest test property ``DISABLED`` will be set. Therefore, CTest will |
| 291 |
# not run the test and will instead list it as "Not Run" when tests are |
| 292 |
# run locally and when submitting test results to CDash (with test details |
| 293 |
# "Not Run (Disabled)"). Also, the message ``<messageWhyDisabled>`` will |
| 294 |
# be printed to STDOUT by cmake after the line stating the test was added |
| 295 |
# when ``${PROJECT_NAME}_TRACE_ADD_TEST=ON`` is set. If |
| 296 |
# ``<messageWhyDisabled>`` evaluates to FALSE in CMake (e.g. "FALSE", |
| 297 |
# "false", "NO", "no", "0", "", etc.), then the ``DISABLED`` property will |
| 298 |
# not be set. This property can also be set with the CMake cache var ``-D |
| 299 |
# <fullTestName>_SET_DISABLED_AND_MSG="<msgSetByVar>"`` and in fact that |
| 300 |
# var will override the value of ``<messageWhyDisabled>`` passed in here |
| 301 |
# (if ``<msgSetByVar>`` is non-empty). This allows a user to enable tests |
| 302 |
# that are disabled in the CMakeList.txt files using this input. |
| 303 |
# |
| 304 |
# ``STANDARD_PASS_OUTPUT`` |
| 305 |
# |
| 306 |
# If specified, then the standard test output string ``End Result: TEST |
| 307 |
# PASSED`` is grepped in the test stdout for to determine success. This |
| 308 |
# is needed for MPI tests on some platforms since the return value from |
| 309 |
# MPI executables is unreliable. This is set using the built-in CTest |
| 310 |
# property ``PASS_REGULAR_EXPRESSION``. |
| 311 |
# |
| 312 |
# ``PASS_REGULAR_EXPRESSION "<regex0>" "<regex1>" ...`` |
| 313 |
# |
| 314 |
# If specified, then the test will be assumed to pass only if one of the |
| 315 |
# regular expressions ``<regex0>``, ``<regex1>`` etc. match the output |
| 316 |
# send to stdout. Otherwise, the test will fail. This is set using the |
| 317 |
# built-in CTest property ``PASS_REGULAR_EXPRESSION``. Consult standard |
| 318 |
# CMake documentation for full behavior. TIPS: Replace ';' with '[;]' or |
| 319 |
# CMake will interpret this as an array element boundary. To match '.', |
| 320 |
# use '[.]'. |
| 321 |
# |
| 322 |
# ``FAIL_REGULAR_EXPRESSION "<regex0>" "<regex1>" ...`` |
| 323 |
# |
| 324 |
# If specified, then a test will be assumed to fail if one of the regular |
| 325 |
# expressions ``<regex0>``, ``<regex1>`` etc. match the output send to |
| 326 |
# stdout. Otherwise, the test will pass. This is set using the built-in |
| 327 |
# CTest property ``FAIL_REGULAR_EXPRESSION``. Consult standard CMake |
| 328 |
# documentation for full behavior (and see above tips for |
| 329 |
# ``PASS_REGULAR_EXPRESSION``). |
| 330 |
# |
| 331 |
# ``WILL_FAIL`` |
| 332 |
# |
| 333 |
# If passed in, then the pass/fail criteria will be inverted. This is set |
| 334 |
# using the built-in CTest property ``WILL_FAIL``. Consult standard CMake |
| 335 |
# documentation for full behavior. |
| 336 |
# |
| 337 |
# ``ENVIRONMENT "<var1>=<value1>" "<var2>=<value2>" ...``. |
| 338 |
# |
| 339 |
# If passed in, the listed environment variables will be set by CTest |
| 340 |
# before calling the test. This is set using the built-in CTest test |
| 341 |
# property ``ENVIRONMENT``. Note, if the env var values contain |
| 342 |
# semi-colons ``';'``, then replace the semi-colons ``';'`` with another |
| 343 |
# separator ``'<sep>'`` and pass in ``LIST_SEPARATOR <sep>`` so ``<sep>`` |
| 344 |
# will be replaced with ``';'`` at point of usage. If the env var values |
| 345 |
# contain any spaces, also quote the entire variable/value pair as |
| 346 |
# ``"<vari>=<valuei>"``. For example, the env var and value |
| 347 |
# ``my_env_var="arg1 b;arg2;I have spaces"`` would need to be passed as |
| 348 |
# ``"my_env_var=arg1 b<sep>arg2<sep>I have spaces"``. |
| 349 |
# |
| 350 |
# ``TIMEOUT <maxSeconds>`` |
| 351 |
# |
| 352 |
# If passed in, gives maximum number of seconds the test will be allowed |
| 353 |
# to run before being timed-out and killed. This sets the CTest property |
| 354 |
# ``TIMEOUT``. The value ``<maxSeconds>`` will be scaled by the value of |
| 355 |
# `${PROJECT_NAME}_SCALE_TEST_TIMEOUT`_. See `Setting timeouts for tests |
| 356 |
# (tribits_add_test())`_ for more details. |
| 357 |
# |
| 358 |
# **WARNING:** Rather than just increasing the timeout for an expensive |
| 359 |
# test, please try to either make the test run faster or relegate the test |
| 360 |
# to being run less often (i.e. set ``CATEGORIES NIGHTLY`` or even |
| 361 |
# ``HEAVY`` for extremely expensive tests). Expensive tests are one of |
| 362 |
# the worse forms of technical debt that a project can have! |
| 363 |
# |
| 364 |
# ``LIST_SEPARATOR <sep>`` |
| 365 |
# |
| 366 |
# String used as placeholder for the semi-colon char ``';'`` in order to |
| 367 |
# allow pass-through. For example, if arguments to the ``ARGS`` or |
| 368 |
# ``ENVIRONMENT`` need to use semi-colons, then replace ``';'`` with |
| 369 |
# ``'<semicolon>'`` (for example) such as with |
| 370 |
# ``"somearg=arg1<semicolon>arg2"``, then at the point of usage, |
| 371 |
# ``'<semicolon>'`` will be replaced with ``';'`` and it will be passed to |
| 372 |
# the final command as ``"somearg=arg1;arg2"`` (with as many preceding |
| 373 |
# escape backslashes ``'\'`` in front of ``';'`` as is needed for the |
| 374 |
# given usage context). |
| 375 |
# |
| 376 |
# ``ADDED_TESTS_NAMES_OUT <testsNames>`` |
| 377 |
# |
| 378 |
# If specified, then on output the variable ``<testsNames>`` will be set |
| 379 |
# with the name(S) of the tests passed to ``add_test()``. If more than |
| 380 |
# one test is added, then this will be a list of test names. Having this |
| 381 |
# name allows the calling ``CMakeLists.txt`` file access and set |
| 382 |
# additional test properties (see `Setting additional test properties |
| 383 |
# (tribits_add_test())`_). |
| 384 |
# |
| 385 |
# In the end, this function just calls the built-in CMake commands |
| 386 |
# ``add_test(${TEST_NAME} ...)`` and ``set_tests_properties(${TEST_NAME} |
| 387 |
# ...)`` to set up a executable process for ``ctest`` to run, determine |
| 388 |
# pass/fail criteria, and set some other test properties. Therefore, this |
| 389 |
# wrapper function does not provide any fundamentally new features that are |
| 390 |
# not already available in the basic usage if CMake/CTest. However, this |
| 391 |
# wrapper function takes care of many of the details and boiler-plate CMake |
| 392 |
# code that it takes to add such a test (or tests) and enforces consistency |
| 393 |
# across a large project for how tests are defined, run, and named (to avoid |
| 394 |
# test name clashes). |
| 395 |
# |
| 396 |
# If more flexibility or control is needed when defining tests, then the |
| 397 |
# function `tribits_add_advanced_test()`_ should be used instead. |
| 398 |
# |
| 399 |
# In the following subsections, more details on how tests are defined and run |
| 400 |
# is given. |
| 401 |
# |
| 402 |
# .. _Determining the Executable or Command to Run (tribits_add_test()): |
| 403 |
# |
| 404 |
# **Determining the Executable or Command to Run (tribits_add_test())** |
| 405 |
# |
| 406 |
# This function is primarily designed to make it easy to run tests for |
| 407 |
# executables built using the function `tribits_add_executable()`_. To set up |
| 408 |
# tests to run arbitrary executables, see below. |
| 409 |
# |
| 410 |
# By default, the executable to run is determined by first getting the |
| 411 |
# executable name which by default is assumed to be:: |
| 412 |
# |
| 413 |
# <fullExeName> = |
| 414 |
# ${PACKAGE_NAME}_<exeRootName>${${PROJECT_NAME}_CMAKE_EXECUTABLE_SUFFIX} |
| 415 |
# |
| 416 |
# which is (by no coincidence) identical to how it is selected in |
| 417 |
# `tribits_add_executable()`_ (see `Executable and Target Name |
| 418 |
# (tribits_add_executable())`_). This name can be altered by passing in |
| 419 |
# ``NOEXEPREFIX``, ``NOEXESUFFIX``, and ``ADD_DIR_TO_NAME`` as described in |
| 420 |
# `Executable and Target Name (tribits_add_executable())`_. |
| 421 |
# |
| 422 |
# By default, this executable is assumed to be in the current CMake binary |
| 423 |
# directory ``${CMAKE_CURRENT_BINARY_DIR}`` but the directory location can be |
| 424 |
# changed using the ``DIRECTORY <dir>`` argument. |
| 425 |
# |
| 426 |
# If an arbitrary executable is to be run (i.e. not build inside of the |
| 427 |
# project), then pass in ``NOEXEPREFIX`` and ``NOEXESUFFIX`` and set |
| 428 |
# ``<exeRootName>`` to the relative or absolute path of the executable to be |
| 429 |
# run. If ``<exeRootName>`` is not an absolute path, then |
| 430 |
# ``${CMAKE_CURRENT_BINARY_DIR}/<exeRootName>`` is set as the executable to |
| 431 |
# run in this case. |
| 432 |
# |
| 433 |
# NOTE: On native Windows platforms, the ``NOEXESUFFIX`` will still allow |
| 434 |
# CTest to run executables that have the ``*.exe`` suffix. |
| 435 |
# |
| 436 |
# Whatever executable path is specified using this logic, if the executable is |
| 437 |
# not found, then when ``ctest`` goes to run the test, it will mark it as |
| 438 |
# ``NOT RUN``. |
| 439 |
# |
| 440 |
# .. _Determining the Full Test Name (tribits_add_test()): |
| 441 |
# |
| 442 |
# **Determining the Full Test Name (tribits_add_test())** |
| 443 |
# |
| 444 |
# By default, the base test name is selected to be:: |
| 445 |
# |
| 446 |
# <fullTestName> = ${PACKAGE_NAME}_<exeRootName> |
| 447 |
# |
| 448 |
# If ``NAME <testRootName>`` is passed in, then ``<testRootName>`` is used |
| 449 |
# instead of ``<exeRootName>`` above. |
| 450 |
# |
| 451 |
# If ``NAME_POSTFIX <testNamePostfix>`` is passed in, then the base test name |
| 452 |
# is selected to be:: |
| 453 |
# |
| 454 |
# <fullTestName> = ${PACKAGE_NAME}_<exeRootName>_<testNamePostfix> |
| 455 |
# |
| 456 |
# If ``ADD_DIR_TO_NAME`` is passed in, then the directory name relative to the |
| 457 |
# package base directory is added to the name as well to help disambiguate the |
| 458 |
# test name (see the above). |
| 459 |
# |
| 460 |
# Let the test name determined as described above be ``<fullTestName>``. If |
| 461 |
# no arguments or only a single set of arguments are passed in through |
| 462 |
# ``ARGS``, then this is the test name actually passed in to ``add_test()``. |
| 463 |
# If multiple tests are defined, then this name becomes the base test name for |
| 464 |
# each of the tests (see `Adding Multiple Tests (tribits_add_test())`_). |
| 465 |
# |
| 466 |
# Finally, for any test that gets defined, if MPI is enabled |
| 467 |
# (i.e. ``TPL_ENABLE_MPI=ON``), then the terminal suffix |
| 468 |
# ``_MPI_${NUM_MPI_PROCS}`` will be added to the end of the test name (even |
| 469 |
# for multiple tests). No such prefix is added for the serial case |
| 470 |
# (i.e. ``TPL_ENABLE_MPI=OFF``). |
| 471 |
# |
| 472 |
# .. _Adding Multiple Tests (tribits_add_test()): |
| 473 |
# |
| 474 |
# **Adding Multiple Tests (tribits_add_test())** |
| 475 |
# |
| 476 |
# Using this function, one can add executable arguments and can even add |
| 477 |
# multiple tests in one of two ways. One can either pass in one or more |
| 478 |
# **quoted** clusters of arguments using:: |
| 479 |
# |
| 480 |
# ARGS "<arg0> <arg1> ..." "<arg2> <arg3> ..." ... |
| 481 |
# |
| 482 |
# or can pass in an explicit test name postfix and arguments with:: |
| 483 |
# |
| 484 |
# POSTFIX_AND_ARGS_0 <postfix0> <arg0> <arg1> ... |
| 485 |
# POSTFIX_AND_ARGS_1 <postfix1> <arg2> ... |
| 486 |
# ... |
| 487 |
# |
| 488 |
# If only one short set of arguments needs to be passed in, then passing:: |
| 489 |
# |
| 490 |
# ARGS "<arg0> <arg1>" |
| 491 |
# |
| 492 |
# may be preferable since it will not add any postfix name to the test. To |
| 493 |
# add more than one test case using ``ARGS``, one will use more than one |
| 494 |
# quoted set of arguments such as with:: |
| 495 |
# |
| 496 |
# ARGS "<arg0> <arg1>" "<arg2> <arg2>" |
| 497 |
# |
| 498 |
# which creates 2 tests with the names ``<fullTestName>_00`` passing |
| 499 |
# arguments ``"<arg0> <arg1>"`` and ``<fullTestName>_01`` passing arguments |
| 500 |
# ``"<arg2> <arg3>"``. However, when passing multiple sets of arguments it is |
| 501 |
# preferable to **not** use ``ARGS`` but instead use:: |
| 502 |
# |
| 503 |
# POSTFIX_AND_ARGS_0 test_a <arg0> <arg1> |
| 504 |
# POSTFIX_AND_ARGS_1 test_b <arg2> <arg2> |
| 505 |
# |
| 506 |
# which also creates the same 2 tests but now with the improved names |
| 507 |
# ``<fullTestName>_test_a`` passing arguments ``"<arg0> <arg1>"`` and |
| 508 |
# ``<fullTestName>_test_b`` passing arguments ``"<arg2> <arg3>"``. In this way, |
| 509 |
# the individual tests can be given more understandable names. |
| 510 |
# |
| 511 |
# The other advantage of the ``POSTFIX_AND_ARGS_<IDX>`` form is that the |
| 512 |
# arguments ``<arg0>``, ``<arg1>``, ... do not need to be quoted and can |
| 513 |
# therefore be extended over multiple lines like:: |
| 514 |
# |
| 515 |
# POSTFIX_AND_ARGS_0 long_args --this-is-the-first-long-arg=very |
| 516 |
# --this-is-the-second-long-arg=verylong |
| 517 |
# |
| 518 |
# If one does not use quotes when using ``ARGS`` one will actually get more |
| 519 |
# than one test. For example, if one passes in:: |
| 520 |
# |
| 521 |
# ARGS --this-is-the-first-long-arg=very |
| 522 |
# --this-is-the-second-long-arg=verylong |
| 523 |
# |
| 524 |
# one actually gets two tests, not one test. This is a common mistake that |
| 525 |
# people make when using the ``ARGS`` form of passing arguments. This can't |
| 526 |
# be fixed or it will break backward compatibility. If this could be designed |
| 527 |
# fresh, the ``ARGS`` argument would only create a single test and the |
| 528 |
# arguments would not be quoted. |
| 529 |
# |
| 530 |
# .. _Determining Pass/Fail (tribits_add_test()): |
| 531 |
# |
| 532 |
# **Determining Pass/Fail (tribits_add_test())** |
| 533 |
# |
| 534 |
# The only means to determine pass/fail is to use the built-in CTest |
| 535 |
# properties ``PASS_REGULAR_EXPRESSION`` and ``FAIL_REGULAR_EXPRESSION`` which |
| 536 |
# can only grep the test's STDOUT/STDERR or to check for a 0 return value (or |
| 537 |
# invert these using ``WILL_FAIL``). For simple tests, that is enough. |
| 538 |
# However, for more complex executables, one may need to examine one or more |
| 539 |
# output files to determine pass/fail. Raw CMake/CTest cannot do this. In |
| 540 |
# this case, one should use `tribits_add_advanced_test()`_ instead to add the |
| 541 |
# test. |
| 542 |
# |
| 543 |
# .. _Setting additional test properties (tribits_add_test()): |
| 544 |
# |
| 545 |
# **Setting additional test properties (tribits_add_test())** |
| 546 |
# |
| 547 |
# After this function returns, any tests that get added using ``add_test()`` |
| 548 |
# can have additional properties set and changed using |
| 549 |
# ``set_tests_properties()``. Therefore, any tests properties that are not |
| 550 |
# directly supported and passed through this wrapper function can be set in |
| 551 |
# the outer ``CMakeLists.txt`` file after the call to ``tribits_add_test()``. |
| 552 |
# |
| 553 |
# If tests are added, then the names of those tests will be returned in the |
| 554 |
# variable ``ADDED_TESTS_NAMES_OUT <testsNames>``. This can be used, for |
| 555 |
# example, to override the ``PROCESSORS`` property for the tests with:: |
| 556 |
# |
| 557 |
# tribits_add_test( someTest ... |
| 558 |
# ADDED_TESTS_NAMES_OUT someTest_TEST_NAME ) |
| 559 |
# |
| 560 |
# if (someTest_TEST_NAME) |
| 561 |
# set_tests_properties( ${someTest_TEST_NAME} |
| 562 |
# PROPERTIES ATTACHED_FILES someTest.log ) |
| 563 |
# endif() |
| 564 |
# |
| 565 |
# where the test writes a log file ``someTest.log`` that we want to submit to |
| 566 |
# CDash also. |
| 567 |
# |
| 568 |
# This approach will work no matter what TriBITS names the individual test(s) |
| 569 |
# or whether the test(s) are added or not (depending on other arguments like |
| 570 |
# ``COMM``, ``XHOST``, etc.). |
| 571 |
# |
| 572 |
# The following built-in CTest test properties are set through `Formal |
| 573 |
# Arguments (tribits_add_test())`_ or are otherwise automatically set by this |
| 574 |
# function and should **NOT** be overridden by direct calls to |
| 575 |
# ``set_tests_properties()``: ``ENVIRONMENT``, ``FAIL_REGULAR_EXPRESSION``, |
| 576 |
# ``LABELS``, ``PASS_REGULAR_EXPRESSION``, ``RUN_SERIAL``, ``TIMEOUT``, and |
| 577 |
# ``WILL_FAIL``. |
| 578 |
# |
| 579 |
# However, generally, other built-in CTest test properties can be set after |
| 580 |
# the test is added like show above. Examples of test properties that can be |
| 581 |
# set using direct calls to ``set_tests_properties()`` include |
| 582 |
# ``ATTACHED_FILES``, ``ATTACHED_FILES_ON_FAIL``, ``COST``, ``DEPENDS``, |
| 583 |
# ``MEASUREMENT``, ``RESOURCE_LOCK`` and ``WORKING_DIRECTORY``. |
| 584 |
# |
| 585 |
# For example, one can set a dependency between two tests using:: |
| 586 |
# |
| 587 |
# tribits_add_test( test_a [...] |
| 588 |
# ADDED_TESTS_NAMES_OUT test_a_TEST_NAME ) |
| 589 |
# |
| 590 |
# tribits_add_test( test_b [...] |
| 591 |
# ADDED_TESTS_NAMES_OUT test_z_TEST_NAME ) |
| 592 |
# |
| 593 |
# if (test_a_TEST_NAME AND test_b_TEST_NAME) |
| 594 |
# set_tests_properties(${test_b_TEST_NAME} |
| 595 |
# PROPERTIES DEPENDS ${test_a_TEST_NAME}) |
| 596 |
# endif() |
| 597 |
# |
| 598 |
# This ensures that test ``test_b`` will always be run after ``test_a`` if |
| 599 |
# both tests are run by CTest. |
| 600 |
# |
| 601 |
# .. _Running multiple tests at the same time (tribits_add_test()): |
| 602 |
# |
| 603 |
# **Running multiple tests at the same time (tribits_add_test())** |
| 604 |
# |
| 605 |
# By default, CTest will run as many tests defined with ``add_test()`` at same |
| 606 |
# time as it can according to its parallel level (e.g. ``'ctest -j<N>'`` or |
| 607 |
# the CTest property ``CTEST_PARALLEL_LEVEL``). For example, when raw |
| 608 |
# ``'ctest -j10'`` is run, CTest will run multiple tests at the same time to |
| 609 |
# try to make usage of 10 processors/cores. If all of the defined tests only |
| 610 |
# used one process (which is assumed by default except for MPI tests), then |
| 611 |
# CTest will run 10 tests at the same time and will launch new tests as |
| 612 |
# running tests finish. One can also define tests which use more than one |
| 613 |
# process or use more cores than the number of MPI processes. When passing in |
| 614 |
# ``NUM_MPI_PROCS <numMpiProcs>`` (see above), this TriBITS function will set the |
| 615 |
# built-in CTest property ``PROCESSORS`` to ``<numMpiProcs>`` using:: |
| 616 |
# |
| 617 |
# set_tests_properties(<fullTestName> PROPERTIES PROCESSORS <numMpiProcs>) |
| 618 |
# |
| 619 |
# This tells CTest that the defined test uses ``<numMpiProcs>`` processes and |
| 620 |
# CTest will use that information to not exceed the requested parallel level. |
| 621 |
# For example, if several ``NUM_MPI_PROCS 3`` tests are defined and CTest is |
| 622 |
# run with ``'ctest -j12'``, then CTest would schedule and run 4 of these |
| 623 |
# tests at a time (to make use of 12 processors/cores on the machine), |
| 624 |
# starting new tests as running tests finish, until all of the tests have been |
| 625 |
# run. |
| 626 |
# |
| 627 |
# There are some situations where a test will use more processes/cores than |
| 628 |
# specified by ``NUM_MPI_PROCS <numMpiProcs>`` such as when the underlying |
| 629 |
# executable fires off more processes in parallel to do processing. Also, an |
| 630 |
# MPI program may use threading and therefore use overall more cores than the |
| 631 |
# number of MPI processes. For these cases, it is critical to set |
| 632 |
# ``NUM_TOTAL_CORES_USED <numTotalCoresUsed>`` to tell TriBITS and CTest how |
| 633 |
# many cores will be used by a threaded test. This is needed to exclude the |
| 634 |
# test if there are too many processes/cores needed to run the test than are |
| 635 |
# available. Also, if the test is added, then this is needed to set the |
| 636 |
# built-in CTest ``PROCESSORS`` property so CTest can avoid overloading the |
| 637 |
# machine. For example, for test where the MPI executable running on 4 |
| 638 |
# processes uses 10 threads per process, one would set:: |
| 639 |
# |
| 640 |
# NUM_MPI_PROCS 4 NUM_TOTAL_CORES_USED 40 |
| 641 |
# |
| 642 |
# In this case, it sets the CTest ``PROCESSORS`` property as:: |
| 643 |
# |
| 644 |
# set_tests_properties(<fullTestName> PROPERTIES PROCESSORS <numTotalCoresUsed>) |
| 645 |
# |
| 646 |
# When the number of processes a test uses does not cleanly divide into the |
| 647 |
# requested CTest parallel level, it is not clear how CTest schedules the |
| 648 |
# tests (hard to find documentation on this but one could always inspect the |
| 649 |
# CTest source code to find out for sure). However, one boundary case that is |
| 650 |
# well observed is that CTest will run all defined tests regardless of the |
| 651 |
# size of the ``PROCESSORS`` property or the value of |
| 652 |
# ``CTEST_PARALLEL_LEVEL``. For example, if there are tests where |
| 653 |
# ``PROCESSORS`` is set to 20 but ```ctest -j10'`` is used, then CTest will |
| 654 |
# still run those tests (using 20 processes) but will not schedule any other |
| 655 |
# tests while the parallel level is exceeded. This can overload the machine |
| 656 |
# obviously. Therefore, always set ``MPI_EXEC_MAX_NUMPROCS`` to the maximum |
| 657 |
# number of cores/processes that can be comfortably run on a given machine. |
| 658 |
# Also note that MPI tests are very fragile to overloaded machines |
| 659 |
# |
| 660 |
# NOTE: **Never** manually override the ``PROCESSORS`` CTest property. |
| 661 |
# Instead, always using ``NUM_TOTAL_CORES_USED <numTotalCoresUsed>`` to set |
| 662 |
# this. This is important because TriBITS needs to know how many |
| 663 |
# processes/cores are required for test so that it can disable the test if the |
| 664 |
# test requires more cores/processes than a given machine can handle or to |
| 665 |
# exceed an imposed budget of the number processes to be used. The latter is |
| 666 |
# important when running multiple ``ctest -J<N>`` invocations on the same test |
| 667 |
# machine. |
| 668 |
# |
| 669 |
# .. _Setting timeouts for tests (tribits_add_test()): |
| 670 |
# |
| 671 |
# **Setting timeouts for tests (tribits_add_test())** |
| 672 |
# |
| 673 |
# By default, all tests have a default timeout (1500 seconds for most |
| 674 |
# projects, see `DART_TESTING_TIMEOUT`_). That means that if they hang |
| 675 |
# (e.g. as is common when deadlocking occurs with multi-process MPI-based |
| 676 |
# tests and multi-threaded tests) then each test may hang for a long time, |
| 677 |
# causing the overall test suite to take a long time to complete. For many |
| 678 |
# CMake projects, this default timeout is way too long. |
| 679 |
# |
| 680 |
# Timeouts for tests can be adjusted in a couple of ways. First, a default |
| 681 |
# timeout for all tests is enforced by CTest given the configured value of the |
| 682 |
# variable `DART_TESTING_TIMEOUT`_ (typically set by the user but set by |
| 683 |
# default to 1500 for most projects). This is a global timeout that applies |
| 684 |
# to all tests that don't otherwise have individual timeouts set using the |
| 685 |
# ``TIMEOUT`` CTest property (see below). The value of |
| 686 |
# ``DART_TESTING_TIMEOUT`` in the CMake cache on input to CMake will get |
| 687 |
# scaled by `${PROJECT_NAME}_SCALE_TEST_TIMEOUT`_ and the scaled value gets |
| 688 |
# written into the file ``DartConfiguration.tcl`` as the field ``TimeOut``. |
| 689 |
# The``ctest`` executable reads ``TimeOut`` from this file when it runs to |
| 690 |
# determine the default global timeout. The value of this default global |
| 691 |
# ``TimeOut`` can be overridden using the ``ctest`` argument ``--timeout |
| 692 |
# <seconds>`` (see `Overriding test timeouts`_). |
| 693 |
# |
| 694 |
# Alternatively, timeouts for individual tests can be set using the input |
| 695 |
# argument ``TIMEOUT`` (see `Formal Arguments (tribits_add_test())`_ above). |
| 696 |
# The timeout value passed in to this function is then scaled by |
| 697 |
# ``${PROJECT_NAME}_SCALE_TEST_TIMEOUT`` and the scaled timeout is then set as |
| 698 |
# the CTest test property ``TIMEOUT``. One can observe the value of this |
| 699 |
# property in the CMake-generated file ``CTestTestfile.cmake`` in the current |
| 700 |
# build directory. Individual test timeouts set this way are not impacted by |
| 701 |
# the global default timeout ``DART_TESTING_TIMEOUT`` or the ``ctest`` |
| 702 |
# argument ``--timeout <seconds>``. |
| 703 |
# |
| 704 |
# In summary, CTest determines the timeout for any individual test as follows: |
| 705 |
# |
| 706 |
# * If the ``TIMEOUT`` CTest property in ``CTestTestfile.cmake`` for that test |
| 707 |
# is set, then that value is used. |
| 708 |
# |
| 709 |
# * Else if the ``ctest`` commandline option ``--timeout <seconds>`` is |
| 710 |
# provided, then that timeout is used. |
| 711 |
# |
| 712 |
# * Else if the property ``TimeOut`` in the base file |
| 713 |
# ``DartConfiguration.tcl`` is set to non-empty, then that timeout is used. |
| 714 |
# |
| 715 |
# * Else, no timeout is set and the test can run (and hang) forever until |
| 716 |
# manually killed by the user. |
| 717 |
# |
| 718 |
# .. _Debugging and Examining Test Generation (tribits_add_test()): |
| 719 |
# |
| 720 |
# **Debugging and Examining Test Generation (tribits_add_test())** |
| 721 |
# |
| 722 |
# In order to see what tests get added and if not then why, configure with |
| 723 |
# ``${PROJECT_NAME}_TRACE_ADD_TEST=ON``. That will print one line per show |
| 724 |
# that the test got added and if not then why the test was not added (i.e. due |
| 725 |
# to ``COMM``, ``NUM_MPI_PROCS``, ``CATEGORIES``, ``HOST``, ``XHOST``, |
| 726 |
# ``HOSTTYPE``, or ``XHOSTTYPE``). |
| 727 |
# |
| 728 |
# Also, CMake writes a file ``CTestTestfile.cmake`` in the current binary |
| 729 |
# directory which contains all of the added tests and test properties that are |
| 730 |
# set. This is the file that is read by ``ctest`` when it runs to determine |
| 731 |
# what tests to run, determine pass/fail and adjust other behavior using test |
| 732 |
# properties. In this file, one can see the exact ``add_test()`` and |
| 733 |
# ``set_tests_properties()`` commands. The is the ultimate way to debug |
| 734 |
# exactly what tests are getting added by this function (or if the test is |
| 735 |
# even being added at all). |
| 736 |
# |
| 737 |
# .. _Disabling Tests Externally (tribits_add_test()): |
| 738 |
# |
| 739 |
# **Disabling Tests Externally (tribits_add_test())** |
| 740 |
# |
| 741 |
# The test can be disabled externally by setting the CMake cache variable |
| 742 |
# ``<fullTestName>_DISABLE=TRUE``. This allows tests to be disabled on a |
| 743 |
# case-by-case basis by the user (for whatever reason). Here, |
| 744 |
# ``<fullTestName>`` must be the *exact* name that shows up in 'ctest -N' when |
| 745 |
# running the test. If multiple tests are added in this function through |
| 746 |
# multiple argument sets to ``ARGS`` or through multiple |
| 747 |
# ``POSTFIX_AND_ARGS_<IDX>`` arguments, then ``<fullTestName>_DISABLE=TRUE`` |
| 748 |
# must be set for each test individually. When a test is disabled in this |
| 749 |
# way, TriBITS will always print a warning to the ``cmake`` stdout at |
| 750 |
# configure time warning that the test is being disabled. |
| 751 |
# |
| 752 |
# .. _Adding extra commandline arguments externally (tribits_add_test()): |
| 753 |
# |
| 754 |
# **Adding extra commandline arguments externally (tribits_add_test())** |
| 755 |
# |
| 756 |
# One can add additional command-line arguments for any ctest test added using |
| 757 |
# this function. In order to do so, set the CMake cache variable:: |
| 758 |
# |
| 759 |
# set(<fullTestName>_EXTRA_ARGS "<earg0>;<earg1>;<earg2>;..." |
| 760 |
# CACHE STRING "Extra args") |
| 761 |
# |
| 762 |
# in a ``*.cmake`` configure options fragment file or:: |
| 763 |
# |
| 764 |
# -D <fullTestName>_EXTRA_ARGS="<earg0>;<earg1>;<earg2>;..." |
| 765 |
# |
| 766 |
# on the CMake command-line. |
| 767 |
# |
| 768 |
# These extra command-line arguments are added after any arguments passed in |
| 769 |
# through ``ARGS "<oarg0> <oarg1> ..."`` or ``POSTFIX_AND_ARGS_<IDX> <oarg0> |
| 770 |
# <oarg1> ...``. This allows these extra arguments to override the earlier |
| 771 |
# arguments. |
| 772 |
# |
| 773 |
# The primary motivating use case for ``<fullTestName>_EXTRA_ARGS`` is to |
| 774 |
# allow one to alter how a test runs on a specific platform or build. For |
| 775 |
# example, this allows one to disable specific individual unit tests for a |
| 776 |
# GTest executable such as with:: |
| 777 |
# |
| 778 |
# set(<fullTestName>_EXTRA_ARGS "--gtest_filter=-<unittest0>:<unittest1>:..." |
| 779 |
# CACHE STRING "Disable specific unit tests" ) |
| 780 |
# |
| 781 |
# For example, this would be an alternative to disabling an entire unit |
| 782 |
# testing executable using ``-D<fullTestName>_DISABLE=ON`` as described above. |
| 783 |
# |
| 784 |
function(tribits_add_test EXE_NAME) |
| 785 |
|
| 786 |
if(${PROJECT_NAME}_VERBOSE_CONFIGURE) |
| 787 |
message("") |
| 788 |
message("TRIBITS_ADD_TEST: ${EXE_NAME} ${ARGN}") |
| 789 |
endif() |
| 790 |
|
| 791 |
tribits_set_tribits_package_name() |
| 792 |
|
| 793 |
global_set(TRIBITS_ADD_TEST_ADD_TEST_INPUT) |
| 794 |
global_set(TRIBITS_SET_TEST_PROPERTIES_INPUT) |
| 795 |
global_set(MESSAGE_WRAPPER_INPUT) |
| 796 |
|
| 797 |
# |
| 798 |
# A) Parse the input arguments |
| 799 |
# |
| 800 |
|
| 801 |
# Allow for a maximum of 20 (0 through 19) postfix and argument blocks |
| 802 |
set(MAX_NUM_POSTFIX_AND_ARGS_IDX 19) |
| 803 |
|
| 804 |
set(POSTFIX_AND_ARGS_LIST "") |
| 805 |
foreach( POSTFIX_AND_ARGS_IDX RANGE ${MAX_NUM_POSTFIX_AND_ARGS_IDX}) |
| 806 |
list( APPEND POSTFIX_AND_ARGS_LIST POSTFIX_AND_ARGS_${POSTFIX_AND_ARGS_IDX} ) |
| 807 |
endforeach() |
| 808 |
#print_var(POSTFIX_AND_ARGS_LIST) |
| 809 |
|
| 810 |
cmake_parse_arguments( |
| 811 |
PARSE_ARGV 1 # One named argument |
| 812 |
#prefix |
| 813 |
PARSE |
| 814 |
# options |
| 815 |
"NOEXEPREFIX;NOEXESUFFIX;STANDARD_PASS_OUTPUT;WILL_FAIL;ADD_DIR_TO_NAME;RUN_SERIAL" |
| 816 |
#one_value_keywords |
| 817 |
"DISABLED;LIST_SEPARATOR" |
| 818 |
#multi_value_keywords |
| 819 |
"DIRECTORY;KEYWORDS;COMM;NUM_MPI_PROCS;NUM_TOTAL_CORES_USED;ARGS;${POSTFIX_AND_ARGS_LIST};NAME;NAME_POSTFIX;CATEGORIES;HOST;XHOST;HOSTTYPE;XHOSTTYPE;EXCLUDE_IF_NOT_TRUE;PASS_REGULAR_EXPRESSION;FAIL_REGULAR_EXPRESSION;TIMEOUT;ENVIRONMENT;ADDED_TESTS_NAMES_OUT" |
| 820 |
) |
| 821 |
|
| 822 |
tribits_check_for_unparsed_arguments() |
| 823 |
|
| 824 |
if (NOT "${PARSE_ARGS}" STREQUAL "") |
| 825 |
list(LENGTH PARSE_ARGS NUM_PARSE_ARGS) |
| 826 |
elseif (PARSE_POSTFIX_AND_ARGS_0) |
| 827 |
# We will use this list instead |
| 828 |
else() |
| 829 |
# Neither 'ARGS' nor 'POSTFIX_AND_ARGS' was selected so just assume one |
| 830 |
# empty arg |
| 831 |
set(PARSE_ARGS " ") |
| 832 |
set(NUM_PARSE_ARGS 1) |
| 833 |
endif() |
| 834 |
|
| 835 |
if (${PROJECT_NAME}_VERBOSE_CONFIGURE) |
| 836 |
message("") |
| 837 |
message("TRIBITS_ADD_TEST: EXE_NAME = ${EXE_NAME}") |
| 838 |
endif() |
| 839 |
|
| 840 |
if(PARSE_ADDED_TESTS_NAMES_OUT) |
| 841 |
set(${PARSE_ADDED_TESTS_NAMES_OUT} "" PARENT_SCOPE ) |
| 842 |
endif() |
| 843 |
|
| 844 |
set(ADDED_TESTS_NAMES_OUT) |
| 845 |
|
| 846 |
# |
| 847 |
# Get test name |
| 848 |
# |
| 849 |
|
| 850 |
# If requested create a modifier for the name that will be inserted between |
| 851 |
# the package name and the given name or exe_name for the test |
| 852 |
set(directoryName "") |
| 853 |
if (PARSE_ADD_DIR_TO_NAME) |
| 854 |
tribits_create_name_from_current_source_directory(directoryName) |
| 855 |
set(directoryName "${directoryName}_") |
| 856 |
endif() |
| 857 |
|
| 858 |
#message("TRIBITS_ADD_TEST: ${EXE_NAME}: EXE_BINARY_NAME = ${EXE_BINARY_NAME}") |
| 859 |
|
| 860 |
if (PARSE_NAME) |
| 861 |
set(TEST_NAME "${directoryName}${PARSE_NAME}") |
| 862 |
elseif (PARSE_NAME_POSTFIX) |
| 863 |
set(TEST_NAME "${directoryName}${EXE_NAME}_${PARSE_NAME_POSTFIX}") |
| 864 |
else() |
| 865 |
set(TEST_NAME "${directoryName}${EXE_NAME}") |
| 866 |
endif() |
| 867 |
|
| 868 |
set(TEST_NAME "${PACKAGE_NAME}_${TEST_NAME}") |
| 869 |
|
| 870 |
# |
| 871 |
# B) Add or don't add tests based on a number of criteria |
| 872 |
# |
| 873 |
|
| 874 |
set(ADD_THE_TEST FALSE) |
| 875 |
tribits_add_test_process_enable_tests(ADD_THE_TEST) |
| 876 |
if (NOT ADD_THE_TEST) |
| 877 |
return() |
| 878 |
endif() |
| 879 |
|
| 880 |
tribits_add_test_process_skip_ctest_add_test(ADD_THE_TEST) |
| 881 |
if (NOT ADD_THE_TEST) |
| 882 |
return() |
| 883 |
endif() |
| 884 |
|
| 885 |
set(ADD_THE_TEST FALSE) |
| 886 |
tribits_add_test_process_categories(ADD_THE_TEST) |
| 887 |
if (NOT ADD_THE_TEST) |
| 888 |
return() |
| 889 |
endif() |
| 890 |
|
| 891 |
set(ADD_THE_TEST FALSE) |
| 892 |
tribits_add_test_process_host_hosttype(ADD_THE_TEST) |
| 893 |
if (NOT ADD_THE_TEST) |
| 894 |
return() |
| 895 |
endif() |
| 896 |
|
| 897 |
# |
| 898 |
# C) Set the name and path of the binary that will be run |
| 899 |
# |
| 900 |
|
| 901 |
tribits_add_test_get_exe_binary_name( |
| 902 |
"${EXE_NAME}" |
| 903 |
${PARSE_NOEXEPREFIX} |
| 904 |
${PARSE_NOEXESUFFIX} |
| 905 |
${PARSE_ADD_DIR_TO_NAME} EXE_BINARY_NAME |
| 906 |
) |
| 907 |
|
| 908 |
tribits_add_test_adjust_directory( ${EXE_BINARY_NAME} "${PARSE_DIRECTORY}" |
| 909 |
EXECUTABLE_PATH) |
| 910 |
|
| 911 |
# |
| 912 |
# D) Determine if we will add the serial or MPI tests based on input COMM |
| 913 |
# and TPL_ENABLE_MPI |
| 914 |
# |
| 915 |
|
| 916 |
tribits_process_comm_args(ADD_SERIAL_TEST ADD_MPI_TEST ${PARSE_COMM}) |
| 917 |
|
| 918 |
# |
| 919 |
# E) Get the MPI options |
| 920 |
# |
| 921 |
|
| 922 |
tribits_add_test_get_num_procs_used("${PARSE_NUM_MPI_PROCS}" |
| 923 |
"NUM_MPI_PROCS" NUM_PROCS_USED NUM_PROCS_USED_NAME) |
| 924 |
if (NUM_PROCS_USED LESS 0) |
| 925 |
set(ADD_SERIAL_TEST FALSE) |
| 926 |
set(ADD_MPI_TEST FALSE) |
| 927 |
endif() |
| 928 |
|
| 929 |
if (TPL_ENABLE_MPI) |
| 930 |
set(MPI_NAME_POSTFIX "_MPI_${NUM_PROCS_USED}") |
| 931 |
else() |
| 932 |
set(MPI_NAME_POSTFIX "") |
| 933 |
endif() |
| 934 |
|
| 935 |
tribits_add_test_get_num_total_cores_used("${TEST_NAME}${MPI_NAME_POSTFIX}" |
| 936 |
"${PARSE_NUM_TOTAL_CORES_USED}" "NUM_TOTAL_CORES_USED" |
| 937 |
"${NUM_PROCS_USED}" "${NUM_PROCS_USED_NAME}" |
| 938 |
NUM_TOTAL_CORES_USED SKIP_TEST) |
| 939 |
if (SKIP_TEST) |
| 940 |
set(ADD_SERIAL_TEST FALSE) |
| 941 |
set(ADD_MPI_TEST FALSE) |
| 942 |
endif() |
| 943 |
|
| 944 |
# |
| 945 |
# F) Add the tests |
| 946 |
# |
| 947 |
|
| 948 |
if (NOT ADD_SERIAL_TEST AND NOT ADD_MPI_TEST) |
| 949 |
return() |
| 950 |
endif() |
| 951 |
|
| 952 |
if (NOT "${PARSE_ARGS}" STREQUAL "") |
| 953 |
|
| 954 |
# F.1) Add tests with simple lists of arguments |
| 955 |
|
| 956 |
set(COUNTER 0) |
| 957 |
|
| 958 |
foreach(PARSE_ARG ${PARSE_ARGS}) |
| 959 |
|
| 960 |
if(${NUM_PARSE_ARGS} EQUAL 1) |
| 961 |
set(TEST_NAME_INSTANCE "${TEST_NAME}${MPI_NAME_POSTFIX}") |
| 962 |
else() |
| 963 |
set(TEST_NAME_INSTANCE "${TEST_NAME}_${COUNTER}${MPI_NAME_POSTFIX}") |
| 964 |
endif() |
| 965 |
if(${PROJECT_NAME}_VERBOSE_CONFIGURE) |
| 966 |
message(STATUS "TEST_NAME = ${TEST_NAME_INSTANCE}") |
| 967 |
endif() |
| 968 |
|
| 969 |
tribits_convert_cmnd_arg_string_to_add_test_arg_array(${PARSE_ARG} INARGS) |
| 970 |
if (${PROJECT_NAME}_VERBOSE_CONFIGURE) |
| 971 |
print_var(INARGS) |
| 972 |
endif() |
| 973 |
|
| 974 |
tribits_set_run_serial(${TEST_NAME_INSTANCE} "${PARSE_RUN_SERIAL}" |
| 975 |
SET_RUN_SERIAL) |
| 976 |
|
| 977 |
tribits_set_disabled_and_msg(${TEST_NAME_INSTANCE} "${PARSE_DISABLED}" |
| 978 |
SET_DISABLED_AND_MSG) |
| 979 |
|
| 980 |
tribits_add_test_add_test_all( ${TEST_NAME_INSTANCE} |
| 981 |
"${EXECUTABLE_PATH}" "${PARSE_CATEGORIES}" "${NUM_PROCS_USED}" |
| 982 |
"${NUM_TOTAL_CORES_USED}" |
| 983 |
"${SET_RUN_SERIAL}" "${SET_DISABLED_AND_MSG}" ADDED_TEST_NAME ${INARGS} |
| 984 |
"${${TEST_NAME_INSTANCE}_EXTRA_ARGS}" ) |
| 985 |
if(PARSE_ADDED_TESTS_NAMES_OUT AND ADDED_TEST_NAME) |
| 986 |
list(APPEND ADDED_TESTS_NAMES_OUT ${ADDED_TEST_NAME}) |
| 987 |
endif() |
| 988 |
|
| 989 |
math(EXPR COUNTER ${COUNTER}+1 ) |
| 990 |
|
| 991 |
endforeach() |
| 992 |
|
| 993 |
elseif (PARSE_POSTFIX_AND_ARGS_0) |
| 994 |
|
| 995 |
# F.2) Add tests with different postfixes for each set of arguments |
| 996 |
|
| 997 |
foreach( POSTFIX_AND_ARGS_IDX RANGE ${MAX_NUM_POSTFIX_AND_ARGS_IDX}) |
| 998 |
|
| 999 |
if(${PROJECT_NAME}_VERBOSE_CONFIGURE) |
| 1000 |
print_var(PARSE_POSTFIX_AND_ARGS_${POSTFIX_AND_ARGS_IDX}) |
| 1001 |
endif() |
| 1002 |
|
| 1003 |
if (NOT PARSE_POSTFIX_AND_ARGS_${POSTFIX_AND_ARGS_IDX}) |
| 1004 |
break() |
| 1005 |
endif() |
| 1006 |
|
| 1007 |
set( POSTFIX_AND_ARGS "${PARSE_POSTFIX_AND_ARGS_${POSTFIX_AND_ARGS_IDX}}" ) |
| 1008 |
|
| 1009 |
list( GET POSTFIX_AND_ARGS 0 POSTFIX ) |
| 1010 |
set( INARGS "${POSTFIX_AND_ARGS}" ) # Initially contains postfix as ele 0 |
| 1011 |
list( REMOVE_AT INARGS 0 ) # Strip off the postfix name |
| 1012 |
|
| 1013 |
if (NOT "${POSTFIX}" STREQUAL "") |
| 1014 |
set(TEST_NAME_INSTANCE "${TEST_NAME}_${POSTFIX}${MPI_NAME_POSTFIX}") |
| 1015 |
else() |
| 1016 |
set(TEST_NAME_INSTANCE "${TEST_NAME}${MPI_NAME_POSTFIX}") |
| 1017 |
endif() |
| 1018 |
|
| 1019 |
tribits_set_run_serial(${TEST_NAME_INSTANCE} "${PARSE_RUN_SERIAL}" |
| 1020 |
SET_RUN_SERIAL) |
| 1021 |
|
| 1022 |
tribits_set_disabled_and_msg(${TEST_NAME_INSTANCE} "${PARSE_DISABLED}" |
| 1023 |
SET_DISABLED_AND_MSG) |
| 1024 |
|
| 1025 |
tribits_add_test_add_test_all( ${TEST_NAME_INSTANCE} |
| 1026 |
"${EXECUTABLE_PATH}" "${PARSE_CATEGORIES}" "${NUM_PROCS_USED}" |
| 1027 |
"${NUM_TOTAL_CORES_USED}" |
| 1028 |
"${SET_RUN_SERIAL}" "${SET_DISABLED_AND_MSG}" ADDED_TEST_NAME ${INARGS} |
| 1029 |
"${${TEST_NAME_INSTANCE}_EXTRA_ARGS}" |
| 1030 |
) |
| 1031 |
if(PARSE_ADDED_TESTS_NAMES_OUT AND ADDED_TEST_NAME) |
| 1032 |
list(APPEND ADDED_TESTS_NAMES_OUT ${ADDED_TEST_NAME}) |
| 1033 |
endif() |
| 1034 |
|
| 1035 |
endforeach() |
| 1036 |
|
| 1037 |
endif() |
| 1038 |
|
| 1039 |
if(PARSE_ADDED_TESTS_NAMES_OUT) |
| 1040 |
set(${PARSE_ADDED_TESTS_NAMES_OUT} "${ADDED_TESTS_NAMES_OUT}" |
| 1041 |
PARENT_SCOPE ) |
| 1042 |
endif() |
| 1043 |
|
| 1044 |
endfunction() |