Debian Codegraph @

cmake/tribits/core/utils/DriveAdvancedTest.cmake

raw
1 # @HEADER
2 # *****************************************************************************
3 # TriBITS: Tribal Build, Integrate, and Test System
4 #
5 # Copyright 2013-2016 NTESS and the TriBITS contributors.
6 # SPDX-License-Identifier: BSD-3-Clause
7 # *****************************************************************************
8 # @HEADER
9
10 include_guard()
11
12 include("${CMAKE_CURRENT_LIST_DIR}/PrintVar.cmake")
13 include("${CMAKE_CURRENT_LIST_DIR}/AppendStringVar.cmake")
14 include("${CMAKE_CURRENT_LIST_DIR}/Join.cmake")
15 include("${CMAKE_CURRENT_LIST_DIR}/TimingUtils.cmake")
16 include("${CMAKE_CURRENT_LIST_DIR}/TribitsGetCategoriesString.cmake")
17
18
19 function(print_current_date_time PREFIX_STR)
20 execute_process( COMMAND date OUTPUT_STRIP_TRAILING_WHITESPACE
21 OUTPUT_VARIABLE DATE_TIME )
22 message("${PREFIX_STR} ${DATE_TIME}\n")
23 endfunction()
24
25
26 function(print_uptime PREFIX_STR)
27 execute_process( COMMAND uptime OUTPUT_STRIP_TRAILING_WHITESPACE
28 OUTPUT_VARIABLE MACHINE_LOAD )
29 message("${PREFIX_STR} ${MACHINE_LOAD}")
30 endfunction()
31
32
33 function(print_single_check_result msgBegin TEST_CASE_PASSED_IN)
34 if (TEST_CASE_PASSED_IN)
35 message("${msgBegin} [PASSED]")
36 else()
37 message("${msgBegin} [FAILED]")
38 endif()
39 endfunction()
40
41
42 function(print_any_regex_pass_match msgBegin regexMatch)
43 if (regexMatch)
44 message("${msgBegin} [PASSED]")
45 else()
46 message("${msgBegin} [not matched]")
47 endif()
48 endfunction()
49
50
51 function(print_any_regex_fail_match msgBegin regexMatch)
52 if (regexMatch)
53 message("${msgBegin} [FAILED]")
54 else()
55 message("${msgBegin} [does not match]")
56 endif()
57 endfunction()
58
59
60 function(delete_create_working_directory WORKING_DIR_IN SKIP_CLEAN)
61 if (EXISTS "${WORKING_DIR_IN}" AND NOT SKIP_CLEAN)
62 message("Removing existing working directory"
63 " '${WORKING_DIR_IN}'\n")
64 if (NOT SHOW_COMMANDS_ONLY)
65 file(REMOVE_RECURSE "${WORKING_DIR_IN}")
66 endif()
67 endif()
68 if (NOT EXISTS "${WORKING_DIR_IN}")
69 message("Creating new working directory"
70 " '${WORKING_DIR_IN}'\n")
71 if (NOT SHOW_COMMANDS_ONLY)
72 file(MAKE_DIRECTORY "${WORKING_DIR_IN}")
73 endif()
74 endif()
75 endfunction()
76
77
78 macro(setup_and_run_test_idx_copy_files_block)
79
80 message(
81 "Copy files from:\n"
82 " ${TEST_${CMND_IDX}_SOURCE_DIR}/\n"
83 " to:\n"
84 " ${TEST_${CMND_IDX}_DEST_DIR}/\n")
85
86 split("${TEST_${CMND_IDX}_COPY_FILES_TO_TEST_DIR}"
87 "," COPY_FILES_TO_TEST_DIR_ARRAY)
88
89 set(TEST_CASE_PASSED TRUE)
90
91 message("${OUTPUT_SEP}\n")
92
93 # Make the dest directory if not exists (cmake -E copy will not create it)
94 if (NOT EXISTS "${TEST_${CMND_IDX}_DEST_DIR}")
95 message("Creating dest directory ${TEST_${CMND_IDX}_DEST_DIR}/ ...")
96 execute_process(
97 COMMAND ${CMAKE_COMMAND} -E make_directory "${TEST_${CMND_IDX}_DEST_DIR}"
98 RESULT_VARIABLE MKDIR_COMMAND_RTN
99 )
100 if (NOT MKDIR_COMMAND_RTN EQUAL 0)
101 set(TEST_CASE_PASSED FALSE)
102 endif()
103 # NOTE: Above, it would have been great to be able use file(MAKE_DIRECTORY
104 # ...) but if that fails it will just abortS the cmake -P script. There
105 # is no way to handle that gracefully. Therefore, we have to use
106 # execute_process(cmake -E make_directory ...).
107 endif()
108
109 # Copy the full list of files to the dest dir
110 foreach(FILENAME ${COPY_FILES_TO_TEST_DIR_ARRAY})
111 message("Copy file ${FILENAME} ...")
112 execute_process(
113 COMMAND ${CMAKE_COMMAND} -E copy
114 "${TEST_${CMND_IDX}_SOURCE_DIR}/${FILENAME}"
115 "${TEST_${CMND_IDX}_DEST_DIR}/"
116 RESULT_VARIABLE COPY_COMMAND_RTN
117 )
118 if (NOT COPY_COMMAND_RTN EQUAL 0)
119 set(TEST_CASE_PASSED FALSE)
120 endif()
121 endforeach()
122
123 # NOTE: Above, it would have been great to be able use
124 # configure_file(... COPYONLY) but if that fails it will just abortS the
125 # cmake -P script. There is no way to handle that gracefully. Therefore,
126 # we have to use execute_process(cmake -E copy ...). Also, it would have
127 # been great to just copy the files all at once with `cmake -E copy
128 # <srcDir>/<file0> <srDir>/<file1> ... <distDir>/` but older versions of
129 # CMake don't support that mode. They only support copying the files one at
130 # a time :-(
131
132 message("\n${OUTPUT_SEP}\n")
133
134 message("") # Need a vertical space for readability of the output
135
136 endmacro()
137
138
139 macro(setup_and_run_test_idx_cmnd_block)
140
141 # Address working directory for this TEST_<IDX> block if set
142 if (TEST_${CMND_IDX}_WORKING_DIRECTORY)
143 if (NOT IS_ABSOLUTE "${TEST_${CMND_IDX}_WORKING_DIRECTORY}")
144 set(TEST_${CMND_IDX}_WORKING_DIRECTORY
145 ${BASE_WORKING_DIRECTORY}/${TEST_${CMND_IDX}_WORKING_DIRECTORY})
146 endif()
147 delete_create_working_directory("${TEST_${CMND_IDX}_WORKING_DIRECTORY}"
148 ${TEST_${CMND_IDX}_SKIP_CLEAN_WORKING_DIRECTORY})
149 endif()
150
151 # Set up the TEST_<IDX> command block
152 join( TEST_CMND_STR " " TRUE ${TEST_${CMND_IDX}_CMND} )
153 string(REPLACE "${LIST_SEPARATOR}" ";" TEST_CMND_STR "${TEST_CMND_STR}")
154 message("Running: ${TEST_CMND_STR}\n")
155 set(EXEC_CMND COMMAND ${TEST_${CMND_IDX}_CMND})
156 string(REPLACE "${LIST_SEPARATOR}" "\\\;" EXEC_CMND "${EXEC_CMND}")
157
158 # Set up the working directory that this TEST_<IDX> CMND block will run in
159
160 set(WORKING_DIR_SET)
161 if (TEST_${CMND_IDX}_WORKING_DIRECTORY)
162 set(WORKING_DIR_SET "${TEST_${CMND_IDX}_WORKING_DIRECTORY}")
163 elseif(OVERALL_WORKING_DIRECTORY)
164 set(WORKING_DIR_SET "${OVERALL_WORKING_DIRECTORY}")
165 endif()
166
167 if (WORKING_DIR_SET)
168 message(" Running in working directory \"${WORKING_DIR_SET}\"\n")
169 set(WORKING_DIR "${WORKING_DIR_SET}")
170 else()
171 set(WORKING_DIR "${CMAKE_CURRENT_BINARY_DIR}")
172 endif()
173
174 # Set the actual command that will be run with execute_proces()
175
176 set(EXEC_CMND ${EXEC_CMND}
177 WORKING_DIRECTORY "${WORKING_DIR}"
178 )
179
180 # Set up the optional output file that the execute_process() command will write to
181
182 if (TEST_${CMND_IDX}_OUTPUT_FILE)
183 if (NOT IS_ABSOLUTE "${TEST_${CMND_IDX}_OUTPUT_FILE}")
184 set(OUTPUT_FILE_USED "${WORKING_DIR}/${TEST_${CMND_IDX}_OUTPUT_FILE}")
185 else()
186 set(OUTPUT_FILE_USED "${TEST_${CMND_IDX}_OUTPUT_FILE}")
187 endif()
188 message(" Writing output to file \"${OUTPUT_FILE_USED}\"\n")
189 endif()
190
191 # Run the actual command with executte_process() (or just print what would run) ...
192
193 if (NOT SHOW_COMMANDS_ONLY)
194
195 # Provide the test configuration in an environment variable.
196 if(TEST_CONFIG)
197 set(ENV{TEST_CONFIG} "${TEST_CONFIG}")
198 endif(TEST_CONFIG)
199
200 execute_process(
201 ${EXEC_CMND}
202 OUTPUT_VARIABLE TEST_CMND_OUT
203 ERROR_VARIABLE TEST_CMND_OUT
204 RESULT_VARIABLE EXEC_RESULT
205 )
206
207 if (TEST_${CMND_IDX}_OUTPUT_FILE)
208 file(WRITE "${OUTPUT_FILE_USED}" "${TEST_CMND_OUT}")
209 endif()
210
211 message("${OUTPUT_SEP}\n")
212
213 if (NOT TEST_${CMND_IDX}_NO_ECHO_OUTPUT)
214 message("${TEST_CMND_OUT}")
215 else()
216 message("NO_ECHO_OUTPUT\n")
217 endif()
218
219 else()
220
221 message("\n*** Not running command on request ***")
222
223 endif()
224
225 message("${OUTPUT_SEP}\n")
226
227 endmacro()
228
229
230 macro(determine_test_idx_cmnd_block_pass_fail)
231
232 message("TEST_${CMND_IDX}: Return code = ${EXEC_RESULT}")
233
234 # A) Apply first set of pass/fail logic
235 set(TEST_CASE_PASSED FALSE)
236 if (TEST_${CMND_IDX}_PASS_ANY)
237 set(TEST_CASE_PASSED TRUE)
238 print_single_check_result(
239 "TEST_${CMND_IDX}: Pass criteria = Pass Any"
240 ${TEST_CASE_PASSED} )
241 elseif (TEST_${CMND_IDX}_PASS_REGULAR_EXPRESSION)
242 set(TEST_CASE_PASSED FALSE)
243 foreach(REGEX_STR ${TEST_${CMND_IDX}_PASS_REGULAR_EXPRESSION})
244 string(REGEX MATCH "${REGEX_STR}" MATCH_STR "${TEST_CMND_OUT}")
245 if (MATCH_STR)
246 set(TEST_CASE_PASSED TRUE)
247 endif()
248 print_any_regex_pass_match(
249 "TEST_${CMND_IDX}: Pass criteria = Match any REGEX {${REGEX_STR}}"
250 "${MATCH_STR}")
251 endforeach()
252 elseif (TEST_${CMND_IDX}_PASS_REGULAR_EXPRESSION_ALL)
253 set(TEST_CASE_PASSED TRUE)
254 foreach(REGEX_STR ${TEST_${CMND_IDX}_PASS_REGULAR_EXPRESSION_ALL})
255 string(REGEX MATCH "${REGEX_STR}" MATCH_STR "${TEST_CMND_OUT}" )
256 if (NOT "${MATCH_STR}" STREQUAL "")
257 set(THIS_REGEX_MATCHED TRUE)
258 else()
259 set(THIS_REGEX_MATCHED FALSE)
260 endif()
261 if (NOT THIS_REGEX_MATCHED)
262 set(TEST_CASE_PASSED FALSE)
263 endif()
264 print_single_check_result(
265 "TEST_${CMND_IDX}: Pass criteria = Match all REGEX {${REGEX_STR}}"
266 ${THIS_REGEX_MATCHED} )
267 endforeach()
268 else()
269 if (EXEC_RESULT EQUAL 0)
270 set(TEST_CASE_PASSED TRUE)
271 else()
272 set(TEST_CASE_PASSED FALSE)
273 endif()
274 print_single_check_result(
275 "TEST_${CMND_IDX}: Pass criteria = Zero return code"
276 ${TEST_CASE_PASSED} )
277 endif()
278
279 # B) Check for failing regex matching?
280 if (TEST_${CMND_IDX}_FAIL_REGULAR_EXPRESSION)
281 foreach(REGEX_STR ${TEST_${CMND_IDX}_FAIL_REGULAR_EXPRESSION})
282 string(REGEX MATCH "${REGEX_STR}" MATCH_STR "${TEST_CMND_OUT}")
283 if (MATCH_STR)
284 set(TEST_CASE_PASSED FALSE)
285 endif()
286 print_any_regex_fail_match(
287 "TEST_${CMND_IDX}: Pass criteria = Not match REGEX {${REGEX_STR}}"
288 "${MATCH_STR}")
289 endforeach()
290 endif()
291
292 # C) Check for return code always 0?
293 if (TEST_${CMND_IDX}_ALWAYS_FAIL_ON_NONZERO_RETURN)
294 if (NOT EXEC_RESULT EQUAL 0)
295 set(ALWAYS_FAIL_ON_NONZERO_RETURN_PASSED FALSE)
296 set(TEST_CASE_PASSED FALSE)
297 else()
298 set(ALWAYS_FAIL_ON_NONZERO_RETURN_PASSED TRUE)
299 endif()
300 print_single_check_result(
301 "TEST_${CMND_IDX}: Pass criteria = ALWAYS_FAIL_ON_NONZERO_RETURN"
302 ${ALWAYS_FAIL_ON_NONZERO_RETURN_PASSED} )
303 elseif (TEST_${CMND_IDX}_ALWAYS_FAIL_ON_ZERO_RETURN)
304 if (EXEC_RESULT EQUAL 0)
305 set(ALWAYS_FAIL_ON_ZERO_RETURN_PASSED FALSE)
306 set(TEST_CASE_PASSED FALSE)
307 else()
308 set(ALWAYS_FAIL_ON_ZERO_RETURN_PASSED TRUE)
309 endif()
310 print_single_check_result(
311 "TEST_${CMND_IDX}: Pass criteria = ALWAYS_FAIL_ON_ZERO_RETURN"
312 ${ALWAYS_FAIL_ON_ZERO_RETURN_PASSED} )
313 endif()
314
315 # D) Invert pass/fail result?
316 if (TEST_${CMND_IDX}_WILL_FAIL)
317 if (TEST_CASE_PASSED)
318 set(TEST_CASE_PASSED FALSE)
319 else()
320 set(TEST_CASE_PASSED TRUE)
321 endif()
322 print_single_check_result(
323 "TEST_${CMND_IDX}: Pass criteria = WILL_FAIL (invert the above 'Pass critera')"
324 ${TEST_CASE_PASSED} )
325 endif()
326
327 endmacro()
328
329
330 function(drive_advanced_test)
331
332 #
333 # A) Print the header for the advanced test
334 #
335
336 set(ADVANDED_TEST_SEP
337 "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX")
338
339 set(TEST_SEP
340 "================================================================================")
341
342 set(OUTPUT_SEP
343 "--------------------------------------------------------------------------------")
344
345 math(EXPR LAST_CMND_IDX ${NUM_CMNDS}-1)
346
347 message("\n${ADVANDED_TEST_SEP}\n")
348 message("Advanced Test: ${TEST_NAME}\n")
349
350 message("Selected Test/CTest Properties:")
351 tribits_get_categories_string("${CATEGORIES}" CATEGORIES_IN_COMMAS)
352 message(" CATEGORIES = ${CATEGORIES_IN_COMMAS}")
353 message(" PROCESSORS = ${PROCESSORS}")
354 if (TIMEOUT)
355 message(" TIMEOUT = ${TIMEOUT}\n")
356 else()
357 message(" TIMEOUT = DEFAULT\n")
358 endif()
359
360 if (SHOW_MACHINE_LOAD AND NOT SHOW_COMMANDS_ONLY)
361 print_uptime("Starting Uptime:")
362 endif()
363
364 if (SHOW_START_END_DATE_TIME AND NOT SHOW_COMMANDS_ONLY)
365 print_current_date_time("Starting at:")
366 endif()
367
368 if (OVERALL_WORKING_DIRECTORY)
369 if (NOT IS_ABSOLUTE "${OVERALL_WORKING_DIRECTORY}")
370 set(OVERALL_WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${OVERALL_WORKING_DIRECTORY})
371 endif()
372 delete_create_working_directory("${OVERALL_WORKING_DIRECTORY}"
373 ${SKIP_CLEAN_OVERALL_WORKING_DIRECTORY})
374 set(BASE_WORKING_DIRECTORY "${OVERALL_WORKING_DIRECTORY}")
375 else()
376 set(BASE_WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}")
377 endif()
378
379 foreach ( CMND_IDX RANGE ${LAST_CMND_IDX} )
380 if (CMND_IDX EQUAL 0)
381 set(TEST_NAMES_STR "TEST_0")
382 else()
383 string(APPEND TEST_NAMES_STR ", TEST_${CMND_IDX}" )
384 endif()
385 endforeach()
386 message("Running test commands: ${TEST_NAMES_STR}")
387
388 if (SHOW_START_END_DATE_TIME AND NOT SHOW_COMMANDS_ONLY)
389 timer_get_raw_seconds(TEST_CMND_START)
390 set(TEST_OVERALL_START ${TEST_CMND_START})
391 endif()
392
393 #
394 # B) Loop over and run the TEST_<IDX> blocks one at a time
395 #
396
397 set(OVERALL_TEST_PASSED TRUE)
398
399 foreach ( CMND_IDX RANGE ${LAST_CMND_IDX} )
400 message("\n${TEST_SEP}\n")
401 message("TEST_${CMND_IDX}\n")
402
403 # Print the message for this TEST_<IDX> block if set
404 if (TEST_${CMND_IDX}_MESSAGE)
405 message("${TEST_${CMND_IDX}_MESSAGE}\n")
406 endif()
407
408 # Run the TEST_<IDX> block (Copy files or CMND)
409 if (TEST_${CMND_IDX}_COPY_FILES_TO_TEST_DIR)
410 setup_and_run_test_idx_copy_files_block()
411 else()
412 setup_and_run_test_idx_cmnd_block()
413 endif()
414
415 # Print the load and/or timing info for TEST_<IDX> block
416
417 if (SHOW_MACHINE_LOAD AND NOT SHOW_COMMANDS_ONLY)
418 print_uptime("TEST_${CMND_IDX}: Uptime:")
419 endif()
420
421 if (SHOW_START_END_DATE_TIME AND NOT SHOW_COMMANDS_ONLY)
422 timer_get_raw_seconds(TEST_CMND_END)
423 if (TEST_CMND_START AND TEST_CMND_END)
424 timer_print_rel_time(${TEST_CMND_START} ${TEST_CMND_END}
425 "TEST_${CMND_IDX}: Time")
426 else()
427 message("ERROR: Not able to return test times! Is 'date' in your path?")
428 endif()
429 set(TEST_CMND_START ${TEST_CMND_END})
430 endif()
431
432 # Determine pass/fail for the TEST_<IDX> CMND block
433
434 if (NOT SHOW_COMMANDS_ONLY)
435
436 # Determine pass/fail for TEST_<IDX> copy files or CMND
437
438 if (TEST_${CMND_IDX}_COPY_FILES_TO_TEST_DIR)
439 # Pass/fail already determined by setting TEST_CASE_PASSED in
440 # SETUP_AND_RUN_TEST_IDX_COPY_FILES_BLOCK above.
441 else()
442 determine_test_idx_cmnd_block_pass_fail()
443 endif()
444
445 # Print final pass/fail for the TEST_<IDX> block
446
447 if (TEST_CASE_PASSED)
448 message("TEST_${CMND_IDX}: Result = PASSED")
449 else()
450 message("TEST_${CMND_IDX}: Result = FAILED")
451 set(OVERALL_TEST_PASSED FALSE)
452 if (FAIL_FAST)
453 message("TEST_${CMND_IDX}: FAIL FAST, SKIPPING REST OF TEST CASES!")
454 break()
455 endif()
456 endif()
457
458 endif(NOT SHOW_COMMANDS_ONLY)
459
460 endforeach()
461
462 message("\n${TEST_SEP}\n")
463
464 #
465 # C) Print the final test data and pass/fail
466 #
467
468 if (NOT SHOW_COMMANDS_ONLY)
469
470 if (SHOW_START_END_DATE_TIME)
471 print_current_date_time("Ending at:")
472 if (TEST_OVERALL_START AND TEST_CMND_END)
473 timer_print_rel_time(${TEST_OVERALL_START} ${TEST_CMND_END}
474 "OVERALL TEST TIME")
475 else()
476 message("ERROR: Not able to return test times! Is 'date' in your path?")
477 endif()
478 message("")
479 endif()
480
481 if (OVERALL_TEST_PASSED)
482 message("OVERALL FINAL RESULT: TEST PASSED (${TEST_NAME})")
483 else()
484 message("OVERALL FINAL RESULT: TEST FAILED (${TEST_NAME})")
485 endif()
486 else()
487 message("OVERALL FINAL RESULT: DID NOT RUN COMMANDS (${TEST_NAME})")
488 endif()
489
490 message("\n${ADVANDED_TEST_SEP}\n")
491
492 endfunction()