Debian Codegraph @

tests/f/cylc-view/test_header

raw
1 #!/usr/bin/env bash
2 # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
3 # Copyright (C) NIWA & British Crown (Met Office) & Contributors.
4 #
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU General Public License as published by
7 # the Free Software Foundation, either version 3 of the License, or
8 # (at your option) any later version.
9 #
10 # This program is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details.
14 #
15 # You should have received a copy of the GNU General Public License
16 # along with this program. If not, see <http://www.gnu.org/licenses/>.
17 #
18 # NAME
19 # test_header
20 #
21 # SYNOPSIS
22 # . $CYLC_REPO_DIR/tests/lib/bash/test_header
23 #
24 # DESCRIPTION
25 # Interface for constructing tests under a TAP harness (the "prove"
26 # command).
27 #
28 # FUNCTIONS
29 # set_test_number N
30 # echo a total number of tests for TAP to read.
31 # ok TEST_NAME
32 # echo a TAP OK message for TEST_NAME.
33 # fail TEST_NAME
34 # echo a TAP fail message for TEST_NAME. If $CYLC_TEST_DEBUG is set,
35 # cat $TEST_NAME.stderr to stderr.
36 # run_ok TEST_NAME COMMAND ...
37 # Run COMMAND with any following options/arguments and store stdout
38 # and stderr in TEST_NAME.stdout and TEST_NAME.stderr.
39 # This is expected to have a return code of 0, which ok's the test.
40 # run_fail TEST_NAME COMMAND ...
41 # Run COMMAND with any following options/arguments and store stdout
42 # and stderr in TEST_NAME.stdout and TEST_NAME.stderr.
43 # This is expected to have a non-zero return code, which ok's the test.
44 # run_graphql_ok TEST_NAME WORKFLOW JSON
45 # Send a graphql query to the workflow, extract the result from
46 # ${TEST_NAME}.stdout
47 # cmp_ok FILE_TEST FILE_CONTROL
48 # Compare FILE_TEST with a file or stdin given by FILE_CONTROL (stdin
49 # if FILE_CONTROL is "-" or missing). By default, it uses "diff -u" to
50 # compare files. However, if an alternate command such as "xxdiff -D"
51 # is desirable (e.g. for debugging),
52 # "export CYLC_TEST_DIFF_CMD=xxdiff -D".
53 # cmp_ok_re FILE_TEST FILE_CONTROL
54 # Match a file against a regex file line by line.
55 # Reads in the test and control files line by line and checks the test
56 # line re.match'es the control line.
57 # cmp_json TEST_NAME FILE_TEST [FILE_CONTROL]
58 # Alternative implementation to cmp_json_ok, compare two JSON files
59 # and display any differences as a unified(ish) diff:
60 # * Works with nested dictionaries.
61 # * Impervious to dictionary order.
62 # contains_ok FILE_TEST [FILE_CONTROL]
63 # Make sure that each line in FILE_TEST is present in FILE_CONTROL
64 # (stdin if FILE_CONTROL is "-" or missing).
65 # grep_ok PATTERN FILE [$OPTS]
66 # Run "grep [$OPTS] -q -e PATTERN FILE".
67 # grep_workflow_log_ok TEST_NAME PATTERN [$OPTS]
68 # Run "grep [$OPTS] -s -e PATTERN <workflow-log>".
69 # named_grep_ok NAME PATTERN FILE [$OPTS]
70 # Run grep_ok with a custom test name.
71 # OPTS: put grep options like '-E' (extended regex) at end of line.
72 # grep_fail PATTERN FILE [$OPTS]
73 # Run "grep [$OPTS] -q -e PATTERN FILE", expect no match.
74 # count_ok PATTERN FILE COUNT
75 # Test that PATTERN occurs in exactly COUNT lines of FILE.
76 # exists_ok FILE
77 # Test that FILE exists
78 # exists_fail FILE
79 # Test that FILE does not exist
80 # init_workflow TEST_NAME [SOURCE] [[RUN_NUMS]]
81 # Create a workflow from SOURCE's "flow.cylc" called:
82 # "cylctb-${CYLC_TEST_TIME_INIT}/${TEST_SOURCE_DIR##*tests/}/${TEST_NAME}"
83 # Provides WORKFLOW_NAME and WORKFLOW_RUN_DIR variables.
84 # RUN_NUMS (defaults to false): If false run cylc install --no-rundir
85 # install_workflow TEST_NAME SOURCE [RUN_NUMS]
86 # Same as init_workflow, but SOURCE must be a directory containing a
87 # "flow.cylc" file.
88 # log_scan TEST_NAME FILE ATTEMPTS DELAY PATTERN...
89 # Monitor FILE polling every DELAY seconds for maximum of ATTEMPTS
90 # tries grepping for each PATTERN in turn. Tests will only pass if the
91 # PATTERNs appear in FILE in the correct order. Runs one test per
92 # pattern, each prefixed by TEST_NAME.
93 # set LOG_SCAN_GREP_OPTS in the environment, e.g. "-E" for "grep -E"
94 # make_rnd_workflow
95 # Create a randomly-named workflow source directory.
96 # mock_smtpd_init
97 # Start a mock SMTP server daemon for testing. Write host:port setting
98 # to the variable TEST_SMTPD_HOST. Write pid of daemon to
99 # TEST_SMTPD_PID. Write log to TEST_SMTPD_LOG.
100 # mock_smtpd_kill
101 # Kill the mock SMTP server daemon process.
102 # poll COMMAND
103 # Run COMMAND in 1 second intervals for a minute until COMMAND returns
104 # a zero value, or exit 1 (abort test) with a timeout message.
105 # poll_grep ...
106 # Shorthand for 'poll grep -s ...'.
107 # poll_grep_workflow_log ...
108 # Shorthand for 'poll_grep ... "${WORKFLOW_RUN_DIR}/log/scheduler/log"'
109 # poll_pid_done PID
110 # Poll until process with PID is done.
111 # poll_workflow_running
112 # Shorthand for 'poll test -e "${WORKFLOW_RUN_DIR}/.service/contact"'
113 # poll_workflow_stopped
114 # Shorthand for 'poll "!" test -e "${WORKFLOW_RUN_DIR}/.service/contact"'.
115 # Note that you might want to call poll_workflow_running before this as
116 # it may return before contact file is created in the first place.
117 # get_workflow_uuid
118 # Extract the UUID from the contact file or echo '' if not present.
119 # poll_workflow_restart [TIMEOUT [UUID]]
120 # Wait for the workflow to restart.
121 # Provide the UUID from the current run if workflow is not
122 # guaranteed to be running.
123 # purge [WORKFLOW_NAME [PLATFORM]]
124 # Remove a workflow from the filesystem.
125 # Defaults to removing $WORKFLOW_NAME on $CYLC_TEST_PLATFORM
126 # purge_rnd_workflow
127 # Remove a workflow source dir and rundir.
128 # Typically used with make_rnd_workflow
129 # skip N SKIP_REASON
130 # echo "ok $((++T)) # skip SKIP_REASON" N times.
131 # skip_all SKIP_REASON
132 # echo "1..0 # SKIP SKIP_REASON" and exit.
133 # ssh_install_cylc HOST
134 # Install cylc on a remote host.
135 # reftest [TEST_NAME [SOURCE]]
136 # Install a reference workflow using `install_workflow`, run a validation
137 # test on the workflow and run the reference workflow with `workflow_run_ok`.
138 # Expect 2 OK tests.
139 # install_and_validate
140 # The first part of reftest, to allow separate use.
141 # Expect 1 OK test.
142 # reftest_run
143 # The guts of reftest, to allow separate use.
144 # Expect 1 OK test.
145 # create_test_global_config [PRE [POST]]
146 # Create a new global config file $PWD/etc from global-tests.cylc
147 # with PRE and POST pre- and ap-pended.
148 # PRE and POST are strings.
149 # The global config starts with #!Jinja2 in case Jinja2 is used.
150 # localhost_fqdn
151 # Get the FQDN of the current host using the same mechanism Cylc uses.
152 # get_fqdn [TARGET]
153 # SSH to TARGET and return `hostname -f`.
154 # dump_std TEST_NAME
155 # Dump stdout and stderr of TEST_NAME to the test log dir.
156 # delete_db [WORKFLOW_RUN_DIR]
157 # Delete private/public workflow databases for a cold start.
158 #
159 # VARIABLES
160 # LOG_INDENT
161 # indentation width of secondary log lines
162 # # TODO: document other variables used by test scripts
163 #
164 #-------------------------------------------------------------------------------
165
166 set -eu
167 shopt -s extglob
168
169 # For indented secondary log lines; must match the width in cylc.flow.loggingutil
170 export LOG_INDENT=" "
171
172 FAILURES=0
173 SIGNALS="EXIT INT"
174 TEST_DIR=
175 TEST_RHOST_CYLC_DIR=
176
177 FINALLY() {
178 for S in ${SIGNALS}; do
179 trap '' "${S}"
180 done
181 if [[ -n "${TEST_DIR}" ]]; then
182 cd ~
183 rm -rf "${TEST_DIR}"
184 fi
185 if [[ -n "${TEST_RHOST_CYLC_DIR}" ]]; then
186 # TEST_RHOST_CYLC_DIR is a local variable of this script
187 # shellcheck disable=SC2029
188 ssh -oBatchMode=yes -oConnectTimeout=5 "${TEST_RHOST_CYLC_DIR%%:*}" \
189 "rm -fr ${TEST_RHOST_CYLC_DIR#*:}"
190 fi
191 if [[ -n "${TEST_SMTPD_PID:-}" ]]; then
192 kill "${TEST_SMTPD_PID}"
193 fi
194 if ((FAILURES > 0)); then
195 echo -e "\n stdout and stderr stored in: ${TEST_LOG_DIR}" >&2
196 if "${WORKFLOW_RUN_FAILS}"; then
197 echo -e " workflow logs can be found under: ${WORKFLOW_LOG_DIR}/" >&2
198 fi
199 fi
200 }
201 for S in ${SIGNALS}; do
202 # Signal as argument to FINALLY
203 # shellcheck disable=SC2064
204 trap "FINALLY ${S}" "${S}"
205 done
206
207 TEST_NUMBER=0
208
209 if command -v lsof >/dev/null; then
210 HAS_LSOF=true
211 else
212 HAS_LSOF=false
213 fi
214
215 set_test_number() {
216 echo "1..$1"
217 }
218
219 ok() {
220 echo "ok $((++TEST_NUMBER)) - $*"
221 }
222
223 fail() {
224 ((++FAILURES))
225 echo "not ok $((++TEST_NUMBER)) - $*"
226 if [[ -n "${CYLC_TEST_DEBUG:-}" ]]; then
227 echo >'/dev/tty'
228 echo "${TEST_NAME_BASE} ${TEST_NAME}" >'/dev/tty'
229 cat "${TEST_NAME}.stderr" >'/dev/tty'
230 fi
231 }
232
233 dump_std() {
234 local TEST_NAME="$1"
235 mkdir -p "${TEST_LOG_DIR}" # directory may not exist if run fails very early
236 if [[ -s "${TEST_NAME}.stdout" ]]; then
237 cp -p "${TEST_NAME}.stdout" "${TEST_LOG_DIR}/${TEST_NAME}.stdout"
238 fi
239 cp -p "${TEST_NAME}.stderr" "${TEST_LOG_DIR}/${TEST_NAME}.stderr"
240 }
241
242 run_ok() {
243 local TEST_NAME="$1"
244 shift 1
245 if ! "$@" 1>"${TEST_NAME}.stdout" 2>"${TEST_NAME}.stderr"; then
246 fail "${TEST_NAME}"
247 dump_std "${TEST_NAME}"
248 return
249 fi
250 ok "${TEST_NAME}"
251 }
252
253 run_fail() {
254 local TEST_NAME="$1"
255 shift 1
256 if "$@" 1>"${TEST_NAME}.stdout" 2>"${TEST_NAME}.stderr"; then
257 fail "${TEST_NAME}"
258 dump_std "${TEST_NAME}"
259 return
260 fi
261 ok "${TEST_NAME}"
262 }
263
264 run_graphql_ok () {
265 # this is a thin wrapper to `cylc client` which strips newlines from $2 to
266 # make it valid JSON
267 local TEST_NAME="$1"
268 local WORKFLOW="$2"
269 local JSON="$3"
270 # shellcheck disable=SC2086
271 run_ok "${TEST_NAME}" cylc client \
272 "${WORKFLOW}" \
273 'graphql' \
274 < <(echo ${JSON})
275 }
276
277 workflow_run_ok() {
278 local TEST_NAME="$1"
279 shift 1
280 if "$@" 1>"${TEST_NAME}.stdout" 2>"${TEST_NAME}.stderr"; then
281 ok "${TEST_NAME}"
282 return
283 fi
284 fail "${TEST_NAME}"
285 WORKFLOW_RUN_FAILS=true
286 WORKFLOW_LOG_DIR="${WORKFLOW_RUN_DIR}/log/scheduler"
287 mkdir -p "${WORKFLOW_LOG_DIR}" # directory may not exist if run fails very early
288 dump_std "${TEST_NAME}"
289 }
290
291 workflow_run_fail() {
292 local TEST_NAME="$1"
293 shift 1
294 if ! "$@" 1>"${TEST_NAME}.stdout" 2>"${TEST_NAME}.stderr"; then
295 ok "${TEST_NAME}"
296 return
297 fi
298 fail "${TEST_NAME}"
299 WORKFLOW_RUN_FAILS=true
300 WORKFLOW_LOG_DIR="${WORKFLOW_RUN_DIR}/log/scheduler"
301 mkdir -p "${WORKFLOW_LOG_DIR}" # directory may not exist if run fails very early
302 dump_std "${TEST_NAME}"
303 }
304
305 cmp_ok() {
306 local FILE_TEST="$1"
307 local FILE_CONTROL="${2:--}"
308 local TEST_NAME
309 TEST_NAME="$(basename "${FILE_TEST}")-cmp-ok"
310 local DIFF_CMD=${CYLC_TEST_DIFF_CMD:-'diff -u'}
311 if ${DIFF_CMD} "${FILE_CONTROL}" "${FILE_TEST}" >"${TEST_NAME}.stderr" 2>&1
312 then
313 ok "${TEST_NAME}"
314 return
315 else
316 cat "${TEST_NAME}.stderr" >&2
317 fi
318 dump_std "$TEST_NAME"
319 fail "${TEST_NAME}"
320 }
321
322 cmp_ok_re() {
323 local FILE_TEST="$1"
324 local FILE_CONTROL="${2:--}"
325 local TEST_NAME
326 TEST_NAME="$(basename "${FILE_TEST}")-cmp-ok"
327 if python3 -c '
328 import re
329 import sys
330
331 def compare(test_file, control_file):
332 for ind, (test_line, control_line) in enumerate(
333 zip(test_file, control_file)
334 ):
335 if test_line is None:
336 sys.stderr.write("Test file is missing lines")
337 sys.exit(1)
338 if control_line is None:
339 sys.stderr.write("Test file has extra lines")
340 sys.exit(1)
341 if not re.match(control_line, test_line):
342 sys.stderr.write(
343 f"Line {ind+1}: \"{test_line}\" does not match /{control_line}/"
344 .replace("\n", "")
345 )
346 sys.exit(1)
347
348 test_filename, control_filename = sys.argv[1:3]
349 with open(test_filename, "r") as test_file:
350 if control_filename == "-":
351 compare(test_file, sys.stdin)
352 else:
353 with open(control_filename, "r") as control_file:
354 compare(test_file, control_file)
355 ' "${FILE_TEST}" "${FILE_CONTROL}"; then
356 ok "${TEST_NAME}"
357 else
358 (
359 echo -e '\n# Test file:'
360 cat "${FILE_TEST}"
361 ) >&2
362 fail "${TEST_NAME}"
363 fi
364 }
365
366 cmp_json() {
367 local TEST_NAME="$1"
368 local FILE_TEST="$2"
369 local FILE_CONTROL="${3:--}"
370
371 if python3 "$CYLC_REPO_DIR/tests/f/lib/python/diffr.py" \
372 "$(cat "$FILE_CONTROL")" "$(cat "$FILE_TEST")" \
373 >"${TEST_NAME}.stdout" 2>"${TEST_NAME}.stderr"
374 then
375 ok "${TEST_NAME}"
376 return
377 else
378 cat "${TEST_NAME}.stderr" >&2
379 fi
380 dump_std "$TEST_NAME"
381 fail "${TEST_NAME}"
382 }
383
384 contains_ok() {
385 local FILE_TEST="$1"
386 local FILE_CONTROL="${2:--}"
387 local TEST_NAME
388 TEST_NAME="$(basename "${FILE_TEST}")-contains-ok"
389 LANG=C comm -13 <(LANG=C sort "${FILE_TEST}") <(LANG=C sort "${FILE_CONTROL}") \
390 1>"${TEST_NAME}.stdout" 2>"${TEST_NAME}.stderr"
391 if [[ -s "${TEST_NAME}.stdout" ]]; then
392 mkdir -p "${TEST_LOG_DIR}"
393 {
394 echo 'Missing lines:'
395 cat "${TEST_NAME}.stdout"
396 echo 'Test File:'
397 cat "${FILE_TEST}"
398 } >>"${TEST_NAME}.stderr"
399 cp -p "${TEST_NAME}.stderr" "${TEST_LOG_DIR}/${TEST_NAME}.stderr"
400 fail "${TEST_NAME}"
401 return
402 fi
403 ok "${TEST_NAME}"
404 }
405
406 count_ok() {
407 local BRE="$1"
408 local FILE="$2"
409 local COUNT="$3"
410 local TEST_NAME NEW_COUNT
411 TEST_NAME="$(basename "${FILE}")-count-ok"
412 NEW_COUNT=$(grep -c "${BRE}" "${FILE}")
413 if (( NEW_COUNT == COUNT )); then
414 ok "${TEST_NAME}"
415 return
416 fi
417 mkdir -p "${TEST_LOG_DIR}"
418 echo "Found ${NEW_COUNT} (not ${COUNT}) of ${BRE} in ${FILE}" \
419 >"${TEST_LOG_DIR}/${TEST_NAME}.stderr"
420 fail "${TEST_NAME}"
421 }
422
423 grep_ok() {
424 # (Put extra grep options like '-E' at end of the command line)
425 local BRE="$1"
426 local FILE="$2"
427 shift 2
428 OPTS="$*"
429 named_grep_ok "$(basename "${FILE}")-grep-ok" "$BRE" "$FILE" "$OPTS"
430 }
431
432 grep_workflow_log_ok() {
433 local TEST_NAME="$1"
434 local PATTERN="$2"
435 shift 2
436 OPTS="$*"
437 local LOG_FILE="${WORKFLOW_RUN_DIR}/log/scheduler/log"
438 # shellcheck disable=SC2086
439 if grep ${OPTS} -s -e "$PATTERN" "$LOG_FILE"; then
440 ok "${TEST_NAME}"
441 return
442 fi
443 mkdir -p "${TEST_LOG_DIR}"
444 {
445 cat <<__ERR__
446 Can't find:
447 ===========
448 ${PATTERN}
449 ===========
450 in:
451 ===========
452 __ERR__
453 cat "$LOG_FILE"
454 } >"${TEST_LOG_DIR}/${TEST_NAME}.stderr"
455 fail "${TEST_NAME}"
456 }
457
458 named_grep_ok() {
459 # (Put extra grep options like '-E' at end of the command line)
460 local NAME="$1"
461 local BRE="$2"
462 local FILE="$3"
463 shift 3
464 local OPTS="$*"
465 local TEST_NAME
466 TEST_NAME="grep-ok: ${NAME}"
467 # shellcheck disable=SC2086
468 if grep ${OPTS} -q -e "${BRE}" "${FILE}"; then
469 ok "${TEST_NAME}"
470 return
471 fi
472 mkdir -p "${TEST_LOG_DIR}"
473 {
474 cat <<__ERR__
475 Can't find:
476 ===========
477 ${BRE}
478 ===========
479 in:
480 ===========
481 __ERR__
482 cat "${FILE}"
483 } >"${TEST_LOG_DIR}/${TEST_NAME}.stderr"
484 fail "${TEST_NAME}"
485 }
486
487 grep_fail() {
488 local BRE="$1"
489 local FILE="$2"
490 shift 2
491 local OPTS="$*"
492 local TEST_NAME
493 TEST_NAME="$(basename "${FILE}")-grep-fail"
494 if [[ ! -f "${FILE}" ]]; then
495 fail "${TEST_NAME}-file to be grepped does not exist."
496 return
497 fi
498 # shellcheck disable=SC2086
499 if grep ${OPTS} -q -e "${BRE}" "${FILE}"; then
500 mkdir -p "${TEST_LOG_DIR}"
501 echo "ERROR: Found ${BRE} in ${FILE}" \
502 >"${TEST_LOG_DIR}/${TEST_NAME}.stderr"
503 fail "${TEST_NAME}"
504 return
505 fi
506 ok "${TEST_NAME}"
507 }
508
509 exists_ok() {
510 local FILE="$1"
511 local TEST_NAME
512 TEST_NAME="$(basename "${FILE}")-file-exists-ok"
513 if [[ -e $FILE ]]; then
514 ok "${TEST_NAME}"
515 return
516 fi
517 mkdir -p "${TEST_LOG_DIR}"
518 echo "Does not exist: ${FILE}" >"${TEST_LOG_DIR}/${TEST_NAME}.stderr"
519 fail "${TEST_NAME}"
520 }
521
522 exists_fail() {
523 local FILE="$1"
524 local TEST_NAME
525 TEST_NAME="$(basename "${FILE}")-file-exists-fail"
526 if [[ ! -e "${FILE}" ]]; then
527 ok "${TEST_NAME}"
528 return
529 fi
530 mkdir -p "${TEST_LOG_DIR}"
531 echo "Exists: ${FILE}" >"${TEST_LOG_DIR}/${TEST_NAME}.stderr"
532 fail "${TEST_NAME}"
533 }
534
535 graph_workflow() {
536 # Generate a graphviz "plain" format graph of a workflow.
537 local WORKFLOW_NAME="${1}"
538 local OUTPUT_FILE="${2}"
539 shift 2
540 mkdir -p "$(dirname "${OUTPUT_FILE}")"
541 cylc graph --reference "${WORKFLOW_NAME}" "$@" \
542 >"${OUTPUT_FILE}" 2>"${OUTPUT_FILE}".err
543 }
544
545 workflow_id() {
546 local TEST_NAME="${1:-${TEST_NAME_BASE}}"
547 local TEST_SOURCE_BASE="${2:-${TEST_NAME}}"
548 echo "${CYLC_TEST_REG_BASE}/${TEST_SOURCE_DIR_BASE}/${TEST_NAME}"
549 }
550
551 init_workflow() {
552 local TEST_NAME="$1"
553 local FLOW_CONFIG="${2:--}"
554 local RUN_NUMS="${3:-false}"
555 WORKFLOW_NAME="$(workflow_id "${TEST_NAME}")"
556 WORKFLOW_RUN_DIR="${RUN_DIR}/${WORKFLOW_NAME}"
557 mkdir -p "${TEST_DIR}/${WORKFLOW_NAME}/"
558 cat "${FLOW_CONFIG}" >"${TEST_DIR}/${WORKFLOW_NAME}/flow.cylc"
559 cd "${TEST_DIR}/${WORKFLOW_NAME}"
560 if "${RUN_NUMS}"; then
561 cylc install "${TEST_DIR}/${WORKFLOW_NAME}" --workflow-name="${WORKFLOW_NAME}"
562 else
563 cylc install "${TEST_DIR}/${WORKFLOW_NAME}" --no-run-name --workflow-name="${WORKFLOW_NAME}"
564 fi
565 }
566
567 install_workflow() {
568 local TEST_NAME="${1:-${TEST_NAME_BASE}}"
569 local TEST_SOURCE_BASE="${2:-${TEST_NAME}}"
570 local RUN_NUMS="${3:-false}"
571 WORKFLOW_NAME="$(workflow_id "${TEST_NAME}" "${TEST_SOURCE_BASE}")"
572 WORKFLOW_RUN_DIR="${RUN_DIR}/${WORKFLOW_NAME}"
573 mkdir -p "${TEST_DIR}/${WORKFLOW_NAME}/" # make source dir
574 cp -r "${TEST_SOURCE_DIR}/${TEST_SOURCE_BASE}/"* "${TEST_DIR}/${WORKFLOW_NAME}/"
575 cd "${TEST_DIR}/${WORKFLOW_NAME}"
576 if "$RUN_NUMS"; then
577 cylc install "${TEST_DIR}/${WORKFLOW_NAME}" --workflow-name="${WORKFLOW_NAME}"
578 else
579 cylc install "${TEST_DIR}/${WORKFLOW_NAME}" --no-run-name --workflow-name="${WORKFLOW_NAME}"
580 fi
581 }
582
583 log_scan () {
584 local TEST_NAME="$1"
585 local FILE="$2"
586 local REPS=$3
587 local DELAY=$4
588 local OPTS=${LOG_SCAN_GREP_OPTS:-}
589 if ${CYLC_TEST_DEBUG:-false}; then
590 local ERR=2
591 else
592 local ERR=1
593 fi
594 shift 4
595 local count=0
596 local success=false
597 local position=0
598 local newposition=
599 for pattern in "$@"; do
600 count=$(( count + 1 ))
601 success=false
602 echo -n "scanning for '${pattern:0:30}'" >& $ERR
603 for _ in $(seq 1 "${REPS}"); do
604 echo -n '.' >& $ERR
605 # shellcheck disable=SC2086
606 newposition=$(grep -n $OPTS "$pattern" "$FILE" | \
607 tail -n 1 | cut -d ':' -f 1)
608 if (( newposition > position )); then
609 position=$newposition
610 echo ': succeeded' >& $ERR
611 ok "${TEST_NAME}-${count}"
612 success=true
613 break
614 fi
615 sleep "${DELAY}"
616 done
617 shift
618 if ! "${success}"; then
619 echo ': failed' >& $ERR
620 fail "${TEST_NAME}-${count}"
621 if "${CYLC_TEST_DEBUG:-false}"; then
622 cat "${FILE}" >&2
623 fi
624 mkdir -p "${TEST_LOG_DIR}"
625 ERR_FILE="${TEST_LOG_DIR}/${TEST_NAME}.stderr"
626 echo "ERR_FILE=$ERR_FILE" >&2
627 echo "Could not find: ${pattern}" > "${ERR_FILE}"
628 echo -e '\n=======================\n' >> "${ERR_FILE}"
629 cat "${FILE}" >> "${ERR_FILE}"
630 skip "$#"
631 return 1
632 fi
633 done
634 }
635
636 make_rnd_workflow() {
637 # Create a randomly-named workflow source directory.
638 # Define its run directory.
639 # TODO: export a WORKFLOW_NAME variable so tests can get cylc-install
640 # to install under the regular test hierarchy
641 RND_WORKFLOW_NAME="cylctb-x$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c6)"
642 RND_WORKFLOW_SOURCE="$PWD/${RND_WORKFLOW_NAME}"
643 mkdir -p "${RND_WORKFLOW_SOURCE}"
644 touch "${RND_WORKFLOW_SOURCE}/flow.cylc"
645 RND_WORKFLOW_RUNDIR="${RUN_DIR}/${RND_WORKFLOW_NAME}"
646 export RND_WORKFLOW_NAME
647 export RND_WORKFLOW_SOURCE
648 export RND_WORKFLOW_RUNDIR
649 }
650
651 # shellcheck disable=2120
652 # these arguments are intentionally optional
653 purge () {
654 # workflow to remove - defaults to $WORKFLOW_NAME
655 local WORKFLOW_NAME="${1:-$WORKFLOW_NAME}"
656 # job platform to remove it on - defaults to $CYLC_TEST_PLATFORM
657 local PLATFORM="${2:-$CYLC_TEST_PLATFORM}"
658
659 if ((FAILURES != 0)); then
660 # if tests have failed then don't clean up
661 return 0
662 fi
663
664 if [[ -z $WORKFLOW_NAME ]]; then
665 echo 'no flow to purge' >&2
666 return 1
667 fi
668
669 local WORKFLOW_RUN_DIR="${RUN_DIR}/${WORKFLOW_NAME}"
670
671 # wait for local processes to let go of their file handles
672 if ${HAS_LSOF}; then
673 # NOTE: lsof can hang, so call with "timeout".
674 # NOTE: lsof can raise warnings with some filesystems so ignore stderr
675 # shellcheck disable=SC2034
676 for try in $(seq 1 5); do
677 if ! grep -q "${WORKFLOW_RUN_DIR}" < <(timeout 5 lsof 2>/dev/null); then
678 break
679 fi
680 sleep 1
681 done
682 fi
683
684 # shellcheck disable=SC2016
685 # shellcheck disable=SC2089
686 local CMD='
687 cd '~/cylc-run' &&
688 rm -rf "'"$WORKFLOW_NAME"'" &&
689 rmdir -p "$(dirname "'"$WORKFLOW_NAME"'")"
690 '
691
692 # remove the workflow run directory on the platform if remote
693 if [[ -n "$PLATFORM" && "${PLATFORM}" != _local* ]]; then
694 local HOST
695 HOST="$(get_host_from_platform "$PLATFORM")"
696 # shellcheck disable=SC2029
697 # we want this to expand client side
698 (ssh "$HOST" "$CMD" 2>/dev/null) || true
699 fi
700
701 # remove the workflow run directory tree locally - if empty
702 # NOTE: do this after remote purge as this removes the test config too
703 bash -c "$CMD" 2>/dev/null || true
704 }
705 purge_rnd_workflow() {
706 if ((FAILURES != 0)); then
707 # if tests have failed then don't clean up
708 return 0
709 fi
710 # Remove the workflow source created by make_rnd_workflow().
711 # And remove its run-directory too.
712 rm -rf "${RND_WORKFLOW_SOURCE}"
713 rm -rf "${RND_WORKFLOW_RUNDIR}"
714 }
715
716 poll() {
717 local TIMEOUT="$(($(date +%s) + 60))" # wait 1 minute
718 while ! "$@"; do
719 sleep 1
720 if (($(date +%s) > TIMEOUT)); then
721 echo "ERROR: poll timed out: $*" >&2
722 exit 1
723 fi
724 done
725 }
726
727 poll_grep() {
728 poll grep -s "$@"
729 }
730
731 poll_grep_workflow_log() {
732 poll grep -s "$@" "${WORKFLOW_RUN_DIR}/log/scheduler/log"
733 }
734
735 poll_pid_done() {
736 local TIMEOUT="$(($(date +%s) + 60))" # wait 1 minute
737 while ps --no-headers "$@" 1>'/dev/null' 2>&1; do
738 sleep 1
739 if (($(date +%s) > TIMEOUT)); then
740 echo "ERROR: poll timed out: ! ps --no-headers $*" >&2
741 exit 1
742 fi
743 done
744 }
745
746 poll_workflow_running() {
747 poll test -e "${WORKFLOW_RUN_DIR}/.service/contact"
748 }
749
750 poll_workflow_stopped() {
751 poll test '!' -e "${WORKFLOW_RUN_DIR}/.service/contact"
752 }
753
754 get_workflow_uuid() {
755 sed -n 's|CYLC_WORKFLOW_UUID=\(.*\)|\1|p' "${WORKFLOW_RUN_DIR}/.service/contact" 2>/dev/null
756 }
757
758 poll_workflow_restart() {
759 TIMEOUT="${1:-10}"
760 STEP=1
761 UUID="${2:-$(get_workflow_uuid)}"
762 TIME=0
763 while true; do
764 if [[ ${TIME} -gt $TIMEOUT ]]; then
765 return 1
766 fi
767 if [[ $(get_workflow_uuid) != "${UUID}" ]]; then
768 return 0
769 else
770 TIME=$(( TIME + STEP ))
771 sleep "${STEP}"
772 fi
773 done
774 }
775
776 skip() {
777 local N_TO_SKIP="$1"
778 shift 1
779 local COUNT=0
780 while ((COUNT++ < N_TO_SKIP)); do
781 echo "ok $((++TEST_NUMBER)) # skip $*"
782 done
783 }
784
785 skip_all() {
786 echo "1..0 # SKIP $*"
787 exit
788 }
789
790 skip_macos_gh_actions() {
791 # https://github.com/cylc/cylc-flow/issues/6276
792 if [[ "${CI:-}" && "$OSTYPE" == "darwin"* ]]; then
793 skip_all "Skipped due to performance issues on GH Actions MacOS runner"
794 fi
795 }
796
797 ssh_install_cylc() {
798 local RHOST="$1"
799 local RHOST_CYLC_DIR=
800 RHOST_CYLC_DIR=$(_ssh_mkdtemp_cylc_dir "${RHOST}")
801 TEST_RHOST_CYLC_DIR="${RHOST}:${RHOST_CYLC_DIR}"
802 rsync -a '--exclude=*.pyc' "${CYLC_REPO_DIR}"/* "${RHOST}:${RHOST_CYLC_DIR}/"
803 # RHOST_CYLC_DIR is a variable of this function
804 # shellcheck disable=SC2029
805 ssh -n -oBatchMode=yes -oConnectTimeout=5 "${RHOST}" \
806 "make -C '${RHOST_CYLC_DIR}' 'version'" 1>'/dev/null' 2>&1
807 }
808
809 _ssh_mkdtemp_cylc_dir() {
810 local RHOST="$1"
811 ssh -oBatchMode=yes -oConnectTimeout=5 "${RHOST}" python3 - <<'__PYTHON__'
812 import os
813 from tempfile import mkdtemp
814 print(mkdtemp(dir=os.path.expanduser("~"), prefix="cylc-"))
815 __PYTHON__
816 }
817
818 mock_smtpd_init() { # Logic borrowed from Rose
819 local SMTPD_PORT=
820 # Try several ports in case any are in use:
821 for SMTPD_PORT in 8025 8125 8225 8325 8425 8525 8625 8725 8825 8925; do
822 local SMTPD_LOG="${TEST_DIR}/smtpd.log"
823 local SMTPD_HOST="localhost:${SMTPD_PORT}"
824 # Set up fake SMTP server to catch outgoing mail & redirect to log:
825 python3 -u -m 'aiosmtpd' \
826 --class aiosmtpd.handlers.Debugging stdout \
827 --debug --nosetuid \
828 --listen "${SMTPD_HOST}" \
829 1>"${SMTPD_LOG}" 2>&1 & # Runs in background
830 local SMTPD_PID="$!"
831 while ! grep -q 'is listening' "${SMTPD_LOG}" 2>'/dev/null'
832 do
833 if ps "${SMTPD_PID}" 1>/dev/null 2>&1; then
834 sleep 1 # Still waiting for fake server to start
835 else # Server process failed
836 rm -f "${SMTPD_LOG}"
837 unset SMTPD_HOST SMTPD_LOG SMTPD_PID
838 break
839 fi
840 done
841 if [[ -n "${SMTPD_PID:-}" ]]; then
842 # Variables used by tests
843 # shellcheck disable=SC2034
844 TEST_SMTPD_HOST="${SMTPD_HOST}"
845 # shellcheck disable=SC2034
846 TEST_SMTPD_LOG="${SMTPD_LOG}"
847 TEST_SMTPD_PID="${SMTPD_PID}"
848 break
849 fi
850 done
851 }
852
853 mock_smtpd_kill() { # Logic borrowed from Rose
854 if [[ -n "${TEST_SMTPD_PID:-}" ]] && ps "${TEST_SMTPD_PID}" >'/dev/null' 2>&1
855 then
856 kill "${TEST_SMTPD_PID}"
857 wait "${TEST_SMTPD_PID}" 2>/dev/null || true
858 unset TEST_SMTPD_HOST TEST_SMTPD_LOG TEST_SMTPD_PID
859 fi
860 }
861
862 install_and_validate() {
863 # First part of the reftest function, to allow separate use.
864 # Expect 1 OK test.
865 local TEST_NAME="${1:-${TEST_NAME_BASE}}"
866 install_workflow "$@"
867 run_ok "${TEST_NAME}-validate" cylc validate "${WORKFLOW_NAME}"
868 }
869
870 reftest_run() {
871 # Guts of the reftest function, to allow separate use.
872 # Expect 1 OK test.
873 local TEST_NAME="${1:-${TEST_NAME_BASE}}-run"
874 if [[ -n "${REFTEST_OPTS:-}" ]]; then
875 # shellcheck disable=SC2086
876 workflow_run_ok "${TEST_NAME}" \
877 cylc play --reference-test --debug --no-detach \
878 ${REFTEST_OPTS} "${WORKFLOW_NAME}"
879 else
880 workflow_run_ok "${TEST_NAME}" \
881 cylc play --reference-test --debug --no-detach \
882 "${WORKFLOW_NAME}"
883 fi
884 }
885
886 reftest() {
887 # Install, validate, run, and purge, a reference test.
888 # Expect 2 OK tests.
889 install_and_validate "$@"
890 reftest_run "$@"
891 # shellcheck disable=SC2119
892 purge
893 }
894
895 _get_test_config_file() {
896 # print the path to the most specific global-tests.cylc on the system
897 # (or print nothing if no test configs are present)
898 python3 -c "
899 from pathlib import Path
900 from cylc.flow.cfgspec.globalcfg import get_version_hierarchy
901 for version in reversed(get_version_hierarchy('$(cylc version)')):
902 path = Path('${HOME}/.cylc/flow', version, 'global-tests.cylc')
903 if path.exists():
904 print(path)
905 break
906 "
907 }
908
909 create_test_global_config() {
910 # (Documented in file header).
911 local PRE=
912 local POST=
913 if (( $# == 1 )); then
914 PRE=$1
915 elif (( $# == 2 )); then
916 PRE=$1
917 POST=$2
918 elif (( $# > 2 )); then
919 echo 'ERROR, create_test_global_config: too many args' >&2
920 exit 1
921 fi
922 # Tidy in case of previous use of this function.
923 rm -fr 'etc'
924 mkdir 'etc'
925 # Start with Jinja2 hashbang. Harmless if not needed.
926 echo "#!Jinja2" >'etc/global.cylc'
927 echo "$PRE" >>'etc/global.cylc'
928 # add defaults
929 cat >>'etc/global.cylc' <<__HERE__
930 # set a default timeout for all flow runs to avoid hanging tests
931 [scheduler]
932 [[events]]
933 inactivity timeout = PT5M
934 stall timeout = PT5M
935 abort on inactivity timeout = true
936 abort on workflow timeout = true
937 [install]
938 max depth = 5
939 __HERE__
940 # add global-tests.cylc
941 USER_TESTS_CONF_FILE="$(_get_test_config_file)"
942 if [[ -n "${USER_TESTS_CONF_FILE}" ]]; then
943 cat "${USER_TESTS_CONF_FILE}" >>'etc/global.cylc'
944 fi
945 # add platform config
946 if [[ -n "${CYLC_TEST_PLATFORM:-}" ]]; then
947 _add_platform_to_test_global_conf "$CYLC_TEST_PLATFORM"
948 fi
949 echo "$POST" >>'etc/global.cylc'
950 export CYLC_CONF_PATH="${PWD}/etc"
951 }
952
953 _parse_platform_spec () {
954 # the requirement string: e.g. comms:tcp
955 local requirement="$1"
956
957 local locality='local'
958 local job_runner='*'
959 local file_system='*'
960 local comms='*'
961 for req in ${requirement}; do
962 case "$req" in
963 loc:*)
964 locality="${req/loc:/}"
965 ;;
966 runner:*)
967 job_runner="${req/runner:/}"
968 ;;
969 fs:*)
970 file_system="${req/fs:/}"
971 ;;
972 comms:*)
973 comms="${req/comms:/}"
974 ;;
975 *)
976 echo "Invalid option $req" >&2
977 return 1
978 ;;
979 esac
980 done
981 echo "_${locality}_${job_runner}_${file_system}_${comms}"
982 }
983
984 _get_test_platform () {
985 # the requirement string: e.g. comms:tcp
986 local requirement="$1"
987
988 local platform
989 local required_platform
990 required_platform="$(_parse_platform_spec "${requirement}")"
991
992 for platform in "${PLATFORMS[@]}"; do
993 # we are purposefully allowing pattern matching here
994 # shellcheck disable=SC2053
995 if [[ "$platform" == $required_platform ]]; then
996 CYLC_TEST_PLATFORM="$platform"
997 CYLC_TEST_JOB_RUNNER="$(cut -d '_' -f 3 <<< "$platform")"
998 export CYLC_TEST_PLATFORM CYLC_TEST_JOB_RUNNER
999 return 0
1000 fi
1001 done
1002 return 1
1003 }
1004
1005 _get_test_host () {
1006 # extract a test host from the test platform
1007 # NOTE: this will cause a failure if the platform is not configured
1008 CYLC_TEST_HOST="$(get_host_from_platform "$CYLC_TEST_PLATFORM")"
1009 if [[ -z "${CYLC_TEST_HOST}" ]]; then
1010 echo "Could not find host for platform $CYLC_TEST_PLATFORM" >&2
1011 exit 1
1012 fi
1013 export CYLC_TEST_HOST
1014 }
1015
1016 _get_test_install_target () {
1017 # extract a test host from the test platform
1018 # NOTE: this will cause a failure if the platform is not configured
1019 CYLC_TEST_INSTALL_TARGET="$(
1020 get_install_target_from_platform "$CYLC_TEST_PLATFORM"
1021 )"
1022 if [[ -z "${CYLC_TEST_INSTALL_TARGET}" ]]; then
1023 CYLC_TEST_INSTALL_TARGET="$CYLC_TEST_PLATFORM"
1024 fi
1025 export CYLC_TEST_INSTALL_TARGET
1026 }
1027
1028 get_host_from_platform () {
1029 local platform="$1"
1030
1031 cylc config -i "[platforms][${platform}]hosts" \
1032 | cut -d ',' -f 1
1033 }
1034
1035 get_install_target_from_platform () {
1036 local platform="$1"
1037
1038 cylc config -d -i "[platforms][${platform}]install target" \
1039 | cut -d ',' -f 1
1040 }
1041
1042 localhost_fqdn () {
1043 python -c "
1044 from cylc.flow.hostuserutil import get_fqdn_by_host
1045 print(get_fqdn_by_host(None))
1046 "
1047 }
1048
1049 get_fqdn () {
1050 local HOST="${1}"
1051 ssh "$HOST" hostname -f
1052 }
1053
1054 _get_test_platforms () {
1055 CYLC_CONF_PATH='' cylc config -i '[platforms]' \
1056 | sed -n 's/\[\[\(_.*\)\]\]/\1/p'
1057 }
1058
1059 _expand_test_platforms () {
1060 # convert sting of space separated platforms to array
1061 read -r -a PLATFORMS <<< "${1}"
1062
1063 local ret=()
1064 local pattern
1065 local platform
1066 local all_platforms
1067
1068 # get list of all configured test platforms
1069 # NOTE: when we drop bash3 convert to
1070 # mapfile -t all_platforms < <(_get_test_platforms)
1071 all_platforms=()
1072 while IFS='' read -r line; do
1073 all_platforms+=("$line");
1074 done < <(_get_test_platforms)
1075
1076 # perform pattern matching on the PLATFORMS variable
1077 if [[ "${#PLATFORMS[@]}" -eq 0 ]]; then
1078 # default to all local platforms
1079 PLATFORMS=('_local_background*' '_local_at*')
1080 fi
1081
1082 # perform pattern matching
1083 for pattern in "${PLATFORMS[@]}"; do
1084 for platform in "${all_platforms[@]}"; do
1085 # shellcheck disable=SC2053
1086 # we are purposefully performing glob matching here
1087 if [[ ${platform} == ${pattern} ]]; then
1088 ret+=("${platform}")
1089 fi
1090 done
1091 done
1092
1093 # set PLATFORMS to the unique results
1094 # NOTE: when we drop bash3 convert to
1095 # mapfile -t PLATFORMS < <(
1096 PLATFORMS=()
1097 while IFS='' read -r line; do
1098 PLATFORMS+=("$line");
1099 done < <(
1100 for platform in "${ret[@]}"; do
1101 echo "${platform}";
1102 done | sort -u
1103 )
1104 export PLATFORMS
1105 }
1106
1107 _check_test_requirements () {
1108 # check the test requirements against the array of test platforms
1109 if [[ -z "${REQUIRE_PLATFORM:-}" ]]; then
1110 # if no requirement string is provided default to local, background
1111 REQUIRE_PLATFORM='loc:local runner:background'
1112 fi
1113 if ! _get_test_platform "${REQUIRE_PLATFORM}"; then
1114 skip_all "requires $REQUIRE_PLATFORM"
1115 fi
1116 _add_platform_to_test_global_conf "$CYLC_TEST_PLATFORM"
1117 }
1118
1119 _add_platform_to_test_global_conf () {
1120 # add a test platform from the global config to the test global config
1121 # NOTE: Uses global.cylc NOT global-tests.cylc
1122 # Do not configure test platforms in the global-tests.cylc file
1123 PLATFORM="$1"
1124 platform_cfg=$(CYLC_CONF_PATH='' cylc config -i "[platforms][$PLATFORM]")
1125 cat >> "${CYLC_CONF_PATH}/global.cylc" <<__HERE__
1126 [platforms]
1127 [[$PLATFORM]]
1128 $platform_cfg
1129 __HERE__
1130 if [[ -n "$platform_cfg" ]]; then
1131 # Ensure install target explicitly defined:
1132 cylc config -i "[platforms][${PLATFORM}]install target" || exit 1
1133 fi
1134 }
1135
1136 delete_db() {
1137 # Delete private/public workflow databases for a cold start
1138 local WORKFLOW_RUN_DIR="${1:-$WORKFLOW_RUN_DIR}"
1139 rm "${WORKFLOW_RUN_DIR}/.service/db" "${WORKFLOW_RUN_DIR}/log/db"
1140 }
1141
1142
1143 # tags
1144
1145 PATH="${CYLC_REPO_DIR}/bin:${PATH}"
1146
1147 RUN_DIR="${HOME}/cylc-run"
1148 export RUN_DIR
1149 TEST_NAME_BASE="$(basename "$0" '.t')"
1150 TEST_SOURCE_DIR="$(cd "$(dirname "$0")" && pwd)"
1151 TEST_DIR="$(mktemp -d)"
1152 cd "${TEST_DIR}"
1153 TEST_SOURCE_DIR_BASE="${TEST_SOURCE_DIR##*tests/}"
1154 # this prevents test runs started at the same time from conflicting
1155 TEST_LOG_DIR_BASE="${TMPDIR:-/tmp}/${USER}/${CYLC_TEST_REG_BASE}"
1156 TEST_LOG_DIR="${TEST_LOG_DIR_BASE}/${TEST_SOURCE_DIR_BASE}/${TEST_NAME_BASE}"
1157 WORKFLOW_RUN_FAILS=false
1158
1159 CYLC_TEST_SKIP="${CYLC_TEST_SKIP:-}"
1160 # Is this test in the skip list?
1161 THIS="${0#./}"
1162 THIS_DIR="$(dirname "${THIS}")"
1163 for SKIP in ${CYLC_TEST_SKIP}; do
1164 RSKIP="${SKIP#./}"
1165 if [[ "${THIS}" == "${RSKIP}" || "${THIS_DIR%/}" == "${RSKIP%/}" ]]; then
1166 # Deliberately print variable substitution syntax unexpanded
1167 # shellcheck disable=SC2016
1168 skip_all 'this test is in $CYLC_TEST_SKIP.'
1169 fi
1170 done
1171
1172 # Ignore the normal site/user global config, use global-tests.cylc.
1173 create_test_global_config "$@"
1174
1175 # get array of test platforms to run tests with
1176 _expand_test_platforms "${PLATFORMS}"
1177
1178 # run or skip tests based on the test requirements and provided test platforms
1179 # provides: CYLC_TEST_PLATFORM
1180 _check_test_requirements
1181
1182 # extract a test host from the test platform
1183 # provides: CYLC_TEST_HOST
1184 _get_test_host
1185
1186 # extract a test install target from the test platform
1187 # provides: CYLC_TEST_INSTALL_TARGET
1188 _get_test_install_target
1189
1190 if ${CYLC_SHOW_TEST_EXECUTION:-false}; then
1191 # write the name of tests as they are run
1192 # (helpful for debugging hanging tests)
1193 echo "# $TEST_NAME_BASE" >&2
1194 fi
1195
1196 set +x
1197 set +e