#!/usr/bin/env bash
# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
#
# NAME
# test_header
#
# SYNOPSIS
# . $CYLC_REPO_DIR/tests/lib/bash/test_header
#
# DESCRIPTION
# Interface for constructing tests under a TAP harness (the "prove"
# command).
#
# FUNCTIONS
# set_test_number N
# echo a total number of tests for TAP to read.
# ok TEST_NAME
# echo a TAP OK message for TEST_NAME.
# fail TEST_NAME
# echo a TAP fail message for TEST_NAME. If $CYLC_TEST_DEBUG is set,
# cat $TEST_NAME.stderr to stderr.
# run_ok TEST_NAME COMMAND ...
# Run COMMAND with any following options/arguments and store stdout
# and stderr in TEST_NAME.stdout and TEST_NAME.stderr.
# This is expected to have a return code of 0, which ok's the test.
# run_fail TEST_NAME COMMAND ...
# Run COMMAND with any following options/arguments and store stdout
# and stderr in TEST_NAME.stdout and TEST_NAME.stderr.
# This is expected to have a non-zero return code, which ok's the test.
# run_graphql_ok TEST_NAME WORKFLOW JSON
# Send a graphql query to the workflow, extract the result from
# ${TEST_NAME}.stdout
# cmp_ok FILE_TEST FILE_CONTROL
# Compare FILE_TEST with a file or stdin given by FILE_CONTROL (stdin
# if FILE_CONTROL is "-" or missing). By default, it uses "diff -u" to
# compare files. However, if an alternate command such as "xxdiff -D"
# is desirable (e.g. for debugging),
# "export CYLC_TEST_DIFF_CMD=xxdiff -D".
# cmp_ok_re FILE_TEST FILE_CONTROL
# Match a file against a regex file line by line.
# Reads in the test and control files line by line and checks the test
# line re.match'es the control line.
# cmp_json TEST_NAME FILE_TEST [FILE_CONTROL]
# Alternative implementation to cmp_json_ok, compare two JSON files
# and display any differences as a unified(ish) diff:
# * Works with nested dictionaries.
# * Impervious to dictionary order.
# contains_ok FILE_TEST [FILE_CONTROL]
# Make sure that each line in FILE_TEST is present in FILE_CONTROL
# (stdin if FILE_CONTROL is "-" or missing).
# grep_ok PATTERN FILE [$OPTS]
# Run "grep [$OPTS] -q -e PATTERN FILE".
# grep_workflow_log_ok TEST_NAME PATTERN [$OPTS]
# Run "grep [$OPTS] -s -e PATTERN ".
# named_grep_ok NAME PATTERN FILE [$OPTS]
# Run grep_ok with a custom test name.
# OPTS: put grep options like '-E' (extended regex) at end of line.
# grep_fail PATTERN FILE [$OPTS]
# Run "grep [$OPTS] -q -e PATTERN FILE", expect no match.
# count_ok PATTERN FILE COUNT
# Test that PATTERN occurs in exactly COUNT lines of FILE.
# exists_ok FILE
# Test that FILE exists
# exists_fail FILE
# Test that FILE does not exist
# init_workflow TEST_NAME [SOURCE] [[RUN_NUMS]]
# Create a workflow from SOURCE's "flow.cylc" called:
# "cylctb-${CYLC_TEST_TIME_INIT}/${TEST_SOURCE_DIR##*tests/}/${TEST_NAME}"
# Provides WORKFLOW_NAME and WORKFLOW_RUN_DIR variables.
# RUN_NUMS (defaults to false): If false run cylc install --no-rundir
# install_workflow TEST_NAME SOURCE [RUN_NUMS]
# Same as init_workflow, but SOURCE must be a directory containing a
# "flow.cylc" file.
# log_scan TEST_NAME FILE ATTEMPTS DELAY PATTERN...
# Monitor FILE polling every DELAY seconds for maximum of ATTEMPTS
# tries grepping for each PATTERN in turn. Tests will only pass if the
# PATTERNs appear in FILE in the correct order. Runs one test per
# pattern, each prefixed by TEST_NAME.
# set LOG_SCAN_GREP_OPTS in the environment, e.g. "-E" for "grep -E"
# make_rnd_workflow
# Create a randomly-named workflow source directory.
# mock_smtpd_init
# Start a mock SMTP server daemon for testing. Write host:port setting
# to the variable TEST_SMTPD_HOST. Write pid of daemon to
# TEST_SMTPD_PID. Write log to TEST_SMTPD_LOG.
# mock_smtpd_kill
# Kill the mock SMTP server daemon process.
# poll COMMAND
# Run COMMAND in 1 second intervals for a minute until COMMAND returns
# a zero value, or exit 1 (abort test) with a timeout message.
# poll_grep ...
# Shorthand for 'poll grep -s ...'.
# poll_grep_workflow_log ...
# Shorthand for 'poll_grep ... "${WORKFLOW_RUN_DIR}/log/scheduler/log"'
# poll_pid_done PID
# Poll until process with PID is done.
# poll_workflow_running
# Shorthand for 'poll test -e "${WORKFLOW_RUN_DIR}/.service/contact"'
# poll_workflow_stopped
# Shorthand for 'poll "!" test -e "${WORKFLOW_RUN_DIR}/.service/contact"'.
# Note that you might want to call poll_workflow_running before this as
# it may return before contact file is created in the first place.
# get_workflow_uuid
# Extract the UUID from the contact file or echo '' if not present.
# poll_workflow_restart [TIMEOUT [UUID]]
# Wait for the workflow to restart.
# Provide the UUID from the current run if workflow is not
# guaranteed to be running.
# purge [WORKFLOW_NAME [PLATFORM]]
# Remove a workflow from the filesystem.
# Defaults to removing $WORKFLOW_NAME on $CYLC_TEST_PLATFORM
# purge_rnd_workflow
# Remove a workflow source dir and rundir.
# Typically used with make_rnd_workflow
# skip N SKIP_REASON
# echo "ok $((++T)) # skip SKIP_REASON" N times.
# skip_all SKIP_REASON
# echo "1..0 # SKIP SKIP_REASON" and exit.
# ssh_install_cylc HOST
# Install cylc on a remote host.
# reftest [TEST_NAME [SOURCE]]
# Install a reference workflow using `install_workflow`, run a validation
# test on the workflow and run the reference workflow with `workflow_run_ok`.
# Expect 2 OK tests.
# install_and_validate
# The first part of reftest, to allow separate use.
# Expect 1 OK test.
# reftest_run
# The guts of reftest, to allow separate use.
# Expect 1 OK test.
# create_test_global_config [PRE [POST]]
# Create a new global config file $PWD/etc from global-tests.cylc
# with PRE and POST pre- and ap-pended.
# PRE and POST are strings.
# The global config starts with #!Jinja2 in case Jinja2 is used.
# localhost_fqdn
# Get the FQDN of the current host using the same mechanism Cylc uses.
# get_fqdn [TARGET]
# SSH to TARGET and return `hostname -f`.
# dump_std TEST_NAME
# Dump stdout and stderr of TEST_NAME to the test log dir.
# delete_db [WORKFLOW_RUN_DIR]
# Delete private/public workflow databases for a cold start.
#
# VARIABLES
# LOG_INDENT
# indentation width of secondary log lines
# # TODO: document other variables used by test scripts
#
#-------------------------------------------------------------------------------
set -eu
shopt -s extglob
# For indented secondary log lines; must match the width in cylc.flow.loggingutil
export LOG_INDENT=" "
FAILURES=0
SIGNALS="EXIT INT"
TEST_DIR=
TEST_RHOST_CYLC_DIR=
FINALLY() {
for S in ${SIGNALS}; do
trap '' "${S}"
done
if [[ -n "${TEST_DIR}" ]]; then
cd ~
rm -rf "${TEST_DIR}"
fi
if [[ -n "${TEST_RHOST_CYLC_DIR}" ]]; then
# TEST_RHOST_CYLC_DIR is a local variable of this script
# shellcheck disable=SC2029
ssh -oBatchMode=yes -oConnectTimeout=5 "${TEST_RHOST_CYLC_DIR%%:*}" \
"rm -fr ${TEST_RHOST_CYLC_DIR#*:}"
fi
if [[ -n "${TEST_SMTPD_PID:-}" ]]; then
kill "${TEST_SMTPD_PID}"
fi
if ((FAILURES > 0)); then
echo -e "\n stdout and stderr stored in: ${TEST_LOG_DIR}" >&2
if "${WORKFLOW_RUN_FAILS}"; then
echo -e " workflow logs can be found under: ${WORKFLOW_LOG_DIR}/" >&2
fi
fi
}
for S in ${SIGNALS}; do
# Signal as argument to FINALLY
# shellcheck disable=SC2064
trap "FINALLY ${S}" "${S}"
done
TEST_NUMBER=0
if command -v lsof >/dev/null; then
HAS_LSOF=true
else
HAS_LSOF=false
fi
set_test_number() {
echo "1..$1"
}
ok() {
echo "ok $((++TEST_NUMBER)) - $*"
}
fail() {
((++FAILURES))
echo "not ok $((++TEST_NUMBER)) - $*"
if [[ -n "${CYLC_TEST_DEBUG:-}" ]]; then
echo >'/dev/tty'
echo "${TEST_NAME_BASE} ${TEST_NAME}" >'/dev/tty'
cat "${TEST_NAME}.stderr" >'/dev/tty'
fi
}
dump_std() {
local TEST_NAME="$1"
mkdir -p "${TEST_LOG_DIR}" # directory may not exist if run fails very early
if [[ -s "${TEST_NAME}.stdout" ]]; then
cp -p "${TEST_NAME}.stdout" "${TEST_LOG_DIR}/${TEST_NAME}.stdout"
fi
cp -p "${TEST_NAME}.stderr" "${TEST_LOG_DIR}/${TEST_NAME}.stderr"
}
run_ok() {
local TEST_NAME="$1"
shift 1
if ! "$@" 1>"${TEST_NAME}.stdout" 2>"${TEST_NAME}.stderr"; then
fail "${TEST_NAME}"
dump_std "${TEST_NAME}"
return
fi
ok "${TEST_NAME}"
}
run_fail() {
local TEST_NAME="$1"
shift 1
if "$@" 1>"${TEST_NAME}.stdout" 2>"${TEST_NAME}.stderr"; then
fail "${TEST_NAME}"
dump_std "${TEST_NAME}"
return
fi
ok "${TEST_NAME}"
}
run_graphql_ok () {
# this is a thin wrapper to `cylc client` which strips newlines from $2 to
# make it valid JSON
local TEST_NAME="$1"
local WORKFLOW="$2"
local JSON="$3"
# shellcheck disable=SC2086
run_ok "${TEST_NAME}" cylc client \
"${WORKFLOW}" \
'graphql' \
< <(echo ${JSON})
}
workflow_run_ok() {
local TEST_NAME="$1"
shift 1
if "$@" 1>"${TEST_NAME}.stdout" 2>"${TEST_NAME}.stderr"; then
ok "${TEST_NAME}"
return
fi
fail "${TEST_NAME}"
WORKFLOW_RUN_FAILS=true
WORKFLOW_LOG_DIR="${WORKFLOW_RUN_DIR}/log/scheduler"
mkdir -p "${WORKFLOW_LOG_DIR}" # directory may not exist if run fails very early
dump_std "${TEST_NAME}"
}
workflow_run_fail() {
local TEST_NAME="$1"
shift 1
if ! "$@" 1>"${TEST_NAME}.stdout" 2>"${TEST_NAME}.stderr"; then
ok "${TEST_NAME}"
return
fi
fail "${TEST_NAME}"
WORKFLOW_RUN_FAILS=true
WORKFLOW_LOG_DIR="${WORKFLOW_RUN_DIR}/log/scheduler"
mkdir -p "${WORKFLOW_LOG_DIR}" # directory may not exist if run fails very early
dump_std "${TEST_NAME}"
}
cmp_ok() {
local FILE_TEST="$1"
local FILE_CONTROL="${2:--}"
local TEST_NAME
TEST_NAME="$(basename "${FILE_TEST}")-cmp-ok"
local DIFF_CMD=${CYLC_TEST_DIFF_CMD:-'diff -u'}
if ${DIFF_CMD} "${FILE_CONTROL}" "${FILE_TEST}" >"${TEST_NAME}.stderr" 2>&1
then
ok "${TEST_NAME}"
return
else
cat "${TEST_NAME}.stderr" >&2
fi
dump_std "$TEST_NAME"
fail "${TEST_NAME}"
}
cmp_ok_re() {
local FILE_TEST="$1"
local FILE_CONTROL="${2:--}"
local TEST_NAME
TEST_NAME="$(basename "${FILE_TEST}")-cmp-ok"
if python3 -c '
import re
import sys
def compare(test_file, control_file):
for ind, (test_line, control_line) in enumerate(
zip(test_file, control_file)
):
if test_line is None:
sys.stderr.write("Test file is missing lines")
sys.exit(1)
if control_line is None:
sys.stderr.write("Test file has extra lines")
sys.exit(1)
if not re.match(control_line, test_line):
sys.stderr.write(
f"Line {ind+1}: \"{test_line}\" does not match /{control_line}/"
.replace("\n", "")
)
sys.exit(1)
test_filename, control_filename = sys.argv[1:3]
with open(test_filename, "r") as test_file:
if control_filename == "-":
compare(test_file, sys.stdin)
else:
with open(control_filename, "r") as control_file:
compare(test_file, control_file)
' "${FILE_TEST}" "${FILE_CONTROL}"; then
ok "${TEST_NAME}"
else
(
echo -e '\n# Test file:'
cat "${FILE_TEST}"
) >&2
fail "${TEST_NAME}"
fi
}
cmp_json() {
local TEST_NAME="$1"
local FILE_TEST="$2"
local FILE_CONTROL="${3:--}"
if python3 "$CYLC_REPO_DIR/tests/f/lib/python/diffr.py" \
"$(cat "$FILE_CONTROL")" "$(cat "$FILE_TEST")" \
>"${TEST_NAME}.stdout" 2>"${TEST_NAME}.stderr"
then
ok "${TEST_NAME}"
return
else
cat "${TEST_NAME}.stderr" >&2
fi
dump_std "$TEST_NAME"
fail "${TEST_NAME}"
}
contains_ok() {
local FILE_TEST="$1"
local FILE_CONTROL="${2:--}"
local TEST_NAME
TEST_NAME="$(basename "${FILE_TEST}")-contains-ok"
LANG=C comm -13 <(LANG=C sort "${FILE_TEST}") <(LANG=C sort "${FILE_CONTROL}") \
1>"${TEST_NAME}.stdout" 2>"${TEST_NAME}.stderr"
if [[ -s "${TEST_NAME}.stdout" ]]; then
mkdir -p "${TEST_LOG_DIR}"
{
echo 'Missing lines:'
cat "${TEST_NAME}.stdout"
echo 'Test File:'
cat "${FILE_TEST}"
} >>"${TEST_NAME}.stderr"
cp -p "${TEST_NAME}.stderr" "${TEST_LOG_DIR}/${TEST_NAME}.stderr"
fail "${TEST_NAME}"
return
fi
ok "${TEST_NAME}"
}
count_ok() {
local BRE="$1"
local FILE="$2"
local COUNT="$3"
local TEST_NAME NEW_COUNT
TEST_NAME="$(basename "${FILE}")-count-ok"
NEW_COUNT=$(grep -c "${BRE}" "${FILE}")
if (( NEW_COUNT == COUNT )); then
ok "${TEST_NAME}"
return
fi
mkdir -p "${TEST_LOG_DIR}"
echo "Found ${NEW_COUNT} (not ${COUNT}) of ${BRE} in ${FILE}" \
>"${TEST_LOG_DIR}/${TEST_NAME}.stderr"
fail "${TEST_NAME}"
}
grep_ok() {
# (Put extra grep options like '-E' at end of the command line)
local BRE="$1"
local FILE="$2"
shift 2
OPTS="$*"
named_grep_ok "$(basename "${FILE}")-grep-ok" "$BRE" "$FILE" "$OPTS"
}
grep_workflow_log_ok() {
local TEST_NAME="$1"
local PATTERN="$2"
shift 2
OPTS="$*"
local LOG_FILE="${WORKFLOW_RUN_DIR}/log/scheduler/log"
# shellcheck disable=SC2086
if grep ${OPTS} -s -e "$PATTERN" "$LOG_FILE"; then
ok "${TEST_NAME}"
return
fi
mkdir -p "${TEST_LOG_DIR}"
{
cat <<__ERR__
Can't find:
===========
${PATTERN}
===========
in:
===========
__ERR__
cat "$LOG_FILE"
} >"${TEST_LOG_DIR}/${TEST_NAME}.stderr"
fail "${TEST_NAME}"
}
named_grep_ok() {
# (Put extra grep options like '-E' at end of the command line)
local NAME="$1"
local BRE="$2"
local FILE="$3"
shift 3
local OPTS="$*"
local TEST_NAME
TEST_NAME="grep-ok: ${NAME}"
# shellcheck disable=SC2086
if grep ${OPTS} -q -e "${BRE}" "${FILE}"; then
ok "${TEST_NAME}"
return
fi
mkdir -p "${TEST_LOG_DIR}"
{
cat <<__ERR__
Can't find:
===========
${BRE}
===========
in:
===========
__ERR__
cat "${FILE}"
} >"${TEST_LOG_DIR}/${TEST_NAME}.stderr"
fail "${TEST_NAME}"
}
grep_fail() {
local BRE="$1"
local FILE="$2"
shift 2
local OPTS="$*"
local TEST_NAME
TEST_NAME="$(basename "${FILE}")-grep-fail"
if [[ ! -f "${FILE}" ]]; then
fail "${TEST_NAME}-file to be grepped does not exist."
return
fi
# shellcheck disable=SC2086
if grep ${OPTS} -q -e "${BRE}" "${FILE}"; then
mkdir -p "${TEST_LOG_DIR}"
echo "ERROR: Found ${BRE} in ${FILE}" \
>"${TEST_LOG_DIR}/${TEST_NAME}.stderr"
fail "${TEST_NAME}"
return
fi
ok "${TEST_NAME}"
}
exists_ok() {
local FILE="$1"
local TEST_NAME
TEST_NAME="$(basename "${FILE}")-file-exists-ok"
if [[ -e $FILE ]]; then
ok "${TEST_NAME}"
return
fi
mkdir -p "${TEST_LOG_DIR}"
echo "Does not exist: ${FILE}" >"${TEST_LOG_DIR}/${TEST_NAME}.stderr"
fail "${TEST_NAME}"
}
exists_fail() {
local FILE="$1"
local TEST_NAME
TEST_NAME="$(basename "${FILE}")-file-exists-fail"
if [[ ! -e "${FILE}" ]]; then
ok "${TEST_NAME}"
return
fi
mkdir -p "${TEST_LOG_DIR}"
echo "Exists: ${FILE}" >"${TEST_LOG_DIR}/${TEST_NAME}.stderr"
fail "${TEST_NAME}"
}
graph_workflow() {
# Generate a graphviz "plain" format graph of a workflow.
local WORKFLOW_NAME="${1}"
local OUTPUT_FILE="${2}"
shift 2
mkdir -p "$(dirname "${OUTPUT_FILE}")"
cylc graph --reference "${WORKFLOW_NAME}" "$@" \
>"${OUTPUT_FILE}" 2>"${OUTPUT_FILE}".err
}
workflow_id() {
local TEST_NAME="${1:-${TEST_NAME_BASE}}"
local TEST_SOURCE_BASE="${2:-${TEST_NAME}}"
echo "${CYLC_TEST_REG_BASE}/${TEST_SOURCE_DIR_BASE}/${TEST_NAME}"
}
init_workflow() {
local TEST_NAME="$1"
local FLOW_CONFIG="${2:--}"
local RUN_NUMS="${3:-false}"
WORKFLOW_NAME="$(workflow_id "${TEST_NAME}")"
WORKFLOW_RUN_DIR="${RUN_DIR}/${WORKFLOW_NAME}"
mkdir -p "${TEST_DIR}/${WORKFLOW_NAME}/"
cat "${FLOW_CONFIG}" >"${TEST_DIR}/${WORKFLOW_NAME}/flow.cylc"
cd "${TEST_DIR}/${WORKFLOW_NAME}"
if "${RUN_NUMS}"; then
cylc install "${TEST_DIR}/${WORKFLOW_NAME}" --workflow-name="${WORKFLOW_NAME}"
else
cylc install "${TEST_DIR}/${WORKFLOW_NAME}" --no-run-name --workflow-name="${WORKFLOW_NAME}"
fi
}
install_workflow() {
local TEST_NAME="${1:-${TEST_NAME_BASE}}"
local TEST_SOURCE_BASE="${2:-${TEST_NAME}}"
local RUN_NUMS="${3:-false}"
WORKFLOW_NAME="$(workflow_id "${TEST_NAME}" "${TEST_SOURCE_BASE}")"
WORKFLOW_RUN_DIR="${RUN_DIR}/${WORKFLOW_NAME}"
mkdir -p "${TEST_DIR}/${WORKFLOW_NAME}/" # make source dir
cp -r "${TEST_SOURCE_DIR}/${TEST_SOURCE_BASE}/"* "${TEST_DIR}/${WORKFLOW_NAME}/"
cd "${TEST_DIR}/${WORKFLOW_NAME}"
if "$RUN_NUMS"; then
cylc install "${TEST_DIR}/${WORKFLOW_NAME}" --workflow-name="${WORKFLOW_NAME}"
else
cylc install "${TEST_DIR}/${WORKFLOW_NAME}" --no-run-name --workflow-name="${WORKFLOW_NAME}"
fi
}
log_scan () {
local TEST_NAME="$1"
local FILE="$2"
local REPS=$3
local DELAY=$4
local OPTS=${LOG_SCAN_GREP_OPTS:-}
if ${CYLC_TEST_DEBUG:-false}; then
local ERR=2
else
local ERR=1
fi
shift 4
local count=0
local success=false
local position=0
local newposition=
for pattern in "$@"; do
count=$(( count + 1 ))
success=false
echo -n "scanning for '${pattern:0:30}'" >& $ERR
for _ in $(seq 1 "${REPS}"); do
echo -n '.' >& $ERR
# shellcheck disable=SC2086
newposition=$(grep -n $OPTS "$pattern" "$FILE" | \
tail -n 1 | cut -d ':' -f 1)
if (( newposition > position )); then
position=$newposition
echo ': succeeded' >& $ERR
ok "${TEST_NAME}-${count}"
success=true
break
fi
sleep "${DELAY}"
done
shift
if ! "${success}"; then
echo ': failed' >& $ERR
fail "${TEST_NAME}-${count}"
if "${CYLC_TEST_DEBUG:-false}"; then
cat "${FILE}" >&2
fi
mkdir -p "${TEST_LOG_DIR}"
ERR_FILE="${TEST_LOG_DIR}/${TEST_NAME}.stderr"
echo "ERR_FILE=$ERR_FILE" >&2
echo "Could not find: ${pattern}" > "${ERR_FILE}"
echo -e '\n=======================\n' >> "${ERR_FILE}"
cat "${FILE}" >> "${ERR_FILE}"
skip "$#"
return 1
fi
done
}
make_rnd_workflow() {
# Create a randomly-named workflow source directory.
# Define its run directory.
# TODO: export a WORKFLOW_NAME variable so tests can get cylc-install
# to install under the regular test hierarchy
RND_WORKFLOW_NAME="cylctb-x$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c6)"
RND_WORKFLOW_SOURCE="$PWD/${RND_WORKFLOW_NAME}"
mkdir -p "${RND_WORKFLOW_SOURCE}"
touch "${RND_WORKFLOW_SOURCE}/flow.cylc"
RND_WORKFLOW_RUNDIR="${RUN_DIR}/${RND_WORKFLOW_NAME}"
export RND_WORKFLOW_NAME
export RND_WORKFLOW_SOURCE
export RND_WORKFLOW_RUNDIR
}
# shellcheck disable=2120
# these arguments are intentionally optional
purge () {
# workflow to remove - defaults to $WORKFLOW_NAME
local WORKFLOW_NAME="${1:-$WORKFLOW_NAME}"
# job platform to remove it on - defaults to $CYLC_TEST_PLATFORM
local PLATFORM="${2:-$CYLC_TEST_PLATFORM}"
if ((FAILURES != 0)); then
# if tests have failed then don't clean up
return 0
fi
if [[ -z $WORKFLOW_NAME ]]; then
echo 'no flow to purge' >&2
return 1
fi
local WORKFLOW_RUN_DIR="${RUN_DIR}/${WORKFLOW_NAME}"
# wait for local processes to let go of their file handles
if ${HAS_LSOF}; then
# NOTE: lsof can hang, so call with "timeout".
# NOTE: lsof can raise warnings with some filesystems so ignore stderr
# shellcheck disable=SC2034
for try in $(seq 1 5); do
if ! grep -q "${WORKFLOW_RUN_DIR}" < <(timeout 5 lsof 2>/dev/null); then
break
fi
sleep 1
done
fi
# shellcheck disable=SC2016
# shellcheck disable=SC2089
local CMD='
cd '~/cylc-run' &&
rm -rf "'"$WORKFLOW_NAME"'" &&
rmdir -p "$(dirname "'"$WORKFLOW_NAME"'")"
'
# remove the workflow run directory on the platform if remote
if [[ -n "$PLATFORM" && "${PLATFORM}" != _local* ]]; then
local HOST
HOST="$(get_host_from_platform "$PLATFORM")"
# shellcheck disable=SC2029
# we want this to expand client side
(ssh "$HOST" "$CMD" 2>/dev/null) || true
fi
# remove the workflow run directory tree locally - if empty
# NOTE: do this after remote purge as this removes the test config too
bash -c "$CMD" 2>/dev/null || true
}
purge_rnd_workflow() {
if ((FAILURES != 0)); then
# if tests have failed then don't clean up
return 0
fi
# Remove the workflow source created by make_rnd_workflow().
# And remove its run-directory too.
rm -rf "${RND_WORKFLOW_SOURCE}"
rm -rf "${RND_WORKFLOW_RUNDIR}"
}
poll() {
local TIMEOUT="$(($(date +%s) + 60))" # wait 1 minute
while ! "$@"; do
sleep 1
if (($(date +%s) > TIMEOUT)); then
echo "ERROR: poll timed out: $*" >&2
exit 1
fi
done
}
poll_grep() {
poll grep -s "$@"
}
poll_grep_workflow_log() {
poll grep -s "$@" "${WORKFLOW_RUN_DIR}/log/scheduler/log"
}
poll_pid_done() {
local TIMEOUT="$(($(date +%s) + 60))" # wait 1 minute
while ps --no-headers "$@" 1>'/dev/null' 2>&1; do
sleep 1
if (($(date +%s) > TIMEOUT)); then
echo "ERROR: poll timed out: ! ps --no-headers $*" >&2
exit 1
fi
done
}
poll_workflow_running() {
poll test -e "${WORKFLOW_RUN_DIR}/.service/contact"
}
poll_workflow_stopped() {
poll test '!' -e "${WORKFLOW_RUN_DIR}/.service/contact"
}
get_workflow_uuid() {
sed -n 's|CYLC_WORKFLOW_UUID=\(.*\)|\1|p' "${WORKFLOW_RUN_DIR}/.service/contact" 2>/dev/null
}
poll_workflow_restart() {
TIMEOUT="${1:-10}"
STEP=1
UUID="${2:-$(get_workflow_uuid)}"
TIME=0
while true; do
if [[ ${TIME} -gt $TIMEOUT ]]; then
return 1
fi
if [[ $(get_workflow_uuid) != "${UUID}" ]]; then
return 0
else
TIME=$(( TIME + STEP ))
sleep "${STEP}"
fi
done
}
skip() {
local N_TO_SKIP="$1"
shift 1
local COUNT=0
while ((COUNT++ < N_TO_SKIP)); do
echo "ok $((++TEST_NUMBER)) # skip $*"
done
}
skip_all() {
echo "1..0 # SKIP $*"
exit
}
skip_macos_gh_actions() {
# https://github.com/cylc/cylc-flow/issues/6276
if [[ "${CI:-}" && "$OSTYPE" == "darwin"* ]]; then
skip_all "Skipped due to performance issues on GH Actions MacOS runner"
fi
}
ssh_install_cylc() {
local RHOST="$1"
local RHOST_CYLC_DIR=
RHOST_CYLC_DIR=$(_ssh_mkdtemp_cylc_dir "${RHOST}")
TEST_RHOST_CYLC_DIR="${RHOST}:${RHOST_CYLC_DIR}"
rsync -a '--exclude=*.pyc' "${CYLC_REPO_DIR}"/* "${RHOST}:${RHOST_CYLC_DIR}/"
# RHOST_CYLC_DIR is a variable of this function
# shellcheck disable=SC2029
ssh -n -oBatchMode=yes -oConnectTimeout=5 "${RHOST}" \
"make -C '${RHOST_CYLC_DIR}' 'version'" 1>'/dev/null' 2>&1
}
_ssh_mkdtemp_cylc_dir() {
local RHOST="$1"
ssh -oBatchMode=yes -oConnectTimeout=5 "${RHOST}" python3 - <<'__PYTHON__'
import os
from tempfile import mkdtemp
print(mkdtemp(dir=os.path.expanduser("~"), prefix="cylc-"))
__PYTHON__
}
mock_smtpd_init() { # Logic borrowed from Rose
local SMTPD_PORT=
# Try several ports in case any are in use:
for SMTPD_PORT in 8025 8125 8225 8325 8425 8525 8625 8725 8825 8925; do
local SMTPD_LOG="${TEST_DIR}/smtpd.log"
local SMTPD_HOST="localhost:${SMTPD_PORT}"
# Set up fake SMTP server to catch outgoing mail & redirect to log:
python3 -u -m 'aiosmtpd' \
--class aiosmtpd.handlers.Debugging stdout \
--debug --nosetuid \
--listen "${SMTPD_HOST}" \
1>"${SMTPD_LOG}" 2>&1 & # Runs in background
local SMTPD_PID="$!"
while ! grep -q 'is listening' "${SMTPD_LOG}" 2>'/dev/null'
do
if ps "${SMTPD_PID}" 1>/dev/null 2>&1; then
sleep 1 # Still waiting for fake server to start
else # Server process failed
rm -f "${SMTPD_LOG}"
unset SMTPD_HOST SMTPD_LOG SMTPD_PID
break
fi
done
if [[ -n "${SMTPD_PID:-}" ]]; then
# Variables used by tests
# shellcheck disable=SC2034
TEST_SMTPD_HOST="${SMTPD_HOST}"
# shellcheck disable=SC2034
TEST_SMTPD_LOG="${SMTPD_LOG}"
TEST_SMTPD_PID="${SMTPD_PID}"
break
fi
done
}
mock_smtpd_kill() { # Logic borrowed from Rose
if [[ -n "${TEST_SMTPD_PID:-}" ]] && ps "${TEST_SMTPD_PID}" >'/dev/null' 2>&1
then
kill "${TEST_SMTPD_PID}"
wait "${TEST_SMTPD_PID}" 2>/dev/null || true
unset TEST_SMTPD_HOST TEST_SMTPD_LOG TEST_SMTPD_PID
fi
}
install_and_validate() {
# First part of the reftest function, to allow separate use.
# Expect 1 OK test.
local TEST_NAME="${1:-${TEST_NAME_BASE}}"
install_workflow "$@"
run_ok "${TEST_NAME}-validate" cylc validate "${WORKFLOW_NAME}"
}
reftest_run() {
# Guts of the reftest function, to allow separate use.
# Expect 1 OK test.
local TEST_NAME="${1:-${TEST_NAME_BASE}}-run"
if [[ -n "${REFTEST_OPTS:-}" ]]; then
# shellcheck disable=SC2086
workflow_run_ok "${TEST_NAME}" \
cylc play --reference-test --debug --no-detach \
${REFTEST_OPTS} "${WORKFLOW_NAME}"
else
workflow_run_ok "${TEST_NAME}" \
cylc play --reference-test --debug --no-detach \
"${WORKFLOW_NAME}"
fi
}
reftest() {
# Install, validate, run, and purge, a reference test.
# Expect 2 OK tests.
install_and_validate "$@"
reftest_run "$@"
# shellcheck disable=SC2119
purge
}
_get_test_config_file() {
# print the path to the most specific global-tests.cylc on the system
# (or print nothing if no test configs are present)
python3 -c "
from pathlib import Path
from cylc.flow.cfgspec.globalcfg import get_version_hierarchy
for version in reversed(get_version_hierarchy('$(cylc version)')):
path = Path('${HOME}/.cylc/flow', version, 'global-tests.cylc')
if path.exists():
print(path)
break
"
}
create_test_global_config() {
# (Documented in file header).
local PRE=
local POST=
if (( $# == 1 )); then
PRE=$1
elif (( $# == 2 )); then
PRE=$1
POST=$2
elif (( $# > 2 )); then
echo 'ERROR, create_test_global_config: too many args' >&2
exit 1
fi
# Tidy in case of previous use of this function.
rm -fr 'etc'
mkdir 'etc'
# Start with Jinja2 hashbang. Harmless if not needed.
echo "#!Jinja2" >'etc/global.cylc'
echo "$PRE" >>'etc/global.cylc'
# add defaults
cat >>'etc/global.cylc' <<__HERE__
# set a default timeout for all flow runs to avoid hanging tests
[scheduler]
[[events]]
inactivity timeout = PT5M
stall timeout = PT5M
abort on inactivity timeout = true
abort on workflow timeout = true
[install]
max depth = 5
__HERE__
# add global-tests.cylc
USER_TESTS_CONF_FILE="$(_get_test_config_file)"
if [[ -n "${USER_TESTS_CONF_FILE}" ]]; then
cat "${USER_TESTS_CONF_FILE}" >>'etc/global.cylc'
fi
# add platform config
if [[ -n "${CYLC_TEST_PLATFORM:-}" ]]; then
_add_platform_to_test_global_conf "$CYLC_TEST_PLATFORM"
fi
echo "$POST" >>'etc/global.cylc'
export CYLC_CONF_PATH="${PWD}/etc"
}
_parse_platform_spec () {
# the requirement string: e.g. comms:tcp
local requirement="$1"
local locality='local'
local job_runner='*'
local file_system='*'
local comms='*'
for req in ${requirement}; do
case "$req" in
loc:*)
locality="${req/loc:/}"
;;
runner:*)
job_runner="${req/runner:/}"
;;
fs:*)
file_system="${req/fs:/}"
;;
comms:*)
comms="${req/comms:/}"
;;
*)
echo "Invalid option $req" >&2
return 1
;;
esac
done
echo "_${locality}_${job_runner}_${file_system}_${comms}"
}
_get_test_platform () {
# the requirement string: e.g. comms:tcp
local requirement="$1"
local platform
local required_platform
required_platform="$(_parse_platform_spec "${requirement}")"
for platform in "${PLATFORMS[@]}"; do
# we are purposefully allowing pattern matching here
# shellcheck disable=SC2053
if [[ "$platform" == $required_platform ]]; then
CYLC_TEST_PLATFORM="$platform"
CYLC_TEST_JOB_RUNNER="$(cut -d '_' -f 3 <<< "$platform")"
export CYLC_TEST_PLATFORM CYLC_TEST_JOB_RUNNER
return 0
fi
done
return 1
}
_get_test_host () {
# extract a test host from the test platform
# NOTE: this will cause a failure if the platform is not configured
CYLC_TEST_HOST="$(get_host_from_platform "$CYLC_TEST_PLATFORM")"
if [[ -z "${CYLC_TEST_HOST}" ]]; then
echo "Could not find host for platform $CYLC_TEST_PLATFORM" >&2
exit 1
fi
export CYLC_TEST_HOST
}
_get_test_install_target () {
# extract a test host from the test platform
# NOTE: this will cause a failure if the platform is not configured
CYLC_TEST_INSTALL_TARGET="$(
get_install_target_from_platform "$CYLC_TEST_PLATFORM"
)"
if [[ -z "${CYLC_TEST_INSTALL_TARGET}" ]]; then
CYLC_TEST_INSTALL_TARGET="$CYLC_TEST_PLATFORM"
fi
export CYLC_TEST_INSTALL_TARGET
}
get_host_from_platform () {
local platform="$1"
cylc config -i "[platforms][${platform}]hosts" \
| cut -d ',' -f 1
}
get_install_target_from_platform () {
local platform="$1"
cylc config -d -i "[platforms][${platform}]install target" \
| cut -d ',' -f 1
}
localhost_fqdn () {
python -c "
from cylc.flow.hostuserutil import get_fqdn_by_host
print(get_fqdn_by_host(None))
"
}
get_fqdn () {
local HOST="${1}"
ssh "$HOST" hostname -f
}
_get_test_platforms () {
CYLC_CONF_PATH='' cylc config -i '[platforms]' \
| sed -n 's/\[\[\(_.*\)\]\]/\1/p'
}
_expand_test_platforms () {
# convert sting of space separated platforms to array
read -r -a PLATFORMS <<< "${1}"
local ret=()
local pattern
local platform
local all_platforms
# get list of all configured test platforms
# NOTE: when we drop bash3 convert to
# mapfile -t all_platforms < <(_get_test_platforms)
all_platforms=()
while IFS='' read -r line; do
all_platforms+=("$line");
done < <(_get_test_platforms)
# perform pattern matching on the PLATFORMS variable
if [[ "${#PLATFORMS[@]}" -eq 0 ]]; then
# default to all local platforms
PLATFORMS=('_local_background*' '_local_at*')
fi
# perform pattern matching
for pattern in "${PLATFORMS[@]}"; do
for platform in "${all_platforms[@]}"; do
# shellcheck disable=SC2053
# we are purposefully performing glob matching here
if [[ ${platform} == ${pattern} ]]; then
ret+=("${platform}")
fi
done
done
# set PLATFORMS to the unique results
# NOTE: when we drop bash3 convert to
# mapfile -t PLATFORMS < <(
PLATFORMS=()
while IFS='' read -r line; do
PLATFORMS+=("$line");
done < <(
for platform in "${ret[@]}"; do
echo "${platform}";
done | sort -u
)
export PLATFORMS
}
_check_test_requirements () {
# check the test requirements against the array of test platforms
if [[ -z "${REQUIRE_PLATFORM:-}" ]]; then
# if no requirement string is provided default to local, background
REQUIRE_PLATFORM='loc:local runner:background'
fi
if ! _get_test_platform "${REQUIRE_PLATFORM}"; then
skip_all "requires $REQUIRE_PLATFORM"
fi
_add_platform_to_test_global_conf "$CYLC_TEST_PLATFORM"
}
_add_platform_to_test_global_conf () {
# add a test platform from the global config to the test global config
# NOTE: Uses global.cylc NOT global-tests.cylc
# Do not configure test platforms in the global-tests.cylc file
PLATFORM="$1"
platform_cfg=$(CYLC_CONF_PATH='' cylc config -i "[platforms][$PLATFORM]")
cat >> "${CYLC_CONF_PATH}/global.cylc" <<__HERE__
[platforms]
[[$PLATFORM]]
$platform_cfg
__HERE__
if [[ -n "$platform_cfg" ]]; then
# Ensure install target explicitly defined:
cylc config -i "[platforms][${PLATFORM}]install target" || exit 1
fi
}
delete_db() {
# Delete private/public workflow databases for a cold start
local WORKFLOW_RUN_DIR="${1:-$WORKFLOW_RUN_DIR}"
rm "${WORKFLOW_RUN_DIR}/.service/db" "${WORKFLOW_RUN_DIR}/log/db"
}
# tags
PATH="${CYLC_REPO_DIR}/bin:${PATH}"
RUN_DIR="${HOME}/cylc-run"
export RUN_DIR
TEST_NAME_BASE="$(basename "$0" '.t')"
TEST_SOURCE_DIR="$(cd "$(dirname "$0")" && pwd)"
TEST_DIR="$(mktemp -d)"
cd "${TEST_DIR}"
TEST_SOURCE_DIR_BASE="${TEST_SOURCE_DIR##*tests/}"
# this prevents test runs started at the same time from conflicting
TEST_LOG_DIR_BASE="${TMPDIR:-/tmp}/${USER}/${CYLC_TEST_REG_BASE}"
TEST_LOG_DIR="${TEST_LOG_DIR_BASE}/${TEST_SOURCE_DIR_BASE}/${TEST_NAME_BASE}"
WORKFLOW_RUN_FAILS=false
CYLC_TEST_SKIP="${CYLC_TEST_SKIP:-}"
# Is this test in the skip list?
THIS="${0#./}"
THIS_DIR="$(dirname "${THIS}")"
for SKIP in ${CYLC_TEST_SKIP}; do
RSKIP="${SKIP#./}"
if [[ "${THIS}" == "${RSKIP}" || "${THIS_DIR%/}" == "${RSKIP%/}" ]]; then
# Deliberately print variable substitution syntax unexpanded
# shellcheck disable=SC2016
skip_all 'this test is in $CYLC_TEST_SKIP.'
fi
done
# Ignore the normal site/user global config, use global-tests.cylc.
create_test_global_config "$@"
# get array of test platforms to run tests with
_expand_test_platforms "${PLATFORMS}"
# run or skip tests based on the test requirements and provided test platforms
# provides: CYLC_TEST_PLATFORM
_check_test_requirements
# extract a test host from the test platform
# provides: CYLC_TEST_HOST
_get_test_host
# extract a test install target from the test platform
# provides: CYLC_TEST_INSTALL_TARGET
_get_test_install_target
if ${CYLC_SHOW_TEST_EXECUTION:-false}; then
# write the name of tests as they are run
# (helpful for debugging hanging tests)
echo "# $TEST_NAME_BASE" >&2
fi
set +x
set +e