#!/bin/bash -e # See ``./devel --help``. set -o pipefail # Library implementation detection detect:0:debian() { [ -f "/etc/debian_version" ]; } _codename() { local release release=$(lsb_release --release --short) if [ -z "${release}" ] || [ "${release}" = "n/a" ]; then printf "sid" else lsb_release --codename --short fi } # Profiles declare -A MBD_PROFILES=( # dist selections [_debian]="export MBD_SETUP+=' --from-origins Debian --archives-from-origins --sources-from-origins';" # Supported (distro-info) Debian codenames [_debian_all]="export MBD_SETUP+=' --from-origins Debian:all --archives-from-origins --sources-from-origins';" # Known-working Debian codenames [_ubuntu]="export MBD_SETUP+=' --from-origins Ubuntu --archives-from-origins --sources-from-origins';" # Supported (distro-info) Ubuntu codenames [_pureos]="export MBD_SETUP+=' --from-origins PureOS:all --archives-from-origins --sources-from-origins --distributions amber:arm64';" # PureOS [_all]="export MBD_SETUP+=' --from-origins Debian:all Ubuntu:all --archives-from-origins --sources-from-origins';" # All known-working (Debian and Ubuntu) codenames [_urold]="export MBD_SETUP+=' --sources squeeze lenny trusty';" # Urold, may still work with some special tweaks [_arm64]="export MBD_SETUP+=' --distributions $(_codename):arm64 --chroots $(_codename):arm64';" # Add arm64 arch (to test non-native arch under amd64) [_repos]="export MBD_SETUP+=' --repositories debdev::sid repo::sid';" # Add special "debdev" and 'normal' "repo" repository # network options [_ssl]="export MBD_HTTPD_SSL=true;" # Encrypted [_remote_localhost]="export MBD_SETUP+=' --remotes tcp:host=localhost:port=8068';" # With remote 'localhost' [_remote_hostname]="export MBD_SETUP+=' --remotes tcp:host=$(hostname -f):port=8066'; export MBD_HOSTNAME=localhost; export MBD_HTTPD_PORT=8068;" # Run as 'localhost:8068' with remote 'hostname' # installation OS workarounds [_buster]="export MBD_SKIP+=' codespell pylint pyflakes sphinx-build tidy build-ourselves';" # Hacks to run under buster [_bullseye]="export MBD_SKIP+=' pylint lintian'" # Hacks to run under bullseye # Misc [_debug]="export MBD_COMMON_OPTIONS+=' --log-level=WARNING --log-level=mini_buildd.DEBUG';" # Normal debug mode (mini-buildd logs only) [_debug_all]="export MBD_COMMON_OPTIONS+=' --log-level=DEBUG'; export PYTHONWARNINGS='all'" # Full debug mode (all logs && python warnings) [_debug_django]="export MBD_EXTRA_OPTIONS+=' --debug=webapp'" # Enable django debug mode (templates) ) _mbd_profile() # { local prof=${1} local code=${MBD_PROFILES[${prof}]} if [ -n "${code}" ]; then eval "${MBD_PROFILES[${prof}]}" logI "Using profile: ${prof}" else logE "No such profile: ${prof}" return 1 fi } mbd_profile() # [ ...] ... { # Load _profiles local prof local -i spos=0 for prof in "${@}"; do [ "${prof:0:1}" == "_" ] || break # profile args need to start with _ _mbd_profile "${prof}" spos+=1 done # Run w/ remaining (target) args ./devel "${@:spos+1}" } mbd_supertestall() { ./devel profile _all updatetestall | tee ../supertestall.log 2>&1 ./devel profile _all _ssl updatetestall | tee ../supertestall_ssl.log 2>&1 } MBD_BROWSERS="firefox-esr firefox chromium google-chrome-stable google-chrome-beta google-chrome-unstable" mbd_browser() { local -a urls=("http://${MBD_HOSTNAME}:8066" "http://localhost:8068" "https://${MBD_HOSTNAME}:8066" "https://localhost:8068" "http://localhost:8000") local -A args=( [firefox]="--no-remote" [firefox-esr]="--no-remote" ) ${1} ${args[${1}]} "${urls[@]}" >devel.browser.log 2>&1 & } # Convenience: Run ipython so we can import directly from working dir (PYTHONPATH already set here) mbd_ipython3() { ipython3; } _bf_=$(tput bold) || true _it_=$(tput sitm) || true _red_=$(tput setaf 1) || true _yellow_=$(tput setaf 3) || true _blue_=$(tput setaf 4) || true _reset_=$(tput sgr0) || true log() { local running="" [ -z "${MBD_RUNNING}" ] || running="${_bf_}${MBD_RUNNING}${_reset_}: " local -A color=(["I"]="${_blue_}" ["W"]="${_yellow_}" ["E"]="${_red_}") printf "$(date --rfc-3339=ns) ${color[${1}]}${1}${_reset_}: ${running}${2}\n" "${@:3}" >&2; } logI() { log "I" "${@}"; } logW() { log "W" "${@}"; } logE() { log "E" "${@}"; } # don't accidentally run this on your host system. check_devsys() { if ! (ischroot || systemd-detect-virt --quiet); then printf "W: $0\n" >&2 printf "W: Never use this script in production systems\n" >&2 printf "W: This may be not a developer system (not chroot, not systemd container)\n" >&2 read -p"Sure to continue (Ctrl-c to abort)?" DUMMY fi } # python local install helpers mbd_pip_uninstall() { local packages=${1:-$(pip3 freeze --user)} [ -z "${packages}" ] || pip3 uninstall --yes ${packages} } mbd_pip_install() { local package for package in "${@}"; do pip3 install --upgrade --ignore-installed "${package}" done } mbd_installdeps() { sudo apt-get update sudo apt-get --yes upgrade # Extras sudo apt-get install \ --no-install-recommends \ --yes \ \ lsb-release \ aptitude \ git \ git-buildpackage \ devscripts \ disorderfs \ diffoscope \ equivs \ pydocstyle \ pycodestyle \ isort \ pylint \ pyflakes3 \ python3-pip \ python3-wheel \ python3-keyrings.alt \ devscripts \ faketime \ tidy \ codespell \ sqlite3 \ moreutils \ curl \ siege \ debmirror \ apache2-utils \ ssl-cert \ libnss3-tools \ libdistro-info-perl \ arch-test \ firefox-esr # Extras: ftp client, preferring ftp-ssl. sudo apt-get install ftp-ssl || sudo apt-get install ftp # Build-Deps mk-build-deps --install --remove --root-cmd=sudo --tool="apt-get --yes --no-install-recommends" } # Force plain text backend for development mbd_pythonkeyringtestconfig() { local keyringrc="${HOME}/.config/python_keyring/keyringrc.cfg" [ -L "${keyringrc}" ] || { mkdir -p "$(dirname "${keyringrc}")" && ln -s -v -f "${MBD_PJPATH}/devel.keyringrc.cfg" "${keyringrc}"; } } # Add custom self-signed cert mbd_setupcert() { sudo ${MBD_PJPATH}/src/mini-buildd-self-signed-certificate create ${MBD_HOSTNAME} # Make trusted for system (ca-certificates) local cert="/etc/ssl/mini-buildd/certs/mini-buildd.crt" ( cd "/usr/local/share/ca-certificates/" && sudo ln -s -f "${cert}" .) sudo update-ca-certificates --fresh # Helper to update certs to some known browsers ./examples/mini-buildd-utils/ca-certificates2browser } declare -g -a _ON_EXIT=() on_exit_run() { local line for line in "${_ON_EXIT[@]}"; do if ${MBD_KEEP}; then logW "MBD_KEEP: Ignoring On Exit: %s" "${line}" else logI "On Exit: %s" "${line}" ${line} fi done [ -z "${MBD_RUNNING}" ] || logE "Failed: $?" } on_exit() { _ON_EXIT+=("${*}") } trap "on_exit_run" EXIT # Avoid some esoteric cases where on_exit is not run (certain emacs compilation buffer kills) trap "exit 2" INT trap "exit 3" TERM _check_prog() { local path for path in $(printf "${PATH}" | tr ":" " "); do local prog="${path}/${1}" if [ -x "${prog}" ]; then logI "Found: ${prog}." return 0 fi done logE "'${1}' not found in path; please install." logI "'./devel prepare-system' should install all deps needed." exit 1 } # Extra python as pip user install (f.e., use 'pylint' here to use it instead of the (usually older) Debian version). : ${MBD_PIPINSTALL:=""} MBD_PJPATH="$(readlink -f $(dirname $0))" MBD_PYPATH="${MBD_PJPATH}/src" export PYTHONPATH="${MBD_PYPATH}" MBD_SETUP_CFG="${MBD_PJPATH}/setup.cfg" MBD_LINTIANRC="${MBD_PJPATH}/devel.lintianrc" MBD_CODENAME=$(_codename) declare -A _MBD_CODEVERSIONS=([buster]=10 [bullseye]=11 [bookworm]=12 [trixie]=13 [forky]=~FORKY [sid]=~SID) [ -z "${MBD_PROFILES[_${MBD_CODENAME}]}" ] || _mbd_profile "_${MBD_CODENAME}" MBD_CODEVERSION="${_MBD_CODEVERSIONS[${MBD_CODENAME}]}" # Autogenerated by us MBD_DPUT_CF="${MBD_PJPATH}/devel.dput.cf" MBD_LAST_CHANGES="${MBD_PJPATH}/devel.last_changes" # # Environment (others) # # Use noninteractive by default -- install, configure && purge w/o confirmation (see --preserve-env=DEBIAN_FRONTEND in code below) export DEBIAN_FRONTEND=${DEBIAN_FRONTEND:-noninteractive} export PYTHONWARNINGS=${PYTHONWARNINGS:-ignore} # # Configurable variables # MBD_HTTPD_SSL_DESC="Toggle 'https mode' (ssl key is managed via 'mini-buildd-self-signed-certificate'. Should just work after './devel prepare-system')." : ${MBD_HTTPD_SSL:=false} MBD_CONFIG+="MBD_HTTPD_SSL " MBD_HTTPD_PORT_DESC="Custom port -- run a second instance on same network stack (i.e, use '8068' and configure ftp to be at '8069')." : ${MBD_HTTPD_PORT:=8066} MBD_CONFIG+="MBD_HTTPD_PORT " MBD_COMMON_OPTIONS_DESC="Common options (used by all python-based tools)." : ${MBD_COMMON_OPTIONS:=} MBD_CONFIG+="MBD_COMMON_OPTIONS " MBD_EXTRA_OPTIONS_DESC="Extra options for the mini-buildd call only." : ${MBD_EXTRA_OPTIONS:=} MBD_CONFIG+="MBD_EXTRA_OPTIONS " MBD_HOSTNAME_DESC="Custom hostname -- for a second instance in ssl mode, you must have a different hostname for the certificate." : ${MBD_HOSTNAME:=$(hostname -f || printf "localhost")} MBD_CONFIG+="MBD_HOSTNAME " MBD_SETUP_COMMON_DESC="Common args for API call 'setup', needed to complete testsuite. Rather don't touch." : ${MBD_SETUP_COMMON:=--update all --pca all --archives-from-proxy --sources ${MBD_CODENAME} --distributions-from-sources --repositories-from-distributions test --chroots-from-sources } MBD_CONFIG+="MBD_SETUP_COMMON " MBD_SETUP_DESC="Extra args for API call 'setup'. Use '--archives-from-origins' if you have no local 'apt-cacher-ng' running." : ${MBD_SETUP:=} MBD_CONFIG+="MBD_SETUP " MBD_KEEP_DESC="Don't run 'on exit' cleanups." : ${MBD_KEEP:=false} MBD_CONFIG+="MBD_KEEP " MBD_SKIP_DESC="Skip these test cases (space separated list)." : ${MBD_SKIP:=} MBD_CONFIG+="MBD_SKIP " if ${MBD_HTTPD_SSL}; then MBD_HTTPD_ENDPOINT="ssl:port=${MBD_HTTPD_PORT}:privateKey=/etc/ssl/mini-buildd/private/mini-buildd.key:certKey=/etc/ssl/mini-buildd/certs/mini-buildd.crt" MBD_HTTPD_PROTO="https" else MBD_HTTPD_ENDPOINT="tcp6:port=${MBD_HTTPD_PORT}" MBD_HTTPD_PROTO="http" fi MBD_HTTPD_URL="${MBD_HTTPD_PROTO}://${MBD_HOSTNAME}:${MBD_HTTPD_PORT}" MBD_HTTPD_ADMIN_URL="${MBD_HTTPD_PROTO}://admin@${MBD_HOSTNAME}:${MBD_HTTPD_PORT}" MBD_DEBCONF="\ set mini-buildd/admin_password admin set mini-buildd/options ${MBD_COMMON_OPTIONS} --hostname=${MBD_HOSTNAME} --http-endpoint=${MBD_HTTPD_ENDPOINT} ${MBD_EXTRA_OPTIONS} set mini-buildd/pythonwarnings ${PYTHONWARNINGS}" # Needed by ``gbp dch`` -- be sure it's not empty : ${DEBEMAIL:="$(id --user --name)@${MBD_HOSTNAME}"} export DEBEMAIL mbd_api() { ${MBD_PYPATH}/mini-buildd-api ${MBD_COMMON_OPTIONS} --auto-save-passwords --auto-confirm "${1}" "${MBD_HTTPD_ADMIN_URL}" "${@:2}"; } mbd_codenames() { ${MBD_PYPATH}/mini-buildd-api ${MBD_COMMON_OPTIONS} --json status "${MBD_HTTPD_URL}" --with-repositories | jq --raw-output ".repositories.test.codenames[]"; } mbd_events() { ${MBD_PYPATH}/mini-buildd-events ${MBD_COMMON_OPTIONS} "${MBD_HTTPD_URL}" "${@:1}"; } mbd_dput() { mbd_api dput_conf >${MBD_DPUT_CF} ${MBD_PYPATH}/mini-buildd-dput ${MBD_COMMON_OPTIONS} --config=${MBD_DPUT_CF} --force mini-buildd-${MBD_HOSTNAME%%.*} "${@:1}" } # Shortcuts to build one test/keyring package mbd_testpackage() { mbd_api test_packages --sources="${1}" --distributions="${MBD_CODENAME}-test-unstable" "${@:2}"; } mbd_mbd-test-cpp() { mbd_testpackage "mbd-test-cpp" "${@}"; } mbd_mbd-test-archall() { mbd_testpackage "mbd-test-archall" "${@}"; } mbd_mbd-test-ftbfs() { mbd_testpackage "mbd-test-ftbfs" "${@}"; } mbd_archive-keyring() { mbd_api keyring_packages --distributions="${MBD_CODENAME}-test-unstable" "${@}"; } # pgrep pid && kill support mbd_pid() { local p pids=$(pgrep --uid=mini-buildd --full /usr/bin/python3.*/usr/sbin/mini-buildd) for p in ${pids}; do local root=$(sudo realpath -m "/proc/${p}/root") if [ "${root}" == "/" ]; then printf "%s\n" "${p}" else logW "Ignoring process %s from chroot %s" "${p}" "${root}" fi done } mbd_kill() { local pid=$(mbd_pid) signal=${1:-SIGTERM} logI "Killing process ${pid:-*no mini-buildd process found*} (${signal})..." [ -z "${pid}" ] || sudo kill --signal=${signal} ${pid} # sysv (chroot) compat (else "service start" will not work) sudo rm -v -f /run/mini-buildd.pid } MBD_PYSOURCES=$(./pysources) MBD_PYMODULES=$(./pysources modules) MBD_PYSCRIPTS=$(./pysources scripts) mbd_pyrpl() { for m in ${MBD_PYSOURCES}; do rpl -F "${1}" "${2}" "${m}" done } mbd_service() { sudo service mini-buildd "${1}" } mbd_changes() # [] { local arch=${1-$(dpkg-architecture --query=DEB_BUILD_ARCH)} printf "../mini-buildd_$(dpkg-parsechangelog --show-field=version)_${arch}.changes" } mbd_curl-admin() { local url="${MBD_HTTPD_URL}/accounts/login/" local cookies=./devel.curl_cookies local curl="curl --silent --cookie-jar ${cookies} --cookie ${cookies} --referer ${url}" rm -v -f "${cookies}" # Generate initial cookies file ${curl} "${url}" >/dev/null local csrf="csrfmiddlewaretoken=$(grep csrftoken "${cookies}" | cut -f7)" # Login. Returns 302 w/ empty body on successful login. We test for the empty body. local login_body="./devel.curl_login" ${curl} --data "${csrf}&username=admin&password=admin" --output "${login_body}" "${url}" if [ -s "${login_body}" ]; then logE "Curl login failed." return 1 fi time ${curl} "${@}" } mbd_icons() { logI "Icon names that need to be supported in used icon theme." logI "Naming oriented on: https://specifications.freedesktop.org/icon-naming-spec/icon-naming-spec-latest.html" grep --recursive --no-filename --only-matching 'mbd_icon[[:space:]]*[^[:space:]]*' src/mini_buildd/templates/ | sort --unique } # A very bad hack to hack HTML without having to install package mbd_html-hack() { local static_path="/usr/lib/python3/dist-packages/mini_buildd/static" local templates_path="/usr/lib/python3/dist-packages/mini_buildd/templates" case $1 in start) sudo adduser mini-buildd "$(id --group --name)" local path for path in ${static_path} ${templates_path}; do sudo rm -rfv "${path}" (cd $(dirname "${path}") && sudo ln -s -v "${MBD_PYPATH}/mini_buildd/$(basename "${path}")" .) done ( cd "${MBD_PYPATH}/mini_buildd/static/" ln -s -v "/usr/share/javascript" . || true ln -s -v /usr/lib/python3/dist-packages/django/contrib/admin/static/admin . || true ) mbd_service restart ;; stop) for path in ${static_path} ${templates_path}; do sudo rm -v -f "${path}" done rm -v "${MBD_PYPATH}/mini_buildd/static/javascript" || true rm -v "${MBD_PYPATH}/mini_buildd/static/admin" || true ;; check) if [ -L "${static_path}" -o -L "${templates_path}" ]; then logE "HTML hack active!" return 1 fi ;; esac } if ! mbd_html-hack check; then read -p"Remove html hack (Ctrl-c to abort)?" DUMMY mbd_html-hack stop fi mbd_makemigrations() { python3 ./setup.py version_py ./src/django-admin makemigrations mini_buildd } mbd_makedatamigrations() { python3 ./setup.py version_py ./src/django-admin makemigrations --empty mini_buildd } # # Runner functions: mbd_run:: # # : Three-digit sequence number; this number also determines in which order to run. # All run in sequence order is the whole testsuite (which, in turn, may be run with different profiles, see supertestall as example). # 0..: system prepare # 1..: static code tests # 2..: dynamic code tests # 3..: build # 4..: static build artefact tests # 5..: deploy # 6..: live tests # 7..: web live tests mbd_run:000:prepare-system() { mbd_installdeps mbd_pythonkeyringtestconfig mbd_pip_uninstall mbd_pip_install ${MBD_PIPINSTALL} } mbd_run:100:version_py() # Needed once to get (at least) version.py setup { python3 ./setup.py version_py } mbd_run:105:codespell() { local ups=$(codespell --ignore-words="./devel.codespell.ignore-words" --exclude-file="./devel.codespell.ignore-lines" --quiet-level=2 ./devel ${MBD_PYSOURCES} manual/*.rst) if [ -n "${ups}" ]; then logE "codespell failed:\n${ups}" return 1 fi } mbd_run:110:pycodestyle() # See setup.cfg { _check_prog pycodestyle pycodestyle ${MBD_PYSOURCES} } # Note: Output not parsable; see https://github.com/PyCQA/pydocstyle/issues/460 mbd_run:115:pydocstyle() # See setup.cfg { _check_prog pydocstyle pydocstyle ${MBD_PYSOURCES} } mbd_run:117:isort() # See setup.cfg { _check_prog isort if [ "${1}" == "fix" ]; then isort ${MBD_PYSOURCES} else isort --check ${MBD_PYSOURCES} fi } MBD_PYLINT_MSG_TEMPLATE="{path}:{line}: {C}: [{msg_id}({symbol}), {obj}] {msg}" mbd_run:120:pylint() # See setup.cfg # See https://github.com/PyCQA/pylint/issues/6943 { _check_prog pylint pylint --jobs=0 --msg-template="${MBD_PYLINT_MSG_TEMPLATE}" ${MBD_PYSOURCES} } mbd_run:121:pylint-migrations() { _check_prog pylint pylint --jobs=0 --msg-template="${MBD_PYLINT_MSG_TEMPLATE}" \ --disable="invalid-name,duplicate-code,line-too-long" \ src/mini_buildd/migrations/ } mbd_run:135:pyflakes() # See setup.cfg { _check_prog pyflakes3 pyflakes3 ${MBD_PYSOURCES} } mbd_run:200:pydoctests() { ./pydoctests } mbd_run:201:migrations-from-20x() { local sqlite_path="src/test-data/config.sqlite" sqlite3 "${sqlite_path}" ".read ${sqlite_path}.20x.sql" ./src/run-migrations --log-level=DEBUG ${sqlite_path} rm -v -f "${sqlite_path}" } # Same call as in debian/rules, but warnings are errors mbd_run:205:sphinx-build() { python3 ./setup.py version_py sphinx-build -N -bhtml -W ./manual/ ./build/sphinx/html/ } mbd_run:300:changelog() { # Checking changelog (must be unchanged)... git diff-index --exit-code HEAD debian/changelog on_exit git checkout debian/changelog ## Variant w/: --snapshot --distribution=xxx working: # gbp dch --snapshot --auto \ # --new-version="$(dpkg-parsechangelog --show-field=Version)~test${MBD_CODEVERSION}+0" --dch-opt="--force-bad-version" \ # --distribution="${MBD_CODENAME}-test-experimental" --force-distribution # Try to get snapshot changelog so package might be uploaded to testsuite mini-buildd later for testing # gbp dch --auto # This gets pretty slow if your are in a longer dev battle stint with many commits gbp dch --since HEAD~~~~ \ --new-version="$(dpkg-parsechangelog --show-field=Version).dev$(date --utc +'%Y%m%d%H%M%S')~test${MBD_CODEVERSION}+0" --dch-opt="--force-bad-version" \ --distribution=${MBD_CODENAME}-test-experimental --force-distribution mbd_changes >"${MBD_LAST_CHANGES}" } mbd_run:305:build() { DEB_BUILD_OPTIONS+="nocheck" debuild --no-lintian -us -uc } mbd_run:400:lintian() { local changes=$(cat "${MBD_LAST_CHANGES}" || mbd_changes) logI "Lintian-checking: %s" "${changes}" local result=$(lintian --cfg="${MBD_LINTIANRC}" "${@}" "${changes}") printf "%s\n---\n" "${result}" if grep "^[EW]:" <<< ${result}; then logE "Lintian FAILED" return 1 fi } # This also checks "full" package building (with doc and check) mbd_run:405:debrepro() { debrepro } mbd_run:500:remove() { mbd_service stop || true sudo --preserve-env=DEBIAN_FRONTEND dpkg --${1:-remove} mini-buildd mini-buildd-utils mini-buildd-doc python3-mini-buildd python-mini-buildd mini-buildd-common } mbd_run:505:purge() { mbd_run:500:remove purge } mbd_run:510:install() { ! ischroot || mbd_kill # In chroot environment, apt will ignore stopping the (old) service, which may lead to errors upgrading sudo --preserve-env=DEBIAN_FRONTEND apt-get --yes --allow-downgrades install $(mbd_changes) ! ${MBD_HTTPD_SSL} || mbd_setupcert # setup certificates late to guarantee user 'mini-buildd' is available and ownership of cert files can be properly set sudo sed -i "s|^\([A-Z_]\+\)=.*|\1=\"\"|g" /etc/default/mini-buildd # Deprive any FOO_BAR=value lines of their value (else ``debian/mini-buildd.config`` honors (old) values from file as user customizations) printf "%s" "${MBD_DEBCONF}" | sudo debconf-communicate sudo --preserve-env=DEBIAN_FRONTEND dpkg-reconfigure mini-buildd } # Workaround: ``apt-get --install-recommends --install-suggests `` does not seem to work (see install() above) mbd_run:511:install-recommends-suggests() { sudo aptitude --assume-yes install '~Rrecommends:^mini-buildd' '~Rsuggests:^mini-buildd' } mbd_run:515:restart() { mbd_service restart sleep 0.8 # Try to avoid "connection refused" warnings in most cases local -i try=1 while ! mbd_api status; do if [ ${try} -gt 5 ]; then logE "Can't get status even after ${try} tries" return 1 fi try+=1 sleep 1 done } mbd_run:600:setup() { mbd_api setup ${MBD_SETUP_COMMON} ${MBD_SETUP} } mbd_run:601:spameventsqueue() { local i for i in {1..50}; do mbd_events >>/dev/null 2>&1 & done sleep 10 # Wait some time to let all clients build up network connection curl --max-time 5 --retry 1 --silent --output /dev/null "${MBD_HTTPD_URL}" # If it does not reply in time, assume error: twisted would stall (TCP connect yes, but no handling) mbd_service restart # This should terminate all clients sleep 10 # Be sure mini-buildd is up again for following tests } mbd_run:605:keyring-packages() { mbd_api keyring_packages } mbd_run:610:test-packages() { mbd_api test_packages --with-check } mbd_run:615:testsuite-packages() { local location="./share/testsuite-packages/" dst changes # Clean out possible debris (cd "${location}" && git clean -x -d -f) # Rebuild DSCs when needed for dst in $(find ${location} -maxdepth 1 -mindepth 1 -type d); do local source=$(basename "${dst}") [ -z "${1}" ] || [ "${source}" == "${1}" ] || continue sed --in-place --expression="1 s/sid/${MBD_CODENAME}/" --expression="1 s/~SID/${MBD_CODEVERSION}/" "${dst}/debian/changelog" ( cd ${dst} dpkg-buildpackage -us -uc ) sed --in-place --expression="1 s/${MBD_CODENAME}/sid/" --expression="1 s/${MBD_CODEVERSION}/~SID/" "${dst}/debian/changelog" done # Tests: Upload and expect event for changes in ${location}/*${1}*.changes; do local after=$(date --rfc-3339=ns) local source=$(basename ${changes} | cut -d'_' -f1) local expect=$(cut -d'-' -f1 <<<${source} | tr '[:lower:]' '[:upper:]') local failOn="FAILED REJECTED" case ${expect} in REJECTED) failOn="INSTALLED BUILT" ;; esac [ -z "${1}" ] || [ "${source}" == "${1}" ] || continue logI "Testing ${source} (expect=${expect})" # Try package removal (make test not fail if it's already installed from a previous run) if [ "${expect}" == "INSTALLED" ]; then mbd_api remove "${source}" "${MBD_CODENAME}-test-unstable" --without-rollback >/dev/null || true fi case ${source} in installed-portext) cp "${location}/installed-portext_1.0.0.dsc" "${location}/installed-portext_1.0.0.tar.gz" "/tmp/" mbd_api port_ext --allow-unauthenticated "file:///tmp/installed-portext_1.0.0.dsc" "${MBD_CODENAME}-test-unstable" ;; *) # Standard dput, dput-ng can't do ftps. mbd_dput "${changes}" ;; esac mbd_events --source="${source}" --after="${after}" --exit-on ${expect} --fail-on ${failOn} if [ "${expect}" == "INSTALLED" ]; then case ${source} in installed-port) # Try to check for success on internal ports where possible local c codenames=$(mbd_codenames) # Available codenames on running mbd for c in bookworm bullseye; do # Needs to be same as in installed-port's ``changelog`` # Note: On rebuilds (c == MBD_CODENAME), the events check won't work (i.e., would always immediately succeed on the original package) if [ "${c}" != "${MBD_CODENAME}" ] && grep "^${c}\$" <<<${codenames}; then logI "Waiting for internal ${c} port..." mbd_events --source "${source}" --distribution "${c}-test-unstable" --after "${after}" --exit-on ${expect} --fail-on ${failOn} mbd_api remove "${source}" "${c}-test-unstable" --without-rollback else logW "Skipping check for internal ${c} port (rebuilt or codename not available)" fi done ;; esac fi done } mbd_run:620:build-ourselves() { local after=$(date --rfc-3339=ns) local changes=$(cat "${MBD_LAST_CHANGES}" || mbd_changes) mbd_dput ${changes} mbd_events --source="mini-buildd" --distribution="${MBD_CODENAME}-test-experimental" --after="${after}" --exit-on INSTALLED --fail-on FAILED REJECTED } mbd_run:625:build-migrate-remove() { __build_migrate() { local after=$(date --rfc-3339=ns) mbd_mbd-test-cpp mbd_events --source="mbd-test-cpp" --distribution="${MBD_CODENAME}-test-unstable" --after="${after}" --exit-on INSTALLED --fail-on FAILED REJECTED mbd_api migrate --full "mbd-test-cpp" "${MBD_CODENAME}-test-unstable" mbd_events --source="mbd-test-cpp" --distribution="${MBD_CODENAME}-test-testing" --after="${after}" --exit-on MIGRATED mbd_events --source="mbd-test-cpp" --distribution="${MBD_CODENAME}-test-stable" --after="${after}" --exit-on MIGRATED } __build_migrate __build_migrate # Build && migrated twice: There should be rollback0 for all three suites local ls=$(mbd_api ls "mbd-test-cpp") grep "${MBD_CODENAME}-test-unstable-rollback0" <<<${ls} grep "${MBD_CODENAME}-test-testing-rollback0" <<<${ls} grep "${MBD_CODENAME}-test-stable-rollback0" <<<${ls} # Test removals (don't remove stable as we do test installs later) mbd_api remove "mbd-test-cpp" "${MBD_CODENAME}-test-unstable-rollback0" mbd_api remove "mbd-test-cpp" "${MBD_CODENAME}-test-unstable" } mbd_run:630:bogus-ftp-uploads() { local -i ftp_port=$((MBD_HTTPD_PORT+1)) # test packager.py bogus upload handling _upload() { timeout 5 ftp -inv ${MBD_HOSTNAME} ${ftp_port} <"${reject}" Distribution: kaputt Version: 1.2.3 Source: kaputt Architecture: kaputt EOF local after=$(date --rfc-3339=ns) _upload "${reject}" mbd_events --source="kaputt" --after="${after}" --exit-on REJECTED # Completely unparsable changes. This file should just be removed by mini-buildd (no feasible way to test that automatically, though). local invalid="/tmp/invalid.changes" cat <"${invalid}" kaputt EOF _upload "${invalid}" } mbd_run:635:api-getters() { # Consume mbd_api status mbd_api pub_key mbd_api dput_conf mbd_api sources_list mbd_api ls "mbd-test-cpp" --repositories "test" mbd_api show "mbd-test-cpp" --repositories "test" --codenames "${MBD_CODENAME}" # Maintain mbd_api start mbd_api uploaders mbd_api snapshot_ls "${MBD_CODENAME}-test-stable" } _test_install() { on_exit sudo dpkg --purge "${1}" sudo apt-get "${@:2}" install "${1}" } mbd_run:640:apt-tofu-bootstrap() { sudo MBD_CODENAME=${MBD_CODENAME} ./src/mini-buildd-bootstrap-apt "${MBD_HTTPD_URL}" auto local sources_list="/etc/apt/sources.list.d/mbd-testsuite.list" on_exit sudo rm -v -f "${sources_list}" on_exit sudo apt-get update # Add sources.list && test apt update mbd_api sources_list --codenames "${MBD_CODENAME}" | sudo tee "${sources_list}" sudo apt-get update # install previously built test package _test_install mbd-test-cpp } mbd_run:645:apt-snapshot() { # Test snapshot gen, del and sources.list local snap="TESTSUITE_$(date --iso-8601=seconds)" mbd_api snapshot_create "${MBD_CODENAME}-test-stable" "${snap}" mbd_api snapshot_delete "${MBD_CODENAME}-test-stable" "${snap}" mbd_api snapshot_create "${MBD_CODENAME}-test-stable" "${snap}" mbd_api snapshot_ls "${MBD_CODENAME}-test-stable" mbd_api sources_list --repositories "test" --codenames "${MBD_CODENAME}" --suites "stable" --snapshot "${snap}" } mbd_test_urls() { local uri for uri in \ /mini_buildd/ \ /mini_buildd/api/ \ /mini_buildd/api/status/?output=html \ /mini_buildd/api/search/?pattern=mbd\&output=html \ /mini_buildd/api/ls/?repository=test\&codename=${MBD_CODENAME}\&source=mbd-test-cpp\&output=html \ /mini_buildd/api/show/?repository=test\&codename=${MBD_CODENAME}\&source=mbd-test-cpp\&output=html \ $(for route in events repositories builders crontab manual api setup sitemap; do printf "/mini_buildd/%s/ " "${route}"; done) \ /mini_buildd/repositories/test/dists/${MBD_CODENAME}-test-stable/ \ $(for route in events repositories; do printf "/mini_buildd/%s-dir/ " "${route}"; done) \ /accounts/login/ \ /admin/ \ /admin/mini_buildd/source/ \ /static/manual/index.html \ /static/manual/abstract.html \ /static/manual/administrator.html \ /static/manual/consumer.html \ /static/manual/developer.html \ /static/manual/roadmap.html \ /static/manual/auto_admonitions.html \ ; do printf "%s\n" "${MBD_HTTPD_URL}${uri}" done } mbd_run:700:tidy() { local url html="./devel.tidy.html" while IFS="" read -r url; do logI "Testing HTML: ${html} from '${url}'" mbd_curl-admin --fail --silent --location --output "${html}" "${url}" local mute="" mute+="PROPRIETARY_ATTRIBUTE," # With django 3, tidy complains about 'proprietary attribute "autocapitalize"', but this seems to be valid html5. mute+="TRIM_EMPTY_ELEMENT," # Just too picky. mute+="TAG_NOT_ALLOWED_IN," # Cries about "