| 1 |
#!/bin/bash -e |
| 2 |
|
| 3 |
# See ``./devel --help``. |
| 4 |
|
| 5 |
set -o pipefail |
| 6 |
|
| 7 |
# Library implementation detection |
| 8 |
detect:0:debian() { [ -f "/etc/debian_version" ]; } |
| 9 |
|
| 10 |
_codename() |
| 11 |
{ |
| 12 |
local release |
| 13 |
release=$(lsb_release --release --short) |
| 14 |
if [ -z "${release}" ] || [ "${release}" = "n/a" ]; then |
| 15 |
printf "sid" |
| 16 |
else |
| 17 |
lsb_release --codename --short |
| 18 |
fi |
| 19 |
} |
| 20 |
|
| 21 |
# Profiles |
| 22 |
declare -A MBD_PROFILES=( |
| 23 |
# dist selections |
| 24 |
[_debian]="export MBD_SETUP+=' --from-origins Debian --archives-from-origins --sources-from-origins';" # Supported (distro-info) Debian codenames |
| 25 |
[_debian_all]="export MBD_SETUP+=' --from-origins Debian:all --archives-from-origins --sources-from-origins';" # Known-working Debian codenames |
| 26 |
[_ubuntu]="export MBD_SETUP+=' --from-origins Ubuntu --archives-from-origins --sources-from-origins';" # Supported (distro-info) Ubuntu codenames |
| 27 |
[_pureos]="export MBD_SETUP+=' --from-origins PureOS:all --archives-from-origins --sources-from-origins --distributions amber:arm64';" # PureOS |
| 28 |
[_all]="export MBD_SETUP+=' --from-origins Debian:all Ubuntu:all --archives-from-origins --sources-from-origins';" # All known-working (Debian and Ubuntu) codenames |
| 29 |
[_urold]="export MBD_SETUP+=' --sources squeeze lenny trusty';" # Urold, may still work with some special tweaks |
| 30 |
[_arm64]="export MBD_SETUP+=' --distributions $(_codename):arm64 --chroots $(_codename):arm64';" # Add arm64 arch (to test non-native arch under amd64) |
| 31 |
[_repos]="export MBD_SETUP+=' --repositories debdev::sid repo::sid';" # Add special "debdev" and 'normal' "repo" repository |
| 32 |
|
| 33 |
# network options |
| 34 |
[_ssl]="export MBD_HTTPD_SSL=true;" # Encrypted |
| 35 |
[_remote_localhost]="export MBD_SETUP+=' --remotes tcp:host=localhost:port=8068';" # With remote 'localhost' |
| 36 |
[_remote_hostname]="export MBD_SETUP+=' --remotes tcp:host=$(hostname -f):port=8066'; export MBD_HOSTNAME=localhost; export MBD_HTTPD_PORT=8068;" # Run as 'localhost:8068' with remote 'hostname' |
| 37 |
|
| 38 |
# installation OS workarounds |
| 39 |
[_buster]="export MBD_SKIP+=' codespell pylint pyflakes sphinx-build tidy build-ourselves';" # Hacks to run under buster |
| 40 |
[_bullseye]="export MBD_SKIP+=' pylint lintian'" # Hacks to run under bullseye |
| 41 |
|
| 42 |
# Misc |
| 43 |
[_debug]="export MBD_COMMON_OPTIONS+=' --log-level=WARNING --log-level=mini_buildd.DEBUG';" # Normal debug mode (mini-buildd logs only) |
| 44 |
[_debug_all]="export MBD_COMMON_OPTIONS+=' --log-level=DEBUG'; export PYTHONWARNINGS='all'" # Full debug mode (all logs && python warnings) |
| 45 |
[_debug_django]="export MBD_EXTRA_OPTIONS+=' --debug=webapp'" # Enable django debug mode (templates) |
| 46 |
) |
| 47 |
|
| 48 |
_mbd_profile() # <prof> |
| 49 |
{ |
| 50 |
local prof=${1} |
| 51 |
|
| 52 |
local code=${MBD_PROFILES[${prof}]} |
| 53 |
if [ -n "${code}" ]; then |
| 54 |
eval "${MBD_PROFILES[${prof}]}" |
| 55 |
logI "Using profile: ${prof}" |
| 56 |
else |
| 57 |
logE "No such profile: ${prof}" |
| 58 |
return 1 |
| 59 |
fi |
| 60 |
} |
| 61 |
|
| 62 |
mbd_profile() # [<prof0> <prof1> ...] <target_arg0> <target_arg1> ... |
| 63 |
{ |
| 64 |
# Load _profiles |
| 65 |
local prof |
| 66 |
local -i spos=0 |
| 67 |
for prof in "${@}"; do |
| 68 |
[ "${prof:0:1}" == "_" ] || break # profile args need to start with _ |
| 69 |
_mbd_profile "${prof}" |
| 70 |
spos+=1 |
| 71 |
done |
| 72 |
|
| 73 |
# Run w/ remaining (target) args |
| 74 |
./devel "${@:spos+1}" |
| 75 |
} |
| 76 |
|
| 77 |
mbd_supertestall() |
| 78 |
{ |
| 79 |
./devel profile _all updatetestall | tee ../supertestall.log 2>&1 |
| 80 |
./devel profile _all _ssl updatetestall | tee ../supertestall_ssl.log 2>&1 |
| 81 |
} |
| 82 |
|
| 83 |
MBD_BROWSERS="firefox-esr firefox chromium google-chrome-stable google-chrome-beta google-chrome-unstable" |
| 84 |
mbd_browser() |
| 85 |
{ |
| 86 |
local -a urls=("http://${MBD_HOSTNAME}:8066" "http://localhost:8068" "https://${MBD_HOSTNAME}:8066" "https://localhost:8068" "http://localhost:8000") |
| 87 |
local -A args=( |
| 88 |
[firefox]="--no-remote" |
| 89 |
[firefox-esr]="--no-remote" |
| 90 |
) |
| 91 |
${1} ${args[${1}]} "${urls[@]}" >devel.browser.log 2>&1 & |
| 92 |
} |
| 93 |
|
| 94 |
# Convenience: Run ipython so we can import directly from working dir (PYTHONPATH already set here) |
| 95 |
mbd_ipython3() { ipython3; } |
| 96 |
|
| 97 |
_bf_=$(tput bold) || true |
| 98 |
_it_=$(tput sitm) || true |
| 99 |
_red_=$(tput setaf 1) || true |
| 100 |
_yellow_=$(tput setaf 3) || true |
| 101 |
_blue_=$(tput setaf 4) || true |
| 102 |
_reset_=$(tput sgr0) || true |
| 103 |
|
| 104 |
log() |
| 105 |
{ |
| 106 |
local running="" |
| 107 |
[ -z "${MBD_RUNNING}" ] || running="${_bf_}${MBD_RUNNING}${_reset_}: " |
| 108 |
local -A color=(["I"]="${_blue_}" ["W"]="${_yellow_}" ["E"]="${_red_}") |
| 109 |
printf "$(date --rfc-3339=ns) ${color[${1}]}${1}${_reset_}: ${running}${2}\n" "${@:3}" >&2; |
| 110 |
} |
| 111 |
logI() { log "I" "${@}"; } |
| 112 |
logW() { log "W" "${@}"; } |
| 113 |
logE() { log "E" "${@}"; } |
| 114 |
|
| 115 |
# don't accidentally run this on your host system. |
| 116 |
check_devsys() |
| 117 |
{ |
| 118 |
if ! (ischroot || systemd-detect-virt --quiet); then |
| 119 |
printf "W: $0\n" >&2 |
| 120 |
printf "W: Never use this script in production systems\n" >&2 |
| 121 |
printf "W: This may be not a developer system (not chroot, not systemd container)\n" >&2 |
| 122 |
read -p"Sure to continue (Ctrl-c to abort)?" DUMMY |
| 123 |
fi |
| 124 |
} |
| 125 |
|
| 126 |
# python local install helpers |
| 127 |
mbd_pip_uninstall() |
| 128 |
{ |
| 129 |
local packages=${1:-$(pip3 freeze --user)} |
| 130 |
[ -z "${packages}" ] || pip3 uninstall --yes ${packages} |
| 131 |
} |
| 132 |
|
| 133 |
mbd_pip_install() |
| 134 |
{ |
| 135 |
local package |
| 136 |
for package in "${@}"; do |
| 137 |
pip3 install --upgrade --ignore-installed "${package}" |
| 138 |
done |
| 139 |
} |
| 140 |
|
| 141 |
mbd_installdeps() |
| 142 |
{ |
| 143 |
sudo apt-get update |
| 144 |
sudo apt-get --yes upgrade |
| 145 |
|
| 146 |
# Extras |
| 147 |
sudo apt-get install \ |
| 148 |
--no-install-recommends \ |
| 149 |
--yes \ |
| 150 |
\ |
| 151 |
lsb-release \ |
| 152 |
aptitude \ |
| 153 |
git \ |
| 154 |
git-buildpackage \ |
| 155 |
devscripts \ |
| 156 |
disorderfs \ |
| 157 |
diffoscope \ |
| 158 |
equivs \ |
| 159 |
pydocstyle \ |
| 160 |
pycodestyle \ |
| 161 |
isort \ |
| 162 |
pylint \ |
| 163 |
pyflakes3 \ |
| 164 |
python3-pip \ |
| 165 |
python3-wheel \ |
| 166 |
python3-keyrings.alt \ |
| 167 |
devscripts \ |
| 168 |
faketime \ |
| 169 |
tidy \ |
| 170 |
codespell \ |
| 171 |
sqlite3 \ |
| 172 |
moreutils \ |
| 173 |
curl \ |
| 174 |
siege \ |
| 175 |
debmirror \ |
| 176 |
apache2-utils \ |
| 177 |
ssl-cert \ |
| 178 |
libnss3-tools \ |
| 179 |
libdistro-info-perl \ |
| 180 |
arch-test \ |
| 181 |
firefox-esr |
| 182 |
|
| 183 |
# Extras: ftp client, preferring ftp-ssl. |
| 184 |
sudo apt-get install ftp-ssl || sudo apt-get install ftp |
| 185 |
|
| 186 |
# Build-Deps |
| 187 |
mk-build-deps --install --remove --root-cmd=sudo --tool="apt-get --yes --no-install-recommends" |
| 188 |
} |
| 189 |
|
| 190 |
# Force plain text backend for development |
| 191 |
mbd_pythonkeyringtestconfig() |
| 192 |
{ |
| 193 |
local keyringrc="${HOME}/.config/python_keyring/keyringrc.cfg" |
| 194 |
[ -L "${keyringrc}" ] || { mkdir -p "$(dirname "${keyringrc}")" && ln -s -v -f "${MBD_PJPATH}/devel.keyringrc.cfg" "${keyringrc}"; } |
| 195 |
} |
| 196 |
|
| 197 |
# Add custom self-signed cert |
| 198 |
mbd_setupcert() |
| 199 |
{ |
| 200 |
sudo ${MBD_PJPATH}/src/mini-buildd-self-signed-certificate create ${MBD_HOSTNAME} |
| 201 |
|
| 202 |
# Make trusted for system (ca-certificates) |
| 203 |
local cert="/etc/ssl/mini-buildd/certs/mini-buildd.crt" |
| 204 |
( cd "/usr/local/share/ca-certificates/" && sudo ln -s -f "${cert}" .) |
| 205 |
sudo update-ca-certificates --fresh |
| 206 |
|
| 207 |
# Helper to update certs to some known browsers |
| 208 |
./examples/mini-buildd-utils/ca-certificates2browser |
| 209 |
} |
| 210 |
|
| 211 |
declare -g -a _ON_EXIT=() |
| 212 |
on_exit_run() |
| 213 |
{ |
| 214 |
local line |
| 215 |
for line in "${_ON_EXIT[@]}"; do |
| 216 |
if ${MBD_KEEP}; then |
| 217 |
logW "MBD_KEEP: Ignoring On Exit: %s" "${line}" |
| 218 |
else |
| 219 |
logI "On Exit: %s" "${line}" |
| 220 |
${line} |
| 221 |
fi |
| 222 |
done |
| 223 |
[ -z "${MBD_RUNNING}" ] || logE "Failed: $?" |
| 224 |
} |
| 225 |
on_exit() |
| 226 |
{ |
| 227 |
_ON_EXIT+=("${*}") |
| 228 |
} |
| 229 |
trap "on_exit_run" EXIT |
| 230 |
# Avoid some esoteric cases where on_exit is not run (certain emacs compilation buffer kills) |
| 231 |
trap "exit 2" INT |
| 232 |
trap "exit 3" TERM |
| 233 |
|
| 234 |
_check_prog() |
| 235 |
{ |
| 236 |
local path |
| 237 |
for path in $(printf "${PATH}" | tr ":" " "); do |
| 238 |
local prog="${path}/${1}" |
| 239 |
if [ -x "${prog}" ]; then |
| 240 |
logI "Found: ${prog}." |
| 241 |
return 0 |
| 242 |
fi |
| 243 |
done |
| 244 |
logE "'${1}' not found in path; please install." |
| 245 |
logI "'./devel prepare-system' should install all deps needed." |
| 246 |
exit 1 |
| 247 |
} |
| 248 |
|
| 249 |
# Extra python as pip user install (f.e., use 'pylint' here to use it instead of the (usually older) Debian version). |
| 250 |
: ${MBD_PIPINSTALL:=""} |
| 251 |
|
| 252 |
MBD_PJPATH="$(readlink -f $(dirname $0))" |
| 253 |
MBD_PYPATH="${MBD_PJPATH}/src" |
| 254 |
export PYTHONPATH="${MBD_PYPATH}" |
| 255 |
MBD_SETUP_CFG="${MBD_PJPATH}/setup.cfg" |
| 256 |
MBD_LINTIANRC="${MBD_PJPATH}/devel.lintianrc" |
| 257 |
MBD_CODENAME=$(_codename) |
| 258 |
declare -A _MBD_CODEVERSIONS=([buster]=10 [bullseye]=11 [bookworm]=12 [trixie]=13 [forky]=~FORKY [sid]=~SID) |
| 259 |
[ -z "${MBD_PROFILES[_${MBD_CODENAME}]}" ] || _mbd_profile "_${MBD_CODENAME}" |
| 260 |
MBD_CODEVERSION="${_MBD_CODEVERSIONS[${MBD_CODENAME}]}" |
| 261 |
|
| 262 |
# Autogenerated by us |
| 263 |
MBD_DPUT_CF="${MBD_PJPATH}/devel.dput.cf" |
| 264 |
MBD_LAST_CHANGES="${MBD_PJPATH}/devel.last_changes" |
| 265 |
|
| 266 |
# |
| 267 |
# Environment (others) |
| 268 |
# |
| 269 |
# Use noninteractive by default -- install, configure && purge w/o confirmation (see --preserve-env=DEBIAN_FRONTEND in code below) |
| 270 |
export DEBIAN_FRONTEND=${DEBIAN_FRONTEND:-noninteractive} |
| 271 |
export PYTHONWARNINGS=${PYTHONWARNINGS:-ignore} |
| 272 |
|
| 273 |
# |
| 274 |
# Configurable variables |
| 275 |
# |
| 276 |
MBD_HTTPD_SSL_DESC="Toggle 'https mode' (ssl key is managed via 'mini-buildd-self-signed-certificate'. Should just work after './devel prepare-system')." |
| 277 |
: ${MBD_HTTPD_SSL:=false} |
| 278 |
MBD_CONFIG+="MBD_HTTPD_SSL " |
| 279 |
|
| 280 |
MBD_HTTPD_PORT_DESC="Custom port -- run a second instance on same network stack (i.e, use '8068' and configure ftp to be at '8069')." |
| 281 |
: ${MBD_HTTPD_PORT:=8066} |
| 282 |
MBD_CONFIG+="MBD_HTTPD_PORT " |
| 283 |
|
| 284 |
MBD_COMMON_OPTIONS_DESC="Common options (used by all python-based tools)." |
| 285 |
: ${MBD_COMMON_OPTIONS:=} |
| 286 |
MBD_CONFIG+="MBD_COMMON_OPTIONS " |
| 287 |
|
| 288 |
MBD_EXTRA_OPTIONS_DESC="Extra options for the mini-buildd call only." |
| 289 |
: ${MBD_EXTRA_OPTIONS:=} |
| 290 |
MBD_CONFIG+="MBD_EXTRA_OPTIONS " |
| 291 |
|
| 292 |
MBD_HOSTNAME_DESC="Custom hostname -- for a second instance in ssl mode, you must have a different hostname for the certificate." |
| 293 |
: ${MBD_HOSTNAME:=$(hostname -f || printf "localhost")} |
| 294 |
MBD_CONFIG+="MBD_HOSTNAME " |
| 295 |
|
| 296 |
MBD_SETUP_COMMON_DESC="Common args for API call 'setup', needed to complete testsuite. Rather don't touch." |
| 297 |
: ${MBD_SETUP_COMMON:=--update all --pca all --archives-from-proxy --sources ${MBD_CODENAME} --distributions-from-sources --repositories-from-distributions test --chroots-from-sources } |
| 298 |
MBD_CONFIG+="MBD_SETUP_COMMON " |
| 299 |
|
| 300 |
MBD_SETUP_DESC="Extra args for API call 'setup'. Use '--archives-from-origins' if you have no local 'apt-cacher-ng' running." |
| 301 |
: ${MBD_SETUP:=} |
| 302 |
MBD_CONFIG+="MBD_SETUP " |
| 303 |
|
| 304 |
MBD_KEEP_DESC="Don't run 'on exit' cleanups." |
| 305 |
: ${MBD_KEEP:=false} |
| 306 |
MBD_CONFIG+="MBD_KEEP " |
| 307 |
|
| 308 |
MBD_SKIP_DESC="Skip these test cases (space separated list)." |
| 309 |
: ${MBD_SKIP:=} |
| 310 |
MBD_CONFIG+="MBD_SKIP " |
| 311 |
|
| 312 |
if ${MBD_HTTPD_SSL}; then |
| 313 |
MBD_HTTPD_ENDPOINT="ssl:port=${MBD_HTTPD_PORT}:privateKey=/etc/ssl/mini-buildd/private/mini-buildd.key:certKey=/etc/ssl/mini-buildd/certs/mini-buildd.crt" |
| 314 |
MBD_HTTPD_PROTO="https" |
| 315 |
else |
| 316 |
MBD_HTTPD_ENDPOINT="tcp6:port=${MBD_HTTPD_PORT}" |
| 317 |
MBD_HTTPD_PROTO="http" |
| 318 |
fi |
| 319 |
MBD_HTTPD_URL="${MBD_HTTPD_PROTO}://${MBD_HOSTNAME}:${MBD_HTTPD_PORT}" |
| 320 |
MBD_HTTPD_ADMIN_URL="${MBD_HTTPD_PROTO}://admin@${MBD_HOSTNAME}:${MBD_HTTPD_PORT}" |
| 321 |
MBD_DEBCONF="\ |
| 322 |
set mini-buildd/admin_password admin |
| 323 |
set mini-buildd/options ${MBD_COMMON_OPTIONS} --hostname=${MBD_HOSTNAME} --http-endpoint=${MBD_HTTPD_ENDPOINT} ${MBD_EXTRA_OPTIONS} |
| 324 |
set mini-buildd/pythonwarnings ${PYTHONWARNINGS}" |
| 325 |
|
| 326 |
# Needed by ``gbp dch`` -- be sure it's not empty |
| 327 |
: ${DEBEMAIL:="$(id --user --name)@${MBD_HOSTNAME}"} |
| 328 |
export DEBEMAIL |
| 329 |
|
| 330 |
mbd_api() { ${MBD_PYPATH}/mini-buildd-api ${MBD_COMMON_OPTIONS} --auto-save-passwords --auto-confirm "${1}" "${MBD_HTTPD_ADMIN_URL}" "${@:2}"; } |
| 331 |
mbd_codenames() { ${MBD_PYPATH}/mini-buildd-api ${MBD_COMMON_OPTIONS} --json status "${MBD_HTTPD_URL}" --with-repositories | jq --raw-output ".repositories.test.codenames[]"; } |
| 332 |
mbd_events() { ${MBD_PYPATH}/mini-buildd-events ${MBD_COMMON_OPTIONS} "${MBD_HTTPD_URL}" "${@:1}"; } |
| 333 |
mbd_dput() |
| 334 |
{ |
| 335 |
mbd_api dput_conf >${MBD_DPUT_CF} |
| 336 |
${MBD_PYPATH}/mini-buildd-dput ${MBD_COMMON_OPTIONS} --config=${MBD_DPUT_CF} --force mini-buildd-${MBD_HOSTNAME%%.*} "${@:1}" |
| 337 |
} |
| 338 |
|
| 339 |
# Shortcuts to build one test/keyring package |
| 340 |
mbd_testpackage() { mbd_api test_packages --sources="${1}" --distributions="${MBD_CODENAME}-test-unstable" "${@:2}"; } |
| 341 |
mbd_mbd-test-cpp() { mbd_testpackage "mbd-test-cpp" "${@}"; } |
| 342 |
mbd_mbd-test-archall() { mbd_testpackage "mbd-test-archall" "${@}"; } |
| 343 |
mbd_mbd-test-ftbfs() { mbd_testpackage "mbd-test-ftbfs" "${@}"; } |
| 344 |
|
| 345 |
mbd_archive-keyring() { mbd_api keyring_packages --distributions="${MBD_CODENAME}-test-unstable" "${@}"; } |
| 346 |
|
| 347 |
# pgrep pid && kill support |
| 348 |
mbd_pid() |
| 349 |
{ |
| 350 |
local p pids=$(pgrep --uid=mini-buildd --full /usr/bin/python3.*/usr/sbin/mini-buildd) |
| 351 |
for p in ${pids}; do |
| 352 |
local root=$(sudo realpath -m "/proc/${p}/root") |
| 353 |
if [ "${root}" == "/" ]; then |
| 354 |
printf "%s\n" "${p}" |
| 355 |
else |
| 356 |
logW "Ignoring process %s from chroot %s" "${p}" "${root}" |
| 357 |
fi |
| 358 |
done |
| 359 |
} |
| 360 |
|
| 361 |
mbd_kill() |
| 362 |
{ |
| 363 |
local pid=$(mbd_pid) signal=${1:-SIGTERM} |
| 364 |
logI "Killing process ${pid:-*no mini-buildd process found*} (${signal})..." |
| 365 |
[ -z "${pid}" ] || sudo kill --signal=${signal} ${pid} |
| 366 |
|
| 367 |
# sysv (chroot) compat (else "service start" will not work) |
| 368 |
sudo rm -v -f /run/mini-buildd.pid |
| 369 |
} |
| 370 |
|
| 371 |
MBD_PYSOURCES=$(./pysources) |
| 372 |
MBD_PYMODULES=$(./pysources modules) |
| 373 |
MBD_PYSCRIPTS=$(./pysources scripts) |
| 374 |
|
| 375 |
mbd_pyrpl() |
| 376 |
{ |
| 377 |
for m in ${MBD_PYSOURCES}; do |
| 378 |
rpl -F "${1}" "${2}" "${m}" |
| 379 |
done |
| 380 |
} |
| 381 |
|
| 382 |
mbd_service() |
| 383 |
{ |
| 384 |
sudo service mini-buildd "${1}" |
| 385 |
} |
| 386 |
|
| 387 |
mbd_changes() # [<arch>] |
| 388 |
{ |
| 389 |
local arch=${1-$(dpkg-architecture --query=DEB_BUILD_ARCH)} |
| 390 |
printf "../mini-buildd_$(dpkg-parsechangelog --show-field=version)_${arch}.changes" |
| 391 |
} |
| 392 |
|
| 393 |
mbd_curl-admin() |
| 394 |
{ |
| 395 |
local url="${MBD_HTTPD_URL}/accounts/login/" |
| 396 |
local cookies=./devel.curl_cookies |
| 397 |
local curl="curl --silent --cookie-jar ${cookies} --cookie ${cookies} --referer ${url}" |
| 398 |
rm -v -f "${cookies}" |
| 399 |
|
| 400 |
# Generate initial cookies file |
| 401 |
${curl} "${url}" >/dev/null |
| 402 |
local csrf="csrfmiddlewaretoken=$(grep csrftoken "${cookies}" | cut -f7)" |
| 403 |
|
| 404 |
# Login. Returns 302 w/ empty body on successful login. We test for the empty body. |
| 405 |
local login_body="./devel.curl_login" |
| 406 |
${curl} --data "${csrf}&username=admin&password=admin" --output "${login_body}" "${url}" |
| 407 |
if [ -s "${login_body}" ]; then |
| 408 |
logE "Curl login failed." |
| 409 |
return 1 |
| 410 |
fi |
| 411 |
time ${curl} "${@}" |
| 412 |
} |
| 413 |
|
| 414 |
mbd_icons() |
| 415 |
{ |
| 416 |
logI "Icon names that need to be supported in used icon theme." |
| 417 |
logI "Naming oriented on: https://specifications.freedesktop.org/icon-naming-spec/icon-naming-spec-latest.html" |
| 418 |
grep --recursive --no-filename --only-matching 'mbd_icon[[:space:]]*[^[:space:]]*' src/mini_buildd/templates/ | sort --unique |
| 419 |
} |
| 420 |
|
| 421 |
# A very bad hack to hack HTML without having to install package |
| 422 |
mbd_html-hack() |
| 423 |
{ |
| 424 |
local static_path="/usr/lib/python3/dist-packages/mini_buildd/static" |
| 425 |
local templates_path="/usr/lib/python3/dist-packages/mini_buildd/templates" |
| 426 |
case $1 in |
| 427 |
start) |
| 428 |
sudo adduser mini-buildd "$(id --group --name)" |
| 429 |
local path |
| 430 |
for path in ${static_path} ${templates_path}; do |
| 431 |
sudo rm -rfv "${path}" |
| 432 |
(cd $(dirname "${path}") && sudo ln -s -v "${MBD_PYPATH}/mini_buildd/$(basename "${path}")" .) |
| 433 |
done |
| 434 |
( |
| 435 |
cd "${MBD_PYPATH}/mini_buildd/static/" |
| 436 |
ln -s -v "/usr/share/javascript" . || true |
| 437 |
ln -s -v /usr/lib/python3/dist-packages/django/contrib/admin/static/admin . || true |
| 438 |
) |
| 439 |
mbd_service restart |
| 440 |
;; |
| 441 |
stop) |
| 442 |
for path in ${static_path} ${templates_path}; do |
| 443 |
sudo rm -v -f "${path}" |
| 444 |
done |
| 445 |
rm -v "${MBD_PYPATH}/mini_buildd/static/javascript" || true |
| 446 |
rm -v "${MBD_PYPATH}/mini_buildd/static/admin" || true |
| 447 |
;; |
| 448 |
check) |
| 449 |
if [ -L "${static_path}" -o -L "${templates_path}" ]; then |
| 450 |
logE "HTML hack active!" |
| 451 |
return 1 |
| 452 |
fi |
| 453 |
;; |
| 454 |
esac |
| 455 |
} |
| 456 |
|
| 457 |
if ! mbd_html-hack check; then |
| 458 |
read -p"Remove html hack (Ctrl-c to abort)?" DUMMY |
| 459 |
mbd_html-hack stop |
| 460 |
fi |
| 461 |
|
| 462 |
mbd_makemigrations() |
| 463 |
{ |
| 464 |
python3 ./setup.py version_py |
| 465 |
./src/django-admin makemigrations mini_buildd |
| 466 |
} |
| 467 |
|
| 468 |
mbd_makedatamigrations() |
| 469 |
{ |
| 470 |
python3 ./setup.py version_py |
| 471 |
./src/django-admin makemigrations --empty mini_buildd |
| 472 |
} |
| 473 |
|
| 474 |
# |
| 475 |
# Runner functions: mbd_run:<sequence>:<name> |
| 476 |
# |
| 477 |
# <sequence>: Three-digit sequence number; this number also determines in which order to run. |
| 478 |
# All run in sequence order is the whole testsuite (which, in turn, may be run with different profiles, see supertestall as example). |
| 479 |
# 0..: system prepare |
| 480 |
# 1..: static code tests |
| 481 |
# 2..: dynamic code tests |
| 482 |
# 3..: build |
| 483 |
# 4..: static build artefact tests |
| 484 |
# 5..: deploy |
| 485 |
# 6..: live tests |
| 486 |
# 7..: web live tests |
| 487 |
|
| 488 |
mbd_run:000:prepare-system() |
| 489 |
{ |
| 490 |
mbd_installdeps |
| 491 |
mbd_pythonkeyringtestconfig |
| 492 |
mbd_pip_uninstall |
| 493 |
mbd_pip_install ${MBD_PIPINSTALL} |
| 494 |
} |
| 495 |
|
| 496 |
mbd_run:100:version_py() # Needed once to get (at least) version.py setup |
| 497 |
{ |
| 498 |
python3 ./setup.py version_py |
| 499 |
} |
| 500 |
|
| 501 |
mbd_run:105:codespell() |
| 502 |
{ |
| 503 |
local ups=$(codespell --ignore-words="./devel.codespell.ignore-words" --exclude-file="./devel.codespell.ignore-lines" --quiet-level=2 ./devel ${MBD_PYSOURCES} manual/*.rst) |
| 504 |
if [ -n "${ups}" ]; then |
| 505 |
logE "codespell failed:\n${ups}" |
| 506 |
return 1 |
| 507 |
fi |
| 508 |
} |
| 509 |
|
| 510 |
mbd_run:110:pycodestyle() # See setup.cfg |
| 511 |
{ |
| 512 |
_check_prog pycodestyle |
| 513 |
pycodestyle ${MBD_PYSOURCES} |
| 514 |
} |
| 515 |
|
| 516 |
# Note: Output not parsable; see https://github.com/PyCQA/pydocstyle/issues/460 |
| 517 |
mbd_run:115:pydocstyle() # See setup.cfg |
| 518 |
{ |
| 519 |
_check_prog pydocstyle |
| 520 |
pydocstyle ${MBD_PYSOURCES} |
| 521 |
} |
| 522 |
|
| 523 |
mbd_run:117:isort() # See setup.cfg |
| 524 |
{ |
| 525 |
_check_prog isort |
| 526 |
if [ "${1}" == "fix" ]; then |
| 527 |
isort ${MBD_PYSOURCES} |
| 528 |
else |
| 529 |
isort --check ${MBD_PYSOURCES} |
| 530 |
fi |
| 531 |
} |
| 532 |
|
| 533 |
MBD_PYLINT_MSG_TEMPLATE="{path}:{line}: {C}: [{msg_id}({symbol}), {obj}] {msg}" |
| 534 |
mbd_run:120:pylint() # See setup.cfg # See https://github.com/PyCQA/pylint/issues/6943 |
| 535 |
{ |
| 536 |
_check_prog pylint |
| 537 |
pylint --jobs=0 --msg-template="${MBD_PYLINT_MSG_TEMPLATE}" ${MBD_PYSOURCES} |
| 538 |
} |
| 539 |
|
| 540 |
mbd_run:121:pylint-migrations() |
| 541 |
{ |
| 542 |
_check_prog pylint |
| 543 |
pylint --jobs=0 --msg-template="${MBD_PYLINT_MSG_TEMPLATE}" \ |
| 544 |
--disable="invalid-name,duplicate-code,line-too-long" \ |
| 545 |
src/mini_buildd/migrations/ |
| 546 |
} |
| 547 |
|
| 548 |
mbd_run:135:pyflakes() # See setup.cfg |
| 549 |
{ |
| 550 |
_check_prog pyflakes3 |
| 551 |
pyflakes3 ${MBD_PYSOURCES} |
| 552 |
} |
| 553 |
|
| 554 |
mbd_run:200:pydoctests() |
| 555 |
{ |
| 556 |
./pydoctests |
| 557 |
} |
| 558 |
|
| 559 |
mbd_run:201:migrations-from-20x() |
| 560 |
{ |
| 561 |
local sqlite_path="src/test-data/config.sqlite" |
| 562 |
sqlite3 "${sqlite_path}" ".read ${sqlite_path}.20x.sql" |
| 563 |
./src/run-migrations --log-level=DEBUG ${sqlite_path} |
| 564 |
rm -v -f "${sqlite_path}" |
| 565 |
} |
| 566 |
|
| 567 |
# Same call as in debian/rules, but warnings are errors |
| 568 |
mbd_run:205:sphinx-build() |
| 569 |
{ |
| 570 |
python3 ./setup.py version_py |
| 571 |
sphinx-build -N -bhtml -W ./manual/ ./build/sphinx/html/ |
| 572 |
} |
| 573 |
|
| 574 |
mbd_run:300:changelog() |
| 575 |
{ |
| 576 |
# Checking changelog (must be unchanged)... |
| 577 |
git diff-index --exit-code HEAD debian/changelog |
| 578 |
on_exit git checkout debian/changelog |
| 579 |
|
| 580 |
## Variant w/: --snapshot --distribution=xxx working: |
| 581 |
# gbp dch --snapshot --auto \ |
| 582 |
# --new-version="$(dpkg-parsechangelog --show-field=Version)~test${MBD_CODEVERSION}+0" --dch-opt="--force-bad-version" \ |
| 583 |
# --distribution="${MBD_CODENAME}-test-experimental" --force-distribution |
| 584 |
|
| 585 |
# Try to get snapshot changelog so package might be uploaded to testsuite mini-buildd later for testing |
| 586 |
# gbp dch --auto # This gets pretty slow if your are in a longer dev battle stint with many commits |
| 587 |
gbp dch --since HEAD~~~~ \ |
| 588 |
--new-version="$(dpkg-parsechangelog --show-field=Version).dev$(date --utc +'%Y%m%d%H%M%S')~test${MBD_CODEVERSION}+0" --dch-opt="--force-bad-version" \ |
| 589 |
--distribution=${MBD_CODENAME}-test-experimental --force-distribution |
| 590 |
mbd_changes >"${MBD_LAST_CHANGES}" |
| 591 |
} |
| 592 |
|
| 593 |
mbd_run:305:build() |
| 594 |
{ |
| 595 |
DEB_BUILD_OPTIONS+="nocheck" debuild --no-lintian -us -uc |
| 596 |
} |
| 597 |
|
| 598 |
mbd_run:400:lintian() |
| 599 |
{ |
| 600 |
local changes=$(cat "${MBD_LAST_CHANGES}" || mbd_changes) |
| 601 |
logI "Lintian-checking: %s" "${changes}" |
| 602 |
local result=$(lintian --cfg="${MBD_LINTIANRC}" "${@}" "${changes}") |
| 603 |
printf "%s\n---\n" "${result}" |
| 604 |
if grep "^[EW]:" <<< ${result}; then |
| 605 |
logE "Lintian FAILED" |
| 606 |
return 1 |
| 607 |
fi |
| 608 |
} |
| 609 |
|
| 610 |
# This also checks "full" package building (with doc and check) |
| 611 |
mbd_run:405:debrepro() |
| 612 |
{ |
| 613 |
debrepro |
| 614 |
} |
| 615 |
|
| 616 |
mbd_run:500:remove() |
| 617 |
{ |
| 618 |
mbd_service stop || true |
| 619 |
sudo --preserve-env=DEBIAN_FRONTEND dpkg --${1:-remove} mini-buildd mini-buildd-utils mini-buildd-doc python3-mini-buildd python-mini-buildd mini-buildd-common |
| 620 |
} |
| 621 |
|
| 622 |
mbd_run:505:purge() |
| 623 |
{ |
| 624 |
mbd_run:500:remove purge |
| 625 |
} |
| 626 |
|
| 627 |
mbd_run:510:install() |
| 628 |
{ |
| 629 |
! ischroot || mbd_kill # In chroot environment, apt will ignore stopping the (old) service, which may lead to errors upgrading |
| 630 |
|
| 631 |
sudo --preserve-env=DEBIAN_FRONTEND apt-get --yes --allow-downgrades install $(mbd_changes) |
| 632 |
! ${MBD_HTTPD_SSL} || mbd_setupcert # setup certificates late to guarantee user 'mini-buildd' is available and ownership of cert files can be properly set |
| 633 |
sudo sed -i "s|^\([A-Z_]\+\)=.*|\1=\"\"|g" /etc/default/mini-buildd # Deprive any FOO_BAR=value lines of their value (else ``debian/mini-buildd.config`` honors (old) values from file as user customizations) |
| 634 |
printf "%s" "${MBD_DEBCONF}" | sudo debconf-communicate |
| 635 |
sudo --preserve-env=DEBIAN_FRONTEND dpkg-reconfigure mini-buildd |
| 636 |
} |
| 637 |
|
| 638 |
# Workaround: ``apt-get --install-recommends --install-suggests <FILE_ON_DISK>`` does not seem to work (see install() above) |
| 639 |
mbd_run:511:install-recommends-suggests() |
| 640 |
{ |
| 641 |
sudo aptitude --assume-yes install '~Rrecommends:^mini-buildd' '~Rsuggests:^mini-buildd' |
| 642 |
} |
| 643 |
|
| 644 |
mbd_run:515:restart() |
| 645 |
{ |
| 646 |
mbd_service restart |
| 647 |
sleep 0.8 # Try to avoid "connection refused" warnings in most cases |
| 648 |
|
| 649 |
local -i try=1 |
| 650 |
while ! mbd_api status; do |
| 651 |
if [ ${try} -gt 5 ]; then |
| 652 |
logE "Can't get status even after ${try} tries" |
| 653 |
return 1 |
| 654 |
fi |
| 655 |
try+=1 |
| 656 |
sleep 1 |
| 657 |
done |
| 658 |
} |
| 659 |
|
| 660 |
mbd_run:600:setup() |
| 661 |
{ |
| 662 |
mbd_api setup ${MBD_SETUP_COMMON} ${MBD_SETUP} |
| 663 |
} |
| 664 |
|
| 665 |
mbd_run:601:spameventsqueue() |
| 666 |
{ |
| 667 |
local i |
| 668 |
for i in {1..50}; do |
| 669 |
mbd_events >>/dev/null 2>&1 & |
| 670 |
done |
| 671 |
sleep 10 # Wait some time to let all clients build up network connection |
| 672 |
curl --max-time 5 --retry 1 --silent --output /dev/null "${MBD_HTTPD_URL}" # If it does not reply in time, assume error: twisted would stall (TCP connect yes, but no handling) |
| 673 |
mbd_service restart # This should terminate all clients |
| 674 |
sleep 10 # Be sure mini-buildd is up again for following tests |
| 675 |
} |
| 676 |
|
| 677 |
mbd_run:605:keyring-packages() |
| 678 |
{ |
| 679 |
mbd_api keyring_packages |
| 680 |
} |
| 681 |
|
| 682 |
mbd_run:610:test-packages() |
| 683 |
{ |
| 684 |
mbd_api test_packages --with-check |
| 685 |
} |
| 686 |
|
| 687 |
mbd_run:615:testsuite-packages() |
| 688 |
{ |
| 689 |
local location="./share/testsuite-packages/" dst changes |
| 690 |
|
| 691 |
# Clean out possible debris |
| 692 |
(cd "${location}" && git clean -x -d -f) |
| 693 |
|
| 694 |
# Rebuild DSCs when needed |
| 695 |
for dst in $(find ${location} -maxdepth 1 -mindepth 1 -type d); do |
| 696 |
local source=$(basename "${dst}") |
| 697 |
[ -z "${1}" ] || [ "${source}" == "${1}" ] || continue |
| 698 |
sed --in-place --expression="1 s/sid/${MBD_CODENAME}/" --expression="1 s/~SID/${MBD_CODEVERSION}/" "${dst}/debian/changelog" |
| 699 |
( |
| 700 |
cd ${dst} |
| 701 |
dpkg-buildpackage -us -uc |
| 702 |
) |
| 703 |
sed --in-place --expression="1 s/${MBD_CODENAME}/sid/" --expression="1 s/${MBD_CODEVERSION}/~SID/" "${dst}/debian/changelog" |
| 704 |
done |
| 705 |
|
| 706 |
# Tests: Upload and expect event |
| 707 |
for changes in ${location}/*${1}*.changes; do |
| 708 |
local after=$(date --rfc-3339=ns) |
| 709 |
local source=$(basename ${changes} | cut -d'_' -f1) |
| 710 |
local expect=$(cut -d'-' -f1 <<<${source} | tr '[:lower:]' '[:upper:]') |
| 711 |
local failOn="FAILED REJECTED" |
| 712 |
case ${expect} in |
| 713 |
REJECTED) |
| 714 |
failOn="INSTALLED BUILT" |
| 715 |
;; |
| 716 |
esac |
| 717 |
[ -z "${1}" ] || [ "${source}" == "${1}" ] || continue |
| 718 |
logI "Testing ${source} (expect=${expect})" |
| 719 |
|
| 720 |
# Try package removal (make test not fail if it's already installed from a previous run) |
| 721 |
if [ "${expect}" == "INSTALLED" ]; then |
| 722 |
mbd_api remove "${source}" "${MBD_CODENAME}-test-unstable" --without-rollback >/dev/null || true |
| 723 |
fi |
| 724 |
|
| 725 |
case ${source} in |
| 726 |
installed-portext) |
| 727 |
cp "${location}/installed-portext_1.0.0.dsc" "${location}/installed-portext_1.0.0.tar.gz" "/tmp/" |
| 728 |
mbd_api port_ext --allow-unauthenticated "file:///tmp/installed-portext_1.0.0.dsc" "${MBD_CODENAME}-test-unstable" |
| 729 |
;; |
| 730 |
*) |
| 731 |
# Standard dput, dput-ng can't do ftps. |
| 732 |
mbd_dput "${changes}" |
| 733 |
;; |
| 734 |
esac |
| 735 |
mbd_events --source="${source}" --after="${after}" --exit-on ${expect} --fail-on ${failOn} |
| 736 |
if [ "${expect}" == "INSTALLED" ]; then |
| 737 |
case ${source} in |
| 738 |
installed-port) |
| 739 |
# Try to check for success on internal ports where possible |
| 740 |
local c codenames=$(mbd_codenames) # Available codenames on running mbd |
| 741 |
for c in bookworm bullseye; do # Needs to be same as in installed-port's ``changelog`` |
| 742 |
# Note: On rebuilds (c == MBD_CODENAME), the events check won't work (i.e., would always immediately succeed on the original package) |
| 743 |
if [ "${c}" != "${MBD_CODENAME}" ] && grep "^${c}\$" <<<${codenames}; then |
| 744 |
logI "Waiting for internal ${c} port..." |
| 745 |
mbd_events --source "${source}" --distribution "${c}-test-unstable" --after "${after}" --exit-on ${expect} --fail-on ${failOn} |
| 746 |
mbd_api remove "${source}" "${c}-test-unstable" --without-rollback |
| 747 |
else |
| 748 |
logW "Skipping check for internal ${c} port (rebuilt or codename not available)" |
| 749 |
fi |
| 750 |
done |
| 751 |
;; |
| 752 |
esac |
| 753 |
fi |
| 754 |
done |
| 755 |
} |
| 756 |
|
| 757 |
mbd_run:620:build-ourselves() |
| 758 |
{ |
| 759 |
local after=$(date --rfc-3339=ns) |
| 760 |
local changes=$(cat "${MBD_LAST_CHANGES}" || mbd_changes) |
| 761 |
mbd_dput ${changes} |
| 762 |
mbd_events --source="mini-buildd" --distribution="${MBD_CODENAME}-test-experimental" --after="${after}" --exit-on INSTALLED --fail-on FAILED REJECTED |
| 763 |
} |
| 764 |
|
| 765 |
mbd_run:625:build-migrate-remove() |
| 766 |
{ |
| 767 |
__build_migrate() |
| 768 |
{ |
| 769 |
local after=$(date --rfc-3339=ns) |
| 770 |
mbd_mbd-test-cpp |
| 771 |
mbd_events --source="mbd-test-cpp" --distribution="${MBD_CODENAME}-test-unstable" --after="${after}" --exit-on INSTALLED --fail-on FAILED REJECTED |
| 772 |
|
| 773 |
mbd_api migrate --full "mbd-test-cpp" "${MBD_CODENAME}-test-unstable" |
| 774 |
mbd_events --source="mbd-test-cpp" --distribution="${MBD_CODENAME}-test-testing" --after="${after}" --exit-on MIGRATED |
| 775 |
mbd_events --source="mbd-test-cpp" --distribution="${MBD_CODENAME}-test-stable" --after="${after}" --exit-on MIGRATED |
| 776 |
} |
| 777 |
__build_migrate |
| 778 |
__build_migrate |
| 779 |
|
| 780 |
# Build && migrated twice: There should be rollback0 for all three suites |
| 781 |
local ls=$(mbd_api ls "mbd-test-cpp") |
| 782 |
grep "${MBD_CODENAME}-test-unstable-rollback0" <<<${ls} |
| 783 |
grep "${MBD_CODENAME}-test-testing-rollback0" <<<${ls} |
| 784 |
grep "${MBD_CODENAME}-test-stable-rollback0" <<<${ls} |
| 785 |
|
| 786 |
# Test removals (don't remove stable as we do test installs later) |
| 787 |
mbd_api remove "mbd-test-cpp" "${MBD_CODENAME}-test-unstable-rollback0" |
| 788 |
mbd_api remove "mbd-test-cpp" "${MBD_CODENAME}-test-unstable" |
| 789 |
} |
| 790 |
|
| 791 |
mbd_run:630:bogus-ftp-uploads() |
| 792 |
{ |
| 793 |
local -i ftp_port=$((MBD_HTTPD_PORT+1)) |
| 794 |
# test packager.py bogus upload handling |
| 795 |
_upload() |
| 796 |
{ |
| 797 |
timeout 5 ftp -inv ${MBD_HOSTNAME} ${ftp_port} <<EOF || true # timeout, true: openssl 3 'unexpected eof while reading' from pyftpd -- still receives the file, but won't close connection, making this halt |
| 798 |
user "anonymous" "dummypass" |
| 799 |
cd incoming/ |
| 800 |
binary |
| 801 |
put "${1}" "$(basename "${1}")" |
| 802 |
bye |
| 803 |
EOF |
| 804 |
} |
| 805 |
|
| 806 |
# Broken but basically acceptable changes. This should log event REJECTED. |
| 807 |
local reject="/tmp/reject.changes" |
| 808 |
cat <<EOF >"${reject}" |
| 809 |
Distribution: kaputt |
| 810 |
Version: 1.2.3 |
| 811 |
Source: kaputt |
| 812 |
Architecture: kaputt |
| 813 |
EOF |
| 814 |
local after=$(date --rfc-3339=ns) |
| 815 |
_upload "${reject}" |
| 816 |
mbd_events --source="kaputt" --after="${after}" --exit-on REJECTED |
| 817 |
|
| 818 |
# Completely unparsable changes. This file should just be removed by mini-buildd (no feasible way to test that automatically, though). |
| 819 |
local invalid="/tmp/invalid.changes" |
| 820 |
cat <<EOF >"${invalid}" |
| 821 |
kaputt |
| 822 |
EOF |
| 823 |
_upload "${invalid}" |
| 824 |
} |
| 825 |
|
| 826 |
mbd_run:635:api-getters() |
| 827 |
{ |
| 828 |
# Consume |
| 829 |
mbd_api status |
| 830 |
mbd_api pub_key |
| 831 |
mbd_api dput_conf |
| 832 |
mbd_api sources_list |
| 833 |
mbd_api ls "mbd-test-cpp" --repositories "test" |
| 834 |
mbd_api show "mbd-test-cpp" --repositories "test" --codenames "${MBD_CODENAME}" |
| 835 |
|
| 836 |
# Maintain |
| 837 |
mbd_api start |
| 838 |
mbd_api uploaders |
| 839 |
mbd_api snapshot_ls "${MBD_CODENAME}-test-stable" |
| 840 |
} |
| 841 |
|
| 842 |
_test_install() |
| 843 |
{ |
| 844 |
on_exit sudo dpkg --purge "${1}" |
| 845 |
sudo apt-get "${@:2}" install "${1}" |
| 846 |
} |
| 847 |
|
| 848 |
mbd_run:640:apt-tofu-bootstrap() |
| 849 |
{ |
| 850 |
sudo MBD_CODENAME=${MBD_CODENAME} ./src/mini-buildd-bootstrap-apt "${MBD_HTTPD_URL}" auto |
| 851 |
|
| 852 |
local sources_list="/etc/apt/sources.list.d/mbd-testsuite.list" |
| 853 |
on_exit sudo rm -v -f "${sources_list}" |
| 854 |
on_exit sudo apt-get update |
| 855 |
|
| 856 |
# Add sources.list && test apt update |
| 857 |
mbd_api sources_list --codenames "${MBD_CODENAME}" | sudo tee "${sources_list}" |
| 858 |
sudo apt-get update |
| 859 |
|
| 860 |
# install previously built test package |
| 861 |
_test_install mbd-test-cpp |
| 862 |
} |
| 863 |
|
| 864 |
mbd_run:645:apt-snapshot() |
| 865 |
{ |
| 866 |
# Test snapshot gen, del and sources.list |
| 867 |
local snap="TESTSUITE_$(date --iso-8601=seconds)" |
| 868 |
mbd_api snapshot_create "${MBD_CODENAME}-test-stable" "${snap}" |
| 869 |
mbd_api snapshot_delete "${MBD_CODENAME}-test-stable" "${snap}" |
| 870 |
mbd_api snapshot_create "${MBD_CODENAME}-test-stable" "${snap}" |
| 871 |
mbd_api snapshot_ls "${MBD_CODENAME}-test-stable" |
| 872 |
mbd_api sources_list --repositories "test" --codenames "${MBD_CODENAME}" --suites "stable" --snapshot "${snap}" |
| 873 |
} |
| 874 |
|
| 875 |
mbd_test_urls() |
| 876 |
{ |
| 877 |
local uri |
| 878 |
for uri in \ |
| 879 |
/mini_buildd/ \ |
| 880 |
/mini_buildd/api/ \ |
| 881 |
/mini_buildd/api/status/?output=html \ |
| 882 |
/mini_buildd/api/search/?pattern=mbd\&output=html \ |
| 883 |
/mini_buildd/api/ls/?repository=test\&codename=${MBD_CODENAME}\&source=mbd-test-cpp\&output=html \ |
| 884 |
/mini_buildd/api/show/?repository=test\&codename=${MBD_CODENAME}\&source=mbd-test-cpp\&output=html \ |
| 885 |
$(for route in events repositories builders crontab manual api setup sitemap; do printf "/mini_buildd/%s/ " "${route}"; done) \ |
| 886 |
/mini_buildd/repositories/test/dists/${MBD_CODENAME}-test-stable/ \ |
| 887 |
$(for route in events repositories; do printf "/mini_buildd/%s-dir/ " "${route}"; done) \ |
| 888 |
/accounts/login/ \ |
| 889 |
/admin/ \ |
| 890 |
/admin/mini_buildd/source/ \ |
| 891 |
/static/manual/index.html \ |
| 892 |
/static/manual/abstract.html \ |
| 893 |
/static/manual/administrator.html \ |
| 894 |
/static/manual/consumer.html \ |
| 895 |
/static/manual/developer.html \ |
| 896 |
/static/manual/roadmap.html \ |
| 897 |
/static/manual/auto_admonitions.html \ |
| 898 |
; do |
| 899 |
printf "%s\n" "${MBD_HTTPD_URL}${uri}" |
| 900 |
done |
| 901 |
} |
| 902 |
|
| 903 |
mbd_run:700:tidy() |
| 904 |
{ |
| 905 |
local url html="./devel.tidy.html" |
| 906 |
while IFS="" read -r url; do |
| 907 |
logI "Testing HTML: ${html} from '${url}'" |
| 908 |
mbd_curl-admin --fail --silent --location --output "${html}" "${url}" |
| 909 |
|
| 910 |
local mute="" |
| 911 |
mute+="PROPRIETARY_ATTRIBUTE," # With django 3, tidy complains about 'proprietary attribute "autocapitalize"', but this seems to be valid html5. |
| 912 |
mute+="TRIM_EMPTY_ELEMENT," # Just too picky. |
| 913 |
mute+="TAG_NOT_ALLOWED_IN," # Cries about "<script> isn't allowed in <table>", but this isn't true for HTML5 afaik. And me wants! |
| 914 |
mute+="MISSING_ENDTAG_BEFORE,INSERTING_TAG," # django 4.2(admin/color_theme_toggle.html) uses <button><div/></button> (not sure if valid, but we should not fail on django upstream) |
| 915 |
mute+="BLANK_TITLE_ELEMENT," # New w/ tidy 5.8.0, no fix known |
| 916 |
if grep --quiet "/manual/" <<<${url}; then |
| 917 |
mute+="MISSING_ATTR_VALUE," # HTML created by sphinx (manual) has this a lot, and seems nothing we can do about it. |
| 918 |
mute+="UNKNOWN_ELEMENT,DISCARDING_UNEXPECTED," # sphinx 7.3.7 seems to use new "search" element |
| 919 |
fi |
| 920 |
|
| 921 |
local result=$(tidy --quiet yes --force-output yes --gnu-emacs yes --mute-id yes -output /dev/null --mute ${mute} "${html}" 2>&1) |
| 922 |
|
| 923 |
if [ -n "${result}" ]; then |
| 924 |
printf "Tidy says: \"%s\"\n" "${result}" |
| 925 |
return 1 |
| 926 |
fi |
| 927 |
done <<<$(mbd_test_urls) |
| 928 |
} |
| 929 |
|
| 930 |
mbd_run:710:httpd-benchmark() |
| 931 |
{ |
| 932 |
siege --benchmark --time=10s --file=<(mbd_test_urls) --json-output |
| 933 |
} |
| 934 |
|
| 935 |
mbd_run:720:debmirror() |
| 936 |
{ |
| 937 |
mbd_api debmirror --codenames "${MBD_CODENAME}" --suites stable |
| 938 |
} |
| 939 |
|
| 940 |
mbd_sequence() # [<levelRegex>=00] [<name>] <hr> |
| 941 |
{ |
| 942 |
local levelRegex="${1:-00}" name="${2}" hr="${3}" |
| 943 |
for func in $(declare -F | cut -d" " -f3- | grep "^mbd_run:${levelRegex}:${name}" | sort || true); do |
| 944 |
if [ -n "${hr}" ]; then |
| 945 |
printf "%s " "$(cut -d: -f3 <<<${func})" |
| 946 |
else |
| 947 |
printf "%s " "${func}" |
| 948 |
fi |
| 949 |
done |
| 950 |
} |
| 951 |
|
| 952 |
mbd_skip() # <func> |
| 953 |
{ |
| 954 |
local short=$(cut -d: -f3 <<<${1}) s |
| 955 |
for s in ${MBD_SKIP}; do |
| 956 |
[ "${short}" != "${s}" ] || return 0 |
| 957 |
done |
| 958 |
return 1 |
| 959 |
} |
| 960 |
|
| 961 |
mbd_run() # [<levelRegex>=00] [<name>] [<customArgs>...] |
| 962 |
{ |
| 963 |
local levelRegex="${1}" name="${2}" |
| 964 |
local -a info=() |
| 965 |
local func totalStartStamp=$(date +%s) |
| 966 |
local -i count=0 |
| 967 |
for func in $(mbd_sequence "${levelRegex}" "${name}"); do |
| 968 |
MBD_RUNNING="${func}" |
| 969 |
local startStamp=$(date +%s) |
| 970 |
if mbd_skip "${func}"; then |
| 971 |
logI "SKIPPED (in skip list: ${MBD_SKIP})" |
| 972 |
else |
| 973 |
logI "Starting..." |
| 974 |
${func} "${@:3}" |
| 975 |
logI "Completed." |
| 976 |
fi |
| 977 |
MBD_RUNNING="" |
| 978 |
|
| 979 |
count+=1 |
| 980 |
info+=("$(printf "OK (%03d seconds): %s" "$(($(date +%s) - startStamp))" "${func}")") |
| 981 |
done |
| 982 |
if ((count <= 0)); then |
| 983 |
logE "No runs for this sequence filter: ${levelRegex} ${name}" |
| 984 |
return 1 |
| 985 |
fi |
| 986 |
logI "Sequence results (%03d seconds):" "$(($(date +%s) - totalStartStamp))" |
| 987 |
logI "%s" "${info[@]}" |
| 988 |
logI "OK, %s runs succeeded ($(date))." "${count}" |
| 989 |
} |
| 990 |
|
| 991 |
# Shortcuts |
| 992 |
declare -A MBD_RUN_SHORTCUTS=( |
| 993 |
["check"]='1..' |
| 994 |
["update"]='\(300\|305\|510\|515\)' |
| 995 |
["updatecheck"]='\(1..\|300\|305\|510\|515\)' |
| 996 |
["updatetest"]='\([12]..\|300\|305\|510\|515\|4..\)' |
| 997 |
["updatetestall"]='.*' |
| 998 |
) |
| 999 |
# We can't iterate through the associative array in the given order later, so we at least want a sorted key list as helper |
| 1000 |
MBD_RUN_SHORTCUTS_SORTED="$(printf '%s\n' "${!MBD_RUN_SHORTCUTS[@]}" | sort -n)" |
| 1001 |
|
| 1002 |
# Some extra targets |
| 1003 |
mbd_service.log() |
| 1004 |
{ |
| 1005 |
if systemctl is-system-running; then |
| 1006 |
journalctl --unit mini-buildd --follow |
| 1007 |
else |
| 1008 |
tail --follow=name --retry /var/log/mini-buildd.log | ts |
| 1009 |
fi |
| 1010 |
} |
| 1011 |
|
| 1012 |
mbd_access.log() |
| 1013 |
{ |
| 1014 |
tail --follow=name --retry /var/lib/mini-buildd/var/log/access.log |
| 1015 |
} |
| 1016 |
|
| 1017 |
mbd_grep() |
| 1018 |
{ |
| 1019 |
grep --recursive \ |
| 1020 |
--exclude-dir=".git" \ |
| 1021 |
--exclude="changelog" \ |
| 1022 |
--binary-files="without-match" "${@}" |
| 1023 |
} |
| 1024 |
|
| 1025 |
mbd_pylintgeneratedmembers() |
| 1026 |
{ |
| 1027 |
# Hack to update identifiers with false-positive "has no xxx member" error (mostly django). |
| 1028 |
# Implies you already have a this as _last_ option in ``setup.cfg``. |
| 1029 |
{ |
| 1030 |
for o in $(pylint src/mini_buildd/ | grep "has no.*member" | cut -d"'" -f4 | sort | uniq); do |
| 1031 |
printf "\t${o},\n" |
| 1032 |
done |
| 1033 |
} >>setup.cfg |
| 1034 |
} |
| 1035 |
|
| 1036 |
# Try to reproduce deadlock |
| 1037 |
# This try w/ events does not yet reproduce the deadlock; saving for later. |
| 1038 |
mbd_deadlock() |
| 1039 |
{ |
| 1040 |
local -i count=0 |
| 1041 |
while mbd_events --source mbd-test-cpp --stop >/dev/null; do |
| 1042 |
# echo ${count}; count+=1; |
| 1043 |
sleep 0; |
| 1044 |
done |
| 1045 |
} |
| 1046 |
|
| 1047 |
mbd_bash-completion() |
| 1048 |
{ |
| 1049 |
local non_runners=$(declare -F | cut -d" " -f3 | grep "^mbd_" | grep --invert-match "^mbd_run" | cut -b5-) |
| 1050 |
local profiles="${!MBD_PROFILES[*]}" |
| 1051 |
printf "complete -W '%s' ./devel\n" "$(printf '%s ' "${!MBD_RUN_SHORTCUTS[@]}") $(mbd_sequence "..." "" "hr") ${non_runners} ${profiles} ${MBD_BROWSERS}" |
| 1052 |
} |
| 1053 |
|
| 1054 |
mbd_run-lintian() |
| 1055 |
{ |
| 1056 |
mbd_run:300:changelog |
| 1057 |
mbd_run:305:build |
| 1058 |
mbd_run:400:lintian |
| 1059 |
} |
| 1060 |
|
| 1061 |
# run-profile [callgrind]: Run with python profiling enabled && show stats after termination |
| 1062 |
# * needs ``python3-yappi`` (for multithreading) |
| 1063 |
# * optional: will use ``kcachegrind`` if called with ``callgrind`` argument |
| 1064 |
mbd_run-profile() |
| 1065 |
{ |
| 1066 |
local format=${1:-pstat} profile=~mini-buildd/mini-buildd.profile |
| 1067 |
mbd_service stop |
| 1068 |
sudo su - mini-buildd --command="yappi --output-format=${format} --output-file=${profile} /usr/sbin/mini-buildd" || true |
| 1069 |
logI "Profiling data: ${profile} (${format})" |
| 1070 |
case ${format} in |
| 1071 |
"pstat") |
| 1072 |
python3 -c "import pstats; pstats.Stats('${profile}').sort_stats('cumtime').print_stats('mini_buildd');" | less |
| 1073 |
;; |
| 1074 |
"callgrind") |
| 1075 |
kcachegrind ${profile} |
| 1076 |
;; |
| 1077 |
esac |
| 1078 |
} |
| 1079 |
|
| 1080 |
main() |
| 1081 |
{ |
| 1082 |
if [ -z "${1}" ]; then |
| 1083 |
local p="./$(basename "${0}")" |
| 1084 |
cat <<EOF |
| 1085 |
Usage: [<env>] ${p} <shortcut-or-runner-or-special> | run <groupRegex><levelRegex>${_reset} [<customArgs>...] |
| 1086 |
|
| 1087 |
Helper script to develop/debug mini-buildd. |
| 1088 |
|
| 1089 |
Prerequisites |
| 1090 |
------------- |
| 1091 |
* Only run in *DEVELOPMENT* systems (it does DO THINGS, and maybe w/o asking) |
| 1092 |
* 'sudo' should be configured |
| 1093 |
* Known-Pre-Depends: lsb-release |
| 1094 |
* eval \$(./devel bash-completion) is your friend |
| 1095 |
|
| 1096 |
Test suite configuration via environment |
| 1097 |
---------------------------------------- |
| 1098 |
$(for VAR in ${MBD_CONFIG}; do VAR_DESC="${VAR}_DESC"; printf "%s\n %s\n" "${VAR}='${!VAR}'" "${!VAR_DESC}"; done) |
| 1099 |
|
| 1100 |
HINT: Combinable profiles exist for common scenarios: ./devel profile _<TAB> |
| 1101 |
|
| 1102 |
Shortcut commands |
| 1103 |
----------------- |
| 1104 |
$(for s in ${MBD_RUN_SHORTCUTS_SORTED}; do printf " ${_it}${p} %-15s${_reset}: %s(%s)\n" "${s}" "$(mbd_sequence "${MBD_RUN_SHORTCUTS[${s}]}" "" "hr")" "${MBD_RUN_SHORTCUTS[${s}]}"; done) |
| 1105 |
|
| 1106 |
Runners |
| 1107 |
------- |
| 1108 |
${p} $(mbd_sequence "..." "" "hr" | tr " " "|") |
| 1109 |
|
| 1110 |
Examples |
| 1111 |
-------- |
| 1112 |
${_it}Install snapshot Debian packages${_reset}: |
| 1113 |
$ ./devel prepare-system # Only 1st time |
| 1114 |
$ ./devel update |
| 1115 |
|
| 1116 |
${_it}Run full testuite (leaves you with a fresh mini-buildd fully set up, any previous install WILL BE PURGED!)${_reset}: |
| 1117 |
$ ./devel updatetestall # w/ default setup |
| 1118 |
$ ./devel profile _all updatetestall # w/ all known sources |
| 1119 |
|
| 1120 |
${_it}Local setup to test remotes (on two systems w/ same network stack like chroots)${_reset}: |
| 1121 |
$ ./devel profile _debian _remote_hostname update # chroot 0 |
| 1122 |
$ ./devel profile _debian _remote_localhost update # chroot 1 |
| 1123 |
|
| 1124 |
${_it}Follow service logs${_reset}: |
| 1125 |
$ ${p} service.log : Follow daemon logs. |
| 1126 |
|
| 1127 |
${_it}Run all fully-automated tests possible${_reset}: |
| 1128 |
$ ${p} supertestall |
| 1129 |
EOF |
| 1130 |
else |
| 1131 |
check_devsys |
| 1132 |
local shortcut="${MBD_RUN_SHORTCUTS[${1}]}" |
| 1133 |
if [ -n "${shortcut}" ]; then |
| 1134 |
mbd_run "${shortcut}" "" "${@:2}" |
| 1135 |
else |
| 1136 |
local func="mbd_${1}" # direct function |
| 1137 |
if [ "$(type -t "${func}")" = "function" ]; then |
| 1138 |
${func} "${@:2}" |
| 1139 |
else |
| 1140 |
mbd_run '.*' "${1}" "${@:2}" |
| 1141 |
fi |
| 1142 |
fi |
| 1143 |
fi |
| 1144 |
} |
| 1145 |
|
| 1146 |
main "${@}" |