Debian Codegraph @

contrib/automation/hgautomation/linux.py

raw
1 # linux.py - Linux specific automation functionality
2 #
3 # Copyright 2019 Gregory Szorc <[email protected]>
4 #
5 # This software may be used and distributed according to the terms of the
6 # GNU General Public License version 2 or any later version.
7
8 # no-check-code because Python 3 native.
9
10 import os
11 import pathlib
12 import shlex
13 import subprocess
14 import tempfile
15
16 from .ssh import exec_command
17
18
19 # Linux distributions that are supported.
20 DISTROS = {
21 'debian9',
22 'debian10',
23 'ubuntu18.04',
24 'ubuntu19.04',
25 }
26
27 INSTALL_PYTHONS = r'''
28 PYENV3_VERSIONS="3.8.10 3.9.5 pypy3.5-7.0.0 pypy3.6-7.3.3 pypy3.7-7.3.3"
29
30 git clone https://github.com/pyenv/pyenv.git /hgdev/pyenv
31 pushd /hgdev/pyenv
32 git checkout 328fd42c3a2fbf14ae46dae2021a087fe27ba7e2
33 popd
34
35 export PYENV_ROOT="/hgdev/pyenv"
36 export PATH="$PYENV_ROOT/bin:$PATH"
37
38 # pip 19.2.3.
39 PIP_SHA256=57e3643ff19f018f8a00dfaa6b7e4620e3c1a7a2171fd218425366ec006b3bfe
40 wget -O get-pip.py --progress dot:mega https://github.com/pypa/get-pip/raw/309a56c5fd94bd1134053a541cb4657a4e47e09d/get-pip.py
41 echo "${PIP_SHA256} get-pip.py" | sha256sum --check -
42
43 VIRTUALENV_SHA256=f78d81b62d3147396ac33fc9d77579ddc42cc2a98dd9ea38886f616b33bc7fb2
44 VIRTUALENV_TARBALL=virtualenv-16.7.5.tar.gz
45 wget -O ${VIRTUALENV_TARBALL} --progress dot:mega https://files.pythonhosted.org/packages/66/f0/6867af06d2e2f511e4e1d7094ff663acdebc4f15d4a0cb0fed1007395124/${VIRTUALENV_TARBALL}
46 echo "${VIRTUALENV_SHA256} ${VIRTUALENV_TARBALL}" | sha256sum --check -
47
48 for v in ${PYENV3_VERSIONS}; do
49 pyenv install -v ${v}
50 ${PYENV_ROOT}/versions/${v}/bin/python get-pip.py
51
52 case ${v} in
53 3.5.*)
54 REQUIREMENTS=requirements-py3.5.txt
55 ;;
56 pypy3.5*)
57 REQUIREMENTS=requirements-py3.5.txt
58 ;;
59 *)
60 REQUIREMENTS=requirements-py3.txt
61 ;;
62 esac
63
64 ${PYENV_ROOT}/versions/${v}/bin/pip install -r /hgdev/${REQUIREMENTS}
65 done
66
67 pyenv global ${PYENV3_VERSIONS} system
68 '''.lstrip().replace(
69 '\r\n', '\n'
70 )
71
72 INSTALL_PYOXIDIZER = r'''
73 PYOXIDIZER_VERSION=0.16.0
74 PYOXIDIZER_SHA256=8875471c270312fbb934007fd30f65f1904cc0f5da6188d61c90ed2129b9f9c1
75 PYOXIDIZER_URL=https://github.com/indygreg/PyOxidizer/releases/download/pyoxidizer%2F${PYOXIDIZER_VERSION}/pyoxidizer-${PYOXIDIZER_VERSION}-linux_x86_64.zip
76
77 wget -O pyoxidizer.zip --progress dot:mega ${PYOXIDIZER_URL}
78 echo "${PYOXIDIZER_SHA256} pyoxidizer.zip" | sha256sum --check -
79
80 unzip pyoxidizer.zip
81 chmod +x pyoxidizer
82 sudo mv pyoxidizer /usr/local/bin/pyoxidizer
83 '''
84
85 INSTALL_RUST = r'''
86 RUSTUP_INIT_SHA256=a46fe67199b7bcbbde2dcbc23ae08db6f29883e260e23899a88b9073effc9076
87 wget -O rustup-init --progress dot:mega https://static.rust-lang.org/rustup/archive/1.18.3/x86_64-unknown-linux-gnu/rustup-init
88 echo "${RUSTUP_INIT_SHA256} rustup-init" | sha256sum --check -
89
90 chmod +x rustup-init
91 sudo -H -u hg -g hg ./rustup-init -y
92 sudo -H -u hg -g hg /home/hg/.cargo/bin/rustup install 1.41.1 1.52.0
93 sudo -H -u hg -g hg /home/hg/.cargo/bin/rustup component add clippy
94 '''
95
96
97 BOOTSTRAP_VIRTUALENV = r'''
98 /usr/bin/virtualenv /hgdev/venv-bootstrap
99
100 HG_SHA256=35fc8ba5e0379c1b3affa2757e83fb0509e8ac314cbd9f1fd133cf265d16e49f
101 HG_TARBALL=mercurial-5.1.1.tar.gz
102
103 wget -O ${HG_TARBALL} --progress dot:mega https://www.mercurial-scm.org/release/${HG_TARBALL}
104 echo "${HG_SHA256} ${HG_TARBALL}" | sha256sum --check -
105
106 /hgdev/venv-bootstrap/bin/pip install ${HG_TARBALL}
107 '''.lstrip().replace(
108 '\r\n', '\n'
109 )
110
111
112 BOOTSTRAP_DEBIAN = (
113 r'''
114 #!/bin/bash
115
116 set -ex
117
118 DISTRO=`grep DISTRIB_ID /etc/lsb-release | awk -F= '{{print $2}}'`
119 DEBIAN_VERSION=`cat /etc/debian_version`
120 LSB_RELEASE=`lsb_release -cs`
121
122 sudo /usr/sbin/groupadd hg
123 sudo /usr/sbin/groupadd docker
124 sudo /usr/sbin/useradd -g hg -G sudo,docker -d /home/hg -m -s /bin/bash hg
125 sudo mkdir /home/hg/.ssh
126 sudo cp ~/.ssh/authorized_keys /home/hg/.ssh/authorized_keys
127 sudo chown -R hg:hg /home/hg/.ssh
128 sudo chmod 700 /home/hg/.ssh
129 sudo chmod 600 /home/hg/.ssh/authorized_keys
130
131 cat << EOF | sudo tee /etc/sudoers.d/90-hg
132 hg ALL=(ALL) NOPASSWD:ALL
133 EOF
134
135 sudo apt-get update
136 sudo DEBIAN_FRONTEND=noninteractive apt-get -yq dist-upgrade
137
138 # Install packages necessary to set up Docker Apt repo.
139 sudo DEBIAN_FRONTEND=noninteractive apt-get -yq install --no-install-recommends \
140 apt-transport-https \
141 gnupg
142
143 cat > docker-apt-key << EOF
144 -----BEGIN PGP PUBLIC KEY BLOCK-----
145
146 mQINBFit2ioBEADhWpZ8/wvZ6hUTiXOwQHXMAlaFHcPH9hAtr4F1y2+OYdbtMuth
147 lqqwp028AqyY+PRfVMtSYMbjuQuu5byyKR01BbqYhuS3jtqQmljZ/bJvXqnmiVXh
148 38UuLa+z077PxyxQhu5BbqntTPQMfiyqEiU+BKbq2WmANUKQf+1AmZY/IruOXbnq
149 L4C1+gJ8vfmXQt99npCaxEjaNRVYfOS8QcixNzHUYnb6emjlANyEVlZzeqo7XKl7
150 UrwV5inawTSzWNvtjEjj4nJL8NsLwscpLPQUhTQ+7BbQXAwAmeHCUTQIvvWXqw0N
151 cmhh4HgeQscQHYgOJjjDVfoY5MucvglbIgCqfzAHW9jxmRL4qbMZj+b1XoePEtht
152 ku4bIQN1X5P07fNWzlgaRL5Z4POXDDZTlIQ/El58j9kp4bnWRCJW0lya+f8ocodo
153 vZZ+Doi+fy4D5ZGrL4XEcIQP/Lv5uFyf+kQtl/94VFYVJOleAv8W92KdgDkhTcTD
154 G7c0tIkVEKNUq48b3aQ64NOZQW7fVjfoKwEZdOqPE72Pa45jrZzvUFxSpdiNk2tZ
155 XYukHjlxxEgBdC/J3cMMNRE1F4NCA3ApfV1Y7/hTeOnmDuDYwr9/obA8t016Yljj
156 q5rdkywPf4JF8mXUW5eCN1vAFHxeg9ZWemhBtQmGxXnw9M+z6hWwc6ahmwARAQAB
157 tCtEb2NrZXIgUmVsZWFzZSAoQ0UgZGViKSA8ZG9ja2VyQGRvY2tlci5jb20+iQI3
158 BBMBCgAhBQJYrefAAhsvBQsJCAcDBRUKCQgLBRYCAwEAAh4BAheAAAoJEI2BgDwO
159 v82IsskP/iQZo68flDQmNvn8X5XTd6RRaUH33kXYXquT6NkHJciS7E2gTJmqvMqd
160 tI4mNYHCSEYxI5qrcYV5YqX9P6+Ko+vozo4nseUQLPH/ATQ4qL0Zok+1jkag3Lgk
161 jonyUf9bwtWxFp05HC3GMHPhhcUSexCxQLQvnFWXD2sWLKivHp2fT8QbRGeZ+d3m
162 6fqcd5Fu7pxsqm0EUDK5NL+nPIgYhN+auTrhgzhK1CShfGccM/wfRlei9Utz6p9P
163 XRKIlWnXtT4qNGZNTN0tR+NLG/6Bqd8OYBaFAUcue/w1VW6JQ2VGYZHnZu9S8LMc
164 FYBa5Ig9PxwGQOgq6RDKDbV+PqTQT5EFMeR1mrjckk4DQJjbxeMZbiNMG5kGECA8
165 g383P3elhn03WGbEEa4MNc3Z4+7c236QI3xWJfNPdUbXRaAwhy/6rTSFbzwKB0Jm
166 ebwzQfwjQY6f55MiI/RqDCyuPj3r3jyVRkK86pQKBAJwFHyqj9KaKXMZjfVnowLh
167 9svIGfNbGHpucATqREvUHuQbNnqkCx8VVhtYkhDb9fEP2xBu5VvHbR+3nfVhMut5
168 G34Ct5RS7Jt6LIfFdtcn8CaSas/l1HbiGeRgc70X/9aYx/V/CEJv0lIe8gP6uDoW
169 FPIZ7d6vH+Vro6xuWEGiuMaiznap2KhZmpkgfupyFmplh0s6knymuQINBFit2ioB
170 EADneL9S9m4vhU3blaRjVUUyJ7b/qTjcSylvCH5XUE6R2k+ckEZjfAMZPLpO+/tF
171 M2JIJMD4SifKuS3xck9KtZGCufGmcwiLQRzeHF7vJUKrLD5RTkNi23ydvWZgPjtx
172 Q+DTT1Zcn7BrQFY6FgnRoUVIxwtdw1bMY/89rsFgS5wwuMESd3Q2RYgb7EOFOpnu
173 w6da7WakWf4IhnF5nsNYGDVaIHzpiqCl+uTbf1epCjrOlIzkZ3Z3Yk5CM/TiFzPk
174 z2lLz89cpD8U+NtCsfagWWfjd2U3jDapgH+7nQnCEWpROtzaKHG6lA3pXdix5zG8
175 eRc6/0IbUSWvfjKxLLPfNeCS2pCL3IeEI5nothEEYdQH6szpLog79xB9dVnJyKJb
176 VfxXnseoYqVrRz2VVbUI5Blwm6B40E3eGVfUQWiux54DspyVMMk41Mx7QJ3iynIa
177 1N4ZAqVMAEruyXTRTxc9XW0tYhDMA/1GYvz0EmFpm8LzTHA6sFVtPm/ZlNCX6P1X
178 zJwrv7DSQKD6GGlBQUX+OeEJ8tTkkf8QTJSPUdh8P8YxDFS5EOGAvhhpMBYD42kQ
179 pqXjEC+XcycTvGI7impgv9PDY1RCC1zkBjKPa120rNhv/hkVk/YhuGoajoHyy4h7
180 ZQopdcMtpN2dgmhEegny9JCSwxfQmQ0zK0g7m6SHiKMwjwARAQABiQQ+BBgBCAAJ
181 BQJYrdoqAhsCAikJEI2BgDwOv82IwV0gBBkBCAAGBQJYrdoqAAoJEH6gqcPyc/zY
182 1WAP/2wJ+R0gE6qsce3rjaIz58PJmc8goKrir5hnElWhPgbq7cYIsW5qiFyLhkdp
183 YcMmhD9mRiPpQn6Ya2w3e3B8zfIVKipbMBnke/ytZ9M7qHmDCcjoiSmwEXN3wKYI
184 mD9VHONsl/CG1rU9Isw1jtB5g1YxuBA7M/m36XN6x2u+NtNMDB9P56yc4gfsZVES
185 KA9v+yY2/l45L8d/WUkUi0YXomn6hyBGI7JrBLq0CX37GEYP6O9rrKipfz73XfO7
186 JIGzOKZlljb/D9RX/g7nRbCn+3EtH7xnk+TK/50euEKw8SMUg147sJTcpQmv6UzZ
187 cM4JgL0HbHVCojV4C/plELwMddALOFeYQzTif6sMRPf+3DSj8frbInjChC3yOLy0
188 6br92KFom17EIj2CAcoeq7UPhi2oouYBwPxh5ytdehJkoo+sN7RIWua6P2WSmon5
189 U888cSylXC0+ADFdgLX9K2zrDVYUG1vo8CX0vzxFBaHwN6Px26fhIT1/hYUHQR1z
190 VfNDcyQmXqkOnZvvoMfz/Q0s9BhFJ/zU6AgQbIZE/hm1spsfgvtsD1frZfygXJ9f
191 irP+MSAI80xHSf91qSRZOj4Pl3ZJNbq4yYxv0b1pkMqeGdjdCYhLU+LZ4wbQmpCk
192 SVe2prlLureigXtmZfkqevRz7FrIZiu9ky8wnCAPwC7/zmS18rgP/17bOtL4/iIz
193 QhxAAoAMWVrGyJivSkjhSGx1uCojsWfsTAm11P7jsruIL61ZzMUVE2aM3Pmj5G+W
194 9AcZ58Em+1WsVnAXdUR//bMmhyr8wL/G1YO1V3JEJTRdxsSxdYa4deGBBY/Adpsw
195 24jxhOJR+lsJpqIUeb999+R8euDhRHG9eFO7DRu6weatUJ6suupoDTRWtr/4yGqe
196 dKxV3qQhNLSnaAzqW/1nA3iUB4k7kCaKZxhdhDbClf9P37qaRW467BLCVO/coL3y
197 Vm50dwdrNtKpMBh3ZpbB1uJvgi9mXtyBOMJ3v8RZeDzFiG8HdCtg9RvIt/AIFoHR
198 H3S+U79NT6i0KPzLImDfs8T7RlpyuMc4Ufs8ggyg9v3Ae6cN3eQyxcK3w0cbBwsh
199 /nQNfsA6uu+9H7NhbehBMhYnpNZyrHzCmzyXkauwRAqoCbGCNykTRwsur9gS41TQ
200 M8ssD1jFheOJf3hODnkKU+HKjvMROl1DK7zdmLdNzA1cvtZH/nCC9KPj1z8QC47S
201 xx+dTZSx4ONAhwbS/LN3PoKtn8LPjY9NP9uDWI+TWYquS2U+KHDrBDlsgozDbs/O
202 jCxcpDzNmXpWQHEtHU7649OXHP7UeNST1mCUCH5qdank0V1iejF6/CfTFU4MfcrG
203 YT90qFF93M3v01BbxP+EIY2/9tiIPbrd
204 =0YYh
205 -----END PGP PUBLIC KEY BLOCK-----
206 EOF
207
208 sudo apt-key add docker-apt-key
209
210 if [ "$LSB_RELEASE" = "stretch" ]; then
211 cat << EOF | sudo tee -a /etc/apt/sources.list
212 # Need backports for clang-format-6.0
213 deb http://deb.debian.org/debian stretch-backports main
214 EOF
215 fi
216
217 if [ "$LSB_RELEASE" = "stretch" -o "$LSB_RELEASE" = "buster" ]; then
218 cat << EOF | sudo tee -a /etc/apt/sources.list
219 # Sources are useful if we want to compile things locally.
220 deb-src http://deb.debian.org/debian $LSB_RELEASE main
221 deb-src http://security.debian.org/debian-security $LSB_RELEASE/updates main
222 deb-src http://deb.debian.org/debian $LSB_RELEASE-updates main
223 deb-src http://deb.debian.org/debian $LSB_RELEASE-backports main
224
225 deb [arch=amd64] https://download.docker.com/linux/debian $LSB_RELEASE stable
226 EOF
227
228 elif [ "$DISTRO" = "Ubuntu" ]; then
229 cat << EOF | sudo tee -a /etc/apt/sources.list
230 deb [arch=amd64] https://download.docker.com/linux/ubuntu $LSB_RELEASE stable
231 EOF
232
233 fi
234
235 sudo apt-get update
236
237 PACKAGES="\
238 awscli \
239 btrfs-progs \
240 build-essential \
241 bzr \
242 clang-format-6.0 \
243 cvs \
244 darcs \
245 debhelper \
246 devscripts \
247 docker-ce \
248 dpkg-dev \
249 dstat \
250 emacs \
251 gettext \
252 git \
253 htop \
254 iotop \
255 jfsutils \
256 libbz2-dev \
257 libexpat1-dev \
258 libffi-dev \
259 libgdbm-dev \
260 liblzma-dev \
261 libncurses5-dev \
262 libnss3-dev \
263 libreadline-dev \
264 libsqlite3-dev \
265 libssl-dev \
266 netbase \
267 ntfs-3g \
268 nvme-cli \
269 pyflakes3 \
270 pylint3 \
271 python3-boto3 \
272 python3-dev \
273 python3-docutils \
274 python3-fuzzywuzzy \
275 python3-pygments \
276 python3-vcr \
277 python3-venv \
278 rsync \
279 sqlite3 \
280 subversion \
281 tcl-dev \
282 tk-dev \
283 tla \
284 unzip \
285 uuid-dev \
286 vim \
287 virtualenv \
288 wget \
289 xfsprogs \
290 zip \
291 zlib1g-dev"
292
293 if [ "LSB_RELEASE" = "stretch" ]; then
294 PACKAGES="$PACKAGES linux-perf"
295 elif [ "$DISTRO" = "Ubuntu" ]; then
296 PACKAGES="$PACKAGES linux-tools-common"
297 fi
298
299 # Monotone only available in older releases.
300 if [ "$LSB_RELEASE" = "stretch" -o "$LSB_RELEASE" = "xenial" ]; then
301 PACKAGES="$PACKAGES monotone"
302 fi
303
304 sudo DEBIAN_FRONTEND=noninteractive apt-get -yq install --no-install-recommends $PACKAGES
305
306 # Create clang-format symlink so test harness finds it.
307 sudo update-alternatives --install /usr/bin/clang-format clang-format \
308 /usr/bin/clang-format-6.0 1000
309
310 sudo mkdir /hgdev
311 # Will be normalized to hg:hg later.
312 sudo chown `whoami` /hgdev
313
314 {install_rust}
315 {install_pyoxidizer}
316
317 cp requirements-*.txt /hgdev/
318
319 # Disable the pip version check because it uses the network and can
320 # be annoying.
321 cat << EOF | sudo tee -a /etc/pip.conf
322 [global]
323 disable-pip-version-check = True
324 EOF
325
326 {install_pythons}
327 {bootstrap_virtualenv}
328
329 /hgdev/venv-bootstrap/bin/hg clone https://www.mercurial-scm.org/repo/hg /hgdev/src
330
331 # Mark the repo as non-publishing.
332 cat >> /hgdev/src/.hg/hgrc << EOF
333 [phases]
334 publish = false
335 EOF
336
337 sudo chown -R hg:hg /hgdev
338 '''.lstrip()
339 .format(
340 install_rust=INSTALL_RUST,
341 install_pyoxidizer=INSTALL_PYOXIDIZER,
342 install_pythons=INSTALL_PYTHONS,
343 bootstrap_virtualenv=BOOTSTRAP_VIRTUALENV,
344 )
345 .replace('\r\n', '\n')
346 )
347
348
349 # Prepares /hgdev for operations.
350 PREPARE_HGDEV = '''
351 #!/bin/bash
352
353 set -e
354
355 FS=$1
356
357 ensure_device() {
358 if [ -z "${DEVICE}" ]; then
359 echo "could not find block device to format"
360 exit 1
361 fi
362 }
363
364 # Determine device to partition for extra filesystem.
365 # If only 1 volume is present, it will be the root volume and
366 # should be /dev/nvme0. If multiple volumes are present, the
367 # root volume could be nvme0 or nvme1. Use whichever one doesn't have
368 # a partition.
369 if [ -e /dev/nvme1n1 ]; then
370 if [ -e /dev/nvme0n1p1 ]; then
371 DEVICE=/dev/nvme1n1
372 else
373 DEVICE=/dev/nvme0n1
374 fi
375 else
376 DEVICE=
377 fi
378
379 sudo mkdir /hgwork
380
381 if [ "${FS}" != "default" -a "${FS}" != "tmpfs" ]; then
382 ensure_device
383 echo "creating ${FS} filesystem on ${DEVICE}"
384 fi
385
386 if [ "${FS}" = "default" ]; then
387 :
388
389 elif [ "${FS}" = "btrfs" ]; then
390 sudo mkfs.btrfs ${DEVICE}
391 sudo mount ${DEVICE} /hgwork
392
393 elif [ "${FS}" = "ext3" ]; then
394 # lazy_journal_init speeds up filesystem creation at the expense of
395 # integrity if things crash. We are an ephemeral instance, so we don't
396 # care about integrity.
397 sudo mkfs.ext3 -E lazy_journal_init=1 ${DEVICE}
398 sudo mount ${DEVICE} /hgwork
399
400 elif [ "${FS}" = "ext4" ]; then
401 sudo mkfs.ext4 -E lazy_journal_init=1 ${DEVICE}
402 sudo mount ${DEVICE} /hgwork
403
404 elif [ "${FS}" = "jfs" ]; then
405 sudo mkfs.jfs ${DEVICE}
406 sudo mount ${DEVICE} /hgwork
407
408 elif [ "${FS}" = "tmpfs" ]; then
409 echo "creating tmpfs volume in /hgwork"
410 sudo mount -t tmpfs -o size=1024M tmpfs /hgwork
411
412 elif [ "${FS}" = "xfs" ]; then
413 sudo mkfs.xfs ${DEVICE}
414 sudo mount ${DEVICE} /hgwork
415
416 else
417 echo "unsupported filesystem: ${FS}"
418 exit 1
419 fi
420
421 echo "/hgwork ready"
422
423 sudo chown hg:hg /hgwork
424 mkdir /hgwork/tmp
425 chown hg:hg /hgwork/tmp
426
427 rsync -a /hgdev/src /hgwork/
428 '''.lstrip().replace(
429 '\r\n', '\n'
430 )
431
432
433 HG_UPDATE_CLEAN = '''
434 set -ex
435
436 HG=/hgdev/venv-bootstrap/bin/hg
437
438 cd /hgwork/src
439 ${HG} --config extensions.purge= purge --all
440 ${HG} update -C $1
441 ${HG} log -r .
442 '''.lstrip().replace(
443 '\r\n', '\n'
444 )
445
446
447 def prepare_exec_environment(ssh_client, filesystem='default'):
448 """Prepare an EC2 instance to execute things.
449
450 The AMI has an ``/hgdev`` bootstrapped with various Python installs
451 and a clone of the Mercurial repo.
452
453 In EC2, EBS volumes launched from snapshots have wonky performance behavior.
454 Notably, blocks have to be copied on first access, which makes volume
455 I/O extremely slow on fresh volumes.
456
457 Furthermore, we may want to run operations, tests, etc on alternative
458 filesystems so we examine behavior on different filesystems.
459
460 This function is used to facilitate executing operations on alternate
461 volumes.
462 """
463 sftp = ssh_client.open_sftp()
464
465 with sftp.open('/hgdev/prepare-hgdev', 'wb') as fh:
466 fh.write(PREPARE_HGDEV)
467 fh.chmod(0o0777)
468
469 command = 'sudo /hgdev/prepare-hgdev %s' % filesystem
470 chan, stdin, stdout = exec_command(ssh_client, command)
471 stdin.close()
472
473 for line in stdout:
474 print(line, end='')
475
476 res = chan.recv_exit_status()
477
478 if res:
479 raise Exception('non-0 exit code updating working directory; %d' % res)
480
481
482 def synchronize_hg(
483 source_path: pathlib.Path, ec2_instance, revision: str = None
484 ):
485 """Synchronize a local Mercurial source path to remote EC2 instance."""
486
487 with tempfile.TemporaryDirectory() as temp_dir:
488 temp_dir = pathlib.Path(temp_dir)
489
490 ssh_dir = temp_dir / '.ssh'
491 ssh_dir.mkdir()
492 ssh_dir.chmod(0o0700)
493
494 public_ip = ec2_instance.public_ip_address
495
496 ssh_config = ssh_dir / 'config'
497
498 with ssh_config.open('w', encoding='utf-8') as fh:
499 fh.write('Host %s\n' % public_ip)
500 fh.write(' User hg\n')
501 fh.write(' StrictHostKeyChecking no\n')
502 fh.write(' UserKnownHostsFile %s\n' % (ssh_dir / 'known_hosts'))
503 fh.write(' IdentityFile %s\n' % ec2_instance.ssh_private_key_path)
504
505 if not (source_path / '.hg').is_dir():
506 raise Exception(
507 '%s is not a Mercurial repository; synchronization '
508 'not yet supported' % source_path
509 )
510
511 env = dict(os.environ)
512 env['HGPLAIN'] = '1'
513 env['HGENCODING'] = 'utf-8'
514
515 hg_bin = source_path / 'hg'
516
517 res = subprocess.run(
518 ['python3', str(hg_bin), 'log', '-r', revision, '-T', '{node}'],
519 cwd=str(source_path),
520 env=env,
521 check=True,
522 capture_output=True,
523 )
524
525 full_revision = res.stdout.decode('ascii')
526
527 args = [
528 'python3',
529 str(hg_bin),
530 '--config',
531 'ui.ssh=ssh -F %s' % ssh_config,
532 '--config',
533 'ui.remotecmd=/hgdev/venv-bootstrap/bin/hg',
534 # Also ensure .hgtags changes are present so auto version
535 # calculation works.
536 'push',
537 '-f',
538 '-r',
539 full_revision,
540 '-r',
541 'file(.hgtags)',
542 'ssh://%s//hgwork/src' % public_ip,
543 ]
544
545 res = subprocess.run(args, cwd=str(source_path), env=env)
546
547 # Allow 1 (no-op) to not trigger error.
548 if res.returncode not in (0, 1):
549 res.check_returncode()
550
551 # TODO support synchronizing dirty working directory.
552
553 sftp = ec2_instance.ssh_client.open_sftp()
554
555 with sftp.open('/hgdev/hgup', 'wb') as fh:
556 fh.write(HG_UPDATE_CLEAN)
557 fh.chmod(0o0700)
558
559 chan, stdin, stdout = exec_command(
560 ec2_instance.ssh_client, '/hgdev/hgup %s' % full_revision
561 )
562 stdin.close()
563
564 for line in stdout:
565 print(line, end='')
566
567 res = chan.recv_exit_status()
568
569 if res:
570 raise Exception(
571 'non-0 exit code updating working directory; %d' % res
572 )
573
574
575 def run_tests(ssh_client, python_version, test_flags=None):
576 """Run tests on a remote Linux machine via an SSH client."""
577 test_flags = test_flags or []
578
579 print('running tests')
580
581 if python_version == 'system3':
582 python = '/usr/bin/python3'
583 elif python_version.startswith('pypy'):
584 python = '/hgdev/pyenv/shims/%s' % python_version
585 else:
586 python = '/hgdev/pyenv/shims/python%s' % python_version
587
588 test_flags = shlex.join(test_flags)
589
590 command = (
591 '/bin/sh -c "export TMPDIR=/hgwork/tmp; '
592 'cd /hgwork/src/tests && %s run-tests.py %s"' % (python, test_flags)
593 )
594
595 chan, stdin, stdout = exec_command(ssh_client, command)
596
597 stdin.close()
598
599 for line in stdout:
600 print(line, end='')
601
602 return chan.recv_exit_status()