Debian Codegraph @

contrib/automation/hgautomation/windows.py

raw
1 # windows.py - Automation specific to Windows
2 #
3 # Copyright 2019 Gregory Szorc <[email protected]>
4 #
5 # This software may be used and distributed according to the terms of the
6 # GNU General Public License version 2 or any later version.
7
8 # no-check-code because Python 3 native.
9
10 import datetime
11 import os
12 import paramiko
13 import pathlib
14 import re
15 import subprocess
16 import tempfile
17
18 from .pypi import upload as pypi_upload
19 from .winrm import run_powershell
20
21
22 HG_PURGE = r'''
23 $Env:PATH = "C:\hgdev\venv-bootstrap\Scripts;$Env:PATH"
24 Set-Location C:\hgdev\src
25 hg.exe --config extensions.purge= purge --all
26 if ($LASTEXITCODE -ne 0) {
27 throw "process exited non-0: $LASTEXITCODE"
28 }
29 Write-Output "purged Mercurial repo"
30 '''
31
32 HG_UPDATE_CLEAN = r'''
33 $Env:PATH = "C:\hgdev\venv-bootstrap\Scripts;$Env:PATH"
34 Set-Location C:\hgdev\src
35 hg.exe --config extensions.purge= purge --all
36 if ($LASTEXITCODE -ne 0) {{
37 throw "process exited non-0: $LASTEXITCODE"
38 }}
39 hg.exe update -C {revision}
40 if ($LASTEXITCODE -ne 0) {{
41 throw "process exited non-0: $LASTEXITCODE"
42 }}
43 hg.exe log -r .
44 Write-Output "updated Mercurial working directory to {revision}"
45 '''.lstrip()
46
47 BUILD_INNO_PYTHON3 = r'''
48 $Env:PATH = "C:\hgdev\venv-bootstrap\Scripts;$Env:PATH"
49 $Env:RUSTUP_HOME = "C:\hgdev\rustup"
50 $Env:CARGO_HOME = "C:\hgdev\cargo"
51 Set-Location C:\hgdev\src
52 C:\hgdev\python37-x64\python.exe contrib\packaging\packaging.py inno --pyoxidizer-target {pyoxidizer_target} --version {version}
53 if ($LASTEXITCODE -ne 0) {{
54 throw "process exited non-0: $LASTEXITCODE"
55 }}
56 '''
57
58
59 BUILD_WHEEL = r'''
60 $Env:PATH = "C:\hgdev\venv-bootstrap\Scripts;$Env:PATH"
61 Set-Location C:\hgdev\src
62 C:\hgdev\python{python_version}-{arch}\python.exe -m pip wheel --wheel-dir dist .
63 if ($LASTEXITCODE -ne 0) {{
64 throw "process exited non-0: $LASTEXITCODE"
65 }}
66 '''
67
68 BUILD_WIX_PYTHON3 = r'''
69 $Env:PATH = "C:\hgdev\venv-bootstrap\Scripts;$Env:PATH"
70 $Env:RUSTUP_HOME = "C:\hgdev\rustup"
71 $Env:CARGO_HOME = "C:\hgdev\cargo"
72 Set-Location C:\hgdev\src
73 C:\hgdev\python37-x64\python.exe contrib\packaging\packaging.py wix --pyoxidizer-target {pyoxidizer_target} --version {version}
74 if ($LASTEXITCODE -ne 0) {{
75 throw "process exited non-0: $LASTEXITCODE"
76 }}
77 '''
78
79
80 RUN_TESTS = r'''
81 C:\hgdev\MinGW\msys\1.0\bin\sh.exe --login -c "cd /c/hgdev/src/tests && /c/hgdev/{python_path}/python.exe run-tests.py {test_flags}"
82 if ($LASTEXITCODE -ne 0) {{
83 throw "process exited non-0: $LASTEXITCODE"
84 }}
85 '''
86
87
88 WHEEL_FILENAME_PYTHON37_X86 = 'mercurial-{version}-cp37-cp37m-win32.whl'
89 WHEEL_FILENAME_PYTHON37_X64 = 'mercurial-{version}-cp37-cp37m-win_amd64.whl'
90 WHEEL_FILENAME_PYTHON38_X86 = 'mercurial-{version}-cp38-cp38-win32.whl'
91 WHEEL_FILENAME_PYTHON38_X64 = 'mercurial-{version}-cp38-cp38-win_amd64.whl'
92 WHEEL_FILENAME_PYTHON39_X86 = 'mercurial-{version}-cp39-cp39-win32.whl'
93 WHEEL_FILENAME_PYTHON39_X64 = 'mercurial-{version}-cp39-cp39-win_amd64.whl'
94 WHEEL_FILENAME_PYTHON310_X86 = 'mercurial-{version}-cp310-cp310-win32.whl'
95 WHEEL_FILENAME_PYTHON310_X64 = 'mercurial-{version}-cp310-cp310-win_amd64.whl'
96
97 EXE_FILENAME_PYTHON3_X86 = 'Mercurial-{version}-x86.exe'
98 EXE_FILENAME_PYTHON3_X64 = 'Mercurial-{version}-x64.exe'
99
100 MSI_FILENAME_PYTHON3_X86 = 'mercurial-{version}-x86.msi'
101 MSI_FILENAME_PYTHON3_X64 = 'mercurial-{version}-x64.msi'
102
103 MERCURIAL_SCM_BASE_URL = 'https://mercurial-scm.org/release/windows'
104
105 X86_USER_AGENT_PATTERN = '.*Windows.*'
106 X64_USER_AGENT_PATTERN = '.*Windows.*(WOW|x)64.*'
107
108 # TODO remove Python version once Python 2 is dropped.
109 EXE_PYTHON3_X86_DESCRIPTION = (
110 'Mercurial {version} Inno Setup installer - x86 Windows (Python 3) '
111 '- does not require admin rights'
112 )
113 EXE_PYTHON3_X64_DESCRIPTION = (
114 'Mercurial {version} Inno Setup installer - x64 Windows (Python 3) '
115 '- does not require admin rights'
116 )
117 MSI_PYTHON3_X86_DESCRIPTION = (
118 'Mercurial {version} MSI installer - x86 Windows (Python 3) '
119 '- requires admin rights'
120 )
121 MSI_PYTHON3_X64_DESCRIPTION = (
122 'Mercurial {version} MSI installer - x64 Windows (Python 3) '
123 '- requires admin rights'
124 )
125
126
127 def fix_authorized_keys_permissions(winrm_client, path):
128 commands = [
129 '$ErrorActionPreference = "Stop"',
130 'Repair-AuthorizedKeyPermission -FilePath %s -Confirm:$false' % path,
131 r'icacls %s /remove:g "NT Service\sshd"' % path,
132 ]
133
134 run_powershell(winrm_client, '\n'.join(commands))
135
136
137 def synchronize_hg(hg_repo: pathlib.Path, revision: str, ec2_instance):
138 """Synchronize local Mercurial repo to remote EC2 instance."""
139
140 winrm_client = ec2_instance.winrm_client
141
142 with tempfile.TemporaryDirectory() as temp_dir:
143 temp_dir = pathlib.Path(temp_dir)
144
145 ssh_dir = temp_dir / '.ssh'
146 ssh_dir.mkdir()
147 ssh_dir.chmod(0o0700)
148
149 # Generate SSH key to use for communication.
150 subprocess.run(
151 [
152 'ssh-keygen',
153 '-t',
154 'rsa',
155 '-b',
156 '4096',
157 '-N',
158 '',
159 '-f',
160 str(ssh_dir / 'id_rsa'),
161 ],
162 check=True,
163 capture_output=True,
164 )
165
166 # Add it to ~/.ssh/authorized_keys on remote.
167 # This assumes the file doesn't already exist.
168 authorized_keys = r'c:\Users\Administrator\.ssh\authorized_keys'
169 winrm_client.execute_cmd(r'mkdir c:\Users\Administrator\.ssh')
170 winrm_client.copy(str(ssh_dir / 'id_rsa.pub'), authorized_keys)
171 fix_authorized_keys_permissions(winrm_client, authorized_keys)
172
173 public_ip = ec2_instance.public_ip_address
174
175 ssh_config = temp_dir / '.ssh' / 'config'
176
177 with open(ssh_config, 'w', encoding='utf-8') as fh:
178 fh.write('Host %s\n' % public_ip)
179 fh.write(' User Administrator\n')
180 fh.write(' StrictHostKeyChecking no\n')
181 fh.write(' UserKnownHostsFile %s\n' % (ssh_dir / 'known_hosts'))
182 fh.write(' IdentityFile %s\n' % (ssh_dir / 'id_rsa'))
183
184 if not (hg_repo / '.hg').is_dir():
185 raise Exception(
186 '%s is not a Mercurial repository; '
187 'synchronization not yet supported' % hg_repo
188 )
189
190 env = dict(os.environ)
191 env['HGPLAIN'] = '1'
192 env['HGENCODING'] = 'utf-8'
193
194 hg_bin = hg_repo / 'hg'
195
196 res = subprocess.run(
197 ['python3', str(hg_bin), 'log', '-r', revision, '-T', '{node}'],
198 cwd=str(hg_repo),
199 env=env,
200 check=True,
201 capture_output=True,
202 )
203
204 full_revision = res.stdout.decode('ascii')
205
206 args = [
207 'python3',
208 hg_bin,
209 '--config',
210 'ui.ssh=ssh -F %s' % ssh_config,
211 '--config',
212 'ui.remotecmd=c:/hgdev/venv-bootstrap/Scripts/hg.exe',
213 # Also ensure .hgtags changes are present so auto version
214 # calculation works.
215 'push',
216 '-f',
217 '-r',
218 full_revision,
219 '-r',
220 'file(.hgtags)',
221 'ssh://%s/c:/hgdev/src' % public_ip,
222 ]
223
224 res = subprocess.run(args, cwd=str(hg_repo), env=env)
225
226 # Allow 1 (no-op) to not trigger error.
227 if res.returncode not in (0, 1):
228 res.check_returncode()
229
230 run_powershell(
231 winrm_client, HG_UPDATE_CLEAN.format(revision=full_revision)
232 )
233
234 # TODO detect dirty local working directory and synchronize accordingly.
235
236
237 def purge_hg(winrm_client):
238 """Purge the Mercurial source repository on an EC2 instance."""
239 run_powershell(winrm_client, HG_PURGE)
240
241
242 def find_latest_dist(winrm_client, pattern):
243 """Find path to newest file in dist/ directory matching a pattern."""
244
245 res = winrm_client.execute_ps(
246 r'$v = Get-ChildItem -Path C:\hgdev\src\dist -Filter "%s" '
247 '| Sort-Object LastWriteTime -Descending '
248 '| Select-Object -First 1\n'
249 '$v.name' % pattern
250 )
251 return res[0]
252
253
254 def copy_latest_dist(winrm_client, pattern, dest_path):
255 """Copy latest file matching pattern in dist/ directory.
256
257 Given a WinRM client and a file pattern, find the latest file on the remote
258 matching that pattern and copy it to the ``dest_path`` directory on the
259 local machine.
260 """
261 latest = find_latest_dist(winrm_client, pattern)
262 source = r'C:\hgdev\src\dist\%s' % latest
263 dest = dest_path / latest
264 print('copying %s to %s' % (source, dest))
265 winrm_client.fetch(source, str(dest))
266
267
268 def build_inno_installer(
269 winrm_client,
270 arch: str,
271 dest_path: pathlib.Path,
272 version=None,
273 ):
274 """Build the Inno Setup installer on a remote machine.
275
276 Using a WinRM client, remote commands are executed to build
277 a Mercurial Inno Setup installer.
278 """
279 print('building Inno Setup installer for %s' % arch)
280
281 # TODO fix this limitation in packaging code
282 if not version:
283 raise Exception("version string is required when building for Python 3")
284
285 if arch == "x86":
286 target_triple = "i686-pc-windows-msvc"
287 elif arch == "x64":
288 target_triple = "x86_64-pc-windows-msvc"
289 else:
290 raise Exception("unhandled arch: %s" % arch)
291
292 ps = BUILD_INNO_PYTHON3.format(
293 pyoxidizer_target=target_triple,
294 version=version,
295 )
296
297 run_powershell(winrm_client, ps)
298 copy_latest_dist(winrm_client, '*.exe', dest_path)
299
300
301 def build_wheel(
302 winrm_client, python_version: str, arch: str, dest_path: pathlib.Path
303 ):
304 """Build Python wheels on a remote machine.
305
306 Using a WinRM client, remote commands are executed to build a Python wheel
307 for Mercurial.
308 """
309 print('Building Windows wheel for Python %s %s' % (python_version, arch))
310
311 ps = BUILD_WHEEL.format(
312 python_version=python_version.replace(".", ""), arch=arch
313 )
314
315 run_powershell(winrm_client, ps)
316 copy_latest_dist(winrm_client, '*.whl', dest_path)
317
318
319 def build_wix_installer(
320 winrm_client,
321 arch: str,
322 dest_path: pathlib.Path,
323 version=None,
324 ):
325 """Build the WiX installer on a remote machine.
326
327 Using a WinRM client, remote commands are executed to build a WiX installer.
328 """
329 print('Building WiX installer for %s' % arch)
330
331 # TODO fix this limitation in packaging code
332 if not version:
333 raise Exception("version string is required when building for Python 3")
334
335 if arch == "x86":
336 target_triple = "i686-pc-windows-msvc"
337 elif arch == "x64":
338 target_triple = "x86_64-pc-windows-msvc"
339 else:
340 raise Exception("unhandled arch: %s" % arch)
341
342 ps = BUILD_WIX_PYTHON3.format(
343 pyoxidizer_target=target_triple,
344 version=version,
345 )
346
347 run_powershell(winrm_client, ps)
348 copy_latest_dist(winrm_client, '*.msi', dest_path)
349
350
351 def run_tests(winrm_client, python_version, arch, test_flags=''):
352 """Run tests on a remote Windows machine.
353
354 ``python_version`` is a ``X.Y`` string like ``2.7`` or ``3.7``.
355 ``arch`` is ``x86`` or ``x64``.
356 ``test_flags`` is a str representing extra arguments to pass to
357 ``run-tests.py``.
358 """
359 if not re.match(r'\d\.\d', python_version):
360 raise ValueError(
361 r'python_version must be \d.\d; got %s' % python_version
362 )
363
364 if arch not in ('x86', 'x64'):
365 raise ValueError('arch must be x86 or x64; got %s' % arch)
366
367 python_path = 'python%s-%s' % (python_version.replace('.', ''), arch)
368
369 ps = RUN_TESTS.format(
370 python_path=python_path,
371 test_flags=test_flags or '',
372 )
373
374 run_powershell(winrm_client, ps)
375
376
377 def resolve_wheel_artifacts(dist_path: pathlib.Path, version: str):
378 return (
379 dist_path / WHEEL_FILENAME_PYTHON37_X86.format(version=version),
380 dist_path / WHEEL_FILENAME_PYTHON37_X64.format(version=version),
381 dist_path / WHEEL_FILENAME_PYTHON38_X86.format(version=version),
382 dist_path / WHEEL_FILENAME_PYTHON38_X64.format(version=version),
383 dist_path / WHEEL_FILENAME_PYTHON39_X86.format(version=version),
384 dist_path / WHEEL_FILENAME_PYTHON39_X64.format(version=version),
385 dist_path / WHEEL_FILENAME_PYTHON310_X86.format(version=version),
386 dist_path / WHEEL_FILENAME_PYTHON310_X64.format(version=version),
387 )
388
389
390 def resolve_all_artifacts(dist_path: pathlib.Path, version: str):
391 return (
392 dist_path / WHEEL_FILENAME_PYTHON37_X86.format(version=version),
393 dist_path / WHEEL_FILENAME_PYTHON37_X64.format(version=version),
394 dist_path / WHEEL_FILENAME_PYTHON38_X86.format(version=version),
395 dist_path / WHEEL_FILENAME_PYTHON38_X64.format(version=version),
396 dist_path / WHEEL_FILENAME_PYTHON39_X86.format(version=version),
397 dist_path / WHEEL_FILENAME_PYTHON39_X64.format(version=version),
398 dist_path / WHEEL_FILENAME_PYTHON310_X86.format(version=version),
399 dist_path / WHEEL_FILENAME_PYTHON310_X64.format(version=version),
400 dist_path / EXE_FILENAME_PYTHON3_X86.format(version=version),
401 dist_path / EXE_FILENAME_PYTHON3_X64.format(version=version),
402 dist_path / MSI_FILENAME_PYTHON3_X86.format(version=version),
403 dist_path / MSI_FILENAME_PYTHON3_X64.format(version=version),
404 )
405
406
407 def generate_latest_dat(version: str):
408 python3_x86_exe_filename = EXE_FILENAME_PYTHON3_X86.format(version=version)
409 python3_x64_exe_filename = EXE_FILENAME_PYTHON3_X64.format(version=version)
410 python3_x86_msi_filename = MSI_FILENAME_PYTHON3_X86.format(version=version)
411 python3_x64_msi_filename = MSI_FILENAME_PYTHON3_X64.format(version=version)
412
413 entries = (
414 (
415 '10',
416 version,
417 X86_USER_AGENT_PATTERN,
418 '%s/%s' % (MERCURIAL_SCM_BASE_URL, python3_x86_exe_filename),
419 EXE_PYTHON3_X86_DESCRIPTION.format(version=version),
420 ),
421 (
422 '10',
423 version,
424 X64_USER_AGENT_PATTERN,
425 '%s/%s' % (MERCURIAL_SCM_BASE_URL, python3_x64_exe_filename),
426 EXE_PYTHON3_X64_DESCRIPTION.format(version=version),
427 ),
428 (
429 '10',
430 version,
431 X86_USER_AGENT_PATTERN,
432 '%s/%s' % (MERCURIAL_SCM_BASE_URL, python3_x86_msi_filename),
433 MSI_PYTHON3_X86_DESCRIPTION.format(version=version),
434 ),
435 (
436 '10',
437 version,
438 X64_USER_AGENT_PATTERN,
439 '%s/%s' % (MERCURIAL_SCM_BASE_URL, python3_x64_msi_filename),
440 MSI_PYTHON3_X64_DESCRIPTION.format(version=version),
441 ),
442 )
443
444 lines = ['\t'.join(e) for e in entries]
445
446 return '\n'.join(lines) + '\n'
447
448
449 def publish_artifacts_pypi(dist_path: pathlib.Path, version: str):
450 """Publish Windows release artifacts to PyPI."""
451
452 wheel_paths = resolve_wheel_artifacts(dist_path, version)
453
454 for p in wheel_paths:
455 if not p.exists():
456 raise Exception('%s not found' % p)
457
458 print('uploading wheels to PyPI (you may be prompted for credentials)')
459 pypi_upload(wheel_paths)
460
461
462 def publish_artifacts_mercurial_scm_org(
463 dist_path: pathlib.Path, version: str, ssh_username=None
464 ):
465 """Publish Windows release artifacts to mercurial-scm.org."""
466 all_paths = resolve_all_artifacts(dist_path, version)
467
468 for p in all_paths:
469 if not p.exists():
470 raise Exception('%s not found' % p)
471
472 client = paramiko.SSHClient()
473 client.load_system_host_keys()
474 # We assume the system SSH configuration knows how to connect.
475 print('connecting to mercurial-scm.org via ssh...')
476 try:
477 client.connect('mercurial-scm.org', username=ssh_username)
478 except paramiko.AuthenticationException:
479 print('error authenticating; is an SSH key available in an SSH agent?')
480 raise
481
482 print('SSH connection established')
483
484 print('opening SFTP client...')
485 sftp = client.open_sftp()
486 print('SFTP client obtained')
487
488 for p in all_paths:
489 dest_path = '/var/www/release/windows/%s' % p.name
490 print('uploading %s to %s' % (p, dest_path))
491
492 with p.open('rb') as fh:
493 data = fh.read()
494
495 with sftp.open(dest_path, 'wb') as fh:
496 fh.write(data)
497 fh.chmod(0o0664)
498
499 latest_dat_path = '/var/www/release/windows/latest.dat'
500
501 now = datetime.datetime.utcnow()
502 backup_path = dist_path / (
503 'latest-windows-%s.dat' % now.strftime('%Y%m%dT%H%M%S')
504 )
505 print('backing up %s to %s' % (latest_dat_path, backup_path))
506
507 with sftp.open(latest_dat_path, 'rb') as fh:
508 latest_dat_old = fh.read()
509
510 with backup_path.open('wb') as fh:
511 fh.write(latest_dat_old)
512
513 print('writing %s with content:' % latest_dat_path)
514 latest_dat_content = generate_latest_dat(version)
515 print(latest_dat_content)
516
517 with sftp.open(latest_dat_path, 'wb') as fh:
518 fh.write(latest_dat_content.encode('ascii'))
519
520
521 def publish_artifacts(
522 dist_path: pathlib.Path,
523 version: str,
524 pypi=True,
525 mercurial_scm_org=True,
526 ssh_username=None,
527 ):
528 """Publish Windows release artifacts.
529
530 Files are found in `dist_path`. We will look for files with version string
531 `version`.
532
533 `pypi` controls whether we upload to PyPI.
534 `mercurial_scm_org` controls whether we upload to mercurial-scm.org.
535 """
536 if pypi:
537 publish_artifacts_pypi(dist_path, version)
538
539 if mercurial_scm_org:
540 publish_artifacts_mercurial_scm_org(
541 dist_path, version, ssh_username=ssh_username
542 )