Debian Codegraph @

contrib/automation/hgautomation/cli.py

raw
1 # cli.py - Command line interface for automation
2 #
3 # Copyright 2019 Gregory Szorc <[email protected]>
4 #
5 # This software may be used and distributed according to the terms of the
6 # GNU General Public License version 2 or any later version.
7
8 # no-check-code because Python 3 native.
9
10 import argparse
11 import concurrent.futures as futures
12 import os
13 import pathlib
14 import time
15
16 from . import (
17 aws,
18 HGAutomation,
19 linux,
20 try_server,
21 windows,
22 )
23
24
25 SOURCE_ROOT = pathlib.Path(
26 os.path.abspath(__file__)
27 ).parent.parent.parent.parent
28 DIST_PATH = SOURCE_ROOT / 'dist'
29
30
31 def bootstrap_linux_dev(
32 hga: HGAutomation, aws_region, distros=None, parallel=False
33 ):
34 c = hga.aws_connection(aws_region)
35
36 if distros:
37 distros = distros.split(',')
38 else:
39 distros = sorted(linux.DISTROS)
40
41 # TODO There is a wonky interaction involving KeyboardInterrupt whereby
42 # the context manager that is supposed to terminate the temporary EC2
43 # instance doesn't run. Until we fix this, make parallel building opt-in
44 # so we don't orphan instances.
45 if parallel:
46 fs = []
47
48 with futures.ThreadPoolExecutor(len(distros)) as e:
49 for distro in distros:
50 fs.append(e.submit(aws.ensure_linux_dev_ami, c, distro=distro))
51
52 for f in fs:
53 f.result()
54 else:
55 for distro in distros:
56 aws.ensure_linux_dev_ami(c, distro=distro)
57
58
59 def bootstrap_windows_dev(hga: HGAutomation, aws_region, base_image_name):
60 c = hga.aws_connection(aws_region)
61 image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name)
62 print('Windows development AMI available as %s' % image.id)
63
64
65 def build_inno(
66 hga: HGAutomation,
67 aws_region,
68 arch,
69 revision,
70 version,
71 base_image_name,
72 ):
73 c = hga.aws_connection(aws_region)
74 image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name)
75 DIST_PATH.mkdir(exist_ok=True)
76
77 with aws.temporary_windows_dev_instances(c, image, 't3.medium') as insts:
78 instance = insts[0]
79
80 windows.synchronize_hg(SOURCE_ROOT, revision, instance)
81
82 for a in arch:
83 windows.build_inno_installer(
84 instance.winrm_client,
85 a,
86 DIST_PATH,
87 version=version,
88 )
89
90
91 def build_wix(
92 hga: HGAutomation,
93 aws_region,
94 arch,
95 revision,
96 version,
97 base_image_name,
98 ):
99 c = hga.aws_connection(aws_region)
100 image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name)
101 DIST_PATH.mkdir(exist_ok=True)
102
103 with aws.temporary_windows_dev_instances(c, image, 't3.medium') as insts:
104 instance = insts[0]
105
106 windows.synchronize_hg(SOURCE_ROOT, revision, instance)
107
108 for a in arch:
109 windows.build_wix_installer(
110 instance.winrm_client,
111 a,
112 DIST_PATH,
113 version=version,
114 )
115
116
117 def build_windows_wheel(
118 hga: HGAutomation,
119 aws_region,
120 python_version,
121 arch,
122 revision,
123 base_image_name,
124 ):
125 c = hga.aws_connection(aws_region)
126 image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name)
127 DIST_PATH.mkdir(exist_ok=True)
128
129 with aws.temporary_windows_dev_instances(c, image, 't3.medium') as insts:
130 instance = insts[0]
131
132 windows.synchronize_hg(SOURCE_ROOT, revision, instance)
133
134 for py_version in python_version:
135 for a in arch:
136 windows.build_wheel(
137 instance.winrm_client, py_version, a, DIST_PATH
138 )
139
140
141 def build_all_windows_packages(
142 hga: HGAutomation, aws_region, revision, version, base_image_name
143 ):
144 c = hga.aws_connection(aws_region)
145 image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name)
146 DIST_PATH.mkdir(exist_ok=True)
147
148 with aws.temporary_windows_dev_instances(c, image, 'm6i.large') as insts:
149 instance = insts[0]
150
151 winrm_client = instance.winrm_client
152
153 windows.synchronize_hg(SOURCE_ROOT, revision, instance)
154
155 for py_version in ("3.7", "3.8", "3.9", "3.10"):
156 for arch in ("x86", "x64"):
157 windows.purge_hg(winrm_client)
158 windows.build_wheel(
159 winrm_client,
160 python_version=py_version,
161 arch=arch,
162 dest_path=DIST_PATH,
163 )
164
165 for arch in ('x86', 'x64'):
166 windows.purge_hg(winrm_client)
167 windows.build_inno_installer(
168 winrm_client, arch, DIST_PATH, version=version
169 )
170 windows.build_wix_installer(
171 winrm_client, arch, DIST_PATH, version=version
172 )
173
174
175 def terminate_ec2_instances(hga: HGAutomation, aws_region):
176 c = hga.aws_connection(aws_region, ensure_ec2_state=False)
177 aws.terminate_ec2_instances(c.ec2resource)
178
179
180 def purge_ec2_resources(hga: HGAutomation, aws_region):
181 c = hga.aws_connection(aws_region, ensure_ec2_state=False)
182 aws.remove_resources(c)
183
184
185 def run_tests_linux(
186 hga: HGAutomation,
187 aws_region,
188 instance_type,
189 python_version,
190 test_flags,
191 distro,
192 filesystem,
193 ):
194 c = hga.aws_connection(aws_region)
195 image = aws.ensure_linux_dev_ami(c, distro=distro)
196
197 t_start = time.time()
198
199 ensure_extra_volume = filesystem not in ('default', 'tmpfs')
200
201 with aws.temporary_linux_dev_instances(
202 c, image, instance_type, ensure_extra_volume=ensure_extra_volume
203 ) as insts:
204 instance = insts[0]
205
206 linux.prepare_exec_environment(
207 instance.ssh_client, filesystem=filesystem
208 )
209 linux.synchronize_hg(SOURCE_ROOT, instance, '.')
210 t_prepared = time.time()
211 linux.run_tests(instance.ssh_client, python_version, test_flags)
212 t_done = time.time()
213
214 t_setup = t_prepared - t_start
215 t_all = t_done - t_start
216
217 print(
218 'total time: %.1fs; setup: %.1fs; tests: %.1fs; setup overhead: %.1f%%'
219 % (t_all, t_setup, t_done - t_prepared, t_setup / t_all * 100.0)
220 )
221
222
223 def run_tests_windows(
224 hga: HGAutomation,
225 aws_region,
226 instance_type,
227 python_version,
228 arch,
229 test_flags,
230 base_image_name,
231 ):
232 c = hga.aws_connection(aws_region)
233 image = aws.ensure_windows_dev_ami(c, base_image_name=base_image_name)
234
235 with aws.temporary_windows_dev_instances(
236 c, image, instance_type, disable_antivirus=True
237 ) as insts:
238 instance = insts[0]
239
240 windows.synchronize_hg(SOURCE_ROOT, '.', instance)
241 windows.run_tests(
242 instance.winrm_client, python_version, arch, test_flags
243 )
244
245
246 def publish_windows_artifacts(
247 hg: HGAutomation,
248 aws_region,
249 version: str,
250 pypi: bool,
251 mercurial_scm_org: bool,
252 ssh_username: str,
253 ):
254 windows.publish_artifacts(
255 DIST_PATH,
256 version,
257 pypi=pypi,
258 mercurial_scm_org=mercurial_scm_org,
259 ssh_username=ssh_username,
260 )
261
262
263 def run_try(hga: HGAutomation, aws_region: str, rev: str):
264 c = hga.aws_connection(aws_region, ensure_ec2_state=False)
265 try_server.trigger_try(c, rev=rev)
266
267
268 def get_parser():
269 parser = argparse.ArgumentParser()
270
271 parser.add_argument(
272 '--state-path',
273 default='~/.hgautomation',
274 help='Path for local state files',
275 )
276 parser.add_argument(
277 '--aws-region',
278 help='AWS region to use',
279 default='us-west-2',
280 )
281
282 subparsers = parser.add_subparsers()
283
284 sp = subparsers.add_parser(
285 'bootstrap-linux-dev',
286 help='Bootstrap Linux development environments',
287 )
288 sp.add_argument(
289 '--distros',
290 help='Comma delimited list of distros to bootstrap',
291 )
292 sp.add_argument(
293 '--parallel',
294 action='store_true',
295 help='Generate AMIs in parallel (not CTRL-c safe)',
296 )
297 sp.set_defaults(func=bootstrap_linux_dev)
298
299 sp = subparsers.add_parser(
300 'bootstrap-windows-dev',
301 help='Bootstrap the Windows development environment',
302 )
303 sp.add_argument(
304 '--base-image-name',
305 help='AMI name of base image',
306 default=aws.WINDOWS_BASE_IMAGE_NAME,
307 )
308 sp.set_defaults(func=bootstrap_windows_dev)
309
310 sp = subparsers.add_parser(
311 'build-all-windows-packages',
312 help='Build all Windows packages',
313 )
314 sp.add_argument(
315 '--revision',
316 help='Mercurial revision to build',
317 default='.',
318 )
319 sp.add_argument(
320 '--version',
321 help='Mercurial version string to use',
322 )
323 sp.add_argument(
324 '--base-image-name',
325 help='AMI name of base image',
326 default=aws.WINDOWS_BASE_IMAGE_NAME,
327 )
328 sp.set_defaults(func=build_all_windows_packages)
329
330 sp = subparsers.add_parser(
331 'build-inno',
332 help='Build Inno Setup installer(s)',
333 )
334 sp.add_argument(
335 '--arch',
336 help='Architecture to build for',
337 choices={'x86', 'x64'},
338 nargs='*',
339 default=['x64'],
340 )
341 sp.add_argument(
342 '--revision',
343 help='Mercurial revision to build',
344 default='.',
345 )
346 sp.add_argument(
347 '--version',
348 help='Mercurial version string to use in installer',
349 )
350 sp.add_argument(
351 '--base-image-name',
352 help='AMI name of base image',
353 default=aws.WINDOWS_BASE_IMAGE_NAME,
354 )
355 sp.set_defaults(func=build_inno)
356
357 sp = subparsers.add_parser(
358 'build-windows-wheel',
359 help='Build Windows wheel(s)',
360 )
361 sp.add_argument(
362 '--python-version',
363 help='Python version to build for',
364 choices={'3.7', '3.8', '3.9', '3.10'},
365 nargs='*',
366 default=['3.8'],
367 )
368 sp.add_argument(
369 '--arch',
370 help='Architecture to build for',
371 choices={'x86', 'x64'},
372 nargs='*',
373 default=['x64'],
374 )
375 sp.add_argument(
376 '--revision',
377 help='Mercurial revision to build',
378 default='.',
379 )
380 sp.add_argument(
381 '--base-image-name',
382 help='AMI name of base image',
383 default=aws.WINDOWS_BASE_IMAGE_NAME,
384 )
385 sp.set_defaults(func=build_windows_wheel)
386
387 sp = subparsers.add_parser('build-wix', help='Build WiX installer(s)')
388 sp.add_argument(
389 '--arch',
390 help='Architecture to build for',
391 choices={'x86', 'x64'},
392 nargs='*',
393 default=['x64'],
394 )
395 sp.add_argument(
396 '--revision',
397 help='Mercurial revision to build',
398 default='.',
399 )
400 sp.add_argument(
401 '--version',
402 help='Mercurial version string to use in installer',
403 )
404 sp.add_argument(
405 '--base-image-name',
406 help='AMI name of base image',
407 default=aws.WINDOWS_BASE_IMAGE_NAME,
408 )
409 sp.set_defaults(func=build_wix)
410
411 sp = subparsers.add_parser(
412 'terminate-ec2-instances',
413 help='Terminate all active EC2 instances managed by us',
414 )
415 sp.set_defaults(func=terminate_ec2_instances)
416
417 sp = subparsers.add_parser(
418 'purge-ec2-resources',
419 help='Purge all EC2 resources managed by us',
420 )
421 sp.set_defaults(func=purge_ec2_resources)
422
423 sp = subparsers.add_parser(
424 'run-tests-linux',
425 help='Run tests on Linux',
426 )
427 sp.add_argument(
428 '--distro',
429 help='Linux distribution to run tests on',
430 choices=linux.DISTROS,
431 default='debian10',
432 )
433 sp.add_argument(
434 '--filesystem',
435 help='Filesystem type to use',
436 choices={'btrfs', 'default', 'ext3', 'ext4', 'jfs', 'tmpfs', 'xfs'},
437 default='default',
438 )
439 sp.add_argument(
440 '--instance-type',
441 help='EC2 instance type to use',
442 default='c5.9xlarge',
443 )
444 sp.add_argument(
445 '--python-version',
446 help='Python version to use',
447 choices={
448 'system3',
449 '3.8',
450 'pypy',
451 'pypy3.5',
452 'pypy3.6',
453 },
454 default='system3',
455 )
456 sp.add_argument(
457 'test_flags',
458 help='Extra command line flags to pass to run-tests.py',
459 nargs='*',
460 )
461 sp.set_defaults(func=run_tests_linux)
462
463 sp = subparsers.add_parser(
464 'run-tests-windows',
465 help='Run tests on Windows',
466 )
467 sp.add_argument(
468 '--instance-type',
469 help='EC2 instance type to use',
470 default='m6i.large',
471 )
472 sp.add_argument(
473 '--python-version',
474 help='Python version to use',
475 choices={'3.8', '3.9', '3.10'},
476 default='3.9',
477 )
478 sp.add_argument(
479 '--arch',
480 help='Architecture to test',
481 choices={'x86', 'x64'},
482 default='x64',
483 )
484 sp.add_argument(
485 '--test-flags',
486 help='Extra command line flags to pass to run-tests.py',
487 )
488 sp.add_argument(
489 '--base-image-name',
490 help='AMI name of base image',
491 default=aws.WINDOWS_BASE_IMAGE_NAME,
492 )
493 sp.set_defaults(func=run_tests_windows)
494
495 sp = subparsers.add_parser(
496 'publish-windows-artifacts',
497 help='Publish built Windows artifacts (wheels, installers, etc)',
498 )
499 sp.add_argument(
500 '--no-pypi',
501 dest='pypi',
502 action='store_false',
503 default=True,
504 help='Skip uploading to PyPI',
505 )
506 sp.add_argument(
507 '--no-mercurial-scm-org',
508 dest='mercurial_scm_org',
509 action='store_false',
510 default=True,
511 help='Skip uploading to www.mercurial-scm.org',
512 )
513 sp.add_argument(
514 '--ssh-username',
515 help='SSH username for mercurial-scm.org',
516 )
517 sp.add_argument(
518 'version',
519 help='Mercurial version string to locate local packages',
520 )
521 sp.set_defaults(func=publish_windows_artifacts)
522
523 sp = subparsers.add_parser(
524 'try', help='Run CI automation against a custom changeset'
525 )
526 sp.add_argument('-r', '--rev', default='.', help='Revision to run CI on')
527 sp.set_defaults(func=run_try)
528
529 return parser
530
531
532 def main():
533 parser = get_parser()
534 args = parser.parse_args()
535
536 local_state_path = pathlib.Path(os.path.expanduser(args.state_path))
537 automation = HGAutomation(local_state_path)
538
539 if not hasattr(args, 'func'):
540 parser.print_help()
541 return
542
543 kwargs = dict(vars(args))
544 del kwargs['func']
545 del kwargs['state_path']
546
547 args.func(automation, **kwargs)