Debian Codegraph @

contrib/automation/hgautomation/aws.py

raw
1 # aws.py - Automation code for Amazon Web Services
2 #
3 # Copyright 2019 Gregory Szorc <[email protected]>
4 #
5 # This software may be used and distributed according to the terms of the
6 # GNU General Public License version 2 or any later version.
7
8 # no-check-code because Python 3 native.
9
10 import contextlib
11 import copy
12 import hashlib
13 import json
14 import os
15 import pathlib
16 import subprocess
17 import time
18
19 import boto3
20 import botocore.exceptions
21
22 from .linux import BOOTSTRAP_DEBIAN
23 from .ssh import (
24 exec_command as ssh_exec_command,
25 wait_for_ssh,
26 )
27 from .winrm import (
28 run_powershell,
29 wait_for_winrm,
30 )
31
32
33 SOURCE_ROOT = pathlib.Path(
34 os.path.abspath(__file__)
35 ).parent.parent.parent.parent
36
37 INSTALL_WINDOWS_DEPENDENCIES = (
38 SOURCE_ROOT / 'contrib' / 'install-windows-dependencies.ps1'
39 )
40
41
42 INSTANCE_TYPES_WITH_STORAGE = {
43 'c5d',
44 'd2',
45 'h1',
46 'i3',
47 'm5ad',
48 'm5d',
49 'r5d',
50 'r5ad',
51 'x1',
52 'z1d',
53 }
54
55
56 AMAZON_ACCOUNT_ID = '801119661308'
57 DEBIAN_ACCOUNT_ID = '379101102735'
58 DEBIAN_ACCOUNT_ID_2 = '136693071363'
59 UBUNTU_ACCOUNT_ID = '099720109477'
60
61
62 WINDOWS_BASE_IMAGE_NAME = 'Windows_Server-2022-English-Full-Base-*'
63
64
65 KEY_PAIRS = {
66 'automation',
67 }
68
69
70 SECURITY_GROUPS = {
71 'linux-dev-1': {
72 'description': 'Mercurial Linux instances that perform build/test automation',
73 'ingress': [
74 {
75 'FromPort': 22,
76 'ToPort': 22,
77 'IpProtocol': 'tcp',
78 'IpRanges': [
79 {
80 'CidrIp': '0.0.0.0/0',
81 'Description': 'SSH from entire Internet',
82 },
83 ],
84 },
85 ],
86 },
87 'windows-dev-1': {
88 'description': 'Mercurial Windows instances that perform build automation',
89 'ingress': [
90 {
91 'FromPort': 22,
92 'ToPort': 22,
93 'IpProtocol': 'tcp',
94 'IpRanges': [
95 {
96 'CidrIp': '0.0.0.0/0',
97 'Description': 'SSH from entire Internet',
98 },
99 ],
100 },
101 {
102 'FromPort': 3389,
103 'ToPort': 3389,
104 'IpProtocol': 'tcp',
105 'IpRanges': [
106 {
107 'CidrIp': '0.0.0.0/0',
108 'Description': 'RDP from entire Internet',
109 },
110 ],
111 },
112 {
113 'FromPort': 5985,
114 'ToPort': 5986,
115 'IpProtocol': 'tcp',
116 'IpRanges': [
117 {
118 'CidrIp': '0.0.0.0/0',
119 'Description': 'PowerShell Remoting (Windows Remote Management)',
120 },
121 ],
122 },
123 ],
124 },
125 }
126
127
128 IAM_ROLES = {
129 'ephemeral-ec2-role-1': {
130 'description': 'Mercurial temporary EC2 instances',
131 'policy_arns': [
132 'arn:aws:iam::aws:policy/service-role/AmazonEC2RoleforSSM',
133 ],
134 },
135 }
136
137
138 ASSUME_ROLE_POLICY_DOCUMENT = '''
139 {
140 "Version": "2012-10-17",
141 "Statement": [
142 {
143 "Effect": "Allow",
144 "Principal": {
145 "Service": "ec2.amazonaws.com"
146 },
147 "Action": "sts:AssumeRole"
148 }
149 ]
150 }
151 '''.strip()
152
153
154 IAM_INSTANCE_PROFILES = {
155 'ephemeral-ec2-1': {
156 'roles': [
157 'ephemeral-ec2-role-1',
158 ],
159 }
160 }
161
162
163 # User Data for Windows EC2 instance. Mainly used to set the password
164 # and configure WinRM.
165 # Inspired by the User Data script used by Packer
166 # (from https://www.packer.io/intro/getting-started/build-image.html).
167 WINDOWS_USER_DATA = r'''
168 <powershell>
169
170 # TODO enable this once we figure out what is failing.
171 #$ErrorActionPreference = "stop"
172
173 # Set administrator password
174 net user Administrator "%s"
175 wmic useraccount where "name='Administrator'" set PasswordExpires=FALSE
176
177 # And set it via EC2Launch so it persists across reboots.
178 $config = & $env:ProgramFiles\Amazon\EC2Launch\EC2Launch.exe get-agent-config --format json | ConvertFrom-Json
179 $config | ConvertTo-Json -Depth 6 | Out-File -encoding UTF8 $env:ProgramData/Amazon/EC2Launch/config/agent-config.yml
180 $setAdminAccount = @"
181 {
182 "task": "setAdminAccount",
183 "inputs": {
184 "password": {
185 "type": "static",
186 "data": "%s"
187 }
188 }
189 }
190 "@
191 $config.config | %%{if($_.stage -eq 'preReady'){$_.tasks += (ConvertFrom-Json -InputObject $setAdminAccount)}}
192 $config | ConvertTo-Json -Depth 6 | Out-File -encoding UTF8 $env:ProgramData/Amazon/EC2Launch/config/agent-config.yml
193
194 # First, make sure WinRM can't be connected to
195 netsh advfirewall firewall set rule name="Windows Remote Management (HTTP-In)" new enable=yes action=block
196
197 # Delete any existing WinRM listeners
198 winrm delete winrm/config/listener?Address=*+Transport=HTTP 2>$Null
199 winrm delete winrm/config/listener?Address=*+Transport=HTTPS 2>$Null
200
201 # Create a new WinRM listener and configure
202 winrm create winrm/config/listener?Address=*+Transport=HTTP
203 winrm set winrm/config/winrs '@{MaxMemoryPerShellMB="0"}'
204 winrm set winrm/config '@{MaxTimeoutms="7200000"}'
205 winrm set winrm/config/service '@{AllowUnencrypted="true"}'
206 winrm set winrm/config/service '@{MaxConcurrentOperationsPerUser="12000"}'
207 winrm set winrm/config/service/auth '@{Basic="true"}'
208 winrm set winrm/config/client/auth '@{Basic="true"}'
209
210 # Configure UAC to allow privilege elevation in remote shells
211 $Key = 'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Policies\System'
212 $Setting = 'LocalAccountTokenFilterPolicy'
213 Set-ItemProperty -Path $Key -Name $Setting -Value 1 -Force
214
215 # Avoid long usernames in the temp directory path because the '~' causes extra quoting in ssh output
216 [System.Environment]::SetEnvironmentVariable('TMP', 'C:\Temp', [System.EnvironmentVariableTarget]::User)
217 [System.Environment]::SetEnvironmentVariable('TEMP', 'C:\Temp', [System.EnvironmentVariableTarget]::User)
218
219 # Configure and restart the WinRM Service; Enable the required firewall exception
220 Stop-Service -Name WinRM
221 Set-Service -Name WinRM -StartupType Automatic
222 netsh advfirewall firewall set rule name="Windows Remote Management (HTTP-In)" new action=allow localip=any remoteip=any
223 Start-Service -Name WinRM
224
225 # Disable firewall on private network interfaces so prompts don't appear.
226 Set-NetFirewallProfile -Name private -Enabled false
227 </powershell>
228 '''.lstrip()
229
230
231 WINDOWS_BOOTSTRAP_POWERSHELL = '''
232 Write-Output "installing PowerShell dependencies"
233 Install-PackageProvider -Name NuGet -MinimumVersion 2.8.5.201 -Force
234 Set-PSRepository -Name PSGallery -InstallationPolicy Trusted
235 Install-Module -Name OpenSSHUtils -RequiredVersion 0.0.2.0
236
237 Write-Output "installing OpenSSL server"
238 Add-WindowsCapability -Online -Name OpenSSH.Server~~~~0.0.1.0
239 # Various tools will attempt to use older versions of .NET. So we enable
240 # the feature that provides them so it doesn't have to be auto-enabled
241 # later.
242 Write-Output "enabling .NET Framework feature"
243 Install-WindowsFeature -Name Net-Framework-Core
244 '''
245
246
247 class AWSConnection:
248 """Manages the state of a connection with AWS."""
249
250 def __init__(self, automation, region: str, ensure_ec2_state: bool = True):
251 self.automation = automation
252 self.local_state_path = automation.state_path
253
254 self.prefix = 'hg-'
255
256 self.session = boto3.session.Session(region_name=region)
257 self.ec2client = self.session.client('ec2')
258 self.ec2resource = self.session.resource('ec2')
259 self.iamclient = self.session.client('iam')
260 self.iamresource = self.session.resource('iam')
261 self.security_groups = {}
262
263 if ensure_ec2_state:
264 ensure_key_pairs(automation.state_path, self.ec2resource)
265 self.security_groups = ensure_security_groups(self.ec2resource)
266 ensure_iam_state(self.iamclient, self.iamresource)
267
268 def key_pair_path_private(self, name):
269 """Path to a key pair private key file."""
270 return self.local_state_path / 'keys' / ('keypair-%s' % name)
271
272 def key_pair_path_public(self, name):
273 return self.local_state_path / 'keys' / ('keypair-%s.pub' % name)
274
275
276 def rsa_key_fingerprint(p: pathlib.Path):
277 """Compute the fingerprint of an RSA private key."""
278
279 # TODO use rsa package.
280 res = subprocess.run(
281 [
282 'openssl',
283 'pkcs8',
284 '-in',
285 str(p),
286 '-nocrypt',
287 '-topk8',
288 '-outform',
289 'DER',
290 ],
291 capture_output=True,
292 check=True,
293 )
294
295 sha1 = hashlib.sha1(res.stdout).hexdigest()
296 return ':'.join(a + b for a, b in zip(sha1[::2], sha1[1::2]))
297
298
299 def ensure_key_pairs(state_path: pathlib.Path, ec2resource, prefix='hg-'):
300 remote_existing = {}
301
302 for kpi in ec2resource.key_pairs.all():
303 if kpi.name.startswith(prefix):
304 remote_existing[kpi.name[len(prefix) :]] = kpi.key_fingerprint
305
306 # Validate that we have these keys locally.
307 key_path = state_path / 'keys'
308 key_path.mkdir(exist_ok=True, mode=0o700)
309
310 def remove_remote(name):
311 print('deleting key pair %s' % name)
312 key = ec2resource.KeyPair(name)
313 key.delete()
314
315 def remove_local(name):
316 pub_full = key_path / ('keypair-%s.pub' % name)
317 priv_full = key_path / ('keypair-%s' % name)
318
319 print('removing %s' % pub_full)
320 pub_full.unlink()
321 print('removing %s' % priv_full)
322 priv_full.unlink()
323
324 local_existing = {}
325
326 for f in sorted(os.listdir(key_path)):
327 if not f.startswith('keypair-') or not f.endswith('.pub'):
328 continue
329
330 name = f[len('keypair-') : -len('.pub')]
331
332 pub_full = key_path / f
333 priv_full = key_path / ('keypair-%s' % name)
334
335 with open(pub_full, encoding='ascii') as fh:
336 data = fh.read()
337
338 if not data.startswith('ssh-rsa '):
339 print(
340 'unexpected format for key pair file: %s; removing' % pub_full
341 )
342 pub_full.unlink()
343 priv_full.unlink()
344 continue
345
346 local_existing[name] = rsa_key_fingerprint(priv_full)
347
348 for name in sorted(set(remote_existing) | set(local_existing)):
349 if name not in local_existing:
350 actual = '%s%s' % (prefix, name)
351 print('remote key %s does not exist locally' % name)
352 remove_remote(actual)
353 del remote_existing[name]
354
355 elif name not in remote_existing:
356 print('local key %s does not exist remotely' % name)
357 remove_local(name)
358 del local_existing[name]
359
360 elif remote_existing[name] != local_existing[name]:
361 print(
362 'key fingerprint mismatch for %s; '
363 'removing from local and remote' % name
364 )
365 remove_local(name)
366 remove_remote('%s%s' % (prefix, name))
367 del local_existing[name]
368 del remote_existing[name]
369
370 missing = KEY_PAIRS - set(remote_existing)
371
372 for name in sorted(missing):
373 actual = '%s%s' % (prefix, name)
374 print('creating key pair %s' % actual)
375
376 priv_full = key_path / ('keypair-%s' % name)
377 pub_full = key_path / ('keypair-%s.pub' % name)
378
379 kp = ec2resource.create_key_pair(KeyName=actual)
380
381 with priv_full.open('w', encoding='ascii') as fh:
382 fh.write(kp.key_material)
383 fh.write('\n')
384
385 priv_full.chmod(0o0600)
386
387 # SSH public key can be extracted via `ssh-keygen`.
388 with pub_full.open('w', encoding='ascii') as fh:
389 subprocess.run(
390 ['ssh-keygen', '-y', '-f', str(priv_full)],
391 stdout=fh,
392 check=True,
393 )
394
395 pub_full.chmod(0o0600)
396
397
398 def delete_instance_profile(profile):
399 for role in profile.roles:
400 print(
401 'removing role %s from instance profile %s'
402 % (role.name, profile.name)
403 )
404 profile.remove_role(RoleName=role.name)
405
406 print('deleting instance profile %s' % profile.name)
407 profile.delete()
408
409
410 def ensure_iam_state(iamclient, iamresource, prefix='hg-'):
411 """Ensure IAM state is in sync with our canonical definition."""
412
413 remote_profiles = {}
414
415 for profile in iamresource.instance_profiles.all():
416 if profile.name.startswith(prefix):
417 remote_profiles[profile.name[len(prefix) :]] = profile
418
419 for name in sorted(set(remote_profiles) - set(IAM_INSTANCE_PROFILES)):
420 delete_instance_profile(remote_profiles[name])
421 del remote_profiles[name]
422
423 remote_roles = {}
424
425 for role in iamresource.roles.all():
426 if role.name.startswith(prefix):
427 remote_roles[role.name[len(prefix) :]] = role
428
429 for name in sorted(set(remote_roles) - set(IAM_ROLES)):
430 role = remote_roles[name]
431
432 print('removing role %s' % role.name)
433 role.delete()
434 del remote_roles[name]
435
436 # We've purged remote state that doesn't belong. Create missing
437 # instance profiles and roles.
438 for name in sorted(set(IAM_INSTANCE_PROFILES) - set(remote_profiles)):
439 actual = '%s%s' % (prefix, name)
440 print('creating IAM instance profile %s' % actual)
441
442 profile = iamresource.create_instance_profile(
443 InstanceProfileName=actual
444 )
445 remote_profiles[name] = profile
446
447 waiter = iamclient.get_waiter('instance_profile_exists')
448 waiter.wait(InstanceProfileName=actual)
449 print('IAM instance profile %s is available' % actual)
450
451 for name in sorted(set(IAM_ROLES) - set(remote_roles)):
452 entry = IAM_ROLES[name]
453
454 actual = '%s%s' % (prefix, name)
455 print('creating IAM role %s' % actual)
456
457 role = iamresource.create_role(
458 RoleName=actual,
459 Description=entry['description'],
460 AssumeRolePolicyDocument=ASSUME_ROLE_POLICY_DOCUMENT,
461 )
462
463 waiter = iamclient.get_waiter('role_exists')
464 waiter.wait(RoleName=actual)
465 print('IAM role %s is available' % actual)
466
467 remote_roles[name] = role
468
469 for arn in entry['policy_arns']:
470 print('attaching policy %s to %s' % (arn, role.name))
471 role.attach_policy(PolicyArn=arn)
472
473 # Now reconcile state of profiles.
474 for name, meta in sorted(IAM_INSTANCE_PROFILES.items()):
475 profile = remote_profiles[name]
476 wanted = {'%s%s' % (prefix, role) for role in meta['roles']}
477 have = {role.name for role in profile.roles}
478
479 for role in sorted(have - wanted):
480 print('removing role %s from %s' % (role, profile.name))
481 profile.remove_role(RoleName=role)
482
483 for role in sorted(wanted - have):
484 print('adding role %s to %s' % (role, profile.name))
485 profile.add_role(RoleName=role)
486
487
488 def find_image(ec2resource, owner_id, name, reverse_sort_field=None):
489 """Find an AMI by its owner ID and name."""
490
491 images = ec2resource.images.filter(
492 Filters=[
493 {
494 'Name': 'owner-id',
495 'Values': [owner_id],
496 },
497 {
498 'Name': 'state',
499 'Values': ['available'],
500 },
501 {
502 'Name': 'image-type',
503 'Values': ['machine'],
504 },
505 {
506 'Name': 'name',
507 'Values': [name],
508 },
509 ]
510 )
511
512 if reverse_sort_field:
513 images = sorted(
514 images,
515 key=lambda image: getattr(image, reverse_sort_field),
516 reverse=True,
517 )
518
519 for image in images:
520 return image
521
522 raise Exception('unable to find image for %s' % name)
523
524
525 def ensure_security_groups(ec2resource, prefix='hg-'):
526 """Ensure all necessary Mercurial security groups are present.
527
528 All security groups are prefixed with ``hg-`` by default. Any security
529 groups having this prefix but aren't in our list are deleted.
530 """
531 existing = {}
532
533 for group in ec2resource.security_groups.all():
534 if group.group_name.startswith(prefix):
535 existing[group.group_name[len(prefix) :]] = group
536
537 purge = set(existing) - set(SECURITY_GROUPS)
538
539 for name in sorted(purge):
540 group = existing[name]
541 print('removing legacy security group: %s' % group.group_name)
542 group.delete()
543
544 security_groups = {}
545
546 for name, group in sorted(SECURITY_GROUPS.items()):
547 if name in existing:
548 security_groups[name] = existing[name]
549 continue
550
551 actual = '%s%s' % (prefix, name)
552 print('adding security group %s' % actual)
553
554 group_res = ec2resource.create_security_group(
555 Description=group['description'],
556 GroupName=actual,
557 )
558
559 group_res.authorize_ingress(
560 IpPermissions=group['ingress'],
561 )
562
563 security_groups[name] = group_res
564
565 return security_groups
566
567
568 def terminate_ec2_instances(ec2resource, prefix='hg-'):
569 """Terminate all EC2 instances managed by us."""
570 waiting = []
571
572 for instance in ec2resource.instances.all():
573 if instance.state['Name'] == 'terminated':
574 continue
575
576 for tag in instance.tags or []:
577 if tag['Key'] == 'Name' and tag['Value'].startswith(prefix):
578 print('terminating %s' % instance.id)
579 instance.terminate()
580 waiting.append(instance)
581
582 for instance in waiting:
583 instance.wait_until_terminated()
584
585
586 def remove_resources(c, prefix='hg-'):
587 """Purge all of our resources in this EC2 region."""
588 ec2resource = c.ec2resource
589 iamresource = c.iamresource
590
591 terminate_ec2_instances(ec2resource, prefix=prefix)
592
593 for image in ec2resource.images.filter(Owners=['self']):
594 if image.name.startswith(prefix):
595 remove_ami(ec2resource, image)
596
597 for group in ec2resource.security_groups.all():
598 if group.group_name.startswith(prefix):
599 print('removing security group %s' % group.group_name)
600 group.delete()
601
602 for profile in iamresource.instance_profiles.all():
603 if profile.name.startswith(prefix):
604 delete_instance_profile(profile)
605
606 for role in iamresource.roles.all():
607 if role.name.startswith(prefix):
608 for p in role.attached_policies.all():
609 print('detaching policy %s from %s' % (p.arn, role.name))
610 role.detach_policy(PolicyArn=p.arn)
611
612 print('removing role %s' % role.name)
613 role.delete()
614
615
616 def wait_for_ip_addresses(instances):
617 """Wait for the public IP addresses of an iterable of instances."""
618 for instance in instances:
619 while True:
620 if not instance.public_ip_address:
621 time.sleep(2)
622 instance.reload()
623 continue
624
625 print(
626 'public IP address for %s: %s'
627 % (instance.id, instance.public_ip_address)
628 )
629 break
630
631
632 def remove_ami(ec2resource, image):
633 """Remove an AMI and its underlying snapshots."""
634 snapshots = []
635
636 for device in image.block_device_mappings:
637 if 'Ebs' in device:
638 snapshots.append(ec2resource.Snapshot(device['Ebs']['SnapshotId']))
639
640 print('deregistering %s' % image.id)
641 image.deregister()
642
643 for snapshot in snapshots:
644 print('deleting snapshot %s' % snapshot.id)
645 snapshot.delete()
646
647
648 def wait_for_ssm(ssmclient, instances):
649 """Wait for SSM to come online for an iterable of instance IDs."""
650 while True:
651 res = ssmclient.describe_instance_information(
652 Filters=[
653 {
654 'Key': 'InstanceIds',
655 'Values': [i.id for i in instances],
656 },
657 ],
658 )
659
660 available = len(res['InstanceInformationList'])
661 wanted = len(instances)
662
663 print('%d/%d instances available in SSM' % (available, wanted))
664
665 if available == wanted:
666 return
667
668 time.sleep(2)
669
670
671 def run_ssm_command(ssmclient, instances, document_name, parameters):
672 """Run a PowerShell script on an EC2 instance."""
673
674 res = ssmclient.send_command(
675 InstanceIds=[i.id for i in instances],
676 DocumentName=document_name,
677 Parameters=parameters,
678 CloudWatchOutputConfig={
679 'CloudWatchOutputEnabled': True,
680 },
681 )
682
683 command_id = res['Command']['CommandId']
684
685 for instance in instances:
686 while True:
687 try:
688 res = ssmclient.get_command_invocation(
689 CommandId=command_id,
690 InstanceId=instance.id,
691 )
692 except botocore.exceptions.ClientError as e:
693 if e.response['Error']['Code'] == 'InvocationDoesNotExist':
694 print('could not find SSM command invocation; waiting')
695 time.sleep(1)
696 continue
697 else:
698 raise
699
700 if res['Status'] == 'Success':
701 break
702 elif res['Status'] in ('Pending', 'InProgress', 'Delayed'):
703 time.sleep(2)
704 else:
705 raise Exception(
706 'command failed on %s: %s' % (instance.id, res['Status'])
707 )
708
709
710 @contextlib.contextmanager
711 def temporary_ec2_instances(ec2resource, config):
712 """Create temporary EC2 instances.
713
714 This is a proxy to ``ec2client.run_instances(**config)`` that takes care of
715 managing the lifecycle of the instances.
716
717 When the context manager exits, the instances are terminated.
718
719 The context manager evaluates to the list of data structures
720 describing each created instance. The instances may not be available
721 for work immediately: it is up to the caller to wait for the instance
722 to start responding.
723 """
724
725 ids = None
726
727 try:
728 res = ec2resource.create_instances(**config)
729
730 ids = [i.id for i in res]
731 print('started instances: %s' % ' '.join(ids))
732
733 yield res
734 finally:
735 if ids:
736 print('terminating instances: %s' % ' '.join(ids))
737 for instance in res:
738 instance.terminate()
739 print('terminated %d instances' % len(ids))
740
741
742 @contextlib.contextmanager
743 def create_temp_windows_ec2_instances(
744 c: AWSConnection, config, bootstrap: bool = False
745 ):
746 """Create temporary Windows EC2 instances.
747
748 This is a higher-level wrapper around ``create_temp_ec2_instances()`` that
749 configures the Windows instance for Windows Remote Management. The emitted
750 instances will have a ``winrm_client`` attribute containing a
751 ``pypsrp.client.Client`` instance bound to the instance.
752 """
753 if 'IamInstanceProfile' in config:
754 raise ValueError('IamInstanceProfile cannot be provided in config')
755 if 'UserData' in config:
756 raise ValueError('UserData cannot be provided in config')
757
758 password = c.automation.default_password()
759
760 config = copy.deepcopy(config)
761 config['IamInstanceProfile'] = {
762 'Name': 'hg-ephemeral-ec2-1',
763 }
764 config.setdefault('TagSpecifications', []).append(
765 {
766 'ResourceType': 'instance',
767 'Tags': [{'Key': 'Name', 'Value': 'hg-temp-windows'}],
768 }
769 )
770
771 if bootstrap:
772 config['UserData'] = WINDOWS_USER_DATA % (password, password)
773
774 with temporary_ec2_instances(c.ec2resource, config) as instances:
775 wait_for_ip_addresses(instances)
776
777 print('waiting for Windows Remote Management service...')
778
779 for instance in instances:
780 client = wait_for_winrm(
781 instance.public_ip_address, 'Administrator', password
782 )
783 print('established WinRM connection to %s' % instance.id)
784 instance.winrm_client = client
785
786 yield instances
787
788
789 def resolve_fingerprint(fingerprint):
790 fingerprint = json.dumps(fingerprint, sort_keys=True)
791 return hashlib.sha256(fingerprint.encode('utf-8')).hexdigest()
792
793
794 def find_and_reconcile_image(ec2resource, name, fingerprint):
795 """Attempt to find an existing EC2 AMI with a name and fingerprint.
796
797 If an image with the specified fingerprint is found, it is returned.
798 Otherwise None is returned.
799
800 Existing images for the specified name that don't have the specified
801 fingerprint or are missing required metadata or deleted.
802 """
803 # Find existing AMIs with this name and delete the ones that are invalid.
804 # Store a reference to a good image so it can be returned one the
805 # image state is reconciled.
806 images = ec2resource.images.filter(
807 Filters=[{'Name': 'name', 'Values': [name]}]
808 )
809
810 existing_image = None
811
812 for image in images:
813 if image.tags is None:
814 print(
815 'image %s for %s lacks required tags; removing'
816 % (image.id, image.name)
817 )
818 remove_ami(ec2resource, image)
819 else:
820 tags = {t['Key']: t['Value'] for t in image.tags}
821
822 if tags.get('HGIMAGEFINGERPRINT') == fingerprint:
823 existing_image = image
824 else:
825 print(
826 'image %s for %s has wrong fingerprint; removing'
827 % (image.id, image.name)
828 )
829 remove_ami(ec2resource, image)
830
831 return existing_image
832
833
834 def create_ami_from_instance(
835 ec2client, instance, name, description, fingerprint
836 ):
837 """Create an AMI from a running instance.
838
839 Returns the ``ec2resource.Image`` representing the created AMI.
840 """
841 instance.stop()
842
843 ec2client.get_waiter('instance_stopped').wait(
844 InstanceIds=[instance.id],
845 WaiterConfig={
846 'Delay': 5,
847 },
848 )
849 print('%s is stopped' % instance.id)
850
851 image = instance.create_image(
852 Name=name,
853 Description=description,
854 )
855
856 image.create_tags(
857 Tags=[
858 {
859 'Key': 'HGIMAGEFINGERPRINT',
860 'Value': fingerprint,
861 },
862 ]
863 )
864
865 print('waiting for image %s' % image.id)
866
867 ec2client.get_waiter('image_available').wait(
868 ImageIds=[image.id],
869 )
870
871 print('image %s available as %s' % (image.id, image.name))
872
873 return image
874
875
876 def ensure_linux_dev_ami(c: AWSConnection, distro='debian10', prefix='hg-'):
877 """Ensures a Linux development AMI is available and up-to-date.
878
879 Returns an ``ec2.Image`` of either an existing AMI or a newly-built one.
880 """
881 ec2client = c.ec2client
882 ec2resource = c.ec2resource
883
884 name = '%s%s-%s' % (prefix, 'linux-dev', distro)
885
886 if distro == 'debian9':
887 image = find_image(
888 ec2resource,
889 DEBIAN_ACCOUNT_ID,
890 'debian-stretch-hvm-x86_64-gp2-2019-09-08-17994',
891 )
892 ssh_username = 'admin'
893 elif distro == 'debian10':
894 image = find_image(
895 ec2resource,
896 DEBIAN_ACCOUNT_ID_2,
897 'debian-10-amd64-20190909-10',
898 )
899 ssh_username = 'admin'
900 elif distro == 'ubuntu18.04':
901 image = find_image(
902 ec2resource,
903 UBUNTU_ACCOUNT_ID,
904 'ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-20190918',
905 )
906 ssh_username = 'ubuntu'
907 elif distro == 'ubuntu19.04':
908 image = find_image(
909 ec2resource,
910 UBUNTU_ACCOUNT_ID,
911 'ubuntu/images/hvm-ssd/ubuntu-disco-19.04-amd64-server-20190918',
912 )
913 ssh_username = 'ubuntu'
914 else:
915 raise ValueError('unsupported Linux distro: %s' % distro)
916
917 config = {
918 'BlockDeviceMappings': [
919 {
920 'DeviceName': image.block_device_mappings[0]['DeviceName'],
921 'Ebs': {
922 'DeleteOnTermination': True,
923 'VolumeSize': 10,
924 'VolumeType': 'gp3',
925 },
926 },
927 ],
928 'EbsOptimized': True,
929 'ImageId': image.id,
930 'InstanceInitiatedShutdownBehavior': 'stop',
931 # 8 VCPUs for compiling Python.
932 'InstanceType': 't3.2xlarge',
933 'KeyName': '%sautomation' % prefix,
934 'MaxCount': 1,
935 'MinCount': 1,
936 'SecurityGroupIds': [c.security_groups['linux-dev-1'].id],
937 }
938
939 requirements3_path = (
940 pathlib.Path(__file__).parent.parent / 'linux-requirements-py3.txt'
941 )
942 requirements35_path = (
943 pathlib.Path(__file__).parent.parent / 'linux-requirements-py3.5.txt'
944 )
945 with requirements3_path.open('r', encoding='utf-8') as fh:
946 requirements3 = fh.read()
947 with requirements35_path.open('r', encoding='utf-8') as fh:
948 requirements35 = fh.read()
949
950 # Compute a deterministic fingerprint to determine whether image needs to
951 # be regenerated.
952 fingerprint = resolve_fingerprint(
953 {
954 'instance_config': config,
955 'bootstrap_script': BOOTSTRAP_DEBIAN,
956 'requirements_py3': requirements3,
957 'requirements_py35': requirements35,
958 }
959 )
960
961 existing_image = find_and_reconcile_image(ec2resource, name, fingerprint)
962
963 if existing_image:
964 return existing_image
965
966 print('no suitable %s image found; creating one...' % name)
967
968 with temporary_ec2_instances(ec2resource, config) as instances:
969 wait_for_ip_addresses(instances)
970
971 instance = instances[0]
972
973 client = wait_for_ssh(
974 instance.public_ip_address,
975 22,
976 username=ssh_username,
977 key_filename=str(c.key_pair_path_private('automation')),
978 )
979
980 home = '/home/%s' % ssh_username
981
982 with client:
983 print('connecting to SSH server')
984 sftp = client.open_sftp()
985
986 print('uploading bootstrap files')
987 with sftp.open('%s/bootstrap' % home, 'wb') as fh:
988 fh.write(BOOTSTRAP_DEBIAN)
989 fh.chmod(0o0700)
990
991 with sftp.open('%s/requirements-py3.txt' % home, 'wb') as fh:
992 fh.write(requirements3)
993 fh.chmod(0o0700)
994
995 with sftp.open('%s/requirements-py3.5.txt' % home, 'wb') as fh:
996 fh.write(requirements35)
997 fh.chmod(0o0700)
998
999 print('executing bootstrap')
1000 chan, stdin, stdout = ssh_exec_command(
1001 client, '%s/bootstrap' % home
1002 )
1003 stdin.close()
1004
1005 for line in stdout:
1006 print(line, end='')
1007
1008 res = chan.recv_exit_status()
1009 if res:
1010 raise Exception('non-0 exit from bootstrap: %d' % res)
1011
1012 print(
1013 'bootstrap completed; stopping %s to create %s'
1014 % (instance.id, name)
1015 )
1016
1017 return create_ami_from_instance(
1018 ec2client,
1019 instance,
1020 name,
1021 'Mercurial Linux development environment',
1022 fingerprint,
1023 )
1024
1025
1026 @contextlib.contextmanager
1027 def temporary_linux_dev_instances(
1028 c: AWSConnection,
1029 image,
1030 instance_type,
1031 prefix='hg-',
1032 ensure_extra_volume=False,
1033 ):
1034 """Create temporary Linux development EC2 instances.
1035
1036 Context manager resolves to a list of ``ec2.Instance`` that were created
1037 and are running.
1038
1039 ``ensure_extra_volume`` can be set to ``True`` to require that instances
1040 have a 2nd storage volume available other than the primary AMI volume.
1041 For instance types with instance storage, this does nothing special.
1042 But for instance types without instance storage, an additional EBS volume
1043 will be added to the instance.
1044
1045 Instances have an ``ssh_client`` attribute containing a paramiko SSHClient
1046 instance bound to the instance.
1047
1048 Instances have an ``ssh_private_key_path`` attributing containing the
1049 str path to the SSH private key to connect to the instance.
1050 """
1051
1052 block_device_mappings = [
1053 {
1054 'DeviceName': image.block_device_mappings[0]['DeviceName'],
1055 'Ebs': {
1056 'DeleteOnTermination': True,
1057 'VolumeSize': 12,
1058 'VolumeType': 'gp3',
1059 },
1060 }
1061 ]
1062
1063 # This is not an exhaustive list of instance types having instance storage.
1064 # But
1065 if ensure_extra_volume and not instance_type.startswith(
1066 tuple(INSTANCE_TYPES_WITH_STORAGE)
1067 ):
1068 main_device = block_device_mappings[0]['DeviceName']
1069
1070 if main_device == 'xvda':
1071 second_device = 'xvdb'
1072 elif main_device == '/dev/sda1':
1073 second_device = '/dev/sdb'
1074 else:
1075 raise ValueError(
1076 'unhandled primary EBS device name: %s' % main_device
1077 )
1078
1079 block_device_mappings.append(
1080 {
1081 'DeviceName': second_device,
1082 'Ebs': {
1083 'DeleteOnTermination': True,
1084 'VolumeSize': 8,
1085 'VolumeType': 'gp3',
1086 },
1087 }
1088 )
1089
1090 config = {
1091 'BlockDeviceMappings': block_device_mappings,
1092 'EbsOptimized': True,
1093 'ImageId': image.id,
1094 'InstanceInitiatedShutdownBehavior': 'terminate',
1095 'InstanceType': instance_type,
1096 'KeyName': '%sautomation' % prefix,
1097 'MaxCount': 1,
1098 'MinCount': 1,
1099 'SecurityGroupIds': [c.security_groups['linux-dev-1'].id],
1100 }
1101
1102 with temporary_ec2_instances(c.ec2resource, config) as instances:
1103 wait_for_ip_addresses(instances)
1104
1105 ssh_private_key_path = str(c.key_pair_path_private('automation'))
1106
1107 for instance in instances:
1108 client = wait_for_ssh(
1109 instance.public_ip_address,
1110 22,
1111 username='hg',
1112 key_filename=ssh_private_key_path,
1113 )
1114
1115 instance.ssh_client = client
1116 instance.ssh_private_key_path = ssh_private_key_path
1117
1118 try:
1119 yield instances
1120 finally:
1121 for instance in instances:
1122 instance.ssh_client.close()
1123
1124
1125 def ensure_windows_dev_ami(
1126 c: AWSConnection,
1127 prefix='hg-',
1128 base_image_name=WINDOWS_BASE_IMAGE_NAME,
1129 ):
1130 """Ensure Windows Development AMI is available and up-to-date.
1131
1132 If necessary, a modern AMI will be built by starting a temporary EC2
1133 instance and bootstrapping it.
1134
1135 Obsolete AMIs will be deleted so there is only a single AMI having the
1136 desired name.
1137
1138 Returns an ``ec2.Image`` of either an existing AMI or a newly-built
1139 one.
1140 """
1141 ec2client = c.ec2client
1142 ec2resource = c.ec2resource
1143 ssmclient = c.session.client('ssm')
1144
1145 name = '%s%s' % (prefix, 'windows-dev')
1146
1147 image = find_image(
1148 ec2resource,
1149 AMAZON_ACCOUNT_ID,
1150 base_image_name,
1151 reverse_sort_field="name",
1152 )
1153
1154 config = {
1155 'BlockDeviceMappings': [
1156 {
1157 'DeviceName': '/dev/sda1',
1158 'Ebs': {
1159 'DeleteOnTermination': True,
1160 'VolumeSize': 32,
1161 'VolumeType': 'gp3',
1162 },
1163 }
1164 ],
1165 'ImageId': image.id,
1166 'InstanceInitiatedShutdownBehavior': 'stop',
1167 'InstanceType': 'm6i.large',
1168 'KeyName': '%sautomation' % prefix,
1169 'MaxCount': 1,
1170 'MinCount': 1,
1171 'SecurityGroupIds': [c.security_groups['windows-dev-1'].id],
1172 }
1173
1174 commands = [
1175 # Need to start the service so sshd_config is generated.
1176 'Start-Service sshd',
1177 'Write-Output "modifying sshd_config"',
1178 r'$content = Get-Content C:\ProgramData\ssh\sshd_config',
1179 '$content = $content -replace "Match Group administrators","" -replace "AuthorizedKeysFile __PROGRAMDATA__/ssh/administrators_authorized_keys",""',
1180 r'$content | Set-Content C:\ProgramData\ssh\sshd_config',
1181 'Import-Module OpenSSHUtils',
1182 r'Repair-SshdConfigPermission C:\ProgramData\ssh\sshd_config -Confirm:$false',
1183 'Restart-Service sshd',
1184 'Write-Output "installing OpenSSL client"',
1185 'Add-WindowsCapability -Online -Name OpenSSH.Client~~~~0.0.1.0',
1186 'Set-Service -Name sshd -StartupType "Automatic"',
1187 'Write-Output "OpenSSH server running"',
1188 ]
1189
1190 with INSTALL_WINDOWS_DEPENDENCIES.open('r', encoding='utf-8') as fh:
1191 commands.extend(l.rstrip() for l in fh)
1192
1193 # Disable Windows Defender when bootstrapping because it just slows
1194 # things down.
1195 commands.insert(0, 'Set-MpPreference -DisableRealtimeMonitoring $true')
1196 commands.append('Set-MpPreference -DisableRealtimeMonitoring $false')
1197
1198 # Trigger shutdown to prepare for imaging.
1199 commands.append(
1200 'Stop-Computer -ComputerName localhost',
1201 )
1202
1203 # Compute a deterministic fingerprint to determine whether image needs
1204 # to be regenerated.
1205 fingerprint = resolve_fingerprint(
1206 {
1207 'instance_config': config,
1208 'user_data': WINDOWS_USER_DATA,
1209 'initial_bootstrap': WINDOWS_BOOTSTRAP_POWERSHELL,
1210 'bootstrap_commands': commands,
1211 'base_image_name': base_image_name,
1212 }
1213 )
1214
1215 existing_image = find_and_reconcile_image(ec2resource, name, fingerprint)
1216
1217 if existing_image:
1218 return existing_image
1219
1220 print('no suitable Windows development image found; creating one...')
1221
1222 with create_temp_windows_ec2_instances(
1223 c, config, bootstrap=True
1224 ) as instances:
1225 assert len(instances) == 1
1226 instance = instances[0]
1227
1228 wait_for_ssm(ssmclient, [instance])
1229
1230 # On first boot, install various Windows updates.
1231 # We would ideally use PowerShell Remoting for this. However, there are
1232 # trust issues that make it difficult to invoke Windows Update
1233 # remotely. So we use SSM, which has a mechanism for running Windows
1234 # Update.
1235 print('installing Windows features...')
1236 run_ssm_command(
1237 ssmclient,
1238 [instance],
1239 'AWS-RunPowerShellScript',
1240 {
1241 'commands': WINDOWS_BOOTSTRAP_POWERSHELL.split('\n'),
1242 },
1243 )
1244
1245 # Reboot so all updates are fully applied.
1246 #
1247 # We don't use instance.reboot() here because it is asynchronous and
1248 # we don't know when exactly the instance has rebooted. It could take
1249 # a while to stop and we may start trying to interact with the instance
1250 # before it has rebooted.
1251 print('rebooting instance %s' % instance.id)
1252 instance.stop()
1253 ec2client.get_waiter('instance_stopped').wait(
1254 InstanceIds=[instance.id],
1255 WaiterConfig={
1256 'Delay': 5,
1257 },
1258 )
1259
1260 instance.start()
1261 wait_for_ip_addresses([instance])
1262
1263 # There is a race condition here between the User Data PS script running
1264 # and us connecting to WinRM. This can manifest as
1265 # "AuthorizationManager check failed" failures during run_powershell().
1266 # TODO figure out a workaround.
1267
1268 print('waiting for Windows Remote Management to come back...')
1269 client = wait_for_winrm(
1270 instance.public_ip_address,
1271 'Administrator',
1272 c.automation.default_password(),
1273 )
1274 print('established WinRM connection to %s' % instance.id)
1275 instance.winrm_client = client
1276
1277 print('bootstrapping instance...')
1278 run_powershell(instance.winrm_client, '\n'.join(commands))
1279
1280 print('bootstrap completed; stopping %s to create image' % instance.id)
1281 return create_ami_from_instance(
1282 ec2client,
1283 instance,
1284 name,
1285 'Mercurial Windows development environment',
1286 fingerprint,
1287 )
1288
1289
1290 @contextlib.contextmanager
1291 def temporary_windows_dev_instances(
1292 c: AWSConnection,
1293 image,
1294 instance_type,
1295 prefix='hg-',
1296 disable_antivirus=False,
1297 ):
1298 """Create a temporary Windows development EC2 instance.
1299
1300 Context manager resolves to the list of ``EC2.Instance`` that were created.
1301 """
1302 config = {
1303 'BlockDeviceMappings': [
1304 {
1305 'DeviceName': '/dev/sda1',
1306 'Ebs': {
1307 'DeleteOnTermination': True,
1308 'VolumeSize': 32,
1309 'VolumeType': 'gp3',
1310 },
1311 }
1312 ],
1313 'ImageId': image.id,
1314 'InstanceInitiatedShutdownBehavior': 'stop',
1315 'InstanceType': instance_type,
1316 'KeyName': '%sautomation' % prefix,
1317 'MaxCount': 1,
1318 'MinCount': 1,
1319 'SecurityGroupIds': [c.security_groups['windows-dev-1'].id],
1320 }
1321
1322 with create_temp_windows_ec2_instances(c, config) as instances:
1323 if disable_antivirus:
1324 for instance in instances:
1325 run_powershell(
1326 instance.winrm_client,
1327 'Set-MpPreference -DisableRealtimeMonitoring $true',
1328 )
1329
1330 yield instances