Debian Codegraph @

boto3/crt.py

raw
1 # Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved.
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License"). You
4 # may not use this file except in compliance with the License. A copy of
5 # the License is located at
6 #
7 # https://aws.amazon.com/apache2.0/
8 #
9 # or in the "license" file accompanying this file. This file is
10 # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
11 # ANY KIND, either express or implied. See the License for the specific
12 # language governing permissions and limitations under the License.
13 """
14 This file contains private functionality for interacting with the AWS
15 Common Runtime library (awscrt) in boto3.
16
17 All code contained within this file is for internal usage within this
18 project and is not intended for external consumption. All interfaces
19 contained within are subject to abrupt breaking changes.
20 """
21
22 import threading
23
24 import botocore.exceptions
25 from botocore.session import Session
26 from s3transfer.crt import (
27 BotocoreCRTCredentialsWrapper,
28 BotocoreCRTRequestSerializer,
29 CRTTransferManager,
30 acquire_crt_s3_process_lock,
31 create_s3_crt_client,
32 )
33
34 # Singletons for CRT-backed transfers
35 CRT_S3_CLIENT = None
36 BOTOCORE_CRT_SERIALIZER = None
37
38 CLIENT_CREATION_LOCK = threading.Lock()
39 PROCESS_LOCK_NAME = 'boto3'
40
41
42 def _create_crt_client(session, config, region_name, cred_provider):
43 """Create a CRT S3 Client for file transfer.
44
45 Instantiating many of these may lead to degraded performance or
46 system resource exhaustion.
47 """
48 create_crt_client_kwargs = {
49 'region': region_name,
50 'use_ssl': True,
51 'crt_credentials_provider': cred_provider,
52 }
53 return create_s3_crt_client(**create_crt_client_kwargs)
54
55
56 def _create_crt_request_serializer(session, region_name):
57 return BotocoreCRTRequestSerializer(
58 session, {'region_name': region_name, 'endpoint_url': None}
59 )
60
61
62 def _create_crt_s3_client(
63 session, config, region_name, credentials, lock, **kwargs
64 ):
65 """Create boto3 wrapper class to manage crt lock reference and S3 client."""
66 cred_wrapper = BotocoreCRTCredentialsWrapper(credentials)
67 cred_provider = cred_wrapper.to_crt_credentials_provider()
68 return CRTS3Client(
69 _create_crt_client(session, config, region_name, cred_provider),
70 lock,
71 region_name,
72 cred_wrapper,
73 )
74
75
76 def _initialize_crt_transfer_primatives(client, config):
77 lock = acquire_crt_s3_process_lock(PROCESS_LOCK_NAME)
78 if lock is None:
79 # If we're unable to acquire the lock, we cannot
80 # use the CRT in this process and should default to
81 # the classic s3transfer manager.
82 return None, None
83
84 session = Session()
85 region_name = client.meta.region_name
86 credentials = client._get_credentials()
87
88 serializer = _create_crt_request_serializer(session, region_name)
89 s3_client = _create_crt_s3_client(
90 session, config, region_name, credentials, lock
91 )
92 return serializer, s3_client
93
94
95 def get_crt_s3_client(client, config):
96 global CRT_S3_CLIENT
97 global BOTOCORE_CRT_SERIALIZER
98
99 with CLIENT_CREATION_LOCK:
100 if CRT_S3_CLIENT is None:
101 serializer, s3_client = _initialize_crt_transfer_primatives(
102 client, config
103 )
104 BOTOCORE_CRT_SERIALIZER = serializer
105 CRT_S3_CLIENT = s3_client
106
107 return CRT_S3_CLIENT
108
109
110 class CRTS3Client:
111 """
112 This wrapper keeps track of our underlying CRT client, the lock used to
113 acquire it and the region we've used to instantiate the client.
114
115 Due to limitations in the existing CRT interfaces, we can only make calls
116 in a single region and does not support redirects. We track the region to
117 ensure we don't use the CRT client when a successful request cannot be made.
118 """
119
120 def __init__(self, crt_client, process_lock, region, cred_provider):
121 self.crt_client = crt_client
122 self.process_lock = process_lock
123 self.region = region
124 self.cred_provider = cred_provider
125
126
127 def is_crt_compatible_request(client, crt_s3_client):
128 """
129 Boto3 client must use same signing region and credentials
130 as the CRT_S3_CLIENT singleton. Otherwise fallback to classic.
131 """
132 if crt_s3_client is None:
133 return False
134
135 boto3_creds = client._get_credentials()
136 if boto3_creds is None:
137 return False
138
139 is_same_identity = compare_identity(
140 boto3_creds.get_frozen_credentials(), crt_s3_client.cred_provider
141 )
142 is_same_region = client.meta.region_name == crt_s3_client.region
143 return is_same_region and is_same_identity
144
145
146 def compare_identity(boto3_creds, crt_s3_creds):
147 try:
148 crt_creds = crt_s3_creds()
149 except botocore.exceptions.NoCredentialsError:
150 return False
151
152 is_matching_identity = (
153 boto3_creds.access_key == crt_creds.access_key_id
154 and boto3_creds.secret_key == crt_creds.secret_access_key
155 and boto3_creds.token == crt_creds.session_token
156 )
157 return is_matching_identity
158
159
160 def create_crt_transfer_manager(client, config):
161 """Create a CRTTransferManager for optimized data transfer."""
162 crt_s3_client = get_crt_s3_client(client, config)
163 if is_crt_compatible_request(client, crt_s3_client):
164 return CRTTransferManager(
165 crt_s3_client.crt_client, BOTOCORE_CRT_SERIALIZER
166 )
167 return None