| 1 |
# Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved. |
| 2 |
# |
| 3 |
# Licensed under the Apache License, Version 2.0 (the "License"). You |
| 4 |
# may not use this file except in compliance with the License. A copy of |
| 5 |
# the License is located at |
| 6 |
# |
| 7 |
# https://aws.amazon.com/apache2.0/ |
| 8 |
# |
| 9 |
# or in the "license" file accompanying this file. This file is |
| 10 |
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF |
| 11 |
# ANY KIND, either express or implied. See the License for the specific |
| 12 |
# language governing permissions and limitations under the License. |
| 13 |
""" |
| 14 |
This file contains private functionality for interacting with the AWS |
| 15 |
Common Runtime library (awscrt) in boto3. |
| 16 |
|
| 17 |
All code contained within this file is for internal usage within this |
| 18 |
project and is not intended for external consumption. All interfaces |
| 19 |
contained within are subject to abrupt breaking changes. |
| 20 |
""" |
| 21 |
|
| 22 |
import threading |
| 23 |
|
| 24 |
import botocore.exceptions |
| 25 |
from botocore.session import Session |
| 26 |
from s3transfer.crt import ( |
| 27 |
BotocoreCRTCredentialsWrapper, |
| 28 |
BotocoreCRTRequestSerializer, |
| 29 |
CRTTransferManager, |
| 30 |
acquire_crt_s3_process_lock, |
| 31 |
create_s3_crt_client, |
| 32 |
) |
| 33 |
|
| 34 |
# Singletons for CRT-backed transfers |
| 35 |
CRT_S3_CLIENT = None |
| 36 |
BOTOCORE_CRT_SERIALIZER = None |
| 37 |
|
| 38 |
CLIENT_CREATION_LOCK = threading.Lock() |
| 39 |
PROCESS_LOCK_NAME = 'boto3' |
| 40 |
|
| 41 |
|
| 42 |
def _create_crt_client(session, config, region_name, cred_provider): |
| 43 |
"""Create a CRT S3 Client for file transfer. |
| 44 |
|
| 45 |
Instantiating many of these may lead to degraded performance or |
| 46 |
system resource exhaustion. |
| 47 |
""" |
| 48 |
create_crt_client_kwargs = { |
| 49 |
'region': region_name, |
| 50 |
'use_ssl': True, |
| 51 |
'crt_credentials_provider': cred_provider, |
| 52 |
} |
| 53 |
return create_s3_crt_client(**create_crt_client_kwargs) |
| 54 |
|
| 55 |
|
| 56 |
def _create_crt_request_serializer(session, region_name): |
| 57 |
return BotocoreCRTRequestSerializer( |
| 58 |
session, {'region_name': region_name, 'endpoint_url': None} |
| 59 |
) |
| 60 |
|
| 61 |
|
| 62 |
def _create_crt_s3_client( |
| 63 |
session, config, region_name, credentials, lock, **kwargs |
| 64 |
): |
| 65 |
"""Create boto3 wrapper class to manage crt lock reference and S3 client.""" |
| 66 |
cred_wrapper = BotocoreCRTCredentialsWrapper(credentials) |
| 67 |
cred_provider = cred_wrapper.to_crt_credentials_provider() |
| 68 |
return CRTS3Client( |
| 69 |
_create_crt_client(session, config, region_name, cred_provider), |
| 70 |
lock, |
| 71 |
region_name, |
| 72 |
cred_wrapper, |
| 73 |
) |
| 74 |
|
| 75 |
|
| 76 |
def _initialize_crt_transfer_primatives(client, config): |
| 77 |
lock = acquire_crt_s3_process_lock(PROCESS_LOCK_NAME) |
| 78 |
if lock is None: |
| 79 |
# If we're unable to acquire the lock, we cannot |
| 80 |
# use the CRT in this process and should default to |
| 81 |
# the classic s3transfer manager. |
| 82 |
return None, None |
| 83 |
|
| 84 |
session = Session() |
| 85 |
region_name = client.meta.region_name |
| 86 |
credentials = client._get_credentials() |
| 87 |
|
| 88 |
serializer = _create_crt_request_serializer(session, region_name) |
| 89 |
s3_client = _create_crt_s3_client( |
| 90 |
session, config, region_name, credentials, lock |
| 91 |
) |
| 92 |
return serializer, s3_client |
| 93 |
|
| 94 |
|
| 95 |
def get_crt_s3_client(client, config): |
| 96 |
global CRT_S3_CLIENT |
| 97 |
global BOTOCORE_CRT_SERIALIZER |
| 98 |
|
| 99 |
with CLIENT_CREATION_LOCK: |
| 100 |
if CRT_S3_CLIENT is None: |
| 101 |
serializer, s3_client = _initialize_crt_transfer_primatives( |
| 102 |
client, config |
| 103 |
) |
| 104 |
BOTOCORE_CRT_SERIALIZER = serializer |
| 105 |
CRT_S3_CLIENT = s3_client |
| 106 |
|
| 107 |
return CRT_S3_CLIENT |
| 108 |
|
| 109 |
|
| 110 |
class CRTS3Client: |
| 111 |
""" |
| 112 |
This wrapper keeps track of our underlying CRT client, the lock used to |
| 113 |
acquire it and the region we've used to instantiate the client. |
| 114 |
|
| 115 |
Due to limitations in the existing CRT interfaces, we can only make calls |
| 116 |
in a single region and does not support redirects. We track the region to |
| 117 |
ensure we don't use the CRT client when a successful request cannot be made. |
| 118 |
""" |
| 119 |
|
| 120 |
def __init__(self, crt_client, process_lock, region, cred_provider): |
| 121 |
self.crt_client = crt_client |
| 122 |
self.process_lock = process_lock |
| 123 |
self.region = region |
| 124 |
self.cred_provider = cred_provider |
| 125 |
|
| 126 |
|
| 127 |
def is_crt_compatible_request(client, crt_s3_client): |
| 128 |
""" |
| 129 |
Boto3 client must use same signing region and credentials |
| 130 |
as the CRT_S3_CLIENT singleton. Otherwise fallback to classic. |
| 131 |
""" |
| 132 |
if crt_s3_client is None: |
| 133 |
return False |
| 134 |
|
| 135 |
boto3_creds = client._get_credentials() |
| 136 |
if boto3_creds is None: |
| 137 |
return False |
| 138 |
|
| 139 |
is_same_identity = compare_identity( |
| 140 |
boto3_creds.get_frozen_credentials(), crt_s3_client.cred_provider |
| 141 |
) |
| 142 |
is_same_region = client.meta.region_name == crt_s3_client.region |
| 143 |
return is_same_region and is_same_identity |
| 144 |
|
| 145 |
|
| 146 |
def compare_identity(boto3_creds, crt_s3_creds): |
| 147 |
try: |
| 148 |
crt_creds = crt_s3_creds() |
| 149 |
except botocore.exceptions.NoCredentialsError: |
| 150 |
return False |
| 151 |
|
| 152 |
is_matching_identity = ( |
| 153 |
boto3_creds.access_key == crt_creds.access_key_id |
| 154 |
and boto3_creds.secret_key == crt_creds.secret_access_key |
| 155 |
and boto3_creds.token == crt_creds.session_token |
| 156 |
) |
| 157 |
return is_matching_identity |
| 158 |
|
| 159 |
|
| 160 |
def create_crt_transfer_manager(client, config): |
| 161 |
"""Create a CRTTransferManager for optimized data transfer.""" |
| 162 |
crt_s3_client = get_crt_s3_client(client, config) |
| 163 |
if is_crt_compatible_request(client, crt_s3_client): |
| 164 |
return CRTTransferManager( |
| 165 |
crt_s3_client.crt_client, BOTOCORE_CRT_SERIALIZER |
| 166 |
) |
| 167 |
return None |