| 1 |
# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. |
| 2 |
# |
| 3 |
# Licensed under the Apache License, Version 2.0 (the "License"). You |
| 4 |
# may not use this file except in compliance with the License. A copy of |
| 5 |
# the License is located at |
| 6 |
# |
| 7 |
# http://aws.amazon.com/apache2.0/ |
| 8 |
# |
| 9 |
# or in the "license" file accompanying this file. This file is |
| 10 |
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF |
| 11 |
# ANY KIND, either express or implied. See the License for the specific |
| 12 |
# language governing permissions and limitations under the License. |
| 13 |
import logging |
| 14 |
import re |
| 15 |
import threading |
| 16 |
from io import BytesIO |
| 17 |
|
| 18 |
import awscrt.http |
| 19 |
import awscrt.s3 |
| 20 |
import botocore.awsrequest |
| 21 |
import botocore.session |
| 22 |
from awscrt.auth import ( |
| 23 |
AwsCredentials, |
| 24 |
AwsCredentialsProvider, |
| 25 |
AwsSigningAlgorithm, |
| 26 |
AwsSigningConfig, |
| 27 |
) |
| 28 |
from awscrt.io import ( |
| 29 |
ClientBootstrap, |
| 30 |
ClientTlsContext, |
| 31 |
DefaultHostResolver, |
| 32 |
EventLoopGroup, |
| 33 |
TlsContextOptions, |
| 34 |
) |
| 35 |
from awscrt.s3 import S3Client, S3RequestTlsMode, S3RequestType |
| 36 |
from botocore import UNSIGNED |
| 37 |
from botocore.compat import urlsplit |
| 38 |
from botocore.config import Config |
| 39 |
from botocore.exceptions import NoCredentialsError |
| 40 |
from botocore.utils import ArnParser, InvalidArnException |
| 41 |
|
| 42 |
from s3transfer.constants import FULL_OBJECT_CHECKSUM_ARGS, MB |
| 43 |
from s3transfer.exceptions import TransferNotDoneError |
| 44 |
from s3transfer.futures import BaseTransferFuture, BaseTransferMeta |
| 45 |
from s3transfer.manager import TransferManager |
| 46 |
from s3transfer.utils import ( |
| 47 |
CallArgs, |
| 48 |
OSUtils, |
| 49 |
create_nested_client, |
| 50 |
get_callbacks, |
| 51 |
is_s3express_bucket, |
| 52 |
) |
| 53 |
|
| 54 |
logger = logging.getLogger(__name__) |
| 55 |
|
| 56 |
CRT_S3_PROCESS_LOCK = None |
| 57 |
|
| 58 |
|
| 59 |
def acquire_crt_s3_process_lock(name): |
| 60 |
# Currently, the CRT S3 client performs best when there is only one |
| 61 |
# instance of it running on a host. This lock allows an application to |
| 62 |
# signal across processes whether there is another process of the same |
| 63 |
# application using the CRT S3 client and prevent spawning more than one |
| 64 |
# CRT S3 clients running on the system for that application. |
| 65 |
# |
| 66 |
# NOTE: When acquiring the CRT process lock, the lock automatically is |
| 67 |
# released when the lock object is garbage collected. So, the CRT process |
| 68 |
# lock is set as a global so that it is not unintentionally garbage |
| 69 |
# collected/released if reference of the lock is lost. |
| 70 |
global CRT_S3_PROCESS_LOCK |
| 71 |
if CRT_S3_PROCESS_LOCK is None: |
| 72 |
crt_lock = awscrt.s3.CrossProcessLock(name) |
| 73 |
try: |
| 74 |
crt_lock.acquire() |
| 75 |
except RuntimeError: |
| 76 |
# If there is another process that is holding the lock, the CRT |
| 77 |
# returns a RuntimeError. We return None here to signal that our |
| 78 |
# current process was not able to acquire the lock. |
| 79 |
return None |
| 80 |
CRT_S3_PROCESS_LOCK = crt_lock |
| 81 |
return CRT_S3_PROCESS_LOCK |
| 82 |
|
| 83 |
|
| 84 |
def create_s3_crt_client( |
| 85 |
region, |
| 86 |
crt_credentials_provider=None, |
| 87 |
num_threads=None, |
| 88 |
target_throughput=None, |
| 89 |
part_size=8 * MB, |
| 90 |
use_ssl=True, |
| 91 |
verify=None, |
| 92 |
): |
| 93 |
""" |
| 94 |
:type region: str |
| 95 |
:param region: The region used for signing |
| 96 |
|
| 97 |
:type crt_credentials_provider: |
| 98 |
Optional[awscrt.auth.AwsCredentialsProvider] |
| 99 |
:param crt_credentials_provider: CRT AWS credentials provider |
| 100 |
to use to sign requests. If not set, requests will not be signed. |
| 101 |
|
| 102 |
:type num_threads: Optional[int] |
| 103 |
:param num_threads: Number of worker threads generated. Default |
| 104 |
is the number of processors in the machine. |
| 105 |
|
| 106 |
:type target_throughput: Optional[int] |
| 107 |
:param target_throughput: Throughput target in bytes per second. |
| 108 |
By default, CRT will automatically attempt to choose a target |
| 109 |
throughput that matches the system's maximum network throughput. |
| 110 |
Currently, if CRT is unable to determine the maximum network |
| 111 |
throughput, a fallback target throughput of ``1_250_000_000`` bytes |
| 112 |
per second (which translates to 10 gigabits per second, or 1.16 |
| 113 |
gibibytes per second) is used. To set a specific target |
| 114 |
throughput, set a value for this parameter. |
| 115 |
|
| 116 |
:type part_size: Optional[int] |
| 117 |
:param part_size: Size, in Bytes, of parts that files will be downloaded |
| 118 |
or uploaded in. |
| 119 |
|
| 120 |
:type use_ssl: boolean |
| 121 |
:param use_ssl: Whether or not to use SSL. By default, SSL is used. |
| 122 |
Note that not all services support non-ssl connections. |
| 123 |
|
| 124 |
:type verify: Optional[boolean/string] |
| 125 |
:param verify: Whether or not to verify SSL certificates. |
| 126 |
By default SSL certificates are verified. You can provide the |
| 127 |
following values: |
| 128 |
|
| 129 |
* False - do not validate SSL certificates. SSL will still be |
| 130 |
used (unless use_ssl is False), but SSL certificates |
| 131 |
will not be verified. |
| 132 |
* path/to/cert/bundle.pem - A filename of the CA cert bundle to |
| 133 |
use. Specify this argument if you want to use a custom CA cert |
| 134 |
bundle instead of the default one on your system. |
| 135 |
""" |
| 136 |
event_loop_group = EventLoopGroup(num_threads) |
| 137 |
host_resolver = DefaultHostResolver(event_loop_group) |
| 138 |
bootstrap = ClientBootstrap(event_loop_group, host_resolver) |
| 139 |
tls_connection_options = None |
| 140 |
|
| 141 |
tls_mode = ( |
| 142 |
S3RequestTlsMode.ENABLED if use_ssl else S3RequestTlsMode.DISABLED |
| 143 |
) |
| 144 |
if verify is not None: |
| 145 |
tls_ctx_options = TlsContextOptions() |
| 146 |
if verify: |
| 147 |
tls_ctx_options.override_default_trust_store_from_path( |
| 148 |
ca_filepath=verify |
| 149 |
) |
| 150 |
else: |
| 151 |
tls_ctx_options.verify_peer = False |
| 152 |
client_tls_option = ClientTlsContext(tls_ctx_options) |
| 153 |
tls_connection_options = client_tls_option.new_connection_options() |
| 154 |
target_gbps = _get_crt_throughput_target_gbps( |
| 155 |
provided_throughput_target_bytes=target_throughput |
| 156 |
) |
| 157 |
return S3Client( |
| 158 |
bootstrap=bootstrap, |
| 159 |
region=region, |
| 160 |
credential_provider=crt_credentials_provider, |
| 161 |
part_size=part_size, |
| 162 |
tls_mode=tls_mode, |
| 163 |
tls_connection_options=tls_connection_options, |
| 164 |
throughput_target_gbps=target_gbps, |
| 165 |
enable_s3express=True, |
| 166 |
) |
| 167 |
|
| 168 |
|
| 169 |
def _get_crt_throughput_target_gbps(provided_throughput_target_bytes=None): |
| 170 |
if provided_throughput_target_bytes is None: |
| 171 |
target_gbps = awscrt.s3.get_recommended_throughput_target_gbps() |
| 172 |
logger.debug( |
| 173 |
'Recommended CRT throughput target in gbps: %s', target_gbps |
| 174 |
) |
| 175 |
if target_gbps is None: |
| 176 |
target_gbps = 10.0 |
| 177 |
else: |
| 178 |
# NOTE: The GB constant in s3transfer is technically a gibibyte. The |
| 179 |
# GB constant is not used here because the CRT interprets gigabits |
| 180 |
# for networking as a base power of 10 |
| 181 |
# (i.e. 1000 ** 3 instead of 1024 ** 3). |
| 182 |
target_gbps = provided_throughput_target_bytes * 8 / 1_000_000_000 |
| 183 |
logger.debug('Using CRT throughput target in gbps: %s', target_gbps) |
| 184 |
return target_gbps |
| 185 |
|
| 186 |
|
| 187 |
class CRTTransferManager: |
| 188 |
ALLOWED_DOWNLOAD_ARGS = TransferManager.ALLOWED_DOWNLOAD_ARGS |
| 189 |
ALLOWED_UPLOAD_ARGS = TransferManager.ALLOWED_UPLOAD_ARGS |
| 190 |
ALLOWED_DELETE_ARGS = TransferManager.ALLOWED_DELETE_ARGS |
| 191 |
|
| 192 |
VALIDATE_SUPPORTED_BUCKET_VALUES = True |
| 193 |
|
| 194 |
_UNSUPPORTED_BUCKET_PATTERNS = TransferManager._UNSUPPORTED_BUCKET_PATTERNS |
| 195 |
|
| 196 |
def __init__(self, crt_s3_client, crt_request_serializer, osutil=None): |
| 197 |
"""A transfer manager interface for Amazon S3 on CRT s3 client. |
| 198 |
|
| 199 |
:type crt_s3_client: awscrt.s3.S3Client |
| 200 |
:param crt_s3_client: The CRT s3 client, handling all the |
| 201 |
HTTP requests and functions under then hood |
| 202 |
|
| 203 |
:type crt_request_serializer: s3transfer.crt.BaseCRTRequestSerializer |
| 204 |
:param crt_request_serializer: Serializer, generates unsigned crt HTTP |
| 205 |
request. |
| 206 |
|
| 207 |
:type osutil: s3transfer.utils.OSUtils |
| 208 |
:param osutil: OSUtils object to use for os-related behavior when |
| 209 |
using with transfer manager. |
| 210 |
""" |
| 211 |
if osutil is None: |
| 212 |
self._osutil = OSUtils() |
| 213 |
self._crt_s3_client = crt_s3_client |
| 214 |
self._s3_args_creator = S3ClientArgsCreator( |
| 215 |
crt_request_serializer, self._osutil |
| 216 |
) |
| 217 |
self._crt_exception_translator = ( |
| 218 |
crt_request_serializer.translate_crt_exception |
| 219 |
) |
| 220 |
self._future_coordinators = [] |
| 221 |
self._semaphore = threading.Semaphore(128) # not configurable |
| 222 |
# A counter to create unique id's for each transfer submitted. |
| 223 |
self._id_counter = 0 |
| 224 |
|
| 225 |
def __enter__(self): |
| 226 |
return self |
| 227 |
|
| 228 |
def __exit__(self, exc_type, exc_value, *args): |
| 229 |
cancel = False |
| 230 |
if exc_type: |
| 231 |
cancel = True |
| 232 |
self._shutdown(cancel) |
| 233 |
|
| 234 |
def download( |
| 235 |
self, bucket, key, fileobj, extra_args=None, subscribers=None |
| 236 |
): |
| 237 |
if extra_args is None: |
| 238 |
extra_args = {} |
| 239 |
if subscribers is None: |
| 240 |
subscribers = {} |
| 241 |
self._validate_all_known_args(extra_args, self.ALLOWED_DOWNLOAD_ARGS) |
| 242 |
self._validate_if_bucket_supported(bucket) |
| 243 |
callargs = CallArgs( |
| 244 |
bucket=bucket, |
| 245 |
key=key, |
| 246 |
fileobj=fileobj, |
| 247 |
extra_args=extra_args, |
| 248 |
subscribers=subscribers, |
| 249 |
) |
| 250 |
return self._submit_transfer("get_object", callargs) |
| 251 |
|
| 252 |
def upload(self, fileobj, bucket, key, extra_args=None, subscribers=None): |
| 253 |
if extra_args is None: |
| 254 |
extra_args = {} |
| 255 |
if subscribers is None: |
| 256 |
subscribers = {} |
| 257 |
self._validate_all_known_args(extra_args, self.ALLOWED_UPLOAD_ARGS) |
| 258 |
self._validate_if_bucket_supported(bucket) |
| 259 |
self._validate_checksum_algorithm_supported(extra_args) |
| 260 |
callargs = CallArgs( |
| 261 |
bucket=bucket, |
| 262 |
key=key, |
| 263 |
fileobj=fileobj, |
| 264 |
extra_args=extra_args, |
| 265 |
subscribers=subscribers, |
| 266 |
) |
| 267 |
return self._submit_transfer("put_object", callargs) |
| 268 |
|
| 269 |
def delete(self, bucket, key, extra_args=None, subscribers=None): |
| 270 |
if extra_args is None: |
| 271 |
extra_args = {} |
| 272 |
if subscribers is None: |
| 273 |
subscribers = {} |
| 274 |
self._validate_all_known_args(extra_args, self.ALLOWED_DELETE_ARGS) |
| 275 |
self._validate_if_bucket_supported(bucket) |
| 276 |
callargs = CallArgs( |
| 277 |
bucket=bucket, |
| 278 |
key=key, |
| 279 |
extra_args=extra_args, |
| 280 |
subscribers=subscribers, |
| 281 |
) |
| 282 |
return self._submit_transfer("delete_object", callargs) |
| 283 |
|
| 284 |
def shutdown(self, cancel=False): |
| 285 |
self._shutdown(cancel) |
| 286 |
|
| 287 |
def _validate_if_bucket_supported(self, bucket): |
| 288 |
# s3 high level operations don't support some resources |
| 289 |
# (eg. S3 Object Lambda) only direct API calls are available |
| 290 |
# for such resources |
| 291 |
if self.VALIDATE_SUPPORTED_BUCKET_VALUES: |
| 292 |
for resource, pattern in self._UNSUPPORTED_BUCKET_PATTERNS.items(): |
| 293 |
match = pattern.match(bucket) |
| 294 |
if match: |
| 295 |
raise ValueError( |
| 296 |
f'TransferManager methods do not support {resource} ' |
| 297 |
'resource. Use direct client calls instead.' |
| 298 |
) |
| 299 |
|
| 300 |
def _validate_all_known_args(self, actual, allowed): |
| 301 |
for kwarg in actual: |
| 302 |
if kwarg not in allowed: |
| 303 |
raise ValueError( |
| 304 |
f"Invalid extra_args key '{kwarg}', " |
| 305 |
f"must be one of: {', '.join(allowed)}" |
| 306 |
) |
| 307 |
|
| 308 |
def _validate_checksum_algorithm_supported(self, extra_args): |
| 309 |
checksum_algorithm = extra_args.get('ChecksumAlgorithm') |
| 310 |
if checksum_algorithm is None: |
| 311 |
return |
| 312 |
supported_algorithms = list(awscrt.s3.S3ChecksumAlgorithm.__members__) |
| 313 |
if checksum_algorithm.upper() not in supported_algorithms: |
| 314 |
raise ValueError( |
| 315 |
f'ChecksumAlgorithm: {checksum_algorithm} not supported. ' |
| 316 |
f'Supported algorithms are: {supported_algorithms}' |
| 317 |
) |
| 318 |
|
| 319 |
def _cancel_transfers(self): |
| 320 |
for coordinator in self._future_coordinators: |
| 321 |
if not coordinator.done(): |
| 322 |
coordinator.cancel() |
| 323 |
|
| 324 |
def _finish_transfers(self): |
| 325 |
for coordinator in self._future_coordinators: |
| 326 |
coordinator.result() |
| 327 |
|
| 328 |
def _wait_transfers_done(self): |
| 329 |
for coordinator in self._future_coordinators: |
| 330 |
coordinator.wait_until_on_done_callbacks_complete() |
| 331 |
|
| 332 |
def _shutdown(self, cancel=False): |
| 333 |
if cancel: |
| 334 |
self._cancel_transfers() |
| 335 |
try: |
| 336 |
self._finish_transfers() |
| 337 |
|
| 338 |
except KeyboardInterrupt: |
| 339 |
self._cancel_transfers() |
| 340 |
except Exception: |
| 341 |
pass |
| 342 |
finally: |
| 343 |
self._wait_transfers_done() |
| 344 |
|
| 345 |
def _release_semaphore(self, **kwargs): |
| 346 |
self._semaphore.release() |
| 347 |
|
| 348 |
def _submit_transfer(self, request_type, call_args): |
| 349 |
on_done_after_calls = [self._release_semaphore] |
| 350 |
coordinator = CRTTransferCoordinator( |
| 351 |
transfer_id=self._id_counter, |
| 352 |
exception_translator=self._crt_exception_translator, |
| 353 |
) |
| 354 |
components = { |
| 355 |
'meta': CRTTransferMeta(self._id_counter, call_args), |
| 356 |
'coordinator': coordinator, |
| 357 |
} |
| 358 |
future = CRTTransferFuture(**components) |
| 359 |
afterdone = AfterDoneHandler(coordinator) |
| 360 |
on_done_after_calls.append(afterdone) |
| 361 |
|
| 362 |
try: |
| 363 |
self._semaphore.acquire() |
| 364 |
on_queued = self._s3_args_creator.get_crt_callback( |
| 365 |
future, 'queued' |
| 366 |
) |
| 367 |
on_queued() |
| 368 |
crt_callargs = self._s3_args_creator.get_make_request_args( |
| 369 |
request_type, |
| 370 |
call_args, |
| 371 |
coordinator, |
| 372 |
future, |
| 373 |
on_done_after_calls, |
| 374 |
) |
| 375 |
crt_s3_request = self._crt_s3_client.make_request(**crt_callargs) |
| 376 |
except Exception as e: |
| 377 |
coordinator.set_exception(e, True) |
| 378 |
on_done = self._s3_args_creator.get_crt_callback( |
| 379 |
future, 'done', after_subscribers=on_done_after_calls |
| 380 |
) |
| 381 |
on_done(error=e) |
| 382 |
else: |
| 383 |
coordinator.set_s3_request(crt_s3_request) |
| 384 |
self._future_coordinators.append(coordinator) |
| 385 |
|
| 386 |
self._id_counter += 1 |
| 387 |
return future |
| 388 |
|
| 389 |
|
| 390 |
class CRTTransferMeta(BaseTransferMeta): |
| 391 |
"""Holds metadata about the CRTTransferFuture""" |
| 392 |
|
| 393 |
def __init__(self, transfer_id=None, call_args=None): |
| 394 |
self._transfer_id = transfer_id |
| 395 |
self._call_args = call_args |
| 396 |
self._user_context = {} |
| 397 |
|
| 398 |
@property |
| 399 |
def call_args(self): |
| 400 |
return self._call_args |
| 401 |
|
| 402 |
@property |
| 403 |
def transfer_id(self): |
| 404 |
return self._transfer_id |
| 405 |
|
| 406 |
@property |
| 407 |
def user_context(self): |
| 408 |
return self._user_context |
| 409 |
|
| 410 |
|
| 411 |
class CRTTransferFuture(BaseTransferFuture): |
| 412 |
def __init__(self, meta=None, coordinator=None): |
| 413 |
"""The future associated to a submitted transfer request via CRT S3 client |
| 414 |
|
| 415 |
:type meta: s3transfer.crt.CRTTransferMeta |
| 416 |
:param meta: The metadata associated to the transfer future. |
| 417 |
|
| 418 |
:type coordinator: s3transfer.crt.CRTTransferCoordinator |
| 419 |
:param coordinator: The coordinator associated to the transfer future. |
| 420 |
""" |
| 421 |
self._meta = meta |
| 422 |
if meta is None: |
| 423 |
self._meta = CRTTransferMeta() |
| 424 |
self._coordinator = coordinator |
| 425 |
|
| 426 |
@property |
| 427 |
def meta(self): |
| 428 |
return self._meta |
| 429 |
|
| 430 |
def done(self): |
| 431 |
return self._coordinator.done() |
| 432 |
|
| 433 |
def result(self, timeout=None): |
| 434 |
self._coordinator.result(timeout) |
| 435 |
|
| 436 |
def cancel(self): |
| 437 |
self._coordinator.cancel() |
| 438 |
|
| 439 |
def set_exception(self, exception): |
| 440 |
"""Sets the exception on the future.""" |
| 441 |
if not self.done(): |
| 442 |
raise TransferNotDoneError( |
| 443 |
'set_exception can only be called once the transfer is ' |
| 444 |
'complete.' |
| 445 |
) |
| 446 |
self._coordinator.set_exception(exception, override=True) |
| 447 |
|
| 448 |
|
| 449 |
class BaseCRTRequestSerializer: |
| 450 |
def serialize_http_request(self, transfer_type, future): |
| 451 |
"""Serialize CRT HTTP requests. |
| 452 |
|
| 453 |
:type transfer_type: string |
| 454 |
:param transfer_type: the type of transfer made, |
| 455 |
e.g 'put_object', 'get_object', 'delete_object' |
| 456 |
|
| 457 |
:type future: s3transfer.crt.CRTTransferFuture |
| 458 |
|
| 459 |
:rtype: awscrt.http.HttpRequest |
| 460 |
:returns: An unsigned HTTP request to be used for the CRT S3 client |
| 461 |
""" |
| 462 |
raise NotImplementedError('serialize_http_request()') |
| 463 |
|
| 464 |
def translate_crt_exception(self, exception): |
| 465 |
raise NotImplementedError('translate_crt_exception()') |
| 466 |
|
| 467 |
|
| 468 |
class BotocoreCRTRequestSerializer(BaseCRTRequestSerializer): |
| 469 |
def __init__(self, session, client_kwargs=None): |
| 470 |
"""Serialize CRT HTTP request using botocore logic |
| 471 |
It also takes into account configuration from both the session |
| 472 |
and any keyword arguments that could be passed to |
| 473 |
`Session.create_client()` when serializing the request. |
| 474 |
|
| 475 |
:type session: botocore.session.Session |
| 476 |
|
| 477 |
:type client_kwargs: Optional[Dict[str, str]]) |
| 478 |
:param client_kwargs: The kwargs for the botocore |
| 479 |
s3 client initialization. |
| 480 |
""" |
| 481 |
self._session = session |
| 482 |
if client_kwargs is None: |
| 483 |
client_kwargs = {} |
| 484 |
self._resolve_client_config(session, client_kwargs) |
| 485 |
self._client = create_nested_client(session, **client_kwargs) |
| 486 |
self._client.meta.events.register( |
| 487 |
'request-created.s3.*', self._capture_http_request |
| 488 |
) |
| 489 |
self._client.meta.events.register( |
| 490 |
'after-call.s3.*', self._change_response_to_serialized_http_request |
| 491 |
) |
| 492 |
self._client.meta.events.register( |
| 493 |
'before-send.s3.*', self._make_fake_http_response |
| 494 |
) |
| 495 |
self._client.meta.events.register( |
| 496 |
'before-call.s3.*', self._remove_checksum_context |
| 497 |
) |
| 498 |
|
| 499 |
def _resolve_client_config(self, session, client_kwargs): |
| 500 |
user_provided_config = None |
| 501 |
if session.get_default_client_config(): |
| 502 |
user_provided_config = session.get_default_client_config() |
| 503 |
if 'config' in client_kwargs: |
| 504 |
user_provided_config = client_kwargs['config'] |
| 505 |
|
| 506 |
client_config = Config(signature_version=UNSIGNED) |
| 507 |
if user_provided_config: |
| 508 |
client_config = user_provided_config.merge(client_config) |
| 509 |
client_kwargs['config'] = client_config |
| 510 |
client_kwargs["service_name"] = "s3" |
| 511 |
|
| 512 |
def _crt_request_from_aws_request(self, aws_request): |
| 513 |
url_parts = urlsplit(aws_request.url) |
| 514 |
crt_path = url_parts.path |
| 515 |
if url_parts.query: |
| 516 |
crt_path = f'{crt_path}?{url_parts.query}' |
| 517 |
headers_list = [] |
| 518 |
for name, value in aws_request.headers.items(): |
| 519 |
if isinstance(value, str): |
| 520 |
headers_list.append((name, value)) |
| 521 |
else: |
| 522 |
headers_list.append((name, str(value, 'utf-8'))) |
| 523 |
|
| 524 |
crt_headers = awscrt.http.HttpHeaders(headers_list) |
| 525 |
|
| 526 |
crt_request = awscrt.http.HttpRequest( |
| 527 |
method=aws_request.method, |
| 528 |
path=crt_path, |
| 529 |
headers=crt_headers, |
| 530 |
body_stream=aws_request.body, |
| 531 |
) |
| 532 |
return crt_request |
| 533 |
|
| 534 |
def _convert_to_crt_http_request(self, botocore_http_request): |
| 535 |
# Logic that does CRTUtils.crt_request_from_aws_request |
| 536 |
crt_request = self._crt_request_from_aws_request(botocore_http_request) |
| 537 |
if crt_request.headers.get("host") is None: |
| 538 |
# If host is not set, set it for the request before using CRT s3 |
| 539 |
url_parts = urlsplit(botocore_http_request.url) |
| 540 |
crt_request.headers.set("host", url_parts.netloc) |
| 541 |
if crt_request.headers.get('Content-MD5') is not None: |
| 542 |
crt_request.headers.remove("Content-MD5") |
| 543 |
|
| 544 |
# In general, the CRT S3 client expects a content length header. It |
| 545 |
# only expects a missing content length header if the body is not |
| 546 |
# seekable. However, botocore does not set the content length header |
| 547 |
# for GetObject API requests and so we set the content length to zero |
| 548 |
# to meet the CRT S3 client's expectation that the content length |
| 549 |
# header is set even if there is no body. |
| 550 |
if crt_request.headers.get('Content-Length') is None: |
| 551 |
if botocore_http_request.body is None: |
| 552 |
crt_request.headers.add('Content-Length', "0") |
| 553 |
|
| 554 |
# Botocore sets the Transfer-Encoding header when it cannot determine |
| 555 |
# the content length of the request body (e.g. it's not seekable). |
| 556 |
# However, CRT does not support this header, but it supports |
| 557 |
# non-seekable bodies. So we remove this header to not cause issues |
| 558 |
# in the downstream CRT S3 request. |
| 559 |
if crt_request.headers.get('Transfer-Encoding') is not None: |
| 560 |
crt_request.headers.remove('Transfer-Encoding') |
| 561 |
|
| 562 |
return crt_request |
| 563 |
|
| 564 |
def _capture_http_request(self, request, **kwargs): |
| 565 |
request.context['http_request'] = request |
| 566 |
|
| 567 |
def _change_response_to_serialized_http_request( |
| 568 |
self, context, parsed, **kwargs |
| 569 |
): |
| 570 |
request = context['http_request'] |
| 571 |
parsed['HTTPRequest'] = request.prepare() |
| 572 |
|
| 573 |
def _make_fake_http_response(self, request, **kwargs): |
| 574 |
return botocore.awsrequest.AWSResponse( |
| 575 |
None, |
| 576 |
200, |
| 577 |
{}, |
| 578 |
FakeRawResponse(b""), |
| 579 |
) |
| 580 |
|
| 581 |
def _get_botocore_http_request(self, client_method, call_args): |
| 582 |
return getattr(self._client, client_method)( |
| 583 |
Bucket=call_args.bucket, Key=call_args.key, **call_args.extra_args |
| 584 |
)['HTTPRequest'] |
| 585 |
|
| 586 |
def serialize_http_request(self, transfer_type, future): |
| 587 |
botocore_http_request = self._get_botocore_http_request( |
| 588 |
transfer_type, future.meta.call_args |
| 589 |
) |
| 590 |
crt_request = self._convert_to_crt_http_request(botocore_http_request) |
| 591 |
return crt_request |
| 592 |
|
| 593 |
def translate_crt_exception(self, exception): |
| 594 |
if isinstance(exception, awscrt.s3.S3ResponseError): |
| 595 |
return self._translate_crt_s3_response_error(exception) |
| 596 |
else: |
| 597 |
return None |
| 598 |
|
| 599 |
def _translate_crt_s3_response_error(self, s3_response_error): |
| 600 |
status_code = s3_response_error.status_code |
| 601 |
if status_code < 301: |
| 602 |
# Botocore's exception parsing only |
| 603 |
# runs on status codes >= 301 |
| 604 |
return None |
| 605 |
|
| 606 |
headers = {k: v for k, v in s3_response_error.headers} |
| 607 |
operation_name = s3_response_error.operation_name |
| 608 |
if operation_name is not None: |
| 609 |
service_model = self._client.meta.service_model |
| 610 |
shape = service_model.operation_model(operation_name).output_shape |
| 611 |
else: |
| 612 |
shape = None |
| 613 |
|
| 614 |
response_dict = { |
| 615 |
'headers': botocore.awsrequest.HeadersDict(headers), |
| 616 |
'status_code': status_code, |
| 617 |
'body': s3_response_error.body, |
| 618 |
} |
| 619 |
parsed_response = self._client._response_parser.parse( |
| 620 |
response_dict, shape=shape |
| 621 |
) |
| 622 |
|
| 623 |
error_code = parsed_response.get("Error", {}).get("Code") |
| 624 |
error_class = self._client.exceptions.from_code(error_code) |
| 625 |
return error_class(parsed_response, operation_name=operation_name) |
| 626 |
|
| 627 |
def _remove_checksum_context(self, params, **kwargs): |
| 628 |
request_context = params.get("context", {}) |
| 629 |
if "checksum" in request_context: |
| 630 |
del request_context["checksum"] |
| 631 |
|
| 632 |
|
| 633 |
class FakeRawResponse(BytesIO): |
| 634 |
def stream(self, amt=1024, decode_content=None): |
| 635 |
while True: |
| 636 |
chunk = self.read(amt) |
| 637 |
if not chunk: |
| 638 |
break |
| 639 |
yield chunk |
| 640 |
|
| 641 |
|
| 642 |
class BotocoreCRTCredentialsWrapper: |
| 643 |
def __init__(self, resolved_botocore_credentials): |
| 644 |
self._resolved_credentials = resolved_botocore_credentials |
| 645 |
|
| 646 |
def __call__(self): |
| 647 |
credentials = self._get_credentials().get_frozen_credentials() |
| 648 |
return AwsCredentials( |
| 649 |
credentials.access_key, credentials.secret_key, credentials.token |
| 650 |
) |
| 651 |
|
| 652 |
def to_crt_credentials_provider(self): |
| 653 |
return AwsCredentialsProvider.new_delegate(self) |
| 654 |
|
| 655 |
def _get_credentials(self): |
| 656 |
if self._resolved_credentials is None: |
| 657 |
raise NoCredentialsError() |
| 658 |
return self._resolved_credentials |
| 659 |
|
| 660 |
|
| 661 |
class CRTTransferCoordinator: |
| 662 |
"""A helper class for managing CRTTransferFuture""" |
| 663 |
|
| 664 |
def __init__( |
| 665 |
self, transfer_id=None, s3_request=None, exception_translator=None |
| 666 |
): |
| 667 |
self.transfer_id = transfer_id |
| 668 |
self._exception_translator = exception_translator |
| 669 |
self._s3_request = s3_request |
| 670 |
self._lock = threading.Lock() |
| 671 |
self._exception = None |
| 672 |
self._crt_future = None |
| 673 |
self._done_event = threading.Event() |
| 674 |
|
| 675 |
@property |
| 676 |
def s3_request(self): |
| 677 |
return self._s3_request |
| 678 |
|
| 679 |
def set_done_callbacks_complete(self): |
| 680 |
self._done_event.set() |
| 681 |
|
| 682 |
def wait_until_on_done_callbacks_complete(self, timeout=None): |
| 683 |
self._done_event.wait(timeout) |
| 684 |
|
| 685 |
def set_exception(self, exception, override=False): |
| 686 |
with self._lock: |
| 687 |
if not self.done() or override: |
| 688 |
self._exception = exception |
| 689 |
|
| 690 |
def cancel(self): |
| 691 |
if self._s3_request: |
| 692 |
self._s3_request.cancel() |
| 693 |
|
| 694 |
def result(self, timeout=None): |
| 695 |
if self._exception: |
| 696 |
raise self._exception |
| 697 |
try: |
| 698 |
self._crt_future.result(timeout) |
| 699 |
except KeyboardInterrupt: |
| 700 |
self.cancel() |
| 701 |
self._crt_future.result(timeout) |
| 702 |
raise |
| 703 |
except Exception as e: |
| 704 |
self.handle_exception(e) |
| 705 |
finally: |
| 706 |
if self._s3_request: |
| 707 |
self._s3_request = None |
| 708 |
|
| 709 |
def handle_exception(self, exc): |
| 710 |
translated_exc = None |
| 711 |
if self._exception_translator: |
| 712 |
try: |
| 713 |
translated_exc = self._exception_translator(exc) |
| 714 |
except Exception as e: |
| 715 |
# Bail out if we hit an issue translating |
| 716 |
# and raise the original error. |
| 717 |
logger.debug("Unable to translate exception.", exc_info=e) |
| 718 |
pass |
| 719 |
if translated_exc is not None: |
| 720 |
raise translated_exc from exc |
| 721 |
else: |
| 722 |
raise exc |
| 723 |
|
| 724 |
def done(self): |
| 725 |
if self._crt_future is None: |
| 726 |
return False |
| 727 |
return self._crt_future.done() |
| 728 |
|
| 729 |
def set_s3_request(self, s3_request): |
| 730 |
self._s3_request = s3_request |
| 731 |
self._crt_future = self._s3_request.finished_future |
| 732 |
|
| 733 |
|
| 734 |
class S3ClientArgsCreator: |
| 735 |
def __init__(self, crt_request_serializer, os_utils): |
| 736 |
self._request_serializer = crt_request_serializer |
| 737 |
self._os_utils = os_utils |
| 738 |
|
| 739 |
def get_make_request_args( |
| 740 |
self, request_type, call_args, coordinator, future, on_done_after_calls |
| 741 |
): |
| 742 |
request_args_handler = getattr( |
| 743 |
self, |
| 744 |
f'_get_make_request_args_{request_type}', |
| 745 |
self._default_get_make_request_args, |
| 746 |
) |
| 747 |
return request_args_handler( |
| 748 |
request_type=request_type, |
| 749 |
call_args=call_args, |
| 750 |
coordinator=coordinator, |
| 751 |
future=future, |
| 752 |
on_done_before_calls=[], |
| 753 |
on_done_after_calls=on_done_after_calls, |
| 754 |
) |
| 755 |
|
| 756 |
def get_crt_callback( |
| 757 |
self, |
| 758 |
future, |
| 759 |
callback_type, |
| 760 |
before_subscribers=None, |
| 761 |
after_subscribers=None, |
| 762 |
): |
| 763 |
def invoke_all_callbacks(*args, **kwargs): |
| 764 |
callbacks_list = [] |
| 765 |
if before_subscribers is not None: |
| 766 |
callbacks_list += before_subscribers |
| 767 |
callbacks_list += get_callbacks(future, callback_type) |
| 768 |
if after_subscribers is not None: |
| 769 |
callbacks_list += after_subscribers |
| 770 |
for callback in callbacks_list: |
| 771 |
# The get_callbacks helper will set the first augment |
| 772 |
# by keyword, the other augments need to be set by keyword |
| 773 |
# as well |
| 774 |
if callback_type == "progress": |
| 775 |
callback(bytes_transferred=args[0]) |
| 776 |
else: |
| 777 |
callback(*args, **kwargs) |
| 778 |
|
| 779 |
return invoke_all_callbacks |
| 780 |
|
| 781 |
def _get_make_request_args_put_object( |
| 782 |
self, |
| 783 |
request_type, |
| 784 |
call_args, |
| 785 |
coordinator, |
| 786 |
future, |
| 787 |
on_done_before_calls, |
| 788 |
on_done_after_calls, |
| 789 |
): |
| 790 |
send_filepath = None |
| 791 |
if isinstance(call_args.fileobj, str): |
| 792 |
send_filepath = call_args.fileobj |
| 793 |
data_len = self._os_utils.get_file_size(send_filepath) |
| 794 |
call_args.extra_args["ContentLength"] = data_len |
| 795 |
else: |
| 796 |
call_args.extra_args["Body"] = call_args.fileobj |
| 797 |
|
| 798 |
checksum_config = None |
| 799 |
if not any( |
| 800 |
checksum_arg in call_args.extra_args |
| 801 |
for checksum_arg in FULL_OBJECT_CHECKSUM_ARGS |
| 802 |
): |
| 803 |
checksum_algorithm = call_args.extra_args.pop( |
| 804 |
'ChecksumAlgorithm', 'CRC32' |
| 805 |
).upper() |
| 806 |
checksum_config = awscrt.s3.S3ChecksumConfig( |
| 807 |
algorithm=awscrt.s3.S3ChecksumAlgorithm[checksum_algorithm], |
| 808 |
location=awscrt.s3.S3ChecksumLocation.TRAILER, |
| 809 |
) |
| 810 |
# Suppress botocore's automatic MD5 calculation by setting an override |
| 811 |
# value that will get deleted in the BotocoreCRTRequestSerializer. |
| 812 |
# As part of the CRT S3 request, we request the CRT S3 client to |
| 813 |
# automatically add trailing checksums to its uploads. |
| 814 |
call_args.extra_args["ContentMD5"] = "override-to-be-removed" |
| 815 |
|
| 816 |
make_request_args = self._default_get_make_request_args( |
| 817 |
request_type=request_type, |
| 818 |
call_args=call_args, |
| 819 |
coordinator=coordinator, |
| 820 |
future=future, |
| 821 |
on_done_before_calls=on_done_before_calls, |
| 822 |
on_done_after_calls=on_done_after_calls, |
| 823 |
) |
| 824 |
make_request_args['send_filepath'] = send_filepath |
| 825 |
make_request_args['checksum_config'] = checksum_config |
| 826 |
return make_request_args |
| 827 |
|
| 828 |
def _get_make_request_args_get_object( |
| 829 |
self, |
| 830 |
request_type, |
| 831 |
call_args, |
| 832 |
coordinator, |
| 833 |
future, |
| 834 |
on_done_before_calls, |
| 835 |
on_done_after_calls, |
| 836 |
): |
| 837 |
recv_filepath = None |
| 838 |
on_body = None |
| 839 |
checksum_config = awscrt.s3.S3ChecksumConfig(validate_response=True) |
| 840 |
if isinstance(call_args.fileobj, str): |
| 841 |
final_filepath = call_args.fileobj |
| 842 |
recv_filepath = self._os_utils.get_temp_filename(final_filepath) |
| 843 |
on_done_before_calls.append( |
| 844 |
RenameTempFileHandler( |
| 845 |
coordinator, final_filepath, recv_filepath, self._os_utils |
| 846 |
) |
| 847 |
) |
| 848 |
else: |
| 849 |
on_body = OnBodyFileObjWriter(call_args.fileobj) |
| 850 |
|
| 851 |
make_request_args = self._default_get_make_request_args( |
| 852 |
request_type=request_type, |
| 853 |
call_args=call_args, |
| 854 |
coordinator=coordinator, |
| 855 |
future=future, |
| 856 |
on_done_before_calls=on_done_before_calls, |
| 857 |
on_done_after_calls=on_done_after_calls, |
| 858 |
) |
| 859 |
make_request_args['recv_filepath'] = recv_filepath |
| 860 |
make_request_args['on_body'] = on_body |
| 861 |
make_request_args['checksum_config'] = checksum_config |
| 862 |
return make_request_args |
| 863 |
|
| 864 |
def _default_get_make_request_args( |
| 865 |
self, |
| 866 |
request_type, |
| 867 |
call_args, |
| 868 |
coordinator, |
| 869 |
future, |
| 870 |
on_done_before_calls, |
| 871 |
on_done_after_calls, |
| 872 |
): |
| 873 |
make_request_args = { |
| 874 |
'request': self._request_serializer.serialize_http_request( |
| 875 |
request_type, future |
| 876 |
), |
| 877 |
'type': getattr( |
| 878 |
S3RequestType, request_type.upper(), S3RequestType.DEFAULT |
| 879 |
), |
| 880 |
'on_done': self.get_crt_callback( |
| 881 |
future, 'done', on_done_before_calls, on_done_after_calls |
| 882 |
), |
| 883 |
'on_progress': self.get_crt_callback(future, 'progress'), |
| 884 |
} |
| 885 |
|
| 886 |
# For DEFAULT requests, CRT requires the official S3 operation name. |
| 887 |
# So transform string like "delete_object" -> "DeleteObject". |
| 888 |
if make_request_args['type'] == S3RequestType.DEFAULT: |
| 889 |
make_request_args['operation_name'] = ''.join( |
| 890 |
x.title() for x in request_type.split('_') |
| 891 |
) |
| 892 |
|
| 893 |
arn_handler = _S3ArnParamHandler() |
| 894 |
if ( |
| 895 |
accesspoint_arn_details := arn_handler.handle_arn(call_args.bucket) |
| 896 |
) and accesspoint_arn_details['region'] == "": |
| 897 |
# Configure our region to `*` to propogate in `x-amz-region-set` |
| 898 |
# for multi-region support in MRAP accesspoints. |
| 899 |
# use_double_uri_encode and should_normalize_uri_path are defaulted to be True |
| 900 |
# But SDK already encoded the URI, and it's for S3, so set both to False |
| 901 |
make_request_args['signing_config'] = AwsSigningConfig( |
| 902 |
algorithm=AwsSigningAlgorithm.V4_ASYMMETRIC, |
| 903 |
region="*", |
| 904 |
use_double_uri_encode=False, |
| 905 |
should_normalize_uri_path=False, |
| 906 |
) |
| 907 |
call_args.bucket = accesspoint_arn_details['resource_name'] |
| 908 |
elif is_s3express_bucket(call_args.bucket): |
| 909 |
# use_double_uri_encode and should_normalize_uri_path are defaulted to be True |
| 910 |
# But SDK already encoded the URI, and it's for S3, so set both to False |
| 911 |
make_request_args['signing_config'] = AwsSigningConfig( |
| 912 |
algorithm=AwsSigningAlgorithm.V4_S3EXPRESS, |
| 913 |
use_double_uri_encode=False, |
| 914 |
should_normalize_uri_path=False, |
| 915 |
) |
| 916 |
return make_request_args |
| 917 |
|
| 918 |
|
| 919 |
class RenameTempFileHandler: |
| 920 |
def __init__(self, coordinator, final_filename, temp_filename, osutil): |
| 921 |
self._coordinator = coordinator |
| 922 |
self._final_filename = final_filename |
| 923 |
self._temp_filename = temp_filename |
| 924 |
self._osutil = osutil |
| 925 |
|
| 926 |
def __call__(self, **kwargs): |
| 927 |
error = kwargs['error'] |
| 928 |
if error: |
| 929 |
self._osutil.remove_file(self._temp_filename) |
| 930 |
else: |
| 931 |
try: |
| 932 |
self._osutil.rename_file( |
| 933 |
self._temp_filename, self._final_filename |
| 934 |
) |
| 935 |
except Exception as e: |
| 936 |
self._osutil.remove_file(self._temp_filename) |
| 937 |
# the CRT future has done already at this point |
| 938 |
self._coordinator.set_exception(e) |
| 939 |
|
| 940 |
|
| 941 |
class AfterDoneHandler: |
| 942 |
def __init__(self, coordinator): |
| 943 |
self._coordinator = coordinator |
| 944 |
|
| 945 |
def __call__(self, **kwargs): |
| 946 |
self._coordinator.set_done_callbacks_complete() |
| 947 |
|
| 948 |
|
| 949 |
class OnBodyFileObjWriter: |
| 950 |
def __init__(self, fileobj): |
| 951 |
self._fileobj = fileobj |
| 952 |
|
| 953 |
def __call__(self, chunk, **kwargs): |
| 954 |
self._fileobj.write(chunk) |
| 955 |
|
| 956 |
|
| 957 |
class _S3ArnParamHandler: |
| 958 |
"""Partial port of S3ArnParamHandler from botocore. |
| 959 |
|
| 960 |
This is used to make a determination on MRAP accesspoints for signing |
| 961 |
purposes. This should be safe to remove once we properly integrate auth |
| 962 |
resolution from Botocore into the CRT transfer integration. |
| 963 |
""" |
| 964 |
|
| 965 |
_RESOURCE_REGEX = re.compile( |
| 966 |
r'^(?P<resource_type>accesspoint|outpost)[/:](?P<resource_name>.+)$' |
| 967 |
) |
| 968 |
|
| 969 |
def __init__(self): |
| 970 |
self._arn_parser = ArnParser() |
| 971 |
|
| 972 |
def handle_arn(self, bucket): |
| 973 |
arn_details = self._get_arn_details_from_bucket(bucket) |
| 974 |
if arn_details is None: |
| 975 |
return |
| 976 |
if arn_details['resource_type'] == 'accesspoint': |
| 977 |
return arn_details |
| 978 |
|
| 979 |
def _get_arn_details_from_bucket(self, bucket): |
| 980 |
try: |
| 981 |
arn_details = self._arn_parser.parse_arn(bucket) |
| 982 |
self._add_resource_type_and_name(arn_details) |
| 983 |
return arn_details |
| 984 |
except InvalidArnException: |
| 985 |
pass |
| 986 |
return None |
| 987 |
|
| 988 |
def _add_resource_type_and_name(self, arn_details): |
| 989 |
match = self._RESOURCE_REGEX.match(arn_details['resource']) |
| 990 |
if match: |
| 991 |
arn_details['resource_type'] = match.group('resource_type') |
| 992 |
arn_details['resource_name'] = match.group('resource_name') |