Debian Codegraph @

s3transfer/crt.py

raw
1 # Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License"). You
4 # may not use this file except in compliance with the License. A copy of
5 # the License is located at
6 #
7 # http://aws.amazon.com/apache2.0/
8 #
9 # or in the "license" file accompanying this file. This file is
10 # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
11 # ANY KIND, either express or implied. See the License for the specific
12 # language governing permissions and limitations under the License.
13 import logging
14 import re
15 import threading
16 from io import BytesIO
17
18 import awscrt.http
19 import awscrt.s3
20 import botocore.awsrequest
21 import botocore.session
22 from awscrt.auth import (
23 AwsCredentials,
24 AwsCredentialsProvider,
25 AwsSigningAlgorithm,
26 AwsSigningConfig,
27 )
28 from awscrt.io import (
29 ClientBootstrap,
30 ClientTlsContext,
31 DefaultHostResolver,
32 EventLoopGroup,
33 TlsContextOptions,
34 )
35 from awscrt.s3 import S3Client, S3RequestTlsMode, S3RequestType
36 from botocore import UNSIGNED
37 from botocore.compat import urlsplit
38 from botocore.config import Config
39 from botocore.exceptions import NoCredentialsError
40 from botocore.utils import ArnParser, InvalidArnException
41
42 from s3transfer.constants import FULL_OBJECT_CHECKSUM_ARGS, MB
43 from s3transfer.exceptions import TransferNotDoneError
44 from s3transfer.futures import BaseTransferFuture, BaseTransferMeta
45 from s3transfer.manager import TransferManager
46 from s3transfer.utils import (
47 CallArgs,
48 OSUtils,
49 create_nested_client,
50 get_callbacks,
51 is_s3express_bucket,
52 )
53
54 logger = logging.getLogger(__name__)
55
56 CRT_S3_PROCESS_LOCK = None
57
58
59 def acquire_crt_s3_process_lock(name):
60 # Currently, the CRT S3 client performs best when there is only one
61 # instance of it running on a host. This lock allows an application to
62 # signal across processes whether there is another process of the same
63 # application using the CRT S3 client and prevent spawning more than one
64 # CRT S3 clients running on the system for that application.
65 #
66 # NOTE: When acquiring the CRT process lock, the lock automatically is
67 # released when the lock object is garbage collected. So, the CRT process
68 # lock is set as a global so that it is not unintentionally garbage
69 # collected/released if reference of the lock is lost.
70 global CRT_S3_PROCESS_LOCK
71 if CRT_S3_PROCESS_LOCK is None:
72 crt_lock = awscrt.s3.CrossProcessLock(name)
73 try:
74 crt_lock.acquire()
75 except RuntimeError:
76 # If there is another process that is holding the lock, the CRT
77 # returns a RuntimeError. We return None here to signal that our
78 # current process was not able to acquire the lock.
79 return None
80 CRT_S3_PROCESS_LOCK = crt_lock
81 return CRT_S3_PROCESS_LOCK
82
83
84 def create_s3_crt_client(
85 region,
86 crt_credentials_provider=None,
87 num_threads=None,
88 target_throughput=None,
89 part_size=8 * MB,
90 use_ssl=True,
91 verify=None,
92 ):
93 """
94 :type region: str
95 :param region: The region used for signing
96
97 :type crt_credentials_provider:
98 Optional[awscrt.auth.AwsCredentialsProvider]
99 :param crt_credentials_provider: CRT AWS credentials provider
100 to use to sign requests. If not set, requests will not be signed.
101
102 :type num_threads: Optional[int]
103 :param num_threads: Number of worker threads generated. Default
104 is the number of processors in the machine.
105
106 :type target_throughput: Optional[int]
107 :param target_throughput: Throughput target in bytes per second.
108 By default, CRT will automatically attempt to choose a target
109 throughput that matches the system's maximum network throughput.
110 Currently, if CRT is unable to determine the maximum network
111 throughput, a fallback target throughput of ``1_250_000_000`` bytes
112 per second (which translates to 10 gigabits per second, or 1.16
113 gibibytes per second) is used. To set a specific target
114 throughput, set a value for this parameter.
115
116 :type part_size: Optional[int]
117 :param part_size: Size, in Bytes, of parts that files will be downloaded
118 or uploaded in.
119
120 :type use_ssl: boolean
121 :param use_ssl: Whether or not to use SSL. By default, SSL is used.
122 Note that not all services support non-ssl connections.
123
124 :type verify: Optional[boolean/string]
125 :param verify: Whether or not to verify SSL certificates.
126 By default SSL certificates are verified. You can provide the
127 following values:
128
129 * False - do not validate SSL certificates. SSL will still be
130 used (unless use_ssl is False), but SSL certificates
131 will not be verified.
132 * path/to/cert/bundle.pem - A filename of the CA cert bundle to
133 use. Specify this argument if you want to use a custom CA cert
134 bundle instead of the default one on your system.
135 """
136 event_loop_group = EventLoopGroup(num_threads)
137 host_resolver = DefaultHostResolver(event_loop_group)
138 bootstrap = ClientBootstrap(event_loop_group, host_resolver)
139 tls_connection_options = None
140
141 tls_mode = (
142 S3RequestTlsMode.ENABLED if use_ssl else S3RequestTlsMode.DISABLED
143 )
144 if verify is not None:
145 tls_ctx_options = TlsContextOptions()
146 if verify:
147 tls_ctx_options.override_default_trust_store_from_path(
148 ca_filepath=verify
149 )
150 else:
151 tls_ctx_options.verify_peer = False
152 client_tls_option = ClientTlsContext(tls_ctx_options)
153 tls_connection_options = client_tls_option.new_connection_options()
154 target_gbps = _get_crt_throughput_target_gbps(
155 provided_throughput_target_bytes=target_throughput
156 )
157 return S3Client(
158 bootstrap=bootstrap,
159 region=region,
160 credential_provider=crt_credentials_provider,
161 part_size=part_size,
162 tls_mode=tls_mode,
163 tls_connection_options=tls_connection_options,
164 throughput_target_gbps=target_gbps,
165 enable_s3express=True,
166 )
167
168
169 def _get_crt_throughput_target_gbps(provided_throughput_target_bytes=None):
170 if provided_throughput_target_bytes is None:
171 target_gbps = awscrt.s3.get_recommended_throughput_target_gbps()
172 logger.debug(
173 'Recommended CRT throughput target in gbps: %s', target_gbps
174 )
175 if target_gbps is None:
176 target_gbps = 10.0
177 else:
178 # NOTE: The GB constant in s3transfer is technically a gibibyte. The
179 # GB constant is not used here because the CRT interprets gigabits
180 # for networking as a base power of 10
181 # (i.e. 1000 ** 3 instead of 1024 ** 3).
182 target_gbps = provided_throughput_target_bytes * 8 / 1_000_000_000
183 logger.debug('Using CRT throughput target in gbps: %s', target_gbps)
184 return target_gbps
185
186
187 class CRTTransferManager:
188 ALLOWED_DOWNLOAD_ARGS = TransferManager.ALLOWED_DOWNLOAD_ARGS
189 ALLOWED_UPLOAD_ARGS = TransferManager.ALLOWED_UPLOAD_ARGS
190 ALLOWED_DELETE_ARGS = TransferManager.ALLOWED_DELETE_ARGS
191
192 VALIDATE_SUPPORTED_BUCKET_VALUES = True
193
194 _UNSUPPORTED_BUCKET_PATTERNS = TransferManager._UNSUPPORTED_BUCKET_PATTERNS
195
196 def __init__(self, crt_s3_client, crt_request_serializer, osutil=None):
197 """A transfer manager interface for Amazon S3 on CRT s3 client.
198
199 :type crt_s3_client: awscrt.s3.S3Client
200 :param crt_s3_client: The CRT s3 client, handling all the
201 HTTP requests and functions under then hood
202
203 :type crt_request_serializer: s3transfer.crt.BaseCRTRequestSerializer
204 :param crt_request_serializer: Serializer, generates unsigned crt HTTP
205 request.
206
207 :type osutil: s3transfer.utils.OSUtils
208 :param osutil: OSUtils object to use for os-related behavior when
209 using with transfer manager.
210 """
211 if osutil is None:
212 self._osutil = OSUtils()
213 self._crt_s3_client = crt_s3_client
214 self._s3_args_creator = S3ClientArgsCreator(
215 crt_request_serializer, self._osutil
216 )
217 self._crt_exception_translator = (
218 crt_request_serializer.translate_crt_exception
219 )
220 self._future_coordinators = []
221 self._semaphore = threading.Semaphore(128) # not configurable
222 # A counter to create unique id's for each transfer submitted.
223 self._id_counter = 0
224
225 def __enter__(self):
226 return self
227
228 def __exit__(self, exc_type, exc_value, *args):
229 cancel = False
230 if exc_type:
231 cancel = True
232 self._shutdown(cancel)
233
234 def download(
235 self, bucket, key, fileobj, extra_args=None, subscribers=None
236 ):
237 if extra_args is None:
238 extra_args = {}
239 if subscribers is None:
240 subscribers = {}
241 self._validate_all_known_args(extra_args, self.ALLOWED_DOWNLOAD_ARGS)
242 self._validate_if_bucket_supported(bucket)
243 callargs = CallArgs(
244 bucket=bucket,
245 key=key,
246 fileobj=fileobj,
247 extra_args=extra_args,
248 subscribers=subscribers,
249 )
250 return self._submit_transfer("get_object", callargs)
251
252 def upload(self, fileobj, bucket, key, extra_args=None, subscribers=None):
253 if extra_args is None:
254 extra_args = {}
255 if subscribers is None:
256 subscribers = {}
257 self._validate_all_known_args(extra_args, self.ALLOWED_UPLOAD_ARGS)
258 self._validate_if_bucket_supported(bucket)
259 self._validate_checksum_algorithm_supported(extra_args)
260 callargs = CallArgs(
261 bucket=bucket,
262 key=key,
263 fileobj=fileobj,
264 extra_args=extra_args,
265 subscribers=subscribers,
266 )
267 return self._submit_transfer("put_object", callargs)
268
269 def delete(self, bucket, key, extra_args=None, subscribers=None):
270 if extra_args is None:
271 extra_args = {}
272 if subscribers is None:
273 subscribers = {}
274 self._validate_all_known_args(extra_args, self.ALLOWED_DELETE_ARGS)
275 self._validate_if_bucket_supported(bucket)
276 callargs = CallArgs(
277 bucket=bucket,
278 key=key,
279 extra_args=extra_args,
280 subscribers=subscribers,
281 )
282 return self._submit_transfer("delete_object", callargs)
283
284 def shutdown(self, cancel=False):
285 self._shutdown(cancel)
286
287 def _validate_if_bucket_supported(self, bucket):
288 # s3 high level operations don't support some resources
289 # (eg. S3 Object Lambda) only direct API calls are available
290 # for such resources
291 if self.VALIDATE_SUPPORTED_BUCKET_VALUES:
292 for resource, pattern in self._UNSUPPORTED_BUCKET_PATTERNS.items():
293 match = pattern.match(bucket)
294 if match:
295 raise ValueError(
296 f'TransferManager methods do not support {resource} '
297 'resource. Use direct client calls instead.'
298 )
299
300 def _validate_all_known_args(self, actual, allowed):
301 for kwarg in actual:
302 if kwarg not in allowed:
303 raise ValueError(
304 f"Invalid extra_args key '{kwarg}', "
305 f"must be one of: {', '.join(allowed)}"
306 )
307
308 def _validate_checksum_algorithm_supported(self, extra_args):
309 checksum_algorithm = extra_args.get('ChecksumAlgorithm')
310 if checksum_algorithm is None:
311 return
312 supported_algorithms = list(awscrt.s3.S3ChecksumAlgorithm.__members__)
313 if checksum_algorithm.upper() not in supported_algorithms:
314 raise ValueError(
315 f'ChecksumAlgorithm: {checksum_algorithm} not supported. '
316 f'Supported algorithms are: {supported_algorithms}'
317 )
318
319 def _cancel_transfers(self):
320 for coordinator in self._future_coordinators:
321 if not coordinator.done():
322 coordinator.cancel()
323
324 def _finish_transfers(self):
325 for coordinator in self._future_coordinators:
326 coordinator.result()
327
328 def _wait_transfers_done(self):
329 for coordinator in self._future_coordinators:
330 coordinator.wait_until_on_done_callbacks_complete()
331
332 def _shutdown(self, cancel=False):
333 if cancel:
334 self._cancel_transfers()
335 try:
336 self._finish_transfers()
337
338 except KeyboardInterrupt:
339 self._cancel_transfers()
340 except Exception:
341 pass
342 finally:
343 self._wait_transfers_done()
344
345 def _release_semaphore(self, **kwargs):
346 self._semaphore.release()
347
348 def _submit_transfer(self, request_type, call_args):
349 on_done_after_calls = [self._release_semaphore]
350 coordinator = CRTTransferCoordinator(
351 transfer_id=self._id_counter,
352 exception_translator=self._crt_exception_translator,
353 )
354 components = {
355 'meta': CRTTransferMeta(self._id_counter, call_args),
356 'coordinator': coordinator,
357 }
358 future = CRTTransferFuture(**components)
359 afterdone = AfterDoneHandler(coordinator)
360 on_done_after_calls.append(afterdone)
361
362 try:
363 self._semaphore.acquire()
364 on_queued = self._s3_args_creator.get_crt_callback(
365 future, 'queued'
366 )
367 on_queued()
368 crt_callargs = self._s3_args_creator.get_make_request_args(
369 request_type,
370 call_args,
371 coordinator,
372 future,
373 on_done_after_calls,
374 )
375 crt_s3_request = self._crt_s3_client.make_request(**crt_callargs)
376 except Exception as e:
377 coordinator.set_exception(e, True)
378 on_done = self._s3_args_creator.get_crt_callback(
379 future, 'done', after_subscribers=on_done_after_calls
380 )
381 on_done(error=e)
382 else:
383 coordinator.set_s3_request(crt_s3_request)
384 self._future_coordinators.append(coordinator)
385
386 self._id_counter += 1
387 return future
388
389
390 class CRTTransferMeta(BaseTransferMeta):
391 """Holds metadata about the CRTTransferFuture"""
392
393 def __init__(self, transfer_id=None, call_args=None):
394 self._transfer_id = transfer_id
395 self._call_args = call_args
396 self._user_context = {}
397
398 @property
399 def call_args(self):
400 return self._call_args
401
402 @property
403 def transfer_id(self):
404 return self._transfer_id
405
406 @property
407 def user_context(self):
408 return self._user_context
409
410
411 class CRTTransferFuture(BaseTransferFuture):
412 def __init__(self, meta=None, coordinator=None):
413 """The future associated to a submitted transfer request via CRT S3 client
414
415 :type meta: s3transfer.crt.CRTTransferMeta
416 :param meta: The metadata associated to the transfer future.
417
418 :type coordinator: s3transfer.crt.CRTTransferCoordinator
419 :param coordinator: The coordinator associated to the transfer future.
420 """
421 self._meta = meta
422 if meta is None:
423 self._meta = CRTTransferMeta()
424 self._coordinator = coordinator
425
426 @property
427 def meta(self):
428 return self._meta
429
430 def done(self):
431 return self._coordinator.done()
432
433 def result(self, timeout=None):
434 self._coordinator.result(timeout)
435
436 def cancel(self):
437 self._coordinator.cancel()
438
439 def set_exception(self, exception):
440 """Sets the exception on the future."""
441 if not self.done():
442 raise TransferNotDoneError(
443 'set_exception can only be called once the transfer is '
444 'complete.'
445 )
446 self._coordinator.set_exception(exception, override=True)
447
448
449 class BaseCRTRequestSerializer:
450 def serialize_http_request(self, transfer_type, future):
451 """Serialize CRT HTTP requests.
452
453 :type transfer_type: string
454 :param transfer_type: the type of transfer made,
455 e.g 'put_object', 'get_object', 'delete_object'
456
457 :type future: s3transfer.crt.CRTTransferFuture
458
459 :rtype: awscrt.http.HttpRequest
460 :returns: An unsigned HTTP request to be used for the CRT S3 client
461 """
462 raise NotImplementedError('serialize_http_request()')
463
464 def translate_crt_exception(self, exception):
465 raise NotImplementedError('translate_crt_exception()')
466
467
468 class BotocoreCRTRequestSerializer(BaseCRTRequestSerializer):
469 def __init__(self, session, client_kwargs=None):
470 """Serialize CRT HTTP request using botocore logic
471 It also takes into account configuration from both the session
472 and any keyword arguments that could be passed to
473 `Session.create_client()` when serializing the request.
474
475 :type session: botocore.session.Session
476
477 :type client_kwargs: Optional[Dict[str, str]])
478 :param client_kwargs: The kwargs for the botocore
479 s3 client initialization.
480 """
481 self._session = session
482 if client_kwargs is None:
483 client_kwargs = {}
484 self._resolve_client_config(session, client_kwargs)
485 self._client = create_nested_client(session, **client_kwargs)
486 self._client.meta.events.register(
487 'request-created.s3.*', self._capture_http_request
488 )
489 self._client.meta.events.register(
490 'after-call.s3.*', self._change_response_to_serialized_http_request
491 )
492 self._client.meta.events.register(
493 'before-send.s3.*', self._make_fake_http_response
494 )
495 self._client.meta.events.register(
496 'before-call.s3.*', self._remove_checksum_context
497 )
498
499 def _resolve_client_config(self, session, client_kwargs):
500 user_provided_config = None
501 if session.get_default_client_config():
502 user_provided_config = session.get_default_client_config()
503 if 'config' in client_kwargs:
504 user_provided_config = client_kwargs['config']
505
506 client_config = Config(signature_version=UNSIGNED)
507 if user_provided_config:
508 client_config = user_provided_config.merge(client_config)
509 client_kwargs['config'] = client_config
510 client_kwargs["service_name"] = "s3"
511
512 def _crt_request_from_aws_request(self, aws_request):
513 url_parts = urlsplit(aws_request.url)
514 crt_path = url_parts.path
515 if url_parts.query:
516 crt_path = f'{crt_path}?{url_parts.query}'
517 headers_list = []
518 for name, value in aws_request.headers.items():
519 if isinstance(value, str):
520 headers_list.append((name, value))
521 else:
522 headers_list.append((name, str(value, 'utf-8')))
523
524 crt_headers = awscrt.http.HttpHeaders(headers_list)
525
526 crt_request = awscrt.http.HttpRequest(
527 method=aws_request.method,
528 path=crt_path,
529 headers=crt_headers,
530 body_stream=aws_request.body,
531 )
532 return crt_request
533
534 def _convert_to_crt_http_request(self, botocore_http_request):
535 # Logic that does CRTUtils.crt_request_from_aws_request
536 crt_request = self._crt_request_from_aws_request(botocore_http_request)
537 if crt_request.headers.get("host") is None:
538 # If host is not set, set it for the request before using CRT s3
539 url_parts = urlsplit(botocore_http_request.url)
540 crt_request.headers.set("host", url_parts.netloc)
541 if crt_request.headers.get('Content-MD5') is not None:
542 crt_request.headers.remove("Content-MD5")
543
544 # In general, the CRT S3 client expects a content length header. It
545 # only expects a missing content length header if the body is not
546 # seekable. However, botocore does not set the content length header
547 # for GetObject API requests and so we set the content length to zero
548 # to meet the CRT S3 client's expectation that the content length
549 # header is set even if there is no body.
550 if crt_request.headers.get('Content-Length') is None:
551 if botocore_http_request.body is None:
552 crt_request.headers.add('Content-Length', "0")
553
554 # Botocore sets the Transfer-Encoding header when it cannot determine
555 # the content length of the request body (e.g. it's not seekable).
556 # However, CRT does not support this header, but it supports
557 # non-seekable bodies. So we remove this header to not cause issues
558 # in the downstream CRT S3 request.
559 if crt_request.headers.get('Transfer-Encoding') is not None:
560 crt_request.headers.remove('Transfer-Encoding')
561
562 return crt_request
563
564 def _capture_http_request(self, request, **kwargs):
565 request.context['http_request'] = request
566
567 def _change_response_to_serialized_http_request(
568 self, context, parsed, **kwargs
569 ):
570 request = context['http_request']
571 parsed['HTTPRequest'] = request.prepare()
572
573 def _make_fake_http_response(self, request, **kwargs):
574 return botocore.awsrequest.AWSResponse(
575 None,
576 200,
577 {},
578 FakeRawResponse(b""),
579 )
580
581 def _get_botocore_http_request(self, client_method, call_args):
582 return getattr(self._client, client_method)(
583 Bucket=call_args.bucket, Key=call_args.key, **call_args.extra_args
584 )['HTTPRequest']
585
586 def serialize_http_request(self, transfer_type, future):
587 botocore_http_request = self._get_botocore_http_request(
588 transfer_type, future.meta.call_args
589 )
590 crt_request = self._convert_to_crt_http_request(botocore_http_request)
591 return crt_request
592
593 def translate_crt_exception(self, exception):
594 if isinstance(exception, awscrt.s3.S3ResponseError):
595 return self._translate_crt_s3_response_error(exception)
596 else:
597 return None
598
599 def _translate_crt_s3_response_error(self, s3_response_error):
600 status_code = s3_response_error.status_code
601 if status_code < 301:
602 # Botocore's exception parsing only
603 # runs on status codes >= 301
604 return None
605
606 headers = {k: v for k, v in s3_response_error.headers}
607 operation_name = s3_response_error.operation_name
608 if operation_name is not None:
609 service_model = self._client.meta.service_model
610 shape = service_model.operation_model(operation_name).output_shape
611 else:
612 shape = None
613
614 response_dict = {
615 'headers': botocore.awsrequest.HeadersDict(headers),
616 'status_code': status_code,
617 'body': s3_response_error.body,
618 }
619 parsed_response = self._client._response_parser.parse(
620 response_dict, shape=shape
621 )
622
623 error_code = parsed_response.get("Error", {}).get("Code")
624 error_class = self._client.exceptions.from_code(error_code)
625 return error_class(parsed_response, operation_name=operation_name)
626
627 def _remove_checksum_context(self, params, **kwargs):
628 request_context = params.get("context", {})
629 if "checksum" in request_context:
630 del request_context["checksum"]
631
632
633 class FakeRawResponse(BytesIO):
634 def stream(self, amt=1024, decode_content=None):
635 while True:
636 chunk = self.read(amt)
637 if not chunk:
638 break
639 yield chunk
640
641
642 class BotocoreCRTCredentialsWrapper:
643 def __init__(self, resolved_botocore_credentials):
644 self._resolved_credentials = resolved_botocore_credentials
645
646 def __call__(self):
647 credentials = self._get_credentials().get_frozen_credentials()
648 return AwsCredentials(
649 credentials.access_key, credentials.secret_key, credentials.token
650 )
651
652 def to_crt_credentials_provider(self):
653 return AwsCredentialsProvider.new_delegate(self)
654
655 def _get_credentials(self):
656 if self._resolved_credentials is None:
657 raise NoCredentialsError()
658 return self._resolved_credentials
659
660
661 class CRTTransferCoordinator:
662 """A helper class for managing CRTTransferFuture"""
663
664 def __init__(
665 self, transfer_id=None, s3_request=None, exception_translator=None
666 ):
667 self.transfer_id = transfer_id
668 self._exception_translator = exception_translator
669 self._s3_request = s3_request
670 self._lock = threading.Lock()
671 self._exception = None
672 self._crt_future = None
673 self._done_event = threading.Event()
674
675 @property
676 def s3_request(self):
677 return self._s3_request
678
679 def set_done_callbacks_complete(self):
680 self._done_event.set()
681
682 def wait_until_on_done_callbacks_complete(self, timeout=None):
683 self._done_event.wait(timeout)
684
685 def set_exception(self, exception, override=False):
686 with self._lock:
687 if not self.done() or override:
688 self._exception = exception
689
690 def cancel(self):
691 if self._s3_request:
692 self._s3_request.cancel()
693
694 def result(self, timeout=None):
695 if self._exception:
696 raise self._exception
697 try:
698 self._crt_future.result(timeout)
699 except KeyboardInterrupt:
700 self.cancel()
701 self._crt_future.result(timeout)
702 raise
703 except Exception as e:
704 self.handle_exception(e)
705 finally:
706 if self._s3_request:
707 self._s3_request = None
708
709 def handle_exception(self, exc):
710 translated_exc = None
711 if self._exception_translator:
712 try:
713 translated_exc = self._exception_translator(exc)
714 except Exception as e:
715 # Bail out if we hit an issue translating
716 # and raise the original error.
717 logger.debug("Unable to translate exception.", exc_info=e)
718 pass
719 if translated_exc is not None:
720 raise translated_exc from exc
721 else:
722 raise exc
723
724 def done(self):
725 if self._crt_future is None:
726 return False
727 return self._crt_future.done()
728
729 def set_s3_request(self, s3_request):
730 self._s3_request = s3_request
731 self._crt_future = self._s3_request.finished_future
732
733
734 class S3ClientArgsCreator:
735 def __init__(self, crt_request_serializer, os_utils):
736 self._request_serializer = crt_request_serializer
737 self._os_utils = os_utils
738
739 def get_make_request_args(
740 self, request_type, call_args, coordinator, future, on_done_after_calls
741 ):
742 request_args_handler = getattr(
743 self,
744 f'_get_make_request_args_{request_type}',
745 self._default_get_make_request_args,
746 )
747 return request_args_handler(
748 request_type=request_type,
749 call_args=call_args,
750 coordinator=coordinator,
751 future=future,
752 on_done_before_calls=[],
753 on_done_after_calls=on_done_after_calls,
754 )
755
756 def get_crt_callback(
757 self,
758 future,
759 callback_type,
760 before_subscribers=None,
761 after_subscribers=None,
762 ):
763 def invoke_all_callbacks(*args, **kwargs):
764 callbacks_list = []
765 if before_subscribers is not None:
766 callbacks_list += before_subscribers
767 callbacks_list += get_callbacks(future, callback_type)
768 if after_subscribers is not None:
769 callbacks_list += after_subscribers
770 for callback in callbacks_list:
771 # The get_callbacks helper will set the first augment
772 # by keyword, the other augments need to be set by keyword
773 # as well
774 if callback_type == "progress":
775 callback(bytes_transferred=args[0])
776 else:
777 callback(*args, **kwargs)
778
779 return invoke_all_callbacks
780
781 def _get_make_request_args_put_object(
782 self,
783 request_type,
784 call_args,
785 coordinator,
786 future,
787 on_done_before_calls,
788 on_done_after_calls,
789 ):
790 send_filepath = None
791 if isinstance(call_args.fileobj, str):
792 send_filepath = call_args.fileobj
793 data_len = self._os_utils.get_file_size(send_filepath)
794 call_args.extra_args["ContentLength"] = data_len
795 else:
796 call_args.extra_args["Body"] = call_args.fileobj
797
798 checksum_config = None
799 if not any(
800 checksum_arg in call_args.extra_args
801 for checksum_arg in FULL_OBJECT_CHECKSUM_ARGS
802 ):
803 checksum_algorithm = call_args.extra_args.pop(
804 'ChecksumAlgorithm', 'CRC32'
805 ).upper()
806 checksum_config = awscrt.s3.S3ChecksumConfig(
807 algorithm=awscrt.s3.S3ChecksumAlgorithm[checksum_algorithm],
808 location=awscrt.s3.S3ChecksumLocation.TRAILER,
809 )
810 # Suppress botocore's automatic MD5 calculation by setting an override
811 # value that will get deleted in the BotocoreCRTRequestSerializer.
812 # As part of the CRT S3 request, we request the CRT S3 client to
813 # automatically add trailing checksums to its uploads.
814 call_args.extra_args["ContentMD5"] = "override-to-be-removed"
815
816 make_request_args = self._default_get_make_request_args(
817 request_type=request_type,
818 call_args=call_args,
819 coordinator=coordinator,
820 future=future,
821 on_done_before_calls=on_done_before_calls,
822 on_done_after_calls=on_done_after_calls,
823 )
824 make_request_args['send_filepath'] = send_filepath
825 make_request_args['checksum_config'] = checksum_config
826 return make_request_args
827
828 def _get_make_request_args_get_object(
829 self,
830 request_type,
831 call_args,
832 coordinator,
833 future,
834 on_done_before_calls,
835 on_done_after_calls,
836 ):
837 recv_filepath = None
838 on_body = None
839 checksum_config = awscrt.s3.S3ChecksumConfig(validate_response=True)
840 if isinstance(call_args.fileobj, str):
841 final_filepath = call_args.fileobj
842 recv_filepath = self._os_utils.get_temp_filename(final_filepath)
843 on_done_before_calls.append(
844 RenameTempFileHandler(
845 coordinator, final_filepath, recv_filepath, self._os_utils
846 )
847 )
848 else:
849 on_body = OnBodyFileObjWriter(call_args.fileobj)
850
851 make_request_args = self._default_get_make_request_args(
852 request_type=request_type,
853 call_args=call_args,
854 coordinator=coordinator,
855 future=future,
856 on_done_before_calls=on_done_before_calls,
857 on_done_after_calls=on_done_after_calls,
858 )
859 make_request_args['recv_filepath'] = recv_filepath
860 make_request_args['on_body'] = on_body
861 make_request_args['checksum_config'] = checksum_config
862 return make_request_args
863
864 def _default_get_make_request_args(
865 self,
866 request_type,
867 call_args,
868 coordinator,
869 future,
870 on_done_before_calls,
871 on_done_after_calls,
872 ):
873 make_request_args = {
874 'request': self._request_serializer.serialize_http_request(
875 request_type, future
876 ),
877 'type': getattr(
878 S3RequestType, request_type.upper(), S3RequestType.DEFAULT
879 ),
880 'on_done': self.get_crt_callback(
881 future, 'done', on_done_before_calls, on_done_after_calls
882 ),
883 'on_progress': self.get_crt_callback(future, 'progress'),
884 }
885
886 # For DEFAULT requests, CRT requires the official S3 operation name.
887 # So transform string like "delete_object" -> "DeleteObject".
888 if make_request_args['type'] == S3RequestType.DEFAULT:
889 make_request_args['operation_name'] = ''.join(
890 x.title() for x in request_type.split('_')
891 )
892
893 arn_handler = _S3ArnParamHandler()
894 if (
895 accesspoint_arn_details := arn_handler.handle_arn(call_args.bucket)
896 ) and accesspoint_arn_details['region'] == "":
897 # Configure our region to `*` to propogate in `x-amz-region-set`
898 # for multi-region support in MRAP accesspoints.
899 # use_double_uri_encode and should_normalize_uri_path are defaulted to be True
900 # But SDK already encoded the URI, and it's for S3, so set both to False
901 make_request_args['signing_config'] = AwsSigningConfig(
902 algorithm=AwsSigningAlgorithm.V4_ASYMMETRIC,
903 region="*",
904 use_double_uri_encode=False,
905 should_normalize_uri_path=False,
906 )
907 call_args.bucket = accesspoint_arn_details['resource_name']
908 elif is_s3express_bucket(call_args.bucket):
909 # use_double_uri_encode and should_normalize_uri_path are defaulted to be True
910 # But SDK already encoded the URI, and it's for S3, so set both to False
911 make_request_args['signing_config'] = AwsSigningConfig(
912 algorithm=AwsSigningAlgorithm.V4_S3EXPRESS,
913 use_double_uri_encode=False,
914 should_normalize_uri_path=False,
915 )
916 return make_request_args
917
918
919 class RenameTempFileHandler:
920 def __init__(self, coordinator, final_filename, temp_filename, osutil):
921 self._coordinator = coordinator
922 self._final_filename = final_filename
923 self._temp_filename = temp_filename
924 self._osutil = osutil
925
926 def __call__(self, **kwargs):
927 error = kwargs['error']
928 if error:
929 self._osutil.remove_file(self._temp_filename)
930 else:
931 try:
932 self._osutil.rename_file(
933 self._temp_filename, self._final_filename
934 )
935 except Exception as e:
936 self._osutil.remove_file(self._temp_filename)
937 # the CRT future has done already at this point
938 self._coordinator.set_exception(e)
939
940
941 class AfterDoneHandler:
942 def __init__(self, coordinator):
943 self._coordinator = coordinator
944
945 def __call__(self, **kwargs):
946 self._coordinator.set_done_callbacks_complete()
947
948
949 class OnBodyFileObjWriter:
950 def __init__(self, fileobj):
951 self._fileobj = fileobj
952
953 def __call__(self, chunk, **kwargs):
954 self._fileobj.write(chunk)
955
956
957 class _S3ArnParamHandler:
958 """Partial port of S3ArnParamHandler from botocore.
959
960 This is used to make a determination on MRAP accesspoints for signing
961 purposes. This should be safe to remove once we properly integrate auth
962 resolution from Botocore into the CRT transfer integration.
963 """
964
965 _RESOURCE_REGEX = re.compile(
966 r'^(?P<resource_type>accesspoint|outpost)[/:](?P<resource_name>.+)$'
967 )
968
969 def __init__(self):
970 self._arn_parser = ArnParser()
971
972 def handle_arn(self, bucket):
973 arn_details = self._get_arn_details_from_bucket(bucket)
974 if arn_details is None:
975 return
976 if arn_details['resource_type'] == 'accesspoint':
977 return arn_details
978
979 def _get_arn_details_from_bucket(self, bucket):
980 try:
981 arn_details = self._arn_parser.parse_arn(bucket)
982 self._add_resource_type_and_name(arn_details)
983 return arn_details
984 except InvalidArnException:
985 pass
986 return None
987
988 def _add_resource_type_and_name(self, arn_details):
989 match = self._RESOURCE_REGEX.match(arn_details['resource'])
990 if match:
991 arn_details['resource_type'] = match.group('resource_type')
992 arn_details['resource_name'] = match.group('resource_name')