Debian Codegraph @

s3transfer/__init__.py

raw ◀ ▶
2 Debian patches touch this file
1 # Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License"). You
4 # may not use this file except in compliance with the License. A copy of
5 # the License is located at
6 #
7 # http://aws.amazon.com/apache2.0/
8 #
9 # or in the "license" file accompanying this file. This file is
10 # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
11 # ANY KIND, either express or implied. See the License for the specific
12 # language governing permissions and limitations under the License.
13 """Abstractions over S3's upload/download operations.
14
15 This module provides high level abstractions for efficient
16 uploads/downloads. It handles several things for the user:
17
18 * Automatically switching to multipart transfers when
19 a file is over a specific size threshold
20 * Uploading/downloading a file in parallel
21 * Throttling based on max bandwidth
22 * Progress callbacks to monitor transfers
23 * Retries. While botocore handles retries for streaming uploads,
24 it is not possible for it to handle retries for streaming
25 downloads. This module handles retries for both cases so
26 you don't need to implement any retry logic yourself.
27
28 This module has a reasonable set of defaults. It also allows you
29 to configure many aspects of the transfer process including:
30
31 * Multipart threshold size
32 * Max parallel downloads
33 * Max bandwidth
34 * Socket timeouts
35 * Retry amounts
36
37 There is no support for s3->s3 multipart copies at this
38 time.
39
40
41 .. _ref_s3transfer_usage:
42
43 Usage
44 =====
45
46 The simplest way to use this module is:
47
48 .. code-block:: python
49
50 client = boto3.client('s3', 'us-west-2')
51 transfer = S3Transfer(client)
52 # Upload /tmp/myfile to s3://bucket/key
53 transfer.upload_file('/tmp/myfile', 'bucket', 'key')
54
55 # Download s3://bucket/key to /tmp/myfile
56 transfer.download_file('bucket', 'key', '/tmp/myfile')
57
58 The ``upload_file`` and ``download_file`` methods also accept
59 ``**kwargs``, which will be forwarded through to the corresponding
60 client operation. Here are a few examples using ``upload_file``::
61
62 # Making the object public
63 transfer.upload_file('/tmp/myfile', 'bucket', 'key',
64 extra_args={'ACL': 'public-read'})
65
66 # Setting metadata
67 transfer.upload_file('/tmp/myfile', 'bucket', 'key',
68 extra_args={'Metadata': {'a': 'b', 'c': 'd'}})
69
70 # Setting content type
71 transfer.upload_file('/tmp/myfile.json', 'bucket', 'key',
72 extra_args={'ContentType': "application/json"})
73
74
75 The ``S3Transfer`` class also supports progress callbacks so you can
76 provide transfer progress to users. Both the ``upload_file`` and
77 ``download_file`` methods take an optional ``callback`` parameter.
78 Here's an example of how to print a simple progress percentage
79 to the user:
80
81 .. code-block:: python
82
83 class ProgressPercentage(object):
84 def __init__(self, filename):
85 self._filename = filename
86 self._size = float(os.path.getsize(filename))
87 self._seen_so_far = 0
88 self._lock = threading.Lock()
89
90 def __call__(self, bytes_amount):
91 # To simplify we'll assume this is hooked up
92 # to a single filename.
93 with self._lock:
94 self._seen_so_far += bytes_amount
95 percentage = (self._seen_so_far / self._size) * 100
96 sys.stdout.write(
97 "\r%s %s / %s (%.2f%%)" % (self._filename, self._seen_so_far,
98 self._size, percentage))
99 sys.stdout.flush()
100
101
102 transfer = S3Transfer(boto3.client('s3', 'us-west-2'))
103 # Upload /tmp/myfile to s3://bucket/key and print upload progress.
104 transfer.upload_file('/tmp/myfile', 'bucket', 'key',
105 callback=ProgressPercentage('/tmp/myfile'))
106
107
108
109 You can also provide a TransferConfig object to the S3Transfer
110 object that gives you more fine grained control over the
111 transfer. For example:
112
113 .. code-block:: python
114
115 client = boto3.client('s3', 'us-west-2')
116 config = TransferConfig(
117 multipart_threshold=8 * 1024 * 1024,
118 max_concurrency=10,
119 num_download_attempts=10,
120 )
121 transfer = S3Transfer(client, config)
122 transfer.upload_file('/tmp/foo', 'bucket', 'key')
123
124
125 """
126
127 import concurrent.futures
128 import functools
129 import logging
130 import math
131 import os
132 import queue
133 import random
134 import socket
135 import string
136 import threading
137 from logging import NullHandler
138
139 from botocore.exceptions import IncompleteReadError, ResponseStreamingError
•140 from urllib3.exceptions import (
141 ReadTimeoutError,
142 )
143
144 import s3transfer.compat
145 from s3transfer.exceptions import RetriesExceededError, S3UploadFailedError
146
147 __author__ = 'Amazon Web Services'
148 __version__ = '0.14.0'
149
150
151 logger = logging.getLogger(__name__)
152 logger.addHandler(NullHandler())
153
154 MB = 1024 * 1024
155 SHUTDOWN_SENTINEL = object()
156
157
158 def random_file_extension(num_digits=8):
159 return ''.join(random.choice(string.hexdigits) for _ in range(num_digits))
160
161
162 def disable_upload_callbacks(request, operation_name, **kwargs):
163 if operation_name in ['PutObject', 'UploadPart'] and hasattr(
164 request.body, 'disable_callback'
165 ):
166 request.body.disable_callback()
167
168
169 def enable_upload_callbacks(request, operation_name, **kwargs):
170 if operation_name in ['PutObject', 'UploadPart'] and hasattr(
171 request.body, 'enable_callback'
172 ):
173 request.body.enable_callback()
174
175
176 class QueueShutdownError(Exception):
177 pass
178
179
180 class ReadFileChunk:
181 def __init__(
182 self,
183 fileobj,
184 start_byte,
185 chunk_size,
186 full_file_size,
187 callback=None,
188 enable_callback=True,
189 ):
190 """
191
192 Given a file object shown below:
193
194 |___________________________________________________|
195 0 | | full_file_size
196 |----chunk_size---|
197 start_byte
198
199 :type fileobj: file
200 :param fileobj: File like object
201
202 :type start_byte: int
203 :param start_byte: The first byte from which to start reading.
204
205 :type chunk_size: int
206 :param chunk_size: The max chunk size to read. Trying to read
207 pass the end of the chunk size will behave like you've
208 reached the end of the file.
209
210 :type full_file_size: int
211 :param full_file_size: The entire content length associated
212 with ``fileobj``.
213
214 :type callback: function(amount_read)
215 :param callback: Called whenever data is read from this object.
216
217 """
218 self._fileobj = fileobj
219 self._start_byte = start_byte
220 self._size = self._calculate_file_size(
221 self._fileobj,
222 requested_size=chunk_size,
223 start_byte=start_byte,
224 actual_file_size=full_file_size,
225 )
226 self._fileobj.seek(self._start_byte)
227 self._amount_read = 0
228 self._callback = callback
229 self._callback_enabled = enable_callback
230
231 @classmethod
232 def from_filename(
233 cls,
234 filename,
235 start_byte,
236 chunk_size,
237 callback=None,
238 enable_callback=True,
239 ):
240 """Convenience factory function to create from a filename.
241
242 :type start_byte: int
243 :param start_byte: The first byte from which to start reading.
244
245 :type chunk_size: int
246 :param chunk_size: The max chunk size to read. Trying to read
247 pass the end of the chunk size will behave like you've
248 reached the end of the file.
249
250 :type full_file_size: int
251 :param full_file_size: The entire content length associated
252 with ``fileobj``.
253
254 :type callback: function(amount_read)
255 :param callback: Called whenever data is read from this object.
256
257 :type enable_callback: bool
258 :param enable_callback: Indicate whether to invoke callback
259 during read() calls.
260
261 :rtype: ``ReadFileChunk``
262 :return: A new instance of ``ReadFileChunk``
263
264 """
265 f = open(filename, 'rb')
266 file_size = os.fstat(f.fileno()).st_size
267 return cls(
268 f, start_byte, chunk_size, file_size, callback, enable_callback
269 )
270
271 def _calculate_file_size(
272 self, fileobj, requested_size, start_byte, actual_file_size
273 ):
274 max_chunk_size = actual_file_size - start_byte
275 return min(max_chunk_size, requested_size)
276
277 def read(self, amount=None):
278 if amount is None:
279 amount_to_read = self._size - self._amount_read
280 else:
281 amount_to_read = min(self._size - self._amount_read, amount)
282 data = self._fileobj.read(amount_to_read)
283 self._amount_read += len(data)
284 if self._callback is not None and self._callback_enabled:
285 self._callback(len(data))
286 return data
287
288 def enable_callback(self):
289 self._callback_enabled = True
290
291 def disable_callback(self):
292 self._callback_enabled = False
293
294 def seek(self, where):
295 self._fileobj.seek(self._start_byte + where)
296 if self._callback is not None and self._callback_enabled:
297 # To also rewind the callback() for an accurate progress report
298 self._callback(where - self._amount_read)
299 self._amount_read = where
300
301 def close(self):
302 self._fileobj.close()
303
304 def tell(self):
305 return self._amount_read
306
307 def __len__(self):
308 # __len__ is defined because requests will try to determine the length
309 # of the stream to set a content length. In the normal case
310 # of the file it will just stat the file, but we need to change that
311 # behavior. By providing a __len__, requests will use that instead
312 # of stat'ing the file.
313 return self._size
314
315 def __enter__(self):
316 return self
317
318 def __exit__(self, *args, **kwargs):
319 self.close()
320
321 def __iter__(self):
322 # This is a workaround for http://bugs.python.org/issue17575
323 # Basically httplib will try to iterate over the contents, even
324 # if its a file like object. This wasn't noticed because we've
325 # already exhausted the stream so iterating over the file immediately
326 # stops, which is what we're simulating here.
327 return iter([])
328
329
330 class StreamReaderProgress:
331 """Wrapper for a read only stream that adds progress callbacks."""
332
333 def __init__(self, stream, callback=None):
334 self._stream = stream
335 self._callback = callback
336
337 def read(self, *args, **kwargs):
338 value = self._stream.read(*args, **kwargs)
339 if self._callback is not None:
340 self._callback(len(value))
341 return value
342
343
344 class OSUtils:
345 def get_file_size(self, filename):
346 return os.path.getsize(filename)
347
348 def open_file_chunk_reader(self, filename, start_byte, size, callback):
349 return ReadFileChunk.from_filename(
350 filename, start_byte, size, callback, enable_callback=False
351 )
352
353 def open(self, filename, mode):
354 return open(filename, mode)
355
356 def remove_file(self, filename):
357 """Remove a file, noop if file does not exist."""
358 # Unlike os.remove, if the file does not exist,
359 # then this method does nothing.
360 try:
361 os.remove(filename)
362 except OSError:
363 pass
364
365 def rename_file(self, current_filename, new_filename):
366 s3transfer.compat.rename_file(current_filename, new_filename)
367
368
369 class MultipartUploader:
370 # These are the extra_args that need to be forwarded onto
371 # subsequent upload_parts.
372 UPLOAD_PART_ARGS = [
373 'SSECustomerKey',
374 'SSECustomerAlgorithm',
375 'SSECustomerKeyMD5',
376 'RequestPayer',
377 ]
378
379 def __init__(
380 self,
381 client,
382 config,
383 osutil,
384 executor_cls=concurrent.futures.ThreadPoolExecutor,
385 ):
386 self._client = client
387 self._config = config
388 self._os = osutil
389 self._executor_cls = executor_cls
390
391 def _extra_upload_part_args(self, extra_args):
392 # Only the args in UPLOAD_PART_ARGS actually need to be passed
393 # onto the upload_part calls.
394 upload_parts_args = {}
395 for key, value in extra_args.items():
396 if key in self.UPLOAD_PART_ARGS:
397 upload_parts_args[key] = value
398 return upload_parts_args
399
400 def upload_file(self, filename, bucket, key, callback, extra_args):
401 response = self._client.create_multipart_upload(
402 Bucket=bucket, Key=key, **extra_args
403 )
404 upload_id = response['UploadId']
405 try:
406 parts = self._upload_parts(
407 upload_id, filename, bucket, key, callback, extra_args
408 )
409 except Exception as e:
410 logger.debug(
411 "Exception raised while uploading parts, "
412 "aborting multipart upload.",
413 exc_info=True,
414 )
415 self._client.abort_multipart_upload(
416 Bucket=bucket, Key=key, UploadId=upload_id
417 )
418 raise S3UploadFailedError(
419 "Failed to upload {} to {}: {}".format(
420 filename, '/'.join([bucket, key]), e
421 )
422 )
423 self._client.complete_multipart_upload(
424 Bucket=bucket,
425 Key=key,
426 UploadId=upload_id,
427 MultipartUpload={'Parts': parts},
428 )
429
430 def _upload_parts(
431 self, upload_id, filename, bucket, key, callback, extra_args
432 ):
433 upload_parts_extra_args = self._extra_upload_part_args(extra_args)
434 parts = []
435 part_size = self._config.multipart_chunksize
436 num_parts = int(
437 math.ceil(self._os.get_file_size(filename) / float(part_size))
438 )
439 max_workers = self._config.max_concurrency
440 with self._executor_cls(max_workers=max_workers) as executor:
441 upload_partial = functools.partial(
442 self._upload_one_part,
443 filename,
444 bucket,
445 key,
446 upload_id,
447 part_size,
448 upload_parts_extra_args,
449 callback,
450 )
451 for part in executor.map(upload_partial, range(1, num_parts + 1)):
452 parts.append(part)
453 return parts
454
455 def _upload_one_part(
456 self,
457 filename,
458 bucket,
459 key,
460 upload_id,
461 part_size,
462 extra_args,
463 callback,
464 part_number,
465 ):
466 open_chunk_reader = self._os.open_file_chunk_reader
467 with open_chunk_reader(
468 filename, part_size * (part_number - 1), part_size, callback
469 ) as body:
470 response = self._client.upload_part(
471 Bucket=bucket,
472 Key=key,
473 UploadId=upload_id,
474 PartNumber=part_number,
475 Body=body,
476 **extra_args,
477 )
478 etag = response['ETag']
479 return {'ETag': etag, 'PartNumber': part_number}
480
481
482 class ShutdownQueue(queue.Queue):
483 """A queue implementation that can be shutdown.
484
485 Shutting down a queue means that this class adds a
486 trigger_shutdown method that will trigger all subsequent
487 calls to put() to fail with a ``QueueShutdownError``.
488
489 It purposefully deviates from queue.Queue, and is *not* meant
490 to be a drop in replacement for ``queue.Queue``.
491
492 """
493
494 def _init(self, maxsize):
495 self._shutdown = False
496 self._shutdown_lock = threading.Lock()
497 # queue.Queue is an old style class so we don't use super().
498 return queue.Queue._init(self, maxsize)
499
500 def trigger_shutdown(self):
501 with self._shutdown_lock:
502 self._shutdown = True
503 logger.debug("The IO queue is now shutdown.")
504
505 def put(self, item):
506 # Note: this is not sufficient, it's still possible to deadlock!
507 # Need to hook into the condition vars used by this class.
508 with self._shutdown_lock:
509 if self._shutdown:
510 raise QueueShutdownError(
511 "Cannot put item to queue when queue has been shutdown."
512 )
513 return queue.Queue.put(self, item)
514
515
516 class MultipartDownloader:
517 def __init__(
518 self,
519 client,
520 config,
521 osutil,
522 executor_cls=concurrent.futures.ThreadPoolExecutor,
523 ):
524 self._client = client
525 self._config = config
526 self._os = osutil
527 self._executor_cls = executor_cls
528 self._ioqueue = ShutdownQueue(self._config.max_io_queue)
529
530 def download_file(
531 self, bucket, key, filename, object_size, extra_args, callback=None
532 ):
533 with self._executor_cls(max_workers=2) as controller:
534 # 1 thread for the future that manages the uploading of files
535 # 1 thread for the future that manages IO writes.
536 download_parts_handler = functools.partial(
537 self._download_file_as_future,
538 bucket,
539 key,
540 filename,
541 object_size,
542 callback,
543 )
544 parts_future = controller.submit(download_parts_handler)
545
546 io_writes_handler = functools.partial(
547 self._perform_io_writes, filename
548 )
549 io_future = controller.submit(io_writes_handler)
550 results = concurrent.futures.wait(
551 [parts_future, io_future],
552 return_when=concurrent.futures.FIRST_EXCEPTION,
553 )
554 self._process_future_results(results)
555
556 def _process_future_results(self, futures):
557 finished, unfinished = futures
558 for future in finished:
559 future.result()
560
561 def _download_file_as_future(
562 self, bucket, key, filename, object_size, callback
563 ):
564 part_size = self._config.multipart_chunksize
565 num_parts = int(math.ceil(object_size / float(part_size)))
566 max_workers = self._config.max_concurrency
567 download_partial = functools.partial(
568 self._download_range,
569 bucket,
570 key,
571 filename,
572 part_size,
573 num_parts,
574 callback,
575 )
576 try:
577 with self._executor_cls(max_workers=max_workers) as executor:
578 list(executor.map(download_partial, range(num_parts)))
579 finally:
580 self._ioqueue.put(SHUTDOWN_SENTINEL)
581
582 def _calculate_range_param(self, part_size, part_index, num_parts):
583 start_range = part_index * part_size
584 if part_index == num_parts - 1:
585 end_range = ''
586 else:
587 end_range = start_range + part_size - 1
588 range_param = f'bytes={start_range}-{end_range}'
589 return range_param
590
591 def _download_range(
592 self, bucket, key, filename, part_size, num_parts, callback, part_index
593 ):
594 try:
595 range_param = self._calculate_range_param(
596 part_size, part_index, num_parts
597 )
598
599 max_attempts = self._config.num_download_attempts
600 last_exception = None
601 for i in range(max_attempts):
602 try:
603 logger.debug("Making get_object call.")
604 response = self._client.get_object(
605 Bucket=bucket, Key=key, Range=range_param
606 )
607 streaming_body = StreamReaderProgress(
608 response['Body'], callback
609 )
610 buffer_size = 1024 * 16
611 current_index = part_size * part_index
612 for chunk in iter(
613 lambda: streaming_body.read(buffer_size), b''
614 ):
615 self._ioqueue.put((current_index, chunk))
616 current_index += len(chunk)
617 return
618 except (
619 socket.timeout,
620 OSError,
621 ReadTimeoutError,
622 IncompleteReadError,
623 ResponseStreamingError,
624 ) as e:
625 logger.debug(
626 "Retrying exception caught (%s), "
627 "retrying request, (attempt %s / %s)",
628 e,
629 i,
630 max_attempts,
631 exc_info=True,
632 )
633 last_exception = e
634 continue
635 raise RetriesExceededError(last_exception)
636 finally:
637 logger.debug("EXITING _download_range for part: %s", part_index)
638
639 def _perform_io_writes(self, filename):
640 with self._os.open(filename, 'wb') as f:
641 while True:
642 task = self._ioqueue.get()
643 if task is SHUTDOWN_SENTINEL:
644 logger.debug(
645 "Shutdown sentinel received in IO handler, "
646 "shutting down IO handler."
647 )
648 return
649 else:
650 try:
651 offset, data = task
652 f.seek(offset)
653 f.write(data)
654 except Exception as e:
655 logger.debug(
656 "Caught exception in IO thread: %s",
657 e,
658 exc_info=True,
659 )
660 self._ioqueue.trigger_shutdown()
661 raise
662
663
664 class TransferConfig:
665 def __init__(
666 self,
667 multipart_threshold=8 * MB,
668 max_concurrency=10,
669 multipart_chunksize=8 * MB,
670 num_download_attempts=5,
671 max_io_queue=100,
672 ):
673 self.multipart_threshold = multipart_threshold
674 self.max_concurrency = max_concurrency
675 self.multipart_chunksize = multipart_chunksize
676 self.num_download_attempts = num_download_attempts
677 self.max_io_queue = max_io_queue
678
679
680 class S3Transfer:
681 ALLOWED_DOWNLOAD_ARGS = [
682 'VersionId',
683 'SSECustomerAlgorithm',
684 'SSECustomerKey',
685 'SSECustomerKeyMD5',
686 'RequestPayer',
687 ]
688
689 ALLOWED_UPLOAD_ARGS = [
690 'ACL',
691 'CacheControl',
692 'ContentDisposition',
693 'ContentEncoding',
694 'ContentLanguage',
695 'ContentType',
696 'Expires',
697 'GrantFullControl',
698 'GrantRead',
699 'GrantReadACP',
700 'GrantWriteACL',
701 'Metadata',
702 'RequestPayer',
703 'ServerSideEncryption',
704 'StorageClass',
705 'SSECustomerAlgorithm',
706 'SSECustomerKey',
707 'SSECustomerKeyMD5',
708 'SSEKMSKeyId',
709 'SSEKMSEncryptionContext',
710 'Tagging',
711 ]
712
713 def __init__(self, client, config=None, osutil=None):
714 self._client = client
715 self._client.meta.events.register(
716 'before-call.s3.*', self._update_checksum_context
717 )
718 if config is None:
719 config = TransferConfig()
720 self._config = config
721 if osutil is None:
722 osutil = OSUtils()
723 self._osutil = osutil
724
725 def _update_checksum_context(self, params, **kwargs):
726 request_context = params.get("context", {})
727 checksum_context = request_context.get("checksum", {})
728 if "request_algorithm" in checksum_context:
729 # Force request checksum algorithm in the header if specified.
730 checksum_context["request_algorithm"]["in"] = "header"
731
732 def upload_file(
733 self, filename, bucket, key, callback=None, extra_args=None
734 ):
735 """Upload a file to an S3 object.
736
737 Variants have also been injected into S3 client, Bucket and Object.
738 You don't have to use S3Transfer.upload_file() directly.
739 """
740 if extra_args is None:
741 extra_args = {}
742 self._validate_all_known_args(extra_args, self.ALLOWED_UPLOAD_ARGS)
743 events = self._client.meta.events
744 events.register_first(
745 'request-created.s3',
746 disable_upload_callbacks,
747 unique_id='s3upload-callback-disable',
748 )
749 events.register_last(
750 'request-created.s3',
751 enable_upload_callbacks,
752 unique_id='s3upload-callback-enable',
753 )
754 if (
755 self._osutil.get_file_size(filename)
756 >= self._config.multipart_threshold
757 ):
758 self._multipart_upload(filename, bucket, key, callback, extra_args)
759 else:
760 self._put_object(filename, bucket, key, callback, extra_args)
761
762 def _put_object(self, filename, bucket, key, callback, extra_args):
763 # We're using open_file_chunk_reader so we can take advantage of the
764 # progress callback functionality.
765 open_chunk_reader = self._osutil.open_file_chunk_reader
766 with open_chunk_reader(
767 filename,
768 0,
769 self._osutil.get_file_size(filename),
770 callback=callback,
771 ) as body:
772 self._client.put_object(
773 Bucket=bucket, Key=key, Body=body, **extra_args
774 )
775
776 def download_file(
777 self, bucket, key, filename, extra_args=None, callback=None
778 ):
779 """Download an S3 object to a file.
780
781 Variants have also been injected into S3 client, Bucket and Object.
782 You don't have to use S3Transfer.download_file() directly.
783 """
784 # This method will issue a ``head_object`` request to determine
785 # the size of the S3 object. This is used to determine if the
786 # object is downloaded in parallel.
787 if extra_args is None:
788 extra_args = {}
789 self._validate_all_known_args(extra_args, self.ALLOWED_DOWNLOAD_ARGS)
790 object_size = self._object_size(bucket, key, extra_args)
791 temp_filename = filename + os.extsep + random_file_extension()
792 try:
793 self._download_file(
794 bucket, key, temp_filename, object_size, extra_args, callback
795 )
796 except Exception:
797 logger.debug(
798 "Exception caught in download_file, removing partial file: %s",
799 temp_filename,
800 exc_info=True,
801 )
802 self._osutil.remove_file(temp_filename)
803 raise
804 else:
805 self._osutil.rename_file(temp_filename, filename)
806
807 def _download_file(
808 self, bucket, key, filename, object_size, extra_args, callback
809 ):
810 if object_size >= self._config.multipart_threshold:
811 self._ranged_download(
812 bucket, key, filename, object_size, extra_args, callback
813 )
814 else:
815 self._get_object(bucket, key, filename, extra_args, callback)
816
817 def _validate_all_known_args(self, actual, allowed):
818 for kwarg in actual:
819 if kwarg not in allowed:
820 raise ValueError(
821 f"Invalid extra_args key '{kwarg}', "
822 f"must be one of: {', '.join(allowed)}"
823 )
824
825 def _ranged_download(
826 self, bucket, key, filename, object_size, extra_args, callback
827 ):
828 downloader = MultipartDownloader(
829 self._client, self._config, self._osutil
830 )
831 downloader.download_file(
832 bucket, key, filename, object_size, extra_args, callback
833 )
834
835 def _get_object(self, bucket, key, filename, extra_args, callback):
836 # precondition: num_download_attempts > 0
837 max_attempts = self._config.num_download_attempts
838 last_exception = None
839 for i in range(max_attempts):
840 try:
841 return self._do_get_object(
842 bucket, key, filename, extra_args, callback
843 )
844 except (
845 socket.timeout,
846 OSError,
847 ReadTimeoutError,
848 IncompleteReadError,
849 ResponseStreamingError,
850 ) as e:
851 # TODO: we need a way to reset the callback if the
852 # download failed.
853 logger.debug(
854 "Retrying exception caught (%s), "
855 "retrying request, (attempt %s / %s)",
856 e,
857 i,
858 max_attempts,
859 exc_info=True,
860 )
861 last_exception = e
862 continue
863 raise RetriesExceededError(last_exception)
864
865 def _do_get_object(self, bucket, key, filename, extra_args, callback):
866 response = self._client.get_object(
867 Bucket=bucket, Key=key, **extra_args
868 )
869 streaming_body = StreamReaderProgress(response['Body'], callback)
870 with self._osutil.open(filename, 'wb') as f:
871 for chunk in iter(lambda: streaming_body.read(8192), b''):
872 f.write(chunk)
873
874 def _object_size(self, bucket, key, extra_args):
875 return self._client.head_object(Bucket=bucket, Key=key, **extra_args)[
876 'ContentLength'
877 ]
878
879 def _multipart_upload(self, filename, bucket, key, callback, extra_args):
880 uploader = MultipartUploader(self._client, self._config, self._osutil)
881 uploader.upload_file(filename, bucket, key, callback, extra_args)