| 1 |
# Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. |
| 2 |
# |
| 3 |
# Licensed under the Apache License, Version 2.0 (the "License"). You |
| 4 |
# may not use this file except in compliance with the License. A copy of |
| 5 |
# the License is located at |
| 6 |
# |
| 7 |
# http://aws.amazon.com/apache2.0/ |
| 8 |
# |
| 9 |
# or in the "license" file accompanying this file. This file is |
| 10 |
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF |
| 11 |
# ANY KIND, either express or implied. See the License for the specific |
| 12 |
# language governing permissions and limitations under the License. |
| 13 |
"""Abstractions over S3's upload/download operations. |
| 14 |
|
| 15 |
This module provides high level abstractions for efficient |
| 16 |
uploads/downloads. It handles several things for the user: |
| 17 |
|
| 18 |
* Automatically switching to multipart transfers when |
| 19 |
a file is over a specific size threshold |
| 20 |
* Uploading/downloading a file in parallel |
| 21 |
* Throttling based on max bandwidth |
| 22 |
* Progress callbacks to monitor transfers |
| 23 |
* Retries. While botocore handles retries for streaming uploads, |
| 24 |
it is not possible for it to handle retries for streaming |
| 25 |
downloads. This module handles retries for both cases so |
| 26 |
you don't need to implement any retry logic yourself. |
| 27 |
|
| 28 |
This module has a reasonable set of defaults. It also allows you |
| 29 |
to configure many aspects of the transfer process including: |
| 30 |
|
| 31 |
* Multipart threshold size |
| 32 |
* Max parallel downloads |
| 33 |
* Max bandwidth |
| 34 |
* Socket timeouts |
| 35 |
* Retry amounts |
| 36 |
|
| 37 |
There is no support for s3->s3 multipart copies at this |
| 38 |
time. |
| 39 |
|
| 40 |
|
| 41 |
.. _ref_s3transfer_usage: |
| 42 |
|
| 43 |
Usage |
| 44 |
===== |
| 45 |
|
| 46 |
The simplest way to use this module is: |
| 47 |
|
| 48 |
.. code-block:: python |
| 49 |
|
| 50 |
client = boto3.client('s3', 'us-west-2') |
| 51 |
transfer = S3Transfer(client) |
| 52 |
# Upload /tmp/myfile to s3://bucket/key |
| 53 |
transfer.upload_file('/tmp/myfile', 'bucket', 'key') |
| 54 |
|
| 55 |
# Download s3://bucket/key to /tmp/myfile |
| 56 |
transfer.download_file('bucket', 'key', '/tmp/myfile') |
| 57 |
|
| 58 |
The ``upload_file`` and ``download_file`` methods also accept |
| 59 |
``**kwargs``, which will be forwarded through to the corresponding |
| 60 |
client operation. Here are a few examples using ``upload_file``:: |
| 61 |
|
| 62 |
# Making the object public |
| 63 |
transfer.upload_file('/tmp/myfile', 'bucket', 'key', |
| 64 |
extra_args={'ACL': 'public-read'}) |
| 65 |
|
| 66 |
# Setting metadata |
| 67 |
transfer.upload_file('/tmp/myfile', 'bucket', 'key', |
| 68 |
extra_args={'Metadata': {'a': 'b', 'c': 'd'}}) |
| 69 |
|
| 70 |
# Setting content type |
| 71 |
transfer.upload_file('/tmp/myfile.json', 'bucket', 'key', |
| 72 |
extra_args={'ContentType': "application/json"}) |
| 73 |
|
| 74 |
|
| 75 |
The ``S3Transfer`` class also supports progress callbacks so you can |
| 76 |
provide transfer progress to users. Both the ``upload_file`` and |
| 77 |
``download_file`` methods take an optional ``callback`` parameter. |
| 78 |
Here's an example of how to print a simple progress percentage |
| 79 |
to the user: |
| 80 |
|
| 81 |
.. code-block:: python |
| 82 |
|
| 83 |
class ProgressPercentage(object): |
| 84 |
def __init__(self, filename): |
| 85 |
self._filename = filename |
| 86 |
self._size = float(os.path.getsize(filename)) |
| 87 |
self._seen_so_far = 0 |
| 88 |
self._lock = threading.Lock() |
| 89 |
|
| 90 |
def __call__(self, bytes_amount): |
| 91 |
# To simplify we'll assume this is hooked up |
| 92 |
# to a single filename. |
| 93 |
with self._lock: |
| 94 |
self._seen_so_far += bytes_amount |
| 95 |
percentage = (self._seen_so_far / self._size) * 100 |
| 96 |
sys.stdout.write( |
| 97 |
"\r%s %s / %s (%.2f%%)" % (self._filename, self._seen_so_far, |
| 98 |
self._size, percentage)) |
| 99 |
sys.stdout.flush() |
| 100 |
|
| 101 |
|
| 102 |
transfer = S3Transfer(boto3.client('s3', 'us-west-2')) |
| 103 |
# Upload /tmp/myfile to s3://bucket/key and print upload progress. |
| 104 |
transfer.upload_file('/tmp/myfile', 'bucket', 'key', |
| 105 |
callback=ProgressPercentage('/tmp/myfile')) |
| 106 |
|
| 107 |
|
| 108 |
|
| 109 |
You can also provide a TransferConfig object to the S3Transfer |
| 110 |
object that gives you more fine grained control over the |
| 111 |
transfer. For example: |
| 112 |
|
| 113 |
.. code-block:: python |
| 114 |
|
| 115 |
client = boto3.client('s3', 'us-west-2') |
| 116 |
config = TransferConfig( |
| 117 |
multipart_threshold=8 * 1024 * 1024, |
| 118 |
max_concurrency=10, |
| 119 |
num_download_attempts=10, |
| 120 |
) |
| 121 |
transfer = S3Transfer(client, config) |
| 122 |
transfer.upload_file('/tmp/foo', 'bucket', 'key') |
| 123 |
|
| 124 |
|
| 125 |
""" |
| 126 |
|
| 127 |
import concurrent.futures |
| 128 |
import functools |
| 129 |
import logging |
| 130 |
import math |
| 131 |
import os |
| 132 |
import queue |
| 133 |
import random |
| 134 |
import socket |
| 135 |
import string |
| 136 |
import threading |
| 137 |
from logging import NullHandler |
| 138 |
|
| 139 |
from botocore.exceptions import IncompleteReadError, ResponseStreamingError |
| 140 |
from urllib3.exceptions import ( |
| 141 |
ReadTimeoutError, |
| 142 |
) |
| 143 |
|
| 144 |
import s3transfer.compat |
| 145 |
from s3transfer.exceptions import RetriesExceededError, S3UploadFailedError |
| 146 |
|
| 147 |
__author__ = 'Amazon Web Services' |
| 148 |
__version__ = '0.14.0' |
| 149 |
|
| 150 |
|
| 151 |
logger = logging.getLogger(__name__) |
| 152 |
logger.addHandler(NullHandler()) |
| 153 |
|
| 154 |
MB = 1024 * 1024 |
| 155 |
SHUTDOWN_SENTINEL = object() |
| 156 |
|
| 157 |
|
| 158 |
def random_file_extension(num_digits=8): |
| 159 |
return ''.join(random.choice(string.hexdigits) for _ in range(num_digits)) |
| 160 |
|
| 161 |
|
| 162 |
def disable_upload_callbacks(request, operation_name, **kwargs): |
| 163 |
if operation_name in ['PutObject', 'UploadPart'] and hasattr( |
| 164 |
request.body, 'disable_callback' |
| 165 |
): |
| 166 |
request.body.disable_callback() |
| 167 |
|
| 168 |
|
| 169 |
def enable_upload_callbacks(request, operation_name, **kwargs): |
| 170 |
if operation_name in ['PutObject', 'UploadPart'] and hasattr( |
| 171 |
request.body, 'enable_callback' |
| 172 |
): |
| 173 |
request.body.enable_callback() |
| 174 |
|
| 175 |
|
| 176 |
class QueueShutdownError(Exception): |
| 177 |
pass |
| 178 |
|
| 179 |
|
| 180 |
class ReadFileChunk: |
| 181 |
def __init__( |
| 182 |
self, |
| 183 |
fileobj, |
| 184 |
start_byte, |
| 185 |
chunk_size, |
| 186 |
full_file_size, |
| 187 |
callback=None, |
| 188 |
enable_callback=True, |
| 189 |
): |
| 190 |
""" |
| 191 |
|
| 192 |
Given a file object shown below: |
| 193 |
|
| 194 |
|___________________________________________________| |
| 195 |
0 | | full_file_size |
| 196 |
|----chunk_size---| |
| 197 |
start_byte |
| 198 |
|
| 199 |
:type fileobj: file |
| 200 |
:param fileobj: File like object |
| 201 |
|
| 202 |
:type start_byte: int |
| 203 |
:param start_byte: The first byte from which to start reading. |
| 204 |
|
| 205 |
:type chunk_size: int |
| 206 |
:param chunk_size: The max chunk size to read. Trying to read |
| 207 |
pass the end of the chunk size will behave like you've |
| 208 |
reached the end of the file. |
| 209 |
|
| 210 |
:type full_file_size: int |
| 211 |
:param full_file_size: The entire content length associated |
| 212 |
with ``fileobj``. |
| 213 |
|
| 214 |
:type callback: function(amount_read) |
| 215 |
:param callback: Called whenever data is read from this object. |
| 216 |
|
| 217 |
""" |
| 218 |
self._fileobj = fileobj |
| 219 |
self._start_byte = start_byte |
| 220 |
self._size = self._calculate_file_size( |
| 221 |
self._fileobj, |
| 222 |
requested_size=chunk_size, |
| 223 |
start_byte=start_byte, |
| 224 |
actual_file_size=full_file_size, |
| 225 |
) |
| 226 |
self._fileobj.seek(self._start_byte) |
| 227 |
self._amount_read = 0 |
| 228 |
self._callback = callback |
| 229 |
self._callback_enabled = enable_callback |
| 230 |
|
| 231 |
@classmethod |
| 232 |
def from_filename( |
| 233 |
cls, |
| 234 |
filename, |
| 235 |
start_byte, |
| 236 |
chunk_size, |
| 237 |
callback=None, |
| 238 |
enable_callback=True, |
| 239 |
): |
| 240 |
"""Convenience factory function to create from a filename. |
| 241 |
|
| 242 |
:type start_byte: int |
| 243 |
:param start_byte: The first byte from which to start reading. |
| 244 |
|
| 245 |
:type chunk_size: int |
| 246 |
:param chunk_size: The max chunk size to read. Trying to read |
| 247 |
pass the end of the chunk size will behave like you've |
| 248 |
reached the end of the file. |
| 249 |
|
| 250 |
:type full_file_size: int |
| 251 |
:param full_file_size: The entire content length associated |
| 252 |
with ``fileobj``. |
| 253 |
|
| 254 |
:type callback: function(amount_read) |
| 255 |
:param callback: Called whenever data is read from this object. |
| 256 |
|
| 257 |
:type enable_callback: bool |
| 258 |
:param enable_callback: Indicate whether to invoke callback |
| 259 |
during read() calls. |
| 260 |
|
| 261 |
:rtype: ``ReadFileChunk`` |
| 262 |
:return: A new instance of ``ReadFileChunk`` |
| 263 |
|
| 264 |
""" |
| 265 |
f = open(filename, 'rb') |
| 266 |
file_size = os.fstat(f.fileno()).st_size |
| 267 |
return cls( |
| 268 |
f, start_byte, chunk_size, file_size, callback, enable_callback |
| 269 |
) |
| 270 |
|
| 271 |
def _calculate_file_size( |
| 272 |
self, fileobj, requested_size, start_byte, actual_file_size |
| 273 |
): |
| 274 |
max_chunk_size = actual_file_size - start_byte |
| 275 |
return min(max_chunk_size, requested_size) |
| 276 |
|
| 277 |
def read(self, amount=None): |
| 278 |
if amount is None: |
| 279 |
amount_to_read = self._size - self._amount_read |
| 280 |
else: |
| 281 |
amount_to_read = min(self._size - self._amount_read, amount) |
| 282 |
data = self._fileobj.read(amount_to_read) |
| 283 |
self._amount_read += len(data) |
| 284 |
if self._callback is not None and self._callback_enabled: |
| 285 |
self._callback(len(data)) |
| 286 |
return data |
| 287 |
|
| 288 |
def enable_callback(self): |
| 289 |
self._callback_enabled = True |
| 290 |
|
| 291 |
def disable_callback(self): |
| 292 |
self._callback_enabled = False |
| 293 |
|
| 294 |
def seek(self, where): |
| 295 |
self._fileobj.seek(self._start_byte + where) |
| 296 |
if self._callback is not None and self._callback_enabled: |
| 297 |
# To also rewind the callback() for an accurate progress report |
| 298 |
self._callback(where - self._amount_read) |
| 299 |
self._amount_read = where |
| 300 |
|
| 301 |
def close(self): |
| 302 |
self._fileobj.close() |
| 303 |
|
| 304 |
def tell(self): |
| 305 |
return self._amount_read |
| 306 |
|
| 307 |
def __len__(self): |
| 308 |
# __len__ is defined because requests will try to determine the length |
| 309 |
# of the stream to set a content length. In the normal case |
| 310 |
# of the file it will just stat the file, but we need to change that |
| 311 |
# behavior. By providing a __len__, requests will use that instead |
| 312 |
# of stat'ing the file. |
| 313 |
return self._size |
| 314 |
|
| 315 |
def __enter__(self): |
| 316 |
return self |
| 317 |
|
| 318 |
def __exit__(self, *args, **kwargs): |
| 319 |
self.close() |
| 320 |
|
| 321 |
def __iter__(self): |
| 322 |
# This is a workaround for http://bugs.python.org/issue17575 |
| 323 |
# Basically httplib will try to iterate over the contents, even |
| 324 |
# if its a file like object. This wasn't noticed because we've |
| 325 |
# already exhausted the stream so iterating over the file immediately |
| 326 |
# stops, which is what we're simulating here. |
| 327 |
return iter([]) |
| 328 |
|
| 329 |
|
| 330 |
class StreamReaderProgress: |
| 331 |
"""Wrapper for a read only stream that adds progress callbacks.""" |
| 332 |
|
| 333 |
def __init__(self, stream, callback=None): |
| 334 |
self._stream = stream |
| 335 |
self._callback = callback |
| 336 |
|
| 337 |
def read(self, *args, **kwargs): |
| 338 |
value = self._stream.read(*args, **kwargs) |
| 339 |
if self._callback is not None: |
| 340 |
self._callback(len(value)) |
| 341 |
return value |
| 342 |
|
| 343 |
|
| 344 |
class OSUtils: |
| 345 |
def get_file_size(self, filename): |
| 346 |
return os.path.getsize(filename) |
| 347 |
|
| 348 |
def open_file_chunk_reader(self, filename, start_byte, size, callback): |
| 349 |
return ReadFileChunk.from_filename( |
| 350 |
filename, start_byte, size, callback, enable_callback=False |
| 351 |
) |
| 352 |
|
| 353 |
def open(self, filename, mode): |
| 354 |
return open(filename, mode) |
| 355 |
|
| 356 |
def remove_file(self, filename): |
| 357 |
"""Remove a file, noop if file does not exist.""" |
| 358 |
# Unlike os.remove, if the file does not exist, |
| 359 |
# then this method does nothing. |
| 360 |
try: |
| 361 |
os.remove(filename) |
| 362 |
except OSError: |
| 363 |
pass |
| 364 |
|
| 365 |
def rename_file(self, current_filename, new_filename): |
| 366 |
s3transfer.compat.rename_file(current_filename, new_filename) |
| 367 |
|
| 368 |
|
| 369 |
class MultipartUploader: |
| 370 |
# These are the extra_args that need to be forwarded onto |
| 371 |
# subsequent upload_parts. |
| 372 |
UPLOAD_PART_ARGS = [ |
| 373 |
'SSECustomerKey', |
| 374 |
'SSECustomerAlgorithm', |
| 375 |
'SSECustomerKeyMD5', |
| 376 |
'RequestPayer', |
| 377 |
] |
| 378 |
|
| 379 |
def __init__( |
| 380 |
self, |
| 381 |
client, |
| 382 |
config, |
| 383 |
osutil, |
| 384 |
executor_cls=concurrent.futures.ThreadPoolExecutor, |
| 385 |
): |
| 386 |
self._client = client |
| 387 |
self._config = config |
| 388 |
self._os = osutil |
| 389 |
self._executor_cls = executor_cls |
| 390 |
|
| 391 |
def _extra_upload_part_args(self, extra_args): |
| 392 |
# Only the args in UPLOAD_PART_ARGS actually need to be passed |
| 393 |
# onto the upload_part calls. |
| 394 |
upload_parts_args = {} |
| 395 |
for key, value in extra_args.items(): |
| 396 |
if key in self.UPLOAD_PART_ARGS: |
| 397 |
upload_parts_args[key] = value |
| 398 |
return upload_parts_args |
| 399 |
|
| 400 |
def upload_file(self, filename, bucket, key, callback, extra_args): |
| 401 |
response = self._client.create_multipart_upload( |
| 402 |
Bucket=bucket, Key=key, **extra_args |
| 403 |
) |
| 404 |
upload_id = response['UploadId'] |
| 405 |
try: |
| 406 |
parts = self._upload_parts( |
| 407 |
upload_id, filename, bucket, key, callback, extra_args |
| 408 |
) |
| 409 |
except Exception as e: |
| 410 |
logger.debug( |
| 411 |
"Exception raised while uploading parts, " |
| 412 |
"aborting multipart upload.", |
| 413 |
exc_info=True, |
| 414 |
) |
| 415 |
self._client.abort_multipart_upload( |
| 416 |
Bucket=bucket, Key=key, UploadId=upload_id |
| 417 |
) |
| 418 |
raise S3UploadFailedError( |
| 419 |
"Failed to upload {} to {}: {}".format( |
| 420 |
filename, '/'.join([bucket, key]), e |
| 421 |
) |
| 422 |
) |
| 423 |
self._client.complete_multipart_upload( |
| 424 |
Bucket=bucket, |
| 425 |
Key=key, |
| 426 |
UploadId=upload_id, |
| 427 |
MultipartUpload={'Parts': parts}, |
| 428 |
) |
| 429 |
|
| 430 |
def _upload_parts( |
| 431 |
self, upload_id, filename, bucket, key, callback, extra_args |
| 432 |
): |
| 433 |
upload_parts_extra_args = self._extra_upload_part_args(extra_args) |
| 434 |
parts = [] |
| 435 |
part_size = self._config.multipart_chunksize |
| 436 |
num_parts = int( |
| 437 |
math.ceil(self._os.get_file_size(filename) / float(part_size)) |
| 438 |
) |
| 439 |
max_workers = self._config.max_concurrency |
| 440 |
with self._executor_cls(max_workers=max_workers) as executor: |
| 441 |
upload_partial = functools.partial( |
| 442 |
self._upload_one_part, |
| 443 |
filename, |
| 444 |
bucket, |
| 445 |
key, |
| 446 |
upload_id, |
| 447 |
part_size, |
| 448 |
upload_parts_extra_args, |
| 449 |
callback, |
| 450 |
) |
| 451 |
for part in executor.map(upload_partial, range(1, num_parts + 1)): |
| 452 |
parts.append(part) |
| 453 |
return parts |
| 454 |
|
| 455 |
def _upload_one_part( |
| 456 |
self, |
| 457 |
filename, |
| 458 |
bucket, |
| 459 |
key, |
| 460 |
upload_id, |
| 461 |
part_size, |
| 462 |
extra_args, |
| 463 |
callback, |
| 464 |
part_number, |
| 465 |
): |
| 466 |
open_chunk_reader = self._os.open_file_chunk_reader |
| 467 |
with open_chunk_reader( |
| 468 |
filename, part_size * (part_number - 1), part_size, callback |
| 469 |
) as body: |
| 470 |
response = self._client.upload_part( |
| 471 |
Bucket=bucket, |
| 472 |
Key=key, |
| 473 |
UploadId=upload_id, |
| 474 |
PartNumber=part_number, |
| 475 |
Body=body, |
| 476 |
**extra_args, |
| 477 |
) |
| 478 |
etag = response['ETag'] |
| 479 |
return {'ETag': etag, 'PartNumber': part_number} |
| 480 |
|
| 481 |
|
| 482 |
class ShutdownQueue(queue.Queue): |
| 483 |
"""A queue implementation that can be shutdown. |
| 484 |
|
| 485 |
Shutting down a queue means that this class adds a |
| 486 |
trigger_shutdown method that will trigger all subsequent |
| 487 |
calls to put() to fail with a ``QueueShutdownError``. |
| 488 |
|
| 489 |
It purposefully deviates from queue.Queue, and is *not* meant |
| 490 |
to be a drop in replacement for ``queue.Queue``. |
| 491 |
|
| 492 |
""" |
| 493 |
|
| 494 |
def _init(self, maxsize): |
| 495 |
self._shutdown = False |
| 496 |
self._shutdown_lock = threading.Lock() |
| 497 |
# queue.Queue is an old style class so we don't use super(). |
| 498 |
return queue.Queue._init(self, maxsize) |
| 499 |
|
| 500 |
def trigger_shutdown(self): |
| 501 |
with self._shutdown_lock: |
| 502 |
self._shutdown = True |
| 503 |
logger.debug("The IO queue is now shutdown.") |
| 504 |
|
| 505 |
def put(self, item): |
| 506 |
# Note: this is not sufficient, it's still possible to deadlock! |
| 507 |
# Need to hook into the condition vars used by this class. |
| 508 |
with self._shutdown_lock: |
| 509 |
if self._shutdown: |
| 510 |
raise QueueShutdownError( |
| 511 |
"Cannot put item to queue when queue has been shutdown." |
| 512 |
) |
| 513 |
return queue.Queue.put(self, item) |
| 514 |
|
| 515 |
|
| 516 |
class MultipartDownloader: |
| 517 |
def __init__( |
| 518 |
self, |
| 519 |
client, |
| 520 |
config, |
| 521 |
osutil, |
| 522 |
executor_cls=concurrent.futures.ThreadPoolExecutor, |
| 523 |
): |
| 524 |
self._client = client |
| 525 |
self._config = config |
| 526 |
self._os = osutil |
| 527 |
self._executor_cls = executor_cls |
| 528 |
self._ioqueue = ShutdownQueue(self._config.max_io_queue) |
| 529 |
|
| 530 |
def download_file( |
| 531 |
self, bucket, key, filename, object_size, extra_args, callback=None |
| 532 |
): |
| 533 |
with self._executor_cls(max_workers=2) as controller: |
| 534 |
# 1 thread for the future that manages the uploading of files |
| 535 |
# 1 thread for the future that manages IO writes. |
| 536 |
download_parts_handler = functools.partial( |
| 537 |
self._download_file_as_future, |
| 538 |
bucket, |
| 539 |
key, |
| 540 |
filename, |
| 541 |
object_size, |
| 542 |
callback, |
| 543 |
) |
| 544 |
parts_future = controller.submit(download_parts_handler) |
| 545 |
|
| 546 |
io_writes_handler = functools.partial( |
| 547 |
self._perform_io_writes, filename |
| 548 |
) |
| 549 |
io_future = controller.submit(io_writes_handler) |
| 550 |
results = concurrent.futures.wait( |
| 551 |
[parts_future, io_future], |
| 552 |
return_when=concurrent.futures.FIRST_EXCEPTION, |
| 553 |
) |
| 554 |
self._process_future_results(results) |
| 555 |
|
| 556 |
def _process_future_results(self, futures): |
| 557 |
finished, unfinished = futures |
| 558 |
for future in finished: |
| 559 |
future.result() |
| 560 |
|
| 561 |
def _download_file_as_future( |
| 562 |
self, bucket, key, filename, object_size, callback |
| 563 |
): |
| 564 |
part_size = self._config.multipart_chunksize |
| 565 |
num_parts = int(math.ceil(object_size / float(part_size))) |
| 566 |
max_workers = self._config.max_concurrency |
| 567 |
download_partial = functools.partial( |
| 568 |
self._download_range, |
| 569 |
bucket, |
| 570 |
key, |
| 571 |
filename, |
| 572 |
part_size, |
| 573 |
num_parts, |
| 574 |
callback, |
| 575 |
) |
| 576 |
try: |
| 577 |
with self._executor_cls(max_workers=max_workers) as executor: |
| 578 |
list(executor.map(download_partial, range(num_parts))) |
| 579 |
finally: |
| 580 |
self._ioqueue.put(SHUTDOWN_SENTINEL) |
| 581 |
|
| 582 |
def _calculate_range_param(self, part_size, part_index, num_parts): |
| 583 |
start_range = part_index * part_size |
| 584 |
if part_index == num_parts - 1: |
| 585 |
end_range = '' |
| 586 |
else: |
| 587 |
end_range = start_range + part_size - 1 |
| 588 |
range_param = f'bytes={start_range}-{end_range}' |
| 589 |
return range_param |
| 590 |
|
| 591 |
def _download_range( |
| 592 |
self, bucket, key, filename, part_size, num_parts, callback, part_index |
| 593 |
): |
| 594 |
try: |
| 595 |
range_param = self._calculate_range_param( |
| 596 |
part_size, part_index, num_parts |
| 597 |
) |
| 598 |
|
| 599 |
max_attempts = self._config.num_download_attempts |
| 600 |
last_exception = None |
| 601 |
for i in range(max_attempts): |
| 602 |
try: |
| 603 |
logger.debug("Making get_object call.") |
| 604 |
response = self._client.get_object( |
| 605 |
Bucket=bucket, Key=key, Range=range_param |
| 606 |
) |
| 607 |
streaming_body = StreamReaderProgress( |
| 608 |
response['Body'], callback |
| 609 |
) |
| 610 |
buffer_size = 1024 * 16 |
| 611 |
current_index = part_size * part_index |
| 612 |
for chunk in iter( |
| 613 |
lambda: streaming_body.read(buffer_size), b'' |
| 614 |
): |
| 615 |
self._ioqueue.put((current_index, chunk)) |
| 616 |
current_index += len(chunk) |
| 617 |
return |
| 618 |
except ( |
| 619 |
socket.timeout, |
| 620 |
OSError, |
| 621 |
ReadTimeoutError, |
| 622 |
IncompleteReadError, |
| 623 |
ResponseStreamingError, |
| 624 |
) as e: |
| 625 |
logger.debug( |
| 626 |
"Retrying exception caught (%s), " |
| 627 |
"retrying request, (attempt %s / %s)", |
| 628 |
e, |
| 629 |
i, |
| 630 |
max_attempts, |
| 631 |
exc_info=True, |
| 632 |
) |
| 633 |
last_exception = e |
| 634 |
continue |
| 635 |
raise RetriesExceededError(last_exception) |
| 636 |
finally: |
| 637 |
logger.debug("EXITING _download_range for part: %s", part_index) |
| 638 |
|
| 639 |
def _perform_io_writes(self, filename): |
| 640 |
with self._os.open(filename, 'wb') as f: |
| 641 |
while True: |
| 642 |
task = self._ioqueue.get() |
| 643 |
if task is SHUTDOWN_SENTINEL: |
| 644 |
logger.debug( |
| 645 |
"Shutdown sentinel received in IO handler, " |
| 646 |
"shutting down IO handler." |
| 647 |
) |
| 648 |
return |
| 649 |
else: |
| 650 |
try: |
| 651 |
offset, data = task |
| 652 |
f.seek(offset) |
| 653 |
f.write(data) |
| 654 |
except Exception as e: |
| 655 |
logger.debug( |
| 656 |
"Caught exception in IO thread: %s", |
| 657 |
e, |
| 658 |
exc_info=True, |
| 659 |
) |
| 660 |
self._ioqueue.trigger_shutdown() |
| 661 |
raise |
| 662 |
|
| 663 |
|
| 664 |
class TransferConfig: |
| 665 |
def __init__( |
| 666 |
self, |
| 667 |
multipart_threshold=8 * MB, |
| 668 |
max_concurrency=10, |
| 669 |
multipart_chunksize=8 * MB, |
| 670 |
num_download_attempts=5, |
| 671 |
max_io_queue=100, |
| 672 |
): |
| 673 |
self.multipart_threshold = multipart_threshold |
| 674 |
self.max_concurrency = max_concurrency |
| 675 |
self.multipart_chunksize = multipart_chunksize |
| 676 |
self.num_download_attempts = num_download_attempts |
| 677 |
self.max_io_queue = max_io_queue |
| 678 |
|
| 679 |
|
| 680 |
class S3Transfer: |
| 681 |
ALLOWED_DOWNLOAD_ARGS = [ |
| 682 |
'VersionId', |
| 683 |
'SSECustomerAlgorithm', |
| 684 |
'SSECustomerKey', |
| 685 |
'SSECustomerKeyMD5', |
| 686 |
'RequestPayer', |
| 687 |
] |
| 688 |
|
| 689 |
ALLOWED_UPLOAD_ARGS = [ |
| 690 |
'ACL', |
| 691 |
'CacheControl', |
| 692 |
'ContentDisposition', |
| 693 |
'ContentEncoding', |
| 694 |
'ContentLanguage', |
| 695 |
'ContentType', |
| 696 |
'Expires', |
| 697 |
'GrantFullControl', |
| 698 |
'GrantRead', |
| 699 |
'GrantReadACP', |
| 700 |
'GrantWriteACL', |
| 701 |
'Metadata', |
| 702 |
'RequestPayer', |
| 703 |
'ServerSideEncryption', |
| 704 |
'StorageClass', |
| 705 |
'SSECustomerAlgorithm', |
| 706 |
'SSECustomerKey', |
| 707 |
'SSECustomerKeyMD5', |
| 708 |
'SSEKMSKeyId', |
| 709 |
'SSEKMSEncryptionContext', |
| 710 |
'Tagging', |
| 711 |
] |
| 712 |
|
| 713 |
def __init__(self, client, config=None, osutil=None): |
| 714 |
self._client = client |
| 715 |
self._client.meta.events.register( |
| 716 |
'before-call.s3.*', self._update_checksum_context |
| 717 |
) |
| 718 |
if config is None: |
| 719 |
config = TransferConfig() |
| 720 |
self._config = config |
| 721 |
if osutil is None: |
| 722 |
osutil = OSUtils() |
| 723 |
self._osutil = osutil |
| 724 |
|
| 725 |
def _update_checksum_context(self, params, **kwargs): |
| 726 |
request_context = params.get("context", {}) |
| 727 |
checksum_context = request_context.get("checksum", {}) |
| 728 |
if "request_algorithm" in checksum_context: |
| 729 |
# Force request checksum algorithm in the header if specified. |
| 730 |
checksum_context["request_algorithm"]["in"] = "header" |
| 731 |
|
| 732 |
def upload_file( |
| 733 |
self, filename, bucket, key, callback=None, extra_args=None |
| 734 |
): |
| 735 |
"""Upload a file to an S3 object. |
| 736 |
|
| 737 |
Variants have also been injected into S3 client, Bucket and Object. |
| 738 |
You don't have to use S3Transfer.upload_file() directly. |
| 739 |
""" |
| 740 |
if extra_args is None: |
| 741 |
extra_args = {} |
| 742 |
self._validate_all_known_args(extra_args, self.ALLOWED_UPLOAD_ARGS) |
| 743 |
events = self._client.meta.events |
| 744 |
events.register_first( |
| 745 |
'request-created.s3', |
| 746 |
disable_upload_callbacks, |
| 747 |
unique_id='s3upload-callback-disable', |
| 748 |
) |
| 749 |
events.register_last( |
| 750 |
'request-created.s3', |
| 751 |
enable_upload_callbacks, |
| 752 |
unique_id='s3upload-callback-enable', |
| 753 |
) |
| 754 |
if ( |
| 755 |
self._osutil.get_file_size(filename) |
| 756 |
>= self._config.multipart_threshold |
| 757 |
): |
| 758 |
self._multipart_upload(filename, bucket, key, callback, extra_args) |
| 759 |
else: |
| 760 |
self._put_object(filename, bucket, key, callback, extra_args) |
| 761 |
|
| 762 |
def _put_object(self, filename, bucket, key, callback, extra_args): |
| 763 |
# We're using open_file_chunk_reader so we can take advantage of the |
| 764 |
# progress callback functionality. |
| 765 |
open_chunk_reader = self._osutil.open_file_chunk_reader |
| 766 |
with open_chunk_reader( |
| 767 |
filename, |
| 768 |
0, |
| 769 |
self._osutil.get_file_size(filename), |
| 770 |
callback=callback, |
| 771 |
) as body: |
| 772 |
self._client.put_object( |
| 773 |
Bucket=bucket, Key=key, Body=body, **extra_args |
| 774 |
) |
| 775 |
|
| 776 |
def download_file( |
| 777 |
self, bucket, key, filename, extra_args=None, callback=None |
| 778 |
): |
| 779 |
"""Download an S3 object to a file. |
| 780 |
|
| 781 |
Variants have also been injected into S3 client, Bucket and Object. |
| 782 |
You don't have to use S3Transfer.download_file() directly. |
| 783 |
""" |
| 784 |
# This method will issue a ``head_object`` request to determine |
| 785 |
# the size of the S3 object. This is used to determine if the |
| 786 |
# object is downloaded in parallel. |
| 787 |
if extra_args is None: |
| 788 |
extra_args = {} |
| 789 |
self._validate_all_known_args(extra_args, self.ALLOWED_DOWNLOAD_ARGS) |
| 790 |
object_size = self._object_size(bucket, key, extra_args) |
| 791 |
temp_filename = filename + os.extsep + random_file_extension() |
| 792 |
try: |
| 793 |
self._download_file( |
| 794 |
bucket, key, temp_filename, object_size, extra_args, callback |
| 795 |
) |
| 796 |
except Exception: |
| 797 |
logger.debug( |
| 798 |
"Exception caught in download_file, removing partial file: %s", |
| 799 |
temp_filename, |
| 800 |
exc_info=True, |
| 801 |
) |
| 802 |
self._osutil.remove_file(temp_filename) |
| 803 |
raise |
| 804 |
else: |
| 805 |
self._osutil.rename_file(temp_filename, filename) |
| 806 |
|
| 807 |
def _download_file( |
| 808 |
self, bucket, key, filename, object_size, extra_args, callback |
| 809 |
): |
| 810 |
if object_size >= self._config.multipart_threshold: |
| 811 |
self._ranged_download( |
| 812 |
bucket, key, filename, object_size, extra_args, callback |
| 813 |
) |
| 814 |
else: |
| 815 |
self._get_object(bucket, key, filename, extra_args, callback) |
| 816 |
|
| 817 |
def _validate_all_known_args(self, actual, allowed): |
| 818 |
for kwarg in actual: |
| 819 |
if kwarg not in allowed: |
| 820 |
raise ValueError( |
| 821 |
f"Invalid extra_args key '{kwarg}', " |
| 822 |
f"must be one of: {', '.join(allowed)}" |
| 823 |
) |
| 824 |
|
| 825 |
def _ranged_download( |
| 826 |
self, bucket, key, filename, object_size, extra_args, callback |
| 827 |
): |
| 828 |
downloader = MultipartDownloader( |
| 829 |
self._client, self._config, self._osutil |
| 830 |
) |
| 831 |
downloader.download_file( |
| 832 |
bucket, key, filename, object_size, extra_args, callback |
| 833 |
) |
| 834 |
|
| 835 |
def _get_object(self, bucket, key, filename, extra_args, callback): |
| 836 |
# precondition: num_download_attempts > 0 |
| 837 |
max_attempts = self._config.num_download_attempts |
| 838 |
last_exception = None |
| 839 |
for i in range(max_attempts): |
| 840 |
try: |
| 841 |
return self._do_get_object( |
| 842 |
bucket, key, filename, extra_args, callback |
| 843 |
) |
| 844 |
except ( |
| 845 |
socket.timeout, |
| 846 |
OSError, |
| 847 |
ReadTimeoutError, |
| 848 |
IncompleteReadError, |
| 849 |
ResponseStreamingError, |
| 850 |
) as e: |
| 851 |
# TODO: we need a way to reset the callback if the |
| 852 |
# download failed. |
| 853 |
logger.debug( |
| 854 |
"Retrying exception caught (%s), " |
| 855 |
"retrying request, (attempt %s / %s)", |
| 856 |
e, |
| 857 |
i, |
| 858 |
max_attempts, |
| 859 |
exc_info=True, |
| 860 |
) |
| 861 |
last_exception = e |
| 862 |
continue |
| 863 |
raise RetriesExceededError(last_exception) |
| 864 |
|
| 865 |
def _do_get_object(self, bucket, key, filename, extra_args, callback): |
| 866 |
response = self._client.get_object( |
| 867 |
Bucket=bucket, Key=key, **extra_args |
| 868 |
) |
| 869 |
streaming_body = StreamReaderProgress(response['Body'], callback) |
| 870 |
with self._osutil.open(filename, 'wb') as f: |
| 871 |
for chunk in iter(lambda: streaming_body.read(8192), b''): |
| 872 |
f.write(chunk) |
| 873 |
|
| 874 |
def _object_size(self, bucket, key, extra_args): |
| 875 |
return self._client.head_object(Bucket=bucket, Key=key, **extra_args)[ |
| 876 |
'ContentLength' |
| 877 |
] |
| 878 |
|
| 879 |
def _multipart_upload(self, filename, bucket, key, callback, extra_args): |
| 880 |
uploader = MultipartUploader(self._client, self._config, self._osutil) |
| 881 |
uploader.upload_file(filename, bucket, key, callback, extra_args) |