| 1 |
# Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. |
| 2 |
# |
| 3 |
# Licensed under the Apache License, Version 2.0 (the "License"). You |
| 4 |
# may not use this file except in compliance with the License. A copy of |
| 5 |
# the License is located at |
| 6 |
# |
| 7 |
# https://aws.amazon.com/apache2.0/ |
| 8 |
# |
| 9 |
# or in the "license" file accompanying this file. This file is |
| 10 |
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF |
| 11 |
# ANY KIND, either express or implied. See the License for the specific |
| 12 |
# language governing permissions and limitations under the License. |
| 13 |
"""Abstractions over S3's upload/download operations. |
| 14 |
|
| 15 |
This module provides high level abstractions for efficient |
| 16 |
uploads/downloads. It handles several things for the user: |
| 17 |
|
| 18 |
* Automatically switching to multipart transfers when |
| 19 |
a file is over a specific size threshold |
| 20 |
* Uploading/downloading a file in parallel |
| 21 |
* Progress callbacks to monitor transfers |
| 22 |
* Retries. While botocore handles retries for streaming uploads, |
| 23 |
it is not possible for it to handle retries for streaming |
| 24 |
downloads. This module handles retries for both cases so |
| 25 |
you don't need to implement any retry logic yourself. |
| 26 |
|
| 27 |
This module has a reasonable set of defaults. It also allows you |
| 28 |
to configure many aspects of the transfer process including: |
| 29 |
|
| 30 |
* Multipart threshold size |
| 31 |
* Max parallel downloads |
| 32 |
* Socket timeouts |
| 33 |
* Retry amounts |
| 34 |
|
| 35 |
There is no support for s3->s3 multipart copies at this |
| 36 |
time. |
| 37 |
|
| 38 |
|
| 39 |
.. _ref_s3transfer_usage: |
| 40 |
|
| 41 |
Usage |
| 42 |
===== |
| 43 |
|
| 44 |
The simplest way to use this module is: |
| 45 |
|
| 46 |
.. code-block:: python |
| 47 |
|
| 48 |
client = boto3.client('s3', 'us-west-2') |
| 49 |
transfer = S3Transfer(client) |
| 50 |
# Upload /tmp/myfile to s3://bucket/key |
| 51 |
transfer.upload_file('/tmp/myfile', 'bucket', 'key') |
| 52 |
|
| 53 |
# Download s3://bucket/key to /tmp/myfile |
| 54 |
transfer.download_file('bucket', 'key', '/tmp/myfile') |
| 55 |
|
| 56 |
The ``upload_file`` and ``download_file`` methods also accept |
| 57 |
``**kwargs``, which will be forwarded through to the corresponding |
| 58 |
client operation. Here are a few examples using ``upload_file``:: |
| 59 |
|
| 60 |
# Making the object public |
| 61 |
transfer.upload_file('/tmp/myfile', 'bucket', 'key', |
| 62 |
extra_args={'ACL': 'public-read'}) |
| 63 |
|
| 64 |
# Setting metadata |
| 65 |
transfer.upload_file('/tmp/myfile', 'bucket', 'key', |
| 66 |
extra_args={'Metadata': {'a': 'b', 'c': 'd'}}) |
| 67 |
|
| 68 |
# Setting content type |
| 69 |
transfer.upload_file('/tmp/myfile.json', 'bucket', 'key', |
| 70 |
extra_args={'ContentType': "application/json"}) |
| 71 |
|
| 72 |
|
| 73 |
The ``S3Transfer`` class also supports progress callbacks so you can |
| 74 |
provide transfer progress to users. Both the ``upload_file`` and |
| 75 |
``download_file`` methods take an optional ``callback`` parameter. |
| 76 |
Here's an example of how to print a simple progress percentage |
| 77 |
to the user: |
| 78 |
|
| 79 |
.. code-block:: python |
| 80 |
|
| 81 |
class ProgressPercentage(object): |
| 82 |
def __init__(self, filename): |
| 83 |
self._filename = filename |
| 84 |
self._size = float(os.path.getsize(filename)) |
| 85 |
self._seen_so_far = 0 |
| 86 |
self._lock = threading.Lock() |
| 87 |
|
| 88 |
def __call__(self, bytes_amount): |
| 89 |
# To simplify we'll assume this is hooked up |
| 90 |
# to a single filename. |
| 91 |
with self._lock: |
| 92 |
self._seen_so_far += bytes_amount |
| 93 |
percentage = (self._seen_so_far / self._size) * 100 |
| 94 |
sys.stdout.write( |
| 95 |
"\r%s %s / %s (%.2f%%)" % ( |
| 96 |
self._filename, self._seen_so_far, self._size, |
| 97 |
percentage)) |
| 98 |
sys.stdout.flush() |
| 99 |
|
| 100 |
|
| 101 |
transfer = S3Transfer(boto3.client('s3', 'us-west-2')) |
| 102 |
# Upload /tmp/myfile to s3://bucket/key and print upload progress. |
| 103 |
transfer.upload_file('/tmp/myfile', 'bucket', 'key', |
| 104 |
callback=ProgressPercentage('/tmp/myfile')) |
| 105 |
|
| 106 |
|
| 107 |
|
| 108 |
You can also provide a TransferConfig object to the S3Transfer |
| 109 |
object that gives you more fine grained control over the |
| 110 |
transfer. For example: |
| 111 |
|
| 112 |
.. code-block:: python |
| 113 |
|
| 114 |
client = boto3.client('s3', 'us-west-2') |
| 115 |
config = TransferConfig( |
| 116 |
multipart_threshold=8 * 1024 * 1024, |
| 117 |
max_concurrency=10, |
| 118 |
num_download_attempts=10, |
| 119 |
) |
| 120 |
transfer = S3Transfer(client, config) |
| 121 |
transfer.upload_file('/tmp/foo', 'bucket', 'key') |
| 122 |
|
| 123 |
|
| 124 |
""" |
| 125 |
|
| 126 |
import logging |
| 127 |
import threading |
| 128 |
from os import PathLike, fspath, getpid |
| 129 |
|
| 130 |
from botocore.compat import HAS_CRT |
| 131 |
from botocore.exceptions import ClientError |
| 132 |
from s3transfer.exceptions import ( |
| 133 |
RetriesExceededError as S3TransferRetriesExceededError, |
| 134 |
) |
| 135 |
from s3transfer.futures import NonThreadedExecutor |
| 136 |
from s3transfer.manager import TransferConfig as S3TransferConfig |
| 137 |
from s3transfer.manager import TransferManager |
| 138 |
from s3transfer.subscribers import BaseSubscriber |
| 139 |
from s3transfer.utils import OSUtils |
| 140 |
|
| 141 |
import boto3.s3.constants as constants |
| 142 |
from boto3.exceptions import RetriesExceededError, S3UploadFailedError |
| 143 |
|
| 144 |
if HAS_CRT: |
| 145 |
import awscrt.s3 |
| 146 |
|
| 147 |
from boto3.crt import create_crt_transfer_manager |
| 148 |
|
| 149 |
KB = 1024 |
| 150 |
MB = KB * KB |
| 151 |
|
| 152 |
logger = logging.getLogger(__name__) |
| 153 |
|
| 154 |
|
| 155 |
def create_transfer_manager(client, config, osutil=None): |
| 156 |
"""Creates a transfer manager based on configuration |
| 157 |
|
| 158 |
:type client: boto3.client |
| 159 |
:param client: The S3 client to use |
| 160 |
|
| 161 |
:type config: boto3.s3.transfer.TransferConfig |
| 162 |
:param config: The transfer config to use |
| 163 |
|
| 164 |
:type osutil: s3transfer.utils.OSUtils |
| 165 |
:param osutil: The os utility to use |
| 166 |
|
| 167 |
:rtype: s3transfer.manager.TransferManager |
| 168 |
:returns: A transfer manager based on parameters provided |
| 169 |
""" |
| 170 |
if _should_use_crt(config): |
| 171 |
crt_transfer_manager = create_crt_transfer_manager(client, config) |
| 172 |
if crt_transfer_manager is not None: |
| 173 |
logger.debug( |
| 174 |
f"Using CRT client. pid: {getpid()}, thread: {threading.get_ident()}" |
| 175 |
) |
| 176 |
return crt_transfer_manager |
| 177 |
|
| 178 |
# If we don't resolve something above, fallback to the default. |
| 179 |
logger.debug( |
| 180 |
f"Using default client. pid: {getpid()}, thread: {threading.get_ident()}" |
| 181 |
) |
| 182 |
return _create_default_transfer_manager(client, config, osutil) |
| 183 |
|
| 184 |
|
| 185 |
def _should_use_crt(config): |
| 186 |
# This feature requires awscrt>=0.19.18 |
| 187 |
if HAS_CRT and has_minimum_crt_version((0, 19, 18)): |
| 188 |
is_optimized_instance = awscrt.s3.is_optimized_for_system() |
| 189 |
else: |
| 190 |
is_optimized_instance = False |
| 191 |
pref_transfer_client = config.preferred_transfer_client.lower() |
| 192 |
|
| 193 |
if ( |
| 194 |
is_optimized_instance |
| 195 |
and pref_transfer_client == constants.AUTO_RESOLVE_TRANSFER_CLIENT |
| 196 |
): |
| 197 |
logger.debug( |
| 198 |
"Attempting to use CRTTransferManager. Config settings may be ignored." |
| 199 |
) |
| 200 |
return True |
| 201 |
|
| 202 |
logger.debug( |
| 203 |
"Opting out of CRT Transfer Manager. Preferred client: " |
| 204 |
f"{pref_transfer_client}, CRT available: {HAS_CRT}, " |
| 205 |
f"Instance Optimized: {is_optimized_instance}." |
| 206 |
) |
| 207 |
return False |
| 208 |
|
| 209 |
|
| 210 |
def has_minimum_crt_version(minimum_version): |
| 211 |
"""Not intended for use outside boto3.""" |
| 212 |
if not HAS_CRT: |
| 213 |
return False |
| 214 |
|
| 215 |
crt_version_str = awscrt.__version__ |
| 216 |
try: |
| 217 |
crt_version_ints = map(int, crt_version_str.split(".")) |
| 218 |
crt_version_tuple = tuple(crt_version_ints) |
| 219 |
except (TypeError, ValueError): |
| 220 |
return False |
| 221 |
|
| 222 |
return crt_version_tuple >= minimum_version |
| 223 |
|
| 224 |
|
| 225 |
def _create_default_transfer_manager(client, config, osutil): |
| 226 |
"""Create the default TransferManager implementation for s3transfer.""" |
| 227 |
executor_cls = None |
| 228 |
if not config.use_threads: |
| 229 |
executor_cls = NonThreadedExecutor |
| 230 |
return TransferManager(client, config, osutil, executor_cls) |
| 231 |
|
| 232 |
|
| 233 |
class TransferConfig(S3TransferConfig): |
| 234 |
ALIAS = { |
| 235 |
'max_concurrency': 'max_request_concurrency', |
| 236 |
'max_io_queue': 'max_io_queue_size', |
| 237 |
} |
| 238 |
|
| 239 |
def __init__( |
| 240 |
self, |
| 241 |
multipart_threshold=8 * MB, |
| 242 |
max_concurrency=10, |
| 243 |
multipart_chunksize=8 * MB, |
| 244 |
num_download_attempts=5, |
| 245 |
max_io_queue=100, |
| 246 |
io_chunksize=256 * KB, |
| 247 |
use_threads=True, |
| 248 |
max_bandwidth=None, |
| 249 |
preferred_transfer_client=constants.AUTO_RESOLVE_TRANSFER_CLIENT, |
| 250 |
): |
| 251 |
"""Configuration object for managed S3 transfers |
| 252 |
|
| 253 |
:param multipart_threshold: The transfer size threshold for which |
| 254 |
multipart uploads, downloads, and copies will automatically be |
| 255 |
triggered. |
| 256 |
|
| 257 |
:param max_concurrency: The maximum number of threads that will be |
| 258 |
making requests to perform a transfer. If ``use_threads`` is |
| 259 |
set to ``False``, the value provided is ignored as the transfer |
| 260 |
will only ever use the current thread. |
| 261 |
|
| 262 |
:param multipart_chunksize: The partition size of each part for a |
| 263 |
multipart transfer. |
| 264 |
|
| 265 |
:param num_download_attempts: The number of download attempts that |
| 266 |
will be retried upon errors with downloading an object in S3. |
| 267 |
Note that these retries account for errors that occur when |
| 268 |
streaming down the data from s3 (i.e. socket errors and read |
| 269 |
timeouts that occur after receiving an OK response from s3). |
| 270 |
Other retryable exceptions such as throttling errors and 5xx |
| 271 |
errors are already retried by botocore (this default is 5). This |
| 272 |
does not take into account the number of exceptions retried by |
| 273 |
botocore. |
| 274 |
|
| 275 |
:param max_io_queue: The maximum amount of read parts that can be |
| 276 |
queued in memory to be written for a download. The size of each |
| 277 |
of these read parts is at most the size of ``io_chunksize``. |
| 278 |
|
| 279 |
:param io_chunksize: The max size of each chunk in the io queue. |
| 280 |
Currently, this is size used when ``read`` is called on the |
| 281 |
downloaded stream as well. |
| 282 |
|
| 283 |
:param use_threads: If True, threads will be used when performing |
| 284 |
S3 transfers. If False, no threads will be used in |
| 285 |
performing transfers; all logic will be run in the current thread. |
| 286 |
|
| 287 |
:param max_bandwidth: The maximum bandwidth that will be consumed |
| 288 |
in uploading and downloading file content. The value is an integer |
| 289 |
in terms of bytes per second. |
| 290 |
|
| 291 |
:param preferred_transfer_client: String specifying preferred transfer |
| 292 |
client for transfer operations. |
| 293 |
|
| 294 |
Current supported settings are: |
| 295 |
* auto (default) - Use the CRTTransferManager when calls |
| 296 |
are made with supported environment and settings. |
| 297 |
* classic - Only use the origin S3TransferManager with |
| 298 |
requests. Disables possible CRT upgrade on requests. |
| 299 |
""" |
| 300 |
super().__init__( |
| 301 |
multipart_threshold=multipart_threshold, |
| 302 |
max_request_concurrency=max_concurrency, |
| 303 |
multipart_chunksize=multipart_chunksize, |
| 304 |
num_download_attempts=num_download_attempts, |
| 305 |
max_io_queue_size=max_io_queue, |
| 306 |
io_chunksize=io_chunksize, |
| 307 |
max_bandwidth=max_bandwidth, |
| 308 |
) |
| 309 |
# Some of the argument names are not the same as the inherited |
| 310 |
# S3TransferConfig so we add aliases so you can still access the |
| 311 |
# old version of the names. |
| 312 |
for alias in self.ALIAS: |
| 313 |
setattr(self, alias, getattr(self, self.ALIAS[alias])) |
| 314 |
self.use_threads = use_threads |
| 315 |
self.preferred_transfer_client = preferred_transfer_client |
| 316 |
|
| 317 |
def __setattr__(self, name, value): |
| 318 |
# If the alias name is used, make sure we set the name that it points |
| 319 |
# to as that is what actually is used in governing the TransferManager. |
| 320 |
if name in self.ALIAS: |
| 321 |
super().__setattr__(self.ALIAS[name], value) |
| 322 |
# Always set the value of the actual name provided. |
| 323 |
super().__setattr__(name, value) |
| 324 |
|
| 325 |
|
| 326 |
class S3Transfer: |
| 327 |
ALLOWED_DOWNLOAD_ARGS = TransferManager.ALLOWED_DOWNLOAD_ARGS |
| 328 |
ALLOWED_UPLOAD_ARGS = TransferManager.ALLOWED_UPLOAD_ARGS |
| 329 |
ALLOWED_COPY_ARGS = TransferManager.ALLOWED_COPY_ARGS |
| 330 |
|
| 331 |
def __init__(self, client=None, config=None, osutil=None, manager=None): |
| 332 |
if not client and not manager: |
| 333 |
raise ValueError( |
| 334 |
'Either a boto3.Client or s3transfer.manager.TransferManager ' |
| 335 |
'must be provided' |
| 336 |
) |
| 337 |
if manager and any([client, config, osutil]): |
| 338 |
raise ValueError( |
| 339 |
'Manager cannot be provided with client, config, ' |
| 340 |
'nor osutil. These parameters are mutually exclusive.' |
| 341 |
) |
| 342 |
if config is None: |
| 343 |
config = TransferConfig() |
| 344 |
if osutil is None: |
| 345 |
osutil = OSUtils() |
| 346 |
if manager: |
| 347 |
self._manager = manager |
| 348 |
else: |
| 349 |
self._manager = create_transfer_manager(client, config, osutil) |
| 350 |
|
| 351 |
def upload_file( |
| 352 |
self, filename, bucket, key, callback=None, extra_args=None |
| 353 |
): |
| 354 |
"""Upload a file to an S3 object. |
| 355 |
|
| 356 |
Variants have also been injected into S3 client, Bucket and Object. |
| 357 |
You don't have to use S3Transfer.upload_file() directly. |
| 358 |
|
| 359 |
.. seealso:: |
| 360 |
:py:meth:`S3.Client.upload_file` |
| 361 |
:py:meth:`S3.Client.upload_fileobj` |
| 362 |
""" |
| 363 |
if isinstance(filename, PathLike): |
| 364 |
filename = fspath(filename) |
| 365 |
if not isinstance(filename, str): |
| 366 |
raise ValueError('Filename must be a string or a path-like object') |
| 367 |
|
| 368 |
subscribers = self._get_subscribers(callback) |
| 369 |
future = self._manager.upload( |
| 370 |
filename, bucket, key, extra_args, subscribers |
| 371 |
) |
| 372 |
try: |
| 373 |
future.result() |
| 374 |
# If a client error was raised, add the backwards compatibility layer |
| 375 |
# that raises a S3UploadFailedError. These specific errors were only |
| 376 |
# ever thrown for upload_parts but now can be thrown for any related |
| 377 |
# client error. |
| 378 |
except ClientError as e: |
| 379 |
raise S3UploadFailedError( |
| 380 |
"Failed to upload {} to {}: {}".format( |
| 381 |
filename, '/'.join([bucket, key]), e |
| 382 |
) |
| 383 |
) |
| 384 |
|
| 385 |
def download_file( |
| 386 |
self, bucket, key, filename, extra_args=None, callback=None |
| 387 |
): |
| 388 |
"""Download an S3 object to a file. |
| 389 |
|
| 390 |
Variants have also been injected into S3 client, Bucket and Object. |
| 391 |
You don't have to use S3Transfer.download_file() directly. |
| 392 |
|
| 393 |
.. seealso:: |
| 394 |
:py:meth:`S3.Client.download_file` |
| 395 |
:py:meth:`S3.Client.download_fileobj` |
| 396 |
""" |
| 397 |
if isinstance(filename, PathLike): |
| 398 |
filename = fspath(filename) |
| 399 |
if not isinstance(filename, str): |
| 400 |
raise ValueError('Filename must be a string or a path-like object') |
| 401 |
|
| 402 |
subscribers = self._get_subscribers(callback) |
| 403 |
future = self._manager.download( |
| 404 |
bucket, key, filename, extra_args, subscribers |
| 405 |
) |
| 406 |
try: |
| 407 |
future.result() |
| 408 |
# This is for backwards compatibility where when retries are |
| 409 |
# exceeded we need to throw the same error from boto3 instead of |
| 410 |
# s3transfer's built in RetriesExceededError as current users are |
| 411 |
# catching the boto3 one instead of the s3transfer exception to do |
| 412 |
# their own retries. |
| 413 |
except S3TransferRetriesExceededError as e: |
| 414 |
raise RetriesExceededError(e.last_exception) |
| 415 |
|
| 416 |
def _get_subscribers(self, callback): |
| 417 |
if not callback: |
| 418 |
return None |
| 419 |
return [ProgressCallbackInvoker(callback)] |
| 420 |
|
| 421 |
def __enter__(self): |
| 422 |
return self |
| 423 |
|
| 424 |
def __exit__(self, *args): |
| 425 |
self._manager.__exit__(*args) |
| 426 |
|
| 427 |
|
| 428 |
class ProgressCallbackInvoker(BaseSubscriber): |
| 429 |
"""A back-compat wrapper to invoke a provided callback via a subscriber |
| 430 |
|
| 431 |
:param callback: A callable that takes a single positional argument for |
| 432 |
how many bytes were transferred. |
| 433 |
""" |
| 434 |
|
| 435 |
def __init__(self, callback): |
| 436 |
self._callback = callback |
| 437 |
|
| 438 |
def on_progress(self, bytes_transferred, **kwargs): |
| 439 |
self._callback(bytes_transferred) |