| 1 |
# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. |
| 2 |
# |
| 3 |
# Licensed under the Apache License, Version 2.0 (the "License"). You |
| 4 |
# may not use this file except in compliance with the License. A copy of |
| 5 |
# the License is located at |
| 6 |
# |
| 7 |
# http://aws.amazon.com/apache2.0/ |
| 8 |
# |
| 9 |
# or in the "license" file accompanying this file. This file is |
| 10 |
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF |
| 11 |
# ANY KIND, either express or implied. See the License for the specific |
| 12 |
# language governing permissions and limitations under the License. |
| 13 |
"""Speeds up S3 throughput by using processes |
| 14 |
|
| 15 |
Getting Started |
| 16 |
=============== |
| 17 |
|
| 18 |
The :class:`ProcessPoolDownloader` can be used to download a single file by |
| 19 |
calling :meth:`ProcessPoolDownloader.download_file`: |
| 20 |
|
| 21 |
.. code:: python |
| 22 |
|
| 23 |
from s3transfer.processpool import ProcessPoolDownloader |
| 24 |
|
| 25 |
with ProcessPoolDownloader() as downloader: |
| 26 |
downloader.download_file('mybucket', 'mykey', 'myfile') |
| 27 |
|
| 28 |
|
| 29 |
This snippet downloads the S3 object located in the bucket ``mybucket`` at the |
| 30 |
key ``mykey`` to the local file ``myfile``. Any errors encountered during the |
| 31 |
transfer are not propagated. To determine if a transfer succeeded or |
| 32 |
failed, use the `Futures`_ interface. |
| 33 |
|
| 34 |
|
| 35 |
The :class:`ProcessPoolDownloader` can be used to download multiple files as |
| 36 |
well: |
| 37 |
|
| 38 |
.. code:: python |
| 39 |
|
| 40 |
from s3transfer.processpool import ProcessPoolDownloader |
| 41 |
|
| 42 |
with ProcessPoolDownloader() as downloader: |
| 43 |
downloader.download_file('mybucket', 'mykey', 'myfile') |
| 44 |
downloader.download_file('mybucket', 'myotherkey', 'myotherfile') |
| 45 |
|
| 46 |
|
| 47 |
When running this snippet, the downloading of ``mykey`` and ``myotherkey`` |
| 48 |
happen in parallel. The first ``download_file`` call does not block the |
| 49 |
second ``download_file`` call. The snippet blocks when exiting |
| 50 |
the context manager and blocks until both downloads are complete. |
| 51 |
|
| 52 |
Alternatively, the ``ProcessPoolDownloader`` can be instantiated |
| 53 |
and explicitly be shutdown using :meth:`ProcessPoolDownloader.shutdown`: |
| 54 |
|
| 55 |
.. code:: python |
| 56 |
|
| 57 |
from s3transfer.processpool import ProcessPoolDownloader |
| 58 |
|
| 59 |
downloader = ProcessPoolDownloader() |
| 60 |
downloader.download_file('mybucket', 'mykey', 'myfile') |
| 61 |
downloader.download_file('mybucket', 'myotherkey', 'myotherfile') |
| 62 |
downloader.shutdown() |
| 63 |
|
| 64 |
|
| 65 |
For this code snippet, the call to ``shutdown`` blocks until both |
| 66 |
downloads are complete. |
| 67 |
|
| 68 |
|
| 69 |
Additional Parameters |
| 70 |
===================== |
| 71 |
|
| 72 |
Additional parameters can be provided to the ``download_file`` method: |
| 73 |
|
| 74 |
* ``extra_args``: A dictionary containing any additional client arguments |
| 75 |
to include in the |
| 76 |
`GetObject <https://botocore.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#S3.Client.get_object>`_ |
| 77 |
API request. For example: |
| 78 |
|
| 79 |
.. code:: python |
| 80 |
|
| 81 |
from s3transfer.processpool import ProcessPoolDownloader |
| 82 |
|
| 83 |
with ProcessPoolDownloader() as downloader: |
| 84 |
downloader.download_file( |
| 85 |
'mybucket', 'mykey', 'myfile', |
| 86 |
extra_args={'VersionId': 'myversion'}) |
| 87 |
|
| 88 |
|
| 89 |
* ``expected_size``: By default, the downloader will make a HeadObject |
| 90 |
call to determine the size of the object. To opt-out of this additional |
| 91 |
API call, you can provide the size of the object in bytes: |
| 92 |
|
| 93 |
.. code:: python |
| 94 |
|
| 95 |
from s3transfer.processpool import ProcessPoolDownloader |
| 96 |
|
| 97 |
MB = 1024 * 1024 |
| 98 |
with ProcessPoolDownloader() as downloader: |
| 99 |
downloader.download_file( |
| 100 |
'mybucket', 'mykey', 'myfile', expected_size=2 * MB) |
| 101 |
|
| 102 |
|
| 103 |
Futures |
| 104 |
======= |
| 105 |
|
| 106 |
When ``download_file`` is called, it immediately returns a |
| 107 |
:class:`ProcessPoolTransferFuture`. The future can be used to poll the state |
| 108 |
of a particular transfer. To get the result of the download, |
| 109 |
call :meth:`ProcessPoolTransferFuture.result`. The method blocks |
| 110 |
until the transfer completes, whether it succeeds or fails. For example: |
| 111 |
|
| 112 |
.. code:: python |
| 113 |
|
| 114 |
from s3transfer.processpool import ProcessPoolDownloader |
| 115 |
|
| 116 |
with ProcessPoolDownloader() as downloader: |
| 117 |
future = downloader.download_file('mybucket', 'mykey', 'myfile') |
| 118 |
print(future.result()) |
| 119 |
|
| 120 |
|
| 121 |
If the download succeeds, the future returns ``None``: |
| 122 |
|
| 123 |
.. code:: python |
| 124 |
|
| 125 |
None |
| 126 |
|
| 127 |
|
| 128 |
If the download fails, the exception causing the failure is raised. For |
| 129 |
example, if ``mykey`` did not exist, the following error would be raised |
| 130 |
|
| 131 |
|
| 132 |
.. code:: python |
| 133 |
|
| 134 |
botocore.exceptions.ClientError: An error occurred (404) when calling the HeadObject operation: Not Found |
| 135 |
|
| 136 |
|
| 137 |
.. note:: |
| 138 |
|
| 139 |
:meth:`ProcessPoolTransferFuture.result` can only be called while the |
| 140 |
``ProcessPoolDownloader`` is running (e.g. before calling ``shutdown`` or |
| 141 |
inside the context manager). |
| 142 |
|
| 143 |
|
| 144 |
Process Pool Configuration |
| 145 |
========================== |
| 146 |
|
| 147 |
By default, the downloader has the following configuration options: |
| 148 |
|
| 149 |
* ``multipart_threshold``: The threshold size for performing ranged downloads |
| 150 |
in bytes. By default, ranged downloads happen for S3 objects that are |
| 151 |
greater than or equal to 8 MB in size. |
| 152 |
|
| 153 |
* ``multipart_chunksize``: The size of each ranged download in bytes. By |
| 154 |
default, the size of each ranged download is 8 MB. |
| 155 |
|
| 156 |
* ``max_request_processes``: The maximum number of processes used to download |
| 157 |
S3 objects. By default, the maximum is 10 processes. |
| 158 |
|
| 159 |
|
| 160 |
To change the default configuration, use the :class:`ProcessTransferConfig`: |
| 161 |
|
| 162 |
.. code:: python |
| 163 |
|
| 164 |
from s3transfer.processpool import ProcessPoolDownloader |
| 165 |
from s3transfer.processpool import ProcessTransferConfig |
| 166 |
|
| 167 |
config = ProcessTransferConfig( |
| 168 |
multipart_threshold=64 * 1024 * 1024, # 64 MB |
| 169 |
max_request_processes=50 |
| 170 |
) |
| 171 |
downloader = ProcessPoolDownloader(config=config) |
| 172 |
|
| 173 |
|
| 174 |
Client Configuration |
| 175 |
==================== |
| 176 |
|
| 177 |
The process pool downloader creates ``botocore`` clients on your behalf. In |
| 178 |
order to affect how the client is created, pass the keyword arguments |
| 179 |
that would have been used in the :meth:`botocore.Session.create_client` call: |
| 180 |
|
| 181 |
.. code:: python |
| 182 |
|
| 183 |
|
| 184 |
from s3transfer.processpool import ProcessPoolDownloader |
| 185 |
from s3transfer.processpool import ProcessTransferConfig |
| 186 |
|
| 187 |
downloader = ProcessPoolDownloader( |
| 188 |
client_kwargs={'region_name': 'us-west-2'}) |
| 189 |
|
| 190 |
|
| 191 |
This snippet ensures that all clients created by the ``ProcessPoolDownloader`` |
| 192 |
are using ``us-west-2`` as their region. |
| 193 |
|
| 194 |
""" |
| 195 |
|
| 196 |
import collections |
| 197 |
import contextlib |
| 198 |
import logging |
| 199 |
import multiprocessing |
| 200 |
import signal |
| 201 |
import threading |
| 202 |
from copy import deepcopy |
| 203 |
|
| 204 |
import botocore.session |
| 205 |
from botocore.config import Config |
| 206 |
|
| 207 |
from s3transfer.compat import MAXINT, BaseManager |
| 208 |
from s3transfer.constants import ALLOWED_DOWNLOAD_ARGS, MB, PROCESS_USER_AGENT |
| 209 |
from s3transfer.exceptions import CancelledError, RetriesExceededError |
| 210 |
from s3transfer.futures import BaseTransferFuture, BaseTransferMeta |
| 211 |
from s3transfer.utils import ( |
| 212 |
S3_RETRYABLE_DOWNLOAD_ERRORS, |
| 213 |
CallArgs, |
| 214 |
OSUtils, |
| 215 |
calculate_num_parts, |
| 216 |
calculate_range_parameter, |
| 217 |
create_nested_client, |
| 218 |
) |
| 219 |
|
| 220 |
logger = logging.getLogger(__name__) |
| 221 |
|
| 222 |
SHUTDOWN_SIGNAL = 'SHUTDOWN' |
| 223 |
|
| 224 |
# The DownloadFileRequest tuple is submitted from the ProcessPoolDownloader |
| 225 |
# to the GetObjectSubmitter in order for the submitter to begin submitting |
| 226 |
# GetObjectJobs to the GetObjectWorkers. |
| 227 |
DownloadFileRequest = collections.namedtuple( |
| 228 |
'DownloadFileRequest', |
| 229 |
[ |
| 230 |
'transfer_id', # The unique id for the transfer |
| 231 |
'bucket', # The bucket to download the object from |
| 232 |
'key', # The key to download the object from |
| 233 |
'filename', # The user-requested download location |
| 234 |
'extra_args', # Extra arguments to provide to client calls |
| 235 |
'expected_size', # The user-provided expected size of the download |
| 236 |
], |
| 237 |
) |
| 238 |
|
| 239 |
# The GetObjectJob tuple is submitted from the GetObjectSubmitter |
| 240 |
# to the GetObjectWorkers to download the file or parts of the file. |
| 241 |
GetObjectJob = collections.namedtuple( |
| 242 |
'GetObjectJob', |
| 243 |
[ |
| 244 |
'transfer_id', # The unique id for the transfer |
| 245 |
'bucket', # The bucket to download the object from |
| 246 |
'key', # The key to download the object from |
| 247 |
'temp_filename', # The temporary file to write the content to via |
| 248 |
# completed GetObject calls. |
| 249 |
'extra_args', # Extra arguments to provide to the GetObject call |
| 250 |
'offset', # The offset to write the content for the temp file. |
| 251 |
'filename', # The user-requested download location. The worker |
| 252 |
# of final GetObjectJob will move the file located at |
| 253 |
# temp_filename to the location of filename. |
| 254 |
], |
| 255 |
) |
| 256 |
|
| 257 |
|
| 258 |
@contextlib.contextmanager |
| 259 |
def ignore_ctrl_c(): |
| 260 |
original_handler = _add_ignore_handler_for_interrupts() |
| 261 |
yield |
| 262 |
signal.signal(signal.SIGINT, original_handler) |
| 263 |
|
| 264 |
|
| 265 |
def _add_ignore_handler_for_interrupts(): |
| 266 |
# Windows is unable to pickle signal.signal directly so it needs to |
| 267 |
# be wrapped in a function defined at the module level |
| 268 |
return signal.signal(signal.SIGINT, signal.SIG_IGN) |
| 269 |
|
| 270 |
|
| 271 |
class ProcessTransferConfig: |
| 272 |
def __init__( |
| 273 |
self, |
| 274 |
multipart_threshold=8 * MB, |
| 275 |
multipart_chunksize=8 * MB, |
| 276 |
max_request_processes=10, |
| 277 |
): |
| 278 |
"""Configuration for the ProcessPoolDownloader |
| 279 |
|
| 280 |
:param multipart_threshold: The threshold for which ranged downloads |
| 281 |
occur. |
| 282 |
|
| 283 |
:param multipart_chunksize: The chunk size of each ranged download. |
| 284 |
|
| 285 |
:param max_request_processes: The maximum number of processes that |
| 286 |
will be making S3 API transfer-related requests at a time. |
| 287 |
""" |
| 288 |
self.multipart_threshold = multipart_threshold |
| 289 |
self.multipart_chunksize = multipart_chunksize |
| 290 |
self.max_request_processes = max_request_processes |
| 291 |
|
| 292 |
|
| 293 |
class ProcessPoolDownloader: |
| 294 |
def __init__(self, client_kwargs=None, config=None): |
| 295 |
"""Downloads S3 objects using process pools |
| 296 |
|
| 297 |
:type client_kwargs: dict |
| 298 |
:param client_kwargs: The keyword arguments to provide when |
| 299 |
instantiating S3 clients. The arguments must match the keyword |
| 300 |
arguments provided to the |
| 301 |
`botocore.session.Session.create_client()` method. |
| 302 |
|
| 303 |
:type config: ProcessTransferConfig |
| 304 |
:param config: Configuration for the downloader |
| 305 |
""" |
| 306 |
if client_kwargs is None: |
| 307 |
client_kwargs = {} |
| 308 |
self._client_factory = ClientFactory(client_kwargs) |
| 309 |
|
| 310 |
self._transfer_config = config |
| 311 |
if config is None: |
| 312 |
self._transfer_config = ProcessTransferConfig() |
| 313 |
|
| 314 |
self._download_request_queue = multiprocessing.Queue(1000) |
| 315 |
self._worker_queue = multiprocessing.Queue(1000) |
| 316 |
self._osutil = OSUtils() |
| 317 |
|
| 318 |
self._started = False |
| 319 |
self._start_lock = threading.Lock() |
| 320 |
|
| 321 |
# These below are initialized in the start() method |
| 322 |
self._manager = None |
| 323 |
self._transfer_monitor = None |
| 324 |
self._submitter = None |
| 325 |
self._workers = [] |
| 326 |
|
| 327 |
def download_file( |
| 328 |
self, bucket, key, filename, extra_args=None, expected_size=None |
| 329 |
): |
| 330 |
"""Downloads the object's contents to a file |
| 331 |
|
| 332 |
:type bucket: str |
| 333 |
:param bucket: The name of the bucket to download from |
| 334 |
|
| 335 |
:type key: str |
| 336 |
:param key: The name of the key to download from |
| 337 |
|
| 338 |
:type filename: str |
| 339 |
:param filename: The name of a file to download to. |
| 340 |
|
| 341 |
:type extra_args: dict |
| 342 |
:param extra_args: Extra arguments that may be passed to the |
| 343 |
client operation |
| 344 |
|
| 345 |
:type expected_size: int |
| 346 |
:param expected_size: The expected size in bytes of the download. If |
| 347 |
provided, the downloader will not call HeadObject to determine the |
| 348 |
object's size and use the provided value instead. The size is |
| 349 |
needed to determine whether to do a multipart download. |
| 350 |
|
| 351 |
:rtype: s3transfer.futures.TransferFuture |
| 352 |
:returns: Transfer future representing the download |
| 353 |
""" |
| 354 |
self._start_if_needed() |
| 355 |
if extra_args is None: |
| 356 |
extra_args = {} |
| 357 |
self._validate_all_known_args(extra_args) |
| 358 |
transfer_id = self._transfer_monitor.notify_new_transfer() |
| 359 |
download_file_request = DownloadFileRequest( |
| 360 |
transfer_id=transfer_id, |
| 361 |
bucket=bucket, |
| 362 |
key=key, |
| 363 |
filename=filename, |
| 364 |
extra_args=extra_args, |
| 365 |
expected_size=expected_size, |
| 366 |
) |
| 367 |
logger.debug( |
| 368 |
'Submitting download file request: %s.', download_file_request |
| 369 |
) |
| 370 |
self._download_request_queue.put(download_file_request) |
| 371 |
call_args = CallArgs( |
| 372 |
bucket=bucket, |
| 373 |
key=key, |
| 374 |
filename=filename, |
| 375 |
extra_args=extra_args, |
| 376 |
expected_size=expected_size, |
| 377 |
) |
| 378 |
future = self._get_transfer_future(transfer_id, call_args) |
| 379 |
return future |
| 380 |
|
| 381 |
def shutdown(self): |
| 382 |
"""Shutdown the downloader |
| 383 |
|
| 384 |
It will wait till all downloads are complete before returning. |
| 385 |
""" |
| 386 |
self._shutdown_if_needed() |
| 387 |
|
| 388 |
def __enter__(self): |
| 389 |
return self |
| 390 |
|
| 391 |
def __exit__(self, exc_type, exc_value, *args): |
| 392 |
if isinstance(exc_value, KeyboardInterrupt): |
| 393 |
if self._transfer_monitor is not None: |
| 394 |
self._transfer_monitor.notify_cancel_all_in_progress() |
| 395 |
self.shutdown() |
| 396 |
|
| 397 |
def _start_if_needed(self): |
| 398 |
with self._start_lock: |
| 399 |
if not self._started: |
| 400 |
self._start() |
| 401 |
|
| 402 |
def _start(self): |
| 403 |
self._start_transfer_monitor_manager() |
| 404 |
self._start_submitter() |
| 405 |
self._start_get_object_workers() |
| 406 |
self._started = True |
| 407 |
|
| 408 |
def _validate_all_known_args(self, provided): |
| 409 |
for kwarg in provided: |
| 410 |
if kwarg not in ALLOWED_DOWNLOAD_ARGS: |
| 411 |
download_args = ', '.join(ALLOWED_DOWNLOAD_ARGS) |
| 412 |
raise ValueError( |
| 413 |
f"Invalid extra_args key '{kwarg}', " |
| 414 |
f"must be one of: {download_args}" |
| 415 |
) |
| 416 |
|
| 417 |
def _get_transfer_future(self, transfer_id, call_args): |
| 418 |
meta = ProcessPoolTransferMeta( |
| 419 |
call_args=call_args, transfer_id=transfer_id |
| 420 |
) |
| 421 |
future = ProcessPoolTransferFuture( |
| 422 |
monitor=self._transfer_monitor, meta=meta |
| 423 |
) |
| 424 |
return future |
| 425 |
|
| 426 |
def _start_transfer_monitor_manager(self): |
| 427 |
logger.debug('Starting the TransferMonitorManager.') |
| 428 |
self._manager = TransferMonitorManager() |
| 429 |
# We do not want Ctrl-C's to cause the manager to shutdown immediately |
| 430 |
# as worker processes will still need to communicate with it when they |
| 431 |
# are shutting down. So instead we ignore Ctrl-C and let the manager |
| 432 |
# be explicitly shutdown when shutting down the downloader. |
| 433 |
self._manager.start(_add_ignore_handler_for_interrupts) |
| 434 |
self._transfer_monitor = self._manager.TransferMonitor() |
| 435 |
|
| 436 |
def _start_submitter(self): |
| 437 |
logger.debug('Starting the GetObjectSubmitter.') |
| 438 |
self._submitter = GetObjectSubmitter( |
| 439 |
transfer_config=self._transfer_config, |
| 440 |
client_factory=self._client_factory, |
| 441 |
transfer_monitor=self._transfer_monitor, |
| 442 |
osutil=self._osutil, |
| 443 |
download_request_queue=self._download_request_queue, |
| 444 |
worker_queue=self._worker_queue, |
| 445 |
) |
| 446 |
self._submitter.start() |
| 447 |
|
| 448 |
def _start_get_object_workers(self): |
| 449 |
logger.debug( |
| 450 |
'Starting %s GetObjectWorkers.', |
| 451 |
self._transfer_config.max_request_processes, |
| 452 |
) |
| 453 |
for _ in range(self._transfer_config.max_request_processes): |
| 454 |
worker = GetObjectWorker( |
| 455 |
queue=self._worker_queue, |
| 456 |
client_factory=self._client_factory, |
| 457 |
transfer_monitor=self._transfer_monitor, |
| 458 |
osutil=self._osutil, |
| 459 |
) |
| 460 |
worker.start() |
| 461 |
self._workers.append(worker) |
| 462 |
|
| 463 |
def _shutdown_if_needed(self): |
| 464 |
with self._start_lock: |
| 465 |
if self._started: |
| 466 |
self._shutdown() |
| 467 |
|
| 468 |
def _shutdown(self): |
| 469 |
self._shutdown_submitter() |
| 470 |
self._shutdown_get_object_workers() |
| 471 |
self._shutdown_transfer_monitor_manager() |
| 472 |
self._started = False |
| 473 |
|
| 474 |
def _shutdown_transfer_monitor_manager(self): |
| 475 |
logger.debug('Shutting down the TransferMonitorManager.') |
| 476 |
self._manager.shutdown() |
| 477 |
|
| 478 |
def _shutdown_submitter(self): |
| 479 |
logger.debug('Shutting down the GetObjectSubmitter.') |
| 480 |
self._download_request_queue.put(SHUTDOWN_SIGNAL) |
| 481 |
self._submitter.join() |
| 482 |
|
| 483 |
def _shutdown_get_object_workers(self): |
| 484 |
logger.debug('Shutting down the GetObjectWorkers.') |
| 485 |
for _ in self._workers: |
| 486 |
self._worker_queue.put(SHUTDOWN_SIGNAL) |
| 487 |
for worker in self._workers: |
| 488 |
worker.join() |
| 489 |
|
| 490 |
|
| 491 |
class ProcessPoolTransferFuture(BaseTransferFuture): |
| 492 |
def __init__(self, monitor, meta): |
| 493 |
"""The future associated to a submitted process pool transfer request |
| 494 |
|
| 495 |
:type monitor: TransferMonitor |
| 496 |
:param monitor: The monitor associated to the process pool downloader |
| 497 |
|
| 498 |
:type meta: ProcessPoolTransferMeta |
| 499 |
:param meta: The metadata associated to the request. This object |
| 500 |
is visible to the requester. |
| 501 |
""" |
| 502 |
self._monitor = monitor |
| 503 |
self._meta = meta |
| 504 |
|
| 505 |
@property |
| 506 |
def meta(self): |
| 507 |
return self._meta |
| 508 |
|
| 509 |
def done(self): |
| 510 |
return self._monitor.is_done(self._meta.transfer_id) |
| 511 |
|
| 512 |
def result(self): |
| 513 |
try: |
| 514 |
return self._monitor.poll_for_result(self._meta.transfer_id) |
| 515 |
except KeyboardInterrupt: |
| 516 |
# For the multiprocessing Manager, a thread is given a single |
| 517 |
# connection to reuse in communicating between the thread in the |
| 518 |
# main process and the Manager's process. If a Ctrl-C happens when |
| 519 |
# polling for the result, it will make the main thread stop trying |
| 520 |
# to receive from the connection, but the Manager process will not |
| 521 |
# know that the main process has stopped trying to receive and |
| 522 |
# will not close the connection. As a result if another message is |
| 523 |
# sent to the Manager process, the listener in the Manager |
| 524 |
# processes will not process the new message as it is still trying |
| 525 |
# trying to process the previous message (that was Ctrl-C'd) and |
| 526 |
# thus cause the thread in the main process to hang on its send. |
| 527 |
# The only way around this is to create a new connection and send |
| 528 |
# messages from that new connection instead. |
| 529 |
self._monitor._connect() |
| 530 |
self.cancel() |
| 531 |
raise |
| 532 |
|
| 533 |
def cancel(self): |
| 534 |
self._monitor.notify_exception( |
| 535 |
self._meta.transfer_id, CancelledError() |
| 536 |
) |
| 537 |
|
| 538 |
|
| 539 |
class ProcessPoolTransferMeta(BaseTransferMeta): |
| 540 |
"""Holds metadata about the ProcessPoolTransferFuture""" |
| 541 |
|
| 542 |
def __init__(self, transfer_id, call_args): |
| 543 |
self._transfer_id = transfer_id |
| 544 |
self._call_args = call_args |
| 545 |
self._user_context = {} |
| 546 |
|
| 547 |
@property |
| 548 |
def call_args(self): |
| 549 |
return self._call_args |
| 550 |
|
| 551 |
@property |
| 552 |
def transfer_id(self): |
| 553 |
return self._transfer_id |
| 554 |
|
| 555 |
@property |
| 556 |
def user_context(self): |
| 557 |
return self._user_context |
| 558 |
|
| 559 |
|
| 560 |
class ClientFactory: |
| 561 |
def __init__(self, client_kwargs=None): |
| 562 |
"""Creates S3 clients for processes |
| 563 |
|
| 564 |
Botocore sessions and clients are not pickleable so they cannot be |
| 565 |
inherited across Process boundaries. Instead, they must be instantiated |
| 566 |
once a process is running. |
| 567 |
""" |
| 568 |
self._client_kwargs = client_kwargs |
| 569 |
if self._client_kwargs is None: |
| 570 |
self._client_kwargs = {} |
| 571 |
|
| 572 |
client_config = deepcopy(self._client_kwargs.get('config', Config())) |
| 573 |
if not client_config.user_agent_extra: |
| 574 |
client_config.user_agent_extra = PROCESS_USER_AGENT |
| 575 |
else: |
| 576 |
client_config.user_agent_extra += " " + PROCESS_USER_AGENT |
| 577 |
self._client_kwargs['config'] = client_config |
| 578 |
|
| 579 |
def create_client(self): |
| 580 |
"""Create a botocore S3 client""" |
| 581 |
session = botocore.session.Session() |
| 582 |
return create_nested_client(session, 's3', **self._client_kwargs) |
| 583 |
|
| 584 |
|
| 585 |
class TransferMonitor: |
| 586 |
def __init__(self): |
| 587 |
"""Monitors transfers for cross-process communication |
| 588 |
|
| 589 |
Notifications can be sent to the monitor and information can be |
| 590 |
retrieved from the monitor for a particular transfer. This abstraction |
| 591 |
is ran in a ``multiprocessing.managers.BaseManager`` in order to be |
| 592 |
shared across processes. |
| 593 |
""" |
| 594 |
# TODO: Add logic that removes the TransferState if the transfer is |
| 595 |
# marked as done and the reference to the future is no longer being |
| 596 |
# held onto. Without this logic, this dictionary will continue to |
| 597 |
# grow in size with no limit. |
| 598 |
self._transfer_states = {} |
| 599 |
self._id_count = 0 |
| 600 |
self._init_lock = threading.Lock() |
| 601 |
|
| 602 |
def notify_new_transfer(self): |
| 603 |
with self._init_lock: |
| 604 |
transfer_id = self._id_count |
| 605 |
self._transfer_states[transfer_id] = TransferState() |
| 606 |
self._id_count += 1 |
| 607 |
return transfer_id |
| 608 |
|
| 609 |
def is_done(self, transfer_id): |
| 610 |
"""Determine a particular transfer is complete |
| 611 |
|
| 612 |
:param transfer_id: Unique identifier for the transfer |
| 613 |
:return: True, if done. False, otherwise. |
| 614 |
""" |
| 615 |
return self._transfer_states[transfer_id].done |
| 616 |
|
| 617 |
def notify_done(self, transfer_id): |
| 618 |
"""Notify a particular transfer is complete |
| 619 |
|
| 620 |
:param transfer_id: Unique identifier for the transfer |
| 621 |
""" |
| 622 |
self._transfer_states[transfer_id].set_done() |
| 623 |
|
| 624 |
def poll_for_result(self, transfer_id): |
| 625 |
"""Poll for the result of a transfer |
| 626 |
|
| 627 |
:param transfer_id: Unique identifier for the transfer |
| 628 |
:return: If the transfer succeeded, it will return the result. If the |
| 629 |
transfer failed, it will raise the exception associated to the |
| 630 |
failure. |
| 631 |
""" |
| 632 |
self._transfer_states[transfer_id].wait_till_done() |
| 633 |
exception = self._transfer_states[transfer_id].exception |
| 634 |
if exception: |
| 635 |
raise exception |
| 636 |
return None |
| 637 |
|
| 638 |
def notify_exception(self, transfer_id, exception): |
| 639 |
"""Notify an exception was encountered for a transfer |
| 640 |
|
| 641 |
:param transfer_id: Unique identifier for the transfer |
| 642 |
:param exception: The exception encountered for that transfer |
| 643 |
""" |
| 644 |
# TODO: Not all exceptions are pickleable so if we are running |
| 645 |
# this in a multiprocessing.BaseManager we will want to |
| 646 |
# make sure to update this signature to ensure pickleability of the |
| 647 |
# arguments or have the ProxyObject do the serialization. |
| 648 |
self._transfer_states[transfer_id].exception = exception |
| 649 |
|
| 650 |
def notify_cancel_all_in_progress(self): |
| 651 |
for transfer_state in self._transfer_states.values(): |
| 652 |
if not transfer_state.done: |
| 653 |
transfer_state.exception = CancelledError() |
| 654 |
|
| 655 |
def get_exception(self, transfer_id): |
| 656 |
"""Retrieve the exception encountered for the transfer |
| 657 |
|
| 658 |
:param transfer_id: Unique identifier for the transfer |
| 659 |
:return: The exception encountered for that transfer. Otherwise |
| 660 |
if there were no exceptions, returns None. |
| 661 |
""" |
| 662 |
return self._transfer_states[transfer_id].exception |
| 663 |
|
| 664 |
def notify_expected_jobs_to_complete(self, transfer_id, num_jobs): |
| 665 |
"""Notify the amount of jobs expected for a transfer |
| 666 |
|
| 667 |
:param transfer_id: Unique identifier for the transfer |
| 668 |
:param num_jobs: The number of jobs to complete the transfer |
| 669 |
""" |
| 670 |
self._transfer_states[transfer_id].jobs_to_complete = num_jobs |
| 671 |
|
| 672 |
def notify_job_complete(self, transfer_id): |
| 673 |
"""Notify that a single job is completed for a transfer |
| 674 |
|
| 675 |
:param transfer_id: Unique identifier for the transfer |
| 676 |
:return: The number of jobs remaining to complete the transfer |
| 677 |
""" |
| 678 |
return self._transfer_states[transfer_id].decrement_jobs_to_complete() |
| 679 |
|
| 680 |
|
| 681 |
class TransferState: |
| 682 |
"""Represents the current state of an individual transfer""" |
| 683 |
|
| 684 |
# NOTE: Ideally the TransferState object would be used directly by the |
| 685 |
# various different abstractions in the ProcessPoolDownloader and remove |
| 686 |
# the need for the TransferMonitor. However, it would then impose the |
| 687 |
# constraint that two hops are required to make or get any changes in the |
| 688 |
# state of a transfer across processes: one hop to get a proxy object for |
| 689 |
# the TransferState and then a second hop to communicate calling the |
| 690 |
# specific TransferState method. |
| 691 |
def __init__(self): |
| 692 |
self._exception = None |
| 693 |
self._done_event = threading.Event() |
| 694 |
self._job_lock = threading.Lock() |
| 695 |
self._jobs_to_complete = 0 |
| 696 |
|
| 697 |
@property |
| 698 |
def done(self): |
| 699 |
return self._done_event.is_set() |
| 700 |
|
| 701 |
def set_done(self): |
| 702 |
self._done_event.set() |
| 703 |
|
| 704 |
def wait_till_done(self): |
| 705 |
self._done_event.wait(MAXINT) |
| 706 |
|
| 707 |
@property |
| 708 |
def exception(self): |
| 709 |
return self._exception |
| 710 |
|
| 711 |
@exception.setter |
| 712 |
def exception(self, val): |
| 713 |
self._exception = val |
| 714 |
|
| 715 |
@property |
| 716 |
def jobs_to_complete(self): |
| 717 |
return self._jobs_to_complete |
| 718 |
|
| 719 |
@jobs_to_complete.setter |
| 720 |
def jobs_to_complete(self, val): |
| 721 |
self._jobs_to_complete = val |
| 722 |
|
| 723 |
def decrement_jobs_to_complete(self): |
| 724 |
with self._job_lock: |
| 725 |
self._jobs_to_complete -= 1 |
| 726 |
return self._jobs_to_complete |
| 727 |
|
| 728 |
|
| 729 |
class TransferMonitorManager(BaseManager): |
| 730 |
pass |
| 731 |
|
| 732 |
|
| 733 |
TransferMonitorManager.register('TransferMonitor', TransferMonitor) |
| 734 |
|
| 735 |
|
| 736 |
class BaseS3TransferProcess(multiprocessing.Process): |
| 737 |
def __init__(self, client_factory): |
| 738 |
super().__init__() |
| 739 |
self._client_factory = client_factory |
| 740 |
self._client = None |
| 741 |
|
| 742 |
def run(self): |
| 743 |
# Clients are not pickleable so their instantiation cannot happen |
| 744 |
# in the __init__ for processes that are created under the |
| 745 |
# spawn method. |
| 746 |
self._client = self._client_factory.create_client() |
| 747 |
with ignore_ctrl_c(): |
| 748 |
# By default these processes are ran as child processes to the |
| 749 |
# main process. Any Ctrl-c encountered in the main process is |
| 750 |
# propagated to the child process and interrupt it at any time. |
| 751 |
# To avoid any potentially bad states caused from an interrupt |
| 752 |
# (i.e. a transfer failing to notify its done or making the |
| 753 |
# communication protocol become out of sync with the |
| 754 |
# TransferMonitor), we ignore all Ctrl-C's and allow the main |
| 755 |
# process to notify these child processes when to stop processing |
| 756 |
# jobs. |
| 757 |
self._do_run() |
| 758 |
|
| 759 |
def _do_run(self): |
| 760 |
raise NotImplementedError('_do_run()') |
| 761 |
|
| 762 |
|
| 763 |
class GetObjectSubmitter(BaseS3TransferProcess): |
| 764 |
def __init__( |
| 765 |
self, |
| 766 |
transfer_config, |
| 767 |
client_factory, |
| 768 |
transfer_monitor, |
| 769 |
osutil, |
| 770 |
download_request_queue, |
| 771 |
worker_queue, |
| 772 |
): |
| 773 |
"""Submit GetObjectJobs to fulfill a download file request |
| 774 |
|
| 775 |
:param transfer_config: Configuration for transfers. |
| 776 |
:param client_factory: ClientFactory for creating S3 clients. |
| 777 |
:param transfer_monitor: Monitor for notifying and retrieving state |
| 778 |
of transfer. |
| 779 |
:param osutil: OSUtils object to use for os-related behavior when |
| 780 |
performing the transfer. |
| 781 |
:param download_request_queue: Queue to retrieve download file |
| 782 |
requests. |
| 783 |
:param worker_queue: Queue to submit GetObjectJobs for workers |
| 784 |
to perform. |
| 785 |
""" |
| 786 |
super().__init__(client_factory) |
| 787 |
self._transfer_config = transfer_config |
| 788 |
self._transfer_monitor = transfer_monitor |
| 789 |
self._osutil = osutil |
| 790 |
self._download_request_queue = download_request_queue |
| 791 |
self._worker_queue = worker_queue |
| 792 |
|
| 793 |
def _do_run(self): |
| 794 |
while True: |
| 795 |
download_file_request = self._download_request_queue.get() |
| 796 |
if download_file_request == SHUTDOWN_SIGNAL: |
| 797 |
logger.debug('Submitter shutdown signal received.') |
| 798 |
return |
| 799 |
try: |
| 800 |
self._submit_get_object_jobs(download_file_request) |
| 801 |
except Exception as e: |
| 802 |
logger.debug( |
| 803 |
'Exception caught when submitting jobs for ' |
| 804 |
'download file request %s: %s', |
| 805 |
download_file_request, |
| 806 |
e, |
| 807 |
exc_info=True, |
| 808 |
) |
| 809 |
self._transfer_monitor.notify_exception( |
| 810 |
download_file_request.transfer_id, e |
| 811 |
) |
| 812 |
self._transfer_monitor.notify_done( |
| 813 |
download_file_request.transfer_id |
| 814 |
) |
| 815 |
|
| 816 |
def _submit_get_object_jobs(self, download_file_request): |
| 817 |
size = self._get_size(download_file_request) |
| 818 |
temp_filename = self._allocate_temp_file(download_file_request, size) |
| 819 |
if size < self._transfer_config.multipart_threshold: |
| 820 |
self._submit_single_get_object_job( |
| 821 |
download_file_request, temp_filename |
| 822 |
) |
| 823 |
else: |
| 824 |
self._submit_ranged_get_object_jobs( |
| 825 |
download_file_request, temp_filename, size |
| 826 |
) |
| 827 |
|
| 828 |
def _get_size(self, download_file_request): |
| 829 |
expected_size = download_file_request.expected_size |
| 830 |
if expected_size is None: |
| 831 |
expected_size = self._client.head_object( |
| 832 |
Bucket=download_file_request.bucket, |
| 833 |
Key=download_file_request.key, |
| 834 |
**download_file_request.extra_args, |
| 835 |
)['ContentLength'] |
| 836 |
return expected_size |
| 837 |
|
| 838 |
def _allocate_temp_file(self, download_file_request, size): |
| 839 |
temp_filename = self._osutil.get_temp_filename( |
| 840 |
download_file_request.filename |
| 841 |
) |
| 842 |
self._osutil.allocate(temp_filename, size) |
| 843 |
return temp_filename |
| 844 |
|
| 845 |
def _submit_single_get_object_job( |
| 846 |
self, download_file_request, temp_filename |
| 847 |
): |
| 848 |
self._notify_jobs_to_complete(download_file_request.transfer_id, 1) |
| 849 |
self._submit_get_object_job( |
| 850 |
transfer_id=download_file_request.transfer_id, |
| 851 |
bucket=download_file_request.bucket, |
| 852 |
key=download_file_request.key, |
| 853 |
temp_filename=temp_filename, |
| 854 |
offset=0, |
| 855 |
extra_args=download_file_request.extra_args, |
| 856 |
filename=download_file_request.filename, |
| 857 |
) |
| 858 |
|
| 859 |
def _submit_ranged_get_object_jobs( |
| 860 |
self, download_file_request, temp_filename, size |
| 861 |
): |
| 862 |
part_size = self._transfer_config.multipart_chunksize |
| 863 |
num_parts = calculate_num_parts(size, part_size) |
| 864 |
self._notify_jobs_to_complete( |
| 865 |
download_file_request.transfer_id, num_parts |
| 866 |
) |
| 867 |
for i in range(num_parts): |
| 868 |
offset = i * part_size |
| 869 |
range_parameter = calculate_range_parameter( |
| 870 |
part_size, i, num_parts |
| 871 |
) |
| 872 |
get_object_kwargs = {'Range': range_parameter} |
| 873 |
get_object_kwargs.update(download_file_request.extra_args) |
| 874 |
self._submit_get_object_job( |
| 875 |
transfer_id=download_file_request.transfer_id, |
| 876 |
bucket=download_file_request.bucket, |
| 877 |
key=download_file_request.key, |
| 878 |
temp_filename=temp_filename, |
| 879 |
offset=offset, |
| 880 |
extra_args=get_object_kwargs, |
| 881 |
filename=download_file_request.filename, |
| 882 |
) |
| 883 |
|
| 884 |
def _submit_get_object_job(self, **get_object_job_kwargs): |
| 885 |
self._worker_queue.put(GetObjectJob(**get_object_job_kwargs)) |
| 886 |
|
| 887 |
def _notify_jobs_to_complete(self, transfer_id, jobs_to_complete): |
| 888 |
logger.debug( |
| 889 |
'Notifying %s job(s) to complete for transfer_id %s.', |
| 890 |
jobs_to_complete, |
| 891 |
transfer_id, |
| 892 |
) |
| 893 |
self._transfer_monitor.notify_expected_jobs_to_complete( |
| 894 |
transfer_id, jobs_to_complete |
| 895 |
) |
| 896 |
|
| 897 |
|
| 898 |
class GetObjectWorker(BaseS3TransferProcess): |
| 899 |
# TODO: It may make sense to expose these class variables as configuration |
| 900 |
# options if users want to tweak them. |
| 901 |
_MAX_ATTEMPTS = 5 |
| 902 |
_IO_CHUNKSIZE = 2 * MB |
| 903 |
|
| 904 |
def __init__(self, queue, client_factory, transfer_monitor, osutil): |
| 905 |
"""Fulfills GetObjectJobs |
| 906 |
|
| 907 |
Downloads the S3 object, writes it to the specified file, and |
| 908 |
renames the file to its final location if it completes the final |
| 909 |
job for a particular transfer. |
| 910 |
|
| 911 |
:param queue: Queue for retrieving GetObjectJob's |
| 912 |
:param client_factory: ClientFactory for creating S3 clients |
| 913 |
:param transfer_monitor: Monitor for notifying |
| 914 |
:param osutil: OSUtils object to use for os-related behavior when |
| 915 |
performing the transfer. |
| 916 |
""" |
| 917 |
super().__init__(client_factory) |
| 918 |
self._queue = queue |
| 919 |
self._client_factory = client_factory |
| 920 |
self._transfer_monitor = transfer_monitor |
| 921 |
self._osutil = osutil |
| 922 |
|
| 923 |
def _do_run(self): |
| 924 |
while True: |
| 925 |
job = self._queue.get() |
| 926 |
if job == SHUTDOWN_SIGNAL: |
| 927 |
logger.debug('Worker shutdown signal received.') |
| 928 |
return |
| 929 |
if not self._transfer_monitor.get_exception(job.transfer_id): |
| 930 |
self._run_get_object_job(job) |
| 931 |
else: |
| 932 |
logger.debug( |
| 933 |
'Skipping get object job %s because there was a previous ' |
| 934 |
'exception.', |
| 935 |
job, |
| 936 |
) |
| 937 |
remaining = self._transfer_monitor.notify_job_complete( |
| 938 |
job.transfer_id |
| 939 |
) |
| 940 |
logger.debug( |
| 941 |
'%s jobs remaining for transfer_id %s.', |
| 942 |
remaining, |
| 943 |
job.transfer_id, |
| 944 |
) |
| 945 |
if not remaining: |
| 946 |
self._finalize_download( |
| 947 |
job.transfer_id, job.temp_filename, job.filename |
| 948 |
) |
| 949 |
|
| 950 |
def _run_get_object_job(self, job): |
| 951 |
try: |
| 952 |
self._do_get_object( |
| 953 |
bucket=job.bucket, |
| 954 |
key=job.key, |
| 955 |
temp_filename=job.temp_filename, |
| 956 |
extra_args=job.extra_args, |
| 957 |
offset=job.offset, |
| 958 |
) |
| 959 |
except Exception as e: |
| 960 |
logger.debug( |
| 961 |
'Exception caught when downloading object for ' |
| 962 |
'get object job %s: %s', |
| 963 |
job, |
| 964 |
e, |
| 965 |
exc_info=True, |
| 966 |
) |
| 967 |
self._transfer_monitor.notify_exception(job.transfer_id, e) |
| 968 |
|
| 969 |
def _do_get_object(self, bucket, key, extra_args, temp_filename, offset): |
| 970 |
last_exception = None |
| 971 |
for i in range(self._MAX_ATTEMPTS): |
| 972 |
try: |
| 973 |
response = self._client.get_object( |
| 974 |
Bucket=bucket, Key=key, **extra_args |
| 975 |
) |
| 976 |
self._write_to_file(temp_filename, offset, response['Body']) |
| 977 |
return |
| 978 |
except S3_RETRYABLE_DOWNLOAD_ERRORS as e: |
| 979 |
logger.debug( |
| 980 |
'Retrying exception caught (%s), ' |
| 981 |
'retrying request, (attempt %s / %s)', |
| 982 |
e, |
| 983 |
i + 1, |
| 984 |
self._MAX_ATTEMPTS, |
| 985 |
exc_info=True, |
| 986 |
) |
| 987 |
last_exception = e |
| 988 |
raise RetriesExceededError(last_exception) |
| 989 |
|
| 990 |
def _write_to_file(self, filename, offset, body): |
| 991 |
with open(filename, 'rb+') as f: |
| 992 |
f.seek(offset) |
| 993 |
chunks = iter(lambda: body.read(self._IO_CHUNKSIZE), b'') |
| 994 |
for chunk in chunks: |
| 995 |
f.write(chunk) |
| 996 |
|
| 997 |
def _finalize_download(self, transfer_id, temp_filename, filename): |
| 998 |
if self._transfer_monitor.get_exception(transfer_id): |
| 999 |
self._osutil.remove_file(temp_filename) |
| 1000 |
else: |
| 1001 |
self._do_file_rename(transfer_id, temp_filename, filename) |
| 1002 |
self._transfer_monitor.notify_done(transfer_id) |
| 1003 |
|
| 1004 |
def _do_file_rename(self, transfer_id, temp_filename, filename): |
| 1005 |
try: |
| 1006 |
self._osutil.rename_file(temp_filename, filename) |
| 1007 |
except Exception as e: |
| 1008 |
self._transfer_monitor.notify_exception(transfer_id, e) |
| 1009 |
self._osutil.remove_file(temp_filename) |