| 1 |
#!/usr/bin/env python |
| 2 |
""" |
| 3 |
Downloads using s3transfer.processpool.ProcessPoolDownloader |
| 4 |
|
| 5 |
Usage |
| 6 |
===== |
| 7 |
|
| 8 |
NOTE: Make sure you run ``pip install -r requirements-dev.txt`` before running. |
| 9 |
|
| 10 |
To download a file:: |
| 11 |
|
| 12 |
./proccesspool-download -f myfilename -b mybucket -k mykey |
| 13 |
|
| 14 |
To download a prefix recursively to a directory:: |
| 15 |
|
| 16 |
./proccesspool-download -d mydirname -b mybucket -p myprefix/ |
| 17 |
|
| 18 |
""" |
| 19 |
|
| 20 |
import argparse |
| 21 |
import os |
| 22 |
|
| 23 |
import botocore.session |
| 24 |
|
| 25 |
from s3transfer.processpool import ProcessPoolDownloader, ProcessTransferConfig |
| 26 |
|
| 27 |
MB = 1024 * 1024 |
| 28 |
|
| 29 |
|
| 30 |
def download(bucket, key, filename, num_processes, mb_chunksize): |
| 31 |
config = ProcessTransferConfig( |
| 32 |
multipart_chunksize=mb_chunksize * MB, |
| 33 |
max_request_processes=num_processes, |
| 34 |
) |
| 35 |
with ProcessPoolDownloader(config=config) as downloader: |
| 36 |
future = downloader.download_file( |
| 37 |
bucket=bucket, key=key, filename=filename |
| 38 |
) |
| 39 |
future.result() |
| 40 |
|
| 41 |
|
| 42 |
def recursive_download(bucket, prefix, dirname, num_processes, mb_chunksize): |
| 43 |
config = ProcessTransferConfig( |
| 44 |
multipart_chunksize=mb_chunksize * MB, |
| 45 |
max_request_processes=num_processes, |
| 46 |
) |
| 47 |
s3 = botocore.session.get_session().create_client('s3') |
| 48 |
with ProcessPoolDownloader(config=config) as downloader: |
| 49 |
paginator = s3.get_paginator('list_objects') |
| 50 |
for response in paginator.paginate(Bucket=bucket, Prefix=prefix): |
| 51 |
contents = response.get('Contents', []) |
| 52 |
for content in contents: |
| 53 |
key = content['Key'] |
| 54 |
filename = os.path.join(dirname, key[len(prefix) :]) |
| 55 |
parent_dirname = os.path.dirname(filename) |
| 56 |
if not os.path.exists(parent_dirname): |
| 57 |
os.makedirs(parent_dirname) |
| 58 |
# An expected size is provided so an additional HeadObject |
| 59 |
# does not need to be made for each of these objects that |
| 60 |
# get downloaded. |
| 61 |
downloader.download_file( |
| 62 |
bucket, |
| 63 |
key, |
| 64 |
filename=filename, |
| 65 |
expected_size=content['Size'], |
| 66 |
) |
| 67 |
|
| 68 |
|
| 69 |
def main(): |
| 70 |
parser = argparse.ArgumentParser(usage=__doc__) |
| 71 |
parser.add_argument( |
| 72 |
'-b', '--bucket', required=True, help='The S3 bucket to download from' |
| 73 |
) |
| 74 |
single_file_group = parser.add_argument_group('Single file downloads') |
| 75 |
single_file_group.add_argument( |
| 76 |
'-k', '--key', help='The key to download from' |
| 77 |
) |
| 78 |
single_file_group.add_argument( |
| 79 |
'-f', '--filename', help='The name of file to download to' |
| 80 |
) |
| 81 |
recursive_file_group = parser.add_argument_group( |
| 82 |
'Recursive file downloads' |
| 83 |
) |
| 84 |
recursive_file_group.add_argument( |
| 85 |
'-p', '--prefix', help='The prefix to download from' |
| 86 |
) |
| 87 |
recursive_file_group.add_argument( |
| 88 |
'-d', '--dirname', help='The directory to download to' |
| 89 |
) |
| 90 |
parser.add_argument( |
| 91 |
'-n', |
| 92 |
'--num-processes', |
| 93 |
type=int, |
| 94 |
default=10, |
| 95 |
help='The number of processes to run the download. 10 by default.', |
| 96 |
) |
| 97 |
parser.add_argument( |
| 98 |
'-c', |
| 99 |
'--mb-chunksize', |
| 100 |
type=int, |
| 101 |
default=8, |
| 102 |
help='The part size in MB to use for the download. 8 MB by default.', |
| 103 |
) |
| 104 |
args = parser.parse_args() |
| 105 |
if args.filename and args.key: |
| 106 |
download( |
| 107 |
args.bucket, |
| 108 |
args.key, |
| 109 |
args.filename, |
| 110 |
args.num_processes, |
| 111 |
args.mb_chunksize, |
| 112 |
) |
| 113 |
elif args.prefix and args.dirname: |
| 114 |
recursive_download( |
| 115 |
args.bucket, |
| 116 |
args.prefix, |
| 117 |
args.dirname, |
| 118 |
args.num_processes, |
| 119 |
args.mb_chunksize, |
| 120 |
) |
| 121 |
else: |
| 122 |
raise ValueError( |
| 123 |
'Either --key and --filename must be provided or ' |
| 124 |
'--prefix and --dirname must be provided.' |
| 125 |
) |
| 126 |
|
| 127 |
|
| 128 |
if __name__ == '__main__': |
| 129 |
main() |