Debian Codegraph @

scripts/performance/processpool-download

raw
1 #!/usr/bin/env python
2 """
3 Downloads using s3transfer.processpool.ProcessPoolDownloader
4
5 Usage
6 =====
7
8 NOTE: Make sure you run ``pip install -r requirements-dev.txt`` before running.
9
10 To download a file::
11
12 ./proccesspool-download -f myfilename -b mybucket -k mykey
13
14 To download a prefix recursively to a directory::
15
16 ./proccesspool-download -d mydirname -b mybucket -p myprefix/
17
18 """
19
20 import argparse
21 import os
22
23 import botocore.session
24
25 from s3transfer.processpool import ProcessPoolDownloader, ProcessTransferConfig
26
27 MB = 1024 * 1024
28
29
30 def download(bucket, key, filename, num_processes, mb_chunksize):
31 config = ProcessTransferConfig(
32 multipart_chunksize=mb_chunksize * MB,
33 max_request_processes=num_processes,
34 )
35 with ProcessPoolDownloader(config=config) as downloader:
36 future = downloader.download_file(
37 bucket=bucket, key=key, filename=filename
38 )
39 future.result()
40
41
42 def recursive_download(bucket, prefix, dirname, num_processes, mb_chunksize):
43 config = ProcessTransferConfig(
44 multipart_chunksize=mb_chunksize * MB,
45 max_request_processes=num_processes,
46 )
47 s3 = botocore.session.get_session().create_client('s3')
48 with ProcessPoolDownloader(config=config) as downloader:
49 paginator = s3.get_paginator('list_objects')
50 for response in paginator.paginate(Bucket=bucket, Prefix=prefix):
51 contents = response.get('Contents', [])
52 for content in contents:
53 key = content['Key']
54 filename = os.path.join(dirname, key[len(prefix) :])
55 parent_dirname = os.path.dirname(filename)
56 if not os.path.exists(parent_dirname):
57 os.makedirs(parent_dirname)
58 # An expected size is provided so an additional HeadObject
59 # does not need to be made for each of these objects that
60 # get downloaded.
61 downloader.download_file(
62 bucket,
63 key,
64 filename=filename,
65 expected_size=content['Size'],
66 )
67
68
69 def main():
70 parser = argparse.ArgumentParser(usage=__doc__)
71 parser.add_argument(
72 '-b', '--bucket', required=True, help='The S3 bucket to download from'
73 )
74 single_file_group = parser.add_argument_group('Single file downloads')
75 single_file_group.add_argument(
76 '-k', '--key', help='The key to download from'
77 )
78 single_file_group.add_argument(
79 '-f', '--filename', help='The name of file to download to'
80 )
81 recursive_file_group = parser.add_argument_group(
82 'Recursive file downloads'
83 )
84 recursive_file_group.add_argument(
85 '-p', '--prefix', help='The prefix to download from'
86 )
87 recursive_file_group.add_argument(
88 '-d', '--dirname', help='The directory to download to'
89 )
90 parser.add_argument(
91 '-n',
92 '--num-processes',
93 type=int,
94 default=10,
95 help='The number of processes to run the download. 10 by default.',
96 )
97 parser.add_argument(
98 '-c',
99 '--mb-chunksize',
100 type=int,
101 default=8,
102 help='The part size in MB to use for the download. 8 MB by default.',
103 )
104 args = parser.parse_args()
105 if args.filename and args.key:
106 download(
107 args.bucket,
108 args.key,
109 args.filename,
110 args.num_processes,
111 args.mb_chunksize,
112 )
113 elif args.prefix and args.dirname:
114 recursive_download(
115 args.bucket,
116 args.prefix,
117 args.dirname,
118 args.num_processes,
119 args.mb_chunksize,
120 )
121 else:
122 raise ValueError(
123 'Either --key and --filename must be provided or '
124 '--prefix and --dirname must be provided.'
125 )
126
127
128 if __name__ == '__main__':
129 main()