Debian Codegraph @

scripts/performance/download-file

raw
1 #!/usr/bin/env python
2 """
3 Downloads a file using s3transfer. You can also chose how type of file that
4 is downloaded (i.e. filename, seekable, nonseekable).
5
6 Usage
7 =====
8
9 NOTE: Make sure you run ``pip install -r requirements-dev.txt`` before running.
10
11 To download a file::
12
13 ./download-file --file-name myfilename --file-type filename \\
14 --s3-bucket mybucket --s3-key mykey
15
16 """
17
18 import argparse
19
20 from botocore.session import get_session
21
22 from s3transfer.manager import TransferManager
23
24
25 class NonSeekableWriter:
26 """A wrapper to hide the ability to seek for a fileobj"""
27
28 def __init__(self, fileobj):
29 self._fileobj = fileobj
30
31 def write(self, b):
32 return self._fileobj.write(b)
33
34
35 class Downloader:
36 def download(self, args):
37 session = get_session()
38 client = session.create_client('s3')
39 file_type = args.file_type
40 if args.debug:
41 session.set_debug_logger('')
42 with TransferManager(client) as manager:
43 getattr(self, 'download_' + file_type)(
44 manager, args.file_name, args.s3_bucket, args.s3_key
45 )
46
47 def download_filename(self, manager, filename, bucket, s3_key):
48 manager.download(bucket, s3_key, filename)
49
50 def download_seekable(self, manager, filename, bucket, s3_key):
51 with open(filename, 'wb') as f:
52 future = manager.download(bucket, s3_key, f)
53 future.result()
54
55 def download_nonseekable(self, manager, filename, bucket, s3_key):
56 with open(filename, 'wb') as f:
57 future = manager.download(bucket, s3_key, NonSeekableWriter(f))
58 future.result()
59
60
61 def main():
62 parser = argparse.ArgumentParser(usage=__doc__)
63 parser.add_argument('--file-name', required=True, help='The name of file')
64 parser.add_argument(
65 '--file-type',
66 choices=['filename', 'seekable', 'nonseekable'],
67 required=True,
68 help='The way to represent the file when downloading',
69 )
70 parser.add_argument(
71 '--s3-bucket',
72 required=True,
73 help='The S3 bucket to download the file to',
74 )
75 parser.add_argument(
76 '--s3-key', required=True, help='The key to download to'
77 )
78 parser.add_argument(
79 '--debug',
80 action='store_true',
81 help='Whether to turn debugging on. This will get printed to stderr',
82 )
83 args = parser.parse_args()
84 Downloader().download(args)
85
86
87 if __name__ == '__main__':
88 main()