Debian Codegraph @

scripts/performance/upload-file

raw
1 #!/usr/bin/env python
2 """
3 Uploads a file using s3transfer. You can also chose how type of file that
4 is uploaded (i.e. filename, seekable, nonseekable).
5
6 Usage
7 =====
8
9 NOTE: Make sure you run ``pip install -r requirements-dev.txt`` before running.
10
11 To upload a file::
12
13 ./upload-file --file-name myfilename --file-type filename \\
14 --s3-bucket mybucket --s3-key mykey
15
16 """
17
18 import argparse
19
20 from botocore.session import get_session
21
22 from s3transfer.manager import TransferManager
23
24
25 class NonSeekableReader:
26 """A wrapper to hide the ability to seek for a fileobj"""
27
28 def __init__(self, fileobj):
29 self._fileobj = fileobj
30
31 def read(self, amt=-1):
32 return self._fileobj.read(amt)
33
34
35 class Uploader:
36 def upload(self, args):
37 session = get_session()
38 client = session.create_client('s3')
39 file_type = args.file_type
40 if args.debug:
41 session.set_debug_logger('')
42 with TransferManager(client) as manager:
43 getattr(self, 'upload_' + file_type)(
44 manager, args.file_name, args.s3_bucket, args.s3_key
45 )
46
47 def upload_filename(self, manager, filename, bucket, s3_key):
48 manager.upload(filename, bucket, s3_key)
49
50 def upload_seekable(self, manager, filename, bucket, s3_key):
51 with open(filename, 'rb') as f:
52 future = manager.upload(f, bucket, s3_key)
53 future.result()
54
55 def upload_nonseekable(self, manager, filename, bucket, s3_key):
56 with open(filename, 'rb') as f:
57 future = manager.upload(NonSeekableReader(f), bucket, s3_key)
58 future.result()
59
60
61 def main():
62 parser = argparse.ArgumentParser(usage=__doc__)
63 parser.add_argument('--file-name', required=True, help='The name of file')
64 parser.add_argument(
65 '--file-type',
66 choices=['filename', 'seekable', 'nonseekable'],
67 required=True,
68 help='The way to represent the file when uploading',
69 )
70 parser.add_argument(
71 '--s3-bucket',
72 required=True,
73 help='The S3 bucket to upload the file to',
74 )
75 parser.add_argument('--s3-key', required=True, help='The key to upload to')
76 parser.add_argument(
77 '--debug',
78 action='store_true',
79 help='Whether to turn debugging on. This will get printed to stderr',
80 )
81 args = parser.parse_args()
82 Uploader().upload(args)
83
84
85 if __name__ == '__main__':
86 main()