Debian Codegraph @

scripts/performance/benchmark-upload

raw
1 #!/usr/bin/env python
2 """
3 Benchmark the uploading of a file using s3transfer. You can also chose how
4 type of file that is uploaded (i.e. filename, seekable, nonseekable).
5
6 Usage
7 =====
8
9 NOTE: Make sure you run ``pip install -r requirements-dev.txt`` before running.
10
11 To benchmark with using a temporary file that is generated for you::
12
13 ./benchmark-upload --file-size 10MB --file-type filename \\
14 --s3-bucket mybucket
15
16 To benchmark with your own local file::
17
18 ./benchmark-upload --source-file myfile --file-type filename \\
19 --s3-bucket mybucket
20
21 """
22
23 import argparse
24 import os
25 import shutil
26 import subprocess
27 import tempfile
28
29 from botocore.session import get_session
30
31 TEMP_FILE = 'temp'
32 TEMP_KEY = 'temp'
33 KB = 1024
34 SIZE_SUFFIX = {
35 'kb': 1024,
36 'mb': 1024**2,
37 'gb': 1024**3,
38 'tb': 1024**4,
39 'kib': 1024,
40 'mib': 1024**2,
41 'gib': 1024**3,
42 'tib': 1024**4,
43 }
44
45
46 def human_readable_to_bytes(value):
47 """Converts a human readable size to bytes.
48 :param value: A string such as "10MB". If a suffix is not included,
49 then the value is assumed to be an integer representing the size
50 in bytes.
51 :returns: The converted value in bytes as an integer
52 """
53 value = value.lower()
54 if value[-2:] == 'ib':
55 # Assume IEC suffix.
56 suffix = value[-3:].lower()
57 else:
58 suffix = value[-2:].lower()
59 has_size_identifier = len(value) >= 2 and suffix in SIZE_SUFFIX
60 if not has_size_identifier:
61 try:
62 return int(value)
63 except ValueError:
64 raise ValueError(f"Invalid size value: {value}")
65 else:
66 multiplier = SIZE_SUFFIX[suffix]
67 return int(value[: -len(suffix)]) * multiplier
68
69
70 def create_file(filename, file_size):
71 with open(filename, 'wb') as f:
72 for i in range(0, file_size, KB):
73 f.write(b'a' * i)
74
75
76 def benchmark_upload(args):
77 source_file = args.source_file
78 session = get_session()
79 client = session.create_client('s3')
80 tempdir = None
81 try:
82 # If a source file was not specified, then create a temporary file of
83 # that size for the user.
84 if not source_file:
85 tempdir = tempfile.mkdtemp()
86 source_file = os.path.join(tempdir, TEMP_FILE)
87 create_file(source_file, args.file_size)
88
89 upload_file_script = (
90 f'./upload-file --file-name {source_file} --file-type {args.file_type} --s3-bucket {args.s3_bucket} '
91 f'--s3-key {TEMP_KEY}'
92 )
93 benchmark_args = ['./benchmark', upload_file_script]
94 if args.output_file:
95 benchmark_args.extend(['--output-file', args.output_file])
96 subprocess.check_call(benchmark_args)
97 finally:
98 if tempdir:
99 shutil.rmtree(tempdir)
100 client.delete_object(Bucket=args.s3_bucket, Key=TEMP_KEY)
101
102
103 def main():
104 parser = argparse.ArgumentParser(usage=__doc__)
105 source_file_group = parser.add_mutually_exclusive_group(required=True)
106 source_file_group.add_argument(
107 '--source-file',
108 help=(
109 'The local file to upload. Note this is optional. You can also '
110 'use --file-size which will create a temporary file for you.'
111 ),
112 )
113 source_file_group.add_argument(
114 '--file-size',
115 type=human_readable_to_bytes,
116 help=(
117 'The size of the temporary file to create. You can also specify '
118 'your own file with --source-file'
119 ),
120 )
121 parser.add_argument(
122 '--file-type',
123 choices=['filename', 'seekable', 'nonseekable'],
124 required=True,
125 help='The way to represent the file when uploading',
126 )
127 parser.add_argument(
128 '--s3-bucket',
129 required=True,
130 help='The S3 bucket to upload the file to',
131 )
132 parser.add_argument(
133 '--output-file',
134 help=(
135 'The file to output the data collected to. The default '
136 'location performance.csv'
137 ),
138 )
139 args = parser.parse_args()
140 benchmark_upload(args)
141
142
143 if __name__ == '__main__':
144 main()