Debian Codegraph @

scripts/performance/benchmark-download

raw
1 #!/usr/bin/env python
2 """
3 Benchmark the downloading of a file using s3transfer. You can also chose how
4 type of file that is downloaded (i.e. filename, seekable, nonseekable).
5
6 Usage
7 =====
8
9 NOTE: Make sure you run ``pip install -r requirements-dev.txt`` before running.
10
11 To benchmark with using a temporary file and key that is generated for you::
12
13 ./benchmark-download --file-size 10MB --file-type filename \\
14 --s3-bucket mybucket
15
16 To benchmark with your own s3 key:
17
18 ./benchmark-upload --existing-s3-key mykey --file-type filename \\
19 --s3-bucket mybucket
20
21 """
22
23 import argparse
24 import os
25 import shutil
26 import subprocess
27 import tempfile
28
29 from botocore.session import get_session
30
31 from s3transfer.manager import TransferManager
32
33 TEMP_FILE = 'temp'
34 TEMP_KEY = 'temp'
35 KB = 1024
36 SIZE_SUFFIX = {
37 'kb': 1024,
38 'mb': 1024**2,
39 'gb': 1024**3,
40 'tb': 1024**4,
41 'kib': 1024,
42 'mib': 1024**2,
43 'gib': 1024**3,
44 'tib': 1024**4,
45 }
46
47
48 def human_readable_to_bytes(value):
49 """Converts a human readable size to bytes.
50 :param value: A string such as "10MB". If a suffix is not included,
51 then the value is assumed to be an integer representing the size
52 in bytes.
53 :returns: The converted value in bytes as an integer
54 """
55 value = value.lower()
56 if value[-2:] == 'ib':
57 # Assume IEC suffix.
58 suffix = value[-3:].lower()
59 else:
60 suffix = value[-2:].lower()
61 has_size_identifier = len(value) >= 2 and suffix in SIZE_SUFFIX
62 if not has_size_identifier:
63 try:
64 return int(value)
65 except ValueError:
66 raise ValueError(f"Invalid size value: {value}")
67 else:
68 multiplier = SIZE_SUFFIX[suffix]
69 return int(value[: -len(suffix)]) * multiplier
70
71
72 def create_file(filename, file_size):
73 with open(filename, 'wb') as f:
74 for i in range(0, file_size, KB):
75 f.write(b'a' * i)
76
77
78 def benchmark_download(args):
79 # Create a temporary directory to use for scratch work.
80 tempdir = tempfile.mkdtemp()
81 temp_file = os.path.join(tempdir, TEMP_FILE)
82
83 if args.target_download:
84 temp_file = os.path.abspath(os.path.expanduser(args.target_download))
85 session = get_session()
86 client = session.create_client('s3')
87 s3_key = args.existing_s3_key
88 try:
89 # If an existing s3 key was not specified, then create a temporary
90 # file of that size for the user and upload it.
91 if not args.existing_s3_key:
92 # Create the temporary file.
93 create_file(temp_file, args.file_size)
94
95 # Create the temporary s3 key
96 s3_key = TEMP_KEY
97 upload_file(client, temp_file, args.s3_bucket)
98
99 download_file_script = (
100 f'./download-file --file-name {temp_file} --file-type {args.file_type} --s3-bucket {args.s3_bucket} '
101 f'--s3-key {s3_key}'
102 )
103 benchmark_args = ['./benchmark', download_file_script]
104 if args.output_file:
105 benchmark_args.extend(['--output-file', args.output_file])
106 subprocess.check_call(benchmark_args)
107 finally:
108 shutil.rmtree(tempdir)
109 if not args.existing_s3_key:
110 client.delete_object(Bucket=args.s3_bucket, Key=s3_key)
111
112
113 def upload_file(client, filename, bucket):
114 with TransferManager(client) as manager:
115 manager.upload(filename, bucket, TEMP_KEY)
116
117
118 def main():
119 parser = argparse.ArgumentParser()
120 file_group = parser.add_mutually_exclusive_group(required=True)
121 file_group.add_argument(
122 '--file-size',
123 type=human_readable_to_bytes,
124 help=(
125 'The size of the temporary file to create and then upload to s3. '
126 'You can also specify your own key with --existing-s3-key to '
127 'avoid going through this setup step.'
128 ),
129 )
130 parser.add_argument(
131 '--file-type',
132 choices=['filename', 'seekable', 'nonseekable'],
133 required=True,
134 help='The way to represent the file when downloading',
135 )
136 parser.add_argument(
137 '--s3-bucket',
138 required=True,
139 help='The S3 bucket to download the file to',
140 )
141 file_group.add_argument(
142 '--existing-s3-key',
143 help=(
144 'The existing s3 key to download from. You can also use '
145 '--file-size to create a temporary file and key to download from.'
146 ),
147 )
148 parser.add_argument(
149 '--target-download',
150 help=(
151 'The filename to download to. Note that this file will '
152 'always be cleaned up for you.'
153 ),
154 )
155 parser.add_argument(
156 '--output-file',
157 help=(
158 'The file to output the data collected to. The default '
159 'location performance.csv'
160 ),
161 )
162 args = parser.parse_args()
163 benchmark_download(args)
164
165
166 if __name__ == '__main__':
167 main()