Debian Codegraph @

test/integration/helpers.py

raw
1 ######################################################################
2 #
3 # File: test/integration/helpers.py
4 #
5 # Copyright 2022 Backblaze Inc. All Rights Reserved.
6 #
7 # License https://www.backblaze.com/using_b2_code.html
8 #
9 ######################################################################
10 from __future__ import annotations
11
12 import dataclasses
13 import json
14 import logging
15 import os
16 import pathlib
17 import platform
18 import random
19 import re
20 import secrets
21 import shutil
22 import string
23 import subprocess
24 import sys
25 import threading
26 import time
27 import warnings
28 from dataclasses import dataclass
29 from datetime import datetime, timedelta
30 from os import environ, linesep
31 from pathlib import Path
32 from tempfile import mkdtemp, mktemp
33
34 import backoff
35 from b2sdk.v2 import (
36 ALL_CAPABILITIES,
37 BUCKET_NAME_CHARS_UNIQ,
38 BUCKET_NAME_LENGTH_RANGE,
39 NO_RETENTION_FILE_SETTING,
40 B2Api,
41 Bucket,
42 EncryptionAlgorithm,
43 EncryptionKey,
44 EncryptionMode,
45 EncryptionSetting,
46 InMemoryAccountInfo,
47 InMemoryCache,
48 LegalHold,
49 RetentionMode,
50 fix_windows_path_limit,
51 )
52 from b2sdk.v2.exception import (
53 BadRequest,
54 BucketIdNotFound,
55 FileNotPresent,
56 TooManyRequests,
57 v3BucketIdNotFound,
58 )
59
60 from b2._internal.console_tool import Command, current_time_millis
61
62 logger = logging.getLogger(__name__)
63
64 # A large period is set here to avoid issues related to clock skew or other time-related issues under CI
65 BUCKET_CLEANUP_PERIOD_MILLIS = timedelta(days=1).total_seconds() * 1000
66 ONE_HOUR_MILLIS = 60 * 60 * 1000
67 ONE_DAY_MILLIS = ONE_HOUR_MILLIS * 24
68
69 BUCKET_NAME_LENGTH = BUCKET_NAME_LENGTH_RANGE[1]
70 BUCKET_CREATED_AT_MILLIS = 'created_at_millis'
71
72 NODE_DESCRIPTION = f"{platform.node()}: {platform.platform()}"
73
74
75 def get_seed():
76 """
77 Get seed for random number generator.
78
79 GH Actions machines seem to offer a very limited entropy pool
80 """
81 return b''.join(
82 (
83 secrets.token_bytes(32),
84 str(time.time_ns()).encode(),
85 NODE_DESCRIPTION.encode(),
86 str(os.getpid()).encode(), # needed due to pytest-xdist
87 str(environ).encode('utf8', errors='ignore'
88 ), # especially helpful under GitHub (and similar) CI
89 )
90 )
91
92
93 RNG = random.Random(get_seed())
94 RNG_SEED = RNG.randint(0, 2 << 31)
95 RNG_COUNTER = 0
96
97 if sys.version_info < (3, 9):
98 RNG.randbytes = lambda n: RNG.getrandbits(n * 8).to_bytes(n, 'little')
99
100 SSE_NONE = EncryptionSetting(mode=EncryptionMode.NONE,)
101 SSE_B2_AES = EncryptionSetting(
102 mode=EncryptionMode.SSE_B2,
103 algorithm=EncryptionAlgorithm.AES256,
104 )
105 _SSE_KEY = RNG.randbytes(32)
106 SSE_C_AES = EncryptionSetting(
107 mode=EncryptionMode.SSE_C,
108 algorithm=EncryptionAlgorithm.AES256,
109 key=EncryptionKey(secret=_SSE_KEY, key_id='user-generated-key-id')
110 )
111 SSE_C_AES_2 = EncryptionSetting(
112 mode=EncryptionMode.SSE_C,
113 algorithm=EncryptionAlgorithm.AES256,
114 key=EncryptionKey(secret=_SSE_KEY, key_id='another-user-generated-key-id')
115 )
116
117
118 def random_token(length: int, chars=string.ascii_letters) -> str:
119 return ''.join(RNG.choice(chars) for _ in range(length))
120
121
122 def bucket_name_part(length: int) -> str:
123 assert length >= 1
124 global RNG_COUNTER
125 RNG_COUNTER += 1
126 name_part = random_token(length, BUCKET_NAME_CHARS_UNIQ)
127 logger.info('RNG_SEED: %s', RNG_SEED)
128 logger.info('RNG_COUNTER: %i, length: %i', RNG_COUNTER, length)
129 logger.info('name_part: %s', name_part)
130 return name_part
131
132
133 @dataclass
134 class Api:
135 account_id: str
136 application_key: str
137 realm: str
138 general_bucket_name_prefix: str
139 this_run_bucket_name_prefix: str
140
141 api: B2Api = None
142 bucket_name_log: list[str] = dataclasses.field(default_factory=list)
143
144 def __post_init__(self):
145 info = InMemoryAccountInfo()
146 cache = InMemoryCache()
147 self.api = B2Api(info, cache=cache)
148 self.api.authorize_account(self.realm, self.account_id, self.application_key)
149 assert BUCKET_NAME_LENGTH - len(
150 self.this_run_bucket_name_prefix
151 ) > 5, self.this_run_bucket_name_prefix
152
153 def new_bucket_name(self) -> str:
154 bucket_name = self.this_run_bucket_name_prefix + bucket_name_part(
155 BUCKET_NAME_LENGTH - len(self.this_run_bucket_name_prefix)
156 )
157 self.bucket_name_log.append(bucket_name)
158 return bucket_name
159
160 def new_bucket_info(self) -> dict:
161 return {
162 BUCKET_CREATED_AT_MILLIS: str(current_time_millis()),
163 "created_by": NODE_DESCRIPTION,
164 }
165
166 def create_bucket(self, bucket_type: str = 'allPublic', **kwargs) -> Bucket:
167 bucket_name = self.new_bucket_name()
168 return self.api.create_bucket(
169 bucket_name,
170 bucket_type=bucket_type,
171 bucket_info=self.new_bucket_info(),
172 **kwargs,
173 )
174
175 def _should_remove_bucket(self, bucket: Bucket) -> tuple[bool, str]:
176 if bucket.name.startswith(self.this_run_bucket_name_prefix):
177 return True, 'it is a bucket for this very run'
178 if bucket.name.startswith(self.general_bucket_name_prefix):
179 if BUCKET_CREATED_AT_MILLIS in bucket.bucket_info:
180 delete_older_than = current_time_millis() - BUCKET_CLEANUP_PERIOD_MILLIS
181 this_bucket_creation_time = int(bucket.bucket_info[BUCKET_CREATED_AT_MILLIS])
182 if this_bucket_creation_time < delete_older_than:
183 return True, f"this_bucket_creation_time={this_bucket_creation_time} < delete_older_than={delete_older_than}"
184 return False, f"this_bucket_creation_time={this_bucket_creation_time} >= delete_older_than={delete_older_than}"
185 else:
186 return True, 'undefined ' + BUCKET_CREATED_AT_MILLIS
187 return False, f'does not start with {self.general_bucket_name_prefix!r}'
188
189 def clean_buckets(self, quick=False):
190 # even with use_cache=True, if cache is empty API call will be made
191 buckets = self.api.list_buckets(use_cache=quick)
192 print('Total bucket count:', len(buckets))
193 remaining_buckets = []
194 for bucket in buckets:
195 should_remove, why = self._should_remove_bucket(bucket)
196 if not should_remove:
197 print(f'Skipping bucket removal {bucket.name!r} because {why}')
198 remaining_buckets.append(bucket)
199 continue
200
201 print('Trying to remove bucket:', bucket.name, 'because', why)
202 try:
203 self.clean_bucket(bucket)
204 except BucketIdNotFound:
205 print(f'It seems that bucket {bucket.name} has already been removed')
206 print('Total bucket count after cleanup:', len(remaining_buckets))
207 for bucket in remaining_buckets:
208 print(bucket)
209
210 @backoff.on_exception(
211 backoff.expo,
212 TooManyRequests,
213 max_tries=8,
214 )
215 def clean_bucket(self, bucket: Bucket | str):
216 if isinstance(bucket, str):
217 bucket = self.api.get_bucket_by_name(bucket)
218
219 # try optimistic bucket removal first, since it is completely free (as opposed to `ls` call)
220 try:
221 return self.api.delete_bucket(bucket)
222 except (BucketIdNotFound, v3BucketIdNotFound):
223 return # bucket was already removed
224 except BadRequest as exc:
225 assert exc.code == 'cannot_delete_non_empty_bucket'
226
227 files_leftover = False
228 file_versions = bucket.ls(latest_only=False, recursive=True)
229
230 for file_version_info, _ in file_versions:
231 if file_version_info.file_retention:
232 if file_version_info.file_retention.mode == RetentionMode.GOVERNANCE:
233 print('Removing retention from file version:', file_version_info.id_)
234 self.api.update_file_retention(
235 file_version_info.id_, file_version_info.file_name,
236 NO_RETENTION_FILE_SETTING, True
237 )
238 elif file_version_info.file_retention.mode == RetentionMode.COMPLIANCE:
239 if file_version_info.file_retention.retain_until > current_time_millis(): # yapf: disable
240 print(
241 f'File version: {file_version_info.id_} cannot be removed due to compliance mode retention'
242 )
243 files_leftover = True
244 continue
245 elif file_version_info.file_retention.mode == RetentionMode.NONE:
246 pass
247 else:
248 raise ValueError(
249 f'Unknown retention mode: {file_version_info.file_retention.mode}'
250 )
251 if file_version_info.legal_hold.is_on():
252 print('Removing legal hold from file version:', file_version_info.id_)
253 self.api.update_file_legal_hold(
254 file_version_info.id_, file_version_info.file_name, LegalHold.OFF
255 )
256 print('Removing file version:', file_version_info.id_)
257 try:
258 self.api.delete_file_version(file_version_info.id_, file_version_info.file_name)
259 except FileNotPresent:
260 print(
261 f'It seems that file version {file_version_info.id_} has already been removed'
262 )
263
264 if files_leftover:
265 print('Unable to remove bucket because some retained files remain')
266 else:
267 print('Removing bucket:', bucket.name)
268 try:
269 self.api.delete_bucket(bucket)
270 except BucketIdNotFound:
271 print(f'It seems that bucket {bucket.name} has already been removed')
272 print()
273
274 def count_and_print_buckets(self) -> int:
275 buckets = self.api.list_buckets()
276 count = len(buckets)
277 print(f'Total bucket count at {datetime.now()}: {count}')
278 for i, bucket in enumerate(buckets, start=1):
279 print(f'- {i}\t{bucket.name} [{bucket.id_}]')
280 return count
281
282
283 def print_text_indented(text):
284 """
285 Prints text that may include weird characters, indented four spaces.
286 """
287 for line in text.split(linesep):
288 Command._print_helper(sys.stdout, sys.stdout.encoding, ' ', repr(line)[1:-1])
289
290
291 def print_output(status, stdout, stderr):
292 print(' status:', status)
293 if stdout != '':
294 print(' stdout:')
295 print_text_indented(stdout)
296 if stderr != '':
297 print(' stderr:')
298 print_text_indented(stderr)
299 print()
300
301
302 def serialize_enc_settings(value):
303 if not isinstance(value, EncryptionSetting):
304 raise TypeError
305 return value.as_dict()
306
307
308 def print_json_indented(value):
309 """
310 Converts the value to JSON, then prints it.
311 """
312 print_text_indented(
313 json.dumps(
314 value, indent=4, sort_keys=True, ensure_ascii=True, default=serialize_enc_settings
315 )
316 )
317
318
319 def remove_warnings(text):
320 return linesep.join(line for line in text.split(linesep) if 'DeprecationWarning' not in line)
321
322
323 class StringReader:
324 def __init__(self):
325 self.string = None
326
327 def get_string(self):
328 return self.string
329
330 def read_from(self, f):
331 try:
332 self.string = f.read()
333 except Exception as e:
334 print(e)
335 self.string = str(e)
336
337
338 def should_equal(expected, actual):
339 print(' expected:')
340 print_json_indented(expected)
341 print(' actual:')
342 print_json_indented(actual)
343 assert expected == actual
344 print()
345
346
347 class CommandLine:
348
349 EXPECTED_STDERR_PATTERNS = [
350 re.compile(r'^Using https?://[\w.]+$'), # account auth
351 re.compile(r'.*B/s]$', re.DOTALL), # progress bar
352 re.compile(r'^\r?$'), # empty line
353 re.compile(
354 r'Encrypting file\(s\) with SSE-C without providing key id. '
355 r'Set B2_DESTINATION_SSE_C_KEY_ID to allow key identification'
356 ),
357 re.compile(
358 r'WARNING: Unable to print unicode. Encoding for stdout is: '
359 r'\'[a-zA-Z0-9]+\''
360 ), # windows-bundle tests on CI use cp1252
361 re.compile(r'Trying to print: .*'),
362 ]
363
364 def __init__(
365 self,
366 command,
367 account_id,
368 application_key,
369 realm,
370 bucket_name_prefix,
371 env_file_cmd_placeholder,
372 api_wrapper: Api,
373 b2_uri_args,
374 ):
375 self.command = command
376 self.account_id = account_id
377 self.application_key = application_key
378 self.realm = realm
379 self.bucket_name_prefix = bucket_name_prefix
380 self.env_file_cmd_placeholder = env_file_cmd_placeholder
381 self.api_wrapper = api_wrapper
382 self.b2_uri_args = b2_uri_args
383
384 def generate_bucket_name(self):
385 return self.api_wrapper.new_bucket_name()
386
387 def get_bucket_info_args(self) -> tuple[str, str]:
388 return '--bucket-info', json.dumps(self.api_wrapper.new_bucket_info(), ensure_ascii=True)
389
390 def run_command(self, args, additional_env: dict | None = None):
391 """
392 Runs the command with the given arguments, returns a tuple in form of
393 (succeeded, stdout)
394 """
395 status, stdout, stderr = self.execute(args, additional_env)
396 return status == 0 and stderr == '', stdout
397
398 def should_succeed(
399 self,
400 args: list[str] | None,
401 expected_pattern: str | None = None,
402 additional_env: dict | None = None,
403 expected_stderr_pattern: str | re.Pattern = None,
404 ) -> str:
405 """
406 Runs the command-line with the given arguments. Raises an exception
407 if there was an error; otherwise, returns the stdout of the command
408 as string.
409 """
410 status, stdout, stderr = self.execute(args, additional_env)
411 assert status == 0, f'FAILED with status {status}, stderr={stderr}'
412
413 if expected_stderr_pattern:
414 assert expected_stderr_pattern.search(stderr), \
415 f'stderr did not match pattern="{expected_stderr_pattern}", stderr="{stderr}"'
416 elif stderr != '':
417 for line in (s.strip() for s in stderr.split(os.linesep)):
418 assert any(p.match(line) for p in self.EXPECTED_STDERR_PATTERNS), \
419 f'Unexpected stderr line: {repr(line)}'
420
421 if expected_pattern is not None:
422 assert re.search(expected_pattern, stdout), \
423 f'did not match pattern="{expected_pattern}", stdout="{stdout}"'
424
425 return stdout.replace(os.linesep, '\n')
426
427 @classmethod
428 def prepare_env(self, additional_env: dict | None = None):
429 environ['PYTHONIOENCODING'] = 'utf-8'
430 env = environ.copy()
431 env.update(additional_env or {})
432 return env
433
434 def parse_command(self, env):
435 """
436 Split `self.command` into a list of strings. If necessary, dump the env vars to a tmp file and substitute
437 one the command's argument with that file's path.
438 """
439 command = self.command.split(' ')
440 if self.env_file_cmd_placeholder:
441 if any('\n' in var_value for var_value in env.values()):
442 raise ValueError(
443 'Env vars containing new line characters will break env file format'
444 )
445 env_file_path = mktemp()
446 pathlib.Path(env_file_path).write_text('\n'.join(f'{k}={v}' for k, v in env.items()))
447 command = [
448 (c if c != self.env_file_cmd_placeholder else env_file_path) for c in command
449 ]
450 return command
451
452 def execute(
453 self,
454 args: list[str | Path | int] | None = None,
455 additional_env: dict | None = None,
456 ):
457 """
458 :param cmd: a command to run
459 :param args: command's arguments
460 :param additional_env: environment variables to pass to the command, overwriting parent process ones
461 :return: (status, stdout, stderr)
462 """
463 # We'll run the b2 command-line by running the b2 module from
464 # the current directory or provided as parameter
465 env = self.prepare_env(additional_env)
466 command = self.parse_command(env)
467
468 args: list[str] = [str(arg) for arg in args] if args else []
469 command.extend(args)
470
471 print('Running:', ' '.join(command))
472
473 stdout = StringReader()
474 stderr = StringReader()
475
476 p = subprocess.Popen(
477 command,
478 stdin=subprocess.PIPE,
479 stdout=subprocess.PIPE,
480 stderr=subprocess.PIPE,
481 close_fds=platform.system() != 'Windows',
482 env=env,
483 )
484 p.stdin.close()
485 reader1 = threading.Thread(target=stdout.read_from, args=[p.stdout])
486 reader1.start()
487 reader2 = threading.Thread(target=stderr.read_from, args=[p.stderr])
488 reader2.start()
489 p.wait()
490 reader1.join()
491 reader2.join()
492
493 stdout_decoded = remove_warnings(stdout.get_string().decode('utf-8', errors='replace'))
494 stderr_decoded = remove_warnings(stderr.get_string().decode('utf-8', errors='replace'))
495
496 print_output(p.returncode, stdout_decoded, stderr_decoded)
497 return p.returncode, stdout_decoded, stderr_decoded
498
499 def should_succeed_json(self, args, additional_env: dict | None = None, **kwargs):
500 """
501 Runs the command-line with the given arguments. Raises an exception
502 if there was an error; otherwise, treats the stdout as JSON and returns
503 the data in it.
504 """
505 result = self.should_succeed(args, additional_env=additional_env, **kwargs)
506 try:
507 loaded_result = json.loads(result)
508 except json.JSONDecodeError:
509 raise ValueError(f'{result} is not a valid json')
510 return loaded_result
511
512 def should_fail(self, args, expected_pattern, additional_env: dict | None = None):
513 """
514 Runs the command-line with the given args, expecting the given pattern
515 to appear in stderr.
516 """
517 status, stdout, stderr = self.execute(args, additional_env)
518 assert status != 0, 'ERROR: should have failed'
519
520 assert re.search(expected_pattern, stdout + stderr), \
521 f'did not match pattern="{expected_pattern}", stdout="{stdout}", stderr="{stderr}"'
522
523 def reauthorize(self, check_key_capabilities=False):
524 """Clear and authorize again to the account."""
525 self.should_succeed(['clear-account'])
526 self.should_succeed(
527 [
528 'authorize-account', '--environment', self.realm, self.account_id,
529 self.application_key
530 ]
531 )
532 if check_key_capabilities:
533 auth_dict = self.should_succeed_json(['get-account-info'])
534 private_preview_caps = {
535 'readBucketNotifications',
536 'writeBucketNotifications',
537 }
538 missing_capabilities = set(ALL_CAPABILITIES) - {
539 'readBuckets', 'listAllBucketNames'
540 } - private_preview_caps - set(auth_dict['allowed']['capabilities'])
541 assert not missing_capabilities, f'it appears that the raw_api integration test is being run with a non-full key. Missing capabilities: {missing_capabilities}'
542
543 def list_file_versions(self, bucket_name):
544 return self.should_succeed_json(
545 ['ls', '--json', '--recursive', '--versions', *self.b2_uri_args(bucket_name)]
546 )
547
548 def cleanup_buckets(self, buckets: dict[str, dict | None]) -> None:
549 for bucket_name, bucket_dict in buckets.items():
550 self.cleanup_bucket(bucket_name, bucket_dict)
551
552 def cleanup_bucket(self, bucket_name: str, bucket_dict: dict | None = None) -> None:
553 """
554 Cleanup bucket
555
556 Since bucket was being handled by the tool, it is safe to assume it is cached in its cache and we don't
557 need to call C class API list_buckets endpoint to get it.
558 """
559 if not bucket_dict:
560 try:
561 bucket_dict = self.should_succeed_json(['get-bucket', bucket_name])
562 except (ValueError, AssertionError): # bucket doesn't exist
563 return
564
565 bucket = self.api_wrapper.api.BUCKET_CLASS(
566 api=self.api_wrapper.api,
567 id_=bucket_dict['bucketId'],
568 name=bucket_name,
569 )
570 self.api_wrapper.clean_bucket(bucket)
571
572
573 class TempDir:
574 def __init__(self):
575 warnings.warn(
576 'TempDir is deprecated; use pytest tmp_path fixture instead',
577 DeprecationWarning,
578 stacklevel=2,
579 )
580 self.dirpath = None
581
582 def __enter__(self):
583 self.dirpath = mkdtemp()
584 return Path(self.dirpath)
585
586 def __exit__(self, exc_type, exc_val, exc_tb):
587 shutil.rmtree(fix_windows_path_limit(self.dirpath))
588
589
590 def read_file(path: str | Path):
591 with open(path, 'rb') as f:
592 return f.read()
593
594
595 def write_file(path: str | Path, contents: bytes):
596 with open(path, 'wb') as f:
597 f.write(contents)
598
599
600 def file_mod_time_millis(path: str | Path) -> int:
601 return int(os.path.getmtime(path) * 1000)
602
603
604 def set_file_mod_time_millis(path: str | Path, time):
605 os.utime(path, (os.path.getatime(path), time / 1000))
606
607
608 def random_hex(length):
609 return ''.join(RNG.choice('0123456789abcdef') for _ in range(length))