Debian Codegraph @

test/integration/conftest.py

raw
1 ######################################################################
2 #
3 # File: test/integration/conftest.py
4 #
5 # Copyright 2020 Backblaze Inc. All Rights Reserved.
6 #
7 # License https://www.backblaze.com/using_b2_code.html
8 #
9 ######################################################################
10 from __future__ import annotations
11
12 import logging
13 import os
14 import pathlib
15 import re
16 import subprocess
17 import sys
18 import tempfile
19 from os import environ, path
20 from tempfile import TemporaryDirectory
21
22 import pytest
23 from b2sdk.v2 import B2_ACCOUNT_INFO_ENV_VAR, XDG_CONFIG_HOME_ENV_VAR, Bucket
24
25 from b2._internal.version_listing import (
26 CLI_VERSIONS,
27 LATEST_STABLE_VERSION,
28 UNSTABLE_CLI_VERSION,
29 get_int_version,
30 )
31
32 from ..helpers import b2_uri_args_v3, b2_uri_args_v4
33 from .helpers import NODE_DESCRIPTION, RNG_SEED, Api, CommandLine, bucket_name_part, random_token
34
35 logger = logging.getLogger(__name__)
36
37 GENERAL_BUCKET_NAME_PREFIX = 'clitst'
38 TEMPDIR = tempfile.gettempdir()
39 ROOT_PATH = pathlib.Path(__file__).parent.parent.parent
40
41
42 @pytest.fixture(scope='session', autouse=True)
43 def summary_notes(request, worker_id):
44 capmanager = request.config.pluginmanager.getplugin("capturemanager")
45 with capmanager.global_and_fixture_disabled():
46 log_handler = logging.StreamHandler(sys.stderr)
47 log_fmt = logging.Formatter(f'{worker_id} %(asctime)s %(levelname).1s %(message)s')
48 log_handler.setFormatter(log_fmt)
49 logger.addHandler(log_handler)
50
51 class Notes:
52 def append(self, note):
53 logger.info(note)
54
55 return Notes()
56
57
58 @pytest.fixture(scope='session', autouse=True)
59 def node_stats(summary_notes):
60 summary_notes.append(f"NODE={NODE_DESCRIPTION} seed={RNG_SEED}")
61
62
63 @pytest.hookimpl
64 def pytest_addoption(parser):
65 parser.addoption(
66 '--sut',
67 default=f'{sys.executable} -m b2._internal.{UNSTABLE_CLI_VERSION}',
68 help='Path to the System Under Test',
69 )
70 parser.addoption(
71 '--env-file-cmd-placeholder',
72 default=None,
73 help=(
74 'If specified, all occurrences of this string in `--sut` will be substituted with a '
75 'path to a tmp file containing env vars to be used when running commands in tests. Useful '
76 'for docker.'
77 )
78 )
79 parser.addoption(
80 '--as_version',
81 default=None,
82 help='Force running tests as a particular version of the CLI, '
83 'useful if version cannot be determined easily from the executable',
84 )
85 parser.addoption('--cleanup', action='store_true', help='Perform full cleanup at exit')
86
87
88 def get_raw_cli_int_version(config) -> int | None:
89 forced_version = config.getoption('--as_version')
90 if forced_version:
91 return int(forced_version)
92
93 executable = config.getoption('--sut')
94 # If the executable contains anything that looks like a proper version, we can try to pick it up.
95 versions_list = '|'.join(CLI_VERSIONS)
96 versions_match = re.search(rf'({versions_list})', executable)
97 if versions_match:
98 return get_int_version(versions_match.group(1))
99
100 return None
101
102
103 def get_cli_int_version(config) -> int:
104 return get_raw_cli_int_version(config) or get_int_version(LATEST_STABLE_VERSION)
105
106
107 @pytest.fixture(scope='session')
108 def apiver_int(request):
109 return get_cli_int_version(request.config)
110
111
112 @pytest.fixture(scope='session')
113 def apiver(apiver_int):
114 return f"v{apiver_int}"
115
116
117 @pytest.hookimpl
118 def pytest_report_header(config):
119 cli_version = get_cli_int_version(config)
120 return f'b2 apiver: {cli_version}'
121
122
123 @pytest.fixture(scope='session')
124 def cli_version(request) -> str:
125 """
126 Get CLI version name, i.e. b2v3, _b2v4, etc.
127 """
128 # The default stable version could be provided directly as e.g.: b2v3, but also indirectly as b2.
129 # In case there is no direct version, we return the default binary name instead.
130 raw_cli_version = get_raw_cli_int_version(request.config)
131 if raw_cli_version is None:
132 return 'b2'
133
134 for version in CLI_VERSIONS:
135 if get_int_version(version) == raw_cli_version:
136 return version
137 raise pytest.UsageError(f'Unknown CLI version: {raw_cli_version}')
138
139
140 @pytest.fixture(scope='session')
141 def application_key() -> str:
142 key = environ.get('B2_TEST_APPLICATION_KEY')
143 assert application_key, 'B2_TEST_APPLICATION_KEY is not set'
144 yield key
145
146
147 @pytest.fixture(scope='session')
148 def application_key_id() -> str:
149 key_id = environ.get('B2_TEST_APPLICATION_KEY_ID')
150 assert key_id, 'B2_TEST_APPLICATION_KEY_ID is not set'
151 yield key_id
152
153
154 @pytest.fixture(scope='session')
155 def realm() -> str:
156 yield environ.get('B2_TEST_ENVIRONMENT', 'production')
157
158
159 @pytest.fixture
160 def bucket(bucket_factory) -> Bucket:
161 return bucket_factory()
162
163
164 @pytest.fixture
165 def bucket_factory(b2_api, schedule_bucket_cleanup):
166 def create_bucket(**kwargs):
167 new_bucket = b2_api.create_bucket(**kwargs)
168 schedule_bucket_cleanup(new_bucket.name, new_bucket.bucket_dict)
169 return new_bucket
170
171 yield create_bucket
172
173
174 @pytest.fixture(scope='function')
175 def bucket_name(bucket) -> str:
176 yield bucket.name
177
178
179 @pytest.fixture(scope='function')
180 def file_name(bucket) -> str:
181 file_ = bucket.upload_bytes(b'test_file', f'{random_token(8)}.txt')
182 yield file_.file_name
183
184
185 @pytest.fixture(scope='function') # , autouse=True)
186 def debug_print_buckets(b2_api):
187 print('-' * 30)
188 print('Buckets before test ' + environ['PYTEST_CURRENT_TEST'])
189 num_buckets = b2_api.count_and_print_buckets()
190 print('-' * 30)
191 try:
192 yield
193 finally:
194 print('-' * 30)
195 print('Buckets after test ' + environ['PYTEST_CURRENT_TEST'])
196 delta = b2_api.count_and_print_buckets() - num_buckets
197 print(f'DELTA: {delta}')
198 print('-' * 30)
199
200
201 @pytest.fixture(scope='session')
202 def this_run_bucket_name_prefix() -> str:
203 yield GENERAL_BUCKET_NAME_PREFIX + bucket_name_part(8)
204
205
206 @pytest.fixture(scope='session')
207 def monkeysession():
208 with pytest.MonkeyPatch.context() as mp:
209 yield mp
210
211
212 @pytest.fixture(scope='session', autouse=True)
213 def auto_change_account_info_dir(monkeysession) -> str:
214 """
215 Automatically for the whole testing:
216 1) temporary remove B2_APPLICATION_KEY and B2_APPLICATION_KEY_ID from environment
217 2) create a temporary directory for storing account info database
218 3) set B2_ACCOUNT_INFO_ENV_VAR to point to the temporary account info file
219 """
220
221 monkeysession.delenv('B2_APPLICATION_KEY_ID', raising=False)
222 monkeysession.delenv('B2_APPLICATION_KEY', raising=False)
223
224 # make b2sdk use temp dir for storing default & per-profile account information
225 with TemporaryDirectory() as temp_dir:
226 monkeysession.setenv(B2_ACCOUNT_INFO_ENV_VAR, path.join(temp_dir, '.b2_account_info'))
227 monkeysession.setenv(XDG_CONFIG_HOME_ENV_VAR, temp_dir)
228 yield temp_dir
229
230
231 @pytest.fixture(scope='session')
232 def b2_api(
233 application_key_id,
234 application_key,
235 realm,
236 this_run_bucket_name_prefix,
237 auto_change_account_info_dir,
238 summary_notes,
239 ) -> Api:
240 api = Api(
241 application_key_id,
242 application_key,
243 realm,
244 general_bucket_name_prefix=GENERAL_BUCKET_NAME_PREFIX,
245 this_run_bucket_name_prefix=this_run_bucket_name_prefix,
246 )
247 yield api
248 api.clean_buckets()
249 # showing account_id in the logs is safe; so we explicitly prevent it from being redacted
250 summary_notes.append(f"B2 Account ID: {api.account_id[:1]!r}{api.account_id[1:]!r}")
251 summary_notes.append(f"Buckets names used during this tests: {api.bucket_name_log!r}")
252
253
254 @pytest.fixture(scope='module')
255 def global_b2_tool(
256 request, application_key_id, application_key, realm, this_run_bucket_name_prefix, b2_api,
257 auto_change_account_info_dir, b2_uri_args
258 ) -> CommandLine:
259 tool = CommandLine(
260 request.config.getoption('--sut'),
261 application_key_id,
262 application_key,
263 realm,
264 this_run_bucket_name_prefix,
265 request.config.getoption('--env-file-cmd-placeholder'),
266 api_wrapper=b2_api,
267 b2_uri_args=b2_uri_args,
268 )
269 tool.reauthorize(check_key_capabilities=True) # reauthorize for the first time (with check)
270 yield tool
271
272
273 @pytest.fixture(scope='function')
274 def b2_tool(global_b2_tool):
275 """Automatically reauthorized b2_tool for each test (without check)"""
276 global_b2_tool.reauthorize(check_key_capabilities=False)
277 return global_b2_tool
278
279
280 @pytest.fixture
281 def account_info_file() -> pathlib.Path:
282 return pathlib.Path(os.environ[B2_ACCOUNT_INFO_ENV_VAR]).expanduser()
283
284
285 @pytest.fixture
286 def schedule_bucket_cleanup(global_b2_tool):
287 """
288 Explicitly ask for buckets cleanup after the test
289
290 This should be only used when testing `create-bucket` command; otherwise use `bucket_factory` fixture.
291 """
292 buckets_to_clean = {}
293
294 def add_bucket_to_cleanup(bucket_name, bucket_dict: dict | None = None):
295 buckets_to_clean[bucket_name] = bucket_dict
296
297 yield add_bucket_to_cleanup
298 if buckets_to_clean:
299 global_b2_tool.reauthorize(
300 check_key_capabilities=False
301 ) # test may have mangled authorization
302 global_b2_tool.cleanup_buckets(buckets_to_clean)
303
304
305 @pytest.fixture(autouse=True, scope='session')
306 def sample_filepath():
307 """Copy the README.md file to /tmp so that docker tests can access it"""
308 tmp_readme = pathlib.Path(TEMPDIR) / 'README.md'
309 if not tmp_readme.exists():
310 tmp_readme.write_text((ROOT_PATH / 'README.md').read_text())
311 return tmp_readme
312
313
314 @pytest.fixture(autouse=True, scope='session')
315 def sample_file(sample_filepath):
316 return str(sample_filepath)
317
318
319 @pytest.fixture(scope='session')
320 def is_running_on_docker(pytestconfig):
321 return pytestconfig.getoption('--sut').startswith('docker')
322
323
324 SECRET_FIXTURES = {'application_key', 'application_key_id'}
325
326
327 @pytest.fixture(scope="session")
328 def homedir(tmp_path_factory):
329 yield tmp_path_factory.mktemp("test_homedir")
330
331
332 @pytest.fixture(scope="session")
333 def b2_in_path(tmp_path_factory):
334 """
335 Create a dummy b2 executable in a temporary directory and add it to PATH.
336
337 This allows us to test the b2 command from shell level even if tested `b2` package was not installed.
338 """
339
340 tempdir = tmp_path_factory.mktemp("temp_bin")
341 temp_executable = tempdir / "b2"
342 with open(temp_executable, "w") as f:
343 f.write(
344 f"#!{sys.executable}\n"
345 "import sys\n"
346 f"sys.path.insert(0, {os.getcwd()!r})\n" # ensure relative imports work even if command is run in different directory
347 "from b2.console_tool import main\n"
348 "main()\n"
349 )
350
351 temp_executable.chmod(0o700)
352
353 original_path = os.environ["PATH"]
354 new_path = f"{tempdir}:{original_path}"
355 yield new_path
356
357
358 @pytest.fixture(scope="module")
359 def env(b2_in_path, homedir, monkeysession, is_running_on_docker):
360 """Get ENV for running b2 command from shell level."""
361 if not is_running_on_docker:
362 monkeysession.setenv('PATH', b2_in_path)
363 monkeysession.setenv('HOME', str(homedir))
364 monkeysession.setenv('SHELL', "/bin/bash") # fix for running under github actions
365 yield os.environ
366
367
368 @pytest.fixture
369 def bash_runner(env):
370 """Run command in bash shell."""
371
372 def run_command(command: str):
373 try:
374 return subprocess.run(
375 ["/bin/bash", "-c", command],
376 capture_output=True,
377 check=True,
378 env=env,
379 text=True,
380 )
381 except subprocess.CalledProcessError as e:
382 print(f"Command {command!r} failed with exit code {e.returncode}")
383 print(e.stdout)
384 print(e.stderr, file=sys.stderr)
385 raise
386
387 return run_command
388
389
390 def pytest_collection_modifyitems(items):
391 """
392 Add 'require_secrets' marker to all tests that use secrets.
393 """
394 for item in items:
395 if SECRET_FIXTURES & set(getattr(item, 'fixturenames', ())):
396 item.add_marker('require_secrets')
397
398
399 @pytest.fixture(scope='module')
400 def b2_uri_args(apiver_int):
401 if apiver_int >= 4:
402 return b2_uri_args_v4
403 else:
404 return b2_uri_args_v3