| 1 |
#!/usr/bin/env python3 |
| 2 |
###################################################################### |
| 3 |
# |
| 4 |
# File: test/integration/test_b2_command_line.py |
| 5 |
# |
| 6 |
# Copyright 2019 Backblaze Inc. All Rights Reserved. |
| 7 |
# |
| 8 |
# License https://www.backblaze.com/using_b2_code.html |
| 9 |
# |
| 10 |
###################################################################### |
| 11 |
from __future__ import annotations |
| 12 |
|
| 13 |
import base64 |
| 14 |
import contextlib |
| 15 |
import hashlib |
| 16 |
import itertools |
| 17 |
import json |
| 18 |
import os |
| 19 |
import os.path |
| 20 |
import pathlib |
| 21 |
import re |
| 22 |
import sys |
| 23 |
import time |
| 24 |
from pathlib import Path |
| 25 |
from tempfile import mkdtemp |
| 26 |
|
| 27 |
import pytest |
| 28 |
from b2sdk.v2 import ( |
| 29 |
B2_ACCOUNT_INFO_ENV_VAR, |
| 30 |
SSE_C_KEY_ID_FILE_INFO_KEY_NAME, |
| 31 |
UNKNOWN_FILE_RETENTION_SETTING, |
| 32 |
EncryptionMode, |
| 33 |
EncryptionSetting, |
| 34 |
FileRetentionSetting, |
| 35 |
LegalHold, |
| 36 |
RetentionMode, |
| 37 |
SqliteAccountInfo, |
| 38 |
fix_windows_path_limit, |
| 39 |
) |
| 40 |
from b2sdk.v2.exception import MissingAccountData |
| 41 |
|
| 42 |
from b2._internal._cli.const import ( |
| 43 |
B2_APPLICATION_KEY_ENV_VAR, |
| 44 |
B2_APPLICATION_KEY_ID_ENV_VAR, |
| 45 |
B2_ENVIRONMENT_ENV_VAR, |
| 46 |
) |
| 47 |
from b2._internal.console_tool import current_time_millis |
| 48 |
|
| 49 |
from ..helpers import skip_on_windows |
| 50 |
from .helpers import ( |
| 51 |
ONE_DAY_MILLIS, |
| 52 |
ONE_HOUR_MILLIS, |
| 53 |
SSE_B2_AES, |
| 54 |
SSE_C_AES, |
| 55 |
SSE_C_AES_2, |
| 56 |
SSE_NONE, |
| 57 |
TempDir, |
| 58 |
file_mod_time_millis, |
| 59 |
random_hex, |
| 60 |
read_file, |
| 61 |
set_file_mod_time_millis, |
| 62 |
should_equal, |
| 63 |
write_file, |
| 64 |
) |
| 65 |
|
| 66 |
|
| 67 |
def test_authorize_account_via_params_saving_credentials( |
| 68 |
b2_tool, |
| 69 |
realm, |
| 70 |
application_key, |
| 71 |
application_key_id, |
| 72 |
account_info_file, |
| 73 |
): |
| 74 |
""" |
| 75 |
When calling `authorize-account` and passing credentials as params, |
| 76 |
we want the credentials to be saved. |
| 77 |
""" |
| 78 |
|
| 79 |
b2_tool.should_succeed(['clear-account']) |
| 80 |
|
| 81 |
assert B2_APPLICATION_KEY_ID_ENV_VAR not in os.environ |
| 82 |
assert B2_APPLICATION_KEY_ENV_VAR not in os.environ |
| 83 |
|
| 84 |
b2_tool.should_succeed( |
| 85 |
['authorize-account', '--environment', realm, application_key_id, application_key] |
| 86 |
) |
| 87 |
|
| 88 |
assert account_info_file.exists() |
| 89 |
account_info = SqliteAccountInfo() |
| 90 |
assert account_info.get_application_key() == application_key |
| 91 |
assert account_info.get_application_key_id() == application_key_id |
| 92 |
|
| 93 |
|
| 94 |
def test_authorize_account_via_env_vars_saving_credentials( |
| 95 |
b2_tool, |
| 96 |
realm, |
| 97 |
application_key, |
| 98 |
application_key_id, |
| 99 |
account_info_file, |
| 100 |
): |
| 101 |
""" |
| 102 |
When calling `authorize-account` and passing credentials |
| 103 |
via env vars, we still want the credentials to be saved. |
| 104 |
""" |
| 105 |
|
| 106 |
b2_tool.should_succeed(['clear-account']) |
| 107 |
|
| 108 |
assert B2_APPLICATION_KEY_ID_ENV_VAR not in os.environ |
| 109 |
assert B2_APPLICATION_KEY_ENV_VAR not in os.environ |
| 110 |
|
| 111 |
b2_tool.should_succeed( |
| 112 |
['authorize-account'], |
| 113 |
additional_env={ |
| 114 |
B2_ENVIRONMENT_ENV_VAR: realm, |
| 115 |
B2_APPLICATION_KEY_ID_ENV_VAR: application_key_id, |
| 116 |
B2_APPLICATION_KEY_ENV_VAR: application_key, |
| 117 |
} |
| 118 |
) |
| 119 |
|
| 120 |
assert account_info_file.exists() |
| 121 |
account_info = SqliteAccountInfo() |
| 122 |
assert account_info.get_application_key() == application_key |
| 123 |
assert account_info.get_application_key_id() == application_key_id |
| 124 |
|
| 125 |
|
| 126 |
def test_clear_account_with_env_vars( |
| 127 |
b2_tool, |
| 128 |
realm, |
| 129 |
application_key, |
| 130 |
application_key_id, |
| 131 |
account_info_file, |
| 132 |
): |
| 133 |
""" |
| 134 |
When calling `clear-account` and passing credentials via env vars, |
| 135 |
we want the credentials to be removed from the file. |
| 136 |
""" |
| 137 |
|
| 138 |
assert account_info_file.exists() |
| 139 |
account_info = SqliteAccountInfo() |
| 140 |
assert account_info.get_application_key() == application_key |
| 141 |
assert account_info.get_application_key_id() == application_key_id |
| 142 |
|
| 143 |
b2_tool.should_succeed( |
| 144 |
['clear-account'], |
| 145 |
additional_env={ |
| 146 |
B2_ENVIRONMENT_ENV_VAR: realm, |
| 147 |
B2_APPLICATION_KEY_ID_ENV_VAR: application_key_id, |
| 148 |
B2_APPLICATION_KEY_ENV_VAR: application_key, |
| 149 |
} |
| 150 |
) |
| 151 |
|
| 152 |
assert account_info_file.exists() |
| 153 |
account_info = SqliteAccountInfo() |
| 154 |
with pytest.raises(MissingAccountData): |
| 155 |
account_info.get_application_key() |
| 156 |
with pytest.raises(MissingAccountData): |
| 157 |
account_info.get_application_key_id() |
| 158 |
|
| 159 |
|
| 160 |
@pytest.mark.apiver(to_ver=3) |
| 161 |
def test_command_with_env_vars_saving_credentials( |
| 162 |
b2_tool, |
| 163 |
realm, |
| 164 |
application_key, |
| 165 |
application_key_id, |
| 166 |
account_info_file, |
| 167 |
bucket_name, |
| 168 |
b2_uri_args, |
| 169 |
): |
| 170 |
""" |
| 171 |
When calling any command other then `authorize-account` and passing credentials |
| 172 |
via env vars, we don't want them to be saved. |
| 173 |
""" |
| 174 |
|
| 175 |
b2_tool.should_succeed(['clear-account']) |
| 176 |
|
| 177 |
assert B2_APPLICATION_KEY_ID_ENV_VAR not in os.environ |
| 178 |
assert B2_APPLICATION_KEY_ENV_VAR not in os.environ |
| 179 |
|
| 180 |
b2_tool.should_succeed( |
| 181 |
['ls', '--long', *b2_uri_args(bucket_name)], |
| 182 |
additional_env={ |
| 183 |
B2_ENVIRONMENT_ENV_VAR: realm, |
| 184 |
B2_APPLICATION_KEY_ID_ENV_VAR: application_key_id, |
| 185 |
B2_APPLICATION_KEY_ENV_VAR: application_key, |
| 186 |
} |
| 187 |
) |
| 188 |
|
| 189 |
assert account_info_file.exists() |
| 190 |
account_info = SqliteAccountInfo() |
| 191 |
assert account_info.get_application_key() == application_key |
| 192 |
assert account_info.get_application_key_id() == application_key_id |
| 193 |
|
| 194 |
|
| 195 |
@pytest.mark.apiver(from_ver=4) |
| 196 |
def test_command_with_env_vars_not_saving_credentials( |
| 197 |
b2_tool, |
| 198 |
realm, |
| 199 |
application_key, |
| 200 |
application_key_id, |
| 201 |
account_info_file, |
| 202 |
bucket_name, |
| 203 |
b2_uri_args, |
| 204 |
): |
| 205 |
""" |
| 206 |
When calling any command other then `authorize-account` and passing credentials |
| 207 |
via env vars, we don't want them to be saved. |
| 208 |
""" |
| 209 |
|
| 210 |
b2_tool.should_succeed(['clear-account']) |
| 211 |
|
| 212 |
assert B2_APPLICATION_KEY_ID_ENV_VAR not in os.environ |
| 213 |
assert B2_APPLICATION_KEY_ENV_VAR not in os.environ |
| 214 |
|
| 215 |
b2_tool.should_succeed( |
| 216 |
['ls', '--long', *b2_uri_args(bucket_name)], |
| 217 |
additional_env={ |
| 218 |
B2_ENVIRONMENT_ENV_VAR: realm, |
| 219 |
B2_APPLICATION_KEY_ID_ENV_VAR: application_key_id, |
| 220 |
B2_APPLICATION_KEY_ENV_VAR: application_key, |
| 221 |
} |
| 222 |
) |
| 223 |
|
| 224 |
assert account_info_file.exists() |
| 225 |
account_info = SqliteAccountInfo() |
| 226 |
with pytest.raises(MissingAccountData): |
| 227 |
account_info.get_application_key() |
| 228 |
with pytest.raises(MissingAccountData): |
| 229 |
account_info.get_application_key_id() |
| 230 |
|
| 231 |
|
| 232 |
@pytest.mark.apiver(from_ver=4) |
| 233 |
def test_command_with_env_vars_reusing_existing_account_info( |
| 234 |
b2_tool, |
| 235 |
realm, |
| 236 |
application_key, |
| 237 |
application_key_id, |
| 238 |
account_info_file, |
| 239 |
bucket_name, |
| 240 |
b2_uri_args, |
| 241 |
): |
| 242 |
""" |
| 243 |
When calling any command with credentials passed via env vars, and the account |
| 244 |
info file already contains the same credentials, we want to use filesystem for |
| 245 |
storing cache, not the in-memory cache. |
| 246 |
""" |
| 247 |
|
| 248 |
assert B2_APPLICATION_KEY_ID_ENV_VAR not in os.environ |
| 249 |
assert B2_APPLICATION_KEY_ENV_VAR not in os.environ |
| 250 |
|
| 251 |
assert account_info_file.exists() |
| 252 |
account_info = SqliteAccountInfo() |
| 253 |
assert account_info.get_application_key() == application_key |
| 254 |
assert account_info.get_application_key_id() == application_key_id |
| 255 |
|
| 256 |
account_info.remove_bucket_name(bucket_name) |
| 257 |
assert account_info.get_bucket_id_or_none_from_bucket_name(bucket_name) is None |
| 258 |
|
| 259 |
b2_tool.should_succeed( |
| 260 |
['ls', '--long', *b2_uri_args(bucket_name)], |
| 261 |
additional_env={ |
| 262 |
B2_ENVIRONMENT_ENV_VAR: realm, |
| 263 |
B2_APPLICATION_KEY_ID_ENV_VAR: application_key_id, |
| 264 |
B2_APPLICATION_KEY_ENV_VAR: application_key, |
| 265 |
} |
| 266 |
) |
| 267 |
|
| 268 |
assert account_info_file.exists() |
| 269 |
account_info = SqliteAccountInfo() |
| 270 |
assert account_info.get_bucket_id_or_none_from_bucket_name(bucket_name) is not None |
| 271 |
|
| 272 |
|
| 273 |
@pytest.fixture |
| 274 |
def uploaded_sample_file(b2_tool, bucket_name, sample_filepath): |
| 275 |
return b2_tool.should_succeed_json( |
| 276 |
['upload-file', '--quiet', bucket_name, |
| 277 |
str(sample_filepath), 'sample_file'] |
| 278 |
) |
| 279 |
|
| 280 |
|
| 281 |
def test_download(b2_tool, bucket_name, sample_filepath, uploaded_sample_file, tmp_path): |
| 282 |
output_a = tmp_path / 'a' |
| 283 |
b2_tool.should_succeed( |
| 284 |
[ |
| 285 |
'download-file', '--quiet', f"b2://{bucket_name}/{uploaded_sample_file['fileName']}", |
| 286 |
str(output_a) |
| 287 |
] |
| 288 |
) |
| 289 |
assert output_a.read_text() == sample_filepath.read_text() |
| 290 |
|
| 291 |
output_b = tmp_path / 'b' |
| 292 |
b2_tool.should_succeed( |
| 293 |
['download-file', '--quiet', f"b2id://{uploaded_sample_file['fileId']}", |
| 294 |
str(output_b)] |
| 295 |
) |
| 296 |
assert output_b.read_text() == sample_filepath.read_text() |
| 297 |
|
| 298 |
|
| 299 |
def test_basic(b2_tool, bucket_name, sample_file, tmp_path, b2_uri_args): |
| 300 |
|
| 301 |
file_mod_time_str = str(file_mod_time_millis(sample_file)) |
| 302 |
|
| 303 |
file_data = read_file(sample_file) |
| 304 |
hex_sha1 = hashlib.sha1(file_data).hexdigest() |
| 305 |
|
| 306 |
list_of_buckets = b2_tool.should_succeed_json(['list-buckets', '--json']) |
| 307 |
should_equal( |
| 308 |
[bucket_name], [b['bucketName'] for b in list_of_buckets if b['bucketName'] == bucket_name] |
| 309 |
) |
| 310 |
|
| 311 |
b2_tool.should_succeed(['upload-file', '--quiet', bucket_name, sample_file, 'a']) |
| 312 |
b2_tool.should_succeed(['ls', '--long', '--replication', *b2_uri_args(bucket_name)]) |
| 313 |
b2_tool.should_succeed(['upload-file', '--no-progress', bucket_name, sample_file, 'a']) |
| 314 |
b2_tool.should_succeed(['upload-file', '--no-progress', bucket_name, sample_file, 'b/1']) |
| 315 |
b2_tool.should_succeed(['upload-file', '--no-progress', bucket_name, sample_file, 'b/2']) |
| 316 |
b2_tool.should_succeed( |
| 317 |
[ |
| 318 |
'upload-file', '--no-progress', '--sha1', hex_sha1, '--info', 'foo=bar=baz', '--info', |
| 319 |
'color=blue', bucket_name, sample_file, 'c' |
| 320 |
] |
| 321 |
) |
| 322 |
b2_tool.should_fail( |
| 323 |
[ |
| 324 |
'upload-file', '--no-progress', '--sha1', hex_sha1, '--info', 'foo-bar', '--info', |
| 325 |
'color=blue', bucket_name, sample_file, 'c' |
| 326 |
], r'ERROR: Bad file info: foo-bar' |
| 327 |
) |
| 328 |
b2_tool.should_succeed( |
| 329 |
[ |
| 330 |
'upload-file', '--no-progress', '--content-type', 'text/plain', bucket_name, |
| 331 |
sample_file, 'd' |
| 332 |
] |
| 333 |
) |
| 334 |
|
| 335 |
b2_tool.should_succeed(['upload-file', '--no-progress', bucket_name, sample_file, 'rm']) |
| 336 |
b2_tool.should_succeed(['upload-file', '--no-progress', bucket_name, sample_file, 'rm1']) |
| 337 |
# with_wildcard allows us to target a single file. rm will be removed, rm1 will be left alone |
| 338 |
b2_tool.should_succeed( |
| 339 |
['rm', '--recursive', '--with-wildcard', *b2_uri_args(bucket_name, 'rm')] |
| 340 |
) |
| 341 |
list_of_files = b2_tool.should_succeed_json( |
| 342 |
['ls', '--json', '--recursive', '--with-wildcard', *b2_uri_args(bucket_name, 'rm*')] |
| 343 |
) |
| 344 |
should_equal(['rm1'], [f['fileName'] for f in list_of_files]) |
| 345 |
b2_tool.should_succeed( |
| 346 |
['rm', '--recursive', '--with-wildcard', *b2_uri_args(bucket_name, 'rm1')] |
| 347 |
) |
| 348 |
|
| 349 |
b2_tool.should_succeed(['download-file', '--quiet', f'b2://{bucket_name}/b/1', tmp_path / 'a']) |
| 350 |
|
| 351 |
b2_tool.should_succeed(['hide-file', bucket_name, 'c']) |
| 352 |
|
| 353 |
list_of_files = b2_tool.should_succeed_json( |
| 354 |
['ls', '--json', '--recursive', *b2_uri_args(bucket_name)] |
| 355 |
) |
| 356 |
should_equal(['a', 'b/1', 'b/2', 'd'], [f['fileName'] for f in list_of_files]) |
| 357 |
|
| 358 |
list_of_files = b2_tool.should_succeed_json( |
| 359 |
['ls', '--json', '--recursive', '--versions', *b2_uri_args(bucket_name)] |
| 360 |
) |
| 361 |
should_equal(['a', 'a', 'b/1', 'b/2', 'c', 'c', 'd'], [f['fileName'] for f in list_of_files]) |
| 362 |
should_equal( |
| 363 |
['upload', 'upload', 'upload', 'upload', 'hide', 'upload', 'upload'], |
| 364 |
[f['action'] for f in list_of_files] |
| 365 |
) |
| 366 |
|
| 367 |
first_a_version = list_of_files[0] |
| 368 |
|
| 369 |
first_c_version = list_of_files[4] |
| 370 |
second_c_version = list_of_files[5] |
| 371 |
list_of_files = b2_tool.should_succeed_json( |
| 372 |
['ls', '--json', '--recursive', '--versions', *b2_uri_args(bucket_name, 'c')] |
| 373 |
) |
| 374 |
should_equal([], [f['fileName'] for f in list_of_files]) |
| 375 |
|
| 376 |
b2_tool.should_succeed(['copy-file-by-id', first_a_version['fileId'], bucket_name, 'x']) |
| 377 |
|
| 378 |
b2_tool.should_succeed(['ls', *b2_uri_args(bucket_name)], '^a{0}b/{0}d{0}'.format(os.linesep)) |
| 379 |
# file_id, action, date, time, size(, replication), name |
| 380 |
b2_tool.should_succeed( |
| 381 |
['ls', '--long', *b2_uri_args(bucket_name)], |
| 382 |
'^4_z.* upload .* {1} a{0}.* - .* b/{0}4_z.* upload .* {1} d{0}'.format( |
| 383 |
os.linesep, len(file_data) |
| 384 |
) |
| 385 |
) |
| 386 |
b2_tool.should_succeed( |
| 387 |
['ls', '--long', '--replication', *b2_uri_args(bucket_name)], |
| 388 |
'^4_z.* upload .* {1} - a{0}.* - .* - b/{0}4_z.* upload .* {1} - d{0}'.format( |
| 389 |
os.linesep, len(file_data) |
| 390 |
) |
| 391 |
) |
| 392 |
b2_tool.should_succeed( |
| 393 |
['ls', '--versions', *b2_uri_args(bucket_name)], |
| 394 |
f'^a{os.linesep}a{os.linesep}b/{os.linesep}c{os.linesep}c{os.linesep}d{os.linesep}' |
| 395 |
) |
| 396 |
b2_tool.should_succeed( |
| 397 |
['ls', *b2_uri_args(bucket_name, 'b')], f'^b/1{os.linesep}b/2{os.linesep}' |
| 398 |
) |
| 399 |
b2_tool.should_succeed( |
| 400 |
['ls', *b2_uri_args(bucket_name, 'b/')], f'^b/1{os.linesep}b/2{os.linesep}' |
| 401 |
) |
| 402 |
|
| 403 |
file_info = b2_tool.should_succeed_json(['file-info', f"b2id://{second_c_version['fileId']}"]) |
| 404 |
expected_info = { |
| 405 |
'color': 'blue', |
| 406 |
'foo': 'bar=baz', |
| 407 |
'src_last_modified_millis': file_mod_time_str |
| 408 |
} |
| 409 |
should_equal(expected_info, file_info['fileInfo']) |
| 410 |
|
| 411 |
b2_tool.should_succeed(['delete-file-version', 'c', first_c_version['fileId']]) |
| 412 |
b2_tool.should_succeed( |
| 413 |
['ls', *b2_uri_args(bucket_name)], f'^a{os.linesep}b/{os.linesep}c{os.linesep}d{os.linesep}' |
| 414 |
) |
| 415 |
|
| 416 |
b2_tool.should_succeed(['get-url', f"b2id://{second_c_version['fileId']}"]) |
| 417 |
|
| 418 |
b2_tool.should_succeed( |
| 419 |
['get-url', f"b2://{bucket_name}/any-file-name"], |
| 420 |
'^https://.*/file/{}/{}\r?$'.format( |
| 421 |
bucket_name, |
| 422 |
'any-file-name', |
| 423 |
), |
| 424 |
) # \r? is for Windows, as $ doesn't match \r\n |
| 425 |
|
| 426 |
|
| 427 |
@pytest.mark.apiver(from_ver=4) |
| 428 |
def test_ls_b2id(b2_tool, uploaded_sample_file): |
| 429 |
b2_tool.should_succeed( |
| 430 |
['ls', f"b2id://{uploaded_sample_file['fileId']}"], |
| 431 |
expected_pattern=f"^{uploaded_sample_file['fileName']}", |
| 432 |
) |
| 433 |
|
| 434 |
|
| 435 |
@pytest.mark.apiver(from_ver=4) |
| 436 |
def test_rm_b2id(b2_tool, bucket_name, uploaded_sample_file): |
| 437 |
# remove the file by id |
| 438 |
b2_tool.should_succeed(['rm', f"b2id://{uploaded_sample_file['fileId']}"]) |
| 439 |
|
| 440 |
# check that the file is gone |
| 441 |
b2_tool.should_succeed( |
| 442 |
['ls', f'b2://{bucket_name}'], |
| 443 |
expected_pattern='^$', |
| 444 |
) |
| 445 |
|
| 446 |
|
| 447 |
def test_debug_logs(b2_tool, is_running_on_docker, tmp_path): |
| 448 |
to_be_removed_bucket_name = b2_tool.generate_bucket_name() |
| 449 |
b2_tool.should_succeed( |
| 450 |
[ |
| 451 |
'create-bucket', |
| 452 |
to_be_removed_bucket_name, |
| 453 |
'allPublic', |
| 454 |
*b2_tool.get_bucket_info_args(), |
| 455 |
], |
| 456 |
) |
| 457 |
b2_tool.should_succeed(['delete-bucket', to_be_removed_bucket_name],) |
| 458 |
b2_tool.should_fail( |
| 459 |
['delete-bucket', to_be_removed_bucket_name], |
| 460 |
re.compile(r'^ERROR: Bucket with id=\w* not found\s*$') |
| 461 |
) |
| 462 |
# Check logging settings |
| 463 |
if not is_running_on_docker: # It's difficult to read the log in docker in CI |
| 464 |
b2_tool.should_fail( |
| 465 |
['delete-bucket', to_be_removed_bucket_name, '--debug-logs'], |
| 466 |
re.compile(r'^ERROR: Bucket with id=\w* not found\s*$') |
| 467 |
) |
| 468 |
stack_trace_in_log = r'Traceback \(most recent call last\):.*Bucket with id=\w* not found' |
| 469 |
|
| 470 |
# the two regexes below depend on log message from urllib3, which is not perfect, but this test needs to |
| 471 |
# check global logging settings |
| 472 |
stderr_regex = re.compile( |
| 473 |
r'DEBUG:urllib3.connectionpool:.* "POST /b2api/v2/b2_delete_bucket HTTP' |
| 474 |
r'.*' + stack_trace_in_log, |
| 475 |
re.DOTALL, |
| 476 |
) |
| 477 |
log_file_regex = re.compile( |
| 478 |
r'urllib3.connectionpool\tDEBUG\t.* "POST /b2api/v2/b2_delete_bucket HTTP' |
| 479 |
r'.*' + stack_trace_in_log, |
| 480 |
re.DOTALL, |
| 481 |
) |
| 482 |
with open('b2_cli.log') as logfile: |
| 483 |
log = logfile.read() |
| 484 |
assert re.search(log_file_regex, log), log |
| 485 |
os.remove('b2_cli.log') |
| 486 |
|
| 487 |
b2_tool.should_fail(['delete-bucket', to_be_removed_bucket_name, '--verbose'], stderr_regex) |
| 488 |
assert not os.path.exists('b2_cli.log') |
| 489 |
|
| 490 |
b2_tool.should_fail( |
| 491 |
['delete-bucket', to_be_removed_bucket_name, '--verbose', '--debug-logs'], stderr_regex |
| 492 |
) |
| 493 |
with open('b2_cli.log') as logfile: |
| 494 |
log = logfile.read() |
| 495 |
assert re.search(log_file_regex, log), log |
| 496 |
|
| 497 |
|
| 498 |
def test_bucket(b2_tool, bucket_name): |
| 499 |
rule = """{ |
| 500 |
"daysFromHidingToDeleting": 1, |
| 501 |
"daysFromUploadingToHiding": null, |
| 502 |
"fileNamePrefix": "" |
| 503 |
}""" |
| 504 |
output = b2_tool.should_succeed_json( |
| 505 |
[ |
| 506 |
'update-bucket', '--lifecycle-rule', rule, bucket_name, 'allPublic', |
| 507 |
*b2_tool.get_bucket_info_args() |
| 508 |
], |
| 509 |
) |
| 510 |
|
| 511 |
########## // doesn't happen on production, but messes up some tests \\ ########## |
| 512 |
for key in output['lifecycleRules'][0]: |
| 513 |
if key[8] == 'S' and len(key) == 47: |
| 514 |
del output['lifecycleRules'][0][key] |
| 515 |
break |
| 516 |
########## \\ doesn't happen on production, but messes up some tests // ########## |
| 517 |
|
| 518 |
assert output["lifecycleRules"] == [ |
| 519 |
{ |
| 520 |
"daysFromHidingToDeleting": 1, |
| 521 |
"daysFromUploadingToHiding": None, |
| 522 |
"fileNamePrefix": "" |
| 523 |
} |
| 524 |
] |
| 525 |
|
| 526 |
|
| 527 |
def test_key_restrictions(b2_tool, bucket_name, sample_file, bucket_factory, b2_uri_args): |
| 528 |
# A single file for rm to fail on. |
| 529 |
b2_tool.should_succeed(['upload-file', '--no-progress', bucket_name, sample_file, 'test']) |
| 530 |
|
| 531 |
key_one_name = 'clt-testKey-01' + random_hex(6) |
| 532 |
created_key_stdout = b2_tool.should_succeed( |
| 533 |
[ |
| 534 |
'create-key', |
| 535 |
key_one_name, |
| 536 |
'listFiles,listBuckets,readFiles,writeKeys', |
| 537 |
] |
| 538 |
) |
| 539 |
key_one_id, key_one = created_key_stdout.split() |
| 540 |
|
| 541 |
b2_tool.should_succeed( |
| 542 |
['authorize-account', '--environment', b2_tool.realm, key_one_id, key_one], |
| 543 |
) |
| 544 |
|
| 545 |
b2_tool.should_succeed(['get-bucket', bucket_name],) |
| 546 |
second_bucket_name = bucket_factory().name |
| 547 |
b2_tool.should_succeed(['get-bucket', second_bucket_name],) |
| 548 |
|
| 549 |
key_two_name = 'clt-testKey-02' + random_hex(6) |
| 550 |
created_key_two_stdout = b2_tool.should_succeed( |
| 551 |
[ |
| 552 |
'create-key', |
| 553 |
'--bucket', |
| 554 |
bucket_name, |
| 555 |
key_two_name, |
| 556 |
'listFiles,listBuckets,readFiles', |
| 557 |
] |
| 558 |
) |
| 559 |
key_two_id, key_two = created_key_two_stdout.split() |
| 560 |
|
| 561 |
b2_tool.should_succeed( |
| 562 |
['authorize-account', '--environment', b2_tool.realm, key_two_id, key_two], |
| 563 |
) |
| 564 |
b2_tool.should_succeed(['get-bucket', bucket_name],) |
| 565 |
b2_tool.should_succeed(['ls', *b2_uri_args(bucket_name)],) |
| 566 |
|
| 567 |
# Capabilities can be listed in any order. While this regex doesn't confirm that all three are present, |
| 568 |
# in ensures that there are three in total. |
| 569 |
failed_bucket_err = r'Deletion of file "test" \([^\)]+\) failed: unauthorized for ' \ |
| 570 |
r'application key with capabilities ' \ |
| 571 |
r"'(.*listFiles.*|.*listBuckets.*|.*readFiles.*){3}', " \ |
| 572 |
r"restricted to bucket '%s' \(unauthorized\)" % bucket_name |
| 573 |
b2_tool.should_fail( |
| 574 |
['rm', '--recursive', '--no-progress', *b2_uri_args(bucket_name)], failed_bucket_err |
| 575 |
) |
| 576 |
|
| 577 |
failed_bucket_err = r'ERROR: Application key is restricted to bucket: ' + bucket_name |
| 578 |
b2_tool.should_fail(['get-bucket', second_bucket_name], failed_bucket_err) |
| 579 |
|
| 580 |
failed_list_files_err = r'ERROR: Application key is restricted to bucket: ' + bucket_name |
| 581 |
b2_tool.should_fail(['ls', *b2_uri_args(second_bucket_name)], failed_list_files_err) |
| 582 |
|
| 583 |
failed_list_files_err = r'ERROR: Application key is restricted to bucket: ' + bucket_name |
| 584 |
b2_tool.should_fail(['rm', *b2_uri_args(second_bucket_name)], failed_list_files_err) |
| 585 |
|
| 586 |
# reauthorize with more capabilities for clean up |
| 587 |
b2_tool.should_succeed( |
| 588 |
[ |
| 589 |
'authorize-account', '--environment', b2_tool.realm, b2_tool.account_id, |
| 590 |
b2_tool.application_key |
| 591 |
] |
| 592 |
) |
| 593 |
b2_tool.should_succeed(['delete-key', key_one_id]) |
| 594 |
b2_tool.should_succeed(['delete-key', key_two_id]) |
| 595 |
|
| 596 |
|
| 597 |
def test_delete_bucket(b2_tool, bucket_name): |
| 598 |
b2_tool.should_succeed(['delete-bucket', bucket_name]) |
| 599 |
b2_tool.should_fail( |
| 600 |
['delete-bucket', bucket_name], re.compile(r'^ERROR: Bucket with id=\w* not found\s*$') |
| 601 |
) |
| 602 |
|
| 603 |
|
| 604 |
def test_rapid_bucket_operations(b2_tool): |
| 605 |
new_bucket_name = b2_tool.generate_bucket_name() |
| 606 |
bucket_info_args = b2_tool.get_bucket_info_args() |
| 607 |
# apparently server behaves erratically when we delete a bucket and recreate it right away |
| 608 |
b2_tool.should_succeed(['create-bucket', new_bucket_name, 'allPrivate', *bucket_info_args]) |
| 609 |
b2_tool.should_succeed(['update-bucket', new_bucket_name, 'allPublic']) |
| 610 |
b2_tool.should_succeed(['delete-bucket', new_bucket_name]) |
| 611 |
|
| 612 |
|
| 613 |
def test_account(b2_tool, cli_version, apiver_int, monkeypatch): |
| 614 |
|
| 615 |
with monkeypatch.context() as mp: |
| 616 |
account_info_file_path = os.path.join(mkdtemp(), 'b2_account_info') |
| 617 |
mp.setenv(B2_ACCOUNT_INFO_ENV_VAR, account_info_file_path) |
| 618 |
|
| 619 |
b2_tool.should_succeed(['clear-account']) |
| 620 |
bad_application_key = random_hex(len(b2_tool.application_key)) |
| 621 |
b2_tool.should_fail( |
| 622 |
['authorize-account', b2_tool.account_id, bad_application_key], r'unauthorized' |
| 623 |
) # this call doesn't use --environment on purpose, so that we check that it is non-mandatory |
| 624 |
b2_tool.should_succeed( |
| 625 |
[ |
| 626 |
'authorize-account', |
| 627 |
'--environment', |
| 628 |
b2_tool.realm, |
| 629 |
b2_tool.account_id, |
| 630 |
b2_tool.application_key, |
| 631 |
] |
| 632 |
) |
| 633 |
|
| 634 |
# Testing (B2_APPLICATION_KEY, B2_APPLICATION_KEY_ID) for commands other than authorize-account |
| 635 |
with monkeypatch.context() as mp: |
| 636 |
account_info_file_path = os.path.join(mkdtemp(), 'b2_account_info') |
| 637 |
mp.setenv(B2_ACCOUNT_INFO_ENV_VAR, account_info_file_path) |
| 638 |
|
| 639 |
# first, let's make sure "create-bucket" doesn't work without auth data - i.e. that the sqlite file has been |
| 640 |
# successfully removed |
| 641 |
bucket_name = b2_tool.generate_bucket_name() |
| 642 |
b2_tool.should_fail( |
| 643 |
['create-bucket', bucket_name, 'allPrivate'], |
| 644 |
r'ERROR: Missing account data: \'NoneType\' object is not subscriptable (\(key 0\) )? ' |
| 645 |
fr'Use: {cli_version}(\.(exe|EXE))? authorize-account or provide auth data with \'B2_APPLICATION_KEY_ID\' and ' |
| 646 |
r'\'B2_APPLICATION_KEY\' environment variables' |
| 647 |
) |
| 648 |
|
| 649 |
with monkeypatch.context() as mp: |
| 650 |
account_info_file_path = os.path.join(mkdtemp(), 'b2_account_info') |
| 651 |
mp.setenv(B2_ACCOUNT_INFO_ENV_VAR, account_info_file_path) |
| 652 |
|
| 653 |
# then, let's see that auth data from env vars works |
| 654 |
os.environ['B2_APPLICATION_KEY'] = os.environ['B2_TEST_APPLICATION_KEY'] |
| 655 |
os.environ['B2_APPLICATION_KEY_ID'] = os.environ['B2_TEST_APPLICATION_KEY_ID'] |
| 656 |
os.environ['B2_ENVIRONMENT'] = b2_tool.realm |
| 657 |
|
| 658 |
bucket_name = b2_tool.generate_bucket_name() |
| 659 |
b2_tool.should_succeed( |
| 660 |
['create-bucket', bucket_name, 'allPrivate', *b2_tool.get_bucket_info_args()] |
| 661 |
) |
| 662 |
b2_tool.should_succeed(['delete-bucket', bucket_name]) |
| 663 |
|
| 664 |
if apiver_int >= 4: |
| 665 |
assert not os.path.exists( |
| 666 |
account_info_file_path |
| 667 |
), 'sqlite file was created while it shouldn\'t' |
| 668 |
else: |
| 669 |
assert os.path.exists(account_info_file_path), 'sqlite file was not created' |
| 670 |
account_info = SqliteAccountInfo(account_info_file_path) |
| 671 |
assert account_info.get_application_key_id() == os.environ['B2_TEST_APPLICATION_KEY_ID'] |
| 672 |
assert account_info.get_application_key() == os.environ['B2_TEST_APPLICATION_KEY'] |
| 673 |
|
| 674 |
os.environ.pop('B2_APPLICATION_KEY') |
| 675 |
os.environ.pop('B2_APPLICATION_KEY_ID') |
| 676 |
|
| 677 |
# last, let's see that providing only one of the env vars results in a failure |
| 678 |
os.environ['B2_APPLICATION_KEY'] = os.environ['B2_TEST_APPLICATION_KEY'] |
| 679 |
b2_tool.should_fail( |
| 680 |
['create-bucket', bucket_name, 'allPrivate'], |
| 681 |
r'Please provide both "B2_APPLICATION_KEY" and "B2_APPLICATION_KEY_ID" environment variables or none of them' |
| 682 |
) |
| 683 |
os.environ.pop('B2_APPLICATION_KEY') |
| 684 |
|
| 685 |
os.environ['B2_APPLICATION_KEY_ID'] = os.environ['B2_TEST_APPLICATION_KEY_ID'] |
| 686 |
b2_tool.should_fail( |
| 687 |
['create-bucket', bucket_name, 'allPrivate'], |
| 688 |
r'Please provide both "B2_APPLICATION_KEY" and "B2_APPLICATION_KEY_ID" environment variables or none of them' |
| 689 |
) |
| 690 |
os.environ.pop('B2_APPLICATION_KEY_ID') |
| 691 |
|
| 692 |
|
| 693 |
def file_version_summary(list_of_files): |
| 694 |
""" |
| 695 |
Given the result of list-file-versions, returns a list |
| 696 |
of all file versions, with "+" for upload and "-" for |
| 697 |
hide, looking like this: |
| 698 |
|
| 699 |
['+ photos/a.jpg', '- photos/b.jpg', '+ photos/c.jpg'] |
| 700 |
|
| 701 |
""" |
| 702 |
return [filename_summary(f) for f in list_of_files] |
| 703 |
|
| 704 |
|
| 705 |
def filename_summary(file_): |
| 706 |
return ('+ ' if (file_['action'] == 'upload') else '- ') + file_['fileName'] |
| 707 |
|
| 708 |
|
| 709 |
def file_version_summary_with_encryption(list_of_files): |
| 710 |
""" |
| 711 |
Given the result of list-file-versions, returns a list |
| 712 |
of all file versions, with "+" for upload and "-" for |
| 713 |
hide, with information about encryption, looking like this: |
| 714 |
|
| 715 |
[ |
| 716 |
('+ photos/a.jpg', 'SSE-C:AES256?sse_c_key_id=user-generated-key-id'), |
| 717 |
('+ photos/a.jpg', 'SSE-B2:AES256'), |
| 718 |
('- photos/b.jpg', None), |
| 719 |
('+ photos/c.jpg', 'none'), |
| 720 |
] |
| 721 |
""" |
| 722 |
result = [] |
| 723 |
for f in list_of_files: |
| 724 |
entry = filename_summary(f) |
| 725 |
encryption = encryption_summary(f['serverSideEncryption'], f['fileInfo']) |
| 726 |
result.append((entry, encryption)) |
| 727 |
return result |
| 728 |
|
| 729 |
|
| 730 |
def find_file_id(list_of_files, file_name): |
| 731 |
for file in list_of_files: |
| 732 |
if file['fileName'] == file_name: |
| 733 |
return file['fileId'] |
| 734 |
assert False, f'file not found: {file_name}' |
| 735 |
|
| 736 |
|
| 737 |
def encryption_summary(sse_dict, file_info): |
| 738 |
if isinstance(sse_dict, EncryptionSetting): |
| 739 |
sse_dict = sse_dict.as_dict() |
| 740 |
encryption = sse_dict['mode'] |
| 741 |
assert encryption in ( |
| 742 |
EncryptionMode.NONE.value, EncryptionMode.SSE_B2.value, EncryptionMode.SSE_C.value |
| 743 |
) |
| 744 |
algorithm = sse_dict.get('algorithm') |
| 745 |
if algorithm is not None: |
| 746 |
encryption += ':' + algorithm |
| 747 |
if sse_dict['mode'] == 'SSE-C': |
| 748 |
sse_c_key_id = file_info.get(SSE_C_KEY_ID_FILE_INFO_KEY_NAME) |
| 749 |
encryption += f'?{SSE_C_KEY_ID_FILE_INFO_KEY_NAME}={sse_c_key_id}' |
| 750 |
|
| 751 |
return encryption |
| 752 |
|
| 753 |
|
| 754 |
def test_sync_up(b2_tool, bucket_name): |
| 755 |
sync_up_helper(b2_tool, bucket_name, 'sync') |
| 756 |
|
| 757 |
|
| 758 |
def test_sync_up_sse_b2(b2_tool, bucket_name): |
| 759 |
sync_up_helper(b2_tool, bucket_name, 'sync', encryption=SSE_B2_AES) |
| 760 |
|
| 761 |
|
| 762 |
def test_sync_up_sse_c(b2_tool, bucket_name): |
| 763 |
sync_up_helper(b2_tool, bucket_name, 'sync', encryption=SSE_C_AES) |
| 764 |
|
| 765 |
|
| 766 |
def test_sync_up_no_prefix(b2_tool, bucket_name): |
| 767 |
sync_up_helper(b2_tool, bucket_name, '') |
| 768 |
|
| 769 |
|
| 770 |
def sync_up_helper(b2_tool, bucket_name, dir_, encryption=None): |
| 771 |
sync_point_parts = [bucket_name] |
| 772 |
if dir_: |
| 773 |
sync_point_parts.append(dir_) |
| 774 |
prefix = dir_ + '/' |
| 775 |
else: |
| 776 |
prefix = '' |
| 777 |
b2_sync_point = 'b2:' + '/'.join(sync_point_parts) |
| 778 |
|
| 779 |
with TempDir() as dir_path: |
| 780 |
file_versions = b2_tool.list_file_versions(bucket_name) |
| 781 |
should_equal([], file_version_summary(file_versions)) |
| 782 |
|
| 783 |
write_file(dir_path / 'a', b'hello') |
| 784 |
write_file(dir_path / 'b', b'hello') |
| 785 |
write_file(dir_path / 'c', b'hello') |
| 786 |
|
| 787 |
# simulate action (nothing should be uploaded) |
| 788 |
b2_tool.should_succeed(['sync', '--no-progress', '--dry-run', dir_path, b2_sync_point]) |
| 789 |
file_versions = b2_tool.list_file_versions(bucket_name) |
| 790 |
should_equal([], file_version_summary(file_versions)) |
| 791 |
|
| 792 |
# |
| 793 |
# A note about OSError: [WinError 1314] |
| 794 |
# |
| 795 |
# If you are seeing this, then probably you ran the integration test suite from |
| 796 |
# a non-admin account which on Windows doesn't by default get to create symlinks. |
| 797 |
# A special permission is needed. Now maybe there is a way to give that permission, |
| 798 |
# but it didn't work for me, so I just ran it as admin. A guide that I've found |
| 799 |
# recommended to go to Control Panel, Administrative Tools, Local Security Policy, |
| 800 |
# Local Policies, User Rights Assignment and there you can find a permission to |
| 801 |
# create symbilic links. Add your user to it (or a group that the user is in). |
| 802 |
# |
| 803 |
# Finally in order to apply the new policy, run `cmd` and execute |
| 804 |
# ``gpupdate /force``. |
| 805 |
# |
| 806 |
# Again, if it still doesn't work, consider just running the shell you are |
| 807 |
# launching ``nox`` as admin. |
| 808 |
|
| 809 |
os.symlink('broken', dir_path / 'd') # OSError: [WinError 1314] ? See the comment above |
| 810 |
|
| 811 |
additional_env = None |
| 812 |
|
| 813 |
# now upload |
| 814 |
if encryption is None: |
| 815 |
command = ['sync', '--no-progress', dir_path, b2_sync_point] |
| 816 |
expected_encryption = SSE_NONE |
| 817 |
expected_encryption_str = encryption_summary(expected_encryption.as_dict(), {}) |
| 818 |
elif encryption == SSE_B2_AES: |
| 819 |
command = [ |
| 820 |
'sync', '--no-progress', '--destination-server-side-encryption', 'SSE-B2', dir_path, |
| 821 |
b2_sync_point |
| 822 |
] |
| 823 |
expected_encryption = encryption |
| 824 |
expected_encryption_str = encryption_summary(expected_encryption.as_dict(), {}) |
| 825 |
elif encryption == SSE_C_AES: |
| 826 |
command = [ |
| 827 |
'sync', '--no-progress', '--destination-server-side-encryption', 'SSE-C', dir_path, |
| 828 |
b2_sync_point |
| 829 |
] |
| 830 |
expected_encryption = encryption |
| 831 |
additional_env = { |
| 832 |
'B2_DESTINATION_SSE_C_KEY_B64': base64.b64encode(SSE_C_AES.key.secret).decode(), |
| 833 |
'B2_DESTINATION_SSE_C_KEY_ID': SSE_C_AES.key.key_id, |
| 834 |
} |
| 835 |
expected_encryption_str = encryption_summary( |
| 836 |
expected_encryption.as_dict(), |
| 837 |
{SSE_C_KEY_ID_FILE_INFO_KEY_NAME: SSE_C_AES.key.key_id} |
| 838 |
) |
| 839 |
else: |
| 840 |
raise NotImplementedError('unsupported encryption mode: %s' % encryption) |
| 841 |
|
| 842 |
b2_tool.should_succeed( |
| 843 |
command, expected_pattern="d could not be accessed", additional_env=additional_env |
| 844 |
) |
| 845 |
file_versions = b2_tool.list_file_versions(bucket_name) |
| 846 |
|
| 847 |
should_equal( |
| 848 |
[ |
| 849 |
('+ ' + prefix + 'a', expected_encryption_str), |
| 850 |
('+ ' + prefix + 'b', expected_encryption_str), |
| 851 |
('+ ' + prefix + 'c', expected_encryption_str), |
| 852 |
], |
| 853 |
file_version_summary_with_encryption(file_versions), |
| 854 |
) |
| 855 |
if encryption and encryption.mode == EncryptionMode.SSE_C: |
| 856 |
b2_tool.should_fail( |
| 857 |
command, |
| 858 |
expected_pattern="ValueError: Using SSE-C requires providing an encryption key via " |
| 859 |
"B2_DESTINATION_SSE_C_KEY_B64 env var" |
| 860 |
) |
| 861 |
if encryption is not None: |
| 862 |
return # that's enough, we've checked that encryption works, no need to repeat the whole sync suite |
| 863 |
|
| 864 |
c_id = find_file_id(file_versions, prefix + 'c') |
| 865 |
file_info = b2_tool.should_succeed_json(['file-info', f"b2id://{c_id}"])['fileInfo'] |
| 866 |
should_equal( |
| 867 |
file_mod_time_millis(dir_path / 'c'), int(file_info['src_last_modified_millis']) |
| 868 |
) |
| 869 |
|
| 870 |
os.unlink(dir_path / 'b') |
| 871 |
write_file(dir_path / 'c', b'hello world') |
| 872 |
|
| 873 |
b2_tool.should_succeed( |
| 874 |
['sync', '--no-progress', '--keep-days', '10', dir_path, b2_sync_point] |
| 875 |
) |
| 876 |
file_versions = b2_tool.list_file_versions(bucket_name) |
| 877 |
should_equal( |
| 878 |
[ |
| 879 |
'+ ' + prefix + 'a', |
| 880 |
'- ' + prefix + 'b', |
| 881 |
'+ ' + prefix + 'b', |
| 882 |
'+ ' + prefix + 'c', |
| 883 |
'+ ' + prefix + 'c', |
| 884 |
], file_version_summary(file_versions) |
| 885 |
) |
| 886 |
|
| 887 |
os.unlink(dir_path / 'a') |
| 888 |
|
| 889 |
b2_tool.should_succeed(['sync', '--no-progress', '--delete', dir_path, b2_sync_point]) |
| 890 |
file_versions = b2_tool.list_file_versions(bucket_name) |
| 891 |
should_equal([ |
| 892 |
'+ ' + prefix + 'c', |
| 893 |
], file_version_summary(file_versions)) |
| 894 |
|
| 895 |
# test --compare-threshold with file size |
| 896 |
write_file(dir_path / 'c', b'hello world!') |
| 897 |
|
| 898 |
# should not upload new version of c |
| 899 |
b2_tool.should_succeed( |
| 900 |
[ |
| 901 |
'sync', '--no-progress', '--keep-days', '10', '--compare-versions', 'size', |
| 902 |
'--compare-threshold', '1', dir_path, b2_sync_point |
| 903 |
] |
| 904 |
) |
| 905 |
file_versions = b2_tool.list_file_versions(bucket_name) |
| 906 |
should_equal([ |
| 907 |
'+ ' + prefix + 'c', |
| 908 |
], file_version_summary(file_versions)) |
| 909 |
|
| 910 |
# should upload new version of c |
| 911 |
b2_tool.should_succeed( |
| 912 |
[ |
| 913 |
'sync', '--no-progress', '--keep-days', '10', '--compare-versions', 'size', |
| 914 |
dir_path, b2_sync_point |
| 915 |
] |
| 916 |
) |
| 917 |
file_versions = b2_tool.list_file_versions(bucket_name) |
| 918 |
should_equal( |
| 919 |
[ |
| 920 |
'+ ' + prefix + 'c', |
| 921 |
'+ ' + prefix + 'c', |
| 922 |
], file_version_summary(file_versions) |
| 923 |
) |
| 924 |
|
| 925 |
set_file_mod_time_millis(dir_path / 'c', file_mod_time_millis(dir_path / 'c') + 2000) |
| 926 |
|
| 927 |
# test --compare-threshold with modTime |
| 928 |
# should not upload new version of c |
| 929 |
b2_tool.should_succeed( |
| 930 |
[ |
| 931 |
'sync', '--no-progress', '--keep-days', '10', '--compare-versions', 'modTime', |
| 932 |
'--compare-threshold', '2000', dir_path, b2_sync_point |
| 933 |
] |
| 934 |
) |
| 935 |
file_versions = b2_tool.list_file_versions(bucket_name) |
| 936 |
should_equal( |
| 937 |
[ |
| 938 |
'+ ' + prefix + 'c', |
| 939 |
'+ ' + prefix + 'c', |
| 940 |
], file_version_summary(file_versions) |
| 941 |
) |
| 942 |
|
| 943 |
# should upload new version of c |
| 944 |
b2_tool.should_succeed( |
| 945 |
[ |
| 946 |
'sync', '--no-progress', '--keep-days', '10', '--compare-versions', 'modTime', |
| 947 |
dir_path, b2_sync_point |
| 948 |
] |
| 949 |
) |
| 950 |
file_versions = b2_tool.list_file_versions(bucket_name) |
| 951 |
should_equal( |
| 952 |
[ |
| 953 |
'+ ' + prefix + 'c', |
| 954 |
'+ ' + prefix + 'c', |
| 955 |
'+ ' + prefix + 'c', |
| 956 |
], file_version_summary(file_versions) |
| 957 |
) |
| 958 |
|
| 959 |
# create one more file |
| 960 |
write_file(dir_path / 'linktarget', b'hello') |
| 961 |
mod_time = str((file_mod_time_millis(dir_path / 'linktarget') - 10) / 1000) |
| 962 |
|
| 963 |
# exclude last created file because of mtime |
| 964 |
b2_tool.should_succeed( |
| 965 |
[ |
| 966 |
'sync', '--no-progress', '--exclude-if-modified-after', mod_time, dir_path, |
| 967 |
b2_sync_point |
| 968 |
] |
| 969 |
) |
| 970 |
file_versions = b2_tool.list_file_versions(bucket_name) |
| 971 |
should_equal( |
| 972 |
[ |
| 973 |
'+ ' + prefix + 'c', |
| 974 |
'+ ' + prefix + 'c', |
| 975 |
'+ ' + prefix + 'c', |
| 976 |
], |
| 977 |
file_version_summary(file_versions), |
| 978 |
) |
| 979 |
|
| 980 |
# confirm symlink is skipped |
| 981 |
os.symlink('linktarget', dir_path / 'alink') |
| 982 |
|
| 983 |
b2_tool.should_succeed( |
| 984 |
['sync', '--no-progress', '--exclude-all-symlinks', dir_path, b2_sync_point], |
| 985 |
) |
| 986 |
file_versions = b2_tool.list_file_versions(bucket_name) |
| 987 |
should_equal( |
| 988 |
[ |
| 989 |
'+ ' + prefix + 'c', |
| 990 |
'+ ' + prefix + 'c', |
| 991 |
'+ ' + prefix + 'c', |
| 992 |
'+ ' + prefix + 'linktarget', |
| 993 |
], |
| 994 |
file_version_summary(file_versions), |
| 995 |
) |
| 996 |
|
| 997 |
# confirm symlink target is uploaded (with symlink's name) |
| 998 |
b2_tool.should_succeed(['sync', '--no-progress', dir_path, b2_sync_point]) |
| 999 |
file_versions = b2_tool.list_file_versions(bucket_name) |
| 1000 |
should_equal( |
| 1001 |
[ |
| 1002 |
'+ ' + prefix + 'alink', |
| 1003 |
'+ ' + prefix + 'c', |
| 1004 |
'+ ' + prefix + 'c', |
| 1005 |
'+ ' + prefix + 'c', |
| 1006 |
'+ ' + prefix + 'linktarget', |
| 1007 |
], |
| 1008 |
file_version_summary(file_versions), |
| 1009 |
) |
| 1010 |
|
| 1011 |
|
| 1012 |
def test_sync_down(b2_tool, bucket_name, sample_file): |
| 1013 |
sync_down_helper(b2_tool, bucket_name, 'sync', sample_file) |
| 1014 |
|
| 1015 |
|
| 1016 |
def test_sync_down_no_prefix(b2_tool, bucket_name, sample_file): |
| 1017 |
sync_down_helper(b2_tool, bucket_name, '', sample_file) |
| 1018 |
|
| 1019 |
|
| 1020 |
def test_sync_down_sse_c_no_prefix(b2_tool, bucket_name, sample_file): |
| 1021 |
sync_down_helper(b2_tool, bucket_name, '', sample_file, SSE_C_AES) |
| 1022 |
|
| 1023 |
|
| 1024 |
def sync_down_helper(b2_tool, bucket_name, folder_in_bucket, sample_file, encryption=None): |
| 1025 |
|
| 1026 |
b2_sync_point = 'b2:%s' % bucket_name |
| 1027 |
if folder_in_bucket: |
| 1028 |
b2_sync_point += '/' + folder_in_bucket |
| 1029 |
b2_file_prefix = folder_in_bucket + '/' |
| 1030 |
else: |
| 1031 |
b2_file_prefix = '' |
| 1032 |
|
| 1033 |
if encryption is None or encryption.mode in (EncryptionMode.NONE, EncryptionMode.SSE_B2): |
| 1034 |
upload_encryption_args = [] |
| 1035 |
upload_additional_env = {} |
| 1036 |
sync_encryption_args = [] |
| 1037 |
sync_additional_env = {} |
| 1038 |
elif encryption.mode == EncryptionMode.SSE_C: |
| 1039 |
upload_encryption_args = ['--destination-server-side-encryption', 'SSE-C'] |
| 1040 |
upload_additional_env = { |
| 1041 |
'B2_DESTINATION_SSE_C_KEY_B64': base64.b64encode(encryption.key.secret).decode(), |
| 1042 |
'B2_DESTINATION_SSE_C_KEY_ID': encryption.key.key_id, |
| 1043 |
} |
| 1044 |
sync_encryption_args = ['--source-server-side-encryption', 'SSE-C'] |
| 1045 |
sync_additional_env = { |
| 1046 |
'B2_SOURCE_SSE_C_KEY_B64': base64.b64encode(encryption.key.secret).decode(), |
| 1047 |
'B2_SOURCE_SSE_C_KEY_ID': encryption.key.key_id, |
| 1048 |
} |
| 1049 |
else: |
| 1050 |
raise NotImplementedError(encryption) |
| 1051 |
|
| 1052 |
with TempDir() as local_path: |
| 1053 |
# Sync from an empty "folder" as a source. |
| 1054 |
b2_tool.should_succeed(['sync', b2_sync_point, local_path]) |
| 1055 |
should_equal([], sorted(local_path.iterdir())) |
| 1056 |
|
| 1057 |
# Put a couple files in B2 |
| 1058 |
b2_tool.should_succeed( |
| 1059 |
['upload-file', '--no-progress', bucket_name, sample_file, b2_file_prefix + 'a'] + |
| 1060 |
upload_encryption_args, |
| 1061 |
additional_env=upload_additional_env, |
| 1062 |
) |
| 1063 |
b2_tool.should_succeed( |
| 1064 |
['upload-file', '--no-progress', bucket_name, sample_file, b2_file_prefix + 'b'] + |
| 1065 |
upload_encryption_args, |
| 1066 |
additional_env=upload_additional_env, |
| 1067 |
) |
| 1068 |
b2_tool.should_succeed( |
| 1069 |
['sync', b2_sync_point, local_path] + sync_encryption_args, |
| 1070 |
additional_env=sync_additional_env, |
| 1071 |
) |
| 1072 |
should_equal(['a', 'b'], sorted(os.listdir(local_path))) |
| 1073 |
|
| 1074 |
b2_tool.should_succeed( |
| 1075 |
['upload-file', '--no-progress', bucket_name, sample_file, b2_file_prefix + 'c'] + |
| 1076 |
upload_encryption_args, |
| 1077 |
additional_env=upload_additional_env, |
| 1078 |
) |
| 1079 |
|
| 1080 |
# Sync the files with one file being excluded because of mtime |
| 1081 |
mod_time = str((file_mod_time_millis(sample_file) - 10) / 1000) |
| 1082 |
b2_tool.should_succeed( |
| 1083 |
[ |
| 1084 |
'sync', '--no-progress', '--exclude-if-modified-after', mod_time, b2_sync_point, |
| 1085 |
local_path |
| 1086 |
] + sync_encryption_args, |
| 1087 |
additional_env=sync_additional_env, |
| 1088 |
) |
| 1089 |
should_equal(['a', 'b'], sorted(os.listdir(local_path))) |
| 1090 |
# Sync all the files |
| 1091 |
b2_tool.should_succeed( |
| 1092 |
['sync', '--no-progress', b2_sync_point, local_path] + sync_encryption_args, |
| 1093 |
additional_env=sync_additional_env, |
| 1094 |
) |
| 1095 |
should_equal(['a', 'b', 'c'], sorted(os.listdir(local_path))) |
| 1096 |
with TempDir() as new_local_path: |
| 1097 |
if encryption and encryption.mode == EncryptionMode.SSE_C: |
| 1098 |
b2_tool.should_fail( |
| 1099 |
['sync', '--no-progress', b2_sync_point, new_local_path] + sync_encryption_args, |
| 1100 |
expected_pattern='ValueError: Using SSE-C requires providing an encryption key via ' |
| 1101 |
'B2_SOURCE_SSE_C_KEY_B64 env var', |
| 1102 |
) |
| 1103 |
b2_tool.should_fail( |
| 1104 |
['sync', '--no-progress', b2_sync_point, new_local_path], |
| 1105 |
expected_pattern= |
| 1106 |
'b2sdk._internal.exception.BadRequest: The object was stored using a form of Server Side ' |
| 1107 |
'Encryption. The correct parameters must be provided to retrieve the object. ' |
| 1108 |
r'\(bad_request\)', |
| 1109 |
) |
| 1110 |
|
| 1111 |
|
| 1112 |
def test_sync_copy(bucket_factory, b2_tool, bucket_name, sample_file): |
| 1113 |
prepare_and_run_sync_copy_tests( |
| 1114 |
bucket_factory, b2_tool, bucket_name, 'sync', sample_file=sample_file |
| 1115 |
) |
| 1116 |
|
| 1117 |
|
| 1118 |
def test_sync_copy_no_prefix_default_encryption(bucket_factory, b2_tool, bucket_name, sample_file): |
| 1119 |
prepare_and_run_sync_copy_tests( |
| 1120 |
bucket_factory, |
| 1121 |
b2_tool, |
| 1122 |
bucket_name, |
| 1123 |
'', |
| 1124 |
sample_file=sample_file, |
| 1125 |
destination_encryption=None, |
| 1126 |
expected_encryption=SSE_NONE |
| 1127 |
) |
| 1128 |
|
| 1129 |
|
| 1130 |
def test_sync_copy_no_prefix_no_encryption(bucket_factory, b2_tool, bucket_name, sample_file): |
| 1131 |
prepare_and_run_sync_copy_tests( |
| 1132 |
bucket_factory, |
| 1133 |
b2_tool, |
| 1134 |
bucket_name, |
| 1135 |
'', |
| 1136 |
sample_file=sample_file, |
| 1137 |
destination_encryption=SSE_NONE, |
| 1138 |
expected_encryption=SSE_NONE |
| 1139 |
) |
| 1140 |
|
| 1141 |
|
| 1142 |
def test_sync_copy_no_prefix_sse_b2(bucket_factory, b2_tool, bucket_name, sample_file): |
| 1143 |
prepare_and_run_sync_copy_tests( |
| 1144 |
bucket_factory, |
| 1145 |
b2_tool, |
| 1146 |
bucket_name, |
| 1147 |
'', |
| 1148 |
sample_file=sample_file, |
| 1149 |
destination_encryption=SSE_B2_AES, |
| 1150 |
expected_encryption=SSE_B2_AES, |
| 1151 |
) |
| 1152 |
|
| 1153 |
|
| 1154 |
def test_sync_copy_no_prefix_sse_c(bucket_factory, b2_tool, bucket_name, sample_file): |
| 1155 |
prepare_and_run_sync_copy_tests( |
| 1156 |
bucket_factory, |
| 1157 |
b2_tool, |
| 1158 |
bucket_name, |
| 1159 |
'', |
| 1160 |
sample_file=sample_file, |
| 1161 |
destination_encryption=SSE_C_AES, |
| 1162 |
expected_encryption=SSE_C_AES, |
| 1163 |
source_encryption=SSE_C_AES_2, |
| 1164 |
) |
| 1165 |
|
| 1166 |
|
| 1167 |
def test_sync_copy_sse_c_single_bucket(b2_tool, bucket_name, sample_file): |
| 1168 |
run_sync_copy_with_basic_checks( |
| 1169 |
b2_tool=b2_tool, |
| 1170 |
b2_file_prefix='first_folder/', |
| 1171 |
b2_sync_point=f'b2:{bucket_name}/first_folder', |
| 1172 |
bucket_name=bucket_name, |
| 1173 |
other_b2_sync_point=f'b2:{bucket_name}/second_folder', |
| 1174 |
destination_encryption=SSE_C_AES_2, |
| 1175 |
source_encryption=SSE_C_AES, |
| 1176 |
sample_file=sample_file, |
| 1177 |
) |
| 1178 |
expected_encryption_first = encryption_summary( |
| 1179 |
SSE_C_AES.as_dict(), |
| 1180 |
{SSE_C_KEY_ID_FILE_INFO_KEY_NAME: SSE_C_AES.key.key_id}, |
| 1181 |
) |
| 1182 |
expected_encryption_second = encryption_summary( |
| 1183 |
SSE_C_AES_2.as_dict(), |
| 1184 |
{SSE_C_KEY_ID_FILE_INFO_KEY_NAME: SSE_C_AES_2.key.key_id}, |
| 1185 |
) |
| 1186 |
|
| 1187 |
file_versions = b2_tool.list_file_versions(bucket_name) |
| 1188 |
should_equal( |
| 1189 |
[ |
| 1190 |
('+ first_folder/a', expected_encryption_first), |
| 1191 |
('+ first_folder/b', expected_encryption_first), |
| 1192 |
('+ second_folder/a', expected_encryption_second), |
| 1193 |
('+ second_folder/b', expected_encryption_second), |
| 1194 |
], |
| 1195 |
file_version_summary_with_encryption(file_versions), |
| 1196 |
) |
| 1197 |
|
| 1198 |
|
| 1199 |
def prepare_and_run_sync_copy_tests( |
| 1200 |
bucket_factory, |
| 1201 |
b2_tool, |
| 1202 |
bucket_name, |
| 1203 |
folder_in_bucket, |
| 1204 |
sample_file, |
| 1205 |
destination_encryption=None, |
| 1206 |
expected_encryption=SSE_NONE, |
| 1207 |
source_encryption=None, |
| 1208 |
): |
| 1209 |
b2_sync_point = 'b2:%s' % bucket_name |
| 1210 |
if folder_in_bucket: |
| 1211 |
b2_sync_point += '/' + folder_in_bucket |
| 1212 |
b2_file_prefix = folder_in_bucket + '/' |
| 1213 |
else: |
| 1214 |
b2_file_prefix = '' |
| 1215 |
|
| 1216 |
other_bucket_name = bucket_factory().name |
| 1217 |
|
| 1218 |
other_b2_sync_point = 'b2:%s' % other_bucket_name |
| 1219 |
if folder_in_bucket: |
| 1220 |
other_b2_sync_point += '/' + folder_in_bucket |
| 1221 |
|
| 1222 |
run_sync_copy_with_basic_checks( |
| 1223 |
b2_tool=b2_tool, |
| 1224 |
b2_file_prefix=b2_file_prefix, |
| 1225 |
b2_sync_point=b2_sync_point, |
| 1226 |
bucket_name=bucket_name, |
| 1227 |
other_b2_sync_point=other_b2_sync_point, |
| 1228 |
destination_encryption=destination_encryption, |
| 1229 |
source_encryption=source_encryption, |
| 1230 |
sample_file=sample_file, |
| 1231 |
) |
| 1232 |
|
| 1233 |
if destination_encryption is None or destination_encryption in (SSE_NONE, SSE_B2_AES): |
| 1234 |
encryption_file_info = {} |
| 1235 |
elif destination_encryption.mode == EncryptionMode.SSE_C: |
| 1236 |
encryption_file_info = {SSE_C_KEY_ID_FILE_INFO_KEY_NAME: destination_encryption.key.key_id} |
| 1237 |
else: |
| 1238 |
raise NotImplementedError(destination_encryption) |
| 1239 |
|
| 1240 |
file_versions = b2_tool.list_file_versions(other_bucket_name) |
| 1241 |
expected_encryption_str = encryption_summary( |
| 1242 |
expected_encryption.as_dict(), encryption_file_info |
| 1243 |
) |
| 1244 |
should_equal( |
| 1245 |
[ |
| 1246 |
('+ ' + b2_file_prefix + 'a', expected_encryption_str), |
| 1247 |
('+ ' + b2_file_prefix + 'b', expected_encryption_str), |
| 1248 |
], |
| 1249 |
file_version_summary_with_encryption(file_versions), |
| 1250 |
) |
| 1251 |
|
| 1252 |
|
| 1253 |
def run_sync_copy_with_basic_checks( |
| 1254 |
b2_tool, |
| 1255 |
b2_file_prefix, |
| 1256 |
b2_sync_point, |
| 1257 |
bucket_name, |
| 1258 |
other_b2_sync_point, |
| 1259 |
destination_encryption, |
| 1260 |
source_encryption, |
| 1261 |
sample_file, |
| 1262 |
): |
| 1263 |
# Put a couple files in B2 |
| 1264 |
if source_encryption is None or source_encryption.mode in ( |
| 1265 |
EncryptionMode.NONE, EncryptionMode.SSE_B2 |
| 1266 |
): |
| 1267 |
b2_tool.should_succeed( |
| 1268 |
[ |
| 1269 |
'upload-file', '--no-progress', '--destination-server-side-encryption', 'SSE-B2', |
| 1270 |
bucket_name, sample_file, b2_file_prefix + 'a' |
| 1271 |
] |
| 1272 |
) |
| 1273 |
b2_tool.should_succeed( |
| 1274 |
['upload-file', '--no-progress', bucket_name, sample_file, b2_file_prefix + 'b'] |
| 1275 |
) |
| 1276 |
elif source_encryption.mode == EncryptionMode.SSE_C: |
| 1277 |
for suffix in ['a', 'b']: |
| 1278 |
b2_tool.should_succeed( |
| 1279 |
[ |
| 1280 |
'upload-file', '--no-progress', '--destination-server-side-encryption', 'SSE-C', |
| 1281 |
bucket_name, sample_file, b2_file_prefix + suffix |
| 1282 |
], |
| 1283 |
additional_env={ |
| 1284 |
'B2_DESTINATION_SSE_C_KEY_B64': |
| 1285 |
base64.b64encode(source_encryption.key.secret).decode(), |
| 1286 |
'B2_DESTINATION_SSE_C_KEY_ID': |
| 1287 |
source_encryption.key.key_id, |
| 1288 |
}, |
| 1289 |
) |
| 1290 |
else: |
| 1291 |
raise NotImplementedError(source_encryption) |
| 1292 |
|
| 1293 |
# Sync all the files |
| 1294 |
if destination_encryption is None or destination_encryption == SSE_NONE: |
| 1295 |
b2_tool.should_succeed(['sync', '--no-progress', b2_sync_point, other_b2_sync_point]) |
| 1296 |
elif destination_encryption == SSE_B2_AES: |
| 1297 |
b2_tool.should_succeed( |
| 1298 |
[ |
| 1299 |
'sync', '--no-progress', '--destination-server-side-encryption', |
| 1300 |
destination_encryption.mode.value, b2_sync_point, other_b2_sync_point |
| 1301 |
] |
| 1302 |
) |
| 1303 |
elif destination_encryption.mode == EncryptionMode.SSE_C: |
| 1304 |
b2_tool.should_fail( |
| 1305 |
[ |
| 1306 |
'sync', '--no-progress', '--destination-server-side-encryption', |
| 1307 |
destination_encryption.mode.value, b2_sync_point, other_b2_sync_point |
| 1308 |
], |
| 1309 |
additional_env={ |
| 1310 |
'B2_DESTINATION_SSE_C_KEY_B64': |
| 1311 |
base64.b64encode(destination_encryption.key.secret).decode(), |
| 1312 |
'B2_DESTINATION_SSE_C_KEY_ID': |
| 1313 |
destination_encryption.key.key_id, |
| 1314 |
}, |
| 1315 |
expected_pattern= |
| 1316 |
'b2sdk._internal.exception.BadRequest: The object was stored using a form of Server Side ' |
| 1317 |
'Encryption. The correct parameters must be provided to retrieve the object. ' |
| 1318 |
r'\(bad_request\)' |
| 1319 |
) |
| 1320 |
b2_tool.should_succeed( |
| 1321 |
[ |
| 1322 |
'sync', '--no-progress', '--destination-server-side-encryption', |
| 1323 |
destination_encryption.mode.value, '--source-server-side-encryption', |
| 1324 |
source_encryption.mode.value, b2_sync_point, other_b2_sync_point |
| 1325 |
], |
| 1326 |
additional_env={ |
| 1327 |
'B2_DESTINATION_SSE_C_KEY_B64': |
| 1328 |
base64.b64encode(destination_encryption.key.secret).decode(), |
| 1329 |
'B2_DESTINATION_SSE_C_KEY_ID': |
| 1330 |
destination_encryption.key.key_id, |
| 1331 |
'B2_SOURCE_SSE_C_KEY_B64': |
| 1332 |
base64.b64encode(source_encryption.key.secret).decode(), |
| 1333 |
'B2_SOURCE_SSE_C_KEY_ID': |
| 1334 |
source_encryption.key.key_id, |
| 1335 |
} |
| 1336 |
) |
| 1337 |
|
| 1338 |
else: |
| 1339 |
raise NotImplementedError(destination_encryption) |
| 1340 |
|
| 1341 |
|
| 1342 |
def test_sync_long_path(b2_tool, bucket_name): |
| 1343 |
""" |
| 1344 |
test sync with very long path (overcome windows 260 character limit) |
| 1345 |
""" |
| 1346 |
b2_sync_point = 'b2://' + bucket_name |
| 1347 |
|
| 1348 |
long_path = '/'.join( |
| 1349 |
( |
| 1350 |
'extremely_long_path_which_exceeds_windows_unfortunate_260_character_path_limit', |
| 1351 |
'and_needs_special_prefixes_containing_backslashes_added_to_overcome_this_limitation', |
| 1352 |
'when_doing_so_beware_leaning_toothpick_syndrome_as_it_can_cause_frustration', |
| 1353 |
'see_also_xkcd_1638' |
| 1354 |
) |
| 1355 |
) |
| 1356 |
|
| 1357 |
with TempDir() as dir_path: |
| 1358 |
local_long_path = (dir_path / long_path).resolve() |
| 1359 |
fixed_local_long_path = Path(fix_windows_path_limit(str(local_long_path))) |
| 1360 |
os.makedirs(fixed_local_long_path.parent) |
| 1361 |
write_file(fixed_local_long_path, b'asdf') |
| 1362 |
|
| 1363 |
b2_tool.should_succeed(['sync', '--no-progress', '--delete', dir_path, b2_sync_point]) |
| 1364 |
file_versions = b2_tool.list_file_versions(bucket_name) |
| 1365 |
should_equal(['+ ' + long_path], file_version_summary(file_versions)) |
| 1366 |
|
| 1367 |
|
| 1368 |
def test_default_sse_b2__update_bucket(b2_tool, bucket_name, schedule_bucket_cleanup): |
| 1369 |
# Set default encryption via update-bucket |
| 1370 |
bucket_info = b2_tool.should_succeed_json(['get-bucket', bucket_name]) |
| 1371 |
bucket_default_sse = {'mode': 'none'} |
| 1372 |
should_equal(bucket_default_sse, bucket_info['defaultServerSideEncryption']) |
| 1373 |
|
| 1374 |
bucket_info = b2_tool.should_succeed_json( |
| 1375 |
['update-bucket', '--default-server-side-encryption=SSE-B2', bucket_name] |
| 1376 |
) |
| 1377 |
bucket_default_sse = { |
| 1378 |
'algorithm': 'AES256', |
| 1379 |
'mode': 'SSE-B2', |
| 1380 |
} |
| 1381 |
should_equal(bucket_default_sse, bucket_info['defaultServerSideEncryption']) |
| 1382 |
|
| 1383 |
bucket_info = b2_tool.should_succeed_json(['get-bucket', bucket_name]) |
| 1384 |
bucket_default_sse = { |
| 1385 |
'algorithm': 'AES256', |
| 1386 |
'mode': 'SSE-B2', |
| 1387 |
} |
| 1388 |
should_equal(bucket_default_sse, bucket_info['defaultServerSideEncryption']) |
| 1389 |
|
| 1390 |
|
| 1391 |
def test_default_sse_b2__create_bucket(b2_tool, schedule_bucket_cleanup): |
| 1392 |
# Set default encryption via create-bucket |
| 1393 |
second_bucket_name = b2_tool.generate_bucket_name() |
| 1394 |
schedule_bucket_cleanup(second_bucket_name) |
| 1395 |
b2_tool.should_succeed( |
| 1396 |
[ |
| 1397 |
'create-bucket', |
| 1398 |
'--default-server-side-encryption=SSE-B2', |
| 1399 |
second_bucket_name, |
| 1400 |
'allPublic', |
| 1401 |
*b2_tool.get_bucket_info_args(), |
| 1402 |
] |
| 1403 |
) |
| 1404 |
second_bucket_info = b2_tool.should_succeed_json(['get-bucket', second_bucket_name]) |
| 1405 |
second_bucket_default_sse = { |
| 1406 |
'algorithm': 'AES256', |
| 1407 |
'mode': 'SSE-B2', |
| 1408 |
} |
| 1409 |
should_equal(second_bucket_default_sse, second_bucket_info['defaultServerSideEncryption']) |
| 1410 |
|
| 1411 |
|
| 1412 |
def test_sse_b2(b2_tool, bucket_name, sample_file, tmp_path, b2_uri_args): |
| 1413 |
b2_tool.should_succeed( |
| 1414 |
[ |
| 1415 |
'upload-file', '--destination-server-side-encryption=SSE-B2', '--quiet', bucket_name, |
| 1416 |
sample_file, 'encrypted' |
| 1417 |
] |
| 1418 |
) |
| 1419 |
b2_tool.should_succeed(['upload-file', '--quiet', bucket_name, sample_file, 'not_encrypted']) |
| 1420 |
|
| 1421 |
b2_tool.should_succeed( |
| 1422 |
['download-file', '--quiet', f'b2://{bucket_name}/encrypted', tmp_path / 'encrypted'] |
| 1423 |
) |
| 1424 |
b2_tool.should_succeed( |
| 1425 |
[ |
| 1426 |
'download-file', '--quiet', f'b2://{bucket_name}/not_encrypted', |
| 1427 |
tmp_path / 'not_encrypted' |
| 1428 |
] |
| 1429 |
) |
| 1430 |
|
| 1431 |
list_of_files = b2_tool.should_succeed_json( |
| 1432 |
['ls', '--json', '--recursive', *b2_uri_args(bucket_name)] |
| 1433 |
) |
| 1434 |
should_equal( |
| 1435 |
[{ |
| 1436 |
'algorithm': 'AES256', |
| 1437 |
'mode': 'SSE-B2' |
| 1438 |
}, { |
| 1439 |
'mode': 'none' |
| 1440 |
}], [f['serverSideEncryption'] for f in list_of_files] |
| 1441 |
) |
| 1442 |
|
| 1443 |
encrypted_version = list_of_files[0] |
| 1444 |
file_info = b2_tool.should_succeed_json(['file-info', f"b2id://{encrypted_version['fileId']}"]) |
| 1445 |
should_equal({'algorithm': 'AES256', 'mode': 'SSE-B2'}, file_info['serverSideEncryption']) |
| 1446 |
not_encrypted_version = list_of_files[1] |
| 1447 |
file_info = b2_tool.should_succeed_json( |
| 1448 |
['file-info', f"b2id://{not_encrypted_version['fileId']}"] |
| 1449 |
) |
| 1450 |
should_equal({'mode': 'none'}, file_info['serverSideEncryption']) |
| 1451 |
|
| 1452 |
b2_tool.should_succeed( |
| 1453 |
[ |
| 1454 |
'copy-file-by-id', '--destination-server-side-encryption=SSE-B2', |
| 1455 |
encrypted_version['fileId'], bucket_name, 'copied_encrypted' |
| 1456 |
] |
| 1457 |
) |
| 1458 |
b2_tool.should_succeed( |
| 1459 |
['copy-file-by-id', not_encrypted_version['fileId'], bucket_name, 'copied_not_encrypted'] |
| 1460 |
) |
| 1461 |
|
| 1462 |
list_of_files = b2_tool.should_succeed_json( |
| 1463 |
['ls', '--json', '--recursive', *b2_uri_args(bucket_name)] |
| 1464 |
) |
| 1465 |
should_equal( |
| 1466 |
[{ |
| 1467 |
'algorithm': 'AES256', |
| 1468 |
'mode': 'SSE-B2' |
| 1469 |
}, { |
| 1470 |
'mode': 'none' |
| 1471 |
}] * 2, [f['serverSideEncryption'] for f in list_of_files] |
| 1472 |
) |
| 1473 |
|
| 1474 |
copied_encrypted_version = list_of_files[2] |
| 1475 |
file_info = b2_tool.should_succeed_json( |
| 1476 |
['file-info', f"b2id://{copied_encrypted_version['fileId']}"] |
| 1477 |
) |
| 1478 |
should_equal({'algorithm': 'AES256', 'mode': 'SSE-B2'}, file_info['serverSideEncryption']) |
| 1479 |
|
| 1480 |
copied_not_encrypted_version = list_of_files[3] |
| 1481 |
file_info = b2_tool.should_succeed_json( |
| 1482 |
['file-info', f"b2id://{copied_not_encrypted_version['fileId']}"] |
| 1483 |
) |
| 1484 |
should_equal({'mode': 'none'}, file_info['serverSideEncryption']) |
| 1485 |
|
| 1486 |
|
| 1487 |
def test_sse_c(b2_tool, bucket_name, is_running_on_docker, sample_file, tmp_path, b2_uri_args): |
| 1488 |
|
| 1489 |
sse_c_key_id = 'user-generated-key-id \nąóźćż\nœøΩ≈ç\nßäöü' |
| 1490 |
if is_running_on_docker: |
| 1491 |
# TODO: fix this once we figure out how to pass env vars with \n in them to docker, docker-compose should work |
| 1492 |
sse_c_key_id = sse_c_key_id.replace('\n', '') |
| 1493 |
|
| 1494 |
secret = os.urandom(32) |
| 1495 |
|
| 1496 |
b2_tool.should_fail( |
| 1497 |
[ |
| 1498 |
'upload-file', '--no-progress', '--quiet', '--destination-server-side-encryption', |
| 1499 |
'SSE-C', bucket_name, sample_file, 'gonna-fail-anyway' |
| 1500 |
], |
| 1501 |
'Using SSE-C requires providing an encryption key via B2_DESTINATION_SSE_C_KEY_B64 env var' |
| 1502 |
) |
| 1503 |
file_version_info = b2_tool.should_succeed_json( |
| 1504 |
[ |
| 1505 |
'upload-file', '--no-progress', '--quiet', '--destination-server-side-encryption', |
| 1506 |
'SSE-C', bucket_name, sample_file, 'uploaded_encrypted' |
| 1507 |
], |
| 1508 |
additional_env={ |
| 1509 |
'B2_DESTINATION_SSE_C_KEY_B64': base64.b64encode(secret).decode(), |
| 1510 |
'B2_DESTINATION_SSE_C_KEY_ID': sse_c_key_id, |
| 1511 |
} |
| 1512 |
) |
| 1513 |
should_equal( |
| 1514 |
{ |
| 1515 |
"algorithm": "AES256", |
| 1516 |
"customerKey": "******", |
| 1517 |
"customerKeyMd5": "******", |
| 1518 |
"mode": "SSE-C" |
| 1519 |
}, file_version_info['serverSideEncryption'] |
| 1520 |
) |
| 1521 |
should_equal(sse_c_key_id, file_version_info['fileInfo'][SSE_C_KEY_ID_FILE_INFO_KEY_NAME]) |
| 1522 |
|
| 1523 |
b2_tool.should_fail( |
| 1524 |
['download-file', '--quiet', f'b2://{bucket_name}/uploaded_encrypted', 'gonna_fail_anyway'], |
| 1525 |
expected_pattern='ERROR: The object was stored using a form of Server Side Encryption. The ' |
| 1526 |
r'correct parameters must be provided to retrieve the object. \(bad_request\)' |
| 1527 |
) |
| 1528 |
b2_tool.should_fail( |
| 1529 |
[ |
| 1530 |
'download-file', '--quiet', '--source-server-side-encryption', 'SSE-C', |
| 1531 |
f'b2://{bucket_name}/uploaded_encrypted', 'gonna_fail_anyway' |
| 1532 |
], |
| 1533 |
expected_pattern='ValueError: Using SSE-C requires providing an encryption key via ' |
| 1534 |
'B2_SOURCE_SSE_C_KEY_B64 env var' |
| 1535 |
) |
| 1536 |
b2_tool.should_fail( |
| 1537 |
[ |
| 1538 |
'download-file', '--quiet', '--source-server-side-encryption', 'SSE-C', |
| 1539 |
f'b2://{bucket_name}/uploaded_encrypted', 'gonna_fail_anyway' |
| 1540 |
], |
| 1541 |
expected_pattern='ERROR: Wrong or no SSE-C key provided when reading a file.', |
| 1542 |
additional_env={'B2_SOURCE_SSE_C_KEY_B64': base64.b64encode(os.urandom(32)).decode()} |
| 1543 |
) |
| 1544 |
with contextlib.nullcontext(tmp_path) as dir_path: |
| 1545 |
b2_tool.should_succeed( |
| 1546 |
[ |
| 1547 |
'download-file', |
| 1548 |
'--no-progress', |
| 1549 |
'--quiet', |
| 1550 |
'--source-server-side-encryption', |
| 1551 |
'SSE-C', |
| 1552 |
f'b2://{bucket_name}/uploaded_encrypted', |
| 1553 |
dir_path / 'a', |
| 1554 |
], |
| 1555 |
additional_env={'B2_SOURCE_SSE_C_KEY_B64': base64.b64encode(secret).decode()} |
| 1556 |
) |
| 1557 |
assert read_file(dir_path / 'a') == read_file(sample_file) |
| 1558 |
b2_tool.should_succeed( |
| 1559 |
[ |
| 1560 |
'download-file', |
| 1561 |
'--no-progress', |
| 1562 |
'--quiet', |
| 1563 |
'--source-server-side-encryption', |
| 1564 |
'SSE-C', |
| 1565 |
f"b2id://{file_version_info['fileId']}", |
| 1566 |
dir_path / 'b', |
| 1567 |
], |
| 1568 |
additional_env={'B2_SOURCE_SSE_C_KEY_B64': base64.b64encode(secret).decode()} |
| 1569 |
) |
| 1570 |
assert read_file(dir_path / 'b') == read_file(sample_file) |
| 1571 |
|
| 1572 |
b2_tool.should_fail( |
| 1573 |
['copy-file-by-id', file_version_info['fileId'], bucket_name, 'gonna-fail-anyway'], |
| 1574 |
expected_pattern= |
| 1575 |
'ERROR: The object was stored using a form of Server Side Encryption. The correct ' |
| 1576 |
r'parameters must be provided to retrieve the object. \(bad_request\)' |
| 1577 |
) |
| 1578 |
b2_tool.should_fail( |
| 1579 |
[ |
| 1580 |
'copy-file-by-id', '--source-server-side-encryption=SSE-C', file_version_info['fileId'], |
| 1581 |
bucket_name, 'gonna-fail-anyway' |
| 1582 |
], |
| 1583 |
expected_pattern='ValueError: Using SSE-C requires providing an encryption key via ' |
| 1584 |
'B2_SOURCE_SSE_C_KEY_B64 env var' |
| 1585 |
) |
| 1586 |
b2_tool.should_fail( |
| 1587 |
[ |
| 1588 |
'copy-file-by-id', '--source-server-side-encryption=SSE-C', |
| 1589 |
'--destination-server-side-encryption=SSE-C', file_version_info['fileId'], bucket_name, |
| 1590 |
'gonna-fail-anyway' |
| 1591 |
], |
| 1592 |
expected_pattern='ValueError: Using SSE-C requires providing an encryption key via ' |
| 1593 |
'B2_DESTINATION_SSE_C_KEY_B64 env var', |
| 1594 |
additional_env={'B2_SOURCE_SSE_C_KEY_B64': base64.b64encode(secret).decode()} |
| 1595 |
) |
| 1596 |
b2_tool.should_fail( |
| 1597 |
[ |
| 1598 |
'copy-file-by-id', '--source-server-side-encryption=SSE-C', file_version_info['fileId'], |
| 1599 |
bucket_name, 'gonna-fail-anyway' |
| 1600 |
], |
| 1601 |
additional_env={'B2_SOURCE_SSE_C_KEY_B64': base64.b64encode(secret).decode()}, |
| 1602 |
expected_pattern= |
| 1603 |
'Attempting to copy file with metadata while either source or destination uses ' |
| 1604 |
'SSE-C. Use --fetch-metadata to fetch source file metadata before copying.', |
| 1605 |
) |
| 1606 |
b2_tool.should_succeed( |
| 1607 |
[ |
| 1608 |
'copy-file-by-id', |
| 1609 |
'--source-server-side-encryption=SSE-C', |
| 1610 |
file_version_info['fileId'], |
| 1611 |
bucket_name, |
| 1612 |
'not_encrypted_copied_from_encrypted_metadata_replace', |
| 1613 |
'--info', |
| 1614 |
'a=b', |
| 1615 |
'--content-type', |
| 1616 |
'text/plain', |
| 1617 |
], |
| 1618 |
additional_env={'B2_SOURCE_SSE_C_KEY_B64': base64.b64encode(secret).decode()} |
| 1619 |
) |
| 1620 |
b2_tool.should_succeed( |
| 1621 |
[ |
| 1622 |
'copy-file-by-id', |
| 1623 |
'--source-server-side-encryption=SSE-C', |
| 1624 |
file_version_info['fileId'], |
| 1625 |
bucket_name, |
| 1626 |
'not_encrypted_copied_from_encrypted_metadata_replace_empty', |
| 1627 |
'--no-info', |
| 1628 |
'--content-type', |
| 1629 |
'text/plain', |
| 1630 |
], |
| 1631 |
additional_env={'B2_SOURCE_SSE_C_KEY_B64': base64.b64encode(secret).decode()} |
| 1632 |
) |
| 1633 |
b2_tool.should_succeed( |
| 1634 |
[ |
| 1635 |
'copy-file-by-id', |
| 1636 |
'--source-server-side-encryption=SSE-C', |
| 1637 |
file_version_info['fileId'], |
| 1638 |
bucket_name, |
| 1639 |
'not_encrypted_copied_from_encrypted_metadata_pseudo_copy', |
| 1640 |
'--fetch-metadata', |
| 1641 |
], |
| 1642 |
additional_env={'B2_SOURCE_SSE_C_KEY_B64': base64.b64encode(secret).decode()} |
| 1643 |
) |
| 1644 |
b2_tool.should_succeed( |
| 1645 |
[ |
| 1646 |
'copy-file-by-id', |
| 1647 |
'--source-server-side-encryption=SSE-C', |
| 1648 |
'--destination-server-side-encryption=SSE-C', |
| 1649 |
file_version_info['fileId'], |
| 1650 |
bucket_name, |
| 1651 |
'encrypted_no_id_copied_from_encrypted', |
| 1652 |
'--fetch-metadata', |
| 1653 |
], |
| 1654 |
additional_env={ |
| 1655 |
'B2_SOURCE_SSE_C_KEY_B64': base64.b64encode(secret).decode(), |
| 1656 |
'B2_DESTINATION_SSE_C_KEY_B64': base64.b64encode(os.urandom(32)).decode(), |
| 1657 |
} |
| 1658 |
) |
| 1659 |
b2_tool.should_succeed( |
| 1660 |
[ |
| 1661 |
'copy-file-by-id', |
| 1662 |
'--source-server-side-encryption=SSE-C', |
| 1663 |
'--destination-server-side-encryption=SSE-C', |
| 1664 |
file_version_info['fileId'], |
| 1665 |
bucket_name, |
| 1666 |
'encrypted_with_id_copied_from_encrypted_metadata_replace', |
| 1667 |
'--no-info', |
| 1668 |
'--content-type', |
| 1669 |
'text/plain', |
| 1670 |
], |
| 1671 |
additional_env={ |
| 1672 |
'B2_SOURCE_SSE_C_KEY_B64': base64.b64encode(secret).decode(), |
| 1673 |
'B2_DESTINATION_SSE_C_KEY_B64': base64.b64encode(os.urandom(32)).decode(), |
| 1674 |
'B2_DESTINATION_SSE_C_KEY_ID': 'another-user-generated-key-id', |
| 1675 |
} |
| 1676 |
) |
| 1677 |
b2_tool.should_succeed( |
| 1678 |
[ |
| 1679 |
'copy-file-by-id', |
| 1680 |
'--source-server-side-encryption=SSE-C', |
| 1681 |
'--destination-server-side-encryption=SSE-C', |
| 1682 |
file_version_info['fileId'], |
| 1683 |
bucket_name, |
| 1684 |
'encrypted_with_id_copied_from_encrypted_metadata_pseudo_copy', |
| 1685 |
'--fetch-metadata', |
| 1686 |
], |
| 1687 |
additional_env={ |
| 1688 |
'B2_SOURCE_SSE_C_KEY_B64': base64.b64encode(secret).decode(), |
| 1689 |
'B2_DESTINATION_SSE_C_KEY_B64': base64.b64encode(os.urandom(32)).decode(), |
| 1690 |
'B2_DESTINATION_SSE_C_KEY_ID': 'another-user-generated-key-id', |
| 1691 |
} |
| 1692 |
) |
| 1693 |
list_of_files = b2_tool.should_succeed_json( |
| 1694 |
['ls', '--json', '--recursive', *b2_uri_args(bucket_name)] |
| 1695 |
) |
| 1696 |
should_equal( |
| 1697 |
[ |
| 1698 |
{ |
| 1699 |
'file_name': 'encrypted_no_id_copied_from_encrypted', |
| 1700 |
'sse_c_key_id': 'missing_key', |
| 1701 |
'serverSideEncryption': |
| 1702 |
{ |
| 1703 |
"algorithm": "AES256", |
| 1704 |
"customerKey": "******", |
| 1705 |
"customerKeyMd5": "******", |
| 1706 |
"mode": "SSE-C" |
| 1707 |
}, |
| 1708 |
}, |
| 1709 |
{ |
| 1710 |
'file_name': 'encrypted_with_id_copied_from_encrypted_metadata_pseudo_copy', |
| 1711 |
'sse_c_key_id': 'another-user-generated-key-id', |
| 1712 |
'serverSideEncryption': |
| 1713 |
{ |
| 1714 |
'algorithm': 'AES256', |
| 1715 |
"customerKey": "******", |
| 1716 |
"customerKeyMd5": "******", |
| 1717 |
'mode': 'SSE-C', |
| 1718 |
}, |
| 1719 |
}, |
| 1720 |
{ |
| 1721 |
'file_name': 'encrypted_with_id_copied_from_encrypted_metadata_replace', |
| 1722 |
'sse_c_key_id': 'another-user-generated-key-id', |
| 1723 |
'serverSideEncryption': |
| 1724 |
{ |
| 1725 |
'algorithm': 'AES256', |
| 1726 |
"customerKey": "******", |
| 1727 |
"customerKeyMd5": "******", |
| 1728 |
'mode': 'SSE-C', |
| 1729 |
}, |
| 1730 |
}, |
| 1731 |
{ |
| 1732 |
'file_name': 'not_encrypted_copied_from_encrypted_metadata_pseudo_copy', |
| 1733 |
'sse_c_key_id': 'missing_key', |
| 1734 |
'serverSideEncryption': { |
| 1735 |
'mode': 'none', |
| 1736 |
}, |
| 1737 |
}, |
| 1738 |
{ |
| 1739 |
'file_name': 'not_encrypted_copied_from_encrypted_metadata_replace', |
| 1740 |
'sse_c_key_id': 'missing_key', |
| 1741 |
'serverSideEncryption': { |
| 1742 |
'mode': 'none', |
| 1743 |
}, |
| 1744 |
}, |
| 1745 |
{ |
| 1746 |
'file_name': 'not_encrypted_copied_from_encrypted_metadata_replace_empty', |
| 1747 |
'sse_c_key_id': 'missing_key', |
| 1748 |
'serverSideEncryption': { |
| 1749 |
'mode': 'none', |
| 1750 |
}, |
| 1751 |
}, |
| 1752 |
{ |
| 1753 |
'file_name': 'uploaded_encrypted', |
| 1754 |
'sse_c_key_id': sse_c_key_id, |
| 1755 |
'serverSideEncryption': |
| 1756 |
{ |
| 1757 |
"algorithm": "AES256", |
| 1758 |
"customerKey": "******", |
| 1759 |
"customerKeyMd5": "******", |
| 1760 |
"mode": "SSE-C" |
| 1761 |
}, |
| 1762 |
}, |
| 1763 |
], |
| 1764 |
sorted( |
| 1765 |
[ |
| 1766 |
{ |
| 1767 |
'sse_c_key_id': |
| 1768 |
f['fileInfo'].get(SSE_C_KEY_ID_FILE_INFO_KEY_NAME, 'missing_key'), |
| 1769 |
'serverSideEncryption': |
| 1770 |
f['serverSideEncryption'], |
| 1771 |
'file_name': |
| 1772 |
f['fileName'] |
| 1773 |
} for f in list_of_files |
| 1774 |
], |
| 1775 |
key=lambda r: r['file_name'] |
| 1776 |
) |
| 1777 |
) |
| 1778 |
|
| 1779 |
|
| 1780 |
@pytest.mark.skipif( |
| 1781 |
(sys.version_info.major, sys.version_info.minor) < (3, 8), |
| 1782 |
reason="License extraction doesn't work on older versions, and we're only " |
| 1783 |
"obliged to provide this " |
| 1784 |
"data in bundled and built packages." |
| 1785 |
) |
| 1786 |
@pytest.mark.parametrize('with_packages', [True, False]) |
| 1787 |
def test_license(b2_tool, with_packages, cli_version): |
| 1788 |
license_text = b2_tool.should_succeed( |
| 1789 |
['license'] + (['--with-packages'] if with_packages else []) |
| 1790 |
) |
| 1791 |
|
| 1792 |
if with_packages: |
| 1793 |
# In the case of e.g.: docker image, it has a license built-in with a `b2`. |
| 1794 |
# It also is unable to generate this license because it lacks required packages. |
| 1795 |
# Thus, I'm allowing here for the test of licenses to pass whenever |
| 1796 |
# the binary is named `b2` or with the proper cli version string (e.g. `_b2v4` or `b2v3`). |
| 1797 |
full_license_re = re.compile( |
| 1798 |
fr'Licenses of all modules used by ({cli_version}|b2)(\.EXE)?, shipped with it in binary form:\r?\n' |
| 1799 |
r'\+-*\+-*\+\r?\n' |
| 1800 |
r'\|\s*Module name\s*\|\s*License text\s*\|\r?\n' |
| 1801 |
r'.*' |
| 1802 |
r'\+-*\+-*\+\r?\n', re.MULTILINE + re.DOTALL |
| 1803 |
) |
| 1804 |
full_license_text = next(full_license_re.finditer(license_text), None) |
| 1805 |
assert full_license_text, license_text |
| 1806 |
assert len( |
| 1807 |
full_license_text.group(0) |
| 1808 |
) > 140_000 # we should know if the length of this block changes dramatically |
| 1809 |
# Note that GitHub CI adds additional packages: |
| 1810 |
# 'colorlog', 'virtualenv', 'nox', 'packaging', 'argcomplete', 'filelock' |
| 1811 |
# that sum up to around 50k characters. Tests ran from docker image are unaffected. |
| 1812 |
|
| 1813 |
# See the explanation above for why both `b2` and `cli_version` are allowed here. |
| 1814 |
license_summary_re = re.compile( |
| 1815 |
fr'Summary of all modules used by ({cli_version}|b2)(\.EXE)?, shipped with it in binary form:\r?\n' |
| 1816 |
r'\+-*\+-*\+-*\+-*\+-*\+\r?\n' |
| 1817 |
r'\|\s*Module name\s*\|\s*Version\s*\|\s*License\s*\|\s*Author\s*\|\s*URL\s*\|\r?\n' |
| 1818 |
r'.*' |
| 1819 |
r'\+-*\+-*\+-*\+-*\+-*\+\r?\n', re.MULTILINE + re.DOTALL |
| 1820 |
) |
| 1821 |
license_summary_text = next(license_summary_re.finditer(license_text), None) |
| 1822 |
assert license_summary_text, license_text |
| 1823 |
assert len( |
| 1824 |
license_summary_text.group(0) |
| 1825 |
) > 6_300 # we should know if the length of this block changes dramatically |
| 1826 |
|
| 1827 |
assert """ license: |
| 1828 |
Backblaze wants developers and organization to copy and re-use our |
| 1829 |
code examples, so we make the samples available by several different |
| 1830 |
licenses. One option is the MIT license (below). Other options are |
| 1831 |
available here: |
| 1832 |
|
| 1833 |
https://www.backblaze.com/using_b2_code.html |
| 1834 |
|
| 1835 |
|
| 1836 |
The MIT License (MIT) |
| 1837 |
|
| 1838 |
Copyright (c) 2015 Backblaze |
| 1839 |
|
| 1840 |
Permission is hereby granted, free of charge, to any person obtaining a copy |
| 1841 |
of this software and associated documentation files (the "Software"), to deal |
| 1842 |
in the Software without restriction, including without limitation the rights |
| 1843 |
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
| 1844 |
copies of the Software, and to permit persons to whom the Software is |
| 1845 |
furnished to do so, subject to the following conditions: |
| 1846 |
|
| 1847 |
The above copyright notice and this permission notice shall be included in all |
| 1848 |
copies or substantial portions of the Software. |
| 1849 |
|
| 1850 |
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| 1851 |
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| 1852 |
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
| 1853 |
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| 1854 |
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
| 1855 |
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| 1856 |
SOFTWARE.""" in license_text.replace(os.linesep, '\n'), repr(license_text[-2000:]) |
| 1857 |
|
| 1858 |
|
| 1859 |
def test_file_lock( |
| 1860 |
b2_tool, application_key_id, application_key, sample_file, bucket_factory, |
| 1861 |
schedule_bucket_cleanup |
| 1862 |
): |
| 1863 |
lock_disabled_bucket_name = bucket_factory(bucket_type='allPrivate').name |
| 1864 |
|
| 1865 |
now_millis = current_time_millis() |
| 1866 |
|
| 1867 |
not_lockable_file = b2_tool.should_succeed_json( # file in a lock disabled bucket |
| 1868 |
['upload-file', '--quiet', lock_disabled_bucket_name, sample_file, 'a'] |
| 1869 |
) |
| 1870 |
|
| 1871 |
_assert_file_lock_configuration( |
| 1872 |
b2_tool, |
| 1873 |
not_lockable_file['fileId'], |
| 1874 |
retention_mode=RetentionMode.NONE, |
| 1875 |
legal_hold=LegalHold.UNSET |
| 1876 |
) |
| 1877 |
|
| 1878 |
b2_tool.should_fail( |
| 1879 |
[ |
| 1880 |
'upload-file', |
| 1881 |
'--quiet', |
| 1882 |
lock_disabled_bucket_name, |
| 1883 |
sample_file, |
| 1884 |
'a', |
| 1885 |
'--file-retention-mode', |
| 1886 |
'governance', |
| 1887 |
'--retain-until', |
| 1888 |
str(now_millis + 1.5 * ONE_HOUR_MILLIS), |
| 1889 |
'--legal-hold', |
| 1890 |
'on', |
| 1891 |
], r'ERROR: The bucket is not file lock enabled \(bucket_missing_file_lock\)' |
| 1892 |
) |
| 1893 |
|
| 1894 |
b2_tool.should_fail( |
| 1895 |
[ |
| 1896 |
'update-bucket', lock_disabled_bucket_name, 'allPrivate', '--default-retention-mode', |
| 1897 |
'compliance' |
| 1898 |
], 'ValueError: must specify period for retention mode RetentionMode.COMPLIANCE' |
| 1899 |
) |
| 1900 |
b2_tool.should_fail( |
| 1901 |
[ |
| 1902 |
'update-bucket', lock_disabled_bucket_name, 'allPrivate', '--default-retention-mode', |
| 1903 |
'compliance', '--default-retention-period', '7 days' |
| 1904 |
], r'ERROR: The bucket is not file lock enabled \(bucket_missing_file_lock\)' |
| 1905 |
) |
| 1906 |
lock_enabled_bucket_name = b2_tool.generate_bucket_name() |
| 1907 |
schedule_bucket_cleanup(lock_enabled_bucket_name) |
| 1908 |
b2_tool.should_succeed( |
| 1909 |
[ |
| 1910 |
'create-bucket', |
| 1911 |
lock_enabled_bucket_name, |
| 1912 |
'allPrivate', |
| 1913 |
'--file-lock-enabled', |
| 1914 |
*b2_tool.get_bucket_info_args(), |
| 1915 |
], |
| 1916 |
) |
| 1917 |
updated_bucket = b2_tool.should_succeed_json( |
| 1918 |
[ |
| 1919 |
'update-bucket', |
| 1920 |
lock_enabled_bucket_name, |
| 1921 |
'allPrivate', |
| 1922 |
'--default-retention-mode', |
| 1923 |
'governance', |
| 1924 |
'--default-retention-period', |
| 1925 |
'1 days', |
| 1926 |
], |
| 1927 |
) |
| 1928 |
assert updated_bucket['defaultRetention'] == { |
| 1929 |
'mode': 'governance', |
| 1930 |
'period': { |
| 1931 |
'duration': 1, |
| 1932 |
'unit': 'days', |
| 1933 |
}, |
| 1934 |
} |
| 1935 |
|
| 1936 |
lockable_file = b2_tool.should_succeed_json( # file in a lock enabled bucket |
| 1937 |
['upload-file', '--no-progress', '--quiet', lock_enabled_bucket_name, sample_file, 'a'] |
| 1938 |
) |
| 1939 |
|
| 1940 |
b2_tool.should_fail( |
| 1941 |
[ |
| 1942 |
'update-file-retention', not_lockable_file['fileName'], not_lockable_file['fileId'], |
| 1943 |
'governance', '--retain-until', |
| 1944 |
str(now_millis + ONE_DAY_MILLIS + ONE_HOUR_MILLIS) |
| 1945 |
], r'ERROR: The bucket is not file lock enabled \(bucket_missing_file_lock\)' |
| 1946 |
) |
| 1947 |
|
| 1948 |
b2_tool.should_succeed( # first let's try with a file name |
| 1949 |
['update-file-retention', lockable_file['fileName'], lockable_file['fileId'], 'governance', |
| 1950 |
'--retain-until', str(now_millis + ONE_DAY_MILLIS + ONE_HOUR_MILLIS)] |
| 1951 |
) |
| 1952 |
|
| 1953 |
_assert_file_lock_configuration( |
| 1954 |
b2_tool, |
| 1955 |
lockable_file['fileId'], |
| 1956 |
retention_mode=RetentionMode.GOVERNANCE, |
| 1957 |
retain_until=now_millis + ONE_DAY_MILLIS + ONE_HOUR_MILLIS |
| 1958 |
) |
| 1959 |
|
| 1960 |
b2_tool.should_succeed( # and now without a file name |
| 1961 |
['update-file-retention', lockable_file['fileId'], 'governance', |
| 1962 |
'--retain-until', str(now_millis + ONE_DAY_MILLIS + 2 * ONE_HOUR_MILLIS)] |
| 1963 |
) |
| 1964 |
|
| 1965 |
_assert_file_lock_configuration( |
| 1966 |
b2_tool, |
| 1967 |
lockable_file['fileId'], |
| 1968 |
retention_mode=RetentionMode.GOVERNANCE, |
| 1969 |
retain_until=now_millis + ONE_DAY_MILLIS + 2 * ONE_HOUR_MILLIS |
| 1970 |
) |
| 1971 |
|
| 1972 |
b2_tool.should_fail( |
| 1973 |
[ |
| 1974 |
'update-file-retention', lockable_file['fileName'], lockable_file['fileId'], |
| 1975 |
'governance', '--retain-until', |
| 1976 |
str(now_millis + ONE_HOUR_MILLIS) |
| 1977 |
], |
| 1978 |
"ERROR: Auth token not authorized to write retention or file already in 'compliance' mode or " |
| 1979 |
"bypassGovernance=true parameter missing", |
| 1980 |
) |
| 1981 |
b2_tool.should_succeed( |
| 1982 |
[ |
| 1983 |
'update-file-retention', lockable_file['fileName'], lockable_file['fileId'], |
| 1984 |
'governance', '--retain-until', |
| 1985 |
str(now_millis + ONE_HOUR_MILLIS), '--bypass-governance' |
| 1986 |
], |
| 1987 |
) |
| 1988 |
|
| 1989 |
_assert_file_lock_configuration( |
| 1990 |
b2_tool, |
| 1991 |
lockable_file['fileId'], |
| 1992 |
retention_mode=RetentionMode.GOVERNANCE, |
| 1993 |
retain_until=now_millis + ONE_HOUR_MILLIS |
| 1994 |
) |
| 1995 |
|
| 1996 |
b2_tool.should_fail( |
| 1997 |
['update-file-retention', lockable_file['fileName'], lockable_file['fileId'], 'none'], |
| 1998 |
"ERROR: Auth token not authorized to write retention or file already in 'compliance' mode or " |
| 1999 |
"bypassGovernance=true parameter missing", |
| 2000 |
) |
| 2001 |
b2_tool.should_succeed( |
| 2002 |
[ |
| 2003 |
'update-file-retention', lockable_file['fileName'], lockable_file['fileId'], 'none', |
| 2004 |
'--bypass-governance' |
| 2005 |
], |
| 2006 |
) |
| 2007 |
|
| 2008 |
_assert_file_lock_configuration( |
| 2009 |
b2_tool, lockable_file['fileId'], retention_mode=RetentionMode.NONE |
| 2010 |
) |
| 2011 |
|
| 2012 |
b2_tool.should_fail( |
| 2013 |
['update-file-legal-hold', not_lockable_file['fileId'], 'on'], |
| 2014 |
r'ERROR: The bucket is not file lock enabled \(bucket_missing_file_lock\)' |
| 2015 |
) |
| 2016 |
|
| 2017 |
b2_tool.should_succeed( # first let's try with a file name |
| 2018 |
['update-file-legal-hold', lockable_file['fileName'], lockable_file['fileId'], 'on'], |
| 2019 |
) |
| 2020 |
|
| 2021 |
_assert_file_lock_configuration(b2_tool, lockable_file['fileId'], legal_hold=LegalHold.ON) |
| 2022 |
|
| 2023 |
b2_tool.should_succeed( # and now without a file name |
| 2024 |
['update-file-legal-hold', lockable_file['fileId'], 'off'], |
| 2025 |
) |
| 2026 |
|
| 2027 |
_assert_file_lock_configuration(b2_tool, lockable_file['fileId'], legal_hold=LegalHold.OFF) |
| 2028 |
|
| 2029 |
updated_bucket = b2_tool.should_succeed_json( |
| 2030 |
[ |
| 2031 |
'update-bucket', |
| 2032 |
lock_enabled_bucket_name, |
| 2033 |
'allPrivate', |
| 2034 |
'--default-retention-mode', |
| 2035 |
'none', |
| 2036 |
], |
| 2037 |
) |
| 2038 |
assert updated_bucket['defaultRetention'] == {'mode': None} |
| 2039 |
|
| 2040 |
b2_tool.should_fail( |
| 2041 |
[ |
| 2042 |
'upload-file', |
| 2043 |
'--no-progress', |
| 2044 |
'--quiet', |
| 2045 |
lock_enabled_bucket_name, |
| 2046 |
sample_file, |
| 2047 |
'a', |
| 2048 |
'--file-retention-mode', |
| 2049 |
'governance', |
| 2050 |
'--retain-until', |
| 2051 |
str(now_millis - 1.5 * ONE_HOUR_MILLIS), |
| 2052 |
], |
| 2053 |
r'ERROR: The retainUntilTimestamp must be in future \(retain_until_timestamp_must_be_in_future\)', |
| 2054 |
) |
| 2055 |
|
| 2056 |
uploaded_file = b2_tool.should_succeed_json( |
| 2057 |
[ |
| 2058 |
'upload-file', |
| 2059 |
'--no-progress', |
| 2060 |
'--quiet', |
| 2061 |
lock_enabled_bucket_name, |
| 2062 |
sample_file, |
| 2063 |
'a', |
| 2064 |
'--file-retention-mode', |
| 2065 |
'governance', |
| 2066 |
'--retain-until', |
| 2067 |
str(now_millis + 1.5 * ONE_HOUR_MILLIS), |
| 2068 |
'--legal-hold', |
| 2069 |
'on', |
| 2070 |
] |
| 2071 |
) |
| 2072 |
|
| 2073 |
_assert_file_lock_configuration( |
| 2074 |
b2_tool, |
| 2075 |
uploaded_file['fileId'], |
| 2076 |
retention_mode=RetentionMode.GOVERNANCE, |
| 2077 |
retain_until=now_millis + 1.5 * ONE_HOUR_MILLIS, |
| 2078 |
legal_hold=LegalHold.ON |
| 2079 |
) |
| 2080 |
|
| 2081 |
b2_tool.should_fail( |
| 2082 |
[ |
| 2083 |
'copy-file-by-id', |
| 2084 |
lockable_file['fileId'], |
| 2085 |
lock_disabled_bucket_name, |
| 2086 |
'copied', |
| 2087 |
'--file-retention-mode', |
| 2088 |
'governance', |
| 2089 |
'--retain-until', |
| 2090 |
str(now_millis + 1.25 * ONE_HOUR_MILLIS), |
| 2091 |
'--legal-hold', |
| 2092 |
'off', |
| 2093 |
], r'ERROR: The bucket is not file lock enabled \(bucket_missing_file_lock\)' |
| 2094 |
) |
| 2095 |
|
| 2096 |
copied_file = b2_tool.should_succeed_json( |
| 2097 |
[ |
| 2098 |
'copy-file-by-id', |
| 2099 |
lockable_file['fileId'], |
| 2100 |
lock_enabled_bucket_name, |
| 2101 |
'copied', |
| 2102 |
'--file-retention-mode', |
| 2103 |
'governance', |
| 2104 |
'--retain-until', |
| 2105 |
str(now_millis + 1.25 * ONE_HOUR_MILLIS), |
| 2106 |
'--legal-hold', |
| 2107 |
'off', |
| 2108 |
] |
| 2109 |
) |
| 2110 |
|
| 2111 |
_assert_file_lock_configuration( |
| 2112 |
b2_tool, |
| 2113 |
copied_file['fileId'], |
| 2114 |
retention_mode=RetentionMode.GOVERNANCE, |
| 2115 |
retain_until=now_millis + 1.25 * ONE_HOUR_MILLIS, |
| 2116 |
legal_hold=LegalHold.OFF |
| 2117 |
) |
| 2118 |
lock_disabled_key_id, lock_disabled_key = make_lock_disabled_key(b2_tool) |
| 2119 |
|
| 2120 |
b2_tool.should_succeed( |
| 2121 |
[ |
| 2122 |
'authorize-account', '--environment', b2_tool.realm, lock_disabled_key_id, |
| 2123 |
lock_disabled_key |
| 2124 |
], |
| 2125 |
) |
| 2126 |
|
| 2127 |
file_lock_without_perms_test( |
| 2128 |
b2_tool, |
| 2129 |
lock_enabled_bucket_name, |
| 2130 |
lock_disabled_bucket_name, |
| 2131 |
lockable_file['fileId'], |
| 2132 |
not_lockable_file['fileId'], |
| 2133 |
sample_file=sample_file |
| 2134 |
) |
| 2135 |
|
| 2136 |
b2_tool.should_succeed( |
| 2137 |
['authorize-account', '--environment', b2_tool.realm, application_key_id, application_key], |
| 2138 |
) |
| 2139 |
|
| 2140 |
deleting_locked_files( |
| 2141 |
b2_tool, lock_enabled_bucket_name, lock_disabled_key_id, lock_disabled_key, sample_file |
| 2142 |
) |
| 2143 |
|
| 2144 |
|
| 2145 |
def make_lock_disabled_key(b2_tool): |
| 2146 |
key_name = 'no-perms-for-file-lock' + random_hex(6) |
| 2147 |
created_key_stdout = b2_tool.should_succeed( |
| 2148 |
[ |
| 2149 |
'create-key', |
| 2150 |
key_name, |
| 2151 |
'listFiles,listBuckets,readFiles,writeKeys,deleteFiles', |
| 2152 |
] |
| 2153 |
) |
| 2154 |
key_id, key = created_key_stdout.split() |
| 2155 |
return key_id, key |
| 2156 |
|
| 2157 |
|
| 2158 |
def file_lock_without_perms_test( |
| 2159 |
b2_tool, lock_enabled_bucket_name, lock_disabled_bucket_name, lockable_file_id, |
| 2160 |
not_lockable_file_id, sample_file |
| 2161 |
): |
| 2162 |
|
| 2163 |
b2_tool.should_fail( |
| 2164 |
[ |
| 2165 |
'update-bucket', lock_enabled_bucket_name, 'allPrivate', '--default-retention-mode', |
| 2166 |
'governance', '--default-retention-period', '1 days' |
| 2167 |
], |
| 2168 |
'ERROR: unauthorized for application key with capabilities', |
| 2169 |
) |
| 2170 |
|
| 2171 |
_assert_file_lock_configuration( |
| 2172 |
b2_tool, |
| 2173 |
lockable_file_id, |
| 2174 |
retention_mode=RetentionMode.UNKNOWN, |
| 2175 |
legal_hold=LegalHold.UNKNOWN |
| 2176 |
) |
| 2177 |
|
| 2178 |
b2_tool.should_fail( |
| 2179 |
[ |
| 2180 |
'update-file-retention', lockable_file_id, 'governance', '--retain-until', |
| 2181 |
str(current_time_millis() + 7 * ONE_DAY_MILLIS) |
| 2182 |
], |
| 2183 |
"ERROR: Auth token not authorized to write retention or file already in 'compliance' mode or " |
| 2184 |
"bypassGovernance=true parameter missing", |
| 2185 |
) |
| 2186 |
|
| 2187 |
b2_tool.should_fail( |
| 2188 |
[ |
| 2189 |
'update-file-retention', not_lockable_file_id, 'governance', '--retain-until', |
| 2190 |
str(current_time_millis() + 7 * ONE_DAY_MILLIS) |
| 2191 |
], |
| 2192 |
"ERROR: Auth token not authorized to write retention or file already in 'compliance' mode or " |
| 2193 |
"bypassGovernance=true parameter missing", |
| 2194 |
) |
| 2195 |
|
| 2196 |
b2_tool.should_fail( |
| 2197 |
['update-file-legal-hold', lockable_file_id, 'on'], |
| 2198 |
"ERROR: Auth token not authorized to write retention or file already in 'compliance' mode or " |
| 2199 |
"bypassGovernance=true parameter missing", |
| 2200 |
) |
| 2201 |
|
| 2202 |
b2_tool.should_fail( |
| 2203 |
['update-file-legal-hold', not_lockable_file_id, 'on'], |
| 2204 |
"ERROR: Auth token not authorized to write retention or file already in 'compliance' mode or " |
| 2205 |
"bypassGovernance=true parameter missing", |
| 2206 |
) |
| 2207 |
|
| 2208 |
b2_tool.should_fail( |
| 2209 |
[ |
| 2210 |
'upload-file', |
| 2211 |
'--no-progress', |
| 2212 |
'--quiet', |
| 2213 |
lock_enabled_bucket_name, |
| 2214 |
sample_file, |
| 2215 |
'bound_to_fail_anyway', |
| 2216 |
'--file-retention-mode', |
| 2217 |
'governance', |
| 2218 |
'--retain-until', |
| 2219 |
str(current_time_millis() + ONE_HOUR_MILLIS), |
| 2220 |
'--legal-hold', |
| 2221 |
'on', |
| 2222 |
], |
| 2223 |
"unauthorized for application key with capabilities", |
| 2224 |
) |
| 2225 |
|
| 2226 |
b2_tool.should_fail( |
| 2227 |
[ |
| 2228 |
'upload-file', |
| 2229 |
'--no-progress', |
| 2230 |
'--quiet', |
| 2231 |
lock_disabled_bucket_name, |
| 2232 |
sample_file, |
| 2233 |
'bound_to_fail_anyway', |
| 2234 |
'--file-retention-mode', |
| 2235 |
'governance', |
| 2236 |
'--retain-until', |
| 2237 |
str(current_time_millis() + ONE_HOUR_MILLIS), |
| 2238 |
'--legal-hold', |
| 2239 |
'on', |
| 2240 |
], |
| 2241 |
"unauthorized for application key with capabilities", |
| 2242 |
) |
| 2243 |
|
| 2244 |
b2_tool.should_fail( |
| 2245 |
[ |
| 2246 |
'copy-file-by-id', |
| 2247 |
lockable_file_id, |
| 2248 |
lock_enabled_bucket_name, |
| 2249 |
'copied', |
| 2250 |
'--file-retention-mode', |
| 2251 |
'governance', |
| 2252 |
'--retain-until', |
| 2253 |
str(current_time_millis() + ONE_HOUR_MILLIS), |
| 2254 |
'--legal-hold', |
| 2255 |
'off', |
| 2256 |
], |
| 2257 |
'ERROR: unauthorized for application key with capabilities', |
| 2258 |
) |
| 2259 |
|
| 2260 |
b2_tool.should_fail( |
| 2261 |
[ |
| 2262 |
'copy-file-by-id', |
| 2263 |
lockable_file_id, |
| 2264 |
lock_disabled_bucket_name, |
| 2265 |
'copied', |
| 2266 |
'--file-retention-mode', |
| 2267 |
'governance', |
| 2268 |
'--retain-until', |
| 2269 |
str(current_time_millis() + ONE_HOUR_MILLIS), |
| 2270 |
'--legal-hold', |
| 2271 |
'off', |
| 2272 |
], |
| 2273 |
'ERROR: unauthorized for application key with capabilities', |
| 2274 |
) |
| 2275 |
|
| 2276 |
|
| 2277 |
def upload_locked_file(b2_tool, bucket_name, sample_file): |
| 2278 |
return b2_tool.should_succeed_json( |
| 2279 |
[ |
| 2280 |
'upload-file', |
| 2281 |
'--no-progress', |
| 2282 |
'--quiet', |
| 2283 |
'--file-retention-mode', |
| 2284 |
'governance', |
| 2285 |
'--retain-until', |
| 2286 |
str(int(time.time()) + 1000), |
| 2287 |
bucket_name, |
| 2288 |
sample_file, |
| 2289 |
'a-locked', |
| 2290 |
] |
| 2291 |
) |
| 2292 |
|
| 2293 |
|
| 2294 |
def deleting_locked_files( |
| 2295 |
b2_tool, lock_enabled_bucket_name, lock_disabled_key_id, lock_disabled_key, sample_file |
| 2296 |
): |
| 2297 |
locked_file = upload_locked_file(b2_tool, lock_enabled_bucket_name, sample_file) |
| 2298 |
b2_tool.should_fail( |
| 2299 |
[ # master key |
| 2300 |
'delete-file-version', |
| 2301 |
locked_file['fileName'], |
| 2302 |
locked_file['fileId'], |
| 2303 |
], |
| 2304 |
"ERROR: Access Denied for application key " |
| 2305 |
) |
| 2306 |
b2_tool.should_succeed([ # master key |
| 2307 |
'delete-file-version', |
| 2308 |
locked_file['fileName'], |
| 2309 |
locked_file['fileId'], |
| 2310 |
'--bypass-governance' |
| 2311 |
]) |
| 2312 |
|
| 2313 |
locked_file = upload_locked_file(b2_tool, lock_enabled_bucket_name, sample_file) |
| 2314 |
|
| 2315 |
b2_tool.should_succeed( |
| 2316 |
[ |
| 2317 |
'authorize-account', '--environment', b2_tool.realm, lock_disabled_key_id, |
| 2318 |
lock_disabled_key |
| 2319 |
], |
| 2320 |
) |
| 2321 |
b2_tool.should_fail([ # lock disabled key |
| 2322 |
'delete-file-version', |
| 2323 |
locked_file['fileName'], |
| 2324 |
locked_file['fileId'], |
| 2325 |
'--bypass-governance', |
| 2326 |
], "ERROR: unauthorized for application key with capabilities '") |
| 2327 |
|
| 2328 |
|
| 2329 |
def test_profile_switch(b2_tool): |
| 2330 |
# this test could be unit, but it adds a lot of complexity because of |
| 2331 |
# necessity to pass mocked B2Api to ConsoleTool; it's much easier to |
| 2332 |
# just have an integration test instead |
| 2333 |
|
| 2334 |
MISSING_ACCOUNT_PATTERN = 'Missing account data' |
| 2335 |
|
| 2336 |
b2_tool.should_succeed( |
| 2337 |
[ |
| 2338 |
'authorize-account', |
| 2339 |
'--environment', |
| 2340 |
b2_tool.realm, |
| 2341 |
b2_tool.account_id, |
| 2342 |
b2_tool.application_key, |
| 2343 |
] |
| 2344 |
) |
| 2345 |
b2_tool.should_succeed(['get-account-info']) |
| 2346 |
b2_tool.should_succeed(['clear-account']) |
| 2347 |
b2_tool.should_fail(['get-account-info'], expected_pattern=MISSING_ACCOUNT_PATTERN) |
| 2348 |
|
| 2349 |
# in order to use --profile flag, we need to temporary |
| 2350 |
# delete B2_ACCOUNT_INFO_ENV_VAR |
| 2351 |
B2_ACCOUNT_INFO = os.environ.pop(B2_ACCOUNT_INFO_ENV_VAR, None) |
| 2352 |
|
| 2353 |
# now authorize a different account |
| 2354 |
profile = 'profile-for-test-' + random_hex(6) |
| 2355 |
b2_tool.should_fail( |
| 2356 |
['get-account-info', '--profile', profile], |
| 2357 |
expected_pattern=MISSING_ACCOUNT_PATTERN, |
| 2358 |
) |
| 2359 |
b2_tool.should_succeed( |
| 2360 |
[ |
| 2361 |
'authorize-account', |
| 2362 |
'--environment', |
| 2363 |
b2_tool.realm, |
| 2364 |
'--profile', |
| 2365 |
profile, |
| 2366 |
b2_tool.account_id, |
| 2367 |
b2_tool.application_key, |
| 2368 |
] |
| 2369 |
) |
| 2370 |
|
| 2371 |
account_info = b2_tool.should_succeed_json(['get-account-info', '--profile', profile]) |
| 2372 |
account_file_path = account_info['accountFilePath'] |
| 2373 |
assert profile in account_file_path, \ |
| 2374 |
f'accountFilePath "{account_file_path}" should contain profile name "{profile}"' |
| 2375 |
|
| 2376 |
b2_tool.should_succeed(['clear-account', '--profile', profile]) |
| 2377 |
b2_tool.should_fail( |
| 2378 |
['get-account-info', '--profile', profile], |
| 2379 |
expected_pattern=MISSING_ACCOUNT_PATTERN, |
| 2380 |
) |
| 2381 |
os.remove(account_file_path) |
| 2382 |
|
| 2383 |
# restore B2_ACCOUNT_INFO_ENV_VAR, if existed |
| 2384 |
if B2_ACCOUNT_INFO: |
| 2385 |
os.environ[B2_ACCOUNT_INFO_ENV_VAR] = B2_ACCOUNT_INFO |
| 2386 |
|
| 2387 |
|
| 2388 |
def test_replication_basic(b2_tool, bucket_name, schedule_bucket_cleanup): |
| 2389 |
key_one_name = 'clt-testKey-01' + random_hex(6) |
| 2390 |
created_key_stdout = b2_tool.should_succeed( |
| 2391 |
[ |
| 2392 |
'create-key', |
| 2393 |
key_one_name, |
| 2394 |
'listBuckets,readFiles', |
| 2395 |
] |
| 2396 |
) |
| 2397 |
key_one_id, _ = created_key_stdout.split() |
| 2398 |
|
| 2399 |
key_two_name = 'clt-testKey-02' + random_hex(6) |
| 2400 |
created_key_stdout = b2_tool.should_succeed( |
| 2401 |
[ |
| 2402 |
'create-key', |
| 2403 |
key_two_name, |
| 2404 |
'listBuckets,writeFiles', |
| 2405 |
] |
| 2406 |
) |
| 2407 |
key_two_id, _ = created_key_stdout.split() |
| 2408 |
|
| 2409 |
destination_bucket_name = bucket_name |
| 2410 |
destination_bucket = b2_tool.should_succeed_json(['get-bucket', destination_bucket_name]) |
| 2411 |
|
| 2412 |
# test that by default there's no `replicationConfiguration` key |
| 2413 |
assert 'replicationConfiguration' not in destination_bucket |
| 2414 |
|
| 2415 |
# ---------------- set up replication destination ---------------- |
| 2416 |
|
| 2417 |
# update destination bucket info |
| 2418 |
destination_replication_configuration = { |
| 2419 |
'asReplicationSource': None, |
| 2420 |
'asReplicationDestination': { |
| 2421 |
'sourceToDestinationKeyMapping': { |
| 2422 |
key_one_id: key_two_id, |
| 2423 |
}, |
| 2424 |
}, |
| 2425 |
} |
| 2426 |
destination_replication_configuration_json = json.dumps(destination_replication_configuration) |
| 2427 |
destination_bucket = b2_tool.should_succeed_json( |
| 2428 |
[ |
| 2429 |
'update-bucket', |
| 2430 |
destination_bucket_name, |
| 2431 |
'allPublic', |
| 2432 |
'--replication', |
| 2433 |
destination_replication_configuration_json, |
| 2434 |
] |
| 2435 |
) |
| 2436 |
|
| 2437 |
# test that destination bucket is registered as replication destination |
| 2438 |
assert destination_bucket['replication'].get('asReplicationSource') is None |
| 2439 |
assert destination_bucket['replication' |
| 2440 |
]['asReplicationDestination' |
| 2441 |
] == destination_replication_configuration['asReplicationDestination'] |
| 2442 |
|
| 2443 |
# ---------------- set up replication source ---------------- |
| 2444 |
source_replication_configuration = { |
| 2445 |
"asReplicationSource": |
| 2446 |
{ |
| 2447 |
"replicationRules": |
| 2448 |
[ |
| 2449 |
{ |
| 2450 |
"destinationBucketId": destination_bucket['bucketId'], |
| 2451 |
"fileNamePrefix": "one/", |
| 2452 |
"includeExistingFiles": False, |
| 2453 |
"isEnabled": True, |
| 2454 |
"priority": 1, |
| 2455 |
"replicationRuleName": "replication-one" |
| 2456 |
}, { |
| 2457 |
"destinationBucketId": destination_bucket['bucketId'], |
| 2458 |
"fileNamePrefix": "two/", |
| 2459 |
"includeExistingFiles": False, |
| 2460 |
"isEnabled": True, |
| 2461 |
"priority": 2, |
| 2462 |
"replicationRuleName": "replication-two" |
| 2463 |
} |
| 2464 |
], |
| 2465 |
"sourceApplicationKeyId": key_one_id, |
| 2466 |
}, |
| 2467 |
} |
| 2468 |
source_replication_configuration_json = json.dumps(source_replication_configuration) |
| 2469 |
|
| 2470 |
# create a source bucket and set up replication to destination bucket |
| 2471 |
source_bucket_name = b2_tool.generate_bucket_name() |
| 2472 |
schedule_bucket_cleanup(source_bucket_name) |
| 2473 |
b2_tool.should_succeed( |
| 2474 |
[ |
| 2475 |
'create-bucket', |
| 2476 |
source_bucket_name, |
| 2477 |
'allPublic', |
| 2478 |
'--replication', |
| 2479 |
source_replication_configuration_json, |
| 2480 |
*b2_tool.get_bucket_info_args(), |
| 2481 |
] |
| 2482 |
) |
| 2483 |
source_bucket = b2_tool.should_succeed_json(['get-bucket', source_bucket_name]) |
| 2484 |
|
| 2485 |
# test that all replication rules are present in source bucket |
| 2486 |
assert source_bucket['replication']['asReplicationSource' |
| 2487 |
] == source_replication_configuration['asReplicationSource'] |
| 2488 |
|
| 2489 |
# test that source bucket is not mentioned as replication destination |
| 2490 |
assert source_bucket['replication'].get('asReplicationDestination') is None |
| 2491 |
|
| 2492 |
# ---------------- attempt enabling object lock ---------------- |
| 2493 |
b2_tool.should_fail( |
| 2494 |
['update-bucket', source_bucket_name, '--file-lock-enabled'], |
| 2495 |
'ERROR: Operation not supported for buckets with source replication' |
| 2496 |
) |
| 2497 |
|
| 2498 |
# ---------------- remove replication source ---------------- |
| 2499 |
|
| 2500 |
no_replication_configuration = { |
| 2501 |
'asReplicationSource': None, |
| 2502 |
'asReplicationDestination': None, |
| 2503 |
} |
| 2504 |
no_replication_configuration_json = json.dumps(no_replication_configuration) |
| 2505 |
source_bucket = b2_tool.should_succeed_json( |
| 2506 |
[ |
| 2507 |
'update-bucket', source_bucket_name, 'allPublic', '--replication', |
| 2508 |
no_replication_configuration_json |
| 2509 |
] |
| 2510 |
) |
| 2511 |
|
| 2512 |
# test that source bucket replication is removed |
| 2513 |
assert source_bucket['replication'] == { |
| 2514 |
'asReplicationDestination': None, |
| 2515 |
'asReplicationSource': None |
| 2516 |
} |
| 2517 |
|
| 2518 |
# ---------------- remove replication destination ---------------- |
| 2519 |
|
| 2520 |
destination_bucket = b2_tool.should_succeed_json( |
| 2521 |
[ |
| 2522 |
'update-bucket', |
| 2523 |
destination_bucket_name, |
| 2524 |
'allPublic', |
| 2525 |
'--replication', |
| 2526 |
'{}', |
| 2527 |
] |
| 2528 |
) |
| 2529 |
|
| 2530 |
# test that destination bucket replication is removed |
| 2531 |
assert destination_bucket['replication'] == { |
| 2532 |
'asReplicationDestination': None, |
| 2533 |
'asReplicationSource': None |
| 2534 |
} |
| 2535 |
|
| 2536 |
b2_tool.should_succeed(['delete-key', key_one_id]) |
| 2537 |
b2_tool.should_succeed(['delete-key', key_two_id]) |
| 2538 |
|
| 2539 |
|
| 2540 |
def test_replication_setup(b2_tool, bucket_name, schedule_bucket_cleanup): |
| 2541 |
source_bucket_name = b2_tool.generate_bucket_name() |
| 2542 |
schedule_bucket_cleanup(source_bucket_name) |
| 2543 |
b2_tool.should_succeed( |
| 2544 |
[ |
| 2545 |
'create-bucket', |
| 2546 |
source_bucket_name, |
| 2547 |
'allPublic', |
| 2548 |
'--file-lock-enabled', |
| 2549 |
*b2_tool.get_bucket_info_args(), |
| 2550 |
] |
| 2551 |
) |
| 2552 |
destination_bucket_name = bucket_name |
| 2553 |
b2_tool.should_succeed(['replication-setup', source_bucket_name, destination_bucket_name]) |
| 2554 |
destination_bucket_old = b2_tool.should_succeed_json(['get-bucket', destination_bucket_name]) |
| 2555 |
|
| 2556 |
b2_tool.should_succeed( |
| 2557 |
[ |
| 2558 |
'replication-setup', |
| 2559 |
'--priority', |
| 2560 |
'132', |
| 2561 |
'--file-name-prefix', |
| 2562 |
'foo', |
| 2563 |
'--name', |
| 2564 |
'my-replication-rule', |
| 2565 |
source_bucket_name, |
| 2566 |
destination_bucket_name, |
| 2567 |
] |
| 2568 |
) |
| 2569 |
source_bucket = b2_tool.should_succeed_json(['get-bucket', source_bucket_name]) |
| 2570 |
destination_bucket = b2_tool.should_succeed_json(['get-bucket', destination_bucket_name]) |
| 2571 |
assert source_bucket['replication']['asReplicationSource']['replicationRules'] == [ |
| 2572 |
{ |
| 2573 |
"destinationBucketId": destination_bucket['bucketId'], |
| 2574 |
"fileNamePrefix": "", |
| 2575 |
"includeExistingFiles": False, |
| 2576 |
"isEnabled": True, |
| 2577 |
"priority": 128, |
| 2578 |
"replicationRuleName": destination_bucket['bucketName'], |
| 2579 |
}, |
| 2580 |
{ |
| 2581 |
"destinationBucketId": destination_bucket['bucketId'], |
| 2582 |
"fileNamePrefix": "foo", |
| 2583 |
"includeExistingFiles": False, |
| 2584 |
"isEnabled": True, |
| 2585 |
"priority": 132, |
| 2586 |
"replicationRuleName": "my-replication-rule", |
| 2587 |
}, |
| 2588 |
] |
| 2589 |
|
| 2590 |
for key_one_id, key_two_id in destination_bucket['replication']['asReplicationDestination'][ |
| 2591 |
'sourceToDestinationKeyMapping'].items(): |
| 2592 |
b2_tool.should_succeed(['delete-key', key_one_id]) |
| 2593 |
b2_tool.should_succeed(['delete-key', key_two_id]) |
| 2594 |
assert destination_bucket_old['replication']['asReplicationDestination'][ |
| 2595 |
'sourceToDestinationKeyMapping'] == destination_bucket['replication'][ |
| 2596 |
'asReplicationDestination']['sourceToDestinationKeyMapping'] |
| 2597 |
|
| 2598 |
|
| 2599 |
def test_replication_monitoring(b2_tool, bucket_name, sample_file, schedule_bucket_cleanup): |
| 2600 |
|
| 2601 |
# ---------------- set up keys ---------------- |
| 2602 |
key_one_name = 'clt-testKey-01' + random_hex(6) |
| 2603 |
created_key_stdout = b2_tool.should_succeed( |
| 2604 |
[ |
| 2605 |
'create-key', |
| 2606 |
key_one_name, |
| 2607 |
'listBuckets,readFiles', |
| 2608 |
] |
| 2609 |
) |
| 2610 |
key_one_id, _ = created_key_stdout.split() |
| 2611 |
|
| 2612 |
key_two_name = 'clt-testKey-02' + random_hex(6) |
| 2613 |
created_key_stdout = b2_tool.should_succeed( |
| 2614 |
[ |
| 2615 |
'create-key', |
| 2616 |
key_two_name, |
| 2617 |
'listBuckets,writeFiles', |
| 2618 |
] |
| 2619 |
) |
| 2620 |
key_two_id, _ = created_key_stdout.split() |
| 2621 |
|
| 2622 |
# ---------------- add test data ---------------- |
| 2623 |
destination_bucket_name = bucket_name |
| 2624 |
uploaded_a = b2_tool.should_succeed_json( |
| 2625 |
['upload-file', '--quiet', destination_bucket_name, sample_file, 'one/a'] |
| 2626 |
) |
| 2627 |
|
| 2628 |
# ---------------- set up replication destination ---------------- |
| 2629 |
|
| 2630 |
# update destination bucket info |
| 2631 |
destination_replication_configuration = { |
| 2632 |
'asReplicationSource': None, |
| 2633 |
'asReplicationDestination': { |
| 2634 |
'sourceToDestinationKeyMapping': { |
| 2635 |
key_one_id: key_two_id, |
| 2636 |
}, |
| 2637 |
}, |
| 2638 |
} |
| 2639 |
destination_replication_configuration_json = json.dumps(destination_replication_configuration) |
| 2640 |
destination_bucket = b2_tool.should_succeed_json( |
| 2641 |
[ |
| 2642 |
'update-bucket', |
| 2643 |
destination_bucket_name, |
| 2644 |
'allPublic', |
| 2645 |
'--replication', |
| 2646 |
destination_replication_configuration_json, |
| 2647 |
] |
| 2648 |
) |
| 2649 |
|
| 2650 |
# ---------------- set up replication source ---------------- |
| 2651 |
source_replication_configuration = { |
| 2652 |
"asReplicationSource": |
| 2653 |
{ |
| 2654 |
"replicationRules": |
| 2655 |
[ |
| 2656 |
{ |
| 2657 |
"destinationBucketId": destination_bucket['bucketId'], |
| 2658 |
"fileNamePrefix": "one/", |
| 2659 |
"includeExistingFiles": False, |
| 2660 |
"isEnabled": True, |
| 2661 |
"priority": 1, |
| 2662 |
"replicationRuleName": "replication-one" |
| 2663 |
}, { |
| 2664 |
"destinationBucketId": destination_bucket['bucketId'], |
| 2665 |
"fileNamePrefix": "two/", |
| 2666 |
"includeExistingFiles": False, |
| 2667 |
"isEnabled": True, |
| 2668 |
"priority": 2, |
| 2669 |
"replicationRuleName": "replication-two" |
| 2670 |
} |
| 2671 |
], |
| 2672 |
"sourceApplicationKeyId": key_one_id, |
| 2673 |
}, |
| 2674 |
} |
| 2675 |
source_replication_configuration_json = json.dumps(source_replication_configuration) |
| 2676 |
|
| 2677 |
# create a source bucket and set up replication to destination bucket |
| 2678 |
source_bucket_name = b2_tool.generate_bucket_name() |
| 2679 |
schedule_bucket_cleanup(source_bucket_name) |
| 2680 |
b2_tool.should_succeed( |
| 2681 |
[ |
| 2682 |
'create-bucket', |
| 2683 |
source_bucket_name, |
| 2684 |
'allPublic', |
| 2685 |
'--file-lock-enabled', |
| 2686 |
'--replication', |
| 2687 |
source_replication_configuration_json, |
| 2688 |
*b2_tool.get_bucket_info_args(), |
| 2689 |
] |
| 2690 |
) |
| 2691 |
|
| 2692 |
# make test data |
| 2693 |
uploaded_a = b2_tool.should_succeed_json( |
| 2694 |
['upload-file', '--quiet', source_bucket_name, sample_file, 'one/a'] |
| 2695 |
) |
| 2696 |
b2_tool.should_succeed_json( |
| 2697 |
[ |
| 2698 |
'upload-file', |
| 2699 |
'--quiet', |
| 2700 |
source_bucket_name, |
| 2701 |
'--legal-hold', |
| 2702 |
'on', |
| 2703 |
sample_file, |
| 2704 |
'two/b', |
| 2705 |
] |
| 2706 |
) |
| 2707 |
|
| 2708 |
# encryption |
| 2709 |
# SSE-B2 |
| 2710 |
upload_encryption_args = ['--destination-server-side-encryption', 'SSE-B2'] |
| 2711 |
upload_additional_env = {} |
| 2712 |
b2_tool.should_succeed_json( |
| 2713 |
['upload-file', '--quiet', source_bucket_name, sample_file, 'two/c'] + |
| 2714 |
upload_encryption_args, |
| 2715 |
additional_env=upload_additional_env, |
| 2716 |
) |
| 2717 |
|
| 2718 |
# SSE-C |
| 2719 |
upload_encryption_args = ['--destination-server-side-encryption', 'SSE-C'] |
| 2720 |
upload_additional_env = { |
| 2721 |
'B2_DESTINATION_SSE_C_KEY_B64': base64.b64encode(SSE_C_AES.key.secret).decode(), |
| 2722 |
'B2_DESTINATION_SSE_C_KEY_ID': SSE_C_AES.key.key_id, |
| 2723 |
} |
| 2724 |
b2_tool.should_succeed_json( |
| 2725 |
['upload-file', '--quiet', source_bucket_name, sample_file, 'two/d'] + |
| 2726 |
upload_encryption_args, |
| 2727 |
additional_env=upload_additional_env, |
| 2728 |
) |
| 2729 |
|
| 2730 |
# encryption + legal hold |
| 2731 |
b2_tool.should_succeed_json( |
| 2732 |
[ |
| 2733 |
'upload-file', |
| 2734 |
'--quiet', |
| 2735 |
source_bucket_name, |
| 2736 |
sample_file, |
| 2737 |
'two/e', |
| 2738 |
'--legal-hold', |
| 2739 |
'on', |
| 2740 |
] + upload_encryption_args, |
| 2741 |
additional_env=upload_additional_env, |
| 2742 |
) |
| 2743 |
|
| 2744 |
# there is just one file, so clean after itself for faster execution |
| 2745 |
b2_tool.should_succeed(['delete-file-version', uploaded_a['fileName'], uploaded_a['fileId']]) |
| 2746 |
|
| 2747 |
# run stats command |
| 2748 |
replication_status_json = b2_tool.should_succeed_json( |
| 2749 |
[ |
| 2750 |
'replication-status', |
| 2751 |
# '--destination-profile', |
| 2752 |
# profile, |
| 2753 |
'--no-progress', |
| 2754 |
# '--columns=count, hash differs', |
| 2755 |
'--output-format', |
| 2756 |
'json', |
| 2757 |
source_bucket_name, |
| 2758 |
] |
| 2759 |
) |
| 2760 |
|
| 2761 |
assert replication_status_json in [ |
| 2762 |
{ |
| 2763 |
"replication-one": |
| 2764 |
[ |
| 2765 |
{ |
| 2766 |
"count": 1, |
| 2767 |
"destination_replication_status": None, |
| 2768 |
"hash_differs": None, |
| 2769 |
"metadata_differs": None, |
| 2770 |
"source_has_file_retention": None, |
| 2771 |
"source_has_hide_marker": None, |
| 2772 |
"source_has_large_metadata": None, |
| 2773 |
"source_has_legal_hold": None, |
| 2774 |
"source_encryption_mode": None, |
| 2775 |
"source_replication_status": None, |
| 2776 |
} |
| 2777 |
], |
| 2778 |
"replication-two": |
| 2779 |
[ |
| 2780 |
{ |
| 2781 |
"count": 1, |
| 2782 |
"destination_replication_status": None, |
| 2783 |
"hash_differs": None, |
| 2784 |
"metadata_differs": None, |
| 2785 |
"source_has_file_retention": False, |
| 2786 |
"source_has_hide_marker": False, |
| 2787 |
"source_has_large_metadata": False, |
| 2788 |
"source_has_legal_hold": True, |
| 2789 |
"source_encryption_mode": 'none', |
| 2790 |
"source_replication_status": first, |
| 2791 |
}, { |
| 2792 |
"count": 1, |
| 2793 |
"destination_replication_status": None, |
| 2794 |
"hash_differs": None, |
| 2795 |
"metadata_differs": None, |
| 2796 |
"source_has_file_retention": False, |
| 2797 |
"source_has_hide_marker": False, |
| 2798 |
"source_has_large_metadata": False, |
| 2799 |
"source_has_legal_hold": False, |
| 2800 |
"source_encryption_mode": 'SSE-B2', |
| 2801 |
"source_replication_status": second, |
| 2802 |
}, { |
| 2803 |
"count": 1, |
| 2804 |
"destination_replication_status": None, |
| 2805 |
"hash_differs": None, |
| 2806 |
"metadata_differs": None, |
| 2807 |
"source_has_file_retention": False, |
| 2808 |
"source_has_hide_marker": False, |
| 2809 |
"source_has_large_metadata": False, |
| 2810 |
"source_has_legal_hold": False, |
| 2811 |
"source_encryption_mode": 'SSE-C', |
| 2812 |
"source_replication_status": None, |
| 2813 |
}, { |
| 2814 |
"count": 1, |
| 2815 |
"destination_replication_status": None, |
| 2816 |
"hash_differs": None, |
| 2817 |
"metadata_differs": None, |
| 2818 |
"source_has_file_retention": False, |
| 2819 |
"source_has_hide_marker": False, |
| 2820 |
"source_has_large_metadata": False, |
| 2821 |
"source_has_legal_hold": True, |
| 2822 |
"source_encryption_mode": 'SSE-C', |
| 2823 |
"source_replication_status": None, |
| 2824 |
} |
| 2825 |
] |
| 2826 |
} for first, second in itertools.product(['FAILED', 'PENDING'], ['FAILED', 'PENDING']) |
| 2827 |
] |
| 2828 |
|
| 2829 |
|
| 2830 |
def test_enable_file_lock_first_retention_second(b2_tool, bucket_name): |
| 2831 |
# enable file lock only |
| 2832 |
b2_tool.should_succeed(['update-bucket', bucket_name, '--file-lock-enabled']) |
| 2833 |
|
| 2834 |
# set retention with file lock already enabled |
| 2835 |
b2_tool.should_succeed( |
| 2836 |
[ |
| 2837 |
'update-bucket', bucket_name, '--default-retention-mode', 'compliance', |
| 2838 |
'--default-retention-period', '7 days' |
| 2839 |
] |
| 2840 |
) |
| 2841 |
|
| 2842 |
# attempt to re-enable should be a noop |
| 2843 |
b2_tool.should_succeed(['update-bucket', bucket_name, '--file-lock-enabled']) |
| 2844 |
|
| 2845 |
|
| 2846 |
def test_enable_file_lock_and_set_retention_at_once(b2_tool, bucket_name): |
| 2847 |
# attempt setting retention without file lock enabled |
| 2848 |
b2_tool.should_fail( |
| 2849 |
[ |
| 2850 |
'update-bucket', bucket_name, '--default-retention-mode', 'compliance', |
| 2851 |
'--default-retention-period', '7 days' |
| 2852 |
], r'ERROR: The bucket is not file lock enabled \(bucket_missing_file_lock\)' |
| 2853 |
) |
| 2854 |
|
| 2855 |
# enable file lock and set retention at once |
| 2856 |
b2_tool.should_succeed( |
| 2857 |
[ |
| 2858 |
'update-bucket', bucket_name, '--default-retention-mode', 'compliance', |
| 2859 |
'--default-retention-period', '7 days', '--file-lock-enabled' |
| 2860 |
] |
| 2861 |
) |
| 2862 |
|
| 2863 |
# attempt to re-enable should be a noop |
| 2864 |
b2_tool.should_succeed(['update-bucket', bucket_name, '--file-lock-enabled']) |
| 2865 |
|
| 2866 |
|
| 2867 |
def _assert_file_lock_configuration( |
| 2868 |
b2_tool, |
| 2869 |
file_id, |
| 2870 |
retention_mode: RetentionMode | None = None, |
| 2871 |
retain_until: int | None = None, |
| 2872 |
legal_hold: LegalHold | None = None |
| 2873 |
): |
| 2874 |
|
| 2875 |
file_version = b2_tool.should_succeed_json(['file-info', f"b2id://{file_id}"]) |
| 2876 |
if retention_mode is not None: |
| 2877 |
if file_version['fileRetention']['mode'] == 'unknown': |
| 2878 |
actual_file_retention = UNKNOWN_FILE_RETENTION_SETTING |
| 2879 |
else: |
| 2880 |
actual_file_retention = FileRetentionSetting.from_file_retention_value_dict( |
| 2881 |
file_version['fileRetention'] |
| 2882 |
) |
| 2883 |
expected_file_retention = FileRetentionSetting(retention_mode, retain_until) |
| 2884 |
assert expected_file_retention == actual_file_retention |
| 2885 |
if legal_hold is not None: |
| 2886 |
if file_version['legalHold'] == 'unknown': |
| 2887 |
actual_legal_hold = LegalHold.UNKNOWN |
| 2888 |
else: |
| 2889 |
actual_legal_hold = LegalHold.from_string_or_none(file_version['legalHold']) |
| 2890 |
assert legal_hold == actual_legal_hold |
| 2891 |
|
| 2892 |
|
| 2893 |
def test_upload_file__custom_upload_time(b2_tool, bucket_name, sample_file, b2_uri_args): |
| 2894 |
file_data = read_file(sample_file) |
| 2895 |
cut = 12345 |
| 2896 |
cut_printable = '1970-01-01 00:00:12' |
| 2897 |
args = [ |
| 2898 |
'upload-file', |
| 2899 |
'--no-progress', |
| 2900 |
'--custom-upload-time', |
| 2901 |
str(cut), |
| 2902 |
'--quiet', |
| 2903 |
bucket_name, |
| 2904 |
sample_file, |
| 2905 |
'a', |
| 2906 |
] |
| 2907 |
succeeded, stdout = b2_tool.run_command(args) |
| 2908 |
if not succeeded: |
| 2909 |
b2_tool.should_fail(args, 'custom_timestamp_not_allowed') |
| 2910 |
else: |
| 2911 |
# file_id, action, date, time, size(, replication), name |
| 2912 |
b2_tool.should_succeed( |
| 2913 |
['ls', '--long', *b2_uri_args(bucket_name)], '^4_z.* upload {} +{} a'.format( |
| 2914 |
cut_printable, |
| 2915 |
len(file_data), |
| 2916 |
) |
| 2917 |
) |
| 2918 |
# file_id, action, date, time, size(, replication), name |
| 2919 |
b2_tool.should_succeed( |
| 2920 |
['ls', '--long', '--replication', *b2_uri_args(bucket_name)], |
| 2921 |
f'^4_z.* upload {cut_printable} +{len(file_data)} - a' |
| 2922 |
) |
| 2923 |
|
| 2924 |
|
| 2925 |
@skip_on_windows |
| 2926 |
def test_upload_file__stdin_pipe_operator(request, bash_runner, b2_tool, bucket_name): |
| 2927 |
"""Test upload-file from stdin using pipe operator.""" |
| 2928 |
content = request.node.name |
| 2929 |
run = bash_runner( |
| 2930 |
f'echo -n {content!r} ' |
| 2931 |
f'| ' |
| 2932 |
f'{" ".join(b2_tool.parse_command(b2_tool.prepare_env()))} upload-file {bucket_name} - {request.node.name}.txt' |
| 2933 |
) |
| 2934 |
assert hashlib.sha1(content.encode()).hexdigest() in run.stdout |
| 2935 |
|
| 2936 |
|
| 2937 |
@skip_on_windows |
| 2938 |
def test_upload_unbound_stream__redirect_operator( |
| 2939 |
request, bash_runner, b2_tool, bucket_name, is_running_on_docker |
| 2940 |
): |
| 2941 |
"""Test upload-unbound-stream from stdin using redirect operator.""" |
| 2942 |
if is_running_on_docker: |
| 2943 |
pytest.skip('Not supported on Docker') |
| 2944 |
content = request.node.name |
| 2945 |
command = request.config.getoption('--sut') |
| 2946 |
run = bash_runner( |
| 2947 |
f'{command} upload-unbound-stream {bucket_name} <(echo -n {content}) {request.node.name}.txt' |
| 2948 |
) |
| 2949 |
assert hashlib.sha1(content.encode()).hexdigest() in run.stdout |
| 2950 |
|
| 2951 |
|
| 2952 |
def test_download_file_stdout( |
| 2953 |
b2_tool, bucket_name, sample_filepath, tmp_path, uploaded_sample_file |
| 2954 |
): |
| 2955 |
assert b2_tool.should_succeed( |
| 2956 |
['download-file', '--quiet', f"b2://{bucket_name}/{uploaded_sample_file['fileName']}", '-'], |
| 2957 |
) == sample_filepath.read_text() |
| 2958 |
assert b2_tool.should_succeed( |
| 2959 |
['download-file', '--quiet', f"b2id://{uploaded_sample_file['fileId']}", '-'], |
| 2960 |
) == sample_filepath.read_text() |
| 2961 |
|
| 2962 |
|
| 2963 |
def test_download_file_to_directory( |
| 2964 |
b2_tool, bucket_name, sample_filepath, tmp_path, uploaded_sample_file |
| 2965 |
): |
| 2966 |
downloads_directory = 'downloads' |
| 2967 |
target_directory = tmp_path / downloads_directory |
| 2968 |
target_directory.mkdir() |
| 2969 |
filename_as_path = pathlib.Path(uploaded_sample_file['fileName']) |
| 2970 |
|
| 2971 |
sample_file_content = sample_filepath.read_text() |
| 2972 |
b2_tool.should_succeed( |
| 2973 |
[ |
| 2974 |
'download-file', |
| 2975 |
'--quiet', |
| 2976 |
f"b2://{bucket_name}/{uploaded_sample_file['fileName']}", |
| 2977 |
str(target_directory), |
| 2978 |
], |
| 2979 |
) |
| 2980 |
downloaded_file = target_directory / filename_as_path |
| 2981 |
assert downloaded_file.read_text() == sample_file_content, \ |
| 2982 |
f'{downloaded_file}, {downloaded_file.read_text()}, {sample_file_content}' |
| 2983 |
|
| 2984 |
b2_tool.should_succeed( |
| 2985 |
[ |
| 2986 |
'download-file', |
| 2987 |
'--quiet', |
| 2988 |
f"b2id://{uploaded_sample_file['fileId']}", |
| 2989 |
str(target_directory), |
| 2990 |
], |
| 2991 |
) |
| 2992 |
# A second file should be created. |
| 2993 |
new_files = [ |
| 2994 |
filepath |
| 2995 |
for filepath in target_directory.glob(f'{filename_as_path.stem}*{filename_as_path.suffix}') |
| 2996 |
if filepath.name != filename_as_path.name |
| 2997 |
] |
| 2998 |
assert len(new_files) == 1, f'{new_files}' |
| 2999 |
assert new_files[0].read_text() == sample_file_content, \ |
| 3000 |
f'{new_files}, {new_files[0].read_text()}, {sample_file_content}' |
| 3001 |
|
| 3002 |
|
| 3003 |
def test_cat(b2_tool, bucket_name, sample_filepath, tmp_path, uploaded_sample_file): |
| 3004 |
assert b2_tool.should_succeed( |
| 3005 |
['cat', f"b2://{bucket_name}/{uploaded_sample_file['fileName']}"], |
| 3006 |
) == sample_filepath.read_text() |
| 3007 |
assert b2_tool.should_succeed(['cat', f"b2id://{uploaded_sample_file['fileId']}"] |
| 3008 |
) == sample_filepath.read_text() |
| 3009 |
|
| 3010 |
|
| 3011 |
def test_header_arguments(b2_tool, bucket_name, sample_filepath, tmp_path): |
| 3012 |
# yapf: disable |
| 3013 |
args = [ |
| 3014 |
'--cache-control', 'max-age=3600', |
| 3015 |
'--content-disposition', 'attachment', |
| 3016 |
'--content-encoding', 'gzip', |
| 3017 |
'--content-language', 'en', |
| 3018 |
'--expires', 'Thu, 01 Dec 2050 16:00:00 GMT', |
| 3019 |
] |
| 3020 |
# yapf: enable |
| 3021 |
expected_file_info = { |
| 3022 |
'b2-cache-control': 'max-age=3600', |
| 3023 |
'b2-content-disposition': 'attachment', |
| 3024 |
'b2-content-encoding': 'gzip', |
| 3025 |
'b2-content-language': 'en', |
| 3026 |
'b2-expires': 'Thu, 01 Dec 2050 16:00:00 GMT', |
| 3027 |
} |
| 3028 |
|
| 3029 |
def assert_expected(file_info, expected=expected_file_info): |
| 3030 |
for key, val in expected.items(): |
| 3031 |
assert file_info[key] == val |
| 3032 |
|
| 3033 |
status, stdout, stderr = b2_tool.execute( |
| 3034 |
[ |
| 3035 |
'upload-file', |
| 3036 |
'--quiet', |
| 3037 |
'--no-progress', |
| 3038 |
bucket_name, |
| 3039 |
str(sample_filepath), |
| 3040 |
'sample_file', |
| 3041 |
*args, |
| 3042 |
'--info', |
| 3043 |
'b2-content-disposition=will-be-overwritten', |
| 3044 |
] |
| 3045 |
) |
| 3046 |
assert status == 0 |
| 3047 |
file_version = json.loads(stdout) |
| 3048 |
assert_expected(file_version['fileInfo']) |
| 3049 |
|
| 3050 |
# Since we used both --info and --content-disposition to set b2-content-disposition, |
| 3051 |
# a warning should be emitted |
| 3052 |
assert 'will be overwritten' in stderr and 'b2-content-disposition = attachment' in stderr |
| 3053 |
|
| 3054 |
copied_version = b2_tool.should_succeed_json( |
| 3055 |
[ |
| 3056 |
'copy-file-by-id', '--quiet', *args, '--content-type', 'text/plain', |
| 3057 |
file_version['fileId'], bucket_name, 'copied_file' |
| 3058 |
] |
| 3059 |
) |
| 3060 |
assert_expected(copied_version['fileInfo']) |
| 3061 |
|
| 3062 |
download_output = b2_tool.should_succeed( |
| 3063 |
['download-file', f"b2id://{file_version['fileId']}", tmp_path / 'downloaded_file'] |
| 3064 |
) |
| 3065 |
assert re.search(r'CacheControl: *max-age=3600', download_output) |
| 3066 |
assert re.search(r'ContentDisposition: *attachment', download_output) |
| 3067 |
assert re.search(r'ContentEncoding: *gzip', download_output) |
| 3068 |
assert re.search(r'ContentLanguage: *en', download_output) |
| 3069 |
assert re.search(r'Expires: *Thu, 01 Dec 2050 16:00:00 GMT', download_output) |
| 3070 |
|
| 3071 |
|
| 3072 |
def test_notification_rules(b2_tool, bucket_name): |
| 3073 |
auth_dict = b2_tool.should_succeed_json(['get-account-info']) |
| 3074 |
if 'writeBucketNotifications' not in auth_dict['allowed']['capabilities']: |
| 3075 |
pytest.skip('Test account does not have writeBucketNotifications capability') |
| 3076 |
|
| 3077 |
private_preview_pattern = re.compile(r'FeaturePreviewWarning') |
| 3078 |
assert b2_tool.should_succeed_json( |
| 3079 |
["notification-rules", "list", f"b2://{bucket_name}", "--json"], |
| 3080 |
expected_stderr_pattern=private_preview_pattern |
| 3081 |
) == [] |
| 3082 |
|
| 3083 |
notification_rule = { |
| 3084 |
"eventTypes": ["b2:ObjectCreated:*"], |
| 3085 |
"isEnabled": True, |
| 3086 |
"name": "test-rule", |
| 3087 |
"objectNamePrefix": "", |
| 3088 |
"targetConfiguration": |
| 3089 |
{ |
| 3090 |
"customHeaders": None, |
| 3091 |
"hmacSha256SigningSecret": None, |
| 3092 |
"targetType": "webhook", |
| 3093 |
"url": "https://example.com/webhook", |
| 3094 |
} |
| 3095 |
} |
| 3096 |
# add rule |
| 3097 |
created_rule = b2_tool.should_succeed_json( |
| 3098 |
[ |
| 3099 |
"notification-rules", |
| 3100 |
"create", |
| 3101 |
"--json", |
| 3102 |
f"b2://{bucket_name}", |
| 3103 |
"test-rule", |
| 3104 |
"--webhook-url", |
| 3105 |
"https://example.com/webhook", |
| 3106 |
"--event-type", |
| 3107 |
"b2:ObjectCreated:*", |
| 3108 |
], |
| 3109 |
expected_stderr_pattern=private_preview_pattern |
| 3110 |
) |
| 3111 |
expected_rules = [{**notification_rule, "isSuspended": False, "suspensionReason": ""}] |
| 3112 |
assert created_rule == expected_rules[0] |
| 3113 |
|
| 3114 |
# modify rule |
| 3115 |
secret = "0testSecret000000000000000000032" |
| 3116 |
modified_rule = b2_tool.should_succeed_json( |
| 3117 |
[ |
| 3118 |
"notification-rules", |
| 3119 |
"update", |
| 3120 |
"--json", |
| 3121 |
f"b2://{bucket_name}/prefix", |
| 3122 |
"test-rule", |
| 3123 |
"--disable", |
| 3124 |
"--sign-secret", |
| 3125 |
secret, |
| 3126 |
], |
| 3127 |
expected_stderr_pattern=private_preview_pattern |
| 3128 |
) |
| 3129 |
expected_rules[0].update({"objectNamePrefix": "prefix", "isEnabled": False}) |
| 3130 |
expected_rules[0]["targetConfiguration"]["hmacSha256SigningSecret"] = secret |
| 3131 |
assert modified_rule == expected_rules[0] |
| 3132 |
|
| 3133 |
# read updated rules |
| 3134 |
assert b2_tool.should_succeed_json( |
| 3135 |
["notification-rules", "list", f"b2://{bucket_name}", "--json"], |
| 3136 |
expected_stderr_pattern=private_preview_pattern |
| 3137 |
) == expected_rules |
| 3138 |
|
| 3139 |
# delete rule by name |
| 3140 |
assert b2_tool.should_succeed( |
| 3141 |
["notification-rules", "delete", f"b2://{bucket_name}", "test-rule"], |
| 3142 |
expected_stderr_pattern=private_preview_pattern |
| 3143 |
) == f"Rule 'test-rule' has been deleted from b2://{bucket_name}/\n" |
| 3144 |
assert b2_tool.should_succeed_json( |
| 3145 |
["notification-rules", "list", f"b2://{bucket_name}", "--json"], |
| 3146 |
expected_stderr_pattern=private_preview_pattern |
| 3147 |
) == [] |