###################################################################### # # File: test/unit/test_console_tool.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### import json import os import pathlib import re from functools import lru_cache from io import StringIO from itertools import chain, product from test.helpers import skip_on_windows from typing import List, Optional from unittest import mock import pytest from b2sdk import v1 from b2sdk.v2 import ( ALL_CAPABILITIES, B2Api, B2HttpApiConfig, ProgressReport, RawSimulator, StubAccountInfo, TempDir, UploadSourceBytes, fix_windows_path_limit, ) from b2sdk.v2.exception import Conflict # Any error for testing fast-fail of the rm command. from more_itertools import one from b2._internal._b2v4.registry import Rm as v4Rm from b2._internal._cli.const import ( B2_APPLICATION_KEY_ENV_VAR, B2_APPLICATION_KEY_ID_ENV_VAR, B2_ENVIRONMENT_ENV_VAR, ) from b2._internal.b2v3.rm import Rm as v3Rm from b2._internal.version import VERSION from .test_base import TestBase def file_mod_time_millis(path): return int(os.path.getmtime(path) * 1000) class BaseConsoleToolTest(TestBase): RE_API_VERSION = re.compile(r"\/v\d\/") json_pattern = re.compile(r'[^{,^\[]*(?P{.*})|(?P\[.*]).*', re.DOTALL) def setUp(self): self.account_info = StubAccountInfo() # this is a hack - B2HttpApiConfig expects a class, but we want to use an instance # which will be reused during the test, thus instead of class we pass a lambda which # returns already predefined instance self.raw_simulator = RawSimulator() self.api_config = B2HttpApiConfig(_raw_api_class=lambda *args, **kwargs: self.raw_simulator) @lru_cache(maxsize=None) def _get_b2api(**kwargs) -> B2Api: kwargs.pop('profile', None) return B2Api( account_info=self.account_info, cache=None, api_config=self.api_config, **kwargs, ) self.console_tool_class._initialize_b2_api = lambda cls, args, kwargs: _get_b2api(**kwargs) self.b2_api = _get_b2api() self.raw_api = self.b2_api.session.raw_api self.account_id, self.master_key = self.raw_api.create_account() for env_var_name in [ B2_APPLICATION_KEY_ID_ENV_VAR, B2_APPLICATION_KEY_ENV_VAR, B2_ENVIRONMENT_ENV_VAR, ]: os.environ.pop(env_var_name, None) def _get_stdouterr(self): stdout = StringIO() stderr = StringIO() return stdout, stderr def _run_command_ignore_output(self, argv): """ Runs the given command in the console tool, checking that it success, but ignoring the stdout. """ stdout, stderr = self._get_stdouterr() actual_status = self.console_tool_class(stdout, stderr).run_command(['b2'] + argv) actual_stderr = self._trim_trailing_spaces(stderr.getvalue()) if actual_stderr != '': print('ACTUAL STDERR: ', repr(actual_stderr)) print(actual_stderr) assert re.match(r'^(|Using https?://[\w.]+)$', actual_stderr), f"stderr: {actual_stderr!r}" self.assertEqual(0, actual_status, 'exit status code') def _trim_leading_spaces(self, s): """ Takes the contents of a triple-quoted string, and removes the leading newline and leading spaces that come from it being indented with code. """ # The first line starts on the line following the triple # quote, so the first line after splitting can be discarded. lines = s.split('\n') if lines[0] == '': lines = lines[1:] if len(lines) == 0: return '' # Count the leading spaces space_count = min((self._leading_spaces(line) for line in lines if line != ''), default=0) # Remove the leading spaces from each line, based on the line # with the fewest leading spaces leading_spaces = ' ' * space_count assert all( line.startswith(leading_spaces) or line == '' for line in lines ), 'all lines have leading spaces' return '\n'.join('' if line == '' else line[space_count:] for line in lines) def _leading_spaces(self, s): space_count = 0 while space_count < len(s) and s[space_count] == ' ': space_count += 1 return space_count def _trim_trailing_spaces(self, s): return '\n'.join(line.rstrip() for line in s.split('\n')) def _make_local_file(self, temp_dir, file_name): local_path = os.path.join(temp_dir, file_name) with open(local_path, 'wb') as f: f.write(b'hello world') return local_path def _read_file(self, local_path): with open(local_path, 'rb') as f: return f.read() def _remove_api_version_number(self, s): return re.sub(self.RE_API_VERSION, '/vx/', s) def _normalize_expected_output(self, text, format_vars=None): if text is None: return None format_vars = format_vars or {} return self._trim_leading_spaces(text).format( account_id=self.account_id, master_key=self.master_key, **format_vars ) def assertDictIsContained(self, subset, superset): """Asserts that all keys in `subset` are present is `superset` and their corresponding values are the same""" truncated_superset = {k: v for k, v in superset.items() if k in subset} self.maxDiff = None self.assertEqual(subset, truncated_superset) def assertListOfDictsIsContained(self, list_of_subsets, list_of_supersets): """Performs the same assertion as assertDictIsContained, but for dicts in two lists itertively""" self.assertEqual(len(list_of_subsets), len(list_of_supersets)) truncated_list_of_supersets = [] for subset, superset in zip(list_of_subsets, list_of_supersets): truncated_list_of_supersets.append({k: v for k, v in superset.items() if k in subset}) self.assertEqual(list_of_subsets, truncated_list_of_supersets) def _authorize_account(self): """ Prepare for a test by authorizing an account and getting an account auth token """ self._run_command_ignore_output(['authorize-account', self.account_id, self.master_key]) def _clear_account(self): """ Clear account auth data """ self._run_command_ignore_output(['clear-account']) def _create_my_bucket(self): self._run_command(['create-bucket', 'my-bucket', 'allPublic'], 'bucket_0\n', '', 0) def _run_command( self, argv, expected_stdout=None, expected_stderr='', expected_status=0, format_vars=None, remove_version=False, expected_json_in_stdout: Optional[dict] = None, expected_part_of_stdout=None, unexpected_part_of_stdout=None, ): """ Runs one command using the ConsoleTool, checking stdout, stderr, and the returned status code. The expected output strings are format strings (as used by str.format), so braces need to be escaped by doubling them. The variables 'account_id' and 'master_key' are set by default, plus any variables passed in the dict format_vars. The ConsoleTool is stateless, so we can make a new one for each call, with a fresh stdout and stderr. However, last instance of ConsoleTool is stored in `self.console_tool`, may be handy for testing internals of the tool after last command invocation. """ expected_stderr = self._normalize_expected_output(expected_stderr, format_vars) stdout, stderr = self._get_stdouterr() self.console_tool = self.console_tool_class(stdout, stderr) try: actual_status = self.console_tool.run_command(['b2'] + argv) except SystemExit as e: actual_status = e.code actual_stdout = self._trim_trailing_spaces(stdout.getvalue()) actual_stderr = self._trim_trailing_spaces(stderr.getvalue()) # ignore any references to specific api version if remove_version: actual_stdout = self._remove_api_version_number(actual_stdout) actual_stderr = self._remove_api_version_number(actual_stderr) if expected_stdout is not None and expected_stdout != actual_stdout: expected_stdout = self._normalize_expected_output(expected_stdout, format_vars) print('EXPECTED STDOUT:', repr(expected_stdout)) print('ACTUAL STDOUT: ', repr(actual_stdout)) print(actual_stdout) if expected_part_of_stdout is not None and expected_part_of_stdout not in actual_stdout: expected_part_of_stdout = self._normalize_expected_output( expected_part_of_stdout, format_vars ) print('EXPECTED TO FIND IN STDOUT:', repr(expected_part_of_stdout)) print('ACTUAL STDOUT: ', repr(actual_stdout)) if expected_stderr is not None and expected_stderr != actual_stderr: print('EXPECTED STDERR:', repr(expected_stderr)) print('ACTUAL STDERR: ', repr(actual_stderr)) print(actual_stderr) if expected_json_in_stdout is not None: json_match = self.json_pattern.match(actual_stdout) if not json_match: self.fail('EXPECTED TO FIND A JSON IN: ' + repr(actual_stdout)) found_json = json.loads(json_match.group('dict_json') or json_match.group('list_json')) if json_match.group('dict_json'): self.assertDictIsContained(expected_json_in_stdout, found_json) else: self.assertListOfDictsIsContained(expected_json_in_stdout, found_json) if expected_stdout is not None: self.assertEqual(expected_stdout, actual_stdout, 'stdout') if expected_part_of_stdout is not None: self.assertIn(expected_part_of_stdout, actual_stdout) if unexpected_part_of_stdout is not None: self.assertNotIn(unexpected_part_of_stdout, actual_stdout) if expected_stderr is not None: self.assertEqual(expected_stderr, actual_stderr, 'stderr') assert expected_status == actual_status return actual_status, actual_stdout, actual_stderr @classmethod def _upload_multiple_files(cls, bucket): data = UploadSourceBytes(b'test-data') bucket.upload(data, 'a/test.csv') bucket.upload(data, 'a/test.tsv') bucket.upload(data, 'b/b/test.csv') bucket.upload(data, 'b/b1/test.csv') bucket.upload(data, 'b/b2/test.tsv') bucket.upload(data, 'b/test.txt') bucket.upload(data, 'c/test.csv') bucket.upload(data, 'c/test.tsv') class TestTTYConsoleTool(BaseConsoleToolTest): def _get_stdouterr(self): class FakeStringIO(StringIO): def isatty(self): return True stdout = FakeStringIO() stderr = FakeStringIO() return stdout, stderr def test_e_c1_char_ls_default_escape_control_chars_setting(self): self._authorize_account() self._run_command(['create-bucket', 'my-bucket-cc', 'allPrivate'], 'bucket_0\n', '', 0) with TempDir() as temp_dir: local_file = self._make_local_file(temp_dir, "x") bad_str = "\u009b2K\u009b7Gb\u009b24Gx\u009b4GH" escaped_bad_str = "\\x9b2K\\x9b7Gb\\x9b24Gx\\x9b4GH" self._run_command(['upload-file', '--no-progress', 'my-bucket-cc', local_file, bad_str]) self._run_command( ['upload-file', '--no-progress', 'my-bucket-cc', local_file, "some_normal_text"] ) self._run_command( ['ls', *self.b2_uri_args('my-bucket-cc')], expected_part_of_stdout=escaped_bad_str ) class TestConsoleTool(BaseConsoleToolTest): @pytest.mark.apiver(to_ver=3) def test_camel_case_supported_in_v3(self): self._authorize_account() self._run_command( ['create-bucket', 'my-bucket', '--bucketInfo', '{"xxx": "123"}', 'allPrivate'], 'bucket_0\n', '', 0 ) self._run_command( ['create-bucket', 'my-bucket-kebab', '--bucket-info', '{"xxx": "123"}', 'allPrivate'], 'bucket_1\n', '', 0 ) @pytest.mark.apiver(from_ver=4) def test_camel_case_not_supported_in_v4(self): self._authorize_account() self._run_command(['create-bucket', 'my-bucket', '--bucketInfo', 'allPrivate'], '', '', 2) self._run_command( ['create-bucket', 'my-bucket-kebab', '--bucket-info', '{"xxx": "123"}', 'allPrivate'], 'bucket_0\n', '', 0 ) def test_create_key_and_authorize_with_it(self): # Start with authorizing with the master key self._authorize_account() # Create a key self._run_command( ['create-key', 'key1', 'listBuckets,listKeys'], 'appKeyId0 appKey0\n', '', 0, ) # Authorize with the key self._run_command( ['authorize-account', 'appKeyId0', 'appKey0'], None, '', 0, ) def test_create_key_with_authorization_from_env_vars(self): # Initial condition assert self.account_info.get_account_auth_token() is None # Authorize an account with a good api key. # Setting up environment variables with mock.patch.dict( 'os.environ', { B2_APPLICATION_KEY_ID_ENV_VAR: self.account_id, B2_APPLICATION_KEY_ENV_VAR: self.master_key, } ): assert B2_APPLICATION_KEY_ID_ENV_VAR in os.environ assert B2_APPLICATION_KEY_ENV_VAR in os.environ # The first time we're running on this cache there will be output from the implicit "authorize-account" call self._run_command( ['create-key', 'key1', 'listBuckets,listKeys'], 'appKeyId0 appKey0\n', '', 0, ) # The second time "authorize-account" is not called self._run_command( ['create-key', 'key1', 'listBuckets,listKeys,writeKeys'], 'appKeyId1 appKey1\n', '', 0, ) with mock.patch.dict( 'os.environ', { B2_APPLICATION_KEY_ID_ENV_VAR: 'appKeyId1', B2_APPLICATION_KEY_ENV_VAR: 'appKey1', } ): # "authorize-account" is called when the key changes self._run_command( ['create-key', 'key1', 'listBuckets,listKeys'], 'appKeyId2 appKey2\n', '', 0, ) # "authorize-account" is also called when the realm changes with mock.patch.dict( 'os.environ', { B2_ENVIRONMENT_ENV_VAR: 'http://custom.example.com', } ): self._run_command( ['create-key', 'key1', 'listBuckets,listKeys'], 'appKeyId3 appKey3\n', 'Using http://custom.example.com\n', 0, ) def test_authorize_key_without_list_buckets(self): self._authorize_account() # Create a key without listBuckets self._run_command(['create-key', 'key1', 'listKeys'], 'appKeyId0 appKey0\n', '', 0) # Authorize with the key self._run_command( ['authorize-account', 'appKeyId0', 'appKey0'], '', 'ERROR: application key has no listBuckets capability, which is required for the b2 command-line tool\n', 1, ) def test_create_bucket__with_lifecycle_rule(self): self._authorize_account() rule = json.dumps( { "daysFromHidingToDeleting": 1, "daysFromUploadingToHiding": None, "fileNamePrefix": "" } ) self._run_command( ['create-bucket', 'my-bucket', 'allPrivate', '--lifecycle-rule', rule], 'bucket_0\n', '', 0 ) def test_create_bucket__with_lifecycle_rules(self): self._authorize_account() rules = json.dumps( [ { "daysFromHidingToDeleting": 1, "daysFromUploadingToHiding": None, "fileNamePrefix": "" } ] ) self._run_command( ['create-bucket', 'my-bucket', 'allPrivate', '--lifecycle-rules', rules], 'bucket_0\n', '', 0 ) def test_create_bucket__mutually_exclusive_lifecycle_rules_options(self): self._authorize_account() rule = json.dumps( { "daysFromHidingToDeleting": 1, "daysFromUploadingToHiding": None, "fileNamePrefix": "" } ) self._run_command( [ 'create-bucket', 'my-bucket', 'allPrivate', '--lifecycle-rule', rule, '--lifecycle-rules', f"[{rule}]" ], '', '', 2 ) def test_create_bucket_key_and_authorize_with_it(self): # Start with authorizing with the master key self._authorize_account() # Make a bucket self._run_command(['create-bucket', 'my-bucket', 'allPrivate'], 'bucket_0\n', '', 0) # Create a key restricted to that bucket self._run_command( ['create-key', '--bucket', 'my-bucket', 'key1', 'listKeys,listBuckets'], 'appKeyId0 appKey0\n', '', 0 ) # Authorize with the key self._run_command( ['authorize-account', 'appKeyId0', 'appKey0'], None, '', 0, ) def test_update_bucket_without_lifecycle(self): # Start with authorizing with the master key self._authorize_account() bucket_name = 'my-bucket-liferules' # Create a bucket with lifecycleRule self._run_command( [ 'create-bucket', '--lifecycle-rule', '{"daysFromHidingToDeleting": 2, "fileNamePrefix": "foo"}', bucket_name, 'allPrivate' ], 'bucket_0\n', '', 0 ) expected_stdout_dict = { "accountId": self.account_id, "bucketId": "bucket_0", "bucketInfo": { "xxx": "123" }, "bucketName": "my-bucket-liferules", "bucketType": "allPrivate", "lifecycleRules": [{ "daysFromHidingToDeleting": 2, "fileNamePrefix": "foo" }], } # Update some other attribute than lifecycleRule, which should remain intact self._run_command( ['update-bucket', bucket_name, '--bucket-info', '{"xxx": "123"}'], expected_json_in_stdout=expected_stdout_dict, ) def test_clear_account(self): # Initial condition self._authorize_account() assert self.account_info.get_account_auth_token() is not None # Clearing the account should remove the auth token # from the account info. self._run_command(['clear-account'], '', '', 0) assert self.account_info.get_account_auth_token() is None def test_buckets(self): self._authorize_account() # Make a bucket with an illegal name expected_stdout = 'ERROR: Bad request: illegal bucket name: bad/bucket/name\n' self._run_command(['create-bucket', 'bad/bucket/name', 'allPublic'], '', expected_stdout, 1) # Make two buckets self._run_command(['create-bucket', 'my-bucket', 'allPrivate'], 'bucket_0\n', '', 0) self._run_command(['create-bucket', 'your-bucket', 'allPrivate'], 'bucket_1\n', '', 0) # Update one of them expected_json = { "accountId": self.account_id, "bucketId": "bucket_0", "bucketInfo": {}, "bucketName": "my-bucket", "bucketType": "allPublic", "corsRules": [], "defaultServerSideEncryption": { "mode": "none" }, "lifecycleRules": [], "options": [], "revision": 2 } self._run_command( ['update-bucket', 'my-bucket', 'allPublic'], expected_json_in_stdout=expected_json ) # Make sure they are there expected_stdout = ''' bucket_0 allPublic my-bucket bucket_1 allPrivate your-bucket ''' self._run_command(['list-buckets'], expected_stdout, '', 0) # Delete one expected_stdout = '' self._run_command(['delete-bucket', 'your-bucket'], expected_stdout, '', 0) def test_encrypted_buckets(self): self._authorize_account() # Make two encrypted buckets self._run_command(['create-bucket', 'my-bucket', 'allPrivate'], 'bucket_0\n', '', 0) self._run_command( [ 'create-bucket', '--default-server-side-encryption=SSE-B2', 'your-bucket', 'allPrivate' ], 'bucket_1\n', '', 0 ) # Update the one without encryption expected_json = { "accountId": self.account_id, "bucketId": "bucket_0", "bucketInfo": {}, "bucketName": "my-bucket", "bucketType": "allPublic", "corsRules": [], "defaultServerSideEncryption": { "algorithm": "AES256", "mode": "SSE-B2" }, "lifecycleRules": [], "options": [], "revision": 2 } self._run_command( ['update-bucket', '--default-server-side-encryption=SSE-B2', 'my-bucket', 'allPublic'], expected_json_in_stdout=expected_json, ) # Update the one with encryption expected_json = { "accountId": self.account_id, "bucketId": "bucket_1", "bucketInfo": {}, "bucketName": "your-bucket", "bucketType": "allPrivate", "corsRules": [], "defaultServerSideEncryption": { "algorithm": "AES256", "mode": "SSE-B2" }, "lifecycleRules": [], "options": [], "revision": 2 } self._run_command( ['update-bucket', 'your-bucket', 'allPrivate'], expected_json_in_stdout=expected_json ) # Make sure they are there expected_stdout = ''' bucket_0 allPublic my-bucket bucket_1 allPrivate your-bucket ''' self._run_command(['list-buckets'], expected_stdout, '', 0) def test_keys(self): self._authorize_account() self._run_command(['create-bucket', 'my-bucket-a', 'allPublic'], 'bucket_0\n', '', 0) self._run_command(['create-bucket', 'my-bucket-b', 'allPublic'], 'bucket_1\n', '', 0) self._run_command(['create-bucket', 'my-bucket-c', 'allPublic'], 'bucket_2\n', '', 0) capabilities = ['readFiles', 'listBuckets'] capabilities_with_commas = ','.join(capabilities) # Make a key with an illegal name expected_stderr = 'ERROR: Bad request: illegal key name: bad_key_name\n' self._run_command( ['create-key', 'bad_key_name', capabilities_with_commas], '', expected_stderr, 1 ) # Make a key with negative validDurationInSeconds expected_stderr = 'ERROR: Bad request: valid duration must be greater than 0, and less than 1000 days in seconds\n' self._run_command( ['create-key', '--duration', '-456', 'goodKeyName', capabilities_with_commas], '', expected_stderr, 1 ) # Make a key with validDurationInSeconds outside of range expected_stderr = 'ERROR: Bad request: valid duration must be greater than 0, ' \ 'and less than 1000 days in seconds\n' self._run_command( ['create-key', '--duration', '0', 'goodKeyName', capabilities_with_commas], '', expected_stderr, 1 ) self._run_command( ['create-key', '--duration', '86400001', 'goodKeyName', capabilities_with_commas], '', expected_stderr, 1 ) # Create three keys self._run_command( ['create-key', 'goodKeyName-One', capabilities_with_commas], 'appKeyId0 appKey0\n', '', 0, ) self._run_command( [ 'create-key', '--bucket', 'my-bucket-a', 'goodKeyName-Two', capabilities_with_commas + ',readBucketEncryption' ], 'appKeyId1 appKey1\n', '', 0, ) self._run_command( [ 'create-key', '--bucket', 'my-bucket-b', 'goodKeyName-Three', capabilities_with_commas ], 'appKeyId2 appKey2\n', '', 0, ) self._run_command( ['create-key', '--all-capabilities', 'goodKeyName-Four'], 'appKeyId3 appKey3\n', '', 0, ) self._run_command( ['create-key', '--bucket', 'my-bucket-b', 'goodKeyName-Five', capabilities_with_commas], 'appKeyId4 appKey4\n', '', 0, ) # Delete one key self._run_command(['delete-key', 'appKeyId2'], 'appKeyId2\n', '', 0) # Delete one bucket, to test listing when a bucket is gone. self._run_command_ignore_output(['delete-bucket', 'my-bucket-b']) # List keys expected_list_keys_out = """ appKeyId0 goodKeyName-One appKeyId1 goodKeyName-Two appKeyId3 goodKeyName-Four appKeyId4 goodKeyName-Five """ expected_list_keys_out_long = """ appKeyId0 goodKeyName-One - - - '' readFiles,listBuckets appKeyId1 goodKeyName-Two my-bucket-a - - '' readFiles,listBuckets,readBucketEncryption appKeyId3 goodKeyName-Four - - - '' {} appKeyId4 goodKeyName-Five id=bucket_1 - - '' readFiles,listBuckets """.format(','.join(sorted(ALL_CAPABILITIES))) self._run_command(['list-keys'], expected_list_keys_out, '', 0) self._run_command(['list-keys', '--long'], expected_list_keys_out_long, '', 0) # authorize and make calls using application key with no restrictions self._run_command(['authorize-account', 'appKeyId0', 'appKey0'], None, '', 0) self._run_command( ['list-buckets'], 'bucket_0 allPublic my-bucket-a\nbucket_2 allPublic my-bucket-c\n', '', 0 ) expected_json = { "accountId": self.account_id, "bucketId": "bucket_0", "bucketInfo": {}, "bucketName": "my-bucket-a", "bucketType": "allPublic", "corsRules": [], "defaultServerSideEncryption": { "mode": None }, "lifecycleRules": [], "options": [], "revision": 1 } self._run_command(['get-bucket', 'my-bucket-a'], expected_json_in_stdout=expected_json) # authorize and make calls using an application key with bucket restrictions self._run_command(['authorize-account', 'appKeyId1', 'appKey1'], None, '', 0) self._run_command( ['list-buckets'], '', 'ERROR: Application key is restricted to bucket: my-bucket-a\n', 1 ) self._run_command( ['get-bucket', 'my-bucket-c'], '', 'ERROR: Application key is restricted to bucket: my-bucket-a\n', 1 ) expected_json = { "accountId": self.account_id, "bucketId": "bucket_0", "bucketInfo": {}, "bucketName": "my-bucket-a", "bucketType": "allPublic", "corsRules": [], "defaultServerSideEncryption": { "mode": "none" }, "lifecycleRules": [], "options": [], "revision": 1 } self._run_command(['get-bucket', 'my-bucket-a'], expected_json_in_stdout=expected_json) self._run_command( ['ls', '--json', *self.b2_uri_args('my-bucket-c')], '', 'ERROR: Application key is restricted to bucket: my-bucket-a\n', 1 ) def test_bucket_info_from_json(self): self._authorize_account() self._run_command(['create-bucket', 'my-bucket', 'allPublic'], 'bucket_0\n', '', 0) bucket_info = {'color': 'blue'} expected_json = { "accountId": self.account_id, "bucketId": "bucket_0", "bucketInfo": { "color": "blue" }, "bucketName": "my-bucket", "bucketType": "allPrivate", "corsRules": [], "defaultServerSideEncryption": { "mode": "none" }, "lifecycleRules": [], "options": [], "revision": 2 } self._run_command( ['update-bucket', '--bucket-info', json.dumps(bucket_info), 'my-bucket', 'allPrivate'], expected_json_in_stdout=expected_json, ) def test_files(self): self._authorize_account() self._run_command(['create-bucket', 'my-bucket', 'allPublic'], 'bucket_0\n', '', 0) with TempDir() as temp_dir: local_file1 = self._make_local_file(temp_dir, 'file1.txt') # For this test, use a mod time without millis. My mac truncates # millis and just leaves seconds. mod_time = 1500111222 os.utime(local_file1, (mod_time, mod_time)) self.assertEqual(1500111222, os.path.getmtime(local_file1)) # Upload a file expected_stdout = ''' URL by file name: http://download.example.com/file/my-bucket/file1.txt URL by fileId: http://download.example.com/b2api/vx/b2_download_file_by_id?fileId=9999''' expected_json = { "action": "upload", "contentSha1": "2aae6c35c94fcfb415dbe95f408b9ce91ee846ed", "contentType": "b2/x-auto", "fileId": "9999", "fileInfo": { "src_last_modified_millis": "1500111222000", "b2-cache-control": "private, max-age=3600" }, "fileName": "file1.txt", "serverSideEncryption": { "mode": "none" }, "size": 11, "uploadTimestamp": 5000 } self._run_command( [ 'upload-file', '--no-progress', 'my-bucket', local_file1, 'file1.txt', '--cache-control=private, max-age=3600' ], expected_json_in_stdout=expected_json, remove_version=True, expected_part_of_stdout=expected_stdout, ) # Get file info mod_time_str = str(file_mod_time_millis(local_file1)) expected_json = { "accountId": self.account_id, "action": "upload", "bucketId": "bucket_0", "size": 11, "contentSha1": "2aae6c35c94fcfb415dbe95f408b9ce91ee846ed", "contentType": "b2/x-auto", "fileId": "9999", "fileInfo": { "src_last_modified_millis": "1500111222000", "b2-cache-control": "private, max-age=3600" }, "fileName": "file1.txt", "serverSideEncryption": { "mode": "none" }, "uploadTimestamp": 5000 } self._run_command( ['file-info', 'b2id://9999'], expected_json_in_stdout=expected_json, ) # Hide the file expected_json = { "action": "hide", "contentSha1": "none", "fileId": "9998", "fileInfo": {}, "fileName": "file1.txt", "serverSideEncryption": { "mode": "none" }, "size": 0, "uploadTimestamp": 5001 } self._run_command( ['hide-file', 'my-bucket', 'file1.txt'], expected_json_in_stdout=expected_json, ) # List the file versions expected_json = [ { "action": "hide", "contentSha1": "none", "fileId": "9998", "fileInfo": {}, "fileName": "file1.txt", "serverSideEncryption": { "mode": "none" }, "size": 0, "uploadTimestamp": 5001 }, { "action": "upload", "contentSha1": "2aae6c35c94fcfb415dbe95f408b9ce91ee846ed", "contentType": "b2/x-auto", "fileId": "9999", "fileInfo": { "src_last_modified_millis": str(mod_time_str), "b2-cache-control": "private, max-age=3600" }, "fileName": "file1.txt", "serverSideEncryption": { "mode": "none" }, "size": 11, "uploadTimestamp": 5000 } ] self._run_command( ['ls', '--json', '--versions', *self.b2_uri_args('my-bucket')], expected_json_in_stdout=expected_json, ) # List the file names expected_stdout = ''' [] ''' self._run_command( ['ls', '--json', *self.b2_uri_args('my-bucket')], expected_stdout, '', 0 ) # Delete one file version, passing the name in expected_json = {"action": "delete", "fileId": "9998", "fileName": "file1.txt"} self._run_command( ['delete-file-version', 'file1.txt', '9998'], expected_json_in_stdout=expected_json ) # Delete one file version, not passing the name in expected_json = {"action": "delete", "fileId": "9999", "fileName": "file1.txt"} self._run_command( ['delete-file-version', '9999'], expected_json_in_stdout=expected_json ) def test_files_encrypted(self): self._authorize_account() self._run_command(['create-bucket', 'my-bucket', 'allPublic'], 'bucket_0\n', '', 0) with TempDir() as temp_dir: local_file1 = self._make_local_file(temp_dir, 'file1.txt') # For this test, use a mod time without millis. My mac truncates # millis and just leaves seconds. mod_time = 1500111222 os.utime(local_file1, (mod_time, mod_time)) self.assertEqual(1500111222, os.path.getmtime(local_file1)) # Upload a file expected_stdout = ''' URL by file name: http://download.example.com/file/my-bucket/file1.txt URL by fileId: http://download.example.com/b2api/vx/b2_download_file_by_id?fileId=9999''' expected_json = { "action": "upload", "contentSha1": "2aae6c35c94fcfb415dbe95f408b9ce91ee846ed", "contentType": "b2/x-auto", "fileId": "9999", "fileInfo": { "src_last_modified_millis": "1500111222000" }, "fileName": "file1.txt", "serverSideEncryption": { "algorithm": "AES256", "mode": "SSE-B2" }, "size": 11, "uploadTimestamp": 5000 } self._run_command( [ 'upload-file', '--no-progress', '--destination-server-side-encryption=SSE-B2', 'my-bucket', local_file1, 'file1.txt' ], expected_json_in_stdout=expected_json, remove_version=True, expected_part_of_stdout=expected_stdout, ) # Get file info mod_time_str = str(file_mod_time_millis(local_file1)) expected_json = { "accountId": self.account_id, "action": "upload", "bucketId": "bucket_0", "size": 11, "contentSha1": "2aae6c35c94fcfb415dbe95f408b9ce91ee846ed", "contentType": "b2/x-auto", "fileId": "9999", "fileInfo": { "src_last_modified_millis": "1500111222000" }, "fileName": "file1.txt", "serverSideEncryption": { "algorithm": "AES256", "mode": "SSE-B2" }, "uploadTimestamp": 5000 } self._run_command( ['file-info', 'b2id://9999'], expected_json_in_stdout=expected_json, ) # Download by name local_download1 = os.path.join(temp_dir, 'download1.txt') expected_stdout_template = ''' File name: file1.txt File id: 9999 Output file path: {output_path} File size: 11 Content type: b2/x-auto Content sha1: 2aae6c35c94fcfb415dbe95f408b9ce91ee846ed Encryption: mode=SSE-B2, algorithm=AES256 Retention: none Legal hold: INFO src_last_modified_millis: 1500111222000 Checksum matches Download finished ''' expected_stdout = expected_stdout_template.format( output_path=pathlib.Path(local_download1).resolve() ) self._run_command( ['download-file', '--no-progress', 'b2://my-bucket/file1.txt', local_download1], expected_stdout, '', 0 ) self.assertEqual(b'hello world', self._read_file(local_download1)) self.assertEqual(mod_time, int(round(os.path.getmtime(local_download1)))) # Download file by ID. (Same expected output as downloading by name) local_download2 = os.path.join(temp_dir, 'download2.txt') expected_stdout = expected_stdout_template.format( output_path=pathlib.Path(local_download2).resolve() ) self._run_command( ['download-file', '--no-progress', 'b2id://9999', local_download2], expected_stdout, '', 0 ) self.assertEqual(b'hello world', self._read_file(local_download2)) # Hide the file expected_json = { "action": "hide", "contentSha1": "none", "fileId": "9998", "fileInfo": {}, "fileName": "file1.txt", "serverSideEncryption": { "mode": "none" }, "size": 0, "uploadTimestamp": 5001 } self._run_command( ['hide-file', 'my-bucket', 'file1.txt'], expected_json_in_stdout=expected_json, ) # List the file versions expected_json = [ { "action": "hide", "contentSha1": "none", "fileId": "9998", "fileInfo": {}, "fileName": "file1.txt", "serverSideEncryption": { "mode": "none" }, "size": 0, "uploadTimestamp": 5001 }, { "action": "upload", "contentSha1": "2aae6c35c94fcfb415dbe95f408b9ce91ee846ed", "contentType": "b2/x-auto", "fileId": "9999", "fileInfo": { "src_last_modified_millis": str(mod_time_str) }, "fileName": "file1.txt", "serverSideEncryption": { "algorithm": "AES256", "mode": "SSE-B2" }, "size": 11, "uploadTimestamp": 5000 } ] self._run_command( ['ls', '--json', '--versions', *self.b2_uri_args('my-bucket')], expected_json_in_stdout=expected_json, ) # List the file names expected_stdout = ''' [] ''' self._run_command( ['ls', '--json', *self.b2_uri_args('my-bucket')], expected_stdout, '', 0 ) # Delete one file version, passing the name in expected_json = {"action": "delete", "fileId": "9998", "fileName": "file1.txt"} self._run_command( ['delete-file-version', 'file1.txt', '9998'], expected_json_in_stdout=expected_json, ) # Delete one file version, not passing the name in expected_json = {"action": "delete", "fileId": "9999", "fileName": "file1.txt"} self._run_command( ['delete-file-version', '9999'], expected_json_in_stdout=expected_json, ) def _test_download_to_directory(self, download_by: str): self._authorize_account() self._create_my_bucket() base_filename = 'file' extension = '.txt' source_filename = f'{base_filename}{extension}' with TempDir() as temp_dir: local_file = self._make_local_file(temp_dir, source_filename) local_file_content = self._read_file(local_file) self._run_command( ['upload-file', '--no-progress', 'my-bucket', local_file, source_filename], remove_version=True, ) b2uri = f'b2://my-bucket/{source_filename}' if download_by == 'name' else 'b2id://9999' command = [ 'download-file', '--no-progress', b2uri, ] target_directory = os.path.join(temp_dir, 'target') os.mkdir(target_directory) command += [target_directory] self._run_command(command) self.assertEqual( local_file_content, self._read_file(os.path.join(target_directory, source_filename)) ) # Download the file second time, to check the override behavior. self._run_command(command) # We should get another file. target_directory_files = [ elem for elem in pathlib.Path(target_directory).glob(f'{base_filename}*{extension}') if elem.name != source_filename ] assert len(target_directory_files) == 1, f'{target_directory_files}' self.assertEqual(local_file_content, self._read_file(target_directory_files[0])) def test_download_by_id_to_directory(self): self._test_download_to_directory(download_by='id') def test_download_by_name_to_directory(self): self._test_download_to_directory(download_by='name') def test_copy_file_by_id(self): self._authorize_account() self._create_my_bucket() with TempDir() as temp_dir: local_file1 = self._make_local_file(temp_dir, 'file1.txt') # For this test, use a mod time without millis. My mac truncates # millis and just leaves seconds. mod_time = 1500111222 os.utime(local_file1, (mod_time, mod_time)) self.assertEqual(1500111222, os.path.getmtime(local_file1)) # Upload a file expected_stdout = ''' URL by file name: http://download.example.com/file/my-bucket/file1.txt URL by fileId: http://download.example.com/b2api/vx/b2_download_file_by_id?fileId=9999''' expected_json = { "action": "upload", "contentSha1": "2aae6c35c94fcfb415dbe95f408b9ce91ee846ed", "contentType": "b2/x-auto", "fileId": "9999", "fileInfo": { "src_last_modified_millis": "1500111222000" }, "fileName": "file1.txt", "serverSideEncryption": { "mode": "none" }, "size": 11, "uploadTimestamp": 5000 } self._run_command( ['upload-file', '--no-progress', 'my-bucket', local_file1, 'file1.txt'], expected_json_in_stdout=expected_json, remove_version=True, expected_part_of_stdout=expected_stdout, ) # Copy File expected_json = { "accountId": self.account_id, "action": "copy", "bucketId": "bucket_0", "size": 11, "contentSha1": "2aae6c35c94fcfb415dbe95f408b9ce91ee846ed", "contentType": "b2/x-auto", "fileId": "9998", "fileInfo": { "src_last_modified_millis": "1500111222000" }, "fileName": "file1_copy.txt", "serverSideEncryption": { "mode": "none" }, "uploadTimestamp": 5001 } self._run_command( ['copy-file-by-id', '9999', 'my-bucket', 'file1_copy.txt'], expected_json_in_stdout=expected_json, ) # Copy File with range parameter expected_json = { "accountId": self.account_id, "action": "copy", "bucketId": "bucket_0", "size": 5, "contentSha1": "4f664540ff30b8d34e037298a84e4736be39d731", "contentType": "b2/x-auto", "fileId": "9997", "fileInfo": { "src_last_modified_millis": "1500111222000" }, "fileName": "file1_copy.txt", "serverSideEncryption": { "mode": "none" }, "uploadTimestamp": 5002 } self._run_command( ['copy-file-by-id', '--range', '3,7', '9999', 'my-bucket', 'file1_copy.txt'], expected_json_in_stdout=expected_json, ) local_download1 = os.path.join(temp_dir, 'file1_copy.txt') self._run_command( ['download-file', '-q', 'b2://my-bucket/file1_copy.txt', local_download1] ) self.assertEqual(b'lo wo', self._read_file(local_download1)) # Invalid metadata copy with file info expected_stderr = "ERROR: File info can be set only when content type is set\n" self._run_command( [ 'copy-file-by-id', '--info', 'a=b', '9999', 'my-bucket', 'file1_copy.txt', ], '', expected_stderr, 1, ) # Invalid metadata replace without file info expected_stderr = "ERROR: File info can be not set only when content type is not set\n" self._run_command( [ 'copy-file-by-id', '--content-type', 'text/plain', '9999', 'my-bucket', 'file1_copy.txt', ], '', expected_stderr, 1, ) # replace with content type and file info expected_json = { "accountId": self.account_id, "action": "copy", "bucketId": "bucket_0", "size": 11, "contentSha1": "2aae6c35c94fcfb415dbe95f408b9ce91ee846ed", "contentType": "text/plain", "fileId": "9996", "fileInfo": { "a": "b" }, "fileName": "file1_copy.txt", "serverSideEncryption": { "mode": "none" }, "uploadTimestamp": 5003 } self._run_command( [ 'copy-file-by-id', '--content-type', 'text/plain', '--info', 'a=b', '9999', 'my-bucket', 'file1_copy.txt', ], expected_json_in_stdout=expected_json, ) # UnsatisfiableRange expected_stderr = "ERROR: The range in the request is outside the size of the file\n" self._run_command( ['copy-file-by-id', '--range', '12,20', '9999', 'my-bucket', 'file1_copy.txt'], '', expected_stderr, 1, ) # Copy in different bucket self._run_command(['create-bucket', 'my-bucket1', 'allPublic'], 'bucket_1\n', '', 0) expected_json = { "accountId": self.account_id, "action": "copy", "bucketId": "bucket_1", "size": 11, "contentSha1": "2aae6c35c94fcfb415dbe95f408b9ce91ee846ed", "contentType": "b2/x-auto", "fileId": "9994", "fileInfo": { "src_last_modified_millis": "1500111222000" }, "fileName": "file1_copy.txt", "serverSideEncryption": { "mode": "none" }, "uploadTimestamp": 5004 } self._run_command( ['copy-file-by-id', '9999', 'my-bucket1', 'file1_copy.txt'], expected_json_in_stdout=expected_json, ) def test_get_download_auth_defaults(self): self._authorize_account() self._create_my_bucket() self._run_command( ['get-download-auth', 'my-bucket'], 'fake_download_auth_token_bucket_0__86400\n', '', 0 ) def test_get_download_auth_explicit(self): self._authorize_account() self._create_my_bucket() self._run_command( ['get-download-auth', '--prefix', 'prefix', '--duration', '12345', 'my-bucket'], 'fake_download_auth_token_bucket_0_prefix_12345\n', '', 0 ) def test_get_download_auth_url(self): self._authorize_account() self._create_my_bucket() self._run_command( ['get-download-url-with-auth', '--duration', '12345', 'my-bucket', 'my-file'], 'http://download.example.com/file/my-bucket/my-file?Authorization=fake_download_auth_token_bucket_0_my-file_12345\n', '', 0 ) def test_get_download_auth_url_with_encoding(self): self._authorize_account() self._create_my_bucket() self._run_command( ['get-download-url-with-auth', '--duration', '12345', 'my-bucket', '\u81ea'], 'http://download.example.com/file/my-bucket/%E8%87%AA?Authorization=fake_download_auth_token_bucket_0_%E8%87%AA_12345\n', '', 0 ) def test_list_unfinished_large_files_with_none(self): self._authorize_account() self._create_my_bucket() self._run_command(['list-unfinished-large-files', 'my-bucket'], '', '', 0) def test_upload_large_file(self): self._authorize_account() self._create_my_bucket() min_part_size = self.account_info.get_recommended_part_size() file_size = min_part_size * 3 with TempDir() as temp_dir: file_path = os.path.join(temp_dir, 'test.txt') text = '*' * file_size with open(file_path, 'wb') as f: f.write(text.encode('utf-8')) mod_time_str = str(file_mod_time_millis(file_path)) expected_stdout = ''' URL by file name: http://download.example.com/file/my-bucket/test.txt URL by fileId: http://download.example.com/b2api/vx/b2_download_file_by_id?fileId=9999''' expected_json = { "action": "upload", "contentSha1": "none", "contentType": "b2/x-auto", "fileId": "9999", "fileInfo": { "large_file_sha1": "cc8954ec25e0c564b6a693fb22200e4f832c18e8", "src_last_modified_millis": str(mod_time_str) }, "fileName": "test.txt", "serverSideEncryption": { "mode": "none" }, "size": 600, "uploadTimestamp": 5000 } self._run_command( [ 'upload-file', '--no-progress', '--threads', '5', 'my-bucket', file_path, 'test.txt' ], expected_json_in_stdout=expected_json, remove_version=True, expected_part_of_stdout=expected_stdout, ) def test_upload_large_file_encrypted(self): self._authorize_account() self._run_command(['create-bucket', 'my-bucket', 'allPublic'], 'bucket_0\n', '', 0) min_part_size = self.account_info.get_recommended_part_size() file_size = min_part_size * 3 with TempDir() as temp_dir: file_path = os.path.join(temp_dir, 'test.txt') text = '*' * file_size with open(file_path, 'wb') as f: f.write(text.encode('utf-8')) mod_time_str = str(file_mod_time_millis(file_path)) expected_stdout = ''' URL by file name: http://download.example.com/file/my-bucket/test.txt URL by fileId: http://download.example.com/b2api/vx/b2_download_file_by_id?fileId=9999''' expected_json = { "action": "upload", "contentSha1": "none", "contentType": "b2/x-auto", "fileId": "9999", "fileInfo": { "large_file_sha1": "cc8954ec25e0c564b6a693fb22200e4f832c18e8", "src_last_modified_millis": str(mod_time_str) }, "fileName": "test.txt", "serverSideEncryption": { "algorithm": "AES256", "mode": "SSE-B2" }, "size": 600, "uploadTimestamp": 5000 } self._run_command( [ 'upload-file', '--no-progress', '--destination-server-side-encryption=SSE-B2', '--threads', '5', 'my-bucket', file_path, 'test.txt' ], expected_json_in_stdout=expected_json, remove_version=True, expected_part_of_stdout=expected_stdout, ) def test_upload_incremental(self): self._authorize_account() self._run_command(['create-bucket', 'my-bucket', 'allPublic'], 'bucket_0\n', '', 0) min_part_size = self.account_info.get_recommended_part_size() file_size = min_part_size * 2 with TempDir() as temp_dir: file_path = pathlib.Path(temp_dir) / 'test.txt' incremental_upload_params = [ 'upload-file', '--no-progress', '--threads', '5', '--incremental-mode', 'my-bucket', str(file_path), 'test.txt', ] file_path.write_bytes(b'*' * file_size) self._run_command(incremental_upload_params) with open(file_path, 'ab') as f: f.write(b'*' * min_part_size) self._run_command(incremental_upload_params) downloaded_path = pathlib.Path(temp_dir) / 'out.txt' self._run_command( [ 'download-file', '-q', 'b2://my-bucket/test.txt', str(downloaded_path), ] ) assert downloaded_path.read_bytes() == file_path.read_bytes() def test_get_account_info(self): self._authorize_account() expected_json = { "accountAuthToken": "auth_token_0", "accountFilePath": getattr(self.account_info, 'filename', None), # missing in StubAccountInfo in tests "accountId": self.account_id, "allowed": { "bucketId": None, "bucketName": None, "capabilities": sorted(ALL_CAPABILITIES), "namePrefix": None }, "apiUrl": "http://api.example.com", "applicationKey": self.master_key, "downloadUrl": "http://download.example.com", "s3endpoint": "http://s3.api.example.com", } self._run_command( ['get-account-info'], expected_json_in_stdout=expected_json, ) def test_get_bucket(self): self._authorize_account() self._create_my_bucket() expected_json = { "accountId": self.account_id, "bucketId": "bucket_0", "bucketInfo": {}, "bucketName": "my-bucket", "bucketType": "allPublic", "corsRules": [], "defaultServerSideEncryption": { "mode": "none" }, "lifecycleRules": [], "options": [], "revision": 1 } self._run_command( ['get-bucket', 'my-bucket'], expected_json_in_stdout=expected_json, ) def test_get_bucket_empty_show_size(self): self._authorize_account() self._create_my_bucket() expected_json = { "accountId": self.account_id, "bucketId": "bucket_0", "bucketInfo": {}, "bucketName": "my-bucket", "bucketType": "allPublic", "corsRules": [], "defaultServerSideEncryption": { "mode": "none" }, "fileCount": 0, "lifecycleRules": [], "options": [], "revision": 1, "totalSize": 0 } self._run_command( ['get-bucket', '--show-size', 'my-bucket'], expected_json_in_stdout=expected_json, ) def test_get_bucket_one_item_show_size(self): self._authorize_account() self._create_my_bucket() with TempDir() as temp_dir: # Upload a standard test file. local_file1 = self._make_local_file(temp_dir, 'file1.txt') mod_time_str = str(file_mod_time_millis(local_file1)) expected_stdout = ''' URL by file name: http://download.example.com/file/my-bucket/file1.txt URL by fileId: http://download.example.com/b2api/vx/b2_download_file_by_id?fileId=9999''' expected_json = { "action": "upload", "contentSha1": "2aae6c35c94fcfb415dbe95f408b9ce91ee846ed", "contentType": "b2/x-auto", "fileId": "9999", "fileInfo": { "src_last_modified_millis": str(mod_time_str) }, "fileName": "file1.txt", "serverSideEncryption": { "mode": "none" }, "size": 11, "uploadTimestamp": 5000 } self._run_command( ['upload-file', '--no-progress', 'my-bucket', local_file1, 'file1.txt'], expected_json_in_stdout=expected_json, remove_version=True, expected_part_of_stdout=expected_stdout, ) # Now check the output of get-bucket against the canon. expected_json = { "accountId": self.account_id, "bucketId": "bucket_0", "bucketInfo": {}, "bucketName": "my-bucket", "bucketType": "allPublic", "corsRules": [], "defaultServerSideEncryption": { "mode": "none" }, "fileCount": 1, "lifecycleRules": [], "options": [], "revision": 1, "totalSize": 11 } self._run_command( ['get-bucket', '--show-size', 'my-bucket'], expected_json_in_stdout=expected_json, ) def test_get_bucket_with_versions(self): self._authorize_account() self._create_my_bucket() # Put many versions of a file into the test bucket. Unroll the loop here for convenience. bucket = self.b2_api.get_bucket_by_name('my-bucket') bucket.upload(UploadSourceBytes(b'test'), 'test') bucket.upload(UploadSourceBytes(b'test'), 'test') bucket.upload(UploadSourceBytes(b'test'), 'test') bucket.upload(UploadSourceBytes(b'test'), 'test') bucket.upload(UploadSourceBytes(b'test'), 'test') bucket.upload(UploadSourceBytes(b'test'), 'test') bucket.upload(UploadSourceBytes(b'test'), 'test') bucket.upload(UploadSourceBytes(b'test'), 'test') bucket.upload(UploadSourceBytes(b'test'), 'test') bucket.upload(UploadSourceBytes(b'test'), 'test') # Now check the output of get-bucket against the canon. expected_json = { "accountId": self.account_id, "bucketId": "bucket_0", "bucketInfo": {}, "bucketName": "my-bucket", "bucketType": "allPublic", "corsRules": [], "defaultServerSideEncryption": { "mode": "none" }, "fileCount": 10, "lifecycleRules": [], "options": [], "revision": 1, "totalSize": 40 } self._run_command( ['get-bucket', '--show-size', 'my-bucket'], expected_json_in_stdout=expected_json, ) def test_get_bucket_with_folders(self): self._authorize_account() self._create_my_bucket() # Create a hierarchical structure within the test bucket. Unroll the loop here for # convenience. bucket = self.b2_api.get_bucket_by_name('my-bucket') bucket.upload(UploadSourceBytes(b'test'), 'test') bucket.upload(UploadSourceBytes(b'test'), '1/test') bucket.upload(UploadSourceBytes(b'test'), '1/2/test') bucket.upload(UploadSourceBytes(b'test'), '1/2/3/test') bucket.upload(UploadSourceBytes(b'test'), '1/2/3/4/test') bucket.upload(UploadSourceBytes(b'test'), '1/2/3/4/5/test') bucket.upload(UploadSourceBytes(b'test'), '1/2/3/4/5/6/test') bucket.upload(UploadSourceBytes(b'test'), '1/2/3/4/5/6/7/test') bucket.upload(UploadSourceBytes(b'test'), '1/2/3/4/5/6/7/8/test') bucket.upload(UploadSourceBytes(b'test'), '1/2/3/4/5/6/7/8/9/test') bucket.upload(UploadSourceBytes(b'check'), 'check') bucket.upload(UploadSourceBytes(b'check'), '1/check') bucket.upload(UploadSourceBytes(b'check'), '1/2/check') bucket.upload(UploadSourceBytes(b'check'), '1/2/3/check') bucket.upload(UploadSourceBytes(b'check'), '1/2/3/4/check') bucket.upload(UploadSourceBytes(b'check'), '1/2/3/4/5/check') bucket.upload(UploadSourceBytes(b'check'), '1/2/3/4/5/6/check') bucket.upload(UploadSourceBytes(b'check'), '1/2/3/4/5/6/7/check') bucket.upload(UploadSourceBytes(b'check'), '1/2/3/4/5/6/7/8/check') bucket.upload(UploadSourceBytes(b'check'), '1/2/3/4/5/6/7/8/9/check') # Now check the output of get-bucket against the canon. expected_json = { "accountId": self.account_id, "bucketId": "bucket_0", "bucketInfo": {}, "bucketName": "my-bucket", "bucketType": "allPublic", "corsRules": [], "defaultServerSideEncryption": { "mode": "none" }, "fileCount": 20, "lifecycleRules": [], "options": [], "revision": 1, "totalSize": 90 } self._run_command( ['get-bucket', '--show-size', 'my-bucket'], expected_json_in_stdout=expected_json, ) def test_get_bucket_with_hidden(self): self._authorize_account() self._create_my_bucket() # Put some files into the test bucket. Unroll the loop for convenience. bucket = self.b2_api.get_bucket_by_name('my-bucket') bucket.upload(UploadSourceBytes(b'test'), 'upload1') bucket.upload(UploadSourceBytes(b'test'), 'upload2') bucket.upload(UploadSourceBytes(b'test'), 'upload3') bucket.upload(UploadSourceBytes(b'test'), 'upload4') bucket.upload(UploadSourceBytes(b'test'), 'upload5') bucket.upload(UploadSourceBytes(b'test'), 'upload6') # Hide some new files. Don't check the results here; it will be clear enough that # something has failed if the output of 'get-bucket' does not match the canon. stdout, stderr = self._get_stdouterr() console_tool = self.console_tool_class(stdout, stderr) console_tool.run_command(['b2', 'hide-file', 'my-bucket', 'hidden1']) console_tool.run_command(['b2', 'hide-file', 'my-bucket', 'hidden2']) console_tool.run_command(['b2', 'hide-file', 'my-bucket', 'hidden3']) console_tool.run_command(['b2', 'hide-file', 'my-bucket', 'hidden4']) # Now check the output of get-bucket against the canon. expected_json = { "accountId": self.account_id, "bucketId": "bucket_0", "bucketInfo": {}, "bucketName": "my-bucket", "bucketType": "allPublic", "corsRules": [], "defaultServerSideEncryption": { "mode": "none" }, "fileCount": 10, "lifecycleRules": [], "options": [], "revision": 1, "totalSize": 24 } self._run_command( ['get-bucket', '--show-size', 'my-bucket'], expected_json_in_stdout=expected_json, ) def test_get_bucket_complex(self): self._authorize_account() self._create_my_bucket() # Create a hierarchical structure within the test bucket. Unroll the loop here for # convenience. bucket = self.b2_api.get_bucket_by_name('my-bucket') bucket.upload(UploadSourceBytes(b'test'), 'test') bucket.upload(UploadSourceBytes(b'test'), 'test') bucket.upload(UploadSourceBytes(b'test'), '1/test') bucket.upload(UploadSourceBytes(b'test'), '1/test') bucket.upload(UploadSourceBytes(b'test'), '1/2/test') bucket.upload(UploadSourceBytes(b'test'), '1/2/test') bucket.upload(UploadSourceBytes(b'test'), '1/2/3/test') bucket.upload(UploadSourceBytes(b'test'), '1/2/3/test') bucket.upload(UploadSourceBytes(b'test'), '1/2/3/test') bucket.upload(UploadSourceBytes(b'test'), '1/2/3/test') bucket.upload(UploadSourceBytes(b'test'), '1/2/3/test') bucket.upload(UploadSourceBytes(b'check'), 'check') bucket.upload(UploadSourceBytes(b'check'), 'check') bucket.upload(UploadSourceBytes(b'check'), '1/check') bucket.upload(UploadSourceBytes(b'check'), '1/check') bucket.upload(UploadSourceBytes(b'check'), '1/2/check') bucket.upload(UploadSourceBytes(b'check'), '1/2/check') bucket.upload(UploadSourceBytes(b'check'), '1/2/check') bucket.upload(UploadSourceBytes(b'check'), '1/2/3/check') bucket.upload(UploadSourceBytes(b'check'), '1/2/3/4/check') bucket.upload(UploadSourceBytes(b'check'), '1/2/3/4/check') bucket.upload(UploadSourceBytes(b'check'), '1/2/3/4/check') # Hide some new files. Don't check the results here; it will be clear enough that # something has failed if the output of 'get-bucket' does not match the canon. stdout, stderr = self._get_stdouterr() console_tool = self.console_tool_class(stdout, stderr) console_tool.run_command(['b2', 'hide-file', 'my-bucket', '1/hidden1']) console_tool.run_command(['b2', 'hide-file', 'my-bucket', '1/hidden1']) console_tool.run_command(['b2', 'hide-file', 'my-bucket', '1/hidden2']) console_tool.run_command(['b2', 'hide-file', 'my-bucket', '1/2/hidden3']) console_tool.run_command(['b2', 'hide-file', 'my-bucket', '1/2/hidden3']) console_tool.run_command(['b2', 'hide-file', 'my-bucket', '1/2/hidden3']) console_tool.run_command(['b2', 'hide-file', 'my-bucket', '1/2/hidden3']) # Now check the output of get-bucket against the canon. expected_json = { "accountId": self.account_id, "bucketId": "bucket_0", "bucketInfo": {}, "bucketName": "my-bucket", "bucketType": "allPublic", "corsRules": [], "defaultServerSideEncryption": { "mode": "none" }, "fileCount": 29, "lifecycleRules": [], "options": [], "revision": 1, "totalSize": 99 } self._run_command( ['get-bucket', '--show-size', 'my-bucket'], expected_json_in_stdout=expected_json, ) def test_get_bucket_encrypted(self): self._authorize_account() self._run_command( [ 'create-bucket', '--default-server-side-encryption=SSE-B2', '--default-server-side-encryption-algorithm=AES256', 'my-bucket', 'allPublic' ], 'bucket_0\n', '', 0 ) expected_json = { "accountId": self.account_id, "bucketId": "bucket_0", "bucketInfo": {}, "bucketName": "my-bucket", "bucketType": "allPublic", "corsRules": [], "defaultServerSideEncryption": { "algorithm": "AES256", "mode": "SSE-B2" }, "fileCount": 0, "lifecycleRules": [], "options": [], "revision": 1, "totalSize": 0 } self._run_command( ['get-bucket', '--show-size', 'my-bucket'], expected_json_in_stdout=expected_json, ) def test_sync(self): self._authorize_account() self._create_my_bucket() with TempDir() as temp_dir: file_path = os.path.join(temp_dir, 'test.txt') with open(file_path, 'wb') as f: f.write(b'hello world') expected_stdout = ''' upload test.txt ''' command = ['sync', '--no-progress', temp_dir, 'b2://my-bucket'] self._run_command(command, expected_stdout, '', 0) def test_sync_empty_folder_when_not_enabled(self): self._authorize_account() self._create_my_bucket() with TempDir() as temp_dir: command = ['sync', '--no-progress', temp_dir, 'b2://my-bucket'] expected_stderr = 'ERROR: Directory %s is empty. Use --allow-empty-source to sync anyway.\n' % fix_windows_path_limit( temp_dir.replace('\\\\', '\\') ) self._run_command(command, '', expected_stderr, 1) def test_sync_empty_folder_when_enabled(self): self._authorize_account() self._create_my_bucket() with TempDir() as temp_dir: command = ['sync', '--no-progress', '--allow-empty-source', temp_dir, 'b2://my-bucket'] self._run_command(command, '', '', 0) def test_sync_dry_run(self): self._authorize_account() self._create_my_bucket() with TempDir() as temp_dir: temp_file = self._make_local_file(temp_dir, 'test-dry-run.txt') # dry-run expected_stdout = ''' upload test-dry-run.txt ''' command = ['sync', '--no-progress', '--dry-run', temp_dir, 'b2://my-bucket'] self._run_command(command, expected_stdout, '', 0) # file should not have been uploaded expected_stdout = ''' [] ''' self._run_command( ['ls', '--json', *self.b2_uri_args('my-bucket')], expected_stdout, '', 0 ) # upload file expected_stdout = ''' upload test-dry-run.txt ''' command = ['sync', '--no-progress', temp_dir, 'b2://my-bucket'] self._run_command(command, expected_stdout, '', 0) # file should have been uploaded mtime = file_mod_time_millis(temp_file) expected_json = [ { "action": "upload", "contentSha1": "2aae6c35c94fcfb415dbe95f408b9ce91ee846ed", "contentType": "b2/x-auto", "fileId": "9999", "fileInfo": { "src_last_modified_millis": str(mtime) }, "fileName": "test-dry-run.txt", "serverSideEncryption": { "mode": "none" }, "size": 11, "uploadTimestamp": 5000 } ] self._run_command( ['ls', '--json', *self.b2_uri_args('my-bucket')], expected_json_in_stdout=expected_json, ) def test_sync_exclude_all_symlinks(self): self._authorize_account() self._create_my_bucket() with TempDir() as temp_dir: self._make_local_file(temp_dir, 'test.txt') os.symlink('test.txt', os.path.join(temp_dir, 'alink')) expected_stdout = ''' upload test.txt ''' command = [ 'sync', '--no-progress', '--exclude-all-symlinks', temp_dir, 'b2://my-bucket' ] self._run_command(command, expected_stdout, '', 0) def test_sync_dont_exclude_all_symlinks(self): self._authorize_account() self._create_my_bucket() with TempDir() as temp_dir: self._make_local_file(temp_dir, 'test.txt') os.symlink('test.txt', os.path.join(temp_dir, 'alink')) # Exact stdout cannot be asserted because line order is non-deterministic expected_part_of_stdout = ''' upload alink ''' command = ['sync', '--no-progress', temp_dir, 'b2://my-bucket'] self._run_command(command, expected_part_of_stdout=expected_part_of_stdout) def test_sync_exclude_if_modified_after_in_range(self): self._authorize_account() self._create_my_bucket() with TempDir() as temp_dir: for file, mtime in (('test.txt', 1367900664.152), ('test2.txt', 1367600664.152)): self._make_local_file(temp_dir, file) path = os.path.join(temp_dir, file) os.utime(path, (mtime, mtime)) expected_stdout = ''' upload test2.txt ''' command = [ 'sync', '--no-progress', '--exclude-if-modified-after', '1367700664.152', temp_dir, 'b2://my-bucket' ] self._run_command(command, expected_stdout, '', 0) def test_sync_exclude_if_modified_after_exact(self): self._authorize_account() self._create_my_bucket() with TempDir() as temp_dir: for file, mtime in (('test.txt', 1367900664.152), ('test2.txt', 1367600664.152)): self._make_local_file(temp_dir, file) path = os.path.join(temp_dir, file) os.utime(path, (mtime, mtime)) expected_stdout = ''' upload test2.txt ''' command = [ 'sync', '--no-progress', '--exclude-if-modified-after', '1367600664.152', temp_dir, 'b2://my-bucket' ] self._run_command(command, expected_stdout, '', 0) def _test_sync_threads( self, threads=None, sync_threads=None, download_threads=None, upload_threads=None, ): self._authorize_account() self._create_my_bucket() with TempDir() as temp_dir: self._make_local_file(temp_dir, 'file.txt') command = ['sync', '--no-progress'] if threads is not None: command += ['--threads', str(threads)] if sync_threads is not None: command += ['--sync-threads', str(sync_threads)] if download_threads is not None: command += ['--download-threads', str(download_threads)] if upload_threads is not None: command += ['--upload-threads', str(upload_threads)] command += [temp_dir, 'b2://my-bucket'] expected_stdout = ''' upload file.txt ''' self._run_command(command, expected_stdout) def test_sync_threads(self): self._test_sync_threads(threads=1) def test_sync_sync_threads(self): self._test_sync_threads(sync_threads=1) def test_sync_download_threads(self): self._test_sync_threads(download_threads=1) def test_sync_upload_threads(self): self._test_sync_threads(upload_threads=1) def test_sync_many_thread_options(self): self._test_sync_threads(sync_threads=1, download_threads=1, upload_threads=1) def test_sync_threads_and_upload_threads(self): # Using --threads is exclusive with other options with self.assertRaises(ValueError): self._test_sync_threads(threads=1, upload_threads=1) def test_sync_threads_and_sync_threads(self): # Using --threads is exclusive with other options with self.assertRaises(ValueError): self._test_sync_threads(threads=1, sync_threads=1) def test_sync_threads_and_download_threads(self): # Using --threads is exclusive with other options with self.assertRaises(ValueError): self._test_sync_threads(threads=1, download_threads=1) def test_sync_all_thread_options(self): # Using --threads is exclusive with other options with self.assertRaises(ValueError): self._test_sync_threads(threads=1, sync_threads=1, download_threads=1, upload_threads=1) def test_ls(self): self._authorize_account() self._create_my_bucket() # Check with no files self._run_command(['ls', *self.b2_uri_args('my-bucket')], '', '', 0) # Create some files, including files in a folder bucket = self.b2_api.get_bucket_by_name('my-bucket') bucket.upload(UploadSourceBytes(b''), 'a') bucket.upload(UploadSourceBytes(b' '), 'b/b1') bucket.upload(UploadSourceBytes(b' '), 'b/b2') bucket.upload(UploadSourceBytes(b' '), 'c') bucket.upload(UploadSourceBytes(b' '), 'c') # Condensed output expected_stdout = ''' a b/ c ''' self._run_command(['ls', *self.b2_uri_args('my-bucket')], expected_stdout, '', 0) # Recursive output expected_stdout = ''' a b/b1 b/b2 c ''' self._run_command( ['ls', '--recursive', *self.b2_uri_args('my-bucket')], expected_stdout, '', 0 ) self._run_command(['ls', '-r', *self.b2_uri_args('my-bucket')], expected_stdout, '', 0) # Check long output. (The format expects full-length file ids, so it causes whitespace here) expected_stdout = ''' 9999 upload 1970-01-01 00:00:05 0 a - - - - 0 b/ 9995 upload 1970-01-01 00:00:05 6 c ''' self._run_command(['ls', '--long', *self.b2_uri_args('my-bucket')], expected_stdout, '', 0) # Check long versions output (The format expects full-length file ids, so it causes whitespace here) expected_stdout = ''' 9999 upload 1970-01-01 00:00:05 0 a - - - - 0 b/ 9995 upload 1970-01-01 00:00:05 6 c 9996 upload 1970-01-01 00:00:05 5 c ''' self._run_command( ['ls', '--long', '--versions', *self.b2_uri_args('my-bucket')], expected_stdout, '', 0 ) def test_ls_wildcard(self): self._authorize_account() self._create_my_bucket() # Check with no files self._run_command( ['ls', '--recursive', '--with-wildcard', *self.b2_uri_args('my-bucket', '*.txt')], '', '', 0 ) # Create some files, including files in a folder bucket = self.b2_api.get_bucket_by_name('my-bucket') self._upload_multiple_files(bucket) expected_stdout = ''' a/test.csv a/test.tsv b/b/test.csv b/b1/test.csv b/b2/test.tsv c/test.csv c/test.tsv ''' self._run_command( ['ls', '--recursive', '--with-wildcard', *self.b2_uri_args('my-bucket', '*.[tc]sv')], expected_stdout, ) expected_stdout = ''' a/test.tsv b/b2/test.tsv c/test.tsv ''' self._run_command( ['ls', '--recursive', '--with-wildcard', *self.b2_uri_args('my-bucket', '*.tsv')], expected_stdout, ) expected_stdout = ''' b/b1/test.csv ''' self._run_command( [ 'ls', '--recursive', '--with-wildcard', *self.b2_uri_args('my-bucket', 'b/b?/test.csv') ], expected_stdout, ) expected_stdout = ''' a/test.csv a/test.tsv c/test.csv c/test.tsv ''' self._run_command( ['ls', '--recursive', '--with-wildcard', *self.b2_uri_args('my-bucket', '?/test.?sv')], expected_stdout, ) expected_stdout = ''' b/b/test.csv b/b1/test.csv ''' self._run_command( [ 'ls', '--recursive', '--with-wildcard', *self.b2_uri_args('my-bucket', '?/*/*.[!t]sv') ], expected_stdout, ) def test_ls_with_wildcard_no_recursive(self): self._authorize_account() self._create_my_bucket() # Check with no files self._run_command( ['ls', '--with-wildcard', *self.b2_uri_args('my-bucket')], expected_stderr='ERROR: with_wildcard requires recursive to be turned on as well\n', expected_status=1, ) def test_restrictions(self): # Initial condition self.assertEqual(None, self.account_info.get_account_auth_token()) # Authorize an account with the master key. account_id = self.account_id self._run_command_ignore_output(['authorize-account', account_id, self.master_key]) # Create a bucket to use bucket_name = 'restrictedBucket' bucket_id = 'bucket_0' self._run_command(['create-bucket', bucket_name, 'allPrivate'], bucket_id + '\n', '', 0) # Create another bucket other_bucket_name = 'otherBucket' self._run_command_ignore_output(['create-bucket', other_bucket_name, 'allPrivate']) # Create a key restricted to a bucket app_key_id = 'appKeyId0' app_key = 'appKey0' capabilities = "listBuckets,readFiles" file_prefix = 'some/file/prefix/' self._run_command( [ 'create-key', '--bucket', bucket_name, '--name-prefix', file_prefix, 'my-key', capabilities ], app_key_id + ' ' + app_key + '\n', '', 0, ) self._run_command_ignore_output(['authorize-account', app_key_id, app_key]) # Auth token should be in account info now self.assertEqual('auth_token_1', self.account_info.get_account_auth_token()) # Assertions that the restrictions not only are saved but what they are supposed to be self.assertEqual( dict( bucketId=bucket_id, bucketName=bucket_name, capabilities=[ 'listBuckets', 'readFiles', ], namePrefix=file_prefix, ), self.account_info.get_allowed(), ) # Test that the application key info gets added to the unauthorized error message. expected_create_key_stderr = "ERROR: unauthorized for application key " \ "with capabilities 'listBuckets,readFiles', " \ "restricted to bucket 'restrictedBucket', " \ "restricted to files that start with 'some/file/prefix/' (unauthorized)\n" self._run_command( ['create-key', 'goodKeyName-One', 'readFiles,listBuckets'], '', expected_create_key_stderr, 1, ) def test_list_buckets_not_allowed_for_app_key(self): # Create a bucket and a key restricted to that bucket. self._authorize_account() self._run_command( ['create-bucket', 'my-bucket', 'allPrivate'], 'bucket_0\n', '', 0, ) # Authorizing with the key will fail because the ConsoleTool needs # to be able to look up the name of the bucket. self._run_command( ['create-key', 'my-key', 'listFiles'], 'appKeyId0 appKey0\n', '', 0, ) # Authorize with the key, which should result in an error. self._run_command( ['authorize-account', 'appKeyId0', 'appKey0'], '', 'ERROR: application key has no listBuckets capability, which is required for the b2 command-line tool\n', 1, ) def test_bucket_missing_for_bucket_key(self): # Create a bucket and a key restricted to that bucket. self._authorize_account() self._run_command( ['create-bucket', 'my-bucket', 'allPrivate'], 'bucket_0\n', '', 0, ) self._run_command( ['create-key', '--bucket', 'my-bucket', 'my-key', 'listBuckets,listFiles'], 'appKeyId0 appKey0\n', '', 0, ) # Get rid of the bucket, leaving the key with a dangling pointer to it. self._run_command_ignore_output(['delete-bucket', 'my-bucket']) # Authorizing with the key will fail because the ConsoleTool needs # to be able to look up the name of the bucket. self._run_command( ['authorize-account', 'appKeyId0', 'appKey0'], '', "ERROR: unable to authorize account: Application key is restricted to a bucket that doesn't exist\n", 1, ) def test_ls_for_restricted_bucket(self): # Create a bucket and a key restricted to that bucket. self._authorize_account() self._run_command( ['create-bucket', 'my-bucket', 'allPrivate'], 'bucket_0\n', '', 0, ) self._run_command( ['create-key', '--bucket', 'my-bucket', 'my-key', 'listBuckets,listFiles'], 'appKeyId0 appKey0\n', '', 0, ) # Authorize with the key and list the files self._run_command_ignore_output(['authorize-account', 'appKeyId0', 'appKey0'],) self._run_command( ['ls', *self.b2_uri_args('my-bucket')], '', '', 0, ) def test_bad_terminal(self): stdout = mock.MagicMock() stdout.write = mock.MagicMock( side_effect=[ UnicodeEncodeError('codec', 'foo', 100, 105, 'artificial UnicodeEncodeError') ] + list(range(25)) ) stderr = mock.MagicMock() console_tool = self.console_tool_class(stdout, stderr) console_tool.run_command(['b2', 'authorize-account', self.account_id, self.master_key]) def test_passing_api_parameters(self): self._authorize_account() commands = [ [ 'b2', 'download-file-by-name', '--profile', 'nonexistent', 'dummy-name', 'dummy-file-name', 'dummy-local-file-name' ], [ 'b2', 'download-file-by-id', '--profile', 'nonexistent', 'dummy-id', 'dummy-local-file-name' ], ['b2', 'sync', '--profile', 'nonexistent', 'b2:dummy-source', 'dummy-destination'], ] parameters = [ { '--write-buffer-size': 123, '--skip-hash-verification': None, '--max-download-streams-per-file': 8, }, { '--write-buffer-size': 321, '--max-download-streams-per-file': 7, }, ] for command, params in product(commands, parameters): console_tool = self.console_tool_class( mock.MagicMock(), mock.MagicMock(), ) args = [str(val) for val in chain.from_iterable(params.items()) if val] console_tool.run_command(command + args) download_manager = console_tool.api.services.download_manager assert download_manager.write_buffer_size == params['--write-buffer-size'] assert download_manager.check_hash is ('--skip-hash-verification' not in params) parallel_strategy = one( strategy for strategy in download_manager.strategies if isinstance(strategy, download_manager.PARALLEL_DOWNLOADER_CLASS) ) assert parallel_strategy.max_streams == params['--max-download-streams-per-file'] def test_passing_api_parameters_with_auth_env_vars(self): os.environ[B2_APPLICATION_KEY_ID_ENV_VAR] = self.account_id os.environ[B2_APPLICATION_KEY_ENV_VAR] = self.master_key command = [ 'b2', 'download-file-by-id', 'dummy-id', 'dummy-local-file-name', '--write-buffer-size', '123', '--max-download-streams-per-file', '5', '--skip-hash-verification', ] console_tool = self.console_tool_class( mock.MagicMock(), mock.MagicMock(), ) console_tool.run_command(command) download_manager = console_tool.api.services.download_manager assert download_manager.write_buffer_size == 123 assert download_manager.check_hash is False parallel_strategy = one( strategy for strategy in download_manager.strategies if isinstance(strategy, download_manager.PARALLEL_DOWNLOADER_CLASS) ) assert parallel_strategy.max_streams == 5 @pytest.mark.apiver(from_ver=4) def test_ls_b2id(self): self._authorize_account() self._create_my_bucket() # Create a file bucket = self.b2_api.get_bucket_by_name('my-bucket') file_version = bucket.upload(UploadSourceBytes(b''), 'test.txt') # Condensed output expected_stdout = ''' test.txt ''' self._run_command(['ls', f'b2id://{file_version.id_}'], expected_stdout, '', 0) def test_ls_filters(self): self._authorize_account() self._create_my_bucket() # Create some files, including files in a folder bucket = self.b2_api.get_bucket_by_name('my-bucket') data = UploadSourceBytes(b'test-data') bucket.upload(data, 'a/test.csv') bucket.upload(data, 'a/test.tsv') bucket.upload(data, 'b/b/test.csv') bucket.upload(data, 'c/test.csv') bucket.upload(data, 'c/test.tsv') bucket.upload(data, 'test.csv') bucket.upload(data, 'test.tsv') expected_stdout = ''' a/ b/ c/ test.csv ''' self._run_command( ['ls', *self.b2_uri_args('my-bucket'), '--include', '*.csv'], expected_stdout, ) self._run_command( ['ls', *self.b2_uri_args('my-bucket'), '--exclude', '*.tsv'], expected_stdout, ) expected_stdout = ''' a/test.csv b/b/test.csv c/test.csv test.csv ''' self._run_command( ['ls', *self.b2_uri_args('my-bucket'), '--recursive', '--include', '*.csv'], expected_stdout, ) self._run_command( ['ls', *self.b2_uri_args('my-bucket'), '--recursive', '--exclude', '*.tsv'], expected_stdout, ) expected_stdout = ''' b/b/test.csv c/test.csv test.csv ''' self._run_command( [ 'ls', *self.b2_uri_args('my-bucket'), '--recursive', '--exclude', '*', '--include', '*.csv', '--exclude', 'a/*' ], expected_stdout, ) @pytest.mark.skip("temporarily disabled") @skip_on_windows def test_escape_c0_char_on_sync_stack_trace(self): self._authorize_account() self._run_command(['create-bucket', 'my-bucket-0', 'allPrivate'], 'bucket_0\n', '', 0) self._run_command(['create-bucket', 'my-bucket-1', 'allPrivate'], 'bucket_1\n', '', 0) with TempDir() as temp_dir: _ = self._make_local_file(temp_dir, "\x1b[32mC\x1b[33mC\x1b[34mI\x1b[0m") self._run_command( [ 'sync', '--no-progress', '--no-escape-control-characters', temp_dir, 'b2://my-bucket-0' ], expected_part_of_stdout='\\x1b[32m', expected_status=0, ) self._run_command( [ 'sync', '--no-progress', '--escape-control-characters', temp_dir, 'b2://my-bucket-1' ], expected_part_of_stdout="upload '\\x1b[32mC\\x1b[33mC\\x1b[34mI\\x1b[0m'\n", expected_status=0, unexpected_part_of_stdout='\x1b[32m', ) def test_escape_c0_char_on_key_restricted_path(self): self._authorize_account() self._run_command(['create-bucket', 'my-bucket-0', 'allPublic'], 'bucket_0\n', '', 0) cc_name = "$'\x1b[31mC\x1b[32mC\x1b[33mI\x1b[0m'" escaped_error = "ERROR: unauthorized for application key with capabilities 'listBuckets,listKeys', restricted to bucket 'my-bucket-0', restricted to files that start with '$'\\x1b[31mC\\x1b[32mC\\x1b[33mI\\x1b[0m'' (unauthorized)\n" # Create a key self._run_command( [ 'create-key', '--bucket', 'my-bucket-0', '--name-prefix', cc_name, 'key1', 'listBuckets,listKeys' ], 'appKeyId0 appKey0\n', expected_status=0, ) # Authorize with the key self._run_command(['authorize-account', 'appKeyId0', 'appKey0'], expected_status=0) self._run_command( ['ls', *self.b2_uri_args('my-bucket-0'), '--no-escape-control-characters'], expected_status=1, expected_stderr=escaped_error ) self._run_command( ['ls', *self.b2_uri_args('my-bucket-0'), '--escape-control-characters'], expected_status=1, expected_stderr=escaped_error ) self._run_command( ['ls', *self.b2_uri_args('my-bucket-0')], expected_status=1, expected_stderr=escaped_error ) def test_escape_c1_char_on_ls_long(self): self._authorize_account() self._run_command(['create-bucket', 'my-bucket-0', 'allPrivate'], 'bucket_0\n', '', 0) with TempDir() as temp_dir: local_file = self._make_local_file(temp_dir, 'file1.txt') cc_filename = '\u009bT\u009bE\u009bS\u009bTtest.txt' escaped_cc_filename = '\\x9bT\\x9bE\\x9bS\\x9bTtest.txt' self._run_command( ['upload-file', '--no-progress', 'my-bucket-0', local_file, cc_filename] ) self._run_command( ['ls', '--long', '--no-escape-control-characters', *self.b2_uri_args('my-bucket-0')], expected_part_of_stdout=cc_filename, ) self._run_command( ['ls', '--long', *self.b2_uri_args('my-bucket-0')], expected_part_of_stdout=cc_filename ) self._run_command( ['ls', '--long', '--escape-control-characters', *self.b2_uri_args('my-bucket-0')], expected_part_of_stdout=escaped_cc_filename, unexpected_part_of_stdout=cc_filename ) def test_escape_c1_char_ls(self): self._authorize_account() self._run_command(['create-bucket', 'my-bucket-cc', 'allPrivate'], 'bucket_0\n', '', 0) with TempDir() as temp_dir: local_file = self._make_local_file(temp_dir, "x") bad_str = "\u009b2K\u009b7Gb\u009b24Gx\u009b4GH" escaped_bad_str = "\\x9b2K\\x9b7Gb\\x9b24Gx\\x9b4GH" self._run_command(['upload-file', '--no-progress', 'my-bucket-cc', local_file, bad_str]) self._run_command( ['upload-file', '--no-progress', 'my-bucket-cc', local_file, "some_normal_text"] ) self._run_command( ['ls', *self.b2_uri_args('my-bucket-cc'), '--no-escape-control-characters'], expected_part_of_stdout=bad_str ) self._run_command( ['ls', *self.b2_uri_args('my-bucket-cc')], expected_part_of_stdout=bad_str ) self._run_command( ['ls', *self.b2_uri_args('my-bucket-cc'), '--escape-control-characters'], expected_part_of_stdout=escaped_bad_str ) class TestConsoleToolWithV1(BaseConsoleToolTest): """These tests use v1 interface to perform various setups before running CLI commands""" def setUp(self): super().setUp() self.v1_account_info = v1.StubAccountInfo() self.v1_b2_api = v1.B2Api(self.v1_account_info, None) self.v1_b2_api.session.raw_api = self.raw_api self.v1_b2_api.authorize_account('production', self.account_id, self.master_key) self._authorize_account() self._create_my_bucket() self.v1_bucket = self.v1_b2_api.create_bucket('my-v1-bucket', 'allPrivate') def test_cancel_large_file(self): file = self.v1_bucket.start_large_file('file1', 'text/plain', {}) self._run_command(['cancel-large-file', file.file_id], '9999 canceled\n', '', 0) def test_cancel_all_large_file(self): self.v1_bucket.start_large_file('file1', 'text/plain', {}) self.v1_bucket.start_large_file('file2', 'text/plain', {}) expected_stdout = ''' 9999 canceled 9998 canceled ''' self._run_command( ['cancel-all-unfinished-large-files', 'my-v1-bucket'], expected_stdout, '', 0 ) def test_list_parts_with_none(self): file = self.v1_bucket.start_large_file('file', 'text/plain', {}) self._run_command(['list-parts', file.file_id], '', '', 0) def test_list_parts_with_parts(self): bucket = self.b2_api.get_bucket_by_name('my-bucket') file = self.v1_bucket.start_large_file('file', 'text/plain', {}) content = b'hello world' large_file_upload_state = mock.MagicMock() large_file_upload_state.has_error.return_value = False bucket.api.services.upload_manager._upload_part( bucket.id_, file.file_id, UploadSourceBytes(content), 1, large_file_upload_state, None, None ) bucket.api.services.upload_manager._upload_part( bucket.id_, file.file_id, UploadSourceBytes(content), 3, large_file_upload_state, None, None ) expected_stdout = ''' 1 11 2aae6c35c94fcfb415dbe95f408b9ce91ee846ed 3 11 2aae6c35c94fcfb415dbe95f408b9ce91ee846ed ''' self._run_command(['list-parts', file.file_id], expected_stdout, '', 0) def test_list_unfinished_large_files_with_some(self): api_url = self.account_info.get_api_url() auth_token = self.account_info.get_account_auth_token() self.raw_api.start_large_file(api_url, auth_token, 'bucket_0', 'file1', 'text/plain', {}) self.raw_api.start_large_file( api_url, auth_token, 'bucket_0', 'file2', 'text/plain', {'color': 'blue'} ) self.raw_api.start_large_file( api_url, auth_token, 'bucket_0', 'file3', 'application/json', {} ) expected_stdout = ''' 9999 file1 text/plain 9998 file2 text/plain color=blue 9997 file3 application/json ''' self._run_command(['list-unfinished-large-files', 'my-bucket'], expected_stdout, '', 0) class TestRmConsoleTool(BaseConsoleToolTest): """ These tests replace default progress reporter of Rm class to ensure that it reports everything as fast as possible. """ class InstantReporter(ProgressReport): UPDATE_INTERVAL = 0.0 @classmethod def setUpClass(cls) -> None: cls.original_v3_progress_class = v3Rm.PROGRESS_REPORT_CLASS cls.original_v4_progress_class = v4Rm.PROGRESS_REPORT_CLASS v3Rm.PROGRESS_REPORT_CLASS = cls.InstantReporter v4Rm.PROGRESS_REPORT_CLASS = cls.InstantReporter def setUp(self): super().setUp() self._authorize_account() self._create_my_bucket() self.bucket = self.b2_api.get_bucket_by_name('my-bucket') self._upload_multiple_files(self.bucket) @classmethod def tearDownClass(cls) -> None: v3Rm.PROGRESS_REPORT_CLASS = cls.original_v3_progress_class v4Rm.PROGRESS_REPORT_CLASS = cls.original_v4_progress_class def test_rm_wildcard(self): self._run_command( [ 'rm', '--recursive', '--with-wildcard', '--no-progress', *self.b2_uri_args('my-bucket', '*.csv') ], ) expected_stdout = ''' a/test.tsv b/b2/test.tsv b/test.txt c/test.tsv ''' self._run_command(['ls', '--recursive', *self.b2_uri_args('my-bucket')], expected_stdout) def test_rm_versions(self): # Uploading content of the bucket again to create second version of each file. self._upload_multiple_files(self.bucket) self._run_command( [ 'rm', '--versions', '--recursive', '--with-wildcard', *self.b2_uri_args('my-bucket', '*.csv') ], ) expected_stdout = ''' a/test.tsv a/test.tsv b/b2/test.tsv b/b2/test.tsv b/test.txt b/test.txt c/test.tsv c/test.tsv ''' self._run_command( ['ls', '--versions', '--recursive', *self.b2_uri_args('my-bucket')], expected_stdout ) def test_rm_no_recursive(self): self._run_command(['rm', '--no-progress', *self.b2_uri_args('my-bucket', 'b/')]) expected_stdout = ''' a/test.csv a/test.tsv b/b/test.csv b/b1/test.csv b/b2/test.tsv c/test.csv c/test.tsv ''' self._run_command(['ls', '--recursive', *self.b2_uri_args('my-bucket')], expected_stdout) def test_rm_dry_run(self): expected_stdout = ''' a/test.csv b/b/test.csv b/b1/test.csv c/test.csv ''' self._run_command( [ 'rm', '--recursive', '--with-wildcard', '--dry-run', *self.b2_uri_args('my-bucket', '*.csv') ], expected_stdout, ) expected_stdout = ''' a/test.csv a/test.tsv b/b/test.csv b/b1/test.csv b/b2/test.tsv b/test.txt c/test.csv c/test.tsv ''' self._run_command(['ls', '--recursive', *self.b2_uri_args('my-bucket')], expected_stdout) def test_rm_exact_filename(self): self._run_command( [ 'rm', '--recursive', '--with-wildcard', '--no-progress', *self.b2_uri_args('my-bucket', 'b/b/test.csv') ], ) expected_stdout = ''' a/test.csv a/test.tsv b/b1/test.csv b/b2/test.tsv b/test.txt c/test.csv c/test.tsv ''' self._run_command(['ls', '--recursive', *self.b2_uri_args('my-bucket')], expected_stdout) def test_rm_no_name_removes_everything(self): self._run_command(['rm', '--recursive', '--no-progress', *self.b2_uri_args('my-bucket')]) self._run_command(['ls', '--recursive', *self.b2_uri_args('my-bucket')], '') def test_rm_with_wildcard_without_recursive(self): self._run_command( ['rm', '--with-wildcard', *self.b2_uri_args('my-bucket')], expected_stderr='ERROR: with_wildcard requires recursive to be turned on as well\n', expected_status=1, ) def test_rm_queue_size_and_number_of_threads(self): self._run_command( [ 'rm', '--recursive', '--threads', '2', '--queue-size', '4', *self.b2_uri_args('my-bucket') ] ) self._run_command(['ls', '--recursive', *self.b2_uri_args('my-bucket')], '') def test_rm_progress(self): expected_in_stdout = ' count: 4/4 ' self._run_command( ['rm', '--recursive', '--with-wildcard', *self.b2_uri_args('my-bucket', '*.csv')], expected_part_of_stdout=expected_in_stdout, ) expected_stdout = ''' a/test.tsv b/b2/test.tsv b/test.txt c/test.tsv ''' self._run_command(['ls', '--recursive', *self.b2_uri_args('my-bucket')], expected_stdout) def _run_problematic_removal( self, additional_parameters: Optional[List[str]] = None, expected_in_stdout: Optional[str] = None, unexpected_in_stdout: Optional[str] = None ): additional_parameters = additional_parameters or [] original_delete_file_version = self.b2_api.raw_api.delete_file_version def mocked_delete_file_version( this, account_auth_token, file_id, file_name, bypass_governance=False, *args, **kwargs ): if file_name == 'b/b1/test.csv': raise Conflict() return original_delete_file_version( this, account_auth_token, file_id, file_name, bypass_governance, *args, **kwargs ) with mock.patch.object( self.b2_api.raw_api, 'delete_file_version', side_effect=mocked_delete_file_version, ): self._run_command( [ 'rm', '--recursive', '--with-wildcard', '--threads', '1', '--queue-size', '1', *additional_parameters, *self.b2_uri_args('my-bucket', '*'), ], expected_status=1, expected_part_of_stdout=expected_in_stdout, unexpected_part_of_stdout=unexpected_in_stdout, ) def test_rm_fail_fast(self): # Since we already have all the jobs submitted to another thread, # we can only rely on the log to tell when it stopped. expected_in_stdout = ''' Deletion of file "b/b1/test.csv" (9996) failed: Conflict: count: 3/4''' unexpected_in_stdout = ' count: 5/5 ' self._run_problematic_removal(['--fail-fast'], expected_in_stdout, unexpected_in_stdout) def test_rm_skipping_over_errors(self): self._run_problematic_removal() expected_stdout = ''' b/b1/test.csv ''' self._run_command(['ls', '--recursive', *self.b2_uri_args('my-bucket')], expected_stdout) @pytest.mark.apiver(from_ver=4) def test_rm_b2id(self): # Create a file file_version = self.bucket.upload(UploadSourceBytes(b''), 'new-file.txt') # Before deleting expected_stdout = ''' a/test.csv a/test.tsv b/b/test.csv b/b1/test.csv b/b2/test.tsv b/test.txt c/test.csv c/test.tsv new-file.txt ''' self._run_command(['ls', '--recursive', 'b2://my-bucket'], expected_stdout) # Delete file self._run_command(['rm', '--no-progress', f'b2id://{file_version.id_}'], '', '', 0) # After deleting expected_stdout = ''' a/test.csv a/test.tsv b/b/test.csv b/b1/test.csv b/b2/test.tsv b/test.txt c/test.csv c/test.tsv ''' self._run_command(['ls', '--recursive', 'b2://my-bucket'], expected_stdout) def rm_filters_helper(self, rm_args: List[str], expected_ls_stdout: str): self._authorize_account() self._run_command(['create-bucket', 'my-rm-bucket', 'allPublic'], 'bucket_1\n', '', 0) bucket = self.b2_api.get_bucket_by_name('my-rm-bucket') # Create some files, including files in a folder data = UploadSourceBytes(b'test-data') bucket.upload(data, 'a/test.csv') bucket.upload(data, 'a/test.tsv') bucket.upload(data, 'b/b/test.csv') bucket.upload(data, 'c/test.csv') bucket.upload(data, 'c/test.tsv') bucket.upload(data, 'test.csv') bucket.upload(data, 'test.tsv') bucket.upload(data, 'test.txt') self._run_command( ['rm', '--no-progress', *self.b2_uri_args('my-rm-bucket'), *rm_args], '', '', 0 ) self._run_command( ['ls', *self.b2_uri_args('my-rm-bucket'), '--recursive'], expected_ls_stdout, ) def test_rm_filters_include(self): expected_ls_stdout = ''' a/test.csv a/test.tsv b/b/test.csv c/test.csv c/test.tsv test.tsv test.txt ''' self.rm_filters_helper(['--include', '*.csv'], expected_ls_stdout) def test_rm_filters_exclude(self): expected_ls_stdout = ''' a/test.csv a/test.tsv b/b/test.csv c/test.csv c/test.tsv test.csv ''' self.rm_filters_helper(['--exclude', '*.csv'], expected_ls_stdout) def test_rm_filters_include_recursive(self): expected_ls_stdout = ''' a/test.tsv c/test.tsv test.tsv test.txt ''' self.rm_filters_helper(['--recursive', '--include', '*.csv'], expected_ls_stdout) def test_rm_filters_exclude_recursive(self): expected_ls_stdout = ''' a/test.csv b/b/test.csv c/test.csv test.csv ''' self.rm_filters_helper(['--recursive', '--exclude', '*.csv'], expected_ls_stdout) def test_rm_filters_mixed(self): expected_ls_stdout = ''' a/test.csv a/test.tsv c/test.tsv test.tsv test.txt ''' self.rm_filters_helper( ['--recursive', '--exclude', '*', '--include', '*.csv', '--exclude', 'a/*'], expected_ls_stdout ) class TestVersionConsoleTool(BaseConsoleToolTest): def test_version(self): self._run_command(['version', '--short'], expected_stdout=f'{VERSION}\n') self._run_command(['version'], expected_stdout=f'b2 command line tool, version {VERSION}\n')