Debian Codegraph @

test/unit/test_console_tool.py

raw
1 ######################################################################
2 #
3 # File: test/unit/test_console_tool.py
4 #
5 # Copyright 2019 Backblaze Inc. All Rights Reserved.
6 #
7 # License https://www.backblaze.com/using_b2_code.html
8 #
9 ######################################################################
10
11 import json
12 import os
13 import pathlib
14 import re
15 from functools import lru_cache
16 from io import StringIO
17 from itertools import chain, product
18 from test.helpers import skip_on_windows
19 from typing import List, Optional
20 from unittest import mock
21
22 import pytest
23 from b2sdk import v1
24 from b2sdk.v2 import (
25 ALL_CAPABILITIES,
26 B2Api,
27 B2HttpApiConfig,
28 ProgressReport,
29 RawSimulator,
30 StubAccountInfo,
31 TempDir,
32 UploadSourceBytes,
33 fix_windows_path_limit,
34 )
35 from b2sdk.v2.exception import Conflict # Any error for testing fast-fail of the rm command.
36 from more_itertools import one
37
38 from b2._internal._b2v4.registry import Rm as v4Rm
39 from b2._internal._cli.const import (
40 B2_APPLICATION_KEY_ENV_VAR,
41 B2_APPLICATION_KEY_ID_ENV_VAR,
42 B2_ENVIRONMENT_ENV_VAR,
43 )
44 from b2._internal.b2v3.rm import Rm as v3Rm
45 from b2._internal.version import VERSION
46
47 from .test_base import TestBase
48
49
50 def file_mod_time_millis(path):
51 return int(os.path.getmtime(path) * 1000)
52
53
54 class BaseConsoleToolTest(TestBase):
55 RE_API_VERSION = re.compile(r"\/v\d\/")
56 json_pattern = re.compile(r'[^{,^\[]*(?P<dict_json>{.*})|(?P<list_json>\[.*]).*', re.DOTALL)
57
58 def setUp(self):
59 self.account_info = StubAccountInfo()
60
61 # this is a hack - B2HttpApiConfig expects a class, but we want to use an instance
62 # which will be reused during the test, thus instead of class we pass a lambda which
63 # returns already predefined instance
64 self.raw_simulator = RawSimulator()
65 self.api_config = B2HttpApiConfig(_raw_api_class=lambda *args, **kwargs: self.raw_simulator)
66
67 @lru_cache(maxsize=None)
68 def _get_b2api(**kwargs) -> B2Api:
69 kwargs.pop('profile', None)
70 return B2Api(
71 account_info=self.account_info,
72 cache=None,
73 api_config=self.api_config,
74 **kwargs,
75 )
76
77 self.console_tool_class._initialize_b2_api = lambda cls, args, kwargs: _get_b2api(**kwargs)
78
79 self.b2_api = _get_b2api()
80 self.raw_api = self.b2_api.session.raw_api
81 self.account_id, self.master_key = self.raw_api.create_account()
82
83 for env_var_name in [
84 B2_APPLICATION_KEY_ID_ENV_VAR,
85 B2_APPLICATION_KEY_ENV_VAR,
86 B2_ENVIRONMENT_ENV_VAR,
87 ]:
88 os.environ.pop(env_var_name, None)
89
90 def _get_stdouterr(self):
91 stdout = StringIO()
92 stderr = StringIO()
93 return stdout, stderr
94
95 def _run_command_ignore_output(self, argv):
96 """
97 Runs the given command in the console tool, checking that it
98 success, but ignoring the stdout.
99 """
100 stdout, stderr = self._get_stdouterr()
101 actual_status = self.console_tool_class(stdout, stderr).run_command(['b2'] + argv)
102 actual_stderr = self._trim_trailing_spaces(stderr.getvalue())
103
104 if actual_stderr != '':
105 print('ACTUAL STDERR: ', repr(actual_stderr))
106 print(actual_stderr)
107
108 assert re.match(r'^(|Using https?://[\w.]+)$', actual_stderr), f"stderr: {actual_stderr!r}"
109 self.assertEqual(0, actual_status, 'exit status code')
110
111 def _trim_leading_spaces(self, s):
112 """
113 Takes the contents of a triple-quoted string, and removes the leading
114 newline and leading spaces that come from it being indented with code.
115 """
116 # The first line starts on the line following the triple
117 # quote, so the first line after splitting can be discarded.
118 lines = s.split('\n')
119 if lines[0] == '':
120 lines = lines[1:]
121 if len(lines) == 0:
122 return ''
123
124 # Count the leading spaces
125 space_count = min((self._leading_spaces(line) for line in lines if line != ''), default=0)
126
127 # Remove the leading spaces from each line, based on the line
128 # with the fewest leading spaces
129 leading_spaces = ' ' * space_count
130 assert all(
131 line.startswith(leading_spaces) or line == '' for line in lines
132 ), 'all lines have leading spaces'
133 return '\n'.join('' if line == '' else line[space_count:] for line in lines)
134
135 def _leading_spaces(self, s):
136 space_count = 0
137 while space_count < len(s) and s[space_count] == ' ':
138 space_count += 1
139 return space_count
140
141 def _trim_trailing_spaces(self, s):
142 return '\n'.join(line.rstrip() for line in s.split('\n'))
143
144 def _make_local_file(self, temp_dir, file_name):
145 local_path = os.path.join(temp_dir, file_name)
146 with open(local_path, 'wb') as f:
147 f.write(b'hello world')
148 return local_path
149
150 def _read_file(self, local_path):
151 with open(local_path, 'rb') as f:
152 return f.read()
153
154 def _remove_api_version_number(self, s):
155 return re.sub(self.RE_API_VERSION, '/vx/', s)
156
157 def _normalize_expected_output(self, text, format_vars=None):
158 if text is None:
159 return None
160 format_vars = format_vars or {}
161 return self._trim_leading_spaces(text).format(
162 account_id=self.account_id, master_key=self.master_key, **format_vars
163 )
164
165 def assertDictIsContained(self, subset, superset):
166 """Asserts that all keys in `subset` are present is `superset` and their corresponding values are the same"""
167 truncated_superset = {k: v for k, v in superset.items() if k in subset}
168 self.maxDiff = None
169 self.assertEqual(subset, truncated_superset)
170
171 def assertListOfDictsIsContained(self, list_of_subsets, list_of_supersets):
172 """Performs the same assertion as assertDictIsContained, but for dicts in two lists itertively"""
173 self.assertEqual(len(list_of_subsets), len(list_of_supersets))
174 truncated_list_of_supersets = []
175 for subset, superset in zip(list_of_subsets, list_of_supersets):
176 truncated_list_of_supersets.append({k: v for k, v in superset.items() if k in subset})
177 self.assertEqual(list_of_subsets, truncated_list_of_supersets)
178
179 def _authorize_account(self):
180 """
181 Prepare for a test by authorizing an account and getting an account auth token
182 """
183 self._run_command_ignore_output(['authorize-account', self.account_id, self.master_key])
184
185 def _clear_account(self):
186 """
187 Clear account auth data
188 """
189 self._run_command_ignore_output(['clear-account'])
190
191 def _create_my_bucket(self):
192 self._run_command(['create-bucket', 'my-bucket', 'allPublic'], 'bucket_0\n', '', 0)
193
194 def _run_command(
195 self,
196 argv,
197 expected_stdout=None,
198 expected_stderr='',
199 expected_status=0,
200 format_vars=None,
201 remove_version=False,
202 expected_json_in_stdout: Optional[dict] = None,
203 expected_part_of_stdout=None,
204 unexpected_part_of_stdout=None,
205 ):
206 """
207 Runs one command using the ConsoleTool, checking stdout, stderr, and
208 the returned status code.
209
210 The expected output strings are format strings (as used by str.format),
211 so braces need to be escaped by doubling them. The variables 'account_id'
212 and 'master_key' are set by default, plus any variables passed in the dict
213 format_vars.
214
215 The ConsoleTool is stateless, so we can make a new one for each
216 call, with a fresh stdout and stderr. However, last instance of ConsoleTool
217 is stored in `self.console_tool`, may be handy for testing internals
218 of the tool after last command invocation.
219 """
220 expected_stderr = self._normalize_expected_output(expected_stderr, format_vars)
221 stdout, stderr = self._get_stdouterr()
222 self.console_tool = self.console_tool_class(stdout, stderr)
223 try:
224 actual_status = self.console_tool.run_command(['b2'] + argv)
225 except SystemExit as e:
226 actual_status = e.code
227
228 actual_stdout = self._trim_trailing_spaces(stdout.getvalue())
229 actual_stderr = self._trim_trailing_spaces(stderr.getvalue())
230
231 # ignore any references to specific api version
232 if remove_version:
233 actual_stdout = self._remove_api_version_number(actual_stdout)
234 actual_stderr = self._remove_api_version_number(actual_stderr)
235
236 if expected_stdout is not None and expected_stdout != actual_stdout:
237 expected_stdout = self._normalize_expected_output(expected_stdout, format_vars)
238 print('EXPECTED STDOUT:', repr(expected_stdout))
239 print('ACTUAL STDOUT: ', repr(actual_stdout))
240 print(actual_stdout)
241 if expected_part_of_stdout is not None and expected_part_of_stdout not in actual_stdout:
242 expected_part_of_stdout = self._normalize_expected_output(
243 expected_part_of_stdout, format_vars
244 )
245 print('EXPECTED TO FIND IN STDOUT:', repr(expected_part_of_stdout))
246 print('ACTUAL STDOUT: ', repr(actual_stdout))
247 if expected_stderr is not None and expected_stderr != actual_stderr:
248 print('EXPECTED STDERR:', repr(expected_stderr))
249 print('ACTUAL STDERR: ', repr(actual_stderr))
250 print(actual_stderr)
251
252 if expected_json_in_stdout is not None:
253 json_match = self.json_pattern.match(actual_stdout)
254 if not json_match:
255 self.fail('EXPECTED TO FIND A JSON IN: ' + repr(actual_stdout))
256
257 found_json = json.loads(json_match.group('dict_json') or json_match.group('list_json'))
258 if json_match.group('dict_json'):
259 self.assertDictIsContained(expected_json_in_stdout, found_json)
260 else:
261 self.assertListOfDictsIsContained(expected_json_in_stdout, found_json)
262
263 if expected_stdout is not None:
264 self.assertEqual(expected_stdout, actual_stdout, 'stdout')
265 if expected_part_of_stdout is not None:
266 self.assertIn(expected_part_of_stdout, actual_stdout)
267 if unexpected_part_of_stdout is not None:
268 self.assertNotIn(unexpected_part_of_stdout, actual_stdout)
269 if expected_stderr is not None:
270 self.assertEqual(expected_stderr, actual_stderr, 'stderr')
271 assert expected_status == actual_status
272 return actual_status, actual_stdout, actual_stderr
273
274 @classmethod
275 def _upload_multiple_files(cls, bucket):
276 data = UploadSourceBytes(b'test-data')
277 bucket.upload(data, 'a/test.csv')
278 bucket.upload(data, 'a/test.tsv')
279 bucket.upload(data, 'b/b/test.csv')
280 bucket.upload(data, 'b/b1/test.csv')
281 bucket.upload(data, 'b/b2/test.tsv')
282 bucket.upload(data, 'b/test.txt')
283 bucket.upload(data, 'c/test.csv')
284 bucket.upload(data, 'c/test.tsv')
285
286
287 class TestTTYConsoleTool(BaseConsoleToolTest):
288 def _get_stdouterr(self):
289 class FakeStringIO(StringIO):
290 def isatty(self):
291 return True
292
293 stdout = FakeStringIO()
294 stderr = FakeStringIO()
295 return stdout, stderr
296
297 def test_e_c1_char_ls_default_escape_control_chars_setting(self):
298 self._authorize_account()
299 self._run_command(['create-bucket', 'my-bucket-cc', 'allPrivate'], 'bucket_0\n', '', 0)
300
301 with TempDir() as temp_dir:
302 local_file = self._make_local_file(temp_dir, "x")
303 bad_str = "\u009b2K\u009b7Gb\u009b24Gx\u009b4GH"
304 escaped_bad_str = "\\x9b2K\\x9b7Gb\\x9b24Gx\\x9b4GH"
305
306 self._run_command(['upload-file', '--no-progress', 'my-bucket-cc', local_file, bad_str])
307 self._run_command(
308 ['upload-file', '--no-progress', 'my-bucket-cc', local_file, "some_normal_text"]
309 )
310
311 self._run_command(
312 ['ls', *self.b2_uri_args('my-bucket-cc')], expected_part_of_stdout=escaped_bad_str
313 )
314
315
316 class TestConsoleTool(BaseConsoleToolTest):
317 @pytest.mark.apiver(to_ver=3)
318 def test_camel_case_supported_in_v3(self):
319 self._authorize_account()
320 self._run_command(
321 ['create-bucket', 'my-bucket', '--bucketInfo', '{"xxx": "123"}', 'allPrivate'],
322 'bucket_0\n', '', 0
323 )
324 self._run_command(
325 ['create-bucket', 'my-bucket-kebab', '--bucket-info', '{"xxx": "123"}', 'allPrivate'],
326 'bucket_1\n', '', 0
327 )
328
329 @pytest.mark.apiver(from_ver=4)
330 def test_camel_case_not_supported_in_v4(self):
331 self._authorize_account()
332 self._run_command(['create-bucket', 'my-bucket', '--bucketInfo', 'allPrivate'], '', '', 2)
333 self._run_command(
334 ['create-bucket', 'my-bucket-kebab', '--bucket-info', '{"xxx": "123"}', 'allPrivate'],
335 'bucket_0\n', '', 0
336 )
337
338 def test_create_key_and_authorize_with_it(self):
339 # Start with authorizing with the master key
340 self._authorize_account()
341
342 # Create a key
343 self._run_command(
344 ['create-key', 'key1', 'listBuckets,listKeys'],
345 'appKeyId0 appKey0\n',
346 '',
347 0,
348 )
349
350 # Authorize with the key
351 self._run_command(
352 ['authorize-account', 'appKeyId0', 'appKey0'],
353 None,
354 '',
355 0,
356 )
357
358 def test_create_key_with_authorization_from_env_vars(self):
359 # Initial condition
360 assert self.account_info.get_account_auth_token() is None
361
362 # Authorize an account with a good api key.
363
364 # Setting up environment variables
365 with mock.patch.dict(
366 'os.environ', {
367 B2_APPLICATION_KEY_ID_ENV_VAR: self.account_id,
368 B2_APPLICATION_KEY_ENV_VAR: self.master_key,
369 }
370 ):
371 assert B2_APPLICATION_KEY_ID_ENV_VAR in os.environ
372 assert B2_APPLICATION_KEY_ENV_VAR in os.environ
373
374 # The first time we're running on this cache there will be output from the implicit "authorize-account" call
375 self._run_command(
376 ['create-key', 'key1', 'listBuckets,listKeys'],
377 'appKeyId0 appKey0\n',
378 '',
379 0,
380 )
381
382 # The second time "authorize-account" is not called
383 self._run_command(
384 ['create-key', 'key1', 'listBuckets,listKeys,writeKeys'],
385 'appKeyId1 appKey1\n',
386 '',
387 0,
388 )
389
390 with mock.patch.dict(
391 'os.environ', {
392 B2_APPLICATION_KEY_ID_ENV_VAR: 'appKeyId1',
393 B2_APPLICATION_KEY_ENV_VAR: 'appKey1',
394 }
395 ):
396 # "authorize-account" is called when the key changes
397 self._run_command(
398 ['create-key', 'key1', 'listBuckets,listKeys'],
399 'appKeyId2 appKey2\n',
400 '',
401 0,
402 )
403
404 # "authorize-account" is also called when the realm changes
405 with mock.patch.dict(
406 'os.environ', {
407 B2_ENVIRONMENT_ENV_VAR: 'http://custom.example.com',
408 }
409 ):
410 self._run_command(
411 ['create-key', 'key1', 'listBuckets,listKeys'],
412 'appKeyId3 appKey3\n',
413 'Using http://custom.example.com\n',
414 0,
415 )
416
417 def test_authorize_key_without_list_buckets(self):
418 self._authorize_account()
419
420 # Create a key without listBuckets
421 self._run_command(['create-key', 'key1', 'listKeys'], 'appKeyId0 appKey0\n', '', 0)
422
423 # Authorize with the key
424 self._run_command(
425 ['authorize-account', 'appKeyId0', 'appKey0'],
426 '',
427 'ERROR: application key has no listBuckets capability, which is required for the b2 command-line tool\n',
428 1,
429 )
430
431 def test_create_bucket__with_lifecycle_rule(self):
432 self._authorize_account()
433
434 rule = json.dumps(
435 {
436 "daysFromHidingToDeleting": 1,
437 "daysFromUploadingToHiding": None,
438 "fileNamePrefix": ""
439 }
440 )
441
442 self._run_command(
443 ['create-bucket', 'my-bucket', 'allPrivate', '--lifecycle-rule', rule], 'bucket_0\n',
444 '', 0
445 )
446
447 def test_create_bucket__with_lifecycle_rules(self):
448 self._authorize_account()
449
450 rules = json.dumps(
451 [
452 {
453 "daysFromHidingToDeleting": 1,
454 "daysFromUploadingToHiding": None,
455 "fileNamePrefix": ""
456 }
457 ]
458 )
459
460 self._run_command(
461 ['create-bucket', 'my-bucket', 'allPrivate', '--lifecycle-rules', rules], 'bucket_0\n',
462 '', 0
463 )
464
465 def test_create_bucket__mutually_exclusive_lifecycle_rules_options(self):
466 self._authorize_account()
467
468 rule = json.dumps(
469 {
470 "daysFromHidingToDeleting": 1,
471 "daysFromUploadingToHiding": None,
472 "fileNamePrefix": ""
473 }
474 )
475
476 self._run_command(
477 [
478 'create-bucket', 'my-bucket', 'allPrivate', '--lifecycle-rule', rule,
479 '--lifecycle-rules', f"[{rule}]"
480 ], '', '', 2
481 )
482
483 def test_create_bucket_key_and_authorize_with_it(self):
484 # Start with authorizing with the master key
485 self._authorize_account()
486
487 # Make a bucket
488 self._run_command(['create-bucket', 'my-bucket', 'allPrivate'], 'bucket_0\n', '', 0)
489
490 # Create a key restricted to that bucket
491 self._run_command(
492 ['create-key', '--bucket', 'my-bucket', 'key1', 'listKeys,listBuckets'],
493 'appKeyId0 appKey0\n', '', 0
494 )
495
496 # Authorize with the key
497 self._run_command(
498 ['authorize-account', 'appKeyId0', 'appKey0'],
499 None,
500 '',
501 0,
502 )
503
504 def test_update_bucket_without_lifecycle(self):
505 # Start with authorizing with the master key
506 self._authorize_account()
507
508 bucket_name = 'my-bucket-liferules'
509 # Create a bucket with lifecycleRule
510 self._run_command(
511 [
512 'create-bucket', '--lifecycle-rule',
513 '{"daysFromHidingToDeleting": 2, "fileNamePrefix": "foo"}', bucket_name,
514 'allPrivate'
515 ], 'bucket_0\n', '', 0
516 )
517
518 expected_stdout_dict = {
519 "accountId": self.account_id,
520 "bucketId": "bucket_0",
521 "bucketInfo": {
522 "xxx": "123"
523 },
524 "bucketName": "my-bucket-liferules",
525 "bucketType": "allPrivate",
526 "lifecycleRules": [{
527 "daysFromHidingToDeleting": 2,
528 "fileNamePrefix": "foo"
529 }],
530 }
531
532 # Update some other attribute than lifecycleRule, which should remain intact
533 self._run_command(
534 ['update-bucket', bucket_name, '--bucket-info', '{"xxx": "123"}'],
535 expected_json_in_stdout=expected_stdout_dict,
536 )
537
538 def test_clear_account(self):
539 # Initial condition
540 self._authorize_account()
541 assert self.account_info.get_account_auth_token() is not None
542
543 # Clearing the account should remove the auth token
544 # from the account info.
545 self._run_command(['clear-account'], '', '', 0)
546 assert self.account_info.get_account_auth_token() is None
547
548 def test_buckets(self):
549 self._authorize_account()
550
551 # Make a bucket with an illegal name
552 expected_stdout = 'ERROR: Bad request: illegal bucket name: bad/bucket/name\n'
553 self._run_command(['create-bucket', 'bad/bucket/name', 'allPublic'], '', expected_stdout, 1)
554
555 # Make two buckets
556 self._run_command(['create-bucket', 'my-bucket', 'allPrivate'], 'bucket_0\n', '', 0)
557 self._run_command(['create-bucket', 'your-bucket', 'allPrivate'], 'bucket_1\n', '', 0)
558
559 # Update one of them
560 expected_json = {
561 "accountId": self.account_id,
562 "bucketId": "bucket_0",
563 "bucketInfo": {},
564 "bucketName": "my-bucket",
565 "bucketType": "allPublic",
566 "corsRules": [],
567 "defaultServerSideEncryption": {
568 "mode": "none"
569 },
570 "lifecycleRules": [],
571 "options": [],
572 "revision": 2
573 }
574
575 self._run_command(
576 ['update-bucket', 'my-bucket', 'allPublic'], expected_json_in_stdout=expected_json
577 )
578
579 # Make sure they are there
580 expected_stdout = '''
581 bucket_0 allPublic my-bucket
582 bucket_1 allPrivate your-bucket
583 '''
584
585 self._run_command(['list-buckets'], expected_stdout, '', 0)
586
587 # Delete one
588 expected_stdout = ''
589
590 self._run_command(['delete-bucket', 'your-bucket'], expected_stdout, '', 0)
591
592 def test_encrypted_buckets(self):
593 self._authorize_account()
594
595 # Make two encrypted buckets
596 self._run_command(['create-bucket', 'my-bucket', 'allPrivate'], 'bucket_0\n', '', 0)
597 self._run_command(
598 [
599 'create-bucket', '--default-server-side-encryption=SSE-B2', 'your-bucket',
600 'allPrivate'
601 ], 'bucket_1\n', '', 0
602 )
603
604 # Update the one without encryption
605 expected_json = {
606 "accountId": self.account_id,
607 "bucketId": "bucket_0",
608 "bucketInfo": {},
609 "bucketName": "my-bucket",
610 "bucketType": "allPublic",
611 "corsRules": [],
612 "defaultServerSideEncryption": {
613 "algorithm": "AES256",
614 "mode": "SSE-B2"
615 },
616 "lifecycleRules": [],
617 "options": [],
618 "revision": 2
619 }
620
621 self._run_command(
622 ['update-bucket', '--default-server-side-encryption=SSE-B2', 'my-bucket', 'allPublic'],
623 expected_json_in_stdout=expected_json,
624 )
625
626 # Update the one with encryption
627 expected_json = {
628 "accountId": self.account_id,
629 "bucketId": "bucket_1",
630 "bucketInfo": {},
631 "bucketName": "your-bucket",
632 "bucketType": "allPrivate",
633 "corsRules": [],
634 "defaultServerSideEncryption": {
635 "algorithm": "AES256",
636 "mode": "SSE-B2"
637 },
638 "lifecycleRules": [],
639 "options": [],
640 "revision": 2
641 }
642
643 self._run_command(
644 ['update-bucket', 'your-bucket', 'allPrivate'], expected_json_in_stdout=expected_json
645 )
646
647 # Make sure they are there
648 expected_stdout = '''
649 bucket_0 allPublic my-bucket
650 bucket_1 allPrivate your-bucket
651 '''
652
653 self._run_command(['list-buckets'], expected_stdout, '', 0)
654
655 def test_keys(self):
656 self._authorize_account()
657
658 self._run_command(['create-bucket', 'my-bucket-a', 'allPublic'], 'bucket_0\n', '', 0)
659 self._run_command(['create-bucket', 'my-bucket-b', 'allPublic'], 'bucket_1\n', '', 0)
660 self._run_command(['create-bucket', 'my-bucket-c', 'allPublic'], 'bucket_2\n', '', 0)
661
662 capabilities = ['readFiles', 'listBuckets']
663 capabilities_with_commas = ','.join(capabilities)
664
665 # Make a key with an illegal name
666 expected_stderr = 'ERROR: Bad request: illegal key name: bad_key_name\n'
667 self._run_command(
668 ['create-key', 'bad_key_name', capabilities_with_commas], '', expected_stderr, 1
669 )
670
671 # Make a key with negative validDurationInSeconds
672 expected_stderr = 'ERROR: Bad request: valid duration must be greater than 0, and less than 1000 days in seconds\n'
673 self._run_command(
674 ['create-key', '--duration', '-456', 'goodKeyName', capabilities_with_commas], '',
675 expected_stderr, 1
676 )
677
678 # Make a key with validDurationInSeconds outside of range
679 expected_stderr = 'ERROR: Bad request: valid duration must be greater than 0, ' \
680 'and less than 1000 days in seconds\n'
681 self._run_command(
682 ['create-key', '--duration', '0', 'goodKeyName', capabilities_with_commas], '',
683 expected_stderr, 1
684 )
685 self._run_command(
686 ['create-key', '--duration', '86400001', 'goodKeyName', capabilities_with_commas], '',
687 expected_stderr, 1
688 )
689
690 # Create three keys
691 self._run_command(
692 ['create-key', 'goodKeyName-One', capabilities_with_commas],
693 'appKeyId0 appKey0\n',
694 '',
695 0,
696 )
697 self._run_command(
698 [
699 'create-key', '--bucket', 'my-bucket-a', 'goodKeyName-Two',
700 capabilities_with_commas + ',readBucketEncryption'
701 ],
702 'appKeyId1 appKey1\n',
703 '',
704 0,
705 )
706 self._run_command(
707 [
708 'create-key', '--bucket', 'my-bucket-b', 'goodKeyName-Three',
709 capabilities_with_commas
710 ],
711 'appKeyId2 appKey2\n',
712 '',
713 0,
714 )
715 self._run_command(
716 ['create-key', '--all-capabilities', 'goodKeyName-Four'],
717 'appKeyId3 appKey3\n',
718 '',
719 0,
720 )
721 self._run_command(
722 ['create-key', '--bucket', 'my-bucket-b', 'goodKeyName-Five', capabilities_with_commas],
723 'appKeyId4 appKey4\n',
724 '',
725 0,
726 )
727
728 # Delete one key
729 self._run_command(['delete-key', 'appKeyId2'], 'appKeyId2\n', '', 0)
730
731 # Delete one bucket, to test listing when a bucket is gone.
732 self._run_command_ignore_output(['delete-bucket', 'my-bucket-b'])
733
734 # List keys
735 expected_list_keys_out = """
736 appKeyId0 goodKeyName-One
737 appKeyId1 goodKeyName-Two
738 appKeyId3 goodKeyName-Four
739 appKeyId4 goodKeyName-Five
740 """
741
742 expected_list_keys_out_long = """
743 appKeyId0 goodKeyName-One - - - '' readFiles,listBuckets
744 appKeyId1 goodKeyName-Two my-bucket-a - - '' readFiles,listBuckets,readBucketEncryption
745 appKeyId3 goodKeyName-Four - - - '' {}
746 appKeyId4 goodKeyName-Five id=bucket_1 - - '' readFiles,listBuckets
747 """.format(','.join(sorted(ALL_CAPABILITIES)))
748
749 self._run_command(['list-keys'], expected_list_keys_out, '', 0)
750 self._run_command(['list-keys', '--long'], expected_list_keys_out_long, '', 0)
751
752 # authorize and make calls using application key with no restrictions
753 self._run_command(['authorize-account', 'appKeyId0', 'appKey0'], None, '', 0)
754 self._run_command(
755 ['list-buckets'],
756 'bucket_0 allPublic my-bucket-a\nbucket_2 allPublic my-bucket-c\n', '', 0
757 )
758
759 expected_json = {
760 "accountId": self.account_id,
761 "bucketId": "bucket_0",
762 "bucketInfo": {},
763 "bucketName": "my-bucket-a",
764 "bucketType": "allPublic",
765 "corsRules": [],
766 "defaultServerSideEncryption": {
767 "mode": None
768 },
769 "lifecycleRules": [],
770 "options": [],
771 "revision": 1
772 }
773 self._run_command(['get-bucket', 'my-bucket-a'], expected_json_in_stdout=expected_json)
774
775 # authorize and make calls using an application key with bucket restrictions
776 self._run_command(['authorize-account', 'appKeyId1', 'appKey1'], None, '', 0)
777
778 self._run_command(
779 ['list-buckets'], '', 'ERROR: Application key is restricted to bucket: my-bucket-a\n', 1
780 )
781 self._run_command(
782 ['get-bucket', 'my-bucket-c'], '',
783 'ERROR: Application key is restricted to bucket: my-bucket-a\n', 1
784 )
785
786 expected_json = {
787 "accountId": self.account_id,
788 "bucketId": "bucket_0",
789 "bucketInfo": {},
790 "bucketName": "my-bucket-a",
791 "bucketType": "allPublic",
792 "corsRules": [],
793 "defaultServerSideEncryption": {
794 "mode": "none"
795 },
796 "lifecycleRules": [],
797 "options": [],
798 "revision": 1
799 }
800
801 self._run_command(['get-bucket', 'my-bucket-a'], expected_json_in_stdout=expected_json)
802 self._run_command(
803 ['ls', '--json', *self.b2_uri_args('my-bucket-c')], '',
804 'ERROR: Application key is restricted to bucket: my-bucket-a\n', 1
805 )
806
807 def test_bucket_info_from_json(self):
808
809 self._authorize_account()
810 self._run_command(['create-bucket', 'my-bucket', 'allPublic'], 'bucket_0\n', '', 0)
811
812 bucket_info = {'color': 'blue'}
813
814 expected_json = {
815 "accountId": self.account_id,
816 "bucketId": "bucket_0",
817 "bucketInfo": {
818 "color": "blue"
819 },
820 "bucketName": "my-bucket",
821 "bucketType": "allPrivate",
822 "corsRules": [],
823 "defaultServerSideEncryption": {
824 "mode": "none"
825 },
826 "lifecycleRules": [],
827 "options": [],
828 "revision": 2
829 }
830 self._run_command(
831 ['update-bucket', '--bucket-info',
832 json.dumps(bucket_info), 'my-bucket', 'allPrivate'],
833 expected_json_in_stdout=expected_json,
834 )
835
836 def test_files(self):
837
838 self._authorize_account()
839 self._run_command(['create-bucket', 'my-bucket', 'allPublic'], 'bucket_0\n', '', 0)
840
841 with TempDir() as temp_dir:
842 local_file1 = self._make_local_file(temp_dir, 'file1.txt')
843 # For this test, use a mod time without millis. My mac truncates
844 # millis and just leaves seconds.
845 mod_time = 1500111222
846 os.utime(local_file1, (mod_time, mod_time))
847 self.assertEqual(1500111222, os.path.getmtime(local_file1))
848
849 # Upload a file
850 expected_stdout = '''
851 URL by file name: http://download.example.com/file/my-bucket/file1.txt
852 URL by fileId: http://download.example.com/b2api/vx/b2_download_file_by_id?fileId=9999'''
853 expected_json = {
854 "action": "upload",
855 "contentSha1": "2aae6c35c94fcfb415dbe95f408b9ce91ee846ed",
856 "contentType": "b2/x-auto",
857 "fileId": "9999",
858 "fileInfo":
859 {
860 "src_last_modified_millis": "1500111222000",
861 "b2-cache-control": "private, max-age=3600"
862 },
863 "fileName": "file1.txt",
864 "serverSideEncryption": {
865 "mode": "none"
866 },
867 "size": 11,
868 "uploadTimestamp": 5000
869 }
870
871 self._run_command(
872 [
873 'upload-file', '--no-progress', 'my-bucket', local_file1, 'file1.txt',
874 '--cache-control=private, max-age=3600'
875 ],
876 expected_json_in_stdout=expected_json,
877 remove_version=True,
878 expected_part_of_stdout=expected_stdout,
879 )
880
881 # Get file info
882 mod_time_str = str(file_mod_time_millis(local_file1))
883 expected_json = {
884 "accountId": self.account_id,
885 "action": "upload",
886 "bucketId": "bucket_0",
887 "size": 11,
888 "contentSha1": "2aae6c35c94fcfb415dbe95f408b9ce91ee846ed",
889 "contentType": "b2/x-auto",
890 "fileId": "9999",
891 "fileInfo":
892 {
893 "src_last_modified_millis": "1500111222000",
894 "b2-cache-control": "private, max-age=3600"
895 },
896 "fileName": "file1.txt",
897 "serverSideEncryption": {
898 "mode": "none"
899 },
900 "uploadTimestamp": 5000
901 }
902
903 self._run_command(
904 ['file-info', 'b2id://9999'],
905 expected_json_in_stdout=expected_json,
906 )
907
908 # Hide the file
909 expected_json = {
910 "action": "hide",
911 "contentSha1": "none",
912 "fileId": "9998",
913 "fileInfo": {},
914 "fileName": "file1.txt",
915 "serverSideEncryption": {
916 "mode": "none"
917 },
918 "size": 0,
919 "uploadTimestamp": 5001
920 }
921
922 self._run_command(
923 ['hide-file', 'my-bucket', 'file1.txt'],
924 expected_json_in_stdout=expected_json,
925 )
926
927 # List the file versions
928 expected_json = [
929 {
930 "action": "hide",
931 "contentSha1": "none",
932 "fileId": "9998",
933 "fileInfo": {},
934 "fileName": "file1.txt",
935 "serverSideEncryption": {
936 "mode": "none"
937 },
938 "size": 0,
939 "uploadTimestamp": 5001
940 }, {
941 "action": "upload",
942 "contentSha1": "2aae6c35c94fcfb415dbe95f408b9ce91ee846ed",
943 "contentType": "b2/x-auto",
944 "fileId": "9999",
945 "fileInfo":
946 {
947 "src_last_modified_millis": str(mod_time_str),
948 "b2-cache-control": "private, max-age=3600"
949 },
950 "fileName": "file1.txt",
951 "serverSideEncryption": {
952 "mode": "none"
953 },
954 "size": 11,
955 "uploadTimestamp": 5000
956 }
957 ]
958
959 self._run_command(
960 ['ls', '--json', '--versions', *self.b2_uri_args('my-bucket')],
961 expected_json_in_stdout=expected_json,
962 )
963
964 # List the file names
965 expected_stdout = '''
966 []
967 '''
968
969 self._run_command(
970 ['ls', '--json', *self.b2_uri_args('my-bucket')], expected_stdout, '', 0
971 )
972
973 # Delete one file version, passing the name in
974 expected_json = {"action": "delete", "fileId": "9998", "fileName": "file1.txt"}
975
976 self._run_command(
977 ['delete-file-version', 'file1.txt', '9998'], expected_json_in_stdout=expected_json
978 )
979
980 # Delete one file version, not passing the name in
981 expected_json = {"action": "delete", "fileId": "9999", "fileName": "file1.txt"}
982
983 self._run_command(
984 ['delete-file-version', '9999'], expected_json_in_stdout=expected_json
985 )
986
987 def test_files_encrypted(self):
988
989 self._authorize_account()
990 self._run_command(['create-bucket', 'my-bucket', 'allPublic'], 'bucket_0\n', '', 0)
991
992 with TempDir() as temp_dir:
993 local_file1 = self._make_local_file(temp_dir, 'file1.txt')
994 # For this test, use a mod time without millis. My mac truncates
995 # millis and just leaves seconds.
996 mod_time = 1500111222
997 os.utime(local_file1, (mod_time, mod_time))
998 self.assertEqual(1500111222, os.path.getmtime(local_file1))
999
1000 # Upload a file
1001 expected_stdout = '''
1002 URL by file name: http://download.example.com/file/my-bucket/file1.txt
1003 URL by fileId: http://download.example.com/b2api/vx/b2_download_file_by_id?fileId=9999'''
1004 expected_json = {
1005 "action": "upload",
1006 "contentSha1": "2aae6c35c94fcfb415dbe95f408b9ce91ee846ed",
1007 "contentType": "b2/x-auto",
1008 "fileId": "9999",
1009 "fileInfo": {
1010 "src_last_modified_millis": "1500111222000"
1011 },
1012 "fileName": "file1.txt",
1013 "serverSideEncryption": {
1014 "algorithm": "AES256",
1015 "mode": "SSE-B2"
1016 },
1017 "size": 11,
1018 "uploadTimestamp": 5000
1019 }
1020
1021 self._run_command(
1022 [
1023 'upload-file', '--no-progress', '--destination-server-side-encryption=SSE-B2',
1024 'my-bucket', local_file1, 'file1.txt'
1025 ],
1026 expected_json_in_stdout=expected_json,
1027 remove_version=True,
1028 expected_part_of_stdout=expected_stdout,
1029 )
1030
1031 # Get file info
1032 mod_time_str = str(file_mod_time_millis(local_file1))
1033 expected_json = {
1034 "accountId": self.account_id,
1035 "action": "upload",
1036 "bucketId": "bucket_0",
1037 "size": 11,
1038 "contentSha1": "2aae6c35c94fcfb415dbe95f408b9ce91ee846ed",
1039 "contentType": "b2/x-auto",
1040 "fileId": "9999",
1041 "fileInfo": {
1042 "src_last_modified_millis": "1500111222000"
1043 },
1044 "fileName": "file1.txt",
1045 "serverSideEncryption": {
1046 "algorithm": "AES256",
1047 "mode": "SSE-B2"
1048 },
1049 "uploadTimestamp": 5000
1050 }
1051
1052 self._run_command(
1053 ['file-info', 'b2id://9999'],
1054 expected_json_in_stdout=expected_json,
1055 )
1056
1057 # Download by name
1058 local_download1 = os.path.join(temp_dir, 'download1.txt')
1059 expected_stdout_template = '''
1060 File name: file1.txt
1061 File id: 9999
1062 Output file path: {output_path}
1063 File size: 11
1064 Content type: b2/x-auto
1065 Content sha1: 2aae6c35c94fcfb415dbe95f408b9ce91ee846ed
1066 Encryption: mode=SSE-B2, algorithm=AES256
1067 Retention: none
1068 Legal hold: <unset>
1069 INFO src_last_modified_millis: 1500111222000
1070 Checksum matches
1071 Download finished
1072 '''
1073 expected_stdout = expected_stdout_template.format(
1074 output_path=pathlib.Path(local_download1).resolve()
1075 )
1076
1077 self._run_command(
1078 ['download-file', '--no-progress', 'b2://my-bucket/file1.txt', local_download1],
1079 expected_stdout, '', 0
1080 )
1081 self.assertEqual(b'hello world', self._read_file(local_download1))
1082 self.assertEqual(mod_time, int(round(os.path.getmtime(local_download1))))
1083
1084 # Download file by ID. (Same expected output as downloading by name)
1085 local_download2 = os.path.join(temp_dir, 'download2.txt')
1086 expected_stdout = expected_stdout_template.format(
1087 output_path=pathlib.Path(local_download2).resolve()
1088 )
1089 self._run_command(
1090 ['download-file', '--no-progress', 'b2id://9999', local_download2], expected_stdout,
1091 '', 0
1092 )
1093 self.assertEqual(b'hello world', self._read_file(local_download2))
1094
1095 # Hide the file
1096 expected_json = {
1097 "action": "hide",
1098 "contentSha1": "none",
1099 "fileId": "9998",
1100 "fileInfo": {},
1101 "fileName": "file1.txt",
1102 "serverSideEncryption": {
1103 "mode": "none"
1104 },
1105 "size": 0,
1106 "uploadTimestamp": 5001
1107 }
1108
1109 self._run_command(
1110 ['hide-file', 'my-bucket', 'file1.txt'],
1111 expected_json_in_stdout=expected_json,
1112 )
1113
1114 # List the file versions
1115 expected_json = [
1116 {
1117 "action": "hide",
1118 "contentSha1": "none",
1119 "fileId": "9998",
1120 "fileInfo": {},
1121 "fileName": "file1.txt",
1122 "serverSideEncryption": {
1123 "mode": "none"
1124 },
1125 "size": 0,
1126 "uploadTimestamp": 5001
1127 }, {
1128 "action": "upload",
1129 "contentSha1": "2aae6c35c94fcfb415dbe95f408b9ce91ee846ed",
1130 "contentType": "b2/x-auto",
1131 "fileId": "9999",
1132 "fileInfo": {
1133 "src_last_modified_millis": str(mod_time_str)
1134 },
1135 "fileName": "file1.txt",
1136 "serverSideEncryption": {
1137 "algorithm": "AES256",
1138 "mode": "SSE-B2"
1139 },
1140 "size": 11,
1141 "uploadTimestamp": 5000
1142 }
1143 ]
1144
1145 self._run_command(
1146 ['ls', '--json', '--versions', *self.b2_uri_args('my-bucket')],
1147 expected_json_in_stdout=expected_json,
1148 )
1149
1150 # List the file names
1151 expected_stdout = '''
1152 []
1153 '''
1154
1155 self._run_command(
1156 ['ls', '--json', *self.b2_uri_args('my-bucket')], expected_stdout, '', 0
1157 )
1158
1159 # Delete one file version, passing the name in
1160 expected_json = {"action": "delete", "fileId": "9998", "fileName": "file1.txt"}
1161
1162 self._run_command(
1163 ['delete-file-version', 'file1.txt', '9998'],
1164 expected_json_in_stdout=expected_json,
1165 )
1166
1167 # Delete one file version, not passing the name in
1168 expected_json = {"action": "delete", "fileId": "9999", "fileName": "file1.txt"}
1169
1170 self._run_command(
1171 ['delete-file-version', '9999'],
1172 expected_json_in_stdout=expected_json,
1173 )
1174
1175 def _test_download_to_directory(self, download_by: str):
1176 self._authorize_account()
1177 self._create_my_bucket()
1178
1179 base_filename = 'file'
1180 extension = '.txt'
1181 source_filename = f'{base_filename}{extension}'
1182
1183 with TempDir() as temp_dir:
1184 local_file = self._make_local_file(temp_dir, source_filename)
1185 local_file_content = self._read_file(local_file)
1186
1187 self._run_command(
1188 ['upload-file', '--no-progress', 'my-bucket', local_file, source_filename],
1189 remove_version=True,
1190 )
1191
1192 b2uri = f'b2://my-bucket/{source_filename}' if download_by == 'name' else 'b2id://9999'
1193 command = [
1194 'download-file',
1195 '--no-progress',
1196 b2uri,
1197 ]
1198
1199 target_directory = os.path.join(temp_dir, 'target')
1200 os.mkdir(target_directory)
1201 command += [target_directory]
1202 self._run_command(command)
1203 self.assertEqual(
1204 local_file_content,
1205 self._read_file(os.path.join(target_directory, source_filename))
1206 )
1207
1208 # Download the file second time, to check the override behavior.
1209 self._run_command(command)
1210 # We should get another file.
1211 target_directory_files = [
1212 elem
1213 for elem in pathlib.Path(target_directory).glob(f'{base_filename}*{extension}')
1214 if elem.name != source_filename
1215 ]
1216 assert len(target_directory_files) == 1, f'{target_directory_files}'
1217 self.assertEqual(local_file_content, self._read_file(target_directory_files[0]))
1218
1219 def test_download_by_id_to_directory(self):
1220 self._test_download_to_directory(download_by='id')
1221
1222 def test_download_by_name_to_directory(self):
1223 self._test_download_to_directory(download_by='name')
1224
1225 def test_copy_file_by_id(self):
1226 self._authorize_account()
1227 self._create_my_bucket()
1228
1229 with TempDir() as temp_dir:
1230 local_file1 = self._make_local_file(temp_dir, 'file1.txt')
1231 # For this test, use a mod time without millis. My mac truncates
1232 # millis and just leaves seconds.
1233 mod_time = 1500111222
1234 os.utime(local_file1, (mod_time, mod_time))
1235 self.assertEqual(1500111222, os.path.getmtime(local_file1))
1236
1237 # Upload a file
1238 expected_stdout = '''
1239 URL by file name: http://download.example.com/file/my-bucket/file1.txt
1240 URL by fileId: http://download.example.com/b2api/vx/b2_download_file_by_id?fileId=9999'''
1241 expected_json = {
1242 "action": "upload",
1243 "contentSha1": "2aae6c35c94fcfb415dbe95f408b9ce91ee846ed",
1244 "contentType": "b2/x-auto",
1245 "fileId": "9999",
1246 "fileInfo": {
1247 "src_last_modified_millis": "1500111222000"
1248 },
1249 "fileName": "file1.txt",
1250 "serverSideEncryption": {
1251 "mode": "none"
1252 },
1253 "size": 11,
1254 "uploadTimestamp": 5000
1255 }
1256
1257 self._run_command(
1258 ['upload-file', '--no-progress', 'my-bucket', local_file1, 'file1.txt'],
1259 expected_json_in_stdout=expected_json,
1260 remove_version=True,
1261 expected_part_of_stdout=expected_stdout,
1262 )
1263
1264 # Copy File
1265 expected_json = {
1266 "accountId": self.account_id,
1267 "action": "copy",
1268 "bucketId": "bucket_0",
1269 "size": 11,
1270 "contentSha1": "2aae6c35c94fcfb415dbe95f408b9ce91ee846ed",
1271 "contentType": "b2/x-auto",
1272 "fileId": "9998",
1273 "fileInfo": {
1274 "src_last_modified_millis": "1500111222000"
1275 },
1276 "fileName": "file1_copy.txt",
1277 "serverSideEncryption": {
1278 "mode": "none"
1279 },
1280 "uploadTimestamp": 5001
1281 }
1282 self._run_command(
1283 ['copy-file-by-id', '9999', 'my-bucket', 'file1_copy.txt'],
1284 expected_json_in_stdout=expected_json,
1285 )
1286
1287 # Copy File with range parameter
1288 expected_json = {
1289 "accountId": self.account_id,
1290 "action": "copy",
1291 "bucketId": "bucket_0",
1292 "size": 5,
1293 "contentSha1": "4f664540ff30b8d34e037298a84e4736be39d731",
1294 "contentType": "b2/x-auto",
1295 "fileId": "9997",
1296 "fileInfo": {
1297 "src_last_modified_millis": "1500111222000"
1298 },
1299 "fileName": "file1_copy.txt",
1300 "serverSideEncryption": {
1301 "mode": "none"
1302 },
1303 "uploadTimestamp": 5002
1304 }
1305 self._run_command(
1306 ['copy-file-by-id', '--range', '3,7', '9999', 'my-bucket', 'file1_copy.txt'],
1307 expected_json_in_stdout=expected_json,
1308 )
1309
1310 local_download1 = os.path.join(temp_dir, 'file1_copy.txt')
1311 self._run_command(
1312 ['download-file', '-q', 'b2://my-bucket/file1_copy.txt', local_download1]
1313 )
1314 self.assertEqual(b'lo wo', self._read_file(local_download1))
1315
1316 # Invalid metadata copy with file info
1317 expected_stderr = "ERROR: File info can be set only when content type is set\n"
1318 self._run_command(
1319 [
1320 'copy-file-by-id',
1321 '--info',
1322 'a=b',
1323 '9999',
1324 'my-bucket',
1325 'file1_copy.txt',
1326 ],
1327 '',
1328 expected_stderr,
1329 1,
1330 )
1331
1332 # Invalid metadata replace without file info
1333 expected_stderr = "ERROR: File info can be not set only when content type is not set\n"
1334 self._run_command(
1335 [
1336 'copy-file-by-id',
1337 '--content-type',
1338 'text/plain',
1339 '9999',
1340 'my-bucket',
1341 'file1_copy.txt',
1342 ],
1343 '',
1344 expected_stderr,
1345 1,
1346 )
1347
1348 # replace with content type and file info
1349 expected_json = {
1350 "accountId": self.account_id,
1351 "action": "copy",
1352 "bucketId": "bucket_0",
1353 "size": 11,
1354 "contentSha1": "2aae6c35c94fcfb415dbe95f408b9ce91ee846ed",
1355 "contentType": "text/plain",
1356 "fileId": "9996",
1357 "fileInfo": {
1358 "a": "b"
1359 },
1360 "fileName": "file1_copy.txt",
1361 "serverSideEncryption": {
1362 "mode": "none"
1363 },
1364 "uploadTimestamp": 5003
1365 }
1366 self._run_command(
1367 [
1368 'copy-file-by-id',
1369 '--content-type',
1370 'text/plain',
1371 '--info',
1372 'a=b',
1373 '9999',
1374 'my-bucket',
1375 'file1_copy.txt',
1376 ],
1377 expected_json_in_stdout=expected_json,
1378 )
1379
1380 # UnsatisfiableRange
1381 expected_stderr = "ERROR: The range in the request is outside the size of the file\n"
1382 self._run_command(
1383 ['copy-file-by-id', '--range', '12,20', '9999', 'my-bucket', 'file1_copy.txt'],
1384 '',
1385 expected_stderr,
1386 1,
1387 )
1388
1389 # Copy in different bucket
1390 self._run_command(['create-bucket', 'my-bucket1', 'allPublic'], 'bucket_1\n', '', 0)
1391 expected_json = {
1392 "accountId": self.account_id,
1393 "action": "copy",
1394 "bucketId": "bucket_1",
1395 "size": 11,
1396 "contentSha1": "2aae6c35c94fcfb415dbe95f408b9ce91ee846ed",
1397 "contentType": "b2/x-auto",
1398 "fileId": "9994",
1399 "fileInfo": {
1400 "src_last_modified_millis": "1500111222000"
1401 },
1402 "fileName": "file1_copy.txt",
1403 "serverSideEncryption": {
1404 "mode": "none"
1405 },
1406 "uploadTimestamp": 5004
1407 }
1408 self._run_command(
1409 ['copy-file-by-id', '9999', 'my-bucket1', 'file1_copy.txt'],
1410 expected_json_in_stdout=expected_json,
1411 )
1412
1413 def test_get_download_auth_defaults(self):
1414 self._authorize_account()
1415 self._create_my_bucket()
1416 self._run_command(
1417 ['get-download-auth', 'my-bucket'], 'fake_download_auth_token_bucket_0__86400\n', '', 0
1418 )
1419
1420 def test_get_download_auth_explicit(self):
1421 self._authorize_account()
1422 self._create_my_bucket()
1423 self._run_command(
1424 ['get-download-auth', '--prefix', 'prefix', '--duration', '12345', 'my-bucket'],
1425 'fake_download_auth_token_bucket_0_prefix_12345\n', '', 0
1426 )
1427
1428 def test_get_download_auth_url(self):
1429 self._authorize_account()
1430 self._create_my_bucket()
1431 self._run_command(
1432 ['get-download-url-with-auth', '--duration', '12345', 'my-bucket', 'my-file'],
1433 'http://download.example.com/file/my-bucket/my-file?Authorization=fake_download_auth_token_bucket_0_my-file_12345\n',
1434 '', 0
1435 )
1436
1437 def test_get_download_auth_url_with_encoding(self):
1438 self._authorize_account()
1439 self._create_my_bucket()
1440 self._run_command(
1441 ['get-download-url-with-auth', '--duration', '12345', 'my-bucket', '\u81ea'],
1442 'http://download.example.com/file/my-bucket/%E8%87%AA?Authorization=fake_download_auth_token_bucket_0_%E8%87%AA_12345\n',
1443 '', 0
1444 )
1445
1446 def test_list_unfinished_large_files_with_none(self):
1447 self._authorize_account()
1448 self._create_my_bucket()
1449 self._run_command(['list-unfinished-large-files', 'my-bucket'], '', '', 0)
1450
1451 def test_upload_large_file(self):
1452 self._authorize_account()
1453 self._create_my_bucket()
1454 min_part_size = self.account_info.get_recommended_part_size()
1455 file_size = min_part_size * 3
1456
1457 with TempDir() as temp_dir:
1458 file_path = os.path.join(temp_dir, 'test.txt')
1459 text = '*' * file_size
1460 with open(file_path, 'wb') as f:
1461 f.write(text.encode('utf-8'))
1462 mod_time_str = str(file_mod_time_millis(file_path))
1463 expected_stdout = '''
1464 URL by file name: http://download.example.com/file/my-bucket/test.txt
1465 URL by fileId: http://download.example.com/b2api/vx/b2_download_file_by_id?fileId=9999'''
1466 expected_json = {
1467 "action": "upload",
1468 "contentSha1": "none",
1469 "contentType": "b2/x-auto",
1470 "fileId": "9999",
1471 "fileInfo":
1472 {
1473 "large_file_sha1": "cc8954ec25e0c564b6a693fb22200e4f832c18e8",
1474 "src_last_modified_millis": str(mod_time_str)
1475 },
1476 "fileName": "test.txt",
1477 "serverSideEncryption": {
1478 "mode": "none"
1479 },
1480 "size": 600,
1481 "uploadTimestamp": 5000
1482 }
1483
1484 self._run_command(
1485 [
1486 'upload-file', '--no-progress', '--threads', '5', 'my-bucket', file_path,
1487 'test.txt'
1488 ],
1489 expected_json_in_stdout=expected_json,
1490 remove_version=True,
1491 expected_part_of_stdout=expected_stdout,
1492 )
1493
1494 def test_upload_large_file_encrypted(self):
1495 self._authorize_account()
1496 self._run_command(['create-bucket', 'my-bucket', 'allPublic'], 'bucket_0\n', '', 0)
1497 min_part_size = self.account_info.get_recommended_part_size()
1498 file_size = min_part_size * 3
1499
1500 with TempDir() as temp_dir:
1501 file_path = os.path.join(temp_dir, 'test.txt')
1502 text = '*' * file_size
1503 with open(file_path, 'wb') as f:
1504 f.write(text.encode('utf-8'))
1505 mod_time_str = str(file_mod_time_millis(file_path))
1506 expected_stdout = '''
1507 URL by file name: http://download.example.com/file/my-bucket/test.txt
1508 URL by fileId: http://download.example.com/b2api/vx/b2_download_file_by_id?fileId=9999'''
1509 expected_json = {
1510 "action": "upload",
1511 "contentSha1": "none",
1512 "contentType": "b2/x-auto",
1513 "fileId": "9999",
1514 "fileInfo":
1515 {
1516 "large_file_sha1": "cc8954ec25e0c564b6a693fb22200e4f832c18e8",
1517 "src_last_modified_millis": str(mod_time_str)
1518 },
1519 "fileName": "test.txt",
1520 "serverSideEncryption": {
1521 "algorithm": "AES256",
1522 "mode": "SSE-B2"
1523 },
1524 "size": 600,
1525 "uploadTimestamp": 5000
1526 }
1527
1528 self._run_command(
1529 [
1530 'upload-file', '--no-progress', '--destination-server-side-encryption=SSE-B2',
1531 '--threads', '5', 'my-bucket', file_path, 'test.txt'
1532 ],
1533 expected_json_in_stdout=expected_json,
1534 remove_version=True,
1535 expected_part_of_stdout=expected_stdout,
1536 )
1537
1538 def test_upload_incremental(self):
1539 self._authorize_account()
1540 self._run_command(['create-bucket', 'my-bucket', 'allPublic'], 'bucket_0\n', '', 0)
1541 min_part_size = self.account_info.get_recommended_part_size()
1542 file_size = min_part_size * 2
1543
1544 with TempDir() as temp_dir:
1545 file_path = pathlib.Path(temp_dir) / 'test.txt'
1546
1547 incremental_upload_params = [
1548 'upload-file',
1549 '--no-progress',
1550 '--threads',
1551 '5',
1552 '--incremental-mode',
1553 'my-bucket',
1554 str(file_path),
1555 'test.txt',
1556 ]
1557
1558 file_path.write_bytes(b'*' * file_size)
1559 self._run_command(incremental_upload_params)
1560
1561 with open(file_path, 'ab') as f:
1562 f.write(b'*' * min_part_size)
1563 self._run_command(incremental_upload_params)
1564
1565 downloaded_path = pathlib.Path(temp_dir) / 'out.txt'
1566 self._run_command(
1567 [
1568 'download-file',
1569 '-q',
1570 'b2://my-bucket/test.txt',
1571 str(downloaded_path),
1572 ]
1573 )
1574 assert downloaded_path.read_bytes() == file_path.read_bytes()
1575
1576 def test_get_account_info(self):
1577 self._authorize_account()
1578 expected_json = {
1579 "accountAuthToken": "auth_token_0",
1580 "accountFilePath": getattr(self.account_info, 'filename',
1581 None), # missing in StubAccountInfo in tests
1582 "accountId": self.account_id,
1583 "allowed":
1584 {
1585 "bucketId": None,
1586 "bucketName": None,
1587 "capabilities": sorted(ALL_CAPABILITIES),
1588 "namePrefix": None
1589 },
1590 "apiUrl": "http://api.example.com",
1591 "applicationKey": self.master_key,
1592 "downloadUrl": "http://download.example.com",
1593 "s3endpoint": "http://s3.api.example.com",
1594 }
1595 self._run_command(
1596 ['get-account-info'],
1597 expected_json_in_stdout=expected_json,
1598 )
1599
1600 def test_get_bucket(self):
1601 self._authorize_account()
1602 self._create_my_bucket()
1603 expected_json = {
1604 "accountId": self.account_id,
1605 "bucketId": "bucket_0",
1606 "bucketInfo": {},
1607 "bucketName": "my-bucket",
1608 "bucketType": "allPublic",
1609 "corsRules": [],
1610 "defaultServerSideEncryption": {
1611 "mode": "none"
1612 },
1613 "lifecycleRules": [],
1614 "options": [],
1615 "revision": 1
1616 }
1617 self._run_command(
1618 ['get-bucket', 'my-bucket'],
1619 expected_json_in_stdout=expected_json,
1620 )
1621
1622 def test_get_bucket_empty_show_size(self):
1623 self._authorize_account()
1624 self._create_my_bucket()
1625 expected_json = {
1626 "accountId": self.account_id,
1627 "bucketId": "bucket_0",
1628 "bucketInfo": {},
1629 "bucketName": "my-bucket",
1630 "bucketType": "allPublic",
1631 "corsRules": [],
1632 "defaultServerSideEncryption": {
1633 "mode": "none"
1634 },
1635 "fileCount": 0,
1636 "lifecycleRules": [],
1637 "options": [],
1638 "revision": 1,
1639 "totalSize": 0
1640 }
1641 self._run_command(
1642 ['get-bucket', '--show-size', 'my-bucket'],
1643 expected_json_in_stdout=expected_json,
1644 )
1645
1646 def test_get_bucket_one_item_show_size(self):
1647 self._authorize_account()
1648 self._create_my_bucket()
1649 with TempDir() as temp_dir:
1650 # Upload a standard test file.
1651 local_file1 = self._make_local_file(temp_dir, 'file1.txt')
1652 mod_time_str = str(file_mod_time_millis(local_file1))
1653 expected_stdout = '''
1654 URL by file name: http://download.example.com/file/my-bucket/file1.txt
1655 URL by fileId: http://download.example.com/b2api/vx/b2_download_file_by_id?fileId=9999'''
1656 expected_json = {
1657 "action": "upload",
1658 "contentSha1": "2aae6c35c94fcfb415dbe95f408b9ce91ee846ed",
1659 "contentType": "b2/x-auto",
1660 "fileId": "9999",
1661 "fileInfo": {
1662 "src_last_modified_millis": str(mod_time_str)
1663 },
1664 "fileName": "file1.txt",
1665 "serverSideEncryption": {
1666 "mode": "none"
1667 },
1668 "size": 11,
1669 "uploadTimestamp": 5000
1670 }
1671 self._run_command(
1672 ['upload-file', '--no-progress', 'my-bucket', local_file1, 'file1.txt'],
1673 expected_json_in_stdout=expected_json,
1674 remove_version=True,
1675 expected_part_of_stdout=expected_stdout,
1676 )
1677
1678 # Now check the output of get-bucket against the canon.
1679 expected_json = {
1680 "accountId": self.account_id,
1681 "bucketId": "bucket_0",
1682 "bucketInfo": {},
1683 "bucketName": "my-bucket",
1684 "bucketType": "allPublic",
1685 "corsRules": [],
1686 "defaultServerSideEncryption": {
1687 "mode": "none"
1688 },
1689 "fileCount": 1,
1690 "lifecycleRules": [],
1691 "options": [],
1692 "revision": 1,
1693 "totalSize": 11
1694 }
1695 self._run_command(
1696 ['get-bucket', '--show-size', 'my-bucket'],
1697 expected_json_in_stdout=expected_json,
1698 )
1699
1700 def test_get_bucket_with_versions(self):
1701 self._authorize_account()
1702 self._create_my_bucket()
1703
1704 # Put many versions of a file into the test bucket. Unroll the loop here for convenience.
1705 bucket = self.b2_api.get_bucket_by_name('my-bucket')
1706 bucket.upload(UploadSourceBytes(b'test'), 'test')
1707 bucket.upload(UploadSourceBytes(b'test'), 'test')
1708 bucket.upload(UploadSourceBytes(b'test'), 'test')
1709 bucket.upload(UploadSourceBytes(b'test'), 'test')
1710 bucket.upload(UploadSourceBytes(b'test'), 'test')
1711 bucket.upload(UploadSourceBytes(b'test'), 'test')
1712 bucket.upload(UploadSourceBytes(b'test'), 'test')
1713 bucket.upload(UploadSourceBytes(b'test'), 'test')
1714 bucket.upload(UploadSourceBytes(b'test'), 'test')
1715 bucket.upload(UploadSourceBytes(b'test'), 'test')
1716
1717 # Now check the output of get-bucket against the canon.
1718 expected_json = {
1719 "accountId": self.account_id,
1720 "bucketId": "bucket_0",
1721 "bucketInfo": {},
1722 "bucketName": "my-bucket",
1723 "bucketType": "allPublic",
1724 "corsRules": [],
1725 "defaultServerSideEncryption": {
1726 "mode": "none"
1727 },
1728 "fileCount": 10,
1729 "lifecycleRules": [],
1730 "options": [],
1731 "revision": 1,
1732 "totalSize": 40
1733 }
1734 self._run_command(
1735 ['get-bucket', '--show-size', 'my-bucket'],
1736 expected_json_in_stdout=expected_json,
1737 )
1738
1739 def test_get_bucket_with_folders(self):
1740 self._authorize_account()
1741 self._create_my_bucket()
1742
1743 # Create a hierarchical structure within the test bucket. Unroll the loop here for
1744 # convenience.
1745 bucket = self.b2_api.get_bucket_by_name('my-bucket')
1746 bucket.upload(UploadSourceBytes(b'test'), 'test')
1747 bucket.upload(UploadSourceBytes(b'test'), '1/test')
1748 bucket.upload(UploadSourceBytes(b'test'), '1/2/test')
1749 bucket.upload(UploadSourceBytes(b'test'), '1/2/3/test')
1750 bucket.upload(UploadSourceBytes(b'test'), '1/2/3/4/test')
1751 bucket.upload(UploadSourceBytes(b'test'), '1/2/3/4/5/test')
1752 bucket.upload(UploadSourceBytes(b'test'), '1/2/3/4/5/6/test')
1753 bucket.upload(UploadSourceBytes(b'test'), '1/2/3/4/5/6/7/test')
1754 bucket.upload(UploadSourceBytes(b'test'), '1/2/3/4/5/6/7/8/test')
1755 bucket.upload(UploadSourceBytes(b'test'), '1/2/3/4/5/6/7/8/9/test')
1756 bucket.upload(UploadSourceBytes(b'check'), 'check')
1757 bucket.upload(UploadSourceBytes(b'check'), '1/check')
1758 bucket.upload(UploadSourceBytes(b'check'), '1/2/check')
1759 bucket.upload(UploadSourceBytes(b'check'), '1/2/3/check')
1760 bucket.upload(UploadSourceBytes(b'check'), '1/2/3/4/check')
1761 bucket.upload(UploadSourceBytes(b'check'), '1/2/3/4/5/check')
1762 bucket.upload(UploadSourceBytes(b'check'), '1/2/3/4/5/6/check')
1763 bucket.upload(UploadSourceBytes(b'check'), '1/2/3/4/5/6/7/check')
1764 bucket.upload(UploadSourceBytes(b'check'), '1/2/3/4/5/6/7/8/check')
1765 bucket.upload(UploadSourceBytes(b'check'), '1/2/3/4/5/6/7/8/9/check')
1766
1767 # Now check the output of get-bucket against the canon.
1768 expected_json = {
1769 "accountId": self.account_id,
1770 "bucketId": "bucket_0",
1771 "bucketInfo": {},
1772 "bucketName": "my-bucket",
1773 "bucketType": "allPublic",
1774 "corsRules": [],
1775 "defaultServerSideEncryption": {
1776 "mode": "none"
1777 },
1778 "fileCount": 20,
1779 "lifecycleRules": [],
1780 "options": [],
1781 "revision": 1,
1782 "totalSize": 90
1783 }
1784 self._run_command(
1785 ['get-bucket', '--show-size', 'my-bucket'],
1786 expected_json_in_stdout=expected_json,
1787 )
1788
1789 def test_get_bucket_with_hidden(self):
1790 self._authorize_account()
1791 self._create_my_bucket()
1792
1793 # Put some files into the test bucket. Unroll the loop for convenience.
1794 bucket = self.b2_api.get_bucket_by_name('my-bucket')
1795 bucket.upload(UploadSourceBytes(b'test'), 'upload1')
1796 bucket.upload(UploadSourceBytes(b'test'), 'upload2')
1797 bucket.upload(UploadSourceBytes(b'test'), 'upload3')
1798 bucket.upload(UploadSourceBytes(b'test'), 'upload4')
1799 bucket.upload(UploadSourceBytes(b'test'), 'upload5')
1800 bucket.upload(UploadSourceBytes(b'test'), 'upload6')
1801
1802 # Hide some new files. Don't check the results here; it will be clear enough that
1803 # something has failed if the output of 'get-bucket' does not match the canon.
1804 stdout, stderr = self._get_stdouterr()
1805 console_tool = self.console_tool_class(stdout, stderr)
1806 console_tool.run_command(['b2', 'hide-file', 'my-bucket', 'hidden1'])
1807 console_tool.run_command(['b2', 'hide-file', 'my-bucket', 'hidden2'])
1808 console_tool.run_command(['b2', 'hide-file', 'my-bucket', 'hidden3'])
1809 console_tool.run_command(['b2', 'hide-file', 'my-bucket', 'hidden4'])
1810
1811 # Now check the output of get-bucket against the canon.
1812 expected_json = {
1813 "accountId": self.account_id,
1814 "bucketId": "bucket_0",
1815 "bucketInfo": {},
1816 "bucketName": "my-bucket",
1817 "bucketType": "allPublic",
1818 "corsRules": [],
1819 "defaultServerSideEncryption": {
1820 "mode": "none"
1821 },
1822 "fileCount": 10,
1823 "lifecycleRules": [],
1824 "options": [],
1825 "revision": 1,
1826 "totalSize": 24
1827 }
1828 self._run_command(
1829 ['get-bucket', '--show-size', 'my-bucket'],
1830 expected_json_in_stdout=expected_json,
1831 )
1832
1833 def test_get_bucket_complex(self):
1834 self._authorize_account()
1835 self._create_my_bucket()
1836
1837 # Create a hierarchical structure within the test bucket. Unroll the loop here for
1838 # convenience.
1839 bucket = self.b2_api.get_bucket_by_name('my-bucket')
1840 bucket.upload(UploadSourceBytes(b'test'), 'test')
1841 bucket.upload(UploadSourceBytes(b'test'), 'test')
1842 bucket.upload(UploadSourceBytes(b'test'), '1/test')
1843 bucket.upload(UploadSourceBytes(b'test'), '1/test')
1844 bucket.upload(UploadSourceBytes(b'test'), '1/2/test')
1845 bucket.upload(UploadSourceBytes(b'test'), '1/2/test')
1846 bucket.upload(UploadSourceBytes(b'test'), '1/2/3/test')
1847 bucket.upload(UploadSourceBytes(b'test'), '1/2/3/test')
1848 bucket.upload(UploadSourceBytes(b'test'), '1/2/3/test')
1849 bucket.upload(UploadSourceBytes(b'test'), '1/2/3/test')
1850 bucket.upload(UploadSourceBytes(b'test'), '1/2/3/test')
1851 bucket.upload(UploadSourceBytes(b'check'), 'check')
1852 bucket.upload(UploadSourceBytes(b'check'), 'check')
1853 bucket.upload(UploadSourceBytes(b'check'), '1/check')
1854 bucket.upload(UploadSourceBytes(b'check'), '1/check')
1855 bucket.upload(UploadSourceBytes(b'check'), '1/2/check')
1856 bucket.upload(UploadSourceBytes(b'check'), '1/2/check')
1857 bucket.upload(UploadSourceBytes(b'check'), '1/2/check')
1858 bucket.upload(UploadSourceBytes(b'check'), '1/2/3/check')
1859 bucket.upload(UploadSourceBytes(b'check'), '1/2/3/4/check')
1860 bucket.upload(UploadSourceBytes(b'check'), '1/2/3/4/check')
1861 bucket.upload(UploadSourceBytes(b'check'), '1/2/3/4/check')
1862
1863 # Hide some new files. Don't check the results here; it will be clear enough that
1864 # something has failed if the output of 'get-bucket' does not match the canon.
1865 stdout, stderr = self._get_stdouterr()
1866 console_tool = self.console_tool_class(stdout, stderr)
1867 console_tool.run_command(['b2', 'hide-file', 'my-bucket', '1/hidden1'])
1868 console_tool.run_command(['b2', 'hide-file', 'my-bucket', '1/hidden1'])
1869 console_tool.run_command(['b2', 'hide-file', 'my-bucket', '1/hidden2'])
1870 console_tool.run_command(['b2', 'hide-file', 'my-bucket', '1/2/hidden3'])
1871 console_tool.run_command(['b2', 'hide-file', 'my-bucket', '1/2/hidden3'])
1872 console_tool.run_command(['b2', 'hide-file', 'my-bucket', '1/2/hidden3'])
1873 console_tool.run_command(['b2', 'hide-file', 'my-bucket', '1/2/hidden3'])
1874
1875 # Now check the output of get-bucket against the canon.
1876 expected_json = {
1877 "accountId": self.account_id,
1878 "bucketId": "bucket_0",
1879 "bucketInfo": {},
1880 "bucketName": "my-bucket",
1881 "bucketType": "allPublic",
1882 "corsRules": [],
1883 "defaultServerSideEncryption": {
1884 "mode": "none"
1885 },
1886 "fileCount": 29,
1887 "lifecycleRules": [],
1888 "options": [],
1889 "revision": 1,
1890 "totalSize": 99
1891 }
1892 self._run_command(
1893 ['get-bucket', '--show-size', 'my-bucket'],
1894 expected_json_in_stdout=expected_json,
1895 )
1896
1897 def test_get_bucket_encrypted(self):
1898 self._authorize_account()
1899 self._run_command(
1900 [
1901 'create-bucket', '--default-server-side-encryption=SSE-B2',
1902 '--default-server-side-encryption-algorithm=AES256', 'my-bucket', 'allPublic'
1903 ], 'bucket_0\n', '', 0
1904 )
1905 expected_json = {
1906 "accountId": self.account_id,
1907 "bucketId": "bucket_0",
1908 "bucketInfo": {},
1909 "bucketName": "my-bucket",
1910 "bucketType": "allPublic",
1911 "corsRules": [],
1912 "defaultServerSideEncryption": {
1913 "algorithm": "AES256",
1914 "mode": "SSE-B2"
1915 },
1916 "fileCount": 0,
1917 "lifecycleRules": [],
1918 "options": [],
1919 "revision": 1,
1920 "totalSize": 0
1921 }
1922 self._run_command(
1923 ['get-bucket', '--show-size', 'my-bucket'],
1924 expected_json_in_stdout=expected_json,
1925 )
1926
1927 def test_sync(self):
1928 self._authorize_account()
1929 self._create_my_bucket()
1930
1931 with TempDir() as temp_dir:
1932 file_path = os.path.join(temp_dir, 'test.txt')
1933 with open(file_path, 'wb') as f:
1934 f.write(b'hello world')
1935 expected_stdout = '''
1936 upload test.txt
1937 '''
1938
1939 command = ['sync', '--no-progress', temp_dir, 'b2://my-bucket']
1940 self._run_command(command, expected_stdout, '', 0)
1941
1942 def test_sync_empty_folder_when_not_enabled(self):
1943 self._authorize_account()
1944 self._create_my_bucket()
1945 with TempDir() as temp_dir:
1946 command = ['sync', '--no-progress', temp_dir, 'b2://my-bucket']
1947 expected_stderr = 'ERROR: Directory %s is empty. Use --allow-empty-source to sync anyway.\n' % fix_windows_path_limit(
1948 temp_dir.replace('\\\\', '\\')
1949 )
1950 self._run_command(command, '', expected_stderr, 1)
1951
1952 def test_sync_empty_folder_when_enabled(self):
1953 self._authorize_account()
1954 self._create_my_bucket()
1955 with TempDir() as temp_dir:
1956 command = ['sync', '--no-progress', '--allow-empty-source', temp_dir, 'b2://my-bucket']
1957 self._run_command(command, '', '', 0)
1958
1959 def test_sync_dry_run(self):
1960 self._authorize_account()
1961 self._create_my_bucket()
1962
1963 with TempDir() as temp_dir:
1964 temp_file = self._make_local_file(temp_dir, 'test-dry-run.txt')
1965
1966 # dry-run
1967 expected_stdout = '''
1968 upload test-dry-run.txt
1969 '''
1970 command = ['sync', '--no-progress', '--dry-run', temp_dir, 'b2://my-bucket']
1971 self._run_command(command, expected_stdout, '', 0)
1972
1973 # file should not have been uploaded
1974 expected_stdout = '''
1975 []
1976 '''
1977 self._run_command(
1978 ['ls', '--json', *self.b2_uri_args('my-bucket')], expected_stdout, '', 0
1979 )
1980
1981 # upload file
1982 expected_stdout = '''
1983 upload test-dry-run.txt
1984 '''
1985 command = ['sync', '--no-progress', temp_dir, 'b2://my-bucket']
1986 self._run_command(command, expected_stdout, '', 0)
1987
1988 # file should have been uploaded
1989 mtime = file_mod_time_millis(temp_file)
1990 expected_json = [
1991 {
1992 "action": "upload",
1993 "contentSha1": "2aae6c35c94fcfb415dbe95f408b9ce91ee846ed",
1994 "contentType": "b2/x-auto",
1995 "fileId": "9999",
1996 "fileInfo": {
1997 "src_last_modified_millis": str(mtime)
1998 },
1999 "fileName": "test-dry-run.txt",
2000 "serverSideEncryption": {
2001 "mode": "none"
2002 },
2003 "size": 11,
2004 "uploadTimestamp": 5000
2005 }
2006 ]
2007 self._run_command(
2008 ['ls', '--json', *self.b2_uri_args('my-bucket')],
2009 expected_json_in_stdout=expected_json,
2010 )
2011
2012 def test_sync_exclude_all_symlinks(self):
2013 self._authorize_account()
2014 self._create_my_bucket()
2015
2016 with TempDir() as temp_dir:
2017 self._make_local_file(temp_dir, 'test.txt')
2018 os.symlink('test.txt', os.path.join(temp_dir, 'alink'))
2019 expected_stdout = '''
2020 upload test.txt
2021 '''
2022
2023 command = [
2024 'sync', '--no-progress', '--exclude-all-symlinks', temp_dir, 'b2://my-bucket'
2025 ]
2026 self._run_command(command, expected_stdout, '', 0)
2027
2028 def test_sync_dont_exclude_all_symlinks(self):
2029 self._authorize_account()
2030 self._create_my_bucket()
2031
2032 with TempDir() as temp_dir:
2033 self._make_local_file(temp_dir, 'test.txt')
2034 os.symlink('test.txt', os.path.join(temp_dir, 'alink'))
2035 # Exact stdout cannot be asserted because line order is non-deterministic
2036 expected_part_of_stdout = '''
2037 upload alink
2038 '''
2039
2040 command = ['sync', '--no-progress', temp_dir, 'b2://my-bucket']
2041 self._run_command(command, expected_part_of_stdout=expected_part_of_stdout)
2042
2043 def test_sync_exclude_if_modified_after_in_range(self):
2044 self._authorize_account()
2045 self._create_my_bucket()
2046
2047 with TempDir() as temp_dir:
2048 for file, mtime in (('test.txt', 1367900664.152), ('test2.txt', 1367600664.152)):
2049 self._make_local_file(temp_dir, file)
2050 path = os.path.join(temp_dir, file)
2051 os.utime(path, (mtime, mtime))
2052
2053 expected_stdout = '''
2054 upload test2.txt
2055 '''
2056
2057 command = [
2058 'sync', '--no-progress', '--exclude-if-modified-after', '1367700664.152', temp_dir,
2059 'b2://my-bucket'
2060 ]
2061 self._run_command(command, expected_stdout, '', 0)
2062
2063 def test_sync_exclude_if_modified_after_exact(self):
2064 self._authorize_account()
2065 self._create_my_bucket()
2066
2067 with TempDir() as temp_dir:
2068 for file, mtime in (('test.txt', 1367900664.152), ('test2.txt', 1367600664.152)):
2069 self._make_local_file(temp_dir, file)
2070 path = os.path.join(temp_dir, file)
2071 os.utime(path, (mtime, mtime))
2072
2073 expected_stdout = '''
2074 upload test2.txt
2075 '''
2076
2077 command = [
2078 'sync', '--no-progress', '--exclude-if-modified-after', '1367600664.152', temp_dir,
2079 'b2://my-bucket'
2080 ]
2081 self._run_command(command, expected_stdout, '', 0)
2082
2083 def _test_sync_threads(
2084 self,
2085 threads=None,
2086 sync_threads=None,
2087 download_threads=None,
2088 upload_threads=None,
2089 ):
2090 self._authorize_account()
2091 self._create_my_bucket()
2092
2093 with TempDir() as temp_dir:
2094 self._make_local_file(temp_dir, 'file.txt')
2095 command = ['sync', '--no-progress']
2096 if threads is not None:
2097 command += ['--threads', str(threads)]
2098 if sync_threads is not None:
2099 command += ['--sync-threads', str(sync_threads)]
2100 if download_threads is not None:
2101 command += ['--download-threads', str(download_threads)]
2102 if upload_threads is not None:
2103 command += ['--upload-threads', str(upload_threads)]
2104 command += [temp_dir, 'b2://my-bucket']
2105 expected_stdout = '''
2106 upload file.txt
2107 '''
2108 self._run_command(command, expected_stdout)
2109
2110 def test_sync_threads(self):
2111 self._test_sync_threads(threads=1)
2112
2113 def test_sync_sync_threads(self):
2114 self._test_sync_threads(sync_threads=1)
2115
2116 def test_sync_download_threads(self):
2117 self._test_sync_threads(download_threads=1)
2118
2119 def test_sync_upload_threads(self):
2120 self._test_sync_threads(upload_threads=1)
2121
2122 def test_sync_many_thread_options(self):
2123 self._test_sync_threads(sync_threads=1, download_threads=1, upload_threads=1)
2124
2125 def test_sync_threads_and_upload_threads(self):
2126 # Using --threads is exclusive with other options
2127 with self.assertRaises(ValueError):
2128 self._test_sync_threads(threads=1, upload_threads=1)
2129
2130 def test_sync_threads_and_sync_threads(self):
2131 # Using --threads is exclusive with other options
2132 with self.assertRaises(ValueError):
2133 self._test_sync_threads(threads=1, sync_threads=1)
2134
2135 def test_sync_threads_and_download_threads(self):
2136 # Using --threads is exclusive with other options
2137 with self.assertRaises(ValueError):
2138 self._test_sync_threads(threads=1, download_threads=1)
2139
2140 def test_sync_all_thread_options(self):
2141 # Using --threads is exclusive with other options
2142 with self.assertRaises(ValueError):
2143 self._test_sync_threads(threads=1, sync_threads=1, download_threads=1, upload_threads=1)
2144
2145 def test_ls(self):
2146 self._authorize_account()
2147 self._create_my_bucket()
2148
2149 # Check with no files
2150 self._run_command(['ls', *self.b2_uri_args('my-bucket')], '', '', 0)
2151
2152 # Create some files, including files in a folder
2153 bucket = self.b2_api.get_bucket_by_name('my-bucket')
2154 bucket.upload(UploadSourceBytes(b''), 'a')
2155 bucket.upload(UploadSourceBytes(b' '), 'b/b1')
2156 bucket.upload(UploadSourceBytes(b' '), 'b/b2')
2157 bucket.upload(UploadSourceBytes(b' '), 'c')
2158 bucket.upload(UploadSourceBytes(b' '), 'c')
2159
2160 # Condensed output
2161 expected_stdout = '''
2162 a
2163 b/
2164 c
2165 '''
2166 self._run_command(['ls', *self.b2_uri_args('my-bucket')], expected_stdout, '', 0)
2167
2168 # Recursive output
2169 expected_stdout = '''
2170 a
2171 b/b1
2172 b/b2
2173 c
2174 '''
2175 self._run_command(
2176 ['ls', '--recursive', *self.b2_uri_args('my-bucket')], expected_stdout, '', 0
2177 )
2178 self._run_command(['ls', '-r', *self.b2_uri_args('my-bucket')], expected_stdout, '', 0)
2179
2180 # Check long output. (The format expects full-length file ids, so it causes whitespace here)
2181 expected_stdout = '''
2182 9999 upload 1970-01-01 00:00:05 0 a
2183 - - - - 0 b/
2184 9995 upload 1970-01-01 00:00:05 6 c
2185 '''
2186 self._run_command(['ls', '--long', *self.b2_uri_args('my-bucket')], expected_stdout, '', 0)
2187
2188 # Check long versions output (The format expects full-length file ids, so it causes whitespace here)
2189 expected_stdout = '''
2190 9999 upload 1970-01-01 00:00:05 0 a
2191 - - - - 0 b/
2192 9995 upload 1970-01-01 00:00:05 6 c
2193 9996 upload 1970-01-01 00:00:05 5 c
2194 '''
2195 self._run_command(
2196 ['ls', '--long', '--versions', *self.b2_uri_args('my-bucket')], expected_stdout, '', 0
2197 )
2198
2199 def test_ls_wildcard(self):
2200 self._authorize_account()
2201 self._create_my_bucket()
2202
2203 # Check with no files
2204 self._run_command(
2205 ['ls', '--recursive', '--with-wildcard', *self.b2_uri_args('my-bucket', '*.txt')], '',
2206 '', 0
2207 )
2208
2209 # Create some files, including files in a folder
2210 bucket = self.b2_api.get_bucket_by_name('my-bucket')
2211 self._upload_multiple_files(bucket)
2212
2213 expected_stdout = '''
2214 a/test.csv
2215 a/test.tsv
2216 b/b/test.csv
2217 b/b1/test.csv
2218 b/b2/test.tsv
2219 c/test.csv
2220 c/test.tsv
2221 '''
2222 self._run_command(
2223 ['ls', '--recursive', '--with-wildcard', *self.b2_uri_args('my-bucket', '*.[tc]sv')],
2224 expected_stdout,
2225 )
2226
2227 expected_stdout = '''
2228 a/test.tsv
2229 b/b2/test.tsv
2230 c/test.tsv
2231 '''
2232 self._run_command(
2233 ['ls', '--recursive', '--with-wildcard', *self.b2_uri_args('my-bucket', '*.tsv')],
2234 expected_stdout,
2235 )
2236
2237 expected_stdout = '''
2238 b/b1/test.csv
2239 '''
2240 self._run_command(
2241 [
2242 'ls', '--recursive', '--with-wildcard',
2243 *self.b2_uri_args('my-bucket', 'b/b?/test.csv')
2244 ],
2245 expected_stdout,
2246 )
2247
2248 expected_stdout = '''
2249 a/test.csv
2250 a/test.tsv
2251 c/test.csv
2252 c/test.tsv
2253 '''
2254 self._run_command(
2255 ['ls', '--recursive', '--with-wildcard', *self.b2_uri_args('my-bucket', '?/test.?sv')],
2256 expected_stdout,
2257 )
2258
2259 expected_stdout = '''
2260 b/b/test.csv
2261 b/b1/test.csv
2262 '''
2263 self._run_command(
2264 [
2265 'ls', '--recursive', '--with-wildcard',
2266 *self.b2_uri_args('my-bucket', '?/*/*.[!t]sv')
2267 ],
2268 expected_stdout,
2269 )
2270
2271 def test_ls_with_wildcard_no_recursive(self):
2272 self._authorize_account()
2273 self._create_my_bucket()
2274
2275 # Check with no files
2276 self._run_command(
2277 ['ls', '--with-wildcard', *self.b2_uri_args('my-bucket')],
2278 expected_stderr='ERROR: with_wildcard requires recursive to be turned on as well\n',
2279 expected_status=1,
2280 )
2281
2282 def test_restrictions(self):
2283 # Initial condition
2284 self.assertEqual(None, self.account_info.get_account_auth_token())
2285
2286 # Authorize an account with the master key.
2287 account_id = self.account_id
2288 self._run_command_ignore_output(['authorize-account', account_id, self.master_key])
2289
2290 # Create a bucket to use
2291 bucket_name = 'restrictedBucket'
2292 bucket_id = 'bucket_0'
2293 self._run_command(['create-bucket', bucket_name, 'allPrivate'], bucket_id + '\n', '', 0)
2294
2295 # Create another bucket
2296 other_bucket_name = 'otherBucket'
2297 self._run_command_ignore_output(['create-bucket', other_bucket_name, 'allPrivate'])
2298
2299 # Create a key restricted to a bucket
2300 app_key_id = 'appKeyId0'
2301 app_key = 'appKey0'
2302 capabilities = "listBuckets,readFiles"
2303 file_prefix = 'some/file/prefix/'
2304 self._run_command(
2305 [
2306 'create-key', '--bucket', bucket_name, '--name-prefix', file_prefix, 'my-key',
2307 capabilities
2308 ],
2309 app_key_id + ' ' + app_key + '\n',
2310 '',
2311 0,
2312 )
2313
2314 self._run_command_ignore_output(['authorize-account', app_key_id, app_key])
2315
2316 # Auth token should be in account info now
2317 self.assertEqual('auth_token_1', self.account_info.get_account_auth_token())
2318
2319 # Assertions that the restrictions not only are saved but what they are supposed to be
2320 self.assertEqual(
2321 dict(
2322 bucketId=bucket_id,
2323 bucketName=bucket_name,
2324 capabilities=[
2325 'listBuckets',
2326 'readFiles',
2327 ],
2328 namePrefix=file_prefix,
2329 ),
2330 self.account_info.get_allowed(),
2331 )
2332
2333 # Test that the application key info gets added to the unauthorized error message.
2334 expected_create_key_stderr = "ERROR: unauthorized for application key " \
2335 "with capabilities 'listBuckets,readFiles', " \
2336 "restricted to bucket 'restrictedBucket', " \
2337 "restricted to files that start with 'some/file/prefix/' (unauthorized)\n"
2338 self._run_command(
2339 ['create-key', 'goodKeyName-One', 'readFiles,listBuckets'],
2340 '',
2341 expected_create_key_stderr,
2342 1,
2343 )
2344
2345 def test_list_buckets_not_allowed_for_app_key(self):
2346 # Create a bucket and a key restricted to that bucket.
2347 self._authorize_account()
2348 self._run_command(
2349 ['create-bucket', 'my-bucket', 'allPrivate'],
2350 'bucket_0\n',
2351 '',
2352 0,
2353 )
2354
2355 # Authorizing with the key will fail because the ConsoleTool needs
2356 # to be able to look up the name of the bucket.
2357 self._run_command(
2358 ['create-key', 'my-key', 'listFiles'],
2359 'appKeyId0 appKey0\n',
2360 '',
2361 0,
2362 )
2363
2364 # Authorize with the key, which should result in an error.
2365 self._run_command(
2366 ['authorize-account', 'appKeyId0', 'appKey0'],
2367 '',
2368 'ERROR: application key has no listBuckets capability, which is required for the b2 command-line tool\n',
2369 1,
2370 )
2371
2372 def test_bucket_missing_for_bucket_key(self):
2373 # Create a bucket and a key restricted to that bucket.
2374 self._authorize_account()
2375 self._run_command(
2376 ['create-bucket', 'my-bucket', 'allPrivate'],
2377 'bucket_0\n',
2378 '',
2379 0,
2380 )
2381 self._run_command(
2382 ['create-key', '--bucket', 'my-bucket', 'my-key', 'listBuckets,listFiles'],
2383 'appKeyId0 appKey0\n',
2384 '',
2385 0,
2386 )
2387
2388 # Get rid of the bucket, leaving the key with a dangling pointer to it.
2389 self._run_command_ignore_output(['delete-bucket', 'my-bucket'])
2390
2391 # Authorizing with the key will fail because the ConsoleTool needs
2392 # to be able to look up the name of the bucket.
2393 self._run_command(
2394 ['authorize-account', 'appKeyId0', 'appKey0'],
2395 '',
2396 "ERROR: unable to authorize account: Application key is restricted to a bucket that doesn't exist\n",
2397 1,
2398 )
2399
2400 def test_ls_for_restricted_bucket(self):
2401 # Create a bucket and a key restricted to that bucket.
2402 self._authorize_account()
2403 self._run_command(
2404 ['create-bucket', 'my-bucket', 'allPrivate'],
2405 'bucket_0\n',
2406 '',
2407 0,
2408 )
2409 self._run_command(
2410 ['create-key', '--bucket', 'my-bucket', 'my-key', 'listBuckets,listFiles'],
2411 'appKeyId0 appKey0\n',
2412 '',
2413 0,
2414 )
2415
2416 # Authorize with the key and list the files
2417 self._run_command_ignore_output(['authorize-account', 'appKeyId0', 'appKey0'],)
2418 self._run_command(
2419 ['ls', *self.b2_uri_args('my-bucket')],
2420 '',
2421 '',
2422 0,
2423 )
2424
2425 def test_bad_terminal(self):
2426 stdout = mock.MagicMock()
2427 stdout.write = mock.MagicMock(
2428 side_effect=[
2429 UnicodeEncodeError('codec', 'foo', 100, 105, 'artificial UnicodeEncodeError')
2430 ] + list(range(25))
2431 )
2432 stderr = mock.MagicMock()
2433 console_tool = self.console_tool_class(stdout, stderr)
2434 console_tool.run_command(['b2', 'authorize-account', self.account_id, self.master_key])
2435
2436 def test_passing_api_parameters(self):
2437 self._authorize_account()
2438 commands = [
2439 [
2440 'b2', 'download-file-by-name', '--profile', 'nonexistent', 'dummy-name',
2441 'dummy-file-name', 'dummy-local-file-name'
2442 ],
2443 [
2444 'b2', 'download-file-by-id', '--profile', 'nonexistent', 'dummy-id',
2445 'dummy-local-file-name'
2446 ],
2447 ['b2', 'sync', '--profile', 'nonexistent', 'b2:dummy-source', 'dummy-destination'],
2448 ]
2449 parameters = [
2450 {
2451 '--write-buffer-size': 123,
2452 '--skip-hash-verification': None,
2453 '--max-download-streams-per-file': 8,
2454 },
2455 {
2456 '--write-buffer-size': 321,
2457 '--max-download-streams-per-file': 7,
2458 },
2459 ]
2460 for command, params in product(commands, parameters):
2461 console_tool = self.console_tool_class(
2462 mock.MagicMock(),
2463 mock.MagicMock(),
2464 )
2465
2466 args = [str(val) for val in chain.from_iterable(params.items()) if val]
2467 console_tool.run_command(command + args)
2468
2469 download_manager = console_tool.api.services.download_manager
2470 assert download_manager.write_buffer_size == params['--write-buffer-size']
2471 assert download_manager.check_hash is ('--skip-hash-verification' not in params)
2472
2473 parallel_strategy = one(
2474 strategy for strategy in download_manager.strategies
2475 if isinstance(strategy, download_manager.PARALLEL_DOWNLOADER_CLASS)
2476 )
2477 assert parallel_strategy.max_streams == params['--max-download-streams-per-file']
2478
2479 def test_passing_api_parameters_with_auth_env_vars(self):
2480
2481 os.environ[B2_APPLICATION_KEY_ID_ENV_VAR] = self.account_id
2482 os.environ[B2_APPLICATION_KEY_ENV_VAR] = self.master_key
2483
2484 command = [
2485 'b2',
2486 'download-file-by-id',
2487 'dummy-id',
2488 'dummy-local-file-name',
2489 '--write-buffer-size',
2490 '123',
2491 '--max-download-streams-per-file',
2492 '5',
2493 '--skip-hash-verification',
2494 ]
2495
2496 console_tool = self.console_tool_class(
2497 mock.MagicMock(),
2498 mock.MagicMock(),
2499 )
2500 console_tool.run_command(command)
2501
2502 download_manager = console_tool.api.services.download_manager
2503 assert download_manager.write_buffer_size == 123
2504 assert download_manager.check_hash is False
2505
2506 parallel_strategy = one(
2507 strategy for strategy in download_manager.strategies
2508 if isinstance(strategy, download_manager.PARALLEL_DOWNLOADER_CLASS)
2509 )
2510 assert parallel_strategy.max_streams == 5
2511
2512 @pytest.mark.apiver(from_ver=4)
2513 def test_ls_b2id(self):
2514 self._authorize_account()
2515 self._create_my_bucket()
2516
2517 # Create a file
2518 bucket = self.b2_api.get_bucket_by_name('my-bucket')
2519 file_version = bucket.upload(UploadSourceBytes(b''), 'test.txt')
2520
2521 # Condensed output
2522 expected_stdout = '''
2523 test.txt
2524 '''
2525 self._run_command(['ls', f'b2id://{file_version.id_}'], expected_stdout, '', 0)
2526
2527 def test_ls_filters(self):
2528 self._authorize_account()
2529 self._create_my_bucket()
2530
2531 # Create some files, including files in a folder
2532 bucket = self.b2_api.get_bucket_by_name('my-bucket')
2533 data = UploadSourceBytes(b'test-data')
2534 bucket.upload(data, 'a/test.csv')
2535 bucket.upload(data, 'a/test.tsv')
2536 bucket.upload(data, 'b/b/test.csv')
2537 bucket.upload(data, 'c/test.csv')
2538 bucket.upload(data, 'c/test.tsv')
2539 bucket.upload(data, 'test.csv')
2540 bucket.upload(data, 'test.tsv')
2541
2542 expected_stdout = '''
2543 a/
2544 b/
2545 c/
2546 test.csv
2547 '''
2548 self._run_command(
2549 ['ls', *self.b2_uri_args('my-bucket'), '--include', '*.csv'],
2550 expected_stdout,
2551 )
2552 self._run_command(
2553 ['ls', *self.b2_uri_args('my-bucket'), '--exclude', '*.tsv'],
2554 expected_stdout,
2555 )
2556
2557 expected_stdout = '''
2558 a/test.csv
2559 b/b/test.csv
2560 c/test.csv
2561 test.csv
2562 '''
2563 self._run_command(
2564 ['ls', *self.b2_uri_args('my-bucket'), '--recursive', '--include', '*.csv'],
2565 expected_stdout,
2566 )
2567 self._run_command(
2568 ['ls', *self.b2_uri_args('my-bucket'), '--recursive', '--exclude', '*.tsv'],
2569 expected_stdout,
2570 )
2571
2572 expected_stdout = '''
2573 b/b/test.csv
2574 c/test.csv
2575 test.csv
2576 '''
2577 self._run_command(
2578 [
2579 'ls', *self.b2_uri_args('my-bucket'), '--recursive', '--exclude', '*', '--include',
2580 '*.csv', '--exclude', 'a/*'
2581 ],
2582 expected_stdout,
2583 )
2584
2585 @pytest.mark.skip("temporarily disabled")
2586 @skip_on_windows
2587 def test_escape_c0_char_on_sync_stack_trace(self):
2588 self._authorize_account()
2589 self._run_command(['create-bucket', 'my-bucket-0', 'allPrivate'], 'bucket_0\n', '', 0)
2590 self._run_command(['create-bucket', 'my-bucket-1', 'allPrivate'], 'bucket_1\n', '', 0)
2591
2592 with TempDir() as temp_dir:
2593 _ = self._make_local_file(temp_dir, "\x1b[32mC\x1b[33mC\x1b[34mI\x1b[0m")
2594 self._run_command(
2595 [
2596 'sync', '--no-progress', '--no-escape-control-characters', temp_dir,
2597 'b2://my-bucket-0'
2598 ],
2599 expected_part_of_stdout='\\x1b[32m',
2600 expected_status=0,
2601 )
2602 self._run_command(
2603 [
2604 'sync', '--no-progress', '--escape-control-characters', temp_dir,
2605 'b2://my-bucket-1'
2606 ],
2607 expected_part_of_stdout="upload '\\x1b[32mC\\x1b[33mC\\x1b[34mI\\x1b[0m'\n",
2608 expected_status=0,
2609 unexpected_part_of_stdout='\x1b[32m',
2610 )
2611
2612 def test_escape_c0_char_on_key_restricted_path(self):
2613 self._authorize_account()
2614 self._run_command(['create-bucket', 'my-bucket-0', 'allPublic'], 'bucket_0\n', '', 0)
2615 cc_name = "$'\x1b[31mC\x1b[32mC\x1b[33mI\x1b[0m'"
2616 escaped_error = "ERROR: unauthorized for application key with capabilities 'listBuckets,listKeys', restricted to bucket 'my-bucket-0', restricted to files that start with '$'\\x1b[31mC\\x1b[32mC\\x1b[33mI\\x1b[0m'' (unauthorized)\n"
2617
2618 # Create a key
2619 self._run_command(
2620 [
2621 'create-key', '--bucket', 'my-bucket-0', '--name-prefix', cc_name, 'key1',
2622 'listBuckets,listKeys'
2623 ],
2624 'appKeyId0 appKey0\n',
2625 expected_status=0,
2626 )
2627
2628 # Authorize with the key
2629 self._run_command(['authorize-account', 'appKeyId0', 'appKey0'], expected_status=0)
2630
2631 self._run_command(
2632 ['ls', *self.b2_uri_args('my-bucket-0'), '--no-escape-control-characters'],
2633 expected_status=1,
2634 expected_stderr=escaped_error
2635 )
2636
2637 self._run_command(
2638 ['ls', *self.b2_uri_args('my-bucket-0'), '--escape-control-characters'],
2639 expected_status=1,
2640 expected_stderr=escaped_error
2641 )
2642
2643 self._run_command(
2644 ['ls', *self.b2_uri_args('my-bucket-0')],
2645 expected_status=1,
2646 expected_stderr=escaped_error
2647 )
2648
2649 def test_escape_c1_char_on_ls_long(self):
2650 self._authorize_account()
2651 self._run_command(['create-bucket', 'my-bucket-0', 'allPrivate'], 'bucket_0\n', '', 0)
2652
2653 with TempDir() as temp_dir:
2654 local_file = self._make_local_file(temp_dir, 'file1.txt')
2655 cc_filename = '\u009bT\u009bE\u009bS\u009bTtest.txt'
2656 escaped_cc_filename = '\\x9bT\\x9bE\\x9bS\\x9bTtest.txt'
2657
2658 self._run_command(
2659 ['upload-file', '--no-progress', 'my-bucket-0', local_file, cc_filename]
2660 )
2661
2662 self._run_command(
2663 ['ls', '--long', '--no-escape-control-characters', *self.b2_uri_args('my-bucket-0')],
2664 expected_part_of_stdout=cc_filename,
2665 )
2666
2667 self._run_command(
2668 ['ls', '--long', *self.b2_uri_args('my-bucket-0')], expected_part_of_stdout=cc_filename
2669 )
2670
2671 self._run_command(
2672 ['ls', '--long', '--escape-control-characters', *self.b2_uri_args('my-bucket-0')],
2673 expected_part_of_stdout=escaped_cc_filename,
2674 unexpected_part_of_stdout=cc_filename
2675 )
2676
2677 def test_escape_c1_char_ls(self):
2678 self._authorize_account()
2679 self._run_command(['create-bucket', 'my-bucket-cc', 'allPrivate'], 'bucket_0\n', '', 0)
2680
2681 with TempDir() as temp_dir:
2682 local_file = self._make_local_file(temp_dir, "x")
2683 bad_str = "\u009b2K\u009b7Gb\u009b24Gx\u009b4GH"
2684 escaped_bad_str = "\\x9b2K\\x9b7Gb\\x9b24Gx\\x9b4GH"
2685
2686 self._run_command(['upload-file', '--no-progress', 'my-bucket-cc', local_file, bad_str])
2687
2688 self._run_command(
2689 ['upload-file', '--no-progress', 'my-bucket-cc', local_file, "some_normal_text"]
2690 )
2691
2692 self._run_command(
2693 ['ls', *self.b2_uri_args('my-bucket-cc'), '--no-escape-control-characters'],
2694 expected_part_of_stdout=bad_str
2695 )
2696
2697 self._run_command(
2698 ['ls', *self.b2_uri_args('my-bucket-cc')], expected_part_of_stdout=bad_str
2699 )
2700
2701 self._run_command(
2702 ['ls', *self.b2_uri_args('my-bucket-cc'), '--escape-control-characters'],
2703 expected_part_of_stdout=escaped_bad_str
2704 )
2705
2706
2707 class TestConsoleToolWithV1(BaseConsoleToolTest):
2708 """These tests use v1 interface to perform various setups before running CLI commands"""
2709
2710 def setUp(self):
2711 super().setUp()
2712 self.v1_account_info = v1.StubAccountInfo()
2713
2714 self.v1_b2_api = v1.B2Api(self.v1_account_info, None)
2715 self.v1_b2_api.session.raw_api = self.raw_api
2716 self.v1_b2_api.authorize_account('production', self.account_id, self.master_key)
2717 self._authorize_account()
2718 self._create_my_bucket()
2719 self.v1_bucket = self.v1_b2_api.create_bucket('my-v1-bucket', 'allPrivate')
2720
2721 def test_cancel_large_file(self):
2722 file = self.v1_bucket.start_large_file('file1', 'text/plain', {})
2723 self._run_command(['cancel-large-file', file.file_id], '9999 canceled\n', '', 0)
2724
2725 def test_cancel_all_large_file(self):
2726 self.v1_bucket.start_large_file('file1', 'text/plain', {})
2727 self.v1_bucket.start_large_file('file2', 'text/plain', {})
2728 expected_stdout = '''
2729 9999 canceled
2730 9998 canceled
2731 '''
2732
2733 self._run_command(
2734 ['cancel-all-unfinished-large-files', 'my-v1-bucket'], expected_stdout, '', 0
2735 )
2736
2737 def test_list_parts_with_none(self):
2738 file = self.v1_bucket.start_large_file('file', 'text/plain', {})
2739 self._run_command(['list-parts', file.file_id], '', '', 0)
2740
2741 def test_list_parts_with_parts(self):
2742
2743 bucket = self.b2_api.get_bucket_by_name('my-bucket')
2744 file = self.v1_bucket.start_large_file('file', 'text/plain', {})
2745 content = b'hello world'
2746 large_file_upload_state = mock.MagicMock()
2747 large_file_upload_state.has_error.return_value = False
2748 bucket.api.services.upload_manager._upload_part(
2749 bucket.id_, file.file_id, UploadSourceBytes(content), 1, large_file_upload_state, None,
2750 None
2751 )
2752 bucket.api.services.upload_manager._upload_part(
2753 bucket.id_, file.file_id, UploadSourceBytes(content), 3, large_file_upload_state, None,
2754 None
2755 )
2756 expected_stdout = '''
2757 1 11 2aae6c35c94fcfb415dbe95f408b9ce91ee846ed
2758 3 11 2aae6c35c94fcfb415dbe95f408b9ce91ee846ed
2759 '''
2760
2761 self._run_command(['list-parts', file.file_id], expected_stdout, '', 0)
2762
2763 def test_list_unfinished_large_files_with_some(self):
2764 api_url = self.account_info.get_api_url()
2765 auth_token = self.account_info.get_account_auth_token()
2766 self.raw_api.start_large_file(api_url, auth_token, 'bucket_0', 'file1', 'text/plain', {})
2767 self.raw_api.start_large_file(
2768 api_url, auth_token, 'bucket_0', 'file2', 'text/plain', {'color': 'blue'}
2769 )
2770 self.raw_api.start_large_file(
2771 api_url, auth_token, 'bucket_0', 'file3', 'application/json', {}
2772 )
2773 expected_stdout = '''
2774 9999 file1 text/plain
2775 9998 file2 text/plain color=blue
2776 9997 file3 application/json
2777 '''
2778
2779 self._run_command(['list-unfinished-large-files', 'my-bucket'], expected_stdout, '', 0)
2780
2781
2782 class TestRmConsoleTool(BaseConsoleToolTest):
2783 """
2784 These tests replace default progress reporter of Rm class
2785 to ensure that it reports everything as fast as possible.
2786 """
2787
2788 class InstantReporter(ProgressReport):
2789 UPDATE_INTERVAL = 0.0
2790
2791 @classmethod
2792 def setUpClass(cls) -> None:
2793 cls.original_v3_progress_class = v3Rm.PROGRESS_REPORT_CLASS
2794 cls.original_v4_progress_class = v4Rm.PROGRESS_REPORT_CLASS
2795
2796 v3Rm.PROGRESS_REPORT_CLASS = cls.InstantReporter
2797 v4Rm.PROGRESS_REPORT_CLASS = cls.InstantReporter
2798
2799 def setUp(self):
2800 super().setUp()
2801
2802 self._authorize_account()
2803 self._create_my_bucket()
2804
2805 self.bucket = self.b2_api.get_bucket_by_name('my-bucket')
2806 self._upload_multiple_files(self.bucket)
2807
2808 @classmethod
2809 def tearDownClass(cls) -> None:
2810 v3Rm.PROGRESS_REPORT_CLASS = cls.original_v3_progress_class
2811 v4Rm.PROGRESS_REPORT_CLASS = cls.original_v4_progress_class
2812
2813 def test_rm_wildcard(self):
2814 self._run_command(
2815 [
2816 'rm', '--recursive', '--with-wildcard', '--no-progress',
2817 *self.b2_uri_args('my-bucket', '*.csv')
2818 ],
2819 )
2820
2821 expected_stdout = '''
2822 a/test.tsv
2823 b/b2/test.tsv
2824 b/test.txt
2825 c/test.tsv
2826 '''
2827 self._run_command(['ls', '--recursive', *self.b2_uri_args('my-bucket')], expected_stdout)
2828
2829 def test_rm_versions(self):
2830 # Uploading content of the bucket again to create second version of each file.
2831 self._upload_multiple_files(self.bucket)
2832
2833 self._run_command(
2834 [
2835 'rm', '--versions', '--recursive', '--with-wildcard',
2836 *self.b2_uri_args('my-bucket', '*.csv')
2837 ],
2838 )
2839
2840 expected_stdout = '''
2841 a/test.tsv
2842 a/test.tsv
2843 b/b2/test.tsv
2844 b/b2/test.tsv
2845 b/test.txt
2846 b/test.txt
2847 c/test.tsv
2848 c/test.tsv
2849 '''
2850 self._run_command(
2851 ['ls', '--versions', '--recursive', *self.b2_uri_args('my-bucket')], expected_stdout
2852 )
2853
2854 def test_rm_no_recursive(self):
2855 self._run_command(['rm', '--no-progress', *self.b2_uri_args('my-bucket', 'b/')])
2856
2857 expected_stdout = '''
2858 a/test.csv
2859 a/test.tsv
2860 b/b/test.csv
2861 b/b1/test.csv
2862 b/b2/test.tsv
2863 c/test.csv
2864 c/test.tsv
2865 '''
2866 self._run_command(['ls', '--recursive', *self.b2_uri_args('my-bucket')], expected_stdout)
2867
2868 def test_rm_dry_run(self):
2869 expected_stdout = '''
2870 a/test.csv
2871 b/b/test.csv
2872 b/b1/test.csv
2873 c/test.csv
2874 '''
2875 self._run_command(
2876 [
2877 'rm', '--recursive', '--with-wildcard', '--dry-run',
2878 *self.b2_uri_args('my-bucket', '*.csv')
2879 ],
2880 expected_stdout,
2881 )
2882
2883 expected_stdout = '''
2884 a/test.csv
2885 a/test.tsv
2886 b/b/test.csv
2887 b/b1/test.csv
2888 b/b2/test.tsv
2889 b/test.txt
2890 c/test.csv
2891 c/test.tsv
2892 '''
2893 self._run_command(['ls', '--recursive', *self.b2_uri_args('my-bucket')], expected_stdout)
2894
2895 def test_rm_exact_filename(self):
2896 self._run_command(
2897 [
2898 'rm', '--recursive', '--with-wildcard', '--no-progress',
2899 *self.b2_uri_args('my-bucket', 'b/b/test.csv')
2900 ],
2901 )
2902
2903 expected_stdout = '''
2904 a/test.csv
2905 a/test.tsv
2906 b/b1/test.csv
2907 b/b2/test.tsv
2908 b/test.txt
2909 c/test.csv
2910 c/test.tsv
2911 '''
2912 self._run_command(['ls', '--recursive', *self.b2_uri_args('my-bucket')], expected_stdout)
2913
2914 def test_rm_no_name_removes_everything(self):
2915 self._run_command(['rm', '--recursive', '--no-progress', *self.b2_uri_args('my-bucket')])
2916 self._run_command(['ls', '--recursive', *self.b2_uri_args('my-bucket')], '')
2917
2918 def test_rm_with_wildcard_without_recursive(self):
2919 self._run_command(
2920 ['rm', '--with-wildcard', *self.b2_uri_args('my-bucket')],
2921 expected_stderr='ERROR: with_wildcard requires recursive to be turned on as well\n',
2922 expected_status=1,
2923 )
2924
2925 def test_rm_queue_size_and_number_of_threads(self):
2926 self._run_command(
2927 [
2928 'rm', '--recursive', '--threads', '2', '--queue-size', '4',
2929 *self.b2_uri_args('my-bucket')
2930 ]
2931 )
2932 self._run_command(['ls', '--recursive', *self.b2_uri_args('my-bucket')], '')
2933
2934 def test_rm_progress(self):
2935 expected_in_stdout = ' count: 4/4 '
2936 self._run_command(
2937 ['rm', '--recursive', '--with-wildcard', *self.b2_uri_args('my-bucket', '*.csv')],
2938 expected_part_of_stdout=expected_in_stdout,
2939 )
2940
2941 expected_stdout = '''
2942 a/test.tsv
2943 b/b2/test.tsv
2944 b/test.txt
2945 c/test.tsv
2946 '''
2947 self._run_command(['ls', '--recursive', *self.b2_uri_args('my-bucket')], expected_stdout)
2948
2949 def _run_problematic_removal(
2950 self,
2951 additional_parameters: Optional[List[str]] = None,
2952 expected_in_stdout: Optional[str] = None,
2953 unexpected_in_stdout: Optional[str] = None
2954 ):
2955 additional_parameters = additional_parameters or []
2956
2957 original_delete_file_version = self.b2_api.raw_api.delete_file_version
2958
2959 def mocked_delete_file_version(
2960 this, account_auth_token, file_id, file_name, bypass_governance=False, *args, **kwargs
2961 ):
2962 if file_name == 'b/b1/test.csv':
2963 raise Conflict()
2964 return original_delete_file_version(
2965 this, account_auth_token, file_id, file_name, bypass_governance, *args, **kwargs
2966 )
2967
2968 with mock.patch.object(
2969 self.b2_api.raw_api,
2970 'delete_file_version',
2971 side_effect=mocked_delete_file_version,
2972 ):
2973 self._run_command(
2974 [
2975 'rm',
2976 '--recursive',
2977 '--with-wildcard',
2978 '--threads',
2979 '1',
2980 '--queue-size',
2981 '1',
2982 *additional_parameters,
2983 *self.b2_uri_args('my-bucket', '*'),
2984 ],
2985 expected_status=1,
2986 expected_part_of_stdout=expected_in_stdout,
2987 unexpected_part_of_stdout=unexpected_in_stdout,
2988 )
2989
2990 def test_rm_fail_fast(self):
2991 # Since we already have all the jobs submitted to another thread,
2992 # we can only rely on the log to tell when it stopped.
2993 expected_in_stdout = '''
2994 Deletion of file "b/b1/test.csv" (9996) failed: Conflict:
2995 count: 3/4'''
2996 unexpected_in_stdout = ' count: 5/5 '
2997 self._run_problematic_removal(['--fail-fast'], expected_in_stdout, unexpected_in_stdout)
2998
2999 def test_rm_skipping_over_errors(self):
3000 self._run_problematic_removal()
3001
3002 expected_stdout = '''
3003 b/b1/test.csv
3004 '''
3005 self._run_command(['ls', '--recursive', *self.b2_uri_args('my-bucket')], expected_stdout)
3006
3007 @pytest.mark.apiver(from_ver=4)
3008 def test_rm_b2id(self):
3009 # Create a file
3010 file_version = self.bucket.upload(UploadSourceBytes(b''), 'new-file.txt')
3011
3012 # Before deleting
3013 expected_stdout = '''
3014 a/test.csv
3015 a/test.tsv
3016 b/b/test.csv
3017 b/b1/test.csv
3018 b/b2/test.tsv
3019 b/test.txt
3020 c/test.csv
3021 c/test.tsv
3022 new-file.txt
3023 '''
3024 self._run_command(['ls', '--recursive', 'b2://my-bucket'], expected_stdout)
3025
3026 # Delete file
3027 self._run_command(['rm', '--no-progress', f'b2id://{file_version.id_}'], '', '', 0)
3028
3029 # After deleting
3030 expected_stdout = '''
3031 a/test.csv
3032 a/test.tsv
3033 b/b/test.csv
3034 b/b1/test.csv
3035 b/b2/test.tsv
3036 b/test.txt
3037 c/test.csv
3038 c/test.tsv
3039 '''
3040 self._run_command(['ls', '--recursive', 'b2://my-bucket'], expected_stdout)
3041
3042 def rm_filters_helper(self, rm_args: List[str], expected_ls_stdout: str):
3043 self._authorize_account()
3044 self._run_command(['create-bucket', 'my-rm-bucket', 'allPublic'], 'bucket_1\n', '', 0)
3045 bucket = self.b2_api.get_bucket_by_name('my-rm-bucket')
3046
3047 # Create some files, including files in a folder
3048 data = UploadSourceBytes(b'test-data')
3049 bucket.upload(data, 'a/test.csv')
3050 bucket.upload(data, 'a/test.tsv')
3051 bucket.upload(data, 'b/b/test.csv')
3052 bucket.upload(data, 'c/test.csv')
3053 bucket.upload(data, 'c/test.tsv')
3054 bucket.upload(data, 'test.csv')
3055 bucket.upload(data, 'test.tsv')
3056 bucket.upload(data, 'test.txt')
3057
3058 self._run_command(
3059 ['rm', '--no-progress', *self.b2_uri_args('my-rm-bucket'), *rm_args], '', '', 0
3060 )
3061 self._run_command(
3062 ['ls', *self.b2_uri_args('my-rm-bucket'), '--recursive'],
3063 expected_ls_stdout,
3064 )
3065
3066 def test_rm_filters_include(self):
3067 expected_ls_stdout = '''
3068 a/test.csv
3069 a/test.tsv
3070 b/b/test.csv
3071 c/test.csv
3072 c/test.tsv
3073 test.tsv
3074 test.txt
3075 '''
3076 self.rm_filters_helper(['--include', '*.csv'], expected_ls_stdout)
3077
3078 def test_rm_filters_exclude(self):
3079 expected_ls_stdout = '''
3080 a/test.csv
3081 a/test.tsv
3082 b/b/test.csv
3083 c/test.csv
3084 c/test.tsv
3085 test.csv
3086 '''
3087 self.rm_filters_helper(['--exclude', '*.csv'], expected_ls_stdout)
3088
3089 def test_rm_filters_include_recursive(self):
3090 expected_ls_stdout = '''
3091 a/test.tsv
3092 c/test.tsv
3093 test.tsv
3094 test.txt
3095 '''
3096 self.rm_filters_helper(['--recursive', '--include', '*.csv'], expected_ls_stdout)
3097
3098 def test_rm_filters_exclude_recursive(self):
3099 expected_ls_stdout = '''
3100 a/test.csv
3101 b/b/test.csv
3102 c/test.csv
3103 test.csv
3104 '''
3105 self.rm_filters_helper(['--recursive', '--exclude', '*.csv'], expected_ls_stdout)
3106
3107 def test_rm_filters_mixed(self):
3108 expected_ls_stdout = '''
3109 a/test.csv
3110 a/test.tsv
3111 c/test.tsv
3112 test.tsv
3113 test.txt
3114 '''
3115 self.rm_filters_helper(
3116 ['--recursive', '--exclude', '*', '--include', '*.csv', '--exclude', 'a/*'],
3117 expected_ls_stdout
3118 )
3119
3120
3121 class TestVersionConsoleTool(BaseConsoleToolTest):
3122 def test_version(self):
3123 self._run_command(['version', '--short'], expected_stdout=f'{VERSION}\n')
3124 self._run_command(['version'], expected_stdout=f'b2 command line tool, version {VERSION}\n')