Debian Codegraph @

Lib/test/test_tarfile.py

raw
1 import errno
2 import sys
3 import os
4 import io
5 from hashlib import sha256
6 from contextlib import contextmanager, ExitStack
7 from random import Random
8 import pathlib
9 import shutil
10 import re
11 import warnings
12 import stat
13
14 import unittest
15 import unittest.mock
16 import tarfile
17
18 from test import archiver_tests
19 from test import support
20 from test.support import os_helper
21 from test.support import script_helper
22 from test.support import warnings_helper
23
24 # Check for our compression modules.
25 try:
26 import gzip
27 except ImportError:
28 gzip = None
29 try:
30 import zlib
31 except ImportError:
32 zlib = None
33 try:
34 import bz2
35 except ImportError:
36 bz2 = None
37 try:
38 import lzma
39 except ImportError:
40 lzma = None
41 try:
42 from compression import zstd
43 except ImportError:
44 zstd = None
45
46 def sha256sum(data):
47 return sha256(data).hexdigest()
48
49 TEMPDIR = os.path.abspath(os_helper.TESTFN) + "-tardir"
50 tarextdir = TEMPDIR + '-extract-test'
51 tarname = support.findfile("testtar.tar", subdir="archivetestdata")
52 gzipname = os.path.join(TEMPDIR, "testtar.tar.gz")
53 bz2name = os.path.join(TEMPDIR, "testtar.tar.bz2")
54 xzname = os.path.join(TEMPDIR, "testtar.tar.xz")
55 zstname = os.path.join(TEMPDIR, "testtar.tar.zst")
56 tmpname = os.path.join(TEMPDIR, "tmp.tar")
57 dotlessname = os.path.join(TEMPDIR, "testtar")
58 SPACE = b" "
59
60 sha256_regtype = (
61 "e09e4bc8b3c9d9177e77256353b36c159f5f040531bbd4b024a8f9b9196c71ce"
62 )
63 sha256_sparse = (
64 "4f05a776071146756345ceee937b33fc5644f5a96b9780d1c7d6a32cdf164d7b"
65 )
66
67
68 class TarTest:
69 tarname = tarname
70 suffix = ''
71 open = io.FileIO
72 taropen = tarfile.TarFile.taropen
73
74 @property
75 def mode(self):
76 return self.prefix + self.suffix
77
78 @support.requires_gzip()
79 class GzipTest:
80 tarname = gzipname
81 suffix = 'gz'
82 open = gzip.GzipFile if gzip else None
83 taropen = tarfile.TarFile.gzopen
84
85 @support.requires_bz2()
86 class Bz2Test:
87 tarname = bz2name
88 suffix = 'bz2'
89 open = bz2.BZ2File if bz2 else None
90 taropen = tarfile.TarFile.bz2open
91
92 @support.requires_lzma()
93 class LzmaTest:
94 tarname = xzname
95 suffix = 'xz'
96 open = lzma.LZMAFile if lzma else None
97 taropen = tarfile.TarFile.xzopen
98
99 @support.requires_zstd()
100 class ZstdTest:
101 tarname = zstname
102 suffix = 'zst'
103 open = zstd.ZstdFile if zstd else None
104 taropen = tarfile.TarFile.zstopen
105
106 class ReadTest(TarTest):
107
108 prefix = "r:"
109
110 def setUp(self):
111 self.tar = tarfile.open(self.tarname, mode=self.mode,
112 encoding="iso8859-1")
113
114 def tearDown(self):
115 self.tar.close()
116
117 class StreamModeTest(ReadTest):
118
119 # Only needs to change how the tarfile is opened to set
120 # stream mode
121 def setUp(self):
122 self.tar = tarfile.open(self.tarname, mode=self.mode,
123 encoding="iso8859-1",
124 stream=True)
125
126 class UstarReadTest(ReadTest, unittest.TestCase):
127
128 def test_fileobj_regular_file(self):
129 tarinfo = self.tar.getmember("ustar/regtype")
130 with self.tar.extractfile(tarinfo) as fobj:
131 data = fobj.read()
132 self.assertEqual(len(data), tarinfo.size,
133 "regular file extraction failed")
134 self.assertEqual(sha256sum(data), sha256_regtype,
135 "regular file extraction failed")
136
137 def test_fileobj_readlines(self):
138 self.tar.extract("ustar/regtype", TEMPDIR, filter='data')
139 tarinfo = self.tar.getmember("ustar/regtype")
140 with open(os.path.join(TEMPDIR, "ustar/regtype"), "r") as fobj1:
141 lines1 = fobj1.readlines()
142
143 with self.tar.extractfile(tarinfo) as fobj:
144 fobj2 = io.TextIOWrapper(fobj)
145 lines2 = fobj2.readlines()
146 self.assertEqual(lines1, lines2,
147 "fileobj.readlines() failed")
148 self.assertEqual(len(lines2), 114,
149 "fileobj.readlines() failed")
150 self.assertEqual(lines2[83],
151 "I will gladly admit that Python is not the fastest "
152 "running scripting language.\n",
153 "fileobj.readlines() failed")
154
155 def test_fileobj_iter(self):
156 self.tar.extract("ustar/regtype", TEMPDIR, filter='data')
157 tarinfo = self.tar.getmember("ustar/regtype")
158 with open(os.path.join(TEMPDIR, "ustar/regtype"), "r") as fobj1:
159 lines1 = fobj1.readlines()
160 with self.tar.extractfile(tarinfo) as fobj2:
161 lines2 = list(io.TextIOWrapper(fobj2))
162 self.assertEqual(lines1, lines2,
163 "fileobj.__iter__() failed")
164
165 def test_fileobj_seek(self):
166 self.tar.extract("ustar/regtype", TEMPDIR,
167 filter='data')
168 with open(os.path.join(TEMPDIR, "ustar/regtype"), "rb") as fobj:
169 data = fobj.read()
170
171 tarinfo = self.tar.getmember("ustar/regtype")
172 with self.tar.extractfile(tarinfo) as fobj:
173 text = fobj.read()
174 fobj.seek(0)
175 self.assertEqual(0, fobj.tell(),
176 "seek() to file's start failed")
177 fobj.seek(2048, 0)
178 self.assertEqual(2048, fobj.tell(),
179 "seek() to absolute position failed")
180 fobj.seek(-1024, 1)
181 self.assertEqual(1024, fobj.tell(),
182 "seek() to negative relative position failed")
183 fobj.seek(1024, 1)
184 self.assertEqual(2048, fobj.tell(),
185 "seek() to positive relative position failed")
186 s = fobj.read(10)
187 self.assertEqual(s, data[2048:2058],
188 "read() after seek failed")
189 fobj.seek(0, 2)
190 self.assertEqual(tarinfo.size, fobj.tell(),
191 "seek() to file's end failed")
192 self.assertEqual(fobj.read(), b"",
193 "read() at file's end did not return empty string")
194 fobj.seek(-tarinfo.size, 2)
195 self.assertEqual(0, fobj.tell(),
196 "relative seek() to file's end failed")
197 fobj.seek(512)
198 s1 = fobj.readlines()
199 fobj.seek(512)
200 s2 = fobj.readlines()
201 self.assertEqual(s1, s2,
202 "readlines() after seek failed")
203 fobj.seek(0)
204 self.assertEqual(len(fobj.readline()), fobj.tell(),
205 "tell() after readline() failed")
206 fobj.seek(512)
207 self.assertEqual(len(fobj.readline()) + 512, fobj.tell(),
208 "tell() after seek() and readline() failed")
209 fobj.seek(0)
210 line = fobj.readline()
211 self.assertEqual(fobj.read(), data[len(line):],
212 "read() after readline() failed")
213
214 def test_fileobj_text(self):
215 with self.tar.extractfile("ustar/regtype") as fobj:
216 fobj = io.TextIOWrapper(fobj)
217 data = fobj.read().encode("iso8859-1")
218 self.assertEqual(sha256sum(data), sha256_regtype)
219 try:
220 fobj.seek(100)
221 except AttributeError:
222 # Issue #13815: seek() complained about a missing
223 # flush() method.
224 self.fail("seeking failed in text mode")
225
226 # Test if symbolic and hard links are resolved by extractfile(). The
227 # test link members each point to a regular member whose data is
228 # supposed to be exported.
229 def _test_fileobj_link(self, lnktype, regtype):
230 with self.tar.extractfile(lnktype) as a, \
231 self.tar.extractfile(regtype) as b:
232 self.assertEqual(a.name, b.name)
233
234 def test_fileobj_link1(self):
235 self._test_fileobj_link("ustar/lnktype", "ustar/regtype")
236
237 def test_fileobj_link2(self):
238 self._test_fileobj_link("./ustar/linktest2/lnktype",
239 "ustar/linktest1/regtype")
240
241 def test_fileobj_symlink1(self):
242 self._test_fileobj_link("ustar/symtype", "ustar/regtype")
243
244 def test_fileobj_symlink2(self):
245 self._test_fileobj_link("./ustar/linktest2/symtype",
246 "ustar/linktest1/regtype")
247
248 def test_issue14160(self):
249 self._test_fileobj_link("symtype2", "ustar/regtype")
250
251 def test_add_dir_getmember(self):
252 # bpo-21987
253 self.add_dir_and_getmember('bar')
254 self.add_dir_and_getmember('a'*101)
255
256 @unittest.skipUnless(hasattr(os, "getuid") and hasattr(os, "getgid"),
257 "Missing getuid or getgid implementation")
258 def add_dir_and_getmember(self, name):
259 def filter(tarinfo):
260 tarinfo.uid = tarinfo.gid = 100
261 return tarinfo
262
263 with os_helper.temp_cwd():
264 with tarfile.open(tmpname, 'w') as tar:
265 tar.format = tarfile.USTAR_FORMAT
266 try:
267 os.mkdir(name)
268 tar.add(name, filter=filter)
269 finally:
270 os.rmdir(name)
271 with tarfile.open(tmpname) as tar:
272 self.assertEqual(
273 tar.getmember(name),
274 tar.getmember(name + '/')
275 )
276
277 class GzipUstarReadTest(GzipTest, UstarReadTest):
278 pass
279
280 class Bz2UstarReadTest(Bz2Test, UstarReadTest):
281 pass
282
283 class LzmaUstarReadTest(LzmaTest, UstarReadTest):
284 pass
285
286 class ZstdUstarReadTest(ZstdTest, UstarReadTest):
287 pass
288
289 class ListTest(ReadTest, unittest.TestCase):
290
291 # Override setUp to use default encoding (UTF-8)
292 def setUp(self):
293 self.tar = tarfile.open(self.tarname, mode=self.mode)
294
295 def test_list(self):
296 tio = io.TextIOWrapper(io.BytesIO(), 'ascii', newline='\n')
297 with support.swap_attr(sys, 'stdout', tio):
298 self.tar.list(verbose=False)
299 out = tio.detach().getvalue()
300 self.assertIn(b'ustar/conttype', out)
301 self.assertIn(b'ustar/regtype', out)
302 self.assertIn(b'ustar/lnktype', out)
303 self.assertIn(b'ustar' + (b'/12345' * 40) + b'67/longname', out)
304 self.assertIn(b'./ustar/linktest2/symtype', out)
305 self.assertIn(b'./ustar/linktest2/lnktype', out)
306 # Make sure it puts trailing slash for directory
307 self.assertIn(b'ustar/dirtype/', out)
308 self.assertIn(b'ustar/dirtype-with-size/', out)
309 # Make sure it is able to print unencodable characters
310 def conv(b):
311 s = b.decode(self.tar.encoding, 'surrogateescape')
312 return s.encode('ascii', 'backslashreplace')
313 self.assertIn(conv(b'ustar/umlauts-\xc4\xd6\xdc\xe4\xf6\xfc\xdf'), out)
314 self.assertIn(conv(b'misc/regtype-hpux-signed-chksum-'
315 b'\xc4\xd6\xdc\xe4\xf6\xfc\xdf'), out)
316 self.assertIn(conv(b'misc/regtype-old-v7-signed-chksum-'
317 b'\xc4\xd6\xdc\xe4\xf6\xfc\xdf'), out)
318 self.assertIn(conv(b'pax/bad-pax-\xe4\xf6\xfc'), out)
319 self.assertIn(conv(b'pax/hdrcharset-\xe4\xf6\xfc'), out)
320 # Make sure it prints files separated by one newline without any
321 # 'ls -l'-like accessories if verbose flag is not being used
322 # ...
323 # ustar/conttype
324 # ustar/regtype
325 # ...
326 self.assertRegex(out, br'ustar/conttype ?\r?\n'
327 br'ustar/regtype ?\r?\n')
328 # Make sure it does not print the source of link without verbose flag
329 self.assertNotIn(b'link to', out)
330 self.assertNotIn(b'->', out)
331
332 def test_list_verbose(self):
333 tio = io.TextIOWrapper(io.BytesIO(), 'ascii', newline='\n')
334 with support.swap_attr(sys, 'stdout', tio):
335 self.tar.list(verbose=True)
336 out = tio.detach().getvalue()
337 # Make sure it prints files separated by one newline with 'ls -l'-like
338 # accessories if verbose flag is being used
339 # ...
340 # ?rw-r--r-- tarfile/tarfile 7011 2003-01-06 07:19:43 ustar/conttype
341 # -rw-r--r-- tarfile/tarfile 7011 2003-01-06 07:19:43 ustar/regtype
342 # drwxr-xr-x tarfile/tarfile 0 2003-01-05 15:19:43 ustar/dirtype/
343 # ...
344 #
345 # Array of values to modify the regex below:
346 # ((file_type, file_permissions, file_length), ...)
347 type_perm_lengths = (
348 (br'\?', b'rw-r--r--', b'7011'), (b'-', b'rw-r--r--', b'7011'),
349 (b'd', b'rwxr-xr-x', b'0'), (b'd', b'rwxr-xr-x', b'255'),
350 (br'\?', b'rw-r--r--', b'0'), (b'l', b'rwxrwxrwx', b'0'),
351 (b'b', b'rw-rw----', b'3,0'), (b'c', b'rw-rw-rw-', b'1,3'),
352 (b'p', b'rw-r--r--', b'0'))
353 self.assertRegex(out, b''.join(
354 [(tp + (br'%s tarfile/tarfile\s+%s ' % (perm, ln) +
355 br'\d{4}-\d\d-\d\d\s+\d\d:\d\d:\d\d '
356 br'ustar/\w+type[/>\sa-z-]*\n')) for tp, perm, ln
357 in type_perm_lengths]))
358 # Make sure it prints the source of link with verbose flag
359 self.assertIn(b'ustar/symtype -> regtype', out)
360 self.assertIn(b'./ustar/linktest2/symtype -> ../linktest1/regtype', out)
361 self.assertIn(b'./ustar/linktest2/lnktype link to '
362 b'./ustar/linktest1/regtype', out)
363 self.assertIn(b'gnu' + (b'/123' * 125) + b'/longlink link to gnu' +
364 (b'/123' * 125) + b'/longname', out)
365 self.assertIn(b'pax' + (b'/123' * 125) + b'/longlink link to pax' +
366 (b'/123' * 125) + b'/longname', out)
367
368 def test_list_members(self):
369 tio = io.TextIOWrapper(io.BytesIO(), 'ascii', newline='\n')
370 def members(tar):
371 for tarinfo in tar.getmembers():
372 if 'reg' in tarinfo.name:
373 yield tarinfo
374 with support.swap_attr(sys, 'stdout', tio):
375 self.tar.list(verbose=False, members=members(self.tar))
376 out = tio.detach().getvalue()
377 self.assertIn(b'ustar/regtype', out)
378 self.assertNotIn(b'ustar/conttype', out)
379
380
381 class GzipListTest(GzipTest, ListTest):
382 pass
383
384
385 class Bz2ListTest(Bz2Test, ListTest):
386 pass
387
388
389 class LzmaListTest(LzmaTest, ListTest):
390 pass
391
392 class ZstdListTest(ZstdTest, ListTest):
393 pass
394
395 class CommonReadTest(ReadTest):
396
397 def test_is_tarfile_erroneous(self):
398 with open(tmpname, "wb"):
399 pass
400
401 # is_tarfile works on filenames
402 self.assertFalse(tarfile.is_tarfile(tmpname))
403
404 # is_tarfile works on path-like objects
405 self.assertFalse(tarfile.is_tarfile(os_helper.FakePath(tmpname)))
406
407 # is_tarfile works on file objects
408 with open(tmpname, "rb") as fobj:
409 self.assertFalse(tarfile.is_tarfile(fobj))
410
411 # is_tarfile works on file-like objects
412 self.assertFalse(tarfile.is_tarfile(io.BytesIO(b"invalid")))
413
414 def test_is_tarfile_valid(self):
415 # is_tarfile works on filenames
416 self.assertTrue(tarfile.is_tarfile(self.tarname))
417
418 # is_tarfile works on path-like objects
419 self.assertTrue(tarfile.is_tarfile(os_helper.FakePath(self.tarname)))
420
421 # is_tarfile works on file objects
422 with open(self.tarname, "rb") as fobj:
423 self.assertTrue(tarfile.is_tarfile(fobj))
424
425 # is_tarfile works on file-like objects
426 with open(self.tarname, "rb") as fobj:
427 self.assertTrue(tarfile.is_tarfile(io.BytesIO(fobj.read())))
428
429 def test_is_tarfile_keeps_position(self):
430 # Test for issue44289: tarfile.is_tarfile() modifies
431 # file object's current position
432 with open(self.tarname, "rb") as fobj:
433 tarfile.is_tarfile(fobj)
434 self.assertEqual(fobj.tell(), 0)
435
436 with open(self.tarname, "rb") as fobj:
437 file_like = io.BytesIO(fobj.read())
438 tarfile.is_tarfile(file_like)
439 self.assertEqual(file_like.tell(), 0)
440
441 def test_empty_tarfile(self):
442 # Test for issue6123: Allow opening empty archives.
443 # This test checks if tarfile.open() is able to open an empty tar
444 # archive successfully. Note that an empty tar archive is not the
445 # same as an empty file!
446 with tarfile.open(tmpname, self.mode.replace("r", "w")):
447 pass
448 try:
449 tar = tarfile.open(tmpname, self.mode)
450 tar.getnames()
451 except tarfile.ReadError:
452 self.fail("tarfile.open() failed on empty archive")
453 else:
454 self.assertListEqual(tar.getmembers(), [])
455 finally:
456 tar.close()
457
458 def test_non_existent_tarfile(self):
459 # Test for issue11513: prevent non-existent gzipped tarfiles raising
460 # multiple exceptions.
461 with self.assertRaisesRegex(FileNotFoundError, "xxx"):
462 tarfile.open("xxx", self.mode)
463
464 def test_null_tarfile(self):
465 # Test for issue6123: Allow opening empty archives.
466 # This test guarantees that tarfile.open() does not treat an empty
467 # file as an empty tar archive.
468 with open(tmpname, "wb"):
469 pass
470 self.assertRaises(tarfile.ReadError, tarfile.open, tmpname, self.mode)
471 self.assertRaises(tarfile.ReadError, tarfile.open, tmpname)
472
473 def test_ignore_zeros(self):
474 # Test TarFile's ignore_zeros option.
475 # generate 512 pseudorandom bytes
476 data = Random(0).randbytes(512)
477 for char in (b'\0', b'a'):
478 # Test if EOFHeaderError ('\0') and InvalidHeaderError ('a')
479 # are ignored correctly.
480 with self.open(tmpname, "w") as fobj:
481 fobj.write(char * 1024)
482 tarinfo = tarfile.TarInfo("foo")
483 tarinfo.size = len(data)
484 fobj.write(tarinfo.tobuf())
485 fobj.write(data)
486
487 tar = tarfile.open(tmpname, mode="r", ignore_zeros=True)
488 try:
489 self.assertListEqual(tar.getnames(), ["foo"],
490 "ignore_zeros=True should have skipped the %r-blocks" %
491 char)
492 finally:
493 tar.close()
494
495 def test_premature_end_of_archive(self):
496 for size in (512, 600, 1024, 1200):
497 with tarfile.open(tmpname, "w:") as tar:
498 t = tarfile.TarInfo("foo")
499 t.size = 1024
500 tar.addfile(t, io.BytesIO(b"a" * 1024))
501
502 with open(tmpname, "r+b") as fobj:
503 fobj.truncate(size)
504
505 with tarfile.open(tmpname) as tar:
506 with self.assertRaisesRegex(tarfile.ReadError, "unexpected end of data"):
507 for t in tar:
508 pass
509
510 with tarfile.open(tmpname) as tar:
511 t = tar.next()
512
513 with self.assertRaisesRegex(tarfile.ReadError, "unexpected end of data"):
514 tar.extract(t, TEMPDIR, filter='data')
515
516 with self.assertRaisesRegex(tarfile.ReadError, "unexpected end of data"):
517 tar.extractfile(t).read()
518
519 def test_length_zero_header(self):
520 # bpo-39017 (CVE-2019-20907): reading a zero-length header should fail
521 # with an exception
522 with self.assertRaisesRegex(tarfile.ReadError, "file could not be opened successfully"):
523 with tarfile.open(support.findfile('recursion.tar', subdir='archivetestdata')):
524 pass
525
526 def test_extractfile_attrs(self):
527 # gh-74468: TarFile.name must name a file, not a parent archive.
528 file = self.tar.getmember('ustar/regtype')
529 with self.tar.extractfile(file) as fobj:
530 self.assertEqual(fobj.name, 'ustar/regtype')
531 self.assertRaises(AttributeError, fobj.fileno)
532 self.assertEqual(fobj.mode, 'rb')
533 self.assertIs(fobj.readable(), True)
534 self.assertIs(fobj.writable(), False)
535 if self.is_stream:
536 self.assertRaises(AttributeError, fobj.seekable)
537 else:
538 self.assertIs(fobj.seekable(), True)
539 self.assertIs(fobj.closed, False)
540 self.assertIs(fobj.closed, True)
541 self.assertEqual(fobj.name, 'ustar/regtype')
542 self.assertRaises(AttributeError, fobj.fileno)
543 self.assertEqual(fobj.mode, 'rb')
544 self.assertIs(fobj.readable(), True)
545 self.assertIs(fobj.writable(), False)
546 if self.is_stream:
547 self.assertRaises(AttributeError, fobj.seekable)
548 else:
549 self.assertIs(fobj.seekable(), True)
550
551
552 class MiscReadTestBase(CommonReadTest):
553 is_stream = False
554
555 def test_no_name_argument(self):
556 with open(self.tarname, "rb") as fobj:
557 self.assertIsInstance(fobj.name, str)
558 with tarfile.open(fileobj=fobj, mode=self.mode) as tar:
559 self.assertIsInstance(tar.name, str)
560 self.assertEqual(tar.name, os.path.abspath(fobj.name))
561
562 def test_no_name_attribute(self):
563 with open(self.tarname, "rb") as fobj:
564 data = fobj.read()
565 fobj = io.BytesIO(data)
566 self.assertRaises(AttributeError, getattr, fobj, "name")
567 tar = tarfile.open(fileobj=fobj, mode=self.mode)
568 self.assertIsNone(tar.name)
569
570 def test_empty_name_attribute(self):
571 with open(self.tarname, "rb") as fobj:
572 data = fobj.read()
573 fobj = io.BytesIO(data)
574 fobj.name = ""
575 with tarfile.open(fileobj=fobj, mode=self.mode) as tar:
576 self.assertIsNone(tar.name)
577
578 def test_int_name_attribute(self):
579 # Issue 21044: tarfile.open() should handle fileobj with an integer
580 # 'name' attribute.
581 fd = os.open(self.tarname, os.O_RDONLY)
582 with open(fd, 'rb') as fobj:
583 self.assertIsInstance(fobj.name, int)
584 with tarfile.open(fileobj=fobj, mode=self.mode) as tar:
585 self.assertIsNone(tar.name)
586
587 def test_bytes_name_attribute(self):
588 tarname = os.fsencode(self.tarname)
589 with open(tarname, 'rb') as fobj:
590 self.assertIsInstance(fobj.name, bytes)
591 with tarfile.open(fileobj=fobj, mode=self.mode) as tar:
592 self.assertIsInstance(tar.name, bytes)
593 self.assertEqual(tar.name, os.path.abspath(fobj.name))
594
595 def test_pathlike_name(self, tarname=None):
596 if tarname is None:
597 tarname = self.tarname
598 expected = os.path.abspath(tarname)
599 tarname = os_helper.FakePath(tarname)
600 with tarfile.open(tarname, mode=self.mode) as tar:
601 self.assertEqual(tar.name, expected)
602 with self.taropen(tarname) as tar:
603 self.assertEqual(tar.name, expected)
604 with tarfile.TarFile.open(tarname, mode=self.mode) as tar:
605 self.assertEqual(tar.name, expected)
606 if self.suffix == '':
607 with tarfile.TarFile(tarname, mode='r') as tar:
608 self.assertEqual(tar.name, expected)
609
610 def test_pathlike_bytes_name(self):
611 self.test_pathlike_name(os.fsencode(self.tarname))
612
613 def test_illegal_mode_arg(self):
614 with open(tmpname, 'wb'):
615 pass
616 with self.assertRaisesRegex(ValueError, 'mode must be '):
617 tar = self.taropen(tmpname, 'q')
618 with self.assertRaisesRegex(ValueError, 'mode must be '):
619 tar = self.taropen(tmpname, 'rw')
620 with self.assertRaisesRegex(ValueError, 'mode must be '):
621 tar = self.taropen(tmpname, '')
622
623 def test_fileobj_with_offset(self):
624 # Skip the first member and store values from the second member
625 # of the testtar.
626 tar = tarfile.open(self.tarname, mode=self.mode)
627 try:
628 tar.next()
629 t = tar.next()
630 name = t.name
631 offset = t.offset
632 with tar.extractfile(t) as f:
633 data = f.read()
634 finally:
635 tar.close()
636
637 # Open the testtar and seek to the offset of the second member.
638 with self.open(self.tarname) as fobj:
639 fobj.seek(offset)
640
641 # Test if the tarfile starts with the second member.
642 with tar.open(self.tarname, mode="r:", fileobj=fobj) as tar:
643 t = tar.next()
644 self.assertEqual(t.name, name)
645 # Read to the end of fileobj and test if seeking back to the
646 # beginning works.
647 tar.getmembers()
648 self.assertEqual(tar.extractfile(t).read(), data,
649 "seek back did not work")
650
651 def test_fail_comp(self):
652 # For Gzip and Bz2 Tests: fail with a ReadError on an uncompressed file.
653 self.assertRaises(tarfile.ReadError, tarfile.open, tarname, self.mode)
654 with open(tarname, "rb") as fobj:
655 self.assertRaises(tarfile.ReadError, tarfile.open,
656 fileobj=fobj, mode=self.mode)
657
658 def test_v7_dirtype(self):
659 # Test old style dirtype member (bug #1336623):
660 # Old V7 tars create directory members using an AREGTYPE
661 # header with a "/" appended to the filename field.
662 tarinfo = self.tar.getmember("misc/dirtype-old-v7")
663 self.assertEqual(tarinfo.type, tarfile.DIRTYPE,
664 "v7 dirtype failed")
665
666 def test_xstar_type(self):
667 # The xstar format stores extra atime and ctime fields inside the
668 # space reserved for the prefix field. The prefix field must be
669 # ignored in this case, otherwise it will mess up the name.
670 try:
671 self.tar.getmember("misc/regtype-xstar")
672 except KeyError:
673 self.fail("failed to find misc/regtype-xstar (mangled prefix?)")
674
675 def test_check_members(self):
676 for tarinfo in self.tar:
677 self.assertEqual(int(tarinfo.mtime), 0o7606136617,
678 "wrong mtime for %s" % tarinfo.name)
679 if not tarinfo.name.startswith("ustar/"):
680 continue
681 self.assertEqual(tarinfo.uname, "tarfile",
682 "wrong uname for %s" % tarinfo.name)
683
684 def test_find_members(self):
685 self.assertEqual(self.tar.getmembers()[-1].name, "misc/eof",
686 "could not find all members")
687
688 @unittest.skipUnless(hasattr(os, "link"),
689 "Missing hardlink implementation")
690 @os_helper.skip_unless_symlink
691 def test_extract_hardlink(self):
692 # Test hardlink extraction (e.g. bug #857297).
693 with tarfile.open(tarname, errorlevel=1, encoding="iso8859-1") as tar:
694 tar.extract("ustar/regtype", TEMPDIR, filter='data')
695 self.addCleanup(os_helper.unlink, os.path.join(TEMPDIR, "ustar/regtype"))
696
697 tar.extract("ustar/lnktype", TEMPDIR, filter='data')
698 self.addCleanup(os_helper.unlink, os.path.join(TEMPDIR, "ustar/lnktype"))
699 with open(os.path.join(TEMPDIR, "ustar/lnktype"), "rb") as f:
700 data = f.read()
701 self.assertEqual(sha256sum(data), sha256_regtype)
702
703 tar.extract("ustar/symtype", TEMPDIR, filter='data')
704 self.addCleanup(os_helper.unlink, os.path.join(TEMPDIR, "ustar/symtype"))
705 with open(os.path.join(TEMPDIR, "ustar/symtype"), "rb") as f:
706 data = f.read()
707 self.assertEqual(sha256sum(data), sha256_regtype)
708
709 @os_helper.skip_unless_working_chmod
710 def test_extractall(self):
711 # Test if extractall() correctly restores directory permissions
712 # and times (see issue1735).
713 tar = tarfile.open(tarname, encoding="iso8859-1")
714 DIR = os.path.join(TEMPDIR, "extractall")
715 os.mkdir(DIR)
716 try:
717 directories = [t for t in tar if t.isdir()]
718 tar.extractall(DIR, directories, filter='fully_trusted')
719 for tarinfo in directories:
720 path = os.path.join(DIR, tarinfo.name)
721 if sys.platform != "win32":
722 # Win32 has no support for fine grained permissions.
723 self.assertEqual(tarinfo.mode & 0o777,
724 os.stat(path).st_mode & 0o777,
725 tarinfo.name)
726 def format_mtime(mtime):
727 if isinstance(mtime, float):
728 return "{} ({})".format(mtime, mtime.hex())
729 else:
730 return "{!r} (int)".format(mtime)
731 file_mtime = os.path.getmtime(path)
732 errmsg = "tar mtime {0} != file time {1} of path {2!a}".format(
733 format_mtime(tarinfo.mtime),
734 format_mtime(file_mtime),
735 path)
736 self.assertEqual(tarinfo.mtime, file_mtime, errmsg)
737 finally:
738 tar.close()
739 os_helper.rmtree(DIR)
740
741 @staticmethod
742 def test_extractall_default_filter():
743 # Test that the default filter is now "data", and the other filter types are not used.
744 DIR = pathlib.Path(TEMPDIR) / "extractall_default_filter"
745 with (
746 os_helper.temp_dir(DIR),
747 tarfile.open(tarname, encoding="iso8859-1") as tar,
748 unittest.mock.patch("tarfile.data_filter", wraps=tarfile.data_filter) as mock_data_filter,
749 unittest.mock.patch("tarfile.tar_filter", wraps=tarfile.tar_filter) as mock_tar_filter,
750 unittest.mock.patch("tarfile.fully_trusted_filter", wraps=tarfile.fully_trusted_filter) as mock_ft_filter
751 ):
752 directories = [t for t in tar if t.isdir()]
753 tar.extractall(DIR, directories)
754
755 mock_data_filter.assert_called()
756 mock_ft_filter.assert_not_called()
757 mock_tar_filter.assert_not_called()
758
759 @os_helper.skip_unless_working_chmod
760 def test_extract_directory(self):
761 dirtype = "ustar/dirtype"
762 DIR = os.path.join(TEMPDIR, "extractdir")
763 os.mkdir(DIR)
764 try:
765 with tarfile.open(tarname, encoding="iso8859-1") as tar:
766 tarinfo = tar.getmember(dirtype)
767 tar.extract(tarinfo, path=DIR, filter='fully_trusted')
768 extracted = os.path.join(DIR, dirtype)
769 self.assertEqual(os.path.getmtime(extracted), tarinfo.mtime)
770 if sys.platform != "win32":
771 self.assertEqual(os.stat(extracted).st_mode & 0o777, 0o755)
772 finally:
773 os_helper.rmtree(DIR)
774
775 def test_extractall_pathlike_dir(self):
776 DIR = os.path.join(TEMPDIR, "extractall")
777 with os_helper.temp_dir(DIR), \
778 tarfile.open(tarname, encoding="iso8859-1") as tar:
779 directories = [t for t in tar if t.isdir()]
780 tar.extractall(os_helper.FakePath(DIR), directories, filter='fully_trusted')
781 for tarinfo in directories:
782 path = os.path.join(DIR, tarinfo.name)
783 self.assertEqual(os.path.getmtime(path), tarinfo.mtime)
784
785 def test_extract_pathlike_dir(self):
786 dirtype = "ustar/dirtype"
787 DIR = os.path.join(TEMPDIR, "extractall")
788 with os_helper.temp_dir(DIR), \
789 tarfile.open(tarname, encoding="iso8859-1") as tar:
790 tarinfo = tar.getmember(dirtype)
791 tar.extract(tarinfo, path=os_helper.FakePath(DIR), filter='fully_trusted')
792 extracted = os.path.join(DIR, dirtype)
793 self.assertEqual(os.path.getmtime(extracted), tarinfo.mtime)
794
795 def test_init_close_fobj(self):
796 # Issue #7341: Close the internal file object in the TarFile
797 # constructor in case of an error. For the test we rely on
798 # the fact that opening an empty file raises a ReadError.
799 empty = os.path.join(TEMPDIR, "empty")
800 with open(empty, "wb") as fobj:
801 fobj.write(b"")
802
803 try:
804 tar = object.__new__(tarfile.TarFile)
805 try:
806 tar.__init__(empty)
807 except tarfile.ReadError:
808 self.assertTrue(tar.fileobj.closed)
809 else:
810 self.fail("ReadError not raised")
811 finally:
812 os_helper.unlink(empty)
813
814 def test_parallel_iteration(self):
815 # Issue #16601: Restarting iteration over tarfile continued
816 # from where it left off.
817 with tarfile.open(self.tarname) as tar:
818 for m1, m2 in zip(tar, tar):
819 self.assertEqual(m1.offset, m2.offset)
820 self.assertEqual(m1.get_info(), m2.get_info())
821
822 @unittest.skipIf(zlib is None, "requires zlib")
823 def test_zlib_error_does_not_leak(self):
824 # bpo-39039: tarfile.open allowed zlib exceptions to bubble up when
825 # parsing certain types of invalid data
826 with unittest.mock.patch("tarfile.TarInfo.fromtarfile") as mock:
827 mock.side_effect = zlib.error
828 with self.assertRaises(tarfile.ReadError):
829 tarfile.open(self.tarname)
830
831 def test_next_on_empty_tarfile(self):
832 fd = io.BytesIO()
833 tf = tarfile.open(fileobj=fd, mode="w")
834 tf.close()
835
836 fd.seek(0)
837 with tarfile.open(fileobj=fd, mode="r|") as tf:
838 self.assertEqual(tf.next(), None)
839
840 fd.seek(0)
841 with tarfile.open(fileobj=fd, mode="r") as tf:
842 self.assertEqual(tf.next(), None)
843
844 def _setup_symlink_to_target(self, temp_dirpath):
845 target_filepath = os.path.join(temp_dirpath, "target")
846 ustar_dirpath = os.path.join(temp_dirpath, "ustar")
847 hardlink_filepath = os.path.join(ustar_dirpath, "lnktype")
848 with open(target_filepath, "wb") as f:
849 f.write(b"target")
850 os.makedirs(ustar_dirpath)
851 os.symlink(target_filepath, hardlink_filepath)
852 return target_filepath, hardlink_filepath
853
854 def _assert_on_file_content(self, filepath, digest):
855 with open(filepath, "rb") as f:
856 data = f.read()
857 self.assertEqual(sha256sum(data), digest)
858
859 @unittest.skipUnless(
860 hasattr(os, "link"), "Missing hardlink implementation"
861 )
862 @os_helper.skip_unless_symlink
863 def test_extract_hardlink_on_symlink(self):
864 """
865 This test verifies that extracting a hardlink will not follow an
866 existing symlink after a FileExistsError on os.link.
867 """
868 with os_helper.temp_dir() as DIR:
869 target_filepath, hardlink_filepath = self._setup_symlink_to_target(DIR)
870 with tarfile.open(tarname, encoding="iso8859-1") as tar:
871 tar.extract("ustar/regtype", DIR, filter="data")
872 tar.extract("ustar/lnktype", DIR, filter="data")
873 self._assert_on_file_content(target_filepath, sha256sum(b"target"))
874 self._assert_on_file_content(hardlink_filepath, sha256_regtype)
875
876 @unittest.skipUnless(
877 hasattr(os, "link"), "Missing hardlink implementation"
878 )
879 @os_helper.skip_unless_symlink
880 def test_extractall_hardlink_on_symlink(self):
881 """
882 This test verifies that extracting a hardlink will not follow an
883 existing symlink after a FileExistsError on os.link.
884 """
885 with os_helper.temp_dir() as DIR:
886 target_filepath, hardlink_filepath = self._setup_symlink_to_target(DIR)
887 with tarfile.open(tarname, encoding="iso8859-1") as tar:
888 tar.extractall(
889 DIR, members=["ustar/regtype", "ustar/lnktype"], filter="data",
890 )
891 self._assert_on_file_content(target_filepath, sha256sum(b"target"))
892 self._assert_on_file_content(hardlink_filepath, sha256_regtype)
893
894
895 class GzipReadTestBase:
896
897 def test_read_with_extra_field(self):
898 with open(self.tarname, 'rb') as f:
899 data = bytearray(f.read())
900 flags = data[3]
901 self.assertEqual(flags, 8)
902 data[3] = flags | 4
903 data[10:10] = b'\x05\x00extra'
904 with open(tmpname, 'wb') as f:
905 f.write(data)
906 print(self.mode)
907 with tarfile.open(tmpname, mode=self.mode):
908 pass
909
910 def test_read_with_file_comment(self):
911 with open(self.tarname, 'rb') as f:
912 data = bytearray(f.read())
913 flags = data[3]
914 self.assertEqual(flags, 8)
915 data[3] = flags | 16
916 i = data.index(0, 10) + 1
917 data[i:i] = b'comment\x00'
918 with open(tmpname, 'wb') as f:
919 f.write(data)
920 with tarfile.open(tmpname, mode=self.mode):
921 pass
922
923
924 class MiscReadTest(MiscReadTestBase, unittest.TestCase):
925 test_fail_comp = None
926
927 class GzipMiscReadTest(GzipTest, GzipReadTestBase, MiscReadTestBase, unittest.TestCase):
928 pass
929
930 class Bz2MiscReadTest(Bz2Test, MiscReadTestBase, unittest.TestCase):
931 pass
932
933 class LzmaMiscReadTest(LzmaTest, MiscReadTestBase, unittest.TestCase):
934 pass
935
936 class ZstdMiscReadTest(ZstdTest, MiscReadTestBase, unittest.TestCase):
937 pass
938
939 class StreamReadTest(CommonReadTest, unittest.TestCase):
940
941 prefix="r|"
942 is_stream = True
943
944 def test_read_through(self):
945 # Issue #11224: A poorly designed _FileInFile.read() method
946 # caused seeking errors with stream tar files.
947 for tarinfo in self.tar:
948 if not tarinfo.isreg():
949 continue
950 with self.tar.extractfile(tarinfo) as fobj:
951 while True:
952 try:
953 buf = fobj.read(512)
954 except tarfile.StreamError:
955 self.fail("simple read-through using "
956 "TarFile.extractfile() failed")
957 if not buf:
958 break
959
960 def test_fileobj_regular_file(self):
961 tarinfo = self.tar.next() # get "regtype" (can't use getmember)
962 with self.tar.extractfile(tarinfo) as fobj:
963 data = fobj.read()
964 self.assertEqual(len(data), tarinfo.size,
965 "regular file extraction failed")
966 self.assertEqual(sha256sum(data), sha256_regtype,
967 "regular file extraction failed")
968
969 def test_provoke_stream_error(self):
970 tarinfos = self.tar.getmembers()
971 with self.tar.extractfile(tarinfos[0]) as f: # read the first member
972 self.assertRaises(tarfile.StreamError, f.read)
973
974 def test_compare_members(self):
975 tar1 = tarfile.open(tarname, encoding="iso8859-1")
976 try:
977 tar2 = self.tar
978
979 while True:
980 t1 = tar1.next()
981 t2 = tar2.next()
982 if t1 is None:
983 break
984 self.assertIsNotNone(t2, "stream.next() failed.")
985
986 if t2.islnk() or t2.issym():
987 with self.assertRaises(tarfile.StreamError):
988 tar2.extractfile(t2)
989 continue
990
991 v1 = tar1.extractfile(t1)
992 v2 = tar2.extractfile(t2)
993 if v1 is None:
994 continue
995 self.assertIsNotNone(v2, "stream.extractfile() failed")
996 self.assertEqual(v1.read(), v2.read(),
997 "stream extraction failed")
998 finally:
999 tar1.close()
1000
1001 class GzipStreamReadTest(GzipTest, GzipReadTestBase, StreamReadTest):
1002 pass
1003
1004 class Bz2StreamReadTest(Bz2Test, StreamReadTest):
1005 pass
1006
1007 class LzmaStreamReadTest(LzmaTest, StreamReadTest):
1008 pass
1009
1010 class ZstdStreamReadTest(ZstdTest, StreamReadTest):
1011 pass
1012
1013 class TarStreamModeReadTest(StreamModeTest, unittest.TestCase):
1014
1015 def test_stream_mode_no_cache(self):
1016 for _ in self.tar:
1017 pass
1018 self.assertEqual(self.tar.members, [])
1019
1020 class GzipStreamModeReadTest(GzipTest, TarStreamModeReadTest):
1021 pass
1022
1023 class Bz2StreamModeReadTest(Bz2Test, TarStreamModeReadTest):
1024 pass
1025
1026 class LzmaStreamModeReadTest(LzmaTest, TarStreamModeReadTest):
1027 pass
1028
1029 class ZstdStreamModeReadTest(ZstdTest, TarStreamModeReadTest):
1030 pass
1031
1032 class DetectReadTest(TarTest, unittest.TestCase):
1033 def _testfunc_file(self, name, mode):
1034 try:
1035 tar = tarfile.open(name, mode)
1036 except tarfile.ReadError as e:
1037 self.fail()
1038 else:
1039 tar.close()
1040
1041 def _testfunc_fileobj(self, name, mode):
1042 try:
1043 with open(name, "rb") as f:
1044 tar = tarfile.open(name, mode, fileobj=f)
1045 except tarfile.ReadError as e:
1046 self.fail()
1047 else:
1048 tar.close()
1049
1050 def _test_modes(self, testfunc):
1051 if self.suffix:
1052 with self.assertRaises(tarfile.ReadError):
1053 tarfile.open(tarname, mode="r:" + self.suffix)
1054 with self.assertRaises(tarfile.ReadError):
1055 tarfile.open(tarname, mode="r|" + self.suffix)
1056 with self.assertRaises(tarfile.ReadError):
1057 tarfile.open(self.tarname, mode="r:")
1058 with self.assertRaises(tarfile.ReadError):
1059 tarfile.open(self.tarname, mode="r|")
1060 testfunc(self.tarname, "r")
1061 testfunc(self.tarname, "r:" + self.suffix)
1062 testfunc(self.tarname, "r:*")
1063 testfunc(self.tarname, "r|" + self.suffix)
1064 testfunc(self.tarname, "r|*")
1065
1066 def test_detect_file(self):
1067 self._test_modes(self._testfunc_file)
1068
1069 def test_detect_fileobj(self):
1070 self._test_modes(self._testfunc_fileobj)
1071
1072 class GzipDetectReadTest(GzipTest, DetectReadTest):
1073 pass
1074
1075 class Bz2DetectReadTest(Bz2Test, DetectReadTest):
1076 def test_detect_stream_bz2(self):
1077 # Originally, tarfile's stream detection looked for the string
1078 # "BZh91" at the start of the file. This is incorrect because
1079 # the '9' represents the blocksize (900,000 bytes). If the file was
1080 # compressed using another blocksize autodetection fails.
1081 with open(tarname, "rb") as fobj:
1082 data = fobj.read()
1083
1084 # Compress with blocksize 100,000 bytes, the file starts with "BZh11".
1085 with bz2.BZ2File(tmpname, "wb", compresslevel=1) as fobj:
1086 fobj.write(data)
1087
1088 self._testfunc_file(tmpname, "r|*")
1089
1090 class LzmaDetectReadTest(LzmaTest, DetectReadTest):
1091 pass
1092
1093 class ZstdDetectReadTest(ZstdTest, DetectReadTest):
1094 pass
1095
1096 class GzipBrokenHeaderCorrectException(GzipTest, unittest.TestCase):
1097 """
1098 See: https://github.com/python/cpython/issues/107396
1099 """
1100 def runTest(self):
1101 f = io.BytesIO(
1102 b'\x1f\x8b' # header
1103 b'\x08' # compression method
1104 b'\x04' # flags
1105 b'\0\0\0\0\0\0' # timestamp, compression data, OS ID
1106 b'\0\x01' # size
1107 b'\0\0\0\0\0' # corrupt data (zeros)
1108 )
1109 with self.assertRaises(tarfile.ReadError):
1110 tarfile.open(fileobj=f, mode='r|gz')
1111
1112
1113 class MemberReadTest(ReadTest, unittest.TestCase):
1114
1115 def _test_member(self, tarinfo, chksum=None, **kwargs):
1116 if chksum is not None:
1117 with self.tar.extractfile(tarinfo) as f:
1118 self.assertEqual(sha256sum(f.read()), chksum,
1119 "wrong sha256sum for %s" % tarinfo.name)
1120
1121 kwargs["mtime"] = 0o7606136617
1122 kwargs["uid"] = 1000
1123 kwargs["gid"] = 100
1124 if "old-v7" not in tarinfo.name:
1125 # V7 tar can't handle alphabetic owners.
1126 kwargs["uname"] = "tarfile"
1127 kwargs["gname"] = "tarfile"
1128 for k, v in kwargs.items():
1129 self.assertEqual(getattr(tarinfo, k), v,
1130 "wrong value in %s field of %s" % (k, tarinfo.name))
1131
1132 def test_find_regtype(self):
1133 tarinfo = self.tar.getmember("ustar/regtype")
1134 self._test_member(tarinfo, size=7011, chksum=sha256_regtype)
1135
1136 def test_find_conttype(self):
1137 tarinfo = self.tar.getmember("ustar/conttype")
1138 self._test_member(tarinfo, size=7011, chksum=sha256_regtype)
1139
1140 def test_find_dirtype(self):
1141 tarinfo = self.tar.getmember("ustar/dirtype")
1142 self._test_member(tarinfo, size=0)
1143
1144 def test_find_dirtype_with_size(self):
1145 tarinfo = self.tar.getmember("ustar/dirtype-with-size")
1146 self._test_member(tarinfo, size=255)
1147
1148 def test_find_lnktype(self):
1149 tarinfo = self.tar.getmember("ustar/lnktype")
1150 self._test_member(tarinfo, size=0, linkname="ustar/regtype")
1151
1152 def test_find_symtype(self):
1153 tarinfo = self.tar.getmember("ustar/symtype")
1154 self._test_member(tarinfo, size=0, linkname="regtype")
1155
1156 def test_find_blktype(self):
1157 tarinfo = self.tar.getmember("ustar/blktype")
1158 self._test_member(tarinfo, size=0, devmajor=3, devminor=0)
1159
1160 def test_find_chrtype(self):
1161 tarinfo = self.tar.getmember("ustar/chrtype")
1162 self._test_member(tarinfo, size=0, devmajor=1, devminor=3)
1163
1164 def test_find_fifotype(self):
1165 tarinfo = self.tar.getmember("ustar/fifotype")
1166 self._test_member(tarinfo, size=0)
1167
1168 def test_find_sparse(self):
1169 tarinfo = self.tar.getmember("ustar/sparse")
1170 self._test_member(tarinfo, size=86016, chksum=sha256_sparse)
1171
1172 def test_find_gnusparse(self):
1173 tarinfo = self.tar.getmember("gnu/sparse")
1174 self._test_member(tarinfo, size=86016, chksum=sha256_sparse)
1175
1176 def test_find_gnusparse_00(self):
1177 tarinfo = self.tar.getmember("gnu/sparse-0.0")
1178 self._test_member(tarinfo, size=86016, chksum=sha256_sparse)
1179
1180 def test_find_gnusparse_01(self):
1181 tarinfo = self.tar.getmember("gnu/sparse-0.1")
1182 self._test_member(tarinfo, size=86016, chksum=sha256_sparse)
1183
1184 def test_find_gnusparse_10(self):
1185 tarinfo = self.tar.getmember("gnu/sparse-1.0")
1186 self._test_member(tarinfo, size=86016, chksum=sha256_sparse)
1187
1188 def test_find_umlauts(self):
1189 tarinfo = self.tar.getmember("ustar/umlauts-"
1190 "\xc4\xd6\xdc\xe4\xf6\xfc\xdf")
1191 self._test_member(tarinfo, size=7011, chksum=sha256_regtype)
1192
1193 def test_find_ustar_longname(self):
1194 name = "ustar/" + "12345/" * 39 + "1234567/longname"
1195 self.assertIn(name, self.tar.getnames())
1196
1197 def test_find_regtype_oldv7(self):
1198 tarinfo = self.tar.getmember("misc/regtype-old-v7")
1199 self._test_member(tarinfo, size=7011, chksum=sha256_regtype)
1200
1201 def test_find_pax_umlauts(self):
1202 self.tar.close()
1203 self.tar = tarfile.open(self.tarname, mode=self.mode,
1204 encoding="iso8859-1")
1205 tarinfo = self.tar.getmember("pax/umlauts-"
1206 "\xc4\xd6\xdc\xe4\xf6\xfc\xdf")
1207 self._test_member(tarinfo, size=7011, chksum=sha256_regtype)
1208
1209
1210 class LongnameTest:
1211
1212 def test_read_longname(self):
1213 # Test reading of longname (bug #1471427).
1214 longname = self.subdir + "/" + "123/" * 125 + "longname"
1215 try:
1216 tarinfo = self.tar.getmember(longname)
1217 except KeyError:
1218 self.fail("longname not found")
1219 self.assertNotEqual(tarinfo.type, tarfile.DIRTYPE,
1220 "read longname as dirtype")
1221
1222 def test_read_longlink(self):
1223 longname = self.subdir + "/" + "123/" * 125 + "longname"
1224 longlink = self.subdir + "/" + "123/" * 125 + "longlink"
1225 try:
1226 tarinfo = self.tar.getmember(longlink)
1227 except KeyError:
1228 self.fail("longlink not found")
1229 self.assertEqual(tarinfo.linkname, longname, "linkname wrong")
1230
1231 def test_truncated_longname(self):
1232 longname = self.subdir + "/" + "123/" * 125 + "longname"
1233 tarinfo = self.tar.getmember(longname)
1234 offset = tarinfo.offset
1235 self.tar.fileobj.seek(offset)
1236 fobj = io.BytesIO(self.tar.fileobj.read(3 * 512))
1237 with self.assertRaises(tarfile.ReadError):
1238 tarfile.open(name="foo.tar", fileobj=fobj)
1239
1240 def test_header_offset(self):
1241 # Test if the start offset of the TarInfo object includes
1242 # the preceding extended header.
1243 longname = self.subdir + "/" + "123/" * 125 + "longname"
1244 offset = self.tar.getmember(longname).offset
1245 with open(tarname, "rb") as fobj:
1246 fobj.seek(offset)
1247 tarinfo = tarfile.TarInfo.frombuf(fobj.read(512),
1248 "iso8859-1", "strict")
1249 self.assertEqual(tarinfo.type, self.longnametype)
1250
1251 def test_longname_directory(self):
1252 # Test reading a longlink directory. Issue #47231.
1253 longdir = ('a' * 101) + '/'
1254 with os_helper.temp_cwd():
1255 with tarfile.open(tmpname, 'w') as tar:
1256 tar.format = self.format
1257 try:
1258 os.mkdir(longdir)
1259 tar.add(longdir)
1260 finally:
1261 os.rmdir(longdir.rstrip("/"))
1262 with tarfile.open(tmpname) as tar:
1263 self.assertIsNotNone(tar.getmember(longdir))
1264 self.assertIsNotNone(tar.getmember(longdir.removesuffix('/')))
1265
1266 def test_longname_file_not_directory(self):
1267 # Test reading a longname file and ensure it is not handled as a directory
1268 # Issue #141707
1269 buf = io.BytesIO()
1270 with tarfile.open(mode='w', fileobj=buf, format=self.format) as tar:
1271 ti = tarfile.TarInfo()
1272 ti.type = tarfile.AREGTYPE
1273 ti.name = ('a' * 99) + '/' + ('b' * 3)
1274 tar.addfile(ti)
1275
1276 expected = {t.name: t.type for t in tar.getmembers()}
1277
1278 buf.seek(0)
1279 with tarfile.open(mode='r', fileobj=buf) as tar:
1280 actual = {t.name: t.type for t in tar.getmembers()}
1281
1282 self.assertEqual(expected, actual)
1283
1284
1285 class GNUReadTest(LongnameTest, ReadTest, unittest.TestCase):
1286
1287 subdir = "gnu"
1288 longnametype = tarfile.GNUTYPE_LONGNAME
1289 format = tarfile.GNU_FORMAT
1290
1291 # Since 3.2 tarfile is supposed to accurately restore sparse members and
1292 # produce files with holes. This is what we actually want to test here.
1293 # Unfortunately, not all platforms/filesystems support sparse files, and
1294 # even on platforms that do it is non-trivial to make reliable assertions
1295 # about holes in files. Therefore, we first do one basic test which works
1296 # an all platforms, and after that a test that will work only on
1297 # platforms/filesystems that prove to support sparse files.
1298 def _test_sparse_file(self, name):
1299 self.tar.extract(name, TEMPDIR, filter='data')
1300 filename = os.path.join(TEMPDIR, name)
1301 with open(filename, "rb") as fobj:
1302 data = fobj.read()
1303 self.assertEqual(sha256sum(data), sha256_sparse,
1304 "wrong sha256sum for %s" % name)
1305
1306 if self._fs_supports_holes():
1307 s = os.stat(filename)
1308 self.assertLess(s.st_blocks * 512, s.st_size)
1309
1310 def test_sparse_file_old(self):
1311 self._test_sparse_file("gnu/sparse")
1312
1313 def test_sparse_file_00(self):
1314 self._test_sparse_file("gnu/sparse-0.0")
1315
1316 def test_sparse_file_01(self):
1317 self._test_sparse_file("gnu/sparse-0.1")
1318
1319 def test_sparse_file_10(self):
1320 self._test_sparse_file("gnu/sparse-1.0")
1321
1322 @staticmethod
1323 def _fs_supports_holes():
1324 # Return True if the platform knows the st_blocks stat attribute and
1325 # uses st_blocks units of 512 bytes, and if the filesystem is able to
1326 # store holes of 4 KiB in files.
1327 #
1328 # The function returns False if page size is larger than 4 KiB.
1329 # For example, ppc64 uses pages of 64 KiB.
1330 if sys.platform.startswith(("linux", "android")):
1331 # Linux evidentially has 512 byte st_blocks units.
1332 name = os.path.join(TEMPDIR, "sparse-test")
1333 with open(name, "wb") as fobj:
1334 # Seek to "punch a hole" of 4 KiB
1335 fobj.seek(4096)
1336 fobj.write(b'x' * 4096)
1337 fobj.truncate()
1338 s = os.stat(name)
1339 os_helper.unlink(name)
1340 return (s.st_blocks * 512 < s.st_size)
1341 else:
1342 return False
1343
1344
1345 class PaxReadTest(LongnameTest, ReadTest, unittest.TestCase):
1346
1347 subdir = "pax"
1348 longnametype = tarfile.XHDTYPE
1349 format = tarfile.PAX_FORMAT
1350
1351 def test_pax_global_headers(self):
1352 tar = tarfile.open(tarname, encoding="iso8859-1")
1353 try:
1354 tarinfo = tar.getmember("pax/regtype1")
1355 self.assertEqual(tarinfo.uname, "foo")
1356 self.assertEqual(tarinfo.gname, "bar")
1357 self.assertEqual(tarinfo.pax_headers.get("VENDOR.umlauts"),
1358 "\xc4\xd6\xdc\xe4\xf6\xfc\xdf")
1359
1360 tarinfo = tar.getmember("pax/regtype2")
1361 self.assertEqual(tarinfo.uname, "")
1362 self.assertEqual(tarinfo.gname, "bar")
1363 self.assertEqual(tarinfo.pax_headers.get("VENDOR.umlauts"),
1364 "\xc4\xd6\xdc\xe4\xf6\xfc\xdf")
1365
1366 tarinfo = tar.getmember("pax/regtype3")
1367 self.assertEqual(tarinfo.uname, "tarfile")
1368 self.assertEqual(tarinfo.gname, "tarfile")
1369 self.assertEqual(tarinfo.pax_headers.get("VENDOR.umlauts"),
1370 "\xc4\xd6\xdc\xe4\xf6\xfc\xdf")
1371 finally:
1372 tar.close()
1373
1374 def test_pax_number_fields(self):
1375 # All following number fields are read from the pax header.
1376 tar = tarfile.open(tarname, encoding="iso8859-1")
1377 try:
1378 tarinfo = tar.getmember("pax/regtype4")
1379 self.assertEqual(tarinfo.size, 7011)
1380 self.assertEqual(tarinfo.uid, 123)
1381 self.assertEqual(tarinfo.gid, 123)
1382 self.assertEqual(tarinfo.mtime, 1041808783.0)
1383 self.assertEqual(type(tarinfo.mtime), float)
1384 self.assertEqual(float(tarinfo.pax_headers["atime"]), 1041808783.0)
1385 self.assertEqual(float(tarinfo.pax_headers["ctime"]), 1041808783.0)
1386 finally:
1387 tar.close()
1388
1389 def test_pax_header_bad_formats(self):
1390 # The fields from the pax header have priority over the
1391 # TarInfo.
1392 pax_header_replacements = (
1393 b" foo=bar\n",
1394 b"0 \n",
1395 b"1 \n",
1396 b"2 \n",
1397 b"3 =\n",
1398 b"4 =a\n",
1399 b"1000000 foo=bar\n",
1400 b"0 foo=bar\n",
1401 b"-12 foo=bar\n",
1402 b"000000000000000000000000036 foo=bar\n",
1403 )
1404 pax_headers = {"foo": "bar"}
1405
1406 for replacement in pax_header_replacements:
1407 with self.subTest(header=replacement):
1408 tar = tarfile.open(tmpname, "w", format=tarfile.PAX_FORMAT,
1409 encoding="iso8859-1")
1410 try:
1411 t = tarfile.TarInfo()
1412 t.name = "pax" # non-ASCII
1413 t.uid = 1
1414 t.pax_headers = pax_headers
1415 tar.addfile(t)
1416 finally:
1417 tar.close()
1418
1419 with open(tmpname, "rb") as f:
1420 data = f.read()
1421 self.assertIn(b"11 foo=bar\n", data)
1422 data = data.replace(b"11 foo=bar\n", replacement)
1423
1424 with open(tmpname, "wb") as f:
1425 f.truncate()
1426 f.write(data)
1427
1428 with self.assertRaisesRegex(tarfile.ReadError, r"method tar: ReadError\('invalid header'\)"):
1429 tarfile.open(tmpname, encoding="iso8859-1")
1430
1431
1432 class WriteTestBase(TarTest):
1433 # Put all write tests in here that are supposed to be tested
1434 # in all possible mode combinations.
1435
1436 def test_fileobj_no_close(self):
1437 fobj = io.BytesIO()
1438 with tarfile.open(fileobj=fobj, mode=self.mode) as tar:
1439 tar.addfile(tarfile.TarInfo("foo"))
1440 self.assertFalse(fobj.closed, "external fileobjs must never closed")
1441 # Issue #20238: Incomplete gzip output with mode="w:gz"
1442 data = fobj.getvalue()
1443 del tar
1444 support.gc_collect()
1445 self.assertFalse(fobj.closed)
1446 self.assertEqual(data, fobj.getvalue())
1447
1448 def test_eof_marker(self):
1449 # Make sure an end of archive marker is written (two zero blocks).
1450 # tarfile insists on aligning archives to a 20 * 512 byte recordsize.
1451 # So, we create an archive that has exactly 10240 bytes without the
1452 # marker, and has 20480 bytes once the marker is written.
1453 with tarfile.open(tmpname, self.mode) as tar:
1454 t = tarfile.TarInfo("foo")
1455 t.size = tarfile.RECORDSIZE - tarfile.BLOCKSIZE
1456 tar.addfile(t, io.BytesIO(b"a" * t.size))
1457
1458 with self.open(tmpname, "rb") as fobj:
1459 self.assertEqual(len(fobj.read()), tarfile.RECORDSIZE * 2)
1460
1461
1462 class WriteTest(WriteTestBase, unittest.TestCase):
1463
1464 prefix = "w:"
1465
1466 def test_100_char_name(self):
1467 # The name field in a tar header stores strings of at most 100 chars.
1468 # If a string is shorter than 100 chars it has to be padded with '\0',
1469 # which implies that a string of exactly 100 chars is stored without
1470 # a trailing '\0'.
1471 name = "0123456789" * 10
1472 tar = tarfile.open(tmpname, self.mode)
1473 try:
1474 t = tarfile.TarInfo(name)
1475 tar.addfile(t)
1476 finally:
1477 tar.close()
1478
1479 tar = tarfile.open(tmpname)
1480 try:
1481 self.assertEqual(tar.getnames()[0], name,
1482 "failed to store 100 char filename")
1483 finally:
1484 tar.close()
1485
1486 def test_tar_size(self):
1487 # Test for bug #1013882.
1488 tar = tarfile.open(tmpname, self.mode)
1489 try:
1490 path = os.path.join(TEMPDIR, "file")
1491 with open(path, "wb") as fobj:
1492 fobj.write(b"aaa")
1493 tar.add(path)
1494 finally:
1495 tar.close()
1496 self.assertGreater(os.path.getsize(tmpname), 0,
1497 "tarfile is empty")
1498
1499 # The test_*_size tests test for bug #1167128.
1500 def test_file_size(self):
1501 tar = tarfile.open(tmpname, self.mode)
1502 try:
1503 path = os.path.join(TEMPDIR, "file")
1504 with open(path, "wb"):
1505 pass
1506 tarinfo = tar.gettarinfo(path)
1507 self.assertEqual(tarinfo.size, 0)
1508
1509 with open(path, "wb") as fobj:
1510 fobj.write(b"aaa")
1511 tarinfo = tar.gettarinfo(path)
1512 self.assertEqual(tarinfo.size, 3)
1513 finally:
1514 tar.close()
1515
1516 def test_directory_size(self):
1517 path = os.path.join(TEMPDIR, "directory")
1518 os.mkdir(path)
1519 try:
1520 tar = tarfile.open(tmpname, self.mode)
1521 try:
1522 tarinfo = tar.gettarinfo(path)
1523 self.assertEqual(tarinfo.size, 0)
1524 finally:
1525 tar.close()
1526 finally:
1527 os_helper.rmdir(path)
1528
1529 # mock the following:
1530 # os.listdir: so we know that files are in the wrong order
1531 def test_ordered_recursion(self):
1532 path = os.path.join(TEMPDIR, "directory")
1533 os.mkdir(path)
1534 open(os.path.join(path, "1"), "a").close()
1535 open(os.path.join(path, "2"), "a").close()
1536 try:
1537 tar = tarfile.open(tmpname, self.mode)
1538 try:
1539 with unittest.mock.patch('os.listdir') as mock_listdir:
1540 mock_listdir.return_value = ["2", "1"]
1541 tar.add(path)
1542 paths = []
1543 for m in tar.getmembers():
1544 paths.append(os.path.split(m.name)[-1])
1545 self.assertEqual(paths, ["directory", "1", "2"]);
1546 finally:
1547 tar.close()
1548 finally:
1549 os_helper.unlink(os.path.join(path, "1"))
1550 os_helper.unlink(os.path.join(path, "2"))
1551 os_helper.rmdir(path)
1552
1553 def test_gettarinfo_pathlike_name(self):
1554 with tarfile.open(tmpname, self.mode) as tar:
1555 path = os.path.join(TEMPDIR, "file")
1556 with open(path, "wb") as fobj:
1557 fobj.write(b"aaa")
1558 tarinfo = tar.gettarinfo(os_helper.FakePath(path))
1559 tarinfo2 = tar.gettarinfo(path)
1560 self.assertIsInstance(tarinfo.name, str)
1561 self.assertEqual(tarinfo.name, tarinfo2.name)
1562 self.assertEqual(tarinfo.size, 3)
1563
1564 @unittest.skipUnless(hasattr(os, "link"),
1565 "Missing hardlink implementation")
1566 def test_link_size(self):
1567 link = os.path.join(TEMPDIR, "link")
1568 target = os.path.join(TEMPDIR, "link_target")
1569 with open(target, "wb") as fobj:
1570 fobj.write(b"aaa")
1571 try:
1572 os.link(target, link)
1573 except PermissionError as e:
1574 self.skipTest('os.link(): %s' % e)
1575 try:
1576 tar = tarfile.open(tmpname, self.mode)
1577 try:
1578 # Record the link target in the inodes list.
1579 tar.gettarinfo(target)
1580 tarinfo = tar.gettarinfo(link)
1581 self.assertEqual(tarinfo.size, 0)
1582 finally:
1583 tar.close()
1584 finally:
1585 os_helper.unlink(target)
1586 os_helper.unlink(link)
1587
1588 @os_helper.skip_unless_symlink
1589 def test_symlink_size(self):
1590 path = os.path.join(TEMPDIR, "symlink")
1591 os.symlink("link_target", path)
1592 try:
1593 tar = tarfile.open(tmpname, self.mode)
1594 try:
1595 tarinfo = tar.gettarinfo(path)
1596 self.assertEqual(tarinfo.size, 0)
1597 finally:
1598 tar.close()
1599 finally:
1600 os_helper.unlink(path)
1601
1602 def test_add_self(self):
1603 # Test for #1257255.
1604 dstname = os.path.abspath(tmpname)
1605 tar = tarfile.open(tmpname, self.mode)
1606 try:
1607 self.assertEqual(tar.name, dstname,
1608 "archive name must be absolute")
1609 tar.add(dstname)
1610 self.assertEqual(tar.getnames(), [],
1611 "added the archive to itself")
1612
1613 with os_helper.change_cwd(TEMPDIR):
1614 tar.add(dstname)
1615 self.assertEqual(tar.getnames(), [],
1616 "added the archive to itself")
1617 finally:
1618 tar.close()
1619
1620 def test_filter(self):
1621 tempdir = os.path.join(TEMPDIR, "filter")
1622 os.mkdir(tempdir)
1623 try:
1624 for name in ("foo", "bar", "baz"):
1625 name = os.path.join(tempdir, name)
1626 os_helper.create_empty_file(name)
1627
1628 def filter(tarinfo):
1629 if os.path.basename(tarinfo.name) == "bar":
1630 return
1631 tarinfo.uid = 123
1632 tarinfo.uname = "foo"
1633 return tarinfo
1634
1635 tar = tarfile.open(tmpname, self.mode, encoding="iso8859-1")
1636 try:
1637 tar.add(tempdir, arcname="empty_dir", filter=filter)
1638 finally:
1639 tar.close()
1640
1641 # Verify that filter is a keyword-only argument
1642 with self.assertRaises(TypeError):
1643 tar.add(tempdir, "empty_dir", True, None, filter)
1644
1645 tar = tarfile.open(tmpname, "r")
1646 try:
1647 for tarinfo in tar:
1648 self.assertEqual(tarinfo.uid, 123)
1649 self.assertEqual(tarinfo.uname, "foo")
1650 self.assertEqual(len(tar.getmembers()), 3)
1651 finally:
1652 tar.close()
1653 finally:
1654 os_helper.rmtree(tempdir)
1655
1656 # Guarantee that stored pathnames are not modified. Don't
1657 # remove ./ or ../ or double slashes. Still make absolute
1658 # pathnames relative.
1659 # For details see bug #6054.
1660 def _test_pathname(self, path, cmp_path=None, dir=False):
1661 # Create a tarfile with an empty member named path
1662 # and compare the stored name with the original.
1663 foo = os.path.join(TEMPDIR, "foo")
1664 if not dir:
1665 os_helper.create_empty_file(foo)
1666 else:
1667 os.mkdir(foo)
1668
1669 tar = tarfile.open(tmpname, self.mode)
1670 try:
1671 tar.add(foo, arcname=path)
1672 finally:
1673 tar.close()
1674
1675 tar = tarfile.open(tmpname, "r")
1676 try:
1677 t = tar.next()
1678 finally:
1679 tar.close()
1680
1681 if not dir:
1682 os_helper.unlink(foo)
1683 else:
1684 os_helper.rmdir(foo)
1685
1686 self.assertEqual(t.name, cmp_path or path.replace(os.sep, "/"))
1687
1688
1689 @os_helper.skip_unless_symlink
1690 def test_extractall_symlinks(self):
1691 # Test if extractall works properly when tarfile contains symlinks
1692 tempdir = os.path.join(TEMPDIR, "testsymlinks")
1693 temparchive = os.path.join(TEMPDIR, "testsymlinks.tar")
1694 os.mkdir(tempdir)
1695 try:
1696 source_file = os.path.join(tempdir,'source')
1697 target_file = os.path.join(tempdir,'symlink')
1698 with open(source_file,'w') as f:
1699 f.write('something\n')
1700 os.symlink(source_file, target_file)
1701 with tarfile.open(temparchive, 'w') as tar:
1702 tar.add(source_file, arcname="source")
1703 tar.add(target_file, arcname="symlink")
1704 # Let's extract it to the location which contains the symlink
1705 with tarfile.open(temparchive, errorlevel=2) as tar:
1706 # this should not raise OSError: [Errno 17] File exists
1707 try:
1708 tar.extractall(path=tempdir,
1709 filter='fully_trusted')
1710 except OSError:
1711 self.fail("extractall failed with symlinked files")
1712 finally:
1713 os_helper.unlink(temparchive)
1714 os_helper.rmtree(tempdir)
1715
1716 def test_pathnames(self):
1717 self._test_pathname("foo")
1718 self._test_pathname(os.path.join("foo", ".", "bar"))
1719 self._test_pathname(os.path.join("foo", "..", "bar"))
1720 self._test_pathname(os.path.join(".", "foo"))
1721 self._test_pathname(os.path.join(".", "foo", "."))
1722 self._test_pathname(os.path.join(".", "foo", ".", "bar"))
1723 self._test_pathname(os.path.join(".", "foo", "..", "bar"))
1724 self._test_pathname(os.path.join(".", "foo", "..", "bar"))
1725 self._test_pathname(os.path.join("..", "foo"))
1726 self._test_pathname(os.path.join("..", "foo", ".."))
1727 self._test_pathname(os.path.join("..", "foo", ".", "bar"))
1728 self._test_pathname(os.path.join("..", "foo", "..", "bar"))
1729
1730 self._test_pathname("foo" + os.sep + os.sep + "bar")
1731 self._test_pathname("foo" + os.sep + os.sep, "foo", dir=True)
1732
1733 def test_abs_pathnames(self):
1734 if sys.platform == "win32":
1735 self._test_pathname("C:\\foo", "foo")
1736 else:
1737 self._test_pathname("/foo", "foo")
1738 self._test_pathname("///foo", "foo")
1739
1740 def test_cwd(self):
1741 # Test adding the current working directory.
1742 with os_helper.change_cwd(TEMPDIR):
1743 tar = tarfile.open(tmpname, self.mode)
1744 try:
1745 tar.add(".")
1746 finally:
1747 tar.close()
1748
1749 tar = tarfile.open(tmpname, "r")
1750 try:
1751 for t in tar:
1752 if t.name != ".":
1753 self.assertStartsWith(t.name, "./")
1754 finally:
1755 tar.close()
1756
1757 def test_open_nonwritable_fileobj(self):
1758 for exctype in OSError, EOFError, RuntimeError:
1759 class BadFile(io.BytesIO):
1760 first = True
1761 def write(self, data):
1762 if self.first:
1763 self.first = False
1764 raise exctype
1765
1766 f = BadFile()
1767 with (
1768 warnings_helper.check_no_resource_warning(self),
1769 self.assertRaises(exctype),
1770 ):
1771 tarfile.open(tmpname, self.mode, fileobj=f,
1772 format=tarfile.PAX_FORMAT,
1773 pax_headers={'non': 'empty'})
1774 self.assertFalse(f.closed)
1775
1776 def test_missing_fileobj(self):
1777 with tarfile.open(tmpname, self.mode) as tar:
1778 tarinfo = tar.gettarinfo(tarname)
1779 with self.assertRaises(ValueError):
1780 tar.addfile(tarinfo)
1781
1782
1783 class GzipWriteTest(GzipTest, WriteTest):
1784 pass
1785
1786
1787 class Bz2WriteTest(Bz2Test, WriteTest):
1788 pass
1789
1790
1791 class LzmaWriteTest(LzmaTest, WriteTest):
1792 pass
1793
1794 class ZstdWriteTest(ZstdTest, WriteTest):
1795 pass
1796
1797 class StreamWriteTest(WriteTestBase, unittest.TestCase):
1798
1799 prefix = "w|"
1800 decompressor = None
1801
1802 def test_stream_padding(self):
1803 # Test for bug #1543303.
1804 tar = tarfile.open(tmpname, self.mode)
1805 tar.close()
1806 if self.decompressor:
1807 dec = self.decompressor()
1808 with open(tmpname, "rb") as fobj:
1809 data = fobj.read()
1810 data = dec.decompress(data)
1811 self.assertFalse(dec.unused_data, "found trailing data")
1812 else:
1813 with self.open(tmpname) as fobj:
1814 data = fobj.read()
1815 self.assertEqual(data.count(b"\0"), tarfile.RECORDSIZE,
1816 "incorrect zero padding")
1817
1818 @unittest.skipUnless(sys.platform != "win32" and hasattr(os, "umask"),
1819 "Missing umask implementation")
1820 @unittest.skipIf(
1821 support.is_emscripten or support.is_wasi,
1822 "Emscripten's/WASI's umask is a stub."
1823 )
1824 def test_file_mode(self):
1825 # Test for issue #8464: Create files with correct
1826 # permissions.
1827 if os.path.exists(tmpname):
1828 os_helper.unlink(tmpname)
1829
1830 original_umask = os.umask(0o022)
1831 try:
1832 tar = tarfile.open(tmpname, self.mode)
1833 tar.close()
1834 mode = os.stat(tmpname).st_mode & 0o777
1835 self.assertEqual(mode, 0o644, "wrong file permissions")
1836 finally:
1837 os.umask(original_umask)
1838
1839 def test_pathlike_name(self):
1840 expected_name = os.path.abspath(tmpname)
1841 tarpath = os_helper.FakePath(tmpname)
1842
1843 for func in (tarfile.open, tarfile.TarFile.open):
1844 with self.subTest():
1845 with func(tarpath, self.mode) as tar:
1846 self.assertEqual(tar.name, expected_name)
1847 os_helper.unlink(tmpname)
1848
1849
1850 class GzipStreamWriteTest(GzipTest, StreamWriteTest):
1851 def test_source_directory_not_leaked(self):
1852 """
1853 Ensure the source directory is not included in the tar header
1854 per bpo-41316.
1855 """
1856 tarfile.open(tmpname, self.mode).close()
1857 payload = pathlib.Path(tmpname).read_text(encoding='latin-1')
1858 assert os.path.dirname(tmpname) not in payload
1859
1860
1861 class Bz2StreamWriteTest(Bz2Test, StreamWriteTest):
1862 decompressor = bz2.BZ2Decompressor if bz2 else None
1863
1864 class LzmaStreamWriteTest(LzmaTest, StreamWriteTest):
1865 decompressor = lzma.LZMADecompressor if lzma else None
1866
1867 class ZstdStreamWriteTest(ZstdTest, StreamWriteTest):
1868 decompressor = zstd.ZstdDecompressor if zstd else None
1869
1870 class _CompressedWriteTest(TarTest):
1871 # This is not actually a standalone test.
1872 # It does not inherit WriteTest because it only makes sense with gz,bz2
1873 source = (b"And we move to Bristol where they have a special, " +
1874 b"Very Silly candidate")
1875
1876 def _compressed_tar(self, compresslevel):
1877 fobj = io.BytesIO()
1878 with tarfile.open(tmpname, self.mode, fobj,
1879 compresslevel=compresslevel) as tarfl:
1880 tarfl.addfile(tarfile.TarInfo("foo"), io.BytesIO(self.source))
1881 return fobj
1882
1883 def _test_bz2_header(self, compresslevel):
1884 fobj = self._compressed_tar(compresslevel)
1885 self.assertEqual(fobj.getvalue()[0:10],
1886 b"BZh%d1AY&SY" % compresslevel)
1887
1888 def _test_gz_header(self, compresslevel):
1889 fobj = self._compressed_tar(compresslevel)
1890 self.assertEqual(fobj.getvalue()[:3], b"\x1f\x8b\x08")
1891
1892 class Bz2CompressWriteTest(Bz2Test, _CompressedWriteTest, unittest.TestCase):
1893 prefix = "w:"
1894 def test_compression_levels(self):
1895 self._test_bz2_header(1)
1896 self._test_bz2_header(5)
1897 self._test_bz2_header(9)
1898
1899 class Bz2CompressStreamWriteTest(Bz2Test, _CompressedWriteTest,
1900 unittest.TestCase):
1901 prefix = "w|"
1902 def test_compression_levels(self):
1903 self._test_bz2_header(1)
1904 self._test_bz2_header(5)
1905 self._test_bz2_header(9)
1906
1907 class GzCompressWriteTest(GzipTest, _CompressedWriteTest, unittest.TestCase):
1908 prefix = "w:"
1909 def test_compression_levels(self):
1910 self._test_gz_header(1)
1911 self._test_gz_header(5)
1912 self._test_gz_header(9)
1913
1914 class GzCompressStreamWriteTest(GzipTest, _CompressedWriteTest,
1915 unittest.TestCase):
1916 prefix = "w|"
1917 def test_compression_levels(self):
1918 self._test_gz_header(1)
1919 self._test_gz_header(5)
1920 self._test_gz_header(9)
1921
1922 class CompressLevelRaises(unittest.TestCase):
1923 def test_compresslevel_wrong_modes(self):
1924 compresslevel = 5
1925 fobj = io.BytesIO()
1926 with self.assertRaises(TypeError):
1927 tarfile.open(tmpname, "w:", fobj, compresslevel=compresslevel)
1928
1929 @support.requires_bz2()
1930 def test_wrong_compresslevels(self):
1931 # BZ2 checks that the compresslevel is in [1,9]. gz does not
1932 fobj = io.BytesIO()
1933 with self.assertRaises(ValueError):
1934 tarfile.open(tmpname, "w:bz2", fobj, compresslevel=0)
1935 with self.assertRaises(ValueError):
1936 tarfile.open(tmpname, "w:bz2", fobj, compresslevel=10)
1937 with self.assertRaises(ValueError):
1938 tarfile.open(tmpname, "w|bz2", fobj, compresslevel=10)
1939
1940 class GNUWriteTest(unittest.TestCase):
1941 # This testcase checks for correct creation of GNU Longname
1942 # and Longlink extended headers (cp. bug #812325).
1943
1944 def _length(self, s):
1945 blocks = len(s) // 512 + 1
1946 return blocks * 512
1947
1948 def _calc_size(self, name, link=None):
1949 # Initial tar header
1950 count = 512
1951
1952 if len(name) > tarfile.LENGTH_NAME:
1953 # GNU longname extended header + longname
1954 count += 512
1955 count += self._length(name)
1956 if link is not None and len(link) > tarfile.LENGTH_LINK:
1957 # GNU longlink extended header + longlink
1958 count += 512
1959 count += self._length(link)
1960 return count
1961
1962 def _test(self, name, link=None):
1963 tarinfo = tarfile.TarInfo(name)
1964 if link:
1965 tarinfo.linkname = link
1966 tarinfo.type = tarfile.LNKTYPE
1967
1968 tar = tarfile.open(tmpname, "w")
1969 try:
1970 tar.format = tarfile.GNU_FORMAT
1971 tar.addfile(tarinfo)
1972
1973 v1 = self._calc_size(name, link)
1974 v2 = tar.offset
1975 self.assertEqual(v1, v2, "GNU longname/longlink creation failed")
1976 finally:
1977 tar.close()
1978
1979 tar = tarfile.open(tmpname)
1980 try:
1981 member = tar.next()
1982 self.assertIsNotNone(member,
1983 "unable to read longname member")
1984 self.assertEqual(tarinfo.name, member.name,
1985 "unable to read longname member")
1986 self.assertEqual(tarinfo.linkname, member.linkname,
1987 "unable to read longname member")
1988 finally:
1989 tar.close()
1990
1991 def test_longname_1023(self):
1992 self._test(("longnam/" * 127) + "longnam")
1993
1994 def test_longname_1024(self):
1995 self._test(("longnam/" * 127) + "longname")
1996
1997 def test_longname_1025(self):
1998 self._test(("longnam/" * 127) + "longname_")
1999
2000 def test_longlink_1023(self):
2001 self._test("name", ("longlnk/" * 127) + "longlnk")
2002
2003 def test_longlink_1024(self):
2004 self._test("name", ("longlnk/" * 127) + "longlink")
2005
2006 def test_longlink_1025(self):
2007 self._test("name", ("longlnk/" * 127) + "longlink_")
2008
2009 def test_longnamelink_1023(self):
2010 self._test(("longnam/" * 127) + "longnam",
2011 ("longlnk/" * 127) + "longlnk")
2012
2013 def test_longnamelink_1024(self):
2014 self._test(("longnam/" * 127) + "longname",
2015 ("longlnk/" * 127) + "longlink")
2016
2017 def test_longnamelink_1025(self):
2018 self._test(("longnam/" * 127) + "longname_",
2019 ("longlnk/" * 127) + "longlink_")
2020
2021
2022 class DeviceHeaderTest(WriteTestBase, unittest.TestCase):
2023
2024 prefix = "w:"
2025
2026 def test_headers_written_only_for_device_files(self):
2027 # Regression test for bpo-18819.
2028 tempdir = os.path.join(TEMPDIR, "device_header_test")
2029 os.mkdir(tempdir)
2030 try:
2031 tar = tarfile.open(tmpname, self.mode)
2032 try:
2033 input_blk = tarfile.TarInfo(name="my_block_device")
2034 input_reg = tarfile.TarInfo(name="my_regular_file")
2035 input_blk.type = tarfile.BLKTYPE
2036 input_reg.type = tarfile.REGTYPE
2037 tar.addfile(input_blk)
2038 tar.addfile(input_reg)
2039 finally:
2040 tar.close()
2041
2042 # devmajor and devminor should be *interpreted* as 0 in both...
2043 tar = tarfile.open(tmpname, "r")
2044 try:
2045 output_blk = tar.getmember("my_block_device")
2046 output_reg = tar.getmember("my_regular_file")
2047 finally:
2048 tar.close()
2049 self.assertEqual(output_blk.devmajor, 0)
2050 self.assertEqual(output_blk.devminor, 0)
2051 self.assertEqual(output_reg.devmajor, 0)
2052 self.assertEqual(output_reg.devminor, 0)
2053
2054 # ...but the fields should not actually be set on regular files:
2055 with open(tmpname, "rb") as infile:
2056 buf = infile.read()
2057 buf_blk = buf[output_blk.offset:output_blk.offset_data]
2058 buf_reg = buf[output_reg.offset:output_reg.offset_data]
2059 # See `struct posixheader` in GNU docs for byte offsets:
2060 # <https://www.gnu.org/software/tar/manual/html_node/Standard.html>
2061 device_headers = slice(329, 329 + 16)
2062 self.assertEqual(buf_blk[device_headers], b"0000000\0" * 2)
2063 self.assertEqual(buf_reg[device_headers], b"\0" * 16)
2064 finally:
2065 os_helper.rmtree(tempdir)
2066
2067
2068 class CreateTest(WriteTestBase, unittest.TestCase):
2069
2070 prefix = "x:"
2071
2072 file_path = os.path.join(TEMPDIR, "spameggs42")
2073
2074 def setUp(self):
2075 os_helper.unlink(tmpname)
2076
2077 @classmethod
2078 def setUpClass(cls):
2079 with open(cls.file_path, "wb") as fobj:
2080 fobj.write(b"aaa")
2081
2082 @classmethod
2083 def tearDownClass(cls):
2084 os_helper.unlink(cls.file_path)
2085
2086 def test_create(self):
2087 with tarfile.open(tmpname, self.mode) as tobj:
2088 tobj.add(self.file_path)
2089
2090 with self.taropen(tmpname) as tobj:
2091 names = tobj.getnames()
2092 self.assertEqual(len(names), 1)
2093 self.assertIn('spameggs42', names[0])
2094
2095 def test_create_existing(self):
2096 with tarfile.open(tmpname, self.mode) as tobj:
2097 tobj.add(self.file_path)
2098
2099 with self.assertRaises(FileExistsError):
2100 tobj = tarfile.open(tmpname, self.mode)
2101
2102 with self.taropen(tmpname) as tobj:
2103 names = tobj.getnames()
2104 self.assertEqual(len(names), 1)
2105 self.assertIn('spameggs42', names[0])
2106
2107 def test_create_taropen(self):
2108 with self.taropen(tmpname, "x") as tobj:
2109 tobj.add(self.file_path)
2110
2111 with self.taropen(tmpname) as tobj:
2112 names = tobj.getnames()
2113 self.assertEqual(len(names), 1)
2114 self.assertIn('spameggs42', names[0])
2115
2116 def test_create_existing_taropen(self):
2117 with self.taropen(tmpname, "x") as tobj:
2118 tobj.add(self.file_path)
2119
2120 with self.assertRaises(FileExistsError):
2121 with self.taropen(tmpname, "x"):
2122 pass
2123
2124 with self.taropen(tmpname) as tobj:
2125 names = tobj.getnames()
2126 self.assertEqual(len(names), 1)
2127 self.assertIn("spameggs42", names[0])
2128
2129 def test_create_pathlike_name(self):
2130 with tarfile.open(os_helper.FakePath(tmpname), self.mode) as tobj:
2131 self.assertIsInstance(tobj.name, str)
2132 self.assertEqual(tobj.name, os.path.abspath(tmpname))
2133 tobj.add(os_helper.FakePath(self.file_path))
2134 names = tobj.getnames()
2135 self.assertEqual(len(names), 1)
2136 self.assertIn('spameggs42', names[0])
2137
2138 with self.taropen(tmpname) as tobj:
2139 names = tobj.getnames()
2140 self.assertEqual(len(names), 1)
2141 self.assertIn('spameggs42', names[0])
2142
2143 def test_create_taropen_pathlike_name(self):
2144 with self.taropen(os_helper.FakePath(tmpname), "x") as tobj:
2145 self.assertIsInstance(tobj.name, str)
2146 self.assertEqual(tobj.name, os.path.abspath(tmpname))
2147 tobj.add(os_helper.FakePath(self.file_path))
2148 names = tobj.getnames()
2149 self.assertEqual(len(names), 1)
2150 self.assertIn('spameggs42', names[0])
2151
2152 with self.taropen(tmpname) as tobj:
2153 names = tobj.getnames()
2154 self.assertEqual(len(names), 1)
2155 self.assertIn('spameggs42', names[0])
2156
2157
2158 class GzipCreateTest(GzipTest, CreateTest):
2159
2160 def test_create_with_compresslevel(self):
2161 with tarfile.open(tmpname, self.mode, compresslevel=1) as tobj:
2162 tobj.add(self.file_path)
2163 with tarfile.open(tmpname, 'r:gz', compresslevel=1) as tobj:
2164 pass
2165
2166
2167 class Bz2CreateTest(Bz2Test, CreateTest):
2168
2169 def test_create_with_compresslevel(self):
2170 with tarfile.open(tmpname, self.mode, compresslevel=1) as tobj:
2171 tobj.add(self.file_path)
2172 with tarfile.open(tmpname, 'r:bz2', compresslevel=1) as tobj:
2173 pass
2174
2175
2176 class LzmaCreateTest(LzmaTest, CreateTest):
2177
2178 # Unlike gz and bz2, xz uses the preset keyword instead of compresslevel.
2179 # It does not allow for preset to be specified when reading.
2180 def test_create_with_preset(self):
2181 with tarfile.open(tmpname, self.mode, preset=1) as tobj:
2182 tobj.add(self.file_path)
2183
2184
2185 class ZstdCreateTest(ZstdTest, CreateTest):
2186
2187 # Unlike gz and bz2, zstd uses the level keyword instead of compresslevel.
2188 # It does not allow for level to be specified when reading.
2189 def test_create_with_level(self):
2190 with tarfile.open(tmpname, self.mode, level=1) as tobj:
2191 tobj.add(self.file_path)
2192
2193 class CreateWithXModeTest(CreateTest):
2194
2195 prefix = "x"
2196
2197 test_create_taropen = None
2198 test_create_existing_taropen = None
2199
2200
2201 @unittest.skipUnless(hasattr(os, "link"), "Missing hardlink implementation")
2202 class HardlinkTest(unittest.TestCase):
2203 # Test the creation of LNKTYPE (hardlink) members in an archive.
2204
2205 def setUp(self):
2206 self.foo = os.path.join(TEMPDIR, "foo")
2207 self.bar = os.path.join(TEMPDIR, "bar")
2208
2209 with open(self.foo, "wb") as fobj:
2210 fobj.write(b"foo")
2211
2212 try:
2213 os.link(self.foo, self.bar)
2214 except PermissionError as e:
2215 self.skipTest('os.link(): %s' % e)
2216
2217 self.tar = tarfile.open(tmpname, "w")
2218 self.tar.add(self.foo)
2219
2220 def tearDown(self):
2221 self.tar.close()
2222 os_helper.unlink(self.foo)
2223 os_helper.unlink(self.bar)
2224
2225 def test_add_twice(self):
2226 # The same name will be added as a REGTYPE every
2227 # time regardless of st_nlink.
2228 tarinfo = self.tar.gettarinfo(self.foo)
2229 self.assertEqual(tarinfo.type, tarfile.REGTYPE,
2230 "add file as regular failed")
2231
2232 def test_add_hardlink(self):
2233 tarinfo = self.tar.gettarinfo(self.bar)
2234 self.assertEqual(tarinfo.type, tarfile.LNKTYPE,
2235 "add file as hardlink failed")
2236
2237 def test_dereference_hardlink(self):
2238 self.tar.dereference = True
2239 tarinfo = self.tar.gettarinfo(self.bar)
2240 self.assertEqual(tarinfo.type, tarfile.REGTYPE,
2241 "dereferencing hardlink failed")
2242
2243
2244 class PaxWriteTest(GNUWriteTest):
2245
2246 def _test(self, name, link=None):
2247 # See GNUWriteTest.
2248 tarinfo = tarfile.TarInfo(name)
2249 if link:
2250 tarinfo.linkname = link
2251 tarinfo.type = tarfile.LNKTYPE
2252
2253 tar = tarfile.open(tmpname, "w", format=tarfile.PAX_FORMAT)
2254 try:
2255 tar.addfile(tarinfo)
2256 finally:
2257 tar.close()
2258
2259 tar = tarfile.open(tmpname)
2260 try:
2261 if link:
2262 l = tar.getmembers()[0].linkname
2263 self.assertEqual(link, l, "PAX longlink creation failed")
2264 else:
2265 n = tar.getmembers()[0].name
2266 self.assertEqual(name, n, "PAX longname creation failed")
2267 finally:
2268 tar.close()
2269
2270 def test_pax_global_header(self):
2271 pax_headers = {
2272 "foo": "bar",
2273 "uid": "0",
2274 "mtime": "1.23",
2275 "test": "\xe4\xf6\xfc",
2276 "\xe4\xf6\xfc": "test"}
2277
2278 tar = tarfile.open(tmpname, "w", format=tarfile.PAX_FORMAT,
2279 pax_headers=pax_headers)
2280 try:
2281 tar.addfile(tarfile.TarInfo("test"))
2282 finally:
2283 tar.close()
2284
2285 # Test if the global header was written correctly.
2286 tar = tarfile.open(tmpname, encoding="iso8859-1")
2287 try:
2288 self.assertEqual(tar.pax_headers, pax_headers)
2289 self.assertEqual(tar.getmembers()[0].pax_headers, pax_headers)
2290 # Test if all the fields are strings.
2291 for key, val in tar.pax_headers.items():
2292 self.assertIsNot(type(key), bytes)
2293 self.assertIsNot(type(val), bytes)
2294 if key in tarfile.PAX_NUMBER_FIELDS:
2295 try:
2296 tarfile.PAX_NUMBER_FIELDS[key](val)
2297 except (TypeError, ValueError):
2298 self.fail("unable to convert pax header field")
2299 finally:
2300 tar.close()
2301
2302 def test_pax_extended_header(self):
2303 # The fields from the pax header have priority over the
2304 # TarInfo.
2305 pax_headers = {"path": "foo", "uid": "123"}
2306
2307 tar = tarfile.open(tmpname, "w", format=tarfile.PAX_FORMAT,
2308 encoding="iso8859-1")
2309 try:
2310 t = tarfile.TarInfo()
2311 t.name = "\xe4\xf6\xfc" # non-ASCII
2312 t.uid = 8**8 # too large
2313 t.pax_headers = pax_headers
2314 tar.addfile(t)
2315 finally:
2316 tar.close()
2317
2318 tar = tarfile.open(tmpname, encoding="iso8859-1")
2319 try:
2320 t = tar.getmembers()[0]
2321 self.assertEqual(t.pax_headers, pax_headers)
2322 self.assertEqual(t.name, "foo")
2323 self.assertEqual(t.uid, 123)
2324 finally:
2325 tar.close()
2326
2327 def test_create_pax_header(self):
2328 # The ustar header should contain values that can be
2329 # represented reasonably, even if a better (e.g. higher
2330 # precision) version is set in the pax header.
2331 # Issue #45863
2332
2333 # values that should be kept
2334 t = tarfile.TarInfo()
2335 t.name = "foo"
2336 t.mtime = 1000.1
2337 t.size = 100
2338 t.uid = 123
2339 t.gid = 124
2340 info = t.get_info()
2341 header = t.create_pax_header(info, encoding="iso8859-1")
2342 self.assertEqual(info['name'], "foo")
2343 # mtime should be rounded to nearest second
2344 self.assertIsInstance(info['mtime'], int)
2345 self.assertEqual(info['mtime'], 1000)
2346 self.assertEqual(info['size'], 100)
2347 self.assertEqual(info['uid'], 123)
2348 self.assertEqual(info['gid'], 124)
2349 self.assertEqual(header,
2350 b'././@PaxHeader' + bytes(86) \
2351 + b'0000000\x000000000\x000000000\x0000000000020\x0000000000000\x00010205\x00 x' \
2352 + bytes(100) + b'ustar\x0000'+ bytes(247) \
2353 + b'16 mtime=1000.1\n' + bytes(496) + b'foo' + bytes(97) \
2354 + b'0000644\x000000173\x000000174\x0000000000144\x0000000001750\x00006516\x00 0' \
2355 + bytes(100) + b'ustar\x0000' + bytes(247))
2356
2357 # values that should be changed
2358 t = tarfile.TarInfo()
2359 t.name = "foo\u3374" # can't be represented in ascii
2360 t.mtime = 10**10 # too big
2361 t.size = 10**10 # too big
2362 t.uid = 8**8 # too big
2363 t.gid = 8**8+1 # too big
2364 info = t.get_info()
2365 header = t.create_pax_header(info, encoding="iso8859-1")
2366 # name is kept as-is in info but should be added to pax header
2367 self.assertEqual(info['name'], "foo\u3374")
2368 self.assertEqual(info['mtime'], 0)
2369 self.assertEqual(info['size'], 0)
2370 self.assertEqual(info['uid'], 0)
2371 self.assertEqual(info['gid'], 0)
2372 self.assertEqual(header,
2373 b'././@PaxHeader' + bytes(86) \
2374 + b'0000000\x000000000\x000000000\x0000000000130\x0000000000000\x00010207\x00 x' \
2375 + bytes(100) + b'ustar\x0000' + bytes(247) \
2376 + b'15 path=foo\xe3\x8d\xb4\n16 uid=16777216\n' \
2377 + b'16 gid=16777217\n20 size=10000000000\n' \
2378 + b'21 mtime=10000000000\n'+ bytes(424) + b'foo?' + bytes(96) \
2379 + b'0000644\x000000000\x000000000\x0000000000000\x0000000000000\x00006540\x00 0' \
2380 + bytes(100) + b'ustar\x0000' + bytes(247))
2381
2382
2383 class UnicodeTest:
2384
2385 def test_iso8859_1_filename(self):
2386 self._test_unicode_filename("iso8859-1")
2387
2388 def test_utf7_filename(self):
2389 self._test_unicode_filename("utf7")
2390
2391 def test_utf8_filename(self):
2392 self._test_unicode_filename("utf-8")
2393
2394 def _test_unicode_filename(self, encoding):
2395 tar = tarfile.open(tmpname, "w", format=self.format,
2396 encoding=encoding, errors="strict")
2397 try:
2398 name = "\xe4\xf6\xfc"
2399 tar.addfile(tarfile.TarInfo(name))
2400 finally:
2401 tar.close()
2402
2403 tar = tarfile.open(tmpname, encoding=encoding)
2404 try:
2405 self.assertEqual(tar.getmembers()[0].name, name)
2406 finally:
2407 tar.close()
2408
2409 def test_unicode_filename_error(self):
2410 tar = tarfile.open(tmpname, "w", format=self.format,
2411 encoding="ascii", errors="strict")
2412 try:
2413 tarinfo = tarfile.TarInfo()
2414
2415 tarinfo.name = "\xe4\xf6\xfc"
2416 self.assertRaises(UnicodeError, tar.addfile, tarinfo)
2417
2418 tarinfo.name = "foo"
2419 tarinfo.uname = "\xe4\xf6\xfc"
2420 self.assertRaises(UnicodeError, tar.addfile, tarinfo)
2421 finally:
2422 tar.close()
2423
2424 def test_unicode_argument(self):
2425 tar = tarfile.open(tarname, "r",
2426 encoding="iso8859-1", errors="strict")
2427 try:
2428 for t in tar:
2429 self.assertIs(type(t.name), str)
2430 self.assertIs(type(t.linkname), str)
2431 self.assertIs(type(t.uname), str)
2432 self.assertIs(type(t.gname), str)
2433 finally:
2434 tar.close()
2435
2436 def test_uname_unicode(self):
2437 t = tarfile.TarInfo("foo")
2438 t.uname = "\xe4\xf6\xfc"
2439 t.gname = "\xe4\xf6\xfc"
2440
2441 tar = tarfile.open(tmpname, mode="w", format=self.format,
2442 encoding="iso8859-1")
2443 try:
2444 tar.addfile(t)
2445 finally:
2446 tar.close()
2447
2448 tar = tarfile.open(tmpname, encoding="iso8859-1")
2449 try:
2450 t = tar.getmember("foo")
2451 self.assertEqual(t.uname, "\xe4\xf6\xfc")
2452 self.assertEqual(t.gname, "\xe4\xf6\xfc")
2453
2454 if self.format != tarfile.PAX_FORMAT:
2455 tar.close()
2456 tar = tarfile.open(tmpname, encoding="ascii")
2457 t = tar.getmember("foo")
2458 self.assertEqual(t.uname, "\udce4\udcf6\udcfc")
2459 self.assertEqual(t.gname, "\udce4\udcf6\udcfc")
2460 finally:
2461 tar.close()
2462
2463
2464 class UstarUnicodeTest(UnicodeTest, unittest.TestCase):
2465
2466 format = tarfile.USTAR_FORMAT
2467
2468 # Test whether the utf-8 encoded version of a filename exceeds the 100
2469 # bytes name field limit (every occurrence of '\xff' will be expanded to 2
2470 # bytes).
2471 def test_unicode_name1(self):
2472 self._test_ustar_name("0123456789" * 10)
2473 self._test_ustar_name("0123456789" * 10 + "0", ValueError)
2474 self._test_ustar_name("0123456789" * 9 + "01234567\xff")
2475 self._test_ustar_name("0123456789" * 9 + "012345678\xff", ValueError)
2476
2477 def test_unicode_name2(self):
2478 self._test_ustar_name("0123456789" * 9 + "012345\xff\xff")
2479 self._test_ustar_name("0123456789" * 9 + "0123456\xff\xff", ValueError)
2480
2481 # Test whether the utf-8 encoded version of a filename exceeds the 155
2482 # bytes prefix + '/' + 100 bytes name limit.
2483 def test_unicode_longname1(self):
2484 self._test_ustar_name("0123456789" * 15 + "01234/" + "0123456789" * 10)
2485 self._test_ustar_name("0123456789" * 15 + "0123/4" + "0123456789" * 10, ValueError)
2486 self._test_ustar_name("0123456789" * 15 + "012\xff/" + "0123456789" * 10)
2487 self._test_ustar_name("0123456789" * 15 + "0123\xff/" + "0123456789" * 10, ValueError)
2488
2489 def test_unicode_longname2(self):
2490 self._test_ustar_name("0123456789" * 15 + "01\xff/2" + "0123456789" * 10, ValueError)
2491 self._test_ustar_name("0123456789" * 15 + "01\xff\xff/" + "0123456789" * 10, ValueError)
2492
2493 def test_unicode_longname3(self):
2494 self._test_ustar_name("0123456789" * 15 + "01\xff\xff/2" + "0123456789" * 10, ValueError)
2495 self._test_ustar_name("0123456789" * 15 + "01234/" + "0123456789" * 9 + "01234567\xff")
2496 self._test_ustar_name("0123456789" * 15 + "01234/" + "0123456789" * 9 + "012345678\xff", ValueError)
2497
2498 def test_unicode_longname4(self):
2499 self._test_ustar_name("0123456789" * 15 + "01234/" + "0123456789" * 9 + "012345\xff\xff")
2500 self._test_ustar_name("0123456789" * 15 + "01234/" + "0123456789" * 9 + "0123456\xff\xff", ValueError)
2501
2502 def _test_ustar_name(self, name, exc=None):
2503 with tarfile.open(tmpname, "w", format=self.format, encoding="utf-8") as tar:
2504 t = tarfile.TarInfo(name)
2505 if exc is None:
2506 tar.addfile(t)
2507 else:
2508 self.assertRaises(exc, tar.addfile, t)
2509
2510 if exc is None:
2511 with tarfile.open(tmpname, "r", encoding="utf-8") as tar:
2512 for t in tar:
2513 self.assertEqual(name, t.name)
2514 break
2515
2516 # Test the same as above for the 100 bytes link field.
2517 def test_unicode_link1(self):
2518 self._test_ustar_link("0123456789" * 10)
2519 self._test_ustar_link("0123456789" * 10 + "0", ValueError)
2520 self._test_ustar_link("0123456789" * 9 + "01234567\xff")
2521 self._test_ustar_link("0123456789" * 9 + "012345678\xff", ValueError)
2522
2523 def test_unicode_link2(self):
2524 self._test_ustar_link("0123456789" * 9 + "012345\xff\xff")
2525 self._test_ustar_link("0123456789" * 9 + "0123456\xff\xff", ValueError)
2526
2527 def _test_ustar_link(self, name, exc=None):
2528 with tarfile.open(tmpname, "w", format=self.format, encoding="utf-8") as tar:
2529 t = tarfile.TarInfo("foo")
2530 t.linkname = name
2531 if exc is None:
2532 tar.addfile(t)
2533 else:
2534 self.assertRaises(exc, tar.addfile, t)
2535
2536 if exc is None:
2537 with tarfile.open(tmpname, "r", encoding="utf-8") as tar:
2538 for t in tar:
2539 self.assertEqual(name, t.linkname)
2540 break
2541
2542
2543 class GNUUnicodeTest(UnicodeTest, unittest.TestCase):
2544
2545 format = tarfile.GNU_FORMAT
2546
2547 def test_bad_pax_header(self):
2548 # Test for issue #8633. GNU tar <= 1.23 creates raw binary fields
2549 # without a hdrcharset=BINARY header.
2550 for encoding, name in (
2551 ("utf-8", "pax/bad-pax-\udce4\udcf6\udcfc"),
2552 ("iso8859-1", "pax/bad-pax-\xe4\xf6\xfc"),):
2553 with tarfile.open(tarname, encoding=encoding,
2554 errors="surrogateescape") as tar:
2555 try:
2556 t = tar.getmember(name)
2557 except KeyError:
2558 self.fail("unable to read bad GNU tar pax header")
2559
2560
2561 class PAXUnicodeTest(UnicodeTest, unittest.TestCase):
2562
2563 format = tarfile.PAX_FORMAT
2564
2565 # PAX_FORMAT ignores encoding in write mode.
2566 test_unicode_filename_error = None
2567
2568 def test_binary_header(self):
2569 # Test a POSIX.1-2008 compatible header with a hdrcharset=BINARY field.
2570 for encoding, name in (
2571 ("utf-8", "pax/hdrcharset-\udce4\udcf6\udcfc"),
2572 ("iso8859-1", "pax/hdrcharset-\xe4\xf6\xfc"),):
2573 with tarfile.open(tarname, encoding=encoding,
2574 errors="surrogateescape") as tar:
2575 try:
2576 t = tar.getmember(name)
2577 except KeyError:
2578 self.fail("unable to read POSIX.1-2008 binary header")
2579
2580
2581 class AppendTestBase:
2582 # Test append mode (cp. patch #1652681).
2583
2584 def setUp(self):
2585 self.tarname = tmpname
2586 if os.path.exists(self.tarname):
2587 os_helper.unlink(self.tarname)
2588
2589 def _create_testtar(self, mode="w:"):
2590 with tarfile.open(tarname, encoding="iso8859-1") as src:
2591 t = src.getmember("ustar/regtype")
2592 t.name = "foo"
2593 with src.extractfile(t) as f:
2594 with tarfile.open(self.tarname, mode) as tar:
2595 tar.addfile(t, f)
2596
2597 def test_append_compressed(self):
2598 self._create_testtar("w:" + self.suffix)
2599 self.assertRaises(tarfile.ReadError, tarfile.open, tmpname, "a")
2600
2601 class AppendTest(AppendTestBase, unittest.TestCase):
2602 test_append_compressed = None
2603
2604 def _add_testfile(self, fileobj=None):
2605 with tarfile.open(self.tarname, "a", fileobj=fileobj) as tar:
2606 tar.addfile(tarfile.TarInfo("bar"))
2607
2608 def _test(self, names=["bar"], fileobj=None):
2609 with tarfile.open(self.tarname, fileobj=fileobj) as tar:
2610 self.assertEqual(tar.getnames(), names)
2611
2612 def test_non_existing(self):
2613 self._add_testfile()
2614 self._test()
2615
2616 def test_empty(self):
2617 tarfile.open(self.tarname, "w:").close()
2618 self._add_testfile()
2619 self._test()
2620
2621 def test_empty_fileobj(self):
2622 fobj = io.BytesIO(b"\0" * 1024)
2623 self._add_testfile(fobj)
2624 fobj.seek(0)
2625 self._test(fileobj=fobj)
2626
2627 def test_fileobj(self):
2628 self._create_testtar()
2629 with open(self.tarname, "rb") as fobj:
2630 data = fobj.read()
2631 fobj = io.BytesIO(data)
2632 self._add_testfile(fobj)
2633 fobj.seek(0)
2634 self._test(names=["foo", "bar"], fileobj=fobj)
2635
2636 def test_existing(self):
2637 self._create_testtar()
2638 self._add_testfile()
2639 self._test(names=["foo", "bar"])
2640
2641 # Append mode is supposed to fail if the tarfile to append to
2642 # does not end with a zero block.
2643 def _test_error(self, data):
2644 with open(self.tarname, "wb") as fobj:
2645 fobj.write(data)
2646 self.assertRaises(tarfile.ReadError, self._add_testfile)
2647
2648 def test_null(self):
2649 self._test_error(b"")
2650
2651 def test_incomplete(self):
2652 self._test_error(b"\0" * 13)
2653
2654 def test_premature_eof(self):
2655 data = tarfile.TarInfo("foo").tobuf()
2656 self._test_error(data)
2657
2658 def test_trailing_garbage(self):
2659 data = tarfile.TarInfo("foo").tobuf()
2660 self._test_error(data + b"\0" * 13)
2661
2662 def test_invalid(self):
2663 self._test_error(b"a" * 512)
2664
2665 class GzipAppendTest(GzipTest, AppendTestBase, unittest.TestCase):
2666 pass
2667
2668 class Bz2AppendTest(Bz2Test, AppendTestBase, unittest.TestCase):
2669 pass
2670
2671 class LzmaAppendTest(LzmaTest, AppendTestBase, unittest.TestCase):
2672 pass
2673
2674 class ZstdAppendTest(ZstdTest, AppendTestBase, unittest.TestCase):
2675 pass
2676
2677 class LimitsTest(unittest.TestCase):
2678
2679 def test_ustar_limits(self):
2680 # 100 char name
2681 tarinfo = tarfile.TarInfo("0123456789" * 10)
2682 tarinfo.tobuf(tarfile.USTAR_FORMAT)
2683
2684 # 101 char name that cannot be stored
2685 tarinfo = tarfile.TarInfo("0123456789" * 10 + "0")
2686 self.assertRaises(ValueError, tarinfo.tobuf, tarfile.USTAR_FORMAT)
2687
2688 # 256 char name with a slash at pos 156
2689 tarinfo = tarfile.TarInfo("123/" * 62 + "longname")
2690 tarinfo.tobuf(tarfile.USTAR_FORMAT)
2691
2692 # 256 char name that cannot be stored
2693 tarinfo = tarfile.TarInfo("1234567/" * 31 + "longname")
2694 self.assertRaises(ValueError, tarinfo.tobuf, tarfile.USTAR_FORMAT)
2695
2696 # 512 char name
2697 tarinfo = tarfile.TarInfo("123/" * 126 + "longname")
2698 self.assertRaises(ValueError, tarinfo.tobuf, tarfile.USTAR_FORMAT)
2699
2700 # 512 char linkname
2701 tarinfo = tarfile.TarInfo("longlink")
2702 tarinfo.linkname = "123/" * 126 + "longname"
2703 self.assertRaises(ValueError, tarinfo.tobuf, tarfile.USTAR_FORMAT)
2704
2705 # uid > 8 digits
2706 tarinfo = tarfile.TarInfo("name")
2707 tarinfo.uid = 0o10000000
2708 self.assertRaises(ValueError, tarinfo.tobuf, tarfile.USTAR_FORMAT)
2709
2710 def test_gnu_limits(self):
2711 tarinfo = tarfile.TarInfo("123/" * 126 + "longname")
2712 tarinfo.tobuf(tarfile.GNU_FORMAT)
2713
2714 tarinfo = tarfile.TarInfo("longlink")
2715 tarinfo.linkname = "123/" * 126 + "longname"
2716 tarinfo.tobuf(tarfile.GNU_FORMAT)
2717
2718 # uid >= 256 ** 7
2719 tarinfo = tarfile.TarInfo("name")
2720 tarinfo.uid = 0o4000000000000000000
2721 self.assertRaises(ValueError, tarinfo.tobuf, tarfile.GNU_FORMAT)
2722
2723 def test_pax_limits(self):
2724 tarinfo = tarfile.TarInfo("123/" * 126 + "longname")
2725 tarinfo.tobuf(tarfile.PAX_FORMAT)
2726
2727 tarinfo = tarfile.TarInfo("longlink")
2728 tarinfo.linkname = "123/" * 126 + "longname"
2729 tarinfo.tobuf(tarfile.PAX_FORMAT)
2730
2731 tarinfo = tarfile.TarInfo("name")
2732 tarinfo.uid = 0o4000000000000000000
2733 tarinfo.tobuf(tarfile.PAX_FORMAT)
2734
2735
2736 class MiscTest(unittest.TestCase):
2737
2738 def test_char_fields(self):
2739 self.assertEqual(tarfile.stn("foo", 8, "ascii", "strict"),
2740 b"foo\0\0\0\0\0")
2741 self.assertEqual(tarfile.stn("foobar", 3, "ascii", "strict"),
2742 b"foo")
2743 self.assertEqual(tarfile.nts(b"foo\0\0\0\0\0", "ascii", "strict"),
2744 "foo")
2745 self.assertEqual(tarfile.nts(b"foo\0bar\0", "ascii", "strict"),
2746 "foo")
2747
2748 def test_read_number_fields(self):
2749 # Issue 13158: Test if GNU tar specific base-256 number fields
2750 # are decoded correctly.
2751 self.assertEqual(tarfile.nti(b"0000001\x00"), 1)
2752 self.assertEqual(tarfile.nti(b"7777777\x00"), 0o7777777)
2753 self.assertEqual(tarfile.nti(b"\x80\x00\x00\x00\x00\x20\x00\x00"),
2754 0o10000000)
2755 self.assertEqual(tarfile.nti(b"\x80\x00\x00\x00\xff\xff\xff\xff"),
2756 0xffffffff)
2757 self.assertEqual(tarfile.nti(b"\xff\xff\xff\xff\xff\xff\xff\xff"),
2758 -1)
2759 self.assertEqual(tarfile.nti(b"\xff\xff\xff\xff\xff\xff\xff\x9c"),
2760 -100)
2761 self.assertEqual(tarfile.nti(b"\xff\x00\x00\x00\x00\x00\x00\x00"),
2762 -0x100000000000000)
2763
2764 # Issue 24514: Test if empty number fields are converted to zero.
2765 self.assertEqual(tarfile.nti(b"\0"), 0)
2766 self.assertEqual(tarfile.nti(b" \0"), 0)
2767
2768 def test_write_number_fields(self):
2769 self.assertEqual(tarfile.itn(1), b"0000001\x00")
2770 self.assertEqual(tarfile.itn(0o7777777), b"7777777\x00")
2771 self.assertEqual(tarfile.itn(0o10000000, format=tarfile.GNU_FORMAT),
2772 b"\x80\x00\x00\x00\x00\x20\x00\x00")
2773 self.assertEqual(tarfile.itn(0xffffffff, format=tarfile.GNU_FORMAT),
2774 b"\x80\x00\x00\x00\xff\xff\xff\xff")
2775 self.assertEqual(tarfile.itn(-1, format=tarfile.GNU_FORMAT),
2776 b"\xff\xff\xff\xff\xff\xff\xff\xff")
2777 self.assertEqual(tarfile.itn(-100, format=tarfile.GNU_FORMAT),
2778 b"\xff\xff\xff\xff\xff\xff\xff\x9c")
2779 self.assertEqual(tarfile.itn(-0x100000000000000,
2780 format=tarfile.GNU_FORMAT),
2781 b"\xff\x00\x00\x00\x00\x00\x00\x00")
2782
2783 # Issue 32713: Test if itn() supports float values outside the
2784 # non-GNU format range
2785 self.assertEqual(tarfile.itn(-100.0, format=tarfile.GNU_FORMAT),
2786 b"\xff\xff\xff\xff\xff\xff\xff\x9c")
2787 self.assertEqual(tarfile.itn(8 ** 12 + 0.0, format=tarfile.GNU_FORMAT),
2788 b"\x80\x00\x00\x10\x00\x00\x00\x00")
2789 self.assertEqual(tarfile.nti(tarfile.itn(-0.1, format=tarfile.GNU_FORMAT)), 0)
2790
2791 def test_number_field_limits(self):
2792 with self.assertRaises(ValueError):
2793 tarfile.itn(-1, 8, tarfile.USTAR_FORMAT)
2794 with self.assertRaises(ValueError):
2795 tarfile.itn(0o10000000, 8, tarfile.USTAR_FORMAT)
2796 with self.assertRaises(ValueError):
2797 tarfile.itn(-0x10000000001, 6, tarfile.GNU_FORMAT)
2798 with self.assertRaises(ValueError):
2799 tarfile.itn(0x10000000000, 6, tarfile.GNU_FORMAT)
2800
2801 def test__all__(self):
2802 not_exported = {
2803 'version', 'grp', 'pwd', 'symlink_exception', 'NUL', 'BLOCKSIZE',
2804 'RECORDSIZE', 'GNU_MAGIC', 'POSIX_MAGIC', 'LENGTH_NAME',
2805 'LENGTH_LINK', 'LENGTH_PREFIX', 'REGTYPE', 'AREGTYPE', 'LNKTYPE',
2806 'SYMTYPE', 'CHRTYPE', 'BLKTYPE', 'DIRTYPE', 'FIFOTYPE', 'CONTTYPE',
2807 'GNUTYPE_LONGNAME', 'GNUTYPE_LONGLINK', 'GNUTYPE_SPARSE',
2808 'XHDTYPE', 'XGLTYPE', 'SOLARIS_XHDTYPE', 'SUPPORTED_TYPES',
2809 'REGULAR_TYPES', 'GNU_TYPES', 'PAX_FIELDS', 'PAX_NAME_FIELDS',
2810 'PAX_NUMBER_FIELDS', 'stn', 'nts', 'nti', 'itn', 'calc_chksums',
2811 'copyfileobj', 'filemode', 'EmptyHeaderError',
2812 'TruncatedHeaderError', 'EOFHeaderError', 'InvalidHeaderError',
2813 'SubsequentHeaderError', 'ExFileObject', 'main'}
2814 support.check__all__(self, tarfile, not_exported=not_exported)
2815
2816 def test_useful_error_message_when_modules_missing(self):
2817 fname = os.path.join(os.path.dirname(__file__), 'archivetestdata', 'testtar.tar.xz')
2818 with self.assertRaises(tarfile.ReadError) as excinfo:
2819 error = tarfile.CompressionError('lzma module is not available'),
2820 with unittest.mock.patch.object(tarfile.TarFile, 'xzopen', side_effect=error):
2821 tarfile.open(fname)
2822
2823 self.assertIn(
2824 "\n- method xz: CompressionError('lzma module is not available')\n",
2825 str(excinfo.exception),
2826 )
2827
2828 @unittest.skipUnless(os_helper.can_symlink(), 'requires symlink support')
2829 @unittest.skipUnless(hasattr(os, 'chmod'), "missing os.chmod")
2830 @unittest.mock.patch('os.chmod')
2831 def test_deferred_directory_attributes_update(self, mock_chmod):
2832 # Regression test for gh-127987: setting attributes on arbitrary files
2833 tempdir = os.path.join(TEMPDIR, 'test127987')
2834 def mock_chmod_side_effect(path, mode, **kwargs):
2835 target_path = os.path.realpath(path)
2836 if os.path.commonpath([target_path, tempdir]) != tempdir:
2837 raise Exception("should not try to chmod anything outside the destination", target_path)
2838 mock_chmod.side_effect = mock_chmod_side_effect
2839
2840 outside_tree_dir = os.path.join(TEMPDIR, 'outside_tree_dir')
2841 with ArchiveMaker() as arc:
2842 arc.add('x', symlink_to='.')
2843 arc.add('x', type=tarfile.DIRTYPE, mode='?rwsrwsrwt')
2844 arc.add('x', symlink_to=outside_tree_dir)
2845
2846 os.makedirs(outside_tree_dir)
2847 try:
2848 arc.open().extractall(path=tempdir, filter='tar')
2849 finally:
2850 os_helper.rmtree(outside_tree_dir)
2851 os_helper.rmtree(tempdir)
2852
2853
2854 class CommandLineTest(unittest.TestCase):
2855
2856 def tarfilecmd(self, *args, **kwargs):
2857 rc, out, err = script_helper.assert_python_ok('-m', 'tarfile', *args,
2858 **kwargs)
2859 return out.replace(os.linesep.encode(), b'\n')
2860
2861 def tarfilecmd_failure(self, *args):
2862 return script_helper.assert_python_failure('-m', 'tarfile', *args)
2863
2864 def make_simple_tarfile(self, tar_name):
2865 files = [support.findfile('tokenize_tests.txt',
2866 subdir='tokenizedata'),
2867 support.findfile('tokenize_tests-no-coding-cookie-'
2868 'and-utf8-bom-sig-only.txt',
2869 subdir='tokenizedata')]
2870 self.addCleanup(os_helper.unlink, tar_name)
2871 with tarfile.open(tar_name, 'w') as tf:
2872 for tardata in files:
2873 tf.add(tardata, arcname=os.path.basename(tardata))
2874
2875 def make_evil_tarfile(self, tar_name):
2876 self.addCleanup(os_helper.unlink, tar_name)
2877 with tarfile.open(tar_name, 'w') as tf:
2878 benign = tarfile.TarInfo('benign')
2879 tf.addfile(benign, fileobj=io.BytesIO(b''))
2880 evil = tarfile.TarInfo('../evil')
2881 tf.addfile(evil, fileobj=io.BytesIO(b''))
2882
2883 def test_bad_use(self):
2884 rc, out, err = self.tarfilecmd_failure()
2885 self.assertEqual(out, b'')
2886 self.assertIn(b'usage', err.lower())
2887 self.assertIn(b'error', err.lower())
2888 self.assertIn(b'required', err.lower())
2889 rc, out, err = self.tarfilecmd_failure('-l', '')
2890 self.assertEqual(out, b'')
2891 self.assertNotEqual(err.strip(), b'')
2892
2893 def test_test_command(self):
2894 for tar_name in testtarnames:
2895 for opt in '-t', '--test':
2896 out = self.tarfilecmd(opt, tar_name)
2897 self.assertEqual(out, b'')
2898
2899 def test_test_command_verbose(self):
2900 for tar_name in testtarnames:
2901 for opt in '-v', '--verbose':
2902 out = self.tarfilecmd(opt, '-t', tar_name,
2903 PYTHONIOENCODING='utf-8')
2904 self.assertIn(b'is a tar archive.\n', out)
2905
2906 def test_test_command_invalid_file(self):
2907 zipname = support.findfile('zipdir.zip', subdir='archivetestdata')
2908 rc, out, err = self.tarfilecmd_failure('-t', zipname)
2909 self.assertIn(b' is not a tar archive.', err)
2910 self.assertEqual(out, b'')
2911 self.assertEqual(rc, 1)
2912
2913 for tar_name in testtarnames:
2914 with self.subTest(tar_name=tar_name):
2915 with open(tar_name, 'rb') as f:
2916 data = f.read()
2917 try:
2918 with open(tmpname, 'wb') as f:
2919 f.write(data[:511])
2920 rc, out, err = self.tarfilecmd_failure('-t', tmpname)
2921 self.assertEqual(out, b'')
2922 self.assertEqual(rc, 1)
2923 finally:
2924 os_helper.unlink(tmpname)
2925
2926 def test_list_command(self):
2927 for tar_name in testtarnames:
2928 with support.captured_stdout() as t:
2929 with tarfile.open(tar_name, 'r') as tf:
2930 tf.list(verbose=False)
2931 expected = t.getvalue().encode('ascii', 'backslashreplace')
2932 for opt in '-l', '--list':
2933 out = self.tarfilecmd(opt, tar_name,
2934 PYTHONIOENCODING='ascii')
2935 self.assertEqual(out, expected)
2936
2937 def test_list_command_verbose(self):
2938 for tar_name in testtarnames:
2939 with support.captured_stdout() as t:
2940 with tarfile.open(tar_name, 'r') as tf:
2941 tf.list(verbose=True)
2942 expected = t.getvalue().encode('ascii', 'backslashreplace')
2943 for opt in '-v', '--verbose':
2944 out = self.tarfilecmd(opt, '-l', tar_name,
2945 PYTHONIOENCODING='ascii')
2946 self.assertEqual(out, expected)
2947
2948 def test_list_command_invalid_file(self):
2949 zipname = support.findfile('zipdir.zip', subdir='archivetestdata')
2950 rc, out, err = self.tarfilecmd_failure('-l', zipname)
2951 self.assertIn(b' is not a tar archive.', err)
2952 self.assertEqual(out, b'')
2953 self.assertEqual(rc, 1)
2954
2955 def test_create_command(self):
2956 files = [support.findfile('tokenize_tests.txt',
2957 subdir='tokenizedata'),
2958 support.findfile('tokenize_tests-no-coding-cookie-'
2959 'and-utf8-bom-sig-only.txt',
2960 subdir='tokenizedata')]
2961 for opt in '-c', '--create':
2962 try:
2963 out = self.tarfilecmd(opt, tmpname, *files)
2964 self.assertEqual(out, b'')
2965 with tarfile.open(tmpname) as tar:
2966 tar.getmembers()
2967 finally:
2968 os_helper.unlink(tmpname)
2969
2970 def test_create_command_verbose(self):
2971 files = [support.findfile('tokenize_tests.txt',
2972 subdir='tokenizedata'),
2973 support.findfile('tokenize_tests-no-coding-cookie-'
2974 'and-utf8-bom-sig-only.txt',
2975 subdir='tokenizedata')]
2976 for opt in '-v', '--verbose':
2977 try:
2978 out = self.tarfilecmd(opt, '-c', tmpname, *files,
2979 PYTHONIOENCODING='utf-8')
2980 self.assertIn(b' file created.', out)
2981 with tarfile.open(tmpname) as tar:
2982 tar.getmembers()
2983 finally:
2984 os_helper.unlink(tmpname)
2985
2986 def test_create_command_dotless_filename(self):
2987 files = [support.findfile('tokenize_tests.txt', subdir='tokenizedata')]
2988 try:
2989 out = self.tarfilecmd('-c', dotlessname, *files)
2990 self.assertEqual(out, b'')
2991 with tarfile.open(dotlessname) as tar:
2992 tar.getmembers()
2993 finally:
2994 os_helper.unlink(dotlessname)
2995
2996 def test_create_command_dot_started_filename(self):
2997 tar_name = os.path.join(TEMPDIR, ".testtar")
2998 files = [support.findfile('tokenize_tests.txt', subdir='tokenizedata')]
2999 try:
3000 out = self.tarfilecmd('-c', tar_name, *files)
3001 self.assertEqual(out, b'')
3002 with tarfile.open(tar_name) as tar:
3003 tar.getmembers()
3004 finally:
3005 os_helper.unlink(tar_name)
3006
3007 def test_create_command_compressed(self):
3008 files = [support.findfile('tokenize_tests.txt',
3009 subdir='tokenizedata'),
3010 support.findfile('tokenize_tests-no-coding-cookie-'
3011 'and-utf8-bom-sig-only.txt',
3012 subdir='tokenizedata')]
3013 for filetype in (GzipTest, Bz2Test, LzmaTest, ZstdTest):
3014 if not filetype.open:
3015 continue
3016 try:
3017 tar_name = tmpname + '.' + filetype.suffix
3018 out = self.tarfilecmd('-c', tar_name, *files)
3019 with filetype.taropen(tar_name) as tar:
3020 tar.getmembers()
3021 finally:
3022 os_helper.unlink(tar_name)
3023
3024 def test_extract_command(self):
3025 self.make_simple_tarfile(tmpname)
3026 for opt in '-e', '--extract':
3027 try:
3028 with os_helper.temp_cwd(tarextdir):
3029 out = self.tarfilecmd(opt, tmpname)
3030 self.assertEqual(out, b'')
3031 finally:
3032 os_helper.rmtree(tarextdir)
3033
3034 def test_extract_command_verbose(self):
3035 self.make_simple_tarfile(tmpname)
3036 for opt in '-v', '--verbose':
3037 try:
3038 with os_helper.temp_cwd(tarextdir):
3039 out = self.tarfilecmd(opt, '-e', tmpname,
3040 PYTHONIOENCODING='utf-8')
3041 self.assertIn(b' file is extracted.', out)
3042 finally:
3043 os_helper.rmtree(tarextdir)
3044
3045 def test_extract_command_filter(self):
3046 self.make_evil_tarfile(tmpname)
3047 # Make an inner directory, so the member named '../evil'
3048 # is still extracted into `tarextdir`
3049 destdir = os.path.join(tarextdir, 'dest')
3050 os.mkdir(tarextdir)
3051 try:
3052 with os_helper.temp_cwd(destdir):
3053 self.tarfilecmd_failure('-e', tmpname,
3054 '-v',
3055 '--filter', 'data')
3056 out = self.tarfilecmd('-e', tmpname,
3057 '-v',
3058 '--filter', 'fully_trusted',
3059 PYTHONIOENCODING='utf-8')
3060 self.assertIn(b' file is extracted.', out)
3061 finally:
3062 os_helper.rmtree(tarextdir)
3063
3064 def test_extract_command_different_directory(self):
3065 self.make_simple_tarfile(tmpname)
3066 try:
3067 with os_helper.temp_cwd(tarextdir):
3068 out = self.tarfilecmd('-e', tmpname, 'spamdir')
3069 self.assertEqual(out, b'')
3070 finally:
3071 os_helper.rmtree(tarextdir)
3072
3073 def test_extract_command_invalid_file(self):
3074 zipname = support.findfile('zipdir.zip', subdir='archivetestdata')
3075 with os_helper.temp_cwd(tarextdir):
3076 rc, out, err = self.tarfilecmd_failure('-e', zipname)
3077 self.assertIn(b' is not a tar archive.', err)
3078 self.assertEqual(out, b'')
3079 self.assertEqual(rc, 1)
3080
3081
3082 class ContextManagerTest(unittest.TestCase):
3083
3084 def test_basic(self):
3085 with tarfile.open(tarname) as tar:
3086 self.assertFalse(tar.closed, "closed inside runtime context")
3087 self.assertTrue(tar.closed, "context manager failed")
3088
3089 def test_closed(self):
3090 # The __enter__() method is supposed to raise OSError
3091 # if the TarFile object is already closed.
3092 tar = tarfile.open(tarname)
3093 tar.close()
3094 with self.assertRaises(OSError):
3095 with tar:
3096 pass
3097
3098 def test_exception(self):
3099 # Test if the OSError exception is passed through properly.
3100 with self.assertRaises(Exception) as exc:
3101 with tarfile.open(tarname) as tar:
3102 raise OSError
3103 self.assertIsInstance(exc.exception, OSError,
3104 "wrong exception raised in context manager")
3105 self.assertTrue(tar.closed, "context manager failed")
3106
3107 def test_no_eof(self):
3108 # __exit__() must not write end-of-archive blocks if an
3109 # exception was raised.
3110 try:
3111 with tarfile.open(tmpname, "w") as tar:
3112 raise Exception
3113 except:
3114 pass
3115 self.assertEqual(os.path.getsize(tmpname), 0,
3116 "context manager wrote an end-of-archive block")
3117 self.assertTrue(tar.closed, "context manager failed")
3118
3119 def test_eof(self):
3120 # __exit__() must write end-of-archive blocks, i.e. call
3121 # TarFile.close() if there was no error.
3122 with tarfile.open(tmpname, "w"):
3123 pass
3124 self.assertNotEqual(os.path.getsize(tmpname), 0,
3125 "context manager wrote no end-of-archive block")
3126
3127 def test_fileobj(self):
3128 # Test that __exit__() did not close the external file
3129 # object.
3130 with open(tmpname, "wb") as fobj:
3131 try:
3132 with tarfile.open(fileobj=fobj, mode="w") as tar:
3133 raise Exception
3134 except:
3135 pass
3136 self.assertFalse(fobj.closed, "external file object was closed")
3137 self.assertTrue(tar.closed, "context manager failed")
3138
3139
3140 @unittest.skipIf(hasattr(os, "link"), "requires os.link to be missing")
3141 class LinkEmulationTest(ReadTest, unittest.TestCase):
3142
3143 # Test for issue #8741 regression. On platforms that do not support
3144 # symbolic or hard links tarfile tries to extract these types of members
3145 # as the regular files they point to.
3146 def _test_link_extraction(self, name):
3147 self.tar.extract(name, TEMPDIR, filter='fully_trusted')
3148 with open(os.path.join(TEMPDIR, name), "rb") as f:
3149 data = f.read()
3150 self.assertEqual(sha256sum(data), sha256_regtype)
3151
3152 # See issues #1578269, #8879, and #17689 for some history on these skips
3153 @unittest.skipIf(hasattr(os.path, "islink"),
3154 "Skip emulation - has os.path.islink but not os.link")
3155 def test_hardlink_extraction1(self):
3156 self._test_link_extraction("ustar/lnktype")
3157
3158 @unittest.skipIf(hasattr(os.path, "islink"),
3159 "Skip emulation - has os.path.islink but not os.link")
3160 def test_hardlink_extraction2(self):
3161 self._test_link_extraction("./ustar/linktest2/lnktype")
3162
3163 @unittest.skipIf(hasattr(os, "symlink"),
3164 "Skip emulation if symlink exists")
3165 def test_symlink_extraction1(self):
3166 self._test_link_extraction("ustar/symtype")
3167
3168 @unittest.skipIf(hasattr(os, "symlink"),
3169 "Skip emulation if symlink exists")
3170 def test_symlink_extraction2(self):
3171 self._test_link_extraction("./ustar/linktest2/symtype")
3172
3173
3174 class Bz2PartialReadTest(Bz2Test, unittest.TestCase):
3175 # Issue5068: The _BZ2Proxy.read() method loops forever
3176 # on an empty or partial bzipped file.
3177
3178 def _test_partial_input(self, mode):
3179 class MyBytesIO(io.BytesIO):
3180 hit_eof = False
3181 def read(self, n):
3182 if self.hit_eof:
3183 raise AssertionError("infinite loop detected in "
3184 "tarfile.open()")
3185 self.hit_eof = self.tell() == len(self.getvalue())
3186 return super(MyBytesIO, self).read(n)
3187 def seek(self, *args):
3188 self.hit_eof = False
3189 return super(MyBytesIO, self).seek(*args)
3190
3191 data = bz2.compress(tarfile.TarInfo("foo").tobuf())
3192 for x in range(len(data) + 1):
3193 try:
3194 tarfile.open(fileobj=MyBytesIO(data[:x]), mode=mode)
3195 except tarfile.ReadError:
3196 pass # we have no interest in ReadErrors
3197
3198 def test_partial_input(self):
3199 self._test_partial_input("r")
3200
3201 def test_partial_input_bz2(self):
3202 self._test_partial_input("r:bz2")
3203
3204
3205 def root_is_uid_gid_0():
3206 try:
3207 import pwd, grp
3208 except ImportError:
3209 return False
3210 if pwd.getpwuid(0)[0] != 'root':
3211 return False
3212 if grp.getgrgid(0)[0] != 'root':
3213 return False
3214 return True
3215
3216
3217 @unittest.skipUnless(hasattr(os, 'chown'), "missing os.chown")
3218 @unittest.skipUnless(hasattr(os, 'geteuid'), "missing os.geteuid")
3219 class NumericOwnerTest(unittest.TestCase):
3220 # mock the following:
3221 # os.chown: so we can test what's being called
3222 # os.chmod: so the modes are not actually changed. if they are, we can't
3223 # delete the files/directories
3224 # os.geteuid: so we can lie and say we're root (uid = 0)
3225
3226 @staticmethod
3227 def _make_test_archive(filename_1, dirname_1, filename_2):
3228 # the file contents to write
3229 fobj = io.BytesIO(b"content")
3230
3231 # create a tar file with a file, a directory, and a file within that
3232 # directory. Assign various .uid/.gid values to them
3233 items = [(filename_1, 99, 98, tarfile.REGTYPE, fobj),
3234 (dirname_1, 77, 76, tarfile.DIRTYPE, None),
3235 (filename_2, 88, 87, tarfile.REGTYPE, fobj),
3236 ]
3237 with tarfile.open(tmpname, 'w') as tarfl:
3238 for name, uid, gid, typ, contents in items:
3239 t = tarfile.TarInfo(name)
3240 t.uid = uid
3241 t.gid = gid
3242 t.uname = 'root'
3243 t.gname = 'root'
3244 t.type = typ
3245 tarfl.addfile(t, contents)
3246
3247 # return the full pathname to the tar file
3248 return tmpname
3249
3250 @staticmethod
3251 @contextmanager
3252 def _setup_test(mock_geteuid):
3253 mock_geteuid.return_value = 0 # lie and say we're root
3254 fname = 'numeric-owner-testfile'
3255 dirname = 'dir'
3256
3257 # the names we want stored in the tarfile
3258 filename_1 = fname
3259 dirname_1 = dirname
3260 filename_2 = os.path.join(dirname, fname)
3261
3262 # create the tarfile with the contents we're after
3263 tar_filename = NumericOwnerTest._make_test_archive(filename_1,
3264 dirname_1,
3265 filename_2)
3266
3267 # open the tarfile for reading. yield it and the names of the items
3268 # we stored into the file
3269 with tarfile.open(tar_filename) as tarfl:
3270 yield tarfl, filename_1, dirname_1, filename_2
3271
3272 @unittest.mock.patch('os.chown')
3273 @unittest.mock.patch('os.chmod')
3274 @unittest.mock.patch('os.geteuid')
3275 def test_extract_with_numeric_owner(self, mock_geteuid, mock_chmod,
3276 mock_chown):
3277 with self._setup_test(mock_geteuid) as (tarfl, filename_1, _,
3278 filename_2):
3279 tarfl.extract(filename_1, TEMPDIR, numeric_owner=True,
3280 filter='fully_trusted')
3281 tarfl.extract(filename_2 , TEMPDIR, numeric_owner=True,
3282 filter='fully_trusted')
3283
3284 # convert to filesystem paths
3285 f_filename_1 = os.path.join(TEMPDIR, filename_1)
3286 f_filename_2 = os.path.join(TEMPDIR, filename_2)
3287
3288 mock_chown.assert_has_calls([unittest.mock.call(f_filename_1, 99, 98),
3289 unittest.mock.call(f_filename_2, 88, 87),
3290 ],
3291 any_order=True)
3292
3293 @unittest.mock.patch('os.chown')
3294 @unittest.mock.patch('os.chmod')
3295 @unittest.mock.patch('os.geteuid')
3296 def test_extractall_with_numeric_owner(self, mock_geteuid, mock_chmod,
3297 mock_chown):
3298 with self._setup_test(mock_geteuid) as (tarfl, filename_1, dirname_1,
3299 filename_2):
3300 tarfl.extractall(TEMPDIR, numeric_owner=True,
3301 filter='fully_trusted')
3302
3303 # convert to filesystem paths
3304 f_filename_1 = os.path.join(TEMPDIR, filename_1)
3305 f_dirname_1 = os.path.join(TEMPDIR, dirname_1)
3306 f_filename_2 = os.path.join(TEMPDIR, filename_2)
3307
3308 mock_chown.assert_has_calls([unittest.mock.call(f_filename_1, 99, 98),
3309 unittest.mock.call(f_dirname_1, 77, 76),
3310 unittest.mock.call(f_filename_2, 88, 87),
3311 ],
3312 any_order=True)
3313
3314 # this test requires that uid=0 and gid=0 really be named 'root'. that's
3315 # because the uname and gname in the test file are 'root', and extract()
3316 # will look them up using pwd and grp to find their uid and gid, which we
3317 # test here to be 0.
3318 @unittest.skipUnless(root_is_uid_gid_0(),
3319 'uid=0,gid=0 must be named "root"')
3320 @unittest.mock.patch('os.chown')
3321 @unittest.mock.patch('os.chmod')
3322 @unittest.mock.patch('os.geteuid')
3323 def test_extract_without_numeric_owner(self, mock_geteuid, mock_chmod,
3324 mock_chown):
3325 with self._setup_test(mock_geteuid) as (tarfl, filename_1, _, _):
3326 tarfl.extract(filename_1, TEMPDIR, numeric_owner=False,
3327 filter='fully_trusted')
3328
3329 # convert to filesystem paths
3330 f_filename_1 = os.path.join(TEMPDIR, filename_1)
3331
3332 mock_chown.assert_called_with(f_filename_1, 0, 0)
3333
3334 @unittest.mock.patch('os.geteuid')
3335 def test_keyword_only(self, mock_geteuid):
3336 with self._setup_test(mock_geteuid) as (tarfl, filename_1, _, _):
3337 self.assertRaises(TypeError,
3338 tarfl.extract, filename_1, TEMPDIR, False, True)
3339
3340
3341 class ReplaceTests(ReadTest, unittest.TestCase):
3342 def test_replace_name(self):
3343 member = self.tar.getmember('ustar/regtype')
3344 replaced = member.replace(name='misc/other')
3345 self.assertEqual(replaced.name, 'misc/other')
3346 self.assertEqual(member.name, 'ustar/regtype')
3347 self.assertEqual(self.tar.getmember('ustar/regtype').name,
3348 'ustar/regtype')
3349
3350 def test_replace_deep(self):
3351 member = self.tar.getmember('pax/regtype1')
3352 replaced = member.replace()
3353 replaced.pax_headers['gname'] = 'not-bar'
3354 self.assertEqual(member.pax_headers['gname'], 'bar')
3355 self.assertEqual(
3356 self.tar.getmember('pax/regtype1').pax_headers['gname'], 'bar')
3357
3358 def test_replace_shallow(self):
3359 member = self.tar.getmember('pax/regtype1')
3360 replaced = member.replace(deep=False)
3361 replaced.pax_headers['gname'] = 'not-bar'
3362 self.assertEqual(member.pax_headers['gname'], 'not-bar')
3363 self.assertEqual(
3364 self.tar.getmember('pax/regtype1').pax_headers['gname'], 'not-bar')
3365
3366 def test_replace_all(self):
3367 member = self.tar.getmember('ustar/regtype')
3368 for attr_name in ('name', 'mtime', 'mode', 'linkname',
3369 'uid', 'gid', 'uname', 'gname'):
3370 with self.subTest(attr_name=attr_name):
3371 replaced = member.replace(**{attr_name: None})
3372 self.assertEqual(getattr(replaced, attr_name), None)
3373 self.assertNotEqual(getattr(member, attr_name), None)
3374
3375 def test_replace_internal(self):
3376 member = self.tar.getmember('ustar/regtype')
3377 with self.assertRaises(TypeError):
3378 member.replace(offset=123456789)
3379
3380
3381 class NoneInfoExtractTests(ReadTest):
3382 # These mainly check that all kinds of members are extracted successfully
3383 # if some metadata is None.
3384 # Some of the methods do additional spot checks.
3385
3386 # We also test that the default filters can deal with None.
3387
3388 extraction_filter = None
3389
3390 @classmethod
3391 def setUpClass(cls):
3392 tar = tarfile.open(tarname, mode='r', encoding="iso8859-1")
3393 cls.control_dir = pathlib.Path(TEMPDIR) / "extractall_ctrl"
3394 tar.errorlevel = 0
3395 with ExitStack() as cm:
3396 if cls.extraction_filter is None:
3397 cm.enter_context(warnings.catch_warnings(
3398 action="ignore", category=DeprecationWarning))
3399 tar.extractall(cls.control_dir, filter=cls.extraction_filter)
3400 tar.close()
3401 cls.control_paths = set(
3402 p.relative_to(cls.control_dir)
3403 for p in pathlib.Path(cls.control_dir).glob('**/*'))
3404
3405 @classmethod
3406 def tearDownClass(cls):
3407 shutil.rmtree(cls.control_dir)
3408
3409 def check_files_present(self, directory):
3410 got_paths = set(
3411 p.relative_to(directory)
3412 for p in pathlib.Path(directory).glob('**/*'))
3413 if self.extraction_filter in (None, 'data'):
3414 # The 'data' filter is expected to reject special files
3415 for path in 'ustar/fifotype', 'ustar/blktype', 'ustar/chrtype':
3416 got_paths.discard(pathlib.Path(path))
3417 self.assertEqual(self.control_paths, got_paths)
3418
3419 @contextmanager
3420 def extract_with_none(self, *attr_names):
3421 DIR = pathlib.Path(TEMPDIR) / "extractall_none"
3422 self.tar.errorlevel = 0
3423 for member in self.tar.getmembers():
3424 for attr_name in attr_names:
3425 setattr(member, attr_name, None)
3426 with os_helper.temp_dir(DIR):
3427 self.tar.extractall(DIR, filter='fully_trusted')
3428 self.check_files_present(DIR)
3429 yield DIR
3430
3431 def test_extractall_none_mtime(self):
3432 # mtimes of extracted files should be later than 'now' -- the mtime
3433 # of a previously created directory.
3434 now = pathlib.Path(TEMPDIR).stat().st_mtime
3435 with self.extract_with_none('mtime') as DIR:
3436 for path in pathlib.Path(DIR).glob('**/*'):
3437 with self.subTest(path=path):
3438 try:
3439 mtime = path.stat().st_mtime
3440 except OSError:
3441 # Some systems can't stat symlinks, ignore those
3442 if not path.is_symlink():
3443 raise
3444 else:
3445 self.assertGreaterEqual(path.stat().st_mtime, now)
3446
3447 def test_extractall_none_mode(self):
3448 # modes of directories and regular files should match the mode
3449 # of a "normally" created directory or regular file
3450 dir_mode = pathlib.Path(TEMPDIR).stat().st_mode
3451 regular_file = pathlib.Path(TEMPDIR) / 'regular_file'
3452 regular_file.write_text('')
3453 regular_file_mode = regular_file.stat().st_mode
3454 with self.extract_with_none('mode') as DIR:
3455 for path in pathlib.Path(DIR).glob('**/*'):
3456 with self.subTest(path=path):
3457 if path.is_dir():
3458 self.assertEqual(path.stat().st_mode, dir_mode)
3459 elif path.is_file():
3460 self.assertEqual(path.stat().st_mode,
3461 regular_file_mode)
3462
3463 def test_extractall_none_uid(self):
3464 with self.extract_with_none('uid'):
3465 pass
3466
3467 def test_extractall_none_gid(self):
3468 with self.extract_with_none('gid'):
3469 pass
3470
3471 def test_extractall_none_uname(self):
3472 with self.extract_with_none('uname'):
3473 pass
3474
3475 def test_extractall_none_gname(self):
3476 with self.extract_with_none('gname'):
3477 pass
3478
3479 def test_extractall_none_ownership(self):
3480 with self.extract_with_none('uid', 'gid', 'uname', 'gname'):
3481 pass
3482
3483 class NoneInfoExtractTests_Data(NoneInfoExtractTests, unittest.TestCase):
3484 extraction_filter = 'data'
3485
3486 class NoneInfoExtractTests_FullyTrusted(NoneInfoExtractTests,
3487 unittest.TestCase):
3488 extraction_filter = 'fully_trusted'
3489
3490 class NoneInfoExtractTests_Tar(NoneInfoExtractTests, unittest.TestCase):
3491 extraction_filter = 'tar'
3492
3493 class NoneInfoExtractTests_Default(NoneInfoExtractTests,
3494 unittest.TestCase):
3495 extraction_filter = None
3496
3497 class NoneInfoTests_Misc(unittest.TestCase):
3498 def test_add(self):
3499 # When addfile() encounters None metadata, it raises a ValueError
3500 bio = io.BytesIO()
3501 for tarformat in (tarfile.USTAR_FORMAT, tarfile.GNU_FORMAT,
3502 tarfile.PAX_FORMAT):
3503 with self.subTest(tarformat=tarformat):
3504 tar = tarfile.open(fileobj=bio, mode='w', format=tarformat)
3505 tarinfo = tar.gettarinfo(tarname)
3506 try:
3507 with open(tarname, 'rb') as f:
3508 tar.addfile(tarinfo, f)
3509 except Exception:
3510 if tarformat == tarfile.USTAR_FORMAT:
3511 # In the old, limited format, adding might fail for
3512 # reasons like the UID being too large
3513 pass
3514 else:
3515 raise
3516 else:
3517 for attr_name in ('mtime', 'mode', 'uid', 'gid',
3518 'uname', 'gname'):
3519 with self.subTest(attr_name=attr_name):
3520 replaced = tarinfo.replace(**{attr_name: None})
3521 with self.assertRaisesRegex(ValueError,
3522 f"{attr_name}"):
3523 with open(tarname, 'rb') as f:
3524 tar.addfile(replaced, f)
3525
3526 def test_list(self):
3527 # Change some metadata to None, then compare list() output
3528 # word-for-word. We want list() to not raise, and to only change
3529 # printout for the affected piece of metadata.
3530 # (n.b.: some contents of the test archive are hardcoded.)
3531 for attr_names in ({'mtime'}, {'mode'}, {'uid'}, {'gid'},
3532 {'uname'}, {'gname'},
3533 {'uid', 'uname'}, {'gid', 'gname'}):
3534 with (self.subTest(attr_names=attr_names),
3535 tarfile.open(tarname, encoding="iso8859-1") as tar):
3536 tio_prev = io.TextIOWrapper(io.BytesIO(), 'ascii', newline='\n')
3537 with support.swap_attr(sys, 'stdout', tio_prev):
3538 tar.list()
3539 for member in tar.getmembers():
3540 for attr_name in attr_names:
3541 setattr(member, attr_name, None)
3542 tio_new = io.TextIOWrapper(io.BytesIO(), 'ascii', newline='\n')
3543 with support.swap_attr(sys, 'stdout', tio_new):
3544 tar.list()
3545 for expected, got in zip(tio_prev.detach().getvalue().split(),
3546 tio_new.detach().getvalue().split()):
3547 if attr_names == {'mtime'} and re.match(rb'2003-01-\d\d', expected):
3548 self.assertEqual(got, b'????-??-??')
3549 elif attr_names == {'mtime'} and re.match(rb'\d\d:\d\d:\d\d', expected):
3550 self.assertEqual(got, b'??:??:??')
3551 elif attr_names == {'mode'} and re.match(
3552 rb'.([r-][w-][x-]){3}', expected):
3553 self.assertEqual(got, b'??????????')
3554 elif attr_names == {'uname'} and expected.startswith(
3555 (b'tarfile/', b'lars/', b'foo/')):
3556 exp_user, exp_group = expected.split(b'/')
3557 got_user, got_group = got.split(b'/')
3558 self.assertEqual(got_group, exp_group)
3559 self.assertRegex(got_user, b'[0-9]+')
3560 elif attr_names == {'gname'} and expected.endswith(
3561 (b'/tarfile', b'/users', b'/bar')):
3562 exp_user, exp_group = expected.split(b'/')
3563 got_user, got_group = got.split(b'/')
3564 self.assertEqual(got_user, exp_user)
3565 self.assertRegex(got_group, b'[0-9]+')
3566 elif attr_names == {'uid'} and expected.startswith(
3567 (b'1000/')):
3568 exp_user, exp_group = expected.split(b'/')
3569 got_user, got_group = got.split(b'/')
3570 self.assertEqual(got_group, exp_group)
3571 self.assertEqual(got_user, b'None')
3572 elif attr_names == {'gid'} and expected.endswith((b'/100')):
3573 exp_user, exp_group = expected.split(b'/')
3574 got_user, got_group = got.split(b'/')
3575 self.assertEqual(got_user, exp_user)
3576 self.assertEqual(got_group, b'None')
3577 elif attr_names == {'uid', 'uname'} and expected.startswith(
3578 (b'tarfile/', b'lars/', b'foo/', b'1000/')):
3579 exp_user, exp_group = expected.split(b'/')
3580 got_user, got_group = got.split(b'/')
3581 self.assertEqual(got_group, exp_group)
3582 self.assertEqual(got_user, b'None')
3583 elif attr_names == {'gname', 'gid'} and expected.endswith(
3584 (b'/tarfile', b'/users', b'/bar', b'/100')):
3585 exp_user, exp_group = expected.split(b'/')
3586 got_user, got_group = got.split(b'/')
3587 self.assertEqual(got_user, exp_user)
3588 self.assertEqual(got_group, b'None')
3589 else:
3590 # In other cases the output should be the same
3591 self.assertEqual(expected, got)
3592
3593 def _filemode_to_int(mode):
3594 """Inverse of `stat.filemode` (for permission bits)
3595
3596 Using mode strings rather than numbers makes the later tests more readable.
3597 """
3598 str_mode = mode[1:]
3599 result = (
3600 {'r': stat.S_IRUSR, '-': 0}[str_mode[0]]
3601 | {'w': stat.S_IWUSR, '-': 0}[str_mode[1]]
3602 | {'x': stat.S_IXUSR, '-': 0,
3603 's': stat.S_IXUSR | stat.S_ISUID,
3604 'S': stat.S_ISUID}[str_mode[2]]
3605 | {'r': stat.S_IRGRP, '-': 0}[str_mode[3]]
3606 | {'w': stat.S_IWGRP, '-': 0}[str_mode[4]]
3607 | {'x': stat.S_IXGRP, '-': 0,
3608 's': stat.S_IXGRP | stat.S_ISGID,
3609 'S': stat.S_ISGID}[str_mode[5]]
3610 | {'r': stat.S_IROTH, '-': 0}[str_mode[6]]
3611 | {'w': stat.S_IWOTH, '-': 0}[str_mode[7]]
3612 | {'x': stat.S_IXOTH, '-': 0,
3613 't': stat.S_IXOTH | stat.S_ISVTX,
3614 'T': stat.S_ISVTX}[str_mode[8]]
3615 )
3616 # check we did this right
3617 assert stat.filemode(result)[1:] == mode[1:]
3618
3619 return result
3620
3621 class ArchiveMaker:
3622 """Helper to create a tar file with specific contents
3623
3624 Usage:
3625
3626 with ArchiveMaker() as t:
3627 t.add('filename', ...)
3628
3629 with t.open() as tar:
3630 ... # `tar` is now a TarFile with 'filename' in it!
3631 """
3632 def __init__(self, **kwargs):
3633 self.bio = io.BytesIO()
3634 self.tar_kwargs = dict(kwargs)
3635
3636 def __enter__(self):
3637 self.tar_w = tarfile.TarFile(mode='w', fileobj=self.bio, **self.tar_kwargs)
3638 return self
3639
3640 def __exit__(self, *exc):
3641 self.tar_w.close()
3642 self.contents = self.bio.getvalue()
3643 self.bio = None
3644
3645 def add(self, name, *, type=None, symlink_to=None, hardlink_to=None,
3646 mode=None, size=None, content=None, **kwargs):
3647 """Add a member to the test archive. Call within `with`.
3648
3649 Provides many shortcuts:
3650 - default `type` is based on symlink_to, hardlink_to, and trailing `/`
3651 in name (which is stripped)
3652 - size & content defaults are based on each other
3653 - content can be str or bytes
3654 - mode should be textual ('-rwxrwxrwx')
3655
3656 (add more! this is unstable internal test-only API)
3657 """
3658 name = str(name)
3659 tarinfo = tarfile.TarInfo(name).replace(**kwargs)
3660 if content is not None:
3661 if isinstance(content, str):
3662 content = content.encode()
3663 size = len(content)
3664 if size is not None:
3665 tarinfo.size = size
3666 if content is None:
3667 content = bytes(tarinfo.size)
3668 if mode:
3669 tarinfo.mode = _filemode_to_int(mode)
3670 if symlink_to is not None:
3671 type = tarfile.SYMTYPE
3672 tarinfo.linkname = str(symlink_to)
3673 if hardlink_to is not None:
3674 type = tarfile.LNKTYPE
3675 tarinfo.linkname = str(hardlink_to)
3676 if name.endswith('/') and type is None:
3677 type = tarfile.DIRTYPE
3678 if type is not None:
3679 tarinfo.type = type
3680 if tarinfo.isreg():
3681 fileobj = io.BytesIO(content)
3682 else:
3683 fileobj = None
3684 self.tar_w.addfile(tarinfo, fileobj)
3685
3686 def open(self, **kwargs):
3687 """Open the resulting archive as TarFile. Call after `with`."""
3688 bio = io.BytesIO(self.contents)
3689 return tarfile.open(fileobj=bio, **kwargs)
3690
3691 # Under WASI, `os_helper.can_symlink` is False to make
3692 # `skip_unless_symlink` skip symlink tests. "
3693 # But in the following tests we use can_symlink to *determine* which
3694 # behavior is expected.
3695 # Like other symlink tests, skip these on WASI for now.
3696 if support.is_wasi:
3697 def symlink_test(f):
3698 return unittest.skip("WASI: Skip symlink test for now")(f)
3699 else:
3700 def symlink_test(f):
3701 return f
3702
3703
3704 class TestExtractionFilters(unittest.TestCase):
3705
3706 # A temporary directory for the extraction results.
3707 # All files that "escape" the destination path should still end
3708 # up in this directory.
3709 outerdir = pathlib.Path(TEMPDIR) / 'outerdir'
3710
3711 # The destination for the extraction, within `outerdir`
3712 destdir = outerdir / 'dest'
3713
3714 @classmethod
3715 def setUpClass(cls):
3716 # Posix and Windows have different pathname resolution:
3717 # either symlink or a '..' component resolve first.
3718 # Let's see which we are on.
3719 if os_helper.can_symlink():
3720 testpath = os.path.join(TEMPDIR, 'resolution_test')
3721 os.mkdir(testpath)
3722
3723 # testpath/current links to `.` which is all of:
3724 # - `testpath`
3725 # - `testpath/current`
3726 # - `testpath/current/current`
3727 # - etc.
3728 os.symlink('.', os.path.join(testpath, 'current'))
3729
3730 # we'll test where `testpath/current/../file` ends up
3731 with open(os.path.join(testpath, 'current', '..', 'file'), 'w'):
3732 pass
3733
3734 if os.path.exists(os.path.join(testpath, 'file')):
3735 # Windows collapses 'current\..' to '.' first, leaving
3736 # 'testpath\file'
3737 cls.dotdot_resolves_early = True
3738 elif os.path.exists(os.path.join(testpath, '..', 'file')):
3739 # Posix resolves 'current' to '.' first, leaving
3740 # 'testpath/../file'
3741 cls.dotdot_resolves_early = False
3742 else:
3743 raise AssertionError('Could not determine link resolution')
3744 else:
3745 cls.dotdot_resolves_early = False
3746
3747 @contextmanager
3748 def check_context(self, tar, filter, *, check_flag=True):
3749 """Extracts `tar` to `self.destdir` and allows checking the result
3750
3751 If an error occurs, it must be checked using `expect_exception`
3752
3753 Otherwise, all resulting files must be checked using `expect_file`,
3754 except the destination directory itself and parent directories of
3755 other files.
3756 When checking directories, do so before their contents.
3757
3758 A file called 'flag' is made in outerdir (i.e. outside destdir)
3759 before extraction; it should not be altered nor should its contents
3760 be read/copied.
3761 """
3762 with os_helper.temp_dir(self.outerdir):
3763 flag_path = self.outerdir / 'flag'
3764 flag_path.write_text('capture me')
3765 try:
3766 tar.extractall(self.destdir, filter=filter)
3767 except Exception as exc:
3768 self.raised_exception = exc
3769 self.reraise_exception = True
3770 self.expected_paths = set()
3771 else:
3772 self.raised_exception = None
3773 self.reraise_exception = False
3774 self.expected_paths = set(self.outerdir.glob('**/*'))
3775 self.expected_paths.discard(self.destdir)
3776 self.expected_paths.discard(flag_path)
3777 try:
3778 yield self
3779 finally:
3780 tar.close()
3781 if self.reraise_exception:
3782 raise self.raised_exception
3783 self.assertEqual(self.expected_paths, set())
3784 if check_flag:
3785 self.assertEqual(flag_path.read_text(), 'capture me')
3786 else:
3787 assert filter == 'fully_trusted'
3788
3789 def expect_file(self, name, type=None, symlink_to=None, mode=None,
3790 size=None, content=None):
3791 """Check a single file. See check_context."""
3792 if self.raised_exception:
3793 raise self.raised_exception
3794 # use normpath() rather than resolve() so we don't follow symlinks
3795 path = pathlib.Path(os.path.normpath(self.destdir / name))
3796 self.assertIn(path, self.expected_paths)
3797 self.expected_paths.remove(path)
3798 if mode is not None and os_helper.can_chmod() and os.name != 'nt':
3799 got = stat.filemode(stat.S_IMODE(path.stat().st_mode))
3800 self.assertEqual(got, mode)
3801 if type is None and isinstance(name, str) and name.endswith('/'):
3802 type = tarfile.DIRTYPE
3803 if symlink_to is not None:
3804 got = (self.destdir / name).readlink()
3805 expected = pathlib.Path(symlink_to)
3806 # The symlink might be the same (textually) as what we expect,
3807 # but some systems change the link to an equivalent path, so
3808 # we fall back to samefile().
3809 try:
3810 if expected != got:
3811 self.assertTrue(got.samefile(expected))
3812 except Exception as e:
3813 # attach a note, so it's shown even if `samefile` fails
3814 e.add_note(f'{expected=}, {got=}')
3815 raise
3816 elif type == tarfile.REGTYPE or type is None:
3817 self.assertTrue(path.is_file())
3818 elif type == tarfile.DIRTYPE:
3819 self.assertTrue(path.is_dir())
3820 elif type == tarfile.FIFOTYPE:
3821 self.assertTrue(path.is_fifo())
3822 elif type == tarfile.SYMTYPE:
3823 self.assertTrue(path.is_symlink())
3824 else:
3825 raise NotImplementedError(type)
3826 if size is not None:
3827 self.assertEqual(path.stat().st_size, size)
3828 if content is not None:
3829 self.assertEqual(path.read_text(), content)
3830 for parent in path.parents:
3831 self.expected_paths.discard(parent)
3832
3833 def expect_any_tree(self, name):
3834 """Check a directory; forget about its contents."""
3835 tree_path = (self.destdir / name).resolve()
3836 self.expect_file(tree_path, type=tarfile.DIRTYPE)
3837 self.expected_paths = {
3838 p for p in self.expected_paths
3839 if tree_path not in p.parents
3840 }
3841
3842 def expect_exception(self, exc_type, message_re='.'):
3843 with self.assertRaisesRegex(exc_type, message_re):
3844 if self.raised_exception is not None:
3845 raise self.raised_exception
3846 self.reraise_exception = False
3847 return self.raised_exception
3848
3849 def test_benign_file(self):
3850 with ArchiveMaker() as arc:
3851 arc.add('benign.txt')
3852 for filter in 'fully_trusted', 'tar', 'data':
3853 with self.check_context(arc.open(), filter):
3854 self.expect_file('benign.txt')
3855
3856 def test_absolute(self):
3857 # Test handling a member with an absolute path
3858 # Inspired by 'absolute1' in https://github.com/jwilk/traversal-archives
3859 with ArchiveMaker() as arc:
3860 arc.add(self.outerdir / 'escaped.evil')
3861
3862 with self.check_context(arc.open(), 'fully_trusted'):
3863 self.expect_file('../escaped.evil')
3864
3865 for filter in 'tar', 'data':
3866 with self.check_context(arc.open(), filter):
3867 if str(self.outerdir).startswith('/'):
3868 # We strip leading slashes, as e.g. GNU tar does
3869 # (without --absolute-filenames).
3870 outerdir_stripped = str(self.outerdir).lstrip('/')
3871 self.expect_file(f'{outerdir_stripped}/escaped.evil')
3872 else:
3873 # On this system, absolute paths don't have leading
3874 # slashes.
3875 # So, there's nothing to strip. We refuse to unpack
3876 # to an absolute path, nonetheless.
3877 self.expect_exception(
3878 tarfile.AbsolutePathError,
3879 """['"].*escaped.evil['"] has an absolute path""")
3880
3881 @symlink_test
3882 def test_parent_symlink(self):
3883 # Test interplaying symlinks
3884 # Inspired by 'dirsymlink2a' in jwilk/traversal-archives
3885 with ArchiveMaker() as arc:
3886
3887 # `current` links to `.` which is both:
3888 # - the destination directory
3889 # - `current` itself
3890 arc.add('current', symlink_to='.')
3891
3892 # effectively points to ./../
3893 arc.add('parent', symlink_to='current/..')
3894
3895 arc.add('parent/evil')
3896
3897 if os_helper.can_symlink():
3898 with self.check_context(arc.open(), 'fully_trusted'):
3899 if self.raised_exception is not None:
3900 # Windows will refuse to create a file that's a symlink to itself
3901 # (and tarfile doesn't swallow that exception)
3902 self.expect_exception(FileExistsError)
3903 # The other cases will fail with this error too.
3904 # Skip the rest of this test.
3905 return
3906 else:
3907 self.expect_file('current', symlink_to='.')
3908 self.expect_file('parent', symlink_to='current/..')
3909 self.expect_file('../evil')
3910
3911 with self.check_context(arc.open(), 'tar'):
3912 self.expect_exception(
3913 tarfile.OutsideDestinationError,
3914 """'parent/evil' would be extracted to ['"].*evil['"], """
3915 + "which is outside the destination")
3916
3917 with self.check_context(arc.open(), 'data'):
3918 if self.dotdot_resolves_early:
3919 # 'current/../..' normalises to '..', which is rejected.
3920 self.expect_exception(
3921 tarfile.LinkOutsideDestinationError,
3922 """'parent' would link to ['"].*outerdir['"], """
3923 + "which is outside the destination")
3924 else:
3925 # 'current/..' normalises to '.'; the rewritten link is
3926 # created and 'parent/evil' lands harmlessly inside the
3927 # destination.
3928 self.expect_file('current', symlink_to='.')
3929 self.expect_file('parent', symlink_to='.')
3930 self.expect_file('evil')
3931
3932 else:
3933 # No symlink support. The symlinks are ignored.
3934 with self.check_context(arc.open(), 'fully_trusted'):
3935 self.expect_file('parent/evil')
3936 with self.check_context(arc.open(), 'tar'):
3937 self.expect_file('parent/evil')
3938 with self.check_context(arc.open(), 'data'):
3939 self.expect_file('parent/evil')
3940
3941 @symlink_test
3942 @os_helper.skip_unless_symlink
3943 def test_realpath_limit_attack(self):
3944 # (CVE-2025-4517)
3945
3946 with ArchiveMaker() as arc:
3947 # populate the symlinks and dirs that expand in os.path.realpath()
3948 # The component length is chosen so that in common cases, the unexpanded
3949 # path fits in PATH_MAX, but it overflows when the final symlink
3950 # is expanded
3951 steps = "abcdefghijklmnop"
3952 if sys.platform == 'win32':
3953 component = 'd' * 25
3954 elif 'PC_PATH_MAX' in os.pathconf_names:
3955 max_path_len = os.pathconf(self.outerdir.parent, "PC_PATH_MAX")
3956 path_sep_len = 1
3957 dest_len = len(str(self.destdir)) + path_sep_len
3958 component_len = (max_path_len - dest_len) // (len(steps) + path_sep_len)
3959 component = 'd' * component_len
3960 else:
3961 raise NotImplementedError("Need to guess component length for {sys.platform}")
3962 path = ""
3963 step_path = ""
3964 for i in steps:
3965 arc.add(os.path.join(path, component), type=tarfile.DIRTYPE,
3966 mode='drwxrwxrwx')
3967 arc.add(os.path.join(path, i), symlink_to=component)
3968 path = os.path.join(path, component)
3969 step_path = os.path.join(step_path, i)
3970 # create the final symlink that exceeds PATH_MAX and simply points
3971 # to the top dir.
3972 # this link will never be expanded by
3973 # os.path.realpath(strict=False), nor anything after it.
3974 linkpath = os.path.join(*steps, "l"*254)
3975 parent_segments = [".."] * len(steps)
3976 arc.add(linkpath, symlink_to=os.path.join(*parent_segments))
3977 # make a symlink outside to keep the tar command happy
3978 arc.add("escape", symlink_to=os.path.join(linkpath, ".."))
3979 # use the symlinks above, that are not checked, to create a hardlink
3980 # to a file outside of the destination path
3981 arc.add("flaglink", hardlink_to=os.path.join("escape", "flag"))
3982 # now that we have the hardlink we can overwrite the file
3983 arc.add("flaglink", content='overwrite')
3984 # we can also create new files as well!
3985 arc.add("escape/newfile", content='new')
3986
3987 with (self.subTest('fully_trusted'),
3988 self.check_context(arc.open(), filter='fully_trusted',
3989 check_flag=False)):
3990 if sys.platform == 'win32':
3991 self.expect_exception((FileNotFoundError, FileExistsError))
3992 elif self.raised_exception:
3993 # Cannot symlink/hardlink: tarfile falls back to getmember()
3994 self.expect_exception(KeyError)
3995 # Otherwise, this block should never enter.
3996 else:
3997 self.expect_any_tree(component)
3998 self.expect_file('flaglink', content='overwrite')
3999 self.expect_file('../newfile', content='new')
4000 self.expect_file('escape', type=tarfile.SYMTYPE)
4001 self.expect_file('a', symlink_to=component)
4002
4003 for filter in 'tar', 'data':
4004 with self.subTest(filter), self.check_context(arc.open(), filter=filter):
4005 exc = self.expect_exception((OSError, KeyError))
4006 if isinstance(exc, OSError):
4007 if sys.platform == 'win32':
4008 # 3: ERROR_PATH_NOT_FOUND
4009 # 5: ERROR_ACCESS_DENIED
4010 # 206: ERROR_FILENAME_EXCED_RANGE
4011 self.assertIn(exc.winerror, (3, 5, 206))
4012 else:
4013 self.assertEqual(exc.errno, errno.ENAMETOOLONG)
4014
4015 @symlink_test
4016 def test_parent_symlink2(self):
4017 # Test interplaying symlinks
4018 # Inspired by 'dirsymlink2b' in jwilk/traversal-archives
4019
4020 with ArchiveMaker() as arc:
4021
4022 # `current` links to `.` which is both the destination directory
4023 # and `current` itself
4024 arc.add('current', symlink_to='.')
4025
4026 # `current/parent` is also available as `./parent`,
4027 # and effectively points to `./../`
4028 arc.add('current/parent', symlink_to='..')
4029
4030 arc.add('parent/evil')
4031
4032 with self.check_context(arc.open(), 'fully_trusted'):
4033 if os_helper.can_symlink():
4034 self.expect_file('current', symlink_to='.')
4035 self.expect_file('parent', symlink_to='..')
4036 self.expect_file('../evil')
4037 else:
4038 self.expect_file('current/')
4039 self.expect_file('parent/evil')
4040
4041 with self.check_context(arc.open(), 'tar'):
4042 if os_helper.can_symlink():
4043 # Fail when extracting a file outside destination
4044 self.expect_exception(
4045 tarfile.OutsideDestinationError,
4046 "'parent/evil' would be extracted to "
4047 + """['"].*evil['"], which is outside """
4048 + "the destination")
4049 else:
4050 self.expect_file('current/')
4051 self.expect_file('parent/evil')
4052
4053 with self.check_context(arc.open(), 'data'):
4054 if os_helper.can_symlink():
4055 if self.dotdot_resolves_early:
4056 # Fail when extracting a file outside destination
4057 self.expect_exception(
4058 tarfile.OutsideDestinationError,
4059 "'parent/evil' would be extracted to "
4060 + """['"].*evil['"], which is outside """
4061 + "the destination")
4062 else:
4063 # Fail as soon as we have a symlink outside the destination
4064 self.expect_exception(
4065 tarfile.LinkOutsideDestinationError,
4066 "'current/parent' would link to "
4067 + """['"].*outerdir['"], which is outside """
4068 + "the destination")
4069 else:
4070 self.expect_file('current/')
4071 self.expect_file('parent/evil')
4072
4073 @symlink_test
4074 def test_absolute_symlink(self):
4075 # Test symlink to an absolute path
4076 # Inspired by 'dirsymlink' in jwilk/traversal-archives
4077 with ArchiveMaker() as arc:
4078 arc.add('parent', symlink_to=self.outerdir)
4079 arc.add('parent/evil')
4080
4081 with self.check_context(arc.open(), 'fully_trusted'):
4082 if os_helper.can_symlink():
4083 self.expect_file('parent', symlink_to=self.outerdir)
4084 self.expect_file('../evil')
4085 else:
4086 self.expect_file('parent/evil')
4087
4088 with self.check_context(arc.open(), 'tar'):
4089 if os_helper.can_symlink():
4090 self.expect_exception(
4091 tarfile.OutsideDestinationError,
4092 "'parent/evil' would be extracted to "
4093 + """['"].*evil['"], which is outside """
4094 + "the destination")
4095 else:
4096 self.expect_file('parent/evil')
4097
4098 with self.check_context(arc.open(), 'data'):
4099 self.expect_exception(
4100 tarfile.AbsoluteLinkError,
4101 "'parent' is a link to an absolute path")
4102
4103 def test_absolute_hardlink(self):
4104 # Test hardlink to an absolute path
4105 # Inspired by 'dirsymlink' in https://github.com/jwilk/traversal-archives
4106 with ArchiveMaker() as arc:
4107 arc.add('parent', hardlink_to=self.outerdir / 'foo')
4108
4109 with self.check_context(arc.open(), 'fully_trusted'):
4110 self.expect_exception(KeyError, ".*foo. not found")
4111
4112 with self.check_context(arc.open(), 'tar'):
4113 self.expect_exception(KeyError, ".*foo. not found")
4114
4115 with self.check_context(arc.open(), 'data'):
4116 self.expect_exception(
4117 tarfile.AbsoluteLinkError,
4118 "'parent' is a link to an absolute path")
4119
4120 @symlink_test
4121 def test_sly_relative0(self):
4122 # Inspired by 'relative0' in jwilk/traversal-archives
4123 with ArchiveMaker() as arc:
4124 # points to `../../tmp/moo`
4125 arc.add('../moo', symlink_to='..//tmp/moo')
4126
4127 try:
4128 with self.check_context(arc.open(), filter='fully_trusted'):
4129 if os_helper.can_symlink():
4130 if isinstance(self.raised_exception, FileExistsError):
4131 # XXX TarFile happens to fail creating a parent
4132 # directory.
4133 # This might be a bug, but fixing it would hurt
4134 # security.
4135 # Note that e.g. GNU `tar` rejects '..' components,
4136 # so you could argue this is an invalid archive and we
4137 # just raise an bad type of exception.
4138 self.expect_exception(FileExistsError)
4139 else:
4140 self.expect_file('../moo', symlink_to='..//tmp/moo')
4141 else:
4142 # The symlink can't be extracted and is ignored
4143 pass
4144 except FileExistsError:
4145 pass
4146
4147 for filter in 'tar', 'data':
4148 with self.check_context(arc.open(), filter):
4149 self.expect_exception(
4150 tarfile.OutsideDestinationError,
4151 "'../moo' would be extracted to "
4152 + "'.*moo', which is outside "
4153 + "the destination")
4154
4155 @symlink_test
4156 def test_sly_relative2(self):
4157 # Inspired by 'relative2' in jwilk/traversal-archives
4158 with ArchiveMaker() as arc:
4159 arc.add('tmp/')
4160 arc.add('tmp/../../moo', symlink_to='tmp/../..//tmp/moo')
4161
4162 with self.check_context(arc.open(), 'fully_trusted'):
4163 self.expect_file('tmp', type=tarfile.DIRTYPE)
4164 if os_helper.can_symlink():
4165 self.expect_file('../moo', symlink_to='tmp/../../tmp/moo')
4166
4167 for filter in 'tar', 'data':
4168 with self.check_context(arc.open(), filter):
4169 self.expect_exception(
4170 tarfile.OutsideDestinationError,
4171 "'tmp/../../moo' would be extracted to "
4172 + """['"].*moo['"], which is outside the """
4173 + "destination")
4174
4175 @symlink_test
4176 @os_helper.skip_unless_symlink
4177 def test_normpath_realpath_mismatch(self):
4178 # The link-target check must validate the value that will actually
4179 # be written to disk (the normalised linkname), not the original.
4180 # Here 'a' is a symlink to a deep nonexistent path, so realpath()
4181 # of 'a/../../...' stays inside the destination while normpath()
4182 # collapses 'a/..' lexically and escapes.
4183 depth = len(self.destdir.parts) + 5
4184 deep = '/'.join(f'p{i}' for i in range(depth))
4185 sneaky = 'a/' + '../' * depth + 'flag'
4186 for kind in 'symlink_to', 'hardlink_to':
4187 with self.subTest(kind):
4188 with ArchiveMaker() as arc:
4189 arc.add('a', symlink_to=deep)
4190 arc.add('escape', **{kind: sneaky})
4191 with self.check_context(arc.open(), 'data'):
4192 self.expect_exception(
4193 tarfile.LinkOutsideDestinationError)
4194
4195 @symlink_test
4196 @os_helper.skip_unless_symlink
4197 def test_symlink_trailing_slash(self):
4198 # A trailing slash on a symlink member's name must not cause the
4199 # link target to be resolved relative to the wrong directory.
4200 with ArchiveMaker() as arc:
4201 t = tarfile.TarInfo('x/')
4202 t.type = tarfile.SYMTYPE
4203 t.linkname = '..'
4204 arc.tar_w.addfile(t)
4205 arc.add('x/escaped', content='hi')
4206
4207 with self.check_context(arc.open(), 'data'):
4208 self.expect_exception(tarfile.LinkOutsideDestinationError)
4209
4210 @symlink_test
4211 @os_helper.skip_unless_symlink
4212 def test_link_at_destination(self):
4213 # A link member whose name resolves to the destination directory
4214 # itself must be rejected: otherwise the destination is replaced
4215 # by a symlink and later members can be redirected through it.
4216 for name in '', '.', './':
4217 with ArchiveMaker() as arc:
4218 t = tarfile.TarInfo(name)
4219 t.type = tarfile.SYMTYPE
4220 t.linkname = '.'
4221 arc.tar_w.addfile(t)
4222
4223 with self.check_context(arc.open(), 'data'):
4224 self.expect_exception(tarfile.OutsideDestinationError)
4225
4226 @symlink_test
4227 @os_helper.skip_unless_symlink
4228 def test_empty_name_symlink_chain(self):
4229 # Regression test for a chain of empty-named symlinks that
4230 # incrementally redirects the destination outwards.
4231 with ArchiveMaker() as arc:
4232 for name, target in [('', ''), ('a/', '..'),
4233 ('', 'dummy'), ('', 'a'),
4234 ('b/', '..'),
4235 ('', 'dummy'), ('', 'a/b')]:
4236 t = tarfile.TarInfo(name)
4237 t.type = tarfile.SYMTYPE
4238 t.linkname = target
4239 arc.tar_w.addfile(t)
4240 arc.add('escaped', content='hi')
4241
4242 with self.check_context(arc.open(), 'data'):
4243 self.expect_exception(tarfile.FilterError)
4244
4245 @symlink_test
4246 def test_deep_symlink(self):
4247 # Test that symlinks and hardlinks inside a directory
4248 # point to the correct file (`target` of size 3).
4249 # If links aren't supported we get a copy of the file.
4250 with ArchiveMaker() as arc:
4251 arc.add('targetdir/target', size=3)
4252 # a hardlink's linkname is relative to the archive
4253 arc.add('linkdir/hardlink', hardlink_to=os.path.join(
4254 'targetdir', 'target'))
4255 # a symlink's linkname is relative to the link's directory
4256 arc.add('linkdir/symlink', symlink_to=os.path.join(
4257 '..', 'targetdir', 'target'))
4258
4259 for filter in 'tar', 'data', 'fully_trusted':
4260 with self.check_context(arc.open(), filter):
4261 self.expect_file('targetdir/target', size=3)
4262 self.expect_file('linkdir/hardlink', size=3)
4263 if os_helper.can_symlink():
4264 self.expect_file('linkdir/symlink', size=3,
4265 symlink_to='../targetdir/target')
4266 else:
4267 self.expect_file('linkdir/symlink', size=3)
4268
4269 @symlink_test
4270 def test_chains(self):
4271 # Test chaining of symlinks/hardlinks.
4272 # Symlinks are created before the files they point to.
4273 with ArchiveMaker() as arc:
4274 arc.add('linkdir/symlink', symlink_to='hardlink')
4275 arc.add('symlink2', symlink_to=os.path.join(
4276 'linkdir', 'hardlink2'))
4277 arc.add('targetdir/target', size=3)
4278 arc.add('linkdir/hardlink', hardlink_to=os.path.join('targetdir', 'target'))
4279 arc.add('linkdir/hardlink2', hardlink_to=os.path.join('linkdir', 'symlink'))
4280
4281 for filter in 'tar', 'data', 'fully_trusted':
4282 with self.check_context(arc.open(), filter):
4283 self.expect_file('targetdir/target', size=3)
4284 self.expect_file('linkdir/hardlink', size=3)
4285 self.expect_file('linkdir/hardlink2', size=3)
4286 if os_helper.can_symlink():
4287 self.expect_file('linkdir/symlink', size=3,
4288 symlink_to='hardlink')
4289 self.expect_file('symlink2', size=3,
4290 symlink_to='linkdir/hardlink2')
4291 else:
4292 self.expect_file('linkdir/symlink', size=3)
4293 self.expect_file('symlink2', size=3)
4294
4295 @symlink_test
4296 def test_sneaky_hardlink_fallback(self):
4297 # (CVE-2025-4330)
4298 # Test that when hardlink extraction falls back to extracting members
4299 # from the archive, the extracted member is (re-)filtered.
4300 with ArchiveMaker() as arc:
4301 # Create a directory structure so the c/escape symlink stays
4302 # inside the path
4303 arc.add("a/t/dummy")
4304 # Create b/ directory
4305 arc.add("b/")
4306 # Point "c" to the bottom of the tree in "a"
4307 arc.add("c", symlink_to=os.path.join("a", "t"))
4308 # link to non-existant location under "a"
4309 arc.add("c/escape", symlink_to=os.path.join("..", "..",
4310 "link_here"))
4311 # Move "c" to point to "b" ("c/escape" no longer exists)
4312 arc.add("c", symlink_to="b")
4313 # Attempt to create a hard link to "c/escape". Since it doesn't
4314 # exist it will attempt to extract "cescape" but at "boom".
4315 arc.add("boom", hardlink_to=os.path.join("c", "escape"))
4316
4317 with self.check_context(arc.open(), 'data'):
4318 if not os_helper.can_symlink():
4319 # When 'c/escape' is extracted, 'c' is a regular
4320 # directory, and 'c/escape' *would* point outside
4321 # the destination if symlinks were allowed.
4322 self.expect_exception(
4323 tarfile.LinkOutsideDestinationError)
4324 elif sys.platform == "win32":
4325 # On Windows, 'c/escape' points outside the destination
4326 self.expect_exception(tarfile.LinkOutsideDestinationError)
4327 else:
4328 e = self.expect_exception(
4329 tarfile.LinkFallbackError,
4330 "link 'boom' would be extracted as a copy of "
4331 + "'c/escape', which was rejected")
4332 self.assertIsInstance(e.__cause__,
4333 tarfile.LinkOutsideDestinationError)
4334 for filter in 'tar', 'fully_trusted':
4335 with self.subTest(filter), self.check_context(arc.open(), filter):
4336 if not os_helper.can_symlink():
4337 self.expect_file("a/t/dummy")
4338 self.expect_file("b/")
4339 self.expect_file("c/")
4340 else:
4341 self.expect_file("a/t/dummy")
4342 self.expect_file("b/")
4343 self.expect_file("a/t/escape", symlink_to='../../link_here')
4344 self.expect_file("boom", symlink_to='../../link_here')
4345 self.expect_file("c", symlink_to='b')
4346
4347 @symlink_test
4348 def test_exfiltration_via_symlink(self):
4349 # (CVE-2025-4138)
4350 # Test changing symlinks that result in a symlink pointing outside
4351 # the extraction directory, unless prevented by 'data' filter's
4352 # normalization.
4353 with ArchiveMaker() as arc:
4354 arc.add("escape", symlink_to=os.path.join('link', 'link', '..', '..', 'link-here'))
4355 arc.add("link", symlink_to='./')
4356
4357 for filter in 'tar', 'data', 'fully_trusted':
4358 with self.check_context(arc.open(), filter):
4359 if os_helper.can_symlink():
4360 self.expect_file("link", symlink_to='./')
4361 if filter == 'data':
4362 self.expect_file("escape", symlink_to='link-here')
4363 else:
4364 self.expect_file("escape",
4365 symlink_to='link/link/../../link-here')
4366 else:
4367 # Nothing is extracted.
4368 pass
4369
4370 @symlink_test
4371 def test_chmod_outside_dir(self):
4372 # (CVE-2024-12718)
4373 # Test that members used for delayed updates of directory metadata
4374 # are (re-)filtered.
4375 with ArchiveMaker() as arc:
4376 # "pwn" is a veeeery innocent symlink:
4377 arc.add("a/pwn", symlink_to='.')
4378 # But now "pwn" is also a directory, so it's scheduled to have its
4379 # metadata updated later:
4380 arc.add("a/pwn/", mode='drwxrwxrwx')
4381 # Oops, "pwn" is not so innocent any more:
4382 arc.add("a/pwn", symlink_to='x/../')
4383 # Newly created symlink points to the dest dir,
4384 # so it's OK for the "data" filter.
4385 arc.add('a/x', symlink_to=('../'))
4386 # But now "pwn" points outside the dest dir
4387
4388 for filter in 'tar', 'data', 'fully_trusted':
4389 with self.check_context(arc.open(), filter) as cc:
4390 if not os_helper.can_symlink():
4391 self.expect_file("a/pwn/")
4392 elif filter == 'data':
4393 self.expect_file("a/x", symlink_to='../')
4394 self.expect_file("a/pwn", symlink_to='.')
4395 else:
4396 self.expect_file("a/x", symlink_to='../')
4397 self.expect_file("a/pwn", symlink_to='x/../')
4398 if sys.platform != "win32":
4399 st_mode = cc.outerdir.stat().st_mode
4400 self.assertNotEqual(st_mode & 0o777, 0o777)
4401
4402 def test_link_fallback_normalizes(self):
4403 # Make sure hardlink fallbacks work for non-normalized paths for all
4404 # filters
4405 with ArchiveMaker() as arc:
4406 arc.add("dir/")
4407 arc.add("dir/../afile")
4408 arc.add("link1", hardlink_to='dir/../afile')
4409 arc.add("link2", hardlink_to='dir/../dir/../afile')
4410
4411 for filter in 'tar', 'data', 'fully_trusted':
4412 with self.check_context(arc.open(), filter) as cc:
4413 self.expect_file("dir/")
4414 self.expect_file("afile")
4415 self.expect_file("link1")
4416 self.expect_file("link2")
4417
4418 def test_modes(self):
4419 # Test how file modes are extracted
4420 # (Note that the modes are ignored on platforms without working chmod)
4421 with ArchiveMaker() as arc:
4422 arc.add('all_bits', mode='?rwsrwsrwt')
4423 arc.add('perm_bits', mode='?rwxrwxrwx')
4424 arc.add('exec_group_other', mode='?rw-rwxrwx')
4425 arc.add('read_group_only', mode='?---r-----')
4426 arc.add('no_bits', mode='?---------')
4427 arc.add('dir/', mode='?---rwsrwt')
4428 arc.add('dir_all_bits/', mode='?rwsrwsrwt')
4429
4430 # On some systems, setting the uid, gid, and/or sticky bit is a no-ops.
4431 # Check which bits we can set, so we can compare tarfile machinery to
4432 # a simple chmod.
4433 tmp_filename = os.path.join(TEMPDIR, "tmp.file")
4434 with open(tmp_filename, 'w'):
4435 pass
4436 try:
4437 new_mode = (os.stat(tmp_filename).st_mode
4438 | stat.S_ISVTX | stat.S_ISGID | stat.S_ISUID)
4439 try:
4440 os.chmod(tmp_filename, new_mode)
4441 except OSError as exc:
4442 if exc.errno == getattr(errno, "EFTYPE", 0):
4443 # gh-108948: On FreeBSD, regular users cannot set
4444 # the sticky bit.
4445 self.skipTest("chmod() failed with EFTYPE: "
4446 "regular users cannot set sticky bit")
4447 else:
4448 raise
4449
4450 got_mode = os.stat(tmp_filename).st_mode
4451 _t_file = 't' if (got_mode & stat.S_ISVTX) else 'x'
4452 _suid_file = 's' if (got_mode & stat.S_ISUID) else 'x'
4453 _sgid_file = 's' if (got_mode & stat.S_ISGID) else 'x'
4454 finally:
4455 os.unlink(tmp_filename)
4456
4457 os.mkdir(tmp_filename)
4458 new_mode = (os.stat(tmp_filename).st_mode
4459 | stat.S_ISVTX | stat.S_ISGID | stat.S_ISUID)
4460 os.chmod(tmp_filename, new_mode)
4461 got_mode = os.stat(tmp_filename).st_mode
4462 _t_dir = 't' if (got_mode & stat.S_ISVTX) else 'x'
4463 _suid_dir = 's' if (got_mode & stat.S_ISUID) else 'x'
4464 _sgid_dir = 's' if (got_mode & stat.S_ISGID) else 'x'
4465 os.rmdir(tmp_filename)
4466
4467 with self.check_context(arc.open(), 'fully_trusted'):
4468 self.expect_file('all_bits',
4469 mode=f'?rw{_suid_file}rw{_sgid_file}rw{_t_file}')
4470 self.expect_file('perm_bits', mode='?rwxrwxrwx')
4471 self.expect_file('exec_group_other', mode='?rw-rwxrwx')
4472 self.expect_file('read_group_only', mode='?---r-----')
4473 self.expect_file('no_bits', mode='?---------')
4474 self.expect_file('dir/', mode=f'?---rw{_sgid_dir}rw{_t_dir}')
4475 self.expect_file('dir_all_bits/',
4476 mode=f'?rw{_suid_dir}rw{_sgid_dir}rw{_t_dir}')
4477
4478 with self.check_context(arc.open(), 'tar'):
4479 self.expect_file('all_bits', mode='?rwxr-xr-x')
4480 self.expect_file('perm_bits', mode='?rwxr-xr-x')
4481 self.expect_file('exec_group_other', mode='?rw-r-xr-x')
4482 self.expect_file('read_group_only', mode='?---r-----')
4483 self.expect_file('no_bits', mode='?---------')
4484 self.expect_file('dir/', mode='?---r-xr-x')
4485 self.expect_file('dir_all_bits/', mode='?rwxr-xr-x')
4486
4487 with self.check_context(arc.open(), 'data'):
4488 normal_dir_mode = stat.filemode(stat.S_IMODE(
4489 self.outerdir.stat().st_mode))
4490 self.expect_file('all_bits', mode='?rwxr-xr-x')
4491 self.expect_file('perm_bits', mode='?rwxr-xr-x')
4492 self.expect_file('exec_group_other', mode='?rw-r--r--')
4493 self.expect_file('read_group_only', mode='?rw-r-----')
4494 self.expect_file('no_bits', mode='?rw-------')
4495 self.expect_file('dir/', mode=normal_dir_mode)
4496 self.expect_file('dir_all_bits/', mode=normal_dir_mode)
4497
4498 def test_pipe(self):
4499 # Test handling of a special file
4500 with ArchiveMaker() as arc:
4501 arc.add('foo', type=tarfile.FIFOTYPE)
4502
4503 for filter in 'fully_trusted', 'tar':
4504 with self.check_context(arc.open(), filter):
4505 if hasattr(os, 'mkfifo'):
4506 self.expect_file('foo', type=tarfile.FIFOTYPE)
4507 else:
4508 # The pipe can't be extracted and is skipped.
4509 pass
4510
4511 with self.check_context(arc.open(), 'data'):
4512 self.expect_exception(
4513 tarfile.SpecialFileError,
4514 "'foo' is a special file")
4515
4516 def test_special_files(self):
4517 # Creating device files is tricky. Instead of attempting that let's
4518 # only check the filter result.
4519 for special_type in tarfile.FIFOTYPE, tarfile.CHRTYPE, tarfile.BLKTYPE:
4520 tarinfo = tarfile.TarInfo('foo')
4521 tarinfo.type = special_type
4522 trusted = tarfile.fully_trusted_filter(tarinfo, '')
4523 self.assertIs(trusted, tarinfo)
4524 tar = tarfile.tar_filter(tarinfo, '')
4525 self.assertEqual(tar.type, special_type)
4526 with self.assertRaises(tarfile.SpecialFileError) as cm:
4527 tarfile.data_filter(tarinfo, '')
4528 self.assertIsInstance(cm.exception.tarinfo, tarfile.TarInfo)
4529 self.assertEqual(cm.exception.tarinfo.name, 'foo')
4530
4531 def test_fully_trusted_filter(self):
4532 # The 'fully_trusted' filter returns the original TarInfo objects.
4533 with tarfile.TarFile.open(tarname) as tar:
4534 for tarinfo in tar.getmembers():
4535 filtered = tarfile.fully_trusted_filter(tarinfo, '')
4536 self.assertIs(filtered, tarinfo)
4537
4538 def test_tar_filter(self):
4539 # The 'tar' filter returns TarInfo objects with the same name/type.
4540 # (It can also fail for particularly "evil" input, but we don't have
4541 # that in the test archive.)
4542 with tarfile.TarFile.open(tarname, encoding="iso8859-1") as tar:
4543 for tarinfo in tar.getmembers():
4544 try:
4545 filtered = tarfile.tar_filter(tarinfo, '')
4546 except UnicodeEncodeError:
4547 continue
4548 self.assertIs(filtered.name, tarinfo.name)
4549 self.assertIs(filtered.type, tarinfo.type)
4550
4551 def test_data_filter(self):
4552 # The 'data' filter either raises, or returns TarInfo with the same
4553 # name/type.
4554 with tarfile.TarFile.open(tarname, encoding="iso8859-1") as tar:
4555 for tarinfo in tar.getmembers():
4556 try:
4557 filtered = tarfile.data_filter(tarinfo, '')
4558 except (tarfile.FilterError, UnicodeEncodeError):
4559 continue
4560 self.assertIs(filtered.name, tarinfo.name)
4561 self.assertIs(filtered.type, tarinfo.type)
4562
4563 @unittest.skipIf(sys.platform == 'win32', 'requires native bytes paths')
4564 def test_filter_unencodable(self):
4565 # Sanity check using a valid path.
4566 tarinfo = tarfile.TarInfo(os_helper.TESTFN)
4567 filtered = tarfile.tar_filter(tarinfo, '')
4568 self.assertIs(filtered.name, tarinfo.name)
4569 filtered = tarfile.data_filter(tarinfo, '')
4570 self.assertIs(filtered.name, tarinfo.name)
4571
4572 tarinfo = tarfile.TarInfo('test\x00')
4573 self.assertRaises(ValueError, tarfile.tar_filter, tarinfo, '')
4574 self.assertRaises(ValueError, tarfile.data_filter, tarinfo, '')
4575 tarinfo = tarfile.TarInfo('\ud800')
4576 self.assertRaises(UnicodeEncodeError, tarfile.tar_filter, tarinfo, '')
4577 self.assertRaises(UnicodeEncodeError, tarfile.data_filter, tarinfo, '')
4578
4579 @unittest.skipIf(sys.platform == 'win32', 'requires native bytes paths')
4580 def test_extract_unencodable(self):
4581 # Create a member with name \xed\xa0\x80 which is UTF-8 encoded
4582 # lone surrogate \ud800.
4583 with ArchiveMaker(encoding='ascii', errors='surrogateescape') as arc:
4584 arc.add('\udced\udca0\udc80')
4585 with os_helper.temp_cwd() as tmp:
4586 tar = arc.open(encoding='utf-8', errors='surrogatepass',
4587 errorlevel=1)
4588 self.assertEqual(tar.getnames(), ['\ud800'])
4589 with self.assertRaises(UnicodeEncodeError):
4590 tar.extractall()
4591 self.assertEqual(os.listdir(), [])
4592
4593 tar = arc.open(encoding='utf-8', errors='surrogatepass',
4594 errorlevel=0, debug=1)
4595 with support.captured_stderr() as stderr:
4596 tar.extractall()
4597 self.assertEqual(os.listdir(), [])
4598 self.assertIn('tarfile: UnicodeEncodeError ', stderr.getvalue())
4599
4600 def test_change_default_filter_on_instance(self):
4601 tar = tarfile.TarFile(tarname, 'r')
4602 def strict_filter(tarinfo, path):
4603 if tarinfo.name == 'ustar/regtype':
4604 return tarinfo
4605 else:
4606 return None
4607 tar.extraction_filter = strict_filter
4608 with self.check_context(tar, None):
4609 self.expect_file('ustar/regtype')
4610
4611 def test_change_default_filter_on_class(self):
4612 def strict_filter(tarinfo, path):
4613 if tarinfo.name == 'ustar/regtype':
4614 return tarinfo
4615 else:
4616 return None
4617 tar = tarfile.TarFile(tarname, 'r')
4618 with support.swap_attr(tarfile.TarFile, 'extraction_filter',
4619 staticmethod(strict_filter)):
4620 with self.check_context(tar, None):
4621 self.expect_file('ustar/regtype')
4622
4623 def test_change_default_filter_on_subclass(self):
4624 class TarSubclass(tarfile.TarFile):
4625 def extraction_filter(self, tarinfo, path):
4626 if tarinfo.name == 'ustar/regtype':
4627 return tarinfo
4628 else:
4629 return None
4630
4631 tar = TarSubclass(tarname, 'r')
4632 with self.check_context(tar, None):
4633 self.expect_file('ustar/regtype')
4634
4635 def test_change_default_filter_to_string(self):
4636 tar = tarfile.TarFile(tarname, 'r')
4637 tar.extraction_filter = 'data'
4638 with self.check_context(tar, None):
4639 self.expect_exception(TypeError)
4640
4641 def test_custom_filter(self):
4642 def custom_filter(tarinfo, path):
4643 self.assertIs(path, self.destdir)
4644 if tarinfo.name == 'move_this':
4645 return tarinfo.replace(name='moved')
4646 if tarinfo.name == 'ignore_this':
4647 return None
4648 return tarinfo
4649
4650 with ArchiveMaker() as arc:
4651 arc.add('move_this')
4652 arc.add('ignore_this')
4653 arc.add('keep')
4654 with self.check_context(arc.open(), custom_filter):
4655 self.expect_file('moved')
4656 self.expect_file('keep')
4657
4658 def test_bad_filter_name(self):
4659 with ArchiveMaker() as arc:
4660 arc.add('foo')
4661 with self.check_context(arc.open(), 'bad filter name'):
4662 self.expect_exception(ValueError)
4663
4664 def test_stateful_filter(self):
4665 # Stateful filters should be possible.
4666 # (This doesn't really test tarfile. Rather, it demonstrates
4667 # that third parties can implement a stateful filter.)
4668 class StatefulFilter:
4669 def __enter__(self):
4670 self.num_files_processed = 0
4671 return self
4672
4673 def __call__(self, tarinfo, path):
4674 try:
4675 tarinfo = tarfile.data_filter(tarinfo, path)
4676 except tarfile.FilterError:
4677 return None
4678 self.num_files_processed += 1
4679 return tarinfo
4680
4681 def __exit__(self, *exc_info):
4682 self.done = True
4683
4684 with ArchiveMaker() as arc:
4685 arc.add('good')
4686 arc.add('bad', symlink_to='/')
4687 arc.add('good')
4688 with StatefulFilter() as custom_filter:
4689 with self.check_context(arc.open(), custom_filter):
4690 self.expect_file('good')
4691 self.assertEqual(custom_filter.num_files_processed, 2)
4692 self.assertEqual(custom_filter.done, True)
4693
4694 def test_errorlevel(self):
4695 def extracterror_filter(tarinfo, path):
4696 raise tarfile.ExtractError('failed with ExtractError')
4697 def filtererror_filter(tarinfo, path):
4698 raise tarfile.FilterError('failed with FilterError')
4699 def oserror_filter(tarinfo, path):
4700 raise OSError('failed with OSError')
4701 def tarerror_filter(tarinfo, path):
4702 raise tarfile.TarError('failed with base TarError')
4703 def valueerror_filter(tarinfo, path):
4704 raise ValueError('failed with ValueError')
4705
4706 with ArchiveMaker() as arc:
4707 arc.add('file')
4708
4709 # If errorlevel is 0, errors affected by errorlevel are ignored
4710
4711 with self.check_context(arc.open(errorlevel=0), extracterror_filter):
4712 pass
4713
4714 with self.check_context(arc.open(errorlevel=0), filtererror_filter):
4715 pass
4716
4717 with self.check_context(arc.open(errorlevel=0), oserror_filter):
4718 pass
4719
4720 with self.check_context(arc.open(errorlevel=0), tarerror_filter):
4721 self.expect_exception(tarfile.TarError)
4722
4723 with self.check_context(arc.open(errorlevel=0), valueerror_filter):
4724 self.expect_exception(ValueError)
4725
4726 # If 1, all fatal errors are raised
4727
4728 with self.check_context(arc.open(errorlevel=1), extracterror_filter):
4729 pass
4730
4731 with self.check_context(arc.open(errorlevel=1), filtererror_filter):
4732 self.expect_exception(tarfile.FilterError)
4733
4734 with self.check_context(arc.open(errorlevel=1), oserror_filter):
4735 self.expect_exception(OSError)
4736
4737 with self.check_context(arc.open(errorlevel=1), tarerror_filter):
4738 self.expect_exception(tarfile.TarError)
4739
4740 with self.check_context(arc.open(errorlevel=1), valueerror_filter):
4741 self.expect_exception(ValueError)
4742
4743 # If 2, all non-fatal errors are raised as well.
4744
4745 with self.check_context(arc.open(errorlevel=2), extracterror_filter):
4746 self.expect_exception(tarfile.ExtractError)
4747
4748 with self.check_context(arc.open(errorlevel=2), filtererror_filter):
4749 self.expect_exception(tarfile.FilterError)
4750
4751 with self.check_context(arc.open(errorlevel=2), oserror_filter):
4752 self.expect_exception(OSError)
4753
4754 with self.check_context(arc.open(errorlevel=2), tarerror_filter):
4755 self.expect_exception(tarfile.TarError)
4756
4757 with self.check_context(arc.open(errorlevel=2), valueerror_filter):
4758 self.expect_exception(ValueError)
4759
4760 # We only handle ExtractionError, FilterError & OSError specially.
4761
4762 with self.check_context(arc.open(errorlevel='boo!'), filtererror_filter):
4763 self.expect_exception(TypeError) # errorlevel is not int
4764
4765
4766 class OverwriteTests(archiver_tests.OverwriteTests, unittest.TestCase):
4767 testdir = os.path.join(TEMPDIR, "testoverwrite")
4768
4769 @classmethod
4770 def setUpClass(cls):
4771 p = cls.ar_with_file = os.path.join(TEMPDIR, 'tar-with-file.tar')
4772 cls.addClassCleanup(os_helper.unlink, p)
4773 with tarfile.open(p, 'w') as tar:
4774 t = tarfile.TarInfo('test')
4775 t.size = 10
4776 tar.addfile(t, io.BytesIO(b'newcontent'))
4777
4778 p = cls.ar_with_dir = os.path.join(TEMPDIR, 'tar-with-dir.tar')
4779 cls.addClassCleanup(os_helper.unlink, p)
4780 with tarfile.open(p, 'w') as tar:
4781 tar.addfile(tar.gettarinfo(os.curdir, 'test'))
4782
4783 p = os.path.join(TEMPDIR, 'tar-with-implicit-dir.tar')
4784 cls.ar_with_implicit_dir = p
4785 cls.addClassCleanup(os_helper.unlink, p)
4786 with tarfile.open(p, 'w') as tar:
4787 t = tarfile.TarInfo('test/file')
4788 t.size = 10
4789 tar.addfile(t, io.BytesIO(b'newcontent'))
4790
4791 def open(self, path):
4792 return tarfile.open(path, 'r')
4793
4794 def extractall(self, ar):
4795 ar.extractall(self.testdir, filter='fully_trusted')
4796
4797
4798 class OffsetValidationTests(unittest.TestCase):
4799 tarname = tmpname
4800 invalid_posix_header = (
4801 # name: 100 bytes
4802 tarfile.NUL * tarfile.LENGTH_NAME
4803 # mode, space, null terminator: 8 bytes
4804 + b"000755" + SPACE + tarfile.NUL
4805 # uid, space, null terminator: 8 bytes
4806 + b"000001" + SPACE + tarfile.NUL
4807 # gid, space, null terminator: 8 bytes
4808 + b"000001" + SPACE + tarfile.NUL
4809 # size, space: 12 bytes
4810 + b"\xff" * 11 + SPACE
4811 # mtime, space: 12 bytes
4812 + tarfile.NUL * 11 + SPACE
4813 # chksum: 8 bytes
4814 + b"0011407" + tarfile.NUL
4815 # type: 1 byte
4816 + tarfile.REGTYPE
4817 # linkname: 100 bytes
4818 + tarfile.NUL * tarfile.LENGTH_LINK
4819 # magic: 6 bytes, version: 2 bytes
4820 + tarfile.POSIX_MAGIC
4821 # uname: 32 bytes
4822 + tarfile.NUL * 32
4823 # gname: 32 bytes
4824 + tarfile.NUL * 32
4825 # devmajor, space, null terminator: 8 bytes
4826 + tarfile.NUL * 6 + SPACE + tarfile.NUL
4827 # devminor, space, null terminator: 8 bytes
4828 + tarfile.NUL * 6 + SPACE + tarfile.NUL
4829 # prefix: 155 bytes
4830 + tarfile.NUL * tarfile.LENGTH_PREFIX
4831 # padding: 12 bytes
4832 + tarfile.NUL * 12
4833 )
4834 invalid_gnu_header = (
4835 # name: 100 bytes
4836 tarfile.NUL * tarfile.LENGTH_NAME
4837 # mode, null terminator: 8 bytes
4838 + b"0000755" + tarfile.NUL
4839 # uid, null terminator: 8 bytes
4840 + b"0000001" + tarfile.NUL
4841 # gid, space, null terminator: 8 bytes
4842 + b"0000001" + tarfile.NUL
4843 # size, space: 12 bytes
4844 + b"\xff" * 11 + SPACE
4845 # mtime, space: 12 bytes
4846 + tarfile.NUL * 11 + SPACE
4847 # chksum: 8 bytes
4848 + b"0011327" + tarfile.NUL
4849 # type: 1 byte
4850 + tarfile.REGTYPE
4851 # linkname: 100 bytes
4852 + tarfile.NUL * tarfile.LENGTH_LINK
4853 # magic: 8 bytes
4854 + tarfile.GNU_MAGIC
4855 # uname: 32 bytes
4856 + tarfile.NUL * 32
4857 # gname: 32 bytes
4858 + tarfile.NUL * 32
4859 # devmajor, null terminator: 8 bytes
4860 + tarfile.NUL * 8
4861 # devminor, null terminator: 8 bytes
4862 + tarfile.NUL * 8
4863 # padding: 167 bytes
4864 + tarfile.NUL * 167
4865 )
4866 invalid_v7_header = (
4867 # name: 100 bytes
4868 tarfile.NUL * tarfile.LENGTH_NAME
4869 # mode, space, null terminator: 8 bytes
4870 + b"000755" + SPACE + tarfile.NUL
4871 # uid, space, null terminator: 8 bytes
4872 + b"000001" + SPACE + tarfile.NUL
4873 # gid, space, null terminator: 8 bytes
4874 + b"000001" + SPACE + tarfile.NUL
4875 # size, space: 12 bytes
4876 + b"\xff" * 11 + SPACE
4877 # mtime, space: 12 bytes
4878 + tarfile.NUL * 11 + SPACE
4879 # chksum: 8 bytes
4880 + b"0010070" + tarfile.NUL
4881 # type: 1 byte
4882 + tarfile.REGTYPE
4883 # linkname: 100 bytes
4884 + tarfile.NUL * tarfile.LENGTH_LINK
4885 # padding: 255 bytes
4886 + tarfile.NUL * 255
4887 )
4888 valid_gnu_header = tarfile.TarInfo("filename").tobuf(tarfile.GNU_FORMAT)
4889 data_block = b"\xff" * tarfile.BLOCKSIZE
4890
4891 def _write_buffer(self, buffer):
4892 with open(self.tarname, "wb") as f:
4893 f.write(buffer)
4894
4895 def _get_members(self, ignore_zeros=None):
4896 with open(self.tarname, "rb") as f:
4897 with tarfile.open(
4898 mode="r", fileobj=f, ignore_zeros=ignore_zeros
4899 ) as tar:
4900 return tar.getmembers()
4901
4902 def _assert_raises_read_error_exception(self):
4903 with self.assertRaisesRegex(
4904 tarfile.ReadError, "file could not be opened successfully"
4905 ):
4906 self._get_members()
4907
4908 def test_invalid_offset_header_validations(self):
4909 for tar_format, invalid_header in (
4910 ("posix", self.invalid_posix_header),
4911 ("gnu", self.invalid_gnu_header),
4912 ("v7", self.invalid_v7_header),
4913 ):
4914 with self.subTest(format=tar_format):
4915 self._write_buffer(invalid_header)
4916 self._assert_raises_read_error_exception()
4917
4918 def test_early_stop_at_invalid_offset_header(self):
4919 buffer = self.valid_gnu_header + self.invalid_gnu_header + self.valid_gnu_header
4920 self._write_buffer(buffer)
4921 members = self._get_members()
4922 self.assertEqual(len(members), 1)
4923 self.assertEqual(members[0].name, "filename")
4924 self.assertEqual(members[0].offset, 0)
4925
4926 def test_ignore_invalid_archive(self):
4927 # 3 invalid headers with their respective data
4928 buffer = (self.invalid_gnu_header + self.data_block) * 3
4929 self._write_buffer(buffer)
4930 members = self._get_members(ignore_zeros=True)
4931 self.assertEqual(len(members), 0)
4932
4933 def test_ignore_invalid_offset_headers(self):
4934 for first_block, second_block, expected_offset in (
4935 (
4936 (self.valid_gnu_header),
4937 (self.invalid_gnu_header + self.data_block),
4938 0,
4939 ),
4940 (
4941 (self.invalid_gnu_header + self.data_block),
4942 (self.valid_gnu_header),
4943 1024,
4944 ),
4945 ):
4946 self._write_buffer(first_block + second_block)
4947 members = self._get_members(ignore_zeros=True)
4948 self.assertEqual(len(members), 1)
4949 self.assertEqual(members[0].name, "filename")
4950 self.assertEqual(members[0].offset, expected_offset)
4951
4952
4953 def setUpModule():
4954 os_helper.unlink(TEMPDIR)
4955 os.makedirs(TEMPDIR)
4956
4957 global testtarnames
4958 testtarnames = [tarname]
4959 with open(tarname, "rb") as fobj:
4960 data = fobj.read()
4961
4962 # Create compressed tarfiles.
4963 for c in GzipTest, Bz2Test, LzmaTest, ZstdTest:
4964 if c.open:
4965 os_helper.unlink(c.tarname)
4966 testtarnames.append(c.tarname)
4967 with c.open(c.tarname, "wb") as tar:
4968 tar.write(data)
4969
4970 def tearDownModule():
4971 if os.path.exists(TEMPDIR):
4972 os_helper.rmtree(TEMPDIR)
4973
4974 if __name__ == "__main__":
4975 unittest.main()