Debian Codegraph @

Lib/test/test_tarfile.py

raw
1 import errno
2 import sys
3 import os
4 import io
5 from hashlib import sha256
6 from contextlib import contextmanager, ExitStack
7 from random import Random
8 import pathlib
9 import shutil
10 import re
11 import warnings
12 import stat
13
14 import unittest
15 import unittest.mock
16 import tarfile
17
18 from test import archiver_tests
19 from test import support
20 from test.support import os_helper
21 from test.support import script_helper
22 from test.support import warnings_helper
23
24 # Check for our compression modules.
25 try:
26 import gzip
27 except ImportError:
28 gzip = None
29 try:
30 import zlib
31 except ImportError:
32 zlib = None
33 try:
34 import bz2
35 except ImportError:
36 bz2 = None
37 try:
38 import lzma
39 except ImportError:
40 lzma = None
41 try:
42 from compression import zstd
43 except ImportError:
44 zstd = None
45
46 def sha256sum(data):
47 return sha256(data).hexdigest()
48
49 TEMPDIR = os.path.abspath(os_helper.TESTFN) + "-tardir"
50 tarextdir = TEMPDIR + '-extract-test'
51 tarname = support.findfile("testtar.tar", subdir="archivetestdata")
52 gzipname = os.path.join(TEMPDIR, "testtar.tar.gz")
53 bz2name = os.path.join(TEMPDIR, "testtar.tar.bz2")
54 xzname = os.path.join(TEMPDIR, "testtar.tar.xz")
55 zstname = os.path.join(TEMPDIR, "testtar.tar.zst")
56 tmpname = os.path.join(TEMPDIR, "tmp.tar")
57 dotlessname = os.path.join(TEMPDIR, "testtar")
58 SPACE = b" "
59
60 sha256_regtype = (
61 "e09e4bc8b3c9d9177e77256353b36c159f5f040531bbd4b024a8f9b9196c71ce"
62 )
63 sha256_sparse = (
64 "4f05a776071146756345ceee937b33fc5644f5a96b9780d1c7d6a32cdf164d7b"
65 )
66
67
68 class TarTest:
69 tarname = tarname
70 suffix = ''
71 open = io.FileIO
72 taropen = tarfile.TarFile.taropen
73
74 @property
75 def mode(self):
76 return self.prefix + self.suffix
77
78 @support.requires_gzip()
79 class GzipTest:
80 tarname = gzipname
81 suffix = 'gz'
82 open = gzip.GzipFile if gzip else None
83 taropen = tarfile.TarFile.gzopen
84
85 @support.requires_bz2()
86 class Bz2Test:
87 tarname = bz2name
88 suffix = 'bz2'
89 open = bz2.BZ2File if bz2 else None
90 taropen = tarfile.TarFile.bz2open
91
92 @support.requires_lzma()
93 class LzmaTest:
94 tarname = xzname
95 suffix = 'xz'
96 open = lzma.LZMAFile if lzma else None
97 taropen = tarfile.TarFile.xzopen
98
99 @support.requires_zstd()
100 class ZstdTest:
101 tarname = zstname
102 suffix = 'zst'
103 open = zstd.ZstdFile if zstd else None
104 taropen = tarfile.TarFile.zstopen
105
106 class ReadTest(TarTest):
107
108 prefix = "r:"
109
110 def setUp(self):
111 self.tar = tarfile.open(self.tarname, mode=self.mode,
112 encoding="iso8859-1")
113
114 def tearDown(self):
115 self.tar.close()
116
117 class StreamModeTest(ReadTest):
118
119 # Only needs to change how the tarfile is opened to set
120 # stream mode
121 def setUp(self):
122 self.tar = tarfile.open(self.tarname, mode=self.mode,
123 encoding="iso8859-1",
124 stream=True)
125
126 class UstarReadTest(ReadTest, unittest.TestCase):
127
128 def test_fileobj_regular_file(self):
129 tarinfo = self.tar.getmember("ustar/regtype")
130 with self.tar.extractfile(tarinfo) as fobj:
131 data = fobj.read()
132 self.assertEqual(len(data), tarinfo.size,
133 "regular file extraction failed")
134 self.assertEqual(sha256sum(data), sha256_regtype,
135 "regular file extraction failed")
136
137 def test_fileobj_readlines(self):
138 self.tar.extract("ustar/regtype", TEMPDIR, filter='data')
139 tarinfo = self.tar.getmember("ustar/regtype")
140 with open(os.path.join(TEMPDIR, "ustar/regtype"), "r") as fobj1:
141 lines1 = fobj1.readlines()
142
143 with self.tar.extractfile(tarinfo) as fobj:
144 fobj2 = io.TextIOWrapper(fobj)
145 lines2 = fobj2.readlines()
146 self.assertEqual(lines1, lines2,
147 "fileobj.readlines() failed")
148 self.assertEqual(len(lines2), 114,
149 "fileobj.readlines() failed")
150 self.assertEqual(lines2[83],
151 "I will gladly admit that Python is not the fastest "
152 "running scripting language.\n",
153 "fileobj.readlines() failed")
154
155 def test_fileobj_iter(self):
156 self.tar.extract("ustar/regtype", TEMPDIR, filter='data')
157 tarinfo = self.tar.getmember("ustar/regtype")
158 with open(os.path.join(TEMPDIR, "ustar/regtype"), "r") as fobj1:
159 lines1 = fobj1.readlines()
160 with self.tar.extractfile(tarinfo) as fobj2:
161 lines2 = list(io.TextIOWrapper(fobj2))
162 self.assertEqual(lines1, lines2,
163 "fileobj.__iter__() failed")
164
165 def test_fileobj_seek(self):
166 self.tar.extract("ustar/regtype", TEMPDIR,
167 filter='data')
168 with open(os.path.join(TEMPDIR, "ustar/regtype"), "rb") as fobj:
169 data = fobj.read()
170
171 tarinfo = self.tar.getmember("ustar/regtype")
172 with self.tar.extractfile(tarinfo) as fobj:
173 text = fobj.read()
174 fobj.seek(0)
175 self.assertEqual(0, fobj.tell(),
176 "seek() to file's start failed")
177 fobj.seek(2048, 0)
178 self.assertEqual(2048, fobj.tell(),
179 "seek() to absolute position failed")
180 fobj.seek(-1024, 1)
181 self.assertEqual(1024, fobj.tell(),
182 "seek() to negative relative position failed")
183 fobj.seek(1024, 1)
184 self.assertEqual(2048, fobj.tell(),
185 "seek() to positive relative position failed")
186 s = fobj.read(10)
187 self.assertEqual(s, data[2048:2058],
188 "read() after seek failed")
189 fobj.seek(0, 2)
190 self.assertEqual(tarinfo.size, fobj.tell(),
191 "seek() to file's end failed")
192 self.assertEqual(fobj.read(), b"",
193 "read() at file's end did not return empty string")
194 fobj.seek(-tarinfo.size, 2)
195 self.assertEqual(0, fobj.tell(),
196 "relative seek() to file's end failed")
197 fobj.seek(512)
198 s1 = fobj.readlines()
199 fobj.seek(512)
200 s2 = fobj.readlines()
201 self.assertEqual(s1, s2,
202 "readlines() after seek failed")
203 fobj.seek(0)
204 self.assertEqual(len(fobj.readline()), fobj.tell(),
205 "tell() after readline() failed")
206 fobj.seek(512)
207 self.assertEqual(len(fobj.readline()) + 512, fobj.tell(),
208 "tell() after seek() and readline() failed")
209 fobj.seek(0)
210 line = fobj.readline()
211 self.assertEqual(fobj.read(), data[len(line):],
212 "read() after readline() failed")
213
214 def test_fileobj_text(self):
215 with self.tar.extractfile("ustar/regtype") as fobj:
216 fobj = io.TextIOWrapper(fobj)
217 data = fobj.read().encode("iso8859-1")
218 self.assertEqual(sha256sum(data), sha256_regtype)
219 try:
220 fobj.seek(100)
221 except AttributeError:
222 # Issue #13815: seek() complained about a missing
223 # flush() method.
224 self.fail("seeking failed in text mode")
225
226 # Test if symbolic and hard links are resolved by extractfile(). The
227 # test link members each point to a regular member whose data is
228 # supposed to be exported.
229 def _test_fileobj_link(self, lnktype, regtype):
230 with self.tar.extractfile(lnktype) as a, \
231 self.tar.extractfile(regtype) as b:
232 self.assertEqual(a.name, b.name)
233
234 def test_fileobj_link1(self):
235 self._test_fileobj_link("ustar/lnktype", "ustar/regtype")
236
237 def test_fileobj_link2(self):
238 self._test_fileobj_link("./ustar/linktest2/lnktype",
239 "ustar/linktest1/regtype")
240
241 def test_fileobj_symlink1(self):
242 self._test_fileobj_link("ustar/symtype", "ustar/regtype")
243
244 def test_fileobj_symlink2(self):
245 self._test_fileobj_link("./ustar/linktest2/symtype",
246 "ustar/linktest1/regtype")
247
248 def test_issue14160(self):
249 self._test_fileobj_link("symtype2", "ustar/regtype")
250
251 def test_add_dir_getmember(self):
252 # bpo-21987
253 self.add_dir_and_getmember('bar')
254 self.add_dir_and_getmember('a'*101)
255
256 @unittest.skipUnless(hasattr(os, "getuid") and hasattr(os, "getgid"),
257 "Missing getuid or getgid implementation")
258 def add_dir_and_getmember(self, name):
259 def filter(tarinfo):
260 tarinfo.uid = tarinfo.gid = 100
261 return tarinfo
262
263 with os_helper.temp_cwd():
264 with tarfile.open(tmpname, 'w') as tar:
265 tar.format = tarfile.USTAR_FORMAT
266 try:
267 os.mkdir(name)
268 tar.add(name, filter=filter)
269 finally:
270 os.rmdir(name)
271 with tarfile.open(tmpname) as tar:
272 self.assertEqual(
273 tar.getmember(name),
274 tar.getmember(name + '/')
275 )
276
277 class GzipUstarReadTest(GzipTest, UstarReadTest):
278 pass
279
280 class Bz2UstarReadTest(Bz2Test, UstarReadTest):
281 pass
282
283 class LzmaUstarReadTest(LzmaTest, UstarReadTest):
284 pass
285
286 class ZstdUstarReadTest(ZstdTest, UstarReadTest):
287 pass
288
289 class ListTest(ReadTest, unittest.TestCase):
290
291 # Override setUp to use default encoding (UTF-8)
292 def setUp(self):
293 self.tar = tarfile.open(self.tarname, mode=self.mode)
294
295 def test_list(self):
296 tio = io.TextIOWrapper(io.BytesIO(), 'ascii', newline='\n')
297 with support.swap_attr(sys, 'stdout', tio):
298 self.tar.list(verbose=False)
299 out = tio.detach().getvalue()
300 self.assertIn(b'ustar/conttype', out)
301 self.assertIn(b'ustar/regtype', out)
302 self.assertIn(b'ustar/lnktype', out)
303 self.assertIn(b'ustar' + (b'/12345' * 40) + b'67/longname', out)
304 self.assertIn(b'./ustar/linktest2/symtype', out)
305 self.assertIn(b'./ustar/linktest2/lnktype', out)
306 # Make sure it puts trailing slash for directory
307 self.assertIn(b'ustar/dirtype/', out)
308 self.assertIn(b'ustar/dirtype-with-size/', out)
309 # Make sure it is able to print unencodable characters
310 def conv(b):
311 s = b.decode(self.tar.encoding, 'surrogateescape')
312 return s.encode('ascii', 'backslashreplace')
313 self.assertIn(conv(b'ustar/umlauts-\xc4\xd6\xdc\xe4\xf6\xfc\xdf'), out)
314 self.assertIn(conv(b'misc/regtype-hpux-signed-chksum-'
315 b'\xc4\xd6\xdc\xe4\xf6\xfc\xdf'), out)
316 self.assertIn(conv(b'misc/regtype-old-v7-signed-chksum-'
317 b'\xc4\xd6\xdc\xe4\xf6\xfc\xdf'), out)
318 self.assertIn(conv(b'pax/bad-pax-\xe4\xf6\xfc'), out)
319 self.assertIn(conv(b'pax/hdrcharset-\xe4\xf6\xfc'), out)
320 # Make sure it prints files separated by one newline without any
321 # 'ls -l'-like accessories if verbose flag is not being used
322 # ...
323 # ustar/conttype
324 # ustar/regtype
325 # ...
326 self.assertRegex(out, br'ustar/conttype ?\r?\n'
327 br'ustar/regtype ?\r?\n')
328 # Make sure it does not print the source of link without verbose flag
329 self.assertNotIn(b'link to', out)
330 self.assertNotIn(b'->', out)
331
332 def test_list_verbose(self):
333 tio = io.TextIOWrapper(io.BytesIO(), 'ascii', newline='\n')
334 with support.swap_attr(sys, 'stdout', tio):
335 self.tar.list(verbose=True)
336 out = tio.detach().getvalue()
337 # Make sure it prints files separated by one newline with 'ls -l'-like
338 # accessories if verbose flag is being used
339 # ...
340 # ?rw-r--r-- tarfile/tarfile 7011 2003-01-06 07:19:43 ustar/conttype
341 # -rw-r--r-- tarfile/tarfile 7011 2003-01-06 07:19:43 ustar/regtype
342 # drwxr-xr-x tarfile/tarfile 0 2003-01-05 15:19:43 ustar/dirtype/
343 # ...
344 #
345 # Array of values to modify the regex below:
346 # ((file_type, file_permissions, file_length), ...)
347 type_perm_lengths = (
348 (br'\?', b'rw-r--r--', b'7011'), (b'-', b'rw-r--r--', b'7011'),
349 (b'd', b'rwxr-xr-x', b'0'), (b'd', b'rwxr-xr-x', b'255'),
350 (br'\?', b'rw-r--r--', b'0'), (b'l', b'rwxrwxrwx', b'0'),
351 (b'b', b'rw-rw----', b'3,0'), (b'c', b'rw-rw-rw-', b'1,3'),
352 (b'p', b'rw-r--r--', b'0'))
353 self.assertRegex(out, b''.join(
354 [(tp + (br'%s tarfile/tarfile\s+%s ' % (perm, ln) +
355 br'\d{4}-\d\d-\d\d\s+\d\d:\d\d:\d\d '
356 br'ustar/\w+type[/>\sa-z-]*\n')) for tp, perm, ln
357 in type_perm_lengths]))
358 # Make sure it prints the source of link with verbose flag
359 self.assertIn(b'ustar/symtype -> regtype', out)
360 self.assertIn(b'./ustar/linktest2/symtype -> ../linktest1/regtype', out)
361 self.assertIn(b'./ustar/linktest2/lnktype link to '
362 b'./ustar/linktest1/regtype', out)
363 self.assertIn(b'gnu' + (b'/123' * 125) + b'/longlink link to gnu' +
364 (b'/123' * 125) + b'/longname', out)
365 self.assertIn(b'pax' + (b'/123' * 125) + b'/longlink link to pax' +
366 (b'/123' * 125) + b'/longname', out)
367
368 def test_list_members(self):
369 tio = io.TextIOWrapper(io.BytesIO(), 'ascii', newline='\n')
370 def members(tar):
371 for tarinfo in tar.getmembers():
372 if 'reg' in tarinfo.name:
373 yield tarinfo
374 with support.swap_attr(sys, 'stdout', tio):
375 self.tar.list(verbose=False, members=members(self.tar))
376 out = tio.detach().getvalue()
377 self.assertIn(b'ustar/regtype', out)
378 self.assertNotIn(b'ustar/conttype', out)
379
380
381 class GzipListTest(GzipTest, ListTest):
382 pass
383
384
385 class Bz2ListTest(Bz2Test, ListTest):
386 pass
387
388
389 class LzmaListTest(LzmaTest, ListTest):
390 pass
391
392 class ZstdListTest(ZstdTest, ListTest):
393 pass
394
395 class CommonReadTest(ReadTest):
396
397 def test_is_tarfile_erroneous(self):
398 with open(tmpname, "wb"):
399 pass
400
401 # is_tarfile works on filenames
402 self.assertFalse(tarfile.is_tarfile(tmpname))
403
404 # is_tarfile works on path-like objects
405 self.assertFalse(tarfile.is_tarfile(os_helper.FakePath(tmpname)))
406
407 # is_tarfile works on file objects
408 with open(tmpname, "rb") as fobj:
409 self.assertFalse(tarfile.is_tarfile(fobj))
410
411 # is_tarfile works on file-like objects
412 self.assertFalse(tarfile.is_tarfile(io.BytesIO(b"invalid")))
413
414 def test_is_tarfile_valid(self):
415 # is_tarfile works on filenames
416 self.assertTrue(tarfile.is_tarfile(self.tarname))
417
418 # is_tarfile works on path-like objects
419 self.assertTrue(tarfile.is_tarfile(os_helper.FakePath(self.tarname)))
420
421 # is_tarfile works on file objects
422 with open(self.tarname, "rb") as fobj:
423 self.assertTrue(tarfile.is_tarfile(fobj))
424
425 # is_tarfile works on file-like objects
426 with open(self.tarname, "rb") as fobj:
427 self.assertTrue(tarfile.is_tarfile(io.BytesIO(fobj.read())))
428
429 def test_is_tarfile_keeps_position(self):
430 # Test for issue44289: tarfile.is_tarfile() modifies
431 # file object's current position
432 with open(self.tarname, "rb") as fobj:
433 tarfile.is_tarfile(fobj)
434 self.assertEqual(fobj.tell(), 0)
435
436 with open(self.tarname, "rb") as fobj:
437 file_like = io.BytesIO(fobj.read())
438 tarfile.is_tarfile(file_like)
439 self.assertEqual(file_like.tell(), 0)
440
441 def test_empty_tarfile(self):
442 # Test for issue6123: Allow opening empty archives.
443 # This test checks if tarfile.open() is able to open an empty tar
444 # archive successfully. Note that an empty tar archive is not the
445 # same as an empty file!
446 with tarfile.open(tmpname, self.mode.replace("r", "w")):
447 pass
448 try:
449 tar = tarfile.open(tmpname, self.mode)
450 tar.getnames()
451 except tarfile.ReadError:
452 self.fail("tarfile.open() failed on empty archive")
453 else:
454 self.assertListEqual(tar.getmembers(), [])
455 finally:
456 tar.close()
457
458 def test_non_existent_tarfile(self):
459 # Test for issue11513: prevent non-existent gzipped tarfiles raising
460 # multiple exceptions.
461 with self.assertRaisesRegex(FileNotFoundError, "xxx"):
462 tarfile.open("xxx", self.mode)
463
464 def test_null_tarfile(self):
465 # Test for issue6123: Allow opening empty archives.
466 # This test guarantees that tarfile.open() does not treat an empty
467 # file as an empty tar archive.
468 with open(tmpname, "wb"):
469 pass
470 self.assertRaises(tarfile.ReadError, tarfile.open, tmpname, self.mode)
471 self.assertRaises(tarfile.ReadError, tarfile.open, tmpname)
472
473 def test_ignore_zeros(self):
474 # Test TarFile's ignore_zeros option.
475 # generate 512 pseudorandom bytes
476 data = Random(0).randbytes(512)
477 for char in (b'\0', b'a'):
478 # Test if EOFHeaderError ('\0') and InvalidHeaderError ('a')
479 # are ignored correctly.
480 with self.open(tmpname, "w") as fobj:
481 fobj.write(char * 1024)
482 tarinfo = tarfile.TarInfo("foo")
483 tarinfo.size = len(data)
484 fobj.write(tarinfo.tobuf())
485 fobj.write(data)
486
487 tar = tarfile.open(tmpname, mode="r", ignore_zeros=True)
488 try:
489 self.assertListEqual(tar.getnames(), ["foo"],
490 "ignore_zeros=True should have skipped the %r-blocks" %
491 char)
492 finally:
493 tar.close()
494
495 def test_premature_end_of_archive(self):
496 for size in (512, 600, 1024, 1200):
497 with tarfile.open(tmpname, "w:") as tar:
498 t = tarfile.TarInfo("foo")
499 t.size = 1024
500 tar.addfile(t, io.BytesIO(b"a" * 1024))
501
502 with open(tmpname, "r+b") as fobj:
503 fobj.truncate(size)
504
505 with tarfile.open(tmpname) as tar:
506 with self.assertRaisesRegex(tarfile.ReadError, "unexpected end of data"):
507 for t in tar:
508 pass
509
510 with tarfile.open(tmpname) as tar:
511 t = tar.next()
512
513 with self.assertRaisesRegex(tarfile.ReadError, "unexpected end of data"):
514 tar.extract(t, TEMPDIR, filter='data')
515
516 with self.assertRaisesRegex(tarfile.ReadError, "unexpected end of data"):
517 tar.extractfile(t).read()
518
519 def test_length_zero_header(self):
520 # bpo-39017 (CVE-2019-20907): reading a zero-length header should fail
521 # with an exception
522 with self.assertRaisesRegex(tarfile.ReadError, "file could not be opened successfully"):
523 with tarfile.open(support.findfile('recursion.tar', subdir='archivetestdata')):
524 pass
525
526 def test_extractfile_attrs(self):
527 # gh-74468: TarFile.name must name a file, not a parent archive.
528 file = self.tar.getmember('ustar/regtype')
529 with self.tar.extractfile(file) as fobj:
530 self.assertEqual(fobj.name, 'ustar/regtype')
531 self.assertRaises(AttributeError, fobj.fileno)
532 self.assertEqual(fobj.mode, 'rb')
533 self.assertIs(fobj.readable(), True)
534 self.assertIs(fobj.writable(), False)
535 if self.is_stream:
536 self.assertRaises(AttributeError, fobj.seekable)
537 else:
538 self.assertIs(fobj.seekable(), True)
539 self.assertIs(fobj.closed, False)
540 self.assertIs(fobj.closed, True)
541 self.assertEqual(fobj.name, 'ustar/regtype')
542 self.assertRaises(AttributeError, fobj.fileno)
543 self.assertEqual(fobj.mode, 'rb')
544 self.assertIs(fobj.readable(), True)
545 self.assertIs(fobj.writable(), False)
546 if self.is_stream:
547 self.assertRaises(AttributeError, fobj.seekable)
548 else:
549 self.assertIs(fobj.seekable(), True)
550
551
552 class MiscReadTestBase(CommonReadTest):
553 is_stream = False
554
555 def test_no_name_argument(self):
556 with open(self.tarname, "rb") as fobj:
557 self.assertIsInstance(fobj.name, str)
558 with tarfile.open(fileobj=fobj, mode=self.mode) as tar:
559 self.assertIsInstance(tar.name, str)
560 self.assertEqual(tar.name, os.path.abspath(fobj.name))
561
562 def test_no_name_attribute(self):
563 with open(self.tarname, "rb") as fobj:
564 data = fobj.read()
565 fobj = io.BytesIO(data)
566 self.assertRaises(AttributeError, getattr, fobj, "name")
567 tar = tarfile.open(fileobj=fobj, mode=self.mode)
568 self.assertIsNone(tar.name)
569
570 def test_empty_name_attribute(self):
571 with open(self.tarname, "rb") as fobj:
572 data = fobj.read()
573 fobj = io.BytesIO(data)
574 fobj.name = ""
575 with tarfile.open(fileobj=fobj, mode=self.mode) as tar:
576 self.assertIsNone(tar.name)
577
578 def test_int_name_attribute(self):
579 # Issue 21044: tarfile.open() should handle fileobj with an integer
580 # 'name' attribute.
581 fd = os.open(self.tarname, os.O_RDONLY)
582 with open(fd, 'rb') as fobj:
583 self.assertIsInstance(fobj.name, int)
584 with tarfile.open(fileobj=fobj, mode=self.mode) as tar:
585 self.assertIsNone(tar.name)
586
587 def test_bytes_name_attribute(self):
588 tarname = os.fsencode(self.tarname)
589 with open(tarname, 'rb') as fobj:
590 self.assertIsInstance(fobj.name, bytes)
591 with tarfile.open(fileobj=fobj, mode=self.mode) as tar:
592 self.assertIsInstance(tar.name, bytes)
593 self.assertEqual(tar.name, os.path.abspath(fobj.name))
594
595 def test_pathlike_name(self, tarname=None):
596 if tarname is None:
597 tarname = self.tarname
598 expected = os.path.abspath(tarname)
599 tarname = os_helper.FakePath(tarname)
600 with tarfile.open(tarname, mode=self.mode) as tar:
601 self.assertEqual(tar.name, expected)
602 with self.taropen(tarname) as tar:
603 self.assertEqual(tar.name, expected)
604 with tarfile.TarFile.open(tarname, mode=self.mode) as tar:
605 self.assertEqual(tar.name, expected)
606 if self.suffix == '':
607 with tarfile.TarFile(tarname, mode='r') as tar:
608 self.assertEqual(tar.name, expected)
609
610 def test_pathlike_bytes_name(self):
611 self.test_pathlike_name(os.fsencode(self.tarname))
612
613 def test_illegal_mode_arg(self):
614 with open(tmpname, 'wb'):
615 pass
616 with self.assertRaisesRegex(ValueError, 'mode must be '):
617 tar = self.taropen(tmpname, 'q')
618 with self.assertRaisesRegex(ValueError, 'mode must be '):
619 tar = self.taropen(tmpname, 'rw')
620 with self.assertRaisesRegex(ValueError, 'mode must be '):
621 tar = self.taropen(tmpname, '')
622
623 def test_fileobj_with_offset(self):
624 # Skip the first member and store values from the second member
625 # of the testtar.
626 tar = tarfile.open(self.tarname, mode=self.mode)
627 try:
628 tar.next()
629 t = tar.next()
630 name = t.name
631 offset = t.offset
632 with tar.extractfile(t) as f:
633 data = f.read()
634 finally:
635 tar.close()
636
637 # Open the testtar and seek to the offset of the second member.
638 with self.open(self.tarname) as fobj:
639 fobj.seek(offset)
640
641 # Test if the tarfile starts with the second member.
642 with tar.open(self.tarname, mode="r:", fileobj=fobj) as tar:
643 t = tar.next()
644 self.assertEqual(t.name, name)
645 # Read to the end of fileobj and test if seeking back to the
646 # beginning works.
647 tar.getmembers()
648 self.assertEqual(tar.extractfile(t).read(), data,
649 "seek back did not work")
650
651 def test_fail_comp(self):
652 # For Gzip and Bz2 Tests: fail with a ReadError on an uncompressed file.
653 self.assertRaises(tarfile.ReadError, tarfile.open, tarname, self.mode)
654 with open(tarname, "rb") as fobj:
655 self.assertRaises(tarfile.ReadError, tarfile.open,
656 fileobj=fobj, mode=self.mode)
657
658 def test_v7_dirtype(self):
659 # Test old style dirtype member (bug #1336623):
660 # Old V7 tars create directory members using an AREGTYPE
661 # header with a "/" appended to the filename field.
662 tarinfo = self.tar.getmember("misc/dirtype-old-v7")
663 self.assertEqual(tarinfo.type, tarfile.DIRTYPE,
664 "v7 dirtype failed")
665
666 def test_xstar_type(self):
667 # The xstar format stores extra atime and ctime fields inside the
668 # space reserved for the prefix field. The prefix field must be
669 # ignored in this case, otherwise it will mess up the name.
670 try:
671 self.tar.getmember("misc/regtype-xstar")
672 except KeyError:
673 self.fail("failed to find misc/regtype-xstar (mangled prefix?)")
674
675 def test_check_members(self):
676 for tarinfo in self.tar:
677 self.assertEqual(int(tarinfo.mtime), 0o7606136617,
678 "wrong mtime for %s" % tarinfo.name)
679 if not tarinfo.name.startswith("ustar/"):
680 continue
681 self.assertEqual(tarinfo.uname, "tarfile",
682 "wrong uname for %s" % tarinfo.name)
683
684 def test_find_members(self):
685 self.assertEqual(self.tar.getmembers()[-1].name, "misc/eof",
686 "could not find all members")
687
688 @unittest.skipUnless(hasattr(os, "link"),
689 "Missing hardlink implementation")
690 @os_helper.skip_unless_symlink
691 def test_extract_hardlink(self):
692 # Test hardlink extraction (e.g. bug #857297).
693 with tarfile.open(tarname, errorlevel=1, encoding="iso8859-1") as tar:
694 tar.extract("ustar/regtype", TEMPDIR, filter='data')
695 self.addCleanup(os_helper.unlink, os.path.join(TEMPDIR, "ustar/regtype"))
696
697 tar.extract("ustar/lnktype", TEMPDIR, filter='data')
698 self.addCleanup(os_helper.unlink, os.path.join(TEMPDIR, "ustar/lnktype"))
699 with open(os.path.join(TEMPDIR, "ustar/lnktype"), "rb") as f:
700 data = f.read()
701 self.assertEqual(sha256sum(data), sha256_regtype)
702
703 tar.extract("ustar/symtype", TEMPDIR, filter='data')
704 self.addCleanup(os_helper.unlink, os.path.join(TEMPDIR, "ustar/symtype"))
705 with open(os.path.join(TEMPDIR, "ustar/symtype"), "rb") as f:
706 data = f.read()
707 self.assertEqual(sha256sum(data), sha256_regtype)
708
709 @os_helper.skip_unless_working_chmod
710 def test_extractall(self):
711 # Test if extractall() correctly restores directory permissions
712 # and times (see issue1735).
713 tar = tarfile.open(tarname, encoding="iso8859-1")
714 DIR = os.path.join(TEMPDIR, "extractall")
715 os.mkdir(DIR)
716 try:
717 directories = [t for t in tar if t.isdir()]
718 tar.extractall(DIR, directories, filter='fully_trusted')
719 for tarinfo in directories:
720 path = os.path.join(DIR, tarinfo.name)
721 if sys.platform != "win32":
722 # Win32 has no support for fine grained permissions.
723 self.assertEqual(tarinfo.mode & 0o777,
724 os.stat(path).st_mode & 0o777,
725 tarinfo.name)
726 def format_mtime(mtime):
727 if isinstance(mtime, float):
728 return "{} ({})".format(mtime, mtime.hex())
729 else:
730 return "{!r} (int)".format(mtime)
731 file_mtime = os.path.getmtime(path)
732 errmsg = "tar mtime {0} != file time {1} of path {2!a}".format(
733 format_mtime(tarinfo.mtime),
734 format_mtime(file_mtime),
735 path)
736 self.assertEqual(tarinfo.mtime, file_mtime, errmsg)
737 finally:
738 tar.close()
739 os_helper.rmtree(DIR)
740
741 @staticmethod
742 def test_extractall_default_filter():
743 # Test that the default filter is now "data", and the other filter types are not used.
744 DIR = pathlib.Path(TEMPDIR) / "extractall_default_filter"
745 with (
746 os_helper.temp_dir(DIR),
747 tarfile.open(tarname, encoding="iso8859-1") as tar,
748 unittest.mock.patch("tarfile.data_filter", wraps=tarfile.data_filter) as mock_data_filter,
749 unittest.mock.patch("tarfile.tar_filter", wraps=tarfile.tar_filter) as mock_tar_filter,
750 unittest.mock.patch("tarfile.fully_trusted_filter", wraps=tarfile.fully_trusted_filter) as mock_ft_filter
751 ):
752 directories = [t for t in tar if t.isdir()]
753 tar.extractall(DIR, directories)
754
755 mock_data_filter.assert_called()
756 mock_ft_filter.assert_not_called()
757 mock_tar_filter.assert_not_called()
758
759 @os_helper.skip_unless_working_chmod
760 def test_extract_directory(self):
761 dirtype = "ustar/dirtype"
762 DIR = os.path.join(TEMPDIR, "extractdir")
763 os.mkdir(DIR)
764 try:
765 with tarfile.open(tarname, encoding="iso8859-1") as tar:
766 tarinfo = tar.getmember(dirtype)
767 tar.extract(tarinfo, path=DIR, filter='fully_trusted')
768 extracted = os.path.join(DIR, dirtype)
769 self.assertEqual(os.path.getmtime(extracted), tarinfo.mtime)
770 if sys.platform != "win32":
771 self.assertEqual(os.stat(extracted).st_mode & 0o777, 0o755)
772 finally:
773 os_helper.rmtree(DIR)
774
775 def test_extractall_pathlike_dir(self):
776 DIR = os.path.join(TEMPDIR, "extractall")
777 with os_helper.temp_dir(DIR), \
778 tarfile.open(tarname, encoding="iso8859-1") as tar:
779 directories = [t for t in tar if t.isdir()]
780 tar.extractall(os_helper.FakePath(DIR), directories, filter='fully_trusted')
781 for tarinfo in directories:
782 path = os.path.join(DIR, tarinfo.name)
783 self.assertEqual(os.path.getmtime(path), tarinfo.mtime)
784
785 def test_extract_pathlike_dir(self):
786 dirtype = "ustar/dirtype"
787 DIR = os.path.join(TEMPDIR, "extractall")
788 with os_helper.temp_dir(DIR), \
789 tarfile.open(tarname, encoding="iso8859-1") as tar:
790 tarinfo = tar.getmember(dirtype)
791 tar.extract(tarinfo, path=os_helper.FakePath(DIR), filter='fully_trusted')
792 extracted = os.path.join(DIR, dirtype)
793 self.assertEqual(os.path.getmtime(extracted), tarinfo.mtime)
794
795 def test_init_close_fobj(self):
796 # Issue #7341: Close the internal file object in the TarFile
797 # constructor in case of an error. For the test we rely on
798 # the fact that opening an empty file raises a ReadError.
799 empty = os.path.join(TEMPDIR, "empty")
800 with open(empty, "wb") as fobj:
801 fobj.write(b"")
802
803 try:
804 tar = object.__new__(tarfile.TarFile)
805 try:
806 tar.__init__(empty)
807 except tarfile.ReadError:
808 self.assertTrue(tar.fileobj.closed)
809 else:
810 self.fail("ReadError not raised")
811 finally:
812 os_helper.unlink(empty)
813
814 def test_parallel_iteration(self):
815 # Issue #16601: Restarting iteration over tarfile continued
816 # from where it left off.
817 with tarfile.open(self.tarname) as tar:
818 for m1, m2 in zip(tar, tar):
819 self.assertEqual(m1.offset, m2.offset)
820 self.assertEqual(m1.get_info(), m2.get_info())
821
822 @unittest.skipIf(zlib is None, "requires zlib")
823 def test_zlib_error_does_not_leak(self):
824 # bpo-39039: tarfile.open allowed zlib exceptions to bubble up when
825 # parsing certain types of invalid data
826 with unittest.mock.patch("tarfile.TarInfo.fromtarfile") as mock:
827 mock.side_effect = zlib.error
828 with self.assertRaises(tarfile.ReadError):
829 tarfile.open(self.tarname)
830
831 def test_next_on_empty_tarfile(self):
832 fd = io.BytesIO()
833 tf = tarfile.open(fileobj=fd, mode="w")
834 tf.close()
835
836 fd.seek(0)
837 with tarfile.open(fileobj=fd, mode="r|") as tf:
838 self.assertEqual(tf.next(), None)
839
840 fd.seek(0)
841 with tarfile.open(fileobj=fd, mode="r") as tf:
842 self.assertEqual(tf.next(), None)
843
844 def _setup_symlink_to_target(self, temp_dirpath):
845 target_filepath = os.path.join(temp_dirpath, "target")
846 ustar_dirpath = os.path.join(temp_dirpath, "ustar")
847 hardlink_filepath = os.path.join(ustar_dirpath, "lnktype")
848 with open(target_filepath, "wb") as f:
849 f.write(b"target")
850 os.makedirs(ustar_dirpath)
851 os.symlink(target_filepath, hardlink_filepath)
852 return target_filepath, hardlink_filepath
853
854 def _assert_on_file_content(self, filepath, digest):
855 with open(filepath, "rb") as f:
856 data = f.read()
857 self.assertEqual(sha256sum(data), digest)
858
859 @unittest.skipUnless(
860 hasattr(os, "link"), "Missing hardlink implementation"
861 )
862 @os_helper.skip_unless_symlink
863 def test_extract_hardlink_on_symlink(self):
864 """
865 This test verifies that extracting a hardlink will not follow an
866 existing symlink after a FileExistsError on os.link.
867 """
868 with os_helper.temp_dir() as DIR:
869 target_filepath, hardlink_filepath = self._setup_symlink_to_target(DIR)
870 with tarfile.open(tarname, encoding="iso8859-1") as tar:
871 tar.extract("ustar/regtype", DIR, filter="data")
872 tar.extract("ustar/lnktype", DIR, filter="data")
873 self._assert_on_file_content(target_filepath, sha256sum(b"target"))
874 self._assert_on_file_content(hardlink_filepath, sha256_regtype)
875
876 @unittest.skipUnless(
877 hasattr(os, "link"), "Missing hardlink implementation"
878 )
879 @os_helper.skip_unless_symlink
880 def test_extractall_hardlink_on_symlink(self):
881 """
882 This test verifies that extracting a hardlink will not follow an
883 existing symlink after a FileExistsError on os.link.
884 """
885 with os_helper.temp_dir() as DIR:
886 target_filepath, hardlink_filepath = self._setup_symlink_to_target(DIR)
887 with tarfile.open(tarname, encoding="iso8859-1") as tar:
888 tar.extractall(
889 DIR, members=["ustar/regtype", "ustar/lnktype"], filter="data",
890 )
891 self._assert_on_file_content(target_filepath, sha256sum(b"target"))
892 self._assert_on_file_content(hardlink_filepath, sha256_regtype)
893
894
895 class MiscReadTest(MiscReadTestBase, unittest.TestCase):
896 test_fail_comp = None
897
898 class GzipMiscReadTest(GzipTest, MiscReadTestBase, unittest.TestCase):
899 pass
900
901 class Bz2MiscReadTest(Bz2Test, MiscReadTestBase, unittest.TestCase):
902 pass
903
904 class LzmaMiscReadTest(LzmaTest, MiscReadTestBase, unittest.TestCase):
905 pass
906
907 class ZstdMiscReadTest(ZstdTest, MiscReadTestBase, unittest.TestCase):
908 pass
909
910 class StreamReadTest(CommonReadTest, unittest.TestCase):
911
912 prefix="r|"
913 is_stream = True
914
915 def test_read_through(self):
916 # Issue #11224: A poorly designed _FileInFile.read() method
917 # caused seeking errors with stream tar files.
918 for tarinfo in self.tar:
919 if not tarinfo.isreg():
920 continue
921 with self.tar.extractfile(tarinfo) as fobj:
922 while True:
923 try:
924 buf = fobj.read(512)
925 except tarfile.StreamError:
926 self.fail("simple read-through using "
927 "TarFile.extractfile() failed")
928 if not buf:
929 break
930
931 def test_fileobj_regular_file(self):
932 tarinfo = self.tar.next() # get "regtype" (can't use getmember)
933 with self.tar.extractfile(tarinfo) as fobj:
934 data = fobj.read()
935 self.assertEqual(len(data), tarinfo.size,
936 "regular file extraction failed")
937 self.assertEqual(sha256sum(data), sha256_regtype,
938 "regular file extraction failed")
939
940 def test_provoke_stream_error(self):
941 tarinfos = self.tar.getmembers()
942 with self.tar.extractfile(tarinfos[0]) as f: # read the first member
943 self.assertRaises(tarfile.StreamError, f.read)
944
945 def test_compare_members(self):
946 tar1 = tarfile.open(tarname, encoding="iso8859-1")
947 try:
948 tar2 = self.tar
949
950 while True:
951 t1 = tar1.next()
952 t2 = tar2.next()
953 if t1 is None:
954 break
955 self.assertIsNotNone(t2, "stream.next() failed.")
956
957 if t2.islnk() or t2.issym():
958 with self.assertRaises(tarfile.StreamError):
959 tar2.extractfile(t2)
960 continue
961
962 v1 = tar1.extractfile(t1)
963 v2 = tar2.extractfile(t2)
964 if v1 is None:
965 continue
966 self.assertIsNotNone(v2, "stream.extractfile() failed")
967 self.assertEqual(v1.read(), v2.read(),
968 "stream extraction failed")
969 finally:
970 tar1.close()
971
972 class GzipStreamReadTest(GzipTest, StreamReadTest):
973 pass
974
975 class Bz2StreamReadTest(Bz2Test, StreamReadTest):
976 pass
977
978 class LzmaStreamReadTest(LzmaTest, StreamReadTest):
979 pass
980
981 class ZstdStreamReadTest(ZstdTest, StreamReadTest):
982 pass
983
984 class TarStreamModeReadTest(StreamModeTest, unittest.TestCase):
985
986 def test_stream_mode_no_cache(self):
987 for _ in self.tar:
988 pass
989 self.assertEqual(self.tar.members, [])
990
991 class GzipStreamModeReadTest(GzipTest, TarStreamModeReadTest):
992 pass
993
994 class Bz2StreamModeReadTest(Bz2Test, TarStreamModeReadTest):
995 pass
996
997 class LzmaStreamModeReadTest(LzmaTest, TarStreamModeReadTest):
998 pass
999
1000 class ZstdStreamModeReadTest(ZstdTest, TarStreamModeReadTest):
1001 pass
1002
1003 class DetectReadTest(TarTest, unittest.TestCase):
1004 def _testfunc_file(self, name, mode):
1005 try:
1006 tar = tarfile.open(name, mode)
1007 except tarfile.ReadError as e:
1008 self.fail()
1009 else:
1010 tar.close()
1011
1012 def _testfunc_fileobj(self, name, mode):
1013 try:
1014 with open(name, "rb") as f:
1015 tar = tarfile.open(name, mode, fileobj=f)
1016 except tarfile.ReadError as e:
1017 self.fail()
1018 else:
1019 tar.close()
1020
1021 def _test_modes(self, testfunc):
1022 if self.suffix:
1023 with self.assertRaises(tarfile.ReadError):
1024 tarfile.open(tarname, mode="r:" + self.suffix)
1025 with self.assertRaises(tarfile.ReadError):
1026 tarfile.open(tarname, mode="r|" + self.suffix)
1027 with self.assertRaises(tarfile.ReadError):
1028 tarfile.open(self.tarname, mode="r:")
1029 with self.assertRaises(tarfile.ReadError):
1030 tarfile.open(self.tarname, mode="r|")
1031 testfunc(self.tarname, "r")
1032 testfunc(self.tarname, "r:" + self.suffix)
1033 testfunc(self.tarname, "r:*")
1034 testfunc(self.tarname, "r|" + self.suffix)
1035 testfunc(self.tarname, "r|*")
1036
1037 def test_detect_file(self):
1038 self._test_modes(self._testfunc_file)
1039
1040 def test_detect_fileobj(self):
1041 self._test_modes(self._testfunc_fileobj)
1042
1043 class GzipDetectReadTest(GzipTest, DetectReadTest):
1044 pass
1045
1046 class Bz2DetectReadTest(Bz2Test, DetectReadTest):
1047 def test_detect_stream_bz2(self):
1048 # Originally, tarfile's stream detection looked for the string
1049 # "BZh91" at the start of the file. This is incorrect because
1050 # the '9' represents the blocksize (900,000 bytes). If the file was
1051 # compressed using another blocksize autodetection fails.
1052 with open(tarname, "rb") as fobj:
1053 data = fobj.read()
1054
1055 # Compress with blocksize 100,000 bytes, the file starts with "BZh11".
1056 with bz2.BZ2File(tmpname, "wb", compresslevel=1) as fobj:
1057 fobj.write(data)
1058
1059 self._testfunc_file(tmpname, "r|*")
1060
1061 class LzmaDetectReadTest(LzmaTest, DetectReadTest):
1062 pass
1063
1064 class ZstdDetectReadTest(ZstdTest, DetectReadTest):
1065 pass
1066
1067 class GzipBrokenHeaderCorrectException(GzipTest, unittest.TestCase):
1068 """
1069 See: https://github.com/python/cpython/issues/107396
1070 """
1071 def runTest(self):
1072 f = io.BytesIO(
1073 b'\x1f\x8b' # header
1074 b'\x08' # compression method
1075 b'\x04' # flags
1076 b'\0\0\0\0\0\0' # timestamp, compression data, OS ID
1077 b'\0\x01' # size
1078 b'\0\0\0\0\0' # corrupt data (zeros)
1079 )
1080 with self.assertRaises(tarfile.ReadError):
1081 tarfile.open(fileobj=f, mode='r|gz')
1082
1083
1084 class MemberReadTest(ReadTest, unittest.TestCase):
1085
1086 def _test_member(self, tarinfo, chksum=None, **kwargs):
1087 if chksum is not None:
1088 with self.tar.extractfile(tarinfo) as f:
1089 self.assertEqual(sha256sum(f.read()), chksum,
1090 "wrong sha256sum for %s" % tarinfo.name)
1091
1092 kwargs["mtime"] = 0o7606136617
1093 kwargs["uid"] = 1000
1094 kwargs["gid"] = 100
1095 if "old-v7" not in tarinfo.name:
1096 # V7 tar can't handle alphabetic owners.
1097 kwargs["uname"] = "tarfile"
1098 kwargs["gname"] = "tarfile"
1099 for k, v in kwargs.items():
1100 self.assertEqual(getattr(tarinfo, k), v,
1101 "wrong value in %s field of %s" % (k, tarinfo.name))
1102
1103 def test_find_regtype(self):
1104 tarinfo = self.tar.getmember("ustar/regtype")
1105 self._test_member(tarinfo, size=7011, chksum=sha256_regtype)
1106
1107 def test_find_conttype(self):
1108 tarinfo = self.tar.getmember("ustar/conttype")
1109 self._test_member(tarinfo, size=7011, chksum=sha256_regtype)
1110
1111 def test_find_dirtype(self):
1112 tarinfo = self.tar.getmember("ustar/dirtype")
1113 self._test_member(tarinfo, size=0)
1114
1115 def test_find_dirtype_with_size(self):
1116 tarinfo = self.tar.getmember("ustar/dirtype-with-size")
1117 self._test_member(tarinfo, size=255)
1118
1119 def test_find_lnktype(self):
1120 tarinfo = self.tar.getmember("ustar/lnktype")
1121 self._test_member(tarinfo, size=0, linkname="ustar/regtype")
1122
1123 def test_find_symtype(self):
1124 tarinfo = self.tar.getmember("ustar/symtype")
1125 self._test_member(tarinfo, size=0, linkname="regtype")
1126
1127 def test_find_blktype(self):
1128 tarinfo = self.tar.getmember("ustar/blktype")
1129 self._test_member(tarinfo, size=0, devmajor=3, devminor=0)
1130
1131 def test_find_chrtype(self):
1132 tarinfo = self.tar.getmember("ustar/chrtype")
1133 self._test_member(tarinfo, size=0, devmajor=1, devminor=3)
1134
1135 def test_find_fifotype(self):
1136 tarinfo = self.tar.getmember("ustar/fifotype")
1137 self._test_member(tarinfo, size=0)
1138
1139 def test_find_sparse(self):
1140 tarinfo = self.tar.getmember("ustar/sparse")
1141 self._test_member(tarinfo, size=86016, chksum=sha256_sparse)
1142
1143 def test_find_gnusparse(self):
1144 tarinfo = self.tar.getmember("gnu/sparse")
1145 self._test_member(tarinfo, size=86016, chksum=sha256_sparse)
1146
1147 def test_find_gnusparse_00(self):
1148 tarinfo = self.tar.getmember("gnu/sparse-0.0")
1149 self._test_member(tarinfo, size=86016, chksum=sha256_sparse)
1150
1151 def test_find_gnusparse_01(self):
1152 tarinfo = self.tar.getmember("gnu/sparse-0.1")
1153 self._test_member(tarinfo, size=86016, chksum=sha256_sparse)
1154
1155 def test_find_gnusparse_10(self):
1156 tarinfo = self.tar.getmember("gnu/sparse-1.0")
1157 self._test_member(tarinfo, size=86016, chksum=sha256_sparse)
1158
1159 def test_find_umlauts(self):
1160 tarinfo = self.tar.getmember("ustar/umlauts-"
1161 "\xc4\xd6\xdc\xe4\xf6\xfc\xdf")
1162 self._test_member(tarinfo, size=7011, chksum=sha256_regtype)
1163
1164 def test_find_ustar_longname(self):
1165 name = "ustar/" + "12345/" * 39 + "1234567/longname"
1166 self.assertIn(name, self.tar.getnames())
1167
1168 def test_find_regtype_oldv7(self):
1169 tarinfo = self.tar.getmember("misc/regtype-old-v7")
1170 self._test_member(tarinfo, size=7011, chksum=sha256_regtype)
1171
1172 def test_find_pax_umlauts(self):
1173 self.tar.close()
1174 self.tar = tarfile.open(self.tarname, mode=self.mode,
1175 encoding="iso8859-1")
1176 tarinfo = self.tar.getmember("pax/umlauts-"
1177 "\xc4\xd6\xdc\xe4\xf6\xfc\xdf")
1178 self._test_member(tarinfo, size=7011, chksum=sha256_regtype)
1179
1180
1181 class LongnameTest:
1182
1183 def test_read_longname(self):
1184 # Test reading of longname (bug #1471427).
1185 longname = self.subdir + "/" + "123/" * 125 + "longname"
1186 try:
1187 tarinfo = self.tar.getmember(longname)
1188 except KeyError:
1189 self.fail("longname not found")
1190 self.assertNotEqual(tarinfo.type, tarfile.DIRTYPE,
1191 "read longname as dirtype")
1192
1193 def test_read_longlink(self):
1194 longname = self.subdir + "/" + "123/" * 125 + "longname"
1195 longlink = self.subdir + "/" + "123/" * 125 + "longlink"
1196 try:
1197 tarinfo = self.tar.getmember(longlink)
1198 except KeyError:
1199 self.fail("longlink not found")
1200 self.assertEqual(tarinfo.linkname, longname, "linkname wrong")
1201
1202 def test_truncated_longname(self):
1203 longname = self.subdir + "/" + "123/" * 125 + "longname"
1204 tarinfo = self.tar.getmember(longname)
1205 offset = tarinfo.offset
1206 self.tar.fileobj.seek(offset)
1207 fobj = io.BytesIO(self.tar.fileobj.read(3 * 512))
1208 with self.assertRaises(tarfile.ReadError):
1209 tarfile.open(name="foo.tar", fileobj=fobj)
1210
1211 def test_header_offset(self):
1212 # Test if the start offset of the TarInfo object includes
1213 # the preceding extended header.
1214 longname = self.subdir + "/" + "123/" * 125 + "longname"
1215 offset = self.tar.getmember(longname).offset
1216 with open(tarname, "rb") as fobj:
1217 fobj.seek(offset)
1218 tarinfo = tarfile.TarInfo.frombuf(fobj.read(512),
1219 "iso8859-1", "strict")
1220 self.assertEqual(tarinfo.type, self.longnametype)
1221
1222 def test_longname_directory(self):
1223 # Test reading a longlink directory. Issue #47231.
1224 longdir = ('a' * 101) + '/'
1225 with os_helper.temp_cwd():
1226 with tarfile.open(tmpname, 'w') as tar:
1227 tar.format = self.format
1228 try:
1229 os.mkdir(longdir)
1230 tar.add(longdir)
1231 finally:
1232 os.rmdir(longdir.rstrip("/"))
1233 with tarfile.open(tmpname) as tar:
1234 self.assertIsNotNone(tar.getmember(longdir))
1235 self.assertIsNotNone(tar.getmember(longdir.removesuffix('/')))
1236
1237 def test_longname_file_not_directory(self):
1238 # Test reading a longname file and ensure it is not handled as a directory
1239 # Issue #141707
1240 buf = io.BytesIO()
1241 with tarfile.open(mode='w', fileobj=buf, format=self.format) as tar:
1242 ti = tarfile.TarInfo()
1243 ti.type = tarfile.AREGTYPE
1244 ti.name = ('a' * 99) + '/' + ('b' * 3)
1245 tar.addfile(ti)
1246
1247 expected = {t.name: t.type for t in tar.getmembers()}
1248
1249 buf.seek(0)
1250 with tarfile.open(mode='r', fileobj=buf) as tar:
1251 actual = {t.name: t.type for t in tar.getmembers()}
1252
1253 self.assertEqual(expected, actual)
1254
1255
1256 class GNUReadTest(LongnameTest, ReadTest, unittest.TestCase):
1257
1258 subdir = "gnu"
1259 longnametype = tarfile.GNUTYPE_LONGNAME
1260 format = tarfile.GNU_FORMAT
1261
1262 # Since 3.2 tarfile is supposed to accurately restore sparse members and
1263 # produce files with holes. This is what we actually want to test here.
1264 # Unfortunately, not all platforms/filesystems support sparse files, and
1265 # even on platforms that do it is non-trivial to make reliable assertions
1266 # about holes in files. Therefore, we first do one basic test which works
1267 # an all platforms, and after that a test that will work only on
1268 # platforms/filesystems that prove to support sparse files.
1269 def _test_sparse_file(self, name):
1270 self.tar.extract(name, TEMPDIR, filter='data')
1271 filename = os.path.join(TEMPDIR, name)
1272 with open(filename, "rb") as fobj:
1273 data = fobj.read()
1274 self.assertEqual(sha256sum(data), sha256_sparse,
1275 "wrong sha256sum for %s" % name)
1276
1277 if self._fs_supports_holes():
1278 s = os.stat(filename)
1279 self.assertLess(s.st_blocks * 512, s.st_size)
1280
1281 def test_sparse_file_old(self):
1282 self._test_sparse_file("gnu/sparse")
1283
1284 def test_sparse_file_00(self):
1285 self._test_sparse_file("gnu/sparse-0.0")
1286
1287 def test_sparse_file_01(self):
1288 self._test_sparse_file("gnu/sparse-0.1")
1289
1290 def test_sparse_file_10(self):
1291 self._test_sparse_file("gnu/sparse-1.0")
1292
1293 @staticmethod
1294 def _fs_supports_holes():
1295 # Return True if the platform knows the st_blocks stat attribute and
1296 # uses st_blocks units of 512 bytes, and if the filesystem is able to
1297 # store holes of 4 KiB in files.
1298 #
1299 # The function returns False if page size is larger than 4 KiB.
1300 # For example, ppc64 uses pages of 64 KiB.
1301 if sys.platform.startswith(("linux", "android")):
1302 # Linux evidentially has 512 byte st_blocks units.
1303 name = os.path.join(TEMPDIR, "sparse-test")
1304 with open(name, "wb") as fobj:
1305 # Seek to "punch a hole" of 4 KiB
1306 fobj.seek(4096)
1307 fobj.write(b'x' * 4096)
1308 fobj.truncate()
1309 s = os.stat(name)
1310 os_helper.unlink(name)
1311 return (s.st_blocks * 512 < s.st_size)
1312 else:
1313 return False
1314
1315
1316 class PaxReadTest(LongnameTest, ReadTest, unittest.TestCase):
1317
1318 subdir = "pax"
1319 longnametype = tarfile.XHDTYPE
1320 format = tarfile.PAX_FORMAT
1321
1322 def test_pax_global_headers(self):
1323 tar = tarfile.open(tarname, encoding="iso8859-1")
1324 try:
1325 tarinfo = tar.getmember("pax/regtype1")
1326 self.assertEqual(tarinfo.uname, "foo")
1327 self.assertEqual(tarinfo.gname, "bar")
1328 self.assertEqual(tarinfo.pax_headers.get("VENDOR.umlauts"),
1329 "\xc4\xd6\xdc\xe4\xf6\xfc\xdf")
1330
1331 tarinfo = tar.getmember("pax/regtype2")
1332 self.assertEqual(tarinfo.uname, "")
1333 self.assertEqual(tarinfo.gname, "bar")
1334 self.assertEqual(tarinfo.pax_headers.get("VENDOR.umlauts"),
1335 "\xc4\xd6\xdc\xe4\xf6\xfc\xdf")
1336
1337 tarinfo = tar.getmember("pax/regtype3")
1338 self.assertEqual(tarinfo.uname, "tarfile")
1339 self.assertEqual(tarinfo.gname, "tarfile")
1340 self.assertEqual(tarinfo.pax_headers.get("VENDOR.umlauts"),
1341 "\xc4\xd6\xdc\xe4\xf6\xfc\xdf")
1342 finally:
1343 tar.close()
1344
1345 def test_pax_number_fields(self):
1346 # All following number fields are read from the pax header.
1347 tar = tarfile.open(tarname, encoding="iso8859-1")
1348 try:
1349 tarinfo = tar.getmember("pax/regtype4")
1350 self.assertEqual(tarinfo.size, 7011)
1351 self.assertEqual(tarinfo.uid, 123)
1352 self.assertEqual(tarinfo.gid, 123)
1353 self.assertEqual(tarinfo.mtime, 1041808783.0)
1354 self.assertEqual(type(tarinfo.mtime), float)
1355 self.assertEqual(float(tarinfo.pax_headers["atime"]), 1041808783.0)
1356 self.assertEqual(float(tarinfo.pax_headers["ctime"]), 1041808783.0)
1357 finally:
1358 tar.close()
1359
1360 def test_pax_header_bad_formats(self):
1361 # The fields from the pax header have priority over the
1362 # TarInfo.
1363 pax_header_replacements = (
1364 b" foo=bar\n",
1365 b"0 \n",
1366 b"1 \n",
1367 b"2 \n",
1368 b"3 =\n",
1369 b"4 =a\n",
1370 b"1000000 foo=bar\n",
1371 b"0 foo=bar\n",
1372 b"-12 foo=bar\n",
1373 b"000000000000000000000000036 foo=bar\n",
1374 )
1375 pax_headers = {"foo": "bar"}
1376
1377 for replacement in pax_header_replacements:
1378 with self.subTest(header=replacement):
1379 tar = tarfile.open(tmpname, "w", format=tarfile.PAX_FORMAT,
1380 encoding="iso8859-1")
1381 try:
1382 t = tarfile.TarInfo()
1383 t.name = "pax" # non-ASCII
1384 t.uid = 1
1385 t.pax_headers = pax_headers
1386 tar.addfile(t)
1387 finally:
1388 tar.close()
1389
1390 with open(tmpname, "rb") as f:
1391 data = f.read()
1392 self.assertIn(b"11 foo=bar\n", data)
1393 data = data.replace(b"11 foo=bar\n", replacement)
1394
1395 with open(tmpname, "wb") as f:
1396 f.truncate()
1397 f.write(data)
1398
1399 with self.assertRaisesRegex(tarfile.ReadError, r"method tar: ReadError\('invalid header'\)"):
1400 tarfile.open(tmpname, encoding="iso8859-1")
1401
1402
1403 class WriteTestBase(TarTest):
1404 # Put all write tests in here that are supposed to be tested
1405 # in all possible mode combinations.
1406
1407 def test_fileobj_no_close(self):
1408 fobj = io.BytesIO()
1409 with tarfile.open(fileobj=fobj, mode=self.mode) as tar:
1410 tar.addfile(tarfile.TarInfo("foo"))
1411 self.assertFalse(fobj.closed, "external fileobjs must never closed")
1412 # Issue #20238: Incomplete gzip output with mode="w:gz"
1413 data = fobj.getvalue()
1414 del tar
1415 support.gc_collect()
1416 self.assertFalse(fobj.closed)
1417 self.assertEqual(data, fobj.getvalue())
1418
1419 def test_eof_marker(self):
1420 # Make sure an end of archive marker is written (two zero blocks).
1421 # tarfile insists on aligning archives to a 20 * 512 byte recordsize.
1422 # So, we create an archive that has exactly 10240 bytes without the
1423 # marker, and has 20480 bytes once the marker is written.
1424 with tarfile.open(tmpname, self.mode) as tar:
1425 t = tarfile.TarInfo("foo")
1426 t.size = tarfile.RECORDSIZE - tarfile.BLOCKSIZE
1427 tar.addfile(t, io.BytesIO(b"a" * t.size))
1428
1429 with self.open(tmpname, "rb") as fobj:
1430 self.assertEqual(len(fobj.read()), tarfile.RECORDSIZE * 2)
1431
1432
1433 class WriteTest(WriteTestBase, unittest.TestCase):
1434
1435 prefix = "w:"
1436
1437 def test_100_char_name(self):
1438 # The name field in a tar header stores strings of at most 100 chars.
1439 # If a string is shorter than 100 chars it has to be padded with '\0',
1440 # which implies that a string of exactly 100 chars is stored without
1441 # a trailing '\0'.
1442 name = "0123456789" * 10
1443 tar = tarfile.open(tmpname, self.mode)
1444 try:
1445 t = tarfile.TarInfo(name)
1446 tar.addfile(t)
1447 finally:
1448 tar.close()
1449
1450 tar = tarfile.open(tmpname)
1451 try:
1452 self.assertEqual(tar.getnames()[0], name,
1453 "failed to store 100 char filename")
1454 finally:
1455 tar.close()
1456
1457 def test_tar_size(self):
1458 # Test for bug #1013882.
1459 tar = tarfile.open(tmpname, self.mode)
1460 try:
1461 path = os.path.join(TEMPDIR, "file")
1462 with open(path, "wb") as fobj:
1463 fobj.write(b"aaa")
1464 tar.add(path)
1465 finally:
1466 tar.close()
1467 self.assertGreater(os.path.getsize(tmpname), 0,
1468 "tarfile is empty")
1469
1470 # The test_*_size tests test for bug #1167128.
1471 def test_file_size(self):
1472 tar = tarfile.open(tmpname, self.mode)
1473 try:
1474 path = os.path.join(TEMPDIR, "file")
1475 with open(path, "wb"):
1476 pass
1477 tarinfo = tar.gettarinfo(path)
1478 self.assertEqual(tarinfo.size, 0)
1479
1480 with open(path, "wb") as fobj:
1481 fobj.write(b"aaa")
1482 tarinfo = tar.gettarinfo(path)
1483 self.assertEqual(tarinfo.size, 3)
1484 finally:
1485 tar.close()
1486
1487 def test_directory_size(self):
1488 path = os.path.join(TEMPDIR, "directory")
1489 os.mkdir(path)
1490 try:
1491 tar = tarfile.open(tmpname, self.mode)
1492 try:
1493 tarinfo = tar.gettarinfo(path)
1494 self.assertEqual(tarinfo.size, 0)
1495 finally:
1496 tar.close()
1497 finally:
1498 os_helper.rmdir(path)
1499
1500 # mock the following:
1501 # os.listdir: so we know that files are in the wrong order
1502 def test_ordered_recursion(self):
1503 path = os.path.join(TEMPDIR, "directory")
1504 os.mkdir(path)
1505 open(os.path.join(path, "1"), "a").close()
1506 open(os.path.join(path, "2"), "a").close()
1507 try:
1508 tar = tarfile.open(tmpname, self.mode)
1509 try:
1510 with unittest.mock.patch('os.listdir') as mock_listdir:
1511 mock_listdir.return_value = ["2", "1"]
1512 tar.add(path)
1513 paths = []
1514 for m in tar.getmembers():
1515 paths.append(os.path.split(m.name)[-1])
1516 self.assertEqual(paths, ["directory", "1", "2"]);
1517 finally:
1518 tar.close()
1519 finally:
1520 os_helper.unlink(os.path.join(path, "1"))
1521 os_helper.unlink(os.path.join(path, "2"))
1522 os_helper.rmdir(path)
1523
1524 def test_gettarinfo_pathlike_name(self):
1525 with tarfile.open(tmpname, self.mode) as tar:
1526 path = os.path.join(TEMPDIR, "file")
1527 with open(path, "wb") as fobj:
1528 fobj.write(b"aaa")
1529 tarinfo = tar.gettarinfo(os_helper.FakePath(path))
1530 tarinfo2 = tar.gettarinfo(path)
1531 self.assertIsInstance(tarinfo.name, str)
1532 self.assertEqual(tarinfo.name, tarinfo2.name)
1533 self.assertEqual(tarinfo.size, 3)
1534
1535 @unittest.skipUnless(hasattr(os, "link"),
1536 "Missing hardlink implementation")
1537 def test_link_size(self):
1538 link = os.path.join(TEMPDIR, "link")
1539 target = os.path.join(TEMPDIR, "link_target")
1540 with open(target, "wb") as fobj:
1541 fobj.write(b"aaa")
1542 try:
1543 os.link(target, link)
1544 except PermissionError as e:
1545 self.skipTest('os.link(): %s' % e)
1546 try:
1547 tar = tarfile.open(tmpname, self.mode)
1548 try:
1549 # Record the link target in the inodes list.
1550 tar.gettarinfo(target)
1551 tarinfo = tar.gettarinfo(link)
1552 self.assertEqual(tarinfo.size, 0)
1553 finally:
1554 tar.close()
1555 finally:
1556 os_helper.unlink(target)
1557 os_helper.unlink(link)
1558
1559 @os_helper.skip_unless_symlink
1560 def test_symlink_size(self):
1561 path = os.path.join(TEMPDIR, "symlink")
1562 os.symlink("link_target", path)
1563 try:
1564 tar = tarfile.open(tmpname, self.mode)
1565 try:
1566 tarinfo = tar.gettarinfo(path)
1567 self.assertEqual(tarinfo.size, 0)
1568 finally:
1569 tar.close()
1570 finally:
1571 os_helper.unlink(path)
1572
1573 def test_add_self(self):
1574 # Test for #1257255.
1575 dstname = os.path.abspath(tmpname)
1576 tar = tarfile.open(tmpname, self.mode)
1577 try:
1578 self.assertEqual(tar.name, dstname,
1579 "archive name must be absolute")
1580 tar.add(dstname)
1581 self.assertEqual(tar.getnames(), [],
1582 "added the archive to itself")
1583
1584 with os_helper.change_cwd(TEMPDIR):
1585 tar.add(dstname)
1586 self.assertEqual(tar.getnames(), [],
1587 "added the archive to itself")
1588 finally:
1589 tar.close()
1590
1591 def test_filter(self):
1592 tempdir = os.path.join(TEMPDIR, "filter")
1593 os.mkdir(tempdir)
1594 try:
1595 for name in ("foo", "bar", "baz"):
1596 name = os.path.join(tempdir, name)
1597 os_helper.create_empty_file(name)
1598
1599 def filter(tarinfo):
1600 if os.path.basename(tarinfo.name) == "bar":
1601 return
1602 tarinfo.uid = 123
1603 tarinfo.uname = "foo"
1604 return tarinfo
1605
1606 tar = tarfile.open(tmpname, self.mode, encoding="iso8859-1")
1607 try:
1608 tar.add(tempdir, arcname="empty_dir", filter=filter)
1609 finally:
1610 tar.close()
1611
1612 # Verify that filter is a keyword-only argument
1613 with self.assertRaises(TypeError):
1614 tar.add(tempdir, "empty_dir", True, None, filter)
1615
1616 tar = tarfile.open(tmpname, "r")
1617 try:
1618 for tarinfo in tar:
1619 self.assertEqual(tarinfo.uid, 123)
1620 self.assertEqual(tarinfo.uname, "foo")
1621 self.assertEqual(len(tar.getmembers()), 3)
1622 finally:
1623 tar.close()
1624 finally:
1625 os_helper.rmtree(tempdir)
1626
1627 # Guarantee that stored pathnames are not modified. Don't
1628 # remove ./ or ../ or double slashes. Still make absolute
1629 # pathnames relative.
1630 # For details see bug #6054.
1631 def _test_pathname(self, path, cmp_path=None, dir=False):
1632 # Create a tarfile with an empty member named path
1633 # and compare the stored name with the original.
1634 foo = os.path.join(TEMPDIR, "foo")
1635 if not dir:
1636 os_helper.create_empty_file(foo)
1637 else:
1638 os.mkdir(foo)
1639
1640 tar = tarfile.open(tmpname, self.mode)
1641 try:
1642 tar.add(foo, arcname=path)
1643 finally:
1644 tar.close()
1645
1646 tar = tarfile.open(tmpname, "r")
1647 try:
1648 t = tar.next()
1649 finally:
1650 tar.close()
1651
1652 if not dir:
1653 os_helper.unlink(foo)
1654 else:
1655 os_helper.rmdir(foo)
1656
1657 self.assertEqual(t.name, cmp_path or path.replace(os.sep, "/"))
1658
1659
1660 @os_helper.skip_unless_symlink
1661 def test_extractall_symlinks(self):
1662 # Test if extractall works properly when tarfile contains symlinks
1663 tempdir = os.path.join(TEMPDIR, "testsymlinks")
1664 temparchive = os.path.join(TEMPDIR, "testsymlinks.tar")
1665 os.mkdir(tempdir)
1666 try:
1667 source_file = os.path.join(tempdir,'source')
1668 target_file = os.path.join(tempdir,'symlink')
1669 with open(source_file,'w') as f:
1670 f.write('something\n')
1671 os.symlink(source_file, target_file)
1672 with tarfile.open(temparchive, 'w') as tar:
1673 tar.add(source_file, arcname="source")
1674 tar.add(target_file, arcname="symlink")
1675 # Let's extract it to the location which contains the symlink
1676 with tarfile.open(temparchive, errorlevel=2) as tar:
1677 # this should not raise OSError: [Errno 17] File exists
1678 try:
1679 tar.extractall(path=tempdir,
1680 filter='fully_trusted')
1681 except OSError:
1682 self.fail("extractall failed with symlinked files")
1683 finally:
1684 os_helper.unlink(temparchive)
1685 os_helper.rmtree(tempdir)
1686
1687 def test_pathnames(self):
1688 self._test_pathname("foo")
1689 self._test_pathname(os.path.join("foo", ".", "bar"))
1690 self._test_pathname(os.path.join("foo", "..", "bar"))
1691 self._test_pathname(os.path.join(".", "foo"))
1692 self._test_pathname(os.path.join(".", "foo", "."))
1693 self._test_pathname(os.path.join(".", "foo", ".", "bar"))
1694 self._test_pathname(os.path.join(".", "foo", "..", "bar"))
1695 self._test_pathname(os.path.join(".", "foo", "..", "bar"))
1696 self._test_pathname(os.path.join("..", "foo"))
1697 self._test_pathname(os.path.join("..", "foo", ".."))
1698 self._test_pathname(os.path.join("..", "foo", ".", "bar"))
1699 self._test_pathname(os.path.join("..", "foo", "..", "bar"))
1700
1701 self._test_pathname("foo" + os.sep + os.sep + "bar")
1702 self._test_pathname("foo" + os.sep + os.sep, "foo", dir=True)
1703
1704 def test_abs_pathnames(self):
1705 if sys.platform == "win32":
1706 self._test_pathname("C:\\foo", "foo")
1707 else:
1708 self._test_pathname("/foo", "foo")
1709 self._test_pathname("///foo", "foo")
1710
1711 def test_cwd(self):
1712 # Test adding the current working directory.
1713 with os_helper.change_cwd(TEMPDIR):
1714 tar = tarfile.open(tmpname, self.mode)
1715 try:
1716 tar.add(".")
1717 finally:
1718 tar.close()
1719
1720 tar = tarfile.open(tmpname, "r")
1721 try:
1722 for t in tar:
1723 if t.name != ".":
1724 self.assertStartsWith(t.name, "./")
1725 finally:
1726 tar.close()
1727
1728 def test_open_nonwritable_fileobj(self):
1729 for exctype in OSError, EOFError, RuntimeError:
1730 class BadFile(io.BytesIO):
1731 first = True
1732 def write(self, data):
1733 if self.first:
1734 self.first = False
1735 raise exctype
1736
1737 f = BadFile()
1738 with (
1739 warnings_helper.check_no_resource_warning(self),
1740 self.assertRaises(exctype),
1741 ):
1742 tarfile.open(tmpname, self.mode, fileobj=f,
1743 format=tarfile.PAX_FORMAT,
1744 pax_headers={'non': 'empty'})
1745 self.assertFalse(f.closed)
1746
1747 def test_missing_fileobj(self):
1748 with tarfile.open(tmpname, self.mode) as tar:
1749 tarinfo = tar.gettarinfo(tarname)
1750 with self.assertRaises(ValueError):
1751 tar.addfile(tarinfo)
1752
1753
1754 class GzipWriteTest(GzipTest, WriteTest):
1755 pass
1756
1757
1758 class Bz2WriteTest(Bz2Test, WriteTest):
1759 pass
1760
1761
1762 class LzmaWriteTest(LzmaTest, WriteTest):
1763 pass
1764
1765 class ZstdWriteTest(ZstdTest, WriteTest):
1766 pass
1767
1768 class StreamWriteTest(WriteTestBase, unittest.TestCase):
1769
1770 prefix = "w|"
1771 decompressor = None
1772
1773 def test_stream_padding(self):
1774 # Test for bug #1543303.
1775 tar = tarfile.open(tmpname, self.mode)
1776 tar.close()
1777 if self.decompressor:
1778 dec = self.decompressor()
1779 with open(tmpname, "rb") as fobj:
1780 data = fobj.read()
1781 data = dec.decompress(data)
1782 self.assertFalse(dec.unused_data, "found trailing data")
1783 else:
1784 with self.open(tmpname) as fobj:
1785 data = fobj.read()
1786 self.assertEqual(data.count(b"\0"), tarfile.RECORDSIZE,
1787 "incorrect zero padding")
1788
1789 @unittest.skipUnless(sys.platform != "win32" and hasattr(os, "umask"),
1790 "Missing umask implementation")
1791 @unittest.skipIf(
1792 support.is_emscripten or support.is_wasi,
1793 "Emscripten's/WASI's umask is a stub."
1794 )
1795 def test_file_mode(self):
1796 # Test for issue #8464: Create files with correct
1797 # permissions.
1798 if os.path.exists(tmpname):
1799 os_helper.unlink(tmpname)
1800
1801 original_umask = os.umask(0o022)
1802 try:
1803 tar = tarfile.open(tmpname, self.mode)
1804 tar.close()
1805 mode = os.stat(tmpname).st_mode & 0o777
1806 self.assertEqual(mode, 0o644, "wrong file permissions")
1807 finally:
1808 os.umask(original_umask)
1809
1810 def test_pathlike_name(self):
1811 expected_name = os.path.abspath(tmpname)
1812 tarpath = os_helper.FakePath(tmpname)
1813
1814 for func in (tarfile.open, tarfile.TarFile.open):
1815 with self.subTest():
1816 with func(tarpath, self.mode) as tar:
1817 self.assertEqual(tar.name, expected_name)
1818 os_helper.unlink(tmpname)
1819
1820
1821 class GzipStreamWriteTest(GzipTest, StreamWriteTest):
1822 def test_source_directory_not_leaked(self):
1823 """
1824 Ensure the source directory is not included in the tar header
1825 per bpo-41316.
1826 """
1827 tarfile.open(tmpname, self.mode).close()
1828 payload = pathlib.Path(tmpname).read_text(encoding='latin-1')
1829 assert os.path.dirname(tmpname) not in payload
1830
1831
1832 class Bz2StreamWriteTest(Bz2Test, StreamWriteTest):
1833 decompressor = bz2.BZ2Decompressor if bz2 else None
1834
1835 class LzmaStreamWriteTest(LzmaTest, StreamWriteTest):
1836 decompressor = lzma.LZMADecompressor if lzma else None
1837
1838 class ZstdStreamWriteTest(ZstdTest, StreamWriteTest):
1839 decompressor = zstd.ZstdDecompressor if zstd else None
1840
1841 class _CompressedWriteTest(TarTest):
1842 # This is not actually a standalone test.
1843 # It does not inherit WriteTest because it only makes sense with gz,bz2
1844 source = (b"And we move to Bristol where they have a special, " +
1845 b"Very Silly candidate")
1846
1847 def _compressed_tar(self, compresslevel):
1848 fobj = io.BytesIO()
1849 with tarfile.open(tmpname, self.mode, fobj,
1850 compresslevel=compresslevel) as tarfl:
1851 tarfl.addfile(tarfile.TarInfo("foo"), io.BytesIO(self.source))
1852 return fobj
1853
1854 def _test_bz2_header(self, compresslevel):
1855 fobj = self._compressed_tar(compresslevel)
1856 self.assertEqual(fobj.getvalue()[0:10],
1857 b"BZh%d1AY&SY" % compresslevel)
1858
1859 def _test_gz_header(self, compresslevel):
1860 fobj = self._compressed_tar(compresslevel)
1861 self.assertEqual(fobj.getvalue()[:3], b"\x1f\x8b\x08")
1862
1863 class Bz2CompressWriteTest(Bz2Test, _CompressedWriteTest, unittest.TestCase):
1864 prefix = "w:"
1865 def test_compression_levels(self):
1866 self._test_bz2_header(1)
1867 self._test_bz2_header(5)
1868 self._test_bz2_header(9)
1869
1870 class Bz2CompressStreamWriteTest(Bz2Test, _CompressedWriteTest,
1871 unittest.TestCase):
1872 prefix = "w|"
1873 def test_compression_levels(self):
1874 self._test_bz2_header(1)
1875 self._test_bz2_header(5)
1876 self._test_bz2_header(9)
1877
1878 class GzCompressWriteTest(GzipTest, _CompressedWriteTest, unittest.TestCase):
1879 prefix = "w:"
1880 def test_compression_levels(self):
1881 self._test_gz_header(1)
1882 self._test_gz_header(5)
1883 self._test_gz_header(9)
1884
1885 class GzCompressStreamWriteTest(GzipTest, _CompressedWriteTest,
1886 unittest.TestCase):
1887 prefix = "w|"
1888 def test_compression_levels(self):
1889 self._test_gz_header(1)
1890 self._test_gz_header(5)
1891 self._test_gz_header(9)
1892
1893 class CompressLevelRaises(unittest.TestCase):
1894 def test_compresslevel_wrong_modes(self):
1895 compresslevel = 5
1896 fobj = io.BytesIO()
1897 with self.assertRaises(TypeError):
1898 tarfile.open(tmpname, "w:", fobj, compresslevel=compresslevel)
1899
1900 @support.requires_bz2()
1901 def test_wrong_compresslevels(self):
1902 # BZ2 checks that the compresslevel is in [1,9]. gz does not
1903 fobj = io.BytesIO()
1904 with self.assertRaises(ValueError):
1905 tarfile.open(tmpname, "w:bz2", fobj, compresslevel=0)
1906 with self.assertRaises(ValueError):
1907 tarfile.open(tmpname, "w:bz2", fobj, compresslevel=10)
1908 with self.assertRaises(ValueError):
1909 tarfile.open(tmpname, "w|bz2", fobj, compresslevel=10)
1910
1911 class GNUWriteTest(unittest.TestCase):
1912 # This testcase checks for correct creation of GNU Longname
1913 # and Longlink extended headers (cp. bug #812325).
1914
1915 def _length(self, s):
1916 blocks = len(s) // 512 + 1
1917 return blocks * 512
1918
1919 def _calc_size(self, name, link=None):
1920 # Initial tar header
1921 count = 512
1922
1923 if len(name) > tarfile.LENGTH_NAME:
1924 # GNU longname extended header + longname
1925 count += 512
1926 count += self._length(name)
1927 if link is not None and len(link) > tarfile.LENGTH_LINK:
1928 # GNU longlink extended header + longlink
1929 count += 512
1930 count += self._length(link)
1931 return count
1932
1933 def _test(self, name, link=None):
1934 tarinfo = tarfile.TarInfo(name)
1935 if link:
1936 tarinfo.linkname = link
1937 tarinfo.type = tarfile.LNKTYPE
1938
1939 tar = tarfile.open(tmpname, "w")
1940 try:
1941 tar.format = tarfile.GNU_FORMAT
1942 tar.addfile(tarinfo)
1943
1944 v1 = self._calc_size(name, link)
1945 v2 = tar.offset
1946 self.assertEqual(v1, v2, "GNU longname/longlink creation failed")
1947 finally:
1948 tar.close()
1949
1950 tar = tarfile.open(tmpname)
1951 try:
1952 member = tar.next()
1953 self.assertIsNotNone(member,
1954 "unable to read longname member")
1955 self.assertEqual(tarinfo.name, member.name,
1956 "unable to read longname member")
1957 self.assertEqual(tarinfo.linkname, member.linkname,
1958 "unable to read longname member")
1959 finally:
1960 tar.close()
1961
1962 def test_longname_1023(self):
1963 self._test(("longnam/" * 127) + "longnam")
1964
1965 def test_longname_1024(self):
1966 self._test(("longnam/" * 127) + "longname")
1967
1968 def test_longname_1025(self):
1969 self._test(("longnam/" * 127) + "longname_")
1970
1971 def test_longlink_1023(self):
1972 self._test("name", ("longlnk/" * 127) + "longlnk")
1973
1974 def test_longlink_1024(self):
1975 self._test("name", ("longlnk/" * 127) + "longlink")
1976
1977 def test_longlink_1025(self):
1978 self._test("name", ("longlnk/" * 127) + "longlink_")
1979
1980 def test_longnamelink_1023(self):
1981 self._test(("longnam/" * 127) + "longnam",
1982 ("longlnk/" * 127) + "longlnk")
1983
1984 def test_longnamelink_1024(self):
1985 self._test(("longnam/" * 127) + "longname",
1986 ("longlnk/" * 127) + "longlink")
1987
1988 def test_longnamelink_1025(self):
1989 self._test(("longnam/" * 127) + "longname_",
1990 ("longlnk/" * 127) + "longlink_")
1991
1992
1993 class DeviceHeaderTest(WriteTestBase, unittest.TestCase):
1994
1995 prefix = "w:"
1996
1997 def test_headers_written_only_for_device_files(self):
1998 # Regression test for bpo-18819.
1999 tempdir = os.path.join(TEMPDIR, "device_header_test")
2000 os.mkdir(tempdir)
2001 try:
2002 tar = tarfile.open(tmpname, self.mode)
2003 try:
2004 input_blk = tarfile.TarInfo(name="my_block_device")
2005 input_reg = tarfile.TarInfo(name="my_regular_file")
2006 input_blk.type = tarfile.BLKTYPE
2007 input_reg.type = tarfile.REGTYPE
2008 tar.addfile(input_blk)
2009 tar.addfile(input_reg)
2010 finally:
2011 tar.close()
2012
2013 # devmajor and devminor should be *interpreted* as 0 in both...
2014 tar = tarfile.open(tmpname, "r")
2015 try:
2016 output_blk = tar.getmember("my_block_device")
2017 output_reg = tar.getmember("my_regular_file")
2018 finally:
2019 tar.close()
2020 self.assertEqual(output_blk.devmajor, 0)
2021 self.assertEqual(output_blk.devminor, 0)
2022 self.assertEqual(output_reg.devmajor, 0)
2023 self.assertEqual(output_reg.devminor, 0)
2024
2025 # ...but the fields should not actually be set on regular files:
2026 with open(tmpname, "rb") as infile:
2027 buf = infile.read()
2028 buf_blk = buf[output_blk.offset:output_blk.offset_data]
2029 buf_reg = buf[output_reg.offset:output_reg.offset_data]
2030 # See `struct posixheader` in GNU docs for byte offsets:
2031 # <https://www.gnu.org/software/tar/manual/html_node/Standard.html>
2032 device_headers = slice(329, 329 + 16)
2033 self.assertEqual(buf_blk[device_headers], b"0000000\0" * 2)
2034 self.assertEqual(buf_reg[device_headers], b"\0" * 16)
2035 finally:
2036 os_helper.rmtree(tempdir)
2037
2038
2039 class CreateTest(WriteTestBase, unittest.TestCase):
2040
2041 prefix = "x:"
2042
2043 file_path = os.path.join(TEMPDIR, "spameggs42")
2044
2045 def setUp(self):
2046 os_helper.unlink(tmpname)
2047
2048 @classmethod
2049 def setUpClass(cls):
2050 with open(cls.file_path, "wb") as fobj:
2051 fobj.write(b"aaa")
2052
2053 @classmethod
2054 def tearDownClass(cls):
2055 os_helper.unlink(cls.file_path)
2056
2057 def test_create(self):
2058 with tarfile.open(tmpname, self.mode) as tobj:
2059 tobj.add(self.file_path)
2060
2061 with self.taropen(tmpname) as tobj:
2062 names = tobj.getnames()
2063 self.assertEqual(len(names), 1)
2064 self.assertIn('spameggs42', names[0])
2065
2066 def test_create_existing(self):
2067 with tarfile.open(tmpname, self.mode) as tobj:
2068 tobj.add(self.file_path)
2069
2070 with self.assertRaises(FileExistsError):
2071 tobj = tarfile.open(tmpname, self.mode)
2072
2073 with self.taropen(tmpname) as tobj:
2074 names = tobj.getnames()
2075 self.assertEqual(len(names), 1)
2076 self.assertIn('spameggs42', names[0])
2077
2078 def test_create_taropen(self):
2079 with self.taropen(tmpname, "x") as tobj:
2080 tobj.add(self.file_path)
2081
2082 with self.taropen(tmpname) as tobj:
2083 names = tobj.getnames()
2084 self.assertEqual(len(names), 1)
2085 self.assertIn('spameggs42', names[0])
2086
2087 def test_create_existing_taropen(self):
2088 with self.taropen(tmpname, "x") as tobj:
2089 tobj.add(self.file_path)
2090
2091 with self.assertRaises(FileExistsError):
2092 with self.taropen(tmpname, "x"):
2093 pass
2094
2095 with self.taropen(tmpname) as tobj:
2096 names = tobj.getnames()
2097 self.assertEqual(len(names), 1)
2098 self.assertIn("spameggs42", names[0])
2099
2100 def test_create_pathlike_name(self):
2101 with tarfile.open(os_helper.FakePath(tmpname), self.mode) as tobj:
2102 self.assertIsInstance(tobj.name, str)
2103 self.assertEqual(tobj.name, os.path.abspath(tmpname))
2104 tobj.add(os_helper.FakePath(self.file_path))
2105 names = tobj.getnames()
2106 self.assertEqual(len(names), 1)
2107 self.assertIn('spameggs42', names[0])
2108
2109 with self.taropen(tmpname) as tobj:
2110 names = tobj.getnames()
2111 self.assertEqual(len(names), 1)
2112 self.assertIn('spameggs42', names[0])
2113
2114 def test_create_taropen_pathlike_name(self):
2115 with self.taropen(os_helper.FakePath(tmpname), "x") as tobj:
2116 self.assertIsInstance(tobj.name, str)
2117 self.assertEqual(tobj.name, os.path.abspath(tmpname))
2118 tobj.add(os_helper.FakePath(self.file_path))
2119 names = tobj.getnames()
2120 self.assertEqual(len(names), 1)
2121 self.assertIn('spameggs42', names[0])
2122
2123 with self.taropen(tmpname) as tobj:
2124 names = tobj.getnames()
2125 self.assertEqual(len(names), 1)
2126 self.assertIn('spameggs42', names[0])
2127
2128
2129 class GzipCreateTest(GzipTest, CreateTest):
2130
2131 def test_create_with_compresslevel(self):
2132 with tarfile.open(tmpname, self.mode, compresslevel=1) as tobj:
2133 tobj.add(self.file_path)
2134 with tarfile.open(tmpname, 'r:gz', compresslevel=1) as tobj:
2135 pass
2136
2137
2138 class Bz2CreateTest(Bz2Test, CreateTest):
2139
2140 def test_create_with_compresslevel(self):
2141 with tarfile.open(tmpname, self.mode, compresslevel=1) as tobj:
2142 tobj.add(self.file_path)
2143 with tarfile.open(tmpname, 'r:bz2', compresslevel=1) as tobj:
2144 pass
2145
2146
2147 class LzmaCreateTest(LzmaTest, CreateTest):
2148
2149 # Unlike gz and bz2, xz uses the preset keyword instead of compresslevel.
2150 # It does not allow for preset to be specified when reading.
2151 def test_create_with_preset(self):
2152 with tarfile.open(tmpname, self.mode, preset=1) as tobj:
2153 tobj.add(self.file_path)
2154
2155
2156 class ZstdCreateTest(ZstdTest, CreateTest):
2157
2158 # Unlike gz and bz2, zstd uses the level keyword instead of compresslevel.
2159 # It does not allow for level to be specified when reading.
2160 def test_create_with_level(self):
2161 with tarfile.open(tmpname, self.mode, level=1) as tobj:
2162 tobj.add(self.file_path)
2163
2164 class CreateWithXModeTest(CreateTest):
2165
2166 prefix = "x"
2167
2168 test_create_taropen = None
2169 test_create_existing_taropen = None
2170
2171
2172 @unittest.skipUnless(hasattr(os, "link"), "Missing hardlink implementation")
2173 class HardlinkTest(unittest.TestCase):
2174 # Test the creation of LNKTYPE (hardlink) members in an archive.
2175
2176 def setUp(self):
2177 self.foo = os.path.join(TEMPDIR, "foo")
2178 self.bar = os.path.join(TEMPDIR, "bar")
2179
2180 with open(self.foo, "wb") as fobj:
2181 fobj.write(b"foo")
2182
2183 try:
2184 os.link(self.foo, self.bar)
2185 except PermissionError as e:
2186 self.skipTest('os.link(): %s' % e)
2187
2188 self.tar = tarfile.open(tmpname, "w")
2189 self.tar.add(self.foo)
2190
2191 def tearDown(self):
2192 self.tar.close()
2193 os_helper.unlink(self.foo)
2194 os_helper.unlink(self.bar)
2195
2196 def test_add_twice(self):
2197 # The same name will be added as a REGTYPE every
2198 # time regardless of st_nlink.
2199 tarinfo = self.tar.gettarinfo(self.foo)
2200 self.assertEqual(tarinfo.type, tarfile.REGTYPE,
2201 "add file as regular failed")
2202
2203 def test_add_hardlink(self):
2204 tarinfo = self.tar.gettarinfo(self.bar)
2205 self.assertEqual(tarinfo.type, tarfile.LNKTYPE,
2206 "add file as hardlink failed")
2207
2208 def test_dereference_hardlink(self):
2209 self.tar.dereference = True
2210 tarinfo = self.tar.gettarinfo(self.bar)
2211 self.assertEqual(tarinfo.type, tarfile.REGTYPE,
2212 "dereferencing hardlink failed")
2213
2214
2215 class PaxWriteTest(GNUWriteTest):
2216
2217 def _test(self, name, link=None):
2218 # See GNUWriteTest.
2219 tarinfo = tarfile.TarInfo(name)
2220 if link:
2221 tarinfo.linkname = link
2222 tarinfo.type = tarfile.LNKTYPE
2223
2224 tar = tarfile.open(tmpname, "w", format=tarfile.PAX_FORMAT)
2225 try:
2226 tar.addfile(tarinfo)
2227 finally:
2228 tar.close()
2229
2230 tar = tarfile.open(tmpname)
2231 try:
2232 if link:
2233 l = tar.getmembers()[0].linkname
2234 self.assertEqual(link, l, "PAX longlink creation failed")
2235 else:
2236 n = tar.getmembers()[0].name
2237 self.assertEqual(name, n, "PAX longname creation failed")
2238 finally:
2239 tar.close()
2240
2241 def test_pax_global_header(self):
2242 pax_headers = {
2243 "foo": "bar",
2244 "uid": "0",
2245 "mtime": "1.23",
2246 "test": "\xe4\xf6\xfc",
2247 "\xe4\xf6\xfc": "test"}
2248
2249 tar = tarfile.open(tmpname, "w", format=tarfile.PAX_FORMAT,
2250 pax_headers=pax_headers)
2251 try:
2252 tar.addfile(tarfile.TarInfo("test"))
2253 finally:
2254 tar.close()
2255
2256 # Test if the global header was written correctly.
2257 tar = tarfile.open(tmpname, encoding="iso8859-1")
2258 try:
2259 self.assertEqual(tar.pax_headers, pax_headers)
2260 self.assertEqual(tar.getmembers()[0].pax_headers, pax_headers)
2261 # Test if all the fields are strings.
2262 for key, val in tar.pax_headers.items():
2263 self.assertIsNot(type(key), bytes)
2264 self.assertIsNot(type(val), bytes)
2265 if key in tarfile.PAX_NUMBER_FIELDS:
2266 try:
2267 tarfile.PAX_NUMBER_FIELDS[key](val)
2268 except (TypeError, ValueError):
2269 self.fail("unable to convert pax header field")
2270 finally:
2271 tar.close()
2272
2273 def test_pax_extended_header(self):
2274 # The fields from the pax header have priority over the
2275 # TarInfo.
2276 pax_headers = {"path": "foo", "uid": "123"}
2277
2278 tar = tarfile.open(tmpname, "w", format=tarfile.PAX_FORMAT,
2279 encoding="iso8859-1")
2280 try:
2281 t = tarfile.TarInfo()
2282 t.name = "\xe4\xf6\xfc" # non-ASCII
2283 t.uid = 8**8 # too large
2284 t.pax_headers = pax_headers
2285 tar.addfile(t)
2286 finally:
2287 tar.close()
2288
2289 tar = tarfile.open(tmpname, encoding="iso8859-1")
2290 try:
2291 t = tar.getmembers()[0]
2292 self.assertEqual(t.pax_headers, pax_headers)
2293 self.assertEqual(t.name, "foo")
2294 self.assertEqual(t.uid, 123)
2295 finally:
2296 tar.close()
2297
2298 def test_create_pax_header(self):
2299 # The ustar header should contain values that can be
2300 # represented reasonably, even if a better (e.g. higher
2301 # precision) version is set in the pax header.
2302 # Issue #45863
2303
2304 # values that should be kept
2305 t = tarfile.TarInfo()
2306 t.name = "foo"
2307 t.mtime = 1000.1
2308 t.size = 100
2309 t.uid = 123
2310 t.gid = 124
2311 info = t.get_info()
2312 header = t.create_pax_header(info, encoding="iso8859-1")
2313 self.assertEqual(info['name'], "foo")
2314 # mtime should be rounded to nearest second
2315 self.assertIsInstance(info['mtime'], int)
2316 self.assertEqual(info['mtime'], 1000)
2317 self.assertEqual(info['size'], 100)
2318 self.assertEqual(info['uid'], 123)
2319 self.assertEqual(info['gid'], 124)
2320 self.assertEqual(header,
2321 b'././@PaxHeader' + bytes(86) \
2322 + b'0000000\x000000000\x000000000\x0000000000020\x0000000000000\x00010205\x00 x' \
2323 + bytes(100) + b'ustar\x0000'+ bytes(247) \
2324 + b'16 mtime=1000.1\n' + bytes(496) + b'foo' + bytes(97) \
2325 + b'0000644\x000000173\x000000174\x0000000000144\x0000000001750\x00006516\x00 0' \
2326 + bytes(100) + b'ustar\x0000' + bytes(247))
2327
2328 # values that should be changed
2329 t = tarfile.TarInfo()
2330 t.name = "foo\u3374" # can't be represented in ascii
2331 t.mtime = 10**10 # too big
2332 t.size = 10**10 # too big
2333 t.uid = 8**8 # too big
2334 t.gid = 8**8+1 # too big
2335 info = t.get_info()
2336 header = t.create_pax_header(info, encoding="iso8859-1")
2337 # name is kept as-is in info but should be added to pax header
2338 self.assertEqual(info['name'], "foo\u3374")
2339 self.assertEqual(info['mtime'], 0)
2340 self.assertEqual(info['size'], 0)
2341 self.assertEqual(info['uid'], 0)
2342 self.assertEqual(info['gid'], 0)
2343 self.assertEqual(header,
2344 b'././@PaxHeader' + bytes(86) \
2345 + b'0000000\x000000000\x000000000\x0000000000130\x0000000000000\x00010207\x00 x' \
2346 + bytes(100) + b'ustar\x0000' + bytes(247) \
2347 + b'15 path=foo\xe3\x8d\xb4\n16 uid=16777216\n' \
2348 + b'16 gid=16777217\n20 size=10000000000\n' \
2349 + b'21 mtime=10000000000\n'+ bytes(424) + b'foo?' + bytes(96) \
2350 + b'0000644\x000000000\x000000000\x0000000000000\x0000000000000\x00006540\x00 0' \
2351 + bytes(100) + b'ustar\x0000' + bytes(247))
2352
2353
2354 class UnicodeTest:
2355
2356 def test_iso8859_1_filename(self):
2357 self._test_unicode_filename("iso8859-1")
2358
2359 def test_utf7_filename(self):
2360 self._test_unicode_filename("utf7")
2361
2362 def test_utf8_filename(self):
2363 self._test_unicode_filename("utf-8")
2364
2365 def _test_unicode_filename(self, encoding):
2366 tar = tarfile.open(tmpname, "w", format=self.format,
2367 encoding=encoding, errors="strict")
2368 try:
2369 name = "\xe4\xf6\xfc"
2370 tar.addfile(tarfile.TarInfo(name))
2371 finally:
2372 tar.close()
2373
2374 tar = tarfile.open(tmpname, encoding=encoding)
2375 try:
2376 self.assertEqual(tar.getmembers()[0].name, name)
2377 finally:
2378 tar.close()
2379
2380 def test_unicode_filename_error(self):
2381 tar = tarfile.open(tmpname, "w", format=self.format,
2382 encoding="ascii", errors="strict")
2383 try:
2384 tarinfo = tarfile.TarInfo()
2385
2386 tarinfo.name = "\xe4\xf6\xfc"
2387 self.assertRaises(UnicodeError, tar.addfile, tarinfo)
2388
2389 tarinfo.name = "foo"
2390 tarinfo.uname = "\xe4\xf6\xfc"
2391 self.assertRaises(UnicodeError, tar.addfile, tarinfo)
2392 finally:
2393 tar.close()
2394
2395 def test_unicode_argument(self):
2396 tar = tarfile.open(tarname, "r",
2397 encoding="iso8859-1", errors="strict")
2398 try:
2399 for t in tar:
2400 self.assertIs(type(t.name), str)
2401 self.assertIs(type(t.linkname), str)
2402 self.assertIs(type(t.uname), str)
2403 self.assertIs(type(t.gname), str)
2404 finally:
2405 tar.close()
2406
2407 def test_uname_unicode(self):
2408 t = tarfile.TarInfo("foo")
2409 t.uname = "\xe4\xf6\xfc"
2410 t.gname = "\xe4\xf6\xfc"
2411
2412 tar = tarfile.open(tmpname, mode="w", format=self.format,
2413 encoding="iso8859-1")
2414 try:
2415 tar.addfile(t)
2416 finally:
2417 tar.close()
2418
2419 tar = tarfile.open(tmpname, encoding="iso8859-1")
2420 try:
2421 t = tar.getmember("foo")
2422 self.assertEqual(t.uname, "\xe4\xf6\xfc")
2423 self.assertEqual(t.gname, "\xe4\xf6\xfc")
2424
2425 if self.format != tarfile.PAX_FORMAT:
2426 tar.close()
2427 tar = tarfile.open(tmpname, encoding="ascii")
2428 t = tar.getmember("foo")
2429 self.assertEqual(t.uname, "\udce4\udcf6\udcfc")
2430 self.assertEqual(t.gname, "\udce4\udcf6\udcfc")
2431 finally:
2432 tar.close()
2433
2434
2435 class UstarUnicodeTest(UnicodeTest, unittest.TestCase):
2436
2437 format = tarfile.USTAR_FORMAT
2438
2439 # Test whether the utf-8 encoded version of a filename exceeds the 100
2440 # bytes name field limit (every occurrence of '\xff' will be expanded to 2
2441 # bytes).
2442 def test_unicode_name1(self):
2443 self._test_ustar_name("0123456789" * 10)
2444 self._test_ustar_name("0123456789" * 10 + "0", ValueError)
2445 self._test_ustar_name("0123456789" * 9 + "01234567\xff")
2446 self._test_ustar_name("0123456789" * 9 + "012345678\xff", ValueError)
2447
2448 def test_unicode_name2(self):
2449 self._test_ustar_name("0123456789" * 9 + "012345\xff\xff")
2450 self._test_ustar_name("0123456789" * 9 + "0123456\xff\xff", ValueError)
2451
2452 # Test whether the utf-8 encoded version of a filename exceeds the 155
2453 # bytes prefix + '/' + 100 bytes name limit.
2454 def test_unicode_longname1(self):
2455 self._test_ustar_name("0123456789" * 15 + "01234/" + "0123456789" * 10)
2456 self._test_ustar_name("0123456789" * 15 + "0123/4" + "0123456789" * 10, ValueError)
2457 self._test_ustar_name("0123456789" * 15 + "012\xff/" + "0123456789" * 10)
2458 self._test_ustar_name("0123456789" * 15 + "0123\xff/" + "0123456789" * 10, ValueError)
2459
2460 def test_unicode_longname2(self):
2461 self._test_ustar_name("0123456789" * 15 + "01\xff/2" + "0123456789" * 10, ValueError)
2462 self._test_ustar_name("0123456789" * 15 + "01\xff\xff/" + "0123456789" * 10, ValueError)
2463
2464 def test_unicode_longname3(self):
2465 self._test_ustar_name("0123456789" * 15 + "01\xff\xff/2" + "0123456789" * 10, ValueError)
2466 self._test_ustar_name("0123456789" * 15 + "01234/" + "0123456789" * 9 + "01234567\xff")
2467 self._test_ustar_name("0123456789" * 15 + "01234/" + "0123456789" * 9 + "012345678\xff", ValueError)
2468
2469 def test_unicode_longname4(self):
2470 self._test_ustar_name("0123456789" * 15 + "01234/" + "0123456789" * 9 + "012345\xff\xff")
2471 self._test_ustar_name("0123456789" * 15 + "01234/" + "0123456789" * 9 + "0123456\xff\xff", ValueError)
2472
2473 def _test_ustar_name(self, name, exc=None):
2474 with tarfile.open(tmpname, "w", format=self.format, encoding="utf-8") as tar:
2475 t = tarfile.TarInfo(name)
2476 if exc is None:
2477 tar.addfile(t)
2478 else:
2479 self.assertRaises(exc, tar.addfile, t)
2480
2481 if exc is None:
2482 with tarfile.open(tmpname, "r", encoding="utf-8") as tar:
2483 for t in tar:
2484 self.assertEqual(name, t.name)
2485 break
2486
2487 # Test the same as above for the 100 bytes link field.
2488 def test_unicode_link1(self):
2489 self._test_ustar_link("0123456789" * 10)
2490 self._test_ustar_link("0123456789" * 10 + "0", ValueError)
2491 self._test_ustar_link("0123456789" * 9 + "01234567\xff")
2492 self._test_ustar_link("0123456789" * 9 + "012345678\xff", ValueError)
2493
2494 def test_unicode_link2(self):
2495 self._test_ustar_link("0123456789" * 9 + "012345\xff\xff")
2496 self._test_ustar_link("0123456789" * 9 + "0123456\xff\xff", ValueError)
2497
2498 def _test_ustar_link(self, name, exc=None):
2499 with tarfile.open(tmpname, "w", format=self.format, encoding="utf-8") as tar:
2500 t = tarfile.TarInfo("foo")
2501 t.linkname = name
2502 if exc is None:
2503 tar.addfile(t)
2504 else:
2505 self.assertRaises(exc, tar.addfile, t)
2506
2507 if exc is None:
2508 with tarfile.open(tmpname, "r", encoding="utf-8") as tar:
2509 for t in tar:
2510 self.assertEqual(name, t.linkname)
2511 break
2512
2513
2514 class GNUUnicodeTest(UnicodeTest, unittest.TestCase):
2515
2516 format = tarfile.GNU_FORMAT
2517
2518 def test_bad_pax_header(self):
2519 # Test for issue #8633. GNU tar <= 1.23 creates raw binary fields
2520 # without a hdrcharset=BINARY header.
2521 for encoding, name in (
2522 ("utf-8", "pax/bad-pax-\udce4\udcf6\udcfc"),
2523 ("iso8859-1", "pax/bad-pax-\xe4\xf6\xfc"),):
2524 with tarfile.open(tarname, encoding=encoding,
2525 errors="surrogateescape") as tar:
2526 try:
2527 t = tar.getmember(name)
2528 except KeyError:
2529 self.fail("unable to read bad GNU tar pax header")
2530
2531
2532 class PAXUnicodeTest(UnicodeTest, unittest.TestCase):
2533
2534 format = tarfile.PAX_FORMAT
2535
2536 # PAX_FORMAT ignores encoding in write mode.
2537 test_unicode_filename_error = None
2538
2539 def test_binary_header(self):
2540 # Test a POSIX.1-2008 compatible header with a hdrcharset=BINARY field.
2541 for encoding, name in (
2542 ("utf-8", "pax/hdrcharset-\udce4\udcf6\udcfc"),
2543 ("iso8859-1", "pax/hdrcharset-\xe4\xf6\xfc"),):
2544 with tarfile.open(tarname, encoding=encoding,
2545 errors="surrogateescape") as tar:
2546 try:
2547 t = tar.getmember(name)
2548 except KeyError:
2549 self.fail("unable to read POSIX.1-2008 binary header")
2550
2551
2552 class AppendTestBase:
2553 # Test append mode (cp. patch #1652681).
2554
2555 def setUp(self):
2556 self.tarname = tmpname
2557 if os.path.exists(self.tarname):
2558 os_helper.unlink(self.tarname)
2559
2560 def _create_testtar(self, mode="w:"):
2561 with tarfile.open(tarname, encoding="iso8859-1") as src:
2562 t = src.getmember("ustar/regtype")
2563 t.name = "foo"
2564 with src.extractfile(t) as f:
2565 with tarfile.open(self.tarname, mode) as tar:
2566 tar.addfile(t, f)
2567
2568 def test_append_compressed(self):
2569 self._create_testtar("w:" + self.suffix)
2570 self.assertRaises(tarfile.ReadError, tarfile.open, tmpname, "a")
2571
2572 class AppendTest(AppendTestBase, unittest.TestCase):
2573 test_append_compressed = None
2574
2575 def _add_testfile(self, fileobj=None):
2576 with tarfile.open(self.tarname, "a", fileobj=fileobj) as tar:
2577 tar.addfile(tarfile.TarInfo("bar"))
2578
2579 def _test(self, names=["bar"], fileobj=None):
2580 with tarfile.open(self.tarname, fileobj=fileobj) as tar:
2581 self.assertEqual(tar.getnames(), names)
2582
2583 def test_non_existing(self):
2584 self._add_testfile()
2585 self._test()
2586
2587 def test_empty(self):
2588 tarfile.open(self.tarname, "w:").close()
2589 self._add_testfile()
2590 self._test()
2591
2592 def test_empty_fileobj(self):
2593 fobj = io.BytesIO(b"\0" * 1024)
2594 self._add_testfile(fobj)
2595 fobj.seek(0)
2596 self._test(fileobj=fobj)
2597
2598 def test_fileobj(self):
2599 self._create_testtar()
2600 with open(self.tarname, "rb") as fobj:
2601 data = fobj.read()
2602 fobj = io.BytesIO(data)
2603 self._add_testfile(fobj)
2604 fobj.seek(0)
2605 self._test(names=["foo", "bar"], fileobj=fobj)
2606
2607 def test_existing(self):
2608 self._create_testtar()
2609 self._add_testfile()
2610 self._test(names=["foo", "bar"])
2611
2612 # Append mode is supposed to fail if the tarfile to append to
2613 # does not end with a zero block.
2614 def _test_error(self, data):
2615 with open(self.tarname, "wb") as fobj:
2616 fobj.write(data)
2617 self.assertRaises(tarfile.ReadError, self._add_testfile)
2618
2619 def test_null(self):
2620 self._test_error(b"")
2621
2622 def test_incomplete(self):
2623 self._test_error(b"\0" * 13)
2624
2625 def test_premature_eof(self):
2626 data = tarfile.TarInfo("foo").tobuf()
2627 self._test_error(data)
2628
2629 def test_trailing_garbage(self):
2630 data = tarfile.TarInfo("foo").tobuf()
2631 self._test_error(data + b"\0" * 13)
2632
2633 def test_invalid(self):
2634 self._test_error(b"a" * 512)
2635
2636 class GzipAppendTest(GzipTest, AppendTestBase, unittest.TestCase):
2637 pass
2638
2639 class Bz2AppendTest(Bz2Test, AppendTestBase, unittest.TestCase):
2640 pass
2641
2642 class LzmaAppendTest(LzmaTest, AppendTestBase, unittest.TestCase):
2643 pass
2644
2645 class ZstdAppendTest(ZstdTest, AppendTestBase, unittest.TestCase):
2646 pass
2647
2648 class LimitsTest(unittest.TestCase):
2649
2650 def test_ustar_limits(self):
2651 # 100 char name
2652 tarinfo = tarfile.TarInfo("0123456789" * 10)
2653 tarinfo.tobuf(tarfile.USTAR_FORMAT)
2654
2655 # 101 char name that cannot be stored
2656 tarinfo = tarfile.TarInfo("0123456789" * 10 + "0")
2657 self.assertRaises(ValueError, tarinfo.tobuf, tarfile.USTAR_FORMAT)
2658
2659 # 256 char name with a slash at pos 156
2660 tarinfo = tarfile.TarInfo("123/" * 62 + "longname")
2661 tarinfo.tobuf(tarfile.USTAR_FORMAT)
2662
2663 # 256 char name that cannot be stored
2664 tarinfo = tarfile.TarInfo("1234567/" * 31 + "longname")
2665 self.assertRaises(ValueError, tarinfo.tobuf, tarfile.USTAR_FORMAT)
2666
2667 # 512 char name
2668 tarinfo = tarfile.TarInfo("123/" * 126 + "longname")
2669 self.assertRaises(ValueError, tarinfo.tobuf, tarfile.USTAR_FORMAT)
2670
2671 # 512 char linkname
2672 tarinfo = tarfile.TarInfo("longlink")
2673 tarinfo.linkname = "123/" * 126 + "longname"
2674 self.assertRaises(ValueError, tarinfo.tobuf, tarfile.USTAR_FORMAT)
2675
2676 # uid > 8 digits
2677 tarinfo = tarfile.TarInfo("name")
2678 tarinfo.uid = 0o10000000
2679 self.assertRaises(ValueError, tarinfo.tobuf, tarfile.USTAR_FORMAT)
2680
2681 def test_gnu_limits(self):
2682 tarinfo = tarfile.TarInfo("123/" * 126 + "longname")
2683 tarinfo.tobuf(tarfile.GNU_FORMAT)
2684
2685 tarinfo = tarfile.TarInfo("longlink")
2686 tarinfo.linkname = "123/" * 126 + "longname"
2687 tarinfo.tobuf(tarfile.GNU_FORMAT)
2688
2689 # uid >= 256 ** 7
2690 tarinfo = tarfile.TarInfo("name")
2691 tarinfo.uid = 0o4000000000000000000
2692 self.assertRaises(ValueError, tarinfo.tobuf, tarfile.GNU_FORMAT)
2693
2694 def test_pax_limits(self):
2695 tarinfo = tarfile.TarInfo("123/" * 126 + "longname")
2696 tarinfo.tobuf(tarfile.PAX_FORMAT)
2697
2698 tarinfo = tarfile.TarInfo("longlink")
2699 tarinfo.linkname = "123/" * 126 + "longname"
2700 tarinfo.tobuf(tarfile.PAX_FORMAT)
2701
2702 tarinfo = tarfile.TarInfo("name")
2703 tarinfo.uid = 0o4000000000000000000
2704 tarinfo.tobuf(tarfile.PAX_FORMAT)
2705
2706
2707 class MiscTest(unittest.TestCase):
2708
2709 def test_char_fields(self):
2710 self.assertEqual(tarfile.stn("foo", 8, "ascii", "strict"),
2711 b"foo\0\0\0\0\0")
2712 self.assertEqual(tarfile.stn("foobar", 3, "ascii", "strict"),
2713 b"foo")
2714 self.assertEqual(tarfile.nts(b"foo\0\0\0\0\0", "ascii", "strict"),
2715 "foo")
2716 self.assertEqual(tarfile.nts(b"foo\0bar\0", "ascii", "strict"),
2717 "foo")
2718
2719 def test_read_number_fields(self):
2720 # Issue 13158: Test if GNU tar specific base-256 number fields
2721 # are decoded correctly.
2722 self.assertEqual(tarfile.nti(b"0000001\x00"), 1)
2723 self.assertEqual(tarfile.nti(b"7777777\x00"), 0o7777777)
2724 self.assertEqual(tarfile.nti(b"\x80\x00\x00\x00\x00\x20\x00\x00"),
2725 0o10000000)
2726 self.assertEqual(tarfile.nti(b"\x80\x00\x00\x00\xff\xff\xff\xff"),
2727 0xffffffff)
2728 self.assertEqual(tarfile.nti(b"\xff\xff\xff\xff\xff\xff\xff\xff"),
2729 -1)
2730 self.assertEqual(tarfile.nti(b"\xff\xff\xff\xff\xff\xff\xff\x9c"),
2731 -100)
2732 self.assertEqual(tarfile.nti(b"\xff\x00\x00\x00\x00\x00\x00\x00"),
2733 -0x100000000000000)
2734
2735 # Issue 24514: Test if empty number fields are converted to zero.
2736 self.assertEqual(tarfile.nti(b"\0"), 0)
2737 self.assertEqual(tarfile.nti(b" \0"), 0)
2738
2739 def test_write_number_fields(self):
2740 self.assertEqual(tarfile.itn(1), b"0000001\x00")
2741 self.assertEqual(tarfile.itn(0o7777777), b"7777777\x00")
2742 self.assertEqual(tarfile.itn(0o10000000, format=tarfile.GNU_FORMAT),
2743 b"\x80\x00\x00\x00\x00\x20\x00\x00")
2744 self.assertEqual(tarfile.itn(0xffffffff, format=tarfile.GNU_FORMAT),
2745 b"\x80\x00\x00\x00\xff\xff\xff\xff")
2746 self.assertEqual(tarfile.itn(-1, format=tarfile.GNU_FORMAT),
2747 b"\xff\xff\xff\xff\xff\xff\xff\xff")
2748 self.assertEqual(tarfile.itn(-100, format=tarfile.GNU_FORMAT),
2749 b"\xff\xff\xff\xff\xff\xff\xff\x9c")
2750 self.assertEqual(tarfile.itn(-0x100000000000000,
2751 format=tarfile.GNU_FORMAT),
2752 b"\xff\x00\x00\x00\x00\x00\x00\x00")
2753
2754 # Issue 32713: Test if itn() supports float values outside the
2755 # non-GNU format range
2756 self.assertEqual(tarfile.itn(-100.0, format=tarfile.GNU_FORMAT),
2757 b"\xff\xff\xff\xff\xff\xff\xff\x9c")
2758 self.assertEqual(tarfile.itn(8 ** 12 + 0.0, format=tarfile.GNU_FORMAT),
2759 b"\x80\x00\x00\x10\x00\x00\x00\x00")
2760 self.assertEqual(tarfile.nti(tarfile.itn(-0.1, format=tarfile.GNU_FORMAT)), 0)
2761
2762 def test_number_field_limits(self):
2763 with self.assertRaises(ValueError):
2764 tarfile.itn(-1, 8, tarfile.USTAR_FORMAT)
2765 with self.assertRaises(ValueError):
2766 tarfile.itn(0o10000000, 8, tarfile.USTAR_FORMAT)
2767 with self.assertRaises(ValueError):
2768 tarfile.itn(-0x10000000001, 6, tarfile.GNU_FORMAT)
2769 with self.assertRaises(ValueError):
2770 tarfile.itn(0x10000000000, 6, tarfile.GNU_FORMAT)
2771
2772 def test__all__(self):
2773 not_exported = {
2774 'version', 'grp', 'pwd', 'symlink_exception', 'NUL', 'BLOCKSIZE',
2775 'RECORDSIZE', 'GNU_MAGIC', 'POSIX_MAGIC', 'LENGTH_NAME',
2776 'LENGTH_LINK', 'LENGTH_PREFIX', 'REGTYPE', 'AREGTYPE', 'LNKTYPE',
2777 'SYMTYPE', 'CHRTYPE', 'BLKTYPE', 'DIRTYPE', 'FIFOTYPE', 'CONTTYPE',
2778 'GNUTYPE_LONGNAME', 'GNUTYPE_LONGLINK', 'GNUTYPE_SPARSE',
2779 'XHDTYPE', 'XGLTYPE', 'SOLARIS_XHDTYPE', 'SUPPORTED_TYPES',
2780 'REGULAR_TYPES', 'GNU_TYPES', 'PAX_FIELDS', 'PAX_NAME_FIELDS',
2781 'PAX_NUMBER_FIELDS', 'stn', 'nts', 'nti', 'itn', 'calc_chksums',
2782 'copyfileobj', 'filemode', 'EmptyHeaderError',
2783 'TruncatedHeaderError', 'EOFHeaderError', 'InvalidHeaderError',
2784 'SubsequentHeaderError', 'ExFileObject', 'main'}
2785 support.check__all__(self, tarfile, not_exported=not_exported)
2786
2787 def test_useful_error_message_when_modules_missing(self):
2788 fname = os.path.join(os.path.dirname(__file__), 'archivetestdata', 'testtar.tar.xz')
2789 with self.assertRaises(tarfile.ReadError) as excinfo:
2790 error = tarfile.CompressionError('lzma module is not available'),
2791 with unittest.mock.patch.object(tarfile.TarFile, 'xzopen', side_effect=error):
2792 tarfile.open(fname)
2793
2794 self.assertIn(
2795 "\n- method xz: CompressionError('lzma module is not available')\n",
2796 str(excinfo.exception),
2797 )
2798
2799 @unittest.skipUnless(os_helper.can_symlink(), 'requires symlink support')
2800 @unittest.skipUnless(hasattr(os, 'chmod'), "missing os.chmod")
2801 @unittest.mock.patch('os.chmod')
2802 def test_deferred_directory_attributes_update(self, mock_chmod):
2803 # Regression test for gh-127987: setting attributes on arbitrary files
2804 tempdir = os.path.join(TEMPDIR, 'test127987')
2805 def mock_chmod_side_effect(path, mode, **kwargs):
2806 target_path = os.path.realpath(path)
2807 if os.path.commonpath([target_path, tempdir]) != tempdir:
2808 raise Exception("should not try to chmod anything outside the destination", target_path)
2809 mock_chmod.side_effect = mock_chmod_side_effect
2810
2811 outside_tree_dir = os.path.join(TEMPDIR, 'outside_tree_dir')
2812 with ArchiveMaker() as arc:
2813 arc.add('x', symlink_to='.')
2814 arc.add('x', type=tarfile.DIRTYPE, mode='?rwsrwsrwt')
2815 arc.add('x', symlink_to=outside_tree_dir)
2816
2817 os.makedirs(outside_tree_dir)
2818 try:
2819 arc.open().extractall(path=tempdir, filter='tar')
2820 finally:
2821 os_helper.rmtree(outside_tree_dir)
2822 os_helper.rmtree(tempdir)
2823
2824
2825 class CommandLineTest(unittest.TestCase):
2826
2827 def tarfilecmd(self, *args, **kwargs):
2828 rc, out, err = script_helper.assert_python_ok('-m', 'tarfile', *args,
2829 **kwargs)
2830 return out.replace(os.linesep.encode(), b'\n')
2831
2832 def tarfilecmd_failure(self, *args):
2833 return script_helper.assert_python_failure('-m', 'tarfile', *args)
2834
2835 def make_simple_tarfile(self, tar_name):
2836 files = [support.findfile('tokenize_tests.txt',
2837 subdir='tokenizedata'),
2838 support.findfile('tokenize_tests-no-coding-cookie-'
2839 'and-utf8-bom-sig-only.txt',
2840 subdir='tokenizedata')]
2841 self.addCleanup(os_helper.unlink, tar_name)
2842 with tarfile.open(tar_name, 'w') as tf:
2843 for tardata in files:
2844 tf.add(tardata, arcname=os.path.basename(tardata))
2845
2846 def make_evil_tarfile(self, tar_name):
2847 self.addCleanup(os_helper.unlink, tar_name)
2848 with tarfile.open(tar_name, 'w') as tf:
2849 benign = tarfile.TarInfo('benign')
2850 tf.addfile(benign, fileobj=io.BytesIO(b''))
2851 evil = tarfile.TarInfo('../evil')
2852 tf.addfile(evil, fileobj=io.BytesIO(b''))
2853
2854 def test_bad_use(self):
2855 rc, out, err = self.tarfilecmd_failure()
2856 self.assertEqual(out, b'')
2857 self.assertIn(b'usage', err.lower())
2858 self.assertIn(b'error', err.lower())
2859 self.assertIn(b'required', err.lower())
2860 rc, out, err = self.tarfilecmd_failure('-l', '')
2861 self.assertEqual(out, b'')
2862 self.assertNotEqual(err.strip(), b'')
2863
2864 def test_test_command(self):
2865 for tar_name in testtarnames:
2866 for opt in '-t', '--test':
2867 out = self.tarfilecmd(opt, tar_name)
2868 self.assertEqual(out, b'')
2869
2870 def test_test_command_verbose(self):
2871 for tar_name in testtarnames:
2872 for opt in '-v', '--verbose':
2873 out = self.tarfilecmd(opt, '-t', tar_name,
2874 PYTHONIOENCODING='utf-8')
2875 self.assertIn(b'is a tar archive.\n', out)
2876
2877 def test_test_command_invalid_file(self):
2878 zipname = support.findfile('zipdir.zip', subdir='archivetestdata')
2879 rc, out, err = self.tarfilecmd_failure('-t', zipname)
2880 self.assertIn(b' is not a tar archive.', err)
2881 self.assertEqual(out, b'')
2882 self.assertEqual(rc, 1)
2883
2884 for tar_name in testtarnames:
2885 with self.subTest(tar_name=tar_name):
2886 with open(tar_name, 'rb') as f:
2887 data = f.read()
2888 try:
2889 with open(tmpname, 'wb') as f:
2890 f.write(data[:511])
2891 rc, out, err = self.tarfilecmd_failure('-t', tmpname)
2892 self.assertEqual(out, b'')
2893 self.assertEqual(rc, 1)
2894 finally:
2895 os_helper.unlink(tmpname)
2896
2897 def test_list_command(self):
2898 for tar_name in testtarnames:
2899 with support.captured_stdout() as t:
2900 with tarfile.open(tar_name, 'r') as tf:
2901 tf.list(verbose=False)
2902 expected = t.getvalue().encode('ascii', 'backslashreplace')
2903 for opt in '-l', '--list':
2904 out = self.tarfilecmd(opt, tar_name,
2905 PYTHONIOENCODING='ascii')
2906 self.assertEqual(out, expected)
2907
2908 def test_list_command_verbose(self):
2909 for tar_name in testtarnames:
2910 with support.captured_stdout() as t:
2911 with tarfile.open(tar_name, 'r') as tf:
2912 tf.list(verbose=True)
2913 expected = t.getvalue().encode('ascii', 'backslashreplace')
2914 for opt in '-v', '--verbose':
2915 out = self.tarfilecmd(opt, '-l', tar_name,
2916 PYTHONIOENCODING='ascii')
2917 self.assertEqual(out, expected)
2918
2919 def test_list_command_invalid_file(self):
2920 zipname = support.findfile('zipdir.zip', subdir='archivetestdata')
2921 rc, out, err = self.tarfilecmd_failure('-l', zipname)
2922 self.assertIn(b' is not a tar archive.', err)
2923 self.assertEqual(out, b'')
2924 self.assertEqual(rc, 1)
2925
2926 def test_create_command(self):
2927 files = [support.findfile('tokenize_tests.txt',
2928 subdir='tokenizedata'),
2929 support.findfile('tokenize_tests-no-coding-cookie-'
2930 'and-utf8-bom-sig-only.txt',
2931 subdir='tokenizedata')]
2932 for opt in '-c', '--create':
2933 try:
2934 out = self.tarfilecmd(opt, tmpname, *files)
2935 self.assertEqual(out, b'')
2936 with tarfile.open(tmpname) as tar:
2937 tar.getmembers()
2938 finally:
2939 os_helper.unlink(tmpname)
2940
2941 def test_create_command_verbose(self):
2942 files = [support.findfile('tokenize_tests.txt',
2943 subdir='tokenizedata'),
2944 support.findfile('tokenize_tests-no-coding-cookie-'
2945 'and-utf8-bom-sig-only.txt',
2946 subdir='tokenizedata')]
2947 for opt in '-v', '--verbose':
2948 try:
2949 out = self.tarfilecmd(opt, '-c', tmpname, *files,
2950 PYTHONIOENCODING='utf-8')
2951 self.assertIn(b' file created.', out)
2952 with tarfile.open(tmpname) as tar:
2953 tar.getmembers()
2954 finally:
2955 os_helper.unlink(tmpname)
2956
2957 def test_create_command_dotless_filename(self):
2958 files = [support.findfile('tokenize_tests.txt', subdir='tokenizedata')]
2959 try:
2960 out = self.tarfilecmd('-c', dotlessname, *files)
2961 self.assertEqual(out, b'')
2962 with tarfile.open(dotlessname) as tar:
2963 tar.getmembers()
2964 finally:
2965 os_helper.unlink(dotlessname)
2966
2967 def test_create_command_dot_started_filename(self):
2968 tar_name = os.path.join(TEMPDIR, ".testtar")
2969 files = [support.findfile('tokenize_tests.txt', subdir='tokenizedata')]
2970 try:
2971 out = self.tarfilecmd('-c', tar_name, *files)
2972 self.assertEqual(out, b'')
2973 with tarfile.open(tar_name) as tar:
2974 tar.getmembers()
2975 finally:
2976 os_helper.unlink(tar_name)
2977
2978 def test_create_command_compressed(self):
2979 files = [support.findfile('tokenize_tests.txt',
2980 subdir='tokenizedata'),
2981 support.findfile('tokenize_tests-no-coding-cookie-'
2982 'and-utf8-bom-sig-only.txt',
2983 subdir='tokenizedata')]
2984 for filetype in (GzipTest, Bz2Test, LzmaTest, ZstdTest):
2985 if not filetype.open:
2986 continue
2987 try:
2988 tar_name = tmpname + '.' + filetype.suffix
2989 out = self.tarfilecmd('-c', tar_name, *files)
2990 with filetype.taropen(tar_name) as tar:
2991 tar.getmembers()
2992 finally:
2993 os_helper.unlink(tar_name)
2994
2995 def test_extract_command(self):
2996 self.make_simple_tarfile(tmpname)
2997 for opt in '-e', '--extract':
2998 try:
2999 with os_helper.temp_cwd(tarextdir):
3000 out = self.tarfilecmd(opt, tmpname)
3001 self.assertEqual(out, b'')
3002 finally:
3003 os_helper.rmtree(tarextdir)
3004
3005 def test_extract_command_verbose(self):
3006 self.make_simple_tarfile(tmpname)
3007 for opt in '-v', '--verbose':
3008 try:
3009 with os_helper.temp_cwd(tarextdir):
3010 out = self.tarfilecmd(opt, '-e', tmpname,
3011 PYTHONIOENCODING='utf-8')
3012 self.assertIn(b' file is extracted.', out)
3013 finally:
3014 os_helper.rmtree(tarextdir)
3015
3016 def test_extract_command_filter(self):
3017 self.make_evil_tarfile(tmpname)
3018 # Make an inner directory, so the member named '../evil'
3019 # is still extracted into `tarextdir`
3020 destdir = os.path.join(tarextdir, 'dest')
3021 os.mkdir(tarextdir)
3022 try:
3023 with os_helper.temp_cwd(destdir):
3024 self.tarfilecmd_failure('-e', tmpname,
3025 '-v',
3026 '--filter', 'data')
3027 out = self.tarfilecmd('-e', tmpname,
3028 '-v',
3029 '--filter', 'fully_trusted',
3030 PYTHONIOENCODING='utf-8')
3031 self.assertIn(b' file is extracted.', out)
3032 finally:
3033 os_helper.rmtree(tarextdir)
3034
3035 def test_extract_command_different_directory(self):
3036 self.make_simple_tarfile(tmpname)
3037 try:
3038 with os_helper.temp_cwd(tarextdir):
3039 out = self.tarfilecmd('-e', tmpname, 'spamdir')
3040 self.assertEqual(out, b'')
3041 finally:
3042 os_helper.rmtree(tarextdir)
3043
3044 def test_extract_command_invalid_file(self):
3045 zipname = support.findfile('zipdir.zip', subdir='archivetestdata')
3046 with os_helper.temp_cwd(tarextdir):
3047 rc, out, err = self.tarfilecmd_failure('-e', zipname)
3048 self.assertIn(b' is not a tar archive.', err)
3049 self.assertEqual(out, b'')
3050 self.assertEqual(rc, 1)
3051
3052
3053 class ContextManagerTest(unittest.TestCase):
3054
3055 def test_basic(self):
3056 with tarfile.open(tarname) as tar:
3057 self.assertFalse(tar.closed, "closed inside runtime context")
3058 self.assertTrue(tar.closed, "context manager failed")
3059
3060 def test_closed(self):
3061 # The __enter__() method is supposed to raise OSError
3062 # if the TarFile object is already closed.
3063 tar = tarfile.open(tarname)
3064 tar.close()
3065 with self.assertRaises(OSError):
3066 with tar:
3067 pass
3068
3069 def test_exception(self):
3070 # Test if the OSError exception is passed through properly.
3071 with self.assertRaises(Exception) as exc:
3072 with tarfile.open(tarname) as tar:
3073 raise OSError
3074 self.assertIsInstance(exc.exception, OSError,
3075 "wrong exception raised in context manager")
3076 self.assertTrue(tar.closed, "context manager failed")
3077
3078 def test_no_eof(self):
3079 # __exit__() must not write end-of-archive blocks if an
3080 # exception was raised.
3081 try:
3082 with tarfile.open(tmpname, "w") as tar:
3083 raise Exception
3084 except:
3085 pass
3086 self.assertEqual(os.path.getsize(tmpname), 0,
3087 "context manager wrote an end-of-archive block")
3088 self.assertTrue(tar.closed, "context manager failed")
3089
3090 def test_eof(self):
3091 # __exit__() must write end-of-archive blocks, i.e. call
3092 # TarFile.close() if there was no error.
3093 with tarfile.open(tmpname, "w"):
3094 pass
3095 self.assertNotEqual(os.path.getsize(tmpname), 0,
3096 "context manager wrote no end-of-archive block")
3097
3098 def test_fileobj(self):
3099 # Test that __exit__() did not close the external file
3100 # object.
3101 with open(tmpname, "wb") as fobj:
3102 try:
3103 with tarfile.open(fileobj=fobj, mode="w") as tar:
3104 raise Exception
3105 except:
3106 pass
3107 self.assertFalse(fobj.closed, "external file object was closed")
3108 self.assertTrue(tar.closed, "context manager failed")
3109
3110
3111 @unittest.skipIf(hasattr(os, "link"), "requires os.link to be missing")
3112 class LinkEmulationTest(ReadTest, unittest.TestCase):
3113
3114 # Test for issue #8741 regression. On platforms that do not support
3115 # symbolic or hard links tarfile tries to extract these types of members
3116 # as the regular files they point to.
3117 def _test_link_extraction(self, name):
3118 self.tar.extract(name, TEMPDIR, filter='fully_trusted')
3119 with open(os.path.join(TEMPDIR, name), "rb") as f:
3120 data = f.read()
3121 self.assertEqual(sha256sum(data), sha256_regtype)
3122
3123 # See issues #1578269, #8879, and #17689 for some history on these skips
3124 @unittest.skipIf(hasattr(os.path, "islink"),
3125 "Skip emulation - has os.path.islink but not os.link")
3126 def test_hardlink_extraction1(self):
3127 self._test_link_extraction("ustar/lnktype")
3128
3129 @unittest.skipIf(hasattr(os.path, "islink"),
3130 "Skip emulation - has os.path.islink but not os.link")
3131 def test_hardlink_extraction2(self):
3132 self._test_link_extraction("./ustar/linktest2/lnktype")
3133
3134 @unittest.skipIf(hasattr(os, "symlink"),
3135 "Skip emulation if symlink exists")
3136 def test_symlink_extraction1(self):
3137 self._test_link_extraction("ustar/symtype")
3138
3139 @unittest.skipIf(hasattr(os, "symlink"),
3140 "Skip emulation if symlink exists")
3141 def test_symlink_extraction2(self):
3142 self._test_link_extraction("./ustar/linktest2/symtype")
3143
3144
3145 class Bz2PartialReadTest(Bz2Test, unittest.TestCase):
3146 # Issue5068: The _BZ2Proxy.read() method loops forever
3147 # on an empty or partial bzipped file.
3148
3149 def _test_partial_input(self, mode):
3150 class MyBytesIO(io.BytesIO):
3151 hit_eof = False
3152 def read(self, n):
3153 if self.hit_eof:
3154 raise AssertionError("infinite loop detected in "
3155 "tarfile.open()")
3156 self.hit_eof = self.tell() == len(self.getvalue())
3157 return super(MyBytesIO, self).read(n)
3158 def seek(self, *args):
3159 self.hit_eof = False
3160 return super(MyBytesIO, self).seek(*args)
3161
3162 data = bz2.compress(tarfile.TarInfo("foo").tobuf())
3163 for x in range(len(data) + 1):
3164 try:
3165 tarfile.open(fileobj=MyBytesIO(data[:x]), mode=mode)
3166 except tarfile.ReadError:
3167 pass # we have no interest in ReadErrors
3168
3169 def test_partial_input(self):
3170 self._test_partial_input("r")
3171
3172 def test_partial_input_bz2(self):
3173 self._test_partial_input("r:bz2")
3174
3175
3176 def root_is_uid_gid_0():
3177 try:
3178 import pwd, grp
3179 except ImportError:
3180 return False
3181 if pwd.getpwuid(0)[0] != 'root':
3182 return False
3183 if grp.getgrgid(0)[0] != 'root':
3184 return False
3185 return True
3186
3187
3188 @unittest.skipUnless(hasattr(os, 'chown'), "missing os.chown")
3189 @unittest.skipUnless(hasattr(os, 'geteuid'), "missing os.geteuid")
3190 class NumericOwnerTest(unittest.TestCase):
3191 # mock the following:
3192 # os.chown: so we can test what's being called
3193 # os.chmod: so the modes are not actually changed. if they are, we can't
3194 # delete the files/directories
3195 # os.geteuid: so we can lie and say we're root (uid = 0)
3196
3197 @staticmethod
3198 def _make_test_archive(filename_1, dirname_1, filename_2):
3199 # the file contents to write
3200 fobj = io.BytesIO(b"content")
3201
3202 # create a tar file with a file, a directory, and a file within that
3203 # directory. Assign various .uid/.gid values to them
3204 items = [(filename_1, 99, 98, tarfile.REGTYPE, fobj),
3205 (dirname_1, 77, 76, tarfile.DIRTYPE, None),
3206 (filename_2, 88, 87, tarfile.REGTYPE, fobj),
3207 ]
3208 with tarfile.open(tmpname, 'w') as tarfl:
3209 for name, uid, gid, typ, contents in items:
3210 t = tarfile.TarInfo(name)
3211 t.uid = uid
3212 t.gid = gid
3213 t.uname = 'root'
3214 t.gname = 'root'
3215 t.type = typ
3216 tarfl.addfile(t, contents)
3217
3218 # return the full pathname to the tar file
3219 return tmpname
3220
3221 @staticmethod
3222 @contextmanager
3223 def _setup_test(mock_geteuid):
3224 mock_geteuid.return_value = 0 # lie and say we're root
3225 fname = 'numeric-owner-testfile'
3226 dirname = 'dir'
3227
3228 # the names we want stored in the tarfile
3229 filename_1 = fname
3230 dirname_1 = dirname
3231 filename_2 = os.path.join(dirname, fname)
3232
3233 # create the tarfile with the contents we're after
3234 tar_filename = NumericOwnerTest._make_test_archive(filename_1,
3235 dirname_1,
3236 filename_2)
3237
3238 # open the tarfile for reading. yield it and the names of the items
3239 # we stored into the file
3240 with tarfile.open(tar_filename) as tarfl:
3241 yield tarfl, filename_1, dirname_1, filename_2
3242
3243 @unittest.mock.patch('os.chown')
3244 @unittest.mock.patch('os.chmod')
3245 @unittest.mock.patch('os.geteuid')
3246 def test_extract_with_numeric_owner(self, mock_geteuid, mock_chmod,
3247 mock_chown):
3248 with self._setup_test(mock_geteuid) as (tarfl, filename_1, _,
3249 filename_2):
3250 tarfl.extract(filename_1, TEMPDIR, numeric_owner=True,
3251 filter='fully_trusted')
3252 tarfl.extract(filename_2 , TEMPDIR, numeric_owner=True,
3253 filter='fully_trusted')
3254
3255 # convert to filesystem paths
3256 f_filename_1 = os.path.join(TEMPDIR, filename_1)
3257 f_filename_2 = os.path.join(TEMPDIR, filename_2)
3258
3259 mock_chown.assert_has_calls([unittest.mock.call(f_filename_1, 99, 98),
3260 unittest.mock.call(f_filename_2, 88, 87),
3261 ],
3262 any_order=True)
3263
3264 @unittest.mock.patch('os.chown')
3265 @unittest.mock.patch('os.chmod')
3266 @unittest.mock.patch('os.geteuid')
3267 def test_extractall_with_numeric_owner(self, mock_geteuid, mock_chmod,
3268 mock_chown):
3269 with self._setup_test(mock_geteuid) as (tarfl, filename_1, dirname_1,
3270 filename_2):
3271 tarfl.extractall(TEMPDIR, numeric_owner=True,
3272 filter='fully_trusted')
3273
3274 # convert to filesystem paths
3275 f_filename_1 = os.path.join(TEMPDIR, filename_1)
3276 f_dirname_1 = os.path.join(TEMPDIR, dirname_1)
3277 f_filename_2 = os.path.join(TEMPDIR, filename_2)
3278
3279 mock_chown.assert_has_calls([unittest.mock.call(f_filename_1, 99, 98),
3280 unittest.mock.call(f_dirname_1, 77, 76),
3281 unittest.mock.call(f_filename_2, 88, 87),
3282 ],
3283 any_order=True)
3284
3285 # this test requires that uid=0 and gid=0 really be named 'root'. that's
3286 # because the uname and gname in the test file are 'root', and extract()
3287 # will look them up using pwd and grp to find their uid and gid, which we
3288 # test here to be 0.
3289 @unittest.skipUnless(root_is_uid_gid_0(),
3290 'uid=0,gid=0 must be named "root"')
3291 @unittest.mock.patch('os.chown')
3292 @unittest.mock.patch('os.chmod')
3293 @unittest.mock.patch('os.geteuid')
3294 def test_extract_without_numeric_owner(self, mock_geteuid, mock_chmod,
3295 mock_chown):
3296 with self._setup_test(mock_geteuid) as (tarfl, filename_1, _, _):
3297 tarfl.extract(filename_1, TEMPDIR, numeric_owner=False,
3298 filter='fully_trusted')
3299
3300 # convert to filesystem paths
3301 f_filename_1 = os.path.join(TEMPDIR, filename_1)
3302
3303 mock_chown.assert_called_with(f_filename_1, 0, 0)
3304
3305 @unittest.mock.patch('os.geteuid')
3306 def test_keyword_only(self, mock_geteuid):
3307 with self._setup_test(mock_geteuid) as (tarfl, filename_1, _, _):
3308 self.assertRaises(TypeError,
3309 tarfl.extract, filename_1, TEMPDIR, False, True)
3310
3311
3312 class ReplaceTests(ReadTest, unittest.TestCase):
3313 def test_replace_name(self):
3314 member = self.tar.getmember('ustar/regtype')
3315 replaced = member.replace(name='misc/other')
3316 self.assertEqual(replaced.name, 'misc/other')
3317 self.assertEqual(member.name, 'ustar/regtype')
3318 self.assertEqual(self.tar.getmember('ustar/regtype').name,
3319 'ustar/regtype')
3320
3321 def test_replace_deep(self):
3322 member = self.tar.getmember('pax/regtype1')
3323 replaced = member.replace()
3324 replaced.pax_headers['gname'] = 'not-bar'
3325 self.assertEqual(member.pax_headers['gname'], 'bar')
3326 self.assertEqual(
3327 self.tar.getmember('pax/regtype1').pax_headers['gname'], 'bar')
3328
3329 def test_replace_shallow(self):
3330 member = self.tar.getmember('pax/regtype1')
3331 replaced = member.replace(deep=False)
3332 replaced.pax_headers['gname'] = 'not-bar'
3333 self.assertEqual(member.pax_headers['gname'], 'not-bar')
3334 self.assertEqual(
3335 self.tar.getmember('pax/regtype1').pax_headers['gname'], 'not-bar')
3336
3337 def test_replace_all(self):
3338 member = self.tar.getmember('ustar/regtype')
3339 for attr_name in ('name', 'mtime', 'mode', 'linkname',
3340 'uid', 'gid', 'uname', 'gname'):
3341 with self.subTest(attr_name=attr_name):
3342 replaced = member.replace(**{attr_name: None})
3343 self.assertEqual(getattr(replaced, attr_name), None)
3344 self.assertNotEqual(getattr(member, attr_name), None)
3345
3346 def test_replace_internal(self):
3347 member = self.tar.getmember('ustar/regtype')
3348 with self.assertRaises(TypeError):
3349 member.replace(offset=123456789)
3350
3351
3352 class NoneInfoExtractTests(ReadTest):
3353 # These mainly check that all kinds of members are extracted successfully
3354 # if some metadata is None.
3355 # Some of the methods do additional spot checks.
3356
3357 # We also test that the default filters can deal with None.
3358
3359 extraction_filter = None
3360
3361 @classmethod
3362 def setUpClass(cls):
3363 tar = tarfile.open(tarname, mode='r', encoding="iso8859-1")
3364 cls.control_dir = pathlib.Path(TEMPDIR) / "extractall_ctrl"
3365 tar.errorlevel = 0
3366 with ExitStack() as cm:
3367 if cls.extraction_filter is None:
3368 cm.enter_context(warnings.catch_warnings(
3369 action="ignore", category=DeprecationWarning))
3370 tar.extractall(cls.control_dir, filter=cls.extraction_filter)
3371 tar.close()
3372 cls.control_paths = set(
3373 p.relative_to(cls.control_dir)
3374 for p in pathlib.Path(cls.control_dir).glob('**/*'))
3375
3376 @classmethod
3377 def tearDownClass(cls):
3378 shutil.rmtree(cls.control_dir)
3379
3380 def check_files_present(self, directory):
3381 got_paths = set(
3382 p.relative_to(directory)
3383 for p in pathlib.Path(directory).glob('**/*'))
3384 if self.extraction_filter in (None, 'data'):
3385 # The 'data' filter is expected to reject special files
3386 for path in 'ustar/fifotype', 'ustar/blktype', 'ustar/chrtype':
3387 got_paths.discard(pathlib.Path(path))
3388 self.assertEqual(self.control_paths, got_paths)
3389
3390 @contextmanager
3391 def extract_with_none(self, *attr_names):
3392 DIR = pathlib.Path(TEMPDIR) / "extractall_none"
3393 self.tar.errorlevel = 0
3394 for member in self.tar.getmembers():
3395 for attr_name in attr_names:
3396 setattr(member, attr_name, None)
3397 with os_helper.temp_dir(DIR):
3398 self.tar.extractall(DIR, filter='fully_trusted')
3399 self.check_files_present(DIR)
3400 yield DIR
3401
3402 def test_extractall_none_mtime(self):
3403 # mtimes of extracted files should be later than 'now' -- the mtime
3404 # of a previously created directory.
3405 now = pathlib.Path(TEMPDIR).stat().st_mtime
3406 with self.extract_with_none('mtime') as DIR:
3407 for path in pathlib.Path(DIR).glob('**/*'):
3408 with self.subTest(path=path):
3409 try:
3410 mtime = path.stat().st_mtime
3411 except OSError:
3412 # Some systems can't stat symlinks, ignore those
3413 if not path.is_symlink():
3414 raise
3415 else:
3416 self.assertGreaterEqual(path.stat().st_mtime, now)
3417
3418 def test_extractall_none_mode(self):
3419 # modes of directories and regular files should match the mode
3420 # of a "normally" created directory or regular file
3421 dir_mode = pathlib.Path(TEMPDIR).stat().st_mode
3422 regular_file = pathlib.Path(TEMPDIR) / 'regular_file'
3423 regular_file.write_text('')
3424 regular_file_mode = regular_file.stat().st_mode
3425 with self.extract_with_none('mode') as DIR:
3426 for path in pathlib.Path(DIR).glob('**/*'):
3427 with self.subTest(path=path):
3428 if path.is_dir():
3429 self.assertEqual(path.stat().st_mode, dir_mode)
3430 elif path.is_file():
3431 self.assertEqual(path.stat().st_mode,
3432 regular_file_mode)
3433
3434 def test_extractall_none_uid(self):
3435 with self.extract_with_none('uid'):
3436 pass
3437
3438 def test_extractall_none_gid(self):
3439 with self.extract_with_none('gid'):
3440 pass
3441
3442 def test_extractall_none_uname(self):
3443 with self.extract_with_none('uname'):
3444 pass
3445
3446 def test_extractall_none_gname(self):
3447 with self.extract_with_none('gname'):
3448 pass
3449
3450 def test_extractall_none_ownership(self):
3451 with self.extract_with_none('uid', 'gid', 'uname', 'gname'):
3452 pass
3453
3454 class NoneInfoExtractTests_Data(NoneInfoExtractTests, unittest.TestCase):
3455 extraction_filter = 'data'
3456
3457 class NoneInfoExtractTests_FullyTrusted(NoneInfoExtractTests,
3458 unittest.TestCase):
3459 extraction_filter = 'fully_trusted'
3460
3461 class NoneInfoExtractTests_Tar(NoneInfoExtractTests, unittest.TestCase):
3462 extraction_filter = 'tar'
3463
3464 class NoneInfoExtractTests_Default(NoneInfoExtractTests,
3465 unittest.TestCase):
3466 extraction_filter = None
3467
3468 class NoneInfoTests_Misc(unittest.TestCase):
3469 def test_add(self):
3470 # When addfile() encounters None metadata, it raises a ValueError
3471 bio = io.BytesIO()
3472 for tarformat in (tarfile.USTAR_FORMAT, tarfile.GNU_FORMAT,
3473 tarfile.PAX_FORMAT):
3474 with self.subTest(tarformat=tarformat):
3475 tar = tarfile.open(fileobj=bio, mode='w', format=tarformat)
3476 tarinfo = tar.gettarinfo(tarname)
3477 try:
3478 with open(tarname, 'rb') as f:
3479 tar.addfile(tarinfo, f)
3480 except Exception:
3481 if tarformat == tarfile.USTAR_FORMAT:
3482 # In the old, limited format, adding might fail for
3483 # reasons like the UID being too large
3484 pass
3485 else:
3486 raise
3487 else:
3488 for attr_name in ('mtime', 'mode', 'uid', 'gid',
3489 'uname', 'gname'):
3490 with self.subTest(attr_name=attr_name):
3491 replaced = tarinfo.replace(**{attr_name: None})
3492 with self.assertRaisesRegex(ValueError,
3493 f"{attr_name}"):
3494 with open(tarname, 'rb') as f:
3495 tar.addfile(replaced, f)
3496
3497 def test_list(self):
3498 # Change some metadata to None, then compare list() output
3499 # word-for-word. We want list() to not raise, and to only change
3500 # printout for the affected piece of metadata.
3501 # (n.b.: some contents of the test archive are hardcoded.)
3502 for attr_names in ({'mtime'}, {'mode'}, {'uid'}, {'gid'},
3503 {'uname'}, {'gname'},
3504 {'uid', 'uname'}, {'gid', 'gname'}):
3505 with (self.subTest(attr_names=attr_names),
3506 tarfile.open(tarname, encoding="iso8859-1") as tar):
3507 tio_prev = io.TextIOWrapper(io.BytesIO(), 'ascii', newline='\n')
3508 with support.swap_attr(sys, 'stdout', tio_prev):
3509 tar.list()
3510 for member in tar.getmembers():
3511 for attr_name in attr_names:
3512 setattr(member, attr_name, None)
3513 tio_new = io.TextIOWrapper(io.BytesIO(), 'ascii', newline='\n')
3514 with support.swap_attr(sys, 'stdout', tio_new):
3515 tar.list()
3516 for expected, got in zip(tio_prev.detach().getvalue().split(),
3517 tio_new.detach().getvalue().split()):
3518 if attr_names == {'mtime'} and re.match(rb'2003-01-\d\d', expected):
3519 self.assertEqual(got, b'????-??-??')
3520 elif attr_names == {'mtime'} and re.match(rb'\d\d:\d\d:\d\d', expected):
3521 self.assertEqual(got, b'??:??:??')
3522 elif attr_names == {'mode'} and re.match(
3523 rb'.([r-][w-][x-]){3}', expected):
3524 self.assertEqual(got, b'??????????')
3525 elif attr_names == {'uname'} and expected.startswith(
3526 (b'tarfile/', b'lars/', b'foo/')):
3527 exp_user, exp_group = expected.split(b'/')
3528 got_user, got_group = got.split(b'/')
3529 self.assertEqual(got_group, exp_group)
3530 self.assertRegex(got_user, b'[0-9]+')
3531 elif attr_names == {'gname'} and expected.endswith(
3532 (b'/tarfile', b'/users', b'/bar')):
3533 exp_user, exp_group = expected.split(b'/')
3534 got_user, got_group = got.split(b'/')
3535 self.assertEqual(got_user, exp_user)
3536 self.assertRegex(got_group, b'[0-9]+')
3537 elif attr_names == {'uid'} and expected.startswith(
3538 (b'1000/')):
3539 exp_user, exp_group = expected.split(b'/')
3540 got_user, got_group = got.split(b'/')
3541 self.assertEqual(got_group, exp_group)
3542 self.assertEqual(got_user, b'None')
3543 elif attr_names == {'gid'} and expected.endswith((b'/100')):
3544 exp_user, exp_group = expected.split(b'/')
3545 got_user, got_group = got.split(b'/')
3546 self.assertEqual(got_user, exp_user)
3547 self.assertEqual(got_group, b'None')
3548 elif attr_names == {'uid', 'uname'} and expected.startswith(
3549 (b'tarfile/', b'lars/', b'foo/', b'1000/')):
3550 exp_user, exp_group = expected.split(b'/')
3551 got_user, got_group = got.split(b'/')
3552 self.assertEqual(got_group, exp_group)
3553 self.assertEqual(got_user, b'None')
3554 elif attr_names == {'gname', 'gid'} and expected.endswith(
3555 (b'/tarfile', b'/users', b'/bar', b'/100')):
3556 exp_user, exp_group = expected.split(b'/')
3557 got_user, got_group = got.split(b'/')
3558 self.assertEqual(got_user, exp_user)
3559 self.assertEqual(got_group, b'None')
3560 else:
3561 # In other cases the output should be the same
3562 self.assertEqual(expected, got)
3563
3564 def _filemode_to_int(mode):
3565 """Inverse of `stat.filemode` (for permission bits)
3566
3567 Using mode strings rather than numbers makes the later tests more readable.
3568 """
3569 str_mode = mode[1:]
3570 result = (
3571 {'r': stat.S_IRUSR, '-': 0}[str_mode[0]]
3572 | {'w': stat.S_IWUSR, '-': 0}[str_mode[1]]
3573 | {'x': stat.S_IXUSR, '-': 0,
3574 's': stat.S_IXUSR | stat.S_ISUID,
3575 'S': stat.S_ISUID}[str_mode[2]]
3576 | {'r': stat.S_IRGRP, '-': 0}[str_mode[3]]
3577 | {'w': stat.S_IWGRP, '-': 0}[str_mode[4]]
3578 | {'x': stat.S_IXGRP, '-': 0,
3579 's': stat.S_IXGRP | stat.S_ISGID,
3580 'S': stat.S_ISGID}[str_mode[5]]
3581 | {'r': stat.S_IROTH, '-': 0}[str_mode[6]]
3582 | {'w': stat.S_IWOTH, '-': 0}[str_mode[7]]
3583 | {'x': stat.S_IXOTH, '-': 0,
3584 't': stat.S_IXOTH | stat.S_ISVTX,
3585 'T': stat.S_ISVTX}[str_mode[8]]
3586 )
3587 # check we did this right
3588 assert stat.filemode(result)[1:] == mode[1:]
3589
3590 return result
3591
3592 class ArchiveMaker:
3593 """Helper to create a tar file with specific contents
3594
3595 Usage:
3596
3597 with ArchiveMaker() as t:
3598 t.add('filename', ...)
3599
3600 with t.open() as tar:
3601 ... # `tar` is now a TarFile with 'filename' in it!
3602 """
3603 def __init__(self, **kwargs):
3604 self.bio = io.BytesIO()
3605 self.tar_kwargs = dict(kwargs)
3606
3607 def __enter__(self):
3608 self.tar_w = tarfile.TarFile(mode='w', fileobj=self.bio, **self.tar_kwargs)
3609 return self
3610
3611 def __exit__(self, *exc):
3612 self.tar_w.close()
3613 self.contents = self.bio.getvalue()
3614 self.bio = None
3615
3616 def add(self, name, *, type=None, symlink_to=None, hardlink_to=None,
3617 mode=None, size=None, content=None, **kwargs):
3618 """Add a member to the test archive. Call within `with`.
3619
3620 Provides many shortcuts:
3621 - default `type` is based on symlink_to, hardlink_to, and trailing `/`
3622 in name (which is stripped)
3623 - size & content defaults are based on each other
3624 - content can be str or bytes
3625 - mode should be textual ('-rwxrwxrwx')
3626
3627 (add more! this is unstable internal test-only API)
3628 """
3629 name = str(name)
3630 tarinfo = tarfile.TarInfo(name).replace(**kwargs)
3631 if content is not None:
3632 if isinstance(content, str):
3633 content = content.encode()
3634 size = len(content)
3635 if size is not None:
3636 tarinfo.size = size
3637 if content is None:
3638 content = bytes(tarinfo.size)
3639 if mode:
3640 tarinfo.mode = _filemode_to_int(mode)
3641 if symlink_to is not None:
3642 type = tarfile.SYMTYPE
3643 tarinfo.linkname = str(symlink_to)
3644 if hardlink_to is not None:
3645 type = tarfile.LNKTYPE
3646 tarinfo.linkname = str(hardlink_to)
3647 if name.endswith('/') and type is None:
3648 type = tarfile.DIRTYPE
3649 if type is not None:
3650 tarinfo.type = type
3651 if tarinfo.isreg():
3652 fileobj = io.BytesIO(content)
3653 else:
3654 fileobj = None
3655 self.tar_w.addfile(tarinfo, fileobj)
3656
3657 def open(self, **kwargs):
3658 """Open the resulting archive as TarFile. Call after `with`."""
3659 bio = io.BytesIO(self.contents)
3660 return tarfile.open(fileobj=bio, **kwargs)
3661
3662 # Under WASI, `os_helper.can_symlink` is False to make
3663 # `skip_unless_symlink` skip symlink tests. "
3664 # But in the following tests we use can_symlink to *determine* which
3665 # behavior is expected.
3666 # Like other symlink tests, skip these on WASI for now.
3667 if support.is_wasi:
3668 def symlink_test(f):
3669 return unittest.skip("WASI: Skip symlink test for now")(f)
3670 else:
3671 def symlink_test(f):
3672 return f
3673
3674
3675 class TestExtractionFilters(unittest.TestCase):
3676
3677 # A temporary directory for the extraction results.
3678 # All files that "escape" the destination path should still end
3679 # up in this directory.
3680 outerdir = pathlib.Path(TEMPDIR) / 'outerdir'
3681
3682 # The destination for the extraction, within `outerdir`
3683 destdir = outerdir / 'dest'
3684
3685 @contextmanager
3686 def check_context(self, tar, filter, *, check_flag=True):
3687 """Extracts `tar` to `self.destdir` and allows checking the result
3688
3689 If an error occurs, it must be checked using `expect_exception`
3690
3691 Otherwise, all resulting files must be checked using `expect_file`,
3692 except the destination directory itself and parent directories of
3693 other files.
3694 When checking directories, do so before their contents.
3695
3696 A file called 'flag' is made in outerdir (i.e. outside destdir)
3697 before extraction; it should not be altered nor should its contents
3698 be read/copied.
3699 """
3700 with os_helper.temp_dir(self.outerdir):
3701 flag_path = self.outerdir / 'flag'
3702 flag_path.write_text('capture me')
3703 try:
3704 tar.extractall(self.destdir, filter=filter)
3705 except Exception as exc:
3706 self.raised_exception = exc
3707 self.reraise_exception = True
3708 self.expected_paths = set()
3709 else:
3710 self.raised_exception = None
3711 self.reraise_exception = False
3712 self.expected_paths = set(self.outerdir.glob('**/*'))
3713 self.expected_paths.discard(self.destdir)
3714 self.expected_paths.discard(flag_path)
3715 try:
3716 yield self
3717 finally:
3718 tar.close()
3719 if self.reraise_exception:
3720 raise self.raised_exception
3721 self.assertEqual(self.expected_paths, set())
3722 if check_flag:
3723 self.assertEqual(flag_path.read_text(), 'capture me')
3724 else:
3725 assert filter == 'fully_trusted'
3726
3727 def expect_file(self, name, type=None, symlink_to=None, mode=None,
3728 size=None, content=None):
3729 """Check a single file. See check_context."""
3730 if self.raised_exception:
3731 raise self.raised_exception
3732 # use normpath() rather than resolve() so we don't follow symlinks
3733 path = pathlib.Path(os.path.normpath(self.destdir / name))
3734 self.assertIn(path, self.expected_paths)
3735 self.expected_paths.remove(path)
3736 if mode is not None and os_helper.can_chmod() and os.name != 'nt':
3737 got = stat.filemode(stat.S_IMODE(path.stat().st_mode))
3738 self.assertEqual(got, mode)
3739 if type is None and isinstance(name, str) and name.endswith('/'):
3740 type = tarfile.DIRTYPE
3741 if symlink_to is not None:
3742 got = (self.destdir / name).readlink()
3743 expected = pathlib.Path(symlink_to)
3744 # The symlink might be the same (textually) as what we expect,
3745 # but some systems change the link to an equivalent path, so
3746 # we fall back to samefile().
3747 try:
3748 if expected != got:
3749 self.assertTrue(got.samefile(expected))
3750 except Exception as e:
3751 # attach a note, so it's shown even if `samefile` fails
3752 e.add_note(f'{expected=}, {got=}')
3753 raise
3754 elif type == tarfile.REGTYPE or type is None:
3755 self.assertTrue(path.is_file())
3756 elif type == tarfile.DIRTYPE:
3757 self.assertTrue(path.is_dir())
3758 elif type == tarfile.FIFOTYPE:
3759 self.assertTrue(path.is_fifo())
3760 elif type == tarfile.SYMTYPE:
3761 self.assertTrue(path.is_symlink())
3762 else:
3763 raise NotImplementedError(type)
3764 if size is not None:
3765 self.assertEqual(path.stat().st_size, size)
3766 if content is not None:
3767 self.assertEqual(path.read_text(), content)
3768 for parent in path.parents:
3769 self.expected_paths.discard(parent)
3770
3771 def expect_any_tree(self, name):
3772 """Check a directory; forget about its contents."""
3773 tree_path = (self.destdir / name).resolve()
3774 self.expect_file(tree_path, type=tarfile.DIRTYPE)
3775 self.expected_paths = {
3776 p for p in self.expected_paths
3777 if tree_path not in p.parents
3778 }
3779
3780 def expect_exception(self, exc_type, message_re='.'):
3781 with self.assertRaisesRegex(exc_type, message_re):
3782 if self.raised_exception is not None:
3783 raise self.raised_exception
3784 self.reraise_exception = False
3785 return self.raised_exception
3786
3787 def test_benign_file(self):
3788 with ArchiveMaker() as arc:
3789 arc.add('benign.txt')
3790 for filter in 'fully_trusted', 'tar', 'data':
3791 with self.check_context(arc.open(), filter):
3792 self.expect_file('benign.txt')
3793
3794 def test_absolute(self):
3795 # Test handling a member with an absolute path
3796 # Inspired by 'absolute1' in https://github.com/jwilk/traversal-archives
3797 with ArchiveMaker() as arc:
3798 arc.add(self.outerdir / 'escaped.evil')
3799
3800 with self.check_context(arc.open(), 'fully_trusted'):
3801 self.expect_file('../escaped.evil')
3802
3803 for filter in 'tar', 'data':
3804 with self.check_context(arc.open(), filter):
3805 if str(self.outerdir).startswith('/'):
3806 # We strip leading slashes, as e.g. GNU tar does
3807 # (without --absolute-filenames).
3808 outerdir_stripped = str(self.outerdir).lstrip('/')
3809 self.expect_file(f'{outerdir_stripped}/escaped.evil')
3810 else:
3811 # On this system, absolute paths don't have leading
3812 # slashes.
3813 # So, there's nothing to strip. We refuse to unpack
3814 # to an absolute path, nonetheless.
3815 self.expect_exception(
3816 tarfile.AbsolutePathError,
3817 """['"].*escaped.evil['"] has an absolute path""")
3818
3819 @symlink_test
3820 def test_parent_symlink(self):
3821 # Test interplaying symlinks
3822 # Inspired by 'dirsymlink2a' in jwilk/traversal-archives
3823 with ArchiveMaker() as arc:
3824
3825 # `current` links to `.` which is both:
3826 # - the destination directory
3827 # - `current` itself
3828 arc.add('current', symlink_to='.')
3829
3830 # effectively points to ./../
3831 arc.add('parent', symlink_to='current/..')
3832
3833 arc.add('parent/evil')
3834
3835 if os_helper.can_symlink():
3836 with self.check_context(arc.open(), 'fully_trusted'):
3837 if self.raised_exception is not None:
3838 # Windows will refuse to create a file that's a symlink to itself
3839 # (and tarfile doesn't swallow that exception)
3840 self.expect_exception(FileExistsError)
3841 # The other cases will fail with this error too.
3842 # Skip the rest of this test.
3843 return
3844 else:
3845 self.expect_file('current', symlink_to='.')
3846 self.expect_file('parent', symlink_to='current/..')
3847 self.expect_file('../evil')
3848
3849 with self.check_context(arc.open(), 'tar'):
3850 self.expect_exception(
3851 tarfile.OutsideDestinationError,
3852 """'parent/evil' would be extracted to ['"].*evil['"], """
3853 + "which is outside the destination")
3854
3855 with self.check_context(arc.open(), 'data'):
3856 self.expect_exception(
3857 tarfile.LinkOutsideDestinationError,
3858 """'parent' would link to ['"].*outerdir['"], """
3859 + "which is outside the destination")
3860
3861 else:
3862 # No symlink support. The symlinks are ignored.
3863 with self.check_context(arc.open(), 'fully_trusted'):
3864 self.expect_file('parent/evil')
3865 with self.check_context(arc.open(), 'tar'):
3866 self.expect_file('parent/evil')
3867 with self.check_context(arc.open(), 'data'):
3868 self.expect_file('parent/evil')
3869
3870 @symlink_test
3871 @os_helper.skip_unless_symlink
3872 def test_realpath_limit_attack(self):
3873 # (CVE-2025-4517)
3874
3875 with ArchiveMaker() as arc:
3876 # populate the symlinks and dirs that expand in os.path.realpath()
3877 # The component length is chosen so that in common cases, the unexpanded
3878 # path fits in PATH_MAX, but it overflows when the final symlink
3879 # is expanded
3880 steps = "abcdefghijklmnop"
3881 if sys.platform == 'win32':
3882 component = 'd' * 25
3883 elif 'PC_PATH_MAX' in os.pathconf_names:
3884 max_path_len = os.pathconf(self.outerdir.parent, "PC_PATH_MAX")
3885 path_sep_len = 1
3886 dest_len = len(str(self.destdir)) + path_sep_len
3887 component_len = (max_path_len - dest_len) // (len(steps) + path_sep_len)
3888 component = 'd' * component_len
3889 else:
3890 raise NotImplementedError("Need to guess component length for {sys.platform}")
3891 path = ""
3892 step_path = ""
3893 for i in steps:
3894 arc.add(os.path.join(path, component), type=tarfile.DIRTYPE,
3895 mode='drwxrwxrwx')
3896 arc.add(os.path.join(path, i), symlink_to=component)
3897 path = os.path.join(path, component)
3898 step_path = os.path.join(step_path, i)
3899 # create the final symlink that exceeds PATH_MAX and simply points
3900 # to the top dir.
3901 # this link will never be expanded by
3902 # os.path.realpath(strict=False), nor anything after it.
3903 linkpath = os.path.join(*steps, "l"*254)
3904 parent_segments = [".."] * len(steps)
3905 arc.add(linkpath, symlink_to=os.path.join(*parent_segments))
3906 # make a symlink outside to keep the tar command happy
3907 arc.add("escape", symlink_to=os.path.join(linkpath, ".."))
3908 # use the symlinks above, that are not checked, to create a hardlink
3909 # to a file outside of the destination path
3910 arc.add("flaglink", hardlink_to=os.path.join("escape", "flag"))
3911 # now that we have the hardlink we can overwrite the file
3912 arc.add("flaglink", content='overwrite')
3913 # we can also create new files as well!
3914 arc.add("escape/newfile", content='new')
3915
3916 with (self.subTest('fully_trusted'),
3917 self.check_context(arc.open(), filter='fully_trusted',
3918 check_flag=False)):
3919 if sys.platform == 'win32':
3920 self.expect_exception((FileNotFoundError, FileExistsError))
3921 elif self.raised_exception:
3922 # Cannot symlink/hardlink: tarfile falls back to getmember()
3923 self.expect_exception(KeyError)
3924 # Otherwise, this block should never enter.
3925 else:
3926 self.expect_any_tree(component)
3927 self.expect_file('flaglink', content='overwrite')
3928 self.expect_file('../newfile', content='new')
3929 self.expect_file('escape', type=tarfile.SYMTYPE)
3930 self.expect_file('a', symlink_to=component)
3931
3932 for filter in 'tar', 'data':
3933 with self.subTest(filter), self.check_context(arc.open(), filter=filter):
3934 exc = self.expect_exception((OSError, KeyError))
3935 if isinstance(exc, OSError):
3936 if sys.platform == 'win32':
3937 # 3: ERROR_PATH_NOT_FOUND
3938 # 5: ERROR_ACCESS_DENIED
3939 # 206: ERROR_FILENAME_EXCED_RANGE
3940 self.assertIn(exc.winerror, (3, 5, 206))
3941 else:
3942 self.assertEqual(exc.errno, errno.ENAMETOOLONG)
3943
3944 @symlink_test
3945 def test_parent_symlink2(self):
3946 # Test interplaying symlinks
3947 # Inspired by 'dirsymlink2b' in jwilk/traversal-archives
3948
3949 # Posix and Windows have different pathname resolution:
3950 # either symlink or a '..' component resolve first.
3951 # Let's see which we are on.
3952 if os_helper.can_symlink():
3953 testpath = os.path.join(TEMPDIR, 'resolution_test')
3954 os.mkdir(testpath)
3955
3956 # testpath/current links to `.` which is all of:
3957 # - `testpath`
3958 # - `testpath/current`
3959 # - `testpath/current/current`
3960 # - etc.
3961 os.symlink('.', os.path.join(testpath, 'current'))
3962
3963 # we'll test where `testpath/current/../file` ends up
3964 with open(os.path.join(testpath, 'current', '..', 'file'), 'w'):
3965 pass
3966
3967 if os.path.exists(os.path.join(testpath, 'file')):
3968 # Windows collapses 'current\..' to '.' first, leaving
3969 # 'testpath\file'
3970 dotdot_resolves_early = True
3971 elif os.path.exists(os.path.join(testpath, '..', 'file')):
3972 # Posix resolves 'current' to '.' first, leaving
3973 # 'testpath/../file'
3974 dotdot_resolves_early = False
3975 else:
3976 raise AssertionError('Could not determine link resolution')
3977
3978 with ArchiveMaker() as arc:
3979
3980 # `current` links to `.` which is both the destination directory
3981 # and `current` itself
3982 arc.add('current', symlink_to='.')
3983
3984 # `current/parent` is also available as `./parent`,
3985 # and effectively points to `./../`
3986 arc.add('current/parent', symlink_to='..')
3987
3988 arc.add('parent/evil')
3989
3990 with self.check_context(arc.open(), 'fully_trusted'):
3991 if os_helper.can_symlink():
3992 self.expect_file('current', symlink_to='.')
3993 self.expect_file('parent', symlink_to='..')
3994 self.expect_file('../evil')
3995 else:
3996 self.expect_file('current/')
3997 self.expect_file('parent/evil')
3998
3999 with self.check_context(arc.open(), 'tar'):
4000 if os_helper.can_symlink():
4001 # Fail when extracting a file outside destination
4002 self.expect_exception(
4003 tarfile.OutsideDestinationError,
4004 "'parent/evil' would be extracted to "
4005 + """['"].*evil['"], which is outside """
4006 + "the destination")
4007 else:
4008 self.expect_file('current/')
4009 self.expect_file('parent/evil')
4010
4011 with self.check_context(arc.open(), 'data'):
4012 if os_helper.can_symlink():
4013 if dotdot_resolves_early:
4014 # Fail when extracting a file outside destination
4015 self.expect_exception(
4016 tarfile.OutsideDestinationError,
4017 "'parent/evil' would be extracted to "
4018 + """['"].*evil['"], which is outside """
4019 + "the destination")
4020 else:
4021 # Fail as soon as we have a symlink outside the destination
4022 self.expect_exception(
4023 tarfile.LinkOutsideDestinationError,
4024 "'current/parent' would link to "
4025 + """['"].*outerdir['"], which is outside """
4026 + "the destination")
4027 else:
4028 self.expect_file('current/')
4029 self.expect_file('parent/evil')
4030
4031 @symlink_test
4032 def test_absolute_symlink(self):
4033 # Test symlink to an absolute path
4034 # Inspired by 'dirsymlink' in jwilk/traversal-archives
4035 with ArchiveMaker() as arc:
4036 arc.add('parent', symlink_to=self.outerdir)
4037 arc.add('parent/evil')
4038
4039 with self.check_context(arc.open(), 'fully_trusted'):
4040 if os_helper.can_symlink():
4041 self.expect_file('parent', symlink_to=self.outerdir)
4042 self.expect_file('../evil')
4043 else:
4044 self.expect_file('parent/evil')
4045
4046 with self.check_context(arc.open(), 'tar'):
4047 if os_helper.can_symlink():
4048 self.expect_exception(
4049 tarfile.OutsideDestinationError,
4050 "'parent/evil' would be extracted to "
4051 + """['"].*evil['"], which is outside """
4052 + "the destination")
4053 else:
4054 self.expect_file('parent/evil')
4055
4056 with self.check_context(arc.open(), 'data'):
4057 self.expect_exception(
4058 tarfile.AbsoluteLinkError,
4059 "'parent' is a link to an absolute path")
4060
4061 def test_absolute_hardlink(self):
4062 # Test hardlink to an absolute path
4063 # Inspired by 'dirsymlink' in https://github.com/jwilk/traversal-archives
4064 with ArchiveMaker() as arc:
4065 arc.add('parent', hardlink_to=self.outerdir / 'foo')
4066
4067 with self.check_context(arc.open(), 'fully_trusted'):
4068 self.expect_exception(KeyError, ".*foo. not found")
4069
4070 with self.check_context(arc.open(), 'tar'):
4071 self.expect_exception(KeyError, ".*foo. not found")
4072
4073 with self.check_context(arc.open(), 'data'):
4074 self.expect_exception(
4075 tarfile.AbsoluteLinkError,
4076 "'parent' is a link to an absolute path")
4077
4078 @symlink_test
4079 def test_sly_relative0(self):
4080 # Inspired by 'relative0' in jwilk/traversal-archives
4081 with ArchiveMaker() as arc:
4082 # points to `../../tmp/moo`
4083 arc.add('../moo', symlink_to='..//tmp/moo')
4084
4085 try:
4086 with self.check_context(arc.open(), filter='fully_trusted'):
4087 if os_helper.can_symlink():
4088 if isinstance(self.raised_exception, FileExistsError):
4089 # XXX TarFile happens to fail creating a parent
4090 # directory.
4091 # This might be a bug, but fixing it would hurt
4092 # security.
4093 # Note that e.g. GNU `tar` rejects '..' components,
4094 # so you could argue this is an invalid archive and we
4095 # just raise an bad type of exception.
4096 self.expect_exception(FileExistsError)
4097 else:
4098 self.expect_file('../moo', symlink_to='..//tmp/moo')
4099 else:
4100 # The symlink can't be extracted and is ignored
4101 pass
4102 except FileExistsError:
4103 pass
4104
4105 for filter in 'tar', 'data':
4106 with self.check_context(arc.open(), filter):
4107 self.expect_exception(
4108 tarfile.OutsideDestinationError,
4109 "'../moo' would be extracted to "
4110 + "'.*moo', which is outside "
4111 + "the destination")
4112
4113 @symlink_test
4114 def test_sly_relative2(self):
4115 # Inspired by 'relative2' in jwilk/traversal-archives
4116 with ArchiveMaker() as arc:
4117 arc.add('tmp/')
4118 arc.add('tmp/../../moo', symlink_to='tmp/../..//tmp/moo')
4119
4120 with self.check_context(arc.open(), 'fully_trusted'):
4121 self.expect_file('tmp', type=tarfile.DIRTYPE)
4122 if os_helper.can_symlink():
4123 self.expect_file('../moo', symlink_to='tmp/../../tmp/moo')
4124
4125 for filter in 'tar', 'data':
4126 with self.check_context(arc.open(), filter):
4127 self.expect_exception(
4128 tarfile.OutsideDestinationError,
4129 "'tmp/../../moo' would be extracted to "
4130 + """['"].*moo['"], which is outside the """
4131 + "destination")
4132
4133 @symlink_test
4134 def test_deep_symlink(self):
4135 # Test that symlinks and hardlinks inside a directory
4136 # point to the correct file (`target` of size 3).
4137 # If links aren't supported we get a copy of the file.
4138 with ArchiveMaker() as arc:
4139 arc.add('targetdir/target', size=3)
4140 # a hardlink's linkname is relative to the archive
4141 arc.add('linkdir/hardlink', hardlink_to=os.path.join(
4142 'targetdir', 'target'))
4143 # a symlink's linkname is relative to the link's directory
4144 arc.add('linkdir/symlink', symlink_to=os.path.join(
4145 '..', 'targetdir', 'target'))
4146
4147 for filter in 'tar', 'data', 'fully_trusted':
4148 with self.check_context(arc.open(), filter):
4149 self.expect_file('targetdir/target', size=3)
4150 self.expect_file('linkdir/hardlink', size=3)
4151 if os_helper.can_symlink():
4152 self.expect_file('linkdir/symlink', size=3,
4153 symlink_to='../targetdir/target')
4154 else:
4155 self.expect_file('linkdir/symlink', size=3)
4156
4157 @symlink_test
4158 def test_chains(self):
4159 # Test chaining of symlinks/hardlinks.
4160 # Symlinks are created before the files they point to.
4161 with ArchiveMaker() as arc:
4162 arc.add('linkdir/symlink', symlink_to='hardlink')
4163 arc.add('symlink2', symlink_to=os.path.join(
4164 'linkdir', 'hardlink2'))
4165 arc.add('targetdir/target', size=3)
4166 arc.add('linkdir/hardlink', hardlink_to=os.path.join('targetdir', 'target'))
4167 arc.add('linkdir/hardlink2', hardlink_to=os.path.join('linkdir', 'symlink'))
4168
4169 for filter in 'tar', 'data', 'fully_trusted':
4170 with self.check_context(arc.open(), filter):
4171 self.expect_file('targetdir/target', size=3)
4172 self.expect_file('linkdir/hardlink', size=3)
4173 self.expect_file('linkdir/hardlink2', size=3)
4174 if os_helper.can_symlink():
4175 self.expect_file('linkdir/symlink', size=3,
4176 symlink_to='hardlink')
4177 self.expect_file('symlink2', size=3,
4178 symlink_to='linkdir/hardlink2')
4179 else:
4180 self.expect_file('linkdir/symlink', size=3)
4181 self.expect_file('symlink2', size=3)
4182
4183 @symlink_test
4184 def test_sneaky_hardlink_fallback(self):
4185 # (CVE-2025-4330)
4186 # Test that when hardlink extraction falls back to extracting members
4187 # from the archive, the extracted member is (re-)filtered.
4188 with ArchiveMaker() as arc:
4189 # Create a directory structure so the c/escape symlink stays
4190 # inside the path
4191 arc.add("a/t/dummy")
4192 # Create b/ directory
4193 arc.add("b/")
4194 # Point "c" to the bottom of the tree in "a"
4195 arc.add("c", symlink_to=os.path.join("a", "t"))
4196 # link to non-existant location under "a"
4197 arc.add("c/escape", symlink_to=os.path.join("..", "..",
4198 "link_here"))
4199 # Move "c" to point to "b" ("c/escape" no longer exists)
4200 arc.add("c", symlink_to="b")
4201 # Attempt to create a hard link to "c/escape". Since it doesn't
4202 # exist it will attempt to extract "cescape" but at "boom".
4203 arc.add("boom", hardlink_to=os.path.join("c", "escape"))
4204
4205 with self.check_context(arc.open(), 'data'):
4206 if not os_helper.can_symlink():
4207 # When 'c/escape' is extracted, 'c' is a regular
4208 # directory, and 'c/escape' *would* point outside
4209 # the destination if symlinks were allowed.
4210 self.expect_exception(
4211 tarfile.LinkOutsideDestinationError)
4212 elif sys.platform == "win32":
4213 # On Windows, 'c/escape' points outside the destination
4214 self.expect_exception(tarfile.LinkOutsideDestinationError)
4215 else:
4216 e = self.expect_exception(
4217 tarfile.LinkFallbackError,
4218 "link 'boom' would be extracted as a copy of "
4219 + "'c/escape', which was rejected")
4220 self.assertIsInstance(e.__cause__,
4221 tarfile.LinkOutsideDestinationError)
4222 for filter in 'tar', 'fully_trusted':
4223 with self.subTest(filter), self.check_context(arc.open(), filter):
4224 if not os_helper.can_symlink():
4225 self.expect_file("a/t/dummy")
4226 self.expect_file("b/")
4227 self.expect_file("c/")
4228 else:
4229 self.expect_file("a/t/dummy")
4230 self.expect_file("b/")
4231 self.expect_file("a/t/escape", symlink_to='../../link_here')
4232 self.expect_file("boom", symlink_to='../../link_here')
4233 self.expect_file("c", symlink_to='b')
4234
4235 @symlink_test
4236 def test_exfiltration_via_symlink(self):
4237 # (CVE-2025-4138)
4238 # Test changing symlinks that result in a symlink pointing outside
4239 # the extraction directory, unless prevented by 'data' filter's
4240 # normalization.
4241 with ArchiveMaker() as arc:
4242 arc.add("escape", symlink_to=os.path.join('link', 'link', '..', '..', 'link-here'))
4243 arc.add("link", symlink_to='./')
4244
4245 for filter in 'tar', 'data', 'fully_trusted':
4246 with self.check_context(arc.open(), filter):
4247 if os_helper.can_symlink():
4248 self.expect_file("link", symlink_to='./')
4249 if filter == 'data':
4250 self.expect_file("escape", symlink_to='link-here')
4251 else:
4252 self.expect_file("escape",
4253 symlink_to='link/link/../../link-here')
4254 else:
4255 # Nothing is extracted.
4256 pass
4257
4258 @symlink_test
4259 def test_chmod_outside_dir(self):
4260 # (CVE-2024-12718)
4261 # Test that members used for delayed updates of directory metadata
4262 # are (re-)filtered.
4263 with ArchiveMaker() as arc:
4264 # "pwn" is a veeeery innocent symlink:
4265 arc.add("a/pwn", symlink_to='.')
4266 # But now "pwn" is also a directory, so it's scheduled to have its
4267 # metadata updated later:
4268 arc.add("a/pwn/", mode='drwxrwxrwx')
4269 # Oops, "pwn" is not so innocent any more:
4270 arc.add("a/pwn", symlink_to='x/../')
4271 # Newly created symlink points to the dest dir,
4272 # so it's OK for the "data" filter.
4273 arc.add('a/x', symlink_to=('../'))
4274 # But now "pwn" points outside the dest dir
4275
4276 for filter in 'tar', 'data', 'fully_trusted':
4277 with self.check_context(arc.open(), filter) as cc:
4278 if not os_helper.can_symlink():
4279 self.expect_file("a/pwn/")
4280 elif filter == 'data':
4281 self.expect_file("a/x", symlink_to='../')
4282 self.expect_file("a/pwn", symlink_to='.')
4283 else:
4284 self.expect_file("a/x", symlink_to='../')
4285 self.expect_file("a/pwn", symlink_to='x/../')
4286 if sys.platform != "win32":
4287 st_mode = cc.outerdir.stat().st_mode
4288 self.assertNotEqual(st_mode & 0o777, 0o777)
4289
4290 def test_link_fallback_normalizes(self):
4291 # Make sure hardlink fallbacks work for non-normalized paths for all
4292 # filters
4293 with ArchiveMaker() as arc:
4294 arc.add("dir/")
4295 arc.add("dir/../afile")
4296 arc.add("link1", hardlink_to='dir/../afile')
4297 arc.add("link2", hardlink_to='dir/../dir/../afile')
4298
4299 for filter in 'tar', 'data', 'fully_trusted':
4300 with self.check_context(arc.open(), filter) as cc:
4301 self.expect_file("dir/")
4302 self.expect_file("afile")
4303 self.expect_file("link1")
4304 self.expect_file("link2")
4305
4306 def test_modes(self):
4307 # Test how file modes are extracted
4308 # (Note that the modes are ignored on platforms without working chmod)
4309 with ArchiveMaker() as arc:
4310 arc.add('all_bits', mode='?rwsrwsrwt')
4311 arc.add('perm_bits', mode='?rwxrwxrwx')
4312 arc.add('exec_group_other', mode='?rw-rwxrwx')
4313 arc.add('read_group_only', mode='?---r-----')
4314 arc.add('no_bits', mode='?---------')
4315 arc.add('dir/', mode='?---rwsrwt')
4316 arc.add('dir_all_bits/', mode='?rwsrwsrwt')
4317
4318 # On some systems, setting the uid, gid, and/or sticky bit is a no-ops.
4319 # Check which bits we can set, so we can compare tarfile machinery to
4320 # a simple chmod.
4321 tmp_filename = os.path.join(TEMPDIR, "tmp.file")
4322 with open(tmp_filename, 'w'):
4323 pass
4324 try:
4325 new_mode = (os.stat(tmp_filename).st_mode
4326 | stat.S_ISVTX | stat.S_ISGID | stat.S_ISUID)
4327 try:
4328 os.chmod(tmp_filename, new_mode)
4329 except OSError as exc:
4330 if exc.errno == getattr(errno, "EFTYPE", 0):
4331 # gh-108948: On FreeBSD, regular users cannot set
4332 # the sticky bit.
4333 self.skipTest("chmod() failed with EFTYPE: "
4334 "regular users cannot set sticky bit")
4335 else:
4336 raise
4337
4338 got_mode = os.stat(tmp_filename).st_mode
4339 _t_file = 't' if (got_mode & stat.S_ISVTX) else 'x'
4340 _suid_file = 's' if (got_mode & stat.S_ISUID) else 'x'
4341 _sgid_file = 's' if (got_mode & stat.S_ISGID) else 'x'
4342 finally:
4343 os.unlink(tmp_filename)
4344
4345 os.mkdir(tmp_filename)
4346 new_mode = (os.stat(tmp_filename).st_mode
4347 | stat.S_ISVTX | stat.S_ISGID | stat.S_ISUID)
4348 os.chmod(tmp_filename, new_mode)
4349 got_mode = os.stat(tmp_filename).st_mode
4350 _t_dir = 't' if (got_mode & stat.S_ISVTX) else 'x'
4351 _suid_dir = 's' if (got_mode & stat.S_ISUID) else 'x'
4352 _sgid_dir = 's' if (got_mode & stat.S_ISGID) else 'x'
4353 os.rmdir(tmp_filename)
4354
4355 with self.check_context(arc.open(), 'fully_trusted'):
4356 self.expect_file('all_bits',
4357 mode=f'?rw{_suid_file}rw{_sgid_file}rw{_t_file}')
4358 self.expect_file('perm_bits', mode='?rwxrwxrwx')
4359 self.expect_file('exec_group_other', mode='?rw-rwxrwx')
4360 self.expect_file('read_group_only', mode='?---r-----')
4361 self.expect_file('no_bits', mode='?---------')
4362 self.expect_file('dir/', mode=f'?---rw{_sgid_dir}rw{_t_dir}')
4363 self.expect_file('dir_all_bits/',
4364 mode=f'?rw{_suid_dir}rw{_sgid_dir}rw{_t_dir}')
4365
4366 with self.check_context(arc.open(), 'tar'):
4367 self.expect_file('all_bits', mode='?rwxr-xr-x')
4368 self.expect_file('perm_bits', mode='?rwxr-xr-x')
4369 self.expect_file('exec_group_other', mode='?rw-r-xr-x')
4370 self.expect_file('read_group_only', mode='?---r-----')
4371 self.expect_file('no_bits', mode='?---------')
4372 self.expect_file('dir/', mode='?---r-xr-x')
4373 self.expect_file('dir_all_bits/', mode='?rwxr-xr-x')
4374
4375 with self.check_context(arc.open(), 'data'):
4376 normal_dir_mode = stat.filemode(stat.S_IMODE(
4377 self.outerdir.stat().st_mode))
4378 self.expect_file('all_bits', mode='?rwxr-xr-x')
4379 self.expect_file('perm_bits', mode='?rwxr-xr-x')
4380 self.expect_file('exec_group_other', mode='?rw-r--r--')
4381 self.expect_file('read_group_only', mode='?rw-r-----')
4382 self.expect_file('no_bits', mode='?rw-------')
4383 self.expect_file('dir/', mode=normal_dir_mode)
4384 self.expect_file('dir_all_bits/', mode=normal_dir_mode)
4385
4386 def test_pipe(self):
4387 # Test handling of a special file
4388 with ArchiveMaker() as arc:
4389 arc.add('foo', type=tarfile.FIFOTYPE)
4390
4391 for filter in 'fully_trusted', 'tar':
4392 with self.check_context(arc.open(), filter):
4393 if hasattr(os, 'mkfifo'):
4394 self.expect_file('foo', type=tarfile.FIFOTYPE)
4395 else:
4396 # The pipe can't be extracted and is skipped.
4397 pass
4398
4399 with self.check_context(arc.open(), 'data'):
4400 self.expect_exception(
4401 tarfile.SpecialFileError,
4402 "'foo' is a special file")
4403
4404 def test_special_files(self):
4405 # Creating device files is tricky. Instead of attempting that let's
4406 # only check the filter result.
4407 for special_type in tarfile.FIFOTYPE, tarfile.CHRTYPE, tarfile.BLKTYPE:
4408 tarinfo = tarfile.TarInfo('foo')
4409 tarinfo.type = special_type
4410 trusted = tarfile.fully_trusted_filter(tarinfo, '')
4411 self.assertIs(trusted, tarinfo)
4412 tar = tarfile.tar_filter(tarinfo, '')
4413 self.assertEqual(tar.type, special_type)
4414 with self.assertRaises(tarfile.SpecialFileError) as cm:
4415 tarfile.data_filter(tarinfo, '')
4416 self.assertIsInstance(cm.exception.tarinfo, tarfile.TarInfo)
4417 self.assertEqual(cm.exception.tarinfo.name, 'foo')
4418
4419 def test_fully_trusted_filter(self):
4420 # The 'fully_trusted' filter returns the original TarInfo objects.
4421 with tarfile.TarFile.open(tarname) as tar:
4422 for tarinfo in tar.getmembers():
4423 filtered = tarfile.fully_trusted_filter(tarinfo, '')
4424 self.assertIs(filtered, tarinfo)
4425
4426 def test_tar_filter(self):
4427 # The 'tar' filter returns TarInfo objects with the same name/type.
4428 # (It can also fail for particularly "evil" input, but we don't have
4429 # that in the test archive.)
4430 with tarfile.TarFile.open(tarname, encoding="iso8859-1") as tar:
4431 for tarinfo in tar.getmembers():
4432 try:
4433 filtered = tarfile.tar_filter(tarinfo, '')
4434 except UnicodeEncodeError:
4435 continue
4436 self.assertIs(filtered.name, tarinfo.name)
4437 self.assertIs(filtered.type, tarinfo.type)
4438
4439 def test_data_filter(self):
4440 # The 'data' filter either raises, or returns TarInfo with the same
4441 # name/type.
4442 with tarfile.TarFile.open(tarname, encoding="iso8859-1") as tar:
4443 for tarinfo in tar.getmembers():
4444 try:
4445 filtered = tarfile.data_filter(tarinfo, '')
4446 except (tarfile.FilterError, UnicodeEncodeError):
4447 continue
4448 self.assertIs(filtered.name, tarinfo.name)
4449 self.assertIs(filtered.type, tarinfo.type)
4450
4451 @unittest.skipIf(sys.platform == 'win32', 'requires native bytes paths')
4452 def test_filter_unencodable(self):
4453 # Sanity check using a valid path.
4454 tarinfo = tarfile.TarInfo(os_helper.TESTFN)
4455 filtered = tarfile.tar_filter(tarinfo, '')
4456 self.assertIs(filtered.name, tarinfo.name)
4457 filtered = tarfile.data_filter(tarinfo, '')
4458 self.assertIs(filtered.name, tarinfo.name)
4459
4460 tarinfo = tarfile.TarInfo('test\x00')
4461 self.assertRaises(ValueError, tarfile.tar_filter, tarinfo, '')
4462 self.assertRaises(ValueError, tarfile.data_filter, tarinfo, '')
4463 tarinfo = tarfile.TarInfo('\ud800')
4464 self.assertRaises(UnicodeEncodeError, tarfile.tar_filter, tarinfo, '')
4465 self.assertRaises(UnicodeEncodeError, tarfile.data_filter, tarinfo, '')
4466
4467 @unittest.skipIf(sys.platform == 'win32', 'requires native bytes paths')
4468 def test_extract_unencodable(self):
4469 # Create a member with name \xed\xa0\x80 which is UTF-8 encoded
4470 # lone surrogate \ud800.
4471 with ArchiveMaker(encoding='ascii', errors='surrogateescape') as arc:
4472 arc.add('\udced\udca0\udc80')
4473 with os_helper.temp_cwd() as tmp:
4474 tar = arc.open(encoding='utf-8', errors='surrogatepass',
4475 errorlevel=1)
4476 self.assertEqual(tar.getnames(), ['\ud800'])
4477 with self.assertRaises(UnicodeEncodeError):
4478 tar.extractall()
4479 self.assertEqual(os.listdir(), [])
4480
4481 tar = arc.open(encoding='utf-8', errors='surrogatepass',
4482 errorlevel=0, debug=1)
4483 with support.captured_stderr() as stderr:
4484 tar.extractall()
4485 self.assertEqual(os.listdir(), [])
4486 self.assertIn('tarfile: UnicodeEncodeError ', stderr.getvalue())
4487
4488 def test_change_default_filter_on_instance(self):
4489 tar = tarfile.TarFile(tarname, 'r')
4490 def strict_filter(tarinfo, path):
4491 if tarinfo.name == 'ustar/regtype':
4492 return tarinfo
4493 else:
4494 return None
4495 tar.extraction_filter = strict_filter
4496 with self.check_context(tar, None):
4497 self.expect_file('ustar/regtype')
4498
4499 def test_change_default_filter_on_class(self):
4500 def strict_filter(tarinfo, path):
4501 if tarinfo.name == 'ustar/regtype':
4502 return tarinfo
4503 else:
4504 return None
4505 tar = tarfile.TarFile(tarname, 'r')
4506 with support.swap_attr(tarfile.TarFile, 'extraction_filter',
4507 staticmethod(strict_filter)):
4508 with self.check_context(tar, None):
4509 self.expect_file('ustar/regtype')
4510
4511 def test_change_default_filter_on_subclass(self):
4512 class TarSubclass(tarfile.TarFile):
4513 def extraction_filter(self, tarinfo, path):
4514 if tarinfo.name == 'ustar/regtype':
4515 return tarinfo
4516 else:
4517 return None
4518
4519 tar = TarSubclass(tarname, 'r')
4520 with self.check_context(tar, None):
4521 self.expect_file('ustar/regtype')
4522
4523 def test_change_default_filter_to_string(self):
4524 tar = tarfile.TarFile(tarname, 'r')
4525 tar.extraction_filter = 'data'
4526 with self.check_context(tar, None):
4527 self.expect_exception(TypeError)
4528
4529 def test_custom_filter(self):
4530 def custom_filter(tarinfo, path):
4531 self.assertIs(path, self.destdir)
4532 if tarinfo.name == 'move_this':
4533 return tarinfo.replace(name='moved')
4534 if tarinfo.name == 'ignore_this':
4535 return None
4536 return tarinfo
4537
4538 with ArchiveMaker() as arc:
4539 arc.add('move_this')
4540 arc.add('ignore_this')
4541 arc.add('keep')
4542 with self.check_context(arc.open(), custom_filter):
4543 self.expect_file('moved')
4544 self.expect_file('keep')
4545
4546 def test_bad_filter_name(self):
4547 with ArchiveMaker() as arc:
4548 arc.add('foo')
4549 with self.check_context(arc.open(), 'bad filter name'):
4550 self.expect_exception(ValueError)
4551
4552 def test_stateful_filter(self):
4553 # Stateful filters should be possible.
4554 # (This doesn't really test tarfile. Rather, it demonstrates
4555 # that third parties can implement a stateful filter.)
4556 class StatefulFilter:
4557 def __enter__(self):
4558 self.num_files_processed = 0
4559 return self
4560
4561 def __call__(self, tarinfo, path):
4562 try:
4563 tarinfo = tarfile.data_filter(tarinfo, path)
4564 except tarfile.FilterError:
4565 return None
4566 self.num_files_processed += 1
4567 return tarinfo
4568
4569 def __exit__(self, *exc_info):
4570 self.done = True
4571
4572 with ArchiveMaker() as arc:
4573 arc.add('good')
4574 arc.add('bad', symlink_to='/')
4575 arc.add('good')
4576 with StatefulFilter() as custom_filter:
4577 with self.check_context(arc.open(), custom_filter):
4578 self.expect_file('good')
4579 self.assertEqual(custom_filter.num_files_processed, 2)
4580 self.assertEqual(custom_filter.done, True)
4581
4582 def test_errorlevel(self):
4583 def extracterror_filter(tarinfo, path):
4584 raise tarfile.ExtractError('failed with ExtractError')
4585 def filtererror_filter(tarinfo, path):
4586 raise tarfile.FilterError('failed with FilterError')
4587 def oserror_filter(tarinfo, path):
4588 raise OSError('failed with OSError')
4589 def tarerror_filter(tarinfo, path):
4590 raise tarfile.TarError('failed with base TarError')
4591 def valueerror_filter(tarinfo, path):
4592 raise ValueError('failed with ValueError')
4593
4594 with ArchiveMaker() as arc:
4595 arc.add('file')
4596
4597 # If errorlevel is 0, errors affected by errorlevel are ignored
4598
4599 with self.check_context(arc.open(errorlevel=0), extracterror_filter):
4600 pass
4601
4602 with self.check_context(arc.open(errorlevel=0), filtererror_filter):
4603 pass
4604
4605 with self.check_context(arc.open(errorlevel=0), oserror_filter):
4606 pass
4607
4608 with self.check_context(arc.open(errorlevel=0), tarerror_filter):
4609 self.expect_exception(tarfile.TarError)
4610
4611 with self.check_context(arc.open(errorlevel=0), valueerror_filter):
4612 self.expect_exception(ValueError)
4613
4614 # If 1, all fatal errors are raised
4615
4616 with self.check_context(arc.open(errorlevel=1), extracterror_filter):
4617 pass
4618
4619 with self.check_context(arc.open(errorlevel=1), filtererror_filter):
4620 self.expect_exception(tarfile.FilterError)
4621
4622 with self.check_context(arc.open(errorlevel=1), oserror_filter):
4623 self.expect_exception(OSError)
4624
4625 with self.check_context(arc.open(errorlevel=1), tarerror_filter):
4626 self.expect_exception(tarfile.TarError)
4627
4628 with self.check_context(arc.open(errorlevel=1), valueerror_filter):
4629 self.expect_exception(ValueError)
4630
4631 # If 2, all non-fatal errors are raised as well.
4632
4633 with self.check_context(arc.open(errorlevel=2), extracterror_filter):
4634 self.expect_exception(tarfile.ExtractError)
4635
4636 with self.check_context(arc.open(errorlevel=2), filtererror_filter):
4637 self.expect_exception(tarfile.FilterError)
4638
4639 with self.check_context(arc.open(errorlevel=2), oserror_filter):
4640 self.expect_exception(OSError)
4641
4642 with self.check_context(arc.open(errorlevel=2), tarerror_filter):
4643 self.expect_exception(tarfile.TarError)
4644
4645 with self.check_context(arc.open(errorlevel=2), valueerror_filter):
4646 self.expect_exception(ValueError)
4647
4648 # We only handle ExtractionError, FilterError & OSError specially.
4649
4650 with self.check_context(arc.open(errorlevel='boo!'), filtererror_filter):
4651 self.expect_exception(TypeError) # errorlevel is not int
4652
4653
4654 class OverwriteTests(archiver_tests.OverwriteTests, unittest.TestCase):
4655 testdir = os.path.join(TEMPDIR, "testoverwrite")
4656
4657 @classmethod
4658 def setUpClass(cls):
4659 p = cls.ar_with_file = os.path.join(TEMPDIR, 'tar-with-file.tar')
4660 cls.addClassCleanup(os_helper.unlink, p)
4661 with tarfile.open(p, 'w') as tar:
4662 t = tarfile.TarInfo('test')
4663 t.size = 10
4664 tar.addfile(t, io.BytesIO(b'newcontent'))
4665
4666 p = cls.ar_with_dir = os.path.join(TEMPDIR, 'tar-with-dir.tar')
4667 cls.addClassCleanup(os_helper.unlink, p)
4668 with tarfile.open(p, 'w') as tar:
4669 tar.addfile(tar.gettarinfo(os.curdir, 'test'))
4670
4671 p = os.path.join(TEMPDIR, 'tar-with-implicit-dir.tar')
4672 cls.ar_with_implicit_dir = p
4673 cls.addClassCleanup(os_helper.unlink, p)
4674 with tarfile.open(p, 'w') as tar:
4675 t = tarfile.TarInfo('test/file')
4676 t.size = 10
4677 tar.addfile(t, io.BytesIO(b'newcontent'))
4678
4679 def open(self, path):
4680 return tarfile.open(path, 'r')
4681
4682 def extractall(self, ar):
4683 ar.extractall(self.testdir, filter='fully_trusted')
4684
4685
4686 class OffsetValidationTests(unittest.TestCase):
4687 tarname = tmpname
4688 invalid_posix_header = (
4689 # name: 100 bytes
4690 tarfile.NUL * tarfile.LENGTH_NAME
4691 # mode, space, null terminator: 8 bytes
4692 + b"000755" + SPACE + tarfile.NUL
4693 # uid, space, null terminator: 8 bytes
4694 + b"000001" + SPACE + tarfile.NUL
4695 # gid, space, null terminator: 8 bytes
4696 + b"000001" + SPACE + tarfile.NUL
4697 # size, space: 12 bytes
4698 + b"\xff" * 11 + SPACE
4699 # mtime, space: 12 bytes
4700 + tarfile.NUL * 11 + SPACE
4701 # chksum: 8 bytes
4702 + b"0011407" + tarfile.NUL
4703 # type: 1 byte
4704 + tarfile.REGTYPE
4705 # linkname: 100 bytes
4706 + tarfile.NUL * tarfile.LENGTH_LINK
4707 # magic: 6 bytes, version: 2 bytes
4708 + tarfile.POSIX_MAGIC
4709 # uname: 32 bytes
4710 + tarfile.NUL * 32
4711 # gname: 32 bytes
4712 + tarfile.NUL * 32
4713 # devmajor, space, null terminator: 8 bytes
4714 + tarfile.NUL * 6 + SPACE + tarfile.NUL
4715 # devminor, space, null terminator: 8 bytes
4716 + tarfile.NUL * 6 + SPACE + tarfile.NUL
4717 # prefix: 155 bytes
4718 + tarfile.NUL * tarfile.LENGTH_PREFIX
4719 # padding: 12 bytes
4720 + tarfile.NUL * 12
4721 )
4722 invalid_gnu_header = (
4723 # name: 100 bytes
4724 tarfile.NUL * tarfile.LENGTH_NAME
4725 # mode, null terminator: 8 bytes
4726 + b"0000755" + tarfile.NUL
4727 # uid, null terminator: 8 bytes
4728 + b"0000001" + tarfile.NUL
4729 # gid, space, null terminator: 8 bytes
4730 + b"0000001" + tarfile.NUL
4731 # size, space: 12 bytes
4732 + b"\xff" * 11 + SPACE
4733 # mtime, space: 12 bytes
4734 + tarfile.NUL * 11 + SPACE
4735 # chksum: 8 bytes
4736 + b"0011327" + tarfile.NUL
4737 # type: 1 byte
4738 + tarfile.REGTYPE
4739 # linkname: 100 bytes
4740 + tarfile.NUL * tarfile.LENGTH_LINK
4741 # magic: 8 bytes
4742 + tarfile.GNU_MAGIC
4743 # uname: 32 bytes
4744 + tarfile.NUL * 32
4745 # gname: 32 bytes
4746 + tarfile.NUL * 32
4747 # devmajor, null terminator: 8 bytes
4748 + tarfile.NUL * 8
4749 # devminor, null terminator: 8 bytes
4750 + tarfile.NUL * 8
4751 # padding: 167 bytes
4752 + tarfile.NUL * 167
4753 )
4754 invalid_v7_header = (
4755 # name: 100 bytes
4756 tarfile.NUL * tarfile.LENGTH_NAME
4757 # mode, space, null terminator: 8 bytes
4758 + b"000755" + SPACE + tarfile.NUL
4759 # uid, space, null terminator: 8 bytes
4760 + b"000001" + SPACE + tarfile.NUL
4761 # gid, space, null terminator: 8 bytes
4762 + b"000001" + SPACE + tarfile.NUL
4763 # size, space: 12 bytes
4764 + b"\xff" * 11 + SPACE
4765 # mtime, space: 12 bytes
4766 + tarfile.NUL * 11 + SPACE
4767 # chksum: 8 bytes
4768 + b"0010070" + tarfile.NUL
4769 # type: 1 byte
4770 + tarfile.REGTYPE
4771 # linkname: 100 bytes
4772 + tarfile.NUL * tarfile.LENGTH_LINK
4773 # padding: 255 bytes
4774 + tarfile.NUL * 255
4775 )
4776 valid_gnu_header = tarfile.TarInfo("filename").tobuf(tarfile.GNU_FORMAT)
4777 data_block = b"\xff" * tarfile.BLOCKSIZE
4778
4779 def _write_buffer(self, buffer):
4780 with open(self.tarname, "wb") as f:
4781 f.write(buffer)
4782
4783 def _get_members(self, ignore_zeros=None):
4784 with open(self.tarname, "rb") as f:
4785 with tarfile.open(
4786 mode="r", fileobj=f, ignore_zeros=ignore_zeros
4787 ) as tar:
4788 return tar.getmembers()
4789
4790 def _assert_raises_read_error_exception(self):
4791 with self.assertRaisesRegex(
4792 tarfile.ReadError, "file could not be opened successfully"
4793 ):
4794 self._get_members()
4795
4796 def test_invalid_offset_header_validations(self):
4797 for tar_format, invalid_header in (
4798 ("posix", self.invalid_posix_header),
4799 ("gnu", self.invalid_gnu_header),
4800 ("v7", self.invalid_v7_header),
4801 ):
4802 with self.subTest(format=tar_format):
4803 self._write_buffer(invalid_header)
4804 self._assert_raises_read_error_exception()
4805
4806 def test_early_stop_at_invalid_offset_header(self):
4807 buffer = self.valid_gnu_header + self.invalid_gnu_header + self.valid_gnu_header
4808 self._write_buffer(buffer)
4809 members = self._get_members()
4810 self.assertEqual(len(members), 1)
4811 self.assertEqual(members[0].name, "filename")
4812 self.assertEqual(members[0].offset, 0)
4813
4814 def test_ignore_invalid_archive(self):
4815 # 3 invalid headers with their respective data
4816 buffer = (self.invalid_gnu_header + self.data_block) * 3
4817 self._write_buffer(buffer)
4818 members = self._get_members(ignore_zeros=True)
4819 self.assertEqual(len(members), 0)
4820
4821 def test_ignore_invalid_offset_headers(self):
4822 for first_block, second_block, expected_offset in (
4823 (
4824 (self.valid_gnu_header),
4825 (self.invalid_gnu_header + self.data_block),
4826 0,
4827 ),
4828 (
4829 (self.invalid_gnu_header + self.data_block),
4830 (self.valid_gnu_header),
4831 1024,
4832 ),
4833 ):
4834 self._write_buffer(first_block + second_block)
4835 members = self._get_members(ignore_zeros=True)
4836 self.assertEqual(len(members), 1)
4837 self.assertEqual(members[0].name, "filename")
4838 self.assertEqual(members[0].offset, expected_offset)
4839
4840
4841 def setUpModule():
4842 os_helper.unlink(TEMPDIR)
4843 os.makedirs(TEMPDIR)
4844
4845 global testtarnames
4846 testtarnames = [tarname]
4847 with open(tarname, "rb") as fobj:
4848 data = fobj.read()
4849
4850 # Create compressed tarfiles.
4851 for c in GzipTest, Bz2Test, LzmaTest, ZstdTest:
4852 if c.open:
4853 os_helper.unlink(c.tarname)
4854 testtarnames.append(c.tarname)
4855 with c.open(c.tarname, "wb") as tar:
4856 tar.write(data)
4857
4858 def tearDownModule():
4859 if os.path.exists(TEMPDIR):
4860 os_helper.rmtree(TEMPDIR)
4861
4862 if __name__ == "__main__":
4863 unittest.main()