Debian Codegraph @

Lib/test/test_gzip.py

raw
1 """Test script for the gzip module.
2 """
3
4 import array
5 import functools
6 import gc
7 import io
8 import os
9 import struct
10 import sys
11 import unittest
12 import warnings
13 from subprocess import PIPE, Popen
14 from test.support import catch_unraisable_exception
15 from test.support import import_helper
16 from test.support import os_helper
17 from test.support import _4G, bigmemtest, requires_subprocess
18 from test.support.script_helper import assert_python_ok, assert_python_failure
19
20 gzip = import_helper.import_module('gzip')
21 zlib = import_helper.import_module('zlib')
22
23 data1 = b""" int length=DEFAULTALLOC, err = Z_OK;
24 PyObject *RetVal;
25 int flushmode = Z_FINISH;
26 unsigned long start_total_out;
27
28 """
29
30 data2 = b"""/* zlibmodule.c -- gzip-compatible data compression */
31 /* See http://www.gzip.org/zlib/
32 /* See http://www.winimage.com/zLibDll for Windows */
33 """
34
35
36 TEMPDIR = os.path.abspath(os_helper.TESTFN) + '-gzdir'
37
38
39 class UnseekableIO(io.BytesIO):
40 def seekable(self):
41 return False
42
43 def tell(self):
44 raise io.UnsupportedOperation
45
46 def seek(self, *args):
47 raise io.UnsupportedOperation
48
49
50 class BaseTest(unittest.TestCase):
51 filename = os_helper.TESTFN
52
53 def setUp(self):
54 os_helper.unlink(self.filename)
55
56 def tearDown(self):
57 os_helper.unlink(self.filename)
58
59
60 class TestGzip(BaseTest):
61 def write_and_read_back(self, data, mode='b'):
62 b_data = bytes(data)
63 with gzip.GzipFile(self.filename, 'w'+mode) as f:
64 l = f.write(data)
65 self.assertEqual(l, len(b_data))
66 with gzip.GzipFile(self.filename, 'r'+mode) as f:
67 self.assertEqual(f.read(), b_data)
68
69 def test_write(self):
70 with gzip.GzipFile(self.filename, 'wb') as f:
71 f.write(data1 * 50)
72
73 # Try flush and fileno.
74 f.flush()
75 f.fileno()
76 if hasattr(os, 'fsync'):
77 os.fsync(f.fileno())
78 f.close()
79
80 # Test multiple close() calls.
81 f.close()
82
83 def test_write_read_with_pathlike_file(self):
84 filename = os_helper.FakePath(self.filename)
85 with gzip.GzipFile(filename, 'w') as f:
86 f.write(data1 * 50)
87 self.assertIsInstance(f.name, str)
88 self.assertEqual(f.name, self.filename)
89 with gzip.GzipFile(filename, 'a') as f:
90 f.write(data1)
91 with gzip.GzipFile(filename) as f:
92 d = f.read()
93 self.assertEqual(d, data1 * 51)
94 self.assertIsInstance(f.name, str)
95 self.assertEqual(f.name, self.filename)
96
97 # The following test_write_xy methods test that write accepts
98 # the corresponding bytes-like object type as input
99 # and that the data written equals bytes(xy) in all cases.
100 def test_write_memoryview(self):
101 self.write_and_read_back(memoryview(data1 * 50))
102 m = memoryview(bytes(range(256)))
103 data = m.cast('B', shape=[8,8,4])
104 self.write_and_read_back(data)
105
106 def test_write_bytearray(self):
107 self.write_and_read_back(bytearray(data1 * 50))
108
109 def test_write_array(self):
110 self.write_and_read_back(array.array('I', data1 * 40))
111
112 def test_write_incompatible_type(self):
113 # Test that non-bytes-like types raise TypeError.
114 # Issue #21560: attempts to write incompatible types
115 # should not affect the state of the fileobject
116 with gzip.GzipFile(self.filename, 'wb') as f:
117 with self.assertRaises(TypeError):
118 f.write('')
119 with self.assertRaises(TypeError):
120 f.write([])
121 f.write(data1)
122 with gzip.GzipFile(self.filename, 'rb') as f:
123 self.assertEqual(f.read(), data1)
124
125 def test_read(self):
126 self.test_write()
127 # Try reading.
128 with gzip.GzipFile(self.filename, 'r') as f:
129 d = f.read()
130 self.assertEqual(d, data1*50)
131
132 def test_read1(self):
133 self.test_write()
134 blocks = []
135 nread = 0
136 with gzip.GzipFile(self.filename, 'r') as f:
137 while True:
138 d = f.read1()
139 if not d:
140 break
141 blocks.append(d)
142 nread += len(d)
143 # Check that position was updated correctly (see issue10791).
144 self.assertEqual(f.tell(), nread)
145 self.assertEqual(b''.join(blocks), data1 * 50)
146
147 def test_readinto(self):
148 # 10MB of uncompressible data to ensure multiple reads
149 large_data = os.urandom(10 * 2**20)
150 with gzip.GzipFile(self.filename, 'wb') as f:
151 f.write(large_data)
152
153 buf = bytearray(len(large_data))
154 with gzip.GzipFile(self.filename, 'r') as f:
155 nbytes = f.readinto(buf)
156 self.assertEqual(nbytes, len(large_data))
157 self.assertEqual(buf, large_data)
158
159 def test_readinto1(self):
160 # 10MB of uncompressible data to ensure multiple reads
161 large_data = os.urandom(10 * 2**20)
162 with gzip.GzipFile(self.filename, 'wb') as f:
163 f.write(large_data)
164
165 nread = 0
166 buf = bytearray(len(large_data))
167 memview = memoryview(buf) # Simplifies slicing
168 with gzip.GzipFile(self.filename, 'r') as f:
169 for count in range(200):
170 nbytes = f.readinto1(memview[nread:])
171 if not nbytes:
172 break
173 nread += nbytes
174 self.assertEqual(f.tell(), nread)
175 self.assertEqual(buf, large_data)
176 # readinto1() should require multiple loops
177 self.assertGreater(count, 1)
178
179 @bigmemtest(size=_4G, memuse=1)
180 def test_read_large(self, size):
181 # Read chunk size over UINT_MAX should be supported, despite zlib's
182 # limitation per low-level call
183 compressed = gzip.compress(data1, compresslevel=1)
184 f = gzip.GzipFile(fileobj=io.BytesIO(compressed), mode='rb')
185 self.assertEqual(f.read(size), data1)
186
187 def test_io_on_closed_object(self):
188 # Test that I/O operations on closed GzipFile objects raise a
189 # ValueError, just like the corresponding functions on file objects.
190
191 # Write to a file, open it for reading, then close it.
192 self.test_write()
193 f = gzip.GzipFile(self.filename, 'r')
194 fileobj = f.fileobj
195 self.assertFalse(fileobj.closed)
196 f.close()
197 self.assertTrue(fileobj.closed)
198 with self.assertRaises(ValueError):
199 f.read(1)
200 with self.assertRaises(ValueError):
201 f.seek(0)
202 with self.assertRaises(ValueError):
203 f.tell()
204 # Open the file for writing, then close it.
205 f = gzip.GzipFile(self.filename, 'w')
206 fileobj = f.fileobj
207 self.assertFalse(fileobj.closed)
208 f.close()
209 self.assertTrue(fileobj.closed)
210 with self.assertRaises(ValueError):
211 f.write(b'')
212 with self.assertRaises(ValueError):
213 f.flush()
214
215 def test_append(self):
216 self.test_write()
217 # Append to the previous file
218 with gzip.GzipFile(self.filename, 'ab') as f:
219 f.write(data2 * 15)
220
221 with gzip.GzipFile(self.filename, 'rb') as f:
222 d = f.read()
223 self.assertEqual(d, (data1*50) + (data2*15))
224
225 def test_many_append(self):
226 # Bug #1074261 was triggered when reading a file that contained
227 # many, many members. Create such a file and verify that reading it
228 # works.
229 with gzip.GzipFile(self.filename, 'wb', 9) as f:
230 f.write(b'a')
231 for i in range(0, 200):
232 with gzip.GzipFile(self.filename, "ab", 9) as f: # append
233 f.write(b'a')
234
235 # Try reading the file
236 with gzip.GzipFile(self.filename, "rb") as zgfile:
237 contents = b""
238 while 1:
239 ztxt = zgfile.read(8192)
240 contents += ztxt
241 if not ztxt: break
242 self.assertEqual(contents, b'a'*201)
243
244 def test_exclusive_write(self):
245 with gzip.GzipFile(self.filename, 'xb') as f:
246 f.write(data1 * 50)
247 with gzip.GzipFile(self.filename, 'rb') as f:
248 self.assertEqual(f.read(), data1 * 50)
249 with self.assertRaises(FileExistsError):
250 gzip.GzipFile(self.filename, 'xb')
251
252 def test_buffered_reader(self):
253 # Issue #7471: a GzipFile can be wrapped in a BufferedReader for
254 # performance.
255 self.test_write()
256
257 with gzip.GzipFile(self.filename, 'rb') as f:
258 with io.BufferedReader(f) as r:
259 lines = [line for line in r]
260
261 self.assertEqual(lines, 50 * data1.splitlines(keepends=True))
262
263 def test_readline(self):
264 self.test_write()
265 # Try .readline() with varying line lengths
266
267 with gzip.GzipFile(self.filename, 'rb') as f:
268 line_length = 0
269 while 1:
270 L = f.readline(line_length)
271 if not L and line_length != 0: break
272 self.assertTrue(len(L) <= line_length)
273 line_length = (line_length + 1) % 50
274
275 def test_readlines(self):
276 self.test_write()
277 # Try .readlines()
278
279 with gzip.GzipFile(self.filename, 'rb') as f:
280 L = f.readlines()
281
282 with gzip.GzipFile(self.filename, 'rb') as f:
283 while 1:
284 L = f.readlines(150)
285 if L == []: break
286
287 def test_seek_read(self):
288 self.test_write()
289 # Try seek, read test
290
291 with gzip.GzipFile(self.filename) as f:
292 while 1:
293 oldpos = f.tell()
294 line1 = f.readline()
295 if not line1: break
296 newpos = f.tell()
297 f.seek(oldpos) # negative seek
298 if len(line1)>10:
299 amount = 10
300 else:
301 amount = len(line1)
302 line2 = f.read(amount)
303 self.assertEqual(line1[:amount], line2)
304 f.seek(newpos) # positive seek
305
306 def test_seek_whence(self):
307 self.test_write()
308 # Try seek(whence=1), read test
309
310 with gzip.GzipFile(self.filename) as f:
311 f.read(10)
312 f.seek(10, whence=1)
313 y = f.read(10)
314 self.assertEqual(y, data1[20:30])
315
316 def test_seek_write(self):
317 # Try seek, write test
318 with gzip.GzipFile(self.filename, 'w') as f:
319 for pos in range(0, 256, 16):
320 f.seek(pos)
321 f.write(b'GZ\n')
322
323 def test_mode(self):
324 self.test_write()
325 with gzip.GzipFile(self.filename, 'r') as f:
326 self.assertEqual(f.myfileobj.mode, 'rb')
327 os_helper.unlink(self.filename)
328 with gzip.GzipFile(self.filename, 'x') as f:
329 self.assertEqual(f.myfileobj.mode, 'xb')
330
331 def test_1647484(self):
332 for mode in ('wb', 'rb'):
333 with gzip.GzipFile(self.filename, mode) as f:
334 self.assertHasAttr(f, "name")
335 self.assertEqual(f.name, self.filename)
336
337 def test_paddedfile_getattr(self):
338 self.test_write()
339 with gzip.GzipFile(self.filename, 'rb') as f:
340 self.assertHasAttr(f.fileobj, "name")
341 self.assertEqual(f.fileobj.name, self.filename)
342
343 def test_mtime(self):
344 mtime = 123456789
345 with gzip.GzipFile(self.filename, 'w', mtime = mtime) as fWrite:
346 fWrite.write(data1)
347 with gzip.GzipFile(self.filename) as fRead:
348 self.assertHasAttr(fRead, 'mtime')
349 self.assertIsNone(fRead.mtime)
350 dataRead = fRead.read()
351 self.assertEqual(dataRead, data1)
352 self.assertEqual(fRead.mtime, mtime)
353
354 def test_metadata(self):
355 mtime = 123456789
356
357 with gzip.GzipFile(self.filename, 'w', mtime = mtime) as fWrite:
358 fWrite.write(data1)
359
360 with open(self.filename, 'rb') as fRead:
361 # see RFC 1952: http://www.faqs.org/rfcs/rfc1952.html
362
363 idBytes = fRead.read(2)
364 self.assertEqual(idBytes, b'\x1f\x8b') # gzip ID
365
366 cmByte = fRead.read(1)
367 self.assertEqual(cmByte, b'\x08') # deflate
368
369 try:
370 expectedname = self.filename.encode('Latin-1') + b'\x00'
371 expectedflags = b'\x08' # only the FNAME flag is set
372 except UnicodeEncodeError:
373 expectedname = b''
374 expectedflags = b'\x00'
375
376 flagsByte = fRead.read(1)
377 self.assertEqual(flagsByte, expectedflags)
378
379 mtimeBytes = fRead.read(4)
380 self.assertEqual(mtimeBytes, struct.pack('<i', mtime)) # little-endian
381
382 xflByte = fRead.read(1)
383 self.assertEqual(xflByte, b'\x02') # maximum compression
384
385 osByte = fRead.read(1)
386 self.assertEqual(osByte, b'\xff') # OS "unknown" (OS-independent)
387
388 # Since the FNAME flag is set, the zero-terminated filename follows.
389 # RFC 1952 specifies that this is the name of the input file, if any.
390 # However, the gzip module defaults to storing the name of the output
391 # file in this field.
392 nameBytes = fRead.read(len(expectedname))
393 self.assertEqual(nameBytes, expectedname)
394
395 # Since no other flags were set, the header ends here.
396 # Rather than process the compressed data, let's seek to the trailer.
397 fRead.seek(os.stat(self.filename).st_size - 8)
398
399 crc32Bytes = fRead.read(4) # CRC32 of uncompressed data [data1]
400 self.assertEqual(crc32Bytes, b'\xaf\xd7d\x83')
401
402 isizeBytes = fRead.read(4)
403 self.assertEqual(isizeBytes, struct.pack('<i', len(data1)))
404
405 def test_metadata_ascii_name(self):
406 self.filename = os_helper.TESTFN_ASCII
407 self.test_metadata()
408
409 def test_compresslevel_metadata(self):
410 # see RFC 1952: http://www.faqs.org/rfcs/rfc1952.html
411 # specifically, discussion of XFL in section 2.3.1
412 cases = [
413 ('fast', 1, b'\x04'),
414 ('best', 9, b'\x02'),
415 ('tradeoff', 6, b'\x00'),
416 ]
417 xflOffset = 8
418
419 for (name, level, expectedXflByte) in cases:
420 with self.subTest(name):
421 fWrite = gzip.GzipFile(self.filename, 'w', compresslevel=level)
422 with fWrite:
423 fWrite.write(data1)
424 with open(self.filename, 'rb') as fRead:
425 fRead.seek(xflOffset)
426 xflByte = fRead.read(1)
427 self.assertEqual(xflByte, expectedXflByte)
428
429 def test_with_open(self):
430 # GzipFile supports the context management protocol
431 with gzip.GzipFile(self.filename, "wb") as f:
432 f.write(b"xxx")
433 f = gzip.GzipFile(self.filename, "rb")
434 f.close()
435 try:
436 with f:
437 pass
438 except ValueError:
439 pass
440 else:
441 self.fail("__enter__ on a closed file didn't raise an exception")
442 try:
443 with gzip.GzipFile(self.filename, "wb") as f:
444 1/0
445 except ZeroDivisionError:
446 pass
447 else:
448 self.fail("1/0 didn't raise an exception")
449
450 def test_zero_padded_file(self):
451 with gzip.GzipFile(self.filename, "wb") as f:
452 f.write(data1 * 50)
453
454 # Pad the file with zeroes
455 with open(self.filename, "ab") as f:
456 f.write(b"\x00" * 50)
457
458 with gzip.GzipFile(self.filename, "rb") as f:
459 d = f.read()
460 self.assertEqual(d, data1 * 50, "Incorrect data in file")
461
462 def test_gzip_BadGzipFile_exception(self):
463 self.assertIsSubclass(gzip.BadGzipFile, OSError)
464
465 def test_bad_gzip_file(self):
466 with open(self.filename, 'wb') as file:
467 file.write(data1 * 50)
468 with gzip.GzipFile(self.filename, 'r') as file:
469 self.assertRaises(gzip.BadGzipFile, file.readlines)
470
471 def test_non_seekable_file(self):
472 uncompressed = data1 * 50
473 buf = UnseekableIO()
474 with gzip.GzipFile(fileobj=buf, mode="wb") as f:
475 f.write(uncompressed)
476 compressed = buf.getvalue()
477 buf = UnseekableIO(compressed)
478 with gzip.GzipFile(fileobj=buf, mode="rb") as f:
479 self.assertEqual(f.read(), uncompressed)
480
481 def test_peek(self):
482 uncompressed = data1 * 200
483 with gzip.GzipFile(self.filename, "wb") as f:
484 f.write(uncompressed)
485
486 def sizes():
487 while True:
488 for n in range(5, 50, 10):
489 yield n
490
491 with gzip.GzipFile(self.filename, "rb") as f:
492 f.max_read_chunk = 33
493 nread = 0
494 for n in sizes():
495 s = f.peek(n)
496 if s == b'':
497 break
498 self.assertEqual(f.read(len(s)), s)
499 nread += len(s)
500 self.assertEqual(f.read(100), b'')
501 self.assertEqual(nread, len(uncompressed))
502
503 def test_textio_readlines(self):
504 # Issue #10791: TextIOWrapper.readlines() fails when wrapping GzipFile.
505 lines = (data1 * 50).decode("ascii").splitlines(keepends=True)
506 self.test_write()
507 with gzip.GzipFile(self.filename, 'r') as f:
508 with io.TextIOWrapper(f, encoding="ascii") as t:
509 self.assertEqual(t.readlines(), lines)
510
511 def test_fileobj_with_name(self):
512 with open(self.filename, "xb") as raw:
513 with gzip.GzipFile(fileobj=raw, mode="x") as f:
514 f.write(b'one')
515 self.assertEqual(f.name, raw.name)
516 self.assertEqual(f.fileno(), raw.fileno())
517 self.assertEqual(f.mode, gzip.WRITE)
518 self.assertIs(f.readable(), False)
519 self.assertIs(f.writable(), True)
520 self.assertIs(f.seekable(), True)
521 self.assertIs(f.closed, False)
522 self.assertIs(f.closed, True)
523 self.assertEqual(f.name, raw.name)
524 self.assertRaises(AttributeError, f.fileno)
525 self.assertEqual(f.mode, gzip.WRITE)
526 self.assertIs(f.readable(), False)
527 self.assertIs(f.writable(), True)
528 self.assertIs(f.seekable(), True)
529
530 with open(self.filename, "wb") as raw:
531 with gzip.GzipFile(fileobj=raw, mode="w") as f:
532 f.write(b'two')
533 self.assertEqual(f.name, raw.name)
534 self.assertEqual(f.fileno(), raw.fileno())
535 self.assertEqual(f.mode, gzip.WRITE)
536 self.assertIs(f.readable(), False)
537 self.assertIs(f.writable(), True)
538 self.assertIs(f.seekable(), True)
539 self.assertIs(f.closed, False)
540 self.assertIs(f.closed, True)
541 self.assertEqual(f.name, raw.name)
542 self.assertRaises(AttributeError, f.fileno)
543 self.assertEqual(f.mode, gzip.WRITE)
544 self.assertIs(f.readable(), False)
545 self.assertIs(f.writable(), True)
546 self.assertIs(f.seekable(), True)
547
548 with open(self.filename, "ab") as raw:
549 with gzip.GzipFile(fileobj=raw, mode="a") as f:
550 f.write(b'three')
551 self.assertEqual(f.name, raw.name)
552 self.assertEqual(f.fileno(), raw.fileno())
553 self.assertEqual(f.mode, gzip.WRITE)
554 self.assertIs(f.readable(), False)
555 self.assertIs(f.writable(), True)
556 self.assertIs(f.seekable(), True)
557 self.assertIs(f.closed, False)
558 self.assertIs(f.closed, True)
559 self.assertEqual(f.name, raw.name)
560 self.assertRaises(AttributeError, f.fileno)
561 self.assertEqual(f.mode, gzip.WRITE)
562 self.assertIs(f.readable(), False)
563 self.assertIs(f.writable(), True)
564 self.assertIs(f.seekable(), True)
565
566 with open(self.filename, "rb") as raw:
567 with gzip.GzipFile(fileobj=raw, mode="r") as f:
568 self.assertEqual(f.read(), b'twothree')
569 self.assertEqual(f.name, raw.name)
570 self.assertEqual(f.fileno(), raw.fileno())
571 self.assertEqual(f.mode, gzip.READ)
572 self.assertIs(f.readable(), True)
573 self.assertIs(f.writable(), False)
574 self.assertIs(f.seekable(), True)
575 self.assertIs(f.closed, False)
576 self.assertIs(f.closed, True)
577 self.assertEqual(f.name, raw.name)
578 self.assertRaises(AttributeError, f.fileno)
579 self.assertEqual(f.mode, gzip.READ)
580 self.assertIs(f.readable(), True)
581 self.assertIs(f.writable(), False)
582 self.assertIs(f.seekable(), True)
583
584 def test_fileobj_from_fdopen(self):
585 # Issue #13781: Opening a GzipFile for writing fails when using a
586 # fileobj created with os.fdopen().
587 fd = os.open(self.filename, os.O_WRONLY | os.O_CREAT | os.O_EXCL)
588 with os.fdopen(fd, "xb") as raw:
589 with gzip.GzipFile(fileobj=raw, mode="x") as f:
590 f.write(b'one')
591 self.assertEqual(f.name, '')
592 self.assertEqual(f.fileno(), raw.fileno())
593 self.assertIs(f.closed, True)
594 self.assertEqual(f.name, '')
595 self.assertRaises(AttributeError, f.fileno)
596
597 fd = os.open(self.filename, os.O_WRONLY | os.O_CREAT | os.O_TRUNC)
598 with os.fdopen(fd, "wb") as raw:
599 with gzip.GzipFile(fileobj=raw, mode="w") as f:
600 f.write(b'two')
601 self.assertEqual(f.name, '')
602 self.assertEqual(f.fileno(), raw.fileno())
603 self.assertEqual(f.name, '')
604 self.assertRaises(AttributeError, f.fileno)
605
606 fd = os.open(self.filename, os.O_WRONLY | os.O_CREAT | os.O_APPEND)
607 with os.fdopen(fd, "ab") as raw:
608 with gzip.GzipFile(fileobj=raw, mode="a") as f:
609 f.write(b'three')
610 self.assertEqual(f.name, '')
611 self.assertEqual(f.fileno(), raw.fileno())
612 self.assertEqual(f.name, '')
613 self.assertRaises(AttributeError, f.fileno)
614
615 fd = os.open(self.filename, os.O_RDONLY)
616 with os.fdopen(fd, "rb") as raw:
617 with gzip.GzipFile(fileobj=raw, mode="r") as f:
618 self.assertEqual(f.read(), b'twothree')
619 self.assertEqual(f.name, '')
620 self.assertEqual(f.fileno(), raw.fileno())
621 self.assertEqual(f.name, '')
622 self.assertRaises(AttributeError, f.fileno)
623
624 def test_fileobj_mode(self):
625 self.assertEqual(gzip.READ, 'rb')
626 self.assertEqual(gzip.WRITE, 'wb')
627 gzip.GzipFile(self.filename, "wb").close()
628 with open(self.filename, "r+b") as f:
629 with gzip.GzipFile(fileobj=f, mode='r') as g:
630 self.assertEqual(g.mode, gzip.READ)
631 with gzip.GzipFile(fileobj=f, mode='w') as g:
632 self.assertEqual(g.mode, gzip.WRITE)
633 with gzip.GzipFile(fileobj=f, mode='a') as g:
634 self.assertEqual(g.mode, gzip.WRITE)
635 with gzip.GzipFile(fileobj=f, mode='x') as g:
636 self.assertEqual(g.mode, gzip.WRITE)
637 with self.assertRaises(ValueError):
638 gzip.GzipFile(fileobj=f, mode='z')
639 for mode in "rb", "r+b":
640 with open(self.filename, mode) as f:
641 with gzip.GzipFile(fileobj=f) as g:
642 self.assertEqual(g.mode, gzip.READ)
643 for mode in "wb", "ab", "xb":
644 if "x" in mode:
645 os_helper.unlink(self.filename)
646 with open(self.filename, mode) as f:
647 with self.assertWarns(FutureWarning):
648 g = gzip.GzipFile(fileobj=f)
649 with g:
650 self.assertEqual(g.mode, gzip.WRITE)
651
652 def test_bytes_filename(self):
653 str_filename = self.filename
654 bytes_filename = os.fsencode(str_filename)
655 with gzip.GzipFile(bytes_filename, "wb") as f:
656 f.write(data1 * 50)
657 self.assertEqual(f.name, bytes_filename)
658 with gzip.GzipFile(bytes_filename, "rb") as f:
659 self.assertEqual(f.read(), data1 * 50)
660 self.assertEqual(f.name, bytes_filename)
661 # Sanity check that we are actually operating on the right file.
662 with gzip.GzipFile(str_filename, "rb") as f:
663 self.assertEqual(f.read(), data1 * 50)
664 self.assertEqual(f.name, str_filename)
665
666 def test_fileobj_without_name(self):
667 bio = io.BytesIO()
668 with gzip.GzipFile(fileobj=bio, mode='wb') as f:
669 f.write(data1 * 50)
670 self.assertEqual(f.name, '')
671 self.assertRaises(io.UnsupportedOperation, f.fileno)
672 self.assertEqual(f.mode, gzip.WRITE)
673 self.assertIs(f.readable(), False)
674 self.assertIs(f.writable(), True)
675 self.assertIs(f.seekable(), True)
676 self.assertIs(f.closed, False)
677 self.assertIs(f.closed, True)
678 self.assertEqual(f.name, '')
679 self.assertRaises(AttributeError, f.fileno)
680 self.assertEqual(f.mode, gzip.WRITE)
681 self.assertIs(f.readable(), False)
682 self.assertIs(f.writable(), True)
683 self.assertIs(f.seekable(), True)
684
685 bio.seek(0)
686 with gzip.GzipFile(fileobj=bio, mode='rb') as f:
687 self.assertEqual(f.read(), data1 * 50)
688 self.assertEqual(f.name, '')
689 self.assertRaises(io.UnsupportedOperation, f.fileno)
690 self.assertEqual(f.mode, gzip.READ)
691 self.assertIs(f.readable(), True)
692 self.assertIs(f.writable(), False)
693 self.assertIs(f.seekable(), True)
694 self.assertIs(f.closed, False)
695 self.assertIs(f.closed, True)
696 self.assertEqual(f.name, '')
697 self.assertRaises(AttributeError, f.fileno)
698 self.assertEqual(f.mode, gzip.READ)
699 self.assertIs(f.readable(), True)
700 self.assertIs(f.writable(), False)
701 self.assertIs(f.seekable(), True)
702
703 def test_fileobj_and_filename(self):
704 filename2 = self.filename + 'new'
705 with (open(self.filename, 'wb') as fileobj,
706 gzip.GzipFile(fileobj=fileobj, filename=filename2, mode='wb') as f):
707 f.write(data1 * 50)
708 self.assertEqual(f.name, filename2)
709 with (open(self.filename, 'rb') as fileobj,
710 gzip.GzipFile(fileobj=fileobj, filename=filename2, mode='rb') as f):
711 self.assertEqual(f.read(), data1 * 50)
712 self.assertEqual(f.name, filename2)
713 # Sanity check that we are actually operating on the right file.
714 with gzip.GzipFile(self.filename, 'rb') as f:
715 self.assertEqual(f.read(), data1 * 50)
716 self.assertEqual(f.name, self.filename)
717
718 def test_decompress_limited(self):
719 """Decompressed data buffering should be limited"""
720 bomb = gzip.compress(b'\0' * int(2e6), compresslevel=9)
721 self.assertLess(len(bomb), io.DEFAULT_BUFFER_SIZE)
722
723 bomb = io.BytesIO(bomb)
724 decomp = gzip.GzipFile(fileobj=bomb)
725 self.assertEqual(decomp.read(1), b'\0')
726 max_decomp = 1 + io.DEFAULT_BUFFER_SIZE
727 self.assertLessEqual(decomp._buffer.raw.tell(), max_decomp,
728 "Excessive amount of data was decompressed")
729
730 # Testing compress/decompress shortcut functions
731
732 def test_compress(self):
733 for data in [data1, data2]:
734 for args in [(), (1,), (6,), (9,)]:
735 datac = gzip.compress(data, *args)
736 self.assertEqual(type(datac), bytes)
737 with gzip.GzipFile(fileobj=io.BytesIO(datac), mode="rb") as f:
738 self.assertEqual(f.read(), data)
739
740 def test_compress_mtime(self):
741 mtime = 123456789
742 for data in [data1, data2]:
743 for args in [(), (1,), (6,), (9,)]:
744 with self.subTest(data=data, args=args):
745 datac = gzip.compress(data, *args, mtime=mtime)
746 self.assertEqual(type(datac), bytes)
747 with gzip.GzipFile(fileobj=io.BytesIO(datac), mode="rb") as f:
748 f.read(1) # to set mtime attribute
749 self.assertEqual(f.mtime, mtime)
750
751 def test_compress_mtime_default(self):
752 # test for gh-125260
753 datac = gzip.compress(data1, mtime=0)
754 datac2 = gzip.compress(data1)
755 self.assertEqual(datac, datac2)
756 datac3 = gzip.compress(data1, mtime=None)
757 self.assertNotEqual(datac, datac3)
758 with gzip.GzipFile(fileobj=io.BytesIO(datac3), mode="rb") as f:
759 f.read(1) # to set mtime attribute
760 self.assertGreater(f.mtime, 1)
761
762 def test_compress_correct_level(self):
763 for mtime in (0, 42):
764 with self.subTest(mtime=mtime):
765 nocompress = gzip.compress(data1, compresslevel=0, mtime=mtime)
766 yescompress = gzip.compress(data1, compresslevel=1, mtime=mtime)
767 self.assertIn(data1, nocompress)
768 self.assertNotIn(data1, yescompress)
769
770 def test_issue112346(self):
771 # The OS byte should be 255, this should not change between Python versions.
772 for mtime in (0, 42):
773 with self.subTest(mtime=mtime):
774 compress = gzip.compress(data1, compresslevel=1, mtime=mtime)
775 self.assertEqual(
776 struct.unpack("<IxB", compress[4:10]),
777 (mtime, 255),
778 "Gzip header does not properly set either mtime or OS byte."
779 )
780
781 def test_decompress(self):
782 for data in (data1, data2):
783 buf = io.BytesIO()
784 with gzip.GzipFile(fileobj=buf, mode="wb") as f:
785 f.write(data)
786 self.assertEqual(gzip.decompress(buf.getvalue()), data)
787 # Roundtrip with compress
788 datac = gzip.compress(data)
789 self.assertEqual(gzip.decompress(datac), data)
790
791 def test_decompress_truncated_trailer(self):
792 compressed_data = gzip.compress(data1)
793 self.assertRaises(EOFError, gzip.decompress, compressed_data[:-4])
794
795 def test_decompress_missing_trailer(self):
796 compressed_data = gzip.compress(data1)
797 self.assertRaises(EOFError, gzip.decompress, compressed_data[:-8])
798
799 def test_read_truncated(self):
800 data = data1*50
801 # Drop the CRC (4 bytes) and file size (4 bytes).
802 truncated = gzip.compress(data)[:-8]
803 with gzip.GzipFile(fileobj=io.BytesIO(truncated)) as f:
804 self.assertRaises(EOFError, f.read)
805 with gzip.GzipFile(fileobj=io.BytesIO(truncated)) as f:
806 self.assertEqual(f.read(len(data)), data)
807 self.assertRaises(EOFError, f.read, 1)
808 # Incomplete 10-byte header.
809 for i in range(2, 10):
810 with gzip.GzipFile(fileobj=io.BytesIO(truncated[:i])) as f:
811 self.assertRaises(EOFError, f.read, 1)
812
813 def test_read_with_extra(self):
814 # Gzip data with an extra field
815 gzdata = (b'\x1f\x8b\x08\x04\xb2\x17cQ\x02\xff'
816 b'\x05\x00Extra'
817 b'\x0bI-.\x01\x002\xd1Mx\x04\x00\x00\x00')
818 with gzip.GzipFile(fileobj=io.BytesIO(gzdata)) as f:
819 self.assertEqual(f.read(), b'Test')
820
821 def test_prepend_error(self):
822 # See issue #20875
823 with gzip.open(self.filename, "wb") as f:
824 f.write(data1)
825 with gzip.open(self.filename, "rb") as f:
826 f._buffer.raw._fp.prepend()
827
828 def test_issue44439(self):
829 q = array.array('Q', [1, 2, 3, 4, 5])
830 LENGTH = len(q) * q.itemsize
831
832 with gzip.GzipFile(fileobj=io.BytesIO(), mode='w') as f:
833 self.assertEqual(f.write(q), LENGTH)
834 self.assertEqual(f.tell(), LENGTH)
835
836 def test_flush_flushes_compressor(self):
837 # See issue GH-105808.
838 b = io.BytesIO()
839 message = b"important message here."
840 with gzip.GzipFile(fileobj=b, mode='w') as f:
841 f.write(message)
842 f.flush()
843 partial_data = b.getvalue()
844 full_data = b.getvalue()
845 self.assertEqual(gzip.decompress(full_data), message)
846 # The partial data should contain the gzip header and the complete
847 # message, but not the end-of-stream markers (so we can't just
848 # decompress it directly).
849 with self.assertRaises(EOFError):
850 gzip.decompress(partial_data)
851 d = zlib.decompressobj(wbits=-zlib.MAX_WBITS)
852 f = io.BytesIO(partial_data)
853 gzip._read_gzip_header(f)
854 read_message = d.decompress(f.read())
855 self.assertEqual(read_message, message)
856
857 def test_flush_modes(self):
858 # Make sure the argument to flush is properly passed to the
859 # zlib.compressobj; see issue GH-105808.
860 class FakeCompressor:
861 def __init__(self):
862 self.modes = []
863 def compress(self, data):
864 return b''
865 def flush(self, mode=-1):
866 self.modes.append(mode)
867 return b''
868 b = io.BytesIO()
869 fc = FakeCompressor()
870 with gzip.GzipFile(fileobj=b, mode='w') as f:
871 f.compress = fc
872 f.flush()
873 f.flush(50)
874 f.flush(zlib_mode=100)
875 # The implicit close will also flush the compressor.
876 expected_modes = [
877 zlib.Z_SYNC_FLUSH,
878 50,
879 100,
880 -1,
881 ]
882 self.assertEqual(fc.modes, expected_modes)
883
884 def test_write_seek_write(self):
885 # Make sure that offset is up-to-date before seeking
886 # See issue GH-108111
887 b = io.BytesIO()
888 message = b"important message here."
889 with gzip.GzipFile(fileobj=b, mode='w') as f:
890 f.write(message)
891 f.seek(len(message))
892 f.write(message)
893 data = b.getvalue()
894 self.assertEqual(gzip.decompress(data), message * 2)
895
896
897 def test_refloop_unraisable(self):
898 # Ensure a GzipFile referring to a temporary fileobj deletes cleanly.
899 # Previously an unraisable exception would occur on close because the
900 # fileobj would be closed before the GzipFile as the result of a
901 # reference loop. See issue gh-129726
902 with catch_unraisable_exception() as cm:
903 with self.assertWarns(ResourceWarning):
904 gzip.GzipFile(fileobj=io.BytesIO(), mode="w")
905 gc.collect()
906 self.assertIsNone(cm.unraisable)
907
908
909 class TestOpen(BaseTest):
910 def test_binary_modes(self):
911 uncompressed = data1 * 50
912
913 with gzip.open(self.filename, "wb") as f:
914 f.write(uncompressed)
915 with open(self.filename, "rb") as f:
916 file_data = gzip.decompress(f.read())
917 self.assertEqual(file_data, uncompressed)
918
919 with gzip.open(self.filename, "rb") as f:
920 self.assertEqual(f.read(), uncompressed)
921
922 with gzip.open(self.filename, "ab") as f:
923 f.write(uncompressed)
924 with open(self.filename, "rb") as f:
925 file_data = gzip.decompress(f.read())
926 self.assertEqual(file_data, uncompressed * 2)
927
928 with self.assertRaises(FileExistsError):
929 gzip.open(self.filename, "xb")
930 os_helper.unlink(self.filename)
931 with gzip.open(self.filename, "xb") as f:
932 f.write(uncompressed)
933 with open(self.filename, "rb") as f:
934 file_data = gzip.decompress(f.read())
935 self.assertEqual(file_data, uncompressed)
936
937 def test_pathlike_file(self):
938 filename = os_helper.FakePath(self.filename)
939 with gzip.open(filename, "wb") as f:
940 f.write(data1 * 50)
941 self.assertEqual(f.name, self.filename)
942 with gzip.open(filename, "ab") as f:
943 f.write(data1)
944 self.assertEqual(f.name, self.filename)
945 with gzip.open(filename) as f:
946 self.assertEqual(f.read(), data1 * 51)
947 self.assertEqual(f.name, self.filename)
948
949 def test_implicit_binary_modes(self):
950 # Test implicit binary modes (no "b" or "t" in mode string).
951 uncompressed = data1 * 50
952
953 with gzip.open(self.filename, "w") as f:
954 f.write(uncompressed)
955 with open(self.filename, "rb") as f:
956 file_data = gzip.decompress(f.read())
957 self.assertEqual(file_data, uncompressed)
958
959 with gzip.open(self.filename, "r") as f:
960 self.assertEqual(f.read(), uncompressed)
961
962 with gzip.open(self.filename, "a") as f:
963 f.write(uncompressed)
964 with open(self.filename, "rb") as f:
965 file_data = gzip.decompress(f.read())
966 self.assertEqual(file_data, uncompressed * 2)
967
968 with self.assertRaises(FileExistsError):
969 gzip.open(self.filename, "x")
970 os_helper.unlink(self.filename)
971 with gzip.open(self.filename, "x") as f:
972 f.write(uncompressed)
973 with open(self.filename, "rb") as f:
974 file_data = gzip.decompress(f.read())
975 self.assertEqual(file_data, uncompressed)
976
977 def test_text_modes(self):
978 uncompressed = data1.decode("ascii") * 50
979 uncompressed_raw = uncompressed.replace("\n", os.linesep)
980 with gzip.open(self.filename, "wt", encoding="ascii") as f:
981 f.write(uncompressed)
982 with open(self.filename, "rb") as f:
983 file_data = gzip.decompress(f.read()).decode("ascii")
984 self.assertEqual(file_data, uncompressed_raw)
985 with gzip.open(self.filename, "rt", encoding="ascii") as f:
986 self.assertEqual(f.read(), uncompressed)
987 with gzip.open(self.filename, "at", encoding="ascii") as f:
988 f.write(uncompressed)
989 with open(self.filename, "rb") as f:
990 file_data = gzip.decompress(f.read()).decode("ascii")
991 self.assertEqual(file_data, uncompressed_raw * 2)
992
993 def test_fileobj(self):
994 uncompressed_bytes = data1 * 50
995 uncompressed_str = uncompressed_bytes.decode("ascii")
996 compressed = gzip.compress(uncompressed_bytes)
997 with gzip.open(io.BytesIO(compressed), "r") as f:
998 self.assertEqual(f.read(), uncompressed_bytes)
999 with gzip.open(io.BytesIO(compressed), "rb") as f:
1000 self.assertEqual(f.read(), uncompressed_bytes)
1001 with gzip.open(io.BytesIO(compressed), "rt", encoding="ascii") as f:
1002 self.assertEqual(f.read(), uncompressed_str)
1003
1004 def test_bad_params(self):
1005 # Test invalid parameter combinations.
1006 with self.assertRaises(TypeError):
1007 gzip.open(123.456)
1008 with self.assertRaises(ValueError):
1009 gzip.open(self.filename, "wbt")
1010 with self.assertRaises(ValueError):
1011 gzip.open(self.filename, "xbt")
1012 with self.assertRaises(ValueError):
1013 gzip.open(self.filename, "rb", encoding="utf-8")
1014 with self.assertRaises(ValueError):
1015 gzip.open(self.filename, "rb", errors="ignore")
1016 with self.assertRaises(ValueError):
1017 gzip.open(self.filename, "rb", newline="\n")
1018
1019 def test_encoding(self):
1020 # Test non-default encoding.
1021 uncompressed = data1.decode("ascii") * 50
1022 uncompressed_raw = uncompressed.replace("\n", os.linesep)
1023 with gzip.open(self.filename, "wt", encoding="utf-16") as f:
1024 f.write(uncompressed)
1025 with open(self.filename, "rb") as f:
1026 file_data = gzip.decompress(f.read()).decode("utf-16")
1027 self.assertEqual(file_data, uncompressed_raw)
1028 with gzip.open(self.filename, "rt", encoding="utf-16") as f:
1029 self.assertEqual(f.read(), uncompressed)
1030
1031 def test_encoding_error_handler(self):
1032 # Test with non-default encoding error handler.
1033 with gzip.open(self.filename, "wb") as f:
1034 f.write(b"foo\xffbar")
1035 with gzip.open(self.filename, "rt", encoding="ascii", errors="ignore") \
1036 as f:
1037 self.assertEqual(f.read(), "foobar")
1038
1039 def test_newline(self):
1040 # Test with explicit newline (universal newline mode disabled).
1041 uncompressed = data1.decode("ascii") * 50
1042 with gzip.open(self.filename, "wt", encoding="ascii", newline="\n") as f:
1043 f.write(uncompressed)
1044 with gzip.open(self.filename, "rt", encoding="ascii", newline="\r") as f:
1045 self.assertEqual(f.readlines(), [uncompressed])
1046
1047
1048 def create_and_remove_directory(directory):
1049 def decorator(function):
1050 @functools.wraps(function)
1051 def wrapper(*args, **kwargs):
1052 os.makedirs(directory)
1053 try:
1054 return function(*args, **kwargs)
1055 finally:
1056 os_helper.rmtree(directory)
1057 return wrapper
1058 return decorator
1059
1060
1061 class TestCommandLine(unittest.TestCase):
1062 data = b'This is a simple test with gzip'
1063
1064 @requires_subprocess()
1065 def test_decompress_stdin_stdout(self):
1066 with io.BytesIO() as bytes_io:
1067 with gzip.GzipFile(fileobj=bytes_io, mode='wb') as gzip_file:
1068 gzip_file.write(self.data)
1069
1070 args = sys.executable, '-m', 'gzip', '-d'
1071 with Popen(args, stdin=PIPE, stdout=PIPE, stderr=PIPE) as proc:
1072 out, err = proc.communicate(bytes_io.getvalue())
1073
1074 self.assertEqual(err, b'')
1075 self.assertEqual(out, self.data)
1076
1077 @create_and_remove_directory(TEMPDIR)
1078 def test_decompress_infile_outfile(self):
1079 gzipname = os.path.join(TEMPDIR, 'testgzip.gz')
1080 self.assertFalse(os.path.exists(gzipname))
1081
1082 with gzip.open(gzipname, mode='wb') as fp:
1083 fp.write(self.data)
1084 rc, out, err = assert_python_ok('-m', 'gzip', '-d', gzipname)
1085
1086 with open(os.path.join(TEMPDIR, "testgzip"), "rb") as gunziped:
1087 self.assertEqual(gunziped.read(), self.data)
1088
1089 self.assertTrue(os.path.exists(gzipname))
1090 self.assertEqual(rc, 0)
1091 self.assertEqual(out, b'')
1092 self.assertEqual(err, b'')
1093
1094 def test_decompress_infile_outfile_error(self):
1095 rc, out, err = assert_python_failure('-m', 'gzip', '-d', 'thisisatest.out')
1096 self.assertEqual(b"filename doesn't end in .gz: 'thisisatest.out'", err.strip())
1097 self.assertEqual(rc, 1)
1098 self.assertEqual(out, b'')
1099
1100 @requires_subprocess()
1101 @create_and_remove_directory(TEMPDIR)
1102 def test_compress_stdin_outfile(self):
1103 args = sys.executable, '-m', 'gzip'
1104 with Popen(args, stdin=PIPE, stdout=PIPE, stderr=PIPE) as proc:
1105 out, err = proc.communicate(self.data)
1106
1107 self.assertEqual(err, b'')
1108 self.assertEqual(out[:2], b"\x1f\x8b")
1109
1110 @create_and_remove_directory(TEMPDIR)
1111 def test_compress_infile_outfile_default(self):
1112 local_testgzip = os.path.join(TEMPDIR, 'testgzip')
1113 gzipname = local_testgzip + '.gz'
1114 self.assertFalse(os.path.exists(gzipname))
1115
1116 with open(local_testgzip, 'wb') as fp:
1117 fp.write(self.data)
1118
1119 rc, out, err = assert_python_ok('-m', 'gzip', local_testgzip)
1120
1121 self.assertTrue(os.path.exists(gzipname))
1122 self.assertEqual(out, b'')
1123 self.assertEqual(err, b'')
1124
1125 @create_and_remove_directory(TEMPDIR)
1126 def test_compress_infile_outfile(self):
1127 for compress_level in ('--fast', '--best'):
1128 with self.subTest(compress_level=compress_level):
1129 local_testgzip = os.path.join(TEMPDIR, 'testgzip')
1130 gzipname = local_testgzip + '.gz'
1131 self.assertFalse(os.path.exists(gzipname))
1132
1133 with open(local_testgzip, 'wb') as fp:
1134 fp.write(self.data)
1135
1136 rc, out, err = assert_python_ok('-m', 'gzip', compress_level, local_testgzip)
1137
1138 self.assertTrue(os.path.exists(gzipname))
1139 self.assertEqual(out, b'')
1140 self.assertEqual(err, b'')
1141 os.remove(gzipname)
1142 self.assertFalse(os.path.exists(gzipname))
1143
1144 def test_compress_fast_best_are_exclusive(self):
1145 rc, out, err = assert_python_failure('-m', 'gzip', '--fast', '--best')
1146 self.assertIn(b"error: argument --best: not allowed with argument --fast", err)
1147 self.assertEqual(out, b'')
1148
1149 def test_decompress_cannot_have_flags_compression(self):
1150 rc, out, err = assert_python_failure('-m', 'gzip', '--fast', '-d')
1151 self.assertIn(b'error: argument -d/--decompress: not allowed with argument --fast', err)
1152 self.assertEqual(out, b'')
1153
1154
1155 if __name__ == "__main__":
1156 unittest.main()