Debian Codegraph @

Lib/test/test_zstd.py

raw
1 import array
2 import gc
3 import io
4 import pathlib
5 import random
6 import re
7 import os
8 import unittest
9 import tempfile
10 import threading
11
12 from test.support.import_helper import import_module
13 from test.support import threading_helper
14 from test.support import _1M
15 from test.support import Py_GIL_DISABLED
16
17 _zstd = import_module("_zstd")
18 zstd = import_module("compression.zstd")
19
20 from compression.zstd import (
21 open,
22 compress,
23 decompress,
24 ZstdCompressor,
25 ZstdDecompressor,
26 ZstdDict,
27 ZstdError,
28 zstd_version,
29 zstd_version_info,
30 COMPRESSION_LEVEL_DEFAULT,
31 get_frame_info,
32 get_frame_size,
33 finalize_dict,
34 train_dict,
35 CompressionParameter,
36 DecompressionParameter,
37 Strategy,
38 ZstdFile,
39 )
40
41 _1K = 1024
42 _130_1K = 130 * _1K
43 DICT_SIZE1 = 3*_1K
44
45 DAT_130K_D = None
46 DAT_130K_C = None
47
48 DECOMPRESSED_DAT = None
49 COMPRESSED_DAT = None
50
51 DECOMPRESSED_100_PLUS_32KB = None
52 COMPRESSED_100_PLUS_32KB = None
53
54 SKIPPABLE_FRAME = None
55
56 THIS_FILE_BYTES = None
57 THIS_FILE_STR = None
58 COMPRESSED_THIS_FILE = None
59
60 COMPRESSED_BOGUS = None
61
62 SAMPLES = None
63
64 TRAINED_DICT = None
65
66 # Cannot be deferred to setup as it is used to check whether or not to skip
67 # tests
68 try:
69 SUPPORT_MULTITHREADING = CompressionParameter.nb_workers.bounds() != (0, 0)
70 except Exception:
71 SUPPORT_MULTITHREADING = False
72
73 C_INT_MIN = -(2**31)
74 C_INT_MAX = (2**31) - 1
75
76
77 def setUpModule():
78 # uncompressed size 130KB, more than a zstd block.
79 # with a frame epilogue, 4 bytes checksum.
80 global DAT_130K_D
81 DAT_130K_D = bytes([random.randint(0, 127) for _ in range(130*_1K)])
82
83 global DAT_130K_C
84 DAT_130K_C = compress(DAT_130K_D, options={CompressionParameter.checksum_flag:1})
85
86 global DECOMPRESSED_DAT
87 DECOMPRESSED_DAT = b'abcdefg123456' * 1000
88
89 global COMPRESSED_DAT
90 COMPRESSED_DAT = compress(DECOMPRESSED_DAT)
91
92 global DECOMPRESSED_100_PLUS_32KB
93 DECOMPRESSED_100_PLUS_32KB = b'a' * (100 + 32*_1K)
94
95 global COMPRESSED_100_PLUS_32KB
96 COMPRESSED_100_PLUS_32KB = compress(DECOMPRESSED_100_PLUS_32KB)
97
98 global SKIPPABLE_FRAME
99 SKIPPABLE_FRAME = (0x184D2A50).to_bytes(4, byteorder='little') + \
100 (32*_1K).to_bytes(4, byteorder='little') + \
101 b'a' * (32*_1K)
102
103 global THIS_FILE_BYTES, THIS_FILE_STR
104 with io.open(os.path.abspath(__file__), 'rb') as f:
105 THIS_FILE_BYTES = f.read()
106 THIS_FILE_BYTES = re.sub(rb'\r?\n', rb'\n', THIS_FILE_BYTES)
107 THIS_FILE_STR = THIS_FILE_BYTES.decode('utf-8')
108
109 global COMPRESSED_THIS_FILE
110 COMPRESSED_THIS_FILE = compress(THIS_FILE_BYTES)
111
112 global COMPRESSED_BOGUS
113 COMPRESSED_BOGUS = DECOMPRESSED_DAT
114
115 # dict data
116 words = [b'red', b'green', b'yellow', b'black', b'withe', b'blue',
117 b'lilac', b'purple', b'navy', b'glod', b'silver', b'olive',
118 b'dog', b'cat', b'tiger', b'lion', b'fish', b'bird']
119 lst = []
120 for i in range(300):
121 sample = [b'%s = %d' % (random.choice(words), random.randrange(100))
122 for j in range(20)]
123 sample = b'\n'.join(sample)
124
125 lst.append(sample)
126 global SAMPLES
127 SAMPLES = lst
128 assert len(SAMPLES) > 10
129
130 global TRAINED_DICT
131 TRAINED_DICT = train_dict(SAMPLES, 3*_1K)
132 assert len(TRAINED_DICT.dict_content) <= 3*_1K
133
134
135 class FunctionsTestCase(unittest.TestCase):
136
137 def test_version(self):
138 s = ".".join((str(i) for i in zstd_version_info))
139 self.assertEqual(s, zstd_version)
140
141 def test_compressionLevel_values(self):
142 min, max = CompressionParameter.compression_level.bounds()
143 self.assertIs(type(COMPRESSION_LEVEL_DEFAULT), int)
144 self.assertIs(type(min), int)
145 self.assertIs(type(max), int)
146 self.assertLess(min, max)
147
148 def test_roundtrip_default(self):
149 raw_dat = THIS_FILE_BYTES[: len(THIS_FILE_BYTES) // 6]
150 dat1 = compress(raw_dat)
151 dat2 = decompress(dat1)
152 self.assertEqual(dat2, raw_dat)
153
154 def test_roundtrip_level(self):
155 raw_dat = THIS_FILE_BYTES[: len(THIS_FILE_BYTES) // 6]
156 level_min, level_max = CompressionParameter.compression_level.bounds()
157
158 for level in range(max(-20, level_min), level_max + 1):
159 dat1 = compress(raw_dat, level)
160 dat2 = decompress(dat1)
161 self.assertEqual(dat2, raw_dat)
162
163 def test_get_frame_info(self):
164 # no dict
165 info = get_frame_info(COMPRESSED_100_PLUS_32KB[:20])
166 self.assertEqual(info.decompressed_size, 32 * _1K + 100)
167 self.assertEqual(info.dictionary_id, 0)
168
169 # use dict
170 dat = compress(b"a" * 345, zstd_dict=TRAINED_DICT)
171 info = get_frame_info(dat)
172 self.assertEqual(info.decompressed_size, 345)
173 self.assertEqual(info.dictionary_id, TRAINED_DICT.dict_id)
174
175 with self.assertRaisesRegex(ZstdError, "not less than the frame header"):
176 get_frame_info(b"aaaaaaaaaaaaaa")
177
178 def test_get_frame_size(self):
179 size = get_frame_size(COMPRESSED_100_PLUS_32KB)
180 self.assertEqual(size, len(COMPRESSED_100_PLUS_32KB))
181
182 with self.assertRaisesRegex(ZstdError, "not less than this complete frame"):
183 get_frame_size(b"aaaaaaaaaaaaaa")
184
185 def test_decompress_2x130_1K(self):
186 decompressed_size = get_frame_info(DAT_130K_C).decompressed_size
187 self.assertEqual(decompressed_size, _130_1K)
188
189 dat = decompress(DAT_130K_C + DAT_130K_C)
190 self.assertEqual(len(dat), 2 * _130_1K)
191
192
193 class CompressorTestCase(unittest.TestCase):
194
195 def test_simple_compress_bad_args(self):
196 # ZstdCompressor
197 self.assertRaises(TypeError, ZstdCompressor, [])
198 self.assertRaises(TypeError, ZstdCompressor, level=3.14)
199 self.assertRaises(TypeError, ZstdCompressor, level="abc")
200 self.assertRaises(TypeError, ZstdCompressor, options=b"abc")
201
202 self.assertRaises(TypeError, ZstdCompressor, zstd_dict=123)
203 self.assertRaises(TypeError, ZstdCompressor, zstd_dict=b"abcd1234")
204 self.assertRaises(TypeError, ZstdCompressor, zstd_dict={1: 2, 3: 4})
205
206 # valid range for compression level is [-(1<<17), 22]
207 msg = r'illegal compression level {}; the valid range is \[-?\d+, -?\d+\]'
208 with self.assertRaisesRegex(ValueError, msg.format(C_INT_MAX)):
209 ZstdCompressor(C_INT_MAX)
210 with self.assertRaisesRegex(ValueError, msg.format(C_INT_MIN)):
211 ZstdCompressor(C_INT_MIN)
212 msg = r'illegal compression level; the valid range is \[-?\d+, -?\d+\]'
213 with self.assertRaisesRegex(ValueError, msg):
214 ZstdCompressor(level=-(2**1000))
215 with self.assertRaisesRegex(ValueError, msg):
216 ZstdCompressor(level=2**1000)
217
218 with self.assertRaises(ValueError):
219 ZstdCompressor(options={CompressionParameter.window_log: 100})
220 with self.assertRaises(ValueError):
221 ZstdCompressor(options={3333: 100})
222
223 # Method bad arguments
224 zc = ZstdCompressor()
225 self.assertRaises(TypeError, zc.compress)
226 self.assertRaises((TypeError, ValueError), zc.compress, b"foo", b"bar")
227 self.assertRaises(TypeError, zc.compress, "str")
228 self.assertRaises((TypeError, ValueError), zc.flush, b"foo")
229 self.assertRaises(TypeError, zc.flush, b"blah", 1)
230
231 self.assertRaises(ValueError, zc.compress, b'', -1)
232 self.assertRaises(ValueError, zc.compress, b'', 3)
233 self.assertRaises(ValueError, zc.flush, zc.CONTINUE) # 0
234 self.assertRaises(ValueError, zc.flush, 3)
235
236 zc.compress(b'')
237 zc.compress(b'', zc.CONTINUE)
238 zc.compress(b'', zc.FLUSH_BLOCK)
239 zc.compress(b'', zc.FLUSH_FRAME)
240 empty = zc.flush()
241 zc.flush(zc.FLUSH_BLOCK)
242 zc.flush(zc.FLUSH_FRAME)
243
244 def test_compress_parameters(self):
245 d = {CompressionParameter.compression_level : 10,
246
247 CompressionParameter.window_log : 12,
248 CompressionParameter.hash_log : 10,
249 CompressionParameter.chain_log : 12,
250 CompressionParameter.search_log : 12,
251 CompressionParameter.min_match : 4,
252 CompressionParameter.target_length : 12,
253 CompressionParameter.strategy : Strategy.lazy,
254
255 CompressionParameter.enable_long_distance_matching : 1,
256 CompressionParameter.ldm_hash_log : 12,
257 CompressionParameter.ldm_min_match : 11,
258 CompressionParameter.ldm_bucket_size_log : 5,
259 CompressionParameter.ldm_hash_rate_log : 12,
260
261 CompressionParameter.content_size_flag : 1,
262 CompressionParameter.checksum_flag : 1,
263 CompressionParameter.dict_id_flag : 0,
264
265 CompressionParameter.nb_workers : 2 if SUPPORT_MULTITHREADING else 0,
266 CompressionParameter.job_size : 5*_1M if SUPPORT_MULTITHREADING else 0,
267 CompressionParameter.overlap_log : 9 if SUPPORT_MULTITHREADING else 0,
268 }
269 ZstdCompressor(options=d)
270
271 d1 = d.copy()
272 # larger than signed int
273 d1[CompressionParameter.ldm_bucket_size_log] = C_INT_MAX
274 with self.assertRaises(ValueError):
275 ZstdCompressor(options=d1)
276 # smaller than signed int
277 d1[CompressionParameter.ldm_bucket_size_log] = C_INT_MIN
278 with self.assertRaises(ValueError):
279 ZstdCompressor(options=d1)
280
281 # out of bounds compression level
282 level_min, level_max = CompressionParameter.compression_level.bounds()
283 with self.assertRaises(ValueError):
284 compress(b'', level_max+1)
285 with self.assertRaises(ValueError):
286 compress(b'', level_min-1)
287 with self.assertRaises(ValueError):
288 compress(b'', 2**1000)
289 with self.assertRaises(ValueError):
290 compress(b'', -(2**1000))
291 with self.assertRaises(ValueError):
292 compress(b'', options={
293 CompressionParameter.compression_level: level_max+1})
294 with self.assertRaises(ValueError):
295 compress(b'', options={
296 CompressionParameter.compression_level: level_min-1})
297
298 # zstd lib doesn't support MT compression
299 if not SUPPORT_MULTITHREADING:
300 with self.assertRaises(ValueError):
301 ZstdCompressor(options={CompressionParameter.nb_workers:4})
302 with self.assertRaises(ValueError):
303 ZstdCompressor(options={CompressionParameter.job_size:4})
304 with self.assertRaises(ValueError):
305 ZstdCompressor(options={CompressionParameter.overlap_log:4})
306
307 # out of bounds error msg
308 option = {CompressionParameter.window_log:100}
309 with self.assertRaisesRegex(
310 ValueError,
311 "compression parameter 'window_log' received an illegal value 100; "
312 r'the valid range is \[-?\d+, -?\d+\]',
313 ):
314 compress(b'', options=option)
315
316 def test_unknown_compression_parameter(self):
317 KEY = 100001234
318 option = {CompressionParameter.compression_level: 10,
319 KEY: 200000000}
320 pattern = rf"invalid compression parameter 'unknown parameter \(key {KEY}\)'"
321 with self.assertRaisesRegex(ValueError, pattern):
322 ZstdCompressor(options=option)
323
324 @unittest.skipIf(not SUPPORT_MULTITHREADING,
325 "zstd build doesn't support multi-threaded compression")
326 def test_zstd_multithread_compress(self):
327 size = 40*_1M
328 b = THIS_FILE_BYTES * (size // len(THIS_FILE_BYTES))
329
330 options = {CompressionParameter.compression_level : 4,
331 CompressionParameter.nb_workers : 2}
332
333 # compress()
334 dat1 = compress(b, options=options)
335 dat2 = decompress(dat1)
336 self.assertEqual(dat2, b)
337
338 # ZstdCompressor
339 c = ZstdCompressor(options=options)
340 dat1 = c.compress(b, c.CONTINUE)
341 dat2 = c.compress(b, c.FLUSH_BLOCK)
342 dat3 = c.compress(b, c.FLUSH_FRAME)
343 dat4 = decompress(dat1+dat2+dat3)
344 self.assertEqual(dat4, b * 3)
345
346 # ZstdFile
347 with ZstdFile(io.BytesIO(), 'w', options=options) as f:
348 f.write(b)
349
350 def test_compress_flushblock(self):
351 point = len(THIS_FILE_BYTES) // 2
352
353 c = ZstdCompressor()
354 self.assertEqual(c.last_mode, c.FLUSH_FRAME)
355 dat1 = c.compress(THIS_FILE_BYTES[:point])
356 self.assertEqual(c.last_mode, c.CONTINUE)
357 dat1 += c.compress(THIS_FILE_BYTES[point:], c.FLUSH_BLOCK)
358 self.assertEqual(c.last_mode, c.FLUSH_BLOCK)
359 dat2 = c.flush()
360 pattern = "Compressed data ended before the end-of-stream marker"
361 with self.assertRaisesRegex(ZstdError, pattern):
362 decompress(dat1)
363
364 dat3 = decompress(dat1 + dat2)
365
366 self.assertEqual(dat3, THIS_FILE_BYTES)
367
368 def test_compress_flushframe(self):
369 # test compress & decompress
370 point = len(THIS_FILE_BYTES) // 2
371
372 c = ZstdCompressor()
373
374 dat1 = c.compress(THIS_FILE_BYTES[:point])
375 self.assertEqual(c.last_mode, c.CONTINUE)
376
377 dat1 += c.compress(THIS_FILE_BYTES[point:], c.FLUSH_FRAME)
378 self.assertEqual(c.last_mode, c.FLUSH_FRAME)
379
380 nt = get_frame_info(dat1)
381 self.assertEqual(nt.decompressed_size, None) # no content size
382
383 dat2 = decompress(dat1)
384
385 self.assertEqual(dat2, THIS_FILE_BYTES)
386
387 # single .FLUSH_FRAME mode has content size
388 c = ZstdCompressor()
389 dat = c.compress(THIS_FILE_BYTES, mode=c.FLUSH_FRAME)
390 self.assertEqual(c.last_mode, c.FLUSH_FRAME)
391
392 nt = get_frame_info(dat)
393 self.assertEqual(nt.decompressed_size, len(THIS_FILE_BYTES))
394
395 def test_compress_empty(self):
396 # output empty content frame
397 self.assertNotEqual(compress(b''), b'')
398
399 c = ZstdCompressor()
400 self.assertNotEqual(c.compress(b'', c.FLUSH_FRAME), b'')
401
402 def test_set_pledged_input_size(self):
403 DAT = DECOMPRESSED_100_PLUS_32KB
404 CHUNK_SIZE = len(DAT) // 3
405
406 # wrong value
407 c = ZstdCompressor()
408 with self.assertRaisesRegex(ValueError,
409 r'should be a positive int less than \d+'):
410 c.set_pledged_input_size(-300)
411 # overflow
412 with self.assertRaisesRegex(ValueError,
413 r'should be a positive int less than \d+'):
414 c.set_pledged_input_size(2**64)
415 # ZSTD_CONTENTSIZE_ERROR is invalid
416 with self.assertRaisesRegex(ValueError,
417 r'should be a positive int less than \d+'):
418 c.set_pledged_input_size(2**64-2)
419 # ZSTD_CONTENTSIZE_UNKNOWN should use None
420 with self.assertRaisesRegex(ValueError,
421 r'should be a positive int less than \d+'):
422 c.set_pledged_input_size(2**64-1)
423
424 # check valid values are settable
425 c.set_pledged_input_size(2**63)
426 c.set_pledged_input_size(2**64-3)
427
428 # check that zero means empty frame
429 c = ZstdCompressor(level=1)
430 c.set_pledged_input_size(0)
431 c.compress(b'')
432 dat = c.flush()
433 ret = get_frame_info(dat)
434 self.assertEqual(ret.decompressed_size, 0)
435
436
437 # wrong mode
438 c = ZstdCompressor(level=1)
439 c.compress(b'123456')
440 self.assertEqual(c.last_mode, c.CONTINUE)
441 with self.assertRaisesRegex(ValueError,
442 r'last_mode == FLUSH_FRAME'):
443 c.set_pledged_input_size(300)
444
445 # None value
446 c = ZstdCompressor(level=1)
447 c.set_pledged_input_size(None)
448 dat = c.compress(DAT) + c.flush()
449
450 ret = get_frame_info(dat)
451 self.assertEqual(ret.decompressed_size, None)
452
453 # correct value
454 c = ZstdCompressor(level=1)
455 c.set_pledged_input_size(len(DAT))
456
457 chunks = []
458 posi = 0
459 while posi < len(DAT):
460 dat = c.compress(DAT[posi:posi+CHUNK_SIZE])
461 posi += CHUNK_SIZE
462 chunks.append(dat)
463
464 dat = c.flush()
465 chunks.append(dat)
466 chunks = b''.join(chunks)
467
468 ret = get_frame_info(chunks)
469 self.assertEqual(ret.decompressed_size, len(DAT))
470 self.assertEqual(decompress(chunks), DAT)
471
472 c.set_pledged_input_size(len(DAT)) # the second frame
473 dat = c.compress(DAT) + c.flush()
474
475 ret = get_frame_info(dat)
476 self.assertEqual(ret.decompressed_size, len(DAT))
477 self.assertEqual(decompress(dat), DAT)
478
479 # not enough data
480 c = ZstdCompressor(level=1)
481 c.set_pledged_input_size(len(DAT)+1)
482
483 for start in range(0, len(DAT), CHUNK_SIZE):
484 end = min(start+CHUNK_SIZE, len(DAT))
485 _dat = c.compress(DAT[start:end])
486
487 with self.assertRaises(ZstdError):
488 c.flush()
489
490 # too much data
491 c = ZstdCompressor(level=1)
492 c.set_pledged_input_size(len(DAT))
493
494 for start in range(0, len(DAT), CHUNK_SIZE):
495 end = min(start+CHUNK_SIZE, len(DAT))
496 _dat = c.compress(DAT[start:end])
497
498 with self.assertRaises(ZstdError):
499 c.compress(b'extra', ZstdCompressor.FLUSH_FRAME)
500
501 # content size not set if content_size_flag == 0
502 c = ZstdCompressor(options={CompressionParameter.content_size_flag: 0})
503 c.set_pledged_input_size(10)
504 dat1 = c.compress(b"hello")
505 dat2 = c.compress(b"world")
506 dat3 = c.flush()
507 frame_data = get_frame_info(dat1 + dat2 + dat3)
508 self.assertIsNone(frame_data.decompressed_size)
509
510
511 class DecompressorTestCase(unittest.TestCase):
512
513 def test_simple_decompress_bad_args(self):
514 # ZstdDecompressor
515 self.assertRaises(TypeError, ZstdDecompressor, ())
516 self.assertRaises(TypeError, ZstdDecompressor, zstd_dict=123)
517 self.assertRaises(TypeError, ZstdDecompressor, zstd_dict=b'abc')
518 self.assertRaises(TypeError, ZstdDecompressor, zstd_dict={1:2, 3:4})
519
520 self.assertRaises(TypeError, ZstdDecompressor, options=123)
521 self.assertRaises(TypeError, ZstdDecompressor, options='abc')
522 self.assertRaises(TypeError, ZstdDecompressor, options=b'abc')
523
524 with self.assertRaises(ValueError):
525 ZstdDecompressor(options={C_INT_MAX: 100})
526 with self.assertRaises(ValueError):
527 ZstdDecompressor(options={C_INT_MIN: 100})
528 with self.assertRaises(ValueError):
529 ZstdDecompressor(options={0: C_INT_MAX})
530 with self.assertRaises(OverflowError):
531 ZstdDecompressor(options={2**1000: 100})
532 with self.assertRaises(OverflowError):
533 ZstdDecompressor(options={-(2**1000): 100})
534 with self.assertRaises(OverflowError):
535 ZstdDecompressor(options={0: -(2**1000)})
536
537 with self.assertRaises(ValueError):
538 ZstdDecompressor(options={DecompressionParameter.window_log_max: 100})
539 with self.assertRaises(ValueError):
540 ZstdDecompressor(options={3333: 100})
541
542 empty = compress(b'')
543 lzd = ZstdDecompressor()
544 self.assertRaises(TypeError, lzd.decompress)
545 self.assertRaises(TypeError, lzd.decompress, b"foo", b"bar")
546 self.assertRaises(TypeError, lzd.decompress, "str")
547 lzd.decompress(empty)
548
549 def test_decompress_parameters(self):
550 d = {DecompressionParameter.window_log_max : 15}
551 ZstdDecompressor(options=d)
552
553 d1 = d.copy()
554 # larger than signed int
555 d1[DecompressionParameter.window_log_max] = 2**1000
556 with self.assertRaises(OverflowError):
557 ZstdDecompressor(None, d1)
558 # smaller than signed int
559 d1[DecompressionParameter.window_log_max] = -(2**1000)
560 with self.assertRaises(OverflowError):
561 ZstdDecompressor(None, d1)
562
563 d1[DecompressionParameter.window_log_max] = C_INT_MAX
564 with self.assertRaises(ValueError):
565 ZstdDecompressor(None, d1)
566 d1[DecompressionParameter.window_log_max] = C_INT_MIN
567 with self.assertRaises(ValueError):
568 ZstdDecompressor(None, d1)
569
570 # out of bounds error msg
571 options = {DecompressionParameter.window_log_max:100}
572 with self.assertRaisesRegex(
573 ValueError,
574 "decompression parameter 'window_log_max' received an illegal value 100; "
575 r'the valid range is \[-?\d+, -?\d+\]',
576 ):
577 decompress(b'', options=options)
578
579 # out of bounds deecompression parameter
580 options[DecompressionParameter.window_log_max] = C_INT_MAX
581 with self.assertRaises(ValueError):
582 decompress(b'', options=options)
583 options[DecompressionParameter.window_log_max] = C_INT_MIN
584 with self.assertRaises(ValueError):
585 decompress(b'', options=options)
586 options[DecompressionParameter.window_log_max] = 2**1000
587 with self.assertRaises(OverflowError):
588 decompress(b'', options=options)
589 options[DecompressionParameter.window_log_max] = -(2**1000)
590 with self.assertRaises(OverflowError):
591 decompress(b'', options=options)
592
593 def test_unknown_decompression_parameter(self):
594 KEY = 100001234
595 options = {DecompressionParameter.window_log_max: DecompressionParameter.window_log_max.bounds()[1],
596 KEY: 200000000}
597 pattern = rf"invalid decompression parameter 'unknown parameter \(key {KEY}\)'"
598 with self.assertRaisesRegex(ValueError, pattern):
599 ZstdDecompressor(options=options)
600
601 def test_decompress_epilogue_flags(self):
602 # DAT_130K_C has a 4 bytes checksum at frame epilogue
603
604 # full unlimited
605 d = ZstdDecompressor()
606 dat = d.decompress(DAT_130K_C)
607 self.assertEqual(len(dat), _130_1K)
608 self.assertFalse(d.needs_input)
609
610 with self.assertRaises(EOFError):
611 dat = d.decompress(b'')
612
613 # full limited
614 d = ZstdDecompressor()
615 dat = d.decompress(DAT_130K_C, _130_1K)
616 self.assertEqual(len(dat), _130_1K)
617 self.assertFalse(d.needs_input)
618
619 with self.assertRaises(EOFError):
620 dat = d.decompress(b'', 0)
621
622 # [:-4] unlimited
623 d = ZstdDecompressor()
624 dat = d.decompress(DAT_130K_C[:-4])
625 self.assertEqual(len(dat), _130_1K)
626 self.assertTrue(d.needs_input)
627
628 dat = d.decompress(b'')
629 self.assertEqual(len(dat), 0)
630 self.assertTrue(d.needs_input)
631
632 # [:-4] limited
633 d = ZstdDecompressor()
634 dat = d.decompress(DAT_130K_C[:-4], _130_1K)
635 self.assertEqual(len(dat), _130_1K)
636 self.assertFalse(d.needs_input)
637
638 dat = d.decompress(b'', 0)
639 self.assertEqual(len(dat), 0)
640 self.assertFalse(d.needs_input)
641
642 # [:-3] unlimited
643 d = ZstdDecompressor()
644 dat = d.decompress(DAT_130K_C[:-3])
645 self.assertEqual(len(dat), _130_1K)
646 self.assertTrue(d.needs_input)
647
648 dat = d.decompress(b'')
649 self.assertEqual(len(dat), 0)
650 self.assertTrue(d.needs_input)
651
652 # [:-3] limited
653 d = ZstdDecompressor()
654 dat = d.decompress(DAT_130K_C[:-3], _130_1K)
655 self.assertEqual(len(dat), _130_1K)
656 self.assertFalse(d.needs_input)
657
658 dat = d.decompress(b'', 0)
659 self.assertEqual(len(dat), 0)
660 self.assertFalse(d.needs_input)
661
662 # [:-1] unlimited
663 d = ZstdDecompressor()
664 dat = d.decompress(DAT_130K_C[:-1])
665 self.assertEqual(len(dat), _130_1K)
666 self.assertTrue(d.needs_input)
667
668 dat = d.decompress(b'')
669 self.assertEqual(len(dat), 0)
670 self.assertTrue(d.needs_input)
671
672 # [:-1] limited
673 d = ZstdDecompressor()
674 dat = d.decompress(DAT_130K_C[:-1], _130_1K)
675 self.assertEqual(len(dat), _130_1K)
676 self.assertFalse(d.needs_input)
677
678 dat = d.decompress(b'', 0)
679 self.assertEqual(len(dat), 0)
680 self.assertFalse(d.needs_input)
681
682 def test_decompressor_arg(self):
683 zd = ZstdDict(b'12345678', is_raw=True)
684
685 with self.assertRaises(TypeError):
686 d = ZstdDecompressor(zstd_dict={})
687
688 with self.assertRaises(TypeError):
689 d = ZstdDecompressor(options=zd)
690
691 ZstdDecompressor()
692 ZstdDecompressor(zd, {})
693 ZstdDecompressor(zstd_dict=zd, options={DecompressionParameter.window_log_max:25})
694
695 def test_decompressor_1(self):
696 # empty
697 d = ZstdDecompressor()
698 dat = d.decompress(b'')
699
700 self.assertEqual(dat, b'')
701 self.assertFalse(d.eof)
702
703 # 130_1K full
704 d = ZstdDecompressor()
705 dat = d.decompress(DAT_130K_C)
706
707 self.assertEqual(len(dat), _130_1K)
708 self.assertTrue(d.eof)
709 self.assertFalse(d.needs_input)
710
711 # 130_1K full, limit output
712 d = ZstdDecompressor()
713 dat = d.decompress(DAT_130K_C, _130_1K)
714
715 self.assertEqual(len(dat), _130_1K)
716 self.assertTrue(d.eof)
717 self.assertFalse(d.needs_input)
718
719 # 130_1K, without 4 bytes checksum
720 d = ZstdDecompressor()
721 dat = d.decompress(DAT_130K_C[:-4])
722
723 self.assertEqual(len(dat), _130_1K)
724 self.assertFalse(d.eof)
725 self.assertTrue(d.needs_input)
726
727 # above, limit output
728 d = ZstdDecompressor()
729 dat = d.decompress(DAT_130K_C[:-4], _130_1K)
730
731 self.assertEqual(len(dat), _130_1K)
732 self.assertFalse(d.eof)
733 self.assertFalse(d.needs_input)
734
735 # full, unused_data
736 TRAIL = b'89234893abcd'
737 d = ZstdDecompressor()
738 dat = d.decompress(DAT_130K_C + TRAIL, _130_1K)
739
740 self.assertEqual(len(dat), _130_1K)
741 self.assertTrue(d.eof)
742 self.assertFalse(d.needs_input)
743 self.assertEqual(d.unused_data, TRAIL)
744
745 def test_decompressor_chunks_read_300(self):
746 TRAIL = b'89234893abcd'
747 DAT = DAT_130K_C + TRAIL
748 d = ZstdDecompressor()
749
750 bi = io.BytesIO(DAT)
751 lst = []
752 while True:
753 if d.needs_input:
754 dat = bi.read(300)
755 if not dat:
756 break
757 else:
758 raise Exception('should not get here')
759
760 ret = d.decompress(dat)
761 lst.append(ret)
762 if d.eof:
763 break
764
765 ret = b''.join(lst)
766
767 self.assertEqual(len(ret), _130_1K)
768 self.assertTrue(d.eof)
769 self.assertFalse(d.needs_input)
770 self.assertEqual(d.unused_data + bi.read(), TRAIL)
771
772 def test_decompressor_chunks_read_3(self):
773 TRAIL = b'89234893'
774 DAT = DAT_130K_C + TRAIL
775 d = ZstdDecompressor()
776
777 bi = io.BytesIO(DAT)
778 lst = []
779 while True:
780 if d.needs_input:
781 dat = bi.read(3)
782 if not dat:
783 break
784 else:
785 dat = b''
786
787 ret = d.decompress(dat, 1)
788 lst.append(ret)
789 if d.eof:
790 break
791
792 ret = b''.join(lst)
793
794 self.assertEqual(len(ret), _130_1K)
795 self.assertTrue(d.eof)
796 self.assertFalse(d.needs_input)
797 self.assertEqual(d.unused_data + bi.read(), TRAIL)
798
799
800 def test_decompress_empty(self):
801 with self.assertRaises(ZstdError):
802 decompress(b'')
803
804 d = ZstdDecompressor()
805 self.assertEqual(d.decompress(b''), b'')
806 self.assertFalse(d.eof)
807
808 def test_decompress_empty_content_frame(self):
809 DAT = compress(b'')
810 # decompress
811 self.assertGreaterEqual(len(DAT), 4)
812 self.assertEqual(decompress(DAT), b'')
813
814 with self.assertRaises(ZstdError):
815 decompress(DAT[:-1])
816
817 # ZstdDecompressor
818 d = ZstdDecompressor()
819 dat = d.decompress(DAT)
820 self.assertEqual(dat, b'')
821 self.assertTrue(d.eof)
822 self.assertFalse(d.needs_input)
823 self.assertEqual(d.unused_data, b'')
824 self.assertEqual(d.unused_data, b'') # twice
825
826 d = ZstdDecompressor()
827 dat = d.decompress(DAT[:-1])
828 self.assertEqual(dat, b'')
829 self.assertFalse(d.eof)
830 self.assertTrue(d.needs_input)
831 self.assertEqual(d.unused_data, b'')
832 self.assertEqual(d.unused_data, b'') # twice
833
834 class DecompressorFlagsTestCase(unittest.TestCase):
835
836 @classmethod
837 def setUpClass(cls):
838 options = {CompressionParameter.checksum_flag:1}
839 c = ZstdCompressor(options=options)
840
841 cls.DECOMPRESSED_42 = b'a'*42
842 cls.FRAME_42 = c.compress(cls.DECOMPRESSED_42, c.FLUSH_FRAME)
843
844 cls.DECOMPRESSED_60 = b'a'*60
845 cls.FRAME_60 = c.compress(cls.DECOMPRESSED_60, c.FLUSH_FRAME)
846
847 cls.FRAME_42_60 = cls.FRAME_42 + cls.FRAME_60
848 cls.DECOMPRESSED_42_60 = cls.DECOMPRESSED_42 + cls.DECOMPRESSED_60
849
850 cls._130_1K = 130*_1K
851
852 c = ZstdCompressor()
853 cls.UNKNOWN_FRAME_42 = c.compress(cls.DECOMPRESSED_42) + c.flush()
854 cls.UNKNOWN_FRAME_60 = c.compress(cls.DECOMPRESSED_60) + c.flush()
855 cls.UNKNOWN_FRAME_42_60 = cls.UNKNOWN_FRAME_42 + cls.UNKNOWN_FRAME_60
856
857 cls.TRAIL = b'12345678abcdefg!@#$%^&*()_+|'
858
859 def test_function_decompress(self):
860
861 self.assertEqual(len(decompress(COMPRESSED_100_PLUS_32KB)), 100+32*_1K)
862
863 # 1 frame
864 self.assertEqual(decompress(self.FRAME_42), self.DECOMPRESSED_42)
865
866 self.assertEqual(decompress(self.UNKNOWN_FRAME_42), self.DECOMPRESSED_42)
867
868 pattern = r"Compressed data ended before the end-of-stream marker"
869 with self.assertRaisesRegex(ZstdError, pattern):
870 decompress(self.FRAME_42[:1])
871
872 with self.assertRaisesRegex(ZstdError, pattern):
873 decompress(self.FRAME_42[:-4])
874
875 with self.assertRaisesRegex(ZstdError, pattern):
876 decompress(self.FRAME_42[:-1])
877
878 # 2 frames
879 self.assertEqual(decompress(self.FRAME_42_60), self.DECOMPRESSED_42_60)
880
881 self.assertEqual(decompress(self.UNKNOWN_FRAME_42_60), self.DECOMPRESSED_42_60)
882
883 self.assertEqual(decompress(self.FRAME_42 + self.UNKNOWN_FRAME_60),
884 self.DECOMPRESSED_42_60)
885
886 self.assertEqual(decompress(self.UNKNOWN_FRAME_42 + self.FRAME_60),
887 self.DECOMPRESSED_42_60)
888
889 with self.assertRaisesRegex(ZstdError, pattern):
890 decompress(self.FRAME_42_60[:-4])
891
892 with self.assertRaisesRegex(ZstdError, pattern):
893 decompress(self.UNKNOWN_FRAME_42_60[:-1])
894
895 # 130_1K
896 self.assertEqual(decompress(DAT_130K_C), DAT_130K_D)
897
898 with self.assertRaisesRegex(ZstdError, pattern):
899 decompress(DAT_130K_C[:-4])
900
901 with self.assertRaisesRegex(ZstdError, pattern):
902 decompress(DAT_130K_C[:-1])
903
904 # Unknown frame descriptor
905 with self.assertRaisesRegex(ZstdError, "Unknown frame descriptor"):
906 decompress(b'aaaaaaaaa')
907
908 with self.assertRaisesRegex(ZstdError, "Unknown frame descriptor"):
909 decompress(self.FRAME_42 + b'aaaaaaaaa')
910
911 with self.assertRaisesRegex(ZstdError, "Unknown frame descriptor"):
912 decompress(self.UNKNOWN_FRAME_42_60 + b'aaaaaaaaa')
913
914 # doesn't match checksum
915 checksum = DAT_130K_C[-4:]
916 if checksum[0] == 255:
917 wrong_checksum = bytes([254]) + checksum[1:]
918 else:
919 wrong_checksum = bytes([checksum[0]+1]) + checksum[1:]
920
921 dat = DAT_130K_C[:-4] + wrong_checksum
922
923 with self.assertRaisesRegex(ZstdError, "doesn't match checksum"):
924 decompress(dat)
925
926 def test_function_skippable(self):
927 self.assertEqual(decompress(SKIPPABLE_FRAME), b'')
928 self.assertEqual(decompress(SKIPPABLE_FRAME + SKIPPABLE_FRAME), b'')
929
930 # 1 frame + 2 skippable
931 self.assertEqual(len(decompress(SKIPPABLE_FRAME + SKIPPABLE_FRAME + DAT_130K_C)),
932 self._130_1K)
933
934 self.assertEqual(len(decompress(DAT_130K_C + SKIPPABLE_FRAME + SKIPPABLE_FRAME)),
935 self._130_1K)
936
937 self.assertEqual(len(decompress(SKIPPABLE_FRAME + DAT_130K_C + SKIPPABLE_FRAME)),
938 self._130_1K)
939
940 # unknown size
941 self.assertEqual(decompress(SKIPPABLE_FRAME + self.UNKNOWN_FRAME_60),
942 self.DECOMPRESSED_60)
943
944 self.assertEqual(decompress(self.UNKNOWN_FRAME_60 + SKIPPABLE_FRAME),
945 self.DECOMPRESSED_60)
946
947 # 2 frames + 1 skippable
948 self.assertEqual(decompress(self.FRAME_42 + SKIPPABLE_FRAME + self.FRAME_60),
949 self.DECOMPRESSED_42_60)
950
951 self.assertEqual(decompress(SKIPPABLE_FRAME + self.FRAME_42_60),
952 self.DECOMPRESSED_42_60)
953
954 self.assertEqual(decompress(self.UNKNOWN_FRAME_42_60 + SKIPPABLE_FRAME),
955 self.DECOMPRESSED_42_60)
956
957 # incomplete
958 with self.assertRaises(ZstdError):
959 decompress(SKIPPABLE_FRAME[:1])
960
961 with self.assertRaises(ZstdError):
962 decompress(SKIPPABLE_FRAME[:-1])
963
964 with self.assertRaises(ZstdError):
965 decompress(self.FRAME_42 + SKIPPABLE_FRAME[:-1])
966
967 # Unknown frame descriptor
968 with self.assertRaisesRegex(ZstdError, "Unknown frame descriptor"):
969 decompress(b'aaaaaaaaa' + SKIPPABLE_FRAME)
970
971 with self.assertRaisesRegex(ZstdError, "Unknown frame descriptor"):
972 decompress(SKIPPABLE_FRAME + b'aaaaaaaaa')
973
974 with self.assertRaisesRegex(ZstdError, "Unknown frame descriptor"):
975 decompress(SKIPPABLE_FRAME + SKIPPABLE_FRAME + b'aaaaaaaaa')
976
977 def test_decompressor_1(self):
978 # empty 1
979 d = ZstdDecompressor()
980
981 dat = d.decompress(b'')
982 self.assertEqual(dat, b'')
983 self.assertFalse(d.eof)
984 self.assertTrue(d.needs_input)
985 self.assertEqual(d.unused_data, b'')
986 self.assertEqual(d.unused_data, b'') # twice
987
988 dat = d.decompress(b'', 0)
989 self.assertEqual(dat, b'')
990 self.assertFalse(d.eof)
991 self.assertFalse(d.needs_input)
992 self.assertEqual(d.unused_data, b'')
993 self.assertEqual(d.unused_data, b'') # twice
994
995 dat = d.decompress(COMPRESSED_100_PLUS_32KB + b'a')
996 self.assertEqual(dat, DECOMPRESSED_100_PLUS_32KB)
997 self.assertTrue(d.eof)
998 self.assertFalse(d.needs_input)
999 self.assertEqual(d.unused_data, b'a')
1000 self.assertEqual(d.unused_data, b'a') # twice
1001
1002 # empty 2
1003 d = ZstdDecompressor()
1004
1005 dat = d.decompress(b'', 0)
1006 self.assertEqual(dat, b'')
1007 self.assertFalse(d.eof)
1008 self.assertFalse(d.needs_input)
1009 self.assertEqual(d.unused_data, b'')
1010 self.assertEqual(d.unused_data, b'') # twice
1011
1012 dat = d.decompress(b'')
1013 self.assertEqual(dat, b'')
1014 self.assertFalse(d.eof)
1015 self.assertTrue(d.needs_input)
1016 self.assertEqual(d.unused_data, b'')
1017 self.assertEqual(d.unused_data, b'') # twice
1018
1019 dat = d.decompress(COMPRESSED_100_PLUS_32KB + b'a')
1020 self.assertEqual(dat, DECOMPRESSED_100_PLUS_32KB)
1021 self.assertTrue(d.eof)
1022 self.assertFalse(d.needs_input)
1023 self.assertEqual(d.unused_data, b'a')
1024 self.assertEqual(d.unused_data, b'a') # twice
1025
1026 # 1 frame
1027 d = ZstdDecompressor()
1028 dat = d.decompress(self.FRAME_42)
1029
1030 self.assertEqual(dat, self.DECOMPRESSED_42)
1031 self.assertTrue(d.eof)
1032 self.assertFalse(d.needs_input)
1033 self.assertEqual(d.unused_data, b'')
1034 self.assertEqual(d.unused_data, b'') # twice
1035
1036 with self.assertRaises(EOFError):
1037 d.decompress(b'')
1038
1039 # 1 frame, trail
1040 d = ZstdDecompressor()
1041 dat = d.decompress(self.FRAME_42 + self.TRAIL)
1042
1043 self.assertEqual(dat, self.DECOMPRESSED_42)
1044 self.assertTrue(d.eof)
1045 self.assertFalse(d.needs_input)
1046 self.assertEqual(d.unused_data, self.TRAIL)
1047 self.assertEqual(d.unused_data, self.TRAIL) # twice
1048
1049 # 1 frame, 32_1K
1050 temp = compress(b'a'*(32*_1K))
1051 d = ZstdDecompressor()
1052 dat = d.decompress(temp, 32*_1K)
1053
1054 self.assertEqual(dat, b'a'*(32*_1K))
1055 self.assertTrue(d.eof)
1056 self.assertFalse(d.needs_input)
1057 self.assertEqual(d.unused_data, b'')
1058 self.assertEqual(d.unused_data, b'') # twice
1059
1060 with self.assertRaises(EOFError):
1061 d.decompress(b'')
1062
1063 # 1 frame, 32_1K+100, trail
1064 d = ZstdDecompressor()
1065 dat = d.decompress(COMPRESSED_100_PLUS_32KB+self.TRAIL, 100) # 100 bytes
1066
1067 self.assertEqual(len(dat), 100)
1068 self.assertFalse(d.eof)
1069 self.assertFalse(d.needs_input)
1070 self.assertEqual(d.unused_data, b'')
1071
1072 dat = d.decompress(b'') # 32_1K
1073
1074 self.assertEqual(len(dat), 32*_1K)
1075 self.assertTrue(d.eof)
1076 self.assertFalse(d.needs_input)
1077 self.assertEqual(d.unused_data, self.TRAIL)
1078 self.assertEqual(d.unused_data, self.TRAIL) # twice
1079
1080 with self.assertRaises(EOFError):
1081 d.decompress(b'')
1082
1083 # incomplete 1
1084 d = ZstdDecompressor()
1085 dat = d.decompress(self.FRAME_60[:1])
1086
1087 self.assertFalse(d.eof)
1088 self.assertTrue(d.needs_input)
1089 self.assertEqual(d.unused_data, b'')
1090 self.assertEqual(d.unused_data, b'') # twice
1091
1092 # incomplete 2
1093 d = ZstdDecompressor()
1094
1095 dat = d.decompress(self.FRAME_60[:-4])
1096 self.assertEqual(dat, self.DECOMPRESSED_60)
1097 self.assertFalse(d.eof)
1098 self.assertTrue(d.needs_input)
1099 self.assertEqual(d.unused_data, b'')
1100 self.assertEqual(d.unused_data, b'') # twice
1101
1102 # incomplete 3
1103 d = ZstdDecompressor()
1104
1105 dat = d.decompress(self.FRAME_60[:-1])
1106 self.assertEqual(dat, self.DECOMPRESSED_60)
1107 self.assertFalse(d.eof)
1108 self.assertTrue(d.needs_input)
1109 self.assertEqual(d.unused_data, b'')
1110
1111 # incomplete 4
1112 d = ZstdDecompressor()
1113
1114 dat = d.decompress(self.FRAME_60[:-4], 60)
1115 self.assertEqual(dat, self.DECOMPRESSED_60)
1116 self.assertFalse(d.eof)
1117 self.assertFalse(d.needs_input)
1118 self.assertEqual(d.unused_data, b'')
1119 self.assertEqual(d.unused_data, b'') # twice
1120
1121 dat = d.decompress(b'')
1122 self.assertEqual(dat, b'')
1123 self.assertFalse(d.eof)
1124 self.assertTrue(d.needs_input)
1125 self.assertEqual(d.unused_data, b'')
1126 self.assertEqual(d.unused_data, b'') # twice
1127
1128 # Unknown frame descriptor
1129 d = ZstdDecompressor()
1130 with self.assertRaisesRegex(ZstdError, "Unknown frame descriptor"):
1131 d.decompress(b'aaaaaaaaa')
1132
1133 def test_decompressor_skippable(self):
1134 # 1 skippable
1135 d = ZstdDecompressor()
1136 dat = d.decompress(SKIPPABLE_FRAME)
1137
1138 self.assertEqual(dat, b'')
1139 self.assertTrue(d.eof)
1140 self.assertFalse(d.needs_input)
1141 self.assertEqual(d.unused_data, b'')
1142 self.assertEqual(d.unused_data, b'') # twice
1143
1144 # 1 skippable, max_length=0
1145 d = ZstdDecompressor()
1146 dat = d.decompress(SKIPPABLE_FRAME, 0)
1147
1148 self.assertEqual(dat, b'')
1149 self.assertTrue(d.eof)
1150 self.assertFalse(d.needs_input)
1151 self.assertEqual(d.unused_data, b'')
1152 self.assertEqual(d.unused_data, b'') # twice
1153
1154 # 1 skippable, trail
1155 d = ZstdDecompressor()
1156 dat = d.decompress(SKIPPABLE_FRAME + self.TRAIL)
1157
1158 self.assertEqual(dat, b'')
1159 self.assertTrue(d.eof)
1160 self.assertFalse(d.needs_input)
1161 self.assertEqual(d.unused_data, self.TRAIL)
1162 self.assertEqual(d.unused_data, self.TRAIL) # twice
1163
1164 # incomplete
1165 d = ZstdDecompressor()
1166 dat = d.decompress(SKIPPABLE_FRAME[:-1])
1167
1168 self.assertEqual(dat, b'')
1169 self.assertFalse(d.eof)
1170 self.assertTrue(d.needs_input)
1171 self.assertEqual(d.unused_data, b'')
1172 self.assertEqual(d.unused_data, b'') # twice
1173
1174 # incomplete
1175 d = ZstdDecompressor()
1176 dat = d.decompress(SKIPPABLE_FRAME[:-1], 0)
1177
1178 self.assertEqual(dat, b'')
1179 self.assertFalse(d.eof)
1180 self.assertFalse(d.needs_input)
1181 self.assertEqual(d.unused_data, b'')
1182 self.assertEqual(d.unused_data, b'') # twice
1183
1184 dat = d.decompress(b'')
1185
1186 self.assertEqual(dat, b'')
1187 self.assertFalse(d.eof)
1188 self.assertTrue(d.needs_input)
1189 self.assertEqual(d.unused_data, b'')
1190 self.assertEqual(d.unused_data, b'') # twice
1191
1192
1193
1194 class ZstdDictTestCase(unittest.TestCase):
1195
1196 def test_is_raw(self):
1197 # must be passed as a keyword argument
1198 with self.assertRaises(TypeError):
1199 ZstdDict(bytes(8), True)
1200
1201 # content < 8
1202 b = b'1234567'
1203 with self.assertRaises(ValueError):
1204 ZstdDict(b)
1205
1206 # content == 8
1207 b = b'12345678'
1208 zd = ZstdDict(b, is_raw=True)
1209 self.assertEqual(zd.dict_id, 0)
1210
1211 temp = compress(b'aaa12345678', level=3, zstd_dict=zd)
1212 self.assertEqual(b'aaa12345678', decompress(temp, zd))
1213
1214 # is_raw == False
1215 b = b'12345678abcd'
1216 with self.assertRaises(ValueError):
1217 ZstdDict(b)
1218
1219 # read only attributes
1220 with self.assertRaises(AttributeError):
1221 zd.dict_content = b
1222
1223 with self.assertRaises(AttributeError):
1224 zd.dict_id = 10000
1225
1226 # ZstdDict arguments
1227 zd = ZstdDict(TRAINED_DICT.dict_content, is_raw=False)
1228 self.assertNotEqual(zd.dict_id, 0)
1229
1230 zd = ZstdDict(TRAINED_DICT.dict_content, is_raw=True)
1231 self.assertNotEqual(zd.dict_id, 0) # note this assertion
1232
1233 with self.assertRaises(TypeError):
1234 ZstdDict("12345678abcdef", is_raw=True)
1235 with self.assertRaises(TypeError):
1236 ZstdDict(TRAINED_DICT)
1237
1238 # invalid parameter
1239 with self.assertRaises(TypeError):
1240 ZstdDict(desk333=345)
1241
1242 def test_invalid_dict(self):
1243 DICT_MAGIC = 0xEC30A437.to_bytes(4, byteorder='little')
1244 dict_content = DICT_MAGIC + b'abcdefghighlmnopqrstuvwxyz'
1245
1246 # corrupted
1247 zd = ZstdDict(dict_content, is_raw=False)
1248 with self.assertRaisesRegex(ZstdError, r'ZSTD_CDict.*?content\.$'):
1249 ZstdCompressor(zstd_dict=zd.as_digested_dict)
1250 with self.assertRaisesRegex(ZstdError, r'ZSTD_DDict.*?content\.$'):
1251 ZstdDecompressor(zd)
1252
1253 # wrong type
1254 with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'):
1255 ZstdCompressor(zstd_dict=[zd, 1])
1256 with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'):
1257 ZstdCompressor(zstd_dict=(zd, 1.0))
1258 with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'):
1259 ZstdCompressor(zstd_dict=(zd,))
1260 with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'):
1261 ZstdCompressor(zstd_dict=(zd, 1, 2))
1262 with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'):
1263 ZstdCompressor(zstd_dict=(zd, -1))
1264 with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'):
1265 ZstdCompressor(zstd_dict=(zd, 3))
1266 with self.assertRaises(OverflowError):
1267 ZstdCompressor(zstd_dict=(zd, 2**1000))
1268 with self.assertRaises(OverflowError):
1269 ZstdCompressor(zstd_dict=(zd, -2**1000))
1270
1271 with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'):
1272 ZstdDecompressor(zstd_dict=[zd, 1])
1273 with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'):
1274 ZstdDecompressor(zstd_dict=(zd, 1.0))
1275 with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'):
1276 ZstdDecompressor((zd,))
1277 with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'):
1278 ZstdDecompressor((zd, 1, 2))
1279 with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'):
1280 ZstdDecompressor((zd, -1))
1281 with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'):
1282 ZstdDecompressor((zd, 3))
1283 with self.assertRaises(OverflowError):
1284 ZstdDecompressor((zd, 2**1000))
1285 with self.assertRaises(OverflowError):
1286 ZstdDecompressor((zd, -2**1000))
1287
1288 def test_train_dict(self):
1289 TRAINED_DICT = train_dict(SAMPLES, DICT_SIZE1)
1290 ZstdDict(TRAINED_DICT.dict_content, is_raw=False)
1291
1292 self.assertNotEqual(TRAINED_DICT.dict_id, 0)
1293 self.assertGreater(len(TRAINED_DICT.dict_content), 0)
1294 self.assertLessEqual(len(TRAINED_DICT.dict_content), DICT_SIZE1)
1295 self.assertTrue(re.match(r'^<ZstdDict dict_id=\d+ dict_size=\d+>$', str(TRAINED_DICT)))
1296
1297 # compress/decompress
1298 c = ZstdCompressor(zstd_dict=TRAINED_DICT)
1299 for sample in SAMPLES:
1300 dat1 = compress(sample, zstd_dict=TRAINED_DICT)
1301 dat2 = decompress(dat1, TRAINED_DICT)
1302 self.assertEqual(sample, dat2)
1303
1304 dat1 = c.compress(sample)
1305 dat1 += c.flush()
1306 dat2 = decompress(dat1, TRAINED_DICT)
1307 self.assertEqual(sample, dat2)
1308
1309 def test_finalize_dict(self):
1310 DICT_SIZE2 = 200*_1K
1311 C_LEVEL = 6
1312
1313 try:
1314 dic2 = finalize_dict(TRAINED_DICT, SAMPLES, DICT_SIZE2, C_LEVEL)
1315 except NotImplementedError:
1316 # < v1.4.5 at compile-time, >= v.1.4.5 at run-time
1317 return
1318
1319 self.assertNotEqual(dic2.dict_id, 0)
1320 self.assertGreater(len(dic2.dict_content), 0)
1321 self.assertLessEqual(len(dic2.dict_content), DICT_SIZE2)
1322
1323 # compress/decompress
1324 c = ZstdCompressor(C_LEVEL, zstd_dict=dic2)
1325 for sample in SAMPLES:
1326 dat1 = compress(sample, C_LEVEL, zstd_dict=dic2)
1327 dat2 = decompress(dat1, dic2)
1328 self.assertEqual(sample, dat2)
1329
1330 dat1 = c.compress(sample)
1331 dat1 += c.flush()
1332 dat2 = decompress(dat1, dic2)
1333 self.assertEqual(sample, dat2)
1334
1335 # dict mismatch
1336 self.assertNotEqual(TRAINED_DICT.dict_id, dic2.dict_id)
1337
1338 dat1 = compress(SAMPLES[0], zstd_dict=TRAINED_DICT)
1339 with self.assertRaises(ZstdError):
1340 decompress(dat1, dic2)
1341
1342 def test_train_dict_arguments(self):
1343 with self.assertRaises(ValueError):
1344 train_dict([], 100*_1K)
1345
1346 with self.assertRaises(ValueError):
1347 train_dict(SAMPLES, -100)
1348
1349 with self.assertRaises(ValueError):
1350 train_dict(SAMPLES, 0)
1351
1352 def test_finalize_dict_arguments(self):
1353 with self.assertRaises(TypeError):
1354 finalize_dict({1:2}, (b'aaa', b'bbb'), 100*_1K, 2)
1355
1356 with self.assertRaises(ValueError):
1357 finalize_dict(TRAINED_DICT, [], 100*_1K, 2)
1358
1359 with self.assertRaises(ValueError):
1360 finalize_dict(TRAINED_DICT, SAMPLES, -100, 2)
1361
1362 with self.assertRaises(ValueError):
1363 finalize_dict(TRAINED_DICT, SAMPLES, 0, 2)
1364
1365 def test_train_dict_c(self):
1366 # argument wrong type
1367 with self.assertRaises(TypeError):
1368 _zstd.train_dict({}, (), 100)
1369 with self.assertRaises(TypeError):
1370 _zstd.train_dict(bytearray(), (), 100)
1371 with self.assertRaises(TypeError):
1372 _zstd.train_dict(b'', 99, 100)
1373 with self.assertRaises(TypeError):
1374 _zstd.train_dict(b'', [], 100)
1375 with self.assertRaises(TypeError):
1376 _zstd.train_dict(b'', (), 100.1)
1377 with self.assertRaises(TypeError):
1378 _zstd.train_dict(b'', (99.1,), 100)
1379 with self.assertRaises(ValueError):
1380 _zstd.train_dict(b'abc', (4, -1), 100)
1381 with self.assertRaises(ValueError):
1382 _zstd.train_dict(b'abc', (2,), 100)
1383 with self.assertRaises(ValueError):
1384 _zstd.train_dict(b'', (99,), 100)
1385
1386 # size > size_t
1387 with self.assertRaises(ValueError):
1388 _zstd.train_dict(b'', (2**1000,), 100)
1389 with self.assertRaises(ValueError):
1390 _zstd.train_dict(b'', (-2**1000,), 100)
1391
1392 # dict_size <= 0
1393 with self.assertRaises(ValueError):
1394 _zstd.train_dict(b'', (), 0)
1395 with self.assertRaises(ValueError):
1396 _zstd.train_dict(b'', (), -1)
1397
1398 with self.assertRaises(ZstdError):
1399 _zstd.train_dict(b'', (), 1)
1400
1401 def test_finalize_dict_c(self):
1402 with self.assertRaises(TypeError):
1403 _zstd.finalize_dict(1, 2, 3, 4, 5)
1404
1405 # argument wrong type
1406 with self.assertRaises(TypeError):
1407 _zstd.finalize_dict({}, b'', (), 100, 5)
1408 with self.assertRaises(TypeError):
1409 _zstd.finalize_dict(bytearray(TRAINED_DICT.dict_content), b'', (), 100, 5)
1410 with self.assertRaises(TypeError):
1411 _zstd.finalize_dict(TRAINED_DICT.dict_content, {}, (), 100, 5)
1412 with self.assertRaises(TypeError):
1413 _zstd.finalize_dict(TRAINED_DICT.dict_content, bytearray(), (), 100, 5)
1414 with self.assertRaises(TypeError):
1415 _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', 99, 100, 5)
1416 with self.assertRaises(TypeError):
1417 _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', [], 100, 5)
1418 with self.assertRaises(TypeError):
1419 _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), 100.1, 5)
1420 with self.assertRaises(TypeError):
1421 _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), 100, 5.1)
1422
1423 with self.assertRaises(ValueError):
1424 _zstd.finalize_dict(TRAINED_DICT.dict_content, b'abc', (4, -1), 100, 5)
1425 with self.assertRaises(ValueError):
1426 _zstd.finalize_dict(TRAINED_DICT.dict_content, b'abc', (2,), 100, 5)
1427 with self.assertRaises(ValueError):
1428 _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (99,), 100, 5)
1429
1430 # size > size_t
1431 with self.assertRaises(ValueError):
1432 _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (2**1000,), 100, 5)
1433 with self.assertRaises(ValueError):
1434 _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (-2**1000,), 100, 5)
1435
1436 # dict_size <= 0
1437 with self.assertRaises(ValueError):
1438 _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), 0, 5)
1439 with self.assertRaises(ValueError):
1440 _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), -1, 5)
1441 with self.assertRaises(OverflowError):
1442 _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), 2**1000, 5)
1443 with self.assertRaises(OverflowError):
1444 _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), -2**1000, 5)
1445
1446 with self.assertRaises(OverflowError):
1447 _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), 100, 2**1000)
1448 with self.assertRaises(OverflowError):
1449 _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), 100, -2**1000)
1450
1451 with self.assertRaises(ZstdError):
1452 _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), 100, 5)
1453
1454 def test_train_buffer_protocol_samples(self):
1455 def _nbytes(dat):
1456 if isinstance(dat, (bytes, bytearray)):
1457 return len(dat)
1458 return memoryview(dat).nbytes
1459
1460 # prepare samples
1461 chunk_lst = []
1462 wrong_size_lst = []
1463 correct_size_lst = []
1464 for _ in range(300):
1465 arr = array.array('Q', [random.randint(0, 20) for i in range(20)])
1466 chunk_lst.append(arr)
1467 correct_size_lst.append(_nbytes(arr))
1468 wrong_size_lst.append(len(arr))
1469 concatenation = b''.join(chunk_lst)
1470
1471 # wrong size list
1472 with self.assertRaisesRegex(ValueError,
1473 "The samples size tuple doesn't match the concatenation's size"):
1474 _zstd.train_dict(concatenation, tuple(wrong_size_lst), 100*_1K)
1475
1476 # correct size list
1477 _zstd.train_dict(concatenation, tuple(correct_size_lst), 3*_1K)
1478
1479 # wrong size list
1480 with self.assertRaisesRegex(ValueError,
1481 "The samples size tuple doesn't match the concatenation's size"):
1482 _zstd.finalize_dict(TRAINED_DICT.dict_content,
1483 concatenation, tuple(wrong_size_lst), 300*_1K, 5)
1484
1485 # correct size list
1486 _zstd.finalize_dict(TRAINED_DICT.dict_content,
1487 concatenation, tuple(correct_size_lst), 300*_1K, 5)
1488
1489 def test_as_prefix(self):
1490 # V1
1491 V1 = THIS_FILE_BYTES
1492 zd = ZstdDict(V1, is_raw=True)
1493
1494 # V2
1495 mid = len(V1) // 2
1496 V2 = V1[:mid] + \
1497 (b'a' if V1[mid] != int.from_bytes(b'a') else b'b') + \
1498 V1[mid+1:]
1499
1500 # compress
1501 dat = compress(V2, zstd_dict=zd.as_prefix)
1502 self.assertEqual(get_frame_info(dat).dictionary_id, 0)
1503
1504 # decompress
1505 self.assertEqual(decompress(dat, zd.as_prefix), V2)
1506
1507 # use wrong prefix
1508 zd2 = ZstdDict(SAMPLES[0], is_raw=True)
1509 try:
1510 decompressed = decompress(dat, zd2.as_prefix)
1511 except ZstdError: # expected
1512 pass
1513 else:
1514 self.assertNotEqual(decompressed, V2)
1515
1516 # read only attribute
1517 with self.assertRaises(AttributeError):
1518 zd.as_prefix = b'1234'
1519
1520 def test_as_digested_dict(self):
1521 zd = TRAINED_DICT
1522
1523 # test .as_digested_dict
1524 dat = compress(SAMPLES[0], zstd_dict=zd.as_digested_dict)
1525 self.assertEqual(decompress(dat, zd.as_digested_dict), SAMPLES[0])
1526 with self.assertRaises(AttributeError):
1527 zd.as_digested_dict = b'1234'
1528
1529 # test .as_undigested_dict
1530 dat = compress(SAMPLES[0], zstd_dict=zd.as_undigested_dict)
1531 self.assertEqual(decompress(dat, zd.as_undigested_dict), SAMPLES[0])
1532 with self.assertRaises(AttributeError):
1533 zd.as_undigested_dict = b'1234'
1534
1535 def test_advanced_compression_parameters(self):
1536 options = {CompressionParameter.compression_level: 6,
1537 CompressionParameter.window_log: 20,
1538 CompressionParameter.enable_long_distance_matching: 1}
1539
1540 # automatically select
1541 dat = compress(SAMPLES[0], options=options, zstd_dict=TRAINED_DICT)
1542 self.assertEqual(decompress(dat, TRAINED_DICT), SAMPLES[0])
1543
1544 # explicitly select
1545 dat = compress(SAMPLES[0], options=options, zstd_dict=TRAINED_DICT.as_digested_dict)
1546 self.assertEqual(decompress(dat, TRAINED_DICT), SAMPLES[0])
1547
1548 def test_len(self):
1549 self.assertEqual(len(TRAINED_DICT), len(TRAINED_DICT.dict_content))
1550 self.assertIn(str(len(TRAINED_DICT)), str(TRAINED_DICT))
1551
1552 class FileTestCase(unittest.TestCase):
1553 def setUp(self):
1554 self.DECOMPRESSED_42 = b'a'*42
1555 self.FRAME_42 = compress(self.DECOMPRESSED_42)
1556
1557 def test_init(self):
1558 with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB)) as f:
1559 pass
1560 with ZstdFile(io.BytesIO(), "w") as f:
1561 pass
1562 with ZstdFile(io.BytesIO(), "x") as f:
1563 pass
1564 with ZstdFile(io.BytesIO(), "a") as f:
1565 pass
1566
1567 with ZstdFile(io.BytesIO(), "w", level=12) as f:
1568 pass
1569 with ZstdFile(io.BytesIO(), "w", options={CompressionParameter.checksum_flag:1}) as f:
1570 pass
1571 with ZstdFile(io.BytesIO(), "w", options={}) as f:
1572 pass
1573 with ZstdFile(io.BytesIO(), "w", level=20, zstd_dict=TRAINED_DICT) as f:
1574 pass
1575
1576 with ZstdFile(io.BytesIO(), "r", options={DecompressionParameter.window_log_max:25}) as f:
1577 pass
1578 with ZstdFile(io.BytesIO(), "r", options={}, zstd_dict=TRAINED_DICT) as f:
1579 pass
1580
1581 def test_init_with_PathLike_filename(self):
1582 with tempfile.NamedTemporaryFile(delete=False) as tmp_f:
1583 filename = pathlib.Path(tmp_f.name)
1584
1585 with ZstdFile(filename, "a") as f:
1586 f.write(DECOMPRESSED_100_PLUS_32KB)
1587 with ZstdFile(filename) as f:
1588 self.assertEqual(f.read(), DECOMPRESSED_100_PLUS_32KB)
1589
1590 with ZstdFile(filename, "a") as f:
1591 f.write(DECOMPRESSED_100_PLUS_32KB)
1592 with ZstdFile(filename) as f:
1593 self.assertEqual(f.read(), DECOMPRESSED_100_PLUS_32KB * 2)
1594
1595 os.remove(filename)
1596
1597 def test_init_with_filename(self):
1598 with tempfile.NamedTemporaryFile(delete=False) as tmp_f:
1599 filename = pathlib.Path(tmp_f.name)
1600
1601 with ZstdFile(filename) as f:
1602 pass
1603 with ZstdFile(filename, "w") as f:
1604 pass
1605 with ZstdFile(filename, "a") as f:
1606 pass
1607
1608 os.remove(filename)
1609
1610 def test_init_mode(self):
1611 bi = io.BytesIO()
1612
1613 with ZstdFile(bi, "r"):
1614 pass
1615 with ZstdFile(bi, "rb"):
1616 pass
1617 with ZstdFile(bi, "w"):
1618 pass
1619 with ZstdFile(bi, "wb"):
1620 pass
1621 with ZstdFile(bi, "a"):
1622 pass
1623 with ZstdFile(bi, "ab"):
1624 pass
1625
1626 def test_init_with_x_mode(self):
1627 with tempfile.NamedTemporaryFile() as tmp_f:
1628 filename = pathlib.Path(tmp_f.name)
1629
1630 for mode in ("x", "xb"):
1631 with ZstdFile(filename, mode):
1632 pass
1633 with self.assertRaises(FileExistsError):
1634 with ZstdFile(filename, mode):
1635 pass
1636 os.remove(filename)
1637
1638 def test_init_bad_mode(self):
1639 with self.assertRaises(ValueError):
1640 ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), (3, "x"))
1641 with self.assertRaises(ValueError):
1642 ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), "")
1643 with self.assertRaises(ValueError):
1644 ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), "xt")
1645 with self.assertRaises(ValueError):
1646 ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), "x+")
1647 with self.assertRaises(ValueError):
1648 ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), "rx")
1649 with self.assertRaises(ValueError):
1650 ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), "wx")
1651 with self.assertRaises(ValueError):
1652 ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), "rt")
1653 with self.assertRaises(ValueError):
1654 ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), "r+")
1655 with self.assertRaises(ValueError):
1656 ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), "wt")
1657 with self.assertRaises(ValueError):
1658 ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), "w+")
1659 with self.assertRaises(ValueError):
1660 ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), "rw")
1661
1662 with self.assertRaisesRegex(TypeError,
1663 r"not be a CompressionParameter"):
1664 ZstdFile(io.BytesIO(), 'rb',
1665 options={CompressionParameter.compression_level:5})
1666 with self.assertRaisesRegex(TypeError,
1667 r"not be a DecompressionParameter"):
1668 ZstdFile(io.BytesIO(), 'wb',
1669 options={DecompressionParameter.window_log_max:21})
1670
1671 with self.assertRaises(TypeError):
1672 ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), "r", level=12)
1673
1674 def test_init_bad_check(self):
1675 with self.assertRaises(TypeError):
1676 ZstdFile(io.BytesIO(), "w", level='asd')
1677 # CHECK_UNKNOWN and anything above CHECK_ID_MAX should be invalid.
1678 with self.assertRaises(ValueError):
1679 ZstdFile(io.BytesIO(), "w", options={999:9999})
1680 with self.assertRaises(ValueError):
1681 ZstdFile(io.BytesIO(), "w", options={CompressionParameter.window_log:99})
1682
1683 with self.assertRaises(TypeError):
1684 ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), "r", options=33)
1685
1686 with self.assertRaises(OverflowError):
1687 ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB),
1688 options={DecompressionParameter.window_log_max:2**31})
1689
1690 with self.assertRaises(ValueError):
1691 ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB),
1692 options={444:333})
1693
1694 with self.assertRaises(TypeError):
1695 ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), zstd_dict={1:2})
1696
1697 with self.assertRaises(TypeError):
1698 ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), zstd_dict=b'dict123456')
1699
1700 def test_init_close_fp(self):
1701 # get a temp file name
1702 with tempfile.NamedTemporaryFile(delete=False) as tmp_f:
1703 tmp_f.write(DAT_130K_C)
1704 filename = tmp_f.name
1705
1706 with self.assertRaises(TypeError):
1707 ZstdFile(filename, options={'a':'b'})
1708
1709 # for PyPy
1710 gc.collect()
1711
1712 os.remove(filename)
1713
1714 def test_close(self):
1715 with io.BytesIO(COMPRESSED_100_PLUS_32KB) as src:
1716 f = ZstdFile(src)
1717 f.close()
1718 # ZstdFile.close() should not close the underlying file object.
1719 self.assertFalse(src.closed)
1720 # Try closing an already-closed ZstdFile.
1721 f.close()
1722 self.assertFalse(src.closed)
1723
1724 # Test with a real file on disk, opened directly by ZstdFile.
1725 with tempfile.NamedTemporaryFile(delete=False) as tmp_f:
1726 filename = pathlib.Path(tmp_f.name)
1727
1728 f = ZstdFile(filename)
1729 fp = f._fp
1730 f.close()
1731 # Here, ZstdFile.close() *should* close the underlying file object.
1732 self.assertTrue(fp.closed)
1733 # Try closing an already-closed ZstdFile.
1734 f.close()
1735
1736 os.remove(filename)
1737
1738 def test_closed(self):
1739 f = ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB))
1740 try:
1741 self.assertFalse(f.closed)
1742 f.read()
1743 self.assertFalse(f.closed)
1744 finally:
1745 f.close()
1746 self.assertTrue(f.closed)
1747
1748 f = ZstdFile(io.BytesIO(), "w")
1749 try:
1750 self.assertFalse(f.closed)
1751 finally:
1752 f.close()
1753 self.assertTrue(f.closed)
1754
1755 def test_fileno(self):
1756 # 1
1757 f = ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB))
1758 try:
1759 self.assertRaises(io.UnsupportedOperation, f.fileno)
1760 finally:
1761 f.close()
1762 self.assertRaises(ValueError, f.fileno)
1763
1764 # 2
1765 with tempfile.NamedTemporaryFile(delete=False) as tmp_f:
1766 filename = pathlib.Path(tmp_f.name)
1767
1768 f = ZstdFile(filename)
1769 try:
1770 self.assertEqual(f.fileno(), f._fp.fileno())
1771 self.assertIsInstance(f.fileno(), int)
1772 finally:
1773 f.close()
1774 self.assertRaises(ValueError, f.fileno)
1775
1776 os.remove(filename)
1777
1778 # 3, no .fileno() method
1779 class C:
1780 def read(self, size=-1):
1781 return b'123'
1782 with ZstdFile(C(), 'rb') as f:
1783 with self.assertRaisesRegex(AttributeError, r'fileno'):
1784 f.fileno()
1785
1786 def test_name(self):
1787 # 1
1788 f = ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB))
1789 try:
1790 with self.assertRaises(AttributeError):
1791 f.name
1792 finally:
1793 f.close()
1794 with self.assertRaises(ValueError):
1795 f.name
1796
1797 # 2
1798 with tempfile.NamedTemporaryFile(delete=False) as tmp_f:
1799 filename = pathlib.Path(tmp_f.name)
1800
1801 f = ZstdFile(filename)
1802 try:
1803 self.assertEqual(f.name, f._fp.name)
1804 self.assertIsInstance(f.name, str)
1805 finally:
1806 f.close()
1807 with self.assertRaises(ValueError):
1808 f.name
1809
1810 os.remove(filename)
1811
1812 # 3, no .filename property
1813 class C:
1814 def read(self, size=-1):
1815 return b'123'
1816 with ZstdFile(C(), 'rb') as f:
1817 with self.assertRaisesRegex(AttributeError, r'name'):
1818 f.name
1819
1820 def test_seekable(self):
1821 f = ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB))
1822 try:
1823 self.assertTrue(f.seekable())
1824 f.read()
1825 self.assertTrue(f.seekable())
1826 finally:
1827 f.close()
1828 self.assertRaises(ValueError, f.seekable)
1829
1830 f = ZstdFile(io.BytesIO(), "w")
1831 try:
1832 self.assertFalse(f.seekable())
1833 finally:
1834 f.close()
1835 self.assertRaises(ValueError, f.seekable)
1836
1837 src = io.BytesIO(COMPRESSED_100_PLUS_32KB)
1838 src.seekable = lambda: False
1839 f = ZstdFile(src)
1840 try:
1841 self.assertFalse(f.seekable())
1842 finally:
1843 f.close()
1844 self.assertRaises(ValueError, f.seekable)
1845
1846 def test_readable(self):
1847 f = ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB))
1848 try:
1849 self.assertTrue(f.readable())
1850 f.read()
1851 self.assertTrue(f.readable())
1852 finally:
1853 f.close()
1854 self.assertRaises(ValueError, f.readable)
1855
1856 f = ZstdFile(io.BytesIO(), "w")
1857 try:
1858 self.assertFalse(f.readable())
1859 finally:
1860 f.close()
1861 self.assertRaises(ValueError, f.readable)
1862
1863 def test_writable(self):
1864 f = ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB))
1865 try:
1866 self.assertFalse(f.writable())
1867 f.read()
1868 self.assertFalse(f.writable())
1869 finally:
1870 f.close()
1871 self.assertRaises(ValueError, f.writable)
1872
1873 f = ZstdFile(io.BytesIO(), "w")
1874 try:
1875 self.assertTrue(f.writable())
1876 finally:
1877 f.close()
1878 self.assertRaises(ValueError, f.writable)
1879
1880 def test_read_0(self):
1881 with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB)) as f:
1882 self.assertEqual(f.read(0), b"")
1883 self.assertEqual(f.read(), DECOMPRESSED_100_PLUS_32KB)
1884 with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB),
1885 options={DecompressionParameter.window_log_max:20}) as f:
1886 self.assertEqual(f.read(0), b"")
1887
1888 # empty file
1889 with ZstdFile(io.BytesIO(b'')) as f:
1890 self.assertEqual(f.read(0), b"")
1891 with self.assertRaises(EOFError):
1892 f.read(10)
1893
1894 with ZstdFile(io.BytesIO(b'')) as f:
1895 with self.assertRaises(EOFError):
1896 f.read(10)
1897
1898 def test_read_10(self):
1899 with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB)) as f:
1900 chunks = []
1901 while True:
1902 result = f.read(10)
1903 if not result:
1904 break
1905 self.assertLessEqual(len(result), 10)
1906 chunks.append(result)
1907 self.assertEqual(b"".join(chunks), DECOMPRESSED_100_PLUS_32KB)
1908
1909 def test_read_multistream(self):
1910 with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB * 5)) as f:
1911 self.assertEqual(f.read(), DECOMPRESSED_100_PLUS_32KB * 5)
1912
1913 with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB + SKIPPABLE_FRAME)) as f:
1914 self.assertEqual(f.read(), DECOMPRESSED_100_PLUS_32KB)
1915
1916 with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB + COMPRESSED_DAT)) as f:
1917 self.assertEqual(f.read(), DECOMPRESSED_100_PLUS_32KB + DECOMPRESSED_DAT)
1918
1919 def test_read_incomplete(self):
1920 with ZstdFile(io.BytesIO(DAT_130K_C[:-200])) as f:
1921 self.assertRaises(EOFError, f.read)
1922
1923 # Trailing data isn't a valid compressed stream
1924 with ZstdFile(io.BytesIO(self.FRAME_42 + b'12345')) as f:
1925 self.assertRaises(ZstdError, f.read)
1926
1927 with ZstdFile(io.BytesIO(SKIPPABLE_FRAME + b'12345')) as f:
1928 self.assertRaises(ZstdError, f.read)
1929
1930 def test_read_truncated(self):
1931 # Drop stream epilogue: 4 bytes checksum
1932 truncated = DAT_130K_C[:-4]
1933 with ZstdFile(io.BytesIO(truncated)) as f:
1934 self.assertRaises(EOFError, f.read)
1935
1936 with ZstdFile(io.BytesIO(truncated)) as f:
1937 # this is an important test, make sure it doesn't raise EOFError.
1938 self.assertEqual(f.read(130*_1K), DAT_130K_D)
1939 with self.assertRaises(EOFError):
1940 f.read(1)
1941
1942 # Incomplete header
1943 for i in range(1, 20):
1944 with ZstdFile(io.BytesIO(truncated[:i])) as f:
1945 self.assertRaises(EOFError, f.read, 1)
1946
1947 def test_read_bad_args(self):
1948 f = ZstdFile(io.BytesIO(COMPRESSED_DAT))
1949 f.close()
1950 self.assertRaises(ValueError, f.read)
1951 with ZstdFile(io.BytesIO(), "w") as f:
1952 self.assertRaises(ValueError, f.read)
1953 with ZstdFile(io.BytesIO(COMPRESSED_DAT)) as f:
1954 self.assertRaises(TypeError, f.read, float())
1955
1956 def test_read_bad_data(self):
1957 with ZstdFile(io.BytesIO(COMPRESSED_BOGUS)) as f:
1958 self.assertRaises(ZstdError, f.read)
1959
1960 def test_read_exception(self):
1961 class C:
1962 def read(self, size=-1):
1963 raise OSError
1964 with ZstdFile(C()) as f:
1965 with self.assertRaises(OSError):
1966 f.read(10)
1967
1968 def test_read1(self):
1969 with ZstdFile(io.BytesIO(DAT_130K_C)) as f:
1970 blocks = []
1971 while True:
1972 result = f.read1()
1973 if not result:
1974 break
1975 blocks.append(result)
1976 self.assertEqual(b"".join(blocks), DAT_130K_D)
1977 self.assertEqual(f.read1(), b"")
1978
1979 def test_read1_0(self):
1980 with ZstdFile(io.BytesIO(COMPRESSED_DAT)) as f:
1981 self.assertEqual(f.read1(0), b"")
1982
1983 def test_read1_10(self):
1984 with ZstdFile(io.BytesIO(COMPRESSED_DAT)) as f:
1985 blocks = []
1986 while True:
1987 result = f.read1(10)
1988 if not result:
1989 break
1990 blocks.append(result)
1991 self.assertEqual(b"".join(blocks), DECOMPRESSED_DAT)
1992 self.assertEqual(f.read1(), b"")
1993
1994 def test_read1_multistream(self):
1995 with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB * 5)) as f:
1996 blocks = []
1997 while True:
1998 result = f.read1()
1999 if not result:
2000 break
2001 blocks.append(result)
2002 self.assertEqual(b"".join(blocks), DECOMPRESSED_100_PLUS_32KB * 5)
2003 self.assertEqual(f.read1(), b"")
2004
2005 def test_read1_bad_args(self):
2006 f = ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB))
2007 f.close()
2008 self.assertRaises(ValueError, f.read1)
2009 with ZstdFile(io.BytesIO(), "w") as f:
2010 self.assertRaises(ValueError, f.read1)
2011 with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB)) as f:
2012 self.assertRaises(TypeError, f.read1, None)
2013
2014 def test_readinto(self):
2015 arr = array.array("I", range(100))
2016 self.assertEqual(len(arr), 100)
2017 self.assertEqual(len(arr) * arr.itemsize, 400)
2018 ba = bytearray(300)
2019 with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB)) as f:
2020 # 0 length output buffer
2021 self.assertEqual(f.readinto(ba[0:0]), 0)
2022
2023 # use correct length for buffer protocol object
2024 self.assertEqual(f.readinto(arr), 400)
2025 self.assertEqual(arr.tobytes(), DECOMPRESSED_100_PLUS_32KB[:400])
2026
2027 # normal readinto
2028 self.assertEqual(f.readinto(ba), 300)
2029 self.assertEqual(ba, DECOMPRESSED_100_PLUS_32KB[400:700])
2030
2031 def test_peek(self):
2032 with ZstdFile(io.BytesIO(DAT_130K_C)) as f:
2033 result = f.peek()
2034 self.assertGreater(len(result), 0)
2035 self.assertTrue(DAT_130K_D.startswith(result))
2036 self.assertEqual(f.read(), DAT_130K_D)
2037 with ZstdFile(io.BytesIO(DAT_130K_C)) as f:
2038 result = f.peek(10)
2039 self.assertGreater(len(result), 0)
2040 self.assertTrue(DAT_130K_D.startswith(result))
2041 self.assertEqual(f.read(), DAT_130K_D)
2042
2043 def test_peek_bad_args(self):
2044 with ZstdFile(io.BytesIO(), "w") as f:
2045 self.assertRaises(ValueError, f.peek)
2046
2047 def test_iterator(self):
2048 with io.BytesIO(THIS_FILE_BYTES) as f:
2049 lines = f.readlines()
2050 compressed = compress(THIS_FILE_BYTES)
2051
2052 # iter
2053 with ZstdFile(io.BytesIO(compressed)) as f:
2054 self.assertListEqual(list(iter(f)), lines)
2055
2056 # readline
2057 with ZstdFile(io.BytesIO(compressed)) as f:
2058 for line in lines:
2059 self.assertEqual(f.readline(), line)
2060 self.assertEqual(f.readline(), b'')
2061 self.assertEqual(f.readline(), b'')
2062
2063 # readlines
2064 with ZstdFile(io.BytesIO(compressed)) as f:
2065 self.assertListEqual(f.readlines(), lines)
2066
2067 def test_decompress_limited(self):
2068 _ZSTD_DStreamInSize = 128*_1K + 3
2069
2070 bomb = compress(b'\0' * int(2e6), level=10)
2071 self.assertLess(len(bomb), _ZSTD_DStreamInSize)
2072
2073 decomp = ZstdFile(io.BytesIO(bomb))
2074 self.assertEqual(decomp.read(1), b'\0')
2075
2076 # BufferedReader uses 128 KiB buffer in __init__.py
2077 max_decomp = 128*_1K
2078 self.assertLessEqual(decomp._buffer.raw.tell(), max_decomp,
2079 "Excessive amount of data was decompressed")
2080
2081 def test_write(self):
2082 raw_data = THIS_FILE_BYTES[: len(THIS_FILE_BYTES) // 6]
2083 with io.BytesIO() as dst:
2084 with ZstdFile(dst, "w") as f:
2085 f.write(raw_data)
2086
2087 comp = ZstdCompressor()
2088 expected = comp.compress(raw_data) + comp.flush()
2089 self.assertEqual(dst.getvalue(), expected)
2090
2091 with io.BytesIO() as dst:
2092 with ZstdFile(dst, "w", level=12) as f:
2093 f.write(raw_data)
2094
2095 comp = ZstdCompressor(12)
2096 expected = comp.compress(raw_data) + comp.flush()
2097 self.assertEqual(dst.getvalue(), expected)
2098
2099 with io.BytesIO() as dst:
2100 with ZstdFile(dst, "w", options={CompressionParameter.checksum_flag:1}) as f:
2101 f.write(raw_data)
2102
2103 comp = ZstdCompressor(options={CompressionParameter.checksum_flag:1})
2104 expected = comp.compress(raw_data) + comp.flush()
2105 self.assertEqual(dst.getvalue(), expected)
2106
2107 with io.BytesIO() as dst:
2108 options = {CompressionParameter.compression_level:-5,
2109 CompressionParameter.checksum_flag:1}
2110 with ZstdFile(dst, "w",
2111 options=options) as f:
2112 f.write(raw_data)
2113
2114 comp = ZstdCompressor(options=options)
2115 expected = comp.compress(raw_data) + comp.flush()
2116 self.assertEqual(dst.getvalue(), expected)
2117
2118 def test_write_empty_frame(self):
2119 # .FLUSH_FRAME generates an empty content frame
2120 c = ZstdCompressor()
2121 self.assertNotEqual(c.flush(c.FLUSH_FRAME), b'')
2122 self.assertNotEqual(c.flush(c.FLUSH_FRAME), b'')
2123
2124 # don't generate empty content frame
2125 bo = io.BytesIO()
2126 with ZstdFile(bo, 'w') as f:
2127 pass
2128 self.assertEqual(bo.getvalue(), b'')
2129
2130 bo = io.BytesIO()
2131 with ZstdFile(bo, 'w') as f:
2132 f.flush(f.FLUSH_FRAME)
2133 self.assertEqual(bo.getvalue(), b'')
2134
2135 # if .write(b''), generate empty content frame
2136 bo = io.BytesIO()
2137 with ZstdFile(bo, 'w') as f:
2138 f.write(b'')
2139 self.assertNotEqual(bo.getvalue(), b'')
2140
2141 # has an empty content frame
2142 bo = io.BytesIO()
2143 with ZstdFile(bo, 'w') as f:
2144 f.flush(f.FLUSH_BLOCK)
2145 self.assertNotEqual(bo.getvalue(), b'')
2146
2147 def test_write_empty_block(self):
2148 # If no internal data, .FLUSH_BLOCK return b''.
2149 c = ZstdCompressor()
2150 self.assertEqual(c.flush(c.FLUSH_BLOCK), b'')
2151 self.assertNotEqual(c.compress(b'123', c.FLUSH_BLOCK),
2152 b'')
2153 self.assertEqual(c.flush(c.FLUSH_BLOCK), b'')
2154 self.assertEqual(c.compress(b''), b'')
2155 self.assertEqual(c.compress(b''), b'')
2156 self.assertEqual(c.flush(c.FLUSH_BLOCK), b'')
2157
2158 # mode = .last_mode
2159 bo = io.BytesIO()
2160 with ZstdFile(bo, 'w') as f:
2161 f.write(b'123')
2162 f.flush(f.FLUSH_BLOCK)
2163 fp_pos = f._fp.tell()
2164 self.assertNotEqual(fp_pos, 0)
2165 f.flush(f.FLUSH_BLOCK)
2166 self.assertEqual(f._fp.tell(), fp_pos)
2167
2168 # mode != .last_mode
2169 bo = io.BytesIO()
2170 with ZstdFile(bo, 'w') as f:
2171 f.flush(f.FLUSH_BLOCK)
2172 self.assertEqual(f._fp.tell(), 0)
2173 f.write(b'')
2174 f.flush(f.FLUSH_BLOCK)
2175 self.assertEqual(f._fp.tell(), 0)
2176
2177 def test_write_101(self):
2178 with io.BytesIO() as dst:
2179 with ZstdFile(dst, "w") as f:
2180 for start in range(0, len(THIS_FILE_BYTES), 101):
2181 f.write(THIS_FILE_BYTES[start:start+101])
2182
2183 comp = ZstdCompressor()
2184 expected = comp.compress(THIS_FILE_BYTES) + comp.flush()
2185 self.assertEqual(dst.getvalue(), expected)
2186
2187 def test_write_append(self):
2188 def comp(data):
2189 comp = ZstdCompressor()
2190 return comp.compress(data) + comp.flush()
2191
2192 part1 = THIS_FILE_BYTES[:_1K]
2193 part2 = THIS_FILE_BYTES[_1K:1536]
2194 part3 = THIS_FILE_BYTES[1536:]
2195 expected = b"".join(comp(x) for x in (part1, part2, part3))
2196 with io.BytesIO() as dst:
2197 with ZstdFile(dst, "w") as f:
2198 f.write(part1)
2199 with ZstdFile(dst, "a") as f:
2200 f.write(part2)
2201 with ZstdFile(dst, "a") as f:
2202 f.write(part3)
2203 self.assertEqual(dst.getvalue(), expected)
2204
2205 def test_write_bad_args(self):
2206 f = ZstdFile(io.BytesIO(), "w")
2207 f.close()
2208 self.assertRaises(ValueError, f.write, b"foo")
2209 with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), "r") as f:
2210 self.assertRaises(ValueError, f.write, b"bar")
2211 with ZstdFile(io.BytesIO(), "w") as f:
2212 self.assertRaises(TypeError, f.write, None)
2213 self.assertRaises(TypeError, f.write, "text")
2214 self.assertRaises(TypeError, f.write, 789)
2215
2216 def test_writelines(self):
2217 def comp(data):
2218 comp = ZstdCompressor()
2219 return comp.compress(data) + comp.flush()
2220
2221 with io.BytesIO(THIS_FILE_BYTES) as f:
2222 lines = f.readlines()
2223 with io.BytesIO() as dst:
2224 with ZstdFile(dst, "w") as f:
2225 f.writelines(lines)
2226 expected = comp(THIS_FILE_BYTES)
2227 self.assertEqual(dst.getvalue(), expected)
2228
2229 def test_seek_forward(self):
2230 with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB)) as f:
2231 f.seek(555)
2232 self.assertEqual(f.read(), DECOMPRESSED_100_PLUS_32KB[555:])
2233
2234 def test_seek_forward_across_streams(self):
2235 with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB * 2)) as f:
2236 f.seek(len(DECOMPRESSED_100_PLUS_32KB) + 123)
2237 self.assertEqual(f.read(), DECOMPRESSED_100_PLUS_32KB[123:])
2238
2239 def test_seek_forward_relative_to_current(self):
2240 with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB)) as f:
2241 f.read(100)
2242 f.seek(1236, 1)
2243 self.assertEqual(f.read(), DECOMPRESSED_100_PLUS_32KB[1336:])
2244
2245 def test_seek_forward_relative_to_end(self):
2246 with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB)) as f:
2247 f.seek(-555, 2)
2248 self.assertEqual(f.read(), DECOMPRESSED_100_PLUS_32KB[-555:])
2249
2250 def test_seek_backward(self):
2251 with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB)) as f:
2252 f.read(1001)
2253 f.seek(211)
2254 self.assertEqual(f.read(), DECOMPRESSED_100_PLUS_32KB[211:])
2255
2256 def test_seek_backward_across_streams(self):
2257 with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB * 2)) as f:
2258 f.read(len(DECOMPRESSED_100_PLUS_32KB) + 333)
2259 f.seek(737)
2260 self.assertEqual(f.read(),
2261 DECOMPRESSED_100_PLUS_32KB[737:] + DECOMPRESSED_100_PLUS_32KB)
2262
2263 def test_seek_backward_relative_to_end(self):
2264 with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB)) as f:
2265 f.seek(-150, 2)
2266 self.assertEqual(f.read(), DECOMPRESSED_100_PLUS_32KB[-150:])
2267
2268 def test_seek_past_end(self):
2269 with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB)) as f:
2270 f.seek(len(DECOMPRESSED_100_PLUS_32KB) + 9001)
2271 self.assertEqual(f.tell(), len(DECOMPRESSED_100_PLUS_32KB))
2272 self.assertEqual(f.read(), b"")
2273
2274 def test_seek_past_start(self):
2275 with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB)) as f:
2276 f.seek(-88)
2277 self.assertEqual(f.tell(), 0)
2278 self.assertEqual(f.read(), DECOMPRESSED_100_PLUS_32KB)
2279
2280 def test_seek_bad_args(self):
2281 f = ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB))
2282 f.close()
2283 self.assertRaises(ValueError, f.seek, 0)
2284 with ZstdFile(io.BytesIO(), "w") as f:
2285 self.assertRaises(ValueError, f.seek, 0)
2286 with ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB)) as f:
2287 self.assertRaises(ValueError, f.seek, 0, 3)
2288 # io.BufferedReader raises TypeError instead of ValueError
2289 self.assertRaises((TypeError, ValueError), f.seek, 9, ())
2290 self.assertRaises(TypeError, f.seek, None)
2291 self.assertRaises(TypeError, f.seek, b"derp")
2292
2293 def test_seek_not_seekable(self):
2294 class C(io.BytesIO):
2295 def seekable(self):
2296 return False
2297 obj = C(COMPRESSED_100_PLUS_32KB)
2298 with ZstdFile(obj, 'r') as f:
2299 d = f.read(1)
2300 self.assertFalse(f.seekable())
2301 with self.assertRaisesRegex(io.UnsupportedOperation,
2302 'File or stream is not seekable'):
2303 f.seek(0)
2304 d += f.read()
2305 self.assertEqual(d, DECOMPRESSED_100_PLUS_32KB)
2306
2307 def test_tell(self):
2308 with ZstdFile(io.BytesIO(DAT_130K_C)) as f:
2309 pos = 0
2310 while True:
2311 self.assertEqual(f.tell(), pos)
2312 result = f.read(random.randint(171, 189))
2313 if not result:
2314 break
2315 pos += len(result)
2316 self.assertEqual(f.tell(), len(DAT_130K_D))
2317 with ZstdFile(io.BytesIO(), "w") as f:
2318 for pos in range(0, len(DAT_130K_D), 143):
2319 self.assertEqual(f.tell(), pos)
2320 f.write(DAT_130K_D[pos:pos+143])
2321 self.assertEqual(f.tell(), len(DAT_130K_D))
2322
2323 def test_tell_bad_args(self):
2324 f = ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB))
2325 f.close()
2326 self.assertRaises(ValueError, f.tell)
2327
2328 def test_file_dict(self):
2329 # default
2330 bi = io.BytesIO()
2331 with ZstdFile(bi, 'w', zstd_dict=TRAINED_DICT) as f:
2332 f.write(SAMPLES[0])
2333 bi.seek(0)
2334 with ZstdFile(bi, zstd_dict=TRAINED_DICT) as f:
2335 dat = f.read()
2336 self.assertEqual(dat, SAMPLES[0])
2337
2338 # .as_(un)digested_dict
2339 bi = io.BytesIO()
2340 with ZstdFile(bi, 'w', zstd_dict=TRAINED_DICT.as_digested_dict) as f:
2341 f.write(SAMPLES[0])
2342 bi.seek(0)
2343 with ZstdFile(bi, zstd_dict=TRAINED_DICT.as_undigested_dict) as f:
2344 dat = f.read()
2345 self.assertEqual(dat, SAMPLES[0])
2346
2347 def test_file_prefix(self):
2348 bi = io.BytesIO()
2349 with ZstdFile(bi, 'w', zstd_dict=TRAINED_DICT.as_prefix) as f:
2350 f.write(SAMPLES[0])
2351 bi.seek(0)
2352 with ZstdFile(bi, zstd_dict=TRAINED_DICT.as_prefix) as f:
2353 dat = f.read()
2354 self.assertEqual(dat, SAMPLES[0])
2355
2356 def test_UnsupportedOperation(self):
2357 # 1
2358 with ZstdFile(io.BytesIO(), 'r') as f:
2359 with self.assertRaises(io.UnsupportedOperation):
2360 f.write(b'1234')
2361
2362 # 2
2363 class T:
2364 def read(self, size):
2365 return b'a' * size
2366
2367 with self.assertRaises(TypeError): # on creation
2368 with ZstdFile(T(), 'w') as f:
2369 pass
2370
2371 # 3
2372 with ZstdFile(io.BytesIO(), 'w') as f:
2373 with self.assertRaises(io.UnsupportedOperation):
2374 f.read(100)
2375 with self.assertRaises(io.UnsupportedOperation):
2376 f.seek(100)
2377 self.assertEqual(f.closed, True)
2378 with self.assertRaises(ValueError):
2379 f.readable()
2380 with self.assertRaises(ValueError):
2381 f.tell()
2382 with self.assertRaises(ValueError):
2383 f.read(100)
2384
2385 def test_read_readinto_readinto1(self):
2386 lst = []
2387 with ZstdFile(io.BytesIO(COMPRESSED_THIS_FILE*5)) as f:
2388 while True:
2389 method = random.randint(0, 2)
2390 size = random.randint(0, 300)
2391
2392 if method == 0:
2393 dat = f.read(size)
2394 if not dat and size:
2395 break
2396 lst.append(dat)
2397 elif method == 1:
2398 ba = bytearray(size)
2399 read_size = f.readinto(ba)
2400 if read_size == 0 and size:
2401 break
2402 lst.append(bytes(ba[:read_size]))
2403 elif method == 2:
2404 ba = bytearray(size)
2405 read_size = f.readinto1(ba)
2406 if read_size == 0 and size:
2407 break
2408 lst.append(bytes(ba[:read_size]))
2409 self.assertEqual(b''.join(lst), THIS_FILE_BYTES*5)
2410
2411 def test_zstdfile_flush(self):
2412 # closed
2413 f = ZstdFile(io.BytesIO(), 'w')
2414 f.close()
2415 with self.assertRaises(ValueError):
2416 f.flush()
2417
2418 # read
2419 with ZstdFile(io.BytesIO(), 'r') as f:
2420 # does nothing for read-only stream
2421 f.flush()
2422
2423 # write
2424 DAT = b'abcd'
2425 bi = io.BytesIO()
2426 with ZstdFile(bi, 'w') as f:
2427 self.assertEqual(f.write(DAT), len(DAT))
2428 self.assertEqual(f.tell(), len(DAT))
2429 self.assertEqual(bi.tell(), 0) # not enough for a block
2430
2431 self.assertEqual(f.flush(), None)
2432 self.assertEqual(f.tell(), len(DAT))
2433 self.assertGreater(bi.tell(), 0) # flushed
2434
2435 # write, no .flush() method
2436 class C:
2437 def write(self, b):
2438 return len(b)
2439 with ZstdFile(C(), 'w') as f:
2440 self.assertEqual(f.write(DAT), len(DAT))
2441 self.assertEqual(f.tell(), len(DAT))
2442
2443 self.assertEqual(f.flush(), None)
2444 self.assertEqual(f.tell(), len(DAT))
2445
2446 def test_zstdfile_flush_mode(self):
2447 self.assertEqual(ZstdFile.FLUSH_BLOCK, ZstdCompressor.FLUSH_BLOCK)
2448 self.assertEqual(ZstdFile.FLUSH_FRAME, ZstdCompressor.FLUSH_FRAME)
2449 with self.assertRaises(AttributeError):
2450 ZstdFile.CONTINUE
2451
2452 bo = io.BytesIO()
2453 with ZstdFile(bo, 'w') as f:
2454 # flush block
2455 self.assertEqual(f.write(b'123'), 3)
2456 self.assertIsNone(f.flush(f.FLUSH_BLOCK))
2457 p1 = bo.tell()
2458 # mode == .last_mode, should return
2459 self.assertIsNone(f.flush())
2460 p2 = bo.tell()
2461 self.assertEqual(p1, p2)
2462 # flush frame
2463 self.assertEqual(f.write(b'456'), 3)
2464 self.assertIsNone(f.flush(mode=f.FLUSH_FRAME))
2465 # flush frame
2466 self.assertEqual(f.write(b'789'), 3)
2467 self.assertIsNone(f.flush(f.FLUSH_FRAME))
2468 p1 = bo.tell()
2469 # mode == .last_mode, should return
2470 self.assertIsNone(f.flush(f.FLUSH_FRAME))
2471 p2 = bo.tell()
2472 self.assertEqual(p1, p2)
2473 self.assertEqual(decompress(bo.getvalue()), b'123456789')
2474
2475 bo = io.BytesIO()
2476 with ZstdFile(bo, 'w') as f:
2477 f.write(b'123')
2478 with self.assertRaisesRegex(ValueError, r'\.FLUSH_.*?\.FLUSH_'):
2479 f.flush(ZstdCompressor.CONTINUE)
2480 with self.assertRaises(ValueError):
2481 f.flush(-1)
2482 with self.assertRaises(ValueError):
2483 f.flush(123456)
2484 with self.assertRaises(TypeError):
2485 f.flush(node=ZstdCompressor.CONTINUE)
2486 with self.assertRaises((TypeError, ValueError)):
2487 f.flush('FLUSH_FRAME')
2488 with self.assertRaises(TypeError):
2489 f.flush(b'456', f.FLUSH_BLOCK)
2490
2491 def test_zstdfile_truncate(self):
2492 with ZstdFile(io.BytesIO(), 'w') as f:
2493 with self.assertRaises(io.UnsupportedOperation):
2494 f.truncate(200)
2495
2496 def test_zstdfile_iter_issue45475(self):
2497 lines = [l for l in ZstdFile(io.BytesIO(COMPRESSED_THIS_FILE))]
2498 self.assertGreater(len(lines), 0)
2499
2500 def test_append_new_file(self):
2501 with tempfile.NamedTemporaryFile(delete=True) as tmp_f:
2502 filename = tmp_f.name
2503
2504 with ZstdFile(filename, 'a') as f:
2505 pass
2506 self.assertTrue(os.path.isfile(filename))
2507
2508 os.remove(filename)
2509
2510 class OpenTestCase(unittest.TestCase):
2511
2512 def test_binary_modes(self):
2513 with open(io.BytesIO(COMPRESSED_100_PLUS_32KB), "rb") as f:
2514 self.assertEqual(f.read(), DECOMPRESSED_100_PLUS_32KB)
2515 with io.BytesIO() as bio:
2516 with open(bio, "wb") as f:
2517 f.write(DECOMPRESSED_100_PLUS_32KB)
2518 file_data = decompress(bio.getvalue())
2519 self.assertEqual(file_data, DECOMPRESSED_100_PLUS_32KB)
2520 with open(bio, "ab") as f:
2521 f.write(DECOMPRESSED_100_PLUS_32KB)
2522 file_data = decompress(bio.getvalue())
2523 self.assertEqual(file_data, DECOMPRESSED_100_PLUS_32KB * 2)
2524
2525 def test_text_modes(self):
2526 # empty input
2527 with self.assertRaises(EOFError):
2528 with open(io.BytesIO(b''), "rt", encoding="utf-8", newline='\n') as reader:
2529 for _ in reader:
2530 pass
2531
2532 # read
2533 uncompressed = THIS_FILE_STR.replace(os.linesep, "\n")
2534 with open(io.BytesIO(COMPRESSED_THIS_FILE), "rt", encoding="utf-8") as f:
2535 self.assertEqual(f.read(), uncompressed)
2536
2537 with io.BytesIO() as bio:
2538 # write
2539 with open(bio, "wt", encoding="utf-8") as f:
2540 f.write(uncompressed)
2541 file_data = decompress(bio.getvalue()).decode("utf-8")
2542 self.assertEqual(file_data.replace(os.linesep, "\n"), uncompressed)
2543 # append
2544 with open(bio, "at", encoding="utf-8") as f:
2545 f.write(uncompressed)
2546 file_data = decompress(bio.getvalue()).decode("utf-8")
2547 self.assertEqual(file_data.replace(os.linesep, "\n"), uncompressed * 2)
2548
2549 def test_bad_params(self):
2550 with tempfile.NamedTemporaryFile(delete=False) as tmp_f:
2551 TESTFN = pathlib.Path(tmp_f.name)
2552
2553 with self.assertRaises(ValueError):
2554 open(TESTFN, "")
2555 with self.assertRaises(ValueError):
2556 open(TESTFN, "rbt")
2557 with self.assertRaises(ValueError):
2558 open(TESTFN, "rb", encoding="utf-8")
2559 with self.assertRaises(ValueError):
2560 open(TESTFN, "rb", errors="ignore")
2561 with self.assertRaises(ValueError):
2562 open(TESTFN, "rb", newline="\n")
2563
2564 os.remove(TESTFN)
2565
2566 def test_option(self):
2567 options = {DecompressionParameter.window_log_max:25}
2568 with open(io.BytesIO(COMPRESSED_100_PLUS_32KB), "rb", options=options) as f:
2569 self.assertEqual(f.read(), DECOMPRESSED_100_PLUS_32KB)
2570
2571 options = {CompressionParameter.compression_level:12}
2572 with io.BytesIO() as bio:
2573 with open(bio, "wb", options=options) as f:
2574 f.write(DECOMPRESSED_100_PLUS_32KB)
2575 file_data = decompress(bio.getvalue())
2576 self.assertEqual(file_data, DECOMPRESSED_100_PLUS_32KB)
2577
2578 def test_encoding(self):
2579 uncompressed = THIS_FILE_STR.replace(os.linesep, "\n")
2580
2581 with io.BytesIO() as bio:
2582 with open(bio, "wt", encoding="utf-16-le") as f:
2583 f.write(uncompressed)
2584 file_data = decompress(bio.getvalue()).decode("utf-16-le")
2585 self.assertEqual(file_data.replace(os.linesep, "\n"), uncompressed)
2586 bio.seek(0)
2587 with open(bio, "rt", encoding="utf-16-le") as f:
2588 self.assertEqual(f.read().replace(os.linesep, "\n"), uncompressed)
2589
2590 def test_encoding_error_handler(self):
2591 with io.BytesIO(compress(b"foo\xffbar")) as bio:
2592 with open(bio, "rt", encoding="ascii", errors="ignore") as f:
2593 self.assertEqual(f.read(), "foobar")
2594
2595 def test_newline(self):
2596 # Test with explicit newline (universal newline mode disabled).
2597 text = THIS_FILE_STR.replace(os.linesep, "\n")
2598 with io.BytesIO() as bio:
2599 with open(bio, "wt", encoding="utf-8", newline="\n") as f:
2600 f.write(text)
2601 bio.seek(0)
2602 with open(bio, "rt", encoding="utf-8", newline="\r") as f:
2603 self.assertEqual(f.readlines(), [text])
2604
2605 def test_x_mode(self):
2606 with tempfile.NamedTemporaryFile(delete=False) as tmp_f:
2607 TESTFN = pathlib.Path(tmp_f.name)
2608
2609 for mode in ("x", "xb", "xt"):
2610 os.remove(TESTFN)
2611
2612 if mode == "xt":
2613 encoding = "utf-8"
2614 else:
2615 encoding = None
2616 with open(TESTFN, mode, encoding=encoding):
2617 pass
2618 with self.assertRaises(FileExistsError):
2619 with open(TESTFN, mode):
2620 pass
2621
2622 os.remove(TESTFN)
2623
2624 def test_open_dict(self):
2625 # default
2626 bi = io.BytesIO()
2627 with open(bi, 'w', zstd_dict=TRAINED_DICT) as f:
2628 f.write(SAMPLES[0])
2629 bi.seek(0)
2630 with open(bi, zstd_dict=TRAINED_DICT) as f:
2631 dat = f.read()
2632 self.assertEqual(dat, SAMPLES[0])
2633
2634 # .as_(un)digested_dict
2635 bi = io.BytesIO()
2636 with open(bi, 'w', zstd_dict=TRAINED_DICT.as_digested_dict) as f:
2637 f.write(SAMPLES[0])
2638 bi.seek(0)
2639 with open(bi, zstd_dict=TRAINED_DICT.as_undigested_dict) as f:
2640 dat = f.read()
2641 self.assertEqual(dat, SAMPLES[0])
2642
2643 # invalid dictionary
2644 bi = io.BytesIO()
2645 with self.assertRaisesRegex(TypeError, 'zstd_dict'):
2646 open(bi, 'w', zstd_dict={1:2, 2:3})
2647
2648 with self.assertRaisesRegex(TypeError, 'zstd_dict'):
2649 open(bi, 'w', zstd_dict=b'1234567890')
2650
2651 def test_open_prefix(self):
2652 bi = io.BytesIO()
2653 with open(bi, 'w', zstd_dict=TRAINED_DICT.as_prefix) as f:
2654 f.write(SAMPLES[0])
2655 bi.seek(0)
2656 with open(bi, zstd_dict=TRAINED_DICT.as_prefix) as f:
2657 dat = f.read()
2658 self.assertEqual(dat, SAMPLES[0])
2659
2660 def test_buffer_protocol(self):
2661 # don't use len() for buffer protocol objects
2662 arr = array.array("i", range(1000))
2663 LENGTH = len(arr) * arr.itemsize
2664
2665 with open(io.BytesIO(), "wb") as f:
2666 self.assertEqual(f.write(arr), LENGTH)
2667 self.assertEqual(f.tell(), LENGTH)
2668
2669 class FreeThreadingMethodTests(unittest.TestCase):
2670
2671 @threading_helper.reap_threads
2672 @threading_helper.requires_working_threading()
2673 def test_compress_locking(self):
2674 input = b'a'* (16*_1K)
2675 num_threads = 8
2676
2677 # gh-136394: the first output of .compress() includes the frame header
2678 # we run the first .compress() call outside of the threaded portion
2679 # to make the test order-independent
2680
2681 comp = ZstdCompressor()
2682 parts = [comp.compress(input, ZstdCompressor.FLUSH_BLOCK)]
2683 for _ in range(num_threads):
2684 res = comp.compress(input, ZstdCompressor.FLUSH_BLOCK)
2685 if res:
2686 parts.append(res)
2687 rest1 = comp.flush()
2688 expected = b''.join(parts) + rest1
2689
2690 comp = ZstdCompressor()
2691 output = [comp.compress(input, ZstdCompressor.FLUSH_BLOCK)]
2692 def run_method(method, input_data, output_data):
2693 res = method(input_data, ZstdCompressor.FLUSH_BLOCK)
2694 if res:
2695 output_data.append(res)
2696 threads = []
2697
2698 for i in range(num_threads):
2699 thread = threading.Thread(target=run_method, args=(comp.compress, input, output))
2700
2701 threads.append(thread)
2702
2703 with threading_helper.start_threads(threads):
2704 pass
2705
2706 rest2 = comp.flush()
2707 self.assertEqual(rest1, rest2)
2708 actual = b''.join(output) + rest2
2709 self.assertEqual(expected, actual)
2710
2711 @threading_helper.reap_threads
2712 @threading_helper.requires_working_threading()
2713 def test_decompress_locking(self):
2714 input = compress(b'a'* (16*_1K))
2715 num_threads = 8
2716 # to ensure we decompress over multiple calls, set maxsize
2717 window_size = _1K * 16//num_threads
2718
2719 decomp = ZstdDecompressor()
2720 parts = []
2721 for _ in range(num_threads):
2722 res = decomp.decompress(input, window_size)
2723 if res:
2724 parts.append(res)
2725 expected = b''.join(parts)
2726
2727 comp = ZstdDecompressor()
2728 output = []
2729 def run_method(method, input_data, output_data):
2730 res = method(input_data, window_size)
2731 if res:
2732 output_data.append(res)
2733 threads = []
2734
2735 for i in range(num_threads):
2736 thread = threading.Thread(target=run_method, args=(comp.decompress, input, output))
2737
2738 threads.append(thread)
2739
2740 with threading_helper.start_threads(threads):
2741 pass
2742
2743 actual = b''.join(output)
2744 self.assertEqual(expected, actual)
2745
2746 @threading_helper.reap_threads
2747 @threading_helper.requires_working_threading()
2748 def test_compress_shared_dict(self):
2749 num_threads = 8
2750
2751 def run_method(b):
2752 level = threading.get_ident() % 4
2753 # sync threads to increase chance of contention on
2754 # capsule storing dictionary levels
2755 b.wait()
2756 ZstdCompressor(level=level,
2757 zstd_dict=TRAINED_DICT.as_digested_dict)
2758 b.wait()
2759 ZstdCompressor(level=level,
2760 zstd_dict=TRAINED_DICT.as_undigested_dict)
2761 b.wait()
2762 ZstdCompressor(level=level,
2763 zstd_dict=TRAINED_DICT.as_prefix)
2764 threads = []
2765
2766 b = threading.Barrier(num_threads)
2767 for i in range(num_threads):
2768 thread = threading.Thread(target=run_method, args=(b,))
2769
2770 threads.append(thread)
2771
2772 with threading_helper.start_threads(threads):
2773 pass
2774
2775 @threading_helper.reap_threads
2776 @threading_helper.requires_working_threading()
2777 def test_decompress_shared_dict(self):
2778 num_threads = 8
2779
2780 def run_method(b):
2781 # sync threads to increase chance of contention on
2782 # decompression dictionary
2783 b.wait()
2784 ZstdDecompressor(zstd_dict=TRAINED_DICT.as_digested_dict)
2785 b.wait()
2786 ZstdDecompressor(zstd_dict=TRAINED_DICT.as_undigested_dict)
2787 b.wait()
2788 ZstdDecompressor(zstd_dict=TRAINED_DICT.as_prefix)
2789 threads = []
2790
2791 b = threading.Barrier(num_threads)
2792 for i in range(num_threads):
2793 thread = threading.Thread(target=run_method, args=(b,))
2794
2795 threads.append(thread)
2796
2797 with threading_helper.start_threads(threads):
2798 pass
2799
2800
2801 if __name__ == "__main__":
2802 unittest.main()