Debian Codegraph @

Lib/compression/zstd/__init__.py

raw
1 """Python bindings to the Zstandard (zstd) compression library (RFC-8878)."""
2
3 __all__ = (
4 # compression.zstd
5 'COMPRESSION_LEVEL_DEFAULT',
6 'compress',
7 'CompressionParameter',
8 'decompress',
9 'DecompressionParameter',
10 'finalize_dict',
11 'get_frame_info',
12 'Strategy',
13 'train_dict',
14
15 # compression.zstd._zstdfile
16 'open',
17 'ZstdFile',
18
19 # _zstd
20 'get_frame_size',
21 'zstd_version',
22 'zstd_version_info',
23 'ZstdCompressor',
24 'ZstdDecompressor',
25 'ZstdDict',
26 'ZstdError',
27 )
28
29 import _zstd
30 import enum
31 from _zstd import (ZstdCompressor, ZstdDecompressor, ZstdDict, ZstdError,
32 get_frame_size, zstd_version)
33 from compression.zstd._zstdfile import ZstdFile, open, _nbytes
34
35 # zstd_version_number is (MAJOR * 100 * 100 + MINOR * 100 + RELEASE)
36 zstd_version_info = (*divmod(_zstd.zstd_version_number // 100, 100),
37 _zstd.zstd_version_number % 100)
38 """Version number of the runtime zstd library as a tuple of integers."""
39
40 COMPRESSION_LEVEL_DEFAULT = _zstd.ZSTD_CLEVEL_DEFAULT
41 """The default compression level for Zstandard, currently '3'."""
42
43
44 class FrameInfo:
45 """Information about a Zstandard frame."""
46
47 __slots__ = 'decompressed_size', 'dictionary_id'
48
49 def __init__(self, decompressed_size, dictionary_id):
50 super().__setattr__('decompressed_size', decompressed_size)
51 super().__setattr__('dictionary_id', dictionary_id)
52
53 def __repr__(self):
54 return (f'FrameInfo(decompressed_size={self.decompressed_size}, '
55 f'dictionary_id={self.dictionary_id})')
56
57 def __setattr__(self, name, _):
58 raise AttributeError(f"can't set attribute {name!r}")
59
60
61 def get_frame_info(frame_buffer):
62 """Get Zstandard frame information from a frame header.
63
64 *frame_buffer* is a bytes-like object. It should start from the
65 beginning of a frame, and needs to include at least the frame header
66 (6 to 18 bytes).
67
68 The returned FrameInfo object has two attributes.
69 'decompressed_size' is the size in bytes of the data in the frame when
70 decompressed, or None when the decompressed size is unknown.
71 'dictionary_id' is an int in the range (0, 2**32). The special value 0
72 means that the dictionary ID was not recorded in the frame header,
73 the frame may or may not need a dictionary to be decoded,
74 and the ID of such a dictionary is not specified.
75 """
76 return FrameInfo(*_zstd.get_frame_info(frame_buffer))
77
78
79 def train_dict(samples, dict_size):
80 """Return a ZstdDict representing a trained Zstandard dictionary.
81
82 *samples* is an iterable of samples, where a sample is a bytes-like
83 object representing a file.
84
85 *dict_size* is the dictionary's maximum size, in bytes.
86 """
87 if not isinstance(dict_size, int):
88 ds_cls = type(dict_size).__qualname__
89 raise TypeError(f'dict_size must be an int object, not {ds_cls!r}.')
90
91 samples = tuple(samples)
92 chunks = b''.join(samples)
93 chunk_sizes = tuple(_nbytes(sample) for sample in samples)
94 if not chunks:
95 raise ValueError("samples contained no data; can't train dictionary.")
96 dict_content = _zstd.train_dict(chunks, chunk_sizes, dict_size)
97 return ZstdDict(dict_content)
98
99
100 def finalize_dict(zstd_dict, /, samples, dict_size, level):
101 """Return a ZstdDict representing a finalized Zstandard dictionary.
102
103 Given a custom content as a basis for dictionary, and a set of samples,
104 finalize *zstd_dict* by adding headers and statistics according to the
105 Zstandard dictionary format.
106
107 You may compose an effective dictionary content by hand, which is used
108 as basis dictionary, and use some samples to finalize a dictionary. The
109 basis dictionary may be a "raw content" dictionary. See *is_raw* in
110 ZstdDict.
111
112 *samples* is an iterable of samples, where a sample is a bytes-like
113 object representing a file.
114 *dict_size* is the dictionary's maximum size, in bytes.
115 *level* is the expected compression level. The statistics for each
116 compression level differ, so tuning the dictionary to the compression
117 level can provide improvements.
118 """
119
120 if not isinstance(zstd_dict, ZstdDict):
121 raise TypeError('zstd_dict argument should be a ZstdDict object.')
122 if not isinstance(dict_size, int):
123 raise TypeError('dict_size argument should be an int object.')
124 if not isinstance(level, int):
125 raise TypeError('level argument should be an int object.')
126
127 samples = tuple(samples)
128 chunks = b''.join(samples)
129 chunk_sizes = tuple(_nbytes(sample) for sample in samples)
130 if not chunks:
131 raise ValueError("The samples are empty content, can't finalize the "
132 "dictionary.")
133 dict_content = _zstd.finalize_dict(zstd_dict.dict_content, chunks,
134 chunk_sizes, dict_size, level)
135 return ZstdDict(dict_content)
136
137
138 def compress(data, level=None, options=None, zstd_dict=None):
139 """Return Zstandard compressed *data* as bytes.
140
141 *level* is an int specifying the compression level to use, defaulting to
142 COMPRESSION_LEVEL_DEFAULT ('3').
143 *options* is a dict object that contains advanced compression
144 parameters. See CompressionParameter for more on options.
145 *zstd_dict* is a ZstdDict object, a pre-trained Zstandard dictionary.
146 See the function train_dict for how to train a ZstdDict on sample data.
147
148 For incremental compression, use a ZstdCompressor instead.
149 """
150 comp = ZstdCompressor(level=level, options=options, zstd_dict=zstd_dict)
151 return comp.compress(data, mode=ZstdCompressor.FLUSH_FRAME)
152
153
154 def decompress(data, zstd_dict=None, options=None):
155 """Decompress one or more frames of Zstandard compressed *data*.
156
157 *zstd_dict* is a ZstdDict object, a pre-trained Zstandard dictionary.
158 See the function train_dict for how to train a ZstdDict on sample data.
159 *options* is a dict object that contains advanced compression
160 parameters. See DecompressionParameter for more on options.
161
162 For incremental decompression, use a ZstdDecompressor instead.
163 """
164 results = []
165 while True:
166 decomp = ZstdDecompressor(options=options, zstd_dict=zstd_dict)
167 results.append(decomp.decompress(data))
168 if not decomp.eof:
169 raise ZstdError('Compressed data ended before the '
170 'end-of-stream marker was reached')
171 data = decomp.unused_data
172 if not data:
173 break
174 return b''.join(results)
175
176
177 class CompressionParameter(enum.IntEnum):
178 """Compression parameters."""
179
180 compression_level = _zstd.ZSTD_c_compressionLevel
181 window_log = _zstd.ZSTD_c_windowLog
182 hash_log = _zstd.ZSTD_c_hashLog
183 chain_log = _zstd.ZSTD_c_chainLog
184 search_log = _zstd.ZSTD_c_searchLog
185 min_match = _zstd.ZSTD_c_minMatch
186 target_length = _zstd.ZSTD_c_targetLength
187 strategy = _zstd.ZSTD_c_strategy
188
189 enable_long_distance_matching = _zstd.ZSTD_c_enableLongDistanceMatching
190 ldm_hash_log = _zstd.ZSTD_c_ldmHashLog
191 ldm_min_match = _zstd.ZSTD_c_ldmMinMatch
192 ldm_bucket_size_log = _zstd.ZSTD_c_ldmBucketSizeLog
193 ldm_hash_rate_log = _zstd.ZSTD_c_ldmHashRateLog
194
195 content_size_flag = _zstd.ZSTD_c_contentSizeFlag
196 checksum_flag = _zstd.ZSTD_c_checksumFlag
197 dict_id_flag = _zstd.ZSTD_c_dictIDFlag
198
199 nb_workers = _zstd.ZSTD_c_nbWorkers
200 job_size = _zstd.ZSTD_c_jobSize
201 overlap_log = _zstd.ZSTD_c_overlapLog
202
203 def bounds(self):
204 """Return the (lower, upper) int bounds of a compression parameter.
205
206 Both the lower and upper bounds are inclusive.
207 """
208 return _zstd.get_param_bounds(self.value, is_compress=True)
209
210
211 class DecompressionParameter(enum.IntEnum):
212 """Decompression parameters."""
213
214 window_log_max = _zstd.ZSTD_d_windowLogMax
215
216 def bounds(self):
217 """Return the (lower, upper) int bounds of a decompression parameter.
218
219 Both the lower and upper bounds are inclusive.
220 """
221 return _zstd.get_param_bounds(self.value, is_compress=False)
222
223
224 class Strategy(enum.IntEnum):
225 """Compression strategies, listed from fastest to strongest.
226
227 Note that new strategies might be added in the future.
228 Only the order (from fast to strong) is guaranteed,
229 the numeric value might change.
230 """
231
232 fast = _zstd.ZSTD_fast
233 dfast = _zstd.ZSTD_dfast
234 greedy = _zstd.ZSTD_greedy
235 lazy = _zstd.ZSTD_lazy
236 lazy2 = _zstd.ZSTD_lazy2
237 btlazy2 = _zstd.ZSTD_btlazy2
238 btopt = _zstd.ZSTD_btopt
239 btultra = _zstd.ZSTD_btultra
240 btultra2 = _zstd.ZSTD_btultra2
241
242
243 # Check validity of the CompressionParameter & DecompressionParameter types
244 _zstd.set_parameter_types(CompressionParameter, DecompressionParameter)