| 1 |
"""Python bindings to the Zstandard (zstd) compression library (RFC-8878).""" |
| 2 |
|
| 3 |
__all__ = ( |
| 4 |
# compression.zstd |
| 5 |
'COMPRESSION_LEVEL_DEFAULT', |
| 6 |
'compress', |
| 7 |
'CompressionParameter', |
| 8 |
'decompress', |
| 9 |
'DecompressionParameter', |
| 10 |
'finalize_dict', |
| 11 |
'get_frame_info', |
| 12 |
'Strategy', |
| 13 |
'train_dict', |
| 14 |
|
| 15 |
# compression.zstd._zstdfile |
| 16 |
'open', |
| 17 |
'ZstdFile', |
| 18 |
|
| 19 |
# _zstd |
| 20 |
'get_frame_size', |
| 21 |
'zstd_version', |
| 22 |
'zstd_version_info', |
| 23 |
'ZstdCompressor', |
| 24 |
'ZstdDecompressor', |
| 25 |
'ZstdDict', |
| 26 |
'ZstdError', |
| 27 |
) |
| 28 |
|
| 29 |
import _zstd |
| 30 |
import enum |
| 31 |
from _zstd import (ZstdCompressor, ZstdDecompressor, ZstdDict, ZstdError, |
| 32 |
get_frame_size, zstd_version) |
| 33 |
from compression.zstd._zstdfile import ZstdFile, open, _nbytes |
| 34 |
|
| 35 |
# zstd_version_number is (MAJOR * 100 * 100 + MINOR * 100 + RELEASE) |
| 36 |
zstd_version_info = (*divmod(_zstd.zstd_version_number // 100, 100), |
| 37 |
_zstd.zstd_version_number % 100) |
| 38 |
"""Version number of the runtime zstd library as a tuple of integers.""" |
| 39 |
|
| 40 |
COMPRESSION_LEVEL_DEFAULT = _zstd.ZSTD_CLEVEL_DEFAULT |
| 41 |
"""The default compression level for Zstandard, currently '3'.""" |
| 42 |
|
| 43 |
|
| 44 |
class FrameInfo: |
| 45 |
"""Information about a Zstandard frame.""" |
| 46 |
|
| 47 |
__slots__ = 'decompressed_size', 'dictionary_id' |
| 48 |
|
| 49 |
def __init__(self, decompressed_size, dictionary_id): |
| 50 |
super().__setattr__('decompressed_size', decompressed_size) |
| 51 |
super().__setattr__('dictionary_id', dictionary_id) |
| 52 |
|
| 53 |
def __repr__(self): |
| 54 |
return (f'FrameInfo(decompressed_size={self.decompressed_size}, ' |
| 55 |
f'dictionary_id={self.dictionary_id})') |
| 56 |
|
| 57 |
def __setattr__(self, name, _): |
| 58 |
raise AttributeError(f"can't set attribute {name!r}") |
| 59 |
|
| 60 |
|
| 61 |
def get_frame_info(frame_buffer): |
| 62 |
"""Get Zstandard frame information from a frame header. |
| 63 |
|
| 64 |
*frame_buffer* is a bytes-like object. It should start from the beginning |
| 65 |
of a frame, and needs to include at least the frame header (6 to 18 bytes). |
| 66 |
|
| 67 |
The returned FrameInfo object has two attributes. |
| 68 |
'decompressed_size' is the size in bytes of the data in the frame when |
| 69 |
decompressed, or None when the decompressed size is unknown. |
| 70 |
'dictionary_id' is an int in the range (0, 2**32). The special value 0 |
| 71 |
means that the dictionary ID was not recorded in the frame header, |
| 72 |
the frame may or may not need a dictionary to be decoded, |
| 73 |
and the ID of such a dictionary is not specified. |
| 74 |
""" |
| 75 |
return FrameInfo(*_zstd.get_frame_info(frame_buffer)) |
| 76 |
|
| 77 |
|
| 78 |
def train_dict(samples, dict_size): |
| 79 |
"""Return a ZstdDict representing a trained Zstandard dictionary. |
| 80 |
|
| 81 |
*samples* is an iterable of samples, where a sample is a bytes-like |
| 82 |
object representing a file. |
| 83 |
|
| 84 |
*dict_size* is the dictionary's maximum size, in bytes. |
| 85 |
""" |
| 86 |
if not isinstance(dict_size, int): |
| 87 |
ds_cls = type(dict_size).__qualname__ |
| 88 |
raise TypeError(f'dict_size must be an int object, not {ds_cls!r}.') |
| 89 |
|
| 90 |
samples = tuple(samples) |
| 91 |
chunks = b''.join(samples) |
| 92 |
chunk_sizes = tuple(_nbytes(sample) for sample in samples) |
| 93 |
if not chunks: |
| 94 |
raise ValueError("samples contained no data; can't train dictionary.") |
| 95 |
dict_content = _zstd.train_dict(chunks, chunk_sizes, dict_size) |
| 96 |
return ZstdDict(dict_content) |
| 97 |
|
| 98 |
|
| 99 |
def finalize_dict(zstd_dict, /, samples, dict_size, level): |
| 100 |
"""Return a ZstdDict representing a finalized Zstandard dictionary. |
| 101 |
|
| 102 |
Given a custom content as a basis for dictionary, and a set of samples, |
| 103 |
finalize *zstd_dict* by adding headers and statistics according to the |
| 104 |
Zstandard dictionary format. |
| 105 |
|
| 106 |
You may compose an effective dictionary content by hand, which is used as |
| 107 |
basis dictionary, and use some samples to finalize a dictionary. The basis |
| 108 |
dictionary may be a "raw content" dictionary. See *is_raw* in ZstdDict. |
| 109 |
|
| 110 |
*samples* is an iterable of samples, where a sample is a bytes-like object |
| 111 |
representing a file. |
| 112 |
*dict_size* is the dictionary's maximum size, in bytes. |
| 113 |
*level* is the expected compression level. The statistics for each |
| 114 |
compression level differ, so tuning the dictionary to the compression level |
| 115 |
can provide improvements. |
| 116 |
""" |
| 117 |
|
| 118 |
if not isinstance(zstd_dict, ZstdDict): |
| 119 |
raise TypeError('zstd_dict argument should be a ZstdDict object.') |
| 120 |
if not isinstance(dict_size, int): |
| 121 |
raise TypeError('dict_size argument should be an int object.') |
| 122 |
if not isinstance(level, int): |
| 123 |
raise TypeError('level argument should be an int object.') |
| 124 |
|
| 125 |
samples = tuple(samples) |
| 126 |
chunks = b''.join(samples) |
| 127 |
chunk_sizes = tuple(_nbytes(sample) for sample in samples) |
| 128 |
if not chunks: |
| 129 |
raise ValueError("The samples are empty content, can't finalize the " |
| 130 |
"dictionary.") |
| 131 |
dict_content = _zstd.finalize_dict(zstd_dict.dict_content, chunks, |
| 132 |
chunk_sizes, dict_size, level) |
| 133 |
return ZstdDict(dict_content) |
| 134 |
|
| 135 |
|
| 136 |
def compress(data, level=None, options=None, zstd_dict=None): |
| 137 |
"""Return Zstandard compressed *data* as bytes. |
| 138 |
|
| 139 |
*level* is an int specifying the compression level to use, defaulting to |
| 140 |
COMPRESSION_LEVEL_DEFAULT ('3'). |
| 141 |
*options* is a dict object that contains advanced compression |
| 142 |
parameters. See CompressionParameter for more on options. |
| 143 |
*zstd_dict* is a ZstdDict object, a pre-trained Zstandard dictionary. See |
| 144 |
the function train_dict for how to train a ZstdDict on sample data. |
| 145 |
|
| 146 |
For incremental compression, use a ZstdCompressor instead. |
| 147 |
""" |
| 148 |
comp = ZstdCompressor(level=level, options=options, zstd_dict=zstd_dict) |
| 149 |
return comp.compress(data, mode=ZstdCompressor.FLUSH_FRAME) |
| 150 |
|
| 151 |
|
| 152 |
def decompress(data, zstd_dict=None, options=None): |
| 153 |
"""Decompress one or more frames of Zstandard compressed *data*. |
| 154 |
|
| 155 |
*zstd_dict* is a ZstdDict object, a pre-trained Zstandard dictionary. See |
| 156 |
the function train_dict for how to train a ZstdDict on sample data. |
| 157 |
*options* is a dict object that contains advanced compression |
| 158 |
parameters. See DecompressionParameter for more on options. |
| 159 |
|
| 160 |
For incremental decompression, use a ZstdDecompressor instead. |
| 161 |
""" |
| 162 |
results = [] |
| 163 |
while True: |
| 164 |
decomp = ZstdDecompressor(options=options, zstd_dict=zstd_dict) |
| 165 |
results.append(decomp.decompress(data)) |
| 166 |
if not decomp.eof: |
| 167 |
raise ZstdError('Compressed data ended before the ' |
| 168 |
'end-of-stream marker was reached') |
| 169 |
data = decomp.unused_data |
| 170 |
if not data: |
| 171 |
break |
| 172 |
return b''.join(results) |
| 173 |
|
| 174 |
|
| 175 |
class CompressionParameter(enum.IntEnum): |
| 176 |
"""Compression parameters.""" |
| 177 |
|
| 178 |
compression_level = _zstd.ZSTD_c_compressionLevel |
| 179 |
window_log = _zstd.ZSTD_c_windowLog |
| 180 |
hash_log = _zstd.ZSTD_c_hashLog |
| 181 |
chain_log = _zstd.ZSTD_c_chainLog |
| 182 |
search_log = _zstd.ZSTD_c_searchLog |
| 183 |
min_match = _zstd.ZSTD_c_minMatch |
| 184 |
target_length = _zstd.ZSTD_c_targetLength |
| 185 |
strategy = _zstd.ZSTD_c_strategy |
| 186 |
|
| 187 |
enable_long_distance_matching = _zstd.ZSTD_c_enableLongDistanceMatching |
| 188 |
ldm_hash_log = _zstd.ZSTD_c_ldmHashLog |
| 189 |
ldm_min_match = _zstd.ZSTD_c_ldmMinMatch |
| 190 |
ldm_bucket_size_log = _zstd.ZSTD_c_ldmBucketSizeLog |
| 191 |
ldm_hash_rate_log = _zstd.ZSTD_c_ldmHashRateLog |
| 192 |
|
| 193 |
content_size_flag = _zstd.ZSTD_c_contentSizeFlag |
| 194 |
checksum_flag = _zstd.ZSTD_c_checksumFlag |
| 195 |
dict_id_flag = _zstd.ZSTD_c_dictIDFlag |
| 196 |
|
| 197 |
nb_workers = _zstd.ZSTD_c_nbWorkers |
| 198 |
job_size = _zstd.ZSTD_c_jobSize |
| 199 |
overlap_log = _zstd.ZSTD_c_overlapLog |
| 200 |
|
| 201 |
def bounds(self): |
| 202 |
"""Return the (lower, upper) int bounds of a compression parameter. |
| 203 |
|
| 204 |
Both the lower and upper bounds are inclusive. |
| 205 |
""" |
| 206 |
return _zstd.get_param_bounds(self.value, is_compress=True) |
| 207 |
|
| 208 |
|
| 209 |
class DecompressionParameter(enum.IntEnum): |
| 210 |
"""Decompression parameters.""" |
| 211 |
|
| 212 |
window_log_max = _zstd.ZSTD_d_windowLogMax |
| 213 |
|
| 214 |
def bounds(self): |
| 215 |
"""Return the (lower, upper) int bounds of a decompression parameter. |
| 216 |
|
| 217 |
Both the lower and upper bounds are inclusive. |
| 218 |
""" |
| 219 |
return _zstd.get_param_bounds(self.value, is_compress=False) |
| 220 |
|
| 221 |
|
| 222 |
class Strategy(enum.IntEnum): |
| 223 |
"""Compression strategies, listed from fastest to strongest. |
| 224 |
|
| 225 |
Note that new strategies might be added in the future. |
| 226 |
Only the order (from fast to strong) is guaranteed, |
| 227 |
the numeric value might change. |
| 228 |
""" |
| 229 |
|
| 230 |
fast = _zstd.ZSTD_fast |
| 231 |
dfast = _zstd.ZSTD_dfast |
| 232 |
greedy = _zstd.ZSTD_greedy |
| 233 |
lazy = _zstd.ZSTD_lazy |
| 234 |
lazy2 = _zstd.ZSTD_lazy2 |
| 235 |
btlazy2 = _zstd.ZSTD_btlazy2 |
| 236 |
btopt = _zstd.ZSTD_btopt |
| 237 |
btultra = _zstd.ZSTD_btultra |
| 238 |
btultra2 = _zstd.ZSTD_btultra2 |
| 239 |
|
| 240 |
|
| 241 |
# Check validity of the CompressionParameter & DecompressionParameter types |
| 242 |
_zstd.set_parameter_types(CompressionParameter, DecompressionParameter) |