| 1 |
/* Low level interface to the Zstandard algorithm & the zstd library. */ |
| 2 |
|
| 3 |
/* ZstdDecompressor class definition */ |
| 4 |
|
| 5 |
/*[clinic input] |
| 6 |
module _zstd |
| 7 |
class _zstd.ZstdDecompressor "ZstdDecompressor *" "&zstd_decompressor_type_spec" |
| 8 |
[clinic start generated code]*/ |
| 9 |
/*[clinic end generated code: output=da39a3ee5e6b4b0d input=e2969ddf48a203e0]*/ |
| 10 |
|
| 11 |
#ifndef Py_BUILD_CORE_BUILTIN |
| 12 |
# define Py_BUILD_CORE_MODULE 1 |
| 13 |
#endif |
| 14 |
|
| 15 |
#include "Python.h" |
| 16 |
|
| 17 |
#include "_zstdmodule.h" |
| 18 |
#include "buffer.h" |
| 19 |
#include "internal/pycore_lock.h" // PyMutex_IsLocked |
| 20 |
|
| 21 |
#include <stdbool.h> // bool |
| 22 |
#include <stddef.h> // offsetof() |
| 23 |
#include <zstd.h> // ZSTD_*() |
| 24 |
|
| 25 |
typedef struct { |
| 26 |
PyObject_HEAD |
| 27 |
|
| 28 |
/* Decompression context */ |
| 29 |
ZSTD_DCtx *dctx; |
| 30 |
|
| 31 |
/* ZstdDict object in use */ |
| 32 |
PyObject *dict; |
| 33 |
|
| 34 |
/* Unconsumed input data */ |
| 35 |
char *input_buffer; |
| 36 |
size_t input_buffer_size; |
| 37 |
size_t in_begin, in_end; |
| 38 |
|
| 39 |
/* Unused data */ |
| 40 |
PyObject *unused_data; |
| 41 |
|
| 42 |
/* 0 if decompressor has (or may has) unconsumed input data, 0 or 1. */ |
| 43 |
bool needs_input; |
| 44 |
|
| 45 |
/* For ZstdDecompressor, 0 or 1. |
| 46 |
1 means the end of the first frame has been reached. */ |
| 47 |
bool eof; |
| 48 |
|
| 49 |
/* Lock to protect the decompression context */ |
| 50 |
PyMutex lock; |
| 51 |
} ZstdDecompressor; |
| 52 |
|
| 53 |
#define ZstdDecompressor_CAST(op) ((ZstdDecompressor *)op) |
| 54 |
|
| 55 |
#include "clinic/decompressor.c.h" |
| 56 |
|
| 57 |
static inline ZSTD_DDict * |
| 58 |
_get_DDict(ZstdDict *self) |
| 59 |
{ |
| 60 |
assert(PyMutex_IsLocked(&self->lock)); |
| 61 |
ZSTD_DDict *ret; |
| 62 |
|
| 63 |
if (self->d_dict == NULL) { |
| 64 |
/* Create ZSTD_DDict instance from dictionary content */ |
| 65 |
Py_BEGIN_ALLOW_THREADS |
| 66 |
ret = ZSTD_createDDict(self->dict_buffer, self->dict_len); |
| 67 |
Py_END_ALLOW_THREADS |
| 68 |
self->d_dict = ret; |
| 69 |
|
| 70 |
if (self->d_dict == NULL) { |
| 71 |
_zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self)); |
| 72 |
if (mod_state != NULL) { |
| 73 |
PyErr_SetString(mod_state->ZstdError, |
| 74 |
"Failed to create a ZSTD_DDict instance from " |
| 75 |
"Zstandard dictionary content."); |
| 76 |
} |
| 77 |
} |
| 78 |
} |
| 79 |
|
| 80 |
return self->d_dict; |
| 81 |
} |
| 82 |
|
| 83 |
static int |
| 84 |
_zstd_set_d_parameters(ZstdDecompressor *self, PyObject *options) |
| 85 |
{ |
| 86 |
_zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self)); |
| 87 |
if (mod_state == NULL) { |
| 88 |
return -1; |
| 89 |
} |
| 90 |
|
| 91 |
if (!PyDict_Check(options)) { |
| 92 |
PyErr_Format(PyExc_TypeError, |
| 93 |
"ZstdDecompressor() argument 'options' must be dict, not %T", |
| 94 |
options); |
| 95 |
return -1; |
| 96 |
} |
| 97 |
|
| 98 |
Py_ssize_t pos = 0; |
| 99 |
PyObject *key, *value; |
| 100 |
while (PyDict_Next(options, &pos, &key, &value)) { |
| 101 |
/* Check key type */ |
| 102 |
if (Py_TYPE(key) == mod_state->CParameter_type) { |
| 103 |
PyErr_SetString(PyExc_TypeError, |
| 104 |
"decompression options dictionary key must not be a " |
| 105 |
"CompressionParameter attribute"); |
| 106 |
return -1; |
| 107 |
} |
| 108 |
|
| 109 |
Py_INCREF(key); |
| 110 |
Py_INCREF(value); |
| 111 |
int key_v = PyLong_AsInt(key); |
| 112 |
Py_DECREF(key); |
| 113 |
if (key_v == -1 && PyErr_Occurred()) { |
| 114 |
Py_DECREF(value); |
| 115 |
return -1; |
| 116 |
} |
| 117 |
|
| 118 |
int value_v = PyLong_AsInt(value); |
| 119 |
Py_DECREF(value); |
| 120 |
if (value_v == -1 && PyErr_Occurred()) { |
| 121 |
return -1; |
| 122 |
} |
| 123 |
|
| 124 |
/* Set parameter to compression context */ |
| 125 |
size_t zstd_ret = ZSTD_DCtx_setParameter(self->dctx, key_v, value_v); |
| 126 |
|
| 127 |
/* Check error */ |
| 128 |
if (ZSTD_isError(zstd_ret)) { |
| 129 |
set_parameter_error(0, key_v, value_v); |
| 130 |
return -1; |
| 131 |
} |
| 132 |
} |
| 133 |
return 0; |
| 134 |
} |
| 135 |
|
| 136 |
static int |
| 137 |
_zstd_load_impl(ZstdDecompressor *self, ZstdDict *zd, |
| 138 |
_zstd_state *mod_state, int type) |
| 139 |
{ |
| 140 |
size_t zstd_ret; |
| 141 |
if (type == DICT_TYPE_DIGESTED) { |
| 142 |
/* Get ZSTD_DDict */ |
| 143 |
ZSTD_DDict *d_dict = _get_DDict(zd); |
| 144 |
if (d_dict == NULL) { |
| 145 |
return -1; |
| 146 |
} |
| 147 |
/* Reference a prepared dictionary */ |
| 148 |
zstd_ret = ZSTD_DCtx_refDDict(self->dctx, d_dict); |
| 149 |
} |
| 150 |
else if (type == DICT_TYPE_UNDIGESTED) { |
| 151 |
/* Load a dictionary */ |
| 152 |
zstd_ret = ZSTD_DCtx_loadDictionary(self->dctx, zd->dict_buffer, |
| 153 |
zd->dict_len); |
| 154 |
} |
| 155 |
else if (type == DICT_TYPE_PREFIX) { |
| 156 |
/* Load a prefix */ |
| 157 |
zstd_ret = ZSTD_DCtx_refPrefix(self->dctx, zd->dict_buffer, |
| 158 |
zd->dict_len); |
| 159 |
} |
| 160 |
else { |
| 161 |
Py_UNREACHABLE(); |
| 162 |
} |
| 163 |
|
| 164 |
/* Check error */ |
| 165 |
if (ZSTD_isError(zstd_ret)) { |
| 166 |
set_zstd_error(mod_state, ERR_LOAD_D_DICT, zstd_ret); |
| 167 |
return -1; |
| 168 |
} |
| 169 |
return 0; |
| 170 |
} |
| 171 |
|
| 172 |
/* Load dictionary or prefix to decompression context */ |
| 173 |
static int |
| 174 |
_zstd_load_d_dict(ZstdDecompressor *self, PyObject *dict) |
| 175 |
{ |
| 176 |
_zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self)); |
| 177 |
/* When decompressing, use digested dictionary by default. */ |
| 178 |
int type = DICT_TYPE_DIGESTED; |
| 179 |
ZstdDict *zd = _Py_parse_zstd_dict(mod_state, dict, &type); |
| 180 |
if (zd == NULL) { |
| 181 |
return -1; |
| 182 |
} |
| 183 |
int ret; |
| 184 |
PyMutex_Lock(&zd->lock); |
| 185 |
ret = _zstd_load_impl(self, zd, mod_state, type); |
| 186 |
PyMutex_Unlock(&zd->lock); |
| 187 |
return ret; |
| 188 |
} |
| 189 |
|
| 190 |
/* |
| 191 |
Decompress implementation in pseudo code: |
| 192 |
|
| 193 |
initialize_output_buffer |
| 194 |
while True: |
| 195 |
decompress_data |
| 196 |
set_object_flag # .eof |
| 197 |
|
| 198 |
if output_buffer_exhausted: |
| 199 |
if output_buffer_reached_max_length: |
| 200 |
finish |
| 201 |
grow_output_buffer |
| 202 |
elif input_buffer_exhausted: |
| 203 |
finish |
| 204 |
|
| 205 |
ZSTD_decompressStream()'s size_t return value: |
| 206 |
- 0 when a frame is completely decoded and fully flushed, |
| 207 |
zstd's internal buffer has no data. |
| 208 |
- An error code, which can be tested using ZSTD_isError(). |
| 209 |
- Or any other value > 0, which means there is still some decoding or |
| 210 |
flushing to do to complete current frame. |
| 211 |
|
| 212 |
Note, decompressing "an empty input" in any case will make it > 0. |
| 213 |
*/ |
| 214 |
static PyObject * |
| 215 |
decompress_lock_held(ZstdDecompressor *self, ZSTD_inBuffer *in, |
| 216 |
Py_ssize_t max_length) |
| 217 |
{ |
| 218 |
size_t zstd_ret; |
| 219 |
ZSTD_outBuffer out; |
| 220 |
_BlocksOutputBuffer buffer = {.list = NULL}; |
| 221 |
PyObject *ret; |
| 222 |
|
| 223 |
/* Initialize the output buffer */ |
| 224 |
if (_OutputBuffer_InitAndGrow(&buffer, &out, max_length) < 0) { |
| 225 |
goto error; |
| 226 |
} |
| 227 |
assert(out.pos == 0); |
| 228 |
|
| 229 |
while (1) { |
| 230 |
/* Decompress */ |
| 231 |
Py_BEGIN_ALLOW_THREADS |
| 232 |
zstd_ret = ZSTD_decompressStream(self->dctx, &out, in); |
| 233 |
Py_END_ALLOW_THREADS |
| 234 |
|
| 235 |
/* Check error */ |
| 236 |
if (ZSTD_isError(zstd_ret)) { |
| 237 |
_zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self)); |
| 238 |
set_zstd_error(mod_state, ERR_DECOMPRESS, zstd_ret); |
| 239 |
goto error; |
| 240 |
} |
| 241 |
|
| 242 |
/* Set .eof flag */ |
| 243 |
if (zstd_ret == 0) { |
| 244 |
/* Stop when a frame is decompressed */ |
| 245 |
self->eof = 1; |
| 246 |
break; |
| 247 |
} |
| 248 |
|
| 249 |
/* Need to check out before in. Maybe zstd's internal buffer still has |
| 250 |
a few bytes that can be output, grow the buffer and continue. */ |
| 251 |
if (out.pos == out.size) { |
| 252 |
/* Output buffer exhausted */ |
| 253 |
|
| 254 |
/* Output buffer reached max_length */ |
| 255 |
if (_OutputBuffer_ReachedMaxLength(&buffer, &out)) { |
| 256 |
break; |
| 257 |
} |
| 258 |
|
| 259 |
/* Grow output buffer */ |
| 260 |
if (_OutputBuffer_Grow(&buffer, &out) < 0) { |
| 261 |
goto error; |
| 262 |
} |
| 263 |
assert(out.pos == 0); |
| 264 |
|
| 265 |
} |
| 266 |
else if (in->pos == in->size) { |
| 267 |
/* Finished */ |
| 268 |
break; |
| 269 |
} |
| 270 |
} |
| 271 |
|
| 272 |
/* Return a bytes object */ |
| 273 |
ret = _OutputBuffer_Finish(&buffer, &out); |
| 274 |
if (ret != NULL) { |
| 275 |
return ret; |
| 276 |
} |
| 277 |
|
| 278 |
error: |
| 279 |
_OutputBuffer_OnError(&buffer); |
| 280 |
return NULL; |
| 281 |
} |
| 282 |
|
| 283 |
static void |
| 284 |
decompressor_reset_session_lock_held(ZstdDecompressor *self) |
| 285 |
{ |
| 286 |
assert(PyMutex_IsLocked(&self->lock)); |
| 287 |
|
| 288 |
/* Reset variables */ |
| 289 |
self->in_begin = 0; |
| 290 |
self->in_end = 0; |
| 291 |
|
| 292 |
Py_CLEAR(self->unused_data); |
| 293 |
|
| 294 |
/* Reset variables in one operation */ |
| 295 |
self->needs_input = 1; |
| 296 |
self->eof = 0; |
| 297 |
|
| 298 |
/* Resetting session is guaranteed to never fail */ |
| 299 |
ZSTD_DCtx_reset(self->dctx, ZSTD_reset_session_only); |
| 300 |
} |
| 301 |
|
| 302 |
static PyObject * |
| 303 |
stream_decompress_lock_held(ZstdDecompressor *self, Py_buffer *data, |
| 304 |
Py_ssize_t max_length) |
| 305 |
{ |
| 306 |
assert(PyMutex_IsLocked(&self->lock)); |
| 307 |
ZSTD_inBuffer in; |
| 308 |
PyObject *ret = NULL; |
| 309 |
int use_input_buffer; |
| 310 |
|
| 311 |
/* Check .eof flag */ |
| 312 |
if (self->eof) { |
| 313 |
PyErr_SetString(PyExc_EOFError, |
| 314 |
"Already at the end of a Zstandard frame."); |
| 315 |
assert(ret == NULL); |
| 316 |
return NULL; |
| 317 |
} |
| 318 |
|
| 319 |
/* Prepare input buffer w/wo unconsumed data */ |
| 320 |
if (self->in_begin == self->in_end) { |
| 321 |
/* No unconsumed data */ |
| 322 |
use_input_buffer = 0; |
| 323 |
|
| 324 |
in.src = data->buf; |
| 325 |
in.size = data->len; |
| 326 |
in.pos = 0; |
| 327 |
} |
| 328 |
else if (data->len == 0) { |
| 329 |
/* Has unconsumed data, fast path for b'' */ |
| 330 |
assert(self->in_begin < self->in_end); |
| 331 |
|
| 332 |
use_input_buffer = 1; |
| 333 |
|
| 334 |
in.src = self->input_buffer + self->in_begin; |
| 335 |
in.size = self->in_end - self->in_begin; |
| 336 |
in.pos = 0; |
| 337 |
} |
| 338 |
else { |
| 339 |
/* Has unconsumed data */ |
| 340 |
use_input_buffer = 1; |
| 341 |
|
| 342 |
/* Unconsumed data size in input_buffer */ |
| 343 |
size_t used_now = self->in_end - self->in_begin; |
| 344 |
assert(self->in_end > self->in_begin); |
| 345 |
|
| 346 |
/* Number of bytes we can append to input buffer */ |
| 347 |
size_t avail_now = self->input_buffer_size - self->in_end; |
| 348 |
assert(self->input_buffer_size >= self->in_end); |
| 349 |
|
| 350 |
/* Number of bytes we can append if we move existing contents to |
| 351 |
beginning of buffer */ |
| 352 |
size_t avail_total = self->input_buffer_size - used_now; |
| 353 |
assert(self->input_buffer_size >= used_now); |
| 354 |
|
| 355 |
if (avail_total < (size_t) data->len) { |
| 356 |
char *tmp; |
| 357 |
size_t new_size = used_now + data->len; |
| 358 |
|
| 359 |
/* Allocate with new size */ |
| 360 |
tmp = PyMem_Malloc(new_size); |
| 361 |
if (tmp == NULL) { |
| 362 |
PyErr_NoMemory(); |
| 363 |
goto error; |
| 364 |
} |
| 365 |
|
| 366 |
/* Copy unconsumed data to the beginning of new buffer */ |
| 367 |
memcpy(tmp, |
| 368 |
self->input_buffer + self->in_begin, |
| 369 |
used_now); |
| 370 |
|
| 371 |
/* Switch to new buffer */ |
| 372 |
PyMem_Free(self->input_buffer); |
| 373 |
self->input_buffer = tmp; |
| 374 |
self->input_buffer_size = new_size; |
| 375 |
|
| 376 |
/* Set begin & end position */ |
| 377 |
self->in_begin = 0; |
| 378 |
self->in_end = used_now; |
| 379 |
} |
| 380 |
else if (avail_now < (size_t) data->len) { |
| 381 |
/* Move unconsumed data to the beginning. |
| 382 |
Overlap is possible, so use memmove(). */ |
| 383 |
memmove(self->input_buffer, |
| 384 |
self->input_buffer + self->in_begin, |
| 385 |
used_now); |
| 386 |
|
| 387 |
/* Set begin & end position */ |
| 388 |
self->in_begin = 0; |
| 389 |
self->in_end = used_now; |
| 390 |
} |
| 391 |
|
| 392 |
/* Copy data to input buffer */ |
| 393 |
memcpy(self->input_buffer + self->in_end, data->buf, data->len); |
| 394 |
self->in_end += data->len; |
| 395 |
|
| 396 |
in.src = self->input_buffer + self->in_begin; |
| 397 |
in.size = used_now + data->len; |
| 398 |
in.pos = 0; |
| 399 |
} |
| 400 |
assert(in.pos == 0); |
| 401 |
|
| 402 |
/* Decompress */ |
| 403 |
ret = decompress_lock_held(self, &in, max_length); |
| 404 |
if (ret == NULL) { |
| 405 |
goto error; |
| 406 |
} |
| 407 |
|
| 408 |
/* Unconsumed input data */ |
| 409 |
if (in.pos == in.size) { |
| 410 |
if (Py_SIZE(ret) == max_length || self->eof) { |
| 411 |
self->needs_input = 0; |
| 412 |
} |
| 413 |
else { |
| 414 |
self->needs_input = 1; |
| 415 |
} |
| 416 |
|
| 417 |
if (use_input_buffer) { |
| 418 |
/* Clear input_buffer */ |
| 419 |
self->in_begin = 0; |
| 420 |
self->in_end = 0; |
| 421 |
} |
| 422 |
} |
| 423 |
else { |
| 424 |
size_t data_size = in.size - in.pos; |
| 425 |
|
| 426 |
self->needs_input = 0; |
| 427 |
|
| 428 |
if (!use_input_buffer) { |
| 429 |
/* Discard buffer if it's too small |
| 430 |
(resizing it may needlessly copy the current contents) */ |
| 431 |
if (self->input_buffer != NULL |
| 432 |
&& self->input_buffer_size < data_size) |
| 433 |
{ |
| 434 |
PyMem_Free(self->input_buffer); |
| 435 |
self->input_buffer = NULL; |
| 436 |
self->input_buffer_size = 0; |
| 437 |
} |
| 438 |
|
| 439 |
/* Allocate if necessary */ |
| 440 |
if (self->input_buffer == NULL) { |
| 441 |
self->input_buffer = PyMem_Malloc(data_size); |
| 442 |
if (self->input_buffer == NULL) { |
| 443 |
PyErr_NoMemory(); |
| 444 |
goto error; |
| 445 |
} |
| 446 |
self->input_buffer_size = data_size; |
| 447 |
} |
| 448 |
|
| 449 |
/* Copy unconsumed data */ |
| 450 |
memcpy(self->input_buffer, (char*)in.src + in.pos, data_size); |
| 451 |
self->in_begin = 0; |
| 452 |
self->in_end = data_size; |
| 453 |
} |
| 454 |
else { |
| 455 |
/* Use input buffer */ |
| 456 |
self->in_begin += in.pos; |
| 457 |
} |
| 458 |
} |
| 459 |
|
| 460 |
return ret; |
| 461 |
|
| 462 |
error: |
| 463 |
/* Reset decompressor's states/session */ |
| 464 |
decompressor_reset_session_lock_held(self); |
| 465 |
|
| 466 |
Py_CLEAR(ret); |
| 467 |
return NULL; |
| 468 |
} |
| 469 |
|
| 470 |
|
| 471 |
/*[clinic input] |
| 472 |
@classmethod |
| 473 |
_zstd.ZstdDecompressor.__new__ as _zstd_ZstdDecompressor_new |
| 474 |
zstd_dict: object = None |
| 475 |
A ZstdDict object, a pre-trained Zstandard dictionary. |
| 476 |
options: object = None |
| 477 |
A dict object that contains advanced decompression parameters. |
| 478 |
|
| 479 |
Create a decompressor object for decompressing data incrementally. |
| 480 |
|
| 481 |
Thread-safe at method level. For one-shot decompression, use the decompress() |
| 482 |
function instead. |
| 483 |
[clinic start generated code]*/ |
| 484 |
|
| 485 |
static PyObject * |
| 486 |
_zstd_ZstdDecompressor_new_impl(PyTypeObject *type, PyObject *zstd_dict, |
| 487 |
PyObject *options) |
| 488 |
/*[clinic end generated code: output=590ca65c1102ff4a input=213daa57e3ea4062]*/ |
| 489 |
{ |
| 490 |
ZstdDecompressor* self = PyObject_GC_New(ZstdDecompressor, type); |
| 491 |
if (self == NULL) { |
| 492 |
goto error; |
| 493 |
} |
| 494 |
|
| 495 |
self->input_buffer = NULL; |
| 496 |
self->input_buffer_size = 0; |
| 497 |
self->in_begin = -1; |
| 498 |
self->in_end = -1; |
| 499 |
self->unused_data = NULL; |
| 500 |
self->eof = 0; |
| 501 |
self->dict = NULL; |
| 502 |
self->lock = (PyMutex){0}; |
| 503 |
|
| 504 |
/* needs_input flag */ |
| 505 |
self->needs_input = 1; |
| 506 |
|
| 507 |
/* Decompression context */ |
| 508 |
self->dctx = ZSTD_createDCtx(); |
| 509 |
if (self->dctx == NULL) { |
| 510 |
_zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self)); |
| 511 |
if (mod_state != NULL) { |
| 512 |
PyErr_SetString(mod_state->ZstdError, |
| 513 |
"Unable to create ZSTD_DCtx instance."); |
| 514 |
} |
| 515 |
goto error; |
| 516 |
} |
| 517 |
|
| 518 |
/* Load Zstandard dictionary to decompression context */ |
| 519 |
if (zstd_dict != Py_None) { |
| 520 |
if (_zstd_load_d_dict(self, zstd_dict) < 0) { |
| 521 |
goto error; |
| 522 |
} |
| 523 |
Py_INCREF(zstd_dict); |
| 524 |
self->dict = zstd_dict; |
| 525 |
} |
| 526 |
|
| 527 |
/* Set options dictionary */ |
| 528 |
if (options != Py_None) { |
| 529 |
if (_zstd_set_d_parameters(self, options) < 0) { |
| 530 |
goto error; |
| 531 |
} |
| 532 |
} |
| 533 |
|
| 534 |
// We can only start GC tracking once self->dict is set. |
| 535 |
PyObject_GC_Track(self); |
| 536 |
|
| 537 |
return (PyObject*)self; |
| 538 |
|
| 539 |
error: |
| 540 |
Py_XDECREF(self); |
| 541 |
return NULL; |
| 542 |
} |
| 543 |
|
| 544 |
static void |
| 545 |
ZstdDecompressor_dealloc(PyObject *ob) |
| 546 |
{ |
| 547 |
ZstdDecompressor *self = ZstdDecompressor_CAST(ob); |
| 548 |
|
| 549 |
PyObject_GC_UnTrack(self); |
| 550 |
|
| 551 |
/* Free decompression context */ |
| 552 |
if (self->dctx) { |
| 553 |
ZSTD_freeDCtx(self->dctx); |
| 554 |
} |
| 555 |
|
| 556 |
assert(!PyMutex_IsLocked(&self->lock)); |
| 557 |
|
| 558 |
/* Py_CLEAR the dict after free decompression context */ |
| 559 |
Py_CLEAR(self->dict); |
| 560 |
|
| 561 |
/* Free unconsumed input data buffer */ |
| 562 |
PyMem_Free(self->input_buffer); |
| 563 |
|
| 564 |
/* Free unused data */ |
| 565 |
Py_CLEAR(self->unused_data); |
| 566 |
|
| 567 |
PyTypeObject *tp = Py_TYPE(self); |
| 568 |
PyObject_GC_Del(ob); |
| 569 |
Py_DECREF(tp); |
| 570 |
} |
| 571 |
|
| 572 |
/*[clinic input] |
| 573 |
@getter |
| 574 |
_zstd.ZstdDecompressor.unused_data |
| 575 |
|
| 576 |
A bytes object of un-consumed input data. |
| 577 |
|
| 578 |
When ZstdDecompressor object stops after a frame is |
| 579 |
decompressed, unused input data after the frame. Otherwise this will be b''. |
| 580 |
[clinic start generated code]*/ |
| 581 |
|
| 582 |
static PyObject * |
| 583 |
_zstd_ZstdDecompressor_unused_data_get_impl(ZstdDecompressor *self) |
| 584 |
/*[clinic end generated code: output=f3a20940f11b6b09 input=54d41ecd681a3444]*/ |
| 585 |
{ |
| 586 |
PyObject *ret; |
| 587 |
|
| 588 |
PyMutex_Lock(&self->lock); |
| 589 |
|
| 590 |
if (!self->eof) { |
| 591 |
PyMutex_Unlock(&self->lock); |
| 592 |
return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); |
| 593 |
} |
| 594 |
else { |
| 595 |
if (self->unused_data == NULL) { |
| 596 |
self->unused_data = PyBytes_FromStringAndSize( |
| 597 |
self->input_buffer + self->in_begin, |
| 598 |
self->in_end - self->in_begin); |
| 599 |
ret = self->unused_data; |
| 600 |
Py_XINCREF(ret); |
| 601 |
} |
| 602 |
else { |
| 603 |
ret = self->unused_data; |
| 604 |
Py_INCREF(ret); |
| 605 |
} |
| 606 |
} |
| 607 |
|
| 608 |
PyMutex_Unlock(&self->lock); |
| 609 |
return ret; |
| 610 |
} |
| 611 |
|
| 612 |
/*[clinic input] |
| 613 |
_zstd.ZstdDecompressor.decompress |
| 614 |
|
| 615 |
data: Py_buffer |
| 616 |
A bytes-like object, Zstandard data to be decompressed. |
| 617 |
max_length: Py_ssize_t = -1 |
| 618 |
Maximum size of returned data. When it is negative, the size of |
| 619 |
output buffer is unlimited. When it is nonnegative, returns at |
| 620 |
most max_length bytes of decompressed data. |
| 621 |
|
| 622 |
Decompress *data*, returning uncompressed bytes if possible, or b'' otherwise. |
| 623 |
|
| 624 |
If *max_length* is nonnegative, returns at most *max_length* bytes of |
| 625 |
decompressed data. If this limit is reached and further output can be |
| 626 |
produced, *self.needs_input* will be set to ``False``. In this case, the next |
| 627 |
call to *decompress()* may provide *data* as b'' to obtain more of the output. |
| 628 |
|
| 629 |
If all of the input data was decompressed and returned (either because this |
| 630 |
was less than *max_length* bytes, or because *max_length* was negative), |
| 631 |
*self.needs_input* will be set to True. |
| 632 |
|
| 633 |
Attempting to decompress data after the end of a frame is reached raises an |
| 634 |
EOFError. Any data found after the end of the frame is ignored and saved in |
| 635 |
the self.unused_data attribute. |
| 636 |
[clinic start generated code]*/ |
| 637 |
|
| 638 |
static PyObject * |
| 639 |
_zstd_ZstdDecompressor_decompress_impl(ZstdDecompressor *self, |
| 640 |
Py_buffer *data, |
| 641 |
Py_ssize_t max_length) |
| 642 |
/*[clinic end generated code: output=a4302b3c940dbec6 input=6463dfdf98091caa]*/ |
| 643 |
{ |
| 644 |
PyObject *ret; |
| 645 |
/* Thread-safe code */ |
| 646 |
PyMutex_Lock(&self->lock); |
| 647 |
ret = stream_decompress_lock_held(self, data, max_length); |
| 648 |
PyMutex_Unlock(&self->lock); |
| 649 |
return ret; |
| 650 |
} |
| 651 |
|
| 652 |
static PyMethodDef ZstdDecompressor_methods[] = { |
| 653 |
_ZSTD_ZSTDDECOMPRESSOR_DECOMPRESS_METHODDEF |
| 654 |
{NULL, NULL} |
| 655 |
}; |
| 656 |
|
| 657 |
PyDoc_STRVAR(ZstdDecompressor_eof_doc, |
| 658 |
"True means the end of the first frame has been reached. If decompress data\n" |
| 659 |
"after that, an EOFError exception will be raised."); |
| 660 |
|
| 661 |
PyDoc_STRVAR(ZstdDecompressor_needs_input_doc, |
| 662 |
"If the max_length output limit in .decompress() method has been reached,\n" |
| 663 |
"and the decompressor has (or may has) unconsumed input data, it will be set\n" |
| 664 |
"to False. In this case, passing b'' to the .decompress() method may output\n" |
| 665 |
"further data."); |
| 666 |
|
| 667 |
static PyMemberDef ZstdDecompressor_members[] = { |
| 668 |
{"eof", Py_T_BOOL, offsetof(ZstdDecompressor, eof), |
| 669 |
Py_READONLY, ZstdDecompressor_eof_doc}, |
| 670 |
{"needs_input", Py_T_BOOL, offsetof(ZstdDecompressor, needs_input), |
| 671 |
Py_READONLY, ZstdDecompressor_needs_input_doc}, |
| 672 |
{NULL} |
| 673 |
}; |
| 674 |
|
| 675 |
static PyGetSetDef ZstdDecompressor_getset[] = { |
| 676 |
_ZSTD_ZSTDDECOMPRESSOR_UNUSED_DATA_GETSETDEF |
| 677 |
{NULL} |
| 678 |
}; |
| 679 |
|
| 680 |
static int |
| 681 |
ZstdDecompressor_traverse(PyObject *ob, visitproc visit, void *arg) |
| 682 |
{ |
| 683 |
ZstdDecompressor *self = ZstdDecompressor_CAST(ob); |
| 684 |
Py_VISIT(self->dict); |
| 685 |
return 0; |
| 686 |
} |
| 687 |
|
| 688 |
static int |
| 689 |
ZstdDecompressor_clear(PyObject *ob) |
| 690 |
{ |
| 691 |
ZstdDecompressor *self = ZstdDecompressor_CAST(ob); |
| 692 |
Py_CLEAR(self->dict); |
| 693 |
Py_CLEAR(self->unused_data); |
| 694 |
return 0; |
| 695 |
} |
| 696 |
|
| 697 |
static PyType_Slot ZstdDecompressor_slots[] = { |
| 698 |
{Py_tp_new, _zstd_ZstdDecompressor_new}, |
| 699 |
{Py_tp_dealloc, ZstdDecompressor_dealloc}, |
| 700 |
{Py_tp_methods, ZstdDecompressor_methods}, |
| 701 |
{Py_tp_members, ZstdDecompressor_members}, |
| 702 |
{Py_tp_getset, ZstdDecompressor_getset}, |
| 703 |
{Py_tp_doc, (void *)_zstd_ZstdDecompressor_new__doc__}, |
| 704 |
{Py_tp_traverse, ZstdDecompressor_traverse}, |
| 705 |
{Py_tp_clear, ZstdDecompressor_clear}, |
| 706 |
{0, 0} |
| 707 |
}; |
| 708 |
|
| 709 |
PyType_Spec zstd_decompressor_type_spec = { |
| 710 |
.name = "compression.zstd.ZstdDecompressor", |
| 711 |
.basicsize = sizeof(ZstdDecompressor), |
| 712 |
.flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE |
| 713 |
| Py_TPFLAGS_HAVE_GC, |
| 714 |
.slots = ZstdDecompressor_slots, |
| 715 |
}; |