Debian Codegraph @

Modules/_zstd/decompressor.c

raw
1 /* Low level interface to the Zstandard algorithm & the zstd library. */
2
3 /* ZstdDecompressor class definition */
4
5 /*[clinic input]
6 module _zstd
7 class _zstd.ZstdDecompressor "ZstdDecompressor *" "&zstd_decompressor_type_spec"
8 [clinic start generated code]*/
9 /*[clinic end generated code: output=da39a3ee5e6b4b0d input=e2969ddf48a203e0]*/
10
11 #ifndef Py_BUILD_CORE_BUILTIN
12 # define Py_BUILD_CORE_MODULE 1
13 #endif
14
15 #include "Python.h"
16
17 #include "_zstdmodule.h"
18 #include "buffer.h"
19 #include "internal/pycore_lock.h" // PyMutex_IsLocked
20
21 #include <stdbool.h> // bool
22 #include <stddef.h> // offsetof()
23 #include <zstd.h> // ZSTD_*()
24
25 typedef struct {
26 PyObject_HEAD
27
28 /* Decompression context */
29 ZSTD_DCtx *dctx;
30
31 /* ZstdDict object in use */
32 PyObject *dict;
33
34 /* Unconsumed input data */
35 char *input_buffer;
36 size_t input_buffer_size;
37 size_t in_begin, in_end;
38
39 /* Unused data */
40 PyObject *unused_data;
41
42 /* 0 if decompressor has (or may has) unconsumed input data, 0 or 1. */
43 bool needs_input;
44
45 /* For ZstdDecompressor, 0 or 1.
46 1 means the end of the first frame has been reached. */
47 bool eof;
48
49 /* Lock to protect the decompression context */
50 PyMutex lock;
51 } ZstdDecompressor;
52
53 #define ZstdDecompressor_CAST(op) ((ZstdDecompressor *)op)
54
55 #include "clinic/decompressor.c.h"
56
57 static inline ZSTD_DDict *
58 _get_DDict(ZstdDict *self)
59 {
60 assert(PyMutex_IsLocked(&self->lock));
61 ZSTD_DDict *ret;
62
63 if (self->d_dict == NULL) {
64 /* Create ZSTD_DDict instance from dictionary content */
65 Py_BEGIN_ALLOW_THREADS
66 ret = ZSTD_createDDict(self->dict_buffer, self->dict_len);
67 Py_END_ALLOW_THREADS
68 self->d_dict = ret;
69
70 if (self->d_dict == NULL) {
71 _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self));
72 if (mod_state != NULL) {
73 PyErr_SetString(mod_state->ZstdError,
74 "Failed to create a ZSTD_DDict instance from "
75 "Zstandard dictionary content.");
76 }
77 }
78 }
79
80 return self->d_dict;
81 }
82
83 static int
84 _zstd_set_d_parameters(ZstdDecompressor *self, PyObject *options)
85 {
86 _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self));
87 if (mod_state == NULL) {
88 return -1;
89 }
90
91 if (!PyDict_Check(options)) {
92 PyErr_Format(PyExc_TypeError,
93 "ZstdDecompressor() argument 'options' must be dict, not %T",
94 options);
95 return -1;
96 }
97
98 Py_ssize_t pos = 0;
99 PyObject *key, *value;
100 while (PyDict_Next(options, &pos, &key, &value)) {
101 /* Check key type */
102 if (Py_TYPE(key) == mod_state->CParameter_type) {
103 PyErr_SetString(PyExc_TypeError,
104 "decompression options dictionary key must not be a "
105 "CompressionParameter attribute");
106 return -1;
107 }
108
109 Py_INCREF(key);
110 Py_INCREF(value);
111 int key_v = PyLong_AsInt(key);
112 Py_DECREF(key);
113 if (key_v == -1 && PyErr_Occurred()) {
114 Py_DECREF(value);
115 return -1;
116 }
117
118 int value_v = PyLong_AsInt(value);
119 Py_DECREF(value);
120 if (value_v == -1 && PyErr_Occurred()) {
121 return -1;
122 }
123
124 /* Set parameter to compression context */
125 size_t zstd_ret = ZSTD_DCtx_setParameter(self->dctx, key_v, value_v);
126
127 /* Check error */
128 if (ZSTD_isError(zstd_ret)) {
129 set_parameter_error(0, key_v, value_v);
130 return -1;
131 }
132 }
133 return 0;
134 }
135
136 static int
137 _zstd_load_impl(ZstdDecompressor *self, ZstdDict *zd,
138 _zstd_state *mod_state, int type)
139 {
140 size_t zstd_ret;
141 if (type == DICT_TYPE_DIGESTED) {
142 /* Get ZSTD_DDict */
143 ZSTD_DDict *d_dict = _get_DDict(zd);
144 if (d_dict == NULL) {
145 return -1;
146 }
147 /* Reference a prepared dictionary */
148 zstd_ret = ZSTD_DCtx_refDDict(self->dctx, d_dict);
149 }
150 else if (type == DICT_TYPE_UNDIGESTED) {
151 /* Load a dictionary */
152 zstd_ret = ZSTD_DCtx_loadDictionary(self->dctx, zd->dict_buffer,
153 zd->dict_len);
154 }
155 else if (type == DICT_TYPE_PREFIX) {
156 /* Load a prefix */
157 zstd_ret = ZSTD_DCtx_refPrefix(self->dctx, zd->dict_buffer,
158 zd->dict_len);
159 }
160 else {
161 Py_UNREACHABLE();
162 }
163
164 /* Check error */
165 if (ZSTD_isError(zstd_ret)) {
166 set_zstd_error(mod_state, ERR_LOAD_D_DICT, zstd_ret);
167 return -1;
168 }
169 return 0;
170 }
171
172 /* Load dictionary or prefix to decompression context */
173 static int
174 _zstd_load_d_dict(ZstdDecompressor *self, PyObject *dict)
175 {
176 _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self));
177 /* When decompressing, use digested dictionary by default. */
178 int type = DICT_TYPE_DIGESTED;
179 ZstdDict *zd = _Py_parse_zstd_dict(mod_state, dict, &type);
180 if (zd == NULL) {
181 return -1;
182 }
183 int ret;
184 PyMutex_Lock(&zd->lock);
185 ret = _zstd_load_impl(self, zd, mod_state, type);
186 PyMutex_Unlock(&zd->lock);
187 return ret;
188 }
189
190 /*
191 Decompress implementation in pseudo code:
192
193 initialize_output_buffer
194 while True:
195 decompress_data
196 set_object_flag # .eof
197
198 if output_buffer_exhausted:
199 if output_buffer_reached_max_length:
200 finish
201 grow_output_buffer
202 elif input_buffer_exhausted:
203 finish
204
205 ZSTD_decompressStream()'s size_t return value:
206 - 0 when a frame is completely decoded and fully flushed,
207 zstd's internal buffer has no data.
208 - An error code, which can be tested using ZSTD_isError().
209 - Or any other value > 0, which means there is still some decoding or
210 flushing to do to complete current frame.
211
212 Note, decompressing "an empty input" in any case will make it > 0.
213 */
214 static PyObject *
215 decompress_lock_held(ZstdDecompressor *self, ZSTD_inBuffer *in,
216 Py_ssize_t max_length)
217 {
218 size_t zstd_ret;
219 ZSTD_outBuffer out;
220 _BlocksOutputBuffer buffer = {.list = NULL};
221 PyObject *ret;
222
223 /* Initialize the output buffer */
224 if (_OutputBuffer_InitAndGrow(&buffer, &out, max_length) < 0) {
225 goto error;
226 }
227 assert(out.pos == 0);
228
229 while (1) {
230 /* Decompress */
231 Py_BEGIN_ALLOW_THREADS
232 zstd_ret = ZSTD_decompressStream(self->dctx, &out, in);
233 Py_END_ALLOW_THREADS
234
235 /* Check error */
236 if (ZSTD_isError(zstd_ret)) {
237 _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self));
238 set_zstd_error(mod_state, ERR_DECOMPRESS, zstd_ret);
239 goto error;
240 }
241
242 /* Set .eof flag */
243 if (zstd_ret == 0) {
244 /* Stop when a frame is decompressed */
245 self->eof = 1;
246 break;
247 }
248
249 /* Need to check out before in. Maybe zstd's internal buffer still has
250 a few bytes that can be output, grow the buffer and continue. */
251 if (out.pos == out.size) {
252 /* Output buffer exhausted */
253
254 /* Output buffer reached max_length */
255 if (_OutputBuffer_ReachedMaxLength(&buffer, &out)) {
256 break;
257 }
258
259 /* Grow output buffer */
260 if (_OutputBuffer_Grow(&buffer, &out) < 0) {
261 goto error;
262 }
263 assert(out.pos == 0);
264
265 }
266 else if (in->pos == in->size) {
267 /* Finished */
268 break;
269 }
270 }
271
272 /* Return a bytes object */
273 ret = _OutputBuffer_Finish(&buffer, &out);
274 if (ret != NULL) {
275 return ret;
276 }
277
278 error:
279 _OutputBuffer_OnError(&buffer);
280 return NULL;
281 }
282
283 static void
284 decompressor_reset_session_lock_held(ZstdDecompressor *self)
285 {
286 assert(PyMutex_IsLocked(&self->lock));
287
288 /* Reset variables */
289 self->in_begin = 0;
290 self->in_end = 0;
291
292 Py_CLEAR(self->unused_data);
293
294 /* Reset variables in one operation */
295 self->needs_input = 1;
296 self->eof = 0;
297
298 /* Resetting session is guaranteed to never fail */
299 ZSTD_DCtx_reset(self->dctx, ZSTD_reset_session_only);
300 }
301
302 static PyObject *
303 stream_decompress_lock_held(ZstdDecompressor *self, Py_buffer *data,
304 Py_ssize_t max_length)
305 {
306 assert(PyMutex_IsLocked(&self->lock));
307 ZSTD_inBuffer in;
308 PyObject *ret = NULL;
309 int use_input_buffer;
310
311 /* Check .eof flag */
312 if (self->eof) {
313 PyErr_SetString(PyExc_EOFError,
314 "Already at the end of a Zstandard frame.");
315 assert(ret == NULL);
316 return NULL;
317 }
318
319 /* Prepare input buffer w/wo unconsumed data */
320 if (self->in_begin == self->in_end) {
321 /* No unconsumed data */
322 use_input_buffer = 0;
323
324 in.src = data->buf;
325 in.size = data->len;
326 in.pos = 0;
327 }
328 else if (data->len == 0) {
329 /* Has unconsumed data, fast path for b'' */
330 assert(self->in_begin < self->in_end);
331
332 use_input_buffer = 1;
333
334 in.src = self->input_buffer + self->in_begin;
335 in.size = self->in_end - self->in_begin;
336 in.pos = 0;
337 }
338 else {
339 /* Has unconsumed data */
340 use_input_buffer = 1;
341
342 /* Unconsumed data size in input_buffer */
343 size_t used_now = self->in_end - self->in_begin;
344 assert(self->in_end > self->in_begin);
345
346 /* Number of bytes we can append to input buffer */
347 size_t avail_now = self->input_buffer_size - self->in_end;
348 assert(self->input_buffer_size >= self->in_end);
349
350 /* Number of bytes we can append if we move existing contents to
351 beginning of buffer */
352 size_t avail_total = self->input_buffer_size - used_now;
353 assert(self->input_buffer_size >= used_now);
354
355 if (avail_total < (size_t) data->len) {
356 char *tmp;
357 size_t new_size = used_now + data->len;
358
359 /* Allocate with new size */
360 tmp = PyMem_Malloc(new_size);
361 if (tmp == NULL) {
362 PyErr_NoMemory();
363 goto error;
364 }
365
366 /* Copy unconsumed data to the beginning of new buffer */
367 memcpy(tmp,
368 self->input_buffer + self->in_begin,
369 used_now);
370
371 /* Switch to new buffer */
372 PyMem_Free(self->input_buffer);
373 self->input_buffer = tmp;
374 self->input_buffer_size = new_size;
375
376 /* Set begin & end position */
377 self->in_begin = 0;
378 self->in_end = used_now;
379 }
380 else if (avail_now < (size_t) data->len) {
381 /* Move unconsumed data to the beginning.
382 Overlap is possible, so use memmove(). */
383 memmove(self->input_buffer,
384 self->input_buffer + self->in_begin,
385 used_now);
386
387 /* Set begin & end position */
388 self->in_begin = 0;
389 self->in_end = used_now;
390 }
391
392 /* Copy data to input buffer */
393 memcpy(self->input_buffer + self->in_end, data->buf, data->len);
394 self->in_end += data->len;
395
396 in.src = self->input_buffer + self->in_begin;
397 in.size = used_now + data->len;
398 in.pos = 0;
399 }
400 assert(in.pos == 0);
401
402 /* Decompress */
403 ret = decompress_lock_held(self, &in, max_length);
404 if (ret == NULL) {
405 goto error;
406 }
407
408 /* Unconsumed input data */
409 if (in.pos == in.size) {
410 if (Py_SIZE(ret) == max_length || self->eof) {
411 self->needs_input = 0;
412 }
413 else {
414 self->needs_input = 1;
415 }
416
417 if (use_input_buffer) {
418 /* Clear input_buffer */
419 self->in_begin = 0;
420 self->in_end = 0;
421 }
422 }
423 else {
424 size_t data_size = in.size - in.pos;
425
426 self->needs_input = 0;
427
428 if (!use_input_buffer) {
429 /* Discard buffer if it's too small
430 (resizing it may needlessly copy the current contents) */
431 if (self->input_buffer != NULL
432 && self->input_buffer_size < data_size)
433 {
434 PyMem_Free(self->input_buffer);
435 self->input_buffer = NULL;
436 self->input_buffer_size = 0;
437 }
438
439 /* Allocate if necessary */
440 if (self->input_buffer == NULL) {
441 self->input_buffer = PyMem_Malloc(data_size);
442 if (self->input_buffer == NULL) {
443 PyErr_NoMemory();
444 goto error;
445 }
446 self->input_buffer_size = data_size;
447 }
448
449 /* Copy unconsumed data */
450 memcpy(self->input_buffer, (char*)in.src + in.pos, data_size);
451 self->in_begin = 0;
452 self->in_end = data_size;
453 }
454 else {
455 /* Use input buffer */
456 self->in_begin += in.pos;
457 }
458 }
459
460 return ret;
461
462 error:
463 /* Reset decompressor's states/session */
464 decompressor_reset_session_lock_held(self);
465
466 Py_CLEAR(ret);
467 return NULL;
468 }
469
470
471 /*[clinic input]
472 @classmethod
473 _zstd.ZstdDecompressor.__new__ as _zstd_ZstdDecompressor_new
474 zstd_dict: object = None
475 A ZstdDict object, a pre-trained Zstandard dictionary.
476 options: object = None
477 A dict object that contains advanced decompression parameters.
478
479 Create a decompressor object for decompressing data incrementally.
480
481 Thread-safe at method level. For one-shot decompression, use the decompress()
482 function instead.
483 [clinic start generated code]*/
484
485 static PyObject *
486 _zstd_ZstdDecompressor_new_impl(PyTypeObject *type, PyObject *zstd_dict,
487 PyObject *options)
488 /*[clinic end generated code: output=590ca65c1102ff4a input=213daa57e3ea4062]*/
489 {
490 ZstdDecompressor* self = PyObject_GC_New(ZstdDecompressor, type);
491 if (self == NULL) {
492 goto error;
493 }
494
495 self->input_buffer = NULL;
496 self->input_buffer_size = 0;
497 self->in_begin = -1;
498 self->in_end = -1;
499 self->unused_data = NULL;
500 self->eof = 0;
501 self->dict = NULL;
502 self->lock = (PyMutex){0};
503
504 /* needs_input flag */
505 self->needs_input = 1;
506
507 /* Decompression context */
508 self->dctx = ZSTD_createDCtx();
509 if (self->dctx == NULL) {
510 _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self));
511 if (mod_state != NULL) {
512 PyErr_SetString(mod_state->ZstdError,
513 "Unable to create ZSTD_DCtx instance.");
514 }
515 goto error;
516 }
517
518 /* Load Zstandard dictionary to decompression context */
519 if (zstd_dict != Py_None) {
520 if (_zstd_load_d_dict(self, zstd_dict) < 0) {
521 goto error;
522 }
523 Py_INCREF(zstd_dict);
524 self->dict = zstd_dict;
525 }
526
527 /* Set options dictionary */
528 if (options != Py_None) {
529 if (_zstd_set_d_parameters(self, options) < 0) {
530 goto error;
531 }
532 }
533
534 // We can only start GC tracking once self->dict is set.
535 PyObject_GC_Track(self);
536
537 return (PyObject*)self;
538
539 error:
540 Py_XDECREF(self);
541 return NULL;
542 }
543
544 static void
545 ZstdDecompressor_dealloc(PyObject *ob)
546 {
547 ZstdDecompressor *self = ZstdDecompressor_CAST(ob);
548
549 PyObject_GC_UnTrack(self);
550
551 /* Free decompression context */
552 if (self->dctx) {
553 ZSTD_freeDCtx(self->dctx);
554 }
555
556 assert(!PyMutex_IsLocked(&self->lock));
557
558 /* Py_CLEAR the dict after free decompression context */
559 Py_CLEAR(self->dict);
560
561 /* Free unconsumed input data buffer */
562 PyMem_Free(self->input_buffer);
563
564 /* Free unused data */
565 Py_CLEAR(self->unused_data);
566
567 PyTypeObject *tp = Py_TYPE(self);
568 PyObject_GC_Del(ob);
569 Py_DECREF(tp);
570 }
571
572 /*[clinic input]
573 @getter
574 _zstd.ZstdDecompressor.unused_data
575
576 A bytes object of un-consumed input data.
577
578 When ZstdDecompressor object stops after a frame is
579 decompressed, unused input data after the frame. Otherwise this will be b''.
580 [clinic start generated code]*/
581
582 static PyObject *
583 _zstd_ZstdDecompressor_unused_data_get_impl(ZstdDecompressor *self)
584 /*[clinic end generated code: output=f3a20940f11b6b09 input=54d41ecd681a3444]*/
585 {
586 PyObject *ret;
587
588 PyMutex_Lock(&self->lock);
589
590 if (!self->eof) {
591 PyMutex_Unlock(&self->lock);
592 return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES);
593 }
594 else {
595 if (self->unused_data == NULL) {
596 self->unused_data = PyBytes_FromStringAndSize(
597 self->input_buffer + self->in_begin,
598 self->in_end - self->in_begin);
599 ret = self->unused_data;
600 Py_XINCREF(ret);
601 }
602 else {
603 ret = self->unused_data;
604 Py_INCREF(ret);
605 }
606 }
607
608 PyMutex_Unlock(&self->lock);
609 return ret;
610 }
611
612 /*[clinic input]
613 _zstd.ZstdDecompressor.decompress
614
615 data: Py_buffer
616 A bytes-like object, Zstandard data to be decompressed.
617 max_length: Py_ssize_t = -1
618 Maximum size of returned data. When it is negative, the size of
619 output buffer is unlimited. When it is nonnegative, returns at
620 most max_length bytes of decompressed data.
621
622 Decompress *data*, returning uncompressed bytes if possible, or b'' otherwise.
623
624 If *max_length* is nonnegative, returns at most *max_length* bytes of
625 decompressed data. If this limit is reached and further output can be
626 produced, *self.needs_input* will be set to ``False``. In this case, the next
627 call to *decompress()* may provide *data* as b'' to obtain more of the output.
628
629 If all of the input data was decompressed and returned (either because this
630 was less than *max_length* bytes, or because *max_length* was negative),
631 *self.needs_input* will be set to True.
632
633 Attempting to decompress data after the end of a frame is reached raises an
634 EOFError. Any data found after the end of the frame is ignored and saved in
635 the self.unused_data attribute.
636 [clinic start generated code]*/
637
638 static PyObject *
639 _zstd_ZstdDecompressor_decompress_impl(ZstdDecompressor *self,
640 Py_buffer *data,
641 Py_ssize_t max_length)
642 /*[clinic end generated code: output=a4302b3c940dbec6 input=6463dfdf98091caa]*/
643 {
644 PyObject *ret;
645 /* Thread-safe code */
646 PyMutex_Lock(&self->lock);
647 ret = stream_decompress_lock_held(self, data, max_length);
648 PyMutex_Unlock(&self->lock);
649 return ret;
650 }
651
652 static PyMethodDef ZstdDecompressor_methods[] = {
653 _ZSTD_ZSTDDECOMPRESSOR_DECOMPRESS_METHODDEF
654 {NULL, NULL}
655 };
656
657 PyDoc_STRVAR(ZstdDecompressor_eof_doc,
658 "True means the end of the first frame has been reached. If decompress data\n"
659 "after that, an EOFError exception will be raised.");
660
661 PyDoc_STRVAR(ZstdDecompressor_needs_input_doc,
662 "If the max_length output limit in .decompress() method has been reached,\n"
663 "and the decompressor has (or may has) unconsumed input data, it will be set\n"
664 "to False. In this case, passing b'' to the .decompress() method may output\n"
665 "further data.");
666
667 static PyMemberDef ZstdDecompressor_members[] = {
668 {"eof", Py_T_BOOL, offsetof(ZstdDecompressor, eof),
669 Py_READONLY, ZstdDecompressor_eof_doc},
670 {"needs_input", Py_T_BOOL, offsetof(ZstdDecompressor, needs_input),
671 Py_READONLY, ZstdDecompressor_needs_input_doc},
672 {NULL}
673 };
674
675 static PyGetSetDef ZstdDecompressor_getset[] = {
676 _ZSTD_ZSTDDECOMPRESSOR_UNUSED_DATA_GETSETDEF
677 {NULL}
678 };
679
680 static int
681 ZstdDecompressor_traverse(PyObject *ob, visitproc visit, void *arg)
682 {
683 ZstdDecompressor *self = ZstdDecompressor_CAST(ob);
684 Py_VISIT(self->dict);
685 return 0;
686 }
687
688 static int
689 ZstdDecompressor_clear(PyObject *ob)
690 {
691 ZstdDecompressor *self = ZstdDecompressor_CAST(ob);
692 Py_CLEAR(self->dict);
693 Py_CLEAR(self->unused_data);
694 return 0;
695 }
696
697 static PyType_Slot ZstdDecompressor_slots[] = {
698 {Py_tp_new, _zstd_ZstdDecompressor_new},
699 {Py_tp_dealloc, ZstdDecompressor_dealloc},
700 {Py_tp_methods, ZstdDecompressor_methods},
701 {Py_tp_members, ZstdDecompressor_members},
702 {Py_tp_getset, ZstdDecompressor_getset},
703 {Py_tp_doc, (void *)_zstd_ZstdDecompressor_new__doc__},
704 {Py_tp_traverse, ZstdDecompressor_traverse},
705 {Py_tp_clear, ZstdDecompressor_clear},
706 {0, 0}
707 };
708
709 PyType_Spec zstd_decompressor_type_spec = {
710 .name = "compression.zstd.ZstdDecompressor",
711 .basicsize = sizeof(ZstdDecompressor),
712 .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE
713 | Py_TPFLAGS_HAVE_GC,
714 .slots = ZstdDecompressor_slots,
715 };