Debian Codegraph @

src/c/compression_zstd/decompressor.c

raw
1 /* Low level interface to the Zstandard algorithm & the zstd library. */
2
3 /* ZstdDecompressor class definition */
4
5 /*[clinic input]
6 module _zstd
7 class _zstd.ZstdDecompressor "ZstdDecompressor *" "&zstd_decompressor_type_spec"
8 [clinic start generated code]*/
9 /*[clinic end generated code: output=da39a3ee5e6b4b0d input=e2969ddf48a203e0]*/
10
11 #ifndef Py_BUILD_CORE_BUILTIN
12 # define Py_BUILD_CORE_MODULE 1
13 #endif
14
15 #include "backports_zstd_compat.h"
16 #include "backports_zstd_edits.h"
17
18 #include "Python.h"
19
20 #include "_zstdmodule.h"
21 #include "buffer.h"
22
23 #include <stdbool.h> // bool
24 #include <stddef.h> // offsetof()
25 #include <zstd.h> // ZSTD_*()
26
27 typedef struct {
28 PyObject_HEAD
29
30 /* Decompression context */
31 ZSTD_DCtx *dctx;
32
33 /* ZstdDict object in use */
34 PyObject *dict;
35
36 /* Unconsumed input data */
37 char *input_buffer;
38 size_t input_buffer_size;
39 size_t in_begin, in_end;
40
41 /* Unused data */
42 PyObject *unused_data;
43
44 /* 0 if decompressor has (or may has) unconsumed input data, 0 or 1. */
45 bool needs_input;
46
47 /* For ZstdDecompressor, 0 or 1.
48 1 means the end of the first frame has been reached. */
49 bool eof;
50
51 /* Lock to protect the decompression context */
52 BACKPORTSZSTD_LOCK lock;
53 } ZstdDecompressor;
54
55 #define ZstdDecompressor_CAST(op) ((ZstdDecompressor *)op)
56
57 #include "clinic/decompressor.c.h"
58
59 static inline ZSTD_DDict *
60 _get_DDict(ZstdDict *self)
61 {
62 assert(BACKPORTSZSTD_LOCK_isLocked(&self->lock));
63 ZSTD_DDict *ret;
64
65 if (self->d_dict == NULL) {
66 /* Create ZSTD_DDict instance from dictionary content */
67 Py_BEGIN_ALLOW_THREADS
68 ret = ZSTD_createDDict(self->dict_buffer, self->dict_len);
69 Py_END_ALLOW_THREADS
70 self->d_dict = ret;
71
72 if (self->d_dict == NULL) {
73 _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self));
74 if (mod_state != NULL) {
75 PyErr_SetString(mod_state->ZstdError,
76 "Failed to create a ZSTD_DDict instance from "
77 "Zstandard dictionary content.");
78 }
79 }
80 }
81
82 return self->d_dict;
83 }
84
85 static int
86 _zstd_set_d_parameters(ZstdDecompressor *self, PyObject *options)
87 {
88 _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self));
89 if (mod_state == NULL) {
90 return -1;
91 }
92
93 if (!PyDict_Check(options)) {
94 BACKPORTSZSTD_PyErr_Format_AppendPT(PyExc_TypeError,
95 "ZstdDecompressor() argument 'options' must be dict, not ",
96 options);
97 return -1;
98 }
99
100 Py_ssize_t pos = 0;
101 PyObject *key, *value;
102 while (PyDict_Next(options, &pos, &key, &value)) {
103 /* Check key type */
104 if (Py_TYPE(key) == mod_state->CParameter_type) {
105 PyErr_SetString(PyExc_TypeError,
106 "decompression options dictionary key must not be a "
107 "CompressionParameter attribute");
108 return -1;
109 }
110
111 Py_INCREF(key);
112 Py_INCREF(value);
113 int key_v = PyLong_AsInt(key);
114 Py_DECREF(key);
115 if (key_v == -1 && PyErr_Occurred()) {
116 Py_DECREF(value);
117 return -1;
118 }
119
120 int value_v = PyLong_AsInt(value);
121 Py_DECREF(value);
122 if (value_v == -1 && PyErr_Occurred()) {
123 return -1;
124 }
125
126 /* Set parameter to compression context */
127 size_t zstd_ret = ZSTD_DCtx_setParameter(self->dctx, key_v, value_v);
128
129 /* Check error */
130 if (ZSTD_isError(zstd_ret)) {
131 set_parameter_error(0, key_v, value_v);
132 return -1;
133 }
134 }
135 return 0;
136 }
137
138 static int
139 _zstd_load_impl(ZstdDecompressor *self, ZstdDict *zd,
140 _zstd_state *mod_state, int type)
141 {
142 size_t zstd_ret;
143 if (type == DICT_TYPE_DIGESTED) {
144 /* Get ZSTD_DDict */
145 ZSTD_DDict *d_dict = _get_DDict(zd);
146 if (d_dict == NULL) {
147 return -1;
148 }
149 /* Reference a prepared dictionary */
150 zstd_ret = ZSTD_DCtx_refDDict(self->dctx, d_dict);
151 }
152 else if (type == DICT_TYPE_UNDIGESTED) {
153 /* Load a dictionary */
154 zstd_ret = ZSTD_DCtx_loadDictionary(self->dctx, zd->dict_buffer,
155 zd->dict_len);
156 }
157 else if (type == DICT_TYPE_PREFIX) {
158 /* Load a prefix */
159 zstd_ret = ZSTD_DCtx_refPrefix(self->dctx, zd->dict_buffer,
160 zd->dict_len);
161 }
162 else {
163 Py_UNREACHABLE();
164 }
165
166 /* Check error */
167 if (ZSTD_isError(zstd_ret)) {
168 set_zstd_error(mod_state, ERR_LOAD_D_DICT, zstd_ret);
169 return -1;
170 }
171 return 0;
172 }
173
174 /* Load dictionary or prefix to decompression context */
175 static int
176 _zstd_load_d_dict(ZstdDecompressor *self, PyObject *dict)
177 {
178 _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self));
179 /* When decompressing, use digested dictionary by default. */
180 int type = DICT_TYPE_DIGESTED;
181 ZstdDict *zd = _Py_parse_zstd_dict(mod_state, dict, &type);
182 if (zd == NULL) {
183 return -1;
184 }
185 int ret;
186 BACKPORTSZSTD_LOCK_lock(&zd->lock);
187 ret = _zstd_load_impl(self, zd, mod_state, type);
188 BACKPORTSZSTD_LOCK_unlock(&zd->lock);
189 return ret;
190 }
191
192 /*
193 Decompress implementation in pseudo code:
194
195 initialize_output_buffer
196 while True:
197 decompress_data
198 set_object_flag # .eof
199
200 if output_buffer_exhausted:
201 if output_buffer_reached_max_length:
202 finish
203 grow_output_buffer
204 elif input_buffer_exhausted:
205 finish
206
207 ZSTD_decompressStream()'s size_t return value:
208 - 0 when a frame is completely decoded and fully flushed,
209 zstd's internal buffer has no data.
210 - An error code, which can be tested using ZSTD_isError().
211 - Or any other value > 0, which means there is still some decoding or
212 flushing to do to complete current frame.
213
214 Note, decompressing "an empty input" in any case will make it > 0.
215 */
216 static PyObject *
217 decompress_lock_held(ZstdDecompressor *self, ZSTD_inBuffer *in,
218 Py_ssize_t max_length)
219 {
220 size_t zstd_ret;
221 ZSTD_outBuffer out;
222 _BlocksOutputBuffer buffer = {.list = NULL};
223 PyObject *ret;
224
225 /* Initialize the output buffer */
226 if (_OutputBuffer_InitAndGrow(&buffer, &out, max_length) < 0) {
227 goto error;
228 }
229 assert(out.pos == 0);
230
231 while (1) {
232 /* Decompress */
233 Py_BEGIN_ALLOW_THREADS
234 zstd_ret = ZSTD_decompressStream(self->dctx, &out, in);
235 Py_END_ALLOW_THREADS
236
237 /* Check error */
238 if (ZSTD_isError(zstd_ret)) {
239 _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self));
240 set_zstd_error(mod_state, ERR_DECOMPRESS, zstd_ret);
241 goto error;
242 }
243
244 /* Set .eof flag */
245 if (zstd_ret == 0) {
246 /* Stop when a frame is decompressed */
247 self->eof = 1;
248 break;
249 }
250
251 /* Need to check out before in. Maybe zstd's internal buffer still has
252 a few bytes that can be output, grow the buffer and continue. */
253 if (out.pos == out.size) {
254 /* Output buffer exhausted */
255
256 /* Output buffer reached max_length */
257 if (_OutputBuffer_ReachedMaxLength(&buffer, &out)) {
258 break;
259 }
260
261 /* Grow output buffer */
262 if (_OutputBuffer_Grow(&buffer, &out) < 0) {
263 goto error;
264 }
265 assert(out.pos == 0);
266
267 }
268 else if (in->pos == in->size) {
269 /* Finished */
270 break;
271 }
272 }
273
274 /* Return a bytes object */
275 ret = _OutputBuffer_Finish(&buffer, &out);
276 if (ret != NULL) {
277 return ret;
278 }
279
280 error:
281 _OutputBuffer_OnError(&buffer);
282 return NULL;
283 }
284
285 static void
286 decompressor_reset_session_lock_held(ZstdDecompressor *self)
287 {
288 assert(BACKPORTSZSTD_LOCK_isLocked(&self->lock));
289
290 /* Reset variables */
291 self->in_begin = 0;
292 self->in_end = 0;
293
294 Py_CLEAR(self->unused_data);
295
296 /* Reset variables in one operation */
297 self->needs_input = 1;
298 self->eof = 0;
299
300 /* Resetting session is guaranteed to never fail */
301 ZSTD_DCtx_reset(self->dctx, ZSTD_reset_session_only);
302 }
303
304 static PyObject *
305 stream_decompress_lock_held(ZstdDecompressor *self, Py_buffer *data,
306 Py_ssize_t max_length)
307 {
308 assert(BACKPORTSZSTD_LOCK_isLocked(&self->lock));
309 ZSTD_inBuffer in;
310 PyObject *ret = NULL;
311 int use_input_buffer;
312
313 /* Check .eof flag */
314 if (self->eof) {
315 PyErr_SetString(PyExc_EOFError,
316 "Already at the end of a Zstandard frame.");
317 assert(ret == NULL);
318 return NULL;
319 }
320
321 /* Prepare input buffer w/wo unconsumed data */
322 if (self->in_begin == self->in_end) {
323 /* No unconsumed data */
324 use_input_buffer = 0;
325
326 in.src = data->buf;
327 in.size = data->len;
328 in.pos = 0;
329 }
330 else if (data->len == 0) {
331 /* Has unconsumed data, fast path for b'' */
332 assert(self->in_begin < self->in_end);
333
334 use_input_buffer = 1;
335
336 in.src = self->input_buffer + self->in_begin;
337 in.size = self->in_end - self->in_begin;
338 in.pos = 0;
339 }
340 else {
341 /* Has unconsumed data */
342 use_input_buffer = 1;
343
344 /* Unconsumed data size in input_buffer */
345 size_t used_now = self->in_end - self->in_begin;
346 assert(self->in_end > self->in_begin);
347
348 /* Number of bytes we can append to input buffer */
349 size_t avail_now = self->input_buffer_size - self->in_end;
350 assert(self->input_buffer_size >= self->in_end);
351
352 /* Number of bytes we can append if we move existing contents to
353 beginning of buffer */
354 size_t avail_total = self->input_buffer_size - used_now;
355 assert(self->input_buffer_size >= used_now);
356
357 if (avail_total < (size_t) data->len) {
358 char *tmp;
359 size_t new_size = used_now + data->len;
360
361 /* Allocate with new size */
362 tmp = PyMem_Malloc(new_size);
363 if (tmp == NULL) {
364 PyErr_NoMemory();
365 goto error;
366 }
367
368 /* Copy unconsumed data to the beginning of new buffer */
369 memcpy(tmp,
370 self->input_buffer + self->in_begin,
371 used_now);
372
373 /* Switch to new buffer */
374 PyMem_Free(self->input_buffer);
375 self->input_buffer = tmp;
376 self->input_buffer_size = new_size;
377
378 /* Set begin & end position */
379 self->in_begin = 0;
380 self->in_end = used_now;
381 }
382 else if (avail_now < (size_t) data->len) {
383 /* Move unconsumed data to the beginning.
384 Overlap is possible, so use memmove(). */
385 memmove(self->input_buffer,
386 self->input_buffer + self->in_begin,
387 used_now);
388
389 /* Set begin & end position */
390 self->in_begin = 0;
391 self->in_end = used_now;
392 }
393
394 /* Copy data to input buffer */
395 memcpy(self->input_buffer + self->in_end, data->buf, data->len);
396 self->in_end += data->len;
397
398 in.src = self->input_buffer + self->in_begin;
399 in.size = used_now + data->len;
400 in.pos = 0;
401 }
402 assert(in.pos == 0);
403
404 /* Decompress */
405 ret = decompress_lock_held(self, &in, max_length);
406 if (ret == NULL) {
407 goto error;
408 }
409
410 /* Unconsumed input data */
411 if (in.pos == in.size) {
412 if (Py_SIZE(ret) == max_length || self->eof) {
413 self->needs_input = 0;
414 }
415 else {
416 self->needs_input = 1;
417 }
418
419 if (use_input_buffer) {
420 /* Clear input_buffer */
421 self->in_begin = 0;
422 self->in_end = 0;
423 }
424 }
425 else {
426 size_t data_size = in.size - in.pos;
427
428 self->needs_input = 0;
429
430 if (!use_input_buffer) {
431 /* Discard buffer if it's too small
432 (resizing it may needlessly copy the current contents) */
433 if (self->input_buffer != NULL
434 && self->input_buffer_size < data_size)
435 {
436 PyMem_Free(self->input_buffer);
437 self->input_buffer = NULL;
438 self->input_buffer_size = 0;
439 }
440
441 /* Allocate if necessary */
442 if (self->input_buffer == NULL) {
443 self->input_buffer = PyMem_Malloc(data_size);
444 if (self->input_buffer == NULL) {
445 PyErr_NoMemory();
446 goto error;
447 }
448 self->input_buffer_size = data_size;
449 }
450
451 /* Copy unconsumed data */
452 memcpy(self->input_buffer, (char*)in.src + in.pos, data_size);
453 self->in_begin = 0;
454 self->in_end = data_size;
455 }
456 else {
457 /* Use input buffer */
458 self->in_begin += in.pos;
459 }
460 }
461
462 return ret;
463
464 error:
465 /* Reset decompressor's states/session */
466 decompressor_reset_session_lock_held(self);
467
468 Py_CLEAR(ret);
469 return NULL;
470 }
471
472
473 /*[clinic input]
474 @classmethod
475 _zstd.ZstdDecompressor.__new__ as _zstd_ZstdDecompressor_new
476 zstd_dict: object = None
477 A ZstdDict object, a pre-trained Zstandard dictionary.
478 options: object = None
479 A dict object that contains advanced decompression parameters.
480
481 Create a decompressor object for decompressing data incrementally.
482
483 Thread-safe at method level. For one-shot decompression, use the decompress()
484 function instead.
485 [clinic start generated code]*/
486
487 static PyObject *
488 _zstd_ZstdDecompressor_new_impl(PyTypeObject *type, PyObject *zstd_dict,
489 PyObject *options)
490 /*[clinic end generated code: output=590ca65c1102ff4a input=213daa57e3ea4062]*/
491 {
492 ZstdDecompressor* self = PyObject_GC_New(ZstdDecompressor, type);
493 if (self == NULL) {
494 goto error;
495 }
496
497 self->input_buffer = NULL;
498 self->input_buffer_size = 0;
499 self->in_begin = -1;
500 self->in_end = -1;
501 self->unused_data = NULL;
502 self->eof = 0;
503 self->dict = NULL;
504 self->lock = BACKPORTSZSTD_LOCK_allocate();
505 if(BACKPORTSZSTD_LOCK_isError(self->lock)) {
506 _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self));
507 if (mod_state != NULL) {
508 PyErr_SetString(mod_state->ZstdError,
509 "Unable to allocate lock");
510 }
511 goto error;
512 }
513
514 /* needs_input flag */
515 self->needs_input = 1;
516
517 /* Decompression context */
518 self->dctx = ZSTD_createDCtx();
519 if (self->dctx == NULL) {
520 _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self));
521 if (mod_state != NULL) {
522 PyErr_SetString(mod_state->ZstdError,
523 "Unable to create ZSTD_DCtx instance.");
524 }
525 goto error;
526 }
527
528 /* Load Zstandard dictionary to decompression context */
529 if (zstd_dict != Py_None) {
530 if (_zstd_load_d_dict(self, zstd_dict) < 0) {
531 goto error;
532 }
533 Py_INCREF(zstd_dict);
534 self->dict = zstd_dict;
535 }
536
537 /* Set options dictionary */
538 if (options != Py_None) {
539 if (_zstd_set_d_parameters(self, options) < 0) {
540 goto error;
541 }
542 }
543
544 // We can only start GC tracking once self->dict is set.
545 PyObject_GC_Track(self);
546
547 return (PyObject*)self;
548
549 error:
550 Py_XDECREF(self);
551 return NULL;
552 }
553
554 static void
555 ZstdDecompressor_dealloc(PyObject *ob)
556 {
557 ZstdDecompressor *self = ZstdDecompressor_CAST(ob);
558
559 PyObject_GC_UnTrack(self);
560
561 /* Free decompression context */
562 if (self->dctx) {
563 ZSTD_freeDCtx(self->dctx);
564 }
565
566 assert(!BACKPORTSZSTD_LOCK_isLocked(&self->lock));
567 BACKPORTSZSTD_LOCK_free(self->lock);
568
569 /* Py_CLEAR the dict after free decompression context */
570 Py_CLEAR(self->dict);
571
572 /* Free unconsumed input data buffer */
573 PyMem_Free(self->input_buffer);
574
575 /* Free unused data */
576 Py_CLEAR(self->unused_data);
577
578 PyTypeObject *tp = Py_TYPE(self);
579 PyObject_GC_Del(ob);
580 Py_DECREF(tp);
581 }
582
583 /*[clinic input]
584 @getter
585 _zstd.ZstdDecompressor.unused_data
586
587 A bytes object of un-consumed input data.
588
589 When ZstdDecompressor object stops after a frame is
590 decompressed, unused input data after the frame. Otherwise this will be b''.
591 [clinic start generated code]*/
592
593 static PyObject *
594 _zstd_ZstdDecompressor_unused_data_get_impl(ZstdDecompressor *self)
595 /*[clinic end generated code: output=f3a20940f11b6b09 input=54d41ecd681a3444]*/
596 {
597 PyObject *ret;
598
599 BACKPORTSZSTD_LOCK_lock(&self->lock);
600
601 if (!self->eof) {
602 BACKPORTSZSTD_LOCK_unlock(&self->lock);
603 return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES);
604 }
605 else {
606 if (self->unused_data == NULL) {
607 self->unused_data = PyBytes_FromStringAndSize(
608 self->input_buffer + self->in_begin,
609 self->in_end - self->in_begin);
610 ret = self->unused_data;
611 Py_XINCREF(ret);
612 }
613 else {
614 ret = self->unused_data;
615 Py_INCREF(ret);
616 }
617 }
618
619 BACKPORTSZSTD_LOCK_unlock(&self->lock);
620 return ret;
621 }
622
623 /*[clinic input]
624 _zstd.ZstdDecompressor.decompress
625
626 data: Py_buffer
627 A bytes-like object, Zstandard data to be decompressed.
628 max_length: Py_ssize_t = -1
629 Maximum size of returned data. When it is negative, the size of
630 output buffer is unlimited. When it is nonnegative, returns at
631 most max_length bytes of decompressed data.
632
633 Decompress *data*, returning uncompressed bytes if possible, or b'' otherwise.
634
635 If *max_length* is nonnegative, returns at most *max_length* bytes of
636 decompressed data. If this limit is reached and further output can be
637 produced, *self.needs_input* will be set to ``False``. In this case, the next
638 call to *decompress()* may provide *data* as b'' to obtain more of the output.
639
640 If all of the input data was decompressed and returned (either because this
641 was less than *max_length* bytes, or because *max_length* was negative),
642 *self.needs_input* will be set to True.
643
644 Attempting to decompress data after the end of a frame is reached raises an
645 EOFError. Any data found after the end of the frame is ignored and saved in
646 the self.unused_data attribute.
647 [clinic start generated code]*/
648
649 static PyObject *
650 _zstd_ZstdDecompressor_decompress_impl(ZstdDecompressor *self,
651 Py_buffer *data,
652 Py_ssize_t max_length)
653 /*[clinic end generated code: output=a4302b3c940dbec6 input=6463dfdf98091caa]*/
654 {
655 PyObject *ret;
656 /* Thread-safe code */
657 BACKPORTSZSTD_LOCK_lock(&self->lock);
658 ret = stream_decompress_lock_held(self, data, max_length);
659 BACKPORTSZSTD_LOCK_unlock(&self->lock);
660 return ret;
661 }
662
663 static PyMethodDef ZstdDecompressor_methods[] = {
664 _ZSTD_ZSTDDECOMPRESSOR_DECOMPRESS_METHODDEF
665 {NULL, NULL}
666 };
667
668 PyDoc_STRVAR(ZstdDecompressor_eof_doc,
669 "True means the end of the first frame has been reached. If decompress data\n"
670 "after that, an EOFError exception will be raised.");
671
672 PyDoc_STRVAR(ZstdDecompressor_needs_input_doc,
673 "If the max_length output limit in .decompress() method has been reached,\n"
674 "and the decompressor has (or may has) unconsumed input data, it will be set\n"
675 "to False. In this case, passing b'' to the .decompress() method may output\n"
676 "further data.");
677
678 static PyMemberDef ZstdDecompressor_members[] = {
679 {"eof", Py_T_BOOL, offsetof(ZstdDecompressor, eof),
680 Py_READONLY, ZstdDecompressor_eof_doc},
681 {"needs_input", Py_T_BOOL, offsetof(ZstdDecompressor, needs_input),
682 Py_READONLY, ZstdDecompressor_needs_input_doc},
683 {NULL}
684 };
685
686 static PyGetSetDef ZstdDecompressor_getset[] = {
687 _ZSTD_ZSTDDECOMPRESSOR_UNUSED_DATA_GETSETDEF
688 {NULL}
689 };
690
691 static int
692 ZstdDecompressor_traverse(PyObject *ob, visitproc visit, void *arg)
693 {
694 ZstdDecompressor *self = ZstdDecompressor_CAST(ob);
695 Py_VISIT(self->dict);
696 return 0;
697 }
698
699 static int
700 ZstdDecompressor_clear(PyObject *ob)
701 {
702 ZstdDecompressor *self = ZstdDecompressor_CAST(ob);
703 Py_CLEAR(self->dict);
704 Py_CLEAR(self->unused_data);
705 return 0;
706 }
707
708 static PyType_Slot ZstdDecompressor_slots[] = {
709 {Py_tp_new, _zstd_ZstdDecompressor_new},
710 {Py_tp_dealloc, ZstdDecompressor_dealloc},
711 {Py_tp_methods, ZstdDecompressor_methods},
712 {Py_tp_members, ZstdDecompressor_members},
713 {Py_tp_getset, ZstdDecompressor_getset},
714 {Py_tp_doc, (void *)_zstd_ZstdDecompressor_new__doc__},
715 {Py_tp_traverse, ZstdDecompressor_traverse},
716 {Py_tp_clear, ZstdDecompressor_clear},
717 {0, 0}
718 };
719
720 PyType_Spec zstd_decompressor_type_spec = {
721 .name = "backports.zstd.ZstdDecompressor",
722 .basicsize = sizeof(ZstdDecompressor),
723 .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE
724 | Py_TPFLAGS_HAVE_GC,
725 .slots = ZstdDecompressor_slots,
726 };