Debian Codegraph @

Modules/_zstd/decompressor.c

raw
1 /* Low level interface to the Zstandard algorithm & the zstd library. */
2
3 /* ZstdDecompressor class definition */
4
5 /*[clinic input]
6 module _zstd
7 class _zstd.ZstdDecompressor "ZstdDecompressor *" "&zstd_decompressor_type_spec"
8 [clinic start generated code]*/
9 /*[clinic end generated code: output=da39a3ee5e6b4b0d input=e2969ddf48a203e0]*/
10
11 #ifndef Py_BUILD_CORE_BUILTIN
12 # define Py_BUILD_CORE_MODULE 1
13 #endif
14
15 #include "Python.h"
16
17 #include "_zstdmodule.h"
18 #include "buffer.h"
19 #include "internal/pycore_lock.h" // PyMutex_IsLocked
20
21 #include <stdbool.h> // bool
22 #include <stddef.h> // offsetof()
23 #include <zstd.h> // ZSTD_*()
24
25 typedef struct {
26 PyObject_HEAD
27
28 /* Decompression context */
29 ZSTD_DCtx *dctx;
30
31 /* ZstdDict object in use */
32 PyObject *dict;
33
34 /* Unconsumed input data */
35 char *input_buffer;
36 size_t input_buffer_size;
37 size_t in_begin, in_end;
38
39 /* Unused data */
40 PyObject *unused_data;
41
42 /* 0 if decompressor has (or may has) unconsumed input data, 0 or 1. */
43 bool needs_input;
44
45 /* For ZstdDecompressor, 0 or 1.
46 1 means the end of the first frame has been reached. */
47 bool eof;
48
49 /* Lock to protect the decompression context */
50 PyMutex lock;
51 } ZstdDecompressor;
52
53 #define ZstdDecompressor_CAST(op) ((ZstdDecompressor *)op)
54
55 #include "clinic/decompressor.c.h"
56
57 static inline ZSTD_DDict *
58 _get_DDict(ZstdDict *self)
59 {
60 assert(PyMutex_IsLocked(&self->lock));
61 ZSTD_DDict *ret;
62
63 if (self->d_dict == NULL) {
64 /* Create ZSTD_DDict instance from dictionary content */
65 Py_BEGIN_ALLOW_THREADS
66 ret = ZSTD_createDDict(self->dict_buffer, self->dict_len);
67 Py_END_ALLOW_THREADS
68 self->d_dict = ret;
69
70 if (self->d_dict == NULL) {
71 _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self));
72 if (mod_state != NULL) {
73 PyErr_SetString(mod_state->ZstdError,
74 "Failed to create a ZSTD_DDict instance from "
75 "Zstandard dictionary content.");
76 }
77 }
78 }
79
80 return self->d_dict;
81 }
82
83 static int
84 _zstd_set_d_parameters(ZstdDecompressor *self, PyObject *options)
85 {
86 _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self));
87 if (mod_state == NULL) {
88 return -1;
89 }
90
91 if (!PyDict_Check(options)) {
92 PyErr_Format(PyExc_TypeError,
93 "ZstdDecompressor() argument 'options' must be dict, not %T",
94 options);
95 return -1;
96 }
97
98 Py_ssize_t pos = 0;
99 PyObject *key, *value;
100 while (PyDict_Next(options, &pos, &key, &value)) {
101 /* Check key type */
102 if (Py_TYPE(key) == mod_state->CParameter_type) {
103 PyErr_SetString(PyExc_TypeError,
104 "decompression options dictionary key must not be a "
105 "CompressionParameter attribute");
106 return -1;
107 }
108
109 Py_INCREF(key);
110 Py_INCREF(value);
111 int key_v = PyLong_AsInt(key);
112 Py_DECREF(key);
113 if (key_v == -1 && PyErr_Occurred()) {
114 Py_DECREF(value);
115 return -1;
116 }
117
118 int value_v = PyLong_AsInt(value);
119 Py_DECREF(value);
120 if (value_v == -1 && PyErr_Occurred()) {
121 return -1;
122 }
123
124 /* Set parameter to compression context */
125 size_t zstd_ret = ZSTD_DCtx_setParameter(self->dctx, key_v, value_v);
126
127 /* Check error */
128 if (ZSTD_isError(zstd_ret)) {
129 set_parameter_error(0, key_v, value_v);
130 return -1;
131 }
132 }
133 return 0;
134 }
135
136 static int
137 _zstd_load_impl(ZstdDecompressor *self, ZstdDict *zd,
138 _zstd_state *mod_state, int type)
139 {
140 size_t zstd_ret;
141 if (type == DICT_TYPE_DIGESTED) {
142 /* Get ZSTD_DDict */
143 ZSTD_DDict *d_dict = _get_DDict(zd);
144 if (d_dict == NULL) {
145 return -1;
146 }
147 /* Reference a prepared dictionary */
148 zstd_ret = ZSTD_DCtx_refDDict(self->dctx, d_dict);
149 }
150 else if (type == DICT_TYPE_UNDIGESTED) {
151 /* Load a dictionary */
152 zstd_ret = ZSTD_DCtx_loadDictionary(self->dctx, zd->dict_buffer,
153 zd->dict_len);
154 }
155 else if (type == DICT_TYPE_PREFIX) {
156 /* Load a prefix */
157 zstd_ret = ZSTD_DCtx_refPrefix(self->dctx, zd->dict_buffer,
158 zd->dict_len);
159 }
160 else {
161 Py_UNREACHABLE();
162 }
163
164 /* Check error */
165 if (ZSTD_isError(zstd_ret)) {
166 set_zstd_error(mod_state, ERR_LOAD_D_DICT, zstd_ret);
167 return -1;
168 }
169 return 0;
170 }
171
172 /* Load dictionary or prefix to decompression context */
173 static int
174 _zstd_load_d_dict(ZstdDecompressor *self, PyObject *dict)
175 {
176 _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self));
177 /* When decompressing, use digested dictionary by default. */
178 int type = DICT_TYPE_DIGESTED;
179 ZstdDict *zd = _Py_parse_zstd_dict(mod_state, dict, &type);
180 if (zd == NULL) {
181 return -1;
182 }
183 int ret;
184 PyMutex_Lock(&zd->lock);
185 ret = _zstd_load_impl(self, zd, mod_state, type);
186 PyMutex_Unlock(&zd->lock);
187 return ret;
188 }
189
190 /*
191 Decompress implementation in pseudo code:
192
193 initialize_output_buffer
194 while True:
195 decompress_data
196 set_object_flag # .eof
197
198 if output_buffer_exhausted:
199 if output_buffer_reached_max_length:
200 finish
201 grow_output_buffer
202 elif input_buffer_exhausted:
203 finish
204
205 ZSTD_decompressStream()'s size_t return value:
206 - 0 when a frame is completely decoded and fully flushed,
207 zstd's internal buffer has no data.
208 - An error code, which can be tested using ZSTD_isError().
209 - Or any other value > 0, which means there is still some decoding or
210 flushing to do to complete current frame.
211
212 Note, decompressing "an empty input" in any case will make it > 0.
213 */
214 static PyObject *
215 decompress_lock_held(ZstdDecompressor *self, ZSTD_inBuffer *in,
216 Py_ssize_t max_length)
217 {
218 size_t zstd_ret;
219 ZSTD_outBuffer out;
220 _BlocksOutputBuffer buffer = {.list = NULL};
221 PyObject *ret;
222
223 /* Initialize the output buffer */
224 if (_OutputBuffer_InitAndGrow(&buffer, &out, max_length) < 0) {
225 goto error;
226 }
227 assert(out.pos == 0);
228
229 while (1) {
230 /* Decompress */
231 Py_BEGIN_ALLOW_THREADS
232 zstd_ret = ZSTD_decompressStream(self->dctx, &out, in);
233 Py_END_ALLOW_THREADS
234
235 /* Check error */
236 if (ZSTD_isError(zstd_ret)) {
237 _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self));
238 set_zstd_error(mod_state, ERR_DECOMPRESS, zstd_ret);
239 goto error;
240 }
241
242 /* Set .eof flag */
243 if (zstd_ret == 0) {
244 /* Stop when a frame is decompressed */
245 self->eof = 1;
246 break;
247 }
248
249 /* Need to check out before in. Maybe zstd's internal buffer still has
250 a few bytes that can be output, grow the buffer and continue. */
251 if (out.pos == out.size) {
252 /* Output buffer exhausted */
253
254 /* Output buffer reached max_length */
255 if (_OutputBuffer_ReachedMaxLength(&buffer, &out)) {
256 break;
257 }
258
259 /* Grow output buffer */
260 if (_OutputBuffer_Grow(&buffer, &out) < 0) {
261 goto error;
262 }
263 assert(out.pos == 0);
264
265 }
266 else if (in->pos == in->size) {
267 /* Finished */
268 break;
269 }
270 }
271
272 /* Return a bytes object */
273 ret = _OutputBuffer_Finish(&buffer, &out);
274 if (ret != NULL) {
275 return ret;
276 }
277
278 error:
279 _OutputBuffer_OnError(&buffer);
280 return NULL;
281 }
282
283 static void
284 decompressor_reset_session_lock_held(ZstdDecompressor *self)
285 {
286 assert(PyMutex_IsLocked(&self->lock));
287
288 /* Reset variables */
289 self->in_begin = 0;
290 self->in_end = 0;
291
292 Py_CLEAR(self->unused_data);
293
294 /* Reset variables in one operation */
295 self->needs_input = 1;
296 self->eof = 0;
297
298 /* Resetting session is guaranteed to never fail */
299 ZSTD_DCtx_reset(self->dctx, ZSTD_reset_session_only);
300 }
301
302 static PyObject *
303 stream_decompress_lock_held(ZstdDecompressor *self, Py_buffer *data,
304 Py_ssize_t max_length)
305 {
306 assert(PyMutex_IsLocked(&self->lock));
307 ZSTD_inBuffer in;
308 PyObject *ret = NULL;
309 int use_input_buffer;
310
311 /* Check .eof flag */
312 if (self->eof) {
313 PyErr_SetString(PyExc_EOFError,
314 "Already at the end of a Zstandard frame.");
315 assert(ret == NULL);
316 return NULL;
317 }
318
319 /* Prepare input buffer w/wo unconsumed data */
320 if (self->in_begin == self->in_end) {
321 /* No unconsumed data */
322 use_input_buffer = 0;
323
324 in.src = data->buf;
325 in.size = data->len;
326 in.pos = 0;
327 }
328 else if (data->len == 0) {
329 /* Has unconsumed data, fast path for b'' */
330 assert(self->in_begin < self->in_end);
331
332 use_input_buffer = 1;
333
334 in.src = self->input_buffer + self->in_begin;
335 in.size = self->in_end - self->in_begin;
336 in.pos = 0;
337 }
338 else {
339 /* Has unconsumed data */
340 use_input_buffer = 1;
341
342 /* Unconsumed data size in input_buffer */
343 size_t used_now = self->in_end - self->in_begin;
344 assert(self->in_end > self->in_begin);
345
346 /* Number of bytes we can append to input buffer */
347 size_t avail_now = self->input_buffer_size - self->in_end;
348 assert(self->input_buffer_size >= self->in_end);
349
350 /* Number of bytes we can append if we move existing contents to
351 beginning of buffer */
352 size_t avail_total = self->input_buffer_size - used_now;
353 assert(self->input_buffer_size >= used_now);
354
355 if (avail_total < (size_t) data->len) {
356 char *tmp;
357 size_t new_size = used_now + data->len;
358
359 /* Allocate with new size */
360 tmp = PyMem_Malloc(new_size);
361 if (tmp == NULL) {
362 PyErr_NoMemory();
363 goto error;
364 }
365
366 /* Copy unconsumed data to the beginning of new buffer */
367 memcpy(tmp,
368 self->input_buffer + self->in_begin,
369 used_now);
370
371 /* Switch to new buffer */
372 PyMem_Free(self->input_buffer);
373 self->input_buffer = tmp;
374 self->input_buffer_size = new_size;
375
376 /* Set begin & end position */
377 self->in_begin = 0;
378 self->in_end = used_now;
379 }
380 else if (avail_now < (size_t) data->len) {
381 /* Move unconsumed data to the beginning.
382 Overlap is possible, so use memmove(). */
383 memmove(self->input_buffer,
384 self->input_buffer + self->in_begin,
385 used_now);
386
387 /* Set begin & end position */
388 self->in_begin = 0;
389 self->in_end = used_now;
390 }
391
392 /* Copy data to input buffer */
393 memcpy(self->input_buffer + self->in_end, data->buf, data->len);
394 self->in_end += data->len;
395
396 in.src = self->input_buffer + self->in_begin;
397 in.size = used_now + data->len;
398 in.pos = 0;
399 }
400 assert(in.pos == 0);
401
402 /* Decompress */
403 ret = decompress_lock_held(self, &in, max_length);
404 if (ret == NULL) {
405 goto error;
406 }
407
408 /* Unconsumed input data */
409 if (in.pos == in.size) {
410 if (Py_SIZE(ret) == max_length || self->eof) {
411 self->needs_input = 0;
412 }
413 else {
414 self->needs_input = 1;
415 }
416
417 if (use_input_buffer) {
418 /* Clear input_buffer */
419 self->in_begin = 0;
420 self->in_end = 0;
421 }
422 }
423 else {
424 size_t data_size = in.size - in.pos;
425
426 self->needs_input = 0;
427
428 if (!use_input_buffer) {
429 /* Discard buffer if it's too small
430 (resizing it may needlessly copy the current contents) */
431 if (self->input_buffer != NULL
432 && self->input_buffer_size < data_size)
433 {
434 PyMem_Free(self->input_buffer);
435 self->input_buffer = NULL;
436 self->input_buffer_size = 0;
437 }
438
439 /* Allocate if necessary */
440 if (self->input_buffer == NULL) {
441 self->input_buffer = PyMem_Malloc(data_size);
442 if (self->input_buffer == NULL) {
443 PyErr_NoMemory();
444 goto error;
445 }
446 self->input_buffer_size = data_size;
447 }
448
449 /* Copy unconsumed data */
450 memcpy(self->input_buffer, (char*)in.src + in.pos, data_size);
451 self->in_begin = 0;
452 self->in_end = data_size;
453 }
454 else {
455 /* Use input buffer */
456 self->in_begin += in.pos;
457 }
458 }
459
460 return ret;
461
462 error:
463 /* Reset decompressor's states/session */
464 decompressor_reset_session_lock_held(self);
465
466 Py_CLEAR(ret);
467 return NULL;
468 }
469
470
471 /*[clinic input]
472 @classmethod
473 _zstd.ZstdDecompressor.__new__ as _zstd_ZstdDecompressor_new
474 zstd_dict: object = None
475 A ZstdDict object, a pre-trained Zstandard dictionary.
476 options: object = None
477 A dict object that contains advanced decompression parameters.
478
479 Create a decompressor object for decompressing data incrementally.
480
481 Thread-safe at method level. For one-shot decompression, use the
482 decompress() function instead.
483 [clinic start generated code]*/
484
485 static PyObject *
486 _zstd_ZstdDecompressor_new_impl(PyTypeObject *type, PyObject *zstd_dict,
487 PyObject *options)
488 /*[clinic end generated code: output=590ca65c1102ff4a input=73879de69bf89f59]*/
489 {
490 ZstdDecompressor* self = PyObject_GC_New(ZstdDecompressor, type);
491 if (self == NULL) {
492 goto error;
493 }
494
495 self->input_buffer = NULL;
496 self->input_buffer_size = 0;
497 self->in_begin = -1;
498 self->in_end = -1;
499 self->unused_data = NULL;
500 self->eof = 0;
501 self->dict = NULL;
502 self->lock = (PyMutex){0};
503
504 /* needs_input flag */
505 self->needs_input = 1;
506
507 /* Decompression context */
508 self->dctx = ZSTD_createDCtx();
509 if (self->dctx == NULL) {
510 _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self));
511 if (mod_state != NULL) {
512 PyErr_SetString(mod_state->ZstdError,
513 "Unable to create ZSTD_DCtx instance.");
514 }
515 goto error;
516 }
517
518 /* Load Zstandard dictionary to decompression context */
519 if (zstd_dict != Py_None) {
520 if (_zstd_load_d_dict(self, zstd_dict) < 0) {
521 goto error;
522 }
523 Py_INCREF(zstd_dict);
524 self->dict = zstd_dict;
525 }
526
527 /* Set options dictionary */
528 if (options != Py_None) {
529 if (_zstd_set_d_parameters(self, options) < 0) {
530 goto error;
531 }
532 }
533
534 // We can only start GC tracking once self->dict is set.
535 PyObject_GC_Track(self);
536
537 return (PyObject*)self;
538
539 error:
540 Py_XDECREF(self);
541 return NULL;
542 }
543
544 static void
545 ZstdDecompressor_dealloc(PyObject *ob)
546 {
547 ZstdDecompressor *self = ZstdDecompressor_CAST(ob);
548
549 PyObject_GC_UnTrack(self);
550
551 /* Free decompression context */
552 if (self->dctx) {
553 ZSTD_freeDCtx(self->dctx);
554 }
555
556 assert(!PyMutex_IsLocked(&self->lock));
557
558 /* Py_CLEAR the dict after free decompression context */
559 Py_CLEAR(self->dict);
560
561 /* Free unconsumed input data buffer */
562 PyMem_Free(self->input_buffer);
563
564 /* Free unused data */
565 Py_CLEAR(self->unused_data);
566
567 PyTypeObject *tp = Py_TYPE(self);
568 PyObject_GC_Del(ob);
569 Py_DECREF(tp);
570 }
571
572 /*[clinic input]
573 @getter
574 _zstd.ZstdDecompressor.unused_data
575
576 A bytes object of un-consumed input data.
577
578 When ZstdDecompressor object stops after a frame is
579 decompressed, unused input data after the frame. Otherwise this
580 will be b''.
581 [clinic start generated code]*/
582
583 static PyObject *
584 _zstd_ZstdDecompressor_unused_data_get_impl(ZstdDecompressor *self)
585 /*[clinic end generated code: output=f3a20940f11b6b09 input=0462065c5e60ba01]*/
586 {
587 PyObject *ret;
588
589 PyMutex_Lock(&self->lock);
590
591 if (!self->eof) {
592 PyMutex_Unlock(&self->lock);
593 return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES);
594 }
595 else {
596 if (self->unused_data == NULL) {
597 self->unused_data = PyBytes_FromStringAndSize(
598 self->input_buffer + self->in_begin,
599 self->in_end - self->in_begin);
600 ret = self->unused_data;
601 Py_XINCREF(ret);
602 }
603 else {
604 ret = self->unused_data;
605 Py_INCREF(ret);
606 }
607 }
608
609 PyMutex_Unlock(&self->lock);
610 return ret;
611 }
612
613 /*[clinic input]
614 _zstd.ZstdDecompressor.decompress
615
616 data: Py_buffer
617 A bytes-like object, Zstandard data to be decompressed.
618 max_length: Py_ssize_t = -1
619 Maximum size of returned data. When it is negative, the size of
620 output buffer is unlimited. When it is nonnegative, returns at
621 most max_length bytes of decompressed data.
622
623 Decompress *data*, returning uncompressed bytes if possible, or b'' otherwise.
624
625 If *max_length* is nonnegative, returns at most *max_length* bytes
626 of decompressed data. If this limit is reached and further output
627 can be produced, *self.needs_input* will be set to ``False``. In
628 this case, the next call to *decompress()* may provide *data* as b''
629 to obtain more of the output.
630
631 If all of the input data was decompressed and returned (either
632 because this was less than *max_length* bytes, or because
633 *max_length* was negative), *self.needs_input* will be set to True.
634
635 Attempting to decompress data after the end of a frame is reached
636 raises an EOFError. Any data found after the end of the frame is
637 ignored and saved in the self.unused_data attribute.
638 [clinic start generated code]*/
639
640 static PyObject *
641 _zstd_ZstdDecompressor_decompress_impl(ZstdDecompressor *self,
642 Py_buffer *data,
643 Py_ssize_t max_length)
644 /*[clinic end generated code: output=a4302b3c940dbec6 input=b55991a007f4c558]*/
645 {
646 PyObject *ret;
647 /* Thread-safe code */
648 PyMutex_Lock(&self->lock);
649 ret = stream_decompress_lock_held(self, data, max_length);
650 PyMutex_Unlock(&self->lock);
651 return ret;
652 }
653
654 static PyMethodDef ZstdDecompressor_methods[] = {
655 _ZSTD_ZSTDDECOMPRESSOR_DECOMPRESS_METHODDEF
656 {NULL, NULL}
657 };
658
659 PyDoc_STRVAR(ZstdDecompressor_eof_doc,
660 "True means the end of the first frame has been reached. If decompress data\n"
661 "after that, an EOFError exception will be raised.");
662
663 PyDoc_STRVAR(ZstdDecompressor_needs_input_doc,
664 "If the max_length output limit in .decompress() method has been reached,\n"
665 "and the decompressor has (or may has) unconsumed input data, it will be set\n"
666 "to False. In this case, passing b'' to the .decompress() method may output\n"
667 "further data.");
668
669 static PyMemberDef ZstdDecompressor_members[] = {
670 {"eof", Py_T_BOOL, offsetof(ZstdDecompressor, eof),
671 Py_READONLY, ZstdDecompressor_eof_doc},
672 {"needs_input", Py_T_BOOL, offsetof(ZstdDecompressor, needs_input),
673 Py_READONLY, ZstdDecompressor_needs_input_doc},
674 {NULL}
675 };
676
677 static PyGetSetDef ZstdDecompressor_getset[] = {
678 _ZSTD_ZSTDDECOMPRESSOR_UNUSED_DATA_GETSETDEF
679 {NULL}
680 };
681
682 static int
683 ZstdDecompressor_traverse(PyObject *ob, visitproc visit, void *arg)
684 {
685 ZstdDecompressor *self = ZstdDecompressor_CAST(ob);
686 Py_VISIT(self->dict);
687 return 0;
688 }
689
690 static int
691 ZstdDecompressor_clear(PyObject *ob)
692 {
693 ZstdDecompressor *self = ZstdDecompressor_CAST(ob);
694 Py_CLEAR(self->dict);
695 Py_CLEAR(self->unused_data);
696 return 0;
697 }
698
699 static PyType_Slot ZstdDecompressor_slots[] = {
700 {Py_tp_new, _zstd_ZstdDecompressor_new},
701 {Py_tp_dealloc, ZstdDecompressor_dealloc},
702 {Py_tp_methods, ZstdDecompressor_methods},
703 {Py_tp_members, ZstdDecompressor_members},
704 {Py_tp_getset, ZstdDecompressor_getset},
705 {Py_tp_doc, (void *)_zstd_ZstdDecompressor_new__doc__},
706 {Py_tp_traverse, ZstdDecompressor_traverse},
707 {Py_tp_clear, ZstdDecompressor_clear},
708 {0, 0}
709 };
710
711 PyType_Spec zstd_decompressor_type_spec = {
712 .name = "compression.zstd.ZstdDecompressor",
713 .basicsize = sizeof(ZstdDecompressor),
714 .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE
715 | Py_TPFLAGS_HAVE_GC,
716 .slots = ZstdDecompressor_slots,
717 };