Debian Codegraph @

Modules/_zstd/zstddict.c

raw
1 /* Low level interface to the Zstandard algorithm & the zstd library. */
2
3 /* ZstdDict class definitions */
4
5 /*[clinic input]
6 module _zstd
7 class _zstd.ZstdDict "ZstdDict *" "&zstd_dict_type_spec"
8 [clinic start generated code]*/
9 /*[clinic end generated code: output=da39a3ee5e6b4b0d input=3dcc175ec974f81c]*/
10
11 #ifndef Py_BUILD_CORE_BUILTIN
12 # define Py_BUILD_CORE_MODULE 1
13 #endif
14
15 #include "Python.h"
16
17 #include "_zstdmodule.h"
18 #include "clinic/zstddict.c.h"
19 #include "internal/pycore_lock.h" // PyMutex_IsLocked
20
21 #include <zstd.h> // ZSTD_freeDDict(), ZSTD_getDictID_fromDict()
22
23 #define ZstdDict_CAST(op) ((ZstdDict *)op)
24
25 /*[clinic input]
26 @classmethod
27 _zstd.ZstdDict.__new__ as _zstd_ZstdDict_new
28 dict_content: Py_buffer
29 The content of a Zstandard dictionary as a bytes-like object.
30 /
31 *
32 is_raw: bool = False
33 If true, perform no checks on *dict_content*, useful for some
34 advanced cases. Otherwise, check that the content represents
35 a Zstandard dictionary created by the zstd library or CLI.
36
37 Represents a Zstandard dictionary.
38
39 The dictionary can be used for compression or decompression, and can be shared
40 by multiple ZstdCompressor or ZstdDecompressor objects.
41 [clinic start generated code]*/
42
43 static PyObject *
44 _zstd_ZstdDict_new_impl(PyTypeObject *type, Py_buffer *dict_content,
45 int is_raw)
46 /*[clinic end generated code: output=685b7406a48b0949 input=9e8c493e31c98383]*/
47 {
48 /* All dictionaries must be at least 8 bytes */
49 if (dict_content->len < 8) {
50 PyErr_SetString(PyExc_ValueError,
51 "Zstandard dictionary content too short "
52 "(must have at least eight bytes)");
53 return NULL;
54 }
55
56 ZstdDict* self = PyObject_GC_New(ZstdDict, type);
57 if (self == NULL) {
58 return NULL;
59 }
60
61 self->d_dict = NULL;
62 self->dict_buffer = NULL;
63 self->dict_id = 0;
64 self->lock = (PyMutex){0};
65
66 /* ZSTD_CDict dict */
67 self->c_dicts = PyDict_New();
68 if (self->c_dicts == NULL) {
69 goto error;
70 }
71
72 self->dict_buffer = PyMem_Malloc(dict_content->len);
73 if (!self->dict_buffer) {
74 PyErr_NoMemory();
75 goto error;
76 }
77 memcpy(self->dict_buffer, dict_content->buf, dict_content->len);
78 self->dict_len = dict_content->len;
79
80 /* Get dict_id, 0 means "raw content" dictionary. */
81 self->dict_id = ZSTD_getDictID_fromDict(self->dict_buffer, self->dict_len);
82
83 /* Check validity for ordinary dictionary */
84 if (!is_raw && self->dict_id == 0) {
85 PyErr_SetString(PyExc_ValueError, "invalid Zstandard dictionary");
86 goto error;
87 }
88
89 PyObject_GC_Track(self);
90
91 return (PyObject *)self;
92
93 error:
94 Py_XDECREF(self);
95 return NULL;
96 }
97
98 static void
99 ZstdDict_dealloc(PyObject *ob)
100 {
101 ZstdDict *self = ZstdDict_CAST(ob);
102
103 PyObject_GC_UnTrack(self);
104
105 /* Free ZSTD_DDict instance */
106 if (self->d_dict) {
107 ZSTD_freeDDict(self->d_dict);
108 }
109
110 assert(!PyMutex_IsLocked(&self->lock));
111
112 /* Release dict_buffer after freeing ZSTD_CDict/ZSTD_DDict instances */
113 PyMem_Free(self->dict_buffer);
114 Py_CLEAR(self->c_dicts);
115
116 PyTypeObject *tp = Py_TYPE(self);
117 tp->tp_free(self);
118 Py_DECREF(tp);
119 }
120
121 PyDoc_STRVAR(ZstdDict_dictid_doc,
122 "The Zstandard dictionary, an int between 0 and 2**32.\n\n"
123 "A non-zero value represents an ordinary Zstandard dictionary,\n"
124 "conforming to the standardised format.\n\n"
125 "A value of zero indicates a 'raw content' dictionary,\n"
126 "without any restrictions on format or content.");
127
128 static PyObject *
129 ZstdDict_repr(PyObject *ob)
130 {
131 ZstdDict *dict = ZstdDict_CAST(ob);
132 return PyUnicode_FromFormat("<ZstdDict dict_id=%u dict_size=%zd>",
133 (unsigned int)dict->dict_id, dict->dict_len);
134 }
135
136 static PyMemberDef ZstdDict_members[] = {
137 {"dict_id", Py_T_UINT, offsetof(ZstdDict, dict_id), Py_READONLY, ZstdDict_dictid_doc},
138 {NULL}
139 };
140
141 /*[clinic input]
142 @getter
143 _zstd.ZstdDict.dict_content
144
145 The content of a Zstandard dictionary, as a bytes object.
146 [clinic start generated code]*/
147
148 static PyObject *
149 _zstd_ZstdDict_dict_content_get_impl(ZstdDict *self)
150 /*[clinic end generated code: output=0d05caa5b550eabb input=4ed526d1c151c596]*/
151 {
152 return PyBytes_FromStringAndSize(self->dict_buffer, self->dict_len);
153 }
154
155 /*[clinic input]
156 @getter
157 _zstd.ZstdDict.as_digested_dict
158
159 Load as a digested dictionary to compressor.
160
161 Pass this attribute as zstd_dict argument:
162 compress(dat, zstd_dict=zd.as_digested_dict)
163
164 1. Some advanced compression parameters of compressor may be overridden
165 by parameters of digested dictionary.
166 2. ZstdDict has a digested dictionaries cache for each compression level.
167 It's faster when loading again a digested dictionary with the same
168 compression level.
169 3. No need to use this for decompression.
170 [clinic start generated code]*/
171
172 static PyObject *
173 _zstd_ZstdDict_as_digested_dict_get_impl(ZstdDict *self)
174 /*[clinic end generated code: output=09b086e7a7320dbb input=ee45e1b4a48f6f2c]*/
175 {
176 return Py_BuildValue("Oi", self, DICT_TYPE_DIGESTED);
177 }
178
179 /*[clinic input]
180 @getter
181 _zstd.ZstdDict.as_undigested_dict
182
183 Load as an undigested dictionary to compressor.
184
185 Pass this attribute as zstd_dict argument:
186 compress(dat, zstd_dict=zd.as_undigested_dict)
187
188 1. The advanced compression parameters of compressor will not be overridden.
189 2. Loading an undigested dictionary is costly. If load an undigested dictionary
190 multiple times, consider reusing a compressor object.
191 3. No need to use this for decompression.
192 [clinic start generated code]*/
193
194 static PyObject *
195 _zstd_ZstdDict_as_undigested_dict_get_impl(ZstdDict *self)
196 /*[clinic end generated code: output=43c7a989e6d4253a input=d39210eedec76fed]*/
197 {
198 return Py_BuildValue("Oi", self, DICT_TYPE_UNDIGESTED);
199 }
200
201 /*[clinic input]
202 @getter
203 _zstd.ZstdDict.as_prefix
204
205 Load as a prefix to compressor/decompressor.
206
207 Pass this attribute as zstd_dict argument:
208 compress(dat, zstd_dict=zd.as_prefix)
209
210 1. Prefix is compatible with long distance matching, while dictionary is not.
211 2. It only works for the first frame, then the compressor/decompressor will
212 return to no prefix state.
213 3. When decompressing, must use the same prefix as when compressing.
214 [clinic start generated code]*/
215
216 static PyObject *
217 _zstd_ZstdDict_as_prefix_get_impl(ZstdDict *self)
218 /*[clinic end generated code: output=6f7130c356595a16 input=45b3b6110f36d127]*/
219 {
220 return Py_BuildValue("Oi", self, DICT_TYPE_PREFIX);
221 }
222
223 static PyGetSetDef ZstdDict_getset[] = {
224 _ZSTD_ZSTDDICT_DICT_CONTENT_GETSETDEF
225 _ZSTD_ZSTDDICT_AS_DIGESTED_DICT_GETSETDEF
226 _ZSTD_ZSTDDICT_AS_UNDIGESTED_DICT_GETSETDEF
227 _ZSTD_ZSTDDICT_AS_PREFIX_GETSETDEF
228 {NULL}
229 };
230
231 static Py_ssize_t
232 ZstdDict_length(PyObject *ob)
233 {
234 ZstdDict *self = ZstdDict_CAST(ob);
235 return self->dict_len;
236 }
237
238 static int
239 ZstdDict_traverse(PyObject *ob, visitproc visit, void *arg)
240 {
241 ZstdDict *self = ZstdDict_CAST(ob);
242 Py_VISIT(self->c_dicts);
243 return 0;
244 }
245
246 static int
247 ZstdDict_clear(PyObject *ob)
248 {
249 ZstdDict *self = ZstdDict_CAST(ob);
250 Py_CLEAR(self->c_dicts);
251 return 0;
252 }
253
254 static PyType_Slot zstddict_slots[] = {
255 {Py_tp_members, ZstdDict_members},
256 {Py_tp_getset, ZstdDict_getset},
257 {Py_tp_new, _zstd_ZstdDict_new},
258 {Py_tp_dealloc, ZstdDict_dealloc},
259 {Py_tp_repr, ZstdDict_repr},
260 {Py_tp_doc, (void *)_zstd_ZstdDict_new__doc__},
261 {Py_sq_length, ZstdDict_length},
262 {Py_tp_traverse, ZstdDict_traverse},
263 {Py_tp_clear, ZstdDict_clear},
264 {0, 0}
265 };
266
267 PyType_Spec zstd_dict_type_spec = {
268 .name = "compression.zstd.ZstdDict",
269 .basicsize = sizeof(ZstdDict),
270 .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE
271 | Py_TPFLAGS_HAVE_GC,
272 .slots = zstddict_slots,
273 };