Debian Codegraph @

Modules/_zstd/zstddict.c

raw
1 /* Low level interface to the Zstandard algorithm & the zstd library. */
2
3 /* ZstdDict class definitions */
4
5 /*[clinic input]
6 module _zstd
7 class _zstd.ZstdDict "ZstdDict *" "&zstd_dict_type_spec"
8 [clinic start generated code]*/
9 /*[clinic end generated code: output=da39a3ee5e6b4b0d input=3dcc175ec974f81c]*/
10
11 #ifndef Py_BUILD_CORE_BUILTIN
12 # define Py_BUILD_CORE_MODULE 1
13 #endif
14
15 #include "Python.h"
16
17 #include "_zstdmodule.h"
18 #include "clinic/zstddict.c.h"
19 #include "internal/pycore_lock.h" // PyMutex_IsLocked
20
21 #include <zstd.h> // ZSTD_freeDDict(), ZSTD_getDictID_fromDict()
22
23 #define ZstdDict_CAST(op) ((ZstdDict *)op)
24
25 /*[clinic input]
26 @classmethod
27 _zstd.ZstdDict.__new__ as _zstd_ZstdDict_new
28 dict_content: Py_buffer
29 The content of a Zstandard dictionary as a bytes-like object.
30 /
31 *
32 is_raw: bool = False
33 If true, perform no checks on *dict_content*, useful for some
34 advanced cases. Otherwise, check that the content represents
35 a Zstandard dictionary created by the zstd library or CLI.
36
37 Represents a Zstandard dictionary.
38
39 The dictionary can be used for compression or decompression, and can be
40 shared by multiple ZstdCompressor or ZstdDecompressor objects.
41 [clinic start generated code]*/
42
43 static PyObject *
44 _zstd_ZstdDict_new_impl(PyTypeObject *type, Py_buffer *dict_content,
45 int is_raw)
46 /*[clinic end generated code: output=685b7406a48b0949 input=3bb66063c0240433]*/
47 {
48 /* All dictionaries must be at least 8 bytes */
49 if (dict_content->len < 8) {
50 PyErr_SetString(PyExc_ValueError,
51 "Zstandard dictionary content too short "
52 "(must have at least eight bytes)");
53 return NULL;
54 }
55
56 ZstdDict* self = PyObject_GC_New(ZstdDict, type);
57 if (self == NULL) {
58 return NULL;
59 }
60
61 self->d_dict = NULL;
62 self->dict_buffer = NULL;
63 self->dict_id = 0;
64 self->lock = (PyMutex){0};
65
66 /* ZSTD_CDict dict */
67 self->c_dicts = PyDict_New();
68 if (self->c_dicts == NULL) {
69 goto error;
70 }
71
72 self->dict_buffer = PyMem_Malloc(dict_content->len);
73 if (!self->dict_buffer) {
74 PyErr_NoMemory();
75 goto error;
76 }
77 memcpy(self->dict_buffer, dict_content->buf, dict_content->len);
78 self->dict_len = dict_content->len;
79
80 /* Get dict_id, 0 means "raw content" dictionary. */
81 self->dict_id = ZSTD_getDictID_fromDict(self->dict_buffer, self->dict_len);
82
83 /* Check validity for ordinary dictionary */
84 if (!is_raw && self->dict_id == 0) {
85 PyErr_SetString(PyExc_ValueError, "invalid Zstandard dictionary");
86 goto error;
87 }
88
89 PyObject_GC_Track(self);
90
91 return (PyObject *)self;
92
93 error:
94 Py_XDECREF(self);
95 return NULL;
96 }
97
98 static void
99 ZstdDict_dealloc(PyObject *ob)
100 {
101 ZstdDict *self = ZstdDict_CAST(ob);
102
103 PyObject_GC_UnTrack(self);
104
105 /* Free ZSTD_DDict instance */
106 if (self->d_dict) {
107 ZSTD_freeDDict(self->d_dict);
108 }
109
110 assert(!PyMutex_IsLocked(&self->lock));
111
112 /* Release dict_buffer after freeing ZSTD_CDict/ZSTD_DDict instances */
113 PyMem_Free(self->dict_buffer);
114 Py_CLEAR(self->c_dicts);
115
116 PyTypeObject *tp = Py_TYPE(self);
117 tp->tp_free(self);
118 Py_DECREF(tp);
119 }
120
121 PyDoc_STRVAR(ZstdDict_dictid_doc,
122 "The Zstandard dictionary, an int between 0 and 2**32.\n\n"
123 "A non-zero value represents an ordinary Zstandard dictionary,\n"
124 "conforming to the standardised format.\n\n"
125 "A value of zero indicates a 'raw content' dictionary,\n"
126 "without any restrictions on format or content.");
127
128 static PyObject *
129 ZstdDict_repr(PyObject *ob)
130 {
131 ZstdDict *dict = ZstdDict_CAST(ob);
132 return PyUnicode_FromFormat("<ZstdDict dict_id=%u dict_size=%zd>",
133 (unsigned int)dict->dict_id, dict->dict_len);
134 }
135
136 static PyMemberDef ZstdDict_members[] = {
137 {"dict_id", Py_T_UINT, offsetof(ZstdDict, dict_id), Py_READONLY, ZstdDict_dictid_doc},
138 {NULL}
139 };
140
141 /*[clinic input]
142 @getter
143 _zstd.ZstdDict.dict_content
144
145 The content of a Zstandard dictionary, as a bytes object.
146 [clinic start generated code]*/
147
148 static PyObject *
149 _zstd_ZstdDict_dict_content_get_impl(ZstdDict *self)
150 /*[clinic end generated code: output=0d05caa5b550eabb input=4ed526d1c151c596]*/
151 {
152 return PyBytes_FromStringAndSize(self->dict_buffer, self->dict_len);
153 }
154
155 /*[clinic input]
156 @getter
157 _zstd.ZstdDict.as_digested_dict
158
159 Load as a digested dictionary to compressor.
160
161 Pass this attribute as zstd_dict argument:
162 compress(dat, zstd_dict=zd.as_digested_dict)
163
164 1. Some advanced compression parameters of compressor may be
165 overridden by parameters of digested dictionary.
166 2. ZstdDict has a digested dictionaries cache for each compression
167 level. It's faster when loading again a digested dictionary with
168 the same compression level.
169 3. No need to use this for decompression.
170 [clinic start generated code]*/
171
172 static PyObject *
173 _zstd_ZstdDict_as_digested_dict_get_impl(ZstdDict *self)
174 /*[clinic end generated code: output=09b086e7a7320dbb input=a9417d40f1d7fedd]*/
175 {
176 return Py_BuildValue("Oi", self, DICT_TYPE_DIGESTED);
177 }
178
179 /*[clinic input]
180 @getter
181 _zstd.ZstdDict.as_undigested_dict
182
183 Load as an undigested dictionary to compressor.
184
185 Pass this attribute as zstd_dict argument:
186 compress(dat, zstd_dict=zd.as_undigested_dict)
187
188 1. The advanced compression parameters of compressor will not be
189 overridden.
190 2. Loading an undigested dictionary is costly. If load an undigested
191 dictionary multiple times, consider reusing a compressor object.
192 3. No need to use this for decompression.
193 [clinic start generated code]*/
194
195 static PyObject *
196 _zstd_ZstdDict_as_undigested_dict_get_impl(ZstdDict *self)
197 /*[clinic end generated code: output=43c7a989e6d4253a input=56443c9c4e589cd5]*/
198 {
199 return Py_BuildValue("Oi", self, DICT_TYPE_UNDIGESTED);
200 }
201
202 /*[clinic input]
203 @getter
204 _zstd.ZstdDict.as_prefix
205
206 Load as a prefix to compressor/decompressor.
207
208 Pass this attribute as zstd_dict argument:
209 compress(dat, zstd_dict=zd.as_prefix)
210
211 1. Prefix is compatible with long distance matching, while
212 dictionary is not.
213 2. It only works for the first frame, then the
214 compressor/decompressor will return to no prefix state.
215 3. When decompressing, must use the same prefix as when compressing.
216 [clinic start generated code]*/
217
218 static PyObject *
219 _zstd_ZstdDict_as_prefix_get_impl(ZstdDict *self)
220 /*[clinic end generated code: output=6f7130c356595a16 input=192681a899c6fad0]*/
221 {
222 return Py_BuildValue("Oi", self, DICT_TYPE_PREFIX);
223 }
224
225 static PyGetSetDef ZstdDict_getset[] = {
226 _ZSTD_ZSTDDICT_DICT_CONTENT_GETSETDEF
227 _ZSTD_ZSTDDICT_AS_DIGESTED_DICT_GETSETDEF
228 _ZSTD_ZSTDDICT_AS_UNDIGESTED_DICT_GETSETDEF
229 _ZSTD_ZSTDDICT_AS_PREFIX_GETSETDEF
230 {NULL}
231 };
232
233 static Py_ssize_t
234 ZstdDict_length(PyObject *ob)
235 {
236 ZstdDict *self = ZstdDict_CAST(ob);
237 return self->dict_len;
238 }
239
240 static int
241 ZstdDict_traverse(PyObject *ob, visitproc visit, void *arg)
242 {
243 ZstdDict *self = ZstdDict_CAST(ob);
244 Py_VISIT(self->c_dicts);
245 return 0;
246 }
247
248 static int
249 ZstdDict_clear(PyObject *ob)
250 {
251 ZstdDict *self = ZstdDict_CAST(ob);
252 Py_CLEAR(self->c_dicts);
253 return 0;
254 }
255
256 static PyType_Slot zstddict_slots[] = {
257 {Py_tp_members, ZstdDict_members},
258 {Py_tp_getset, ZstdDict_getset},
259 {Py_tp_new, _zstd_ZstdDict_new},
260 {Py_tp_dealloc, ZstdDict_dealloc},
261 {Py_tp_repr, ZstdDict_repr},
262 {Py_tp_doc, (void *)_zstd_ZstdDict_new__doc__},
263 {Py_sq_length, ZstdDict_length},
264 {Py_tp_traverse, ZstdDict_traverse},
265 {Py_tp_clear, ZstdDict_clear},
266 {0, 0}
267 };
268
269 PyType_Spec zstd_dict_type_spec = {
270 .name = "compression.zstd.ZstdDict",
271 .basicsize = sizeof(ZstdDict),
272 .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE
273 | Py_TPFLAGS_HAVE_GC,
274 .slots = zstddict_slots,
275 };