Debian Codegraph @

src/c/compression_zstd/zstddict.c

raw
1 /* Low level interface to the Zstandard algorithm & the zstd library. */
2
3 /* ZstdDict class definitions */
4
5 /*[clinic input]
6 module _zstd
7 class _zstd.ZstdDict "ZstdDict *" "&zstd_dict_type_spec"
8 [clinic start generated code]*/
9 /*[clinic end generated code: output=da39a3ee5e6b4b0d input=3dcc175ec974f81c]*/
10
11 #ifndef Py_BUILD_CORE_BUILTIN
12 # define Py_BUILD_CORE_MODULE 1
13 #endif
14
15 #include "Python.h"
16
17 #include "backports_zstd_edits.h"
18
19 #include "_zstdmodule.h"
20 #include "clinic/zstddict.c.h"
21
22 #include <zstd.h> // ZSTD_freeDDict(), ZSTD_getDictID_fromDict()
23
24 #define ZstdDict_CAST(op) ((ZstdDict *)op)
25
26 /*[clinic input]
27 @classmethod
28 _zstd.ZstdDict.__new__ as _zstd_ZstdDict_new
29 dict_content: Py_buffer
30 The content of a Zstandard dictionary as a bytes-like object.
31 /
32 *
33 is_raw: bool = False
34 If true, perform no checks on *dict_content*, useful for some
35 advanced cases. Otherwise, check that the content represents
36 a Zstandard dictionary created by the zstd library or CLI.
37
38 Represents a Zstandard dictionary.
39
40 The dictionary can be used for compression or decompression, and can be shared
41 by multiple ZstdCompressor or ZstdDecompressor objects.
42 [clinic start generated code]*/
43
44 static PyObject *
45 _zstd_ZstdDict_new_impl(PyTypeObject *type, Py_buffer *dict_content,
46 int is_raw)
47 /*[clinic end generated code: output=685b7406a48b0949 input=9e8c493e31c98383]*/
48 {
49 /* All dictionaries must be at least 8 bytes */
50 if (dict_content->len < 8) {
51 PyErr_SetString(PyExc_ValueError,
52 "Zstandard dictionary content too short "
53 "(must have at least eight bytes)");
54 return NULL;
55 }
56
57 ZstdDict* self = PyObject_GC_New(ZstdDict, type);
58 if (self == NULL) {
59 return NULL;
60 }
61
62 self->d_dict = NULL;
63 self->dict_buffer = NULL;
64 self->dict_id = 0;
65 self->lock = BACKPORTSZSTD_LOCK_allocate();
66 if(BACKPORTSZSTD_LOCK_isError(self->lock)) {
67 _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self));
68 if (mod_state != NULL) {
69 PyErr_SetString(mod_state->ZstdError,
70 "Unable to allocate lock");
71 }
72 goto error;
73 }
74
75 /* ZSTD_CDict dict */
76 self->c_dicts = PyDict_New();
77 if (self->c_dicts == NULL) {
78 goto error;
79 }
80
81 self->dict_buffer = PyMem_Malloc(dict_content->len);
82 if (!self->dict_buffer) {
83 PyErr_NoMemory();
84 goto error;
85 }
86 memcpy(self->dict_buffer, dict_content->buf, dict_content->len);
87 self->dict_len = dict_content->len;
88
89 /* Get dict_id, 0 means "raw content" dictionary. */
90 self->dict_id = ZSTD_getDictID_fromDict(self->dict_buffer, self->dict_len);
91
92 /* Check validity for ordinary dictionary */
93 if (!is_raw && self->dict_id == 0) {
94 PyErr_SetString(PyExc_ValueError, "invalid Zstandard dictionary");
95 goto error;
96 }
97
98 PyObject_GC_Track(self);
99
100 return (PyObject *)self;
101
102 error:
103 Py_XDECREF(self);
104 return NULL;
105 }
106
107 static void
108 ZstdDict_dealloc(PyObject *ob)
109 {
110 ZstdDict *self = ZstdDict_CAST(ob);
111
112 PyObject_GC_UnTrack(self);
113
114 /* Free ZSTD_DDict instance */
115 if (self->d_dict) {
116 ZSTD_freeDDict(self->d_dict);
117 }
118
119 assert(!BACKPORTSZSTD_LOCK_isLocked(&self->lock));
120 BACKPORTSZSTD_LOCK_free(self->lock);
121
122 /* Release dict_buffer after freeing ZSTD_CDict/ZSTD_DDict instances */
123 PyMem_Free(self->dict_buffer);
124 Py_CLEAR(self->c_dicts);
125
126 PyTypeObject *tp = Py_TYPE(self);
127 tp->tp_free(self);
128 Py_DECREF(tp);
129 }
130
131 PyDoc_STRVAR(ZstdDict_dictid_doc,
132 "The Zstandard dictionary, an int between 0 and 2**32.\n\n"
133 "A non-zero value represents an ordinary Zstandard dictionary,\n"
134 "conforming to the standardised format.\n\n"
135 "A value of zero indicates a 'raw content' dictionary,\n"
136 "without any restrictions on format or content.");
137
138 static PyObject *
139 ZstdDict_repr(PyObject *ob)
140 {
141 ZstdDict *dict = ZstdDict_CAST(ob);
142 return PyUnicode_FromFormat("<ZstdDict dict_id=%u dict_size=%zd>",
143 (unsigned int)dict->dict_id, dict->dict_len);
144 }
145
146 static PyMemberDef ZstdDict_members[] = {
147 {"dict_id", Py_T_UINT, offsetof(ZstdDict, dict_id), Py_READONLY, ZstdDict_dictid_doc},
148 {NULL}
149 };
150
151 /*[clinic input]
152 @getter
153 _zstd.ZstdDict.dict_content
154
155 The content of a Zstandard dictionary, as a bytes object.
156 [clinic start generated code]*/
157
158 static PyObject *
159 _zstd_ZstdDict_dict_content_get_impl(ZstdDict *self)
160 /*[clinic end generated code: output=0d05caa5b550eabb input=4ed526d1c151c596]*/
161 {
162 return PyBytes_FromStringAndSize(self->dict_buffer, self->dict_len);
163 }
164
165 /*[clinic input]
166 @getter
167 _zstd.ZstdDict.as_digested_dict
168
169 Load as a digested dictionary to compressor.
170
171 Pass this attribute as zstd_dict argument:
172 compress(dat, zstd_dict=zd.as_digested_dict)
173
174 1. Some advanced compression parameters of compressor may be overridden
175 by parameters of digested dictionary.
176 2. ZstdDict has a digested dictionaries cache for each compression level.
177 It's faster when loading again a digested dictionary with the same
178 compression level.
179 3. No need to use this for decompression.
180 [clinic start generated code]*/
181
182 static PyObject *
183 _zstd_ZstdDict_as_digested_dict_get_impl(ZstdDict *self)
184 /*[clinic end generated code: output=09b086e7a7320dbb input=ee45e1b4a48f6f2c]*/
185 {
186 return Py_BuildValue("Oi", self, DICT_TYPE_DIGESTED);
187 }
188
189 /*[clinic input]
190 @getter
191 _zstd.ZstdDict.as_undigested_dict
192
193 Load as an undigested dictionary to compressor.
194
195 Pass this attribute as zstd_dict argument:
196 compress(dat, zstd_dict=zd.as_undigested_dict)
197
198 1. The advanced compression parameters of compressor will not be overridden.
199 2. Loading an undigested dictionary is costly. If load an undigested dictionary
200 multiple times, consider reusing a compressor object.
201 3. No need to use this for decompression.
202 [clinic start generated code]*/
203
204 static PyObject *
205 _zstd_ZstdDict_as_undigested_dict_get_impl(ZstdDict *self)
206 /*[clinic end generated code: output=43c7a989e6d4253a input=d39210eedec76fed]*/
207 {
208 return Py_BuildValue("Oi", self, DICT_TYPE_UNDIGESTED);
209 }
210
211 /*[clinic input]
212 @getter
213 _zstd.ZstdDict.as_prefix
214
215 Load as a prefix to compressor/decompressor.
216
217 Pass this attribute as zstd_dict argument:
218 compress(dat, zstd_dict=zd.as_prefix)
219
220 1. Prefix is compatible with long distance matching, while dictionary is not.
221 2. It only works for the first frame, then the compressor/decompressor will
222 return to no prefix state.
223 3. When decompressing, must use the same prefix as when compressing.
224 [clinic start generated code]*/
225
226 static PyObject *
227 _zstd_ZstdDict_as_prefix_get_impl(ZstdDict *self)
228 /*[clinic end generated code: output=6f7130c356595a16 input=45b3b6110f36d127]*/
229 {
230 return Py_BuildValue("Oi", self, DICT_TYPE_PREFIX);
231 }
232
233 static PyGetSetDef ZstdDict_getset[] = {
234 _ZSTD_ZSTDDICT_DICT_CONTENT_GETSETDEF
235 _ZSTD_ZSTDDICT_AS_DIGESTED_DICT_GETSETDEF
236 _ZSTD_ZSTDDICT_AS_UNDIGESTED_DICT_GETSETDEF
237 _ZSTD_ZSTDDICT_AS_PREFIX_GETSETDEF
238 {NULL}
239 };
240
241 static Py_ssize_t
242 ZstdDict_length(PyObject *ob)
243 {
244 ZstdDict *self = ZstdDict_CAST(ob);
245 return self->dict_len;
246 }
247
248 static int
249 ZstdDict_traverse(PyObject *ob, visitproc visit, void *arg)
250 {
251 ZstdDict *self = ZstdDict_CAST(ob);
252 Py_VISIT(self->c_dicts);
253 return 0;
254 }
255
256 static int
257 ZstdDict_clear(PyObject *ob)
258 {
259 ZstdDict *self = ZstdDict_CAST(ob);
260 Py_CLEAR(self->c_dicts);
261 return 0;
262 }
263
264 static PyType_Slot zstddict_slots[] = {
265 {Py_tp_members, ZstdDict_members},
266 {Py_tp_getset, ZstdDict_getset},
267 {Py_tp_new, _zstd_ZstdDict_new},
268 {Py_tp_dealloc, ZstdDict_dealloc},
269 {Py_tp_repr, ZstdDict_repr},
270 {Py_tp_doc, (void *)_zstd_ZstdDict_new__doc__},
271 {Py_sq_length, ZstdDict_length},
272 {Py_tp_traverse, ZstdDict_traverse},
273 {Py_tp_clear, ZstdDict_clear},
274 {0, 0}
275 };
276
277 PyType_Spec zstd_dict_type_spec = {
278 .name = "backports.zstd.ZstdDict",
279 .basicsize = sizeof(ZstdDict),
280 .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE
281 | Py_TPFLAGS_HAVE_GC,
282 .slots = zstddict_slots,
283 };