Debian Codegraph @

c_impl/blake3module.c

raw
1 #define PY_SSIZE_T_CLEAN
2 #include <Python.h>
3
4 #include <stdbool.h>
5 #include <stdio.h>
6
7 #include "blake3.h"
8
9 #define AUTO -1
10
11 #define BUFSIZE 65536
12
13 // CPython defines HASHLIB_GIL_MINSIZE in hashlib.h. We'll want to remove this
14 // definition if this code is added to CPython.
15 #ifdef HASHLIB_GIL_MINSIZE
16 #error Already defined. Delete these lines?
17 #else
18 #define HASHLIB_GIL_MINSIZE 2048
19 #endif
20
21 static void release_buf_if_acquired(Py_buffer *buf) {
22 if (buf != NULL && buf->obj != NULL) {
23 PyBuffer_Release(buf);
24 }
25 }
26
27 static bool weird_buffer(const Py_buffer *buf) {
28 if (buf != NULL && buf->obj != NULL && buf->itemsize != 1) {
29 PyErr_SetString(PyExc_ValueError, "buffer elements must be bytes");
30 return true;
31 }
32 return false;
33 }
34
35 typedef struct {
36 // clang-format gets confused by the (correct, documented) missing semicolon
37 // after PyObject_HEAD.
38 // clang-format off
39 PyObject_HEAD
40 blake3_hasher hasher;
41 PyThread_type_lock lock;
42 // NOTE: Any new fields here need to be handled in both _init() and _copy().
43 // clang-format on
44 } Blake3Object;
45
46 static void Blake3_dealloc(Blake3Object *self) {
47 PyThread_free_lock(self->lock);
48 Py_TYPE(self)->tp_free((PyObject *)self);
49 }
50
51 static PyObject *Blake3_new(PyTypeObject *type, PyObject *args,
52 PyObject *kwds) {
53 Blake3Object *self = NULL;
54 PyThread_type_lock self_lock = NULL;
55 Py_buffer data = {0};
56 Py_buffer key = {0};
57 const char *derive_key_context = NULL;
58 Py_ssize_t max_threads = 1;
59 int usedforsecurity = 1;
60
61 PyObject *ret = NULL;
62
63 static char *kwlist[] = {
64 "", // data, positional-only
65 "key",
66 "derive_key_context",
67 "max_threads", // currently ignored
68 "usedforsecurity", // currently ignored
69 NULL,
70 };
71 if (!PyArg_ParseTupleAndKeywords(args, kwds, "|y*$y*snp", kwlist, &data, &key,
72 &derive_key_context, &max_threads,
73 &usedforsecurity)) {
74 goto exit;
75 }
76
77 if (weird_buffer(&data)) {
78 goto exit;
79 }
80
81 if (weird_buffer(&key)) {
82 goto exit;
83 }
84
85 if (key.obj != NULL && derive_key_context != NULL) {
86 PyErr_SetString(PyExc_ValueError,
87 "key and derive_key_context can't be used together");
88 goto exit;
89 }
90
91 if (key.obj != NULL && key.len != BLAKE3_KEY_LEN) {
92 PyErr_SetString(PyExc_ValueError, "keys must be 32 bytes");
93 goto exit;
94 }
95
96 if (max_threads < 1 && max_threads != AUTO) {
97 PyErr_SetString(PyExc_ValueError, "invalid value for max_threads");
98 goto exit;
99 }
100
101 self = (Blake3Object *)type->tp_alloc(type, 0);
102 if (self == NULL) {
103 goto exit;
104 }
105
106 // TODO: Hashlib implementations do an optimization where they avoid
107 // allocating this lock unless it's needed. Is that worth it? It would mean
108 // we'd need to handle the possible allocation failure at every lock site.
109 // (Hashlib itself handles these by retaining the GIL rather than reporting
110 // the error.)
111 self_lock = PyThread_allocate_lock();
112 if (self_lock == NULL) {
113 goto exit;
114 }
115
116 if (key.obj != NULL) {
117 blake3_hasher_init_keyed(&self->hasher, key.buf);
118 } else if (derive_key_context != NULL) {
119 blake3_hasher_init_derive_key(&self->hasher, derive_key_context);
120 } else {
121 blake3_hasher_init(&self->hasher);
122 }
123
124 if (data.obj != NULL) {
125 if (data.len >= HASHLIB_GIL_MINSIZE) {
126 // clang-format off
127 Py_BEGIN_ALLOW_THREADS
128 // This instance is not yet shared, so we don't need self->lock.
129 blake3_hasher_update(&self->hasher, data.buf, data.len);
130 Py_END_ALLOW_THREADS
131 // clang-format on
132 } else {
133 // Don't bother releasing the GIL for short inputs.
134 blake3_hasher_update(&self->hasher, data.buf, data.len);
135 }
136 }
137
138 // success
139 self->lock = self_lock;
140 ret = (PyObject *)self;
141 self_lock = NULL; // pass ownership
142 self = NULL; // pass ownership
143
144 exit:
145 if (self != NULL) {
146 Py_TYPE(self)->tp_free((PyObject *)self);
147 }
148 if (self_lock != NULL) {
149 PyThread_free_lock(self_lock);
150 }
151 release_buf_if_acquired(&data);
152 release_buf_if_acquired(&key);
153 return ret;
154 }
155
156 // Used for long updates and long outputs.
157 static void Blake3_release_gil_and_lock_self(Blake3Object *self,
158 PyThreadState **thread_state) {
159 *thread_state = PyEval_SaveThread();
160 PyThread_acquire_lock(self->lock, WAIT_LOCK);
161 }
162
163 static void Blake3_unlock_self_and_acquire_gil(Blake3Object *self,
164 PyThreadState **thread_state) {
165 PyThread_release_lock(self->lock);
166 PyEval_RestoreThread(*thread_state);
167 }
168
169 // Used for shorter operations that touch self.
170 static void Blake3_lock_self(Blake3Object *self) {
171 // The optimistic locking strategy here is copied from CPython's ENTER_HASHLIB
172 // macro. If we port this code to hashlib, we should probably use that.
173 #ifdef ENTER_HASHLIB
174 #error Delete this helper function?
175 #endif
176 if (!PyThread_acquire_lock(self->lock, NOWAIT_LOCK)) {
177 // clang-format off
178 Py_BEGIN_ALLOW_THREADS
179 PyThread_acquire_lock(self->lock, WAIT_LOCK);
180 Py_END_ALLOW_THREADS
181 // clang-format on
182 }
183 }
184
185 static void Blake3_unlock_self(Blake3Object *self) {
186 #ifdef LEAVE_HASHLIB
187 #error Delete this helper function too?
188 #endif
189 PyThread_release_lock(self->lock);
190 }
191
192 static PyObject *Blake3_update(Blake3Object *self, PyObject *args) {
193 Py_buffer data = {0};
194
195 PyObject *ret = NULL;
196
197 if (!PyArg_ParseTuple(args, "y*", &data)) {
198 goto exit;
199 }
200
201 if (weird_buffer(&data)) {
202 goto exit;
203 }
204
205 if (data.len >= HASHLIB_GIL_MINSIZE) {
206 PyThreadState *thread_state;
207 Blake3_release_gil_and_lock_self(self, &thread_state);
208 blake3_hasher_update(&self->hasher, data.buf, data.len);
209 Blake3_unlock_self_and_acquire_gil(self, &thread_state);
210 } else {
211 // Don't bother releasing the GIL for short inputs.
212 Blake3_lock_self(self);
213 blake3_hasher_update(&self->hasher, data.buf, data.len);
214 Blake3_unlock_self(self);
215 }
216
217 // Success. We need to increment the refcount on self to return it, see:
218 // https://docs.python.org/3/extending/extending.html#ownership-rules.
219 Py_INCREF(self);
220 ret = (PyObject *)self;
221
222 exit:
223 release_buf_if_acquired(&data);
224 return ret;
225 }
226
227 // This implementation doesn't actually use mmap; it just falls back to regular
228 // file reading. This mainly exists for compatibility with the Rust
229 // implementation's Python test suite.
230 // TODO: actually mmap
231 static PyObject *Blake3_update_mmap(Blake3Object *self, PyObject *args,
232 PyObject *kwds) {
233 PyBytesObject *path_bytes = NULL;
234 FILE *file = NULL;
235 PyObject *ret = NULL;
236
237 static char *kwlist[] = {
238 "path",
239 NULL,
240 };
241 if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&", kwlist,
242 PyUnicode_FSConverter, &path_bytes)) {
243 return NULL;
244 }
245
246 PyThreadState *thread_state;
247 Blake3_release_gil_and_lock_self(self, &thread_state);
248
249 file = fopen(PyBytes_AS_STRING(path_bytes), "r");
250 if (!file) {
251 goto exit;
252 }
253
254 char *buf[BUFSIZE];
255 while (1) {
256 size_t n = fread(buf, sizeof(char), BUFSIZE, file);
257 if (ferror(file)) {
258 goto exit;
259 }
260 blake3_hasher_update(&self->hasher, buf, n);
261 if (feof(file)) {
262 break;
263 }
264 }
265
266 int fclose_ret = fclose(file);
267 file = NULL;
268 if (fclose_ret != 0) {
269 goto exit;
270 }
271
272 // Success. We need to increment the refcount on self to return it, see:
273 // https://docs.python.org/3/extending/extending.html#ownership-rules.
274 Py_INCREF(self);
275 ret = (PyObject *)self;
276
277 exit:
278 if (file) {
279 fclose(file);
280 }
281 Blake3_unlock_self_and_acquire_gil(self, &thread_state);
282 if (!ret) {
283 // XXX: The caller must hold the GIL to call PyErr_SetFromErrno, although
284 // this is not documented.
285 PyErr_SetFromErrno(PyExc_OSError);
286 }
287 Py_XDECREF(path_bytes);
288 return ret;
289 }
290
291 static PyObject *Blake3_digest(Blake3Object *self, PyObject *args,
292 PyObject *kwds) {
293 static char *kwlist[] = {
294 "length",
295 "seek",
296 NULL,
297 };
298 Py_ssize_t length = BLAKE3_OUT_LEN;
299 unsigned long long seek = 0;
300 if (!PyArg_ParseTupleAndKeywords(args, kwds, "|n$K", kwlist, &length,
301 &seek)) {
302 return NULL;
303 }
304 // Create a bytes object as per https://stackoverflow.com/a/55876332/823869.
305 PyObject *output = PyBytes_FromStringAndSize(NULL, length);
306 if (output == NULL) {
307 return NULL;
308 }
309
310 if (length >= HASHLIB_GIL_MINSIZE) {
311 PyThreadState *thread_state;
312 Blake3_release_gil_and_lock_self(self, &thread_state);
313 blake3_hasher_finalize_seek(&self->hasher, seek,
314 (uint8_t *)PyBytes_AsString(output), length);
315 Blake3_unlock_self_and_acquire_gil(self, &thread_state);
316 } else {
317 // Don't bother releasing the GIL for short outputs.
318 Blake3_lock_self(self);
319 blake3_hasher_finalize_seek(&self->hasher, seek,
320 (uint8_t *)PyBytes_AsString(output), length);
321 Blake3_unlock_self(self);
322 }
323
324 return output;
325 }
326
327 static PyObject *Blake3_hexdigest(Blake3Object *self, PyObject *args,
328 PyObject *kwds) {
329 PyObject *bytes = Blake3_digest(self, args, kwds);
330 if (bytes == NULL) {
331 return NULL;
332 }
333 PyObject *hex = PyObject_CallMethod(bytes, "hex", NULL);
334 Py_DECREF(bytes);
335 return hex;
336 }
337
338 // Implemented below, because it needs to refer to Blake3Type.
339 static PyObject *Blake3_copy(Blake3Object *self, PyObject *args);
340
341 static PyObject *Blake3_reset(Blake3Object *self, PyObject *args) {
342 Blake3_lock_self(self);
343 blake3_hasher_reset(&self->hasher);
344 Blake3_unlock_self(self);
345 Py_RETURN_NONE;
346 }
347
348 static PyMethodDef Blake3_methods[] = {
349 {"update", (PyCFunction)Blake3_update, METH_VARARGS, "add input bytes"},
350 {"update_mmap", (PyCFunction)Blake3_update_mmap,
351 METH_VARARGS | METH_KEYWORDS, "add input bytes from a filepath"},
352 {"digest", (PyCFunction)Blake3_digest,
353 METH_VARARGS | METH_KEYWORDS, "finalize the hash"},
354 {"hexdigest", (PyCFunction)Blake3_hexdigest,
355 METH_VARARGS | METH_KEYWORDS,
356 "finalize the hash and encode the result as hex"},
357 {"copy", (PyCFunction)Blake3_copy, METH_VARARGS,
358 "make a copy of this hasher"},
359 {"reset", (PyCFunction)Blake3_reset, METH_VARARGS,
360 "reset this hasher to its initial state"},
361 {NULL, NULL, 0, NULL} // sentinel
362 };
363
364 static PyTypeObject Blake3Type = {
365 // clang-format gets confused by the (correct, documented) missing
366 // semicolon after PyObject_HEAD_INIT.
367 // clang-format off
368 PyVarObject_HEAD_INIT(NULL, 0)
369 .tp_name = "blake3.blake3",
370 .tp_doc = "an incremental BLAKE3 hasher",
371 .tp_basicsize = sizeof(Blake3Object),
372 .tp_itemsize = 0,
373 .tp_flags = Py_TPFLAGS_DEFAULT,
374 .tp_new = Blake3_new,
375 .tp_dealloc = (destructor) Blake3_dealloc,
376 .tp_methods = Blake3_methods,
377 // clang-format on
378 };
379
380 // Declared above but implemented here, because it needs to refer to Blake3Type.
381 static PyObject *Blake3_copy(Blake3Object *self, PyObject *args) {
382 Blake3Object *copy = NULL;
383 PyThread_type_lock copy_lock = NULL;
384
385 PyObject *ret = NULL;
386
387 copy = PyObject_New(Blake3Object, &Blake3Type);
388 if (copy == NULL) {
389 goto exit;
390 }
391
392 copy_lock = PyThread_allocate_lock();
393 if (copy_lock == NULL) {
394 goto exit;
395 }
396
397 Blake3_lock_self(self);
398 memcpy(&copy->hasher, &self->hasher, sizeof(blake3_hasher));
399 copy->lock = copy_lock;
400 Blake3_unlock_self(self);
401
402 // success
403 ret = (PyObject *)copy;
404 copy = NULL; // pass the refcount
405 copy_lock = NULL; // pass ownership
406
407 exit:
408 Py_XDECREF(copy);
409 if (copy_lock != NULL) {
410 PyThread_free_lock(copy_lock);
411 }
412 return ret;
413 }
414
415 static struct PyModuleDef blake3module = {
416 PyModuleDef_HEAD_INIT,
417 .m_name = "blake3",
418 .m_doc = SETUP_PY_DESCRIPTION,
419 .m_size = -1,
420 };
421
422 PyMODINIT_FUNC PyInit_blake3(void) {
423 PyObject *attr_dict = NULL;
424 PyObject *name_str = NULL;
425 PyObject *block_size_int = NULL;
426 PyObject *digest_size_int = NULL;
427 PyObject *key_size_int = NULL;
428 PyObject *auto_int = NULL;
429 PyObject *module = NULL;
430
431 PyObject *ret = NULL;
432
433 // Feedback needed: Handling all these possible allocation failures is
434 // annoying and error-prone. Is this really necessary?
435 attr_dict = PyDict_New();
436 if (attr_dict == NULL) {
437 goto exit;
438 }
439 name_str = PyUnicode_FromString("blake3");
440 if (name_str == NULL) {
441 goto exit;
442 }
443 if (PyDict_SetItemString(attr_dict, "name", name_str) < 0) {
444 goto exit;
445 }
446 block_size_int = PyLong_FromLong(BLAKE3_BLOCK_LEN);
447 if (block_size_int == NULL) {
448 goto exit;
449 }
450 if (PyDict_SetItemString(attr_dict, "block_size", block_size_int) < 0) {
451 goto exit;
452 }
453 digest_size_int = PyLong_FromLong(BLAKE3_OUT_LEN);
454 if (digest_size_int == NULL) {
455 goto exit;
456 }
457 if (PyDict_SetItemString(attr_dict, "digest_size", digest_size_int) < 0) {
458 goto exit;
459 }
460 key_size_int = PyLong_FromLong(BLAKE3_KEY_LEN);
461 if (key_size_int == NULL) {
462 goto exit;
463 }
464 if (PyDict_SetItemString(attr_dict, "key_size", key_size_int) < 0) {
465 goto exit;
466 }
467 auto_int = PyLong_FromLong(AUTO);
468 if (auto_int == NULL) {
469 goto exit;
470 }
471 if (PyDict_SetItemString(attr_dict, "AUTO", auto_int) < 0) {
472 goto exit;
473 }
474
475 Blake3Type.tp_dict = attr_dict;
476 attr_dict = NULL; // pass the refcount
477
478 if (PyType_Ready(&Blake3Type) < 0) {
479 goto exit;
480 }
481
482 module = PyModule_Create(&blake3module);
483 if (module == NULL) {
484 goto exit;
485 }
486
487 if (PyModule_AddObject(module, "blake3", (PyObject *)&Blake3Type) < 0) {
488 Py_DECREF((PyObject *)&Blake3Type);
489 goto exit;
490 }
491
492 if (PyModule_AddStringConstant(module, "__version__", SETUP_PY_VERSION) < 0) {
493 goto exit;
494 }
495
496 // success
497 ret = module;
498 module = NULL; // pass the refcount
499
500 exit:
501 Py_XDECREF(attr_dict);
502 Py_XDECREF(name_str);
503 Py_XDECREF(block_size_int);
504 Py_XDECREF(digest_size_int);
505 Py_XDECREF(key_size_int);
506 Py_XDECREF(auto_int);
507 Py_XDECREF(module);
508 return ret;
509 }