Debian Codegraph @

src/lib.rs

raw
1 extern crate blake3 as upstream_blake3;
2
3 use pyo3::buffer::PyBuffer;
4 use pyo3::exceptions::{PyBufferError, PyOverflowError, PyValueError};
5 use pyo3::prelude::*;
6 use pyo3::types::{PyAny, PyBytes, PyString};
7 use std::path::PathBuf;
8 use std::sync::Mutex;
9
10 // This is the same as HASHLIB_GIL_MINSIZE in CPython.
11 const GIL_MINSIZE: usize = 2048;
12
13 // We want to support buffers of both signed and unsigned bytes, and for hashing
14 // purposes we'll pointer cast both to &[u8]. PyO3 gives us typed buffers, so we
15 // use this enum to wrap them.
16 enum BytesPyBuffer {
17 U8(PyBuffer<u8>),
18 I8(PyBuffer<i8>),
19 }
20
21 impl BytesPyBuffer {
22 /// Try to get a BytesPyBuffer from any Python object. This succeeds for
23 /// Python types like `bytes` and `bytearray` that support the buffer
24 /// protocol, but it fails for most other types.
25 fn get(data: &Bound<PyAny>) -> PyResult<Self> {
26 // First see if we can get a u8 buffer. This is the common case.
27 match PyBuffer::<u8>::get(data) {
28 Ok(pybuffer) => Ok(Self::U8(pybuffer)),
29 // If not, then see if we can get an i8 buffer.
30 Err(u8_err) => match PyBuffer::<i8>::get(data) {
31 Ok(pybuffer) => Ok(Self::I8(pybuffer)),
32 // That didn't work either. Return the first error from above.
33 // If they're different, the first one is more likely to be
34 // relevant to the caller.
35 Err(_) => Err(u8_err),
36 },
37 }
38 }
39
40 /// Get a &[u8] from a PyBuffer<u8> or a PyBuffer<i8>. This function is
41 /// unsafe, because the returned slice could be mutably aliased. See the
42 /// comments below.
43 unsafe fn as_bytes(&self) -> PyResult<&[u8]> {
44 match self {
45 // These arms look identical, but note that they call
46 // bytes_from_pybuffer with different type parameters. That's why
47 // the enum is needed.
48 Self::U8(pybuffer) => unsafe { bytes_from_pybuffer(pybuffer) },
49 Self::I8(pybuffer) => unsafe { bytes_from_pybuffer(pybuffer) },
50 }
51 }
52 }
53
54 unsafe fn bytes_from_pybuffer<T: pyo3::buffer::Element>(pybuffer: &PyBuffer<T>) -> PyResult<&[u8]> {
55 // This function lets us support both signed and unsigned byte arrays
56 // without copy-pasting too much code. Assert that we don't use it for
57 // anything larger than a byte. There's nothing necessarily evil about
58 // hashing an array of u32's, but it would raise endianness issues, and in
59 // any case we don't plan to do it.
60 assert_eq!(std::mem::size_of::<T>(), 1);
61
62 // Check that the buffer is contiguous. Regular bytes/bytearray types are
63 // always contiguous, but e.g. some NumPy arrays can be "strided", and we
64 // want to reject those. This is the same check that `PyBuffer::as_slice`
65 // does internally.
66 if !pybuffer.is_c_contiguous() {
67 return Err(PyBufferError::new_err("buffer is not contiguous"));
68 }
69
70 // Construct a &[u8] slice over the buffer. This is different from
71 // `PyBuffer::as_slice` in a couple ways:
72 //
73 // - We're potentially pointer-casting from &[i8] to &[u8]. This is fine,
74 // and there are safe APIs for this sort of thing (e.g. `bytemuck`).
75 // - `as_slice` returns `&[ReadOnlyCell<T>]` instead of `&[T]`, because the
76 // slice might be mutably aliased. We're exposing &[u8] directly. This is
77 // not obviously fine, and we need to be careful.
78 //
79 // There are several different types of issues that `PyBuffer` (which we're
80 // keeping) and `ReadOnlyCell` (which we're casting away) protect us from:
81 //
82 // - Someone might try to *resize* the buffer while we're reading it, making
83 // our reads UB. The `PyBuffer` "locks" the buffer to prevent this as long
84 // as we keep it alive. This slice is lifetime-bound to the `PyBuffer`
85 // once we return it, so we're ok here.
86 // - We might accidentally write to this buffer ourselves, by writing to a
87 // "different" buffer that turns out to be an alias, or by running any
88 // Python code we don't control that happens to do the same thing. We
89 // won't do any writes to buffers we didn't create internally, and we
90 // won't run any Python code (even finalizers) while this slice is alive,
91 // so I think we're ok here too. (We also won't use any invariant-checking
92 // functions like `std::str::from_utf8` that could get fooled by illegal
93 // mutable aliasing.)
94 // - Another thread might do the same thing, totally out of our control.
95 // In addition to problems that we can cause by doing this ourselves, this
96 // is also a *data race*, which is per se UB.
97 //
98 // The third problem above is the complicated one, because we need to think
99 // about the GIL:
100 //
101 // - The GIL normally blocks other Python threads from running while our
102 // Rust code is running. That would make this whole problem go away.
103 // - However, we don't *want* to block other Python threads while we hash
104 // large inputs, so like the C code in Python's `hashlib` module we
105 // release the GIL, which makes the problem come back.
106 // - Even if we didn't release the GIL, other Rust extensions or C
107 // extensions might release it, and then we might race with them.
108 // - Freethreaded builds of Python 3.13+ don't even have a GIL, so the
109 // question of whether anyone releases it is kind of irrelevant.
110 //
111 // I think the right way to think about this is to forget about the GIL and
112 // just ask what happens if another thread writes to the bytes that we're
113 // hashing, while we're hashing them. Clearly part of the story is that
114 // we'll read and hash "junk bytes" (some unpredictable mix of before and
115 // after, maybe even an "out of thin air" read under exotic circumstances).
116 // That much is expected, and it's almost certainly a bug that the caller
117 // need to fix on their end. But the more vexing question is, can anything
118 // *else* happen? Could this data race trigger "real" UB like corrupting
119 // arbitrary memory or executing an attacker's bitcoin mining virus?
120 //
121 // This is a thorny problem, and the relevant standards make no guarantees.
122 // However, it isn't just our problem. The C implementations in Python's
123 // `hashlib` have the exact same behavior, and you can trigger a real data
124 // race with standard types like this:
125 // https://gist.github.com/oconnor663/c69cb4dbffb9b13bbced3fe8ce2181ac.
126 // This data race violates the requirements of the C memory model also.
127 //
128 // At the end of the day, even if this race turns out to be exploitable
129 // (which I think is unlikely), only pathological programs should be able to
130 // trigger it. Writing to a buffer concurrently from another thread while
131 // hashing it is a very weird thing to do, and it's almost guaranteed to be
132 // a correctness bug, regardless of whether it's also a soundness bug.
133 unsafe {
134 Ok(std::slice::from_raw_parts(
135 pybuffer.buf_ptr() as *const u8,
136 pybuffer.len_bytes(),
137 ))
138 }
139 }
140
141 fn new_thread_pool(max_threads: usize) -> rayon::ThreadPool {
142 rayon::ThreadPoolBuilder::new()
143 .num_threads(max_threads)
144 .build()
145 .unwrap()
146 }
147
148 enum ThreadingMode {
149 Single,
150 Auto,
151 Pool {
152 pool: rayon::ThreadPool,
153 max_threads: usize,
154 },
155 }
156
157 impl Clone for ThreadingMode {
158 fn clone(&self) -> Self {
159 match self {
160 ThreadingMode::Single => ThreadingMode::Single,
161 ThreadingMode::Auto => ThreadingMode::Auto,
162 ThreadingMode::Pool { max_threads, .. } => ThreadingMode::Pool {
163 max_threads: *max_threads,
164 pool: new_thread_pool(*max_threads),
165 },
166 }
167 }
168 }
169
170 /// An incremental BLAKE3 hasher, which can accept any number of writes.
171 /// The interface is similar to `hashlib.blake2b` or `hashlib.md5` from the
172 /// standard library.
173 ///
174 /// Arguments:
175 /// - `data`: Input bytes to hash. Setting this to non-None is equivalent
176 /// to calling `update` on the returned hasher.
177 /// - `key`: A 32-byte key. Setting this to non-None enables the BLAKE3
178 /// keyed hashing mode.
179 /// - `derive_key_context`: A hardcoded, globally unique,
180 /// application-specific context string. Setting this to non-None enables
181 /// the BLAKE3 key derivation mode. `derive_key_context` and `key` cannot
182 /// be used at the same time.
183 /// - `max_threads`: The maximum number of threads that the implementation
184 /// may use for hashing. The default value is 1, meaning single-threaded.
185 /// `max_threads` may be any positive integer, or the value of the class
186 /// attribute `blake3.AUTO`, which lets the implementation use as many
187 /// threads as it likes. (Currently this means a number of threads equal
188 /// to the number of logical CPU cores, but this is not guaranteed.) The
189 /// actual number of threads used may be less than the maximum and may
190 /// change over time. API-compatible reimplementations of this library
191 /// may also ignore this parameter entirely, if they don't support
192 /// multithreading.
193 /// - `usedforsecurity`: Currently ignored. See the standard hashlib docs.
194 // Note: The "blake3.blake3.blake3" canonical path is a Maturin implementation detail. See
195 // https://github.com/mkdocstrings/mkdocstrings/issues/451 for why we expose it here. That means
196 // that both of these work today, though most callers should prefer the first one:
197 //
198 // # Import the re-exported blake3 class from the top-level module. This is stable. Do this.
199 // from blake3 import blake3
200 //
201 // # Import the blake3 class from its canonical path. Avoid this in regular code, because the
202 // # canonical path is an internal implementation detail, and it could change in the future.
203 // from blake3.blake3 import blake3
204 #[pyclass(name = "blake3", module = "blake3.blake3", frozen)]
205 struct Blake3Class {
206 // By default (currently), PyO3 pyclass objects use an atomic RefCell-like
207 // pattern to detect/prevent mutable aliasing. Rather than blocking, any
208 // mutably aliasing accesses instead raises an exception. Normally the GIL
209 // prevents this, but we release the GIL to hash long inputs, so ordinary
210 // races between Python threads can trigger this. And freethreaded Python
211 // builds have no GIL, so under those you can trigger this with any input
212 // length.
213 //
214 // Now, we *could* decide that that's fine. Racing to update a hasher gives
215 // you nondeterministic outputs, and it's hard to imagine a use case where
216 // that's not a bug. However, someone using BLAKE3 as a CSPRNG might reseed
217 // it periodically, and they might not care if other callers race with that.
218 // Also, reliably raising exceptions is defensible, but *occasionally*
219 // raising them is terrible. Users' tests will pass, and then their apps
220 // will crash in prod.
221 //
222 // Instead, we declare this class "frozen" above, meaning that only shared
223 // access is allowed. Then we use a Mutex internally to let us mutate the
224 // Hasher. This means that users will never see exceptions about mutable
225 // borrowing, and the PyO3 docs mention that they want to push the ecosystem
226 // in this direction. See: https://pyo3.rs/main/class.html#frozen-classes-opting-out-of-interior-mutability
227 rust_hasher: Mutex<upstream_blake3::Hasher>,
228 threading_mode: ThreadingMode,
229 }
230
231 #[pymethods]
232 impl Blake3Class {
233 /// The lowercase name of this hashing algorithm, "blake3".
234 #[classattr]
235 #[allow(non_upper_case_globals)]
236 const name: &'static str = "blake3";
237
238 /// The default size of the resulting hash in bytes, 32.
239 #[classattr]
240 #[allow(non_upper_case_globals)]
241 const digest_size: usize = 32;
242
243 /// The internal block size in bytes, 64.
244 #[classattr]
245 #[allow(non_upper_case_globals)]
246 const block_size: usize = 64;
247
248 /// The key size in bytes, 32.
249 #[classattr]
250 #[allow(non_upper_case_globals)]
251 const key_size: usize = 32;
252
253 /// Used as a `max_threads` value, to let the implementation choose the number of threads.
254 ///
255 /// This currently uses a number of threads equal to the number of logical cores, but that
256 /// behavior could change in the future.
257 #[classattr]
258 const AUTO: isize = -1;
259
260 #[new]
261 #[pyo3(signature = (
262 data = None,
263 /,
264 *,
265 key = None,
266 derive_key_context = None,
267 max_threads = 1,
268 usedforsecurity = true
269 ))]
270 fn new<'py>(
271 py: Python<'py>,
272 data: Option<&Bound<'py, PyAny>>,
273 key: Option<&Bound<'py, PyAny>>,
274 derive_key_context: Option<&str>,
275 max_threads: isize,
276 usedforsecurity: bool,
277 ) -> PyResult<Blake3Class> {
278 let _ = usedforsecurity; // currently ignored
279
280 let mut rust_hasher = match (key, derive_key_context) {
281 // The default, unkeyed hash function.
282 (None, None) => upstream_blake3::Hasher::new(),
283 // The keyed hash function.
284 (Some(key_obj), None) => {
285 // Use the same `as_bytes` helper function to get the key buffer
286 // as `update` uses to get the data buffer. Even though we just
287 // copy the bytes immediately here, technically this risks the
288 // same race conditions.
289 let key_buf = BytesPyBuffer::get(key_obj)?;
290 let key_slice: &[u8] = unsafe { key_buf.as_bytes()? };
291 let key_array: &[u8; 32] = if let Ok(array) = key_slice.try_into() {
292 array
293 } else {
294 let msg = format!("expected a {}-byte key, found {}", 32, key_slice.len());
295 return Err(PyValueError::new_err(msg));
296 };
297 upstream_blake3::Hasher::new_keyed(key_array)
298 }
299 // The key derivation function.
300 (None, Some(context)) => upstream_blake3::Hasher::new_derive_key(context),
301 // Error: can't use both modes at the same time.
302 (Some(_), Some(_)) => {
303 return Err(PyValueError::new_err(
304 "cannot use key and derive_key_context at the same time",
305 ));
306 }
307 };
308
309 let threading_mode = match max_threads {
310 1 => ThreadingMode::Single,
311 Self::AUTO => ThreadingMode::Auto,
312 n if n > 1 => ThreadingMode::Pool {
313 max_threads: n as usize,
314 pool: new_thread_pool(n as usize),
315 },
316 _ => return Err(PyValueError::new_err("not a valid number of threads")),
317 };
318
319 if let Some(data_obj) = data {
320 // XXX: Get a &[u8] slice of the data bytes. The safety situation
321 // here is complicated. See all the comments in bytes_from_pybuffer.
322 let data_buf = BytesPyBuffer::get(data_obj)?;
323 let data_slice: &[u8] = unsafe { data_buf.as_bytes()? };
324
325 // Since rust_hasher isn't yet shared, we don't need to access it
326 // through the Mutex here like we do in update() below.
327 let mut update_closure = || match &threading_mode {
328 ThreadingMode::Single => {
329 rust_hasher.update(data_slice);
330 }
331 ThreadingMode::Auto => {
332 rust_hasher.update_rayon(data_slice);
333 }
334 ThreadingMode::Pool { pool, .. } => pool.install(|| {
335 rust_hasher.update_rayon(data_slice);
336 }),
337 };
338
339 if data_slice.len() >= GIL_MINSIZE {
340 // Release the GIL while we hash this slice, so that we don't
341 // block other threads. But again, see all the comments above
342 // about data race risks.
343 py.detach(update_closure);
344 } else {
345 // Don't bother releasing the GIL for short updates.
346 update_closure();
347 }
348 }
349
350 Ok(Blake3Class {
351 rust_hasher: Mutex::new(rust_hasher),
352 threading_mode,
353 })
354 }
355
356 /// Add input bytes to the hasher. You can call this any number of
357 /// times.
358 ///
359 /// Arguments:
360 /// - `data` (required): The input bytes.
361 #[pyo3(signature=(data, /))]
362 fn update<'py>(
363 this: Bound<'py, Self>,
364 py: Python,
365 data: &Bound<PyAny>,
366 ) -> PyResult<Bound<'py, Self>> {
367 let self_ = this.get();
368
369 // XXX: Get a &[u8] slice of the data bytes. The safety situation here
370 // is complicated. See all the comments in bytes_from_pybuffer.
371 let data_buf = BytesPyBuffer::get(data)?;
372 let data_slice: &[u8] = unsafe { data_buf.as_bytes()? };
373
374 let update_closure = || match &self_.threading_mode {
375 ThreadingMode::Single => {
376 self_.rust_hasher.lock().unwrap().update(data_slice);
377 }
378 ThreadingMode::Auto => {
379 self_.rust_hasher.lock().unwrap().update_rayon(data_slice);
380 }
381 ThreadingMode::Pool { pool, .. } => pool.install(|| {
382 self_.rust_hasher.lock().unwrap().update_rayon(data_slice);
383 }),
384 };
385
386 if data_slice.len() >= GIL_MINSIZE {
387 // Release the GIL while we hash this slice, so that we don't
388 // block other threads. But again, see all the comments above
389 // about data race risks.
390 py.detach(update_closure);
391 } else {
392 // Don't bother releasing the GIL for short updates.
393 update_closure();
394 }
395
396 Ok(this)
397 }
398
399 /// Read a file using memory mapping and add its bytes to the hasher. You can call this any
400 /// number of times.
401 ///
402 /// Arguments:
403 /// - `path` (required): The filepath to read.
404 #[pyo3(signature=(path))]
405 fn update_mmap<'py>(
406 this: Bound<'py, Self>,
407 py: Python,
408 path: PathBuf,
409 ) -> PyResult<Bound<'py, Self>> {
410 let self_ = this.get();
411
412 py.detach(|| -> PyResult<()> {
413 match &self_.threading_mode {
414 ThreadingMode::Single => {
415 self_.rust_hasher.lock().unwrap().update_mmap(&path)?;
416 }
417 ThreadingMode::Auto => {
418 self_.rust_hasher.lock().unwrap().update_mmap_rayon(&path)?;
419 }
420 ThreadingMode::Pool { pool, .. } => {
421 pool.install(|| -> PyResult<()> {
422 self_.rust_hasher.lock().unwrap().update_mmap_rayon(&path)?;
423 Ok(())
424 })?;
425 }
426 }
427 Ok(())
428 })?;
429 Ok(this)
430 }
431
432 /// Return a copy (“clone”) of the hasher. This can be used to
433 /// efficiently compute the digests of data sharing a common initial
434 /// substring.
435 #[pyo3(signature=())]
436 fn copy(&self) -> Blake3Class {
437 Blake3Class {
438 rust_hasher: Mutex::new(self.rust_hasher.lock().unwrap().clone()),
439 threading_mode: self.threading_mode.clone(),
440 }
441 }
442
443 /// Reset the hasher to its initial empty state. If the hasher contains
444 /// an internal threadpool (as it currently does if `max_threads` is
445 /// greater than 1), resetting the hasher lets you reuse that pool.
446 /// Note that if any input bytes were supplied in the original
447 /// construction of the hasher, those bytes are *not* replayed.
448 #[pyo3(signature=())]
449 fn reset(&self) {
450 self.rust_hasher.lock().unwrap().reset();
451 }
452
453 /// Finalize the hasher and return the resulting hash as bytes. This
454 /// does not modify the hasher, and calling it twice will give the same
455 /// result. You can also add more input and finalize again.
456 ///
457 /// Arguments:
458 /// - `length`: The number of bytes in the final hash. BLAKE3 supports
459 /// any output length up to 2**64-1. Note that shorter outputs are
460 /// prefixes of longer ones. Defaults to 32.
461 /// - `seek`: The starting byte position in the output stream. Defaults
462 /// to 0.
463 #[pyo3(signature=(length=32, *, seek=0))]
464 fn digest<'p>(&self, py: Python<'p>, length: usize, seek: u64) -> PyResult<Bound<'p, PyBytes>> {
465 if length > isize::MAX as usize {
466 return Err(PyOverflowError::new_err("length overflows isize"));
467 }
468 let mut reader = self.rust_hasher.lock().unwrap().finalize_xof();
469 reader.set_position(seek);
470 PyBytes::new_with(py, length, |slice| {
471 debug_assert_eq!(length, slice.len());
472 if length >= GIL_MINSIZE {
473 // This could be a long-running operation. Release the GIL.
474 py.detach(|| reader.fill(slice));
475 } else {
476 // Don't bother releasing the GIL for short outputs.
477 reader.fill(slice);
478 }
479 Ok(())
480 })
481 }
482
483 /// Finalize the hasher and return the resulting hash as a hexadecimal
484 /// string. This does not modify the hasher, and calling it twice will
485 /// give the same result. You can also add more input and finalize
486 /// again.
487 ///
488 /// Arguments:
489 /// - `length`: The number of bytes in the final hash, prior to hex
490 /// encoding. BLAKE3 supports any output length up to 2**64-1. Note
491 /// that shorter outputs are prefixes of longer ones. Defaults to 32.
492 /// - `seek`: The starting byte position in the output stream, prior to
493 /// hex encoding. Defaults to 0.
494 #[pyo3(signature=(length=32, *, seek=0))]
495 fn hexdigest<'p>(
496 &self,
497 py: Python<'p>,
498 length: usize,
499 seek: u64,
500 ) -> PyResult<Bound<'p, PyString>> {
501 if length > (isize::MAX / 2) as usize {
502 return Err(PyOverflowError::new_err("length overflows isize"));
503 }
504 let bytes = self.digest(py, length, seek)?;
505 let hex = hex::encode(bytes.as_bytes());
506 Ok(PyString::new(py, &hex))
507 }
508 }
509
510 /// Python bindings for the official Rust implementation of BLAKE3
511 /// (https://github.com/BLAKE3-team/BLAKE3). This module provides a single
512 /// class, also called `blake3.` The interface is similar to `hashlib` from
513 /// the standard library, which provides `blake2b`, `md5`, etc.
514 #[pymodule(gil_used = false)]
515 fn blake3(_: Python, m: &Bound<PyModule>) -> PyResult<()> {
516 m.add_class::<Blake3Class>()?;
517 m.add("__version__", env!("CARGO_PKG_VERSION"))?;
518 Ok(())
519 }