Debian Codegraph @

rust/hg-pyo3/src/utils.rs

raw
1 use std::path::Path;
2
3 use hg::errors::HgError;
4 use hg::revlog::index::Index as CoreIndex;
5 use hg::revlog::inner_revlog::RevisionBuffer;
6 use hg::warnings::format::write_warning;
7 use hg::warnings::HgWarningContext;
8 use pyo3::buffer::Element;
9 use pyo3::buffer::PyBuffer;
10 use pyo3::exceptions::PyIOError;
11 use pyo3::exceptions::PyKeyboardInterrupt;
12 use pyo3::exceptions::PyValueError;
13 use pyo3::intern;
14 use pyo3::prelude::*;
15 use pyo3::types::PyBytes;
16 use pyo3::types::PyDict;
17 use pyo3::types::PyList;
18 use pyo3_sharedref::SharedByPyObject;
19 use stable_deref_trait::StableDeref;
20
21 use crate::exceptions::FallbackError;
22 use crate::revlog::InnerRevlog;
23 use crate::revlog::PySharedIndex;
24
25 #[allow(unused)]
26 pub fn print_python_trace(py: Python) -> PyResult<Py<PyAny>> {
27 eprintln!("===============================");
28 eprintln!("Printing Python stack from Rust");
29 eprintln!("===============================");
30 let traceback = py.import("traceback")?;
31 let sys = py.import("sys")?;
32 let kwargs = PyDict::new(py);
33 kwargs.set_item("file", sys.getattr("stderr")?)?;
34 traceback.call_method("print_stack", (), Some(&kwargs)).map(|r| r.unbind())
35 }
36
37 /// Create the module, with `__package__` given from parent
38 ///
39 /// According to PyO3 documentation, which links to
40 /// <https://github.com/PyO3/pyo3/issues/1517>, the convoluted write to
41 /// `sys.modules` has to be made.
42 pub(crate) fn new_submodule<'py>(
43 py: Python<'py>,
44 package_name: &str,
45 name: &str,
46 ) -> PyResult<Bound<'py, PyModule>> {
47 let dotted_name = &format!("{}.{}", package_name, name);
48 let m = PyModule::new(py, name)?;
49 m.add("__package__", package_name)?;
50
51 let sys = PyModule::import(py, "sys")?;
52 // according to the doc, we could make a static PyString out of
53 // "modules" with the `intern!` macro, but this is used only at
54 // registration so it may not be worth the effort.
55 let sys_modules: Bound<'_, PyDict> = sys.getattr("modules")?.extract()?;
56 sys_modules.set_item(dotted_name, &m)?;
57 // Example C code (see pyexpat.c and import.c) will "give away the
58 // reference", but we won't because it will be consumed once the
59 // Rust PyObject is dropped.
60 Ok(m)
61 }
62
63 /// Retrieve the shared index wrapper (which contains the core index)
64 /// from the Python index proxy.
65 pub fn py_rust_index_to_graph(
66 index_proxy: &Bound<'_, PyAny>,
67 ) -> PyResult<SharedByPyObject<PySharedIndex>> {
68 let py_irl = index_proxy.getattr("inner")?;
69 let py_irl_ref = py_irl.cast::<InnerRevlog>()?.borrow();
70 let shareable_irl = &py_irl_ref.irl;
71
72 // Safety: the owner is the actual one and we do not leak any
73 // internal reference.
74 let index =
75 unsafe { shareable_irl.share_map(&py_irl, |irl| (&irl.index).into()) };
76 Ok(index)
77 }
78
79 /// Error propagation for an [`SharedByPyObject`] wrapping a [`Result`]
80 ///
81 /// It would be nice for [`SharedByPyObject`] to provide this directly as
82 /// a variant of the `map` method with a signature such as:
83 ///
84 /// ```
85 /// unsafe fn map_or_err(&self,
86 /// py: Python,
87 /// f: impl FnOnce(T) -> Result(U, E),
88 /// convert_err: impl FnOnce(E) -> PyErr)
89 /// ```
90 ///
91 /// This would spare users of the `pyo3` crate the additional `unsafe` deref
92 /// to inspect the error and return it outside `SharedByPyObject`, and the
93 /// subsequent unwrapping that this function performs.
94 pub(crate) fn py_shared_or_map_err<T, E: std::fmt::Debug + Copy>(
95 py: Python,
96 leaked: SharedByPyObject<Result<T, E>>,
97 convert_err: impl FnOnce(E) -> PyErr,
98 ) -> PyResult<SharedByPyObject<T>> {
99 // Safety: we don't leak the "faked" reference out of `SharedByPyObject`
100 if let Err(e) = *unsafe { leaked.try_borrow(py)? } {
101 return Err(convert_err(e));
102 }
103 // Safety: we don't leak the "faked" reference out of `SharedByPyObject`
104 Ok(unsafe {
105 leaked.map(py, |res| {
106 res.expect("Error case should have already be treated")
107 })
108 })
109 }
110
111 /// Full extraction of the proxy index object as received in PyO3 to a
112 /// [`CoreIndex`] reference.
113 ///
114 /// # Safety
115 ///
116 /// The invariants to maintain are those of the underlying
117 /// [`SharedByPyObject::try_borrow`]: the caller must not leak the inner
118 /// reference.
119 pub(crate) unsafe fn proxy_index_extract<'py>(
120 index_proxy: &Bound<'py, PyAny>,
121 ) -> PyResult<&'py CoreIndex> {
122 let py_shared = py_rust_index_to_graph(index_proxy)?;
123 let py_shared = &*unsafe { py_shared.try_borrow(index_proxy.py())? };
124 Ok(unsafe { py_shared.static_inner() })
125 }
126
127 /// Type shortcut for the kind of bytes slice trait objects that are used in
128 /// particular for mmap data
129 type BoxedBytesSlice =
130 Box<dyn std::ops::Deref<Target = [u8]> + Send + Sync + 'static>;
131
132 /// Take a Python object backed by a Python buffer, and return the underlying
133 /// [`PyBuffer`] along with the Rust slice into said buffer.
134 ///
135 /// The caller needs to make sure that the Python buffer is not freed before
136 /// the slice, otherwise we'd get a dangling pointer once the incoming
137 /// object is freed from Python side. This can be achieved by storing it a
138 /// Python object.
139 ///
140 /// The typical use case is to extract mmap data to make it useable in the
141 /// constructs from the `hg` crate.
142 ///
143 /// # Safety
144 ///
145 /// The caller must make sure that the incoming Python object is kept around
146 /// for at least as long as the returned [`BoxedBytesSlice`].
147 // TODO in PyO3, we already get a reference with two lifetimes, and we
148 // could even take a `Borrowed<'a, 'py, T>`.
149 // So perhaps we could tie everything together with a lifetime so that is
150 // is, after all, safe, and this could be called something like `share_buffer`.
151 #[deny(unsafe_op_in_unsafe_fn)]
152 pub unsafe fn take_buffer_with_slice(
153 data: &Bound<'_, PyAny>,
154 ) -> PyResult<(PyBuffer<u8>, BoxedBytesSlice)> {
155 let buf = PyBuffer::<u8>::get(data)?;
156 let len = buf.item_count();
157
158 // Build a slice from the buffer data
159 let cbuf = buf.buf_ptr();
160 let bytes = if std::mem::size_of::<u8>() == buf.item_size()
161 && buf.is_c_contiguous()
162 && u8::is_compatible_format(buf.format())
163 && buf.dimensions() == 1
164 && buf.readonly()
165 {
166 unsafe { std::slice::from_raw_parts(cbuf as *const u8, len) }
167 } else {
168 return Err(PyValueError::new_err(
169 "buffer has an invalid memory representation",
170 ));
171 };
172
173 Ok((buf, Box::new(bytes)))
174 }
175
176 /// Takes an initialization function `init` which writes bytes to a
177 /// Python-backed buffer, to save on a (potentially large) memory allocation
178 /// and copy. If `init` fails to write the full expected length `len`, an error
179 /// is raised.
180 pub fn with_pybytes_buffer<F>(
181 py: Python,
182 len: usize,
183 init: F,
184 ) -> Result<Py<PyBytes>, hg::revlog::RevlogError>
185 where
186 F: FnOnce(
187 &mut dyn RevisionBuffer<Target = Py<PyBytes>>,
188 ) -> Result<(), hg::revlog::RevlogError>,
189 {
190 // Largely inspired by code in PyO3
191 // https://pyo3.rs/main/doc/pyo3/types/struct.pybytes#method.new_bound_with
192 unsafe {
193 let pyptr = pyo3::ffi::PyBytes_FromStringAndSize(
194 std::ptr::null(),
195 len as pyo3::ffi::Py_ssize_t,
196 );
197 let pybytes = Bound::from_owned_ptr_or_err(py, pyptr)
198 .map_err(|e| HgError::abort_simple(e.to_string()))?
199 .cast_into_unchecked();
200 let buffer: *mut u8 = pyo3::ffi::PyBytes_AsString(pyptr).cast();
201 debug_assert!(!buffer.is_null());
202 let mut rev_buf = PyRevisionBuffer::new(pybytes.unbind(), buffer, len);
203 // Initialise the bytestring in init
204 // If init returns an Err, the buffer is deallocated by `pybytes`
205 init(&mut rev_buf).map(|_| rev_buf.finish())
206 }
207 }
208
209 /// Safe abstraction over a `PyBuffer` together with the `&[u8]` slice
210 /// that borrows it. Implements `Deref<Target = [u8]>`.
211 ///
212 /// Calling `PyBuffer::data` requires a GIL marker but we want to access the
213 /// data in a thread that (ideally) does not need to acquire the GIL.
214 /// This type allows separating the call an the use.
215 ///
216 /// It also enables using a (wrapped) `PyBuffer` in GIL-unaware generic code.
217 pub struct PyBufferDeref {
218 #[allow(unused)]
219 keep_alive: PyBuffer<u8>,
220
221 /// Borrows the buffer inside `self.keep_alive`,
222 /// but the borrow-checker cannot express self-referential structs.
223 data: *const [u8],
224 }
225
226 #[deny(unsafe_op_in_unsafe_fn)]
227 /// Only safe if the buffers are read-only on the Pytho side
228 /// Marking this unsafe so we can think about it
229 unsafe fn get_buffer(buf: &PyBuffer<u8>) -> PyResult<*const [u8]> {
230 let len = buf.item_count();
231
232 // Build a slice from the mmap'ed buffer data
233 let cbuf = buf.buf_ptr();
234 let bytes = if std::mem::size_of::<u8>() == buf.item_size()
235 && buf.readonly()
236 && buf.is_c_contiguous()
237 && u8::is_compatible_format(buf.format())
238 {
239 unsafe { std::slice::from_raw_parts(cbuf as *const u8, len) }
240 } else {
241 return Err(PyValueError::new_err(
242 "Chunk buffer has an invalid memory representation",
243 ));
244 };
245 Ok(bytes)
246 }
247
248 impl PyBufferDeref {
249 pub fn new(buf: PyBuffer<u8>) -> Self {
250 Self { data: unsafe { get_buffer(&buf).unwrap() }, keep_alive: buf }
251 }
252 }
253
254 impl std::ops::Deref for PyBufferDeref {
255 type Target = [u8];
256
257 fn deref(&self) -> &[u8] {
258 // Safety: the raw pointer is valid as long as the PyBuffer is still
259 // alive, and the returned slice borrows `self`.
260 unsafe { &*self.data }
261 }
262 }
263
264 unsafe impl StableDeref for PyBufferDeref {}
265
266 #[allow(unused)]
267 fn static_assert_pybuffer_is_send() {
268 #[allow(clippy::no_effect)]
269 require_send::<PyBuffer<u8>>;
270 }
271
272 // Safety: PyBuffer is Send. Raw pointers are not by default,
273 // but here sending one to another thread is fine since we ensure it stays
274 // valid.
275 unsafe impl Send for PyBufferDeref {}
276
277 /// Safe abstraction over a `PyBytes` together with the `&[u8]` slice
278 /// that borrows it. Implements `Deref<Target = [u8]>`.
279 ///
280 /// Calling `PyBytes::data` requires a GIL marker but we want to access the
281 /// data in a thread that (ideally) does not need to acquire the GIL.
282 /// This type allows separating the call an the use.
283 ///
284 /// It also enables using a (wrapped) `PyBytes` in GIL-unaware generic code.
285 pub struct PyBytesDeref {
286 #[allow(unused)]
287 keep_alive: Py<PyBytes>,
288
289 /// Borrows the buffer inside `self.keep_alive`,
290 /// but the borrow-checker cannot express self-referential structs.
291 data: &'static [u8],
292 }
293
294 impl PyBytesDeref {
295 pub fn new(py: Python, bytes: Py<PyBytes>) -> Self {
296 let as_raw: *const [u8] = bytes.as_bytes(py);
297 Self {
298 // Safety: the raw pointer is valid as long as the PyBytes is still
299 // alive, and the objecs owns it.
300 data: unsafe { &*as_raw },
301 keep_alive: bytes,
302 }
303 }
304
305 #[allow(unused)]
306 pub fn unwrap(self) -> Py<PyBytes> {
307 self.keep_alive
308 }
309 }
310
311 impl std::ops::Deref for PyBytesDeref {
312 type Target = [u8];
313
314 fn deref(&self) -> &[u8] {
315 self.data
316 }
317 }
318
319 unsafe impl StableDeref for PyBytesDeref {}
320
321 fn require_send<T: Send>() {}
322
323 #[allow(unused)]
324 fn static_assert_pybytes_is_send() {
325 #[allow(clippy::no_effect)]
326 require_send::<Py<PyBytes>>;
327 }
328
329 // Safety: `[Py<PyBytes>]` is Send. Raw pointers are not by default,
330 // but here sending one to another thread is fine since we ensure it stays
331 // valid.
332 unsafe impl Send for PyBytesDeref {}
333
334 /// Wrapper around a Python-provided buffer into which the revision contents
335 /// will be written. Done for speed in order to save a large allocation + copy.
336 struct PyRevisionBuffer {
337 py_bytes: Py<PyBytes>,
338 _buf: *mut u8,
339 len: usize,
340 current_buf: *mut u8,
341 current_len: usize,
342 }
343
344 impl PyRevisionBuffer {
345 /// # Safety
346 ///
347 /// `buf` should be the start of the allocated bytes of `bytes`, and `len`
348 /// exactly the length of said allocated bytes.
349 #[inline]
350 unsafe fn new(bytes: Py<PyBytes>, buf: *mut u8, len: usize) -> Self {
351 Self {
352 py_bytes: bytes,
353 _buf: buf,
354 len,
355 current_len: 0,
356 current_buf: buf,
357 }
358 }
359
360 /// Number of bytes that have been copied to. Will be different to the
361 /// total allocated length of the buffer unless the revision is done being
362 /// written.
363 #[inline]
364 fn current_len(&self) -> usize {
365 self.current_len
366 }
367 }
368
369 impl RevisionBuffer for PyRevisionBuffer {
370 type Target = Py<PyBytes>;
371
372 #[inline]
373 fn extend_from_slice(&mut self, slice: &[u8]) {
374 assert!(self.current_len + slice.len() <= self.len);
375 unsafe {
376 // We cannot use `copy_from_nonoverlapping` since it's *possible*
377 // to create a slice from the same Python memory region using
378 // [`PyBytesDeref`]. Probable that LLVM has an optimization anyway?
379 self.current_buf.copy_from(slice.as_ptr(), slice.len());
380 self.current_buf = self.current_buf.add(slice.len());
381 }
382 self.current_len += slice.len()
383 }
384
385 #[inline]
386 fn finish(self) -> Self::Target {
387 // catch unzeroed bytes before it becomes undefined behavior
388 assert_eq!(
389 self.current_len(),
390 self.len,
391 "not enough bytes read for revision"
392 );
393 self.py_bytes
394 }
395 }
396
397 /// Extension trait to help with generic error conversions from hg-core to
398 /// Python.
399 pub(crate) trait HgPyErrExt<T> {
400 fn into_pyerr(self, py: Python) -> PyResult<T>;
401 }
402
403 impl<T, E> HgPyErrExt<T> for Result<T, E>
404 where
405 HgError: From<E>,
406 {
407 fn into_pyerr(self, py: Python) -> PyResult<T> {
408 self.map_err(|e| match e.into() {
409 err @ HgError::IoError { .. } => {
410 PyIOError::new_err(err.to_string())
411 }
412 err @ HgError::UnsupportedFeature(..) => {
413 FallbackError::new_err(err.to_string())
414 }
415 HgError::RaceDetected(_) => {
416 unreachable!("must not surface to the user")
417 }
418 HgError::Path(path_error) => {
419 let msg = PyBytes::new(py, path_error.to_string().as_bytes());
420 let cls = py
421 .import(intern!(py, "mercurial.error"))
422 .and_then(|m| m.getattr(intern!(py, "InputError")))
423 .expect("failed to import InputError");
424 PyErr::from_value(
425 cls.call1((msg,))
426 .expect("initializing an InputError failed"),
427 )
428 }
429 HgError::InterruptReceived => PyKeyboardInterrupt::new_err(()),
430 err @ HgError::Abort { detailed_exit_code, .. } => {
431 let cls = py
432 .import(intern!(py, "mercurial.error"))
433 .and_then(|m| match detailed_exit_code {
434 hg::exit_codes::STATE_ERROR => {
435 m.getattr(intern!(py, "StateError"))
436 }
437 hg::exit_codes::CONFIG_ERROR_ABORT => {
438 m.getattr(intern!(py, "ConfigError"))
439 }
440 // TODO more errors
441 _ => m.getattr(intern!(py, "Abort")),
442 })
443 .expect("failed to import error.Abort");
444 PyErr::from_value(
445 cls.call1((err.to_string().as_bytes(),))
446 .expect("initializing an error.Abort failed"),
447 )
448 }
449 e => {
450 let cls = py
451 .import(intern!(py, "mercurial.error"))
452 .and_then(|m| m.getattr(intern!(py, "Abort")))
453 .expect("failed to import error.Abort");
454 PyErr::from_value(
455 cls.call1((e.to_string().as_bytes(),))
456 .expect("initializing an error.Abort failed"),
457 )
458 }
459 })
460 }
461 }
462
463 /// Wrap a call to `func` so that Python's `SIGINT` handler is first stored,
464 /// then restored after the call to `func` and finally raised if
465 /// `func` returns a [`HgError::InterruptReceived`].
466 ///
467 /// We cannot use [`Python::check_signals`] because it only works from the main
468 /// thread of the main interpreter. To that end, long-running Rust functions
469 /// need to cooperate by listening to their own `SIGINT` signal and return
470 /// the appropriate error on catching that signal: this is especially helpful
471 /// in multithreaded operations.
472 pub fn with_sigint_wrapper<R>(
473 py: Python,
474 func: impl FnOnce() -> Result<R, HgError>,
475 ) -> PyResult<Result<R, HgError>> {
476 let threading_py_mod = py.import(intern!(py, "threading"))?;
477 let current_py_thread =
478 threading_py_mod.call_method0(intern!(py, "current_thread"))?;
479 let main_py_thread =
480 threading_py_mod.call_method0(intern!(py, "main_thread"))?;
481 if !current_py_thread.is(&main_py_thread) {
482 tracing::debug!(
483 "not running in the main Python thread, signal handling skipped"
484 );
485 return Ok(func());
486 }
487
488 let signal_py_mod = py.import(intern!(py, "signal"))?;
489 let sigint_py_const = signal_py_mod.getattr(intern!(py, "SIGINT"))?;
490 let old_handler = signal_py_mod
491 .call_method1(intern!(py, "getsignal"), (sigint_py_const.clone(),))?;
492 let res = func();
493 // Reset the old signal handler in Python because we may have changed it
494 signal_py_mod.call_method1(
495 intern!(py, "signal"),
496 (sigint_py_const.clone(), old_handler),
497 )?;
498 if let Err(HgError::InterruptReceived) = res {
499 // Trigger the signal in Python
500 signal_py_mod
501 .call_method1(intern!(py, "raise_signal"), (sigint_py_const,))?;
502 }
503 Ok(res)
504 }
505
506 /// Transform hg-core warnings to Python bytes
507 pub fn hg_warnings_to_py_warnings(
508 py: Python<'_>,
509 warnings: HgWarningContext,
510 root_dir: &Path,
511 ) -> Result<Py<PyList>, PyErr> {
512 let py_warnings = PyList::empty(py);
513 warnings.finish(|w| {
514 let mut buf = vec![];
515 write_warning(&w, &mut buf, root_dir)?;
516 py_warnings.append(PyBytes::new(py, &buf))
517 })?;
518 Ok(py_warnings.unbind())
519 }
520
521 /// Call the warnings callback from Python with translated warnings
522 pub fn handle_warnings(
523 py: Python<'_>,
524 warning_context: HgWarningContext,
525 working_directory: &Path,
526 on_warnings: Py<PyAny>,
527 ) -> PyResult<()> {
528 let py_warnings =
529 hg_warnings_to_py_warnings(py, warning_context, working_directory)?;
530 on_warnings.call1(py, (py_warnings,))?;
531 Ok(())
532 }