Debian Codegraph @

rust/hg-pyo3/src/file_index.rs

raw
1 //! Bindings for the `hg::file_index` module provided by the `hg-core` package.
2 //!
3 //! From Python, this will be seen as `mercurial.pyo3_rustext.file_index`
4
5 use std::sync::atomic::AtomicBool;
6 use std::sync::atomic::Ordering;
7 use std::sync::RwLockReadGuard;
8 use std::sync::RwLockWriteGuard;
9
10 use hg::file_index::DebugTreeChild;
11 use hg::file_index::DebugTreeNode;
12 use hg::file_index::DebugTreeNodeIter;
13 use hg::file_index::FileIndex;
14 use hg::file_index::FileToken;
15 use hg::file_index::PathTokenIter;
16 use hg::file_index::VacuumMode;
17 use hg::revlog::path_encode::PathEncoding;
18 use hg::utils::files::get_bytes_from_path;
19 use hg::utils::files::get_path_from_bytes;
20 use hg::utils::hg_path::HgPath;
21 use hg::utils::u32_u;
22 use hg::vfs::VfsImpl;
23 use pyo3::exceptions::PyOSError;
24 use pyo3::prelude::*;
25 use pyo3::types::PyBytes;
26 use pyo3::types::PyDict;
27 use pyo3::types::PyList;
28 use pyo3::types::PyTuple;
29 use pyo3_sharedref::py_shared_iterator;
30 use pyo3_sharedref::PyShareable;
31
32 use crate::exceptions::map_try_lock_error;
33 use crate::exceptions::to_string_value_error;
34 use crate::transaction::PyTransaction;
35 use crate::utils::new_submodule;
36
37 /// Implementation of `mercurial.interfaces.IFileIndex`.
38 /// This is a drop-in replacement for `mercurial.store_utils.FileIndex`.
39 #[pyclass(name = "FileIndex", mapping, frozen)]
40 struct PyFileIndex {
41 inner: PyShareable<FileIndex>,
42 /// True if we've already called [`Self::_add_file_generator`], since it
43 /// only needs to be done once per transaction.
44 added_file_generator: AtomicBool,
45 }
46
47 #[pymethods]
48 impl PyFileIndex {
49 #[new]
50 #[allow(clippy::too_many_arguments)]
51 fn new(
52 py: Python<'_>,
53 ui: &Bound<'_, PyAny>,
54 opener: &Bound<'_, PyAny>,
55 try_pending: bool,
56 vacuum_mode: &Bound<'_, PyAny>,
57 max_unused_ratio: f64,
58 gc_retention_s: u32,
59 garbage_timestamp: Option<u32>,
60 ) -> PyResult<Self> {
61 let base = opener.getattr("base")?;
62 let base = get_path_from_bytes(base.extract::<&[u8]>()?).to_owned();
63 let raw_store_vfs = VfsImpl::new(base, false, PathEncoding::None);
64 let vacuum_mode = vacuum_mode.getattr("value")?;
65 let vacuum_mode = VacuumMode::parse(vacuum_mode.extract::<&[u8]>()?)
66 .expect("Python should give a valid vacuum mode");
67 let config = hg::file_index::Config {
68 vacuum_mode,
69 max_unused_ratio,
70 gc_retention_s,
71 garbage_timestamp,
72 };
73 let testing = PyModule::import(py, "mercurial.testing")?;
74 let wait_on_cfg = testing.getattr("wait_on_cfg")?;
75 let devel_sync_point = || {
76 let _ = wait_on_cfg.call1((ui, b"fileindex.pre-read-data-files"));
77 };
78 let file_index = FileIndex::open(
79 raw_store_vfs,
80 try_pending,
81 config,
82 devel_sync_point,
83 )
84 .map_err(to_string_value_error)?;
85 Ok(Self {
86 inner: file_index.into(),
87 added_file_generator: AtomicBool::new(false),
88 })
89 }
90
91 fn has_token(slf: &Bound<'_, Self>, token: u32) -> PyResult<bool> {
92 Self::with_inner_read(slf, |_self_ref, inner| {
93 Ok(inner.has_token(FileToken(token)))
94 })
95 }
96
97 fn has_path(slf: &Bound<'_, Self>, path: &[u8]) -> PyResult<bool> {
98 Self::with_inner_read(slf, |_self_ref, inner| {
99 inner.has_path(HgPath::new(path)).map_err(file_index_error)
100 })
101 }
102
103 fn get_path<'py>(
104 slf: &Bound<'py, Self>,
105 py: Python<'py>,
106 token: u32,
107 ) -> PyResult<Option<Bound<'py, PyBytes>>> {
108 Self::with_inner_read(slf, |_self_ref, inner| {
109 Ok(inner
110 .get_path(FileToken(token))
111 .map_err(file_index_error)?
112 .map(|path| PyBytes::new(py, path.as_bytes())))
113 })
114 }
115
116 fn get_token(slf: &Bound<'_, Self>, path: &[u8]) -> PyResult<Option<u32>> {
117 Self::with_inner_read(slf, |_self_ref, inner| {
118 Ok(inner
119 .get_token(HgPath::new(path))
120 .map_err(file_index_error)?
121 .map(|token| token.0))
122 })
123 }
124
125 fn __contains__(slf: &Bound<'_, Self>, path: &[u8]) -> PyResult<bool> {
126 Self::has_path(slf, path)
127 }
128
129 fn __len__(slf: &Bound<'_, Self>) -> PyResult<usize> {
130 Self::with_inner_read(slf, |_self_ref, inner| Ok(inner.len()))
131 }
132
133 fn __iter__(slf: &Bound<'_, Self>) -> PyResult<PyPathIter> {
134 PyPathIter::new(slf)
135 }
136
137 fn items(slf: &Bound<'_, Self>) -> PyResult<PyPathTokenIter> {
138 PyPathTokenIter::new(slf)
139 }
140
141 fn add(
142 slf: &Bound<'_, Self>,
143 path: &[u8],
144 tr: &Bound<'_, PyAny>,
145 ) -> PyResult<u32> {
146 Self::with_inner_write(slf, |_self_ref, mut inner| {
147 let (token, added) =
148 inner.add(HgPath::new(path)).map_err(file_index_error)?;
149 if added {
150 Self::_add_file_generator(slf, tr)?;
151 }
152 Ok(token.0)
153 })
154 }
155
156 fn remove(
157 slf: &Bound<'_, Self>,
158 path: &[u8],
159 tr: &Bound<'_, PyAny>,
160 ) -> PyResult<()> {
161 Self::with_inner_write(slf, |_self_ref, mut inner| {
162 let removed =
163 inner.remove(HgPath::new(path)).map_err(file_index_error)?;
164 if removed {
165 Self::_add_file_generator(slf, tr)?;
166 }
167 Ok(())
168 })
169 }
170
171 fn vacuum(slf: &Bound<'_, Self>, tr: &Bound<'_, PyAny>) -> PyResult<()> {
172 Self::with_inner_write(slf, |_self_ref, mut inner| {
173 inner.vacuum();
174 Self::_add_file_generator(slf, tr)?;
175 Ok(())
176 })
177 }
178
179 #[pyo3(signature = (tr, force=false))]
180 fn garbage_collect(
181 slf: &Bound<'_, Self>,
182 tr: &Bound<'_, PyAny>,
183 force: bool,
184 ) -> PyResult<()> {
185 Self::with_inner_write(slf, |_self_ref, mut inner| {
186 let changed = inner.garbage_collect(force);
187 if changed {
188 Self::_add_file_generator(slf, tr)?;
189 }
190 Ok(())
191 })
192 }
193
194 fn data_files<'py>(
195 slf: &Bound<'py, Self>,
196 py: Python<'py>,
197 ) -> PyResult<Bound<'py, PyList>> {
198 Self::with_inner_read(slf, |_self_ref, inner| {
199 PyList::new(py, inner.data_files().iter().map(get_bytes_from_path))
200 })
201 }
202
203 fn debug_docket<'py>(
204 slf: &Bound<'py, Self>,
205 py: Python<'py>,
206 ) -> PyResult<Bound<'py, PyDict>> {
207 Self::with_inner_read(slf, |_self_ref, inner| {
208 let docket = inner.docket();
209 let header = &docket.header;
210 let dict = PyDict::new(py);
211 dict.set_item("marker", header.marker)?;
212 dict.set_item("list_file_size", header.list_file_size.get())?;
213 dict.set_item("meta_file_size", header.meta_file_size.get())?;
214 dict.set_item("tree_file_size", header.tree_file_size.get())?;
215 dict.set_item("list_file_id", header.list_file_id.as_bytes())?;
216 dict.set_item("meta_file_id", header.meta_file_id.as_bytes())?;
217 dict.set_item("tree_file_id", header.tree_file_id.as_bytes())?;
218 dict.set_item("tree_root_pointer", header.tree_root_pointer.get())?;
219 dict.set_item("tree_unused_bytes", header.tree_unused_bytes.get())?;
220 dict.set_item(
221 "reserved_flags",
222 u32::from_be_bytes(header.reserved_flags),
223 )?;
224 let garbage_entries = docket
225 .garbage_entries
226 .iter()
227 .map(|entry| -> PyResult<Bound<'_, PyDict>> {
228 let dict = PyDict::new(py);
229 dict.set_item("ttl", entry.ttl)?;
230 dict.set_item("timestamp", entry.timestamp)?;
231 let path = entry.path.as_os_str().as_encoded_bytes();
232 dict.set_item("path", path)?;
233 Ok(dict)
234 })
235 .collect::<Result<Vec<_>, _>>()?;
236 dict.set_item(
237 "garbage_entries",
238 PyList::new(py, garbage_entries)?,
239 )?;
240 Ok(dict)
241 })
242 }
243
244 fn debug_tree_file_size(slf: &Bound<'_, Self>) -> PyResult<usize> {
245 Self::with_inner_read(slf, |_self_ref, inner| {
246 Ok(u32_u(inner.docket().header.tree_file_size.get()))
247 })
248 }
249
250 fn debug_tree_unused_bytes(slf: &Bound<'_, Self>) -> PyResult<usize> {
251 Self::with_inner_read(slf, |_self_ref, inner| {
252 Ok(u32_u(inner.docket().header.tree_unused_bytes.get()))
253 })
254 }
255
256 fn debug_iter_tree_nodes(
257 slf: &Bound<'_, Self>,
258 ) -> PyResult<PyDebugTreeNodeIter> {
259 PyDebugTreeNodeIter::new(slf)
260 }
261
262 fn _add_file_generator(
263 slf: &Bound<'_, Self>,
264 tr: &Bound<'_, PyAny>,
265 ) -> PyResult<()> {
266 // Ordering does not matter since we have the GIL.
267 if slf.borrow().added_file_generator.swap(true, Ordering::Relaxed) {
268 return Ok(());
269 }
270 let kwargs = PyDict::new(slf.py());
271 let genfunc = PyFileIndexWriteClosure {
272 file_index: slf.clone().unbind(),
273 tr: tr.clone().unbind(),
274 };
275 kwargs.set_item("genid", b"fileindex")?;
276 kwargs.set_item("filenames", (b"fileindex",))?;
277 kwargs.set_item("genfunc", genfunc)?;
278 kwargs.set_item("location", b"store")?;
279 kwargs.set_item("post_finalize", true)?;
280 tr.call_method("addfilegenerator", (), Some(&kwargs))?;
281 Ok(())
282 }
283
284 fn _write(
285 slf: &Bound<'_, Self>,
286 f: &Bound<'_, PyAny>,
287 tr: Py<PyAny>,
288 ) -> PyResult<()> {
289 Self::with_inner_write(slf, |self_ref, mut inner| {
290 // Ordering does not matter since we have the GIL.
291 self_ref.added_file_generator.store(false, Ordering::Relaxed);
292 let py_tr = &mut PyTransaction::new(tr);
293 inner.write(py_tr).map_err(to_string_value_error)?;
294 let docket_bytes = inner.docket().serialize();
295 f.call_method1("write", (&docket_bytes,))?;
296 Ok(())
297 })
298 }
299 }
300
301 impl PyFileIndex {
302 /// Helper to get read-only access to [`Self::inner`].
303 fn with_inner_read<'py, T, F>(slf: &Bound<'py, Self>, f: F) -> PyResult<T>
304 where
305 F: FnOnce(&PyRef<'py, Self>, RwLockReadGuard<FileIndex>) -> PyResult<T>,
306 {
307 let self_ref = slf.borrow();
308 // Safety: We are the owner.
309 let shareable_ref = unsafe { self_ref.inner.borrow_with_owner(slf) };
310 let guard = shareable_ref.try_read().map_err(map_try_lock_error)?;
311 f(&self_ref, guard)
312 }
313
314 /// Helper to get read-write access to [`Self::inner`].
315 fn with_inner_write<'py, T, F>(slf: &Bound<'py, Self>, f: F) -> PyResult<T>
316 where
317 F: FnOnce(
318 &PyRef<'py, Self>,
319 RwLockWriteGuard<FileIndex>,
320 ) -> PyResult<T>,
321 {
322 let self_ref = slf.borrow();
323 // Safety: We are the owner.
324 let shareable_ref = unsafe { self_ref.inner.borrow_with_owner(slf) };
325 let guard = shareable_ref.try_write().map_err(map_try_lock_error)?;
326 f(&self_ref, guard)
327 }
328 }
329
330 #[pyclass(name = "FileIndexWriteClosure", frozen)]
331 struct PyFileIndexWriteClosure {
332 file_index: Py<PyFileIndex>,
333 tr: Py<PyAny>,
334 }
335
336 #[pymethods]
337 impl PyFileIndexWriteClosure {
338 fn __call__(&self, py: Python<'_>, f: &Bound<'_, PyAny>) -> PyResult<()> {
339 self.file_index.call_method1(py, "_write", (f, &self.tr))?;
340 Ok(())
341 }
342 }
343
344 py_shared_iterator!(
345 PyPathIter,
346 PyBytes,
347 PyFileIndex,
348 inner,
349 PathTokenIter<'static>,
350 |file_index| file_index.iter(),
351 convert_path_iter_item
352 );
353
354 fn convert_path_iter_item(
355 py: Python,
356 item: Result<(&HgPath, FileToken), hg::file_index::Error>,
357 ) -> PyResult<Option<Py<PyBytes>>> {
358 let (path, _token) = item.map_err(file_index_error)?;
359 Ok(Some(PyBytes::new(py, path.as_bytes()).unbind()))
360 }
361
362 py_shared_iterator!(
363 PyPathTokenIter,
364 PyTuple,
365 PyFileIndex,
366 inner,
367 PathTokenIter<'static>,
368 |file_index| file_index.iter(),
369 convert_path_token_iter_item
370 );
371
372 fn convert_path_token_iter_item(
373 py: Python,
374 item: Result<(&HgPath, FileToken), hg::file_index::Error>,
375 ) -> PyResult<Option<Py<PyTuple>>> {
376 let (path, token) = item.map_err(file_index_error)?;
377 Ok(Some((path.as_bytes(), token.0).into_pyobject(py)?.unbind()))
378 }
379
380 py_shared_iterator!(
381 PyDebugTreeNodeIter,
382 PyTuple,
383 PyFileIndex,
384 inner,
385 DebugTreeNodeIter<'static>,
386 |file_index| file_index.debug_iter_tree_nodes(),
387 convert_debug_tree_node_iter_item
388 );
389
390 fn convert_debug_tree_node_iter_item(
391 py: Python,
392 item: Result<DebugTreeNode, hg::file_index::Error>,
393 ) -> PyResult<Option<Py<PyTuple>>> {
394 let DebugTreeNode { pointer, token, label, children } =
395 item.map_err(file_index_error)?;
396 let children = children
397 .into_iter()
398 .map(|(char, child)| -> Result<(&[u8; 1], Bound<PyAny>), PyErr> {
399 let rhs = match child {
400 DebugTreeChild::Pointer(ptr) => {
401 ptr.into_pyobject(py)?.into_any()
402 }
403 DebugTreeChild::Leaf(label, token) => {
404 (label, token.0).into_pyobject(py)?.into_any()
405 }
406 };
407 Ok((char, rhs))
408 })
409 .collect::<Result<Vec<_>, _>>()?;
410 Ok(Some((pointer, token.0, label, children).into_pyobject(py)?.unbind()))
411 }
412
413 pub fn init_module<'py>(
414 py: Python<'py>,
415 package: &str,
416 ) -> PyResult<Bound<'py, PyModule>> {
417 let m = new_submodule(py, package, "file_index")?;
418 m.add("__doc__", "File index - Rust implementation exposed via PyO3")?;
419 m.add_class::<PyFileIndex>()?;
420 Ok(m)
421 }
422
423 fn file_index_error(err: hg::file_index::Error) -> PyErr {
424 PyOSError::new_err(format!("FileIndex error: {:?}", err))
425 }