Debian Codegraph @

llvm-project/mlir/lib/Bindings/Python/PybindUtils.h

raw
1 //===- PybindUtils.h - Utilities for interop with pybind11 ------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #ifndef MLIR_BINDINGS_PYTHON_PYBINDUTILS_H
10 #define MLIR_BINDINGS_PYTHON_PYBINDUTILS_H
11
12 #include "mlir-c/Support.h"
13 #include "llvm/ADT/Twine.h"
14 #include "llvm/Support/DataTypes.h"
15
16 #include <pybind11/pybind11.h>
17 #include <pybind11/stl.h>
18
19 namespace mlir {
20 namespace python {
21
22 // Sets a python error, ready to be thrown to return control back to the
23 // python runtime.
24 // Correct usage:
25 // throw SetPyError(PyExc_ValueError, "Foobar'd");
26 pybind11::error_already_set SetPyError(PyObject *excClass,
27 const llvm::Twine &message);
28
29 /// CRTP template for special wrapper types that are allowed to be passed in as
30 /// 'None' function arguments and can be resolved by some global mechanic if
31 /// so. Such types will raise an error if this global resolution fails, and
32 /// it is actually illegal for them to ever be unresolved. From a user
33 /// perspective, they behave like a smart ptr to the underlying type (i.e.
34 /// 'get' method and operator-> overloaded).
35 ///
36 /// Derived types must provide a method, which is called when an environmental
37 /// resolution is required. It must raise an exception if resolution fails:
38 /// static ReferrentTy &resolve()
39 ///
40 /// They must also provide a parameter description that will be used in
41 /// error messages about mismatched types:
42 /// static constexpr const char kTypeDescription[] = "<Description>";
43
44 template <typename DerivedTy, typename T>
45 class Defaulting {
46 public:
47 using ReferrentTy = T;
48 /// Type casters require the type to be default constructible, but using
49 /// such an instance is illegal.
50 Defaulting() = default;
51 Defaulting(ReferrentTy &referrent) : referrent(&referrent) {}
52
53 ReferrentTy *get() const { return referrent; }
54 ReferrentTy *operator->() { return referrent; }
55
56 private:
57 ReferrentTy *referrent = nullptr;
58 };
59
60 } // namespace python
61 } // namespace mlir
62
63 namespace pybind11 {
64 namespace detail {
65
66 template <typename DefaultingTy>
67 struct MlirDefaultingCaster {
68 PYBIND11_TYPE_CASTER(DefaultingTy, _(DefaultingTy::kTypeDescription));
69
70 bool load(pybind11::handle src, bool) {
71 if (src.is_none()) {
72 // Note that we do want an exception to propagate from here as it will be
73 // the most informative.
74 value = DefaultingTy{DefaultingTy::resolve()};
75 return true;
76 }
77
78 // Unlike many casters that chain, these casters are expected to always
79 // succeed, so instead of doing an isinstance check followed by a cast,
80 // just cast in one step and handle the exception. Returning false (vs
81 // letting the exception propagate) causes higher level signature parsing
82 // code to produce nice error messages (other than "Cannot cast...").
83 try {
84 value = DefaultingTy{
85 pybind11::cast<typename DefaultingTy::ReferrentTy &>(src)};
86 return true;
87 } catch (std::exception &) {
88 return false;
89 }
90 }
91
92 static handle cast(DefaultingTy src, return_value_policy policy,
93 handle parent) {
94 return pybind11::cast(src, policy);
95 }
96 };
97 } // namespace detail
98 } // namespace pybind11
99
100 //------------------------------------------------------------------------------
101 // Conversion utilities.
102 //------------------------------------------------------------------------------
103
104 namespace mlir {
105
106 /// Accumulates into a python string from a method that accepts an
107 /// MlirStringCallback.
108 struct PyPrintAccumulator {
109 pybind11::list parts;
110
111 void *getUserData() { return this; }
112
113 MlirStringCallback getCallback() {
114 return [](MlirStringRef part, void *userData) {
115 PyPrintAccumulator *printAccum =
116 static_cast<PyPrintAccumulator *>(userData);
117 pybind11::str pyPart(part.data,
118 part.length); // Decodes as UTF-8 by default.
119 printAccum->parts.append(std::move(pyPart));
120 };
121 }
122
123 pybind11::str join() {
124 pybind11::str delim("", 0);
125 return delim.attr("join")(parts);
126 }
127 };
128
129 /// Accumulates int a python file-like object, either writing text (default)
130 /// or binary.
131 class PyFileAccumulator {
132 public:
133 PyFileAccumulator(const pybind11::object &fileObject, bool binary)
134 : pyWriteFunction(fileObject.attr("write")), binary(binary) {}
135
136 void *getUserData() { return this; }
137
138 MlirStringCallback getCallback() {
139 return [](MlirStringRef part, void *userData) {
140 pybind11::gil_scoped_acquire acquire;
141 PyFileAccumulator *accum = static_cast<PyFileAccumulator *>(userData);
142 if (accum->binary) {
143 // Note: Still has to copy and not avoidable with this API.
144 pybind11::bytes pyBytes(part.data, part.length);
145 accum->pyWriteFunction(pyBytes);
146 } else {
147 pybind11::str pyStr(part.data,
148 part.length); // Decodes as UTF-8 by default.
149 accum->pyWriteFunction(pyStr);
150 }
151 };
152 }
153
154 private:
155 pybind11::object pyWriteFunction;
156 bool binary;
157 };
158
159 /// Accumulates into a python string from a method that is expected to make
160 /// one (no more, no less) call to the callback (asserts internally on
161 /// violation).
162 struct PySinglePartStringAccumulator {
163 void *getUserData() { return this; }
164
165 MlirStringCallback getCallback() {
166 return [](MlirStringRef part, void *userData) {
167 PySinglePartStringAccumulator *accum =
168 static_cast<PySinglePartStringAccumulator *>(userData);
169 assert(!accum->invoked &&
170 "PySinglePartStringAccumulator called back multiple times");
171 accum->invoked = true;
172 accum->value = pybind11::str(part.data, part.length);
173 };
174 }
175
176 pybind11::str takeValue() {
177 assert(invoked && "PySinglePartStringAccumulator not called back");
178 return std::move(value);
179 }
180
181 private:
182 pybind11::str value;
183 bool invoked = false;
184 };
185
186 /// A CRTP base class for pseudo-containers willing to support Python-type
187 /// slicing access on top of indexed access. Calling ::bind on this class
188 /// will define `__len__` as well as `__getitem__` with integer and slice
189 /// arguments.
190 ///
191 /// This is intended for pseudo-containers that can refer to arbitrary slices of
192 /// underlying storage indexed by a single integer. Indexing those with an
193 /// integer produces an instance of ElementTy. Indexing those with a slice
194 /// produces a new instance of Derived, which can be sliced further.
195 ///
196 /// A derived class must provide the following:
197 /// - a `static const char *pyClassName ` field containing the name of the
198 /// Python class to bind;
199 /// - an instance method `intptr_t getRawNumElements()` that returns the
200 /// number
201 /// of elements in the backing container (NOT that of the slice);
202 /// - an instance method `ElementTy getRawElement(intptr_t)` that returns a
203 /// single element at the given linear index (NOT slice index);
204 /// - an instance method `Derived slice(intptr_t, intptr_t, intptr_t)` that
205 /// constructs a new instance of the derived pseudo-container with the
206 /// given slice parameters (to be forwarded to the Sliceable constructor).
207 ///
208 /// The getRawNumElements() and getRawElement(intptr_t) callbacks must not
209 /// throw.
210 ///
211 /// A derived class may additionally define:
212 /// - a `static void bindDerived(ClassTy &)` method to bind additional methods
213 /// the python class.
214 template <typename Derived, typename ElementTy>
215 class Sliceable {
216 protected:
217 using ClassTy = pybind11::class_<Derived>;
218
219 /// Transforms `index` into a legal value to access the underlying sequence.
220 /// Returns <0 on failure.
221 intptr_t wrapIndex(intptr_t index) {
222 if (index < 0)
223 index = length + index;
224 if (index < 0 || index >= length)
225 return -1;
226 return index;
227 }
228
229 /// Computes the linear index given the current slice properties.
230 intptr_t linearizeIndex(intptr_t index) {
231 intptr_t linearIndex = index * step + startIndex;
232 assert(linearIndex >= 0 &&
233 linearIndex < static_cast<Derived *>(this)->getRawNumElements() &&
234 "linear index out of bounds, the slice is ill-formed");
235 return linearIndex;
236 }
237
238 /// Returns the element at the given slice index. Supports negative indices
239 /// by taking elements in inverse order. Returns a nullptr object if out
240 /// of bounds.
241 pybind11::object getItem(intptr_t index) {
242 // Negative indices mean we count from the end.
243 index = wrapIndex(index);
244 if (index < 0) {
245 PyErr_SetString(PyExc_IndexError, "index out of range");
246 return {};
247 }
248
249 return pybind11::cast(
250 static_cast<Derived *>(this)->getRawElement(linearizeIndex(index)));
251 }
252
253 /// Returns a new instance of the pseudo-container restricted to the given
254 /// slice. Returns a nullptr object on failure.
255 pybind11::object getItemSlice(PyObject *slice) {
256 ssize_t start, stop, extraStep, sliceLength;
257 if (PySlice_GetIndicesEx(slice, length, &start, &stop, &extraStep,
258 &sliceLength) != 0) {
259 PyErr_SetString(PyExc_IndexError, "index out of range");
260 return {};
261 }
262 return pybind11::cast(static_cast<Derived *>(this)->slice(
263 startIndex + start * step, sliceLength, step * extraStep));
264 }
265
266 public:
267 explicit Sliceable(intptr_t startIndex, intptr_t length, intptr_t step)
268 : startIndex(startIndex), length(length), step(step) {
269 assert(length >= 0 && "expected non-negative slice length");
270 }
271
272 /// Returns the `index`-th element in the slice, supports negative indices.
273 /// Throws if the index is out of bounds.
274 ElementTy getElement(intptr_t index) {
275 // Negative indices mean we count from the end.
276 index = wrapIndex(index);
277 if (index < 0) {
278 throw pybind11::index_error("index out of range");
279 }
280
281 return static_cast<Derived *>(this)->getRawElement(linearizeIndex(index));
282 }
283
284 /// Returns the size of slice.
285 intptr_t size() { return length; }
286
287 /// Returns a new vector (mapped to Python list) containing elements from two
288 /// slices. The new vector is necessary because slices may not be contiguous
289 /// or even come from the same original sequence.
290 std::vector<ElementTy> dunderAdd(Derived &other) {
291 std::vector<ElementTy> elements;
292 elements.reserve(length + other.length);
293 for (intptr_t i = 0; i < length; ++i) {
294 elements.push_back(static_cast<Derived *>(this)->getElement(i));
295 }
296 for (intptr_t i = 0; i < other.length; ++i) {
297 elements.push_back(static_cast<Derived *>(&other)->getElement(i));
298 }
299 return elements;
300 }
301
302 /// Binds the indexing and length methods in the Python class.
303 static void bind(pybind11::module &m) {
304 auto clazz = pybind11::class_<Derived>(m, Derived::pyClassName,
305 pybind11::module_local())
306 .def("__add__", &Sliceable::dunderAdd);
307 Derived::bindDerived(clazz);
308
309 // Manually implement the sequence protocol via the C API. We do this
310 // because it is approx 4x faster than via pybind11, largely because that
311 // formulation requires a C++ exception to be thrown to detect end of
312 // sequence.
313 // Since we are in a C-context, any C++ exception that happens here
314 // will terminate the program. There is nothing in this implementation
315 // that should throw in a non-terminal way, so we forgo further
316 // exception marshalling.
317 // See: https://github.com/pybind/pybind11/issues/2842
318 auto heap_type = reinterpret_cast<PyHeapTypeObject *>(clazz.ptr());
319 assert(heap_type->ht_type.tp_flags & Py_TPFLAGS_HEAPTYPE &&
320 "must be heap type");
321 heap_type->as_sequence.sq_length = +[](PyObject *rawSelf) -> Py_ssize_t {
322 auto self = pybind11::cast<Derived *>(rawSelf);
323 return self->length;
324 };
325 // sq_item is called as part of the sequence protocol for iteration,
326 // list construction, etc.
327 heap_type->as_sequence.sq_item =
328 +[](PyObject *rawSelf, Py_ssize_t index) -> PyObject * {
329 auto self = pybind11::cast<Derived *>(rawSelf);
330 return self->getItem(index).release().ptr();
331 };
332 // mp_subscript is used for both slices and integer lookups.
333 heap_type->as_mapping.mp_subscript =
334 +[](PyObject *rawSelf, PyObject *rawSubscript) -> PyObject * {
335 auto self = pybind11::cast<Derived *>(rawSelf);
336 Py_ssize_t index = PyNumber_AsSsize_t(rawSubscript, PyExc_IndexError);
337 if (!PyErr_Occurred()) {
338 // Integer indexing.
339 return self->getItem(index).release().ptr();
340 }
341 PyErr_Clear();
342
343 // Assume slice-based indexing.
344 if (PySlice_Check(rawSubscript)) {
345 return self->getItemSlice(rawSubscript).release().ptr();
346 }
347
348 PyErr_SetString(PyExc_ValueError, "expected integer or slice");
349 return nullptr;
350 };
351 }
352
353 /// Hook for derived classes willing to bind more methods.
354 static void bindDerived(ClassTy &) {}
355
356 private:
357 intptr_t startIndex;
358 intptr_t length;
359 intptr_t step;
360 };
361
362 } // namespace mlir
363
364 #endif // MLIR_BINDINGS_PYTHON_PYBINDUTILS_H