| 1 |
//===- PybindUtils.h - Utilities for interop with pybind11 ------*- C++ -*-===// |
| 2 |
// |
| 3 |
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 |
// See https://llvm.org/LICENSE.txt for license information. |
| 5 |
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 |
// |
| 7 |
//===----------------------------------------------------------------------===// |
| 8 |
|
| 9 |
#ifndef MLIR_BINDINGS_PYTHON_PYBINDUTILS_H |
| 10 |
#define MLIR_BINDINGS_PYTHON_PYBINDUTILS_H |
| 11 |
|
| 12 |
#include "mlir-c/Support.h" |
| 13 |
#include "llvm/ADT/Twine.h" |
| 14 |
#include "llvm/Support/DataTypes.h" |
| 15 |
|
| 16 |
#include <pybind11/pybind11.h> |
| 17 |
#include <pybind11/stl.h> |
| 18 |
|
| 19 |
namespace mlir { |
| 20 |
namespace python { |
| 21 |
|
| 22 |
// Sets a python error, ready to be thrown to return control back to the |
| 23 |
// python runtime. |
| 24 |
// Correct usage: |
| 25 |
// throw SetPyError(PyExc_ValueError, "Foobar'd"); |
| 26 |
pybind11::error_already_set SetPyError(PyObject *excClass, |
| 27 |
const llvm::Twine &message); |
| 28 |
|
| 29 |
/// CRTP template for special wrapper types that are allowed to be passed in as |
| 30 |
/// 'None' function arguments and can be resolved by some global mechanic if |
| 31 |
/// so. Such types will raise an error if this global resolution fails, and |
| 32 |
/// it is actually illegal for them to ever be unresolved. From a user |
| 33 |
/// perspective, they behave like a smart ptr to the underlying type (i.e. |
| 34 |
/// 'get' method and operator-> overloaded). |
| 35 |
/// |
| 36 |
/// Derived types must provide a method, which is called when an environmental |
| 37 |
/// resolution is required. It must raise an exception if resolution fails: |
| 38 |
/// static ReferrentTy &resolve() |
| 39 |
/// |
| 40 |
/// They must also provide a parameter description that will be used in |
| 41 |
/// error messages about mismatched types: |
| 42 |
/// static constexpr const char kTypeDescription[] = "<Description>"; |
| 43 |
|
| 44 |
template <typename DerivedTy, typename T> |
| 45 |
class Defaulting { |
| 46 |
public: |
| 47 |
using ReferrentTy = T; |
| 48 |
/// Type casters require the type to be default constructible, but using |
| 49 |
/// such an instance is illegal. |
| 50 |
Defaulting() = default; |
| 51 |
Defaulting(ReferrentTy &referrent) : referrent(&referrent) {} |
| 52 |
|
| 53 |
ReferrentTy *get() const { return referrent; } |
| 54 |
ReferrentTy *operator->() { return referrent; } |
| 55 |
|
| 56 |
private: |
| 57 |
ReferrentTy *referrent = nullptr; |
| 58 |
}; |
| 59 |
|
| 60 |
} // namespace python |
| 61 |
} // namespace mlir |
| 62 |
|
| 63 |
namespace pybind11 { |
| 64 |
namespace detail { |
| 65 |
|
| 66 |
template <typename DefaultingTy> |
| 67 |
struct MlirDefaultingCaster { |
| 68 |
PYBIND11_TYPE_CASTER(DefaultingTy, _(DefaultingTy::kTypeDescription)); |
| 69 |
|
| 70 |
bool load(pybind11::handle src, bool) { |
| 71 |
if (src.is_none()) { |
| 72 |
// Note that we do want an exception to propagate from here as it will be |
| 73 |
// the most informative. |
| 74 |
value = DefaultingTy{DefaultingTy::resolve()}; |
| 75 |
return true; |
| 76 |
} |
| 77 |
|
| 78 |
// Unlike many casters that chain, these casters are expected to always |
| 79 |
// succeed, so instead of doing an isinstance check followed by a cast, |
| 80 |
// just cast in one step and handle the exception. Returning false (vs |
| 81 |
// letting the exception propagate) causes higher level signature parsing |
| 82 |
// code to produce nice error messages (other than "Cannot cast..."). |
| 83 |
try { |
| 84 |
value = DefaultingTy{ |
| 85 |
pybind11::cast<typename DefaultingTy::ReferrentTy &>(src)}; |
| 86 |
return true; |
| 87 |
} catch (std::exception &) { |
| 88 |
return false; |
| 89 |
} |
| 90 |
} |
| 91 |
|
| 92 |
static handle cast(DefaultingTy src, return_value_policy policy, |
| 93 |
handle parent) { |
| 94 |
return pybind11::cast(src, policy); |
| 95 |
} |
| 96 |
}; |
| 97 |
} // namespace detail |
| 98 |
} // namespace pybind11 |
| 99 |
|
| 100 |
//------------------------------------------------------------------------------ |
| 101 |
// Conversion utilities. |
| 102 |
//------------------------------------------------------------------------------ |
| 103 |
|
| 104 |
namespace mlir { |
| 105 |
|
| 106 |
/// Accumulates into a python string from a method that accepts an |
| 107 |
/// MlirStringCallback. |
| 108 |
struct PyPrintAccumulator { |
| 109 |
pybind11::list parts; |
| 110 |
|
| 111 |
void *getUserData() { return this; } |
| 112 |
|
| 113 |
MlirStringCallback getCallback() { |
| 114 |
return [](MlirStringRef part, void *userData) { |
| 115 |
PyPrintAccumulator *printAccum = |
| 116 |
static_cast<PyPrintAccumulator *>(userData); |
| 117 |
pybind11::str pyPart(part.data, |
| 118 |
part.length); // Decodes as UTF-8 by default. |
| 119 |
printAccum->parts.append(std::move(pyPart)); |
| 120 |
}; |
| 121 |
} |
| 122 |
|
| 123 |
pybind11::str join() { |
| 124 |
pybind11::str delim("", 0); |
| 125 |
return delim.attr("join")(parts); |
| 126 |
} |
| 127 |
}; |
| 128 |
|
| 129 |
/// Accumulates int a python file-like object, either writing text (default) |
| 130 |
/// or binary. |
| 131 |
class PyFileAccumulator { |
| 132 |
public: |
| 133 |
PyFileAccumulator(const pybind11::object &fileObject, bool binary) |
| 134 |
: pyWriteFunction(fileObject.attr("write")), binary(binary) {} |
| 135 |
|
| 136 |
void *getUserData() { return this; } |
| 137 |
|
| 138 |
MlirStringCallback getCallback() { |
| 139 |
return [](MlirStringRef part, void *userData) { |
| 140 |
pybind11::gil_scoped_acquire acquire; |
| 141 |
PyFileAccumulator *accum = static_cast<PyFileAccumulator *>(userData); |
| 142 |
if (accum->binary) { |
| 143 |
// Note: Still has to copy and not avoidable with this API. |
| 144 |
pybind11::bytes pyBytes(part.data, part.length); |
| 145 |
accum->pyWriteFunction(pyBytes); |
| 146 |
} else { |
| 147 |
pybind11::str pyStr(part.data, |
| 148 |
part.length); // Decodes as UTF-8 by default. |
| 149 |
accum->pyWriteFunction(pyStr); |
| 150 |
} |
| 151 |
}; |
| 152 |
} |
| 153 |
|
| 154 |
private: |
| 155 |
pybind11::object pyWriteFunction; |
| 156 |
bool binary; |
| 157 |
}; |
| 158 |
|
| 159 |
/// Accumulates into a python string from a method that is expected to make |
| 160 |
/// one (no more, no less) call to the callback (asserts internally on |
| 161 |
/// violation). |
| 162 |
struct PySinglePartStringAccumulator { |
| 163 |
void *getUserData() { return this; } |
| 164 |
|
| 165 |
MlirStringCallback getCallback() { |
| 166 |
return [](MlirStringRef part, void *userData) { |
| 167 |
PySinglePartStringAccumulator *accum = |
| 168 |
static_cast<PySinglePartStringAccumulator *>(userData); |
| 169 |
assert(!accum->invoked && |
| 170 |
"PySinglePartStringAccumulator called back multiple times"); |
| 171 |
accum->invoked = true; |
| 172 |
accum->value = pybind11::str(part.data, part.length); |
| 173 |
}; |
| 174 |
} |
| 175 |
|
| 176 |
pybind11::str takeValue() { |
| 177 |
assert(invoked && "PySinglePartStringAccumulator not called back"); |
| 178 |
return std::move(value); |
| 179 |
} |
| 180 |
|
| 181 |
private: |
| 182 |
pybind11::str value; |
| 183 |
bool invoked = false; |
| 184 |
}; |
| 185 |
|
| 186 |
/// A CRTP base class for pseudo-containers willing to support Python-type |
| 187 |
/// slicing access on top of indexed access. Calling ::bind on this class |
| 188 |
/// will define `__len__` as well as `__getitem__` with integer and slice |
| 189 |
/// arguments. |
| 190 |
/// |
| 191 |
/// This is intended for pseudo-containers that can refer to arbitrary slices of |
| 192 |
/// underlying storage indexed by a single integer. Indexing those with an |
| 193 |
/// integer produces an instance of ElementTy. Indexing those with a slice |
| 194 |
/// produces a new instance of Derived, which can be sliced further. |
| 195 |
/// |
| 196 |
/// A derived class must provide the following: |
| 197 |
/// - a `static const char *pyClassName ` field containing the name of the |
| 198 |
/// Python class to bind; |
| 199 |
/// - an instance method `intptr_t getRawNumElements()` that returns the |
| 200 |
/// number |
| 201 |
/// of elements in the backing container (NOT that of the slice); |
| 202 |
/// - an instance method `ElementTy getRawElement(intptr_t)` that returns a |
| 203 |
/// single element at the given linear index (NOT slice index); |
| 204 |
/// - an instance method `Derived slice(intptr_t, intptr_t, intptr_t)` that |
| 205 |
/// constructs a new instance of the derived pseudo-container with the |
| 206 |
/// given slice parameters (to be forwarded to the Sliceable constructor). |
| 207 |
/// |
| 208 |
/// The getRawNumElements() and getRawElement(intptr_t) callbacks must not |
| 209 |
/// throw. |
| 210 |
/// |
| 211 |
/// A derived class may additionally define: |
| 212 |
/// - a `static void bindDerived(ClassTy &)` method to bind additional methods |
| 213 |
/// the python class. |
| 214 |
template <typename Derived, typename ElementTy> |
| 215 |
class Sliceable { |
| 216 |
protected: |
| 217 |
using ClassTy = pybind11::class_<Derived>; |
| 218 |
|
| 219 |
/// Transforms `index` into a legal value to access the underlying sequence. |
| 220 |
/// Returns <0 on failure. |
| 221 |
intptr_t wrapIndex(intptr_t index) { |
| 222 |
if (index < 0) |
| 223 |
index = length + index; |
| 224 |
if (index < 0 || index >= length) |
| 225 |
return -1; |
| 226 |
return index; |
| 227 |
} |
| 228 |
|
| 229 |
/// Computes the linear index given the current slice properties. |
| 230 |
intptr_t linearizeIndex(intptr_t index) { |
| 231 |
intptr_t linearIndex = index * step + startIndex; |
| 232 |
assert(linearIndex >= 0 && |
| 233 |
linearIndex < static_cast<Derived *>(this)->getRawNumElements() && |
| 234 |
"linear index out of bounds, the slice is ill-formed"); |
| 235 |
return linearIndex; |
| 236 |
} |
| 237 |
|
| 238 |
/// Returns the element at the given slice index. Supports negative indices |
| 239 |
/// by taking elements in inverse order. Returns a nullptr object if out |
| 240 |
/// of bounds. |
| 241 |
pybind11::object getItem(intptr_t index) { |
| 242 |
// Negative indices mean we count from the end. |
| 243 |
index = wrapIndex(index); |
| 244 |
if (index < 0) { |
| 245 |
PyErr_SetString(PyExc_IndexError, "index out of range"); |
| 246 |
return {}; |
| 247 |
} |
| 248 |
|
| 249 |
return pybind11::cast( |
| 250 |
static_cast<Derived *>(this)->getRawElement(linearizeIndex(index))); |
| 251 |
} |
| 252 |
|
| 253 |
/// Returns a new instance of the pseudo-container restricted to the given |
| 254 |
/// slice. Returns a nullptr object on failure. |
| 255 |
pybind11::object getItemSlice(PyObject *slice) { |
| 256 |
ssize_t start, stop, extraStep, sliceLength; |
| 257 |
if (PySlice_GetIndicesEx(slice, length, &start, &stop, &extraStep, |
| 258 |
&sliceLength) != 0) { |
| 259 |
PyErr_SetString(PyExc_IndexError, "index out of range"); |
| 260 |
return {}; |
| 261 |
} |
| 262 |
return pybind11::cast(static_cast<Derived *>(this)->slice( |
| 263 |
startIndex + start * step, sliceLength, step * extraStep)); |
| 264 |
} |
| 265 |
|
| 266 |
public: |
| 267 |
explicit Sliceable(intptr_t startIndex, intptr_t length, intptr_t step) |
| 268 |
: startIndex(startIndex), length(length), step(step) { |
| 269 |
assert(length >= 0 && "expected non-negative slice length"); |
| 270 |
} |
| 271 |
|
| 272 |
/// Returns the `index`-th element in the slice, supports negative indices. |
| 273 |
/// Throws if the index is out of bounds. |
| 274 |
ElementTy getElement(intptr_t index) { |
| 275 |
// Negative indices mean we count from the end. |
| 276 |
index = wrapIndex(index); |
| 277 |
if (index < 0) { |
| 278 |
throw pybind11::index_error("index out of range"); |
| 279 |
} |
| 280 |
|
| 281 |
return static_cast<Derived *>(this)->getRawElement(linearizeIndex(index)); |
| 282 |
} |
| 283 |
|
| 284 |
/// Returns the size of slice. |
| 285 |
intptr_t size() { return length; } |
| 286 |
|
| 287 |
/// Returns a new vector (mapped to Python list) containing elements from two |
| 288 |
/// slices. The new vector is necessary because slices may not be contiguous |
| 289 |
/// or even come from the same original sequence. |
| 290 |
std::vector<ElementTy> dunderAdd(Derived &other) { |
| 291 |
std::vector<ElementTy> elements; |
| 292 |
elements.reserve(length + other.length); |
| 293 |
for (intptr_t i = 0; i < length; ++i) { |
| 294 |
elements.push_back(static_cast<Derived *>(this)->getElement(i)); |
| 295 |
} |
| 296 |
for (intptr_t i = 0; i < other.length; ++i) { |
| 297 |
elements.push_back(static_cast<Derived *>(&other)->getElement(i)); |
| 298 |
} |
| 299 |
return elements; |
| 300 |
} |
| 301 |
|
| 302 |
/// Binds the indexing and length methods in the Python class. |
| 303 |
static void bind(pybind11::module &m) { |
| 304 |
auto clazz = pybind11::class_<Derived>(m, Derived::pyClassName, |
| 305 |
pybind11::module_local()) |
| 306 |
.def("__add__", &Sliceable::dunderAdd); |
| 307 |
Derived::bindDerived(clazz); |
| 308 |
|
| 309 |
// Manually implement the sequence protocol via the C API. We do this |
| 310 |
// because it is approx 4x faster than via pybind11, largely because that |
| 311 |
// formulation requires a C++ exception to be thrown to detect end of |
| 312 |
// sequence. |
| 313 |
// Since we are in a C-context, any C++ exception that happens here |
| 314 |
// will terminate the program. There is nothing in this implementation |
| 315 |
// that should throw in a non-terminal way, so we forgo further |
| 316 |
// exception marshalling. |
| 317 |
// See: https://github.com/pybind/pybind11/issues/2842 |
| 318 |
auto heap_type = reinterpret_cast<PyHeapTypeObject *>(clazz.ptr()); |
| 319 |
assert(heap_type->ht_type.tp_flags & Py_TPFLAGS_HEAPTYPE && |
| 320 |
"must be heap type"); |
| 321 |
heap_type->as_sequence.sq_length = +[](PyObject *rawSelf) -> Py_ssize_t { |
| 322 |
auto self = pybind11::cast<Derived *>(rawSelf); |
| 323 |
return self->length; |
| 324 |
}; |
| 325 |
// sq_item is called as part of the sequence protocol for iteration, |
| 326 |
// list construction, etc. |
| 327 |
heap_type->as_sequence.sq_item = |
| 328 |
+[](PyObject *rawSelf, Py_ssize_t index) -> PyObject * { |
| 329 |
auto self = pybind11::cast<Derived *>(rawSelf); |
| 330 |
return self->getItem(index).release().ptr(); |
| 331 |
}; |
| 332 |
// mp_subscript is used for both slices and integer lookups. |
| 333 |
heap_type->as_mapping.mp_subscript = |
| 334 |
+[](PyObject *rawSelf, PyObject *rawSubscript) -> PyObject * { |
| 335 |
auto self = pybind11::cast<Derived *>(rawSelf); |
| 336 |
Py_ssize_t index = PyNumber_AsSsize_t(rawSubscript, PyExc_IndexError); |
| 337 |
if (!PyErr_Occurred()) { |
| 338 |
// Integer indexing. |
| 339 |
return self->getItem(index).release().ptr(); |
| 340 |
} |
| 341 |
PyErr_Clear(); |
| 342 |
|
| 343 |
// Assume slice-based indexing. |
| 344 |
if (PySlice_Check(rawSubscript)) { |
| 345 |
return self->getItemSlice(rawSubscript).release().ptr(); |
| 346 |
} |
| 347 |
|
| 348 |
PyErr_SetString(PyExc_ValueError, "expected integer or slice"); |
| 349 |
return nullptr; |
| 350 |
}; |
| 351 |
} |
| 352 |
|
| 353 |
/// Hook for derived classes willing to bind more methods. |
| 354 |
static void bindDerived(ClassTy &) {} |
| 355 |
|
| 356 |
private: |
| 357 |
intptr_t startIndex; |
| 358 |
intptr_t length; |
| 359 |
intptr_t step; |
| 360 |
}; |
| 361 |
|
| 362 |
} // namespace mlir |
| 363 |
|
| 364 |
#endif // MLIR_BINDINGS_PYTHON_PYBINDUTILS_H |