| 1 |
#ifndef Py_INTERNAL_OBJECT_H |
| 2 |
#define Py_INTERNAL_OBJECT_H |
| 3 |
#ifdef __cplusplus |
| 4 |
extern "C" { |
| 5 |
#endif |
| 6 |
|
| 7 |
#ifndef Py_BUILD_CORE |
| 8 |
# error "this header requires Py_BUILD_CORE define" |
| 9 |
#endif |
| 10 |
|
| 11 |
#include "pycore_emscripten_trampoline.h" // _PyCFunction_TrampolineCall() |
| 12 |
#include "pycore_gc.h" // _PyObject_GC_TRACK() |
| 13 |
#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_PTR_ACQUIRE() |
| 14 |
#include "pycore_pystate.h" // _PyInterpreterState_GET() |
| 15 |
#include "pycore_runtime.h" // _PyRuntime |
| 16 |
#include "pycore_typeobject.h" // _PyStaticType_GetState() |
| 17 |
#include "pycore_uniqueid.h" // _PyObject_ThreadIncrefSlow() |
| 18 |
|
| 19 |
#include <stdbool.h> // bool |
| 20 |
|
| 21 |
|
| 22 |
// This value is added to `ob_ref_shared` for objects that use deferred |
| 23 |
// reference counting so that they are not immediately deallocated when the |
| 24 |
// non-deferred reference count drops to zero. |
| 25 |
// |
| 26 |
// The value is half the maximum shared refcount because the low two bits of |
| 27 |
// `ob_ref_shared` are used for flags. |
| 28 |
#define _Py_REF_DEFERRED (PY_SSIZE_T_MAX / 8) |
| 29 |
|
| 30 |
/* For backwards compatibility -- Do not use this */ |
| 31 |
#define _Py_IsImmortalLoose(op) _Py_IsImmortal |
| 32 |
|
| 33 |
|
| 34 |
/* Check if an object is consistent. For example, ensure that the reference |
| 35 |
counter is greater than or equal to 1, and ensure that ob_type is not NULL. |
| 36 |
|
| 37 |
Call _PyObject_AssertFailed() if the object is inconsistent. |
| 38 |
|
| 39 |
If check_content is zero, only check header fields: reduce the overhead. |
| 40 |
|
| 41 |
The function always return 1. The return value is just here to be able to |
| 42 |
write: |
| 43 |
|
| 44 |
assert(_PyObject_CheckConsistency(obj, 1)); */ |
| 45 |
extern int _PyObject_CheckConsistency(PyObject *op, int check_content); |
| 46 |
|
| 47 |
extern void _PyDebugAllocatorStats(FILE *out, const char *block_name, |
| 48 |
int num_blocks, size_t sizeof_block); |
| 49 |
|
| 50 |
extern void _PyObject_DebugTypeStats(FILE *out); |
| 51 |
|
| 52 |
#ifdef Py_TRACE_REFS |
| 53 |
// Forget a reference registered by _Py_NewReference(). Function called by |
| 54 |
// _Py_Dealloc(). |
| 55 |
// |
| 56 |
// On a free list, the function can be used before modifying an object to |
| 57 |
// remove the object from traced objects. Then _Py_NewReference() or |
| 58 |
// _Py_NewReferenceNoTotal() should be called again on the object to trace |
| 59 |
// it again. |
| 60 |
extern void _Py_ForgetReference(PyObject *); |
| 61 |
#endif |
| 62 |
|
| 63 |
// Export for shared _testinternalcapi extension |
| 64 |
PyAPI_FUNC(int) _PyObject_IsFreed(PyObject *); |
| 65 |
|
| 66 |
/* We need to maintain an internal copy of Py{Var}Object_HEAD_INIT to avoid |
| 67 |
designated initializer conflicts in C++20. If we use the definition in |
| 68 |
object.h, we will be mixing designated and non-designated initializers in |
| 69 |
pycore objects which is forbiddent in C++20. However, if we then use |
| 70 |
designated initializers in object.h then Extensions without designated break. |
| 71 |
Furthermore, we can't use designated initializers in Extensions since these |
| 72 |
are not supported pre-C++20. Thus, keeping an internal copy here is the most |
| 73 |
backwards compatible solution */ |
| 74 |
#if defined(Py_GIL_DISABLED) |
| 75 |
#define _PyObject_HEAD_INIT(type) \ |
| 76 |
{ \ |
| 77 |
.ob_ref_local = _Py_IMMORTAL_REFCNT_LOCAL, \ |
| 78 |
.ob_flags = _Py_STATICALLY_ALLOCATED_FLAG, \ |
| 79 |
.ob_gc_bits = _PyGC_BITS_DEFERRED, \ |
| 80 |
.ob_type = (type) \ |
| 81 |
} |
| 82 |
#else |
| 83 |
#if SIZEOF_VOID_P > 4 |
| 84 |
#define _PyObject_HEAD_INIT(type) \ |
| 85 |
{ \ |
| 86 |
.ob_refcnt = _Py_IMMORTAL_INITIAL_REFCNT, \ |
| 87 |
.ob_flags = _Py_STATIC_FLAG_BITS, \ |
| 88 |
.ob_type = (type) \ |
| 89 |
} |
| 90 |
#else |
| 91 |
#define _PyObject_HEAD_INIT(type) \ |
| 92 |
{ \ |
| 93 |
.ob_refcnt = _Py_STATIC_IMMORTAL_INITIAL_REFCNT, \ |
| 94 |
.ob_type = (type) \ |
| 95 |
} |
| 96 |
#endif |
| 97 |
#endif |
| 98 |
#define _PyVarObject_HEAD_INIT(type, size) \ |
| 99 |
{ \ |
| 100 |
.ob_base = _PyObject_HEAD_INIT(type), \ |
| 101 |
.ob_size = size \ |
| 102 |
} |
| 103 |
|
| 104 |
PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalRefcountErrorFunc( |
| 105 |
const char *func, |
| 106 |
const char *message); |
| 107 |
|
| 108 |
#define _Py_FatalRefcountError(message) \ |
| 109 |
_Py_FatalRefcountErrorFunc(__func__, (message)) |
| 110 |
|
| 111 |
#define _PyReftracerTrack(obj, operation) \ |
| 112 |
do { \ |
| 113 |
struct _reftracer_runtime_state *tracer = &_PyRuntime.ref_tracer; \ |
| 114 |
if (tracer->tracer_func != NULL) { \ |
| 115 |
void *data = tracer->tracer_data; \ |
| 116 |
tracer->tracer_func((obj), (operation), data); \ |
| 117 |
} \ |
| 118 |
} while(0) |
| 119 |
|
| 120 |
#ifdef Py_REF_DEBUG |
| 121 |
/* The symbol is only exposed in the API for the sake of extensions |
| 122 |
built against the pre-3.12 stable ABI. */ |
| 123 |
PyAPI_DATA(Py_ssize_t) _Py_RefTotal; |
| 124 |
|
| 125 |
extern void _Py_AddRefTotal(PyThreadState *, Py_ssize_t); |
| 126 |
extern PyAPI_FUNC(void) _Py_IncRefTotal(PyThreadState *); |
| 127 |
extern PyAPI_FUNC(void) _Py_DecRefTotal(PyThreadState *); |
| 128 |
|
| 129 |
# define _Py_DEC_REFTOTAL(interp) \ |
| 130 |
interp->object_state.reftotal-- |
| 131 |
#endif |
| 132 |
|
| 133 |
// Increment reference count by n |
| 134 |
static inline void _Py_RefcntAdd(PyObject* op, Py_ssize_t n) |
| 135 |
{ |
| 136 |
if (_Py_IsImmortal(op)) { |
| 137 |
_Py_INCREF_IMMORTAL_STAT_INC(); |
| 138 |
return; |
| 139 |
} |
| 140 |
#ifndef Py_GIL_DISABLED |
| 141 |
Py_ssize_t refcnt = _Py_REFCNT(op); |
| 142 |
Py_ssize_t new_refcnt = refcnt + n; |
| 143 |
if (new_refcnt >= (Py_ssize_t)_Py_IMMORTAL_MINIMUM_REFCNT) { |
| 144 |
new_refcnt = _Py_IMMORTAL_INITIAL_REFCNT; |
| 145 |
} |
| 146 |
# if SIZEOF_VOID_P > 4 |
| 147 |
op->ob_refcnt = (PY_UINT32_T)new_refcnt; |
| 148 |
# else |
| 149 |
op->ob_refcnt = new_refcnt; |
| 150 |
# endif |
| 151 |
# ifdef Py_REF_DEBUG |
| 152 |
_Py_AddRefTotal(_PyThreadState_GET(), new_refcnt - refcnt); |
| 153 |
# endif |
| 154 |
#else |
| 155 |
if (_Py_IsOwnedByCurrentThread(op)) { |
| 156 |
uint32_t local = op->ob_ref_local; |
| 157 |
Py_ssize_t refcnt = (Py_ssize_t)local + n; |
| 158 |
# if PY_SSIZE_T_MAX > UINT32_MAX |
| 159 |
if (refcnt > (Py_ssize_t)UINT32_MAX) { |
| 160 |
// Make the object immortal if the 32-bit local reference count |
| 161 |
// would overflow. |
| 162 |
refcnt = _Py_IMMORTAL_REFCNT_LOCAL; |
| 163 |
} |
| 164 |
# endif |
| 165 |
_Py_atomic_store_uint32_relaxed(&op->ob_ref_local, (uint32_t)refcnt); |
| 166 |
} |
| 167 |
else { |
| 168 |
_Py_atomic_add_ssize(&op->ob_ref_shared, (n << _Py_REF_SHARED_SHIFT)); |
| 169 |
} |
| 170 |
# ifdef Py_REF_DEBUG |
| 171 |
_Py_AddRefTotal(_PyThreadState_GET(), n); |
| 172 |
# endif |
| 173 |
#endif |
| 174 |
// Although the ref count was increased by `n` (which may be greater than 1) |
| 175 |
// it is only a single increment (i.e. addition) operation, so only 1 refcnt |
| 176 |
// increment operation is counted. |
| 177 |
_Py_INCREF_STAT_INC(); |
| 178 |
} |
| 179 |
#define _Py_RefcntAdd(op, n) _Py_RefcntAdd(_PyObject_CAST(op), n) |
| 180 |
|
| 181 |
// Checks if an object has a single, unique reference. If the caller holds a |
| 182 |
// unique reference, it may be able to safely modify the object in-place. |
| 183 |
static inline int |
| 184 |
_PyObject_IsUniquelyReferenced(PyObject *ob) |
| 185 |
{ |
| 186 |
#if !defined(Py_GIL_DISABLED) |
| 187 |
return Py_REFCNT(ob) == 1; |
| 188 |
#else |
| 189 |
// NOTE: the entire ob_ref_shared field must be zero, including flags, to |
| 190 |
// ensure that other threads cannot concurrently create new references to |
| 191 |
// this object. |
| 192 |
return (_Py_IsOwnedByCurrentThread(ob) && |
| 193 |
_Py_atomic_load_uint32_relaxed(&ob->ob_ref_local) == 1 && |
| 194 |
_Py_atomic_load_ssize_relaxed(&ob->ob_ref_shared) == 0); |
| 195 |
#endif |
| 196 |
} |
| 197 |
|
| 198 |
PyAPI_FUNC(void) _Py_SetImmortal(PyObject *op); |
| 199 |
PyAPI_FUNC(void) _Py_SetImmortalUntracked(PyObject *op); |
| 200 |
|
| 201 |
// Makes an immortal object mortal again with the specified refcnt. Should only |
| 202 |
// be used during runtime finalization. |
| 203 |
static inline void _Py_SetMortal(PyObject *op, short refcnt) |
| 204 |
{ |
| 205 |
if (op) { |
| 206 |
assert(_Py_IsImmortal(op)); |
| 207 |
#ifdef Py_GIL_DISABLED |
| 208 |
op->ob_tid = _Py_UNOWNED_TID; |
| 209 |
op->ob_ref_local = 0; |
| 210 |
op->ob_ref_shared = _Py_REF_SHARED(refcnt, _Py_REF_MERGED); |
| 211 |
#else |
| 212 |
op->ob_refcnt = refcnt; |
| 213 |
#endif |
| 214 |
} |
| 215 |
} |
| 216 |
|
| 217 |
/* _Py_ClearImmortal() should only be used during runtime finalization. */ |
| 218 |
static inline void _Py_ClearImmortal(PyObject *op) |
| 219 |
{ |
| 220 |
if (op) { |
| 221 |
_Py_SetMortal(op, 1); |
| 222 |
Py_DECREF(op); |
| 223 |
} |
| 224 |
} |
| 225 |
#define _Py_ClearImmortal(op) \ |
| 226 |
do { \ |
| 227 |
_Py_ClearImmortal(_PyObject_CAST(op)); \ |
| 228 |
op = NULL; \ |
| 229 |
} while (0) |
| 230 |
|
| 231 |
#if !defined(Py_GIL_DISABLED) |
| 232 |
static inline void |
| 233 |
_Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct) |
| 234 |
{ |
| 235 |
if (_Py_IsImmortal(op)) { |
| 236 |
_Py_DECREF_IMMORTAL_STAT_INC(); |
| 237 |
return; |
| 238 |
} |
| 239 |
_Py_DECREF_STAT_INC(); |
| 240 |
#ifdef Py_REF_DEBUG |
| 241 |
_Py_DEC_REFTOTAL(PyInterpreterState_Get()); |
| 242 |
#endif |
| 243 |
if (--op->ob_refcnt != 0) { |
| 244 |
assert(op->ob_refcnt > 0); |
| 245 |
} |
| 246 |
else { |
| 247 |
#ifdef Py_TRACE_REFS |
| 248 |
_Py_ForgetReference(op); |
| 249 |
#endif |
| 250 |
_PyReftracerTrack(op, PyRefTracer_DESTROY); |
| 251 |
destruct(op); |
| 252 |
} |
| 253 |
} |
| 254 |
|
| 255 |
static inline void |
| 256 |
_Py_DECREF_NO_DEALLOC(PyObject *op) |
| 257 |
{ |
| 258 |
if (_Py_IsImmortal(op)) { |
| 259 |
_Py_DECREF_IMMORTAL_STAT_INC(); |
| 260 |
return; |
| 261 |
} |
| 262 |
_Py_DECREF_STAT_INC(); |
| 263 |
#ifdef Py_REF_DEBUG |
| 264 |
_Py_DEC_REFTOTAL(PyInterpreterState_Get()); |
| 265 |
#endif |
| 266 |
op->ob_refcnt--; |
| 267 |
#ifdef Py_DEBUG |
| 268 |
if (op->ob_refcnt <= 0) { |
| 269 |
_Py_FatalRefcountError("Expected a positive remaining refcount"); |
| 270 |
} |
| 271 |
#endif |
| 272 |
} |
| 273 |
|
| 274 |
#else |
| 275 |
// TODO: implement Py_DECREF specializations for Py_GIL_DISABLED build |
| 276 |
static inline void |
| 277 |
_Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct) |
| 278 |
{ |
| 279 |
Py_DECREF(op); |
| 280 |
} |
| 281 |
|
| 282 |
static inline void |
| 283 |
_Py_DECREF_NO_DEALLOC(PyObject *op) |
| 284 |
{ |
| 285 |
Py_DECREF(op); |
| 286 |
} |
| 287 |
|
| 288 |
static inline int |
| 289 |
_Py_REF_IS_MERGED(Py_ssize_t ob_ref_shared) |
| 290 |
{ |
| 291 |
return (ob_ref_shared & _Py_REF_SHARED_FLAG_MASK) == _Py_REF_MERGED; |
| 292 |
} |
| 293 |
|
| 294 |
static inline int |
| 295 |
_Py_REF_IS_QUEUED(Py_ssize_t ob_ref_shared) |
| 296 |
{ |
| 297 |
return (ob_ref_shared & _Py_REF_SHARED_FLAG_MASK) == _Py_REF_QUEUED; |
| 298 |
} |
| 299 |
|
| 300 |
// Merge the local and shared reference count fields and add `extra` to the |
| 301 |
// refcount when merging. |
| 302 |
Py_ssize_t _Py_ExplicitMergeRefcount(PyObject *op, Py_ssize_t extra); |
| 303 |
#endif // !defined(Py_GIL_DISABLED) |
| 304 |
|
| 305 |
#ifdef Py_REF_DEBUG |
| 306 |
# undef _Py_DEC_REFTOTAL |
| 307 |
#endif |
| 308 |
|
| 309 |
|
| 310 |
extern int _PyType_CheckConsistency(PyTypeObject *type); |
| 311 |
extern int _PyDict_CheckConsistency(PyObject *mp, int check_content); |
| 312 |
|
| 313 |
// Fast inlined version of PyType_HasFeature() |
| 314 |
static inline int |
| 315 |
_PyType_HasFeature(PyTypeObject *type, unsigned long feature) { |
| 316 |
return ((FT_ATOMIC_LOAD_ULONG_RELAXED(type->tp_flags) & feature) != 0); |
| 317 |
} |
| 318 |
|
| 319 |
extern void _PyType_InitCache(PyInterpreterState *interp); |
| 320 |
|
| 321 |
extern PyStatus _PyObject_InitState(PyInterpreterState *interp); |
| 322 |
extern void _PyObject_FiniState(PyInterpreterState *interp); |
| 323 |
extern bool _PyRefchain_IsTraced(PyInterpreterState *interp, PyObject *obj); |
| 324 |
|
| 325 |
// Macros used for per-thread reference counting in the free threading build. |
| 326 |
// They resolve to normal Py_INCREF/DECREF calls in the default build. |
| 327 |
// |
| 328 |
// The macros are used for only a few references that would otherwise cause |
| 329 |
// scaling bottlenecks in the free threading build: |
| 330 |
// - The reference from an object to `ob_type`. |
| 331 |
// - The reference from a function to `func_code`. |
| 332 |
// - The reference from a function to `func_globals` and `func_builtins`. |
| 333 |
// |
| 334 |
// It's safe, but not performant or necessary, to use these macros for other |
| 335 |
// references to code, type, or dict objects. It's also safe to mix their |
| 336 |
// usage with normal Py_INCREF/DECREF calls. |
| 337 |
// |
| 338 |
// See also Include/internal/pycore_dict.h for _Py_INCREF_DICT/_Py_DECREF_DICT. |
| 339 |
#ifndef Py_GIL_DISABLED |
| 340 |
# define _Py_INCREF_TYPE Py_INCREF |
| 341 |
# define _Py_DECREF_TYPE Py_DECREF |
| 342 |
# define _Py_INCREF_CODE Py_INCREF |
| 343 |
# define _Py_DECREF_CODE Py_DECREF |
| 344 |
#else |
| 345 |
static inline void |
| 346 |
_Py_THREAD_INCREF_OBJECT(PyObject *obj, Py_ssize_t unique_id) |
| 347 |
{ |
| 348 |
_PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); |
| 349 |
|
| 350 |
// The table index is `unique_id - 1` because 0 is not a valid unique id. |
| 351 |
// Unsigned comparison so that `idx=-1` is handled by the "else". |
| 352 |
size_t idx = (size_t)(unique_id - 1); |
| 353 |
if (idx < (size_t)tstate->refcounts.size) { |
| 354 |
# ifdef Py_REF_DEBUG |
| 355 |
_Py_INCREF_IncRefTotal(); |
| 356 |
# endif |
| 357 |
_Py_INCREF_STAT_INC(); |
| 358 |
tstate->refcounts.values[idx]++; |
| 359 |
} |
| 360 |
else { |
| 361 |
// The slow path resizes the per-thread refcount array if necessary. |
| 362 |
// It handles the unique_id=0 case to keep the inlinable function smaller. |
| 363 |
_PyObject_ThreadIncrefSlow(obj, idx); |
| 364 |
} |
| 365 |
} |
| 366 |
|
| 367 |
static inline void |
| 368 |
_Py_INCREF_TYPE(PyTypeObject *type) |
| 369 |
{ |
| 370 |
if (!_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { |
| 371 |
assert(_Py_IsImmortal(type)); |
| 372 |
_Py_INCREF_IMMORTAL_STAT_INC(); |
| 373 |
return; |
| 374 |
} |
| 375 |
|
| 376 |
// gh-122974: GCC 11 warns about the access to PyHeapTypeObject fields when |
| 377 |
// _Py_INCREF_TYPE() is called on a statically allocated type. The |
| 378 |
// _PyType_HasFeature check above ensures that the type is a heap type. |
| 379 |
#if defined(__GNUC__) && __GNUC__ >= 11 |
| 380 |
# pragma GCC diagnostic push |
| 381 |
# pragma GCC diagnostic ignored "-Warray-bounds" |
| 382 |
#endif |
| 383 |
_Py_THREAD_INCREF_OBJECT((PyObject *)type, ((PyHeapTypeObject *)type)->unique_id); |
| 384 |
#if defined(__GNUC__) && __GNUC__ >= 11 |
| 385 |
# pragma GCC diagnostic pop |
| 386 |
#endif |
| 387 |
} |
| 388 |
|
| 389 |
static inline void |
| 390 |
_Py_INCREF_CODE(PyCodeObject *co) |
| 391 |
{ |
| 392 |
_Py_THREAD_INCREF_OBJECT((PyObject *)co, co->_co_unique_id); |
| 393 |
} |
| 394 |
|
| 395 |
static inline void |
| 396 |
_Py_THREAD_DECREF_OBJECT(PyObject *obj, Py_ssize_t unique_id) |
| 397 |
{ |
| 398 |
_PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); |
| 399 |
|
| 400 |
// The table index is `unique_id - 1` because 0 is not a valid unique id. |
| 401 |
// Unsigned comparison so that `idx=-1` is handled by the "else". |
| 402 |
size_t idx = (size_t)(unique_id - 1); |
| 403 |
if (idx < (size_t)tstate->refcounts.size) { |
| 404 |
# ifdef Py_REF_DEBUG |
| 405 |
_Py_DECREF_DecRefTotal(); |
| 406 |
# endif |
| 407 |
_Py_DECREF_STAT_INC(); |
| 408 |
tstate->refcounts.values[idx]--; |
| 409 |
} |
| 410 |
else { |
| 411 |
// Directly decref the object if the id is not assigned or if |
| 412 |
// per-thread refcounting has been disabled on this object. |
| 413 |
Py_DECREF(obj); |
| 414 |
} |
| 415 |
} |
| 416 |
|
| 417 |
static inline void |
| 418 |
_Py_DECREF_TYPE(PyTypeObject *type) |
| 419 |
{ |
| 420 |
if (!_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { |
| 421 |
assert(_Py_IsImmortal(type)); |
| 422 |
_Py_DECREF_IMMORTAL_STAT_INC(); |
| 423 |
return; |
| 424 |
} |
| 425 |
PyHeapTypeObject *ht = (PyHeapTypeObject *)type; |
| 426 |
_Py_THREAD_DECREF_OBJECT((PyObject *)type, ht->unique_id); |
| 427 |
} |
| 428 |
|
| 429 |
static inline void |
| 430 |
_Py_DECREF_CODE(PyCodeObject *co) |
| 431 |
{ |
| 432 |
_Py_THREAD_DECREF_OBJECT((PyObject *)co, co->_co_unique_id); |
| 433 |
} |
| 434 |
#endif |
| 435 |
|
| 436 |
#ifndef Py_GIL_DISABLED |
| 437 |
#ifdef Py_REF_DEBUG |
| 438 |
|
| 439 |
static inline void Py_DECREF_MORTAL(const char *filename, int lineno, PyObject *op) |
| 440 |
{ |
| 441 |
if (op->ob_refcnt <= 0) { |
| 442 |
_Py_NegativeRefcount(filename, lineno, op); |
| 443 |
} |
| 444 |
_Py_DECREF_STAT_INC(); |
| 445 |
assert(!_Py_IsStaticImmortal(op)); |
| 446 |
if (!_Py_IsImmortal(op)) { |
| 447 |
_Py_DECREF_DecRefTotal(); |
| 448 |
} |
| 449 |
if (--op->ob_refcnt == 0) { |
| 450 |
_Py_Dealloc(op); |
| 451 |
} |
| 452 |
} |
| 453 |
#define Py_DECREF_MORTAL(op) Py_DECREF_MORTAL(__FILE__, __LINE__, _PyObject_CAST(op)) |
| 454 |
|
| 455 |
static inline void _Py_DECREF_MORTAL_SPECIALIZED(const char *filename, int lineno, PyObject *op, destructor destruct) |
| 456 |
{ |
| 457 |
if (op->ob_refcnt <= 0) { |
| 458 |
_Py_NegativeRefcount(filename, lineno, op); |
| 459 |
} |
| 460 |
_Py_DECREF_STAT_INC(); |
| 461 |
assert(!_Py_IsStaticImmortal(op)); |
| 462 |
if (!_Py_IsImmortal(op)) { |
| 463 |
_Py_DECREF_DecRefTotal(); |
| 464 |
} |
| 465 |
if (--op->ob_refcnt == 0) { |
| 466 |
#ifdef Py_TRACE_REFS |
| 467 |
_Py_ForgetReference(op); |
| 468 |
#endif |
| 469 |
_PyReftracerTrack(op, PyRefTracer_DESTROY); |
| 470 |
destruct(op); |
| 471 |
} |
| 472 |
} |
| 473 |
#define Py_DECREF_MORTAL_SPECIALIZED(op, destruct) _Py_DECREF_MORTAL_SPECIALIZED(__FILE__, __LINE__, op, destruct) |
| 474 |
|
| 475 |
#else |
| 476 |
|
| 477 |
static inline void Py_DECREF_MORTAL(PyObject *op) |
| 478 |
{ |
| 479 |
assert(!_Py_IsStaticImmortal(op)); |
| 480 |
_Py_DECREF_STAT_INC(); |
| 481 |
if (--op->ob_refcnt == 0) { |
| 482 |
_Py_Dealloc(op); |
| 483 |
} |
| 484 |
} |
| 485 |
#define Py_DECREF_MORTAL(op) Py_DECREF_MORTAL(_PyObject_CAST(op)) |
| 486 |
|
| 487 |
static inline void Py_DECREF_MORTAL_SPECIALIZED(PyObject *op, destructor destruct) |
| 488 |
{ |
| 489 |
assert(!_Py_IsStaticImmortal(op)); |
| 490 |
_Py_DECREF_STAT_INC(); |
| 491 |
if (--op->ob_refcnt == 0) { |
| 492 |
_PyReftracerTrack(op, PyRefTracer_DESTROY); |
| 493 |
destruct(op); |
| 494 |
} |
| 495 |
} |
| 496 |
#define Py_DECREF_MORTAL_SPECIALIZED(op, destruct) Py_DECREF_MORTAL_SPECIALIZED(_PyObject_CAST(op), destruct) |
| 497 |
|
| 498 |
#endif |
| 499 |
#endif |
| 500 |
|
| 501 |
/* Inline functions trading binary compatibility for speed: |
| 502 |
_PyObject_Init() is the fast version of PyObject_Init(), and |
| 503 |
_PyObject_InitVar() is the fast version of PyObject_InitVar(). |
| 504 |
|
| 505 |
These inline functions must not be called with op=NULL. */ |
| 506 |
static inline void |
| 507 |
_PyObject_Init(PyObject *op, PyTypeObject *typeobj) |
| 508 |
{ |
| 509 |
assert(op != NULL); |
| 510 |
Py_SET_TYPE(op, typeobj); |
| 511 |
assert(_PyType_HasFeature(typeobj, Py_TPFLAGS_HEAPTYPE) || _Py_IsImmortal(typeobj)); |
| 512 |
_Py_INCREF_TYPE(typeobj); |
| 513 |
_Py_NewReference(op); |
| 514 |
} |
| 515 |
|
| 516 |
static inline void |
| 517 |
_PyObject_InitVar(PyVarObject *op, PyTypeObject *typeobj, Py_ssize_t size) |
| 518 |
{ |
| 519 |
assert(op != NULL); |
| 520 |
assert(typeobj != &PyLong_Type); |
| 521 |
_PyObject_Init((PyObject *)op, typeobj); |
| 522 |
Py_SET_SIZE(op, size); |
| 523 |
} |
| 524 |
|
| 525 |
// Macros to accept any type for the parameter, and to automatically pass |
| 526 |
// the filename and the filename (if NDEBUG is not defined) where the macro |
| 527 |
// is called. |
| 528 |
#ifdef NDEBUG |
| 529 |
# define _PyObject_GC_TRACK(op) \ |
| 530 |
_PyObject_GC_TRACK(_PyObject_CAST(op)) |
| 531 |
# define _PyObject_GC_UNTRACK(op) \ |
| 532 |
_PyObject_GC_UNTRACK(_PyObject_CAST(op)) |
| 533 |
#else |
| 534 |
# define _PyObject_GC_TRACK(op) \ |
| 535 |
_PyObject_GC_TRACK(__FILE__, __LINE__, _PyObject_CAST(op)) |
| 536 |
# define _PyObject_GC_UNTRACK(op) \ |
| 537 |
_PyObject_GC_UNTRACK(__FILE__, __LINE__, _PyObject_CAST(op)) |
| 538 |
#endif |
| 539 |
|
| 540 |
#ifdef Py_GIL_DISABLED |
| 541 |
|
| 542 |
/* Tries to increment an object's reference count |
| 543 |
* |
| 544 |
* This is a specialized version of _Py_TryIncref that only succeeds if the |
| 545 |
* object is immortal or local to this thread. It does not handle the case |
| 546 |
* where the reference count modification requires an atomic operation. This |
| 547 |
* allows call sites to specialize for the immortal/local case. |
| 548 |
*/ |
| 549 |
static inline int |
| 550 |
_Py_TryIncrefFast(PyObject *op) { |
| 551 |
uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local); |
| 552 |
local += 1; |
| 553 |
if (local == 0) { |
| 554 |
// immortal |
| 555 |
_Py_INCREF_IMMORTAL_STAT_INC(); |
| 556 |
return 1; |
| 557 |
} |
| 558 |
if (_Py_IsOwnedByCurrentThread(op)) { |
| 559 |
_Py_INCREF_STAT_INC(); |
| 560 |
_Py_atomic_store_uint32_relaxed(&op->ob_ref_local, local); |
| 561 |
#ifdef Py_REF_DEBUG |
| 562 |
_Py_IncRefTotal(_PyThreadState_GET()); |
| 563 |
#endif |
| 564 |
return 1; |
| 565 |
} |
| 566 |
return 0; |
| 567 |
} |
| 568 |
|
| 569 |
static inline int |
| 570 |
_Py_TryIncRefShared(PyObject *op) |
| 571 |
{ |
| 572 |
Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared); |
| 573 |
for (;;) { |
| 574 |
// If the shared refcount is zero and the object is either merged |
| 575 |
// or may not have weak references, then we cannot incref it. |
| 576 |
if (shared == 0 || shared == _Py_REF_MERGED) { |
| 577 |
return 0; |
| 578 |
} |
| 579 |
|
| 580 |
if (_Py_atomic_compare_exchange_ssize( |
| 581 |
&op->ob_ref_shared, |
| 582 |
&shared, |
| 583 |
shared + (1 << _Py_REF_SHARED_SHIFT))) { |
| 584 |
#ifdef Py_REF_DEBUG |
| 585 |
_Py_IncRefTotal(_PyThreadState_GET()); |
| 586 |
#endif |
| 587 |
_Py_INCREF_STAT_INC(); |
| 588 |
return 1; |
| 589 |
} |
| 590 |
} |
| 591 |
} |
| 592 |
|
| 593 |
/* Tries to incref the object op and ensures that *src still points to it. */ |
| 594 |
static inline int |
| 595 |
_Py_TryIncrefCompare(PyObject **src, PyObject *op) |
| 596 |
{ |
| 597 |
if (_Py_TryIncrefFast(op)) { |
| 598 |
return 1; |
| 599 |
} |
| 600 |
if (!_Py_TryIncRefShared(op)) { |
| 601 |
return 0; |
| 602 |
} |
| 603 |
if (op != _Py_atomic_load_ptr(src)) { |
| 604 |
Py_DECREF(op); |
| 605 |
return 0; |
| 606 |
} |
| 607 |
return 1; |
| 608 |
} |
| 609 |
|
| 610 |
/* Loads and increfs an object from ptr, which may contain a NULL value. |
| 611 |
Safe with concurrent (atomic) updates to ptr. |
| 612 |
NOTE: The writer must set maybe-weakref on the stored object! */ |
| 613 |
static inline PyObject * |
| 614 |
_Py_XGetRef(PyObject **ptr) |
| 615 |
{ |
| 616 |
for (;;) { |
| 617 |
PyObject *value = _PyObject_CAST(_Py_atomic_load_ptr(ptr)); |
| 618 |
if (value == NULL) { |
| 619 |
return value; |
| 620 |
} |
| 621 |
if (_Py_TryIncrefCompare(ptr, value)) { |
| 622 |
return value; |
| 623 |
} |
| 624 |
} |
| 625 |
} |
| 626 |
|
| 627 |
/* Attempts to loads and increfs an object from ptr. Returns NULL |
| 628 |
on failure, which may be due to a NULL value or a concurrent update. */ |
| 629 |
static inline PyObject * |
| 630 |
_Py_TryXGetRef(PyObject **ptr) |
| 631 |
{ |
| 632 |
PyObject *value = _PyObject_CAST(_Py_atomic_load_ptr(ptr)); |
| 633 |
if (value == NULL) { |
| 634 |
return value; |
| 635 |
} |
| 636 |
if (_Py_TryIncrefCompare(ptr, value)) { |
| 637 |
return value; |
| 638 |
} |
| 639 |
return NULL; |
| 640 |
} |
| 641 |
|
| 642 |
/* Like Py_NewRef but also optimistically sets _Py_REF_MAYBE_WEAKREF |
| 643 |
on objects owned by a different thread. */ |
| 644 |
static inline PyObject * |
| 645 |
_Py_NewRefWithLock(PyObject *op) |
| 646 |
{ |
| 647 |
if (_Py_TryIncrefFast(op)) { |
| 648 |
return op; |
| 649 |
} |
| 650 |
#ifdef Py_REF_DEBUG |
| 651 |
_Py_IncRefTotal(_PyThreadState_GET()); |
| 652 |
#endif |
| 653 |
_Py_INCREF_STAT_INC(); |
| 654 |
for (;;) { |
| 655 |
Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared); |
| 656 |
Py_ssize_t new_shared = shared + (1 << _Py_REF_SHARED_SHIFT); |
| 657 |
if ((shared & _Py_REF_SHARED_FLAG_MASK) == 0) { |
| 658 |
new_shared |= _Py_REF_MAYBE_WEAKREF; |
| 659 |
} |
| 660 |
if (_Py_atomic_compare_exchange_ssize( |
| 661 |
&op->ob_ref_shared, |
| 662 |
&shared, |
| 663 |
new_shared)) { |
| 664 |
return op; |
| 665 |
} |
| 666 |
} |
| 667 |
} |
| 668 |
|
| 669 |
static inline PyObject * |
| 670 |
_Py_XNewRefWithLock(PyObject *obj) |
| 671 |
{ |
| 672 |
if (obj == NULL) { |
| 673 |
return NULL; |
| 674 |
} |
| 675 |
return _Py_NewRefWithLock(obj); |
| 676 |
} |
| 677 |
|
| 678 |
static inline void |
| 679 |
_PyObject_SetMaybeWeakref(PyObject *op) |
| 680 |
{ |
| 681 |
if (_Py_IsImmortal(op)) { |
| 682 |
return; |
| 683 |
} |
| 684 |
for (;;) { |
| 685 |
Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared); |
| 686 |
if ((shared & _Py_REF_SHARED_FLAG_MASK) != 0) { |
| 687 |
// Nothing to do if it's in WEAKREFS, QUEUED, or MERGED states. |
| 688 |
return; |
| 689 |
} |
| 690 |
if (_Py_atomic_compare_exchange_ssize( |
| 691 |
&op->ob_ref_shared, &shared, shared | _Py_REF_MAYBE_WEAKREF)) { |
| 692 |
return; |
| 693 |
} |
| 694 |
} |
| 695 |
} |
| 696 |
|
| 697 |
extern PyAPI_FUNC(int) _PyObject_ResurrectEndSlow(PyObject *op); |
| 698 |
#endif |
| 699 |
|
| 700 |
// Temporarily resurrects an object during deallocation. The refcount is set |
| 701 |
// to one. |
| 702 |
static inline void |
| 703 |
_PyObject_ResurrectStart(PyObject *op) |
| 704 |
{ |
| 705 |
assert(Py_REFCNT(op) == 0); |
| 706 |
#ifdef Py_REF_DEBUG |
| 707 |
_Py_IncRefTotal(_PyThreadState_GET()); |
| 708 |
#endif |
| 709 |
#ifdef Py_GIL_DISABLED |
| 710 |
_Py_atomic_store_uintptr_relaxed(&op->ob_tid, _Py_ThreadId()); |
| 711 |
_Py_atomic_store_uint32_relaxed(&op->ob_ref_local, 1); |
| 712 |
_Py_atomic_store_ssize_relaxed(&op->ob_ref_shared, 0); |
| 713 |
#else |
| 714 |
Py_SET_REFCNT(op, 1); |
| 715 |
#endif |
| 716 |
#ifdef Py_TRACE_REFS |
| 717 |
_Py_ResurrectReference(op); |
| 718 |
#endif |
| 719 |
} |
| 720 |
|
| 721 |
// Undoes an object resurrection by decrementing the refcount without calling |
| 722 |
// _Py_Dealloc(). Returns 0 if the object is dead (the normal case), and |
| 723 |
// deallocation should continue. Returns 1 if the object is still alive. |
| 724 |
static inline int |
| 725 |
_PyObject_ResurrectEnd(PyObject *op) |
| 726 |
{ |
| 727 |
#ifdef Py_REF_DEBUG |
| 728 |
_Py_DecRefTotal(_PyThreadState_GET()); |
| 729 |
#endif |
| 730 |
#ifndef Py_GIL_DISABLED |
| 731 |
Py_SET_REFCNT(op, Py_REFCNT(op) - 1); |
| 732 |
if (Py_REFCNT(op) == 0) { |
| 733 |
# ifdef Py_TRACE_REFS |
| 734 |
_Py_ForgetReference(op); |
| 735 |
# endif |
| 736 |
return 0; |
| 737 |
} |
| 738 |
return 1; |
| 739 |
#else |
| 740 |
uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local); |
| 741 |
Py_ssize_t shared = _Py_atomic_load_ssize_acquire(&op->ob_ref_shared); |
| 742 |
if (_Py_IsOwnedByCurrentThread(op) && local == 1 && shared == 0) { |
| 743 |
// Fast-path: object has a single refcount and is owned by this thread |
| 744 |
_Py_atomic_store_uint32_relaxed(&op->ob_ref_local, 0); |
| 745 |
# ifdef Py_TRACE_REFS |
| 746 |
_Py_ForgetReference(op); |
| 747 |
# endif |
| 748 |
return 0; |
| 749 |
} |
| 750 |
// Slow-path: object has a shared refcount or is not owned by this thread |
| 751 |
return _PyObject_ResurrectEndSlow(op); |
| 752 |
#endif |
| 753 |
} |
| 754 |
|
| 755 |
/* Tries to incref op and returns 1 if successful or 0 otherwise. */ |
| 756 |
static inline int |
| 757 |
_Py_TryIncref(PyObject *op) |
| 758 |
{ |
| 759 |
#ifdef Py_GIL_DISABLED |
| 760 |
return _Py_TryIncrefFast(op) || _Py_TryIncRefShared(op); |
| 761 |
#else |
| 762 |
if (Py_REFCNT(op) > 0) { |
| 763 |
Py_INCREF(op); |
| 764 |
return 1; |
| 765 |
} |
| 766 |
return 0; |
| 767 |
#endif |
| 768 |
} |
| 769 |
|
| 770 |
#ifdef Py_REF_DEBUG |
| 771 |
extern void _PyInterpreterState_FinalizeRefTotal(PyInterpreterState *); |
| 772 |
extern void _Py_FinalizeRefTotal(_PyRuntimeState *); |
| 773 |
extern void _PyDebug_PrintTotalRefs(void); |
| 774 |
#endif |
| 775 |
|
| 776 |
#ifdef Py_TRACE_REFS |
| 777 |
extern void _Py_AddToAllObjects(PyObject *op); |
| 778 |
extern void _Py_PrintReferences(PyInterpreterState *, FILE *); |
| 779 |
extern void _Py_PrintReferenceAddresses(PyInterpreterState *, FILE *); |
| 780 |
#endif |
| 781 |
|
| 782 |
|
| 783 |
/* Return the *address* of the object's weaklist. The address may be |
| 784 |
* dereferenced to get the current head of the weaklist. This is useful |
| 785 |
* for iterating over the linked list of weakrefs, especially when the |
| 786 |
* list is being modified externally (e.g. refs getting removed). |
| 787 |
* |
| 788 |
* The returned pointer should not be used to change the head of the list |
| 789 |
* nor should it be used to add, remove, or swap any refs in the list. |
| 790 |
* That is the sole responsibility of the code in weakrefobject.c. |
| 791 |
*/ |
| 792 |
static inline PyObject ** |
| 793 |
_PyObject_GET_WEAKREFS_LISTPTR(PyObject *op) |
| 794 |
{ |
| 795 |
if (PyType_Check(op) && |
| 796 |
((PyTypeObject *)op)->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { |
| 797 |
PyInterpreterState *interp = _PyInterpreterState_GET(); |
| 798 |
managed_static_type_state *state = _PyStaticType_GetState( |
| 799 |
interp, (PyTypeObject *)op); |
| 800 |
return _PyStaticType_GET_WEAKREFS_LISTPTR(state); |
| 801 |
} |
| 802 |
// Essentially _PyObject_GET_WEAKREFS_LISTPTR_FROM_OFFSET(): |
| 803 |
Py_ssize_t offset = Py_TYPE(op)->tp_weaklistoffset; |
| 804 |
return (PyObject **)((char *)op + offset); |
| 805 |
} |
| 806 |
|
| 807 |
/* This is a special case of _PyObject_GET_WEAKREFS_LISTPTR(). |
| 808 |
* Only the most fundamental lookup path is used. |
| 809 |
* Consequently, static types should not be used. |
| 810 |
* |
| 811 |
* For static builtin types the returned pointer will always point |
| 812 |
* to a NULL tp_weaklist. This is fine for any deallocation cases, |
| 813 |
* since static types are never deallocated and static builtin types |
| 814 |
* are only finalized at the end of runtime finalization. |
| 815 |
* |
| 816 |
* If the weaklist for static types is actually needed then use |
| 817 |
* _PyObject_GET_WEAKREFS_LISTPTR(). |
| 818 |
*/ |
| 819 |
static inline PyWeakReference ** |
| 820 |
_PyObject_GET_WEAKREFS_LISTPTR_FROM_OFFSET(PyObject *op) |
| 821 |
{ |
| 822 |
assert(!PyType_Check(op) || |
| 823 |
((PyTypeObject *)op)->tp_flags & Py_TPFLAGS_HEAPTYPE); |
| 824 |
Py_ssize_t offset = Py_TYPE(op)->tp_weaklistoffset; |
| 825 |
return (PyWeakReference **)((char *)op + offset); |
| 826 |
} |
| 827 |
|
| 828 |
// Fast inlined version of PyType_IS_GC() |
| 829 |
#define _PyType_IS_GC(t) _PyType_HasFeature((t), Py_TPFLAGS_HAVE_GC) |
| 830 |
|
| 831 |
// Fast inlined version of PyObject_IS_GC() |
| 832 |
static inline int |
| 833 |
_PyObject_IS_GC(PyObject *obj) |
| 834 |
{ |
| 835 |
PyTypeObject *type = Py_TYPE(obj); |
| 836 |
return (_PyType_IS_GC(type) |
| 837 |
&& (type->tp_is_gc == NULL || type->tp_is_gc(obj))); |
| 838 |
} |
| 839 |
|
| 840 |
// Fast inlined version of PyObject_Hash() |
| 841 |
static inline Py_hash_t |
| 842 |
_PyObject_HashFast(PyObject *op) |
| 843 |
{ |
| 844 |
if (PyUnicode_CheckExact(op)) { |
| 845 |
Py_hash_t hash = FT_ATOMIC_LOAD_SSIZE_RELAXED( |
| 846 |
_PyASCIIObject_CAST(op)->hash); |
| 847 |
if (hash != -1) { |
| 848 |
return hash; |
| 849 |
} |
| 850 |
} |
| 851 |
return PyObject_Hash(op); |
| 852 |
} |
| 853 |
|
| 854 |
static inline size_t |
| 855 |
_PyType_PreHeaderSize(PyTypeObject *tp) |
| 856 |
{ |
| 857 |
return ( |
| 858 |
#ifndef Py_GIL_DISABLED |
| 859 |
(size_t)_PyType_IS_GC(tp) * sizeof(PyGC_Head) + |
| 860 |
#endif |
| 861 |
(size_t)_PyType_HasFeature(tp, Py_TPFLAGS_PREHEADER) * 2 * sizeof(PyObject *) |
| 862 |
); |
| 863 |
} |
| 864 |
|
| 865 |
void _PyObject_GC_Link(PyObject *op); |
| 866 |
|
| 867 |
// Usage: assert(_Py_CheckSlotResult(obj, "__getitem__", result != NULL)); |
| 868 |
extern int _Py_CheckSlotResult( |
| 869 |
PyObject *obj, |
| 870 |
const char *slot_name, |
| 871 |
int success); |
| 872 |
|
| 873 |
// Test if a type supports weak references |
| 874 |
static inline int _PyType_SUPPORTS_WEAKREFS(PyTypeObject *type) { |
| 875 |
return (type->tp_weaklistoffset != 0); |
| 876 |
} |
| 877 |
|
| 878 |
extern PyObject* _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t nitems); |
| 879 |
PyAPI_FUNC(PyObject *) _PyType_NewManagedObject(PyTypeObject *type); |
| 880 |
|
| 881 |
extern PyTypeObject* _PyType_CalculateMetaclass(PyTypeObject *, PyObject *); |
| 882 |
extern PyObject* _PyType_GetDocFromInternalDoc(const char *, const char *); |
| 883 |
extern PyObject* _PyType_GetTextSignatureFromInternalDoc(const char *, const char *, int); |
| 884 |
extern int _PyObject_SetAttributeErrorContext(PyObject *v, PyObject* name); |
| 885 |
|
| 886 |
void _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp); |
| 887 |
extern int _PyObject_StoreInstanceAttribute(PyObject *obj, |
| 888 |
PyObject *name, PyObject *value); |
| 889 |
extern bool _PyObject_TryGetInstanceAttribute(PyObject *obj, PyObject *name, |
| 890 |
PyObject **attr); |
| 891 |
extern PyObject *_PyType_LookupRefAndVersion(PyTypeObject *, PyObject *, |
| 892 |
unsigned int *); |
| 893 |
|
| 894 |
// Internal API to look for a name through the MRO. |
| 895 |
// This stores a stack reference in out and returns the value of |
| 896 |
// type->tp_version or zero if name is missing. It doesn't set an exception! |
| 897 |
extern unsigned int |
| 898 |
_PyType_LookupStackRefAndVersion(PyTypeObject *type, PyObject *name, _PyStackRef *out); |
| 899 |
|
| 900 |
extern int _PyObject_GetMethodStackRef(PyThreadState *ts, _PyStackRef *self, |
| 901 |
PyObject *name, _PyStackRef *method); |
| 902 |
|
| 903 |
// Cache the provided init method in the specialization cache of type if the |
| 904 |
// provided type version matches the current version of the type. |
| 905 |
// |
| 906 |
// The cached value is borrowed and is only valid if guarded by a type |
| 907 |
// version check. In free-threaded builds the init method must also use |
| 908 |
// deferred reference counting. |
| 909 |
// |
| 910 |
// Returns 1 if the value was cached or 0 otherwise. |
| 911 |
extern int _PyType_CacheInitForSpecialization(PyHeapTypeObject *type, |
| 912 |
PyObject *init, |
| 913 |
unsigned int tp_version); |
| 914 |
|
| 915 |
#ifdef Py_GIL_DISABLED |
| 916 |
# define MANAGED_DICT_OFFSET (((Py_ssize_t)sizeof(PyObject *))*-1) |
| 917 |
# define MANAGED_WEAKREF_OFFSET (((Py_ssize_t)sizeof(PyObject *))*-2) |
| 918 |
#else |
| 919 |
# define MANAGED_DICT_OFFSET (((Py_ssize_t)sizeof(PyObject *))*-3) |
| 920 |
# define MANAGED_WEAKREF_OFFSET (((Py_ssize_t)sizeof(PyObject *))*-4) |
| 921 |
#endif |
| 922 |
|
| 923 |
typedef union { |
| 924 |
PyDictObject *dict; |
| 925 |
} PyManagedDictPointer; |
| 926 |
|
| 927 |
static inline PyManagedDictPointer * |
| 928 |
_PyObject_ManagedDictPointer(PyObject *obj) |
| 929 |
{ |
| 930 |
assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT); |
| 931 |
return (PyManagedDictPointer *)((char *)obj + MANAGED_DICT_OFFSET); |
| 932 |
} |
| 933 |
|
| 934 |
static inline PyDictObject * |
| 935 |
_PyObject_GetManagedDict(PyObject *obj) |
| 936 |
{ |
| 937 |
PyManagedDictPointer *dorv = _PyObject_ManagedDictPointer(obj); |
| 938 |
return (PyDictObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(dorv->dict); |
| 939 |
} |
| 940 |
|
| 941 |
static inline PyDictValues * |
| 942 |
_PyObject_InlineValues(PyObject *obj) |
| 943 |
{ |
| 944 |
PyTypeObject *tp = Py_TYPE(obj); |
| 945 |
assert(tp->tp_basicsize > 0 && (size_t)tp->tp_basicsize % sizeof(PyObject *) == 0); |
| 946 |
assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES); |
| 947 |
assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT); |
| 948 |
return (PyDictValues *)((char *)obj + tp->tp_basicsize); |
| 949 |
} |
| 950 |
|
| 951 |
extern PyObject ** _PyObject_ComputedDictPointer(PyObject *); |
| 952 |
extern int _PyObject_IsInstanceDictEmpty(PyObject *); |
| 953 |
|
| 954 |
// Export for 'math' shared extension |
| 955 |
PyAPI_FUNC(PyObject*) _PyObject_LookupSpecial(PyObject *, PyObject *); |
| 956 |
PyAPI_FUNC(int) _PyObject_LookupSpecialMethod(PyObject *attr, _PyStackRef *method_and_self); |
| 957 |
|
| 958 |
// Calls the method named `attr` on `self`, but does not set an exception if |
| 959 |
// the attribute does not exist. |
| 960 |
PyAPI_FUNC(PyObject *) |
| 961 |
_PyObject_MaybeCallSpecialNoArgs(PyObject *self, PyObject *attr); |
| 962 |
|
| 963 |
PyAPI_FUNC(PyObject *) |
| 964 |
_PyObject_MaybeCallSpecialOneArg(PyObject *self, PyObject *attr, PyObject *arg); |
| 965 |
|
| 966 |
extern int _PyObject_IsAbstract(PyObject *); |
| 967 |
|
| 968 |
PyAPI_FUNC(int) _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method); |
| 969 |
extern PyObject* _PyObject_NextNotImplemented(PyObject *); |
| 970 |
|
| 971 |
// Pickle support. |
| 972 |
// Export for '_datetime' shared extension |
| 973 |
PyAPI_FUNC(PyObject*) _PyObject_GetState(PyObject *); |
| 974 |
|
| 975 |
/* C function call trampolines to mitigate bad function pointer casts. |
| 976 |
* |
| 977 |
* Typical native ABIs ignore additional arguments or fill in missing |
| 978 |
* values with 0/NULL in function pointer cast. Compilers do not show |
| 979 |
* warnings when a function pointer is explicitly casted to an |
| 980 |
* incompatible type. |
| 981 |
* |
| 982 |
* Bad fpcasts are an issue in WebAssembly. WASM's indirect_call has strict |
| 983 |
* function signature checks. Argument count, types, and return type must |
| 984 |
* match. |
| 985 |
* |
| 986 |
* Third party code unintentionally rely on problematic fpcasts. The call |
| 987 |
* trampoline mitigates common occurrences of bad fpcasts on Emscripten. |
| 988 |
*/ |
| 989 |
#if !(defined(__EMSCRIPTEN__) && defined(PY_CALL_TRAMPOLINE)) |
| 990 |
#define _PyCFunction_TrampolineCall(meth, self, args) \ |
| 991 |
(meth)((self), (args)) |
| 992 |
#define _PyCFunctionWithKeywords_TrampolineCall(meth, self, args, kw) \ |
| 993 |
(meth)((self), (args), (kw)) |
| 994 |
#endif // __EMSCRIPTEN__ && PY_CALL_TRAMPOLINE |
| 995 |
|
| 996 |
// Export these 2 symbols for '_pickle' shared extension |
| 997 |
PyAPI_DATA(PyTypeObject) _PyNone_Type; |
| 998 |
PyAPI_DATA(PyTypeObject) _PyNotImplemented_Type; |
| 999 |
|
| 1000 |
// Maps Py_LT to Py_GT, ..., Py_GE to Py_LE. |
| 1001 |
// Export for the stable ABI. |
| 1002 |
PyAPI_DATA(int) _Py_SwappedOp[]; |
| 1003 |
|
| 1004 |
extern void _Py_GetConstant_Init(void); |
| 1005 |
|
| 1006 |
enum _PyAnnotateFormat { |
| 1007 |
_Py_ANNOTATE_FORMAT_VALUE = 1, |
| 1008 |
_Py_ANNOTATE_FORMAT_VALUE_WITH_FAKE_GLOBALS = 2, |
| 1009 |
_Py_ANNOTATE_FORMAT_FORWARDREF = 3, |
| 1010 |
_Py_ANNOTATE_FORMAT_STRING = 4, |
| 1011 |
}; |
| 1012 |
|
| 1013 |
int _PyObject_SetDict(PyObject *obj, PyObject *value); |
| 1014 |
|
| 1015 |
#ifndef Py_GIL_DISABLED |
| 1016 |
static inline Py_ALWAYS_INLINE void _Py_INCREF_MORTAL(PyObject *op) |
| 1017 |
{ |
| 1018 |
assert(!_Py_IsStaticImmortal(op)); |
| 1019 |
op->ob_refcnt++; |
| 1020 |
_Py_INCREF_STAT_INC(); |
| 1021 |
#if defined(Py_REF_DEBUG) && !defined(Py_LIMITED_API) |
| 1022 |
if (!_Py_IsImmortal(op)) { |
| 1023 |
_Py_INCREF_IncRefTotal(); |
| 1024 |
} |
| 1025 |
#endif |
| 1026 |
} |
| 1027 |
#endif |
| 1028 |
|
| 1029 |
#ifdef __cplusplus |
| 1030 |
} |
| 1031 |
#endif |
| 1032 |
#endif /* !Py_INTERNAL_OBJECT_H */ |