Debian Codegraph @

Include/internal/pycore_object.h

raw
1 #ifndef Py_INTERNAL_OBJECT_H
2 #define Py_INTERNAL_OBJECT_H
3 #ifdef __cplusplus
4 extern "C" {
5 #endif
6
7 #ifndef Py_BUILD_CORE
8 # error "this header requires Py_BUILD_CORE define"
9 #endif
10
11 #include "pycore_emscripten_trampoline.h" // _PyCFunction_TrampolineCall()
12 #include "pycore_gc.h" // _PyObject_GC_TRACK()
13 #include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_PTR_ACQUIRE()
14 #include "pycore_pystate.h" // _PyInterpreterState_GET()
15 #include "pycore_runtime.h" // _PyRuntime
16 #include "pycore_typeobject.h" // _PyStaticType_GetState()
17 #include "pycore_uniqueid.h" // _PyObject_ThreadIncrefSlow()
18
19 #include <stdbool.h> // bool
20
21
22 // This value is added to `ob_ref_shared` for objects that use deferred
23 // reference counting so that they are not immediately deallocated when the
24 // non-deferred reference count drops to zero.
25 //
26 // The value is half the maximum shared refcount because the low two bits of
27 // `ob_ref_shared` are used for flags.
28 #define _Py_REF_DEFERRED (PY_SSIZE_T_MAX / 8)
29
30 /* For backwards compatibility -- Do not use this */
31 #define _Py_IsImmortalLoose(op) _Py_IsImmortal
32
33
34 /* Check if an object is consistent. For example, ensure that the reference
35 counter is greater than or equal to 1, and ensure that ob_type is not NULL.
36
37 Call _PyObject_AssertFailed() if the object is inconsistent.
38
39 If check_content is zero, only check header fields: reduce the overhead.
40
41 The function always return 1. The return value is just here to be able to
42 write:
43
44 assert(_PyObject_CheckConsistency(obj, 1)); */
45 extern int _PyObject_CheckConsistency(PyObject *op, int check_content);
46
47 extern void _PyDebugAllocatorStats(FILE *out, const char *block_name,
48 int num_blocks, size_t sizeof_block);
49
50 extern void _PyObject_DebugTypeStats(FILE *out);
51
52 #ifdef Py_TRACE_REFS
53 // Forget a reference registered by _Py_NewReference(). Function called by
54 // _Py_Dealloc().
55 //
56 // On a free list, the function can be used before modifying an object to
57 // remove the object from traced objects. Then _Py_NewReference() or
58 // _Py_NewReferenceNoTotal() should be called again on the object to trace
59 // it again.
60 extern void _Py_ForgetReference(PyObject *);
61 #endif
62
63 // Export for shared _testinternalcapi extension
64 PyAPI_FUNC(int) _PyObject_IsFreed(PyObject *);
65
66 /* We need to maintain an internal copy of Py{Var}Object_HEAD_INIT to avoid
67 designated initializer conflicts in C++20. If we use the definition in
68 object.h, we will be mixing designated and non-designated initializers in
69 pycore objects which is forbiddent in C++20. However, if we then use
70 designated initializers in object.h then Extensions without designated break.
71 Furthermore, we can't use designated initializers in Extensions since these
72 are not supported pre-C++20. Thus, keeping an internal copy here is the most
73 backwards compatible solution */
74 #if defined(Py_GIL_DISABLED)
75 #define _PyObject_HEAD_INIT(type) \
76 { \
77 .ob_ref_local = _Py_IMMORTAL_REFCNT_LOCAL, \
78 .ob_flags = _Py_STATICALLY_ALLOCATED_FLAG, \
79 .ob_gc_bits = _PyGC_BITS_DEFERRED, \
80 .ob_type = (type) \
81 }
82 #else
83 #if SIZEOF_VOID_P > 4
84 #define _PyObject_HEAD_INIT(type) \
85 { \
86 .ob_refcnt = _Py_IMMORTAL_INITIAL_REFCNT, \
87 .ob_flags = _Py_STATIC_FLAG_BITS, \
88 .ob_type = (type) \
89 }
90 #else
91 #define _PyObject_HEAD_INIT(type) \
92 { \
93 .ob_refcnt = _Py_STATIC_IMMORTAL_INITIAL_REFCNT, \
94 .ob_type = (type) \
95 }
96 #endif
97 #endif
98 #define _PyVarObject_HEAD_INIT(type, size) \
99 { \
100 .ob_base = _PyObject_HEAD_INIT(type), \
101 .ob_size = size \
102 }
103
104 PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalRefcountErrorFunc(
105 const char *func,
106 const char *message);
107
108 #define _Py_FatalRefcountError(message) \
109 _Py_FatalRefcountErrorFunc(__func__, (message))
110
111 #define _PyReftracerTrack(obj, operation) \
112 do { \
113 struct _reftracer_runtime_state *tracer = &_PyRuntime.ref_tracer; \
114 if (tracer->tracer_func != NULL) { \
115 void *data = tracer->tracer_data; \
116 tracer->tracer_func((obj), (operation), data); \
117 } \
118 } while(0)
119
120 #ifdef Py_REF_DEBUG
121 /* The symbol is only exposed in the API for the sake of extensions
122 built against the pre-3.12 stable ABI. */
123 PyAPI_DATA(Py_ssize_t) _Py_RefTotal;
124
125 extern void _Py_AddRefTotal(PyThreadState *, Py_ssize_t);
126 extern PyAPI_FUNC(void) _Py_IncRefTotal(PyThreadState *);
127 extern PyAPI_FUNC(void) _Py_DecRefTotal(PyThreadState *);
128
129 # define _Py_DEC_REFTOTAL(interp) \
130 interp->object_state.reftotal--
131 #endif
132
133 // Increment reference count by n
134 static inline void _Py_RefcntAdd(PyObject* op, Py_ssize_t n)
135 {
136 if (_Py_IsImmortal(op)) {
137 _Py_INCREF_IMMORTAL_STAT_INC();
138 return;
139 }
140 #ifndef Py_GIL_DISABLED
141 Py_ssize_t refcnt = _Py_REFCNT(op);
142 Py_ssize_t new_refcnt = refcnt + n;
143 if (new_refcnt >= (Py_ssize_t)_Py_IMMORTAL_MINIMUM_REFCNT) {
144 new_refcnt = _Py_IMMORTAL_INITIAL_REFCNT;
145 }
146 # if SIZEOF_VOID_P > 4
147 op->ob_refcnt = (PY_UINT32_T)new_refcnt;
148 # else
149 op->ob_refcnt = new_refcnt;
150 # endif
151 # ifdef Py_REF_DEBUG
152 _Py_AddRefTotal(_PyThreadState_GET(), new_refcnt - refcnt);
153 # endif
154 #else
155 if (_Py_IsOwnedByCurrentThread(op)) {
156 uint32_t local = op->ob_ref_local;
157 Py_ssize_t refcnt = (Py_ssize_t)local + n;
158 # if PY_SSIZE_T_MAX > UINT32_MAX
159 if (refcnt > (Py_ssize_t)UINT32_MAX) {
160 // Make the object immortal if the 32-bit local reference count
161 // would overflow.
162 refcnt = _Py_IMMORTAL_REFCNT_LOCAL;
163 }
164 # endif
165 _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, (uint32_t)refcnt);
166 }
167 else {
168 _Py_atomic_add_ssize(&op->ob_ref_shared, (n << _Py_REF_SHARED_SHIFT));
169 }
170 # ifdef Py_REF_DEBUG
171 _Py_AddRefTotal(_PyThreadState_GET(), n);
172 # endif
173 #endif
174 // Although the ref count was increased by `n` (which may be greater than 1)
175 // it is only a single increment (i.e. addition) operation, so only 1 refcnt
176 // increment operation is counted.
177 _Py_INCREF_STAT_INC();
178 }
179 #define _Py_RefcntAdd(op, n) _Py_RefcntAdd(_PyObject_CAST(op), n)
180
181 // Checks if an object has a single, unique reference. If the caller holds a
182 // unique reference, it may be able to safely modify the object in-place.
183 static inline int
184 _PyObject_IsUniquelyReferenced(PyObject *ob)
185 {
186 #if !defined(Py_GIL_DISABLED)
187 return Py_REFCNT(ob) == 1;
188 #else
189 // NOTE: the entire ob_ref_shared field must be zero, including flags, to
190 // ensure that other threads cannot concurrently create new references to
191 // this object.
192 return (_Py_IsOwnedByCurrentThread(ob) &&
193 _Py_atomic_load_uint32_relaxed(&ob->ob_ref_local) == 1 &&
194 _Py_atomic_load_ssize_relaxed(&ob->ob_ref_shared) == 0);
195 #endif
196 }
197
198 PyAPI_FUNC(void) _Py_SetImmortal(PyObject *op);
199 PyAPI_FUNC(void) _Py_SetImmortalUntracked(PyObject *op);
200
201 // Makes an immortal object mortal again with the specified refcnt. Should only
202 // be used during runtime finalization.
203 static inline void _Py_SetMortal(PyObject *op, short refcnt)
204 {
205 if (op) {
206 assert(_Py_IsImmortal(op));
207 #ifdef Py_GIL_DISABLED
208 op->ob_tid = _Py_UNOWNED_TID;
209 op->ob_ref_local = 0;
210 op->ob_ref_shared = _Py_REF_SHARED(refcnt, _Py_REF_MERGED);
211 #else
212 op->ob_refcnt = refcnt;
213 #endif
214 }
215 }
216
217 /* _Py_ClearImmortal() should only be used during runtime finalization. */
218 static inline void _Py_ClearImmortal(PyObject *op)
219 {
220 if (op) {
221 _Py_SetMortal(op, 1);
222 Py_DECREF(op);
223 }
224 }
225 #define _Py_ClearImmortal(op) \
226 do { \
227 _Py_ClearImmortal(_PyObject_CAST(op)); \
228 op = NULL; \
229 } while (0)
230
231 #if !defined(Py_GIL_DISABLED)
232 static inline void
233 _Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct)
234 {
235 if (_Py_IsImmortal(op)) {
236 _Py_DECREF_IMMORTAL_STAT_INC();
237 return;
238 }
239 _Py_DECREF_STAT_INC();
240 #ifdef Py_REF_DEBUG
241 _Py_DEC_REFTOTAL(PyInterpreterState_Get());
242 #endif
243 if (--op->ob_refcnt != 0) {
244 assert(op->ob_refcnt > 0);
245 }
246 else {
247 #ifdef Py_TRACE_REFS
248 _Py_ForgetReference(op);
249 #endif
250 _PyReftracerTrack(op, PyRefTracer_DESTROY);
251 destruct(op);
252 }
253 }
254
255 static inline void
256 _Py_DECREF_NO_DEALLOC(PyObject *op)
257 {
258 if (_Py_IsImmortal(op)) {
259 _Py_DECREF_IMMORTAL_STAT_INC();
260 return;
261 }
262 _Py_DECREF_STAT_INC();
263 #ifdef Py_REF_DEBUG
264 _Py_DEC_REFTOTAL(PyInterpreterState_Get());
265 #endif
266 op->ob_refcnt--;
267 #ifdef Py_DEBUG
268 if (op->ob_refcnt <= 0) {
269 _Py_FatalRefcountError("Expected a positive remaining refcount");
270 }
271 #endif
272 }
273
274 #else
275 // TODO: implement Py_DECREF specializations for Py_GIL_DISABLED build
276 static inline void
277 _Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct)
278 {
279 Py_DECREF(op);
280 }
281
282 static inline void
283 _Py_DECREF_NO_DEALLOC(PyObject *op)
284 {
285 Py_DECREF(op);
286 }
287
288 static inline int
289 _Py_REF_IS_MERGED(Py_ssize_t ob_ref_shared)
290 {
291 return (ob_ref_shared & _Py_REF_SHARED_FLAG_MASK) == _Py_REF_MERGED;
292 }
293
294 static inline int
295 _Py_REF_IS_QUEUED(Py_ssize_t ob_ref_shared)
296 {
297 return (ob_ref_shared & _Py_REF_SHARED_FLAG_MASK) == _Py_REF_QUEUED;
298 }
299
300 // Merge the local and shared reference count fields and add `extra` to the
301 // refcount when merging.
302 Py_ssize_t _Py_ExplicitMergeRefcount(PyObject *op, Py_ssize_t extra);
303 #endif // !defined(Py_GIL_DISABLED)
304
305 #ifdef Py_REF_DEBUG
306 # undef _Py_DEC_REFTOTAL
307 #endif
308
309
310 extern int _PyType_CheckConsistency(PyTypeObject *type);
311 extern int _PyDict_CheckConsistency(PyObject *mp, int check_content);
312
313 // Fast inlined version of PyType_HasFeature()
314 static inline int
315 _PyType_HasFeature(PyTypeObject *type, unsigned long feature) {
316 return ((FT_ATOMIC_LOAD_ULONG_RELAXED(type->tp_flags) & feature) != 0);
317 }
318
319 extern void _PyType_InitCache(PyInterpreterState *interp);
320
321 extern PyStatus _PyObject_InitState(PyInterpreterState *interp);
322 extern void _PyObject_FiniState(PyInterpreterState *interp);
323 extern bool _PyRefchain_IsTraced(PyInterpreterState *interp, PyObject *obj);
324
325 // Macros used for per-thread reference counting in the free threading build.
326 // They resolve to normal Py_INCREF/DECREF calls in the default build.
327 //
328 // The macros are used for only a few references that would otherwise cause
329 // scaling bottlenecks in the free threading build:
330 // - The reference from an object to `ob_type`.
331 // - The reference from a function to `func_code`.
332 // - The reference from a function to `func_globals` and `func_builtins`.
333 //
334 // It's safe, but not performant or necessary, to use these macros for other
335 // references to code, type, or dict objects. It's also safe to mix their
336 // usage with normal Py_INCREF/DECREF calls.
337 //
338 // See also Include/internal/pycore_dict.h for _Py_INCREF_DICT/_Py_DECREF_DICT.
339 #ifndef Py_GIL_DISABLED
340 # define _Py_INCREF_TYPE Py_INCREF
341 # define _Py_DECREF_TYPE Py_DECREF
342 # define _Py_INCREF_CODE Py_INCREF
343 # define _Py_DECREF_CODE Py_DECREF
344 #else
345 static inline void
346 _Py_THREAD_INCREF_OBJECT(PyObject *obj, Py_ssize_t unique_id)
347 {
348 _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET();
349
350 // The table index is `unique_id - 1` because 0 is not a valid unique id.
351 // Unsigned comparison so that `idx=-1` is handled by the "else".
352 size_t idx = (size_t)(unique_id - 1);
353 if (idx < (size_t)tstate->refcounts.size) {
354 # ifdef Py_REF_DEBUG
355 _Py_INCREF_IncRefTotal();
356 # endif
357 _Py_INCREF_STAT_INC();
358 tstate->refcounts.values[idx]++;
359 }
360 else {
361 // The slow path resizes the per-thread refcount array if necessary.
362 // It handles the unique_id=0 case to keep the inlinable function smaller.
363 _PyObject_ThreadIncrefSlow(obj, idx);
364 }
365 }
366
367 static inline void
368 _Py_INCREF_TYPE(PyTypeObject *type)
369 {
370 if (!_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) {
371 assert(_Py_IsImmortal(type));
372 _Py_INCREF_IMMORTAL_STAT_INC();
373 return;
374 }
375
376 // gh-122974: GCC 11 warns about the access to PyHeapTypeObject fields when
377 // _Py_INCREF_TYPE() is called on a statically allocated type. The
378 // _PyType_HasFeature check above ensures that the type is a heap type.
379 #if defined(__GNUC__) && __GNUC__ >= 11
380 # pragma GCC diagnostic push
381 # pragma GCC diagnostic ignored "-Warray-bounds"
382 #endif
383 _Py_THREAD_INCREF_OBJECT((PyObject *)type, ((PyHeapTypeObject *)type)->unique_id);
384 #if defined(__GNUC__) && __GNUC__ >= 11
385 # pragma GCC diagnostic pop
386 #endif
387 }
388
389 static inline void
390 _Py_INCREF_CODE(PyCodeObject *co)
391 {
392 _Py_THREAD_INCREF_OBJECT((PyObject *)co, co->_co_unique_id);
393 }
394
395 static inline void
396 _Py_THREAD_DECREF_OBJECT(PyObject *obj, Py_ssize_t unique_id)
397 {
398 _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET();
399
400 // The table index is `unique_id - 1` because 0 is not a valid unique id.
401 // Unsigned comparison so that `idx=-1` is handled by the "else".
402 size_t idx = (size_t)(unique_id - 1);
403 if (idx < (size_t)tstate->refcounts.size) {
404 # ifdef Py_REF_DEBUG
405 _Py_DECREF_DecRefTotal();
406 # endif
407 _Py_DECREF_STAT_INC();
408 tstate->refcounts.values[idx]--;
409 }
410 else {
411 // Directly decref the object if the id is not assigned or if
412 // per-thread refcounting has been disabled on this object.
413 Py_DECREF(obj);
414 }
415 }
416
417 static inline void
418 _Py_DECREF_TYPE(PyTypeObject *type)
419 {
420 if (!_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) {
421 assert(_Py_IsImmortal(type));
422 _Py_DECREF_IMMORTAL_STAT_INC();
423 return;
424 }
425 PyHeapTypeObject *ht = (PyHeapTypeObject *)type;
426 _Py_THREAD_DECREF_OBJECT((PyObject *)type, ht->unique_id);
427 }
428
429 static inline void
430 _Py_DECREF_CODE(PyCodeObject *co)
431 {
432 _Py_THREAD_DECREF_OBJECT((PyObject *)co, co->_co_unique_id);
433 }
434 #endif
435
436 #ifndef Py_GIL_DISABLED
437 #ifdef Py_REF_DEBUG
438
439 static inline void Py_DECREF_MORTAL(const char *filename, int lineno, PyObject *op)
440 {
441 if (op->ob_refcnt <= 0) {
442 _Py_NegativeRefcount(filename, lineno, op);
443 }
444 _Py_DECREF_STAT_INC();
445 assert(!_Py_IsStaticImmortal(op));
446 if (!_Py_IsImmortal(op)) {
447 _Py_DECREF_DecRefTotal();
448 }
449 if (--op->ob_refcnt == 0) {
450 _Py_Dealloc(op);
451 }
452 }
453 #define Py_DECREF_MORTAL(op) Py_DECREF_MORTAL(__FILE__, __LINE__, _PyObject_CAST(op))
454
455 static inline void _Py_DECREF_MORTAL_SPECIALIZED(const char *filename, int lineno, PyObject *op, destructor destruct)
456 {
457 if (op->ob_refcnt <= 0) {
458 _Py_NegativeRefcount(filename, lineno, op);
459 }
460 _Py_DECREF_STAT_INC();
461 assert(!_Py_IsStaticImmortal(op));
462 if (!_Py_IsImmortal(op)) {
463 _Py_DECREF_DecRefTotal();
464 }
465 if (--op->ob_refcnt == 0) {
466 #ifdef Py_TRACE_REFS
467 _Py_ForgetReference(op);
468 #endif
469 _PyReftracerTrack(op, PyRefTracer_DESTROY);
470 destruct(op);
471 }
472 }
473 #define Py_DECREF_MORTAL_SPECIALIZED(op, destruct) _Py_DECREF_MORTAL_SPECIALIZED(__FILE__, __LINE__, op, destruct)
474
475 #else
476
477 static inline void Py_DECREF_MORTAL(PyObject *op)
478 {
479 assert(!_Py_IsStaticImmortal(op));
480 _Py_DECREF_STAT_INC();
481 if (--op->ob_refcnt == 0) {
482 _Py_Dealloc(op);
483 }
484 }
485 #define Py_DECREF_MORTAL(op) Py_DECREF_MORTAL(_PyObject_CAST(op))
486
487 static inline void Py_DECREF_MORTAL_SPECIALIZED(PyObject *op, destructor destruct)
488 {
489 assert(!_Py_IsStaticImmortal(op));
490 _Py_DECREF_STAT_INC();
491 if (--op->ob_refcnt == 0) {
492 _PyReftracerTrack(op, PyRefTracer_DESTROY);
493 destruct(op);
494 }
495 }
496 #define Py_DECREF_MORTAL_SPECIALIZED(op, destruct) Py_DECREF_MORTAL_SPECIALIZED(_PyObject_CAST(op), destruct)
497
498 #endif
499 #endif
500
501 /* Inline functions trading binary compatibility for speed:
502 _PyObject_Init() is the fast version of PyObject_Init(), and
503 _PyObject_InitVar() is the fast version of PyObject_InitVar().
504
505 These inline functions must not be called with op=NULL. */
506 static inline void
507 _PyObject_Init(PyObject *op, PyTypeObject *typeobj)
508 {
509 assert(op != NULL);
510 Py_SET_TYPE(op, typeobj);
511 assert(_PyType_HasFeature(typeobj, Py_TPFLAGS_HEAPTYPE) || _Py_IsImmortal(typeobj));
512 _Py_INCREF_TYPE(typeobj);
513 _Py_NewReference(op);
514 }
515
516 static inline void
517 _PyObject_InitVar(PyVarObject *op, PyTypeObject *typeobj, Py_ssize_t size)
518 {
519 assert(op != NULL);
520 assert(typeobj != &PyLong_Type);
521 _PyObject_Init((PyObject *)op, typeobj);
522 Py_SET_SIZE(op, size);
523 }
524
525 // Macros to accept any type for the parameter, and to automatically pass
526 // the filename and the filename (if NDEBUG is not defined) where the macro
527 // is called.
528 #ifdef NDEBUG
529 # define _PyObject_GC_TRACK(op) \
530 _PyObject_GC_TRACK(_PyObject_CAST(op))
531 # define _PyObject_GC_UNTRACK(op) \
532 _PyObject_GC_UNTRACK(_PyObject_CAST(op))
533 #else
534 # define _PyObject_GC_TRACK(op) \
535 _PyObject_GC_TRACK(__FILE__, __LINE__, _PyObject_CAST(op))
536 # define _PyObject_GC_UNTRACK(op) \
537 _PyObject_GC_UNTRACK(__FILE__, __LINE__, _PyObject_CAST(op))
538 #endif
539
540 #ifdef Py_GIL_DISABLED
541
542 /* Tries to increment an object's reference count
543 *
544 * This is a specialized version of _Py_TryIncref that only succeeds if the
545 * object is immortal or local to this thread. It does not handle the case
546 * where the reference count modification requires an atomic operation. This
547 * allows call sites to specialize for the immortal/local case.
548 */
549 static inline int
550 _Py_TryIncrefFast(PyObject *op) {
551 uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local);
552 local += 1;
553 if (local == 0) {
554 // immortal
555 _Py_INCREF_IMMORTAL_STAT_INC();
556 return 1;
557 }
558 if (_Py_IsOwnedByCurrentThread(op)) {
559 _Py_INCREF_STAT_INC();
560 _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, local);
561 #ifdef Py_REF_DEBUG
562 _Py_IncRefTotal(_PyThreadState_GET());
563 #endif
564 return 1;
565 }
566 return 0;
567 }
568
569 static inline int
570 _Py_TryIncRefShared(PyObject *op)
571 {
572 Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared);
573 for (;;) {
574 // If the shared refcount is zero and the object is either merged
575 // or may not have weak references, then we cannot incref it.
576 if (shared == 0 || shared == _Py_REF_MERGED) {
577 return 0;
578 }
579
580 if (_Py_atomic_compare_exchange_ssize(
581 &op->ob_ref_shared,
582 &shared,
583 shared + (1 << _Py_REF_SHARED_SHIFT))) {
584 #ifdef Py_REF_DEBUG
585 _Py_IncRefTotal(_PyThreadState_GET());
586 #endif
587 _Py_INCREF_STAT_INC();
588 return 1;
589 }
590 }
591 }
592
593 /* Tries to incref the object op and ensures that *src still points to it. */
594 static inline int
595 _Py_TryIncrefCompare(PyObject **src, PyObject *op)
596 {
597 if (_Py_TryIncrefFast(op)) {
598 return 1;
599 }
600 if (!_Py_TryIncRefShared(op)) {
601 return 0;
602 }
603 if (op != _Py_atomic_load_ptr(src)) {
604 Py_DECREF(op);
605 return 0;
606 }
607 return 1;
608 }
609
610 /* Loads and increfs an object from ptr, which may contain a NULL value.
611 Safe with concurrent (atomic) updates to ptr.
612 NOTE: The writer must set maybe-weakref on the stored object! */
613 static inline PyObject *
614 _Py_XGetRef(PyObject **ptr)
615 {
616 for (;;) {
617 PyObject *value = _PyObject_CAST(_Py_atomic_load_ptr(ptr));
618 if (value == NULL) {
619 return value;
620 }
621 if (_Py_TryIncrefCompare(ptr, value)) {
622 return value;
623 }
624 }
625 }
626
627 /* Attempts to loads and increfs an object from ptr. Returns NULL
628 on failure, which may be due to a NULL value or a concurrent update. */
629 static inline PyObject *
630 _Py_TryXGetRef(PyObject **ptr)
631 {
632 PyObject *value = _PyObject_CAST(_Py_atomic_load_ptr(ptr));
633 if (value == NULL) {
634 return value;
635 }
636 if (_Py_TryIncrefCompare(ptr, value)) {
637 return value;
638 }
639 return NULL;
640 }
641
642 /* Like Py_NewRef but also optimistically sets _Py_REF_MAYBE_WEAKREF
643 on objects owned by a different thread. */
644 static inline PyObject *
645 _Py_NewRefWithLock(PyObject *op)
646 {
647 if (_Py_TryIncrefFast(op)) {
648 return op;
649 }
650 #ifdef Py_REF_DEBUG
651 _Py_IncRefTotal(_PyThreadState_GET());
652 #endif
653 _Py_INCREF_STAT_INC();
654 for (;;) {
655 Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared);
656 Py_ssize_t new_shared = shared + (1 << _Py_REF_SHARED_SHIFT);
657 if ((shared & _Py_REF_SHARED_FLAG_MASK) == 0) {
658 new_shared |= _Py_REF_MAYBE_WEAKREF;
659 }
660 if (_Py_atomic_compare_exchange_ssize(
661 &op->ob_ref_shared,
662 &shared,
663 new_shared)) {
664 return op;
665 }
666 }
667 }
668
669 static inline PyObject *
670 _Py_XNewRefWithLock(PyObject *obj)
671 {
672 if (obj == NULL) {
673 return NULL;
674 }
675 return _Py_NewRefWithLock(obj);
676 }
677
678 static inline void
679 _PyObject_SetMaybeWeakref(PyObject *op)
680 {
681 if (_Py_IsImmortal(op)) {
682 return;
683 }
684 for (;;) {
685 Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared);
686 if ((shared & _Py_REF_SHARED_FLAG_MASK) != 0) {
687 // Nothing to do if it's in WEAKREFS, QUEUED, or MERGED states.
688 return;
689 }
690 if (_Py_atomic_compare_exchange_ssize(
691 &op->ob_ref_shared, &shared, shared | _Py_REF_MAYBE_WEAKREF)) {
692 return;
693 }
694 }
695 }
696
697 extern PyAPI_FUNC(int) _PyObject_ResurrectEndSlow(PyObject *op);
698 #endif
699
700 // Temporarily resurrects an object during deallocation. The refcount is set
701 // to one.
702 static inline void
703 _PyObject_ResurrectStart(PyObject *op)
704 {
705 assert(Py_REFCNT(op) == 0);
706 #ifdef Py_REF_DEBUG
707 _Py_IncRefTotal(_PyThreadState_GET());
708 #endif
709 #ifdef Py_GIL_DISABLED
710 _Py_atomic_store_uintptr_relaxed(&op->ob_tid, _Py_ThreadId());
711 _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, 1);
712 _Py_atomic_store_ssize_relaxed(&op->ob_ref_shared, 0);
713 #else
714 Py_SET_REFCNT(op, 1);
715 #endif
716 #ifdef Py_TRACE_REFS
717 _Py_ResurrectReference(op);
718 #endif
719 }
720
721 // Undoes an object resurrection by decrementing the refcount without calling
722 // _Py_Dealloc(). Returns 0 if the object is dead (the normal case), and
723 // deallocation should continue. Returns 1 if the object is still alive.
724 static inline int
725 _PyObject_ResurrectEnd(PyObject *op)
726 {
727 #ifdef Py_REF_DEBUG
728 _Py_DecRefTotal(_PyThreadState_GET());
729 #endif
730 #ifndef Py_GIL_DISABLED
731 Py_SET_REFCNT(op, Py_REFCNT(op) - 1);
732 if (Py_REFCNT(op) == 0) {
733 # ifdef Py_TRACE_REFS
734 _Py_ForgetReference(op);
735 # endif
736 return 0;
737 }
738 return 1;
739 #else
740 uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local);
741 Py_ssize_t shared = _Py_atomic_load_ssize_acquire(&op->ob_ref_shared);
742 if (_Py_IsOwnedByCurrentThread(op) && local == 1 && shared == 0) {
743 // Fast-path: object has a single refcount and is owned by this thread
744 _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, 0);
745 # ifdef Py_TRACE_REFS
746 _Py_ForgetReference(op);
747 # endif
748 return 0;
749 }
750 // Slow-path: object has a shared refcount or is not owned by this thread
751 return _PyObject_ResurrectEndSlow(op);
752 #endif
753 }
754
755 /* Tries to incref op and returns 1 if successful or 0 otherwise. */
756 static inline int
757 _Py_TryIncref(PyObject *op)
758 {
759 #ifdef Py_GIL_DISABLED
760 return _Py_TryIncrefFast(op) || _Py_TryIncRefShared(op);
761 #else
762 if (Py_REFCNT(op) > 0) {
763 Py_INCREF(op);
764 return 1;
765 }
766 return 0;
767 #endif
768 }
769
770 #ifdef Py_REF_DEBUG
771 extern void _PyInterpreterState_FinalizeRefTotal(PyInterpreterState *);
772 extern void _Py_FinalizeRefTotal(_PyRuntimeState *);
773 extern void _PyDebug_PrintTotalRefs(void);
774 #endif
775
776 #ifdef Py_TRACE_REFS
777 extern void _Py_AddToAllObjects(PyObject *op);
778 extern void _Py_PrintReferences(PyInterpreterState *, FILE *);
779 extern void _Py_PrintReferenceAddresses(PyInterpreterState *, FILE *);
780 #endif
781
782
783 /* Return the *address* of the object's weaklist. The address may be
784 * dereferenced to get the current head of the weaklist. This is useful
785 * for iterating over the linked list of weakrefs, especially when the
786 * list is being modified externally (e.g. refs getting removed).
787 *
788 * The returned pointer should not be used to change the head of the list
789 * nor should it be used to add, remove, or swap any refs in the list.
790 * That is the sole responsibility of the code in weakrefobject.c.
791 */
792 static inline PyObject **
793 _PyObject_GET_WEAKREFS_LISTPTR(PyObject *op)
794 {
795 if (PyType_Check(op) &&
796 ((PyTypeObject *)op)->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) {
797 PyInterpreterState *interp = _PyInterpreterState_GET();
798 managed_static_type_state *state = _PyStaticType_GetState(
799 interp, (PyTypeObject *)op);
800 return _PyStaticType_GET_WEAKREFS_LISTPTR(state);
801 }
802 // Essentially _PyObject_GET_WEAKREFS_LISTPTR_FROM_OFFSET():
803 Py_ssize_t offset = Py_TYPE(op)->tp_weaklistoffset;
804 return (PyObject **)((char *)op + offset);
805 }
806
807 /* This is a special case of _PyObject_GET_WEAKREFS_LISTPTR().
808 * Only the most fundamental lookup path is used.
809 * Consequently, static types should not be used.
810 *
811 * For static builtin types the returned pointer will always point
812 * to a NULL tp_weaklist. This is fine for any deallocation cases,
813 * since static types are never deallocated and static builtin types
814 * are only finalized at the end of runtime finalization.
815 *
816 * If the weaklist for static types is actually needed then use
817 * _PyObject_GET_WEAKREFS_LISTPTR().
818 */
819 static inline PyWeakReference **
820 _PyObject_GET_WEAKREFS_LISTPTR_FROM_OFFSET(PyObject *op)
821 {
822 assert(!PyType_Check(op) ||
823 ((PyTypeObject *)op)->tp_flags & Py_TPFLAGS_HEAPTYPE);
824 Py_ssize_t offset = Py_TYPE(op)->tp_weaklistoffset;
825 return (PyWeakReference **)((char *)op + offset);
826 }
827
828 // Fast inlined version of PyType_IS_GC()
829 #define _PyType_IS_GC(t) _PyType_HasFeature((t), Py_TPFLAGS_HAVE_GC)
830
831 // Fast inlined version of PyObject_IS_GC()
832 static inline int
833 _PyObject_IS_GC(PyObject *obj)
834 {
835 PyTypeObject *type = Py_TYPE(obj);
836 return (_PyType_IS_GC(type)
837 && (type->tp_is_gc == NULL || type->tp_is_gc(obj)));
838 }
839
840 // Fast inlined version of PyObject_Hash()
841 static inline Py_hash_t
842 _PyObject_HashFast(PyObject *op)
843 {
844 if (PyUnicode_CheckExact(op)) {
845 Py_hash_t hash = FT_ATOMIC_LOAD_SSIZE_RELAXED(
846 _PyASCIIObject_CAST(op)->hash);
847 if (hash != -1) {
848 return hash;
849 }
850 }
851 return PyObject_Hash(op);
852 }
853
854 static inline size_t
855 _PyType_PreHeaderSize(PyTypeObject *tp)
856 {
857 return (
858 #ifndef Py_GIL_DISABLED
859 (size_t)_PyType_IS_GC(tp) * sizeof(PyGC_Head) +
860 #endif
861 (size_t)_PyType_HasFeature(tp, Py_TPFLAGS_PREHEADER) * 2 * sizeof(PyObject *)
862 );
863 }
864
865 void _PyObject_GC_Link(PyObject *op);
866
867 // Usage: assert(_Py_CheckSlotResult(obj, "__getitem__", result != NULL));
868 extern int _Py_CheckSlotResult(
869 PyObject *obj,
870 const char *slot_name,
871 int success);
872
873 // Test if a type supports weak references
874 static inline int _PyType_SUPPORTS_WEAKREFS(PyTypeObject *type) {
875 return (type->tp_weaklistoffset != 0);
876 }
877
878 extern PyObject* _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t nitems);
879 PyAPI_FUNC(PyObject *) _PyType_NewManagedObject(PyTypeObject *type);
880
881 extern PyTypeObject* _PyType_CalculateMetaclass(PyTypeObject *, PyObject *);
882 extern PyObject* _PyType_GetDocFromInternalDoc(const char *, const char *);
883 extern PyObject* _PyType_GetTextSignatureFromInternalDoc(const char *, const char *, int);
884 extern int _PyObject_SetAttributeErrorContext(PyObject *v, PyObject* name);
885
886 void _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp);
887 extern int _PyObject_StoreInstanceAttribute(PyObject *obj,
888 PyObject *name, PyObject *value);
889 extern bool _PyObject_TryGetInstanceAttribute(PyObject *obj, PyObject *name,
890 PyObject **attr);
891 extern PyObject *_PyType_LookupRefAndVersion(PyTypeObject *, PyObject *,
892 unsigned int *);
893
894 // Internal API to look for a name through the MRO.
895 // This stores a stack reference in out and returns the value of
896 // type->tp_version or zero if name is missing. It doesn't set an exception!
897 extern unsigned int
898 _PyType_LookupStackRefAndVersion(PyTypeObject *type, PyObject *name, _PyStackRef *out);
899
900 extern int _PyObject_GetMethodStackRef(PyThreadState *ts, _PyStackRef *self,
901 PyObject *name, _PyStackRef *method);
902
903 // Cache the provided init method in the specialization cache of type if the
904 // provided type version matches the current version of the type.
905 //
906 // The cached value is borrowed and is only valid if guarded by a type
907 // version check. In free-threaded builds the init method must also use
908 // deferred reference counting.
909 //
910 // Returns 1 if the value was cached or 0 otherwise.
911 extern int _PyType_CacheInitForSpecialization(PyHeapTypeObject *type,
912 PyObject *init,
913 unsigned int tp_version);
914
915 #ifdef Py_GIL_DISABLED
916 # define MANAGED_DICT_OFFSET (((Py_ssize_t)sizeof(PyObject *))*-1)
917 # define MANAGED_WEAKREF_OFFSET (((Py_ssize_t)sizeof(PyObject *))*-2)
918 #else
919 # define MANAGED_DICT_OFFSET (((Py_ssize_t)sizeof(PyObject *))*-3)
920 # define MANAGED_WEAKREF_OFFSET (((Py_ssize_t)sizeof(PyObject *))*-4)
921 #endif
922
923 typedef union {
924 PyDictObject *dict;
925 } PyManagedDictPointer;
926
927 static inline PyManagedDictPointer *
928 _PyObject_ManagedDictPointer(PyObject *obj)
929 {
930 assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
931 return (PyManagedDictPointer *)((char *)obj + MANAGED_DICT_OFFSET);
932 }
933
934 static inline PyDictObject *
935 _PyObject_GetManagedDict(PyObject *obj)
936 {
937 PyManagedDictPointer *dorv = _PyObject_ManagedDictPointer(obj);
938 return (PyDictObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(dorv->dict);
939 }
940
941 static inline PyDictValues *
942 _PyObject_InlineValues(PyObject *obj)
943 {
944 PyTypeObject *tp = Py_TYPE(obj);
945 assert(tp->tp_basicsize > 0 && (size_t)tp->tp_basicsize % sizeof(PyObject *) == 0);
946 assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES);
947 assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
948 return (PyDictValues *)((char *)obj + tp->tp_basicsize);
949 }
950
951 extern PyObject ** _PyObject_ComputedDictPointer(PyObject *);
952 extern int _PyObject_IsInstanceDictEmpty(PyObject *);
953
954 // Export for 'math' shared extension
955 PyAPI_FUNC(PyObject*) _PyObject_LookupSpecial(PyObject *, PyObject *);
956 PyAPI_FUNC(int) _PyObject_LookupSpecialMethod(PyObject *attr, _PyStackRef *method_and_self);
957
958 // Calls the method named `attr` on `self`, but does not set an exception if
959 // the attribute does not exist.
960 PyAPI_FUNC(PyObject *)
961 _PyObject_MaybeCallSpecialNoArgs(PyObject *self, PyObject *attr);
962
963 PyAPI_FUNC(PyObject *)
964 _PyObject_MaybeCallSpecialOneArg(PyObject *self, PyObject *attr, PyObject *arg);
965
966 extern int _PyObject_IsAbstract(PyObject *);
967
968 PyAPI_FUNC(int) _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method);
969 extern PyObject* _PyObject_NextNotImplemented(PyObject *);
970
971 // Pickle support.
972 // Export for '_datetime' shared extension
973 PyAPI_FUNC(PyObject*) _PyObject_GetState(PyObject *);
974
975 /* C function call trampolines to mitigate bad function pointer casts.
976 *
977 * Typical native ABIs ignore additional arguments or fill in missing
978 * values with 0/NULL in function pointer cast. Compilers do not show
979 * warnings when a function pointer is explicitly casted to an
980 * incompatible type.
981 *
982 * Bad fpcasts are an issue in WebAssembly. WASM's indirect_call has strict
983 * function signature checks. Argument count, types, and return type must
984 * match.
985 *
986 * Third party code unintentionally rely on problematic fpcasts. The call
987 * trampoline mitigates common occurrences of bad fpcasts on Emscripten.
988 */
989 #if !(defined(__EMSCRIPTEN__) && defined(PY_CALL_TRAMPOLINE))
990 #define _PyCFunction_TrampolineCall(meth, self, args) \
991 (meth)((self), (args))
992 #define _PyCFunctionWithKeywords_TrampolineCall(meth, self, args, kw) \
993 (meth)((self), (args), (kw))
994 #endif // __EMSCRIPTEN__ && PY_CALL_TRAMPOLINE
995
996 // Export these 2 symbols for '_pickle' shared extension
997 PyAPI_DATA(PyTypeObject) _PyNone_Type;
998 PyAPI_DATA(PyTypeObject) _PyNotImplemented_Type;
999
1000 // Maps Py_LT to Py_GT, ..., Py_GE to Py_LE.
1001 // Export for the stable ABI.
1002 PyAPI_DATA(int) _Py_SwappedOp[];
1003
1004 extern void _Py_GetConstant_Init(void);
1005
1006 enum _PyAnnotateFormat {
1007 _Py_ANNOTATE_FORMAT_VALUE = 1,
1008 _Py_ANNOTATE_FORMAT_VALUE_WITH_FAKE_GLOBALS = 2,
1009 _Py_ANNOTATE_FORMAT_FORWARDREF = 3,
1010 _Py_ANNOTATE_FORMAT_STRING = 4,
1011 };
1012
1013 int _PyObject_SetDict(PyObject *obj, PyObject *value);
1014
1015 #ifndef Py_GIL_DISABLED
1016 static inline Py_ALWAYS_INLINE void _Py_INCREF_MORTAL(PyObject *op)
1017 {
1018 assert(!_Py_IsStaticImmortal(op));
1019 op->ob_refcnt++;
1020 _Py_INCREF_STAT_INC();
1021 #if defined(Py_REF_DEBUG) && !defined(Py_LIMITED_API)
1022 if (!_Py_IsImmortal(op)) {
1023 _Py_INCREF_IncRefTotal();
1024 }
1025 #endif
1026 }
1027 #endif
1028
1029 #ifdef __cplusplus
1030 }
1031 #endif
1032 #endif /* !Py_INTERNAL_OBJECT_H */