Debian Codegraph @

Include/internal/pycore_stackref.h

raw
1 #ifndef Py_INTERNAL_STACKREF_H
2 #define Py_INTERNAL_STACKREF_H
3 #ifdef __cplusplus
4 extern "C" {
5 #endif
6
7 #ifndef Py_BUILD_CORE
8 # error "this header requires Py_BUILD_CORE define"
9 #endif
10
11 #include "pycore_object.h" // Py_DECREF_MORTAL
12 #include "pycore_object_deferred.h" // _PyObject_HasDeferredRefcount()
13
14 #include <stdbool.h> // bool
15
16
17 /*
18 This file introduces a new API for handling references on the stack, called
19 _PyStackRef. This API is inspired by HPy.
20
21 There are 3 main operations, that convert _PyStackRef to PyObject* and
22 vice versa:
23
24 1. Borrow (discouraged)
25 2. Steal
26 3. New
27
28 Borrow means that the reference is converted without any change in ownership.
29 This is discouraged because it makes verification much harder. It also makes
30 unboxed integers harder in the future.
31
32 Steal means that ownership is transferred to something else. The total
33 number of references to the object stays the same. The old reference is no
34 longer valid.
35
36 New creates a new reference from the old reference. The old reference
37 is still valid.
38
39 All _PyStackRef must be operated on by the new reference operations:
40
41 1. DUP
42 2. CLOSE
43
44 DUP is roughly equivalent to Py_NewRef. It creates a new reference from an old
45 reference. The old reference remains unchanged.
46
47 CLOSE is roughly equivalent to Py_DECREF. It destroys a reference.
48
49 Note that it is unsafe to borrow a _PyStackRef and then do normal
50 CPython refcounting operations on it!
51 */
52
53
54 #if !defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG)
55
56 #define Py_TAG_BITS 0
57
58 PyAPI_FUNC(PyObject *) _Py_stackref_get_object(_PyStackRef ref);
59 PyAPI_FUNC(PyObject *) _Py_stackref_close(_PyStackRef ref, const char *filename, int linenumber);
60 PyAPI_FUNC(_PyStackRef) _Py_stackref_create(PyObject *obj, const char *filename, int linenumber);
61 PyAPI_FUNC(void) _Py_stackref_record_borrow(_PyStackRef ref, const char *filename, int linenumber);
62 extern void _Py_stackref_associate(PyInterpreterState *interp, PyObject *obj, _PyStackRef ref);
63
64 static const _PyStackRef PyStackRef_NULL = { .index = 0 };
65
66 // Use the first 3 even numbers for None, True and False.
67 // Odd numbers are reserved for (tagged) integers
68 #define PyStackRef_None ((_PyStackRef){ .index = 2 } )
69 #define PyStackRef_False ((_PyStackRef){ .index = 4 })
70 #define PyStackRef_True ((_PyStackRef){ .index = 6 })
71
72 #define INITIAL_STACKREF_INDEX 8
73
74 static inline int
75 PyStackRef_IsNull(_PyStackRef ref)
76 {
77 return ref.index == 0;
78 }
79
80 static inline int
81 PyStackRef_IsTrue(_PyStackRef ref)
82 {
83 return _Py_stackref_get_object(ref) == Py_True;
84 }
85
86 static inline int
87 PyStackRef_IsFalse(_PyStackRef ref)
88 {
89 return _Py_stackref_get_object(ref) == Py_False;
90 }
91
92 static inline int
93 PyStackRef_IsNone(_PyStackRef ref)
94 {
95 return _Py_stackref_get_object(ref) == Py_None;
96 }
97
98 static inline PyObject *
99 _PyStackRef_AsPyObjectBorrow(_PyStackRef ref, const char *filename, int linenumber)
100 {
101 assert((ref.index & 1) == 0);
102 _Py_stackref_record_borrow(ref, filename, linenumber);
103 return _Py_stackref_get_object(ref);
104 }
105
106 #define PyStackRef_AsPyObjectBorrow(REF) _PyStackRef_AsPyObjectBorrow((REF), __FILE__, __LINE__)
107
108 static inline PyObject *
109 _PyStackRef_AsPyObjectSteal(_PyStackRef ref, const char *filename, int linenumber)
110 {
111 return _Py_stackref_close(ref, filename, linenumber);
112 }
113 #define PyStackRef_AsPyObjectSteal(REF) _PyStackRef_AsPyObjectSteal((REF), __FILE__, __LINE__)
114
115 static inline _PyStackRef
116 _PyStackRef_FromPyObjectNew(PyObject *obj, const char *filename, int linenumber)
117 {
118 Py_INCREF(obj);
119 return _Py_stackref_create(obj, filename, linenumber);
120 }
121 #define PyStackRef_FromPyObjectNew(obj) _PyStackRef_FromPyObjectNew(_PyObject_CAST(obj), __FILE__, __LINE__)
122
123 static inline _PyStackRef
124 _PyStackRef_FromPyObjectSteal(PyObject *obj, const char *filename, int linenumber)
125 {
126 return _Py_stackref_create(obj, filename, linenumber);
127 }
128 #define PyStackRef_FromPyObjectSteal(obj) _PyStackRef_FromPyObjectSteal(_PyObject_CAST(obj), __FILE__, __LINE__)
129
130 static inline _PyStackRef
131 _PyStackRef_FromPyObjectBorrow(PyObject *obj, const char *filename, int linenumber)
132 {
133 return _Py_stackref_create(obj, filename, linenumber);
134 }
135 #define PyStackRef_FromPyObjectBorrow(obj) _PyStackRef_FromPyObjectBorrow(_PyObject_CAST(obj), __FILE__, __LINE__)
136
137 static inline _PyStackRef
138 _PyStackRef_FromPyObjectImmortal(PyObject *obj, const char *filename, int linenumber)
139 {
140 assert(_Py_IsImmortal(obj));
141 return _Py_stackref_create(obj, filename, linenumber);
142 }
143 #define PyStackRef_FromPyObjectImmortal(obj) _PyStackRef_FromPyObjectImmortal(_PyObject_CAST(obj), __FILE__, __LINE__)
144
145 static inline bool
146 PyStackRef_IsTaggedInt(_PyStackRef ref)
147 {
148 return (ref.index & 1) == 1;
149 }
150
151 static inline void
152 _PyStackRef_CLOSE(_PyStackRef ref, const char *filename, int linenumber)
153 {
154 if (PyStackRef_IsTaggedInt(ref)) {
155 return;
156 }
157 PyObject *obj = _Py_stackref_close(ref, filename, linenumber);
158 Py_DECREF(obj);
159 }
160 #define PyStackRef_CLOSE(REF) _PyStackRef_CLOSE((REF), __FILE__, __LINE__)
161
162
163 static inline void
164 _PyStackRef_XCLOSE(_PyStackRef ref, const char *filename, int linenumber)
165 {
166 if (PyStackRef_IsNull(ref)) {
167 return;
168 }
169 _PyStackRef_CLOSE(ref, filename, linenumber);
170 }
171 #define PyStackRef_XCLOSE(REF) _PyStackRef_XCLOSE((REF), __FILE__, __LINE__)
172
173 static inline _PyStackRef
174 _PyStackRef_DUP(_PyStackRef ref, const char *filename, int linenumber)
175 {
176 if (PyStackRef_IsTaggedInt(ref)) {
177 return ref;
178 }
179 else {
180 PyObject *obj = _Py_stackref_get_object(ref);
181 Py_INCREF(obj);
182 return _Py_stackref_create(obj, filename, linenumber);
183 }
184 }
185 #define PyStackRef_DUP(REF) _PyStackRef_DUP(REF, __FILE__, __LINE__)
186
187 extern void _PyStackRef_CLOSE_SPECIALIZED(_PyStackRef ref, destructor destruct, const char *filename, int linenumber);
188 #define PyStackRef_CLOSE_SPECIALIZED(REF, DESTRUCT) _PyStackRef_CLOSE_SPECIALIZED(REF, DESTRUCT, __FILE__, __LINE__)
189
190 static inline _PyStackRef
191 PyStackRef_MakeHeapSafe(_PyStackRef ref)
192 {
193 return ref;
194 }
195
196 static inline _PyStackRef
197 PyStackRef_Borrow(_PyStackRef ref)
198 {
199 return PyStackRef_DUP(ref);
200 }
201
202 #define PyStackRef_CLEAR(REF) \
203 do { \
204 _PyStackRef *_tmp_op_ptr = &(REF); \
205 _PyStackRef _tmp_old_op = (*_tmp_op_ptr); \
206 *_tmp_op_ptr = PyStackRef_NULL; \
207 PyStackRef_XCLOSE(_tmp_old_op); \
208 } while (0)
209
210 static inline _PyStackRef
211 _PyStackRef_FromPyObjectStealMortal(PyObject *obj, const char *filename, int linenumber)
212 {
213 assert(!_Py_IsImmortal(obj));
214 return _Py_stackref_create(obj, filename, linenumber);
215 }
216 #define PyStackRef_FromPyObjectStealMortal(obj) _PyStackRef_FromPyObjectStealMortal(_PyObject_CAST(obj), __FILE__, __LINE__)
217
218 static inline bool
219 PyStackRef_IsHeapSafe(_PyStackRef ref)
220 {
221 return true;
222 }
223
224 static inline _PyStackRef
225 _PyStackRef_FromPyObjectNewMortal(PyObject *obj, const char *filename, int linenumber)
226 {
227 assert(!_Py_IsStaticImmortal(obj));
228 Py_INCREF(obj);
229 return _Py_stackref_create(obj, filename, linenumber);
230 }
231 #define PyStackRef_FromPyObjectNewMortal(obj) _PyStackRef_FromPyObjectNewMortal(_PyObject_CAST(obj), __FILE__, __LINE__)
232
233 #define PyStackRef_RefcountOnObject(REF) 1
234
235 extern int PyStackRef_Is(_PyStackRef a, _PyStackRef b);
236
237 extern bool PyStackRef_IsTaggedInt(_PyStackRef ref);
238
239 extern intptr_t PyStackRef_UntagInt(_PyStackRef ref);
240
241 extern _PyStackRef PyStackRef_TagInt(intptr_t i);
242
243 extern bool
244 PyStackRef_IsNullOrInt(_PyStackRef ref);
245
246 #else
247
248 #define Py_INT_TAG 3
249 #define Py_TAG_REFCNT 1
250
251 static inline bool
252 PyStackRef_IsTaggedInt(_PyStackRef i)
253 {
254 return (i.bits & Py_INT_TAG) == Py_INT_TAG;
255 }
256
257 static inline _PyStackRef
258 PyStackRef_TagInt(intptr_t i)
259 {
260 assert(Py_ARITHMETIC_RIGHT_SHIFT(intptr_t, (i << 2), 2) == i);
261 return (_PyStackRef){ .bits = ((((uintptr_t)i) << 2) | Py_INT_TAG) };
262 }
263
264 static inline intptr_t
265 PyStackRef_UntagInt(_PyStackRef i)
266 {
267 assert((i.bits & Py_INT_TAG) == Py_INT_TAG);
268 intptr_t val = (intptr_t)i.bits;
269 return Py_ARITHMETIC_RIGHT_SHIFT(intptr_t, val, 2);
270 }
271
272
273 #ifdef Py_GIL_DISABLED
274
275 #define Py_TAG_DEFERRED Py_TAG_REFCNT
276
277 #define Py_TAG_PTR ((uintptr_t)0)
278 #define Py_TAG_BITS ((uintptr_t)1)
279
280
281 static const _PyStackRef PyStackRef_NULL = { .bits = Py_TAG_DEFERRED};
282 #define PyStackRef_IsNull(stackref) ((stackref).bits == PyStackRef_NULL.bits)
283 #define PyStackRef_True ((_PyStackRef){.bits = ((uintptr_t)&_Py_TrueStruct) | Py_TAG_DEFERRED })
284 #define PyStackRef_False ((_PyStackRef){.bits = ((uintptr_t)&_Py_FalseStruct) | Py_TAG_DEFERRED })
285 #define PyStackRef_None ((_PyStackRef){.bits = ((uintptr_t)&_Py_NoneStruct) | Py_TAG_DEFERRED })
286
287 // Checks that mask out the deferred bit in the free threading build.
288 #define PyStackRef_IsNone(ref) (PyStackRef_AsPyObjectBorrow(ref) == Py_None)
289 #define PyStackRef_IsTrue(ref) (PyStackRef_AsPyObjectBorrow(ref) == Py_True)
290 #define PyStackRef_IsFalse(ref) (PyStackRef_AsPyObjectBorrow(ref) == Py_False)
291
292 #define PyStackRef_IsNullOrInt(stackref) (PyStackRef_IsNull(stackref) || PyStackRef_IsTaggedInt(stackref))
293
294 static inline PyObject *
295 PyStackRef_AsPyObjectBorrow(_PyStackRef stackref)
296 {
297 PyObject *cleared = ((PyObject *)((stackref).bits & (~Py_TAG_BITS)));
298 return cleared;
299 }
300
301 #define PyStackRef_IsDeferred(ref) (((ref).bits & Py_TAG_BITS) == Py_TAG_DEFERRED)
302
303 static inline PyObject *
304 PyStackRef_NotDeferred_AsPyObject(_PyStackRef stackref)
305 {
306 assert(!PyStackRef_IsDeferred(stackref));
307 return (PyObject *)stackref.bits;
308 }
309
310 static inline PyObject *
311 PyStackRef_AsPyObjectSteal(_PyStackRef stackref)
312 {
313 assert(!PyStackRef_IsNull(stackref));
314 if (PyStackRef_IsDeferred(stackref)) {
315 return Py_NewRef(PyStackRef_AsPyObjectBorrow(stackref));
316 }
317 return PyStackRef_AsPyObjectBorrow(stackref);
318 }
319
320 static inline _PyStackRef
321 _PyStackRef_FromPyObjectSteal(PyObject *obj)
322 {
323 assert(obj != NULL);
324 // Make sure we don't take an already tagged value.
325 assert(((uintptr_t)obj & Py_TAG_BITS) == 0);
326 return (_PyStackRef){ .bits = (uintptr_t)obj };
327 }
328 # define PyStackRef_FromPyObjectSteal(obj) _PyStackRef_FromPyObjectSteal(_PyObject_CAST(obj))
329
330 static inline _PyStackRef
331 PyStackRef_FromPyObjectBorrow(PyObject *obj)
332 {
333 assert(obj != NULL);
334 assert(((uintptr_t)obj & Py_TAG_BITS) == 0);
335 return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_DEFERRED };
336 }
337
338 static inline bool
339 PyStackRef_IsHeapSafe(_PyStackRef stackref)
340 {
341 if (PyStackRef_IsDeferred(stackref) && !PyStackRef_IsTaggedInt(stackref)) {
342 PyObject *obj = PyStackRef_AsPyObjectBorrow(stackref);
343 return obj == NULL || _Py_IsImmortal(obj) || _PyObject_HasDeferredRefcount(obj);
344 }
345 return true;
346 }
347
348 static inline _PyStackRef
349 PyStackRef_MakeHeapSafe(_PyStackRef stackref)
350 {
351 if (PyStackRef_IsHeapSafe(stackref)) {
352 return stackref;
353 }
354 PyObject *obj = PyStackRef_AsPyObjectBorrow(stackref);
355 return (_PyStackRef){ .bits = (uintptr_t)(Py_NewRef(obj)) | Py_TAG_PTR };
356 }
357
358 static inline _PyStackRef
359 PyStackRef_FromPyObjectStealMortal(PyObject *obj)
360 {
361 assert(obj != NULL);
362 assert(!_Py_IsImmortal(obj));
363 // Make sure we don't take an already tagged value.
364 assert(((uintptr_t)obj & Py_TAG_BITS) == 0);
365 return (_PyStackRef){ .bits = (uintptr_t)obj };
366 }
367
368 static inline _PyStackRef
369 PyStackRef_FromPyObjectNew(PyObject *obj)
370 {
371 // Make sure we don't take an already tagged value.
372 assert(((uintptr_t)obj & Py_TAG_BITS) == 0);
373 assert(obj != NULL);
374 if (_PyObject_HasDeferredRefcount(obj)) {
375 return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_DEFERRED };
376 }
377 else {
378 return (_PyStackRef){ .bits = (uintptr_t)(Py_NewRef(obj)) | Py_TAG_PTR };
379 }
380 }
381 #define PyStackRef_FromPyObjectNew(obj) PyStackRef_FromPyObjectNew(_PyObject_CAST(obj))
382
383 static inline _PyStackRef
384 PyStackRef_FromPyObjectImmortal(PyObject *obj)
385 {
386 // Make sure we don't take an already tagged value.
387 assert(((uintptr_t)obj & Py_TAG_BITS) == 0);
388 assert(obj != NULL);
389 assert(_Py_IsImmortal(obj));
390 return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_DEFERRED };
391 }
392 #define PyStackRef_FromPyObjectImmortal(obj) PyStackRef_FromPyObjectImmortal(_PyObject_CAST(obj))
393
394 #define PyStackRef_CLOSE(REF) \
395 do { \
396 _PyStackRef _close_tmp = (REF); \
397 assert(!PyStackRef_IsNull(_close_tmp)); \
398 if (!PyStackRef_IsDeferred(_close_tmp)) { \
399 Py_DECREF(PyStackRef_AsPyObjectBorrow(_close_tmp)); \
400 } \
401 } while (0)
402
403 static inline void
404 PyStackRef_CLOSE_SPECIALIZED(_PyStackRef ref, destructor destruct)
405 {
406 (void)destruct;
407 PyStackRef_CLOSE(ref);
408 }
409
410 static inline _PyStackRef
411 PyStackRef_DUP(_PyStackRef stackref)
412 {
413 assert(!PyStackRef_IsNull(stackref));
414 if (PyStackRef_IsDeferred(stackref)) {
415 return stackref;
416 }
417 Py_INCREF(PyStackRef_AsPyObjectBorrow(stackref));
418 return stackref;
419 }
420
421 static inline _PyStackRef
422 PyStackRef_Borrow(_PyStackRef stackref)
423 {
424 return (_PyStackRef){ .bits = stackref.bits | Py_TAG_DEFERRED };
425 }
426
427 // Convert a possibly deferred reference to a strong reference.
428 static inline _PyStackRef
429 PyStackRef_AsStrongReference(_PyStackRef stackref)
430 {
431 return PyStackRef_FromPyObjectSteal(PyStackRef_AsPyObjectSteal(stackref));
432 }
433
434 #define PyStackRef_XCLOSE(stackref) \
435 do { \
436 _PyStackRef _tmp = (stackref); \
437 if (!PyStackRef_IsNull(_tmp)) { \
438 PyStackRef_CLOSE(_tmp); \
439 } \
440 } while (0);
441
442 #define PyStackRef_CLEAR(op) \
443 do { \
444 _PyStackRef *_tmp_op_ptr = &(op); \
445 _PyStackRef _tmp_old_op = (*_tmp_op_ptr); \
446 if (!PyStackRef_IsNull(_tmp_old_op)) { \
447 *_tmp_op_ptr = PyStackRef_NULL; \
448 PyStackRef_CLOSE(_tmp_old_op); \
449 } \
450 } while (0)
451
452 #define PyStackRef_FromPyObjectNewMortal PyStackRef_FromPyObjectNew
453
454 #else // Py_GIL_DISABLED
455
456 // With GIL
457
458 /* References to immortal objects always have their tag bit set to Py_TAG_REFCNT
459 * as they can (must) have their reclamation deferred */
460
461 #define Py_TAG_BITS 3
462 #if _Py_IMMORTAL_FLAGS != Py_TAG_REFCNT
463 # error "_Py_IMMORTAL_FLAGS != Py_TAG_REFCNT"
464 #endif
465
466 #define BITS_TO_PTR(REF) ((PyObject *)((REF).bits))
467 #define BITS_TO_PTR_MASKED(REF) ((PyObject *)(((REF).bits) & (~Py_TAG_REFCNT)))
468
469 #define PyStackRef_NULL_BITS Py_TAG_REFCNT
470 static const _PyStackRef PyStackRef_NULL = { .bits = PyStackRef_NULL_BITS };
471
472 #define PyStackRef_IsNull(ref) ((ref).bits == PyStackRef_NULL_BITS)
473 #define PyStackRef_True ((_PyStackRef){.bits = ((uintptr_t)&_Py_TrueStruct) | Py_TAG_REFCNT })
474 #define PyStackRef_False ((_PyStackRef){.bits = ((uintptr_t)&_Py_FalseStruct) | Py_TAG_REFCNT })
475 #define PyStackRef_None ((_PyStackRef){.bits = ((uintptr_t)&_Py_NoneStruct) | Py_TAG_REFCNT })
476
477 #define PyStackRef_IsTrue(REF) ((REF).bits == (((uintptr_t)&_Py_TrueStruct) | Py_TAG_REFCNT))
478 #define PyStackRef_IsFalse(REF) ((REF).bits == (((uintptr_t)&_Py_FalseStruct) | Py_TAG_REFCNT))
479 #define PyStackRef_IsNone(REF) ((REF).bits == (((uintptr_t)&_Py_NoneStruct) | Py_TAG_REFCNT))
480
481 #ifdef Py_DEBUG
482
483 static inline void PyStackRef_CheckValid(_PyStackRef ref) {
484 assert(ref.bits != 0);
485 int tag = ref.bits & Py_TAG_BITS;
486 PyObject *obj = BITS_TO_PTR_MASKED(ref);
487 switch (tag) {
488 case 0:
489 /* Can be immortal if object was made immortal after reference came into existence */
490 assert(!_Py_IsStaticImmortal(obj));
491 break;
492 case Py_TAG_REFCNT:
493 break;
494 default:
495 assert(0);
496 }
497 }
498
499 #else
500
501 #define PyStackRef_CheckValid(REF) ((void)0)
502
503 #endif
504
505 #ifdef _WIN32
506 #define PyStackRef_RefcountOnObject(REF) (((REF).bits & Py_TAG_REFCNT) == 0)
507 #define PyStackRef_AsPyObjectBorrow BITS_TO_PTR_MASKED
508 #define PyStackRef_Borrow(REF) (_PyStackRef){ .bits = ((REF).bits) | Py_TAG_REFCNT};
509 #else
510 /* Does this ref not have an embedded refcount and thus not refer to a declared immmortal object? */
511 static inline int
512 PyStackRef_RefcountOnObject(_PyStackRef ref)
513 {
514 return (ref.bits & Py_TAG_REFCNT) == 0;
515 }
516
517 static inline PyObject *
518 PyStackRef_AsPyObjectBorrow(_PyStackRef ref)
519 {
520 assert(!PyStackRef_IsTaggedInt(ref));
521 return BITS_TO_PTR_MASKED(ref);
522 }
523
524 static inline _PyStackRef
525 PyStackRef_Borrow(_PyStackRef ref)
526 {
527 return (_PyStackRef){ .bits = ref.bits | Py_TAG_REFCNT };
528 }
529 #endif
530
531 static inline PyObject *
532 PyStackRef_AsPyObjectSteal(_PyStackRef ref)
533 {
534 if (PyStackRef_RefcountOnObject(ref)) {
535 return BITS_TO_PTR(ref);
536 }
537 else {
538 return Py_NewRef(BITS_TO_PTR_MASKED(ref));
539 }
540 }
541
542 static inline _PyStackRef
543 PyStackRef_FromPyObjectSteal(PyObject *obj)
544 {
545 assert(obj != NULL);
546 #if SIZEOF_VOID_P > 4
547 unsigned int tag = obj->ob_flags & Py_TAG_REFCNT;
548 #else
549 unsigned int tag = _Py_IsImmortal(obj) ? Py_TAG_REFCNT : 0;
550 #endif
551 _PyStackRef ref = ((_PyStackRef){.bits = ((uintptr_t)(obj)) | tag});
552 PyStackRef_CheckValid(ref);
553 return ref;
554 }
555
556 static inline _PyStackRef
557 PyStackRef_FromPyObjectBorrow(PyObject *obj)
558 {
559 assert(obj != NULL);
560 return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_REFCNT };
561 }
562
563 static inline _PyStackRef
564 PyStackRef_FromPyObjectStealMortal(PyObject *obj)
565 {
566 assert(obj != NULL);
567 assert(!_Py_IsImmortal(obj));
568 _PyStackRef ref = ((_PyStackRef){.bits = ((uintptr_t)(obj)) });
569 PyStackRef_CheckValid(ref);
570 return ref;
571 }
572
573 static inline _PyStackRef
574 _PyStackRef_FromPyObjectNew(PyObject *obj)
575 {
576 assert(obj != NULL);
577 if (_Py_IsImmortal(obj)) {
578 return (_PyStackRef){ .bits = ((uintptr_t)obj) | Py_TAG_REFCNT};
579 }
580 _Py_INCREF_MORTAL(obj);
581 _PyStackRef ref = (_PyStackRef){ .bits = (uintptr_t)obj };
582 PyStackRef_CheckValid(ref);
583 return ref;
584 }
585 #define PyStackRef_FromPyObjectNew(obj) _PyStackRef_FromPyObjectNew(_PyObject_CAST(obj))
586
587 static inline _PyStackRef
588 _PyStackRef_FromPyObjectNewMortal(PyObject *obj)
589 {
590 assert(obj != NULL);
591 _Py_INCREF_MORTAL(obj);
592 _PyStackRef ref = (_PyStackRef){ .bits = (uintptr_t)obj };
593 PyStackRef_CheckValid(ref);
594 return ref;
595 }
596 #define PyStackRef_FromPyObjectNewMortal(obj) _PyStackRef_FromPyObjectNewMortal(_PyObject_CAST(obj))
597
598 /* Create a new reference from an object with an embedded reference count */
599 static inline _PyStackRef
600 PyStackRef_FromPyObjectImmortal(PyObject *obj)
601 {
602 assert(_Py_IsImmortal(obj));
603 return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_REFCNT};
604 }
605
606 /* WARNING: This macro evaluates its argument more than once */
607 #ifdef _WIN32
608 #define PyStackRef_DUP(REF) \
609 (PyStackRef_RefcountOnObject(REF) ? (_Py_INCREF_MORTAL(BITS_TO_PTR(REF)), (REF)) : (REF))
610 #else
611 static inline _PyStackRef
612 PyStackRef_DUP(_PyStackRef ref)
613 {
614 assert(!PyStackRef_IsNull(ref));
615 if (PyStackRef_RefcountOnObject(ref)) {
616 _Py_INCREF_MORTAL(BITS_TO_PTR(ref));
617 }
618 return ref;
619 }
620 #endif
621
622 static inline bool
623 PyStackRef_IsHeapSafe(_PyStackRef ref)
624 {
625 return (ref.bits & Py_TAG_BITS) != Py_TAG_REFCNT || ref.bits == PyStackRef_NULL_BITS || _Py_IsImmortal(BITS_TO_PTR_MASKED(ref));
626 }
627
628 static inline _PyStackRef
629 PyStackRef_MakeHeapSafe(_PyStackRef ref)
630 {
631 if (PyStackRef_IsHeapSafe(ref)) {
632 return ref;
633 }
634 PyObject *obj = BITS_TO_PTR_MASKED(ref);
635 Py_INCREF(obj);
636 ref.bits = (uintptr_t)obj;
637 PyStackRef_CheckValid(ref);
638 return ref;
639 }
640
641 #ifdef _WIN32
642 #define PyStackRef_CLOSE(REF) \
643 do { \
644 _PyStackRef _temp = (REF); \
645 if (PyStackRef_RefcountOnObject(_temp)) Py_DECREF_MORTAL(BITS_TO_PTR(_temp)); \
646 } while (0)
647 #else
648 static inline void
649 PyStackRef_CLOSE(_PyStackRef ref)
650 {
651 assert(!PyStackRef_IsNull(ref));
652 if (PyStackRef_RefcountOnObject(ref)) {
653 Py_DECREF_MORTAL(BITS_TO_PTR(ref));
654 }
655 }
656 #endif
657
658 static inline bool
659 PyStackRef_IsNullOrInt(_PyStackRef ref)
660 {
661 return PyStackRef_IsNull(ref) || PyStackRef_IsTaggedInt(ref);
662 }
663
664 static inline void
665 PyStackRef_CLOSE_SPECIALIZED(_PyStackRef ref, destructor destruct)
666 {
667 assert(!PyStackRef_IsNull(ref));
668 if (PyStackRef_RefcountOnObject(ref)) {
669 Py_DECREF_MORTAL_SPECIALIZED(BITS_TO_PTR(ref), destruct);
670 }
671 }
672
673 #ifdef _WIN32
674 #define PyStackRef_XCLOSE PyStackRef_CLOSE
675 #else
676 static inline void
677 PyStackRef_XCLOSE(_PyStackRef ref)
678 {
679 assert(ref.bits != 0);
680 if (PyStackRef_RefcountOnObject(ref)) {
681 assert(!PyStackRef_IsNull(ref));
682 Py_DECREF_MORTAL(BITS_TO_PTR(ref));
683 }
684 }
685 #endif
686
687 #define PyStackRef_CLEAR(REF) \
688 do { \
689 _PyStackRef *_tmp_op_ptr = &(REF); \
690 _PyStackRef _tmp_old_op = (*_tmp_op_ptr); \
691 *_tmp_op_ptr = PyStackRef_NULL; \
692 PyStackRef_XCLOSE(_tmp_old_op); \
693 } while (0)
694
695
696 #endif // Py_GIL_DISABLED
697
698 // Note: this is a macro because MSVC (Windows) has trouble inlining it.
699
700 #define PyStackRef_Is(a, b) (((a).bits & (~Py_TAG_REFCNT)) == ((b).bits & (~Py_TAG_REFCNT)))
701
702
703 #endif // !defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG)
704
705 #define PyStackRef_TYPE(stackref) Py_TYPE(PyStackRef_AsPyObjectBorrow(stackref))
706
707 // Converts a PyStackRef back to a PyObject *, converting the
708 // stackref to a new reference.
709 #define PyStackRef_AsPyObjectNew(stackref) Py_NewRef(PyStackRef_AsPyObjectBorrow(stackref))
710
711 // StackRef type checks
712
713 static inline bool
714 PyStackRef_GenCheck(_PyStackRef stackref)
715 {
716 return PyGen_Check(PyStackRef_AsPyObjectBorrow(stackref));
717 }
718
719 static inline bool
720 PyStackRef_BoolCheck(_PyStackRef stackref)
721 {
722 return PyBool_Check(PyStackRef_AsPyObjectBorrow(stackref));
723 }
724
725 static inline bool
726 PyStackRef_LongCheck(_PyStackRef stackref)
727 {
728 return PyLong_Check(PyStackRef_AsPyObjectBorrow(stackref));
729 }
730
731 static inline bool
732 PyStackRef_ExceptionInstanceCheck(_PyStackRef stackref)
733 {
734 return PyExceptionInstance_Check(PyStackRef_AsPyObjectBorrow(stackref));
735 }
736
737 static inline bool
738 PyStackRef_CodeCheck(_PyStackRef stackref)
739 {
740 return PyCode_Check(PyStackRef_AsPyObjectBorrow(stackref));
741 }
742
743 static inline bool
744 PyStackRef_FunctionCheck(_PyStackRef stackref)
745 {
746 return PyFunction_Check(PyStackRef_AsPyObjectBorrow(stackref));
747 }
748
749 static inline void
750 _PyThreadState_PushCStackRef(PyThreadState *tstate, _PyCStackRef *ref)
751 {
752 #ifdef Py_GIL_DISABLED
753 _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate;
754 ref->next = tstate_impl->c_stack_refs;
755 tstate_impl->c_stack_refs = ref;
756 #endif
757 ref->ref = PyStackRef_NULL;
758 }
759
760 static inline void
761 _PyThreadState_PushCStackRefNew(PyThreadState *tstate, _PyCStackRef *ref, PyObject *obj)
762 {
763 _PyThreadState_PushCStackRef(tstate, ref);
764 ref->ref = PyStackRef_FromPyObjectNew(obj);
765 }
766
767 static inline void
768 _PyThreadState_PopCStackRef(PyThreadState *tstate, _PyCStackRef *ref)
769 {
770 #ifdef Py_GIL_DISABLED
771 _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate;
772 assert(tstate_impl->c_stack_refs == ref);
773 tstate_impl->c_stack_refs = ref->next;
774 #endif
775 PyStackRef_XCLOSE(ref->ref);
776 }
777
778 static inline _PyStackRef
779 _PyThreadState_PopCStackRefSteal(PyThreadState *tstate, _PyCStackRef *ref)
780 {
781 #ifdef Py_GIL_DISABLED
782 _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate;
783 assert(tstate_impl->c_stack_refs == ref);
784 tstate_impl->c_stack_refs = ref->next;
785 #endif
786 return ref->ref;
787 }
788
789 #ifdef Py_GIL_DISABLED
790
791 static inline int
792 _Py_TryIncrefCompareStackRef(PyObject **src, PyObject *op, _PyStackRef *out)
793 {
794 if (_PyObject_HasDeferredRefcount(op)) {
795 *out = (_PyStackRef){ .bits = (uintptr_t)op | Py_TAG_DEFERRED };
796 return 1;
797 }
798 if (_Py_TryIncrefCompare(src, op)) {
799 *out = PyStackRef_FromPyObjectSteal(op);
800 return 1;
801 }
802 return 0;
803 }
804
805 static inline int
806 _Py_TryXGetStackRef(PyObject **src, _PyStackRef *out)
807 {
808 PyObject *op = _PyObject_CAST(_Py_atomic_load_ptr_relaxed(src));
809 if (op == NULL) {
810 *out = PyStackRef_NULL;
811 return 1;
812 }
813 return _Py_TryIncrefCompareStackRef(src, op, out);
814 }
815
816 #endif
817
818 #define PyStackRef_XSETREF(dst, src) \
819 do { \
820 _PyStackRef _tmp_dst_ref = (dst); \
821 (dst) = (src); \
822 PyStackRef_XCLOSE(_tmp_dst_ref); \
823 } while(0)
824
825 // Like Py_VISIT but for _PyStackRef fields
826 #define _Py_VISIT_STACKREF(ref) \
827 do { \
828 if (!PyStackRef_IsNullOrInt(ref)) { \
829 int vret = _PyGC_VisitStackRef(&(ref), visit, arg); \
830 if (vret) \
831 return vret; \
832 } \
833 } while (0)
834
835 #ifdef __cplusplus
836 }
837 #endif
838 #endif /* !Py_INTERNAL_STACKREF_H */