Debian Codegraph @

Include/internal/pycore_critical_section.h

raw
1 #ifndef Py_INTERNAL_CRITICAL_SECTION_H
2 #define Py_INTERNAL_CRITICAL_SECTION_H
3
4 #ifndef Py_BUILD_CORE
5 # error "this header requires Py_BUILD_CORE define"
6 #endif
7
8 #include "pycore_lock.h" // PyMutex_LockFast()
9 #include "pycore_pystate.h" // _PyThreadState_GET()
10 #include <stdint.h>
11
12 #ifdef __cplusplus
13 extern "C" {
14 #endif
15
16 // Tagged pointers to critical sections use the two least significant bits to
17 // mark if the pointed-to critical section is inactive and whether it is a
18 // PyCriticalSection2 object.
19 #define _Py_CRITICAL_SECTION_INACTIVE 0x1
20 #define _Py_CRITICAL_SECTION_TWO_MUTEXES 0x2
21 #define _Py_CRITICAL_SECTION_MASK 0x3
22
23 #ifdef Py_GIL_DISABLED
24 // Specialized version of critical section locking to safely use
25 // PySequence_Fast APIs without the GIL. For performance, the argument *to*
26 // PySequence_Fast() is provided to the macro, not the *result* of
27 // PySequence_Fast(), which would require an extra test to determine if the
28 // lock must be acquired.
29 # define Py_BEGIN_CRITICAL_SECTION_SEQUENCE_FAST(original) \
30 { \
31 PyObject *_orig_seq = _PyObject_CAST(original); \
32 const bool _should_lock_cs = PyList_CheckExact(_orig_seq); \
33 PyCriticalSection _cs; \
34 if (_should_lock_cs) { \
35 _PyCriticalSection_Begin(&_cs, _orig_seq); \
36 }
37
38 # define Py_END_CRITICAL_SECTION_SEQUENCE_FAST() \
39 if (_should_lock_cs) { \
40 PyCriticalSection_End(&_cs); \
41 } \
42 }
43
44 // Asserts that the mutex is locked. The mutex must be held by the
45 // top-most critical section otherwise there's the possibility
46 // that the mutex would be swalled out in some code paths.
47 #define _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(mutex) \
48 _PyCriticalSection_AssertHeld(mutex)
49
50 // Asserts that the mutex for the given object is locked. The mutex must
51 // be held by the top-most critical section otherwise there's the
52 // possibility that the mutex would be swalled out in some code paths.
53 #ifdef Py_DEBUG
54
55 # define _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op) \
56 if (Py_REFCNT(op) != 1) { \
57 _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(&_PyObject_CAST(op)->ob_mutex); \
58 }
59
60 #else /* Py_DEBUG */
61
62 # define _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op)
63
64 #endif /* Py_DEBUG */
65
66 #else /* !Py_GIL_DISABLED */
67 // The critical section APIs are no-ops with the GIL.
68 # define Py_BEGIN_CRITICAL_SECTION_SEQUENCE_FAST(original) {
69 # define Py_END_CRITICAL_SECTION_SEQUENCE_FAST() }
70 # define _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(mutex)
71 # define _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op)
72 #endif /* !Py_GIL_DISABLED */
73
74 // Resumes the top-most critical section.
75 PyAPI_FUNC(void)
76 _PyCriticalSection_Resume(PyThreadState *tstate);
77
78 // (private) slow path for locking the mutex
79 PyAPI_FUNC(void)
80 _PyCriticalSection_BeginSlow(PyCriticalSection *c, PyMutex *m);
81
82 PyAPI_FUNC(void)
83 _PyCriticalSection2_BeginSlow(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2,
84 int is_m1_locked);
85
86 PyAPI_FUNC(void)
87 _PyCriticalSection_SuspendAll(PyThreadState *tstate);
88
89 #ifdef Py_GIL_DISABLED
90
91 static inline int
92 _PyCriticalSection_IsActive(uintptr_t tag)
93 {
94 return tag != 0 && (tag & _Py_CRITICAL_SECTION_INACTIVE) == 0;
95 }
96
97 static inline void
98 _PyCriticalSection_BeginMutex(PyCriticalSection *c, PyMutex *m)
99 {
100 if (PyMutex_LockFast(m)) {
101 PyThreadState *tstate = _PyThreadState_GET();
102 c->_cs_mutex = m;
103 c->_cs_prev = tstate->critical_section;
104 tstate->critical_section = (uintptr_t)c;
105 }
106 else {
107 _PyCriticalSection_BeginSlow(c, m);
108 }
109 }
110 #define PyCriticalSection_BeginMutex _PyCriticalSection_BeginMutex
111
112 static inline void
113 _PyCriticalSection_Begin(PyCriticalSection *c, PyObject *op)
114 {
115 _PyCriticalSection_BeginMutex(c, &op->ob_mutex);
116 }
117 #define PyCriticalSection_Begin _PyCriticalSection_Begin
118
119 // Removes the top-most critical section from the thread's stack of critical
120 // sections. If the new top-most critical section is inactive, then it is
121 // resumed.
122 static inline void
123 _PyCriticalSection_Pop(PyCriticalSection *c)
124 {
125 PyThreadState *tstate = _PyThreadState_GET();
126 uintptr_t prev = c->_cs_prev;
127 tstate->critical_section = prev;
128
129 if ((prev & _Py_CRITICAL_SECTION_INACTIVE) != 0) {
130 _PyCriticalSection_Resume(tstate);
131 }
132 }
133
134 static inline void
135 _PyCriticalSection_End(PyCriticalSection *c)
136 {
137 // If the mutex is NULL, we used the fast path in
138 // _PyCriticalSection_BeginSlow for locks already held in the top-most
139 // critical section, and we shouldn't unlock or pop this critical section.
140 if (c->_cs_mutex == NULL) {
141 return;
142 }
143 PyMutex_Unlock(c->_cs_mutex);
144 _PyCriticalSection_Pop(c);
145 }
146 #define PyCriticalSection_End _PyCriticalSection_End
147
148 static inline void
149 _PyCriticalSection2_BeginMutex(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2)
150 {
151 if (m1 == m2) {
152 // If the two mutex arguments are the same, treat this as a critical
153 // section with a single mutex.
154 c->_cs_mutex2 = NULL;
155 _PyCriticalSection_BeginMutex(&c->_cs_base, m1);
156 return;
157 }
158
159 if ((uintptr_t)m2 < (uintptr_t)m1) {
160 // Sort the mutexes so that the lower address is locked first.
161 // The exact order does not matter, but we need to acquire the mutexes
162 // in a consistent order to avoid lock ordering deadlocks.
163 PyMutex *tmp = m1;
164 m1 = m2;
165 m2 = tmp;
166 }
167
168 if (PyMutex_LockFast(m1)) {
169 if (PyMutex_LockFast(m2)) {
170 PyThreadState *tstate = _PyThreadState_GET();
171 c->_cs_base._cs_mutex = m1;
172 c->_cs_mutex2 = m2;
173 c->_cs_base._cs_prev = tstate->critical_section;
174
175 uintptr_t p = (uintptr_t)c | _Py_CRITICAL_SECTION_TWO_MUTEXES;
176 tstate->critical_section = p;
177 }
178 else {
179 _PyCriticalSection2_BeginSlow(c, m1, m2, 1);
180 }
181 }
182 else {
183 _PyCriticalSection2_BeginSlow(c, m1, m2, 0);
184 }
185 }
186 #define PyCriticalSection2_BeginMutex _PyCriticalSection2_BeginMutex
187
188 static inline void
189 _PyCriticalSection2_Begin(PyCriticalSection2 *c, PyObject *a, PyObject *b)
190 {
191 _PyCriticalSection2_BeginMutex(c, &a->ob_mutex, &b->ob_mutex);
192 }
193 #define PyCriticalSection2_Begin _PyCriticalSection2_Begin
194
195 static inline void
196 _PyCriticalSection2_End(PyCriticalSection2 *c)
197 {
198 // if mutex1 is NULL, we used the fast path in
199 // _PyCriticalSection_BeginSlow for mutexes that are already held,
200 // which should only happen when mutex1 and mutex2 were the same mutex,
201 // and mutex2 should also be NULL.
202 if (c->_cs_base._cs_mutex == NULL) {
203 assert(c->_cs_mutex2 == NULL);
204 return;
205 }
206 if (c->_cs_mutex2) {
207 PyMutex_Unlock(c->_cs_mutex2);
208 }
209 PyMutex_Unlock(c->_cs_base._cs_mutex);
210 _PyCriticalSection_Pop(&c->_cs_base);
211 }
212 #define PyCriticalSection2_End _PyCriticalSection2_End
213
214 static inline void
215 _PyCriticalSection_AssertHeld(PyMutex *mutex)
216 {
217 #ifdef Py_DEBUG
218 PyThreadState *tstate = _PyThreadState_GET();
219 uintptr_t prev = tstate->critical_section;
220 if (prev & _Py_CRITICAL_SECTION_TWO_MUTEXES) {
221 PyCriticalSection2 *cs = (PyCriticalSection2 *)(prev & ~_Py_CRITICAL_SECTION_MASK);
222 assert(cs != NULL && (cs->_cs_base._cs_mutex == mutex || cs->_cs_mutex2 == mutex));
223 }
224 else {
225 PyCriticalSection *cs = (PyCriticalSection *)(tstate->critical_section & ~_Py_CRITICAL_SECTION_MASK);
226 assert(cs != NULL && cs->_cs_mutex == mutex);
227 }
228
229 #endif
230 }
231
232 #endif /* Py_GIL_DISABLED */
233
234 #ifdef __cplusplus
235 }
236 #endif
237 #endif /* !Py_INTERNAL_CRITICAL_SECTION_H */