| 1 |
// Lightweight locks and other synchronization mechanisms. |
| 2 |
// |
| 3 |
// These implementations are based on WebKit's WTF::Lock. See |
| 4 |
// https://webkit.org/blog/6161/locking-in-webkit/ for a description of the |
| 5 |
// design. |
| 6 |
#ifndef Py_INTERNAL_LOCK_H |
| 7 |
#define Py_INTERNAL_LOCK_H |
| 8 |
#ifdef __cplusplus |
| 9 |
extern "C" { |
| 10 |
#endif |
| 11 |
|
| 12 |
#ifndef Py_BUILD_CORE |
| 13 |
# error "this header requires Py_BUILD_CORE define" |
| 14 |
#endif |
| 15 |
|
| 16 |
//_Py_UNLOCKED is defined as 0 and _Py_LOCKED as 1 in Include/cpython/lock.h |
| 17 |
#define _Py_HAS_PARKED 2 |
| 18 |
#define _Py_ONCE_INITIALIZED 4 |
| 19 |
|
| 20 |
static inline int |
| 21 |
PyMutex_LockFast(PyMutex *m) |
| 22 |
{ |
| 23 |
uint8_t expected = _Py_UNLOCKED; |
| 24 |
uint8_t *lock_bits = &m->_bits; |
| 25 |
return _Py_atomic_compare_exchange_uint8(lock_bits, &expected, _Py_LOCKED); |
| 26 |
} |
| 27 |
|
| 28 |
// Re-initializes the mutex after a fork to the unlocked state. |
| 29 |
static inline void |
| 30 |
_PyMutex_at_fork_reinit(PyMutex *m) |
| 31 |
{ |
| 32 |
memset(m, 0, sizeof(*m)); |
| 33 |
} |
| 34 |
|
| 35 |
typedef enum _PyLockFlags { |
| 36 |
// Do not detach/release the GIL when waiting on the lock. |
| 37 |
_Py_LOCK_DONT_DETACH = 0, |
| 38 |
|
| 39 |
// Detach/release the GIL while waiting on the lock. |
| 40 |
_PY_LOCK_DETACH = 1, |
| 41 |
|
| 42 |
// Handle signals if interrupted while waiting on the lock. |
| 43 |
_PY_LOCK_HANDLE_SIGNALS = 2, |
| 44 |
} _PyLockFlags; |
| 45 |
|
| 46 |
// Lock a mutex with an optional timeout and additional options. See |
| 47 |
// _PyLockFlags for details. |
| 48 |
extern PyAPI_FUNC(PyLockStatus) |
| 49 |
_PyMutex_LockTimed(PyMutex *m, PyTime_t timeout_ns, _PyLockFlags flags); |
| 50 |
|
| 51 |
// Lock a mutex with additional options. See _PyLockFlags for details. |
| 52 |
static inline void |
| 53 |
PyMutex_LockFlags(PyMutex *m, _PyLockFlags flags) |
| 54 |
{ |
| 55 |
uint8_t expected = _Py_UNLOCKED; |
| 56 |
if (!_Py_atomic_compare_exchange_uint8(&m->_bits, &expected, _Py_LOCKED)) { |
| 57 |
_PyMutex_LockTimed(m, -1, flags); |
| 58 |
} |
| 59 |
} |
| 60 |
|
| 61 |
// Unlock a mutex, returns -1 if the mutex is not locked (used for improved |
| 62 |
// error messages) otherwise returns 0. |
| 63 |
extern int _PyMutex_TryUnlock(PyMutex *m); |
| 64 |
|
| 65 |
|
| 66 |
// PyEvent is a one-time event notification |
| 67 |
typedef struct { |
| 68 |
uint8_t v; |
| 69 |
} PyEvent; |
| 70 |
|
| 71 |
// Check if the event is set without blocking. Returns 1 if the event is set or |
| 72 |
// 0 otherwise. |
| 73 |
PyAPI_FUNC(int) _PyEvent_IsSet(PyEvent *evt); |
| 74 |
|
| 75 |
// Set the event and notify any waiting threads. |
| 76 |
// Export for '_testinternalcapi' shared extension |
| 77 |
PyAPI_FUNC(void) _PyEvent_Notify(PyEvent *evt); |
| 78 |
|
| 79 |
// Wait for the event to be set. If the event is already set, then this returns |
| 80 |
// immediately. |
| 81 |
PyAPI_FUNC(void) PyEvent_Wait(PyEvent *evt); |
| 82 |
|
| 83 |
// Wait for the event to be set, or until the timeout expires. If the event is |
| 84 |
// already set, then this returns immediately. Returns 1 if the event was set, |
| 85 |
// and 0 if the timeout expired or thread was interrupted. If `detach` is |
| 86 |
// true, then the thread will detach/release the GIL while waiting. |
| 87 |
PyAPI_FUNC(int) |
| 88 |
PyEvent_WaitTimed(PyEvent *evt, PyTime_t timeout_ns, int detach); |
| 89 |
|
| 90 |
// _PyRawMutex implements a word-sized mutex that that does not depend on the |
| 91 |
// parking lot API, and therefore can be used in the parking lot |
| 92 |
// implementation. |
| 93 |
// |
| 94 |
// The mutex uses a packed representation: the least significant bit is used to |
| 95 |
// indicate whether the mutex is locked or not. The remaining bits are either |
| 96 |
// zero or a pointer to a `struct raw_mutex_entry` (see lock.c). |
| 97 |
typedef struct { |
| 98 |
uintptr_t v; |
| 99 |
} _PyRawMutex; |
| 100 |
|
| 101 |
// Slow paths for lock/unlock |
| 102 |
extern void _PyRawMutex_LockSlow(_PyRawMutex *m); |
| 103 |
extern void _PyRawMutex_UnlockSlow(_PyRawMutex *m); |
| 104 |
|
| 105 |
static inline void |
| 106 |
_PyRawMutex_Lock(_PyRawMutex *m) |
| 107 |
{ |
| 108 |
uintptr_t unlocked = _Py_UNLOCKED; |
| 109 |
if (_Py_atomic_compare_exchange_uintptr(&m->v, &unlocked, _Py_LOCKED)) { |
| 110 |
return; |
| 111 |
} |
| 112 |
_PyRawMutex_LockSlow(m); |
| 113 |
} |
| 114 |
|
| 115 |
static inline void |
| 116 |
_PyRawMutex_Unlock(_PyRawMutex *m) |
| 117 |
{ |
| 118 |
uintptr_t locked = _Py_LOCKED; |
| 119 |
if (_Py_atomic_compare_exchange_uintptr(&m->v, &locked, _Py_UNLOCKED)) { |
| 120 |
return; |
| 121 |
} |
| 122 |
_PyRawMutex_UnlockSlow(m); |
| 123 |
} |
| 124 |
|
| 125 |
// Type signature for one-time initialization functions. The function should |
| 126 |
// return 0 on success and -1 on failure. |
| 127 |
typedef int _Py_once_fn_t(void *arg); |
| 128 |
|
| 129 |
// (private) slow path for one time initialization |
| 130 |
PyAPI_FUNC(int) |
| 131 |
_PyOnceFlag_CallOnceSlow(_PyOnceFlag *flag, _Py_once_fn_t *fn, void *arg); |
| 132 |
|
| 133 |
// Calls `fn` once using `flag`. The `arg` is passed to the call to `fn`. |
| 134 |
// |
| 135 |
// Returns 0 on success and -1 on failure. |
| 136 |
// |
| 137 |
// If `fn` returns 0 (success), then subsequent calls immediately return 0. |
| 138 |
// If `fn` returns -1 (failure), then subsequent calls will retry the call. |
| 139 |
static inline int |
| 140 |
_PyOnceFlag_CallOnce(_PyOnceFlag *flag, _Py_once_fn_t *fn, void *arg) |
| 141 |
{ |
| 142 |
if (_Py_atomic_load_uint8(&flag->v) == _Py_ONCE_INITIALIZED) { |
| 143 |
return 0; |
| 144 |
} |
| 145 |
return _PyOnceFlag_CallOnceSlow(flag, fn, arg); |
| 146 |
} |
| 147 |
|
| 148 |
// A recursive mutex. The mutex should zero-initialized. |
| 149 |
typedef struct { |
| 150 |
PyMutex mutex; |
| 151 |
unsigned long long thread; // i.e., PyThread_get_thread_ident_ex() |
| 152 |
size_t level; |
| 153 |
} _PyRecursiveMutex; |
| 154 |
|
| 155 |
PyAPI_FUNC(int) _PyRecursiveMutex_IsLockedByCurrentThread(_PyRecursiveMutex *m); |
| 156 |
PyAPI_FUNC(void) _PyRecursiveMutex_Lock(_PyRecursiveMutex *m); |
| 157 |
extern PyLockStatus _PyRecursiveMutex_LockTimed(_PyRecursiveMutex *m, PyTime_t timeout, _PyLockFlags flags); |
| 158 |
PyAPI_FUNC(void) _PyRecursiveMutex_Unlock(_PyRecursiveMutex *m); |
| 159 |
extern int _PyRecursiveMutex_TryUnlock(_PyRecursiveMutex *m); |
| 160 |
|
| 161 |
// A readers-writer (RW) lock. The lock supports multiple concurrent readers or |
| 162 |
// a single writer. The lock is write-preferring: if a writer is waiting while |
| 163 |
// the lock is read-locked then, new readers will be blocked. This avoids |
| 164 |
// starvation of writers. |
| 165 |
// |
| 166 |
// In C++, the equivalent synchronization primitive is std::shared_mutex |
| 167 |
// with shared ("read") and exclusive ("write") locking. |
| 168 |
// |
| 169 |
// The two least significant bits are used to indicate if the lock is |
| 170 |
// write-locked and if there are parked threads (either readers or writers) |
| 171 |
// waiting to acquire the lock. The remaining bits are used to indicate the |
| 172 |
// number of readers holding the lock. |
| 173 |
// |
| 174 |
// 0b000..00000: unlocked |
| 175 |
// 0bnnn..nnn00: nnn..nnn readers holding the lock |
| 176 |
// 0bnnn..nnn10: nnn..nnn readers holding the lock and a writer is waiting |
| 177 |
// 0b00000..010: unlocked with awoken writer about to acquire lock |
| 178 |
// 0b00000..001: write-locked |
| 179 |
// 0b00000..011: write-locked and readers or other writers are waiting |
| 180 |
// |
| 181 |
// Note that reader_count must be zero if the lock is held by a writer, and |
| 182 |
// vice versa. The lock can only be held by readers or a writer, but not both. |
| 183 |
// |
| 184 |
// The design is optimized for simplicity of the implementation. The lock is |
| 185 |
// not fair: if fairness is desired, use an additional PyMutex to serialize |
| 186 |
// writers. The lock is also not reentrant. |
| 187 |
typedef struct { |
| 188 |
uintptr_t bits; |
| 189 |
} _PyRWMutex; |
| 190 |
|
| 191 |
// Read lock (i.e., shared lock) |
| 192 |
PyAPI_FUNC(void) _PyRWMutex_RLock(_PyRWMutex *rwmutex); |
| 193 |
PyAPI_FUNC(void) _PyRWMutex_RUnlock(_PyRWMutex *rwmutex); |
| 194 |
|
| 195 |
// Write lock (i.e., exclusive lock) |
| 196 |
PyAPI_FUNC(void) _PyRWMutex_Lock(_PyRWMutex *rwmutex); |
| 197 |
PyAPI_FUNC(void) _PyRWMutex_Unlock(_PyRWMutex *rwmutex); |
| 198 |
|
| 199 |
// Similar to linux seqlock: https://en.wikipedia.org/wiki/Seqlock |
| 200 |
// We use a sequence number to lock the writer, an even sequence means we're unlocked, an odd |
| 201 |
// sequence means we're locked. Readers will read the sequence before attempting to read the |
| 202 |
// underlying data and then read the sequence number again after reading the data. If the |
| 203 |
// sequence has not changed the data is valid. |
| 204 |
// |
| 205 |
// Differs a little bit in that we use CAS on sequence as the lock, instead of a separate spin lock. |
| 206 |
// The writer can also detect that the undelering data has not changed and abandon the write |
| 207 |
// and restore the previous sequence. |
| 208 |
typedef struct { |
| 209 |
uint32_t sequence; |
| 210 |
} _PySeqLock; |
| 211 |
|
| 212 |
// Lock the sequence lock for the writer |
| 213 |
PyAPI_FUNC(void) _PySeqLock_LockWrite(_PySeqLock *seqlock); |
| 214 |
|
| 215 |
// Unlock the sequence lock and move to the next sequence number. |
| 216 |
PyAPI_FUNC(void) _PySeqLock_UnlockWrite(_PySeqLock *seqlock); |
| 217 |
|
| 218 |
// Abandon the current update indicating that no mutations have occurred |
| 219 |
// and restore the previous sequence value. |
| 220 |
PyAPI_FUNC(void) _PySeqLock_AbandonWrite(_PySeqLock *seqlock); |
| 221 |
|
| 222 |
// Begin a read operation and return the current sequence number. |
| 223 |
PyAPI_FUNC(uint32_t) _PySeqLock_BeginRead(_PySeqLock *seqlock); |
| 224 |
|
| 225 |
// End the read operation and confirm that the sequence number has not changed. |
| 226 |
// Returns 1 if the read was successful or 0 if the read should be retried. |
| 227 |
PyAPI_FUNC(int) _PySeqLock_EndRead(_PySeqLock *seqlock, uint32_t previous); |
| 228 |
|
| 229 |
// Check if the lock was held during a fork and clear the lock. Returns 1 |
| 230 |
// if the lock was held and any associated data should be cleared. |
| 231 |
PyAPI_FUNC(int) _PySeqLock_AfterFork(_PySeqLock *seqlock); |
| 232 |
|
| 233 |
#ifdef __cplusplus |
| 234 |
} |
| 235 |
#endif |
| 236 |
#endif /* !Py_INTERNAL_LOCK_H */ |