Debian Codegraph @

Include/cpython/pyatomic.h

raw
1 // This header provides cross-platform low-level atomic operations
2 // similar to C11 atomics.
3 //
4 // Operations are sequentially consistent unless they have a suffix indicating
5 // otherwise. If in doubt, prefer the sequentially consistent operations.
6 //
7 // The "_relaxed" suffix for load and store operations indicates the "relaxed"
8 // memory order. They don't provide synchronization, but (roughly speaking)
9 // guarantee somewhat sane behavior for races instead of undefined behavior.
10 // In practice, they correspond to "normal" hardware load and store
11 // instructions, so they are almost as inexpensive as plain loads and stores
12 // in C.
13 //
14 // Note that atomic read-modify-write operations like _Py_atomic_add_* return
15 // the previous value of the atomic variable, not the new value.
16 //
17 // See https://en.cppreference.com/w/c/atomic for more information on C11
18 // atomics.
19 // See https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2020/p2055r0.pdf
20 // "A Relaxed Guide to memory_order_relaxed" for discussion of and common usage
21 // or relaxed atomics.
22 //
23 // Functions with pseudo Python code:
24 //
25 // def _Py_atomic_load(obj):
26 // return obj # sequential consistency
27 //
28 // def _Py_atomic_load_relaxed(obj):
29 // return obj # relaxed consistency
30 //
31 // def _Py_atomic_store(obj, value):
32 // obj = value # sequential consistency
33 //
34 // def _Py_atomic_store_relaxed(obj, value):
35 // obj = value # relaxed consistency
36 //
37 // def _Py_atomic_exchange(obj, value):
38 // # sequential consistency
39 // old_obj = obj
40 // obj = value
41 // return old_obj
42 //
43 // def _Py_atomic_compare_exchange(obj, expected, desired):
44 // # sequential consistency
45 // if obj == expected:
46 // obj = desired
47 // return True
48 // else:
49 // expected = obj
50 // return False
51 //
52 // def _Py_atomic_add(obj, value):
53 // # sequential consistency
54 // old_obj = obj
55 // obj += value
56 // return old_obj
57 //
58 // def _Py_atomic_and(obj, value):
59 // # sequential consistency
60 // old_obj = obj
61 // obj &= value
62 // return old_obj
63 //
64 // def _Py_atomic_or(obj, value):
65 // # sequential consistency
66 // old_obj = obj
67 // obj |= value
68 // return old_obj
69 //
70 // Other functions:
71 //
72 // def _Py_atomic_load_ptr_acquire(obj):
73 // return obj # acquire
74 //
75 // def _Py_atomic_store_ptr_release(obj, value):
76 // obj = value # release
77 //
78 // def _Py_atomic_fence_seq_cst():
79 // # sequential consistency
80 // ...
81 //
82 // def _Py_atomic_fence_release():
83 // # release
84 // ...
85
86 #ifndef Py_CPYTHON_ATOMIC_H
87 # error "this header file must not be included directly"
88 #endif
89
90 // --- _Py_atomic_add --------------------------------------------------------
91 // Atomically adds `value` to `obj` and returns the previous value
92
93 static inline int
94 _Py_atomic_add_int(int *obj, int value);
95
96 static inline int8_t
97 _Py_atomic_add_int8(int8_t *obj, int8_t value);
98
99 static inline int16_t
100 _Py_atomic_add_int16(int16_t *obj, int16_t value);
101
102 static inline int32_t
103 _Py_atomic_add_int32(int32_t *obj, int32_t value);
104
105 static inline int64_t
106 _Py_atomic_add_int64(int64_t *obj, int64_t value);
107
108 static inline intptr_t
109 _Py_atomic_add_intptr(intptr_t *obj, intptr_t value);
110
111 static inline unsigned int
112 _Py_atomic_add_uint(unsigned int *obj, unsigned int value);
113
114 static inline uint8_t
115 _Py_atomic_add_uint8(uint8_t *obj, uint8_t value);
116
117 static inline uint16_t
118 _Py_atomic_add_uint16(uint16_t *obj, uint16_t value);
119
120 static inline uint32_t
121 _Py_atomic_add_uint32(uint32_t *obj, uint32_t value);
122
123 static inline uint64_t
124 _Py_atomic_add_uint64(uint64_t *obj, uint64_t value);
125
126 static inline uintptr_t
127 _Py_atomic_add_uintptr(uintptr_t *obj, uintptr_t value);
128
129 static inline Py_ssize_t
130 _Py_atomic_add_ssize(Py_ssize_t *obj, Py_ssize_t value);
131
132
133 // --- _Py_atomic_compare_exchange -------------------------------------------
134 // Performs an atomic compare-and-exchange.
135 //
136 // - If `*obj` and `*expected` are equal, store `desired` into `*obj`
137 // and return 1 (success).
138 // - Otherwise, store the `*obj` current value into `*expected`
139 // and return 0 (failure).
140 //
141 // These correspond to the C11 atomic_compare_exchange_strong() function.
142
143 static inline int
144 _Py_atomic_compare_exchange_int(int *obj, int *expected, int desired);
145
146 static inline int
147 _Py_atomic_compare_exchange_int8(int8_t *obj, int8_t *expected, int8_t desired);
148
149 static inline int
150 _Py_atomic_compare_exchange_int16(int16_t *obj, int16_t *expected, int16_t desired);
151
152 static inline int
153 _Py_atomic_compare_exchange_int32(int32_t *obj, int32_t *expected, int32_t desired);
154
155 static inline int
156 _Py_atomic_compare_exchange_int64(int64_t *obj, int64_t *expected, int64_t desired);
157
158 static inline int
159 _Py_atomic_compare_exchange_intptr(intptr_t *obj, intptr_t *expected, intptr_t desired);
160
161 static inline int
162 _Py_atomic_compare_exchange_uint(unsigned int *obj, unsigned int *expected, unsigned int desired);
163
164 static inline int
165 _Py_atomic_compare_exchange_uint8(uint8_t *obj, uint8_t *expected, uint8_t desired);
166
167 static inline int
168 _Py_atomic_compare_exchange_uint16(uint16_t *obj, uint16_t *expected, uint16_t desired);
169
170 static inline int
171 _Py_atomic_compare_exchange_uint32(uint32_t *obj, uint32_t *expected, uint32_t desired);
172
173 static inline int
174 _Py_atomic_compare_exchange_uint64(uint64_t *obj, uint64_t *expected, uint64_t desired);
175
176 static inline int
177 _Py_atomic_compare_exchange_uintptr(uintptr_t *obj, uintptr_t *expected, uintptr_t desired);
178
179 static inline int
180 _Py_atomic_compare_exchange_ssize(Py_ssize_t *obj, Py_ssize_t *expected, Py_ssize_t desired);
181
182 // NOTE: `obj` and `expected` are logically `void**` types, but we use `void*`
183 // so that we can pass types like `PyObject**` without a cast.
184 static inline int
185 _Py_atomic_compare_exchange_ptr(void *obj, void *expected, void *value);
186
187
188 // --- _Py_atomic_exchange ---------------------------------------------------
189 // Atomically replaces `*obj` with `value` and returns the previous value of `*obj`.
190
191 static inline int
192 _Py_atomic_exchange_int(int *obj, int value);
193
194 static inline int8_t
195 _Py_atomic_exchange_int8(int8_t *obj, int8_t value);
196
197 static inline int16_t
198 _Py_atomic_exchange_int16(int16_t *obj, int16_t value);
199
200 static inline int32_t
201 _Py_atomic_exchange_int32(int32_t *obj, int32_t value);
202
203 static inline int64_t
204 _Py_atomic_exchange_int64(int64_t *obj, int64_t value);
205
206 static inline intptr_t
207 _Py_atomic_exchange_intptr(intptr_t *obj, intptr_t value);
208
209 static inline unsigned int
210 _Py_atomic_exchange_uint(unsigned int *obj, unsigned int value);
211
212 static inline uint8_t
213 _Py_atomic_exchange_uint8(uint8_t *obj, uint8_t value);
214
215 static inline uint16_t
216 _Py_atomic_exchange_uint16(uint16_t *obj, uint16_t value);
217
218 static inline uint32_t
219 _Py_atomic_exchange_uint32(uint32_t *obj, uint32_t value);
220
221 static inline uint64_t
222 _Py_atomic_exchange_uint64(uint64_t *obj, uint64_t value);
223
224 static inline uintptr_t
225 _Py_atomic_exchange_uintptr(uintptr_t *obj, uintptr_t value);
226
227 static inline Py_ssize_t
228 _Py_atomic_exchange_ssize(Py_ssize_t *obj, Py_ssize_t value);
229
230 static inline void *
231 _Py_atomic_exchange_ptr(void *obj, void *value);
232
233
234 // --- _Py_atomic_and --------------------------------------------------------
235 // Performs `*obj &= value` atomically and returns the previous value of `*obj`.
236
237 static inline uint8_t
238 _Py_atomic_and_uint8(uint8_t *obj, uint8_t value);
239
240 static inline uint16_t
241 _Py_atomic_and_uint16(uint16_t *obj, uint16_t value);
242
243 static inline uint32_t
244 _Py_atomic_and_uint32(uint32_t *obj, uint32_t value);
245
246 static inline uint64_t
247 _Py_atomic_and_uint64(uint64_t *obj, uint64_t value);
248
249 static inline uintptr_t
250 _Py_atomic_and_uintptr(uintptr_t *obj, uintptr_t value);
251
252
253 // --- _Py_atomic_or ---------------------------------------------------------
254 // Performs `*obj |= value` atomically and returns the previous value of `*obj`.
255
256 static inline uint8_t
257 _Py_atomic_or_uint8(uint8_t *obj, uint8_t value);
258
259 static inline uint16_t
260 _Py_atomic_or_uint16(uint16_t *obj, uint16_t value);
261
262 static inline uint32_t
263 _Py_atomic_or_uint32(uint32_t *obj, uint32_t value);
264
265 static inline uint64_t
266 _Py_atomic_or_uint64(uint64_t *obj, uint64_t value);
267
268 static inline uintptr_t
269 _Py_atomic_or_uintptr(uintptr_t *obj, uintptr_t value);
270
271
272 // --- _Py_atomic_load -------------------------------------------------------
273 // Atomically loads `*obj` (sequential consistency)
274
275 static inline int
276 _Py_atomic_load_int(const int *obj);
277
278 static inline int8_t
279 _Py_atomic_load_int8(const int8_t *obj);
280
281 static inline int16_t
282 _Py_atomic_load_int16(const int16_t *obj);
283
284 static inline int32_t
285 _Py_atomic_load_int32(const int32_t *obj);
286
287 static inline int64_t
288 _Py_atomic_load_int64(const int64_t *obj);
289
290 static inline intptr_t
291 _Py_atomic_load_intptr(const intptr_t *obj);
292
293 static inline uint8_t
294 _Py_atomic_load_uint8(const uint8_t *obj);
295
296 static inline uint16_t
297 _Py_atomic_load_uint16(const uint16_t *obj);
298
299 static inline uint32_t
300 _Py_atomic_load_uint32(const uint32_t *obj);
301
302 static inline uint64_t
303 _Py_atomic_load_uint64(const uint64_t *obj);
304
305 static inline uintptr_t
306 _Py_atomic_load_uintptr(const uintptr_t *obj);
307
308 static inline unsigned int
309 _Py_atomic_load_uint(const unsigned int *obj);
310
311 static inline Py_ssize_t
312 _Py_atomic_load_ssize(const Py_ssize_t *obj);
313
314 static inline void *
315 _Py_atomic_load_ptr(const void *obj);
316
317
318 // --- _Py_atomic_load_relaxed -----------------------------------------------
319 // Loads `*obj` (relaxed consistency, i.e., no ordering)
320
321 static inline int
322 _Py_atomic_load_int_relaxed(const int *obj);
323
324 static inline char
325 _Py_atomic_load_char_relaxed(const char *obj);
326
327 static inline unsigned char
328 _Py_atomic_load_uchar_relaxed(const unsigned char *obj);
329
330 static inline short
331 _Py_atomic_load_short_relaxed(const short *obj);
332
333 static inline unsigned short
334 _Py_atomic_load_ushort_relaxed(const unsigned short *obj);
335
336 static inline long
337 _Py_atomic_load_long_relaxed(const long *obj);
338
339 static inline double
340 _Py_atomic_load_double_relaxed(const double *obj);
341
342 static inline long long
343 _Py_atomic_load_llong_relaxed(const long long *obj);
344
345 static inline int8_t
346 _Py_atomic_load_int8_relaxed(const int8_t *obj);
347
348 static inline int16_t
349 _Py_atomic_load_int16_relaxed(const int16_t *obj);
350
351 static inline int32_t
352 _Py_atomic_load_int32_relaxed(const int32_t *obj);
353
354 static inline int64_t
355 _Py_atomic_load_int64_relaxed(const int64_t *obj);
356
357 static inline intptr_t
358 _Py_atomic_load_intptr_relaxed(const intptr_t *obj);
359
360 static inline uint8_t
361 _Py_atomic_load_uint8_relaxed(const uint8_t *obj);
362
363 static inline uint16_t
364 _Py_atomic_load_uint16_relaxed(const uint16_t *obj);
365
366 static inline uint32_t
367 _Py_atomic_load_uint32_relaxed(const uint32_t *obj);
368
369 static inline uint64_t
370 _Py_atomic_load_uint64_relaxed(const uint64_t *obj);
371
372 static inline uintptr_t
373 _Py_atomic_load_uintptr_relaxed(const uintptr_t *obj);
374
375 static inline unsigned int
376 _Py_atomic_load_uint_relaxed(const unsigned int *obj);
377
378 static inline Py_ssize_t
379 _Py_atomic_load_ssize_relaxed(const Py_ssize_t *obj);
380
381 static inline void *
382 _Py_atomic_load_ptr_relaxed(const void *obj);
383
384 static inline unsigned long long
385 _Py_atomic_load_ullong_relaxed(const unsigned long long *obj);
386
387 // --- _Py_atomic_store ------------------------------------------------------
388 // Atomically performs `*obj = value` (sequential consistency)
389
390 static inline void
391 _Py_atomic_store_int(int *obj, int value);
392
393 static inline void
394 _Py_atomic_store_int8(int8_t *obj, int8_t value);
395
396 static inline void
397 _Py_atomic_store_int16(int16_t *obj, int16_t value);
398
399 static inline void
400 _Py_atomic_store_int32(int32_t *obj, int32_t value);
401
402 static inline void
403 _Py_atomic_store_int64(int64_t *obj, int64_t value);
404
405 static inline void
406 _Py_atomic_store_intptr(intptr_t *obj, intptr_t value);
407
408 static inline void
409 _Py_atomic_store_uint8(uint8_t *obj, uint8_t value);
410
411 static inline void
412 _Py_atomic_store_uint16(uint16_t *obj, uint16_t value);
413
414 static inline void
415 _Py_atomic_store_uint32(uint32_t *obj, uint32_t value);
416
417 static inline void
418 _Py_atomic_store_uint64(uint64_t *obj, uint64_t value);
419
420 static inline void
421 _Py_atomic_store_uintptr(uintptr_t *obj, uintptr_t value);
422
423 static inline void
424 _Py_atomic_store_uint(unsigned int *obj, unsigned int value);
425
426 static inline void
427 _Py_atomic_store_ptr(void *obj, void *value);
428
429 static inline void
430 _Py_atomic_store_ssize(Py_ssize_t* obj, Py_ssize_t value);
431
432
433 // --- _Py_atomic_store_relaxed ----------------------------------------------
434 // Stores `*obj = value` (relaxed consistency, i.e., no ordering)
435
436 static inline void
437 _Py_atomic_store_int_relaxed(int *obj, int value);
438
439 static inline void
440 _Py_atomic_store_int8_relaxed(int8_t *obj, int8_t value);
441
442 static inline void
443 _Py_atomic_store_int16_relaxed(int16_t *obj, int16_t value);
444
445 static inline void
446 _Py_atomic_store_int32_relaxed(int32_t *obj, int32_t value);
447
448 static inline void
449 _Py_atomic_store_int64_relaxed(int64_t *obj, int64_t value);
450
451 static inline void
452 _Py_atomic_store_intptr_relaxed(intptr_t *obj, intptr_t value);
453
454 static inline void
455 _Py_atomic_store_uint8_relaxed(uint8_t* obj, uint8_t value);
456
457 static inline void
458 _Py_atomic_store_uint16_relaxed(uint16_t *obj, uint16_t value);
459
460 static inline void
461 _Py_atomic_store_uint32_relaxed(uint32_t *obj, uint32_t value);
462
463 static inline void
464 _Py_atomic_store_uint64_relaxed(uint64_t *obj, uint64_t value);
465
466 static inline void
467 _Py_atomic_store_uintptr_relaxed(uintptr_t *obj, uintptr_t value);
468
469 static inline void
470 _Py_atomic_store_uint_relaxed(unsigned int *obj, unsigned int value);
471
472 static inline void
473 _Py_atomic_store_ptr_relaxed(void *obj, void *value);
474
475 static inline void
476 _Py_atomic_store_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value);
477
478 static inline void
479 _Py_atomic_store_ullong_relaxed(unsigned long long *obj,
480 unsigned long long value);
481
482 static inline void
483 _Py_atomic_store_char_relaxed(char *obj, char value);
484
485 static inline void
486 _Py_atomic_store_uchar_relaxed(unsigned char *obj, unsigned char value);
487
488 static inline void
489 _Py_atomic_store_short_relaxed(short *obj, short value);
490
491 static inline void
492 _Py_atomic_store_ushort_relaxed(unsigned short *obj, unsigned short value);
493
494 static inline void
495 _Py_atomic_store_long_relaxed(long *obj, long value);
496
497 static inline void
498 _Py_atomic_store_float_relaxed(float *obj, float value);
499
500 static inline void
501 _Py_atomic_store_double_relaxed(double *obj, double value);
502
503 static inline void
504 _Py_atomic_store_llong_relaxed(long long *obj, long long value);
505
506
507 // --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------
508
509 // Loads `*obj` (acquire operation)
510 static inline void *
511 _Py_atomic_load_ptr_acquire(const void *obj);
512
513 static inline uintptr_t
514 _Py_atomic_load_uintptr_acquire(const uintptr_t *obj);
515
516 // Stores `*obj = value` (release operation)
517 static inline void
518 _Py_atomic_store_ptr_release(void *obj, void *value);
519
520 static inline void
521 _Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value);
522
523 static inline void
524 _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value);
525
526 static inline void
527 _Py_atomic_store_int_release(int *obj, int value);
528
529 static inline int
530 _Py_atomic_load_int_acquire(const int *obj);
531
532 static inline void
533 _Py_atomic_store_uint_release(unsigned int *obj, unsigned int value);
534
535 static inline void
536 _Py_atomic_store_uint32_release(uint32_t *obj, uint32_t value);
537
538 static inline void
539 _Py_atomic_store_uint64_release(uint64_t *obj, uint64_t value);
540
541 static inline uint64_t
542 _Py_atomic_load_uint64_acquire(const uint64_t *obj);
543
544 static inline uint32_t
545 _Py_atomic_load_uint32_acquire(const uint32_t *obj);
546
547 static inline Py_ssize_t
548 _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj);
549
550
551
552
553 // --- _Py_atomic_fence ------------------------------------------------------
554
555 // Sequential consistency fence. C11 fences have complex semantics. When
556 // possible, use the atomic operations on variables defined above, which
557 // generally do not require explicit use of a fence.
558 // See https://en.cppreference.com/w/cpp/atomic/atomic_thread_fence
559 static inline void _Py_atomic_fence_seq_cst(void);
560
561 // Acquire fence
562 static inline void _Py_atomic_fence_acquire(void);
563
564 // Release fence
565 static inline void _Py_atomic_fence_release(void);
566
567
568 #ifndef _Py_USE_GCC_BUILTIN_ATOMICS
569 # if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8))
570 # define _Py_USE_GCC_BUILTIN_ATOMICS 1
571 # elif defined(__clang__)
572 # if __has_builtin(__atomic_load)
573 # define _Py_USE_GCC_BUILTIN_ATOMICS 1
574 # endif
575 # endif
576 #endif
577
578 #if _Py_USE_GCC_BUILTIN_ATOMICS
579 # define Py_ATOMIC_GCC_H
580 # include "pyatomic_gcc.h"
581 # undef Py_ATOMIC_GCC_H
582 #elif __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_ATOMICS__)
583 # define Py_ATOMIC_STD_H
584 # include "pyatomic_std.h"
585 # undef Py_ATOMIC_STD_H
586 #elif defined(_MSC_VER)
587 # define Py_ATOMIC_MSC_H
588 # include "pyatomic_msc.h"
589 # undef Py_ATOMIC_MSC_H
590 #else
591 # error "no available pyatomic implementation for this platform/compiler"
592 #endif
593
594
595 // --- aliases ---------------------------------------------------------------
596
597 // Compilers don't really support "consume" semantics, so we fake it. Use
598 // "acquire" with TSan to support false positives. Use "relaxed" otherwise,
599 // because CPUs on all platforms we support respect address dependencies without
600 // extra barriers.
601 // See 2.6.7 in https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2020/p2055r0.pdf
602 #if defined(_Py_THREAD_SANITIZER)
603 # define _Py_atomic_load_ptr_consume _Py_atomic_load_ptr_acquire
604 #else
605 # define _Py_atomic_load_ptr_consume _Py_atomic_load_ptr_relaxed
606 #endif
607
608 #if SIZEOF_LONG == 8
609 # define _Py_atomic_load_ulong(p) \
610 _Py_atomic_load_uint64((uint64_t *)p)
611 # define _Py_atomic_load_ulong_relaxed(p) \
612 _Py_atomic_load_uint64_relaxed((uint64_t *)p)
613 # define _Py_atomic_store_ulong(p, v) \
614 _Py_atomic_store_uint64((uint64_t *)p, v)
615 # define _Py_atomic_store_ulong_relaxed(p, v) \
616 _Py_atomic_store_uint64_relaxed((uint64_t *)p, v)
617 #elif SIZEOF_LONG == 4
618 # define _Py_atomic_load_ulong(p) \
619 _Py_atomic_load_uint32((uint32_t *)p)
620 # define _Py_atomic_load_ulong_relaxed(p) \
621 _Py_atomic_load_uint32_relaxed((uint32_t *)p)
622 # define _Py_atomic_store_ulong(p, v) \
623 _Py_atomic_store_uint32((uint32_t *)p, v)
624 # define _Py_atomic_store_ulong_relaxed(p, v) \
625 _Py_atomic_store_uint32_relaxed((uint32_t *)p, v)
626 #else
627 # error "long must be 4 or 8 bytes in size"
628 #endif // SIZEOF_LONG