Debian Codegraph @

Include/cpython/pyatomic_msc.h

raw
1 // This is the implementation of Python atomic operations for MSVC if the
2 // compiler does not support C11 or C++11 atomics.
3 //
4 // MSVC intrinsics are defined on char, short, long, __int64, and pointer
5 // types. Note that long and int are both 32-bits even on 64-bit Windows,
6 // so operations on int are cast to long.
7 //
8 // The volatile keyword has additional memory ordering semantics on MSVC. On
9 // x86 and x86-64, volatile accesses have acquire-release semantics. On ARM64,
10 // volatile accesses behave like C11's memory_order_relaxed.
11
12 #ifndef Py_ATOMIC_MSC_H
13 # error "this header file must not be included directly"
14 #endif
15
16 #include <intrin.h>
17
18 #define _Py_atomic_ASSERT_ARG_TYPE(TYPE) \
19 Py_BUILD_ASSERT(sizeof(*obj) == sizeof(TYPE))
20
21
22 // --- _Py_atomic_add --------------------------------------------------------
23
24 static inline int8_t
25 _Py_atomic_add_int8(int8_t *obj, int8_t value)
26 {
27 _Py_atomic_ASSERT_ARG_TYPE(char);
28 return (int8_t)_InterlockedExchangeAdd8((volatile char *)obj, (char)value);
29 }
30
31 static inline int16_t
32 _Py_atomic_add_int16(int16_t *obj, int16_t value)
33 {
34 _Py_atomic_ASSERT_ARG_TYPE(short);
35 return (int16_t)_InterlockedExchangeAdd16((volatile short *)obj, (short)value);
36 }
37
38 static inline int32_t
39 _Py_atomic_add_int32(int32_t *obj, int32_t value)
40 {
41 _Py_atomic_ASSERT_ARG_TYPE(long);
42 return (int32_t)_InterlockedExchangeAdd((volatile long *)obj, (long)value);
43 }
44
45 static inline int64_t
46 _Py_atomic_add_int64(int64_t *obj, int64_t value)
47 {
48 #if defined(_M_X64) || defined(_M_ARM64)
49 _Py_atomic_ASSERT_ARG_TYPE(__int64);
50 return (int64_t)_InterlockedExchangeAdd64((volatile __int64 *)obj, (__int64)value);
51 #else
52 int64_t old_value = _Py_atomic_load_int64_relaxed(obj);
53 for (;;) {
54 int64_t new_value = old_value + value;
55 if (_Py_atomic_compare_exchange_int64(obj, &old_value, new_value)) {
56 return old_value;
57 }
58 }
59 #endif
60 }
61
62
63 static inline uint8_t
64 _Py_atomic_add_uint8(uint8_t *obj, uint8_t value)
65 {
66 return (uint8_t)_Py_atomic_add_int8((int8_t *)obj, (int8_t)value);
67 }
68
69 static inline uint16_t
70 _Py_atomic_add_uint16(uint16_t *obj, uint16_t value)
71 {
72 return (uint16_t)_Py_atomic_add_int16((int16_t *)obj, (int16_t)value);
73 }
74
75 static inline uint32_t
76 _Py_atomic_add_uint32(uint32_t *obj, uint32_t value)
77 {
78 return (uint32_t)_Py_atomic_add_int32((int32_t *)obj, (int32_t)value);
79 }
80
81 static inline int
82 _Py_atomic_add_int(int *obj, int value)
83 {
84 _Py_atomic_ASSERT_ARG_TYPE(int32_t);
85 return (int)_Py_atomic_add_int32((int32_t *)obj, (int32_t)value);
86 }
87
88 static inline unsigned int
89 _Py_atomic_add_uint(unsigned int *obj, unsigned int value)
90 {
91 _Py_atomic_ASSERT_ARG_TYPE(int32_t);
92 return (unsigned int)_Py_atomic_add_int32((int32_t *)obj, (int32_t)value);
93 }
94
95 static inline uint64_t
96 _Py_atomic_add_uint64(uint64_t *obj, uint64_t value)
97 {
98 return (uint64_t)_Py_atomic_add_int64((int64_t *)obj, (int64_t)value);
99 }
100
101 static inline intptr_t
102 _Py_atomic_add_intptr(intptr_t *obj, intptr_t value)
103 {
104 #if SIZEOF_VOID_P == 8
105 _Py_atomic_ASSERT_ARG_TYPE(int64_t);
106 return (intptr_t)_Py_atomic_add_int64((int64_t *)obj, (int64_t)value);
107 #else
108 _Py_atomic_ASSERT_ARG_TYPE(int32_t);
109 return (intptr_t)_Py_atomic_add_int32((int32_t *)obj, (int32_t)value);
110 #endif
111 }
112
113 static inline uintptr_t
114 _Py_atomic_add_uintptr(uintptr_t *obj, uintptr_t value)
115 {
116 _Py_atomic_ASSERT_ARG_TYPE(intptr_t);
117 return (uintptr_t)_Py_atomic_add_intptr((intptr_t *)obj, (intptr_t)value);
118 }
119
120 static inline Py_ssize_t
121 _Py_atomic_add_ssize(Py_ssize_t *obj, Py_ssize_t value)
122 {
123 _Py_atomic_ASSERT_ARG_TYPE(intptr_t);
124 return (Py_ssize_t)_Py_atomic_add_intptr((intptr_t *)obj, (intptr_t)value);
125 }
126
127
128 // --- _Py_atomic_compare_exchange -------------------------------------------
129
130 static inline int
131 _Py_atomic_compare_exchange_int8(int8_t *obj, int8_t *expected, int8_t value)
132 {
133 _Py_atomic_ASSERT_ARG_TYPE(char);
134 int8_t initial = (int8_t)_InterlockedCompareExchange8(
135 (volatile char *)obj,
136 (char)value,
137 (char)*expected);
138 if (initial == *expected) {
139 return 1;
140 }
141 *expected = initial;
142 return 0;
143 }
144
145 static inline int
146 _Py_atomic_compare_exchange_int16(int16_t *obj, int16_t *expected, int16_t value)
147 {
148 _Py_atomic_ASSERT_ARG_TYPE(short);
149 int16_t initial = (int16_t)_InterlockedCompareExchange16(
150 (volatile short *)obj,
151 (short)value,
152 (short)*expected);
153 if (initial == *expected) {
154 return 1;
155 }
156 *expected = initial;
157 return 0;
158 }
159
160 static inline int
161 _Py_atomic_compare_exchange_int32(int32_t *obj, int32_t *expected, int32_t value)
162 {
163 _Py_atomic_ASSERT_ARG_TYPE(long);
164 int32_t initial = (int32_t)_InterlockedCompareExchange(
165 (volatile long *)obj,
166 (long)value,
167 (long)*expected);
168 if (initial == *expected) {
169 return 1;
170 }
171 *expected = initial;
172 return 0;
173 }
174
175 static inline int
176 _Py_atomic_compare_exchange_int64(int64_t *obj, int64_t *expected, int64_t value)
177 {
178 _Py_atomic_ASSERT_ARG_TYPE(__int64);
179 int64_t initial = (int64_t)_InterlockedCompareExchange64(
180 (volatile __int64 *)obj,
181 (__int64)value,
182 (__int64)*expected);
183 if (initial == *expected) {
184 return 1;
185 }
186 *expected = initial;
187 return 0;
188 }
189
190 static inline int
191 _Py_atomic_compare_exchange_ptr(void *obj, void *expected, void *value)
192 {
193 void *initial = _InterlockedCompareExchangePointer(
194 (void**)obj,
195 value,
196 *(void**)expected);
197 if (initial == *(void**)expected) {
198 return 1;
199 }
200 *(void**)expected = initial;
201 return 0;
202 }
203
204
205 static inline int
206 _Py_atomic_compare_exchange_uint8(uint8_t *obj, uint8_t *expected, uint8_t value)
207 {
208 return _Py_atomic_compare_exchange_int8((int8_t *)obj,
209 (int8_t *)expected,
210 (int8_t)value);
211 }
212
213 static inline int
214 _Py_atomic_compare_exchange_uint16(uint16_t *obj, uint16_t *expected, uint16_t value)
215 {
216 return _Py_atomic_compare_exchange_int16((int16_t *)obj,
217 (int16_t *)expected,
218 (int16_t)value);
219 }
220
221 static inline int
222 _Py_atomic_compare_exchange_uint32(uint32_t *obj, uint32_t *expected, uint32_t value)
223 {
224 return _Py_atomic_compare_exchange_int32((int32_t *)obj,
225 (int32_t *)expected,
226 (int32_t)value);
227 }
228
229 static inline int
230 _Py_atomic_compare_exchange_int(int *obj, int *expected, int value)
231 {
232 _Py_atomic_ASSERT_ARG_TYPE(int32_t);
233 return _Py_atomic_compare_exchange_int32((int32_t *)obj,
234 (int32_t *)expected,
235 (int32_t)value);
236 }
237
238 static inline int
239 _Py_atomic_compare_exchange_uint(unsigned int *obj, unsigned int *expected, unsigned int value)
240 {
241 _Py_atomic_ASSERT_ARG_TYPE(int32_t);
242 return _Py_atomic_compare_exchange_int32((int32_t *)obj,
243 (int32_t *)expected,
244 (int32_t)value);
245 }
246
247 static inline int
248 _Py_atomic_compare_exchange_uint64(uint64_t *obj, uint64_t *expected, uint64_t value)
249 {
250 return _Py_atomic_compare_exchange_int64((int64_t *)obj,
251 (int64_t *)expected,
252 (int64_t)value);
253 }
254
255 static inline int
256 _Py_atomic_compare_exchange_intptr(intptr_t *obj, intptr_t *expected, intptr_t value)
257 {
258 _Py_atomic_ASSERT_ARG_TYPE(void*);
259 return _Py_atomic_compare_exchange_ptr((void**)obj,
260 (void**)expected,
261 (void*)value);
262 }
263
264 static inline int
265 _Py_atomic_compare_exchange_uintptr(uintptr_t *obj, uintptr_t *expected, uintptr_t value)
266 {
267 _Py_atomic_ASSERT_ARG_TYPE(void*);
268 return _Py_atomic_compare_exchange_ptr((void**)obj,
269 (void**)expected,
270 (void*)value);
271 }
272
273 static inline int
274 _Py_atomic_compare_exchange_ssize(Py_ssize_t *obj, Py_ssize_t *expected, Py_ssize_t value)
275 {
276 _Py_atomic_ASSERT_ARG_TYPE(void*);
277 return _Py_atomic_compare_exchange_ptr((void**)obj,
278 (void**)expected,
279 (void*)value);
280 }
281
282
283 // --- _Py_atomic_exchange ---------------------------------------------------
284
285 static inline int8_t
286 _Py_atomic_exchange_int8(int8_t *obj, int8_t value)
287 {
288 _Py_atomic_ASSERT_ARG_TYPE(char);
289 return (int8_t)_InterlockedExchange8((volatile char *)obj, (char)value);
290 }
291
292 static inline int16_t
293 _Py_atomic_exchange_int16(int16_t *obj, int16_t value)
294 {
295 _Py_atomic_ASSERT_ARG_TYPE(short);
296 return (int16_t)_InterlockedExchange16((volatile short *)obj, (short)value);
297 }
298
299 static inline int32_t
300 _Py_atomic_exchange_int32(int32_t *obj, int32_t value)
301 {
302 _Py_atomic_ASSERT_ARG_TYPE(long);
303 return (int32_t)_InterlockedExchange((volatile long *)obj, (long)value);
304 }
305
306 static inline int64_t
307 _Py_atomic_exchange_int64(int64_t *obj, int64_t value)
308 {
309 #if defined(_M_X64) || defined(_M_ARM64)
310 _Py_atomic_ASSERT_ARG_TYPE(__int64);
311 return (int64_t)_InterlockedExchange64((volatile __int64 *)obj, (__int64)value);
312 #else
313 int64_t old_value = _Py_atomic_load_int64_relaxed(obj);
314 for (;;) {
315 if (_Py_atomic_compare_exchange_int64(obj, &old_value, value)) {
316 return old_value;
317 }
318 }
319 #endif
320 }
321
322 static inline void*
323 _Py_atomic_exchange_ptr(void *obj, void *value)
324 {
325 return (void*)_InterlockedExchangePointer((void * volatile *)obj, (void *)value);
326 }
327
328
329 static inline uint8_t
330 _Py_atomic_exchange_uint8(uint8_t *obj, uint8_t value)
331 {
332 return (uint8_t)_Py_atomic_exchange_int8((int8_t *)obj,
333 (int8_t)value);
334 }
335
336 static inline uint16_t
337 _Py_atomic_exchange_uint16(uint16_t *obj, uint16_t value)
338 {
339 return (uint16_t)_Py_atomic_exchange_int16((int16_t *)obj,
340 (int16_t)value);
341 }
342
343 static inline uint32_t
344 _Py_atomic_exchange_uint32(uint32_t *obj, uint32_t value)
345 {
346 return (uint32_t)_Py_atomic_exchange_int32((int32_t *)obj,
347 (int32_t)value);
348 }
349
350 static inline int
351 _Py_atomic_exchange_int(int *obj, int value)
352 {
353 _Py_atomic_ASSERT_ARG_TYPE(int32_t);
354 return (int)_Py_atomic_exchange_int32((int32_t *)obj,
355 (int32_t)value);
356 }
357
358 static inline unsigned int
359 _Py_atomic_exchange_uint(unsigned int *obj, unsigned int value)
360 {
361 _Py_atomic_ASSERT_ARG_TYPE(int32_t);
362 return (unsigned int)_Py_atomic_exchange_int32((int32_t *)obj,
363 (int32_t)value);
364 }
365
366 static inline uint64_t
367 _Py_atomic_exchange_uint64(uint64_t *obj, uint64_t value)
368 {
369 return (uint64_t)_Py_atomic_exchange_int64((int64_t *)obj,
370 (int64_t)value);
371 }
372
373 static inline intptr_t
374 _Py_atomic_exchange_intptr(intptr_t *obj, intptr_t value)
375 {
376 _Py_atomic_ASSERT_ARG_TYPE(void*);
377 return (intptr_t)_Py_atomic_exchange_ptr((void**)obj,
378 (void*)value);
379 }
380
381 static inline uintptr_t
382 _Py_atomic_exchange_uintptr(uintptr_t *obj, uintptr_t value)
383 {
384 _Py_atomic_ASSERT_ARG_TYPE(void*);
385 return (uintptr_t)_Py_atomic_exchange_ptr((void**)obj,
386 (void*)value);
387 }
388
389 static inline Py_ssize_t
390 _Py_atomic_exchange_ssize(Py_ssize_t *obj, Py_ssize_t value)
391 {
392 _Py_atomic_ASSERT_ARG_TYPE(void*);
393 return (Py_ssize_t)_Py_atomic_exchange_ptr((void**)obj,
394 (void*)value);
395 }
396
397
398 // --- _Py_atomic_and --------------------------------------------------------
399
400 static inline uint8_t
401 _Py_atomic_and_uint8(uint8_t *obj, uint8_t value)
402 {
403 _Py_atomic_ASSERT_ARG_TYPE(char);
404 return (uint8_t)_InterlockedAnd8((volatile char *)obj, (char)value);
405 }
406
407 static inline uint16_t
408 _Py_atomic_and_uint16(uint16_t *obj, uint16_t value)
409 {
410 _Py_atomic_ASSERT_ARG_TYPE(short);
411 return (uint16_t)_InterlockedAnd16((volatile short *)obj, (short)value);
412 }
413
414 static inline uint32_t
415 _Py_atomic_and_uint32(uint32_t *obj, uint32_t value)
416 {
417 _Py_atomic_ASSERT_ARG_TYPE(long);
418 return (uint32_t)_InterlockedAnd((volatile long *)obj, (long)value);
419 }
420
421 static inline uint64_t
422 _Py_atomic_and_uint64(uint64_t *obj, uint64_t value)
423 {
424 #if defined(_M_X64) || defined(_M_ARM64)
425 _Py_atomic_ASSERT_ARG_TYPE(__int64);
426 return (uint64_t)_InterlockedAnd64((volatile __int64 *)obj, (__int64)value);
427 #else
428 uint64_t old_value = _Py_atomic_load_uint64_relaxed(obj);
429 for (;;) {
430 uint64_t new_value = old_value & value;
431 if (_Py_atomic_compare_exchange_uint64(obj, &old_value, new_value)) {
432 return old_value;
433 }
434 }
435 #endif
436 }
437
438 static inline uintptr_t
439 _Py_atomic_and_uintptr(uintptr_t *obj, uintptr_t value)
440 {
441 #if SIZEOF_VOID_P == 8
442 _Py_atomic_ASSERT_ARG_TYPE(uint64_t);
443 return (uintptr_t)_Py_atomic_and_uint64((uint64_t *)obj,
444 (uint64_t)value);
445 #else
446 _Py_atomic_ASSERT_ARG_TYPE(uint32_t);
447 return (uintptr_t)_Py_atomic_and_uint32((uint32_t *)obj,
448 (uint32_t)value);
449 #endif
450 }
451
452
453 // --- _Py_atomic_or ---------------------------------------------------------
454
455 static inline uint8_t
456 _Py_atomic_or_uint8(uint8_t *obj, uint8_t value)
457 {
458 _Py_atomic_ASSERT_ARG_TYPE(char);
459 return (uint8_t)_InterlockedOr8((volatile char *)obj, (char)value);
460 }
461
462 static inline uint16_t
463 _Py_atomic_or_uint16(uint16_t *obj, uint16_t value)
464 {
465 _Py_atomic_ASSERT_ARG_TYPE(short);
466 return (uint16_t)_InterlockedOr16((volatile short *)obj, (short)value);
467 }
468
469 static inline uint32_t
470 _Py_atomic_or_uint32(uint32_t *obj, uint32_t value)
471 {
472 _Py_atomic_ASSERT_ARG_TYPE(long);
473 return (uint32_t)_InterlockedOr((volatile long *)obj, (long)value);
474 }
475
476 static inline uint64_t
477 _Py_atomic_or_uint64(uint64_t *obj, uint64_t value)
478 {
479 #if defined(_M_X64) || defined(_M_ARM64)
480 _Py_atomic_ASSERT_ARG_TYPE(__int64);
481 return (uint64_t)_InterlockedOr64((volatile __int64 *)obj, (__int64)value);
482 #else
483 uint64_t old_value = _Py_atomic_load_uint64_relaxed(obj);
484 for (;;) {
485 uint64_t new_value = old_value | value;
486 if (_Py_atomic_compare_exchange_uint64(obj, &old_value, new_value)) {
487 return old_value;
488 }
489 }
490 #endif
491 }
492
493
494 static inline uintptr_t
495 _Py_atomic_or_uintptr(uintptr_t *obj, uintptr_t value)
496 {
497 #if SIZEOF_VOID_P == 8
498 _Py_atomic_ASSERT_ARG_TYPE(uint64_t);
499 return (uintptr_t)_Py_atomic_or_uint64((uint64_t *)obj,
500 (uint64_t)value);
501 #else
502 _Py_atomic_ASSERT_ARG_TYPE(uint32_t);
503 return (uintptr_t)_Py_atomic_or_uint32((uint32_t *)obj,
504 (uint32_t)value);
505 #endif
506 }
507
508
509 // --- _Py_atomic_load -------------------------------------------------------
510
511 static inline uint8_t
512 _Py_atomic_load_uint8(const uint8_t *obj)
513 {
514 #if defined(_M_X64) || defined(_M_IX86)
515 return *(volatile uint8_t *)obj;
516 #elif defined(_M_ARM64)
517 return (uint8_t)__ldar8((unsigned __int8 volatile *)obj);
518 #else
519 # error "no implementation of _Py_atomic_load_uint8"
520 #endif
521 }
522
523 static inline uint16_t
524 _Py_atomic_load_uint16(const uint16_t *obj)
525 {
526 #if defined(_M_X64) || defined(_M_IX86)
527 return *(volatile uint16_t *)obj;
528 #elif defined(_M_ARM64)
529 return (uint16_t)__ldar16((unsigned __int16 volatile *)obj);
530 #else
531 # error "no implementation of _Py_atomic_load_uint16"
532 #endif
533 }
534
535 static inline uint32_t
536 _Py_atomic_load_uint32(const uint32_t *obj)
537 {
538 #if defined(_M_X64) || defined(_M_IX86)
539 return *(volatile uint32_t *)obj;
540 #elif defined(_M_ARM64)
541 return (uint32_t)__ldar32((unsigned __int32 volatile *)obj);
542 #else
543 # error "no implementation of _Py_atomic_load_uint32"
544 #endif
545 }
546
547 static inline uint64_t
548 _Py_atomic_load_uint64(const uint64_t *obj)
549 {
550 #if defined(_M_X64) || defined(_M_IX86)
551 return *(volatile uint64_t *)obj;
552 #elif defined(_M_ARM64)
553 return (uint64_t)__ldar64((unsigned __int64 volatile *)obj);
554 #else
555 # error "no implementation of _Py_atomic_load_uint64"
556 #endif
557 }
558
559 static inline int8_t
560 _Py_atomic_load_int8(const int8_t *obj)
561 {
562 return (int8_t)_Py_atomic_load_uint8((const uint8_t *)obj);
563 }
564
565 static inline int16_t
566 _Py_atomic_load_int16(const int16_t *obj)
567 {
568 return (int16_t)_Py_atomic_load_uint16((const uint16_t *)obj);
569 }
570
571 static inline int32_t
572 _Py_atomic_load_int32(const int32_t *obj)
573 {
574 return (int32_t)_Py_atomic_load_uint32((const uint32_t *)obj);
575 }
576
577 static inline int
578 _Py_atomic_load_int(const int *obj)
579 {
580 _Py_atomic_ASSERT_ARG_TYPE(uint32_t);
581 return (int)_Py_atomic_load_uint32((uint32_t *)obj);
582 }
583
584 static inline unsigned int
585 _Py_atomic_load_uint(const unsigned int *obj)
586 {
587 _Py_atomic_ASSERT_ARG_TYPE(uint32_t);
588 return (unsigned int)_Py_atomic_load_uint32((uint32_t *)obj);
589 }
590
591 static inline int64_t
592 _Py_atomic_load_int64(const int64_t *obj)
593 {
594 return (int64_t)_Py_atomic_load_uint64((const uint64_t *)obj);
595 }
596
597 static inline void*
598 _Py_atomic_load_ptr(const void *obj)
599 {
600 #if SIZEOF_VOID_P == 8
601 return (void*)_Py_atomic_load_uint64((const uint64_t *)obj);
602 #else
603 return (void*)_Py_atomic_load_uint32((const uint32_t *)obj);
604 #endif
605 }
606
607 static inline intptr_t
608 _Py_atomic_load_intptr(const intptr_t *obj)
609 {
610 _Py_atomic_ASSERT_ARG_TYPE(void*);
611 return (intptr_t)_Py_atomic_load_ptr((void*)obj);
612 }
613
614 static inline uintptr_t
615 _Py_atomic_load_uintptr(const uintptr_t *obj)
616 {
617 _Py_atomic_ASSERT_ARG_TYPE(void*);
618 return (uintptr_t)_Py_atomic_load_ptr((void*)obj);
619 }
620
621 static inline Py_ssize_t
622 _Py_atomic_load_ssize(const Py_ssize_t *obj)
623 {
624 _Py_atomic_ASSERT_ARG_TYPE(void*);
625 return (Py_ssize_t)_Py_atomic_load_ptr((void*)obj);
626 }
627
628
629 // --- _Py_atomic_load_relaxed -----------------------------------------------
630
631 static inline int
632 _Py_atomic_load_int_relaxed(const int *obj)
633 {
634 return *(volatile int *)obj;
635 }
636
637 static inline char
638 _Py_atomic_load_char_relaxed(const char *obj)
639 {
640 return *(volatile char *)obj;
641 }
642
643 static inline unsigned char
644 _Py_atomic_load_uchar_relaxed(const unsigned char *obj)
645 {
646 return *(volatile unsigned char *)obj;
647 }
648
649 static inline short
650 _Py_atomic_load_short_relaxed(const short *obj)
651 {
652 return *(volatile short *)obj;
653 }
654
655 static inline unsigned short
656 _Py_atomic_load_ushort_relaxed(const unsigned short *obj)
657 {
658 return *(volatile unsigned short *)obj;
659 }
660
661 static inline long
662 _Py_atomic_load_long_relaxed(const long *obj)
663 {
664 return *(volatile long *)obj;
665 }
666
667 static inline float
668 _Py_atomic_load_float_relaxed(const float *obj)
669 {
670 return *(volatile float *)obj;
671 }
672
673 static inline double
674 _Py_atomic_load_double_relaxed(const double *obj)
675 {
676 return *(volatile double *)obj;
677 }
678
679 static inline int8_t
680 _Py_atomic_load_int8_relaxed(const int8_t *obj)
681 {
682 return *(volatile int8_t *)obj;
683 }
684
685 static inline int16_t
686 _Py_atomic_load_int16_relaxed(const int16_t *obj)
687 {
688 return *(volatile int16_t *)obj;
689 }
690
691 static inline int32_t
692 _Py_atomic_load_int32_relaxed(const int32_t *obj)
693 {
694 return *(volatile int32_t *)obj;
695 }
696
697 static inline int64_t
698 _Py_atomic_load_int64_relaxed(const int64_t *obj)
699 {
700 return *(volatile int64_t *)obj;
701 }
702
703 static inline intptr_t
704 _Py_atomic_load_intptr_relaxed(const intptr_t *obj)
705 {
706 return *(volatile intptr_t *)obj;
707 }
708
709 static inline uint8_t
710 _Py_atomic_load_uint8_relaxed(const uint8_t *obj)
711 {
712 return *(volatile uint8_t *)obj;
713 }
714
715 static inline uint16_t
716 _Py_atomic_load_uint16_relaxed(const uint16_t *obj)
717 {
718 return *(volatile uint16_t *)obj;
719 }
720
721 static inline uint32_t
722 _Py_atomic_load_uint32_relaxed(const uint32_t *obj)
723 {
724 return *(volatile uint32_t *)obj;
725 }
726
727 static inline uint64_t
728 _Py_atomic_load_uint64_relaxed(const uint64_t *obj)
729 {
730 return *(volatile uint64_t *)obj;
731 }
732
733 static inline uintptr_t
734 _Py_atomic_load_uintptr_relaxed(const uintptr_t *obj)
735 {
736 return *(volatile uintptr_t *)obj;
737 }
738
739 static inline unsigned int
740 _Py_atomic_load_uint_relaxed(const unsigned int *obj)
741 {
742 return *(volatile unsigned int *)obj;
743 }
744
745 static inline Py_ssize_t
746 _Py_atomic_load_ssize_relaxed(const Py_ssize_t *obj)
747 {
748 return *(volatile Py_ssize_t *)obj;
749 }
750
751 static inline void*
752 _Py_atomic_load_ptr_relaxed(const void *obj)
753 {
754 return *(void * volatile *)obj;
755 }
756
757 static inline unsigned long long
758 _Py_atomic_load_ullong_relaxed(const unsigned long long *obj)
759 {
760 return *(volatile unsigned long long *)obj;
761 }
762
763 static inline long long
764 _Py_atomic_load_llong_relaxed(const long long *obj)
765 {
766 return *(volatile long long *)obj;
767 }
768
769
770 // --- _Py_atomic_store ------------------------------------------------------
771
772 static inline void
773 _Py_atomic_store_int(int *obj, int value)
774 {
775 (void)_Py_atomic_exchange_int(obj, value);
776 }
777
778 static inline void
779 _Py_atomic_store_int8(int8_t *obj, int8_t value)
780 {
781 (void)_Py_atomic_exchange_int8(obj, value);
782 }
783
784 static inline void
785 _Py_atomic_store_int16(int16_t *obj, int16_t value)
786 {
787 (void)_Py_atomic_exchange_int16(obj, value);
788 }
789
790 static inline void
791 _Py_atomic_store_int32(int32_t *obj, int32_t value)
792 {
793 (void)_Py_atomic_exchange_int32(obj, value);
794 }
795
796 static inline void
797 _Py_atomic_store_int64(int64_t *obj, int64_t value)
798 {
799 (void)_Py_atomic_exchange_int64(obj, value);
800 }
801
802 static inline void
803 _Py_atomic_store_intptr(intptr_t *obj, intptr_t value)
804 {
805 (void)_Py_atomic_exchange_intptr(obj, value);
806 }
807
808 static inline void
809 _Py_atomic_store_uint8(uint8_t *obj, uint8_t value)
810 {
811 (void)_Py_atomic_exchange_uint8(obj, value);
812 }
813
814 static inline void
815 _Py_atomic_store_uint16(uint16_t *obj, uint16_t value)
816 {
817 (void)_Py_atomic_exchange_uint16(obj, value);
818 }
819
820 static inline void
821 _Py_atomic_store_uint32(uint32_t *obj, uint32_t value)
822 {
823 (void)_Py_atomic_exchange_uint32(obj, value);
824 }
825
826 static inline void
827 _Py_atomic_store_uint64(uint64_t *obj, uint64_t value)
828 {
829 (void)_Py_atomic_exchange_uint64(obj, value);
830 }
831
832 static inline void
833 _Py_atomic_store_uintptr(uintptr_t *obj, uintptr_t value)
834 {
835 (void)_Py_atomic_exchange_uintptr(obj, value);
836 }
837
838 static inline void
839 _Py_atomic_store_uint(unsigned int *obj, unsigned int value)
840 {
841 (void)_Py_atomic_exchange_uint(obj, value);
842 }
843
844 static inline void
845 _Py_atomic_store_ptr(void *obj, void *value)
846 {
847 (void)_Py_atomic_exchange_ptr(obj, value);
848 }
849
850 static inline void
851 _Py_atomic_store_ssize(Py_ssize_t *obj, Py_ssize_t value)
852 {
853 (void)_Py_atomic_exchange_ssize(obj, value);
854 }
855
856
857 // --- _Py_atomic_store_relaxed ----------------------------------------------
858
859 static inline void
860 _Py_atomic_store_int_relaxed(int *obj, int value)
861 {
862 *(volatile int *)obj = value;
863 }
864
865 static inline void
866 _Py_atomic_store_int8_relaxed(int8_t *obj, int8_t value)
867 {
868 *(volatile int8_t *)obj = value;
869 }
870
871 static inline void
872 _Py_atomic_store_int16_relaxed(int16_t *obj, int16_t value)
873 {
874 *(volatile int16_t *)obj = value;
875 }
876
877 static inline void
878 _Py_atomic_store_int32_relaxed(int32_t *obj, int32_t value)
879 {
880 *(volatile int32_t *)obj = value;
881 }
882
883 static inline void
884 _Py_atomic_store_int64_relaxed(int64_t *obj, int64_t value)
885 {
886 *(volatile int64_t *)obj = value;
887 }
888
889 static inline void
890 _Py_atomic_store_intptr_relaxed(intptr_t *obj, intptr_t value)
891 {
892 *(volatile intptr_t *)obj = value;
893 }
894
895 static inline void
896 _Py_atomic_store_uint8_relaxed(uint8_t *obj, uint8_t value)
897 {
898 *(volatile uint8_t *)obj = value;
899 }
900
901 static inline void
902 _Py_atomic_store_uint16_relaxed(uint16_t *obj, uint16_t value)
903 {
904 *(volatile uint16_t *)obj = value;
905 }
906
907 static inline void
908 _Py_atomic_store_uint32_relaxed(uint32_t *obj, uint32_t value)
909 {
910 *(volatile uint32_t *)obj = value;
911 }
912
913 static inline void
914 _Py_atomic_store_uint64_relaxed(uint64_t *obj, uint64_t value)
915 {
916 *(volatile uint64_t *)obj = value;
917 }
918
919 static inline void
920 _Py_atomic_store_uintptr_relaxed(uintptr_t *obj, uintptr_t value)
921 {
922 *(volatile uintptr_t *)obj = value;
923 }
924
925 static inline void
926 _Py_atomic_store_uint_relaxed(unsigned int *obj, unsigned int value)
927 {
928 *(volatile unsigned int *)obj = value;
929 }
930
931 static inline void
932 _Py_atomic_store_ptr_relaxed(void *obj, void* value)
933 {
934 *(void * volatile *)obj = value;
935 }
936
937 static inline void
938 _Py_atomic_store_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value)
939 {
940 *(volatile Py_ssize_t *)obj = value;
941 }
942
943 static inline void
944 _Py_atomic_store_ullong_relaxed(unsigned long long *obj,
945 unsigned long long value)
946 {
947 *(volatile unsigned long long *)obj = value;
948 }
949
950 static inline void
951 _Py_atomic_store_char_relaxed(char *obj, char value)
952 {
953 *(volatile char *)obj = value;
954 }
955
956 static inline void
957 _Py_atomic_store_uchar_relaxed(unsigned char *obj, unsigned char value)
958 {
959 *(volatile unsigned char *)obj = value;
960 }
961
962 static inline void
963 _Py_atomic_store_short_relaxed(short *obj, short value)
964 {
965 *(volatile short *)obj = value;
966 }
967
968 static inline void
969 _Py_atomic_store_ushort_relaxed(unsigned short *obj, unsigned short value)
970 {
971 *(volatile unsigned short *)obj = value;
972 }
973
974 static inline void
975 _Py_atomic_store_long_relaxed(long *obj, long value)
976 {
977 *(volatile long *)obj = value;
978 }
979
980 static inline void
981 _Py_atomic_store_float_relaxed(float *obj, float value)
982 {
983 *(volatile float *)obj = value;
984 }
985
986 static inline void
987 _Py_atomic_store_double_relaxed(double *obj, double value)
988 {
989 *(volatile double *)obj = value;
990 }
991
992 static inline void
993 _Py_atomic_store_llong_relaxed(long long *obj, long long value)
994 {
995 *(volatile long long *)obj = value;
996 }
997
998
999 // --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------
1000
1001 static inline void *
1002 _Py_atomic_load_ptr_acquire(const void *obj)
1003 {
1004 #if defined(_M_X64) || defined(_M_IX86)
1005 return *(void * volatile *)obj;
1006 #elif defined(_M_ARM64)
1007 return (void *)__ldar64((unsigned __int64 volatile *)obj);
1008 #else
1009 # error "no implementation of _Py_atomic_load_ptr_acquire"
1010 #endif
1011 }
1012
1013 static inline uintptr_t
1014 _Py_atomic_load_uintptr_acquire(const uintptr_t *obj)
1015 {
1016 #if defined(_M_X64) || defined(_M_IX86)
1017 return *(uintptr_t volatile *)obj;
1018 #elif defined(_M_ARM64)
1019 return (uintptr_t)__ldar64((unsigned __int64 volatile *)obj);
1020 #else
1021 # error "no implementation of _Py_atomic_load_uintptr_acquire"
1022 #endif
1023 }
1024
1025 static inline void
1026 _Py_atomic_store_ptr_release(void *obj, void *value)
1027 {
1028 #if defined(_M_X64) || defined(_M_IX86)
1029 *(void * volatile *)obj = value;
1030 #elif defined(_M_ARM64)
1031 __stlr64((unsigned __int64 volatile *)obj, (uintptr_t)value);
1032 #else
1033 # error "no implementation of _Py_atomic_store_ptr_release"
1034 #endif
1035 }
1036
1037 static inline void
1038 _Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value)
1039 {
1040 #if defined(_M_X64) || defined(_M_IX86)
1041 *(uintptr_t volatile *)obj = value;
1042 #elif defined(_M_ARM64)
1043 _Py_atomic_ASSERT_ARG_TYPE(unsigned __int64);
1044 __stlr64((unsigned __int64 volatile *)obj, (unsigned __int64)value);
1045 #else
1046 # error "no implementation of _Py_atomic_store_uintptr_release"
1047 #endif
1048 }
1049
1050 static inline void
1051 _Py_atomic_store_int_release(int *obj, int value)
1052 {
1053 #if defined(_M_X64) || defined(_M_IX86)
1054 *(int volatile *)obj = value;
1055 #elif defined(_M_ARM64)
1056 _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32);
1057 __stlr32((unsigned __int32 volatile *)obj, (unsigned __int32)value);
1058 #else
1059 # error "no implementation of _Py_atomic_store_int_release"
1060 #endif
1061 }
1062
1063 static inline void
1064 _Py_atomic_store_uint_release(unsigned int *obj, unsigned int value)
1065 {
1066 #if defined(_M_X64) || defined(_M_IX86)
1067 *(volatile unsigned int *)obj = value;
1068 #elif defined(_M_ARM64)
1069 _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32);
1070 __stlr32((unsigned __int32 volatile *)obj, (unsigned __int32)value);
1071 #else
1072 # error "no implementation of _Py_atomic_store_uint_release"
1073 #endif
1074 }
1075
1076 static inline void
1077 _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value)
1078 {
1079 #if defined(_M_X64) || defined(_M_IX86)
1080 *(Py_ssize_t volatile *)obj = value;
1081 #elif defined(_M_ARM64)
1082 __stlr64((unsigned __int64 volatile *)obj, (unsigned __int64)value);
1083 #else
1084 # error "no implementation of _Py_atomic_store_ssize_release"
1085 #endif
1086 }
1087
1088 static inline int
1089 _Py_atomic_load_int_acquire(const int *obj)
1090 {
1091 #if defined(_M_X64) || defined(_M_IX86)
1092 return *(int volatile *)obj;
1093 #elif defined(_M_ARM64)
1094 _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32);
1095 return (int)__ldar32((unsigned __int32 volatile *)obj);
1096 #else
1097 # error "no implementation of _Py_atomic_load_int_acquire"
1098 #endif
1099 }
1100
1101 static inline void
1102 _Py_atomic_store_uint32_release(uint32_t *obj, uint32_t value)
1103 {
1104 #if defined(_M_X64) || defined(_M_IX86)
1105 *(uint32_t volatile *)obj = value;
1106 #elif defined(_M_ARM64)
1107 _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32);
1108 __stlr32((unsigned __int32 volatile *)obj, (unsigned __int32)value);
1109 #else
1110 # error "no implementation of _Py_atomic_store_uint32_release"
1111 #endif
1112 }
1113
1114 static inline void
1115 _Py_atomic_store_uint64_release(uint64_t *obj, uint64_t value)
1116 {
1117 #if defined(_M_X64) || defined(_M_IX86)
1118 *(uint64_t volatile *)obj = value;
1119 #elif defined(_M_ARM64)
1120 _Py_atomic_ASSERT_ARG_TYPE(unsigned __int64);
1121 __stlr64((unsigned __int64 volatile *)obj, (unsigned __int64)value);
1122 #else
1123 # error "no implementation of _Py_atomic_store_uint64_release"
1124 #endif
1125 }
1126
1127 static inline uint64_t
1128 _Py_atomic_load_uint64_acquire(const uint64_t *obj)
1129 {
1130 #if defined(_M_X64) || defined(_M_IX86)
1131 return *(uint64_t volatile *)obj;
1132 #elif defined(_M_ARM64)
1133 _Py_atomic_ASSERT_ARG_TYPE(__int64);
1134 return (uint64_t)__ldar64((unsigned __int64 volatile *)obj);
1135 #else
1136 # error "no implementation of _Py_atomic_load_uint64_acquire"
1137 #endif
1138 }
1139
1140 static inline uint32_t
1141 _Py_atomic_load_uint32_acquire(const uint32_t *obj)
1142 {
1143 #if defined(_M_X64) || defined(_M_IX86)
1144 return *(uint32_t volatile *)obj;
1145 #elif defined(_M_ARM64)
1146 return (uint32_t)__ldar32((uint32_t volatile *)obj);
1147 #else
1148 # error "no implementation of _Py_atomic_load_uint32_acquire"
1149 #endif
1150 }
1151
1152 static inline Py_ssize_t
1153 _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj)
1154 {
1155 #if defined(_M_X64) || defined(_M_IX86)
1156 return *(Py_ssize_t volatile *)obj;
1157 #elif defined(_M_ARM64)
1158 return (Py_ssize_t)__ldar64((unsigned __int64 volatile *)obj);
1159 #else
1160 # error "no implementation of _Py_atomic_load_ssize_acquire"
1161 #endif
1162 }
1163
1164 // --- _Py_atomic_fence ------------------------------------------------------
1165
1166 static inline void
1167 _Py_atomic_fence_seq_cst(void)
1168 {
1169 #if defined(_M_ARM64)
1170 __dmb(_ARM64_BARRIER_ISH);
1171 #elif defined(_M_X64)
1172 __faststorefence();
1173 #elif defined(_M_IX86)
1174 _mm_mfence();
1175 #else
1176 # error "no implementation of _Py_atomic_fence_seq_cst"
1177 #endif
1178 }
1179
1180 static inline void
1181 _Py_atomic_fence_acquire(void)
1182 {
1183 #if defined(_M_ARM64)
1184 __dmb(_ARM64_BARRIER_ISHLD);
1185 #elif defined(_M_X64) || defined(_M_IX86)
1186 _ReadBarrier();
1187 #else
1188 # error "no implementation of _Py_atomic_fence_acquire"
1189 #endif
1190 }
1191
1192 static inline void
1193 _Py_atomic_fence_release(void)
1194 {
1195 #if defined(_M_ARM64)
1196 __dmb(_ARM64_BARRIER_ISH);
1197 #elif defined(_M_X64) || defined(_M_IX86)
1198 _ReadWriteBarrier();
1199 #else
1200 # error "no implementation of _Py_atomic_fence_release"
1201 #endif
1202 }
1203
1204 #undef _Py_atomic_ASSERT_ARG_TYPE