Debian Codegraph @

src/imagination/pco/usclib/sync.cl

raw
1 /*
2 * Copyright © 2025 Imagination Technologies Ltd.
3 * SPDX-License-Identifier: MIT
4 */
5
6 #ifndef __pvr_address_type
7 #define __pvr_address_type uint64_t
8 #define __pvr_get_address(addr) addr
9 #define __pvr_make_address(addr) addr
10 #endif /* __pvr_address_type */
11
12 #include "csbgen/rogue/cdm.h"
13 #include "hwdef/rogue_hw_defs.h"
14 #include "libcl.h"
15
16 /*
17 * Emulates atomic operations by serializing execution to each slot via a
18 * mutex, and to each instance via a per-instance loop.
19 */
20 #define usclib_foreach_instance_atomic() \
21 nir_mutex_pco(PCO_MUTEX_ID_ATOMIC_EMU, PCO_MUTEX_OP_LOCK); \
22 for (bool __done = false; !__done; ({ nir_mutex_pco(PCO_MUTEX_ID_ATOMIC_EMU, PCO_MUTEX_OP_RELEASE); __done = true; })) \
23 for (uint __u = 0; __u < ROGUE_MAX_INSTANCES_PER_TASK; ++__u) \
24 if (__u == nir_load_instance_num_pco())
25
26 uint32_t
27 usclib_emu_ssbo_atomic_comp_swap(uint2 ssbo_buffer, uint ssbo_offset, uint compare, uint data)
28 {
29 uint32_t result;
30
31 usclib_foreach_instance_atomic() {
32 uint32_t pre_val = nir_load_ssbo(ssbo_buffer, ssbo_offset, ACCESS_COHERENT, 4, 0, 0);
33 result = pre_val;
34
35 uint32_t post_val = (pre_val == compare) ? data : pre_val;
36 nir_store_ssbo(post_val, ssbo_buffer, ssbo_offset, 0x1, ACCESS_COHERENT, 4, 0, 0);
37 }
38
39 return result;
40 }
41
42 uint32_t
43 usclib_emu_global_atomic_comp_swap(uint2 addr, uint compare, uint data)
44 {
45 uint32_t result;
46
47 usclib_foreach_instance_atomic() {
48 uint32_t pre_val = nir_dma_ld_pco(1, addr);
49 result = pre_val;
50
51 uint32_t post_val = (pre_val == compare) ? data : pre_val;
52 nir_dma_st_pco(false, addr, post_val);
53 }
54
55 return result;
56 }
57
58 void
59 usclib_barrier(uint num_slots, uint counter_offset)
60 {
61 #define load_barrier_counter() nir_load_shared(counter_offset, 0, 0, 4, 0)
62 #define store_barrier_counter(value) nir_store_shared(value, counter_offset, 0, 0, 0x1, 4, 0)
63
64 bool is_inst_zero = !nir_load_instance_num_pco();
65
66 nir_mutex_pco(PCO_MUTEX_ID_BARRIER, PCO_MUTEX_OP_LOCK);
67
68 if (is_inst_zero)
69 store_barrier_counter(load_barrier_counter() + 1);
70
71 bool all_slots_done = load_barrier_counter() == num_slots;
72 if (all_slots_done) {
73 if (is_inst_zero)
74 store_barrier_counter(0);
75 } else {
76 do {
77 nir_mutex_pco(PCO_MUTEX_ID_BARRIER, PCO_MUTEX_OP_RELEASE_SLEEP);
78 nir_mutex_pco(PCO_MUTEX_ID_BARRIER, PCO_MUTEX_OP_LOCK);
79 } while (load_barrier_counter() != 0);
80 }
81
82 nir_mutex_pco(PCO_MUTEX_ID_BARRIER, PCO_MUTEX_OP_RELEASE_WAKEUP);
83 }
84
85 void
86 usclib_zero_init_wg_mem(uint count)
87 {
88 for (unsigned u = 0; u < count; ++u)
89 nir_store_shared(0, u * sizeof(uint32_t), 0, 0, 0x1, 4, 0);
90 }