| 1 |
/* |
| 2 |
* Copyright © 2025 Imagination Technologies Ltd. |
| 3 |
* SPDX-License-Identifier: MIT |
| 4 |
*/ |
| 5 |
|
| 6 |
#ifndef __pvr_address_type |
| 7 |
#define __pvr_address_type uint64_t |
| 8 |
#define __pvr_get_address(addr) addr |
| 9 |
#define __pvr_make_address(addr) addr |
| 10 |
#endif /* __pvr_address_type */ |
| 11 |
|
| 12 |
#include "csbgen/rogue/cdm.h" |
| 13 |
#include "hwdef/rogue_hw_defs.h" |
| 14 |
#include "libcl.h" |
| 15 |
|
| 16 |
/* |
| 17 |
* Emulates atomic operations by serializing execution to each slot via a |
| 18 |
* mutex, and to each instance via a per-instance loop. |
| 19 |
*/ |
| 20 |
#define usclib_foreach_instance_atomic() \ |
| 21 |
nir_mutex_pco(PCO_MUTEX_ID_ATOMIC_EMU, PCO_MUTEX_OP_LOCK); \ |
| 22 |
for (bool __done = false; !__done; ({ nir_mutex_pco(PCO_MUTEX_ID_ATOMIC_EMU, PCO_MUTEX_OP_RELEASE); __done = true; })) \ |
| 23 |
for (uint __u = 0; __u < ROGUE_MAX_INSTANCES_PER_TASK; ++__u) \ |
| 24 |
if (__u == nir_load_instance_num_pco()) |
| 25 |
|
| 26 |
uint32_t |
| 27 |
usclib_emu_ssbo_atomic_comp_swap(uint2 ssbo_buffer, uint ssbo_offset, uint compare, uint data) |
| 28 |
{ |
| 29 |
uint32_t result; |
| 30 |
|
| 31 |
usclib_foreach_instance_atomic() { |
| 32 |
uint32_t pre_val = nir_load_ssbo(ssbo_buffer, ssbo_offset, ACCESS_COHERENT, 4, 0, 0); |
| 33 |
result = pre_val; |
| 34 |
|
| 35 |
uint32_t post_val = (pre_val == compare) ? data : pre_val; |
| 36 |
nir_store_ssbo(post_val, ssbo_buffer, ssbo_offset, 0x1, ACCESS_COHERENT, 4, 0, 0); |
| 37 |
} |
| 38 |
|
| 39 |
return result; |
| 40 |
} |
| 41 |
|
| 42 |
uint32_t |
| 43 |
usclib_emu_global_atomic_comp_swap(uint2 addr, uint compare, uint data) |
| 44 |
{ |
| 45 |
uint32_t result; |
| 46 |
|
| 47 |
usclib_foreach_instance_atomic() { |
| 48 |
uint32_t pre_val = nir_dma_ld_pco(1, addr); |
| 49 |
result = pre_val; |
| 50 |
|
| 51 |
uint32_t post_val = (pre_val == compare) ? data : pre_val; |
| 52 |
nir_dma_st_pco(false, addr, post_val); |
| 53 |
} |
| 54 |
|
| 55 |
return result; |
| 56 |
} |
| 57 |
|
| 58 |
void |
| 59 |
usclib_barrier(uint num_slots, uint counter_offset) |
| 60 |
{ |
| 61 |
#define load_barrier_counter() nir_load_shared(counter_offset, 0, 0, 4, 0) |
| 62 |
#define store_barrier_counter(value) nir_store_shared(value, counter_offset, 0, 0, 0x1, 4, 0) |
| 63 |
|
| 64 |
bool is_inst_zero = !nir_load_instance_num_pco(); |
| 65 |
|
| 66 |
nir_mutex_pco(PCO_MUTEX_ID_BARRIER, PCO_MUTEX_OP_LOCK); |
| 67 |
|
| 68 |
if (is_inst_zero) |
| 69 |
store_barrier_counter(load_barrier_counter() + 1); |
| 70 |
|
| 71 |
bool all_slots_done = load_barrier_counter() == num_slots; |
| 72 |
if (all_slots_done) { |
| 73 |
if (is_inst_zero) |
| 74 |
store_barrier_counter(0); |
| 75 |
} else { |
| 76 |
do { |
| 77 |
nir_mutex_pco(PCO_MUTEX_ID_BARRIER, PCO_MUTEX_OP_RELEASE_SLEEP); |
| 78 |
nir_mutex_pco(PCO_MUTEX_ID_BARRIER, PCO_MUTEX_OP_LOCK); |
| 79 |
} while (load_barrier_counter() != 0); |
| 80 |
} |
| 81 |
|
| 82 |
nir_mutex_pco(PCO_MUTEX_ID_BARRIER, PCO_MUTEX_OP_RELEASE_WAKEUP); |
| 83 |
} |
| 84 |
|
| 85 |
void |
| 86 |
usclib_zero_init_wg_mem(uint count) |
| 87 |
{ |
| 88 |
for (unsigned u = 0; u < count; ++u) |
| 89 |
nir_store_shared(0, u * sizeof(uint32_t), 0, 0, 0x1, 4, 0); |
| 90 |
} |