| 1 |
/* |
| 2 |
* Copyright © 2025 Imagination Technologies Ltd. |
| 3 |
* SPDX-License-Identifier: MIT |
| 4 |
*/ |
| 5 |
|
| 6 |
#include "libcl.h" |
| 7 |
#include "compiler/libcl/libcl_vk.h" |
| 8 |
|
| 9 |
KERNEL(1) |
| 10 |
vs_nop_common(void) |
| 11 |
{ |
| 12 |
return; |
| 13 |
} |
| 14 |
|
| 15 |
KERNEL(1) |
| 16 |
fs_nop_common(void) |
| 17 |
{ |
| 18 |
return; |
| 19 |
} |
| 20 |
|
| 21 |
KERNEL(1) |
| 22 |
cs_nop_common(void) |
| 23 |
{ |
| 24 |
return; |
| 25 |
} |
| 26 |
|
| 27 |
/* gl_Position = vec4(in.xyz, 1.0f); */ |
| 28 |
KERNEL(1) |
| 29 |
vs_passthrough_common(void) |
| 30 |
{ |
| 31 |
nir_uvsw_write_pco(0, nir_load_vtxin_pco(3, 0)); |
| 32 |
nir_uvsw_write_pco(3, 1.0f); |
| 33 |
} |
| 34 |
|
| 35 |
/* gl_Position = vec4(in.xyz, 1.0f); rta_out = in.w; */ |
| 36 |
KERNEL(1) |
| 37 |
vs_passthrough_rta_common(void) |
| 38 |
{ |
| 39 |
vs_passthrough_common(); |
| 40 |
nir_uvsw_write_pco(4, nir_load_vtxin_pco(1, 3)); |
| 41 |
} |
| 42 |
|
| 43 |
/* TODO: uint index = cl_global_id.x; |
| 44 |
* instead of this function once things |
| 45 |
* are properly hooked up. |
| 46 |
*/ |
| 47 |
static inline uint |
| 48 |
query_calc_global_id(void) |
| 49 |
{ |
| 50 |
uint local_invoc_index = nir_load_vtxin_pco(1, 0); |
| 51 |
local_invoc_index &= get_local_size(0) - 1; |
| 52 |
uint wg_id = nir_load_coeff_pco(1, 0); |
| 53 |
return nir_imad(wg_id, get_local_size(0), local_invoc_index); |
| 54 |
} |
| 55 |
|
| 56 |
/* TODO: support parameter passing. */ |
| 57 |
/* TODO: switch to common implementation. */ |
| 58 |
KERNEL(32) |
| 59 |
cs_query_availability_common(void) |
| 60 |
{ |
| 61 |
uint index_count = nir_load_preamble(1, PVR_QUERY_AVAILABILITY_DATA_INDEX_COUNT, 0); |
| 62 |
|
| 63 |
uint index_base_addr_lo = nir_load_preamble(1, PVR_QUERY_AVAILABILITY_DATA_INDEX_BO_LO, 0); |
| 64 |
uint index_base_addr_hi = nir_load_preamble(1, PVR_QUERY_AVAILABILITY_DATA_INDEX_BO_HI, 0); |
| 65 |
|
| 66 |
uint avail_base_addr_lo = nir_load_preamble(1, PVR_QUERY_AVAILABILITY_DATA_BO_LO, 0); |
| 67 |
uint avail_base_addr_hi = nir_load_preamble(1, PVR_QUERY_AVAILABILITY_DATA_BO_HI, 0); |
| 68 |
|
| 69 |
uint index = query_calc_global_id(); |
| 70 |
|
| 71 |
if (index < index_count) { |
| 72 |
uint2 index_addr = nir_uadd64_32(index_base_addr_lo, index_base_addr_hi, index * sizeof(uint32_t)); |
| 73 |
uint offset = nir_dma_ld_pco(1, index_addr); |
| 74 |
|
| 75 |
uint2 avail_addr = nir_uadd64_32(avail_base_addr_lo, avail_base_addr_hi, offset * sizeof(uint32_t)); |
| 76 |
|
| 77 |
nir_dma_st_pco(false, avail_addr, ~0U); |
| 78 |
} |
| 79 |
} |
| 80 |
|
| 81 |
KERNEL(32) |
| 82 |
cs_query_copy_common(void) |
| 83 |
{ |
| 84 |
uint index_count = nir_load_preamble(1, PVR_QUERY_COPY_DATA_INDEX_COUNT, 0); |
| 85 |
|
| 86 |
uint dest_base_addr_lo = nir_load_preamble(1, PVR_QUERY_COPY_DATA_DEST_BO_LO, 0); |
| 87 |
uint dest_base_addr_hi = nir_load_preamble(1, PVR_QUERY_COPY_DATA_DEST_BO_HI, 0); |
| 88 |
|
| 89 |
uint avail_base_addr_lo = nir_load_preamble(1, PVR_QUERY_COPY_DATA_AVAILABILITY_BO_LO, 0); |
| 90 |
uint avail_base_addr_hi = nir_load_preamble(1, PVR_QUERY_COPY_DATA_AVAILABILITY_BO_HI, 0); |
| 91 |
|
| 92 |
uint result_base_addr_lo = nir_load_preamble(1, PVR_QUERY_COPY_DATA_RESULT_BO_LO, 0); |
| 93 |
uint result_base_addr_hi = nir_load_preamble(1, PVR_QUERY_COPY_DATA_RESULT_BO_HI, 0); |
| 94 |
|
| 95 |
uint dest_stride = nir_load_preamble(1, PVR_QUERY_COPY_DATA_DEST_STRIDE, 0); |
| 96 |
|
| 97 |
uint flags = nir_load_preamble(1, PVR_QUERY_COPY_DATA_FLAGS, 0); |
| 98 |
|
| 99 |
uint index = query_calc_global_id(); |
| 100 |
|
| 101 |
if (index < index_count) { |
| 102 |
uint2 avail_addr = nir_uadd64_32(avail_base_addr_lo, avail_base_addr_hi, index * sizeof(uint32_t)); |
| 103 |
uint available = nir_dma_ld_pco(1, avail_addr); |
| 104 |
|
| 105 |
uint2 dest_addr = nir_umad64_32(dest_stride, index, dest_base_addr_lo, dest_base_addr_hi); |
| 106 |
|
| 107 |
if (available || (flags & VK_QUERY_RESULT_PARTIAL_BIT)) { |
| 108 |
uint2 result_addr = nir_uadd64_32(result_base_addr_lo, result_base_addr_hi, index * sizeof(uint32_t)); |
| 109 |
uint result = nir_dma_ld_pco(1, result_addr); |
| 110 |
|
| 111 |
/* TODO: for 64/32-bit writes, just prep the 64-bit one and set the burst-length variably. */ |
| 112 |
if (flags & VK_QUERY_RESULT_64_BIT) { |
| 113 |
/* TODO: check if data should be (result, 0) or (0, result) */ |
| 114 |
nir_dma_st_pco(false, dest_addr, result, 0); |
| 115 |
} else { |
| 116 |
nir_dma_st_pco(false, dest_addr, result); |
| 117 |
} |
| 118 |
} |
| 119 |
|
| 120 |
if (flags & VK_QUERY_RESULT_WITH_AVAILABILITY_BIT) { |
| 121 |
if (flags & VK_QUERY_RESULT_64_BIT) { |
| 122 |
dest_addr = nir_uadd64_32(dest_addr.x, dest_addr.y, sizeof(uint64_t)); |
| 123 |
/* TODO: check if data should be (available, 0) or (0, available) */ |
| 124 |
nir_dma_st_pco(false, dest_addr, available, 0); |
| 125 |
} else { |
| 126 |
dest_addr = nir_uadd64_32(dest_addr.x, dest_addr.y, sizeof(uint32_t)); |
| 127 |
nir_dma_st_pco(false, dest_addr, available); |
| 128 |
} |
| 129 |
} |
| 130 |
} |
| 131 |
} |
| 132 |
|
| 133 |
KERNEL(32) |
| 134 |
cs_query_reset_common(void) |
| 135 |
{ |
| 136 |
uint index_count = nir_load_preamble(1, PVR_QUERY_RESET_DATA_INDEX_COUNT, 0); |
| 137 |
|
| 138 |
uint result_base_addr_lo = nir_load_preamble(1, PVR_QUERY_RESET_DATA_RESULT_BO_LO, 0); |
| 139 |
uint result_base_addr_hi = nir_load_preamble(1, PVR_QUERY_RESET_DATA_RESULT_BO_HI, 0); |
| 140 |
|
| 141 |
uint avail_base_addr_lo = nir_load_preamble(1, PVR_QUERY_RESET_DATA_AVAILABILITY_BO_LO, 0); |
| 142 |
uint avail_base_addr_hi = nir_load_preamble(1, PVR_QUERY_RESET_DATA_AVAILABILITY_BO_HI, 0); |
| 143 |
|
| 144 |
uint index = query_calc_global_id(); |
| 145 |
|
| 146 |
if (index < index_count) { |
| 147 |
uint2 result_addr = nir_uadd64_32(result_base_addr_lo, result_base_addr_hi, index * sizeof(uint32_t)); |
| 148 |
nir_dma_st_pco(false, result_addr, 0); |
| 149 |
|
| 150 |
uint2 avail_addr = nir_uadd64_32(avail_base_addr_lo, avail_base_addr_hi, index * sizeof(uint32_t)); |
| 151 |
nir_dma_st_pco(false, avail_addr, 0); |
| 152 |
} |
| 153 |
} |
| 154 |
|
| 155 |
static inline void |
| 156 |
load_store_sr(uint block_size_dw, |
| 157 |
bool store, |
| 158 |
bool use_loaded_size, |
| 159 |
bool use_temp_storage) |
| 160 |
{ |
| 161 |
/* static_assert(!use_temp_storage || (!store && block_size_dw == 16), "Invalid arguments."); */ |
| 162 |
|
| 163 |
uint size_addr_lo = nir_load_vtxin_pco(1, PVR_LOAD_STORE_SR_DATA_SIZE_ADDR_LO); |
| 164 |
uint size_addr_hi = nir_load_vtxin_pco(1, PVR_LOAD_STORE_SR_DATA_SIZE_ADDR_HI); |
| 165 |
uint2 size_addr = (uint2)(size_addr_lo, size_addr_hi); |
| 166 |
|
| 167 |
/* Load the allocation size in 16 dword blocks. */ |
| 168 |
uint alloc_size_16dw = !store && use_loaded_size ? |
| 169 |
nir_dma_ld_pco(1, size_addr) : |
| 170 |
nir_load_shared_reg_alloc_size_pco(); |
| 171 |
|
| 172 |
/* Store the allocation size in 16 dword blocks. */ |
| 173 |
if (store) |
| 174 |
nir_dma_st_pco(true, size_addr, alloc_size_16dw); |
| 175 |
|
| 176 |
if (alloc_size_16dw) { |
| 177 |
uint store_addr_lo = nir_load_vtxin_pco(1, PVR_LOAD_STORE_SR_DATA_STORE_ADDR_LO); |
| 178 |
uint store_addr_hi = nir_load_vtxin_pco(1, PVR_LOAD_STORE_SR_DATA_STORE_ADDR_HI); |
| 179 |
uint2 store_addr = (uint2)(store_addr_lo, store_addr_hi); |
| 180 |
|
| 181 |
/* TODO: if multicore, offset store_addr accordingly. */ |
| 182 |
|
| 183 |
uint size_remaining_dw = alloc_size_16dw * 16; |
| 184 |
uint offset = 0; |
| 185 |
|
| 186 |
while (size_remaining_dw) { |
| 187 |
uint burst_len = size_remaining_dw; |
| 188 |
if (size_remaining_dw > block_size_dw) |
| 189 |
burst_len = block_size_dw == 1024 ? 0 : block_size_dw; |
| 190 |
|
| 191 |
if (store) { |
| 192 |
nir_dma_st_shregs_pco(store_addr, burst_len, offset, true); |
| 193 |
} else if (use_temp_storage) { |
| 194 |
uint16 temp = nir_dma_ld_pco(16, store_addr); |
| 195 |
for (unsigned u = 0; u < 16; ++u) |
| 196 |
nir_store_preamble_dynamic(temp[u], offset + u, 0); |
| 197 |
} else { |
| 198 |
nir_dma_ld_shregs_pco(store_addr, burst_len, offset); |
| 199 |
} |
| 200 |
|
| 201 |
offset += block_size_dw; |
| 202 |
size_remaining_dw -= block_size_dw; |
| 203 |
store_addr = nir_uadd64_32(store_addr.x, store_addr.y, block_size_dw * sizeof(uint32_t)); |
| 204 |
} |
| 205 |
} |
| 206 |
} |
| 207 |
|
| 208 |
KERNEL(1) |
| 209 |
cs_store_sr_1024_common(void) |
| 210 |
{ |
| 211 |
load_store_sr(1024, true, false, false); |
| 212 |
} |
| 213 |
|
| 214 |
KERNEL(1) |
| 215 |
cs_load_sr_256_common(void) |
| 216 |
{ |
| 217 |
load_store_sr(256, false, true, false); |
| 218 |
} |
| 219 |
|
| 220 |
KERNEL(1) |
| 221 |
cs_load_sr_16_common(void) |
| 222 |
{ |
| 223 |
load_store_sr(16, false, false, true); |
| 224 |
} |
| 225 |
|
| 226 |
KERNEL(1) |
| 227 |
cs_idfwdf_common(void) |
| 228 |
{ |
| 229 |
/* Do a memory store and a texture read to fence any loads/texture writes from previous kernels */ |
| 230 |
|
| 231 |
uint addr_lo = nir_load_preamble(1, PVR_IDFWDF_DATA_ADDR_LO, 0); |
| 232 |
uint addr_hi = nir_load_preamble(1, PVR_IDFWDF_DATA_ADDR_HI, 0); |
| 233 |
uint2 addr = (uint2)(addr_lo, addr_hi); |
| 234 |
|
| 235 |
nir_dma_st_pco(true, addr, 0U); |
| 236 |
|
| 237 |
uint16 smp_data = (uint16)(0); |
| 238 |
uint4 tex_state = nir_load_preamble(4, PVR_IDFWDF_DATA_TEX, 0); |
| 239 |
uint4 smp_state = nir_load_preamble(4, PVR_IDFWDF_DATA_SMP, 0); |
| 240 |
|
| 241 |
/* TODO: improve flag/range emission. */ |
| 242 |
#define FLAGS 0xc9 /* 2d pplod fcnorm replace */ |
| 243 |
#define RANGE 3 /* data comps: x, y, lod */ |
| 244 |
|
| 245 |
#pragma unroll |
| 246 |
for (unsigned y = 0; y < PVR_IDFWDF_TEX_HEIGHT; ++y) { |
| 247 |
#pragma unroll |
| 248 |
for (unsigned x = 0; x < PVR_IDFWDF_TEX_WIDTH; ++x) { |
| 249 |
smp_data.xy = (uint2)(x, y); |
| 250 |
nir_smp_pco(smp_data, tex_state, smp_state, FLAGS, RANGE); |
| 251 |
} |
| 252 |
} |
| 253 |
} |