Debian Codegraph @

src/imagination/pco/usclib/common.cl

raw
1 /*
2 * Copyright © 2025 Imagination Technologies Ltd.
3 * SPDX-License-Identifier: MIT
4 */
5
6 #include "libcl.h"
7 #include "compiler/libcl/libcl_vk.h"
8
9 KERNEL(1)
10 vs_nop_common(void)
11 {
12 return;
13 }
14
15 KERNEL(1)
16 fs_nop_common(void)
17 {
18 return;
19 }
20
21 KERNEL(1)
22 cs_nop_common(void)
23 {
24 return;
25 }
26
27 /* gl_Position = vec4(in.xyz, 1.0f); */
28 KERNEL(1)
29 vs_passthrough_common(void)
30 {
31 nir_uvsw_write_pco(0, nir_load_vtxin_pco(3, 0));
32 nir_uvsw_write_pco(3, 1.0f);
33 }
34
35 /* gl_Position = vec4(in.xyz, 1.0f); rta_out = in.w; */
36 KERNEL(1)
37 vs_passthrough_rta_common(void)
38 {
39 vs_passthrough_common();
40 nir_uvsw_write_pco(4, nir_load_vtxin_pco(1, 3));
41 }
42
43 /* TODO: uint index = cl_global_id.x;
44 * instead of this function once things
45 * are properly hooked up.
46 */
47 static inline uint
48 query_calc_global_id(void)
49 {
50 uint local_invoc_index = nir_load_vtxin_pco(1, 0);
51 local_invoc_index &= get_local_size(0) - 1;
52 uint wg_id = nir_load_coeff_pco(1, 0);
53 return nir_imad(wg_id, get_local_size(0), local_invoc_index);
54 }
55
56 /* TODO: support parameter passing. */
57 /* TODO: switch to common implementation. */
58 KERNEL(32)
59 cs_query_availability_common(void)
60 {
61 uint index_count = nir_load_preamble(1, PVR_QUERY_AVAILABILITY_DATA_INDEX_COUNT, 0);
62
63 uint index_base_addr_lo = nir_load_preamble(1, PVR_QUERY_AVAILABILITY_DATA_INDEX_BO_LO, 0);
64 uint index_base_addr_hi = nir_load_preamble(1, PVR_QUERY_AVAILABILITY_DATA_INDEX_BO_HI, 0);
65
66 uint avail_base_addr_lo = nir_load_preamble(1, PVR_QUERY_AVAILABILITY_DATA_BO_LO, 0);
67 uint avail_base_addr_hi = nir_load_preamble(1, PVR_QUERY_AVAILABILITY_DATA_BO_HI, 0);
68
69 uint index = query_calc_global_id();
70
71 if (index < index_count) {
72 uint2 index_addr = nir_uadd64_32(index_base_addr_lo, index_base_addr_hi, index * sizeof(uint32_t));
73 uint offset = nir_dma_ld_pco(1, index_addr);
74
75 uint2 avail_addr = nir_uadd64_32(avail_base_addr_lo, avail_base_addr_hi, offset * sizeof(uint32_t));
76
77 nir_dma_st_pco(false, avail_addr, ~0U);
78 }
79 }
80
81 KERNEL(32)
82 cs_query_copy_common(void)
83 {
84 uint index_count = nir_load_preamble(1, PVR_QUERY_COPY_DATA_INDEX_COUNT, 0);
85
86 uint dest_base_addr_lo = nir_load_preamble(1, PVR_QUERY_COPY_DATA_DEST_BO_LO, 0);
87 uint dest_base_addr_hi = nir_load_preamble(1, PVR_QUERY_COPY_DATA_DEST_BO_HI, 0);
88
89 uint avail_base_addr_lo = nir_load_preamble(1, PVR_QUERY_COPY_DATA_AVAILABILITY_BO_LO, 0);
90 uint avail_base_addr_hi = nir_load_preamble(1, PVR_QUERY_COPY_DATA_AVAILABILITY_BO_HI, 0);
91
92 uint result_base_addr_lo = nir_load_preamble(1, PVR_QUERY_COPY_DATA_RESULT_BO_LO, 0);
93 uint result_base_addr_hi = nir_load_preamble(1, PVR_QUERY_COPY_DATA_RESULT_BO_HI, 0);
94
95 uint dest_stride = nir_load_preamble(1, PVR_QUERY_COPY_DATA_DEST_STRIDE, 0);
96
97 uint flags = nir_load_preamble(1, PVR_QUERY_COPY_DATA_FLAGS, 0);
98
99 uint index = query_calc_global_id();
100
101 if (index < index_count) {
102 uint2 avail_addr = nir_uadd64_32(avail_base_addr_lo, avail_base_addr_hi, index * sizeof(uint32_t));
103 uint available = nir_dma_ld_pco(1, avail_addr);
104
105 uint2 dest_addr = nir_umad64_32(dest_stride, index, dest_base_addr_lo, dest_base_addr_hi);
106
107 if (available || (flags & VK_QUERY_RESULT_PARTIAL_BIT)) {
108 uint2 result_addr = nir_uadd64_32(result_base_addr_lo, result_base_addr_hi, index * sizeof(uint32_t));
109 uint result = nir_dma_ld_pco(1, result_addr);
110
111 /* TODO: for 64/32-bit writes, just prep the 64-bit one and set the burst-length variably. */
112 if (flags & VK_QUERY_RESULT_64_BIT) {
113 /* TODO: check if data should be (result, 0) or (0, result) */
114 nir_dma_st_pco(false, dest_addr, result, 0);
115 } else {
116 nir_dma_st_pco(false, dest_addr, result);
117 }
118 }
119
120 if (flags & VK_QUERY_RESULT_WITH_AVAILABILITY_BIT) {
121 if (flags & VK_QUERY_RESULT_64_BIT) {
122 dest_addr = nir_uadd64_32(dest_addr.x, dest_addr.y, sizeof(uint64_t));
123 /* TODO: check if data should be (available, 0) or (0, available) */
124 nir_dma_st_pco(false, dest_addr, available, 0);
125 } else {
126 dest_addr = nir_uadd64_32(dest_addr.x, dest_addr.y, sizeof(uint32_t));
127 nir_dma_st_pco(false, dest_addr, available);
128 }
129 }
130 }
131 }
132
133 KERNEL(32)
134 cs_query_reset_common(void)
135 {
136 uint index_count = nir_load_preamble(1, PVR_QUERY_RESET_DATA_INDEX_COUNT, 0);
137
138 uint result_base_addr_lo = nir_load_preamble(1, PVR_QUERY_RESET_DATA_RESULT_BO_LO, 0);
139 uint result_base_addr_hi = nir_load_preamble(1, PVR_QUERY_RESET_DATA_RESULT_BO_HI, 0);
140
141 uint avail_base_addr_lo = nir_load_preamble(1, PVR_QUERY_RESET_DATA_AVAILABILITY_BO_LO, 0);
142 uint avail_base_addr_hi = nir_load_preamble(1, PVR_QUERY_RESET_DATA_AVAILABILITY_BO_HI, 0);
143
144 uint index = query_calc_global_id();
145
146 if (index < index_count) {
147 uint2 result_addr = nir_uadd64_32(result_base_addr_lo, result_base_addr_hi, index * sizeof(uint32_t));
148 nir_dma_st_pco(false, result_addr, 0);
149
150 uint2 avail_addr = nir_uadd64_32(avail_base_addr_lo, avail_base_addr_hi, index * sizeof(uint32_t));
151 nir_dma_st_pco(false, avail_addr, 0);
152 }
153 }
154
155 static inline void
156 load_store_sr(uint block_size_dw,
157 bool store,
158 bool use_loaded_size,
159 bool use_temp_storage)
160 {
161 /* static_assert(!use_temp_storage || (!store && block_size_dw == 16), "Invalid arguments."); */
162
163 uint size_addr_lo = nir_load_vtxin_pco(1, PVR_LOAD_STORE_SR_DATA_SIZE_ADDR_LO);
164 uint size_addr_hi = nir_load_vtxin_pco(1, PVR_LOAD_STORE_SR_DATA_SIZE_ADDR_HI);
165 uint2 size_addr = (uint2)(size_addr_lo, size_addr_hi);
166
167 /* Load the allocation size in 16 dword blocks. */
168 uint alloc_size_16dw = !store && use_loaded_size ?
169 nir_dma_ld_pco(1, size_addr) :
170 nir_load_shared_reg_alloc_size_pco();
171
172 /* Store the allocation size in 16 dword blocks. */
173 if (store)
174 nir_dma_st_pco(true, size_addr, alloc_size_16dw);
175
176 if (alloc_size_16dw) {
177 uint store_addr_lo = nir_load_vtxin_pco(1, PVR_LOAD_STORE_SR_DATA_STORE_ADDR_LO);
178 uint store_addr_hi = nir_load_vtxin_pco(1, PVR_LOAD_STORE_SR_DATA_STORE_ADDR_HI);
179 uint2 store_addr = (uint2)(store_addr_lo, store_addr_hi);
180
181 /* TODO: if multicore, offset store_addr accordingly. */
182
183 uint size_remaining_dw = alloc_size_16dw * 16;
184 uint offset = 0;
185
186 while (size_remaining_dw) {
187 uint burst_len = size_remaining_dw;
188 if (size_remaining_dw > block_size_dw)
189 burst_len = block_size_dw == 1024 ? 0 : block_size_dw;
190
191 if (store) {
192 nir_dma_st_shregs_pco(store_addr, burst_len, offset, true);
193 } else if (use_temp_storage) {
194 uint16 temp = nir_dma_ld_pco(16, store_addr);
195 for (unsigned u = 0; u < 16; ++u)
196 nir_store_preamble_dynamic(temp[u], offset + u, 0);
197 } else {
198 nir_dma_ld_shregs_pco(store_addr, burst_len, offset);
199 }
200
201 offset += block_size_dw;
202 size_remaining_dw -= block_size_dw;
203 store_addr = nir_uadd64_32(store_addr.x, store_addr.y, block_size_dw * sizeof(uint32_t));
204 }
205 }
206 }
207
208 KERNEL(1)
209 cs_store_sr_1024_common(void)
210 {
211 load_store_sr(1024, true, false, false);
212 }
213
214 KERNEL(1)
215 cs_load_sr_256_common(void)
216 {
217 load_store_sr(256, false, true, false);
218 }
219
220 KERNEL(1)
221 cs_load_sr_16_common(void)
222 {
223 load_store_sr(16, false, false, true);
224 }
225
226 KERNEL(1)
227 cs_idfwdf_common(void)
228 {
229 /* Do a memory store and a texture read to fence any loads/texture writes from previous kernels */
230
231 uint addr_lo = nir_load_preamble(1, PVR_IDFWDF_DATA_ADDR_LO, 0);
232 uint addr_hi = nir_load_preamble(1, PVR_IDFWDF_DATA_ADDR_HI, 0);
233 uint2 addr = (uint2)(addr_lo, addr_hi);
234
235 nir_dma_st_pco(true, addr, 0U);
236
237 uint16 smp_data = (uint16)(0);
238 uint4 tex_state = nir_load_preamble(4, PVR_IDFWDF_DATA_TEX, 0);
239 uint4 smp_state = nir_load_preamble(4, PVR_IDFWDF_DATA_SMP, 0);
240
241 /* TODO: improve flag/range emission. */
242 #define FLAGS 0xc9 /* 2d pplod fcnorm replace */
243 #define RANGE 3 /* data comps: x, y, lod */
244
245 #pragma unroll
246 for (unsigned y = 0; y < PVR_IDFWDF_TEX_HEIGHT; ++y) {
247 #pragma unroll
248 for (unsigned x = 0; x < PVR_IDFWDF_TEX_WIDTH; ++x) {
249 smp_data.xy = (uint2)(x, y);
250 nir_smp_pco(smp_data, tex_state, smp_state, FLAGS, RANGE);
251 }
252 }
253 }