Debian Codegraph @

src/imagination/vulkan/pvr_arch_csb.c

raw
1 /*
2 * Copyright © 2022 Imagination Technologies Ltd.
3 *
4 * based in part on anv driver which is:
5 * Copyright © 2015 Intel Corporation
6 *
7 * based in part on v3dv_cl.c which is:
8 * Copyright © 2019 Raspberry Pi
9 *
10 * SPDX-License-Identifier: MIT
11 */
12
13 #include "pvr_csb.h"
14
15 #include <assert.h>
16 #include <stdbool.h>
17 #include <stdint.h>
18 #include <string.h>
19 #include <vulkan/vulkan.h>
20
21 #include "hwdef/rogue_hw_utils.h"
22 #include "pvr_bo.h"
23 #include "pvr_debug.h"
24 #include "pvr_device.h"
25 #include "pvr_device_info.h"
26 #include "pvr_macros.h"
27 #include "pvr_physical_device.h"
28 #include "pvr_types.h"
29 #include "util/list.h"
30 #include "util/u_dynarray.h"
31 #include "vk_log.h"
32
33 /**
34 * \brief Adds VDMCTRL_STREAM_LINK/CDMCTRL_STREAM_LINK dwords into the control
35 * stream pointed by csb object without setting a relocation mark.
36 *
37 * \warning This does not set the relocation mark.
38 *
39 * \param[in] csb Control Stream Builder object to add LINK dwords to.
40 * \param[in] addr Device virtual address of the sub control stream to link to.
41 * \param[in] ret Selects whether the sub control stream will return or
42 * terminate.
43 */
44 static void
45 pvr_csb_emit_link_unmarked(struct pvr_csb *csb, pvr_dev_addr_t addr, bool ret)
46 {
47 /* Not supported for deferred control stream. */
48 assert(csb->stream_type != PVR_CMD_STREAM_TYPE_GRAPHICS_DEFERRED);
49
50 /* Stream return is only supported for graphics control stream. */
51 assert(!ret || csb->stream_type == PVR_CMD_STREAM_TYPE_GRAPHICS);
52
53 switch (csb->stream_type) {
54 case PVR_CMD_STREAM_TYPE_GRAPHICS:
55 pvr_csb_emit (csb, VDMCTRL_STREAM_LINK0, link) {
56 link.link_addrmsb = addr;
57 link.with_return = ret;
58 }
59
60 pvr_csb_emit (csb, VDMCTRL_STREAM_LINK1, link) {
61 link.link_addrlsb = addr;
62 }
63
64 break;
65
66 case PVR_CMD_STREAM_TYPE_COMPUTE:
67 pvr_csb_emit (csb, CDMCTRL_STREAM_LINK0, link) {
68 link.link_addrmsb = addr;
69 }
70
71 pvr_csb_emit (csb, CDMCTRL_STREAM_LINK1, link) {
72 link.link_addrlsb = addr;
73 }
74
75 break;
76
77 default:
78 UNREACHABLE("Unknown stream type");
79 break;
80 }
81 }
82
83 /**
84 * \brief Helper function to extend csb memory.
85 *
86 * Allocates a new buffer object and links it with the previous buffer object
87 * using STREAM_LINK dwords and updates csb object to use the new buffer.
88 *
89 * To make sure that we have enough space to emit STREAM_LINK dwords in the
90 * current buffer, a few bytes including guard padding size are reserved at the
91 * end, every time a buffer is created. Every time we allocate a new buffer we
92 * fix the current buffer in use to emit the stream link dwords. This makes sure
93 * that when #pvr_arch_csb_alloc_dwords() is called from #pvr_csb_emit() to add
94 * STREAM_LINK0 and STREAM_LINK1, it succeeds without trying to allocate new
95 * pages.
96 *
97 * \param[in] csb Control Stream Builder object to extend.
98 * \return true on success and false otherwise.
99 */
100 static bool pvr_csb_buffer_extend(struct pvr_csb *csb)
101 {
102 const uint8_t stream_link_space =
103 PVR_DW_TO_BYTES(pvr_cmd_length(VDMCTRL_STREAM_LINK0) +
104 pvr_cmd_length(VDMCTRL_STREAM_LINK1));
105 const uint8_t stream_reserved_space =
106 stream_link_space + ROGUE_VDMCTRL_GUARD_SIZE_DEFAULT;
107 const uint32_t cache_line_size =
108 pvr_get_slc_cache_line_size(&csb->device->pdevice->dev_info);
109 size_t current_state_update_size = 0;
110 struct pvr_bo *pvr_bo;
111 VkResult result;
112
113 /* Make sure extra space allocated for stream links is sufficient for both
114 * stream types.
115 */
116 STATIC_ASSERT((pvr_cmd_length(VDMCTRL_STREAM_LINK0) +
117 pvr_cmd_length(VDMCTRL_STREAM_LINK1)) ==
118 (pvr_cmd_length(CDMCTRL_STREAM_LINK0) +
119 pvr_cmd_length(CDMCTRL_STREAM_LINK1)));
120
121 STATIC_ASSERT(ROGUE_VDMCTRL_GUARD_SIZE_DEFAULT ==
122 ROGUE_CDMCTRL_GUARD_SIZE_DEFAULT);
123
124 result = pvr_bo_alloc(csb->device,
125 csb->device->heaps.general_heap,
126 PVR_CMD_BUFFER_CSB_BO_SIZE,
127 cache_line_size,
128 PVR_BO_ALLOC_FLAG_CPU_MAPPED,
129 &pvr_bo);
130 if (result != VK_SUCCESS) {
131 vk_error(csb->device, result);
132 csb->status = result;
133 return false;
134 }
135
136 /* if this is not the first BO in csb */
137 if (csb->pvr_bo) {
138 bool zero_after_move = PVR_IS_DEBUG_SET(DUMP_CONTROL_STREAM);
139 void *new_buffer = pvr_bo->bo->map;
140
141 current_state_update_size =
142 (uint8_t *)csb->next - (uint8_t *)csb->relocation_mark;
143
144 assert(csb->relocation_mark != NULL);
145 assert(csb->next >= csb->relocation_mark);
146
147 memcpy(new_buffer, csb->relocation_mark, current_state_update_size);
148
149 #if MESA_DEBUG
150 assert(csb->relocation_mark_status == PVR_CSB_RELOCATION_MARK_SET);
151 csb->relocation_mark_status = PVR_CSB_RELOCATION_MARK_SET_AND_CONSUMED;
152 zero_after_move = true;
153 #endif
154
155 if (zero_after_move)
156 memset(csb->relocation_mark, 0, current_state_update_size);
157
158 csb->next = csb->relocation_mark;
159
160 csb->end = (uint8_t *)csb->end + stream_link_space;
161 assert((uint8_t *)csb->next + stream_link_space <= (uint8_t *)csb->end);
162
163 pvr_csb_emit_link_unmarked(csb, pvr_bo->vma->dev_addr, false);
164 }
165
166 csb->pvr_bo = pvr_bo;
167 csb->start = pvr_bo->bo->map;
168
169 /* Reserve space at the end, including the default guard padding, to make
170 * sure we don't run out of space when a stream link is required.
171 */
172 csb->end = (uint8_t *)csb->start + pvr_bo->bo->size - stream_reserved_space;
173 csb->next = (uint8_t *)csb->start + current_state_update_size;
174
175 list_addtail(&pvr_bo->link, &csb->pvr_bo_list);
176
177 return true;
178 }
179
180 /**
181 * \brief Provides a chunk of memory from the current csb buffer. In cases where
182 * the buffer is not able to fulfill the required amount of memory,
183 * #pvr_csb_buffer_extend() is called to allocate a new buffer. Maximum size
184 * allocable in bytes is #PVR_CMD_BUFFER_CSB_BO_SIZE - size of STREAM_LINK0
185 * and STREAM_LINK1 dwords.
186 *
187 * \param[in] csb Control Stream Builder object to allocate from.
188 * \param[in] num_dwords Number of dwords to allocate.
189 * \return Valid host virtual address or NULL otherwise.
190 */
191 void *pvr_arch_csb_alloc_dwords(struct pvr_csb *csb, uint32_t num_dwords)
192 {
193 const uint32_t required_space = PVR_DW_TO_BYTES(num_dwords);
194 void *p;
195
196 if (csb->status != VK_SUCCESS)
197 return NULL;
198
199 if (csb->stream_type == PVR_CMD_STREAM_TYPE_GRAPHICS_DEFERRED) {
200 p = util_dynarray_grow_bytes(&csb->deferred_cs_mem, 1, required_space);
201 if (!p)
202 csb->status = vk_error(csb->device, VK_ERROR_OUT_OF_HOST_MEMORY);
203
204 return p;
205 }
206
207 #if MESA_DEBUG
208 if (csb->relocation_mark_status == PVR_CSB_RELOCATION_MARK_CLEARED)
209 mesa_logd_once("CS memory without relocation mark detected.");
210 #endif
211
212 if ((uint8_t *)csb->next + required_space > (uint8_t *)csb->end) {
213 bool ret = pvr_csb_buffer_extend(csb);
214 if (!ret)
215 return NULL;
216 }
217
218 p = csb->next;
219
220 csb->next = (uint8_t *)csb->next + required_space;
221 assert(csb->next <= csb->end);
222
223 return p;
224 }
225
226 /**
227 * \brief Copies control stream words from src csb into dst csb.
228 *
229 * The intended use is to copy PVR_CMD_STREAM_TYPE_GRAPHICS_DEFERRED type
230 * control stream into PVR_CMD_STREAM_TYPE_GRAPHICS type device accessible
231 * control stream for processing.
232 *
233 * This is mainly for secondary command buffers created with
234 * VK_COMMAND_BUFFER_USAGE_SIMULTANEOUS_USE_BIT flag. In that case we need to
235 * copy secondary control stream into the primary control stream for processing.
236 * This is done as part of vkCmdExecuteCommands.
237 *
238 * We create deferred control stream which is basically the same control stream
239 * but based in host side memory to avoid reserving device side resource.
240 *
241 * \param[in,out] csb_dst Destination control Stream Builder object.
242 * \param[in] csb_src Source Control Stream Builder object.
243 */
244 VkResult pvr_arch_csb_copy(struct pvr_csb *csb_dst, struct pvr_csb *csb_src)
245 {
246 const uint8_t stream_reserved_space =
247 PVR_DW_TO_BYTES(pvr_cmd_length(VDMCTRL_STREAM_LINK0) +
248 pvr_cmd_length(VDMCTRL_STREAM_LINK1)) +
249 ROGUE_VDMCTRL_GUARD_SIZE_DEFAULT;
250 const uint32_t size =
251 util_dynarray_num_elements(&csb_src->deferred_cs_mem, char);
252 const uint8_t *start = util_dynarray_begin(&csb_src->deferred_cs_mem);
253 void *destination;
254
255 /* Only deferred control stream supported as src. */
256 assert(csb_src->stream_type == PVR_CMD_STREAM_TYPE_GRAPHICS_DEFERRED);
257
258 /* Only graphics control stream supported as dst. */
259 assert(csb_dst->stream_type == PVR_CMD_STREAM_TYPE_GRAPHICS);
260
261 if (size >= (PVR_CMD_BUFFER_CSB_BO_SIZE - stream_reserved_space)) {
262 /* TODO: For now we don't support deferred streams bigger than one csb
263 * buffer object size.
264 *
265 * While adding support for this make sure to not break the words/dwords
266 * over two csb buffers.
267 */
268 pvr_finishme("Add support to copy streams bigger than one csb buffer");
269 assert(!"CSB source buffer too large to do a full copy");
270 }
271
272 destination = pvr_arch_csb_alloc_dwords(csb_dst, size);
273 if (!destination) {
274 assert(csb_dst->status != VK_SUCCESS);
275 return csb_dst->status;
276 }
277
278 memcpy(destination, start, size);
279
280 return VK_SUCCESS;
281 }
282
283 /**
284 * \brief Adds VDMCTRL_STREAM_LINK/CDMCTRL_STREAM_LINK dwords into the control
285 * stream pointed by csb object.
286 *
287 * \param[in] csb Control Stream Builder object to add LINK dwords to.
288 * \param[in] addr Device virtual address of the sub control stream to link to.
289 * \param[in] ret Selects whether the sub control stream will return or
290 * terminate.
291 */
292 void pvr_arch_csb_emit_link(struct pvr_csb *csb, pvr_dev_addr_t addr, bool ret)
293 {
294 pvr_csb_set_relocation_mark(csb);
295 pvr_csb_emit_link_unmarked(csb, addr, ret);
296 pvr_csb_clear_relocation_mark(csb);
297 }
298
299 /**
300 * \brief Adds VDMCTRL_STREAM_RETURN dword into the control stream pointed by
301 * csb object. Given a VDMCTRL_STREAM_RETURN marks the end of the sub control
302 * stream, we return the status of the control stream as well.
303 *
304 * \param[in] csb Control Stream Builder object to add VDMCTRL_STREAM_RETURN to.
305 * \return VK_SUCCESS on success, or error code otherwise.
306 */
307 VkResult pvr_arch_csb_emit_return(struct pvr_csb *csb)
308 {
309 /* STREAM_RETURN is only supported by graphics control stream. */
310 assert(csb->stream_type == PVR_CMD_STREAM_TYPE_GRAPHICS ||
311 csb->stream_type == PVR_CMD_STREAM_TYPE_GRAPHICS_DEFERRED);
312
313 pvr_csb_set_relocation_mark(csb);
314 /* clang-format off */
315 pvr_csb_emit(csb, VDMCTRL_STREAM_RETURN, ret);
316 /* clang-format on */
317 pvr_csb_clear_relocation_mark(csb);
318
319 return csb->status;
320 }
321
322 /**
323 * \brief Adds STREAM_TERMINATE dword into the control stream pointed by csb
324 * object. Given a STREAM_TERMINATE marks the end of the control stream, we
325 * return the status of the control stream as well.
326 *
327 * \param[in] csb Control Stream Builder object to terminate.
328 * \return VK_SUCCESS on success, or error code otherwise.
329 */
330 VkResult pvr_arch_csb_emit_terminate(struct pvr_csb *csb)
331 {
332 pvr_csb_set_relocation_mark(csb);
333
334 switch (csb->stream_type) {
335 case PVR_CMD_STREAM_TYPE_GRAPHICS:
336 /* clang-format off */
337 pvr_csb_emit(csb, VDMCTRL_STREAM_TERMINATE, terminate);
338 /* clang-format on */
339 break;
340
341 case PVR_CMD_STREAM_TYPE_COMPUTE:
342 /* clang-format off */
343 pvr_csb_emit(csb, CDMCTRL_STREAM_TERMINATE, terminate);
344 /* clang-format on */
345 break;
346
347 default:
348 UNREACHABLE("Unknown stream type");
349 break;
350 }
351
352 pvr_csb_clear_relocation_mark(csb);
353
354 return csb->status;
355 }