| 1 |
/* |
| 2 |
* Copyright © 2022 Imagination Technologies Ltd. |
| 3 |
* |
| 4 |
* based in part on anv driver which is: |
| 5 |
* Copyright © 2015 Intel Corporation |
| 6 |
* |
| 7 |
* based in part on v3dv_cl.c which is: |
| 8 |
* Copyright © 2019 Raspberry Pi |
| 9 |
* |
| 10 |
* SPDX-License-Identifier: MIT |
| 11 |
*/ |
| 12 |
|
| 13 |
#include "pvr_csb.h" |
| 14 |
|
| 15 |
#include <assert.h> |
| 16 |
#include <stdbool.h> |
| 17 |
#include <stdint.h> |
| 18 |
#include <string.h> |
| 19 |
#include <vulkan/vulkan.h> |
| 20 |
|
| 21 |
#include "hwdef/rogue_hw_utils.h" |
| 22 |
#include "pvr_bo.h" |
| 23 |
#include "pvr_debug.h" |
| 24 |
#include "pvr_device.h" |
| 25 |
#include "pvr_device_info.h" |
| 26 |
#include "pvr_macros.h" |
| 27 |
#include "pvr_physical_device.h" |
| 28 |
#include "pvr_types.h" |
| 29 |
#include "util/list.h" |
| 30 |
#include "util/u_dynarray.h" |
| 31 |
#include "vk_log.h" |
| 32 |
|
| 33 |
/** |
| 34 |
* \brief Adds VDMCTRL_STREAM_LINK/CDMCTRL_STREAM_LINK dwords into the control |
| 35 |
* stream pointed by csb object without setting a relocation mark. |
| 36 |
* |
| 37 |
* \warning This does not set the relocation mark. |
| 38 |
* |
| 39 |
* \param[in] csb Control Stream Builder object to add LINK dwords to. |
| 40 |
* \param[in] addr Device virtual address of the sub control stream to link to. |
| 41 |
* \param[in] ret Selects whether the sub control stream will return or |
| 42 |
* terminate. |
| 43 |
*/ |
| 44 |
static void |
| 45 |
pvr_csb_emit_link_unmarked(struct pvr_csb *csb, pvr_dev_addr_t addr, bool ret) |
| 46 |
{ |
| 47 |
/* Not supported for deferred control stream. */ |
| 48 |
assert(csb->stream_type != PVR_CMD_STREAM_TYPE_GRAPHICS_DEFERRED); |
| 49 |
|
| 50 |
/* Stream return is only supported for graphics control stream. */ |
| 51 |
assert(!ret || csb->stream_type == PVR_CMD_STREAM_TYPE_GRAPHICS); |
| 52 |
|
| 53 |
switch (csb->stream_type) { |
| 54 |
case PVR_CMD_STREAM_TYPE_GRAPHICS: |
| 55 |
pvr_csb_emit (csb, VDMCTRL_STREAM_LINK0, link) { |
| 56 |
link.link_addrmsb = addr; |
| 57 |
link.with_return = ret; |
| 58 |
} |
| 59 |
|
| 60 |
pvr_csb_emit (csb, VDMCTRL_STREAM_LINK1, link) { |
| 61 |
link.link_addrlsb = addr; |
| 62 |
} |
| 63 |
|
| 64 |
break; |
| 65 |
|
| 66 |
case PVR_CMD_STREAM_TYPE_COMPUTE: |
| 67 |
pvr_csb_emit (csb, CDMCTRL_STREAM_LINK0, link) { |
| 68 |
link.link_addrmsb = addr; |
| 69 |
} |
| 70 |
|
| 71 |
pvr_csb_emit (csb, CDMCTRL_STREAM_LINK1, link) { |
| 72 |
link.link_addrlsb = addr; |
| 73 |
} |
| 74 |
|
| 75 |
break; |
| 76 |
|
| 77 |
default: |
| 78 |
UNREACHABLE("Unknown stream type"); |
| 79 |
break; |
| 80 |
} |
| 81 |
} |
| 82 |
|
| 83 |
/** |
| 84 |
* \brief Helper function to extend csb memory. |
| 85 |
* |
| 86 |
* Allocates a new buffer object and links it with the previous buffer object |
| 87 |
* using STREAM_LINK dwords and updates csb object to use the new buffer. |
| 88 |
* |
| 89 |
* To make sure that we have enough space to emit STREAM_LINK dwords in the |
| 90 |
* current buffer, a few bytes including guard padding size are reserved at the |
| 91 |
* end, every time a buffer is created. Every time we allocate a new buffer we |
| 92 |
* fix the current buffer in use to emit the stream link dwords. This makes sure |
| 93 |
* that when #pvr_arch_csb_alloc_dwords() is called from #pvr_csb_emit() to add |
| 94 |
* STREAM_LINK0 and STREAM_LINK1, it succeeds without trying to allocate new |
| 95 |
* pages. |
| 96 |
* |
| 97 |
* \param[in] csb Control Stream Builder object to extend. |
| 98 |
* \return true on success and false otherwise. |
| 99 |
*/ |
| 100 |
static bool pvr_csb_buffer_extend(struct pvr_csb *csb) |
| 101 |
{ |
| 102 |
const uint8_t stream_link_space = |
| 103 |
PVR_DW_TO_BYTES(pvr_cmd_length(VDMCTRL_STREAM_LINK0) + |
| 104 |
pvr_cmd_length(VDMCTRL_STREAM_LINK1)); |
| 105 |
const uint8_t stream_reserved_space = |
| 106 |
stream_link_space + ROGUE_VDMCTRL_GUARD_SIZE_DEFAULT; |
| 107 |
const uint32_t cache_line_size = |
| 108 |
pvr_get_slc_cache_line_size(&csb->device->pdevice->dev_info); |
| 109 |
size_t current_state_update_size = 0; |
| 110 |
struct pvr_bo *pvr_bo; |
| 111 |
VkResult result; |
| 112 |
|
| 113 |
/* Make sure extra space allocated for stream links is sufficient for both |
| 114 |
* stream types. |
| 115 |
*/ |
| 116 |
STATIC_ASSERT((pvr_cmd_length(VDMCTRL_STREAM_LINK0) + |
| 117 |
pvr_cmd_length(VDMCTRL_STREAM_LINK1)) == |
| 118 |
(pvr_cmd_length(CDMCTRL_STREAM_LINK0) + |
| 119 |
pvr_cmd_length(CDMCTRL_STREAM_LINK1))); |
| 120 |
|
| 121 |
STATIC_ASSERT(ROGUE_VDMCTRL_GUARD_SIZE_DEFAULT == |
| 122 |
ROGUE_CDMCTRL_GUARD_SIZE_DEFAULT); |
| 123 |
|
| 124 |
result = pvr_bo_alloc(csb->device, |
| 125 |
csb->device->heaps.general_heap, |
| 126 |
PVR_CMD_BUFFER_CSB_BO_SIZE, |
| 127 |
cache_line_size, |
| 128 |
PVR_BO_ALLOC_FLAG_CPU_MAPPED, |
| 129 |
&pvr_bo); |
| 130 |
if (result != VK_SUCCESS) { |
| 131 |
vk_error(csb->device, result); |
| 132 |
csb->status = result; |
| 133 |
return false; |
| 134 |
} |
| 135 |
|
| 136 |
/* if this is not the first BO in csb */ |
| 137 |
if (csb->pvr_bo) { |
| 138 |
bool zero_after_move = PVR_IS_DEBUG_SET(DUMP_CONTROL_STREAM); |
| 139 |
void *new_buffer = pvr_bo->bo->map; |
| 140 |
|
| 141 |
current_state_update_size = |
| 142 |
(uint8_t *)csb->next - (uint8_t *)csb->relocation_mark; |
| 143 |
|
| 144 |
assert(csb->relocation_mark != NULL); |
| 145 |
assert(csb->next >= csb->relocation_mark); |
| 146 |
|
| 147 |
memcpy(new_buffer, csb->relocation_mark, current_state_update_size); |
| 148 |
|
| 149 |
#if MESA_DEBUG |
| 150 |
assert(csb->relocation_mark_status == PVR_CSB_RELOCATION_MARK_SET); |
| 151 |
csb->relocation_mark_status = PVR_CSB_RELOCATION_MARK_SET_AND_CONSUMED; |
| 152 |
zero_after_move = true; |
| 153 |
#endif |
| 154 |
|
| 155 |
if (zero_after_move) |
| 156 |
memset(csb->relocation_mark, 0, current_state_update_size); |
| 157 |
|
| 158 |
csb->next = csb->relocation_mark; |
| 159 |
|
| 160 |
csb->end = (uint8_t *)csb->end + stream_link_space; |
| 161 |
assert((uint8_t *)csb->next + stream_link_space <= (uint8_t *)csb->end); |
| 162 |
|
| 163 |
pvr_csb_emit_link_unmarked(csb, pvr_bo->vma->dev_addr, false); |
| 164 |
} |
| 165 |
|
| 166 |
csb->pvr_bo = pvr_bo; |
| 167 |
csb->start = pvr_bo->bo->map; |
| 168 |
|
| 169 |
/* Reserve space at the end, including the default guard padding, to make |
| 170 |
* sure we don't run out of space when a stream link is required. |
| 171 |
*/ |
| 172 |
csb->end = (uint8_t *)csb->start + pvr_bo->bo->size - stream_reserved_space; |
| 173 |
csb->next = (uint8_t *)csb->start + current_state_update_size; |
| 174 |
|
| 175 |
list_addtail(&pvr_bo->link, &csb->pvr_bo_list); |
| 176 |
|
| 177 |
return true; |
| 178 |
} |
| 179 |
|
| 180 |
/** |
| 181 |
* \brief Provides a chunk of memory from the current csb buffer. In cases where |
| 182 |
* the buffer is not able to fulfill the required amount of memory, |
| 183 |
* #pvr_csb_buffer_extend() is called to allocate a new buffer. Maximum size |
| 184 |
* allocable in bytes is #PVR_CMD_BUFFER_CSB_BO_SIZE - size of STREAM_LINK0 |
| 185 |
* and STREAM_LINK1 dwords. |
| 186 |
* |
| 187 |
* \param[in] csb Control Stream Builder object to allocate from. |
| 188 |
* \param[in] num_dwords Number of dwords to allocate. |
| 189 |
* \return Valid host virtual address or NULL otherwise. |
| 190 |
*/ |
| 191 |
void *pvr_arch_csb_alloc_dwords(struct pvr_csb *csb, uint32_t num_dwords) |
| 192 |
{ |
| 193 |
const uint32_t required_space = PVR_DW_TO_BYTES(num_dwords); |
| 194 |
void *p; |
| 195 |
|
| 196 |
if (csb->status != VK_SUCCESS) |
| 197 |
return NULL; |
| 198 |
|
| 199 |
if (csb->stream_type == PVR_CMD_STREAM_TYPE_GRAPHICS_DEFERRED) { |
| 200 |
p = util_dynarray_grow_bytes(&csb->deferred_cs_mem, 1, required_space); |
| 201 |
if (!p) |
| 202 |
csb->status = vk_error(csb->device, VK_ERROR_OUT_OF_HOST_MEMORY); |
| 203 |
|
| 204 |
return p; |
| 205 |
} |
| 206 |
|
| 207 |
#if MESA_DEBUG |
| 208 |
if (csb->relocation_mark_status == PVR_CSB_RELOCATION_MARK_CLEARED) |
| 209 |
mesa_logd_once("CS memory without relocation mark detected."); |
| 210 |
#endif |
| 211 |
|
| 212 |
if ((uint8_t *)csb->next + required_space > (uint8_t *)csb->end) { |
| 213 |
bool ret = pvr_csb_buffer_extend(csb); |
| 214 |
if (!ret) |
| 215 |
return NULL; |
| 216 |
} |
| 217 |
|
| 218 |
p = csb->next; |
| 219 |
|
| 220 |
csb->next = (uint8_t *)csb->next + required_space; |
| 221 |
assert(csb->next <= csb->end); |
| 222 |
|
| 223 |
return p; |
| 224 |
} |
| 225 |
|
| 226 |
/** |
| 227 |
* \brief Copies control stream words from src csb into dst csb. |
| 228 |
* |
| 229 |
* The intended use is to copy PVR_CMD_STREAM_TYPE_GRAPHICS_DEFERRED type |
| 230 |
* control stream into PVR_CMD_STREAM_TYPE_GRAPHICS type device accessible |
| 231 |
* control stream for processing. |
| 232 |
* |
| 233 |
* This is mainly for secondary command buffers created with |
| 234 |
* VK_COMMAND_BUFFER_USAGE_SIMULTANEOUS_USE_BIT flag. In that case we need to |
| 235 |
* copy secondary control stream into the primary control stream for processing. |
| 236 |
* This is done as part of vkCmdExecuteCommands. |
| 237 |
* |
| 238 |
* We create deferred control stream which is basically the same control stream |
| 239 |
* but based in host side memory to avoid reserving device side resource. |
| 240 |
* |
| 241 |
* \param[in,out] csb_dst Destination control Stream Builder object. |
| 242 |
* \param[in] csb_src Source Control Stream Builder object. |
| 243 |
*/ |
| 244 |
VkResult pvr_arch_csb_copy(struct pvr_csb *csb_dst, struct pvr_csb *csb_src) |
| 245 |
{ |
| 246 |
const uint8_t stream_reserved_space = |
| 247 |
PVR_DW_TO_BYTES(pvr_cmd_length(VDMCTRL_STREAM_LINK0) + |
| 248 |
pvr_cmd_length(VDMCTRL_STREAM_LINK1)) + |
| 249 |
ROGUE_VDMCTRL_GUARD_SIZE_DEFAULT; |
| 250 |
const uint32_t size = |
| 251 |
util_dynarray_num_elements(&csb_src->deferred_cs_mem, char); |
| 252 |
const uint8_t *start = util_dynarray_begin(&csb_src->deferred_cs_mem); |
| 253 |
void *destination; |
| 254 |
|
| 255 |
/* Only deferred control stream supported as src. */ |
| 256 |
assert(csb_src->stream_type == PVR_CMD_STREAM_TYPE_GRAPHICS_DEFERRED); |
| 257 |
|
| 258 |
/* Only graphics control stream supported as dst. */ |
| 259 |
assert(csb_dst->stream_type == PVR_CMD_STREAM_TYPE_GRAPHICS); |
| 260 |
|
| 261 |
if (size >= (PVR_CMD_BUFFER_CSB_BO_SIZE - stream_reserved_space)) { |
| 262 |
/* TODO: For now we don't support deferred streams bigger than one csb |
| 263 |
* buffer object size. |
| 264 |
* |
| 265 |
* While adding support for this make sure to not break the words/dwords |
| 266 |
* over two csb buffers. |
| 267 |
*/ |
| 268 |
pvr_finishme("Add support to copy streams bigger than one csb buffer"); |
| 269 |
assert(!"CSB source buffer too large to do a full copy"); |
| 270 |
} |
| 271 |
|
| 272 |
destination = pvr_arch_csb_alloc_dwords(csb_dst, size); |
| 273 |
if (!destination) { |
| 274 |
assert(csb_dst->status != VK_SUCCESS); |
| 275 |
return csb_dst->status; |
| 276 |
} |
| 277 |
|
| 278 |
memcpy(destination, start, size); |
| 279 |
|
| 280 |
return VK_SUCCESS; |
| 281 |
} |
| 282 |
|
| 283 |
/** |
| 284 |
* \brief Adds VDMCTRL_STREAM_LINK/CDMCTRL_STREAM_LINK dwords into the control |
| 285 |
* stream pointed by csb object. |
| 286 |
* |
| 287 |
* \param[in] csb Control Stream Builder object to add LINK dwords to. |
| 288 |
* \param[in] addr Device virtual address of the sub control stream to link to. |
| 289 |
* \param[in] ret Selects whether the sub control stream will return or |
| 290 |
* terminate. |
| 291 |
*/ |
| 292 |
void pvr_arch_csb_emit_link(struct pvr_csb *csb, pvr_dev_addr_t addr, bool ret) |
| 293 |
{ |
| 294 |
pvr_csb_set_relocation_mark(csb); |
| 295 |
pvr_csb_emit_link_unmarked(csb, addr, ret); |
| 296 |
pvr_csb_clear_relocation_mark(csb); |
| 297 |
} |
| 298 |
|
| 299 |
/** |
| 300 |
* \brief Adds VDMCTRL_STREAM_RETURN dword into the control stream pointed by |
| 301 |
* csb object. Given a VDMCTRL_STREAM_RETURN marks the end of the sub control |
| 302 |
* stream, we return the status of the control stream as well. |
| 303 |
* |
| 304 |
* \param[in] csb Control Stream Builder object to add VDMCTRL_STREAM_RETURN to. |
| 305 |
* \return VK_SUCCESS on success, or error code otherwise. |
| 306 |
*/ |
| 307 |
VkResult pvr_arch_csb_emit_return(struct pvr_csb *csb) |
| 308 |
{ |
| 309 |
/* STREAM_RETURN is only supported by graphics control stream. */ |
| 310 |
assert(csb->stream_type == PVR_CMD_STREAM_TYPE_GRAPHICS || |
| 311 |
csb->stream_type == PVR_CMD_STREAM_TYPE_GRAPHICS_DEFERRED); |
| 312 |
|
| 313 |
pvr_csb_set_relocation_mark(csb); |
| 314 |
/* clang-format off */ |
| 315 |
pvr_csb_emit(csb, VDMCTRL_STREAM_RETURN, ret); |
| 316 |
/* clang-format on */ |
| 317 |
pvr_csb_clear_relocation_mark(csb); |
| 318 |
|
| 319 |
return csb->status; |
| 320 |
} |
| 321 |
|
| 322 |
/** |
| 323 |
* \brief Adds STREAM_TERMINATE dword into the control stream pointed by csb |
| 324 |
* object. Given a STREAM_TERMINATE marks the end of the control stream, we |
| 325 |
* return the status of the control stream as well. |
| 326 |
* |
| 327 |
* \param[in] csb Control Stream Builder object to terminate. |
| 328 |
* \return VK_SUCCESS on success, or error code otherwise. |
| 329 |
*/ |
| 330 |
VkResult pvr_arch_csb_emit_terminate(struct pvr_csb *csb) |
| 331 |
{ |
| 332 |
pvr_csb_set_relocation_mark(csb); |
| 333 |
|
| 334 |
switch (csb->stream_type) { |
| 335 |
case PVR_CMD_STREAM_TYPE_GRAPHICS: |
| 336 |
/* clang-format off */ |
| 337 |
pvr_csb_emit(csb, VDMCTRL_STREAM_TERMINATE, terminate); |
| 338 |
/* clang-format on */ |
| 339 |
break; |
| 340 |
|
| 341 |
case PVR_CMD_STREAM_TYPE_COMPUTE: |
| 342 |
/* clang-format off */ |
| 343 |
pvr_csb_emit(csb, CDMCTRL_STREAM_TERMINATE, terminate); |
| 344 |
/* clang-format on */ |
| 345 |
break; |
| 346 |
|
| 347 |
default: |
| 348 |
UNREACHABLE("Unknown stream type"); |
| 349 |
break; |
| 350 |
} |
| 351 |
|
| 352 |
pvr_csb_clear_relocation_mark(csb); |
| 353 |
|
| 354 |
return csb->status; |
| 355 |
} |