| 1 |
/* |
| 2 |
* Copyright © 2022 Imagination Technologies Ltd. |
| 3 |
* |
| 4 |
* Permission is hereby granted, free of charge, to any person obtaining a copy |
| 5 |
* of this software and associated documentation files (the "Software"), to deal |
| 6 |
* in the Software without restriction, including without limitation the rights |
| 7 |
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
| 8 |
* copies of the Software, and to permit persons to whom the Software is |
| 9 |
* furnished to do so, subject to the following conditions: |
| 10 |
* |
| 11 |
* The above copyright notice and this permission notice (including the next |
| 12 |
* paragraph) shall be included in all copies or substantial portions of the |
| 13 |
* Software. |
| 14 |
* |
| 15 |
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| 16 |
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| 17 |
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
| 18 |
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| 19 |
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
| 20 |
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| 21 |
* SOFTWARE. |
| 22 |
*/ |
| 23 |
|
| 24 |
#include <assert.h> |
| 25 |
#include <stdbool.h> |
| 26 |
#include <stdint.h> |
| 27 |
#include <xf86drm.h> |
| 28 |
|
| 29 |
#include "hwdef/rogue_hw_utils.h" |
| 30 |
#include "pvr_csb.h" |
| 31 |
#include "pvr_device.h" |
| 32 |
#include "pvr_device_info.h" |
| 33 |
#include "pvr_physical_device.h" |
| 34 |
#include "pvr_srv.h" |
| 35 |
#include "pvr_srv_bo.h" |
| 36 |
#include "pvr_srv_bridge.h" |
| 37 |
#include "pvr_srv_job_common.h" |
| 38 |
#include "pvr_srv_job_compute.h" |
| 39 |
#include "pvr_srv_job_render.h" |
| 40 |
#include "pvr_srv_job_transfer.h" |
| 41 |
#include "pvr_srv_public.h" |
| 42 |
#include "pvr_srv_sync.h" |
| 43 |
#include "pvr_srv_sync_prim.h" |
| 44 |
#include "pvr_srv_job_null.h" |
| 45 |
#include "pvr_types.h" |
| 46 |
#include "pvr_winsys.h" |
| 47 |
#include "pvr_winsys_helper.h" |
| 48 |
#include "util/log.h" |
| 49 |
#include "util/macros.h" |
| 50 |
#include "util/os_misc.h" |
| 51 |
#include "util/u_atomic.h" |
| 52 |
#include "vk_log.h" |
| 53 |
#include "vk_sync.h" |
| 54 |
#include "vk_sync_timeline.h" |
| 55 |
|
| 56 |
/* carveout_size can be 0 when no carveout is needed. carveout_address must |
| 57 |
* be 0 if carveout_size is 0. |
| 58 |
*/ |
| 59 |
static VkResult pvr_winsys_heap_init( |
| 60 |
struct pvr_winsys *const ws, |
| 61 |
pvr_dev_addr_t base_address, |
| 62 |
uint64_t size, |
| 63 |
pvr_dev_addr_t carveout_address, |
| 64 |
uint64_t carveout_size, |
| 65 |
uint32_t log2_page_size, |
| 66 |
const struct pvr_winsys_static_data_offsets *const static_data_offsets, |
| 67 |
struct pvr_winsys_heap *const heap) |
| 68 |
{ |
| 69 |
const bool carveout_area_bottom_of_heap = carveout_address.addr == |
| 70 |
base_address.addr; |
| 71 |
const pvr_dev_addr_t vma_heap_begin_addr = |
| 72 |
carveout_area_bottom_of_heap |
| 73 |
? PVR_DEV_ADDR_OFFSET(base_address, carveout_size) |
| 74 |
: base_address; |
| 75 |
const uint64_t vma_heap_size = size - carveout_size; |
| 76 |
|
| 77 |
assert(base_address.addr); |
| 78 |
assert(carveout_size <= size); |
| 79 |
|
| 80 |
/* As per the static_data_carveout_base powervr-km uapi documentation the |
| 81 |
* carveout region can only be at the beginning of the heap or at the end. |
| 82 |
* carveout_address is 0 if there is no carveout region. |
| 83 |
* pvrsrv-km doesn't explicitly provide this info and it's assumed that it's |
| 84 |
* always at the beginning. |
| 85 |
*/ |
| 86 |
assert(carveout_area_bottom_of_heap || |
| 87 |
carveout_address.addr + carveout_size == base_address.addr + size || |
| 88 |
(!carveout_address.addr && !carveout_size)); |
| 89 |
|
| 90 |
heap->ws = ws; |
| 91 |
heap->base_addr = base_address; |
| 92 |
heap->static_data_carveout_addr = carveout_address; |
| 93 |
|
| 94 |
heap->size = size; |
| 95 |
heap->static_data_carveout_size = carveout_size; |
| 96 |
|
| 97 |
heap->page_size = 1 << log2_page_size; |
| 98 |
heap->log2_page_size = log2_page_size; |
| 99 |
|
| 100 |
util_vma_heap_init(&heap->vma_heap, vma_heap_begin_addr.addr, vma_heap_size); |
| 101 |
|
| 102 |
heap->vma_heap.alloc_high = false; |
| 103 |
|
| 104 |
/* It's expected that the heap destroy function to be the last thing that's |
| 105 |
* called, so we start the ref_count at 0. |
| 106 |
*/ |
| 107 |
p_atomic_set(&heap->ref_count, 0); |
| 108 |
|
| 109 |
if (pthread_mutex_init(&heap->lock, NULL)) |
| 110 |
return vk_error(NULL, VK_ERROR_INITIALIZATION_FAILED); |
| 111 |
|
| 112 |
heap->static_data_offsets = *static_data_offsets; |
| 113 |
|
| 114 |
return VK_SUCCESS; |
| 115 |
} |
| 116 |
|
| 117 |
/** |
| 118 |
* Maximum PB free list size supported by RGX and Services. |
| 119 |
* |
| 120 |
* Maximum PB free list size must ensure that no PM address space can be fully |
| 121 |
* used, because if the full address space was used it would wrap and corrupt |
| 122 |
* itself. Since there are two freelists (local is always minimum sized) this |
| 123 |
* can be described as following three conditions being met: |
| 124 |
* |
| 125 |
* Minimum PB + Maximum PB < ALIST PM address space size (16GB) |
| 126 |
* Minimum PB + Maximum PB < TE PM address space size (16GB) / NUM_TE_PIPES |
| 127 |
* Minimum PB + Maximum PB < VCE PM address space size (16GB) / NUM_VCE_PIPES |
| 128 |
* |
| 129 |
* Since the max of NUM_TE_PIPES and NUM_VCE_PIPES is 4, we have a hard limit |
| 130 |
* of 4GB minus the Minimum PB. For convenience we take the smaller power-of-2 |
| 131 |
* value of 2GB. This is far more than any normal application would request |
| 132 |
* or use. |
| 133 |
*/ |
| 134 |
#define PVR_SRV_FREE_LIST_MAX_SIZE (2ULL * 1024ULL * 1024ULL * 1024ULL) |
| 135 |
|
| 136 |
static VkResult pvr_srv_heap_init( |
| 137 |
struct pvr_srv_winsys *srv_ws, |
| 138 |
struct pvr_srv_winsys_heap *srv_heap, |
| 139 |
uint32_t heap_idx, |
| 140 |
const struct pvr_winsys_static_data_offsets *const static_data_offsets) |
| 141 |
{ |
| 142 |
pvr_dev_addr_t base_address; |
| 143 |
uint32_t log2_page_size; |
| 144 |
uint64_t carveout_size; |
| 145 |
VkResult result; |
| 146 |
uint64_t size; |
| 147 |
|
| 148 |
result = pvr_srv_get_heap_details(srv_ws->base.render_fd, |
| 149 |
heap_idx, |
| 150 |
0, |
| 151 |
NULL, |
| 152 |
&base_address, |
| 153 |
&size, |
| 154 |
&carveout_size, |
| 155 |
&log2_page_size); |
| 156 |
if (result != VK_SUCCESS) |
| 157 |
return result; |
| 158 |
|
| 159 |
result = pvr_winsys_heap_init(&srv_ws->base, |
| 160 |
base_address, |
| 161 |
size, |
| 162 |
base_address, |
| 163 |
carveout_size, |
| 164 |
log2_page_size, |
| 165 |
static_data_offsets, |
| 166 |
&srv_heap->base); |
| 167 |
if (result != VK_SUCCESS) |
| 168 |
return result; |
| 169 |
|
| 170 |
assert(srv_heap->base.page_size == srv_ws->base.page_size); |
| 171 |
assert(srv_heap->base.log2_page_size == srv_ws->base.log2_page_size); |
| 172 |
assert(srv_heap->base.static_data_carveout_size % |
| 173 |
PVR_SRV_CARVEOUT_SIZE_GRANULARITY == |
| 174 |
0); |
| 175 |
|
| 176 |
/* Create server-side counterpart of Device Memory heap */ |
| 177 |
result = pvr_srv_int_heap_create(srv_ws->base.render_fd, |
| 178 |
srv_heap->base.base_addr, |
| 179 |
srv_heap->base.size, |
| 180 |
srv_heap->base.log2_page_size, |
| 181 |
srv_ws->server_memctx, |
| 182 |
&srv_heap->server_heap); |
| 183 |
if (result != VK_SUCCESS) { |
| 184 |
pvr_winsys_helper_winsys_heap_finish(&srv_heap->base); |
| 185 |
return result; |
| 186 |
} |
| 187 |
|
| 188 |
return VK_SUCCESS; |
| 189 |
} |
| 190 |
|
| 191 |
static bool pvr_srv_heap_finish(struct pvr_srv_winsys *srv_ws, |
| 192 |
struct pvr_srv_winsys_heap *srv_heap) |
| 193 |
{ |
| 194 |
if (!pvr_winsys_helper_winsys_heap_finish(&srv_heap->base)) |
| 195 |
return false; |
| 196 |
|
| 197 |
pvr_srv_int_heap_destroy(srv_ws->base.render_fd, srv_heap->server_heap); |
| 198 |
|
| 199 |
return true; |
| 200 |
} |
| 201 |
|
| 202 |
static VkResult pvr_srv_memctx_init(struct pvr_srv_winsys *srv_ws) |
| 203 |
{ |
| 204 |
const struct pvr_winsys_static_data_offsets |
| 205 |
general_heap_static_data_offsets = { |
| 206 |
.yuv_csc = FWIF_GENERAL_HEAP_YUV_CSC_OFFSET_BYTES, |
| 207 |
}; |
| 208 |
const struct pvr_winsys_static_data_offsets pds_heap_static_data_offsets = { |
| 209 |
.eot = FWIF_PDS_HEAP_EOT_OFFSET_BYTES, |
| 210 |
.vdm_sync = FWIF_PDS_HEAP_VDM_SYNC_OFFSET_BYTES, |
| 211 |
}; |
| 212 |
const struct pvr_winsys_static_data_offsets usc_heap_static_data_offsets = { |
| 213 |
.vdm_sync = FWIF_USC_HEAP_VDM_SYNC_OFFSET_BYTES, |
| 214 |
}; |
| 215 |
const struct pvr_winsys_static_data_offsets no_static_data_offsets = { 0 }; |
| 216 |
|
| 217 |
char heap_name[PVR_SRV_DEVMEM_HEAPNAME_MAXLENGTH]; |
| 218 |
int transfer_3d_heap_idx = -1; |
| 219 |
int vis_test_heap_idx = -1; |
| 220 |
int general_heap_idx = -1; |
| 221 |
int rgn_hdr_heap_idx = -1; |
| 222 |
int pds_heap_idx = -1; |
| 223 |
int usc_heap_idx = -1; |
| 224 |
uint32_t heap_count; |
| 225 |
VkResult result; |
| 226 |
|
| 227 |
result = pvr_srv_int_ctx_create(srv_ws->base.render_fd, |
| 228 |
&srv_ws->server_memctx, |
| 229 |
&srv_ws->server_memctx_data); |
| 230 |
if (result != VK_SUCCESS) |
| 231 |
return result; |
| 232 |
|
| 233 |
os_get_page_size(&srv_ws->base.page_size); |
| 234 |
srv_ws->base.log2_page_size = util_logbase2(srv_ws->base.page_size); |
| 235 |
|
| 236 |
result = pvr_srv_get_heap_count(srv_ws->base.render_fd, &heap_count); |
| 237 |
if (result != VK_SUCCESS) |
| 238 |
goto err_pvr_srv_int_ctx_destroy; |
| 239 |
|
| 240 |
assert(heap_count > 0); |
| 241 |
|
| 242 |
for (uint32_t i = 0; i < heap_count; i++) { |
| 243 |
result = pvr_srv_get_heap_details(srv_ws->base.render_fd, |
| 244 |
i, |
| 245 |
sizeof(heap_name), |
| 246 |
heap_name, |
| 247 |
NULL, |
| 248 |
NULL, |
| 249 |
NULL, |
| 250 |
NULL); |
| 251 |
if (result != VK_SUCCESS) |
| 252 |
goto err_pvr_srv_int_ctx_destroy; |
| 253 |
|
| 254 |
if (general_heap_idx == -1 && |
| 255 |
strncmp(heap_name, |
| 256 |
PVR_SRV_GENERAL_HEAP_IDENT, |
| 257 |
sizeof(PVR_SRV_GENERAL_HEAP_IDENT)) == 0) { |
| 258 |
general_heap_idx = i; |
| 259 |
} else if (pds_heap_idx == -1 && |
| 260 |
strncmp(heap_name, |
| 261 |
PVR_SRV_PDSCODEDATA_HEAP_IDENT, |
| 262 |
sizeof(PVR_SRV_PDSCODEDATA_HEAP_IDENT)) == 0) { |
| 263 |
pds_heap_idx = i; |
| 264 |
} else if (rgn_hdr_heap_idx == -1 && |
| 265 |
strncmp(heap_name, |
| 266 |
PVR_SRV_RGNHDR_BRN_63142_HEAP_IDENT, |
| 267 |
sizeof(PVR_SRV_RGNHDR_BRN_63142_HEAP_IDENT)) == 0) { |
| 268 |
rgn_hdr_heap_idx = i; |
| 269 |
} else if (transfer_3d_heap_idx == -1 && |
| 270 |
strncmp(heap_name, |
| 271 |
PVR_SRV_TRANSFER_3D_HEAP_IDENT, |
| 272 |
sizeof(PVR_SRV_TRANSFER_3D_HEAP_IDENT)) == 0) { |
| 273 |
transfer_3d_heap_idx = i; |
| 274 |
} else if (usc_heap_idx == -1 && |
| 275 |
strncmp(heap_name, |
| 276 |
PVR_SRV_USCCODE_HEAP_IDENT, |
| 277 |
sizeof(PVR_SRV_USCCODE_HEAP_IDENT)) == 0) { |
| 278 |
usc_heap_idx = i; |
| 279 |
} else if (vis_test_heap_idx == -1 && |
| 280 |
strncmp(heap_name, |
| 281 |
PVR_SRV_VISIBILITY_TEST_HEAP_IDENT, |
| 282 |
sizeof(PVR_SRV_VISIBILITY_TEST_HEAP_IDENT)) == 0) { |
| 283 |
vis_test_heap_idx = i; |
| 284 |
} |
| 285 |
} |
| 286 |
|
| 287 |
/* Check for and initialise required heaps. */ |
| 288 |
if (general_heap_idx == -1 || pds_heap_idx == -1 || |
| 289 |
transfer_3d_heap_idx == -1 || usc_heap_idx == -1 || |
| 290 |
vis_test_heap_idx == -1) { |
| 291 |
result = vk_error(NULL, VK_ERROR_INITIALIZATION_FAILED); |
| 292 |
goto err_pvr_srv_int_ctx_destroy; |
| 293 |
} |
| 294 |
|
| 295 |
result = pvr_srv_heap_init(srv_ws, |
| 296 |
&srv_ws->general_heap, |
| 297 |
general_heap_idx, |
| 298 |
&general_heap_static_data_offsets); |
| 299 |
if (result != VK_SUCCESS) |
| 300 |
goto err_pvr_srv_int_ctx_destroy; |
| 301 |
|
| 302 |
result = pvr_srv_heap_init(srv_ws, |
| 303 |
&srv_ws->pds_heap, |
| 304 |
pds_heap_idx, |
| 305 |
&pds_heap_static_data_offsets); |
| 306 |
if (result != VK_SUCCESS) |
| 307 |
goto err_pvr_srv_heap_finish_general; |
| 308 |
|
| 309 |
result = pvr_srv_heap_init(srv_ws, |
| 310 |
&srv_ws->transfer_3d_heap, |
| 311 |
transfer_3d_heap_idx, |
| 312 |
&no_static_data_offsets); |
| 313 |
if (result != VK_SUCCESS) |
| 314 |
goto err_pvr_srv_heap_finish_pds; |
| 315 |
|
| 316 |
result = pvr_srv_heap_init(srv_ws, |
| 317 |
&srv_ws->usc_heap, |
| 318 |
usc_heap_idx, |
| 319 |
&usc_heap_static_data_offsets); |
| 320 |
if (result != VK_SUCCESS) |
| 321 |
goto err_pvr_srv_heap_finish_transfer_3d; |
| 322 |
|
| 323 |
result = pvr_srv_heap_init(srv_ws, |
| 324 |
&srv_ws->vis_test_heap, |
| 325 |
vis_test_heap_idx, |
| 326 |
&no_static_data_offsets); |
| 327 |
if (result != VK_SUCCESS) |
| 328 |
goto err_pvr_srv_heap_finish_usc; |
| 329 |
|
| 330 |
/* Check for and set up optional heaps. */ |
| 331 |
if (rgn_hdr_heap_idx != -1) { |
| 332 |
result = pvr_srv_heap_init(srv_ws, |
| 333 |
&srv_ws->rgn_hdr_heap, |
| 334 |
rgn_hdr_heap_idx, |
| 335 |
&no_static_data_offsets); |
| 336 |
if (result != VK_SUCCESS) |
| 337 |
goto err_pvr_srv_heap_finish_vis_test; |
| 338 |
|
| 339 |
srv_ws->rgn_hdr_heap_present = true; |
| 340 |
} else { |
| 341 |
srv_ws->rgn_hdr_heap_present = false; |
| 342 |
} |
| 343 |
|
| 344 |
result = |
| 345 |
pvr_winsys_helper_allocate_static_memory(&srv_ws->base, |
| 346 |
pvr_srv_heap_alloc_carveout, |
| 347 |
&srv_ws->general_heap.base, |
| 348 |
&srv_ws->pds_heap.base, |
| 349 |
&srv_ws->usc_heap.base, |
| 350 |
&srv_ws->general_vma, |
| 351 |
&srv_ws->pds_vma, |
| 352 |
&srv_ws->usc_vma); |
| 353 |
if (result != VK_SUCCESS) |
| 354 |
goto err_pvr_srv_heap_finish_rgn_hdr; |
| 355 |
|
| 356 |
result = pvr_winsys_helper_fill_static_memory(&srv_ws->base, |
| 357 |
srv_ws->general_vma, |
| 358 |
srv_ws->pds_vma, |
| 359 |
srv_ws->usc_vma); |
| 360 |
if (result != VK_SUCCESS) |
| 361 |
goto err_pvr_srv_free_static_memory; |
| 362 |
|
| 363 |
return VK_SUCCESS; |
| 364 |
|
| 365 |
err_pvr_srv_free_static_memory: |
| 366 |
pvr_winsys_helper_free_static_memory(srv_ws->general_vma, |
| 367 |
srv_ws->pds_vma, |
| 368 |
srv_ws->usc_vma); |
| 369 |
|
| 370 |
err_pvr_srv_heap_finish_rgn_hdr: |
| 371 |
if (srv_ws->rgn_hdr_heap_present) |
| 372 |
pvr_srv_heap_finish(srv_ws, &srv_ws->rgn_hdr_heap); |
| 373 |
|
| 374 |
err_pvr_srv_heap_finish_vis_test: |
| 375 |
pvr_srv_heap_finish(srv_ws, &srv_ws->vis_test_heap); |
| 376 |
|
| 377 |
err_pvr_srv_heap_finish_usc: |
| 378 |
pvr_srv_heap_finish(srv_ws, &srv_ws->usc_heap); |
| 379 |
|
| 380 |
err_pvr_srv_heap_finish_transfer_3d: |
| 381 |
pvr_srv_heap_finish(srv_ws, &srv_ws->transfer_3d_heap); |
| 382 |
|
| 383 |
err_pvr_srv_heap_finish_pds: |
| 384 |
pvr_srv_heap_finish(srv_ws, &srv_ws->pds_heap); |
| 385 |
|
| 386 |
err_pvr_srv_heap_finish_general: |
| 387 |
pvr_srv_heap_finish(srv_ws, &srv_ws->general_heap); |
| 388 |
|
| 389 |
err_pvr_srv_int_ctx_destroy: |
| 390 |
pvr_srv_int_ctx_destroy(srv_ws->base.render_fd, srv_ws->server_memctx); |
| 391 |
|
| 392 |
return result; |
| 393 |
} |
| 394 |
|
| 395 |
static void pvr_srv_memctx_finish(struct pvr_srv_winsys *srv_ws) |
| 396 |
{ |
| 397 |
pvr_winsys_helper_free_static_memory(srv_ws->general_vma, |
| 398 |
srv_ws->pds_vma, |
| 399 |
srv_ws->usc_vma); |
| 400 |
|
| 401 |
if (srv_ws->rgn_hdr_heap_present) { |
| 402 |
if (!pvr_srv_heap_finish(srv_ws, &srv_ws->rgn_hdr_heap)) { |
| 403 |
vk_errorf(NULL, |
| 404 |
VK_ERROR_UNKNOWN, |
| 405 |
"Region header heap in use, can not deinit"); |
| 406 |
} |
| 407 |
} |
| 408 |
|
| 409 |
if (!pvr_srv_heap_finish(srv_ws, &srv_ws->vis_test_heap)) { |
| 410 |
vk_errorf(NULL, |
| 411 |
VK_ERROR_UNKNOWN, |
| 412 |
"Visibility test heap in use, can not deinit"); |
| 413 |
} |
| 414 |
|
| 415 |
if (!pvr_srv_heap_finish(srv_ws, &srv_ws->usc_heap)) |
| 416 |
vk_errorf(NULL, VK_ERROR_UNKNOWN, "USC heap in use, can not deinit"); |
| 417 |
|
| 418 |
if (!pvr_srv_heap_finish(srv_ws, &srv_ws->transfer_3d_heap)) { |
| 419 |
vk_errorf(NULL, |
| 420 |
VK_ERROR_UNKNOWN, |
| 421 |
"Transfer 3D heap in use, can not deinit"); |
| 422 |
} |
| 423 |
|
| 424 |
if (!pvr_srv_heap_finish(srv_ws, &srv_ws->pds_heap)) |
| 425 |
vk_errorf(NULL, VK_ERROR_UNKNOWN, "PDS heap in use, can not deinit"); |
| 426 |
|
| 427 |
if (!pvr_srv_heap_finish(srv_ws, &srv_ws->general_heap)) { |
| 428 |
vk_errorf(NULL, VK_ERROR_UNKNOWN, "General heap in use, can not deinit"); |
| 429 |
} |
| 430 |
|
| 431 |
pvr_srv_int_ctx_destroy(srv_ws->base.render_fd, srv_ws->server_memctx); |
| 432 |
} |
| 433 |
|
| 434 |
static void pvr_srv_winsys_destroy(struct pvr_winsys *ws) |
| 435 |
{ |
| 436 |
struct pvr_srv_winsys *srv_ws = to_pvr_srv_winsys(ws); |
| 437 |
int fd = ws->render_fd; |
| 438 |
|
| 439 |
if (srv_ws->presignaled_sync) { |
| 440 |
vk_sync_destroy(&srv_ws->presignaled_sync_device->vk, |
| 441 |
&srv_ws->presignaled_sync->base); |
| 442 |
} |
| 443 |
|
| 444 |
pvr_srv_sync_prim_block_finish(srv_ws); |
| 445 |
pvr_srv_memctx_finish(srv_ws); |
| 446 |
vk_free(ws->alloc, srv_ws); |
| 447 |
pvr_srv_connection_destroy(fd); |
| 448 |
} |
| 449 |
|
| 450 |
static uint64_t |
| 451 |
pvr_srv_get_min_free_list_size(const struct pvr_device_info *dev_info) |
| 452 |
{ |
| 453 |
uint64_t min_num_pages; |
| 454 |
|
| 455 |
if (PVR_HAS_FEATURE(dev_info, roguexe)) { |
| 456 |
if (PVR_HAS_QUIRK(dev_info, 66011)) |
| 457 |
min_num_pages = 40U; |
| 458 |
else |
| 459 |
min_num_pages = 25U; |
| 460 |
} else { |
| 461 |
min_num_pages = 50U; |
| 462 |
} |
| 463 |
|
| 464 |
return min_num_pages << ROGUE_BIF_PM_PHYSICAL_PAGE_SHIFT; |
| 465 |
} |
| 466 |
|
| 467 |
static inline uint64_t |
| 468 |
pvr_srv_get_num_phantoms(const struct pvr_device_info *dev_info) |
| 469 |
{ |
| 470 |
return DIV_ROUND_UP(PVR_GET_FEATURE_VALUE(dev_info, num_clusters, 1U), 4U); |
| 471 |
} |
| 472 |
|
| 473 |
/* Return the total reserved size of partition in dwords. */ |
| 474 |
static inline uint64_t pvr_srv_get_total_reserved_partition_size( |
| 475 |
const struct pvr_device_info *dev_info) |
| 476 |
{ |
| 477 |
uint32_t tile_size_x = PVR_GET_FEATURE_VALUE(dev_info, tile_size_x, 0); |
| 478 |
uint32_t tile_size_y = PVR_GET_FEATURE_VALUE(dev_info, tile_size_y, 0); |
| 479 |
uint32_t max_partitions = PVR_GET_FEATURE_VALUE(dev_info, max_partitions, 0); |
| 480 |
|
| 481 |
if (tile_size_x == 16 && tile_size_y == 16) { |
| 482 |
return tile_size_x * tile_size_y * max_partitions * |
| 483 |
PVR_GET_FEATURE_VALUE(dev_info, |
| 484 |
usc_min_output_registers_per_pix, |
| 485 |
0); |
| 486 |
} |
| 487 |
|
| 488 |
return (uint64_t)max_partitions * 1024U; |
| 489 |
} |
| 490 |
|
| 491 |
static inline uint64_t |
| 492 |
pvr_srv_get_reserved_shared_size(const struct pvr_device_info *dev_info) |
| 493 |
{ |
| 494 |
uint32_t common_store_size_in_dwords = |
| 495 |
PVR_GET_FEATURE_VALUE(dev_info, |
| 496 |
common_store_size_in_dwords, |
| 497 |
512U * 4U * 4U); |
| 498 |
uint32_t reserved_shared_size = |
| 499 |
common_store_size_in_dwords - (256U * 4U) - |
| 500 |
pvr_srv_get_total_reserved_partition_size(dev_info); |
| 501 |
|
| 502 |
if (PVR_HAS_QUIRK(dev_info, 44079)) { |
| 503 |
uint32_t common_store_split_point = (768U * 4U * 4U); |
| 504 |
|
| 505 |
return MIN2(common_store_split_point - (256U * 4U), reserved_shared_size); |
| 506 |
} |
| 507 |
|
| 508 |
return reserved_shared_size; |
| 509 |
} |
| 510 |
|
| 511 |
static inline uint64_t |
| 512 |
pvr_srv_get_max_coeffs(const struct pvr_device_info *dev_info) |
| 513 |
{ |
| 514 |
uint32_t max_coeff_additional_portion = ROGUE_MAX_VERTEX_SHARED_REGISTERS; |
| 515 |
uint32_t pending_allocation_shared_regs = 2U * 1024U; |
| 516 |
uint32_t pending_allocation_coeff_regs = 0U; |
| 517 |
uint32_t num_phantoms = pvr_srv_get_num_phantoms(dev_info); |
| 518 |
uint32_t tiles_in_flight = |
| 519 |
PVR_GET_FEATURE_VALUE(dev_info, isp_max_tiles_in_flight, 0); |
| 520 |
uint32_t max_coeff_pixel_portion = |
| 521 |
DIV_ROUND_UP(tiles_in_flight, num_phantoms); |
| 522 |
|
| 523 |
max_coeff_pixel_portion *= ROGUE_MAX_PIXEL_SHARED_REGISTERS; |
| 524 |
|
| 525 |
/* Compute tasks on cores with BRN48492 and without compute overlap may lock |
| 526 |
* up without two additional lines of coeffs. |
| 527 |
*/ |
| 528 |
if (PVR_HAS_QUIRK(dev_info, 48492) && |
| 529 |
!PVR_HAS_FEATURE(dev_info, compute_overlap)) { |
| 530 |
pending_allocation_coeff_regs = 2U * 1024U; |
| 531 |
} |
| 532 |
|
| 533 |
if (PVR_HAS_ENHANCEMENT(dev_info, 38748)) |
| 534 |
pending_allocation_shared_regs = 0U; |
| 535 |
|
| 536 |
if (PVR_HAS_ENHANCEMENT(dev_info, 38020)) { |
| 537 |
max_coeff_additional_portion += |
| 538 |
rogue_max_compute_shared_registers(dev_info); |
| 539 |
} |
| 540 |
|
| 541 |
return pvr_srv_get_reserved_shared_size(dev_info) + |
| 542 |
pending_allocation_coeff_regs - |
| 543 |
(max_coeff_pixel_portion + max_coeff_additional_portion + |
| 544 |
pending_allocation_shared_regs); |
| 545 |
} |
| 546 |
|
| 547 |
static inline uint64_t |
| 548 |
pvr_srv_get_cdm_max_local_mem_size_regs(const struct pvr_device_info *dev_info) |
| 549 |
{ |
| 550 |
uint32_t available_coeffs_in_dwords = pvr_srv_get_max_coeffs(dev_info); |
| 551 |
|
| 552 |
if (PVR_HAS_QUIRK(dev_info, 48492) && PVR_HAS_FEATURE(dev_info, roguexe) && |
| 553 |
!PVR_HAS_FEATURE(dev_info, compute_overlap)) { |
| 554 |
/* Driver must not use the 2 reserved lines. */ |
| 555 |
available_coeffs_in_dwords -= ROGUE_CSRM_LINE_SIZE_IN_DWORDS * 2; |
| 556 |
} |
| 557 |
|
| 558 |
/* The maximum amount of local memory available to a kernel is the minimum |
| 559 |
* of the total number of coefficient registers available and the max common |
| 560 |
* store allocation size which can be made by the CDM. |
| 561 |
* |
| 562 |
* If any coeff lines are reserved for tessellation or pixel then we need to |
| 563 |
* subtract those too. |
| 564 |
*/ |
| 565 |
return MIN2(available_coeffs_in_dwords, |
| 566 |
ROGUE_MAX_PER_KERNEL_LOCAL_MEM_SIZE_REGS); |
| 567 |
} |
| 568 |
|
| 569 |
static VkResult |
| 570 |
pvr_srv_winsys_device_info_init(struct pvr_winsys *ws, |
| 571 |
struct pvr_device_info *dev_info, |
| 572 |
struct pvr_device_runtime_info *runtime_info) |
| 573 |
{ |
| 574 |
struct pvr_srv_winsys *srv_ws = to_pvr_srv_winsys(ws); |
| 575 |
VkResult result; |
| 576 |
|
| 577 |
*dev_info = srv_ws->dev_info; |
| 578 |
|
| 579 |
runtime_info->min_free_list_size = pvr_srv_get_min_free_list_size(dev_info); |
| 580 |
runtime_info->max_free_list_size = PVR_SRV_FREE_LIST_MAX_SIZE; |
| 581 |
runtime_info->reserved_shared_size = |
| 582 |
pvr_srv_get_reserved_shared_size(dev_info); |
| 583 |
runtime_info->total_reserved_partition_size = |
| 584 |
pvr_srv_get_total_reserved_partition_size(dev_info); |
| 585 |
runtime_info->num_phantoms = pvr_srv_get_num_phantoms(dev_info); |
| 586 |
runtime_info->max_coeffs = pvr_srv_get_max_coeffs(dev_info); |
| 587 |
runtime_info->cdm_max_local_mem_size_regs = |
| 588 |
pvr_srv_get_cdm_max_local_mem_size_regs(dev_info); |
| 589 |
|
| 590 |
if (PVR_HAS_FEATURE(dev_info, gpu_multicore_support)) { |
| 591 |
result = pvr_srv_get_multicore_info(ws->render_fd, |
| 592 |
0, |
| 593 |
NULL, |
| 594 |
&runtime_info->core_count); |
| 595 |
if (result != VK_SUCCESS) |
| 596 |
return result; |
| 597 |
} else { |
| 598 |
runtime_info->core_count = 1; |
| 599 |
} |
| 600 |
|
| 601 |
return 0; |
| 602 |
} |
| 603 |
|
| 604 |
static void pvr_srv_winsys_get_heaps_info(struct pvr_winsys *ws, |
| 605 |
struct pvr_winsys_heaps *heaps) |
| 606 |
{ |
| 607 |
struct pvr_srv_winsys *srv_ws = to_pvr_srv_winsys(ws); |
| 608 |
|
| 609 |
heaps->general_heap = &srv_ws->general_heap.base; |
| 610 |
heaps->pds_heap = &srv_ws->pds_heap.base; |
| 611 |
heaps->transfer_frag_heap = &srv_ws->transfer_3d_heap.base; |
| 612 |
heaps->usc_heap = &srv_ws->usc_heap.base; |
| 613 |
heaps->vis_test_heap = &srv_ws->vis_test_heap.base; |
| 614 |
|
| 615 |
if (srv_ws->rgn_hdr_heap_present) |
| 616 |
heaps->rgn_hdr_heap = &srv_ws->rgn_hdr_heap.base; |
| 617 |
else |
| 618 |
heaps->rgn_hdr_heap = &srv_ws->general_heap.base; |
| 619 |
} |
| 620 |
|
| 621 |
#define PER_ARCH_FUNCS(arch) \ |
| 622 |
VkResult pvr_##arch##_srv_render_target_dataset_create( \ |
| 623 |
struct pvr_winsys *ws, \ |
| 624 |
const struct pvr_winsys_rt_dataset_create_info *create_info, \ |
| 625 |
const struct pvr_device_info *dev_info, \ |
| 626 |
struct pvr_winsys_rt_dataset **const rt_dataset_out); \ |
| 627 |
\ |
| 628 |
VkResult pvr_##arch##_srv_winsys_render_submit( \ |
| 629 |
const struct pvr_winsys_render_ctx *ctx, \ |
| 630 |
const struct pvr_winsys_render_submit_info *submit_info, \ |
| 631 |
const struct pvr_device_info *dev_info, \ |
| 632 |
struct vk_sync *signal_sync_geom, \ |
| 633 |
struct vk_sync *signal_sync_frag); \ |
| 634 |
\ |
| 635 |
VkResult pvr_##arch##_srv_winsys_transfer_submit( \ |
| 636 |
const struct pvr_winsys_transfer_ctx *ctx, \ |
| 637 |
const struct pvr_winsys_transfer_submit_info *submit_info, \ |
| 638 |
const struct pvr_device_info *const dev_info, \ |
| 639 |
struct vk_sync *signal_sync); \ |
| 640 |
\ |
| 641 |
VkResult pvr_##arch##_srv_winsys_compute_submit( \ |
| 642 |
const struct pvr_winsys_compute_ctx *ctx, \ |
| 643 |
const struct pvr_winsys_compute_submit_info *submit_info, \ |
| 644 |
const struct pvr_device_info *dev_info, \ |
| 645 |
struct vk_sync *signal_sync) |
| 646 |
|
| 647 |
PER_ARCH_FUNCS(rogue); |
| 648 |
|
| 649 |
#define SRV_WINSYS(arch) \ |
| 650 |
static const struct pvr_winsys_ops pvr_##arch##_srv_winsys_ops = { \ |
| 651 |
.destroy = pvr_srv_winsys_destroy, \ |
| 652 |
.device_info_init = pvr_srv_winsys_device_info_init, \ |
| 653 |
.get_heaps_info = pvr_srv_winsys_get_heaps_info, \ |
| 654 |
.buffer_create = pvr_srv_winsys_buffer_create, \ |
| 655 |
.buffer_create_from_fd = pvr_srv_winsys_buffer_create_from_fd, \ |
| 656 |
.buffer_destroy = pvr_srv_winsys_buffer_destroy, \ |
| 657 |
.buffer_get_fd = pvr_srv_winsys_buffer_get_fd, \ |
| 658 |
.buffer_map = pvr_srv_winsys_buffer_map, \ |
| 659 |
.buffer_unmap = pvr_srv_winsys_buffer_unmap, \ |
| 660 |
.heap_alloc = pvr_srv_winsys_heap_alloc, \ |
| 661 |
.heap_free = pvr_srv_winsys_heap_free, \ |
| 662 |
.vma_map = pvr_srv_winsys_vma_map, \ |
| 663 |
.vma_unmap = pvr_srv_winsys_vma_unmap, \ |
| 664 |
.free_list_create = pvr_srv_winsys_free_list_create, \ |
| 665 |
.free_list_destroy = pvr_srv_winsys_free_list_destroy, \ |
| 666 |
.render_target_dataset_create = \ |
| 667 |
pvr_##arch##_srv_render_target_dataset_create, \ |
| 668 |
.render_target_dataset_destroy = pvr_srv_render_target_dataset_destroy, \ |
| 669 |
.render_ctx_create = pvr_srv_winsys_render_ctx_create, \ |
| 670 |
.render_ctx_destroy = pvr_srv_winsys_render_ctx_destroy, \ |
| 671 |
.render_submit = pvr_##arch##_srv_winsys_render_submit, \ |
| 672 |
.compute_ctx_create = pvr_srv_winsys_compute_ctx_create, \ |
| 673 |
.compute_ctx_destroy = pvr_srv_winsys_compute_ctx_destroy, \ |
| 674 |
.compute_submit = pvr_##arch##_srv_winsys_compute_submit, \ |
| 675 |
.transfer_ctx_create = pvr_srv_winsys_transfer_ctx_create, \ |
| 676 |
.transfer_ctx_destroy = pvr_srv_winsys_transfer_ctx_destroy, \ |
| 677 |
.transfer_submit = pvr_##arch##_srv_winsys_transfer_submit, \ |
| 678 |
.null_job_submit = pvr_srv_winsys_null_job_submit, \ |
| 679 |
} |
| 680 |
|
| 681 |
SRV_WINSYS(rogue); |
| 682 |
|
| 683 |
static bool pvr_is_driver_compatible(int render_fd) |
| 684 |
{ |
| 685 |
drmVersionPtr version; |
| 686 |
|
| 687 |
version = drmGetVersion(render_fd); |
| 688 |
if (!version) |
| 689 |
return false; |
| 690 |
|
| 691 |
assert(strcmp(version->name, PVR_SRV_DRIVER_NAME) == 0); |
| 692 |
|
| 693 |
/* Only the 1.17 driver is supported for now. */ |
| 694 |
if (version->version_major != PVR_SRV_VERSION_MAJ || |
| 695 |
version->version_minor != PVR_SRV_VERSION_MIN) { |
| 696 |
vk_errorf(NULL, |
| 697 |
VK_ERROR_INCOMPATIBLE_DRIVER, |
| 698 |
"Unsupported downstream driver version (%u.%u)", |
| 699 |
version->version_major, |
| 700 |
version->version_minor); |
| 701 |
drmFreeVersion(version); |
| 702 |
|
| 703 |
return false; |
| 704 |
} |
| 705 |
|
| 706 |
drmFreeVersion(version); |
| 707 |
|
| 708 |
return true; |
| 709 |
} |
| 710 |
|
| 711 |
VkResult pvr_srv_winsys_create(const int render_fd, |
| 712 |
const int display_fd, |
| 713 |
const VkAllocationCallbacks *alloc, |
| 714 |
struct pvr_winsys **const ws_out) |
| 715 |
{ |
| 716 |
struct pvr_srv_winsys *srv_ws; |
| 717 |
VkResult result; |
| 718 |
uint64_t bvnc; |
| 719 |
|
| 720 |
if (!pvr_is_driver_compatible(render_fd)) |
| 721 |
return VK_ERROR_INCOMPATIBLE_DRIVER; |
| 722 |
|
| 723 |
result = pvr_srv_init_module(render_fd, PVR_SRVKM_MODULE_TYPE_SERVICES); |
| 724 |
if (result != VK_SUCCESS) |
| 725 |
goto err_out; |
| 726 |
|
| 727 |
result = pvr_srv_connection_create(render_fd, &bvnc); |
| 728 |
if (result != VK_SUCCESS) |
| 729 |
goto err_out; |
| 730 |
|
| 731 |
srv_ws = |
| 732 |
vk_zalloc(alloc, sizeof(*srv_ws), 8, VK_SYSTEM_ALLOCATION_SCOPE_DEVICE); |
| 733 |
if (!srv_ws) { |
| 734 |
result = vk_error(NULL, VK_ERROR_OUT_OF_HOST_MEMORY); |
| 735 |
goto err_pvr_srv_connection_destroy; |
| 736 |
} |
| 737 |
|
| 738 |
int ret = pvr_device_info_init(&srv_ws->dev_info, bvnc); |
| 739 |
if (ret) { |
| 740 |
return vk_errorf(NULL, |
| 741 |
VK_ERROR_INCOMPATIBLE_DRIVER, |
| 742 |
"Unsupported BVNC: %u.%u.%u.%u\n", |
| 743 |
PVR_BVNC_UNPACK_B(bvnc), |
| 744 |
PVR_BVNC_UNPACK_V(bvnc), |
| 745 |
PVR_BVNC_UNPACK_N(bvnc), |
| 746 |
PVR_BVNC_UNPACK_C(bvnc)); |
| 747 |
} |
| 748 |
|
| 749 |
switch (srv_ws->dev_info.ident.arch) { |
| 750 |
case PVR_DEVICE_ARCH_ROGUE: |
| 751 |
srv_ws->base.ops = &pvr_rogue_srv_winsys_ops; |
| 752 |
break; |
| 753 |
} |
| 754 |
|
| 755 |
srv_ws->base.render_fd = render_fd; |
| 756 |
srv_ws->base.display_fd = display_fd; |
| 757 |
srv_ws->base.alloc = alloc; |
| 758 |
|
| 759 |
srv_ws->base.syncobj_type = pvr_srv_sync_type; |
| 760 |
srv_ws->base.sync_types[0] = &srv_ws->base.syncobj_type; |
| 761 |
|
| 762 |
srv_ws->base.timeline_syncobj_type = |
| 763 |
vk_sync_timeline_get_type(srv_ws->base.sync_types[0]); |
| 764 |
srv_ws->base.sync_types[1] = &srv_ws->base.timeline_syncobj_type.sync; |
| 765 |
srv_ws->base.sync_types[2] = NULL; |
| 766 |
|
| 767 |
/* Threaded submit requires VK_SYNC_FEATURE_WAIT_PENDING which pvrsrv |
| 768 |
* doesn't support. |
| 769 |
*/ |
| 770 |
srv_ws->base.features.supports_threaded_submit = false; |
| 771 |
|
| 772 |
result = pvr_srv_memctx_init(srv_ws); |
| 773 |
if (result != VK_SUCCESS) |
| 774 |
goto err_vk_free_srv_ws; |
| 775 |
|
| 776 |
result = pvr_srv_sync_prim_block_init(srv_ws); |
| 777 |
if (result != VK_SUCCESS) |
| 778 |
goto err_pvr_srv_memctx_finish; |
| 779 |
|
| 780 |
*ws_out = &srv_ws->base; |
| 781 |
|
| 782 |
return VK_SUCCESS; |
| 783 |
|
| 784 |
err_pvr_srv_memctx_finish: |
| 785 |
pvr_srv_memctx_finish(srv_ws); |
| 786 |
|
| 787 |
err_vk_free_srv_ws: |
| 788 |
vk_free(alloc, srv_ws); |
| 789 |
|
| 790 |
err_pvr_srv_connection_destroy: |
| 791 |
pvr_srv_connection_destroy(render_fd); |
| 792 |
|
| 793 |
err_out: |
| 794 |
return result; |
| 795 |
} |
| 796 |
|
| 797 |
static VkResult pvr_srv_create_presignaled_sync(struct pvr_device *device, |
| 798 |
struct pvr_srv_sync **out_sync) |
| 799 |
{ |
| 800 |
struct pvr_srv_winsys *srv_ws = to_pvr_srv_winsys(device->ws); |
| 801 |
struct vk_sync *sync; |
| 802 |
|
| 803 |
int timeline_fd; |
| 804 |
int sync_fd; |
| 805 |
|
| 806 |
VkResult result; |
| 807 |
|
| 808 |
result = pvr_srv_create_timeline(srv_ws->base.render_fd, &timeline_fd); |
| 809 |
if (result != VK_SUCCESS) |
| 810 |
return result; |
| 811 |
|
| 812 |
result = pvr_srv_set_timeline_sw_only(timeline_fd); |
| 813 |
if (result != VK_SUCCESS) |
| 814 |
goto err_close_timeline; |
| 815 |
|
| 816 |
result = pvr_srv_create_sw_fence(timeline_fd, &sync_fd, NULL); |
| 817 |
if (result != VK_SUCCESS) |
| 818 |
goto err_close_timeline; |
| 819 |
|
| 820 |
result = pvr_srv_sw_sync_timeline_increment(timeline_fd, NULL); |
| 821 |
if (result != VK_SUCCESS) |
| 822 |
goto err_close_sw_fence; |
| 823 |
|
| 824 |
result = vk_sync_create(&device->vk, |
| 825 |
&device->pdevice->ws->syncobj_type, |
| 826 |
0U, |
| 827 |
0UL, |
| 828 |
&sync); |
| 829 |
if (result != VK_SUCCESS) |
| 830 |
goto err_close_sw_fence; |
| 831 |
|
| 832 |
result = vk_sync_import_sync_file(&device->vk, sync, sync_fd); |
| 833 |
if (result != VK_SUCCESS) |
| 834 |
goto err_destroy_sync; |
| 835 |
|
| 836 |
*out_sync = to_srv_sync(sync); |
| 837 |
(*out_sync)->signaled = true; |
| 838 |
|
| 839 |
close(timeline_fd); |
| 840 |
|
| 841 |
return VK_SUCCESS; |
| 842 |
|
| 843 |
err_destroy_sync: |
| 844 |
vk_sync_destroy(&device->vk, sync); |
| 845 |
|
| 846 |
err_close_sw_fence: |
| 847 |
close(sync_fd); |
| 848 |
|
| 849 |
err_close_timeline: |
| 850 |
close(timeline_fd); |
| 851 |
|
| 852 |
return result; |
| 853 |
} |
| 854 |
|
| 855 |
VkResult pvr_srv_sync_get_presignaled_sync(struct pvr_device *device, |
| 856 |
struct pvr_srv_sync **out_sync) |
| 857 |
{ |
| 858 |
struct pvr_srv_winsys *srv_ws = to_pvr_srv_winsys(device->ws); |
| 859 |
VkResult result; |
| 860 |
|
| 861 |
if (!srv_ws->presignaled_sync) { |
| 862 |
result = |
| 863 |
pvr_srv_create_presignaled_sync(device, &srv_ws->presignaled_sync); |
| 864 |
if (result != VK_SUCCESS) |
| 865 |
return result; |
| 866 |
|
| 867 |
srv_ws->presignaled_sync_device = device; |
| 868 |
} |
| 869 |
|
| 870 |
assert(device == srv_ws->presignaled_sync_device); |
| 871 |
|
| 872 |
*out_sync = srv_ws->presignaled_sync; |
| 873 |
|
| 874 |
return VK_SUCCESS; |
| 875 |
} |