Debian Codegraph @

src/imagination/vulkan/pvr_arch_device.c

raw
1 /*
2 * Copyright © 2022 Imagination Technologies Ltd.
3 *
4 * based in part on anv driver which is:
5 * Copyright © 2015 Intel Corporation
6 *
7 * based in part on v3dv driver which is:
8 * Copyright © 2019 Raspberry Pi
9 *
10 * SPDX-License-Identifier: MIT
11 */
12
13 #include "pvr_device.h"
14
15 #include "vk_log.h"
16
17 #include "hwdef/pvr_hw_utils.h"
18
19 #include "pco_uscgen_programs.h"
20
21 #include "pvr_border.h"
22 #include "pvr_clear.h"
23 #include "pvr_entrypoints.h"
24 #include "pvr_framebuffer.h"
25 #include "pvr_free_list.h"
26 #include "pvr_instance.h"
27 #include "pvr_job_render.h"
28 #include "pvr_macros.h"
29 #include "pvr_physical_device.h"
30 #include "pvr_query.h"
31 #include "pvr_queue.h"
32 #include "pvr_robustness.h"
33 #include "pvr_tex_state.h"
34
35 #define PVR_GLOBAL_FREE_LIST_INITIAL_SIZE (2U * 1024U * 1024U)
36 #define PVR_GLOBAL_FREE_LIST_MAX_SIZE (256U * 1024U * 1024U)
37 #define PVR_GLOBAL_FREE_LIST_GROW_SIZE (1U * 1024U * 1024U)
38
39 /* After PVR_SECONDARY_DEVICE_THRESHOLD devices per instance are created,
40 * devices will have a smaller global free list size, as usually this use-case
41 * implies smaller amounts of work spread out. The free list can still grow as
42 * required.
43 */
44 #define PVR_SECONDARY_DEVICE_THRESHOLD (4U)
45 #define PVR_SECONDARY_DEVICE_FREE_LIST_INITAL_SIZE (512U * 1024U)
46
47 /* The grow threshold is a percentage. This is intended to be 12.5%, but has
48 * been rounded up since the percentage is treated as an integer.
49 */
50 #define PVR_GLOBAL_FREE_LIST_GROW_THRESHOLD 13U
51
52 /* Default size in bytes used by pvr_CreateDevice() for setting up the
53 * suballoc_general, suballoc_pds and suballoc_usc suballocators.
54 *
55 * TODO: Investigate if a different default size can improve the overall
56 * performance of internal driver allocations.
57 */
58 #define PVR_SUBALLOCATOR_GENERAL_SIZE (128 * 1024)
59 #define PVR_SUBALLOCATOR_PDS_SIZE (128 * 1024)
60 #define PVR_SUBALLOCATOR_TRANSFER_SIZE (128 * 1024)
61 #define PVR_SUBALLOCATOR_USC_SIZE (128 * 1024)
62 #define PVR_SUBALLOCATOR_VIS_TEST_SIZE (128 * 1024)
63
64 static uint32_t pvr_get_simultaneous_num_allocs(
65 const struct pvr_device_info *dev_info,
66 ASSERTED const struct pvr_device_runtime_info *dev_runtime_info)
67 {
68 uint32_t min_cluster_per_phantom;
69
70 if (PVR_HAS_FEATURE(dev_info, s8xe))
71 return PVR_GET_FEATURE_VALUE(dev_info, num_raster_pipes, 0U);
72
73 assert(dev_runtime_info->num_phantoms == 1);
74 min_cluster_per_phantom = PVR_GET_FEATURE_VALUE(dev_info, num_clusters, 1U);
75
76 if (min_cluster_per_phantom >= 4)
77 return 1;
78 else if (min_cluster_per_phantom == 2)
79 return 2;
80 else
81 return 4;
82 }
83
84 uint32_t pvr_arch_calc_fscommon_size_and_tiles_in_flight(
85 const struct pvr_device_info *dev_info,
86 const struct pvr_device_runtime_info *dev_runtime_info,
87 uint32_t fs_common_size,
88 uint32_t min_tiles_in_flight)
89 {
90 const uint32_t available_shareds =
91 dev_runtime_info->reserved_shared_size - dev_runtime_info->max_coeffs;
92 const uint32_t max_tiles_in_flight =
93 PVR_GET_FEATURE_VALUE(dev_info, isp_max_tiles_in_flight, 1U);
94 uint32_t num_tile_in_flight;
95 uint32_t num_allocs;
96
97 if (fs_common_size == 0)
98 return max_tiles_in_flight;
99
100 num_allocs = pvr_get_simultaneous_num_allocs(dev_info, dev_runtime_info);
101
102 if (fs_common_size == UINT32_MAX) {
103 uint32_t max_common_size = available_shareds;
104
105 num_allocs *= MIN2(min_tiles_in_flight, max_tiles_in_flight);
106
107 if (!PVR_HAS_ENHANCEMENT(dev_info, 38748)) {
108 /* Hardware needs space for one extra shared allocation. */
109 num_allocs += 1;
110 }
111
112 /* Double resource requirements to deal with fragmentation. */
113 max_common_size /= num_allocs * 2;
114 max_common_size = MIN2(max_common_size, ROGUE_MAX_PIXEL_SHARED_REGISTERS);
115 max_common_size =
116 ROUND_DOWN_TO(max_common_size,
117 ROGUE_TA_STATE_PDS_SIZEINFO2_USC_SHAREDSIZE_UNIT_SIZE);
118
119 return max_common_size;
120 }
121
122 num_tile_in_flight = available_shareds / (fs_common_size * 2);
123
124 if (!PVR_HAS_ENHANCEMENT(dev_info, 38748))
125 num_tile_in_flight -= 1;
126
127 num_tile_in_flight /= num_allocs;
128
129 #if MESA_DEBUG
130 /* Validate the above result. */
131
132 assert(num_tile_in_flight >= MIN2(num_tile_in_flight, max_tiles_in_flight));
133 num_allocs *= num_tile_in_flight;
134
135 if (!PVR_HAS_ENHANCEMENT(dev_info, 38748)) {
136 /* Hardware needs space for one extra shared allocation. */
137 num_allocs += 1;
138 }
139
140 assert(fs_common_size <= available_shareds / (num_allocs * 2));
141 #endif
142
143 return MIN2(num_tile_in_flight, max_tiles_in_flight);
144 }
145
146 VkResult pvr_arch_pds_compute_shader_create_and_upload(
147 struct pvr_device *device,
148 struct pvr_pds_compute_shader_program *program,
149 struct pvr_pds_upload *const pds_upload_out)
150 {
151 const struct pvr_device_info *dev_info = &device->pdevice->dev_info;
152 const uint32_t cache_line_size = pvr_get_slc_cache_line_size(dev_info);
153 size_t staging_buffer_size;
154 uint32_t *staging_buffer;
155 uint32_t *data_buffer;
156 uint32_t *code_buffer;
157 VkResult result;
158
159 /* Calculate how much space we'll need for the compute shader PDS program.
160 */
161 pvr_pds_compute_shader(program, NULL, PDS_GENERATE_SIZES, dev_info);
162
163 /* FIXME: Fix the below inconsistency of code size being in bytes whereas
164 * data size being in dwords.
165 */
166 /* Code size is in bytes, data size in dwords. */
167 staging_buffer_size =
168 PVR_DW_TO_BYTES(program->data_size) + program->code_size;
169
170 staging_buffer = vk_alloc(&device->vk.alloc,
171 staging_buffer_size,
172 8U,
173 VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
174 if (!staging_buffer)
175 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
176
177 data_buffer = staging_buffer;
178 code_buffer = pvr_pds_compute_shader(program,
179 data_buffer,
180 PDS_GENERATE_DATA_SEGMENT,
181 dev_info);
182
183 pvr_pds_compute_shader(program,
184 code_buffer,
185 PDS_GENERATE_CODE_SEGMENT,
186 dev_info);
187
188 for (unsigned u = 0; u < PVR_WORKGROUP_DIMENSIONS; ++u) {
189 unsigned offset = program->num_workgroups_constant_offset_in_dwords[0];
190 if (program->num_work_groups_regs[u] != PVR_PDS_REG_UNUSED)
191 data_buffer[offset + u] = 0;
192
193 offset = program->base_workgroup_constant_offset_in_dwords[0];
194 if (program->work_group_input_regs[u] != PVR_PDS_REG_UNUSED)
195 data_buffer[offset + u] = 0;
196 }
197
198 result = pvr_gpu_upload_pds(device,
199 data_buffer,
200 program->data_size,
201 ROGUE_CDMCTRL_KERNEL1_DATA_ADDR_ALIGNMENT,
202 code_buffer,
203 program->code_size / sizeof(uint32_t),
204 ROGUE_CDMCTRL_KERNEL2_CODE_ADDR_ALIGNMENT,
205 cache_line_size,
206 pds_upload_out);
207
208 vk_free(&device->vk.alloc, staging_buffer);
209
210 return result;
211 }
212
213 static VkResult pvr_device_init_compute_fence_program(struct pvr_device *device)
214 {
215 struct pvr_pds_compute_shader_program program;
216
217 pvr_pds_compute_shader_program_init(&program);
218 /* Fence kernel. */
219 program.fence = true;
220 program.clear_pds_barrier = true;
221
222 return pvr_arch_pds_compute_shader_create_and_upload(
223 device,
224 &program,
225 &device->pds_compute_fence_program);
226 }
227
228 static VkResult pvr_device_init_compute_empty_program(struct pvr_device *device)
229 {
230 struct pvr_pds_compute_shader_program program;
231
232 pvr_pds_compute_shader_program_init(&program);
233 program.clear_pds_barrier = true;
234
235 return pvr_arch_pds_compute_shader_create_and_upload(
236 device,
237 &program,
238 &device->pds_compute_empty_program);
239 }
240
241 static VkResult pvr_pds_idfwdf_programs_create_and_upload(
242 struct pvr_device *device,
243 pvr_dev_addr_t usc_addr,
244 uint32_t shareds,
245 uint32_t temps,
246 pvr_dev_addr_t shareds_buffer_addr,
247 struct pvr_pds_upload *const upload_out,
248 struct pvr_pds_upload *const sw_compute_barrier_upload_out)
249 {
250 const struct pvr_device_info *dev_info = &device->pdevice->dev_info;
251 struct pvr_pds_vertex_shader_sa_program program = {
252 .kick_usc = true,
253 .clear_pds_barrier = PVR_NEED_SW_COMPUTE_PDS_BARRIER(dev_info),
254 };
255 size_t staging_buffer_size;
256 uint32_t *staging_buffer;
257 VkResult result;
258
259 /* We'll need to DMA the shareds into the USC's Common Store. */
260 program.num_dma_kicks = pvr_pds_encode_dma_burst(program.dma_control,
261 program.dma_address,
262 0,
263 shareds,
264 shareds_buffer_addr.addr,
265 false,
266 dev_info);
267
268 /* DMA temp regs. */
269 pvr_pds_setup_doutu(&program.usc_task_control,
270 usc_addr.addr,
271 temps,
272 ROGUE_PDSINST_DOUTU_SAMPLE_RATE_INSTANCE,
273 false);
274
275 pvr_pds_vertex_shader_sa(&program, NULL, PDS_GENERATE_SIZES, dev_info);
276
277 staging_buffer_size = PVR_DW_TO_BYTES(program.code_size + program.data_size);
278
279 staging_buffer = vk_alloc(&device->vk.alloc,
280 staging_buffer_size,
281 8,
282 VK_SYSTEM_ALLOCATION_SCOPE_COMMAND);
283 if (!staging_buffer)
284 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
285
286 /* FIXME: Add support for PDS_GENERATE_CODEDATA_SEGMENTS? */
287 pvr_pds_vertex_shader_sa(&program,
288 staging_buffer,
289 PDS_GENERATE_DATA_SEGMENT,
290 dev_info);
291 pvr_pds_vertex_shader_sa(&program,
292 &staging_buffer[program.data_size],
293 PDS_GENERATE_CODE_SEGMENT,
294 dev_info);
295
296 /* At the time of writing, the SW_COMPUTE_PDS_BARRIER variant of the program
297 * is bigger so we handle it first (if needed) and realloc() for a smaller
298 * size.
299 */
300 if (PVR_NEED_SW_COMPUTE_PDS_BARRIER(dev_info)) {
301 /* FIXME: Figure out the define for alignment of 16. */
302 result = pvr_gpu_upload_pds(device,
303 &staging_buffer[0],
304 program.data_size,
305 16,
306 &staging_buffer[program.data_size],
307 program.code_size,
308 16,
309 16,
310 sw_compute_barrier_upload_out);
311 if (result != VK_SUCCESS) {
312 vk_free(&device->vk.alloc, staging_buffer);
313 return result;
314 }
315
316 program.clear_pds_barrier = false;
317
318 pvr_pds_vertex_shader_sa(&program, NULL, PDS_GENERATE_SIZES, dev_info);
319
320 staging_buffer_size =
321 PVR_DW_TO_BYTES(program.code_size + program.data_size);
322
323 staging_buffer = vk_realloc(&device->vk.alloc,
324 staging_buffer,
325 staging_buffer_size,
326 8,
327 VK_SYSTEM_ALLOCATION_SCOPE_COMMAND);
328 if (!staging_buffer) {
329 pvr_bo_suballoc_free(sw_compute_barrier_upload_out->pvr_bo);
330
331 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
332 }
333
334 /* FIXME: Add support for PDS_GENERATE_CODEDATA_SEGMENTS? */
335 pvr_pds_vertex_shader_sa(&program,
336 staging_buffer,
337 PDS_GENERATE_DATA_SEGMENT,
338 dev_info);
339 pvr_pds_vertex_shader_sa(&program,
340 &staging_buffer[program.data_size],
341 PDS_GENERATE_CODE_SEGMENT,
342 dev_info);
343 } else {
344 *sw_compute_barrier_upload_out = (struct pvr_pds_upload){
345 .pvr_bo = NULL,
346 };
347 }
348
349 /* FIXME: Figure out the define for alignment of 16. */
350 result = pvr_gpu_upload_pds(device,
351 &staging_buffer[0],
352 program.data_size,
353 16,
354 &staging_buffer[program.data_size],
355 program.code_size,
356 16,
357 16,
358 upload_out);
359 if (result != VK_SUCCESS) {
360 vk_free(&device->vk.alloc, staging_buffer);
361 pvr_bo_suballoc_free(sw_compute_barrier_upload_out->pvr_bo);
362
363 return result;
364 }
365
366 vk_free(&device->vk.alloc, staging_buffer);
367
368 return VK_SUCCESS;
369 }
370
371 static VkResult pvr_device_init_compute_idfwdf_state(struct pvr_device *device)
372 {
373 struct pvr_sampler_descriptor sampler_state;
374 struct pvr_image_descriptor image_state;
375 struct pvr_texture_state_info tex_info;
376 const pco_precomp_data *precomp_data;
377 uint32_t *dword_ptr;
378 VkResult result;
379
380 precomp_data = (pco_precomp_data *)pco_usclib_common[CS_IDFWDF_COMMON];
381 device->idfwdf_state.usc_shareds = _PVR_IDFWDF_DATA_COUNT;
382
383 /* FIXME: Figure out the define for alignment of 16. */
384 result = pvr_gpu_upload_usc(device,
385 precomp_data->binary,
386 precomp_data->size_dwords * sizeof(uint32_t),
387 16,
388 &device->idfwdf_state.usc);
389
390 if (result != VK_SUCCESS)
391 return result;
392
393 result = pvr_bo_alloc(device,
394 device->heaps.general_heap,
395 PVR_IDFWDF_TEX_WIDTH * PVR_IDFWDF_TEX_HEIGHT *
396 vk_format_get_blocksize(PVR_IDFWDF_TEX_FORMAT),
397 4,
398 0,
399 &device->idfwdf_state.store_bo);
400 if (result != VK_SUCCESS)
401 goto err_free_usc_program;
402
403 result = pvr_bo_alloc(device,
404 device->heaps.general_heap,
405 _PVR_IDFWDF_DATA_COUNT * ROGUE_REG_SIZE_BYTES,
406 ROGUE_REG_SIZE_BYTES,
407 PVR_BO_ALLOC_FLAG_CPU_MAPPED,
408 &device->idfwdf_state.shareds_bo);
409 if (result != VK_SUCCESS)
410 goto err_free_store_buffer;
411
412 /* Pack state words. */
413
414 pvr_csb_pack (&sampler_state.words[0], TEXSTATE_SAMPLER_WORD0, sampler) {
415 sampler.dadjust = ROGUE_TEXSTATE_DADJUST_ZERO_UINT;
416 sampler.magfilter = ROGUE_TEXSTATE_FILTER_POINT;
417 sampler.addrmode_u = ROGUE_TEXSTATE_ADDRMODE_CLAMP_TO_EDGE;
418 sampler.addrmode_v = ROGUE_TEXSTATE_ADDRMODE_CLAMP_TO_EDGE;
419 }
420
421 /* clang-format off */
422 pvr_csb_pack (&sampler_state.words[1], TEXSTATE_SAMPLER_WORD1, sampler_word1) {}
423 /* clang-format on */
424
425 tex_info = (struct pvr_texture_state_info){
426 .format = PVR_IDFWDF_TEX_FORMAT,
427 .mem_layout = PVR_MEMLAYOUT_LINEAR,
428 .flags = PVR_TEXFLAGS_INDEX_LOOKUP,
429 .type = VK_IMAGE_VIEW_TYPE_2D,
430 .extent = { .width = PVR_IDFWDF_TEX_WIDTH,
431 .height = PVR_IDFWDF_TEX_HEIGHT },
432 .mip_levels = 1,
433 .sample_count = 1,
434 .stride = PVR_IDFWDF_TEX_STRIDE,
435 .swizzle = { PIPE_SWIZZLE_X,
436 PIPE_SWIZZLE_Y,
437 PIPE_SWIZZLE_Z,
438 PIPE_SWIZZLE_W },
439 .addr = device->idfwdf_state.store_bo->vma->dev_addr,
440 };
441
442 result = pvr_arch_pack_tex_state(device, &tex_info, &image_state);
443 if (result != VK_SUCCESS)
444 goto err_free_shareds_buffer;
445
446 /* Fill the shareds buffer. */
447 dword_ptr = (uint32_t *)device->idfwdf_state.shareds_bo->bo->map;
448
449 memcpy(&dword_ptr[PVR_IDFWDF_DATA_TEX],
450 image_state.words,
451 sizeof(image_state.words));
452 memcpy(&dword_ptr[PVR_IDFWDF_DATA_SMP],
453 sampler_state.words,
454 sizeof(sampler_state.words));
455
456 dword_ptr[PVR_IDFWDF_DATA_ADDR_LO] =
457 device->idfwdf_state.store_bo->vma->dev_addr.addr & 0xffffffff;
458 dword_ptr[PVR_IDFWDF_DATA_ADDR_HI] =
459 device->idfwdf_state.store_bo->vma->dev_addr.addr >> 32;
460
461 pvr_bo_cpu_unmap(device, device->idfwdf_state.shareds_bo);
462 dword_ptr = NULL;
463
464 /* Generate and upload PDS programs. */
465 result = pvr_pds_idfwdf_programs_create_and_upload(
466 device,
467 device->idfwdf_state.usc->dev_addr,
468 _PVR_IDFWDF_DATA_COUNT,
469 precomp_data->temps,
470 device->idfwdf_state.shareds_bo->vma->dev_addr,
471 &device->idfwdf_state.pds,
472 &device->idfwdf_state.sw_compute_barrier_pds);
473
474 if (result != VK_SUCCESS)
475 goto err_free_shareds_buffer;
476
477 return VK_SUCCESS;
478
479 err_free_shareds_buffer:
480 pvr_bo_free(device, device->idfwdf_state.shareds_bo);
481
482 err_free_store_buffer:
483 pvr_bo_free(device, device->idfwdf_state.store_bo);
484
485 err_free_usc_program:
486 pvr_bo_suballoc_free(device->idfwdf_state.usc);
487
488 return result;
489 }
490
491 static void pvr_device_finish_compute_idfwdf_state(struct pvr_device *device)
492 {
493 pvr_bo_suballoc_free(device->idfwdf_state.pds.pvr_bo);
494 pvr_bo_suballoc_free(device->idfwdf_state.sw_compute_barrier_pds.pvr_bo);
495 pvr_bo_free(device, device->idfwdf_state.shareds_bo);
496 pvr_bo_free(device, device->idfwdf_state.store_bo);
497 pvr_bo_suballoc_free(device->idfwdf_state.usc);
498 }
499
500 /* FIXME: We should be calculating the size when we upload the code in
501 * pvr_srv_setup_static_pixel_event_program().
502 */
503 static void pvr_device_get_pixel_event_pds_program_data_size(
504 const struct pvr_device_info *dev_info,
505 uint32_t *const data_size_in_dwords_out)
506 {
507 struct pvr_pds_event_program program = {
508 /* No data to DMA, just a DOUTU needed. */
509 .num_emit_word_pairs = 0,
510 };
511
512 pvr_pds_set_sizes_pixel_event(&program, dev_info);
513
514 *data_size_in_dwords_out = program.data_size;
515 }
516
517 static VkResult pvr_device_init_nop_program(struct pvr_device *device)
518 {
519 const uint32_t cache_line_size =
520 pvr_get_slc_cache_line_size(&device->pdevice->dev_info);
521 struct pvr_pds_kickusc_program program = { 0 };
522 const pco_precomp_data *precomp_data;
523 uint32_t staging_buffer_size;
524 uint32_t *staging_buffer;
525 VkResult result;
526
527 precomp_data = (pco_precomp_data *)pco_usclib_common[FS_NOP_COMMON];
528 result = pvr_gpu_upload_usc(device,
529 precomp_data->binary,
530 precomp_data->size_dwords * sizeof(uint32_t),
531 cache_line_size,
532 &device->nop_program.usc);
533 if (result != VK_SUCCESS)
534 return result;
535
536 /* Setup a PDS program that kicks the static USC program. */
537 pvr_pds_setup_doutu(&program.usc_task_control,
538 device->nop_program.usc->dev_addr.addr,
539 precomp_data->temps,
540 ROGUE_PDSINST_DOUTU_SAMPLE_RATE_INSTANCE,
541 false);
542
543 pvr_pds_set_sizes_pixel_shader(&program);
544
545 staging_buffer_size = PVR_DW_TO_BYTES(program.code_size + program.data_size);
546
547 staging_buffer = vk_alloc(&device->vk.alloc,
548 staging_buffer_size,
549 8U,
550 VK_SYSTEM_ALLOCATION_SCOPE_COMMAND);
551 if (!staging_buffer) {
552 result = vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
553 goto err_free_nop_usc_bo;
554 }
555
556 pvr_pds_generate_pixel_shader_program(&program, staging_buffer);
557
558 /* FIXME: Figure out the define for alignment of 16. */
559 result = pvr_gpu_upload_pds(device,
560 staging_buffer,
561 program.data_size,
562 16U,
563 &staging_buffer[program.data_size],
564 program.code_size,
565 16U,
566 16U,
567 &device->nop_program.pds);
568 if (result != VK_SUCCESS)
569 goto err_free_staging_buffer;
570
571 vk_free(&device->vk.alloc, staging_buffer);
572
573 return VK_SUCCESS;
574
575 err_free_staging_buffer:
576 vk_free(&device->vk.alloc, staging_buffer);
577
578 err_free_nop_usc_bo:
579 pvr_bo_suballoc_free(device->nop_program.usc);
580
581 return result;
582 }
583
584 static VkResult
585 pvr_device_init_view_index_init_programs(struct pvr_device *device)
586 {
587 uint32_t staging_buffer_size = 0;
588 uint32_t *staging_buffer = NULL;
589 VkResult result;
590 unsigned i;
591
592 for (i = 0; i < PVR_MAX_MULTIVIEW; ++i) {
593 struct pvr_pds_view_index_init_program *program =
594 &device->view_index_init_info[i];
595
596 program->view_index = i;
597
598 pvr_pds_generate_view_index_init_program(program,
599 NULL,
600 PDS_GENERATE_SIZES);
601
602 if (program->data_size + program->code_size > staging_buffer_size) {
603 staging_buffer_size = program->data_size + program->code_size;
604
605 staging_buffer = vk_realloc(&device->vk.alloc,
606 staging_buffer,
607 staging_buffer_size,
608 8U,
609 VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
610
611 if (!staging_buffer) {
612 result = vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
613 break;
614 }
615 }
616
617 pvr_pds_generate_view_index_init_program(program,
618 staging_buffer,
619 PDS_GENERATE_DATA_SEGMENT);
620 pvr_pds_generate_view_index_init_program(
621 program,
622 &staging_buffer[program->data_size],
623 PDS_GENERATE_CODE_SEGMENT);
624
625 result =
626 pvr_gpu_upload_pds(device,
627 (program->data_size == 0 ? NULL : staging_buffer),
628 program->data_size / sizeof(uint32_t),
629 16U,
630 &staging_buffer[program->data_size],
631 program->code_size / sizeof(uint32_t),
632 16U,
633 16U,
634 &device->view_index_init_programs[i]);
635
636 if (result != VK_SUCCESS)
637 break;
638 }
639
640 vk_free(&device->vk.alloc, staging_buffer);
641
642 if (result != VK_SUCCESS)
643 for (uint32_t u = 0; u < i; ++u)
644 pvr_bo_suballoc_free(device->view_index_init_programs[u].pvr_bo);
645
646 return result;
647 }
648
649 static void pvr_device_init_tile_buffer_state(struct pvr_device *device)
650 {
651 simple_mtx_init(&device->tile_buffer_state.mtx, mtx_plain);
652
653 for (uint32_t i = 0; i < ARRAY_SIZE(device->tile_buffer_state.buffers); i++)
654 device->tile_buffer_state.buffers[i] = NULL;
655
656 device->tile_buffer_state.buffer_count = 0;
657 }
658
659 static void pvr_device_finish_tile_buffer_state(struct pvr_device *device)
660 {
661 /* Destroy the mutex first to trigger asserts in case it's still locked so
662 * that we don't put things in an inconsistent state by freeing buffers that
663 * might be in use or attempt to free buffers while new buffers are being
664 * allocated.
665 */
666 simple_mtx_destroy(&device->tile_buffer_state.mtx);
667 pvr_device_free_tile_buffer_state(device);
668 }
669
670 static void pvr_device_init_default_sampler_state(struct pvr_device *device)
671 {
672 pvr_csb_pack (&device->input_attachment_sampler,
673 TEXSTATE_SAMPLER_WORD0,
674 sampler) {
675 sampler.addrmode_u = ROGUE_TEXSTATE_ADDRMODE_CLAMP_TO_EDGE;
676 sampler.addrmode_v = ROGUE_TEXSTATE_ADDRMODE_CLAMP_TO_EDGE;
677 sampler.addrmode_w = ROGUE_TEXSTATE_ADDRMODE_CLAMP_TO_EDGE;
678 sampler.dadjust = ROGUE_TEXSTATE_DADJUST_ZERO_UINT;
679 sampler.magfilter = ROGUE_TEXSTATE_FILTER_POINT;
680 sampler.minfilter = ROGUE_TEXSTATE_FILTER_POINT;
681 sampler.anisoctl = ROGUE_TEXSTATE_ANISOCTL_DISABLED;
682 sampler.non_normalized_coords = true;
683 }
684 }
685
686 VkResult PVR_PER_ARCH(create_device)(struct pvr_physical_device *pdevice,
687 const VkDeviceCreateInfo *pCreateInfo,
688 const VkAllocationCallbacks *pAllocator,
689 VkDevice *pDevice)
690 {
691 uint32_t initial_free_list_size = PVR_GLOBAL_FREE_LIST_INITIAL_SIZE;
692 struct pvr_instance *instance = pdevice->instance;
693 struct vk_device_dispatch_table dispatch_table;
694 struct pvr_device *device;
695 struct pvr_winsys *ws;
696 VkResult result;
697
698 assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO);
699
700 result = pvr_winsys_create(pdevice->render_path,
701 pdevice->display_path, false,
702 pAllocator ? pAllocator : &instance->vk.alloc,
703 &ws);
704 if (result != VK_SUCCESS)
705 goto err_out;
706
707 device = vk_alloc2(&instance->vk.alloc,
708 pAllocator,
709 sizeof(*device),
710 8,
711 VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
712 if (!device) {
713 result = vk_error(instance, VK_ERROR_OUT_OF_HOST_MEMORY);
714 goto err_pvr_winsys_destroy;
715 }
716
717 vk_device_dispatch_table_from_entrypoints(&dispatch_table,
718 &PVR_PER_ARCH(device_entrypoints),
719 true);
720 vk_device_dispatch_table_from_entrypoints(&dispatch_table,
721 &pvr_device_entrypoints,
722 false);
723
724 vk_device_dispatch_table_from_entrypoints(&dispatch_table,
725 &wsi_device_entrypoints,
726 false);
727
728 result = vk_device_init(&device->vk,
729 &pdevice->vk,
730 &dispatch_table,
731 pCreateInfo,
732 pAllocator);
733 if (result != VK_SUCCESS)
734 goto err_free_device;
735
736 device->instance = instance;
737 device->pdevice = pdevice;
738 device->ws = ws;
739
740 vk_device_set_drm_fd(&device->vk, ws->render_fd);
741
742 if (ws->features.supports_threaded_submit) {
743 /* Queue submission can be blocked if the kernel CCBs become full,
744 * so enable threaded submit to not block the submitter.
745 */
746 vk_device_enable_threaded_submit(&device->vk);
747 }
748
749 ws->ops->get_heaps_info(ws, &device->heaps);
750
751 result = pvr_bo_store_create(device);
752 if (result != VK_SUCCESS)
753 goto err_vk_device_finish;
754
755 pvr_bo_suballocator_init(&device->suballoc_general,
756 device->heaps.general_heap,
757 device,
758 PVR_SUBALLOCATOR_GENERAL_SIZE);
759 pvr_bo_suballocator_init(&device->suballoc_pds,
760 device->heaps.pds_heap,
761 device,
762 PVR_SUBALLOCATOR_PDS_SIZE);
763 pvr_bo_suballocator_init(&device->suballoc_transfer,
764 device->heaps.transfer_frag_heap,
765 device,
766 PVR_SUBALLOCATOR_TRANSFER_SIZE);
767 pvr_bo_suballocator_init(&device->suballoc_usc,
768 device->heaps.usc_heap,
769 device,
770 PVR_SUBALLOCATOR_USC_SIZE);
771 pvr_bo_suballocator_init(&device->suballoc_vis_test,
772 device->heaps.vis_test_heap,
773 device,
774 PVR_SUBALLOCATOR_VIS_TEST_SIZE);
775
776 if (p_atomic_inc_return(&instance->active_device_count) >
777 PVR_SECONDARY_DEVICE_THRESHOLD) {
778 initial_free_list_size = PVR_SECONDARY_DEVICE_FREE_LIST_INITAL_SIZE;
779 }
780
781 result = pvr_free_list_create(device,
782 initial_free_list_size,
783 PVR_GLOBAL_FREE_LIST_MAX_SIZE,
784 PVR_GLOBAL_FREE_LIST_GROW_SIZE,
785 PVR_GLOBAL_FREE_LIST_GROW_THRESHOLD,
786 NULL /* parent_free_list */,
787 &device->global_free_list);
788 if (result != VK_SUCCESS)
789 goto err_dec_device_count;
790
791 result = pvr_device_init_nop_program(device);
792 if (result != VK_SUCCESS)
793 goto err_pvr_free_list_destroy;
794
795 result = pvr_device_init_compute_fence_program(device);
796 if (result != VK_SUCCESS)
797 goto err_pvr_free_nop_program;
798
799 result = pvr_device_init_compute_empty_program(device);
800 if (result != VK_SUCCESS)
801 goto err_pvr_free_compute_fence;
802
803 result = pvr_device_init_view_index_init_programs(device);
804 if (result != VK_SUCCESS)
805 goto err_pvr_free_compute_empty;
806
807 result = pvr_arch_device_create_compute_query_programs(device);
808 if (result != VK_SUCCESS)
809 goto err_pvr_free_view_index;
810
811 result = pvr_device_init_compute_idfwdf_state(device);
812 if (result != VK_SUCCESS)
813 goto err_pvr_destroy_compute_query_programs;
814
815 result = pvr_device_init_graphics_static_clear_state(device);
816 if (result != VK_SUCCESS)
817 goto err_pvr_finish_compute_idfwdf;
818
819 result = pvr_arch_device_init_spm_load_state(device);
820 if (result != VK_SUCCESS)
821 goto err_pvr_finish_graphics_static_clear_state;
822
823 pvr_device_init_tile_buffer_state(device);
824
825 result = pvr_arch_queues_create(device, pCreateInfo);
826 if (result != VK_SUCCESS)
827 goto err_pvr_finish_tile_buffer_state;
828
829 pvr_device_init_default_sampler_state(device);
830
831 pvr_spm_init_scratch_buffer_store(device);
832
833 result = pvr_init_robustness_buffer(device);
834 if (result != VK_SUCCESS)
835 goto err_pvr_spm_finish_scratch_buffer_store;
836
837 result = pvr_arch_border_color_table_init(device);
838 if (result != VK_SUCCESS)
839 goto err_pvr_robustness_buffer_finish;
840
841 /* FIXME: Move this to a later stage and possibly somewhere other than
842 * pvr_device. The purpose of this is so that we don't have to get the size
843 * on each kick.
844 */
845 pvr_device_get_pixel_event_pds_program_data_size(
846 &pdevice->dev_info,
847 &device->pixel_event_data_size_in_dwords);
848
849 device->global_cmd_buffer_submit_count = 0;
850 device->global_queue_present_count = 0;
851
852 simple_mtx_init(&device->rs_mtx, mtx_plain);
853 list_inithead(&device->render_states);
854
855 *pDevice = pvr_device_to_handle(device);
856
857 return VK_SUCCESS;
858
859 err_pvr_robustness_buffer_finish:
860 pvr_robustness_buffer_finish(device);
861
862 err_pvr_spm_finish_scratch_buffer_store:
863 pvr_spm_finish_scratch_buffer_store(device);
864
865 pvr_arch_queues_destroy(device);
866
867 err_pvr_finish_tile_buffer_state:
868 pvr_device_finish_tile_buffer_state(device);
869 pvr_arch_device_finish_spm_load_state(device);
870
871 err_pvr_finish_graphics_static_clear_state:
872 pvr_device_finish_graphics_static_clear_state(device);
873
874 err_pvr_finish_compute_idfwdf:
875 pvr_device_finish_compute_idfwdf_state(device);
876
877 err_pvr_destroy_compute_query_programs:
878 pvr_arch_device_destroy_compute_query_programs(device);
879
880 err_pvr_free_view_index:
881 for (uint32_t u = 0; u < PVR_MAX_MULTIVIEW; ++u)
882 pvr_bo_suballoc_free(device->view_index_init_programs[u].pvr_bo);
883
884 err_pvr_free_compute_empty:
885 pvr_bo_suballoc_free(device->pds_compute_empty_program.pvr_bo);
886
887 err_pvr_free_compute_fence:
888 pvr_bo_suballoc_free(device->pds_compute_fence_program.pvr_bo);
889
890 err_pvr_free_nop_program:
891 pvr_bo_suballoc_free(device->nop_program.pds.pvr_bo);
892 pvr_bo_suballoc_free(device->nop_program.usc);
893
894 err_pvr_free_list_destroy:
895 pvr_free_list_destroy(device->global_free_list);
896
897 err_dec_device_count:
898 p_atomic_dec(&device->instance->active_device_count);
899
900 pvr_bo_suballocator_fini(&device->suballoc_vis_test);
901 pvr_bo_suballocator_fini(&device->suballoc_usc);
902 pvr_bo_suballocator_fini(&device->suballoc_transfer);
903 pvr_bo_suballocator_fini(&device->suballoc_pds);
904 pvr_bo_suballocator_fini(&device->suballoc_general);
905
906 pvr_bo_store_destroy(device);
907
908 err_vk_device_finish:
909 vk_device_finish(&device->vk);
910
911 err_free_device:
912 vk_free(&device->vk.alloc, device);
913
914 err_pvr_winsys_destroy:
915 pvr_winsys_destroy(ws);
916
917 err_out:
918 return result;
919 }
920
921 void PVR_PER_ARCH(destroy_device)(struct pvr_device *device,
922 const VkAllocationCallbacks *pAllocator)
923 {
924 if (!device)
925 return;
926
927 simple_mtx_lock(&device->rs_mtx);
928 list_for_each_entry_safe (struct pvr_render_state,
929 rstate,
930 &device->render_states,
931 link) {
932 pvr_render_state_cleanup(device, pAllocator, rstate);
933 list_del(&rstate->link);
934
935 vk_free(&device->vk.alloc, rstate);
936 }
937 simple_mtx_unlock(&device->rs_mtx);
938 simple_mtx_destroy(&device->rs_mtx);
939
940 pvr_arch_border_color_table_finish(device);
941 pvr_robustness_buffer_finish(device);
942 pvr_spm_finish_scratch_buffer_store(device);
943 pvr_arch_queues_destroy(device);
944 pvr_device_finish_tile_buffer_state(device);
945 pvr_arch_device_finish_spm_load_state(device);
946 pvr_device_finish_graphics_static_clear_state(device);
947 pvr_device_finish_compute_idfwdf_state(device);
948 pvr_arch_device_destroy_compute_query_programs(device);
949 pvr_bo_suballoc_free(device->pds_compute_empty_program.pvr_bo);
950
951 for (uint32_t u = 0; u < PVR_MAX_MULTIVIEW; ++u)
952 pvr_bo_suballoc_free(device->view_index_init_programs[u].pvr_bo);
953
954 pvr_bo_suballoc_free(device->pds_compute_fence_program.pvr_bo);
955 pvr_bo_suballoc_free(device->nop_program.pds.pvr_bo);
956 pvr_bo_suballoc_free(device->nop_program.usc);
957 pvr_free_list_destroy(device->global_free_list);
958 pvr_bo_suballocator_fini(&device->suballoc_vis_test);
959 pvr_bo_suballocator_fini(&device->suballoc_usc);
960 pvr_bo_suballocator_fini(&device->suballoc_transfer);
961 pvr_bo_suballocator_fini(&device->suballoc_pds);
962 pvr_bo_suballocator_fini(&device->suballoc_general);
963 pvr_bo_store_destroy(device);
964 pvr_winsys_destroy(device->ws);
965 p_atomic_dec(&device->instance->active_device_count);
966 vk_device_finish(&device->vk);
967 vk_free(&device->vk.alloc, device);
968 }