Debian Codegraph @

src/imagination/vulkan/pvr_arch_hw_pass.c

raw
1 /*
2 * Copyright © 2022 Imagination Technologies Ltd.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a copy
5 * of this software and associated documentation files (the "Software"), to deal
6 * in the Software without restriction, including without limitation the rights
7 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 * copies of the Software, and to permit persons to whom the Software is
9 * furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 * SOFTWARE.
22 */
23
24 #include <assert.h>
25 #include <limits.h>
26 #include <stdbool.h>
27 #include <stddef.h>
28 #include <string.h>
29 #include <vulkan/vulkan.h>
30
31 #include "hwdef/rogue_hw_defs.h"
32 #include "hwdef/rogue_hw_utils.h"
33 #include "pvr_device.h"
34 #include "pvr_hw_pass.h"
35 #include "pvr_formats.h"
36 #include "pvr_pass.h"
37 #include "pvr_physical_device.h"
38 #include "util/bitset.h"
39 #include "util/list.h"
40 #include "util/macros.h"
41 #include "util/u_math.h"
42 #include "vk_alloc.h"
43 #include "vk_format.h"
44 #include "vk_log.h"
45
46 struct pvr_render_int_subpass {
47 /* Points to the input subpass. This is set to NULL when the subpass is
48 * unscheduled.
49 */
50 struct pvr_render_subpass *subpass;
51
52 /* Count of other subpasses which have this subpass as a dependency. */
53 uint32_t out_subpass_count;
54
55 /* Pointers to the other subpasses which have this subpass as a dependency.
56 */
57 struct pvr_render_int_subpass **out_subpasses;
58
59 /* Count of subpasses on which this subpass is dependent and which haven't
60 * been scheduled yet.
61 */
62 uint32_t in_subpass_count;
63 };
64
65 struct pvr_renderpass_resource {
66 /* Resource type allocated for render target. */
67 enum usc_mrt_resource_type type;
68
69 union {
70 /* If type == USC_MRT_RESOURCE_TYPE_OUTPUT_REG. */
71 struct {
72 /* The output register to use. */
73 uint32_t output_reg;
74
75 /* The offset in bytes within the output register. */
76 uint32_t offset;
77 } reg;
78
79 /* If type == USC_MRT_RESOURCE_TYPE_MEMORY. */
80 struct {
81 /* The index of the tile buffer to use. */
82 uint32_t tile_buffer;
83
84 /* The offset (in dwords) within the tile buffer. */
85 uint32_t offset_dw;
86 } mem;
87 };
88 };
89
90 struct pvr_render_int_attachment {
91 /* Points to the corresponding input attachment. */
92 struct pvr_render_pass_attachment *attachment;
93
94 /* True if this attachment is referenced in the currently open render. */
95 bool is_used;
96
97 /* Operation to use when this attachment is non-resident and referenced as a
98 * color or depth attachment.
99 */
100 VkAttachmentLoadOp load_op;
101
102 /* Operation to use for the stencil component when this attachment is
103 * non-resident and referenced as a color or depth attachment.
104 */
105 VkAttachmentLoadOp stencil_load_op;
106
107 /* Count of uses of this attachment in unscheduled subpasses. */
108 uint32_t remaining_count;
109
110 /* Count of uses of the stencil component of this attachment in unscheduled
111 * subpasses.
112 */
113 uint32_t stencil_remaining_count;
114
115 /* If this attachment has currently allocated on-chip storage then details of
116 * the allocated location.
117 */
118 struct usc_mrt_resource resource;
119
120 /* Index of the subpass in the current render where the attachment is first
121 * used. VK_ATTACHMENT_UNUSED if the attachment isn't used in the current
122 * render.
123 */
124 int32_t first_use;
125
126 /* Index of the subpass in the current render where the attachment is last
127 * used.
128 */
129 int32_t last_use;
130
131 /* Index of the subpass (global) where the attachment is last read. */
132 int32_t last_read;
133
134 /* If this attachment has currently allocated on-chip storage then the entry
135 * in context.active_surf_list.
136 */
137 struct list_head link;
138
139 /* During pvr_close_render: if this attachment has allocated on-chip storage
140 * then the index in pvr_renderpass_hwsetup_render.eot_setup.mrt_resources
141 * with details of the storage location. Otherwise -1.
142 */
143 int32_t mrt_idx;
144
145 /* Index of the last render where the attachment was the source of an MSAA
146 * resolve.
147 */
148 int32_t last_resolve_src_render;
149
150 /* Index of the last render where the attachment was the destination of an
151 * MSAA resolve.
152 */
153 int32_t last_resolve_dst_render;
154
155 /* true if the attachment is used with a z replicate in the current render.
156 */
157 bool z_replicate;
158
159 /* true if this attachment can be resolved by the PBE. */
160 bool is_pbe_downscalable;
161
162 /* true if this attachment requires an EOT attachment. */
163 bool eot_surf_required;
164 };
165
166 /* Which parts of the output registers/a tile buffer are currently allocated. */
167 struct pvr_renderpass_alloc_buffer {
168 /* Bit array. A bit is set if the corresponding dword is allocated. */
169 BITSET_DECLARE(allocs, 8U);
170 };
171
172 struct pvr_renderpass_alloc {
173 /* Which pixel output registers are allocated. */
174 struct pvr_renderpass_alloc_buffer output_reg;
175
176 /* Range of allocated output registers. */
177 uint32_t output_regs_count;
178
179 /* Number of tile buffers allocated. */
180 uint32_t tile_buffers_count;
181
182 /* Which parts of each tile buffer are allocated. Length is
183 * tile_buffers_count.
184 */
185 struct pvr_renderpass_alloc_buffer tile_buffers[PVR_MAX_TILE_BUFFER_COUNT];
186 };
187
188 struct pvr_renderpass_subpass {
189 /* A pointer to the input subpass description. */
190 struct pvr_render_subpass *input_subpass;
191
192 /* true if the depth attachment for this subpass has z replication enabled.
193 */
194 bool z_replicate;
195
196 /* Which pixel output registers/tile buffer locations are allocated during
197 * this subpass.
198 */
199 struct pvr_renderpass_alloc alloc;
200 };
201
202 struct pvr_renderpass_context {
203 /* Internal information about each input attachment. */
204 struct pvr_render_int_attachment *int_attach;
205
206 /* Internal information about each input subpass. */
207 struct pvr_render_int_subpass *int_subpasses;
208
209 /* Input structure. */
210 struct pvr_render_pass *pass;
211
212 /* Output structure. */
213 struct pvr_renderpass_hwsetup *hw_setup;
214
215 /* In-progress render. */
216 struct pvr_renderpass_hwsetup_render *hw_render;
217
218 /* Information about each subpass in the current render. */
219 struct pvr_renderpass_subpass *subpasses;
220
221 /* Which parts of color storage are currently allocated. */
222 struct pvr_renderpass_alloc alloc;
223
224 /* Attachment which is currently allocated the on-chip depth/stencil. */
225 struct pvr_render_int_attachment *int_ds_attach;
226
227 /* Attachment which is loaded into the on-chip depth/stencil at the start of
228 * the render.
229 */
230 struct pvr_render_int_attachment *ds_load_surface;
231
232 /* Attachment which the depth/stencil attachment should be resolved to at the
233 * end of the render.
234 */
235 struct pvr_render_int_attachment *ds_resolve_surface;
236
237 /* Count of surfaces which are allocated on-chip color storage. */
238 uint32_t active_surfaces;
239
240 /* List of attachment/ranges which are allocated on-chip color storage. */
241 struct list_head active_surf_list;
242
243 const VkAllocationCallbacks *allocator;
244 };
245
246 struct pvr_render_int_subpass_dsts {
247 struct pvr_renderpass_resource *color;
248 struct pvr_renderpass_resource incoming_zrep;
249 struct pvr_renderpass_resource existing_zrep;
250 };
251
252 struct pvr_render_subpass_depth_params {
253 bool existing_ds_is_input;
254 bool incoming_ds_is_input;
255 uint32_t existing_ds_attach;
256 };
257
258 struct pvr_renderpass_storage_firstuse_buffer {
259 /* For each pixel output register/tile buffer location: true if the output
260 * register has been allocated in the current render.
261 */
262 bool used[8U];
263 };
264
265 struct pvr_renderpass_storage_firstuse {
266 /* First use information for pixel output registers. */
267 struct pvr_renderpass_storage_firstuse_buffer output_reg;
268
269 /* First use information for tile buffers. */
270 struct pvr_renderpass_storage_firstuse_buffer *tile_buffers;
271 };
272
273 static uint32_t pvr_get_accum_format_bitsize(VkFormat vk_format)
274 {
275 if (util_format_has_depth(vk_format_description(vk_format)))
276 return vk_format_get_blocksizebits(vk_format);
277
278 if (!vk_format_has_stencil(vk_format))
279 return pvr_get_pbe_accum_format_size_in_bytes(vk_format) * 8;
280
281 return 0;
282 }
283
284 /** Copy information about allocated color storage. */
285 static VkResult pvr_copy_alloc(struct pvr_renderpass_context *ctx,
286 struct pvr_renderpass_alloc *dst,
287 struct pvr_renderpass_alloc *src)
288 {
289 dst->output_reg = src->output_reg;
290 dst->output_regs_count = src->output_regs_count;
291
292 dst->tile_buffers_count = src->tile_buffers_count;
293 if (dst->tile_buffers_count > 0U) {
294 memcpy(dst->tile_buffers,
295 src->tile_buffers,
296 sizeof(dst->tile_buffers[0U]) * dst->tile_buffers_count);
297 }
298
299 return VK_SUCCESS;
300 }
301
302 /** Free information about allocated color storage. */
303 static void pvr_free_alloc(struct pvr_renderpass_context *ctx,
304 struct pvr_renderpass_alloc *alloc)
305 {
306 memset(alloc, 0U, sizeof(*alloc));
307 }
308
309 static void pvr_reset_render(struct pvr_renderpass_context *ctx)
310 {
311 ctx->int_ds_attach = NULL;
312 ctx->active_surfaces = 0U;
313 list_inithead(&ctx->active_surf_list);
314
315 memset(&ctx->alloc.output_reg, 0U, sizeof(ctx->alloc.output_reg));
316 ctx->alloc.output_regs_count = 0U;
317 ctx->alloc.tile_buffers_count = 0U;
318 memset(&ctx->alloc.tile_buffers, 0U, sizeof(ctx->alloc.tile_buffers));
319
320 ctx->hw_render = NULL;
321 ctx->subpasses = NULL;
322 ctx->ds_load_surface = NULL;
323 }
324
325 static void
326 pvr_finalise_mrt_setup(const struct pvr_device *device,
327 struct pvr_renderpass_hwsetup_render *hw_render,
328 struct usc_mrt_setup *mrt)
329 {
330 mrt->num_output_regs = hw_render->output_regs_count;
331 mrt->num_tile_buffers = hw_render->tile_buffers_count;
332 }
333
334 /**
335 * Copy information about the number of pixel output registers and tile buffers
336 * required for the current render to the output structure.
337 */
338 static void pvr_finalise_po_alloc(const struct pvr_device *device,
339 struct pvr_renderpass_context *ctx)
340 {
341 struct pvr_renderpass_hwsetup_render *hw_render = ctx->hw_render;
342
343 /* The number of output registers must be a power of two. */
344 hw_render->output_regs_count =
345 util_next_power_of_two(ctx->alloc.output_regs_count);
346
347 assert(ctx->alloc.tile_buffers_count <= ctx->pass->max_tilebuffer_count);
348 hw_render->tile_buffers_count = ctx->alloc.tile_buffers_count;
349
350 /* Copy the number of output registers and tile buffers to each subpass. */
351 for (uint32_t i = 0U; i < hw_render->subpass_count; i++) {
352 struct pvr_renderpass_hwsetup_subpass *hw_subpass =
353 &hw_render->subpasses[i];
354
355 pvr_finalise_mrt_setup(device, hw_render, &hw_subpass->setup);
356 }
357
358 pvr_finalise_mrt_setup(device, hw_render, &hw_render->init_setup);
359 pvr_finalise_mrt_setup(device, hw_render, &hw_render->eot_setup);
360 }
361
362 /** Mark that device memory must be allocated for an attachment. */
363 static void pvr_mark_surface_alloc(struct pvr_renderpass_context *ctx,
364 struct pvr_render_int_attachment *int_attach)
365 {
366 const uint32_t attach_idx = int_attach - ctx->int_attach;
367
368 assert(attach_idx < ctx->pass->attachment_count);
369 ctx->hw_setup->surface_allocate[attach_idx] = true;
370 }
371
372 /**
373 * Check if there is space in a buffer for storing a render target of a
374 * specified size.
375 */
376 static int32_t
377 pvr_is_space_in_buffer(const struct pvr_device_info *dev_info,
378 struct pvr_renderpass_alloc_buffer *buffer,
379 uint32_t pixel_size)
380 {
381 const uint32_t max_out_regs = rogue_get_max_output_regs_per_pixel(dev_info);
382 uint32_t alignment = 1U;
383
384 if (PVR_HAS_FEATURE(dev_info, pbe2_in_xe)) {
385 /* For a 64-bit/128-bit source format: the start offset must be even. */
386 if (pixel_size == 2U || pixel_size == 4U)
387 alignment = 2U;
388 }
389
390 assert(pixel_size <= max_out_regs);
391
392 for (uint32_t i = 0U; i <= (max_out_regs - pixel_size); i += alignment) {
393 if (!BITSET_TEST_COUNT(buffer->allocs, i, pixel_size))
394 return i;
395 }
396
397 return -1;
398 }
399
400 static VkResult
401 pvr_surface_setup_render_init(struct pvr_renderpass_context *ctx,
402 struct pvr_renderpass_storage_firstuse *first_use,
403 struct usc_mrt_resource const *resource,
404 struct pvr_render_pass_attachment *attachment,
405 VkAttachmentLoadOp load_op,
406 bool *use_render_init)
407 {
408 const uint32_t pixel_size =
409 DIV_ROUND_UP(pvr_get_accum_format_bitsize(attachment->vk_format), 32U);
410 struct pvr_renderpass_hwsetup_render *hw_render = ctx->hw_render;
411 struct pvr_renderpass_storage_firstuse_buffer *buffer;
412 uint32_t start;
413
414 /* Check if this is the first use of all the allocated registers. */
415 if (resource->type == USC_MRT_RESOURCE_TYPE_OUTPUT_REG) {
416 buffer = &first_use->output_reg;
417 start = resource->reg.output_reg;
418 } else {
419 assert(resource->mem.tile_buffer < ctx->alloc.tile_buffers_count);
420 buffer = &first_use->tile_buffers[resource->mem.tile_buffer];
421 start = resource->mem.offset_dw;
422 }
423
424 *use_render_init = true;
425 for (uint32_t i = 0U; i < pixel_size; i++) {
426 /* Don't initialize at the render level if the output registers were
427 * previously allocated a different attachment.
428 */
429 if (buffer->used[start + i])
430 *use_render_init = false;
431
432 /* Don't use render init for future attachments allocated to the same
433 * registers.
434 */
435 buffer->used[start + i] = true;
436 }
437
438 if (load_op == VK_ATTACHMENT_LOAD_OP_DONT_CARE)
439 *use_render_init = false;
440
441 if (*use_render_init) {
442 struct pvr_renderpass_colorinit *new_color_init;
443 struct usc_mrt_resource *new_mrt;
444
445 /* Initialize the storage at the start of the render. */
446 new_color_init = vk_realloc(ctx->allocator,
447 hw_render->color_init,
448 sizeof(hw_render->color_init[0U]) *
449 (hw_render->color_init_count + 1U),
450 8U,
451 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
452 if (!new_color_init)
453 return vk_error(NULL, VK_ERROR_OUT_OF_HOST_MEMORY);
454
455 hw_render->color_init = new_color_init;
456 hw_render->color_init[hw_render->color_init_count].index =
457 attachment->index;
458 hw_render->color_init[hw_render->color_init_count].op = load_op;
459
460 /* Set the destination for the attachment load/clear. */
461 assert(hw_render->init_setup.num_render_targets ==
462 hw_render->color_init_count);
463
464 new_mrt = vk_realloc(ctx->allocator,
465 hw_render->init_setup.mrt_resources,
466 sizeof(hw_render->init_setup.mrt_resources[0U]) *
467 (hw_render->init_setup.num_render_targets + 1U),
468 8U,
469 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
470 if (!new_mrt)
471 return vk_error(NULL, VK_ERROR_OUT_OF_HOST_MEMORY);
472
473 hw_render->init_setup.mrt_resources = new_mrt;
474 hw_render->init_setup
475 .mrt_resources[hw_render->init_setup.num_render_targets] = *resource;
476 hw_render->init_setup.num_render_targets++;
477
478 hw_render->color_init_count++;
479 }
480
481 return VK_SUCCESS;
482 }
483
484 static VkResult
485 pvr_subpass_setup_render_init(struct pvr_renderpass_context *ctx)
486 {
487 struct pvr_renderpass_hwsetup_render *hw_render = ctx->hw_render;
488 struct pvr_renderpass_storage_firstuse first_use = { 0 };
489 bool first_ds = true;
490 VkResult result;
491
492 if (ctx->alloc.tile_buffers_count > 0U) {
493 first_use.tile_buffers = vk_zalloc(ctx->allocator,
494 sizeof(first_use.tile_buffers[0U]) *
495 ctx->alloc.tile_buffers_count,
496 8,
497 VK_SYSTEM_ALLOCATION_SCOPE_COMMAND);
498 if (!first_use.tile_buffers)
499 return vk_error(NULL, VK_ERROR_OUT_OF_HOST_MEMORY);
500 }
501
502 for (uint32_t i = 0U; i < hw_render->subpass_count; i++) {
503 struct pvr_renderpass_hwsetup_subpass *hw_subpass =
504 &hw_render->subpasses[i];
505 struct pvr_renderpass_subpass *subpass = &ctx->subpasses[i];
506 struct pvr_render_subpass *input_subpass = subpass->input_subpass;
507
508 /* If this is the first depth attachment in the render then clear at the
509 * render level, not the subpass level.
510 */
511 if (first_ds &&
512 (hw_subpass->depth_initop == VK_ATTACHMENT_LOAD_OP_CLEAR ||
513 hw_subpass->stencil_clear)) {
514 struct pvr_render_int_attachment *int_ds_attach;
515
516 assert(input_subpass->depth_stencil_attachment !=
517 VK_ATTACHMENT_UNUSED);
518 assert(input_subpass->depth_stencil_attachment <
519 ctx->pass->attachment_count);
520 int_ds_attach =
521 &ctx->int_attach[input_subpass->depth_stencil_attachment];
522
523 assert(hw_render->ds_attach_idx == VK_ATTACHMENT_UNUSED ||
524 hw_render->ds_attach_idx == int_ds_attach->attachment->index);
525 hw_render->ds_attach_idx = int_ds_attach->attachment->index;
526
527 if (hw_subpass->depth_initop == VK_ATTACHMENT_LOAD_OP_CLEAR)
528 hw_render->depth_init = VK_ATTACHMENT_LOAD_OP_CLEAR;
529
530 if (hw_subpass->stencil_clear) {
531 hw_render->stencil_init = VK_ATTACHMENT_LOAD_OP_CLEAR;
532 hw_subpass->stencil_clear = false;
533 }
534 }
535
536 if (input_subpass->depth_stencil_attachment != VK_ATTACHMENT_UNUSED)
537 first_ds = false;
538
539 for (uint32_t j = 0U; j < input_subpass->color_count; j++) {
540 struct usc_mrt_resource *mrt = &hw_subpass->setup.mrt_resources[j];
541 const uint32_t attach_idx = input_subpass->color_attachments[j];
542 struct pvr_render_int_attachment *int_attach;
543
544 if (attach_idx == VK_ATTACHMENT_UNUSED)
545 continue;
546
547 int_attach = &ctx->int_attach[attach_idx];
548
549 assert(pvr_get_accum_format_bitsize(
550 int_attach->attachment->vk_format) > 0U);
551
552 /* Is this the first use of the attachment? */
553 if (int_attach->first_use == (int32_t)i) {
554 /* Set if we should initialize the attachment storage at the
555 * render level.
556 */
557 bool use_render_init;
558 result = pvr_surface_setup_render_init(ctx,
559 &first_use,
560 mrt,
561 int_attach->attachment,
562 hw_subpass->color_initops[j],
563 &use_render_init);
564 if (result != VK_SUCCESS) {
565 vk_free(ctx->allocator, first_use.tile_buffers);
566 return result;
567 }
568
569 /* On success don't initialize the attachment at the subpass level.
570 */
571 if (use_render_init)
572 hw_subpass->color_initops[j] = VK_ATTACHMENT_LOAD_OP_DONT_CARE;
573 } else {
574 /* This attachment is already present in on-chip storage so don't
575 * do anything.
576 */
577 assert(hw_subpass->color_initops[j] ==
578 VK_ATTACHMENT_LOAD_OP_DONT_CARE);
579 }
580 }
581 }
582
583 vk_free(ctx->allocator, first_use.tile_buffers);
584
585 return VK_SUCCESS;
586 }
587
588 static void
589 pvr_mark_storage_allocated_in_buffer(struct pvr_renderpass_alloc_buffer *buffer,
590 uint32_t start,
591 uint32_t pixel_size)
592 {
593 assert(!BITSET_TEST_COUNT(buffer->allocs, start, pixel_size));
594 BITSET_SET_COUNT(buffer->allocs, start, pixel_size);
595 }
596
597 static VkResult
598 pvr_mark_storage_allocated(struct pvr_renderpass_context *ctx,
599 struct pvr_renderpass_alloc *alloc,
600 struct pvr_render_pass_attachment *attachment,
601 struct pvr_renderpass_resource *resource)
602 {
603 /* Number of dwords to allocate for the attachment. */
604 const uint32_t pixel_size =
605 DIV_ROUND_UP(pvr_get_accum_format_bitsize(attachment->vk_format), 32U);
606
607 if (resource->type == USC_MRT_RESOURCE_TYPE_OUTPUT_REG) {
608 /* Update the locations used in the pixel output registers. */
609 pvr_mark_storage_allocated_in_buffer(&alloc->output_reg,
610 resource->reg.output_reg,
611 pixel_size);
612
613 /* Update the range of pixel output registers used. */
614 alloc->output_regs_count =
615 MAX2(alloc->output_regs_count, resource->reg.output_reg + pixel_size);
616 } else {
617 assert(resource->type == USC_MRT_RESOURCE_TYPE_MEMORY);
618
619 if (resource->mem.tile_buffer >= alloc->tile_buffers_count) {
620 memset(
621 &alloc->tile_buffers[alloc->tile_buffers_count],
622 0U,
623 sizeof(alloc->tile_buffers[0U]) *
624 (resource->mem.tile_buffer + 1U - alloc->tile_buffers_count));
625 alloc->tile_buffers_count = resource->mem.tile_buffer + 1U;
626 assert(alloc->tile_buffers_count <= ctx->pass->max_tilebuffer_count);
627 }
628
629 /* Update the locations used in the tile buffer. */
630 pvr_mark_storage_allocated_in_buffer(
631 &alloc->tile_buffers[resource->mem.tile_buffer],
632 resource->mem.offset_dw,
633 pixel_size);
634
635 /* The hardware makes the bit depth of the on-chip storage and memory
636 * storage the same so make sure the memory storage is large enough to
637 * accommodate the largest render target.
638 */
639 alloc->output_regs_count =
640 MAX2(alloc->output_regs_count, resource->mem.offset_dw + pixel_size);
641 }
642
643 return VK_SUCCESS;
644 }
645
646 static VkResult
647 pvr_surface_alloc_color_storage(const struct pvr_device_info *dev_info,
648 struct pvr_renderpass_context *ctx,
649 struct pvr_renderpass_alloc *alloc,
650 struct pvr_render_pass_attachment *attachment,
651 struct pvr_renderpass_resource *resource)
652 {
653 /* Number of dwords to allocate for the attachment. */
654 const uint32_t pixel_size =
655 DIV_ROUND_UP(pvr_get_accum_format_bitsize(attachment->vk_format), 32U);
656
657 /* Try allocating pixel output registers. */
658 const int32_t output_reg =
659 pvr_is_space_in_buffer(dev_info, &alloc->output_reg, pixel_size);
660 if (output_reg != -1) {
661 resource->type = USC_MRT_RESOURCE_TYPE_OUTPUT_REG;
662 resource->reg.output_reg = (uint32_t)output_reg;
663 resource->reg.offset = 0U;
664 } else {
665 uint32_t i;
666
667 /* Mark the attachment as using a tile buffer. */
668 resource->type = USC_MRT_RESOURCE_TYPE_MEMORY;
669
670 /* Try allocating from an existing tile buffer. */
671 for (i = 0U; i < alloc->tile_buffers_count; i++) {
672 const int32_t tile_buffer_offset =
673 pvr_is_space_in_buffer(dev_info,
674 &alloc->tile_buffers[i],
675 pixel_size);
676
677 if (tile_buffer_offset != -1) {
678 resource->mem.tile_buffer = i;
679 resource->mem.offset_dw = (uint32_t)tile_buffer_offset;
680 break;
681 }
682 }
683
684 if (i == alloc->tile_buffers_count) {
685 /* Check for reaching the maximum number of tile buffers. */
686 if (alloc->tile_buffers_count == ctx->pass->max_tilebuffer_count)
687 return vk_error(NULL, VK_ERROR_TOO_MANY_OBJECTS);
688
689 /* Use a newly allocated tile buffer. */
690 resource->mem.tile_buffer = i;
691 resource->mem.offset_dw = 0U;
692 }
693 }
694
695 /* Update which parts of the pixel outputs/tile buffers are used. */
696 return pvr_mark_storage_allocated(ctx, alloc, attachment, resource);
697 }
698
699 /** Free the storage allocated to an attachment. */
700 static void
701 pvr_free_buffer_storage(struct pvr_renderpass_alloc_buffer *buffer,
702 struct pvr_render_int_attachment *int_attach,
703 uint32_t start)
704 {
705 const uint32_t pixel_size = DIV_ROUND_UP(
706 pvr_get_accum_format_bitsize(int_attach->attachment->vk_format),
707 32U);
708
709 BITSET_CLEAR_COUNT(buffer->allocs, start, pixel_size);
710 }
711
712 /** Free the storage allocated to an attachment. */
713 static void
714 pvr_free_surface_storage(struct pvr_renderpass_context *ctx,
715 struct pvr_render_int_attachment *int_attach)
716 {
717 struct usc_mrt_resource *resource = &int_attach->resource;
718 struct pvr_renderpass_alloc *alloc = &ctx->alloc;
719
720 assert(resource->type != USC_MRT_RESOURCE_TYPE_INVALID);
721
722 /* Mark the storage as free. */
723 if (resource->type == USC_MRT_RESOURCE_TYPE_OUTPUT_REG) {
724 pvr_free_buffer_storage(&alloc->output_reg,
725 int_attach,
726 resource->reg.output_reg);
727 } else {
728 struct pvr_renderpass_alloc_buffer *tile_buffer;
729
730 assert(resource->type == USC_MRT_RESOURCE_TYPE_MEMORY);
731
732 assert(resource->mem.tile_buffer < alloc->tile_buffers_count);
733 tile_buffer = &alloc->tile_buffers[resource->mem.tile_buffer];
734 pvr_free_buffer_storage(tile_buffer, int_attach, resource->mem.offset_dw);
735 }
736
737 /* Mark that the attachment doesn't have allocated storage. */
738 resource->type = USC_MRT_RESOURCE_TYPE_INVALID;
739
740 /* Remove from the list of surfaces with allocated on-chip storage. */
741 assert(ctx->active_surfaces > 0U);
742 ctx->active_surfaces--;
743 list_del(&int_attach->link);
744 }
745
746 static void pvr_reset_surface(struct pvr_renderpass_context *ctx,
747 struct pvr_render_int_attachment *int_attach)
748 {
749 /* Reset information about the range of uses. */
750 int_attach->first_use = int_attach->last_use = -1;
751 int_attach->z_replicate = false;
752
753 pvr_free_surface_storage(ctx, int_attach);
754 }
755
756 static void
757 pvr_make_surface_active(struct pvr_renderpass_context *ctx,
758 struct pvr_render_int_attachment *int_attach,
759 uint32_t subpass_num)
760 {
761 /* Add to the list of surfaces with on-chip storage. */
762 assert(int_attach->first_use == -1);
763 int_attach->first_use = subpass_num;
764 ctx->active_surfaces++;
765 list_addtail(&int_attach->link, &ctx->active_surf_list);
766 }
767
768 /**
769 * For a subpass copy details of storage locations for the input/color to the
770 * output structure.
771 */
772 static VkResult
773 pvr_copy_storage_details(struct pvr_renderpass_context *ctx,
774 struct pvr_renderpass_hwsetup_subpass *hw_subpass,
775 struct pvr_renderpass_subpass *subpass)
776 {
777 struct pvr_render_subpass *input_subpass = subpass->input_subpass;
778 const uint32_t max_rts =
779 input_subpass->color_count + input_subpass->input_count;
780 VkResult result;
781
782 if (max_rts == 0)
783 return VK_SUCCESS;
784
785 hw_subpass->setup.mrt_resources =
786 vk_zalloc(ctx->allocator,
787 sizeof(hw_subpass->setup.mrt_resources[0U]) * max_rts,
788 8,
789 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
790 if (!hw_subpass->setup.mrt_resources) {
791 result = vk_error(NULL, VK_ERROR_OUT_OF_HOST_MEMORY);
792 goto end_copy_storage_details;
793 }
794
795 for (uint32_t i = 0U; i < input_subpass->color_count; i++) {
796 const uint32_t attach_idx = input_subpass->color_attachments[i];
797 struct pvr_render_int_attachment *int_attach;
798
799 if (attach_idx == VK_ATTACHMENT_UNUSED)
800 continue;
801
802 int_attach = &ctx->int_attach[attach_idx];
803
804 /* Record for the subpass where the color attachment is stored. */
805 assert(int_attach->resource.type != USC_MRT_RESOURCE_TYPE_INVALID);
806 hw_subpass->setup.mrt_resources[i] = int_attach->resource;
807 }
808
809 hw_subpass->setup.num_render_targets = input_subpass->color_count;
810
811 if (input_subpass->input_count == 0)
812 return VK_SUCCESS;
813
814 /* For this subpass's input attachments. */
815 hw_subpass->input_access = vk_alloc(ctx->allocator,
816 sizeof(hw_subpass->input_access[0U]) *
817 input_subpass->input_count,
818 8,
819 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
820 if (!hw_subpass->input_access) {
821 result = vk_error(NULL, VK_ERROR_OUT_OF_HOST_MEMORY);
822 goto end_copy_storage_details;
823 }
824
825 for (uint32_t i = 0U; i < input_subpass->input_count; i++) {
826 const uint32_t attach_idx =
827 input_subpass->input_attachments[i].attachment_idx;
828 struct pvr_render_int_attachment *int_attach;
829
830 if (attach_idx == VK_ATTACHMENT_UNUSED)
831 continue;
832
833 int_attach = &ctx->int_attach[attach_idx];
834
835 if (int_attach->resource.type != USC_MRT_RESOURCE_TYPE_INVALID) {
836 bool is_color = false;
837
838 /* Access the input attachment from on-chip storage. */
839 if (int_attach->z_replicate) {
840 hw_subpass->input_access[i].type =
841 PVR_RENDERPASS_HWSETUP_INPUT_ACCESS_ONCHIP_ZREPLICATE;
842 } else {
843 hw_subpass->input_access[i].type =
844 PVR_RENDERPASS_HWSETUP_INPUT_ACCESS_ONCHIP;
845 }
846
847 /* If this attachment is also a color attachment then point to the
848 * color attachment's resource.
849 */
850 for (uint32_t j = 0U; j < input_subpass->color_count; j++) {
851 if (input_subpass->color_attachments[j] == (int32_t)attach_idx) {
852 hw_subpass->input_access[i].on_chip_rt = j;
853 is_color = true;
854 break;
855 }
856 }
857
858 if (!is_color) {
859 const uint32_t num_rts = hw_subpass->setup.num_render_targets;
860
861 hw_subpass->input_access[i].on_chip_rt = num_rts;
862 hw_subpass->setup.num_render_targets++;
863
864 /* Record the location of the storage for the attachment. */
865 hw_subpass->setup.mrt_resources[num_rts] = int_attach->resource;
866 }
867 } else {
868 /* Access the input attachment from memory. */
869 hw_subpass->input_access[i].type =
870 PVR_RENDERPASS_HWSETUP_INPUT_ACCESS_OFFCHIP;
871 hw_subpass->input_access[i].on_chip_rt = -1;
872 }
873 }
874
875 return VK_SUCCESS;
876
877 end_copy_storage_details:
878 if (hw_subpass->input_access) {
879 vk_free(ctx->allocator, hw_subpass->input_access);
880 hw_subpass->input_access = NULL;
881 }
882
883 if (hw_subpass->setup.mrt_resources) {
884 vk_free(ctx->allocator, hw_subpass->setup.mrt_resources);
885 hw_subpass->setup.mrt_resources = NULL;
886 }
887
888 return result;
889 }
890
891 /**
892 * For a subpass copy details of any storage location for a replicated version
893 * of the depth attachment to the output structure.
894 */
895 static VkResult
896 pvr_copy_z_replicate_details(struct pvr_renderpass_context *ctx,
897 struct pvr_renderpass_hwsetup_subpass *hw_subpass,
898 struct pvr_renderpass_subpass *subpass)
899 {
900 struct pvr_render_subpass *input_subpass = subpass->input_subpass;
901 struct pvr_render_int_attachment *int_ds_attach;
902 uint32_t z_replicate;
903 bool found = false;
904
905 assert(input_subpass->depth_stencil_attachment >= 0U &&
906 input_subpass->depth_stencil_attachment <
907 (int32_t)ctx->pass->attachment_count);
908
909 int_ds_attach = &ctx->int_attach[input_subpass->depth_stencil_attachment];
910
911 assert(hw_subpass->z_replicate == -1);
912
913 /* Is the replicated depth also an input attachment? */
914 for (uint32_t i = 0U; i < input_subpass->input_count; i++) {
915 const uint32_t attach_idx =
916 input_subpass->input_attachments[i].attachment_idx;
917 struct pvr_render_int_attachment *int_attach;
918
919 if (attach_idx == VK_ATTACHMENT_UNUSED)
920 continue;
921
922 int_attach = &ctx->int_attach[attach_idx];
923
924 if (int_attach == int_ds_attach) {
925 z_replicate = hw_subpass->input_access[i].on_chip_rt;
926 found = true;
927 break;
928 }
929 }
930
931 if (!found)
932 z_replicate = hw_subpass->setup.num_render_targets;
933
934 /* If the Z replicate attachment isn't also an input attachment then grow the
935 * array of locations.
936 */
937 assert(z_replicate <= hw_subpass->setup.num_render_targets);
938 if (z_replicate == hw_subpass->setup.num_render_targets) {
939 struct usc_mrt_resource *mrt =
940 vk_realloc(ctx->allocator,
941 hw_subpass->setup.mrt_resources,
942 sizeof(hw_subpass->setup.mrt_resources[0U]) *
943 (hw_subpass->setup.num_render_targets + 1U),
944 8U,
945 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
946 if (!mrt)
947 return vk_error(NULL, VK_ERROR_OUT_OF_HOST_MEMORY);
948
949 hw_subpass->setup.mrt_resources = mrt;
950 hw_subpass->setup.num_render_targets++;
951 }
952
953 /* Copy the location of the Z replicate. */
954 assert(int_ds_attach->resource.type != USC_MRT_RESOURCE_TYPE_INVALID);
955 hw_subpass->setup.mrt_resources[z_replicate] = int_ds_attach->resource;
956 hw_subpass->z_replicate = z_replicate;
957
958 return VK_SUCCESS;
959 }
960
961 static void pvr_dereference_surface(struct pvr_renderpass_context *ctx,
962 int32_t attach_idx,
963 uint32_t subpass_num)
964 {
965 struct pvr_render_int_attachment *int_attach = &ctx->int_attach[attach_idx];
966
967 assert(int_attach->remaining_count > 0U);
968 int_attach->remaining_count--;
969
970 if (int_attach->remaining_count == 0U) {
971 if (int_attach->first_use != -1)
972 int_attach->last_use = subpass_num;
973
974 if (int_attach->resource.type != USC_MRT_RESOURCE_TYPE_INVALID)
975 pvr_free_surface_storage(ctx, int_attach);
976 }
977
978 if (int_attach->attachment->aspects & VK_IMAGE_ASPECT_STENCIL_BIT) {
979 assert(int_attach->stencil_remaining_count > 0U);
980 int_attach->stencil_remaining_count--;
981 }
982 }
983
984 static void pvr_free_render(struct pvr_renderpass_context *ctx)
985 {
986 pvr_free_alloc(ctx, &ctx->alloc);
987
988 if (ctx->subpasses) {
989 for (uint32_t i = 0U; i < ctx->hw_render->subpass_count; i++)
990 pvr_free_alloc(ctx, &ctx->subpasses[i].alloc);
991
992 vk_free(ctx->allocator, ctx->subpasses);
993 ctx->subpasses = NULL;
994 }
995 }
996
997 static bool pvr_render_has_side_effects(struct pvr_renderpass_context *ctx)
998 {
999 struct pvr_renderpass_hwsetup_render *hw_render = ctx->hw_render;
1000 struct pvr_render_pass *pass = ctx->pass;
1001
1002 if ((hw_render->depth_init == VK_ATTACHMENT_LOAD_OP_CLEAR &&
1003 hw_render->depth_store) ||
1004 (hw_render->stencil_init == VK_ATTACHMENT_LOAD_OP_CLEAR &&
1005 hw_render->stencil_store)) {
1006 return true;
1007 }
1008
1009 if (hw_render->stencil_resolve_mode || hw_render->depth_resolve_mode)
1010 return true;
1011
1012 for (uint32_t i = 0U; i < hw_render->eot_surface_count; i++) {
1013 const struct pvr_renderpass_hwsetup_eot_surface *eot_attach =
1014 &hw_render->eot_surfaces[i];
1015 const struct pvr_render_pass_attachment *attachment =
1016 &pass->attachments[eot_attach->attachment_idx];
1017
1018 if (attachment->load_op == VK_ATTACHMENT_LOAD_OP_CLEAR &&
1019 attachment->store_op == VK_ATTACHMENT_STORE_OP_STORE) {
1020 return true;
1021 }
1022
1023 if (eot_attach->need_resolve)
1024 return true;
1025 }
1026
1027 return false;
1028 }
1029
1030 static VkResult pvr_close_render(const struct pvr_device *device,
1031 struct pvr_renderpass_context *ctx)
1032 {
1033 struct pvr_renderpass_hwsetup_render *hw_render = ctx->hw_render;
1034 struct pvr_renderpass_hwsetup_eot_surface *eot_attach;
1035 struct usc_mrt_setup *eot_setup;
1036 int32_t mrt_idx;
1037 VkResult result;
1038
1039 /* Render already closed. */
1040 if (!hw_render)
1041 return VK_SUCCESS;
1042
1043 /* Setup render and allocate resources for color/depth loads and clears. */
1044 result = pvr_subpass_setup_render_init(ctx);
1045 if (result != VK_SUCCESS)
1046 return result;
1047
1048 /* Reset surfaces whose last use was in the current render. */
1049 list_for_each_entry_safe (struct pvr_render_int_attachment,
1050 int_attach,
1051 &ctx->active_surf_list,
1052 link) {
1053 if (int_attach->last_use != -1) {
1054 assert(int_attach->resource.type == USC_MRT_RESOURCE_TYPE_INVALID);
1055 pvr_reset_surface(ctx, int_attach);
1056 }
1057 }
1058
1059 /* Check if the depth attachment has uses in future subpasses. */
1060 if (ctx->int_ds_attach) {
1061 /* Store the depth to the attachment at the end of the render. */
1062 if (ctx->int_ds_attach->remaining_count > 0U)
1063 hw_render->depth_store = true;
1064
1065 /* Store the stencil to the attachment at the end of the render. */
1066 if (ctx->int_ds_attach->stencil_remaining_count > 0U)
1067 hw_render->stencil_store = true;
1068
1069 if (hw_render->depth_store || hw_render->stencil_store) {
1070 assert(hw_render->ds_attach_idx == VK_ATTACHMENT_UNUSED ||
1071 hw_render->ds_attach_idx ==
1072 ctx->int_ds_attach->attachment->index);
1073 hw_render->ds_attach_idx = ctx->int_ds_attach->attachment->index;
1074
1075 /* Allocate memory for the attachment. */
1076 pvr_mark_surface_alloc(ctx, ctx->int_ds_attach);
1077 }
1078
1079 if (ctx->hw_render->stencil_resolve_mode ||
1080 ctx->hw_render->depth_resolve_mode) {
1081 assert(ctx->ds_resolve_surface);
1082
1083 hw_render->ds_attach_resolve_idx =
1084 ctx->ds_resolve_surface - ctx->int_attach;
1085
1086 pvr_mark_surface_alloc(ctx, ctx->ds_resolve_surface);
1087 }
1088
1089 /* Load the depth and stencil before the next use. */
1090 ctx->int_ds_attach->load_op = VK_ATTACHMENT_LOAD_OP_LOAD;
1091 ctx->int_ds_attach->stencil_load_op = VK_ATTACHMENT_LOAD_OP_LOAD;
1092 }
1093
1094 eot_setup = &hw_render->eot_setup;
1095 memset(eot_setup, 0U, sizeof(*eot_setup));
1096
1097 /* Set the number of pixel output registers/tile buffers allocated for the
1098 * render and copy the information to all subpasses and the EOT program.
1099 */
1100 pvr_finalise_po_alloc(device, ctx);
1101
1102 /* If any attachment are used with z replicate then they will be stored to by
1103 * the ISP. So remove them from the list to store to using the PBE.
1104 */
1105 list_for_each_entry_safe (struct pvr_render_int_attachment,
1106 int_attach,
1107 &ctx->active_surf_list,
1108 link) {
1109 if (int_attach->z_replicate)
1110 pvr_reset_surface(ctx, int_attach);
1111 }
1112
1113 /* Number of surfaces with allocated on-chip storage. */
1114 eot_setup->num_render_targets = ctx->active_surfaces;
1115 eot_setup->mrt_resources = vk_alloc(ctx->allocator,
1116 sizeof(eot_setup->mrt_resources[0U]) *
1117 eot_setup->num_render_targets,
1118 8,
1119 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
1120 if (!eot_setup->mrt_resources)
1121 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
1122
1123 /* Record the location of the on-chip storage. */
1124 mrt_idx = 0U;
1125 list_for_each_entry_safe (struct pvr_render_int_attachment,
1126 int_attach,
1127 &ctx->active_surf_list,
1128 link) {
1129 assert(int_attach->resource.type != USC_MRT_RESOURCE_TYPE_INVALID);
1130 assert(int_attach->remaining_count > 0U);
1131 if (int_attach->attachment->aspects & VK_IMAGE_ASPECT_STENCIL_BIT)
1132 assert(int_attach->stencil_remaining_count > 0U);
1133
1134 /* Copy the location of the source data for this attachment. */
1135 eot_setup->mrt_resources[mrt_idx] = int_attach->resource;
1136
1137 assert(int_attach->mrt_idx == -1);
1138 int_attach->mrt_idx = mrt_idx;
1139
1140 mrt_idx++;
1141 }
1142 assert(mrt_idx == (int32_t)eot_setup->num_render_targets);
1143
1144 hw_render->eot_surface_count = 0U;
1145 hw_render->pbe_emits = 0U;
1146
1147 /* Count the number of surfaces to store to at the end of the subpass. */
1148 for (uint32_t i = 0U; i < hw_render->subpass_count; i++) {
1149 struct pvr_renderpass_subpass *subpass = &ctx->subpasses[i];
1150 struct pvr_render_subpass *input_subpass = subpass->input_subpass;
1151
1152 for (uint32_t j = 0U; j < input_subpass->color_count; j++) {
1153 const uint32_t resolve_output =
1154 input_subpass->resolve_attachments
1155 ? input_subpass->resolve_attachments[j]
1156 : VK_ATTACHMENT_UNUSED;
1157 struct pvr_render_int_attachment *color_attach;
1158
1159 if (input_subpass->color_attachments[j] == VK_ATTACHMENT_UNUSED)
1160 continue;
1161
1162 color_attach = &ctx->int_attach[input_subpass->color_attachments[j]];
1163
1164 if (list_is_linked(&color_attach->link)) {
1165 uint32_t rem_count = resolve_output == VK_ATTACHMENT_UNUSED ? 0U
1166 : 1U;
1167
1168 /* If a color attachment is resolved it will have an extra
1169 * remaining usage.
1170 */
1171 if (color_attach->remaining_count > rem_count &&
1172 !color_attach->eot_surf_required) {
1173 color_attach->eot_surf_required = true;
1174 hw_render->eot_surface_count++;
1175 }
1176 }
1177
1178 if (resolve_output != VK_ATTACHMENT_UNUSED) {
1179 struct pvr_render_int_attachment *int_resolve_attach =
1180 &ctx->int_attach[resolve_output];
1181
1182 if (!int_resolve_attach->eot_surf_required) {
1183 int_resolve_attach->eot_surf_required = true;
1184 hw_render->eot_surface_count++;
1185 }
1186 }
1187 }
1188 }
1189
1190 assert(hw_render->eot_surface_count <= 16U);
1191
1192 hw_render->eot_surfaces = vk_alloc(ctx->allocator,
1193 sizeof(hw_render->eot_surfaces[0U]) *
1194 hw_render->eot_surface_count,
1195 8,
1196 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
1197 if (!hw_render->eot_surfaces)
1198 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
1199
1200 eot_attach = hw_render->eot_surfaces;
1201
1202 for (uint32_t i = 0U; i < hw_render->subpass_count; i++) {
1203 struct pvr_renderpass_subpass *subpass = &ctx->subpasses[i];
1204 struct pvr_render_subpass *input_subpass = subpass->input_subpass;
1205
1206 for (uint32_t j = 0U; j < input_subpass->color_count; j++) {
1207 const uint32_t resolve_output =
1208 input_subpass->resolve_attachments
1209 ? input_subpass->resolve_attachments[j]
1210 : VK_ATTACHMENT_UNUSED;
1211 struct pvr_render_int_attachment *color_attach;
1212
1213 if (input_subpass->color_attachments[j] == VK_ATTACHMENT_UNUSED)
1214 continue;
1215
1216 color_attach = &ctx->int_attach[input_subpass->color_attachments[j]];
1217
1218 if (resolve_output != VK_ATTACHMENT_UNUSED) {
1219 struct pvr_render_int_attachment *resolve_src =
1220 &ctx->int_attach[input_subpass->color_attachments[j]];
1221 struct pvr_render_int_attachment *resolve_dst =
1222 &ctx->int_attach[resolve_output];
1223
1224 assert(resolve_dst->eot_surf_required);
1225 resolve_dst->eot_surf_required = false;
1226
1227 /* Dereference the source to the resolve. */
1228 assert(resolve_src->remaining_count > 0U);
1229 resolve_src->remaining_count--;
1230
1231 /* Allocate device memory for the resolve destination. */
1232 pvr_mark_surface_alloc(ctx, resolve_dst);
1233
1234 /* The attachment has been written so load the attachment the
1235 * next time it is referenced.
1236 */
1237 resolve_dst->load_op = VK_ATTACHMENT_LOAD_OP_LOAD;
1238
1239 eot_attach->mrt_idx = resolve_src->mrt_idx;
1240 eot_attach->attachment_idx = resolve_dst->attachment->index;
1241 eot_attach->src_attachment_idx = resolve_src->attachment->index;
1242
1243 eot_attach->need_resolve = true;
1244
1245 if (!resolve_src->is_pbe_downscalable) {
1246 /* Resolve src must be stored for transfer resolve. */
1247 assert(resolve_src->remaining_count > 0U);
1248
1249 eot_attach->resolve_type = PVR_RESOLVE_TYPE_TRANSFER;
1250 } else if (resolve_src->remaining_count == 0U) {
1251 eot_attach->resolve_type = PVR_RESOLVE_TYPE_PBE;
1252 hw_render->pbe_emits++;
1253 } else {
1254 eot_attach->resolve_type = PVR_RESOLVE_TYPE_INVALID;
1255 }
1256
1257 eot_attach++;
1258 }
1259
1260 if (color_attach->eot_surf_required) {
1261 assert(color_attach->remaining_count > 0U);
1262
1263 pvr_mark_surface_alloc(ctx, color_attach);
1264
1265 assert(color_attach->mrt_idx >= 0);
1266 assert(color_attach->mrt_idx <
1267 (int32_t)hw_render->eot_setup.num_render_targets);
1268
1269 eot_attach->mrt_idx = color_attach->mrt_idx;
1270 eot_attach->attachment_idx = color_attach->attachment->index;
1271 eot_attach->need_resolve = false;
1272 eot_attach++;
1273
1274 hw_render->pbe_emits++;
1275
1276 color_attach->eot_surf_required = false;
1277 }
1278 }
1279 }
1280
1281 assert(hw_render->pbe_emits <= PVR_NUM_PBE_EMIT_REGS);
1282
1283 /* Count the number of extra resolves we can do through the PBE. */
1284 for (uint32_t i = 0U; i < hw_render->eot_surface_count; i++) {
1285 eot_attach = &hw_render->eot_surfaces[i];
1286
1287 if (eot_attach->need_resolve &&
1288 eot_attach->resolve_type == PVR_RESOLVE_TYPE_INVALID) {
1289 if (hw_render->pbe_emits == PVR_NUM_PBE_EMIT_REGS) {
1290 eot_attach->resolve_type = PVR_RESOLVE_TYPE_TRANSFER;
1291 } else {
1292 eot_attach->resolve_type = PVR_RESOLVE_TYPE_PBE;
1293 hw_render->pbe_emits++;
1294 }
1295 }
1296 }
1297
1298 assert(hw_render->pbe_emits <= PVR_NUM_PBE_EMIT_REGS);
1299
1300 /* Check for side effects in the final render. */
1301 hw_render->has_side_effects = pvr_render_has_side_effects(ctx);
1302
1303 /* Reset active surfaces. */
1304 list_for_each_entry_safe (struct pvr_render_int_attachment,
1305 int_attach,
1306 &ctx->active_surf_list,
1307 link) {
1308 int_attach->mrt_idx = -1;
1309 pvr_reset_surface(ctx, int_attach);
1310 }
1311
1312 assert(ctx->active_surfaces == 0U);
1313 assert(list_is_empty(&ctx->active_surf_list));
1314
1315 pvr_free_render(ctx);
1316 pvr_reset_render(ctx);
1317
1318 return VK_SUCCESS;
1319 }
1320
1321 static bool pvr_is_input(struct pvr_render_subpass *subpass,
1322 uint32_t attach_idx)
1323 {
1324 if (attach_idx == VK_ATTACHMENT_UNUSED)
1325 return false;
1326
1327 for (uint32_t i = 0U; i < subpass->input_count; i++) {
1328 if (subpass->input_attachments[i].attachment_idx == attach_idx)
1329 return true;
1330 }
1331
1332 return false;
1333 }
1334
1335 static bool
1336 pvr_depth_zls_conflict(struct pvr_renderpass_context *ctx,
1337 struct pvr_render_int_attachment *int_ds_attach,
1338 bool existing_ds_is_input)
1339 {
1340 if (!ctx->int_ds_attach)
1341 return false;
1342
1343 /* No conflict if the incoming subpass doesn't have a depth/stencil
1344 * attachment.
1345 */
1346 if (!int_ds_attach)
1347 return false;
1348
1349 /* No conflict if the incoming depth/stencil attachment is the same as the
1350 * existing one.
1351 */
1352 if (ctx->int_ds_attach == int_ds_attach)
1353 return false;
1354
1355 /* If the existing depth/stencil attachment is used later, then we can't
1356 * overwrite it.
1357 *
1358 * The exception is if the only use is as an input attachment in the incoming
1359 * subpass in which case we can use the Z replicate feature to save the
1360 * value.
1361 */
1362 if (ctx->int_ds_attach->remaining_count > 0U &&
1363 !(existing_ds_is_input && ctx->int_ds_attach->remaining_count == 1U)) {
1364 return true;
1365 }
1366
1367 if (ctx->int_ds_attach->attachment->aspects & VK_IMAGE_ASPECT_STENCIL_BIT &&
1368 ctx->int_ds_attach->stencil_remaining_count > 0U) {
1369 return true;
1370 }
1371
1372 /* We can't load midrender so fail if the new depth/stencil attachment is
1373 * already initialized.
1374 */
1375 if (int_ds_attach->load_op == VK_ATTACHMENT_LOAD_OP_LOAD)
1376 return true;
1377
1378 if (int_ds_attach->attachment->aspects & VK_IMAGE_ASPECT_STENCIL_BIT &&
1379 int_ds_attach->stencil_load_op == VK_ATTACHMENT_LOAD_OP_LOAD) {
1380 return true;
1381 }
1382
1383 return false;
1384 }
1385
1386 static void
1387 pvr_set_surface_resource(struct pvr_render_int_attachment *int_attach,
1388 struct pvr_renderpass_resource *resource)
1389 {
1390 int_attach->resource.type = resource->type;
1391
1392 switch (resource->type) {
1393 case USC_MRT_RESOURCE_TYPE_OUTPUT_REG:
1394 int_attach->resource.reg.output_reg = resource->reg.output_reg;
1395 int_attach->resource.reg.offset = resource->reg.offset;
1396 break;
1397
1398 case USC_MRT_RESOURCE_TYPE_MEMORY:
1399 int_attach->resource.mem.tile_buffer = resource->mem.tile_buffer;
1400 int_attach->resource.mem.offset_dw = resource->mem.offset_dw;
1401 break;
1402
1403 default:
1404 break;
1405 }
1406 }
1407
1408 static bool pvr_equal_resources(struct pvr_renderpass_resource *resource1,
1409 struct pvr_renderpass_resource *resource2)
1410 {
1411 if (resource1->type != resource2->type)
1412 return false;
1413
1414 switch (resource1->type) {
1415 case USC_MRT_RESOURCE_TYPE_OUTPUT_REG:
1416 return resource1->reg.output_reg == resource2->reg.output_reg &&
1417 resource1->reg.offset == resource2->reg.offset;
1418
1419 case USC_MRT_RESOURCE_TYPE_MEMORY:
1420 return resource1->mem.tile_buffer == resource2->mem.tile_buffer &&
1421 resource1->mem.offset_dw == resource2->mem.offset_dw;
1422
1423 default:
1424 return true;
1425 }
1426 }
1427
1428 static VkResult
1429 pvr_enable_z_replicate(struct pvr_renderpass_context *ctx,
1430 struct pvr_renderpass_hwsetup_render *hw_render,
1431 int32_t replicate_attach_idx,
1432 struct pvr_renderpass_resource *replicate_dst)
1433 {
1434 struct pvr_render_int_attachment *int_attach =
1435 &ctx->int_attach[replicate_attach_idx];
1436 int32_t first_use = -1;
1437
1438 /* If Z replication was already enabled for the attachment then nothing more
1439 * to do.
1440 */
1441 if (!int_attach->z_replicate) {
1442 /* Copy details of the storage for the replicated value to the attachment.
1443 */
1444 assert(int_attach->resource.type == USC_MRT_RESOURCE_TYPE_INVALID);
1445 assert(replicate_dst->type != USC_MRT_RESOURCE_TYPE_INVALID);
1446 pvr_set_surface_resource(int_attach, replicate_dst);
1447 } else {
1448 assert(int_attach->resource.type != USC_MRT_RESOURCE_TYPE_INVALID);
1449 assert(replicate_dst->type == USC_MRT_RESOURCE_TYPE_INVALID);
1450 }
1451
1452 /* Find the first subpass where the attachment is written. */
1453 for (uint32_t i = 0U; i < hw_render->subpass_count; i++) {
1454 struct pvr_renderpass_subpass *subpass = &ctx->subpasses[i];
1455 struct pvr_render_subpass *input_subpass = subpass->input_subpass;
1456
1457 if (input_subpass->depth_stencil_attachment == replicate_attach_idx) {
1458 first_use = i;
1459 break;
1460 }
1461 }
1462 assert(first_use >= 0);
1463
1464 /* For all subpasses from the first write. */
1465 for (uint32_t i = first_use; i < hw_render->subpass_count; i++) {
1466 struct pvr_renderpass_subpass *subpass = &ctx->subpasses[i];
1467 struct pvr_render_subpass *input_subpass = subpass->input_subpass;
1468
1469 /* If the subpass writes to the attachment then enable z replication. */
1470 if (input_subpass->depth_stencil_attachment == replicate_attach_idx &&
1471 !subpass->z_replicate) {
1472 subpass->z_replicate = true;
1473
1474 if (i != (hw_render->subpass_count - 1U)) {
1475 /* Copy the details of the storage for replicated value. */
1476 const VkResult result =
1477 pvr_copy_z_replicate_details(ctx,
1478 &ctx->hw_render->subpasses[i],
1479 subpass);
1480 if (result != VK_SUCCESS)
1481 return result;
1482 }
1483 }
1484 }
1485
1486 if (!int_attach->z_replicate) {
1487 /* Add the storage for the replicated value to locations in use at each
1488 * subpass.
1489 */
1490 for (uint32_t i = first_use; i < (hw_render->subpass_count - 1U); i++) {
1491 struct pvr_renderpass_subpass *subpass = &ctx->subpasses[i];
1492
1493 pvr_mark_storage_allocated(ctx,
1494 &subpass->alloc,
1495 int_attach->attachment,
1496 replicate_dst);
1497 }
1498
1499 /* Add the depth attachment to the list of surfaces with allocated
1500 * storage.
1501 */
1502 pvr_make_surface_active(ctx, int_attach, first_use);
1503
1504 int_attach->z_replicate = true;
1505 }
1506
1507 return VK_SUCCESS;
1508 }
1509
1510 static bool pvr_is_pending_resolve_dest(struct pvr_renderpass_context *ctx,
1511 uint32_t attach_idx)
1512 {
1513 struct pvr_render_int_attachment *int_attach = &ctx->int_attach[attach_idx];
1514
1515 return int_attach->last_resolve_dst_render != -1 &&
1516 int_attach->last_resolve_dst_render ==
1517 (int32_t)(ctx->hw_setup->render_count - 1U);
1518 }
1519
1520 static bool pvr_is_pending_resolve_src(struct pvr_renderpass_context *ctx,
1521 uint32_t attach_idx)
1522 {
1523 struct pvr_render_int_attachment *int_attach = &ctx->int_attach[attach_idx];
1524
1525 return int_attach->last_resolve_src_render != -1 &&
1526 int_attach->last_resolve_src_render ==
1527 (int32_t)(ctx->hw_setup->render_count - 1U);
1528 }
1529
1530 static bool pvr_exceeds_pbe_registers(struct pvr_renderpass_context *ctx,
1531 struct pvr_render_subpass *subpass)
1532 {
1533 int32_t live_outputs[PVR_NUM_PBE_EMIT_REGS];
1534 uint32_t num_live_outputs = 0U;
1535
1536 /* Count all color outputs so far. */
1537 for (uint32_t i = 0U; i < ctx->hw_render->subpass_count; i++) {
1538 struct pvr_render_subpass *input_subpass =
1539 ctx->subpasses[i].input_subpass;
1540
1541 for (uint32_t j = 0U; j < input_subpass->color_count; j++) {
1542 const uint32_t global_color_attach =
1543 input_subpass->color_attachments[j];
1544 struct pvr_render_int_attachment *int_attach;
1545 bool found = false;
1546
1547 if (global_color_attach == VK_ATTACHMENT_UNUSED)
1548 continue;
1549
1550 int_attach = &ctx->int_attach[global_color_attach];
1551
1552 if (int_attach->last_read <= (int32_t)subpass->index)
1553 continue;
1554
1555 for (uint32_t k = 0U; k < num_live_outputs; k++) {
1556 if (live_outputs[k] == global_color_attach) {
1557 found = true;
1558 break;
1559 }
1560 }
1561
1562 if (!found)
1563 live_outputs[num_live_outputs++] = global_color_attach;
1564 }
1565 }
1566
1567 assert(num_live_outputs <= PVR_NUM_PBE_EMIT_REGS);
1568
1569 /* Check if adding all the color outputs of the new subpass to the render
1570 * would exceed the limit.
1571 */
1572 for (uint32_t i = 0U; i < subpass->color_count; i++) {
1573 const uint32_t global_color_attach = subpass->color_attachments[i];
1574 struct pvr_render_int_attachment *int_attach;
1575 bool found = false;
1576
1577 if (global_color_attach == VK_ATTACHMENT_UNUSED)
1578 continue;
1579
1580 int_attach = &ctx->int_attach[global_color_attach];
1581
1582 if (int_attach->last_read <= (int32_t)subpass->index)
1583 continue;
1584
1585 for (uint32_t j = 0U; j < num_live_outputs; j++) {
1586 if (live_outputs[j] == global_color_attach) {
1587 found = true;
1588 break;
1589 }
1590 }
1591
1592 if (!found) {
1593 if (num_live_outputs >= PVR_NUM_PBE_EMIT_REGS)
1594 return true;
1595
1596 live_outputs[num_live_outputs++] = global_color_attach;
1597 }
1598 }
1599
1600 return false;
1601 }
1602
1603 static void pvr_merge_alloc_buffer(struct pvr_renderpass_alloc_buffer *dst,
1604 struct pvr_renderpass_alloc_buffer *src)
1605 {
1606 for (uint32_t i = 0U; i < ARRAY_SIZE(dst->allocs); i++)
1607 dst->allocs[i] |= src->allocs[i];
1608 }
1609
1610 static VkResult pvr_merge_alloc(struct pvr_renderpass_context *ctx,
1611 struct pvr_renderpass_alloc *dst,
1612 struct pvr_renderpass_alloc *src)
1613 {
1614 pvr_merge_alloc_buffer(&dst->output_reg, &src->output_reg);
1615
1616 dst->output_regs_count =
1617 MAX2(dst->output_regs_count, src->output_regs_count);
1618
1619 if (dst->tile_buffers_count < src->tile_buffers_count) {
1620 memset(dst->tile_buffers + dst->tile_buffers_count,
1621 0U,
1622 sizeof(dst->tile_buffers[0U]) *
1623 (src->tile_buffers_count - dst->tile_buffers_count));
1624 dst->tile_buffers_count = src->tile_buffers_count;
1625 }
1626
1627 for (uint32_t i = 0U; i < src->tile_buffers_count; i++)
1628 pvr_merge_alloc_buffer(&dst->tile_buffers[i], &src->tile_buffers[i]);
1629
1630 return VK_SUCCESS;
1631 }
1632
1633 static VkResult
1634 pvr_is_z_replicate_space_available(const struct pvr_device_info *dev_info,
1635 struct pvr_renderpass_context *ctx,
1636 struct pvr_renderpass_alloc *alloc,
1637 uint32_t attach_idx,
1638 struct pvr_renderpass_resource *resource)
1639 {
1640 struct pvr_renderpass_hwsetup_render *hw_render = ctx->hw_render;
1641 struct pvr_render_int_attachment *int_attach;
1642 struct pvr_renderpass_alloc combined_alloc;
1643 uint32_t first_use;
1644 VkResult result;
1645
1646 /* If z replication was already enabled by a previous subpass then storage
1647 * will already be allocated.
1648 */
1649 assert(attach_idx < ctx->pass->attachment_count);
1650
1651 int_attach = &ctx->int_attach[attach_idx];
1652 if (int_attach->z_replicate) {
1653 assert(int_attach->resource.type != USC_MRT_RESOURCE_TYPE_INVALID);
1654 return VK_SUCCESS;
1655 }
1656
1657 /* Get the registers used in any subpass after the depth is first written.
1658 * Start with registers used in the incoming subpass.
1659 */
1660 result = pvr_copy_alloc(ctx, &combined_alloc, alloc);
1661 if (result != VK_SUCCESS)
1662 return result;
1663
1664 if (hw_render) {
1665 /* Find the subpass where the depth is first written. */
1666 first_use = hw_render->subpass_count;
1667 for (uint32_t i = 0U; i < hw_render->subpass_count; i++) {
1668 struct pvr_renderpass_subpass *subpass = &ctx->subpasses[i];
1669 struct pvr_render_subpass *input_subpass = subpass->input_subpass;
1670
1671 if (input_subpass->depth_stencil_attachment == (int32_t)attach_idx) {
1672 first_use = i;
1673 break;
1674 }
1675 }
1676
1677 /* Merge in registers used in previous subpasses. */
1678 for (uint32_t i = first_use; i < hw_render->subpass_count; i++) {
1679 struct pvr_renderpass_subpass *subpass = &ctx->subpasses[i];
1680
1681 result = pvr_merge_alloc(ctx, &combined_alloc, &subpass->alloc);
1682 if (result != VK_SUCCESS) {
1683 pvr_free_alloc(ctx, &combined_alloc);
1684 return result;
1685 }
1686 }
1687 }
1688
1689 result = pvr_surface_alloc_color_storage(dev_info,
1690 ctx,
1691 &combined_alloc,
1692 int_attach->attachment,
1693 resource);
1694
1695 pvr_free_alloc(ctx, &combined_alloc);
1696 if (result != VK_SUCCESS)
1697 return result;
1698
1699 return pvr_mark_storage_allocated(ctx,
1700 alloc,
1701 int_attach->attachment,
1702 resource);
1703 }
1704
1705 static VkResult
1706 pvr_is_subpass_space_available(const struct pvr_device_info *dev_info,
1707 struct pvr_renderpass_context *ctx,
1708 struct pvr_render_subpass *subpass,
1709 struct pvr_render_subpass_depth_params *sp_depth,
1710 struct pvr_renderpass_alloc *alloc,
1711 struct pvr_render_int_subpass_dsts *sp_dsts)
1712 {
1713 VkResult result;
1714
1715 /* Mark pointers in return structures as not allocated. */
1716 sp_dsts->color = NULL;
1717
1718 /* Allocate space for which locations are in use after this subpass. */
1719 result = pvr_copy_alloc(ctx, alloc, &ctx->alloc);
1720 if (result != VK_SUCCESS)
1721 return result;
1722
1723 /* Allocate space to store our results. */
1724 if (subpass->color_count > 0U) {
1725 sp_dsts->color =
1726 vk_alloc(ctx->allocator,
1727 sizeof(sp_dsts->color[0U]) * subpass->color_count,
1728 8,
1729 VK_SYSTEM_ALLOCATION_SCOPE_COMMAND);
1730 if (!sp_dsts->color) {
1731 result = vk_error(NULL, VK_ERROR_OUT_OF_HOST_MEMORY);
1732 goto err_free_alloc;
1733 }
1734 } else {
1735 sp_dsts->color = NULL;
1736 }
1737
1738 sp_dsts->existing_zrep.type = USC_MRT_RESOURCE_TYPE_INVALID;
1739 sp_dsts->incoming_zrep.type = USC_MRT_RESOURCE_TYPE_INVALID;
1740
1741 for (uint32_t i = 0U; i < subpass->color_count; i++) {
1742 const uint32_t attach_idx = subpass->color_attachments[i];
1743 struct pvr_render_int_attachment *int_attach;
1744
1745 if (attach_idx == VK_ATTACHMENT_UNUSED)
1746 continue;
1747
1748 int_attach = &ctx->int_attach[attach_idx];
1749
1750 assert(pvr_get_accum_format_bitsize(int_attach->attachment->vk_format) >
1751 0U);
1752
1753 /* Is the attachment not allocated on-chip storage? */
1754 if (int_attach->resource.type == USC_MRT_RESOURCE_TYPE_INVALID) {
1755 result = pvr_surface_alloc_color_storage(dev_info,
1756 ctx,
1757 alloc,
1758 int_attach->attachment,
1759 &sp_dsts->color[i]);
1760 if (result != VK_SUCCESS)
1761 goto err_free_alloc;
1762 } else {
1763 sp_dsts->color[i].type = USC_MRT_RESOURCE_TYPE_INVALID;
1764 }
1765 }
1766
1767 if (sp_depth->existing_ds_is_input) {
1768 result = pvr_is_z_replicate_space_available(dev_info,
1769 ctx,
1770 alloc,
1771 sp_depth->existing_ds_attach,
1772 &sp_dsts->existing_zrep);
1773 if (result != VK_SUCCESS)
1774 goto err_free_alloc;
1775 }
1776
1777 if (sp_depth->incoming_ds_is_input) {
1778 if (sp_depth->existing_ds_attach != subpass->depth_stencil_attachment) {
1779 result = pvr_is_z_replicate_space_available(
1780 dev_info,
1781 ctx,
1782 alloc,
1783 subpass->depth_stencil_attachment,
1784 &sp_dsts->incoming_zrep);
1785 if (result != VK_SUCCESS)
1786 goto err_free_alloc;
1787 } else {
1788 sp_dsts->incoming_zrep = sp_dsts->existing_zrep;
1789 }
1790 }
1791
1792 return VK_SUCCESS;
1793
1794 err_free_alloc:
1795 pvr_free_alloc(ctx, alloc);
1796 if (sp_dsts->color)
1797 vk_free(ctx->allocator, sp_dsts->color);
1798
1799 sp_dsts->color = NULL;
1800
1801 return result;
1802 }
1803
1804 static bool
1805 pvr_can_combine_with_render(const struct pvr_device_info *dev_info,
1806 struct pvr_renderpass_context *ctx,
1807 struct pvr_render_subpass *subpass,
1808 struct pvr_render_subpass_depth_params *sp_depth,
1809 struct pvr_render_int_attachment *int_ds_attach,
1810 struct pvr_renderpass_alloc *new_alloc,
1811 struct pvr_render_int_subpass_dsts *sp_dsts)
1812 {
1813 VkResult result;
1814 bool ret;
1815
1816 /* Mark pointers in return structures as not allocated. */
1817 sp_dsts->color = NULL;
1818
1819 if (ctx->hw_render && (ctx->hw_render->view_mask != subpass->view_mask))
1820 return false;
1821
1822 /* The hardware doesn't support replicating the stencil, so we need to store
1823 * the depth to memory if a stencil attachment is used as an input
1824 * attachment.
1825 */
1826 if (sp_depth->existing_ds_is_input &&
1827 ctx->int_ds_attach->attachment->aspects & VK_IMAGE_ASPECT_STENCIL_BIT) {
1828 return false;
1829 }
1830
1831 if (sp_depth->incoming_ds_is_input && int_ds_attach &&
1832 int_ds_attach->attachment->aspects & VK_IMAGE_ASPECT_STENCIL_BIT &&
1833 ctx->hw_render) {
1834 return false;
1835 }
1836
1837 /* Can't mix multiple sample counts into same render. */
1838 if (ctx->hw_render &&
1839 ctx->hw_render->sample_count != subpass->sample_count) {
1840 return false;
1841 }
1842
1843 /* If the depth is used by both the render and the incoming subpass and
1844 * either the existing depth must be saved or the new depth must be loaded
1845 * then we can't merge.
1846 */
1847 ret = pvr_depth_zls_conflict(ctx,
1848 int_ds_attach,
1849 sp_depth->existing_ds_is_input);
1850 if (ret)
1851 return false;
1852
1853 /* Check if any of the subpass's dependencies are marked that the two
1854 * subpasses can't be in the same render.
1855 */
1856 for (uint32_t i = 0U; i < subpass->dep_count; i++) {
1857 const uint32_t dep = subpass->dep_list[i];
1858 if (subpass->flush_on_dep[i] && ctx->hw_setup->subpass_map[dep].render ==
1859 (ctx->hw_setup->render_count - 1U)) {
1860 return false;
1861 }
1862 }
1863
1864 /* Check if one of the input/color attachments is written by an MSAA resolve
1865 * in an existing subpass in the current render.
1866 */
1867 for (uint32_t i = 0U; i < subpass->input_count; i++) {
1868 const uint32_t attach_idx = subpass->input_attachments[i].attachment_idx;
1869 if (attach_idx != VK_ATTACHMENT_UNUSED &&
1870 pvr_is_pending_resolve_dest(ctx, attach_idx)) {
1871 return false;
1872 }
1873 }
1874
1875 for (uint32_t i = 0U; i < subpass->color_count; i++) {
1876 if (subpass->color_attachments[i] != VK_ATTACHMENT_UNUSED &&
1877 (pvr_is_pending_resolve_dest(ctx, subpass->color_attachments[i]) ||
1878 pvr_is_pending_resolve_src(ctx, subpass->color_attachments[i]))) {
1879 return false;
1880 }
1881
1882 if (subpass->resolve_attachments &&
1883 subpass->resolve_attachments[i] != VK_ATTACHMENT_UNUSED &&
1884 pvr_is_pending_resolve_dest(ctx, subpass->resolve_attachments[i])) {
1885 return false;
1886 }
1887 }
1888
1889 /* No chance of exceeding PBE registers in a single subpass. */
1890 if (ctx->hw_render) {
1891 ret = pvr_exceeds_pbe_registers(ctx, subpass);
1892 if (ret)
1893 return false;
1894 }
1895
1896 /* Check we can allocate storage for the new subpass's color attachments and
1897 * any z replications.
1898 */
1899 result = pvr_is_subpass_space_available(dev_info,
1900 ctx,
1901 subpass,
1902 sp_depth,
1903 new_alloc,
1904 sp_dsts);
1905 if (result != VK_SUCCESS)
1906 return false;
1907
1908 /* Avoid merging subpasses which result in tile buffers having to be
1909 * used. The benefit of merging must be weighed against the cost of
1910 * writing/reading to tile buffers.
1911 */
1912 if (ctx->hw_render && new_alloc->tile_buffers_count > 0)
1913 return false;
1914
1915 return true;
1916 }
1917
1918 static VkResult
1919 pvr_merge_subpass(const struct pvr_device *device,
1920 struct pvr_renderpass_context *ctx,
1921 struct pvr_render_subpass *input_subpass,
1922 struct pvr_renderpass_hwsetup_subpass **const hw_subpass_out)
1923 {
1924 struct pvr_renderpass_hwsetup_subpass *new_hw_subpasses;
1925 struct pvr_renderpass_hwsetup_subpass *hw_subpass;
1926 struct pvr_render_int_attachment *int_ds_attach;
1927 struct pvr_renderpass_hwsetup_render *hw_render;
1928 struct pvr_render_subpass_depth_params sp_depth;
1929 struct pvr_renderpass_subpass *new_subpasses;
1930 struct pvr_render_int_subpass_dsts sp_dsts;
1931 struct pvr_renderpass_subpass *subpass;
1932 struct pvr_renderpass_alloc alloc;
1933 VkResult result;
1934 bool ret;
1935
1936 /* Depth attachment for the incoming subpass. */
1937 if (input_subpass->depth_stencil_attachment != VK_ATTACHMENT_UNUSED)
1938 int_ds_attach = &ctx->int_attach[input_subpass->depth_stencil_attachment];
1939 else
1940 int_ds_attach = NULL;
1941
1942 /* Attachment ID for the existing depth attachment. */
1943 if (ctx->int_ds_attach)
1944 sp_depth.existing_ds_attach = ctx->int_ds_attach - ctx->int_attach;
1945 else
1946 sp_depth.existing_ds_attach = VK_ATTACHMENT_UNUSED;
1947
1948 /* Is the incoming depth attachment used as an input to the incoming subpass?
1949 */
1950 sp_depth.incoming_ds_is_input =
1951 pvr_is_input(input_subpass, input_subpass->depth_stencil_attachment);
1952
1953 /* Is the current depth attachment used as an input to the incoming subpass?
1954 */
1955 sp_depth.existing_ds_is_input =
1956 pvr_is_input(input_subpass, sp_depth.existing_ds_attach);
1957
1958 /* Can the incoming subpass be combined with the existing render? Also checks
1959 * if space is available for the subpass results and return the allocated
1960 * locations.
1961 */
1962 ret = pvr_can_combine_with_render(&device->pdevice->dev_info,
1963 ctx,
1964 input_subpass,
1965 &sp_depth,
1966 int_ds_attach,
1967 &alloc,
1968 &sp_dsts);
1969 if (!ret) {
1970 result = pvr_close_render(device, ctx);
1971 if (result != VK_SUCCESS)
1972 goto end_merge_subpass;
1973
1974 sp_depth.existing_ds_is_input = false;
1975 sp_depth.existing_ds_attach = VK_ATTACHMENT_UNUSED;
1976
1977 /* Allocate again in a new render. */
1978 result = pvr_is_subpass_space_available(&device->pdevice->dev_info,
1979 ctx,
1980 input_subpass,
1981 &sp_depth,
1982 &alloc,
1983 &sp_dsts);
1984 assert(result != VK_ERROR_TOO_MANY_OBJECTS);
1985 if (result != VK_SUCCESS)
1986 goto end_merge_subpass;
1987 }
1988
1989 /* If there isn't an in-progress render then allocate one. */
1990 if (!ctx->hw_render) {
1991 struct pvr_renderpass_hwsetup *hw_setup = ctx->hw_setup;
1992 struct pvr_renderpass_hwsetup_render *new_hw_render = vk_realloc(
1993 ctx->allocator,
1994 hw_setup->renders,
1995 sizeof(hw_setup->renders[0U]) * (hw_setup->render_count + 1U),
1996 8U,
1997 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
1998 if (!new_hw_render) {
1999 result = vk_error(NULL, VK_ERROR_OUT_OF_HOST_MEMORY);
2000 goto end_merge_subpass;
2001 }
2002
2003 hw_setup->renders = new_hw_render;
2004
2005 ctx->hw_render = &hw_setup->renders[hw_setup->render_count];
2006 memset(ctx->hw_render, 0U, sizeof(*hw_render));
2007 ctx->hw_render->ds_attach_idx = VK_ATTACHMENT_UNUSED;
2008 hw_setup->render_count++;
2009 ctx->hw_render->depth_init = VK_ATTACHMENT_LOAD_OP_DONT_CARE;
2010 ctx->hw_render->stencil_init = VK_ATTACHMENT_LOAD_OP_DONT_CARE;
2011 ctx->hw_render->sample_count = input_subpass->sample_count;
2012 ctx->hw_render->view_mask = input_subpass->view_mask;
2013 }
2014
2015 /* Allocate a new subpass in the in-progress render. */
2016 hw_render = ctx->hw_render;
2017
2018 new_hw_subpasses = vk_realloc(ctx->allocator,
2019 hw_render->subpasses,
2020 sizeof(hw_render->subpasses[0U]) *
2021 (hw_render->subpass_count + 1U),
2022 8U,
2023 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
2024 if (!new_hw_subpasses) {
2025 result = vk_error(NULL, VK_ERROR_OUT_OF_HOST_MEMORY);
2026 goto end_merge_subpass;
2027 }
2028
2029 hw_render->subpasses = new_hw_subpasses;
2030 hw_subpass = &hw_render->subpasses[hw_render->subpass_count];
2031
2032 new_subpasses =
2033 vk_realloc(ctx->allocator,
2034 ctx->subpasses,
2035 sizeof(ctx->subpasses[0U]) * (hw_render->subpass_count + 1U),
2036 8U,
2037 VK_SYSTEM_ALLOCATION_SCOPE_COMMAND);
2038 if (!new_subpasses) {
2039 result = vk_error(NULL, VK_ERROR_OUT_OF_HOST_MEMORY);
2040 goto end_merge_subpass;
2041 }
2042
2043 ctx->subpasses = new_subpasses;
2044
2045 subpass = &ctx->subpasses[hw_render->subpass_count];
2046 subpass->input_subpass = input_subpass;
2047 subpass->z_replicate = false;
2048
2049 /* Save the allocation state at the subpass. */
2050 result = pvr_copy_alloc(ctx, &subpass->alloc, &alloc);
2051 if (result != VK_SUCCESS)
2052 goto end_merge_subpass;
2053
2054 hw_render->subpass_count++;
2055
2056 memset(hw_subpass, 0U, sizeof(*hw_subpass));
2057 hw_subpass->index = input_subpass->index;
2058 hw_subpass->z_replicate = -1;
2059 hw_subpass->depth_initop = VK_ATTACHMENT_LOAD_OP_DONT_CARE;
2060
2061 if (int_ds_attach && ctx->int_ds_attach != int_ds_attach) {
2062 bool setup_render_ds = false;
2063 bool stencil_load = false;
2064 bool depth_load = false;
2065
2066 if (int_ds_attach->load_op == VK_ATTACHMENT_LOAD_OP_LOAD) {
2067 depth_load = true;
2068 setup_render_ds = true;
2069 hw_render->depth_init = VK_ATTACHMENT_LOAD_OP_LOAD;
2070 hw_subpass->depth_initop = VK_ATTACHMENT_LOAD_OP_LOAD;
2071
2072 assert(!ctx->ds_load_surface);
2073 ctx->ds_load_surface = int_ds_attach;
2074 } else if (int_ds_attach->load_op == VK_ATTACHMENT_LOAD_OP_CLEAR) {
2075 hw_subpass->depth_initop = VK_ATTACHMENT_LOAD_OP_CLEAR;
2076 }
2077
2078 if (int_ds_attach->attachment->aspects & VK_IMAGE_ASPECT_STENCIL_BIT) {
2079 if (int_ds_attach->stencil_load_op == VK_ATTACHMENT_LOAD_OP_LOAD) {
2080 stencil_load = true;
2081 setup_render_ds = true;
2082 hw_render->stencil_init = VK_ATTACHMENT_LOAD_OP_LOAD;
2083 } else if (int_ds_attach->stencil_load_op ==
2084 VK_ATTACHMENT_LOAD_OP_CLEAR) {
2085 hw_subpass->stencil_clear = true;
2086 }
2087 }
2088
2089 /* If the depth is loaded then allocate external memory for the depth
2090 * attachment.
2091 */
2092 if (depth_load || stencil_load)
2093 pvr_mark_surface_alloc(ctx, int_ds_attach);
2094
2095 if (setup_render_ds) {
2096 assert(hw_render->ds_attach_idx == VK_ATTACHMENT_UNUSED);
2097 hw_render->ds_attach_idx = int_ds_attach->attachment->index;
2098 }
2099
2100 ctx->int_ds_attach = int_ds_attach;
2101 }
2102
2103 /* Set up the initialization operations for subpasses. */
2104 hw_subpass->color_initops = vk_alloc(ctx->allocator,
2105 sizeof(hw_subpass->color_initops[0U]) *
2106 input_subpass->color_count,
2107 8,
2108 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
2109 if (!hw_subpass->color_initops) {
2110 result = vk_error(NULL, VK_ERROR_OUT_OF_HOST_MEMORY);
2111 goto end_merge_subpass;
2112 }
2113
2114 for (uint32_t i = 0U; i < input_subpass->color_count; i++) {
2115 const uint32_t attach_idx = input_subpass->color_attachments[i];
2116 struct pvr_render_int_attachment *int_attach;
2117
2118 if (attach_idx == VK_ATTACHMENT_UNUSED)
2119 continue;
2120
2121 int_attach = &ctx->int_attach[attach_idx];
2122
2123 if (int_attach->first_use == -1) {
2124 hw_subpass->color_initops[i] = int_attach->load_op;
2125
2126 /* If the attachment is loaded then off-chip memory must be
2127 * allocated for it.
2128 */
2129 if (int_attach->load_op == VK_ATTACHMENT_LOAD_OP_LOAD)
2130 pvr_mark_surface_alloc(ctx, int_attach);
2131
2132 /* The attachment has been written so load the attachment the next
2133 * time it is referenced.
2134 */
2135 int_attach->load_op = VK_ATTACHMENT_LOAD_OP_LOAD;
2136 } else {
2137 hw_subpass->color_initops[i] = VK_ATTACHMENT_LOAD_OP_DONT_CARE;
2138 }
2139 }
2140
2141 /* Copy the destinations allocated for the color attachments. */
2142 for (uint32_t i = 0U; i < input_subpass->color_count; i++) {
2143 const uint32_t attach_idx = input_subpass->color_attachments[i];
2144 struct pvr_render_int_attachment *int_attach;
2145 struct pvr_renderpass_resource *attach_dst;
2146
2147 if (attach_idx == VK_ATTACHMENT_UNUSED)
2148 continue;
2149
2150 int_attach = &ctx->int_attach[attach_idx];
2151 attach_dst = &sp_dsts.color[i];
2152
2153 if (int_attach->first_use == -1) {
2154 assert(int_attach->resource.type == USC_MRT_RESOURCE_TYPE_INVALID);
2155 assert(attach_dst->type != USC_MRT_RESOURCE_TYPE_INVALID);
2156 pvr_set_surface_resource(int_attach, attach_dst);
2157
2158 /* If this attachment is being used for the first time then add it
2159 * to the active list.
2160 */
2161 pvr_make_surface_active(ctx,
2162 int_attach,
2163 hw_render->subpass_count - 1U);
2164 } else {
2165 assert(attach_dst->type == USC_MRT_RESOURCE_TYPE_INVALID);
2166 }
2167 }
2168
2169 /* We can't directly read the on-chip depth so mark subpasses where the depth
2170 * is written to replicate the value into part of the color storage.
2171 */
2172 if (sp_depth.existing_ds_is_input) {
2173 result = pvr_enable_z_replicate(ctx,
2174 hw_render,
2175 sp_depth.existing_ds_attach,
2176 &sp_dsts.existing_zrep);
2177 if (result != VK_SUCCESS)
2178 goto end_merge_subpass;
2179 }
2180
2181 if (sp_depth.incoming_ds_is_input) {
2182 if (input_subpass->depth_stencil_attachment !=
2183 sp_depth.existing_ds_attach) {
2184 result =
2185 pvr_enable_z_replicate(ctx,
2186 hw_render,
2187 input_subpass->depth_stencil_attachment,
2188 &sp_dsts.incoming_zrep);
2189 if (result != VK_SUCCESS)
2190 goto end_merge_subpass;
2191 } else {
2192 assert(pvr_equal_resources(&sp_dsts.existing_zrep,
2193 &sp_dsts.incoming_zrep));
2194 }
2195 }
2196
2197 /* Copy the locations of color/input attachments to the output structure.
2198 * N.B. Need to do this after Z replication in case the replicated depth is
2199 * an input attachment for the incoming subpass.
2200 */
2201 result = pvr_copy_storage_details(ctx, hw_subpass, subpass);
2202 if (result != VK_SUCCESS)
2203 goto end_merge_subpass;
2204
2205 if (subpass->z_replicate) {
2206 result = pvr_copy_z_replicate_details(ctx, hw_subpass, subpass);
2207 if (result != VK_SUCCESS)
2208 goto end_merge_subpass;
2209 }
2210
2211 /* Copy the allocation at the subpass. This will then be updated if this was
2212 * last use of any attachment.
2213 */
2214 pvr_free_alloc(ctx, &ctx->alloc);
2215 ctx->alloc = alloc;
2216
2217 /* Free information about subpass destinations. */
2218 if (sp_dsts.color)
2219 vk_free(ctx->allocator, sp_dsts.color);
2220
2221 *hw_subpass_out = hw_subpass;
2222
2223 return VK_SUCCESS;
2224
2225 end_merge_subpass:
2226 if (sp_dsts.color)
2227 vk_free(ctx->allocator, sp_dsts.color);
2228
2229 pvr_free_alloc(ctx, &alloc);
2230
2231 return result;
2232 }
2233
2234 static void
2235 pvr_dereference_color_output_list(struct pvr_renderpass_context *ctx,
2236 uint32_t subpass_num,
2237 struct pvr_render_subpass *subpass)
2238 {
2239 for (uint32_t i = 0U; i < subpass->color_count; i++) {
2240 const uint32_t attach_idx = subpass->color_attachments[i];
2241
2242 if (attach_idx != VK_ATTACHMENT_UNUSED)
2243 pvr_dereference_surface(ctx, attach_idx, subpass_num);
2244 }
2245 }
2246
2247 static void
2248 pvr_dereference_surface_list(struct pvr_renderpass_context *ctx,
2249 uint32_t subpass_num,
2250 struct pvr_render_attachment *attachments,
2251 uint32_t count)
2252 {
2253 for (uint32_t i = 0U; i < count; i++) {
2254 if (attachments[i].attachment_idx != VK_ATTACHMENT_UNUSED)
2255 pvr_dereference_surface(ctx,
2256 attachments[i].attachment_idx,
2257 subpass_num);
2258 }
2259 }
2260
2261 static VkResult pvr_schedule_subpass(const struct pvr_device *device,
2262 struct pvr_renderpass_context *ctx,
2263 uint32_t subpass_idx)
2264 {
2265 struct pvr_renderpass_hwsetup_subpass *hw_subpass;
2266 struct pvr_renderpass_hwsetup_render *hw_render;
2267 struct pvr_render_int_subpass *int_subpass;
2268 struct pvr_render_subpass *subpass;
2269 uint32_t subpass_num;
2270 VkResult result;
2271
2272 int_subpass = &ctx->int_subpasses[subpass_idx];
2273 subpass = int_subpass->subpass;
2274
2275 result = pvr_merge_subpass(device, ctx, subpass, &hw_subpass);
2276 if (result != VK_SUCCESS)
2277 return result;
2278
2279 hw_render = ctx->hw_render;
2280 subpass_num = hw_render->subpass_count - 1U;
2281
2282 /* Record where the subpass was scheduled. */
2283 ctx->hw_setup->subpass_map[subpass_idx].render =
2284 ctx->hw_setup->render_count - 1U;
2285 ctx->hw_setup->subpass_map[subpass_idx].subpass = subpass_num;
2286
2287 /* Check this subpass was the last use of any attachments. */
2288 pvr_dereference_color_output_list(ctx, subpass_num, subpass);
2289 pvr_dereference_surface_list(ctx,
2290 subpass_num,
2291 subpass->input_attachments,
2292 subpass->input_count);
2293 pvr_dereference_surface_list(ctx,
2294 subpass_num,
2295 subpass->preserve_attachments,
2296 subpass->preserve_count);
2297 if (subpass->depth_stencil_attachment != VK_ATTACHMENT_UNUSED) {
2298 struct pvr_render_int_attachment *int_depth_attach =
2299 &ctx->int_attach[subpass->depth_stencil_attachment];
2300
2301 assert(int_depth_attach->remaining_count > 0U);
2302 int_depth_attach->remaining_count--;
2303
2304 if (int_depth_attach->remaining_count == 0U) {
2305 if (int_depth_attach->first_use != -1)
2306 int_depth_attach->last_use = subpass_num;
2307
2308 if (int_depth_attach->z_replicate)
2309 pvr_free_surface_storage(ctx, int_depth_attach);
2310 }
2311
2312 if (int_depth_attach->attachment->aspects & VK_IMAGE_ASPECT_STENCIL_BIT) {
2313 assert(int_depth_attach->stencil_remaining_count > 0U);
2314 int_depth_attach->stencil_remaining_count--;
2315 }
2316
2317 /* The depth attachment has initialized data so load it from memory if it
2318 * is referenced again.
2319 */
2320 int_depth_attach->load_op = VK_ATTACHMENT_LOAD_OP_LOAD;
2321 int_depth_attach->stencil_load_op = VK_ATTACHMENT_LOAD_OP_LOAD;
2322
2323 if (subpass->depth_stencil_resolve_attachment != VK_ATTACHMENT_UNUSED) {
2324 struct pvr_render_int_attachment *int_resolve_attach =
2325 &ctx->int_attach[subpass->depth_stencil_resolve_attachment];
2326
2327 assert(int_depth_attach->last_resolve_src_render <
2328 (int32_t)ctx->hw_setup->render_count - 1);
2329 int_depth_attach->last_resolve_src_render =
2330 ctx->hw_setup->render_count - 1;
2331
2332 assert(int_depth_attach->last_resolve_dst_render <
2333 (int32_t)ctx->hw_setup->render_count - 1);
2334 int_depth_attach->last_resolve_dst_render =
2335 ctx->hw_setup->render_count - 1;
2336
2337 assert(!ctx->ds_resolve_surface);
2338 ctx->ds_resolve_surface = int_resolve_attach;
2339
2340 hw_render->stencil_resolve_mode = subpass->stencil_resolve_mode;
2341 hw_render->depth_resolve_mode = subpass->depth_resolve_mode;
2342
2343 if (hw_render->depth_resolve_mode)
2344 int_depth_attach->remaining_count++;
2345
2346 if (hw_render->stencil_resolve_mode)
2347 int_depth_attach->stencil_remaining_count++;
2348 }
2349 }
2350
2351 /* Mark surfaces which have been the source or destination of an MSAA resolve
2352 * in the current render.
2353 */
2354 for (uint32_t i = 0U; i < subpass->color_count; i++) {
2355 struct pvr_render_int_attachment *resolve_src;
2356 struct pvr_render_int_attachment *resolve_dst;
2357
2358 if (!subpass->resolve_attachments)
2359 break;
2360
2361 if (subpass->resolve_attachments[i] == VK_ATTACHMENT_UNUSED)
2362 continue;
2363
2364 assert(subpass->color_attachments[i] <
2365 (int32_t)ctx->pass->attachment_count);
2366 resolve_src = &ctx->int_attach[subpass->color_attachments[i]];
2367
2368 assert(subpass->resolve_attachments[i] <
2369 (int32_t)ctx->pass->attachment_count);
2370 resolve_dst = &ctx->int_attach[subpass->resolve_attachments[i]];
2371
2372 /* Mark the resolve source. */
2373 assert(resolve_src->last_resolve_src_render <
2374 (int32_t)(ctx->hw_setup->render_count - 1U));
2375 resolve_src->last_resolve_src_render = ctx->hw_setup->render_count - 1U;
2376
2377 /* Mark the resolve destination. */
2378 assert(resolve_dst->last_resolve_dst_render <
2379 (int32_t)(ctx->hw_setup->render_count - 1U));
2380 resolve_dst->last_resolve_dst_render = ctx->hw_setup->render_count - 1U;
2381
2382 /* If we can't down scale through the PBE then the src must be stored
2383 * for transfer down scale.
2384 */
2385 if (!resolve_src->is_pbe_downscalable &&
2386 resolve_src->last_read < (int32_t)ctx->pass->subpass_count) {
2387 resolve_src->last_read = (int32_t)ctx->pass->subpass_count;
2388 resolve_src->remaining_count++;
2389 }
2390 }
2391
2392 /* For subpasses dependent on this subpass decrement the unscheduled
2393 * dependency count.
2394 */
2395 for (uint32_t i = 0U; i < int_subpass->out_subpass_count; i++) {
2396 struct pvr_render_int_subpass *int_dst_subpass =
2397 int_subpass->out_subpasses[i];
2398
2399 assert(int_dst_subpass->in_subpass_count > 0U);
2400 int_dst_subpass->in_subpass_count--;
2401 }
2402
2403 return VK_SUCCESS;
2404 }
2405
2406 static uint32_t pvr_count_uses_in_attachment_list(
2407 struct pvr_render_attachment *attachments,
2408 uint32_t size,
2409 uint32_t attach_idx)
2410 {
2411 uint32_t count = 0U;
2412
2413 for (uint32_t i = 0U; i < size; i++) {
2414 if (attachments[i].attachment_idx == attach_idx)
2415 count++;
2416 }
2417
2418 return count;
2419 }
2420
2421 static uint32_t pvr_count_uses_in_list(uint32_t *attachments,
2422 uint32_t size,
2423 uint32_t attach_idx)
2424 {
2425 uint32_t count = 0U;
2426
2427 for (uint32_t i = 0U; i < size; i++) {
2428 if (attachments[i] == attach_idx)
2429 count++;
2430 }
2431
2432 return count;
2433 }
2434
2435 static void
2436 pvr_count_uses_in_color_output_list(struct pvr_render_subpass *subpass,
2437 uint32_t attach_idx,
2438 uint32_t *color_output_count_out,
2439 uint32_t *resolve_output_count_out)
2440 {
2441 uint32_t resolve_count = 0U;
2442 uint32_t color_count = 0U;
2443
2444 for (uint32_t i = 0U; i < subpass->color_count; i++) {
2445 if (subpass->color_attachments[i] == attach_idx) {
2446 color_count++;
2447
2448 if (subpass->resolve_attachments &&
2449 subpass->resolve_attachments[i] != VK_ATTACHMENT_UNUSED)
2450 resolve_count++;
2451 }
2452 }
2453
2454 *color_output_count_out = color_count;
2455 *resolve_output_count_out = resolve_count;
2456 }
2457
2458 void pvr_arch_destroy_renderpass_hwsetup(const VkAllocationCallbacks *alloc,
2459 struct pvr_renderpass_hwsetup *hw_setup)
2460 {
2461 for (uint32_t i = 0U; i < hw_setup->render_count; i++) {
2462 struct pvr_renderpass_hwsetup_render *hw_render = &hw_setup->renders[i];
2463
2464 vk_free(alloc, hw_render->eot_surfaces);
2465 vk_free(alloc, hw_render->eot_setup.mrt_resources);
2466 vk_free(alloc, hw_render->init_setup.mrt_resources);
2467 vk_free(alloc, hw_render->color_init);
2468
2469 for (uint32_t j = 0U; j < hw_render->subpass_count; j++) {
2470 struct pvr_renderpass_hwsetup_subpass *subpass =
2471 &hw_render->subpasses[j];
2472
2473 vk_free(alloc, subpass->color_initops);
2474 vk_free(alloc, subpass->input_access);
2475 vk_free(alloc, subpass->setup.mrt_resources);
2476 }
2477
2478 vk_free(alloc, hw_render->subpasses);
2479 }
2480
2481 vk_free(alloc, hw_setup->renders);
2482 vk_free(alloc, hw_setup);
2483 }
2484
2485 VkResult pvr_arch_create_renderpass_hwsetup(
2486 struct pvr_device *device,
2487 const VkAllocationCallbacks *alloc,
2488 struct pvr_render_pass *pass,
2489 bool disable_merge,
2490 struct pvr_renderpass_hwsetup **const hw_setup_out)
2491 {
2492 struct pvr_render_int_attachment *int_attachments;
2493 struct pvr_render_int_subpass *int_subpasses;
2494 struct pvr_renderpass_hw_map *subpass_map;
2495 struct pvr_renderpass_hwsetup *hw_setup;
2496 struct pvr_renderpass_context *ctx;
2497 bool requires_frag_pr = false;
2498 bool *surface_allocate;
2499 VkResult result;
2500
2501 VK_MULTIALLOC(ma);
2502 vk_multialloc_add(&ma, &hw_setup, __typeof__(*hw_setup), 1);
2503 vk_multialloc_add(&ma,
2504 &surface_allocate,
2505 __typeof__(*surface_allocate),
2506 pass->attachment_count);
2507 vk_multialloc_add(&ma,
2508 &subpass_map,
2509 __typeof__(*subpass_map),
2510 pass->subpass_count);
2511
2512 if (!vk_multialloc_zalloc(&ma, alloc, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT))
2513 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
2514
2515 hw_setup->surface_allocate = surface_allocate;
2516 hw_setup->subpass_map = subpass_map;
2517
2518 VK_MULTIALLOC(ma_ctx);
2519 vk_multialloc_add(&ma_ctx, &ctx, __typeof__(*ctx), 1);
2520 vk_multialloc_add(&ma_ctx,
2521 &int_attachments,
2522 __typeof__(*int_attachments),
2523 pass->attachment_count);
2524 vk_multialloc_add(&ma_ctx,
2525 &int_subpasses,
2526 __typeof__(*int_subpasses),
2527 pass->subpass_count);
2528
2529 if (!vk_multialloc_zalloc(&ma_ctx,
2530 alloc,
2531 VK_SYSTEM_ALLOCATION_SCOPE_COMMAND)) {
2532 vk_free(alloc, hw_setup);
2533 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
2534 }
2535
2536 ctx->pass = pass;
2537 ctx->hw_setup = hw_setup;
2538 ctx->int_attach = int_attachments;
2539 ctx->int_subpasses = int_subpasses;
2540 ctx->allocator = alloc;
2541
2542 for (uint32_t i = 0U; i < pass->attachment_count; i++) {
2543 struct pvr_render_pass_attachment *attachment = &pass->attachments[i];
2544 struct pvr_render_int_attachment *int_attach = &ctx->int_attach[i];
2545 const VkFormat format = attachment->vk_format;
2546 uint32_t pixel_size_in_chunks;
2547 uint32_t pixel_size_in_bits;
2548
2549 /* TODO: Add support for packing multiple attachments into the same
2550 * register.
2551 */
2552 const uint32_t part_bits = 0;
2553
2554 if (vk_format_is_color(format) &&
2555 pvr_arch_get_pbe_accum_format(attachment->vk_format) ==
2556 PVR_PBE_ACCUM_FORMAT_INVALID) {
2557 /* The VkFormat is not supported as a color attachment so `0`.
2558 * Vulkan doesn't seems to restrict vkCreateRenderPass() to supported
2559 * formats only.
2560 */
2561 pixel_size_in_bits = 0;
2562 } else {
2563 pixel_size_in_bits =
2564 pvr_get_accum_format_bitsize(attachment->vk_format);
2565 }
2566
2567 int_attach->resource.type = USC_MRT_RESOURCE_TYPE_INVALID;
2568 int_attach->resource.intermediate_size =
2569 DIV_ROUND_UP(pixel_size_in_bits, CHAR_BIT);
2570 int_attach->resource.mrt_desc.intermediate_size =
2571 int_attach->resource.intermediate_size;
2572
2573 pixel_size_in_chunks = DIV_ROUND_UP(pixel_size_in_bits, 32U);
2574 for (uint32_t j = 0U; j < pixel_size_in_chunks; j++)
2575 int_attach->resource.mrt_desc.valid_mask[j] = ~0;
2576
2577 if (part_bits > 0U) {
2578 int_attach->resource.mrt_desc.valid_mask[pixel_size_in_chunks] =
2579 BITFIELD_MASK(part_bits);
2580 }
2581
2582 int_attach->load_op = pass->attachments[i].load_op;
2583 int_attach->stencil_load_op = pass->attachments[i].stencil_load_op;
2584 int_attach->attachment = attachment;
2585 int_attach->first_use = -1;
2586 int_attach->last_use = -1;
2587 int_attach->last_read = -1;
2588 int_attach->mrt_idx = -1;
2589 int_attach->last_resolve_dst_render = -1;
2590 int_attach->last_resolve_src_render = -1;
2591 int_attach->z_replicate = false;
2592 int_attach->is_pbe_downscalable = attachment->is_pbe_downscalable;
2593
2594 /* Count the number of references to this attachment in subpasses. */
2595 for (uint32_t j = 0U; j < pass->subpass_count; j++) {
2596 struct pvr_render_subpass *subpass = &pass->subpasses[j];
2597 const uint32_t input_attachment_uses =
2598 pvr_count_uses_in_attachment_list(subpass->input_attachments,
2599 subpass->input_count,
2600 i);
2601
2602 const uint32_t preserve_attachment_uses =
2603 pvr_count_uses_in_attachment_list(subpass->preserve_attachments,
2604 subpass->preserve_count,
2605 i);
2606
2607 uint32_t resolve_output_uses;
2608 uint32_t color_output_uses;
2609 uint32_t total_output_uses;
2610
2611 pvr_count_uses_in_color_output_list(subpass,
2612 i,
2613 &color_output_uses,
2614 &resolve_output_uses);
2615
2616 total_output_uses = color_output_uses + resolve_output_uses +
2617 input_attachment_uses + preserve_attachment_uses;
2618
2619 if (total_output_uses != 0U)
2620 int_attach->last_read = j;
2621
2622 int_attach->remaining_count += total_output_uses;
2623
2624 if ((uint32_t)subpass->depth_stencil_attachment == i)
2625 int_attach->remaining_count++;
2626
2627 requires_frag_pr |= resolve_output_uses != 0;
2628 /* TODO: Should this be checking the normal attachment store op? */
2629 requires_frag_pr |= color_output_uses != 0 &&
2630 pass->attachments[i].stencil_store_op !=
2631 VK_ATTACHMENT_STORE_OP_STORE;
2632 }
2633
2634 if (int_attach->attachment->aspects & VK_IMAGE_ASPECT_STENCIL_BIT) {
2635 int_attach->stencil_remaining_count = int_attach->remaining_count;
2636 if (pass->attachments[i].stencil_store_op ==
2637 VK_ATTACHMENT_STORE_OP_STORE) {
2638 int_attach->stencil_remaining_count++;
2639 }
2640 }
2641
2642 if (pass->attachments[i].store_op == VK_ATTACHMENT_STORE_OP_STORE) {
2643 int_attach->remaining_count++;
2644 int_attach->last_read = pass->subpass_count;
2645 }
2646 }
2647
2648 for (uint32_t i = 0U; i < pass->subpass_count; i++) {
2649 struct pvr_render_int_subpass *int_subpass = &ctx->int_subpasses[i];
2650
2651 int_subpass->subpass = &pass->subpasses[i];
2652 int_subpass->out_subpass_count = 0U;
2653 int_subpass->out_subpasses = NULL;
2654 int_subpass->in_subpass_count = int_subpass->subpass->dep_count;
2655 }
2656
2657 /* For each dependency of a subpass create an edge in the opposite
2658 * direction.
2659 */
2660 for (uint32_t i = 0U; i < pass->subpass_count; i++) {
2661 struct pvr_render_int_subpass *int_subpass = &ctx->int_subpasses[i];
2662
2663 for (uint32_t j = 0U; j < int_subpass->in_subpass_count; j++) {
2664 uint32_t src_idx = int_subpass->subpass->dep_list[j];
2665 struct pvr_render_int_subpass *int_src_subpass;
2666 struct pvr_render_int_subpass **out_subpasses;
2667
2668 assert(src_idx < pass->subpass_count);
2669
2670 int_src_subpass = &ctx->int_subpasses[src_idx];
2671
2672 out_subpasses =
2673 vk_realloc(ctx->allocator,
2674 int_src_subpass->out_subpasses,
2675 sizeof(int_src_subpass->out_subpasses[0U]) *
2676 (int_src_subpass->out_subpass_count + 1U),
2677 8U,
2678 VK_SYSTEM_ALLOCATION_SCOPE_COMMAND);
2679 if (!out_subpasses) {
2680 result = vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
2681 goto end_create_renderpass_hwsetup;
2682 }
2683
2684 int_src_subpass->out_subpasses = out_subpasses;
2685 int_src_subpass->out_subpasses[int_src_subpass->out_subpass_count] =
2686 int_subpass;
2687 int_src_subpass->out_subpass_count++;
2688 }
2689 }
2690
2691 pvr_reset_render(ctx);
2692
2693 for (uint32_t i = 0U; i < pass->subpass_count; i++) {
2694 uint32_t j;
2695
2696 /* Find a subpass with no unscheduled dependencies. */
2697 for (j = 0U; j < pass->subpass_count; j++) {
2698 struct pvr_render_int_subpass *int_subpass = &ctx->int_subpasses[j];
2699
2700 if (int_subpass->subpass && int_subpass->in_subpass_count == 0U)
2701 break;
2702 }
2703 assert(j < pass->subpass_count);
2704
2705 result = pvr_schedule_subpass(device, ctx, j);
2706 if (result != VK_SUCCESS)
2707 goto end_create_renderpass_hwsetup;
2708
2709 if (disable_merge) {
2710 result = pvr_close_render(device, ctx);
2711 if (result != VK_SUCCESS)
2712 goto end_create_renderpass_hwsetup;
2713 }
2714
2715 ctx->int_subpasses[j].subpass = NULL;
2716 }
2717
2718 /* Finalise the last in-progress render. */
2719 result = pvr_close_render(device, ctx);
2720
2721 for (uint32_t i = 0; i < hw_setup->render_count; i++)
2722 hw_setup->renders[i].requires_frag_pr = requires_frag_pr;
2723
2724 end_create_renderpass_hwsetup:
2725 if (result != VK_SUCCESS) {
2726 pvr_free_render(ctx);
2727
2728 if (hw_setup) {
2729 pvr_arch_destroy_renderpass_hwsetup(alloc, hw_setup);
2730 hw_setup = NULL;
2731 }
2732 }
2733
2734 for (uint32_t i = 0U; i < pass->subpass_count; i++) {
2735 struct pvr_render_int_subpass *int_subpass = &ctx->int_subpasses[i];
2736
2737 if (int_subpass->out_subpass_count > 0U)
2738 vk_free(alloc, int_subpass->out_subpasses);
2739 }
2740
2741 vk_free(alloc, ctx);
2742
2743 *hw_setup_out = hw_setup;
2744
2745 return result;
2746 }