Debian Codegraph @

media_softlet/linux/common/os/i915_production/mos_bufmgr.c

raw
1 /**************************************************************************
2 *
3 * Copyright © 2007 Red Hat Inc.
4 * Copyright © 2007-2022 Intel Corporation
5 * Copyright 2006 Tungsten Graphics, Inc., Bismarck, ND., USA
6 * All Rights Reserved.
7 *
8 * Permission is hereby granted, free of charge, to any person obtaining a
9 * copy of this software and associated documentation files (the
10 * "Software"), to deal in the Software without restriction, including
11 * without limitation the rights to use, copy, modify, merge, publish,
12 * distribute, sub license, and/or sell copies of the Software, and to
13 * permit persons to whom the Software is furnished to do so, subject to
14 * the following conditions:
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
20 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
21 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
22 * USE OR OTHER DEALINGS IN THE SOFTWARE.
23 *
24 * The above copyright notice and this permission notice (including the
25 * next paragraph) shall be included in all copies or substantial portions
26 * of the Software.
27 *
28 *
29 **************************************************************************/
30 /*
31 * Authors: Thomas Hellström <thomas-at-tungstengraphics-dot-com>
32 * Keith Whitwell <keithw-at-tungstengraphics-dot-com>
33 * Eric Anholt <[email protected]>
34 * Dave Airlie <[email protected]>
35 */
36
37 #ifdef HAVE_CONFIG_H
38 #include "config.h"
39 #endif
40
41 #include "xf86drm.h"
42 #include "xf86atomic.h"
43 #include <fcntl.h>
44 #include <stdio.h>
45 #include <stdlib.h>
46 #include <string.h>
47 #include <unistd.h>
48 #include <assert.h>
49 #include <pthread.h>
50 #include <sys/ioctl.h>
51 #include <sys/stat.h>
52 #include <sys/types.h>
53 #include <stdbool.h>
54
55 #include "errno.h"
56 #ifndef ETIME
57 #define ETIME ETIMEDOUT
58 #endif
59 #include "libdrm_macros.h"
60 #include "libdrm_lists.h"
61 #include "mos_bufmgr.h"
62 #include "mos_bufmgr_priv.h"
63 #include "mos_bufmgr_xe.h"
64 #include "string.h"
65 #include "i915_drm.h"
66 #include "mos_vma.h"
67 #include "mos_util_debug.h"
68 #include "mos_bufmgr_prelim.h"
69 #include "mos_oca_defs_specific.h"
70 #include "intel_hwconfig_types.h"
71 #include "mos_utilities.h"
72 #include "linux_system_info.h"
73 #include "mos_os_specific.h"
74
75 #ifdef HAVE_VALGRIND
76 #include <valgrind.h>
77 #include <memcheck.h>
78 #define VG(x) x
79 #else
80 #define VG(x)
81 #endif
82
83 #define memclear(s) memset(&s, 0, sizeof(s))
84
85 #define MOS_DBG(...) do { \
86 if (bufmgr_gem != nullptr && bufmgr_gem->bufmgr.debug) \
87 fprintf(stderr, __VA_ARGS__); \
88 } while (0)
89
90 #define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
91 #define MAX2(A, B) ((A) > (B) ? (A) : (B))
92
93 /**
94 * upper_32_bits - return bits 32-63 of a number
95 * @n: the number we're accessing
96 *
97 * A basic shift-right of a 64- or 32-bit quantity. Use this to suppress
98 * the "right shift count >= width of type" warning when that quantity is
99 * 32-bits.
100 */
101 #define upper_32_bits(n) ((__u32)(((n) >> 16) >> 16))
102
103 /**
104 * lower_32_bits - return bits 0-31 of a number
105 * @n: the number we're accessing
106 */
107 #define lower_32_bits(n) ((__u32)(n))
108
109 #define PCI_CHIP_I915_G 0x2582
110 #define PCI_CHIP_E7221_G 0x258A
111 #define PCI_CHIP_I915_GM 0x2592
112
113 #define IS_915(devid) ((devid) == PCI_CHIP_I915_G || \
114 (devid) == PCI_CHIP_E7221_G || \
115 (devid) == PCI_CHIP_I915_GM)
116
117 #define INITIAL_SOFTPIN_TARGET_COUNT 1024
118
119 struct mos_gem_bo_bucket {
120 drmMMListHead head;
121 unsigned long size;
122 };
123
124 typedef struct mos_bufmgr_gem {
125 struct mos_bufmgr bufmgr;
126
127 atomic_t refcount;
128
129 int fd;
130
131 int max_relocs;
132
133 pthread_mutex_t lock;
134
135 struct drm_i915_gem_exec_object *exec_objects;
136 struct drm_i915_gem_exec_object2 *exec2_objects;
137 struct mos_linux_bo **exec_bos;
138 int exec_size;
139 int exec_count;
140
141 /** Array of lists of cached gem objects of power-of-two sizes */
142 struct mos_gem_bo_bucket cache_bucket[14 * 4];
143 int num_buckets;
144 time_t time;
145
146 drmMMListHead managers;
147
148 drmMMListHead named;
149
150 uint64_t gtt_size;
151 int available_fences;
152 int pci_device;
153 unsigned int has_bsd : 1;
154 unsigned int has_bsd2 : 1;
155 unsigned int has_blt : 1;
156 unsigned int has_relaxed_fencing : 1;
157 unsigned int has_llc : 1;
158 unsigned int has_wait_timeout : 1;
159 unsigned int bo_reuse : 1;
160 unsigned int no_exec : 1;
161 unsigned int has_vebox : 1;
162 unsigned int has_ext_mmap : 1;
163 unsigned int has_fence_reg : 1;
164 unsigned int has_lmem : 1;
165 unsigned int has_mmap_offset : 1;
166 bool fenced_relocs;
167
168 struct {
169 void *ptr;
170 uint32_t handle;
171 } userptr_active;
172
173 // manage address for softpin buffer object
174 mos_vma_heap vma_heap[MEMZONE_COUNT];
175 bool use_softpin;
176 bool softpin_va1Malign;
177
178 bool object_capture_disabled;
179
180 BufmgrPrelim *prelim;
181
182 #define MEM_PROFILER_BUFFER_SIZE 256
183 char mem_profiler_buffer[MEM_PROFILER_BUFFER_SIZE];
184 char* mem_profiler_path;
185 int mem_profiler_fd;
186
187 int device_type;
188
189 uint32_t ts_freq;
190 } mos_bufmgr_gem;
191
192 #define DRM_INTEL_RELOC_FENCE (1<<0)
193
194 struct mos_reloc_target {
195 struct mos_linux_bo *bo;
196 int flags;
197 };
198
199 struct mos_softpin_target {
200 struct mos_linux_bo *bo;
201 int flags;
202 };
203
204 struct mos_bo_gem {
205 struct mos_linux_bo bo;
206
207 atomic_t refcount;
208 uint32_t gem_handle;
209 const char *name;
210
211 /**
212 * Kenel-assigned global name for this object
213 *
214 * List contains both flink named and prime fd'd objects
215 */
216 unsigned int global_name;
217 drmMMListHead name_list;
218
219 /**
220 * Index of the buffer within the validation list while preparing a
221 * batchbuffer execution.
222 */
223 int validate_index;
224
225 /**
226 * Current tiling mode
227 */
228 uint32_t tiling_mode;
229 uint32_t swizzle_mode;
230 unsigned long stride;
231
232 time_t free_time;
233
234 /** Array passed to the DRM containing relocation information. */
235 struct drm_i915_gem_relocation_entry *relocs;
236 /**
237 * Array of info structs corresponding to relocs[i].target_handle etc
238 */
239 struct mos_reloc_target *reloc_target_info;
240 /** Number of entries in relocs */
241 int reloc_count;
242 /** Array of BOs that are referenced by this buffer and will be softpinned */
243 struct mos_softpin_target *softpin_target;
244 /** Number softpinned BOs that are referenced by this buffer */
245 int softpin_target_count;
246 /** Maximum amount of softpinned BOs that are referenced by this buffer */
247 int max_softpin_target_count;
248
249 /** Mapped address for the buffer, saved across map/unmap cycles */
250 void *mem_virtual;
251 /** Uncached Mapped address for the buffer, saved across map/unmap cycles */
252 void *mem_wc_virtual;
253 /** GTT virtual address for the buffer, saved across map/unmap cycles */
254 void *gtt_virtual;
255 /**
256 * Virtual address of the buffer allocated by user, used for userptr
257 * objects only.
258 */
259 void *user_virtual;
260 int map_count;
261
262 /** BO cache list */
263 drmMMListHead head;
264
265 /**
266 * Boolean of whether this BO and its children have been included in
267 * the current drm_intel_bufmgr_check_aperture_space() total.
268 */
269 bool included_in_check_aperture;
270
271 /**
272 * Boolean of whether this buffer has been used as a relocation
273 * target and had its size accounted for, and thus can't have any
274 * further relocations added to it.
275 */
276 bool used_as_reloc_target;
277
278 /**
279 * Boolean of whether we have encountered an error whilst building the relocation tree.
280 */
281 bool has_error;
282
283 /**
284 * Boolean of whether this buffer can be re-used
285 */
286 bool reusable;
287
288 /**
289 * Boolean of whether the GPU is definitely not accessing the buffer.
290 *
291 * This is only valid when reusable, since non-reusable
292 * buffers are those that have been shared wth other
293 * processes, so we don't know their state.
294 */
295 bool idle;
296
297 /**
298 * Boolean of whether this buffer was allocated with userptr
299 */
300 bool is_userptr;
301
302 /**
303 * Boolean of whether this buffer can be placed in the full 48-bit
304 * address range on gen8+.
305 *
306 * By default, buffers will be keep in a 32-bit range, unless this
307 * flag is explicitly set.
308 */
309 bool use_48b_address_range;
310
311 /**
312 * Whether this buffer is softpinned at offset specified by the user
313 */
314 bool is_softpin;
315
316 /*
317 * Whether to remove the dependency of this bo in exebuf.
318 */
319 bool exec_async;
320
321 /*
322 * Whether to remove the dependency of this bo in exebuf.
323 */
324 bool exec_capture;
325
326 /**
327 * Size in bytes of this buffer and its relocation descendents.
328 *
329 * Used to avoid costly tree walking in
330 * drm_intel_bufmgr_check_aperture in the common case.
331 */
332 int reloc_tree_size;
333
334 /**
335 * Number of potential fence registers required by this buffer and its
336 * relocations.
337 */
338 int reloc_tree_fences;
339
340 /** Flags that we may need to do the SW_FINSIH ioctl on unmap. */
341 bool mapped_cpu_write;
342
343 /**
344 * Size to pad the object to.
345 *
346 */
347 uint64_t pad_to_size;
348
349 /**
350 * Memory region on created the surfaces for local/system memory
351 */
352 int mem_region;
353 };
354
355 struct mos_exec_info {
356 /* save all exec2_objects for res*/
357 struct drm_i915_gem_exec_object2* obj;
358 /* save batch buffer*/
359 struct drm_i915_gem_exec_object2* batch_obj;
360 /* save previous ptr to void mem leak when free*/
361 struct drm_i915_gem_exec_object2* pSavePreviousExec2Objects;
362 /*bo resource count*/
363 uint32_t obj_count;
364 /*batch buffer bo count*/
365 uint32_t batch_count;
366 /*remain size of 'obj'*/
367 uint32_t obj_remain_size;
368 #define OBJ512_SIZE 512
369 };
370
371 static unsigned int
372 mos_gem_estimate_batch_space(struct mos_linux_bo ** bo_array, int count);
373
374 static unsigned int
375 mos_gem_compute_batch_space(struct mos_linux_bo ** bo_array, int count);
376
377 static int
378 mos_gem_bo_get_tiling(struct mos_linux_bo *bo, uint32_t * tiling_mode,
379 uint32_t * swizzle_mode);
380
381 static int
382 mos_gem_bo_check_mem_region_internal(struct mos_linux_bo *bo,
383 int mem_type);
384
385 static int
386 mos_gem_bo_set_tiling_internal(struct mos_linux_bo *bo,
387 uint32_t tiling_mode,
388 uint32_t stride);
389
390 static void mos_gem_bo_unreference_locked_timed(struct mos_linux_bo *bo,
391 time_t time);
392
393 static void mos_gem_bo_unreference(struct mos_linux_bo *bo);
394 static bool mos_gem_bo_is_softpin(struct mos_linux_bo *bo);
395 static void mos_gem_bo_start_gtt_access(struct mos_linux_bo *bo, int write_enable);
396 static void mos_gem_bo_free(struct mos_linux_bo *bo);
397
398 static int mos_bufmgr_get_driver_info(struct mos_bufmgr *bufmgr, struct LinuxDriverInfo *drvInfo);
399
400 static inline struct mos_bo_gem *to_bo_gem(struct mos_linux_bo *bo)
401 {
402 return (struct mos_bo_gem *)bo;
403 }
404
405 static unsigned long
406 mos_gem_bo_tile_size(struct mos_bufmgr_gem *bufmgr_gem, unsigned long size,
407 uint32_t *tiling_mode)
408 {
409 unsigned long min_size, max_size;
410 unsigned long i;
411
412 if (*tiling_mode == I915_TILING_NONE)
413 return size;
414
415 /* 965+ just need multiples of page size for tiling */
416 return ROUND_UP_TO(size, 4096);
417
418 }
419
420 /*
421 * Round a given pitch up to the minimum required for X tiling on a
422 * given chip. We use 512 as the minimum to allow for a later tiling
423 * change.
424 */
425 static unsigned long
426 mos_gem_bo_tile_pitch(struct mos_bufmgr_gem *bufmgr_gem,
427 unsigned long pitch, uint32_t *tiling_mode)
428 {
429 unsigned long tile_width;
430 unsigned long i;
431
432 /* If untiled, then just align it so that we can do rendering
433 * to it with the 3D engine.
434 */
435 if (*tiling_mode == I915_TILING_NONE)
436 return ALIGN(pitch, 64);
437
438 if (*tiling_mode == I915_TILING_X
439 || (IS_915(bufmgr_gem->pci_device)
440 && *tiling_mode == I915_TILING_Y))
441 tile_width = 512;
442 else
443 tile_width = 128;
444
445 /* 965 is flexible */
446 return ROUND_UP_TO(pitch, tile_width);
447 }
448
449 static struct mos_gem_bo_bucket *
450 mos_gem_bo_bucket_for_size(struct mos_bufmgr_gem *bufmgr_gem,
451 unsigned long size)
452 {
453 int i;
454
455 for (i = 0; i < bufmgr_gem->num_buckets; i++) {
456 struct mos_gem_bo_bucket *bucket =
457 &bufmgr_gem->cache_bucket[i];
458 if (bucket->size >= size) {
459 return bucket;
460 }
461 }
462
463 return nullptr;
464 }
465
466 static void
467 mos_gem_dump_validation_list(struct mos_bufmgr_gem *bufmgr_gem)
468 {
469 int i, j;
470
471 for (i = 0; i < bufmgr_gem->exec_count; i++) {
472 struct mos_linux_bo *bo = bufmgr_gem->exec_bos[i];
473 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
474
475 if (bo_gem->relocs == nullptr || bo_gem->softpin_target == nullptr) {
476 MOS_DBG("%2d: %d %s(%s)\n", i, bo_gem->gem_handle,
477 bo_gem->is_softpin ? "*" : "",
478 bo_gem->name);
479 continue;
480 }
481
482 for (j = 0; j < bo_gem->reloc_count; j++) {
483 struct mos_linux_bo *target_bo = bo_gem->reloc_target_info[j].bo;
484 struct mos_bo_gem *target_gem =
485 (struct mos_bo_gem *) target_bo;
486
487 MOS_DBG("%2d: %d %s(%s)@0x%08x %08x -> "
488 "%d (%s)@0x%08x %08x + 0x%08x\n",
489 i,
490 bo_gem->gem_handle,
491 bo_gem->is_softpin ? "*" : "",
492 bo_gem->name,
493 upper_32_bits(bo_gem->relocs[j].offset),
494 lower_32_bits(bo_gem->relocs[j].offset),
495 target_gem->gem_handle,
496 target_gem->name,
497 upper_32_bits(target_bo->offset64),
498 lower_32_bits(target_bo->offset64),
499 bo_gem->relocs[j].delta);
500 }
501
502 for (j = 0; j < bo_gem->softpin_target_count; j++) {
503 struct mos_linux_bo *target_bo = bo_gem->softpin_target[j].bo;
504 struct mos_bo_gem *target_gem =
505 (struct mos_bo_gem *) target_bo;
506 MOS_DBG("%2d: %d %s(%s) -> "
507 "%d *(%s)@0x%08x %08x\n",
508 i,
509 bo_gem->gem_handle,
510 bo_gem->is_softpin ? "*" : "",
511 bo_gem->name,
512 target_gem->gem_handle,
513 target_gem->name,
514 upper_32_bits(target_bo->offset64),
515 lower_32_bits(target_bo->offset64));
516 }
517 }
518 }
519
520 static inline void
521 mos_gem_bo_reference(struct mos_linux_bo *bo)
522 {
523 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
524
525 atomic_inc(&bo_gem->refcount);
526 }
527
528 /**
529 * Adds the given buffer to the list of buffers to be validated (moved into the
530 * appropriate memory type) with the next batch submission.
531 *
532 * If a buffer is validated multiple times in a batch submission, it ends up
533 * with the intersection of the memory type flags and the union of the
534 * access flags.
535 */
536 static void
537 mos_add_validate_buffer(struct mos_linux_bo *bo)
538 {
539 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
540 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
541 int index;
542 struct drm_i915_gem_exec_object *exec_objects;
543 struct mos_linux_bo **exec_bos;
544
545 if (bo_gem->validate_index != -1)
546 return;
547
548 /* Extend the array of validation entries as necessary. */
549 if (bufmgr_gem->exec_count == bufmgr_gem->exec_size) {
550 int new_size = bufmgr_gem->exec_size * 2;
551
552 if (new_size == 0)
553 new_size = ARRAY_INIT_SIZE;
554
555 exec_objects = (struct drm_i915_gem_exec_object *)realloc(bufmgr_gem->exec_objects,
556 sizeof(*bufmgr_gem->exec_objects) * new_size);
557 if (!exec_objects)
558 return;
559
560 bufmgr_gem->exec_objects = exec_objects;
561
562 exec_bos = (struct mos_linux_bo **)realloc(bufmgr_gem->exec_bos,
563 sizeof(*bufmgr_gem->exec_bos) * new_size);
564 if (!exec_bos)
565 return;
566
567 bufmgr_gem->exec_bos = exec_bos;
568 bufmgr_gem->exec_size = new_size;
569 }
570
571 index = bufmgr_gem->exec_count;
572 bo_gem->validate_index = index;
573 /* Fill in array entry */
574 bufmgr_gem->exec_objects[index].handle = bo_gem->gem_handle;
575 bufmgr_gem->exec_objects[index].relocation_count = bo_gem->reloc_count;
576 bufmgr_gem->exec_objects[index].relocs_ptr = (uintptr_t) bo_gem->relocs;
577 bufmgr_gem->exec_objects[index].alignment = bo->align;
578 bufmgr_gem->exec_objects[index].offset = 0;
579 bufmgr_gem->exec_bos[index] = bo;
580 bufmgr_gem->exec_count++;
581 }
582
583 static void
584 mos_add_validate_buffer2(struct mos_linux_bo *bo, int need_fence)
585 {
586 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bo->bufmgr;
587 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *)bo;
588 int index;
589 struct drm_i915_gem_exec_object2 *exec2_objects;
590 struct mos_linux_bo **exec_bos;
591 int flags = 0;
592
593 if (need_fence)
594 flags |= EXEC_OBJECT_NEEDS_FENCE;
595 if (bo_gem->pad_to_size)
596 flags |= EXEC_OBJECT_PAD_TO_SIZE;
597 if (bo_gem->use_48b_address_range)
598 flags |= EXEC_OBJECT_SUPPORTS_48B_ADDRESS;
599 if (bo_gem->is_softpin)
600 flags |= EXEC_OBJECT_PINNED;
601 if (bo_gem->exec_async)
602 flags |= EXEC_OBJECT_ASYNC;
603 if (bo_gem->exec_capture)
604 flags |= EXEC_OBJECT_CAPTURE;
605
606 if (bo_gem->validate_index != -1) {
607 bufmgr_gem->exec2_objects[bo_gem->validate_index].flags |= flags;
608 return;
609 }
610
611 /* Extend the array of validation entries as necessary. */
612 if (bufmgr_gem->exec_count == bufmgr_gem->exec_size) {
613 int new_size = bufmgr_gem->exec_size * 2;
614
615 if (new_size == 0)
616 new_size = ARRAY_INIT_SIZE;
617 exec2_objects = (struct drm_i915_gem_exec_object2 *)
618 realloc(bufmgr_gem->exec2_objects,
619 sizeof(*bufmgr_gem->exec2_objects) * new_size);
620 if (!exec2_objects)
621 return;
622
623 bufmgr_gem->exec2_objects = exec2_objects;
624
625 exec_bos = (struct mos_linux_bo **)realloc(bufmgr_gem->exec_bos,
626 sizeof(*bufmgr_gem->exec_bos) * new_size);
627 if (!exec_bos)
628 return;
629
630 bufmgr_gem->exec_bos = exec_bos;
631 bufmgr_gem->exec_size = new_size;
632 }
633
634 index = bufmgr_gem->exec_count;
635 bo_gem->validate_index = index;
636 /* Fill in array entry */
637 bufmgr_gem->exec2_objects[index].handle = bo_gem->gem_handle;
638 bufmgr_gem->exec2_objects[index].relocation_count = bo_gem->reloc_count;
639 bufmgr_gem->exec2_objects[index].relocs_ptr = (uintptr_t)bo_gem->relocs;
640 bufmgr_gem->exec2_objects[index].alignment = bo->align;
641 bufmgr_gem->exec2_objects[index].offset = bo_gem->is_softpin ?
642 bo->offset64 : 0;
643 bufmgr_gem->exec_bos[index] = bo;
644 bufmgr_gem->exec2_objects[index].flags = flags;
645 bufmgr_gem->exec2_objects[index].rsvd1 = 0;
646 bufmgr_gem->exec2_objects[index].pad_to_size = bo_gem->pad_to_size;
647 bufmgr_gem->exec2_objects[index].rsvd2 = 0;
648 bufmgr_gem->exec_count++;
649 }
650
651 static void
652 mos_add_reloc_objects(struct mos_reloc_target reloc_target)
653 {
654 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)reloc_target.bo->bufmgr;
655 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *)reloc_target.bo;
656 int index;
657 struct drm_i915_gem_exec_object2 *exec2_objects;
658 struct mos_linux_bo **exec_bos;
659
660 if (bo_gem->validate_index != -1) {
661 bufmgr_gem->exec2_objects[bo_gem->validate_index].flags |= reloc_target.flags;
662 return;
663 }
664
665 /* Extend the array of validation entries as necessary. */
666 if (bufmgr_gem->exec_count == bufmgr_gem->exec_size) {
667 int new_size = bufmgr_gem->exec_size * 2;
668
669 if (new_size == 0)
670 new_size = ARRAY_INIT_SIZE;
671 exec2_objects = (struct drm_i915_gem_exec_object2 *)
672 realloc(bufmgr_gem->exec2_objects,
673 sizeof(*bufmgr_gem->exec2_objects) * new_size);
674 if (!exec2_objects)
675 {
676 MOS_DBG("realloc exec2_objects failed!\n");
677 return;
678 }
679
680 bufmgr_gem->exec2_objects = exec2_objects;
681
682 exec_bos = (struct mos_linux_bo **)realloc(bufmgr_gem->exec_bos,
683 sizeof(*bufmgr_gem->exec_bos) * new_size);
684 if (!exec_bos)
685 {
686 MOS_DBG("realloc exec_bo failed!\n");
687 return;
688 }
689
690 bufmgr_gem->exec_bos = exec_bos;
691 bufmgr_gem->exec_size = new_size;
692 }
693
694 index = bufmgr_gem->exec_count;
695 bo_gem->validate_index = index;
696 /* Fill in array entry */
697 bufmgr_gem->exec2_objects[index].handle = bo_gem->gem_handle;
698 bufmgr_gem->exec2_objects[index].relocation_count = bo_gem->reloc_count;
699 bufmgr_gem->exec2_objects[index].relocs_ptr = (uintptr_t)bo_gem->relocs;
700 bufmgr_gem->exec2_objects[index].alignment = reloc_target.bo->align;
701 bufmgr_gem->exec2_objects[index].offset = 0;
702 bufmgr_gem->exec_bos[index] = reloc_target.bo;
703 bufmgr_gem->exec2_objects[index].flags = reloc_target.flags;
704 bufmgr_gem->exec2_objects[index].rsvd1 = 0;
705 bufmgr_gem->exec2_objects[index].pad_to_size = bo_gem->pad_to_size;
706 bufmgr_gem->exec2_objects[index].rsvd2 = 0;
707 bufmgr_gem->exec_count++;
708 }
709
710 static void
711 mos_add_softpin_objects(struct mos_softpin_target softpin_target)
712 {
713 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)softpin_target.bo->bufmgr;
714 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *)softpin_target.bo;
715 int index;
716 struct drm_i915_gem_exec_object2 *exec2_objects;
717 struct mos_linux_bo **exec_bos;
718
719 if (bo_gem->validate_index != -1) {
720 bufmgr_gem->exec2_objects[bo_gem->validate_index].flags |= softpin_target.flags;
721 return;
722 }
723
724 /* Extend the array of validation entries as necessary. */
725 if (bufmgr_gem->exec_count == bufmgr_gem->exec_size) {
726 int new_size = bufmgr_gem->exec_size * 2;
727
728 if (new_size == 0)
729 new_size = ARRAY_INIT_SIZE;
730 exec2_objects = (struct drm_i915_gem_exec_object2 *)
731 realloc(bufmgr_gem->exec2_objects,
732 sizeof(*bufmgr_gem->exec2_objects) * new_size);
733 if (!exec2_objects)
734 {
735 MOS_DBG("realloc exec2_objects failed!\n");
736 return;
737 }
738
739 bufmgr_gem->exec2_objects = exec2_objects;
740
741 exec_bos = (struct mos_linux_bo **)realloc(bufmgr_gem->exec_bos,
742 sizeof(*bufmgr_gem->exec_bos) * new_size);
743 if (!exec_bos)
744 {
745 MOS_DBG("realloc exec_bo failed!\n");
746 return;
747 }
748
749 bufmgr_gem->exec_bos = exec_bos;
750 bufmgr_gem->exec_size = new_size;
751 }
752
753 index = bufmgr_gem->exec_count;
754 bo_gem->validate_index = index;
755 /* Fill in array entry */
756 bufmgr_gem->exec2_objects[index].handle = bo_gem->gem_handle;
757 bufmgr_gem->exec2_objects[index].relocation_count = bo_gem->reloc_count;
758 bufmgr_gem->exec2_objects[index].relocs_ptr = (uintptr_t)bo_gem->relocs;
759 bufmgr_gem->exec2_objects[index].alignment = softpin_target.bo->align;
760 bufmgr_gem->exec2_objects[index].offset = softpin_target.bo->offset64;
761 bufmgr_gem->exec2_objects[index].flags = softpin_target.flags;
762 bufmgr_gem->exec2_objects[index].pad_to_size = bo_gem->pad_to_size;
763 bufmgr_gem->exec2_objects[index].rsvd1 = 0;
764 bufmgr_gem->exec2_objects[index].rsvd2 = 0;
765 bufmgr_gem->exec_bos[index] = softpin_target.bo;
766 bufmgr_gem->exec_count++;
767 }
768
769 #define RELOC_BUF_SIZE(x) ((I915_RELOC_HEADER + x * I915_RELOC0_STRIDE) * \
770 sizeof(uint32_t))
771
772 static void
773 mos_bo_gem_set_in_aperture_size(struct mos_bufmgr_gem *bufmgr_gem,
774 struct mos_bo_gem *bo_gem,
775 unsigned int alignment)
776 {
777 unsigned int size;
778
779 assert(!bo_gem->used_as_reloc_target);
780
781 /* The older chipsets are far-less flexible in terms of tiling,
782 * and require tiled buffer to be size aligned in the aperture.
783 * This means that in the worst possible case we will need a hole
784 * twice as large as the object in order for it to fit into the
785 * aperture. Optimal packing is for wimps.
786 */
787 size = bo_gem->bo.size;
788
789 bo_gem->reloc_tree_size = size + alignment;
790 }
791
792 static int
793 mos_setup_reloc_list(struct mos_linux_bo *bo)
794 {
795 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
796 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
797 unsigned int max_relocs = bufmgr_gem->max_relocs;
798
799 if (bo->size / 4 < max_relocs)
800 max_relocs = bo->size / 4;
801
802 bo_gem->relocs = (struct drm_i915_gem_relocation_entry *)malloc(max_relocs *
803 sizeof(struct drm_i915_gem_relocation_entry));
804 bo_gem->reloc_target_info = (struct mos_reloc_target *)malloc(max_relocs *
805 sizeof(struct mos_reloc_target));
806 if (bo_gem->relocs == nullptr || bo_gem->reloc_target_info == nullptr) {
807 bo_gem->has_error = true;
808
809 free (bo_gem->relocs);
810 bo_gem->relocs = nullptr;
811
812 free (bo_gem->reloc_target_info);
813 bo_gem->reloc_target_info = nullptr;
814
815 return 1;
816 }
817
818 return 0;
819 }
820
821 static int
822 mos_gem_bo_busy(struct mos_linux_bo *bo)
823 {
824 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
825 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
826 struct drm_i915_gem_busy busy;
827 int ret;
828
829 if (bo_gem->reusable && bo_gem->idle)
830 return false;
831
832 memclear(busy);
833 busy.handle = bo_gem->gem_handle;
834
835 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_BUSY, &busy);
836 if (ret == 0) {
837 bo_gem->idle = !busy.busy;
838 return busy.busy;
839 } else {
840 return false;
841 }
842 return (ret == 0 && busy.busy);
843 }
844
845 static int
846 mos_gem_bo_madvise_internal(struct mos_bufmgr_gem *bufmgr_gem,
847 struct mos_bo_gem *bo_gem, int state)
848 {
849 struct drm_i915_gem_madvise madv;
850
851 memclear(madv);
852 madv.handle = bo_gem->gem_handle;
853 madv.madv = state;
854 madv.retained = 1;
855 drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_MADVISE, &madv);
856
857 return madv.retained;
858 }
859
860 static int
861 mos_gem_bo_madvise(struct mos_linux_bo *bo, int madv)
862 {
863 return mos_gem_bo_madvise_internal
864 ((struct mos_bufmgr_gem *) bo->bufmgr,
865 (struct mos_bo_gem *) bo,
866 madv);
867 }
868
869 /* drop the oldest entries that have been purged by the kernel */
870 static void
871 mos_gem_bo_cache_purge_bucket(struct mos_bufmgr_gem *bufmgr_gem,
872 struct mos_gem_bo_bucket *bucket)
873 {
874 while (!DRMLISTEMPTY(&bucket->head)) {
875 struct mos_bo_gem *bo_gem;
876
877 bo_gem = DRMLISTENTRY(struct mos_bo_gem,
878 bucket->head.next, head);
879 if (mos_gem_bo_madvise_internal
880 (bufmgr_gem, bo_gem, I915_MADV_DONTNEED))
881 break;
882
883 DRMLISTDEL(&bo_gem->head);
884 mos_gem_bo_free(&bo_gem->bo);
885 }
886 }
887
888 static int
889 mos_gem_query_items(int fd, struct drm_i915_query_item *items, uint32_t n_items)
890 {
891 struct drm_i915_query q;
892
893 memclear(q);
894 q.num_items = n_items;
895 q.items_ptr = (uintptr_t)items;
896
897 return drmIoctl(fd, DRM_IOCTL_I915_QUERY, &q);
898 }
899
900 /**
901 * query mechanism for memory regions.
902 */
903 static struct drm_i915_query_memory_regions *mos_gem_get_query_memory_regions(int fd)
904 {
905 int ret;
906 struct drm_i915_query_item item;
907 struct drm_i915_query_memory_regions *query_info;
908
909 memclear(item);
910 item.query_id = DRM_I915_QUERY_MEMORY_REGIONS;
911 ret = mos_gem_query_items(fd, &item, 1);
912 if (ret != 0 || item.length <= 0)
913 return nullptr;
914
915 query_info = (drm_i915_query_memory_regions*)calloc(1, item.length);
916
917 item.data_ptr = (uintptr_t)query_info;
918 ret = mos_gem_query_items(fd, &item, 1);
919 if (ret != 0) {
920 free(query_info);
921 return nullptr;
922 }
923
924 return query_info;
925 }
926
927 /**
928 * check how many lmem regions are available on device.
929 */
930 static uint8_t mos_gem_get_lmem_region_count(struct mos_bufmgr_gem *bufmgr_gem)
931 {
932 assert(nullptr != bufmgr_gem);
933
934 if (BufmgrPrelim::IsPrelimSupported()) {
935 return bufmgr_gem->prelim->GetLmemRegionCount();
936 }
937
938 struct drm_i915_query_memory_regions *query_info;
939 uint8_t num_regions = 0;
940 uint8_t lmem_regions = 0;
941
942 query_info = mos_gem_get_query_memory_regions(bufmgr_gem->fd);
943
944 if(nullptr != query_info)
945 {
946 num_regions = query_info->num_regions;
947
948 for (int i = 0; i < num_regions; i++) {
949 if (query_info->regions[i].region.memory_class == I915_MEMORY_CLASS_DEVICE)
950 {
951 lmem_regions += 1;
952 }
953 }
954 free(query_info);
955 }
956
957 return lmem_regions;
958 }
959
960 /**
961 * check if lmem is available on device.
962 */
963 static bool mos_gem_has_lmem(struct mos_bufmgr_gem *bufmgr_gem)
964 {
965 return mos_gem_get_lmem_region_count(bufmgr_gem) > 0;
966 }
967
968 static enum mos_memory_zone
969 mos_gem_bo_memzone_for_address(struct mos_bufmgr *bufmgr, uint64_t address)
970 {
971 assert(nullptr != bufmgr);
972 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bufmgr;
973
974 if (BufmgrPrelim::IsPrelimSupported()) {
975 return bufmgr_gem->prelim->GetMemzoneForAddress(address);
976 }
977 if (address >= MEMZONE_DEVICE_START)
978 return MEMZONE_DEVICE;
979 else
980 return MEMZONE_SYS;
981 }
982
983 /**
984 * Allocate a section of virtual memory for a buffer, assigning an address.
985 */
986 static uint64_t
987 mos_gem_bo_vma_alloc(struct mos_bufmgr *bufmgr,
988 enum mos_memory_zone memzone,
989 uint64_t size,
990 uint64_t alignment)
991 {
992 CHK_CONDITION(bufmgr == nullptr, "nullptr bufmgr.\n", 0);
993 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bufmgr;
994 /* Force alignment to be some number of pages */
995 alignment = ALIGN(alignment, PAGE_SIZE);
996
997 uint64_t addr = mos_vma_heap_alloc(&bufmgr_gem->vma_heap[memzone], size, alignment);
998
999 // currently only support 48bit range address
1000 CHK_CONDITION((addr >> 48ull) != 0, "invalid address, over 48bit range.\n", 0);
1001 CHK_CONDITION((addr >> (memzone == MEMZONE_SYS ? 40ull : (memzone == MEMZONE_DEVICE ? 41ull:42ull))) != 0, "invalid address, over memory zone range.\n", 0);
1002 CHK_CONDITION((addr % alignment) != 0, "invalid address, not meet aligment requirement.\n", 0);
1003
1004 return addr;
1005 }
1006
1007 static void
1008 mos_gem_bo_vma_free(struct mos_bufmgr *bufmgr,
1009 uint64_t address,
1010 uint64_t size)
1011 {
1012 CHK_CONDITION(bufmgr == nullptr, "nullptr bufmgr.\n", );
1013 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bufmgr;
1014
1015 CHK_CONDITION(address == 0ull, "invalid address.\n", );
1016 enum mos_memory_zone memzone = mos_gem_bo_memzone_for_address(bufmgr, address);
1017 mos_vma_heap_free(&bufmgr_gem->vma_heap[memzone], address, size);
1018 }
1019
1020 static int __mos_gem_create_gem(struct mos_bufmgr_gem *bufmgr_gem,
1021 struct mos_bo_gem *bo_gem,
1022 uint64_t bo_size)
1023 {
1024 int ret = 0;
1025
1026 struct drm_i915_gem_memory_class_instance mem_region;
1027 memclear(mem_region);
1028 mem_region.memory_class = I915_MEMORY_CLASS_DEVICE;
1029 mem_region.memory_instance = 0;
1030
1031 struct drm_i915_gem_create_ext_memory_regions regions;
1032 memclear(regions);
1033 regions.base.name = I915_GEM_CREATE_EXT_MEMORY_REGIONS;
1034 regions.num_regions = 1;
1035 regions.regions = (uintptr_t)(&mem_region);
1036
1037 struct drm_i915_gem_create_ext create;
1038 memclear(create);
1039 create.size = bo_size;
1040 create.extensions = (uintptr_t)(&regions);
1041
1042 ret = drmIoctl(bufmgr_gem->fd,
1043 DRM_IOCTL_I915_GEM_CREATE_EXT,
1044 &create);
1045 bo_gem->gem_handle = create.handle;
1046 bo_gem->bo.handle = bo_gem->gem_handle;
1047 bo_gem->bo.size = create.size;
1048 bo_gem->mem_region = I915_MEMORY_CLASS_DEVICE;
1049
1050 return ret;
1051 }
1052
1053 drm_export struct mos_linux_bo *
1054 mos_gem_bo_alloc_internal(struct mos_bufmgr *bufmgr,
1055 struct mos_drm_bo_alloc *alloc)
1056 {
1057 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bufmgr;
1058 struct mos_bo_gem *bo_gem;
1059 unsigned int page_size = getpagesize();
1060 int ret;
1061 struct mos_gem_bo_bucket *bucket;
1062 bool alloc_from_cache;
1063 unsigned long bo_size;
1064 bool for_render = false;
1065
1066 if (alloc->ext.flags & BO_ALLOC_FOR_RENDER)
1067 for_render = true;
1068
1069 /* Round the allocated size up to a power of two number of pages. */
1070 bucket = mos_gem_bo_bucket_for_size(bufmgr_gem, alloc->size);
1071
1072 /* If we don't have caching at this size, don't actually round the
1073 * allocation up.
1074 */
1075 if (bucket == nullptr) {
1076 bo_size = alloc->size;
1077 if (bo_size < page_size)
1078 bo_size = page_size;
1079 } else {
1080 bo_size = bucket->size;
1081 }
1082
1083 pthread_mutex_lock(&bufmgr_gem->lock);
1084 /* Get a buffer out of the cache if available */
1085 retry:
1086 alloc_from_cache = false;
1087 if (bucket != nullptr && !DRMLISTEMPTY(&bucket->head)) {
1088 if (for_render) {
1089 /* Allocate new render-target BOs from the tail (MRU)
1090 * of the list, as it will likely be hot in the GPU
1091 * cache and in the aperture for us.
1092 */
1093 bo_gem = DRMLISTENTRY(struct mos_bo_gem,
1094 bucket->head.prev, head);
1095 DRMLISTDEL(&bo_gem->head);
1096 alloc_from_cache = true;
1097 bo_gem->bo.align = alloc->alignment;
1098 } else {
1099 assert(alloc->alignment == 0);
1100 /* For non-render-target BOs (where we're probably
1101 * going to map it first thing in order to fill it
1102 * with data), check if the last BO in the cache is
1103 * unbusy, and only reuse in that case. Otherwise,
1104 * allocating a new buffer is probably faster than
1105 * waiting for the GPU to finish.
1106 */
1107 bo_gem = DRMLISTENTRY(struct mos_bo_gem,
1108 bucket->head.next, head);
1109 if (!mos_gem_bo_busy(&bo_gem->bo)) {
1110 alloc_from_cache = true;
1111 DRMLISTDEL(&bo_gem->head);
1112 }
1113 }
1114
1115 if (alloc_from_cache) {
1116 if (!mos_gem_bo_madvise_internal
1117 (bufmgr_gem, bo_gem, I915_MADV_WILLNEED)) {
1118 mos_gem_bo_free(&bo_gem->bo);
1119 mos_gem_bo_cache_purge_bucket(bufmgr_gem,
1120 bucket);
1121 goto retry;
1122 }
1123
1124 if (mos_gem_bo_set_tiling_internal(&bo_gem->bo,
1125 alloc->ext.tiling_mode,
1126 alloc->stride)) {
1127 mos_gem_bo_free(&bo_gem->bo);
1128 goto retry;
1129 }
1130
1131 /* when re-cycle the gem_bo, need to keep the VA space is keeping consistent on memory type
1132 i915 cannot switch same VA b/w system/Local
1133 */
1134 if (bufmgr_gem->has_lmem && mos_gem_bo_check_mem_region_internal(&bo_gem->bo, alloc->ext.mem_type)) {
1135 mos_gem_bo_free(&bo_gem->bo);
1136 goto retry;
1137 }
1138 }
1139 }
1140 pthread_mutex_unlock(&bufmgr_gem->lock);
1141
1142 if (!alloc_from_cache) {
1143
1144 bo_gem = (struct mos_bo_gem *)calloc(1, sizeof(*bo_gem));
1145 if (!bo_gem)
1146 return nullptr;
1147
1148 bo_gem->bo.size = bo_size;
1149 bo_gem->mem_region = BufmgrPrelim::IsPrelimSupported() ?
1150 bufmgr_gem->prelim->GetSystemMemRegionId() :
1151 I915_MEMORY_CLASS_SYSTEM;
1152
1153 if(bufmgr_gem->has_lmem &&
1154 (alloc->ext.mem_type == MOS_MEMPOOL_VIDEOMEMORY || alloc->ext.mem_type == MOS_MEMPOOL_DEVICEMEMORY)) {
1155 if (BufmgrPrelim::IsPrelimSupported()) {
1156 uint32_t handle;
1157 uint64_t size;
1158 uint32_t region;
1159 ret = bufmgr_gem->prelim->CreateGem(bo_size, handle, size, region);
1160 if (ret == 0) {
1161 bo_gem->gem_handle = handle;
1162 bo_gem->bo.handle = bo_gem->gem_handle;
1163 bo_gem->bo.size = size;
1164 bo_gem->mem_region = region;
1165 }
1166 } else {
1167 ret = __mos_gem_create_gem(bufmgr_gem, bo_gem, bo_size);
1168 }
1169 }
1170 else {
1171 struct drm_i915_gem_create create;
1172 memclear(create);
1173 create.size = bo_size;
1174 ret = drmIoctl(bufmgr_gem->fd,
1175 DRM_IOCTL_I915_GEM_CREATE,
1176 &create);
1177 bo_gem->gem_handle = create.handle;
1178 bo_gem->bo.handle = bo_gem->gem_handle;
1179 }
1180 if (ret != 0) {
1181 free(bo_gem);
1182 return nullptr;
1183 }
1184 bo_gem->bo.bufmgr = bufmgr;
1185 bo_gem->bo.align = alloc->alignment;
1186
1187 bo_gem->tiling_mode = I915_TILING_NONE;
1188 bo_gem->swizzle_mode = I915_BIT_6_SWIZZLE_NONE;
1189 bo_gem->stride = 0;
1190 if (bufmgr_gem->mem_profiler_fd != -1)
1191 {
1192 snprintf(bufmgr_gem->mem_profiler_buffer, MEM_PROFILER_BUFFER_SIZE, "GEM_CREATE, %d, %d, %lu, %d, %s\n", getpid(), bo_gem->bo.handle, bo_gem->bo.size,bo_gem->mem_region, alloc->name);
1193 ret = write(bufmgr_gem->mem_profiler_fd, bufmgr_gem->mem_profiler_buffer, strnlen(bufmgr_gem->mem_profiler_buffer, MEM_PROFILER_BUFFER_SIZE));
1194 if (ret == -1)
1195 {
1196 MOS_DBG("Failed to write to %s: %s\n", bufmgr_gem->mem_profiler_path, strerror(errno));
1197 }
1198 }
1199
1200 /* drm_intel_gem_bo_free calls DRMLISTDEL() for an uninitialized
1201 list (vma_list), so better set the list head here */
1202 DRMINITLISTHEAD(&bo_gem->name_list);
1203 if (mos_gem_bo_set_tiling_internal(&bo_gem->bo,
1204 alloc->ext.tiling_mode,
1205 alloc->stride)) {
1206 mos_gem_bo_free(&bo_gem->bo);
1207 return nullptr;
1208 }
1209 }
1210
1211 bo_gem->name = alloc->name;
1212 atomic_set(&bo_gem->refcount, 1);
1213 bo_gem->validate_index = -1;
1214 bo_gem->reloc_tree_fences = 0;
1215 bo_gem->used_as_reloc_target = false;
1216 bo_gem->has_error = false;
1217 bo_gem->reusable = true;
1218 bo_gem->use_48b_address_range = bufmgr_gem->bufmgr.bo_use_48b_address_range ? true : false;
1219
1220 mos_bo_gem_set_in_aperture_size(bufmgr_gem, bo_gem, alloc->alignment);
1221
1222 if (bufmgr_gem->use_softpin)
1223 {
1224 mos_bo_set_softpin(&bo_gem->bo);
1225 }
1226
1227 MOS_DBG("bo_create: buf %d (%s) %ldb\n",
1228 bo_gem->gem_handle, bo_gem->name, alloc->size);
1229
1230 return &bo_gem->bo;
1231 }
1232
1233 static struct mos_linux_bo *
1234 mos_gem_bo_alloc(struct mos_bufmgr *bufmgr,
1235 struct mos_drm_bo_alloc *alloc)
1236 {
1237 alloc->ext.flags = 0;
1238 alloc->alignment = 0;
1239 alloc->stride = 0;
1240 return mos_gem_bo_alloc_internal(bufmgr, alloc);
1241 }
1242
1243 static struct mos_linux_bo *
1244 mos_gem_bo_alloc_tiled(struct mos_bufmgr *bufmgr,
1245 struct mos_drm_bo_alloc_tiled *alloc_tiled)
1246 {
1247 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bufmgr;
1248 unsigned long size, stride;
1249 uint32_t tiling;
1250
1251 do {
1252 unsigned long aligned_y, height_alignment;
1253
1254 tiling = alloc_tiled->ext.tiling_mode;
1255
1256 /* If we're tiled, our allocations are in 8 or 32-row blocks,
1257 * so failure to align our height means that we won't allocate
1258 * enough pages.
1259 *
1260 * If we're untiled, we still have to align to 2 rows high
1261 * because the data port accesses 2x2 blocks even if the
1262 * bottom row isn't to be rendered, so failure to align means
1263 * we could walk off the end of the GTT and fault. This is
1264 * documented on 965, and may be the case on older chipsets
1265 * too so we try to be careful.
1266 */
1267 aligned_y = alloc_tiled->y;
1268 height_alignment = 2;
1269
1270 if (tiling == I915_TILING_X
1271 || (IS_915(bufmgr_gem->pci_device)
1272 && tiling == I915_TILING_Y))
1273 height_alignment = 8;
1274 else if (tiling == I915_TILING_Y)
1275 height_alignment = 32;
1276 aligned_y = ALIGN(alloc_tiled->y, height_alignment);
1277
1278 stride = alloc_tiled->x * alloc_tiled->cpp;
1279 stride = mos_gem_bo_tile_pitch(bufmgr_gem, stride, &alloc_tiled->ext.tiling_mode);
1280 size = stride * aligned_y;
1281 size = mos_gem_bo_tile_size(bufmgr_gem, size, &alloc_tiled->ext.tiling_mode);
1282 } while (alloc_tiled->ext.tiling_mode != tiling);
1283 alloc_tiled->pitch = stride;
1284
1285 if (tiling == I915_TILING_NONE)
1286 stride = 0;
1287
1288 struct mos_drm_bo_alloc alloc;
1289 alloc.name = alloc_tiled->name;
1290 alloc.size = size;
1291 alloc.stride = stride;
1292 alloc.ext = alloc_tiled->ext;
1293 return mos_gem_bo_alloc_internal(bufmgr, &alloc);
1294 }
1295
1296 static struct mos_linux_bo *
1297 mos_gem_bo_alloc_userptr(struct mos_bufmgr *bufmgr,
1298 struct mos_drm_bo_alloc_userptr *alloc_uptr)
1299 {
1300 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bufmgr;
1301 struct mos_bo_gem *bo_gem;
1302 int ret;
1303 struct drm_i915_gem_userptr userptr;
1304
1305 /* Tiling with userptr surfaces is not supported
1306 * on all hardware so refuse it for time being.
1307 */
1308 if (alloc_uptr->tiling_mode != I915_TILING_NONE)
1309 return nullptr;
1310
1311 bo_gem = (struct mos_bo_gem *)calloc(1, sizeof(*bo_gem));
1312 if (!bo_gem)
1313 return nullptr;
1314
1315 bo_gem->bo.size = alloc_uptr->size;
1316
1317 memclear(userptr);
1318 userptr.user_ptr = (__u64)((unsigned long)alloc_uptr->addr);
1319 userptr.user_size = alloc_uptr->size;
1320 userptr.flags = 0;
1321
1322 ret = drmIoctl(bufmgr_gem->fd,
1323 DRM_IOCTL_I915_GEM_USERPTR,
1324 &userptr);
1325 if (ret != 0) {
1326 MOS_DBG("bo_create_userptr: "
1327 "ioctl failed with user ptr %p size 0x%lx, "
1328 "user flags 0x%lx\n", alloc_uptr->addr, alloc_uptr->size, alloc_uptr->flags);
1329 free(bo_gem);
1330 return nullptr;
1331 }
1332
1333 bo_gem->gem_handle = userptr.handle;
1334 bo_gem->bo.handle = bo_gem->gem_handle;
1335 bo_gem->bo.bufmgr = bufmgr;
1336 bo_gem->is_userptr = true;
1337 #ifdef __cplusplus
1338 bo_gem->bo.virt = alloc_uptr->addr;
1339 #else
1340 bo_gem->bo.virtual = alloc_uptr->addr;
1341 #endif
1342 /* Save the address provided by user */
1343 bo_gem->user_virtual = alloc_uptr->addr;
1344 bo_gem->tiling_mode = I915_TILING_NONE;
1345 bo_gem->swizzle_mode = I915_BIT_6_SWIZZLE_NONE;
1346 bo_gem->stride = 0;
1347
1348 DRMINITLISTHEAD(&bo_gem->name_list);
1349
1350 bo_gem->name = alloc_uptr->name;
1351 atomic_set(&bo_gem->refcount, 1);
1352 bo_gem->validate_index = -1;
1353 bo_gem->reloc_tree_fences = 0;
1354 bo_gem->used_as_reloc_target = false;
1355 bo_gem->has_error = false;
1356 bo_gem->reusable = false;
1357 bo_gem->use_48b_address_range = bufmgr_gem->bufmgr.bo_use_48b_address_range ? true : false;
1358
1359 mos_bo_gem_set_in_aperture_size(bufmgr_gem, bo_gem, 0);
1360
1361 if (bufmgr_gem->use_softpin)
1362 {
1363 mos_bo_set_softpin(&bo_gem->bo);
1364 }
1365
1366 MOS_DBG("bo_create_userptr: "
1367 "ptr %p buf %d (%s) size %ldb, stride 0x%x, tile mode %d\n",
1368 alloc_uptr->addr, bo_gem->gem_handle, bo_gem->name,
1369 alloc_uptr->size, alloc_uptr->stride, alloc_uptr->tiling_mode);
1370
1371 return &bo_gem->bo;
1372 }
1373
1374 static bool
1375 has_userptr(struct mos_bufmgr_gem *bufmgr_gem)
1376 {
1377 int ret;
1378 void *ptr;
1379 long pgsz;
1380 struct drm_i915_gem_userptr userptr;
1381
1382 pgsz = sysconf(_SC_PAGESIZE);
1383 assert(pgsz > 0);
1384
1385 ret = posix_memalign(&ptr, pgsz, pgsz);
1386 if (ret) {
1387 MOS_DBG("Failed to get a page (%ld) for userptr detection!\n",
1388 pgsz);
1389 return false;
1390 }
1391
1392 memclear(userptr);
1393 userptr.user_ptr = (__u64)(unsigned long)ptr;
1394 userptr.user_size = pgsz;
1395
1396 retry:
1397 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_USERPTR, &userptr);
1398 if (ret) {
1399 if (errno == ENODEV && userptr.flags == 0) {
1400 userptr.flags = I915_USERPTR_UNSYNCHRONIZED;
1401 goto retry;
1402 }
1403 free(ptr);
1404 return false;
1405 }
1406
1407 /* We don't release the userptr bo here as we want to keep the
1408 * kernel mm tracking alive for our lifetime. The first time we
1409 * create a userptr object the kernel has to install a mmu_notifer
1410 * which is a heavyweight operation (e.g. it requires taking all
1411 * mm_locks and stop_machine()).
1412 */
1413
1414 bufmgr_gem->userptr_active.ptr = ptr;
1415 bufmgr_gem->userptr_active.handle = userptr.handle;
1416 return true;
1417 }
1418
1419 static struct mos_linux_bo *
1420 check_bo_alloc_userptr(struct mos_bufmgr *bufmgr,
1421 struct mos_drm_bo_alloc_userptr *alloc_uptr)
1422 {
1423 if (has_userptr((struct mos_bufmgr_gem *)bufmgr))
1424 bufmgr->bo_alloc_userptr = mos_gem_bo_alloc_userptr;
1425 else
1426 bufmgr->bo_alloc_userptr = nullptr;
1427
1428 return mos_bo_alloc_userptr(bufmgr, alloc_uptr);
1429 }
1430
1431 /**
1432 * Returns a drm_intel_bo wrapping the given buffer object handle.
1433 *
1434 * This can be used when one application needs to pass a buffer object
1435 * to another.
1436 */
1437 static struct mos_linux_bo *
1438 mos_bufmgr_bo_gem_create_from_name(struct mos_bufmgr *bufmgr,
1439 const char *name,
1440 unsigned int handle)
1441 {
1442 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bufmgr;
1443 struct mos_bo_gem *bo_gem;
1444 int ret;
1445 struct drm_gem_open open_arg;
1446 struct drm_i915_gem_get_tiling get_tiling;
1447 drmMMListHead *list;
1448
1449 /* At the moment most applications only have a few named bo.
1450 * For instance, in a DRI client only the render buffers passed
1451 * between X and the client are named. And since X returns the
1452 * alternating names for the front/back buffer a linear search
1453 * provides a sufficiently fast match.
1454 */
1455 pthread_mutex_lock(&bufmgr_gem->lock);
1456 for (list = bufmgr_gem->named.next;
1457 list != &bufmgr_gem->named;
1458 list = list->next) {
1459 bo_gem = DRMLISTENTRY(struct mos_bo_gem, list, name_list);
1460 if (bo_gem->global_name == handle) {
1461 mos_gem_bo_reference(&bo_gem->bo);
1462 pthread_mutex_unlock(&bufmgr_gem->lock);
1463 return &bo_gem->bo;
1464 }
1465 }
1466
1467 memclear(open_arg);
1468 open_arg.name = handle;
1469 ret = drmIoctl(bufmgr_gem->fd,
1470 DRM_IOCTL_GEM_OPEN,
1471 &open_arg);
1472 if (ret != 0) {
1473 MOS_DBG("Couldn't reference %s handle 0x%08x: %s\n",
1474 name, handle, strerror(errno));
1475 pthread_mutex_unlock(&bufmgr_gem->lock);
1476 return nullptr;
1477 }
1478 /* Now see if someone has used a prime handle to get this
1479 * object from the kernel before by looking through the list
1480 * again for a matching gem_handle
1481 */
1482 for (list = bufmgr_gem->named.next;
1483 list != &bufmgr_gem->named;
1484 list = list->next) {
1485 bo_gem = DRMLISTENTRY(struct mos_bo_gem, list, name_list);
1486 if (bo_gem->gem_handle == open_arg.handle) {
1487 mos_gem_bo_reference(&bo_gem->bo);
1488 pthread_mutex_unlock(&bufmgr_gem->lock);
1489 return &bo_gem->bo;
1490 }
1491 }
1492
1493 bo_gem = (struct mos_bo_gem *)calloc(1, sizeof(*bo_gem));
1494 if (!bo_gem) {
1495 pthread_mutex_unlock(&bufmgr_gem->lock);
1496 return nullptr;
1497 }
1498
1499 bo_gem->bo.size = open_arg.size;
1500 bo_gem->bo.offset = 0;
1501 bo_gem->bo.offset64 = 0;
1502 #if defined(__cplusplus)
1503 bo_gem->bo.virt = nullptr;
1504 #else
1505 bo_gem->bo.virtual = nullptr;
1506 #endif
1507 bo_gem->bo.bufmgr = bufmgr;
1508 bo_gem->name = name;
1509 atomic_set(&bo_gem->refcount, 1);
1510 bo_gem->validate_index = -1;
1511 bo_gem->gem_handle = open_arg.handle;
1512 bo_gem->bo.handle = open_arg.handle;
1513 bo_gem->global_name = handle;
1514 bo_gem->reusable = false;
1515 bo_gem->use_48b_address_range = bufmgr_gem->bufmgr.bo_use_48b_address_range ? true : false;
1516
1517 memclear(get_tiling);
1518 if (bufmgr_gem->has_fence_reg) {
1519 get_tiling.handle = bo_gem->gem_handle;
1520 ret = drmIoctl(bufmgr_gem->fd,
1521 DRM_IOCTL_I915_GEM_GET_TILING,
1522 &get_tiling);
1523 if (ret != 0) {
1524 MOS_DBG("create_from_name: failed to get tiling: %s\n", strerror(errno));
1525 mos_gem_bo_unreference(&bo_gem->bo);
1526 pthread_mutex_unlock(&bufmgr_gem->lock);
1527 return nullptr;
1528 }
1529 }
1530 else
1531 {
1532 MOS_DBG("create_from_name: driver ignored to get tiling from kernel\n");
1533 }
1534
1535 bo_gem->tiling_mode = get_tiling.tiling_mode;
1536 bo_gem->swizzle_mode = get_tiling.swizzle_mode;
1537 /* XXX stride is unknown */
1538
1539 mos_bo_gem_set_in_aperture_size(bufmgr_gem, bo_gem, 0);
1540
1541 DRMLISTADDTAIL(&bo_gem->name_list, &bufmgr_gem->named);
1542 pthread_mutex_unlock(&bufmgr_gem->lock);
1543
1544 if (bufmgr_gem->use_softpin)
1545 {
1546 mos_bo_set_softpin(&bo_gem->bo);
1547 }
1548
1549 MOS_DBG("bo_create_from_handle: %d (%s)\n", handle, bo_gem->name);
1550
1551 return &bo_gem->bo;
1552 }
1553
1554 static void
1555 mos_gem_bo_free(struct mos_linux_bo *bo)
1556 {
1557 struct mos_bufmgr_gem *bufmgr_gem = nullptr;
1558 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
1559 struct drm_gem_close close;
1560 int ret;
1561
1562 CHK_CONDITION(bo_gem == nullptr, "bo_gem == nullptr\n", );
1563
1564 bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
1565
1566 CHK_CONDITION(bufmgr_gem == nullptr, "bufmgr_gem == nullptr\n", );
1567
1568 if (bo_gem->mem_virtual) {
1569 VG(VALGRIND_MAKE_MEM_NOACCESS(bo_gem->mem_virtual, 0));
1570 drm_munmap(bo_gem->mem_virtual, bo_gem->bo.size);
1571 bo_gem->mem_virtual = nullptr;
1572 }
1573 if (bo_gem->gtt_virtual) {
1574 VG(VALGRIND_MAKE_MEM_NOACCESS(bo_gem->gtt_virtual, 0));
1575 drm_munmap(bo_gem->gtt_virtual, bo_gem->bo.size);
1576 bo_gem->gtt_virtual = nullptr;
1577 }
1578 if (bo_gem->mem_wc_virtual) {
1579 VG(VALGRIND_MAKE_MEM_NOACCESS(bo_gem->mem_wc_virtual, 0));
1580 drm_munmap(bo_gem->mem_wc_virtual, bo_gem->bo.size);
1581 bo_gem->mem_wc_virtual = nullptr;
1582 }
1583
1584 if(bufmgr_gem->bufmgr.bo_wait_rendering && mos_gem_bo_busy(bo))
1585 {
1586 bufmgr_gem->bufmgr.bo_wait_rendering(bo);
1587 }
1588
1589 /* Close this object */
1590 memclear(close);
1591 close.handle = bo_gem->gem_handle;
1592 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_GEM_CLOSE, &close);
1593 if (ret != 0) {
1594 MOS_DBG("DRM_IOCTL_GEM_CLOSE %d failed (%s): %s\n",
1595 bo_gem->gem_handle, bo_gem->name, strerror(errno));
1596 }
1597 if (bufmgr_gem->mem_profiler_fd != -1)
1598 {
1599 snprintf(bufmgr_gem->mem_profiler_buffer, MEM_PROFILER_BUFFER_SIZE, "GEM_CLOSE, %d, %d, %lu, %d\n", getpid(), bo->handle,bo->size,bo_gem->mem_region);
1600 ret = write(bufmgr_gem->mem_profiler_fd, bufmgr_gem->mem_profiler_buffer, strnlen(bufmgr_gem->mem_profiler_buffer, MEM_PROFILER_BUFFER_SIZE));
1601 if (ret == -1)
1602 {
1603 MOS_DBG("Failed to write to %s: %s\n", bufmgr_gem->mem_profiler_path, strerror(errno));
1604 }
1605 }
1606
1607 if (bufmgr_gem->use_softpin)
1608 {
1609 /* Return the VMA for reuse */
1610 mos_gem_bo_vma_free(bo->bufmgr, bo->offset64, bo->size);
1611 }
1612
1613 free(bo);
1614 }
1615
1616 static void
1617 mos_gem_bo_mark_mmaps_incoherent(struct mos_linux_bo *bo)
1618 {
1619 #if HAVE_VALGRIND
1620 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
1621
1622 if (bo_gem->mem_virtual)
1623 VALGRIND_MAKE_MEM_NOACCESS(bo_gem->mem_virtual, bo->size);
1624
1625 if (bo_gem->gtt_virtual)
1626 VALGRIND_MAKE_MEM_NOACCESS(bo_gem->gtt_virtual, bo->size);
1627
1628 if (bo_gem->mem_wc_virtual)
1629 VALGRIND_MAKE_MEM_NOACCESS(bo_gem->mem_wc_virtual, bo->size);
1630 #endif
1631 }
1632
1633 /** Frees all cached buffers significantly older than @time. */
1634 static void
1635 mos_gem_cleanup_bo_cache(struct mos_bufmgr_gem *bufmgr_gem, time_t time)
1636 {
1637 int i;
1638
1639 if (bufmgr_gem->time == time)
1640 return;
1641
1642 for (i = 0; i < bufmgr_gem->num_buckets; i++) {
1643 struct mos_gem_bo_bucket *bucket =
1644 &bufmgr_gem->cache_bucket[i];
1645
1646 while (!DRMLISTEMPTY(&bucket->head)) {
1647 struct mos_bo_gem *bo_gem;
1648
1649 bo_gem = DRMLISTENTRY(struct mos_bo_gem,
1650 bucket->head.next, head);
1651 if (time - bo_gem->free_time <= 1)
1652 break;
1653
1654 DRMLISTDEL(&bo_gem->head);
1655
1656 mos_gem_bo_free(&bo_gem->bo);
1657 }
1658 }
1659
1660 bufmgr_gem->time = time;
1661 }
1662
1663 drm_export void
1664 mos_gem_bo_unreference_final(struct mos_linux_bo *bo, time_t time)
1665 {
1666 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
1667 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
1668 struct mos_gem_bo_bucket *bucket;
1669 int i;
1670
1671 /* Unreference all the target buffers */
1672 for (i = 0; i < bo_gem->reloc_count; i++) {
1673 if (bo_gem->reloc_target_info[i].bo != bo) {
1674 mos_gem_bo_unreference_locked_timed(bo_gem->
1675 reloc_target_info[i].bo,
1676 time);
1677 }
1678 }
1679 for (i = 0; i < bo_gem->softpin_target_count; i++)
1680 mos_gem_bo_unreference_locked_timed(bo_gem->softpin_target[i].bo,
1681 time);
1682 bo_gem->reloc_count = 0;
1683 bo_gem->used_as_reloc_target = false;
1684 bo_gem->softpin_target_count = 0;
1685 bo_gem->exec_async = false;
1686
1687 MOS_DBG("bo_unreference final: %d (%s)\n",
1688 bo_gem->gem_handle, bo_gem->name);
1689 bo_gem->pad_to_size = 0;
1690
1691 /* release memory associated with this object */
1692 if (bo_gem->reloc_target_info) {
1693 free(bo_gem->reloc_target_info);
1694 bo_gem->reloc_target_info = nullptr;
1695 }
1696 if (bo_gem->relocs) {
1697 free(bo_gem->relocs);
1698 bo_gem->relocs = nullptr;
1699 }
1700 if (bo_gem->softpin_target) {
1701 free(bo_gem->softpin_target);
1702 bo_gem->softpin_target = nullptr;
1703 bo_gem->max_softpin_target_count = 0;
1704 }
1705
1706 /* Clear any left-over mappings */
1707 if (bo_gem->map_count) {
1708 MOS_DBG("bo freed with non-zero map-count %d\n", bo_gem->map_count);
1709 bo_gem->map_count = 0;
1710 mos_gem_bo_mark_mmaps_incoherent(bo);
1711 }
1712
1713 DRMLISTDEL(&bo_gem->name_list);
1714
1715 bucket = mos_gem_bo_bucket_for_size(bufmgr_gem, bo->size);
1716 /* Put the buffer into our internal cache for reuse if we can. */
1717 if (bufmgr_gem->bo_reuse && bo_gem->reusable && bucket != nullptr &&
1718 mos_gem_bo_madvise_internal(bufmgr_gem, bo_gem,
1719 I915_MADV_DONTNEED)) {
1720 bo_gem->free_time = time;
1721
1722 bo_gem->name = nullptr;
1723 bo_gem->validate_index = -1;
1724
1725 DRMLISTADDTAIL(&bo_gem->head, &bucket->head);
1726 } else {
1727 mos_gem_bo_free(bo);
1728 }
1729 }
1730
1731 static void mos_gem_bo_unreference_locked_timed(struct mos_linux_bo *bo,
1732 time_t time)
1733 {
1734 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
1735
1736 assert(atomic_read(&bo_gem->refcount) > 0);
1737 if (atomic_dec_and_test(&bo_gem->refcount))
1738 mos_gem_bo_unreference_final(bo, time);
1739 }
1740
1741 static void mos_gem_bo_unreference(struct mos_linux_bo *bo)
1742 {
1743 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
1744
1745 assert(atomic_read(&bo_gem->refcount) > 0);
1746
1747 if (atomic_add_unless(&bo_gem->refcount, -1, 1)) {
1748 struct mos_bufmgr_gem *bufmgr_gem =
1749 (struct mos_bufmgr_gem *) bo->bufmgr;
1750 struct timespec time;
1751
1752 clock_gettime(CLOCK_MONOTONIC, &time);
1753
1754 pthread_mutex_lock(&bufmgr_gem->lock);
1755
1756 if (atomic_dec_and_test(&bo_gem->refcount)) {
1757 mos_gem_bo_unreference_final(bo, time.tv_sec);
1758 mos_gem_cleanup_bo_cache(bufmgr_gem, time.tv_sec);
1759 }
1760
1761 pthread_mutex_unlock(&bufmgr_gem->lock);
1762 }
1763 }
1764
1765 static int
1766 map_wc(struct mos_linux_bo *bo)
1767 {
1768 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
1769 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
1770 int ret;
1771
1772 if (bo_gem->is_userptr)
1773 return -EINVAL;
1774
1775 if (!bufmgr_gem->has_ext_mmap)
1776 return -EINVAL;
1777
1778 /* Get a mapping of the buffer if we haven't before. */
1779 if (bo_gem->mem_wc_virtual == nullptr && bufmgr_gem->has_mmap_offset) {
1780 struct drm_i915_gem_mmap_offset mmap_arg;
1781
1782 MOS_DBG("bo_map_wc: mmap_offset %d (%s), map_count=%d\n",
1783 bo_gem->gem_handle, bo_gem->name, bo_gem->map_count);
1784
1785 memclear(mmap_arg);
1786 mmap_arg.handle = bo_gem->gem_handle;
1787 /* To indicate the uncached virtual mapping to KMD */
1788 if (BufmgrPrelim::IsPrelimSupported()) {
1789 bufmgr_gem->prelim->SetMmapOffset(bo_gem->mem_region, mmap_arg);
1790 } else if (bufmgr_gem->has_lmem) {
1791 mmap_arg.flags = I915_MMAP_OFFSET_FIXED;
1792 }
1793 else
1794 {
1795 mmap_arg.flags = I915_MMAP_OFFSET_WC;
1796 }
1797 ret = drmIoctl(bufmgr_gem->fd,
1798 DRM_IOCTL_I915_GEM_MMAP_OFFSET,
1799 &mmap_arg);
1800 if (ret != 0) {
1801 ret = -errno;
1802 MOS_DBG("%s:%d: Error mapping buffer %d (%s): %s .\n",
1803 __FILE__, __LINE__, bo_gem->gem_handle,
1804 bo_gem->name, strerror(errno));
1805 return ret;
1806 }
1807
1808 /* and mmap it */
1809 bo_gem->mem_wc_virtual = drm_mmap(0, bo->size, PROT_READ | PROT_WRITE,
1810 MAP_SHARED, bufmgr_gem->fd,
1811 mmap_arg.offset);
1812 if (bo_gem->mem_wc_virtual == MAP_FAILED) {
1813 bo_gem->mem_wc_virtual = nullptr;
1814 ret = -errno;
1815 MOS_DBG("%s:%d: Error mapping buffer %d (%s): %s .\n",
1816 __FILE__, __LINE__,
1817 bo_gem->gem_handle, bo_gem->name,
1818 strerror(errno));
1819 }
1820 }
1821 else if (bo_gem->mem_wc_virtual == nullptr) {
1822 struct drm_i915_gem_mmap mmap_arg;
1823
1824 MOS_DBG("bo_map_wc: mmap %d (%s), map_count=%d\n",
1825 bo_gem->gem_handle, bo_gem->name, bo_gem->map_count);
1826
1827 memclear(mmap_arg);
1828 mmap_arg.handle = bo_gem->gem_handle;
1829 /* To indicate the uncached virtual mapping to KMD */
1830 mmap_arg.flags = I915_MMAP_WC;
1831 mmap_arg.size = bo->size;
1832 ret = drmIoctl(bufmgr_gem->fd,
1833 DRM_IOCTL_I915_GEM_MMAP,
1834 &mmap_arg);
1835 if (ret != 0) {
1836 ret = -errno;
1837 MOS_DBG("%s:%d: Error mapping buffer %d (%s): %s .\n",
1838 __FILE__, __LINE__, bo_gem->gem_handle,
1839 bo_gem->name, strerror(errno));
1840 return ret;
1841 }
1842 VG(VALGRIND_MALLOCLIKE_BLOCK(mmap_arg.addr_ptr, mmap_arg.size, 0, 1));
1843 bo_gem->mem_wc_virtual = (void *)(uintptr_t) mmap_arg.addr_ptr;
1844 }
1845 #ifdef __cplusplus
1846 bo->virt = bo_gem->mem_wc_virtual;
1847 #else
1848 bo->virtual = bo_gem->mem_wc_virtual;
1849 #endif
1850
1851 MOS_DBG("bo_map_wc: %d (%s) -> %p\n", bo_gem->gem_handle, bo_gem->name,
1852 bo_gem->mem_wc_virtual);
1853
1854 return 0;
1855 }
1856
1857 /* To be used in a similar way to mmap_gtt */
1858 drm_export int
1859 mos_gem_bo_map_wc(struct mos_linux_bo *bo) {
1860 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
1861 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
1862 struct drm_i915_gem_set_domain set_domain;
1863 struct drm_i915_gem_wait wait;
1864 int ret;
1865
1866 pthread_mutex_lock(&bufmgr_gem->lock);
1867
1868 ret = map_wc(bo);
1869 if (ret) {
1870 pthread_mutex_unlock(&bufmgr_gem->lock);
1871 return ret;
1872 }
1873
1874 if (bufmgr_gem->has_lmem) {
1875 assert(bufmgr_gem->has_wait_timeout);
1876 memclear(wait);
1877 wait.bo_handle = bo_gem->gem_handle;
1878 wait.timeout_ns = -1; // infinite wait
1879 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_WAIT, &wait);
1880 if (ret == -1) {
1881 MOS_DBG("%s:%d: DRM_IOCTL_I915_GEM_WAIT failed (%d)\n",
1882 __FILE__, __LINE__, errno);
1883 }
1884 } else {
1885 /* Now move it to the GTT domain so that the GPU and CPU
1886 * caches are flushed and the GPU isn't actively using the
1887 * buffer.
1888 *
1889 * The domain change is done even for the objects which
1890 * are not bounded. For them first the pages are acquired,
1891 * before the domain change.
1892 */
1893 memclear(set_domain);
1894 set_domain.handle = bo_gem->gem_handle;
1895 set_domain.read_domains = I915_GEM_DOMAIN_GTT;
1896 set_domain.write_domain = I915_GEM_DOMAIN_GTT;
1897 ret = drmIoctl(bufmgr_gem->fd,
1898 DRM_IOCTL_I915_GEM_SET_DOMAIN,
1899 &set_domain);
1900 if (ret != 0) {
1901 MOS_DBG("%s:%d: Error setting domain %d: %s\n",
1902 __FILE__, __LINE__, bo_gem->gem_handle,
1903 strerror(errno));
1904 }
1905 }
1906
1907 mos_gem_bo_mark_mmaps_incoherent(bo);
1908 VG(VALGRIND_MAKE_MEM_DEFINED(bo_gem->mem_wc_virtual, bo->size));
1909 pthread_mutex_unlock(&bufmgr_gem->lock);
1910
1911 return 0;
1912 }
1913
1914 drm_export int mos_gem_bo_map(struct mos_linux_bo *bo, int write_enable)
1915 {
1916 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
1917 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
1918 int ret;
1919
1920 if (bo_gem->is_userptr) {
1921 /* Return the same user ptr */
1922 #ifdef __cplusplus
1923 bo->virt = bo_gem->user_virtual;
1924 #else
1925 bo->virtual = bo_gem->user_virtual;
1926 #endif
1927 return 0;
1928 }
1929
1930 pthread_mutex_lock(&bufmgr_gem->lock);
1931
1932 if (bufmgr_gem->has_mmap_offset) {
1933 struct drm_i915_gem_wait wait;
1934
1935 if (!bo_gem->mem_virtual) {
1936 struct drm_i915_gem_mmap_offset mmap_arg;
1937
1938 MOS_DBG("bo_map: %d (%s), map_count=%d\n",
1939 bo_gem->gem_handle, bo_gem->name, bo_gem->map_count);
1940
1941 memclear(mmap_arg);
1942 mmap_arg.handle = bo_gem->gem_handle;
1943 if (BufmgrPrelim::IsPrelimSupported()) {
1944 bufmgr_gem->prelim->SetMmapOffset(bo_gem->mem_region, mmap_arg);
1945 } else if (bufmgr_gem->has_lmem) {
1946 mmap_arg.flags = I915_MMAP_OFFSET_FIXED;
1947 } else {
1948 mmap_arg.flags = I915_MMAP_OFFSET_WB;
1949 }
1950 ret = drmIoctl(bufmgr_gem->fd,
1951 DRM_IOCTL_I915_GEM_MMAP_OFFSET,
1952 &mmap_arg);
1953 if (ret != 0) {
1954 ret = -errno;
1955 MOS_DBG("%s:%d: Error mapping buffer %d (%s): %s .\n",
1956 __FILE__, __LINE__, bo_gem->gem_handle,
1957 bo_gem->name, strerror(errno));
1958 pthread_mutex_unlock(&bufmgr_gem->lock);
1959 return ret;
1960 }
1961
1962 /* and mmap it */
1963 bo_gem->mem_virtual = drm_mmap(0, bo->size, PROT_READ | PROT_WRITE,
1964 MAP_SHARED, bufmgr_gem->fd,
1965 mmap_arg.offset);
1966 if (bo_gem->mem_virtual == MAP_FAILED) {
1967 bo_gem->mem_virtual = nullptr;
1968 ret = -errno;
1969 MOS_DBG("%s:%d: Error mapping buffer %d (%s): %s .\n",
1970 __FILE__, __LINE__,
1971 bo_gem->gem_handle, bo_gem->name,
1972 strerror(errno));
1973 }
1974 }
1975
1976 assert(bufmgr_gem->has_wait_timeout);
1977 memclear(wait);
1978 wait.bo_handle = bo_gem->gem_handle;
1979 wait.timeout_ns = -1; // infinite wait
1980 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_WAIT, &wait);
1981 if (ret == -1) {
1982 MOS_DBG("%s:%d: DRM_IOCTL_I915_GEM_WAIT failed (%d)\n",
1983 __FILE__, __LINE__, errno);
1984 }
1985 } else { /*!has_mmap_offset*/
1986 struct drm_i915_gem_set_domain set_domain;
1987
1988 if (!bo_gem->mem_virtual) {
1989 struct drm_i915_gem_mmap mmap_arg;
1990
1991 MOS_DBG("bo_map: %d (%s), map_count=%d\n",
1992 bo_gem->gem_handle, bo_gem->name, bo_gem->map_count);
1993
1994 memclear(mmap_arg);
1995 mmap_arg.handle = bo_gem->gem_handle;
1996 mmap_arg.size = bo->size;
1997 ret = drmIoctl(bufmgr_gem->fd,
1998 DRM_IOCTL_I915_GEM_MMAP,
1999 &mmap_arg);
2000 if (ret != 0) {
2001 ret = -errno;
2002 MOS_DBG("%s:%d: Error mapping buffer %d (%s): %s .\n",
2003 __FILE__, __LINE__, bo_gem->gem_handle,
2004 bo_gem->name, strerror(errno));
2005 pthread_mutex_unlock(&bufmgr_gem->lock);
2006 return ret;
2007 }
2008 VG(VALGRIND_MALLOCLIKE_BLOCK(mmap_arg.addr_ptr, mmap_arg.size, 0, 1));
2009 bo_gem->mem_virtual = (void *)(uintptr_t) mmap_arg.addr_ptr;
2010 }
2011
2012 memclear(set_domain);
2013 set_domain.handle = bo_gem->gem_handle;
2014 set_domain.read_domains = I915_GEM_DOMAIN_CPU;
2015 if (write_enable)
2016 set_domain.write_domain = I915_GEM_DOMAIN_CPU;
2017 else
2018 set_domain.write_domain = 0;
2019 ret = drmIoctl(bufmgr_gem->fd,
2020 DRM_IOCTL_I915_GEM_SET_DOMAIN,
2021 &set_domain);
2022 if (ret != 0) {
2023 MOS_DBG("%s:%d: Error setting to CPU domain %d: %s\n",
2024 __FILE__, __LINE__, bo_gem->gem_handle,
2025 strerror(errno));
2026 }
2027 }
2028 MOS_DBG("bo_map: %d (%s) -> %p\n", bo_gem->gem_handle, bo_gem->name,
2029 bo_gem->mem_virtual);
2030 #ifdef __cplusplus
2031 bo->virt = bo_gem->mem_virtual;
2032 #else
2033 bo->virtual = bo_gem->mem_virtual;
2034 #endif
2035
2036 if (write_enable)
2037 bo_gem->mapped_cpu_write = true;
2038
2039 mos_gem_bo_mark_mmaps_incoherent(bo);
2040 VG(VALGRIND_MAKE_MEM_DEFINED(bo_gem->mem_virtual, bo->size));
2041 pthread_mutex_unlock(&bufmgr_gem->lock);
2042
2043 return 0;
2044 }
2045
2046 drm_export int
2047 map_gtt(struct mos_linux_bo *bo)
2048 {
2049 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
2050 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2051 int ret;
2052
2053 if (bo_gem->is_userptr)
2054 return -EINVAL;
2055
2056 /* Get a mapping of the buffer if we haven't before. */
2057 if (bo_gem->gtt_virtual == nullptr) {
2058 __u64 offset = 0;
2059 if (bufmgr_gem->has_lmem) {
2060 struct drm_i915_gem_mmap_offset mmap_arg;
2061
2062 MOS_DBG("map_gtt: mmap_offset %d (%s), map_count=%d\n",
2063 bo_gem->gem_handle, bo_gem->name, bo_gem->map_count);
2064
2065 memclear(mmap_arg);
2066 mmap_arg.handle = bo_gem->gem_handle;
2067 if (BufmgrPrelim::IsPrelimSupported()) {
2068 bufmgr_gem->prelim->SetMmapOffset(bo_gem->mem_region, mmap_arg);
2069 } else {
2070 mmap_arg.flags = I915_MMAP_OFFSET_FIXED;
2071 }
2072
2073 /* Get the fake offset back... */
2074 ret = drmIoctl(bufmgr_gem->fd,
2075 DRM_IOCTL_I915_GEM_MMAP_OFFSET,
2076 &mmap_arg);
2077 offset = mmap_arg.offset;
2078 }
2079 else
2080 {
2081 struct drm_i915_gem_mmap_gtt mmap_arg;
2082
2083 MOS_DBG("bo_map_gtt: mmap %d (%s), map_count=%d\n",
2084 bo_gem->gem_handle, bo_gem->name, bo_gem->map_count);
2085
2086 memclear(mmap_arg);
2087 mmap_arg.handle = bo_gem->gem_handle;
2088
2089 /* Get the fake offset back... */
2090 ret = drmIoctl(bufmgr_gem->fd,
2091 DRM_IOCTL_I915_GEM_MMAP_GTT,
2092 &mmap_arg);
2093 offset = mmap_arg.offset;
2094 }
2095 if (ret != 0) {
2096 ret = -errno;
2097 MOS_DBG("%s:%d: Error preparing buffer map %d (%s): %s .\n",
2098 __FILE__, __LINE__,
2099 bo_gem->gem_handle, bo_gem->name,
2100 strerror(errno));
2101 return ret;
2102 }
2103
2104 /* and mmap it */
2105 bo_gem->gtt_virtual = drm_mmap(0, bo->size, PROT_READ | PROT_WRITE,
2106 MAP_SHARED, bufmgr_gem->fd,
2107 offset);
2108 if (bo_gem->gtt_virtual == MAP_FAILED) {
2109 bo_gem->gtt_virtual = nullptr;
2110 ret = -errno;
2111 MOS_DBG("%s:%d: Error mapping buffer %d (%s): %s .\n",
2112 __FILE__, __LINE__,
2113 bo_gem->gem_handle, bo_gem->name,
2114 strerror(errno));
2115 return ret;
2116 }
2117 }
2118 #ifdef __cplusplus
2119 bo->virt = bo_gem->gtt_virtual;
2120 #else
2121 bo->virtual = bo_gem->gtt_virtual;
2122 #endif
2123
2124 MOS_DBG("bo_map_gtt: %d (%s) -> %p\n", bo_gem->gem_handle, bo_gem->name,
2125 bo_gem->gtt_virtual);
2126
2127 return 0;
2128 }
2129
2130 static int
2131 mos_gem_bo_map_gtt(struct mos_linux_bo *bo)
2132 {
2133 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
2134 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2135 struct drm_i915_gem_set_domain set_domain;
2136 struct drm_i915_gem_wait wait;
2137 int ret;
2138
2139 pthread_mutex_lock(&bufmgr_gem->lock);
2140
2141 ret = map_gtt(bo);
2142 if (ret) {
2143 pthread_mutex_unlock(&bufmgr_gem->lock);
2144 return ret;
2145 }
2146
2147 if (bufmgr_gem->has_lmem) {
2148 assert(bufmgr_gem->has_wait_timeout);
2149 memclear(wait);
2150 wait.bo_handle = bo_gem->gem_handle;
2151 wait.timeout_ns = -1; // infinite wait
2152 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_WAIT, &wait);
2153 if (ret == -1) {
2154 MOS_DBG("%s:%d: DRM_IOCTL_I915_GEM_WAIT failed (%d)\n",
2155 __FILE__, __LINE__, errno);
2156 }
2157 } else {
2158 /* Now move it to the GTT domain so that the GPU and CPU
2159 * caches are flushed and the GPU isn't actively using the
2160 * buffer.
2161 *
2162 * The pagefault handler does this domain change for us when
2163 * it has unbound the BO from the GTT, but it's up to us to
2164 * tell it when we're about to use things if we had done
2165 * rendering and it still happens to be bound to the GTT.
2166 */
2167 memclear(set_domain);
2168 set_domain.handle = bo_gem->gem_handle;
2169 set_domain.read_domains = I915_GEM_DOMAIN_GTT;
2170 set_domain.write_domain = I915_GEM_DOMAIN_GTT;
2171 ret = drmIoctl(bufmgr_gem->fd,
2172 DRM_IOCTL_I915_GEM_SET_DOMAIN,
2173 &set_domain);
2174 if (ret != 0) {
2175 MOS_DBG("%s:%d: Error setting domain %d: %s\n",
2176 __FILE__, __LINE__, bo_gem->gem_handle,
2177 strerror(errno));
2178 }
2179 }
2180 mos_gem_bo_mark_mmaps_incoherent(bo);
2181 VG(VALGRIND_MAKE_MEM_DEFINED(bo_gem->gtt_virtual, bo->size));
2182 pthread_mutex_unlock(&bufmgr_gem->lock);
2183
2184 return 0;
2185 }
2186
2187 /**
2188 * Performs a mapping of the buffer object like the normal GTT
2189 * mapping, but avoids waiting for the GPU to be done reading from or
2190 * rendering to the buffer.
2191 *
2192 * This is used in the implementation of GL_ARB_map_buffer_range: The
2193 * user asks to create a buffer, then does a mapping, fills some
2194 * space, runs a drawing command, then asks to map it again without
2195 * synchronizing because it guarantees that it won't write over the
2196 * data that the GPU is busy using (or, more specifically, that if it
2197 * does write over the data, it acknowledges that rendering is
2198 * undefined).
2199 */
2200
2201 static int
2202 mos_gem_bo_map_unsynchronized(struct mos_linux_bo *bo)
2203 {
2204 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
2205 #ifdef HAVE_VALGRIND
2206 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2207 #endif
2208 int ret;
2209
2210 /* If the CPU cache isn't coherent with the GTT, then use a
2211 * regular synchronized mapping. The problem is that we don't
2212 * track where the buffer was last used on the CPU side in
2213 * terms of drm_intel_bo_map vs drm_intel_gem_bo_map_gtt, so
2214 * we would potentially corrupt the buffer even when the user
2215 * does reasonable things.
2216 */
2217 if (!bufmgr_gem->has_llc)
2218 return mos_gem_bo_map_gtt(bo);
2219
2220 pthread_mutex_lock(&bufmgr_gem->lock);
2221
2222 ret = map_gtt(bo);
2223 if (ret == 0) {
2224 mos_gem_bo_mark_mmaps_incoherent(bo);
2225 VG(VALGRIND_MAKE_MEM_DEFINED(bo_gem->gtt_virtual, bo->size));
2226 }
2227
2228 pthread_mutex_unlock(&bufmgr_gem->lock);
2229
2230 return ret;
2231 }
2232
2233 static int mos_gem_bo_unmap(struct mos_linux_bo *bo)
2234 {
2235 struct mos_bufmgr_gem *bufmgr_gem;
2236 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2237 int ret = 0;
2238
2239 if (bo == nullptr)
2240 return 0;
2241
2242 if (bo_gem->is_userptr)
2243 return 0;
2244
2245 bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
2246
2247 pthread_mutex_lock(&bufmgr_gem->lock);
2248
2249 if (bo_gem->map_count <= 0) {
2250 MOS_DBG("attempted to unmap an unmapped bo\n");
2251 pthread_mutex_unlock(&bufmgr_gem->lock);
2252 /* Preserve the old behaviour of just treating this as a
2253 * no-op rather than reporting the error.
2254 */
2255 return 0;
2256 }
2257
2258 if (bo_gem->mapped_cpu_write) {
2259 struct drm_i915_gem_sw_finish sw_finish;
2260
2261 /* Cause a flush to happen if the buffer's pinned for
2262 * scanout, so the results show up in a timely manner.
2263 * Unlike GTT set domains, this only does work if the
2264 * buffer should be scanout-related.
2265 */
2266 memclear(sw_finish);
2267 sw_finish.handle = bo_gem->gem_handle;
2268 ret = drmIoctl(bufmgr_gem->fd,
2269 DRM_IOCTL_I915_GEM_SW_FINISH,
2270 &sw_finish);
2271 ret = ret == -1 ? -errno : 0;
2272
2273 bo_gem->mapped_cpu_write = false;
2274 }
2275
2276 /* We need to unmap after every innovation as we cannot track
2277 * an open vma for every bo as that will exhaasut the system
2278 * limits and cause later failures.
2279 */
2280 if (--bo_gem->map_count == 0) {
2281 mos_gem_bo_mark_mmaps_incoherent(bo);
2282 #ifdef __cplusplus
2283 bo->virt = nullptr;
2284 #else
2285 bo->virtual = nullptr;
2286 #endif
2287 }
2288 pthread_mutex_unlock(&bufmgr_gem->lock);
2289
2290 return ret;
2291 }
2292
2293 static int
2294 mos_gem_bo_unmap_wc(struct mos_linux_bo *bo)
2295 {
2296 return mos_gem_bo_unmap(bo);
2297 }
2298
2299 static int
2300 mos_gem_bo_unmap_gtt(struct mos_linux_bo *bo)
2301 {
2302 return mos_gem_bo_unmap(bo);
2303 }
2304
2305 /** Waits for all GPU rendering with the object to have completed. */
2306 static void
2307 mos_gem_bo_wait_rendering(struct mos_linux_bo *bo)
2308 {
2309 mos_gem_bo_start_gtt_access(bo, 1);
2310 }
2311
2312 /**
2313 * Waits on a BO for the given amount of time.
2314 *
2315 * @bo: buffer object to wait for
2316 * @timeout_ns: amount of time to wait in nanoseconds.
2317 * If value is less than 0, an infinite wait will occur.
2318 *
2319 * Returns 0 if the wait was successful ie. the last batch referencing the
2320 * object has completed within the allotted time. Otherwise some negative return
2321 * value describes the error. Of particular interest is -ETIME when the wait has
2322 * failed to yield the desired result.
2323 *
2324 * Similar to drm_intel_gem_bo_wait_rendering except a timeout parameter allows
2325 * the operation to give up after a certain amount of time. Another subtle
2326 * difference is the internal locking semantics are different (this variant does
2327 * not hold the lock for the duration of the wait). This makes the wait subject
2328 * to a larger userspace race window.
2329 *
2330 * The implementation shall wait until the object is no longer actively
2331 * referenced within a batch buffer at the time of the call. The wait will
2332 * not guarantee that the buffer is re-issued via another thread, or an flinked
2333 * handle. Userspace must make sure this race does not occur if such precision
2334 * is important.
2335 *
2336 * Note that some kernels have broken the inifite wait for negative values
2337 * promise, upgrade to latest stable kernels if this is the case.
2338 */
2339 drm_export int
2340 mos_gem_bo_wait(struct mos_linux_bo *bo, int64_t timeout_ns)
2341 {
2342
2343 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
2344 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2345 struct drm_i915_gem_wait wait;
2346 int ret;
2347
2348 if (!bufmgr_gem->has_wait_timeout) {
2349 MOS_DBG("%s:%d: Timed wait is not supported. Falling back to "
2350 "infinite wait\n", __FILE__, __LINE__);
2351 if (timeout_ns) {
2352 mos_gem_bo_wait_rendering(bo);
2353 return 0;
2354 } else {
2355 return mos_gem_bo_busy(bo) ? -ETIME : 0;
2356 }
2357 }
2358
2359 memclear(wait);
2360 wait.bo_handle = bo_gem->gem_handle;
2361 wait.timeout_ns = timeout_ns;
2362 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_WAIT, &wait);
2363 if (ret == -1)
2364 return -errno;
2365
2366 return ret;
2367 }
2368
2369 /**
2370 * Sets the object to the GTT read and possibly write domain, used by the X
2371 * 2D driver in the absence of kernel support to do drm_intel_gem_bo_map_gtt().
2372 *
2373 * In combination with drm_intel_gem_bo_pin() and manual fence management, we
2374 * can do tiled pixmaps this way.
2375 */
2376 static void
2377 mos_gem_bo_start_gtt_access(struct mos_linux_bo *bo, int write_enable)
2378 {
2379 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
2380 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2381 struct drm_i915_gem_set_domain set_domain;
2382 struct drm_i915_gem_wait wait;
2383 int ret;
2384
2385 if (bufmgr_gem->has_lmem) {
2386 assert(bufmgr_gem->has_wait_timeout);
2387 memclear(wait);
2388 wait.bo_handle = bo_gem->gem_handle;
2389 wait.timeout_ns = -1; // infinite wait
2390 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_WAIT, &wait);
2391 if (ret == -1) {
2392 MOS_DBG("%s:%d: DRM_IOCTL_I915_GEM_WAIT failed (%d)\n",
2393 __FILE__, __LINE__, errno);
2394 }
2395 } else {
2396 memclear(set_domain);
2397 set_domain.handle = bo_gem->gem_handle;
2398 set_domain.read_domains = I915_GEM_DOMAIN_GTT;
2399 set_domain.write_domain = write_enable ? I915_GEM_DOMAIN_GTT : 0;
2400 ret = drmIoctl(bufmgr_gem->fd,
2401 DRM_IOCTL_I915_GEM_SET_DOMAIN,
2402 &set_domain);
2403 if (ret != 0) {
2404 MOS_DBG("%s:%d: Error setting memory domains %d (%08x %08x): %s .\n",
2405 __FILE__, __LINE__, bo_gem->gem_handle,
2406 set_domain.read_domains, set_domain.write_domain,
2407 strerror(errno));
2408 }
2409 }
2410 }
2411
2412 static void
2413 mos_bufmgr_gem_destroy(struct mos_bufmgr *bufmgr)
2414 {
2415 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bufmgr;
2416 struct drm_gem_close close_bo;
2417 int i, ret;
2418
2419 free(bufmgr_gem->exec2_objects);
2420 free(bufmgr_gem->exec_objects);
2421 free(bufmgr_gem->exec_bos);
2422 pthread_mutex_destroy(&bufmgr_gem->lock);
2423
2424 /* Free any cached buffer objects we were going to reuse */
2425 for (i = 0; i < bufmgr_gem->num_buckets; i++) {
2426 struct mos_gem_bo_bucket *bucket =
2427 &bufmgr_gem->cache_bucket[i];
2428 struct mos_bo_gem *bo_gem;
2429
2430 while (!DRMLISTEMPTY(&bucket->head)) {
2431 bo_gem = DRMLISTENTRY(struct mos_bo_gem,
2432 bucket->head.next, head);
2433 DRMLISTDEL(&bo_gem->head);
2434
2435 mos_gem_bo_free(&bo_gem->bo);
2436 }
2437 }
2438
2439 /* Release userptr bo kept hanging around for optimisation. */
2440 if (bufmgr_gem->userptr_active.ptr) {
2441 memclear(close_bo);
2442 close_bo.handle = bufmgr_gem->userptr_active.handle;
2443 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_GEM_CLOSE, &close_bo);
2444 free(bufmgr_gem->userptr_active.ptr);
2445 if (ret)
2446 fprintf(stderr,
2447 "Failed to release test userptr object! (%d) "
2448 "i915 kernel driver may not be sane!\n", errno);
2449 }
2450
2451 mos_vma_heap_finish(&bufmgr_gem->vma_heap[MEMZONE_SYS]);
2452 mos_vma_heap_finish(&bufmgr_gem->vma_heap[MEMZONE_DEVICE]);
2453 if (BufmgrPrelim::IsPrelimSupported()) {
2454 bufmgr_gem->prelim->UninitVmaHeap(&bufmgr_gem->vma_heap[MEMZONE_PRIME]);
2455
2456 BufmgrPrelim::DestroyPrelim(bufmgr_gem->prelim);
2457 bufmgr_gem->prelim = nullptr;
2458 }
2459
2460 if (bufmgr_gem->mem_profiler_fd != -1)
2461 {
2462 close(bufmgr_gem->mem_profiler_fd);
2463 }
2464
2465 free(bufmgr);
2466 }
2467
2468 static int
2469 do_bo_emit_reloc(struct mos_linux_bo *bo, uint32_t offset,
2470 struct mos_linux_bo *target_bo, uint32_t target_offset,
2471 uint32_t read_domains, uint32_t write_domain,
2472 bool need_fence, uint64_t presumed_offset)
2473 {
2474 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
2475 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2476 struct mos_bo_gem *target_bo_gem = (struct mos_bo_gem *) target_bo;
2477
2478 if (bo_gem->has_error)
2479 return -ENOMEM;
2480
2481 if (target_bo_gem->has_error) {
2482 bo_gem->has_error = true;
2483 return -ENOMEM;
2484 }
2485
2486 /* Create a new relocation list if needed */
2487 if (bo_gem->relocs == nullptr && mos_setup_reloc_list(bo))
2488 return -ENOMEM;
2489
2490 /* Check overflow */
2491 assert(bo_gem->reloc_count < bufmgr_gem->max_relocs);
2492
2493 /* Check args */
2494 assert(offset <= bo->size - 4);
2495 assert((write_domain & (write_domain - 1)) == 0);
2496
2497 /* An object needing a fence is a tiled buffer, so it won't have
2498 * relocs to other buffers.
2499 */
2500 if (need_fence) {
2501 assert(target_bo_gem->reloc_count == 0);
2502 target_bo_gem->reloc_tree_fences = 1;
2503 }
2504
2505 /* Make sure that we're not adding a reloc to something whose size has
2506 * already been accounted for.
2507 */
2508 assert(!bo_gem->used_as_reloc_target);
2509 if (target_bo_gem != bo_gem) {
2510 target_bo_gem->used_as_reloc_target = true;
2511 bo_gem->reloc_tree_size += target_bo_gem->reloc_tree_size;
2512 bo_gem->reloc_tree_fences += target_bo_gem->reloc_tree_fences;
2513 }
2514
2515 int flags = 0;
2516 if (target_bo_gem->pad_to_size)
2517 flags |= EXEC_OBJECT_PAD_TO_SIZE;
2518 if (target_bo_gem->use_48b_address_range)
2519 flags |= EXEC_OBJECT_SUPPORTS_48B_ADDRESS;
2520 if (target_bo_gem->exec_async)
2521 flags |= EXEC_OBJECT_ASYNC;
2522 if (target_bo_gem->exec_capture)
2523 flags |= EXEC_OBJECT_CAPTURE;
2524
2525 if (target_bo != bo)
2526 mos_gem_bo_reference(target_bo);
2527
2528 bo_gem->reloc_target_info[bo_gem->reloc_count].bo = target_bo;
2529 bo_gem->reloc_target_info[bo_gem->reloc_count].flags = flags;
2530 bo_gem->relocs[bo_gem->reloc_count].offset = offset;
2531 bo_gem->relocs[bo_gem->reloc_count].delta = target_offset;
2532 bo_gem->relocs[bo_gem->reloc_count].target_handle =
2533 target_bo_gem->gem_handle;
2534 bo_gem->relocs[bo_gem->reloc_count].read_domains = read_domains;
2535 if (BufmgrPrelim::IsPrelimSupported()) {
2536 // if reloc handle is batch buffer itself, cannot set write domain
2537 bo_gem->relocs[bo_gem->reloc_count].write_domain = (bo_gem->bo.handle == target_bo_gem->gem_handle ? 0 : write_domain);
2538 } else {
2539 bo_gem->relocs[bo_gem->reloc_count].write_domain = write_domain;
2540 }
2541 bo_gem->relocs[bo_gem->reloc_count].presumed_offset = presumed_offset;
2542 bo_gem->reloc_count++;
2543
2544 return 0;
2545 }
2546
2547 static void
2548 mos_gem_bo_use_48b_address_range(struct mos_linux_bo *bo, uint32_t enable)
2549 {
2550 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2551 bo_gem->use_48b_address_range = enable;
2552 }
2553
2554 static void
2555 mos_gem_bo_set_object_async(struct mos_linux_bo *bo)
2556 {
2557 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *)bo;
2558 bo_gem->exec_async = true;
2559 }
2560
2561 static void
2562 mos_gem_bo_set_exec_object_async(struct mos_linux_bo *bo, struct mos_linux_bo *target_bo)
2563 {
2564 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
2565 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2566 struct mos_bo_gem *target_bo_gem = (struct mos_bo_gem *) target_bo;
2567 int i;
2568 for (i = 0; i < bo_gem->reloc_count; i++)
2569 {
2570 if (bo_gem->reloc_target_info[i].bo == target_bo)
2571 {
2572 bo_gem->reloc_target_info[i].flags |= EXEC_OBJECT_ASYNC;
2573 break;
2574 }
2575 }
2576
2577 for (i = 0; i < bo_gem->softpin_target_count; i++)
2578 {
2579 if (bo_gem->softpin_target[i].bo == target_bo)
2580 {
2581 bo_gem->softpin_target[i].flags |= EXEC_OBJECT_ASYNC;
2582 break;
2583 }
2584 }
2585 }
2586
2587 static void
2588 mos_gem_bo_set_object_capture(struct mos_linux_bo *bo)
2589 {
2590 // Do nothing if bo is nullptr
2591 if (bo == nullptr)
2592 {
2593 return;
2594 }
2595 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
2596 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *)bo;
2597 if (bufmgr_gem != nullptr &&
2598 bo_gem != nullptr &&
2599 !bufmgr_gem->object_capture_disabled)
2600 {
2601 bo_gem->exec_capture = true;
2602 }
2603 }
2604
2605 static int
2606 mos_gem_bo_add_softpin_target(struct mos_linux_bo *bo, struct mos_linux_bo *target_bo, bool write_flag)
2607 {
2608 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
2609 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2610 struct mos_bo_gem *target_bo_gem = (struct mos_bo_gem *) target_bo;
2611 if (bo_gem->has_error)
2612 return -ENOMEM;
2613
2614 if (target_bo_gem->has_error) {
2615 bo_gem->has_error = true;
2616 return -ENOMEM;
2617 }
2618
2619 if (!target_bo_gem->is_softpin)
2620 return -EINVAL;
2621 if (target_bo_gem == bo_gem)
2622 return -EINVAL;
2623
2624 if (bo_gem->softpin_target_count == bo_gem->max_softpin_target_count) {
2625 int max_softpin_target_count = bo_gem->max_softpin_target_count * 2;
2626
2627 /* initial softpin target count*/
2628 if (max_softpin_target_count == 0){
2629 max_softpin_target_count = INITIAL_SOFTPIN_TARGET_COUNT;
2630 }
2631
2632 bo_gem->softpin_target = (struct mos_softpin_target *)realloc(bo_gem->softpin_target, max_softpin_target_count *
2633 sizeof(struct mos_softpin_target));
2634 if (!bo_gem->softpin_target)
2635 return -ENOMEM;
2636
2637 bo_gem->max_softpin_target_count = max_softpin_target_count;
2638 }
2639
2640 int flags = EXEC_OBJECT_PINNED;
2641 if (target_bo_gem->pad_to_size)
2642 flags |= EXEC_OBJECT_PAD_TO_SIZE;
2643 if (target_bo_gem->use_48b_address_range)
2644 flags |= EXEC_OBJECT_SUPPORTS_48B_ADDRESS;
2645 if (target_bo_gem->exec_async)
2646 flags |= EXEC_OBJECT_ASYNC;
2647 if (target_bo_gem->exec_capture)
2648 flags |= EXEC_OBJECT_CAPTURE;
2649 if (write_flag)
2650 flags |= EXEC_OBJECT_WRITE;
2651
2652 bo_gem->softpin_target[bo_gem->softpin_target_count].bo = target_bo;
2653 bo_gem->softpin_target[bo_gem->softpin_target_count].flags = flags;
2654 mos_gem_bo_reference(target_bo);
2655 bo_gem->softpin_target_count++;
2656
2657 return 0;
2658 }
2659
2660 static mos_oca_exec_list_info*
2661 mos_bufmgr_bo_get_softpin_targets_info(struct mos_linux_bo *bo, int *count)
2662 {
2663 if(bo == nullptr || count == nullptr)
2664 {
2665 return nullptr;
2666 }
2667 mos_oca_exec_list_info *info = nullptr;
2668 std::vector<int> bo_added;
2669 int counter = 0;
2670 int MAX_COUNT = 50;
2671 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
2672 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *)bo;
2673 int softpin_target_count = bo_gem->softpin_target_count;
2674 if(softpin_target_count == 0 || softpin_target_count > MAX_COUNT)
2675 {
2676 return info;
2677 }
2678 info = (mos_oca_exec_list_info *)malloc((softpin_target_count + 1) * sizeof(mos_oca_exec_list_info));
2679 if(info == nullptr)
2680 {
2681 return info;
2682 }
2683
2684 for(int i = 0; i < softpin_target_count; i++)
2685 {
2686 /*note: set capture for each bo*/
2687 struct mos_softpin_target *target = (struct mos_softpin_target *)&bo_gem->softpin_target[i];
2688 struct mos_bo_gem *target_gem = (struct mos_bo_gem *)target->bo;
2689 if(std::find(bo_added.begin(), bo_added.end(), target->bo->handle) == bo_added.end())
2690 {
2691 info[counter].handle = target->bo->handle;
2692 info[counter].size = target->bo->size;
2693 info[counter].offset64 = target->bo->offset64;
2694
2695 if (!bufmgr_gem->object_capture_disabled)
2696 target->flags |= EXEC_OBJECT_CAPTURE;
2697
2698 info[counter].flags = target->flags;
2699 info[counter].mem_region = target_gem->mem_region;
2700 info[counter].is_batch = false;
2701 bo_added.push_back(target->bo->handle);
2702 counter++;
2703 }
2704 }
2705
2706 /*note: bo is cmd bo, also need to be added*/
2707 int bb_flags = 0;
2708 if (bo_gem->pad_to_size)
2709 bb_flags |= EXEC_OBJECT_PAD_TO_SIZE;
2710 if (bo_gem->use_48b_address_range)
2711 bb_flags |= EXEC_OBJECT_SUPPORTS_48B_ADDRESS;
2712 if (bo_gem->is_softpin)
2713 bb_flags |= EXEC_OBJECT_PINNED;
2714 if (bo_gem->exec_async)
2715 bb_flags |= EXEC_OBJECT_ASYNC;
2716 if (bo_gem->exec_capture)
2717 bb_flags |= EXEC_OBJECT_CAPTURE;
2718
2719 info[counter].handle = bo->handle;
2720 info[counter].size = bo->size;
2721 info[counter].offset64 = bo->offset64;
2722 info[counter].flags = bb_flags;
2723 info[counter].mem_region = bo_gem->mem_region;
2724 info[counter].is_batch = true;
2725 counter++;
2726
2727 *count = counter;
2728
2729 return info;
2730 }
2731
2732 static int
2733 mos_gem_bo_pad_to_size(struct mos_linux_bo *bo, uint64_t pad_to_size)
2734 {
2735 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2736
2737 if (pad_to_size && pad_to_size < bo->size)
2738 return -EINVAL;
2739
2740 bo_gem->pad_to_size = pad_to_size;
2741 return 0;
2742 }
2743
2744 static int
2745 mos_gem_bo_emit_reloc(struct mos_linux_bo *bo, uint32_t offset,
2746 struct mos_linux_bo *target_bo, uint32_t target_offset,
2747 uint32_t read_domains, uint32_t write_domain,
2748 uint64_t presumed_offset)
2749 {
2750 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bo->bufmgr;
2751
2752 return do_bo_emit_reloc(bo, offset, target_bo, target_offset,
2753 read_domains, write_domain,
2754 false,
2755 presumed_offset);
2756 }
2757
2758 /**
2759 * Removes existing relocation entries in the BO after "start".
2760 *
2761 * This allows a user to avoid a two-step process for state setup with
2762 * counting up all the buffer objects and doing a
2763 * drm_intel_bufmgr_check_aperture_space() before emitting any of the
2764 * relocations for the state setup. Instead, save the state of the
2765 * batchbuffer including drm_intel_gem_get_reloc_count(), emit all the
2766 * state, and then check if it still fits in the aperture.
2767 *
2768 * Any further drm_intel_bufmgr_check_aperture_space() queries
2769 * involving this buffer in the tree are undefined after this call.
2770 *
2771 * This also removes all softpinned targets being referenced by the BO.
2772 */
2773 static void
2774 mos_gem_bo_clear_relocs(struct mos_linux_bo *bo, int start)
2775 {
2776 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
2777 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2778 int i;
2779 struct timespec time;
2780
2781 clock_gettime(CLOCK_MONOTONIC, &time);
2782
2783 assert(bo_gem->reloc_count >= start);
2784
2785 /* Unreference the cleared target buffers */
2786 pthread_mutex_lock(&bufmgr_gem->lock);
2787
2788 for (i = start; i < bo_gem->reloc_count; i++) {
2789 struct mos_bo_gem *target_bo_gem = (struct mos_bo_gem *) bo_gem->reloc_target_info[i].bo;
2790 if (&target_bo_gem->bo != bo) {
2791 bo_gem->reloc_tree_fences -= target_bo_gem->reloc_tree_fences;
2792 target_bo_gem->used_as_reloc_target = false;
2793 target_bo_gem->reloc_count = 0;
2794 mos_gem_bo_unreference_locked_timed(&target_bo_gem->bo,
2795 time.tv_sec);
2796 }
2797 }
2798 bo_gem->reloc_count = start;
2799
2800 for (i = 0; i < bo_gem->softpin_target_count; i++) {
2801 struct mos_bo_gem *target_bo_gem = (struct mos_bo_gem *) bo_gem->softpin_target[i].bo;
2802 mos_gem_bo_unreference_locked_timed(&target_bo_gem->bo, time.tv_sec);
2803 }
2804 bo_gem->softpin_target_count = 0;
2805
2806 pthread_mutex_unlock(&bufmgr_gem->lock);
2807
2808 }
2809
2810 /**
2811 * Walk the tree of relocations rooted at BO and accumulate the list of
2812 * validations to be performed and update the relocation buffers with
2813 * index values into the validation list.
2814 */
2815 static void
2816 mos_gem_bo_process_reloc(struct mos_linux_bo *bo)
2817 {
2818 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2819 int i;
2820
2821 if (bo_gem->relocs == nullptr)
2822 return;
2823
2824 for (i = 0; i < bo_gem->reloc_count; i++) {
2825 struct mos_linux_bo *target_bo = bo_gem->reloc_target_info[i].bo;
2826
2827 if (target_bo == bo)
2828 continue;
2829
2830 mos_gem_bo_mark_mmaps_incoherent(bo);
2831
2832 /* Continue walking the tree depth-first. */
2833 mos_gem_bo_process_reloc(target_bo);
2834
2835 /* Add the target to the validate list */
2836 mos_add_validate_buffer(target_bo);
2837 }
2838 }
2839
2840 static void
2841 mos_gem_bo_process_reloc2(struct mos_linux_bo *bo)
2842 {
2843 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *)bo;
2844 int i;
2845
2846 if (bo_gem->relocs == nullptr && bo_gem->softpin_target == nullptr)
2847 return;
2848
2849 for (i = 0; i < bo_gem->reloc_count; i++) {
2850 struct mos_linux_bo *target_bo = bo_gem->reloc_target_info[i].bo;
2851
2852 if (target_bo == bo)
2853 continue;
2854
2855 mos_gem_bo_mark_mmaps_incoherent(bo);
2856
2857 /* Continue walking the tree depth-first. */
2858 mos_gem_bo_process_reloc2(target_bo);
2859
2860 /* Add the target to the validate list */
2861 mos_add_reloc_objects(bo_gem->reloc_target_info[i]);
2862 }
2863
2864 for (i = 0; i < bo_gem->softpin_target_count; i++) {
2865 struct mos_linux_bo *target_bo = bo_gem->softpin_target[i].bo;
2866
2867 if (target_bo == bo)
2868 continue;
2869
2870 mos_gem_bo_mark_mmaps_incoherent(bo);
2871 mos_gem_bo_process_reloc2(target_bo);
2872 mos_add_softpin_objects(bo_gem->softpin_target[i]);
2873 }
2874 }
2875
2876 static void
2877 mos_update_buffer_offsets(struct mos_bufmgr_gem *bufmgr_gem)
2878 {
2879 int i;
2880
2881 for (i = 0; i < bufmgr_gem->exec_count; i++) {
2882 struct mos_linux_bo *bo = bufmgr_gem->exec_bos[i];
2883 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2884
2885 /* Update the buffer offset */
2886 if (bufmgr_gem->exec_objects[i].offset != bo->offset64) {
2887 MOS_DBG("BO %d (%s) migrated: 0x%08x %08x -> 0x%08x %08x\n",
2888 bo_gem->gem_handle, bo_gem->name,
2889 upper_32_bits(bo->offset64),
2890 lower_32_bits(bo->offset64),
2891 upper_32_bits(bufmgr_gem->exec_objects[i].offset),
2892 lower_32_bits(bufmgr_gem->exec_objects[i].offset));
2893 bo->offset64 = bufmgr_gem->exec_objects[i].offset;
2894 bo->offset = bufmgr_gem->exec_objects[i].offset;
2895 }
2896 }
2897 }
2898
2899 static void
2900 mos_update_buffer_offsets2 (struct mos_bufmgr_gem *bufmgr_gem, mos_linux_context *ctx, mos_linux_bo *cmd_bo)
2901 {
2902 int i;
2903
2904 for (i = 0; i < bufmgr_gem->exec_count; i++) {
2905 struct mos_linux_bo *bo = bufmgr_gem->exec_bos[i];
2906 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *)bo;
2907
2908 /* Update the buffer offset */
2909 if (bufmgr_gem->exec2_objects[i].offset != bo->offset64) {
2910 /* If we're seeing softpinned object here it means that the kernel
2911 * has relocated our object... Indicating a programming error
2912 */
2913 assert(!bo_gem->is_softpin);
2914 MOS_DBG("BO %d (%s) migrated: 0x%08x %08x -> 0x%08x %08x\n",
2915 bo_gem->gem_handle, bo_gem->name,
2916 upper_32_bits(bo->offset64),
2917 lower_32_bits(bo->offset64),
2918 upper_32_bits(bufmgr_gem->exec2_objects[i].offset),
2919 lower_32_bits(bufmgr_gem->exec2_objects[i].offset));
2920 bo->offset64 = bufmgr_gem->exec2_objects[i].offset;
2921 bo->offset = bufmgr_gem->exec2_objects[i].offset;
2922 }
2923
2924 if(!bufmgr_gem->use_softpin)
2925 {
2926 if (cmd_bo != bo) {
2927 auto item_ctx = ctx->pOsContext->contextOffsetList.begin();
2928 for (; item_ctx != ctx->pOsContext->contextOffsetList.end(); item_ctx++) {
2929 if (item_ctx->intel_context == ctx && item_ctx->target_bo == bo) {
2930 item_ctx->offset64 = bo->offset64;
2931 break;
2932 }
2933 }
2934 if ( item_ctx == ctx->pOsContext->contextOffsetList.end()) {
2935 struct MOS_CONTEXT_OFFSET newContext = {ctx,
2936 bo,
2937 bo->offset64};
2938 ctx->pOsContext->contextOffsetList.push_back(newContext);
2939 }
2940 }
2941 }
2942 }
2943 }
2944
2945 drm_export int
2946 do_exec2(struct mos_linux_bo *bo, int used, struct mos_linux_context *ctx,
2947 drm_clip_rect_t *cliprects, int num_cliprects, int DR4,
2948 unsigned int flags, int *fence
2949 )
2950 {
2951
2952 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bo->bufmgr;
2953 struct drm_i915_gem_execbuffer2 execbuf;
2954 int ret = 0;
2955 int i;
2956
2957 if (to_bo_gem(bo)->has_error)
2958 return -ENOMEM;
2959
2960 switch (flags & 0x7) {
2961 default:
2962 return -EINVAL;
2963 case I915_EXEC_BLT:
2964 if (!bufmgr_gem->has_blt)
2965 return -EINVAL;
2966 break;
2967 case I915_EXEC_BSD:
2968 if (!bufmgr_gem->has_bsd)
2969 return -EINVAL;
2970 break;
2971 case I915_EXEC_VEBOX:
2972 if (!bufmgr_gem->has_vebox)
2973 return -EINVAL;
2974 break;
2975 case I915_EXEC_RENDER:
2976 case I915_EXEC_DEFAULT:
2977 break;
2978 }
2979
2980 pthread_mutex_lock(&bufmgr_gem->lock);
2981 /* Update indices and set up the validate list. */
2982 mos_gem_bo_process_reloc2(bo);
2983
2984 /* Add the batch buffer to the validation list. There are no relocations
2985 * pointing to it.
2986 */
2987 mos_add_validate_buffer2(bo, 0);
2988
2989 memclear(execbuf);
2990 execbuf.buffers_ptr = (uintptr_t)bufmgr_gem->exec2_objects;
2991 execbuf.buffer_count = bufmgr_gem->exec_count;
2992 execbuf.batch_start_offset = 0;
2993 execbuf.batch_len = used;
2994 execbuf.cliprects_ptr = (uintptr_t)cliprects;
2995 execbuf.num_cliprects = num_cliprects;
2996 execbuf.DR1 = 0;
2997 execbuf.DR4 = DR4;
2998 execbuf.flags = flags;
2999 if (ctx == nullptr)
3000 i915_execbuffer2_set_context_id(execbuf, 0);
3001 else
3002 i915_execbuffer2_set_context_id(execbuf, ctx->ctx_id);
3003 execbuf.rsvd2 = 0;
3004 if((flags & I915_EXEC_FENCE_SUBMIT) || (flags & I915_EXEC_FENCE_IN))
3005 {
3006 execbuf.rsvd2 = *fence;
3007 }
3008 else if(flags & I915_EXEC_FENCE_OUT)
3009 {
3010 execbuf.rsvd2 = -1;
3011 }
3012
3013 if (bufmgr_gem->no_exec)
3014 goto skip_execution;
3015
3016 ret = drmIoctl(bufmgr_gem->fd,
3017 DRM_IOCTL_I915_GEM_EXECBUFFER2_WR,
3018 &execbuf);
3019 if (ret != 0) {
3020 ret = -errno;
3021 if (ret == -ENOSPC) {
3022 MOS_DBG("Execbuffer fails to pin. "
3023 "Estimate: %u. Actual: %u. Available: %u\n",
3024 mos_gem_estimate_batch_space(bufmgr_gem->exec_bos,
3025 bufmgr_gem->exec_count),
3026 mos_gem_compute_batch_space(bufmgr_gem->exec_bos,
3027 bufmgr_gem->exec_count),
3028 (unsigned int) bufmgr_gem->gtt_size);
3029 }
3030 }
3031
3032 if (ctx != nullptr)
3033 {
3034 mos_update_buffer_offsets2(bufmgr_gem, ctx, bo);
3035 }
3036
3037 if(flags & I915_EXEC_FENCE_OUT)
3038 {
3039 *fence = execbuf.rsvd2 >> 32;
3040 }
3041
3042 skip_execution:
3043 if (bufmgr_gem->bufmgr.debug)
3044 mos_gem_dump_validation_list(bufmgr_gem);
3045
3046 for (i = 0; i < bufmgr_gem->exec_count; i++) {
3047 struct mos_bo_gem *bo_gem = to_bo_gem(bufmgr_gem->exec_bos[i]);
3048
3049 bo_gem->idle = false;
3050
3051 /* Disconnect the buffer from the validate list */
3052 bo_gem->validate_index = -1;
3053 bufmgr_gem->exec_bos[i] = nullptr;
3054 }
3055 bufmgr_gem->exec_count = 0;
3056 pthread_mutex_unlock(&bufmgr_gem->lock);
3057
3058 return ret;
3059 }
3060
3061 static int
3062 do_exec3(struct mos_linux_bo **bo, int _num_bo, struct mos_linux_context *ctx,
3063 drm_clip_rect_t *cliprects, int num_cliprects, int DR4,
3064 unsigned int _flags, int *fence
3065 )
3066 {
3067 uint64_t flags = _flags;
3068 uint64_t num_bo = _num_bo;
3069 if((bo == nullptr) || (ctx == nullptr) || (num_bo == 0))
3070 {
3071 return -EINVAL;
3072 }
3073
3074 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bo[0]->bufmgr;
3075 struct drm_i915_gem_execbuffer2 execbuf;
3076 int ret = 0;
3077 int i;
3078
3079 pthread_mutex_lock(&bufmgr_gem->lock);
3080
3081 struct mos_exec_info exec_info;
3082 memset(static_cast<void*>(&exec_info), 0, sizeof(exec_info));
3083 exec_info.batch_obj = (struct drm_i915_gem_exec_object2 *) calloc (num_bo, sizeof(struct drm_i915_gem_exec_object2));
3084 if(exec_info.batch_obj == nullptr)
3085 {
3086 ret = -ENOMEM;
3087 goto skip_execution;
3088 }
3089 exec_info.obj_remain_size = OBJ512_SIZE;
3090 exec_info.obj = (struct drm_i915_gem_exec_object2 *) calloc (OBJ512_SIZE, sizeof(struct drm_i915_gem_exec_object2));
3091 if(exec_info.obj == nullptr)
3092 {
3093 ret = -ENOMEM;
3094 goto skip_execution;
3095 }
3096
3097 for(i = 0; i < num_bo; i++)
3098 {
3099 if (to_bo_gem(bo[i])->has_error)
3100 {
3101 ret = -ENOMEM;
3102 goto skip_execution;
3103 }
3104
3105 /* Update indices and set up the validate list. */
3106 mos_gem_bo_process_reloc2(bo[i]);
3107
3108 /* Add the batch buffer to the validation list. There are no relocations
3109 * pointing to it.
3110 */
3111 mos_add_validate_buffer2(bo[i], 0);
3112
3113 if((bufmgr_gem->exec_count - 1 + num_bo) > exec_info.obj_remain_size)
3114 {
3115 // origin size + OBJ512_SIZE + obj_count + batch_count;
3116 uint32_t new_obj_size = exec_info.obj_count + exec_info.obj_remain_size + OBJ512_SIZE + bufmgr_gem->exec_count - 1 + num_bo;
3117 struct drm_i915_gem_exec_object2 *new_obj = (struct drm_i915_gem_exec_object2 *)realloc(exec_info.obj, new_obj_size * sizeof(struct drm_i915_gem_exec_object2));
3118 if(new_obj == nullptr)
3119 {
3120 ret = -ENOMEM;
3121 goto skip_execution;
3122 }
3123 exec_info.obj_remain_size = new_obj_size - exec_info.obj_count;
3124 exec_info.obj = new_obj;
3125 }
3126 if(0 == i)
3127 {
3128 uint32_t cp_size = (bufmgr_gem->exec_count - 1) * sizeof(struct drm_i915_gem_exec_object2);
3129 memcpy(exec_info.obj, bufmgr_gem->exec2_objects, cp_size);
3130 exec_info.obj_count += (bufmgr_gem->exec_count - 1);
3131 exec_info.obj_remain_size -= (bufmgr_gem->exec_count - 1);
3132 }
3133 else
3134 {
3135 for(int e2 = 0; e2 < bufmgr_gem->exec_count - 1; e2++)
3136 {
3137 int e1;
3138 for(e1 = 0; e1 < exec_info.obj_count; e1++)
3139 {
3140 // skip the duplicated bo if it is already in the list of exec_info.obj
3141 if(bufmgr_gem->exec2_objects[e2].handle == exec_info.obj[e1].handle)
3142 {
3143 break;
3144 }
3145 }
3146 //if no duplicated bo found, add it into list of exec_info.obj
3147 if(e1 == exec_info.obj_count)
3148 {
3149 exec_info.obj[exec_info.obj_count] = bufmgr_gem->exec2_objects[e2];
3150 exec_info.obj_count++;
3151 exec_info.obj_remain_size--;
3152 }
3153 }
3154 }
3155 memcpy(&exec_info.batch_obj[i], &bufmgr_gem->exec2_objects[bufmgr_gem->exec_count - 1], sizeof(struct drm_i915_gem_exec_object2));
3156 exec_info.batch_count++;
3157 uint32_t reloc_count = bufmgr_gem->exec2_objects[bufmgr_gem->exec_count - 1].relocation_count;
3158 uint32_t cp_size = (reloc_count * sizeof(struct drm_i915_gem_relocation_entry));
3159
3160 struct drm_i915_gem_relocation_entry* ptr_reloc = (struct drm_i915_gem_relocation_entry *)calloc(reloc_count,sizeof(struct drm_i915_gem_relocation_entry));
3161 if(ptr_reloc == nullptr)
3162 {
3163 ret = -ENOMEM;
3164 goto skip_execution;
3165 }
3166 memcpy(ptr_reloc, (struct drm_i915_gem_relocation_entry *)bufmgr_gem->exec2_objects[bufmgr_gem->exec_count - 1].relocs_ptr, cp_size);
3167
3168 exec_info.batch_obj[i].relocs_ptr = (uintptr_t)ptr_reloc;
3169 exec_info.batch_obj[i].relocation_count = reloc_count;
3170
3171 //clear bo
3172 if (bufmgr_gem->bufmgr.debug)
3173 {
3174 mos_gem_dump_validation_list(bufmgr_gem);
3175 }
3176
3177 for (int j = 0; j < bufmgr_gem->exec_count; j++) {
3178 struct mos_bo_gem *bo_gem = to_bo_gem(bufmgr_gem->exec_bos[j]);
3179
3180 if(bo_gem)
3181 {
3182 bo_gem->idle = false;
3183
3184 /* Disconnect the buffer from the validate list */
3185 bo_gem->validate_index = -1;
3186 bufmgr_gem->exec_bos[j] = nullptr;
3187 }
3188 }
3189 bufmgr_gem->exec_count = 0;
3190 }
3191
3192 //add back batch obj to the last position
3193 for(i = 0; i < num_bo; i++)
3194 {
3195 exec_info.obj[exec_info.obj_count] = exec_info.batch_obj[i];
3196 exec_info.obj_count++;
3197 exec_info.obj_remain_size--;
3198 }
3199
3200 //save previous ptr
3201 exec_info.pSavePreviousExec2Objects = bufmgr_gem->exec2_objects;
3202 bufmgr_gem->exec_count = exec_info.obj_count;
3203 bufmgr_gem->exec2_objects = exec_info.obj;
3204
3205 memclear(execbuf);
3206 execbuf.buffers_ptr = (uintptr_t)bufmgr_gem->exec2_objects;
3207 execbuf.buffer_count = bufmgr_gem->exec_count;
3208 execbuf.batch_start_offset = 0;
3209 execbuf.cliprects_ptr = (uintptr_t)cliprects;
3210 execbuf.num_cliprects = num_cliprects;
3211 execbuf.DR1 = 0;
3212 execbuf.DR4 = DR4;
3213 execbuf.flags = flags;
3214 if (ctx == nullptr)
3215 i915_execbuffer2_set_context_id(execbuf, 0);
3216 else
3217 i915_execbuffer2_set_context_id(execbuf, ctx->ctx_id);
3218 execbuf.rsvd2 = 0;
3219 if((flags & I915_EXEC_FENCE_SUBMIT) || (flags & I915_EXEC_FENCE_IN))
3220 {
3221 execbuf.rsvd2 = *fence;
3222 }
3223 else if(flags & I915_EXEC_FENCE_OUT)
3224 {
3225 execbuf.rsvd2 = -1;
3226 }
3227
3228 if (bufmgr_gem->no_exec)
3229 goto skip_execution;
3230
3231 ret = drmIoctl(bufmgr_gem->fd,
3232 DRM_IOCTL_I915_GEM_EXECBUFFER2_WR,
3233 &execbuf);
3234
3235 if (ret != 0) {
3236 ret = -errno;
3237 if (ret == -ENOSPC) {
3238 MOS_DBG("Execbuffer fails to pin. "
3239 "Estimate: %u. Actual: %u. Available: %u\n",
3240 mos_gem_estimate_batch_space(bufmgr_gem->exec_bos,
3241 bufmgr_gem->exec_count),
3242 mos_gem_compute_batch_space(bufmgr_gem->exec_bos,
3243 bufmgr_gem->exec_count),
3244 (unsigned int) bufmgr_gem->gtt_size);
3245 }
3246 }
3247
3248 bufmgr_gem->exec2_objects = exec_info.pSavePreviousExec2Objects;
3249
3250 if(flags & I915_EXEC_FENCE_OUT)
3251 {
3252 *fence = execbuf.rsvd2 >> 32;
3253 }
3254
3255 skip_execution:
3256 if (bufmgr_gem->bufmgr.debug)
3257 mos_gem_dump_validation_list(bufmgr_gem);
3258
3259 bufmgr_gem->exec_count = 0;
3260 if(exec_info.batch_obj)
3261 {
3262 for(i = 0; i < num_bo; i++)
3263 {
3264 mos_safe_free((struct drm_i915_gem_relocation_entry *)exec_info.batch_obj[i].relocs_ptr);
3265 }
3266 }
3267 mos_safe_free(exec_info.obj);
3268 mos_safe_free(exec_info.batch_obj);
3269 pthread_mutex_unlock(&bufmgr_gem->lock);
3270
3271 return ret;
3272 }
3273
3274 static int
3275 mos_gem_bo_exec2(struct mos_linux_bo *bo, int used,
3276 drm_clip_rect_t *cliprects, int num_cliprects,
3277 int DR4)
3278 {
3279 return do_exec2(bo, used, nullptr, cliprects, num_cliprects, DR4,
3280 I915_EXEC_RENDER, nullptr);
3281 }
3282
3283 static int
3284 mos_gem_bo_mrb_exec2(struct mos_linux_bo *bo, int used,
3285 drm_clip_rect_t *cliprects, int num_cliprects, int DR4,
3286 unsigned int flags)
3287 {
3288 return do_exec2(bo, used, nullptr, cliprects, num_cliprects, DR4,
3289 flags, nullptr);
3290 }
3291
3292 static int
3293 mos_gem_bo_context_exec2(struct mos_linux_bo *bo, int used, struct mos_linux_context *ctx,
3294 drm_clip_rect_t *cliprects, int num_cliprects, int DR4,
3295 unsigned int flags, int *fence)
3296 {
3297 return do_exec2(bo, used, ctx, cliprects, num_cliprects, DR4,
3298 flags, fence);
3299 }
3300
3301 static int
3302 mos_gem_bo_context_exec3(struct mos_linux_bo **bo, int num_bo, struct mos_linux_context *ctx,
3303 struct drm_clip_rect *cliprects, int num_cliprects, int DR4,
3304 unsigned int flags, int *fence)
3305 {
3306 return do_exec3(bo, num_bo, ctx, cliprects, num_cliprects, DR4,
3307 flags, fence);
3308 }
3309
3310 static int
3311 mos_gem_bo_check_mem_region_internal(struct mos_linux_bo *bo,
3312 int mem_type)
3313 {
3314 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
3315 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
3316
3317 if (BufmgrPrelim::IsPrelimSupported()) {
3318 return bufmgr_gem->prelim->CheckMemRegion(bo_gem->mem_region, mem_type);
3319 }
3320
3321 // when re-cycle the gem_bo, need to keep the VA space is keeping consistent on memory type
3322 if (bo_gem->mem_region == I915_MEMORY_CLASS_SYSTEM &&
3323 (mem_type == MOS_MEMPOOL_VIDEOMEMORY || mem_type == MOS_MEMPOOL_DEVICEMEMORY))
3324 return -EINVAL;
3325
3326 if (bo_gem->mem_region == I915_MEMORY_CLASS_DEVICE &&
3327 (mem_type == MOS_MEMPOOL_SYSTEMMEMORY))
3328 return -EINVAL;
3329
3330 return 0;
3331 }
3332
3333 static int
3334 mos_gem_bo_set_tiling_internal(struct mos_linux_bo *bo,
3335 uint32_t tiling_mode,
3336 uint32_t stride)
3337 {
3338 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
3339 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
3340 struct drm_i915_gem_set_tiling set_tiling;
3341 int ret;
3342
3343 if (!bufmgr_gem->has_fence_reg)
3344 return 0;
3345
3346 if (bo_gem->global_name == 0 &&
3347 tiling_mode == bo_gem->tiling_mode &&
3348 stride == bo_gem->stride)
3349 return 0;
3350
3351 memset(&set_tiling, 0, sizeof(set_tiling));
3352 /* set_tiling is slightly broken and overwrites the
3353 * input on the error path, so we have to open code
3354 * rmIoctl.
3355 */
3356 set_tiling.handle = bo_gem->gem_handle;
3357 set_tiling.tiling_mode = tiling_mode;
3358 set_tiling.stride = stride;
3359
3360 ret = drmIoctl(bufmgr_gem->fd,
3361 DRM_IOCTL_I915_GEM_SET_TILING,
3362 &set_tiling);
3363 if (ret == -1)
3364 return -errno;
3365
3366 bo_gem->tiling_mode = set_tiling.tiling_mode;
3367 bo_gem->swizzle_mode = set_tiling.swizzle_mode;
3368 bo_gem->stride = set_tiling.stride;
3369 return 0;
3370 }
3371
3372 static int
3373 mos_gem_bo_set_tiling(struct mos_linux_bo *bo, uint32_t * tiling_mode,
3374 uint32_t stride)
3375 {
3376 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
3377 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
3378 int ret;
3379
3380 /* Tiling with userptr surfaces is not supported
3381 * on all hardware so refuse it for time being.
3382 */
3383 if (bo_gem->is_userptr)
3384 return -EINVAL;
3385
3386 /* Linear buffers have no stride. By ensuring that we only ever use
3387 * stride 0 with linear buffers, we simplify our code.
3388 */
3389 if (*tiling_mode == I915_TILING_NONE)
3390 stride = 0;
3391
3392 ret = mos_gem_bo_set_tiling_internal(bo, *tiling_mode, stride);
3393 if (ret == 0)
3394 mos_bo_gem_set_in_aperture_size(bufmgr_gem, bo_gem, 0);
3395 *tiling_mode = bo_gem->tiling_mode;
3396 return ret;
3397 }
3398
3399 static int
3400 mos_gem_bo_get_tiling(struct mos_linux_bo *bo, uint32_t * tiling_mode,
3401 uint32_t * swizzle_mode)
3402 {
3403 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
3404
3405 *tiling_mode = bo_gem->tiling_mode;
3406 *swizzle_mode = bo_gem->swizzle_mode;
3407 return 0;
3408 }
3409
3410 static int
3411 mos_gem_bo_set_softpin_offset(struct mos_linux_bo *bo, uint64_t offset)
3412 {
3413 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
3414
3415 bo_gem->is_softpin = true;
3416 bo->offset64 = offset;
3417 bo->offset = offset;
3418 return 0;
3419 }
3420
3421 static int
3422 mos_gem_bo_set_softpin(MOS_LINUX_BO *bo)
3423 {
3424 int ret = 0;
3425 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
3426 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
3427
3428 pthread_mutex_lock(&bufmgr_gem->lock);
3429 if (!mos_gem_bo_is_softpin(bo))
3430 {
3431 uint64_t offset = 0;
3432 if (BufmgrPrelim::IsPrelimSupported() && bo_gem->mem_region == MEMZONE_PRIME) {
3433 offset = mos_gem_bo_vma_alloc(bo->bufmgr, (enum mos_memory_zone)bo_gem->mem_region, bo->size, PAGE_SIZE_2M);
3434 } else {
3435 uint64_t alignment = (bufmgr_gem->softpin_va1Malign) ? PAGE_SIZE_1M : PAGE_SIZE_64K;
3436 offset = mos_gem_bo_vma_alloc(bo->bufmgr, (enum mos_memory_zone)bo_gem->mem_region, bo->size, alignment);
3437 }
3438 ret = mos_gem_bo_set_softpin_offset(bo, offset);
3439 }
3440 pthread_mutex_unlock(&bufmgr_gem->lock);
3441
3442 if (ret == 0)
3443 {
3444 ret = mos_bo_use_48b_address_range(bo, 1);
3445 }
3446
3447 return ret;
3448 }
3449
3450 static struct mos_linux_bo *
3451 mos_gem_bo_create_from_prime(struct mos_bufmgr *bufmgr, struct mos_drm_bo_alloc_prime *alloc_prime)
3452 {
3453 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bufmgr;
3454 int ret;
3455 uint32_t handle;
3456 struct mos_bo_gem *bo_gem;
3457 struct drm_i915_gem_get_tiling get_tiling;
3458 int prime_fd = alloc_prime->prime_fd;
3459 int size = alloc_prime->size;
3460 drmMMListHead *list;
3461
3462 pthread_mutex_lock(&bufmgr_gem->lock);
3463 ret = drmPrimeFDToHandle(bufmgr_gem->fd, prime_fd, &handle);
3464 if (ret) {
3465 MOS_DBG("create_from_prime: failed to obtain handle from fd: %s\n", strerror(errno));
3466 pthread_mutex_unlock(&bufmgr_gem->lock);
3467 return nullptr;
3468 }
3469
3470 /*
3471 * See if the kernel has already returned this buffer to us. Just as
3472 * for named buffers, we must not create two bo's pointing at the same
3473 * kernel object
3474 */
3475 for (list = bufmgr_gem->named.next;
3476 list != &bufmgr_gem->named;
3477 list = list->next) {
3478 bo_gem = DRMLISTENTRY(struct mos_bo_gem, list, name_list);
3479 if (bo_gem->gem_handle == handle) {
3480 mos_gem_bo_reference(&bo_gem->bo);
3481 pthread_mutex_unlock(&bufmgr_gem->lock);
3482 return &bo_gem->bo;
3483 }
3484 }
3485
3486 bo_gem = (struct mos_bo_gem *)calloc(1, sizeof(*bo_gem));
3487 if (!bo_gem) {
3488 pthread_mutex_unlock(&bufmgr_gem->lock);
3489 return nullptr;
3490 }
3491 /* Determine size of bo. The fd-to-handle ioctl really should
3492 * return the size, but it doesn't. If we have kernel 3.12 or
3493 * later, we can lseek on the prime fd to get the size. Older
3494 * kernels will just fail, in which case we fall back to the
3495 * provided (estimated or guess size). */
3496 ret = lseek(prime_fd, 0, SEEK_END);
3497 if (ret != -1)
3498 bo_gem->bo.size = ret;
3499 else
3500 bo_gem->bo.size = size;
3501
3502 bo_gem->bo.handle = handle;
3503 bo_gem->bo.bufmgr = bufmgr;
3504
3505 bo_gem->gem_handle = handle;
3506
3507 atomic_set(&bo_gem->refcount, 1);
3508
3509 bo_gem->name = alloc_prime->name;
3510 bo_gem->validate_index = -1;
3511 bo_gem->reloc_tree_fences = 0;
3512 bo_gem->used_as_reloc_target = false;
3513 bo_gem->has_error = false;
3514 bo_gem->reusable = false;
3515 bo_gem->use_48b_address_range = bufmgr_gem->bufmgr.bo_use_48b_address_range ? true : false;
3516 if (BufmgrPrelim::IsPrelimSupported()) {
3517 bo_gem->mem_region = MEMZONE_PRIME;
3518 }
3519
3520 DRMLISTADDTAIL(&bo_gem->name_list, &bufmgr_gem->named);
3521 pthread_mutex_unlock(&bufmgr_gem->lock);
3522
3523 memclear(get_tiling);
3524 if(bufmgr_gem->has_fence_reg) {
3525 get_tiling.handle = bo_gem->gem_handle;
3526 ret = drmIoctl(bufmgr_gem->fd,
3527 DRM_IOCTL_I915_GEM_GET_TILING,
3528 &get_tiling);
3529 if (ret != 0) {
3530 MOS_DBG("create_from_prime: failed to get tiling: %s\n", strerror(errno));
3531 mos_gem_bo_unreference(&bo_gem->bo);
3532 return nullptr;
3533 }
3534 }
3535 else
3536 {
3537 MOS_DBG("create_from_prime: driver ignored to get tiling from kernel\n");
3538 }
3539
3540 bo_gem->tiling_mode = get_tiling.tiling_mode;
3541 bo_gem->swizzle_mode = get_tiling.swizzle_mode;
3542 /* XXX stride is unknown */
3543 mos_bo_gem_set_in_aperture_size(bufmgr_gem, bo_gem, 0);
3544 if (bufmgr_gem->use_softpin)
3545 {
3546 mos_bo_set_softpin(&bo_gem->bo);
3547 }
3548
3549 return &bo_gem->bo;
3550 }
3551
3552 static int
3553 mos_gem_bo_export_to_prime(struct mos_linux_bo *bo, int *prime_fd)
3554 {
3555 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
3556 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
3557
3558 pthread_mutex_lock(&bufmgr_gem->lock);
3559 if (DRMLISTEMPTY(&bo_gem->name_list))
3560 DRMLISTADDTAIL(&bo_gem->name_list, &bufmgr_gem->named);
3561 pthread_mutex_unlock(&bufmgr_gem->lock);
3562
3563 if (drmPrimeHandleToFD(bufmgr_gem->fd, bo_gem->gem_handle,
3564 DRM_CLOEXEC, prime_fd) != 0)
3565 return -errno;
3566
3567 bo_gem->reusable = false;
3568
3569 return 0;
3570 }
3571
3572 static int
3573 mos_gem_bo_flink(struct mos_linux_bo *bo, uint32_t * name)
3574 {
3575 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
3576 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
3577 int ret;
3578
3579 if (!bo_gem->global_name) {
3580 struct drm_gem_flink flink;
3581
3582 memclear(flink);
3583 flink.handle = bo_gem->gem_handle;
3584
3585 pthread_mutex_lock(&bufmgr_gem->lock);
3586
3587 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_GEM_FLINK, &flink);
3588 if (ret != 0) {
3589 pthread_mutex_unlock(&bufmgr_gem->lock);
3590 return -errno;
3591 }
3592
3593 bo_gem->global_name = flink.name;
3594 bo_gem->reusable = false;
3595
3596 if (DRMLISTEMPTY(&bo_gem->name_list))
3597 DRMLISTADDTAIL(&bo_gem->name_list, &bufmgr_gem->named);
3598 pthread_mutex_unlock(&bufmgr_gem->lock);
3599 }
3600
3601 *name = bo_gem->global_name;
3602 return 0;
3603 }
3604
3605 /**
3606 * Enables unlimited caching of buffer objects for reuse.
3607 *
3608 * This is potentially very memory expensive, as the cache at each bucket
3609 * size is only bounded by how many buffers of that size we've managed to have
3610 * in flight at once.
3611 */
3612 static void
3613 mos_gem_enable_reuse(struct mos_bufmgr *bufmgr)
3614 {
3615 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bufmgr;
3616
3617 bufmgr_gem->bo_reuse = true;
3618 }
3619
3620 /**
3621 * Return the additional aperture space required by the tree of buffer objects
3622 * rooted at bo.
3623 */
3624 static int
3625 mos_gem_bo_get_aperture_space(struct mos_linux_bo *bo)
3626 {
3627 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
3628 int i;
3629 int total = 0;
3630
3631 if (bo == nullptr || bo_gem->included_in_check_aperture)
3632 return 0;
3633
3634 total += bo->size;
3635 bo_gem->included_in_check_aperture = true;
3636
3637 for (i = 0; i < bo_gem->reloc_count; i++)
3638 total +=
3639 mos_gem_bo_get_aperture_space(bo_gem->
3640 reloc_target_info[i].bo);
3641
3642 return total;
3643 }
3644
3645 /**
3646 * Count the number of buffers in this list that need a fence reg
3647 *
3648 * If the count is greater than the number of available regs, we'll have
3649 * to ask the caller to resubmit a batch with fewer tiled buffers.
3650 *
3651 * This function over-counts if the same buffer is used multiple times.
3652 */
3653 static unsigned int
3654 mos_gem_total_fences(struct mos_linux_bo ** bo_array, int count)
3655 {
3656 int i;
3657 unsigned int total = 0;
3658
3659 for (i = 0; i < count; i++) {
3660 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo_array[i];
3661
3662 if (bo_gem == nullptr)
3663 continue;
3664
3665 total += bo_gem->reloc_tree_fences;
3666 }
3667 return total;
3668 }
3669
3670 /**
3671 * Clear the flag set by drm_intel_gem_bo_get_aperture_space() so we're ready
3672 * for the next drm_intel_bufmgr_check_aperture_space() call.
3673 */
3674 static void
3675 mos_gem_bo_clear_aperture_space_flag(struct mos_linux_bo *bo)
3676 {
3677 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
3678 int i;
3679
3680 if (bo == nullptr || !bo_gem->included_in_check_aperture)
3681 return;
3682
3683 bo_gem->included_in_check_aperture = false;
3684
3685 for (i = 0; i < bo_gem->reloc_count; i++)
3686 mos_gem_bo_clear_aperture_space_flag(bo_gem->
3687 reloc_target_info[i].bo);
3688 }
3689
3690 /**
3691 * Return a conservative estimate for the amount of aperture required
3692 * for a collection of buffers. This may double-count some buffers.
3693 */
3694 static unsigned int
3695 mos_gem_estimate_batch_space(struct mos_linux_bo **bo_array, int count)
3696 {
3697 int i;
3698 unsigned int total = 0;
3699
3700 for (i = 0; i < count; i++) {
3701 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo_array[i];
3702 if (bo_gem != nullptr)
3703 total += bo_gem->reloc_tree_size;
3704 }
3705 return total;
3706 }
3707
3708 /**
3709 * Return the amount of aperture needed for a collection of buffers.
3710 * This avoids double counting any buffers, at the cost of looking
3711 * at every buffer in the set.
3712 */
3713 static unsigned int
3714 mos_gem_compute_batch_space(struct mos_linux_bo **bo_array, int count)
3715 {
3716 int i;
3717 unsigned int total = 0;
3718
3719 for (i = 0; i < count; i++) {
3720 total += mos_gem_bo_get_aperture_space(bo_array[i]);
3721 /* For the first buffer object in the array, we get an
3722 * accurate count back for its reloc_tree size (since nothing
3723 * had been flagged as being counted yet). We can save that
3724 * value out as a more conservative reloc_tree_size that
3725 * avoids double-counting target buffers. Since the first
3726 * buffer happens to usually be the batch buffer in our
3727 * callers, this can pull us back from doing the tree
3728 * walk on every new batch emit.
3729 */
3730 if (i == 0) {
3731 struct mos_bo_gem *bo_gem =
3732 (struct mos_bo_gem *) bo_array[i];
3733 bo_gem->reloc_tree_size = total;
3734 }
3735 }
3736
3737 for (i = 0; i < count; i++)
3738 mos_gem_bo_clear_aperture_space_flag(bo_array[i]);
3739 return total;
3740 }
3741
3742 /**
3743 * Return -1 if the batchbuffer should be flushed before attempting to
3744 * emit rendering referencing the buffers pointed to by bo_array.
3745 *
3746 * This is required because if we try to emit a batchbuffer with relocations
3747 * to a tree of buffers that won't simultaneously fit in the aperture,
3748 * the rendering will return an error at a point where the software is not
3749 * prepared to recover from it.
3750 *
3751 * However, we also want to emit the batchbuffer significantly before we reach
3752 * the limit, as a series of batchbuffers each of which references buffers
3753 * covering almost all of the aperture means that at each emit we end up
3754 * waiting to evict a buffer from the last rendering, and we get synchronous
3755 * performance. By emitting smaller batchbuffers, we eat some CPU overhead to
3756 * get better parallelism.
3757 */
3758 static int
3759 mos_gem_check_aperture_space(struct mos_linux_bo **bo_array, int count)
3760 {
3761 struct mos_bufmgr_gem *bufmgr_gem =
3762 (struct mos_bufmgr_gem *) bo_array[0]->bufmgr;
3763 unsigned int total = 0;
3764 unsigned int threshold = bufmgr_gem->gtt_size * 3 / 4;
3765 int total_fences;
3766
3767 /* Check for fence reg constraints if necessary */
3768 if (bufmgr_gem->available_fences) {
3769 total_fences = mos_gem_total_fences(bo_array, count);
3770 if (total_fences > bufmgr_gem->available_fences)
3771 return -ENOSPC;
3772 }
3773
3774 total = mos_gem_estimate_batch_space(bo_array, count);
3775
3776 if (total > threshold)
3777 total = mos_gem_compute_batch_space(bo_array, count);
3778
3779 if (total > threshold) {
3780 MOS_DBG("check_space: overflowed available aperture, "
3781 "%dkb vs %dkb\n",
3782 total / 1024, (int)bufmgr_gem->gtt_size / 1024);
3783 return -ENOSPC;
3784 } else {
3785 MOS_DBG("drm_check_space: total %dkb vs bufgr %dkb\n", total / 1024,
3786 (int)bufmgr_gem->gtt_size / 1024);
3787 return 0;
3788 }
3789 }
3790
3791 /*
3792 * Disable buffer reuse for objects which are shared with the kernel
3793 * as scanout buffers
3794 */
3795 static int
3796 mos_gem_bo_disable_reuse(struct mos_linux_bo *bo)
3797 {
3798 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
3799
3800 bo_gem->reusable = false;
3801 return 0;
3802 }
3803
3804 static int
3805 mos_gem_bo_is_reusable(struct mos_linux_bo *bo)
3806 {
3807 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
3808
3809 return bo_gem->reusable;
3810 }
3811
3812 static int
3813 _mos_gem_bo_references(struct mos_linux_bo *bo, struct mos_linux_bo *target_bo)
3814 {
3815 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
3816 int i;
3817
3818 for (i = 0; i < bo_gem->reloc_count; i++) {
3819 if (bo_gem->reloc_target_info[i].bo == target_bo)
3820 return 1;
3821 if (bo == bo_gem->reloc_target_info[i].bo)
3822 continue;
3823 if (_mos_gem_bo_references(bo_gem->reloc_target_info[i].bo,
3824 target_bo))
3825 return 1;
3826 }
3827
3828 for (i = 0; i< bo_gem->softpin_target_count; i++) {
3829 if (bo_gem->softpin_target[i].bo == target_bo)
3830 return 1;
3831 if (_mos_gem_bo_references(bo_gem->softpin_target[i].bo, target_bo))
3832 return 1;
3833 }
3834
3835 return 0;
3836 }
3837
3838 /** Return true if target_bo is referenced by bo's relocation tree. */
3839 static int
3840 mos_gem_bo_references(struct mos_linux_bo *bo, struct mos_linux_bo *target_bo)
3841 {
3842 struct mos_bo_gem *target_bo_gem = (struct mos_bo_gem *) target_bo;
3843
3844 if (bo == nullptr || target_bo == nullptr)
3845 return 0;
3846 if (target_bo_gem->used_as_reloc_target)
3847 return _mos_gem_bo_references(bo, target_bo);
3848 return 0;
3849 }
3850
3851 static void
3852 add_bucket(struct mos_bufmgr_gem *bufmgr_gem, int size)
3853 {
3854 unsigned int i = bufmgr_gem->num_buckets;
3855
3856 assert(i < ARRAY_SIZE(bufmgr_gem->cache_bucket));
3857
3858 DRMINITLISTHEAD(&bufmgr_gem->cache_bucket[i].head);
3859 bufmgr_gem->cache_bucket[i].size = size;
3860 bufmgr_gem->num_buckets++;
3861 }
3862
3863 static void
3864 init_cache_buckets(struct mos_bufmgr_gem *bufmgr_gem)
3865 {
3866 unsigned long size, cache_max_size = 64 * 1024 * 1024;
3867
3868 /* OK, so power of two buckets was too wasteful of memory.
3869 * Give 3 other sizes between each power of two, to hopefully
3870 * cover things accurately enough. (The alternative is
3871 * probably to just go for exact matching of sizes, and assume
3872 * that for things like composited window resize the tiled
3873 * width/height alignment and rounding of sizes to pages will
3874 * get us useful cache hit rates anyway)
3875 */
3876 add_bucket(bufmgr_gem, 4096);
3877 add_bucket(bufmgr_gem, 4096 * 2);
3878 add_bucket(bufmgr_gem, 4096 * 3);
3879
3880 /* Initialize the linked lists for BO reuse cache. */
3881 for (size = 4 * 4096; size <= cache_max_size; size *= 2) {
3882 add_bucket(bufmgr_gem, size);
3883
3884 add_bucket(bufmgr_gem, size + size * 1 / 4);
3885 add_bucket(bufmgr_gem, size + size * 2 / 4);
3886 add_bucket(bufmgr_gem, size + size * 3 / 4);
3887 }
3888 }
3889
3890 /**
3891 * Get the PCI ID for the device. This can be overridden by setting the
3892 * INTEL_DEVID_OVERRIDE environment variable to the desired ID.
3893 */
3894 static int
3895 get_pci_device_id(struct mos_bufmgr_gem *bufmgr_gem)
3896 {
3897 char *devid_override;
3898 int devid = 0;
3899 int ret;
3900 drm_i915_getparam_t gp;
3901
3902 if (geteuid() == getuid()) {
3903 devid_override = getenv("INTEL_DEVID_OVERRIDE");
3904 if (devid_override) {
3905 bufmgr_gem->no_exec = true;
3906 return strtod(devid_override, nullptr);
3907 }
3908 }
3909
3910 memclear(gp);
3911 gp.param = I915_PARAM_CHIPSET_ID;
3912 gp.value = &devid;
3913 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
3914 if (ret) {
3915 fprintf(stderr, "get chip id failed: %d [%d]\n", ret, errno);
3916 fprintf(stderr, "param: %d, val: %d\n", gp.param, *gp.value);
3917 }
3918 return devid;
3919 }
3920
3921 static int
3922 mos_gem_get_devid(struct mos_bufmgr *bufmgr)
3923 {
3924 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bufmgr;
3925
3926 return bufmgr_gem->pci_device;
3927 }
3928
3929 int mos_gem_ctx_set_user_ctx_params(struct mos_linux_context *context)
3930 {
3931 /*
3932 * INTEL_I915_CTX_CONTROL=0, 1, 2, 3
3933 * 0: default, do nothing
3934 * 1: disable ctx recoverable
3935 * 2: disable ctx bannable
3936 * 3: disable both
3937 * */
3938
3939 if (context == nullptr)
3940 {
3941 return -EINVAL;
3942 }
3943
3944 int ret = 0;
3945 char *user_ctx_env = getenv("INTEL_I915_CTX_CONTROL");
3946 bool disable_recoverable = false;
3947 bool disable_bannable = false;
3948 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)context->bufmgr;
3949
3950 if (user_ctx_env != nullptr)
3951 {
3952 uint8_t user_ctx_env_value = (uint8_t)atoi(user_ctx_env);
3953 if(user_ctx_env_value > 3)
3954 {
3955 MOS_DBG("INTEL_I915_CTX_CONTROL: invalid value %u setting\n",
3956 user_ctx_env_value);
3957 }
3958 else
3959 {
3960 if (user_ctx_env_value & 0x1)
3961 {
3962 disable_recoverable = true;
3963 }
3964
3965 if(user_ctx_env_value & 0x2)
3966 {
3967 disable_bannable = true;
3968 }
3969 }
3970 }
3971
3972 if(disable_recoverable)
3973 {
3974 ret = mos_set_context_param(context,
3975 0,
3976 I915_CONTEXT_PARAM_RECOVERABLE,
3977 0);
3978 if(ret != 0) {
3979 MOS_DBG("I915_CONTEXT_PARAM_RECOVERABLE failed: %s\n",
3980 strerror(errno));
3981 }
3982 else
3983 {
3984 MOS_DBG("successfull to disable context recoverable\n");
3985 }
3986 }
3987
3988 if(disable_bannable)
3989 {
3990 ret = mos_set_context_param(context,
3991 0,
3992 I915_CONTEXT_PARAM_BANNABLE,
3993 0);
3994 if(ret != 0) {
3995 MOS_DBG("I915_CONTEXT_PARAM_BANNABLE failed: %s\n",
3996 strerror(errno));
3997 }
3998 else
3999 {
4000 MOS_DBG("successfull to disable context bannable\n");
4001 }
4002 }
4003
4004 return ret;
4005 }
4006
4007 static struct mos_linux_context *
4008 mos_gem_context_create(struct mos_bufmgr *bufmgr)
4009 {
4010 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bufmgr;
4011 struct drm_i915_gem_context_create create;
4012 struct mos_linux_context *context = nullptr;
4013 int ret;
4014
4015 context = (struct mos_linux_context *)calloc(1, sizeof(*context));
4016 if (!context)
4017 return nullptr;
4018
4019 memclear(create);
4020 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_CONTEXT_CREATE, &create);
4021 if (ret != 0) {
4022 MOS_DBG("DRM_IOCTL_I915_GEM_CONTEXT_CREATE failed: %s\n",
4023 strerror(errno));
4024 free(context);
4025 return nullptr;
4026 }
4027
4028 context->ctx_id = create.ctx_id;
4029 context->bufmgr = bufmgr;
4030
4031 ret = mos_gem_ctx_set_user_ctx_params(context);
4032
4033 return context;
4034 }
4035
4036 static void
4037 mos_gem_context_destroy(struct mos_linux_context *ctx)
4038 {
4039 struct mos_bufmgr_gem *bufmgr_gem;
4040 struct drm_i915_gem_context_destroy destroy;
4041 int ret;
4042
4043 if (ctx == nullptr)
4044 return;
4045
4046 memclear(destroy);
4047
4048 bufmgr_gem = (struct mos_bufmgr_gem *)ctx->bufmgr;
4049 destroy.ctx_id = ctx->ctx_id;
4050 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_CONTEXT_DESTROY,
4051 &destroy);
4052 if (ret != 0)
4053 fprintf(stderr, "DRM_IOCTL_I915_GEM_CONTEXT_DESTROY failed: %s\n",
4054 strerror(errno));
4055
4056 free(ctx);
4057 }
4058
4059 static int
4060 mos_bufmg_get_reset_stats(struct mos_linux_context *ctx,
4061 uint32_t *reset_count,
4062 uint32_t *active,
4063 uint32_t *pending)
4064 {
4065 struct mos_bufmgr_gem *bufmgr_gem;
4066 struct drm_i915_reset_stats stats;
4067 int ret;
4068
4069 if (ctx == nullptr)
4070 return -EINVAL;
4071
4072 memclear(stats);
4073
4074 bufmgr_gem = (struct mos_bufmgr_gem *)ctx->bufmgr;
4075 stats.ctx_id = ctx->ctx_id;
4076 ret = drmIoctl(bufmgr_gem->fd,
4077 DRM_IOCTL_I915_GET_RESET_STATS,
4078 &stats);
4079 if (ret == 0) {
4080 if (reset_count != nullptr)
4081 *reset_count = stats.reset_count;
4082
4083 if (active != nullptr)
4084 *active = stats.batch_active;
4085
4086 if (pending != nullptr)
4087 *pending = stats.batch_pending;
4088 }
4089
4090 return ret;
4091 }
4092
4093 static unsigned int mos_bufmgr_hweight8(struct mos_linux_context *ctx, uint8_t w)
4094 {
4095 uint32_t i, weight = 0;
4096
4097 for (i=0; i<8; i++)
4098 {
4099 weight += !!((w) & (1UL << i));
4100 }
4101 return weight;
4102 }
4103
4104 static uint8_t mos_bufmgr_switch_off_n_bits(struct mos_linux_context *ctx, uint8_t in_mask, int n)
4105 {
4106 int i,count;
4107 uint8_t bi,out_mask;
4108
4109 assert (n>0 && n<=8);
4110
4111 out_mask = in_mask;
4112 count = n;
4113 for(i=0; i<8; i++)
4114 {
4115 bi = 1UL<<i;
4116 if (bi & in_mask)
4117 {
4118 out_mask &= ~bi;
4119 count--;
4120 }
4121 if (count==0)
4122 {
4123 break;
4124 }
4125 }
4126 return out_mask;
4127 }
4128
4129 static int
4130 mos_bufmgr_get_context_param_sseu(struct mos_linux_context *ctx,
4131 struct drm_i915_gem_context_param_sseu *sseu)
4132 {
4133 struct mos_bufmgr_gem *bufmgr_gem;
4134 struct drm_i915_gem_context_param context_param;
4135 int ret;
4136
4137 if (ctx == nullptr)
4138 return -EINVAL;
4139
4140 bufmgr_gem = (struct mos_bufmgr_gem *)ctx->bufmgr;
4141 memset(&context_param, 0, sizeof(context_param));
4142 context_param.ctx_id = ctx->ctx_id;
4143 context_param.param = I915_CONTEXT_PARAM_SSEU;
4144 context_param.value = (uint64_t) sseu;
4145 context_param.size = sizeof(struct drm_i915_gem_context_param_sseu);
4146
4147 ret = drmIoctl(bufmgr_gem->fd,
4148 DRM_IOCTL_I915_GEM_CONTEXT_GETPARAM,
4149 &context_param);
4150
4151 return ret;
4152 }
4153
4154 static int
4155 mos_bufmgr_set_context_param_sseu(struct mos_linux_context *ctx,
4156 struct drm_i915_gem_context_param_sseu sseu)
4157 {
4158 struct mos_bufmgr_gem *bufmgr_gem;
4159 struct drm_i915_gem_context_param context_param;
4160 int ret;
4161
4162 if (ctx == nullptr)
4163 return -EINVAL;
4164
4165 bufmgr_gem = (struct mos_bufmgr_gem *)ctx->bufmgr;
4166 memset(&context_param, 0, sizeof(context_param));
4167 context_param.ctx_id = ctx->ctx_id;
4168 context_param.param = I915_CONTEXT_PARAM_SSEU;
4169 context_param.value = (uint64_t) &sseu;
4170 context_param.size = sizeof(struct drm_i915_gem_context_param_sseu);
4171
4172 ret = drmIoctl(bufmgr_gem->fd,
4173 DRM_IOCTL_I915_GEM_CONTEXT_SETPARAM,
4174 &context_param);
4175
4176 return ret;
4177 }
4178
4179 static int
4180 mos_gem_get_context_param(struct mos_linux_context *ctx,
4181 uint32_t size,
4182 uint64_t param,
4183 uint64_t *value)
4184 {
4185 struct mos_bufmgr_gem *bufmgr_gem;
4186 struct drm_i915_gem_context_param context_param;
4187 int ret;
4188
4189 if (ctx == nullptr)
4190 return -EINVAL;
4191
4192 bufmgr_gem = (struct mos_bufmgr_gem *)ctx->bufmgr;
4193 context_param.ctx_id = ctx->ctx_id;
4194 context_param.size = size;
4195 context_param.param = param;
4196 context_param.value = 0;
4197
4198 ret = drmIoctl(bufmgr_gem->fd,
4199 DRM_IOCTL_I915_GEM_CONTEXT_GETPARAM,
4200 &context_param);
4201 *value = context_param.value;
4202
4203 return ret;
4204 }
4205
4206 static int
4207 mos_gem_set_context_param(struct mos_linux_context *ctx,
4208 uint32_t size,
4209 uint64_t param,
4210 uint64_t value)
4211 {
4212 struct mos_bufmgr_gem *bufmgr_gem;
4213 struct drm_i915_gem_context_param context_param;
4214 int ret;
4215
4216 if (ctx == nullptr)
4217 return -EINVAL;
4218
4219 bufmgr_gem = (struct mos_bufmgr_gem *)ctx->bufmgr;
4220 context_param.ctx_id = ctx->ctx_id;
4221 context_param.size = size;
4222 context_param.param = param;
4223 context_param.value = value;
4224
4225 ret = drmIoctl(bufmgr_gem->fd,
4226 DRM_IOCTL_I915_GEM_CONTEXT_SETPARAM,
4227 &context_param);
4228
4229 return ret;
4230 }
4231
4232 static int
4233 mos_bufmg_reg_read(struct mos_bufmgr *bufmgr,
4234 uint32_t offset,
4235 uint64_t *result)
4236 {
4237 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bufmgr;
4238 struct drm_i915_reg_read reg_read;
4239 int ret;
4240
4241 memclear(reg_read);
4242 reg_read.offset = offset;
4243
4244 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_REG_READ, &reg_read);
4245
4246 *result = reg_read.val;
4247 return ret;
4248 }
4249
4250 static pthread_mutex_t bufmgr_list_mutex = PTHREAD_MUTEX_INITIALIZER;
4251 static drmMMListHead bufmgr_list = { &bufmgr_list, &bufmgr_list };
4252
4253 static struct mos_bufmgr_gem *
4254 mos_bufmgr_gem_find(int fd)
4255 {
4256 struct mos_bufmgr_gem *bufmgr_gem;
4257
4258 DRMLISTFOREACHENTRY(bufmgr_gem, &bufmgr_list, managers) {
4259 if (bufmgr_gem->fd == fd) {
4260 atomic_inc(&bufmgr_gem->refcount);
4261 return bufmgr_gem;
4262 }
4263 }
4264
4265 return nullptr;
4266 }
4267
4268 static void
4269 mos_bufmgr_gem_unref(struct mos_bufmgr *bufmgr)
4270 {
4271 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bufmgr;
4272
4273 if (atomic_add_unless(&bufmgr_gem->refcount, -1, 1)) {
4274 pthread_mutex_lock(&bufmgr_list_mutex);
4275
4276 if (atomic_dec_and_test(&bufmgr_gem->refcount)) {
4277 DRMLISTDEL(&bufmgr_gem->managers);
4278 mos_bufmgr_gem_destroy(bufmgr);
4279 }
4280
4281 pthread_mutex_unlock(&bufmgr_list_mutex);
4282 }
4283 }
4284
4285 static int
4286 mos_gem_get_memory_info(struct mos_bufmgr *bufmgr, char *info, uint32_t length)
4287 {
4288 #if (_DEBUG || _RELEASE_INTERNAL)
4289 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bufmgr;
4290 assert(bufmgr_gem != nullptr);
4291 assert(info != nullptr);
4292
4293 if (length < MOS_MAX_MSG_BUF_SIZE) {
4294 fprintf(stderr, "Buffer size not enough.\n");
4295 return -1;
4296 }
4297
4298 if (BufmgrPrelim::IsPrelimSupported()) {
4299 return bufmgr_gem->prelim->GetMemoryInfo(bufmgr_gem->has_lmem, info, length);
4300 }
4301 #endif
4302 return 0;
4303 }
4304
4305 static void mos_gem_enable_softpin(struct mos_bufmgr *bufmgr, bool va1m_align)
4306 {
4307 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bufmgr;
4308 bufmgr_gem->use_softpin = true;
4309 bufmgr_gem->softpin_va1Malign = va1m_align;
4310 }
4311
4312 static void mos_gem_enable_vmbind(struct mos_bufmgr *bufmgr)
4313 {
4314 }
4315
4316 static void mos_gem_disable_object_capture(struct mos_bufmgr *bufmgr)
4317 {
4318 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bufmgr;
4319 if (bufmgr_gem != nullptr)
4320 {
4321 bufmgr_gem->object_capture_disabled = true;
4322 }
4323 }
4324
4325 static struct mos_linux_context *
4326 mos_gem_context_create_ext(struct mos_bufmgr *bufmgr, __u32 flags, bool bContextProtected)
4327 {
4328 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bufmgr;
4329 struct drm_i915_gem_context_create_ext create;
4330 struct mos_linux_context *context = nullptr;
4331 struct drm_i915_gem_context_create_ext_setparam p_protected;
4332 struct drm_i915_gem_context_create_ext_setparam p_norecover;
4333 int ret;
4334
4335 context = (struct mos_linux_context *)calloc(1, sizeof(*context));
4336 if (!context)
4337 return nullptr;
4338
4339 memclear(create);
4340 create.flags = flags;
4341 create.extensions = 0;
4342
4343 if (bContextProtected)
4344 {
4345 memclear(p_protected);
4346 memclear(p_norecover);
4347 p_protected.base.next_extension = 0;
4348 p_protected.base.name = I915_CONTEXT_CREATE_EXT_SETPARAM;
4349 p_protected.param.param = I915_CONTEXT_PARAM_PROTECTED_CONTENT;
4350 p_protected.param.value = 1;
4351
4352 p_norecover.base.next_extension = (uintptr_t)&p_protected;
4353 p_norecover.base.name = I915_CONTEXT_CREATE_EXT_SETPARAM;
4354 p_norecover.param.param = I915_CONTEXT_PARAM_RECOVERABLE;
4355 p_norecover.param.value = 0;
4356
4357 create.flags = flags|I915_CONTEXT_CREATE_FLAGS_USE_EXTENSIONS;
4358 create.extensions = (uintptr_t)&p_norecover;
4359 }
4360
4361 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_CONTEXT_CREATE_EXT, &create);
4362 if (ret != 0) {
4363 MOS_DBG("DRM_IOCTL_I915_GEM_CONTEXT_CREATE failed: %s\n",
4364 strerror(errno));
4365 free(context);
4366 return nullptr;
4367 }
4368
4369 context->ctx_id = create.ctx_id;
4370 context->bufmgr = bufmgr;
4371
4372 ret = mos_gem_ctx_set_user_ctx_params(context);
4373
4374 return context;
4375 }
4376
4377 static __u32 mos_gem_vm_create(struct mos_bufmgr *bufmgr)
4378 {
4379 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bufmgr;
4380 struct drm_i915_gem_vm_control *vm = nullptr;
4381 __u32 vm_id;
4382 int ret;
4383
4384 vm = (struct drm_i915_gem_vm_control *)calloc(1, sizeof(struct drm_i915_gem_vm_control));
4385 if (nullptr == vm)
4386 {
4387 MOS_DBG("vm calloc failed\n" );
4388 return INVALID_VM;
4389 }
4390
4391 memset(vm, 0, sizeof(*vm));
4392
4393 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_VM_CREATE, vm);
4394 if (ret != 0) {
4395 MOS_DBG("DRM_IOCTL_I915_GEM_VM_CREATE failed: %s\n",
4396 strerror(errno));
4397 free(vm);
4398 return INVALID_VM;
4399 }
4400
4401 vm_id = vm->vm_id;
4402 free(vm);
4403
4404 return vm_id;
4405 }
4406
4407 static void mos_gem_vm_destroy(struct mos_bufmgr *bufmgr, __u32 vm_id)
4408 {
4409 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bufmgr;
4410 struct drm_i915_gem_vm_control *vm = nullptr;
4411 int ret;
4412
4413 if (vm_id == INVALID_VM)
4414 {
4415 MOS_DBG("input invalid param\n" );
4416 return;
4417 }
4418
4419 vm = (struct drm_i915_gem_vm_control *)calloc(1, sizeof(struct drm_i915_gem_vm_control));
4420
4421 if (nullptr == vm)
4422 {
4423 MOS_DBG("vm calloc failed\n" );
4424 return;
4425 }
4426
4427 vm->vm_id = vm_id;
4428 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_VM_DESTROY, vm);
4429 if (ret != 0) {
4430 MOS_DBG("DRM_IOCTL_I915_GEM_VM_DESTROY failed: %s\n",
4431 strerror(errno));
4432 }
4433
4434 free(vm);
4435 }
4436
4437
4438 static struct mos_linux_context *
4439 mos_gem_context_create_shared(struct mos_bufmgr *bufmgr,
4440 mos_linux_context* ctx,
4441 __u32 flags,
4442 bool bContextProtected,
4443 void *engine_map,
4444 uint8_t ctx_width,
4445 uint8_t num_placements,
4446 uint32_t ctx_type)
4447 {
4448 MOS_UNUSED(engine_map);
4449 MOS_UNUSED(ctx_width);
4450 MOS_UNUSED(num_placements);
4451 MOS_UNUSED(ctx_type);
4452
4453 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bufmgr;
4454 struct drm_i915_gem_context_create_ext create;
4455 struct mos_linux_context *context = nullptr;
4456 struct drm_i915_gem_context_create_ext_setparam p_protected;
4457 struct drm_i915_gem_context_create_ext_setparam p_norecover;
4458 int ret;
4459
4460 if (ctx == nullptr || ctx->vm_id == INVALID_VM)
4461 return nullptr;
4462
4463 if (BufmgrPrelim::IsPrelimSupported()) {
4464 bufmgr_gem->prelim->WaDisableSingleTimeline(bufmgr_gem->has_lmem, flags);
4465 }
4466
4467 context = (struct mos_linux_context *)calloc(1, sizeof(*context));
4468 if (!context)
4469 return nullptr;
4470
4471 memclear(create);
4472 create.flags = flags;
4473 create.extensions = 0;
4474 if (bContextProtected)
4475 {
4476 memclear(p_protected);
4477 memclear(p_norecover);
4478 p_protected.base.next_extension = 0;
4479 p_protected.base.name = I915_CONTEXT_CREATE_EXT_SETPARAM;
4480 p_protected.param.param = I915_CONTEXT_PARAM_PROTECTED_CONTENT;
4481 p_protected.param.value = 1;
4482
4483 p_norecover.base.next_extension = (uintptr_t)&p_protected;
4484 p_norecover.base.name = I915_CONTEXT_CREATE_EXT_SETPARAM;
4485 p_norecover.param.param = I915_CONTEXT_PARAM_RECOVERABLE;
4486 p_norecover.param.value = 0;
4487
4488 create.flags = flags|I915_CONTEXT_CREATE_FLAGS_USE_EXTENSIONS;
4489 create.extensions = (uintptr_t)&p_norecover;
4490 }
4491 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_CONTEXT_CREATE_EXT, &create);
4492 if (ret != 0) {
4493 MOS_DBG("DRM_IOCTL_I915_GEM_CONTEXT_CREATE failed: %s\n",
4494 strerror(errno));
4495 free(context);
4496 return nullptr;
4497 }
4498
4499 context->ctx_id = create.ctx_id;
4500 context->bufmgr = bufmgr;
4501
4502 ret = mos_set_context_param(context,
4503 0,
4504 I915_CONTEXT_PARAM_VM,
4505 ctx->vm_id);
4506 if(ret != 0) {
4507 MOS_DBG("I915_CONTEXT_PARAM_VM failed: %s\n",
4508 strerror(errno));
4509 free(context);
4510 return nullptr;
4511 }
4512
4513 ret = mos_gem_ctx_set_user_ctx_params(context);
4514
4515 return context;
4516 }
4517
4518 static int mos_bufmgr_query_engines_count(struct mos_bufmgr *bufmgr,
4519 unsigned int *nengine)
4520 {
4521 assert(bufmgr);
4522 assert(nengine);
4523 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bufmgr;
4524 if (BufmgrPrelim::IsPrelimSupported()) {
4525 return bufmgr_gem->prelim->QueryEnginesCount(nengine);
4526 }
4527 int fd = ((struct mos_bufmgr_gem*)bufmgr)->fd;
4528 struct drm_i915_query query;
4529 struct drm_i915_query_item query_item;
4530 struct drm_i915_query_engine_info *engines = nullptr;
4531 int ret, len;
4532
4533 memclear(query_item);
4534 query_item.query_id = DRM_I915_QUERY_ENGINE_INFO;
4535 query_item.length = 0;
4536 memclear(query);
4537 query.num_items = 1;
4538 query.items_ptr = (uintptr_t)&query_item;
4539
4540 ret = drmIoctl(fd, DRM_IOCTL_I915_QUERY, &query);
4541 if (ret || query_item.length == 0)
4542 {
4543 *nengine = 0;
4544 return ret;
4545 }
4546
4547 len = query_item.length;
4548
4549 engines = (drm_i915_query_engine_info *)malloc(len);
4550 if (nullptr == engines)
4551 {
4552 *nengine = 0;
4553 ret = -ENOMEM;
4554 return ret;
4555 }
4556
4557 memset(engines, 0, len);
4558 memclear(query_item);
4559 query_item.query_id = DRM_I915_QUERY_ENGINE_INFO;
4560 query_item.length = len;
4561 query_item.data_ptr = (uintptr_t)engines;
4562 memclear(query);
4563 query.num_items = 1;
4564 query.items_ptr = (uintptr_t)&query_item;
4565
4566 ret = drmIoctl(fd, DRM_IOCTL_I915_QUERY, &query);
4567
4568 *nengine = ret ? 0 : engines->num_engines;
4569
4570 if(engines)
4571 {
4572 free(engines);
4573 }
4574 return ret;
4575 }
4576
4577 static int mos_bufmgr_query_engines(struct mos_bufmgr *bufmgr,
4578 __u16 engine_class,
4579 __u64 caps,
4580 unsigned int *nengine,
4581 void *engine_map)
4582 {
4583 struct i915_engine_class_instance *ci = (struct i915_engine_class_instance *)engine_map;
4584 if((bufmgr == nullptr) || (nengine == nullptr) || (ci == nullptr))
4585 {
4586 return -EINVAL;
4587 }
4588
4589 struct drm_i915_query query;
4590 struct drm_i915_query_item query_item;
4591 struct drm_i915_query_engine_info *engines = nullptr;
4592 int ret, len;
4593 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bufmgr;
4594 if (BufmgrPrelim::IsPrelimSupported()) {
4595 return bufmgr_gem->prelim->QueryEngines(bufmgr_gem->has_lmem,
4596 engine_class,
4597 caps,
4598 nengine,
4599 ci);
4600 }
4601
4602 int fd = bufmgr_gem->fd;
4603
4604 memclear(query_item);
4605 query_item.query_id = DRM_I915_QUERY_ENGINE_INFO;
4606 query_item.length = 0;
4607 memclear(query);
4608 query.num_items = 1;
4609 query.items_ptr = (uintptr_t)&query_item;
4610
4611 ret = drmIoctl(fd, DRM_IOCTL_I915_QUERY, &query);
4612 if (ret)
4613 {
4614 goto fini;
4615 }
4616
4617 len = query_item.length;
4618 if(len == 0)
4619 {
4620 goto fini;
4621 }
4622
4623 engines = (drm_i915_query_engine_info *)malloc(len);
4624 if (nullptr == engines)
4625 {
4626 ret = -ENOMEM;
4627 goto fini;
4628 }
4629 memset(engines,0,len);
4630 memclear(query_item);
4631 query_item.query_id = DRM_I915_QUERY_ENGINE_INFO;
4632 query_item.length = len;
4633 query_item.data_ptr = (uintptr_t)engines;
4634 memclear(query);
4635 query.num_items = 1;
4636 query.items_ptr = (uintptr_t)&query_item;
4637
4638 ret = drmIoctl(fd, DRM_IOCTL_I915_QUERY, &query);
4639 if (ret)
4640 {
4641 goto fini;
4642 }
4643
4644 int i, num;
4645 for (i = 0, num = 0; i < engines->num_engines; i++) {
4646 struct drm_i915_engine_info *engine =
4647 (struct drm_i915_engine_info *)&engines->engines[i];
4648 if ( engine_class == engine->engine.engine_class
4649 && ((caps & engine->capabilities) == caps ))
4650 {
4651 ci->engine_class = engine_class;
4652 ci->engine_instance = engine->engine.engine_instance;
4653 ci++;
4654 num++;
4655 }
4656 if (num > *nengine)
4657 {
4658 fprintf(stderr,"%s: Number of engine instances out of range, %d,%d\n",
4659 __FUNCTION__, num, *nengine);
4660 goto fini;
4661 }
4662 }
4663 *nengine = num;
4664
4665
4666 fini:
4667 if (engines)
4668 free(engines);
4669 return ret;
4670 }
4671
4672 static size_t mos_bufmgr_get_engine_class_size()
4673 {
4674 return sizeof(struct i915_engine_class_instance);
4675 }
4676
4677 static int
4678 mos_bufmgr_query_sys_engines(struct mos_bufmgr *bufmgr, MEDIA_SYSTEM_INFO* gfx_info)
4679 {
4680 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bufmgr;
4681 int ret;
4682
4683 if (nullptr == gfx_info)
4684 {
4685 return -EINVAL;
4686 }
4687
4688 unsigned int maxNengine = 0;
4689 if((gfx_info->VDBoxInfo.NumberOfVDBoxEnabled == 0)
4690 || (gfx_info->VEBoxInfo.NumberOfVEBoxEnabled == 0))
4691 {
4692 if (mos_query_engines_count(bufmgr, &maxNengine) || (maxNengine == 0))
4693 {
4694 MOS_OS_ASSERTMESSAGE("Failed to query engines count.\n");
4695 return -ENODEV;
4696 }
4697 }
4698
4699 if (gfx_info->VDBoxInfo.NumberOfVDBoxEnabled == 0)
4700 {
4701 unsigned int nengine = maxNengine;
4702 struct i915_engine_class_instance *uengines = nullptr;
4703 uengines = (struct i915_engine_class_instance *)MOS_AllocAndZeroMemory(nengine * sizeof(struct i915_engine_class_instance));
4704 if (nullptr == uengines)
4705 {
4706 return -ENOMEM;
4707 }
4708 ret = mos_bufmgr_query_engines(bufmgr, I915_ENGINE_CLASS_VIDEO, 0, &nengine, (void *)uengines);
4709 if (ret)
4710 {
4711 MOS_OS_ASSERTMESSAGE("Failed to query vdbox engine\n");
4712 MOS_SafeFreeMemory(uengines);
4713 return -ENODEV;
4714 }
4715 else
4716 {
4717 gfx_info->VDBoxInfo.NumberOfVDBoxEnabled = nengine;
4718 }
4719
4720 for (int i=0; i<nengine; i++)
4721 {
4722 gfx_info->VDBoxInfo.Instances.VDBoxEnableMask |= 1<<uengines[i].engine_instance;
4723 }
4724
4725 MOS_SafeFreeMemory(uengines);
4726 }
4727
4728 if (gfx_info->VEBoxInfo.NumberOfVEBoxEnabled == 0)
4729 {
4730 unsigned int nengine = maxNengine;
4731 struct i915_engine_class_instance *uengines = nullptr;
4732 uengines = (struct i915_engine_class_instance *)MOS_AllocAndZeroMemory(nengine * sizeof(struct i915_engine_class_instance));
4733 if (nullptr == uengines)
4734 {
4735 return -ENOMEM;
4736 }
4737 ret = mos_bufmgr_query_engines(bufmgr, I915_ENGINE_CLASS_VIDEO_ENHANCE, 0, &nengine, (void *)uengines);
4738 if (ret)
4739 {
4740 MOS_OS_ASSERTMESSAGE("Failed to query vebox engine\n");
4741 MOS_SafeFreeMemory(uengines);
4742 return -ENODEV;
4743 }
4744 else
4745 {
4746 MOS_OS_ASSERT(nengine <= maxNengine);
4747 gfx_info->VEBoxInfo.NumberOfVEBoxEnabled = nengine;
4748 }
4749
4750 MOS_SafeFreeMemory(uengines);
4751 }
4752
4753 return 0;
4754 }
4755
4756 void mos_gem_select_fixed_engine(struct mos_bufmgr *bufmgr,
4757 void *engine_map,
4758 uint32_t *nengine,
4759 uint32_t fixed_instance_mask)
4760 {
4761 MOS_UNUSED(bufmgr);
4762 #if (DEBUG || _RELEASE_INTERNAL)
4763 if (fixed_instance_mask)
4764 {
4765 struct i915_engine_class_instance *_engine_map = (struct i915_engine_class_instance *)engine_map;
4766 auto unselect_index = 0;
4767 for(auto bit = 0; bit < *nengine; bit++)
4768 {
4769 if(((fixed_instance_mask >> bit) & 0x1) && (bit > unselect_index))
4770 {
4771 _engine_map[unselect_index].engine_class = _engine_map[bit].engine_class;
4772 _engine_map[unselect_index].engine_instance = _engine_map[bit].engine_instance;
4773 _engine_map[bit].engine_class = 0;
4774 _engine_map[bit].engine_instance = 0;
4775 unselect_index++;
4776 }
4777 else if(((fixed_instance_mask >> bit) & 0x1) && (bit == unselect_index))
4778 {
4779 unselect_index++;
4780 }
4781 else if(!((fixed_instance_mask >> bit) & 0x1))
4782 {
4783 _engine_map[bit].engine_class = 0;
4784 _engine_map[bit].engine_instance = 0;
4785 }
4786 }
4787 *nengine = unselect_index;
4788 }
4789 #else
4790 MOS_UNUSED(engine_map);
4791 MOS_UNUSED(nengine);
4792 MOS_UNUSED(fixed_instance_mask);
4793 #endif
4794
4795 }
4796
4797 static int mos_gem_set_context_param_parallel(struct mos_linux_context *ctx,
4798 struct i915_engine_class_instance *ci,
4799 unsigned int count)
4800 {
4801 if((ctx == nullptr) || (ci == nullptr) || (count <= 0))
4802 {
4803 return -EINVAL;
4804 }
4805
4806 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)ctx->bufmgr;
4807 if (BufmgrPrelim::IsPrelimSupported()) {
4808 return BufmgrPrelim::SetContextParamParallel(
4809 ctx,
4810 ci,
4811 count);
4812 }
4813
4814 int ret = 0;
4815 uint32_t size = 0;
4816 int n;
4817 struct i915_context_engines_parallel_submit* parallel_submit = nullptr;
4818 struct i915_context_param_engines* set_engines = nullptr;
4819
4820 size = sizeof(struct i915_context_engines_parallel_submit) + count * sizeof(*ci);
4821 parallel_submit = (struct i915_context_engines_parallel_submit*)malloc(size);
4822 if(parallel_submit == nullptr)
4823 {
4824 ret = -ENOMEM;
4825 goto fini;
4826 }
4827 memset(parallel_submit, 0, size);
4828 parallel_submit->base.name = I915_CONTEXT_ENGINES_EXT_PARALLEL_SUBMIT;
4829 parallel_submit->engine_index = 0;
4830 parallel_submit->width = count;
4831 parallel_submit->num_siblings = 1;
4832 for(int i = 0; i < count; i++)
4833 {
4834 parallel_submit->engines[i] = ci[i];
4835 }
4836
4837 /* I915_DEFINE_CONTEXT_PARAM_ENGINES */
4838 size = sizeof(struct i915_context_param_engines) + sizeof(struct i915_engine_class_instance);
4839 set_engines = (struct i915_context_param_engines*) malloc(size);
4840 if(set_engines == nullptr)
4841 {
4842 ret = -ENOMEM;
4843 goto fini;
4844 }
4845 set_engines->extensions = (uintptr_t)(parallel_submit);
4846 set_engines->engines[0].engine_class = I915_ENGINE_CLASS_INVALID;
4847 set_engines->engines[0].engine_instance = I915_ENGINE_CLASS_INVALID_NONE;
4848
4849 ret = mos_set_context_param(ctx,
4850 size,
4851 I915_CONTEXT_PARAM_ENGINES,
4852 (uintptr_t)set_engines);
4853 fini:
4854 if (set_engines)
4855 free(set_engines);
4856 if (parallel_submit)
4857 free(parallel_submit);
4858 return ret;
4859 }
4860
4861 static int mos_gem_set_context_param_load_balance(struct mos_linux_context *ctx,
4862 struct i915_engine_class_instance *ci,
4863 unsigned int count)
4864 {
4865 int ret;
4866 uint32_t size;
4867 struct i915_context_engines_load_balance* balancer = nullptr;
4868 struct i915_context_param_engines* set_engines = nullptr;
4869
4870 MOS_OS_CHECK_CONDITION(ci == nullptr, "Invalid (nullptr) Pointer.", EINVAL);
4871 MOS_OS_CHECK_CONDITION(count == 0, "Invalid input parameter. Number of engines must be > 0.", EINVAL);
4872
4873 /* I915_DEFINE_CONTEXT_ENGINES_LOAD_BALANCE */
4874 size = sizeof(struct i915_context_engines_load_balance) + count * sizeof(*ci);
4875 balancer = (struct i915_context_engines_load_balance*)malloc(size);
4876 if (nullptr == balancer)
4877 {
4878 ret = -ENOMEM;
4879 goto fini;
4880 }
4881 memset(balancer, 0, size);
4882 balancer->base.name = I915_CONTEXT_ENGINES_EXT_LOAD_BALANCE;
4883 balancer->num_siblings = count;
4884 memcpy(balancer->engines, ci, count * sizeof(*ci));
4885
4886 /* I915_DEFINE_CONTEXT_PARAM_ENGINES */
4887 size = sizeof(uint64_t) + sizeof(*ci);
4888 set_engines = (struct i915_context_param_engines*) malloc(size);
4889 if (nullptr == set_engines)
4890 {
4891 ret = -ENOMEM;
4892 goto fini;
4893 }
4894 set_engines->extensions = (uintptr_t)(balancer);
4895 set_engines->engines[0].engine_class = I915_ENGINE_CLASS_INVALID;
4896 set_engines->engines[0].engine_instance = I915_ENGINE_CLASS_INVALID_NONE;
4897
4898 ret = mos_set_context_param(ctx,
4899 size,
4900 I915_CONTEXT_PARAM_ENGINES,
4901 (uintptr_t)set_engines);
4902 fini:
4903 if (nullptr != set_engines)
4904 free(set_engines);
4905 if (nullptr != balancer)
4906 free(balancer);
4907 return ret;
4908 }
4909
4910 static int mos_gem_set_context_param_bond(struct mos_linux_context *ctx,
4911 struct i915_engine_class_instance master_ci,
4912 struct i915_engine_class_instance *bond_ci,
4913 unsigned int bond_count)
4914 {
4915 int ret;
4916 uint32_t size;
4917 struct i915_context_engines_load_balance* balancer = nullptr;
4918 struct i915_context_engines_bond *bond = nullptr;
4919 struct i915_context_param_engines* set_engines = nullptr;
4920
4921 assert(bond_ci);
4922
4923 /* I915_DEFINE_CONTEXT_ENGINES_LOAD_BALANCE */
4924 size = sizeof(struct i915_context_engines_load_balance) + bond_count * sizeof(bond_ci);
4925 balancer = (struct i915_context_engines_load_balance*)malloc(size);
4926 if (nullptr == balancer)
4927 {
4928 ret = -ENOMEM;
4929 goto fini;
4930 }
4931 memset(balancer, 0, size);
4932 balancer->base.name = I915_CONTEXT_ENGINES_EXT_LOAD_BALANCE;
4933 balancer->num_siblings = bond_count;
4934 memcpy(balancer->engines, bond_ci, bond_count * sizeof(*bond_ci));
4935
4936 /* I915_DEFINE_CONTEXT_ENGINES_BOND */
4937 size = sizeof(struct i915_context_engines_bond) + bond_count * sizeof(*bond_ci);
4938 bond = (struct i915_context_engines_bond*)malloc(size);
4939 if (nullptr == bond)
4940 {
4941 ret = -ENOMEM;
4942 goto fini;
4943 }
4944 memset(bond, 0, size);
4945 bond->base.name = I915_CONTEXT_ENGINES_EXT_BOND;
4946 bond->master = master_ci;
4947 bond->num_bonds = bond_count;
4948 memcpy(bond->engines, bond_ci, bond_count * sizeof(*bond_ci));
4949
4950 /* I915_DEFINE_CONTEXT_PARAM_ENGINES */
4951 size = sizeof(uint64_t) + sizeof(struct i915_engine_class_instance);
4952 set_engines = (struct i915_context_param_engines*) malloc(size);
4953 if (nullptr == set_engines)
4954 {
4955 ret = -ENOMEM;
4956 goto fini;
4957 }
4958 set_engines->extensions = (uintptr_t)(balancer);
4959 balancer->base.next_extension = (uintptr_t)(bond);
4960 set_engines->engines[0].engine_class = I915_ENGINE_CLASS_INVALID;
4961 set_engines->engines[0].engine_instance = I915_ENGINE_CLASS_INVALID_NONE;
4962
4963 ret = mos_set_context_param(ctx,
4964 size,
4965 I915_CONTEXT_PARAM_ENGINES,
4966 (uintptr_t)set_engines);
4967 fini:
4968 if (nullptr != set_engines)
4969 free(set_engines);
4970 if (nullptr != bond)
4971 free(bond);
4972 if (nullptr != balancer)
4973 free(balancer);
4974 return ret;
4975 }
4976
4977 static bool mos_gem_bo_is_softpin(struct mos_linux_bo *bo)
4978 {
4979 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
4980 if (bo_gem == nullptr)
4981 {
4982 return false;
4983 }
4984
4985 return bo_gem->is_softpin;
4986 }
4987
4988 static bool
4989 mos_gem_bo_is_exec_object_async(struct mos_linux_bo *bo)
4990 {
4991 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *)bo;
4992
4993 return bo_gem->exec_async;
4994 }
4995
4996 static int mos_bufmgr_query_device_blob(struct mos_bufmgr *bufmgr, MEDIA_SYSTEM_INFO* gfx_info)
4997 {
4998 if ((bufmgr == nullptr) || (gfx_info == nullptr))
4999 {
5000 return -EINVAL;
5001 }
5002
5003 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem*)bufmgr;
5004 if (bufmgr_gem == nullptr)
5005 {
5006 return -EINVAL;
5007 }
5008 int fd = bufmgr_gem->fd;
5009 uint32_t *hw_info = nullptr;
5010 uint32_t ulength= 0;
5011 int ret, i;
5012 struct drm_i915_query_item query_item;
5013
5014 memclear(query_item);
5015 query_item.length = 0;
5016 query_item.query_id = DRM_I915_QUERY_HWCONFIG_BLOB;
5017 ret = mos_gem_query_items(fd, &query_item, 1);
5018 if (ret != 0 || query_item.length <= 0)
5019 {
5020 mos_safe_free(hw_info);
5021 return (ret != 0) ? ret : -1;
5022 }
5023
5024 hw_info = (uint32_t*) malloc(query_item.length);
5025 if (hw_info != nullptr)
5026 {
5027 memset(hw_info, 0, query_item.length);
5028 }
5029 else
5030 {
5031 mos_safe_free(hw_info);
5032 return -ENOMEM;
5033 }
5034 query_item.data_ptr = (uintptr_t) hw_info;
5035 ret = mos_gem_query_items(fd, &query_item, 1);
5036 if (ret != 0 || query_item.length <= 0)
5037 {
5038 mos_safe_free(hw_info);
5039 return (ret != 0) ? ret : -1;
5040 }
5041 ulength = query_item.length / sizeof(uint32_t);
5042 i = 0;
5043 while (i < ulength) {
5044 /* Attribute ID starts with 1 */
5045 if (hw_info[i] <= 0)
5046 {
5047 mos_safe_free(hw_info);
5048 return -EINVAL;
5049 }
5050
5051 #if DEBUG_BLOB_QUERY
5052 fprintf("%s: %d\n", key_string[hw_info[i]], hw_info[i+2]);
5053 #endif
5054 if (INTEL_HWCONFIG_MAX_SLICES_SUPPORTED == hw_info[i])
5055 {
5056 if (hw_info[i+1] != 1)
5057 {
5058 mos_safe_free(hw_info);
5059 return -EINVAL;
5060 }
5061 gfx_info->SliceCount = hw_info[i+2];
5062 gfx_info->MaxSlicesSupported = hw_info[i+2];
5063 }
5064
5065 if (INTEL_HWCONFIG_MAX_DUAL_SUBSLICES_SUPPORTED == hw_info[i])
5066 {
5067 if (hw_info[i+1] != 1)
5068 {
5069 mos_safe_free(hw_info);
5070 return -EINVAL;
5071 }
5072 gfx_info->SubSliceCount = hw_info[i+2];
5073 gfx_info->MaxSubSlicesSupported = hw_info[i+2];
5074 }
5075
5076 if (INTEL_HWCONFIG_MAX_NUM_EU_PER_DSS == hw_info[i])
5077 {
5078 if (hw_info[i+1] != 1)
5079 {
5080 mos_safe_free(hw_info);
5081 return -EINVAL;
5082 }
5083 gfx_info->MaxEuPerSubSlice = hw_info[i+2];
5084 }
5085
5086 if (INTEL_HWCONFIG_DEPRECATED_L3_CACHE_SIZE_IN_KB == hw_info[i])
5087 {
5088 if (hw_info[i+1] != 1)
5089 {
5090 mos_safe_free(hw_info);
5091 return -EINVAL;
5092 }
5093 gfx_info->L3CacheSizeInKb = hw_info[i+2];
5094 }
5095
5096 if (INTEL_HWCONFIG_NUM_THREADS_PER_EU == hw_info[i])
5097 {
5098 if (hw_info[i+1] != 1)
5099 {
5100 mos_safe_free(hw_info);
5101 return -EINVAL;
5102 }
5103 gfx_info->NumThreadsPerEu = hw_info[i+2];
5104 }
5105
5106 if (INTEL_HWCONFIG_MAX_VECS == hw_info[i])
5107 {
5108 if (hw_info[i+1] != 1)
5109 {
5110 mos_safe_free(hw_info);
5111 return -EINVAL;
5112 }
5113 gfx_info->MaxVECS = hw_info[i+2];
5114 }
5115
5116 /* Advance to next key */
5117 i += hw_info[i + 1]; // value size
5118 i += 2;// KL size
5119 }
5120
5121 if (hw_info != nullptr)
5122 {
5123 mos_safe_free(hw_info);
5124 hw_info = nullptr;
5125 }
5126
5127 return ret;
5128 }
5129
5130 static int mos_bufmgr_query_hw_ip_version(struct mos_bufmgr *bufmgr, __u16 engine_class, void *ip_ver_info)
5131 {
5132 return -1;
5133 }
5134
5135 static uint64_t mos_bufmgr_get_platform_information(struct mos_bufmgr *bufmgr)
5136 {
5137 assert(bufmgr);
5138 return bufmgr->platform_information;
5139 }
5140
5141 static void mos_bufmgr_set_platform_information(struct mos_bufmgr *bufmgr, uint64_t p)
5142 {
5143 assert(bufmgr);
5144 bufmgr->platform_information |= p;
5145 }
5146
5147 static int
5148 mos_bufmgr_get_ts_frequency(struct mos_bufmgr *bufmgr, uint32_t *ts_freq)
5149 {
5150 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem*)bufmgr;
5151
5152 *ts_freq = bufmgr_gem->ts_freq;
5153
5154 return 0;
5155 }
5156
5157 static bool
5158 mos_bufmgr_has_bsd2(struct mos_bufmgr *bufmgr)
5159 {
5160 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem*)bufmgr;
5161
5162 return bufmgr_gem->has_bsd2;
5163 }
5164
5165 #define I915_CONTEXT_PRIVATE_PARAM_BOOST 0x80000000
5166 void
5167 mos_bufmgr_enable_turbo_boost(struct mos_bufmgr *bufmgr)
5168 {
5169 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem*)bufmgr;
5170 struct drm_i915_gem_context_param ctxParam;
5171 int32_t retVal = 0;
5172
5173 MOS_ZeroMemory( &ctxParam, sizeof( ctxParam ) );
5174 ctxParam.param = I915_CONTEXT_PRIVATE_PARAM_BOOST;
5175 ctxParam.value = 1;
5176 retVal = drmIoctl(bufmgr_gem->fd,
5177 DRM_IOCTL_I915_GEM_CONTEXT_SETPARAM, &ctxParam );
5178 }
5179
5180 /**
5181 * Initializes the GEM buffer manager, which uses the kernel to allocate, map,
5182 * and manage map buffer objections.
5183 *
5184 * \param fd File descriptor of the opened DRM device.
5185 */
5186 static struct mos_bufmgr *
5187 mos_bufmgr_gem_init_i915(int fd, int batch_size)
5188 {
5189 struct mos_bufmgr_gem *bufmgr_gem;
5190 struct drm_i915_gem_get_aperture aperture;
5191 drm_i915_getparam_t gp;
5192 int ret, tmp;
5193 bool exec2 = false;
5194
5195 pthread_mutex_lock(&bufmgr_list_mutex);
5196
5197 bufmgr_gem = mos_bufmgr_gem_find(fd);
5198 if (bufmgr_gem)
5199 goto exit;
5200
5201 bufmgr_gem = (struct mos_bufmgr_gem *)calloc(1, sizeof(*bufmgr_gem));
5202 if (bufmgr_gem == nullptr)
5203 goto exit;
5204
5205 bufmgr_gem->fd = fd;
5206 atomic_set(&bufmgr_gem->refcount, 1);
5207
5208 if (pthread_mutex_init(&bufmgr_gem->lock, nullptr) != 0) {
5209 free(bufmgr_gem);
5210 bufmgr_gem = nullptr;
5211 goto exit;
5212 }
5213
5214 bufmgr_gem->bufmgr.bo_alloc = mos_gem_bo_alloc;
5215 bufmgr_gem->bufmgr.bo_alloc_tiled = mos_gem_bo_alloc_tiled;
5216 bufmgr_gem->bufmgr.bo_reference = mos_gem_bo_reference;
5217 bufmgr_gem->bufmgr.bo_unreference = mos_gem_bo_unreference;
5218 bufmgr_gem->bufmgr.bo_map = mos_gem_bo_map;
5219 bufmgr_gem->bufmgr.bo_unmap = mos_gem_bo_unmap;
5220 bufmgr_gem->bufmgr.bo_wait_rendering = mos_gem_bo_wait_rendering;
5221 bufmgr_gem->bufmgr.bo_pad_to_size = mos_gem_bo_pad_to_size;
5222 bufmgr_gem->bufmgr.bo_emit_reloc = mos_gem_bo_emit_reloc;
5223 bufmgr_gem->bufmgr.bo_get_tiling = mos_gem_bo_get_tiling;
5224 bufmgr_gem->bufmgr.bo_set_tiling = mos_gem_bo_set_tiling;
5225 bufmgr_gem->bufmgr.bo_flink = mos_gem_bo_flink;
5226 bufmgr_gem->bufmgr.bo_exec = mos_gem_bo_exec2;
5227 bufmgr_gem->bufmgr.bo_mrb_exec = mos_gem_bo_mrb_exec2;
5228 bufmgr_gem->bufmgr.bo_busy = mos_gem_bo_busy;
5229 bufmgr_gem->bufmgr.bo_madvise = mos_gem_bo_madvise;
5230 bufmgr_gem->bufmgr.destroy = mos_bufmgr_gem_unref;
5231 bufmgr_gem->bufmgr.debug = 0;
5232 bufmgr_gem->bufmgr.check_aperture_space =
5233 mos_gem_check_aperture_space;
5234 bufmgr_gem->bufmgr.bo_disable_reuse = mos_gem_bo_disable_reuse;
5235 bufmgr_gem->bufmgr.bo_is_reusable = mos_gem_bo_is_reusable;
5236 bufmgr_gem->bufmgr.bo_references = mos_gem_bo_references;
5237
5238 bufmgr_gem->bufmgr.bo_wait = mos_gem_bo_wait;
5239 bufmgr_gem->bufmgr.bo_clear_relocs = mos_gem_bo_clear_relocs;
5240 bufmgr_gem->bufmgr.context_create = mos_gem_context_create;
5241 bufmgr_gem->bufmgr.context_create_ext = mos_gem_context_create_ext;
5242 bufmgr_gem->bufmgr.context_create_shared = mos_gem_context_create_shared;
5243 bufmgr_gem->bufmgr.context_destroy = mos_gem_context_destroy;
5244 bufmgr_gem->bufmgr.vm_create = mos_gem_vm_create;
5245 bufmgr_gem->bufmgr.vm_destroy = mos_gem_vm_destroy;
5246 bufmgr_gem->bufmgr.bo_context_exec2 = mos_gem_bo_context_exec2;
5247 bufmgr_gem->bufmgr.bo_context_exec3 = mos_gem_bo_context_exec3;
5248 bufmgr_gem->bufmgr.bo_is_exec_object_async = mos_gem_bo_is_exec_object_async;
5249 bufmgr_gem->bufmgr.bo_is_softpin = mos_gem_bo_is_softpin;
5250 bufmgr_gem->bufmgr.bo_map_gtt = mos_gem_bo_map_gtt;
5251 bufmgr_gem->bufmgr.bo_unmap_gtt = mos_gem_bo_unmap_gtt;
5252 bufmgr_gem->bufmgr.bo_map_wc = mos_gem_bo_map_wc;
5253 bufmgr_gem->bufmgr.bo_unmap_wc = mos_gem_bo_unmap_wc;
5254 bufmgr_gem->bufmgr.bo_map_unsynchronized = mos_gem_bo_map_unsynchronized;
5255 bufmgr_gem->bufmgr.bo_start_gtt_access = mos_gem_bo_start_gtt_access;
5256 bufmgr_gem->bufmgr.bo_get_softpin_targets_info = mos_bufmgr_bo_get_softpin_targets_info;
5257 bufmgr_gem->bufmgr.bo_create_from_name = mos_bufmgr_bo_gem_create_from_name;
5258 bufmgr_gem->bufmgr.enable_reuse = mos_gem_enable_reuse;
5259 bufmgr_gem->bufmgr.enable_softpin = mos_gem_enable_softpin;
5260 bufmgr_gem->bufmgr.enable_vmbind = mos_gem_enable_vmbind;
5261 bufmgr_gem->bufmgr.disable_object_capture = mos_gem_disable_object_capture;
5262 bufmgr_gem->bufmgr.get_memory_info = mos_gem_get_memory_info;
5263 bufmgr_gem->bufmgr.get_devid = mos_gem_get_devid;
5264 bufmgr_gem->bufmgr.set_context_param = mos_gem_set_context_param;
5265 bufmgr_gem->bufmgr.set_context_param_parallel = mos_gem_set_context_param_parallel;
5266 bufmgr_gem->bufmgr.set_context_param_load_balance = mos_gem_set_context_param_load_balance;
5267 bufmgr_gem->bufmgr.set_context_param_bond = mos_gem_set_context_param_bond;
5268 bufmgr_gem->bufmgr.get_context_param = mos_gem_get_context_param;
5269 bufmgr_gem->bufmgr.bo_create_from_prime = mos_gem_bo_create_from_prime;
5270 bufmgr_gem->bufmgr.bo_export_to_prime = mos_gem_bo_export_to_prime;
5271 bufmgr_gem->bufmgr.reg_read = mos_bufmg_reg_read;
5272 bufmgr_gem->bufmgr.get_reset_stats = mos_bufmg_get_reset_stats;
5273 bufmgr_gem->bufmgr.get_context_param_sseu = mos_bufmgr_get_context_param_sseu;
5274 bufmgr_gem->bufmgr.set_context_param_sseu = mos_bufmgr_set_context_param_sseu;
5275 bufmgr_gem->bufmgr.query_sys_engines = mos_bufmgr_query_sys_engines;
5276 bufmgr_gem->bufmgr.query_device_blob = mos_bufmgr_query_device_blob;
5277 bufmgr_gem->bufmgr.get_driver_info = mos_bufmgr_get_driver_info;
5278 bufmgr_gem->bufmgr.query_hw_ip_version = mos_bufmgr_query_hw_ip_version;
5279 bufmgr_gem->bufmgr.get_platform_information = mos_bufmgr_get_platform_information;
5280 bufmgr_gem->bufmgr.set_platform_information = mos_bufmgr_set_platform_information;
5281 bufmgr_gem->bufmgr.query_engines_count = mos_bufmgr_query_engines_count;
5282 bufmgr_gem->bufmgr.query_engines = mos_bufmgr_query_engines;
5283 bufmgr_gem->bufmgr.get_engine_class_size = mos_bufmgr_get_engine_class_size;
5284 bufmgr_gem->bufmgr.select_fixed_engine = mos_gem_select_fixed_engine;
5285 bufmgr_gem->bufmgr.switch_off_n_bits = mos_bufmgr_switch_off_n_bits;
5286 bufmgr_gem->bufmgr.hweight8 = mos_bufmgr_hweight8;
5287 bufmgr_gem->bufmgr.get_ts_frequency = mos_bufmgr_get_ts_frequency;
5288 bufmgr_gem->bufmgr.has_bsd2 = mos_bufmgr_has_bsd2;
5289 bufmgr_gem->bufmgr.enable_turbo_boost = mos_bufmgr_enable_turbo_boost;
5290
5291 bufmgr_gem->mem_profiler_path = getenv("MEDIA_MEMORY_PROFILER_LOG");
5292 if (bufmgr_gem->mem_profiler_path != nullptr)
5293 {
5294 if (strcmp(bufmgr_gem->mem_profiler_path, "/sys/kernel/debug/tracing/trace_marker") == 0)
5295 {
5296 ret = bufmgr_gem->mem_profiler_fd = open(bufmgr_gem->mem_profiler_path, O_WRONLY );
5297 }
5298 else
5299 {
5300 ret = bufmgr_gem->mem_profiler_fd = open(bufmgr_gem->mem_profiler_path, O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR);
5301 }
5302
5303 if (ret == -1)
5304 {
5305 MOS_DBG("Failed to open %s: %s\n", bufmgr_gem->mem_profiler_path, strerror(errno));
5306 }
5307 }
5308 else
5309 {
5310 bufmgr_gem->mem_profiler_fd = -1;
5311 }
5312
5313 memclear(aperture);
5314 ret = drmIoctl(bufmgr_gem->fd,
5315 DRM_IOCTL_I915_GEM_GET_APERTURE,
5316 &aperture);
5317
5318 if (ret == 0)
5319 bufmgr_gem->gtt_size = aperture.aper_available_size;
5320 else {
5321 fprintf(stderr, "DRM_IOCTL_I915_GEM_APERTURE failed: %s\n",
5322 strerror(errno));
5323 bufmgr_gem->gtt_size = 128 * 1024 * 1024;
5324 fprintf(stderr, "Assuming %dkB available aperture size.\n"
5325 "May lead to reduced performance or incorrect "
5326 "rendering.\n",
5327 (int)bufmgr_gem->gtt_size / 1024);
5328 }
5329
5330 /* support Gen 8+ */
5331 bufmgr_gem->pci_device = get_pci_device_id(bufmgr_gem);
5332
5333 if (bufmgr_gem->pci_device == 0) {
5334 pthread_mutex_destroy(&bufmgr_gem->lock);
5335 if (bufmgr_gem->mem_profiler_fd != -1)
5336 {
5337 close(bufmgr_gem->mem_profiler_fd);
5338 }
5339 free(bufmgr_gem);
5340 bufmgr_gem = nullptr;
5341 goto exit;
5342 }
5343
5344 memclear(gp);
5345 gp.value = &tmp;
5346
5347 gp.param = I915_PARAM_HAS_EXECBUF2;
5348 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
5349 if (!ret)
5350 exec2 = true;
5351
5352 gp.param = I915_PARAM_HAS_BSD;
5353 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
5354 bufmgr_gem->has_bsd = ret == 0;
5355
5356 gp.param = I915_PARAM_HAS_BSD2;
5357 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
5358 bufmgr_gem->has_bsd2 = (ret == 0 && *gp.value != 0);
5359
5360 gp.param = I915_PARAM_HAS_BLT;
5361 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
5362 bufmgr_gem->has_blt = ret == 0;
5363
5364 gp.param = I915_PARAM_HAS_RELAXED_FENCING;
5365 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
5366 bufmgr_gem->has_relaxed_fencing = ret == 0;
5367
5368 bufmgr_gem->bufmgr.bo_alloc_userptr = check_bo_alloc_userptr;
5369
5370 gp.param = I915_PARAM_HAS_WAIT_TIMEOUT;
5371 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
5372 bufmgr_gem->has_wait_timeout = ret == 0;
5373
5374 gp.param = I915_PARAM_HAS_LLC;
5375 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
5376 bufmgr_gem->has_llc = *gp.value;
5377
5378 gp.param = I915_PARAM_HAS_VEBOX;
5379 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
5380 bufmgr_gem->has_vebox = (ret == 0) & (*gp.value > 0);
5381
5382 gp.param = I915_PARAM_MMAP_VERSION;
5383 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
5384 bufmgr_gem->has_ext_mmap = (ret == 0) & (*gp.value > 0);
5385
5386 gp.param = I915_PARAM_NUM_FENCES_AVAIL;
5387 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
5388 bufmgr_gem->has_fence_reg = (ret == 0) & (*gp.value > 0);
5389
5390 gp.param = I915_PARAM_HAS_EXEC_SOFTPIN;
5391 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
5392 if (ret == 0 && *gp.value > 0) {
5393 bufmgr_gem->bufmgr.bo_set_softpin = mos_gem_bo_set_softpin;
5394 bufmgr_gem->bufmgr.bo_add_softpin_target = mos_gem_bo_add_softpin_target;
5395 }
5396
5397 gp.param = I915_PARAM_HAS_EXEC_ASYNC;
5398 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
5399 if (ret == 0 && *gp.value > 0) {
5400 bufmgr_gem->bufmgr.set_object_async = mos_gem_bo_set_object_async;
5401 bufmgr_gem->bufmgr.set_exec_object_async = mos_gem_bo_set_exec_object_async;
5402 }
5403
5404 gp.param = I915_PARAM_HAS_EXEC_CAPTURE;
5405 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
5406 if (ret == 0 && *gp.value > 0)
5407 {
5408 bufmgr_gem->bufmgr.set_object_capture = mos_gem_bo_set_object_capture;
5409 }
5410 else
5411 {
5412 bufmgr_gem->object_capture_disabled = true;
5413 }
5414
5415 gp.param = I915_PARAM_MMAP_GTT_VERSION;
5416 ret = drmIoctl(fd, DRM_IOCTL_I915_GETPARAM, &gp);
5417 bufmgr_gem->has_mmap_offset = (ret == 0) && (*gp.value >= 4);
5418
5419 gp.param = I915_PARAM_CS_TIMESTAMP_FREQUENCY;
5420 ret = drmIoctl(fd, DRM_IOCTL_I915_GETPARAM, &gp);
5421 bufmgr_gem->ts_freq = (ret == 0) ? *gp.value : 0;
5422
5423 struct drm_i915_gem_context_param context_param;
5424 memset(&context_param, 0, sizeof(context_param));
5425 context_param.param = I915_CONTEXT_PARAM_GTT_SIZE;
5426 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_CONTEXT_GETPARAM, &context_param);
5427 if (ret == 0){
5428 uint64_t gtt_size_4g = (uint64_t)1 << 32;
5429 if (context_param.value > gtt_size_4g)
5430 {
5431 bufmgr_gem->bufmgr.bo_use_48b_address_range = mos_gem_bo_use_48b_address_range;
5432 }
5433 }
5434
5435 bufmgr_gem->prelim = BufmgrPrelim::CreatePrelim(bufmgr_gem->fd);
5436
5437 bufmgr_gem->has_lmem = mos_gem_has_lmem(bufmgr_gem);
5438
5439 bufmgr_gem->device_type = DEVICE_TYPE_I915;
5440
5441 if (nullptr != bufmgr_gem->prelim) {
5442 bufmgr_gem->prelim->Init(bufmgr_gem->bufmgr, bufmgr_gem->has_lmem);
5443 }
5444
5445 if (!BufmgrPrelim::IsPrelimSupported()) {
5446 bufmgr_gem->bufmgr.has_full_vd = true;
5447 }
5448
5449 /* Let's go with one relocation per every 2 dwords (but round down a bit
5450 * since a power of two will mean an extra page allocation for the reloc
5451 * buffer).
5452 *
5453 * Every 4 was too few for the blender benchmark.
5454 */
5455 batch_size &= 0xffffff00;
5456 bufmgr_gem->max_relocs = batch_size / sizeof(uint32_t) / 2 - 2;
5457
5458 DRMINITLISTHEAD(&bufmgr_gem->named);
5459 init_cache_buckets(bufmgr_gem);
5460
5461 DRMLISTADD(&bufmgr_gem->managers, &bufmgr_list);
5462
5463 bufmgr_gem->use_softpin = false;
5464 mos_vma_heap_init(&bufmgr_gem->vma_heap[MEMZONE_SYS], MEMZONE_SYS_START, MEMZONE_SYS_SIZE);
5465 mos_vma_heap_init(&bufmgr_gem->vma_heap[MEMZONE_DEVICE], MEMZONE_DEVICE_START, MEMZONE_DEVICE_SIZE);
5466 if (BufmgrPrelim::IsPrelimSupported()) {
5467 bufmgr_gem->prelim->InitVmaHeap(&bufmgr_gem->vma_heap[MEMZONE_PRIME]);
5468 }
5469
5470 exit:
5471 pthread_mutex_unlock(&bufmgr_list_mutex);
5472
5473 return bufmgr_gem != nullptr ? &bufmgr_gem->bufmgr : nullptr;
5474 }
5475
5476 struct mos_bufmgr *
5477 mos_bufmgr_gem_init(int fd, int batch_size, int *device_type)
5478 {
5479 int type = mos_query_device_type(fd);
5480 if (device_type != nullptr)
5481 {
5482 *device_type = type;
5483 }
5484 if(DEVICE_TYPE_I915 == type)
5485 {
5486 return mos_bufmgr_gem_init_i915(fd, batch_size);
5487 }
5488 else if (DEVICE_TYPE_XE == type)
5489 {
5490 return mos_bufmgr_gem_init_xe(fd, batch_size);
5491 }
5492
5493 return nullptr;
5494 }
5495
5496 int mos_get_param(int fd, int32_t param, uint32_t *param_value)
5497 {
5498 if((fd < 0) || (param_value == nullptr))
5499 {
5500 return -EINVAL;
5501 }
5502
5503 struct drm_i915_getparam gp;
5504 gp.param = param;
5505 gp.value = (int32_t *)param_value;
5506
5507 return drmIoctl(fd, DRM_IOCTL_I915_GETPARAM, &gp) == 0;
5508 }
5509
5510 static int mos_bufmgr_get_driver_info(struct mos_bufmgr *bufmgr, struct LinuxDriverInfo *drvInfo)
5511 {
5512 if (bufmgr == nullptr || drvInfo == nullptr)
5513 {
5514 return -EINVAL;
5515 }
5516 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem*)bufmgr;
5517 int fd = bufmgr_gem->fd;
5518 if (fd < 0)
5519 {
5520 return -EINVAL;
5521 }
5522 uint32_t retValue = 0;
5523
5524 drvInfo->hasBsd = 0;
5525 if (mos_get_param(fd, I915_PARAM_HAS_BSD, &retValue))
5526 {
5527 drvInfo->hasBsd = !!retValue;
5528 }
5529
5530 drvInfo->hasBsd2 = 0;
5531 retValue = 0;
5532 if (mos_get_param(fd, I915_PARAM_HAS_BSD2, &retValue))
5533 {
5534 drvInfo->hasBsd2 = !!retValue;
5535 }
5536
5537 drvInfo->hasVebox = 0;
5538 retValue = 0;
5539 if (mos_get_param(fd, I915_PARAM_HAS_VEBOX, &retValue))
5540 {
5541 drvInfo->hasVebox = !!retValue;
5542 }
5543
5544 drvInfo->hasPpgtt = 1;
5545 retValue = 0;
5546 if (mos_get_param(fd, I915_PARAM_HAS_ALIASING_PPGTT, &retValue))
5547 {
5548 drvInfo->hasPpgtt = !!retValue;
5549 }
5550
5551 drvInfo->hasHuc = 0;
5552 retValue = 0;
5553 if (mos_get_param(fd, I915_PARAM_HUC_STATUS, &retValue))
5554 {
5555 drvInfo->hasHuc = !!retValue;
5556 if (retValue == 1)
5557 {
5558 drvInfo->hasProtectedHuc = 1;
5559 }
5560 }
5561
5562 drvInfo->devId = 0;
5563 retValue = 0;
5564 if (mos_get_param(fd, I915_PARAM_CHIPSET_ID, &retValue))
5565 {
5566 drvInfo->devId = retValue;
5567 }
5568 drvInfo->devRev = 0;
5569 retValue = 0;
5570 if (mos_get_param(fd, I915_PARAM_REVISION, &retValue))
5571 {
5572 drvInfo->devRev = retValue;
5573 }
5574
5575 drvInfo->euCount = 0;
5576 retValue = 0;
5577 if (mos_get_param(fd, I915_PARAM_EU_TOTAL, &retValue))
5578 {
5579 drvInfo->euCount = retValue;
5580 }
5581
5582 drvInfo->subSliceCount = 0;
5583 retValue = 0;
5584 if (mos_get_param(fd, I915_PARAM_SUBSLICE_TOTAL, &retValue))
5585 {
5586 drvInfo->subSliceCount = retValue;
5587 }
5588
5589 // There is no interface to read total slice count from drm/i915, so we
5590 // will set the slice count in InitMediaSysInfo accordint to Device ID.
5591 drvInfo->sliceCount = 0;
5592
5593 return 0;
5594 }
5595 static int mos_get_dev_id_i915(int fd, uint32_t *device_id)
5596 {
5597 if (nullptr == device_id)
5598 {
5599 return -EINVAL;
5600 }
5601 uint32_t retValue = 0;
5602 if (mos_get_param(fd, I915_PARAM_CHIPSET_ID, &retValue))
5603 {
5604 *device_id = retValue;
5605 }
5606
5607 return 0;
5608 }
5609
5610 int mos_get_device_id(int fd, uint32_t *deviceId)
5611 {
5612 int device_type = mos_query_device_type(fd);
5613
5614 if (DEVICE_TYPE_I915 == device_type)
5615 {
5616 return mos_get_dev_id_i915(fd, deviceId);
5617 }
5618 else if (DEVICE_TYPE_XE == device_type)
5619 {
5620 return mos_get_dev_id_xe(fd, deviceId);
5621 }
5622
5623 return -ENODEV;
5624 }