Debian Codegraph @

media_softlet/linux/common/os/i915/mos_bufmgr.c

raw
1 /**************************************************************************
2 *
3 * Copyright © 2007 Red Hat Inc.
4 * Copyright © 2007-2022 Intel Corporation
5 * Copyright 2006 Tungsten Graphics, Inc., Bismarck, ND., USA
6 * All Rights Reserved.
7 *
8 * Permission is hereby granted, free of charge, to any person obtaining a
9 * copy of this software and associated documentation files (the
10 * "Software"), to deal in the Software without restriction, including
11 * without limitation the rights to use, copy, modify, merge, publish,
12 * distribute, sub license, and/or sell copies of the Software, and to
13 * permit persons to whom the Software is furnished to do so, subject to
14 * the following conditions:
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
20 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
21 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
22 * USE OR OTHER DEALINGS IN THE SOFTWARE.
23 *
24 * The above copyright notice and this permission notice (including the
25 * next paragraph) shall be included in all copies or substantial portions
26 * of the Software.
27 *
28 *
29 **************************************************************************/
30 /*
31 * Authors: Thomas Hellström <thomas-at-tungstengraphics-dot-com>
32 * Keith Whitwell <keithw-at-tungstengraphics-dot-com>
33 * Eric Anholt <[email protected]>
34 * Dave Airlie <[email protected]>
35 */
36
37 #ifdef HAVE_CONFIG_H
38 #include "config.h"
39 #endif
40
41 #include "xf86drm.h"
42 #include "xf86atomic.h"
43 #include <fcntl.h>
44 #include <stdio.h>
45 #include <stdlib.h>
46 #include <string.h>
47 #include <unistd.h>
48 #include <assert.h>
49 #include <pthread.h>
50 #include <sys/ioctl.h>
51 #include <sys/stat.h>
52 #include <sys/types.h>
53 #include <stdbool.h>
54
55 #include "errno.h"
56 #ifndef ETIME
57 #define ETIME ETIMEDOUT
58 #endif
59 #include "libdrm_macros.h"
60 #include "libdrm_lists.h"
61 #include "mos_bufmgr.h"
62 #include "mos_bufmgr_priv.h"
63 #include "mos_bufmgr_xe.h"
64 #include "string.h"
65
66 #include "i915_drm.h"
67 #include "mos_vma.h"
68 #include "mos_util_debug.h"
69 #include "mos_oca_defs_specific.h"
70 #include "intel_hwconfig_types.h"
71 #include "mos_utilities.h"
72 #include "linux_system_info.h"
73 #include "mos_os_specific.h"
74
75 #ifdef HAVE_VALGRIND
76 #include <valgrind.h>
77 #include <memcheck.h>
78 #define VG(x) x
79 #else
80 #define VG(x)
81 #endif
82
83 #define memclear(s) memset(&s, 0, sizeof(s))
84
85 #define MOS_DBG(...) do { \
86 if (bufmgr_gem != nullptr && bufmgr_gem->bufmgr.debug) \
87 fprintf(stderr, __VA_ARGS__); \
88 } while (0)
89
90 #define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
91 #define MAX2(A, B) ((A) > (B) ? (A) : (B))
92
93 /**
94 * upper_32_bits - return bits 32-63 of a number
95 * @n: the number we're accessing
96 *
97 * A basic shift-right of a 64- or 32-bit quantity. Use this to suppress
98 * the "right shift count >= width of type" warning when that quantity is
99 * 32-bits.
100 */
101 #define upper_32_bits(n) ((__u32)(((n) >> 16) >> 16))
102
103 /**
104 * lower_32_bits - return bits 0-31 of a number
105 * @n: the number we're accessing
106 */
107 #define lower_32_bits(n) ((__u32)(n))
108
109 #define PCI_CHIP_I915_G 0x2582
110 #define PCI_CHIP_E7221_G 0x258A
111 #define PCI_CHIP_I915_GM 0x2592
112
113 #define IS_915(devid) ((devid) == PCI_CHIP_I915_G || \
114 (devid) == PCI_CHIP_E7221_G || \
115 (devid) == PCI_CHIP_I915_GM)
116
117 #define INITIAL_SOFTPIN_TARGET_COUNT 1024
118
119 struct mos_gem_bo_bucket {
120 drmMMListHead head;
121 unsigned long size;
122 };
123
124 struct mos_bufmgr_gem {
125 struct mos_bufmgr bufmgr;
126
127 atomic_t refcount;
128
129 int fd;
130
131 int max_relocs;
132
133 pthread_mutex_t lock;
134
135 struct drm_i915_gem_exec_object *exec_objects;
136 struct drm_i915_gem_exec_object2 *exec2_objects;
137 struct mos_linux_bo **exec_bos;
138 int exec_size;
139 int exec_count;
140
141 /** Array of lists of cached gem objects of power-of-two sizes */
142 struct mos_gem_bo_bucket cache_bucket[64];
143 int num_buckets;
144 time_t time;
145
146 drmMMListHead managers;
147
148 drmMMListHead named;
149
150 uint64_t gtt_size;
151 int available_fences;
152 int pci_device;
153 unsigned int has_bsd : 1;
154 unsigned int has_bsd2 : 1;
155 unsigned int has_blt : 1;
156 unsigned int has_relaxed_fencing : 1;
157 unsigned int has_llc : 1;
158 unsigned int has_wait_timeout : 1;
159 unsigned int bo_reuse : 1;
160 unsigned int no_exec : 1;
161 unsigned int has_vebox : 1;
162 unsigned int has_ext_mmap : 1;
163 unsigned int has_fence_reg : 1;
164 unsigned int has_lmem : 1;
165 unsigned int has_mmap_offset : 1;
166 bool fenced_relocs;
167
168 struct {
169 void *ptr;
170 uint32_t handle;
171 } userptr_active;
172
173 // manage address for softpin buffer object
174 mos_vma_heap vma_heap[MEMZONE_COUNT];
175 bool use_softpin;
176 bool softpin_va1Malign;
177
178 bool object_capture_disabled;
179
180 #define MEM_PROFILER_BUFFER_SIZE 256
181 char mem_profiler_buffer[MEM_PROFILER_BUFFER_SIZE];
182 char* mem_profiler_path;
183 int mem_profiler_fd;
184
185 int device_type;
186
187 uint32_t ts_freq;
188 } mos_bufmgr_gem;
189
190 #define DRM_INTEL_RELOC_FENCE (1<<0)
191
192 struct mos_reloc_target {
193 struct mos_linux_bo *bo;
194 int flags;
195 };
196
197 struct mos_softpin_target {
198 struct mos_linux_bo *bo;
199 int flags;
200 };
201
202 struct mos_bo_gem {
203 struct mos_linux_bo bo;
204
205 atomic_t refcount;
206 uint32_t gem_handle;
207 const char *name;
208
209 /**
210 * Kenel-assigned global name for this object
211 *
212 * List contains both flink named and prime fd'd objects
213 */
214 unsigned int global_name;
215 drmMMListHead name_list;
216
217 /**
218 * Index of the buffer within the validation list while preparing a
219 * batchbuffer execution.
220 */
221 int validate_index;
222
223 /**
224 * Current tiling mode
225 */
226 uint32_t tiling_mode;
227 uint32_t swizzle_mode;
228 unsigned long stride;
229
230 time_t free_time;
231
232 /** Array passed to the DRM containing relocation information. */
233 struct drm_i915_gem_relocation_entry *relocs;
234 /**
235 * Array of info structs corresponding to relocs[i].target_handle etc
236 */
237 struct mos_reloc_target *reloc_target_info;
238 /** Number of entries in relocs */
239 int reloc_count;
240 /** Array of BOs that are referenced by this buffer and will be softpinned */
241 struct mos_softpin_target *softpin_target;
242 /** Number softpinned BOs that are referenced by this buffer */
243 int softpin_target_count;
244 /** Maximum amount of softpinned BOs that are referenced by this buffer */
245 int max_softpin_target_count;
246
247 /** Mapped address for the buffer, saved across map/unmap cycles */
248 void *mem_virtual;
249 /** Uncached Mapped address for the buffer, saved across map/unmap cycles */
250 void *mem_wc_virtual;
251 /** GTT virtual address for the buffer, saved across map/unmap cycles */
252 void *gtt_virtual;
253 /**
254 * Virtual address of the buffer allocated by user, used for userptr
255 * objects only.
256 */
257 void *user_virtual;
258 int map_count;
259
260 /** BO cache list */
261 drmMMListHead head;
262
263 /**
264 * Boolean of whether this BO and its children have been included in
265 * the current drm_intel_bufmgr_check_aperture_space() total.
266 */
267 bool included_in_check_aperture;
268
269 /**
270 * Boolean of whether this buffer has been used as a relocation
271 * target and had its size accounted for, and thus can't have any
272 * further relocations added to it.
273 */
274 bool used_as_reloc_target;
275
276 /**
277 * Boolean of whether we have encountered an error whilst building the relocation tree.
278 */
279 bool has_error;
280
281 /**
282 * Boolean of whether this buffer can be re-used
283 */
284 bool reusable;
285
286 /**
287 * Boolean of whether the GPU is definitely not accessing the buffer.
288 *
289 * This is only valid when reusable, since non-reusable
290 * buffers are those that have been shared wth other
291 * processes, so we don't know their state.
292 */
293 bool idle;
294
295 /**
296 * Boolean of whether this buffer was allocated with userptr
297 */
298 bool is_userptr;
299
300 /**
301 * Boolean of whether this buffer can be placed in the full 48-bit
302 * address range on gen8+.
303 *
304 * By default, buffers will be keep in a 32-bit range, unless this
305 * flag is explicitly set.
306 */
307 bool use_48b_address_range;
308
309 /**
310 * Whether this buffer is softpinned at offset specified by the user
311 */
312 bool is_softpin;
313
314 /*
315 * Whether to remove the dependency of this bo in exebuf.
316 */
317 bool exec_async;
318
319 /*
320 * Whether to remove the dependency of this bo in exebuf.
321 */
322 bool exec_capture;
323
324 /**
325 * Size in bytes of this buffer and its relocation descendents.
326 *
327 * Used to avoid costly tree walking in
328 * drm_intel_bufmgr_check_aperture in the common case.
329 */
330 int reloc_tree_size;
331
332 /**
333 * Number of potential fence registers required by this buffer and its
334 * relocations.
335 */
336 int reloc_tree_fences;
337
338 /** Flags that we may need to do the SW_FINSIH ioctl on unmap. */
339 bool mapped_cpu_write;
340
341 /**
342 * Size to pad the object to.
343 *
344 */
345 uint64_t pad_to_size;
346
347 /**
348 * Memory Type on created the surfaces for local/system memory
349 */
350 int mem_region;
351
352 /**
353 * PAT Index
354 */
355 unsigned int pat_index;
356
357 /**
358 * Is cpu cacheable
359 */
360 bool cpu_cacheable;
361 };
362
363 struct mos_exec_info {
364 /* save all exec2_objects for res*/
365 struct drm_i915_gem_exec_object2* obj;
366 /* save batch buffer*/
367 struct drm_i915_gem_exec_object2* batch_obj;
368 /* save previous ptr to void mem leak when free*/
369 struct drm_i915_gem_exec_object2* pSavePreviousExec2Objects;
370 /*bo resource count*/
371 uint32_t obj_count;
372 /*batch buffer bo count*/
373 uint32_t batch_count;
374 /*remain size of 'obj'*/
375 uint32_t obj_remain_size;
376 #define OBJ512_SIZE 512
377 };
378
379 static unsigned int
380 mos_gem_estimate_batch_space(struct mos_linux_bo ** bo_array, int count);
381
382 static unsigned int
383 mos_gem_compute_batch_space(struct mos_linux_bo ** bo_array, int count);
384
385 static int
386 mos_gem_bo_get_tiling(struct mos_linux_bo *bo, uint32_t * tiling_mode,
387 uint32_t * swizzle_mode);
388
389 static int
390 mos_gem_bo_check_mem_region_internal(struct mos_linux_bo *bo,
391 int mem_type);
392
393 static int
394 mos_gem_bo_set_tiling_internal(struct mos_linux_bo *bo,
395 uint32_t tiling_mode,
396 uint32_t stride);
397
398 static void mos_gem_bo_unreference_locked_timed(struct mos_linux_bo *bo,
399 time_t time);
400
401 static void mos_gem_bo_unreference(struct mos_linux_bo *bo);
402 static bool mos_gem_bo_is_softpin(struct mos_linux_bo *bo);
403 static void mos_gem_bo_start_gtt_access(struct mos_linux_bo *bo, int write_enable);
404 static void mos_gem_bo_free(struct mos_linux_bo *bo);
405
406 static int mos_bufmgr_get_driver_info(struct mos_bufmgr *bufmgr, struct LinuxDriverInfo *drvInfo);
407
408 static inline struct mos_bo_gem *to_bo_gem(struct mos_linux_bo *bo)
409 {
410 return (struct mos_bo_gem *)bo;
411 }
412
413 static unsigned long
414 mos_gem_bo_tile_size(struct mos_bufmgr_gem *bufmgr_gem, unsigned long size,
415 uint32_t *tiling_mode)
416 {
417 unsigned long min_size, max_size;
418 unsigned long i;
419
420 if (*tiling_mode == I915_TILING_NONE)
421 return size;
422
423 /* 965+ just need multiples of page size for tiling */
424 return ROUND_UP_TO(size, 4096);
425
426 }
427
428 /*
429 * Round a given pitch up to the minimum required for X tiling on a
430 * given chip. We use 512 as the minimum to allow for a later tiling
431 * change.
432 */
433 static unsigned long
434 mos_gem_bo_tile_pitch(struct mos_bufmgr_gem *bufmgr_gem,
435 unsigned long pitch, uint32_t *tiling_mode)
436 {
437 unsigned long tile_width;
438 unsigned long i;
439
440 /* If untiled, then just align it so that we can do rendering
441 * to it with the 3D engine.
442 */
443 if (*tiling_mode == I915_TILING_NONE)
444 return ALIGN(pitch, 64);
445
446 if (*tiling_mode == I915_TILING_X
447 || (IS_915(bufmgr_gem->pci_device)
448 && *tiling_mode == I915_TILING_Y))
449 tile_width = 512;
450 else
451 tile_width = 128;
452
453 /* 965 is flexible */
454 return ROUND_UP_TO(pitch, tile_width);
455 }
456
457 static struct mos_gem_bo_bucket *
458 mos_gem_bo_bucket_for_size(struct mos_bufmgr_gem *bufmgr_gem,
459 unsigned long size)
460 {
461 int i;
462
463 for (i = 0; i < bufmgr_gem->num_buckets; i++) {
464 struct mos_gem_bo_bucket *bucket =
465 &bufmgr_gem->cache_bucket[i];
466 if (bucket->size >= size) {
467 return bucket;
468 }
469 }
470
471 return nullptr;
472 }
473
474 static void
475 mos_gem_dump_validation_list(struct mos_bufmgr_gem *bufmgr_gem)
476 {
477 int i, j;
478
479 for (i = 0; i < bufmgr_gem->exec_count; i++) {
480 struct mos_linux_bo *bo = bufmgr_gem->exec_bos[i];
481 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
482
483 if (bo_gem->relocs == nullptr || bo_gem->softpin_target == nullptr) {
484 MOS_DBG("%2d: %d %s(%s)\n", i, bo_gem->gem_handle,
485 bo_gem->is_softpin ? "*" : "",
486 bo_gem->name);
487 continue;
488 }
489
490 for (j = 0; j < bo_gem->reloc_count; j++) {
491 struct mos_linux_bo *target_bo = bo_gem->reloc_target_info[j].bo;
492 struct mos_bo_gem *target_gem =
493 (struct mos_bo_gem *) target_bo;
494
495 MOS_DBG("%2d: %d %s(%s)@0x%08x %08x -> "
496 "%d (%s)@0x%08x %08x + 0x%08x\n",
497 i,
498 bo_gem->gem_handle,
499 bo_gem->is_softpin ? "*" : "",
500 bo_gem->name,
501 upper_32_bits(bo_gem->relocs[j].offset),
502 lower_32_bits(bo_gem->relocs[j].offset),
503 target_gem->gem_handle,
504 target_gem->name,
505 upper_32_bits(target_bo->offset64),
506 lower_32_bits(target_bo->offset64),
507 bo_gem->relocs[j].delta);
508 }
509
510 for (j = 0; j < bo_gem->softpin_target_count; j++) {
511 struct mos_linux_bo *target_bo = bo_gem->softpin_target[j].bo;
512 struct mos_bo_gem *target_gem =
513 (struct mos_bo_gem *) target_bo;
514 MOS_DBG("%2d: %d %s(%s) -> "
515 "%d *(%s)@0x%08x %08x\n",
516 i,
517 bo_gem->gem_handle,
518 bo_gem->is_softpin ? "*" : "",
519 bo_gem->name,
520 target_gem->gem_handle,
521 target_gem->name,
522 upper_32_bits(target_bo->offset64),
523 lower_32_bits(target_bo->offset64));
524 }
525 }
526 }
527
528 static inline void
529 mos_gem_bo_reference(struct mos_linux_bo *bo)
530 {
531 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
532
533 atomic_inc(&bo_gem->refcount);
534 }
535
536 /**
537 * Adds the given buffer to the list of buffers to be validated (moved into the
538 * appropriate memory type) with the next batch submission.
539 *
540 * If a buffer is validated multiple times in a batch submission, it ends up
541 * with the intersection of the memory type flags and the union of the
542 * access flags.
543 */
544 static void
545 mos_add_validate_buffer(struct mos_linux_bo *bo)
546 {
547 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
548 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
549 int index;
550 struct drm_i915_gem_exec_object *exec_objects;
551 struct mos_linux_bo **exec_bos;
552
553 if (bo_gem->validate_index != -1)
554 return;
555
556 /* Extend the array of validation entries as necessary. */
557 if (bufmgr_gem->exec_count == bufmgr_gem->exec_size) {
558 int new_size = bufmgr_gem->exec_size * 2;
559
560 if (new_size == 0)
561 new_size = ARRAY_INIT_SIZE;
562
563 exec_objects = (struct drm_i915_gem_exec_object *)realloc(bufmgr_gem->exec_objects,
564 sizeof(*bufmgr_gem->exec_objects) * new_size);
565 if (!exec_objects)
566 return;
567
568 bufmgr_gem->exec_objects = exec_objects;
569
570 exec_bos = (struct mos_linux_bo **)realloc(bufmgr_gem->exec_bos,
571 sizeof(*bufmgr_gem->exec_bos) * new_size);
572 if (!exec_bos)
573 return;
574
575 bufmgr_gem->exec_bos = exec_bos;
576 bufmgr_gem->exec_size = new_size;
577 }
578
579 index = bufmgr_gem->exec_count;
580 bo_gem->validate_index = index;
581 /* Fill in array entry */
582 bufmgr_gem->exec_objects[index].handle = bo_gem->gem_handle;
583 bufmgr_gem->exec_objects[index].relocation_count = bo_gem->reloc_count;
584 bufmgr_gem->exec_objects[index].relocs_ptr = (uintptr_t) bo_gem->relocs;
585 bufmgr_gem->exec_objects[index].alignment = bo->align;
586 bufmgr_gem->exec_objects[index].offset = 0;
587 bufmgr_gem->exec_bos[index] = bo;
588 bufmgr_gem->exec_count++;
589 }
590
591 static void
592 mos_add_validate_buffer2(struct mos_linux_bo *bo, int need_fence)
593 {
594 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bo->bufmgr;
595 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *)bo;
596 int index;
597 struct drm_i915_gem_exec_object2 *exec2_objects;
598 struct mos_linux_bo **exec_bos;
599 int flags = 0;
600
601 if (need_fence)
602 flags |= EXEC_OBJECT_NEEDS_FENCE;
603 if (bo_gem->pad_to_size)
604 flags |= EXEC_OBJECT_PAD_TO_SIZE;
605 if (bo_gem->use_48b_address_range)
606 flags |= EXEC_OBJECT_SUPPORTS_48B_ADDRESS;
607 if (bo_gem->is_softpin)
608 flags |= EXEC_OBJECT_PINNED;
609 if (bo_gem->exec_async)
610 flags |= EXEC_OBJECT_ASYNC;
611 if (bo_gem->exec_capture)
612 flags |= EXEC_OBJECT_CAPTURE;
613
614 if (bo_gem->validate_index != -1) {
615 bufmgr_gem->exec2_objects[bo_gem->validate_index].flags |= flags;
616 return;
617 }
618
619 /* Extend the array of validation entries as necessary. */
620 if (bufmgr_gem->exec_count == bufmgr_gem->exec_size) {
621 int new_size = bufmgr_gem->exec_size * 2;
622
623 if (new_size == 0)
624 new_size = ARRAY_INIT_SIZE;
625 exec2_objects = (struct drm_i915_gem_exec_object2 *)
626 realloc(bufmgr_gem->exec2_objects,
627 sizeof(*bufmgr_gem->exec2_objects) * new_size);
628 if (!exec2_objects)
629 return;
630
631 bufmgr_gem->exec2_objects = exec2_objects;
632
633 exec_bos = (struct mos_linux_bo **)realloc(bufmgr_gem->exec_bos,
634 sizeof(*bufmgr_gem->exec_bos) * new_size);
635 if (!exec_bos)
636 return;
637
638 bufmgr_gem->exec_bos = exec_bos;
639 bufmgr_gem->exec_size = new_size;
640 }
641
642 index = bufmgr_gem->exec_count;
643 bo_gem->validate_index = index;
644 /* Fill in array entry */
645 bufmgr_gem->exec2_objects[index].handle = bo_gem->gem_handle;
646 bufmgr_gem->exec2_objects[index].relocation_count = bo_gem->reloc_count;
647 bufmgr_gem->exec2_objects[index].relocs_ptr = (uintptr_t)bo_gem->relocs;
648 bufmgr_gem->exec2_objects[index].alignment = bo->align;
649 bufmgr_gem->exec2_objects[index].offset = bo_gem->is_softpin ?
650 bo->offset64 : 0;
651 bufmgr_gem->exec_bos[index] = bo;
652 bufmgr_gem->exec2_objects[index].flags = flags;
653 bufmgr_gem->exec2_objects[index].rsvd1 = 0;
654 bufmgr_gem->exec2_objects[index].pad_to_size = bo_gem->pad_to_size;
655 bufmgr_gem->exec2_objects[index].rsvd2 = 0;
656 bufmgr_gem->exec_count++;
657 }
658
659 static void
660 mos_add_reloc_objects(struct mos_reloc_target reloc_target)
661 {
662 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)reloc_target.bo->bufmgr;
663 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *)reloc_target.bo;
664 int index;
665 struct drm_i915_gem_exec_object2 *exec2_objects;
666 struct mos_linux_bo **exec_bos;
667
668 if (bo_gem->validate_index != -1) {
669 bufmgr_gem->exec2_objects[bo_gem->validate_index].flags |= reloc_target.flags;
670 return;
671 }
672
673 /* Extend the array of validation entries as necessary. */
674 if (bufmgr_gem->exec_count == bufmgr_gem->exec_size) {
675 int new_size = bufmgr_gem->exec_size * 2;
676
677 if (new_size == 0)
678 new_size = ARRAY_INIT_SIZE;
679 exec2_objects = (struct drm_i915_gem_exec_object2 *)
680 realloc(bufmgr_gem->exec2_objects,
681 sizeof(*bufmgr_gem->exec2_objects) * new_size);
682 if (!exec2_objects)
683 {
684 MOS_DBG("realloc exec2_objects failed!\n");
685 return;
686 }
687
688 bufmgr_gem->exec2_objects = exec2_objects;
689
690 exec_bos = (struct mos_linux_bo **)realloc(bufmgr_gem->exec_bos,
691 sizeof(*bufmgr_gem->exec_bos) * new_size);
692 if (!exec_bos)
693 {
694 MOS_DBG("realloc exec_bo failed!\n");
695 return;
696 }
697
698 bufmgr_gem->exec_bos = exec_bos;
699 bufmgr_gem->exec_size = new_size;
700 }
701
702 index = bufmgr_gem->exec_count;
703 bo_gem->validate_index = index;
704 /* Fill in array entry */
705 bufmgr_gem->exec2_objects[index].handle = bo_gem->gem_handle;
706 bufmgr_gem->exec2_objects[index].relocation_count = bo_gem->reloc_count;
707 bufmgr_gem->exec2_objects[index].relocs_ptr = (uintptr_t)bo_gem->relocs;
708 bufmgr_gem->exec2_objects[index].alignment = reloc_target.bo->align;
709 bufmgr_gem->exec2_objects[index].offset = 0;
710 bufmgr_gem->exec_bos[index] = reloc_target.bo;
711 bufmgr_gem->exec2_objects[index].flags = reloc_target.flags;
712 bufmgr_gem->exec2_objects[index].rsvd1 = 0;
713 bufmgr_gem->exec2_objects[index].pad_to_size = bo_gem->pad_to_size;
714 bufmgr_gem->exec2_objects[index].rsvd2 = 0;
715 bufmgr_gem->exec_count++;
716 }
717
718 static void
719 mos_add_softpin_objects(struct mos_softpin_target softpin_target)
720 {
721 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)softpin_target.bo->bufmgr;
722 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *)softpin_target.bo;
723 int index;
724 struct drm_i915_gem_exec_object2 *exec2_objects;
725 struct mos_linux_bo **exec_bos;
726
727 if (bo_gem->validate_index != -1) {
728 bufmgr_gem->exec2_objects[bo_gem->validate_index].flags |= softpin_target.flags;
729 return;
730 }
731
732 /* Extend the array of validation entries as necessary. */
733 if (bufmgr_gem->exec_count == bufmgr_gem->exec_size) {
734 int new_size = bufmgr_gem->exec_size * 2;
735
736 if (new_size == 0)
737 new_size = ARRAY_INIT_SIZE;
738 exec2_objects = (struct drm_i915_gem_exec_object2 *)
739 realloc(bufmgr_gem->exec2_objects,
740 sizeof(*bufmgr_gem->exec2_objects) * new_size);
741 if (!exec2_objects)
742 {
743 MOS_DBG("realloc exec2_objects failed!\n");
744 return;
745 }
746
747 bufmgr_gem->exec2_objects = exec2_objects;
748
749 exec_bos = (struct mos_linux_bo **)realloc(bufmgr_gem->exec_bos,
750 sizeof(*bufmgr_gem->exec_bos) * new_size);
751 if (!exec_bos)
752 {
753 MOS_DBG("realloc exec_bo failed!\n");
754 return;
755 }
756
757 bufmgr_gem->exec_bos = exec_bos;
758 bufmgr_gem->exec_size = new_size;
759 }
760
761 index = bufmgr_gem->exec_count;
762 bo_gem->validate_index = index;
763 /* Fill in array entry */
764 bufmgr_gem->exec2_objects[index].handle = bo_gem->gem_handle;
765 bufmgr_gem->exec2_objects[index].relocation_count = bo_gem->reloc_count;
766 bufmgr_gem->exec2_objects[index].relocs_ptr = (uintptr_t)bo_gem->relocs;
767 bufmgr_gem->exec2_objects[index].alignment = softpin_target.bo->align;
768 bufmgr_gem->exec2_objects[index].offset = softpin_target.bo->offset64;
769 bufmgr_gem->exec2_objects[index].flags = softpin_target.flags;
770 bufmgr_gem->exec2_objects[index].pad_to_size = bo_gem->pad_to_size;
771 bufmgr_gem->exec2_objects[index].rsvd1 = 0;
772 bufmgr_gem->exec2_objects[index].rsvd2 = 0;
773 bufmgr_gem->exec_bos[index] = softpin_target.bo;
774 bufmgr_gem->exec_count++;
775 }
776
777 #define RELOC_BUF_SIZE(x) ((I915_RELOC_HEADER + x * I915_RELOC0_STRIDE) * \
778 sizeof(uint32_t))
779
780 static void
781 mos_bo_gem_set_in_aperture_size(struct mos_bufmgr_gem *bufmgr_gem,
782 struct mos_bo_gem *bo_gem,
783 unsigned int alignment)
784 {
785 unsigned int size;
786
787 assert(!bo_gem->used_as_reloc_target);
788
789 /* The older chipsets are far-less flexible in terms of tiling,
790 * and require tiled buffer to be size aligned in the aperture.
791 * This means that in the worst possible case we will need a hole
792 * twice as large as the object in order for it to fit into the
793 * aperture. Optimal packing is for wimps.
794 */
795 size = bo_gem->bo.size;
796
797 bo_gem->reloc_tree_size = size + alignment;
798 }
799
800 static int
801 mos_setup_reloc_list(struct mos_linux_bo *bo)
802 {
803 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
804 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
805 unsigned int max_relocs = bufmgr_gem->max_relocs;
806
807 if (bo->size / 4 < max_relocs)
808 max_relocs = bo->size / 4;
809
810 bo_gem->relocs = (struct drm_i915_gem_relocation_entry *)malloc(max_relocs *
811 sizeof(struct drm_i915_gem_relocation_entry));
812 bo_gem->reloc_target_info = (struct mos_reloc_target *)malloc(max_relocs *
813 sizeof(struct mos_reloc_target));
814 if (bo_gem->relocs == nullptr || bo_gem->reloc_target_info == nullptr) {
815 bo_gem->has_error = true;
816
817 free (bo_gem->relocs);
818 bo_gem->relocs = nullptr;
819
820 free (bo_gem->reloc_target_info);
821 bo_gem->reloc_target_info = nullptr;
822
823 return 1;
824 }
825
826 return 0;
827 }
828
829 static int
830 mos_gem_bo_busy(struct mos_linux_bo *bo)
831 {
832 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
833 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
834 struct drm_i915_gem_busy busy;
835 int ret;
836
837 if (bo_gem->reusable && bo_gem->idle)
838 return false;
839
840 memclear(busy);
841 busy.handle = bo_gem->gem_handle;
842
843 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_BUSY, &busy);
844 if (ret == 0) {
845 bo_gem->idle = !busy.busy;
846 return busy.busy;
847 } else {
848 return false;
849 }
850 return (ret == 0 && busy.busy);
851 }
852
853 static int
854 mos_gem_bo_madvise_internal(struct mos_bufmgr_gem *bufmgr_gem,
855 struct mos_bo_gem *bo_gem, int state)
856 {
857 struct drm_i915_gem_madvise madv;
858
859 memclear(madv);
860 madv.handle = bo_gem->gem_handle;
861 madv.madv = state;
862 madv.retained = 1;
863 drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_MADVISE, &madv);
864
865 return madv.retained;
866 }
867
868 static int
869 mos_gem_bo_madvise(struct mos_linux_bo *bo, int madv)
870 {
871 return mos_gem_bo_madvise_internal
872 ((struct mos_bufmgr_gem *) bo->bufmgr,
873 (struct mos_bo_gem *) bo,
874 madv);
875 }
876
877 /* drop the oldest entries that have been purged by the kernel */
878 static void
879 mos_gem_bo_cache_purge_bucket(struct mos_bufmgr_gem *bufmgr_gem,
880 struct mos_gem_bo_bucket *bucket)
881 {
882 while (!DRMLISTEMPTY(&bucket->head)) {
883 struct mos_bo_gem *bo_gem;
884
885 bo_gem = DRMLISTENTRY(struct mos_bo_gem,
886 bucket->head.next, head);
887 if (mos_gem_bo_madvise_internal
888 (bufmgr_gem, bo_gem, I915_MADV_DONTNEED))
889 break;
890
891 DRMLISTDEL(&bo_gem->head);
892 mos_gem_bo_free(&bo_gem->bo);
893 }
894 }
895
896 static int
897 mos_gem_query_items(int fd, struct drm_i915_query_item *items, uint32_t n_items)
898 {
899 struct drm_i915_query q;
900
901 memclear(q);
902 q.num_items = n_items;
903 q.items_ptr = (uintptr_t)items;
904
905 return drmIoctl(fd, DRM_IOCTL_I915_QUERY, &q);
906 }
907
908 /**
909 * query mechanism for memory regions.
910 */
911 static struct drm_i915_query_memory_regions *mos_gem_get_query_memory_regions(int fd)
912 {
913 int ret;
914 struct drm_i915_query_item item;
915 struct drm_i915_query_memory_regions *query_info;
916
917 memclear(item);
918 item.query_id = DRM_I915_QUERY_MEMORY_REGIONS;
919 ret = mos_gem_query_items(fd, &item, 1);
920 if (ret != 0 || item.length <= 0)
921 return NULL;
922
923 query_info = (drm_i915_query_memory_regions*)calloc(1, item.length);
924
925 item.data_ptr = (uintptr_t)query_info;
926 ret = mos_gem_query_items(fd, &item, 1);
927 if (ret != 0) {
928 free(query_info);
929 return NULL;
930 }
931
932 return query_info;
933 }
934
935 /**
936 * check how many lmem regions are available on device.
937 */
938 static uint8_t mos_gem_get_lmem_region_count(int fd)
939 {
940 struct drm_i915_query_memory_regions *query_info;
941 uint8_t num_regions = 0;
942 uint8_t lmem_regions = 0;
943
944 query_info = mos_gem_get_query_memory_regions(fd);
945
946 if(query_info)
947 {
948 num_regions = query_info->num_regions;
949
950 for (int i = 0; i < num_regions; i++) {
951 if (query_info->regions[i].region.memory_class == I915_MEMORY_CLASS_DEVICE)
952 {
953 lmem_regions += 1;
954 }
955 }
956 free(query_info);
957 }
958
959 return lmem_regions;
960 }
961
962 /**
963 * check if lmem is available on device.
964 */
965 static bool mos_gem_has_lmem(int fd)
966 {
967 return mos_gem_get_lmem_region_count(fd) > 0;
968 }
969
970 static enum mos_memory_zone
971 mos_gem_bo_memzone_for_address(uint64_t address)
972 {
973 if (address >= MEMZONE_DEVICE_START)
974 return MEMZONE_DEVICE;
975 else
976 return MEMZONE_SYS;
977 }
978
979 /**
980 * Allocate a section of virtual memory for a buffer, assigning an address.
981 */
982 static uint64_t
983 mos_gem_bo_vma_alloc(struct mos_bufmgr *bufmgr,
984 enum mos_memory_zone memzone,
985 uint64_t size,
986 uint64_t alignment)
987 {
988 CHK_CONDITION(bufmgr == nullptr, "nullptr bufmgr.\n", 0);
989 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bufmgr;
990 /* Force alignment to be some number of pages */
991 alignment = ALIGN(alignment, PAGE_SIZE);
992
993 uint64_t addr = mos_vma_heap_alloc(&bufmgr_gem->vma_heap[memzone], size, alignment);
994
995 // currently only support 48bit range address
996 CHK_CONDITION((addr >> 48ull) != 0, "invalid address, over 48bit range.\n", 0);
997 CHK_CONDITION((addr >> (memzone == MEMZONE_SYS ? 40ull : 41ull)) != 0, "invalid address, over memory zone range.\n", 0);
998 CHK_CONDITION((addr % alignment) != 0, "invalid address, not meet aligment requirement.\n", 0);
999
1000 return addr;
1001 }
1002
1003 static void
1004 mos_gem_bo_vma_free(struct mos_bufmgr *bufmgr,
1005 uint64_t address,
1006 uint64_t size)
1007 {
1008 CHK_CONDITION(bufmgr == nullptr, "nullptr bufmgr.\n", );
1009 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bufmgr;
1010
1011 CHK_CONDITION(address == 0ull, "invalid address.\n", );
1012 enum mos_memory_zone memzone = mos_gem_bo_memzone_for_address(address);
1013 mos_vma_heap_free(&bufmgr_gem->vma_heap[memzone], address, size);
1014 }
1015
1016 drm_export struct mos_linux_bo *
1017 mos_gem_bo_alloc_internal(struct mos_bufmgr *bufmgr,
1018 struct mos_drm_bo_alloc *alloc)
1019 {
1020 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bufmgr;
1021 struct mos_bo_gem *bo_gem;
1022 unsigned int page_size = getpagesize();
1023 static bool support_pat_index = true;
1024 int ret;
1025 struct mos_gem_bo_bucket *bucket;
1026 bool alloc_from_cache;
1027 unsigned long bo_size;
1028 bool for_render = false;
1029
1030 if (alloc->ext.flags & BO_ALLOC_FOR_RENDER)
1031 for_render = true;
1032
1033 /* Round the allocated size up to a power of two number of pages. */
1034 bucket = mos_gem_bo_bucket_for_size(bufmgr_gem, alloc->size);
1035
1036 /* If we don't have caching at this size, don't actually round the
1037 * allocation up.
1038 */
1039 if (bucket == nullptr) {
1040 bo_size = alloc->size;
1041 if (bo_size < page_size)
1042 bo_size = page_size;
1043 } else {
1044 bo_size = bucket->size;
1045 }
1046 if (!support_pat_index)
1047 {
1048 /* For old kernel without pat index support,
1049 * We need to reset pat_index for bo reuse policy
1050 */
1051 alloc->ext.pat_index = PAT_INDEX_INVALID;
1052 }
1053 pthread_mutex_lock(&bufmgr_gem->lock);
1054 /* Get a buffer out of the cache if available */
1055 retry:
1056 alloc_from_cache = false;
1057 if (bucket != nullptr && !DRMLISTEMPTY(&bucket->head)) {
1058 if (for_render) {
1059 /* Allocate new render-target BOs from the tail (MRU)
1060 * of the list, as it will likely be hot in the GPU
1061 * cache and in the aperture for us.
1062 */
1063 bo_gem = DRMLISTENTRY(struct mos_bo_gem,
1064 bucket->head.prev, head);
1065 DRMLISTDEL(&bo_gem->head);
1066 alloc_from_cache = true;
1067 bo_gem->bo.align = alloc->alignment;
1068 } else {
1069 assert(alloc->alignment == 0);
1070 /* For non-render-target BOs (where we're probably
1071 * going to map it first thing in order to fill it
1072 * with data), check if the last BO in the cache is
1073 * unbusy, and only reuse in that case. Otherwise,
1074 * allocating a new buffer is probably faster than
1075 * waiting for the GPU to finish.
1076 */
1077 bo_gem = DRMLISTENTRY(struct mos_bo_gem,
1078 bucket->head.next, head);
1079 if (!mos_gem_bo_busy(&bo_gem->bo)) {
1080 alloc_from_cache = true;
1081 DRMLISTDEL(&bo_gem->head);
1082 }
1083 }
1084
1085 if (alloc_from_cache) {
1086 if (!mos_gem_bo_madvise_internal
1087 (bufmgr_gem, bo_gem, I915_MADV_WILLNEED)) {
1088 mos_gem_bo_free(&bo_gem->bo);
1089 mos_gem_bo_cache_purge_bucket(bufmgr_gem,
1090 bucket);
1091 goto retry;
1092 }
1093 if (bo_gem->pat_index != alloc->ext.pat_index)
1094 {
1095 mos_gem_bo_free(&bo_gem->bo);
1096 goto retry;
1097 }
1098 if (mos_gem_bo_set_tiling_internal(&bo_gem->bo,
1099 alloc->ext.tiling_mode,
1100 alloc->stride)) {
1101 mos_gem_bo_free(&bo_gem->bo);
1102 goto retry;
1103 }
1104 if (bufmgr_gem->has_lmem && mos_gem_bo_check_mem_region_internal(&bo_gem->bo, alloc->ext.mem_type)) {
1105 mos_gem_bo_free(&bo_gem->bo);
1106 goto retry;
1107 }
1108 }
1109 }
1110 pthread_mutex_unlock(&bufmgr_gem->lock);
1111
1112 if (!alloc_from_cache) {
1113
1114 bo_gem = (struct mos_bo_gem *)calloc(1, sizeof(*bo_gem));
1115 if (!bo_gem)
1116 return nullptr;
1117
1118 bo_gem->bo.size = bo_size;
1119 bo_gem->mem_region = I915_MEMORY_CLASS_SYSTEM;
1120 bo_gem->pat_index = PAT_INDEX_INVALID;
1121 bo_gem->cpu_cacheable = true;
1122
1123 if (bufmgr_gem->has_lmem &&
1124 (alloc->ext.mem_type == MOS_MEMPOOL_VIDEOMEMORY || alloc->ext.mem_type == MOS_MEMPOOL_DEVICEMEMORY)) {
1125 struct drm_i915_gem_memory_class_instance mem_region;
1126 memclear(mem_region);
1127 mem_region.memory_class = I915_MEMORY_CLASS_DEVICE;
1128 mem_region.memory_instance = 0;
1129
1130 struct drm_i915_gem_create_ext_memory_regions regions;
1131 memclear(regions);
1132 regions.base.name = I915_GEM_CREATE_EXT_MEMORY_REGIONS;
1133 regions.num_regions = 1;
1134 regions.regions = (uintptr_t)(&mem_region);
1135
1136 struct drm_i915_gem_create_ext create;
1137 memclear(create);
1138 create.size = bo_size;
1139 create.extensions = (uintptr_t)(&regions);
1140
1141 ret = drmIoctl(bufmgr_gem->fd,
1142 DRM_IOCTL_I915_GEM_CREATE_EXT,
1143 &create);
1144 bo_gem->gem_handle = create.handle;
1145 bo_gem->bo.handle = bo_gem->gem_handle;
1146 bo_gem->mem_region = I915_MEMORY_CLASS_DEVICE;
1147 }
1148 else
1149 {
1150 ret = -EINVAL;
1151 if (support_pat_index && alloc->ext.pat_index != PAT_INDEX_INVALID)
1152 {
1153 struct drm_i915_gem_create_ext_set_pat set_pat_ext;
1154 memclear(set_pat_ext);
1155 set_pat_ext.base.name = I915_GEM_CREATE_EXT_SET_PAT;
1156 set_pat_ext.pat_index = alloc->ext.pat_index;
1157
1158 struct drm_i915_gem_create_ext create;
1159 memclear(create);
1160 create.size = bo_size;
1161 create.extensions = (uintptr_t)(&set_pat_ext);
1162 ret = drmIoctl(bufmgr_gem->fd,
1163 DRM_IOCTL_I915_GEM_CREATE_EXT,
1164 &create);
1165 bo_gem->gem_handle = create.handle;
1166 bo_gem->bo.handle = bo_gem->gem_handle;
1167 bo_gem->pat_index = alloc->ext.pat_index;
1168 if (ret != 0)
1169 {
1170 /* For old kernel without pat_index support,
1171 * DRM_IOCTL_I915_GEM_CREATE_EXT with unknown
1172 * set_pat_ext extension will return -EINVAL
1173 * support_pat_index need to be set false.
1174 */
1175 support_pat_index = false;
1176 }
1177 }
1178 if (ret != 0)
1179 {
1180 struct drm_i915_gem_create create;
1181 memclear(create);
1182 create.size = bo_size;
1183 ret = drmIoctl(bufmgr_gem->fd,
1184 DRM_IOCTL_I915_GEM_CREATE,
1185 &create);
1186 bo_gem->gem_handle = create.handle;
1187 bo_gem->bo.handle = bo_gem->gem_handle;
1188 bo_gem->pat_index = PAT_INDEX_INVALID;
1189 }
1190 }
1191 if (ret != 0) {
1192 free(bo_gem);
1193 return nullptr;
1194 }
1195 bo_gem->bo.bufmgr = bufmgr;
1196 bo_gem->bo.align = alloc->alignment;
1197
1198 bo_gem->tiling_mode = I915_TILING_NONE;
1199 bo_gem->swizzle_mode = I915_BIT_6_SWIZZLE_NONE;
1200 bo_gem->stride = 0;
1201 if (bufmgr_gem->mem_profiler_fd != -1)
1202 {
1203 snprintf(bufmgr_gem->mem_profiler_buffer, MEM_PROFILER_BUFFER_SIZE, "GEM_CREATE, %d, %d, %lu, %d, %s\n", getpid(), bo_gem->bo.handle, bo_gem->bo.size,bo_gem->mem_region, alloc->name);
1204 ret = write(bufmgr_gem->mem_profiler_fd, bufmgr_gem->mem_profiler_buffer, strnlen(bufmgr_gem->mem_profiler_buffer, MEM_PROFILER_BUFFER_SIZE));
1205 if (ret == -1)
1206 {
1207 MOS_DBG("Failed to write to %s: %s\n", bufmgr_gem->mem_profiler_path, strerror(errno));
1208 }
1209 }
1210
1211 /* drm_intel_gem_bo_free calls DRMLISTDEL() for an uninitialized
1212 list (vma_list), so better set the list head here */
1213 DRMINITLISTHEAD(&bo_gem->name_list);
1214 if (mos_gem_bo_set_tiling_internal(&bo_gem->bo,
1215 alloc->ext.tiling_mode,
1216 alloc->stride)) {
1217 mos_gem_bo_free(&bo_gem->bo);
1218 return nullptr;
1219 }
1220 }
1221
1222 bo_gem->name = alloc->name;
1223 atomic_set(&bo_gem->refcount, 1);
1224 bo_gem->validate_index = -1;
1225 bo_gem->reloc_tree_fences = 0;
1226 bo_gem->used_as_reloc_target = false;
1227 bo_gem->has_error = false;
1228 bo_gem->reusable = true;
1229 bo_gem->use_48b_address_range = bufmgr_gem->bufmgr.bo_use_48b_address_range ? true : false;
1230
1231 if (bo_gem->pat_index != PAT_INDEX_INVALID)
1232 {
1233 bo_gem->cpu_cacheable = alloc->ext.cpu_cacheable;
1234 }
1235
1236 mos_bo_gem_set_in_aperture_size(bufmgr_gem, bo_gem, alloc->alignment);
1237
1238 if (bufmgr_gem->use_softpin)
1239 {
1240 mos_bo_set_softpin(&bo_gem->bo);
1241 }
1242
1243 MOS_DBG("bo_create: buf %d (%s) %ldb\n",
1244 bo_gem->gem_handle, bo_gem->name, alloc->size);
1245
1246 return &bo_gem->bo;
1247 }
1248
1249 static struct mos_linux_bo *
1250 mos_gem_bo_alloc(struct mos_bufmgr *bufmgr,
1251 struct mos_drm_bo_alloc *alloc)
1252 {
1253 alloc->ext.flags = 0;
1254 alloc->alignment = 0;
1255 alloc->stride = 0;
1256 return mos_gem_bo_alloc_internal(bufmgr, alloc);
1257 }
1258
1259 static struct mos_linux_bo *
1260 mos_gem_bo_alloc_tiled(struct mos_bufmgr *bufmgr,
1261 struct mos_drm_bo_alloc_tiled *alloc_tiled)
1262 {
1263 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bufmgr;
1264 unsigned long size, stride;
1265 uint32_t tiling;
1266
1267 do {
1268 unsigned long aligned_y, height_alignment;
1269
1270 tiling = alloc_tiled->ext.tiling_mode;
1271
1272 /* If we're tiled, our allocations are in 8 or 32-row blocks,
1273 * so failure to align our height means that we won't allocate
1274 * enough pages.
1275 *
1276 * If we're untiled, we still have to align to 2 rows high
1277 * because the data port accesses 2x2 blocks even if the
1278 * bottom row isn't to be rendered, so failure to align means
1279 * we could walk off the end of the GTT and fault. This is
1280 * documented on 965, and may be the case on older chipsets
1281 * too so we try to be careful.
1282 */
1283 aligned_y = alloc_tiled->y;
1284 height_alignment = 2;
1285
1286 if (tiling == I915_TILING_X
1287 || (IS_915(bufmgr_gem->pci_device)
1288 && tiling == I915_TILING_Y))
1289 height_alignment = 8;
1290 else if (tiling == I915_TILING_Y)
1291 height_alignment = 32;
1292 aligned_y = ALIGN(alloc_tiled->y, height_alignment);
1293
1294 stride = alloc_tiled->x * alloc_tiled->cpp;
1295 stride = mos_gem_bo_tile_pitch(bufmgr_gem, stride, &alloc_tiled->ext.tiling_mode);
1296 size = stride * aligned_y;
1297 size = mos_gem_bo_tile_size(bufmgr_gem, size, &alloc_tiled->ext.tiling_mode);
1298 } while (alloc_tiled->ext.tiling_mode != tiling);
1299 alloc_tiled->pitch = stride;
1300
1301 if (tiling == I915_TILING_NONE)
1302 stride = 0;
1303
1304 struct mos_drm_bo_alloc alloc;
1305 alloc.name = alloc_tiled->name;
1306 alloc.size = size;
1307 alloc.stride = stride;
1308 alloc.ext = alloc_tiled->ext;
1309 return mos_gem_bo_alloc_internal(bufmgr, &alloc);
1310 }
1311
1312 static struct mos_linux_bo *
1313 mos_gem_bo_alloc_userptr(struct mos_bufmgr *bufmgr,
1314 struct mos_drm_bo_alloc_userptr *alloc_uptr)
1315 {
1316 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bufmgr;
1317 struct mos_bo_gem *bo_gem;
1318 int ret;
1319 struct drm_i915_gem_userptr userptr;
1320
1321 /* Tiling with userptr surfaces is not supported
1322 * on all hardware so refuse it for time being.
1323 */
1324 if (alloc_uptr->tiling_mode != I915_TILING_NONE)
1325 return nullptr;
1326
1327 bo_gem = (struct mos_bo_gem *)calloc(1, sizeof(*bo_gem));
1328 if (!bo_gem)
1329 return nullptr;
1330
1331 bo_gem->bo.size = alloc_uptr->size;
1332
1333 memclear(userptr);
1334 userptr.user_ptr = (__u64)((unsigned long)alloc_uptr->addr);
1335 userptr.user_size = alloc_uptr->size;
1336 userptr.flags = 0;
1337
1338 ret = drmIoctl(bufmgr_gem->fd,
1339 DRM_IOCTL_I915_GEM_USERPTR,
1340 &userptr);
1341 if (ret != 0) {
1342 MOS_DBG("bo_create_userptr: "
1343 "ioctl failed with user ptr %p size 0x%lx, "
1344 "user flags 0x%lx\n", alloc_uptr->addr, alloc_uptr->size, alloc_uptr->flags);
1345 free(bo_gem);
1346 return nullptr;
1347 }
1348
1349 bo_gem->gem_handle = userptr.handle;
1350 bo_gem->bo.handle = bo_gem->gem_handle;
1351 bo_gem->bo.bufmgr = bufmgr;
1352 bo_gem->is_userptr = true;
1353 bo_gem->pat_index = PAT_INDEX_INVALID;
1354 bo_gem->cpu_cacheable = true;
1355 #ifdef __cplusplus
1356 bo_gem->bo.virt = alloc_uptr->addr;
1357 #else
1358 bo_gem->bo.virtual = alloc_uptr->addr;
1359 #endif
1360 /* Save the address provided by user */
1361 bo_gem->user_virtual = alloc_uptr->addr;
1362 bo_gem->tiling_mode = I915_TILING_NONE;
1363 bo_gem->swizzle_mode = I915_BIT_6_SWIZZLE_NONE;
1364 bo_gem->stride = 0;
1365
1366 DRMINITLISTHEAD(&bo_gem->name_list);
1367
1368 bo_gem->name = alloc_uptr->name;
1369 atomic_set(&bo_gem->refcount, 1);
1370 bo_gem->validate_index = -1;
1371 bo_gem->reloc_tree_fences = 0;
1372 bo_gem->used_as_reloc_target = false;
1373 bo_gem->has_error = false;
1374 bo_gem->reusable = false;
1375 bo_gem->use_48b_address_range = bufmgr_gem->bufmgr.bo_use_48b_address_range ? true : false;
1376
1377 mos_bo_gem_set_in_aperture_size(bufmgr_gem, bo_gem, 0);
1378
1379 if (bufmgr_gem->use_softpin)
1380 {
1381 mos_bo_set_softpin(&bo_gem->bo);
1382 }
1383
1384 MOS_DBG("bo_create_userptr: "
1385 "ptr %p buf %d (%s) size %ldb, stride 0x%x, tile mode %d\n",
1386 alloc_uptr->addr, bo_gem->gem_handle, bo_gem->name,
1387 alloc_uptr->size, alloc_uptr->stride, alloc_uptr->tiling_mode);
1388
1389 return &bo_gem->bo;
1390 }
1391
1392 static bool
1393 has_userptr(struct mos_bufmgr_gem *bufmgr_gem)
1394 {
1395 int ret;
1396 void *ptr;
1397 long pgsz;
1398 struct drm_i915_gem_userptr userptr;
1399
1400 pgsz = sysconf(_SC_PAGESIZE);
1401 assert(pgsz > 0);
1402
1403 ret = posix_memalign(&ptr, pgsz, pgsz);
1404 if (ret) {
1405 MOS_DBG("Failed to get a page (%ld) for userptr detection!\n",
1406 pgsz);
1407 return false;
1408 }
1409
1410 memclear(userptr);
1411 userptr.user_ptr = (__u64)(unsigned long)ptr;
1412 userptr.user_size = pgsz;
1413
1414 retry:
1415 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_USERPTR, &userptr);
1416 if (ret) {
1417 if (errno == ENODEV && userptr.flags == 0) {
1418 userptr.flags = I915_USERPTR_UNSYNCHRONIZED;
1419 goto retry;
1420 }
1421 free(ptr);
1422 return false;
1423 }
1424
1425 /* We don't release the userptr bo here as we want to keep the
1426 * kernel mm tracking alive for our lifetime. The first time we
1427 * create a userptr object the kernel has to install a mmu_notifer
1428 * which is a heavyweight operation (e.g. it requires taking all
1429 * mm_locks and stop_machine()).
1430 */
1431
1432 bufmgr_gem->userptr_active.ptr = ptr;
1433 bufmgr_gem->userptr_active.handle = userptr.handle;
1434 return true;
1435 }
1436
1437 static struct mos_linux_bo *
1438 check_bo_alloc_userptr(struct mos_bufmgr *bufmgr,
1439 struct mos_drm_bo_alloc_userptr *alloc_uptr)
1440 {
1441 if (has_userptr((struct mos_bufmgr_gem *)bufmgr))
1442 bufmgr->bo_alloc_userptr = mos_gem_bo_alloc_userptr;
1443 else
1444 bufmgr->bo_alloc_userptr = nullptr;
1445
1446 return mos_bo_alloc_userptr(bufmgr, alloc_uptr);
1447 }
1448
1449 /**
1450 * Returns a drm_intel_bo wrapping the given buffer object handle.
1451 *
1452 * This can be used when one application needs to pass a buffer object
1453 * to another.
1454 */
1455 static struct mos_linux_bo *
1456 mos_bufmgr_bo_gem_create_from_name(struct mos_bufmgr *bufmgr,
1457 const char *name,
1458 unsigned int handle)
1459 {
1460 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bufmgr;
1461 struct mos_bo_gem *bo_gem;
1462 int ret;
1463 struct drm_gem_open open_arg;
1464 struct drm_i915_gem_get_tiling get_tiling;
1465 drmMMListHead *list;
1466
1467 /* At the moment most applications only have a few named bo.
1468 * For instance, in a DRI client only the render buffers passed
1469 * between X and the client are named. And since X returns the
1470 * alternating names for the front/back buffer a linear search
1471 * provides a sufficiently fast match.
1472 */
1473 pthread_mutex_lock(&bufmgr_gem->lock);
1474 for (list = bufmgr_gem->named.next;
1475 list != &bufmgr_gem->named;
1476 list = list->next) {
1477 bo_gem = DRMLISTENTRY(struct mos_bo_gem, list, name_list);
1478 if (bo_gem->global_name == handle) {
1479 mos_gem_bo_reference(&bo_gem->bo);
1480 pthread_mutex_unlock(&bufmgr_gem->lock);
1481 return &bo_gem->bo;
1482 }
1483 }
1484
1485 memclear(open_arg);
1486 open_arg.name = handle;
1487 ret = drmIoctl(bufmgr_gem->fd,
1488 DRM_IOCTL_GEM_OPEN,
1489 &open_arg);
1490 if (ret != 0) {
1491 MOS_DBG("Couldn't reference %s handle 0x%08x: %s\n",
1492 name, handle, strerror(errno));
1493 pthread_mutex_unlock(&bufmgr_gem->lock);
1494 return nullptr;
1495 }
1496 /* Now see if someone has used a prime handle to get this
1497 * object from the kernel before by looking through the list
1498 * again for a matching gem_handle
1499 */
1500 for (list = bufmgr_gem->named.next;
1501 list != &bufmgr_gem->named;
1502 list = list->next) {
1503 bo_gem = DRMLISTENTRY(struct mos_bo_gem, list, name_list);
1504 if (bo_gem->gem_handle == open_arg.handle) {
1505 mos_gem_bo_reference(&bo_gem->bo);
1506 pthread_mutex_unlock(&bufmgr_gem->lock);
1507 return &bo_gem->bo;
1508 }
1509 }
1510
1511 bo_gem = (struct mos_bo_gem *)calloc(1, sizeof(*bo_gem));
1512 if (!bo_gem) {
1513 pthread_mutex_unlock(&bufmgr_gem->lock);
1514 return nullptr;
1515 }
1516
1517 bo_gem->bo.size = open_arg.size;
1518 bo_gem->bo.offset = 0;
1519 bo_gem->bo.offset64 = 0;
1520 #if defined(__cplusplus)
1521 bo_gem->bo.virt = nullptr;
1522 #else
1523 bo_gem->bo.virtual = nullptr;
1524 #endif
1525 bo_gem->bo.bufmgr = bufmgr;
1526 bo_gem->name = name;
1527 bo_gem->pat_index = PAT_INDEX_INVALID;
1528 bo_gem->cpu_cacheable = true;
1529 atomic_set(&bo_gem->refcount, 1);
1530 bo_gem->validate_index = -1;
1531 bo_gem->gem_handle = open_arg.handle;
1532 bo_gem->bo.handle = open_arg.handle;
1533 bo_gem->global_name = handle;
1534 bo_gem->reusable = false;
1535 bo_gem->use_48b_address_range = bufmgr_gem->bufmgr.bo_use_48b_address_range ? true : false;
1536
1537 memclear(get_tiling);
1538 if (bufmgr_gem->has_fence_reg) {
1539 get_tiling.handle = bo_gem->gem_handle;
1540 ret = drmIoctl(bufmgr_gem->fd,
1541 DRM_IOCTL_I915_GEM_GET_TILING,
1542 &get_tiling);
1543 if (ret != 0) {
1544 MOS_DBG("create_from_name: failed to get tiling: %s\n", strerror(errno));
1545 mos_gem_bo_unreference(&bo_gem->bo);
1546 pthread_mutex_unlock(&bufmgr_gem->lock);
1547 return nullptr;
1548 }
1549 }
1550 else
1551 {
1552 MOS_DBG("create_from_name: driver ignored to get tiling from kernel\n");
1553 }
1554
1555 bo_gem->tiling_mode = get_tiling.tiling_mode;
1556 bo_gem->swizzle_mode = get_tiling.swizzle_mode;
1557 /* XXX stride is unknown */
1558
1559 mos_bo_gem_set_in_aperture_size(bufmgr_gem, bo_gem, 0);
1560
1561 DRMLISTADDTAIL(&bo_gem->name_list, &bufmgr_gem->named);
1562 pthread_mutex_unlock(&bufmgr_gem->lock);
1563
1564 if (bufmgr_gem->use_softpin)
1565 {
1566 mos_bo_set_softpin(&bo_gem->bo);
1567 }
1568
1569 MOS_DBG("bo_create_from_handle: %d (%s)\n", handle, bo_gem->name);
1570
1571 return &bo_gem->bo;
1572 }
1573
1574 static void
1575 mos_gem_bo_free(struct mos_linux_bo *bo)
1576 {
1577 struct mos_bufmgr_gem *bufmgr_gem = nullptr;
1578 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
1579 struct drm_gem_close close;
1580 int ret;
1581
1582 CHK_CONDITION(bo_gem == nullptr, "bo_gem == nullptr\n", );
1583
1584 bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
1585
1586 CHK_CONDITION(bufmgr_gem == nullptr, "bufmgr_gem == nullptr\n", );
1587
1588 if (bo_gem->mem_virtual) {
1589 VG(VALGRIND_MAKE_MEM_NOACCESS(bo_gem->mem_virtual, 0));
1590 drm_munmap(bo_gem->mem_virtual, bo_gem->bo.size);
1591 bo_gem->mem_virtual = nullptr;
1592 }
1593 if (bo_gem->gtt_virtual) {
1594 VG(VALGRIND_MAKE_MEM_NOACCESS(bo_gem->gtt_virtual, 0));
1595 drm_munmap(bo_gem->gtt_virtual, bo_gem->bo.size);
1596 bo_gem->gtt_virtual = nullptr;
1597 }
1598 if (bo_gem->mem_wc_virtual) {
1599 VG(VALGRIND_MAKE_MEM_NOACCESS(bo_gem->mem_wc_virtual, 0));
1600 drm_munmap(bo_gem->mem_wc_virtual, bo_gem->bo.size);
1601 bo_gem->mem_wc_virtual = nullptr;
1602 }
1603
1604 if(bufmgr_gem->bufmgr.bo_wait_rendering && mos_gem_bo_busy(bo))
1605 {
1606 bufmgr_gem->bufmgr.bo_wait_rendering(bo);
1607 }
1608
1609 /* Close this object */
1610 memclear(close);
1611 close.handle = bo_gem->gem_handle;
1612 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_GEM_CLOSE, &close);
1613 if (ret != 0) {
1614 MOS_DBG("DRM_IOCTL_GEM_CLOSE %d failed (%s): %s\n",
1615 bo_gem->gem_handle, bo_gem->name, strerror(errno));
1616 }
1617 if (bufmgr_gem->mem_profiler_fd != -1)
1618 {
1619 snprintf(bufmgr_gem->mem_profiler_buffer, MEM_PROFILER_BUFFER_SIZE, "GEM_CLOSE, %d, %d, %lu, %d\n", getpid(), bo->handle,bo->size,bo_gem->mem_region);
1620 ret = write(bufmgr_gem->mem_profiler_fd, bufmgr_gem->mem_profiler_buffer, strnlen(bufmgr_gem->mem_profiler_buffer, MEM_PROFILER_BUFFER_SIZE));
1621 if (ret == -1)
1622 {
1623 MOS_DBG("Failed to write to %s: %s\n", bufmgr_gem->mem_profiler_path, strerror(errno));
1624 }
1625 }
1626
1627 if (bufmgr_gem->use_softpin)
1628 {
1629 /* Return the VMA for reuse */
1630 mos_gem_bo_vma_free(bo->bufmgr, bo->offset64, bo->size);
1631 }
1632
1633 free(bo);
1634 }
1635
1636 static void
1637 mos_gem_bo_mark_mmaps_incoherent(struct mos_linux_bo *bo)
1638 {
1639 #if HAVE_VALGRIND
1640 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
1641
1642 if (bo_gem->mem_virtual)
1643 VALGRIND_MAKE_MEM_NOACCESS(bo_gem->mem_virtual, bo->size);
1644
1645 if (bo_gem->gtt_virtual)
1646 VALGRIND_MAKE_MEM_NOACCESS(bo_gem->gtt_virtual, bo->size);
1647
1648 if (bo_gem->mem_wc_virtual)
1649 VALGRIND_MAKE_MEM_NOACCESS(bo_gem->mem_wc_virtual, bo->size);
1650 #endif
1651 }
1652
1653 /** Frees all cached buffers significantly older than @time. */
1654 static void
1655 mos_gem_cleanup_bo_cache(struct mos_bufmgr_gem *bufmgr_gem, time_t time)
1656 {
1657 int i;
1658
1659 if (bufmgr_gem->time == time)
1660 return;
1661
1662 for (i = 0; i < bufmgr_gem->num_buckets; i++) {
1663 struct mos_gem_bo_bucket *bucket =
1664 &bufmgr_gem->cache_bucket[i];
1665
1666 while (!DRMLISTEMPTY(&bucket->head)) {
1667 struct mos_bo_gem *bo_gem;
1668
1669 bo_gem = DRMLISTENTRY(struct mos_bo_gem,
1670 bucket->head.next, head);
1671 if (time - bo_gem->free_time <= 1)
1672 break;
1673
1674 DRMLISTDEL(&bo_gem->head);
1675
1676 mos_gem_bo_free(&bo_gem->bo);
1677 }
1678 }
1679
1680 bufmgr_gem->time = time;
1681 }
1682
1683 drm_export void
1684 mos_gem_bo_unreference_final(struct mos_linux_bo *bo, time_t time)
1685 {
1686 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
1687 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
1688 struct mos_gem_bo_bucket *bucket;
1689 int i;
1690
1691 /* Unreference all the target buffers */
1692 for (i = 0; i < bo_gem->reloc_count; i++) {
1693 if (bo_gem->reloc_target_info[i].bo != bo) {
1694 mos_gem_bo_unreference_locked_timed(bo_gem->
1695 reloc_target_info[i].bo,
1696 time);
1697 }
1698 }
1699 for (i = 0; i < bo_gem->softpin_target_count; i++)
1700 mos_gem_bo_unreference_locked_timed(bo_gem->softpin_target[i].bo,
1701 time);
1702 bo_gem->reloc_count = 0;
1703 bo_gem->used_as_reloc_target = false;
1704 bo_gem->softpin_target_count = 0;
1705 bo_gem->exec_async = false;
1706
1707 MOS_DBG("bo_unreference final: %d (%s)\n",
1708 bo_gem->gem_handle, bo_gem->name);
1709 bo_gem->pad_to_size = 0;
1710
1711 /* release memory associated with this object */
1712 if (bo_gem->reloc_target_info) {
1713 free(bo_gem->reloc_target_info);
1714 bo_gem->reloc_target_info = nullptr;
1715 }
1716 if (bo_gem->relocs) {
1717 free(bo_gem->relocs);
1718 bo_gem->relocs = nullptr;
1719 }
1720 if (bo_gem->softpin_target) {
1721 free(bo_gem->softpin_target);
1722 bo_gem->softpin_target = nullptr;
1723 bo_gem->max_softpin_target_count = 0;
1724 }
1725
1726 /* Clear any left-over mappings */
1727 if (bo_gem->map_count) {
1728 MOS_DBG("bo freed with non-zero map-count %d\n", bo_gem->map_count);
1729 bo_gem->map_count = 0;
1730 mos_gem_bo_mark_mmaps_incoherent(bo);
1731 }
1732
1733 DRMLISTDEL(&bo_gem->name_list);
1734
1735 bucket = mos_gem_bo_bucket_for_size(bufmgr_gem, bo->size);
1736 /* Put the buffer into our internal cache for reuse if we can. */
1737 if (bufmgr_gem->bo_reuse && bo_gem->reusable && bucket != nullptr &&
1738 mos_gem_bo_madvise_internal(bufmgr_gem, bo_gem,
1739 I915_MADV_DONTNEED)) {
1740 bo_gem->free_time = time;
1741
1742 bo_gem->name = nullptr;
1743 bo_gem->validate_index = -1;
1744
1745 DRMLISTADDTAIL(&bo_gem->head, &bucket->head);
1746 } else {
1747 mos_gem_bo_free(bo);
1748 }
1749 }
1750
1751 static void mos_gem_bo_unreference_locked_timed(struct mos_linux_bo *bo,
1752 time_t time)
1753 {
1754 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
1755
1756 assert(atomic_read(&bo_gem->refcount) > 0);
1757 if (atomic_dec_and_test(&bo_gem->refcount))
1758 mos_gem_bo_unreference_final(bo, time);
1759 }
1760
1761 static void mos_gem_bo_unreference(struct mos_linux_bo *bo)
1762 {
1763 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
1764
1765 assert(atomic_read(&bo_gem->refcount) > 0);
1766
1767 if (atomic_add_unless(&bo_gem->refcount, -1, 1)) {
1768 struct mos_bufmgr_gem *bufmgr_gem =
1769 (struct mos_bufmgr_gem *) bo->bufmgr;
1770 struct timespec time;
1771
1772 clock_gettime(CLOCK_MONOTONIC, &time);
1773
1774 pthread_mutex_lock(&bufmgr_gem->lock);
1775
1776 if (atomic_dec_and_test(&bo_gem->refcount)) {
1777 mos_gem_bo_unreference_final(bo, time.tv_sec);
1778 mos_gem_cleanup_bo_cache(bufmgr_gem, time.tv_sec);
1779 }
1780
1781 pthread_mutex_unlock(&bufmgr_gem->lock);
1782 }
1783 }
1784
1785 static int
1786 map_wc(struct mos_linux_bo *bo)
1787 {
1788 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
1789 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
1790 int ret;
1791
1792 if (bo_gem->is_userptr)
1793 return -EINVAL;
1794
1795 if (!bufmgr_gem->has_ext_mmap)
1796 return -EINVAL;
1797
1798 /* Get a mapping of the buffer if we haven't before. */
1799 if (bo_gem->mem_wc_virtual == nullptr && bufmgr_gem->has_mmap_offset) {
1800 struct drm_i915_gem_mmap_offset mmap_arg;
1801
1802 MOS_DBG("bo_map_wc: mmap_offset %d (%s), map_count=%d\n",
1803 bo_gem->gem_handle, bo_gem->name, bo_gem->map_count);
1804
1805 memclear(mmap_arg);
1806 mmap_arg.handle = bo_gem->gem_handle;
1807 /* To indicate the uncached virtual mapping to KMD */
1808 if (bufmgr_gem->has_lmem)
1809 {
1810 mmap_arg.flags = I915_MMAP_OFFSET_FIXED;
1811 }
1812 else
1813 {
1814 mmap_arg.flags = I915_MMAP_OFFSET_WC;
1815 }
1816 ret = drmIoctl(bufmgr_gem->fd,
1817 DRM_IOCTL_I915_GEM_MMAP_OFFSET,
1818 &mmap_arg);
1819 if (ret != 0) {
1820 ret = -errno;
1821 MOS_DBG("%s:%d: Error mapping buffer %d (%s): %s .\n",
1822 __FILE__, __LINE__, bo_gem->gem_handle,
1823 bo_gem->name, strerror(errno));
1824 return ret;
1825 }
1826
1827 /* and mmap it */
1828 bo_gem->mem_wc_virtual = drm_mmap(0, bo->size, PROT_READ | PROT_WRITE,
1829 MAP_SHARED, bufmgr_gem->fd,
1830 mmap_arg.offset);
1831 if (bo_gem->mem_wc_virtual == MAP_FAILED) {
1832 bo_gem->mem_wc_virtual = nullptr;
1833 ret = -errno;
1834 MOS_DBG("%s:%d: Error mapping buffer %d (%s): %s .\n",
1835 __FILE__, __LINE__,
1836 bo_gem->gem_handle, bo_gem->name,
1837 strerror(errno));
1838 }
1839 }
1840 else if (bo_gem->mem_wc_virtual == nullptr) {
1841 struct drm_i915_gem_mmap mmap_arg;
1842
1843 MOS_DBG("bo_map_wc: mmap %d (%s), map_count=%d\n",
1844 bo_gem->gem_handle, bo_gem->name, bo_gem->map_count);
1845
1846 memclear(mmap_arg);
1847 mmap_arg.handle = bo_gem->gem_handle;
1848 /* To indicate the uncached virtual mapping to KMD */
1849 mmap_arg.flags = I915_MMAP_WC;
1850 mmap_arg.size = bo->size;
1851 ret = drmIoctl(bufmgr_gem->fd,
1852 DRM_IOCTL_I915_GEM_MMAP,
1853 &mmap_arg);
1854 if (ret != 0) {
1855 ret = -errno;
1856 MOS_DBG("%s:%d: Error mapping buffer %d (%s): %s .\n",
1857 __FILE__, __LINE__, bo_gem->gem_handle,
1858 bo_gem->name, strerror(errno));
1859 return ret;
1860 }
1861 VG(VALGRIND_MALLOCLIKE_BLOCK(mmap_arg.addr_ptr, mmap_arg.size, 0, 1));
1862 bo_gem->mem_wc_virtual = (void *)(uintptr_t) mmap_arg.addr_ptr;
1863 }
1864 #ifdef __cplusplus
1865 bo->virt = bo_gem->mem_wc_virtual;
1866 #else
1867 bo->virtual = bo_gem->mem_wc_virtual;
1868 #endif
1869
1870 MOS_DBG("bo_map_wc: %d (%s) -> %p\n", bo_gem->gem_handle, bo_gem->name,
1871 bo_gem->mem_wc_virtual);
1872
1873 return 0;
1874 }
1875
1876 /* To be used in a similar way to mmap_gtt */
1877 drm_export int
1878 mos_gem_bo_map_wc(struct mos_linux_bo *bo) {
1879 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
1880 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
1881 struct drm_i915_gem_set_domain set_domain;
1882 struct drm_i915_gem_wait wait;
1883 int ret;
1884
1885 pthread_mutex_lock(&bufmgr_gem->lock);
1886
1887 ret = map_wc(bo);
1888 if (ret) {
1889 pthread_mutex_unlock(&bufmgr_gem->lock);
1890 return ret;
1891 }
1892
1893 if (bufmgr_gem->has_lmem) {
1894 assert(bufmgr_gem->has_wait_timeout);
1895 memclear(wait);
1896 wait.bo_handle = bo_gem->gem_handle;
1897 wait.timeout_ns = -1; // infinite wait
1898 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_WAIT, &wait);
1899 if (ret == -1) {
1900 MOS_DBG("%s:%d: DRM_IOCTL_I915_GEM_WAIT failed (%d)\n",
1901 __FILE__, __LINE__, errno);
1902 }
1903 } else {
1904 /* Now move it to the GTT domain so that the GPU and CPU
1905 * caches are flushed and the GPU isn't actively using the
1906 * buffer.
1907 *
1908 * The domain change is done even for the objects which
1909 * are not bounded. For them first the pages are acquired,
1910 * before the domain change.
1911 */
1912 memclear(set_domain);
1913 set_domain.handle = bo_gem->gem_handle;
1914 set_domain.read_domains = I915_GEM_DOMAIN_GTT;
1915 set_domain.write_domain = I915_GEM_DOMAIN_GTT;
1916 ret = drmIoctl(bufmgr_gem->fd,
1917 DRM_IOCTL_I915_GEM_SET_DOMAIN,
1918 &set_domain);
1919 if (ret != 0) {
1920 MOS_DBG("%s:%d: Error setting domain %d: %s\n",
1921 __FILE__, __LINE__, bo_gem->gem_handle,
1922 strerror(errno));
1923 }
1924 }
1925
1926 mos_gem_bo_mark_mmaps_incoherent(bo);
1927 VG(VALGRIND_MAKE_MEM_DEFINED(bo_gem->mem_wc_virtual, bo->size));
1928 pthread_mutex_unlock(&bufmgr_gem->lock);
1929
1930 return 0;
1931 }
1932
1933 drm_export int mos_gem_bo_map(struct mos_linux_bo *bo, int write_enable)
1934 {
1935 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
1936 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
1937 int ret;
1938
1939 if (bo_gem->is_userptr) {
1940 /* Return the same user ptr */
1941 #ifdef __cplusplus
1942 bo->virt = bo_gem->user_virtual;
1943 #else
1944 bo->virtual = bo_gem->user_virtual;
1945 #endif
1946 return 0;
1947 }
1948 /* If cpu cacheable is false, it means bo is Non-Coherent. */
1949 if (!bo_gem->cpu_cacheable)
1950 {
1951 return mos_gem_bo_map_wc(bo);
1952 }
1953
1954 pthread_mutex_lock(&bufmgr_gem->lock);
1955
1956 if (bufmgr_gem->has_mmap_offset) {
1957 struct drm_i915_gem_wait wait;
1958
1959 if (!bo_gem->mem_virtual) {
1960 struct drm_i915_gem_mmap_offset mmap_arg;
1961
1962 MOS_DBG("bo_map: %d (%s), map_count=%d\n",
1963 bo_gem->gem_handle, bo_gem->name, bo_gem->map_count);
1964
1965 memclear(mmap_arg);
1966 mmap_arg.handle = bo_gem->gem_handle;
1967 if (bufmgr_gem->has_lmem)
1968 {
1969 mmap_arg.flags = I915_MMAP_OFFSET_FIXED;
1970 }
1971 else
1972 {
1973 mmap_arg.flags = I915_MMAP_OFFSET_WB;
1974 }
1975 ret = drmIoctl(bufmgr_gem->fd,
1976 DRM_IOCTL_I915_GEM_MMAP_OFFSET,
1977 &mmap_arg);
1978 if (ret != 0) {
1979 ret = -errno;
1980 MOS_DBG("%s:%d: Error mapping buffer %d (%s): %s .\n",
1981 __FILE__, __LINE__, bo_gem->gem_handle,
1982 bo_gem->name, strerror(errno));
1983 pthread_mutex_unlock(&bufmgr_gem->lock);
1984 return ret;
1985 }
1986
1987 /* and mmap it */
1988 bo_gem->mem_virtual = drm_mmap(0, bo->size, PROT_READ | PROT_WRITE,
1989 MAP_SHARED, bufmgr_gem->fd,
1990 mmap_arg.offset);
1991 if (bo_gem->mem_virtual == MAP_FAILED) {
1992 bo_gem->mem_virtual = nullptr;
1993 ret = -errno;
1994 MOS_DBG("%s:%d: Error mapping buffer %d (%s): %s .\n",
1995 __FILE__, __LINE__,
1996 bo_gem->gem_handle, bo_gem->name,
1997 strerror(errno));
1998 }
1999 }
2000
2001 assert(bufmgr_gem->has_wait_timeout);
2002 memclear(wait);
2003 wait.bo_handle = bo_gem->gem_handle;
2004 wait.timeout_ns = -1; // infinite wait
2005 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_WAIT, &wait);
2006 if (ret == -1) {
2007 MOS_DBG("%s:%d: DRM_IOCTL_I915_GEM_WAIT failed (%d)\n",
2008 __FILE__, __LINE__, errno);
2009 }
2010 } else { /*!has_mmap_offset*/
2011 struct drm_i915_gem_set_domain set_domain;
2012
2013 if (!bo_gem->mem_virtual) {
2014 struct drm_i915_gem_mmap mmap_arg;
2015
2016 MOS_DBG("bo_map: %d (%s), map_count=%d\n",
2017 bo_gem->gem_handle, bo_gem->name, bo_gem->map_count);
2018
2019 memclear(mmap_arg);
2020 mmap_arg.handle = bo_gem->gem_handle;
2021 mmap_arg.size = bo->size;
2022 ret = drmIoctl(bufmgr_gem->fd,
2023 DRM_IOCTL_I915_GEM_MMAP,
2024 &mmap_arg);
2025 if (ret != 0) {
2026 ret = -errno;
2027 MOS_DBG("%s:%d: Error mapping buffer %d (%s): %s .\n",
2028 __FILE__, __LINE__, bo_gem->gem_handle,
2029 bo_gem->name, strerror(errno));
2030 pthread_mutex_unlock(&bufmgr_gem->lock);
2031 return ret;
2032 }
2033 VG(VALGRIND_MALLOCLIKE_BLOCK(mmap_arg.addr_ptr, mmap_arg.size, 0, 1));
2034 bo_gem->mem_virtual = (void *)(uintptr_t) mmap_arg.addr_ptr;
2035 }
2036
2037 memclear(set_domain);
2038 set_domain.handle = bo_gem->gem_handle;
2039 set_domain.read_domains = I915_GEM_DOMAIN_CPU;
2040 if (write_enable)
2041 set_domain.write_domain = I915_GEM_DOMAIN_CPU;
2042 else
2043 set_domain.write_domain = 0;
2044 ret = drmIoctl(bufmgr_gem->fd,
2045 DRM_IOCTL_I915_GEM_SET_DOMAIN,
2046 &set_domain);
2047 if (ret != 0) {
2048 MOS_DBG("%s:%d: Error setting to CPU domain %d: %s\n",
2049 __FILE__, __LINE__, bo_gem->gem_handle,
2050 strerror(errno));
2051 }
2052 }
2053 MOS_DBG("bo_map: %d (%s) -> %p\n", bo_gem->gem_handle, bo_gem->name,
2054 bo_gem->mem_virtual);
2055 #ifdef __cplusplus
2056 bo->virt = bo_gem->mem_virtual;
2057 #else
2058 bo->virtual = bo_gem->mem_virtual;
2059 #endif
2060
2061 if (write_enable)
2062 bo_gem->mapped_cpu_write = true;
2063
2064 mos_gem_bo_mark_mmaps_incoherent(bo);
2065 VG(VALGRIND_MAKE_MEM_DEFINED(bo_gem->mem_virtual, bo->size));
2066 pthread_mutex_unlock(&bufmgr_gem->lock);
2067
2068 return 0;
2069 }
2070
2071 drm_export int
2072 map_gtt(struct mos_linux_bo *bo)
2073 {
2074 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
2075 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2076 int ret;
2077
2078 if (bo_gem->is_userptr)
2079 return -EINVAL;
2080
2081 /* Get a mapping of the buffer if we haven't before. */
2082 if (bo_gem->gtt_virtual == nullptr) {
2083 __u64 offset = 0;
2084 if (bufmgr_gem->has_lmem) {
2085 struct drm_i915_gem_mmap_offset mmap_arg;
2086
2087 MOS_DBG("map_gtt: mmap_offset %d (%s), map_count=%d\n",
2088 bo_gem->gem_handle, bo_gem->name, bo_gem->map_count);
2089
2090 memclear(mmap_arg);
2091 mmap_arg.handle = bo_gem->gem_handle;
2092 mmap_arg.flags = I915_MMAP_OFFSET_FIXED;
2093
2094 /* Get the fake offset back... */
2095 ret = drmIoctl(bufmgr_gem->fd,
2096 DRM_IOCTL_I915_GEM_MMAP_OFFSET,
2097 &mmap_arg);
2098 offset = mmap_arg.offset;
2099 }
2100 else
2101 {
2102 struct drm_i915_gem_mmap_gtt mmap_arg;
2103
2104 MOS_DBG("bo_map_gtt: mmap %d (%s), map_count=%d\n",
2105 bo_gem->gem_handle, bo_gem->name, bo_gem->map_count);
2106
2107 memclear(mmap_arg);
2108 mmap_arg.handle = bo_gem->gem_handle;
2109
2110 /* Get the fake offset back... */
2111 ret = drmIoctl(bufmgr_gem->fd,
2112 DRM_IOCTL_I915_GEM_MMAP_GTT,
2113 &mmap_arg);
2114 offset = mmap_arg.offset;
2115 }
2116 if (ret != 0) {
2117 ret = -errno;
2118 MOS_DBG("%s:%d: Error preparing buffer map %d (%s): %s .\n",
2119 __FILE__, __LINE__,
2120 bo_gem->gem_handle, bo_gem->name,
2121 strerror(errno));
2122 return ret;
2123 }
2124
2125 /* and mmap it */
2126 bo_gem->gtt_virtual = drm_mmap(0, bo->size, PROT_READ | PROT_WRITE,
2127 MAP_SHARED, bufmgr_gem->fd,
2128 offset);
2129 if (bo_gem->gtt_virtual == MAP_FAILED) {
2130 bo_gem->gtt_virtual = nullptr;
2131 ret = -errno;
2132 MOS_DBG("%s:%d: Error mapping buffer %d (%s): %s .\n",
2133 __FILE__, __LINE__,
2134 bo_gem->gem_handle, bo_gem->name,
2135 strerror(errno));
2136 return ret;
2137 }
2138 }
2139 #ifdef __cplusplus
2140 bo->virt = bo_gem->gtt_virtual;
2141 #else
2142 bo->virtual = bo_gem->gtt_virtual;
2143 #endif
2144
2145 MOS_DBG("bo_map_gtt: %d (%s) -> %p\n", bo_gem->gem_handle, bo_gem->name,
2146 bo_gem->gtt_virtual);
2147
2148 return 0;
2149 }
2150
2151 static int
2152 mos_gem_bo_map_gtt(struct mos_linux_bo *bo)
2153 {
2154 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
2155 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2156 struct drm_i915_gem_set_domain set_domain;
2157 struct drm_i915_gem_wait wait;
2158 int ret;
2159
2160 pthread_mutex_lock(&bufmgr_gem->lock);
2161
2162 ret = map_gtt(bo);
2163 if (ret) {
2164 pthread_mutex_unlock(&bufmgr_gem->lock);
2165 return ret;
2166 }
2167
2168 if (bufmgr_gem->has_lmem) {
2169 assert(bufmgr_gem->has_wait_timeout);
2170 memclear(wait);
2171 wait.bo_handle = bo_gem->gem_handle;
2172 wait.timeout_ns = -1; // infinite wait
2173 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_WAIT, &wait);
2174 if (ret == -1) {
2175 MOS_DBG("%s:%d: DRM_IOCTL_I915_GEM_WAIT failed (%d)\n",
2176 __FILE__, __LINE__, errno);
2177 }
2178 } else {
2179 /* Now move it to the GTT domain so that the GPU and CPU
2180 * caches are flushed and the GPU isn't actively using the
2181 * buffer.
2182 *
2183 * The pagefault handler does this domain change for us when
2184 * it has unbound the BO from the GTT, but it's up to us to
2185 * tell it when we're about to use things if we had done
2186 * rendering and it still happens to be bound to the GTT.
2187 */
2188 memclear(set_domain);
2189 set_domain.handle = bo_gem->gem_handle;
2190 set_domain.read_domains = I915_GEM_DOMAIN_GTT;
2191 set_domain.write_domain = I915_GEM_DOMAIN_GTT;
2192 ret = drmIoctl(bufmgr_gem->fd,
2193 DRM_IOCTL_I915_GEM_SET_DOMAIN,
2194 &set_domain);
2195 if (ret != 0) {
2196 MOS_DBG("%s:%d: Error setting domain %d: %s\n",
2197 __FILE__, __LINE__, bo_gem->gem_handle,
2198 strerror(errno));
2199 }
2200 }
2201 mos_gem_bo_mark_mmaps_incoherent(bo);
2202 VG(VALGRIND_MAKE_MEM_DEFINED(bo_gem->gtt_virtual, bo->size));
2203 pthread_mutex_unlock(&bufmgr_gem->lock);
2204
2205 return 0;
2206 }
2207
2208 /**
2209 * Performs a mapping of the buffer object like the normal GTT
2210 * mapping, but avoids waiting for the GPU to be done reading from or
2211 * rendering to the buffer.
2212 *
2213 * This is used in the implementation of GL_ARB_map_buffer_range: The
2214 * user asks to create a buffer, then does a mapping, fills some
2215 * space, runs a drawing command, then asks to map it again without
2216 * synchronizing because it guarantees that it won't write over the
2217 * data that the GPU is busy using (or, more specifically, that if it
2218 * does write over the data, it acknowledges that rendering is
2219 * undefined).
2220 */
2221
2222 static int
2223 mos_gem_bo_map_unsynchronized(struct mos_linux_bo *bo)
2224 {
2225 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
2226 #ifdef HAVE_VALGRIND
2227 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2228 #endif
2229 int ret;
2230
2231 /* If the CPU cache isn't coherent with the GTT, then use a
2232 * regular synchronized mapping. The problem is that we don't
2233 * track where the buffer was last used on the CPU side in
2234 * terms of drm_intel_bo_map vs drm_intel_gem_bo_map_gtt, so
2235 * we would potentially corrupt the buffer even when the user
2236 * does reasonable things.
2237 */
2238 if (!bufmgr_gem->has_llc)
2239 return mos_gem_bo_map_gtt(bo);
2240
2241 pthread_mutex_lock(&bufmgr_gem->lock);
2242
2243 ret = map_gtt(bo);
2244 if (ret == 0) {
2245 mos_gem_bo_mark_mmaps_incoherent(bo);
2246 VG(VALGRIND_MAKE_MEM_DEFINED(bo_gem->gtt_virtual, bo->size));
2247 }
2248
2249 pthread_mutex_unlock(&bufmgr_gem->lock);
2250
2251 return ret;
2252 }
2253
2254 static int mos_gem_bo_unmap(struct mos_linux_bo *bo)
2255 {
2256 struct mos_bufmgr_gem *bufmgr_gem;
2257 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2258 int ret = 0;
2259
2260 if (bo == nullptr)
2261 return 0;
2262
2263 if (bo_gem->is_userptr)
2264 return 0;
2265
2266 bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
2267
2268 pthread_mutex_lock(&bufmgr_gem->lock);
2269
2270 if (bo_gem->map_count <= 0) {
2271 MOS_DBG("attempted to unmap an unmapped bo\n");
2272 pthread_mutex_unlock(&bufmgr_gem->lock);
2273 /* Preserve the old behaviour of just treating this as a
2274 * no-op rather than reporting the error.
2275 */
2276 return 0;
2277 }
2278
2279 if (bo_gem->mapped_cpu_write) {
2280 struct drm_i915_gem_sw_finish sw_finish;
2281
2282 /* Cause a flush to happen if the buffer's pinned for
2283 * scanout, so the results show up in a timely manner.
2284 * Unlike GTT set domains, this only does work if the
2285 * buffer should be scanout-related.
2286 */
2287 memclear(sw_finish);
2288 sw_finish.handle = bo_gem->gem_handle;
2289 ret = drmIoctl(bufmgr_gem->fd,
2290 DRM_IOCTL_I915_GEM_SW_FINISH,
2291 &sw_finish);
2292 ret = ret == -1 ? -errno : 0;
2293
2294 bo_gem->mapped_cpu_write = false;
2295 }
2296
2297 /* We need to unmap after every innovation as we cannot track
2298 * an open vma for every bo as that will exhaasut the system
2299 * limits and cause later failures.
2300 */
2301 if (--bo_gem->map_count == 0) {
2302 mos_gem_bo_mark_mmaps_incoherent(bo);
2303 #ifdef __cplusplus
2304 bo->virt = nullptr;
2305 #else
2306 bo->virtual = nullptr;
2307 #endif
2308 }
2309 pthread_mutex_unlock(&bufmgr_gem->lock);
2310
2311 return ret;
2312 }
2313
2314 static int
2315 mos_gem_bo_unmap_wc(struct mos_linux_bo *bo)
2316 {
2317 return mos_gem_bo_unmap(bo);
2318 }
2319
2320 static int
2321 mos_gem_bo_unmap_gtt(struct mos_linux_bo *bo)
2322 {
2323 return mos_gem_bo_unmap(bo);
2324 }
2325
2326 /** Waits for all GPU rendering with the object to have completed. */
2327 static void
2328 mos_gem_bo_wait_rendering(struct mos_linux_bo *bo)
2329 {
2330 mos_gem_bo_start_gtt_access(bo, 1);
2331 }
2332
2333 /**
2334 * Waits on a BO for the given amount of time.
2335 *
2336 * @bo: buffer object to wait for
2337 * @timeout_ns: amount of time to wait in nanoseconds.
2338 * If value is less than 0, an infinite wait will occur.
2339 *
2340 * Returns 0 if the wait was successful ie. the last batch referencing the
2341 * object has completed within the allotted time. Otherwise some negative return
2342 * value describes the error. Of particular interest is -ETIME when the wait has
2343 * failed to yield the desired result.
2344 *
2345 * Similar to drm_intel_gem_bo_wait_rendering except a timeout parameter allows
2346 * the operation to give up after a certain amount of time. Another subtle
2347 * difference is the internal locking semantics are different (this variant does
2348 * not hold the lock for the duration of the wait). This makes the wait subject
2349 * to a larger userspace race window.
2350 *
2351 * The implementation shall wait until the object is no longer actively
2352 * referenced within a batch buffer at the time of the call. The wait will
2353 * not guarantee that the buffer is re-issued via another thread, or an flinked
2354 * handle. Userspace must make sure this race does not occur if such precision
2355 * is important.
2356 *
2357 * Note that some kernels have broken the inifite wait for negative values
2358 * promise, upgrade to latest stable kernels if this is the case.
2359 */
2360 drm_export int
2361 mos_gem_bo_wait(struct mos_linux_bo *bo, int64_t timeout_ns)
2362 {
2363
2364 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
2365 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2366 struct drm_i915_gem_wait wait;
2367 int ret;
2368
2369 if (!bufmgr_gem->has_wait_timeout) {
2370 MOS_DBG("%s:%d: Timed wait is not supported. Falling back to "
2371 "infinite wait\n", __FILE__, __LINE__);
2372 if (timeout_ns) {
2373 mos_gem_bo_wait_rendering(bo);
2374 return 0;
2375 } else {
2376 return mos_gem_bo_busy(bo) ? -ETIME : 0;
2377 }
2378 }
2379
2380 memclear(wait);
2381 wait.bo_handle = bo_gem->gem_handle;
2382 wait.timeout_ns = timeout_ns;
2383 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_WAIT, &wait);
2384 if (ret == -1)
2385 return -errno;
2386
2387 return ret;
2388 }
2389
2390 /**
2391 * Sets the object to the GTT read and possibly write domain, used by the X
2392 * 2D driver in the absence of kernel support to do drm_intel_gem_bo_map_gtt().
2393 *
2394 * In combination with drm_intel_gem_bo_pin() and manual fence management, we
2395 * can do tiled pixmaps this way.
2396 */
2397 static void
2398 mos_gem_bo_start_gtt_access(struct mos_linux_bo *bo, int write_enable)
2399 {
2400 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
2401 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2402 struct drm_i915_gem_set_domain set_domain;
2403 struct drm_i915_gem_wait wait;
2404 int ret;
2405
2406 if (bufmgr_gem->has_lmem) {
2407 assert(bufmgr_gem->has_wait_timeout);
2408 memclear(wait);
2409 wait.bo_handle = bo_gem->gem_handle;
2410 wait.timeout_ns = -1; // infinite wait
2411 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_WAIT, &wait);
2412 if (ret == -1) {
2413 MOS_DBG("%s:%d: DRM_IOCTL_I915_GEM_WAIT failed (%d)\n",
2414 __FILE__, __LINE__, errno);
2415 }
2416 } else {
2417 memclear(set_domain);
2418 set_domain.handle = bo_gem->gem_handle;
2419 set_domain.read_domains = I915_GEM_DOMAIN_GTT;
2420 set_domain.write_domain = write_enable ? I915_GEM_DOMAIN_GTT : 0;
2421 ret = drmIoctl(bufmgr_gem->fd,
2422 DRM_IOCTL_I915_GEM_SET_DOMAIN,
2423 &set_domain);
2424 if (ret != 0) {
2425 MOS_DBG("%s:%d: Error setting memory domains %d (%08x %08x): %s .\n",
2426 __FILE__, __LINE__, bo_gem->gem_handle,
2427 set_domain.read_domains, set_domain.write_domain,
2428 strerror(errno));
2429 }
2430 }
2431 }
2432
2433 static void
2434 mos_bufmgr_cleanup_cache(struct mos_bufmgr_gem *bufmgr_gem)
2435 {
2436 for (int i = 0; i < bufmgr_gem->num_buckets; i++) {
2437 struct mos_gem_bo_bucket *bucket =
2438 &bufmgr_gem->cache_bucket[i];
2439 struct mos_bo_gem *bo_gem;
2440
2441 while (!DRMLISTEMPTY(&bucket->head)) {
2442 bo_gem = DRMLISTENTRY(struct mos_bo_gem,
2443 bucket->head.next, head);
2444 DRMLISTDEL(&bo_gem->head);
2445
2446 mos_gem_bo_free(&bo_gem->bo);
2447 }
2448 bufmgr_gem->cache_bucket[i].size = 0;
2449 }
2450 bufmgr_gem->num_buckets = 0;
2451 }
2452
2453 static void
2454 mos_bufmgr_gem_destroy(struct mos_bufmgr *bufmgr)
2455 {
2456 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bufmgr;
2457 struct drm_gem_close close_bo;
2458 int ret;
2459
2460 free(bufmgr_gem->exec2_objects);
2461 free(bufmgr_gem->exec_objects);
2462 free(bufmgr_gem->exec_bos);
2463 pthread_mutex_destroy(&bufmgr_gem->lock);
2464
2465 /* Free any cached buffer objects we were going to reuse */
2466 mos_bufmgr_cleanup_cache(bufmgr_gem);
2467
2468 /* Release userptr bo kept hanging around for optimisation. */
2469 if (bufmgr_gem->userptr_active.ptr) {
2470 memclear(close_bo);
2471 close_bo.handle = bufmgr_gem->userptr_active.handle;
2472 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_GEM_CLOSE, &close_bo);
2473 free(bufmgr_gem->userptr_active.ptr);
2474 if (ret)
2475 fprintf(stderr,
2476 "Failed to release test userptr object! (%d) "
2477 "i915 kernel driver may not be sane!\n", errno);
2478 }
2479
2480 mos_vma_heap_finish(&bufmgr_gem->vma_heap[MEMZONE_SYS]);
2481 mos_vma_heap_finish(&bufmgr_gem->vma_heap[MEMZONE_DEVICE]);
2482
2483 if (bufmgr_gem->mem_profiler_fd != -1)
2484 {
2485 close(bufmgr_gem->mem_profiler_fd);
2486 }
2487
2488 free(bufmgr);
2489 }
2490
2491 static int
2492 do_bo_emit_reloc(struct mos_linux_bo *bo, uint32_t offset,
2493 struct mos_linux_bo *target_bo, uint32_t target_offset,
2494 uint32_t read_domains, uint32_t write_domain,
2495 bool need_fence, uint64_t presumed_offset)
2496 {
2497 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
2498 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2499 struct mos_bo_gem *target_bo_gem = (struct mos_bo_gem *) target_bo;
2500
2501 if (bo_gem->has_error)
2502 return -ENOMEM;
2503
2504 if (target_bo_gem->has_error) {
2505 bo_gem->has_error = true;
2506 return -ENOMEM;
2507 }
2508
2509 /* Create a new relocation list if needed */
2510 if (bo_gem->relocs == nullptr && mos_setup_reloc_list(bo))
2511 return -ENOMEM;
2512
2513 /* Check overflow */
2514 assert(bo_gem->reloc_count < bufmgr_gem->max_relocs);
2515
2516 /* Check args */
2517 assert(offset <= bo->size - 4);
2518 assert((write_domain & (write_domain - 1)) == 0);
2519
2520 /* An object needing a fence is a tiled buffer, so it won't have
2521 * relocs to other buffers.
2522 */
2523 if (need_fence) {
2524 assert(target_bo_gem->reloc_count == 0);
2525 target_bo_gem->reloc_tree_fences = 1;
2526 }
2527
2528 /* Make sure that we're not adding a reloc to something whose size has
2529 * already been accounted for.
2530 */
2531 assert(!bo_gem->used_as_reloc_target);
2532 if (target_bo_gem != bo_gem) {
2533 target_bo_gem->used_as_reloc_target = true;
2534 bo_gem->reloc_tree_size += target_bo_gem->reloc_tree_size;
2535 bo_gem->reloc_tree_fences += target_bo_gem->reloc_tree_fences;
2536 }
2537
2538 int flags = 0;
2539 if (target_bo_gem->pad_to_size)
2540 flags |= EXEC_OBJECT_PAD_TO_SIZE;
2541 if (target_bo_gem->use_48b_address_range)
2542 flags |= EXEC_OBJECT_SUPPORTS_48B_ADDRESS;
2543 if (target_bo_gem->exec_async)
2544 flags |= EXEC_OBJECT_ASYNC;
2545 if (target_bo_gem->exec_capture)
2546 flags |= EXEC_OBJECT_CAPTURE;
2547
2548 if (target_bo != bo)
2549 mos_gem_bo_reference(target_bo);
2550
2551 bo_gem->reloc_target_info[bo_gem->reloc_count].bo = target_bo;
2552 bo_gem->reloc_target_info[bo_gem->reloc_count].flags = flags;
2553 bo_gem->relocs[bo_gem->reloc_count].offset = offset;
2554 bo_gem->relocs[bo_gem->reloc_count].delta = target_offset;
2555 bo_gem->relocs[bo_gem->reloc_count].target_handle =
2556 target_bo_gem->gem_handle;
2557 bo_gem->relocs[bo_gem->reloc_count].read_domains = read_domains;
2558 bo_gem->relocs[bo_gem->reloc_count].write_domain = write_domain;
2559 bo_gem->relocs[bo_gem->reloc_count].presumed_offset = presumed_offset;
2560 bo_gem->reloc_count++;
2561
2562 return 0;
2563 }
2564
2565 static void
2566 mos_gem_bo_use_48b_address_range(struct mos_linux_bo *bo, uint32_t enable)
2567 {
2568 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2569 bo_gem->use_48b_address_range = enable;
2570 }
2571
2572 static void
2573 mos_gem_bo_set_object_async(struct mos_linux_bo *bo)
2574 {
2575 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *)bo;
2576 bo_gem->exec_async = true;
2577 }
2578
2579 static void
2580 mos_gem_bo_set_exec_object_async(struct mos_linux_bo *bo, struct mos_linux_bo *target_bo)
2581 {
2582 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
2583 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2584 struct mos_bo_gem *target_bo_gem = (struct mos_bo_gem *) target_bo;
2585 int i;
2586 for (i = 0; i < bo_gem->reloc_count; i++)
2587 {
2588 if (bo_gem->reloc_target_info[i].bo == target_bo)
2589 {
2590 bo_gem->reloc_target_info[i].flags |= EXEC_OBJECT_ASYNC;
2591 break;
2592 }
2593 }
2594
2595 for (i = 0; i < bo_gem->softpin_target_count; i++)
2596 {
2597 if (bo_gem->softpin_target[i].bo == target_bo)
2598 {
2599 bo_gem->softpin_target[i].flags |= EXEC_OBJECT_ASYNC;
2600 break;
2601 }
2602 }
2603 }
2604
2605 static void
2606 mos_gem_bo_set_object_capture(struct mos_linux_bo *bo)
2607 {
2608 // Do nothing if bo is nullptr
2609 if (bo == nullptr)
2610 {
2611 return;
2612 }
2613 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
2614 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *)bo;
2615 if (bufmgr_gem != nullptr &&
2616 bo_gem != nullptr &&
2617 !bufmgr_gem->object_capture_disabled)
2618 {
2619 bo_gem->exec_capture = true;
2620 }
2621 }
2622
2623 static int
2624 mos_gem_bo_add_softpin_target(struct mos_linux_bo *bo, struct mos_linux_bo *target_bo, bool write_flag)
2625 {
2626 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
2627 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2628 struct mos_bo_gem *target_bo_gem = (struct mos_bo_gem *) target_bo;
2629 if (bo_gem->has_error)
2630 return -ENOMEM;
2631
2632 if (target_bo_gem->has_error) {
2633 bo_gem->has_error = true;
2634 return -ENOMEM;
2635 }
2636
2637 if (!target_bo_gem->is_softpin)
2638 return -EINVAL;
2639 if (target_bo_gem == bo_gem)
2640 return -EINVAL;
2641
2642 if (bo_gem->softpin_target_count == bo_gem->max_softpin_target_count) {
2643 int max_softpin_target_count = bo_gem->max_softpin_target_count * 2;
2644
2645 /* initial softpin target count*/
2646 if (max_softpin_target_count == 0){
2647 max_softpin_target_count = INITIAL_SOFTPIN_TARGET_COUNT;
2648 }
2649
2650 bo_gem->softpin_target = (struct mos_softpin_target *)realloc(bo_gem->softpin_target, max_softpin_target_count *
2651 sizeof(struct mos_softpin_target));
2652 if (!bo_gem->softpin_target)
2653 return -ENOMEM;
2654
2655 bo_gem->max_softpin_target_count = max_softpin_target_count;
2656 }
2657
2658 int flags = EXEC_OBJECT_PINNED;
2659 if (target_bo_gem->pad_to_size)
2660 flags |= EXEC_OBJECT_PAD_TO_SIZE;
2661 if (target_bo_gem->use_48b_address_range)
2662 flags |= EXEC_OBJECT_SUPPORTS_48B_ADDRESS;
2663 if (target_bo_gem->exec_async)
2664 flags |= EXEC_OBJECT_ASYNC;
2665 if (target_bo_gem->exec_capture)
2666 flags |= EXEC_OBJECT_CAPTURE;
2667 if (write_flag)
2668 flags |= EXEC_OBJECT_WRITE;
2669
2670 bo_gem->softpin_target[bo_gem->softpin_target_count].bo = target_bo;
2671 bo_gem->softpin_target[bo_gem->softpin_target_count].flags = flags;
2672 mos_gem_bo_reference(target_bo);
2673 bo_gem->softpin_target_count++;
2674
2675 return 0;
2676 }
2677
2678 static mos_oca_exec_list_info*
2679 mos_bufmgr_bo_get_softpin_targets_info(struct mos_linux_bo *bo, int *count)
2680 {
2681 if(bo == nullptr || count == nullptr)
2682 {
2683 return nullptr;
2684 }
2685 mos_oca_exec_list_info *info = nullptr;
2686 std::vector<int> bo_added;
2687 int counter = 0;
2688 int MAX_COUNT = 50;
2689 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
2690 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *)bo;
2691 int softpin_target_count = bo_gem->softpin_target_count;
2692 if(softpin_target_count == 0 || softpin_target_count > MAX_COUNT)
2693 {
2694 return info;
2695 }
2696 info = (mos_oca_exec_list_info *)malloc((softpin_target_count + 1) * sizeof(mos_oca_exec_list_info));
2697 if(info == nullptr)
2698 {
2699 return info;
2700 }
2701
2702 for(int i = 0; i < softpin_target_count; i++)
2703 {
2704 /*note: set capture for each bo*/
2705 struct mos_softpin_target *target = (struct mos_softpin_target *)&bo_gem->softpin_target[i];
2706 struct mos_bo_gem *target_gem = (struct mos_bo_gem *)target->bo;
2707 if(std::find(bo_added.begin(), bo_added.end(), target->bo->handle) == bo_added.end())
2708 {
2709 info[counter].handle = target->bo->handle;
2710 info[counter].size = target->bo->size;
2711 info[counter].offset64 = target->bo->offset64;
2712
2713 if (!bufmgr_gem->object_capture_disabled)
2714 target->flags |= EXEC_OBJECT_CAPTURE;
2715
2716 info[counter].flags = target->flags;
2717 info[counter].mem_region = target_gem->mem_region;
2718 info[counter].is_batch = false;
2719 bo_added.push_back(target->bo->handle);
2720 counter++;
2721 }
2722 }
2723
2724 /*note: bo is cmd bo, also need to be added*/
2725 int bb_flags = 0;
2726 if (bo_gem->pad_to_size)
2727 bb_flags |= EXEC_OBJECT_PAD_TO_SIZE;
2728 if (bo_gem->use_48b_address_range)
2729 bb_flags |= EXEC_OBJECT_SUPPORTS_48B_ADDRESS;
2730 if (bo_gem->is_softpin)
2731 bb_flags |= EXEC_OBJECT_PINNED;
2732 if (bo_gem->exec_async)
2733 bb_flags |= EXEC_OBJECT_ASYNC;
2734 if (bo_gem->exec_capture)
2735 bb_flags |= EXEC_OBJECT_CAPTURE;
2736
2737 info[counter].handle = bo->handle;
2738 info[counter].size = bo->size;
2739 info[counter].offset64 = bo->offset64;
2740 info[counter].flags = bb_flags;
2741 info[counter].mem_region = bo_gem->mem_region;
2742 info[counter].is_batch = true;
2743 counter++;
2744
2745 *count = counter;
2746
2747 return info;
2748 }
2749
2750 static int
2751 mos_gem_bo_pad_to_size(struct mos_linux_bo *bo, uint64_t pad_to_size)
2752 {
2753 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2754
2755 if (pad_to_size && pad_to_size < bo->size)
2756 return -EINVAL;
2757
2758 bo_gem->pad_to_size = pad_to_size;
2759 return 0;
2760 }
2761
2762 static int
2763 mos_gem_bo_emit_reloc(struct mos_linux_bo *bo, uint32_t offset,
2764 struct mos_linux_bo *target_bo, uint32_t target_offset,
2765 uint32_t read_domains, uint32_t write_domain,
2766 uint64_t presumed_offset)
2767 {
2768 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bo->bufmgr;
2769
2770 return do_bo_emit_reloc(bo, offset, target_bo, target_offset,
2771 read_domains, write_domain,
2772 false,
2773 presumed_offset);
2774 }
2775
2776 /**
2777 * Removes existing relocation entries in the BO after "start".
2778 *
2779 * This allows a user to avoid a two-step process for state setup with
2780 * counting up all the buffer objects and doing a
2781 * drm_intel_bufmgr_check_aperture_space() before emitting any of the
2782 * relocations for the state setup. Instead, save the state of the
2783 * batchbuffer including drm_intel_gem_get_reloc_count(), emit all the
2784 * state, and then check if it still fits in the aperture.
2785 *
2786 * Any further drm_intel_bufmgr_check_aperture_space() queries
2787 * involving this buffer in the tree are undefined after this call.
2788 *
2789 * This also removes all softpinned targets being referenced by the BO.
2790 */
2791 static void
2792 mos_gem_bo_clear_relocs(struct mos_linux_bo *bo, int start)
2793 {
2794 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
2795 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2796 int i;
2797 struct timespec time;
2798
2799 clock_gettime(CLOCK_MONOTONIC, &time);
2800
2801 assert(bo_gem->reloc_count >= start);
2802
2803 /* Unreference the cleared target buffers */
2804 pthread_mutex_lock(&bufmgr_gem->lock);
2805
2806 for (i = start; i < bo_gem->reloc_count; i++) {
2807 struct mos_bo_gem *target_bo_gem = (struct mos_bo_gem *) bo_gem->reloc_target_info[i].bo;
2808 if (&target_bo_gem->bo != bo) {
2809 bo_gem->reloc_tree_fences -= target_bo_gem->reloc_tree_fences;
2810 target_bo_gem->used_as_reloc_target = false;
2811 target_bo_gem->reloc_count = 0;
2812 mos_gem_bo_unreference_locked_timed(&target_bo_gem->bo,
2813 time.tv_sec);
2814 }
2815 }
2816 bo_gem->reloc_count = start;
2817
2818 for (i = 0; i < bo_gem->softpin_target_count; i++) {
2819 struct mos_bo_gem *target_bo_gem = (struct mos_bo_gem *) bo_gem->softpin_target[i].bo;
2820 mos_gem_bo_unreference_locked_timed(&target_bo_gem->bo, time.tv_sec);
2821 }
2822 bo_gem->softpin_target_count = 0;
2823
2824 pthread_mutex_unlock(&bufmgr_gem->lock);
2825
2826 }
2827
2828 /**
2829 * Walk the tree of relocations rooted at BO and accumulate the list of
2830 * validations to be performed and update the relocation buffers with
2831 * index values into the validation list.
2832 */
2833 static void
2834 mos_gem_bo_process_reloc(struct mos_linux_bo *bo)
2835 {
2836 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2837 int i;
2838
2839 if (bo_gem->relocs == nullptr)
2840 return;
2841
2842 for (i = 0; i < bo_gem->reloc_count; i++) {
2843 struct mos_linux_bo *target_bo = bo_gem->reloc_target_info[i].bo;
2844
2845 if (target_bo == bo)
2846 continue;
2847
2848 mos_gem_bo_mark_mmaps_incoherent(bo);
2849
2850 /* Continue walking the tree depth-first. */
2851 mos_gem_bo_process_reloc(target_bo);
2852
2853 /* Add the target to the validate list */
2854 mos_add_validate_buffer(target_bo);
2855 }
2856 }
2857
2858 static void
2859 mos_gem_bo_process_reloc2(struct mos_linux_bo *bo)
2860 {
2861 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *)bo;
2862 int i;
2863
2864 if (bo_gem->relocs == nullptr && bo_gem->softpin_target == nullptr)
2865 return;
2866
2867 for (i = 0; i < bo_gem->reloc_count; i++) {
2868 struct mos_linux_bo *target_bo = bo_gem->reloc_target_info[i].bo;
2869
2870 if (target_bo == bo)
2871 continue;
2872
2873 mos_gem_bo_mark_mmaps_incoherent(bo);
2874
2875 /* Continue walking the tree depth-first. */
2876 mos_gem_bo_process_reloc2(target_bo);
2877
2878 /* Add the target to the validate list */
2879 mos_add_reloc_objects(bo_gem->reloc_target_info[i]);
2880 }
2881
2882 for (i = 0; i < bo_gem->softpin_target_count; i++) {
2883 struct mos_linux_bo *target_bo = bo_gem->softpin_target[i].bo;
2884
2885 if (target_bo == bo)
2886 continue;
2887
2888 mos_gem_bo_mark_mmaps_incoherent(bo);
2889 mos_gem_bo_process_reloc2(target_bo);
2890 mos_add_softpin_objects(bo_gem->softpin_target[i]);
2891 }
2892 }
2893
2894 static void
2895 mos_update_buffer_offsets(struct mos_bufmgr_gem *bufmgr_gem)
2896 {
2897 int i;
2898
2899 for (i = 0; i < bufmgr_gem->exec_count; i++) {
2900 struct mos_linux_bo *bo = bufmgr_gem->exec_bos[i];
2901 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2902
2903 /* Update the buffer offset */
2904 if (bufmgr_gem->exec_objects[i].offset != bo->offset64) {
2905 MOS_DBG("BO %d (%s) migrated: 0x%08x %08x -> 0x%08x %08x\n",
2906 bo_gem->gem_handle, bo_gem->name,
2907 upper_32_bits(bo->offset64),
2908 lower_32_bits(bo->offset64),
2909 upper_32_bits(bufmgr_gem->exec_objects[i].offset),
2910 lower_32_bits(bufmgr_gem->exec_objects[i].offset));
2911 bo->offset64 = bufmgr_gem->exec_objects[i].offset;
2912 bo->offset = bufmgr_gem->exec_objects[i].offset;
2913 }
2914 }
2915 }
2916
2917 static void
2918 mos_update_buffer_offsets2 (struct mos_bufmgr_gem *bufmgr_gem, mos_linux_context *ctx, mos_linux_bo *cmd_bo)
2919 {
2920 int i;
2921
2922 for (i = 0; i < bufmgr_gem->exec_count; i++) {
2923 struct mos_linux_bo *bo = bufmgr_gem->exec_bos[i];
2924 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *)bo;
2925
2926 /* Update the buffer offset */
2927 if (bufmgr_gem->exec2_objects[i].offset != bo->offset64) {
2928 /* If we're seeing softpinned object here it means that the kernel
2929 * has relocated our object... Indicating a programming error
2930 */
2931 assert(!bo_gem->is_softpin);
2932 MOS_DBG("BO %d (%s) migrated: 0x%08x %08x -> 0x%08x %08x\n",
2933 bo_gem->gem_handle, bo_gem->name,
2934 upper_32_bits(bo->offset64),
2935 lower_32_bits(bo->offset64),
2936 upper_32_bits(bufmgr_gem->exec2_objects[i].offset),
2937 lower_32_bits(bufmgr_gem->exec2_objects[i].offset));
2938 bo->offset64 = bufmgr_gem->exec2_objects[i].offset;
2939 bo->offset = bufmgr_gem->exec2_objects[i].offset;
2940 }
2941
2942 if(!bufmgr_gem->use_softpin)
2943 {
2944 if (cmd_bo != bo) {
2945 auto item_ctx = ctx->pOsContext->contextOffsetList.begin();
2946 for (; item_ctx != ctx->pOsContext->contextOffsetList.end(); item_ctx++) {
2947 if (item_ctx->intel_context == ctx && item_ctx->target_bo == bo) {
2948 item_ctx->offset64 = bo->offset64;
2949 break;
2950 }
2951 }
2952 if ( item_ctx == ctx->pOsContext->contextOffsetList.end()) {
2953 struct MOS_CONTEXT_OFFSET newContext = {ctx,
2954 bo,
2955 bo->offset64};
2956 ctx->pOsContext->contextOffsetList.push_back(newContext);
2957 }
2958 }
2959 }
2960 }
2961 }
2962
2963 drm_export int
2964 do_exec2(struct mos_linux_bo *bo, int used, struct mos_linux_context *ctx,
2965 drm_clip_rect_t *cliprects, int num_cliprects, int DR4,
2966 unsigned int flags, int *fence
2967 )
2968 {
2969
2970 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bo->bufmgr;
2971 struct drm_i915_gem_execbuffer2 execbuf;
2972 int ret = 0;
2973 int i;
2974
2975 if (to_bo_gem(bo)->has_error)
2976 return -ENOMEM;
2977
2978 switch (flags & 0x7) {
2979 default:
2980 return -EINVAL;
2981 case I915_EXEC_BLT:
2982 if (!bufmgr_gem->has_blt)
2983 return -EINVAL;
2984 break;
2985 case I915_EXEC_BSD:
2986 if (!bufmgr_gem->has_bsd)
2987 return -EINVAL;
2988 break;
2989 case I915_EXEC_VEBOX:
2990 if (!bufmgr_gem->has_vebox)
2991 return -EINVAL;
2992 break;
2993 case I915_EXEC_RENDER:
2994 case I915_EXEC_DEFAULT:
2995 break;
2996 }
2997
2998 pthread_mutex_lock(&bufmgr_gem->lock);
2999 /* Update indices and set up the validate list. */
3000 mos_gem_bo_process_reloc2(bo);
3001
3002 /* Add the batch buffer to the validation list. There are no relocations
3003 * pointing to it.
3004 */
3005 mos_add_validate_buffer2(bo, 0);
3006
3007 memclear(execbuf);
3008 execbuf.buffers_ptr = (uintptr_t)bufmgr_gem->exec2_objects;
3009 execbuf.buffer_count = bufmgr_gem->exec_count;
3010 execbuf.batch_start_offset = 0;
3011 execbuf.batch_len = used;
3012 execbuf.cliprects_ptr = (uintptr_t)cliprects;
3013 execbuf.num_cliprects = num_cliprects;
3014 execbuf.DR1 = 0;
3015 execbuf.DR4 = DR4;
3016 execbuf.flags = flags;
3017 if (ctx == nullptr)
3018 i915_execbuffer2_set_context_id(execbuf, 0);
3019 else
3020 i915_execbuffer2_set_context_id(execbuf, ctx->ctx_id);
3021 execbuf.rsvd2 = 0;
3022 if(flags & I915_EXEC_FENCE_SUBMIT)
3023 {
3024 execbuf.rsvd2 = *fence;
3025 }
3026 if(flags & I915_EXEC_FENCE_OUT)
3027 {
3028 execbuf.rsvd2 = -1;
3029 }
3030
3031 if (bufmgr_gem->no_exec)
3032 goto skip_execution;
3033
3034 ret = drmIoctl(bufmgr_gem->fd,
3035 DRM_IOCTL_I915_GEM_EXECBUFFER2_WR,
3036 &execbuf);
3037 if (ret != 0) {
3038 ret = -errno;
3039 if (ret == -ENOSPC) {
3040 MOS_DBG("Execbuffer fails to pin. "
3041 "Estimate: %u. Actual: %u. Available: %u\n",
3042 mos_gem_estimate_batch_space(bufmgr_gem->exec_bos,
3043 bufmgr_gem->exec_count),
3044 mos_gem_compute_batch_space(bufmgr_gem->exec_bos,
3045 bufmgr_gem->exec_count),
3046 (unsigned int) bufmgr_gem->gtt_size);
3047 }
3048 }
3049
3050 if (ctx != nullptr)
3051 {
3052 mos_update_buffer_offsets2(bufmgr_gem, ctx, bo);
3053 }
3054
3055 if(flags & I915_EXEC_FENCE_OUT)
3056 {
3057 *fence = execbuf.rsvd2 >> 32;
3058 }
3059
3060 skip_execution:
3061 if (bufmgr_gem->bufmgr.debug)
3062 mos_gem_dump_validation_list(bufmgr_gem);
3063
3064 for (i = 0; i < bufmgr_gem->exec_count; i++) {
3065 struct mos_bo_gem *bo_gem = to_bo_gem(bufmgr_gem->exec_bos[i]);
3066
3067 bo_gem->idle = false;
3068
3069 /* Disconnect the buffer from the validate list */
3070 bo_gem->validate_index = -1;
3071 bufmgr_gem->exec_bos[i] = nullptr;
3072 }
3073 bufmgr_gem->exec_count = 0;
3074 pthread_mutex_unlock(&bufmgr_gem->lock);
3075
3076 return ret;
3077 }
3078
3079 static int
3080 do_exec3(struct mos_linux_bo **bo, int _num_bo, struct mos_linux_context *ctx,
3081 drm_clip_rect_t *cliprects, int num_cliprects, int DR4,
3082 unsigned int _flags, int *fence
3083 )
3084 {
3085 uint64_t flags = _flags;
3086 uint64_t num_bo = _num_bo;
3087 if((bo == nullptr) || (ctx == nullptr) || (num_bo == 0))
3088 {
3089 return -EINVAL;
3090 }
3091
3092 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bo[0]->bufmgr;
3093 struct drm_i915_gem_execbuffer2 execbuf;
3094 int ret = 0;
3095 int i;
3096
3097 pthread_mutex_lock(&bufmgr_gem->lock);
3098
3099 struct mos_exec_info exec_info;
3100 memset(static_cast<void*>(&exec_info), 0, sizeof(exec_info));
3101 exec_info.batch_obj = (struct drm_i915_gem_exec_object2 *) calloc (num_bo, sizeof(struct drm_i915_gem_exec_object2));
3102 if(exec_info.batch_obj == nullptr)
3103 {
3104 ret = -ENOMEM;
3105 goto skip_execution;
3106 }
3107 exec_info.obj_remain_size = OBJ512_SIZE;
3108 exec_info.obj = (struct drm_i915_gem_exec_object2 *) calloc (OBJ512_SIZE, sizeof(struct drm_i915_gem_exec_object2));
3109 if(exec_info.obj == nullptr)
3110 {
3111 ret = -ENOMEM;
3112 goto skip_execution;
3113 }
3114
3115 for(i = 0; i < num_bo; i++)
3116 {
3117 if (to_bo_gem(bo[i])->has_error)
3118 {
3119 ret = -ENOMEM;
3120 goto skip_execution;
3121 }
3122
3123 /* Update indices and set up the validate list. */
3124 mos_gem_bo_process_reloc2(bo[i]);
3125
3126 /* Add the batch buffer to the validation list. There are no relocations
3127 * pointing to it.
3128 */
3129 mos_add_validate_buffer2(bo[i], 0);
3130
3131 if((bufmgr_gem->exec_count - 1 + num_bo) > exec_info.obj_remain_size)
3132 {
3133 // origin size + OBJ512_SIZE + obj_count + batch_count;
3134 uint32_t new_obj_size = exec_info.obj_count + exec_info.obj_remain_size + OBJ512_SIZE + bufmgr_gem->exec_count - 1 + num_bo;
3135 struct drm_i915_gem_exec_object2 *new_obj = (struct drm_i915_gem_exec_object2 *)realloc(exec_info.obj, new_obj_size * sizeof(struct drm_i915_gem_exec_object2));
3136 if(new_obj == nullptr)
3137 {
3138 ret = -ENOMEM;
3139 goto skip_execution;
3140 }
3141 exec_info.obj_remain_size = new_obj_size - exec_info.obj_count;
3142 exec_info.obj = new_obj;
3143 }
3144 if(0 == i)
3145 {
3146 uint32_t cp_size = (bufmgr_gem->exec_count - 1) * sizeof(struct drm_i915_gem_exec_object2);
3147 memcpy(exec_info.obj, bufmgr_gem->exec2_objects, cp_size);
3148 exec_info.obj_count += (bufmgr_gem->exec_count - 1);
3149 exec_info.obj_remain_size -= (bufmgr_gem->exec_count - 1);
3150 }
3151 else
3152 {
3153 for(int e2 = 0; e2 < bufmgr_gem->exec_count - 1; e2++)
3154 {
3155 int e1;
3156 for(e1 = 0; e1 < exec_info.obj_count; e1++)
3157 {
3158 // skip the duplicated bo if it is already in the list of exec_info.obj
3159 if(bufmgr_gem->exec2_objects[e2].handle == exec_info.obj[e1].handle)
3160 {
3161 break;
3162 }
3163 }
3164 //if no duplicated bo found, add it into list of exec_info.obj
3165 if(e1 == exec_info.obj_count)
3166 {
3167 exec_info.obj[exec_info.obj_count] = bufmgr_gem->exec2_objects[e2];
3168 exec_info.obj_count++;
3169 exec_info.obj_remain_size--;
3170 }
3171 }
3172 }
3173 memcpy(&exec_info.batch_obj[i], &bufmgr_gem->exec2_objects[bufmgr_gem->exec_count - 1], sizeof(struct drm_i915_gem_exec_object2));
3174 exec_info.batch_count++;
3175 uint32_t reloc_count = bufmgr_gem->exec2_objects[bufmgr_gem->exec_count - 1].relocation_count;
3176 uint32_t cp_size = (reloc_count * sizeof(struct drm_i915_gem_relocation_entry));
3177
3178 struct drm_i915_gem_relocation_entry* ptr_reloc = (struct drm_i915_gem_relocation_entry *)calloc(reloc_count,sizeof(struct drm_i915_gem_relocation_entry));
3179 if(ptr_reloc == nullptr)
3180 {
3181 ret = -ENOMEM;
3182 goto skip_execution;
3183 }
3184 memcpy(ptr_reloc, (struct drm_i915_gem_relocation_entry *)bufmgr_gem->exec2_objects[bufmgr_gem->exec_count - 1].relocs_ptr, cp_size);
3185
3186 exec_info.batch_obj[i].relocs_ptr = (uintptr_t)ptr_reloc;
3187 exec_info.batch_obj[i].relocation_count = reloc_count;
3188
3189 //clear bo
3190 if (bufmgr_gem->bufmgr.debug)
3191 {
3192 mos_gem_dump_validation_list(bufmgr_gem);
3193 }
3194
3195 for (int j = 0; j < bufmgr_gem->exec_count; j++) {
3196 struct mos_bo_gem *bo_gem = to_bo_gem(bufmgr_gem->exec_bos[j]);
3197
3198 if(bo_gem)
3199 {
3200 bo_gem->idle = false;
3201
3202 /* Disconnect the buffer from the validate list */
3203 bo_gem->validate_index = -1;
3204 bufmgr_gem->exec_bos[j] = nullptr;
3205 }
3206 }
3207 bufmgr_gem->exec_count = 0;
3208 }
3209
3210 //add back batch obj to the last position
3211 for(i = 0; i < num_bo; i++)
3212 {
3213 exec_info.obj[exec_info.obj_count] = exec_info.batch_obj[i];
3214 exec_info.obj_count++;
3215 exec_info.obj_remain_size--;
3216 }
3217
3218 //save previous ptr
3219 exec_info.pSavePreviousExec2Objects = bufmgr_gem->exec2_objects;
3220 bufmgr_gem->exec_count = exec_info.obj_count;
3221 bufmgr_gem->exec2_objects = exec_info.obj;
3222
3223 memclear(execbuf);
3224 execbuf.buffers_ptr = (uintptr_t)bufmgr_gem->exec2_objects;
3225 execbuf.buffer_count = bufmgr_gem->exec_count;
3226 execbuf.batch_start_offset = 0;
3227 execbuf.cliprects_ptr = (uintptr_t)cliprects;
3228 execbuf.num_cliprects = num_cliprects;
3229 execbuf.DR1 = 0;
3230 execbuf.DR4 = DR4;
3231 execbuf.flags = flags;
3232 if (ctx == nullptr)
3233 i915_execbuffer2_set_context_id(execbuf, 0);
3234 else
3235 i915_execbuffer2_set_context_id(execbuf, ctx->ctx_id);
3236 execbuf.rsvd2 = 0;
3237 if((flags & I915_EXEC_FENCE_SUBMIT) || (flags & I915_EXEC_FENCE_IN))
3238 {
3239 execbuf.rsvd2 = *fence;
3240 }
3241 else if(flags & I915_EXEC_FENCE_OUT)
3242 {
3243 execbuf.rsvd2 = -1;
3244 }
3245
3246 if (bufmgr_gem->no_exec)
3247 goto skip_execution;
3248
3249 ret = drmIoctl(bufmgr_gem->fd,
3250 DRM_IOCTL_I915_GEM_EXECBUFFER2_WR,
3251 &execbuf);
3252
3253 if (ret != 0) {
3254 ret = -errno;
3255 if (ret == -ENOSPC) {
3256 MOS_DBG("Execbuffer fails to pin. "
3257 "Estimate: %u. Actual: %u. Available: %u\n",
3258 mos_gem_estimate_batch_space(bufmgr_gem->exec_bos,
3259 bufmgr_gem->exec_count),
3260 mos_gem_compute_batch_space(bufmgr_gem->exec_bos,
3261 bufmgr_gem->exec_count),
3262 (unsigned int) bufmgr_gem->gtt_size);
3263 }
3264 }
3265
3266 bufmgr_gem->exec2_objects = exec_info.pSavePreviousExec2Objects;
3267
3268 if(flags & I915_EXEC_FENCE_OUT)
3269 {
3270 *fence = execbuf.rsvd2 >> 32;
3271 }
3272
3273 skip_execution:
3274 if (bufmgr_gem->bufmgr.debug)
3275 mos_gem_dump_validation_list(bufmgr_gem);
3276
3277 bufmgr_gem->exec_count = 0;
3278 if(exec_info.batch_obj)
3279 {
3280 for(i = 0; i < num_bo; i++)
3281 {
3282 mos_safe_free((struct drm_i915_gem_relocation_entry *)exec_info.batch_obj[i].relocs_ptr);
3283 }
3284 }
3285 mos_safe_free(exec_info.obj);
3286 mos_safe_free(exec_info.batch_obj);
3287 pthread_mutex_unlock(&bufmgr_gem->lock);
3288
3289 return ret;
3290 }
3291
3292 static int
3293 mos_gem_bo_exec2(struct mos_linux_bo *bo, int used,
3294 drm_clip_rect_t *cliprects, int num_cliprects,
3295 int DR4)
3296 {
3297 return do_exec2(bo, used, nullptr, cliprects, num_cliprects, DR4,
3298 I915_EXEC_RENDER, nullptr);
3299 }
3300
3301 static int
3302 mos_gem_bo_mrb_exec2(struct mos_linux_bo *bo, int used,
3303 drm_clip_rect_t *cliprects, int num_cliprects, int DR4,
3304 unsigned int flags)
3305 {
3306 return do_exec2(bo, used, nullptr, cliprects, num_cliprects, DR4,
3307 flags, nullptr);
3308 }
3309
3310 static int
3311 mos_gem_bo_context_exec2(struct mos_linux_bo *bo, int used, struct mos_linux_context *ctx,
3312 drm_clip_rect_t *cliprects, int num_cliprects, int DR4,
3313 unsigned int flags, int *fence)
3314 {
3315 return do_exec2(bo, used, ctx, cliprects, num_cliprects, DR4,
3316 flags, fence);
3317 }
3318
3319 static int
3320 mos_gem_bo_context_exec3(struct mos_linux_bo **bo, int num_bo, struct mos_linux_context *ctx,
3321 struct drm_clip_rect *cliprects, int num_cliprects, int DR4,
3322 unsigned int flags, int *fence)
3323 {
3324 return do_exec3(bo, num_bo, ctx, cliprects, num_cliprects, DR4,
3325 flags, fence);
3326 }
3327
3328 static int
3329 mos_gem_bo_check_mem_region_internal(struct mos_linux_bo *bo,
3330 int mem_type)
3331 {
3332 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
3333
3334 // when re-cycle the gem_bo, need to keep the VA space is keeping consistent on memory type
3335 if (bo_gem->mem_region == I915_MEMORY_CLASS_SYSTEM &&
3336 (mem_type == MOS_MEMPOOL_VIDEOMEMORY || mem_type == MOS_MEMPOOL_DEVICEMEMORY))
3337 return -EINVAL;
3338
3339 if (bo_gem->mem_region == I915_MEMORY_CLASS_DEVICE &&
3340 (mem_type == MOS_MEMPOOL_SYSTEMMEMORY))
3341 return -EINVAL;
3342
3343 return 0;
3344 }
3345
3346 static int
3347 mos_gem_bo_set_tiling_internal(struct mos_linux_bo *bo,
3348 uint32_t tiling_mode,
3349 uint32_t stride)
3350 {
3351 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
3352 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
3353 struct drm_i915_gem_set_tiling set_tiling;
3354 int ret;
3355
3356 if (!bufmgr_gem->has_fence_reg)
3357 return 0;
3358
3359 if (bo_gem->global_name == 0 &&
3360 tiling_mode == bo_gem->tiling_mode &&
3361 stride == bo_gem->stride)
3362 return 0;
3363
3364 memset(&set_tiling, 0, sizeof(set_tiling));
3365 /* set_tiling is slightly broken and overwrites the
3366 * input on the error path, so we have to open code
3367 * rmIoctl.
3368 */
3369 set_tiling.handle = bo_gem->gem_handle;
3370 set_tiling.tiling_mode = tiling_mode;
3371 set_tiling.stride = stride;
3372
3373 ret = drmIoctl(bufmgr_gem->fd,
3374 DRM_IOCTL_I915_GEM_SET_TILING,
3375 &set_tiling);
3376 if (ret == -1)
3377 return -errno;
3378
3379 bo_gem->tiling_mode = set_tiling.tiling_mode;
3380 bo_gem->swizzle_mode = set_tiling.swizzle_mode;
3381 bo_gem->stride = set_tiling.stride;
3382 return 0;
3383 }
3384
3385 static int
3386 mos_gem_bo_set_tiling(struct mos_linux_bo *bo, uint32_t * tiling_mode,
3387 uint32_t stride)
3388 {
3389 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
3390 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
3391 int ret;
3392
3393 /* Tiling with userptr surfaces is not supported
3394 * on all hardware so refuse it for time being.
3395 */
3396 if (bo_gem->is_userptr)
3397 return -EINVAL;
3398
3399 /* Linear buffers have no stride. By ensuring that we only ever use
3400 * stride 0 with linear buffers, we simplify our code.
3401 */
3402 if (*tiling_mode == I915_TILING_NONE)
3403 stride = 0;
3404
3405 ret = mos_gem_bo_set_tiling_internal(bo, *tiling_mode, stride);
3406 if (ret == 0)
3407 mos_bo_gem_set_in_aperture_size(bufmgr_gem, bo_gem, 0);
3408 *tiling_mode = bo_gem->tiling_mode;
3409 return ret;
3410 }
3411
3412 static int
3413 mos_gem_bo_get_tiling(struct mos_linux_bo *bo, uint32_t * tiling_mode,
3414 uint32_t * swizzle_mode)
3415 {
3416 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
3417
3418 *tiling_mode = bo_gem->tiling_mode;
3419 *swizzle_mode = bo_gem->swizzle_mode;
3420 return 0;
3421 }
3422
3423 static int
3424 mos_gem_bo_set_softpin_offset(struct mos_linux_bo *bo, uint64_t offset)
3425 {
3426 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
3427
3428 bo_gem->is_softpin = true;
3429 bo->offset64 = offset;
3430 bo->offset = offset;
3431 return 0;
3432 }
3433
3434 static int
3435 mos_gem_bo_set_softpin(MOS_LINUX_BO *bo)
3436 {
3437 int ret = 0;
3438 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
3439 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
3440
3441 pthread_mutex_lock(&bufmgr_gem->lock);
3442 if (!mos_gem_bo_is_softpin(bo))
3443 {
3444 uint64_t alignment = (bufmgr_gem->softpin_va1Malign) ? PAGE_SIZE_1M : PAGE_SIZE_64K;
3445 uint64_t offset = mos_gem_bo_vma_alloc(bo->bufmgr, (enum mos_memory_zone)bo_gem->mem_region, bo->size, alignment);
3446 ret = mos_gem_bo_set_softpin_offset(bo, offset);
3447 }
3448 pthread_mutex_unlock(&bufmgr_gem->lock);
3449
3450 if (ret == 0)
3451 {
3452 ret = mos_bo_use_48b_address_range(bo, 1);
3453 }
3454
3455 return ret;
3456 }
3457
3458 static struct mos_linux_bo *
3459 mos_gem_bo_create_from_prime(struct mos_bufmgr *bufmgr, struct mos_drm_bo_alloc_prime *alloc_prime)
3460 {
3461 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bufmgr;
3462 int ret;
3463 uint32_t handle;
3464 struct mos_bo_gem *bo_gem;
3465 struct drm_i915_gem_get_tiling get_tiling;
3466 int prime_fd = alloc_prime->prime_fd;
3467 int size = alloc_prime->size;
3468 drmMMListHead *list;
3469
3470 pthread_mutex_lock(&bufmgr_gem->lock);
3471 ret = drmPrimeFDToHandle(bufmgr_gem->fd, prime_fd, &handle);
3472 if (ret) {
3473 MOS_DBG("create_from_prime: failed to obtain handle from fd: %s\n", strerror(errno));
3474 pthread_mutex_unlock(&bufmgr_gem->lock);
3475 return nullptr;
3476 }
3477
3478 /*
3479 * See if the kernel has already returned this buffer to us. Just as
3480 * for named buffers, we must not create two bo's pointing at the same
3481 * kernel object
3482 */
3483 for (list = bufmgr_gem->named.next;
3484 list != &bufmgr_gem->named;
3485 list = list->next) {
3486 bo_gem = DRMLISTENTRY(struct mos_bo_gem, list, name_list);
3487 if (bo_gem->gem_handle == handle) {
3488 mos_gem_bo_reference(&bo_gem->bo);
3489 pthread_mutex_unlock(&bufmgr_gem->lock);
3490 return &bo_gem->bo;
3491 }
3492 }
3493
3494 bo_gem = (struct mos_bo_gem *)calloc(1, sizeof(*bo_gem));
3495 if (!bo_gem) {
3496 pthread_mutex_unlock(&bufmgr_gem->lock);
3497 return nullptr;
3498 }
3499 /* Determine size of bo. The fd-to-handle ioctl really should
3500 * return the size, but it doesn't. If we have kernel 3.12 or
3501 * later, we can lseek on the prime fd to get the size. Older
3502 * kernels will just fail, in which case we fall back to the
3503 * provided (estimated or guess size). */
3504 ret = lseek(prime_fd, 0, SEEK_END);
3505 if (ret != -1)
3506 bo_gem->bo.size = ret;
3507 else
3508 bo_gem->bo.size = size;
3509
3510 bo_gem->bo.handle = handle;
3511 bo_gem->bo.bufmgr = bufmgr;
3512
3513 bo_gem->gem_handle = handle;
3514 bo_gem->pat_index = PAT_INDEX_INVALID;
3515 bo_gem->cpu_cacheable = true;
3516 atomic_set(&bo_gem->refcount, 1);
3517
3518 bo_gem->name = alloc_prime->name;
3519 bo_gem->validate_index = -1;
3520 bo_gem->reloc_tree_fences = 0;
3521 bo_gem->used_as_reloc_target = false;
3522 bo_gem->has_error = false;
3523 bo_gem->reusable = false;
3524 bo_gem->use_48b_address_range = bufmgr_gem->bufmgr.bo_use_48b_address_range ? true : false;
3525
3526 DRMLISTADDTAIL(&bo_gem->name_list, &bufmgr_gem->named);
3527 pthread_mutex_unlock(&bufmgr_gem->lock);
3528
3529 memclear(get_tiling);
3530 if(bufmgr_gem->has_fence_reg) {
3531 get_tiling.handle = bo_gem->gem_handle;
3532 ret = drmIoctl(bufmgr_gem->fd,
3533 DRM_IOCTL_I915_GEM_GET_TILING,
3534 &get_tiling);
3535 if (ret != 0) {
3536 MOS_DBG("create_from_prime: failed to get tiling: %s\n", strerror(errno));
3537 mos_gem_bo_unreference(&bo_gem->bo);
3538 return nullptr;
3539 }
3540 }
3541 else
3542 {
3543 MOS_DBG("create_from_prime: driver ignored to get tiling from kernel\n");
3544 }
3545
3546 bo_gem->tiling_mode = get_tiling.tiling_mode;
3547 bo_gem->swizzle_mode = get_tiling.swizzle_mode;
3548 /* XXX stride is unknown */
3549 mos_bo_gem_set_in_aperture_size(bufmgr_gem, bo_gem, 0);
3550 if (bufmgr_gem->use_softpin)
3551 {
3552 mos_bo_set_softpin(&bo_gem->bo);
3553 }
3554
3555 return &bo_gem->bo;
3556 }
3557
3558 static int
3559 mos_gem_bo_export_to_prime(struct mos_linux_bo *bo, int *prime_fd)
3560 {
3561 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
3562 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
3563
3564 pthread_mutex_lock(&bufmgr_gem->lock);
3565 if (DRMLISTEMPTY(&bo_gem->name_list))
3566 DRMLISTADDTAIL(&bo_gem->name_list, &bufmgr_gem->named);
3567 pthread_mutex_unlock(&bufmgr_gem->lock);
3568
3569 if (drmPrimeHandleToFD(bufmgr_gem->fd, bo_gem->gem_handle,
3570 DRM_CLOEXEC, prime_fd) != 0)
3571 return -errno;
3572
3573 bo_gem->reusable = false;
3574
3575 return 0;
3576 }
3577
3578 static int
3579 mos_gem_bo_flink(struct mos_linux_bo *bo, uint32_t * name)
3580 {
3581 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
3582 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
3583 int ret;
3584
3585 if (!bo_gem->global_name) {
3586 struct drm_gem_flink flink;
3587
3588 memclear(flink);
3589 flink.handle = bo_gem->gem_handle;
3590
3591 pthread_mutex_lock(&bufmgr_gem->lock);
3592
3593 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_GEM_FLINK, &flink);
3594 if (ret != 0) {
3595 pthread_mutex_unlock(&bufmgr_gem->lock);
3596 return -errno;
3597 }
3598
3599 bo_gem->global_name = flink.name;
3600 bo_gem->reusable = false;
3601
3602 if (DRMLISTEMPTY(&bo_gem->name_list))
3603 DRMLISTADDTAIL(&bo_gem->name_list, &bufmgr_gem->named);
3604 pthread_mutex_unlock(&bufmgr_gem->lock);
3605 }
3606
3607 *name = bo_gem->global_name;
3608 return 0;
3609 }
3610
3611 /**
3612 * Enables unlimited caching of buffer objects for reuse.
3613 *
3614 * This is potentially very memory expensive, as the cache at each bucket
3615 * size is only bounded by how many buffers of that size we've managed to have
3616 * in flight at once.
3617 */
3618 static void
3619 mos_gem_enable_reuse(struct mos_bufmgr *bufmgr)
3620 {
3621 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bufmgr;
3622
3623 bufmgr_gem->bo_reuse = true;
3624 }
3625
3626 /**
3627 * Return the additional aperture space required by the tree of buffer objects
3628 * rooted at bo.
3629 */
3630 static int
3631 mos_gem_bo_get_aperture_space(struct mos_linux_bo *bo)
3632 {
3633 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
3634 int i;
3635 int total = 0;
3636
3637 if (bo == nullptr || bo_gem->included_in_check_aperture)
3638 return 0;
3639
3640 total += bo->size;
3641 bo_gem->included_in_check_aperture = true;
3642
3643 for (i = 0; i < bo_gem->reloc_count; i++)
3644 total +=
3645 mos_gem_bo_get_aperture_space(bo_gem->
3646 reloc_target_info[i].bo);
3647
3648 return total;
3649 }
3650
3651 /**
3652 * Count the number of buffers in this list that need a fence reg
3653 *
3654 * If the count is greater than the number of available regs, we'll have
3655 * to ask the caller to resubmit a batch with fewer tiled buffers.
3656 *
3657 * This function over-counts if the same buffer is used multiple times.
3658 */
3659 static unsigned int
3660 mos_gem_total_fences(struct mos_linux_bo ** bo_array, int count)
3661 {
3662 int i;
3663 unsigned int total = 0;
3664
3665 for (i = 0; i < count; i++) {
3666 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo_array[i];
3667
3668 if (bo_gem == nullptr)
3669 continue;
3670
3671 total += bo_gem->reloc_tree_fences;
3672 }
3673 return total;
3674 }
3675
3676 /**
3677 * Clear the flag set by drm_intel_gem_bo_get_aperture_space() so we're ready
3678 * for the next drm_intel_bufmgr_check_aperture_space() call.
3679 */
3680 static void
3681 mos_gem_bo_clear_aperture_space_flag(struct mos_linux_bo *bo)
3682 {
3683 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
3684 int i;
3685
3686 if (bo == nullptr || !bo_gem->included_in_check_aperture)
3687 return;
3688
3689 bo_gem->included_in_check_aperture = false;
3690
3691 for (i = 0; i < bo_gem->reloc_count; i++)
3692 mos_gem_bo_clear_aperture_space_flag(bo_gem->
3693 reloc_target_info[i].bo);
3694 }
3695
3696 /**
3697 * Return a conservative estimate for the amount of aperture required
3698 * for a collection of buffers. This may double-count some buffers.
3699 */
3700 static unsigned int
3701 mos_gem_estimate_batch_space(struct mos_linux_bo **bo_array, int count)
3702 {
3703 int i;
3704 unsigned int total = 0;
3705
3706 for (i = 0; i < count; i++) {
3707 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo_array[i];
3708 if (bo_gem != nullptr)
3709 total += bo_gem->reloc_tree_size;
3710 }
3711 return total;
3712 }
3713
3714 /**
3715 * Return the amount of aperture needed for a collection of buffers.
3716 * This avoids double counting any buffers, at the cost of looking
3717 * at every buffer in the set.
3718 */
3719 static unsigned int
3720 mos_gem_compute_batch_space(struct mos_linux_bo **bo_array, int count)
3721 {
3722 int i;
3723 unsigned int total = 0;
3724
3725 for (i = 0; i < count; i++) {
3726 total += mos_gem_bo_get_aperture_space(bo_array[i]);
3727 /* For the first buffer object in the array, we get an
3728 * accurate count back for its reloc_tree size (since nothing
3729 * had been flagged as being counted yet). We can save that
3730 * value out as a more conservative reloc_tree_size that
3731 * avoids double-counting target buffers. Since the first
3732 * buffer happens to usually be the batch buffer in our
3733 * callers, this can pull us back from doing the tree
3734 * walk on every new batch emit.
3735 */
3736 if (i == 0) {
3737 struct mos_bo_gem *bo_gem =
3738 (struct mos_bo_gem *) bo_array[i];
3739 bo_gem->reloc_tree_size = total;
3740 }
3741 }
3742
3743 for (i = 0; i < count; i++)
3744 mos_gem_bo_clear_aperture_space_flag(bo_array[i]);
3745 return total;
3746 }
3747
3748 /**
3749 * Return -1 if the batchbuffer should be flushed before attempting to
3750 * emit rendering referencing the buffers pointed to by bo_array.
3751 *
3752 * This is required because if we try to emit a batchbuffer with relocations
3753 * to a tree of buffers that won't simultaneously fit in the aperture,
3754 * the rendering will return an error at a point where the software is not
3755 * prepared to recover from it.
3756 *
3757 * However, we also want to emit the batchbuffer significantly before we reach
3758 * the limit, as a series of batchbuffers each of which references buffers
3759 * covering almost all of the aperture means that at each emit we end up
3760 * waiting to evict a buffer from the last rendering, and we get synchronous
3761 * performance. By emitting smaller batchbuffers, we eat some CPU overhead to
3762 * get better parallelism.
3763 */
3764 static int
3765 mos_gem_check_aperture_space(struct mos_linux_bo **bo_array, int count)
3766 {
3767 struct mos_bufmgr_gem *bufmgr_gem =
3768 (struct mos_bufmgr_gem *) bo_array[0]->bufmgr;
3769 unsigned int total = 0;
3770 unsigned int threshold = bufmgr_gem->gtt_size * 3 / 4;
3771 int total_fences;
3772
3773 /* Check for fence reg constraints if necessary */
3774 if (bufmgr_gem->available_fences) {
3775 total_fences = mos_gem_total_fences(bo_array, count);
3776 if (total_fences > bufmgr_gem->available_fences)
3777 return -ENOSPC;
3778 }
3779
3780 total = mos_gem_estimate_batch_space(bo_array, count);
3781
3782 if (total > threshold)
3783 total = mos_gem_compute_batch_space(bo_array, count);
3784
3785 if (total > threshold) {
3786 MOS_DBG("check_space: overflowed available aperture, "
3787 "%dkb vs %dkb\n",
3788 total / 1024, (int)bufmgr_gem->gtt_size / 1024);
3789 return -ENOSPC;
3790 } else {
3791 MOS_DBG("drm_check_space: total %dkb vs bufgr %dkb\n", total / 1024,
3792 (int)bufmgr_gem->gtt_size / 1024);
3793 return 0;
3794 }
3795 }
3796
3797 /*
3798 * Disable buffer reuse for objects which are shared with the kernel
3799 * as scanout buffers
3800 */
3801 static int
3802 mos_gem_bo_disable_reuse(struct mos_linux_bo *bo)
3803 {
3804 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
3805
3806 bo_gem->reusable = false;
3807 return 0;
3808 }
3809
3810 static int
3811 mos_gem_bo_is_reusable(struct mos_linux_bo *bo)
3812 {
3813 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
3814
3815 return bo_gem->reusable;
3816 }
3817
3818 static int
3819 _mos_gem_bo_references(struct mos_linux_bo *bo, struct mos_linux_bo *target_bo)
3820 {
3821 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
3822 int i;
3823
3824 for (i = 0; i < bo_gem->reloc_count; i++) {
3825 if (bo_gem->reloc_target_info[i].bo == target_bo)
3826 return 1;
3827 if (bo == bo_gem->reloc_target_info[i].bo)
3828 continue;
3829 if (_mos_gem_bo_references(bo_gem->reloc_target_info[i].bo,
3830 target_bo))
3831 return 1;
3832 }
3833
3834 for (i = 0; i< bo_gem->softpin_target_count; i++) {
3835 if (bo_gem->softpin_target[i].bo == target_bo)
3836 return 1;
3837 if (_mos_gem_bo_references(bo_gem->softpin_target[i].bo, target_bo))
3838 return 1;
3839 }
3840
3841 return 0;
3842 }
3843
3844 /** Return true if target_bo is referenced by bo's relocation tree. */
3845 static int
3846 mos_gem_bo_references(struct mos_linux_bo *bo, struct mos_linux_bo *target_bo)
3847 {
3848 struct mos_bo_gem *target_bo_gem = (struct mos_bo_gem *) target_bo;
3849
3850 if (bo == nullptr || target_bo == nullptr)
3851 return 0;
3852 if (target_bo_gem->used_as_reloc_target)
3853 return _mos_gem_bo_references(bo, target_bo);
3854 return 0;
3855 }
3856
3857 static void
3858 add_bucket(struct mos_bufmgr_gem *bufmgr_gem, int size)
3859 {
3860 unsigned int i = bufmgr_gem->num_buckets;
3861
3862 assert(i < ARRAY_SIZE(bufmgr_gem->cache_bucket));
3863
3864 DRMINITLISTHEAD(&bufmgr_gem->cache_bucket[i].head);
3865 bufmgr_gem->cache_bucket[i].size = size;
3866 bufmgr_gem->num_buckets++;
3867 }
3868
3869 static void
3870 init_cache_buckets(struct mos_bufmgr_gem *bufmgr_gem)
3871 {
3872 unsigned long size, cache_max_size = 64 * 1024 * 1024;
3873
3874 /* OK, so power of two buckets was too wasteful of memory.
3875 * Give 3 other sizes between each power of two, to hopefully
3876 * cover things accurately enough. (The alternative is
3877 * probably to just go for exact matching of sizes, and assume
3878 * that for things like composited window resize the tiled
3879 * width/height alignment and rounding of sizes to pages will
3880 * get us useful cache hit rates anyway)
3881 */
3882 add_bucket(bufmgr_gem, 4096);
3883 add_bucket(bufmgr_gem, 4096 * 2);
3884 add_bucket(bufmgr_gem, 4096 * 3);
3885
3886 /* Initialize the linked lists for BO reuse cache. */
3887 for (size = 4 * 4096; size <= cache_max_size; size *= 2) {
3888 add_bucket(bufmgr_gem, size);
3889
3890 add_bucket(bufmgr_gem, size + size * 1 / 4);
3891 add_bucket(bufmgr_gem, size + size * 2 / 4);
3892 add_bucket(bufmgr_gem, size + size * 3 / 4);
3893 }
3894 }
3895
3896 static void
3897 mos_gem_realloc_cache(struct mos_bufmgr *bufmgr, uint8_t alloc_mode)
3898 {
3899 unsigned long size, cache_max_size = 64 * 1024 * 1024, unit_size;
3900 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bufmgr;
3901
3902 // Clean up the pre-allocated cache before re-allocating according
3903 // to alloc_mode
3904 mos_bufmgr_cleanup_cache(bufmgr_gem);
3905
3906 /* OK, so power of two buckets was too wasteful of memory.
3907 * Give 3 other sizes between each power of two, to hopefully
3908 * cover things accurately enough. (The alternative is
3909 * probably to just go for exact matching of sizes, and assume
3910 * that for things like composited window resize the tiled
3911 * width/height alignment and rounding of sizes to pages will
3912 * get us useful cache hit rates anyway)
3913 */
3914 /* alloc_mode 0 is default alloc_mode
3915 * alloc_mode 1 rounding up to 64K for all < 1M
3916 * alloc_mode 2 rounding up to 2M for size> 1M
3917 * alloc_mode 3 rounding up to 2M for size > 1M and 64K for size <= 1M */
3918 if( alloc_mode > 3 )
3919 alloc_mode = 0;
3920
3921 if ( 0 == alloc_mode || 2 == alloc_mode)
3922 {
3923 // < 1M normal alloc_mode
3924 add_bucket(bufmgr_gem, 4096);
3925 add_bucket(bufmgr_gem, 4096 * 2);
3926 add_bucket(bufmgr_gem, 4096 * 3);
3927 /* Initialize the linked lists for BO reuse cache. */
3928 for (size = 4 * 4096; size < 1024 * 1024; size *= 2) {
3929 add_bucket(bufmgr_gem, size);
3930 add_bucket(bufmgr_gem, size + size * 1 / 4);
3931 add_bucket(bufmgr_gem, size + size * 2 / 4);
3932 add_bucket(bufmgr_gem, size + size * 3 / 4);
3933 }
3934
3935 add_bucket(bufmgr_gem, 1024 * 1024);
3936 }
3937 if (1 == alloc_mode || 3 == alloc_mode)
3938 {
3939 // < 1M 64k alignment
3940 unit_size = 64 * 1024;
3941 for (size = unit_size; size <= 1024 * 1024; size += unit_size)
3942 {
3943 add_bucket(bufmgr_gem, size);
3944 }
3945 }
3946 if( 0 == alloc_mode || 1 == alloc_mode)
3947 {
3948 //> 1M is normal alloc_mode
3949 add_bucket(bufmgr_gem, 1280 * 1024);
3950 add_bucket(bufmgr_gem, 1536 * 1024);
3951 add_bucket(bufmgr_gem, 1792 * 1024);
3952
3953 for (size = 2 * 1024 * 1024; size < cache_max_size; size *= 2) {
3954 add_bucket(bufmgr_gem, size);
3955 add_bucket(bufmgr_gem, size + size * 1 / 4);
3956 add_bucket(bufmgr_gem, size + size * 2 / 4);
3957 add_bucket(bufmgr_gem, size + size * 3 / 4);
3958 }
3959 }
3960 if( 2 == alloc_mode || 3 == alloc_mode)
3961 {
3962 //> 1M rolling to 2M
3963 unit_size = 2 * 1024 * 1024;
3964 add_bucket(bufmgr_gem, unit_size);
3965 add_bucket(bufmgr_gem, 3 * 1024 * 1024);
3966
3967 for (size = 4 * 1024 * 1024; size <= cache_max_size; size += unit_size)
3968 {
3969 add_bucket(bufmgr_gem, size);
3970 }
3971 }
3972 }
3973
3974 /**
3975 * Get the PCI ID for the device. This can be overridden by setting the
3976 * INTEL_DEVID_OVERRIDE environment variable to the desired ID.
3977 */
3978 static int
3979 get_pci_device_id(struct mos_bufmgr_gem *bufmgr_gem)
3980 {
3981 char *devid_override;
3982 int devid = 0;
3983 int ret;
3984 drm_i915_getparam_t gp;
3985
3986 if (geteuid() == getuid()) {
3987 devid_override = getenv("INTEL_DEVID_OVERRIDE");
3988 if (devid_override) {
3989 bufmgr_gem->no_exec = true;
3990 return strtod(devid_override, nullptr);
3991 }
3992 }
3993
3994 memclear(gp);
3995 gp.param = I915_PARAM_CHIPSET_ID;
3996 gp.value = &devid;
3997 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
3998 if (ret) {
3999 fprintf(stderr, "get chip id failed: %d [%d]\n", ret, errno);
4000 fprintf(stderr, "param: %d, val: %d\n", gp.param, *gp.value);
4001 }
4002 return devid;
4003 }
4004
4005 static int
4006 mos_gem_get_devid(struct mos_bufmgr *bufmgr)
4007 {
4008 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bufmgr;
4009
4010 return bufmgr_gem->pci_device;
4011 }
4012
4013 int mos_gem_ctx_set_user_ctx_params(struct mos_linux_context *context)
4014 {
4015 /*
4016 * INTEL_I915_CTX_CONTROL=0, 1, 2, 3
4017 * 0: default, do nothing
4018 * 1: disable ctx recoverable
4019 * 2: disable ctx bannable
4020 * 3: disable both
4021 * */
4022
4023 if (context == nullptr)
4024 {
4025 return -EINVAL;
4026 }
4027
4028 int ret = 0;
4029 char *user_ctx_env = getenv("INTEL_I915_CTX_CONTROL");
4030 bool disable_recoverable = false;
4031 bool disable_bannable = false;
4032 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)context->bufmgr;
4033
4034 if (user_ctx_env != nullptr)
4035 {
4036 uint8_t user_ctx_env_value = (uint8_t)atoi(user_ctx_env);
4037 if(user_ctx_env_value > 3)
4038 {
4039 MOS_DBG("INTEL_I915_CTX_CONTROL: invalid value %u setting\n",
4040 user_ctx_env_value);
4041 }
4042 else
4043 {
4044 if (user_ctx_env_value & 0x1)
4045 {
4046 disable_recoverable = true;
4047 }
4048
4049 if(user_ctx_env_value & 0x2)
4050 {
4051 disable_bannable = true;
4052 }
4053 }
4054 }
4055
4056 if(disable_recoverable)
4057 {
4058 ret = mos_set_context_param(context,
4059 0,
4060 I915_CONTEXT_PARAM_RECOVERABLE,
4061 0);
4062 if(ret != 0) {
4063 MOS_DBG("I915_CONTEXT_PARAM_RECOVERABLE failed: %s\n",
4064 strerror(errno));
4065 }
4066 else
4067 {
4068 MOS_DBG("successfull to disable context recoverable\n");
4069 }
4070 }
4071
4072 if(disable_bannable)
4073 {
4074 ret = mos_set_context_param(context,
4075 0,
4076 I915_CONTEXT_PARAM_BANNABLE,
4077 0);
4078 if(ret != 0) {
4079 MOS_DBG("I915_CONTEXT_PARAM_BANNABLE failed: %s\n",
4080 strerror(errno));
4081 }
4082 else
4083 {
4084 MOS_DBG("successfull to disable context bannable\n");
4085 }
4086 }
4087
4088 return ret;
4089 }
4090
4091 static struct mos_linux_context *
4092 mos_gem_context_create(struct mos_bufmgr *bufmgr)
4093 {
4094 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bufmgr;
4095 struct drm_i915_gem_context_create create;
4096 struct mos_linux_context *context = nullptr;
4097 int ret;
4098
4099 context = (struct mos_linux_context *)calloc(1, sizeof(*context));
4100 if (!context)
4101 return nullptr;
4102
4103 memclear(create);
4104 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_CONTEXT_CREATE, &create);
4105 if (ret != 0) {
4106 MOS_DBG("DRM_IOCTL_I915_GEM_CONTEXT_CREATE failed: %s\n",
4107 strerror(errno));
4108 free(context);
4109 return nullptr;
4110 }
4111
4112 context->ctx_id = create.ctx_id;
4113 context->bufmgr = bufmgr;
4114
4115 ret = mos_gem_ctx_set_user_ctx_params(context);
4116
4117 return context;
4118 }
4119
4120 static void
4121 mos_gem_context_destroy(struct mos_linux_context *ctx)
4122 {
4123 struct mos_bufmgr_gem *bufmgr_gem;
4124 struct drm_i915_gem_context_destroy destroy;
4125 int ret;
4126
4127 if (ctx == nullptr)
4128 return;
4129
4130 memclear(destroy);
4131
4132 bufmgr_gem = (struct mos_bufmgr_gem *)ctx->bufmgr;
4133 destroy.ctx_id = ctx->ctx_id;
4134 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_CONTEXT_DESTROY,
4135 &destroy);
4136 if (ret != 0)
4137 fprintf(stderr, "DRM_IOCTL_I915_GEM_CONTEXT_DESTROY failed: %s\n",
4138 strerror(errno));
4139
4140 free(ctx);
4141 }
4142
4143 static int
4144 mos_bufmg_get_reset_stats(struct mos_linux_context *ctx,
4145 uint32_t *reset_count,
4146 uint32_t *active,
4147 uint32_t *pending)
4148 {
4149 struct mos_bufmgr_gem *bufmgr_gem;
4150 struct drm_i915_reset_stats stats;
4151 int ret;
4152
4153 if (ctx == nullptr)
4154 return -EINVAL;
4155
4156 memclear(stats);
4157
4158 bufmgr_gem = (struct mos_bufmgr_gem *)ctx->bufmgr;
4159 stats.ctx_id = ctx->ctx_id;
4160 ret = drmIoctl(bufmgr_gem->fd,
4161 DRM_IOCTL_I915_GET_RESET_STATS,
4162 &stats);
4163 if (ret == 0) {
4164 if (reset_count != nullptr)
4165 *reset_count = stats.reset_count;
4166
4167 if (active != nullptr)
4168 *active = stats.batch_active;
4169
4170 if (pending != nullptr)
4171 *pending = stats.batch_pending;
4172 }
4173
4174 return ret;
4175 }
4176
4177 static unsigned int mos_bufmgr_hweight8(struct mos_linux_context *ctx, uint8_t w)
4178 {
4179 uint32_t i, weight = 0;
4180
4181 for (i=0; i<8; i++)
4182 {
4183 weight += !!((w) & (1UL << i));
4184 }
4185 return weight;
4186 }
4187
4188 static uint8_t mos_bufmgr_switch_off_n_bits(struct mos_linux_context *ctx, uint8_t in_mask, int n)
4189 {
4190 int i,count;
4191 uint8_t bi,out_mask;
4192
4193 assert (n>0 && n<=8);
4194
4195 out_mask = in_mask;
4196 count = n;
4197 for(i=0; i<8; i++)
4198 {
4199 bi = 1UL<<i;
4200 if (bi & in_mask)
4201 {
4202 out_mask &= ~bi;
4203 count--;
4204 }
4205 if (count==0)
4206 {
4207 break;
4208 }
4209 }
4210 return out_mask;
4211 }
4212
4213 static int
4214 mos_bufmgr_get_context_param_sseu(struct mos_linux_context *ctx,
4215 struct drm_i915_gem_context_param_sseu *sseu)
4216 {
4217 struct mos_bufmgr_gem *bufmgr_gem;
4218 struct drm_i915_gem_context_param context_param;
4219 int ret;
4220
4221 if (ctx == nullptr)
4222 return -EINVAL;
4223
4224 bufmgr_gem = (struct mos_bufmgr_gem *)ctx->bufmgr;
4225 memset(&context_param, 0, sizeof(context_param));
4226 context_param.ctx_id = ctx->ctx_id;
4227 context_param.param = I915_CONTEXT_PARAM_SSEU;
4228 context_param.value = (uint64_t) sseu;
4229 context_param.size = sizeof(struct drm_i915_gem_context_param_sseu);
4230
4231 ret = drmIoctl(bufmgr_gem->fd,
4232 DRM_IOCTL_I915_GEM_CONTEXT_GETPARAM,
4233 &context_param);
4234
4235 return ret;
4236 }
4237
4238 static int
4239 mos_bufmgr_set_context_param_sseu(struct mos_linux_context *ctx,
4240 struct drm_i915_gem_context_param_sseu sseu)
4241 {
4242 struct mos_bufmgr_gem *bufmgr_gem;
4243 struct drm_i915_gem_context_param context_param;
4244 int ret;
4245
4246 if (ctx == nullptr)
4247 return -EINVAL;
4248
4249 bufmgr_gem = (struct mos_bufmgr_gem *)ctx->bufmgr;
4250 memset(&context_param, 0, sizeof(context_param));
4251 context_param.ctx_id = ctx->ctx_id;
4252 context_param.param = I915_CONTEXT_PARAM_SSEU;
4253 context_param.value = (uint64_t) &sseu;
4254 context_param.size = sizeof(struct drm_i915_gem_context_param_sseu);
4255
4256 ret = drmIoctl(bufmgr_gem->fd,
4257 DRM_IOCTL_I915_GEM_CONTEXT_SETPARAM,
4258 &context_param);
4259
4260 return ret;
4261 }
4262
4263 static int
4264 mos_gem_get_context_param(struct mos_linux_context *ctx,
4265 uint32_t size,
4266 uint64_t param,
4267 uint64_t *value)
4268 {
4269 struct mos_bufmgr_gem *bufmgr_gem;
4270 struct drm_i915_gem_context_param context_param;
4271 int ret;
4272
4273 if (ctx == nullptr)
4274 return -EINVAL;
4275
4276 bufmgr_gem = (struct mos_bufmgr_gem *)ctx->bufmgr;
4277 context_param.ctx_id = ctx->ctx_id;
4278 context_param.size = size;
4279 context_param.param = param;
4280 context_param.value = 0;
4281
4282 ret = drmIoctl(bufmgr_gem->fd,
4283 DRM_IOCTL_I915_GEM_CONTEXT_GETPARAM,
4284 &context_param);
4285 *value = context_param.value;
4286
4287 return ret;
4288 }
4289
4290 static int
4291 mos_gem_set_context_param(struct mos_linux_context *ctx,
4292 uint32_t size,
4293 uint64_t param,
4294 uint64_t value)
4295 {
4296 struct mos_bufmgr_gem *bufmgr_gem;
4297 struct drm_i915_gem_context_param context_param;
4298 int ret;
4299
4300 if (ctx == nullptr)
4301 return -EINVAL;
4302
4303 bufmgr_gem = (struct mos_bufmgr_gem *)ctx->bufmgr;
4304 context_param.ctx_id = ctx->ctx_id;
4305 context_param.size = size;
4306 context_param.param = param;
4307 context_param.value = value;
4308
4309 ret = drmIoctl(bufmgr_gem->fd,
4310 DRM_IOCTL_I915_GEM_CONTEXT_SETPARAM,
4311 &context_param);
4312
4313 return ret;
4314 }
4315
4316 static int
4317 mos_bufmg_reg_read(struct mos_bufmgr *bufmgr,
4318 uint32_t offset,
4319 uint64_t *result)
4320 {
4321 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bufmgr;
4322 struct drm_i915_reg_read reg_read;
4323 int ret;
4324
4325 memclear(reg_read);
4326 reg_read.offset = offset;
4327
4328 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_REG_READ, &reg_read);
4329
4330 *result = reg_read.val;
4331 return ret;
4332 }
4333
4334 static pthread_mutex_t bufmgr_list_mutex = PTHREAD_MUTEX_INITIALIZER;
4335 static drmMMListHead bufmgr_list = { &bufmgr_list, &bufmgr_list };
4336
4337 static struct mos_bufmgr_gem *
4338 mos_bufmgr_gem_find(int fd)
4339 {
4340 struct mos_bufmgr_gem *bufmgr_gem;
4341
4342 DRMLISTFOREACHENTRY(bufmgr_gem, &bufmgr_list, managers) {
4343 if (bufmgr_gem->fd == fd) {
4344 atomic_inc(&bufmgr_gem->refcount);
4345 return bufmgr_gem;
4346 }
4347 }
4348
4349 return nullptr;
4350 }
4351
4352 static void
4353 mos_bufmgr_gem_unref(struct mos_bufmgr *bufmgr)
4354 {
4355 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bufmgr;
4356
4357 if (atomic_add_unless(&bufmgr_gem->refcount, -1, 1)) {
4358 pthread_mutex_lock(&bufmgr_list_mutex);
4359
4360 if (atomic_dec_and_test(&bufmgr_gem->refcount)) {
4361 DRMLISTDEL(&bufmgr_gem->managers);
4362 mos_bufmgr_gem_destroy(bufmgr);
4363 }
4364
4365 pthread_mutex_unlock(&bufmgr_list_mutex);
4366 }
4367 }
4368
4369 static int
4370 mos_gem_get_memory_info(struct mos_bufmgr *bufmgr, char *info, uint32_t length)
4371 {
4372 return 0;
4373 }
4374
4375 static void mos_gem_enable_softpin(struct mos_bufmgr *bufmgr, bool va1m_align)
4376 {
4377 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bufmgr;
4378 bufmgr_gem->use_softpin = true;
4379 bufmgr_gem->softpin_va1Malign = va1m_align;
4380 }
4381
4382 static void mos_gem_enable_vmbind(struct mos_bufmgr *bufmgr)
4383 {
4384 }
4385
4386 static void mos_gem_disable_object_capture(struct mos_bufmgr *bufmgr)
4387 {
4388 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bufmgr;
4389 if (bufmgr_gem != nullptr)
4390 {
4391 bufmgr_gem->object_capture_disabled = true;
4392 }
4393 }
4394
4395 static struct mos_linux_context *
4396 mos_gem_context_create_ext(struct mos_bufmgr *bufmgr, __u32 flags, bool bContextProtected)
4397 {
4398 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bufmgr;
4399 struct drm_i915_gem_context_create_ext create;
4400 struct mos_linux_context *context = nullptr;
4401 struct drm_i915_gem_context_create_ext_setparam p_protected;
4402 struct drm_i915_gem_context_create_ext_setparam p_norecover;
4403 int ret;
4404
4405 context = (struct mos_linux_context *)calloc(1, sizeof(*context));
4406 if (!context)
4407 return nullptr;
4408
4409 memclear(create);
4410 create.flags = flags;
4411 create.extensions = 0;
4412
4413 if (bContextProtected)
4414 {
4415 memclear(p_protected);
4416 memclear(p_norecover);
4417 p_protected.base.next_extension = 0;
4418 p_protected.base.name = I915_CONTEXT_CREATE_EXT_SETPARAM;
4419 p_protected.param.param = I915_CONTEXT_PARAM_PROTECTED_CONTENT;
4420 p_protected.param.value = 1;
4421
4422 p_norecover.base.next_extension = (uintptr_t)&p_protected;
4423 p_norecover.base.name = I915_CONTEXT_CREATE_EXT_SETPARAM;
4424 p_norecover.param.param = I915_CONTEXT_PARAM_RECOVERABLE;
4425 p_norecover.param.value = 0;
4426
4427 create.flags = flags|I915_CONTEXT_CREATE_FLAGS_USE_EXTENSIONS;
4428 create.extensions = (uintptr_t)&p_norecover;
4429 }
4430 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_CONTEXT_CREATE_EXT, &create);
4431 if (ret != 0) {
4432 MOS_DBG("DRM_IOCTL_I915_GEM_CONTEXT_CREATE failed: %s\n",
4433 strerror(errno));
4434 free(context);
4435 return nullptr;
4436 }
4437
4438 context->ctx_id = create.ctx_id;
4439 context->bufmgr = bufmgr;
4440
4441 ret = mos_gem_ctx_set_user_ctx_params(context);
4442
4443 return context;
4444 }
4445
4446 static __u32 mos_gem_vm_create(struct mos_bufmgr *bufmgr)
4447 {
4448 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bufmgr;
4449 struct drm_i915_gem_vm_control *vm = nullptr;
4450 __u32 vm_id;
4451 int ret;
4452
4453 vm = (struct drm_i915_gem_vm_control *)calloc(1, sizeof(struct drm_i915_gem_vm_control));
4454 if (nullptr == vm)
4455 {
4456 MOS_DBG("vm calloc failed\n" );
4457 return INVALID_VM;
4458 }
4459
4460 memset(vm, 0, sizeof(*vm));
4461
4462 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_VM_CREATE, vm);
4463 if (ret != 0) {
4464 MOS_DBG("DRM_IOCTL_I915_GEM_VM_CREATE failed: %s\n",
4465 strerror(errno));
4466 free(vm);
4467 return INVALID_VM;
4468 }
4469
4470 vm_id = vm->vm_id;
4471 free(vm);
4472
4473 return vm_id;
4474 }
4475
4476 static void mos_gem_vm_destroy(struct mos_bufmgr *bufmgr, __u32 vm_id)
4477 {
4478 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bufmgr;
4479 struct drm_i915_gem_vm_control *vm = nullptr;
4480 int ret;
4481
4482 if (vm_id == INVALID_VM)
4483 {
4484 MOS_DBG("input invalid param\n" );
4485 return;
4486 }
4487
4488 vm = (struct drm_i915_gem_vm_control *)calloc(1, sizeof(struct drm_i915_gem_vm_control));
4489
4490 if (nullptr == vm)
4491 {
4492 MOS_DBG("vm calloc failed\n" );
4493 return;
4494 }
4495
4496 vm->vm_id = vm_id;
4497 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_VM_DESTROY, vm);
4498 if (ret != 0) {
4499 MOS_DBG("DRM_IOCTL_I915_GEM_VM_DESTROY failed: %s\n",
4500 strerror(errno));
4501 }
4502
4503 free(vm);
4504 }
4505
4506 static struct mos_linux_context *
4507 mos_gem_context_create_shared(struct mos_bufmgr *bufmgr,
4508 mos_linux_context* ctx,
4509 __u32 flags,
4510 bool bContextProtected,
4511 void *engine_map,
4512 uint8_t ctx_width,
4513 uint8_t num_placements,
4514 uint32_t ctx_type)
4515 {
4516 MOS_UNUSED(engine_map);
4517 MOS_UNUSED(ctx_width);
4518 MOS_UNUSED(num_placements);
4519 MOS_UNUSED(ctx_type);
4520
4521 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bufmgr;
4522 struct drm_i915_gem_context_create_ext create;
4523 struct mos_linux_context *context = nullptr;
4524 struct drm_i915_gem_context_create_ext_setparam p_protected;
4525 struct drm_i915_gem_context_create_ext_setparam p_norecover;
4526 int ret;
4527
4528 if (ctx == nullptr || ctx->vm_id == INVALID_VM)
4529 return nullptr;
4530
4531 context = (struct mos_linux_context *)calloc(1, sizeof(*context));
4532 if (!context)
4533 return nullptr;
4534
4535 memclear(create);
4536 create.flags = flags;
4537 create.extensions = 0;
4538 if (bContextProtected)
4539 {
4540 memclear(p_protected);
4541 memclear(p_norecover);
4542 p_protected.base.next_extension = 0;
4543 p_protected.base.name = I915_CONTEXT_CREATE_EXT_SETPARAM;
4544 p_protected.param.param = I915_CONTEXT_PARAM_PROTECTED_CONTENT;
4545 p_protected.param.value = 1;
4546
4547 p_norecover.base.next_extension = (uintptr_t)&p_protected;
4548 p_norecover.base.name = I915_CONTEXT_CREATE_EXT_SETPARAM;
4549 p_norecover.param.param = I915_CONTEXT_PARAM_RECOVERABLE;
4550 p_norecover.param.value = 0;
4551
4552 create.flags = flags|I915_CONTEXT_CREATE_FLAGS_USE_EXTENSIONS;
4553 create.extensions = (uintptr_t)&p_norecover;
4554 }
4555 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_CONTEXT_CREATE_EXT, &create);
4556 if (ret != 0) {
4557 MOS_DBG("DRM_IOCTL_I915_GEM_CONTEXT_CREATE failed: %s\n",
4558 strerror(errno));
4559 free(context);
4560 return nullptr;
4561 }
4562
4563 context->ctx_id = create.ctx_id;
4564 context->bufmgr = bufmgr;
4565
4566 ret = mos_set_context_param(context,
4567 0,
4568 I915_CONTEXT_PARAM_VM,
4569 ctx->vm_id);
4570 if(ret != 0) {
4571 MOS_DBG("I915_CONTEXT_PARAM_VM failed: %s\n",
4572 strerror(errno));
4573 free(context);
4574 return nullptr;
4575 }
4576
4577 ret = mos_gem_ctx_set_user_ctx_params(context);
4578
4579 return context;
4580 }
4581
4582 static int mos_bufmgr_query_engines_count(struct mos_bufmgr *bufmgr,
4583 unsigned int *nengine)
4584 {
4585 assert(bufmgr);
4586 assert(nengine);
4587 int fd = ((struct mos_bufmgr_gem*)bufmgr)->fd;
4588 struct drm_i915_query query;
4589 struct drm_i915_query_item query_item;
4590 struct drm_i915_query_engine_info *engines = nullptr;
4591 int ret, len;
4592
4593 memclear(query_item);
4594 query_item.query_id = DRM_I915_QUERY_ENGINE_INFO;
4595 query_item.length = 0;
4596 memclear(query);
4597 query.num_items = 1;
4598 query.items_ptr = (uintptr_t)&query_item;
4599
4600 ret = drmIoctl(fd, DRM_IOCTL_I915_QUERY, &query);
4601 if (ret || query_item.length == 0)
4602 {
4603 *nengine = 0;
4604 return ret;
4605 }
4606
4607 len = query_item.length;
4608
4609 engines = (drm_i915_query_engine_info *)malloc(len);
4610 if (nullptr == engines)
4611 {
4612 *nengine = 0;
4613 ret = -ENOMEM;
4614 return ret;
4615 }
4616
4617 memset(engines, 0, len);
4618 memclear(query_item);
4619 query_item.query_id = DRM_I915_QUERY_ENGINE_INFO;
4620 query_item.length = len;
4621 query_item.data_ptr = (uintptr_t)engines;
4622 memclear(query);
4623 query.num_items = 1;
4624 query.items_ptr = (uintptr_t)&query_item;
4625
4626 ret = drmIoctl(fd, DRM_IOCTL_I915_QUERY, &query);
4627
4628 *nengine = ret ? 0 : engines->num_engines;
4629
4630 if(engines)
4631 {
4632 free(engines);
4633 }
4634 return ret;
4635 }
4636
4637 static int mos_bufmgr_query_engines(struct mos_bufmgr *bufmgr,
4638 __u16 engine_class,
4639 __u64 caps,
4640 unsigned int *nengine,
4641 void *engine_map)
4642 {
4643 struct i915_engine_class_instance *ci = (struct i915_engine_class_instance *)engine_map;
4644 if((bufmgr == nullptr) || (nengine == nullptr) || (ci == nullptr))
4645 {
4646 return -EINVAL;
4647 }
4648
4649 struct drm_i915_query query;
4650 struct drm_i915_query_item query_item;
4651 struct drm_i915_query_engine_info *engines = nullptr;
4652 int ret, len;
4653 int fd = ((struct mos_bufmgr_gem*)bufmgr)->fd;
4654
4655 memclear(query_item);
4656 query_item.query_id = DRM_I915_QUERY_ENGINE_INFO;
4657 query_item.length = 0;
4658 memclear(query);
4659 query.num_items = 1;
4660 query.items_ptr = (uintptr_t)&query_item;
4661
4662 ret = drmIoctl(fd, DRM_IOCTL_I915_QUERY, &query);
4663 if (ret)
4664 {
4665 goto fini;
4666 }
4667
4668 len = query_item.length;
4669 if(len == 0)
4670 {
4671 goto fini;
4672 }
4673
4674 engines = (drm_i915_query_engine_info *)malloc(len);
4675 if (nullptr == engines)
4676 {
4677 ret = -ENOMEM;
4678 goto fini;
4679 }
4680 memset(engines,0,len);
4681 memclear(query_item);
4682 query_item.query_id = DRM_I915_QUERY_ENGINE_INFO;
4683 query_item.length = len;
4684 query_item.data_ptr = (uintptr_t)engines;
4685 memclear(query);
4686 query.num_items = 1;
4687 query.items_ptr = (uintptr_t)&query_item;
4688
4689 ret = drmIoctl(fd, DRM_IOCTL_I915_QUERY, &query);
4690 if (ret)
4691 {
4692 goto fini;
4693 }
4694
4695 int i, num;
4696 for (i = 0, num = 0; i < engines->num_engines; i++) {
4697 struct drm_i915_engine_info *engine =
4698 (struct drm_i915_engine_info *)&engines->engines[i];
4699 if ( engine_class == engine->engine.engine_class
4700 && ((caps & engine->capabilities) == caps ))
4701 {
4702 ci->engine_class = engine_class;
4703 ci->engine_instance = engine->engine.engine_instance;
4704 ci++;
4705 num++;
4706 }
4707 if (num > *nengine)
4708 {
4709 fprintf(stderr,"%s: Number of engine instances out of range, %d,%d\n",
4710 __FUNCTION__, num, *nengine);
4711 goto fini;
4712 }
4713 }
4714 *nengine = num;
4715
4716
4717 fini:
4718 if (engines)
4719 free(engines);
4720 return ret;
4721 }
4722
4723 static size_t mos_bufmgr_get_engine_class_size()
4724 {
4725 return sizeof(struct i915_engine_class_instance);
4726 }
4727
4728 static int
4729 mos_bufmgr_query_sys_engines(struct mos_bufmgr *bufmgr, MEDIA_SYSTEM_INFO* gfx_info)
4730 {
4731 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bufmgr;
4732 int ret;
4733
4734 if (nullptr == gfx_info)
4735 {
4736 return -EINVAL;
4737 }
4738
4739 unsigned int maxNengine = 0;
4740 if((gfx_info->VDBoxInfo.NumberOfVDBoxEnabled == 0)
4741 || (gfx_info->VEBoxInfo.NumberOfVEBoxEnabled == 0))
4742 {
4743 if (mos_query_engines_count(bufmgr, &maxNengine) || (maxNengine == 0))
4744 {
4745 MOS_OS_ASSERTMESSAGE("Failed to query engines count.\n");
4746 return -ENODEV;
4747 }
4748 }
4749
4750 if (gfx_info->VDBoxInfo.NumberOfVDBoxEnabled == 0)
4751 {
4752 unsigned int nengine = maxNengine;
4753 struct i915_engine_class_instance *uengines = nullptr;
4754 uengines = (struct i915_engine_class_instance *)MOS_AllocAndZeroMemory(nengine * sizeof(struct i915_engine_class_instance));
4755 if (nullptr == uengines)
4756 {
4757 return -ENOMEM;
4758 }
4759 ret = mos_bufmgr_query_engines(bufmgr, I915_ENGINE_CLASS_VIDEO, 0, &nengine, (void *)uengines);
4760 if (ret)
4761 {
4762 MOS_OS_ASSERTMESSAGE("Failed to query vdbox engine\n");
4763 MOS_SafeFreeMemory(uengines);
4764 return -ENODEV;
4765 }
4766 else
4767 {
4768 gfx_info->VDBoxInfo.NumberOfVDBoxEnabled = nengine;
4769 }
4770
4771 for (int i=0; i<nengine; i++)
4772 {
4773 gfx_info->VDBoxInfo.Instances.VDBoxEnableMask |= 1<<uengines[i].engine_instance;
4774 }
4775
4776 MOS_SafeFreeMemory(uengines);
4777 }
4778
4779 if (gfx_info->VEBoxInfo.NumberOfVEBoxEnabled == 0)
4780 {
4781 unsigned int nengine = maxNengine;
4782 struct i915_engine_class_instance *uengines = nullptr;
4783 uengines = (struct i915_engine_class_instance *)MOS_AllocAndZeroMemory(nengine * sizeof(struct i915_engine_class_instance));
4784 if (nullptr == uengines)
4785 {
4786 return -ENOMEM;
4787 }
4788 ret = mos_bufmgr_query_engines(bufmgr, I915_ENGINE_CLASS_VIDEO_ENHANCE, 0, &nengine, (void *)uengines);
4789 if (ret)
4790 {
4791 MOS_OS_ASSERTMESSAGE("Failed to query vebox engine\n");
4792 MOS_SafeFreeMemory(uengines);
4793 return -ENODEV;
4794 }
4795 else
4796 {
4797 MOS_OS_ASSERT(nengine <= maxNengine);
4798 gfx_info->VEBoxInfo.NumberOfVEBoxEnabled = nengine;
4799 }
4800
4801 MOS_SafeFreeMemory(uengines);
4802 }
4803
4804 return 0;
4805 }
4806
4807 void mos_gem_select_fixed_engine(struct mos_bufmgr *bufmgr,
4808 void *engine_map,
4809 uint32_t *nengine,
4810 uint32_t fixed_instance_mask)
4811 {
4812 MOS_UNUSED(bufmgr);
4813 #if (DEBUG || _RELEASE_INTERNAL)
4814 if (fixed_instance_mask)
4815 {
4816 struct i915_engine_class_instance *_engine_map = (struct i915_engine_class_instance *)engine_map;
4817 auto unselect_index = 0;
4818 for(auto bit = 0; bit < *nengine; bit++)
4819 {
4820 if(((fixed_instance_mask >> bit) & 0x1) && (bit > unselect_index))
4821 {
4822 _engine_map[unselect_index].engine_class = _engine_map[bit].engine_class;
4823 _engine_map[unselect_index].engine_instance = _engine_map[bit].engine_instance;
4824 _engine_map[bit].engine_class = 0;
4825 _engine_map[bit].engine_instance = 0;
4826 unselect_index++;
4827 }
4828 else if(((fixed_instance_mask >> bit) & 0x1) && (bit == unselect_index))
4829 {
4830 unselect_index++;
4831 }
4832 else if(!((fixed_instance_mask >> bit) & 0x1))
4833 {
4834 _engine_map[bit].engine_class = 0;
4835 _engine_map[bit].engine_instance = 0;
4836 }
4837 }
4838 *nengine = unselect_index;
4839 }
4840 #else
4841 MOS_UNUSED(engine_map);
4842 MOS_UNUSED(nengine);
4843 MOS_UNUSED(fixed_instance_mask);
4844 #endif
4845
4846 }
4847
4848 static int mos_gem_set_context_param_parallel(struct mos_linux_context *ctx,
4849 struct i915_engine_class_instance *ci,
4850 unsigned int count)
4851 {
4852 if((ctx == nullptr) || (ci == nullptr) || (count <= 0))
4853 {
4854 return -EINVAL;
4855 }
4856
4857 int ret = 0;
4858 uint32_t size = 0;
4859 int n;
4860 struct i915_context_engines_parallel_submit* parallel_submit = nullptr;
4861 struct i915_context_param_engines* set_engines = nullptr;
4862
4863 size = sizeof(struct i915_context_engines_parallel_submit) + count * sizeof(*ci);
4864 parallel_submit = (struct i915_context_engines_parallel_submit*)malloc(size);
4865 if(parallel_submit == nullptr)
4866 {
4867 ret = -ENOMEM;
4868 goto fini;
4869 }
4870 memset(parallel_submit, 0, size);
4871 parallel_submit->base.name = I915_CONTEXT_ENGINES_EXT_PARALLEL_SUBMIT;
4872 parallel_submit->engine_index = 0;
4873 parallel_submit->width = count;
4874 parallel_submit->num_siblings = 1;
4875 for(int i = 0; i < count; i++)
4876 {
4877 parallel_submit->engines[i] = ci[i];
4878 }
4879
4880 /* I915_DEFINE_CONTEXT_PARAM_ENGINES */
4881 size = sizeof(struct i915_context_param_engines) + sizeof(struct i915_engine_class_instance);
4882 set_engines = (struct i915_context_param_engines*) malloc(size);
4883 if(set_engines == nullptr)
4884 {
4885 ret = -ENOMEM;
4886 goto fini;
4887 }
4888 set_engines->extensions = (uintptr_t)(parallel_submit);
4889 set_engines->engines[0].engine_class = I915_ENGINE_CLASS_INVALID;
4890 set_engines->engines[0].engine_instance = I915_ENGINE_CLASS_INVALID_NONE;
4891
4892 ret = mos_set_context_param(ctx,
4893 size,
4894 I915_CONTEXT_PARAM_ENGINES,
4895 (uintptr_t)set_engines);
4896 fini:
4897 if (set_engines)
4898 free(set_engines);
4899 if (parallel_submit)
4900 free(parallel_submit);
4901 return ret;
4902 }
4903
4904 static int mos_gem_set_context_param_load_balance(struct mos_linux_context *ctx,
4905 struct i915_engine_class_instance *ci,
4906 unsigned int count)
4907 {
4908 int ret;
4909 uint32_t size;
4910 struct i915_context_engines_load_balance* balancer = nullptr;
4911 struct i915_context_param_engines* set_engines = nullptr;
4912
4913 MOS_OS_CHECK_CONDITION(ci == nullptr, "Invalid (nullptr) Pointer.", EINVAL);
4914 MOS_OS_CHECK_CONDITION(count == 0, "Invalid input parameter. Number of engines must be > 0.", EINVAL);
4915
4916 /* I915_DEFINE_CONTEXT_ENGINES_LOAD_BALANCE */
4917 size = sizeof(struct i915_context_engines_load_balance) + count * sizeof(*ci);
4918 balancer = (struct i915_context_engines_load_balance*)malloc(size);
4919 if (NULL == balancer)
4920 {
4921 ret = -ENOMEM;
4922 goto fini;
4923 }
4924 memset(balancer, 0, size);
4925 balancer->base.name = I915_CONTEXT_ENGINES_EXT_LOAD_BALANCE;
4926 balancer->num_siblings = count;
4927 memcpy(balancer->engines, ci, count * sizeof(*ci));
4928
4929 /* I915_DEFINE_CONTEXT_PARAM_ENGINES */
4930 size = sizeof(uint64_t) + sizeof(*ci);
4931 set_engines = (struct i915_context_param_engines*) malloc(size);
4932 if (NULL == set_engines)
4933 {
4934 ret = -ENOMEM;
4935 goto fini;
4936 }
4937 set_engines->extensions = (uintptr_t)(balancer);
4938 set_engines->engines[0].engine_class = I915_ENGINE_CLASS_INVALID;
4939 set_engines->engines[0].engine_instance = I915_ENGINE_CLASS_INVALID_NONE;
4940
4941 ret = mos_set_context_param(ctx,
4942 size,
4943 I915_CONTEXT_PARAM_ENGINES,
4944 (uintptr_t)set_engines);
4945 fini:
4946 if (set_engines)
4947 free(set_engines);
4948 if (balancer)
4949 free(balancer);
4950 return ret;
4951 }
4952
4953 static int mos_gem_set_context_param_bond(struct mos_linux_context *ctx,
4954 struct i915_engine_class_instance master_ci,
4955 struct i915_engine_class_instance *bond_ci,
4956 unsigned int bond_count)
4957 {
4958 int ret;
4959 uint32_t size;
4960 struct i915_context_engines_load_balance* balancer = nullptr;
4961 struct i915_context_engines_bond *bond = nullptr;
4962 struct i915_context_param_engines* set_engines = nullptr;
4963
4964 assert(bond_ci);
4965
4966 /* I915_DEFINE_CONTEXT_ENGINES_LOAD_BALANCE */
4967 size = sizeof(struct i915_context_engines_load_balance) + bond_count * sizeof(bond_ci);
4968 balancer = (struct i915_context_engines_load_balance*)malloc(size);
4969 if (NULL == balancer)
4970 {
4971 ret = -ENOMEM;
4972 goto fini;
4973 }
4974 memset(balancer, 0, size);
4975 balancer->base.name = I915_CONTEXT_ENGINES_EXT_LOAD_BALANCE;
4976 balancer->num_siblings = bond_count;
4977 memcpy(balancer->engines, bond_ci, bond_count * sizeof(*bond_ci));
4978
4979 /* I915_DEFINE_CONTEXT_ENGINES_BOND */
4980 size = sizeof(struct i915_context_engines_bond) + bond_count * sizeof(*bond_ci);
4981 bond = (struct i915_context_engines_bond*)malloc(size);
4982 if (NULL == bond)
4983 {
4984 ret = -ENOMEM;
4985 goto fini;
4986 }
4987 memset(bond, 0, size);
4988 bond->base.name = I915_CONTEXT_ENGINES_EXT_BOND;
4989 bond->master = master_ci;
4990 bond->num_bonds = bond_count;
4991 memcpy(bond->engines, bond_ci, bond_count * sizeof(*bond_ci));
4992
4993 /* I915_DEFINE_CONTEXT_PARAM_ENGINES */
4994 size = sizeof(uint64_t) + sizeof(struct i915_engine_class_instance);
4995 set_engines = (struct i915_context_param_engines*) malloc(size);
4996 if (NULL == set_engines)
4997 {
4998 ret = -ENOMEM;
4999 goto fini;
5000 }
5001 set_engines->extensions = (uintptr_t)(balancer);
5002 balancer->base.next_extension = (uintptr_t)(bond);
5003 set_engines->engines[0].engine_class = I915_ENGINE_CLASS_INVALID;
5004 set_engines->engines[0].engine_instance = I915_ENGINE_CLASS_INVALID_NONE;
5005
5006 ret = mos_set_context_param(ctx,
5007 size,
5008 I915_CONTEXT_PARAM_ENGINES,
5009 (uintptr_t)set_engines);
5010 fini:
5011 if (set_engines)
5012 free(set_engines);
5013 if (bond)
5014 free(bond);
5015 if (balancer)
5016 free(balancer);
5017 return ret;
5018 }
5019
5020 static bool mos_gem_bo_is_softpin(struct mos_linux_bo *bo)
5021 {
5022 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
5023 if (bo_gem == nullptr)
5024 {
5025 return false;
5026 }
5027
5028 return bo_gem->is_softpin;
5029 }
5030
5031 static bool
5032 mos_gem_bo_is_exec_object_async(struct mos_linux_bo *bo)
5033 {
5034 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *)bo;
5035
5036 return bo_gem->exec_async;
5037 }
5038
5039
5040 static int mos_bufmgr_query_device_blob(struct mos_bufmgr *bufmgr, MEDIA_SYSTEM_INFO* gfx_info)
5041 {
5042 if ((bufmgr == nullptr) || (gfx_info == nullptr))
5043 {
5044 return -EINVAL;
5045 }
5046
5047 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem*)bufmgr;
5048 if (bufmgr_gem == nullptr)
5049 {
5050 return -EINVAL;
5051 }
5052 int fd = bufmgr_gem->fd;
5053 uint32_t *hw_info = nullptr;
5054 uint32_t ulength= 0;
5055 int ret, i;
5056 struct drm_i915_query_item query_item;
5057
5058 memclear(query_item);
5059 query_item.length = 0;
5060 query_item.query_id = DRM_I915_QUERY_HWCONFIG_BLOB;
5061 ret = mos_gem_query_items(fd, &query_item, 1);
5062 if (ret != 0 || query_item.length <= 0)
5063 {
5064 mos_safe_free(hw_info);
5065 return (ret != 0) ? ret : -1;
5066 }
5067
5068 hw_info = (uint32_t*) malloc(query_item.length);
5069 if (hw_info != nullptr)
5070 {
5071 memset(hw_info, 0, query_item.length);
5072 }
5073 else
5074 {
5075 mos_safe_free(hw_info);
5076 return -ENOMEM;
5077 }
5078 query_item.data_ptr = (uintptr_t) hw_info;
5079 ret = mos_gem_query_items(fd, &query_item, 1);
5080 if (ret != 0 || query_item.length <= 0)
5081 {
5082 mos_safe_free(hw_info);
5083 return (ret != 0) ? ret : -1;
5084 }
5085 ulength = query_item.length / sizeof(uint32_t);
5086 i = 0;
5087 while (i < ulength) {
5088 /* Attribute ID starts with 1 */
5089 if (hw_info[i] <= 0)
5090 {
5091 mos_safe_free(hw_info);
5092 return -EINVAL;
5093 }
5094
5095 #if DEBUG_BLOB_QUERY
5096 fprintf("%s: %d\n", key_string[hw_info[i]], hw_info[i+2]);
5097 #endif
5098 if (INTEL_HWCONFIG_MAX_SLICES_SUPPORTED == hw_info[i])
5099 {
5100 if (hw_info[i+1] != 1)
5101 {
5102 mos_safe_free(hw_info);
5103 return -EINVAL;
5104 }
5105 gfx_info->SliceCount = hw_info[i+2];
5106 gfx_info->MaxSlicesSupported = hw_info[i+2];
5107 }
5108
5109 if (INTEL_HWCONFIG_MAX_DUAL_SUBSLICES_SUPPORTED == hw_info[i])
5110 {
5111 if (hw_info[i+1] != 1)
5112 {
5113 mos_safe_free(hw_info);
5114 return -EINVAL;
5115 }
5116 gfx_info->SubSliceCount = hw_info[i+2];
5117 gfx_info->MaxSubSlicesSupported = hw_info[i+2];
5118 }
5119
5120 if (INTEL_HWCONFIG_MAX_NUM_EU_PER_DSS == hw_info[i])
5121 {
5122 if (hw_info[i+1] != 1)
5123 {
5124 mos_safe_free(hw_info);
5125 return -EINVAL;
5126 }
5127 gfx_info->MaxEuPerSubSlice = hw_info[i+2];
5128 }
5129
5130 if (INTEL_HWCONFIG_DEPRECATED_L3_CACHE_SIZE_IN_KB == hw_info[i])
5131 {
5132 if (hw_info[i+1] != 1)
5133 {
5134 mos_safe_free(hw_info);
5135 return -EINVAL;
5136 }
5137 gfx_info->L3CacheSizeInKb = hw_info[i+2];
5138 }
5139
5140 if (INTEL_HWCONFIG_NUM_THREADS_PER_EU == hw_info[i])
5141 {
5142 if (hw_info[i+1] != 1)
5143 {
5144 mos_safe_free(hw_info);
5145 return -EINVAL;
5146 }
5147 gfx_info->NumThreadsPerEu = hw_info[i+2];
5148 }
5149
5150 if (INTEL_HWCONFIG_MAX_VECS == hw_info[i])
5151 {
5152 if (hw_info[i+1] != 1)
5153 {
5154 mos_safe_free(hw_info);
5155 return -EINVAL;
5156 }
5157 gfx_info->MaxVECS = hw_info[i+2];
5158 }
5159
5160 /* Advance to next key */
5161 i += hw_info[i + 1]; // value size
5162 i += 2;// KL size
5163 }
5164
5165 if (hw_info != nullptr)
5166 {
5167 mos_safe_free(hw_info);
5168 hw_info = nullptr;
5169 }
5170
5171 return ret;
5172 }
5173
5174 static int mos_bufmgr_query_hw_ip_version(struct mos_bufmgr *bufmgr, __u16 engine_class, void *ip_ver_info)
5175 {
5176 return -1;
5177 }
5178
5179 static uint64_t mos_bufmgr_get_platform_information(struct mos_bufmgr *bufmgr)
5180 {
5181 assert(bufmgr);
5182 return bufmgr->platform_information;
5183 }
5184
5185 static void mos_bufmgr_set_platform_information(struct mos_bufmgr *bufmgr, uint64_t p)
5186 {
5187 assert(bufmgr);
5188 bufmgr->platform_information |= p;
5189 }
5190
5191 static int
5192 mos_bufmgr_get_ts_frequency(struct mos_bufmgr *bufmgr, uint32_t *ts_freq)
5193 {
5194 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem*)bufmgr;
5195
5196 *ts_freq = bufmgr_gem->ts_freq;
5197
5198 return 0;
5199 }
5200
5201 static bool
5202 mos_bufmgr_has_bsd2(struct mos_bufmgr *bufmgr)
5203 {
5204 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem*)bufmgr;
5205
5206 return bufmgr_gem->has_bsd2;
5207 }
5208
5209 #define I915_CONTEXT_PRIVATE_PARAM_BOOST 0x80000000
5210 void
5211 mos_bufmgr_enable_turbo_boost(struct mos_bufmgr *bufmgr)
5212 {
5213 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem*)bufmgr;
5214 struct drm_i915_gem_context_param ctxParam;
5215 int32_t retVal = 0;
5216
5217 MOS_ZeroMemory( &ctxParam, sizeof( ctxParam ) );
5218 ctxParam.param = I915_CONTEXT_PRIVATE_PARAM_BOOST;
5219 ctxParam.value = 1;
5220 retVal = drmIoctl(bufmgr_gem->fd,
5221 DRM_IOCTL_I915_GEM_CONTEXT_SETPARAM, &ctxParam );
5222 }
5223
5224 /**
5225 * Initializes the GEM buffer manager, which uses the kernel to allocate, map,
5226 * and manage map buffer objections.
5227 *
5228 * \param fd File descriptor of the opened DRM device.
5229 */
5230 static struct mos_bufmgr *
5231 mos_bufmgr_gem_init_i915(int fd, int batch_size)
5232 {
5233 struct mos_bufmgr_gem *bufmgr_gem;
5234 struct drm_i915_gem_get_aperture aperture;
5235 drm_i915_getparam_t gp;
5236 int ret, tmp;
5237 uint8_t alloc_mode;
5238 bool exec2 = false;
5239
5240 pthread_mutex_lock(&bufmgr_list_mutex);
5241
5242 bufmgr_gem = mos_bufmgr_gem_find(fd);
5243 if (bufmgr_gem)
5244 goto exit;
5245
5246 bufmgr_gem = (struct mos_bufmgr_gem *)calloc(1, sizeof(*bufmgr_gem));
5247 if (bufmgr_gem == nullptr)
5248 goto exit;
5249
5250 bufmgr_gem->fd = fd;
5251 atomic_set(&bufmgr_gem->refcount, 1);
5252
5253 if (pthread_mutex_init(&bufmgr_gem->lock, nullptr) != 0) {
5254 free(bufmgr_gem);
5255 bufmgr_gem = nullptr;
5256 goto exit;
5257 }
5258
5259 bufmgr_gem->bufmgr.bo_alloc = mos_gem_bo_alloc;
5260 bufmgr_gem->bufmgr.bo_alloc_tiled = mos_gem_bo_alloc_tiled;
5261 bufmgr_gem->bufmgr.bo_reference = mos_gem_bo_reference;
5262 bufmgr_gem->bufmgr.bo_unreference = mos_gem_bo_unreference;
5263 bufmgr_gem->bufmgr.bo_map = mos_gem_bo_map;
5264 bufmgr_gem->bufmgr.bo_unmap = mos_gem_bo_unmap;
5265 bufmgr_gem->bufmgr.bo_wait_rendering = mos_gem_bo_wait_rendering;
5266 bufmgr_gem->bufmgr.bo_pad_to_size = mos_gem_bo_pad_to_size;
5267 bufmgr_gem->bufmgr.bo_emit_reloc = mos_gem_bo_emit_reloc;
5268 bufmgr_gem->bufmgr.bo_get_tiling = mos_gem_bo_get_tiling;
5269 bufmgr_gem->bufmgr.bo_set_tiling = mos_gem_bo_set_tiling;
5270 bufmgr_gem->bufmgr.bo_flink = mos_gem_bo_flink;
5271 bufmgr_gem->bufmgr.bo_exec = mos_gem_bo_exec2;
5272 bufmgr_gem->bufmgr.bo_mrb_exec = mos_gem_bo_mrb_exec2;
5273 bufmgr_gem->bufmgr.bo_busy = mos_gem_bo_busy;
5274 bufmgr_gem->bufmgr.bo_madvise = mos_gem_bo_madvise;
5275 bufmgr_gem->bufmgr.destroy = mos_bufmgr_gem_unref;
5276 bufmgr_gem->bufmgr.debug = 0;
5277 bufmgr_gem->bufmgr.check_aperture_space =
5278 mos_gem_check_aperture_space;
5279 bufmgr_gem->bufmgr.bo_disable_reuse = mos_gem_bo_disable_reuse;
5280 bufmgr_gem->bufmgr.bo_is_reusable = mos_gem_bo_is_reusable;
5281 bufmgr_gem->bufmgr.bo_references = mos_gem_bo_references;
5282
5283 bufmgr_gem->bufmgr.bo_wait = mos_gem_bo_wait;
5284 bufmgr_gem->bufmgr.bo_clear_relocs = mos_gem_bo_clear_relocs;
5285 bufmgr_gem->bufmgr.context_create = mos_gem_context_create;
5286 bufmgr_gem->bufmgr.context_create_ext = mos_gem_context_create_ext;
5287 bufmgr_gem->bufmgr.context_create_shared = mos_gem_context_create_shared;
5288 bufmgr_gem->bufmgr.context_destroy = mos_gem_context_destroy;
5289 bufmgr_gem->bufmgr.vm_create = mos_gem_vm_create;
5290 bufmgr_gem->bufmgr.vm_destroy = mos_gem_vm_destroy;
5291 bufmgr_gem->bufmgr.bo_context_exec2 = mos_gem_bo_context_exec2;
5292 bufmgr_gem->bufmgr.bo_context_exec3 = mos_gem_bo_context_exec3;
5293 bufmgr_gem->bufmgr.bo_is_exec_object_async = mos_gem_bo_is_exec_object_async;
5294 bufmgr_gem->bufmgr.bo_is_softpin = mos_gem_bo_is_softpin;
5295 bufmgr_gem->bufmgr.bo_map_gtt = mos_gem_bo_map_gtt;
5296 bufmgr_gem->bufmgr.bo_unmap_gtt = mos_gem_bo_unmap_gtt;
5297 bufmgr_gem->bufmgr.bo_map_wc = mos_gem_bo_map_wc;
5298 bufmgr_gem->bufmgr.bo_unmap_wc = mos_gem_bo_unmap_wc;
5299 bufmgr_gem->bufmgr.bo_map_unsynchronized = mos_gem_bo_map_unsynchronized;
5300 bufmgr_gem->bufmgr.bo_start_gtt_access = mos_gem_bo_start_gtt_access;
5301 bufmgr_gem->bufmgr.bo_get_softpin_targets_info = mos_bufmgr_bo_get_softpin_targets_info;
5302 bufmgr_gem->bufmgr.bo_create_from_name = mos_bufmgr_bo_gem_create_from_name;
5303 bufmgr_gem->bufmgr.enable_reuse = mos_gem_enable_reuse;
5304 bufmgr_gem->bufmgr.enable_softpin = mos_gem_enable_softpin;
5305 bufmgr_gem->bufmgr.enable_vmbind = mos_gem_enable_vmbind;
5306 bufmgr_gem->bufmgr.disable_object_capture = mos_gem_disable_object_capture;
5307 bufmgr_gem->bufmgr.get_memory_info = mos_gem_get_memory_info;
5308 bufmgr_gem->bufmgr.get_devid = mos_gem_get_devid;
5309 bufmgr_gem->bufmgr.realloc_cache = mos_gem_realloc_cache;
5310 bufmgr_gem->bufmgr.set_context_param = mos_gem_set_context_param;
5311 bufmgr_gem->bufmgr.set_context_param_parallel = mos_gem_set_context_param_parallel;
5312 bufmgr_gem->bufmgr.set_context_param_load_balance = mos_gem_set_context_param_load_balance;
5313 bufmgr_gem->bufmgr.set_context_param_bond = mos_gem_set_context_param_bond;
5314 bufmgr_gem->bufmgr.get_context_param = mos_gem_get_context_param;
5315 bufmgr_gem->bufmgr.bo_create_from_prime = mos_gem_bo_create_from_prime;
5316 bufmgr_gem->bufmgr.bo_export_to_prime = mos_gem_bo_export_to_prime;
5317 bufmgr_gem->bufmgr.reg_read = mos_bufmg_reg_read;
5318 bufmgr_gem->bufmgr.get_reset_stats = mos_bufmg_get_reset_stats;
5319 bufmgr_gem->bufmgr.get_context_param_sseu = mos_bufmgr_get_context_param_sseu;
5320 bufmgr_gem->bufmgr.set_context_param_sseu = mos_bufmgr_set_context_param_sseu;
5321 bufmgr_gem->bufmgr.query_sys_engines = mos_bufmgr_query_sys_engines;
5322 bufmgr_gem->bufmgr.query_device_blob = mos_bufmgr_query_device_blob;
5323 bufmgr_gem->bufmgr.get_driver_info = mos_bufmgr_get_driver_info;
5324 bufmgr_gem->bufmgr.query_hw_ip_version = mos_bufmgr_query_hw_ip_version;
5325 bufmgr_gem->bufmgr.get_platform_information = mos_bufmgr_get_platform_information;
5326 bufmgr_gem->bufmgr.set_platform_information = mos_bufmgr_set_platform_information;
5327 bufmgr_gem->bufmgr.query_engines_count = mos_bufmgr_query_engines_count;
5328 bufmgr_gem->bufmgr.query_engines = mos_bufmgr_query_engines;
5329 bufmgr_gem->bufmgr.get_engine_class_size = mos_bufmgr_get_engine_class_size;
5330 bufmgr_gem->bufmgr.select_fixed_engine = mos_gem_select_fixed_engine;
5331 bufmgr_gem->bufmgr.switch_off_n_bits = mos_bufmgr_switch_off_n_bits;
5332 bufmgr_gem->bufmgr.hweight8 = mos_bufmgr_hweight8;
5333 bufmgr_gem->bufmgr.get_ts_frequency = mos_bufmgr_get_ts_frequency;
5334 bufmgr_gem->bufmgr.has_bsd2 = mos_bufmgr_has_bsd2;
5335 bufmgr_gem->bufmgr.enable_turbo_boost = mos_bufmgr_enable_turbo_boost;
5336
5337 bufmgr_gem->mem_profiler_path = getenv("MEDIA_MEMORY_PROFILER_LOG");
5338 if (bufmgr_gem->mem_profiler_path != nullptr)
5339 {
5340 if (strcmp(bufmgr_gem->mem_profiler_path, "/sys/kernel/debug/tracing/trace_marker") == 0)
5341 {
5342 ret = bufmgr_gem->mem_profiler_fd = open(bufmgr_gem->mem_profiler_path, O_WRONLY );
5343 }
5344 else
5345 {
5346 ret = bufmgr_gem->mem_profiler_fd = open(bufmgr_gem->mem_profiler_path, O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR);
5347 }
5348
5349 if (ret == -1)
5350 {
5351 MOS_DBG("Failed to open %s: %s\n", bufmgr_gem->mem_profiler_path, strerror(errno));
5352 }
5353 }
5354 else
5355 {
5356 bufmgr_gem->mem_profiler_fd = -1;
5357 }
5358
5359 memclear(aperture);
5360 ret = drmIoctl(bufmgr_gem->fd,
5361 DRM_IOCTL_I915_GEM_GET_APERTURE,
5362 &aperture);
5363
5364 if (ret == 0)
5365 bufmgr_gem->gtt_size = aperture.aper_available_size;
5366 else {
5367 fprintf(stderr, "DRM_IOCTL_I915_GEM_APERTURE failed: %s\n",
5368 strerror(errno));
5369 bufmgr_gem->gtt_size = 128 * 1024 * 1024;
5370 fprintf(stderr, "Assuming %dkB available aperture size.\n"
5371 "May lead to reduced performance or incorrect "
5372 "rendering.\n",
5373 (int)bufmgr_gem->gtt_size / 1024);
5374 }
5375
5376 /* support Gen 8+ */
5377 bufmgr_gem->pci_device = get_pci_device_id(bufmgr_gem);
5378
5379 if (bufmgr_gem->pci_device == 0) {
5380 pthread_mutex_destroy(&bufmgr_gem->lock);
5381 if (bufmgr_gem->mem_profiler_fd != -1)
5382 {
5383 close(bufmgr_gem->mem_profiler_fd);
5384 }
5385 free(bufmgr_gem);
5386 bufmgr_gem = nullptr;
5387 goto exit;
5388 }
5389
5390 memclear(gp);
5391 gp.value = &tmp;
5392
5393 gp.param = I915_PARAM_HAS_EXECBUF2;
5394 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
5395 if (!ret)
5396 exec2 = true;
5397
5398 gp.param = I915_PARAM_HAS_BSD;
5399 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
5400 bufmgr_gem->has_bsd = ret == 0;
5401
5402 gp.param = I915_PARAM_HAS_BSD2;
5403 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
5404 bufmgr_gem->has_bsd2 = (ret == 0 && *gp.value != 0);
5405
5406 gp.param = I915_PARAM_HAS_BLT;
5407 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
5408 bufmgr_gem->has_blt = ret == 0;
5409
5410 gp.param = I915_PARAM_HAS_RELAXED_FENCING;
5411 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
5412 bufmgr_gem->has_relaxed_fencing = ret == 0;
5413
5414 bufmgr_gem->bufmgr.bo_alloc_userptr = check_bo_alloc_userptr;
5415
5416 gp.param = I915_PARAM_HAS_WAIT_TIMEOUT;
5417 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
5418 bufmgr_gem->has_wait_timeout = ret == 0;
5419
5420 gp.param = I915_PARAM_HAS_LLC;
5421 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
5422 bufmgr_gem->has_llc = *gp.value;
5423
5424 gp.param = I915_PARAM_HAS_VEBOX;
5425 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
5426 bufmgr_gem->has_vebox = (ret == 0) & (*gp.value > 0);
5427
5428 gp.param = I915_PARAM_MMAP_VERSION;
5429 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
5430 bufmgr_gem->has_ext_mmap = (ret == 0) & (*gp.value > 0);
5431
5432 gp.param = I915_PARAM_NUM_FENCES_AVAIL;
5433 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
5434 bufmgr_gem->has_fence_reg = (ret == 0) & (*gp.value > 0);
5435
5436 gp.param = I915_PARAM_HAS_EXEC_SOFTPIN;
5437 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
5438 if (ret == 0 && *gp.value > 0) {
5439 bufmgr_gem->bufmgr.bo_set_softpin = mos_gem_bo_set_softpin;
5440 bufmgr_gem->bufmgr.bo_add_softpin_target = mos_gem_bo_add_softpin_target;
5441 }
5442
5443 gp.param = I915_PARAM_HAS_EXEC_ASYNC;
5444 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
5445 if (ret == 0 && *gp.value > 0) {
5446 bufmgr_gem->bufmgr.set_object_async = mos_gem_bo_set_object_async;
5447 bufmgr_gem->bufmgr.set_exec_object_async = mos_gem_bo_set_exec_object_async;
5448 }
5449
5450 gp.param = I915_PARAM_HAS_EXEC_CAPTURE;
5451 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
5452 if (ret == 0 && *gp.value > 0)
5453 {
5454 bufmgr_gem->bufmgr.set_object_capture = mos_gem_bo_set_object_capture;
5455 }
5456 else
5457 {
5458 bufmgr_gem->object_capture_disabled = true;
5459 }
5460
5461 gp.param = I915_PARAM_MMAP_GTT_VERSION;
5462 ret = drmIoctl(fd, DRM_IOCTL_I915_GETPARAM, &gp);
5463 bufmgr_gem->has_mmap_offset = (ret == 0) && (*gp.value >= 4);
5464
5465 gp.param = I915_PARAM_CS_TIMESTAMP_FREQUENCY;
5466 ret = drmIoctl(fd, DRM_IOCTL_I915_GETPARAM, &gp);
5467 bufmgr_gem->ts_freq = (ret == 0) ? *gp.value : 0;
5468
5469 struct drm_i915_gem_context_param context_param;
5470 memset(&context_param, 0, sizeof(context_param));
5471 context_param.param = I915_CONTEXT_PARAM_GTT_SIZE;
5472 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_CONTEXT_GETPARAM, &context_param);
5473 if (ret == 0){
5474 uint64_t gtt_size_4g = (uint64_t)1 << 32;
5475 if (context_param.value > gtt_size_4g)
5476 {
5477 bufmgr_gem->bufmgr.bo_use_48b_address_range = mos_gem_bo_use_48b_address_range;
5478 }
5479 }
5480
5481 bufmgr_gem->device_type = DEVICE_TYPE_I915;
5482 bufmgr_gem->has_lmem = mos_gem_has_lmem(bufmgr_gem->fd);
5483
5484 bufmgr_gem->bufmgr.has_full_vd = true;
5485
5486 /* Let's go with one relocation per every 2 dwords (but round down a bit
5487 * since a power of two will mean an extra page allocation for the reloc
5488 * buffer).
5489 *
5490 * Every 4 was too few for the blender benchmark.
5491 */
5492 alloc_mode = (uint8_t)(batch_size & 0xff);
5493 batch_size &= 0xffffff00;
5494 bufmgr_gem->max_relocs = batch_size / sizeof(uint32_t) / 2 - 2;
5495
5496 DRMINITLISTHEAD(&bufmgr_gem->named);
5497 init_cache_buckets(bufmgr_gem);
5498
5499 DRMLISTADD(&bufmgr_gem->managers, &bufmgr_list);
5500
5501 bufmgr_gem->use_softpin = false;
5502 mos_vma_heap_init(&bufmgr_gem->vma_heap[MEMZONE_SYS], MEMZONE_SYS_START, MEMZONE_SYS_SIZE);
5503 mos_vma_heap_init(&bufmgr_gem->vma_heap[MEMZONE_DEVICE], MEMZONE_DEVICE_START, MEMZONE_DEVICE_SIZE);
5504
5505 exit:
5506 pthread_mutex_unlock(&bufmgr_list_mutex);
5507
5508 return bufmgr_gem != nullptr ? &bufmgr_gem->bufmgr : nullptr;
5509 }
5510
5511 struct mos_bufmgr *
5512 mos_bufmgr_gem_init(int fd, int batch_size, int *device_type)
5513 {
5514 int type = mos_query_device_type(fd);
5515 if (device_type != nullptr)
5516 {
5517 *device_type = type;
5518 }
5519 if(DEVICE_TYPE_I915 == type)
5520 {
5521 return mos_bufmgr_gem_init_i915(fd, batch_size);
5522 }
5523 else if (DEVICE_TYPE_XE == type)
5524 {
5525 return mos_bufmgr_gem_init_xe(fd, batch_size);
5526 }
5527
5528 return nullptr;
5529 }
5530
5531 int mos_get_param(int fd, int32_t param, uint32_t *param_value)
5532 {
5533 if((fd < 0) || (param_value == nullptr))
5534 {
5535 return -EINVAL;
5536 }
5537
5538 struct drm_i915_getparam gp;
5539 gp.param = param;
5540 gp.value = (int32_t *)param_value;
5541
5542 return drmIoctl(fd, DRM_IOCTL_I915_GETPARAM, &gp) == 0;
5543 }
5544
5545 static int mos_bufmgr_get_driver_info(struct mos_bufmgr *bufmgr, struct LinuxDriverInfo *drvInfo)
5546 {
5547 if (bufmgr == nullptr || drvInfo == nullptr)
5548 {
5549 return -EINVAL;
5550 }
5551 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem*)bufmgr;
5552 int fd = bufmgr_gem->fd;
5553 if (fd < 0)
5554 {
5555 return -EINVAL;
5556 }
5557 uint32_t retValue = 0;
5558
5559 drvInfo->hasBsd = 0;
5560 if (mos_get_param(fd, I915_PARAM_HAS_BSD, &retValue))
5561 {
5562 drvInfo->hasBsd = !!retValue;
5563 }
5564
5565 drvInfo->hasBsd2 = 0;
5566 retValue = 0;
5567 if (mos_get_param(fd, I915_PARAM_HAS_BSD2, &retValue))
5568 {
5569 drvInfo->hasBsd2 = !!retValue;
5570 }
5571
5572 drvInfo->hasVebox = 0;
5573 retValue = 0;
5574 if (mos_get_param(fd, I915_PARAM_HAS_VEBOX, &retValue))
5575 {
5576 drvInfo->hasVebox = !!retValue;
5577 }
5578
5579 drvInfo->hasPpgtt = 1;
5580 retValue = 0;
5581 if (mos_get_param(fd, I915_PARAM_HAS_ALIASING_PPGTT, &retValue))
5582 {
5583 drvInfo->hasPpgtt = !!retValue;
5584 }
5585
5586 drvInfo->hasHuc = 0;
5587 retValue = 0;
5588 if (mos_get_param(fd, I915_PARAM_HUC_STATUS, &retValue))
5589 {
5590 drvInfo->hasHuc = !!retValue;
5591 if (retValue == 1)
5592 {
5593 drvInfo->hasProtectedHuc = 1;
5594 }
5595 }
5596
5597 drvInfo->devId = 0;
5598 retValue = 0;
5599 if (mos_get_param(fd, I915_PARAM_CHIPSET_ID, &retValue))
5600 {
5601 drvInfo->devId = retValue;
5602 }
5603 drvInfo->devRev = 0;
5604 retValue = 0;
5605 if (mos_get_param(fd, I915_PARAM_REVISION, &retValue))
5606 {
5607 drvInfo->devRev = retValue;
5608 }
5609
5610 drvInfo->euCount = 0;
5611 retValue = 0;
5612 if (mos_get_param(fd, I915_PARAM_EU_TOTAL, &retValue))
5613 {
5614 drvInfo->euCount = retValue;
5615 }
5616
5617 drvInfo->subSliceCount = 0;
5618 retValue = 0;
5619 if (mos_get_param(fd, I915_PARAM_SUBSLICE_TOTAL, &retValue))
5620 {
5621 drvInfo->subSliceCount = retValue;
5622 }
5623
5624 // There is no interface to read total slice count from drm/i915, so we
5625 // will set the slice count in InitMediaSysInfo accordint to Device ID.
5626 drvInfo->sliceCount = 0;
5627
5628 return 0;
5629 }
5630
5631 static int mos_get_dev_id_i915(int fd, uint32_t *device_id)
5632 {
5633 if (nullptr == device_id)
5634 {
5635 return -EINVAL;
5636 }
5637 uint32_t retValue = 0;
5638 if (mos_get_param(fd, I915_PARAM_CHIPSET_ID, &retValue))
5639 {
5640 *device_id = retValue;
5641 }
5642
5643 return 0;
5644 }
5645
5646 int mos_get_device_id(int fd, uint32_t *deviceId)
5647 {
5648 int device_type = mos_query_device_type(fd);
5649
5650 if (DEVICE_TYPE_I915 == device_type)
5651 {
5652 return mos_get_dev_id_i915(fd, deviceId);
5653 }
5654 else if (DEVICE_TYPE_XE == device_type)
5655 {
5656 return mos_get_dev_id_xe(fd, deviceId);
5657 }
5658
5659 return -ENODEV;
5660 }