Debian Codegraph @

media_softlet/linux/common/os/xe/mos_bufmgr_xe.c

raw
1 /*
2 * Copyright © 2023 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Xu, Zhengguo <[email protected]>
25 */
26
27 #ifdef HAVE_LIBGEN_H
28 #include <libgen.h>
29 #endif
30 #include <stdlib.h>
31 #include <fcntl.h>
32 #include <sys/stat.h>
33 #include <sys/mman.h>
34 #include <signal.h>
35 #include <getopt.h>
36 #include <stdlib.h>
37 #include <unistd.h>
38 #include <sys/wait.h>
39 #include <sys/types.h>
40 #include <sys/syscall.h>
41 #include <sys/utsname.h>
42 #include <termios.h>
43 #ifndef ETIME
44 #define ETIME ETIMEDOUT
45 #endif
46
47 #include <map>
48 #include <vector>
49 #include <queue>
50 #include <list>
51 #include <mutex>
52 #include <shared_mutex>
53 #include <algorithm>
54
55 #ifdef HAVE_VALGRIND
56 #include <valgrind/valgrind.h>
57 #include <valgrind/memcheck.h>
58
59 #define VG(x) x
60 #else
61 #define VG(x) do {} while (0)
62 #endif
63
64 #ifdef HAVE_CONFIG_H
65 #include "config.h"
66 #endif
67 #include "mos_bufmgr_api.h"
68 #include "mos_util_debug.h"
69 #include "intel_hwconfig_types.h"
70 #include "xf86drm.h"
71 #include "mos_vma.h"
72 #include "libdrm_lists.h"
73 #include "mos_bufmgr_xe.h"
74 #include "mos_synchronization_xe.h"
75 #include "mos_utilities.h"
76 #include "mos_bufmgr_util_debug.h"
77 #include "media_user_setting_value.h"
78 #include "linux_system_info.h"
79 #include "mos_oca_defs_specific.h"
80
81 //These two struct used by mos_bufmgr_priv.h
82 typedef struct MOS_OCA_EXEC_LIST_INFO mos_oca_exec_list_info;
83 //struct MEDIA_SYSTEM_INFO;
84
85 #include "mos_bufmgr_priv.h"
86
87 #define MAX(a, b) ((a) > (b) ? (a) : (b))
88 #define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
89
90 //mos_xe_mem_class currently used as index of default_alignment
91 enum mos_xe_mem_class
92 {
93 MOS_XE_MEM_CLASS_SYSMEM = 0, //For DRM_XE_MEM_REGION_CLASS_SYSMEM
94 MOS_XE_MEM_CLASS_VRAM, //For DRM_XE_MEM_REGION_CLASS_VRAM
95 MOS_XE_MEM_CLASS_MAX
96 };
97
98 struct mos_xe_context {
99 struct mos_linux_context ctx;
100
101 /**
102 * Always keep the latest avaiable timeline index for
103 * such execution's fence out point.
104 */
105 struct mos_xe_dep* timeline_dep;
106
107 /**
108 * The UMD's dummy exec_queue id for exec_queue ctx.
109 */
110 uint32_t dummy_exec_queue_id;
111
112 /**
113 * Indicate to the ctx width.
114 */
115 uint8_t ctx_width;
116 /**
117 * Indicate to num placements when creating exec_queue.
118 */
119 uint8_t num_placements;
120 /**
121 * Indicate to engine class used to create exec_queue.
122 */
123 uint16_t engine_class;
124 /**
125 * Indicate to engine capability of queried exec_queue.
126 */
127 uint64_t engine_caps;
128 /**
129 * Indicate to creation flags, current value should be always zero.
130 */
131 uint32_t flags;
132 /**
133 * Indicate whether it is protected ctx.
134 */
135 bool is_protected;
136
137 /**
138 * Indicate to exec_queue reset count on this context;
139 * Note, this count depends on context restore, if uplayer tries to query
140 * reset statue before context restore, this value may be incorrect.
141 */
142 uint32_t reset_count;
143 };
144
145 typedef struct mos_xe_device {
146 /**
147 * Note: we agree that hw_config[0] points to the number of hw config in total
148 * And hw config data starts from hw_config[1]
149 */
150 uint32_t *hw_config = nullptr;
151 struct drm_xe_query_config *config = nullptr;
152 struct drm_xe_query_engines *engines = nullptr;
153 struct drm_xe_query_mem_regions *mem_regions = nullptr;
154 struct drm_xe_query_gt_list *gt_list = nullptr;
155
156 /**
157 * Note: we agree here that uc_versions[0] for guc version and uc_versions[1] for huc version
158 */
159 struct drm_xe_query_uc_fw_version uc_versions[UC_TYPE_MAX];
160 } mos_xe_device;
161
162 struct mos_xe_gem_bo_bucket {
163 drmMMListHead sys_head;
164 drmMMListHead vram_head;
165 unsigned long size;
166 };
167
168 struct mos_xe_bucket_lab {
169 /**
170 * 4K aligned for all
171 */
172 #define CACHE_BUCKET_MODE_DEFAULT 0
173 /**
174 * 64K aligned for size < 1M and 4K aligned for size >=1M
175 */
176 #define CACHE_BUCKET_MODE_64K 1
177 /**
178 * 2M aligned for size > 1M and 4K aligned for size >= 1M
179 */
180 #define CACHE_BUCKET_MODE_2M 2
181 /**
182 * 64K aligned for size <=1M and 2M aligned for size > 1M
183 */
184 #define CACHE_BUCKET_MODE_64K_2M 3
185 struct mos_xe_gem_bo_bucket cache_buckets[64];
186 int num_buckets;
187 uint64_t max_cache_size;
188 uint8_t cache_mode;
189 bool enable_bo_reuse;
190 };
191
192 typedef struct mos_xe_bufmgr_gem {
193 struct mos_bufmgr bufmgr;
194
195 atomic_t ref_count;
196
197 int fd;
198
199 std::recursive_mutex m_lock;
200
201 drmMMListHead managers;
202 drmMMListHead named;
203
204 mos_vma_heap vma_heap[MEMZONE_COUNT];
205
206 bool object_capture_disabled; // Note: useless on xe and remove it in furture.
207
208 #define MEM_PROFILER_BUFFER_SIZE 256
209 char mem_profiler_buffer[MEM_PROFILER_BUFFER_SIZE];
210 char* mem_profiler_path;
211 int mem_profiler_fd;
212
213 uint32_t gt_id;
214
215 /**
216 * This RW lock is used for avoid reading or writing the same sync obj in KMD.
217 * Reading sync obj ioctl: exec and syncobj wait.
218 * Writing sync obj ioctl: reset sync obj, destroy sync obj and create sync obj.
219 */
220 std::shared_timed_mutex sync_obj_rw_lock;
221
222 /**
223 * Save the pair of UMD dummy exec_queue id and ctx pointer.
224 */
225 std::map<uint32_t, struct mos_xe_context*> global_ctx_info;
226
227 uint32_t vm_id;
228
229 /**
230 * Everything queried from kmd that indicates to hw infomation.
231 */
232 struct mos_xe_device xe_device;
233
234 //Note: DON't put these fields in xe_device
235 bool has_vram;
236 uint8_t va_bits;
237 /** bitmask of all memory regions */
238 uint64_t mem_regions_mask;
239 /** @default_alignment: safe alignment regardless region location */
240 uint32_t default_alignment[MOS_XE_MEM_CLASS_MAX] = {PAGE_SIZE_4K, PAGE_SIZE_4K};
241 //End of Note
242
243 struct mos_xe_bucket_lab bucket_lab;
244
245 /**
246 * Indicates whether gpu-gpu and cpu-gpu synchronization is disabled.
247 * This is mainly for debug purpose, and synchronizarion should be always enabled by default.
248 * It could be disabled by env INTEL_SYNCHRONIZATION_DISABLE.
249 */
250 bool is_disable_synchronization;
251
252 /** indicate to exec_queue property of timeslice */
253 #define EXEC_QUEUE_TIMESLICE_DEFAULT -1
254 #define EXEC_QUEUE_TIMESLICE_MAX 100000 //100ms
255 int32_t exec_queue_timeslice;
256 } mos_xe_bufmgr_gem;
257
258 typedef struct mos_xe_exec_bo {
259 /** indicate to real exec bo*/
260 struct mos_linux_bo *bo;
261
262 /**
263 * Save read, write flags etc.
264 * Two flags defined here: EXEC_OBJECT_READ_XE and EXEC_OBJECT_WRITE_XE.
265 * Whether this bo needs exec sync depends on this flags.
266 */
267 uint32_t flags;
268 } mos_xe_exec_bo;
269
270 typedef struct mos_xe_bo_gem {
271 /**
272 * Maximun size for bo name
273 */
274 #define MAX_NAME_SIZE 128
275
276 struct mos_linux_bo bo;
277
278 /**
279 * Reference count
280 */
281 atomic_t ref_count;
282 /**
283 * Map count when map bo is called
284 */
285 atomic_t map_count;
286
287 //Note7: unify gem_handle and bo.handle by deleting this one; Refine mos_linux_bo.handle to typt of uint32_t
288 /**
289 * Bo handle allocared from drm
290 * Note: conbine with bo.handle to use same one.
291 */
292 uint32_t gem_handle;
293 /**
294 * Save bo name, this is for debug usage;
295 * Suggest giving bo name when allocating bo.
296 */
297 char name[MAX_NAME_SIZE];
298
299 /**
300 *
301 * List contains prime fd'd objects
302 */
303 drmMMListHead name_list;
304
305 /**
306 * Mapped address for the buffer, saved across map/unmap cycles
307 */
308 void *mem_virtual;
309
310 /**
311 * Boolean of whether this buffer was allocated with userptr
312 */
313 bool is_userptr;
314
315 /**
316 * Memory region on created the surfaces for local/system memory;
317 * This field only indicates to memory region type, it not memory region instance.
318 */
319 int mem_region;
320
321 /**
322 * We should always get the syncobj handle from the bo handle by bellow 4 steps in each time:
323 * 1. get the prime_handle from bo.handle
324 * 2. get syncfile fd from prime_fd
325 * 3. get syncobj_handle from syncfile by
326 * 4. close prime_fd and syncfile fd.
327 *
328 * If umd wants external process to sync between them, umd should always import its batch
329 * syncobj handle into each external bo's dma sync buffer.
330 *
331 * Boolean of whether this buffer is imported from external
332 */
333 bool is_imported;
334 /**
335 * @cpu_caching: The CPU caching mode to select for this object. If
336 * mmaping the object the mode selected here will also be used.
337 *
338 * Supported values:
339 *
340 * DRM_XE_GEM_CPU_CACHING_WB: Allocate the pages with write-back
341 * caching. On iGPU this can't be used for scanout surfaces. Currently
342 * not allowed for objects placed in VRAM.
343 *
344 * DRM_XE_GEM_CPU_CACHING_WC: Allocate the pages as write-combined. This
345 * is uncached. Scanout surfaces should likely use this. All objects
346 * that can be placed in VRAM must use this.
347 */
348 uint16_t cpu_caching;
349
350 /**
351 * @pat_index: The platform defined @pat_index to use for this mapping.
352 * The index basically maps to some predefined memory attributes,
353 * including things like caching, coherency, compression etc. The exact
354 * meaning of the pat_index is platform specific. When the KMD sets up
355 * the binding the index here is encoded into the ppGTT PTE.
356 *
357 * For coherency the @pat_index needs to be at least 1way coherent when
358 * drm_xe_gem_create.cpu_caching is DRM_XE_GEM_CPU_CACHING_WB. The KMD
359 * will extract the coherency mode from the @pat_index and reject if
360 * there is a mismatch (see note below for pre-MTL platforms).
361 *
362 * Note: On pre-MTL platforms there is only a caching mode and no
363 * explicit coherency mode, but on such hardware there is always a
364 * shared-LLC (or is dgpu) so all GT memory accesses are coherent with
365 * CPU caches even with the caching mode set as uncached. It's only the
366 * display engine that is incoherent (on dgpu it must be in VRAM which
367 * is always mapped as WC on the CPU). However to keep the uapi somewhat
368 * consistent with newer platforms the KMD groups the different cache
369 * levels into the following coherency buckets on all pre-MTL platforms:
370 *
371 * ppGTT UC -> COH_NONE
372 * ppGTT WC -> COH_NONE
373 * ppGTT WT -> COH_NONE
374 * ppGTT WB -> COH_AT_LEAST_1WAY
375 *
376 * In practice UC/WC/WT should only ever used for scanout surfaces on
377 * such platforms (or perhaps in general for dma-buf if shared with
378 * another device) since it is only the display engine that is actually
379 * incoherent. Everything else should typically use WB given that we
380 * have a shared-LLC. On MTL+ this completely changes and the HW
381 * defines the coherency mode as part of the @pat_index, where
382 * incoherent GT access is possible.
383 *
384 * Note: For userptr and externally imported dma-buf the kernel expects
385 * either 1WAY or 2WAY for the @pat_index.
386 */
387 uint16_t pat_index;
388
389 /**
390 * Boolean of whether this buffer is exported to external
391 */
392 bool is_exported;
393
394 /**
395 * For cmd bo, it has an exec bo list which saves all exec bo in it.
396 * Uplayer caller should alway update this list before exec submission and clear the list after exec submission.
397 */
398 std::map<uintptr_t, struct mos_xe_exec_bo> exec_list;
399
400 #define INVALID_EXEC_QUEUE_ID -1
401 /**
402 * Save last dummy write exec_queue id.
403 * Init this field as INVALID_EXEC_QUEUE_ID at begining.
404 */
405 uint32_t last_exec_write_exec_queue;
406
407 /**
408 * Save last dummy read exec_queue id.
409 * Init this field as INVALID_EXEC_QUEUE_ID at begining.
410 */
411 uint32_t last_exec_read_exec_queue;
412
413 /**
414 * Read dependents, pair of dummy EXEC_QUEUE_ID and mos_xe_bo_dep
415 * This map saves read deps of this bo on all exec exec_queue;
416 * Exec will check opration flags to get the dep from the map to add into exec sync array and updated the map after exec.
417 * Refer to exec call to get more details.
418 */
419 std::map<uint32_t, struct mos_xe_bo_dep> read_deps;
420
421 /**
422 * Write dependents, pair of dummy EXEC_QUEUE_ID and mos_xe_bo_dep
423 * This map saves write deps of this bo on all exec exec_queue;
424 * Exec will check opration flags to get the dep from the map to add into exec sync array and updated the map after exec.
425 * Refer to exec call to get more details.
426 */
427 std::map<uint32_t, struct mos_xe_bo_dep> write_deps;
428
429 } mos_xe_bo_gem;
430
431 struct mos_xe_external_bo_info {
432 /**
433 * syncobj handle created by umd to import external bo syncfile
434 */
435 int syncobj_handle;
436 /**
437 * prime fd export from external bo handle
438 */
439 int prime_fd;
440 };
441
442 #define MOS_UNIMPLEMENT(param) (void)(param)
443
444 static pthread_mutex_t bufmgr_list_mutex = PTHREAD_MUTEX_INITIALIZER;
445 static drmMMListHead bufmgr_list = { &bufmgr_list, &bufmgr_list };
446
447 static void mos_bo_free_xe(struct mos_linux_bo *bo);
448 static int mos_query_engines_count_xe(struct mos_bufmgr *bufmgr, unsigned int *nengine);
449 int mos_query_engines_xe(struct mos_bufmgr *bufmgr,
450 __u16 engine_class,
451 __u64 caps,
452 unsigned int *nengine,
453 void *engine_map);
454 static void mos_gem_bo_wait_rendering_xe(struct mos_linux_bo *bo);
455
456 static struct mos_xe_gem_bo_bucket*
457 __mos_gem_find_bucket_xe(
458 struct mos_xe_gem_bo_bucket *buckets,
459 int num_buckets, uint64_t alloc_size);
460
461 static struct mos_xe_bufmgr_gem *
462 mos_bufmgr_gem_find(int fd)
463 {
464 struct mos_xe_bufmgr_gem *bufmgr_gem;
465
466 DRMLISTFOREACHENTRY(bufmgr_gem, &bufmgr_list, managers) {
467 if (bufmgr_gem->fd == fd) {
468 atomic_inc(&bufmgr_gem->ref_count);
469 return bufmgr_gem;
470 }
471 }
472
473 return nullptr;
474 }
475
476 #define MOS_DRM_CHK_XE_DEV(xe_dev, info, query_func, retval) \
477 MOS_DRM_CHK_NULL_RETURN_VALUE(xe_dev, retval); \
478 if (xe_dev->info == nullptr) \
479 { \
480 xe_dev->info = query_func(fd); \
481 MOS_DRM_CHK_NULL_RETURN_VALUE(xe_dev->info, retval); \
482 }
483
484 static struct drm_xe_query_gt_list *
485 __mos_query_gt_list_xe(int fd)
486 {
487 int ret = 0;
488 struct drm_xe_query_gt_list *gt_list;
489 struct drm_xe_device_query query;
490 memclear(query);
491 query.query = DRM_XE_DEVICE_QUERY_GT_LIST;
492
493 ret = drmIoctl(fd, DRM_IOCTL_XE_DEVICE_QUERY,
494 &query);
495 if (ret || !query.size)
496 {
497 return nullptr;
498 }
499
500 gt_list = (drm_xe_query_gt_list *)calloc(1, query.size);
501 MOS_DRM_CHK_NULL_RETURN_VALUE(gt_list, nullptr);
502
503 query.data = (uintptr_t)(gt_list);
504 ret = drmIoctl(fd, DRM_IOCTL_XE_DEVICE_QUERY,
505 &query);
506 if (ret || !query.size || 0 == gt_list->num_gt)
507 {
508 MOS_XE_SAFE_FREE(gt_list);
509 return nullptr;
510 }
511
512 return gt_list;
513 }
514
515 static uint32_t __mos_query_mem_regions_instance_mask_xe(struct mos_bufmgr *bufmgr)
516 {
517 MOS_DRM_CHK_NULL_RETURN_VALUE(bufmgr, 0)
518 struct mos_xe_bufmgr_gem *bufmgr_gem = (struct mos_xe_bufmgr_gem *)bufmgr;
519 struct mos_xe_device *dev = &bufmgr_gem->xe_device;
520 int fd = bufmgr_gem->fd;
521 uint64_t __memory_regions = 0;
522
523 MOS_DRM_CHK_XE_DEV(dev, gt_list, __mos_query_gt_list_xe, 0)
524
525 struct drm_xe_query_gt_list *gt_list = dev->gt_list;
526 for (int i = 0; i < gt_list->num_gt; i++) {
527 /**
528 * Note: __memory_regions is the mem region instance mask on all tiles and gts
529 */
530 __memory_regions |= gt_list->gt_list[i].near_mem_regions |
531 gt_list->gt_list[i].far_mem_regions;
532 }
533
534 bufmgr_gem->mem_regions_mask = __memory_regions;
535
536 return __memory_regions;
537 }
538
539 static struct drm_xe_query_mem_regions *
540 __mos_query_mem_regions_xe(int fd)
541 {
542 int ret = 0;
543 struct drm_xe_query_mem_regions *mem_regions;
544 struct drm_xe_device_query query;
545 memclear(query);
546 query.query = DRM_XE_DEVICE_QUERY_MEM_REGIONS;
547
548 ret = drmIoctl(fd, DRM_IOCTL_XE_DEVICE_QUERY,
549 &query);
550 if (ret || !query.size)
551 {
552 return nullptr;
553 }
554
555 mem_regions = (drm_xe_query_mem_regions *)calloc(1, query.size);
556 MOS_DRM_CHK_NULL_RETURN_VALUE(mem_regions, nullptr);
557
558 query.data = (uintptr_t)(mem_regions);
559 ret = drmIoctl(fd, DRM_IOCTL_XE_DEVICE_QUERY, &query);
560 if (ret || !query.size || 0 == mem_regions->num_mem_regions)
561 {
562 MOS_XE_SAFE_FREE(mem_regions);
563 return nullptr;
564 }
565
566 return mem_regions;
567 }
568
569 uint8_t __mos_query_vram_region_count_xe(struct mos_xe_device *dev, int fd)
570 {
571 uint8_t vram_regions = 0;
572
573 MOS_DRM_CHK_XE_DEV(dev, mem_regions, __mos_query_mem_regions_xe, 0)
574
575 struct drm_xe_query_mem_regions *mem_regions = dev->mem_regions;
576 for (int i =0; i < mem_regions->num_mem_regions; i++)
577 {
578 if (mem_regions->mem_regions[i].mem_class == DRM_XE_MEM_REGION_CLASS_VRAM)
579 {
580 vram_regions++;
581 }
582 }
583
584 return vram_regions;
585 }
586
587 int mos_force_gt_reset_xe(int fd, int gt_id)
588 {
589 char reset_string[128];
590
591 sprintf(reset_string, "cat /sys/kernel/debug/dri/0/gt%d/force_reset", gt_id);
592 return system(reset_string);
593 }
594
595 static struct drm_xe_query_config *
596 __mos_query_config_xe(int fd)
597 {
598 struct drm_xe_query_config *config;
599 struct drm_xe_device_query query;
600 int ret = 0;
601
602 memclear(query);
603 query.query = DRM_XE_DEVICE_QUERY_CONFIG;
604 ret = drmIoctl(fd, DRM_IOCTL_XE_DEVICE_QUERY, (void *)&query);
605 if (ret || !query.size)
606 {
607 return nullptr;
608 }
609
610 config = (drm_xe_query_config *) malloc(query.size);
611 if (config != nullptr)
612 {
613 memset(config, 0, query.size);
614 }
615 else
616 {
617 MOS_DRM_ASSERTMESSAGE("malloc config failed");
618 return nullptr;
619 }
620
621 query.data = (uintptr_t)config;
622 ret = drmIoctl(fd, DRM_IOCTL_XE_DEVICE_QUERY, (void *)&query);
623 if (ret || !query.size || 0 == config->num_params)
624 {
625 MOS_XE_SAFE_FREE(config);
626 return nullptr;
627 }
628
629 return config;
630 }
631
632 static int
633 __mos_get_default_alignment_xe(struct mos_bufmgr *bufmgr)
634 {
635 MOS_DRM_CHK_NULL_RETURN_VALUE(bufmgr, -EINVAL)
636 struct mos_xe_bufmgr_gem *bufmgr_gem = (struct mos_xe_bufmgr_gem *)bufmgr;
637 struct mos_xe_device *dev = &bufmgr_gem->xe_device;
638 int fd = bufmgr_gem->fd;
639 MOS_DRM_CHK_XE_DEV(dev, mem_regions, __mos_query_mem_regions_xe, -ENODEV)
640 struct drm_xe_query_mem_regions *mem_regions = dev->mem_regions;
641 uint16_t mem_class;
642
643 for (int i = 0; i < mem_regions->num_mem_regions; i++)
644 {
645 if (DRM_XE_MEM_REGION_CLASS_SYSMEM == mem_regions->mem_regions[i].mem_class)
646 {
647 mem_class = MOS_XE_MEM_CLASS_SYSMEM;
648 }
649 else if (DRM_XE_MEM_REGION_CLASS_VRAM == mem_regions->mem_regions[i].mem_class)
650 {
651 mem_class = MOS_XE_MEM_CLASS_VRAM;
652 }
653 else
654 {
655 MOS_DRM_ASSERTMESSAGE("Unsupported mem class");
656 return -EINVAL;
657 }
658
659 if (bufmgr_gem->default_alignment[mem_class] < mem_regions->mem_regions[i].min_page_size)
660 {
661 bufmgr_gem->default_alignment[mem_class] = mem_regions->mem_regions[i].min_page_size;
662 }
663 }
664
665 return 0;
666 }
667
668 /**
669 * Note: Need to add this func to bufmgr api later
670 */
671 static int
672 mos_query_uc_version_xe(struct mos_bufmgr *bufmgr, struct mos_drm_uc_version *version)
673 {
674 int ret = 0;
675 struct mos_xe_bufmgr_gem *bufmgr_gem = (struct mos_xe_bufmgr_gem *)bufmgr;
676 int fd = bufmgr_gem->fd;
677 struct mos_xe_device *dev = &bufmgr_gem->xe_device;
678
679 if (bufmgr && version && version->uc_type < UC_TYPE_MAX)
680 {
681 /**
682 * Note: query uc version from kmd if no historic data in bufmgr, otherwise using historic data.
683 */
684 if (dev->uc_versions[version->uc_type].uc_type != version->uc_type)
685 {
686 struct drm_xe_device_query query;
687 memclear(query);
688 query.size = sizeof(struct drm_xe_query_uc_fw_version);
689 query.query = DRM_XE_DEVICE_QUERY_UC_FW_VERSION;
690 memclear(dev->uc_versions[version->uc_type]);
691 dev->uc_versions[version->uc_type].uc_type = version->uc_type;
692 query.data = (uintptr_t)&dev->uc_versions[version->uc_type];
693
694 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_XE_DEVICE_QUERY,
695 &query);
696 if (ret)
697 {
698 memclear(dev->uc_versions[version->uc_type]);
699 dev->uc_versions[version->uc_type].uc_type = UC_TYPE_INVALID;
700 MOS_DRM_ASSERTMESSAGE("Failed to query UC version, uc type: %d, errno: %d", version->uc_type, ret);
701 return ret;
702 }
703 }
704
705 version->major_version = dev->uc_versions[version->uc_type].major_ver;
706 version->minor_version = dev->uc_versions[version->uc_type].minor_ver;
707 }
708
709 return ret;
710 }
711
712 bool __mos_has_vram_xe(struct mos_bufmgr *bufmgr)
713 {
714 MOS_DRM_CHK_NULL_RETURN_VALUE(bufmgr, 0)
715 struct mos_xe_bufmgr_gem *bufmgr_gem = (struct mos_xe_bufmgr_gem *)bufmgr;
716 struct mos_xe_device *dev = &bufmgr_gem->xe_device;
717 int fd = bufmgr_gem->fd;
718 MOS_DRM_CHK_XE_DEV(dev, config, __mos_query_config_xe, 0)
719 struct drm_xe_query_config *config = dev->config;
720 bool has_vram = ((config->info[DRM_XE_QUERY_CONFIG_FLAGS] & DRM_XE_QUERY_CONFIG_FLAG_HAS_VRAM) > 0);
721 bufmgr_gem->has_vram = has_vram;
722 return has_vram;
723 }
724
725 uint8_t __mos_query_va_bits_xe(struct mos_bufmgr *bufmgr)
726 {
727 uint8_t va_bits = 48;
728 MOS_DRM_CHK_NULL_RETURN_VALUE(bufmgr, va_bits)
729 struct mos_xe_bufmgr_gem *bufmgr_gem = (struct mos_xe_bufmgr_gem *)bufmgr;
730 struct mos_xe_device *dev = &bufmgr_gem->xe_device;
731 int fd = bufmgr_gem->fd;
732 bufmgr_gem->va_bits = va_bits;
733 MOS_DRM_CHK_XE_DEV(dev, config, __mos_query_config_xe, va_bits)
734 struct drm_xe_query_config *config = dev->config;
735 va_bits = config->info[DRM_XE_QUERY_CONFIG_VA_BITS] & 0xff;
736 bufmgr_gem->va_bits = va_bits;
737 return va_bits;
738 }
739
740 static uint64_t
741 mos_get_platform_information_xe(struct mos_bufmgr *bufmgr)
742 {
743 MOS_DRM_CHK_NULL_RETURN_VALUE(bufmgr, 0)
744 return bufmgr->platform_information;
745 }
746
747 static void
748 mos_set_platform_information_xe(struct mos_bufmgr *bufmgr, uint64_t p)
749 {
750 if (bufmgr)
751 bufmgr->platform_information |= p;
752 }
753
754 static enum mos_memory_zone
755 __mos_bo_memzone_for_address_xe(uint64_t address)
756 {
757 if (address >= MEMZONE_PRIME_START)
758 return MEMZONE_PRIME;
759 else if (address >= MEMZONE_DEVICE_START)
760 return MEMZONE_DEVICE;
761 else
762 return MEMZONE_SYS;
763 }
764
765 static void
766 __mos_bo_vma_free_xe(struct mos_bufmgr *bufmgr,
767 uint64_t address,
768 uint64_t size)
769 {
770 CHK_CONDITION(nullptr == bufmgr, "nullptr bufmgr.\n", );
771 struct mos_xe_bufmgr_gem *bufmgr_gem = (struct mos_xe_bufmgr_gem *) bufmgr;
772
773 CHK_CONDITION(0ull == address, "invalid address.\n", );
774 enum mos_memory_zone memzone = __mos_bo_memzone_for_address_xe(address);
775 mos_vma_heap_free(&bufmgr_gem->vma_heap[memzone], address, size);
776 }
777
778 static void
779 __mos_bo_mark_mmaps_incoherent_xe(struct mos_linux_bo *bo)
780 {
781 #if HAVE_VALGRIND
782 struct mos_xe_bo_gem *bo_gem = (struct mos_xe_bo_gem *) bo;
783
784 if (bo_gem->mem_virtual)
785 VALGRIND_MAKE_MEM_NOACCESS(bo_gem->mem_virtual, bo->size);
786 #endif
787 }
788
789 static inline void
790 mos_bo_reference_xe(struct mos_linux_bo *bo)
791 {
792 struct mos_xe_bo_gem *bo_gem = (struct mos_xe_bo_gem *) bo;
793
794 atomic_inc(&bo_gem->ref_count);
795 }
796
797 drm_export void mos_bo_unreference_xe(struct mos_linux_bo *bo)
798 {
799 struct mos_xe_bo_gem *bo_gem = (struct mos_xe_bo_gem *) bo;
800
801 if (atomic_read(&bo_gem->ref_count) <= 0)
802 return;
803
804 if (atomic_dec_and_test(&bo_gem->ref_count))
805 {
806 /* release memory associated with this object */
807 /* Clear any left-over mappings */
808 if (atomic_read(&bo_gem->map_count) > 0)
809 {
810 atomic_set(&bo_gem->map_count, 0);
811 __mos_bo_mark_mmaps_incoherent_xe(bo);
812 }
813
814 DRMLISTDEL(&bo_gem->name_list);
815
816 mos_bo_free_xe(bo);
817 }
818 }
819
820 static uint32_t
821 __mos_vm_create_xe(struct mos_bufmgr *bufmgr)
822 {
823 struct mos_xe_bufmgr_gem *bufmgr_gem = (struct mos_xe_bufmgr_gem *)bufmgr;
824 struct drm_xe_vm_create vm;
825 int ret;
826
827 memclear(vm);
828 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_XE_VM_CREATE, &vm);
829 if (ret != 0)
830 {
831 MOS_DRM_ASSERTMESSAGE("DRM_IOCTL_XE_VM_CREATE failed: %s",
832 strerror(errno));
833 return INVALID_VM;
834 }
835
836 return vm.vm_id;
837 }
838
839 static void
840 __mos_vm_destroy_xe(struct mos_bufmgr *bufmgr, uint32_t vm_id)
841 {
842 struct mos_xe_bufmgr_gem *bufmgr_gem = (struct mos_xe_bufmgr_gem *)bufmgr;
843 struct drm_xe_vm_destroy vm_destroy;
844 int ret;
845
846 if (INVALID_VM == vm_id)
847 {
848 MOS_DRM_ASSERTMESSAGE("invalid vm_id");
849 return;
850 }
851
852 memclear(vm_destroy);
853 vm_destroy.vm_id = vm_id;
854 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_XE_VM_DESTROY, &vm_destroy);
855 if (ret != 0)
856 {
857 MOS_DRM_ASSERTMESSAGE("DRM_IOCTL_XE_VM_DESTROY failed: %s",
858 strerror(errno));
859 }
860 }
861
862
863 static uint32_t
864 mos_vm_create_xe(struct mos_bufmgr *bufmgr)
865 {
866 struct mos_xe_bufmgr_gem *bufmgr_gem = (struct mos_xe_bufmgr_gem *)bufmgr;
867
868 if (bufmgr_gem->vm_id != INVALID_VM)
869 {
870 return bufmgr_gem->vm_id;
871 }
872 else
873 {
874 return __mos_vm_create_xe(bufmgr);
875 }
876 }
877
878 static void
879 mos_vm_destroy_xe(struct mos_bufmgr *bufmgr, uint32_t vm_id)
880 {
881 struct mos_xe_bufmgr_gem *bufmgr_gem = (struct mos_xe_bufmgr_gem *)bufmgr;
882
883 if (vm_id != bufmgr_gem->vm_id)
884 {
885 __mos_vm_destroy_xe(bufmgr, vm_id);
886 }
887 }
888
889 static struct mos_linux_context *
890 mos_context_create_shared_xe(
891 struct mos_bufmgr *bufmgr,
892 mos_linux_context* ctx,
893 __u32 flags,
894 bool bContextProtected,
895 void *engine_map,
896 uint8_t ctx_width,
897 uint8_t num_placements,
898 uint32_t ctx_type)
899 {
900 MOS_UNUSED(ctx);
901 MOS_UNUSED(ctx_type);
902
903 MOS_DRM_CHK_NULL_RETURN_VALUE(bufmgr, nullptr)
904 MOS_DRM_CHK_NULL_RETURN_VALUE(engine_map, nullptr)
905
906 static uint32_t dummy_exec_queue_id = 0;
907 struct mos_xe_bufmgr_gem *bufmgr_gem = (struct mos_xe_bufmgr_gem *)bufmgr;
908 struct mos_xe_context *context = nullptr;
909 struct drm_xe_exec_queue_create create;
910 int ret;
911 uint16_t engine_class = ((struct drm_xe_engine_class_instance *)engine_map)[0].engine_class;
912
913 memclear(create);
914 create.width = ctx_width;
915 create.num_placements = num_placements;
916 create.vm_id = bufmgr_gem->vm_id;
917 create.flags = flags;
918 create.instances = (uintptr_t)engine_map;
919
920 /**
921 * Note: must use MOS_New to allocate buffer instead of malloc since mos_xe_context
922 * contains std::vector and std::queue. Otherwise both will have no instance.
923 */
924 context = MOS_New(mos_xe_context);
925 MOS_DRM_CHK_NULL_RETURN_VALUE(context, nullptr)
926
927 struct drm_xe_ext_set_property* ext = nullptr;
928 struct drm_xe_ext_set_property timeslice;
929 struct drm_xe_ext_set_property protect;
930
931 /**
932 * Set exec_queue timeslice for render/ compute only as WA to ensure exec sequence.
933 * Note, this is caused by a potential issue in kmd since exec_queue preemption by plenty of WL w/ same priority.
934 */
935 if ((engine_class == DRM_XE_ENGINE_CLASS_RENDER
936 || engine_class == DRM_XE_ENGINE_CLASS_COMPUTE)
937 && (ctx_width * num_placements == 1)
938 && bufmgr_gem->exec_queue_timeslice != EXEC_QUEUE_TIMESLICE_DEFAULT)
939 {
940 memclear(timeslice);
941 timeslice.property = DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE;
942 /**
943 * Note, this value indicates to maximum of time slice for WL instead of real waiting time.
944 */
945 timeslice.value = bufmgr_gem->exec_queue_timeslice;
946 timeslice.base.name = DRM_XE_EXEC_QUEUE_EXTENSION_SET_PROPERTY;
947 ext = &timeslice;
948 MOS_DRM_NORMALMESSAGE("WA: exec_queue timeslice set by engine class(%d), value(%d)",
949 engine_class, bufmgr_gem->exec_queue_timeslice);
950 }
951
952 /**
953 * Set exec_queue protect for PXP usage.
954 */
955 if (bContextProtected)
956 {
957 memclear(protect);
958 protect.base.name = DRM_XE_EXEC_QUEUE_EXTENSION_SET_PROPERTY,
959 protect.property = DRM_XE_EXEC_QUEUE_SET_PROPERTY_PXP_TYPE,
960 protect.value = DRM_XE_PXP_TYPE_HWDRM;
961
962 protect.base.next_extension = (uintptr_t)ext;
963 ext = &protect;
964 }
965
966 create.extensions = (uintptr_t)ext;
967
968 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_XE_EXEC_QUEUE_CREATE, &create);
969
970 MOS_DRM_CHK_STATUS_MESSAGE_RETURN_VALUE_WH_OP(ret, context, MOS_Delete, nullptr,
971 "ioctl failed in DRM_IOCTL_XE_EXEC_QUEUE_CREATE, return error(%d)", ret);
972
973 context->ctx.ctx_id = create.exec_queue_id;
974 context->ctx_width = ctx_width;
975 context->num_placements = num_placements;
976 context->engine_class = ((struct drm_xe_engine_class_instance *)engine_map)[0].engine_class;
977 context->is_protected = bContextProtected;
978 context->flags = flags;
979 context->ctx.bufmgr = bufmgr;
980 context->ctx.vm_id = bufmgr_gem->vm_id;
981 context->reset_count = 0;
982 context->timeline_dep = nullptr;
983
984 bufmgr_gem->m_lock.lock();
985 context->dummy_exec_queue_id = ++dummy_exec_queue_id;
986 bufmgr_gem->global_ctx_info[context->dummy_exec_queue_id] = context;
987 bufmgr_gem->m_lock.unlock();
988 return &context->ctx;
989 }
990
991 static struct mos_linux_context *
992 mos_context_create_xe(struct mos_bufmgr *bufmgr)
993 {
994 struct mos_xe_bufmgr_gem *bufmgr_gem = (struct mos_xe_bufmgr_gem *)bufmgr;
995 struct mos_xe_context *context = nullptr;
996
997 /**
998 * Note: must use MOS_New to allocate buffer instead of malloc since mos_xe_context
999 * contains std::queue. Otherwise queue will have no instance.
1000 */
1001 context = MOS_New(mos_xe_context);
1002 MOS_DRM_CHK_NULL_RETURN_VALUE(context, nullptr)
1003
1004 context->ctx.ctx_id = INVALID_EXEC_QUEUE_ID;
1005 context->ctx_width = 0;
1006 context->ctx.bufmgr = bufmgr;
1007 context->ctx.vm_id = bufmgr_gem->vm_id;
1008 context->reset_count = 0;
1009 context->timeline_dep = nullptr;
1010 context->dummy_exec_queue_id = INVALID_EXEC_QUEUE_ID;
1011 return &context->ctx;
1012 }
1013
1014 static struct mos_linux_context *
1015 mos_context_create_ext_xe(
1016 struct mos_bufmgr *bufmgr,
1017 __u32 flags,
1018 bool bContextProtected)
1019 {
1020 MOS_UNUSED(flags);
1021 MOS_UNUSED(bContextProtected);
1022
1023 return mos_context_create_xe(bufmgr);
1024 }
1025
1026 static void
1027 mos_context_destroy_xe(struct mos_linux_context *ctx)
1028 {
1029 if (nullptr == ctx)
1030 {
1031 return;
1032 }
1033
1034 struct mos_xe_bufmgr_gem *bufmgr_gem = (struct mos_xe_bufmgr_gem *)(ctx->bufmgr);
1035 if (nullptr == bufmgr_gem)
1036 {
1037 return;
1038 }
1039 struct mos_xe_context *context = (struct mos_xe_context *)ctx;
1040 struct drm_xe_exec_queue_destroy exec_queue_destroy;
1041 int ret;
1042 bufmgr_gem->m_lock.lock();
1043 bufmgr_gem->sync_obj_rw_lock.lock();
1044 mos_sync_destroy_timeline_dep(bufmgr_gem->fd, context->timeline_dep);
1045 context->timeline_dep = nullptr;
1046 bufmgr_gem->global_ctx_info.erase(context->dummy_exec_queue_id);
1047 bufmgr_gem->sync_obj_rw_lock.unlock();
1048 bufmgr_gem->m_lock.unlock();
1049
1050 if (INVALID_EXEC_QUEUE_ID == ctx->ctx_id)
1051 {
1052 MOS_Delete(context);
1053 return;
1054 }
1055
1056 memclear(exec_queue_destroy);
1057 exec_queue_destroy.exec_queue_id = ctx->ctx_id;
1058
1059 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_XE_EXEC_QUEUE_DESTROY, &exec_queue_destroy);
1060 if (ret != 0)
1061 MOS_DRM_ASSERTMESSAGE("DRM_IOCTL_XE_EXEC_QUEUE_DESTROY failed: %s", strerror(errno));
1062
1063 MOS_Delete(context);
1064 }
1065
1066 /**
1067 * Restore banned exec_queue with newly created one
1068 * Note: this call is only for banned context restore, if using it
1069 * as other purpose, MUST pay attention to context->reset_count here.
1070 */
1071 static int
1072 __mos_context_restore_xe(struct mos_bufmgr *bufmgr,
1073 struct mos_linux_context *ctx)
1074 {
1075 MOS_DRM_CHK_NULL_RETURN_VALUE(bufmgr, -EINVAL);
1076 MOS_DRM_CHK_NULL_RETURN_VALUE(ctx, -EINVAL);
1077 if (INVALID_EXEC_QUEUE_ID == ctx->ctx_id)
1078 {
1079 MOS_DRM_ASSERTMESSAGE("Unable to restore intel context, it is not supported");
1080 return -EINVAL;
1081 }
1082 struct mos_xe_bufmgr_gem *bufmgr_gem = (struct mos_xe_bufmgr_gem *)bufmgr;
1083 struct mos_xe_context *context = (struct mos_xe_context *)ctx;
1084 int ret;
1085
1086 //query engine firstly
1087 uint32_t nengine = 0;
1088 ret = mos_query_engines_count_xe(bufmgr, &nengine);
1089 MOS_DRM_CHK_STATUS_MESSAGE_RETURN(ret,
1090 "query engine count of restore failed, return error(%d)", ret)
1091 struct drm_xe_engine_class_instance *engine_map = nullptr;
1092 engine_map = (struct drm_xe_engine_class_instance *)MOS_AllocAndZeroMemory(nengine * sizeof(struct drm_xe_engine_class_instance));
1093 ret = mos_query_engines_xe(bufmgr,
1094 context->engine_class,
1095 context->engine_caps,
1096 &nengine,
1097 (void*)engine_map);
1098 MOS_DRM_CHK_STATUS_MESSAGE_RETURN(ret,
1099 "query engine of restore failed, return error(%d)", ret)
1100
1101 //create new exec queue
1102 struct drm_xe_exec_queue_create create;
1103 memclear(create);
1104 create.width = context->ctx_width;
1105 create.num_placements = context->num_placements;
1106 create.vm_id = context->ctx.vm_id;
1107 create.flags = context->flags;
1108 create.instances = (uintptr_t)engine_map;
1109 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_XE_EXEC_QUEUE_CREATE, &create);
1110 MOS_DRM_CHK_STATUS_MESSAGE_RETURN(ret,
1111 "ioctl failed in DRM_IOCTL_XE_EXEC_QUEUE_CREATE of restore, return error(%d)", ret)
1112
1113 //destroy old exec_queue
1114 struct drm_xe_exec_queue_destroy exec_queue_destroy;
1115 memclear(exec_queue_destroy);
1116 exec_queue_destroy.exec_queue_id = ctx->ctx_id;
1117 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_XE_EXEC_QUEUE_DESTROY, &exec_queue_destroy);
1118 MOS_DRM_CHK_STATUS_MESSAGE_RETURN(ret,
1119 "ioctl failed in DRM_IOCTL_XE_EXEC_QUEUE_DESTROY of restore, return error(%d)", ret)
1120
1121 //restore
1122 context->ctx.ctx_id = create.exec_queue_id;
1123 context->reset_count += 1;
1124 MOS_SafeFreeMemory(engine_map);
1125 return MOS_XE_SUCCESS;
1126 }
1127
1128 /**
1129 * Get the property of the ctx
1130 *
1131 * @ctx indicates to the context that to query
1132 * @property indicates to what property that to query
1133 * @value indicates to quired value with given property
1134 */
1135 static int
1136 __mos_get_context_property_xe(struct mos_bufmgr *bufmgr,
1137 struct mos_linux_context *ctx,
1138 uint32_t property,
1139 uint64_t &value)
1140 {
1141 MOS_DRM_CHK_NULL_RETURN_VALUE(bufmgr, -EINVAL);
1142 MOS_DRM_CHK_NULL_RETURN_VALUE(ctx, -EINVAL);
1143 struct mos_xe_bufmgr_gem *bufmgr_gem = (struct mos_xe_bufmgr_gem *)bufmgr;
1144 struct drm_xe_exec_queue_get_property p;
1145 memclear(p);
1146 p.property = property;
1147 p.exec_queue_id = ctx->ctx_id;
1148
1149 int ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_XE_EXEC_QUEUE_GET_PROPERTY, &p);
1150
1151 value = p.value;
1152 return ret;
1153 }
1154
1155 /**
1156 * Allocate a section of virtual memory for a buffer, assigning an address.
1157 */
1158 static uint64_t
1159 __mos_bo_vma_alloc_xe(struct mos_bufmgr *bufmgr,
1160 enum mos_memory_zone memzone,
1161 uint64_t size,
1162 uint64_t alignment)
1163 {
1164 CHK_CONDITION(nullptr == bufmgr, "nullptr bufmgr.\n", 0);
1165 struct mos_xe_bufmgr_gem *bufmgr_gem = (struct mos_xe_bufmgr_gem *) bufmgr;
1166 /* Force alignment to be some number of pages */
1167 alignment = ALIGN(alignment, PAGE_SIZE);
1168
1169 uint64_t addr = mos_vma_heap_alloc(&bufmgr_gem->vma_heap[memzone], size, alignment);
1170
1171 // currently only support 48bit range address
1172 CHK_CONDITION((addr >> 48ull) != 0, "invalid address, over 48bit range.\n", 0);
1173 CHK_CONDITION((addr >> (MEMZONE_SYS == memzone ? 40ull : (MEMZONE_DEVICE == memzone ? 41ull:42ull))) != 0, "invalid address, over memory zone range.\n", 0);
1174 CHK_CONDITION((addr % alignment) != 0, "invalid address, not meet aligment requirement.\n", 0);
1175
1176 return addr;
1177 }
1178
1179 static int
1180 __mos_bo_set_offset_xe(MOS_LINUX_BO *bo)
1181 {
1182 struct mos_xe_bo_gem *bo_gem = (struct mos_xe_bo_gem *) bo;
1183 struct mos_xe_bufmgr_gem *bufmgr_gem = (struct mos_xe_bufmgr_gem *) bo->bufmgr;
1184 MOS_DRM_CHK_NULL_RETURN_VALUE(bo_gem, -EINVAL)
1185 MOS_DRM_CHK_NULL_RETURN_VALUE(bufmgr_gem, -EINVAL)
1186
1187 uint64_t offset = 0;
1188 uint64_t alignment = 0;
1189
1190 if (0 == bo->offset64)
1191 {
1192 bufmgr_gem->m_lock.lock();
1193
1194 /* On platforms where lmem only supports 64K pages, kmd requires us
1195 * to either align the va to 2M or seperate the lmem objects and smem
1196 * objects into different va zones to avoid mixing up lmem object and
1197 * smem object into same page table. For imported object, we don't know
1198 * if it's in lmem or smem. So, we need to align the va to 2M.
1199 */
1200 if (MEMZONE_PRIME == bo_gem->mem_region)
1201 {
1202 offset = __mos_bo_vma_alloc_xe(bo->bufmgr, (enum mos_memory_zone)bo_gem->mem_region, bo->size, PAGE_SIZE_2M);
1203 }
1204 else if (MEMZONE_DEVICE == bo_gem->mem_region)
1205 {
1206 alignment = MAX(bufmgr_gem->default_alignment[MOS_XE_MEM_CLASS_VRAM], PAGE_SIZE_64K);
1207 offset = __mos_bo_vma_alloc_xe(bo->bufmgr, (enum mos_memory_zone)bo_gem->mem_region, bo->size, alignment);
1208 }
1209 else if (MEMZONE_SYS == bo_gem->mem_region)
1210 {
1211 alignment = MAX(bufmgr_gem->default_alignment[MOS_XE_MEM_CLASS_SYSMEM], PAGE_SIZE_64K);
1212 offset = __mos_bo_vma_alloc_xe(bo->bufmgr, (enum mos_memory_zone)bo_gem->mem_region, bo->size, alignment);
1213 }
1214 else
1215 {
1216 MOS_DRM_ASSERTMESSAGE("Invalid mem_region:%d", bo_gem->mem_region);
1217 }
1218
1219 bo->offset64 = offset;
1220 bo->offset = offset;
1221
1222 bufmgr_gem->m_lock.unlock();
1223 }
1224
1225 return 0;
1226 }
1227
1228 static int __mos_vm_bind_xe(int fd, uint32_t vm_id, uint32_t exec_queue_id, uint32_t bo_handle,
1229 uint64_t offset, uint64_t addr, uint64_t size, uint16_t pat_index, uint32_t op, uint32_t flags,
1230 struct drm_xe_sync *sync, uint32_t num_syncs, uint64_t ext)
1231 {
1232 int ret;
1233
1234 struct drm_xe_vm_bind bind;
1235 memclear(bind);
1236 bind.extensions = ext;
1237 bind.vm_id = vm_id;
1238 bind.exec_queue_id = exec_queue_id;
1239 bind.num_binds = 1;
1240 bind.bind.obj = bo_handle;
1241 bind.bind.obj_offset = offset;
1242 bind.bind.range = size;
1243 bind.bind.pat_index = pat_index;
1244 bind.bind.addr = addr;
1245 bind.bind.op = op;
1246 bind.bind.flags = flags;
1247 bind.num_syncs = num_syncs;
1248 bind.syncs = (uintptr_t)sync;
1249
1250 ret = drmIoctl(fd, DRM_IOCTL_XE_VM_BIND, &bind);
1251 if (ret)
1252 {
1253 MOS_DRM_ASSERTMESSAGE("Failed to bind vm, vm_id:%d, exec_queue_id:%d, op:0x%x, flags:0x%x, bo_handle:%d, offset:%lx, addr:0x%lx, size:%ld, pat_index:%d, errno(%d)",
1254 vm_id, exec_queue_id, op, flags, bo_handle, offset, addr, size, pat_index, -errno);
1255 }
1256
1257 return ret;
1258 }
1259
1260 static int mos_vm_bind_sync_xe(int fd, uint32_t vm_id, uint32_t bo, uint64_t offset,
1261 uint64_t addr, uint64_t size, uint16_t pat_index, uint32_t op)
1262 {
1263 struct drm_xe_sync sync;
1264
1265 memclear(sync);
1266 sync.flags = DRM_XE_SYNC_FLAG_SIGNAL;
1267 sync.type = DRM_XE_SYNC_TYPE_SYNCOBJ;
1268 sync.handle = mos_sync_syncobj_create(fd, 0);
1269
1270 int ret = __mos_vm_bind_xe(fd, vm_id, 0, bo, offset, addr, size, pat_index,
1271 op, 0, &sync, 1, 0);
1272 if (ret)
1273 {
1274 MOS_DRM_ASSERTMESSAGE("ret:%d, error:%d", ret, -errno);
1275 mos_sync_syncobj_destroy(fd, sync.handle);
1276 return ret;
1277 }
1278
1279 ret = mos_sync_syncobj_wait_err(fd, &sync.handle, 1, INT64_MAX, 0, NULL);
1280 if (ret)
1281 {
1282 MOS_DRM_ASSERTMESSAGE("syncobj_wait error:%d", -errno);
1283 }
1284
1285 mos_sync_syncobj_destroy(fd, sync.handle);
1286
1287 return ret;
1288 }
1289
1290 static int mos_vm_bind_async_xe(int fd, uint32_t vm_id, uint32_t bo, uint64_t offset,
1291 uint64_t addr, uint64_t size, uint16_t pat_index, uint32_t op,
1292 struct drm_xe_sync *sync, uint32_t num_syncs)
1293 {
1294 return __mos_vm_bind_xe(fd, vm_id, 0, bo, offset, addr, size, pat_index,
1295 op, 0, sync, num_syncs, 0);
1296 }
1297
1298 drm_export struct mos_linux_bo *
1299 mos_bo_alloc_xe(struct mos_bufmgr *bufmgr,
1300 struct mos_drm_bo_alloc *alloc)
1301 {
1302 struct mos_xe_bufmgr_gem *bufmgr_gem = (struct mos_xe_bufmgr_gem *) bufmgr;
1303 struct mos_xe_bo_gem *bo_gem;
1304 struct drm_xe_gem_create create;
1305 uint32_t bo_align = alloc->alignment;
1306 struct mos_xe_bucket_lab *lab = &bufmgr_gem->bucket_lab;
1307 struct mos_xe_gem_bo_bucket *bucket = nullptr;
1308 int ret;
1309
1310 /**
1311 * Note: must use MOS_New to allocate buffer instead of malloc since mos_xe_bo_gem
1312 * contains std::vector and std::map. Otherwise both will have no instance.
1313 */
1314 bo_gem = MOS_New(mos_xe_bo_gem);
1315 MOS_DRM_CHK_NULL_RETURN_VALUE(bo_gem, nullptr)
1316 memclear(bo_gem->bo);
1317 bo_gem->is_exported = false;
1318 bo_gem->is_imported = false;
1319 bo_gem->is_userptr = false;
1320 bo_gem->last_exec_read_exec_queue = INVALID_EXEC_QUEUE_ID;
1321 bo_gem->last_exec_write_exec_queue = INVALID_EXEC_QUEUE_ID;
1322 atomic_set(&bo_gem->map_count, 0);
1323 bo_gem->mem_virtual = nullptr;
1324 bo_gem->mem_region = MEMZONE_SYS;
1325 bo_align = MAX(alloc->alignment, bufmgr_gem->default_alignment[MOS_XE_MEM_CLASS_SYSMEM]);
1326
1327 if (bufmgr_gem->has_vram &&
1328 (MOS_MEMPOOL_VIDEOMEMORY == alloc->ext.mem_type || MOS_MEMPOOL_DEVICEMEMORY == alloc->ext.mem_type))
1329 {
1330 bo_gem->mem_region = MEMZONE_DEVICE;
1331 bo_align = MAX(alloc->alignment, bufmgr_gem->default_alignment[MOS_XE_MEM_CLASS_VRAM]);
1332 alloc->ext.cpu_cacheable = false;
1333 }
1334
1335 memclear(create);
1336 if (MEMZONE_DEVICE == bo_gem->mem_region)
1337 {
1338 //Note: memory_region is related to gt_id for multi-tiles gpu, take gt_id into consideration in case of multi-tiles
1339 create.placement = bufmgr_gem->mem_regions_mask & (~0x1);
1340 }
1341 else
1342 {
1343 create.placement = bufmgr_gem->mem_regions_mask & 0x1;
1344 }
1345
1346 //Note: We suggest vm_id=0 here as default, otherwise this bo cannot be exported as prelim fd.
1347 create.vm_id = 0;
1348 create.size = ALIGN(alloc->size, bo_align);
1349
1350 /**
1351 * Find a bucket
1352 * 1, 4K cache bucket:
1353 * 1) 4K aligned size -> all buckets workable
1354 * 2) 64K aligned size -> only bucket whose bucket.size % 64k == 0 workable
1355 * 2, 64K cache bucket:
1356 * 1) 4K/64K aligned size -> all buckets workable
1357 * 3, 2M cache bucket:
1358 * 1) 4K/64K aligned size -> all bucket workable
1359 */
1360 bucket = __mos_gem_find_bucket_xe(lab->cache_buckets, lab->num_buckets, create.size);
1361 if (bucket)
1362 {
1363 create.size = bucket->size;
1364 }
1365
1366 /**
1367 * Note: current, it only supports WB/ WC while UC and other cache are not allowed.
1368 */
1369 create.cpu_caching = alloc->ext.cpu_cacheable ? DRM_XE_GEM_CPU_CACHING_WB : DRM_XE_GEM_CPU_CACHING_WC;
1370
1371 if (alloc->ext.scanout_surf)
1372 {
1373 create.flags |= DRM_XE_GEM_CREATE_FLAG_SCANOUT;
1374 }
1375
1376 ret = drmIoctl(bufmgr_gem->fd,
1377 DRM_IOCTL_XE_GEM_CREATE,
1378 &create);
1379 MOS_DRM_CHK_STATUS_MESSAGE_RETURN_VALUE_WH_OP(ret, bo_gem, MOS_Delete, nullptr,
1380 "ioctl failed in DRM_IOCTL_XE_GEM_CREATE, return error(%d)", ret);
1381
1382 bo_gem->gem_handle = create.handle;
1383 bo_gem->bo.handle = bo_gem->gem_handle;
1384 bo_gem->bo.size = create.size;
1385 bo_gem->bo.vm_id = INVALID_VM;
1386 bo_gem->bo.bufmgr = bufmgr;
1387 bo_gem->bo.align = bo_align;
1388 bo_gem->cpu_caching = create.cpu_caching;
1389 /**
1390 * Note: Better to get a default pat_index to overwite invalid argv. Normally it should not happen.
1391 */
1392 bo_gem->pat_index = alloc->ext.pat_index == PAT_INDEX_INVALID ? 0 : alloc->ext.pat_index;
1393
1394 if (bufmgr_gem->mem_profiler_fd != -1)
1395 {
1396 snprintf(bufmgr_gem->mem_profiler_buffer, MEM_PROFILER_BUFFER_SIZE, "GEM_CREATE, %d, %d, %lu, %d, %s\n",
1397 getpid(), bo_gem->bo.handle, bo_gem->bo.size,bo_gem->mem_region, alloc->name);
1398 ret = write(bufmgr_gem->mem_profiler_fd,
1399 bufmgr_gem->mem_profiler_buffer,
1400 strnlen(bufmgr_gem->mem_profiler_buffer, MEM_PROFILER_BUFFER_SIZE));
1401 if (-1 == ret)
1402 {
1403 MOS_DRM_ASSERTMESSAGE("Failed to write to %s: %s",
1404 bufmgr_gem->mem_profiler_path, strerror(errno));
1405 }
1406 }
1407
1408 /* drm_intel_gem_bo_free calls DRMLISTDEL() for an uninitialized
1409 list (vma_list), so better set the list head here */
1410 DRMINITLISTHEAD(&bo_gem->name_list);
1411
1412 memcpy(bo_gem->name, alloc->name, (strlen(alloc->name) + 1) > MAX_NAME_SIZE ? MAX_NAME_SIZE : (strlen(alloc->name) + 1));
1413 atomic_set(&bo_gem->ref_count, 1);
1414
1415 MOS_DRM_NORMALMESSAGE("buf %d (%s) %ldb, bo:0x%lx",
1416 bo_gem->gem_handle, alloc->name, alloc->size, (uint64_t)&bo_gem->bo);
1417
1418 __mos_bo_set_offset_xe(&bo_gem->bo);
1419
1420 ret = mos_vm_bind_sync_xe(bufmgr_gem->fd,
1421 bufmgr_gem->vm_id,
1422 bo_gem->gem_handle,
1423 0,
1424 bo_gem->bo.offset64,
1425 bo_gem->bo.size,
1426 bo_gem->pat_index,
1427 DRM_XE_VM_BIND_OP_MAP);
1428 if (ret)
1429 {
1430 MOS_DRM_ASSERTMESSAGE("mos_vm_bind_sync_xe ret: %d", ret);
1431 mos_bo_free_xe(&bo_gem->bo);
1432 return nullptr;
1433 }
1434 else
1435 {
1436 bo_gem->bo.vm_id = bufmgr_gem->vm_id;
1437 }
1438
1439 return &bo_gem->bo;
1440 }
1441
1442 static unsigned long
1443 __mos_bo_tile_size_xe(struct mos_xe_bufmgr_gem *bufmgr_gem, unsigned long size,
1444 uint32_t *tiling_mode, uint32_t alignment)
1445 {
1446 unsigned long min_size, max_size;
1447 unsigned long i;
1448
1449 if (TILING_NONE == *tiling_mode)
1450 return size;
1451
1452 /* 965+ just need multiples of page size for tiling */
1453 return ROUND_UP_TO(size, alignment);
1454
1455 }
1456
1457 /*
1458 * Round a given pitch up to the minimum required for X tiling on a
1459 * given chip. We use 512 as the minimum to allow for a later tiling
1460 * change.
1461 */
1462 static unsigned long
1463 __mos_bo_tile_pitch_xe(struct mos_xe_bufmgr_gem *bufmgr_gem,
1464 unsigned long pitch, uint32_t *tiling_mode)
1465 {
1466 unsigned long tile_width;
1467 unsigned long i;
1468
1469 /* If untiled, then just align it so that we can do rendering
1470 * to it with the 3D engine.
1471 */
1472 if (TILING_NONE == *tiling_mode)
1473 return ALIGN(pitch, 64);
1474
1475 if (TILING_X == *tiling_mode)
1476 tile_width = 512;
1477 else
1478 tile_width = 128;
1479
1480 /* 965 is flexible */
1481 return ROUND_UP_TO(pitch, tile_width);
1482 }
1483
1484 static struct mos_linux_bo *
1485 mos_bo_alloc_tiled_xe(struct mos_bufmgr *bufmgr,
1486 struct mos_drm_bo_alloc_tiled *alloc_tiled)
1487 {
1488 struct mos_xe_bufmgr_gem *bufmgr_gem = (struct mos_xe_bufmgr_gem *) bufmgr;
1489 unsigned long size, stride;
1490 uint32_t tiling;
1491
1492 uint32_t alignment = MAX(alloc_tiled->alignment, bufmgr_gem->default_alignment[MOS_XE_MEM_CLASS_SYSMEM]);
1493
1494 if (bufmgr_gem->has_vram &&
1495 (MOS_MEMPOOL_VIDEOMEMORY == alloc_tiled->ext.mem_type || MOS_MEMPOOL_DEVICEMEMORY == alloc_tiled->ext.mem_type))
1496 {
1497 alignment = MAX(alloc_tiled->alignment, bufmgr_gem->default_alignment[MOS_XE_MEM_CLASS_VRAM]);
1498 }
1499
1500 do {
1501 unsigned long aligned_y, height_alignment;
1502
1503 tiling = alloc_tiled->ext.tiling_mode;
1504
1505 /* If we're tiled, our allocations are in 8 or 32-row blocks,
1506 * so failure to align our height means that we won't allocate
1507 * enough pages.
1508 *
1509 * If we're untiled, we still have to align to 2 rows high
1510 * because the data port accesses 2x2 blocks even if the
1511 * bottom row isn't to be rendered, so failure to align means
1512 * we could walk off the end of the GTT and fault. This is
1513 * documented on 965, and may be the case on older chipsets
1514 * too so we try to be careful.
1515 */
1516 aligned_y = alloc_tiled->y;
1517 height_alignment = 2;
1518
1519 if (TILING_X == tiling)
1520 height_alignment = 8;
1521 else if (TILING_Y == tiling)
1522 height_alignment = 32;
1523 aligned_y = ALIGN(alloc_tiled->y, height_alignment);
1524
1525 stride = alloc_tiled->x * alloc_tiled->cpp;
1526 stride = __mos_bo_tile_pitch_xe(bufmgr_gem, stride, &alloc_tiled->ext.tiling_mode);
1527 size = stride * aligned_y;
1528 size = __mos_bo_tile_size_xe(bufmgr_gem, size, &alloc_tiled->ext.tiling_mode, alignment);
1529 } while (alloc_tiled->ext.tiling_mode != tiling);
1530
1531 alloc_tiled->pitch = stride;
1532
1533 struct mos_drm_bo_alloc alloc;
1534 alloc.name = alloc_tiled->name;
1535 alloc.size = size;
1536 alloc.alignment = alignment;
1537 alloc.ext = alloc_tiled->ext;
1538
1539 return mos_bo_alloc_xe(bufmgr, &alloc);
1540 }
1541
1542 drm_export struct mos_linux_bo *
1543 mos_bo_alloc_userptr_xe(struct mos_bufmgr *bufmgr,
1544 struct mos_drm_bo_alloc_userptr *alloc_uptr)
1545 {
1546 struct mos_xe_bufmgr_gem *bufmgr_gem = (struct mos_xe_bufmgr_gem *) bufmgr;
1547 struct mos_xe_bo_gem *bo_gem;
1548 int ret;
1549
1550 /**
1551 * Note: must use MOS_New to allocate buffer instead of malloc since mos_xe_bo_gem
1552 * contains std::vector and std::map. Otherwise both will have no instance.
1553 */
1554 bo_gem = MOS_New(mos_xe_bo_gem);
1555 MOS_DRM_CHK_NULL_RETURN_VALUE(bo_gem, nullptr)
1556 memclear(bo_gem->bo);
1557 bo_gem->is_exported = false;
1558 bo_gem->is_imported = false;
1559 bo_gem->is_userptr = true;
1560 bo_gem->last_exec_read_exec_queue = INVALID_EXEC_QUEUE_ID;
1561 bo_gem->last_exec_write_exec_queue = INVALID_EXEC_QUEUE_ID;
1562 atomic_set(&bo_gem->map_count, 0);
1563 bo_gem->mem_virtual = alloc_uptr->addr;
1564 bo_gem->gem_handle = INVALID_HANDLE;
1565 bo_gem->bo.handle = INVALID_HANDLE;
1566 bo_gem->bo.size = alloc_uptr->size;
1567 bo_gem->pat_index = alloc_uptr->pat_index == PAT_INDEX_INVALID ? 0 : alloc_uptr->pat_index;
1568 bo_gem->bo.bufmgr = bufmgr;
1569 bo_gem->bo.vm_id = INVALID_VM;
1570 bo_gem->mem_region = MEMZONE_SYS;
1571
1572 /* Save the address provided by user */
1573 #ifdef __cplusplus
1574 bo_gem->bo.virt = alloc_uptr->addr;
1575 #else
1576 bo_gem->bo.virtual = alloc_uptr->addr;
1577 #endif
1578
1579 /* drm_intel_gem_bo_free calls DRMLISTDEL() for an uninitialized
1580 list (vma_list), so better set the list head here */
1581 DRMINITLISTHEAD(&bo_gem->name_list);
1582
1583 memcpy(bo_gem->name, alloc_uptr->name, (strlen(alloc_uptr->name) + 1) > MAX_NAME_SIZE ? MAX_NAME_SIZE : (strlen(alloc_uptr->name) + 1));
1584 atomic_set(&bo_gem->ref_count, 1);
1585
1586 __mos_bo_set_offset_xe(&bo_gem->bo);
1587
1588 ret = mos_vm_bind_sync_xe(bufmgr_gem->fd,
1589 bufmgr_gem->vm_id,
1590 0,
1591 (uint64_t)alloc_uptr->addr,
1592 bo_gem->bo.offset64,
1593 bo_gem->bo.size,
1594 bo_gem->pat_index,
1595 DRM_XE_VM_BIND_OP_MAP_USERPTR);
1596
1597 if (ret)
1598 {
1599 MOS_DRM_ASSERTMESSAGE("mos_xe_vm_bind_userptr_sync ret: %d", ret);
1600 mos_bo_free_xe(&bo_gem->bo);
1601 return nullptr;
1602 }
1603 else
1604 {
1605 bo_gem->bo.vm_id = bufmgr_gem->vm_id;
1606 }
1607
1608 MOS_DRM_NORMALMESSAGE("mos_bo_alloc_userptr_xe: buf (%s) %ldb, bo:0x%lx",
1609 alloc_uptr->name, alloc_uptr->size, (uint64_t)&bo_gem->bo);
1610
1611
1612 return &bo_gem->bo;
1613 }
1614
1615 static struct mos_linux_bo *
1616 mos_bo_create_from_prime_xe(struct mos_bufmgr *bufmgr, struct mos_drm_bo_alloc_prime *alloc_prime)
1617 {
1618 struct mos_xe_bufmgr_gem *bufmgr_gem = (struct mos_xe_bufmgr_gem *) bufmgr;
1619 int ret;
1620 uint32_t handle;
1621 struct mos_xe_bo_gem *bo_gem;
1622 int prime_fd = alloc_prime->prime_fd;
1623 int size = alloc_prime->size;
1624 uint16_t pat_index = alloc_prime->pat_index;
1625 drmMMListHead *list;
1626
1627 bufmgr_gem->m_lock.lock();
1628 ret = drmPrimeFDToHandle(bufmgr_gem->fd, prime_fd, &handle);
1629 if (ret)
1630 {
1631 MOS_DRM_ASSERTMESSAGE("create_from_prime: failed to obtain handle from fd: %s", strerror(errno));
1632 bufmgr_gem->m_lock.unlock();
1633 return nullptr;
1634 }
1635
1636 /*
1637 * See if the kernel has already returned this buffer to us. Just as
1638 * for named buffers, we must not create two bo's pointing at the same
1639 * kernel object
1640 */
1641 for (list = bufmgr_gem->named.next; list != &bufmgr_gem->named; list = list->next)
1642 {
1643 bo_gem = DRMLISTENTRY(struct mos_xe_bo_gem, list, name_list);
1644 if (bo_gem->gem_handle == handle)
1645 {
1646 mos_bo_reference_xe(&bo_gem->bo);
1647 bufmgr_gem->m_lock.unlock();
1648 return &bo_gem->bo;
1649 }
1650 }
1651
1652 bo_gem = MOS_New(mos_xe_bo_gem);
1653 if (!bo_gem)
1654 {
1655 bufmgr_gem->m_lock.unlock();
1656 return nullptr;
1657 }
1658
1659 memclear(bo_gem->bo);
1660 bo_gem->is_exported = false;
1661 bo_gem->is_imported = true;
1662 bo_gem->is_userptr = false;
1663 bo_gem->last_exec_read_exec_queue = INVALID_EXEC_QUEUE_ID;
1664 bo_gem->last_exec_write_exec_queue = INVALID_EXEC_QUEUE_ID;
1665 atomic_set(&bo_gem->map_count, 0);
1666 bo_gem->mem_virtual = nullptr;
1667
1668 /* Determine size of bo. The fd-to-handle ioctl really should
1669 * return the size, but it doesn't. If we have kernel 3.12 or
1670 * later, we can lseek on the prime fd to get the size. Older
1671 * kernels will just fail, in which case we fall back to the
1672 * provided (estimated or guess size). */
1673 ret = lseek(prime_fd, 0, SEEK_END);
1674 if (ret != -1)
1675 bo_gem->bo.size = ret;
1676 else
1677 bo_gem->bo.size = size;
1678
1679 bo_gem->bo.handle = handle;
1680 /*
1681 * Note: Need to get the pat_index by the customer_gmminfo with 1way coherency at least.
1682 */
1683 bo_gem->pat_index = pat_index == PAT_INDEX_INVALID ? 0 : pat_index;
1684 bo_gem->bo.bufmgr = bufmgr;
1685
1686 bo_gem->gem_handle = handle;
1687 atomic_set(&bo_gem->ref_count, 1);
1688
1689 /**
1690 * change bo_gem->name to const char*
1691 */
1692 memcpy(bo_gem->name, alloc_prime->name, sizeof("prime"));
1693 bo_gem->mem_region = MEMZONE_PRIME;
1694
1695 DRMLISTADDTAIL(&bo_gem->name_list, &bufmgr_gem->named);
1696 bufmgr_gem->m_lock.unlock();
1697
1698 __mos_bo_set_offset_xe(&bo_gem->bo);
1699
1700 ret = mos_vm_bind_sync_xe(bufmgr_gem->fd,
1701 bufmgr_gem->vm_id,
1702 bo_gem->gem_handle,
1703 0,
1704 bo_gem->bo.offset64,
1705 bo_gem->bo.size,
1706 bo_gem->pat_index,
1707 DRM_XE_VM_BIND_OP_MAP);
1708 if (ret)
1709 {
1710 MOS_DRM_ASSERTMESSAGE("mos_vm_bind_sync_xe ret: %d", ret);
1711 mos_bo_free_xe(&bo_gem->bo);
1712 return nullptr;
1713 }
1714 else
1715 {
1716 bo_gem->bo.vm_id = bufmgr_gem->vm_id;
1717 }
1718
1719 return &bo_gem->bo;
1720 }
1721
1722 static int
1723 mos_bo_export_to_prime_xe(struct mos_linux_bo *bo, int *prime_fd)
1724 {
1725 struct mos_xe_bufmgr_gem *bufmgr_gem = (struct mos_xe_bufmgr_gem *) bo->bufmgr;
1726 struct mos_xe_bo_gem *bo_gem = (struct mos_xe_bo_gem *) bo;
1727
1728 bufmgr_gem->m_lock.lock();
1729 if (DRMLISTEMPTY(&bo_gem->name_list))
1730 DRMLISTADDTAIL(&bo_gem->name_list, &bufmgr_gem->named);
1731 bufmgr_gem->m_lock.unlock();
1732
1733 mos_gem_bo_wait_rendering_xe(bo);
1734
1735 if (drmPrimeHandleToFD(bufmgr_gem->fd, bo_gem->gem_handle,
1736 DRM_CLOEXEC, prime_fd) != 0)
1737 return -errno;
1738
1739 bo_gem->is_exported = true;
1740
1741 return 0;
1742 }
1743
1744 /**
1745 * Update exec list for submission.
1746 *
1747 * @cmd_bo indicates to cmd bo for the exec submission.
1748 * @exec_bo indicates to the gpu resource for exec submission.
1749 * @write_flag indicates to whether exec bo's operation write on GPU.
1750 */
1751 static int
1752 mos_gem_bo_update_exec_list_xe(struct mos_linux_bo *cmd_bo, struct mos_linux_bo *exec_bo, bool write_flag)
1753 {
1754 MOS_DRM_CHK_NULL_RETURN_VALUE(cmd_bo, -EINVAL)
1755 MOS_DRM_CHK_NULL_RETURN_VALUE(exec_bo, -EINVAL)
1756 struct mos_xe_bo_gem *cmd_bo_gem = (struct mos_xe_bo_gem *) cmd_bo;
1757 struct mos_xe_bufmgr_gem *bufmgr_gem = (struct mos_xe_bufmgr_gem *) cmd_bo->bufmgr;
1758 MOS_DRM_CHK_NULL_RETURN_VALUE(bufmgr_gem, -EINVAL)
1759 std::map<uintptr_t, struct mos_xe_exec_bo> &exec_list = cmd_bo_gem->exec_list;
1760
1761 if (exec_bo->handle == cmd_bo->handle)
1762 {
1763 MOS_DRM_NORMALMESSAGE("cmd bo should not add into exec list, skip it");
1764 return MOS_XE_SUCCESS;
1765 }
1766 uintptr_t key = (uintptr_t)exec_bo;
1767 if (exec_list.count(key) > 0)
1768 {
1769 /**
1770 * This exec bo has added before, but need to update its exec flags.
1771 */
1772
1773 // For all BOs with read and write usages, we could just assign write flag to reduce read deps size.
1774 if (write_flag || (exec_list[key].flags & EXEC_OBJECT_WRITE_XE))
1775 {
1776 exec_list[key].flags = EXEC_OBJECT_WRITE_XE;
1777 }
1778 else
1779 {
1780 // For BOs only with read usage, we should assign read flag.
1781 exec_list[key].flags |= EXEC_OBJECT_READ_XE;
1782 }
1783 }
1784 else
1785 {
1786 struct mos_xe_exec_bo target;
1787 target.bo = exec_bo;
1788 target.flags = write_flag ? EXEC_OBJECT_WRITE_XE : EXEC_OBJECT_READ_XE;
1789 exec_list[key] = target;
1790 mos_bo_reference_xe(exec_bo);
1791 }
1792 return MOS_XE_SUCCESS;
1793 }
1794
1795 /**
1796 * Clear the exec bo from the list after submission.
1797 *
1798 * @cmd_bo indicates to cmd bo for the exec submission.
1799 * @start is unused.
1800 */
1801 static void
1802 mos_gem_bo_clear_exec_list_xe(struct mos_linux_bo *cmd_bo, int start)
1803 {
1804 MOS_UNUSED(start);
1805 if (cmd_bo != nullptr && cmd_bo->bufmgr != nullptr)
1806 {
1807 struct mos_xe_bufmgr_gem *bufmgr_gem = (struct mos_xe_bufmgr_gem *) cmd_bo->bufmgr;
1808 struct mos_xe_bo_gem *bo_gem = (struct mos_xe_bo_gem *) cmd_bo;
1809 std::map<uintptr_t, struct mos_xe_exec_bo> &exec_list = bo_gem->exec_list;
1810
1811 for (auto &it : exec_list) {
1812 mos_bo_unreference_xe(it.second.bo);
1813 }
1814 exec_list.clear();
1815 }
1816 }
1817
1818 /**
1819 * This is to dump all pending execution timeline done on such bo
1820 */
1821 int
1822 __mos_dump_bo_wait_rendering_timeline_xe(uint32_t bo_handle,
1823 uint32_t *handles,
1824 uint64_t *points,
1825 uint32_t count,
1826 int64_t timeout_nsec,
1827 uint32_t wait_flags,
1828 uint32_t rw_flags)
1829 {
1830 #if (_DEBUG || _RELEASE_INTERNAL)
1831 if (__XE_TEST_DEBUG(XE_DEBUG_SYNCHRONIZATION))
1832 {
1833 MOS_DRM_CHK_NULL_RETURN_VALUE(handles, -EINVAL)
1834 char log_msg[MOS_MAX_MSG_BUF_SIZE] = { 0 };
1835 int offset = 0;
1836 offset += MOS_SecureStringPrint(log_msg + offset, MOS_MAX_MSG_BUF_SIZE,
1837 MOS_MAX_MSG_BUF_SIZE - offset,
1838 "\n\t\t\tdump bo wait rendering: bo handle = %d, timeout_nsec = %ld, wait_flags = %d, rw_flags = %d",
1839 bo_handle,
1840 timeout_nsec,
1841 wait_flags,
1842 rw_flags);
1843
1844 for (int i = 0; i < count; i++)
1845 {
1846 offset += MOS_SecureStringPrint(log_msg + offset, MOS_MAX_MSG_BUF_SIZE,
1847 MOS_MAX_MSG_BUF_SIZE - offset,
1848 "\n\t\t\t-syncobj handle = %d, timeline = %ld",
1849 handles[i],
1850 points[i]);
1851 }
1852
1853 offset > MOS_MAX_MSG_BUF_SIZE ?
1854 MOS_DRM_NORMALMESSAGE("imcomplete dump since log msg buffer overwrite %s", log_msg) : MOS_DRM_NORMALMESSAGE("%s", log_msg);
1855 }
1856 #endif
1857 return MOS_XE_SUCCESS;
1858 }
1859
1860 /**
1861 * @bo indicates to bo object that need to wait
1862 * @timeout_nsec indicates to timeout in nanosecond:
1863 * if timeout_nsec > 0, waiting for given time, if timeout, return -ETIME;
1864 * if timeout_nsec ==0, check bo busy state, if busy, return -ETIME imediately;
1865 * @wait_flags indicates wait operation, it supports wait all, wait submit, wait available or wait any;
1866 * refer drm syncobj to get more details in drm.h
1867 * @rw_flags indicates to read/write operation:
1868 * if rw_flags & EXEC_OBJECT_WRITE_XE, means bo write. Otherwise it means bo read.
1869 * @first_signaled indicates to first signaled syncobj handle in the handls array.
1870 */
1871 static int
1872 __mos_gem_bo_wait_timeline_rendering_with_flags_xe(struct mos_linux_bo *bo,
1873 int64_t timeout_nsec,
1874 uint32_t wait_flags,
1875 uint32_t rw_flags,
1876 uint32_t *first_signaled)
1877 {
1878 MOS_DRM_CHK_NULL_RETURN_VALUE(bo, -EINVAL)
1879
1880 mos_xe_bufmgr_gem *bufmgr_gem = (mos_xe_bufmgr_gem *)bo->bufmgr;
1881 MOS_DRM_CHK_NULL_RETURN_VALUE(bufmgr_gem, -EINVAL)
1882
1883 int ret = MOS_XE_SUCCESS;
1884 uint32_t count = 0;
1885 mos_xe_bo_gem *bo_gem = (mos_xe_bo_gem *)bo;
1886 std::map<uint32_t, uint64_t> timeline_data; //pair(syncobj, point)
1887 std::vector<uint32_t> handles;
1888 std::vector<uint64_t> points;
1889 std::set<uint32_t> exec_queue_ids;
1890 bufmgr_gem->m_lock.lock();
1891 bufmgr_gem->sync_obj_rw_lock.lock_shared();
1892 MOS_XE_GET_KEYS_FROM_MAP(bufmgr_gem->global_ctx_info, exec_queue_ids);
1893
1894 mos_sync_get_bo_wait_timeline_deps(exec_queue_ids,
1895 bo_gem->read_deps,
1896 bo_gem->write_deps,
1897 timeline_data,
1898 bo_gem->last_exec_write_exec_queue,
1899 rw_flags);
1900 bufmgr_gem->m_lock.unlock();
1901
1902 for (auto it : timeline_data)
1903 {
1904 handles.push_back(it.first);
1905 points.push_back(it.second);
1906 }
1907
1908 count = handles.size();
1909 if (count > 0)
1910 {
1911 ret = mos_sync_syncobj_timeline_wait(bufmgr_gem->fd,
1912 handles.data(),
1913 points.data(),
1914 count,
1915 timeout_nsec,
1916 wait_flags,
1917 first_signaled);
1918
1919 __mos_dump_bo_wait_rendering_timeline_xe(bo_gem->gem_handle,
1920 handles.data(),
1921 points.data(),
1922 count,
1923 timeout_nsec,
1924 wait_flags,
1925 rw_flags);
1926 }
1927 bufmgr_gem->sync_obj_rw_lock.unlock_shared();
1928
1929 return ret;
1930 }
1931
1932 /**
1933 * Check if bo is still busy state.
1934 *
1935 * Check if read dep on all exec_queue and write dep on last write exec_queue are signaled.
1936 * If any one dep is not signaled, that means this bo is busy and return -ETIME immediately.
1937 * Otheriwise, move all dep on this bo from busy queue to free queue for reuse.
1938 */
1939 static int
1940 mos_gem_bo_busy_xe(struct mos_linux_bo *bo)
1941 {
1942 MOS_DRM_CHK_NULL_RETURN_VALUE(bo, -EINVAL);
1943 mos_xe_bufmgr_gem *bufmgr_gem = (mos_xe_bufmgr_gem *)bo->bufmgr;
1944 MOS_DRM_CHK_NULL_RETURN_VALUE(bufmgr_gem, -EINVAL)
1945
1946 int64_t timeout_nsec = 0;
1947 uint32_t wait_flags = DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL;
1948 uint32_t rw_flags = EXEC_OBJECT_READ_XE | EXEC_OBJECT_WRITE_XE;
1949
1950 int ret = __mos_gem_bo_wait_timeline_rendering_with_flags_xe(bo, timeout_nsec, wait_flags, rw_flags, nullptr);
1951
1952 if (ret)
1953 {
1954 //busy
1955 if (errno != ETIME)
1956 {
1957 MOS_DRM_ASSERTMESSAGE("bo_busy_xe ret:%d, error:%d", ret, -errno);
1958 }
1959 return true;
1960 }
1961 else if (MOS_XE_SUCCESS == ret)
1962 {
1963 //free
1964 return false;
1965 }
1966
1967 return false;
1968 }
1969
1970 /**
1971 * Waits for all GPU rendering with the object to have completed.
1972 *
1973 * Wait read dep on all exec_queue and write dep on last write exec_queue are signaled.
1974 * And move all dep on this bo from busy queue to free queue for reuse after rendering completed.
1975 */
1976 static void
1977 mos_gem_bo_wait_rendering_xe(struct mos_linux_bo *bo)
1978 {
1979 if (bo == nullptr || bo->bufmgr == nullptr)
1980 {
1981 MOS_DRM_ASSERTMESSAGE("ptr is null pointer");
1982 return;
1983 }
1984 mos_xe_bufmgr_gem *bufmgr_gem = (mos_xe_bufmgr_gem *)bo->bufmgr;
1985
1986 int64_t timeout_nsec = INT64_MAX;
1987 uint32_t wait_flags = DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL;
1988 uint32_t rw_flags = EXEC_OBJECT_READ_XE | EXEC_OBJECT_WRITE_XE;
1989
1990 int ret = __mos_gem_bo_wait_timeline_rendering_with_flags_xe(bo, timeout_nsec, wait_flags, rw_flags, nullptr);
1991 if (ret)
1992 {
1993 MOS_DRM_ASSERTMESSAGE("bo_wait_rendering_xe ret:%d, error:%d", ret, -errno);
1994 }
1995 }
1996
1997 /**
1998 * @timeout_ns indicates to timeout for waiting, but it is fake timeout;
1999 * it only indicates to wait bo rendering completed or check bo busy state.
2000 * if timeout_ns != 0, wait bo rendering completed.
2001 * if timeout_ns == 0. check bo busy state.
2002 */
2003 static int
2004 mos_gem_bo_wait_xe(struct mos_linux_bo *bo, int64_t timeout_ns)
2005 {
2006 if (timeout_ns)
2007 {
2008 mos_gem_bo_wait_rendering_xe(bo);
2009 return 0;
2010 }
2011 else
2012 {
2013 return mos_gem_bo_busy_xe(bo) ? -ETIME : 0;
2014 }
2015 return 0;
2016 }
2017
2018 /**
2019 * Map gpu resource for CPU read or write.
2020 *
2021 * 1. if map for write, it should wait read dep on all exec_queue and write dep on last write exec_queue signaled.
2022 * 2. if map for read, it should only wait write dep on last write exec_queue signaled.
2023 *
2024 * After bo rendering completed on GPU, then CPU could continue its read or write operation.
2025 */
2026 static int
2027 mos_bo_map_xe(struct mos_linux_bo *bo, int write_enable)
2028 {
2029 MOS_DRM_CHK_NULL_RETURN_VALUE(bo, -EINVAL)
2030 struct mos_xe_bufmgr_gem *bufmgr_gem = (struct mos_xe_bufmgr_gem *) bo->bufmgr;
2031 MOS_DRM_CHK_NULL_RETURN_VALUE(bufmgr_gem, -EINVAL)
2032 struct mos_xe_bo_gem *bo_gem = (struct mos_xe_bo_gem *) bo;
2033 int ret;
2034
2035 int64_t timeout_nsec = INT64_MAX;
2036 uint32_t wait_flags = DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL;
2037 uint32_t rw_flags = write_enable ? EXEC_OBJECT_WRITE_XE : EXEC_OBJECT_READ_XE;
2038
2039 ret = __mos_gem_bo_wait_timeline_rendering_with_flags_xe(bo, timeout_nsec, wait_flags, rw_flags, nullptr);
2040 if (ret)
2041 {
2042 MOS_DRM_ASSERTMESSAGE("bo wait rendering error(%d ns)", -errno);
2043 }
2044
2045 if (bo_gem->is_userptr)
2046 {
2047 /* Return the same user ptr */
2048 return 0;
2049 }
2050
2051 bufmgr_gem->m_lock.lock();
2052 if (nullptr == bo_gem->mem_virtual)
2053 {
2054 struct drm_xe_gem_mmap_offset mmo;
2055 memclear(mmo);
2056 mmo.handle = bo->handle;
2057 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_XE_GEM_MMAP_OFFSET, &mmo);
2058 if (ret)
2059 {
2060 bufmgr_gem->m_lock.unlock();
2061 return ret;
2062 }
2063
2064 bo_gem->mem_virtual = drm_mmap(0, bo->size, PROT_READ | PROT_WRITE,
2065 MAP_SHARED, bufmgr_gem->fd, mmo.offset);
2066 if (MAP_FAILED == bo_gem->mem_virtual)
2067 {
2068 bo_gem->mem_virtual = nullptr;
2069 ret = -errno;
2070 MOS_DRM_ASSERTMESSAGE("Error mapping buffer %d (%s): %s .",
2071 bo_gem->gem_handle, bo_gem->name,
2072 strerror(errno));
2073 }
2074 }
2075
2076 #ifdef __cplusplus
2077 bo->virt = bo_gem->mem_virtual;
2078 #else
2079 bo->virtual = bo_gem->mem_virtual;
2080 #endif
2081
2082 atomic_inc(&bo_gem->map_count);
2083
2084 __mos_bo_mark_mmaps_incoherent_xe(bo);
2085 VG(VALGRIND_MAKE_MEM_DEFINED(bo_gem->mem_virtual, bo->size));
2086 bufmgr_gem->m_lock.unlock();
2087
2088 return 0;
2089 }
2090
2091 static int
2092 mos_bo_map_wc_xe(struct mos_linux_bo *bo)
2093 {
2094 return mos_bo_map_xe(bo, false);
2095 }
2096
2097 static int mos_bo_unmap_xe(struct mos_linux_bo *bo)
2098 {
2099 struct mos_xe_bo_gem *bo_gem = (struct mos_xe_bo_gem *) bo;
2100 MOS_DRM_CHK_NULL_RETURN_VALUE(bo_gem, 0)
2101 struct mos_xe_bufmgr_gem *bufmgr_gem = (struct mos_xe_bufmgr_gem *) bo->bufmgr;
2102 MOS_DRM_CHK_NULL_RETURN_VALUE(bufmgr_gem, 0)
2103
2104 if (bo_gem->is_userptr)
2105 return 0;
2106
2107 bufmgr_gem->m_lock.lock();
2108
2109 if (atomic_dec_and_test(&bo_gem->map_count))
2110 {
2111 __mos_bo_mark_mmaps_incoherent_xe(bo);
2112 #ifdef __cplusplus
2113 bo->virt = nullptr;
2114 #else
2115 bo->virtual = nullptr;
2116 #endif
2117 }
2118 bufmgr_gem->m_lock.unlock();
2119
2120 return 0;
2121 }
2122
2123 static int
2124 mos_bo_unmap_wc_xe(struct mos_linux_bo *bo)
2125 {
2126 return mos_bo_unmap_xe(bo);
2127 }
2128
2129 /**
2130 *This aims to dump the sync info on such execution.
2131 *@syncs contains fence in from bo who has dependency on
2132 *currect execution and a fence out in @dep from current execution.
2133 */
2134 int __mos_dump_syncs_array_xe(struct drm_xe_sync *syncs,
2135 uint32_t count,
2136 mos_xe_dep *dep)
2137 {
2138 #if (_DEBUG || _RELEASE_INTERNAL)
2139 if (__XE_TEST_DEBUG(XE_DEBUG_SYNCHRONIZATION))
2140 {
2141 MOS_DRM_CHK_NULL_RETURN_VALUE(syncs, -EINVAL)
2142 MOS_DRM_CHK_NULL_RETURN_VALUE(dep, -EINVAL)
2143 char log_msg[MOS_MAX_MSG_BUF_SIZE] = { 0 };
2144 int offset = 0;
2145 offset += MOS_SecureStringPrint(log_msg + offset, MOS_MAX_MSG_BUF_SIZE,
2146 MOS_MAX_MSG_BUF_SIZE - offset,
2147 "\n\t\t\tdump fence out syncobj: handle = %d, timeline = %ld",
2148 dep->timeline_index);
2149 if (count > 0)
2150 {
2151 offset += MOS_SecureStringPrint(log_msg + offset, MOS_MAX_MSG_BUF_SIZE,
2152 MOS_MAX_MSG_BUF_SIZE - offset,
2153 "\n\t\t\tdump exec syncs array, num sync = %d",
2154 count);
2155 }
2156 for (int i = 0; i < count; i++)
2157 {
2158 /**
2159 * Note: we assume all are timeline sync here, and change later when any other
2160 * types sync in use.
2161 */
2162 offset += MOS_SecureStringPrint(log_msg + offset, MOS_MAX_MSG_BUF_SIZE,
2163 MOS_MAX_MSG_BUF_SIZE - offset,
2164 "\n\t\t\t-syncobj_handle = %d, timeline = %ld, sync type = %d, sync flags = %d",
2165 syncs[i].handle, syncs[i].timeline_value, syncs[i].type, syncs[i].flags);
2166 }
2167 offset > MOS_MAX_MSG_BUF_SIZE ?
2168 MOS_DRM_NORMALMESSAGE("imcomplete dump since log msg buffer overwrite %s", log_msg) : MOS_DRM_NORMALMESSAGE("%s", log_msg);
2169 }
2170 #endif
2171 return MOS_XE_SUCCESS;
2172 }
2173
2174 /**
2175 * This is to dump timeline for each exec bo on such execution,
2176 * pair of execed_queue_id & timeline_value will be dumped.
2177 */
2178 int
2179 __mos_dump_bo_deps_map_xe(struct mos_linux_bo **bo,
2180 int num_bo,
2181 std::vector<mos_xe_exec_bo> &exec_list,
2182 uint32_t curr_exec_queue_id,
2183 std::map<uint32_t, struct mos_xe_context*> ctx_infos)
2184 {
2185 #if (_DEBUG || _RELEASE_INTERNAL)
2186 if (__XE_TEST_DEBUG(XE_DEBUG_SYNCHRONIZATION))
2187 {
2188 MOS_DRM_CHK_NULL_RETURN_VALUE(bo, -EINVAL)
2189 uint32_t exec_list_size = exec_list.size();
2190 for (int i = 0; i < exec_list_size + num_bo; i++)
2191 {
2192 mos_xe_bo_gem *exec_bo_gem = nullptr;
2193 uint32_t exec_flags = 0;
2194 if (i < exec_list_size)
2195 {
2196 exec_bo_gem = (mos_xe_bo_gem *)exec_list[i].bo;
2197 exec_flags = exec_list[i].flags;
2198 }
2199 else
2200 {
2201 exec_bo_gem = (mos_xe_bo_gem *)bo[i - exec_list_size];
2202 exec_flags = EXEC_OBJECT_WRITE_XE; //use write flags for batch bo as default.
2203 }
2204 if (exec_bo_gem)
2205 {
2206 if (exec_bo_gem->is_imported || exec_bo_gem->is_exported)
2207 {
2208 MOS_DRM_NORMALMESSAGE("\n\t\t\tdump external bo, handle=%d, without deps map, skip dump", exec_bo_gem->bo.handle);
2209 }
2210 else
2211 {
2212 char log_msg[MOS_MAX_MSG_BUF_SIZE] = { 0 };
2213 int offset = 0;
2214 offset += MOS_SecureStringPrint(log_msg + offset, MOS_MAX_MSG_BUF_SIZE,
2215 MOS_MAX_MSG_BUF_SIZE - offset,
2216 "\n\t\t\tdump %s dep: bo handle=%d, curr_exec_queue_id=%d, curr_op_flags=%d",
2217 i >= exec_list_size ? "batch bo" : "exec bo",
2218 exec_bo_gem->bo.handle,
2219 curr_exec_queue_id,
2220 exec_flags);
2221
2222 auto it = exec_bo_gem->read_deps.begin();
2223 while (it != exec_bo_gem->read_deps.end())
2224 {
2225 if (ctx_infos.count(it->first) > 0)
2226 {
2227 offset += MOS_SecureStringPrint(log_msg + offset, MOS_MAX_MSG_BUF_SIZE,
2228 MOS_MAX_MSG_BUF_SIZE - offset,
2229 "\n\t\t\t-read deps: execed_exec_queue_id=%d, syncobj_handle=%d", "timeline = %ld",
2230 it->first,
2231 it->second.dep ? it->second.dep->syncobj_handle : INVALID_HANDLE,
2232 it->second.dep ? it->second.exec_timeline_index : INVALID_HANDLE);
2233 }
2234 it++;
2235 }
2236
2237 it = exec_bo_gem->write_deps.begin();
2238 while (it != exec_bo_gem->write_deps.end())
2239 {
2240 if (ctx_infos.count(it->first) > 0)
2241 {
2242 offset += MOS_SecureStringPrint(log_msg + offset, MOS_MAX_MSG_BUF_SIZE,
2243 MOS_MAX_MSG_BUF_SIZE - offset,
2244 "\n\t\t\t-write deps: execed_exec_queue_id=%d, syncobj_handle=%d", "timeline = %ld",
2245 it->first,
2246 it->second.dep ? it->second.dep->syncobj_handle : INVALID_HANDLE,
2247 it->second.dep ? it->second.exec_timeline_index : INVALID_HANDLE);
2248 }
2249 it++;
2250 }
2251 offset > MOS_MAX_MSG_BUF_SIZE ?
2252 MOS_DRM_NORMALMESSAGE("imcomplete dump since log msg buffer overwrite %s", log_msg) : MOS_DRM_NORMALMESSAGE("%s", log_msg);
2253 }
2254 }
2255 }
2256 }
2257 #endif
2258 return MOS_XE_SUCCESS;
2259 }
2260
2261 static int
2262 __mos_context_exec_update_syncs_xe(struct mos_xe_bufmgr_gem *bufmgr_gem,
2263 struct mos_linux_bo **bo,
2264 int num_bo,
2265 struct mos_xe_context *ctx,
2266 std::vector<mos_xe_exec_bo> &exec_list,
2267 std::vector<struct drm_xe_sync> &syncs,
2268 std::vector<struct mos_xe_external_bo_info> &external_bos)
2269 {
2270 MOS_DRM_CHK_NULL_RETURN_VALUE(ctx, -EINVAL);
2271 uint32_t curr_dummy_exec_queue_id = ctx->dummy_exec_queue_id;
2272 uint32_t exec_list_size = exec_list.size();
2273 int ret = 0;
2274 std::set<uint32_t> exec_queue_ids;
2275 MOS_DRM_CHK_NULL_RETURN_VALUE(bufmgr_gem, -EINVAL);
2276 MOS_XE_GET_KEYS_FROM_MAP(bufmgr_gem->global_ctx_info, exec_queue_ids);
2277
2278 for (int i = 0; i < exec_list_size + num_bo; i++)
2279 {
2280 mos_xe_bo_gem *exec_bo_gem = nullptr;
2281 uint32_t exec_flags = 0;
2282 if (i < exec_list_size)
2283 {
2284 //exec list bo
2285 exec_bo_gem = (mos_xe_bo_gem *)exec_list[i].bo;
2286 exec_flags = exec_list[i].flags;
2287 }
2288 else
2289 {
2290 //batch bo
2291 exec_bo_gem = (mos_xe_bo_gem *)bo[i - exec_list_size];
2292 exec_flags = EXEC_OBJECT_WRITE_XE; //use write flags for batch bo as default
2293 }
2294
2295 if (exec_bo_gem)
2296 {
2297 if (exec_flags == 0)
2298 {
2299 //Add an assert message here in case of potential thread safety issue.
2300 //Currently, exec bo's flags could only be in (0, EXEC_OBJECT_READ_XE | EXEC_OBJECT_WRITE_XE]
2301 MOS_DRM_ASSERTMESSAGE("Invalid op flags(0x0) for exec bo(handle=%d)", exec_bo_gem->bo.handle);
2302 }
2303
2304 if (exec_bo_gem->is_imported || exec_bo_gem->is_exported)
2305 {
2306 //external bo, need to export its syncobj everytime.
2307 int prime_fd = INVALID_HANDLE;
2308 ret = mos_sync_update_exec_syncs_from_handle(
2309 bufmgr_gem->fd,
2310 exec_bo_gem->bo.handle,
2311 exec_flags,
2312 syncs,
2313 prime_fd);
2314 if (ret == MOS_XE_SUCCESS)
2315 {
2316 /**
2317 * Note, must import batch syncobj for each external bo
2318 * and close the syncobj created for them after exec submission.
2319 */
2320 int count = syncs.size();
2321 struct mos_xe_external_bo_info infos;
2322 memclear(infos);
2323 infos.syncobj_handle = syncs[count - 1].handle;
2324 infos.prime_fd = prime_fd;
2325 external_bos.push_back(infos);
2326 }
2327 else
2328 {
2329 //Note: continue process even failed.
2330 //This may only cause potential synchronization issue, DONT't crash umd here.
2331 MOS_DRM_ASSERTMESSAGE("Failed to update syncobj for external bo(%d)",
2332 exec_bo_gem->bo.handle);
2333 }
2334 }
2335 else
2336 {
2337 //internal bo
2338 ret = mos_sync_update_exec_syncs_from_timeline_deps(
2339 curr_dummy_exec_queue_id,
2340 exec_bo_gem->last_exec_write_exec_queue,
2341 exec_flags,
2342 exec_queue_ids,
2343 exec_bo_gem->read_deps,
2344 exec_bo_gem->write_deps,
2345 syncs);
2346 }
2347 }
2348 }
2349 return MOS_XE_SUCCESS;
2350 }
2351
2352 static int
2353 __mos_context_exec_update_bo_deps_xe(struct mos_linux_bo **bo,
2354 int num_bo,
2355 std::vector<mos_xe_exec_bo> &exec_list,
2356 uint32_t curr_exec_queue_id,
2357 struct mos_xe_dep *dep)
2358 {
2359 uint32_t exec_list_size = exec_list.size();
2360
2361 for (int i = 0; i < exec_list_size + num_bo; i++)
2362 {
2363 mos_xe_bo_gem *exec_bo_gem = nullptr;
2364 uint32_t exec_flags = 0;
2365 if (i < exec_list_size)
2366 {
2367 //exec list bo
2368 exec_bo_gem = (mos_xe_bo_gem *)exec_list[i].bo;
2369 exec_flags = exec_list[i].flags;
2370 }
2371 else
2372 {
2373 //batch bo
2374 exec_bo_gem = (mos_xe_bo_gem *)bo[i - exec_list_size];
2375 exec_flags = EXEC_OBJECT_WRITE_XE; //use write flags for batch bo as default.
2376 }
2377 if (exec_bo_gem)
2378 {
2379 mos_sync_update_bo_deps(curr_exec_queue_id, exec_flags, dep, exec_bo_gem->read_deps, exec_bo_gem->write_deps);
2380 if (exec_flags & EXEC_OBJECT_READ_XE)
2381 {
2382 exec_bo_gem->last_exec_read_exec_queue = curr_exec_queue_id;
2383 }
2384 if (exec_flags & EXEC_OBJECT_WRITE_XE)
2385 {
2386 exec_bo_gem->last_exec_write_exec_queue = curr_exec_queue_id;
2387 }
2388 }
2389 }
2390
2391 return MOS_XE_SUCCESS;
2392 }
2393
2394 /**
2395 * @ctx indicates to guity ctx that needs to recover for re-submission
2396 * @exec indicates to exec data in previous failed submission to re-submit
2397 * @curr_exec_queue_id indicates to guilty exec_queue_id, it will be replaced by newly creating one
2398 */
2399 static int
2400 __mos_bo_context_exec_retry_xe(struct mos_bufmgr *bufmgr,
2401 struct mos_linux_context *ctx,
2402 struct drm_xe_exec &exec,
2403 uint32_t &curr_exec_queue_id)
2404 {
2405 MOS_DRM_CHK_NULL_RETURN_VALUE(bufmgr, -EINVAL);
2406 MOS_DRM_CHK_NULL_RETURN_VALUE(ctx, -EINVAL);
2407
2408 struct mos_xe_bufmgr_gem *bufmgr_gem = (struct mos_xe_bufmgr_gem *)bufmgr;
2409 int ret = MOS_XE_SUCCESS;
2410
2411 //query ctx property firstly to check if failure is caused by exec_queue ban
2412 uint64_t property_value = 0;
2413 ret = __mos_get_context_property_xe(bufmgr, ctx, DRM_XE_EXEC_QUEUE_GET_PROPERTY_BAN, property_value);
2414
2415 /**
2416 * if exec_queue is banned, queried value is 1, otherwise it is zero;
2417 * if exec failure is not caused by exec_queue ban, umd could not help recover it.
2418 */
2419 if (ret || !property_value)
2420 {
2421 MOS_DRM_ASSERTMESSAGE("Failed to retore ctx(%d) with error(%d)",
2422 curr_exec_queue_id, -EPERM);
2423 return -EPERM;
2424 }
2425
2426 ret = __mos_context_restore_xe(bufmgr, ctx);
2427
2428 if (ret == MOS_XE_SUCCESS)
2429 {
2430 curr_exec_queue_id = ctx->ctx_id;
2431 exec.exec_queue_id = curr_exec_queue_id;
2432 //try once again to submit
2433 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_XE_EXEC, &exec);
2434 if (ret)
2435 {
2436 MOS_DRM_ASSERTMESSAGE("Failed to re-submission in DRM_IOCTL_XE_EXEC(errno:%d): new exec_queue_id = %d",
2437 ret, curr_exec_queue_id);
2438 }
2439 }
2440 else
2441 {
2442 MOS_DRM_ASSERTMESSAGE("Failed to retore context with error(%d), exec_queue_id = %d",
2443 ret, curr_exec_queue_id);
2444 }
2445 return ret;
2446 }
2447
2448 /**
2449 * @bo contains batch bo only.
2450 * @num_bo indicates to batch bo num.
2451 * @ctx indicates to the exec exec_queue.
2452
2453 *GPU<->GPU synchronization:
2454 * Exec must ensure the synchronization between GPU->GPU with bellow 8 steps:
2455 * 1. Get the deps from read_deps and write_deps by checking bo's op flags and add it into syncs array;
2456 * a) if flags & READ: get write_deps[last_write_exec_queue != ctx->dummy_exec_queue_id] & STATUS_DEP_BUSY only;
2457 * b) if flags & WRITE: get read_deps[all_exec_queue exclude ctx->dummy_exec_queue_id] & STATUS_DEP_BUSY
2458 * and write_deps[last_write_exec_queue != ctx->dummy_exec_queue_id] & STATUS_DEP_BUSY;
2459 * 2. Export a syncobj from external bo as dep and add it indo syncs array.
2460 * 3. Initial a new timeline dep object for exec queue if it doesn't have and add it to syncs array, otherwise add timeline
2461 * dep from context->timeline_dep directly while it has latest avaiable timeline point in it;
2462 * 4. Exec submittion with batches and syncs.
2463 * 5. Update read_deps[ctx->dummy_exec_queue_id] and write_deps[ctx->dummy_exec_queue_id] with the new deps from the dep_queue;
2464 * 6. Update timeline dep's timeline index to be latest avaiable one for currect exec queue.
2465 * 7. Import syncobj from batch bo for each external bo's DMA buffer for external process to wait media process on demand.
2466 * 8. Close syncobj handle and syncobj fd for external bo to avoid leak.
2467 * GPU->CPU(optional):
2468 * If bo->map_deps.dep exist:
2469 * get it and add it to exec syncs array
2470 */
2471 static int
2472 mos_bo_context_exec_with_sync_xe(struct mos_linux_bo **bo, int num_bo, struct mos_linux_context *ctx,
2473 struct drm_clip_rect *cliprects, int num_cliprects, int DR4,
2474 unsigned int flags, int *fence)
2475 {
2476
2477 MOS_DRM_CHK_NULL_RETURN_VALUE(bo, -EINVAL)
2478 MOS_DRM_CHK_NULL_RETURN_VALUE(ctx, -EINVAL)
2479 if (num_bo <= 0)
2480 {
2481 MOS_DRM_ASSERTMESSAGE("invalid batch bo num(%d)", num_bo);
2482 return -EINVAL;
2483 }
2484
2485 struct mos_xe_bufmgr_gem *bufmgr_gem = (struct mos_xe_bufmgr_gem *) bo[0]->bufmgr;
2486 MOS_DRM_CHK_NULL_RETURN_VALUE(bufmgr_gem, -EINVAL)
2487
2488 uint64_t *batch_addrs = (uint64_t*)MOS_AllocAndZeroMemory(num_bo * sizeof(uint64_t));
2489
2490 std::vector<mos_xe_exec_bo> exec_list;
2491 for (int i = 0; i < num_bo; i++)
2492 {
2493 MOS_DRM_CHK_NULL_RETURN_VALUE(bo[i], -EINVAL)
2494 batch_addrs[i] = bo[i]->offset64;
2495 struct mos_xe_bo_gem *batch_bo_gem = (struct mos_xe_bo_gem *) bo[i];
2496 MOS_XE_GET_VALUES_FROM_MAP(batch_bo_gem->exec_list, exec_list);
2497 }
2498
2499 struct mos_xe_context *context = (struct mos_xe_context *) ctx;
2500 uint32_t curr_exec_queue_id = context->ctx.ctx_id;
2501 std::vector<struct mos_xe_external_bo_info> external_bos;
2502 std::vector<struct drm_xe_sync> syncs;
2503 uint64_t curr_timeline = 0;
2504 int ret = 0;
2505
2506 uint32_t exec_list_size = exec_list.size();
2507 if (exec_list_size == 0)
2508 {
2509 MOS_DRM_NORMALMESSAGE("invalid exec list count(%d)", exec_list_size);
2510 }
2511
2512 bufmgr_gem->m_lock.lock();
2513
2514 if (context->timeline_dep == nullptr)
2515 {
2516 context->timeline_dep = mos_sync_create_timeline_dep(bufmgr_gem->fd);
2517
2518 if (context->timeline_dep == nullptr)
2519 {
2520 MOS_DRM_ASSERTMESSAGE("Failed to initial context timeline dep");
2521 bufmgr_gem->m_lock.unlock();
2522 MOS_SafeFreeMemory(batch_addrs);
2523 return -ENOMEM;
2524 }
2525 }
2526
2527 struct mos_xe_dep *dep = context->timeline_dep;
2528 //add latest avaiable timeline point(dep) into syncs as fence out point.
2529 mos_sync_update_exec_syncs_from_timeline_dep(
2530 bufmgr_gem->fd,
2531 dep,
2532 syncs);
2533
2534 bufmgr_gem->sync_obj_rw_lock.lock_shared();
2535 //update exec syncs array by external and interbal bo dep
2536 __mos_context_exec_update_syncs_xe(
2537 bufmgr_gem,
2538 bo,
2539 num_bo,
2540 context,
2541 exec_list,
2542 syncs,
2543 external_bos);
2544
2545 //exec submit
2546 uint32_t sync_count = syncs.size();
2547 struct drm_xe_sync *syncs_array = syncs.data();
2548
2549 //dump bo deps map
2550 __mos_dump_bo_deps_map_xe(bo, num_bo, exec_list, curr_exec_queue_id, bufmgr_gem->global_ctx_info);
2551 //dump fence in and fence out info
2552 __mos_dump_syncs_array_xe(syncs_array, sync_count, dep);
2553
2554 struct drm_xe_exec exec;
2555 memclear(exec);
2556 exec.extensions = 0;
2557 exec.exec_queue_id = curr_exec_queue_id;
2558 exec.num_syncs = sync_count;
2559 exec.syncs = (uintptr_t)syncs_array;
2560 /**
2561 * exec.address only accepts batch->offset64 when num bo == 1;
2562 * and it only accepts batch array when num bo > 1
2563 */
2564 exec.address = (num_bo == 1 ? (uintptr_t)batch_addrs[0] : (uintptr_t)batch_addrs);
2565 exec.num_batch_buffer = num_bo;
2566 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_XE_EXEC, &exec);
2567 if (ret)
2568 {
2569 MOS_DRM_ASSERTMESSAGE("Failed to submission in DRM_IOCTL_XE_EXEC(errno:%d): exec_queue_id = %d, num_syncs = %d, num_bo = %d",
2570 -errno, curr_exec_queue_id, sync_count, num_bo);
2571
2572 //check if it caused by guilty exec_queue_id, if so, could restore the exec_queue_id/ queue here and re-try exec again.
2573 if (ret == -EPERM)
2574 {
2575 ret = __mos_bo_context_exec_retry_xe(&bufmgr_gem->bufmgr, ctx, exec, curr_exec_queue_id);
2576 }
2577 }
2578 curr_timeline = dep->timeline_index;
2579
2580 //update bos' read and write dep with new timeline
2581 __mos_context_exec_update_bo_deps_xe(bo, num_bo, exec_list, context->dummy_exec_queue_id, dep);
2582
2583 //Update dep with latest available timeline
2584 mos_sync_update_timeline_dep(dep);
2585
2586 bufmgr_gem->sync_obj_rw_lock.unlock_shared();
2587 bufmgr_gem->m_lock.unlock();
2588
2589 //import batch syncobj or its point for external bos and close syncobj created for external bo before.
2590 uint32_t external_bo_count = external_bos.size();
2591 int sync_file_fd = INVALID_HANDLE;
2592 int temp_syncobj = INVALID_HANDLE;
2593
2594 if (external_bo_count > 0)
2595 {
2596 temp_syncobj = mos_sync_syncobj_create(bufmgr_gem->fd, 0);
2597 if (temp_syncobj > 0)
2598 {
2599 mos_sync_syncobj_timeline_to_binary(bufmgr_gem->fd, temp_syncobj, dep->syncobj_handle, curr_timeline, 0);
2600 sync_file_fd = mos_sync_syncobj_handle_to_syncfile_fd(bufmgr_gem->fd, temp_syncobj);
2601 }
2602 }
2603 for (int i = 0; i < external_bo_count; i++)
2604 {
2605 //import syncobj for external bos
2606 if (sync_file_fd >= 0)
2607 {
2608 mos_sync_import_syncfile_to_external_bo(bufmgr_gem->fd, external_bos[i].prime_fd, sync_file_fd);
2609 }
2610 if (external_bos[i].prime_fd != INVALID_HANDLE)
2611 {
2612 close(external_bos[i].prime_fd);
2613 }
2614 mos_sync_syncobj_destroy(bufmgr_gem->fd, external_bos[i].syncobj_handle);
2615 }
2616 if (sync_file_fd >= 0)
2617 {
2618 close(sync_file_fd);
2619 }
2620 if (temp_syncobj > 0)
2621 {
2622 mos_sync_syncobj_destroy(bufmgr_gem->fd, temp_syncobj);
2623 }
2624 MOS_SafeFreeMemory(batch_addrs);
2625 //Note: keep exec return value for final return value.
2626 return ret;
2627 }
2628
2629 /**
2630 * Get the DEVICE ID for the device. This can be overridden by setting the
2631 * INTEL_DEVID_OVERRIDE environment variable to the desired ID.
2632 */
2633 static int
2634 mos_get_devid_xe(struct mos_bufmgr *bufmgr)
2635 {
2636 struct mos_xe_bufmgr_gem *bufmgr_gem = (struct mos_xe_bufmgr_gem *)bufmgr;
2637 int fd = bufmgr_gem->fd;
2638 struct mos_xe_device *dev = &bufmgr_gem->xe_device;
2639
2640 MOS_DRM_CHK_XE_DEV(dev, config, __mos_query_config_xe, 0)
2641 struct drm_xe_query_config *config = dev->config;
2642
2643 return (config->info[DRM_XE_QUERY_CONFIG_REV_AND_DEVICE_ID] & 0xffff);
2644 }
2645
2646 static struct drm_xe_query_engines *
2647 __mos_query_engines_xe(int fd)
2648 {
2649 if (fd < 0)
2650 {
2651 return nullptr;
2652 }
2653
2654 struct drm_xe_device_query query;
2655 struct drm_xe_query_engines *engines;
2656 int ret;
2657
2658 memclear(query);
2659 query.extensions = 0;
2660 query.query = DRM_XE_DEVICE_QUERY_ENGINES;
2661 query.size = 0;
2662 query.data = 0;
2663
2664 ret = drmIoctl(fd, DRM_IOCTL_XE_DEVICE_QUERY, &query);
2665 if (ret || !query.size)
2666 {
2667 MOS_DRM_ASSERTMESSAGE("ret:%d, length:%d", ret, query.size);
2668 return nullptr;
2669 }
2670
2671 engines = (drm_xe_query_engines *)calloc(1, query.size);
2672 MOS_DRM_CHK_NULL_RETURN_VALUE(engines, nullptr)
2673
2674 query.data = (uintptr_t)engines;
2675 ret = drmIoctl(fd, DRM_IOCTL_XE_DEVICE_QUERY, &query);
2676 if (ret || !query.size)
2677 {
2678 MOS_DRM_ASSERTMESSAGE("ret:%d, length:%d", ret, query.size);
2679 MOS_XE_SAFE_FREE(engines);
2680 return nullptr;
2681 }
2682
2683 return engines;
2684 }
2685
2686 static int
2687 mos_query_engines_count_xe(struct mos_bufmgr *bufmgr, unsigned int *nengine)
2688 {
2689 MOS_DRM_CHK_NULL_RETURN_VALUE(nengine, -EINVAL);
2690 struct mos_xe_bufmgr_gem *bufmgr_gem = (struct mos_xe_bufmgr_gem *)bufmgr;
2691 int fd = bufmgr_gem->fd;
2692 struct mos_xe_device *dev = &bufmgr_gem->xe_device;
2693
2694 MOS_DRM_CHK_XE_DEV(dev, engines, __mos_query_engines_xe, -ENODEV)
2695 *nengine = dev->engines->num_engines;
2696
2697 return MOS_XE_SUCCESS;
2698 }
2699
2700 int
2701 mos_query_engines_xe(struct mos_bufmgr *bufmgr,
2702 __u16 engine_class,
2703 __u64 caps,
2704 unsigned int *nengine,
2705 void *engine_map)
2706 {
2707 MOS_DRM_CHK_NULL_RETURN_VALUE(nengine, -EINVAL);
2708 MOS_DRM_CHK_NULL_RETURN_VALUE(engine_map, -EINVAL);
2709
2710 struct mos_xe_bufmgr_gem *bufmgr_gem = (struct mos_xe_bufmgr_gem *)bufmgr;
2711 struct drm_xe_engine_class_instance *ci = (struct drm_xe_engine_class_instance *)engine_map;
2712 int fd = bufmgr_gem->fd;
2713 struct mos_xe_device *dev = &bufmgr_gem->xe_device;
2714
2715 MOS_DRM_CHK_XE_DEV(dev, engines, __mos_query_engines_xe, -ENODEV)
2716 struct drm_xe_query_engines *engines = dev->engines;
2717
2718 int i, num;
2719 struct drm_xe_engine *engine;
2720 for (i = 0, num = 0; i < engines->num_engines; i++)
2721 {
2722 engine = (struct drm_xe_engine *)&engines->engines[i];
2723 if (engine_class == engine->instance.engine_class)
2724 {
2725 ci->engine_class = engine_class;
2726 ci->engine_instance = engine->instance.engine_instance;
2727 ci->gt_id = engine->instance.gt_id;
2728 ci++;
2729 num++;
2730 }
2731
2732 if (num > *nengine)
2733 {
2734 MOS_DRM_ASSERTMESSAGE("Number of engine instances out of range, %d,%d", num, *nengine);
2735 return -1;
2736 }
2737 }
2738
2739 //Note30: need to confirm if engine_instance is ordered, otherwise re-order needed.
2740
2741 *nengine = num;
2742
2743 return 0;
2744 }
2745
2746 static size_t
2747 mos_get_engine_class_size_xe()
2748 {
2749 return sizeof(struct drm_xe_engine_class_instance);
2750 }
2751
2752 static int
2753 mos_query_sysinfo_xe(struct mos_bufmgr *bufmgr, MEDIA_SYSTEM_INFO* gfx_info)
2754 {
2755 MOS_DRM_CHK_NULL_RETURN_VALUE(bufmgr, -EINVAL);
2756 MOS_DRM_CHK_NULL_RETURN_VALUE(gfx_info, -EINVAL);
2757
2758 struct mos_xe_bufmgr_gem *bufmgr_gem = (struct mos_xe_bufmgr_gem *)bufmgr;
2759 int fd = bufmgr_gem->fd;
2760 struct mos_xe_device *dev = &bufmgr_gem->xe_device;
2761 int ret;
2762
2763 MOS_DRM_CHK_XE_DEV(dev, engines, __mos_query_engines_xe, -ENODEV)
2764
2765 if (0 == gfx_info->VDBoxInfo.NumberOfVDBoxEnabled
2766 || 0 == gfx_info->VEBoxInfo.NumberOfVEBoxEnabled)
2767 {
2768 unsigned int num_vd = 0;
2769 unsigned int num_ve = 0;
2770
2771 for (unsigned int i = 0; i < dev->engines->num_engines; i++)
2772 {
2773 if (0 == gfx_info->VDBoxInfo.NumberOfVDBoxEnabled
2774 && dev->engines->engines[i].instance.engine_class == DRM_XE_ENGINE_CLASS_VIDEO_DECODE)
2775 {
2776 gfx_info->VDBoxInfo.Instances.VDBoxEnableMask |=
2777 1 << dev->engines->engines[i].instance.engine_instance;
2778 num_vd++;
2779 }
2780
2781 if (0 == gfx_info->VEBoxInfo.NumberOfVEBoxEnabled
2782 && dev->engines->engines[i].instance.engine_class == DRM_XE_ENGINE_CLASS_VIDEO_ENHANCE)
2783 {
2784 num_ve++;
2785 }
2786 }
2787
2788 if (num_vd > 0)
2789 {
2790 gfx_info->VDBoxInfo.NumberOfVDBoxEnabled = num_vd;
2791 }
2792
2793 if (num_vd > 0)
2794 {
2795 gfx_info->VEBoxInfo.NumberOfVEBoxEnabled = num_ve;
2796 }
2797 }
2798
2799 return 0;
2800 }
2801
2802 void mos_select_fixed_engine_xe(struct mos_bufmgr *bufmgr,
2803 void *engine_map,
2804 uint32_t *nengine,
2805 uint32_t fixed_instance_mask)
2806 {
2807 MOS_UNUSED(bufmgr);
2808 #if (DEBUG || _RELEASE_INTERNAL)
2809 if (fixed_instance_mask)
2810 {
2811 struct drm_xe_engine_class_instance *_engine_map = (struct drm_xe_engine_class_instance *)engine_map;
2812 auto unselect_index = 0;
2813 for (auto bit = 0; bit < *nengine; bit++)
2814 {
2815 if (((fixed_instance_mask >> bit) & 0x1) && (bit > unselect_index))
2816 {
2817 _engine_map[unselect_index].engine_class = _engine_map[bit].engine_class;
2818 _engine_map[unselect_index].engine_instance = _engine_map[bit].engine_instance;
2819 _engine_map[unselect_index].gt_id = _engine_map[bit].gt_id;
2820 _engine_map[unselect_index].pad = _engine_map[bit].pad;
2821 _engine_map[bit].engine_class = 0;
2822 _engine_map[bit].engine_instance = 0;
2823 _engine_map[bit].gt_id = 0;
2824 _engine_map[bit].pad = 0;
2825 unselect_index++;
2826 }
2827 else if (((fixed_instance_mask >> bit) & 0x1) && (bit == unselect_index))
2828 {
2829 unselect_index++;
2830 }
2831 else if (!((fixed_instance_mask >> bit) & 0x1))
2832 {
2833 _engine_map[bit].engine_class = 0;
2834 _engine_map[bit].engine_instance = 0;
2835 _engine_map[bit].gt_id = 0;
2836 _engine_map[bit].pad = 0;
2837 }
2838 }
2839 *nengine = unselect_index;
2840 }
2841 #else
2842 MOS_UNUSED(engine_map);
2843 MOS_UNUSED(nengine);
2844 MOS_UNUSED(fixed_instance_mask);
2845 #endif
2846
2847 }
2848
2849
2850 /**
2851 * Note: xe kmd doesn't support query blob before dg2.
2852 */
2853 static uint32_t *
2854 __mos_query_hw_config_xe(int fd)
2855 {
2856 struct drm_xe_device_query query;
2857 uint32_t *hw_config;
2858 int ret;
2859
2860 if (fd < 0)
2861 {
2862 return nullptr;
2863 }
2864
2865 memclear(query);
2866 query.query = DRM_XE_DEVICE_QUERY_HWCONFIG;
2867
2868 ret = drmIoctl(fd, DRM_IOCTL_XE_DEVICE_QUERY, &query);
2869 if (ret || !query.size)
2870 {
2871 MOS_DRM_ASSERTMESSAGE("ret:%d, length:%d", ret, query.size);
2872 return nullptr;
2873 }
2874
2875 hw_config = (uint32_t *)calloc(1, query.size + sizeof(uint32_t));
2876 MOS_DRM_CHK_NULL_RETURN_VALUE(hw_config, nullptr)
2877
2878 query.data = (uintptr_t)&hw_config[1];
2879 ret = drmIoctl(fd, DRM_IOCTL_XE_DEVICE_QUERY, &query);
2880 if (ret != 0 || query.size <= 0)
2881 {
2882 MOS_DRM_ASSERTMESSAGE("ret:%d, length:%d", ret, query.size);
2883 MOS_XE_SAFE_FREE(hw_config);
2884 return nullptr;
2885 }
2886
2887 hw_config[0] = query.size / sizeof(uint32_t);
2888
2889 return hw_config;
2890 }
2891
2892 static int
2893 mos_query_device_blob_xe(struct mos_bufmgr *bufmgr, MEDIA_SYSTEM_INFO* gfx_info)
2894 {
2895 MOS_DRM_CHK_NULL_RETURN_VALUE(gfx_info, -EINVAL)
2896
2897 struct mos_xe_bufmgr_gem *bufmgr_gem = (struct mos_xe_bufmgr_gem *)bufmgr;
2898 int fd = bufmgr_gem->fd;
2899 struct mos_xe_device *dev = &bufmgr_gem->xe_device;
2900
2901 MOS_DRM_CHK_XE_DEV(dev, hw_config, __mos_query_hw_config_xe, -ENODEV)
2902
2903 uint32_t *hwconfig = &dev->hw_config[1];
2904 uint32_t num_config = dev->hw_config[0];
2905
2906 int i = 0;
2907 while (i < num_config) {
2908 /* Attribute ID starts with 1 */
2909 assert(hwconfig[i] > 0);
2910
2911 #if DEBUG_BLOB_QUERY
2912 MOS_DRM_NORMALMESSAGE("query blob: key=%s, value=%d", key_string[hwconfig[i]], hwconfig[i+2]);
2913 #endif
2914 if (INTEL_HWCONFIG_MAX_SLICES_SUPPORTED == hwconfig[i])
2915 {
2916 assert(hwconfig[i+1] == 1);
2917 gfx_info->SliceCount = hwconfig[i+2];
2918 gfx_info->MaxSlicesSupported = hwconfig[i+2];
2919 }
2920
2921 if ((INTEL_HWCONFIG_MAX_DUAL_SUBSLICES_SUPPORTED == hwconfig[i])
2922 || (INTEL_HWCONFIG_MAX_SUBSLICE == hwconfig[i]))
2923 {
2924 assert(hwconfig[i+1] == 1);
2925 gfx_info->SubSliceCount = hwconfig[i+2];
2926 gfx_info->MaxSubSlicesSupported = hwconfig[i+2];
2927 }
2928
2929 if ((INTEL_HWCONFIG_MAX_NUM_EU_PER_DSS == hwconfig[i])
2930 || (INTEL_HWCONFIG_MAX_EU_PER_SUBSLICE == hwconfig[i]))
2931 {
2932 assert(hwconfig[i+1] == 1);
2933 gfx_info->MaxEuPerSubSlice = hwconfig[i+2];
2934 }
2935
2936 if (INTEL_HWCONFIG_DEPRECATED_L3_CACHE_SIZE_IN_KB == hwconfig[i])
2937 {
2938 assert(hwconfig[i+1] == 1);
2939 gfx_info->L3CacheSizeInKb = hwconfig[i+2];
2940 }
2941
2942 if (INTEL_HWCONFIG_NUM_THREADS_PER_EU == hwconfig[i])
2943 {
2944 assert(hwconfig[i+1] == 1);
2945 gfx_info->NumThreadsPerEu = hwconfig[i+2];
2946 }
2947
2948 if (INTEL_HWCONFIG_MAX_VECS == hwconfig[i])
2949 {
2950 assert(hwconfig[i+1] == 1);
2951 gfx_info->MaxVECS = hwconfig[i+2];
2952 }
2953
2954 if (INTEL_HWCONFIG_SLM_SIZE_PER_SS_IN_KB == hwconfig[i])
2955 {
2956 assert(hwconfig[i + 1] == 1);
2957 gfx_info->SLMSizeInKb = hwconfig[i + 2];
2958 }
2959
2960 /* Advance to next key */
2961 i += hwconfig[i + 1]; // value size
2962 i += 2;// KL size
2963 }
2964
2965 return 0;
2966 }
2967
2968 static void
2969 mos_enable_reuse_xe(struct mos_bufmgr *bufmgr)
2970 {
2971 MOS_UNIMPLEMENT(bufmgr);
2972 }
2973
2974 // The function is not supported on KMD
2975 static int mos_query_hw_ip_version_xe(struct mos_bufmgr *bufmgr, __u16 engine_class, void *ip_ver_info)
2976 {
2977 MOS_UNIMPLEMENT(bufmgr);
2978 MOS_UNIMPLEMENT(engine_class);
2979 MOS_UNIMPLEMENT(ip_ver_info);
2980 return 0;
2981 }
2982
2983 static void
2984 mos_bo_free_xe(struct mos_linux_bo *bo)
2985 {
2986 struct mos_xe_bufmgr_gem *bufmgr_gem = nullptr;
2987 struct mos_xe_bo_gem *bo_gem = (struct mos_xe_bo_gem *) bo;
2988 struct drm_gem_close close_ioctl;
2989 int ret;
2990
2991 if (nullptr == bo_gem)
2992 {
2993 MOS_DRM_ASSERTMESSAGE("bo == nullptr");
2994 return;
2995 }
2996
2997 bufmgr_gem = (struct mos_xe_bufmgr_gem *) bo->bufmgr;
2998
2999 if (nullptr == bufmgr_gem)
3000 {
3001 MOS_DRM_ASSERTMESSAGE("bufmgr_gem == nullptr");
3002 return;
3003 }
3004
3005 mos_gem_bo_wait_rendering_xe(bo);
3006
3007 bufmgr_gem->m_lock.lock();
3008
3009 if (!bo_gem->is_userptr)
3010 {
3011 if (bo_gem->mem_virtual)
3012 {
3013 VG(VALGRIND_MAKE_MEM_NOACCESS(bo_gem->mem_virtual, 0));
3014 drm_munmap(bo_gem->mem_virtual, bo_gem->bo.size);
3015 bo_gem->mem_virtual = nullptr;
3016 }
3017 }
3018
3019 if (bo->vm_id != INVALID_VM)
3020 {
3021 ret = mos_vm_bind_sync_xe(bufmgr_gem->fd,
3022 bo->vm_id,
3023 0,
3024 0,
3025 bo->offset64,
3026 bo->size,
3027 bo_gem->pat_index,
3028 DRM_XE_VM_BIND_OP_UNMAP);
3029 if (ret)
3030 {
3031 MOS_DRM_ASSERTMESSAGE("mos_gem_bo_free mos_vm_unbind ret error. bo:0x%lx, vm_id:%d\r",
3032 (uint64_t)bo,
3033 bo->vm_id);
3034 }
3035 else
3036 {
3037 bo->vm_id = INVALID_VM;
3038 }
3039 }
3040
3041 if (!bo_gem->is_userptr)
3042 {
3043 /* Close this object */
3044 memclear(close_ioctl);
3045 close_ioctl.handle = bo_gem->gem_handle;
3046 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_GEM_CLOSE, &close_ioctl);
3047 if (ret != 0)
3048 {
3049 MOS_DRM_ASSERTMESSAGE("DRM_IOCTL_GEM_CLOSE %d failed (%s): %s",
3050 bo_gem->gem_handle, bo_gem->name, strerror(errno));
3051 }
3052 }
3053
3054 if (bufmgr_gem->mem_profiler_fd != -1)
3055 {
3056 snprintf(bufmgr_gem->mem_profiler_buffer, MEM_PROFILER_BUFFER_SIZE, "GEM_CLOSE, %d, %d, %lu, %d\n", getpid(), bo->handle,bo->size,bo_gem->mem_region);
3057 ret = write(bufmgr_gem->mem_profiler_fd, bufmgr_gem->mem_profiler_buffer, strnlen(bufmgr_gem->mem_profiler_buffer, MEM_PROFILER_BUFFER_SIZE));
3058 if (-1 == ret)
3059 {
3060 snprintf(bufmgr_gem->mem_profiler_buffer, MEM_PROFILER_BUFFER_SIZE, "GEM_CLOSE, %d, %d, %lu, %d\n", getpid(), bo->handle,bo->size,bo_gem->mem_region);
3061 ret = write(bufmgr_gem->mem_profiler_fd, bufmgr_gem->mem_profiler_buffer, strnlen(bufmgr_gem->mem_profiler_buffer, MEM_PROFILER_BUFFER_SIZE));
3062 if (-1 == ret)
3063 {
3064 MOS_DRM_ASSERTMESSAGE("Failed to write to %s: %s", bufmgr_gem->mem_profiler_path, strerror(errno));
3065 }
3066 }
3067 }
3068
3069 /* Return the VMA for reuse */
3070 __mos_bo_vma_free_xe(bo->bufmgr, bo->offset64, bo->size);
3071 bufmgr_gem->m_lock.unlock();
3072
3073 MOS_Delete(bo_gem);
3074 }
3075
3076 static int
3077 mos_bo_set_softpin_xe(MOS_LINUX_BO *bo)
3078 {
3079 MOS_UNIMPLEMENT(bo);
3080 return 0;
3081 }
3082
3083 static void
3084 mos_bufmgr_gem_destroy_xe(struct mos_bufmgr *bufmgr)
3085 {
3086 if (nullptr == bufmgr)
3087 return;
3088
3089 struct mos_xe_bufmgr_gem *bufmgr_gem = (struct mos_xe_bufmgr_gem *) bufmgr;
3090 struct mos_xe_device *dev = &bufmgr_gem->xe_device;
3091 int i, ret;
3092
3093 /* Release userptr bo kept hanging around for optimisation. */
3094
3095 mos_vma_heap_finish(&bufmgr_gem->vma_heap[MEMZONE_SYS]);
3096 mos_vma_heap_finish(&bufmgr_gem->vma_heap[MEMZONE_DEVICE]);
3097 mos_vma_heap_finish(&bufmgr_gem->vma_heap[MEMZONE_PRIME]);
3098
3099 if (bufmgr_gem->vm_id != INVALID_VM)
3100 {
3101 __mos_vm_destroy_xe(bufmgr, bufmgr_gem->vm_id);
3102 bufmgr_gem->vm_id = INVALID_VM;
3103 }
3104
3105 if (bufmgr_gem->mem_profiler_fd != -1)
3106 {
3107 close(bufmgr_gem->mem_profiler_fd);
3108 }
3109
3110 MOS_XE_SAFE_FREE(dev->hw_config);
3111 dev->hw_config = nullptr;
3112
3113 MOS_XE_SAFE_FREE(dev->config);
3114 dev->config = nullptr;
3115
3116 MOS_XE_SAFE_FREE(dev->engines);
3117 dev->engines = nullptr;
3118
3119 MOS_XE_SAFE_FREE(dev->mem_regions);
3120 dev->mem_regions = nullptr;
3121
3122 MOS_XE_SAFE_FREE(dev->gt_list);
3123 dev->gt_list = nullptr;
3124
3125 MOS_Delete(bufmgr_gem);
3126 }
3127
3128 static void
3129 mos_bufmgr_gem_unref_xe(struct mos_bufmgr *bufmgr)
3130 {
3131 struct mos_xe_bufmgr_gem *bufmgr_gem = (struct mos_xe_bufmgr_gem *)bufmgr;
3132
3133 if (bufmgr_gem && atomic_add_unless(&bufmgr_gem->ref_count, -1, 1))
3134 {
3135 pthread_mutex_lock(&bufmgr_list_mutex);
3136
3137 if (atomic_dec_and_test(&bufmgr_gem->ref_count))
3138 {
3139 DRMLISTDEL(&bufmgr_gem->managers);
3140 mos_bufmgr_gem_destroy_xe(bufmgr);
3141 }
3142
3143 pthread_mutex_unlock(&bufmgr_list_mutex);
3144 }
3145 }
3146
3147 static int
3148 mo_get_context_param_xe(struct mos_linux_context *ctx,
3149 uint32_t size,
3150 uint64_t param,
3151 uint64_t *value)
3152 {
3153 MOS_UNIMPLEMENT(ctx);
3154 MOS_UNIMPLEMENT(size);
3155 MOS_UNIMPLEMENT(param);
3156 MOS_UNIMPLEMENT(value);
3157 return 0;
3158 }
3159
3160 static void mos_enable_softpin_xe(struct mos_bufmgr *bufmgr, bool va1m_align)
3161 {
3162 MOS_UNIMPLEMENT(bufmgr);
3163 MOS_UNIMPLEMENT(va1m_align);
3164 }
3165
3166 static int
3167 mos_get_reset_stats_xe(struct mos_linux_context *ctx,
3168 uint32_t *reset_count,
3169 uint32_t *active,
3170 uint32_t *pending)
3171 {
3172 MOS_DRM_CHK_NULL_RETURN_VALUE(ctx, -EINVAL);
3173
3174 struct mos_xe_context *context = (struct mos_xe_context *)ctx;
3175 if (reset_count)
3176 *reset_count = context->reset_count;
3177 if (active)
3178 *active = 0;
3179 if (pending)
3180 *pending = 0;
3181 return 0;
3182 }
3183
3184 static mos_oca_exec_list_info*
3185 mos_bo_get_oca_exec_list_info_xe(struct mos_linux_bo *bo, int *count)
3186 {
3187 if (nullptr == bo || nullptr == count)
3188 {
3189 return nullptr;
3190 }
3191
3192 mos_oca_exec_list_info *info = nullptr;
3193 int counter = 0;
3194 int MAX_COUNT = 50;
3195 struct mos_xe_bufmgr_gem *bufmgr_gem = (struct mos_xe_bufmgr_gem *) bo->bufmgr;
3196 struct mos_xe_bo_gem *bo_gem = (struct mos_xe_bo_gem *)bo;
3197 int exec_list_count = bo_gem->exec_list.size();
3198
3199 if (exec_list_count == 0 || exec_list_count > MAX_COUNT)
3200 {
3201 return nullptr;
3202 }
3203
3204 info = (mos_oca_exec_list_info *)malloc((exec_list_count + 1) * sizeof(mos_oca_exec_list_info));
3205 if (!info)
3206 {
3207 MOS_DRM_ASSERTMESSAGE("malloc mos_oca_exec_list_info failed");
3208 return info;
3209 }
3210
3211 for (auto &it : bo_gem->exec_list)
3212 {
3213 /*note: set capture for each bo*/
3214 struct mos_xe_bo_gem *exec_bo_gem = (struct mos_xe_bo_gem *)it.second.bo;
3215 uint32_t exec_flags = it.second.flags;
3216 if (exec_bo_gem)
3217 {
3218 info[counter].handle = exec_bo_gem->bo.handle;
3219 info[counter].size = exec_bo_gem->bo.size;
3220 info[counter].offset64 = exec_bo_gem->bo.offset64;
3221 info[counter].flags = exec_flags;
3222 info[counter].mem_region = exec_bo_gem->mem_region;
3223 info[counter].is_batch = false;
3224 counter++;
3225 }
3226 }
3227
3228 /*note: bo is cmd bo, also need to be added*/
3229 info[counter].handle = bo->handle;
3230 info[counter].size = bo->size;
3231 info[counter].offset64 = bo->offset64;
3232 info[counter].flags = EXEC_OBJECT_WRITE_XE; // use write flags for batch bo as default.
3233 info[counter].mem_region = bo_gem->mem_region;
3234 info[counter].is_batch = true;
3235 counter++;
3236
3237 *count = counter;
3238
3239 return info;
3240 }
3241
3242 static bool
3243 mos_has_bsd2_xe(struct mos_bufmgr *bufmgr)
3244 {
3245 MOS_UNUSED(bufmgr);
3246 return true;
3247 }
3248
3249 static void
3250 mos_bo_set_object_capture_xe(struct mos_linux_bo *bo)
3251 {
3252 MOS_UNIMPLEMENT(bo);
3253 }
3254
3255 static void
3256 mos_bo_set_object_async_xe(struct mos_linux_bo *bo)
3257 {
3258 MOS_UNIMPLEMENT(bo);
3259 }
3260
3261 static int
3262 mos_get_driver_info_xe(struct mos_bufmgr *bufmgr, struct LinuxDriverInfo *drvInfo)
3263 {
3264 MOS_DRM_CHK_NULL_RETURN_VALUE(drvInfo, -EINVAL)
3265 struct mos_xe_bufmgr_gem *bufmgr_gem = (struct mos_xe_bufmgr_gem *)bufmgr;
3266 struct mos_xe_device *dev = &bufmgr_gem->xe_device;
3267 int fd = bufmgr_gem->fd;
3268
3269 uint32_t MaxEuPerSubSlice = 0;
3270 int i = 0;
3271 drvInfo->hasBsd = 1;
3272 drvInfo->hasBsd2 = 1;
3273 drvInfo->hasVebox = 1;
3274
3275 //For XE driver always has ppgtt
3276 drvInfo->hasPpgtt = 1;
3277
3278 /**
3279 * query blob
3280 * Note: xe kmd doesn't support query blob before dg2, so don't check null and return here.
3281 */
3282 if (dev->hw_config == nullptr)
3283 {
3284 dev->hw_config = __mos_query_hw_config_xe(fd);
3285 }
3286
3287 if (dev->hw_config)
3288 {
3289 uint32_t *hw_config = &dev->hw_config[1];
3290 uint32_t num_config = dev->hw_config[0];
3291
3292 while (i < num_config)
3293 {
3294 /* Attribute ID starts with 1 */
3295 assert(hw_config[i] > 0);
3296
3297 #if DEBUG_BLOB_QUERY
3298 MOS_DRM_NORMALMESSAGE("query blob: key=%s, value=%d", key_string[hw_config[i]], hw_config[i+2]);
3299 #endif
3300 if (INTEL_HWCONFIG_MAX_SLICES_SUPPORTED == hw_config[i])
3301 {
3302 assert(hw_config[i+1] == 1);
3303 drvInfo->sliceCount = hw_config[i+2];
3304 }
3305
3306 if ((INTEL_HWCONFIG_MAX_DUAL_SUBSLICES_SUPPORTED == hw_config[i])
3307 || (INTEL_HWCONFIG_MAX_SUBSLICE == hw_config[i]))
3308 {
3309 assert(hw_config[i+1] == 1);
3310 drvInfo->subSliceCount = hw_config[i+2];
3311 }
3312
3313 if ((INTEL_HWCONFIG_MAX_NUM_EU_PER_DSS == hw_config[i])
3314 || (INTEL_HWCONFIG_MAX_EU_PER_SUBSLICE == hw_config[i]))
3315 {
3316 assert(hw_config[i+1] == 1);
3317 MaxEuPerSubSlice = hw_config[i+2];
3318 }
3319
3320 /* Advance to next key */
3321 i += hw_config[i + 1]; // value size
3322 i += 2;// KL size
3323 }
3324
3325 drvInfo->euCount = drvInfo->subSliceCount * MaxEuPerSubSlice;
3326 }
3327 else
3328 {
3329 drvInfo->euCount = 96;
3330 drvInfo->subSliceCount = 6;
3331 drvInfo->sliceCount = 1;
3332 }
3333
3334 // query engines info
3335 MOS_DRM_CHK_XE_DEV(dev, engines, __mos_query_engines_xe, -ENODEV)
3336 struct drm_xe_query_engines *engines = dev->engines;
3337 int num_vd = 0;
3338 int num_ve = 0;
3339 for (i = 0; i < engines->num_engines; i++)
3340 {
3341 if (DRM_XE_ENGINE_CLASS_VIDEO_DECODE == engines->engines[i].instance.engine_class)
3342 {
3343 num_vd++;
3344 }
3345 else if (DRM_XE_ENGINE_CLASS_VIDEO_ENHANCE == engines->engines[i].instance.engine_class)
3346 {
3347 num_ve++;
3348 }
3349 }
3350
3351 if (num_vd >= 1)
3352 {
3353 drvInfo->hasBsd = 1;
3354 }
3355
3356 if (num_vd >= 2)
3357 {
3358 drvInfo->hasBsd2 = 1;
3359 }
3360
3361 if (num_ve >= 1)
3362 {
3363 drvInfo->hasVebox = 1;
3364 }
3365
3366 drvInfo->hasHuc = 1;
3367 if (1 == drvInfo->hasHuc)
3368 {
3369 drvInfo->hasProtectedHuc = 1;
3370 }
3371
3372 // query config
3373 MOS_DRM_CHK_XE_DEV(dev, config, __mos_query_config_xe, -ENODEV)
3374 struct drm_xe_query_config *config = dev->config;
3375 drvInfo->devId = config->info[DRM_XE_QUERY_CONFIG_REV_AND_DEVICE_ID] & 0xffff;
3376 drvInfo->devRev = config->info[DRM_XE_QUERY_CONFIG_REV_AND_DEVICE_ID] >> 16;
3377
3378 return MOS_XE_SUCCESS;
3379 }
3380
3381 static void
3382 __mos_gem_add_bucket_xe(struct mos_xe_bufmgr_gem *bufmgr_gem, int size)
3383 {
3384 mos_xe_bucket_lab *lab = &bufmgr_gem->bucket_lab;
3385 unsigned int i = lab->num_buckets;
3386
3387 if (i < ARRAY_SIZE(lab->cache_buckets))
3388 {
3389 DRMINITLISTHEAD(&lab->cache_buckets[i].sys_head);
3390 DRMINITLISTHEAD(&lab->cache_buckets[i].vram_head);
3391 lab->cache_buckets[i].size = size;
3392 lab->num_buckets++;
3393 }
3394 else
3395 {
3396 MOS_DRM_ASSERTMESSAGE("Unable to add more bucket because of cache bucket full");
3397 }
3398 }
3399
3400 static struct mos_xe_gem_bo_bucket*
3401 __mos_gem_find_bucket_xe(
3402 struct mos_xe_gem_bo_bucket *buckets,
3403 int num_buckets, uint64_t alloc_size)
3404 {
3405 int l = 0, r = num_buckets;
3406 while (l < r)
3407 {
3408 int mid = l + (r - l) / 2;
3409 if (buckets[mid].size < alloc_size)
3410 {
3411 l = mid + 1;
3412 }
3413 else
3414 {
3415 r = mid;
3416 }
3417 }
3418 if (buckets[l].size >= alloc_size)
3419 {
3420 return &buckets[l];
3421 }
3422 else
3423 {
3424 return nullptr;
3425 }
3426 }
3427
3428 static void
3429 __mos_gem_init_cache_buckets(struct mos_xe_bufmgr_gem *bufmgr_gem)
3430 {
3431 unsigned long size, max_cache_size = 64 * 1024 * 1024;
3432 struct mos_xe_bucket_lab *lab = &bufmgr_gem->bucket_lab;
3433 lab->cache_mode = CACHE_BUCKET_MODE_DEFAULT;
3434 lab->max_cache_size = max_cache_size;
3435
3436 /* OK, so power of two buckets was too wasteful of memory.
3437 * Give 3 other sizes between each power of two, to hopefully
3438 * cover things accurately enough. (The alternative is
3439 * probably to just go for exact matching of sizes, and assume
3440 * that for things like composited window resize the tiled
3441 * width/height alignment and rounding of sizes to pages will
3442 * get us useful cache hit rates anyway)
3443 */
3444 __mos_gem_add_bucket_xe(bufmgr_gem, 4096);
3445 __mos_gem_add_bucket_xe(bufmgr_gem, 4096 * 2);
3446 __mos_gem_add_bucket_xe(bufmgr_gem, 4096 * 3);
3447
3448 /* Initialize the linked lists for BO reuse cache. */
3449 for (size = 4 * 4096; size <= max_cache_size; size *= 2)
3450 {
3451 __mos_gem_add_bucket_xe(bufmgr_gem, size);
3452
3453 __mos_gem_add_bucket_xe(bufmgr_gem, size + size * 1 / 4);
3454 __mos_gem_add_bucket_xe(bufmgr_gem, size + size * 2 / 4);
3455 __mos_gem_add_bucket_xe(bufmgr_gem, size + size * 3 / 4);
3456 }
3457 }
3458
3459 static void
3460 __mos_gem_cleanup_cache_bucket_xe(struct mos_xe_bufmgr_gem *bufmgr_gem)
3461 {
3462 struct mos_xe_bucket_lab *lab = &bufmgr_gem->bucket_lab;
3463 for (int i = 0; i < lab->num_buckets; i++) {
3464 struct mos_xe_gem_bo_bucket *bucket =
3465 &lab->cache_buckets[i];
3466
3467 lab->cache_buckets[i].size = 0;
3468 }
3469 lab->num_buckets = 0;
3470 lab->cache_mode = CACHE_BUCKET_MODE_DEFAULT;
3471 lab->max_cache_size = 0;
3472 }
3473
3474 static void
3475 mos_gem_realloc_cache_bucket_xe(struct mos_bufmgr *bufmgr, uint8_t alloc_mode)
3476 {
3477 unsigned long size, max_cache_size = 64 * 1024 * 1024, unit_size;
3478 struct mos_xe_bufmgr_gem *bufmgr_gem = (struct mos_xe_bufmgr_gem *)bufmgr;
3479 struct mos_xe_bucket_lab *lab = &bufmgr_gem->bucket_lab;
3480
3481 __mos_gem_cleanup_cache_bucket_xe(bufmgr_gem);
3482
3483 lab->cache_mode = alloc_mode;
3484 lab->max_cache_size = max_cache_size;
3485
3486
3487 /* OK, so power of two buckets was too wasteful of memory.
3488 * Give 3 other sizes between each power of two, to hopefully
3489 * cover things accurately enough. (The alternative is
3490 * probably to just go for exact matching of sizes, and assume
3491 * that for things like composited window resize the tiled
3492 * width/height alignment and rounding of sizes to pages will
3493 * get us useful cache hit rates anyway)
3494 */
3495 /* alloc_mode 0 is default alloc_mode
3496 * alloc_mode 1 rounding up to 64K for all < 1M
3497 * alloc_mode 2 rounding up to 2M for size> 1M
3498 * alloc_mode 3 rounding up to 2M for size > 1M and 64K for size <= 1M */
3499 if ( alloc_mode > 3 )
3500 alloc_mode = 0;
3501
3502 if ( CACHE_BUCKET_MODE_DEFAULT == alloc_mode
3503 || CACHE_BUCKET_MODE_2M == alloc_mode)
3504 {
3505 // < 1M normal alloc_mode
3506 __mos_gem_add_bucket_xe(bufmgr_gem, 4096);
3507 __mos_gem_add_bucket_xe(bufmgr_gem, 4096 * 2);
3508 __mos_gem_add_bucket_xe(bufmgr_gem, 4096 * 3);
3509 /* Initialize the linked lists for BO reuse cache. */
3510 for (size = 4 * 4096; size < 1024 * 1024; size *= 2)
3511 {
3512 __mos_gem_add_bucket_xe(bufmgr_gem, size);
3513 __mos_gem_add_bucket_xe(bufmgr_gem, size + size * 1 / 4);
3514 __mos_gem_add_bucket_xe(bufmgr_gem, size + size * 2 / 4);
3515 __mos_gem_add_bucket_xe(bufmgr_gem, size + size * 3 / 4);
3516 }
3517
3518 __mos_gem_add_bucket_xe(bufmgr_gem, 1024 * 1024);
3519 }
3520 if (CACHE_BUCKET_MODE_64K == alloc_mode
3521 || CACHE_BUCKET_MODE_64K_2M == alloc_mode)
3522 {
3523 // < 1M 64k alignment
3524 unit_size = 64 * 1024;
3525 for (size = unit_size; size <= 1024 * 1024; size += unit_size)
3526 {
3527 __mos_gem_add_bucket_xe(bufmgr_gem, size);
3528 }
3529 }
3530 if ( CACHE_BUCKET_MODE_DEFAULT == alloc_mode
3531 || CACHE_BUCKET_MODE_64K == alloc_mode)
3532 {
3533 //> 1M is normal alloc_mode
3534 __mos_gem_add_bucket_xe(bufmgr_gem, 1280 * 1024);
3535 __mos_gem_add_bucket_xe(bufmgr_gem, 1536 * 1024);
3536 __mos_gem_add_bucket_xe(bufmgr_gem, 1792 * 1024);
3537
3538 for (size = 2 * 1024 * 1024; size < max_cache_size; size *= 2)
3539 {
3540 __mos_gem_add_bucket_xe(bufmgr_gem, size);
3541 __mos_gem_add_bucket_xe(bufmgr_gem, size + size * 1 / 4);
3542 __mos_gem_add_bucket_xe(bufmgr_gem, size + size * 2 / 4);
3543 __mos_gem_add_bucket_xe(bufmgr_gem, size + size * 3 / 4);
3544 }
3545 }
3546 if ( CACHE_BUCKET_MODE_2M == alloc_mode
3547 || CACHE_BUCKET_MODE_64K_2M == alloc_mode)
3548 {
3549 //> 1M rolling to 2M
3550 unit_size = 2 * 1024 * 1024;
3551 __mos_gem_add_bucket_xe(bufmgr_gem, unit_size);
3552 __mos_gem_add_bucket_xe(bufmgr_gem, 3 * 1024 * 1024);
3553
3554 for (size = 4 * 1024 * 1024; size <= max_cache_size; size += unit_size)
3555 {
3556 __mos_gem_add_bucket_xe(bufmgr_gem, size);
3557 }
3558 }
3559 }
3560
3561 /**
3562 * Initializes the GEM buffer manager, which uses the kernel to allocate, map,
3563 * and manage map buffer objections.
3564 *
3565 * \param fd File descriptor of the opened DRM device.
3566 */
3567 struct mos_bufmgr *
3568 mos_bufmgr_gem_init_xe(int fd, int batch_size)
3569 {
3570 //Note: don't put this field in bufmgr in case of bufmgr inaccessable in some functions
3571 #if (_DEBUG || _RELEASE_INTERNAL)
3572 MOS_READ_ENV_VARIABLE(INTEL_XE_BUFMGR_DEBUG, MOS_USER_FEATURE_VALUE_TYPE_INT64, __xe_bufmgr_debug__);
3573 if (__xe_bufmgr_debug__ < 0)
3574 {
3575 __xe_bufmgr_debug__ = 0;
3576 }
3577 #endif
3578
3579 struct mos_xe_bufmgr_gem *bufmgr_gem;
3580 int ret, tmp;
3581 struct mos_xe_device *dev = nullptr;
3582
3583 pthread_mutex_lock(&bufmgr_list_mutex);
3584
3585 bufmgr_gem = mos_bufmgr_gem_find(fd);
3586 if (bufmgr_gem)
3587 goto exit;
3588
3589 bufmgr_gem = MOS_New(mos_xe_bufmgr_gem);
3590 if (nullptr == bufmgr_gem)
3591 goto exit;
3592
3593 bufmgr_gem->bufmgr = {};
3594 bufmgr_gem->xe_device = {};
3595 dev = &bufmgr_gem->xe_device;
3596
3597 bufmgr_gem->fd = fd;
3598 bufmgr_gem->vm_id = INVALID_VM;
3599 atomic_set(&bufmgr_gem->ref_count, 1);
3600
3601 bufmgr_gem->bufmgr.vm_create = mos_vm_create_xe;
3602 bufmgr_gem->bufmgr.vm_destroy = mos_vm_destroy_xe;
3603 bufmgr_gem->bufmgr.context_create = mos_context_create_xe;
3604 bufmgr_gem->bufmgr.context_create_ext = mos_context_create_ext_xe;
3605 bufmgr_gem->bufmgr.context_create_shared = mos_context_create_shared_xe;
3606 bufmgr_gem->bufmgr.context_destroy = mos_context_destroy_xe;
3607 bufmgr_gem->bufmgr.bo_alloc = mos_bo_alloc_xe;
3608 bufmgr_gem->bufmgr.bo_add_softpin_target = mos_gem_bo_update_exec_list_xe;
3609 bufmgr_gem->bufmgr.bo_clear_relocs = mos_gem_bo_clear_exec_list_xe;
3610 bufmgr_gem->bufmgr.bo_alloc_userptr = mos_bo_alloc_userptr_xe;
3611 bufmgr_gem->bufmgr.bo_alloc_tiled = mos_bo_alloc_tiled_xe;
3612 bufmgr_gem->bufmgr.bo_map = mos_bo_map_xe;
3613 bufmgr_gem->bufmgr.bo_busy = mos_gem_bo_busy_xe;
3614 bufmgr_gem->bufmgr.bo_wait_rendering = mos_gem_bo_wait_rendering_xe;
3615 bufmgr_gem->bufmgr.bo_wait = mos_gem_bo_wait_xe;
3616 bufmgr_gem->bufmgr.bo_map_wc = mos_bo_map_wc_xe;
3617 bufmgr_gem->bufmgr.bo_unmap = mos_bo_unmap_xe;
3618 bufmgr_gem->bufmgr.bo_unmap_wc = mos_bo_unmap_wc_xe;
3619 bufmgr_gem->bufmgr.bo_create_from_prime = mos_bo_create_from_prime_xe;
3620 bufmgr_gem->bufmgr.bo_export_to_prime = mos_bo_export_to_prime_xe;
3621 bufmgr_gem->bufmgr.realloc_cache = mos_gem_realloc_cache_bucket_xe;
3622 bufmgr_gem->bufmgr.get_devid = mos_get_devid_xe;
3623 bufmgr_gem->bufmgr.query_engines_count = mos_query_engines_count_xe;
3624 bufmgr_gem->bufmgr.query_engines = mos_query_engines_xe;
3625 bufmgr_gem->bufmgr.get_engine_class_size = mos_get_engine_class_size_xe;
3626 bufmgr_gem->bufmgr.query_sys_engines = mos_query_sysinfo_xe;
3627 bufmgr_gem->bufmgr.select_fixed_engine = mos_select_fixed_engine_xe;
3628 bufmgr_gem->bufmgr.query_device_blob = mos_query_device_blob_xe;
3629 bufmgr_gem->bufmgr.get_driver_info = mos_get_driver_info_xe;
3630 bufmgr_gem->bufmgr.destroy = mos_bufmgr_gem_unref_xe;
3631 bufmgr_gem->bufmgr.query_hw_ip_version = mos_query_hw_ip_version_xe;
3632 bufmgr_gem->bufmgr.get_platform_information = mos_get_platform_information_xe;
3633 bufmgr_gem->bufmgr.set_platform_information = mos_set_platform_information_xe;
3634 bufmgr_gem->bufmgr.enable_reuse = mos_enable_reuse_xe;
3635 bufmgr_gem->bufmgr.bo_reference = mos_bo_reference_xe;
3636 bufmgr_gem->bufmgr.bo_unreference = mos_bo_unreference_xe;
3637 bufmgr_gem->bufmgr.bo_set_softpin = mos_bo_set_softpin_xe;
3638 bufmgr_gem->bufmgr.enable_softpin = mos_enable_softpin_xe;
3639 bufmgr_gem->bufmgr.get_context_param = mo_get_context_param_xe;
3640 bufmgr_gem->bufmgr.get_reset_stats = mos_get_reset_stats_xe;
3641 bufmgr_gem->bufmgr.bo_get_softpin_targets_info = mos_bo_get_oca_exec_list_info_xe;
3642 bufmgr_gem->bufmgr.has_bsd2= mos_has_bsd2_xe;
3643 bufmgr_gem->bufmgr.set_object_capture = mos_bo_set_object_capture_xe;
3644 bufmgr_gem->bufmgr.set_object_async = mos_bo_set_object_async_xe;
3645 bufmgr_gem->bufmgr.bo_context_exec3 = mos_bo_context_exec_with_sync_xe;
3646
3647 bufmgr_gem->exec_queue_timeslice = EXEC_QUEUE_TIMESLICE_DEFAULT;
3648 MOS_READ_ENV_VARIABLE(INTEL_ENGINE_TIMESLICE, MOS_USER_FEATURE_VALUE_TYPE_INT32, bufmgr_gem->exec_queue_timeslice);
3649 if (bufmgr_gem->exec_queue_timeslice <= 0
3650 || bufmgr_gem->exec_queue_timeslice >= EXEC_QUEUE_TIMESLICE_MAX)
3651 {
3652 bufmgr_gem->exec_queue_timeslice = EXEC_QUEUE_TIMESLICE_DEFAULT;
3653 }
3654
3655 bufmgr_gem->mem_profiler_fd = -1;
3656 bufmgr_gem->mem_profiler_path = getenv("MEDIA_MEMORY_PROFILER_LOG");
3657 if (bufmgr_gem->mem_profiler_path != nullptr)
3658 {
3659 if (strcmp(bufmgr_gem->mem_profiler_path, "/sys/kernel/debug/tracing/trace_marker") == 0)
3660 {
3661 ret = bufmgr_gem->mem_profiler_fd = open(bufmgr_gem->mem_profiler_path, O_WRONLY );
3662 }
3663 else
3664 {
3665 ret = bufmgr_gem->mem_profiler_fd = open(bufmgr_gem->mem_profiler_path, O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR);
3666 }
3667
3668 if ( -1 == ret)
3669 {
3670 MOS_DRM_ASSERTMESSAGE("Failed to open %s: %s", bufmgr_gem->mem_profiler_path, strerror(errno));
3671 }
3672 }
3673
3674 dev->uc_versions[UC_TYPE_GUC_SUBMISSION].uc_type = UC_TYPE_INVALID;
3675 dev->uc_versions[UC_TYPE_HUC].uc_type = UC_TYPE_INVALID;
3676
3677 bufmgr_gem->vm_id = __mos_vm_create_xe(&bufmgr_gem->bufmgr);
3678 __mos_query_mem_regions_instance_mask_xe(&bufmgr_gem->bufmgr);
3679 __mos_has_vram_xe(&bufmgr_gem->bufmgr);
3680 __mos_get_default_alignment_xe(&bufmgr_gem->bufmgr);
3681
3682 DRMLISTADD(&bufmgr_gem->managers, &bufmgr_list);
3683 DRMINITLISTHEAD(&bufmgr_gem->named);
3684
3685 __mos_gem_init_cache_buckets(bufmgr_gem);
3686
3687 mos_vma_heap_init(&bufmgr_gem->vma_heap[MEMZONE_SYS], MEMZONE_SYS_START, MEMZONE_SYS_SIZE);
3688 mos_vma_heap_init(&bufmgr_gem->vma_heap[MEMZONE_DEVICE], MEMZONE_DEVICE_START, MEMZONE_DEVICE_SIZE);
3689 mos_vma_heap_init(&bufmgr_gem->vma_heap[MEMZONE_PRIME], MEMZONE_PRIME_START, MEMZONE_PRIME_SIZE);
3690
3691 exit:
3692 pthread_mutex_unlock(&bufmgr_list_mutex);
3693
3694 return bufmgr_gem != nullptr ? &bufmgr_gem->bufmgr : nullptr;
3695 }
3696
3697 int mos_get_dev_id_xe(int fd, uint32_t *device_id)
3698 {
3699 if (fd < 0 || nullptr == device_id)
3700 {
3701 return -EINVAL;
3702 }
3703 struct drm_xe_query_config *config = __mos_query_config_xe(fd);
3704 MOS_DRM_CHK_NULL_RETURN_VALUE(config, -ENODEV)
3705
3706 *device_id = config->info[DRM_XE_QUERY_CONFIG_REV_AND_DEVICE_ID] & 0xffff;
3707 MOS_XE_SAFE_FREE(config);
3708
3709 return MOS_XE_SUCCESS;
3710 }