| 1 |
/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ |
| 2 |
|
| 3 |
#ifndef __DRM_GPUVM_H__ |
| 4 |
#define __DRM_GPUVM_H__ |
| 5 |
|
| 6 |
/* |
| 7 |
* Copyright (c) 2022 Red Hat. |
| 8 |
* |
| 9 |
* Permission is hereby granted, free of charge, to any person obtaining a |
| 10 |
* copy of this software and associated documentation files (the "Software"), |
| 11 |
* to deal in the Software without restriction, including without limitation |
| 12 |
* the rights to use, copy, modify, merge, publish, distribute, sublicense, |
| 13 |
* and/or sell copies of the Software, and to permit persons to whom the |
| 14 |
* Software is furnished to do so, subject to the following conditions: |
| 15 |
* |
| 16 |
* The above copyright notice and this permission notice shall be included in |
| 17 |
* all copies or substantial portions of the Software. |
| 18 |
* |
| 19 |
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| 20 |
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| 21 |
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
| 22 |
* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR |
| 23 |
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
| 24 |
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
| 25 |
* OTHER DEALINGS IN THE SOFTWARE. |
| 26 |
*/ |
| 27 |
|
| 28 |
#include <linux/dma-resv.h> |
| 29 |
#include <linux/list.h> |
| 30 |
#include <linux/llist.h> |
| 31 |
#include <linux/rbtree.h> |
| 32 |
#include <linux/types.h> |
| 33 |
|
| 34 |
#include <drm/drm_device.h> |
| 35 |
#include <drm/drm_gem.h> |
| 36 |
#include <drm/drm_exec.h> |
| 37 |
|
| 38 |
struct drm_gpuvm; |
| 39 |
struct drm_gpuvm_bo; |
| 40 |
struct drm_gpuvm_ops; |
| 41 |
|
| 42 |
/** |
| 43 |
* enum drm_gpuva_flags - flags for struct drm_gpuva |
| 44 |
*/ |
| 45 |
enum drm_gpuva_flags { |
| 46 |
/** |
| 47 |
* @DRM_GPUVA_INVALIDATED: |
| 48 |
* |
| 49 |
* Flag indicating that the &drm_gpuva's backing GEM is invalidated. |
| 50 |
*/ |
| 51 |
DRM_GPUVA_INVALIDATED = (1 << 0), |
| 52 |
|
| 53 |
/** |
| 54 |
* @DRM_GPUVA_SPARSE: |
| 55 |
* |
| 56 |
* Flag indicating that the &drm_gpuva is a sparse mapping. |
| 57 |
*/ |
| 58 |
DRM_GPUVA_SPARSE = (1 << 1), |
| 59 |
|
| 60 |
/** |
| 61 |
* @DRM_GPUVA_USERBITS: user defined bits |
| 62 |
*/ |
| 63 |
DRM_GPUVA_USERBITS = (1 << 2), |
| 64 |
}; |
| 65 |
|
| 66 |
/** |
| 67 |
* struct drm_gpuva - structure to track a GPU VA mapping |
| 68 |
* |
| 69 |
* This structure represents a GPU VA mapping and is associated with a |
| 70 |
* &drm_gpuvm. |
| 71 |
* |
| 72 |
* Typically, this structure is embedded in bigger driver structures. |
| 73 |
*/ |
| 74 |
struct drm_gpuva { |
| 75 |
/** |
| 76 |
* @vm: the &drm_gpuvm this object is associated with |
| 77 |
*/ |
| 78 |
struct drm_gpuvm *vm; |
| 79 |
|
| 80 |
/** |
| 81 |
* @vm_bo: the &drm_gpuvm_bo abstraction for the mapped |
| 82 |
* &drm_gem_object |
| 83 |
*/ |
| 84 |
struct drm_gpuvm_bo *vm_bo; |
| 85 |
|
| 86 |
/** |
| 87 |
* @flags: the &drm_gpuva_flags for this mapping |
| 88 |
*/ |
| 89 |
enum drm_gpuva_flags flags; |
| 90 |
|
| 91 |
/** |
| 92 |
* @va: structure containing the address and range of the &drm_gpuva |
| 93 |
*/ |
| 94 |
struct { |
| 95 |
/** |
| 96 |
* @va.addr: the start address |
| 97 |
*/ |
| 98 |
u64 addr; |
| 99 |
|
| 100 |
/* |
| 101 |
* @range: the range |
| 102 |
*/ |
| 103 |
u64 range; |
| 104 |
} va; |
| 105 |
|
| 106 |
/** |
| 107 |
* @gem: structure containing the &drm_gem_object and its offset |
| 108 |
*/ |
| 109 |
struct { |
| 110 |
/** |
| 111 |
* @gem.offset: the offset within the &drm_gem_object |
| 112 |
*/ |
| 113 |
u64 offset; |
| 114 |
|
| 115 |
/** |
| 116 |
* @gem.obj: the mapped &drm_gem_object |
| 117 |
*/ |
| 118 |
struct drm_gem_object *obj; |
| 119 |
|
| 120 |
/** |
| 121 |
* @gem.entry: the &list_head to attach this object to a &drm_gpuvm_bo |
| 122 |
*/ |
| 123 |
struct list_head entry; |
| 124 |
} gem; |
| 125 |
|
| 126 |
/** |
| 127 |
* @rb: structure containing data to store &drm_gpuvas in a rb-tree |
| 128 |
*/ |
| 129 |
struct { |
| 130 |
/** |
| 131 |
* @rb.node: the rb-tree node |
| 132 |
*/ |
| 133 |
struct rb_node node; |
| 134 |
|
| 135 |
/** |
| 136 |
* @rb.entry: The &list_head to additionally connect &drm_gpuvas |
| 137 |
* in the same order they appear in the interval tree. This is |
| 138 |
* useful to keep iterating &drm_gpuvas from a start node found |
| 139 |
* through the rb-tree while doing modifications on the rb-tree |
| 140 |
* itself. |
| 141 |
*/ |
| 142 |
struct list_head entry; |
| 143 |
|
| 144 |
/** |
| 145 |
* @rb.__subtree_last: needed by the interval tree, holding last-in-subtree |
| 146 |
*/ |
| 147 |
u64 __subtree_last; |
| 148 |
} rb; |
| 149 |
}; |
| 150 |
|
| 151 |
int drm_gpuva_insert(struct drm_gpuvm *gpuvm, struct drm_gpuva *va); |
| 152 |
void drm_gpuva_remove(struct drm_gpuva *va); |
| 153 |
|
| 154 |
void drm_gpuva_link(struct drm_gpuva *va, struct drm_gpuvm_bo *vm_bo); |
| 155 |
void drm_gpuva_unlink(struct drm_gpuva *va); |
| 156 |
void drm_gpuva_unlink_defer(struct drm_gpuva *va); |
| 157 |
|
| 158 |
struct drm_gpuva *drm_gpuva_find(struct drm_gpuvm *gpuvm, |
| 159 |
u64 addr, u64 range); |
| 160 |
struct drm_gpuva *drm_gpuva_find_first(struct drm_gpuvm *gpuvm, |
| 161 |
u64 addr, u64 range); |
| 162 |
struct drm_gpuva *drm_gpuva_find_prev(struct drm_gpuvm *gpuvm, u64 start); |
| 163 |
struct drm_gpuva *drm_gpuva_find_next(struct drm_gpuvm *gpuvm, u64 end); |
| 164 |
|
| 165 |
/** |
| 166 |
* drm_gpuva_invalidate() - sets whether the backing GEM of this &drm_gpuva is |
| 167 |
* invalidated |
| 168 |
* @va: the &drm_gpuva to set the invalidate flag for |
| 169 |
* @invalidate: indicates whether the &drm_gpuva is invalidated |
| 170 |
*/ |
| 171 |
static inline void drm_gpuva_invalidate(struct drm_gpuva *va, bool invalidate) |
| 172 |
{ |
| 173 |
if (invalidate) |
| 174 |
va->flags |= DRM_GPUVA_INVALIDATED; |
| 175 |
else |
| 176 |
va->flags &= ~DRM_GPUVA_INVALIDATED; |
| 177 |
} |
| 178 |
|
| 179 |
/** |
| 180 |
* drm_gpuva_invalidated() - indicates whether the backing BO of this &drm_gpuva |
| 181 |
* is invalidated |
| 182 |
* @va: the &drm_gpuva to check |
| 183 |
* |
| 184 |
* Returns: %true if the GPU VA is invalidated, %false otherwise |
| 185 |
*/ |
| 186 |
static inline bool drm_gpuva_invalidated(struct drm_gpuva *va) |
| 187 |
{ |
| 188 |
return va->flags & DRM_GPUVA_INVALIDATED; |
| 189 |
} |
| 190 |
|
| 191 |
/** |
| 192 |
* enum drm_gpuvm_flags - flags for struct drm_gpuvm |
| 193 |
*/ |
| 194 |
enum drm_gpuvm_flags { |
| 195 |
/** |
| 196 |
* @DRM_GPUVM_RESV_PROTECTED: GPUVM is protected externally by the |
| 197 |
* GPUVM's &dma_resv lock |
| 198 |
*/ |
| 199 |
DRM_GPUVM_RESV_PROTECTED = BIT(0), |
| 200 |
|
| 201 |
/** |
| 202 |
* @DRM_GPUVM_IMMEDIATE_MODE: use the locking scheme for GEMs designed |
| 203 |
* for modifying the GPUVM during the fence signalling path |
| 204 |
* |
| 205 |
* When set, gpuva.lock is used to protect gpuva.list in all GEM |
| 206 |
* objects associated with this GPUVM. Otherwise, the GEMs dma-resv is |
| 207 |
* used. |
| 208 |
*/ |
| 209 |
DRM_GPUVM_IMMEDIATE_MODE = BIT(1), |
| 210 |
|
| 211 |
/** |
| 212 |
* @DRM_GPUVM_USERBITS: user defined bits |
| 213 |
*/ |
| 214 |
DRM_GPUVM_USERBITS = BIT(2), |
| 215 |
}; |
| 216 |
|
| 217 |
/** |
| 218 |
* struct drm_gpuvm - DRM GPU VA Manager |
| 219 |
* |
| 220 |
* The DRM GPU VA Manager keeps track of a GPU's virtual address space by using |
| 221 |
* &maple_tree structures. Typically, this structure is embedded in bigger |
| 222 |
* driver structures. |
| 223 |
* |
| 224 |
* Drivers can pass addresses and ranges in an arbitrary unit, e.g. bytes or |
| 225 |
* pages. |
| 226 |
* |
| 227 |
* There should be one manager instance per GPU virtual address space. |
| 228 |
*/ |
| 229 |
struct drm_gpuvm { |
| 230 |
/** |
| 231 |
* @name: the name of the DRM GPU VA space |
| 232 |
*/ |
| 233 |
const char *name; |
| 234 |
|
| 235 |
/** |
| 236 |
* @flags: the &drm_gpuvm_flags of this GPUVM |
| 237 |
*/ |
| 238 |
enum drm_gpuvm_flags flags; |
| 239 |
|
| 240 |
/** |
| 241 |
* @drm: the &drm_device this VM lives in |
| 242 |
*/ |
| 243 |
struct drm_device *drm; |
| 244 |
|
| 245 |
/** |
| 246 |
* @mm_start: start of the VA space |
| 247 |
*/ |
| 248 |
u64 mm_start; |
| 249 |
|
| 250 |
/** |
| 251 |
* @mm_range: length of the VA space |
| 252 |
*/ |
| 253 |
u64 mm_range; |
| 254 |
|
| 255 |
/** |
| 256 |
* @rb: structures to track &drm_gpuva entries |
| 257 |
*/ |
| 258 |
struct { |
| 259 |
/** |
| 260 |
* @rb.tree: the rb-tree to track GPU VA mappings |
| 261 |
*/ |
| 262 |
struct rb_root_cached tree; |
| 263 |
|
| 264 |
/** |
| 265 |
* @rb.list: the &list_head to track GPU VA mappings |
| 266 |
*/ |
| 267 |
struct list_head list; |
| 268 |
} rb; |
| 269 |
|
| 270 |
/** |
| 271 |
* @kref: reference count of this object |
| 272 |
*/ |
| 273 |
struct kref kref; |
| 274 |
|
| 275 |
/** |
| 276 |
* @kernel_alloc_node: |
| 277 |
* |
| 278 |
* &drm_gpuva representing the address space cutout reserved for |
| 279 |
* the kernel |
| 280 |
*/ |
| 281 |
struct drm_gpuva kernel_alloc_node; |
| 282 |
|
| 283 |
/** |
| 284 |
* @ops: &drm_gpuvm_ops providing the split/merge steps to drivers |
| 285 |
*/ |
| 286 |
const struct drm_gpuvm_ops *ops; |
| 287 |
|
| 288 |
/** |
| 289 |
* @r_obj: Resv GEM object; representing the GPUVM's common &dma_resv. |
| 290 |
*/ |
| 291 |
struct drm_gem_object *r_obj; |
| 292 |
|
| 293 |
/** |
| 294 |
* @extobj: structure holding the extobj list |
| 295 |
*/ |
| 296 |
struct { |
| 297 |
/** |
| 298 |
* @extobj.list: &list_head storing &drm_gpuvm_bos serving as |
| 299 |
* external object |
| 300 |
*/ |
| 301 |
struct list_head list; |
| 302 |
|
| 303 |
/** |
| 304 |
* @extobj.local_list: pointer to the local list temporarily |
| 305 |
* storing entries from the external object list |
| 306 |
*/ |
| 307 |
struct list_head *local_list; |
| 308 |
|
| 309 |
/** |
| 310 |
* @extobj.lock: spinlock to protect the extobj list |
| 311 |
*/ |
| 312 |
spinlock_t lock; |
| 313 |
} extobj; |
| 314 |
|
| 315 |
/** |
| 316 |
* @evict: structure holding the evict list and evict list lock |
| 317 |
*/ |
| 318 |
struct { |
| 319 |
/** |
| 320 |
* @evict.list: &list_head storing &drm_gpuvm_bos currently |
| 321 |
* being evicted |
| 322 |
*/ |
| 323 |
struct list_head list; |
| 324 |
|
| 325 |
/** |
| 326 |
* @evict.local_list: pointer to the local list temporarily |
| 327 |
* storing entries from the evicted object list |
| 328 |
*/ |
| 329 |
struct list_head *local_list; |
| 330 |
|
| 331 |
/** |
| 332 |
* @evict.lock: spinlock to protect the evict list |
| 333 |
*/ |
| 334 |
spinlock_t lock; |
| 335 |
} evict; |
| 336 |
|
| 337 |
/** |
| 338 |
* @bo_defer: structure holding vm_bos that need to be destroyed |
| 339 |
*/ |
| 340 |
struct llist_head bo_defer; |
| 341 |
}; |
| 342 |
|
| 343 |
void drm_gpuvm_init(struct drm_gpuvm *gpuvm, const char *name, |
| 344 |
enum drm_gpuvm_flags flags, |
| 345 |
struct drm_device *drm, |
| 346 |
struct drm_gem_object *r_obj, |
| 347 |
u64 start_offset, u64 range, |
| 348 |
u64 reserve_offset, u64 reserve_range, |
| 349 |
const struct drm_gpuvm_ops *ops); |
| 350 |
|
| 351 |
/** |
| 352 |
* drm_gpuvm_get() - acquire a struct drm_gpuvm reference |
| 353 |
* @gpuvm: the &drm_gpuvm to acquire the reference of |
| 354 |
* |
| 355 |
* This function acquires an additional reference to @gpuvm. It is illegal to |
| 356 |
* call this without already holding a reference. No locks required. |
| 357 |
* |
| 358 |
* Returns: the &struct drm_gpuvm pointer |
| 359 |
*/ |
| 360 |
static inline struct drm_gpuvm * |
| 361 |
drm_gpuvm_get(struct drm_gpuvm *gpuvm) |
| 362 |
{ |
| 363 |
kref_get(&gpuvm->kref); |
| 364 |
|
| 365 |
return gpuvm; |
| 366 |
} |
| 367 |
|
| 368 |
void drm_gpuvm_put(struct drm_gpuvm *gpuvm); |
| 369 |
|
| 370 |
bool drm_gpuvm_range_valid(struct drm_gpuvm *gpuvm, u64 addr, u64 range); |
| 371 |
bool drm_gpuvm_interval_empty(struct drm_gpuvm *gpuvm, u64 addr, u64 range); |
| 372 |
|
| 373 |
struct drm_gem_object * |
| 374 |
drm_gpuvm_resv_object_alloc(struct drm_device *drm); |
| 375 |
|
| 376 |
/** |
| 377 |
* drm_gpuvm_resv_protected() - indicates whether &DRM_GPUVM_RESV_PROTECTED is |
| 378 |
* set |
| 379 |
* @gpuvm: the &drm_gpuvm |
| 380 |
* |
| 381 |
* Returns: true if &DRM_GPUVM_RESV_PROTECTED is set, false otherwise. |
| 382 |
*/ |
| 383 |
static inline bool |
| 384 |
drm_gpuvm_resv_protected(struct drm_gpuvm *gpuvm) |
| 385 |
{ |
| 386 |
return gpuvm->flags & DRM_GPUVM_RESV_PROTECTED; |
| 387 |
} |
| 388 |
|
| 389 |
/** |
| 390 |
* drm_gpuvm_immediate_mode() - indicates whether &DRM_GPUVM_IMMEDIATE_MODE is |
| 391 |
* set |
| 392 |
* @gpuvm: the &drm_gpuvm |
| 393 |
* |
| 394 |
* Returns: true if &DRM_GPUVM_IMMEDIATE_MODE is set, false otherwise. |
| 395 |
*/ |
| 396 |
static inline bool |
| 397 |
drm_gpuvm_immediate_mode(struct drm_gpuvm *gpuvm) |
| 398 |
{ |
| 399 |
return gpuvm->flags & DRM_GPUVM_IMMEDIATE_MODE; |
| 400 |
} |
| 401 |
|
| 402 |
/** |
| 403 |
* drm_gpuvm_resv() - returns the &drm_gpuvm's &dma_resv |
| 404 |
* @gpuvm__: the &drm_gpuvm |
| 405 |
* |
| 406 |
* Returns: a pointer to the &drm_gpuvm's shared &dma_resv |
| 407 |
*/ |
| 408 |
#define drm_gpuvm_resv(gpuvm__) ((gpuvm__)->r_obj->resv) |
| 409 |
|
| 410 |
/** |
| 411 |
* drm_gpuvm_resv_obj() - returns the &drm_gem_object holding the &drm_gpuvm's |
| 412 |
* &dma_resv |
| 413 |
* @gpuvm__: the &drm_gpuvm |
| 414 |
* |
| 415 |
* Returns: a pointer to the &drm_gem_object holding the &drm_gpuvm's shared |
| 416 |
* &dma_resv |
| 417 |
*/ |
| 418 |
#define drm_gpuvm_resv_obj(gpuvm__) ((gpuvm__)->r_obj) |
| 419 |
|
| 420 |
#define drm_gpuvm_resv_held(gpuvm__) \ |
| 421 |
dma_resv_held(drm_gpuvm_resv(gpuvm__)) |
| 422 |
|
| 423 |
#define drm_gpuvm_resv_assert_held(gpuvm__) \ |
| 424 |
dma_resv_assert_held(drm_gpuvm_resv(gpuvm__)) |
| 425 |
|
| 426 |
#define drm_gpuvm_resv_held(gpuvm__) \ |
| 427 |
dma_resv_held(drm_gpuvm_resv(gpuvm__)) |
| 428 |
|
| 429 |
#define drm_gpuvm_resv_assert_held(gpuvm__) \ |
| 430 |
dma_resv_assert_held(drm_gpuvm_resv(gpuvm__)) |
| 431 |
|
| 432 |
/** |
| 433 |
* drm_gpuvm_is_extobj() - indicates whether the given &drm_gem_object is an |
| 434 |
* external object |
| 435 |
* @gpuvm: the &drm_gpuvm to check |
| 436 |
* @obj: the &drm_gem_object to check |
| 437 |
* |
| 438 |
* Returns: true if the &drm_gem_object &dma_resv differs from the |
| 439 |
* &drm_gpuvms &dma_resv, false otherwise |
| 440 |
*/ |
| 441 |
static inline bool |
| 442 |
drm_gpuvm_is_extobj(struct drm_gpuvm *gpuvm, |
| 443 |
struct drm_gem_object *obj) |
| 444 |
{ |
| 445 |
return obj && obj->resv != drm_gpuvm_resv(gpuvm); |
| 446 |
} |
| 447 |
|
| 448 |
static inline struct drm_gpuva * |
| 449 |
__drm_gpuva_next(struct drm_gpuva *va) |
| 450 |
{ |
| 451 |
if (va && !list_is_last(&va->rb.entry, &va->vm->rb.list)) |
| 452 |
return list_next_entry(va, rb.entry); |
| 453 |
|
| 454 |
return NULL; |
| 455 |
} |
| 456 |
|
| 457 |
/** |
| 458 |
* drm_gpuvm_for_each_va_range() - iterate over a range of &drm_gpuvas |
| 459 |
* @va__: &drm_gpuva structure to assign to in each iteration step |
| 460 |
* @gpuvm__: &drm_gpuvm to walk over |
| 461 |
* @start__: starting offset, the first gpuva will overlap this |
| 462 |
* @end__: ending offset, the last gpuva will start before this (but may |
| 463 |
* overlap) |
| 464 |
* |
| 465 |
* This iterator walks over all &drm_gpuvas in the &drm_gpuvm that lie |
| 466 |
* between @start__ and @end__. It is implemented similarly to list_for_each(), |
| 467 |
* but is using the &drm_gpuvm's internal interval tree to accelerate |
| 468 |
* the search for the starting &drm_gpuva, and hence isn't safe against removal |
| 469 |
* of elements. It assumes that @end__ is within (or is the upper limit of) the |
| 470 |
* &drm_gpuvm. This iterator does not skip over the &drm_gpuvm's |
| 471 |
* @kernel_alloc_node. |
| 472 |
*/ |
| 473 |
#define drm_gpuvm_for_each_va_range(va__, gpuvm__, start__, end__) \ |
| 474 |
for (va__ = drm_gpuva_find_first((gpuvm__), (start__), (end__) - (start__)); \ |
| 475 |
va__ && (va__->va.addr < (end__)); \ |
| 476 |
va__ = __drm_gpuva_next(va__)) |
| 477 |
|
| 478 |
/** |
| 479 |
* drm_gpuvm_for_each_va_range_safe() - safely iterate over a range of |
| 480 |
* &drm_gpuvas |
| 481 |
* @va__: &drm_gpuva to assign to in each iteration step |
| 482 |
* @next__: another &drm_gpuva to use as temporary storage |
| 483 |
* @gpuvm__: &drm_gpuvm to walk over |
| 484 |
* @start__: starting offset, the first gpuva will overlap this |
| 485 |
* @end__: ending offset, the last gpuva will start before this (but may |
| 486 |
* overlap) |
| 487 |
* |
| 488 |
* This iterator walks over all &drm_gpuvas in the &drm_gpuvm that lie |
| 489 |
* between @start__ and @end__. It is implemented similarly to |
| 490 |
* list_for_each_safe(), but is using the &drm_gpuvm's internal interval |
| 491 |
* tree to accelerate the search for the starting &drm_gpuva, and hence is safe |
| 492 |
* against removal of elements. It assumes that @end__ is within (or is the |
| 493 |
* upper limit of) the &drm_gpuvm. This iterator does not skip over the |
| 494 |
* &drm_gpuvm's @kernel_alloc_node. |
| 495 |
*/ |
| 496 |
#define drm_gpuvm_for_each_va_range_safe(va__, next__, gpuvm__, start__, end__) \ |
| 497 |
for (va__ = drm_gpuva_find_first((gpuvm__), (start__), (end__) - (start__)), \ |
| 498 |
next__ = __drm_gpuva_next(va__); \ |
| 499 |
va__ && (va__->va.addr < (end__)); \ |
| 500 |
va__ = next__, next__ = __drm_gpuva_next(va__)) |
| 501 |
|
| 502 |
/** |
| 503 |
* drm_gpuvm_for_each_va() - iterate over all &drm_gpuvas |
| 504 |
* @va__: &drm_gpuva to assign to in each iteration step |
| 505 |
* @gpuvm__: &drm_gpuvm to walk over |
| 506 |
* |
| 507 |
* This iterator walks over all &drm_gpuva structures associated with the given |
| 508 |
* &drm_gpuvm. |
| 509 |
*/ |
| 510 |
#define drm_gpuvm_for_each_va(va__, gpuvm__) \ |
| 511 |
list_for_each_entry(va__, &(gpuvm__)->rb.list, rb.entry) |
| 512 |
|
| 513 |
/** |
| 514 |
* drm_gpuvm_for_each_va_safe() - safely iterate over all &drm_gpuvas |
| 515 |
* @va__: &drm_gpuva to assign to in each iteration step |
| 516 |
* @next__: another &drm_gpuva to use as temporary storage |
| 517 |
* @gpuvm__: &drm_gpuvm to walk over |
| 518 |
* |
| 519 |
* This iterator walks over all &drm_gpuva structures associated with the given |
| 520 |
* &drm_gpuvm. It is implemented with list_for_each_entry_safe(), and |
| 521 |
* hence safe against the removal of elements. |
| 522 |
*/ |
| 523 |
#define drm_gpuvm_for_each_va_safe(va__, next__, gpuvm__) \ |
| 524 |
list_for_each_entry_safe(va__, next__, &(gpuvm__)->rb.list, rb.entry) |
| 525 |
|
| 526 |
/** |
| 527 |
* struct drm_gpuvm_exec - &drm_gpuvm abstraction of &drm_exec |
| 528 |
* |
| 529 |
* This structure should be created on the stack as &drm_exec should be. |
| 530 |
* |
| 531 |
* Optionally, @extra can be set in order to lock additional &drm_gem_objects. |
| 532 |
*/ |
| 533 |
struct drm_gpuvm_exec { |
| 534 |
/** |
| 535 |
* @exec: the &drm_exec structure |
| 536 |
*/ |
| 537 |
struct drm_exec exec; |
| 538 |
|
| 539 |
/** |
| 540 |
* @flags: the flags for the struct drm_exec |
| 541 |
*/ |
| 542 |
u32 flags; |
| 543 |
|
| 544 |
/** |
| 545 |
* @vm: the &drm_gpuvm to lock its DMA reservations |
| 546 |
*/ |
| 547 |
struct drm_gpuvm *vm; |
| 548 |
|
| 549 |
/** |
| 550 |
* @num_fences: the number of fences to reserve for the &dma_resv of the |
| 551 |
* locked &drm_gem_objects |
| 552 |
*/ |
| 553 |
unsigned int num_fences; |
| 554 |
|
| 555 |
/** |
| 556 |
* @extra: Callback and corresponding private data for the driver to |
| 557 |
* lock arbitrary additional &drm_gem_objects. |
| 558 |
*/ |
| 559 |
struct { |
| 560 |
/** |
| 561 |
* @extra.fn: The driver callback to lock additional |
| 562 |
* &drm_gem_objects. |
| 563 |
*/ |
| 564 |
int (*fn)(struct drm_gpuvm_exec *vm_exec); |
| 565 |
|
| 566 |
/** |
| 567 |
* @extra.priv: driver private data for the @fn callback |
| 568 |
*/ |
| 569 |
void *priv; |
| 570 |
} extra; |
| 571 |
}; |
| 572 |
|
| 573 |
int drm_gpuvm_prepare_vm(struct drm_gpuvm *gpuvm, |
| 574 |
struct drm_exec *exec, |
| 575 |
unsigned int num_fences); |
| 576 |
|
| 577 |
int drm_gpuvm_prepare_objects(struct drm_gpuvm *gpuvm, |
| 578 |
struct drm_exec *exec, |
| 579 |
unsigned int num_fences); |
| 580 |
|
| 581 |
int drm_gpuvm_prepare_range(struct drm_gpuvm *gpuvm, |
| 582 |
struct drm_exec *exec, |
| 583 |
u64 addr, u64 range, |
| 584 |
unsigned int num_fences); |
| 585 |
|
| 586 |
int drm_gpuvm_exec_lock(struct drm_gpuvm_exec *vm_exec); |
| 587 |
|
| 588 |
int drm_gpuvm_exec_lock_array(struct drm_gpuvm_exec *vm_exec, |
| 589 |
struct drm_gem_object **objs, |
| 590 |
unsigned int num_objs); |
| 591 |
|
| 592 |
int drm_gpuvm_exec_lock_range(struct drm_gpuvm_exec *vm_exec, |
| 593 |
u64 addr, u64 range); |
| 594 |
|
| 595 |
/** |
| 596 |
* drm_gpuvm_exec_unlock() - lock all dma-resv of all assoiciated BOs |
| 597 |
* @vm_exec: the &drm_gpuvm_exec wrapper |
| 598 |
* |
| 599 |
* Releases all dma-resv locks of all &drm_gem_objects previously acquired |
| 600 |
* through drm_gpuvm_exec_lock() or its variants. |
| 601 |
* |
| 602 |
* Returns: 0 on success, negative error code on failure. |
| 603 |
*/ |
| 604 |
static inline void |
| 605 |
drm_gpuvm_exec_unlock(struct drm_gpuvm_exec *vm_exec) |
| 606 |
{ |
| 607 |
drm_exec_fini(&vm_exec->exec); |
| 608 |
} |
| 609 |
|
| 610 |
int drm_gpuvm_validate(struct drm_gpuvm *gpuvm, struct drm_exec *exec); |
| 611 |
void drm_gpuvm_resv_add_fence(struct drm_gpuvm *gpuvm, |
| 612 |
struct drm_exec *exec, |
| 613 |
struct dma_fence *fence, |
| 614 |
enum dma_resv_usage private_usage, |
| 615 |
enum dma_resv_usage extobj_usage); |
| 616 |
|
| 617 |
/** |
| 618 |
* drm_gpuvm_exec_resv_add_fence() - add fence to private and all extobj |
| 619 |
* @vm_exec: the &drm_gpuvm_exec wrapper |
| 620 |
* @fence: fence to add |
| 621 |
* @private_usage: private dma-resv usage |
| 622 |
* @extobj_usage: extobj dma-resv usage |
| 623 |
* |
| 624 |
* See drm_gpuvm_resv_add_fence(). |
| 625 |
*/ |
| 626 |
static inline void |
| 627 |
drm_gpuvm_exec_resv_add_fence(struct drm_gpuvm_exec *vm_exec, |
| 628 |
struct dma_fence *fence, |
| 629 |
enum dma_resv_usage private_usage, |
| 630 |
enum dma_resv_usage extobj_usage) |
| 631 |
{ |
| 632 |
drm_gpuvm_resv_add_fence(vm_exec->vm, &vm_exec->exec, fence, |
| 633 |
private_usage, extobj_usage); |
| 634 |
} |
| 635 |
|
| 636 |
/** |
| 637 |
* drm_gpuvm_exec_validate() - validate all BOs marked as evicted |
| 638 |
* @vm_exec: the &drm_gpuvm_exec wrapper |
| 639 |
* |
| 640 |
* See drm_gpuvm_validate(). |
| 641 |
* |
| 642 |
* Returns: 0 on success, negative error code on failure. |
| 643 |
*/ |
| 644 |
static inline int |
| 645 |
drm_gpuvm_exec_validate(struct drm_gpuvm_exec *vm_exec) |
| 646 |
{ |
| 647 |
return drm_gpuvm_validate(vm_exec->vm, &vm_exec->exec); |
| 648 |
} |
| 649 |
|
| 650 |
/** |
| 651 |
* struct drm_gpuvm_bo - structure representing a &drm_gpuvm and |
| 652 |
* &drm_gem_object combination |
| 653 |
* |
| 654 |
* This structure is an abstraction representing a &drm_gpuvm and |
| 655 |
* &drm_gem_object combination. It serves as an indirection to accelerate |
| 656 |
* iterating all &drm_gpuvas within a &drm_gpuvm backed by the same |
| 657 |
* &drm_gem_object. |
| 658 |
* |
| 659 |
* Furthermore it is used cache evicted GEM objects for a certain GPU-VM to |
| 660 |
* accelerate validation. |
| 661 |
* |
| 662 |
* Typically, drivers want to create an instance of a struct drm_gpuvm_bo once |
| 663 |
* a GEM object is mapped first in a GPU-VM and release the instance once the |
| 664 |
* last mapping of the GEM object in this GPU-VM is unmapped. |
| 665 |
*/ |
| 666 |
struct drm_gpuvm_bo { |
| 667 |
/** |
| 668 |
* @vm: The &drm_gpuvm the @obj is mapped in. This is a reference |
| 669 |
* counted pointer. |
| 670 |
*/ |
| 671 |
struct drm_gpuvm *vm; |
| 672 |
|
| 673 |
/** |
| 674 |
* @obj: The &drm_gem_object being mapped in @vm. This is a reference |
| 675 |
* counted pointer. |
| 676 |
*/ |
| 677 |
struct drm_gem_object *obj; |
| 678 |
|
| 679 |
/** |
| 680 |
* @evicted: Indicates whether the &drm_gem_object is evicted; field |
| 681 |
* protected by the &drm_gem_object's dma-resv lock. |
| 682 |
*/ |
| 683 |
bool evicted; |
| 684 |
|
| 685 |
/** |
| 686 |
* @kref: The reference count for this &drm_gpuvm_bo. |
| 687 |
*/ |
| 688 |
struct kref kref; |
| 689 |
|
| 690 |
/** |
| 691 |
* @list: Structure containing all &list_heads. |
| 692 |
*/ |
| 693 |
struct { |
| 694 |
/** |
| 695 |
* @list.gpuva: The list of linked &drm_gpuvas. |
| 696 |
* |
| 697 |
* It is safe to access entries from this list as long as the |
| 698 |
* GEM's gpuva lock is held. See also struct drm_gem_object. |
| 699 |
*/ |
| 700 |
struct list_head gpuva; |
| 701 |
|
| 702 |
/** |
| 703 |
* @list.entry: Structure containing all &list_heads serving as |
| 704 |
* entry. |
| 705 |
*/ |
| 706 |
struct { |
| 707 |
/** |
| 708 |
* @list.entry.gem: List entry to attach to the |
| 709 |
* &drm_gem_objects gpuva list. |
| 710 |
*/ |
| 711 |
struct list_head gem; |
| 712 |
|
| 713 |
/** |
| 714 |
* @list.entry.evict: List entry to attach to the |
| 715 |
* &drm_gpuvms extobj list. |
| 716 |
*/ |
| 717 |
struct list_head extobj; |
| 718 |
|
| 719 |
/** |
| 720 |
* @list.entry.evict: List entry to attach to the |
| 721 |
* &drm_gpuvms evict list. |
| 722 |
*/ |
| 723 |
struct list_head evict; |
| 724 |
|
| 725 |
/** |
| 726 |
* @list.entry.bo_defer: List entry to attach to |
| 727 |
* the &drm_gpuvms bo_defer list. |
| 728 |
*/ |
| 729 |
struct llist_node bo_defer; |
| 730 |
} entry; |
| 731 |
} list; |
| 732 |
}; |
| 733 |
|
| 734 |
struct drm_gpuvm_bo * |
| 735 |
drm_gpuvm_bo_create(struct drm_gpuvm *gpuvm, |
| 736 |
struct drm_gem_object *obj); |
| 737 |
|
| 738 |
struct drm_gpuvm_bo * |
| 739 |
drm_gpuvm_bo_obtain_locked(struct drm_gpuvm *gpuvm, |
| 740 |
struct drm_gem_object *obj); |
| 741 |
struct drm_gpuvm_bo * |
| 742 |
drm_gpuvm_bo_obtain_prealloc(struct drm_gpuvm_bo *vm_bo); |
| 743 |
|
| 744 |
/** |
| 745 |
* drm_gpuvm_bo_get() - acquire a struct drm_gpuvm_bo reference |
| 746 |
* @vm_bo: the &drm_gpuvm_bo to acquire the reference of |
| 747 |
* |
| 748 |
* This function acquires an additional reference to @vm_bo. It is illegal to |
| 749 |
* call this without already holding a reference. No locks required. |
| 750 |
* |
| 751 |
* Returns: the &struct vm_bo pointer |
| 752 |
*/ |
| 753 |
static inline struct drm_gpuvm_bo * |
| 754 |
drm_gpuvm_bo_get(struct drm_gpuvm_bo *vm_bo) |
| 755 |
{ |
| 756 |
kref_get(&vm_bo->kref); |
| 757 |
return vm_bo; |
| 758 |
} |
| 759 |
|
| 760 |
bool drm_gpuvm_bo_put(struct drm_gpuvm_bo *vm_bo); |
| 761 |
|
| 762 |
bool drm_gpuvm_bo_put_deferred(struct drm_gpuvm_bo *vm_bo); |
| 763 |
void drm_gpuvm_bo_deferred_cleanup(struct drm_gpuvm *gpuvm); |
| 764 |
|
| 765 |
struct drm_gpuvm_bo * |
| 766 |
drm_gpuvm_bo_find(struct drm_gpuvm *gpuvm, |
| 767 |
struct drm_gem_object *obj); |
| 768 |
|
| 769 |
void drm_gpuvm_bo_evict(struct drm_gpuvm_bo *vm_bo, bool evict); |
| 770 |
|
| 771 |
/** |
| 772 |
* drm_gpuvm_bo_gem_evict() - add/remove all &drm_gpuvm_bo's in the list |
| 773 |
* to/from the &drm_gpuvms evicted list |
| 774 |
* @obj: the &drm_gem_object |
| 775 |
* @evict: indicates whether @obj is evicted |
| 776 |
* |
| 777 |
* See drm_gpuvm_bo_evict(). |
| 778 |
*/ |
| 779 |
static inline void |
| 780 |
drm_gpuvm_bo_gem_evict(struct drm_gem_object *obj, bool evict) |
| 781 |
{ |
| 782 |
struct drm_gpuvm_bo *vm_bo; |
| 783 |
|
| 784 |
drm_gem_for_each_gpuvm_bo(vm_bo, obj) { |
| 785 |
drm_gem_gpuva_assert_lock_held(vm_bo->vm, obj); |
| 786 |
drm_gpuvm_bo_evict(vm_bo, evict); |
| 787 |
} |
| 788 |
} |
| 789 |
|
| 790 |
void drm_gpuvm_bo_extobj_add(struct drm_gpuvm_bo *vm_bo); |
| 791 |
|
| 792 |
/** |
| 793 |
* drm_gpuvm_bo_for_each_va() - iterator to walk over a list of &drm_gpuva |
| 794 |
* @va__: &drm_gpuva structure to assign to in each iteration step |
| 795 |
* @vm_bo__: the &drm_gpuvm_bo the &drm_gpuva to walk are associated with |
| 796 |
* |
| 797 |
* This iterator walks over all &drm_gpuva structures associated with the |
| 798 |
* &drm_gpuvm_bo. |
| 799 |
* |
| 800 |
* The caller must hold the GEM's gpuva lock. |
| 801 |
*/ |
| 802 |
#define drm_gpuvm_bo_for_each_va(va__, vm_bo__) \ |
| 803 |
list_for_each_entry(va__, &(vm_bo)->list.gpuva, gem.entry) |
| 804 |
|
| 805 |
/** |
| 806 |
* drm_gpuvm_bo_for_each_va_safe() - iterator to safely walk over a list of |
| 807 |
* &drm_gpuva |
| 808 |
* @va__: &drm_gpuva structure to assign to in each iteration step |
| 809 |
* @next__: &next &drm_gpuva to store the next step |
| 810 |
* @vm_bo__: the &drm_gpuvm_bo the &drm_gpuva to walk are associated with |
| 811 |
* |
| 812 |
* This iterator walks over all &drm_gpuva structures associated with the |
| 813 |
* &drm_gpuvm_bo. It is implemented with list_for_each_entry_safe(), hence |
| 814 |
* it is save against removal of elements. |
| 815 |
* |
| 816 |
* The caller must hold the GEM's gpuva lock. |
| 817 |
*/ |
| 818 |
#define drm_gpuvm_bo_for_each_va_safe(va__, next__, vm_bo__) \ |
| 819 |
list_for_each_entry_safe(va__, next__, &(vm_bo)->list.gpuva, gem.entry) |
| 820 |
|
| 821 |
/** |
| 822 |
* enum drm_gpuva_op_type - GPU VA operation type |
| 823 |
* |
| 824 |
* Operations to alter the GPU VA mappings tracked by the &drm_gpuvm. |
| 825 |
*/ |
| 826 |
enum drm_gpuva_op_type { |
| 827 |
/** |
| 828 |
* @DRM_GPUVA_OP_MAP: the map op type |
| 829 |
*/ |
| 830 |
DRM_GPUVA_OP_MAP, |
| 831 |
|
| 832 |
/** |
| 833 |
* @DRM_GPUVA_OP_REMAP: the remap op type |
| 834 |
*/ |
| 835 |
DRM_GPUVA_OP_REMAP, |
| 836 |
|
| 837 |
/** |
| 838 |
* @DRM_GPUVA_OP_UNMAP: the unmap op type |
| 839 |
*/ |
| 840 |
DRM_GPUVA_OP_UNMAP, |
| 841 |
|
| 842 |
/** |
| 843 |
* @DRM_GPUVA_OP_PREFETCH: the prefetch op type |
| 844 |
*/ |
| 845 |
DRM_GPUVA_OP_PREFETCH, |
| 846 |
|
| 847 |
/** |
| 848 |
* @DRM_GPUVA_OP_DRIVER: the driver defined op type |
| 849 |
*/ |
| 850 |
DRM_GPUVA_OP_DRIVER, |
| 851 |
}; |
| 852 |
|
| 853 |
/** |
| 854 |
* struct drm_gpuva_op_map - GPU VA map operation |
| 855 |
* |
| 856 |
* This structure represents a single map operation generated by the |
| 857 |
* DRM GPU VA manager. |
| 858 |
*/ |
| 859 |
struct drm_gpuva_op_map { |
| 860 |
/** |
| 861 |
* @va: structure containing address and range of a map |
| 862 |
* operation |
| 863 |
*/ |
| 864 |
struct { |
| 865 |
/** |
| 866 |
* @va.addr: the base address of the new mapping |
| 867 |
*/ |
| 868 |
u64 addr; |
| 869 |
|
| 870 |
/** |
| 871 |
* @va.range: the range of the new mapping |
| 872 |
*/ |
| 873 |
u64 range; |
| 874 |
} va; |
| 875 |
|
| 876 |
/** |
| 877 |
* @gem: structure containing the &drm_gem_object and its offset |
| 878 |
*/ |
| 879 |
struct { |
| 880 |
/** |
| 881 |
* @gem.offset: the offset within the &drm_gem_object |
| 882 |
*/ |
| 883 |
u64 offset; |
| 884 |
|
| 885 |
/** |
| 886 |
* @gem.obj: the &drm_gem_object to map |
| 887 |
*/ |
| 888 |
struct drm_gem_object *obj; |
| 889 |
} gem; |
| 890 |
}; |
| 891 |
|
| 892 |
/** |
| 893 |
* struct drm_gpuva_op_unmap - GPU VA unmap operation |
| 894 |
* |
| 895 |
* This structure represents a single unmap operation generated by the |
| 896 |
* DRM GPU VA manager. |
| 897 |
*/ |
| 898 |
struct drm_gpuva_op_unmap { |
| 899 |
/** |
| 900 |
* @va: the &drm_gpuva to unmap |
| 901 |
*/ |
| 902 |
struct drm_gpuva *va; |
| 903 |
|
| 904 |
/** |
| 905 |
* @keep: |
| 906 |
* |
| 907 |
* Indicates whether this &drm_gpuva is physically contiguous with the |
| 908 |
* original mapping request. |
| 909 |
* |
| 910 |
* Optionally, if &keep is set, drivers may keep the actual page table |
| 911 |
* mappings for this &drm_gpuva, adding the missing page table entries |
| 912 |
* only and update the &drm_gpuvm accordingly. |
| 913 |
*/ |
| 914 |
bool keep; |
| 915 |
}; |
| 916 |
|
| 917 |
/** |
| 918 |
* struct drm_gpuva_op_remap - GPU VA remap operation |
| 919 |
* |
| 920 |
* This represents a single remap operation generated by the DRM GPU VA manager. |
| 921 |
* |
| 922 |
* A remap operation is generated when an existing GPU VA mmapping is split up |
| 923 |
* by inserting a new GPU VA mapping or by partially unmapping existent |
| 924 |
* mapping(s), hence it consists of a maximum of two map and one unmap |
| 925 |
* operation. |
| 926 |
* |
| 927 |
* The @unmap operation takes care of removing the original existing mapping. |
| 928 |
* @prev is used to remap the preceding part, @next the subsequent part. |
| 929 |
* |
| 930 |
* If either a new mapping's start address is aligned with the start address |
| 931 |
* of the old mapping or the new mapping's end address is aligned with the |
| 932 |
* end address of the old mapping, either @prev or @next is NULL. |
| 933 |
* |
| 934 |
* Note, the reason for a dedicated remap operation, rather than arbitrary |
| 935 |
* unmap and map operations, is to give drivers the chance of extracting driver |
| 936 |
* specific data for creating the new mappings from the unmap operations's |
| 937 |
* &drm_gpuva structure which typically is embedded in larger driver specific |
| 938 |
* structures. |
| 939 |
*/ |
| 940 |
struct drm_gpuva_op_remap { |
| 941 |
/** |
| 942 |
* @prev: the preceding part of a split mapping |
| 943 |
*/ |
| 944 |
struct drm_gpuva_op_map *prev; |
| 945 |
|
| 946 |
/** |
| 947 |
* @next: the subsequent part of a split mapping |
| 948 |
*/ |
| 949 |
struct drm_gpuva_op_map *next; |
| 950 |
|
| 951 |
/** |
| 952 |
* @unmap: the unmap operation for the original existing mapping |
| 953 |
*/ |
| 954 |
struct drm_gpuva_op_unmap *unmap; |
| 955 |
}; |
| 956 |
|
| 957 |
/** |
| 958 |
* struct drm_gpuva_op_prefetch - GPU VA prefetch operation |
| 959 |
* |
| 960 |
* This structure represents a single prefetch operation generated by the |
| 961 |
* DRM GPU VA manager. |
| 962 |
*/ |
| 963 |
struct drm_gpuva_op_prefetch { |
| 964 |
/** |
| 965 |
* @va: the &drm_gpuva to prefetch |
| 966 |
*/ |
| 967 |
struct drm_gpuva *va; |
| 968 |
}; |
| 969 |
|
| 970 |
/** |
| 971 |
* struct drm_gpuva_op - GPU VA operation |
| 972 |
* |
| 973 |
* This structure represents a single generic operation. |
| 974 |
* |
| 975 |
* The particular type of the operation is defined by @op. |
| 976 |
*/ |
| 977 |
struct drm_gpuva_op { |
| 978 |
/** |
| 979 |
* @entry: |
| 980 |
* |
| 981 |
* The &list_head used to distribute instances of this struct within |
| 982 |
* &drm_gpuva_ops. |
| 983 |
*/ |
| 984 |
struct list_head entry; |
| 985 |
|
| 986 |
/** |
| 987 |
* @op: the type of the operation |
| 988 |
*/ |
| 989 |
enum drm_gpuva_op_type op; |
| 990 |
|
| 991 |
union { |
| 992 |
/** |
| 993 |
* @map: the map operation |
| 994 |
*/ |
| 995 |
struct drm_gpuva_op_map map; |
| 996 |
|
| 997 |
/** |
| 998 |
* @remap: the remap operation |
| 999 |
*/ |
| 1000 |
struct drm_gpuva_op_remap remap; |
| 1001 |
|
| 1002 |
/** |
| 1003 |
* @unmap: the unmap operation |
| 1004 |
*/ |
| 1005 |
struct drm_gpuva_op_unmap unmap; |
| 1006 |
|
| 1007 |
/** |
| 1008 |
* @prefetch: the prefetch operation |
| 1009 |
*/ |
| 1010 |
struct drm_gpuva_op_prefetch prefetch; |
| 1011 |
}; |
| 1012 |
}; |
| 1013 |
|
| 1014 |
/** |
| 1015 |
* struct drm_gpuva_ops - wraps a list of &drm_gpuva_op |
| 1016 |
*/ |
| 1017 |
struct drm_gpuva_ops { |
| 1018 |
/** |
| 1019 |
* @list: the &list_head |
| 1020 |
*/ |
| 1021 |
struct list_head list; |
| 1022 |
}; |
| 1023 |
|
| 1024 |
/** |
| 1025 |
* drm_gpuva_for_each_op() - iterator to walk over &drm_gpuva_ops |
| 1026 |
* @op: &drm_gpuva_op to assign in each iteration step |
| 1027 |
* @ops: &drm_gpuva_ops to walk |
| 1028 |
* |
| 1029 |
* This iterator walks over all ops within a given list of operations. |
| 1030 |
*/ |
| 1031 |
#define drm_gpuva_for_each_op(op, ops) list_for_each_entry(op, &(ops)->list, entry) |
| 1032 |
|
| 1033 |
/** |
| 1034 |
* drm_gpuva_for_each_op_safe() - iterator to safely walk over &drm_gpuva_ops |
| 1035 |
* @op: &drm_gpuva_op to assign in each iteration step |
| 1036 |
* @next: &next &drm_gpuva_op to store the next step |
| 1037 |
* @ops: &drm_gpuva_ops to walk |
| 1038 |
* |
| 1039 |
* This iterator walks over all ops within a given list of operations. It is |
| 1040 |
* implemented with list_for_each_safe(), so save against removal of elements. |
| 1041 |
*/ |
| 1042 |
#define drm_gpuva_for_each_op_safe(op, next, ops) \ |
| 1043 |
list_for_each_entry_safe(op, next, &(ops)->list, entry) |
| 1044 |
|
| 1045 |
/** |
| 1046 |
* drm_gpuva_for_each_op_from_reverse() - iterate backwards from the given point |
| 1047 |
* @op: &drm_gpuva_op to assign in each iteration step |
| 1048 |
* @ops: &drm_gpuva_ops to walk |
| 1049 |
* |
| 1050 |
* This iterator walks over all ops within a given list of operations beginning |
| 1051 |
* from the given operation in reverse order. |
| 1052 |
*/ |
| 1053 |
#define drm_gpuva_for_each_op_from_reverse(op, ops) \ |
| 1054 |
list_for_each_entry_from_reverse(op, &(ops)->list, entry) |
| 1055 |
|
| 1056 |
/** |
| 1057 |
* drm_gpuva_for_each_op_reverse - iterator to walk over &drm_gpuva_ops in reverse |
| 1058 |
* @op: &drm_gpuva_op to assign in each iteration step |
| 1059 |
* @ops: &drm_gpuva_ops to walk |
| 1060 |
* |
| 1061 |
* This iterator walks over all ops within a given list of operations in reverse |
| 1062 |
*/ |
| 1063 |
#define drm_gpuva_for_each_op_reverse(op, ops) \ |
| 1064 |
list_for_each_entry_reverse(op, &(ops)->list, entry) |
| 1065 |
|
| 1066 |
/** |
| 1067 |
* drm_gpuva_first_op() - returns the first &drm_gpuva_op from &drm_gpuva_ops |
| 1068 |
* @ops: the &drm_gpuva_ops to get the fist &drm_gpuva_op from |
| 1069 |
*/ |
| 1070 |
#define drm_gpuva_first_op(ops) \ |
| 1071 |
list_first_entry(&(ops)->list, struct drm_gpuva_op, entry) |
| 1072 |
|
| 1073 |
/** |
| 1074 |
* drm_gpuva_last_op() - returns the last &drm_gpuva_op from &drm_gpuva_ops |
| 1075 |
* @ops: the &drm_gpuva_ops to get the last &drm_gpuva_op from |
| 1076 |
*/ |
| 1077 |
#define drm_gpuva_last_op(ops) \ |
| 1078 |
list_last_entry(&(ops)->list, struct drm_gpuva_op, entry) |
| 1079 |
|
| 1080 |
/** |
| 1081 |
* drm_gpuva_prev_op() - previous &drm_gpuva_op in the list |
| 1082 |
* @op: the current &drm_gpuva_op |
| 1083 |
*/ |
| 1084 |
#define drm_gpuva_prev_op(op) list_prev_entry(op, entry) |
| 1085 |
|
| 1086 |
/** |
| 1087 |
* drm_gpuva_next_op() - next &drm_gpuva_op in the list |
| 1088 |
* @op: the current &drm_gpuva_op |
| 1089 |
*/ |
| 1090 |
#define drm_gpuva_next_op(op) list_next_entry(op, entry) |
| 1091 |
|
| 1092 |
/** |
| 1093 |
* struct drm_gpuvm_map_req - arguments passed to drm_gpuvm_sm_map[_ops_create]() |
| 1094 |
*/ |
| 1095 |
struct drm_gpuvm_map_req { |
| 1096 |
/** |
| 1097 |
* @map: struct drm_gpuva_op_map |
| 1098 |
*/ |
| 1099 |
struct drm_gpuva_op_map map; |
| 1100 |
}; |
| 1101 |
|
| 1102 |
struct drm_gpuva_ops * |
| 1103 |
drm_gpuvm_sm_map_ops_create(struct drm_gpuvm *gpuvm, |
| 1104 |
const struct drm_gpuvm_map_req *req); |
| 1105 |
struct drm_gpuva_ops * |
| 1106 |
drm_gpuvm_madvise_ops_create(struct drm_gpuvm *gpuvm, |
| 1107 |
const struct drm_gpuvm_map_req *req); |
| 1108 |
|
| 1109 |
struct drm_gpuva_ops * |
| 1110 |
drm_gpuvm_sm_unmap_ops_create(struct drm_gpuvm *gpuvm, |
| 1111 |
u64 addr, u64 range); |
| 1112 |
|
| 1113 |
struct drm_gpuva_ops * |
| 1114 |
drm_gpuvm_prefetch_ops_create(struct drm_gpuvm *gpuvm, |
| 1115 |
u64 addr, u64 range); |
| 1116 |
|
| 1117 |
struct drm_gpuva_ops * |
| 1118 |
drm_gpuvm_bo_unmap_ops_create(struct drm_gpuvm_bo *vm_bo); |
| 1119 |
|
| 1120 |
void drm_gpuva_ops_free(struct drm_gpuvm *gpuvm, |
| 1121 |
struct drm_gpuva_ops *ops); |
| 1122 |
|
| 1123 |
static inline void drm_gpuva_init_from_op(struct drm_gpuva *va, |
| 1124 |
const struct drm_gpuva_op_map *op) |
| 1125 |
{ |
| 1126 |
va->va.addr = op->va.addr; |
| 1127 |
va->va.range = op->va.range; |
| 1128 |
va->gem.obj = op->gem.obj; |
| 1129 |
va->gem.offset = op->gem.offset; |
| 1130 |
} |
| 1131 |
|
| 1132 |
/** |
| 1133 |
* struct drm_gpuvm_ops - callbacks for split/merge steps |
| 1134 |
* |
| 1135 |
* This structure defines the callbacks used by &drm_gpuvm_sm_map and |
| 1136 |
* &drm_gpuvm_sm_unmap to provide the split/merge steps for map and unmap |
| 1137 |
* operations to drivers. |
| 1138 |
*/ |
| 1139 |
struct drm_gpuvm_ops { |
| 1140 |
/** |
| 1141 |
* @vm_free: called when the last reference of a struct drm_gpuvm is |
| 1142 |
* dropped |
| 1143 |
* |
| 1144 |
* This callback is mandatory. |
| 1145 |
*/ |
| 1146 |
void (*vm_free)(struct drm_gpuvm *gpuvm); |
| 1147 |
|
| 1148 |
/** |
| 1149 |
* @op_alloc: called when the &drm_gpuvm allocates |
| 1150 |
* a struct drm_gpuva_op |
| 1151 |
* |
| 1152 |
* Some drivers may want to embed struct drm_gpuva_op into driver |
| 1153 |
* specific structures. By implementing this callback drivers can |
| 1154 |
* allocate memory accordingly. |
| 1155 |
* |
| 1156 |
* This callback is optional. |
| 1157 |
*/ |
| 1158 |
struct drm_gpuva_op *(*op_alloc)(void); |
| 1159 |
|
| 1160 |
/** |
| 1161 |
* @op_free: called when the &drm_gpuvm frees a |
| 1162 |
* struct drm_gpuva_op |
| 1163 |
* |
| 1164 |
* Some drivers may want to embed struct drm_gpuva_op into driver |
| 1165 |
* specific structures. By implementing this callback drivers can |
| 1166 |
* free the previously allocated memory accordingly. |
| 1167 |
* |
| 1168 |
* This callback is optional. |
| 1169 |
*/ |
| 1170 |
void (*op_free)(struct drm_gpuva_op *op); |
| 1171 |
|
| 1172 |
/** |
| 1173 |
* @vm_bo_alloc: called when the &drm_gpuvm allocates |
| 1174 |
* a struct drm_gpuvm_bo |
| 1175 |
* |
| 1176 |
* Some drivers may want to embed struct drm_gpuvm_bo into driver |
| 1177 |
* specific structures. By implementing this callback drivers can |
| 1178 |
* allocate memory accordingly. |
| 1179 |
* |
| 1180 |
* This callback is optional. |
| 1181 |
*/ |
| 1182 |
struct drm_gpuvm_bo *(*vm_bo_alloc)(void); |
| 1183 |
|
| 1184 |
/** |
| 1185 |
* @vm_bo_free: called when the &drm_gpuvm frees a |
| 1186 |
* struct drm_gpuvm_bo |
| 1187 |
* |
| 1188 |
* Some drivers may want to embed struct drm_gpuvm_bo into driver |
| 1189 |
* specific structures. By implementing this callback drivers can |
| 1190 |
* free the previously allocated memory accordingly. |
| 1191 |
* |
| 1192 |
* This callback is optional. |
| 1193 |
*/ |
| 1194 |
void (*vm_bo_free)(struct drm_gpuvm_bo *vm_bo); |
| 1195 |
|
| 1196 |
/** |
| 1197 |
* @vm_bo_validate: called from drm_gpuvm_validate() |
| 1198 |
* |
| 1199 |
* Drivers receive this callback for every evicted &drm_gem_object being |
| 1200 |
* mapped in the corresponding &drm_gpuvm. |
| 1201 |
* |
| 1202 |
* Typically, drivers would call their driver specific variant of |
| 1203 |
* ttm_bo_validate() from within this callback. |
| 1204 |
*/ |
| 1205 |
int (*vm_bo_validate)(struct drm_gpuvm_bo *vm_bo, |
| 1206 |
struct drm_exec *exec); |
| 1207 |
|
| 1208 |
/** |
| 1209 |
* @sm_step_map: called from &drm_gpuvm_sm_map to finally insert the |
| 1210 |
* mapping once all previous steps were completed |
| 1211 |
* |
| 1212 |
* The &priv pointer matches the one the driver passed to |
| 1213 |
* &drm_gpuvm_sm_map or &drm_gpuvm_sm_unmap, respectively. |
| 1214 |
* |
| 1215 |
* Can be NULL if &drm_gpuvm_sm_map is used. |
| 1216 |
*/ |
| 1217 |
int (*sm_step_map)(struct drm_gpuva_op *op, void *priv); |
| 1218 |
|
| 1219 |
/** |
| 1220 |
* @sm_step_remap: called from &drm_gpuvm_sm_map and |
| 1221 |
* &drm_gpuvm_sm_unmap to split up an existent mapping |
| 1222 |
* |
| 1223 |
* This callback is called when existent mapping needs to be split up. |
| 1224 |
* This is the case when either a newly requested mapping overlaps or |
| 1225 |
* is enclosed by an existent mapping or a partial unmap of an existent |
| 1226 |
* mapping is requested. |
| 1227 |
* |
| 1228 |
* The &priv pointer matches the one the driver passed to |
| 1229 |
* &drm_gpuvm_sm_map or &drm_gpuvm_sm_unmap, respectively. |
| 1230 |
* |
| 1231 |
* Can be NULL if neither &drm_gpuvm_sm_map nor &drm_gpuvm_sm_unmap is |
| 1232 |
* used. |
| 1233 |
*/ |
| 1234 |
int (*sm_step_remap)(struct drm_gpuva_op *op, void *priv); |
| 1235 |
|
| 1236 |
/** |
| 1237 |
* @sm_step_unmap: called from &drm_gpuvm_sm_map and |
| 1238 |
* &drm_gpuvm_sm_unmap to unmap an existing mapping |
| 1239 |
* |
| 1240 |
* This callback is called when existing mapping needs to be unmapped. |
| 1241 |
* This is the case when either a newly requested mapping encloses an |
| 1242 |
* existing mapping or an unmap of an existing mapping is requested. |
| 1243 |
* |
| 1244 |
* The &priv pointer matches the one the driver passed to |
| 1245 |
* &drm_gpuvm_sm_map or &drm_gpuvm_sm_unmap, respectively. |
| 1246 |
* |
| 1247 |
* Can be NULL if neither &drm_gpuvm_sm_map nor &drm_gpuvm_sm_unmap is |
| 1248 |
* used. |
| 1249 |
*/ |
| 1250 |
int (*sm_step_unmap)(struct drm_gpuva_op *op, void *priv); |
| 1251 |
}; |
| 1252 |
|
| 1253 |
int drm_gpuvm_sm_map(struct drm_gpuvm *gpuvm, void *priv, |
| 1254 |
const struct drm_gpuvm_map_req *req); |
| 1255 |
|
| 1256 |
int drm_gpuvm_sm_unmap(struct drm_gpuvm *gpuvm, void *priv, |
| 1257 |
u64 addr, u64 range); |
| 1258 |
|
| 1259 |
int drm_gpuvm_sm_map_exec_lock(struct drm_gpuvm *gpuvm, |
| 1260 |
struct drm_exec *exec, unsigned int num_fences, |
| 1261 |
struct drm_gpuvm_map_req *req); |
| 1262 |
|
| 1263 |
int drm_gpuvm_sm_unmap_exec_lock(struct drm_gpuvm *gpuvm, struct drm_exec *exec, |
| 1264 |
u64 req_addr, u64 req_range); |
| 1265 |
|
| 1266 |
void drm_gpuva_map(struct drm_gpuvm *gpuvm, |
| 1267 |
struct drm_gpuva *va, |
| 1268 |
const struct drm_gpuva_op_map *op); |
| 1269 |
|
| 1270 |
void drm_gpuva_remap(struct drm_gpuva *prev, |
| 1271 |
struct drm_gpuva *next, |
| 1272 |
const struct drm_gpuva_op_remap *op); |
| 1273 |
|
| 1274 |
void drm_gpuva_unmap(const struct drm_gpuva_op_unmap *op); |
| 1275 |
|
| 1276 |
/** |
| 1277 |
* drm_gpuva_op_remap_to_unmap_range() - Helper to get the start and range of |
| 1278 |
* the unmap stage of a remap op. |
| 1279 |
* @op: Remap op. |
| 1280 |
* @start_addr: Output pointer for the start of the required unmap. |
| 1281 |
* @range: Output pointer for the length of the required unmap. |
| 1282 |
* |
| 1283 |
* The given start address and range will be set such that they represent the |
| 1284 |
* range of the address space that was previously covered by the mapping being |
| 1285 |
* re-mapped, but is now empty. |
| 1286 |
*/ |
| 1287 |
static inline void |
| 1288 |
drm_gpuva_op_remap_to_unmap_range(const struct drm_gpuva_op_remap *op, |
| 1289 |
u64 *start_addr, u64 *range) |
| 1290 |
{ |
| 1291 |
const u64 va_start = op->prev ? |
| 1292 |
op->prev->va.addr + op->prev->va.range : |
| 1293 |
op->unmap->va->va.addr; |
| 1294 |
const u64 va_end = op->next ? |
| 1295 |
op->next->va.addr : |
| 1296 |
op->unmap->va->va.addr + op->unmap->va->va.range; |
| 1297 |
|
| 1298 |
if (start_addr) |
| 1299 |
*start_addr = va_start; |
| 1300 |
if (range) |
| 1301 |
*range = va_end - va_start; |
| 1302 |
} |
| 1303 |
|
| 1304 |
#endif /* __DRM_GPUVM_H__ */ |