| 1 |
/* |
| 2 |
* Copyright (C) 2014 Etnaviv Project |
| 3 |
* |
| 4 |
* Permission is hereby granted, free of charge, to any person obtaining a |
| 5 |
* copy of this software and associated documentation files (the "Software"), |
| 6 |
* to deal in the Software without restriction, including without limitation |
| 7 |
* the rights to use, copy, modify, merge, publish, distribute, sublicense, |
| 8 |
* and/or sell copies of the Software, and to permit persons to whom the |
| 9 |
* Software is furnished to do so, subject to the following conditions: |
| 10 |
* |
| 11 |
* The above copyright notice and this permission notice (including the next |
| 12 |
* paragraph) shall be included in all copies or substantial portions of the |
| 13 |
* Software. |
| 14 |
* |
| 15 |
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| 16 |
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| 17 |
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
| 18 |
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| 19 |
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
| 20 |
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| 21 |
* SOFTWARE. |
| 22 |
* |
| 23 |
* Authors: |
| 24 |
* Christian Gmeiner <[email protected]> |
| 25 |
*/ |
| 26 |
|
| 27 |
#include "util/os_mman.h" |
| 28 |
#include "util/hash_table.h" |
| 29 |
|
| 30 |
#include "etnaviv_priv.h" |
| 31 |
#include "etnaviv_drmif.h" |
| 32 |
|
| 33 |
simple_mtx_t etna_device_lock = SIMPLE_MTX_INITIALIZER; |
| 34 |
|
| 35 |
/* set buffer name, and add to table, call w/ etna_drm_table_lock held: */ |
| 36 |
static void set_name(struct etna_bo *bo, uint32_t name) |
| 37 |
{ |
| 38 |
simple_mtx_assert_locked(&etna_device_lock); |
| 39 |
|
| 40 |
bo->name = name; |
| 41 |
/* add ourself into the name table: */ |
| 42 |
_mesa_hash_table_insert(bo->dev->name_table, &bo->name, bo); |
| 43 |
} |
| 44 |
|
| 45 |
int etna_bo_is_idle(struct etna_bo *bo) |
| 46 |
{ |
| 47 |
return etna_bo_cpu_prep(bo, |
| 48 |
DRM_ETNA_PREP_READ | |
| 49 |
DRM_ETNA_PREP_WRITE | |
| 50 |
DRM_ETNA_PREP_NOSYNC) == 0; |
| 51 |
} |
| 52 |
|
| 53 |
/* Called under etna_drm_table_lock */ |
| 54 |
static void _etna_bo_free(struct etna_bo *bo) |
| 55 |
{ |
| 56 |
DEBUG_BO("Del bo:", bo); |
| 57 |
VG_BO_FREE(bo); |
| 58 |
|
| 59 |
simple_mtx_assert_locked(&etna_device_lock); |
| 60 |
|
| 61 |
if (bo->va) |
| 62 |
util_vma_heap_free(&bo->dev->address_space, bo->va, bo->size); |
| 63 |
|
| 64 |
if (bo->map) |
| 65 |
os_munmap(bo->map, bo->size); |
| 66 |
|
| 67 |
if (bo->handle) { |
| 68 |
struct drm_gem_close req = { |
| 69 |
.handle = bo->handle, |
| 70 |
}; |
| 71 |
|
| 72 |
if (bo->name) |
| 73 |
_mesa_hash_table_remove_key(bo->dev->name_table, &bo->name); |
| 74 |
|
| 75 |
_mesa_hash_table_remove_key(bo->dev->handle_table, &bo->handle); |
| 76 |
drmIoctl(bo->dev->fd, DRM_IOCTL_GEM_CLOSE, &req); |
| 77 |
} |
| 78 |
|
| 79 |
free(bo); |
| 80 |
} |
| 81 |
|
| 82 |
void etna_bo_kill_zombies(struct etna_device *dev) |
| 83 |
{ |
| 84 |
simple_mtx_assert_locked(&etna_device_lock); |
| 85 |
|
| 86 |
list_for_each_entry_safe(struct etna_bo, bo, &dev->zombie_list, list) { |
| 87 |
VG_BO_OBTAIN(bo); |
| 88 |
list_del(&bo->list); |
| 89 |
_etna_bo_free(bo); |
| 90 |
} |
| 91 |
} |
| 92 |
|
| 93 |
|
| 94 |
static void etna_bo_cleanup_zombies(struct etna_device *dev) |
| 95 |
{ |
| 96 |
simple_mtx_assert_locked(&etna_device_lock); |
| 97 |
|
| 98 |
list_for_each_entry_safe(struct etna_bo, bo, &dev->zombie_list, list) { |
| 99 |
/* Stop once we reach a busy BO - all others past this point were |
| 100 |
* freed more recently so are likely also busy. |
| 101 |
*/ |
| 102 |
if (!etna_bo_is_idle(bo)) |
| 103 |
break; |
| 104 |
|
| 105 |
VG_BO_OBTAIN(bo); |
| 106 |
list_del(&bo->list); |
| 107 |
_etna_bo_free(bo); |
| 108 |
} |
| 109 |
} |
| 110 |
|
| 111 |
void etna_bo_free(struct etna_bo *bo) { |
| 112 |
struct etna_device *dev = bo->dev; |
| 113 |
|
| 114 |
/* If the BO has a userspace managed address we don't free it immediately, |
| 115 |
* but keep it on a deferred destroy list until all submits with the buffer |
| 116 |
* have finished, at which point we can reuse the VMA space. |
| 117 |
*/ |
| 118 |
if (dev->use_softpin) { |
| 119 |
etna_bo_cleanup_zombies(dev); |
| 120 |
VG_BO_RELEASE(bo); |
| 121 |
assert(!list_is_linked(&bo->list)); |
| 122 |
list_addtail(&bo->list, &dev->zombie_list); |
| 123 |
} else { |
| 124 |
_etna_bo_free(bo); |
| 125 |
} |
| 126 |
} |
| 127 |
|
| 128 |
/* lookup a buffer from it's handle, call w/ etna_drm_table_lock held: */ |
| 129 |
static struct etna_bo *lookup_bo(void *tbl, uint32_t handle) |
| 130 |
{ |
| 131 |
struct etna_bo *bo = NULL; |
| 132 |
struct hash_entry *entry; |
| 133 |
|
| 134 |
simple_mtx_assert_locked(&etna_device_lock); |
| 135 |
|
| 136 |
entry = _mesa_hash_table_search(tbl, &handle); |
| 137 |
|
| 138 |
if (entry) { |
| 139 |
/* found, incr refcnt and return: */ |
| 140 |
bo = etna_bo_ref(entry->data); |
| 141 |
|
| 142 |
/* don't break the bucket/zombie list if this bo was found in one */ |
| 143 |
if (list_is_linked(&bo->list)) { |
| 144 |
VG_BO_OBTAIN(bo); |
| 145 |
etna_device_ref(bo->dev); |
| 146 |
list_del(&bo->list); |
| 147 |
} |
| 148 |
} |
| 149 |
|
| 150 |
return bo; |
| 151 |
} |
| 152 |
|
| 153 |
/* allocate a new buffer object, call w/ etna_drm_table_lock held */ |
| 154 |
static struct etna_bo *bo_from_handle(struct etna_device *dev, |
| 155 |
uint32_t size, uint32_t handle, uint32_t flags) |
| 156 |
{ |
| 157 |
struct etna_bo *bo = calloc(sizeof(*bo), 1); |
| 158 |
|
| 159 |
simple_mtx_assert_locked(&etna_device_lock); |
| 160 |
|
| 161 |
if (!bo) { |
| 162 |
struct drm_gem_close req = { |
| 163 |
.handle = handle, |
| 164 |
}; |
| 165 |
|
| 166 |
drmIoctl(dev->fd, DRM_IOCTL_GEM_CLOSE, &req); |
| 167 |
|
| 168 |
return NULL; |
| 169 |
} |
| 170 |
|
| 171 |
bo->dev = etna_device_ref(dev); |
| 172 |
bo->size = size; |
| 173 |
bo->handle = handle; |
| 174 |
bo->flags = flags; |
| 175 |
p_atomic_set(&bo->refcnt, 1); |
| 176 |
/* add ourselves to the handle table: */ |
| 177 |
_mesa_hash_table_insert(dev->handle_table, &bo->handle, bo); |
| 178 |
|
| 179 |
if (dev->use_softpin) |
| 180 |
bo->va = util_vma_heap_alloc(&dev->address_space, bo->size, 4096); |
| 181 |
|
| 182 |
return bo; |
| 183 |
} |
| 184 |
|
| 185 |
/* allocate a new (un-tiled) buffer object */ |
| 186 |
struct etna_bo *etna_bo_new(struct etna_device *dev, uint32_t size, |
| 187 |
uint32_t flags) |
| 188 |
{ |
| 189 |
struct etna_bo *bo; |
| 190 |
int ret; |
| 191 |
struct drm_etnaviv_gem_new req = { |
| 192 |
.flags = flags, |
| 193 |
}; |
| 194 |
|
| 195 |
bo = etna_bo_cache_alloc(&dev->bo_cache, &size, flags); |
| 196 |
if (bo) |
| 197 |
return bo; |
| 198 |
|
| 199 |
req.size = size; |
| 200 |
ret = drmCommandWriteRead(dev->fd, DRM_ETNAVIV_GEM_NEW, |
| 201 |
&req, sizeof(req)); |
| 202 |
if (ret) |
| 203 |
return NULL; |
| 204 |
|
| 205 |
simple_mtx_lock(&etna_device_lock); |
| 206 |
bo = bo_from_handle(dev, size, req.handle, flags); |
| 207 |
bo->reuse = 1; |
| 208 |
simple_mtx_unlock(&etna_device_lock); |
| 209 |
|
| 210 |
DEBUG_BO("New bo:", bo); |
| 211 |
VG_BO_ALLOC(bo); |
| 212 |
|
| 213 |
return bo; |
| 214 |
} |
| 215 |
|
| 216 |
struct etna_bo *etna_bo_ref(struct etna_bo *bo) |
| 217 |
{ |
| 218 |
p_atomic_inc(&bo->refcnt); |
| 219 |
|
| 220 |
return bo; |
| 221 |
} |
| 222 |
|
| 223 |
/* import a buffer object from DRI2 name */ |
| 224 |
struct etna_bo *etna_bo_from_name(struct etna_device *dev, |
| 225 |
uint32_t name) |
| 226 |
{ |
| 227 |
struct etna_bo *bo; |
| 228 |
struct drm_gem_open req = { |
| 229 |
.name = name, |
| 230 |
}; |
| 231 |
|
| 232 |
simple_mtx_lock(&etna_device_lock); |
| 233 |
|
| 234 |
/* check name table first, to see if bo is already open: */ |
| 235 |
bo = lookup_bo(dev->name_table, name); |
| 236 |
if (bo) |
| 237 |
goto out_unlock; |
| 238 |
|
| 239 |
if (drmIoctl(dev->fd, DRM_IOCTL_GEM_OPEN, &req)) { |
| 240 |
ERROR_MSG("gem-open failed: %s", strerror(errno)); |
| 241 |
goto out_unlock; |
| 242 |
} |
| 243 |
|
| 244 |
bo = lookup_bo(dev->handle_table, req.handle); |
| 245 |
if (bo) |
| 246 |
goto out_unlock; |
| 247 |
|
| 248 |
bo = bo_from_handle(dev, req.size, req.handle, 0); |
| 249 |
if (bo) { |
| 250 |
set_name(bo, name); |
| 251 |
DEBUG_BO("New from name:", bo); |
| 252 |
VG_BO_ALLOC(bo); |
| 253 |
} |
| 254 |
|
| 255 |
out_unlock: |
| 256 |
simple_mtx_unlock(&etna_device_lock); |
| 257 |
|
| 258 |
return bo; |
| 259 |
} |
| 260 |
|
| 261 |
/* import a buffer from dmabuf fd, does not take ownership of the |
| 262 |
* fd so caller should close() the fd when it is otherwise done |
| 263 |
* with it (even if it is still using the 'struct etna_bo *') |
| 264 |
*/ |
| 265 |
struct etna_bo *etna_bo_from_dmabuf(struct etna_device *dev, int fd) |
| 266 |
{ |
| 267 |
struct etna_bo *bo; |
| 268 |
int ret, size; |
| 269 |
uint32_t handle; |
| 270 |
|
| 271 |
/* take the lock before calling drmPrimeFDToHandle to avoid |
| 272 |
* racing against etna_bo_del, which might invalidate the |
| 273 |
* returned handle. |
| 274 |
*/ |
| 275 |
simple_mtx_lock(&etna_device_lock); |
| 276 |
|
| 277 |
ret = drmPrimeFDToHandle(dev->fd, fd, &handle); |
| 278 |
if (ret) { |
| 279 |
simple_mtx_unlock(&etna_device_lock); |
| 280 |
return NULL; |
| 281 |
} |
| 282 |
|
| 283 |
bo = lookup_bo(dev->handle_table, handle); |
| 284 |
if (bo) |
| 285 |
goto out_unlock; |
| 286 |
|
| 287 |
/* lseek() to get bo size */ |
| 288 |
size = lseek(fd, 0, SEEK_END); |
| 289 |
lseek(fd, 0, SEEK_CUR); |
| 290 |
|
| 291 |
bo = bo_from_handle(dev, size, handle, 0); |
| 292 |
|
| 293 |
DEBUG_BO("New from dmabuf:", bo); |
| 294 |
VG_BO_ALLOC(bo); |
| 295 |
|
| 296 |
out_unlock: |
| 297 |
simple_mtx_unlock(&etna_device_lock); |
| 298 |
|
| 299 |
return bo; |
| 300 |
} |
| 301 |
|
| 302 |
/* destroy a buffer object */ |
| 303 |
void etna_bo_del(struct etna_bo *bo) |
| 304 |
{ |
| 305 |
if (!bo) |
| 306 |
return; |
| 307 |
|
| 308 |
struct etna_device *dev = bo->dev; |
| 309 |
|
| 310 |
simple_mtx_lock(&etna_device_lock); |
| 311 |
|
| 312 |
/* Must test under table lock to avoid racing with the from_dmabuf/name |
| 313 |
* paths, which rely on the BO refcount to be stable over the lookup, so |
| 314 |
* they can grab a reference when the BO is found in the hash. |
| 315 |
*/ |
| 316 |
if (!p_atomic_dec_zero(&bo->refcnt)) |
| 317 |
goto out; |
| 318 |
|
| 319 |
if (bo->reuse && (etna_bo_cache_free(&dev->bo_cache, bo) == 0)) |
| 320 |
goto out; |
| 321 |
|
| 322 |
etna_bo_free(bo); |
| 323 |
etna_device_del_locked(dev); |
| 324 |
out: |
| 325 |
simple_mtx_unlock(&etna_device_lock); |
| 326 |
} |
| 327 |
|
| 328 |
/* get the global flink/DRI2 buffer name */ |
| 329 |
int etna_bo_get_name(struct etna_bo *bo, uint32_t *name) |
| 330 |
{ |
| 331 |
if (!bo->name) { |
| 332 |
struct drm_gem_flink req = { |
| 333 |
.handle = bo->handle, |
| 334 |
}; |
| 335 |
int ret; |
| 336 |
|
| 337 |
ret = drmIoctl(bo->dev->fd, DRM_IOCTL_GEM_FLINK, &req); |
| 338 |
if (ret) { |
| 339 |
return ret; |
| 340 |
} |
| 341 |
|
| 342 |
simple_mtx_lock(&etna_device_lock); |
| 343 |
set_name(bo, req.name); |
| 344 |
simple_mtx_unlock(&etna_device_lock); |
| 345 |
bo->reuse = 0; |
| 346 |
} |
| 347 |
|
| 348 |
*name = bo->name; |
| 349 |
|
| 350 |
return 0; |
| 351 |
} |
| 352 |
|
| 353 |
uint32_t etna_bo_handle(struct etna_bo *bo) |
| 354 |
{ |
| 355 |
return bo->handle; |
| 356 |
} |
| 357 |
|
| 358 |
/* caller owns the dmabuf fd that is returned and is responsible |
| 359 |
* to close() it when done |
| 360 |
*/ |
| 361 |
int etna_bo_dmabuf(struct etna_bo *bo) |
| 362 |
{ |
| 363 |
int ret, prime_fd; |
| 364 |
|
| 365 |
ret = drmPrimeHandleToFD(bo->dev->fd, bo->handle, DRM_CLOEXEC | DRM_RDWR, |
| 366 |
&prime_fd); |
| 367 |
if (ret) { |
| 368 |
ERROR_MSG("failed to get dmabuf fd: %d", ret); |
| 369 |
return ret; |
| 370 |
} |
| 371 |
|
| 372 |
bo->reuse = 0; |
| 373 |
|
| 374 |
return prime_fd; |
| 375 |
} |
| 376 |
|
| 377 |
uint32_t etna_bo_size(struct etna_bo *bo) |
| 378 |
{ |
| 379 |
return bo->size; |
| 380 |
} |
| 381 |
|
| 382 |
uint32_t etna_bo_gpu_va(struct etna_bo *bo) |
| 383 |
{ |
| 384 |
return bo->va; |
| 385 |
} |
| 386 |
|
| 387 |
void *etna_bo_map(struct etna_bo *bo) |
| 388 |
{ |
| 389 |
if (!bo->map) { |
| 390 |
int ret; |
| 391 |
void *map; |
| 392 |
struct drm_etnaviv_gem_info req = { |
| 393 |
.handle = bo->handle, |
| 394 |
}; |
| 395 |
|
| 396 |
ret = drmCommandWriteRead(bo->dev->fd, DRM_ETNAVIV_GEM_INFO, |
| 397 |
&req, sizeof(req)); |
| 398 |
if (ret) |
| 399 |
return NULL; |
| 400 |
|
| 401 |
map = os_mmap(0, bo->size, PROT_READ | PROT_WRITE, |
| 402 |
MAP_SHARED, bo->dev->fd, req.offset); |
| 403 |
if (map == MAP_FAILED) { |
| 404 |
ERROR_MSG("mmap failed: %s", strerror(errno)); |
| 405 |
return NULL; |
| 406 |
} |
| 407 |
|
| 408 |
if (p_atomic_cmpxchg(&bo->map, NULL, map)) |
| 409 |
munmap(map, bo->size); |
| 410 |
} |
| 411 |
|
| 412 |
return bo->map; |
| 413 |
} |
| 414 |
|
| 415 |
int etna_bo_cpu_prep(struct etna_bo *bo, uint32_t op) |
| 416 |
{ |
| 417 |
struct drm_etnaviv_gem_cpu_prep req = { |
| 418 |
.handle = bo->handle, |
| 419 |
.op = op, |
| 420 |
}; |
| 421 |
|
| 422 |
get_abs_timeout(&req.timeout, 5000000000); |
| 423 |
|
| 424 |
return drmCommandWrite(bo->dev->fd, DRM_ETNAVIV_GEM_CPU_PREP, |
| 425 |
&req, sizeof(req)); |
| 426 |
} |
| 427 |
|
| 428 |
void etna_bo_cpu_fini(struct etna_bo *bo) |
| 429 |
{ |
| 430 |
struct drm_etnaviv_gem_cpu_fini req = { |
| 431 |
.handle = bo->handle, |
| 432 |
}; |
| 433 |
|
| 434 |
drmCommandWrite(bo->dev->fd, DRM_ETNAVIV_GEM_CPU_FINI, |
| 435 |
&req, sizeof(req)); |
| 436 |
} |