| 1 |
#include <assert.h> |
| 2 |
#include <errno.h> |
| 3 |
#include <fcntl.h> |
| 4 |
#include <inttypes.h> |
| 5 |
#include <stdio.h> |
| 6 |
#include <stdlib.h> |
| 7 |
#include <xf86drm.h> |
| 8 |
|
| 9 |
#include "drm-uapi/nouveau_drm.h" |
| 10 |
#include "nouveau.h" |
| 11 |
#include "nvif/class.h" |
| 12 |
#include "nvif/cl0080.h" |
| 13 |
#include "nvif/ioctl.h" |
| 14 |
|
| 15 |
#include "nv_push.h" |
| 16 |
|
| 17 |
#include "util/bitscan.h" |
| 18 |
#include "util/list.h" |
| 19 |
#include "util/os_mman.h" |
| 20 |
#include "util/simple_mtx.h" |
| 21 |
#include "util/u_atomic.h" |
| 22 |
#include "util/u_memory.h" |
| 23 |
|
| 24 |
static FILE *nouveau_out = NULL; |
| 25 |
static uint32_t nouveau_debug = 0; |
| 26 |
|
| 27 |
#define dbg_on(lvl) (nouveau_debug & (1 << lvl)) |
| 28 |
#define dbg(lvl, fmt, args...) do { \ |
| 29 |
if (dbg_on((lvl))) \ |
| 30 |
fprintf(nouveau_out, "nouveau: "fmt, ##args); \ |
| 31 |
} while(0) |
| 32 |
#define err(fmt, args...) fprintf(nouveau_out, "nouveau: "fmt, ##args) |
| 33 |
|
| 34 |
static void |
| 35 |
debug_init(void) |
| 36 |
{ |
| 37 |
static bool once = false; |
| 38 |
const char *debug, *out; |
| 39 |
|
| 40 |
if (once) |
| 41 |
return; |
| 42 |
once = true; |
| 43 |
|
| 44 |
debug = os_get_option("NOUVEAU_LIBDRM_DEBUG"); |
| 45 |
if (debug) { |
| 46 |
int n = strtol(debug, NULL, 0); |
| 47 |
if (n >= 0) |
| 48 |
nouveau_debug = n; |
| 49 |
} |
| 50 |
|
| 51 |
nouveau_out = stderr; |
| 52 |
out = os_get_option("NOUVEAU_LIBDRM_OUT"); |
| 53 |
if (out) { |
| 54 |
FILE *fout = fopen(out, "w"); |
| 55 |
if (fout) |
| 56 |
nouveau_out = fout; |
| 57 |
} |
| 58 |
} |
| 59 |
|
| 60 |
int |
| 61 |
nouveau_drm_new(int fd, struct nouveau_drm **pdrm) |
| 62 |
{ |
| 63 |
debug_init(); |
| 64 |
|
| 65 |
struct nouveau_drm *drm = calloc(1, sizeof(*drm)); |
| 66 |
if (!drm) |
| 67 |
return -ENOMEM; |
| 68 |
drm->fd = fd; |
| 69 |
*pdrm = drm; |
| 70 |
|
| 71 |
drmVersionPtr ver = drmGetVersion(fd); |
| 72 |
if (!ver) |
| 73 |
goto out_err; |
| 74 |
|
| 75 |
drm->version = (ver->version_major << 24) | |
| 76 |
(ver->version_minor << 8) | |
| 77 |
ver->version_patchlevel; |
| 78 |
|
| 79 |
drmFreeVersion(ver); |
| 80 |
|
| 81 |
if (drm->version < 0x01000301) |
| 82 |
goto out_err; |
| 83 |
|
| 84 |
return 0; |
| 85 |
|
| 86 |
out_err: |
| 87 |
nouveau_drm_del(pdrm); |
| 88 |
return -EINVAL; |
| 89 |
} |
| 90 |
|
| 91 |
void |
| 92 |
nouveau_drm_del(struct nouveau_drm **pdrm) |
| 93 |
{ |
| 94 |
free(*pdrm); |
| 95 |
*pdrm = NULL; |
| 96 |
} |
| 97 |
|
| 98 |
static int |
| 99 |
nouveau_object_channel_new(struct nouveau_object *parent, uint64_t handle, uint32_t oclass, |
| 100 |
struct nouveau_object *obj) |
| 101 |
{ |
| 102 |
// TODO nv04? |
| 103 |
struct nouveau_drm *drm = nouveau_drm(parent); |
| 104 |
struct nouveau_device *dev = (void*)obj->parent; |
| 105 |
struct nouveau_fifo *fifo = obj->data; |
| 106 |
struct drm_nouveau_channel_alloc req = { }; |
| 107 |
|
| 108 |
/* nvc0 doesn't need any special handling here */ |
| 109 |
if (dev->chipset < 0xc0) { |
| 110 |
struct nv04_fifo *nv04 = obj->data; |
| 111 |
req.fb_ctxdma_handle = nv04->vram; |
| 112 |
req.tt_ctxdma_handle = nv04->gart; |
| 113 |
} else if (dev->chipset >= 0xe0) { |
| 114 |
struct nve0_fifo *nve0 = obj->data; |
| 115 |
req.fb_ctxdma_handle = 0xffffffff; |
| 116 |
req.tt_ctxdma_handle = nve0->engine; |
| 117 |
} |
| 118 |
|
| 119 |
int ret = drmCommandWriteRead(drm->fd, DRM_NOUVEAU_CHANNEL_ALLOC, &req, sizeof(req)); |
| 120 |
if (ret) |
| 121 |
return ret; |
| 122 |
|
| 123 |
fifo->pushbuf = req.pushbuf_domains; |
| 124 |
fifo->notify = req.notifier_handle; |
| 125 |
obj->handle = req.channel; |
| 126 |
return 0; |
| 127 |
} |
| 128 |
|
| 129 |
static int |
| 130 |
nouveau_object_notifier_new(struct nouveau_object *parent, uint64_t handle, |
| 131 |
struct nouveau_object *obj) |
| 132 |
{ |
| 133 |
struct nouveau_drm *drm = nouveau_drm(obj); |
| 134 |
struct nv04_notify *ntfy = obj->data; |
| 135 |
struct drm_nouveau_notifierobj_alloc req = { |
| 136 |
.channel = parent->handle, |
| 137 |
.handle = handle, |
| 138 |
.size = ntfy->length, |
| 139 |
}; |
| 140 |
|
| 141 |
int ret = drmCommandWriteRead(drm->fd, DRM_NOUVEAU_NOTIFIEROBJ_ALLOC, &req, sizeof(req)); |
| 142 |
if (ret) |
| 143 |
return ret; |
| 144 |
|
| 145 |
ntfy->offset = req.offset; |
| 146 |
return 0; |
| 147 |
} |
| 148 |
|
| 149 |
static int |
| 150 |
nouveau_object_subchan_new(struct nouveau_object *parent, uint64_t handle, uint32_t oclass, |
| 151 |
struct nouveau_object *obj) |
| 152 |
{ |
| 153 |
struct nouveau_drm *drm = nouveau_drm(parent); |
| 154 |
struct { |
| 155 |
struct nvif_ioctl_v0 ioctl; |
| 156 |
struct nvif_ioctl_new_v0 new; |
| 157 |
} args = { |
| 158 |
.ioctl = { |
| 159 |
.route = 0xff, |
| 160 |
.token = parent->handle, |
| 161 |
.type = NVIF_IOCTL_V0_NEW, |
| 162 |
.version = 0, |
| 163 |
}, |
| 164 |
.new = { |
| 165 |
.handle = handle, |
| 166 |
.object = (uintptr_t)obj, |
| 167 |
.oclass = oclass, |
| 168 |
.route = NVIF_IOCTL_V0_ROUTE_NVIF, |
| 169 |
.token = (uintptr_t)obj, |
| 170 |
.version = 0, |
| 171 |
}, |
| 172 |
}; |
| 173 |
|
| 174 |
return drmCommandWrite(drm->fd, DRM_NOUVEAU_NVIF, &args, sizeof(args)); |
| 175 |
} |
| 176 |
|
| 177 |
/* TODO: split this interfaces up so we can verify the right parent object type gets passed in */ |
| 178 |
int |
| 179 |
nouveau_object_new(struct nouveau_object *parent, uint64_t handle, uint32_t oclass, void *data, |
| 180 |
uint32_t length, struct nouveau_object **pobj) |
| 181 |
{ |
| 182 |
struct nouveau_object *obj = calloc(1, sizeof(*obj)); |
| 183 |
if (!obj) |
| 184 |
return -ENOMEM; |
| 185 |
|
| 186 |
obj->parent = parent; |
| 187 |
obj->handle = handle; |
| 188 |
obj->oclass = oclass; |
| 189 |
if (length) { |
| 190 |
obj->data = malloc(length); |
| 191 |
memcpy(obj->data, data, length); |
| 192 |
} |
| 193 |
|
| 194 |
int ret; |
| 195 |
switch (oclass) { |
| 196 |
case NOUVEAU_FIFO_CHANNEL_CLASS: |
| 197 |
ret = nouveau_object_channel_new(parent, handle, oclass, obj); |
| 198 |
break; |
| 199 |
case NOUVEAU_NOTIFIER_CLASS: |
| 200 |
ret = nouveau_object_notifier_new(parent, handle, obj); |
| 201 |
break; |
| 202 |
default: |
| 203 |
ret = nouveau_object_subchan_new(parent, handle, oclass, obj); |
| 204 |
break; |
| 205 |
} |
| 206 |
|
| 207 |
if (ret) { |
| 208 |
free(obj->data); |
| 209 |
free(obj); |
| 210 |
return ret; |
| 211 |
} |
| 212 |
|
| 213 |
*pobj = obj; |
| 214 |
return 0; |
| 215 |
} |
| 216 |
|
| 217 |
static void |
| 218 |
nouveau_object_channel_del(struct nouveau_object *obj) |
| 219 |
{ |
| 220 |
struct nouveau_drm *drm = nouveau_drm(obj->parent); |
| 221 |
struct drm_nouveau_channel_free req = { |
| 222 |
.channel = obj->handle, |
| 223 |
}; |
| 224 |
|
| 225 |
int ret = drmCommandWrite(drm->fd, DRM_NOUVEAU_CHANNEL_FREE, &req, sizeof(req)); |
| 226 |
assert(!ret); |
| 227 |
} |
| 228 |
|
| 229 |
static void |
| 230 |
nouveau_object_subchan_del(struct nouveau_object *obj) |
| 231 |
{ |
| 232 |
struct { |
| 233 |
struct nvif_ioctl_v0 ioctl; |
| 234 |
struct nvif_ioctl_del del; |
| 235 |
} args = { |
| 236 |
.ioctl = { |
| 237 |
.object = (uintptr_t)obj, |
| 238 |
.owner = NVIF_IOCTL_V0_OWNER_ANY, |
| 239 |
.route = 0x00, |
| 240 |
.type = NVIF_IOCTL_V0_DEL, |
| 241 |
.version = 0, |
| 242 |
}, |
| 243 |
}; |
| 244 |
|
| 245 |
drmCommandWrite(obj->parent->handle, DRM_NOUVEAU_NVIF, &args, sizeof(args)); |
| 246 |
} |
| 247 |
|
| 248 |
static void |
| 249 |
nouveau_object_gpuobj_del(struct nouveau_object *obj) |
| 250 |
{ |
| 251 |
struct nouveau_drm *drm = nouveau_drm(obj->parent); |
| 252 |
struct drm_nouveau_gpuobj_free req = { |
| 253 |
.channel = obj->parent->handle, |
| 254 |
.handle = obj->handle, |
| 255 |
}; |
| 256 |
|
| 257 |
drmCommandWrite(drm->fd, DRM_NOUVEAU_GPUOBJ_FREE, &req, sizeof(req)); |
| 258 |
} |
| 259 |
|
| 260 |
void |
| 261 |
nouveau_object_del(struct nouveau_object **pobj) |
| 262 |
{ |
| 263 |
if (!*pobj) |
| 264 |
return; |
| 265 |
|
| 266 |
struct nouveau_object *obj = *pobj; |
| 267 |
switch (obj->oclass) { |
| 268 |
case NOUVEAU_FIFO_CHANNEL_CLASS: |
| 269 |
nouveau_object_channel_del(obj); |
| 270 |
break; |
| 271 |
case NOUVEAU_NOTIFIER_CLASS: |
| 272 |
nouveau_object_gpuobj_del(obj); |
| 273 |
break; |
| 274 |
default: |
| 275 |
nouveau_object_subchan_del(obj); |
| 276 |
break; |
| 277 |
} |
| 278 |
free(obj->data); |
| 279 |
free(obj); |
| 280 |
*pobj = NULL; |
| 281 |
} |
| 282 |
|
| 283 |
#define NOUVEAU_OBJECT_MAX_CLASSES 16 |
| 284 |
int |
| 285 |
nouveau_object_mclass(struct nouveau_object *obj, const struct nouveau_mclass *mclass) |
| 286 |
{ |
| 287 |
struct nouveau_drm *drm = nouveau_drm(obj->parent); |
| 288 |
struct { |
| 289 |
struct nvif_ioctl_v0 ioctl; |
| 290 |
struct nvif_ioctl_sclass_v0 sclass; |
| 291 |
struct nvif_ioctl_sclass_oclass_v0 list[NOUVEAU_OBJECT_MAX_CLASSES]; |
| 292 |
} args = { |
| 293 |
.ioctl = { |
| 294 |
.route = 0xff, |
| 295 |
.token = obj->handle, |
| 296 |
.type = NVIF_IOCTL_V0_SCLASS, |
| 297 |
.version = 0, |
| 298 |
}, |
| 299 |
.sclass = { |
| 300 |
.count = NOUVEAU_OBJECT_MAX_CLASSES, |
| 301 |
.version = 0, |
| 302 |
}, |
| 303 |
}; |
| 304 |
|
| 305 |
int ret = drmCommandWriteRead(drm->fd, DRM_NOUVEAU_NVIF, &args, sizeof(args)); |
| 306 |
if (ret) |
| 307 |
return ret; |
| 308 |
|
| 309 |
int i; |
| 310 |
for (i = 0; mclass[i].oclass; i++) { |
| 311 |
for (int j = 0; j < args.sclass.count; j++) { |
| 312 |
if (args.list[j].oclass == mclass[i].oclass) |
| 313 |
return i; |
| 314 |
} |
| 315 |
} |
| 316 |
|
| 317 |
return -ENODEV; |
| 318 |
} |
| 319 |
|
| 320 |
struct nouveau_device_priv { |
| 321 |
struct nouveau_device base; |
| 322 |
simple_mtx_t lock; |
| 323 |
struct list_head bo_list; |
| 324 |
int gart_limit_percent; |
| 325 |
int vram_limit_percent; |
| 326 |
}; |
| 327 |
|
| 328 |
static inline struct nouveau_device_priv * |
| 329 |
nouveau_device(struct nouveau_device *dev) |
| 330 |
{ |
| 331 |
return (struct nouveau_device_priv *)dev; |
| 332 |
} |
| 333 |
|
| 334 |
int |
| 335 |
nouveau_device_info(struct nouveau_device *dev, struct nv_device_info_v0 *info) |
| 336 |
{ |
| 337 |
struct nouveau_drm *drm = nouveau_drm(dev->object.parent); |
| 338 |
struct { |
| 339 |
struct nvif_ioctl_v0 ioctl; |
| 340 |
struct nvif_ioctl_mthd_v0 mthd; |
| 341 |
struct nv_device_info_v0 info; |
| 342 |
} args = { |
| 343 |
.ioctl = { |
| 344 |
.object = (uintptr_t)dev, |
| 345 |
.owner = NVIF_IOCTL_V0_OWNER_ANY, |
| 346 |
.route = 0x00, |
| 347 |
.type = NVIF_IOCTL_V0_MTHD, |
| 348 |
.version = 0, |
| 349 |
}, |
| 350 |
.mthd = { |
| 351 |
.method = NV_DEVICE_V0_INFO, |
| 352 |
.version = 0, |
| 353 |
}, |
| 354 |
.info = { |
| 355 |
.version = 0, |
| 356 |
}, |
| 357 |
}; |
| 358 |
|
| 359 |
int ret = drmCommandWriteRead(drm->fd, DRM_NOUVEAU_NVIF, &args, sizeof(args)); |
| 360 |
if (ret) |
| 361 |
return ret; |
| 362 |
|
| 363 |
*info = args.info; |
| 364 |
return 0; |
| 365 |
} |
| 366 |
|
| 367 |
int |
| 368 |
nouveau_device_new(struct nouveau_object *parent, struct nouveau_device **pdev) |
| 369 |
{ |
| 370 |
struct nouveau_drm *drm = nouveau_drm(parent); |
| 371 |
struct nouveau_device *dev; |
| 372 |
uint64_t v; |
| 373 |
const char *tmp; |
| 374 |
|
| 375 |
struct nouveau_device_priv *nvdev = calloc(1, sizeof(*nvdev)); |
| 376 |
if (!nvdev) |
| 377 |
return -ENOMEM; |
| 378 |
dev = *pdev = &nvdev->base; |
| 379 |
dev->object.parent = parent; |
| 380 |
|
| 381 |
struct { |
| 382 |
struct nvif_ioctl_v0 ioctl; |
| 383 |
struct nvif_ioctl_new_v0 new; |
| 384 |
struct nv_device_v0 dev; |
| 385 |
} args = { |
| 386 |
.ioctl = { |
| 387 |
.object = 0, |
| 388 |
.owner = NVIF_IOCTL_V0_OWNER_ANY, |
| 389 |
.route = 0x00, |
| 390 |
.type = NVIF_IOCTL_V0_NEW, |
| 391 |
.version = 0, |
| 392 |
}, |
| 393 |
.new = { |
| 394 |
.handle = 0, |
| 395 |
.object = (uintptr_t)&nvdev->base.object, |
| 396 |
.oclass = NV_DEVICE, |
| 397 |
.route = NVIF_IOCTL_V0_ROUTE_NVIF, |
| 398 |
.token = (uintptr_t)&nvdev->base.object, |
| 399 |
.version = 0, |
| 400 |
}, |
| 401 |
.dev = { |
| 402 |
.device = ~0ULL, |
| 403 |
}, |
| 404 |
}; |
| 405 |
|
| 406 |
int ret = drmCommandWrite(drm->fd, DRM_NOUVEAU_NVIF, &args, sizeof(args)); |
| 407 |
if (ret) |
| 408 |
goto done; |
| 409 |
|
| 410 |
struct nv_device_info_v0 info; |
| 411 |
ret = nouveau_device_info(dev, &info); |
| 412 |
if (ret) |
| 413 |
goto done; |
| 414 |
|
| 415 |
nvdev->base.chipset = info.chipset; |
| 416 |
nvdev->base.info.chipset = info.chipset; |
| 417 |
switch (info.platform) { |
| 418 |
case NV_DEVICE_INFO_V0_PCI: |
| 419 |
case NV_DEVICE_INFO_V0_AGP: |
| 420 |
case NV_DEVICE_INFO_V0_PCIE: |
| 421 |
nvdev->base.info.type = NV_DEVICE_TYPE_DIS; |
| 422 |
break; |
| 423 |
case NV_DEVICE_INFO_V0_IGP: |
| 424 |
nvdev->base.info.type = NV_DEVICE_TYPE_IGP; |
| 425 |
break; |
| 426 |
case NV_DEVICE_INFO_V0_SOC: |
| 427 |
nvdev->base.info.type = NV_DEVICE_TYPE_SOC; |
| 428 |
break; |
| 429 |
default: |
| 430 |
UNREACHABLE("unhandled nvidia device type"); |
| 431 |
break; |
| 432 |
} |
| 433 |
|
| 434 |
drmDevicePtr drm_device; |
| 435 |
ret = drmGetDevice2(drm->fd, 0, &drm_device); |
| 436 |
if (ret) |
| 437 |
goto done; |
| 438 |
|
| 439 |
if (drm_device->bustype == DRM_BUS_PCI) { |
| 440 |
nvdev->base.info.pci.domain = drm_device->businfo.pci->domain; |
| 441 |
nvdev->base.info.pci.bus = drm_device->businfo.pci->bus; |
| 442 |
nvdev->base.info.pci.dev = drm_device->businfo.pci->dev; |
| 443 |
nvdev->base.info.pci.func = drm_device->businfo.pci->func; |
| 444 |
nvdev->base.info.pci.revision_id = drm_device->deviceinfo.pci->revision_id; |
| 445 |
nvdev->base.info.device_id = drm_device->deviceinfo.pci->device_id; |
| 446 |
} |
| 447 |
|
| 448 |
drmFreeDevice(&drm_device); |
| 449 |
|
| 450 |
ret = nouveau_getparam(dev, NOUVEAU_GETPARAM_FB_SIZE, &v); |
| 451 |
if (ret) |
| 452 |
goto done; |
| 453 |
nvdev->base.vram_size = v; |
| 454 |
|
| 455 |
ret = nouveau_getparam(dev, NOUVEAU_GETPARAM_AGP_SIZE, &v); |
| 456 |
if (ret) |
| 457 |
goto done; |
| 458 |
nvdev->base.gart_size = v; |
| 459 |
|
| 460 |
tmp = os_get_option("NOUVEAU_LIBDRM_VRAM_LIMIT_PERCENT"); |
| 461 |
if (tmp) |
| 462 |
nvdev->vram_limit_percent = atoi(tmp); |
| 463 |
else |
| 464 |
nvdev->vram_limit_percent = 80; |
| 465 |
nvdev->base.vram_limit = (nvdev->base.vram_size * nvdev->vram_limit_percent) / 100; |
| 466 |
|
| 467 |
tmp = os_get_option("NOUVEAU_LIBDRM_GART_LIMIT_PERCENT"); |
| 468 |
if (tmp) |
| 469 |
nvdev->gart_limit_percent = atoi(tmp); |
| 470 |
else |
| 471 |
nvdev->gart_limit_percent = 80; |
| 472 |
nvdev->base.gart_limit = (nvdev->base.gart_size * nvdev->gart_limit_percent) / 100; |
| 473 |
|
| 474 |
simple_mtx_init(&nvdev->lock, mtx_plain); |
| 475 |
list_inithead(&nvdev->bo_list); |
| 476 |
done: |
| 477 |
if (ret) |
| 478 |
nouveau_device_del(pdev); |
| 479 |
return ret; |
| 480 |
} |
| 481 |
|
| 482 |
void |
| 483 |
nouveau_device_set_classes_for_debug(struct nouveau_device *dev, |
| 484 |
uint32_t cls_eng3d, |
| 485 |
uint32_t cls_compute, |
| 486 |
uint32_t cls_m2mf, |
| 487 |
uint32_t cls_copy) |
| 488 |
{ |
| 489 |
dev->info.cls_eng3d = cls_eng3d; |
| 490 |
dev->info.cls_compute = cls_compute; |
| 491 |
dev->info.cls_m2mf = cls_m2mf; |
| 492 |
dev->info.cls_copy = cls_copy; |
| 493 |
} |
| 494 |
|
| 495 |
void |
| 496 |
nouveau_device_del(struct nouveau_device **pdev) |
| 497 |
{ |
| 498 |
struct nouveau_device_priv *nvdev = nouveau_device(*pdev); |
| 499 |
if (!nvdev) |
| 500 |
return; |
| 501 |
|
| 502 |
simple_mtx_destroy(&nvdev->lock); |
| 503 |
free(nvdev); |
| 504 |
*pdev = NULL; |
| 505 |
} |
| 506 |
|
| 507 |
int |
| 508 |
nouveau_getparam(struct nouveau_device *dev, uint64_t param, uint64_t *value) |
| 509 |
{ |
| 510 |
struct nouveau_drm *drm = nouveau_drm(&dev->object); |
| 511 |
struct drm_nouveau_getparam r = { .param = param }; |
| 512 |
int ret = drmCommandWriteRead(drm->fd, DRM_NOUVEAU_GETPARAM, &r, sizeof(r)); |
| 513 |
*value = r.value; |
| 514 |
return ret; |
| 515 |
} |
| 516 |
|
| 517 |
struct nouveau_client_kref { |
| 518 |
struct drm_nouveau_gem_pushbuf_bo *kref; |
| 519 |
struct nouveau_pushbuf *push; |
| 520 |
}; |
| 521 |
|
| 522 |
struct nouveau_client_priv { |
| 523 |
struct nouveau_client base; |
| 524 |
struct nouveau_client_kref *kref; |
| 525 |
unsigned kref_nr; |
| 526 |
}; |
| 527 |
|
| 528 |
static inline struct nouveau_client_priv * |
| 529 |
nouveau_client(struct nouveau_client *client) |
| 530 |
{ |
| 531 |
return (struct nouveau_client_priv *)client; |
| 532 |
} |
| 533 |
|
| 534 |
int |
| 535 |
nouveau_client_new(struct nouveau_device *dev, struct nouveau_client **pclient) |
| 536 |
{ |
| 537 |
struct nouveau_client_priv *pcli = calloc(1, sizeof(*pcli)); |
| 538 |
if (!pcli) |
| 539 |
return -ENOMEM; |
| 540 |
|
| 541 |
pcli->base.device = dev; |
| 542 |
*pclient = &pcli->base; |
| 543 |
|
| 544 |
return 0; |
| 545 |
} |
| 546 |
|
| 547 |
void |
| 548 |
nouveau_client_del(struct nouveau_client **pclient) |
| 549 |
{ |
| 550 |
struct nouveau_client_priv *pcli = nouveau_client(*pclient); |
| 551 |
if (pcli) { |
| 552 |
free(pcli->kref); |
| 553 |
free(pcli); |
| 554 |
} |
| 555 |
} |
| 556 |
|
| 557 |
struct nouveau_bo_priv { |
| 558 |
struct nouveau_bo base; |
| 559 |
struct list_head head; |
| 560 |
uint32_t refcnt; |
| 561 |
uint64_t map_handle; |
| 562 |
uint32_t name; |
| 563 |
uint32_t access; |
| 564 |
}; |
| 565 |
|
| 566 |
static inline struct nouveau_bo_priv * |
| 567 |
nouveau_bo(struct nouveau_bo *bo) |
| 568 |
{ |
| 569 |
return (struct nouveau_bo_priv *)bo; |
| 570 |
} |
| 571 |
|
| 572 |
static void |
| 573 |
bo_info(struct nouveau_bo *bo, struct drm_nouveau_gem_info *info) |
| 574 |
{ |
| 575 |
struct nouveau_bo_priv *nvbo = nouveau_bo(bo); |
| 576 |
|
| 577 |
nvbo->map_handle = info->map_handle; |
| 578 |
bo->handle = info->handle; |
| 579 |
bo->size = info->size; |
| 580 |
bo->offset = info->offset; |
| 581 |
|
| 582 |
bo->flags = 0; |
| 583 |
if (info->domain & NOUVEAU_GEM_DOMAIN_VRAM) |
| 584 |
bo->flags |= NOUVEAU_BO_VRAM; |
| 585 |
if (info->domain & NOUVEAU_GEM_DOMAIN_GART) |
| 586 |
bo->flags |= NOUVEAU_BO_GART; |
| 587 |
if (!(info->tile_flags & NOUVEAU_GEM_TILE_NONCONTIG)) |
| 588 |
bo->flags |= NOUVEAU_BO_CONTIG; |
| 589 |
if (nvbo->map_handle) |
| 590 |
bo->flags |= NOUVEAU_BO_MAP; |
| 591 |
|
| 592 |
if (bo->device->chipset >= 0xc0) { |
| 593 |
bo->config.nvc0.memtype = (info->tile_flags & 0xff00) >> 8; |
| 594 |
bo->config.nvc0.tile_mode = info->tile_mode; |
| 595 |
} else |
| 596 |
if (bo->device->chipset >= 0x80 || bo->device->chipset == 0x50) { |
| 597 |
bo->config.nv50.memtype = (info->tile_flags & 0x07f00) >> 8 | |
| 598 |
(info->tile_flags & 0x30000) >> 9; |
| 599 |
bo->config.nv50.tile_mode = info->tile_mode << 4; |
| 600 |
} else { |
| 601 |
//bo->config.nv04.surf_flags = info->tile_flags & 7; |
| 602 |
//bo->config.nv04.surf_pitch = info->tile_mode; |
| 603 |
} |
| 604 |
} |
| 605 |
|
| 606 |
static void |
| 607 |
nouveau_bo_make_global(struct nouveau_bo_priv *nvbo) |
| 608 |
{ |
| 609 |
if (!nvbo->head.next) { |
| 610 |
struct nouveau_device_priv *nvdev = nouveau_device(nvbo->base.device); |
| 611 |
simple_mtx_lock(&nvdev->lock); |
| 612 |
if (!nvbo->head.next) |
| 613 |
list_add(&nvbo->head, &nvdev->bo_list); |
| 614 |
simple_mtx_unlock(&nvdev->lock); |
| 615 |
} |
| 616 |
} |
| 617 |
|
| 618 |
static int |
| 619 |
nouveau_bo_wrap_locked(struct nouveau_device *dev, uint32_t handle, |
| 620 |
struct nouveau_bo **pbo, int name) |
| 621 |
{ |
| 622 |
struct nouveau_drm *drm = nouveau_drm(&dev->object); |
| 623 |
struct nouveau_device_priv *nvdev = nouveau_device(dev); |
| 624 |
struct drm_nouveau_gem_info req = { .handle = handle }; |
| 625 |
struct nouveau_bo_priv *nvbo; |
| 626 |
int ret; |
| 627 |
|
| 628 |
list_for_each_entry(struct nouveau_bo_priv, nvbo, &nvdev->bo_list, head) { |
| 629 |
if (nvbo->base.handle == handle) { |
| 630 |
if (p_atomic_inc_return(&nvbo->refcnt) == 1) { |
| 631 |
/* |
| 632 |
* Uh oh, this bo is dead and someone else |
| 633 |
* will free it, but because refcnt is |
| 634 |
* now non-zero fortunately they won't |
| 635 |
* call the ioctl to close the bo. |
| 636 |
* |
| 637 |
* Remove this bo from the list so other |
| 638 |
* calls to nouveau_bo_wrap_locked will |
| 639 |
* see our replacement nvbo. |
| 640 |
*/ |
| 641 |
list_del(&nvbo->head); |
| 642 |
if (!name) |
| 643 |
name = nvbo->name; |
| 644 |
break; |
| 645 |
} |
| 646 |
|
| 647 |
*pbo = &nvbo->base; |
| 648 |
return 0; |
| 649 |
} |
| 650 |
} |
| 651 |
|
| 652 |
ret = drmCommandWriteRead(drm->fd, DRM_NOUVEAU_GEM_INFO, &req, sizeof(req)); |
| 653 |
if (ret) |
| 654 |
return ret; |
| 655 |
|
| 656 |
nvbo = calloc(1, sizeof(*nvbo)); |
| 657 |
if (nvbo) { |
| 658 |
p_atomic_set(&nvbo->refcnt, 1); |
| 659 |
nvbo->base.device = dev; |
| 660 |
bo_info(&nvbo->base, &req); |
| 661 |
nvbo->name = name; |
| 662 |
list_add(&nvbo->head, &nvdev->bo_list); |
| 663 |
*pbo = &nvbo->base; |
| 664 |
return 0; |
| 665 |
} |
| 666 |
|
| 667 |
return -ENOMEM; |
| 668 |
} |
| 669 |
|
| 670 |
int |
| 671 |
nouveau_bo_new(struct nouveau_device *dev, uint32_t flags, uint32_t align, uint64_t size, |
| 672 |
union nouveau_bo_config *config, struct nouveau_bo **pbo) |
| 673 |
{ |
| 674 |
struct nouveau_drm *drm = nouveau_drm(&dev->object); |
| 675 |
struct drm_nouveau_gem_new req = {}; |
| 676 |
struct drm_nouveau_gem_info *info = &req.info; |
| 677 |
int ret; |
| 678 |
|
| 679 |
struct nouveau_bo_priv *nvbo = calloc(1, sizeof(*nvbo)); |
| 680 |
if (!nvbo) |
| 681 |
return -ENOMEM; |
| 682 |
|
| 683 |
struct nouveau_bo *bo = &nvbo->base; |
| 684 |
p_atomic_set(&nvbo->refcnt, 1); |
| 685 |
bo->device = dev; |
| 686 |
bo->flags = flags; |
| 687 |
bo->size = size; |
| 688 |
|
| 689 |
if (bo->flags & NOUVEAU_BO_VRAM) |
| 690 |
info->domain |= NOUVEAU_GEM_DOMAIN_VRAM; |
| 691 |
if (bo->flags & NOUVEAU_BO_GART) |
| 692 |
info->domain |= NOUVEAU_GEM_DOMAIN_GART; |
| 693 |
if (!info->domain) |
| 694 |
info->domain |= NOUVEAU_GEM_DOMAIN_VRAM | NOUVEAU_GEM_DOMAIN_GART; |
| 695 |
|
| 696 |
if (bo->flags & NOUVEAU_BO_MAP) |
| 697 |
info->domain |= NOUVEAU_GEM_DOMAIN_MAPPABLE; |
| 698 |
|
| 699 |
if (bo->flags & NOUVEAU_BO_COHERENT) |
| 700 |
info->domain |= NOUVEAU_GEM_DOMAIN_COHERENT; |
| 701 |
|
| 702 |
if (!(bo->flags & NOUVEAU_BO_CONTIG)) |
| 703 |
info->tile_flags = NOUVEAU_GEM_TILE_NONCONTIG; |
| 704 |
|
| 705 |
info->size = bo->size; |
| 706 |
req.align = align; |
| 707 |
|
| 708 |
if (config) { |
| 709 |
if (dev->chipset >= 0xc0) { |
| 710 |
info->tile_flags = (config->nvc0.memtype & 0xff) << 8; |
| 711 |
info->tile_mode = config->nvc0.tile_mode; |
| 712 |
} else |
| 713 |
if (dev->chipset >= 0x80 || dev->chipset == 0x50) { |
| 714 |
info->tile_flags = (config->nv50.memtype & 0x07f) << 8 | |
| 715 |
(config->nv50.memtype & 0x180) << 9; |
| 716 |
info->tile_mode = config->nv50.tile_mode >> 4; |
| 717 |
} |
| 718 |
} |
| 719 |
|
| 720 |
ret = drmCommandWriteRead(drm->fd, DRM_NOUVEAU_GEM_NEW, &req, sizeof(req)); |
| 721 |
if (ret) { |
| 722 |
free(nvbo); |
| 723 |
return ret; |
| 724 |
} |
| 725 |
|
| 726 |
bo_info(bo, info); |
| 727 |
|
| 728 |
*pbo = bo; |
| 729 |
return 0; |
| 730 |
} |
| 731 |
|
| 732 |
static void |
| 733 |
nouveau_bo_del(struct nouveau_bo *bo) |
| 734 |
{ |
| 735 |
struct nouveau_drm *drm = nouveau_drm(&bo->device->object); |
| 736 |
struct nouveau_device_priv *nvdev = nouveau_device(bo->device); |
| 737 |
struct nouveau_bo_priv *nvbo = nouveau_bo(bo); |
| 738 |
|
| 739 |
if (nvbo->head.next) { |
| 740 |
simple_mtx_lock(&nvdev->lock); |
| 741 |
if (p_atomic_read(&nvbo->refcnt) == 0) { |
| 742 |
list_del(&nvbo->head); |
| 743 |
drmCloseBufferHandle(drm->fd, bo->handle); |
| 744 |
} |
| 745 |
simple_mtx_unlock(&nvdev->lock); |
| 746 |
} else { |
| 747 |
drmCloseBufferHandle(drm->fd, bo->handle); |
| 748 |
} |
| 749 |
if (bo->map) |
| 750 |
os_munmap(bo->map, bo->size); |
| 751 |
free(nvbo); |
| 752 |
} |
| 753 |
|
| 754 |
int |
| 755 |
nouveau_bo_map(struct nouveau_bo *bo, uint32_t access, struct nouveau_client *client) |
| 756 |
{ |
| 757 |
struct nouveau_drm *drm = nouveau_drm(&bo->device->object); |
| 758 |
struct nouveau_bo_priv *nvbo = nouveau_bo(bo); |
| 759 |
if (bo->map == NULL) { |
| 760 |
bo->map = os_mmap(0, bo->size, PROT_READ | PROT_WRITE, MAP_SHARED, drm->fd, nvbo->map_handle); |
| 761 |
if (bo->map == MAP_FAILED) { |
| 762 |
bo->map = NULL; |
| 763 |
return -errno; |
| 764 |
} |
| 765 |
} |
| 766 |
return nouveau_bo_wait(bo, access, client); |
| 767 |
} |
| 768 |
|
| 769 |
int |
| 770 |
nouveau_bo_name_get(struct nouveau_bo *bo, uint32_t *name) |
| 771 |
{ |
| 772 |
struct drm_gem_flink req = { .handle = bo->handle }; |
| 773 |
struct nouveau_drm *drm = nouveau_drm(&bo->device->object); |
| 774 |
struct nouveau_bo_priv *nvbo = nouveau_bo(bo); |
| 775 |
|
| 776 |
*name = nvbo->name; |
| 777 |
if (!*name) { |
| 778 |
int ret = drmIoctl(drm->fd, DRM_IOCTL_GEM_FLINK, &req); |
| 779 |
|
| 780 |
if (ret) { |
| 781 |
*name = 0; |
| 782 |
return ret; |
| 783 |
} |
| 784 |
nvbo->name = *name = req.name; |
| 785 |
|
| 786 |
nouveau_bo_make_global(nvbo); |
| 787 |
} |
| 788 |
return 0; |
| 789 |
} |
| 790 |
|
| 791 |
int |
| 792 |
nouveau_bo_name_ref(struct nouveau_device *dev, uint32_t name, struct nouveau_bo **pbo) |
| 793 |
{ |
| 794 |
struct nouveau_drm *drm = nouveau_drm(&dev->object); |
| 795 |
struct nouveau_device_priv *nvdev = nouveau_device(dev); |
| 796 |
struct drm_gem_open req = { .name = name }; |
| 797 |
int ret; |
| 798 |
|
| 799 |
simple_mtx_lock(&nvdev->lock); |
| 800 |
list_for_each_entry(struct nouveau_bo_priv, nvbo, &nvdev->bo_list, head) { |
| 801 |
if (nvbo->name == name) { |
| 802 |
ret = nouveau_bo_wrap_locked(dev, nvbo->base.handle, pbo, name); |
| 803 |
simple_mtx_unlock(&nvdev->lock); |
| 804 |
return ret; |
| 805 |
} |
| 806 |
} |
| 807 |
|
| 808 |
ret = drmIoctl(drm->fd, DRM_IOCTL_GEM_OPEN, &req); |
| 809 |
if (ret == 0) { |
| 810 |
ret = nouveau_bo_wrap_locked(dev, req.handle, pbo, name); |
| 811 |
} |
| 812 |
|
| 813 |
simple_mtx_unlock(&nvdev->lock); |
| 814 |
return ret; |
| 815 |
} |
| 816 |
|
| 817 |
int |
| 818 |
nouveau_bo_prime_handle_ref(struct nouveau_device *dev, int prime_fd, struct nouveau_bo **bo) |
| 819 |
{ |
| 820 |
struct nouveau_drm *drm = nouveau_drm(&dev->object); |
| 821 |
struct nouveau_device_priv *nvdev = nouveau_device(dev); |
| 822 |
int ret; |
| 823 |
unsigned int handle; |
| 824 |
|
| 825 |
nouveau_bo_ref(NULL, bo); |
| 826 |
|
| 827 |
simple_mtx_lock(&nvdev->lock); |
| 828 |
ret = drmPrimeFDToHandle(drm->fd, prime_fd, &handle); |
| 829 |
if (ret == 0) { |
| 830 |
ret = nouveau_bo_wrap_locked(dev, handle, bo, 0); |
| 831 |
} |
| 832 |
simple_mtx_unlock(&nvdev->lock); |
| 833 |
return ret; |
| 834 |
} |
| 835 |
|
| 836 |
void |
| 837 |
nouveau_bo_ref(struct nouveau_bo *bo, struct nouveau_bo **pref) |
| 838 |
{ |
| 839 |
struct nouveau_bo *ref = *pref; |
| 840 |
if (bo) { |
| 841 |
p_atomic_inc(&nouveau_bo(bo)->refcnt); |
| 842 |
} |
| 843 |
if (ref) { |
| 844 |
if (p_atomic_dec_zero(&nouveau_bo(ref)->refcnt)) |
| 845 |
nouveau_bo_del(ref); |
| 846 |
} |
| 847 |
*pref = bo; |
| 848 |
} |
| 849 |
|
| 850 |
int |
| 851 |
nouveau_bo_set_prime(struct nouveau_bo *bo, int *prime_fd) |
| 852 |
{ |
| 853 |
struct nouveau_drm *drm = nouveau_drm(&bo->device->object); |
| 854 |
struct nouveau_bo_priv *nvbo = nouveau_bo(bo); |
| 855 |
int ret; |
| 856 |
|
| 857 |
ret = drmPrimeHandleToFD(drm->fd, nvbo->base.handle, DRM_CLOEXEC | DRM_RDWR, prime_fd); |
| 858 |
if (ret) |
| 859 |
return ret; |
| 860 |
|
| 861 |
nouveau_bo_make_global(nvbo); |
| 862 |
return 0; |
| 863 |
} |
| 864 |
|
| 865 |
static inline struct nouveau_pushbuf * |
| 866 |
cli_push_get(struct nouveau_client *client, struct nouveau_bo *bo) |
| 867 |
{ |
| 868 |
struct nouveau_client_priv *pcli = nouveau_client(client); |
| 869 |
struct nouveau_pushbuf *push = NULL; |
| 870 |
if (pcli->kref_nr > bo->handle) |
| 871 |
push = pcli->kref[bo->handle].push; |
| 872 |
return push; |
| 873 |
} |
| 874 |
|
| 875 |
static inline struct drm_nouveau_gem_pushbuf_bo * |
| 876 |
cli_kref_get(struct nouveau_client *client, struct nouveau_bo *bo) |
| 877 |
{ |
| 878 |
struct nouveau_client_priv *pcli = nouveau_client(client); |
| 879 |
struct drm_nouveau_gem_pushbuf_bo *kref = NULL; |
| 880 |
if (pcli->kref_nr > bo->handle) |
| 881 |
kref = pcli->kref[bo->handle].kref; |
| 882 |
return kref; |
| 883 |
} |
| 884 |
|
| 885 |
static inline int |
| 886 |
cli_kref_set(struct nouveau_client *client, struct nouveau_bo *bo, |
| 887 |
struct drm_nouveau_gem_pushbuf_bo *kref, struct nouveau_pushbuf *push) |
| 888 |
{ |
| 889 |
struct nouveau_client_priv *pcli = nouveau_client(client); |
| 890 |
if (pcli->kref_nr <= bo->handle) { |
| 891 |
void *new_ptr = realloc(pcli->kref, sizeof(*pcli->kref) * bo->handle * 2); |
| 892 |
if (!new_ptr) { |
| 893 |
err("Failed to realloc memory, expect faulty rendering.\n"); |
| 894 |
return -ENOMEM; |
| 895 |
} |
| 896 |
pcli->kref = new_ptr; |
| 897 |
while (pcli->kref_nr < bo->handle * 2) { |
| 898 |
pcli->kref[pcli->kref_nr].kref = NULL; |
| 899 |
pcli->kref[pcli->kref_nr].push = NULL; |
| 900 |
pcli->kref_nr++; |
| 901 |
} |
| 902 |
} |
| 903 |
pcli->kref[bo->handle].kref = kref; |
| 904 |
pcli->kref[bo->handle].push = push; |
| 905 |
return 0; |
| 906 |
} |
| 907 |
|
| 908 |
int |
| 909 |
nouveau_bo_wait(struct nouveau_bo *bo, uint32_t access, struct nouveau_client *client) |
| 910 |
{ |
| 911 |
struct nouveau_drm *drm = nouveau_drm(&bo->device->object); |
| 912 |
struct nouveau_bo_priv *nvbo = nouveau_bo(bo); |
| 913 |
struct drm_nouveau_gem_cpu_prep req; |
| 914 |
struct nouveau_pushbuf *push; |
| 915 |
int ret = 0; |
| 916 |
|
| 917 |
if (!(access & NOUVEAU_BO_RDWR)) |
| 918 |
return 0; |
| 919 |
|
| 920 |
push = cli_push_get(client, bo); |
| 921 |
if (push) |
| 922 |
nouveau_pushbuf_kick(push); |
| 923 |
|
| 924 |
if (!nvbo->head.next && !(nvbo->access & NOUVEAU_BO_WR) && !(access & NOUVEAU_BO_WR)) |
| 925 |
return 0; |
| 926 |
|
| 927 |
req.handle = bo->handle; |
| 928 |
req.flags = 0; |
| 929 |
if (access & NOUVEAU_BO_WR) |
| 930 |
req.flags |= NOUVEAU_GEM_CPU_PREP_WRITE; |
| 931 |
if (access & NOUVEAU_BO_NOBLOCK) |
| 932 |
req.flags |= NOUVEAU_GEM_CPU_PREP_NOWAIT; |
| 933 |
|
| 934 |
ret = drmCommandWrite(drm->fd, DRM_NOUVEAU_GEM_CPU_PREP, &req, sizeof(req)); |
| 935 |
if (ret == 0) |
| 936 |
nvbo->access = 0; |
| 937 |
return ret; |
| 938 |
} |
| 939 |
|
| 940 |
int |
| 941 |
nouveau_bo_wrap(struct nouveau_device *dev, uint32_t handle, struct nouveau_bo **pbo) |
| 942 |
{ |
| 943 |
struct nouveau_device_priv *nvdev = nouveau_device(dev); |
| 944 |
int ret; |
| 945 |
simple_mtx_lock(&nvdev->lock); |
| 946 |
ret = nouveau_bo_wrap_locked(dev, handle, pbo, 0); |
| 947 |
simple_mtx_unlock(&nvdev->lock); |
| 948 |
return ret; |
| 949 |
} |
| 950 |
|
| 951 |
struct nouveau_bufref_priv { |
| 952 |
struct nouveau_bufref base; |
| 953 |
struct nouveau_bufref_priv *next; |
| 954 |
struct nouveau_bufctx *bufctx; |
| 955 |
}; |
| 956 |
|
| 957 |
struct nouveau_bufbin_priv { |
| 958 |
struct nouveau_bufref_priv *list; |
| 959 |
int relocs; |
| 960 |
}; |
| 961 |
|
| 962 |
struct nouveau_bufctx_priv { |
| 963 |
struct nouveau_bufctx base; |
| 964 |
struct nouveau_bufref_priv *free; |
| 965 |
int nr_bins; |
| 966 |
struct nouveau_bufbin_priv bins[]; |
| 967 |
}; |
| 968 |
|
| 969 |
static inline struct nouveau_bufctx_priv * |
| 970 |
nouveau_bufctx(struct nouveau_bufctx *bctx) |
| 971 |
{ |
| 972 |
return (struct nouveau_bufctx_priv *)bctx; |
| 973 |
} |
| 974 |
|
| 975 |
int |
| 976 |
nouveau_bufctx_new(struct nouveau_client *client, int bins, struct nouveau_bufctx **pbctx) |
| 977 |
{ |
| 978 |
struct nouveau_bufctx_priv *priv; |
| 979 |
|
| 980 |
priv = CALLOC_VARIANT_LENGTH_STRUCT(nouveau_bufctx_priv, sizeof(priv->bins[0]) * bins); |
| 981 |
if (priv) { |
| 982 |
list_inithead(&priv->base.head); |
| 983 |
list_inithead(&priv->base.pending); |
| 984 |
list_inithead(&priv->base.current); |
| 985 |
priv->base.client = client; |
| 986 |
priv->nr_bins = bins; |
| 987 |
*pbctx = &priv->base; |
| 988 |
return 0; |
| 989 |
} |
| 990 |
|
| 991 |
return -ENOMEM; |
| 992 |
} |
| 993 |
|
| 994 |
void |
| 995 |
nouveau_bufctx_del(struct nouveau_bufctx **pbctx) |
| 996 |
{ |
| 997 |
struct nouveau_bufctx_priv *pctx = nouveau_bufctx(*pbctx); |
| 998 |
struct nouveau_bufref_priv *pref; |
| 999 |
if (pctx) { |
| 1000 |
while (pctx->nr_bins--) |
| 1001 |
nouveau_bufctx_reset(&pctx->base, pctx->nr_bins); |
| 1002 |
while ((pref = pctx->free)) { |
| 1003 |
pctx->free = pref->next; |
| 1004 |
free(pref); |
| 1005 |
} |
| 1006 |
free(pctx); |
| 1007 |
*pbctx = NULL; |
| 1008 |
} |
| 1009 |
} |
| 1010 |
|
| 1011 |
struct nouveau_bufref * |
| 1012 |
nouveau_bufctx_mthd(struct nouveau_bufctx *bctx, int bin, uint32_t packet, struct nouveau_bo *bo, |
| 1013 |
uint64_t data, uint32_t flags, uint32_t vor, uint32_t tor) |
| 1014 |
{ |
| 1015 |
struct nouveau_bufctx_priv *pctx = nouveau_bufctx(bctx); |
| 1016 |
struct nouveau_bufbin_priv *pbin = &pctx->bins[bin]; |
| 1017 |
struct nouveau_bufref *bref = nouveau_bufctx_refn(bctx, bin, bo, flags); |
| 1018 |
if (bref) { |
| 1019 |
bref->packet = packet; |
| 1020 |
bref->data = data; |
| 1021 |
bref->vor = vor; |
| 1022 |
bref->tor = tor; |
| 1023 |
pbin->relocs++; |
| 1024 |
bctx->relocs++; |
| 1025 |
} |
| 1026 |
return bref; |
| 1027 |
} |
| 1028 |
|
| 1029 |
struct nouveau_bufref * |
| 1030 |
nouveau_bufctx_refn(struct nouveau_bufctx *bctx, int bin, |
| 1031 |
struct nouveau_bo *bo, uint32_t flags) |
| 1032 |
{ |
| 1033 |
struct nouveau_bufctx_priv *pctx = nouveau_bufctx(bctx); |
| 1034 |
struct nouveau_bufbin_priv *pbin = &pctx->bins[bin]; |
| 1035 |
struct nouveau_bufref_priv *pref = pctx->free; |
| 1036 |
|
| 1037 |
if (!pref) |
| 1038 |
pref = malloc(sizeof(*pref)); |
| 1039 |
else |
| 1040 |
pctx->free = pref->next; |
| 1041 |
|
| 1042 |
if (!pref) |
| 1043 |
return NULL; |
| 1044 |
|
| 1045 |
pref->base.bo = bo; |
| 1046 |
pref->base.flags = flags; |
| 1047 |
pref->base.packet = 0; |
| 1048 |
|
| 1049 |
list_addtail(&pref->base.thead, &bctx->pending); |
| 1050 |
pref->bufctx = bctx; |
| 1051 |
pref->next = pbin->list; |
| 1052 |
pbin->list = pref; |
| 1053 |
|
| 1054 |
return &pref->base; |
| 1055 |
} |
| 1056 |
|
| 1057 |
struct nouveau_pushbuf_krec { |
| 1058 |
struct nouveau_pushbuf_krec *next; |
| 1059 |
struct drm_nouveau_gem_pushbuf_bo buffer[NOUVEAU_GEM_MAX_BUFFERS]; |
| 1060 |
struct drm_nouveau_gem_pushbuf_reloc reloc[NOUVEAU_GEM_MAX_RELOCS]; |
| 1061 |
struct drm_nouveau_gem_pushbuf_push push[NOUVEAU_GEM_MAX_PUSH]; |
| 1062 |
int nr_buffer; |
| 1063 |
int nr_reloc; |
| 1064 |
int nr_push; |
| 1065 |
uint64_t vram_used; |
| 1066 |
uint64_t gart_used; |
| 1067 |
}; |
| 1068 |
|
| 1069 |
struct nouveau_pushbuf_priv { |
| 1070 |
struct nouveau_pushbuf base; |
| 1071 |
struct nouveau_pushbuf_krec *list; |
| 1072 |
struct nouveau_pushbuf_krec *krec; |
| 1073 |
struct list_head bctx_list; |
| 1074 |
struct nouveau_bo *bo; |
| 1075 |
uint32_t type; |
| 1076 |
uint32_t suffix0; |
| 1077 |
uint32_t suffix1; |
| 1078 |
uint32_t *ptr; |
| 1079 |
uint32_t *bgn; |
| 1080 |
int bo_next; |
| 1081 |
int bo_nr; |
| 1082 |
struct nouveau_bo *bos[]; |
| 1083 |
}; |
| 1084 |
|
| 1085 |
static inline struct nouveau_pushbuf_priv * |
| 1086 |
nouveau_pushbuf(struct nouveau_pushbuf *push) |
| 1087 |
{ |
| 1088 |
return (struct nouveau_pushbuf_priv *)push; |
| 1089 |
} |
| 1090 |
|
| 1091 |
static void |
| 1092 |
pushbuf_dump(struct nouveau_device *dev, |
| 1093 |
struct nouveau_pushbuf_krec *krec, int krec_id, int chid) |
| 1094 |
{ |
| 1095 |
struct drm_nouveau_gem_pushbuf_reloc *krel; |
| 1096 |
struct drm_nouveau_gem_pushbuf_push *kpsh; |
| 1097 |
struct drm_nouveau_gem_pushbuf_bo *kref; |
| 1098 |
struct nouveau_bo *bo; |
| 1099 |
uint32_t *bgn, *end; |
| 1100 |
int i; |
| 1101 |
|
| 1102 |
err("ch%d: krec %d pushes %d bufs %d relocs %d\n", |
| 1103 |
chid, krec_id, krec->nr_push, krec->nr_buffer, krec->nr_reloc); |
| 1104 |
|
| 1105 |
kref = krec->buffer; |
| 1106 |
for (i = 0; i < krec->nr_buffer; i++, kref++) { |
| 1107 |
bo = (void *)(uintptr_t)kref->user_priv; |
| 1108 |
err("ch%d: buf %08x %08x %08x %08x %08x %p 0x%"PRIx64" 0x%"PRIx64"\n", |
| 1109 |
chid, i, kref->handle, kref->valid_domains, |
| 1110 |
kref->read_domains, kref->write_domains, bo->map, bo->offset, bo->size); |
| 1111 |
} |
| 1112 |
|
| 1113 |
krel = krec->reloc; |
| 1114 |
for (i = 0; i < krec->nr_reloc; i++, krel++) { |
| 1115 |
err("ch%d: rel %08x %08x %08x %08x %08x %08x %08x\n", |
| 1116 |
chid, krel->reloc_bo_index, krel->reloc_bo_offset, |
| 1117 |
krel->bo_index, krel->flags, krel->data, |
| 1118 |
krel->vor, krel->tor); |
| 1119 |
} |
| 1120 |
|
| 1121 |
kpsh = krec->push; |
| 1122 |
for (i = 0; i < krec->nr_push; i++, kpsh++) { |
| 1123 |
kref = krec->buffer + kpsh->bo_index; |
| 1124 |
bo = (void *)(unsigned long)kref->user_priv; |
| 1125 |
bgn = (uint32_t *)((char *)bo->map + kpsh->offset); |
| 1126 |
end = bgn + ((kpsh->length & 0x7fffff) /4); |
| 1127 |
|
| 1128 |
err("ch%d: psh %s%08x %010llx %010llx\n", chid, |
| 1129 |
bo->map ? "" : "(unmapped) ", kpsh->bo_index, |
| 1130 |
(unsigned long long)kpsh->offset, |
| 1131 |
(unsigned long long)(kpsh->offset + kpsh->length)); |
| 1132 |
if (!bo->map) |
| 1133 |
continue; |
| 1134 |
|
| 1135 |
if (dev->info.cls_eng3d) { |
| 1136 |
struct nv_push push = { |
| 1137 |
.start = bgn, |
| 1138 |
.end = end |
| 1139 |
}; |
| 1140 |
vk_push_print(nouveau_out, &push, &dev->info); |
| 1141 |
} else { |
| 1142 |
while (bgn < end) |
| 1143 |
err("\t0x%08x\n", *bgn++); |
| 1144 |
} |
| 1145 |
} |
| 1146 |
} |
| 1147 |
|
| 1148 |
static int |
| 1149 |
pushbuf_submit(struct nouveau_pushbuf *push, struct nouveau_object *chan) |
| 1150 |
{ |
| 1151 |
struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push); |
| 1152 |
struct nouveau_pushbuf_krec *krec = nvpb->list; |
| 1153 |
struct nouveau_device *dev = push->client->device; |
| 1154 |
struct nouveau_drm *drm = nouveau_drm(&dev->object); |
| 1155 |
struct drm_nouveau_gem_pushbuf_bo_presumed *info; |
| 1156 |
struct drm_nouveau_gem_pushbuf_bo *kref; |
| 1157 |
struct drm_nouveau_gem_pushbuf req; |
| 1158 |
int channel = chan->handle; |
| 1159 |
struct nouveau_bo *bo; |
| 1160 |
int krec_id = 0; |
| 1161 |
int ret = 0, i; |
| 1162 |
|
| 1163 |
if (chan->oclass != NOUVEAU_FIFO_CHANNEL_CLASS) |
| 1164 |
return -EINVAL; |
| 1165 |
|
| 1166 |
if (push->kick_notify) |
| 1167 |
push->kick_notify(push); |
| 1168 |
|
| 1169 |
nouveau_pushbuf_data(push, NULL, 0, 0); |
| 1170 |
|
| 1171 |
while (krec && krec->nr_push) { |
| 1172 |
req.channel = channel; |
| 1173 |
req.nr_buffers = krec->nr_buffer; |
| 1174 |
req.buffers = (uint64_t)(unsigned long)krec->buffer; |
| 1175 |
req.nr_relocs = krec->nr_reloc; |
| 1176 |
req.nr_push = krec->nr_push; |
| 1177 |
req.relocs = (uint64_t)(unsigned long)krec->reloc; |
| 1178 |
req.push = (uint64_t)(unsigned long)krec->push; |
| 1179 |
req.suffix0 = nvpb->suffix0; |
| 1180 |
req.suffix1 = nvpb->suffix1; |
| 1181 |
req.vram_available = 0; |
| 1182 |
if (dbg_on(1)) |
| 1183 |
req.vram_available |= NOUVEAU_GEM_PUSHBUF_SYNC; |
| 1184 |
req.gart_available = 0; |
| 1185 |
|
| 1186 |
if (dbg_on(0)) |
| 1187 |
pushbuf_dump(dev, krec, krec_id++, channel); |
| 1188 |
|
| 1189 |
ret = drmCommandWriteRead(drm->fd, DRM_NOUVEAU_GEM_PUSHBUF, &req, sizeof(req)); |
| 1190 |
nvpb->suffix0 = req.suffix0; |
| 1191 |
nvpb->suffix1 = req.suffix1; |
| 1192 |
dev->vram_limit = (req.vram_available * nouveau_device(dev)->vram_limit_percent) / 100; |
| 1193 |
dev->gart_limit = (req.gart_available * nouveau_device(dev)->gart_limit_percent) / 100; |
| 1194 |
|
| 1195 |
if (ret) { |
| 1196 |
err("kernel rejected pushbuf: %s\n", strerror(-ret)); |
| 1197 |
pushbuf_dump(dev, krec, krec_id++, channel); |
| 1198 |
break; |
| 1199 |
} |
| 1200 |
|
| 1201 |
kref = krec->buffer; |
| 1202 |
for (i = 0; i < krec->nr_buffer; i++, kref++) { |
| 1203 |
bo = (void *)(unsigned long)kref->user_priv; |
| 1204 |
|
| 1205 |
info = &kref->presumed; |
| 1206 |
if (!info->valid) { |
| 1207 |
bo->flags &= ~NOUVEAU_BO_APER; |
| 1208 |
if (info->domain == NOUVEAU_GEM_DOMAIN_VRAM) |
| 1209 |
bo->flags |= NOUVEAU_BO_VRAM; |
| 1210 |
else |
| 1211 |
bo->flags |= NOUVEAU_BO_GART; |
| 1212 |
bo->offset = info->offset; |
| 1213 |
} |
| 1214 |
|
| 1215 |
if (kref->write_domains) |
| 1216 |
nouveau_bo(bo)->access |= NOUVEAU_BO_WR; |
| 1217 |
if (kref->read_domains) |
| 1218 |
nouveau_bo(bo)->access |= NOUVEAU_BO_RD; |
| 1219 |
} |
| 1220 |
|
| 1221 |
krec = krec->next; |
| 1222 |
} |
| 1223 |
|
| 1224 |
return ret; |
| 1225 |
} |
| 1226 |
|
| 1227 |
static int |
| 1228 |
pushbuf_flush(struct nouveau_pushbuf *push) |
| 1229 |
{ |
| 1230 |
struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push); |
| 1231 |
struct nouveau_pushbuf_krec *krec = nvpb->krec; |
| 1232 |
struct drm_nouveau_gem_pushbuf_bo *kref; |
| 1233 |
int i; |
| 1234 |
|
| 1235 |
int ret = pushbuf_submit(push, push->channel); |
| 1236 |
|
| 1237 |
kref = krec->buffer; |
| 1238 |
for (i = 0; i < krec->nr_buffer; i++, kref++) { |
| 1239 |
struct nouveau_bo *bo = (void *)(unsigned long)kref->user_priv; |
| 1240 |
ret = cli_kref_set(push->client, bo, NULL, NULL); |
| 1241 |
if (ret) |
| 1242 |
return ret; |
| 1243 |
nouveau_bo_ref(NULL, &bo); |
| 1244 |
} |
| 1245 |
|
| 1246 |
krec = nvpb->krec; |
| 1247 |
krec->vram_used = 0; |
| 1248 |
krec->gart_used = 0; |
| 1249 |
krec->nr_buffer = 0; |
| 1250 |
krec->nr_reloc = 0; |
| 1251 |
krec->nr_push = 0; |
| 1252 |
|
| 1253 |
list_for_each_entry_safe(struct nouveau_bufctx, bctx, &nvpb->bctx_list, head) { |
| 1254 |
list_splice(&bctx->current, &bctx->pending); |
| 1255 |
list_inithead(&bctx->current); |
| 1256 |
list_delinit(&bctx->head); |
| 1257 |
} |
| 1258 |
|
| 1259 |
return ret; |
| 1260 |
} |
| 1261 |
|
| 1262 |
static bool |
| 1263 |
pushbuf_kref_fits(struct nouveau_pushbuf *push, struct nouveau_bo *bo, uint32_t *domains) |
| 1264 |
{ |
| 1265 |
struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push); |
| 1266 |
struct nouveau_pushbuf_krec *krec = nvpb->krec; |
| 1267 |
struct nouveau_device *dev = push->client->device; |
| 1268 |
struct nouveau_bo *kbo; |
| 1269 |
struct drm_nouveau_gem_pushbuf_bo *kref; |
| 1270 |
int i; |
| 1271 |
|
| 1272 |
/* VRAM is the only valid domain. GART and VRAM|GART buffers |
| 1273 |
* are all accounted to GART, so if this doesn't fit in VRAM |
| 1274 |
* straight up, a flush is needed. |
| 1275 |
*/ |
| 1276 |
if (*domains == NOUVEAU_GEM_DOMAIN_VRAM) { |
| 1277 |
if (krec->vram_used + bo->size > dev->vram_limit) |
| 1278 |
return false; |
| 1279 |
krec->vram_used += bo->size; |
| 1280 |
return true; |
| 1281 |
} |
| 1282 |
|
| 1283 |
/* GART or VRAM|GART buffer. Account both of these buffer types |
| 1284 |
* to GART only for the moment, which simplifies things. If the |
| 1285 |
* buffer can fit already, we're done here. |
| 1286 |
*/ |
| 1287 |
if (krec->gart_used + bo->size <= dev->gart_limit) { |
| 1288 |
krec->gart_used += bo->size; |
| 1289 |
return true; |
| 1290 |
} |
| 1291 |
|
| 1292 |
/* Ran out of GART space, if it's a VRAM|GART buffer and it'll |
| 1293 |
* fit into available VRAM, turn it into a VRAM buffer |
| 1294 |
*/ |
| 1295 |
if ((*domains & NOUVEAU_GEM_DOMAIN_VRAM) && |
| 1296 |
krec->vram_used + bo->size <= dev->vram_limit) { |
| 1297 |
*domains &= NOUVEAU_GEM_DOMAIN_VRAM; |
| 1298 |
krec->vram_used += bo->size; |
| 1299 |
return true; |
| 1300 |
} |
| 1301 |
|
| 1302 |
/* Still couldn't fit the buffer in anywhere, so as a last resort; |
| 1303 |
* scan the buffer list for VRAM|GART buffers and turn them into |
| 1304 |
* VRAM buffers until we have enough space in GART for this one |
| 1305 |
*/ |
| 1306 |
kref = krec->buffer; |
| 1307 |
for (i = 0; i < krec->nr_buffer; i++, kref++) { |
| 1308 |
if (!(kref->valid_domains & NOUVEAU_GEM_DOMAIN_GART)) |
| 1309 |
continue; |
| 1310 |
|
| 1311 |
kbo = (void *)(unsigned long)kref->user_priv; |
| 1312 |
if (!(kref->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) || |
| 1313 |
krec->vram_used + kbo->size > dev->vram_limit) |
| 1314 |
continue; |
| 1315 |
|
| 1316 |
kref->valid_domains &= NOUVEAU_GEM_DOMAIN_VRAM; |
| 1317 |
krec->gart_used -= kbo->size; |
| 1318 |
krec->vram_used += kbo->size; |
| 1319 |
if (krec->gart_used + bo->size <= dev->gart_limit) { |
| 1320 |
krec->gart_used += bo->size; |
| 1321 |
return true; |
| 1322 |
} |
| 1323 |
} |
| 1324 |
|
| 1325 |
/* Couldn't resolve a placement, need to force a flush */ |
| 1326 |
return false; |
| 1327 |
} |
| 1328 |
|
| 1329 |
static struct drm_nouveau_gem_pushbuf_bo * |
| 1330 |
pushbuf_kref(struct nouveau_pushbuf *push, struct nouveau_bo *bo, uint32_t flags) |
| 1331 |
{ |
| 1332 |
struct nouveau_device *dev = push->client->device; |
| 1333 |
struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push); |
| 1334 |
struct nouveau_pushbuf_krec *krec = nvpb->krec; |
| 1335 |
struct nouveau_pushbuf *fpush; |
| 1336 |
struct drm_nouveau_gem_pushbuf_bo *kref; |
| 1337 |
uint32_t domains, domains_wr, domains_rd; |
| 1338 |
int ret; |
| 1339 |
|
| 1340 |
domains = 0; |
| 1341 |
if (flags & NOUVEAU_BO_VRAM) |
| 1342 |
domains |= NOUVEAU_GEM_DOMAIN_VRAM; |
| 1343 |
if (flags & NOUVEAU_BO_GART) |
| 1344 |
domains |= NOUVEAU_GEM_DOMAIN_GART; |
| 1345 |
domains_wr = domains * !!(flags & NOUVEAU_BO_WR); |
| 1346 |
domains_rd = domains * !!(flags & NOUVEAU_BO_RD); |
| 1347 |
|
| 1348 |
/* if buffer is referenced on another pushbuf that is owned by the |
| 1349 |
* same client, we need to flush the other pushbuf first to ensure |
| 1350 |
* the correct ordering of commands |
| 1351 |
*/ |
| 1352 |
fpush = cli_push_get(push->client, bo); |
| 1353 |
if (fpush && fpush != push) |
| 1354 |
pushbuf_flush(fpush); |
| 1355 |
|
| 1356 |
kref = cli_kref_get(push->client, bo); |
| 1357 |
if (kref) { |
| 1358 |
/* possible conflict in memory types - flush and retry */ |
| 1359 |
if (!(kref->valid_domains & domains)) |
| 1360 |
return NULL; |
| 1361 |
|
| 1362 |
/* VRAM|GART buffer turning into a VRAM buffer. Make sure |
| 1363 |
* it'll fit in VRAM and force a flush if not. |
| 1364 |
*/ |
| 1365 |
if ((kref->valid_domains & NOUVEAU_GEM_DOMAIN_GART) && |
| 1366 |
( domains == NOUVEAU_GEM_DOMAIN_VRAM)) { |
| 1367 |
if (krec->vram_used + bo->size > dev->vram_limit) |
| 1368 |
return NULL; |
| 1369 |
krec->vram_used += bo->size; |
| 1370 |
krec->gart_used -= bo->size; |
| 1371 |
} |
| 1372 |
|
| 1373 |
kref->valid_domains &= domains; |
| 1374 |
kref->write_domains |= domains_wr; |
| 1375 |
kref->read_domains |= domains_rd; |
| 1376 |
} else { |
| 1377 |
if (krec->nr_buffer == NOUVEAU_GEM_MAX_BUFFERS || |
| 1378 |
!pushbuf_kref_fits(push, bo, &domains)) |
| 1379 |
return NULL; |
| 1380 |
|
| 1381 |
kref = &krec->buffer[krec->nr_buffer++]; |
| 1382 |
kref->user_priv = (unsigned long)bo; |
| 1383 |
kref->handle = bo->handle; |
| 1384 |
kref->valid_domains = domains; |
| 1385 |
kref->write_domains = domains_wr; |
| 1386 |
kref->read_domains = domains_rd; |
| 1387 |
kref->presumed.valid = 1; |
| 1388 |
kref->presumed.offset = bo->offset; |
| 1389 |
if (bo->flags & NOUVEAU_BO_VRAM) |
| 1390 |
kref->presumed.domain = NOUVEAU_GEM_DOMAIN_VRAM; |
| 1391 |
else |
| 1392 |
kref->presumed.domain = NOUVEAU_GEM_DOMAIN_GART; |
| 1393 |
|
| 1394 |
ret = cli_kref_set(push->client, bo, kref, push); |
| 1395 |
if (ret) |
| 1396 |
return NULL; |
| 1397 |
p_atomic_inc(&nouveau_bo(bo)->refcnt); |
| 1398 |
} |
| 1399 |
|
| 1400 |
return kref; |
| 1401 |
} |
| 1402 |
|
| 1403 |
static uint32_t |
| 1404 |
pushbuf_krel(struct nouveau_pushbuf *push, struct nouveau_bo *bo, |
| 1405 |
uint32_t data, uint32_t flags, uint32_t vor, uint32_t tor) |
| 1406 |
{ |
| 1407 |
struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push); |
| 1408 |
struct nouveau_pushbuf_krec *krec = nvpb->krec; |
| 1409 |
struct drm_nouveau_gem_pushbuf_reloc *krel; |
| 1410 |
struct drm_nouveau_gem_pushbuf_bo *pkref; |
| 1411 |
struct drm_nouveau_gem_pushbuf_bo *bkref; |
| 1412 |
uint32_t reloc = data; |
| 1413 |
|
| 1414 |
pkref = cli_kref_get(push->client, nvpb->bo); |
| 1415 |
bkref = cli_kref_get(push->client, bo); |
| 1416 |
krel = &krec->reloc[krec->nr_reloc++]; |
| 1417 |
|
| 1418 |
assert(pkref); |
| 1419 |
assert(bkref); |
| 1420 |
krel->reloc_bo_index = pkref - krec->buffer; |
| 1421 |
krel->reloc_bo_offset = (push->cur - nvpb->ptr) * 4; |
| 1422 |
krel->bo_index = bkref - krec->buffer; |
| 1423 |
krel->flags = 0; |
| 1424 |
krel->data = data; |
| 1425 |
krel->vor = vor; |
| 1426 |
krel->tor = tor; |
| 1427 |
|
| 1428 |
if (flags & NOUVEAU_BO_LOW) { |
| 1429 |
reloc = (bkref->presumed.offset + data); |
| 1430 |
krel->flags |= NOUVEAU_GEM_RELOC_LOW; |
| 1431 |
} |
| 1432 |
if (flags & NOUVEAU_BO_OR) { |
| 1433 |
if (bkref->presumed.domain & NOUVEAU_GEM_DOMAIN_VRAM) |
| 1434 |
reloc |= vor; |
| 1435 |
else |
| 1436 |
reloc |= tor; |
| 1437 |
krel->flags |= NOUVEAU_GEM_RELOC_OR; |
| 1438 |
} |
| 1439 |
|
| 1440 |
return reloc; |
| 1441 |
} |
| 1442 |
|
| 1443 |
static int |
| 1444 |
pushbuf_refn_fail(struct nouveau_pushbuf *push, int sref, int srel) |
| 1445 |
{ |
| 1446 |
struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push); |
| 1447 |
struct nouveau_pushbuf_krec *krec = nvpb->krec; |
| 1448 |
struct drm_nouveau_gem_pushbuf_bo *kref; |
| 1449 |
int ret; |
| 1450 |
|
| 1451 |
kref = krec->buffer + sref; |
| 1452 |
while (krec->nr_buffer-- > sref) { |
| 1453 |
struct nouveau_bo *bo = (void *)(unsigned long)kref->user_priv; |
| 1454 |
ret = cli_kref_set(push->client, bo, NULL, NULL); |
| 1455 |
if (ret) |
| 1456 |
return ret; |
| 1457 |
nouveau_bo_ref(NULL, &bo); |
| 1458 |
kref++; |
| 1459 |
} |
| 1460 |
krec->nr_buffer = sref; |
| 1461 |
krec->nr_reloc = srel; |
| 1462 |
return 0; |
| 1463 |
} |
| 1464 |
|
| 1465 |
static int |
| 1466 |
pushbuf_refn(struct nouveau_pushbuf *push, bool retry, |
| 1467 |
struct nouveau_pushbuf_refn *refs, int nr) |
| 1468 |
{ |
| 1469 |
struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push); |
| 1470 |
struct nouveau_pushbuf_krec *krec = nvpb->krec; |
| 1471 |
struct drm_nouveau_gem_pushbuf_bo *kref; |
| 1472 |
int sref = krec->nr_buffer; |
| 1473 |
int ret = 0, i; |
| 1474 |
|
| 1475 |
for (i = 0; i < nr; i++) { |
| 1476 |
kref = pushbuf_kref(push, refs[i].bo, refs[i].flags); |
| 1477 |
if (!kref) { |
| 1478 |
ret = -ENOSPC; |
| 1479 |
break; |
| 1480 |
} |
| 1481 |
} |
| 1482 |
|
| 1483 |
if (ret) { |
| 1484 |
ret = pushbuf_refn_fail(push, sref, krec->nr_reloc); |
| 1485 |
if (ret) |
| 1486 |
return ret; |
| 1487 |
if (retry) { |
| 1488 |
pushbuf_flush(push); |
| 1489 |
nouveau_pushbuf_space(push, 0, 0, 0); |
| 1490 |
return pushbuf_refn(push, false, refs, nr); |
| 1491 |
} |
| 1492 |
} |
| 1493 |
|
| 1494 |
return ret; |
| 1495 |
} |
| 1496 |
|
| 1497 |
static int |
| 1498 |
pushbuf_validate(struct nouveau_pushbuf *push, bool retry) |
| 1499 |
{ |
| 1500 |
struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push); |
| 1501 |
struct nouveau_pushbuf_krec *krec = nvpb->krec; |
| 1502 |
struct drm_nouveau_gem_pushbuf_bo *kref; |
| 1503 |
struct nouveau_bufctx *bctx = push->bufctx; |
| 1504 |
int relocs = bctx ? bctx->relocs * 2: 0; |
| 1505 |
int sref, srel, ret; |
| 1506 |
|
| 1507 |
ret = nouveau_pushbuf_space(push, relocs, relocs, 0); |
| 1508 |
if (ret || bctx == NULL) |
| 1509 |
return ret; |
| 1510 |
|
| 1511 |
sref = krec->nr_buffer; |
| 1512 |
srel = krec->nr_reloc; |
| 1513 |
|
| 1514 |
list_del(&bctx->head); |
| 1515 |
list_add(&bctx->head, &nvpb->bctx_list); |
| 1516 |
|
| 1517 |
list_for_each_entry(struct nouveau_bufref, bref, &bctx->pending, thead) { |
| 1518 |
kref = pushbuf_kref(push, bref->bo, bref->flags); |
| 1519 |
if (!kref) { |
| 1520 |
ret = -ENOSPC; |
| 1521 |
break; |
| 1522 |
} |
| 1523 |
|
| 1524 |
if (bref->packet) { |
| 1525 |
pushbuf_krel(push, bref->bo, bref->packet, 0, 0, 0); |
| 1526 |
*push->cur++ = 0; |
| 1527 |
pushbuf_krel(push, bref->bo, bref->data, bref->flags, |
| 1528 |
bref->vor, bref->tor); |
| 1529 |
*push->cur++ = 0; |
| 1530 |
} |
| 1531 |
} |
| 1532 |
|
| 1533 |
list_splice(&bctx->pending, &bctx->current); |
| 1534 |
list_inithead(&bctx->pending); |
| 1535 |
|
| 1536 |
if (ret) { |
| 1537 |
ret = pushbuf_refn_fail(push, sref, srel); |
| 1538 |
if (ret) |
| 1539 |
return ret; |
| 1540 |
if (retry) { |
| 1541 |
pushbuf_flush(push); |
| 1542 |
return pushbuf_validate(push, false); |
| 1543 |
} |
| 1544 |
} |
| 1545 |
|
| 1546 |
return ret; |
| 1547 |
} |
| 1548 |
|
| 1549 |
int |
| 1550 |
nouveau_pushbuf_new(struct nouveau_client *client, struct nouveau_object *chan, int nr, |
| 1551 |
uint32_t size, struct nouveau_pushbuf **ppush) |
| 1552 |
{ |
| 1553 |
struct nouveau_drm *drm = nouveau_drm(&client->device->object); |
| 1554 |
struct nouveau_fifo *fifo = chan->data; |
| 1555 |
struct nouveau_pushbuf_priv *nvpb; |
| 1556 |
struct nouveau_pushbuf *push; |
| 1557 |
struct drm_nouveau_gem_pushbuf req = {}; |
| 1558 |
int ret; |
| 1559 |
|
| 1560 |
if (chan->oclass != NOUVEAU_FIFO_CHANNEL_CLASS) |
| 1561 |
return -EINVAL; |
| 1562 |
|
| 1563 |
/* nop pushbuf call, to get the current "return to main" sequence |
| 1564 |
* we need to append to the pushbuf on early chipsets |
| 1565 |
*/ |
| 1566 |
req.channel = chan->handle; |
| 1567 |
req.nr_push = 0; |
| 1568 |
ret = drmCommandWriteRead(drm->fd, DRM_NOUVEAU_GEM_PUSHBUF, &req, sizeof(req)); |
| 1569 |
if (ret) |
| 1570 |
return ret; |
| 1571 |
|
| 1572 |
nvpb = calloc(1, sizeof(*nvpb) + nr * sizeof(*nvpb->bos)); |
| 1573 |
if (!nvpb) |
| 1574 |
return -ENOMEM; |
| 1575 |
|
| 1576 |
nvpb->suffix0 = req.suffix0; |
| 1577 |
nvpb->suffix1 = req.suffix1; |
| 1578 |
nvpb->krec = calloc(1, sizeof(*nvpb->krec)); |
| 1579 |
nvpb->list = nvpb->krec; |
| 1580 |
if (!nvpb->krec) { |
| 1581 |
free(nvpb); |
| 1582 |
return -ENOMEM; |
| 1583 |
} |
| 1584 |
|
| 1585 |
push = &nvpb->base; |
| 1586 |
push->client = client; |
| 1587 |
push->channel = chan; |
| 1588 |
push->flags = NOUVEAU_BO_RD; |
| 1589 |
if (fifo->pushbuf & NOUVEAU_GEM_DOMAIN_GART) { |
| 1590 |
push->flags |= NOUVEAU_BO_GART; |
| 1591 |
nvpb->type = NOUVEAU_BO_GART; |
| 1592 |
} else |
| 1593 |
if (fifo->pushbuf & NOUVEAU_GEM_DOMAIN_VRAM) { |
| 1594 |
push->flags |= NOUVEAU_BO_VRAM; |
| 1595 |
nvpb->type = NOUVEAU_BO_VRAM; |
| 1596 |
} |
| 1597 |
nvpb->type |= NOUVEAU_BO_MAP; |
| 1598 |
|
| 1599 |
for (nvpb->bo_nr = 0; nvpb->bo_nr < nr; nvpb->bo_nr++) { |
| 1600 |
ret = nouveau_bo_new(client->device, nvpb->type, 0, size, NULL, &nvpb->bos[nvpb->bo_nr]); |
| 1601 |
if (ret) { |
| 1602 |
nouveau_pushbuf_del(&push); |
| 1603 |
return ret; |
| 1604 |
} |
| 1605 |
} |
| 1606 |
|
| 1607 |
list_inithead(&nvpb->bctx_list); |
| 1608 |
*ppush = push; |
| 1609 |
return 0; |
| 1610 |
} |
| 1611 |
|
| 1612 |
void |
| 1613 |
nouveau_pushbuf_del(struct nouveau_pushbuf **ppush) |
| 1614 |
{ |
| 1615 |
struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(*ppush); |
| 1616 |
if (nvpb) { |
| 1617 |
struct drm_nouveau_gem_pushbuf_bo *kref; |
| 1618 |
struct nouveau_pushbuf_krec *krec; |
| 1619 |
while ((krec = nvpb->list)) { |
| 1620 |
kref = krec->buffer; |
| 1621 |
while (krec->nr_buffer--) { |
| 1622 |
unsigned long priv = kref++->user_priv; |
| 1623 |
struct nouveau_bo *bo = (void *)priv; |
| 1624 |
cli_kref_set(nvpb->base.client, bo, NULL, NULL); |
| 1625 |
nouveau_bo_ref(NULL, &bo); |
| 1626 |
} |
| 1627 |
nvpb->list = krec->next; |
| 1628 |
free(krec); |
| 1629 |
} |
| 1630 |
while (nvpb->bo_nr--) |
| 1631 |
nouveau_bo_ref(NULL, &nvpb->bos[nvpb->bo_nr]); |
| 1632 |
nouveau_bo_ref(NULL, &nvpb->bo); |
| 1633 |
free(nvpb); |
| 1634 |
} |
| 1635 |
*ppush = NULL; |
| 1636 |
} |
| 1637 |
|
| 1638 |
void |
| 1639 |
nouveau_bufctx_reset(struct nouveau_bufctx *bctx, int bin) |
| 1640 |
{ |
| 1641 |
struct nouveau_bufctx_priv *pctx = nouveau_bufctx(bctx); |
| 1642 |
struct nouveau_bufbin_priv *pbin = &pctx->bins[bin]; |
| 1643 |
struct nouveau_bufref_priv *pref; |
| 1644 |
|
| 1645 |
while ((pref = pbin->list)) { |
| 1646 |
list_delinit(&pref->base.thead); |
| 1647 |
pbin->list = pref->next; |
| 1648 |
pref->next = pctx->free; |
| 1649 |
pctx->free = pref; |
| 1650 |
} |
| 1651 |
|
| 1652 |
bctx->relocs -= pbin->relocs; |
| 1653 |
pbin->relocs = 0; |
| 1654 |
} |
| 1655 |
|
| 1656 |
struct nouveau_bufctx * |
| 1657 |
nouveau_pushbuf_bufctx(struct nouveau_pushbuf *push, struct nouveau_bufctx *ctx) |
| 1658 |
{ |
| 1659 |
struct nouveau_bufctx *prev = push->bufctx; |
| 1660 |
push->bufctx = ctx; |
| 1661 |
return prev; |
| 1662 |
} |
| 1663 |
|
| 1664 |
void |
| 1665 |
nouveau_pushbuf_data(struct nouveau_pushbuf *push, struct nouveau_bo *bo, uint64_t offset, |
| 1666 |
uint64_t length) |
| 1667 |
{ |
| 1668 |
struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push); |
| 1669 |
struct nouveau_pushbuf_krec *krec = nvpb->krec; |
| 1670 |
struct drm_nouveau_gem_pushbuf_push *kpsh; |
| 1671 |
struct drm_nouveau_gem_pushbuf_bo *kref; |
| 1672 |
|
| 1673 |
if (bo != nvpb->bo && nvpb->bgn != push->cur) { |
| 1674 |
if (nvpb->suffix0 || nvpb->suffix1) { |
| 1675 |
*push->cur++ = nvpb->suffix0; |
| 1676 |
*push->cur++ = nvpb->suffix1; |
| 1677 |
} |
| 1678 |
|
| 1679 |
nouveau_pushbuf_data(push, nvpb->bo, (nvpb->bgn - nvpb->ptr) * 4, (push->cur - nvpb->bgn) * 4); |
| 1680 |
nvpb->bgn = push->cur; |
| 1681 |
} |
| 1682 |
|
| 1683 |
if (bo) { |
| 1684 |
kref = cli_kref_get(push->client, bo); |
| 1685 |
assert(kref); |
| 1686 |
kpsh = &krec->push[krec->nr_push++]; |
| 1687 |
kpsh->bo_index = kref - krec->buffer; |
| 1688 |
kpsh->offset = offset; |
| 1689 |
kpsh->length = length; |
| 1690 |
} |
| 1691 |
} |
| 1692 |
|
| 1693 |
int |
| 1694 |
nouveau_pushbuf_kick(struct nouveau_pushbuf *push) |
| 1695 |
{ |
| 1696 |
pushbuf_flush(push); |
| 1697 |
return pushbuf_validate(push, false); |
| 1698 |
} |
| 1699 |
|
| 1700 |
int |
| 1701 |
nouveau_pushbuf_refn(struct nouveau_pushbuf *push, struct nouveau_pushbuf_refn *refs, int nr) |
| 1702 |
{ |
| 1703 |
return pushbuf_refn(push, true, refs, nr); |
| 1704 |
} |
| 1705 |
|
| 1706 |
void |
| 1707 |
nouveau_pushbuf_reloc(struct nouveau_pushbuf *push, struct nouveau_bo *bo, uint32_t data, |
| 1708 |
uint32_t flags, uint32_t vor, uint32_t tor) |
| 1709 |
{ |
| 1710 |
*push->cur = pushbuf_krel(push, bo, data, flags, vor, tor); |
| 1711 |
push->cur++; |
| 1712 |
} |
| 1713 |
|
| 1714 |
int |
| 1715 |
nouveau_pushbuf_space(struct nouveau_pushbuf *push, uint32_t dwords, uint32_t relocs, |
| 1716 |
uint32_t pushes) |
| 1717 |
{ |
| 1718 |
struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push); |
| 1719 |
struct nouveau_pushbuf_krec *krec = nvpb->krec; |
| 1720 |
struct nouveau_client *client = push->client; |
| 1721 |
struct nouveau_bo *bo = NULL; |
| 1722 |
bool flushed = false; |
| 1723 |
int ret = 0; |
| 1724 |
|
| 1725 |
/* switch to next buffer if insufficient space in the current one */ |
| 1726 |
if (push->cur + dwords >= push->end) { |
| 1727 |
if (nvpb->bo_next < nvpb->bo_nr) { |
| 1728 |
nouveau_bo_ref(nvpb->bos[nvpb->bo_next++], &bo); |
| 1729 |
if (nvpb->bo_next == nvpb->bo_nr) |
| 1730 |
nvpb->bo_next = 0; |
| 1731 |
} else { |
| 1732 |
ret = nouveau_bo_new(client->device, nvpb->type, 0, nvpb->bos[0]->size, NULL, &bo); |
| 1733 |
if (ret) |
| 1734 |
return ret; |
| 1735 |
} |
| 1736 |
} |
| 1737 |
|
| 1738 |
/* make sure there's always enough space to queue up the pending |
| 1739 |
* data in the pushbuf proper |
| 1740 |
*/ |
| 1741 |
pushes++; |
| 1742 |
|
| 1743 |
/* need to flush if we've run out of space on an immediate pushbuf, |
| 1744 |
* if the new buffer won't fit, or if the kernel push/reloc limits |
| 1745 |
* have been hit |
| 1746 |
*/ |
| 1747 |
if (bo || |
| 1748 |
krec->nr_reloc + relocs >= NOUVEAU_GEM_MAX_RELOCS || |
| 1749 |
krec->nr_push + pushes >= NOUVEAU_GEM_MAX_PUSH) { |
| 1750 |
if (nvpb->bo && krec->nr_buffer) |
| 1751 |
pushbuf_flush(push); |
| 1752 |
flushed = true; |
| 1753 |
} |
| 1754 |
|
| 1755 |
/* if necessary, switch to new buffer */ |
| 1756 |
if (bo) { |
| 1757 |
ret = nouveau_bo_map(bo, NOUVEAU_BO_WR, push->client); |
| 1758 |
if (ret) |
| 1759 |
return ret; |
| 1760 |
|
| 1761 |
nouveau_pushbuf_data(push, NULL, 0, 0); |
| 1762 |
nouveau_bo_ref(bo, &nvpb->bo); |
| 1763 |
nouveau_bo_ref(NULL, &bo); |
| 1764 |
|
| 1765 |
nvpb->bgn = nvpb->bo->map; |
| 1766 |
nvpb->ptr = nvpb->bgn; |
| 1767 |
push->cur = nvpb->bgn; |
| 1768 |
push->end = push->cur + (nvpb->bo->size / 4); |
| 1769 |
push->end -= 2 + push->rsvd_kick; /* space for suffix */ |
| 1770 |
} |
| 1771 |
|
| 1772 |
pushbuf_kref(push, nvpb->bo, push->flags); |
| 1773 |
return flushed ? pushbuf_validate(push, false) : 0; |
| 1774 |
} |
| 1775 |
|
| 1776 |
int |
| 1777 |
nouveau_pushbuf_validate(struct nouveau_pushbuf *push) |
| 1778 |
{ |
| 1779 |
return pushbuf_validate(push, true); |
| 1780 |
} |