Debian Codegraph @

src/gallium/winsys/nouveau/drm/nouveau.c

raw
1 #include <assert.h>
2 #include <errno.h>
3 #include <fcntl.h>
4 #include <inttypes.h>
5 #include <stdio.h>
6 #include <stdlib.h>
7 #include <xf86drm.h>
8
9 #include "drm-uapi/nouveau_drm.h"
10 #include "nouveau.h"
11 #include "nvif/class.h"
12 #include "nvif/cl0080.h"
13 #include "nvif/ioctl.h"
14
15 #include "nv_push.h"
16
17 #include "util/bitscan.h"
18 #include "util/list.h"
19 #include "util/os_mman.h"
20 #include "util/simple_mtx.h"
21 #include "util/u_atomic.h"
22 #include "util/u_memory.h"
23
24 static FILE *nouveau_out = NULL;
25 static uint32_t nouveau_debug = 0;
26
27 #define dbg_on(lvl) (nouveau_debug & (1 << lvl))
28 #define dbg(lvl, fmt, args...) do { \
29 if (dbg_on((lvl))) \
30 fprintf(nouveau_out, "nouveau: "fmt, ##args); \
31 } while(0)
32 #define err(fmt, args...) fprintf(nouveau_out, "nouveau: "fmt, ##args)
33
34 static void
35 debug_init(void)
36 {
37 static bool once = false;
38 const char *debug, *out;
39
40 if (once)
41 return;
42 once = true;
43
44 debug = os_get_option("NOUVEAU_LIBDRM_DEBUG");
45 if (debug) {
46 int n = strtol(debug, NULL, 0);
47 if (n >= 0)
48 nouveau_debug = n;
49 }
50
51 nouveau_out = stderr;
52 out = os_get_option("NOUVEAU_LIBDRM_OUT");
53 if (out) {
54 FILE *fout = fopen(out, "w");
55 if (fout)
56 nouveau_out = fout;
57 }
58 }
59
60 int
61 nouveau_drm_new(int fd, struct nouveau_drm **pdrm)
62 {
63 debug_init();
64
65 struct nouveau_drm *drm = calloc(1, sizeof(*drm));
66 if (!drm)
67 return -ENOMEM;
68 drm->fd = fd;
69 *pdrm = drm;
70
71 drmVersionPtr ver = drmGetVersion(fd);
72 if (!ver)
73 goto out_err;
74
75 drm->version = (ver->version_major << 24) |
76 (ver->version_minor << 8) |
77 ver->version_patchlevel;
78
79 drmFreeVersion(ver);
80
81 if (drm->version < 0x01000301)
82 goto out_err;
83
84 return 0;
85
86 out_err:
87 nouveau_drm_del(pdrm);
88 return -EINVAL;
89 }
90
91 void
92 nouveau_drm_del(struct nouveau_drm **pdrm)
93 {
94 free(*pdrm);
95 *pdrm = NULL;
96 }
97
98 static int
99 nouveau_object_channel_new(struct nouveau_object *parent, uint64_t handle, uint32_t oclass,
100 struct nouveau_object *obj)
101 {
102 // TODO nv04?
103 struct nouveau_drm *drm = nouveau_drm(parent);
104 struct nouveau_device *dev = (void*)obj->parent;
105 struct nouveau_fifo *fifo = obj->data;
106 struct drm_nouveau_channel_alloc req = { };
107
108 /* nvc0 doesn't need any special handling here */
109 if (dev->chipset < 0xc0) {
110 struct nv04_fifo *nv04 = obj->data;
111 req.fb_ctxdma_handle = nv04->vram;
112 req.tt_ctxdma_handle = nv04->gart;
113 } else if (dev->chipset >= 0xe0) {
114 struct nve0_fifo *nve0 = obj->data;
115 req.fb_ctxdma_handle = 0xffffffff;
116 req.tt_ctxdma_handle = nve0->engine;
117 }
118
119 int ret = drmCommandWriteRead(drm->fd, DRM_NOUVEAU_CHANNEL_ALLOC, &req, sizeof(req));
120 if (ret)
121 return ret;
122
123 fifo->pushbuf = req.pushbuf_domains;
124 fifo->notify = req.notifier_handle;
125 obj->handle = req.channel;
126 return 0;
127 }
128
129 static int
130 nouveau_object_notifier_new(struct nouveau_object *parent, uint64_t handle,
131 struct nouveau_object *obj)
132 {
133 struct nouveau_drm *drm = nouveau_drm(obj);
134 struct nv04_notify *ntfy = obj->data;
135 struct drm_nouveau_notifierobj_alloc req = {
136 .channel = parent->handle,
137 .handle = handle,
138 .size = ntfy->length,
139 };
140
141 int ret = drmCommandWriteRead(drm->fd, DRM_NOUVEAU_NOTIFIEROBJ_ALLOC, &req, sizeof(req));
142 if (ret)
143 return ret;
144
145 ntfy->offset = req.offset;
146 return 0;
147 }
148
149 static int
150 nouveau_object_subchan_new(struct nouveau_object *parent, uint64_t handle, uint32_t oclass,
151 struct nouveau_object *obj)
152 {
153 struct nouveau_drm *drm = nouveau_drm(parent);
154 struct {
155 struct nvif_ioctl_v0 ioctl;
156 struct nvif_ioctl_new_v0 new;
157 } args = {
158 .ioctl = {
159 .route = 0xff,
160 .token = parent->handle,
161 .type = NVIF_IOCTL_V0_NEW,
162 .version = 0,
163 },
164 .new = {
165 .handle = handle,
166 .object = (uintptr_t)obj,
167 .oclass = oclass,
168 .route = NVIF_IOCTL_V0_ROUTE_NVIF,
169 .token = (uintptr_t)obj,
170 .version = 0,
171 },
172 };
173
174 return drmCommandWrite(drm->fd, DRM_NOUVEAU_NVIF, &args, sizeof(args));
175 }
176
177 /* TODO: split this interfaces up so we can verify the right parent object type gets passed in */
178 int
179 nouveau_object_new(struct nouveau_object *parent, uint64_t handle, uint32_t oclass, void *data,
180 uint32_t length, struct nouveau_object **pobj)
181 {
182 struct nouveau_object *obj = calloc(1, sizeof(*obj));
183 if (!obj)
184 return -ENOMEM;
185
186 obj->parent = parent;
187 obj->handle = handle;
188 obj->oclass = oclass;
189 if (length) {
190 obj->data = malloc(length);
191 memcpy(obj->data, data, length);
192 }
193
194 int ret;
195 switch (oclass) {
196 case NOUVEAU_FIFO_CHANNEL_CLASS:
197 ret = nouveau_object_channel_new(parent, handle, oclass, obj);
198 break;
199 case NOUVEAU_NOTIFIER_CLASS:
200 ret = nouveau_object_notifier_new(parent, handle, obj);
201 break;
202 default:
203 ret = nouveau_object_subchan_new(parent, handle, oclass, obj);
204 break;
205 }
206
207 if (ret) {
208 free(obj->data);
209 free(obj);
210 return ret;
211 }
212
213 *pobj = obj;
214 return 0;
215 }
216
217 static void
218 nouveau_object_channel_del(struct nouveau_object *obj)
219 {
220 struct nouveau_drm *drm = nouveau_drm(obj->parent);
221 struct drm_nouveau_channel_free req = {
222 .channel = obj->handle,
223 };
224
225 int ret = drmCommandWrite(drm->fd, DRM_NOUVEAU_CHANNEL_FREE, &req, sizeof(req));
226 assert(!ret);
227 }
228
229 static void
230 nouveau_object_subchan_del(struct nouveau_object *obj)
231 {
232 struct {
233 struct nvif_ioctl_v0 ioctl;
234 struct nvif_ioctl_del del;
235 } args = {
236 .ioctl = {
237 .object = (uintptr_t)obj,
238 .owner = NVIF_IOCTL_V0_OWNER_ANY,
239 .route = 0x00,
240 .type = NVIF_IOCTL_V0_DEL,
241 .version = 0,
242 },
243 };
244
245 drmCommandWrite(obj->parent->handle, DRM_NOUVEAU_NVIF, &args, sizeof(args));
246 }
247
248 static void
249 nouveau_object_gpuobj_del(struct nouveau_object *obj)
250 {
251 struct nouveau_drm *drm = nouveau_drm(obj->parent);
252 struct drm_nouveau_gpuobj_free req = {
253 .channel = obj->parent->handle,
254 .handle = obj->handle,
255 };
256
257 drmCommandWrite(drm->fd, DRM_NOUVEAU_GPUOBJ_FREE, &req, sizeof(req));
258 }
259
260 void
261 nouveau_object_del(struct nouveau_object **pobj)
262 {
263 if (!*pobj)
264 return;
265
266 struct nouveau_object *obj = *pobj;
267 switch (obj->oclass) {
268 case NOUVEAU_FIFO_CHANNEL_CLASS:
269 nouveau_object_channel_del(obj);
270 break;
271 case NOUVEAU_NOTIFIER_CLASS:
272 nouveau_object_gpuobj_del(obj);
273 break;
274 default:
275 nouveau_object_subchan_del(obj);
276 break;
277 }
278 free(obj->data);
279 free(obj);
280 *pobj = NULL;
281 }
282
283 #define NOUVEAU_OBJECT_MAX_CLASSES 16
284 int
285 nouveau_object_mclass(struct nouveau_object *obj, const struct nouveau_mclass *mclass)
286 {
287 struct nouveau_drm *drm = nouveau_drm(obj->parent);
288 struct {
289 struct nvif_ioctl_v0 ioctl;
290 struct nvif_ioctl_sclass_v0 sclass;
291 struct nvif_ioctl_sclass_oclass_v0 list[NOUVEAU_OBJECT_MAX_CLASSES];
292 } args = {
293 .ioctl = {
294 .route = 0xff,
295 .token = obj->handle,
296 .type = NVIF_IOCTL_V0_SCLASS,
297 .version = 0,
298 },
299 .sclass = {
300 .count = NOUVEAU_OBJECT_MAX_CLASSES,
301 .version = 0,
302 },
303 };
304
305 int ret = drmCommandWriteRead(drm->fd, DRM_NOUVEAU_NVIF, &args, sizeof(args));
306 if (ret)
307 return ret;
308
309 int i;
310 for (i = 0; mclass[i].oclass; i++) {
311 for (int j = 0; j < args.sclass.count; j++) {
312 if (args.list[j].oclass == mclass[i].oclass)
313 return i;
314 }
315 }
316
317 return -ENODEV;
318 }
319
320 struct nouveau_device_priv {
321 struct nouveau_device base;
322 simple_mtx_t lock;
323 struct list_head bo_list;
324 int gart_limit_percent;
325 int vram_limit_percent;
326 };
327
328 static inline struct nouveau_device_priv *
329 nouveau_device(struct nouveau_device *dev)
330 {
331 return (struct nouveau_device_priv *)dev;
332 }
333
334 int
335 nouveau_device_info(struct nouveau_device *dev, struct nv_device_info_v0 *info)
336 {
337 struct nouveau_drm *drm = nouveau_drm(dev->object.parent);
338 struct {
339 struct nvif_ioctl_v0 ioctl;
340 struct nvif_ioctl_mthd_v0 mthd;
341 struct nv_device_info_v0 info;
342 } args = {
343 .ioctl = {
344 .object = (uintptr_t)dev,
345 .owner = NVIF_IOCTL_V0_OWNER_ANY,
346 .route = 0x00,
347 .type = NVIF_IOCTL_V0_MTHD,
348 .version = 0,
349 },
350 .mthd = {
351 .method = NV_DEVICE_V0_INFO,
352 .version = 0,
353 },
354 .info = {
355 .version = 0,
356 },
357 };
358
359 int ret = drmCommandWriteRead(drm->fd, DRM_NOUVEAU_NVIF, &args, sizeof(args));
360 if (ret)
361 return ret;
362
363 *info = args.info;
364 return 0;
365 }
366
367 int
368 nouveau_device_new(struct nouveau_object *parent, struct nouveau_device **pdev)
369 {
370 struct nouveau_drm *drm = nouveau_drm(parent);
371 struct nouveau_device *dev;
372 uint64_t v;
373 const char *tmp;
374
375 struct nouveau_device_priv *nvdev = calloc(1, sizeof(*nvdev));
376 if (!nvdev)
377 return -ENOMEM;
378 dev = *pdev = &nvdev->base;
379 dev->object.parent = parent;
380
381 struct {
382 struct nvif_ioctl_v0 ioctl;
383 struct nvif_ioctl_new_v0 new;
384 struct nv_device_v0 dev;
385 } args = {
386 .ioctl = {
387 .object = 0,
388 .owner = NVIF_IOCTL_V0_OWNER_ANY,
389 .route = 0x00,
390 .type = NVIF_IOCTL_V0_NEW,
391 .version = 0,
392 },
393 .new = {
394 .handle = 0,
395 .object = (uintptr_t)&nvdev->base.object,
396 .oclass = NV_DEVICE,
397 .route = NVIF_IOCTL_V0_ROUTE_NVIF,
398 .token = (uintptr_t)&nvdev->base.object,
399 .version = 0,
400 },
401 .dev = {
402 .device = ~0ULL,
403 },
404 };
405
406 int ret = drmCommandWrite(drm->fd, DRM_NOUVEAU_NVIF, &args, sizeof(args));
407 if (ret)
408 goto done;
409
410 struct nv_device_info_v0 info;
411 ret = nouveau_device_info(dev, &info);
412 if (ret)
413 goto done;
414
415 nvdev->base.chipset = info.chipset;
416 nvdev->base.info.chipset = info.chipset;
417 switch (info.platform) {
418 case NV_DEVICE_INFO_V0_PCI:
419 case NV_DEVICE_INFO_V0_AGP:
420 case NV_DEVICE_INFO_V0_PCIE:
421 nvdev->base.info.type = NV_DEVICE_TYPE_DIS;
422 break;
423 case NV_DEVICE_INFO_V0_IGP:
424 nvdev->base.info.type = NV_DEVICE_TYPE_IGP;
425 break;
426 case NV_DEVICE_INFO_V0_SOC:
427 nvdev->base.info.type = NV_DEVICE_TYPE_SOC;
428 break;
429 default:
430 UNREACHABLE("unhandled nvidia device type");
431 break;
432 }
433
434 drmDevicePtr drm_device;
435 ret = drmGetDevice2(drm->fd, 0, &drm_device);
436 if (ret)
437 goto done;
438
439 if (drm_device->bustype == DRM_BUS_PCI) {
440 nvdev->base.info.pci.domain = drm_device->businfo.pci->domain;
441 nvdev->base.info.pci.bus = drm_device->businfo.pci->bus;
442 nvdev->base.info.pci.dev = drm_device->businfo.pci->dev;
443 nvdev->base.info.pci.func = drm_device->businfo.pci->func;
444 nvdev->base.info.pci.revision_id = drm_device->deviceinfo.pci->revision_id;
445 nvdev->base.info.device_id = drm_device->deviceinfo.pci->device_id;
446 }
447
448 drmFreeDevice(&drm_device);
449
450 ret = nouveau_getparam(dev, NOUVEAU_GETPARAM_FB_SIZE, &v);
451 if (ret)
452 goto done;
453 nvdev->base.vram_size = v;
454
455 ret = nouveau_getparam(dev, NOUVEAU_GETPARAM_AGP_SIZE, &v);
456 if (ret)
457 goto done;
458 nvdev->base.gart_size = v;
459
460 tmp = os_get_option("NOUVEAU_LIBDRM_VRAM_LIMIT_PERCENT");
461 if (tmp)
462 nvdev->vram_limit_percent = atoi(tmp);
463 else
464 nvdev->vram_limit_percent = 80;
465 nvdev->base.vram_limit = (nvdev->base.vram_size * nvdev->vram_limit_percent) / 100;
466
467 tmp = os_get_option("NOUVEAU_LIBDRM_GART_LIMIT_PERCENT");
468 if (tmp)
469 nvdev->gart_limit_percent = atoi(tmp);
470 else
471 nvdev->gart_limit_percent = 80;
472 nvdev->base.gart_limit = (nvdev->base.gart_size * nvdev->gart_limit_percent) / 100;
473
474 simple_mtx_init(&nvdev->lock, mtx_plain);
475 list_inithead(&nvdev->bo_list);
476 done:
477 if (ret)
478 nouveau_device_del(pdev);
479 return ret;
480 }
481
482 void
483 nouveau_device_set_classes_for_debug(struct nouveau_device *dev,
484 uint32_t cls_eng3d,
485 uint32_t cls_compute,
486 uint32_t cls_m2mf,
487 uint32_t cls_copy)
488 {
489 dev->info.cls_eng3d = cls_eng3d;
490 dev->info.cls_compute = cls_compute;
491 dev->info.cls_m2mf = cls_m2mf;
492 dev->info.cls_copy = cls_copy;
493 }
494
495 void
496 nouveau_device_del(struct nouveau_device **pdev)
497 {
498 struct nouveau_device_priv *nvdev = nouveau_device(*pdev);
499 if (!nvdev)
500 return;
501
502 simple_mtx_destroy(&nvdev->lock);
503 free(nvdev);
504 *pdev = NULL;
505 }
506
507 int
508 nouveau_getparam(struct nouveau_device *dev, uint64_t param, uint64_t *value)
509 {
510 struct nouveau_drm *drm = nouveau_drm(&dev->object);
511 struct drm_nouveau_getparam r = { .param = param };
512 int ret = drmCommandWriteRead(drm->fd, DRM_NOUVEAU_GETPARAM, &r, sizeof(r));
513 *value = r.value;
514 return ret;
515 }
516
517 struct nouveau_client_kref {
518 struct drm_nouveau_gem_pushbuf_bo *kref;
519 struct nouveau_pushbuf *push;
520 };
521
522 struct nouveau_client_priv {
523 struct nouveau_client base;
524 struct nouveau_client_kref *kref;
525 unsigned kref_nr;
526 };
527
528 static inline struct nouveau_client_priv *
529 nouveau_client(struct nouveau_client *client)
530 {
531 return (struct nouveau_client_priv *)client;
532 }
533
534 int
535 nouveau_client_new(struct nouveau_device *dev, struct nouveau_client **pclient)
536 {
537 struct nouveau_client_priv *pcli = calloc(1, sizeof(*pcli));
538 if (!pcli)
539 return -ENOMEM;
540
541 pcli->base.device = dev;
542 *pclient = &pcli->base;
543
544 return 0;
545 }
546
547 void
548 nouveau_client_del(struct nouveau_client **pclient)
549 {
550 struct nouveau_client_priv *pcli = nouveau_client(*pclient);
551 if (pcli) {
552 free(pcli->kref);
553 free(pcli);
554 }
555 }
556
557 struct nouveau_bo_priv {
558 struct nouveau_bo base;
559 struct list_head head;
560 uint32_t refcnt;
561 uint64_t map_handle;
562 uint32_t name;
563 uint32_t access;
564 };
565
566 static inline struct nouveau_bo_priv *
567 nouveau_bo(struct nouveau_bo *bo)
568 {
569 return (struct nouveau_bo_priv *)bo;
570 }
571
572 static void
573 bo_info(struct nouveau_bo *bo, struct drm_nouveau_gem_info *info)
574 {
575 struct nouveau_bo_priv *nvbo = nouveau_bo(bo);
576
577 nvbo->map_handle = info->map_handle;
578 bo->handle = info->handle;
579 bo->size = info->size;
580 bo->offset = info->offset;
581
582 bo->flags = 0;
583 if (info->domain & NOUVEAU_GEM_DOMAIN_VRAM)
584 bo->flags |= NOUVEAU_BO_VRAM;
585 if (info->domain & NOUVEAU_GEM_DOMAIN_GART)
586 bo->flags |= NOUVEAU_BO_GART;
587 if (!(info->tile_flags & NOUVEAU_GEM_TILE_NONCONTIG))
588 bo->flags |= NOUVEAU_BO_CONTIG;
589 if (nvbo->map_handle)
590 bo->flags |= NOUVEAU_BO_MAP;
591
592 if (bo->device->chipset >= 0xc0) {
593 bo->config.nvc0.memtype = (info->tile_flags & 0xff00) >> 8;
594 bo->config.nvc0.tile_mode = info->tile_mode;
595 } else
596 if (bo->device->chipset >= 0x80 || bo->device->chipset == 0x50) {
597 bo->config.nv50.memtype = (info->tile_flags & 0x07f00) >> 8 |
598 (info->tile_flags & 0x30000) >> 9;
599 bo->config.nv50.tile_mode = info->tile_mode << 4;
600 } else {
601 //bo->config.nv04.surf_flags = info->tile_flags & 7;
602 //bo->config.nv04.surf_pitch = info->tile_mode;
603 }
604 }
605
606 static void
607 nouveau_bo_make_global(struct nouveau_bo_priv *nvbo)
608 {
609 if (!nvbo->head.next) {
610 struct nouveau_device_priv *nvdev = nouveau_device(nvbo->base.device);
611 simple_mtx_lock(&nvdev->lock);
612 if (!nvbo->head.next)
613 list_add(&nvbo->head, &nvdev->bo_list);
614 simple_mtx_unlock(&nvdev->lock);
615 }
616 }
617
618 static int
619 nouveau_bo_wrap_locked(struct nouveau_device *dev, uint32_t handle,
620 struct nouveau_bo **pbo, int name)
621 {
622 struct nouveau_drm *drm = nouveau_drm(&dev->object);
623 struct nouveau_device_priv *nvdev = nouveau_device(dev);
624 struct drm_nouveau_gem_info req = { .handle = handle };
625 struct nouveau_bo_priv *nvbo;
626 int ret;
627
628 list_for_each_entry(struct nouveau_bo_priv, nvbo, &nvdev->bo_list, head) {
629 if (nvbo->base.handle == handle) {
630 if (p_atomic_inc_return(&nvbo->refcnt) == 1) {
631 /*
632 * Uh oh, this bo is dead and someone else
633 * will free it, but because refcnt is
634 * now non-zero fortunately they won't
635 * call the ioctl to close the bo.
636 *
637 * Remove this bo from the list so other
638 * calls to nouveau_bo_wrap_locked will
639 * see our replacement nvbo.
640 */
641 list_del(&nvbo->head);
642 if (!name)
643 name = nvbo->name;
644 break;
645 }
646
647 *pbo = &nvbo->base;
648 return 0;
649 }
650 }
651
652 ret = drmCommandWriteRead(drm->fd, DRM_NOUVEAU_GEM_INFO, &req, sizeof(req));
653 if (ret)
654 return ret;
655
656 nvbo = calloc(1, sizeof(*nvbo));
657 if (nvbo) {
658 p_atomic_set(&nvbo->refcnt, 1);
659 nvbo->base.device = dev;
660 bo_info(&nvbo->base, &req);
661 nvbo->name = name;
662 list_add(&nvbo->head, &nvdev->bo_list);
663 *pbo = &nvbo->base;
664 return 0;
665 }
666
667 return -ENOMEM;
668 }
669
670 int
671 nouveau_bo_new(struct nouveau_device *dev, uint32_t flags, uint32_t align, uint64_t size,
672 union nouveau_bo_config *config, struct nouveau_bo **pbo)
673 {
674 struct nouveau_drm *drm = nouveau_drm(&dev->object);
675 struct drm_nouveau_gem_new req = {};
676 struct drm_nouveau_gem_info *info = &req.info;
677 int ret;
678
679 struct nouveau_bo_priv *nvbo = calloc(1, sizeof(*nvbo));
680 if (!nvbo)
681 return -ENOMEM;
682
683 struct nouveau_bo *bo = &nvbo->base;
684 p_atomic_set(&nvbo->refcnt, 1);
685 bo->device = dev;
686 bo->flags = flags;
687 bo->size = size;
688
689 if (bo->flags & NOUVEAU_BO_VRAM)
690 info->domain |= NOUVEAU_GEM_DOMAIN_VRAM;
691 if (bo->flags & NOUVEAU_BO_GART)
692 info->domain |= NOUVEAU_GEM_DOMAIN_GART;
693 if (!info->domain)
694 info->domain |= NOUVEAU_GEM_DOMAIN_VRAM | NOUVEAU_GEM_DOMAIN_GART;
695
696 if (bo->flags & NOUVEAU_BO_MAP)
697 info->domain |= NOUVEAU_GEM_DOMAIN_MAPPABLE;
698
699 if (bo->flags & NOUVEAU_BO_COHERENT)
700 info->domain |= NOUVEAU_GEM_DOMAIN_COHERENT;
701
702 if (!(bo->flags & NOUVEAU_BO_CONTIG))
703 info->tile_flags = NOUVEAU_GEM_TILE_NONCONTIG;
704
705 info->size = bo->size;
706 req.align = align;
707
708 if (config) {
709 if (dev->chipset >= 0xc0) {
710 info->tile_flags = (config->nvc0.memtype & 0xff) << 8;
711 info->tile_mode = config->nvc0.tile_mode;
712 } else
713 if (dev->chipset >= 0x80 || dev->chipset == 0x50) {
714 info->tile_flags = (config->nv50.memtype & 0x07f) << 8 |
715 (config->nv50.memtype & 0x180) << 9;
716 info->tile_mode = config->nv50.tile_mode >> 4;
717 }
718 }
719
720 ret = drmCommandWriteRead(drm->fd, DRM_NOUVEAU_GEM_NEW, &req, sizeof(req));
721 if (ret) {
722 free(nvbo);
723 return ret;
724 }
725
726 bo_info(bo, info);
727
728 *pbo = bo;
729 return 0;
730 }
731
732 static void
733 nouveau_bo_del(struct nouveau_bo *bo)
734 {
735 struct nouveau_drm *drm = nouveau_drm(&bo->device->object);
736 struct nouveau_device_priv *nvdev = nouveau_device(bo->device);
737 struct nouveau_bo_priv *nvbo = nouveau_bo(bo);
738
739 if (nvbo->head.next) {
740 simple_mtx_lock(&nvdev->lock);
741 if (p_atomic_read(&nvbo->refcnt) == 0) {
742 list_del(&nvbo->head);
743 drmCloseBufferHandle(drm->fd, bo->handle);
744 }
745 simple_mtx_unlock(&nvdev->lock);
746 } else {
747 drmCloseBufferHandle(drm->fd, bo->handle);
748 }
749 if (bo->map)
750 os_munmap(bo->map, bo->size);
751 free(nvbo);
752 }
753
754 int
755 nouveau_bo_map(struct nouveau_bo *bo, uint32_t access, struct nouveau_client *client)
756 {
757 struct nouveau_drm *drm = nouveau_drm(&bo->device->object);
758 struct nouveau_bo_priv *nvbo = nouveau_bo(bo);
759 if (bo->map == NULL) {
760 bo->map = os_mmap(0, bo->size, PROT_READ | PROT_WRITE, MAP_SHARED, drm->fd, nvbo->map_handle);
761 if (bo->map == MAP_FAILED) {
762 bo->map = NULL;
763 return -errno;
764 }
765 }
766 return nouveau_bo_wait(bo, access, client);
767 }
768
769 int
770 nouveau_bo_name_get(struct nouveau_bo *bo, uint32_t *name)
771 {
772 struct drm_gem_flink req = { .handle = bo->handle };
773 struct nouveau_drm *drm = nouveau_drm(&bo->device->object);
774 struct nouveau_bo_priv *nvbo = nouveau_bo(bo);
775
776 *name = nvbo->name;
777 if (!*name) {
778 int ret = drmIoctl(drm->fd, DRM_IOCTL_GEM_FLINK, &req);
779
780 if (ret) {
781 *name = 0;
782 return ret;
783 }
784 nvbo->name = *name = req.name;
785
786 nouveau_bo_make_global(nvbo);
787 }
788 return 0;
789 }
790
791 int
792 nouveau_bo_name_ref(struct nouveau_device *dev, uint32_t name, struct nouveau_bo **pbo)
793 {
794 struct nouveau_drm *drm = nouveau_drm(&dev->object);
795 struct nouveau_device_priv *nvdev = nouveau_device(dev);
796 struct drm_gem_open req = { .name = name };
797 int ret;
798
799 simple_mtx_lock(&nvdev->lock);
800 list_for_each_entry(struct nouveau_bo_priv, nvbo, &nvdev->bo_list, head) {
801 if (nvbo->name == name) {
802 ret = nouveau_bo_wrap_locked(dev, nvbo->base.handle, pbo, name);
803 simple_mtx_unlock(&nvdev->lock);
804 return ret;
805 }
806 }
807
808 ret = drmIoctl(drm->fd, DRM_IOCTL_GEM_OPEN, &req);
809 if (ret == 0) {
810 ret = nouveau_bo_wrap_locked(dev, req.handle, pbo, name);
811 }
812
813 simple_mtx_unlock(&nvdev->lock);
814 return ret;
815 }
816
817 int
818 nouveau_bo_prime_handle_ref(struct nouveau_device *dev, int prime_fd, struct nouveau_bo **bo)
819 {
820 struct nouveau_drm *drm = nouveau_drm(&dev->object);
821 struct nouveau_device_priv *nvdev = nouveau_device(dev);
822 int ret;
823 unsigned int handle;
824
825 nouveau_bo_ref(NULL, bo);
826
827 simple_mtx_lock(&nvdev->lock);
828 ret = drmPrimeFDToHandle(drm->fd, prime_fd, &handle);
829 if (ret == 0) {
830 ret = nouveau_bo_wrap_locked(dev, handle, bo, 0);
831 }
832 simple_mtx_unlock(&nvdev->lock);
833 return ret;
834 }
835
836 void
837 nouveau_bo_ref(struct nouveau_bo *bo, struct nouveau_bo **pref)
838 {
839 struct nouveau_bo *ref = *pref;
840 if (bo) {
841 p_atomic_inc(&nouveau_bo(bo)->refcnt);
842 }
843 if (ref) {
844 if (p_atomic_dec_zero(&nouveau_bo(ref)->refcnt))
845 nouveau_bo_del(ref);
846 }
847 *pref = bo;
848 }
849
850 int
851 nouveau_bo_set_prime(struct nouveau_bo *bo, int *prime_fd)
852 {
853 struct nouveau_drm *drm = nouveau_drm(&bo->device->object);
854 struct nouveau_bo_priv *nvbo = nouveau_bo(bo);
855 int ret;
856
857 ret = drmPrimeHandleToFD(drm->fd, nvbo->base.handle, DRM_CLOEXEC | DRM_RDWR, prime_fd);
858 if (ret)
859 return ret;
860
861 nouveau_bo_make_global(nvbo);
862 return 0;
863 }
864
865 static inline struct nouveau_pushbuf *
866 cli_push_get(struct nouveau_client *client, struct nouveau_bo *bo)
867 {
868 struct nouveau_client_priv *pcli = nouveau_client(client);
869 struct nouveau_pushbuf *push = NULL;
870 if (pcli->kref_nr > bo->handle)
871 push = pcli->kref[bo->handle].push;
872 return push;
873 }
874
875 static inline struct drm_nouveau_gem_pushbuf_bo *
876 cli_kref_get(struct nouveau_client *client, struct nouveau_bo *bo)
877 {
878 struct nouveau_client_priv *pcli = nouveau_client(client);
879 struct drm_nouveau_gem_pushbuf_bo *kref = NULL;
880 if (pcli->kref_nr > bo->handle)
881 kref = pcli->kref[bo->handle].kref;
882 return kref;
883 }
884
885 static inline int
886 cli_kref_set(struct nouveau_client *client, struct nouveau_bo *bo,
887 struct drm_nouveau_gem_pushbuf_bo *kref, struct nouveau_pushbuf *push)
888 {
889 struct nouveau_client_priv *pcli = nouveau_client(client);
890 if (pcli->kref_nr <= bo->handle) {
891 void *new_ptr = realloc(pcli->kref, sizeof(*pcli->kref) * bo->handle * 2);
892 if (!new_ptr) {
893 err("Failed to realloc memory, expect faulty rendering.\n");
894 return -ENOMEM;
895 }
896 pcli->kref = new_ptr;
897 while (pcli->kref_nr < bo->handle * 2) {
898 pcli->kref[pcli->kref_nr].kref = NULL;
899 pcli->kref[pcli->kref_nr].push = NULL;
900 pcli->kref_nr++;
901 }
902 }
903 pcli->kref[bo->handle].kref = kref;
904 pcli->kref[bo->handle].push = push;
905 return 0;
906 }
907
908 int
909 nouveau_bo_wait(struct nouveau_bo *bo, uint32_t access, struct nouveau_client *client)
910 {
911 struct nouveau_drm *drm = nouveau_drm(&bo->device->object);
912 struct nouveau_bo_priv *nvbo = nouveau_bo(bo);
913 struct drm_nouveau_gem_cpu_prep req;
914 struct nouveau_pushbuf *push;
915 int ret = 0;
916
917 if (!(access & NOUVEAU_BO_RDWR))
918 return 0;
919
920 push = cli_push_get(client, bo);
921 if (push)
922 nouveau_pushbuf_kick(push);
923
924 if (!nvbo->head.next && !(nvbo->access & NOUVEAU_BO_WR) && !(access & NOUVEAU_BO_WR))
925 return 0;
926
927 req.handle = bo->handle;
928 req.flags = 0;
929 if (access & NOUVEAU_BO_WR)
930 req.flags |= NOUVEAU_GEM_CPU_PREP_WRITE;
931 if (access & NOUVEAU_BO_NOBLOCK)
932 req.flags |= NOUVEAU_GEM_CPU_PREP_NOWAIT;
933
934 ret = drmCommandWrite(drm->fd, DRM_NOUVEAU_GEM_CPU_PREP, &req, sizeof(req));
935 if (ret == 0)
936 nvbo->access = 0;
937 return ret;
938 }
939
940 int
941 nouveau_bo_wrap(struct nouveau_device *dev, uint32_t handle, struct nouveau_bo **pbo)
942 {
943 struct nouveau_device_priv *nvdev = nouveau_device(dev);
944 int ret;
945 simple_mtx_lock(&nvdev->lock);
946 ret = nouveau_bo_wrap_locked(dev, handle, pbo, 0);
947 simple_mtx_unlock(&nvdev->lock);
948 return ret;
949 }
950
951 struct nouveau_bufref_priv {
952 struct nouveau_bufref base;
953 struct nouveau_bufref_priv *next;
954 struct nouveau_bufctx *bufctx;
955 };
956
957 struct nouveau_bufbin_priv {
958 struct nouveau_bufref_priv *list;
959 int relocs;
960 };
961
962 struct nouveau_bufctx_priv {
963 struct nouveau_bufctx base;
964 struct nouveau_bufref_priv *free;
965 int nr_bins;
966 struct nouveau_bufbin_priv bins[];
967 };
968
969 static inline struct nouveau_bufctx_priv *
970 nouveau_bufctx(struct nouveau_bufctx *bctx)
971 {
972 return (struct nouveau_bufctx_priv *)bctx;
973 }
974
975 int
976 nouveau_bufctx_new(struct nouveau_client *client, int bins, struct nouveau_bufctx **pbctx)
977 {
978 struct nouveau_bufctx_priv *priv;
979
980 priv = CALLOC_VARIANT_LENGTH_STRUCT(nouveau_bufctx_priv, sizeof(priv->bins[0]) * bins);
981 if (priv) {
982 list_inithead(&priv->base.head);
983 list_inithead(&priv->base.pending);
984 list_inithead(&priv->base.current);
985 priv->base.client = client;
986 priv->nr_bins = bins;
987 *pbctx = &priv->base;
988 return 0;
989 }
990
991 return -ENOMEM;
992 }
993
994 void
995 nouveau_bufctx_del(struct nouveau_bufctx **pbctx)
996 {
997 struct nouveau_bufctx_priv *pctx = nouveau_bufctx(*pbctx);
998 struct nouveau_bufref_priv *pref;
999 if (pctx) {
1000 while (pctx->nr_bins--)
1001 nouveau_bufctx_reset(&pctx->base, pctx->nr_bins);
1002 while ((pref = pctx->free)) {
1003 pctx->free = pref->next;
1004 free(pref);
1005 }
1006 free(pctx);
1007 *pbctx = NULL;
1008 }
1009 }
1010
1011 struct nouveau_bufref *
1012 nouveau_bufctx_mthd(struct nouveau_bufctx *bctx, int bin, uint32_t packet, struct nouveau_bo *bo,
1013 uint64_t data, uint32_t flags, uint32_t vor, uint32_t tor)
1014 {
1015 struct nouveau_bufctx_priv *pctx = nouveau_bufctx(bctx);
1016 struct nouveau_bufbin_priv *pbin = &pctx->bins[bin];
1017 struct nouveau_bufref *bref = nouveau_bufctx_refn(bctx, bin, bo, flags);
1018 if (bref) {
1019 bref->packet = packet;
1020 bref->data = data;
1021 bref->vor = vor;
1022 bref->tor = tor;
1023 pbin->relocs++;
1024 bctx->relocs++;
1025 }
1026 return bref;
1027 }
1028
1029 struct nouveau_bufref *
1030 nouveau_bufctx_refn(struct nouveau_bufctx *bctx, int bin,
1031 struct nouveau_bo *bo, uint32_t flags)
1032 {
1033 struct nouveau_bufctx_priv *pctx = nouveau_bufctx(bctx);
1034 struct nouveau_bufbin_priv *pbin = &pctx->bins[bin];
1035 struct nouveau_bufref_priv *pref = pctx->free;
1036
1037 if (!pref)
1038 pref = malloc(sizeof(*pref));
1039 else
1040 pctx->free = pref->next;
1041
1042 if (!pref)
1043 return NULL;
1044
1045 pref->base.bo = bo;
1046 pref->base.flags = flags;
1047 pref->base.packet = 0;
1048
1049 list_addtail(&pref->base.thead, &bctx->pending);
1050 pref->bufctx = bctx;
1051 pref->next = pbin->list;
1052 pbin->list = pref;
1053
1054 return &pref->base;
1055 }
1056
1057 struct nouveau_pushbuf_krec {
1058 struct nouveau_pushbuf_krec *next;
1059 struct drm_nouveau_gem_pushbuf_bo buffer[NOUVEAU_GEM_MAX_BUFFERS];
1060 struct drm_nouveau_gem_pushbuf_reloc reloc[NOUVEAU_GEM_MAX_RELOCS];
1061 struct drm_nouveau_gem_pushbuf_push push[NOUVEAU_GEM_MAX_PUSH];
1062 int nr_buffer;
1063 int nr_reloc;
1064 int nr_push;
1065 uint64_t vram_used;
1066 uint64_t gart_used;
1067 };
1068
1069 struct nouveau_pushbuf_priv {
1070 struct nouveau_pushbuf base;
1071 struct nouveau_pushbuf_krec *list;
1072 struct nouveau_pushbuf_krec *krec;
1073 struct list_head bctx_list;
1074 struct nouveau_bo *bo;
1075 uint32_t type;
1076 uint32_t suffix0;
1077 uint32_t suffix1;
1078 uint32_t *ptr;
1079 uint32_t *bgn;
1080 int bo_next;
1081 int bo_nr;
1082 struct nouveau_bo *bos[];
1083 };
1084
1085 static inline struct nouveau_pushbuf_priv *
1086 nouveau_pushbuf(struct nouveau_pushbuf *push)
1087 {
1088 return (struct nouveau_pushbuf_priv *)push;
1089 }
1090
1091 static void
1092 pushbuf_dump(struct nouveau_device *dev,
1093 struct nouveau_pushbuf_krec *krec, int krec_id, int chid)
1094 {
1095 struct drm_nouveau_gem_pushbuf_reloc *krel;
1096 struct drm_nouveau_gem_pushbuf_push *kpsh;
1097 struct drm_nouveau_gem_pushbuf_bo *kref;
1098 struct nouveau_bo *bo;
1099 uint32_t *bgn, *end;
1100 int i;
1101
1102 err("ch%d: krec %d pushes %d bufs %d relocs %d\n",
1103 chid, krec_id, krec->nr_push, krec->nr_buffer, krec->nr_reloc);
1104
1105 kref = krec->buffer;
1106 for (i = 0; i < krec->nr_buffer; i++, kref++) {
1107 bo = (void *)(uintptr_t)kref->user_priv;
1108 err("ch%d: buf %08x %08x %08x %08x %08x %p 0x%"PRIx64" 0x%"PRIx64"\n",
1109 chid, i, kref->handle, kref->valid_domains,
1110 kref->read_domains, kref->write_domains, bo->map, bo->offset, bo->size);
1111 }
1112
1113 krel = krec->reloc;
1114 for (i = 0; i < krec->nr_reloc; i++, krel++) {
1115 err("ch%d: rel %08x %08x %08x %08x %08x %08x %08x\n",
1116 chid, krel->reloc_bo_index, krel->reloc_bo_offset,
1117 krel->bo_index, krel->flags, krel->data,
1118 krel->vor, krel->tor);
1119 }
1120
1121 kpsh = krec->push;
1122 for (i = 0; i < krec->nr_push; i++, kpsh++) {
1123 kref = krec->buffer + kpsh->bo_index;
1124 bo = (void *)(unsigned long)kref->user_priv;
1125 bgn = (uint32_t *)((char *)bo->map + kpsh->offset);
1126 end = bgn + ((kpsh->length & 0x7fffff) /4);
1127
1128 err("ch%d: psh %s%08x %010llx %010llx\n", chid,
1129 bo->map ? "" : "(unmapped) ", kpsh->bo_index,
1130 (unsigned long long)kpsh->offset,
1131 (unsigned long long)(kpsh->offset + kpsh->length));
1132 if (!bo->map)
1133 continue;
1134
1135 if (dev->info.cls_eng3d) {
1136 struct nv_push push = {
1137 .start = bgn,
1138 .end = end
1139 };
1140 vk_push_print(nouveau_out, &push, &dev->info);
1141 } else {
1142 while (bgn < end)
1143 err("\t0x%08x\n", *bgn++);
1144 }
1145 }
1146 }
1147
1148 static int
1149 pushbuf_submit(struct nouveau_pushbuf *push, struct nouveau_object *chan)
1150 {
1151 struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push);
1152 struct nouveau_pushbuf_krec *krec = nvpb->list;
1153 struct nouveau_device *dev = push->client->device;
1154 struct nouveau_drm *drm = nouveau_drm(&dev->object);
1155 struct drm_nouveau_gem_pushbuf_bo_presumed *info;
1156 struct drm_nouveau_gem_pushbuf_bo *kref;
1157 struct drm_nouveau_gem_pushbuf req;
1158 int channel = chan->handle;
1159 struct nouveau_bo *bo;
1160 int krec_id = 0;
1161 int ret = 0, i;
1162
1163 if (chan->oclass != NOUVEAU_FIFO_CHANNEL_CLASS)
1164 return -EINVAL;
1165
1166 if (push->kick_notify)
1167 push->kick_notify(push);
1168
1169 nouveau_pushbuf_data(push, NULL, 0, 0);
1170
1171 while (krec && krec->nr_push) {
1172 req.channel = channel;
1173 req.nr_buffers = krec->nr_buffer;
1174 req.buffers = (uint64_t)(unsigned long)krec->buffer;
1175 req.nr_relocs = krec->nr_reloc;
1176 req.nr_push = krec->nr_push;
1177 req.relocs = (uint64_t)(unsigned long)krec->reloc;
1178 req.push = (uint64_t)(unsigned long)krec->push;
1179 req.suffix0 = nvpb->suffix0;
1180 req.suffix1 = nvpb->suffix1;
1181 req.vram_available = 0;
1182 if (dbg_on(1))
1183 req.vram_available |= NOUVEAU_GEM_PUSHBUF_SYNC;
1184 req.gart_available = 0;
1185
1186 if (dbg_on(0))
1187 pushbuf_dump(dev, krec, krec_id++, channel);
1188
1189 ret = drmCommandWriteRead(drm->fd, DRM_NOUVEAU_GEM_PUSHBUF, &req, sizeof(req));
1190 nvpb->suffix0 = req.suffix0;
1191 nvpb->suffix1 = req.suffix1;
1192 dev->vram_limit = (req.vram_available * nouveau_device(dev)->vram_limit_percent) / 100;
1193 dev->gart_limit = (req.gart_available * nouveau_device(dev)->gart_limit_percent) / 100;
1194
1195 if (ret) {
1196 err("kernel rejected pushbuf: %s\n", strerror(-ret));
1197 pushbuf_dump(dev, krec, krec_id++, channel);
1198 break;
1199 }
1200
1201 kref = krec->buffer;
1202 for (i = 0; i < krec->nr_buffer; i++, kref++) {
1203 bo = (void *)(unsigned long)kref->user_priv;
1204
1205 info = &kref->presumed;
1206 if (!info->valid) {
1207 bo->flags &= ~NOUVEAU_BO_APER;
1208 if (info->domain == NOUVEAU_GEM_DOMAIN_VRAM)
1209 bo->flags |= NOUVEAU_BO_VRAM;
1210 else
1211 bo->flags |= NOUVEAU_BO_GART;
1212 bo->offset = info->offset;
1213 }
1214
1215 if (kref->write_domains)
1216 nouveau_bo(bo)->access |= NOUVEAU_BO_WR;
1217 if (kref->read_domains)
1218 nouveau_bo(bo)->access |= NOUVEAU_BO_RD;
1219 }
1220
1221 krec = krec->next;
1222 }
1223
1224 return ret;
1225 }
1226
1227 static int
1228 pushbuf_flush(struct nouveau_pushbuf *push)
1229 {
1230 struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push);
1231 struct nouveau_pushbuf_krec *krec = nvpb->krec;
1232 struct drm_nouveau_gem_pushbuf_bo *kref;
1233 int i;
1234
1235 int ret = pushbuf_submit(push, push->channel);
1236
1237 kref = krec->buffer;
1238 for (i = 0; i < krec->nr_buffer; i++, kref++) {
1239 struct nouveau_bo *bo = (void *)(unsigned long)kref->user_priv;
1240 ret = cli_kref_set(push->client, bo, NULL, NULL);
1241 if (ret)
1242 return ret;
1243 nouveau_bo_ref(NULL, &bo);
1244 }
1245
1246 krec = nvpb->krec;
1247 krec->vram_used = 0;
1248 krec->gart_used = 0;
1249 krec->nr_buffer = 0;
1250 krec->nr_reloc = 0;
1251 krec->nr_push = 0;
1252
1253 list_for_each_entry_safe(struct nouveau_bufctx, bctx, &nvpb->bctx_list, head) {
1254 list_splice(&bctx->current, &bctx->pending);
1255 list_inithead(&bctx->current);
1256 list_delinit(&bctx->head);
1257 }
1258
1259 return ret;
1260 }
1261
1262 static bool
1263 pushbuf_kref_fits(struct nouveau_pushbuf *push, struct nouveau_bo *bo, uint32_t *domains)
1264 {
1265 struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push);
1266 struct nouveau_pushbuf_krec *krec = nvpb->krec;
1267 struct nouveau_device *dev = push->client->device;
1268 struct nouveau_bo *kbo;
1269 struct drm_nouveau_gem_pushbuf_bo *kref;
1270 int i;
1271
1272 /* VRAM is the only valid domain. GART and VRAM|GART buffers
1273 * are all accounted to GART, so if this doesn't fit in VRAM
1274 * straight up, a flush is needed.
1275 */
1276 if (*domains == NOUVEAU_GEM_DOMAIN_VRAM) {
1277 if (krec->vram_used + bo->size > dev->vram_limit)
1278 return false;
1279 krec->vram_used += bo->size;
1280 return true;
1281 }
1282
1283 /* GART or VRAM|GART buffer. Account both of these buffer types
1284 * to GART only for the moment, which simplifies things. If the
1285 * buffer can fit already, we're done here.
1286 */
1287 if (krec->gart_used + bo->size <= dev->gart_limit) {
1288 krec->gart_used += bo->size;
1289 return true;
1290 }
1291
1292 /* Ran out of GART space, if it's a VRAM|GART buffer and it'll
1293 * fit into available VRAM, turn it into a VRAM buffer
1294 */
1295 if ((*domains & NOUVEAU_GEM_DOMAIN_VRAM) &&
1296 krec->vram_used + bo->size <= dev->vram_limit) {
1297 *domains &= NOUVEAU_GEM_DOMAIN_VRAM;
1298 krec->vram_used += bo->size;
1299 return true;
1300 }
1301
1302 /* Still couldn't fit the buffer in anywhere, so as a last resort;
1303 * scan the buffer list for VRAM|GART buffers and turn them into
1304 * VRAM buffers until we have enough space in GART for this one
1305 */
1306 kref = krec->buffer;
1307 for (i = 0; i < krec->nr_buffer; i++, kref++) {
1308 if (!(kref->valid_domains & NOUVEAU_GEM_DOMAIN_GART))
1309 continue;
1310
1311 kbo = (void *)(unsigned long)kref->user_priv;
1312 if (!(kref->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) ||
1313 krec->vram_used + kbo->size > dev->vram_limit)
1314 continue;
1315
1316 kref->valid_domains &= NOUVEAU_GEM_DOMAIN_VRAM;
1317 krec->gart_used -= kbo->size;
1318 krec->vram_used += kbo->size;
1319 if (krec->gart_used + bo->size <= dev->gart_limit) {
1320 krec->gart_used += bo->size;
1321 return true;
1322 }
1323 }
1324
1325 /* Couldn't resolve a placement, need to force a flush */
1326 return false;
1327 }
1328
1329 static struct drm_nouveau_gem_pushbuf_bo *
1330 pushbuf_kref(struct nouveau_pushbuf *push, struct nouveau_bo *bo, uint32_t flags)
1331 {
1332 struct nouveau_device *dev = push->client->device;
1333 struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push);
1334 struct nouveau_pushbuf_krec *krec = nvpb->krec;
1335 struct nouveau_pushbuf *fpush;
1336 struct drm_nouveau_gem_pushbuf_bo *kref;
1337 uint32_t domains, domains_wr, domains_rd;
1338 int ret;
1339
1340 domains = 0;
1341 if (flags & NOUVEAU_BO_VRAM)
1342 domains |= NOUVEAU_GEM_DOMAIN_VRAM;
1343 if (flags & NOUVEAU_BO_GART)
1344 domains |= NOUVEAU_GEM_DOMAIN_GART;
1345 domains_wr = domains * !!(flags & NOUVEAU_BO_WR);
1346 domains_rd = domains * !!(flags & NOUVEAU_BO_RD);
1347
1348 /* if buffer is referenced on another pushbuf that is owned by the
1349 * same client, we need to flush the other pushbuf first to ensure
1350 * the correct ordering of commands
1351 */
1352 fpush = cli_push_get(push->client, bo);
1353 if (fpush && fpush != push)
1354 pushbuf_flush(fpush);
1355
1356 kref = cli_kref_get(push->client, bo);
1357 if (kref) {
1358 /* possible conflict in memory types - flush and retry */
1359 if (!(kref->valid_domains & domains))
1360 return NULL;
1361
1362 /* VRAM|GART buffer turning into a VRAM buffer. Make sure
1363 * it'll fit in VRAM and force a flush if not.
1364 */
1365 if ((kref->valid_domains & NOUVEAU_GEM_DOMAIN_GART) &&
1366 ( domains == NOUVEAU_GEM_DOMAIN_VRAM)) {
1367 if (krec->vram_used + bo->size > dev->vram_limit)
1368 return NULL;
1369 krec->vram_used += bo->size;
1370 krec->gart_used -= bo->size;
1371 }
1372
1373 kref->valid_domains &= domains;
1374 kref->write_domains |= domains_wr;
1375 kref->read_domains |= domains_rd;
1376 } else {
1377 if (krec->nr_buffer == NOUVEAU_GEM_MAX_BUFFERS ||
1378 !pushbuf_kref_fits(push, bo, &domains))
1379 return NULL;
1380
1381 kref = &krec->buffer[krec->nr_buffer++];
1382 kref->user_priv = (unsigned long)bo;
1383 kref->handle = bo->handle;
1384 kref->valid_domains = domains;
1385 kref->write_domains = domains_wr;
1386 kref->read_domains = domains_rd;
1387 kref->presumed.valid = 1;
1388 kref->presumed.offset = bo->offset;
1389 if (bo->flags & NOUVEAU_BO_VRAM)
1390 kref->presumed.domain = NOUVEAU_GEM_DOMAIN_VRAM;
1391 else
1392 kref->presumed.domain = NOUVEAU_GEM_DOMAIN_GART;
1393
1394 ret = cli_kref_set(push->client, bo, kref, push);
1395 if (ret)
1396 return NULL;
1397 p_atomic_inc(&nouveau_bo(bo)->refcnt);
1398 }
1399
1400 return kref;
1401 }
1402
1403 static uint32_t
1404 pushbuf_krel(struct nouveau_pushbuf *push, struct nouveau_bo *bo,
1405 uint32_t data, uint32_t flags, uint32_t vor, uint32_t tor)
1406 {
1407 struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push);
1408 struct nouveau_pushbuf_krec *krec = nvpb->krec;
1409 struct drm_nouveau_gem_pushbuf_reloc *krel;
1410 struct drm_nouveau_gem_pushbuf_bo *pkref;
1411 struct drm_nouveau_gem_pushbuf_bo *bkref;
1412 uint32_t reloc = data;
1413
1414 pkref = cli_kref_get(push->client, nvpb->bo);
1415 bkref = cli_kref_get(push->client, bo);
1416 krel = &krec->reloc[krec->nr_reloc++];
1417
1418 assert(pkref);
1419 assert(bkref);
1420 krel->reloc_bo_index = pkref - krec->buffer;
1421 krel->reloc_bo_offset = (push->cur - nvpb->ptr) * 4;
1422 krel->bo_index = bkref - krec->buffer;
1423 krel->flags = 0;
1424 krel->data = data;
1425 krel->vor = vor;
1426 krel->tor = tor;
1427
1428 if (flags & NOUVEAU_BO_LOW) {
1429 reloc = (bkref->presumed.offset + data);
1430 krel->flags |= NOUVEAU_GEM_RELOC_LOW;
1431 }
1432 if (flags & NOUVEAU_BO_OR) {
1433 if (bkref->presumed.domain & NOUVEAU_GEM_DOMAIN_VRAM)
1434 reloc |= vor;
1435 else
1436 reloc |= tor;
1437 krel->flags |= NOUVEAU_GEM_RELOC_OR;
1438 }
1439
1440 return reloc;
1441 }
1442
1443 static int
1444 pushbuf_refn_fail(struct nouveau_pushbuf *push, int sref, int srel)
1445 {
1446 struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push);
1447 struct nouveau_pushbuf_krec *krec = nvpb->krec;
1448 struct drm_nouveau_gem_pushbuf_bo *kref;
1449 int ret;
1450
1451 kref = krec->buffer + sref;
1452 while (krec->nr_buffer-- > sref) {
1453 struct nouveau_bo *bo = (void *)(unsigned long)kref->user_priv;
1454 ret = cli_kref_set(push->client, bo, NULL, NULL);
1455 if (ret)
1456 return ret;
1457 nouveau_bo_ref(NULL, &bo);
1458 kref++;
1459 }
1460 krec->nr_buffer = sref;
1461 krec->nr_reloc = srel;
1462 return 0;
1463 }
1464
1465 static int
1466 pushbuf_refn(struct nouveau_pushbuf *push, bool retry,
1467 struct nouveau_pushbuf_refn *refs, int nr)
1468 {
1469 struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push);
1470 struct nouveau_pushbuf_krec *krec = nvpb->krec;
1471 struct drm_nouveau_gem_pushbuf_bo *kref;
1472 int sref = krec->nr_buffer;
1473 int ret = 0, i;
1474
1475 for (i = 0; i < nr; i++) {
1476 kref = pushbuf_kref(push, refs[i].bo, refs[i].flags);
1477 if (!kref) {
1478 ret = -ENOSPC;
1479 break;
1480 }
1481 }
1482
1483 if (ret) {
1484 ret = pushbuf_refn_fail(push, sref, krec->nr_reloc);
1485 if (ret)
1486 return ret;
1487 if (retry) {
1488 pushbuf_flush(push);
1489 nouveau_pushbuf_space(push, 0, 0, 0);
1490 return pushbuf_refn(push, false, refs, nr);
1491 }
1492 }
1493
1494 return ret;
1495 }
1496
1497 static int
1498 pushbuf_validate(struct nouveau_pushbuf *push, bool retry)
1499 {
1500 struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push);
1501 struct nouveau_pushbuf_krec *krec = nvpb->krec;
1502 struct drm_nouveau_gem_pushbuf_bo *kref;
1503 struct nouveau_bufctx *bctx = push->bufctx;
1504 int relocs = bctx ? bctx->relocs * 2: 0;
1505 int sref, srel, ret;
1506
1507 ret = nouveau_pushbuf_space(push, relocs, relocs, 0);
1508 if (ret || bctx == NULL)
1509 return ret;
1510
1511 sref = krec->nr_buffer;
1512 srel = krec->nr_reloc;
1513
1514 list_del(&bctx->head);
1515 list_add(&bctx->head, &nvpb->bctx_list);
1516
1517 list_for_each_entry(struct nouveau_bufref, bref, &bctx->pending, thead) {
1518 kref = pushbuf_kref(push, bref->bo, bref->flags);
1519 if (!kref) {
1520 ret = -ENOSPC;
1521 break;
1522 }
1523
1524 if (bref->packet) {
1525 pushbuf_krel(push, bref->bo, bref->packet, 0, 0, 0);
1526 *push->cur++ = 0;
1527 pushbuf_krel(push, bref->bo, bref->data, bref->flags,
1528 bref->vor, bref->tor);
1529 *push->cur++ = 0;
1530 }
1531 }
1532
1533 list_splice(&bctx->pending, &bctx->current);
1534 list_inithead(&bctx->pending);
1535
1536 if (ret) {
1537 ret = pushbuf_refn_fail(push, sref, srel);
1538 if (ret)
1539 return ret;
1540 if (retry) {
1541 pushbuf_flush(push);
1542 return pushbuf_validate(push, false);
1543 }
1544 }
1545
1546 return ret;
1547 }
1548
1549 int
1550 nouveau_pushbuf_new(struct nouveau_client *client, struct nouveau_object *chan, int nr,
1551 uint32_t size, struct nouveau_pushbuf **ppush)
1552 {
1553 struct nouveau_drm *drm = nouveau_drm(&client->device->object);
1554 struct nouveau_fifo *fifo = chan->data;
1555 struct nouveau_pushbuf_priv *nvpb;
1556 struct nouveau_pushbuf *push;
1557 struct drm_nouveau_gem_pushbuf req = {};
1558 int ret;
1559
1560 if (chan->oclass != NOUVEAU_FIFO_CHANNEL_CLASS)
1561 return -EINVAL;
1562
1563 /* nop pushbuf call, to get the current "return to main" sequence
1564 * we need to append to the pushbuf on early chipsets
1565 */
1566 req.channel = chan->handle;
1567 req.nr_push = 0;
1568 ret = drmCommandWriteRead(drm->fd, DRM_NOUVEAU_GEM_PUSHBUF, &req, sizeof(req));
1569 if (ret)
1570 return ret;
1571
1572 nvpb = calloc(1, sizeof(*nvpb) + nr * sizeof(*nvpb->bos));
1573 if (!nvpb)
1574 return -ENOMEM;
1575
1576 nvpb->suffix0 = req.suffix0;
1577 nvpb->suffix1 = req.suffix1;
1578 nvpb->krec = calloc(1, sizeof(*nvpb->krec));
1579 nvpb->list = nvpb->krec;
1580 if (!nvpb->krec) {
1581 free(nvpb);
1582 return -ENOMEM;
1583 }
1584
1585 push = &nvpb->base;
1586 push->client = client;
1587 push->channel = chan;
1588 push->flags = NOUVEAU_BO_RD;
1589 if (fifo->pushbuf & NOUVEAU_GEM_DOMAIN_GART) {
1590 push->flags |= NOUVEAU_BO_GART;
1591 nvpb->type = NOUVEAU_BO_GART;
1592 } else
1593 if (fifo->pushbuf & NOUVEAU_GEM_DOMAIN_VRAM) {
1594 push->flags |= NOUVEAU_BO_VRAM;
1595 nvpb->type = NOUVEAU_BO_VRAM;
1596 }
1597 nvpb->type |= NOUVEAU_BO_MAP;
1598
1599 for (nvpb->bo_nr = 0; nvpb->bo_nr < nr; nvpb->bo_nr++) {
1600 ret = nouveau_bo_new(client->device, nvpb->type, 0, size, NULL, &nvpb->bos[nvpb->bo_nr]);
1601 if (ret) {
1602 nouveau_pushbuf_del(&push);
1603 return ret;
1604 }
1605 }
1606
1607 list_inithead(&nvpb->bctx_list);
1608 *ppush = push;
1609 return 0;
1610 }
1611
1612 void
1613 nouveau_pushbuf_del(struct nouveau_pushbuf **ppush)
1614 {
1615 struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(*ppush);
1616 if (nvpb) {
1617 struct drm_nouveau_gem_pushbuf_bo *kref;
1618 struct nouveau_pushbuf_krec *krec;
1619 while ((krec = nvpb->list)) {
1620 kref = krec->buffer;
1621 while (krec->nr_buffer--) {
1622 unsigned long priv = kref++->user_priv;
1623 struct nouveau_bo *bo = (void *)priv;
1624 cli_kref_set(nvpb->base.client, bo, NULL, NULL);
1625 nouveau_bo_ref(NULL, &bo);
1626 }
1627 nvpb->list = krec->next;
1628 free(krec);
1629 }
1630 while (nvpb->bo_nr--)
1631 nouveau_bo_ref(NULL, &nvpb->bos[nvpb->bo_nr]);
1632 nouveau_bo_ref(NULL, &nvpb->bo);
1633 free(nvpb);
1634 }
1635 *ppush = NULL;
1636 }
1637
1638 void
1639 nouveau_bufctx_reset(struct nouveau_bufctx *bctx, int bin)
1640 {
1641 struct nouveau_bufctx_priv *pctx = nouveau_bufctx(bctx);
1642 struct nouveau_bufbin_priv *pbin = &pctx->bins[bin];
1643 struct nouveau_bufref_priv *pref;
1644
1645 while ((pref = pbin->list)) {
1646 list_delinit(&pref->base.thead);
1647 pbin->list = pref->next;
1648 pref->next = pctx->free;
1649 pctx->free = pref;
1650 }
1651
1652 bctx->relocs -= pbin->relocs;
1653 pbin->relocs = 0;
1654 }
1655
1656 struct nouveau_bufctx *
1657 nouveau_pushbuf_bufctx(struct nouveau_pushbuf *push, struct nouveau_bufctx *ctx)
1658 {
1659 struct nouveau_bufctx *prev = push->bufctx;
1660 push->bufctx = ctx;
1661 return prev;
1662 }
1663
1664 void
1665 nouveau_pushbuf_data(struct nouveau_pushbuf *push, struct nouveau_bo *bo, uint64_t offset,
1666 uint64_t length)
1667 {
1668 struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push);
1669 struct nouveau_pushbuf_krec *krec = nvpb->krec;
1670 struct drm_nouveau_gem_pushbuf_push *kpsh;
1671 struct drm_nouveau_gem_pushbuf_bo *kref;
1672
1673 if (bo != nvpb->bo && nvpb->bgn != push->cur) {
1674 if (nvpb->suffix0 || nvpb->suffix1) {
1675 *push->cur++ = nvpb->suffix0;
1676 *push->cur++ = nvpb->suffix1;
1677 }
1678
1679 nouveau_pushbuf_data(push, nvpb->bo, (nvpb->bgn - nvpb->ptr) * 4, (push->cur - nvpb->bgn) * 4);
1680 nvpb->bgn = push->cur;
1681 }
1682
1683 if (bo) {
1684 kref = cli_kref_get(push->client, bo);
1685 assert(kref);
1686 kpsh = &krec->push[krec->nr_push++];
1687 kpsh->bo_index = kref - krec->buffer;
1688 kpsh->offset = offset;
1689 kpsh->length = length;
1690 }
1691 }
1692
1693 int
1694 nouveau_pushbuf_kick(struct nouveau_pushbuf *push)
1695 {
1696 pushbuf_flush(push);
1697 return pushbuf_validate(push, false);
1698 }
1699
1700 int
1701 nouveau_pushbuf_refn(struct nouveau_pushbuf *push, struct nouveau_pushbuf_refn *refs, int nr)
1702 {
1703 return pushbuf_refn(push, true, refs, nr);
1704 }
1705
1706 void
1707 nouveau_pushbuf_reloc(struct nouveau_pushbuf *push, struct nouveau_bo *bo, uint32_t data,
1708 uint32_t flags, uint32_t vor, uint32_t tor)
1709 {
1710 *push->cur = pushbuf_krel(push, bo, data, flags, vor, tor);
1711 push->cur++;
1712 }
1713
1714 int
1715 nouveau_pushbuf_space(struct nouveau_pushbuf *push, uint32_t dwords, uint32_t relocs,
1716 uint32_t pushes)
1717 {
1718 struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push);
1719 struct nouveau_pushbuf_krec *krec = nvpb->krec;
1720 struct nouveau_client *client = push->client;
1721 struct nouveau_bo *bo = NULL;
1722 bool flushed = false;
1723 int ret = 0;
1724
1725 /* switch to next buffer if insufficient space in the current one */
1726 if (push->cur + dwords >= push->end) {
1727 if (nvpb->bo_next < nvpb->bo_nr) {
1728 nouveau_bo_ref(nvpb->bos[nvpb->bo_next++], &bo);
1729 if (nvpb->bo_next == nvpb->bo_nr)
1730 nvpb->bo_next = 0;
1731 } else {
1732 ret = nouveau_bo_new(client->device, nvpb->type, 0, nvpb->bos[0]->size, NULL, &bo);
1733 if (ret)
1734 return ret;
1735 }
1736 }
1737
1738 /* make sure there's always enough space to queue up the pending
1739 * data in the pushbuf proper
1740 */
1741 pushes++;
1742
1743 /* need to flush if we've run out of space on an immediate pushbuf,
1744 * if the new buffer won't fit, or if the kernel push/reloc limits
1745 * have been hit
1746 */
1747 if (bo ||
1748 krec->nr_reloc + relocs >= NOUVEAU_GEM_MAX_RELOCS ||
1749 krec->nr_push + pushes >= NOUVEAU_GEM_MAX_PUSH) {
1750 if (nvpb->bo && krec->nr_buffer)
1751 pushbuf_flush(push);
1752 flushed = true;
1753 }
1754
1755 /* if necessary, switch to new buffer */
1756 if (bo) {
1757 ret = nouveau_bo_map(bo, NOUVEAU_BO_WR, push->client);
1758 if (ret)
1759 return ret;
1760
1761 nouveau_pushbuf_data(push, NULL, 0, 0);
1762 nouveau_bo_ref(bo, &nvpb->bo);
1763 nouveau_bo_ref(NULL, &bo);
1764
1765 nvpb->bgn = nvpb->bo->map;
1766 nvpb->ptr = nvpb->bgn;
1767 push->cur = nvpb->bgn;
1768 push->end = push->cur + (nvpb->bo->size / 4);
1769 push->end -= 2 + push->rsvd_kick; /* space for suffix */
1770 }
1771
1772 pushbuf_kref(push, nvpb->bo, push->flags);
1773 return flushed ? pushbuf_validate(push, false) : 0;
1774 }
1775
1776 int
1777 nouveau_pushbuf_validate(struct nouveau_pushbuf *push)
1778 {
1779 return pushbuf_validate(push, true);
1780 }