Debian Codegraph @

src/gallium/drivers/nouveau/nvc0/nve4_compute.c

raw
1 /*
2 * Copyright 2012 Nouveau Project
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Christoph Bumiller
23 */
24
25 #include "nvc0/nvc0_context.h"
26 #include "nvc0/nve4_compute.h"
27
28 #include "nv50_ir_driver.h"
29
30 #include "drf.h"
31 #include "qmd.h"
32 #include "cla0c0qmd.h"
33 #include "clc0c0qmd.h"
34 #include "clc3c0qmd.h"
35
36 #define NVA0C0_QMDV00_06_VAL_SET(p,a...) NVVAL_MW_SET((p), NVA0C0, QMDV00_06, ##a)
37 #define NVA0C0_QMDV00_06_DEF_SET(p,a...) NVDEF_MW_SET((p), NVA0C0, QMDV00_06, ##a)
38 #define NVC0C0_QMDV02_01_VAL_SET(p,a...) NVVAL_MW_SET((p), NVC0C0, QMDV02_01, ##a)
39 #define NVC0C0_QMDV02_01_DEF_SET(p,a...) NVDEF_MW_SET((p), NVC0C0, QMDV02_01, ##a)
40 #define NVC3C0_QMDV02_02_VAL_SET(p,a...) NVVAL_MW_SET((p), NVC3C0, QMDV02_02, ##a)
41 #define NVC3C0_QMDV02_02_DEF_SET(p,a...) NVDEF_MW_SET((p), NVC3C0, QMDV02_02, ##a)
42
43 int
44 nve4_screen_compute_setup(struct nvc0_screen *screen,
45 struct nouveau_pushbuf *push)
46 {
47 int i;
48 uint32_t obj_class = screen->compute->oclass;
49 uint64_t address;
50
51 BEGIN_NVC0(push, SUBC_CP(NV01_SUBCHAN_OBJECT), 1);
52 PUSH_DATA (push, screen->compute->oclass);
53
54 BEGIN_NVC0(push, NVE4_CP(TEMP_ADDRESS_HIGH), 2);
55 PUSH_DATAh(push, screen->tls->offset);
56 PUSH_DATA (push, screen->tls->offset);
57 /* No idea why there are 2. Divide size by 2 to be safe.
58 * Actually this might be per-MP TEMP size and looks like I'm only using
59 * 2 MPs instead of all 8.
60 */
61 BEGIN_NVC0(push, NVE4_CP(MP_TEMP_SIZE_HIGH(0)), 3);
62 PUSH_DATAh(push, screen->tls->size / screen->mp_count);
63 PUSH_DATA (push, (screen->tls->size / screen->mp_count) & ~0x7fff);
64 PUSH_DATA (push, 0xff);
65 if (obj_class < GV100_COMPUTE_CLASS) {
66 BEGIN_NVC0(push, NVE4_CP(MP_TEMP_SIZE_HIGH(1)), 3);
67 PUSH_DATAh(push, screen->tls->size / screen->mp_count);
68 PUSH_DATA (push, (screen->tls->size / screen->mp_count) & ~0x7fff);
69 PUSH_DATA (push, 0xff);
70 }
71
72 /* Unified address space ? Who needs that ? Certainly not OpenCL.
73 *
74 * FATAL: Buffers with addresses inside [0x1000000, 0x3000000] will NOT be
75 * accessible. We cannot prevent that at the moment, so expect failure.
76 */
77 if (obj_class < GV100_COMPUTE_CLASS) {
78 BEGIN_NVC0(push, NVE4_CP(LOCAL_BASE), 1);
79 PUSH_DATA (push, 0xff << 24);
80 BEGIN_NVC0(push, NVE4_CP(SHARED_BASE), 1);
81 PUSH_DATA (push, 0xfe << 24);
82
83 BEGIN_NVC0(push, NVE4_CP(CODE_ADDRESS_HIGH), 2);
84 PUSH_DATAh(push, screen->text->offset);
85 PUSH_DATA (push, screen->text->offset);
86 } else {
87 BEGIN_NVC0(push, SUBC_CP(0x2a0), 2);
88 PUSH_DATAh(push, 0xfeULL << 24);
89 PUSH_DATA (push, 0xfeULL << 24);
90 BEGIN_NVC0(push, SUBC_CP(0x7b0), 2);
91 PUSH_DATAh(push, 0xffULL << 24);
92 PUSH_DATA (push, 0xffULL << 24);
93 }
94
95 BEGIN_NVC0(push, SUBC_CP(0x0310), 1);
96 PUSH_DATA (push, (obj_class >= NVF0_COMPUTE_CLASS) ? 0x400 : 0x300);
97
98 /* NOTE: these do not affect the state used by the 3D object */
99 BEGIN_NVC0(push, NVE4_CP(TIC_ADDRESS_HIGH), 3);
100 PUSH_DATAh(push, screen->txc->offset);
101 PUSH_DATA (push, screen->txc->offset);
102 PUSH_DATA (push, NVC0_TIC_MAX_ENTRIES - 1);
103 BEGIN_NVC0(push, NVE4_CP(TSC_ADDRESS_HIGH), 3);
104 PUSH_DATAh(push, screen->txc->offset + 65536);
105 PUSH_DATA (push, screen->txc->offset + 65536);
106 PUSH_DATA (push, NVC0_TSC_MAX_ENTRIES - 1);
107
108 if (obj_class >= NVF0_COMPUTE_CLASS) {
109 /* The blob calls GK110_COMPUTE.FIRMWARE[0x6], along with the args (0x1)
110 * passed with GK110_COMPUTE.GRAPH.SCRATCH[0x2]. This is currently
111 * disabled because our firmware doesn't support these commands and the
112 * GPU hangs if they are used. */
113 BEGIN_NIC0(push, SUBC_CP(0x0248), 64);
114 for (i = 63; i >= 0; i--)
115 PUSH_DATA(push, 0x38000 | i);
116 IMMED_NVC0(push, SUBC_CP(NV50_GRAPH_SERIALIZE), 0);
117 }
118
119 BEGIN_NVC0(push, NVE4_CP(TEX_CB_INDEX), 1);
120 PUSH_DATA (push, 7); /* does not interfere with 3D */
121
122 /* Disabling this UNK command avoid a read fault when using texelFetch()
123 * from a compute shader for weird reasons.
124 if (obj_class == NVF0_COMPUTE_CLASS)
125 IMMED_NVC0(push, SUBC_CP(0x02c4), 1);
126 */
127
128 address = screen->uniform_bo->offset + NVC0_CB_AUX_INFO(5);
129
130 /* MS sample coordinate offsets: these do not work with _ALT modes ! */
131 BEGIN_NVC0(push, NVE4_CP(UPLOAD_DST_ADDRESS_HIGH), 2);
132 PUSH_DATAh(push, address + NVC0_CB_AUX_MS_INFO);
133 PUSH_DATA (push, address + NVC0_CB_AUX_MS_INFO);
134 BEGIN_NVC0(push, NVE4_CP(UPLOAD_LINE_LENGTH_IN), 2);
135 PUSH_DATA (push, 64);
136 PUSH_DATA (push, 1);
137 BEGIN_1IC0(push, NVE4_CP(UPLOAD_EXEC), 17);
138 PUSH_DATA (push, NVE4_COMPUTE_UPLOAD_EXEC_LINEAR | (0x20 << 1));
139 PUSH_DATA (push, 0); /* 0 */
140 PUSH_DATA (push, 0);
141 PUSH_DATA (push, 1); /* 1 */
142 PUSH_DATA (push, 0);
143 PUSH_DATA (push, 0); /* 2 */
144 PUSH_DATA (push, 1);
145 PUSH_DATA (push, 1); /* 3 */
146 PUSH_DATA (push, 1);
147 PUSH_DATA (push, 2); /* 4 */
148 PUSH_DATA (push, 0);
149 PUSH_DATA (push, 3); /* 5 */
150 PUSH_DATA (push, 0);
151 PUSH_DATA (push, 2); /* 6 */
152 PUSH_DATA (push, 1);
153 PUSH_DATA (push, 3); /* 7 */
154 PUSH_DATA (push, 1);
155
156 #ifdef NOUVEAU_NVE4_MP_TRAP_HANDLER
157 BEGIN_NVC0(push, NVE4_CP(UPLOAD_DST_ADDRESS_HIGH), 2);
158 PUSH_DATAh(push, screen->parm->offset + NVE4_CP_INPUT_TRAP_INFO_PTR);
159 PUSH_DATA (push, screen->parm->offset + NVE4_CP_INPUT_TRAP_INFO_PTR);
160 BEGIN_NVC0(push, NVE4_CP(UPLOAD_LINE_LENGTH_IN), 2);
161 PUSH_DATA (push, 28);
162 PUSH_DATA (push, 1);
163 BEGIN_1IC0(push, NVE4_CP(UPLOAD_EXEC), 8);
164 PUSH_DATA (push, 1);
165 PUSH_DATA (push, screen->parm->offset + NVE4_CP_PARAM_TRAP_INFO);
166 PUSH_DATAh(push, screen->parm->offset + NVE4_CP_PARAM_TRAP_INFO);
167 PUSH_DATA (push, screen->tls->offset);
168 PUSH_DATAh(push, screen->tls->offset);
169 PUSH_DATA (push, screen->tls->size / 2); /* MP TEMP block size */
170 PUSH_DATA (push, screen->tls->size / 2 / 64); /* warp TEMP block size */
171 PUSH_DATA (push, 0); /* warp cfstack size */
172 #endif
173
174 BEGIN_NVC0(push, NVE4_CP(FLUSH), 1);
175 PUSH_DATA (push, NVE4_COMPUTE_FLUSH_CB);
176
177 return 0;
178 }
179
180 static void
181 gm107_compute_validate_surfaces(struct nvc0_context *nvc0,
182 struct pipe_image_view *view, int slot)
183 {
184 struct nv04_resource *res = nv04_resource(view->resource);
185 struct nouveau_pushbuf *push = nvc0->base.pushbuf;
186 struct nvc0_screen *screen = nvc0->screen;
187 struct nouveau_bo *txc = nvc0->screen->txc;
188 struct nv50_tic_entry *tic;
189 uint64_t address;
190 const int s = 5;
191
192 tic = nv50_tic_entry(nvc0->images_tic[s][slot]);
193
194 res = nv04_resource(tic->pipe.texture);
195 nvc0_update_tic(nvc0, tic, res);
196
197 if (tic->id < 0) {
198 tic->id = nvc0_screen_tic_alloc(nvc0->screen, tic);
199
200 /* upload the texture view */
201 PUSH_SPACE(push, 16);
202 BEGIN_NVC0(push, NVE4_CP(UPLOAD_DST_ADDRESS_HIGH), 2);
203 PUSH_DATAh(push, txc->offset + (tic->id * 32));
204 PUSH_DATA (push, txc->offset + (tic->id * 32));
205 BEGIN_NVC0(push, NVE4_CP(UPLOAD_LINE_LENGTH_IN), 2);
206 PUSH_DATA (push, 32);
207 PUSH_DATA (push, 1);
208 BEGIN_1IC0(push, NVE4_CP(UPLOAD_EXEC), 9);
209 PUSH_DATA (push, NVE4_COMPUTE_UPLOAD_EXEC_LINEAR | (0x20 << 1));
210 PUSH_DATAp(push, &tic->tic[0], 8);
211
212 BEGIN_NIC0(push, NVE4_CP(TIC_FLUSH), 1);
213 PUSH_DATA (push, (tic->id << 4) | 1);
214 } else
215 if (res->status & NOUVEAU_BUFFER_STATUS_GPU_WRITING) {
216 BEGIN_NIC0(push, NVE4_CP(TEX_CACHE_CTL), 1);
217 PUSH_DATA (push, (tic->id << 4) | 1);
218 }
219 nvc0->screen->tic.lock[tic->id / 32] |= 1 << (tic->id % 32);
220
221 res->status &= ~NOUVEAU_BUFFER_STATUS_GPU_WRITING;
222 res->status |= NOUVEAU_BUFFER_STATUS_GPU_READING;
223
224 BCTX_REFN(nvc0->bufctx_cp, CP_SUF, res, RD);
225
226 address = screen->uniform_bo->offset + NVC0_CB_AUX_INFO(s);
227
228 /* upload the texture handle */
229 BEGIN_NVC0(push, NVE4_CP(UPLOAD_DST_ADDRESS_HIGH), 2);
230 PUSH_DATAh(push, address + NVC0_CB_AUX_TEX_INFO(slot + 32));
231 PUSH_DATA (push, address + NVC0_CB_AUX_TEX_INFO(slot + 32));
232 BEGIN_NVC0(push, NVE4_CP(UPLOAD_LINE_LENGTH_IN), 2);
233 PUSH_DATA (push, 4);
234 PUSH_DATA (push, 0x1);
235 BEGIN_1IC0(push, NVE4_CP(UPLOAD_EXEC), 2);
236 PUSH_DATA (push, NVE4_COMPUTE_UPLOAD_EXEC_LINEAR | (0x20 << 1));
237 PUSH_DATA (push, tic->id);
238
239 BEGIN_NVC0(push, NVE4_CP(FLUSH), 1);
240 PUSH_DATA (push, NVE4_COMPUTE_FLUSH_CB);
241 }
242
243 static void
244 nve4_compute_validate_surfaces(struct nvc0_context *nvc0)
245 {
246 struct nouveau_pushbuf *push = nvc0->base.pushbuf;
247 uint64_t address;
248 const int s = 5;
249 int i, j;
250
251 if (!nvc0->images_dirty[s])
252 return;
253
254 address = nvc0->screen->uniform_bo->offset + NVC0_CB_AUX_INFO(s);
255
256 for (i = 0; i < NVC0_MAX_IMAGES; ++i) {
257 struct pipe_image_view *view = &nvc0->images[s][i];
258
259 BEGIN_NVC0(push, NVE4_CP(UPLOAD_DST_ADDRESS_HIGH), 2);
260 PUSH_DATAh(push, address + NVC0_CB_AUX_SU_INFO(i));
261 PUSH_DATA (push, address + NVC0_CB_AUX_SU_INFO(i));
262 BEGIN_NVC0(push, NVE4_CP(UPLOAD_LINE_LENGTH_IN), 2);
263 PUSH_DATA (push, 16 * 4);
264 PUSH_DATA (push, 0x1);
265 BEGIN_1IC0(push, NVE4_CP(UPLOAD_EXEC), 1 + 16);
266 PUSH_DATA (push, NVE4_COMPUTE_UPLOAD_EXEC_LINEAR | (0x20 << 1));
267
268 if (view->resource) {
269 struct nv04_resource *res = nv04_resource(view->resource);
270
271 if (res->base.target == PIPE_BUFFER) {
272 if (view->access & PIPE_IMAGE_ACCESS_WRITE)
273 nvc0_mark_image_range_valid(view);
274 }
275
276 nve4_set_surface_info(push, view, nvc0);
277 BCTX_REFN(nvc0->bufctx_cp, CP_SUF, res, RDWR);
278
279 if (nvc0->screen->base.class_3d >= GM107_3D_CLASS)
280 gm107_compute_validate_surfaces(nvc0, view, i);
281 } else {
282 for (j = 0; j < 16; j++)
283 PUSH_DATA(push, 0);
284 }
285 }
286 }
287
288 /* Thankfully, textures with samplers follow the normal rules. */
289 static void
290 nve4_compute_validate_samplers(struct nvc0_context *nvc0)
291 {
292 bool need_flush = nve4_validate_tsc(nvc0, 5);
293 if (need_flush) {
294 BEGIN_NVC0(nvc0->base.pushbuf, NVE4_CP(TSC_FLUSH), 1);
295 PUSH_DATA (nvc0->base.pushbuf, 0);
296 }
297
298 /* Invalidate all 3D samplers because they are aliased. */
299 for (int s = 0; s < 5; s++)
300 nvc0->samplers_dirty[s] = ~0;
301 nvc0->dirty_3d |= NVC0_NEW_3D_SAMPLERS;
302 }
303
304 /* (Code duplicated at bottom for various non-convincing reasons.
305 * E.g. we might want to use the COMPUTE subchannel to upload TIC/TSC
306 * entries to avoid a subchannel switch.
307 * Same for texture cache flushes.
308 * Also, the bufctx differs, and more IFs in the 3D version looks ugly.)
309 */
310 static void nve4_compute_validate_textures(struct nvc0_context *);
311
312 static void
313 nve4_compute_set_tex_handles(struct nvc0_context *nvc0)
314 {
315 struct nouveau_pushbuf *push = nvc0->base.pushbuf;
316 struct nvc0_screen *screen = nvc0->screen;
317 uint64_t address;
318 const unsigned s = nvc0_shader_stage(MESA_SHADER_COMPUTE);
319 unsigned i, n;
320 uint32_t dirty = nvc0->textures_dirty[s] | nvc0->samplers_dirty[s];
321
322 if (!dirty)
323 return;
324 i = ffs(dirty) - 1;
325 n = util_logbase2(dirty) + 1 - i;
326 assert(n);
327
328 address = screen->uniform_bo->offset + NVC0_CB_AUX_INFO(s);
329
330 BEGIN_NVC0(push, NVE4_CP(UPLOAD_DST_ADDRESS_HIGH), 2);
331 PUSH_DATAh(push, address + NVC0_CB_AUX_TEX_INFO(i));
332 PUSH_DATA (push, address + NVC0_CB_AUX_TEX_INFO(i));
333 BEGIN_NVC0(push, NVE4_CP(UPLOAD_LINE_LENGTH_IN), 2);
334 PUSH_DATA (push, n * 4);
335 PUSH_DATA (push, 0x1);
336 BEGIN_1IC0(push, NVE4_CP(UPLOAD_EXEC), 1 + n);
337 PUSH_DATA (push, NVE4_COMPUTE_UPLOAD_EXEC_LINEAR | (0x20 << 1));
338 PUSH_DATAp(push, &nvc0->tex_handles[s][i], n);
339
340 BEGIN_NVC0(push, NVE4_CP(FLUSH), 1);
341 PUSH_DATA (push, NVE4_COMPUTE_FLUSH_CB);
342
343 nvc0->textures_dirty[s] = 0;
344 nvc0->samplers_dirty[s] = 0;
345 }
346
347 static void
348 nve4_compute_validate_constbufs(struct nvc0_context *nvc0)
349 {
350 struct nouveau_pushbuf *push = nvc0->base.pushbuf;
351 const int s = 5;
352
353 while (nvc0->constbuf_dirty[s]) {
354 int i = ffs(nvc0->constbuf_dirty[s]) - 1;
355 nvc0->constbuf_dirty[s] &= ~(1 << i);
356
357 if (nvc0->constbuf[s][i].user) {
358 struct nouveau_bo *bo = nvc0->screen->uniform_bo;
359 const unsigned base = NVC0_CB_USR_INFO(s);
360 const unsigned size = nvc0->constbuf[s][0].size;
361 assert(i == 0); /* we really only want OpenGL uniforms here */
362 assert(nvc0->constbuf[s][0].u.data);
363
364 BEGIN_NVC0(push, NVE4_CP(UPLOAD_DST_ADDRESS_HIGH), 2);
365 PUSH_DATAh(push, bo->offset + base);
366 PUSH_DATA (push, bo->offset + base);
367 BEGIN_NVC0(push, NVE4_CP(UPLOAD_LINE_LENGTH_IN), 2);
368 PUSH_DATA (push, size);
369 PUSH_DATA (push, 0x1);
370 BEGIN_1IC0(push, NVE4_CP(UPLOAD_EXEC), 1 + (size / 4));
371 PUSH_DATA (push, NVE4_COMPUTE_UPLOAD_EXEC_LINEAR | (0x20 << 1));
372 PUSH_DATAp(push, nvc0->constbuf[s][0].u.data, size / 4);
373 }
374 else {
375 struct nv04_resource *res =
376 nv04_resource(nvc0->constbuf[s][i].u.buf);
377 if (res) {
378 uint64_t address
379 = nvc0->screen->uniform_bo->offset + NVC0_CB_AUX_INFO(s);
380
381 /* constbufs above 0 will are fetched via ubo info in the shader */
382 if (i > 0) {
383 BEGIN_NVC0(push, NVE4_CP(UPLOAD_DST_ADDRESS_HIGH), 2);
384 PUSH_DATAh(push, address + NVC0_CB_AUX_UBO_INFO(i - 1));
385 PUSH_DATA (push, address + NVC0_CB_AUX_UBO_INFO(i - 1));
386 BEGIN_NVC0(push, NVE4_CP(UPLOAD_LINE_LENGTH_IN), 2);
387 PUSH_DATA (push, 4 * 4);
388 PUSH_DATA (push, 0x1);
389 BEGIN_1IC0(push, NVE4_CP(UPLOAD_EXEC), 1 + 4);
390 PUSH_DATA (push, NVE4_COMPUTE_UPLOAD_EXEC_LINEAR | (0x20 << 1));
391
392 PUSH_DATA (push, res->address + nvc0->constbuf[s][i].offset);
393 PUSH_DATAh(push, res->address + nvc0->constbuf[s][i].offset);
394 PUSH_DATA (push, nvc0->constbuf[s][i].size);
395 PUSH_DATA (push, 0);
396 }
397
398 BCTX_REFN(nvc0->bufctx_cp, CP_CB(i), res, RD);
399 res->cb_bindings[s] |= 1 << i;
400 }
401 }
402 }
403
404 BEGIN_NVC0(push, NVE4_CP(FLUSH), 1);
405 PUSH_DATA (push, NVE4_COMPUTE_FLUSH_CB);
406 }
407
408 static void
409 nve4_compute_validate_buffers(struct nvc0_context *nvc0)
410 {
411 struct nouveau_pushbuf *push = nvc0->base.pushbuf;
412 uint64_t address;
413 const int s = 5;
414 int i;
415
416 address = nvc0->screen->uniform_bo->offset + NVC0_CB_AUX_INFO(s);
417
418 BEGIN_NVC0(push, NVE4_CP(UPLOAD_DST_ADDRESS_HIGH), 2);
419 PUSH_DATAh(push, address + NVC0_CB_AUX_BUF_INFO(0));
420 PUSH_DATA (push, address + NVC0_CB_AUX_BUF_INFO(0));
421 BEGIN_NVC0(push, NVE4_CP(UPLOAD_LINE_LENGTH_IN), 2);
422 PUSH_DATA (push, 4 * NVC0_MAX_BUFFERS * 4);
423 PUSH_DATA (push, 0x1);
424 BEGIN_1IC0(push, NVE4_CP(UPLOAD_EXEC), 1 + 4 * NVC0_MAX_BUFFERS);
425 PUSH_DATA (push, NVE4_COMPUTE_UPLOAD_EXEC_LINEAR | (0x20 << 1));
426
427 for (i = 0; i < NVC0_MAX_BUFFERS; i++) {
428 if (nvc0->buffers[s][i].buffer) {
429 struct nv04_resource *res =
430 nv04_resource(nvc0->buffers[s][i].buffer);
431 PUSH_DATA (push, res->address + nvc0->buffers[s][i].buffer_offset);
432 PUSH_DATAh(push, res->address + nvc0->buffers[s][i].buffer_offset);
433 PUSH_DATA (push, nvc0->buffers[s][i].buffer_size);
434 PUSH_DATA (push, 0);
435 BCTX_REFN(nvc0->bufctx_cp, CP_BUF, res, RDWR);
436 util_range_add(&res->base, &res->valid_buffer_range,
437 nvc0->buffers[s][i].buffer_offset,
438 nvc0->buffers[s][i].buffer_offset +
439 nvc0->buffers[s][i].buffer_size);
440 } else {
441 PUSH_DATA (push, 0);
442 PUSH_DATA (push, 0);
443 PUSH_DATA (push, 0);
444 PUSH_DATA (push, 0);
445 }
446 }
447 }
448
449 static struct nvc0_state_validate
450 validate_list_cp[] = {
451 { nvc0_compprog_validate, NVC0_NEW_CP_PROGRAM },
452 { nve4_compute_validate_textures, NVC0_NEW_CP_TEXTURES },
453 { nve4_compute_validate_samplers, NVC0_NEW_CP_SAMPLERS },
454 { nve4_compute_set_tex_handles, NVC0_NEW_CP_TEXTURES |
455 NVC0_NEW_CP_SAMPLERS },
456 { nve4_compute_validate_surfaces, NVC0_NEW_CP_SURFACES },
457 { nvc0_compute_validate_globals, NVC0_NEW_CP_GLOBALS },
458 { nve4_compute_validate_buffers, NVC0_NEW_CP_BUFFERS },
459 { nve4_compute_validate_constbufs, NVC0_NEW_CP_CONSTBUF },
460 };
461
462 static bool
463 nve4_state_validate_cp(struct nvc0_context *nvc0, uint32_t mask)
464 {
465 bool ret;
466
467 ret = nvc0_state_validate(nvc0, mask, validate_list_cp,
468 ARRAY_SIZE(validate_list_cp), &nvc0->dirty_cp,
469 nvc0->bufctx_cp);
470
471 if (unlikely(nvc0->state.flushed))
472 nvc0_bufctx_fence(nvc0, nvc0->bufctx_cp, true);
473 return ret;
474 }
475
476 static void
477 nve4_compute_upload_input(struct nvc0_context *nvc0,
478 const struct pipe_grid_info *info)
479 {
480 struct nvc0_screen *screen = nvc0->screen;
481 struct nouveau_pushbuf *push = nvc0->base.pushbuf;
482 uint64_t address;
483
484 address = screen->uniform_bo->offset + NVC0_CB_AUX_INFO(5);
485
486 BEGIN_NVC0(push, NVE4_CP(UPLOAD_DST_ADDRESS_HIGH), 2);
487 PUSH_DATAh(push, address + NVC0_CB_AUX_GRID_INFO(0));
488 PUSH_DATA (push, address + NVC0_CB_AUX_GRID_INFO(0));
489 BEGIN_NVC0(push, NVE4_CP(UPLOAD_LINE_LENGTH_IN), 2);
490 PUSH_DATA (push, 8 * 4);
491 PUSH_DATA (push, 0x1);
492
493 if (unlikely(info->indirect)) {
494 struct nv04_resource *res = nv04_resource(info->indirect);
495 uint32_t offset = res->offset + info->indirect_offset;
496
497 PUSH_SPACE_EX(push, 32, 0, 1);
498 PUSH_REF1(push, res->bo, NOUVEAU_BO_RD | res->domain);
499
500 BEGIN_1IC0(push, NVE4_CP(UPLOAD_EXEC), 1 + 8);
501 PUSH_DATA (push, NVE4_COMPUTE_UPLOAD_EXEC_LINEAR | (0x20 << 1));
502 PUSH_DATAp(push, info->block, 3);
503 nouveau_pushbuf_data(push, res->bo, offset,
504 NVC0_IB_ENTRY_1_NO_PREFETCH | 3 * 4);
505 } else {
506 BEGIN_1IC0(push, NVE4_CP(UPLOAD_EXEC), 1 + 8);
507 PUSH_DATA (push, NVE4_COMPUTE_UPLOAD_EXEC_LINEAR | (0x20 << 1));
508 PUSH_DATAp(push, info->block, 3);
509 PUSH_DATAp(push, info->grid, 3);
510 }
511 PUSH_DATA (push, 0);
512 PUSH_DATA (push, info->work_dim);
513
514 BEGIN_NVC0(push, NVE4_CP(FLUSH), 1);
515 PUSH_DATA (push, NVE4_COMPUTE_FLUSH_CB);
516 }
517
518 static inline void
519 gp100_cp_launch_desc_set_cb(uint32_t *qmd, unsigned index,
520 struct nouveau_bo *bo, uint32_t base, uint32_t size)
521 {
522 uint64_t address = bo->offset + base;
523
524 assert(index < 8);
525 assert(!(base & 0xff));
526
527 NVC0C0_QMDV02_01_VAL_SET(qmd, CONSTANT_BUFFER_ADDR_LOWER, index, address);
528 NVC0C0_QMDV02_01_VAL_SET(qmd, CONSTANT_BUFFER_ADDR_UPPER, index, address >> 32);
529 NVC0C0_QMDV02_01_VAL_SET(qmd, CONSTANT_BUFFER_SIZE_SHIFTED4, index,
530 DIV_ROUND_UP(size, 16));
531 NVC0C0_QMDV02_01_DEF_SET(qmd, CONSTANT_BUFFER_VALID, index, TRUE);
532 }
533
534 static inline void
535 nve4_cp_launch_desc_set_cb(uint32_t *qmd, unsigned index, struct nouveau_bo *bo,
536 uint32_t base, uint32_t size)
537 {
538 uint64_t address = bo->offset + base;
539
540 assert(index < 8);
541 assert(!(base & 0xff));
542
543 NVA0C0_QMDV00_06_VAL_SET(qmd, CONSTANT_BUFFER_ADDR_LOWER, index, address);
544 NVA0C0_QMDV00_06_VAL_SET(qmd, CONSTANT_BUFFER_ADDR_UPPER, index, address >> 32);
545 NVA0C0_QMDV00_06_VAL_SET(qmd, CONSTANT_BUFFER_SIZE, index, size);
546 NVA0C0_QMDV00_06_DEF_SET(qmd, CONSTANT_BUFFER_VALID, index, TRUE);
547 }
548
549 static void
550 nve4_compute_setup_buf_cb(struct nvc0_context *nvc0, bool gp100, void *desc)
551 {
552 // only user constant buffers 0-6 can be put in the descriptor, the rest are
553 // loaded through global memory
554 for (int i = 0; i <= 6; i++) {
555 if (nvc0->constbuf[5][i].user || !nvc0->constbuf[5][i].u.buf)
556 continue;
557
558 struct nv04_resource *res =
559 nv04_resource(nvc0->constbuf[5][i].u.buf);
560
561 uint32_t base = res->offset + nvc0->constbuf[5][i].offset;
562 uint32_t size = nvc0->constbuf[5][i].size;
563 if (gp100)
564 gp100_cp_launch_desc_set_cb(desc, i, res->bo, base, size);
565 else
566 nve4_cp_launch_desc_set_cb(desc, i, res->bo, base, size);
567 }
568
569 // there is no need to do FLUSH(NVE4_COMPUTE_FLUSH_CB) because
570 // nve4_compute_upload_input() does it later
571 }
572
573 static void
574 nve4_compute_setup_launch_desc(struct nvc0_context *nvc0, uint32_t *qmd,
575 const struct pipe_grid_info *info)
576 {
577 const struct nvc0_screen *screen = nvc0->screen;
578 const struct nvc0_program *cp = nvc0->compprog;
579 uint32_t shared_size = cp->cp.smem_size + info->variable_shared_mem;
580
581 NVA0C0_QMDV00_06_DEF_SET(qmd, INVALIDATE_TEXTURE_HEADER_CACHE, TRUE);
582 NVA0C0_QMDV00_06_DEF_SET(qmd, INVALIDATE_TEXTURE_SAMPLER_CACHE, TRUE);
583 NVA0C0_QMDV00_06_DEF_SET(qmd, INVALIDATE_TEXTURE_DATA_CACHE, TRUE);
584 NVA0C0_QMDV00_06_DEF_SET(qmd, INVALIDATE_SHADER_DATA_CACHE, TRUE);
585 NVA0C0_QMDV00_06_DEF_SET(qmd, INVALIDATE_SHADER_CONSTANT_CACHE, TRUE);
586 NVA0C0_QMDV00_06_DEF_SET(qmd, RELEASE_MEMBAR_TYPE, FE_SYSMEMBAR);
587 NVA0C0_QMDV00_06_DEF_SET(qmd, CWD_MEMBAR_TYPE, L1_SYSMEMBAR);
588 NVA0C0_QMDV00_06_DEF_SET(qmd, API_VISIBLE_CALL_LIMIT, NO_CHECK);
589 NVA0C0_QMDV00_06_VAL_SET(qmd, SASS_VERSION, 0x30);
590
591 NVA0C0_QMDV00_06_VAL_SET(qmd, PROGRAM_OFFSET, cp->code_base);
592
593 NVA0C0_QMDV00_06_VAL_SET(qmd, CTA_RASTER_WIDTH, info->grid[0]);
594 NVA0C0_QMDV00_06_VAL_SET(qmd, CTA_RASTER_HEIGHT, info->grid[1]);
595 NVA0C0_QMDV00_06_VAL_SET(qmd, CTA_RASTER_DEPTH, info->grid[2]);
596 NVA0C0_QMDV00_06_VAL_SET(qmd, CTA_THREAD_DIMENSION0, info->block[0]);
597 NVA0C0_QMDV00_06_VAL_SET(qmd, CTA_THREAD_DIMENSION1, info->block[1]);
598 NVA0C0_QMDV00_06_VAL_SET(qmd, CTA_THREAD_DIMENSION2, info->block[2]);
599
600 NVA0C0_QMDV00_06_VAL_SET(qmd, SHARED_MEMORY_SIZE, align(shared_size, 0x100));
601 NVA0C0_QMDV00_06_VAL_SET(qmd, SHADER_LOCAL_MEMORY_LOW_SIZE, cp->hdr[1] & 0xfffff0);
602 NVA0C0_QMDV00_06_VAL_SET(qmd, SHADER_LOCAL_MEMORY_HIGH_SIZE, 0);
603 NVA0C0_QMDV00_06_VAL_SET(qmd, SHADER_LOCAL_MEMORY_CRS_SIZE, 0x800);
604
605 if (shared_size > (32 << 10))
606 NVA0C0_QMDV00_06_DEF_SET(qmd, L1_CONFIGURATION,
607 DIRECTLY_ADDRESSABLE_MEMORY_SIZE_48KB);
608 else
609 if (shared_size > (16 << 10))
610 NVA0C0_QMDV00_06_DEF_SET(qmd, L1_CONFIGURATION,
611 DIRECTLY_ADDRESSABLE_MEMORY_SIZE_32KB);
612 else
613 NVA0C0_QMDV00_06_DEF_SET(qmd, L1_CONFIGURATION,
614 DIRECTLY_ADDRESSABLE_MEMORY_SIZE_16KB);
615
616 NVA0C0_QMDV00_06_VAL_SET(qmd, REGISTER_COUNT, cp->num_gprs);
617 NVA0C0_QMDV00_06_VAL_SET(qmd, BARRIER_COUNT, cp->num_barriers);
618
619 // Only bind user uniforms and the driver constant buffer through the
620 // launch descriptor because UBOs are sticked to the driver cb to avoid the
621 // limitation of 8 CBs.
622 if (nvc0->constbuf[5][0].user) {
623 nve4_cp_launch_desc_set_cb(qmd, 0, screen->uniform_bo,
624 NVC0_CB_USR_INFO(5), 1 << 16);
625
626 // Later logic will attempt to bind a real buffer at position 0. That
627 // should not happen if we've bound a user buffer.
628 assert(nvc0->constbuf[5][0].user || !nvc0->constbuf[5][0].u.buf);
629 }
630 nve4_cp_launch_desc_set_cb(qmd, 7, screen->uniform_bo,
631 NVC0_CB_AUX_INFO(5), 1 << 11);
632
633 nve4_compute_setup_buf_cb(nvc0, false, qmd);
634 }
635
636 static void
637 gp100_compute_setup_launch_desc(struct nvc0_context *nvc0, uint32_t *qmd,
638 const struct pipe_grid_info *info)
639 {
640 const struct nvc0_screen *screen = nvc0->screen;
641 const struct nvc0_program *cp = nvc0->compprog;
642 uint32_t shared_size = cp->cp.smem_size + info->variable_shared_mem;
643
644 NVC0C0_QMDV02_01_VAL_SET(qmd, SM_GLOBAL_CACHING_ENABLE, 1);
645 NVC0C0_QMDV02_01_DEF_SET(qmd, RELEASE_MEMBAR_TYPE, FE_SYSMEMBAR);
646 NVC0C0_QMDV02_01_DEF_SET(qmd, CWD_MEMBAR_TYPE, L1_SYSMEMBAR);
647 NVC0C0_QMDV02_01_DEF_SET(qmd, API_VISIBLE_CALL_LIMIT, NO_CHECK);
648
649 NVC0C0_QMDV02_01_VAL_SET(qmd, PROGRAM_OFFSET, cp->code_base);
650
651 NVC0C0_QMDV02_01_VAL_SET(qmd, CTA_RASTER_WIDTH, info->grid[0]);
652 NVC0C0_QMDV02_01_VAL_SET(qmd, CTA_RASTER_HEIGHT, info->grid[1]);
653 NVC0C0_QMDV02_01_VAL_SET(qmd, CTA_RASTER_DEPTH, info->grid[2]);
654 NVC0C0_QMDV02_01_VAL_SET(qmd, CTA_THREAD_DIMENSION0, info->block[0]);
655 NVC0C0_QMDV02_01_VAL_SET(qmd, CTA_THREAD_DIMENSION1, info->block[1]);
656 NVC0C0_QMDV02_01_VAL_SET(qmd, CTA_THREAD_DIMENSION2, info->block[2]);
657
658 NVC0C0_QMDV02_01_VAL_SET(qmd, SHARED_MEMORY_SIZE, align(shared_size, 0x100));
659 NVC0C0_QMDV02_01_VAL_SET(qmd, SHADER_LOCAL_MEMORY_LOW_SIZE, cp->hdr[1] & 0xfffff0);
660 NVC0C0_QMDV02_01_VAL_SET(qmd, SHADER_LOCAL_MEMORY_HIGH_SIZE, 0);
661 NVC0C0_QMDV02_01_VAL_SET(qmd, SHADER_LOCAL_MEMORY_CRS_SIZE, 0x800);
662
663 NVC0C0_QMDV02_01_VAL_SET(qmd, REGISTER_COUNT, cp->num_gprs);
664 NVC0C0_QMDV02_01_VAL_SET(qmd, BARRIER_COUNT, cp->num_barriers);
665
666 // Only bind user uniforms and the driver constant buffer through the
667 // launch descriptor because UBOs are sticked to the driver cb to avoid the
668 // limitation of 8 CBs.
669 if (nvc0->constbuf[5][0].user) {
670 gp100_cp_launch_desc_set_cb(qmd, 0, screen->uniform_bo,
671 NVC0_CB_USR_INFO(5), 1 << 16);
672
673 // Later logic will attempt to bind a real buffer at position 0. That
674 // should not happen if we've bound a user buffer.
675 assert(nvc0->constbuf[5][0].user || !nvc0->constbuf[5][0].u.buf);
676 }
677 gp100_cp_launch_desc_set_cb(qmd, 7, screen->uniform_bo,
678 NVC0_CB_AUX_INFO(5), 1 << 11);
679
680 nve4_compute_setup_buf_cb(nvc0, true, qmd);
681 }
682
683 static int
684 gv100_sm_config_smem_size(u32 size)
685 {
686 if (size > 64 * 1024) size = 96 * 1024;
687 else if (size > 32 * 1024) size = 64 * 1024;
688 else if (size > 16 * 1024) size = 32 * 1024;
689 else if (size > 8 * 1024) size = 16 * 1024;
690 else size = 8 * 1024;
691 return (size / 4096) + 1;
692 }
693
694 static void
695 gv100_compute_setup_launch_desc(struct nvc0_context *nvc0, u32 *qmd,
696 const struct pipe_grid_info *info)
697 {
698 struct nvc0_program *cp = nvc0->compprog;
699 struct nvc0_screen *screen = nvc0->screen;
700 uint64_t entry = screen->text->offset + cp->code_base;
701 uint32_t shared_size = cp->cp.smem_size + info->variable_shared_mem;
702
703 NVC3C0_QMDV02_02_VAL_SET(qmd, SM_GLOBAL_CACHING_ENABLE, 1);
704 NVC3C0_QMDV02_02_DEF_SET(qmd, API_VISIBLE_CALL_LIMIT, NO_CHECK);
705 NVC3C0_QMDV02_02_DEF_SET(qmd, SAMPLER_INDEX, INDEPENDENTLY);
706 NVC3C0_QMDV02_02_VAL_SET(qmd, SHARED_MEMORY_SIZE, align(shared_size, 0x100));
707 NVC3C0_QMDV02_02_VAL_SET(qmd, SHADER_LOCAL_MEMORY_LOW_SIZE, cp->hdr[1] & 0xfffff0);
708 NVC3C0_QMDV02_02_VAL_SET(qmd, SHADER_LOCAL_MEMORY_HIGH_SIZE, 0);
709 NVC3C0_QMDV02_02_VAL_SET(qmd, MIN_SM_CONFIG_SHARED_MEM_SIZE,
710 gv100_sm_config_smem_size(8 * 1024));
711 NVC3C0_QMDV02_02_VAL_SET(qmd, MAX_SM_CONFIG_SHARED_MEM_SIZE,
712 gv100_sm_config_smem_size(96 * 1024));
713 NVC3C0_QMDV02_02_VAL_SET(qmd, QMD_VERSION, 2);
714 NVC3C0_QMDV02_02_VAL_SET(qmd, QMD_MAJOR_VERSION, 2);
715 NVC3C0_QMDV02_02_VAL_SET(qmd, TARGET_SM_CONFIG_SHARED_MEM_SIZE,
716 gv100_sm_config_smem_size(shared_size));
717
718 NVC3C0_QMDV02_02_VAL_SET(qmd, CTA_RASTER_WIDTH, info->grid[0]);
719 NVC3C0_QMDV02_02_VAL_SET(qmd, CTA_RASTER_HEIGHT, info->grid[1]);
720 NVC3C0_QMDV02_02_VAL_SET(qmd, CTA_RASTER_DEPTH, info->grid[2]);
721 NVC3C0_QMDV02_02_VAL_SET(qmd, CTA_THREAD_DIMENSION0, info->block[0]);
722 NVC3C0_QMDV02_02_VAL_SET(qmd, CTA_THREAD_DIMENSION1, info->block[1]);
723 NVC3C0_QMDV02_02_VAL_SET(qmd, CTA_THREAD_DIMENSION2, info->block[2]);
724 NVC3C0_QMDV02_02_VAL_SET(qmd, REGISTER_COUNT_V, cp->num_gprs);
725 NVC3C0_QMDV02_02_VAL_SET(qmd, BARRIER_COUNT, cp->num_barriers);
726
727 // Only bind user uniforms and the driver constant buffer through the
728 // launch descriptor because UBOs are sticked to the driver cb to avoid the
729 // limitation of 8 CBs.
730 if (nvc0->constbuf[5][0].user) {
731 gp100_cp_launch_desc_set_cb(qmd, 0, screen->uniform_bo,
732 NVC0_CB_USR_INFO(5), 1 << 16);
733
734 // Later logic will attempt to bind a real buffer at position 0. That
735 // should not happen if we've bound a user buffer.
736 assert(nvc0->constbuf[5][0].user || !nvc0->constbuf[5][0].u.buf);
737 }
738 gp100_cp_launch_desc_set_cb(qmd, 7, screen->uniform_bo,
739 NVC0_CB_AUX_INFO(5), 1 << 11);
740
741 nve4_compute_setup_buf_cb(nvc0, true, qmd);
742
743 NVC3C0_QMDV02_02_VAL_SET(qmd, PROGRAM_ADDRESS_LOWER, entry & 0xffffffff);
744 NVC3C0_QMDV02_02_VAL_SET(qmd, PROGRAM_ADDRESS_UPPER, entry >> 32);
745 }
746
747 static inline void *
748 nve4_compute_alloc_launch_desc(struct nouveau_context *nv,
749 struct nouveau_bo **pbo, uint64_t *pgpuaddr)
750 {
751 uint8_t *ptr = nouveau_scratch_get(nv, 512, pgpuaddr, pbo);
752 if (!ptr)
753 return NULL;
754 if (*pgpuaddr & 255) {
755 unsigned adj = 256 - (*pgpuaddr & 255);
756 ptr += adj;
757 *pgpuaddr += adj;
758 }
759 memset(ptr, 0x00, 256);
760 return ptr;
761 }
762
763 static void
764 nve4_upload_indirect_desc(struct nouveau_pushbuf *push,
765 struct nv04_resource *res, uint64_t gpuaddr,
766 uint32_t length, uint32_t bo_offset)
767 {
768 BEGIN_NVC0(push, NVE4_CP(UPLOAD_DST_ADDRESS_HIGH), 2);
769 PUSH_DATAh(push, gpuaddr);
770 PUSH_DATA (push, gpuaddr);
771 BEGIN_NVC0(push, NVE4_CP(UPLOAD_LINE_LENGTH_IN), 2);
772 PUSH_DATA (push, length);
773 PUSH_DATA (push, 1);
774
775 PUSH_SPACE_EX(push, 32, 0, 1);
776 PUSH_REF1(push, res->bo, NOUVEAU_BO_RD | res->domain);
777
778 BEGIN_1IC0(push, NVE4_CP(UPLOAD_EXEC), 1 + (length / 4));
779 PUSH_DATA (push, NVE4_COMPUTE_UPLOAD_EXEC_LINEAR | (0x08 << 1));
780 nouveau_pushbuf_data(push, res->bo, bo_offset,
781 NVC0_IB_ENTRY_1_NO_PREFETCH | length);
782 }
783
784 void
785 nve4_launch_grid(struct pipe_context *pipe, const struct pipe_grid_info *info)
786 {
787 struct nvc0_context *nvc0 = nvc0_context(pipe);
788 struct nvc0_screen *screen = nvc0->screen;
789 struct nouveau_pushbuf *push = nvc0->base.pushbuf;
790 void *desc;
791 uint64_t desc_gpuaddr;
792 struct nouveau_bo *desc_bo;
793 int ret;
794
795 desc = nve4_compute_alloc_launch_desc(&nvc0->base, &desc_bo, &desc_gpuaddr);
796 if (!desc) {
797 ret = -1;
798 goto out;
799 }
800 BCTX_REFN_bo(nvc0->bufctx_cp, CP_DESC, NOUVEAU_BO_GART | NOUVEAU_BO_RD,
801 desc_bo);
802
803 list_for_each_entry(struct nvc0_resident, resident, &nvc0->tex_head, list) {
804 nvc0_add_resident(nvc0->bufctx_cp, NVC0_BIND_CP_BINDLESS, resident->buf,
805 resident->flags);
806 }
807
808 list_for_each_entry(struct nvc0_resident, resident, &nvc0->img_head, list) {
809 nvc0_add_resident(nvc0->bufctx_cp, NVC0_BIND_CP_BINDLESS, resident->buf,
810 resident->flags);
811 }
812
813 simple_mtx_lock(&screen->state_lock);
814 ret = !nve4_state_validate_cp(nvc0, ~0);
815 if (ret)
816 goto out_unlock;
817
818 if (nvc0->screen->compute->oclass >= GV100_COMPUTE_CLASS)
819 gv100_compute_setup_launch_desc(nvc0, desc, info);
820 else
821 if (nvc0->screen->compute->oclass >= GP100_COMPUTE_CLASS)
822 gp100_compute_setup_launch_desc(nvc0, desc, info);
823 else
824 nve4_compute_setup_launch_desc(nvc0, desc, info);
825
826 nve4_compute_upload_input(nvc0, info);
827
828 #ifndef NDEBUG
829 if (debug_get_num_option("NV50_PROG_DEBUG", 0)) {
830 debug_printf("Queue Meta Data:\n");
831 if (nvc0->screen->compute->oclass >= GV100_COMPUTE_CLASS)
832 NVC3C0QmdDump_V02_02(desc);
833 else
834 if (nvc0->screen->compute->oclass >= GP100_COMPUTE_CLASS)
835 NVC0C0QmdDump_V02_01(desc);
836 else
837 NVA0C0QmdDump_V00_06(desc);
838 }
839 #endif
840
841 if (unlikely(info->indirect)) {
842 struct nv04_resource *res = nv04_resource(info->indirect);
843 uint32_t offset = res->offset + info->indirect_offset;
844
845 /* upload the descriptor */
846 BEGIN_NVC0(push, NVE4_CP(UPLOAD_DST_ADDRESS_HIGH), 2);
847 PUSH_DATAh(push, desc_gpuaddr);
848 PUSH_DATA (push, desc_gpuaddr);
849 BEGIN_NVC0(push, NVE4_CP(UPLOAD_LINE_LENGTH_IN), 2);
850 PUSH_DATA (push, 256);
851 PUSH_DATA (push, 1);
852 BEGIN_1IC0(push, NVE4_CP(UPLOAD_EXEC), 1 + (256 / 4));
853 PUSH_DATA (push, NVE4_COMPUTE_UPLOAD_EXEC_LINEAR | (0x08 << 1));
854 PUSH_DATAp(push, (const uint32_t *)desc, 256 / 4);
855
856 if (nvc0->screen->compute->oclass >= GP100_COMPUTE_CLASS) {
857 nve4_upload_indirect_desc(push, res, desc_gpuaddr + 48, 12, offset);
858 } else {
859 /* overwrite griddim_x and griddim_y as two 32-bits integers even
860 * if griddim_y must be a 16-bits integer */
861 nve4_upload_indirect_desc(push, res, desc_gpuaddr + 48, 8, offset);
862
863 /* overwrite the 16 high bits of griddim_y with griddim_z because
864 * we need (z << 16) | x */
865 nve4_upload_indirect_desc(push, res, desc_gpuaddr + 54, 4, offset + 8);
866 }
867 }
868
869 /* upload descriptor and flush */
870 PUSH_SPACE_EX(push, 32, 1, 0);
871 PUSH_REF1(push, screen->text, NV_VRAM_DOMAIN(&screen->base) | NOUVEAU_BO_RD);
872 BEGIN_NVC0(push, NVE4_CP(LAUNCH_DESC_ADDRESS), 1);
873 PUSH_DATA (push, desc_gpuaddr >> 8);
874 if (screen->compute->oclass < GA102_COMPUTE_CLASS) {
875 BEGIN_NVC0(push, NVE4_CP(LAUNCH), 1);
876 PUSH_DATA (push, 0x3);
877 } else {
878 BEGIN_NIC0(push, SUBC_CP(0x02c0), 2);
879 PUSH_DATA (push, 1);
880 PUSH_DATA (push, 2);
881 }
882 BEGIN_NVC0(push, SUBC_CP(NV50_GRAPH_SERIALIZE), 1);
883 PUSH_DATA (push, 0);
884
885 nvc0_update_compute_invocations_counter(nvc0, info);
886
887 out_unlock:
888 PUSH_KICK(push);
889 simple_mtx_unlock(&screen->state_lock);
890
891 out:
892 if (ret)
893 NOUVEAU_ERR("Failed to launch grid !\n");
894 nouveau_scratch_done(&nvc0->base);
895 nouveau_bufctx_reset(nvc0->bufctx_cp, NVC0_BIND_CP_DESC);
896 nouveau_bufctx_reset(nvc0->bufctx_cp, NVC0_BIND_CP_BINDLESS);
897 }
898
899
900 #define NVE4_TIC_ENTRY_INVALID 0x000fffff
901
902 static void
903 nve4_compute_validate_textures(struct nvc0_context *nvc0)
904 {
905 struct nouveau_bo *txc = nvc0->screen->txc;
906 struct nouveau_pushbuf *push = nvc0->base.pushbuf;
907 const unsigned s = 5;
908 unsigned i;
909 uint32_t commands[2][32];
910 unsigned n[2] = { 0, 0 };
911
912 for (i = 0; i < nvc0->num_textures[s]; ++i) {
913 struct nv50_tic_entry *tic = nv50_tic_entry(nvc0->textures[s][i]);
914 struct nv04_resource *res;
915 const bool dirty = !!(nvc0->textures_dirty[s] & (1 << i));
916
917 if (!tic) {
918 nvc0->tex_handles[s][i] |= NVE4_TIC_ENTRY_INVALID;
919 continue;
920 }
921 res = nv04_resource(tic->pipe.texture);
922 nvc0_update_tic(nvc0, tic, res);
923
924 if (tic->id < 0) {
925 tic->id = nvc0_screen_tic_alloc(nvc0->screen, tic);
926
927 PUSH_SPACE(push, 16);
928 BEGIN_NVC0(push, NVE4_CP(UPLOAD_DST_ADDRESS_HIGH), 2);
929 PUSH_DATAh(push, txc->offset + (tic->id * 32));
930 PUSH_DATA (push, txc->offset + (tic->id * 32));
931 BEGIN_NVC0(push, NVE4_CP(UPLOAD_LINE_LENGTH_IN), 2);
932 PUSH_DATA (push, 32);
933 PUSH_DATA (push, 1);
934 BEGIN_1IC0(push, NVE4_CP(UPLOAD_EXEC), 9);
935 PUSH_DATA (push, NVE4_COMPUTE_UPLOAD_EXEC_LINEAR | (0x20 << 1));
936 PUSH_DATAp(push, &tic->tic[0], 8);
937
938 commands[0][n[0]++] = (tic->id << 4) | 1;
939 } else
940 if (res->status & NOUVEAU_BUFFER_STATUS_GPU_WRITING) {
941 commands[1][n[1]++] = (tic->id << 4) | 1;
942 }
943 nvc0->screen->tic.lock[tic->id / 32] |= 1 << (tic->id % 32);
944
945 res->status &= ~NOUVEAU_BUFFER_STATUS_GPU_WRITING;
946 res->status |= NOUVEAU_BUFFER_STATUS_GPU_READING;
947
948 nvc0->tex_handles[s][i] &= ~NVE4_TIC_ENTRY_INVALID;
949 nvc0->tex_handles[s][i] |= tic->id;
950 if (dirty)
951 BCTX_REFN(nvc0->bufctx_cp, CP_TEX(i), res, RD);
952 }
953 for (; i < nvc0->state.num_textures[s]; ++i) {
954 nvc0->tex_handles[s][i] |= NVE4_TIC_ENTRY_INVALID;
955 nvc0->textures_dirty[s] |= 1 << i;
956 }
957
958 if (n[0]) {
959 BEGIN_NIC0(push, NVE4_CP(TIC_FLUSH), n[0]);
960 PUSH_DATAp(push, commands[0], n[0]);
961 }
962 if (n[1]) {
963 BEGIN_NIC0(push, NVE4_CP(TEX_CACHE_CTL), n[1]);
964 PUSH_DATAp(push, commands[1], n[1]);
965 }
966
967 nvc0->state.num_textures[s] = nvc0->num_textures[s];
968
969 /* Invalidate all 3D textures because they are aliased. */
970 for (int s = 0; s < 5; s++) {
971 for (int i = 0; i < nvc0->num_textures[s]; i++)
972 nouveau_bufctx_reset(nvc0->bufctx_3d, NVC0_BIND_3D_TEX(s, i));
973 nvc0->textures_dirty[s] = ~0;
974 }
975 nvc0->dirty_3d |= NVC0_NEW_3D_TEXTURES;
976 }
977
978 #ifdef NOUVEAU_NVE4_MP_TRAP_HANDLER
979 static void
980 nve4_compute_trap_info(struct nvc0_context *nvc0)
981 {
982 struct nvc0_screen *screen = nvc0->screen;
983 struct nouveau_bo *bo = screen->parm;
984 int ret, i;
985 volatile struct nve4_mp_trap_info *info;
986 uint8_t *map;
987
988 ret = BO_MAP(&screen->base, bo, NOUVEAU_BO_RDWR, nvc0->base.client);
989 if (ret)
990 return;
991 map = (uint8_t *)bo->map;
992 info = (volatile struct nve4_mp_trap_info *)(map + NVE4_CP_PARAM_TRAP_INFO);
993
994 if (info->lock) {
995 debug_printf("trapstat = %08x\n", info->trapstat);
996 debug_printf("warperr = %08x\n", info->warperr);
997 debug_printf("PC = %x\n", info->pc);
998 debug_printf("tid = %u %u %u\n",
999 info->tid[0], info->tid[1], info->tid[2]);
1000 debug_printf("ctaid = %u %u %u\n",
1001 info->ctaid[0], info->ctaid[1], info->ctaid[2]);
1002 for (i = 0; i <= 63; ++i)
1003 debug_printf("$r%i = %08x\n", i, info->r[i]);
1004 for (i = 0; i <= 6; ++i)
1005 debug_printf("$p%i = %i\n", i, (info->flags >> i) & 1);
1006 debug_printf("$c = %x\n", info->flags >> 12);
1007 }
1008 info->lock = 0;
1009 }
1010 #endif