Debian Codegraph @

src/nouveau/mme/tests/mme_tu104_sim_hw_test.cpp

raw
1 /*
2 * Copyright © 2022 Collabora Ltd.
3 * SPDX-License-Identifier: MIT
4 */
5 #include "mme_runner.h"
6 #include "mme_tu104_sim.h"
7
8 #include "clcd97.h"
9 #include "nv_push_clc597.h"
10
11 class mme_tu104_sim_test : public ::testing::Test, public mme_hw_runner {
12 public:
13 mme_tu104_sim_test();
14 ~mme_tu104_sim_test();
15
16 void SetUp();
17 void test_macro(const mme_builder *b,
18 const std::vector<uint32_t>& macro,
19 const std::vector<uint32_t>& params);
20 };
21
22 mme_tu104_sim_test::mme_tu104_sim_test() :
23 ::testing::Test(),
24 mme_hw_runner()
25 { }
26
27 mme_tu104_sim_test::~mme_tu104_sim_test()
28 { }
29
30 void
31 mme_tu104_sim_test::SetUp()
32 {
33 ASSERT_TRUE(set_up_hw(TURING_A, UINT16_MAX));
34 }
35
36 void
37 mme_tu104_sim_test::test_macro(const mme_builder *b,
38 const std::vector<uint32_t>& macro,
39 const std::vector<uint32_t>& params)
40 {
41 const uint32_t data_dwords = DATA_BO_SIZE / sizeof(uint32_t);
42
43 std::vector<mme_tu104_inst> insts(macro.size() / 3);
44 mme_tu104_decode(&insts[0], &macro[0], macro.size() / 3);
45
46 /* First, make a copy of the data and simulate the macro */
47 std::vector<uint32_t> sim_data(data, data + (DATA_BO_SIZE / 4));
48 mme_tu104_sim_mem sim_mem = {
49 .addr = data_addr,
50 .data = &sim_data[0],
51 .size = DATA_BO_SIZE,
52 };
53 mme_tu104_sim(insts.size(), &insts[0],
54 params.size(), params.size() ? &params[0] : NULL,
55 1, &sim_mem);
56
57 /* Now run the macro on the GPU */
58 push_macro(0, macro);
59
60 P_1INC(p, NVC597, CALL_MME_MACRO(0));
61 if (params.empty()) {
62 P_NVC597_CALL_MME_MACRO(p, 0, 0);
63 } else {
64 P_INLINE_ARRAY(p, &params[0], params.size());
65 }
66
67 submit_push();
68
69 /* Check the results */
70 for (uint32_t i = 0; i < data_dwords; i++)
71 ASSERT_EQ(data[i], sim_data[i]);
72 }
73
74 static mme_tu104_reg
75 mme_value_as_reg(mme_value val)
76 {
77 assert(val.type == MME_VALUE_TYPE_REG);
78 return (mme_tu104_reg)(MME_TU104_REG_R0 + val.reg);
79 }
80
81 TEST_F(mme_tu104_sim_test, sanity)
82 {
83 const uint32_t canary = 0xc0ffee01;
84
85 mme_builder b;
86 mme_builder_init(&b, devinfo);
87
88 mme_store_imm_addr(&b, data_addr, mme_imm(canary));
89
90 auto macro = mme_builder_finish_vec(&b);
91
92 std::vector<uint32_t> params;
93 test_macro(&b, macro, params);
94 }
95
96 TEST_F(mme_tu104_sim_test, multi_param)
97 {
98 mme_builder b;
99 mme_builder_init(&b, devinfo);
100
101 mme_value v0 = mme_alloc_reg(&b);
102 mme_value v1 = mme_alloc_reg(&b);
103
104 mme_tu104_asm(&b, i) {
105 i.alu[0].dst = mme_value_as_reg(v0);
106 i.alu[0].src[0] = MME_TU104_REG_LOAD1;
107 i.alu[1].dst = mme_value_as_reg(v1);
108 i.alu[1].src[0] = MME_TU104_REG_LOAD0;
109 i.imm[0] = (1<<12) | (NVC597_SET_MME_SHADOW_SCRATCH(12) >> 2);
110 i.out[0].mthd = MME_TU104_OUT_OP_IMM0;
111 i.out[0].emit = MME_TU104_OUT_OP_LOAD0;
112 i.imm[1] = (1<<12) | (NVC597_SET_MME_SHADOW_SCRATCH(35) >> 2);
113 i.out[1].mthd = MME_TU104_OUT_OP_IMM1;
114 i.out[1].emit = MME_TU104_OUT_OP_LOAD1;
115 }
116
117 mme_value v2 = mme_state(&b, NVC597_SET_MME_SHADOW_SCRATCH(12));
118 mme_value v3 = mme_state(&b, NVC597_SET_MME_SHADOW_SCRATCH(35));
119
120 mme_store_imm_addr(&b, data_addr + 0, v0);
121 mme_store_imm_addr(&b, data_addr + 4, v1);
122 mme_store_imm_addr(&b, data_addr + 8, v2);
123 mme_store_imm_addr(&b, data_addr + 12, v3);
124
125 auto macro = mme_builder_finish_vec(&b);
126
127 std::vector<uint32_t> params;
128 params.push_back(2581);
129 params.push_back(3048);
130
131 test_macro(&b, macro, params);
132 }
133
134 TEST_F(mme_tu104_sim_test, pred_param)
135 {
136 mme_builder b;
137 mme_builder_init(&b, devinfo);
138
139 mme_value v0 = mme_load(&b);
140 mme_value v1 = mme_mov(&b, mme_imm(240));
141
142 mme_tu104_asm(&b, i) {
143 i.pred_mode = MME_TU104_PRED_TTTT;
144 i.alu[0].dst = mme_value_as_reg(v1);
145 i.alu[0].src[0] = MME_TU104_REG_LOAD0;
146 }
147
148 mme_value v2 = mme_load(&b);
149
150 mme_store_imm_addr(&b, data_addr + 0, v0);
151 mme_store_imm_addr(&b, data_addr + 4, v1);
152 mme_store_imm_addr(&b, data_addr + 8, v2);
153
154 auto macro = mme_builder_finish_vec(&b);
155
156 for (uint32_t j = 0; j < 4; j++) {
157 reset_push();
158
159 std::vector<uint32_t> params;
160 params.push_back((j & 1) * 2043);
161 params.push_back((j & 2) * 523);
162 params.push_back(2581);
163 params.push_back(3048);
164
165 test_macro(&b, macro, params);
166 }
167 }
168
169 TEST_F(mme_tu104_sim_test, out_imm0)
170 {
171 mme_builder b;
172 mme_builder_init(&b, devinfo);
173
174 mme_mthd(&b, NVC597_SET_REPORT_SEMAPHORE_A);
175 mme_emit(&b, mme_imm(high32(data_addr + 0)));
176 mme_emit(&b, mme_imm(low32(data_addr + 0)));
177 mme_tu104_asm(&b, i) {
178 i.imm[0] = 0x1234;
179 i.out[0].emit = MME_TU104_OUT_OP_IMM0;
180 }
181 mme_emit(&b, mme_imm(0x10000000));
182
183 mme_mthd(&b, NVC597_SET_REPORT_SEMAPHORE_A);
184 mme_emit(&b, mme_imm(high32(data_addr + 4)));
185 mme_emit(&b, mme_imm(low32(data_addr + 4)));
186 mme_tu104_asm(&b, i) {
187 i.imm[0] = 0x8765;
188 i.out[0].emit = MME_TU104_OUT_OP_IMM0;
189 }
190 mme_emit(&b, mme_imm(0x10000000));
191
192 auto macro = mme_builder_finish_vec(&b);
193
194 std::vector<uint32_t> params;
195 test_macro(&b, macro, params);
196 }
197
198 TEST_F(mme_tu104_sim_test, out_imm1)
199 {
200 mme_builder b;
201 mme_builder_init(&b, devinfo);
202
203 mme_mthd(&b, NVC597_SET_REPORT_SEMAPHORE_A);
204 mme_emit(&b, mme_imm(high32(data_addr + 0)));
205 mme_emit(&b, mme_imm(low32(data_addr + 0)));
206 mme_tu104_asm(&b, i) {
207 i.imm[1] = 0x1234;
208 i.out[0].emit = MME_TU104_OUT_OP_IMM1;
209 }
210 mme_emit(&b, mme_imm(0x10000000));
211
212 mme_mthd(&b, NVC597_SET_REPORT_SEMAPHORE_A);
213 mme_emit(&b, mme_imm(high32(data_addr + 4)));
214 mme_emit(&b, mme_imm(low32(data_addr + 4)));
215 mme_tu104_asm(&b, i) {
216 i.imm[1] = 0x8765;
217 i.out[0].emit = MME_TU104_OUT_OP_IMM1;
218 }
219 mme_emit(&b, mme_imm(0x10000000));
220
221 auto macro = mme_builder_finish_vec(&b);
222
223 std::vector<uint32_t> params;
224 test_macro(&b, macro, params);
225 }
226
227 TEST_F(mme_tu104_sim_test, out_immhigh0)
228 {
229 mme_builder b;
230 mme_builder_init(&b, devinfo);
231
232 mme_mthd(&b, NVC597_SET_REPORT_SEMAPHORE_A);
233 mme_emit(&b, mme_imm(high32(data_addr + 0)));
234 mme_emit(&b, mme_imm(low32(data_addr + 0)));
235 mme_tu104_asm(&b, i) {
236 i.imm[0] = 0x1234;
237 i.out[0].emit = MME_TU104_OUT_OP_IMMHIGH0;
238 }
239 mme_emit(&b, mme_imm(0x10000000));
240
241 mme_mthd(&b, NVC597_SET_REPORT_SEMAPHORE_A);
242 mme_emit(&b, mme_imm(high32(data_addr + 4)));
243 mme_emit(&b, mme_imm(low32(data_addr + 4)));
244 mme_tu104_asm(&b, i) {
245 i.imm[0] = 0x8765;
246 i.out[1].emit = MME_TU104_OUT_OP_IMMHIGH0;
247 }
248 mme_emit(&b, mme_imm(0x10000000));
249
250 auto macro = mme_builder_finish_vec(&b);
251
252 std::vector<uint32_t> params;
253 test_macro(&b, macro, params);
254 }
255
256 TEST_F(mme_tu104_sim_test, out_immhigh1)
257 {
258 mme_builder b;
259 mme_builder_init(&b, devinfo);
260
261 mme_mthd(&b, NVC597_SET_REPORT_SEMAPHORE_A);
262 mme_emit(&b, mme_imm(high32(data_addr + 0)));
263 mme_emit(&b, mme_imm(low32(data_addr + 0)));
264 mme_tu104_asm(&b, i) {
265 i.imm[1] = 0x1234;
266 i.out[0].emit = MME_TU104_OUT_OP_IMMHIGH1;
267 }
268 mme_emit(&b, mme_imm(0x10000000));
269
270 mme_mthd(&b, NVC597_SET_REPORT_SEMAPHORE_A);
271 mme_emit(&b, mme_imm(high32(data_addr + 4)));
272 mme_emit(&b, mme_imm(low32(data_addr + 4)));
273 mme_tu104_asm(&b, i) {
274 i.imm[1] = 0x8765;
275 i.out[1].emit = MME_TU104_OUT_OP_IMMHIGH1;
276 }
277 mme_emit(&b, mme_imm(0x10000000));
278
279 auto macro = mme_builder_finish_vec(&b);
280
281 std::vector<uint32_t> params;
282 test_macro(&b, macro, params);
283 }
284
285 TEST_F(mme_tu104_sim_test, out_imm32)
286 {
287 mme_builder b;
288 mme_builder_init(&b, devinfo);
289
290 mme_mthd(&b, NVC597_SET_REPORT_SEMAPHORE_A);
291 mme_emit(&b, mme_imm(high32(data_addr + 0)));
292 mme_emit(&b, mme_imm(low32(data_addr + 0)));
293 mme_tu104_asm(&b, i) {
294 i.imm[0] = 0x1234;
295 i.imm[1] = 0x7654;
296 i.out[0].emit = MME_TU104_OUT_OP_IMM32;
297 }
298 mme_emit(&b, mme_imm(0x10000000));
299
300 mme_mthd(&b, NVC597_SET_REPORT_SEMAPHORE_A);
301 mme_emit(&b, mme_imm(high32(data_addr + 4)));
302 mme_emit(&b, mme_imm(low32(data_addr + 4)));
303 mme_tu104_asm(&b, i) {
304 i.imm[0] = 0x1234;
305 i.imm[1] = 0x7654;
306 i.out[1].emit = MME_TU104_OUT_OP_IMM32;
307 }
308 mme_emit(&b, mme_imm(0x10000000));
309
310 auto macro = mme_builder_finish_vec(&b);
311
312 std::vector<uint32_t> params;
313 test_macro(&b, macro, params);
314 }
315
316 TEST_F(mme_tu104_sim_test, reg_imm32)
317 {
318 const uint32_t canary = 0xc0ffee01;
319
320 mme_builder b;
321 mme_builder_init(&b, devinfo);
322
323 mme_value v = mme_alloc_reg(&b);
324
325 mme_tu104_asm(&b, i) {
326 i.alu[0].dst = mme_value_as_reg(v);
327 i.alu[0].op = MME_TU104_ALU_OP_ADD;
328 i.alu[0].src[0] = MME_TU104_REG_IMM32,
329 i.imm[0] = (uint16_t)canary;
330 i.imm[1] = (uint16_t)(canary >> 16);
331 }
332
333 mme_store_imm_addr(&b, data_addr, v);
334
335 auto macro = mme_builder_finish_vec(&b);
336
337 std::vector<uint32_t> params;
338 test_macro(&b, macro, params);
339 }
340
341 TEST_F(mme_tu104_sim_test, pred_alu)
342 {
343 static const enum mme_tu104_pred preds[] = {
344 MME_TU104_PRED_UUUU,
345 MME_TU104_PRED_TTTT,
346 MME_TU104_PRED_FFFF,
347 MME_TU104_PRED_TTUU,
348 MME_TU104_PRED_FFUU,
349 MME_TU104_PRED_TFUU,
350 MME_TU104_PRED_TUUU,
351 MME_TU104_PRED_FUUU,
352 MME_TU104_PRED_UUTT,
353 MME_TU104_PRED_UUTF,
354 MME_TU104_PRED_UUTU,
355 MME_TU104_PRED_UUFT,
356 MME_TU104_PRED_UUFF,
357 MME_TU104_PRED_UUFU,
358 MME_TU104_PRED_UUUT,
359 MME_TU104_PRED_UUUF,
360 };
361
362 for (uint32_t i = 0; i < ARRAY_SIZE(preds); i++) {
363 mme_builder b;
364 mme_builder_init(&b, devinfo);
365
366 mme_value pred = mme_load(&b);
367 mme_value v0 = mme_mov(&b, mme_imm(i * 100 + 13));
368 mme_value v1 = mme_mov(&b, mme_imm(i * 100 + 62));
369
370 mme_tu104_asm(&b, inst) {
371 inst.pred = mme_value_as_reg(pred);
372 inst.pred_mode = preds[i];
373 inst.alu[0].dst = mme_value_as_reg(v0);
374 inst.alu[0].src[0] = MME_TU104_REG_IMM;
375 inst.imm[0] = i * 100 + 25;
376 inst.alu[1].dst = mme_value_as_reg(v1);
377 inst.alu[1].src[0] = MME_TU104_REG_IMM;
378 inst.imm[1] = i * 100 + 73;
379 }
380
381 mme_store_imm_addr(&b, data_addr + i * 8 + 0, v0);
382 mme_store_imm_addr(&b, data_addr + i * 8 + 4, v1);
383
384 auto macro = mme_builder_finish_vec(&b);
385
386 for (uint32_t j = 0; j < 2; j++) {
387 reset_push();
388
389 std::vector<uint32_t> params;
390 params.push_back(j * 25894);
391
392 test_macro(&b, macro, params);
393 }
394 }
395 }
396
397 TEST_F(mme_tu104_sim_test, pred_out)
398 {
399 static const enum mme_tu104_pred preds[] = {
400 MME_TU104_PRED_UUUU,
401 MME_TU104_PRED_TTTT,
402 MME_TU104_PRED_FFFF,
403 MME_TU104_PRED_TTUU,
404 MME_TU104_PRED_FFUU,
405 MME_TU104_PRED_TFUU,
406 MME_TU104_PRED_TUUU,
407 MME_TU104_PRED_FUUU,
408 MME_TU104_PRED_UUTT,
409 MME_TU104_PRED_UUTF,
410 MME_TU104_PRED_UUTU,
411 MME_TU104_PRED_UUFT,
412 MME_TU104_PRED_UUFF,
413 MME_TU104_PRED_UUFU,
414 MME_TU104_PRED_UUUT,
415 MME_TU104_PRED_UUUF,
416 };
417
418 for (uint32_t i = 0; i < ARRAY_SIZE(preds); i++) {
419 mme_builder b;
420 mme_builder_init(&b, devinfo);
421
422 mme_value pred = mme_load(&b);
423
424 mme_tu104_asm(&b, inst) {
425 inst.imm[0] = (1<<12) | (NVC597_SET_MME_SHADOW_SCRATCH(i*2 + 0) >> 2);
426 inst.imm[1] = i * 100 + 25;
427 inst.out[0].mthd = MME_TU104_OUT_OP_IMM0;
428 inst.out[0].emit = MME_TU104_OUT_OP_IMM1;
429 }
430
431 mme_tu104_asm(&b, inst) {
432 inst.imm[0] = (1<<12) | (NVC597_SET_MME_SHADOW_SCRATCH(i*2 + 1) >> 2);
433 inst.imm[1] = i * 100 + 75;
434 inst.out[0].mthd = MME_TU104_OUT_OP_IMM0;
435 inst.out[0].emit = MME_TU104_OUT_OP_IMM1;
436 }
437
438 mme_tu104_asm(&b, inst) {
439 inst.pred = mme_value_as_reg(pred);
440 inst.pred_mode = preds[i];
441 inst.imm[0] = (1<<12) | (NVC597_SET_MME_SHADOW_SCRATCH(i*2 + 0) >> 2);
442 inst.imm[1] = (1<<12) | (NVC597_SET_MME_SHADOW_SCRATCH(i*2 + 1) >> 2);
443 inst.out[0].mthd = MME_TU104_OUT_OP_IMM0;
444 inst.out[0].emit = MME_TU104_OUT_OP_IMM1;
445 inst.out[1].mthd = MME_TU104_OUT_OP_IMM1;
446 inst.out[1].emit = MME_TU104_OUT_OP_IMM0;
447 }
448
449 mme_value v0 = mme_state(&b, NVC597_SET_MME_SHADOW_SCRATCH(i*2 + 0));
450 mme_value v1 = mme_state(&b, NVC597_SET_MME_SHADOW_SCRATCH(i*2 + 1));
451
452 mme_store_imm_addr(&b, data_addr + i * 8 + 0, v0);
453 mme_store_imm_addr(&b, data_addr + i * 8 + 4, v1);
454
455 auto macro = mme_builder_finish_vec(&b);
456
457 for (uint32_t j = 0; j < 2; j++) {
458 reset_push();
459
460 std::vector<uint32_t> params;
461 params.push_back(j * 25894);
462
463 test_macro(&b, macro, params);
464 }
465 }
466 }
467
468 TEST_F(mme_tu104_sim_test, add)
469 {
470 mme_builder b;
471 mme_builder_init(&b, devinfo);
472
473 mme_value x = mme_load(&b);
474 mme_value y = mme_load(&b);
475 mme_value sum = mme_add(&b, x, y);
476 mme_store_imm_addr(&b, data_addr, sum);
477
478 auto macro = mme_builder_finish_vec(&b);
479
480 std::vector<uint32_t> params;
481 params.push_back(25);
482 params.push_back(138);
483
484 test_macro(&b, macro, params);
485 }
486
487 TEST_F(mme_tu104_sim_test, add_imm)
488 {
489 mme_builder b;
490 mme_builder_init(&b, devinfo);
491
492 mme_value x = mme_load(&b);
493
494 mme_value v0 = mme_add(&b, x, mme_imm(0x00000001));
495 mme_value v1 = mme_add(&b, x, mme_imm(0xffffffff));
496 mme_value v2 = mme_add(&b, x, mme_imm(0xffff8000));
497 mme_value v3 = mme_add(&b, mme_imm(0x00000001), x);
498 mme_value v4 = mme_add(&b, mme_imm(0xffffffff), x);
499 mme_value v5 = mme_add(&b, mme_imm(0xffff8000), x);
500 mme_value v6 = mme_add(&b, mme_zero(), mme_imm(0x00000001));
501 mme_value v7 = mme_add(&b, mme_zero(), mme_imm(0xffffffff));
502 mme_value v8 = mme_add(&b, mme_zero(), mme_imm(0xffff8000));
503
504 mme_store_imm_addr(&b, data_addr + 0, v0);
505 mme_store_imm_addr(&b, data_addr + 4, v1);
506 mme_store_imm_addr(&b, data_addr + 8, v2);
507 mme_store_imm_addr(&b, data_addr + 12, v3);
508 mme_store_imm_addr(&b, data_addr + 16, v4);
509 mme_store_imm_addr(&b, data_addr + 20, v5);
510 mme_store_imm_addr(&b, data_addr + 24, v6);
511 mme_store_imm_addr(&b, data_addr + 28, v7);
512 mme_store_imm_addr(&b, data_addr + 32, v8);
513
514 auto macro = mme_builder_finish_vec(&b);
515
516 uint32_t vals[] = {
517 0x0000ffff,
518 0x00008000,
519 0x0001ffff,
520 0xffffffff,
521 };
522
523 for (uint32_t i = 0; i < ARRAY_SIZE(vals); i++) {
524 reset_push();
525
526 std::vector<uint32_t> params;
527 params.push_back(vals[i]);
528
529 test_macro(&b, macro, params);
530 }
531 }
532
533 TEST_F(mme_tu104_sim_test, addc)
534 {
535 mme_builder b;
536 mme_builder_init(&b, devinfo);
537
538 struct mme_value64 x = { mme_load(&b), mme_load(&b) };
539 struct mme_value64 y = { mme_load(&b), mme_load(&b) };
540
541 struct mme_value64 sum = mme_add64(&b, x, y);
542
543 mme_store_imm_addr(&b, data_addr + 0, sum.lo);
544 mme_store_imm_addr(&b, data_addr + 4, sum.hi);
545
546 auto macro = mme_builder_finish_vec(&b);
547
548 std::vector<uint32_t> params;
549 params.push_back(0x80008650);
550 params.push_back(0x596);
551 params.push_back(0x8000a8f6);
552 params.push_back(0x836);
553
554 test_macro(&b, macro, params);
555 }
556
557 TEST_F(mme_tu104_sim_test, addc_imm)
558 {
559 mme_builder b;
560 mme_builder_init(&b, devinfo);
561
562 mme_value x_lo = mme_load(&b);
563 mme_value x_hi = mme_load(&b);
564
565 mme_value v1_lo = mme_alloc_reg(&b);
566 mme_value v1_hi = mme_alloc_reg(&b);
567 mme_tu104_asm(&b, i) {
568 i.alu[0].dst = mme_value_as_reg(v1_lo);
569 i.alu[0].op = MME_TU104_ALU_OP_ADD;
570 i.alu[0].src[0] = mme_value_as_reg(x_lo);
571 i.alu[0].src[1] = MME_TU104_REG_IMM;
572 i.imm[0] = 0x0001;
573 i.alu[1].dst = mme_value_as_reg(v1_hi);
574 i.alu[1].op = MME_TU104_ALU_OP_ADDC;
575 i.alu[1].src[0] = mme_value_as_reg(x_hi);
576 i.alu[1].src[1] = MME_TU104_REG_IMM;
577 i.imm[1] = 0x0000;
578 }
579
580 mme_value v2_lo = mme_alloc_reg(&b);
581 mme_value v2_hi = mme_alloc_reg(&b);
582 mme_tu104_asm(&b, i) {
583 i.alu[0].dst = mme_value_as_reg(v2_lo);
584 i.alu[0].op = MME_TU104_ALU_OP_ADD;
585 i.alu[0].src[0] = mme_value_as_reg(x_lo);
586 i.alu[0].src[1] = MME_TU104_REG_IMM;
587 i.imm[0] = 0x0000;
588 i.alu[1].dst = mme_value_as_reg(v2_hi);
589 i.alu[1].op = MME_TU104_ALU_OP_ADDC;
590 i.alu[1].src[0] = mme_value_as_reg(x_hi);
591 i.alu[1].src[1] = MME_TU104_REG_IMM;
592 i.imm[1] = 0x0001;
593 }
594
595 mme_value v3_lo = mme_alloc_reg(&b);
596 mme_value v3_hi = mme_alloc_reg(&b);
597 mme_tu104_asm(&b, i) {
598 i.alu[0].dst = mme_value_as_reg(v3_lo);
599 i.alu[0].op = MME_TU104_ALU_OP_ADD;
600 i.alu[0].src[0] = mme_value_as_reg(x_lo);
601 i.alu[0].src[1] = MME_TU104_REG_IMM;
602 i.imm[0] = 0x0000;
603 i.alu[1].dst = mme_value_as_reg(v3_hi);
604 i.alu[1].op = MME_TU104_ALU_OP_ADDC;
605 i.alu[1].src[0] = mme_value_as_reg(x_hi);
606 i.alu[1].src[1] = MME_TU104_REG_IMM;
607 i.imm[1] = 0xffff;
608 }
609
610 mme_value v4_lo = mme_alloc_reg(&b);
611 mme_value v4_hi = mme_alloc_reg(&b);
612 mme_tu104_asm(&b, i) {
613 i.alu[0].dst = mme_value_as_reg(v4_lo);
614 i.alu[0].op = MME_TU104_ALU_OP_ADD;
615 i.alu[0].src[0] = mme_value_as_reg(x_lo);
616 i.alu[0].src[1] = MME_TU104_REG_IMM;
617 i.imm[0] = 0x0000;
618 i.alu[1].dst = mme_value_as_reg(v4_hi);
619 i.alu[1].op = MME_TU104_ALU_OP_ADDC;
620 i.alu[1].src[0] = mme_value_as_reg(x_hi);
621 i.alu[1].src[1] = MME_TU104_REG_IMM;
622 i.imm[1] = 0x8000;
623 }
624
625 mme_store_imm_addr(&b, data_addr + 0, v1_lo);
626 mme_store_imm_addr(&b, data_addr + 4, v1_hi);
627 mme_store_imm_addr(&b, data_addr + 8, v2_lo);
628 mme_store_imm_addr(&b, data_addr + 12, v2_hi);
629 mme_store_imm_addr(&b, data_addr + 16, v3_lo);
630 mme_store_imm_addr(&b, data_addr + 20, v3_hi);
631 mme_store_imm_addr(&b, data_addr + 24, v4_lo);
632 mme_store_imm_addr(&b, data_addr + 28, v4_hi);
633
634 auto macro = mme_builder_finish_vec(&b);
635
636 uint64_t vals[] = {
637 0x0000ffffffffffffull,
638 0x0000ffffffff8000ull,
639 0x0000ffff00000000ull,
640 0x0000800000000000ull,
641 0x00008000ffffffffull,
642 0x0001ffff00000000ull,
643 0xffffffff00000000ull,
644 0xffffffffffffffffull,
645 };
646
647 for (uint32_t i = 0; i < ARRAY_SIZE(vals); i++) {
648 reset_push();
649
650 std::vector<uint32_t> params;
651 params.push_back(low32(vals[i]));
652 params.push_back(high32(vals[i]));
653
654 test_macro(&b, macro, params);
655 }
656 }
657
658 TEST_F(mme_tu104_sim_test, sub)
659 {
660 mme_builder b;
661 mme_builder_init(&b, devinfo);
662
663 mme_value x = mme_load(&b);
664 mme_value y = mme_load(&b);
665 mme_value diff = mme_sub(&b, x, y);
666 mme_store_imm_addr(&b, data_addr, diff);
667
668 auto macro = mme_builder_finish_vec(&b);
669
670 std::vector<uint32_t> params;
671 params.push_back(25);
672 params.push_back(138);
673
674 test_macro(&b, macro, params);
675 }
676
677 TEST_F(mme_tu104_sim_test, subb)
678 {
679 mme_builder b;
680 mme_builder_init(&b, devinfo);
681
682 struct mme_value64 x = { mme_load(&b), mme_load(&b) };
683 struct mme_value64 y = { mme_load(&b), mme_load(&b) };
684
685 struct mme_value64 diff = mme_sub64(&b, x, y);
686
687 mme_store_imm_addr(&b, data_addr + 0, diff.lo);
688 mme_store_imm_addr(&b, data_addr + 4, diff.hi);
689
690 auto macro = mme_builder_finish_vec(&b);
691
692 std::vector<uint32_t> params;
693 params.push_back(0x80008650);
694 params.push_back(0x596);
695 params.push_back(0x8000a8f6);
696 params.push_back(0x836);
697
698 test_macro(&b, macro, params);
699 }
700
701 TEST_F(mme_tu104_sim_test, mul)
702 {
703 mme_builder b;
704 mme_builder_init(&b, devinfo);
705
706 mme_value x = mme_load(&b);
707 mme_value y = mme_load(&b);
708 mme_value sum = mme_mul(&b, x, y);
709 mme_store_imm_addr(&b, data_addr, sum);
710
711 auto macro = mme_builder_finish_vec(&b);
712
713 std::vector<uint32_t> params;
714 params.push_back(25);
715 params.push_back(138);
716
717 test_macro(&b, macro, params);
718 }
719
720 TEST_F(mme_tu104_sim_test, mul_imm)
721 {
722 mme_builder b;
723 mme_builder_init(&b, devinfo);
724
725 mme_value x = mme_load(&b);
726
727 mme_value v0 = mme_mul(&b, x, mme_imm(0x00000001));
728 mme_value v1 = mme_mul(&b, x, mme_imm(0xffffffff));
729 mme_value v2 = mme_mul(&b, x, mme_imm(0xffff8000));
730 mme_value v3 = mme_mul(&b, mme_imm(0x00000001), x);
731 mme_value v4 = mme_mul(&b, mme_imm(0xffffffff), x);
732 mme_value v5 = mme_mul(&b, mme_imm(0xffff8000), x);
733
734 mme_store_imm_addr(&b, data_addr + 0, v0);
735 mme_store_imm_addr(&b, data_addr + 4, v1);
736 mme_store_imm_addr(&b, data_addr + 8, v2);
737 mme_store_imm_addr(&b, data_addr + 12, v3);
738 mme_store_imm_addr(&b, data_addr + 16, v4);
739 mme_store_imm_addr(&b, data_addr + 20, v5);
740
741 auto macro = mme_builder_finish_vec(&b);
742
743 int32_t vals[] = { 1, -5, -1, 5 };
744
745 for (uint32_t i = 0; i < ARRAY_SIZE(vals); i++) {
746 reset_push();
747
748 std::vector<uint32_t> params;
749 params.push_back(vals[i]);
750
751 test_macro(&b, macro, params);
752 }
753 }
754
755 TEST_F(mme_tu104_sim_test, mul_mulh)
756 {
757 mme_builder b;
758 mme_builder_init(&b, devinfo);
759
760 mme_value x = mme_load(&b);
761 mme_value y = mme_load(&b);
762
763 struct mme_value64 prod = mme_imul_32x32_64(&b, x, y);
764
765 mme_store_imm_addr(&b, data_addr + 0, prod.lo);
766 mme_store_imm_addr(&b, data_addr + 4, prod.hi);
767
768 auto macro = mme_builder_finish_vec(&b);
769
770 std::vector<uint32_t> params;
771 params.push_back(0x80008650);
772 params.push_back(0x596);
773
774 test_macro(&b, macro, params);
775 }
776
777 static inline struct mme_value
778 mme_mulu(struct mme_builder *b, struct mme_value x, struct mme_value y)
779 {
780 return mme_alu(b, MME_ALU_OP_MULU, x, y);
781 }
782
783 TEST_F(mme_tu104_sim_test, mulu_imm)
784 {
785 mme_builder b;
786 mme_builder_init(&b, devinfo);
787
788 mme_value x = mme_load(&b);
789
790 mme_value v0 = mme_mulu(&b, x, mme_imm(0x00000001));
791 mme_value v1 = mme_mulu(&b, x, mme_imm(0xffffffff));
792 mme_value v2 = mme_mulu(&b, x, mme_imm(0xffff8000));
793 mme_value v3 = mme_mulu(&b, mme_imm(0x00000001), x);
794 mme_value v4 = mme_mulu(&b, mme_imm(0xffffffff), x);
795 mme_value v5 = mme_mulu(&b, mme_imm(0xffff8000), x);
796
797 mme_store_imm_addr(&b, data_addr + 0, v0);
798 mme_store_imm_addr(&b, data_addr + 4, v1);
799 mme_store_imm_addr(&b, data_addr + 8, v2);
800 mme_store_imm_addr(&b, data_addr + 12, v3);
801 mme_store_imm_addr(&b, data_addr + 16, v4);
802 mme_store_imm_addr(&b, data_addr + 20, v5);
803
804 auto macro = mme_builder_finish_vec(&b);
805
806 int32_t vals[] = { 1, -5, -1, 5 };
807
808 for (uint32_t i = 0; i < ARRAY_SIZE(vals); i++) {
809 reset_push();
810
811 std::vector<uint32_t> params;
812 params.push_back(vals[i]);
813
814 test_macro(&b, macro, params);
815 }
816 }
817
818 TEST_F(mme_tu104_sim_test, mulu_mulh)
819 {
820 mme_builder b;
821 mme_builder_init(&b, devinfo);
822
823 mme_value x = mme_load(&b);
824 mme_value y = mme_load(&b);
825
826 struct mme_value64 prod = mme_umul_32x32_64(&b, x, y);
827
828 mme_store_imm_addr(&b, data_addr + 0, prod.lo);
829 mme_store_imm_addr(&b, data_addr + 4, prod.hi);
830
831 auto macro = mme_builder_finish_vec(&b);
832
833 std::vector<uint32_t> params;
834 params.push_back(0x80008650);
835 params.push_back(0x596);
836
837 test_macro(&b, macro, params);
838 }
839
840 TEST_F(mme_tu104_sim_test, clz)
841 {
842 mme_builder b;
843 mme_builder_init(&b, devinfo);
844
845 mme_value bits = mme_clz(&b, mme_load(&b));
846 mme_store_imm_addr(&b, data_addr, bits);
847
848 auto macro = mme_builder_finish_vec(&b);
849
850 std::vector<uint32_t> params;
851 params.push_back(0x00406fe0);
852
853 test_macro(&b, macro, params);
854 }
855
856 #define SHIFT_TEST(op) \
857 TEST_F(mme_tu104_sim_test, op) \
858 { \
859 mme_builder b; \
860 mme_builder_init(&b, devinfo); \
861 \
862 mme_value val = mme_load(&b); \
863 mme_value shift1 = mme_load(&b); \
864 mme_value shift2 = mme_load(&b); \
865 mme_store_imm_addr(&b, data_addr + 0, mme_##op(&b, val, shift1)); \
866 mme_store_imm_addr(&b, data_addr + 4, mme_##op(&b, val, shift2)); \
867 \
868 auto macro = mme_builder_finish_vec(&b); \
869 \
870 std::vector<uint32_t> params; \
871 params.push_back(0x0c406fe0); \
872 params.push_back(5); \
873 params.push_back(51); \
874 \
875 test_macro(&b, macro, params); \
876 }
877
878 SHIFT_TEST(sll)
879 SHIFT_TEST(srl)
880 SHIFT_TEST(sra)
881
882 #undef SHIFT_TEST
883
884 TEST_F(mme_tu104_sim_test, bfe)
885 {
886 const uint32_t canary = 0xc0ffee01;
887
888 mme_builder b;
889 mme_builder_init(&b, devinfo);
890
891 mme_value val = mme_load(&b);
892 mme_value pos = mme_load(&b);
893
894 mme_store_imm_addr(&b, data_addr + 0, mme_bfe(&b, val, pos, 1), true);
895 mme_store_imm_addr(&b, data_addr + 4, mme_bfe(&b, val, pos, 2), true);
896 mme_store_imm_addr(&b, data_addr + 8, mme_bfe(&b, val, pos, 5), true);
897
898 auto macro = mme_builder_finish_vec(&b);
899
900 for (unsigned i = 0; i < 31; i++) {
901 std::vector<uint32_t> params;
902 params.push_back(canary);
903 params.push_back(i);
904
905 test_macro(&b, macro, params);
906
907 ASSERT_EQ(data[0], (canary >> i) & 0x1);
908 ASSERT_EQ(data[1], (canary >> i) & 0x3);
909 ASSERT_EQ(data[2], (canary >> i) & 0x1f);
910 }
911 }
912
913 TEST_F(mme_tu104_sim_test, not)
914 {
915 mme_builder b;
916 mme_builder_init(&b, devinfo);
917
918 mme_value x = mme_load(&b);
919 mme_value v1 = mme_not(&b, x);
920 mme_store_imm_addr(&b, data_addr + 0, v1);
921
922 auto macro = mme_builder_finish_vec(&b);
923
924 std::vector<uint32_t> params;
925 params.push_back(0x0c406fe0);
926
927 test_macro(&b, macro, params);
928 }
929
930 #define BITOP_TEST(op) \
931 TEST_F(mme_tu104_sim_test, op) \
932 { \
933 mme_builder b; \
934 mme_builder_init(&b, devinfo); \
935 \
936 mme_value x = mme_load(&b); \
937 mme_value y = mme_load(&b); \
938 mme_value v1 = mme_##op(&b, x, y); \
939 mme_value v2 = mme_##op(&b, x, mme_imm(0xffff8000)); \
940 mme_value v3 = mme_##op(&b, x, mme_imm(0xffffffff)); \
941 mme_store_imm_addr(&b, data_addr + 0, v1); \
942 mme_store_imm_addr(&b, data_addr + 4, v2); \
943 mme_store_imm_addr(&b, data_addr + 8, v3); \
944 \
945 auto macro = mme_builder_finish_vec(&b); \
946 \
947 std::vector<uint32_t> params; \
948 params.push_back(0x0c406fe0); \
949 params.push_back(0x00fff0c0); \
950 \
951 test_macro(&b, macro, params); \
952 }
953
954 BITOP_TEST(and)
955 BITOP_TEST(and_not)
956 BITOP_TEST(nand)
957 BITOP_TEST(or)
958 BITOP_TEST(xor)
959
960 #undef BITOP_TEST
961
962 TEST_F(mme_tu104_sim_test, merge)
963 {
964 mme_builder b;
965 mme_builder_init(&b, devinfo);
966
967 mme_value x = mme_load(&b);
968 mme_value y = mme_load(&b);
969
970 mme_value m1 = mme_merge(&b, x, y, 12, 12, 20);
971 mme_value m2 = mme_merge(&b, x, y, 12, 8, 20);
972 mme_value m3 = mme_merge(&b, x, y, 8, 12, 20);
973 mme_value m4 = mme_merge(&b, x, y, 12, 16, 8);
974 mme_value m5 = mme_merge(&b, x, y, 24, 12, 8);
975
976 mme_store_imm_addr(&b, data_addr + 0, m1);
977 mme_store_imm_addr(&b, data_addr + 4, m2);
978 mme_store_imm_addr(&b, data_addr + 8, m3);
979 mme_store_imm_addr(&b, data_addr + 12, m4);
980 mme_store_imm_addr(&b, data_addr + 16, m5);
981
982 auto macro = mme_builder_finish_vec(&b);
983
984 std::vector<uint32_t> params;
985 params.push_back(0x0c406fe0);
986 params.push_back(0x76543210u);
987
988 test_macro(&b, macro, params);
989 }
990
991 #define COMPARISON_TEST(op) \
992 TEST_F(mme_tu104_sim_test, op) \
993 { \
994 mme_builder b; \
995 mme_builder_init(&b, devinfo); \
996 \
997 mme_value x = mme_load(&b); \
998 mme_value y = mme_load(&b); \
999 mme_value z = mme_load(&b); \
1000 mme_value w = mme_load(&b); \
1001 \
1002 mme_value v1 = mme_##op(&b, x, y); \
1003 mme_value v2 = mme_##op(&b, y, x); \
1004 mme_value v3 = mme_##op(&b, y, z); \
1005 mme_value v4 = mme_##op(&b, z, y); \
1006 mme_value v5 = mme_##op(&b, w, z); \
1007 mme_value v6 = mme_##op(&b, z, w); \
1008 mme_value v7 = mme_##op(&b, w, w); \
1009 \
1010 mme_store_imm_addr(&b, data_addr + 0, v1); \
1011 mme_store_imm_addr(&b, data_addr + 4, v2); \
1012 mme_store_imm_addr(&b, data_addr + 8, v3); \
1013 mme_store_imm_addr(&b, data_addr + 12, v4); \
1014 mme_store_imm_addr(&b, data_addr + 16, v5); \
1015 mme_store_imm_addr(&b, data_addr + 20, v6); \
1016 mme_store_imm_addr(&b, data_addr + 24, v7); \
1017 \
1018 auto macro = mme_builder_finish_vec(&b); \
1019 \
1020 std::vector<uint32_t> params; \
1021 params.push_back(-5); \
1022 params.push_back(-10); \
1023 params.push_back(5); \
1024 params.push_back(10); \
1025 \
1026 test_macro(&b, macro, params); \
1027 }
1028
1029 COMPARISON_TEST(slt)
1030 COMPARISON_TEST(sltu)
1031 COMPARISON_TEST(sle)
1032 COMPARISON_TEST(sleu)
1033 COMPARISON_TEST(seq)
1034
1035 #undef COMPARISON_TEST
1036
1037 static inline void
1038 mme_inc_whole_inst(mme_builder *b, mme_value val)
1039 {
1040 mme_tu104_asm(b, i) {
1041 i.alu[0].dst = mme_value_as_reg(val);
1042 i.alu[0].op = MME_TU104_ALU_OP_ADD;
1043 i.alu[0].src[0] = mme_value_as_reg(val);
1044 i.alu[0].src[1] = MME_TU104_REG_IMM;
1045 i.imm[0] = 1;
1046 }
1047 }
1048
1049 TEST_F(mme_tu104_sim_test, loop)
1050 {
1051 mme_builder b;
1052 mme_builder_init(&b, devinfo);
1053
1054 mme_value count = mme_load(&b);
1055
1056 mme_value x = mme_mov(&b, mme_zero());
1057 mme_value y = mme_mov(&b, mme_zero());
1058
1059 mme_loop(&b, count) {
1060 mme_tu104_asm(&b, i) { } /* noop */
1061 mme_add_to(&b, x, x, count);
1062 }
1063 mme_add_to(&b, y, y, mme_imm(1));
1064 mme_tu104_asm(&b, i) { } /* noop */
1065 mme_tu104_asm(&b, i) { } /* noop */
1066 mme_tu104_asm(&b, i) { } /* noop */
1067
1068 mme_store_imm_addr(&b, data_addr + 0, count);
1069 mme_store_imm_addr(&b, data_addr + 4, x);
1070 mme_store_imm_addr(&b, data_addr + 8, y);
1071
1072 auto macro = mme_builder_finish_vec(&b);
1073
1074 uint32_t counts[] = {0, 1, 5, 9};
1075
1076 for (uint32_t i = 0; i < ARRAY_SIZE(counts); i++) {
1077 reset_push();
1078
1079 std::vector<uint32_t> params;
1080 params.push_back(counts[i]);
1081
1082 test_macro(&b, macro, params);
1083 ASSERT_EQ(data[0], counts[i]);
1084 ASSERT_EQ(data[1], counts[i] * counts[i]);
1085 ASSERT_EQ(data[2], 1);
1086 }
1087 }
1088
1089 TEST_F(mme_tu104_sim_test, jal)
1090 {
1091 mme_builder b;
1092 mme_builder_init(&b, devinfo);
1093
1094 mme_value x = mme_mov(&b, mme_zero());
1095 mme_value y = mme_mov(&b, mme_zero());
1096
1097 mme_tu104_asm(&b, i) {
1098 i.alu[0].op = MME_TU104_ALU_OP_JAL;
1099 i.imm[0] = (1 << 15) | 6;
1100 }
1101
1102 for (uint32_t j = 0; j < 10; j++)
1103 mme_inc_whole_inst(&b, x);
1104
1105 // mme_tu104_asm(&b, i) {
1106 // i.alu[0].op = MME_TU104_ALU_OP_JAL;
1107 // i.imm[0] = 6;
1108 // }
1109 //
1110 // for (uint32_t j = 0; j < 10; j++)
1111 // mme_inc_whole_inst(&b, y);
1112
1113 mme_store_imm_addr(&b, data_addr + 0, x);
1114 mme_store_imm_addr(&b, data_addr + 4, y);
1115
1116 auto macro = mme_builder_finish_vec(&b);
1117
1118 std::vector<uint32_t> params;
1119 test_macro(&b, macro, params);
1120 ASSERT_EQ(data[0], 5);
1121 }
1122
1123 TEST_F(mme_tu104_sim_test, bxx_fwd)
1124 {
1125 mme_builder b;
1126 mme_builder_init(&b, devinfo);
1127
1128 mme_value vals[10];
1129 for (uint32_t i = 0; i < 10; i++)
1130 vals[i] = mme_mov(&b, mme_zero());
1131
1132 mme_tu104_asm(&b, i) {
1133 i.alu[0].op = MME_TU104_ALU_OP_BEQ;
1134 i.imm[0] = (1 << 15) | 6;
1135 }
1136
1137 for (uint32_t j = 0; j < 10; j++)
1138 mme_inc_whole_inst(&b, vals[j]);
1139
1140 for (uint32_t j = 0; j < 10; j++)
1141 mme_store_imm_addr(&b, data_addr + j * 4, vals[j]);
1142
1143 auto macro = mme_builder_finish_vec(&b);
1144
1145 std::vector<uint32_t> params;
1146 test_macro(&b, macro, params);
1147 }
1148
1149 TEST_F(mme_tu104_sim_test, bxx_bwd)
1150 {
1151 mme_builder b;
1152 mme_builder_init(&b, devinfo);
1153
1154 mme_value vals[15];
1155 for (uint32_t i = 0; i < 15; i++)
1156 vals[i] = mme_mov(&b, mme_zero());
1157
1158 mme_tu104_asm(&b, i) {
1159 i.alu[0].op = MME_TU104_ALU_OP_JAL;
1160 i.imm[0] = (1 << 15) | 12;
1161 }
1162
1163 for (uint32_t j = 0; j < 10; j++)
1164 mme_inc_whole_inst(&b, vals[j]);
1165
1166 mme_tu104_asm(&b, i) {
1167 i.alu[0].op = MME_TU104_ALU_OP_JAL;
1168 i.imm[0] = (1 << 15) | 2;
1169 }
1170
1171 mme_tu104_asm(&b, i) {
1172 i.alu[0].op = MME_TU104_ALU_OP_BEQ;
1173 i.imm[0] = (1 << 15) | ((-8) & 0x1fff);
1174 }
1175
1176 for (uint32_t j = 10; j < 15; j++)
1177 mme_inc_whole_inst(&b, vals[j]);
1178
1179 for (uint32_t j = 0; j < 15; j++)
1180 mme_store_imm_addr(&b, data_addr + j * 4, vals[j]);
1181
1182 auto macro = mme_builder_finish_vec(&b);
1183
1184 std::vector<uint32_t> params;
1185 test_macro(&b, macro, params);
1186 for (uint32_t j = 0; j < 3; j++)
1187 ASSERT_EQ(data[j], 0);
1188 for (uint32_t j = 3; j < 15; j++)
1189 ASSERT_EQ(data[j], 1);
1190 }
1191
1192 TEST_F(mme_tu104_sim_test, bxx_exit)
1193 {
1194 mme_builder b;
1195 mme_builder_init(&b, devinfo);
1196
1197 mme_value vals[10];
1198 for (uint32_t i = 0; i < 10; i++)
1199 vals[i] = mme_mov(&b, mme_zero());
1200
1201 for (uint32_t i = 0; i < 10; i++)
1202 mme_store_imm_addr(&b, data_addr + i * 4, mme_imm(0));
1203
1204 mme_tu104_asm(&b, i) {
1205 i.alu[0].op = MME_TU104_ALU_OP_BEQ;
1206 i.imm[0] = (1 << 15) | 0x1000;
1207 }
1208
1209 /* those writes won't be visible */
1210 for (uint32_t j = 0; j < 10; j++)
1211 mme_inc_whole_inst(&b, vals[j]);
1212
1213 for (uint32_t i = 0; i < 10; i++)
1214 mme_store_imm_addr(&b, data_addr + i * 4, vals[i]);
1215
1216 std::vector<uint32_t> params;
1217
1218 auto macro = mme_builder_finish_vec(&b);
1219 test_macro(&b, macro, params);
1220
1221 uint32_t i;
1222 for (i = 0; i < 10; i++)
1223 ASSERT_EQ(data[i], 0);
1224 }
1225
1226 TEST_F(mme_tu104_sim_test, mme_exit)
1227 {
1228 mme_builder b;
1229 mme_builder_init(&b, devinfo);
1230
1231 mme_value vals[10];
1232 for (uint32_t i = 0; i < 10; i++)
1233 vals[i] = mme_mov(&b, mme_zero());
1234
1235 for (uint32_t i = 0; i < 10; i++)
1236 mme_store_imm_addr(&b, data_addr + i * 4, mme_imm(0));
1237
1238 /* abort */
1239 mme_exit(&b);
1240
1241 /* those writes won't be visible */
1242 for (uint32_t i = 0; i < 10; i++)
1243 vals[i] = mme_mov(&b, mme_imm(i));
1244
1245 for (uint32_t i = 0; i < 10; i++) {
1246 mme_store_imm_addr(&b, data_addr + i * 4, vals[i]);
1247 }
1248
1249 std::vector<uint32_t> params;
1250
1251 auto macro = mme_builder_finish_vec(&b);
1252 test_macro(&b, macro, params);
1253
1254 uint32_t i;
1255 for (i = 0; i < 10; i++)
1256 ASSERT_EQ(data[i], 0);
1257 }
1258
1259 TEST_F(mme_tu104_sim_test, mme_exit_if)
1260 {
1261 mme_builder b;
1262 mme_builder_init(&b, devinfo);
1263
1264 mme_value vals[10];
1265 for (uint32_t i = 0; i < 10; i++)
1266 vals[i] = mme_mov(&b, mme_zero());
1267
1268 for (uint32_t i = 0; i < 10; i++)
1269 mme_store_imm_addr(&b, data_addr + i * 4, mme_imm(0));
1270
1271 /* shouldn't do anything */
1272 mme_exit_if(&b, ieq, mme_zero(), mme_imm(1));
1273
1274 for (uint32_t i = 0; i < 10; i++)
1275 vals[i] = mme_mov(&b, mme_imm(i));
1276
1277 for (uint32_t i = 0; i < 10; i++) {
1278 /* abort on reaching 5 */
1279 mme_exit_if(&b, ile, mme_imm(5), vals[i]);
1280 mme_store_imm_addr(&b, data_addr + i * 4, vals[i]);
1281 }
1282
1283 std::vector<uint32_t> params;
1284
1285 auto macro = mme_builder_finish_vec(&b);
1286 test_macro(&b, macro, params);
1287
1288 uint32_t i;
1289 for (i = 0; i < 10; i++)
1290 ASSERT_EQ(data[i], i < 5 ? i : 0);
1291 }
1292
1293 static bool c_ilt(int32_t x, int32_t y) { return x < y; };
1294 static bool c_ult(uint32_t x, uint32_t y) { return x < y; };
1295 static bool c_ile(int32_t x, int32_t y) { return x <= y; };
1296 static bool c_ule(uint32_t x, uint32_t y) { return x <= y; };
1297 static bool c_ieq(int32_t x, int32_t y) { return x == y; };
1298 static bool c_ige(int32_t x, int32_t y) { return x >= y; };
1299 static bool c_uge(uint32_t x, uint32_t y) { return x >= y; };
1300 static bool c_igt(int32_t x, int32_t y) { return x > y; };
1301 static bool c_ugt(uint32_t x, uint32_t y) { return x > y; };
1302 static bool c_ine(int32_t x, int32_t y) { return x != y; };
1303
1304 #define IF_TEST(op) \
1305 TEST_F(mme_tu104_sim_test, if_##op) \
1306 { \
1307 mme_builder b; \
1308 mme_builder_init(&b, devinfo); \
1309 \
1310 mme_value x = mme_load(&b); \
1311 mme_value y = mme_load(&b); \
1312 mme_value i = mme_mov(&b, mme_zero()); \
1313 \
1314 mme_start_if_##op(&b, x, y); \
1315 { \
1316 mme_add_to(&b, i, i, mme_imm(1)); \
1317 mme_add_to(&b, i, i, mme_imm(1)); \
1318 } \
1319 mme_end_if(&b); \
1320 mme_start_if_##op(&b, x, mme_imm(56)); \
1321 { \
1322 mme_add_to(&b, i, i, mme_imm(1)); \
1323 } \
1324 mme_end_if(&b); \
1325 mme_add_to(&b, i, i, mme_imm(1)); \
1326 mme_add_to(&b, i, i, mme_imm(1)); \
1327 mme_add_to(&b, i, i, mme_imm(1)); \
1328 \
1329 mme_store_imm_addr(&b, data_addr + 0, i); \
1330 \
1331 auto macro = mme_builder_finish_vec(&b); \
1332 \
1333 uint32_t vals[] = {23, 56, (uint32_t)-5, (uint32_t)-10, 56, 14}; \
1334 \
1335 for (uint32_t i = 0; i < ARRAY_SIZE(vals) - 1; i++) { \
1336 reset_push(); \
1337 \
1338 std::vector<uint32_t> params; \
1339 params.push_back(vals[i + 0]); \
1340 params.push_back(vals[i + 1]); \
1341 \
1342 test_macro(&b, macro, params); \
1343 \
1344 uint32_t expected = 3; \
1345 if (c_##op(params[0], params[1])) \
1346 expected += 2; \
1347 if (c_##op(params[0], 56)) \
1348 expected += 1; \
1349 \
1350 ASSERT_EQ(data[0], expected); \
1351 } \
1352 }
1353
1354 IF_TEST(ilt)
1355 IF_TEST(ult)
1356 IF_TEST(ile)
1357 IF_TEST(ule)
1358 IF_TEST(ieq)
1359 IF_TEST(ige)
1360 IF_TEST(uge)
1361 IF_TEST(igt)
1362 IF_TEST(ugt)
1363 IF_TEST(ine)
1364
1365 #undef IF_TEST
1366
1367 #define WHILE_TEST(op, start, step, bound) \
1368 TEST_F(mme_tu104_sim_test, while_##op) \
1369 { \
1370 mme_builder b; \
1371 mme_builder_init(&b, devinfo); \
1372 \
1373 mme_value x = mme_mov(&b, mme_zero()); \
1374 mme_value y = mme_mov(&b, mme_zero()); \
1375 mme_value z = mme_mov(&b, mme_imm(start)); \
1376 mme_value w = mme_mov(&b, mme_zero()); \
1377 mme_value v = mme_mov(&b, mme_zero()); \
1378 \
1379 for (uint32_t j = 0; j < 5; j++) \
1380 mme_inc_whole_inst(&b, x); \
1381 \
1382 mme_while(&b, op, z, mme_imm(bound)) { \
1383 for (uint32_t j = 0; j < 5; j++) \
1384 mme_inc_whole_inst(&b, y); \
1385 \
1386 mme_add_to(&b, z, z, mme_imm(step)); \
1387 \
1388 for (uint32_t j = 0; j < 5; j++) \
1389 mme_inc_whole_inst(&b, w); \
1390 } \
1391 \
1392 for (uint32_t j = 0; j < 5; j++) \
1393 mme_inc_whole_inst(&b, v); \
1394 \
1395 mme_store_imm_addr(&b, data_addr + 0, x); \
1396 mme_store_imm_addr(&b, data_addr + 4, y); \
1397 mme_store_imm_addr(&b, data_addr + 8, z); \
1398 mme_store_imm_addr(&b, data_addr + 12, w); \
1399 mme_store_imm_addr(&b, data_addr + 16, v); \
1400 \
1401 auto macro = mme_builder_finish_vec(&b); \
1402 \
1403 uint32_t end = (uint32_t)(start), count = 0; \
1404 while (c_##op(end, (bound))) { \
1405 end += (uint32_t)(step); \
1406 count++; \
1407 } \
1408 \
1409 std::vector<uint32_t> params; \
1410 test_macro(&b, macro, params); \
1411 ASSERT_EQ(data[0], 5); \
1412 ASSERT_EQ(data[1], 5 * count); \
1413 ASSERT_EQ(data[2], end); \
1414 ASSERT_EQ(data[3], 5 * count); \
1415 ASSERT_EQ(data[4], 5); \
1416 }
1417
1418 WHILE_TEST(ilt, 0, 1, 7)
1419 WHILE_TEST(ult, 0, 1, 7)
1420 WHILE_TEST(ile, -10, 2, 0)
1421 WHILE_TEST(ule, 0, 1, 7)
1422 WHILE_TEST(ieq, 0, 5, 0)
1423 WHILE_TEST(ige, 5, -1, -5)
1424 WHILE_TEST(uge, 15, -2, 2)
1425 WHILE_TEST(igt, 7, -3, -10)
1426 WHILE_TEST(ugt, 1604, -30, 1000)
1427 WHILE_TEST(ine, 0, 1, 7)
1428
1429 #undef WHILE_TEST
1430
1431 TEST_F(mme_tu104_sim_test, nested_while)
1432 {
1433 mme_builder b;
1434 mme_builder_init(&b, devinfo);
1435
1436 mme_value n = mme_load(&b);
1437 mme_value m = mme_load(&b);
1438
1439 mme_value count = mme_mov(&b, mme_zero());
1440
1441 mme_value i = mme_mov(&b, mme_zero());
1442 mme_value j = mme_mov(&b, mme_imm(0xffff));
1443 mme_while(&b, ine, i, n) {
1444 mme_mov_to(&b, j, mme_zero());
1445 mme_while(&b, ine, j, m) {
1446 mme_add_to(&b, count, count, mme_imm(1));
1447 mme_add_to(&b, j, j, mme_imm(1));
1448 }
1449
1450 mme_add_to(&b, i, i, mme_imm(1));
1451 }
1452
1453 mme_store_imm_addr(&b, data_addr + 0, i);
1454 mme_store_imm_addr(&b, data_addr + 4, j);
1455 mme_store_imm_addr(&b, data_addr + 8, count);
1456
1457 auto macro = mme_builder_finish_vec(&b);
1458
1459 std::vector<uint32_t> params;
1460 params.push_back(3);
1461 params.push_back(5);
1462
1463 test_macro(&b, macro, params);
1464 ASSERT_EQ(data[0], 3);
1465 ASSERT_EQ(data[1], 5);
1466 ASSERT_EQ(data[2], 15);
1467 }
1468
1469 #if 0
1470 TEST_F(mme_tu104_sim_test, do_ble)
1471 {
1472 mme_builder b;
1473 mme_builder_init(&b, devinfo);
1474
1475 mme_alu(&b, R5, ADD, LOAD0, ZERO);
1476 mme_alu(&b, R6, ADD, ZERO, ZERO);
1477 mme_alu(&b, R7, ADD, ZERO, ZERO);
1478
1479 mme_alu_imm(&b, R7, ADD, R7, IMM, 1);
1480 mme_alu_imm(&b, R7, ADD, R7, IMM, 1);
1481 mme_alu_imm(&b, R7, ADD, R7, IMM, 1);
1482 mme_alu_imm(&b, R7, ADD, R7, IMM, 1);
1483 mme_alu_imm(&b, R6, ADD, R6, IMM, 1);
1484 mme_branch(&b, BLE, R6, R5, -3, 2);
1485 mme_alu_imm(&b, R7, ADD, R7, IMM, 1);
1486 mme_alu_imm(&b, R7, ADD, R7, IMM, 1);
1487
1488 mme_store_imm_addr(&b, data_addr + 0, MME_TU104_REG_R7);
1489
1490 mme_end(&b);
1491
1492 uint32_t counts[] = {0, 1, 5, 9};
1493
1494 for (uint32_t i = 0; i < ARRAY_SIZE(counts); i++) {
1495 reset_push();
1496
1497 std::vector<uint32_t> params;
1498 params.push_back(counts[i]);
1499
1500 test_macro(&b, params);
1501 }
1502 }
1503 #endif
1504
1505 TEST_F(mme_tu104_sim_test, dread_dwrite)
1506 {
1507 mme_builder b;
1508 mme_builder_init(&b, devinfo);
1509
1510 mme_value x = mme_load(&b);
1511 mme_value y = mme_load(&b);
1512
1513 mme_dwrite(&b, mme_imm(5), x);
1514 mme_dwrite(&b, mme_imm(8), y);
1515
1516 mme_value y2 = mme_dread(&b, mme_imm(8));
1517 mme_value x2 = mme_dread(&b, mme_imm(5));
1518
1519 mme_store_imm_addr(&b, data_addr + 0, y2);
1520 mme_store_imm_addr(&b, data_addr + 4, x2);
1521
1522 auto macro = mme_builder_finish_vec(&b);
1523
1524 std::vector<uint32_t> params;
1525 params.push_back(-10);
1526 params.push_back(5);
1527
1528 test_macro(&b, macro, params);
1529 }
1530
1531 TEST_F(mme_tu104_sim_test, dwrite_dma)
1532 {
1533 const uint32_t canary5 = 0xc0ffee01;
1534 const uint32_t canary8 = canary5 & 0x00ffff00;
1535
1536 mme_builder b;
1537 mme_builder_init(&b, devinfo);
1538
1539 mme_value x = mme_load(&b);
1540 mme_value y = mme_load(&b);
1541
1542 mme_dwrite(&b, mme_imm(5), x);
1543 mme_dwrite(&b, mme_imm(8), y);
1544
1545 auto macro = mme_builder_finish_vec(&b);
1546
1547 push_macro(0, macro);
1548
1549 P_1INC(p, NVC597, CALL_MME_MACRO(0));
1550 P_INLINE_DATA(p, canary5);
1551 P_INLINE_DATA(p, canary8);
1552
1553 P_MTHD(p, NVC597, SET_MME_MEM_ADDRESS_A);
1554 P_NVC597_SET_MME_MEM_ADDRESS_A(p, high32(data_addr));
1555 P_NVC597_SET_MME_MEM_ADDRESS_B(p, low32(data_addr));
1556 /* Start 3 dwords into MME RAM */
1557 P_NVC597_SET_MME_DATA_RAM_ADDRESS(p, 3);
1558 P_IMMD(p, NVC597, MME_DMA_WRITE, 20);
1559
1560 submit_push();
1561
1562 for (uint32_t i = 0; i < 20; i++) {
1563 if (i == 5 - 3) {
1564 ASSERT_EQ(data[i], canary5);
1565 } else if (i == 8 - 3) {
1566 ASSERT_EQ(data[i], canary8);
1567 } else {
1568 ASSERT_EQ(data[i], 0);
1569 }
1570 }
1571 }
1572
1573 TEST_F(mme_tu104_sim_test, dram_limit)
1574 {
1575 static const uint32_t chunk_size = 32;
1576
1577 mme_builder b;
1578 mme_builder_init(&b, devinfo);
1579
1580 mme_value start = mme_load(&b);
1581 mme_value count = mme_load(&b);
1582
1583 mme_value i = mme_mov(&b, start);
1584 mme_loop(&b, count) {
1585 mme_dwrite(&b, i, i);
1586 mme_add_to(&b, i, i, mme_imm(1));
1587 }
1588
1589 mme_value j = mme_mov(&b, start);
1590 struct mme_value64 addr = mme_mov64(&b, mme_imm64(data_addr));
1591
1592 mme_loop(&b, count) {
1593 mme_value x = mme_dread(&b, j);
1594 mme_store(&b, addr, x);
1595 mme_add_to(&b, j, j, mme_imm(1));
1596 mme_add64_to(&b, addr, addr, mme_imm64(4));
1597 }
1598
1599 auto macro = mme_builder_finish_vec(&b);
1600
1601 for (uint32_t i = 0; i < MME_TU104_DRAM_COUNT; i += chunk_size) {
1602 reset_push();
1603
1604 push_macro(0, macro);
1605
1606 P_1INC(p, NVC597, CALL_MME_MACRO(0));
1607 P_INLINE_DATA(p, i);
1608 P_INLINE_DATA(p, chunk_size);
1609
1610 submit_push();
1611
1612 for (uint32_t j = 0; j < chunk_size; j++)
1613 ASSERT_EQ(data[j], i + j);
1614 }
1615 }
1616
1617 TEST_F(mme_tu104_sim_test, dma_read_fifoed)
1618 {
1619 mme_builder b;
1620 mme_builder_init(&b, devinfo);
1621
1622 mme_mthd(&b, NVC597_SET_MME_DATA_RAM_ADDRESS);
1623 mme_emit(&b, mme_zero());
1624
1625 mme_mthd(&b, NVC597_SET_MME_MEM_ADDRESS_A);
1626 mme_emit(&b, mme_imm(high32(data_addr)));
1627 mme_emit(&b, mme_imm(low32(data_addr)));
1628
1629 mme_mthd(&b, NVC597_MME_DMA_READ_FIFOED);
1630 mme_emit(&b, mme_imm(2));
1631
1632 mme_tu104_load_barrier(&b);
1633
1634 mme_value x = mme_load(&b);
1635 mme_value y = mme_load(&b);
1636
1637 mme_store_imm_addr(&b, data_addr + 256 + 0, x);
1638 mme_store_imm_addr(&b, data_addr + 256 + 4, y);
1639
1640 auto macro = mme_builder_finish_vec(&b);
1641
1642 if (devinfo->cls_eng3d < BLACKWELL_A)
1643 P_IMMD(p, NVC597, SET_MME_DATA_FIFO_CONFIG, FIFO_SIZE_SIZE_4KB);
1644
1645 for (uint32_t i = 0; i < 64; i++)
1646 data[i] = 1000 + i;
1647
1648 std::vector<uint32_t> params;
1649 params.push_back(7);
1650
1651 test_macro(&b, macro, params);
1652 }
1653
1654 TEST_F(mme_tu104_sim_test, scratch_limit)
1655 {
1656 static const uint32_t chunk_size = 32;
1657
1658 mme_builder b;
1659 mme_builder_init(&b, devinfo);
1660
1661 mme_value start = mme_load(&b);
1662 mme_value count = mme_load(&b);
1663
1664 mme_value i = mme_mov(&b, start);
1665 mme_loop(&b, count) {
1666 mme_mthd_arr(&b, NVC597_SET_MME_SHADOW_SCRATCH(0), i);
1667 mme_emit(&b, i);
1668 mme_add_to(&b, i, i, mme_imm(1));
1669 }
1670
1671 mme_value j = mme_mov(&b, start);
1672 struct mme_value64 addr = mme_mov64(&b, mme_imm64(data_addr));
1673
1674 mme_loop(&b, count) {
1675 mme_value x = mme_state_arr(&b, NVC597_SET_MME_SHADOW_SCRATCH(0), j);
1676 mme_store(&b, addr, x);
1677 mme_add_to(&b, j, j, mme_imm(1));
1678 mme_add64_to(&b, addr, addr, mme_imm64(4));
1679 }
1680
1681 auto macro = mme_builder_finish_vec(&b);
1682
1683 for (uint32_t i = 0; i < MME_TU104_SCRATCH_COUNT; i += chunk_size) {
1684 reset_push();
1685
1686 push_macro(0, macro);
1687
1688 P_1INC(p, NVC597, CALL_MME_MACRO(0));
1689 P_INLINE_DATA(p, i);
1690 P_INLINE_DATA(p, chunk_size);
1691
1692 submit_push();
1693
1694 for (uint32_t j = 0; j < chunk_size; j++)
1695 ASSERT_EQ(data[j], i + j);
1696 }
1697 }
1698
1699 TEST_F(mme_tu104_sim_test, sanity_compute)
1700 {
1701 const uint32_t canary = 0xc0ffee01;
1702
1703 mme_builder b;
1704 mme_builder_init(&b, devinfo);
1705
1706 mme_store_compute_imm_addr(&b, data_addr, mme_imm(canary));
1707 auto macro = mme_builder_finish_vec(&b);
1708
1709 reset_push();
1710 push_macro(0, macro);
1711
1712 P_1INC(p, NVC7C0, CALL_MME_MACRO(0));
1713 P_NVC7C0_CALL_MME_MACRO(p, 0, 0);
1714 submit_push();
1715
1716 ASSERT_EQ(data[0], canary);
1717 }
1718
1719 TEST_F(mme_tu104_sim_test, scratch_limit_compute)
1720 {
1721 static const uint32_t chunk_size = 4;
1722
1723 mme_builder b;
1724 mme_builder_init(&b, devinfo);
1725
1726 mme_value start = mme_load(&b);
1727 mme_value count = mme_load(&b);
1728
1729 mme_value i = mme_mov(&b, start);
1730 mme_loop(&b, count) {
1731 mme_mthd_arr(&b, NVC7C0_SET_MME_SHADOW_SCRATCH(0), i);
1732 mme_emit(&b, i);
1733 mme_add_to(&b, i, i, mme_imm(1));
1734 }
1735
1736 mme_value j = mme_mov(&b, start);
1737 struct mme_value64 addr = mme_mov64(&b, mme_imm64(data_addr));
1738
1739 mme_loop(&b, count) {
1740 mme_value x = mme_state_arr(&b, NVC7C0_SET_MME_SHADOW_SCRATCH(0), j);
1741 mme_store_compute(&b, addr, x);
1742 mme_add_to(&b, j, j, mme_imm(1));
1743 mme_add64_to(&b, addr, addr, mme_imm64(4));
1744 }
1745
1746 auto macro = mme_builder_finish_vec(&b);
1747
1748 for (uint32_t i = 0; i < 8; i += chunk_size) {
1749 reset_push();
1750
1751 push_macro(0, macro);
1752
1753 P_1INC(p, NVC7C0, CALL_MME_MACRO(1));
1754 P_INLINE_DATA(p, i);
1755 P_INLINE_DATA(p, chunk_size);
1756
1757 submit_push();
1758
1759 for (uint32_t j = 0; j < chunk_size; j++)
1760 ASSERT_EQ(data[j], i + j);
1761 }
1762 }
1763
1764 TEST_F(mme_tu104_sim_test, scratch_share_3d_to_compute)
1765 {
1766 static const uint32_t chunk_size = 4;
1767
1768 mme_builder b;
1769 mme_builder_init(&b, devinfo);
1770
1771 mme_value start = mme_load(&b);
1772 mme_value count = mme_load(&b);
1773 mme_value channel = mme_load(&b);
1774
1775 mme_if(&b, ieq, channel, mme_zero()) {
1776 mme_value i = mme_mov(&b, start);
1777 mme_loop(&b, count) {
1778 mme_mthd_arr(&b, NVC597_SET_MME_SHADOW_SCRATCH(0), i);
1779 mme_emit(&b, i);
1780 mme_add_to(&b, i, i, mme_imm(1));
1781 }
1782 }
1783
1784 mme_if(&b, ieq, channel, mme_imm(1)) {
1785 mme_value i = mme_mov(&b, start);
1786 struct mme_value64 addr = mme_mov64(&b, mme_imm64(data_addr));
1787
1788 mme_loop(&b, count) {
1789 mme_value val = mme_state_arr(&b, NVC7C0_SET_MME_SHADOW_SCRATCH(0), i);
1790 mme_store_compute(&b, addr, val);
1791 mme_add_to(&b, i, i, mme_imm(1));
1792 mme_add64_to(&b, addr, addr, mme_imm64(4));
1793 }
1794 }
1795
1796 auto macro = mme_builder_finish_vec(&b);
1797
1798 for (uint32_t i = 0; i < 8; i += chunk_size) {
1799 reset_push();
1800
1801 push_macro(0, macro);
1802
1803 P_1INC(p, NVC597, CALL_MME_MACRO(0));
1804 P_INLINE_DATA(p, i);
1805 P_INLINE_DATA(p, chunk_size);
1806 P_INLINE_DATA(p, 0);
1807
1808 P_1INC(p, NVC7C0, CALL_MME_MACRO(0));
1809 P_INLINE_DATA(p, i);
1810 P_INLINE_DATA(p, chunk_size);
1811 P_INLINE_DATA(p, 1);
1812
1813 submit_push();
1814
1815 for (uint32_t j = 0; j < chunk_size; j++)
1816 ASSERT_EQ(data[j], i + j);
1817 }
1818 }