Debian Codegraph @

origami/tests/include/testing_origami.hpp

raw
1 // Copyright Advanced Micro Devices, Inc., or its affiliates.
2 // SPDX-License-Identifier: MIT
3
4 #pragma once
5 #include "origami/gemm.hpp"
6 #include "origami/hardware.hpp"
7 #include "origami/utils.hpp"
8 #include <gtest/gtest.h>
9 #include <map>
10 #include <string>
11 #include <vector>
12
13 struct InputWithExpected
14 {
15 std::map<std::string, int> values;
16 std::optional<int> expected;
17 std::optional<int> expected_gt;
18 std::optional<int> expected_lt;
19 };
20
21 struct MyTestData
22 {
23 std::string name;
24 std::vector<InputWithExpected> inputs;
25 };
26
27 // Parameterized test class declaration
28 class AnalyticalGtest : public ::testing::TestWithParam<MyTestData>
29 {
30 };
31
32 void ComputeLoads(
33 int MT_M, int MT_N, int MT_K, const std::optional<int> expected)
34 {
35 auto a_loads = origami::compute_A_loads(MT_M, MT_K);
36 auto b_loads = origami::compute_B_loads(MT_N, MT_K);
37 EXPECT_EQ(a_loads, expected);
38 EXPECT_EQ(b_loads, expected);
39 }
40
41 void EstimateL2Hit(const origami::hardware_t& hardware,
42 int M,
43 int N,
44 int K,
45 int batch,
46 int MT_M,
47 int MT_N,
48 int MT_K,
49 size_t element_size,
50 int splittingFactor,
51 const std::optional<int> expected_gt,
52 const std::optional<int> expected_lt)
53 {
54 double l2_hit;
55 for(int i = 1; i < 1025; i++)
56 {
57 l2_hit = origami::estimate_l2_hit(
58 hardware, M, N, K, batch, MT_M, MT_N, MT_K, element_size, i, splittingFactor);
59 EXPECT_GT(l2_hit, expected_gt);
60 EXPECT_LT(l2_hit, expected_lt);
61 }
62 }
63
64 void ComputeNumMatrixInstructions(const origami::hardware_t& hardware,
65 int MT_M,
66 int MT_N,
67 int MT_K,
68 int MI_M,
69 int MI_N,
70 int MI_K,
71 const std::optional<int> expected)
72 {
73 auto NumberMatrixInstructions = origami::compute_number_matrix_instructions(
74 hardware, MT_M, MT_N, MT_K, MI_M, MI_N, MI_K);
75 EXPECT_EQ(NumberMatrixInstructions, expected);
76 }
77
78 void ComputeMTComputeLatency(const origami::hardware_t& hardware,
79 size_t M,
80 size_t N,
81 size_t K,
82 bool transA,
83 bool transB,
84 size_t MT_M,
85 size_t MT_N,
86 size_t MT_K,
87 size_t MI_M,
88 size_t MI_N,
89 size_t MI_K,
90 size_t element_size_A,
91 size_t element_size_B,
92 const std::optional<int> expected,
93 const std::optional<int> expected_gt)
94 {
95 auto latency = origami::compute_mt_compute_latency(
96 hardware, M, N, K, transA, transB, MT_M, MT_N, MT_K, MI_M, MI_N, MI_K, element_size_A, element_size_B, origami::data_type_t::BFloat16);
97
98 if (expected.has_value())
99 EXPECT_EQ(latency, expected);
100 else if(expected_gt.has_value())
101 EXPECT_GT(latency, expected_gt);
102 }
103
104 void ComputeMemoryLatency(const origami::hardware_t& hardware,
105 size_t M,
106 size_t N,
107 size_t K,
108 size_t batch,
109 bool transA,
110 bool transB,
111 size_t MT_M,
112 size_t MT_N,
113 size_t MT_K,
114 size_t element_size_A,
115 size_t element_size_B,
116 size_t mx_block_size,
117 int wgm,
118 int numActiveCUs,
119 int splittingFactor)
120 {
121 auto mem_latency_small = origami::compute_memory_latency(
122 hardware, M, N, K, transA, transB, batch, MT_M, MT_N, MT_K, element_size_A, element_size_B, mx_block_size, wgm, numActiveCUs, splittingFactor);
123
124 auto mem_latency_large = origami::compute_memory_latency(
125 hardware, M, N, K, transA, transB, batch, MT_M * 2, MT_N * 2, MT_K * 2, element_size_A, element_size_B, mx_block_size, wgm, numActiveCUs, splittingFactor);
126
127 EXPECT_LT(mem_latency_small, mem_latency_large);
128 }
129
130 void ComputeTileLatency(const origami::hardware_t& hardware,
131 size_t M,
132 size_t N,
133 size_t K,
134 size_t batch,
135 bool transA,
136 bool transB,
137 size_t MT_M,
138 size_t MT_N,
139 size_t MT_K,
140 size_t MI_M,
141 size_t MI_N,
142 size_t MI_K,
143 size_t element_size_A, //In bits
144 size_t element_size_B, //In bits,
145 size_t element_size_out, //In bits
146 size_t mx_block_size,
147 int WGM,
148 size_t numActiveCUs,
149 size_t splittingFactor)
150 {
151 auto tile_latency_small = origami::compute_tile_latency(hardware,
152 M,
153 N,
154 K,
155 batch,
156 transA,
157 transB,
158 MT_M,
159 MT_N,
160 MT_K,
161 MI_M,
162 MI_N,
163 MI_K,
164 element_size_A,
165 element_size_B,
166 element_size_out,
167 origami::data_type_t::BFloat16,
168 mx_block_size,
169 WGM,
170 numActiveCUs,
171 splittingFactor);
172
173 auto tile_latency_large = origami::compute_tile_latency(hardware,
174 M,
175 N,
176 K,
177 batch,
178 transA,
179 transB,
180 MT_M * 2,
181 MT_N * 2,
182 MT_K * 2,
183 MI_M,
184 MI_N,
185 MI_K,
186 element_size_A,
187 element_size_B,
188 element_size_out,
189 origami::data_type_t::BFloat16,
190 mx_block_size,
191 WGM,
192 numActiveCUs,
193 splittingFactor);
194
195 EXPECT_GT(tile_latency_large, tile_latency_small);
196 }
197
198 void ComputeWaveLatency(const origami::hardware_t& hardware,
199 size_t M,
200 size_t N,
201 size_t K,
202 size_t batch,
203 bool transA,
204 bool transB,
205 size_t MT_M,
206 size_t MT_N,
207 size_t MT_K,
208 size_t MI_M,
209 size_t MI_N,
210 size_t MI_K,
211 size_t element_size_A, //In bits
212 size_t element_size_B, //In bits,
213 size_t element_size_out, //In bits
214 size_t mx_block_size,
215 int WGM,
216 size_t numActiveCUs,
217 size_t splittingFactor)
218 {
219 auto tile_latency = origami::compute_tile_latency(hardware,
220 M,
221 N,
222 K,
223 batch,
224 transA,
225 transB,
226 MT_M,
227 MT_N,
228 MT_K,
229 MI_M,
230 MI_N,
231 MI_K,
232 element_size_A,
233 element_size_B,
234 element_size_out,
235 origami::data_type_t::BFloat16,
236 mx_block_size,
237 WGM,
238 numActiveCUs,
239 splittingFactor);
240 auto wave_latency = origami::compute_wave_latency(hardware,
241 M,
242 N,
243 K,
244 batch,
245 transA,
246 transB,
247 MT_M,
248 MT_N,
249 MT_K,
250 MI_M,
251 MI_N,
252 MI_K,
253 element_size_A,
254 element_size_B,
255 element_size_out,
256 origami::data_type_t::BFloat16,
257 mx_block_size,
258 WGM,
259 numActiveCUs,
260 splittingFactor);
261 EXPECT_DOUBLE_EQ(wave_latency, tile_latency);
262 }
263
264 void ComputeTotalLatency(const origami::hardware_t& hardware,
265 size_t M,
266 size_t N,
267 size_t K,
268 size_t batch,
269 bool transA,
270 bool transB,
271 size_t MT_M,
272 size_t MT_N,
273 size_t MT_K,
274 size_t MI_M,
275 size_t MI_N,
276 size_t MI_K,
277 size_t element_size_A, //In bits
278 size_t element_size_B, //In bits,
279 size_t element_size_out, //In bits
280 size_t mx_block_size,
281 int WGM,
282 size_t splittingFactor)
283 {
284 double latency_cycles_small = origami::compute_total_latency(hardware,
285 M,
286 N,
287 K,
288 batch,
289 transA,
290 transB,
291 MT_M,
292 MT_N,
293 MT_K,
294 MI_M,
295 MI_N,
296 MI_K,
297 element_size_A,
298 element_size_B,
299 element_size_out,
300 origami::data_type_t::BFloat16,
301 mx_block_size,
302 WGM,
303 splittingFactor);
304
305 double latency_cycles_large = origami::compute_total_latency(hardware,
306 M,
307 N,
308 K,
309 batch,
310 transA,
311 transB,
312 MT_M * 2,
313 MT_N * 2,
314 MT_K * 2,
315 MI_M,
316 MI_N,
317 MI_K,
318 element_size_A,
319 element_size_B,
320 element_size_out,
321 origami::data_type_t::BFloat16,
322 mx_block_size,
323 WGM,
324 splittingFactor);
325 EXPECT_LT(latency_cycles_small, latency_cycles_large);
326 }
327
328 void ComputePerfGflops(size_t M,
329 size_t N,
330 size_t K,
331 size_t batch,
332 bool transA,
333 bool transB,
334 size_t MT_M,
335 size_t MT_N,
336 size_t MT_K,
337 size_t MI_M,
338 size_t MI_N,
339 size_t MI_K,
340 size_t element_size_A, //In bits
341 size_t element_size_B, //In bits,
342 size_t element_size_out, //In bits
343 int WGM)
344 {
345 auto gfx942arch = origami::hardware_t::arch_name_to_enum("gfx942");
346 auto gfx942_slow = origami::hardware_t(
347 gfx942arch, 304, 65536, 8, 1.0, 1.0, 1.0, 4000000, 1.4, 1, std::make_tuple(0, 0.015, 0));
348 auto gfx942_fast = origami::hardware_t(
349 gfx942arch, 304, 65536, 8, 1.0, 1.0, 1.0, 4000000, 1.8, 1, std::make_tuple(0, 0.015, 0));
350 double flops_slow = origami::compute_perf_gflops(gfx942_slow,
351 M,
352 N,
353 K,
354 batch,
355 transA,
356 transB,
357 MT_M,
358 MT_N,
359 MT_K,
360 MI_M,
361 MI_N,
362 MI_K,
363 element_size_A,
364 element_size_B,
365 element_size_out,
366 origami::data_type_t::BFloat16,
367 WGM);
368 double flops_fast = origami::compute_perf_gflops(gfx942_fast,
369 M,
370 N,
371 K,
372 batch,
373 transA,
374 transB,
375 MT_M,
376 MT_N,
377 MT_K,
378 MI_M,
379 MI_N,
380 MI_K,
381 element_size_A,
382 element_size_B,
383 element_size_out,
384 origami::data_type_t::BFloat16,
385 WGM);
386 EXPECT_GT(flops_fast, flops_slow); // faster clock = higher flops
387 }
388
389 void EstimateMallHit(const origami::hardware_t& hardware,
390 int M,
391 int N,
392 int K,
393 int batch,
394 int MT_M,
395 int MT_N,
396 int MT_K,
397 size_t numActiveCUs,
398 size_t splittingFactor,
399 const std::optional<int> expected_gt)
400 {
401 double mall_hit;
402 for(int i = 1; i < 1025; i++)
403 {
404 mall_hit = origami::estimate_mall_hit(
405 hardware, M, N, K, batch, MT_M, MT_N, MT_K, i, numActiveCUs, splittingFactor);
406 EXPECT_GT(mall_hit, expected_gt);
407 }
408 }
409
410 void CheckLDSCapacity(const origami::hardware_t& hardware,
411 int MT_M,
412 int MT_N,
413 int MT_K,
414 size_t element_size)
415 {
416 auto fit_lds_memory
417 = origami::check_lds_capacity(hardware, MT_M, MT_N, MT_K, element_size);
418 EXPECT_TRUE(fit_lds_memory);
419 }
420
421 // hardware_t
422 void HardwareArchEnum(const std::string gpuArchNumber)
423 {
424 auto gpuArchEnum = origami::hardware_t::arch_name_to_enum("gfx" + gpuArchNumber);
425 EXPECT_EQ(gpuArchEnum, origami::hardware_t::architecture_t::gfx942);
426 }
427
428 // Utils
429 void BestGridSize(const origami::hardware_t& hardware,
430 size_t M,
431 size_t N,
432 size_t K,
433 size_t batch,
434 bool transA,
435 bool transB,
436 size_t MT_M,
437 size_t MT_N,
438 size_t MT_K,
439 size_t MI_M,
440 size_t MI_N,
441 size_t MI_K,
442 size_t element_size_A,
443 size_t element_size_B,
444 size_t element_size_out,
445 size_t mx_block_size,
446 double H_L2,
447 size_t WGM,
448 size_t biggest_allowable_split,
449 const std::optional<int> expected_gt)
450 {
451 size_t grid_size = origami::select_best_grid_size(M,
452 N,
453 K,
454 batch,
455 transA,
456 transB,
457 hardware,
458 MT_M,
459 MT_N,
460 MT_K,
461 MI_M,
462 MI_N,
463 MI_K,
464 element_size_A,
465 element_size_B,
466 element_size_out,
467 origami::data_type_t::BFloat16,
468 mx_block_size,
469 H_L2,
470 WGM,
471 biggest_allowable_split);
472 EXPECT_GT(grid_size, expected_gt);
473 }
474
475 void BestMacroTileSize(const origami::hardware_t& hardware,
476 size_t M,
477 size_t N,
478 size_t K,
479 size_t batch,
480 bool transA,
481 bool transB,
482 size_t element_size_A, //In bits
483 size_t element_size_B, //In bits
484 size_t element_size_out, //In bits
485 size_t mx_block_size,
486 double H_L2,
487 size_t WGM)
488 {
489 const std::vector<std::tuple<size_t, // MT_M
490 size_t, // MT_N
491 size_t, // MT_K
492 size_t, // MI_M
493 size_t, // MI_N
494 size_t, // MI_K
495 size_t, // Occupancy
496 int, // wgm
497 size_t, // non_temporal_a
498 size_t // non_temporal_b
499 >>
500 MT_list
501 = {{256, 256, 32, 32, 32, 8, 1, 6, 0, 0}, {128, 128, 64, 32, 32, 8, 1, 6, 0, 0}, {64, 64, 64, 32, 32, 8, 1, 6, 0, 0}};
502 auto results = select_best_macro_tile_size(
503 M, N, K, batch, transA, transB, hardware, MT_list, element_size_A, element_size_B, element_size_out, origami::data_type_t::BFloat16, mx_block_size, H_L2, false, WGM);
504
505 EXPECT_EQ(results.size(), MT_list.size());
506 for(int i = 0; i < results.size() - 1; i++)
507 EXPECT_LT(std::get<0>(results[i]), std::get<0>(results[i + 1]));
508 }
509
510 void BestWGM(const origami::hardware_t& hardware,
511 size_t M,
512 size_t N,
513 size_t K,
514 size_t batch,
515 size_t MT_M,
516 size_t MT_N,
517 size_t MT_K,
518 size_t MI_M,
519 size_t MI_N,
520 size_t MI_K,
521 size_t element_size,
522 double H_L2) // not needed for L2 hit rate but retained if your code expects it
523 {
524 std::vector<size_t> WGM_list = {1, 2, 4, 6, 8, 12};
525
526 auto best_wgm_large_tile = select_best_wgm(
527 M, N, K, batch, hardware, MT_M, MT_N, MT_K, MI_M, MI_N, MI_K, WGM_list, element_size, H_L2, false);
528
529 auto best_wgm_small_tile = select_best_wgm(
530 M, N, K, batch, hardware, MT_M/2, MT_N/2, MT_K * 2, MI_M, MI_N, MI_K, WGM_list, element_size, H_L2, false);
531
532 auto best_wgm_nonsquare = select_best_wgm(
533 2048, 5120, K, batch, hardware, MT_M, MT_N, MT_K, MI_M, MI_N, MI_K, WGM_list, element_size, H_L2, false);
534
535 EXPECT_GT(best_wgm_small_tile.second, best_wgm_large_tile.second);
536 EXPECT_NE(best_wgm_large_tile.second, best_wgm_nonsquare.second);
537 }
538
539 void UtilsTFlopsFromLatency(size_t M, size_t N, size_t K, double latency_cycles, double clock_GHz)
540 {
541 auto tflops = origami::compute_tflops_from_latency(
542 latency_cycles, M, N, K, clock_GHz);
543 double Expected = 1.99;
544 EXPECT_LT(std::abs(tflops - Expected) / std::abs(Expected), 0.01);
545 }