| 1 |
// Copyright Advanced Micro Devices, Inc., or its affiliates. |
| 2 |
// SPDX-License-Identifier: MIT |
| 3 |
|
| 4 |
#pragma once |
| 5 |
#include "origami/gemm.hpp" |
| 6 |
#include "origami/hardware.hpp" |
| 7 |
#include "origami/utils.hpp" |
| 8 |
#include <gtest/gtest.h> |
| 9 |
#include <map> |
| 10 |
#include <string> |
| 11 |
#include <vector> |
| 12 |
|
| 13 |
struct InputWithExpected |
| 14 |
{ |
| 15 |
std::map<std::string, int> values; |
| 16 |
std::optional<int> expected; |
| 17 |
std::optional<int> expected_gt; |
| 18 |
std::optional<int> expected_lt; |
| 19 |
}; |
| 20 |
|
| 21 |
struct MyTestData |
| 22 |
{ |
| 23 |
std::string name; |
| 24 |
std::vector<InputWithExpected> inputs; |
| 25 |
}; |
| 26 |
|
| 27 |
// Parameterized test class declaration |
| 28 |
class AnalyticalGtest : public ::testing::TestWithParam<MyTestData> |
| 29 |
{ |
| 30 |
}; |
| 31 |
|
| 32 |
void ComputeLoads( |
| 33 |
int MT_M, int MT_N, int MT_K, const std::optional<int> expected) |
| 34 |
{ |
| 35 |
auto a_loads = origami::compute_A_loads(MT_M, MT_K); |
| 36 |
auto b_loads = origami::compute_B_loads(MT_N, MT_K); |
| 37 |
EXPECT_EQ(a_loads, expected); |
| 38 |
EXPECT_EQ(b_loads, expected); |
| 39 |
} |
| 40 |
|
| 41 |
void EstimateL2Hit(const origami::hardware_t& hardware, |
| 42 |
int M, |
| 43 |
int N, |
| 44 |
int K, |
| 45 |
int batch, |
| 46 |
int MT_M, |
| 47 |
int MT_N, |
| 48 |
int MT_K, |
| 49 |
size_t element_size, |
| 50 |
int splittingFactor, |
| 51 |
const std::optional<int> expected_gt, |
| 52 |
const std::optional<int> expected_lt) |
| 53 |
{ |
| 54 |
double l2_hit; |
| 55 |
for(int i = 1; i < 1025; i++) |
| 56 |
{ |
| 57 |
l2_hit = origami::estimate_l2_hit( |
| 58 |
hardware, M, N, K, batch, MT_M, MT_N, MT_K, element_size, i, splittingFactor); |
| 59 |
EXPECT_GT(l2_hit, expected_gt); |
| 60 |
EXPECT_LT(l2_hit, expected_lt); |
| 61 |
} |
| 62 |
} |
| 63 |
|
| 64 |
void ComputeNumMatrixInstructions(const origami::hardware_t& hardware, |
| 65 |
int MT_M, |
| 66 |
int MT_N, |
| 67 |
int MT_K, |
| 68 |
int MI_M, |
| 69 |
int MI_N, |
| 70 |
int MI_K, |
| 71 |
const std::optional<int> expected) |
| 72 |
{ |
| 73 |
auto NumberMatrixInstructions = origami::compute_number_matrix_instructions( |
| 74 |
hardware, MT_M, MT_N, MT_K, MI_M, MI_N, MI_K); |
| 75 |
EXPECT_EQ(NumberMatrixInstructions, expected); |
| 76 |
} |
| 77 |
|
| 78 |
void ComputeMTComputeLatency(const origami::hardware_t& hardware, |
| 79 |
size_t M, |
| 80 |
size_t N, |
| 81 |
size_t K, |
| 82 |
bool transA, |
| 83 |
bool transB, |
| 84 |
size_t MT_M, |
| 85 |
size_t MT_N, |
| 86 |
size_t MT_K, |
| 87 |
size_t MI_M, |
| 88 |
size_t MI_N, |
| 89 |
size_t MI_K, |
| 90 |
size_t element_size_A, |
| 91 |
size_t element_size_B, |
| 92 |
const std::optional<int> expected, |
| 93 |
const std::optional<int> expected_gt) |
| 94 |
{ |
| 95 |
auto latency = origami::compute_mt_compute_latency( |
| 96 |
hardware, M, N, K, transA, transB, MT_M, MT_N, MT_K, MI_M, MI_N, MI_K, element_size_A, element_size_B, origami::data_type_t::BFloat16); |
| 97 |
|
| 98 |
if (expected.has_value()) |
| 99 |
EXPECT_EQ(latency, expected); |
| 100 |
else if(expected_gt.has_value()) |
| 101 |
EXPECT_GT(latency, expected_gt); |
| 102 |
} |
| 103 |
|
| 104 |
void ComputeMemoryLatency(const origami::hardware_t& hardware, |
| 105 |
size_t M, |
| 106 |
size_t N, |
| 107 |
size_t K, |
| 108 |
size_t batch, |
| 109 |
bool transA, |
| 110 |
bool transB, |
| 111 |
size_t MT_M, |
| 112 |
size_t MT_N, |
| 113 |
size_t MT_K, |
| 114 |
size_t element_size_A, |
| 115 |
size_t element_size_B, |
| 116 |
size_t mx_block_size, |
| 117 |
int wgm, |
| 118 |
int numActiveCUs, |
| 119 |
int splittingFactor) |
| 120 |
{ |
| 121 |
auto mem_latency_small = origami::compute_memory_latency( |
| 122 |
hardware, M, N, K, transA, transB, batch, MT_M, MT_N, MT_K, element_size_A, element_size_B, mx_block_size, wgm, numActiveCUs, splittingFactor); |
| 123 |
|
| 124 |
auto mem_latency_large = origami::compute_memory_latency( |
| 125 |
hardware, M, N, K, transA, transB, batch, MT_M * 2, MT_N * 2, MT_K * 2, element_size_A, element_size_B, mx_block_size, wgm, numActiveCUs, splittingFactor); |
| 126 |
|
| 127 |
EXPECT_LT(mem_latency_small, mem_latency_large); |
| 128 |
} |
| 129 |
|
| 130 |
void ComputeTileLatency(const origami::hardware_t& hardware, |
| 131 |
size_t M, |
| 132 |
size_t N, |
| 133 |
size_t K, |
| 134 |
size_t batch, |
| 135 |
bool transA, |
| 136 |
bool transB, |
| 137 |
size_t MT_M, |
| 138 |
size_t MT_N, |
| 139 |
size_t MT_K, |
| 140 |
size_t MI_M, |
| 141 |
size_t MI_N, |
| 142 |
size_t MI_K, |
| 143 |
size_t element_size_A, //In bits |
| 144 |
size_t element_size_B, //In bits, |
| 145 |
size_t element_size_out, //In bits |
| 146 |
size_t mx_block_size, |
| 147 |
int WGM, |
| 148 |
size_t numActiveCUs, |
| 149 |
size_t splittingFactor) |
| 150 |
{ |
| 151 |
auto tile_latency_small = origami::compute_tile_latency(hardware, |
| 152 |
M, |
| 153 |
N, |
| 154 |
K, |
| 155 |
batch, |
| 156 |
transA, |
| 157 |
transB, |
| 158 |
MT_M, |
| 159 |
MT_N, |
| 160 |
MT_K, |
| 161 |
MI_M, |
| 162 |
MI_N, |
| 163 |
MI_K, |
| 164 |
element_size_A, |
| 165 |
element_size_B, |
| 166 |
element_size_out, |
| 167 |
origami::data_type_t::BFloat16, |
| 168 |
mx_block_size, |
| 169 |
WGM, |
| 170 |
numActiveCUs, |
| 171 |
splittingFactor); |
| 172 |
|
| 173 |
auto tile_latency_large = origami::compute_tile_latency(hardware, |
| 174 |
M, |
| 175 |
N, |
| 176 |
K, |
| 177 |
batch, |
| 178 |
transA, |
| 179 |
transB, |
| 180 |
MT_M * 2, |
| 181 |
MT_N * 2, |
| 182 |
MT_K * 2, |
| 183 |
MI_M, |
| 184 |
MI_N, |
| 185 |
MI_K, |
| 186 |
element_size_A, |
| 187 |
element_size_B, |
| 188 |
element_size_out, |
| 189 |
origami::data_type_t::BFloat16, |
| 190 |
mx_block_size, |
| 191 |
WGM, |
| 192 |
numActiveCUs, |
| 193 |
splittingFactor); |
| 194 |
|
| 195 |
EXPECT_GT(tile_latency_large, tile_latency_small); |
| 196 |
} |
| 197 |
|
| 198 |
void ComputeWaveLatency(const origami::hardware_t& hardware, |
| 199 |
size_t M, |
| 200 |
size_t N, |
| 201 |
size_t K, |
| 202 |
size_t batch, |
| 203 |
bool transA, |
| 204 |
bool transB, |
| 205 |
size_t MT_M, |
| 206 |
size_t MT_N, |
| 207 |
size_t MT_K, |
| 208 |
size_t MI_M, |
| 209 |
size_t MI_N, |
| 210 |
size_t MI_K, |
| 211 |
size_t element_size_A, //In bits |
| 212 |
size_t element_size_B, //In bits, |
| 213 |
size_t element_size_out, //In bits |
| 214 |
size_t mx_block_size, |
| 215 |
int WGM, |
| 216 |
size_t numActiveCUs, |
| 217 |
size_t splittingFactor) |
| 218 |
{ |
| 219 |
auto tile_latency = origami::compute_tile_latency(hardware, |
| 220 |
M, |
| 221 |
N, |
| 222 |
K, |
| 223 |
batch, |
| 224 |
transA, |
| 225 |
transB, |
| 226 |
MT_M, |
| 227 |
MT_N, |
| 228 |
MT_K, |
| 229 |
MI_M, |
| 230 |
MI_N, |
| 231 |
MI_K, |
| 232 |
element_size_A, |
| 233 |
element_size_B, |
| 234 |
element_size_out, |
| 235 |
origami::data_type_t::BFloat16, |
| 236 |
mx_block_size, |
| 237 |
WGM, |
| 238 |
numActiveCUs, |
| 239 |
splittingFactor); |
| 240 |
auto wave_latency = origami::compute_wave_latency(hardware, |
| 241 |
M, |
| 242 |
N, |
| 243 |
K, |
| 244 |
batch, |
| 245 |
transA, |
| 246 |
transB, |
| 247 |
MT_M, |
| 248 |
MT_N, |
| 249 |
MT_K, |
| 250 |
MI_M, |
| 251 |
MI_N, |
| 252 |
MI_K, |
| 253 |
element_size_A, |
| 254 |
element_size_B, |
| 255 |
element_size_out, |
| 256 |
origami::data_type_t::BFloat16, |
| 257 |
mx_block_size, |
| 258 |
WGM, |
| 259 |
numActiveCUs, |
| 260 |
splittingFactor); |
| 261 |
EXPECT_DOUBLE_EQ(wave_latency, tile_latency); |
| 262 |
} |
| 263 |
|
| 264 |
void ComputeTotalLatency(const origami::hardware_t& hardware, |
| 265 |
size_t M, |
| 266 |
size_t N, |
| 267 |
size_t K, |
| 268 |
size_t batch, |
| 269 |
bool transA, |
| 270 |
bool transB, |
| 271 |
size_t MT_M, |
| 272 |
size_t MT_N, |
| 273 |
size_t MT_K, |
| 274 |
size_t MI_M, |
| 275 |
size_t MI_N, |
| 276 |
size_t MI_K, |
| 277 |
size_t element_size_A, //In bits |
| 278 |
size_t element_size_B, //In bits, |
| 279 |
size_t element_size_out, //In bits |
| 280 |
size_t mx_block_size, |
| 281 |
int WGM, |
| 282 |
size_t splittingFactor) |
| 283 |
{ |
| 284 |
double latency_cycles_small = origami::compute_total_latency(hardware, |
| 285 |
M, |
| 286 |
N, |
| 287 |
K, |
| 288 |
batch, |
| 289 |
transA, |
| 290 |
transB, |
| 291 |
MT_M, |
| 292 |
MT_N, |
| 293 |
MT_K, |
| 294 |
MI_M, |
| 295 |
MI_N, |
| 296 |
MI_K, |
| 297 |
element_size_A, |
| 298 |
element_size_B, |
| 299 |
element_size_out, |
| 300 |
origami::data_type_t::BFloat16, |
| 301 |
mx_block_size, |
| 302 |
WGM, |
| 303 |
splittingFactor); |
| 304 |
|
| 305 |
double latency_cycles_large = origami::compute_total_latency(hardware, |
| 306 |
M, |
| 307 |
N, |
| 308 |
K, |
| 309 |
batch, |
| 310 |
transA, |
| 311 |
transB, |
| 312 |
MT_M * 2, |
| 313 |
MT_N * 2, |
| 314 |
MT_K * 2, |
| 315 |
MI_M, |
| 316 |
MI_N, |
| 317 |
MI_K, |
| 318 |
element_size_A, |
| 319 |
element_size_B, |
| 320 |
element_size_out, |
| 321 |
origami::data_type_t::BFloat16, |
| 322 |
mx_block_size, |
| 323 |
WGM, |
| 324 |
splittingFactor); |
| 325 |
EXPECT_LT(latency_cycles_small, latency_cycles_large); |
| 326 |
} |
| 327 |
|
| 328 |
void ComputePerfGflops(size_t M, |
| 329 |
size_t N, |
| 330 |
size_t K, |
| 331 |
size_t batch, |
| 332 |
bool transA, |
| 333 |
bool transB, |
| 334 |
size_t MT_M, |
| 335 |
size_t MT_N, |
| 336 |
size_t MT_K, |
| 337 |
size_t MI_M, |
| 338 |
size_t MI_N, |
| 339 |
size_t MI_K, |
| 340 |
size_t element_size_A, //In bits |
| 341 |
size_t element_size_B, //In bits, |
| 342 |
size_t element_size_out, //In bits |
| 343 |
int WGM) |
| 344 |
{ |
| 345 |
auto gfx942arch = origami::hardware_t::arch_name_to_enum("gfx942"); |
| 346 |
auto gfx942_slow = origami::hardware_t( |
| 347 |
gfx942arch, 304, 65536, 8, 1.0, 1.0, 1.0, 4000000, 1.4, 1, std::make_tuple(0, 0.015, 0)); |
| 348 |
auto gfx942_fast = origami::hardware_t( |
| 349 |
gfx942arch, 304, 65536, 8, 1.0, 1.0, 1.0, 4000000, 1.8, 1, std::make_tuple(0, 0.015, 0)); |
| 350 |
double flops_slow = origami::compute_perf_gflops(gfx942_slow, |
| 351 |
M, |
| 352 |
N, |
| 353 |
K, |
| 354 |
batch, |
| 355 |
transA, |
| 356 |
transB, |
| 357 |
MT_M, |
| 358 |
MT_N, |
| 359 |
MT_K, |
| 360 |
MI_M, |
| 361 |
MI_N, |
| 362 |
MI_K, |
| 363 |
element_size_A, |
| 364 |
element_size_B, |
| 365 |
element_size_out, |
| 366 |
origami::data_type_t::BFloat16, |
| 367 |
WGM); |
| 368 |
double flops_fast = origami::compute_perf_gflops(gfx942_fast, |
| 369 |
M, |
| 370 |
N, |
| 371 |
K, |
| 372 |
batch, |
| 373 |
transA, |
| 374 |
transB, |
| 375 |
MT_M, |
| 376 |
MT_N, |
| 377 |
MT_K, |
| 378 |
MI_M, |
| 379 |
MI_N, |
| 380 |
MI_K, |
| 381 |
element_size_A, |
| 382 |
element_size_B, |
| 383 |
element_size_out, |
| 384 |
origami::data_type_t::BFloat16, |
| 385 |
WGM); |
| 386 |
EXPECT_GT(flops_fast, flops_slow); // faster clock = higher flops |
| 387 |
} |
| 388 |
|
| 389 |
void EstimateMallHit(const origami::hardware_t& hardware, |
| 390 |
int M, |
| 391 |
int N, |
| 392 |
int K, |
| 393 |
int batch, |
| 394 |
int MT_M, |
| 395 |
int MT_N, |
| 396 |
int MT_K, |
| 397 |
size_t numActiveCUs, |
| 398 |
size_t splittingFactor, |
| 399 |
const std::optional<int> expected_gt) |
| 400 |
{ |
| 401 |
double mall_hit; |
| 402 |
for(int i = 1; i < 1025; i++) |
| 403 |
{ |
| 404 |
mall_hit = origami::estimate_mall_hit( |
| 405 |
hardware, M, N, K, batch, MT_M, MT_N, MT_K, i, numActiveCUs, splittingFactor); |
| 406 |
EXPECT_GT(mall_hit, expected_gt); |
| 407 |
} |
| 408 |
} |
| 409 |
|
| 410 |
void CheckLDSCapacity(const origami::hardware_t& hardware, |
| 411 |
int MT_M, |
| 412 |
int MT_N, |
| 413 |
int MT_K, |
| 414 |
size_t element_size) |
| 415 |
{ |
| 416 |
auto fit_lds_memory |
| 417 |
= origami::check_lds_capacity(hardware, MT_M, MT_N, MT_K, element_size); |
| 418 |
EXPECT_TRUE(fit_lds_memory); |
| 419 |
} |
| 420 |
|
| 421 |
// hardware_t |
| 422 |
void HardwareArchEnum(const std::string gpuArchNumber) |
| 423 |
{ |
| 424 |
auto gpuArchEnum = origami::hardware_t::arch_name_to_enum("gfx" + gpuArchNumber); |
| 425 |
EXPECT_EQ(gpuArchEnum, origami::hardware_t::architecture_t::gfx942); |
| 426 |
} |
| 427 |
|
| 428 |
// Utils |
| 429 |
void BestGridSize(const origami::hardware_t& hardware, |
| 430 |
size_t M, |
| 431 |
size_t N, |
| 432 |
size_t K, |
| 433 |
size_t batch, |
| 434 |
bool transA, |
| 435 |
bool transB, |
| 436 |
size_t MT_M, |
| 437 |
size_t MT_N, |
| 438 |
size_t MT_K, |
| 439 |
size_t MI_M, |
| 440 |
size_t MI_N, |
| 441 |
size_t MI_K, |
| 442 |
size_t element_size_A, |
| 443 |
size_t element_size_B, |
| 444 |
size_t element_size_out, |
| 445 |
size_t mx_block_size, |
| 446 |
double H_L2, |
| 447 |
size_t WGM, |
| 448 |
size_t biggest_allowable_split, |
| 449 |
const std::optional<int> expected_gt) |
| 450 |
{ |
| 451 |
size_t grid_size = origami::select_best_grid_size(M, |
| 452 |
N, |
| 453 |
K, |
| 454 |
batch, |
| 455 |
transA, |
| 456 |
transB, |
| 457 |
hardware, |
| 458 |
MT_M, |
| 459 |
MT_N, |
| 460 |
MT_K, |
| 461 |
MI_M, |
| 462 |
MI_N, |
| 463 |
MI_K, |
| 464 |
element_size_A, |
| 465 |
element_size_B, |
| 466 |
element_size_out, |
| 467 |
origami::data_type_t::BFloat16, |
| 468 |
mx_block_size, |
| 469 |
H_L2, |
| 470 |
WGM, |
| 471 |
biggest_allowable_split); |
| 472 |
EXPECT_GT(grid_size, expected_gt); |
| 473 |
} |
| 474 |
|
| 475 |
void BestMacroTileSize(const origami::hardware_t& hardware, |
| 476 |
size_t M, |
| 477 |
size_t N, |
| 478 |
size_t K, |
| 479 |
size_t batch, |
| 480 |
bool transA, |
| 481 |
bool transB, |
| 482 |
size_t element_size_A, //In bits |
| 483 |
size_t element_size_B, //In bits |
| 484 |
size_t element_size_out, //In bits |
| 485 |
size_t mx_block_size, |
| 486 |
double H_L2, |
| 487 |
size_t WGM) |
| 488 |
{ |
| 489 |
const std::vector<std::tuple<size_t, // MT_M |
| 490 |
size_t, // MT_N |
| 491 |
size_t, // MT_K |
| 492 |
size_t, // MI_M |
| 493 |
size_t, // MI_N |
| 494 |
size_t, // MI_K |
| 495 |
size_t, // Occupancy |
| 496 |
int, // wgm |
| 497 |
size_t, // non_temporal_a |
| 498 |
size_t // non_temporal_b |
| 499 |
>> |
| 500 |
MT_list |
| 501 |
= {{256, 256, 32, 32, 32, 8, 1, 6, 0, 0}, {128, 128, 64, 32, 32, 8, 1, 6, 0, 0}, {64, 64, 64, 32, 32, 8, 1, 6, 0, 0}}; |
| 502 |
auto results = select_best_macro_tile_size( |
| 503 |
M, N, K, batch, transA, transB, hardware, MT_list, element_size_A, element_size_B, element_size_out, origami::data_type_t::BFloat16, mx_block_size, H_L2, false, WGM); |
| 504 |
|
| 505 |
EXPECT_EQ(results.size(), MT_list.size()); |
| 506 |
for(int i = 0; i < results.size() - 1; i++) |
| 507 |
EXPECT_LT(std::get<0>(results[i]), std::get<0>(results[i + 1])); |
| 508 |
} |
| 509 |
|
| 510 |
void BestWGM(const origami::hardware_t& hardware, |
| 511 |
size_t M, |
| 512 |
size_t N, |
| 513 |
size_t K, |
| 514 |
size_t batch, |
| 515 |
size_t MT_M, |
| 516 |
size_t MT_N, |
| 517 |
size_t MT_K, |
| 518 |
size_t MI_M, |
| 519 |
size_t MI_N, |
| 520 |
size_t MI_K, |
| 521 |
size_t element_size, |
| 522 |
double H_L2) // not needed for L2 hit rate but retained if your code expects it |
| 523 |
{ |
| 524 |
std::vector<size_t> WGM_list = {1, 2, 4, 6, 8, 12}; |
| 525 |
|
| 526 |
auto best_wgm_large_tile = select_best_wgm( |
| 527 |
M, N, K, batch, hardware, MT_M, MT_N, MT_K, MI_M, MI_N, MI_K, WGM_list, element_size, H_L2, false); |
| 528 |
|
| 529 |
auto best_wgm_small_tile = select_best_wgm( |
| 530 |
M, N, K, batch, hardware, MT_M/2, MT_N/2, MT_K * 2, MI_M, MI_N, MI_K, WGM_list, element_size, H_L2, false); |
| 531 |
|
| 532 |
auto best_wgm_nonsquare = select_best_wgm( |
| 533 |
2048, 5120, K, batch, hardware, MT_M, MT_N, MT_K, MI_M, MI_N, MI_K, WGM_list, element_size, H_L2, false); |
| 534 |
|
| 535 |
EXPECT_GT(best_wgm_small_tile.second, best_wgm_large_tile.second); |
| 536 |
EXPECT_NE(best_wgm_large_tile.second, best_wgm_nonsquare.second); |
| 537 |
} |
| 538 |
|
| 539 |
void UtilsTFlopsFromLatency(size_t M, size_t N, size_t K, double latency_cycles, double clock_GHz) |
| 540 |
{ |
| 541 |
auto tflops = origami::compute_tflops_from_latency( |
| 542 |
latency_cycles, M, N, K, clock_GHz); |
| 543 |
double Expected = 1.99; |
| 544 |
EXPECT_LT(std::abs(tflops - Expected) / std::abs(Expected), 0.01); |
| 545 |
} |