| 1 |
//===----------------------------------------------------------------------===// |
| 2 |
// |
| 3 |
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 |
// See https://llvm.org/LICENSE.txt for license information. |
| 5 |
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 |
// |
| 7 |
//===----------------------------------------------------------------------===// |
| 8 |
/// |
| 9 |
/// \file |
| 10 |
/// This file contains the definition of the runTests function, which executes a |
| 11 |
/// a suite of tests and print a formatted report for each. |
| 12 |
/// |
| 13 |
//===----------------------------------------------------------------------===// |
| 14 |
|
| 15 |
#ifndef MATHTEST_TESTRUNNER_HPP |
| 16 |
#define MATHTEST_TESTRUNNER_HPP |
| 17 |
|
| 18 |
#include "mathtest/DeviceContext.hpp" |
| 19 |
#include "mathtest/GpuMathTest.hpp" |
| 20 |
#include "mathtest/Numerics.hpp" |
| 21 |
#include "mathtest/TestConfig.hpp" |
| 22 |
|
| 23 |
#include "llvm/ADT/STLExtras.h" |
| 24 |
#include "llvm/ADT/SmallVector.h" |
| 25 |
#include "llvm/ADT/StringRef.h" |
| 26 |
#include "llvm/ADT/Twine.h" |
| 27 |
#include "llvm/Support/Error.h" |
| 28 |
#include "llvm/Support/FormatVariadic.h" |
| 29 |
#include "llvm/Support/raw_ostream.h" |
| 30 |
|
| 31 |
#include <chrono> |
| 32 |
#include <cstddef> |
| 33 |
#include <memory> |
| 34 |
#include <tuple> |
| 35 |
|
| 36 |
namespace mathtest { |
| 37 |
namespace detail { |
| 38 |
|
| 39 |
template <auto Func> |
| 40 |
void printPreamble(const TestConfig &Config, size_t Index, |
| 41 |
size_t Total) noexcept { |
| 42 |
using FunctionConfig = FunctionConfig<Func>; |
| 43 |
|
| 44 |
llvm::errs() << "[" << (Index + 1) << "/" << Total << "] " |
| 45 |
<< "Running conformance test '" << FunctionConfig::Name |
| 46 |
<< "' with '" << Config.Provider << "' on '" << Config.Platform |
| 47 |
<< "'\n"; |
| 48 |
llvm::errs().flush(); |
| 49 |
} |
| 50 |
|
| 51 |
template <typename T> |
| 52 |
void printValue(llvm::raw_ostream &OS, const T &Value) noexcept { |
| 53 |
if constexpr (IsFloatingPoint_v<T>) { |
| 54 |
if constexpr (sizeof(T) < sizeof(float)) |
| 55 |
OS << float(Value); |
| 56 |
else |
| 57 |
OS << Value; |
| 58 |
|
| 59 |
const FPBits<T> Bits(Value); |
| 60 |
OS << llvm::formatv(" (0x{0})", llvm::Twine::utohexstr(Bits.uintval())); |
| 61 |
} else { |
| 62 |
OS << Value; |
| 63 |
} |
| 64 |
} |
| 65 |
|
| 66 |
template <typename... Ts> |
| 67 |
void printValues(llvm::raw_ostream &OS, |
| 68 |
const std::tuple<Ts...> &ValuesTuple) noexcept { |
| 69 |
std::apply( |
| 70 |
[&OS](const auto &...Values) { |
| 71 |
bool IsFirst = true; |
| 72 |
auto PrintWithComma = [&](const auto &Value) { |
| 73 |
if (!IsFirst) |
| 74 |
OS << ", "; |
| 75 |
printValue(OS, Value); |
| 76 |
IsFirst = false; |
| 77 |
}; |
| 78 |
(PrintWithComma(Values), ...); |
| 79 |
}, |
| 80 |
ValuesTuple); |
| 81 |
} |
| 82 |
|
| 83 |
template <typename TestCaseType> |
| 84 |
void printWorstFailingCase(llvm::raw_ostream &OS, |
| 85 |
const TestCaseType &TestCase) noexcept { |
| 86 |
OS << "--- Worst Failing Case ---\n"; |
| 87 |
OS << llvm::formatv(" {0,-14} : ", "Input(s)"); |
| 88 |
printValues(OS, TestCase.Inputs); |
| 89 |
OS << "\n"; |
| 90 |
|
| 91 |
OS << llvm::formatv(" {0,-14} : ", "Actual"); |
| 92 |
printValue(OS, TestCase.Actual); |
| 93 |
OS << "\n"; |
| 94 |
|
| 95 |
OS << llvm::formatv(" {0,-14} : ", "Expected"); |
| 96 |
printValue(OS, TestCase.Expected); |
| 97 |
OS << "\n"; |
| 98 |
} |
| 99 |
|
| 100 |
template <typename TestType, typename ResultType> |
| 101 |
void printReport(const TestType &Test, const ResultType &Result, |
| 102 |
const std::chrono::steady_clock::duration &Duration) noexcept { |
| 103 |
using FunctionConfig = typename TestType::FunctionConfig; |
| 104 |
|
| 105 |
const auto Context = Test.getContext(); |
| 106 |
const auto ElapsedMilliseconds = |
| 107 |
std::chrono::duration_cast<std::chrono::milliseconds>(Duration).count(); |
| 108 |
const bool Passed = Result.hasPassed(); |
| 109 |
|
| 110 |
llvm::errs() << llvm::formatv("=== Test Report for '{0}' === \n", |
| 111 |
FunctionConfig::Name); |
| 112 |
llvm::errs() << llvm::formatv("{0,-17}: {1}\n", "Provider", |
| 113 |
Test.getProvider()); |
| 114 |
llvm::errs() << llvm::formatv("{0,-17}: {1}\n", "Platform", |
| 115 |
Context->getPlatform()); |
| 116 |
llvm::errs() << llvm::formatv("{0,-17}: {1}\n", "Device", Context->getName()); |
| 117 |
llvm::errs() << llvm::formatv("{0,-17}: {1} ms\n", "Elapsed time", |
| 118 |
ElapsedMilliseconds); |
| 119 |
llvm::errs() << llvm::formatv("{0,-17}: {1}\n", "ULP tolerance", |
| 120 |
FunctionConfig::UlpTolerance); |
| 121 |
llvm::errs() << llvm::formatv("{0,-17}: {1}\n", "Max ULP distance", |
| 122 |
Result.getMaxUlpDistance()); |
| 123 |
llvm::errs() << llvm::formatv("{0,-17}: {1}\n", "Test cases", |
| 124 |
Result.getTestCaseCount()); |
| 125 |
llvm::errs() << llvm::formatv("{0,-17}: {1}\n", "Failures", |
| 126 |
Result.getFailureCount()); |
| 127 |
llvm::errs() << llvm::formatv("{0,-17}: {1}\n", "Status", |
| 128 |
Passed ? "PASSED" : "FAILED"); |
| 129 |
|
| 130 |
if (const auto &Worst = Result.getWorstFailingCase()) |
| 131 |
printWorstFailingCase(llvm::errs(), Worst.value()); |
| 132 |
|
| 133 |
llvm::errs().flush(); |
| 134 |
} |
| 135 |
|
| 136 |
template <auto Func, typename TestType = GpuMathTest<Func>> |
| 137 |
[[nodiscard]] llvm::Expected<bool> |
| 138 |
runTest(typename TestType::GeneratorType &Generator, const TestConfig &Config, |
| 139 |
llvm::StringRef DeviceBinaryDir) { |
| 140 |
const auto &Platforms = getPlatforms(); |
| 141 |
|
| 142 |
if (!llvm::any_of(Platforms, [&](llvm::StringRef Platform) { |
| 143 |
return Platform.equals_insensitive(Config.Platform); |
| 144 |
})) |
| 145 |
return llvm::createStringError("Platform '" + Config.Platform + |
| 146 |
"' is not available on this system"); |
| 147 |
|
| 148 |
auto Context = |
| 149 |
std::make_shared<DeviceContext>(Config.Platform, /*DeviceId=*/0); |
| 150 |
auto ExpectedTest = |
| 151 |
TestType::create(Context, Config.Provider, DeviceBinaryDir); |
| 152 |
|
| 153 |
if (!ExpectedTest) |
| 154 |
return ExpectedTest.takeError(); |
| 155 |
|
| 156 |
const auto StartTime = std::chrono::steady_clock::now(); |
| 157 |
|
| 158 |
auto Result = ExpectedTest->run(Generator); |
| 159 |
|
| 160 |
const auto EndTime = std::chrono::steady_clock::now(); |
| 161 |
const auto Duration = EndTime - StartTime; |
| 162 |
|
| 163 |
printReport(*ExpectedTest, Result, Duration); |
| 164 |
|
| 165 |
return Result.hasPassed(); |
| 166 |
} |
| 167 |
} // namespace detail |
| 168 |
|
| 169 |
template <auto Func, typename TestType = GpuMathTest<Func>> |
| 170 |
[[nodiscard]] bool runTests(typename TestType::GeneratorType &Generator, |
| 171 |
const llvm::SmallVector<TestConfig, 4> &Configs, |
| 172 |
llvm::StringRef DeviceBinaryDir, |
| 173 |
bool IsVerbose = false) { |
| 174 |
const size_t NumConfigs = Configs.size(); |
| 175 |
|
| 176 |
if (NumConfigs == 0) |
| 177 |
llvm::errs() << "There is no test configuration to run a test\n"; |
| 178 |
|
| 179 |
bool Passed = true; |
| 180 |
|
| 181 |
for (const auto &[Index, Config] : llvm::enumerate(Configs)) { |
| 182 |
detail::printPreamble<Func>(Config, Index, NumConfigs); |
| 183 |
|
| 184 |
Generator.reset(); |
| 185 |
|
| 186 |
auto ExpectedPassed = |
| 187 |
detail::runTest<Func, TestType>(Generator, Config, DeviceBinaryDir); |
| 188 |
|
| 189 |
if (!ExpectedPassed) { |
| 190 |
const auto Details = llvm::toString(ExpectedPassed.takeError()); |
| 191 |
llvm::errs() |
| 192 |
<< "WARNING: Conformance test not supported on this system\n"; |
| 193 |
|
| 194 |
if (IsVerbose) |
| 195 |
llvm::errs() << "Details: " << Details << "\n"; |
| 196 |
} else { |
| 197 |
Passed &= *ExpectedPassed; |
| 198 |
} |
| 199 |
|
| 200 |
llvm::errs() << "\n"; |
| 201 |
} |
| 202 |
|
| 203 |
return Passed; |
| 204 |
} |
| 205 |
} // namespace mathtest |
| 206 |
|
| 207 |
#endif // MATHTEST_TESTRUNNER_HPP |