Debian Codegraph @

offload/unittests/Conformance/include/mathtest/TestRunner.hpp

raw
1 //===----------------------------------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 ///
9 /// \file
10 /// This file contains the definition of the runTests function, which executes a
11 /// a suite of tests and print a formatted report for each.
12 ///
13 //===----------------------------------------------------------------------===//
14
15 #ifndef MATHTEST_TESTRUNNER_HPP
16 #define MATHTEST_TESTRUNNER_HPP
17
18 #include "mathtest/DeviceContext.hpp"
19 #include "mathtest/GpuMathTest.hpp"
20 #include "mathtest/Numerics.hpp"
21 #include "mathtest/TestConfig.hpp"
22
23 #include "llvm/ADT/STLExtras.h"
24 #include "llvm/ADT/SmallVector.h"
25 #include "llvm/ADT/StringRef.h"
26 #include "llvm/ADT/Twine.h"
27 #include "llvm/Support/Error.h"
28 #include "llvm/Support/FormatVariadic.h"
29 #include "llvm/Support/raw_ostream.h"
30
31 #include <chrono>
32 #include <cstddef>
33 #include <memory>
34 #include <tuple>
35
36 namespace mathtest {
37 namespace detail {
38
39 template <auto Func>
40 void printPreamble(const TestConfig &Config, size_t Index,
41 size_t Total) noexcept {
42 using FunctionConfig = FunctionConfig<Func>;
43
44 llvm::errs() << "[" << (Index + 1) << "/" << Total << "] "
45 << "Running conformance test '" << FunctionConfig::Name
46 << "' with '" << Config.Provider << "' on '" << Config.Platform
47 << "'\n";
48 llvm::errs().flush();
49 }
50
51 template <typename T>
52 void printValue(llvm::raw_ostream &OS, const T &Value) noexcept {
53 if constexpr (IsFloatingPoint_v<T>) {
54 if constexpr (sizeof(T) < sizeof(float))
55 OS << float(Value);
56 else
57 OS << Value;
58
59 const FPBits<T> Bits(Value);
60 OS << llvm::formatv(" (0x{0})", llvm::Twine::utohexstr(Bits.uintval()));
61 } else {
62 OS << Value;
63 }
64 }
65
66 template <typename... Ts>
67 void printValues(llvm::raw_ostream &OS,
68 const std::tuple<Ts...> &ValuesTuple) noexcept {
69 std::apply(
70 [&OS](const auto &...Values) {
71 bool IsFirst = true;
72 auto PrintWithComma = [&](const auto &Value) {
73 if (!IsFirst)
74 OS << ", ";
75 printValue(OS, Value);
76 IsFirst = false;
77 };
78 (PrintWithComma(Values), ...);
79 },
80 ValuesTuple);
81 }
82
83 template <typename TestCaseType>
84 void printWorstFailingCase(llvm::raw_ostream &OS,
85 const TestCaseType &TestCase) noexcept {
86 OS << "--- Worst Failing Case ---\n";
87 OS << llvm::formatv(" {0,-14} : ", "Input(s)");
88 printValues(OS, TestCase.Inputs);
89 OS << "\n";
90
91 OS << llvm::formatv(" {0,-14} : ", "Actual");
92 printValue(OS, TestCase.Actual);
93 OS << "\n";
94
95 OS << llvm::formatv(" {0,-14} : ", "Expected");
96 printValue(OS, TestCase.Expected);
97 OS << "\n";
98 }
99
100 template <typename TestType, typename ResultType>
101 void printReport(const TestType &Test, const ResultType &Result,
102 const std::chrono::steady_clock::duration &Duration) noexcept {
103 using FunctionConfig = typename TestType::FunctionConfig;
104
105 const auto Context = Test.getContext();
106 const auto ElapsedMilliseconds =
107 std::chrono::duration_cast<std::chrono::milliseconds>(Duration).count();
108 const bool Passed = Result.hasPassed();
109
110 llvm::errs() << llvm::formatv("=== Test Report for '{0}' === \n",
111 FunctionConfig::Name);
112 llvm::errs() << llvm::formatv("{0,-17}: {1}\n", "Provider",
113 Test.getProvider());
114 llvm::errs() << llvm::formatv("{0,-17}: {1}\n", "Platform",
115 Context->getPlatform());
116 llvm::errs() << llvm::formatv("{0,-17}: {1}\n", "Device", Context->getName());
117 llvm::errs() << llvm::formatv("{0,-17}: {1} ms\n", "Elapsed time",
118 ElapsedMilliseconds);
119 llvm::errs() << llvm::formatv("{0,-17}: {1}\n", "ULP tolerance",
120 FunctionConfig::UlpTolerance);
121 llvm::errs() << llvm::formatv("{0,-17}: {1}\n", "Max ULP distance",
122 Result.getMaxUlpDistance());
123 llvm::errs() << llvm::formatv("{0,-17}: {1}\n", "Test cases",
124 Result.getTestCaseCount());
125 llvm::errs() << llvm::formatv("{0,-17}: {1}\n", "Failures",
126 Result.getFailureCount());
127 llvm::errs() << llvm::formatv("{0,-17}: {1}\n", "Status",
128 Passed ? "PASSED" : "FAILED");
129
130 if (const auto &Worst = Result.getWorstFailingCase())
131 printWorstFailingCase(llvm::errs(), Worst.value());
132
133 llvm::errs().flush();
134 }
135
136 template <auto Func, typename TestType = GpuMathTest<Func>>
137 [[nodiscard]] llvm::Expected<bool>
138 runTest(typename TestType::GeneratorType &Generator, const TestConfig &Config,
139 llvm::StringRef DeviceBinaryDir) {
140 const auto &Platforms = getPlatforms();
141
142 if (!llvm::any_of(Platforms, [&](llvm::StringRef Platform) {
143 return Platform.equals_insensitive(Config.Platform);
144 }))
145 return llvm::createStringError("Platform '" + Config.Platform +
146 "' is not available on this system");
147
148 auto Context =
149 std::make_shared<DeviceContext>(Config.Platform, /*DeviceId=*/0);
150 auto ExpectedTest =
151 TestType::create(Context, Config.Provider, DeviceBinaryDir);
152
153 if (!ExpectedTest)
154 return ExpectedTest.takeError();
155
156 const auto StartTime = std::chrono::steady_clock::now();
157
158 auto Result = ExpectedTest->run(Generator);
159
160 const auto EndTime = std::chrono::steady_clock::now();
161 const auto Duration = EndTime - StartTime;
162
163 printReport(*ExpectedTest, Result, Duration);
164
165 return Result.hasPassed();
166 }
167 } // namespace detail
168
169 template <auto Func, typename TestType = GpuMathTest<Func>>
170 [[nodiscard]] bool runTests(typename TestType::GeneratorType &Generator,
171 const llvm::SmallVector<TestConfig, 4> &Configs,
172 llvm::StringRef DeviceBinaryDir,
173 bool IsVerbose = false) {
174 const size_t NumConfigs = Configs.size();
175
176 if (NumConfigs == 0)
177 llvm::errs() << "There is no test configuration to run a test\n";
178
179 bool Passed = true;
180
181 for (const auto &[Index, Config] : llvm::enumerate(Configs)) {
182 detail::printPreamble<Func>(Config, Index, NumConfigs);
183
184 Generator.reset();
185
186 auto ExpectedPassed =
187 detail::runTest<Func, TestType>(Generator, Config, DeviceBinaryDir);
188
189 if (!ExpectedPassed) {
190 const auto Details = llvm::toString(ExpectedPassed.takeError());
191 llvm::errs()
192 << "WARNING: Conformance test not supported on this system\n";
193
194 if (IsVerbose)
195 llvm::errs() << "Details: " << Details << "\n";
196 } else {
197 Passed &= *ExpectedPassed;
198 }
199
200 llvm::errs() << "\n";
201 }
202
203 return Passed;
204 }
205 } // namespace mathtest
206
207 #endif // MATHTEST_TESTRUNNER_HPP