| 1 |
#ifndef TEST_OUTPUT_TEST_H |
| 2 |
#define TEST_OUTPUT_TEST_H |
| 3 |
|
| 4 |
#undef NDEBUG |
| 5 |
#include <functional> |
| 6 |
#include <initializer_list> |
| 7 |
#include <memory> |
| 8 |
#include <sstream> |
| 9 |
#include <string> |
| 10 |
#include <utility> |
| 11 |
#include <vector> |
| 12 |
|
| 13 |
#include "../src/re.h" |
| 14 |
#include "benchmark/benchmark.h" |
| 15 |
|
| 16 |
#define CONCAT2(x, y) x##y |
| 17 |
#define CONCAT(x, y) CONCAT2(x, y) |
| 18 |
|
| 19 |
#define ADD_CASES(...) int CONCAT(dummy, __LINE__) = ::AddCases(__VA_ARGS__) |
| 20 |
|
| 21 |
#define SET_SUBSTITUTIONS(...) \ |
| 22 |
int CONCAT(dummy, __LINE__) = ::SetSubstitutions(__VA_ARGS__) |
| 23 |
|
| 24 |
enum MatchRules { |
| 25 |
MR_Default, // Skip non-matching lines until a match is found. |
| 26 |
MR_Next, // Match must occur on the next line. |
| 27 |
MR_Not // No line between the current position and the next match matches |
| 28 |
// the regex |
| 29 |
}; |
| 30 |
|
| 31 |
struct TestCase { |
| 32 |
TestCase(std::string re, int rule = MR_Default); |
| 33 |
|
| 34 |
std::string regex_str; |
| 35 |
int match_rule; |
| 36 |
std::string substituted_regex; |
| 37 |
std::shared_ptr<benchmark::Regex> regex; |
| 38 |
}; |
| 39 |
|
| 40 |
enum TestCaseID { |
| 41 |
TC_ConsoleOut, |
| 42 |
TC_ConsoleErr, |
| 43 |
TC_JSONOut, |
| 44 |
TC_JSONErr, |
| 45 |
TC_CSVOut, |
| 46 |
TC_CSVErr, |
| 47 |
|
| 48 |
TC_NumID // PRIVATE |
| 49 |
}; |
| 50 |
|
| 51 |
// Add a list of test cases to be run against the output specified by |
| 52 |
// 'ID' |
| 53 |
int AddCases(TestCaseID ID, std::initializer_list<TestCase> il); |
| 54 |
|
| 55 |
// Add or set a list of substitutions to be performed on constructed regex's |
| 56 |
// See 'output_test_helper.cc' for a list of default substitutions. |
| 57 |
int SetSubstitutions( |
| 58 |
std::initializer_list<std::pair<std::string, std::string>> il); |
| 59 |
|
| 60 |
// Run all output tests. |
| 61 |
void RunOutputTests(int argc, char* argv[]); |
| 62 |
|
| 63 |
// Count the number of 'pat' substrings in the 'haystack' string. |
| 64 |
int SubstrCnt(const std::string& haystack, const std::string& pat); |
| 65 |
|
| 66 |
// Run registered benchmarks with file reporter enabled, and return the content |
| 67 |
// outputted by the file reporter. |
| 68 |
std::string GetFileReporterOutput(int argc, char* argv[]); |
| 69 |
|
| 70 |
// ========================================================================= // |
| 71 |
// ------------------------- Results checking ------------------------------ // |
| 72 |
// ========================================================================= // |
| 73 |
|
| 74 |
// Call this macro to register a benchmark for checking its results. This |
| 75 |
// should be all that's needed. It subscribes a function to check the (CSV) |
| 76 |
// results of a benchmark. This is done only after verifying that the output |
| 77 |
// strings are really as expected. |
| 78 |
// bm_name_pattern: a name or a regex pattern which will be matched against |
| 79 |
// all the benchmark names. Matching benchmarks |
| 80 |
// will be the subject of a call to checker_function |
| 81 |
// checker_function: should be of type ResultsCheckFn (see below) |
| 82 |
#define CHECK_BENCHMARK_RESULTS(bm_name_pattern, checker_function) \ |
| 83 |
size_t CONCAT(dummy, __LINE__) = AddChecker(bm_name_pattern, checker_function) |
| 84 |
|
| 85 |
struct Results; |
| 86 |
typedef std::function<void(Results const&)> ResultsCheckFn; |
| 87 |
|
| 88 |
size_t AddChecker(const std::string& bm_name_pattern, const ResultsCheckFn& fn); |
| 89 |
|
| 90 |
// Class holding the results of a benchmark. |
| 91 |
// It is passed in calls to checker functions. |
| 92 |
struct Results { |
| 93 |
// the benchmark name |
| 94 |
std::string name; |
| 95 |
// the benchmark fields |
| 96 |
std::map<std::string, std::string> values; |
| 97 |
|
| 98 |
Results(const std::string& n) : name(n) {} |
| 99 |
|
| 100 |
int NumThreads() const; |
| 101 |
|
| 102 |
double NumIterations() const; |
| 103 |
|
| 104 |
typedef enum { kCpuTime, kRealTime } BenchmarkTime; |
| 105 |
|
| 106 |
// get cpu_time or real_time in seconds |
| 107 |
double GetTime(BenchmarkTime which) const; |
| 108 |
|
| 109 |
// get the real_time duration of the benchmark in seconds. |
| 110 |
// it is better to use fuzzy float checks for this, as the float |
| 111 |
// ASCII formatting is lossy. |
| 112 |
double DurationRealTime() const { |
| 113 |
return NumIterations() * GetTime(kRealTime); |
| 114 |
} |
| 115 |
// get the cpu_time duration of the benchmark in seconds |
| 116 |
double DurationCPUTime() const { return NumIterations() * GetTime(kCpuTime); } |
| 117 |
|
| 118 |
// get the string for a result by name, or nullptr if the name |
| 119 |
// is not found |
| 120 |
const std::string* Get(const std::string& entry_name) const { |
| 121 |
auto it = values.find(entry_name); |
| 122 |
if (it == values.end()) return nullptr; |
| 123 |
return &it->second; |
| 124 |
} |
| 125 |
|
| 126 |
// get a result by name, parsed as a specific type. |
| 127 |
// NOTE: for counters, use GetCounterAs instead. |
| 128 |
template <class T> |
| 129 |
T GetAs(const std::string& entry_name) const; |
| 130 |
|
| 131 |
// counters are written as doubles, so they have to be read first |
| 132 |
// as a double, and only then converted to the asked type. |
| 133 |
template <class T> |
| 134 |
T GetCounterAs(const std::string& entry_name) const { |
| 135 |
double dval = GetAs<double>(entry_name); |
| 136 |
T tval = static_cast<T>(dval); |
| 137 |
return tval; |
| 138 |
} |
| 139 |
}; |
| 140 |
|
| 141 |
template <class T> |
| 142 |
T Results::GetAs(const std::string& entry_name) const { |
| 143 |
auto* sv = Get(entry_name); |
| 144 |
BM_CHECK(sv != nullptr && !sv->empty()); |
| 145 |
std::stringstream ss; |
| 146 |
ss << *sv; |
| 147 |
T out; |
| 148 |
ss >> out; |
| 149 |
BM_CHECK(!ss.fail()); |
| 150 |
return out; |
| 151 |
} |
| 152 |
|
| 153 |
//---------------------------------- |
| 154 |
// Macros to help in result checking. Do not use them with arguments causing |
| 155 |
// side-effects. |
| 156 |
|
| 157 |
// clang-format off |
| 158 |
|
| 159 |
#define CHECK_RESULT_VALUE_IMPL(entry, getfn, var_type, var_name, relationship, value) \ |
| 160 |
CONCAT(BM_CHECK_, relationship) \ |
| 161 |
(entry.getfn< var_type >(var_name), (value)) << "\n" \ |
| 162 |
<< __FILE__ << ":" << __LINE__ << ": " << (entry).name << ":\n" \ |
| 163 |
<< __FILE__ << ":" << __LINE__ << ": " \ |
| 164 |
<< "expected (" << #var_type << ")" << (var_name) \ |
| 165 |
<< "=" << (entry).getfn< var_type >(var_name) \ |
| 166 |
<< " to be " #relationship " to " << (value) << "\n" |
| 167 |
|
| 168 |
// check with tolerance. eps_factor is the tolerance window, which is |
| 169 |
// interpreted relative to value (eg, 0.1 means 10% of value). |
| 170 |
#define CHECK_FLOAT_RESULT_VALUE_IMPL(entry, getfn, var_type, var_name, relationship, value, eps_factor) \ |
| 171 |
CONCAT(BM_CHECK_FLOAT_, relationship) \ |
| 172 |
(entry.getfn< var_type >(var_name), (value), (eps_factor) * (value)) << "\n" \ |
| 173 |
<< __FILE__ << ":" << __LINE__ << ": " << (entry).name << ":\n" \ |
| 174 |
<< __FILE__ << ":" << __LINE__ << ": " \ |
| 175 |
<< "expected (" << #var_type << ")" << (var_name) \ |
| 176 |
<< "=" << (entry).getfn< var_type >(var_name) \ |
| 177 |
<< " to be " #relationship " to " << (value) << "\n" \ |
| 178 |
<< __FILE__ << ":" << __LINE__ << ": " \ |
| 179 |
<< "with tolerance of " << (eps_factor) * (value) \ |
| 180 |
<< " (" << (eps_factor)*100. << "%), " \ |
| 181 |
<< "but delta was " << ((entry).getfn< var_type >(var_name) - (value)) \ |
| 182 |
<< " (" << (((entry).getfn< var_type >(var_name) - (value)) \ |
| 183 |
/ \ |
| 184 |
((value) > 1.e-5 || value < -1.e-5 ? value : 1.e-5)*100.) \ |
| 185 |
<< "%)" |
| 186 |
|
| 187 |
#define CHECK_RESULT_VALUE(entry, var_type, var_name, relationship, value) \ |
| 188 |
CHECK_RESULT_VALUE_IMPL(entry, GetAs, var_type, var_name, relationship, value) |
| 189 |
|
| 190 |
#define CHECK_COUNTER_VALUE(entry, var_type, var_name, relationship, value) \ |
| 191 |
CHECK_RESULT_VALUE_IMPL(entry, GetCounterAs, var_type, var_name, relationship, value) |
| 192 |
|
| 193 |
#define CHECK_FLOAT_RESULT_VALUE(entry, var_name, relationship, value, eps_factor) \ |
| 194 |
CHECK_FLOAT_RESULT_VALUE_IMPL(entry, GetAs, double, var_name, relationship, value, eps_factor) |
| 195 |
|
| 196 |
#define CHECK_FLOAT_COUNTER_VALUE(entry, var_name, relationship, value, eps_factor) \ |
| 197 |
CHECK_FLOAT_RESULT_VALUE_IMPL(entry, GetCounterAs, double, var_name, relationship, value, eps_factor) |
| 198 |
|
| 199 |
// clang-format on |
| 200 |
|
| 201 |
// ========================================================================= // |
| 202 |
// --------------------------- Misc Utilities ------------------------------ // |
| 203 |
// ========================================================================= // |
| 204 |
|
| 205 |
namespace { |
| 206 |
|
| 207 |
const char* const dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"; |
| 208 |
|
| 209 |
} // end namespace |
| 210 |
|
| 211 |
#endif // TEST_OUTPUT_TEST_H |