| 1 |
|
| 2 |
#undef NDEBUG |
| 3 |
#include <cassert> |
| 4 |
#include <vector> |
| 5 |
|
| 6 |
#include "../src/check.h" // NOTE: check.h is for internal use only! |
| 7 |
#include "benchmark/benchmark.h" |
| 8 |
|
| 9 |
namespace { |
| 10 |
|
| 11 |
class TestReporter : public benchmark::ConsoleReporter { |
| 12 |
public: |
| 13 |
bool ReportContext(const Context& context) override { |
| 14 |
return ConsoleReporter::ReportContext(context); |
| 15 |
}; |
| 16 |
|
| 17 |
void ReportRuns(const std::vector<Run>& report) override { |
| 18 |
all_runs_.insert(all_runs_.end(), begin(report), end(report)); |
| 19 |
ConsoleReporter::ReportRuns(report); |
| 20 |
} |
| 21 |
|
| 22 |
TestReporter() {} |
| 23 |
~TestReporter() override {} |
| 24 |
|
| 25 |
mutable std::vector<Run> all_runs_; |
| 26 |
}; |
| 27 |
|
| 28 |
struct TestCase { |
| 29 |
std::string name; |
| 30 |
bool error_occurred; |
| 31 |
std::string error_message; |
| 32 |
|
| 33 |
typedef benchmark::BenchmarkReporter::Run Run; |
| 34 |
|
| 35 |
void CheckRun(Run const& run) const { |
| 36 |
BM_CHECK(name == run.benchmark_name()) |
| 37 |
<< "expected " << name << " got " << run.benchmark_name(); |
| 38 |
BM_CHECK_EQ(error_occurred, |
| 39 |
benchmark::internal::SkippedWithError == run.skipped); |
| 40 |
BM_CHECK(error_message == run.skip_message); |
| 41 |
if (error_occurred) { |
| 42 |
// BM_CHECK(run.iterations == 0); |
| 43 |
} else { |
| 44 |
BM_CHECK(run.iterations != 0); |
| 45 |
} |
| 46 |
} |
| 47 |
}; |
| 48 |
|
| 49 |
std::vector<TestCase> ExpectedResults; |
| 50 |
|
| 51 |
int AddCases(const std::string& base_name, |
| 52 |
std::initializer_list<TestCase> const& v) { |
| 53 |
for (auto TC : v) { |
| 54 |
TC.name = base_name + TC.name; |
| 55 |
ExpectedResults.push_back(std::move(TC)); |
| 56 |
} |
| 57 |
return 0; |
| 58 |
} |
| 59 |
|
| 60 |
#define CONCAT(x, y) CONCAT2(x, y) |
| 61 |
#define CONCAT2(x, y) x##y |
| 62 |
#define ADD_CASES(...) int CONCAT(dummy, __LINE__) = AddCases(__VA_ARGS__) |
| 63 |
|
| 64 |
} // end namespace |
| 65 |
|
| 66 |
void BM_error_no_running(benchmark::State& state) { |
| 67 |
state.SkipWithError("error message"); |
| 68 |
} |
| 69 |
BENCHMARK(BM_error_no_running); |
| 70 |
ADD_CASES("BM_error_no_running", {{"", true, "error message"}}); |
| 71 |
|
| 72 |
void BM_error_before_running(benchmark::State& state) { |
| 73 |
state.SkipWithError("error message"); |
| 74 |
while (state.KeepRunning()) { |
| 75 |
assert(false); |
| 76 |
} |
| 77 |
} |
| 78 |
BENCHMARK(BM_error_before_running); |
| 79 |
ADD_CASES("BM_error_before_running", {{"", true, "error message"}}); |
| 80 |
|
| 81 |
void BM_error_before_running_batch(benchmark::State& state) { |
| 82 |
state.SkipWithError("error message"); |
| 83 |
while (state.KeepRunningBatch(17)) { |
| 84 |
assert(false); |
| 85 |
} |
| 86 |
} |
| 87 |
BENCHMARK(BM_error_before_running_batch); |
| 88 |
ADD_CASES("BM_error_before_running_batch", {{"", true, "error message"}}); |
| 89 |
|
| 90 |
void BM_error_before_running_range_for(benchmark::State& state) { |
| 91 |
state.SkipWithError("error message"); |
| 92 |
for (auto _ : state) { |
| 93 |
assert(false); |
| 94 |
} |
| 95 |
} |
| 96 |
BENCHMARK(BM_error_before_running_range_for); |
| 97 |
ADD_CASES("BM_error_before_running_range_for", {{"", true, "error message"}}); |
| 98 |
|
| 99 |
void BM_error_during_running(benchmark::State& state) { |
| 100 |
int first_iter = true; |
| 101 |
while (state.KeepRunning()) { |
| 102 |
if (state.range(0) == 1 && state.thread_index() <= (state.threads() / 2)) { |
| 103 |
assert(first_iter); |
| 104 |
first_iter = false; |
| 105 |
state.SkipWithError("error message"); |
| 106 |
} else { |
| 107 |
state.PauseTiming(); |
| 108 |
state.ResumeTiming(); |
| 109 |
} |
| 110 |
} |
| 111 |
} |
| 112 |
BENCHMARK(BM_error_during_running)->Arg(1)->Arg(2)->ThreadRange(1, 8); |
| 113 |
ADD_CASES("BM_error_during_running", {{"/1/threads:1", true, "error message"}, |
| 114 |
{"/1/threads:2", true, "error message"}, |
| 115 |
{"/1/threads:4", true, "error message"}, |
| 116 |
{"/1/threads:8", true, "error message"}, |
| 117 |
{"/2/threads:1", false, ""}, |
| 118 |
{"/2/threads:2", false, ""}, |
| 119 |
{"/2/threads:4", false, ""}, |
| 120 |
{"/2/threads:8", false, ""}}); |
| 121 |
|
| 122 |
void BM_error_during_running_ranged_for(benchmark::State& state) { |
| 123 |
assert(state.max_iterations > 3 && "test requires at least a few iterations"); |
| 124 |
bool first_iter = true; |
| 125 |
// NOTE: Users should not write the for loop explicitly. |
| 126 |
for (auto It = state.begin(), End = state.end(); It != End; ++It) { |
| 127 |
if (state.range(0) == 1) { |
| 128 |
assert(first_iter); |
| 129 |
first_iter = false; |
| 130 |
(void)first_iter; |
| 131 |
state.SkipWithError("error message"); |
| 132 |
// Test the unfortunate but documented behavior that the ranged-for loop |
| 133 |
// doesn't automatically terminate when SkipWithError is set. |
| 134 |
assert(++It != End); |
| 135 |
break; // Required behavior |
| 136 |
} |
| 137 |
} |
| 138 |
} |
| 139 |
BENCHMARK(BM_error_during_running_ranged_for)->Arg(1)->Arg(2)->Iterations(5); |
| 140 |
ADD_CASES("BM_error_during_running_ranged_for", |
| 141 |
{{"/1/iterations:5", true, "error message"}, |
| 142 |
{"/2/iterations:5", false, ""}}); |
| 143 |
|
| 144 |
void BM_error_after_running(benchmark::State& state) { |
| 145 |
for (auto _ : state) { |
| 146 |
auto iterations = double(state.iterations()) * double(state.iterations()); |
| 147 |
benchmark::DoNotOptimize(iterations); |
| 148 |
} |
| 149 |
if (state.thread_index() <= (state.threads() / 2)) |
| 150 |
state.SkipWithError("error message"); |
| 151 |
} |
| 152 |
BENCHMARK(BM_error_after_running)->ThreadRange(1, 8); |
| 153 |
ADD_CASES("BM_error_after_running", {{"/threads:1", true, "error message"}, |
| 154 |
{"/threads:2", true, "error message"}, |
| 155 |
{"/threads:4", true, "error message"}, |
| 156 |
{"/threads:8", true, "error message"}}); |
| 157 |
|
| 158 |
void BM_error_while_paused(benchmark::State& state) { |
| 159 |
bool first_iter = true; |
| 160 |
while (state.KeepRunning()) { |
| 161 |
if (state.range(0) == 1 && state.thread_index() <= (state.threads() / 2)) { |
| 162 |
assert(first_iter); |
| 163 |
first_iter = false; |
| 164 |
state.PauseTiming(); |
| 165 |
state.SkipWithError("error message"); |
| 166 |
} else { |
| 167 |
state.PauseTiming(); |
| 168 |
state.ResumeTiming(); |
| 169 |
} |
| 170 |
} |
| 171 |
} |
| 172 |
BENCHMARK(BM_error_while_paused)->Arg(1)->Arg(2)->ThreadRange(1, 8); |
| 173 |
ADD_CASES("BM_error_while_paused", {{"/1/threads:1", true, "error message"}, |
| 174 |
{"/1/threads:2", true, "error message"}, |
| 175 |
{"/1/threads:4", true, "error message"}, |
| 176 |
{"/1/threads:8", true, "error message"}, |
| 177 |
{"/2/threads:1", false, ""}, |
| 178 |
{"/2/threads:2", false, ""}, |
| 179 |
{"/2/threads:4", false, ""}, |
| 180 |
{"/2/threads:8", false, ""}}); |
| 181 |
|
| 182 |
int main(int argc, char* argv[]) { |
| 183 |
benchmark::Initialize(&argc, argv); |
| 184 |
|
| 185 |
TestReporter test_reporter; |
| 186 |
benchmark::RunSpecifiedBenchmarks(&test_reporter); |
| 187 |
|
| 188 |
typedef benchmark::BenchmarkReporter::Run Run; |
| 189 |
auto EB = ExpectedResults.begin(); |
| 190 |
|
| 191 |
for (Run const& run : test_reporter.all_runs_) { |
| 192 |
assert(EB != ExpectedResults.end()); |
| 193 |
EB->CheckRun(run); |
| 194 |
++EB; |
| 195 |
} |
| 196 |
assert(EB == ExpectedResults.end()); |
| 197 |
|
| 198 |
return 0; |
| 199 |
} |