| 1 |
// Copyright 2015 Google Inc. All rights reserved. |
| 2 |
// |
| 3 |
// Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 |
// you may not use this file except in compliance with the License. |
| 5 |
// You may obtain a copy of the License at |
| 6 |
// |
| 7 |
// http://www.apache.org/licenses/LICENSE-2.0 |
| 8 |
// |
| 9 |
// Unless required by applicable law or agreed to in writing, software |
| 10 |
// distributed under the License is distributed on an "AS IS" BASIS, |
| 11 |
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 |
// See the License for the specific language governing permissions and |
| 13 |
// limitations under the License. |
| 14 |
|
| 15 |
#include "benchmark/benchmark.h" |
| 16 |
|
| 17 |
#include "benchmark_api_internal.h" |
| 18 |
#include "benchmark_runner.h" |
| 19 |
#include "internal_macros.h" |
| 20 |
|
| 21 |
#ifndef BENCHMARK_OS_WINDOWS |
| 22 |
#if !defined(BENCHMARK_OS_FUCHSIA) && !defined(BENCHMARK_OS_QURT) |
| 23 |
#include <sys/resource.h> |
| 24 |
#endif |
| 25 |
#include <sys/time.h> |
| 26 |
#include <unistd.h> |
| 27 |
#endif |
| 28 |
|
| 29 |
#include <algorithm> |
| 30 |
#include <atomic> |
| 31 |
#include <condition_variable> |
| 32 |
#include <cstdio> |
| 33 |
#include <cstdlib> |
| 34 |
#include <fstream> |
| 35 |
#include <iostream> |
| 36 |
#include <limits> |
| 37 |
#include <map> |
| 38 |
#include <memory> |
| 39 |
#include <random> |
| 40 |
#include <string> |
| 41 |
#include <thread> |
| 42 |
#include <utility> |
| 43 |
|
| 44 |
#include "check.h" |
| 45 |
#include "colorprint.h" |
| 46 |
#include "commandlineflags.h" |
| 47 |
#include "complexity.h" |
| 48 |
#include "counter.h" |
| 49 |
#include "internal_macros.h" |
| 50 |
#include "log.h" |
| 51 |
#include "mutex.h" |
| 52 |
#include "perf_counters.h" |
| 53 |
#include "re.h" |
| 54 |
#include "statistics.h" |
| 55 |
#include "string_util.h" |
| 56 |
#include "thread_manager.h" |
| 57 |
#include "thread_timer.h" |
| 58 |
|
| 59 |
namespace benchmark { |
| 60 |
// Print a list of benchmarks. This option overrides all other options. |
| 61 |
BM_DEFINE_bool(benchmark_list_tests, false); |
| 62 |
|
| 63 |
// A regular expression that specifies the set of benchmarks to execute. If |
| 64 |
// this flag is empty, or if this flag is the string \"all\", all benchmarks |
| 65 |
// linked into the binary are run. |
| 66 |
BM_DEFINE_string(benchmark_filter, ""); |
| 67 |
|
| 68 |
// Specification of how long to run the benchmark. |
| 69 |
// |
| 70 |
// It can be either an exact number of iterations (specified as `<integer>x`), |
| 71 |
// or a minimum number of seconds (specified as `<float>s`). If the latter |
| 72 |
// format (ie., min seconds) is used, the system may run the benchmark longer |
| 73 |
// until the results are considered significant. |
| 74 |
// |
| 75 |
// For backward compatibility, the `s` suffix may be omitted, in which case, |
| 76 |
// the specified number is interpreted as the number of seconds. |
| 77 |
// |
| 78 |
// For cpu-time based tests, this is the lower bound |
| 79 |
// on the total cpu time used by all threads that make up the test. For |
| 80 |
// real-time based tests, this is the lower bound on the elapsed time of the |
| 81 |
// benchmark execution, regardless of number of threads. |
| 82 |
BM_DEFINE_string(benchmark_min_time, kDefaultMinTimeStr); |
| 83 |
|
| 84 |
// Minimum number of seconds a benchmark should be run before results should be |
| 85 |
// taken into account. This e.g can be necessary for benchmarks of code which |
| 86 |
// needs to fill some form of cache before performance is of interest. |
| 87 |
// Note: results gathered within this period are discarded and not used for |
| 88 |
// reported result. |
| 89 |
BM_DEFINE_double(benchmark_min_warmup_time, 0.0); |
| 90 |
|
| 91 |
// The number of runs of each benchmark. If greater than 1, the mean and |
| 92 |
// standard deviation of the runs will be reported. |
| 93 |
BM_DEFINE_int32(benchmark_repetitions, 1); |
| 94 |
|
| 95 |
// If set, enable random interleaving of repetitions of all benchmarks. |
| 96 |
// See http://github.com/google/benchmark/issues/1051 for details. |
| 97 |
BM_DEFINE_bool(benchmark_enable_random_interleaving, false); |
| 98 |
|
| 99 |
// Report the result of each benchmark repetitions. When 'true' is specified |
| 100 |
// only the mean, standard deviation, and other statistics are reported for |
| 101 |
// repeated benchmarks. Affects all reporters. |
| 102 |
BM_DEFINE_bool(benchmark_report_aggregates_only, false); |
| 103 |
|
| 104 |
// Display the result of each benchmark repetitions. When 'true' is specified |
| 105 |
// only the mean, standard deviation, and other statistics are displayed for |
| 106 |
// repeated benchmarks. Unlike benchmark_report_aggregates_only, only affects |
| 107 |
// the display reporter, but *NOT* file reporter, which will still contain |
| 108 |
// all the output. |
| 109 |
BM_DEFINE_bool(benchmark_display_aggregates_only, false); |
| 110 |
|
| 111 |
// The format to use for console output. |
| 112 |
// Valid values are 'console', 'json', or 'csv'. |
| 113 |
BM_DEFINE_string(benchmark_format, "console"); |
| 114 |
|
| 115 |
// The format to use for file output. |
| 116 |
// Valid values are 'console', 'json', or 'csv'. |
| 117 |
BM_DEFINE_string(benchmark_out_format, "json"); |
| 118 |
|
| 119 |
// The file to write additional output to. |
| 120 |
BM_DEFINE_string(benchmark_out, ""); |
| 121 |
|
| 122 |
// Whether to use colors in the output. Valid values: |
| 123 |
// 'true'/'yes'/1, 'false'/'no'/0, and 'auto'. 'auto' means to use colors if |
| 124 |
// the output is being sent to a terminal and the TERM environment variable is |
| 125 |
// set to a terminal type that supports colors. |
| 126 |
BM_DEFINE_string(benchmark_color, "auto"); |
| 127 |
|
| 128 |
// Whether to use tabular format when printing user counters to the console. |
| 129 |
// Valid values: 'true'/'yes'/1, 'false'/'no'/0. Defaults to false. |
| 130 |
BM_DEFINE_bool(benchmark_counters_tabular, false); |
| 131 |
|
| 132 |
// List of additional perf counters to collect, in libpfm format. For more |
| 133 |
// information about libpfm: https://man7.org/linux/man-pages/man3/libpfm.3.html |
| 134 |
BM_DEFINE_string(benchmark_perf_counters, ""); |
| 135 |
|
| 136 |
// Extra context to include in the output formatted as comma-separated key-value |
| 137 |
// pairs. Kept internal as it's only used for parsing from env/command line. |
| 138 |
BM_DEFINE_kvpairs(benchmark_context, {}); |
| 139 |
|
| 140 |
// Set the default time unit to use for reports |
| 141 |
// Valid values are 'ns', 'us', 'ms' or 's' |
| 142 |
BM_DEFINE_string(benchmark_time_unit, ""); |
| 143 |
|
| 144 |
// The level of verbose logging to output |
| 145 |
BM_DEFINE_int32(v, 0); |
| 146 |
|
| 147 |
namespace internal { |
| 148 |
|
| 149 |
std::map<std::string, std::string>* global_context = nullptr; |
| 150 |
|
| 151 |
BENCHMARK_EXPORT std::map<std::string, std::string>*& GetGlobalContext() { |
| 152 |
return global_context; |
| 153 |
} |
| 154 |
|
| 155 |
// FIXME: wouldn't LTO mess this up? |
| 156 |
void UseCharPointer(char const volatile*) {} |
| 157 |
|
| 158 |
} // namespace internal |
| 159 |
|
| 160 |
State::State(std::string name, IterationCount max_iters, |
| 161 |
const std::vector<int64_t>& ranges, int thread_i, int n_threads, |
| 162 |
internal::ThreadTimer* timer, internal::ThreadManager* manager, |
| 163 |
internal::PerfCountersMeasurement* perf_counters_measurement) |
| 164 |
: total_iterations_(0), |
| 165 |
batch_leftover_(0), |
| 166 |
max_iterations(max_iters), |
| 167 |
started_(false), |
| 168 |
finished_(false), |
| 169 |
skipped_(internal::NotSkipped), |
| 170 |
range_(ranges), |
| 171 |
complexity_n_(0), |
| 172 |
name_(std::move(name)), |
| 173 |
thread_index_(thread_i), |
| 174 |
threads_(n_threads), |
| 175 |
timer_(timer), |
| 176 |
manager_(manager), |
| 177 |
perf_counters_measurement_(perf_counters_measurement) { |
| 178 |
BM_CHECK(max_iterations != 0) << "At least one iteration must be run"; |
| 179 |
BM_CHECK_LT(thread_index_, threads_) |
| 180 |
<< "thread_index must be less than threads"; |
| 181 |
|
| 182 |
// Add counters with correct flag now. If added with `counters[name]` in |
| 183 |
// `PauseTiming`, a new `Counter` will be inserted the first time, which |
| 184 |
// won't have the flag. Inserting them now also reduces the allocations |
| 185 |
// during the benchmark. |
| 186 |
if (perf_counters_measurement_) { |
| 187 |
for (const std::string& counter_name : |
| 188 |
perf_counters_measurement_->names()) { |
| 189 |
counters[counter_name] = Counter(0.0, Counter::kAvgIterations); |
| 190 |
} |
| 191 |
} |
| 192 |
|
| 193 |
// Note: The use of offsetof below is technically undefined until C++17 |
| 194 |
// because State is not a standard layout type. However, all compilers |
| 195 |
// currently provide well-defined behavior as an extension (which is |
| 196 |
// demonstrated since constexpr evaluation must diagnose all undefined |
| 197 |
// behavior). However, GCC and Clang also warn about this use of offsetof, |
| 198 |
// which must be suppressed. |
| 199 |
#if defined(__INTEL_COMPILER) |
| 200 |
#pragma warning push |
| 201 |
#pragma warning(disable : 1875) |
| 202 |
#elif defined(__GNUC__) |
| 203 |
#pragma GCC diagnostic push |
| 204 |
#pragma GCC diagnostic ignored "-Winvalid-offsetof" |
| 205 |
#elif defined(__clang__) |
| 206 |
#pragma clang diagnostic push |
| 207 |
#pragma clang diagnostic ignored "-Winvalid-offsetof" |
| 208 |
#endif |
| 209 |
#if defined(__NVCC__) |
| 210 |
#pragma nv_diagnostic push |
| 211 |
#pragma nv_diag_suppress 1427 |
| 212 |
#endif |
| 213 |
#if defined(__NVCOMPILER) |
| 214 |
#pragma diagnostic push |
| 215 |
#pragma diag_suppress offset_in_non_POD_nonstandard |
| 216 |
#endif |
| 217 |
// Offset tests to ensure commonly accessed data is on the first cache line. |
| 218 |
const int cache_line_size = 64; |
| 219 |
static_assert( |
| 220 |
offsetof(State, skipped_) <= (cache_line_size - sizeof(skipped_)), ""); |
| 221 |
#if defined(__INTEL_COMPILER) |
| 222 |
#pragma warning pop |
| 223 |
#elif defined(__GNUC__) |
| 224 |
#pragma GCC diagnostic pop |
| 225 |
#elif defined(__clang__) |
| 226 |
#pragma clang diagnostic pop |
| 227 |
#endif |
| 228 |
#if defined(__NVCC__) |
| 229 |
#pragma nv_diagnostic pop |
| 230 |
#endif |
| 231 |
#if defined(__NVCOMPILER) |
| 232 |
#pragma diagnostic pop |
| 233 |
#endif |
| 234 |
} |
| 235 |
|
| 236 |
void State::PauseTiming() { |
| 237 |
// Add in time accumulated so far |
| 238 |
BM_CHECK(started_ && !finished_ && !skipped()); |
| 239 |
timer_->StopTimer(); |
| 240 |
if (perf_counters_measurement_) { |
| 241 |
std::vector<std::pair<std::string, double>> measurements; |
| 242 |
if (!perf_counters_measurement_->Stop(measurements)) { |
| 243 |
BM_CHECK(false) << "Perf counters read the value failed."; |
| 244 |
} |
| 245 |
for (const auto& name_and_measurement : measurements) { |
| 246 |
const std::string& name = name_and_measurement.first; |
| 247 |
const double measurement = name_and_measurement.second; |
| 248 |
// Counter was inserted with `kAvgIterations` flag by the constructor. |
| 249 |
assert(counters.find(name) != counters.end()); |
| 250 |
counters[name].value += measurement; |
| 251 |
} |
| 252 |
} |
| 253 |
} |
| 254 |
|
| 255 |
void State::ResumeTiming() { |
| 256 |
BM_CHECK(started_ && !finished_ && !skipped()); |
| 257 |
timer_->StartTimer(); |
| 258 |
if (perf_counters_measurement_) { |
| 259 |
perf_counters_measurement_->Start(); |
| 260 |
} |
| 261 |
} |
| 262 |
|
| 263 |
void State::SkipWithMessage(const std::string& msg) { |
| 264 |
skipped_ = internal::SkippedWithMessage; |
| 265 |
{ |
| 266 |
MutexLock l(manager_->GetBenchmarkMutex()); |
| 267 |
if (internal::NotSkipped == manager_->results.skipped_) { |
| 268 |
manager_->results.skip_message_ = msg; |
| 269 |
manager_->results.skipped_ = skipped_; |
| 270 |
} |
| 271 |
} |
| 272 |
total_iterations_ = 0; |
| 273 |
if (timer_->running()) timer_->StopTimer(); |
| 274 |
} |
| 275 |
|
| 276 |
void State::SkipWithError(const std::string& msg) { |
| 277 |
skipped_ = internal::SkippedWithError; |
| 278 |
{ |
| 279 |
MutexLock l(manager_->GetBenchmarkMutex()); |
| 280 |
if (internal::NotSkipped == manager_->results.skipped_) { |
| 281 |
manager_->results.skip_message_ = msg; |
| 282 |
manager_->results.skipped_ = skipped_; |
| 283 |
} |
| 284 |
} |
| 285 |
total_iterations_ = 0; |
| 286 |
if (timer_->running()) timer_->StopTimer(); |
| 287 |
} |
| 288 |
|
| 289 |
void State::SetIterationTime(double seconds) { |
| 290 |
timer_->SetIterationTime(seconds); |
| 291 |
} |
| 292 |
|
| 293 |
void State::SetLabel(const std::string& label) { |
| 294 |
MutexLock l(manager_->GetBenchmarkMutex()); |
| 295 |
manager_->results.report_label_ = label; |
| 296 |
} |
| 297 |
|
| 298 |
void State::StartKeepRunning() { |
| 299 |
BM_CHECK(!started_ && !finished_); |
| 300 |
started_ = true; |
| 301 |
total_iterations_ = skipped() ? 0 : max_iterations; |
| 302 |
manager_->StartStopBarrier(); |
| 303 |
if (!skipped()) ResumeTiming(); |
| 304 |
} |
| 305 |
|
| 306 |
void State::FinishKeepRunning() { |
| 307 |
BM_CHECK(started_ && (!finished_ || skipped())); |
| 308 |
if (!skipped()) { |
| 309 |
PauseTiming(); |
| 310 |
} |
| 311 |
// Total iterations has now wrapped around past 0. Fix this. |
| 312 |
total_iterations_ = 0; |
| 313 |
finished_ = true; |
| 314 |
manager_->StartStopBarrier(); |
| 315 |
} |
| 316 |
|
| 317 |
namespace internal { |
| 318 |
namespace { |
| 319 |
|
| 320 |
// Flushes streams after invoking reporter methods that write to them. This |
| 321 |
// ensures users get timely updates even when streams are not line-buffered. |
| 322 |
void FlushStreams(BenchmarkReporter* reporter) { |
| 323 |
if (!reporter) return; |
| 324 |
std::flush(reporter->GetOutputStream()); |
| 325 |
std::flush(reporter->GetErrorStream()); |
| 326 |
} |
| 327 |
|
| 328 |
// Reports in both display and file reporters. |
| 329 |
void Report(BenchmarkReporter* display_reporter, |
| 330 |
BenchmarkReporter* file_reporter, const RunResults& run_results) { |
| 331 |
auto report_one = [](BenchmarkReporter* reporter, bool aggregates_only, |
| 332 |
const RunResults& results) { |
| 333 |
assert(reporter); |
| 334 |
// If there are no aggregates, do output non-aggregates. |
| 335 |
aggregates_only &= !results.aggregates_only.empty(); |
| 336 |
if (!aggregates_only) reporter->ReportRuns(results.non_aggregates); |
| 337 |
if (!results.aggregates_only.empty()) |
| 338 |
reporter->ReportRuns(results.aggregates_only); |
| 339 |
}; |
| 340 |
|
| 341 |
report_one(display_reporter, run_results.display_report_aggregates_only, |
| 342 |
run_results); |
| 343 |
if (file_reporter) |
| 344 |
report_one(file_reporter, run_results.file_report_aggregates_only, |
| 345 |
run_results); |
| 346 |
|
| 347 |
FlushStreams(display_reporter); |
| 348 |
FlushStreams(file_reporter); |
| 349 |
} |
| 350 |
|
| 351 |
void RunBenchmarks(const std::vector<BenchmarkInstance>& benchmarks, |
| 352 |
BenchmarkReporter* display_reporter, |
| 353 |
BenchmarkReporter* file_reporter) { |
| 354 |
// Note the file_reporter can be null. |
| 355 |
BM_CHECK(display_reporter != nullptr); |
| 356 |
|
| 357 |
// Determine the width of the name field using a minimum width of 10. |
| 358 |
bool might_have_aggregates = FLAGS_benchmark_repetitions > 1; |
| 359 |
size_t name_field_width = 10; |
| 360 |
size_t stat_field_width = 0; |
| 361 |
for (const BenchmarkInstance& benchmark : benchmarks) { |
| 362 |
name_field_width = |
| 363 |
std::max<size_t>(name_field_width, benchmark.name().str().size()); |
| 364 |
might_have_aggregates |= benchmark.repetitions() > 1; |
| 365 |
|
| 366 |
for (const auto& Stat : benchmark.statistics()) |
| 367 |
stat_field_width = std::max<size_t>(stat_field_width, Stat.name_.size()); |
| 368 |
} |
| 369 |
if (might_have_aggregates) name_field_width += 1 + stat_field_width; |
| 370 |
|
| 371 |
// Print header here |
| 372 |
BenchmarkReporter::Context context; |
| 373 |
context.name_field_width = name_field_width; |
| 374 |
|
| 375 |
// Keep track of running times of all instances of each benchmark family. |
| 376 |
std::map<int /*family_index*/, BenchmarkReporter::PerFamilyRunReports> |
| 377 |
per_family_reports; |
| 378 |
|
| 379 |
if (display_reporter->ReportContext(context) && |
| 380 |
(!file_reporter || file_reporter->ReportContext(context))) { |
| 381 |
FlushStreams(display_reporter); |
| 382 |
FlushStreams(file_reporter); |
| 383 |
|
| 384 |
size_t num_repetitions_total = 0; |
| 385 |
|
| 386 |
// This perfcounters object needs to be created before the runners vector |
| 387 |
// below so it outlasts their lifetime. |
| 388 |
PerfCountersMeasurement perfcounters( |
| 389 |
StrSplit(FLAGS_benchmark_perf_counters, ',')); |
| 390 |
|
| 391 |
// Vector of benchmarks to run |
| 392 |
std::vector<internal::BenchmarkRunner> runners; |
| 393 |
runners.reserve(benchmarks.size()); |
| 394 |
|
| 395 |
// Count the number of benchmarks with threads to warn the user in case |
| 396 |
// performance counters are used. |
| 397 |
int benchmarks_with_threads = 0; |
| 398 |
|
| 399 |
// Loop through all benchmarks |
| 400 |
for (const BenchmarkInstance& benchmark : benchmarks) { |
| 401 |
BenchmarkReporter::PerFamilyRunReports* reports_for_family = nullptr; |
| 402 |
if (benchmark.complexity() != oNone) |
| 403 |
reports_for_family = &per_family_reports[benchmark.family_index()]; |
| 404 |
benchmarks_with_threads += (benchmark.threads() > 1); |
| 405 |
runners.emplace_back(benchmark, &perfcounters, reports_for_family); |
| 406 |
int num_repeats_of_this_instance = runners.back().GetNumRepeats(); |
| 407 |
num_repetitions_total += num_repeats_of_this_instance; |
| 408 |
if (reports_for_family) |
| 409 |
reports_for_family->num_runs_total += num_repeats_of_this_instance; |
| 410 |
} |
| 411 |
assert(runners.size() == benchmarks.size() && "Unexpected runner count."); |
| 412 |
|
| 413 |
// The use of performance counters with threads would be unintuitive for |
| 414 |
// the average user so we need to warn them about this case |
| 415 |
if ((benchmarks_with_threads > 0) && (perfcounters.num_counters() > 0)) { |
| 416 |
GetErrorLogInstance() |
| 417 |
<< "***WARNING*** There are " << benchmarks_with_threads |
| 418 |
<< " benchmarks with threads and " << perfcounters.num_counters() |
| 419 |
<< " performance counters were requested. Beware counters will " |
| 420 |
"reflect the combined usage across all " |
| 421 |
"threads.\n"; |
| 422 |
} |
| 423 |
|
| 424 |
std::vector<size_t> repetition_indices; |
| 425 |
repetition_indices.reserve(num_repetitions_total); |
| 426 |
for (size_t runner_index = 0, num_runners = runners.size(); |
| 427 |
runner_index != num_runners; ++runner_index) { |
| 428 |
const internal::BenchmarkRunner& runner = runners[runner_index]; |
| 429 |
std::fill_n(std::back_inserter(repetition_indices), |
| 430 |
runner.GetNumRepeats(), runner_index); |
| 431 |
} |
| 432 |
assert(repetition_indices.size() == num_repetitions_total && |
| 433 |
"Unexpected number of repetition indexes."); |
| 434 |
|
| 435 |
if (FLAGS_benchmark_enable_random_interleaving) { |
| 436 |
std::random_device rd; |
| 437 |
std::mt19937 g(rd()); |
| 438 |
std::shuffle(repetition_indices.begin(), repetition_indices.end(), g); |
| 439 |
} |
| 440 |
|
| 441 |
for (size_t repetition_index : repetition_indices) { |
| 442 |
internal::BenchmarkRunner& runner = runners[repetition_index]; |
| 443 |
runner.DoOneRepetition(); |
| 444 |
if (runner.HasRepeatsRemaining()) continue; |
| 445 |
// FIXME: report each repetition separately, not all of them in bulk. |
| 446 |
|
| 447 |
display_reporter->ReportRunsConfig( |
| 448 |
runner.GetMinTime(), runner.HasExplicitIters(), runner.GetIters()); |
| 449 |
if (file_reporter) |
| 450 |
file_reporter->ReportRunsConfig( |
| 451 |
runner.GetMinTime(), runner.HasExplicitIters(), runner.GetIters()); |
| 452 |
|
| 453 |
RunResults run_results = runner.GetResults(); |
| 454 |
|
| 455 |
// Maybe calculate complexity report |
| 456 |
if (const auto* reports_for_family = runner.GetReportsForFamily()) { |
| 457 |
if (reports_for_family->num_runs_done == |
| 458 |
reports_for_family->num_runs_total) { |
| 459 |
auto additional_run_stats = ComputeBigO(reports_for_family->Runs); |
| 460 |
run_results.aggregates_only.insert(run_results.aggregates_only.end(), |
| 461 |
additional_run_stats.begin(), |
| 462 |
additional_run_stats.end()); |
| 463 |
per_family_reports.erase( |
| 464 |
static_cast<int>(reports_for_family->Runs.front().family_index)); |
| 465 |
} |
| 466 |
} |
| 467 |
|
| 468 |
Report(display_reporter, file_reporter, run_results); |
| 469 |
} |
| 470 |
} |
| 471 |
display_reporter->Finalize(); |
| 472 |
if (file_reporter) file_reporter->Finalize(); |
| 473 |
FlushStreams(display_reporter); |
| 474 |
FlushStreams(file_reporter); |
| 475 |
} |
| 476 |
|
| 477 |
// Disable deprecated warnings temporarily because we need to reference |
| 478 |
// CSVReporter but don't want to trigger -Werror=-Wdeprecated-declarations |
| 479 |
BENCHMARK_DISABLE_DEPRECATED_WARNING |
| 480 |
|
| 481 |
std::unique_ptr<BenchmarkReporter> CreateReporter( |
| 482 |
std::string const& name, ConsoleReporter::OutputOptions output_opts) { |
| 483 |
typedef std::unique_ptr<BenchmarkReporter> PtrType; |
| 484 |
if (name == "console") { |
| 485 |
return PtrType(new ConsoleReporter(output_opts)); |
| 486 |
} |
| 487 |
if (name == "json") { |
| 488 |
return PtrType(new JSONReporter()); |
| 489 |
} |
| 490 |
if (name == "csv") { |
| 491 |
return PtrType(new CSVReporter()); |
| 492 |
} |
| 493 |
std::cerr << "Unexpected format: '" << name << "'\n"; |
| 494 |
std::exit(1); |
| 495 |
} |
| 496 |
|
| 497 |
BENCHMARK_RESTORE_DEPRECATED_WARNING |
| 498 |
|
| 499 |
} // end namespace |
| 500 |
|
| 501 |
bool IsZero(double n) { |
| 502 |
return std::abs(n) < std::numeric_limits<double>::epsilon(); |
| 503 |
} |
| 504 |
|
| 505 |
ConsoleReporter::OutputOptions GetOutputOptions(bool force_no_color) { |
| 506 |
int output_opts = ConsoleReporter::OO_Defaults; |
| 507 |
auto is_benchmark_color = [force_no_color]() -> bool { |
| 508 |
if (force_no_color) { |
| 509 |
return false; |
| 510 |
} |
| 511 |
if (FLAGS_benchmark_color == "auto") { |
| 512 |
return IsColorTerminal(); |
| 513 |
} |
| 514 |
return IsTruthyFlagValue(FLAGS_benchmark_color); |
| 515 |
}; |
| 516 |
if (is_benchmark_color()) { |
| 517 |
output_opts |= ConsoleReporter::OO_Color; |
| 518 |
} else { |
| 519 |
output_opts &= ~ConsoleReporter::OO_Color; |
| 520 |
} |
| 521 |
if (FLAGS_benchmark_counters_tabular) { |
| 522 |
output_opts |= ConsoleReporter::OO_Tabular; |
| 523 |
} else { |
| 524 |
output_opts &= ~ConsoleReporter::OO_Tabular; |
| 525 |
} |
| 526 |
return static_cast<ConsoleReporter::OutputOptions>(output_opts); |
| 527 |
} |
| 528 |
|
| 529 |
} // end namespace internal |
| 530 |
|
| 531 |
BenchmarkReporter* CreateDefaultDisplayReporter() { |
| 532 |
static auto default_display_reporter = |
| 533 |
internal::CreateReporter(FLAGS_benchmark_format, |
| 534 |
internal::GetOutputOptions()) |
| 535 |
.release(); |
| 536 |
return default_display_reporter; |
| 537 |
} |
| 538 |
|
| 539 |
size_t RunSpecifiedBenchmarks() { |
| 540 |
return RunSpecifiedBenchmarks(nullptr, nullptr, FLAGS_benchmark_filter); |
| 541 |
} |
| 542 |
|
| 543 |
size_t RunSpecifiedBenchmarks(std::string spec) { |
| 544 |
return RunSpecifiedBenchmarks(nullptr, nullptr, std::move(spec)); |
| 545 |
} |
| 546 |
|
| 547 |
size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter) { |
| 548 |
return RunSpecifiedBenchmarks(display_reporter, nullptr, |
| 549 |
FLAGS_benchmark_filter); |
| 550 |
} |
| 551 |
|
| 552 |
size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter, |
| 553 |
std::string spec) { |
| 554 |
return RunSpecifiedBenchmarks(display_reporter, nullptr, std::move(spec)); |
| 555 |
} |
| 556 |
|
| 557 |
size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter, |
| 558 |
BenchmarkReporter* file_reporter) { |
| 559 |
return RunSpecifiedBenchmarks(display_reporter, file_reporter, |
| 560 |
FLAGS_benchmark_filter); |
| 561 |
} |
| 562 |
|
| 563 |
size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter, |
| 564 |
BenchmarkReporter* file_reporter, |
| 565 |
std::string spec) { |
| 566 |
if (spec.empty() || spec == "all") |
| 567 |
spec = "."; // Regexp that matches all benchmarks |
| 568 |
|
| 569 |
// Setup the reporters |
| 570 |
std::ofstream output_file; |
| 571 |
std::unique_ptr<BenchmarkReporter> default_display_reporter; |
| 572 |
std::unique_ptr<BenchmarkReporter> default_file_reporter; |
| 573 |
if (!display_reporter) { |
| 574 |
default_display_reporter.reset(CreateDefaultDisplayReporter()); |
| 575 |
display_reporter = default_display_reporter.get(); |
| 576 |
} |
| 577 |
auto& Out = display_reporter->GetOutputStream(); |
| 578 |
auto& Err = display_reporter->GetErrorStream(); |
| 579 |
|
| 580 |
std::string const& fname = FLAGS_benchmark_out; |
| 581 |
if (fname.empty() && file_reporter) { |
| 582 |
Err << "A custom file reporter was provided but " |
| 583 |
"--benchmark_out=<file> was not specified." |
| 584 |
<< std::endl; |
| 585 |
Out.flush(); |
| 586 |
Err.flush(); |
| 587 |
std::exit(1); |
| 588 |
} |
| 589 |
if (!fname.empty()) { |
| 590 |
output_file.open(fname); |
| 591 |
if (!output_file.is_open()) { |
| 592 |
Err << "invalid file name: '" << fname << "'" << std::endl; |
| 593 |
Out.flush(); |
| 594 |
Err.flush(); |
| 595 |
std::exit(1); |
| 596 |
} |
| 597 |
if (!file_reporter) { |
| 598 |
default_file_reporter = internal::CreateReporter( |
| 599 |
FLAGS_benchmark_out_format, FLAGS_benchmark_counters_tabular |
| 600 |
? ConsoleReporter::OO_Tabular |
| 601 |
: ConsoleReporter::OO_None); |
| 602 |
file_reporter = default_file_reporter.get(); |
| 603 |
} |
| 604 |
file_reporter->SetOutputStream(&output_file); |
| 605 |
file_reporter->SetErrorStream(&output_file); |
| 606 |
} |
| 607 |
|
| 608 |
std::vector<internal::BenchmarkInstance> benchmarks; |
| 609 |
if (!FindBenchmarksInternal(spec, &benchmarks, &Err)) { |
| 610 |
Out.flush(); |
| 611 |
Err.flush(); |
| 612 |
return 0; |
| 613 |
} |
| 614 |
|
| 615 |
if (benchmarks.empty()) { |
| 616 |
Err << "Failed to match any benchmarks against regex: " << spec << "\n"; |
| 617 |
Out.flush(); |
| 618 |
Err.flush(); |
| 619 |
return 0; |
| 620 |
} |
| 621 |
|
| 622 |
if (FLAGS_benchmark_list_tests) { |
| 623 |
for (auto const& benchmark : benchmarks) |
| 624 |
Out << benchmark.name().str() << "\n"; |
| 625 |
} else { |
| 626 |
internal::RunBenchmarks(benchmarks, display_reporter, file_reporter); |
| 627 |
} |
| 628 |
|
| 629 |
Out.flush(); |
| 630 |
Err.flush(); |
| 631 |
return benchmarks.size(); |
| 632 |
} |
| 633 |
|
| 634 |
namespace { |
| 635 |
// stores the time unit benchmarks use by default |
| 636 |
TimeUnit default_time_unit = kNanosecond; |
| 637 |
} // namespace |
| 638 |
|
| 639 |
TimeUnit GetDefaultTimeUnit() { return default_time_unit; } |
| 640 |
|
| 641 |
void SetDefaultTimeUnit(TimeUnit unit) { default_time_unit = unit; } |
| 642 |
|
| 643 |
std::string GetBenchmarkFilter() { return FLAGS_benchmark_filter; } |
| 644 |
|
| 645 |
void SetBenchmarkFilter(std::string value) { |
| 646 |
FLAGS_benchmark_filter = std::move(value); |
| 647 |
} |
| 648 |
|
| 649 |
int32_t GetBenchmarkVerbosity() { return FLAGS_v; } |
| 650 |
|
| 651 |
void RegisterMemoryManager(MemoryManager* manager) { |
| 652 |
internal::memory_manager = manager; |
| 653 |
} |
| 654 |
|
| 655 |
void AddCustomContext(const std::string& key, const std::string& value) { |
| 656 |
if (internal::global_context == nullptr) { |
| 657 |
internal::global_context = new std::map<std::string, std::string>(); |
| 658 |
} |
| 659 |
if (!internal::global_context->emplace(key, value).second) { |
| 660 |
std::cerr << "Failed to add custom context \"" << key << "\" as it already " |
| 661 |
<< "exists with value \"" << value << "\"\n"; |
| 662 |
} |
| 663 |
} |
| 664 |
|
| 665 |
namespace internal { |
| 666 |
|
| 667 |
void (*HelperPrintf)(); |
| 668 |
|
| 669 |
void PrintUsageAndExit() { |
| 670 |
HelperPrintf(); |
| 671 |
exit(0); |
| 672 |
} |
| 673 |
|
| 674 |
void SetDefaultTimeUnitFromFlag(const std::string& time_unit_flag) { |
| 675 |
if (time_unit_flag == "s") { |
| 676 |
return SetDefaultTimeUnit(kSecond); |
| 677 |
} |
| 678 |
if (time_unit_flag == "ms") { |
| 679 |
return SetDefaultTimeUnit(kMillisecond); |
| 680 |
} |
| 681 |
if (time_unit_flag == "us") { |
| 682 |
return SetDefaultTimeUnit(kMicrosecond); |
| 683 |
} |
| 684 |
if (time_unit_flag == "ns") { |
| 685 |
return SetDefaultTimeUnit(kNanosecond); |
| 686 |
} |
| 687 |
if (!time_unit_flag.empty()) { |
| 688 |
PrintUsageAndExit(); |
| 689 |
} |
| 690 |
} |
| 691 |
|
| 692 |
void ParseCommandLineFlags(int* argc, char** argv) { |
| 693 |
using namespace benchmark; |
| 694 |
BenchmarkReporter::Context::executable_name = |
| 695 |
(argc && *argc > 0) ? argv[0] : "unknown"; |
| 696 |
for (int i = 1; argc && i < *argc; ++i) { |
| 697 |
if (ParseBoolFlag(argv[i], "benchmark_list_tests", |
| 698 |
&FLAGS_benchmark_list_tests) || |
| 699 |
ParseStringFlag(argv[i], "benchmark_filter", &FLAGS_benchmark_filter) || |
| 700 |
ParseStringFlag(argv[i], "benchmark_min_time", |
| 701 |
&FLAGS_benchmark_min_time) || |
| 702 |
ParseDoubleFlag(argv[i], "benchmark_min_warmup_time", |
| 703 |
&FLAGS_benchmark_min_warmup_time) || |
| 704 |
ParseInt32Flag(argv[i], "benchmark_repetitions", |
| 705 |
&FLAGS_benchmark_repetitions) || |
| 706 |
ParseBoolFlag(argv[i], "benchmark_enable_random_interleaving", |
| 707 |
&FLAGS_benchmark_enable_random_interleaving) || |
| 708 |
ParseBoolFlag(argv[i], "benchmark_report_aggregates_only", |
| 709 |
&FLAGS_benchmark_report_aggregates_only) || |
| 710 |
ParseBoolFlag(argv[i], "benchmark_display_aggregates_only", |
| 711 |
&FLAGS_benchmark_display_aggregates_only) || |
| 712 |
ParseStringFlag(argv[i], "benchmark_format", &FLAGS_benchmark_format) || |
| 713 |
ParseStringFlag(argv[i], "benchmark_out", &FLAGS_benchmark_out) || |
| 714 |
ParseStringFlag(argv[i], "benchmark_out_format", |
| 715 |
&FLAGS_benchmark_out_format) || |
| 716 |
ParseStringFlag(argv[i], "benchmark_color", &FLAGS_benchmark_color) || |
| 717 |
ParseBoolFlag(argv[i], "benchmark_counters_tabular", |
| 718 |
&FLAGS_benchmark_counters_tabular) || |
| 719 |
ParseStringFlag(argv[i], "benchmark_perf_counters", |
| 720 |
&FLAGS_benchmark_perf_counters) || |
| 721 |
ParseKeyValueFlag(argv[i], "benchmark_context", |
| 722 |
&FLAGS_benchmark_context) || |
| 723 |
ParseStringFlag(argv[i], "benchmark_time_unit", |
| 724 |
&FLAGS_benchmark_time_unit) || |
| 725 |
ParseInt32Flag(argv[i], "v", &FLAGS_v)) { |
| 726 |
for (int j = i; j != *argc - 1; ++j) argv[j] = argv[j + 1]; |
| 727 |
|
| 728 |
--(*argc); |
| 729 |
--i; |
| 730 |
} else if (IsFlag(argv[i], "help")) { |
| 731 |
PrintUsageAndExit(); |
| 732 |
} |
| 733 |
} |
| 734 |
for (auto const* flag : |
| 735 |
{&FLAGS_benchmark_format, &FLAGS_benchmark_out_format}) { |
| 736 |
if (*flag != "console" && *flag != "json" && *flag != "csv") { |
| 737 |
PrintUsageAndExit(); |
| 738 |
} |
| 739 |
} |
| 740 |
SetDefaultTimeUnitFromFlag(FLAGS_benchmark_time_unit); |
| 741 |
if (FLAGS_benchmark_color.empty()) { |
| 742 |
PrintUsageAndExit(); |
| 743 |
} |
| 744 |
for (const auto& kv : FLAGS_benchmark_context) { |
| 745 |
AddCustomContext(kv.first, kv.second); |
| 746 |
} |
| 747 |
} |
| 748 |
|
| 749 |
int InitializeStreams() { |
| 750 |
static std::ios_base::Init init; |
| 751 |
return 0; |
| 752 |
} |
| 753 |
|
| 754 |
} // end namespace internal |
| 755 |
|
| 756 |
std::string GetBenchmarkVersion() { return {BENCHMARK_VERSION}; } |
| 757 |
|
| 758 |
void PrintDefaultHelp() { |
| 759 |
fprintf(stdout, |
| 760 |
"benchmark" |
| 761 |
" [--benchmark_list_tests={true|false}]\n" |
| 762 |
" [--benchmark_filter=<regex>]\n" |
| 763 |
" [--benchmark_min_time=`<integer>x` OR `<float>s` ]\n" |
| 764 |
" [--benchmark_min_warmup_time=<min_warmup_time>]\n" |
| 765 |
" [--benchmark_repetitions=<num_repetitions>]\n" |
| 766 |
" [--benchmark_enable_random_interleaving={true|false}]\n" |
| 767 |
" [--benchmark_report_aggregates_only={true|false}]\n" |
| 768 |
" [--benchmark_display_aggregates_only={true|false}]\n" |
| 769 |
" [--benchmark_format=<console|json|csv>]\n" |
| 770 |
" [--benchmark_out=<filename>]\n" |
| 771 |
" [--benchmark_out_format=<json|console|csv>]\n" |
| 772 |
" [--benchmark_color={auto|true|false}]\n" |
| 773 |
" [--benchmark_counters_tabular={true|false}]\n" |
| 774 |
#if defined HAVE_LIBPFM |
| 775 |
" [--benchmark_perf_counters=<counter>,...]\n" |
| 776 |
#endif |
| 777 |
" [--benchmark_context=<key>=<value>,...]\n" |
| 778 |
" [--benchmark_time_unit={ns|us|ms|s}]\n" |
| 779 |
" [--v=<verbosity>]\n"); |
| 780 |
} |
| 781 |
|
| 782 |
void Initialize(int* argc, char** argv, void (*HelperPrintf)()) { |
| 783 |
internal::HelperPrintf = HelperPrintf; |
| 784 |
internal::ParseCommandLineFlags(argc, argv); |
| 785 |
internal::LogLevel() = FLAGS_v; |
| 786 |
} |
| 787 |
|
| 788 |
void Shutdown() { delete internal::global_context; } |
| 789 |
|
| 790 |
bool ReportUnrecognizedArguments(int argc, char** argv) { |
| 791 |
for (int i = 1; i < argc; ++i) { |
| 792 |
fprintf(stderr, "%s: error: unrecognized command-line flag: %s\n", argv[0], |
| 793 |
argv[i]); |
| 794 |
} |
| 795 |
return argc > 1; |
| 796 |
} |
| 797 |
|
| 798 |
} // end namespace benchmark |