bench.cpp raw

   1  // Copyright (c) 2015-present The Bitcoin Core developers
   2  // Distributed under the MIT software license, see the accompanying
   3  // file COPYING or http://www.opensource.org/licenses/mit-license.php.
   4  
   5  #include <bench/bench.h>
   6  
   7  #include <test/util/setup_common.h> // IWYU pragma: keep
   8  #include <util/check.h>
   9  #include <util/fs.h>
  10  #include <util/time.h>
  11  
  12  #include <compare>
  13  #include <fstream>
  14  #include <functional>
  15  #include <iostream>
  16  #include <regex>
  17  #include <string>
  18  #include <string_view>
  19  #include <utility>
  20  #include <vector>
  21  
  22  /**
  23   * Retrieves the available test setup command line arguments that may be used
  24   * in the benchmark. They will be used only if the benchmark utilizes a
  25   * 'BasicTestingSetup' or any child of it.
  26   */
  27  static std::function<std::vector<const char*>()> g_bench_command_line_args{};
  28  const std::function<std::vector<const char*>()> G_TEST_COMMAND_LINE_ARGUMENTS = []() {
  29      return g_bench_command_line_args();
  30  };
  31  
  32  /**
  33   * Retrieve the name of the currently in-use benchmark.
  34   * This is applicable only to benchmarks that utilize the unit test
  35   * framework context setup (e.g. ones using 'MakeNoLogFileContext<TestingSetup>()').
  36   * It places the datadir of each benchmark run within their respective benchmark name.
  37   */
  38  static std::string g_running_benchmark_name;
  39  const std::function<std::string()> G_TEST_GET_FULL_NAME = []() {
  40      return g_running_benchmark_name;
  41  };
  42  
  43  namespace {
  44  
  45  void GenerateTemplateResults(const std::vector<ankerl::nanobench::Result>& benchmarkResults, const fs::path& file, const char* tpl)
  46  {
  47      if (benchmarkResults.empty() || file.empty()) {
  48          // nothing to write, bail out
  49          return;
  50      }
  51      std::ofstream fout{file.std_path()};
  52      if (fout.is_open()) {
  53          ankerl::nanobench::render(tpl, benchmarkResults, fout);
  54          std::cout << "Created " << file << std::endl;
  55      } else {
  56          std::cout << "Could not write to file " << file << std::endl;
  57      }
  58  }
  59  
  60  } // namespace
  61  
  62  namespace benchmark {
  63  
  64  BenchRunner::BenchmarkMap& BenchRunner::benchmarks()
  65  {
  66      static BenchmarkMap benchmarks_map;
  67      return benchmarks_map;
  68  }
  69  
  70  BenchRunner::BenchRunner(std::string_view name, BenchFunction func)
  71  {
  72      Assert(benchmarks().try_emplace(std::string{name}, std::move(func)).second);
  73  }
  74  
  75  void BenchRunner::RunAll(const Args& args)
  76  {
  77      std::regex reFilter(args.regex_filter);
  78      std::smatch baseMatch;
  79  
  80      if (args.sanity_check) {
  81          std::cout << "Running with -sanity-check option, output is being suppressed as benchmark results will be useless." << std::endl;
  82      }
  83  
  84      // Load inner test setup args
  85      g_bench_command_line_args = [&args]() {
  86          std::vector<const char*> ret;
  87          ret.reserve(args.setup_args.size());
  88          for (const auto& arg : args.setup_args) ret.emplace_back(arg.c_str());
  89          return ret;
  90      };
  91  
  92      std::vector<ankerl::nanobench::Result> benchmarkResults;
  93      for (const auto& [name, func] : benchmarks()) {
  94  
  95          if (!std::regex_match(name, baseMatch, reFilter)) {
  96              continue;
  97          }
  98  
  99          if (args.is_list_only) {
 100              std::cout << name << std::endl;
 101              continue;
 102          }
 103  
 104          Bench bench;
 105          if (args.sanity_check) {
 106              bench.epochs(1).epochIterations(1);
 107              bench.output(nullptr);
 108          }
 109          bench.name(name);
 110          g_running_benchmark_name = name;
 111          if (args.min_time > 0ms) {
 112              // convert to nanos before dividing to reduce rounding errors
 113              std::chrono::nanoseconds min_time_ns = args.min_time;
 114              bench.minEpochTime(min_time_ns / bench.epochs());
 115          }
 116  
 117          if (args.asymptote.empty()) {
 118              func(bench);
 119          } else {
 120              for (auto n : args.asymptote) {
 121                  bench.complexityN(n);
 122                  func(bench);
 123              }
 124              std::cout << bench.complexityBigO() << std::endl;
 125          }
 126  
 127          if (!bench.results().empty()) {
 128              benchmarkResults.push_back(bench.results().back());
 129          }
 130      }
 131  
 132      GenerateTemplateResults(benchmarkResults, args.output_csv, "# Benchmark, evals, iterations, total, min, max, median\n"
 133                                                                 "{{#result}}{{name}}, {{epochs}}, {{average(iterations)}}, {{sumProduct(iterations, elapsed)}}, {{minimum(elapsed)}}, {{maximum(elapsed)}}, {{median(elapsed)}}\n"
 134                                                                 "{{/result}}");
 135      GenerateTemplateResults(benchmarkResults, args.output_json, ankerl::nanobench::templates::json());
 136  }
 137  
 138  } // namespace benchmark
 139