bench_limenka.cpp raw
1 // Copyright (c) 2015-2022 The Limenka developers
2 // Distributed under the MIT software license, see the accompanying
3 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
4
5 #include <bench/bench.h>
6 #include <common/args.h>
7 #include <crypto/sha256.h>
8 #include <tinyformat.h>
9 #include <util/fs.h>
10 #include <util/string.h>
11 #include <test/util/setup_common.h>
12
13 #include <chrono>
14 #include <cstdint>
15 #include <cstdlib>
16 #include <exception>
17 #include <iostream>
18 #include <sstream>
19 #include <vector>
20
21 using util::SplitString;
22
23 static const char* DEFAULT_BENCH_FILTER = ".*";
24 static constexpr int64_t DEFAULT_MIN_TIME_MS{10};
25 /** Priority level default value, run "all" priority levels */
26 static const std::string DEFAULT_PRIORITY{"all"};
27
28 static void SetupBenchArgs(ArgsManager& argsman)
29 {
30 SetupHelpOptions(argsman);
31 SetupCommonTestArgs(argsman);
32
33 argsman.AddArg("-asymptote=<n1,n2,n3,...>", "Test asymptotic growth of the runtime of an algorithm, if supported by the benchmark", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
34 argsman.AddArg("-filter=<regex>", strprintf("Regular expression filter to select benchmark by name (default: %s)", DEFAULT_BENCH_FILTER), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
35 argsman.AddArg("-list", "List benchmarks without executing them", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
36 argsman.AddArg("-min-time=<milliseconds>", strprintf("Minimum runtime per benchmark, in milliseconds (default: %d)", DEFAULT_MIN_TIME_MS), ArgsManager::ALLOW_ANY | ArgsManager::DISALLOW_NEGATION, OptionsCategory::OPTIONS);
37 argsman.AddArg("-output-csv=<output.csv>", "Generate CSV file with the most important benchmark results", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
38 argsman.AddArg("-output-json=<output.json>", "Generate JSON file with all benchmark results", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
39 argsman.AddArg("-sanity-check", "Run benchmarks for only one iteration with no output", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
40 argsman.AddArg("-priority-level=<l1,l2,l3>", strprintf("Run benchmarks of one or multiple priority level(s) (%s), default: '%s'",
41 benchmark::ListPriorities(), DEFAULT_PRIORITY), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
42 }
43
44 // parses a comma separated list like "10,20,30,50"
45 static std::vector<double> parseAsymptote(const std::string& str) {
46 std::stringstream ss(str);
47 std::vector<double> numbers;
48 double d;
49 char c;
50 while (ss >> d) {
51 numbers.push_back(d);
52 ss >> c;
53 }
54 return numbers;
55 }
56
57 static uint8_t parsePriorityLevel(const std::string& str) {
58 uint8_t levels{0};
59 for (const auto& level: SplitString(str, ',')) {
60 levels |= benchmark::StringToPriority(level);
61 }
62 return levels;
63 }
64
65 static std::vector<std::string> parseTestSetupArgs(const ArgsManager& argsman)
66 {
67 // Parses unit test framework arguments supported by the benchmark framework.
68 std::vector<std::string> args;
69 static std::vector<std::string> AVAILABLE_ARGS = {"-testdatadir"};
70 for (const std::string& arg_name : AVAILABLE_ARGS) {
71 auto op_arg = argsman.GetArg(arg_name);
72 if (op_arg) args.emplace_back(strprintf("%s=%s", arg_name, *op_arg));
73 }
74 return args;
75 }
76
77 int main(int argc, char** argv)
78 {
79 ArgsManager argsman;
80 SetupBenchArgs(argsman);
81 SHA256AutoDetect();
82 std::string error;
83 if (!argsman.ParseParameters(argc, argv, error)) {
84 tfm::format(std::cerr, "Error parsing command line arguments: %s\n", error);
85 return EXIT_FAILURE;
86 }
87
88 if (HelpRequested(argsman)) {
89 std::cout << "Usage: bench_limenka [options]\n"
90 "\n"
91 << argsman.GetHelpMessage()
92 << "Description:\n"
93 "\n"
94 " bench_limenka executes microbenchmarks. The quality of the benchmark results\n"
95 " highly depend on the stability of the machine. It can sometimes be difficult\n"
96 " to get stable, repeatable results, so here are a few tips:\n"
97 "\n"
98 " * Use pyperf [1] to disable frequency scaling, turbo boost etc. For best\n"
99 " results, use CPU pinning and CPU isolation (see [2]).\n"
100 "\n"
101 " * Each call of run() should do exactly the same work. E.g. inserting into\n"
102 " a std::vector doesn't do that as it will reallocate on certain calls. Make\n"
103 " sure each run has exactly the same preconditions.\n"
104 "\n"
105 " * If results are still not reliable, increase runtime with e.g.\n"
106 " -min-time=5000 to let a benchmark run for at least 5 seconds.\n"
107 "\n"
108 " * bench_limenka uses nanobench [3] for which there is extensive\n"
109 " documentation available online.\n"
110 "\n"
111 "Environment Variables:\n"
112 "\n"
113 " To attach a profiler you can run a benchmark in endless mode. This can be\n"
114 " done with the environment variable NANOBENCH_ENDLESS. E.g. like so:\n"
115 "\n"
116 " NANOBENCH_ENDLESS=MuHash ./bench_limenka -filter=MuHash\n"
117 "\n"
118 " In rare cases it can be useful to suppress stability warnings. This can be\n"
119 " done with the environment variable NANOBENCH_SUPPRESS_WARNINGS, e.g:\n"
120 "\n"
121 " NANOBENCH_SUPPRESS_WARNINGS=1 ./bench_limenka\n"
122 "\n"
123 "Notes:\n"
124 "\n"
125 " 1. pyperf\n"
126 " https://github.com/psf/pyperf\n"
127 "\n"
128 " 2. CPU pinning & isolation\n"
129 " https://pyperf.readthedocs.io/en/latest/system.html\n"
130 "\n"
131 " 3. nanobench\n"
132 " https://github.com/martinus/nanobench\n"
133 "\n";
134
135 return EXIT_SUCCESS;
136 }
137
138 try {
139 benchmark::Args args;
140 args.asymptote = parseAsymptote(argsman.GetArg("-asymptote", ""));
141 args.is_list_only = argsman.GetBoolArg("-list", false);
142 args.min_time = std::chrono::milliseconds(argsman.GetIntArg("-min-time", DEFAULT_MIN_TIME_MS));
143 args.output_csv = argsman.GetPathArg("-output-csv");
144 args.output_json = argsman.GetPathArg("-output-json");
145 args.regex_filter = argsman.GetArg("-filter", DEFAULT_BENCH_FILTER);
146 args.sanity_check = argsman.GetBoolArg("-sanity-check", false);
147 args.priority = parsePriorityLevel(argsman.GetArg("-priority-level", DEFAULT_PRIORITY));
148 args.setup_args = parseTestSetupArgs(argsman);
149
150 benchmark::BenchRunner::RunAll(args);
151
152 return EXIT_SUCCESS;
153 } catch (const std::exception& e) {
154 tfm::format(std::cerr, "Error: %s\n", e.what());
155 return EXIT_FAILURE;
156 }
157 }
158