test_runner.py raw
1 #!/usr/bin/env python3
2 # Copyright (c) 2019-present The Bitcoin Core developers
3 # Distributed under the MIT software license, see the accompanying
4 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
5 """Run fuzz test targets.
6 """
7
8 from concurrent.futures import ThreadPoolExecutor, as_completed
9 from pathlib import Path
10 import argparse
11 import configparser
12 import logging
13 import os
14 import random
15 import subprocess
16 import sys
17
18
19 def get_fuzz_env(*, target, source_dir):
20 symbolizer = os.environ.get('LLVM_SYMBOLIZER_PATH', "/usr/bin/llvm-symbolizer")
21 fuzz_env = os.environ | {
22 'FUZZ': target,
23 'UBSAN_OPTIONS':
24 f'suppressions={source_dir}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1',
25 'UBSAN_SYMBOLIZER_PATH': symbolizer,
26 "ASAN_OPTIONS": "detect_leaks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1",
27 'ASAN_SYMBOLIZER_PATH': symbolizer,
28 'MSAN_SYMBOLIZER_PATH': symbolizer,
29 }
30 return fuzz_env
31
32
33 def main():
34 parser = argparse.ArgumentParser(
35 formatter_class=argparse.ArgumentDefaultsHelpFormatter,
36 description='''Run the fuzz targets with all inputs from the corpus_dir once.''',
37 )
38 parser.add_argument(
39 "-l",
40 "--loglevel",
41 dest="loglevel",
42 default="INFO",
43 help="log events at this level and higher to the console. Can be set to DEBUG, INFO, WARNING, ERROR or CRITICAL. Passing --loglevel DEBUG will output all logs to console.",
44 )
45 parser.add_argument(
46 '--valgrind',
47 action='store_true',
48 help='If true, run fuzzing binaries under the valgrind memory error detector',
49 )
50 parser.add_argument(
51 "--empty_min_time",
52 type=int,
53 help="If set, run at least this long, if the existing fuzz inputs directory is empty.",
54 )
55 parser.add_argument(
56 '-x',
57 '--exclude',
58 help="A comma-separated list of targets to exclude",
59 )
60 parser.add_argument(
61 '--par',
62 '-j',
63 type=int,
64 default=4,
65 help='How many targets to merge or execute in parallel.',
66 )
67 parser.add_argument(
68 'corpus_dir',
69 help='The corpus to run on (must contain subfolders for each fuzz target).',
70 )
71 parser.add_argument(
72 'target',
73 nargs='*',
74 help='The target(s) to run. Default is to run all targets.',
75 )
76 parser.add_argument(
77 '--m_dir',
78 action="append",
79 help="Merge inputs from these directories into the corpus_dir.",
80 )
81 parser.add_argument(
82 '-g',
83 '--generate',
84 action='store_true',
85 help='Create new corpus (or extend the existing ones) by running'
86 ' the given targets for a finite number of times. Outputs them to'
87 ' the passed corpus_dir.'
88 )
89
90 args = parser.parse_args()
91 args.corpus_dir = Path(args.corpus_dir)
92
93 # Set up logging
94 logging.basicConfig(
95 format='%(message)s',
96 level=int(args.loglevel) if args.loglevel.isdigit() else args.loglevel.upper(),
97 )
98
99 # Read config generated by configure.
100 config = configparser.ConfigParser()
101 configfile = os.path.abspath(os.path.dirname(__file__)) + "/../config.ini"
102 config.read_file(open(configfile))
103
104 if not config["components"].getboolean("ENABLE_FUZZ_BINARY"):
105 logging.error("Must have fuzz executable built")
106 sys.exit(1)
107
108 fuzz_bin=os.getenv("BITCOINFUZZ", default=os.path.join(config["environment"]["BUILDDIR"], 'bin', 'fuzz'))
109
110 # Build list of tests
111 test_list_all = parse_test_list(
112 fuzz_bin=fuzz_bin,
113 source_dir=config['environment']['SRCDIR'],
114 )
115
116 if not test_list_all:
117 logging.error("No fuzz targets found")
118 sys.exit(1)
119
120 logging.debug("{} fuzz target(s) found: {}".format(len(test_list_all), " ".join(sorted(test_list_all))))
121
122 args.target = args.target or test_list_all # By default run all
123 test_list_error = list(set(args.target).difference(set(test_list_all)))
124 if test_list_error:
125 logging.error("Unknown fuzz targets selected: {}".format(test_list_error))
126 test_list_selection = list(set(test_list_all).intersection(set(args.target)))
127 if not test_list_selection:
128 logging.error("No fuzz targets selected")
129 if args.exclude:
130 for excluded_target in args.exclude.split(","):
131 if excluded_target not in test_list_selection:
132 logging.error("Target \"{}\" not found in current target list.".format(excluded_target))
133 continue
134 test_list_selection.remove(excluded_target)
135 test_list_selection.sort()
136
137 logging.info("{} of {} detected fuzz target(s) selected: {}".format(len(test_list_selection), len(test_list_all), " ".join(test_list_selection)))
138
139 if not args.generate:
140 test_list_missing_corpus = []
141 for t in test_list_selection:
142 corpus_path = os.path.join(args.corpus_dir, t)
143 if not os.path.exists(corpus_path) or len(os.listdir(corpus_path)) == 0:
144 test_list_missing_corpus.append(t)
145 test_list_missing_corpus.sort()
146 if test_list_missing_corpus:
147 logging.info(
148 "Fuzzing harnesses lacking a corpus: {}".format(
149 " ".join(test_list_missing_corpus)
150 )
151 )
152 logging.info("Please consider adding a fuzz corpus at https://github.com/bitcoin-core/qa-assets")
153
154 print("Check if using libFuzzer ... ", end='')
155 help_output = subprocess.run(
156 args=[
157 fuzz_bin,
158 '-help=1',
159 ],
160 env=get_fuzz_env(target=test_list_selection[0], source_dir=config['environment']['SRCDIR']),
161 check=False,
162 stderr=subprocess.PIPE,
163 text=True,
164 ).stderr
165 using_libfuzzer = "libFuzzer" in help_output
166 print(using_libfuzzer)
167 if (args.generate or args.m_dir) and not using_libfuzzer:
168 logging.error("Must be built with libFuzzer")
169 sys.exit(1)
170
171 with ThreadPoolExecutor(max_workers=args.par) as fuzz_pool:
172 if args.generate:
173 return generate_corpus(
174 fuzz_pool=fuzz_pool,
175 src_dir=config['environment']['SRCDIR'],
176 fuzz_bin=fuzz_bin,
177 corpus_dir=args.corpus_dir,
178 targets=test_list_selection,
179 )
180
181 if args.m_dir:
182 merge_inputs(
183 fuzz_pool=fuzz_pool,
184 corpus=args.corpus_dir,
185 test_list=test_list_selection,
186 src_dir=config['environment']['SRCDIR'],
187 fuzz_bin=fuzz_bin,
188 merge_dirs=[Path(m_dir) for m_dir in args.m_dir],
189 )
190 return
191
192 run_once(
193 fuzz_pool=fuzz_pool,
194 corpus=args.corpus_dir,
195 test_list=test_list_selection,
196 src_dir=config['environment']['SRCDIR'],
197 fuzz_bin=fuzz_bin,
198 using_libfuzzer=using_libfuzzer,
199 use_valgrind=args.valgrind,
200 empty_min_time=args.empty_min_time,
201 )
202
203
204 def transform_process_message_target(targets, src_dir):
205 """Add a target per process message, and also keep ("process_message", {}) to allow for
206 cross-pollination, or unlimited search"""
207
208 p2p_msg_target = "process_message"
209 if (p2p_msg_target, {}) in targets:
210 lines = subprocess.run(
211 ["git", "grep", "--function-context", "ALL_NET_MESSAGE_TYPES{", "src/protocol.h"],
212 check=True,
213 stdout=subprocess.PIPE,
214 text=True,
215 cwd=src_dir,
216 ).stdout.splitlines()
217 lines = [l.split("::", 1)[1].split(",")[0].lower() for l in lines if l.startswith("src/protocol.h- NetMsgType::")]
218 assert len(lines)
219 targets += [(p2p_msg_target, {"LIMIT_TO_MESSAGE_TYPE": m}) for m in lines]
220 return targets
221
222
223 def transform_rpc_target(targets, src_dir):
224 """Add a target per RPC command, and also keep ("rpc", {}) to allow for cross-pollination,
225 or unlimited search"""
226
227 rpc_target = "rpc"
228 if (rpc_target, {}) in targets:
229 lines = subprocess.run(
230 ["git", "grep", "--function-context", "RPC_COMMANDS_SAFE_FOR_FUZZING{", "src/test/fuzz/rpc.cpp"],
231 check=True,
232 stdout=subprocess.PIPE,
233 text=True,
234 cwd=src_dir,
235 ).stdout.splitlines()
236 lines = [l.split("\"", 1)[1].split("\"")[0] for l in lines if l.startswith("src/test/fuzz/rpc.cpp- \"")]
237 assert len(lines)
238 targets += [(rpc_target, {"LIMIT_TO_RPC_COMMAND": r}) for r in lines]
239 return targets
240
241
242 def generate_corpus(*, fuzz_pool, src_dir, fuzz_bin, corpus_dir, targets):
243 """Generates new corpus.
244
245 Run {targets} without input, and outputs the generated corpus to
246 {corpus_dir}.
247 """
248 logging.info("Generating corpus to {}".format(corpus_dir))
249 targets = [(t, {}) for t in targets] # expand to add dictionary for target-specific env variables
250 targets = transform_process_message_target(targets, Path(src_dir))
251 targets = transform_rpc_target(targets, Path(src_dir))
252
253 def job(command, t, t_env):
254 logging.debug(f"Running '{command}'")
255 logging.debug("Command '{}' output:\n'{}'\n".format(
256 command,
257 subprocess.run(
258 command,
259 env={
260 **t_env,
261 **get_fuzz_env(target=t, source_dir=src_dir),
262 },
263 check=True,
264 stderr=subprocess.PIPE,
265 text=True,
266 ).stderr,
267 ))
268
269 futures = []
270 for target, t_env in targets:
271 target_corpus_dir = corpus_dir / target
272 os.makedirs(target_corpus_dir, exist_ok=True)
273 use_value_profile = int(random.random() < .3)
274 command = [
275 fuzz_bin,
276 "-rss_limit_mb=8000",
277 "-max_total_time=6000",
278 "-reload=0",
279 f"-use_value_profile={use_value_profile}",
280 target_corpus_dir,
281 ]
282 futures.append(fuzz_pool.submit(job, command, target, t_env))
283
284 for future in as_completed(futures):
285 future.result()
286
287
288 def merge_inputs(*, fuzz_pool, corpus, test_list, src_dir, fuzz_bin, merge_dirs):
289 logging.info(f"Merge the inputs from the passed dir into the corpus_dir. Passed dirs {merge_dirs}")
290 jobs = []
291 for t in test_list:
292 args = [
293 fuzz_bin,
294 '-rss_limit_mb=8000',
295 '-set_cover_merge=1',
296 # set_cover_merge is used instead of -merge=1 to reduce the overall
297 # size of the qa-assets git repository a bit, but more importantly,
298 # to cut the runtime to iterate over all fuzz inputs [0].
299 # [0] https://github.com/bitcoin-core/qa-assets/issues/130#issuecomment-1761760866
300 '-shuffle=0',
301 '-prefer_small=1',
302 '-use_value_profile=0',
303 # use_value_profile is enabled by oss-fuzz [0], but disabled for
304 # now to avoid bloating the qa-assets git repository [1].
305 # [0] https://github.com/google/oss-fuzz/issues/1406#issuecomment-387790487
306 # [1] https://github.com/bitcoin-core/qa-assets/issues/130#issuecomment-1749075891
307 os.path.join(corpus, t),
308 ] + [str(m_dir / t) for m_dir in merge_dirs]
309 os.makedirs(os.path.join(corpus, t), exist_ok=True)
310 for m_dir in merge_dirs:
311 (m_dir / t).mkdir(exist_ok=True)
312
313 def job(t, args):
314 output = 'Run {} with args {}\n'.format(t, " ".join(args))
315 output += subprocess.run(
316 args,
317 env=get_fuzz_env(target=t, source_dir=src_dir),
318 check=True,
319 stderr=subprocess.PIPE,
320 text=True,
321 ).stderr
322 logging.debug(output)
323
324 jobs.append(fuzz_pool.submit(job, t, args))
325
326 for future in as_completed(jobs):
327 future.result()
328
329
330 def run_once(*, fuzz_pool, corpus, test_list, src_dir, fuzz_bin, using_libfuzzer, use_valgrind, empty_min_time):
331 jobs = []
332 for t in test_list:
333 corpus_path = corpus / t
334 os.makedirs(corpus_path, exist_ok=True)
335 args = [
336 fuzz_bin,
337 ]
338 empty_dir = not any(corpus_path.iterdir())
339 if using_libfuzzer:
340 if empty_min_time and empty_dir:
341 args += [f"-max_total_time={empty_min_time}"]
342 else:
343 args += [
344 "-runs=1",
345 corpus_path,
346 ]
347 else:
348 args += [corpus_path]
349 if use_valgrind:
350 args = ['valgrind', '--quiet', '--error-exitcode=1'] + args
351
352 def job(t, args):
353 output = 'Run {} with args {}'.format(t, args)
354 result = subprocess.run(
355 args,
356 env=get_fuzz_env(target=t, source_dir=src_dir),
357 stderr=subprocess.PIPE,
358 text=True,
359 )
360 output += result.stderr
361 return output, result, t
362
363 jobs.append(fuzz_pool.submit(job, t, args))
364
365 stats = []
366 for future in as_completed(jobs):
367 output, result, target = future.result()
368 logging.debug(output)
369 try:
370 result.check_returncode()
371 except subprocess.CalledProcessError as e:
372 if e.stdout:
373 logging.info(e.stdout)
374 if e.stderr:
375 logging.info(e.stderr)
376 logging.info(f"⚠️ Failure generated from target with exit code {e.returncode}: {result.args}")
377 sys.exit(1)
378 if using_libfuzzer:
379 done_stat = [l for l in output.splitlines() if "DONE" in l]
380 assert len(done_stat) == 1
381 stats.append((target, done_stat[0]))
382
383 if using_libfuzzer:
384 print("Summary:")
385 max_len = max(len(t[0]) for t in stats)
386 for t, s in sorted(stats):
387 t = t.ljust(max_len + 1)
388 print(f"{t}{s}")
389
390
391 def parse_test_list(*, fuzz_bin, source_dir):
392 test_list_all = subprocess.run(
393 fuzz_bin,
394 env={
395 'PRINT_ALL_FUZZ_TARGETS_AND_ABORT': '',
396 **get_fuzz_env(target="", source_dir=source_dir)
397 },
398 stdout=subprocess.PIPE,
399 text=True,
400 check=True,
401 ).stdout.splitlines()
402 return test_list_all
403
404
405 if __name__ == '__main__':
406 main()
407