test_runner.py raw
1 #!/usr/bin/env python3
2 # Copyright (c) 2019-present The Limenka developers
3 # Distributed under the MIT software license, see the accompanying
4 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
5 """Run fuzz test targets.
6 """
7
8 from concurrent.futures import ThreadPoolExecutor, as_completed
9 from pathlib import Path
10 import argparse
11 import configparser
12 import logging
13 import os
14 import platform
15 import random
16 import subprocess
17 import sys
18
19
20 def get_fuzz_env(*, target, source_dir):
21 symbolizer = os.environ.get('LLVM_SYMBOLIZER_PATH', "/usr/bin/llvm-symbolizer")
22 fuzz_env = {
23 'FUZZ': target,
24 'UBSAN_OPTIONS':
25 f'suppressions={source_dir}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1',
26 'UBSAN_SYMBOLIZER_PATH': symbolizer,
27 "ASAN_OPTIONS": "detect_leaks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1",
28 'ASAN_SYMBOLIZER_PATH': symbolizer,
29 'MSAN_SYMBOLIZER_PATH': symbolizer,
30 }
31 if platform.system() == "Windows":
32 # On Windows, `env` option must include valid `SystemRoot`.
33 fuzz_env = {**fuzz_env, 'SystemRoot': os.environ.get('SystemRoot')}
34 return fuzz_env
35
36
37 def main():
38 parser = argparse.ArgumentParser(
39 formatter_class=argparse.ArgumentDefaultsHelpFormatter,
40 description='''Run the fuzz targets with all inputs from the corpus_dir once.''',
41 )
42 parser.add_argument(
43 "-l",
44 "--loglevel",
45 dest="loglevel",
46 default="INFO",
47 help="log events at this level and higher to the console. Can be set to DEBUG, INFO, WARNING, ERROR or CRITICAL. Passing --loglevel DEBUG will output all logs to console.",
48 )
49 parser.add_argument(
50 '--valgrind',
51 action='store_true',
52 help='If true, run fuzzing binaries under the valgrind memory error detector',
53 )
54 parser.add_argument(
55 "--empty_min_time",
56 type=int,
57 help="If set, run at least this long, if the existing fuzz inputs directory is empty.",
58 )
59 parser.add_argument(
60 '-x',
61 '--exclude',
62 help="A comma-separated list of targets to exclude",
63 )
64 parser.add_argument(
65 '--par',
66 '-j',
67 type=int,
68 default=4,
69 help='How many targets to merge or execute in parallel.',
70 )
71 parser.add_argument(
72 'corpus_dir',
73 help='The corpus to run on (must contain subfolders for each fuzz target).',
74 )
75 parser.add_argument(
76 'target',
77 nargs='*',
78 help='The target(s) to run. Default is to run all targets.',
79 )
80 parser.add_argument(
81 '--m_dir',
82 action="append",
83 help="Merge inputs from these directories into the corpus_dir.",
84 )
85 parser.add_argument(
86 '-g',
87 '--generate',
88 action='store_true',
89 help='Create new corpus (or extend the existing ones) by running'
90 ' the given targets for a finite number of times. Outputs them to'
91 ' the passed corpus_dir.'
92 )
93
94 args = parser.parse_args()
95 args.corpus_dir = Path(args.corpus_dir)
96
97 # Set up logging
98 logging.basicConfig(
99 format='%(message)s',
100 level=int(args.loglevel) if args.loglevel.isdigit() else args.loglevel.upper(),
101 )
102
103 # Read config generated by configure.
104 config = configparser.ConfigParser()
105 configfile = os.path.abspath(os.path.dirname(__file__)) + "/../config.ini"
106 config.read_file(open(configfile, encoding="utf8"))
107
108 if not config["components"].getboolean("ENABLE_FUZZ_BINARY"):
109 logging.error("Must have fuzz executable built")
110 sys.exit(1)
111
112 fuzz_bin=os.getenv("LIMENKAFUZZ", default=os.path.join(config["environment"]["BUILDDIR"], 'bin', 'fuzz'))
113
114 # Build list of tests
115 test_list_all = parse_test_list(
116 fuzz_bin=fuzz_bin,
117 source_dir=config['environment']['SRCDIR'],
118 )
119
120 if not test_list_all:
121 logging.error("No fuzz targets found")
122 sys.exit(1)
123
124 logging.debug("{} fuzz target(s) found: {}".format(len(test_list_all), " ".join(sorted(test_list_all))))
125
126 args.target = args.target or test_list_all # By default run all
127 test_list_error = list(set(args.target).difference(set(test_list_all)))
128 if test_list_error:
129 logging.error("Unknown fuzz targets selected: {}".format(test_list_error))
130 test_list_selection = list(set(test_list_all).intersection(set(args.target)))
131 if not test_list_selection:
132 logging.error("No fuzz targets selected")
133 if args.exclude:
134 for excluded_target in args.exclude.split(","):
135 if excluded_target not in test_list_selection:
136 logging.error("Target \"{}\" not found in current target list.".format(excluded_target))
137 continue
138 test_list_selection.remove(excluded_target)
139 test_list_selection.sort()
140
141 logging.info("{} of {} detected fuzz target(s) selected: {}".format(len(test_list_selection), len(test_list_all), " ".join(test_list_selection)))
142
143 if not args.generate:
144 test_list_missing_corpus = []
145 for t in test_list_selection:
146 corpus_path = os.path.join(args.corpus_dir, t)
147 if not os.path.exists(corpus_path) or len(os.listdir(corpus_path)) == 0:
148 test_list_missing_corpus.append(t)
149 test_list_missing_corpus.sort()
150 if test_list_missing_corpus:
151 logging.info(
152 "Fuzzing harnesses lacking a corpus: {}".format(
153 " ".join(test_list_missing_corpus)
154 )
155 )
156 logging.info("Please consider adding a fuzz corpus at https://github.com/limenka/qa-assets")
157
158 try:
159 help_output = subprocess.run(
160 args=[
161 fuzz_bin,
162 '-help=1',
163 ],
164 env=get_fuzz_env(target=test_list_selection[0], source_dir=config['environment']['SRCDIR']),
165 timeout=20,
166 check=False,
167 stderr=subprocess.PIPE,
168 text=True,
169 ).stderr
170 using_libfuzzer = "libFuzzer" in help_output
171 if (args.generate or args.m_dir) and not using_libfuzzer:
172 logging.error("Must be built with libFuzzer")
173 sys.exit(1)
174 except subprocess.TimeoutExpired:
175 logging.error("subprocess timed out: Currently only libFuzzer is supported")
176 sys.exit(1)
177
178 with ThreadPoolExecutor(max_workers=args.par) as fuzz_pool:
179 if args.generate:
180 return generate_corpus(
181 fuzz_pool=fuzz_pool,
182 src_dir=config['environment']['SRCDIR'],
183 fuzz_bin=fuzz_bin,
184 corpus_dir=args.corpus_dir,
185 targets=test_list_selection,
186 )
187
188 if args.m_dir:
189 merge_inputs(
190 fuzz_pool=fuzz_pool,
191 corpus=args.corpus_dir,
192 test_list=test_list_selection,
193 src_dir=config['environment']['SRCDIR'],
194 fuzz_bin=fuzz_bin,
195 merge_dirs=[Path(m_dir) for m_dir in args.m_dir],
196 )
197 return
198
199 run_once(
200 fuzz_pool=fuzz_pool,
201 corpus=args.corpus_dir,
202 test_list=test_list_selection,
203 src_dir=config['environment']['SRCDIR'],
204 fuzz_bin=fuzz_bin,
205 using_libfuzzer=using_libfuzzer,
206 use_valgrind=args.valgrind,
207 empty_min_time=args.empty_min_time,
208 )
209
210
211 def transform_process_message_target(targets, src_dir):
212 """Add a target per process message, and also keep ("process_message", {}) to allow for
213 cross-pollination, or unlimited search"""
214
215 p2p_msg_target = "process_message"
216 if (p2p_msg_target, {}) in targets:
217 lines = subprocess.run(
218 ["git", "grep", "--function-context", "ALL_NET_MESSAGE_TYPES{", src_dir / "src" / "protocol.h"],
219 check=True,
220 stdout=subprocess.PIPE,
221 text=True,
222 ).stdout.splitlines()
223 lines = [l.split("::", 1)[1].split(",")[0].lower() for l in lines if l.startswith("src/protocol.h- NetMsgType::")]
224 assert len(lines)
225 targets += [(p2p_msg_target, {"LIMIT_TO_MESSAGE_TYPE": m}) for m in lines]
226 return targets
227
228
229 def transform_rpc_target(targets, src_dir):
230 """Add a target per RPC command, and also keep ("rpc", {}) to allow for cross-pollination,
231 or unlimited search"""
232
233 rpc_target = "rpc"
234 if (rpc_target, {}) in targets:
235 lines = subprocess.run(
236 ["git", "grep", "--function-context", "RPC_COMMANDS_SAFE_FOR_FUZZING{", src_dir / "src" / "test" / "fuzz" / "rpc.cpp"],
237 check=True,
238 stdout=subprocess.PIPE,
239 text=True,
240 ).stdout.splitlines()
241 lines = [l.split("\"", 1)[1].split("\"")[0] for l in lines if l.startswith("src/test/fuzz/rpc.cpp- \"")]
242 assert len(lines)
243 targets += [(rpc_target, {"LIMIT_TO_RPC_COMMAND": r}) for r in lines]
244 return targets
245
246
247 def generate_corpus(*, fuzz_pool, src_dir, fuzz_bin, corpus_dir, targets):
248 """Generates new corpus.
249
250 Run {targets} without input, and outputs the generated corpus to
251 {corpus_dir}.
252 """
253 logging.info("Generating corpus to {}".format(corpus_dir))
254 targets = [(t, {}) for t in targets] # expand to add dictionary for target-specific env variables
255 targets = transform_process_message_target(targets, Path(src_dir))
256 targets = transform_rpc_target(targets, Path(src_dir))
257
258 def job(command, t, t_env):
259 logging.debug(f"Running '{command}'")
260 logging.debug("Command '{}' output:\n'{}'\n".format(
261 command,
262 subprocess.run(
263 command,
264 env={
265 **t_env,
266 **get_fuzz_env(target=t, source_dir=src_dir),
267 },
268 check=True,
269 stderr=subprocess.PIPE,
270 text=True,
271 ).stderr,
272 ))
273
274 futures = []
275 for target, t_env in targets:
276 target_corpus_dir = corpus_dir / target
277 os.makedirs(target_corpus_dir, exist_ok=True)
278 use_value_profile = int(random.random() < .3)
279 command = [
280 fuzz_bin,
281 "-rss_limit_mb=8000",
282 "-max_total_time=6000",
283 "-reload=0",
284 f"-use_value_profile={use_value_profile}",
285 target_corpus_dir,
286 ]
287 futures.append(fuzz_pool.submit(job, command, target, t_env))
288
289 for future in as_completed(futures):
290 future.result()
291
292
293 def merge_inputs(*, fuzz_pool, corpus, test_list, src_dir, fuzz_bin, merge_dirs):
294 logging.info(f"Merge the inputs from the passed dir into the corpus_dir. Passed dirs {merge_dirs}")
295 jobs = []
296 for t in test_list:
297 args = [
298 fuzz_bin,
299 '-rss_limit_mb=8000',
300 '-set_cover_merge=1',
301 # set_cover_merge is used instead of -merge=1 to reduce the overall
302 # size of the qa-assets git repository a bit, but more importantly,
303 # to cut the runtime to iterate over all fuzz inputs [0].
304 # [0] https://github.com/limenka/qa-assets/issues/130#issuecomment-1761760866
305 '-shuffle=0',
306 '-prefer_small=1',
307 '-use_value_profile=0',
308 # use_value_profile is enabled by oss-fuzz [0], but disabled for
309 # now to avoid bloating the qa-assets git repository [1].
310 # [0] https://github.com/google/oss-fuzz/issues/1406#issuecomment-387790487
311 # [1] https://github.com/limenka/qa-assets/issues/130#issuecomment-1749075891
312 os.path.join(corpus, t),
313 ] + [str(m_dir / t) for m_dir in merge_dirs]
314 os.makedirs(os.path.join(corpus, t), exist_ok=True)
315 for m_dir in merge_dirs:
316 (m_dir / t).mkdir(exist_ok=True)
317
318 def job(t, args):
319 output = 'Run {} with args {}\n'.format(t, " ".join(args))
320 output += subprocess.run(
321 args,
322 env=get_fuzz_env(target=t, source_dir=src_dir),
323 check=True,
324 stderr=subprocess.PIPE,
325 text=True,
326 ).stderr
327 logging.debug(output)
328
329 jobs.append(fuzz_pool.submit(job, t, args))
330
331 for future in as_completed(jobs):
332 future.result()
333
334
335 def run_once(*, fuzz_pool, corpus, test_list, src_dir, fuzz_bin, using_libfuzzer, use_valgrind, empty_min_time):
336 jobs = []
337 for t in test_list:
338 corpus_path = corpus / t
339 os.makedirs(corpus_path, exist_ok=True)
340 args = [
341 fuzz_bin,
342 ]
343 empty_dir = not any(corpus_path.iterdir())
344 if using_libfuzzer:
345 if empty_min_time and empty_dir:
346 args += [f"-max_total_time={empty_min_time}"]
347 else:
348 args += [
349 "-runs=1",
350 corpus_path,
351 ]
352 else:
353 args += [corpus_path]
354 if use_valgrind:
355 args = ['valgrind', '--quiet', '--error-exitcode=1'] + args
356
357 def job(t, args):
358 output = 'Run {} with args {}'.format(t, args)
359 result = subprocess.run(
360 args,
361 env=get_fuzz_env(target=t, source_dir=src_dir),
362 stderr=subprocess.PIPE,
363 text=True,
364 )
365 output += result.stderr
366 return output, result, t
367
368 jobs.append(fuzz_pool.submit(job, t, args))
369
370 stats = []
371 for future in as_completed(jobs):
372 output, result, target = future.result()
373 logging.debug(output)
374 try:
375 result.check_returncode()
376 except subprocess.CalledProcessError as e:
377 if e.stdout:
378 logging.info(e.stdout)
379 if e.stderr:
380 logging.info(e.stderr)
381 logging.info(f"⚠️ Failure generated from target with exit code {e.returncode}: {result.args}")
382 sys.exit(1)
383 if using_libfuzzer:
384 done_stat = [l for l in output.splitlines() if "DONE" in l]
385 assert len(done_stat) == 1
386 stats.append((target, done_stat[0]))
387
388 if using_libfuzzer:
389 print("Summary:")
390 max_len = max(len(t[0]) for t in stats)
391 for t, s in sorted(stats):
392 t = t.ljust(max_len + 1)
393 print(f"{t}{s}")
394
395
396 def parse_test_list(*, fuzz_bin, source_dir):
397 test_list_all = subprocess.run(
398 fuzz_bin,
399 env={
400 'PRINT_ALL_FUZZ_TARGETS_AND_ABORT': '',
401 **get_fuzz_env(target="", source_dir=source_dir)
402 },
403 stdout=subprocess.PIPE,
404 text=True,
405 check=True,
406 ).stdout.splitlines()
407 return test_list_all
408
409
410 if __name__ == '__main__':
411 main()
412