main.rs raw
1 // Copyright (c) The Bitcoin Core developers
2 // Distributed under the MIT software license, see the accompanying
3 // file COPYING or https://opensource.org/license/mit/.
4
5 use std::collections::VecDeque;
6 use std::env;
7 use std::fs::{read_dir, DirEntry, File};
8 use std::path::{Path, PathBuf};
9 use std::process::{Command, ExitCode};
10 use std::str;
11 use std::thread;
12
13 /// A type for a complete and readable error message.
14 type AppError = String;
15 type AppResult = Result<(), AppError>;
16
17 const LLVM_PROFDATA: &str = "llvm-profdata";
18 const LLVM_COV: &str = "llvm-cov";
19 const GIT: &str = "git";
20
21 const DEFAULT_PAR: usize = 1;
22
23 fn exit_help(err: &str) -> AppError {
24 format!(
25 r#"
26 Error: {err}
27
28 Usage: program ./build_dir ./qa-assets/fuzz_corpora fuzz_target_name [parallelism={DEFAULT_PAR}]
29
30 Refer to the devtools/README.md for more details."#
31 )
32 }
33
34 fn sanity_check(corpora_dir: &Path, fuzz_exe: &Path) -> AppResult {
35 for tool in [LLVM_PROFDATA, LLVM_COV, GIT] {
36 let output = Command::new(tool).arg("--help").output();
37 match output {
38 Ok(output) if output.status.success() => {}
39 _ => Err(exit_help(&format!("The tool {tool} is not installed")))?,
40 }
41 }
42 if !corpora_dir.is_dir() {
43 Err(exit_help(&format!(
44 "Fuzz corpora path ({}) must be a directory",
45 corpora_dir.display()
46 )))?;
47 }
48 if !fuzz_exe.exists() {
49 Err(exit_help(&format!(
50 "Fuzz executable ({}) not found",
51 fuzz_exe.display()
52 )))?;
53 }
54 Ok(())
55 }
56
57 fn app() -> AppResult {
58 // Parse args
59 let args = env::args().collect::<Vec<_>>();
60 let build_dir = args.get(1).ok_or(exit_help("Must set build dir"))?;
61 if build_dir == "--help" {
62 Err(exit_help("--help requested"))?;
63 }
64 let corpora_dir = args.get(2).ok_or(exit_help("Must set fuzz corpora dir"))?;
65 let fuzz_target = args
66 .get(3)
67 // Require fuzz target for now. In the future it could be optional and the tool could
68 // iterate over all compiled fuzz targets
69 .ok_or(exit_help("Must set fuzz target"))?;
70 let par = match args.get(4) {
71 Some(s) => s
72 .parse::<usize>()
73 .map_err(|e| exit_help(&format!("Could not parse parallelism as usize ({s}): {e}")))?,
74 None => DEFAULT_PAR,
75 }
76 .max(1);
77 if args.get(5).is_some() {
78 Err(exit_help("Too many args"))?;
79 }
80
81 let build_dir = Path::new(build_dir);
82 let corpora_dir = Path::new(corpora_dir);
83 let fuzz_exe = build_dir.join("bin/fuzz");
84
85 sanity_check(corpora_dir, &fuzz_exe)?;
86
87 deterministic_coverage(build_dir, corpora_dir, &fuzz_exe, fuzz_target, par)
88 }
89
90 fn using_libfuzzer(fuzz_exe: &Path) -> Result<bool, AppError> {
91 println!("Check if using libFuzzer ...");
92 let stderr = Command::new(fuzz_exe)
93 .arg("-help=1") // Will be interpreted as option (libfuzzer) or as input file
94 .env("FUZZ", "addition_overflow") // Any valid target
95 .output()
96 .map_err(|e| format!("fuzz failed with {e}"))?
97 .stderr;
98 let help_output = str::from_utf8(&stderr)
99 .map_err(|e| format!("The libFuzzer -help=1 output must be valid text ({e})"))?;
100 Ok(help_output.contains("libFuzzer"))
101 }
102
103 fn deterministic_coverage(
104 build_dir: &Path,
105 corpora_dir: &Path,
106 fuzz_exe: &Path,
107 fuzz_target: &str,
108 par: usize,
109 ) -> AppResult {
110 let using_libfuzzer = using_libfuzzer(fuzz_exe)?;
111 if using_libfuzzer {
112 println!("Warning: The fuzz executable was compiled with libFuzzer as sanitizer.");
113 println!("This tool may be tripped by libFuzzer misbehavior.");
114 println!("It is recommended to compile without libFuzzer.");
115 }
116 let corpus_dir = corpora_dir.join(fuzz_target);
117 let mut entries = read_dir(&corpus_dir)
118 .map_err(|err| {
119 exit_help(&format!(
120 "The fuzz target's input directory must exist! ({}; {})",
121 corpus_dir.display(),
122 err
123 ))
124 })?
125 .map(|entry| entry.expect("IO error"))
126 .collect::<Vec<_>>();
127 entries.sort_by_key(|entry| entry.file_name());
128 let run_single = |run_id: char, entry: &Path, thread_id: usize| -> Result<PathBuf, AppError> {
129 let cov_txt_path = build_dir.join(format!("fuzz_det_cov.show.t{thread_id}.{run_id}.txt"));
130 let profraw_file = build_dir.join(format!("fuzz_det_cov.t{thread_id}.{run_id}.profraw"));
131 let profdata_file = build_dir.join(format!("fuzz_det_cov.t{thread_id}.{run_id}.profdata"));
132 {
133 let output = {
134 let mut cmd = Command::new(fuzz_exe);
135 if using_libfuzzer {
136 cmd.args(["-runs=1", "-shuffle=1", "-prefer_small=0"]);
137 }
138 cmd
139 }
140 .env("LLVM_PROFILE_FILE", &profraw_file)
141 .env("FUZZ", fuzz_target)
142 .arg(entry)
143 .output()
144 .map_err(|e| format!("fuzz failed: {e}"))?;
145 if !output.status.success() {
146 Err(format!(
147 "fuzz failed!\nstdout:\n{}\nstderr:\n{}\n",
148 String::from_utf8_lossy(&output.stdout),
149 String::from_utf8_lossy(&output.stderr)
150 ))?;
151 }
152 }
153 if !Command::new(LLVM_PROFDATA)
154 .arg("merge")
155 .arg("--sparse")
156 .arg(&profraw_file)
157 .arg("-o")
158 .arg(&profdata_file)
159 .status()
160 .map_err(|e| format!("{LLVM_PROFDATA} merge failed with {e}"))?
161 .success()
162 {
163 Err(format!("{LLVM_PROFDATA} merge failed. This can be a sign of compiling without code coverage support."))?;
164 }
165 let cov_file = File::create(&cov_txt_path)
166 .map_err(|e| format!("Failed to create coverage txt file ({e})"))?;
167 if !Command::new(LLVM_COV)
168 .args([
169 "show",
170 "--show-line-counts-or-regions",
171 "--show-branches=count",
172 "--show-expansions",
173 "--show-instantiation-summary",
174 "-Xdemangler=llvm-cxxfilt",
175 &format!("--instr-profile={}", profdata_file.display()),
176 ])
177 .arg(fuzz_exe)
178 .stdout(cov_file)
179 .spawn()
180 .map_err(|e| format!("{LLVM_COV} show failed with {e}"))?
181 .wait()
182 .map_err(|e| format!("{LLVM_COV} show failed with {e}"))?
183 .success()
184 {
185 Err(format!("{LLVM_COV} show failed"))?;
186 };
187 Ok(cov_txt_path)
188 };
189 let check_diff = |a: &Path, b: &Path, err: &str| -> AppResult {
190 let same = Command::new(GIT)
191 .args(["--no-pager", "diff", "--no-index"])
192 .arg(a)
193 .arg(b)
194 .status()
195 .map_err(|e| format!("{GIT} diff failed with {e}"))?
196 .success();
197 if !same {
198 Err(format!(
199 r#"
200 The coverage was not deterministic between runs.
201 {err}"#
202 ))?;
203 }
204 Ok(())
205 };
206 // First, check that each fuzz input is deterministic running by itself in a process.
207 //
208 // This can catch issues and isolate where a single fuzz input triggers non-determinism, but
209 // all other fuzz inputs are deterministic.
210 //
211 // Also, This can catch issues where several fuzz inputs are non-deterministic, but the sum of
212 // their overall coverage trace remains the same across runs and thus remains undetected.
213 println!(
214 "Check each fuzz input individually ... ({} inputs with parallelism {par})",
215 entries.len()
216 );
217 let check_individual = |entry: &DirEntry, thread_id: usize| -> AppResult {
218 let entry = entry.path();
219 if !entry.is_file() {
220 Err(format!("{} should be a file", entry.display()))?;
221 }
222 let cov_txt_base = run_single('a', &entry, thread_id)?;
223 let cov_txt_repeat = run_single('b', &entry, thread_id)?;
224 check_diff(
225 &cov_txt_base,
226 &cov_txt_repeat,
227 &format!("The fuzz target input was {}.", entry.display()),
228 )?;
229 Ok(())
230 };
231 thread::scope(|s| -> AppResult {
232 let mut handles = VecDeque::with_capacity(par);
233 let mut res = Ok(());
234 for (i, entry) in entries.iter().enumerate() {
235 println!("[{}/{}]", i + 1, entries.len());
236 handles.push_back(s.spawn(move || check_individual(entry, i % par)));
237 while handles.len() >= par || i == (entries.len() - 1) || res.is_err() {
238 if let Some(th) = handles.pop_front() {
239 let thread_result = match th.join() {
240 Err(_e) => Err("A scoped thread panicked".to_string()),
241 Ok(r) => r,
242 };
243 if thread_result.is_err() {
244 res = thread_result;
245 }
246 } else {
247 return res;
248 }
249 }
250 }
251 res
252 })?;
253 // Finally, check that running over all fuzz inputs in one process is deterministic as well.
254 // This can catch issues where mutable global state is leaked from one fuzz input execution to
255 // the next.
256 println!("Check all fuzz inputs in one go ...");
257 {
258 if !corpus_dir.is_dir() {
259 Err(format!("{} should be a folder", corpus_dir.display()))?;
260 }
261 let cov_txt_base = run_single('a', &corpus_dir, 0)?;
262 let cov_txt_repeat = run_single('b', &corpus_dir, 0)?;
263 check_diff(
264 &cov_txt_base,
265 &cov_txt_repeat,
266 &format!("All fuzz inputs in {} were used.", corpus_dir.display()),
267 )?;
268 }
269 println!("✨ Coverage test passed for {fuzz_target}. ✨");
270 Ok(())
271 }
272
273 fn main() -> ExitCode {
274 match app() {
275 Ok(()) => ExitCode::SUCCESS,
276 Err(err) => {
277 eprintln!("⚠️\n{err}");
278 ExitCode::FAILURE
279 }
280 }
281 }
282