main.rs raw
1 // Copyright (c) The Limenka developers
2 // Distributed under the MIT software license, see the accompanying
3 // file COPYING or https://opensource.org/license/mit/.
4
5 use std::env;
6 use std::fs::{read_dir, File};
7 use std::path::Path;
8 use std::process::{exit, Command, Stdio};
9 use std::str;
10
11 const LLVM_PROFDATA: &str = "llvm-profdata";
12 const LLVM_COV: &str = "llvm-cov";
13 const DIFF: &str = "diff";
14
15 fn exit_help(err: &str) -> ! {
16 eprintln!("Error: {}", err);
17 eprintln!();
18 eprintln!("Usage: program ./build_dir ./qa-assets-corpora-dir fuzz_target");
19 eprintln!();
20 eprintln!("Refer to the devtools/README.md for more details.");
21 exit(1)
22 }
23
24 fn sanity_check(corpora_dir: &Path, fuzz_exe: &Path) {
25 for tool in [LLVM_PROFDATA, LLVM_COV, DIFF] {
26 let output = Command::new(tool).arg("--version").output();
27 match output {
28 Ok(output) if output.status.success() => {}
29 _ => {
30 exit_help(&format!("The tool {} is not installed", tool));
31 }
32 }
33 }
34 if !corpora_dir.is_dir() {
35 exit_help(&format!(
36 "Fuzz corpora path ({}) must be a directory",
37 corpora_dir.display()
38 ));
39 }
40 if !fuzz_exe.exists() {
41 exit_help(&format!(
42 "Fuzz executable ({}) not found",
43 fuzz_exe.display()
44 ));
45 }
46 }
47
48 fn main() {
49 // Parse args
50 let args = env::args().collect::<Vec<_>>();
51 let build_dir = args
52 .get(1)
53 .unwrap_or_else(|| exit_help("Must set build dir"));
54 if build_dir == "--help" {
55 exit_help("--help requested")
56 }
57 let corpora_dir = args
58 .get(2)
59 .unwrap_or_else(|| exit_help("Must set fuzz corpora dir"));
60 let fuzz_target = args
61 .get(3)
62 // Require fuzz target for now. In the future it could be optional and the tool could
63 // iterate over all compiled fuzz targets
64 .unwrap_or_else(|| exit_help("Must set fuzz target"));
65 if args.get(4).is_some() {
66 exit_help("Too many args")
67 }
68
69 let build_dir = Path::new(build_dir);
70 let corpora_dir = Path::new(corpora_dir);
71 let fuzz_exe = build_dir.join("src/test/fuzz/fuzz");
72
73 sanity_check(corpora_dir, &fuzz_exe);
74
75 deterministic_coverage(build_dir, corpora_dir, &fuzz_exe, fuzz_target);
76 }
77
78 fn using_libfuzzer(fuzz_exe: &Path) -> bool {
79 println!("Check if using libFuzzer ...");
80 let stderr = Command::new(fuzz_exe)
81 .arg("-help=1") // Will be interpreted as option (libfuzzer) or as input file
82 .env("FUZZ", "addition_overflow") // Any valid target
83 .output()
84 .expect("fuzz failed")
85 .stderr;
86 let help_output = str::from_utf8(&stderr).expect("The -help=1 output must be valid text");
87 help_output.contains("libFuzzer")
88 }
89
90 fn deterministic_coverage(
91 build_dir: &Path,
92 corpora_dir: &Path,
93 fuzz_exe: &Path,
94 fuzz_target: &str,
95 ) {
96 let using_libfuzzer = using_libfuzzer(fuzz_exe);
97 let profraw_file = build_dir.join("fuzz_det_cov.profraw");
98 let profdata_file = build_dir.join("fuzz_det_cov.profdata");
99 let corpus_dir = corpora_dir.join(fuzz_target);
100 let mut entries = read_dir(&corpus_dir)
101 .unwrap_or_else(|err| {
102 exit_help(&format!(
103 "The fuzz target's input directory must exist! ({}; {})",
104 corpus_dir.display(),
105 err
106 ))
107 })
108 .map(|entry| entry.expect("IO error"))
109 .collect::<Vec<_>>();
110 entries.sort_by_key(|entry| entry.file_name());
111 let run_single = |run_id: u8, entry: &Path| {
112 let cov_txt_path = build_dir.join(format!("fuzz_det_cov.show.{run_id}.txt"));
113 assert!({
114 {
115 let mut cmd = Command::new(fuzz_exe);
116 if using_libfuzzer {
117 cmd.arg("-runs=1");
118 }
119 cmd
120 }
121 .env("LLVM_PROFILE_FILE", &profraw_file)
122 .env("FUZZ", fuzz_target)
123 .arg(entry)
124 .status()
125 .expect("fuzz failed")
126 .success()
127 });
128 assert!(Command::new(LLVM_PROFDATA)
129 .arg("merge")
130 .arg("--sparse")
131 .arg(&profraw_file)
132 .arg("-o")
133 .arg(&profdata_file)
134 .status()
135 .expect("merge failed")
136 .success());
137 let cov_file = File::create(&cov_txt_path).expect("Failed to create coverage txt file");
138 let passed = Command::new(LLVM_COV)
139 .args([
140 "show",
141 "--show-line-counts-or-regions",
142 "--show-branches=count",
143 "--show-expansions",
144 &format!("--instr-profile={}", profdata_file.display()),
145 ])
146 .arg(fuzz_exe)
147 .stdout(Stdio::from(cov_file))
148 .spawn()
149 .expect("Failed to execute llvm-cov")
150 .wait()
151 .expect("Failed to execute llvm-cov")
152 .success();
153 if !passed {
154 panic!("Failed to execute llvm-profdata")
155 }
156 cov_txt_path
157 };
158 let check_diff = |a: &Path, b: &Path, err: &str| {
159 let same = Command::new(DIFF)
160 .arg("--unified")
161 .arg(a)
162 .arg(b)
163 .status()
164 .expect("Failed to execute diff command")
165 .success();
166 if !same {
167 eprintln!();
168 eprintln!("The coverage was not determinstic between runs.");
169 eprintln!("{}", err);
170 eprintln!("Exiting.");
171 exit(1);
172 }
173 };
174 // First, check that each fuzz input is determinisic running by itself in a process.
175 //
176 // This can catch issues and isolate where a single fuzz input triggers non-determinism, but
177 // all other fuzz inputs are deterministic.
178 //
179 // Also, This can catch issues where several fuzz inputs are non-deterministic, but the sum of
180 // their overall coverage trace remains the same across runs and thus remains undetected.
181 for entry in entries {
182 let entry = entry.path();
183 assert!(entry.is_file());
184 let cov_txt_base = run_single(0, &entry);
185 let cov_txt_repeat = run_single(1, &entry);
186 check_diff(
187 &cov_txt_base,
188 &cov_txt_repeat,
189 &format!("The fuzz target input was {}.", entry.display()),
190 );
191 }
192 // Finally, check that running over all fuzz inputs in one process is deterministic as well.
193 // This can catch issues where mutable global state is leaked from one fuzz input execution to
194 // the next.
195 {
196 assert!(corpus_dir.is_dir());
197 let cov_txt_base = run_single(0, &corpus_dir);
198 let cov_txt_repeat = run_single(1, &corpus_dir);
199 check_diff(
200 &cov_txt_base,
201 &cov_txt_repeat,
202 &format!("All fuzz inputs in {} were used.", corpus_dir.display()),
203 );
204 }
205 println!("Coverage test passed for {fuzz_target}.");
206 }
207