From fa7e93113052d09cc5ce4332d1c15904341bd132 Mon Sep 17 00:00:00 2001 From: MarcoFalke <*~=`'#}+{/-|&$^_@721217.xyz> Date: Fri, 28 Mar 2025 07:51:09 +0100 Subject: [PATCH 1/5] contrib: Add optional parallelism to deterministic-fuzz-coverage Co-Authored-By: Hodlinator <172445034+hodlinator@users.noreply.github.com> --- .../deterministic-fuzz-coverage/src/main.rs | 68 +++++++++++++++---- 1 file changed, 53 insertions(+), 15 deletions(-) diff --git a/contrib/devtools/deterministic-fuzz-coverage/src/main.rs b/contrib/devtools/deterministic-fuzz-coverage/src/main.rs index 84725d2606c..c5cf06a1e0d 100644 --- a/contrib/devtools/deterministic-fuzz-coverage/src/main.rs +++ b/contrib/devtools/deterministic-fuzz-coverage/src/main.rs @@ -2,11 +2,13 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or https://opensource.org/license/mit/. +use std::collections::VecDeque; use std::env; -use std::fs::{read_dir, File}; +use std::fs::{read_dir, DirEntry, File}; use std::path::{Path, PathBuf}; use std::process::{Command, ExitCode}; use std::str; +use std::thread; /// A type for a complete and readable error message. type AppError = String; @@ -16,12 +18,14 @@ const LLVM_PROFDATA: &str = "llvm-profdata"; const LLVM_COV: &str = "llvm-cov"; const GIT: &str = "git"; +const DEFAULT_PAR: usize = 1; + fn exit_help(err: &str) -> AppError { format!( r#" Error: {err} -Usage: program ./build_dir ./qa-assets/fuzz_corpora fuzz_target_name +Usage: program ./build_dir ./qa-assets/fuzz_corpora fuzz_target_name [parallelism={DEFAULT_PAR}] Refer to the devtools/README.md for more details."# ) @@ -63,7 +67,14 @@ fn app() -> AppResult { // Require fuzz target for now. In the future it could be optional and the tool could // iterate over all compiled fuzz targets .ok_or(exit_help("Must set fuzz target"))?; - if args.get(4).is_some() { + let par = match args.get(4) { + Some(s) => s + .parse::() + .map_err(|e| exit_help(&format!("Could not parse parallelism as usize ({s}): {e}")))?, + None => DEFAULT_PAR, + } + .max(1); + if args.get(5).is_some() { Err(exit_help("Too many args"))?; } @@ -73,7 +84,7 @@ fn app() -> AppResult { sanity_check(corpora_dir, &fuzz_exe)?; - deterministic_coverage(build_dir, corpora_dir, &fuzz_exe, fuzz_target) + deterministic_coverage(build_dir, corpora_dir, &fuzz_exe, fuzz_target, par) } fn using_libfuzzer(fuzz_exe: &Path) -> Result { @@ -94,10 +105,9 @@ fn deterministic_coverage( corpora_dir: &Path, fuzz_exe: &Path, fuzz_target: &str, + par: usize, ) -> AppResult { let using_libfuzzer = using_libfuzzer(fuzz_exe)?; - let profraw_file = build_dir.join("fuzz_det_cov.profraw"); - let profdata_file = build_dir.join("fuzz_det_cov.profdata"); let corpus_dir = corpora_dir.join(fuzz_target); let mut entries = read_dir(&corpus_dir) .map_err(|err| { @@ -110,8 +120,10 @@ fn deterministic_coverage( .map(|entry| entry.expect("IO error")) .collect::>(); entries.sort_by_key(|entry| entry.file_name()); - let run_single = |run_id: u8, entry: &Path| -> Result { - let cov_txt_path = build_dir.join(format!("fuzz_det_cov.show.{run_id}.txt")); + let run_single = |run_id: u8, entry: &Path, thread_id: usize| -> Result { + let cov_txt_path = build_dir.join(format!("fuzz_det_cov.show.t{thread_id}.r{run_id}.txt")); + let profraw_file = build_dir.join(format!("fuzz_det_cov.t{thread_id}.r{run_id}.profraw")); + let profdata_file = build_dir.join(format!("fuzz_det_cov.t{thread_id}.r{run_id}.profdata")); if !{ { let mut cmd = Command::new(fuzz_exe); @@ -187,20 +199,46 @@ The coverage was not deterministic between runs. // // Also, This can catch issues where several fuzz inputs are non-deterministic, but the sum of // their overall coverage trace remains the same across runs and thus remains undetected. - println!("Check each fuzz input individually ..."); - for entry in entries { + println!( + "Check each fuzz input individually ... ({} inputs with parallelism {par})", + entries.len() + ); + let check_individual = |entry: &DirEntry, thread_id: usize| -> AppResult { let entry = entry.path(); if !entry.is_file() { Err(format!("{} should be a file", entry.display()))?; } - let cov_txt_base = run_single(0, &entry)?; - let cov_txt_repeat = run_single(1, &entry)?; + let cov_txt_base = run_single(0, &entry, thread_id)?; + let cov_txt_repeat = run_single(1, &entry, thread_id)?; check_diff( &cov_txt_base, &cov_txt_repeat, &format!("The fuzz target input was {}.", entry.display()), )?; - } + Ok(()) + }; + thread::scope(|s| -> AppResult { + let mut handles = VecDeque::with_capacity(par); + let mut res = Ok(()); + for (i, entry) in entries.iter().enumerate() { + println!("[{}/{}]", i + 1, entries.len()); + handles.push_back(s.spawn(move || check_individual(entry, i % par))); + while handles.len() >= par || i == (entries.len() - 1) || res.is_err() { + if let Some(th) = handles.pop_front() { + let thread_result = match th.join() { + Err(_e) => Err("A scoped thread panicked".to_string()), + Ok(r) => r, + }; + if thread_result.is_err() { + res = thread_result; + } + } else { + return res; + } + } + } + res + })?; // Finally, check that running over all fuzz inputs in one process is deterministic as well. // This can catch issues where mutable global state is leaked from one fuzz input execution to // the next. @@ -209,8 +247,8 @@ The coverage was not deterministic between runs. if !corpus_dir.is_dir() { Err(format!("{} should be a folder", corpus_dir.display()))?; } - let cov_txt_base = run_single(0, &corpus_dir)?; - let cov_txt_repeat = run_single(1, &corpus_dir)?; + let cov_txt_base = run_single(0, &corpus_dir, 0)?; + let cov_txt_repeat = run_single(1, &corpus_dir, 0)?; check_diff( &cov_txt_base, &cov_txt_repeat, From fa82fe2c7364226a758c4a59e1a4a62e3ac4846b Mon Sep 17 00:00:00 2001 From: MarcoFalke <*~=`'#}+{/-|&$^_@721217.xyz> Date: Fri, 28 Mar 2025 14:15:15 +0100 Subject: [PATCH 2/5] contrib: Use -Xdemangler=llvm-cxxfilt in deterministic-*-coverage This makes the result more readable. --- contrib/devtools/deterministic-fuzz-coverage/src/main.rs | 2 ++ contrib/devtools/deterministic-unittest-coverage/src/main.rs | 2 ++ 2 files changed, 4 insertions(+) diff --git a/contrib/devtools/deterministic-fuzz-coverage/src/main.rs b/contrib/devtools/deterministic-fuzz-coverage/src/main.rs index c5cf06a1e0d..df52c497639 100644 --- a/contrib/devtools/deterministic-fuzz-coverage/src/main.rs +++ b/contrib/devtools/deterministic-fuzz-coverage/src/main.rs @@ -161,6 +161,8 @@ fn deterministic_coverage( "--show-line-counts-or-regions", "--show-branches=count", "--show-expansions", + "--show-instantiation-summary", + "-Xdemangler=llvm-cxxfilt", &format!("--instr-profile={}", profdata_file.display()), ]) .arg(fuzz_exe) diff --git a/contrib/devtools/deterministic-unittest-coverage/src/main.rs b/contrib/devtools/deterministic-unittest-coverage/src/main.rs index 58dbebcce49..2da483aa7ff 100644 --- a/contrib/devtools/deterministic-unittest-coverage/src/main.rs +++ b/contrib/devtools/deterministic-unittest-coverage/src/main.rs @@ -104,6 +104,8 @@ fn deterministic_coverage(build_dir: &Path, test_exe: &Path, filter: &str) -> Ap "--show-line-counts-or-regions", "--show-branches=count", "--show-expansions", + "--show-instantiation-summary", + "-Xdemangler=llvm-cxxfilt", &format!("--instr-profile={}", profdata_file.display()), ]) .arg(test_exe) From fa900bb2dce8ef3ee11d5980f008995d66877155 Mon Sep 17 00:00:00 2001 From: MarcoFalke <*~=`'#}+{/-|&$^_@721217.xyz> Date: Mon, 31 Mar 2025 14:02:45 +0200 Subject: [PATCH 3/5] contrib: Only print fuzz output on failure This makes it humanly possible to track progress as only "[N/M]"-lines are printed as long as we succeed. Also, use char (a, b) to indicate run_id instead of u8 (0, 1). Also, use emojis to indicate final success or error. Co-Authored-By: Hodlinator <172445034+hodlinator@users.noreply.github.com> --- .../deterministic-fuzz-coverage/src/main.rs | 38 ++++++++++--------- .../src/main.rs | 10 ++--- 2 files changed, 26 insertions(+), 22 deletions(-) diff --git a/contrib/devtools/deterministic-fuzz-coverage/src/main.rs b/contrib/devtools/deterministic-fuzz-coverage/src/main.rs index df52c497639..285f6cddc93 100644 --- a/contrib/devtools/deterministic-fuzz-coverage/src/main.rs +++ b/contrib/devtools/deterministic-fuzz-coverage/src/main.rs @@ -120,12 +120,12 @@ fn deterministic_coverage( .map(|entry| entry.expect("IO error")) .collect::>(); entries.sort_by_key(|entry| entry.file_name()); - let run_single = |run_id: u8, entry: &Path, thread_id: usize| -> Result { - let cov_txt_path = build_dir.join(format!("fuzz_det_cov.show.t{thread_id}.r{run_id}.txt")); - let profraw_file = build_dir.join(format!("fuzz_det_cov.t{thread_id}.r{run_id}.profraw")); - let profdata_file = build_dir.join(format!("fuzz_det_cov.t{thread_id}.r{run_id}.profdata")); - if !{ - { + let run_single = |run_id: char, entry: &Path, thread_id: usize| -> Result { + let cov_txt_path = build_dir.join(format!("fuzz_det_cov.show.t{thread_id}.{run_id}.txt")); + let profraw_file = build_dir.join(format!("fuzz_det_cov.t{thread_id}.{run_id}.profraw")); + let profdata_file = build_dir.join(format!("fuzz_det_cov.t{thread_id}.{run_id}.profdata")); + { + let output = { let mut cmd = Command::new(fuzz_exe); if using_libfuzzer { cmd.arg("-runs=1"); @@ -135,11 +135,15 @@ fn deterministic_coverage( .env("LLVM_PROFILE_FILE", &profraw_file) .env("FUZZ", fuzz_target) .arg(entry) - .status() - .map_err(|e| format!("fuzz failed with {e}"))? - .success() - } { - Err("fuzz failed".to_string())?; + .output() + .map_err(|e| format!("fuzz failed: {e}"))?; + if !output.status.success() { + Err(format!( + "fuzz failed!\nstdout:\n{}\nstderr:\n{}\n", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ))?; + } } if !Command::new(LLVM_PROFDATA) .arg("merge") @@ -210,8 +214,8 @@ The coverage was not deterministic between runs. if !entry.is_file() { Err(format!("{} should be a file", entry.display()))?; } - let cov_txt_base = run_single(0, &entry, thread_id)?; - let cov_txt_repeat = run_single(1, &entry, thread_id)?; + let cov_txt_base = run_single('a', &entry, thread_id)?; + let cov_txt_repeat = run_single('b', &entry, thread_id)?; check_diff( &cov_txt_base, &cov_txt_repeat, @@ -249,15 +253,15 @@ The coverage was not deterministic between runs. if !corpus_dir.is_dir() { Err(format!("{} should be a folder", corpus_dir.display()))?; } - let cov_txt_base = run_single(0, &corpus_dir, 0)?; - let cov_txt_repeat = run_single(1, &corpus_dir, 0)?; + let cov_txt_base = run_single('a', &corpus_dir, 0)?; + let cov_txt_repeat = run_single('b', &corpus_dir, 0)?; check_diff( &cov_txt_base, &cov_txt_repeat, &format!("All fuzz inputs in {} were used.", corpus_dir.display()), )?; } - println!("Coverage test passed for {fuzz_target}."); + println!("✨ Coverage test passed for {fuzz_target}. ✨"); Ok(()) } @@ -265,7 +269,7 @@ fn main() -> ExitCode { match app() { Ok(()) => ExitCode::SUCCESS, Err(err) => { - eprintln!("{}", err); + eprintln!("⚠️\n{}", err); ExitCode::FAILURE } } diff --git a/contrib/devtools/deterministic-unittest-coverage/src/main.rs b/contrib/devtools/deterministic-unittest-coverage/src/main.rs index 2da483aa7ff..047c8d24edd 100644 --- a/contrib/devtools/deterministic-unittest-coverage/src/main.rs +++ b/contrib/devtools/deterministic-unittest-coverage/src/main.rs @@ -71,7 +71,7 @@ fn app() -> AppResult { fn deterministic_coverage(build_dir: &Path, test_exe: &Path, filter: &str) -> AppResult { let profraw_file = build_dir.join("test_det_cov.profraw"); let profdata_file = build_dir.join("test_det_cov.profdata"); - let run_single = |run_id: u8| -> Result { + let run_single = |run_id: char| -> Result { println!("Run with id {run_id}"); let cov_txt_path = build_dir.join(format!("test_det_cov.show.{run_id}.txt")); if !Command::new(test_exe) @@ -131,10 +131,10 @@ fn deterministic_coverage(build_dir: &Path, test_exe: &Path, filter: &str) -> Ap } Ok(()) }; - let r0 = run_single(0)?; - let r1 = run_single(1)?; + let r0 = run_single('a')?; + let r1 = run_single('b')?; check_diff(&r0, &r1)?; - println!("The coverage was deterministic across two runs."); + println!("✨ The coverage was deterministic across two runs. ✨"); Ok(()) } @@ -142,7 +142,7 @@ fn main() -> ExitCode { match app() { Ok(()) => ExitCode::SUCCESS, Err(err) => { - eprintln!("{}", err); + eprintln!("⚠️\n{}", err); ExitCode::FAILURE } } From fa17cdb191de71cdb4151408450aa9b3eaea6cb8 Mon Sep 17 00:00:00 2001 From: MarcoFalke <*~=`'#}+{/-|&$^_@721217.xyz> Date: Fri, 28 Mar 2025 14:25:34 +0100 Subject: [PATCH 4/5] test: Avoid script check worker threads while fuzzing Threads may execute their function any time after they are spawned, so coverage could be non-deterministic. Fix this, * for the script check worker threads by disabling them while fuzzing. * for the scheduler thread by waiting for it to fully start and run the service queue. --- src/test/util/setup_common.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index 5d723a2c468..87f0b31cad8 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -60,6 +60,7 @@ #include #include +#include #include #include @@ -230,6 +231,12 @@ ChainTestingSetup::ChainTestingSetup(const ChainType chainType, TestOpts opts) // Use synchronous task runner while fuzzing to avoid non-determinism G_FUZZING ? std::make_unique(std::make_unique()) : std::make_unique(std::make_unique(*m_node.scheduler)); + { + // Ensure deterministic coverage by waiting for m_service_thread to be running + std::promise promise; + m_node.scheduler->scheduleFromNow([&promise] { promise.set_value(); }, 0ms); + promise.get_future().wait(); + } } bilingual_str error{}; @@ -247,7 +254,8 @@ ChainTestingSetup::ChainTestingSetup(const ChainType chainType, TestOpts opts) .check_block_index = 1, .notifications = *m_node.notifications, .signals = m_node.validation_signals.get(), - .worker_threads_num = 2, + // Use no worker threads while fuzzing to avoid non-determinism + .worker_threads_num = G_FUZZING ? 0 : 2, }; if (opts.min_validation_cache) { chainman_opts.script_execution_cache_bytes = 0; From fa513101212327f45965092652f6497aa28362ec Mon Sep 17 00:00:00 2001 From: MarcoFalke <*~=`'#}+{/-|&$^_@721217.xyz> Date: Tue, 1 Apr 2025 11:34:56 +0200 Subject: [PATCH 5/5] contrib: Warn about using libFuzzer for coverage check --- contrib/devtools/deterministic-fuzz-coverage/src/main.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/contrib/devtools/deterministic-fuzz-coverage/src/main.rs b/contrib/devtools/deterministic-fuzz-coverage/src/main.rs index 285f6cddc93..9c1738396b5 100644 --- a/contrib/devtools/deterministic-fuzz-coverage/src/main.rs +++ b/contrib/devtools/deterministic-fuzz-coverage/src/main.rs @@ -108,6 +108,11 @@ fn deterministic_coverage( par: usize, ) -> AppResult { let using_libfuzzer = using_libfuzzer(fuzz_exe)?; + if using_libfuzzer { + println!("Warning: The fuzz executable was compiled with libFuzzer as sanitizer."); + println!("This tool may be tripped by libFuzzer misbehavior."); + println!("It is recommended to compile without libFuzzer."); + } let corpus_dir = corpora_dir.join(fuzz_target); let mut entries = read_dir(&corpus_dir) .map_err(|err| {