From 2ee2a0d3a890394be4d996b83d71d27e93becbc7 Mon Sep 17 00:00:00 2001 From: Guilherme Bernardo Date: Sat, 24 Oct 2026 16:21:40 -0400 Subject: [PATCH] fix(testcomp): the smoke test requires evidence that each engine ran Review of #86: the "Executing AFL++/Klee with map2check" banners are logged before the engine is launched, so they also appear when it fails to start, and the other engine can still produce the suite the smoke test asks for. With afl-fuzz unable to start, KLEE found Cover-Error's 42 and the check passed. The frontend now logs, after each engine, what the engine itself wrote: - "AFL++ ran: N executions, M inputs in the corpus", from execs_done and corpus_count in afl-fuzz's fuzzer_stats; with no executions, a warning carrying the last lines of fuzzer.output, which the scratch cleanup would otherwise remove; - "KLEE ran: N tests generated", from the .ktest files in klee-last. smoketest.sh requires those lines with a positive count instead of the banners: AFL++ in both examples, KLEE in Cover-Branches. Checked on the archive rebuilt from this branch: the smoke test passes on ubuntu:24.04 and 26.04 with python3 and gcc-multilib, and fails with only python3, with an afl-fuzz that exits at once ("AFL++ did not run", its output in the log) and with a klee that exits at once ("KLEE did not run"). Unit tests: 14/14. Co-Authored-By: Claude Opus 5.5 (1M context) --- modules/frontend/caller.cpp | 77 +++++++++++++++++++++++++++++ utils/testcomp-archive/README.txt | 4 ++ utils/testcomp-archive/smoketest.sh | 11 +++-- 3 files changed, 88 insertions(+), 4 deletions(-) diff --git a/modules/frontend/caller.cpp b/modules/frontend/caller.cpp index 8485e0682..bc2bd7d3c 100644 --- a/modules/frontend/caller.cpp +++ b/modules/frontend/caller.cpp @@ -198,6 +198,53 @@ long long kleeCoveredInstructions(const std::string &statsPath) { return -1; #endif } + +/** One field of afl-fuzz's fuzzer_stats ("name : value" lines), which it + * rewrites while fuzzing and once more on exit; -1 when the file or the field + * is missing, as when afl-fuzz exited before fuzzing anything. */ +long long aflFuzzerStat(const std::string &statsPath, const std::string &name) { + std::ifstream stats(statsPath); + std::string line; + while (std::getline(stats, line)) { + const size_t colon = line.find(':'); + if (colon == std::string::npos) continue; + std::string field = line.substr(0, colon); + field.erase(field.find_last_not_of(' ') + 1); + if (field != name) continue; + try { + return std::stoll(line.substr(colon + 1)); + } catch (const std::exception &) { + return -1; + } + } + return -1; +} + +/** The .ktest files KLEE wrote, one per path it terminated. */ +size_t kleeTestCount(const std::string &outputDir) { + size_t count = 0; + std::error_code error; + for (const auto &entry : + std::filesystem::directory_iterator(outputDir, error)) { + if (entry.path().extension() == ".ktest") ++count; + } + return count; +} + +/** The last `count` lines of a file. For the log: the scratch directory that + * holds an engine's own output is removed at the end of the run. */ +std::string lastLines(const std::string &path, size_t count) { + std::ifstream file(path); + std::vector lines; + std::string line; + while (std::getline(file, line)) { + lines.push_back(line); + if (lines.size() > count) lines.erase(lines.begin()); + } + std::string tail; + for (const std::string &kept : lines) tail += " " + kept + "\n"; + return tail.empty() ? " (no output)\n" : tail; +} } // namespace int Caller::runKleeWatched(const std::string &command) { @@ -1446,6 +1493,16 @@ void Caller::executeAnalysis(std::string solvername) { exportKleeVectorsAsSeeds(); } Map2Check::Log::Warning("Exited klee with " + std::to_string(result)); + // "Executing Klee" is printed before KLEE starts, so it does not say + // that KLEE ran. The tests it wrote do, and this line is what the + // archive's smoke test requires. + const size_t kleeTests = kleeTestCount(Map2Check::kleeOutputDir); + if (kleeTests > 0) { + Map2Check::Log::Info("KLEE ran: " + std::to_string(kleeTests) + + " tests generated"); + } else { + Map2Check::Log::Warning("KLEE generated no tests"); + } // KLEE found nothing: its vectors, run on natively. Not for // Cover-Branches, whose goal is no violation. if (map2checkMode != Map2CheckMode::COVER_BRANCHES_MODE && @@ -1577,6 +1634,26 @@ void Caller::executeAnalysis(std::string solvername) { // its findings live under afl-out/default/, not afl-out/. const std::string aflFindings = "afl-out/default"; + // As for KLEE, "Executing AFL++" precedes the launch. The execution + // count afl-fuzz recorded is the evidence that it fuzzed, and this line + // is what the archive's smoke test requires. When there is none, what + // afl-fuzz said goes to the log: fuzzer.output is removed with the + // scratch directory. + const long long aflExecs = + aflFuzzerStat(aflFindings + "/fuzzer_stats", "execs_done"); + if (aflExecs > 0) { + const long long corpus = + aflFuzzerStat(aflFindings + "/fuzzer_stats", "corpus_count"); + Map2Check::Log::Info("AFL++ ran: " + std::to_string(aflExecs) + + " executions, " + + std::to_string(std::max(0LL, corpus)) + + " inputs in the corpus"); + } else { + Map2Check::Log::Warning( + "AFL++ recorded no executions; its output ended with:\n" + + lastLines("fuzzer.output", 15)); + } + // Replay crashes with the witness binary to confirm a real violation. // Standalone, the persistent binary reads its input from stdin (see // NonDetGeneratorAFL.c), so the crash file is redirected in; the names diff --git a/utils/testcomp-archive/README.txt b/utils/testcomp-archive/README.txt index a6099f321..e4d6976d3 100644 --- a/utils/testcomp-archive/README.txt +++ b/utils/testcomp-archive/README.txt @@ -45,6 +45,10 @@ runs until the budget ends or KLEE has explored every path, and writes at most 50 test cases: up to half from the fuzzer's corpus (the inputs that found new edges), the rest one per path KLEE explored. +The log records what each engine did, from its own output after the run: +"AFL++ ran: N executions, M inputs in the corpus" and "KLEE ran: N tests +generated". ./smoketest.sh requires both lines. + Bundled components ------------------ Map2Check GPL-2.0 LICENSE diff --git a/utils/testcomp-archive/smoketest.sh b/utils/testcomp-archive/smoketest.sh index 6db3e0170..f074d95e1 100755 --- a/utils/testcomp-archive/smoketest.sh +++ b/utils/testcomp-archive/smoketest.sh @@ -44,12 +44,15 @@ fail() { # exit 1 } -# The run log must show that ran: map2check prints this line only -# after the engine's binary (AFL++) or bitcode (KLEE) was built. +# The run log must show that ran. Not the "Executing ..." banners: +# map2check prints those before launching the engine, so they appear when it +# fails to start too, and the other engine can still produce the suite (review +# of #86). These lines are printed after the run, from what the engine itself +# wrote: afl-fuzz's execution count in fuzzer_stats, KLEE's .ktest files. ran() { # case "$3" in - AFL++) line="Executing AFL++ with map2check" ;; - KLEE) line="Executing Klee with map2check" ;; + AFL++) line="AFL++ ran: [1-9][0-9]* executions" ;; + KLEE) line="KLEE ran: [1-9][0-9]* tests generated" ;; esac grep -q "$line" "$1/run.log" || fail "$1" "$2: $3 did not run" }