diff --git a/modules/frontend/caller.cpp b/modules/frontend/caller.cpp index 8485e0682..bc2bd7d3c 100644 --- a/modules/frontend/caller.cpp +++ b/modules/frontend/caller.cpp @@ -198,6 +198,53 @@ long long kleeCoveredInstructions(const std::string &statsPath) { return -1; #endif } + +/** One field of afl-fuzz's fuzzer_stats ("name : value" lines), which it + * rewrites while fuzzing and once more on exit; -1 when the file or the field + * is missing, as when afl-fuzz exited before fuzzing anything. */ +long long aflFuzzerStat(const std::string &statsPath, const std::string &name) { + std::ifstream stats(statsPath); + std::string line; + while (std::getline(stats, line)) { + const size_t colon = line.find(':'); + if (colon == std::string::npos) continue; + std::string field = line.substr(0, colon); + field.erase(field.find_last_not_of(' ') + 1); + if (field != name) continue; + try { + return std::stoll(line.substr(colon + 1)); + } catch (const std::exception &) { + return -1; + } + } + return -1; +} + +/** The .ktest files KLEE wrote, one per path it terminated. */ +size_t kleeTestCount(const std::string &outputDir) { + size_t count = 0; + std::error_code error; + for (const auto &entry : + std::filesystem::directory_iterator(outputDir, error)) { + if (entry.path().extension() == ".ktest") ++count; + } + return count; +} + +/** The last `count` lines of a file. For the log: the scratch directory that + * holds an engine's own output is removed at the end of the run. */ +std::string lastLines(const std::string &path, size_t count) { + std::ifstream file(path); + std::vector lines; + std::string line; + while (std::getline(file, line)) { + lines.push_back(line); + if (lines.size() > count) lines.erase(lines.begin()); + } + std::string tail; + for (const std::string &kept : lines) tail += " " + kept + "\n"; + return tail.empty() ? " (no output)\n" : tail; +} } // namespace int Caller::runKleeWatched(const std::string &command) { @@ -1446,6 +1493,16 @@ void Caller::executeAnalysis(std::string solvername) { exportKleeVectorsAsSeeds(); } Map2Check::Log::Warning("Exited klee with " + std::to_string(result)); + // "Executing Klee" is printed before KLEE starts, so it does not say + // that KLEE ran. The tests it wrote do, and this line is what the + // archive's smoke test requires. + const size_t kleeTests = kleeTestCount(Map2Check::kleeOutputDir); + if (kleeTests > 0) { + Map2Check::Log::Info("KLEE ran: " + std::to_string(kleeTests) + + " tests generated"); + } else { + Map2Check::Log::Warning("KLEE generated no tests"); + } // KLEE found nothing: its vectors, run on natively. Not for // Cover-Branches, whose goal is no violation. if (map2checkMode != Map2CheckMode::COVER_BRANCHES_MODE && @@ -1577,6 +1634,26 @@ void Caller::executeAnalysis(std::string solvername) { // its findings live under afl-out/default/, not afl-out/. const std::string aflFindings = "afl-out/default"; + // As for KLEE, "Executing AFL++" precedes the launch. The execution + // count afl-fuzz recorded is the evidence that it fuzzed, and this line + // is what the archive's smoke test requires. When there is none, what + // afl-fuzz said goes to the log: fuzzer.output is removed with the + // scratch directory. + const long long aflExecs = + aflFuzzerStat(aflFindings + "/fuzzer_stats", "execs_done"); + if (aflExecs > 0) { + const long long corpus = + aflFuzzerStat(aflFindings + "/fuzzer_stats", "corpus_count"); + Map2Check::Log::Info("AFL++ ran: " + std::to_string(aflExecs) + + " executions, " + + std::to_string(std::max(0LL, corpus)) + + " inputs in the corpus"); + } else { + Map2Check::Log::Warning( + "AFL++ recorded no executions; its output ended with:\n" + + lastLines("fuzzer.output", 15)); + } + // Replay crashes with the witness binary to confirm a real violation. // Standalone, the persistent binary reads its input from stdin (see // NonDetGeneratorAFL.c), so the crash file is redirected in; the names diff --git a/utils/testcomp-archive/README.txt b/utils/testcomp-archive/README.txt index a6099f321..e4d6976d3 100644 --- a/utils/testcomp-archive/README.txt +++ b/utils/testcomp-archive/README.txt @@ -45,6 +45,10 @@ runs until the budget ends or KLEE has explored every path, and writes at most 50 test cases: up to half from the fuzzer's corpus (the inputs that found new edges), the rest one per path KLEE explored. +The log records what each engine did, from its own output after the run: +"AFL++ ran: N executions, M inputs in the corpus" and "KLEE ran: N tests +generated". ./smoketest.sh requires both lines. + Bundled components ------------------ Map2Check GPL-2.0 LICENSE diff --git a/utils/testcomp-archive/smoketest.sh b/utils/testcomp-archive/smoketest.sh index 6db3e0170..f074d95e1 100755 --- a/utils/testcomp-archive/smoketest.sh +++ b/utils/testcomp-archive/smoketest.sh @@ -44,12 +44,15 @@ fail() { # exit 1 } -# The run log must show that ran: map2check prints this line only -# after the engine's binary (AFL++) or bitcode (KLEE) was built. +# The run log must show that ran. Not the "Executing ..." banners: +# map2check prints those before launching the engine, so they appear when it +# fails to start too, and the other engine can still produce the suite (review +# of #86). These lines are printed after the run, from what the engine itself +# wrote: afl-fuzz's execution count in fuzzer_stats, KLEE's .ktest files. ran() { # case "$3" in - AFL++) line="Executing AFL++ with map2check" ;; - KLEE) line="Executing Klee with map2check" ;; + AFL++) line="AFL++ ran: [1-9][0-9]* executions" ;; + KLEE) line="KLEE ran: [1-9][0-9]* tests generated" ;; esac grep -q "$line" "$1/run.log" || fail "$1" "$2: $3 did not run" }