fix(testcomp): the smoke test requires evidence that each engine ran

modules/frontend/caller.cpp

@@ -198,6 +198,53 @@ long long kleeCoveredInstructions(const std::string &statsPath) {
   return -1;
 #endif
 }
+
+/** One field of afl-fuzz's fuzzer_stats ("name : value" lines), which it
+ * rewrites while fuzzing and once more on exit; -1 when the file or the field
+ * is missing, as when afl-fuzz exited before fuzzing anything. */
+long long aflFuzzerStat(const std::string &statsPath, const std::string &name) {
+  std::ifstream stats(statsPath);
+  std::string line;
+  while (std::getline(stats, line)) {
+    const size_t colon = line.find(':');
+    if (colon == std::string::npos) continue;
+    std::string field = line.substr(0, colon);
+    field.erase(field.find_last_not_of(' ') + 1);
+    if (field != name) continue;
+    try {
+      return std::stoll(line.substr(colon + 1));
+    } catch (const std::exception &) {
+      return -1;
+    }
+  }
+  return -1;
+}
+
+/** The .ktest files KLEE wrote, one per path it terminated. */
+size_t kleeTestCount(const std::string &outputDir) {
+  size_t count = 0;
+  std::error_code error;
+  for (const auto &entry :
+       std::filesystem::directory_iterator(outputDir, error)) {
+    if (entry.path().extension() == ".ktest") ++count;
+  }
+  return count;
+}
+
+/** The last `count` lines of a file. For the log: the scratch directory that
+ * holds an engine's own output is removed at the end of the run. */
+std::string lastLines(const std::string &path, size_t count) {
+  std::ifstream file(path);
+  std::vector<std::string> lines;
+  std::string line;
+  while (std::getline(file, line)) {
+    lines.push_back(line);
+    if (lines.size() > count) lines.erase(lines.begin());
+  }
+  std::string tail;
+  for (const std::string &kept : lines) tail += "  " + kept + "\n";
+  return tail.empty() ? "  (no output)\n" : tail;
+}
 }  // namespace
 
 int Caller::runKleeWatched(const std::string &command) {
@@ -1446,6 +1493,16 @@ void Caller::executeAnalysis(std::string solvername) {
         exportKleeVectorsAsSeeds();
       }
       Map2Check::Log::Warning("Exited klee with " + std::to_string(result));
+      // "Executing Klee" is printed before KLEE starts, so it does not say
+      // that KLEE ran. The tests it wrote do, and this line is what the
+      // archive's smoke test requires.
+      const size_t kleeTests = kleeTestCount(Map2Check::kleeOutputDir);
+      if (kleeTests > 0) {
+        Map2Check::Log::Info("KLEE ran: " + std::to_string(kleeTests) +
+                             " tests generated");
+      } else {
+        Map2Check::Log::Warning("KLEE generated no tests");
+      }
       // KLEE found nothing: its vectors, run on natively. Not for
       // Cover-Branches, whose goal is no violation.
       if (map2checkMode != Map2CheckMode::COVER_BRANCHES_MODE &&
@@ -1577,6 +1634,26 @@ void Caller::executeAnalysis(std::string solvername) {
       // its findings live under afl-out/default/, not afl-out/.
       const std::string aflFindings = "afl-out/default";
 
+      // As for KLEE, "Executing AFL++" precedes the launch. The execution
+      // count afl-fuzz recorded is the evidence that it fuzzed, and this line
+      // is what the archive's smoke test requires. When there is none, what
+      // afl-fuzz said goes to the log: fuzzer.output is removed with the
+      // scratch directory.
+      const long long aflExecs =
+          aflFuzzerStat(aflFindings + "/fuzzer_stats", "execs_done");
+      if (aflExecs > 0) {
+        const long long corpus =
+            aflFuzzerStat(aflFindings + "/fuzzer_stats", "corpus_count");
+        Map2Check::Log::Info("AFL++ ran: " + std::to_string(aflExecs) +
+                             " executions, " +
+                             std::to_string(std::max(0LL, corpus)) +
+                             " inputs in the corpus");
+      } else {
+        Map2Check::Log::Warning(
+            "AFL++ recorded no executions; its output ended with:\n" +
+            lastLines("fuzzer.output", 15));
+      }
+
       // Replay crashes with the witness binary to confirm a real violation.
       // Standalone, the persistent binary reads its input from stdin (see
       // NonDetGeneratorAFL.c), so the crash file is redirected in; the names

utils/testcomp-archive/README.txt

@@ -45,6 +45,10 @@ runs until the budget ends or KLEE has explored every path, and writes at most
 50 test cases: up to half from the fuzzer's corpus (the inputs that found new
 edges), the rest one per path KLEE explored.
 
+The log records what each engine did, from its own output after the run:
+"AFL++ ran: N executions, M inputs in the corpus" and "KLEE ran: N tests
+generated". ./smoketest.sh requires both lines.
+
 Bundled components
 ------------------
   Map2Check            GPL-2.0                         LICENSE

utils/testcomp-archive/smoketest.sh

@@ -44,12 +44,15 @@ fail() { # <work> <message>
   exit 1
 }
 
-# The run log must show that <engine> ran: map2check prints this line only
-# after the engine's binary (AFL++) or bitcode (KLEE) was built.
+# The run log must show that <engine> ran. Not the "Executing ..." banners:
+# map2check prints those before launching the engine, so they appear when it
+# fails to start too, and the other engine can still produce the suite (review
+# of #86). These lines are printed after the run, from what the engine itself
+# wrote: afl-fuzz's execution count in fuzzer_stats, KLEE's .ktest files.
 ran() { # <work> <program> <engine>
   case "$3" in
-    AFL++) line="Executing AFL++ with map2check" ;;
-    KLEE) line="Executing Klee with map2check" ;;
+    AFL++) line="AFL++ ran: [1-9][0-9]* executions" ;;
+    KLEE) line="KLEE ran: [1-9][0-9]* tests generated" ;;
   esac
   grep -q "$line" "$1/run.log" || fail "$1" "$2: $3 did not run"
 }