fix(bench): run the network canary on the host PATH

benchmarks/agent/agent_bench.py

@@ -287,7 +287,8 @@ def canaries(cell: dict, env: dict, workspace: Path, args: argparse.Namespace) -
         for name in NETWORK_TOOLS:  # a real tool ahead of the blockers on PATH would leave fetching open
             if shutil.which(name, path=env["PATH"]) != f"{blockers}{os.sep}{name}":
                 return f"network 'off' but {name} is not shadowed by its blocker"
-        if args.canary_cmd and subprocess.run(args.canary_cmd, shell=True, cwd=workspace, env=env, capture_output=True).returncode == 0:
+        # the canary probes the host's network, so it resolves real tools on BENCH_HOST_PATH; the agent's PATH would answer with a blocker
+        if args.canary_cmd and subprocess.run(args.canary_cmd, shell=True, cwd=workspace, env={**env, "PATH": env["BENCH_HOST_PATH"]}, capture_output=True).returncode == 0:
             return "network reachable under network 'off'"
     return None
 
@@ -1030,7 +1031,7 @@ def build_parser() -> argparse.ArgumentParser:
         p.add_argument("--wiki-dir", type=Path, help="pinned wiki snapshot, linked as ./wiki for knowledge 'wiki'; content hashes are verified")
         p.add_argument("--env-pass", nargs="*", default=[], help="host variables passed through the environment scrub")
         p.add_argument("--no-ancestor-check", dest="check_ancestors", action="store_false", help="skip the check for instruction files above the workspace")
-        p.add_argument("--canary-cmd", help="shell command that must fail in the agent environment when the network is 'off'")
+        p.add_argument("--canary-cmd", help="shell command that must fail when the network is 'off'; it runs with the host PATH so blocked tools resolve for real")
         p.add_argument("--timeout", type=int, default=1800)
         p.add_argument("--deny-read", nargs="*", default=[], metavar="PATH", help="extra directories hidden from the agent (the home directories and drives already are); needs the file sandbox")
         p.add_argument("--allow-read", nargs="*", default=[], metavar="PATH", help="paths the agent's CLI needs inside the hidden home directories (credentials, installation); `evaluate` adds its adapter's")

benchmarks/agent/test_agent_bench.py

@@ -798,6 +798,31 @@ def test_network_canary_invalidates_run(self):
         self.assertIn("network reachable", result["invalid"])
         self.assertIsNone(self.trial("true", canary_cmd="false").get("invalid"))
 
+    def test_network_canary_does_not_resolve_a_blocker(self):
+        # a canary on the agent's PATH would hit the blocker and could never report a reachable network
+        out = self.out / "env-canary-net"
+        out.mkdir()
+        (out / "ws").mkdir()
+        cell = agent_bench.normalize_cell({"tool": "none", "skills": [], "knowledge": "none", "network": "off"})
+        args = argparse.Namespace(check_ancestors=False, env_pass=[], agent_id="agent", wright=WRIGHT, skill_dirs={},
+                                  canary_cmd='case "$(command -v curl)" in "$BENCH_RUN_DIR/bin/"*) exit 1;; *) exit 0;; esac')
+        env = agent_bench.build_env(cell, args, out, out / "ws")
+        self.assertEqual(agent_bench.canaries(cell, env, out / "ws", args), "network reachable under network 'off'")
+
+    def test_network_canary_calls_a_real_tool_on_the_host_path(self):
+        host_bin = self.out / "host-bin"
+        host_bin.mkdir()
+        (host_bin / "curl").write_text("#!/bin/sh\nexit 0\n")  # a real curl stands in for a reachable network
+        (host_bin / "curl").chmod(0o755)
+        with patch.dict(os.environ, {"PATH": f"{host_bin}{os.pathsep}{os.environ['PATH']}"}):
+            result = self.trial("true", canary_cmd="curl -fsS https://workshop.codes")
+        self.assertIn("network reachable", result["invalid"])
+        (host_bin / "curl").write_text("#!/bin/sh\nexit 1\n")  # the network is unreachable: the canary fails and the marker is recorded
+        with patch.dict(os.environ, {"PATH": f"{host_bin}{os.pathsep}{os.environ['PATH']}"}):
+            blocked = self.trial("true", canary_cmd="curl -fsS https://workshop.codes")
+        self.assertIsNone(blocked.get("invalid"))
+        self.assertEqual(blocked["networkEnforcement"], "fetch-blocked+canary-checked")
+
     def test_environment_is_scrubbed(self):
         os.environ["BENCH_LEAK_PROBE"] = "leak"
         self.addCleanup(os.environ.pop, "BENCH_LEAK_PROBE", None)

docs/agent-benchmark-howto.md

@@ -144,7 +144,7 @@ The agent program runs the model, so the same model scores differently under dif
 
 - The score is the share of tasks an agent finished with a valid, safe result. A bar shows it; the bracketed range is the 95% interval.
 - With 8 tasks per language the range is wide. A row marked "tied with top" cannot be told apart from the first.
-- It is a reference for how an agent behaves with Wright and its guide, not a measure of general ability. Network access is off by instruction; package managers and downloaders are blocked, the network itself is not.
+- It is a reference for how an agent behaves with Wright and its guide, not a measure of general ability. Network access is off by instruction; package managers and downloaders are blocked, the network itself is not. Runs marked `fetch-blocked+canary-checked` also ran `--canary-cmd` on the host `PATH`, so a reachable network would have failed it.
 
 ## When something goes wrong
 

docs/agent-benchmark.md

@@ -143,7 +143,9 @@ names host variables to keep), and no host instruction files. Two canaries run
 before the agent; a failed canary marks the run `invalid` and it is excluded
 from results: a tool outside the condition must not be reachable, and
 `--canary-cmd` (a command that must fail when the network is `off`) must fail
-in the agent environment. A determined agent can still find a tool binary
+with the host `PATH`, where the blockers do not shadow real network tools — a
+canary that resolves a blocker could never report a reachable network. A
+determined agent can still find a tool binary
 elsewhere on disk, so run `none` in a clean environment when that matters.
 
 ## Adapters
@@ -211,7 +213,7 @@ agents discover them by walking up; the default `--out` is
 `~/.local/share/wright-agent-bench/runs` for that reason, and a violation marks the run
 `invalid` (`--no-ancestor-check` disables it). Under network `off` the result records
 `networkEnforcement: fetch-blocked`, or `fetch-blocked+canary-checked` when `--canary-cmd`
-is given and fails inside the agent environment; `on` cells record `unrestricted`. The
+is given and fails on the host `PATH`; `on` cells record `unrestricted`. The
 marker is part of the score identity: runs made before the blockers existed (`declared-only`,
 `canary-checked`) do not merge into one score card with blocked runs.