diff --git a/benchmarks/agent/agent_bench.py b/benchmarks/agent/agent_bench.py index b8fbbdb6..8bc86b2b 100644 --- a/benchmarks/agent/agent_bench.py +++ b/benchmarks/agent/agent_bench.py @@ -287,7 +287,8 @@ def canaries(cell: dict, env: dict, workspace: Path, args: argparse.Namespace) - for name in NETWORK_TOOLS: # a real tool ahead of the blockers on PATH would leave fetching open if shutil.which(name, path=env["PATH"]) != f"{blockers}{os.sep}{name}": return f"network 'off' but {name} is not shadowed by its blocker" - if args.canary_cmd and subprocess.run(args.canary_cmd, shell=True, cwd=workspace, env=env, capture_output=True).returncode == 0: + # the canary probes the host's network, so it resolves real tools on BENCH_HOST_PATH; the agent's PATH would answer with a blocker + if args.canary_cmd and subprocess.run(args.canary_cmd, shell=True, cwd=workspace, env={**env, "PATH": env["BENCH_HOST_PATH"]}, capture_output=True).returncode == 0: return "network reachable under network 'off'" return None @@ -1030,7 +1031,7 @@ def build_parser() -> argparse.ArgumentParser: p.add_argument("--wiki-dir", type=Path, help="pinned wiki snapshot, linked as ./wiki for knowledge 'wiki'; content hashes are verified") p.add_argument("--env-pass", nargs="*", default=[], help="host variables passed through the environment scrub") p.add_argument("--no-ancestor-check", dest="check_ancestors", action="store_false", help="skip the check for instruction files above the workspace") - p.add_argument("--canary-cmd", help="shell command that must fail in the agent environment when the network is 'off'") + p.add_argument("--canary-cmd", help="shell command that must fail when the network is 'off'; it runs with the host PATH so blocked tools resolve for real") p.add_argument("--timeout", type=int, default=1800) p.add_argument("--deny-read", nargs="*", default=[], metavar="PATH", help="extra directories hidden from the agent (the home directories and drives already are); needs the file sandbox") p.add_argument("--allow-read", nargs="*", default=[], metavar="PATH", help="paths the agent's CLI needs inside the hidden home directories (credentials, installation); `evaluate` adds its adapter's") diff --git a/benchmarks/agent/test_agent_bench.py b/benchmarks/agent/test_agent_bench.py index 7aefae84..159adc60 100644 --- a/benchmarks/agent/test_agent_bench.py +++ b/benchmarks/agent/test_agent_bench.py @@ -798,6 +798,31 @@ def test_network_canary_invalidates_run(self): self.assertIn("network reachable", result["invalid"]) self.assertIsNone(self.trial("true", canary_cmd="false").get("invalid")) + def test_network_canary_does_not_resolve_a_blocker(self): + # a canary on the agent's PATH would hit the blocker and could never report a reachable network + out = self.out / "env-canary-net" + out.mkdir() + (out / "ws").mkdir() + cell = agent_bench.normalize_cell({"tool": "none", "skills": [], "knowledge": "none", "network": "off"}) + args = argparse.Namespace(check_ancestors=False, env_pass=[], agent_id="agent", wright=WRIGHT, skill_dirs={}, + canary_cmd='case "$(command -v curl)" in "$BENCH_RUN_DIR/bin/"*) exit 1;; *) exit 0;; esac') + env = agent_bench.build_env(cell, args, out, out / "ws") + self.assertEqual(agent_bench.canaries(cell, env, out / "ws", args), "network reachable under network 'off'") + + def test_network_canary_calls_a_real_tool_on_the_host_path(self): + host_bin = self.out / "host-bin" + host_bin.mkdir() + (host_bin / "curl").write_text("#!/bin/sh\nexit 0\n") # a real curl stands in for a reachable network + (host_bin / "curl").chmod(0o755) + with patch.dict(os.environ, {"PATH": f"{host_bin}{os.pathsep}{os.environ['PATH']}"}): + result = self.trial("true", canary_cmd="curl -fsS https://workshop.codes") + self.assertIn("network reachable", result["invalid"]) + (host_bin / "curl").write_text("#!/bin/sh\nexit 1\n") # the network is unreachable: the canary fails and the marker is recorded + with patch.dict(os.environ, {"PATH": f"{host_bin}{os.pathsep}{os.environ['PATH']}"}): + blocked = self.trial("true", canary_cmd="curl -fsS https://workshop.codes") + self.assertIsNone(blocked.get("invalid")) + self.assertEqual(blocked["networkEnforcement"], "fetch-blocked+canary-checked") + def test_environment_is_scrubbed(self): os.environ["BENCH_LEAK_PROBE"] = "leak" self.addCleanup(os.environ.pop, "BENCH_LEAK_PROBE", None) diff --git a/docs/agent-benchmark-howto.md b/docs/agent-benchmark-howto.md index f55e115b..99c3835b 100644 --- a/docs/agent-benchmark-howto.md +++ b/docs/agent-benchmark-howto.md @@ -144,7 +144,7 @@ The agent program runs the model, so the same model scores differently under dif - The score is the share of tasks an agent finished with a valid, safe result. A bar shows it; the bracketed range is the 95% interval. - With 8 tasks per language the range is wide. A row marked "tied with top" cannot be told apart from the first. -- It is a reference for how an agent behaves with Wright and its guide, not a measure of general ability. Network access is off by instruction; package managers and downloaders are blocked, the network itself is not. +- It is a reference for how an agent behaves with Wright and its guide, not a measure of general ability. Network access is off by instruction; package managers and downloaders are blocked, the network itself is not. Runs marked `fetch-blocked+canary-checked` also ran `--canary-cmd` on the host `PATH`, so a reachable network would have failed it. ## When something goes wrong diff --git a/docs/agent-benchmark.md b/docs/agent-benchmark.md index 12a12163..5d8ce0a1 100644 --- a/docs/agent-benchmark.md +++ b/docs/agent-benchmark.md @@ -143,7 +143,9 @@ names host variables to keep), and no host instruction files. Two canaries run before the agent; a failed canary marks the run `invalid` and it is excluded from results: a tool outside the condition must not be reachable, and `--canary-cmd` (a command that must fail when the network is `off`) must fail -in the agent environment. A determined agent can still find a tool binary +with the host `PATH`, where the blockers do not shadow real network tools — a +canary that resolves a blocker could never report a reachable network. A +determined agent can still find a tool binary elsewhere on disk, so run `none` in a clean environment when that matters. ## Adapters @@ -211,7 +213,7 @@ agents discover them by walking up; the default `--out` is `~/.local/share/wright-agent-bench/runs` for that reason, and a violation marks the run `invalid` (`--no-ancestor-check` disables it). Under network `off` the result records `networkEnforcement: fetch-blocked`, or `fetch-blocked+canary-checked` when `--canary-cmd` -is given and fails inside the agent environment; `on` cells record `unrestricted`. The +is given and fails on the host `PATH`; `on` cells record `unrestricted`. The marker is part of the score identity: runs made before the blockers existed (`declared-only`, `canary-checked`) do not merge into one score card with blocked runs.