diff --git a/benchmarks/agent/bench_grade.py b/benchmarks/agent/bench_grade.py index cf56bd02..ecb578c4 100644 --- a/benchmarks/agent/bench_grade.py +++ b/benchmarks/agent/bench_grade.py @@ -140,7 +140,8 @@ def call_nodes(node): def count_calls(rules: list[dict], check: dict) -> int: - """Calls matching `call` and the positional `args` patterns, across every enabled rule (or those with the `event` id).""" + """Calls matching any call/args alternative, across enabled rules (or those with the `event` id).""" + alternatives = check.get("alternatives", [check]) count = 0 for rule in rules: content = rule["content"] @@ -151,9 +152,12 @@ def count_calls(rules: list[dict], check: dict) -> int: ] + [action for action in content["actions"] if not action.get("disabled")] for statement in statements: for node in call_nodes(statement): - if node["call"] == check["call"] and all( - int(position) < len(node["args"]) and pattern_matches(pattern, node["args"][int(position)]) - for position, pattern in check.get("args", {}).items() + if any( + node["call"] == alternative["call"] and all( + int(position) < len(node["args"]) and pattern_matches(pattern, node["args"][int(position)]) + for position, pattern in alternative.get("args", {}).items() + ) + for alternative in alternatives ): count += 1 return count @@ -179,7 +183,8 @@ def program_calls(check: dict, entry: Path, wright: str, state: dict) -> dict: if result["selection"]["withheld"]: return check_result(check, False, f"incomplete: {result['selection']['withheld']} rule(s) withheld by maxRules") count = count_calls(result["rules"], check) - return check_result(check, check.get("min", 1) <= count <= check.get("max", count), f"{count} matching '{check['call']}' call(s) in {source} output") + target = "alternatives" if "alternatives" in check else f"'{check['call']}'" + return check_result(check, check.get("min", 1) <= count <= check.get("max", count), f"{count} matching {target} call(s) in {source} output") def check_result(check: dict, passed: bool, detail: str, **extra) -> dict: diff --git a/benchmarks/agent/fixtures/program-calls/exclude-event-player.en-US.ow b/benchmarks/agent/fixtures/program-calls/exclude-event-player.en-US.ow new file mode 100644 index 00000000..b12631ba --- /dev/null +++ b/benchmarks/agent/fixtures/program-calls/exclude-event-player.en-US.ow @@ -0,0 +1,32 @@ +rule ("Array removal") { + event { + Ongoing - Each Player; + All; + All; + } + actions { + Small Message(Remove From Array(All Living Players(Team 2), Event Player), Custom String("Other players")); + } +} + +rule ("Filter") { + event { + Ongoing - Each Player; + All; + All; + } + actions { + Small Message(Filtered Array(All Living Players(Team 2), Current Array Element != Event Player), Custom String("Other players")); + } +} + +rule ("Reversed filter") { + event { + Ongoing - Each Player; + All; + All; + } + actions { + Small Message(Filtered Array(All Living Players(Team 2), Event Player != Current Array Element), Custom String("Other players")); + } +} diff --git a/benchmarks/agent/fixtures/program-calls/includes-event-player.en-US.ow b/benchmarks/agent/fixtures/program-calls/includes-event-player.en-US.ow new file mode 100644 index 00000000..7d60db2a --- /dev/null +++ b/benchmarks/agent/fixtures/program-calls/includes-event-player.en-US.ow @@ -0,0 +1,10 @@ +rule ("Includes event player") { + event { + Ongoing - Each Player; + All; + All; + } + actions { + Small Message(Filtered Array(All Living Players(Team 2), Current Array Element == Event Player), Custom String("Event player")); + } +} diff --git a/benchmarks/agent/test_agent_bench.py b/benchmarks/agent/test_agent_bench.py index 4e32bdde..37403331 100644 --- a/benchmarks/agent/test_agent_bench.py +++ b/benchmarks/agent/test_agent_bench.py @@ -392,6 +392,48 @@ def test_program_calls_alternatives_and_event_restriction(self): any_of = {"call": "setStatusEffect", "args": {"3": {"any": [0.5, 0.75]}}, "min": 2} self.assertEqual(self.program_calls("repel.opy", [any_of, {**any_of, "event": "global"}, {**any_of, "event": "eachPlayer"}]), [True, False, True]) + def test_program_calls_counts_each_node_once_across_call_alternatives(self): + alternatives = {"alternatives": list(self.REPEL_CHECKS), "min": 4, "max": 4} + for fixture in ("repel.opy", "repel.en-US.ow", "repel.zh-CN.ow"): + checks = [alternatives, {**alternatives, "min": 5}, {**alternatives, "max": 3}, + {**alternatives, "event": "global"}, + {**alternatives, "alternatives": [*alternatives["alternatives"], self.REPEL_CHECKS[0]]}] + self.assertEqual(self.program_calls(fixture, checks), [True, False, False, False, True], fixture) + + def test_program_calls_accepts_equivalent_player_exclusions_and_rejects_inclusion(self): + event_player = {"call": "eventPlayer"} + element = {"call": "currentArrayElement"} + players = {"call": "getLivingPlayers", "args": [{"enum": "Team", "member": "TEAM_2"}]} + alternatives = [ + {"call": "removeFromArray", "args": {"0": players, "1": event_player}}, + {"call": "filteredArray", "args": {"0": players, "1": {"call": "!=", "args": [element, event_player]}}}, + {"call": "filteredArray", "args": {"0": players, "1": {"call": "!=", "args": [event_player, element]}}}, + ] + check = {"id": "exclude-player", "kind": "program-calls", "alternatives": alternatives, "min": 1, "max": 1} + entry = agent_bench.HERE / "fixtures/program-calls/exclude-event-player.en-US.ow" + authorities = bench_grade.authorities(WRIGHT, entry, self.out) + response = bench_grade.rule_content(WRIGHT, entry, 200) + for rule in response["result"]["rules"]: + with self.subTest(rule=rule["name"]): + state = {"scratch": self.out, "authorities": authorities, + "ruleContent": {("wright", 200): {"result": {"rules": [rule], "selection": {"withheld": 0}}}}} + self.assertTrue(bench_grade.run_check(check, entry.parent, entry, WRIGHT, state)["passed"]) + combined = {"alternatives": alternatives, "min": 3, "max": 3} + self.assertEqual(self.program_calls(entry.name, [combined, {**combined, "max": 2}]), [True, False]) + self.assertEqual(self.program_calls("includes-event-player.en-US.ow", [check]), [False]) + + def test_program_call_alternatives_are_unavailable_without_rule_content(self): + entry = agent_bench.HERE / "fixtures/program-calls/repel.en-US.ow" + check = {"id": "c", "kind": "program-calls", "alternatives": list(self.REPEL_CHECKS)} + authorities = bench_grade.authorities(WRIGHT, entry, self.out) + for response in ({"result": [{"id": 0, "name": "r"}]}, + {"error": {"code": "invalid-request", "message": "unknown field"}}): + state = {"scratch": self.out, "authorities": authorities} + with patch.object(bench_grade, "rule_content", return_value=response): + result = bench_grade.run_check(check, entry.parent, entry, WRIGHT, state) + self.assertFalse(result["passed"]) + self.assertTrue(result["unavailable"]) + @unittest.skipUnless(bench_grade.oracle_available(), "run `agent_bench.py setup-oracle`") def test_program_calls_reads_the_upstream_oracle_output(self): self.assertEqual(self.program_calls("repel.opy", list(self.REPEL_CHECKS), source="oracle"), [True, True]) diff --git a/docs/agent-benchmark.md b/docs/agent-benchmark.md index b9d2ef7a..b2e9e3e3 100644 --- a/docs/agent-benchmark.md +++ b/docs/agent-benchmark.md @@ -276,11 +276,29 @@ of alternatives, `min`/`max` occurrences), `answer` (`answer.json` key equals the compiled text of `source` is read through `rules` with `content`, and the check counts calls whose catalog id equals `call` and whose positional `args` patterns match, across all rules or those with the `event` id, within -`min`/`max`). A pattern is a rule-content node: numbers compare within 1e-9, +`min`/`max`). Instead of a single `call`/`args`, a check can list +`alternatives: [{"call": ..., "args": {...}}, ...]`. Each call node matching +any alternative counts once toward the shared `min`/`max`, even if several +alternatives match it. `event` and `maxRules` apply to the whole check. +A pattern is a rule-content node: numbers compare within 1e-9, objects as a recursive subset, and `{"any": [...]}` lists alternatives. The check is stable across spelling, locale, and macro expansion, but not across restructured equivalents such as a value held in a variable, so list those -under `any` or assert the pieces separately. A Wright that does not serve +as call `alternatives` or argument patterns under `any`, or assert the pieces +separately. For example, excluding the event player can use array removal or +a filter with either operand order: + +```json +{"kind": "program-calls", "alternatives": [ + {"call": "removeFromArray", "args": {"1": {"call": "eventPlayer"}}}, + {"call": "filteredArray", "args": {"1": {"call": "!=", "args": [ + {"call": "currentArrayElement"}, {"call": "eventPlayer"}]}}}, + {"call": "filteredArray", "args": {"1": {"call": "!=", "args": [ + {"call": "eventPlayer"}, {"call": "currentArrayElement"}]}}} +], "min": 1} +``` + +A Wright that does not serve rule content makes the check `unavailable`, never a pass; `maxRules` (default 200) bounds the read and an incomplete read fails. Prefer `program-calls` over a regex; a regex over compiled text needs a `reference` that matches and a diff --git a/docs/specs/SPEC-414-agent-benchmark-comparison.md b/docs/specs/SPEC-414-agent-benchmark-comparison.md index 0656df07..6b22dd0e 100644 --- a/docs/specs/SPEC-414-agent-benchmark-comparison.md +++ b/docs/specs/SPEC-414-agent-benchmark-comparison.md @@ -84,6 +84,10 @@ Pilot observations that shaped this spec (Sonnet only, 2 scenarios, 18 runs; not (`inspect` JSON or `serve` operations). A regex over emitted text is allowed only where no structured field exists and ships with a positive and a negative fixture. The `program-calls` check kind (ADR-0024) reads the canonical rule content through `rules` with `content` and asserts on calls by catalog id and argument position. + A check uses either a single `call`/`args` or `alternatives: [{"call": ..., "args": {...}}, ...]`. + Each node matching any alternative counts once toward the shared `min`/`max`; `event` restricts all alternatives. + Equivalent forms are listed explicitly, without normalization or dataflow matching. Missing rule content + makes the check `unavailable`, never passed. See [the check format](../agent-benchmark.md#scenarios). - REQ-011: Reported outcome metrics are: `usable` (L1a passes, all required requirement checks pass, no error-severity lint finding), requirement coverage over all runs and over L1a-passing runs, time to first valid snapshot, valid-to-invalid regressions, wall time, and the context and token metrics of REQ-020 to REQ-024. Runtime-only claims are listed as