From cf19335439b622975dcdd1dd290a1cb9a7877cd0 Mon Sep 17 00:00:00 2001 From: Eirik Botten Nicolaysen Date: Sat, 31 Oct 2026 08:42:25 +0100 Subject: [PATCH] Rules: accept the unsupported rules migration 0004 stores Migration 0004 keeps rules that need order or grouping as {"type": "check", "check": "unsupported", "legacy": {...}} and says the engine reports them as INCONCLUSIVE. validate_rules rejected that shape, so validate_agentic_metadata raised for every such revision: the agentic evaluation ended in ERROR and the scenario could not be saved. Skip validation for exactly that shape. --- audits/agentic/rules.py | 2 ++ audits/agentic/tests/test_rule_migration.py | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/audits/agentic/rules.py b/audits/agentic/rules.py index 3b1bba72..0806077e 100644 --- a/audits/agentic/rules.py +++ b/audits/agentic/rules.py @@ -73,6 +73,8 @@ def validate_rules(rules: Any) -> list[RuleError]: errors.append(RuleError(index, "", "each rule must be an object")) elif rule.get("type") != "check": errors.append(RuleError(index, str(rule.get("type", "")), "rules must be custom checks")) + elif set(rule) == {"type", "check", "legacy"} and rule["check"] == "unsupported" and isinstance(rule["legacy"], dict): + continue # kept by migration 0004; the engine reports it as INCONCLUSIVE else: errors.extend(RuleError(index, "check", problem) for problem in validate_check(rule)) return errors diff --git a/audits/agentic/tests/test_rule_migration.py b/audits/agentic/tests/test_rule_migration.py index 5aabda47..d493ab75 100644 --- a/audits/agentic/tests/test_rule_migration.py +++ b/audits/agentic/tests/test_rule_migration.py @@ -38,3 +38,22 @@ def test_all_of_groups_are_flattened_into_separate_rules(): def test_metadata_without_rules_is_unchanged(): metadata = {"agentic": {"trace": {"required": True}}} assert migration.convert_metadata(metadata) == metadata + + +def test_unsupported_rules_kept_by_the_migration_pass_validation_and_are_inconclusive(): + from audits.agentic.checks.rules import rule_checks + from audits.agentic.expectations import validate_agentic_metadata + from audits.agentic.trajectory import normalize + + metadata = migration.convert_metadata({"agentic": {"schema_version": 2, "enforcement": {}, "rules": [ + {"type": "precedes", "first": "a", "then": "b"}]}}) + converted = metadata["agentic"]["rules"] + assert validate_agentic_metadata(metadata).rules == converted + trace = normalize([{"span_id": "t1", "name": "execute_tool", "attributes": { + "gen_ai.operation.name": "execute_tool", "gen_ai.tool.name": "a"}}]) + assert [check.status for check in rule_checks(trace, converted)] == ["INCONCLUSIVE"] + + +def test_only_the_migration_shape_of_an_unsupported_rule_skips_validation(): + junk = {"type": "check", "check": "unsupported", "select": "junk"} + assert validate_rules([junk]) != []