From cf19335439b622975dcdd1dd290a1cb9a7877cd0 Mon Sep 17 00:00:00 2001 From: Eirik Botten Nicolaysen Date: Sat, 31 Oct 2026 08:42:25 +0100 Subject: [PATCH 1/2] Rules: accept the unsupported rules migration 0004 stores Migration 0004 keeps rules that need order or grouping as {"type": "check", "check": "unsupported", "legacy": {...}} and says the engine reports them as INCONCLUSIVE. validate_rules rejected that shape, so validate_agentic_metadata raised for every such revision: the agentic evaluation ended in ERROR and the scenario could not be saved. Skip validation for exactly that shape. --- audits/agentic/rules.py | 2 ++ audits/agentic/tests/test_rule_migration.py | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/audits/agentic/rules.py b/audits/agentic/rules.py index 3b1bba72..0806077e 100644 --- a/audits/agentic/rules.py +++ b/audits/agentic/rules.py @@ -73,6 +73,8 @@ def validate_rules(rules: Any) -> list[RuleError]: errors.append(RuleError(index, "", "each rule must be an object")) elif rule.get("type") != "check": errors.append(RuleError(index, str(rule.get("type", "")), "rules must be custom checks")) + elif set(rule) == {"type", "check", "legacy"} and rule["check"] == "unsupported" and isinstance(rule["legacy"], dict): + continue # kept by migration 0004; the engine reports it as INCONCLUSIVE else: errors.extend(RuleError(index, "check", problem) for problem in validate_check(rule)) return errors diff --git a/audits/agentic/tests/test_rule_migration.py b/audits/agentic/tests/test_rule_migration.py index 5aabda47..d493ab75 100644 --- a/audits/agentic/tests/test_rule_migration.py +++ b/audits/agentic/tests/test_rule_migration.py @@ -38,3 +38,22 @@ def test_all_of_groups_are_flattened_into_separate_rules(): def test_metadata_without_rules_is_unchanged(): metadata = {"agentic": {"trace": {"required": True}}} assert migration.convert_metadata(metadata) == metadata + + +def test_unsupported_rules_kept_by_the_migration_pass_validation_and_are_inconclusive(): + from audits.agentic.checks.rules import rule_checks + from audits.agentic.expectations import validate_agentic_metadata + from audits.agentic.trajectory import normalize + + metadata = migration.convert_metadata({"agentic": {"schema_version": 2, "enforcement": {}, "rules": [ + {"type": "precedes", "first": "a", "then": "b"}]}}) + converted = metadata["agentic"]["rules"] + assert validate_agentic_metadata(metadata).rules == converted + trace = normalize([{"span_id": "t1", "name": "execute_tool", "attributes": { + "gen_ai.operation.name": "execute_tool", "gen_ai.tool.name": "a"}}]) + assert [check.status for check in rule_checks(trace, converted)] == ["INCONCLUSIVE"] + + +def test_only_the_migration_shape_of_an_unsupported_rule_skips_validation(): + junk = {"type": "check", "check": "unsupported", "select": "junk"} + assert validate_rules([junk]) != [] From 537878fe6917c778e1a61cad32cbc492daa98ac5 Mon Sep 17 00:00:00 2001 From: Eirik Botten Nicolaysen Date: Sat, 31 Oct 2026 08:59:49 +0100 Subject: [PATCH 2/2] Rule editor: show unsupported rules read-only and keep them on save renderCustom read rule.select.kind, which the unsupported rules kept by migration 0004 do not have. render() threw on them, so no card after the rule was drawn and sync() never ran: a rule added afterwards never reached agentic_rules_json and was lost on save. Draw an unsupported rule as a read-only card with the reason and a Remove button, and leave it in the rule list unchanged. --- infra/tests/test_rules_form.py | 7 +++++++ templates/partials/agentic_rules_editor.html | 12 +++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/infra/tests/test_rules_form.py b/infra/tests/test_rules_form.py index 9935a237..0962e081 100644 --- a/infra/tests/test_rules_form.py +++ b/infra/tests/test_rules_form.py @@ -46,3 +46,10 @@ def test_custom_check_built_in_the_editor_is_saved(): "check": "count", "expect": {"op": "at_least", "value": 1}} merged = _merge_structured_agentic_fields({}, _post(agentic_rules_json=json.dumps([rule]))) assert merged["agentic"]["rules"] == [rule] + + +def test_unsupported_rule_from_the_migration_is_saved_unchanged_beside_a_new_rule(): + unsupported = {"type": "check", "check": "unsupported", "legacy": {"type": "precedes", "first": "a", "then": "b"}} + rule = {"type": "check", "select": {"kind": "tool", "where": []}, "check": "count", "expect": {"op": "at_least", "value": 1}} + merged = _merge_structured_agentic_fields({}, _post(agentic_rules_json=json.dumps([unsupported, rule]))) + assert merged["agentic"]["rules"] == [unsupported, rule] diff --git a/templates/partials/agentic_rules_editor.html b/templates/partials/agentic_rules_editor.html index 4afedb8c..0a233d23 100644 --- a/templates/partials/agentic_rules_editor.html +++ b/templates/partials/agentic_rules_editor.html @@ -173,6 +173,15 @@ card.querySelector('[data-remove]').addEventListener('click', () => { rules.splice(index, 1); render(); sync(); }); } + // Mirrors rule_engine.unsupported_text. The rule is shown as stored and kept unchanged on save. + function renderUnsupported(card, rule, index) { + const legacy = rule.legacy && typeof rule.legacy === 'object' ? rule.legacy : {}; + const name = legacy.type || rule.check || rule.type || 'rule'; + card.querySelector('[data-summary]').textContent = `No longer supported (${name}): edit this rule to a count, exists or none check.`; + card.querySelector('[data-code]').textContent = JSON.stringify(legacy); + card.querySelector('[data-remove]').addEventListener('click', () => { rules.splice(index, 1); render(); sync(); }); + } + function render() { list.innerHTML = ''; rules.forEach((rule, index) => { @@ -186,7 +195,8 @@
`; - renderCustom(card, card.querySelector('[data-fields]'), rule, index); + if (rule.check === 'unsupported') renderUnsupported(card, rule, index); + else renderCustom(card, card.querySelector('[data-fields]'), rule, index); list.appendChild(card); }); sync();