diff --git a/audits/agentic/rules.py b/audits/agentic/rules.py index 3b1bba72..0806077e 100644 --- a/audits/agentic/rules.py +++ b/audits/agentic/rules.py @@ -73,6 +73,8 @@ def validate_rules(rules: Any) -> list[RuleError]: errors.append(RuleError(index, "", "each rule must be an object")) elif rule.get("type") != "check": errors.append(RuleError(index, str(rule.get("type", "")), "rules must be custom checks")) + elif set(rule) == {"type", "check", "legacy"} and rule["check"] == "unsupported" and isinstance(rule["legacy"], dict): + continue # kept by migration 0004; the engine reports it as INCONCLUSIVE else: errors.extend(RuleError(index, "check", problem) for problem in validate_check(rule)) return errors diff --git a/audits/agentic/tests/test_rule_migration.py b/audits/agentic/tests/test_rule_migration.py index 5aabda47..d493ab75 100644 --- a/audits/agentic/tests/test_rule_migration.py +++ b/audits/agentic/tests/test_rule_migration.py @@ -38,3 +38,22 @@ def test_all_of_groups_are_flattened_into_separate_rules(): def test_metadata_without_rules_is_unchanged(): metadata = {"agentic": {"trace": {"required": True}}} assert migration.convert_metadata(metadata) == metadata + + +def test_unsupported_rules_kept_by_the_migration_pass_validation_and_are_inconclusive(): + from audits.agentic.checks.rules import rule_checks + from audits.agentic.expectations import validate_agentic_metadata + from audits.agentic.trajectory import normalize + + metadata = migration.convert_metadata({"agentic": {"schema_version": 2, "enforcement": {}, "rules": [ + {"type": "precedes", "first": "a", "then": "b"}]}}) + converted = metadata["agentic"]["rules"] + assert validate_agentic_metadata(metadata).rules == converted + trace = normalize([{"span_id": "t1", "name": "execute_tool", "attributes": { + "gen_ai.operation.name": "execute_tool", "gen_ai.tool.name": "a"}}]) + assert [check.status for check in rule_checks(trace, converted)] == ["INCONCLUSIVE"] + + +def test_only_the_migration_shape_of_an_unsupported_rule_skips_validation(): + junk = {"type": "check", "check": "unsupported", "select": "junk"} + assert validate_rules([junk]) != [] diff --git a/infra/tests/test_rules_form.py b/infra/tests/test_rules_form.py index 9935a237..0962e081 100644 --- a/infra/tests/test_rules_form.py +++ b/infra/tests/test_rules_form.py @@ -46,3 +46,10 @@ def test_custom_check_built_in_the_editor_is_saved(): "check": "count", "expect": {"op": "at_least", "value": 1}} merged = _merge_structured_agentic_fields({}, _post(agentic_rules_json=json.dumps([rule]))) assert merged["agentic"]["rules"] == [rule] + + +def test_unsupported_rule_from_the_migration_is_saved_unchanged_beside_a_new_rule(): + unsupported = {"type": "check", "check": "unsupported", "legacy": {"type": "precedes", "first": "a", "then": "b"}} + rule = {"type": "check", "select": {"kind": "tool", "where": []}, "check": "count", "expect": {"op": "at_least", "value": 1}} + merged = _merge_structured_agentic_fields({}, _post(agentic_rules_json=json.dumps([unsupported, rule]))) + assert merged["agentic"]["rules"] == [unsupported, rule] diff --git a/templates/partials/agentic_rules_editor.html b/templates/partials/agentic_rules_editor.html index 4afedb8c..0a233d23 100644 --- a/templates/partials/agentic_rules_editor.html +++ b/templates/partials/agentic_rules_editor.html @@ -173,6 +173,15 @@ card.querySelector('[data-remove]').addEventListener('click', () => { rules.splice(index, 1); render(); sync(); }); } + // Mirrors rule_engine.unsupported_text. The rule is shown as stored and kept unchanged on save. + function renderUnsupported(card, rule, index) { + const legacy = rule.legacy && typeof rule.legacy === 'object' ? rule.legacy : {}; + const name = legacy.type || rule.check || rule.type || 'rule'; + card.querySelector('[data-summary]').textContent = `No longer supported (${name}): edit this rule to a count, exists or none check.`; + card.querySelector('[data-code]').textContent = JSON.stringify(legacy); + card.querySelector('[data-remove]').addEventListener('click', () => { rules.splice(index, 1); render(); sync(); }); + } + function render() { list.innerHTML = ''; rules.forEach((rule, index) => { @@ -186,7 +195,8 @@
`; - renderCustom(card, card.querySelector('[data-fields]'), rule, index); + if (rule.check === 'unsupported') renderUnsupported(card, rule, index); + else renderCustom(card, card.querySelector('[data-fields]'), rule, index); list.appendChild(card); }); sync();