From 3bf8ff0631cbc9d441a3b99ddbcd160ab52f1463 Mon Sep 17 00:00:00 2001 From: Alberto Vara Date: Wed, 21 Oct 2026 13:26:51 +0200 Subject: [PATCH 1/5] AI Guard: tests for backend-provided extra_tags on the ai_guard span The evaluate response can carry an extra_tags object so AI Guard policies can enrich traces without a tracer release per tag (APPSEC-70724). Scalars must land as ai_guard span tags under the key exactly as returned, and structured values in the ai_guard meta struct; the backend owns the ai_guard prefix, so tracers neither add it nor filter by namespace. A malformed payload must be ignored without changing the evaluation. Cassettes are hand-crafted because the backend does not return the field yet; gen_extra_tags_cassettes.py generates them together with the scenario sidecar and is checked by format.sh. Enabled for python flask-poc (dd-trace-py#21145), missing_feature for every other tracer. Co-Authored-By: Claude Opus 5.5 --- format.sh | 8 + manifests/cpp_httpd.yml | 2 + manifests/cpp_nginx.yml | 2 + manifests/dotnet.yml | 2 + manifests/golang.yml | 2 + manifests/java.yml | 2 + manifests/nodejs.yml | 2 + manifests/php.yml | 2 + manifests/python.yml | 8 + manifests/ruby.yml | 2 + tests/ai_guard/extra_tags_scenarios.json | 143 ++++++++++++ tests/ai_guard/test_ai_guard_sdk.py | 170 ++++++++++++++ .../aiguard_evaluate_post_28917f95.json | 22 ++ .../aiguard_evaluate_post_2b336647.json | 22 ++ .../aiguard_evaluate_post_354106da.json | 22 ++ .../aiguard_evaluate_post_827d1449.json | 22 ++ .../aiguard_evaluate_post_bf6868f9.json | 22 ++ .../aiguard_evaluate_post_e2c13e67.json | 22 ++ .../aiguard_evaluate_post_f0ef2bd8.json | 22 ++ utils/scripts/gen_extra_tags_cassettes.py | 220 ++++++++++++++++++ 20 files changed, 719 insertions(+) create mode 100644 tests/ai_guard/extra_tags_scenarios.json create mode 100644 utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_28917f95.json create mode 100644 utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_2b336647.json create mode 100644 utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_354106da.json create mode 100644 utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_827d1449.json create mode 100644 utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_bf6868f9.json create mode 100644 utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_e2c13e67.json create mode 100644 utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_f0ef2bd8.json create mode 100755 utils/scripts/gen_extra_tags_cassettes.py diff --git a/format.sh b/format.sh index 6f634787937..d8ed6b1f2aa 100755 --- a/format.sh +++ b/format.sh @@ -210,6 +210,14 @@ if ! gen_redaction_output="$(python utils/scripts/gen_redaction_cassettes.py $ge exit 1 fi +echo "Checking AI Guard extra_tags fixtures..." +if ! gen_extra_tags_output="$(python utils/scripts/gen_extra_tags_cassettes.py $gen_redaction_args 2>&1)"; then + echo "$gen_extra_tags_output" + echo "AI Guard extra_tags fixtures are out of date or invalid. Regenerate them with" + echo "python utils/scripts/gen_extra_tags_cassettes.py 💥 💔 💥" + exit 1 +fi + echo "Running shellcheck checks..." if ! ./utils/scripts/shellcheck.sh; then echo "shellcheck checks failed. Please fix the errors above. 💥 💔 💥" diff --git a/manifests/cpp_httpd.yml b/manifests/cpp_httpd.yml index 47030f0bcc9..4e18ff2917c 100644 --- a/manifests/cpp_httpd.yml +++ b/manifests/cpp_httpd.yml @@ -21,6 +21,8 @@ manifest: tests/ai_guard/test_ai_guard_sdk.py::Test_ClientIPTagsCollected: missing_feature tests/ai_guard/test_ai_guard_sdk.py::Test_ContentParts: missing_feature tests/ai_guard/test_ai_guard_sdk.py::Test_Evaluation: missing_feature + tests/ai_guard/test_ai_guard_sdk.py::Test_ExtraTags: missing_feature (APPSEC-70724) + tests/ai_guard/test_ai_guard_sdk.py::Test_ExtraTagsFailSafe: missing_feature (APPSEC-70724) tests/ai_guard/test_ai_guard_sdk.py::Test_Full_Response_And_Tags: missing_feature tests/ai_guard/test_ai_guard_sdk.py::Test_NoRedaction: missing_feature tests/ai_guard/test_ai_guard_sdk.py::Test_RedactedMessagesInSDKResponse: missing_feature diff --git a/manifests/cpp_nginx.yml b/manifests/cpp_nginx.yml index 30648f6498b..5b88e3ea88d 100644 --- a/manifests/cpp_nginx.yml +++ b/manifests/cpp_nginx.yml @@ -19,6 +19,8 @@ manifest: tests/ai_guard/test_ai_guard_sdk.py::Test_ClientIPTagsCollected: missing_feature tests/ai_guard/test_ai_guard_sdk.py::Test_ContentParts: missing_feature tests/ai_guard/test_ai_guard_sdk.py::Test_Evaluation: missing_feature + tests/ai_guard/test_ai_guard_sdk.py::Test_ExtraTags: missing_feature (APPSEC-70724) + tests/ai_guard/test_ai_guard_sdk.py::Test_ExtraTagsFailSafe: missing_feature (APPSEC-70724) tests/ai_guard/test_ai_guard_sdk.py::Test_Full_Response_And_Tags: missing_feature tests/ai_guard/test_ai_guard_sdk.py::Test_NoRedaction: missing_feature tests/ai_guard/test_ai_guard_sdk.py::Test_RedactedMessagesInSDKResponse: missing_feature diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml index 7db6bf29e45..05a6646529f 100644 --- a/manifests/dotnet.yml +++ b/manifests/dotnet.yml @@ -11,6 +11,8 @@ manifest: tests/ai_guard/test_ai_guard_sdk.py::Test_ClientIPTagsCollected: missing_feature tests/ai_guard/test_ai_guard_sdk.py::Test_ContentParts: missing_feature tests/ai_guard/test_ai_guard_sdk.py::Test_Evaluation: missing_feature + tests/ai_guard/test_ai_guard_sdk.py::Test_ExtraTags: missing_feature (APPSEC-70724) + tests/ai_guard/test_ai_guard_sdk.py::Test_ExtraTagsFailSafe: missing_feature (APPSEC-70724) tests/ai_guard/test_ai_guard_sdk.py::Test_Full_Response_And_Tags: missing_feature tests/ai_guard/test_ai_guard_sdk.py::Test_NoRedaction: missing_feature tests/ai_guard/test_ai_guard_sdk.py::Test_RedactedMessagesInSDKResponse: missing_feature diff --git a/manifests/golang.yml b/manifests/golang.yml index d498b6a39ab..569f657a4b9 100644 --- a/manifests/golang.yml +++ b/manifests/golang.yml @@ -11,6 +11,8 @@ manifest: tests/ai_guard/test_ai_guard_sdk.py::Test_ClientIPTagsCollected: missing_feature tests/ai_guard/test_ai_guard_sdk.py::Test_ContentParts: missing_feature tests/ai_guard/test_ai_guard_sdk.py::Test_Evaluation: missing_feature + tests/ai_guard/test_ai_guard_sdk.py::Test_ExtraTags: missing_feature (APPSEC-70724) + tests/ai_guard/test_ai_guard_sdk.py::Test_ExtraTagsFailSafe: missing_feature (APPSEC-70724) tests/ai_guard/test_ai_guard_sdk.py::Test_Full_Response_And_Tags: missing_feature tests/ai_guard/test_ai_guard_sdk.py::Test_NoRedaction: missing_feature tests/ai_guard/test_ai_guard_sdk.py::Test_RedactedMessagesInSDKResponse: missing_feature diff --git a/manifests/java.yml b/manifests/java.yml index bd3559e9466..bd471125d2f 100644 --- a/manifests/java.yml +++ b/manifests/java.yml @@ -47,6 +47,8 @@ manifest: - weblog_declaration: "*": irrelevant (just one weblog is enough to test the SDK) "spring-boot": v1.54.0 + tests/ai_guard/test_ai_guard_sdk.py::Test_ExtraTags: missing_feature (APPSEC-70724) + tests/ai_guard/test_ai_guard_sdk.py::Test_ExtraTagsFailSafe: missing_feature (APPSEC-70724) tests/ai_guard/test_ai_guard_sdk.py::Test_Full_Response_And_Tags: - weblog_declaration: "*": irrelevant (just one weblog is enough to test the SDK) diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index 94b6334410c..7e6bfb0a674 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -151,6 +151,8 @@ manifest: - weblog_declaration: "*": irrelevant (just one weblog is enough to test the SDK) express5: *ref_5_69_0 + tests/ai_guard/test_ai_guard_sdk.py::Test_ExtraTags: missing_feature (APPSEC-70724) + tests/ai_guard/test_ai_guard_sdk.py::Test_ExtraTagsFailSafe: missing_feature (APPSEC-70724) tests/ai_guard/test_ai_guard_sdk.py::Test_Full_Response_And_Tags: - weblog_declaration: "*": irrelevant (just one weblog is enough to test the SDK) diff --git a/manifests/php.yml b/manifests/php.yml index 64d0e027b3d..783f47d7049 100644 --- a/manifests/php.yml +++ b/manifests/php.yml @@ -11,6 +11,8 @@ manifest: tests/ai_guard/test_ai_guard_sdk.py::Test_ClientIPTagsCollected: missing_feature tests/ai_guard/test_ai_guard_sdk.py::Test_ContentParts: missing_feature tests/ai_guard/test_ai_guard_sdk.py::Test_Evaluation: missing_feature + tests/ai_guard/test_ai_guard_sdk.py::Test_ExtraTags: missing_feature (APPSEC-70724) + tests/ai_guard/test_ai_guard_sdk.py::Test_ExtraTagsFailSafe: missing_feature (APPSEC-70724) tests/ai_guard/test_ai_guard_sdk.py::Test_Full_Response_And_Tags: missing_feature tests/ai_guard/test_ai_guard_sdk.py::Test_NoRedaction: missing_feature tests/ai_guard/test_ai_guard_sdk.py::Test_RedactedMessagesInSDKResponse: missing_feature diff --git a/manifests/python.yml b/manifests/python.yml index 22b9f37c023..fbd70ad59aa 100644 --- a/manifests/python.yml +++ b/manifests/python.yml @@ -60,6 +60,14 @@ manifest: - weblog_declaration: "*": irrelevant (just one weblog is enough to test the SDK) "flask-poc": v3.18.1 + tests/ai_guard/test_ai_guard_sdk.py::Test_ExtraTags: + - weblog_declaration: + "*": irrelevant (just one weblog is enough to test the SDK) + "flask-poc": v4.17.0-dev + tests/ai_guard/test_ai_guard_sdk.py::Test_ExtraTagsFailSafe: + - weblog_declaration: + "*": irrelevant (just one weblog is enough to test the SDK) + "flask-poc": v4.17.0-dev tests/ai_guard/test_ai_guard_sdk.py::Test_Full_Response_And_Tags: - weblog_declaration: "*": irrelevant (just one weblog is enough to test the SDK) diff --git a/manifests/ruby.yml b/manifests/ruby.yml index 28350f8caea..946cb3b1de4 100644 --- a/manifests/ruby.yml +++ b/manifests/ruby.yml @@ -88,6 +88,8 @@ manifest: sinatra32: irrelevant sinatra41: irrelevant uds-sinatra: irrelevant + tests/ai_guard/test_ai_guard_sdk.py::Test_ExtraTags: missing_feature (APPSEC-70724) + tests/ai_guard/test_ai_guard_sdk.py::Test_ExtraTagsFailSafe: missing_feature (APPSEC-70724) tests/ai_guard/test_ai_guard_sdk.py::Test_Full_Response_And_Tags: - weblog_declaration: "*": v2.25.0-dev diff --git a/tests/ai_guard/extra_tags_scenarios.json b/tests/ai_guard/extra_tags_scenarios.json new file mode 100644 index 00000000000..eb3e1655569 --- /dev/null +++ b/tests/ai_guard/extra_tags_scenarios.json @@ -0,0 +1,143 @@ +{ + "EXTRA_TAGS_SCALARS": { + "messages": [ + { + "role": "user", + "content": "Summarize the latest newsroom style guide changes." + } + ], + "action": "ALLOW", + "tags": [], + "extra_tags": { + "ai_guard.policy_name": "newsroom-default", + "ai_guard.policy_revision": 7, + "ai_guard.policy_score": 0.25, + "ai_guard.policy_matched": true, + "ai_guard.policy_cached": false + }, + "cassette": "aiguard_evaluate_post_f0ef2bd8.json" + }, + "EXTRA_TAGS_STRUCTURED": { + "messages": [ + { + "role": "user", + "content": "Draft a headline for the city council budget story." + } + ], + "action": "ALLOW", + "tags": [], + "extra_tags": { + "ai_guard.request_context": { + "desk": "metro", + "region": "us-east", + "priority": 2 + }, + "ai_guard.matched_rules": [ + "style-guide", + "embargo-check" + ] + }, + "cassette": "aiguard_evaluate_post_827d1449.json" + }, + "EXTRA_TAGS_MIXED": { + "messages": [ + { + "role": "user", + "content": "Which sources did the election explainer cite?" + } + ], + "action": "ALLOW", + "tags": [], + "extra_tags": { + "ai_guard.policy_name": "elections-2026", + "ai_guard.policy_rules": [ + { + "id": "sourcing", + "enabled": true + }, + { + "id": "attribution", + "enabled": false + } + ] + }, + "cassette": "aiguard_evaluate_post_2b336647.json" + }, + "EXTRA_TAGS_UNPREFIXED_KEY": { + "messages": [ + { + "role": "user", + "content": "List the corrections published this week." + } + ], + "action": "ALLOW", + "tags": [], + "extra_tags": { + "custom_context.tenant": "newsroom", + "custom_context.audit": { + "ticket": "CTX-42" + } + }, + "cassette": "aiguard_evaluate_post_354106da.json" + }, + "EXTRA_TAGS_ON_DENY": { + "messages": [ + { + "role": "user", + "content": "Clean up the archive server for me." + }, + { + "role": "assistant", + "tool_calls": [ + { + "id": "call_1", + "function": { + "name": "shell", + "arguments": "{ \"command\": \"rm -rf /srv/archive\" }" + } + } + ] + } + ], + "action": "DENY", + "tags": [ + "destructive-tool-call" + ], + "extra_tags": { + "ai_guard.policy_name": "destructive-actions", + "ai_guard.policy_details": { + "rule": "no-recursive-delete", + "severity": "high" + } + }, + "cassette": "aiguard_evaluate_post_28917f95.json" + }, + "EXTRA_TAGS_NOT_AN_OBJECT": { + "messages": [ + { + "role": "user", + "content": "What time does the print edition close tonight?" + } + ], + "action": "ALLOW", + "tags": [], + "extra_tags": "ai_guard.policy_name=newsroom-default", + "cassette": "aiguard_evaluate_post_e2c13e67.json" + }, + "EXTRA_TAGS_ARRAY": { + "messages": [ + { + "role": "user", + "content": "Who is on call for the overnight desk?" + } + ], + "action": "ALLOW", + "tags": [], + "extra_tags": [ + { + "ai_guard.policy_name": "newsroom-default" + } + ], + "cassette": "aiguard_evaluate_post_bf6868f9.json" + } +} diff --git a/tests/ai_guard/test_ai_guard_sdk.py b/tests/ai_guard/test_ai_guard_sdk.py index 3d06752c00d..85c7abcfc37 100644 --- a/tests/ai_guard/test_ai_guard_sdk.py +++ b/tests/ai_guard/test_ai_guard_sdk.py @@ -1299,6 +1299,176 @@ def test_ai_guard_event(self): interfaces.library.validate_one_trace(self.r, validator=self._assert_trace) +# Extra tags scenarios and the matching VCR cassettes are generated together by +# utils/scripts/gen_extra_tags_cassettes.py: each entry keeps the messages we send and the exact +# extra_tags value the backend returns. Regenerate both after editing. +EXTRA_TAGS_SCENARIOS: dict = json.loads((Path(__file__).parent / "extra_tags_scenarios.json").read_text()) + +# What the tracer itself writes to the ai_guard meta struct. Anything else there came from extra_tags. +TRACER_META_STRUCT_KEYS = frozenset(("messages", "attack_categories", "sds", "tag_probs")) + + +def _post_extra_tags_scenario(scenario_key: str, *, block: str = "false") -> tuple[dict, HttpResponse]: + scenario = EXTRA_TAGS_SCENARIOS[scenario_key] + response = weblog.post("/ai_guard/evaluate", headers={BLOCKING_HEADER: block}, json=scenario["messages"]) + return scenario, response + + +def _assert_scalar_extra_tag(span: DataDogLibrarySpan, key: str, expected: str | float | bool) -> None: # noqa: FBT001 + """A scalar extra tag is on the span under the key exactly as returned, in any faithful encoding. + + Tracers differ in where a number lands (a string in meta or a numeric metric) and how a boolean + is spelled, so the value is compared semantically rather than byte for byte. + """ + raw = _span_tag(span, key) + assert raw is not None, f"Extra tag '{key}' not set on the ai_guard span (meta: {span.get('meta')})" + if isinstance(expected, bool): + assert isinstance(raw, (bool, int, str)), f"Extra tag '{key}' should be a boolean, got '{raw}'" + assert is_same_boolean(actual=raw, expected=expected), f"Extra tag '{key}' should be '{expected}', got '{raw}'" + elif isinstance(expected, (int, float)): + assert isinstance(raw, (int, float, str)), f"Extra tag '{key}' should be a number, got '{raw}'" + assert float(raw) == expected, f"Extra tag '{key}' should be '{expected}', got '{raw}'" + else: + assert raw == expected, f"Extra tag '{key}' should be '{expected}', got '{raw}'" + + +def _assert_extra_tags_applied(extra_tags: dict): + """Validator asserting every extra tag landed where the contract puts it. + + Scalars become span tags and structured values go to the ai_guard meta struct, both under the + key exactly as the backend returned it: the backend owns the ai_guard prefix, so the tracer + neither adds it nor filters keys by namespace. + """ + + def validate(span: DataDogLibrarySpan): + if span["resource"] != "ai_guard": + return False + + ai_guard = _assert_key(span["meta_struct"], "ai_guard") + for key, value in extra_tags.items(): + if isinstance(value, (dict, list)): + assert key in ai_guard, f"Structured extra tag '{key}' not found in the ai_guard meta struct {ai_guard}" + assert ai_guard[key] == value, f"Extra tag '{key}' in meta struct: {ai_guard[key]} != {value}" + assert _span_tag(span, key) is None, f"Structured extra tag '{key}' should not be a span tag" + else: + _assert_scalar_extra_tag(span, key, value) + assert _span_tag(span, f"ai_guard.{key}") is None, f"The tracer must not prefix extra tag '{key}'" + return True + + return validate + + +@rfc("https://datadoghq.atlassian.net/browse/APPSEC-70724") +@features.ai_guard +@scenarios.ai_guard +class Test_ExtraTags: + """The AI Guard backend enriches the ai_guard span through extra_tags in the evaluate response. + + Lets an AI Guard policy attach its own context to traces without a tracer change per tag. + """ + + def setup_scalar_extra_tags(self): + self.scenario, self.r = _post_extra_tags_scenario("EXTRA_TAGS_SCALARS") + + def test_scalar_extra_tags(self): + """String, integer, float and boolean values are set as tags on the ai_guard span.""" + assert self.r.status_code == 200 + interfaces.library.validate_one_span( + self.r, validator=_assert_extra_tags_applied(self.scenario["extra_tags"]), full_trace=True + ) + + def setup_structured_extra_tags(self): + self.scenario, self.r = _post_extra_tags_scenario("EXTRA_TAGS_STRUCTURED") + + def test_structured_extra_tags(self): + """Object and array values are added to the ai_guard meta struct, not flattened into tags.""" + assert self.r.status_code == 200 + interfaces.library.validate_one_span( + self.r, validator=_assert_extra_tags_applied(self.scenario["extra_tags"]), full_trace=True + ) + + def setup_mixed_extra_tags(self): + self.scenario, self.r = _post_extra_tags_scenario("EXTRA_TAGS_MIXED") + + def test_mixed_extra_tags(self): + """One response mixing both kinds routes each value by its own type.""" + assert self.r.status_code == 200 + interfaces.library.validate_one_span( + self.r, validator=_assert_extra_tags_applied(self.scenario["extra_tags"]), full_trace=True + ) + + def setup_keys_not_prefixed_or_filtered(self): + self.scenario, self.r = _post_extra_tags_scenario("EXTRA_TAGS_UNPREFIXED_KEY") + + def test_keys_not_prefixed_or_filtered(self): + """Keys outside the ai_guard namespace land verbatim: namespacing is the backend's job, not the tracer's.""" + assert self.r.status_code == 200 + interfaces.library.validate_one_span( + self.r, validator=_assert_extra_tags_applied(self.scenario["extra_tags"]), full_trace=True + ) + + def setup_extra_tags_on_block(self): + self.scenario, self.r = _post_extra_tags_scenario("EXTRA_TAGS_ON_DENY", block="true") + + def test_extra_tags_on_block(self): + """A blocked evaluation still carries the extra tags, alongside ai_guard.blocked.""" + assert self.r.status_code == 403 + extra_tags_validator = _assert_extra_tags_applied(self.scenario["extra_tags"]) + + def validate(span: DataDogLibrarySpan): + if not extra_tags_validator(span): + return False + _assert_key(span["meta"], "ai_guard.action", "DENY") + assert is_same_boolean(actual=span["meta"].get("ai_guard.blocked"), expected="true"), ( + f"'ai_guard.blocked' with value 'true' not found in '{span['meta']}'" + ) + return True + + interfaces.library.validate_one_span(self.r, validator=validate, full_trace=True) + + +@rfc("https://datadoghq.atlassian.net/browse/APPSEC-70724") +@features.ai_guard +@scenarios.ai_guard +class Test_ExtraTagsFailSafe: + """A malformed extra_tags payload is ignored and never changes the evaluation. + + The field only enriches the span, so a backend shape the tracer does not understand must leave + the action, the SDK response and the span exactly as they would be without it. + """ + + def _assert_ignored(self, response: HttpResponse, scenario: dict) -> None: + assert response.status_code == 200 + _assert_key(json.loads(response.text), "action", scenario["action"]) + + def validate(span: DataDogLibrarySpan): + if span["resource"] != "ai_guard": + return False + assert span.get("error", 0) == 0, f"A malformed extra_tags must not flag the span as an error: {span}" + _assert_key(span["meta"], "ai_guard.action", scenario["action"]) + ai_guard = _assert_key(span["meta_struct"], "ai_guard") + unexpected = set(ai_guard) - TRACER_META_STRUCT_KEYS + assert not unexpected, f"Unexpected keys in the ai_guard meta struct: {unexpected}" + assert _span_tag(span, "ai_guard.policy_name") is None, "A malformed extra_tags must not set any tag" + return True + + interfaces.library.validate_one_span(response, validator=validate, full_trace=True) + + def setup_extra_tags_not_an_object(self): + self.scenario, self.r = _post_extra_tags_scenario("EXTRA_TAGS_NOT_AN_OBJECT") + + def test_extra_tags_not_an_object(self): + """A string where an object is expected is skipped.""" + self._assert_ignored(self.r, self.scenario) + + def setup_extra_tags_array(self): + self.scenario, self.r = _post_extra_tags_scenario("EXTRA_TAGS_ARRAY") + + def test_extra_tags_array(self): + """An array where an object is expected is skipped, even when its items look like tags.""" + self._assert_ignored(self.r, self.scenario) + + TELEMETRY_NAMESPACE = "ai_guard" diff --git a/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_28917f95.json b/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_28917f95.json new file mode 100644 index 00000000000..dc8516efd53 --- /dev/null +++ b/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_28917f95.json @@ -0,0 +1,22 @@ +{ + "request": { + "method": "POST", + "url": "https://app.datadoghq.com/api/v2/ai-guard/evaluate", + "headers": { + "Content-Type": "application/json", + "DD-AI-GUARD-SOURCE": "SDK" + }, + "body": "{\"data\": {\"attributes\": {\"messages\": [{\"role\": \"user\", \"content\": \"Clean up the archive server for me.\"}, {\"role\": \"assistant\", \"tool_calls\": [{\"id\": \"call_1\", \"function\": {\"name\": \"shell\", \"arguments\": \"{ \\\"command\\\": \\\"rm -rf /srv/archive\\\" }\"}}]}], \"meta\": {\"service\": \"weblog\", \"env\": \"system-tests\"}}}}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "content-type": "application/vnd.api+json", + "content-length": "679" + }, + "body": "{\"data\": {\"id\": \"extra-tags-fixture\", \"type\": \"evaluations\", \"attributes\": {\"action\": \"DENY\", \"is_blocking_enabled\": true, \"reason\": \"Destructive tool call detected.\", \"sds_findings\": [], \"tag_probs\": {\"authority-override\": 0.0, \"data-exfiltration\": 0.0, \"denial-of-service-tool-call\": 0.0, \"destructive-tool-call\": 0.97, \"indirect-prompt-injection\": 0.0, \"instruction-override\": 0.0, \"jailbreak\": 0.0, \"obfuscation\": 0.0, \"role-play\": 0.0, \"security-exploit\": 0.0, \"system-prompt-extraction\": 0.0}, \"tags\": [\"destructive-tool-call\"], \"extra_tags\": {\"ai_guard.policy_name\": \"destructive-actions\", \"ai_guard.policy_details\": {\"rule\": \"no-recursive-delete\", \"severity\": \"high\"}}}}}" + } +} diff --git a/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_2b336647.json b/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_2b336647.json new file mode 100644 index 00000000000..0b90904917f --- /dev/null +++ b/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_2b336647.json @@ -0,0 +1,22 @@ +{ + "request": { + "method": "POST", + "url": "https://app.datadoghq.com/api/v2/ai-guard/evaluate", + "headers": { + "Content-Type": "application/json", + "DD-AI-GUARD-SOURCE": "SDK" + }, + "body": "{\"data\": {\"attributes\": {\"messages\": [{\"role\": \"user\", \"content\": \"Which sources did the election explainer cite?\"}], \"meta\": {\"service\": \"weblog\", \"env\": \"system-tests\"}}}}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "content-type": "application/vnd.api+json", + "content-length": "659" + }, + "body": "{\"data\": {\"id\": \"extra-tags-fixture\", \"type\": \"evaluations\", \"attributes\": {\"action\": \"ALLOW\", \"is_blocking_enabled\": true, \"reason\": \"No rule match.\", \"sds_findings\": [], \"tag_probs\": {\"authority-override\": 0.0, \"data-exfiltration\": 0.0, \"denial-of-service-tool-call\": 0.0, \"destructive-tool-call\": 0.0, \"indirect-prompt-injection\": 0.0, \"instruction-override\": 0.0, \"jailbreak\": 0.0, \"obfuscation\": 0.0, \"role-play\": 0.0, \"security-exploit\": 0.0, \"system-prompt-extraction\": 0.0}, \"tags\": [], \"extra_tags\": {\"ai_guard.policy_name\": \"elections-2026\", \"ai_guard.policy_rules\": [{\"id\": \"sourcing\", \"enabled\": true}, {\"id\": \"attribution\", \"enabled\": false}]}}}}" + } +} diff --git a/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_354106da.json b/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_354106da.json new file mode 100644 index 00000000000..5f327011102 --- /dev/null +++ b/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_354106da.json @@ -0,0 +1,22 @@ +{ + "request": { + "method": "POST", + "url": "https://app.datadoghq.com/api/v2/ai-guard/evaluate", + "headers": { + "Content-Type": "application/json", + "DD-AI-GUARD-SOURCE": "SDK" + }, + "body": "{\"data\": {\"attributes\": {\"messages\": [{\"role\": \"user\", \"content\": \"List the corrections published this week.\"}], \"meta\": {\"service\": \"weblog\", \"env\": \"system-tests\"}}}}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "content-type": "application/vnd.api+json", + "content-length": "595" + }, + "body": "{\"data\": {\"id\": \"extra-tags-fixture\", \"type\": \"evaluations\", \"attributes\": {\"action\": \"ALLOW\", \"is_blocking_enabled\": true, \"reason\": \"No rule match.\", \"sds_findings\": [], \"tag_probs\": {\"authority-override\": 0.0, \"data-exfiltration\": 0.0, \"denial-of-service-tool-call\": 0.0, \"destructive-tool-call\": 0.0, \"indirect-prompt-injection\": 0.0, \"instruction-override\": 0.0, \"jailbreak\": 0.0, \"obfuscation\": 0.0, \"role-play\": 0.0, \"security-exploit\": 0.0, \"system-prompt-extraction\": 0.0}, \"tags\": [], \"extra_tags\": {\"custom_context.tenant\": \"newsroom\", \"custom_context.audit\": {\"ticket\": \"CTX-42\"}}}}}" + } +} diff --git a/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_827d1449.json b/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_827d1449.json new file mode 100644 index 00000000000..a18c8a60dc6 --- /dev/null +++ b/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_827d1449.json @@ -0,0 +1,22 @@ +{ + "request": { + "method": "POST", + "url": "https://app.datadoghq.com/api/v2/ai-guard/evaluate", + "headers": { + "Content-Type": "application/json", + "DD-AI-GUARD-SOURCE": "SDK" + }, + "body": "{\"data\": {\"attributes\": {\"messages\": [{\"role\": \"user\", \"content\": \"Draft a headline for the city council budget story.\"}], \"meta\": {\"service\": \"weblog\", \"env\": \"system-tests\"}}}}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "content-type": "application/vnd.api+json", + "content-length": "655" + }, + "body": "{\"data\": {\"id\": \"extra-tags-fixture\", \"type\": \"evaluations\", \"attributes\": {\"action\": \"ALLOW\", \"is_blocking_enabled\": true, \"reason\": \"No rule match.\", \"sds_findings\": [], \"tag_probs\": {\"authority-override\": 0.0, \"data-exfiltration\": 0.0, \"denial-of-service-tool-call\": 0.0, \"destructive-tool-call\": 0.0, \"indirect-prompt-injection\": 0.0, \"instruction-override\": 0.0, \"jailbreak\": 0.0, \"obfuscation\": 0.0, \"role-play\": 0.0, \"security-exploit\": 0.0, \"system-prompt-extraction\": 0.0}, \"tags\": [], \"extra_tags\": {\"ai_guard.request_context\": {\"desk\": \"metro\", \"region\": \"us-east\", \"priority\": 2}, \"ai_guard.matched_rules\": [\"style-guide\", \"embargo-check\"]}}}}" + } +} diff --git a/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_bf6868f9.json b/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_bf6868f9.json new file mode 100644 index 00000000000..6d55035586e --- /dev/null +++ b/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_bf6868f9.json @@ -0,0 +1,22 @@ +{ + "request": { + "method": "POST", + "url": "https://app.datadoghq.com/api/v2/ai-guard/evaluate", + "headers": { + "Content-Type": "application/json", + "DD-AI-GUARD-SOURCE": "SDK" + }, + "body": "{\"data\": {\"attributes\": {\"messages\": [{\"role\": \"user\", \"content\": \"Who is on call for the overnight desk?\"}], \"meta\": {\"service\": \"weblog\", \"env\": \"system-tests\"}}}}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "content-type": "application/vnd.api+json", + "content-length": "558" + }, + "body": "{\"data\": {\"id\": \"extra-tags-fixture\", \"type\": \"evaluations\", \"attributes\": {\"action\": \"ALLOW\", \"is_blocking_enabled\": true, \"reason\": \"No rule match.\", \"sds_findings\": [], \"tag_probs\": {\"authority-override\": 0.0, \"data-exfiltration\": 0.0, \"denial-of-service-tool-call\": 0.0, \"destructive-tool-call\": 0.0, \"indirect-prompt-injection\": 0.0, \"instruction-override\": 0.0, \"jailbreak\": 0.0, \"obfuscation\": 0.0, \"role-play\": 0.0, \"security-exploit\": 0.0, \"system-prompt-extraction\": 0.0}, \"tags\": [], \"extra_tags\": [{\"ai_guard.policy_name\": \"newsroom-default\"}]}}}" + } +} diff --git a/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_e2c13e67.json b/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_e2c13e67.json new file mode 100644 index 00000000000..936648f304c --- /dev/null +++ b/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_e2c13e67.json @@ -0,0 +1,22 @@ +{ + "request": { + "method": "POST", + "url": "https://app.datadoghq.com/api/v2/ai-guard/evaluate", + "headers": { + "Content-Type": "application/json", + "DD-AI-GUARD-SOURCE": "SDK" + }, + "body": "{\"data\": {\"attributes\": {\"messages\": [{\"role\": \"user\", \"content\": \"What time does the print edition close tonight?\"}], \"meta\": {\"service\": \"weblog\", \"env\": \"system-tests\"}}}}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "content-type": "application/vnd.api+json", + "content-length": "551" + }, + "body": "{\"data\": {\"id\": \"extra-tags-fixture\", \"type\": \"evaluations\", \"attributes\": {\"action\": \"ALLOW\", \"is_blocking_enabled\": true, \"reason\": \"No rule match.\", \"sds_findings\": [], \"tag_probs\": {\"authority-override\": 0.0, \"data-exfiltration\": 0.0, \"denial-of-service-tool-call\": 0.0, \"destructive-tool-call\": 0.0, \"indirect-prompt-injection\": 0.0, \"instruction-override\": 0.0, \"jailbreak\": 0.0, \"obfuscation\": 0.0, \"role-play\": 0.0, \"security-exploit\": 0.0, \"system-prompt-extraction\": 0.0}, \"tags\": [], \"extra_tags\": \"ai_guard.policy_name=newsroom-default\"}}}" + } +} diff --git a/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_f0ef2bd8.json b/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_f0ef2bd8.json new file mode 100644 index 00000000000..23fe1bbdd47 --- /dev/null +++ b/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_f0ef2bd8.json @@ -0,0 +1,22 @@ +{ + "request": { + "method": "POST", + "url": "https://app.datadoghq.com/api/v2/ai-guard/evaluate", + "headers": { + "Content-Type": "application/json", + "DD-AI-GUARD-SOURCE": "SDK" + }, + "body": "{\"data\": {\"attributes\": {\"messages\": [{\"role\": \"user\", \"content\": \"Summarize the latest newsroom style guide changes.\"}], \"meta\": {\"service\": \"weblog\", \"env\": \"system-tests\"}}}}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "content-type": "application/vnd.api+json", + "content-length": "684" + }, + "body": "{\"data\": {\"id\": \"extra-tags-fixture\", \"type\": \"evaluations\", \"attributes\": {\"action\": \"ALLOW\", \"is_blocking_enabled\": true, \"reason\": \"No rule match.\", \"sds_findings\": [], \"tag_probs\": {\"authority-override\": 0.0, \"data-exfiltration\": 0.0, \"denial-of-service-tool-call\": 0.0, \"destructive-tool-call\": 0.0, \"indirect-prompt-injection\": 0.0, \"instruction-override\": 0.0, \"jailbreak\": 0.0, \"obfuscation\": 0.0, \"role-play\": 0.0, \"security-exploit\": 0.0, \"system-prompt-extraction\": 0.0}, \"tags\": [], \"extra_tags\": {\"ai_guard.policy_name\": \"newsroom-default\", \"ai_guard.policy_revision\": 7, \"ai_guard.policy_score\": 0.25, \"ai_guard.policy_matched\": true, \"ai_guard.policy_cached\": false}}}}" + } +} diff --git a/utils/scripts/gen_extra_tags_cassettes.py b/utils/scripts/gen_extra_tags_cassettes.py new file mode 100755 index 00000000000..4718bcd484d --- /dev/null +++ b/utils/scripts/gen_extra_tags_cassettes.py @@ -0,0 +1,220 @@ +#!/usr/bin/env python3 +"""Generate hand-crafted AI Guard VCR cassettes for backend-provided extra_tags. + +The evaluate response can carry an optional data.attributes.extra_tags object: tags the AI Guard +backend wants on the ai_guard span, so a new evaluation detail reaches traces without a tracer +release per tag (APPSEC-70724). The backend owns the ai_guard prefix and the namespace, so the +tracer contract is thin: + + - scalar values (string, number, boolean) become ai_guard span tags under the key exactly as + returned, with no prefix added and no namespace filtering; + - structured values (objects, arrays) are added to the ai_guard meta struct under their key. + +The backend does not return extra_tags yet, so no cassette can be recorded: this script writes them +by hand, reusing the request hashing and rendering of gen_redaction_cassettes.py, and writes +tests/ai_guard/extra_tags_scenarios.json, the sidecar the tests read the scenarios from. The two +are generated together because a cassette is addressed by the hash of its request body: a scenario +whose messages drift from its cassette would never be replayed again. + +Run from the repo root: python3 utils/scripts/gen_extra_tags_cassettes.py +Pass --check to report drift without writing anything (used by format.sh --check). +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +from gen_redaction_cassettes import ( + CASSETTES_DIR, + FOREIGN_CASSETTES, + NO_MATCH_REASON, + _cassette_name, + _tag_probs_for, + _wrap_request_body, + render_cassette, +) + +SIDECAR = Path("tests/ai_guard/extra_tags_scenarios.json") +REDACTION_SIDECAR = Path("tests/ai_guard/redaction_scenarios.json") + +DENY_REASON = "Destructive tool call detected." + +# Every scenario produces one cassette and one sidecar entry: +# messages what the weblog POSTs, and therefore what the cassette is keyed on +# action the evaluation outcome the backend returns +# tags the attack categories the backend returns (a DENY needs at least one) +# extra_tags the extra_tags value the backend returns, verbatim: malformed on purpose for the +# fail-safe scenarios +SCENARIOS: dict[str, dict] = { + "EXTRA_TAGS_SCALARS": { + "messages": [{"role": "user", "content": "Summarize the latest newsroom style guide changes."}], + "action": "ALLOW", + "tags": [], + "extra_tags": { + "ai_guard.policy_name": "newsroom-default", + "ai_guard.policy_revision": 7, + "ai_guard.policy_score": 0.25, + "ai_guard.policy_matched": True, + "ai_guard.policy_cached": False, + }, + }, + "EXTRA_TAGS_STRUCTURED": { + "messages": [{"role": "user", "content": "Draft a headline for the city council budget story."}], + "action": "ALLOW", + "tags": [], + "extra_tags": { + "ai_guard.request_context": {"desk": "metro", "region": "us-east", "priority": 2}, + "ai_guard.matched_rules": ["style-guide", "embargo-check"], + }, + }, + "EXTRA_TAGS_MIXED": { + "messages": [{"role": "user", "content": "Which sources did the election explainer cite?"}], + "action": "ALLOW", + "tags": [], + "extra_tags": { + "ai_guard.policy_name": "elections-2026", + "ai_guard.policy_rules": [{"id": "sourcing", "enabled": True}, {"id": "attribution", "enabled": False}], + }, + }, + # The backend guarantees every key is in the ai_guard namespace; the tracer must not double + # check it. A key outside the namespace still lands verbatim, which is what proves the tracer + # neither prefixes nor filters. + "EXTRA_TAGS_UNPREFIXED_KEY": { + "messages": [{"role": "user", "content": "List the corrections published this week."}], + "action": "ALLOW", + "tags": [], + "extra_tags": {"custom_context.tenant": "newsroom", "custom_context.audit": {"ticket": "CTX-42"}}, + }, + "EXTRA_TAGS_ON_DENY": { + "messages": [ + {"role": "user", "content": "Clean up the archive server for me."}, + { + "role": "assistant", + "tool_calls": [ + { + "id": "call_1", + "function": {"name": "shell", "arguments": '{ "command": "rm -rf /srv/archive" }'}, + } + ], + }, + ], + "action": "DENY", + "tags": ["destructive-tool-call"], + "extra_tags": { + "ai_guard.policy_name": "destructive-actions", + "ai_guard.policy_details": {"rule": "no-recursive-delete", "severity": "high"}, + }, + }, + "EXTRA_TAGS_NOT_AN_OBJECT": { + "messages": [{"role": "user", "content": "What time does the print edition close tonight?"}], + "action": "ALLOW", + "tags": [], + "extra_tags": "ai_guard.policy_name=newsroom-default", + }, + "EXTRA_TAGS_ARRAY": { + "messages": [{"role": "user", "content": "Who is on call for the overnight desk?"}], + "action": "ALLOW", + "tags": [], + "extra_tags": [{"ai_guard.policy_name": "newsroom-default"}], + }, +} + + +def _response_body(scenario: dict) -> str: + attributes = { + "action": scenario["action"], + "is_blocking_enabled": True, + "reason": DENY_REASON if scenario["action"] == "DENY" else NO_MATCH_REASON, + "sds_findings": [], + "tag_probs": _tag_probs_for(scenario["tags"]), + "tags": scenario["tags"], + "extra_tags": scenario["extra_tags"], + } + return json.dumps({"data": {"id": "extra-tags-fixture", "type": "evaluations", "attributes": attributes}}) + + +def _owned_by_redaction() -> set[str]: + if not REDACTION_SIDECAR.is_file(): + return set() + return {entry["cassette"] for entry in json.loads(REDACTION_SIDECAR.read_text()).values()} + + +def _previously_generated() -> set[str]: + if not SIDECAR.is_file(): + return set() + return {entry["cassette"] for entry in json.loads(SIDECAR.read_text()).values()} + + +def build_corpus(previously_generated: set[str]) -> tuple[dict[str, str], dict[str, dict], list[str]]: + """Build every cassette and the sidecar in memory, raising on any collision with another test.""" + foreign = dict.fromkeys(_owned_by_redaction(), "the redaction scenarios") | FOREIGN_CASSETTES + contents: dict[str, str] = {} + sidecar: dict[str, dict] = {} + owners: dict[str, str] = {} + for name, scenario in SCENARIOS.items(): + assert scenario["action"] == "ALLOW" or scenario["tags"], f"{name}: a {scenario['action']} needs a tag" + request_body = _wrap_request_body(scenario["messages"]) + cassette = _cassette_name(request_body) + assert cassette not in owners, f"{name} and {owners[cassette]} collide on {cassette}" + assert cassette not in foreign, f"{name} sends the same messages as {foreign[cassette]}: change them" + path = CASSETTES_DIR / cassette + assert not (path.is_file() and cassette not in previously_generated and previously_generated), ( + f"{name} would overwrite {cassette}, which this script did not generate: change its messages" + ) + contents[cassette] = render_cassette(request_body, _response_body(scenario)) + owners[cassette] = name + sidecar[name] = {**scenario, "cassette": cassette} + + orphans = [name for name in sorted(previously_generated - set(contents)) if (CASSETTES_DIR / name).is_file()] + return contents, sidecar, orphans + + +def _render_sidecar(sidecar: dict[str, dict]) -> str: + return json.dumps(sidecar, indent=2) + "\n" + + +def _drift(contents: dict[str, str], sidecar: dict[str, dict], orphans: list[str]) -> list[str]: + differences = [f"{name} (would be deleted)" for name in orphans] + for name, content in sorted(contents.items()): + path = CASSETTES_DIR / name + if not path.is_file(): + differences.append(f"{name} (missing)") + elif path.read_text() != content: + differences.append(f"{name} (outdated)") + if not SIDECAR.is_file() or SIDECAR.read_text() != _render_sidecar(sidecar): + differences.append(f"{SIDECAR} (outdated)") + return differences + + +def main(argv: list[str]) -> int: + check = "--check" in argv[1:] + assert CASSETTES_DIR.is_dir(), f"missing {CASSETTES_DIR}; run from repo root" + contents, sidecar, orphans = build_corpus(_previously_generated()) + + if check: + differences = _drift(contents, sidecar, orphans) + if differences: + print("AI Guard extra_tags fixtures are out of date:") + for difference in differences: + print(f" {difference}") + return 1 + print(f"{len(sidecar)} scenarios in sync with {SIDECAR}") + return 0 + + for name, content in contents.items(): + (CASSETTES_DIR / name).write_text(content) + for orphan in orphans: + (CASSETTES_DIR / orphan).unlink() + print(f"removed orphan cassette {orphan}") + SIDECAR.write_text(_render_sidecar(sidecar)) + + for name, entry in sidecar.items(): + print(f"{name:28s} -> {entry['cassette']}") + print(f"\nwrote {len(sidecar)} scenarios -> {SIDECAR}") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) From 8d0001f2108a3c1ebc0f0c5174bf0a5a2d5608c9 Mon Sep 17 00:00:00 2001 From: Alberto Vara Date: Thu, 22 Oct 2026 16:01:26 +0200 Subject: [PATCH 2/5] AI Guard: keep extra_tags fail-safe tests out of easy-win activation A tracer without extra_tags support also ignores a malformed payload, so Test_ExtraTagsFailSafe XPASSes under missing_feature (verified on Node.js) and the easy-win bot would wrongly declare the feature supported. Co-Authored-By: Claude Opus 5.5 --- utils/scripts/activate_easy_wins/skip.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/utils/scripts/activate_easy_wins/skip.yml b/utils/scripts/activate_easy_wins/skip.yml index a83e8ee3b19..811cd864fbf 100644 --- a/utils/scripts/activate_easy_wins/skip.yml +++ b/utils/scripts/activate_easy_wins/skip.yml @@ -1,5 +1,8 @@ "*": - tests/parametric/test_crashtracking.py::Test_Crashtracking::test_disable_crashtracking + # A tracer without extra_tags support also ignores a malformed payload, so these pass vacuously + - tests/ai_guard/test_ai_guard_sdk.py::Test_ExtraTagsFailSafe::test_extra_tags_not_an_object + - tests/ai_guard/test_ai_guard_sdk.py::Test_ExtraTagsFailSafe::test_extra_tags_array "nodejs": - tests/test_telemetry.py::Test_DependencyEnable::test_app_dependency_loaded_not_sent_dependency_collection_disabled From c0a1b8e4d9cc71e4066d5b6d066bdfa3f4f01bd3 Mon Sep 17 00:00:00 2001 From: Alberto Vara Date: Thu, 22 Oct 2026 16:13:54 +0200 Subject: [PATCH 3/5] AI Guard: accept 1.0/0.0 metrics for boolean extra tags Span metrics decode as float64, so a tracer that stores a boolean extra tag as a metric reports 1.0/0.0, which the type guard rejected. Reuse the redacted-tag check, which accepts exactly 1/0 from metrics, for both. Co-Authored-By: Claude Opus 5.5 --- tests/ai_guard/test_ai_guard_sdk.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/tests/ai_guard/test_ai_guard_sdk.py b/tests/ai_guard/test_ai_guard_sdk.py index 85c7abcfc37..f3b8de8e3ff 100644 --- a/tests/ai_guard/test_ai_guard_sdk.py +++ b/tests/ai_guard/test_ai_guard_sdk.py @@ -541,19 +541,25 @@ def _span_tag(span: DataDogLibrarySpan, key: str) -> object | None: return None -def _assert_redacted_tag(span: DataDogLibrarySpan, *, expected: bool) -> None: - """The ai_guard.redacted span tag must state whether the evaluation redacted anything.""" - raw = _span_tag(span, REDACTED_TAG) - assert raw is not None, f"'{REDACTED_TAG}' not set on the ai_guard span" +def _assert_boolean_tag(span: DataDogLibrarySpan, key: str, *, expected: bool) -> None: + """A boolean span tag, whether the tracer wrote it to meta or to metrics, must have the expected value.""" + raw = _span_tag(span, key) + assert raw is not None, f"'{key}' not set on the ai_guard span" if isinstance(raw, (int, float)) and not isinstance(raw, bool): - # A tracer reporting the tag as a metric uses exactly 1 or 0: the tag is a boolean, so - # any other number is a bug rather than a truthy value to be coerced. - assert raw in (0, 1), f"'{REDACTED_TAG}' reported as a metric should be 1 or 0, got '{raw}'" + # A tracer reporting the tag as a metric uses exactly 1 or 0 (decoded as 1.0 or 0.0, metrics + # being float64): the tag is a boolean, so any other number is a bug rather than a truthy + # value to be coerced. + assert raw in (0, 1), f"'{key}' reported as a metric should be 1 or 0, got '{raw}'" actual: bool | str = raw == 1 else: - assert isinstance(raw, (bool, str)), f"'{REDACTED_TAG}' should be a boolean or a string, got '{raw}'" + assert isinstance(raw, (bool, str)), f"'{key}' should be a boolean or a string, got '{raw}'" actual = raw - assert is_same_boolean(actual=actual, expected=expected), f"'{REDACTED_TAG}' should be '{expected}', got '{actual}'" + assert is_same_boolean(actual=actual, expected=expected), f"'{key}' should be '{expected}', got '{actual}'" + + +def _assert_redacted_tag(span: DataDogLibrarySpan, *, expected: bool) -> None: + """The ai_guard.redacted span tag must state whether the evaluation redacted anything.""" + _assert_boolean_tag(span, REDACTED_TAG, expected=expected) def _assert_redaction_scenario(scenario: dict): @@ -1323,8 +1329,7 @@ def _assert_scalar_extra_tag(span: DataDogLibrarySpan, key: str, expected: str | raw = _span_tag(span, key) assert raw is not None, f"Extra tag '{key}' not set on the ai_guard span (meta: {span.get('meta')})" if isinstance(expected, bool): - assert isinstance(raw, (bool, int, str)), f"Extra tag '{key}' should be a boolean, got '{raw}'" - assert is_same_boolean(actual=raw, expected=expected), f"Extra tag '{key}' should be '{expected}', got '{raw}'" + _assert_boolean_tag(span, key, expected=expected) elif isinstance(expected, (int, float)): assert isinstance(raw, (int, float, str)), f"Extra tag '{key}' should be a number, got '{raw}'" assert float(raw) == expected, f"Extra tag '{key}' should be '{expected}', got '{raw}'" From 2be6bea45a66c673bb1c2df0559e7eed39769631 Mon Sep 17 00:00:00 2001 From: Alberto Vara Date: Mon, 26 Oct 2026 15:59:36 +0100 Subject: [PATCH 4/5] AI Guard: test extra_tags and extra_metastruct as key/value lists The backend contract for APPSEC-70724 changed: the evaluate response carries two separate attributes, both lists of {key, value} objects, since dotted keys such as ai_guard.evaluate_id are not valid JSON:API member names. - extra_tags: string values, set as ai_guard span tags under the key exactly as returned - extra_metastruct: any value, added to the ai_guard meta_struct under the key exactly as returned - the evaluation ID is sent in both, as ai_guard.evaluate_id and evaluate_id, matching data.id Regenerate the hand-written cassettes for the list shape, with a stable UUID as data.id per scenario, and rewrite Test_ExtraTags and Test_ExtraTagsFailSafe accordingly. The fail-safe tests now also assert that the well-formed part of the response is still applied, so they no longer pass vacuously and the easy-win skip entries are dropped. Co-Authored-By: Claude Opus 5.5 --- tests/ai_guard/extra_tags_scenarios.json | 315 ++++++++++++++++-- tests/ai_guard/test_ai_guard_sdk.py | 184 +++++----- .../aiguard_evaluate_post_19cfc9ce.json | 22 ++ .../aiguard_evaluate_post_28917f95.json | 4 +- .../aiguard_evaluate_post_2b336647.json | 4 +- .../aiguard_evaluate_post_354106da.json | 4 +- .../aiguard_evaluate_post_827d1449.json | 4 +- .../aiguard_evaluate_post_bf6868f9.json | 4 +- .../aiguard_evaluate_post_e2c13e67.json | 4 +- .../aiguard_evaluate_post_f0ef2bd8.json | 4 +- utils/scripts/activate_easy_wins/skip.yml | 3 - utils/scripts/gen_extra_tags_cassettes.py | 192 ++++++++--- 12 files changed, 549 insertions(+), 195 deletions(-) create mode 100644 utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_19cfc9ce.json diff --git a/tests/ai_guard/extra_tags_scenarios.json b/tests/ai_guard/extra_tags_scenarios.json index eb3e1655569..5bacda63f85 100644 --- a/tests/ai_guard/extra_tags_scenarios.json +++ b/tests/ai_guard/extra_tags_scenarios.json @@ -1,5 +1,6 @@ { - "EXTRA_TAGS_SCALARS": { + "EXTRA_TAGS": { + "evaluation_id": "13c92662-52c8-5959-8fec-fb7d6bb5b5d0", "messages": [ { "role": "user", @@ -8,16 +9,26 @@ ], "action": "ALLOW", "tags": [], - "extra_tags": { + "extra_tags": [ + { + "key": "ai_guard.policy_name", + "value": "newsroom-default" + }, + { + "key": "ai_guard.policy_revision", + "value": "7" + } + ], + "expected_tags": { "ai_guard.policy_name": "newsroom-default", - "ai_guard.policy_revision": 7, - "ai_guard.policy_score": 0.25, - "ai_guard.policy_matched": true, - "ai_guard.policy_cached": false + "ai_guard.policy_revision": "7" }, + "expected_metastruct": {}, + "skipped_tags": [], "cassette": "aiguard_evaluate_post_f0ef2bd8.json" }, - "EXTRA_TAGS_STRUCTURED": { + "EXTRA_METASTRUCT": { + "evaluation_id": "73606e72-26b6-52f8-8652-7f321db98dda", "messages": [ { "role": "user", @@ -26,20 +37,73 @@ ], "action": "ALLOW", "tags": [], - "extra_tags": { - "ai_guard.request_context": { + "extra_metastruct": [ + { + "key": "request_context", + "value": { + "desk": "metro", + "region": "us-east", + "priority": 2 + } + }, + { + "key": "matched_rules", + "value": [ + "style-guide", + "embargo-check" + ] + }, + { + "key": "policy_rules", + "value": [ + { + "id": "sourcing", + "enabled": true + }, + { + "id": "attribution", + "enabled": false + } + ] + }, + { + "key": "policy_score", + "value": 0.25 + }, + { + "key": "policy_name", + "value": "newsroom-default" + } + ], + "expected_tags": {}, + "expected_metastruct": { + "request_context": { "desk": "metro", "region": "us-east", "priority": 2 }, - "ai_guard.matched_rules": [ + "matched_rules": [ "style-guide", "embargo-check" - ] + ], + "policy_rules": [ + { + "id": "sourcing", + "enabled": true + }, + { + "id": "attribution", + "enabled": false + } + ], + "policy_score": 0.25, + "policy_name": "newsroom-default" }, + "skipped_tags": [], "cassette": "aiguard_evaluate_post_827d1449.json" }, - "EXTRA_TAGS_MIXED": { + "EVALUATE_ID": { + "evaluation_id": "9c9036fc-d7a1-5d5e-bcb3-7d62cc9fa218", "messages": [ { "role": "user", @@ -48,22 +112,43 @@ ], "action": "ALLOW", "tags": [], - "extra_tags": { - "ai_guard.policy_name": "elections-2026", - "ai_guard.policy_rules": [ - { - "id": "sourcing", - "enabled": true - }, - { - "id": "attribution", - "enabled": false + "extra_tags": [ + { + "key": "ai_guard.evaluate_id", + "value": "9c9036fc-d7a1-5d5e-bcb3-7d62cc9fa218" + }, + { + "key": "ai_guard.policy_name", + "value": "elections-2026" + } + ], + "extra_metastruct": [ + { + "key": "evaluate_id", + "value": "9c9036fc-d7a1-5d5e-bcb3-7d62cc9fa218" + }, + { + "key": "policy", + "value": { + "name": "elections-2026" } - ] + } + ], + "expected_tags": { + "ai_guard.evaluate_id": "9c9036fc-d7a1-5d5e-bcb3-7d62cc9fa218", + "ai_guard.policy_name": "elections-2026" + }, + "expected_metastruct": { + "evaluate_id": "9c9036fc-d7a1-5d5e-bcb3-7d62cc9fa218", + "policy": { + "name": "elections-2026" + } }, + "skipped_tags": [], "cassette": "aiguard_evaluate_post_2b336647.json" }, - "EXTRA_TAGS_UNPREFIXED_KEY": { + "KEYS_VERBATIM": { + "evaluation_id": "92a51623-de7f-5850-8f99-2a91d343bc0a", "messages": [ { "role": "user", @@ -72,15 +157,33 @@ ], "action": "ALLOW", "tags": [], - "extra_tags": { - "custom_context.tenant": "newsroom", - "custom_context.audit": { + "extra_tags": [ + { + "key": "custom_context.tenant", + "value": "newsroom" + } + ], + "extra_metastruct": [ + { + "key": "audit", + "value": { + "ticket": "CTX-42" + } + } + ], + "expected_tags": { + "custom_context.tenant": "newsroom" + }, + "expected_metastruct": { + "audit": { "ticket": "CTX-42" } }, + "skipped_tags": [], "cassette": "aiguard_evaluate_post_354106da.json" }, - "EXTRA_TAGS_ON_DENY": { + "ON_DENY": { + "evaluation_id": "55e3505d-93eb-5171-8166-90c709865d83", "messages": [ { "role": "user", @@ -103,16 +206,45 @@ "tags": [ "destructive-tool-call" ], - "extra_tags": { - "ai_guard.policy_name": "destructive-actions", - "ai_guard.policy_details": { + "extra_tags": [ + { + "key": "ai_guard.evaluate_id", + "value": "55e3505d-93eb-5171-8166-90c709865d83" + }, + { + "key": "ai_guard.policy_name", + "value": "destructive-actions" + } + ], + "extra_metastruct": [ + { + "key": "evaluate_id", + "value": "55e3505d-93eb-5171-8166-90c709865d83" + }, + { + "key": "policy_details", + "value": { + "rule": "no-recursive-delete", + "severity": "high" + } + } + ], + "expected_tags": { + "ai_guard.evaluate_id": "55e3505d-93eb-5171-8166-90c709865d83", + "ai_guard.policy_name": "destructive-actions" + }, + "expected_metastruct": { + "evaluate_id": "55e3505d-93eb-5171-8166-90c709865d83", + "policy_details": { "rule": "no-recursive-delete", "severity": "high" } }, + "skipped_tags": [], "cassette": "aiguard_evaluate_post_28917f95.json" }, - "EXTRA_TAGS_NOT_AN_OBJECT": { + "EXTRA_TAGS_NOT_A_LIST": { + "evaluation_id": "a8524c91-cec1-5ee6-812f-bd3b4fefae2b", "messages": [ { "role": "user", @@ -121,10 +253,30 @@ ], "action": "ALLOW", "tags": [], - "extra_tags": "ai_guard.policy_name=newsroom-default", + "extra_tags": { + "ai_guard.policy_name": "newsroom-default" + }, + "extra_metastruct": [ + { + "key": "policy", + "value": { + "name": "newsroom-default" + } + } + ], + "expected_tags": {}, + "expected_metastruct": { + "policy": { + "name": "newsroom-default" + } + }, + "skipped_tags": [ + "ai_guard.policy_name" + ], "cassette": "aiguard_evaluate_post_e2c13e67.json" }, - "EXTRA_TAGS_ARRAY": { + "EXTRA_METASTRUCT_NOT_A_LIST": { + "evaluation_id": "e72731f7-4c56-56b4-95a9-ed8d009cebcb", "messages": [ { "role": "user", @@ -135,9 +287,102 @@ "tags": [], "extra_tags": [ { - "ai_guard.policy_name": "newsroom-default" + "key": "ai_guard.policy_name", + "value": "newsroom-default" } ], + "extra_metastruct": { + "policy": { + "name": "newsroom-default" + } + }, + "expected_tags": { + "ai_guard.policy_name": "newsroom-default" + }, + "expected_metastruct": {}, + "skipped_tags": [], "cassette": "aiguard_evaluate_post_bf6868f9.json" + }, + "MALFORMED_ENTRIES": { + "evaluation_id": "f55f1430-86f7-5441-96b5-9938f4a10261", + "messages": [ + { + "role": "user", + "content": "When does the weekend magazine go to print?" + } + ], + "action": "ALLOW", + "tags": [], + "extra_tags": [ + "ai_guard.policy_name=newsroom-default", + [ + "ai_guard.policy_name", + "newsroom-default" + ], + { + "value": "missing key" + }, + { + "key": 1, + "value": "integer key" + }, + { + "key": "", + "value": "empty key" + }, + { + "key": "ai_guard.policy_revision", + "value": 7 + }, + { + "key": "ai_guard.policy_details", + "value": { + "rule": "embargo-check" + } + }, + { + "key": "ai_guard.policy_name", + "value": "newsroom-default" + } + ], + "extra_metastruct": [ + "policy=newsroom-default", + { + "value": { + "name": "missing key" + } + }, + { + "key": 1, + "value": { + "name": "integer key" + } + }, + { + "key": "", + "value": { + "name": "empty key" + } + }, + { + "key": "policy", + "value": { + "name": "newsroom-default" + } + } + ], + "expected_tags": { + "ai_guard.policy_name": "newsroom-default" + }, + "expected_metastruct": { + "policy": { + "name": "newsroom-default" + } + }, + "skipped_tags": [ + "ai_guard.policy_revision", + "ai_guard.policy_details" + ], + "cassette": "aiguard_evaluate_post_19cfc9ce.json" } } diff --git a/tests/ai_guard/test_ai_guard_sdk.py b/tests/ai_guard/test_ai_guard_sdk.py index f3b8de8e3ff..064ec50f39e 100644 --- a/tests/ai_guard/test_ai_guard_sdk.py +++ b/tests/ai_guard/test_ai_guard_sdk.py @@ -541,25 +541,19 @@ def _span_tag(span: DataDogLibrarySpan, key: str) -> object | None: return None -def _assert_boolean_tag(span: DataDogLibrarySpan, key: str, *, expected: bool) -> None: - """A boolean span tag, whether the tracer wrote it to meta or to metrics, must have the expected value.""" - raw = _span_tag(span, key) - assert raw is not None, f"'{key}' not set on the ai_guard span" +def _assert_redacted_tag(span: DataDogLibrarySpan, *, expected: bool) -> None: + """The ai_guard.redacted span tag must state whether the evaluation redacted anything.""" + raw = _span_tag(span, REDACTED_TAG) + assert raw is not None, f"'{REDACTED_TAG}' not set on the ai_guard span" if isinstance(raw, (int, float)) and not isinstance(raw, bool): - # A tracer reporting the tag as a metric uses exactly 1 or 0 (decoded as 1.0 or 0.0, metrics - # being float64): the tag is a boolean, so any other number is a bug rather than a truthy - # value to be coerced. - assert raw in (0, 1), f"'{key}' reported as a metric should be 1 or 0, got '{raw}'" + # A tracer reporting the tag as a metric uses exactly 1 or 0: the tag is a boolean, so + # any other number is a bug rather than a truthy value to be coerced. + assert raw in (0, 1), f"'{REDACTED_TAG}' reported as a metric should be 1 or 0, got '{raw}'" actual: bool | str = raw == 1 else: - assert isinstance(raw, (bool, str)), f"'{key}' should be a boolean or a string, got '{raw}'" + assert isinstance(raw, (bool, str)), f"'{REDACTED_TAG}' should be a boolean or a string, got '{raw}'" actual = raw - assert is_same_boolean(actual=actual, expected=expected), f"'{key}' should be '{expected}', got '{actual}'" - - -def _assert_redacted_tag(span: DataDogLibrarySpan, *, expected: bool) -> None: - """The ai_guard.redacted span tag must state whether the evaluation redacted anything.""" - _assert_boolean_tag(span, REDACTED_TAG, expected=expected) + assert is_same_boolean(actual=actual, expected=expected), f"'{REDACTED_TAG}' should be '{expected}', got '{actual}'" def _assert_redaction_scenario(scenario: dict): @@ -1306,11 +1300,12 @@ def test_ai_guard_event(self): # Extra tags scenarios and the matching VCR cassettes are generated together by -# utils/scripts/gen_extra_tags_cassettes.py: each entry keeps the messages we send and the exact -# extra_tags value the backend returns. Regenerate both after editing. +# utils/scripts/gen_extra_tags_cassettes.py: each entry keeps the messages we send, the exact +# extra_tags and extra_metastruct values the backend returns, and what the tracer must apply from +# them. Regenerate both after editing. EXTRA_TAGS_SCENARIOS: dict = json.loads((Path(__file__).parent / "extra_tags_scenarios.json").read_text()) -# What the tracer itself writes to the ai_guard meta struct. Anything else there came from extra_tags. +# What the tracer itself writes to the ai_guard meta struct. Anything else there came from extra_metastruct. TRACER_META_STRUCT_KEYS = frozenset(("messages", "attack_categories", "sds", "tag_probs")) @@ -1320,44 +1315,42 @@ def _post_extra_tags_scenario(scenario_key: str, *, block: str = "false") -> tup return scenario, response -def _assert_scalar_extra_tag(span: DataDogLibrarySpan, key: str, expected: str | float | bool) -> None: # noqa: FBT001 - """A scalar extra tag is on the span under the key exactly as returned, in any faithful encoding. - - Tracers differ in where a number lands (a string in meta or a numeric metric) and how a boolean - is spelled, so the value is compared semantically rather than byte for byte. - """ - raw = _span_tag(span, key) - assert raw is not None, f"Extra tag '{key}' not set on the ai_guard span (meta: {span.get('meta')})" - if isinstance(expected, bool): - _assert_boolean_tag(span, key, expected=expected) - elif isinstance(expected, (int, float)): - assert isinstance(raw, (int, float, str)), f"Extra tag '{key}' should be a number, got '{raw}'" - assert float(raw) == expected, f"Extra tag '{key}' should be '{expected}', got '{raw}'" - else: - assert raw == expected, f"Extra tag '{key}' should be '{expected}', got '{raw}'" - - -def _assert_extra_tags_applied(extra_tags: dict): - """Validator asserting every extra tag landed where the contract puts it. +def _assert_extra_tags_applied(scenario: dict): + """Validator asserting the backend-provided entries landed exactly where the contract puts them. - Scalars become span tags and structured values go to the ai_guard meta struct, both under the - key exactly as the backend returned it: the backend owns the ai_guard prefix, so the tracer - neither adds it nor filters keys by namespace. + Each extra_tags entry is a string tag on the ai_guard span and each extra_metastruct entry is a + value in the ai_guard meta struct, both under the key exactly as returned: the backend owns the + ai_guard prefix, so the tracer neither adds it nor filters keys by namespace. Nothing crosses + over: a tag never reaches the meta struct, and a meta struct value is never flattened into a tag. """ + expected_tags: dict[str, str] = scenario["expected_tags"] + expected_metastruct: dict = scenario["expected_metastruct"] def validate(span: DataDogLibrarySpan): if span["resource"] != "ai_guard": return False - ai_guard = _assert_key(span["meta_struct"], "ai_guard") - for key, value in extra_tags.items(): - if isinstance(value, (dict, list)): - assert key in ai_guard, f"Structured extra tag '{key}' not found in the ai_guard meta struct {ai_guard}" - assert ai_guard[key] == value, f"Extra tag '{key}' in meta struct: {ai_guard[key]} != {value}" - assert _span_tag(span, key) is None, f"Structured extra tag '{key}' should not be a span tag" - else: - _assert_scalar_extra_tag(span, key, value) + meta = span.get("meta") or {} + for key, value in expected_tags.items(): + assert meta.get(key) == value, ( + f"Extra tag '{key}' should be '{value}' on the ai_guard span, got '{meta.get(key)}'" + ) assert _span_tag(span, f"ai_guard.{key}") is None, f"The tracer must not prefix extra tag '{key}'" + + ai_guard = _assert_key(span["meta_struct"], "ai_guard") + for key, value in expected_metastruct.items(): + assert key in ai_guard, f"Extra meta struct key '{key}' not found in the ai_guard meta struct {ai_guard}" + assert ai_guard[key] == value, f"Extra meta struct key '{key}': {ai_guard[key]} != {value}" + assert f"ai_guard.{key}" not in ai_guard, f"The tracer must not prefix extra meta struct key '{key}'" + assert _span_tag(span, key) is None, f"Extra meta struct key '{key}' should not be a span tag" + + unexpected = set(ai_guard) - TRACER_META_STRUCT_KEYS - set(expected_metastruct) + assert not unexpected, f"Unexpected keys in the ai_guard meta struct: {unexpected}" + + for key in scenario["skipped_tags"]: + assert _span_tag(span, key) is None, ( + f"Malformed extra tag '{key}' should be skipped, got '{_span_tag(span, key)}'" + ) return True return validate @@ -1367,58 +1360,62 @@ def validate(span: DataDogLibrarySpan): @features.ai_guard @scenarios.ai_guard class Test_ExtraTags: - """The AI Guard backend enriches the ai_guard span through extra_tags in the evaluate response. + """The AI Guard backend enriches the ai_guard span through extra_tags and extra_metastruct. - Lets an AI Guard policy attach its own context to traces without a tracer change per tag. + Lets an AI Guard policy attach its own context to traces without a tracer change per tag. Both + attributes of the evaluate response are lists of {key, value} objects. """ - def setup_scalar_extra_tags(self): - self.scenario, self.r = _post_extra_tags_scenario("EXTRA_TAGS_SCALARS") + def setup_extra_tags(self): + self.scenario, self.r = _post_extra_tags_scenario("EXTRA_TAGS") - def test_scalar_extra_tags(self): - """String, integer, float and boolean values are set as tags on the ai_guard span.""" + def test_extra_tags(self): + """Each extra_tags entry is set as a string tag on the ai_guard span.""" assert self.r.status_code == 200 interfaces.library.validate_one_span( - self.r, validator=_assert_extra_tags_applied(self.scenario["extra_tags"]), full_trace=True + self.r, validator=_assert_extra_tags_applied(self.scenario), full_trace=True ) - def setup_structured_extra_tags(self): - self.scenario, self.r = _post_extra_tags_scenario("EXTRA_TAGS_STRUCTURED") + def setup_extra_metastruct(self): + self.scenario, self.r = _post_extra_tags_scenario("EXTRA_METASTRUCT") - def test_structured_extra_tags(self): - """Object and array values are added to the ai_guard meta struct, not flattened into tags.""" + def test_extra_metastruct(self): + """Each extra_metastruct entry, object, array or scalar, is added to the ai_guard meta struct.""" assert self.r.status_code == 200 interfaces.library.validate_one_span( - self.r, validator=_assert_extra_tags_applied(self.scenario["extra_tags"]), full_trace=True + self.r, validator=_assert_extra_tags_applied(self.scenario), full_trace=True ) - def setup_mixed_extra_tags(self): - self.scenario, self.r = _post_extra_tags_scenario("EXTRA_TAGS_MIXED") + def setup_evaluate_id(self): + self.scenario, self.r = _post_extra_tags_scenario("EVALUATE_ID") - def test_mixed_extra_tags(self): - """One response mixing both kinds routes each value by its own type.""" + def test_evaluate_id(self): + """The evaluation ID lands as ai_guard.evaluate_id and as evaluate_id, correlating the span with the backend.""" assert self.r.status_code == 200 + evaluation_id = self.scenario["evaluation_id"] + assert self.scenario["expected_tags"]["ai_guard.evaluate_id"] == evaluation_id + assert self.scenario["expected_metastruct"]["evaluate_id"] == evaluation_id interfaces.library.validate_one_span( - self.r, validator=_assert_extra_tags_applied(self.scenario["extra_tags"]), full_trace=True + self.r, validator=_assert_extra_tags_applied(self.scenario), full_trace=True ) def setup_keys_not_prefixed_or_filtered(self): - self.scenario, self.r = _post_extra_tags_scenario("EXTRA_TAGS_UNPREFIXED_KEY") + self.scenario, self.r = _post_extra_tags_scenario("KEYS_VERBATIM") def test_keys_not_prefixed_or_filtered(self): - """Keys outside the ai_guard namespace land verbatim: namespacing is the backend's job, not the tracer's.""" + """Keys land verbatim, even outside the ai_guard namespace: namespacing is the backend's job.""" assert self.r.status_code == 200 interfaces.library.validate_one_span( - self.r, validator=_assert_extra_tags_applied(self.scenario["extra_tags"]), full_trace=True + self.r, validator=_assert_extra_tags_applied(self.scenario), full_trace=True ) def setup_extra_tags_on_block(self): - self.scenario, self.r = _post_extra_tags_scenario("EXTRA_TAGS_ON_DENY", block="true") + self.scenario, self.r = _post_extra_tags_scenario("ON_DENY", block="true") def test_extra_tags_on_block(self): - """A blocked evaluation still carries the extra tags, alongside ai_guard.blocked.""" + """A blocked evaluation still carries both kinds of entries, alongside ai_guard.blocked.""" assert self.r.status_code == 403 - extra_tags_validator = _assert_extra_tags_applied(self.scenario["extra_tags"]) + extra_tags_validator = _assert_extra_tags_applied(self.scenario) def validate(span: DataDogLibrarySpan): if not extra_tags_validator(span): @@ -1436,42 +1433,47 @@ def validate(span: DataDogLibrarySpan): @features.ai_guard @scenarios.ai_guard class Test_ExtraTagsFailSafe: - """A malformed extra_tags payload is ignored and never changes the evaluation. + """A malformed extra_tags or extra_metastruct is skipped and never changes the evaluation. - The field only enriches the span, so a backend shape the tracer does not understand must leave - the action, the SDK response and the span exactly as they would be without it. + Both fields only enrich the span, so a shape the tracer does not understand must leave the + action, the SDK response and the span as they would be without it, while every well-formed + part of the response is still applied. """ - def _assert_ignored(self, response: HttpResponse, scenario: dict) -> None: + def _assert_skipped(self, response: HttpResponse, scenario: dict) -> None: assert response.status_code == 200 _assert_key(json.loads(response.text), "action", scenario["action"]) + extra_tags_validator = _assert_extra_tags_applied(scenario) def validate(span: DataDogLibrarySpan): - if span["resource"] != "ai_guard": + if not extra_tags_validator(span): return False - assert span.get("error", 0) == 0, f"A malformed extra_tags must not flag the span as an error: {span}" + assert span.get("error", 0) == 0, f"A malformed payload must not flag the span as an error: {span}" _assert_key(span["meta"], "ai_guard.action", scenario["action"]) - ai_guard = _assert_key(span["meta_struct"], "ai_guard") - unexpected = set(ai_guard) - TRACER_META_STRUCT_KEYS - assert not unexpected, f"Unexpected keys in the ai_guard meta struct: {unexpected}" - assert _span_tag(span, "ai_guard.policy_name") is None, "A malformed extra_tags must not set any tag" return True interfaces.library.validate_one_span(response, validator=validate, full_trace=True) - def setup_extra_tags_not_an_object(self): - self.scenario, self.r = _post_extra_tags_scenario("EXTRA_TAGS_NOT_AN_OBJECT") + def setup_extra_tags_not_a_list(self): + self.scenario, self.r = _post_extra_tags_scenario("EXTRA_TAGS_NOT_A_LIST") + + def test_extra_tags_not_a_list(self): + """An extra_tags object instead of a list is skipped, and extra_metastruct is still applied.""" + self._assert_skipped(self.r, self.scenario) + + def setup_extra_metastruct_not_a_list(self): + self.scenario, self.r = _post_extra_tags_scenario("EXTRA_METASTRUCT_NOT_A_LIST") - def test_extra_tags_not_an_object(self): - """A string where an object is expected is skipped.""" - self._assert_ignored(self.r, self.scenario) + def test_extra_metastruct_not_a_list(self): + """An extra_metastruct object instead of a list is skipped, and extra_tags is still applied.""" + self._assert_skipped(self.r, self.scenario) - def setup_extra_tags_array(self): - self.scenario, self.r = _post_extra_tags_scenario("EXTRA_TAGS_ARRAY") + def setup_malformed_entries(self): + self.scenario, self.r = _post_extra_tags_scenario("MALFORMED_ENTRIES") - def test_extra_tags_array(self): - """An array where an object is expected is skipped, even when its items look like tags.""" - self._assert_ignored(self.r, self.scenario) + def test_malformed_entries(self): + """Malformed entries and non-string extra_tags values are skipped one by one, the rest still lands.""" + self._assert_skipped(self.r, self.scenario) TELEMETRY_NAMESPACE = "ai_guard" diff --git a/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_19cfc9ce.json b/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_19cfc9ce.json new file mode 100644 index 00000000000..6cbb64d3c9f --- /dev/null +++ b/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_19cfc9ce.json @@ -0,0 +1,22 @@ +{ + "request": { + "method": "POST", + "url": "https://app.datadoghq.com/api/v2/ai-guard/evaluate", + "headers": { + "Content-Type": "application/json", + "DD-AI-GUARD-SOURCE": "SDK" + }, + "body": "{\"data\": {\"attributes\": {\"messages\": [{\"role\": \"user\", \"content\": \"When does the weekend magazine go to print?\"}], \"meta\": {\"service\": \"weblog\", \"env\": \"system-tests\"}}}}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "content-type": "application/vnd.api+json", + "content-length": "1131" + }, + "body": "{\"data\": {\"id\": \"f55f1430-86f7-5441-96b5-9938f4a10261\", \"type\": \"evaluations\", \"attributes\": {\"action\": \"ALLOW\", \"is_blocking_enabled\": true, \"reason\": \"No rule match.\", \"sds_findings\": [], \"tag_probs\": {\"authority-override\": 0.0, \"data-exfiltration\": 0.0, \"denial-of-service-tool-call\": 0.0, \"destructive-tool-call\": 0.0, \"indirect-prompt-injection\": 0.0, \"instruction-override\": 0.0, \"jailbreak\": 0.0, \"obfuscation\": 0.0, \"role-play\": 0.0, \"security-exploit\": 0.0, \"system-prompt-extraction\": 0.0}, \"tags\": [], \"extra_tags\": [\"ai_guard.policy_name=newsroom-default\", [\"ai_guard.policy_name\", \"newsroom-default\"], {\"value\": \"missing key\"}, {\"key\": 1, \"value\": \"integer key\"}, {\"key\": \"\", \"value\": \"empty key\"}, {\"key\": \"ai_guard.policy_revision\", \"value\": 7}, {\"key\": \"ai_guard.policy_details\", \"value\": {\"rule\": \"embargo-check\"}}, {\"key\": \"ai_guard.policy_name\", \"value\": \"newsroom-default\"}], \"extra_metastruct\": [\"policy=newsroom-default\", {\"value\": {\"name\": \"missing key\"}}, {\"key\": 1, \"value\": {\"name\": \"integer key\"}}, {\"key\": \"\", \"value\": {\"name\": \"empty key\"}}, {\"key\": \"policy\", \"value\": {\"name\": \"newsroom-default\"}}]}}}" + } +} diff --git a/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_28917f95.json b/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_28917f95.json index dc8516efd53..d0ee30f946e 100644 --- a/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_28917f95.json +++ b/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_28917f95.json @@ -15,8 +15,8 @@ }, "headers": { "content-type": "application/vnd.api+json", - "content-length": "679" + "content-length": "901" }, - "body": "{\"data\": {\"id\": \"extra-tags-fixture\", \"type\": \"evaluations\", \"attributes\": {\"action\": \"DENY\", \"is_blocking_enabled\": true, \"reason\": \"Destructive tool call detected.\", \"sds_findings\": [], \"tag_probs\": {\"authority-override\": 0.0, \"data-exfiltration\": 0.0, \"denial-of-service-tool-call\": 0.0, \"destructive-tool-call\": 0.97, \"indirect-prompt-injection\": 0.0, \"instruction-override\": 0.0, \"jailbreak\": 0.0, \"obfuscation\": 0.0, \"role-play\": 0.0, \"security-exploit\": 0.0, \"system-prompt-extraction\": 0.0}, \"tags\": [\"destructive-tool-call\"], \"extra_tags\": {\"ai_guard.policy_name\": \"destructive-actions\", \"ai_guard.policy_details\": {\"rule\": \"no-recursive-delete\", \"severity\": \"high\"}}}}}" + "body": "{\"data\": {\"id\": \"55e3505d-93eb-5171-8166-90c709865d83\", \"type\": \"evaluations\", \"attributes\": {\"action\": \"DENY\", \"is_blocking_enabled\": true, \"reason\": \"Destructive tool call detected.\", \"sds_findings\": [], \"tag_probs\": {\"authority-override\": 0.0, \"data-exfiltration\": 0.0, \"denial-of-service-tool-call\": 0.0, \"destructive-tool-call\": 0.97, \"indirect-prompt-injection\": 0.0, \"instruction-override\": 0.0, \"jailbreak\": 0.0, \"obfuscation\": 0.0, \"role-play\": 0.0, \"security-exploit\": 0.0, \"system-prompt-extraction\": 0.0}, \"tags\": [\"destructive-tool-call\"], \"extra_tags\": [{\"key\": \"ai_guard.evaluate_id\", \"value\": \"55e3505d-93eb-5171-8166-90c709865d83\"}, {\"key\": \"ai_guard.policy_name\", \"value\": \"destructive-actions\"}], \"extra_metastruct\": [{\"key\": \"evaluate_id\", \"value\": \"55e3505d-93eb-5171-8166-90c709865d83\"}, {\"key\": \"policy_details\", \"value\": {\"rule\": \"no-recursive-delete\", \"severity\": \"high\"}}]}}}" } } diff --git a/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_2b336647.json b/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_2b336647.json index 0b90904917f..5f4665efd88 100644 --- a/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_2b336647.json +++ b/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_2b336647.json @@ -15,8 +15,8 @@ }, "headers": { "content-type": "application/vnd.api+json", - "content-length": "659" + "content-length": "823" }, - "body": "{\"data\": {\"id\": \"extra-tags-fixture\", \"type\": \"evaluations\", \"attributes\": {\"action\": \"ALLOW\", \"is_blocking_enabled\": true, \"reason\": \"No rule match.\", \"sds_findings\": [], \"tag_probs\": {\"authority-override\": 0.0, \"data-exfiltration\": 0.0, \"denial-of-service-tool-call\": 0.0, \"destructive-tool-call\": 0.0, \"indirect-prompt-injection\": 0.0, \"instruction-override\": 0.0, \"jailbreak\": 0.0, \"obfuscation\": 0.0, \"role-play\": 0.0, \"security-exploit\": 0.0, \"system-prompt-extraction\": 0.0}, \"tags\": [], \"extra_tags\": {\"ai_guard.policy_name\": \"elections-2026\", \"ai_guard.policy_rules\": [{\"id\": \"sourcing\", \"enabled\": true}, {\"id\": \"attribution\", \"enabled\": false}]}}}}" + "body": "{\"data\": {\"id\": \"9c9036fc-d7a1-5d5e-bcb3-7d62cc9fa218\", \"type\": \"evaluations\", \"attributes\": {\"action\": \"ALLOW\", \"is_blocking_enabled\": true, \"reason\": \"No rule match.\", \"sds_findings\": [], \"tag_probs\": {\"authority-override\": 0.0, \"data-exfiltration\": 0.0, \"denial-of-service-tool-call\": 0.0, \"destructive-tool-call\": 0.0, \"indirect-prompt-injection\": 0.0, \"instruction-override\": 0.0, \"jailbreak\": 0.0, \"obfuscation\": 0.0, \"role-play\": 0.0, \"security-exploit\": 0.0, \"system-prompt-extraction\": 0.0}, \"tags\": [], \"extra_tags\": [{\"key\": \"ai_guard.evaluate_id\", \"value\": \"9c9036fc-d7a1-5d5e-bcb3-7d62cc9fa218\"}, {\"key\": \"ai_guard.policy_name\", \"value\": \"elections-2026\"}], \"extra_metastruct\": [{\"key\": \"evaluate_id\", \"value\": \"9c9036fc-d7a1-5d5e-bcb3-7d62cc9fa218\"}, {\"key\": \"policy\", \"value\": {\"name\": \"elections-2026\"}}]}}}" } } diff --git a/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_354106da.json b/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_354106da.json index 5f327011102..8c1040884ca 100644 --- a/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_354106da.json +++ b/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_354106da.json @@ -15,8 +15,8 @@ }, "headers": { "content-type": "application/vnd.api+json", - "content-length": "595" + "content-length": "656" }, - "body": "{\"data\": {\"id\": \"extra-tags-fixture\", \"type\": \"evaluations\", \"attributes\": {\"action\": \"ALLOW\", \"is_blocking_enabled\": true, \"reason\": \"No rule match.\", \"sds_findings\": [], \"tag_probs\": {\"authority-override\": 0.0, \"data-exfiltration\": 0.0, \"denial-of-service-tool-call\": 0.0, \"destructive-tool-call\": 0.0, \"indirect-prompt-injection\": 0.0, \"instruction-override\": 0.0, \"jailbreak\": 0.0, \"obfuscation\": 0.0, \"role-play\": 0.0, \"security-exploit\": 0.0, \"system-prompt-extraction\": 0.0}, \"tags\": [], \"extra_tags\": {\"custom_context.tenant\": \"newsroom\", \"custom_context.audit\": {\"ticket\": \"CTX-42\"}}}}}" + "body": "{\"data\": {\"id\": \"92a51623-de7f-5850-8f99-2a91d343bc0a\", \"type\": \"evaluations\", \"attributes\": {\"action\": \"ALLOW\", \"is_blocking_enabled\": true, \"reason\": \"No rule match.\", \"sds_findings\": [], \"tag_probs\": {\"authority-override\": 0.0, \"data-exfiltration\": 0.0, \"denial-of-service-tool-call\": 0.0, \"destructive-tool-call\": 0.0, \"indirect-prompt-injection\": 0.0, \"instruction-override\": 0.0, \"jailbreak\": 0.0, \"obfuscation\": 0.0, \"role-play\": 0.0, \"security-exploit\": 0.0, \"system-prompt-extraction\": 0.0}, \"tags\": [], \"extra_tags\": [{\"key\": \"custom_context.tenant\", \"value\": \"newsroom\"}], \"extra_metastruct\": [{\"key\": \"audit\", \"value\": {\"ticket\": \"CTX-42\"}}]}}}" } } diff --git a/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_827d1449.json b/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_827d1449.json index a18c8a60dc6..ddb025e8df5 100644 --- a/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_827d1449.json +++ b/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_827d1449.json @@ -15,8 +15,8 @@ }, "headers": { "content-type": "application/vnd.api+json", - "content-length": "655" + "content-length": "904" }, - "body": "{\"data\": {\"id\": \"extra-tags-fixture\", \"type\": \"evaluations\", \"attributes\": {\"action\": \"ALLOW\", \"is_blocking_enabled\": true, \"reason\": \"No rule match.\", \"sds_findings\": [], \"tag_probs\": {\"authority-override\": 0.0, \"data-exfiltration\": 0.0, \"denial-of-service-tool-call\": 0.0, \"destructive-tool-call\": 0.0, \"indirect-prompt-injection\": 0.0, \"instruction-override\": 0.0, \"jailbreak\": 0.0, \"obfuscation\": 0.0, \"role-play\": 0.0, \"security-exploit\": 0.0, \"system-prompt-extraction\": 0.0}, \"tags\": [], \"extra_tags\": {\"ai_guard.request_context\": {\"desk\": \"metro\", \"region\": \"us-east\", \"priority\": 2}, \"ai_guard.matched_rules\": [\"style-guide\", \"embargo-check\"]}}}}" + "body": "{\"data\": {\"id\": \"73606e72-26b6-52f8-8652-7f321db98dda\", \"type\": \"evaluations\", \"attributes\": {\"action\": \"ALLOW\", \"is_blocking_enabled\": true, \"reason\": \"No rule match.\", \"sds_findings\": [], \"tag_probs\": {\"authority-override\": 0.0, \"data-exfiltration\": 0.0, \"denial-of-service-tool-call\": 0.0, \"destructive-tool-call\": 0.0, \"indirect-prompt-injection\": 0.0, \"instruction-override\": 0.0, \"jailbreak\": 0.0, \"obfuscation\": 0.0, \"role-play\": 0.0, \"security-exploit\": 0.0, \"system-prompt-extraction\": 0.0}, \"tags\": [], \"extra_metastruct\": [{\"key\": \"request_context\", \"value\": {\"desk\": \"metro\", \"region\": \"us-east\", \"priority\": 2}}, {\"key\": \"matched_rules\", \"value\": [\"style-guide\", \"embargo-check\"]}, {\"key\": \"policy_rules\", \"value\": [{\"id\": \"sourcing\", \"enabled\": true}, {\"id\": \"attribution\", \"enabled\": false}]}, {\"key\": \"policy_score\", \"value\": 0.25}, {\"key\": \"policy_name\", \"value\": \"newsroom-default\"}]}}}" } } diff --git a/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_bf6868f9.json b/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_bf6868f9.json index 6d55035586e..6c4c92ca100 100644 --- a/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_bf6868f9.json +++ b/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_bf6868f9.json @@ -15,8 +15,8 @@ }, "headers": { "content-type": "application/vnd.api+json", - "content-length": "558" + "content-length": "654" }, - "body": "{\"data\": {\"id\": \"extra-tags-fixture\", \"type\": \"evaluations\", \"attributes\": {\"action\": \"ALLOW\", \"is_blocking_enabled\": true, \"reason\": \"No rule match.\", \"sds_findings\": [], \"tag_probs\": {\"authority-override\": 0.0, \"data-exfiltration\": 0.0, \"denial-of-service-tool-call\": 0.0, \"destructive-tool-call\": 0.0, \"indirect-prompt-injection\": 0.0, \"instruction-override\": 0.0, \"jailbreak\": 0.0, \"obfuscation\": 0.0, \"role-play\": 0.0, \"security-exploit\": 0.0, \"system-prompt-extraction\": 0.0}, \"tags\": [], \"extra_tags\": [{\"ai_guard.policy_name\": \"newsroom-default\"}]}}}" + "body": "{\"data\": {\"id\": \"e72731f7-4c56-56b4-95a9-ed8d009cebcb\", \"type\": \"evaluations\", \"attributes\": {\"action\": \"ALLOW\", \"is_blocking_enabled\": true, \"reason\": \"No rule match.\", \"sds_findings\": [], \"tag_probs\": {\"authority-override\": 0.0, \"data-exfiltration\": 0.0, \"denial-of-service-tool-call\": 0.0, \"destructive-tool-call\": 0.0, \"indirect-prompt-injection\": 0.0, \"instruction-override\": 0.0, \"jailbreak\": 0.0, \"obfuscation\": 0.0, \"role-play\": 0.0, \"security-exploit\": 0.0, \"system-prompt-extraction\": 0.0}, \"tags\": [], \"extra_tags\": [{\"key\": \"ai_guard.policy_name\", \"value\": \"newsroom-default\"}], \"extra_metastruct\": {\"policy\": {\"name\": \"newsroom-default\"}}}}}" } } diff --git a/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_e2c13e67.json b/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_e2c13e67.json index 936648f304c..1e34054357e 100644 --- a/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_e2c13e67.json +++ b/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_e2c13e67.json @@ -15,8 +15,8 @@ }, "headers": { "content-type": "application/vnd.api+json", - "content-length": "551" + "content-length": "654" }, - "body": "{\"data\": {\"id\": \"extra-tags-fixture\", \"type\": \"evaluations\", \"attributes\": {\"action\": \"ALLOW\", \"is_blocking_enabled\": true, \"reason\": \"No rule match.\", \"sds_findings\": [], \"tag_probs\": {\"authority-override\": 0.0, \"data-exfiltration\": 0.0, \"denial-of-service-tool-call\": 0.0, \"destructive-tool-call\": 0.0, \"indirect-prompt-injection\": 0.0, \"instruction-override\": 0.0, \"jailbreak\": 0.0, \"obfuscation\": 0.0, \"role-play\": 0.0, \"security-exploit\": 0.0, \"system-prompt-extraction\": 0.0}, \"tags\": [], \"extra_tags\": \"ai_guard.policy_name=newsroom-default\"}}}" + "body": "{\"data\": {\"id\": \"a8524c91-cec1-5ee6-812f-bd3b4fefae2b\", \"type\": \"evaluations\", \"attributes\": {\"action\": \"ALLOW\", \"is_blocking_enabled\": true, \"reason\": \"No rule match.\", \"sds_findings\": [], \"tag_probs\": {\"authority-override\": 0.0, \"data-exfiltration\": 0.0, \"denial-of-service-tool-call\": 0.0, \"destructive-tool-call\": 0.0, \"indirect-prompt-injection\": 0.0, \"instruction-override\": 0.0, \"jailbreak\": 0.0, \"obfuscation\": 0.0, \"role-play\": 0.0, \"security-exploit\": 0.0, \"system-prompt-extraction\": 0.0}, \"tags\": [], \"extra_tags\": {\"ai_guard.policy_name\": \"newsroom-default\"}, \"extra_metastruct\": [{\"key\": \"policy\", \"value\": {\"name\": \"newsroom-default\"}}]}}}" } } diff --git a/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_f0ef2bd8.json b/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_f0ef2bd8.json index 23fe1bbdd47..73e7dce2329 100644 --- a/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_f0ef2bd8.json +++ b/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_f0ef2bd8.json @@ -15,8 +15,8 @@ }, "headers": { "content-type": "application/vnd.api+json", - "content-length": "684" + "content-length": "643" }, - "body": "{\"data\": {\"id\": \"extra-tags-fixture\", \"type\": \"evaluations\", \"attributes\": {\"action\": \"ALLOW\", \"is_blocking_enabled\": true, \"reason\": \"No rule match.\", \"sds_findings\": [], \"tag_probs\": {\"authority-override\": 0.0, \"data-exfiltration\": 0.0, \"denial-of-service-tool-call\": 0.0, \"destructive-tool-call\": 0.0, \"indirect-prompt-injection\": 0.0, \"instruction-override\": 0.0, \"jailbreak\": 0.0, \"obfuscation\": 0.0, \"role-play\": 0.0, \"security-exploit\": 0.0, \"system-prompt-extraction\": 0.0}, \"tags\": [], \"extra_tags\": {\"ai_guard.policy_name\": \"newsroom-default\", \"ai_guard.policy_revision\": 7, \"ai_guard.policy_score\": 0.25, \"ai_guard.policy_matched\": true, \"ai_guard.policy_cached\": false}}}}" + "body": "{\"data\": {\"id\": \"13c92662-52c8-5959-8fec-fb7d6bb5b5d0\", \"type\": \"evaluations\", \"attributes\": {\"action\": \"ALLOW\", \"is_blocking_enabled\": true, \"reason\": \"No rule match.\", \"sds_findings\": [], \"tag_probs\": {\"authority-override\": 0.0, \"data-exfiltration\": 0.0, \"denial-of-service-tool-call\": 0.0, \"destructive-tool-call\": 0.0, \"indirect-prompt-injection\": 0.0, \"instruction-override\": 0.0, \"jailbreak\": 0.0, \"obfuscation\": 0.0, \"role-play\": 0.0, \"security-exploit\": 0.0, \"system-prompt-extraction\": 0.0}, \"tags\": [], \"extra_tags\": [{\"key\": \"ai_guard.policy_name\", \"value\": \"newsroom-default\"}, {\"key\": \"ai_guard.policy_revision\", \"value\": \"7\"}]}}}" } } diff --git a/utils/scripts/activate_easy_wins/skip.yml b/utils/scripts/activate_easy_wins/skip.yml index 811cd864fbf..a83e8ee3b19 100644 --- a/utils/scripts/activate_easy_wins/skip.yml +++ b/utils/scripts/activate_easy_wins/skip.yml @@ -1,8 +1,5 @@ "*": - tests/parametric/test_crashtracking.py::Test_Crashtracking::test_disable_crashtracking - # A tracer without extra_tags support also ignores a malformed payload, so these pass vacuously - - tests/ai_guard/test_ai_guard_sdk.py::Test_ExtraTagsFailSafe::test_extra_tags_not_an_object - - tests/ai_guard/test_ai_guard_sdk.py::Test_ExtraTagsFailSafe::test_extra_tags_array "nodejs": - tests/test_telemetry.py::Test_DependencyEnable::test_app_dependency_loaded_not_sent_dependency_collection_disabled diff --git a/utils/scripts/gen_extra_tags_cassettes.py b/utils/scripts/gen_extra_tags_cassettes.py index 4718bcd484d..22bd856f86d 100755 --- a/utils/scripts/gen_extra_tags_cassettes.py +++ b/utils/scripts/gen_extra_tags_cassettes.py @@ -1,16 +1,22 @@ #!/usr/bin/env python3 -"""Generate hand-crafted AI Guard VCR cassettes for backend-provided extra_tags. +"""Generate hand-crafted AI Guard VCR cassettes for backend-provided extra_tags and extra_metastruct. -The evaluate response can carry an optional data.attributes.extra_tags object: tags the AI Guard -backend wants on the ai_guard span, so a new evaluation detail reaches traces without a tracer -release per tag (APPSEC-70724). The backend owns the ai_guard prefix and the namespace, so the -tracer contract is thin: +The evaluate response can carry two optional attributes so an AI Guard policy can enrich the +ai_guard span without a tracer release per tag (APPSEC-70724). Both are lists of {key, value} +objects rather than maps, because dotted keys such as ai_guard.evaluate_id are not valid JSON:API +member names: - - scalar values (string, number, boolean) become ai_guard span tags under the key exactly as - returned, with no prefix added and no namespace filtering; - - structured values (objects, arrays) are added to the ai_guard meta struct under their key. + - extra_tags holds string values, each set as an ai_guard span tag under the key exactly as + returned; + - extra_metastruct holds any value (object, array or scalar), each added to the ai_guard meta + struct under the key exactly as returned. -The backend does not return extra_tags yet, so no cassette can be recorded: this script writes them +The backend owns the ai_guard prefix and the namespace, so tracers never add the prefix or filter +keys. The first entry the backend sends is the evaluation ID (the same value as data.id), as +ai_guard.evaluate_id in extra_tags and as evaluate_id in extra_metastruct. A malformed field or +entry is skipped and never changes the evaluation. + +The backend does not return these attributes in the recorded cassettes, so this script writes them by hand, reusing the request hashing and rendering of gen_redaction_cassettes.py, and writes tests/ai_guard/extra_tags_scenarios.json, the sidecar the tests read the scenarios from. The two are generated together because a cassette is addressed by the hash of its request body: a scenario @@ -24,6 +30,7 @@ import json import sys +import uuid from pathlib import Path from gen_redaction_cassettes import ( @@ -41,53 +48,72 @@ DENY_REASON = "Destructive tool call detected." +# Fixed namespace so every scenario gets a stable evaluation ID (data.id) across regenerations. +EVALUATION_ID_NAMESPACE = uuid.UUID("6f0e8c1a-3b9d-4c55-9a51-0c2f4e7d8b13") + + +def _evaluation_id(name: str) -> str: + return str(uuid.uuid5(EVALUATION_ID_NAMESPACE, name)) + + +def _kv(**entries: object) -> list[dict]: + return [{"key": key, "value": value} for key, value in entries.items()] + + +def _evaluate_id_entries(name: str) -> tuple[list[dict], list[dict]]: + evaluation_id = _evaluation_id(name) + return ( + [{"key": "ai_guard.evaluate_id", "value": evaluation_id}], + [{"key": "evaluate_id", "value": evaluation_id}], + ) + + # Every scenario produces one cassette and one sidecar entry: -# messages what the weblog POSTs, and therefore what the cassette is keyed on -# action the evaluation outcome the backend returns -# tags the attack categories the backend returns (a DENY needs at least one) -# extra_tags the extra_tags value the backend returns, verbatim: malformed on purpose for the -# fail-safe scenarios +# messages what the weblog POSTs, and therefore what the cassette is keyed on +# action the evaluation outcome the backend returns +# tags the attack categories the backend returns (a DENY needs at least one) +# extra_tags the extra_tags value the backend returns, verbatim (omitted when absent) +# extra_metastruct the extra_metastruct value the backend returns, verbatim (omitted when absent) +# expected_tags, expected_metastruct, skipped_tags +# what the tracer must apply, only spelled out for the fail-safe scenarios, whose +# payloads are malformed on purpose; for the others it is derived from the lists SCENARIOS: dict[str, dict] = { - "EXTRA_TAGS_SCALARS": { + "EXTRA_TAGS": { "messages": [{"role": "user", "content": "Summarize the latest newsroom style guide changes."}], "action": "ALLOW", "tags": [], - "extra_tags": { - "ai_guard.policy_name": "newsroom-default", - "ai_guard.policy_revision": 7, - "ai_guard.policy_score": 0.25, - "ai_guard.policy_matched": True, - "ai_guard.policy_cached": False, - }, + "extra_tags": _kv(**{"ai_guard.policy_name": "newsroom-default", "ai_guard.policy_revision": "7"}), }, - "EXTRA_TAGS_STRUCTURED": { + "EXTRA_METASTRUCT": { "messages": [{"role": "user", "content": "Draft a headline for the city council budget story."}], "action": "ALLOW", "tags": [], - "extra_tags": { - "ai_guard.request_context": {"desk": "metro", "region": "us-east", "priority": 2}, - "ai_guard.matched_rules": ["style-guide", "embargo-check"], - }, + "extra_metastruct": _kv( + request_context={"desk": "metro", "region": "us-east", "priority": 2}, + matched_rules=["style-guide", "embargo-check"], + policy_rules=[{"id": "sourcing", "enabled": True}, {"id": "attribution", "enabled": False}], + policy_score=0.25, + policy_name="newsroom-default", + ), }, - "EXTRA_TAGS_MIXED": { + "EVALUATE_ID": { "messages": [{"role": "user", "content": "Which sources did the election explainer cite?"}], "action": "ALLOW", "tags": [], - "extra_tags": { - "ai_guard.policy_name": "elections-2026", - "ai_guard.policy_rules": [{"id": "sourcing", "enabled": True}, {"id": "attribution", "enabled": False}], - }, + "extra_tags": _evaluate_id_entries("EVALUATE_ID")[0] + _kv(**{"ai_guard.policy_name": "elections-2026"}), + "extra_metastruct": _evaluate_id_entries("EVALUATE_ID")[1] + _kv(policy={"name": "elections-2026"}), }, - # The backend guarantees every key is in the ai_guard namespace; the tracer must not double - # check it. A key outside the namespace still lands verbatim, which is what proves the tracer - # neither prefixes nor filters. - "EXTRA_TAGS_UNPREFIXED_KEY": { + # The backend guarantees every extra_tags key is in the ai_guard namespace; the tracer must not + # double check it. A key outside the namespace still lands verbatim, which is what proves the + # tracer neither prefixes nor filters. + "KEYS_VERBATIM": { "messages": [{"role": "user", "content": "List the corrections published this week."}], "action": "ALLOW", "tags": [], - "extra_tags": {"custom_context.tenant": "newsroom", "custom_context.audit": {"ticket": "CTX-42"}}, + "extra_tags": _kv(**{"custom_context.tenant": "newsroom"}), + "extra_metastruct": _kv(audit={"ticket": "CTX-42"}), }, - "EXTRA_TAGS_ON_DENY": { + "ON_DENY": { "messages": [ {"role": "user", "content": "Clean up the archive server for me."}, { @@ -102,27 +128,86 @@ ], "action": "DENY", "tags": ["destructive-tool-call"], - "extra_tags": { - "ai_guard.policy_name": "destructive-actions", - "ai_guard.policy_details": {"rule": "no-recursive-delete", "severity": "high"}, - }, + "extra_tags": _evaluate_id_entries("ON_DENY")[0] + _kv(**{"ai_guard.policy_name": "destructive-actions"}), + "extra_metastruct": _evaluate_id_entries("ON_DENY")[1] + + _kv(policy_details={"rule": "no-recursive-delete", "severity": "high"}), }, - "EXTRA_TAGS_NOT_AN_OBJECT": { + # extra_tags in the map shape of the first draft of the contract: ignored as a whole, while the + # well-formed extra_metastruct next to it is still applied. + "EXTRA_TAGS_NOT_A_LIST": { "messages": [{"role": "user", "content": "What time does the print edition close tonight?"}], "action": "ALLOW", "tags": [], - "extra_tags": "ai_guard.policy_name=newsroom-default", + "extra_tags": {"ai_guard.policy_name": "newsroom-default"}, + "extra_metastruct": _kv(policy={"name": "newsroom-default"}), + "expected_tags": {}, + "expected_metastruct": {"policy": {"name": "newsroom-default"}}, + "skipped_tags": ["ai_guard.policy_name"], }, - "EXTRA_TAGS_ARRAY": { + "EXTRA_METASTRUCT_NOT_A_LIST": { "messages": [{"role": "user", "content": "Who is on call for the overnight desk?"}], "action": "ALLOW", "tags": [], - "extra_tags": [{"ai_guard.policy_name": "newsroom-default"}], + "extra_tags": _kv(**{"ai_guard.policy_name": "newsroom-default"}), + "extra_metastruct": {"policy": {"name": "newsroom-default"}}, + "expected_tags": {"ai_guard.policy_name": "newsroom-default"}, + "expected_metastruct": {}, + "skipped_tags": [], + }, + # Each malformed entry is skipped on its own: the well-formed entries around it still land. + # extra_tags only carries strings, so a number or an object there is skipped too, and never + # rerouted to the meta struct. + "MALFORMED_ENTRIES": { + "messages": [{"role": "user", "content": "When does the weekend magazine go to print?"}], + "action": "ALLOW", + "tags": [], + "extra_tags": [ + "ai_guard.policy_name=newsroom-default", + ["ai_guard.policy_name", "newsroom-default"], + {"value": "missing key"}, + {"key": 1, "value": "integer key"}, + {"key": "", "value": "empty key"}, + {"key": "ai_guard.policy_revision", "value": 7}, + {"key": "ai_guard.policy_details", "value": {"rule": "embargo-check"}}, + {"key": "ai_guard.policy_name", "value": "newsroom-default"}, + ], + "extra_metastruct": [ + "policy=newsroom-default", + {"value": {"name": "missing key"}}, + {"key": 1, "value": {"name": "integer key"}}, + {"key": "", "value": {"name": "empty key"}}, + {"key": "policy", "value": {"name": "newsroom-default"}}, + ], + "expected_tags": {"ai_guard.policy_name": "newsroom-default"}, + "expected_metastruct": {"policy": {"name": "newsroom-default"}}, + "skipped_tags": ["ai_guard.policy_revision", "ai_guard.policy_details"], }, } -def _response_body(scenario: dict) -> str: +def _as_mapping(name: str, field: str, entries: list[dict] | None) -> dict: + """The {key: value} a well-formed list of {key, value} objects must produce on the span.""" + if entries is None: + return {} + keys = [entry["key"] for entry in entries] + assert len(keys) == len(set(keys)), f"{name}: the backend never repeats a key in {field}" + return {entry["key"]: entry["value"] for entry in entries} + + +def _with_expectations(name: str, scenario: dict) -> dict: + """Return the scenario with its evaluation ID and the tags and meta struct values the tracer must apply.""" + scenario = {"evaluation_id": _evaluation_id(name), **scenario} + if "expected_tags" not in scenario: + scenario["expected_tags"] = _as_mapping(name, "extra_tags", scenario.get("extra_tags")) + scenario["expected_metastruct"] = _as_mapping(name, "extra_metastruct", scenario.get("extra_metastruct")) + scenario["skipped_tags"] = [] + assert all(isinstance(value, str) for value in scenario["expected_tags"].values()), ( + f"{name}: extra_tags only carries strings" + ) + return scenario + + +def _response_body(name: str, scenario: dict) -> str: attributes = { "action": scenario["action"], "is_blocking_enabled": True, @@ -130,9 +215,12 @@ def _response_body(scenario: dict) -> str: "sds_findings": [], "tag_probs": _tag_probs_for(scenario["tags"]), "tags": scenario["tags"], - "extra_tags": scenario["extra_tags"], } - return json.dumps({"data": {"id": "extra-tags-fixture", "type": "evaluations", "attributes": attributes}}) + # Both attributes are omitted, not null, when the backend has nothing to send. + for field in ("extra_tags", "extra_metastruct"): + if field in scenario: + attributes[field] = scenario[field] + return json.dumps({"data": {"id": _evaluation_id(name), "type": "evaluations", "attributes": attributes}}) def _owned_by_redaction() -> set[str]: @@ -163,9 +251,9 @@ def build_corpus(previously_generated: set[str]) -> tuple[dict[str, str], dict[s assert not (path.is_file() and cassette not in previously_generated and previously_generated), ( f"{name} would overwrite {cassette}, which this script did not generate: change its messages" ) - contents[cassette] = render_cassette(request_body, _response_body(scenario)) + contents[cassette] = render_cassette(request_body, _response_body(name, scenario)) owners[cassette] = name - sidecar[name] = {**scenario, "cassette": cassette} + sidecar[name] = {**_with_expectations(name, scenario), "cassette": cassette} orphans = [name for name in sorted(previously_generated - set(contents)) if (CASSETTES_DIR / name).is_file()] return contents, sidecar, orphans @@ -211,7 +299,7 @@ def main(argv: list[str]) -> int: SIDECAR.write_text(_render_sidecar(sidecar)) for name, entry in sidecar.items(): - print(f"{name:28s} -> {entry['cassette']}") + print(f"{name:30s} -> {entry['cassette']}") print(f"\nwrote {len(sidecar)} scenarios -> {SIDECAR}") return 0 From 77ac7f5d9b934f783047470aa177804a1b0733a3 Mon Sep 17 00:00:00 2001 From: Alberto Vara Date: Tue, 27 Oct 2026 15:53:19 +0100 Subject: [PATCH 5/5] AI Guard: send the evaluate ID only in extra_tags and cover boolean/null skips The backend now sends the evaluation ID only as the ai_guard.evaluate_id span tag; extra_metastruct is empty for now (ddoghq/dd-source#139566). - EVALUATE_ID mirrors today's response: ai_guard.evaluate_id in extra_tags, equal to data.id, and no extra_metastruct. The test also asserts the ID never reaches the ai_guard meta_struct. - ON_DENY no longer sends evaluate_id in extra_metastruct. - EXTRA_METASTRUCT uses the illustrative backend example (objects, arrays of objects, a scalar). - MALFORMED_ENTRIES adds a boolean extra_tags value and a null extra_metastruct value, and checks that a non-string or empty key is never stringified or kept. Regenerated with utils/scripts/gen_extra_tags_cassettes.py; cassette names are unchanged. Co-Authored-By: Claude Opus 5.5 --- tests/ai_guard/extra_tags_scenarios.json | 102 +++++++----------- tests/ai_guard/test_ai_guard_sdk.py | 21 ++-- .../aiguard_evaluate_post_19cfc9ce.json | 4 +- .../aiguard_evaluate_post_28917f95.json | 4 +- .../aiguard_evaluate_post_2b336647.json | 4 +- .../aiguard_evaluate_post_827d1449.json | 4 +- utils/scripts/gen_extra_tags_cassettes.py | 44 ++++---- 7 files changed, 82 insertions(+), 101 deletions(-) diff --git a/tests/ai_guard/extra_tags_scenarios.json b/tests/ai_guard/extra_tags_scenarios.json index 5bacda63f85..d3b51ec3654 100644 --- a/tests/ai_guard/extra_tags_scenarios.json +++ b/tests/ai_guard/extra_tags_scenarios.json @@ -39,65 +39,53 @@ "tags": [], "extra_metastruct": [ { - "key": "request_context", + "key": "policy_match", "value": { - "desk": "metro", - "region": "us-east", - "priority": 2 + "rule": "credit_card", + "categories": [ + "pii", + "financial" + ] } }, { "key": "matched_rules", - "value": [ - "style-guide", - "embargo-check" - ] - }, - { - "key": "policy_rules", "value": [ { - "id": "sourcing", - "enabled": true + "id": "r1", + "action": "DENY" }, { - "id": "attribution", - "enabled": false + "id": "r2", + "action": "ALLOW" } ] }, { - "key": "policy_score", - "value": 0.25 - }, - { - "key": "policy_name", - "value": "newsroom-default" + "key": "policy_version", + "value": "v3" } ], "expected_tags": {}, "expected_metastruct": { - "request_context": { - "desk": "metro", - "region": "us-east", - "priority": 2 + "policy_match": { + "rule": "credit_card", + "categories": [ + "pii", + "financial" + ] }, "matched_rules": [ - "style-guide", - "embargo-check" - ], - "policy_rules": [ { - "id": "sourcing", - "enabled": true + "id": "r1", + "action": "DENY" }, { - "id": "attribution", - "enabled": false + "id": "r2", + "action": "ALLOW" } ], - "policy_score": 0.25, - "policy_name": "newsroom-default" + "policy_version": "v3" }, "skipped_tags": [], "cassette": "aiguard_evaluate_post_827d1449.json" @@ -116,34 +104,12 @@ { "key": "ai_guard.evaluate_id", "value": "9c9036fc-d7a1-5d5e-bcb3-7d62cc9fa218" - }, - { - "key": "ai_guard.policy_name", - "value": "elections-2026" - } - ], - "extra_metastruct": [ - { - "key": "evaluate_id", - "value": "9c9036fc-d7a1-5d5e-bcb3-7d62cc9fa218" - }, - { - "key": "policy", - "value": { - "name": "elections-2026" - } } ], "expected_tags": { - "ai_guard.evaluate_id": "9c9036fc-d7a1-5d5e-bcb3-7d62cc9fa218", - "ai_guard.policy_name": "elections-2026" - }, - "expected_metastruct": { - "evaluate_id": "9c9036fc-d7a1-5d5e-bcb3-7d62cc9fa218", - "policy": { - "name": "elections-2026" - } + "ai_guard.evaluate_id": "9c9036fc-d7a1-5d5e-bcb3-7d62cc9fa218" }, + "expected_metastruct": {}, "skipped_tags": [], "cassette": "aiguard_evaluate_post_2b336647.json" }, @@ -217,10 +183,6 @@ } ], "extra_metastruct": [ - { - "key": "evaluate_id", - "value": "55e3505d-93eb-5171-8166-90c709865d83" - }, { "key": "policy_details", "value": { @@ -234,7 +196,6 @@ "ai_guard.policy_name": "destructive-actions" }, "expected_metastruct": { - "evaluate_id": "55e3505d-93eb-5171-8166-90c709865d83", "policy_details": { "rule": "no-recursive-delete", "severity": "high" @@ -340,6 +301,10 @@ "rule": "embargo-check" } }, + { + "key": "ai_guard.policy_enforced", + "value": true + }, { "key": "ai_guard.policy_name", "value": "newsroom-default" @@ -364,6 +329,10 @@ "name": "empty key" } }, + { + "key": "policy_note", + "value": null + }, { "key": "policy", "value": { @@ -381,7 +350,10 @@ }, "skipped_tags": [ "ai_guard.policy_revision", - "ai_guard.policy_details" + "ai_guard.policy_details", + "ai_guard.policy_enforced", + "1", + "" ], "cassette": "aiguard_evaluate_post_19cfc9ce.json" } diff --git a/tests/ai_guard/test_ai_guard_sdk.py b/tests/ai_guard/test_ai_guard_sdk.py index 064ec50f39e..717d80eab2b 100644 --- a/tests/ai_guard/test_ai_guard_sdk.py +++ b/tests/ai_guard/test_ai_guard_sdk.py @@ -1390,14 +1390,23 @@ def setup_evaluate_id(self): self.scenario, self.r = _post_extra_tags_scenario("EVALUATE_ID") def test_evaluate_id(self): - """The evaluation ID lands as ai_guard.evaluate_id and as evaluate_id, correlating the span with the backend.""" + """The evaluation ID lands only as the ai_guard.evaluate_id tag, correlating the span with the backend.""" assert self.r.status_code == 200 evaluation_id = self.scenario["evaluation_id"] - assert self.scenario["expected_tags"]["ai_guard.evaluate_id"] == evaluation_id - assert self.scenario["expected_metastruct"]["evaluate_id"] == evaluation_id - interfaces.library.validate_one_span( - self.r, validator=_assert_extra_tags_applied(self.scenario), full_trace=True - ) + assert self.scenario["expected_tags"] == {"ai_guard.evaluate_id": evaluation_id} + assert self.scenario["expected_metastruct"] == {} + extra_tags_validator = _assert_extra_tags_applied(self.scenario) + + def validate(span: DataDogLibrarySpan): + if not extra_tags_validator(span): + return False + _assert_key(span["meta"], "ai_guard.evaluate_id", evaluation_id) + ai_guard = _assert_key(span["meta_struct"], "ai_guard") + for key in ("evaluate_id", "ai_guard.evaluate_id"): + assert key not in ai_guard, f"The evaluation ID must not be in the ai_guard meta struct: {ai_guard}" + return True + + interfaces.library.validate_one_span(self.r, validator=validate, full_trace=True) def setup_keys_not_prefixed_or_filtered(self): self.scenario, self.r = _post_extra_tags_scenario("KEYS_VERBATIM") diff --git a/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_19cfc9ce.json b/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_19cfc9ce.json index 6cbb64d3c9f..9a3069ff6ea 100644 --- a/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_19cfc9ce.json +++ b/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_19cfc9ce.json @@ -15,8 +15,8 @@ }, "headers": { "content-type": "application/vnd.api+json", - "content-length": "1131" + "content-length": "1222" }, - "body": "{\"data\": {\"id\": \"f55f1430-86f7-5441-96b5-9938f4a10261\", \"type\": \"evaluations\", \"attributes\": {\"action\": \"ALLOW\", \"is_blocking_enabled\": true, \"reason\": \"No rule match.\", \"sds_findings\": [], \"tag_probs\": {\"authority-override\": 0.0, \"data-exfiltration\": 0.0, \"denial-of-service-tool-call\": 0.0, \"destructive-tool-call\": 0.0, \"indirect-prompt-injection\": 0.0, \"instruction-override\": 0.0, \"jailbreak\": 0.0, \"obfuscation\": 0.0, \"role-play\": 0.0, \"security-exploit\": 0.0, \"system-prompt-extraction\": 0.0}, \"tags\": [], \"extra_tags\": [\"ai_guard.policy_name=newsroom-default\", [\"ai_guard.policy_name\", \"newsroom-default\"], {\"value\": \"missing key\"}, {\"key\": 1, \"value\": \"integer key\"}, {\"key\": \"\", \"value\": \"empty key\"}, {\"key\": \"ai_guard.policy_revision\", \"value\": 7}, {\"key\": \"ai_guard.policy_details\", \"value\": {\"rule\": \"embargo-check\"}}, {\"key\": \"ai_guard.policy_name\", \"value\": \"newsroom-default\"}], \"extra_metastruct\": [\"policy=newsroom-default\", {\"value\": {\"name\": \"missing key\"}}, {\"key\": 1, \"value\": {\"name\": \"integer key\"}}, {\"key\": \"\", \"value\": {\"name\": \"empty key\"}}, {\"key\": \"policy\", \"value\": {\"name\": \"newsroom-default\"}}]}}}" + "body": "{\"data\": {\"id\": \"f55f1430-86f7-5441-96b5-9938f4a10261\", \"type\": \"evaluations\", \"attributes\": {\"action\": \"ALLOW\", \"is_blocking_enabled\": true, \"reason\": \"No rule match.\", \"sds_findings\": [], \"tag_probs\": {\"authority-override\": 0.0, \"data-exfiltration\": 0.0, \"denial-of-service-tool-call\": 0.0, \"destructive-tool-call\": 0.0, \"indirect-prompt-injection\": 0.0, \"instruction-override\": 0.0, \"jailbreak\": 0.0, \"obfuscation\": 0.0, \"role-play\": 0.0, \"security-exploit\": 0.0, \"system-prompt-extraction\": 0.0}, \"tags\": [], \"extra_tags\": [\"ai_guard.policy_name=newsroom-default\", [\"ai_guard.policy_name\", \"newsroom-default\"], {\"value\": \"missing key\"}, {\"key\": 1, \"value\": \"integer key\"}, {\"key\": \"\", \"value\": \"empty key\"}, {\"key\": \"ai_guard.policy_revision\", \"value\": 7}, {\"key\": \"ai_guard.policy_details\", \"value\": {\"rule\": \"embargo-check\"}}, {\"key\": \"ai_guard.policy_enforced\", \"value\": true}, {\"key\": \"ai_guard.policy_name\", \"value\": \"newsroom-default\"}], \"extra_metastruct\": [\"policy=newsroom-default\", {\"value\": {\"name\": \"missing key\"}}, {\"key\": 1, \"value\": {\"name\": \"integer key\"}}, {\"key\": \"\", \"value\": {\"name\": \"empty key\"}}, {\"key\": \"policy_note\", \"value\": null}, {\"key\": \"policy\", \"value\": {\"name\": \"newsroom-default\"}}]}}}" } } diff --git a/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_28917f95.json b/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_28917f95.json index d0ee30f946e..a3a352eb70e 100644 --- a/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_28917f95.json +++ b/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_28917f95.json @@ -15,8 +15,8 @@ }, "headers": { "content-type": "application/vnd.api+json", - "content-length": "901" + "content-length": "828" }, - "body": "{\"data\": {\"id\": \"55e3505d-93eb-5171-8166-90c709865d83\", \"type\": \"evaluations\", \"attributes\": {\"action\": \"DENY\", \"is_blocking_enabled\": true, \"reason\": \"Destructive tool call detected.\", \"sds_findings\": [], \"tag_probs\": {\"authority-override\": 0.0, \"data-exfiltration\": 0.0, \"denial-of-service-tool-call\": 0.0, \"destructive-tool-call\": 0.97, \"indirect-prompt-injection\": 0.0, \"instruction-override\": 0.0, \"jailbreak\": 0.0, \"obfuscation\": 0.0, \"role-play\": 0.0, \"security-exploit\": 0.0, \"system-prompt-extraction\": 0.0}, \"tags\": [\"destructive-tool-call\"], \"extra_tags\": [{\"key\": \"ai_guard.evaluate_id\", \"value\": \"55e3505d-93eb-5171-8166-90c709865d83\"}, {\"key\": \"ai_guard.policy_name\", \"value\": \"destructive-actions\"}], \"extra_metastruct\": [{\"key\": \"evaluate_id\", \"value\": \"55e3505d-93eb-5171-8166-90c709865d83\"}, {\"key\": \"policy_details\", \"value\": {\"rule\": \"no-recursive-delete\", \"severity\": \"high\"}}]}}}" + "body": "{\"data\": {\"id\": \"55e3505d-93eb-5171-8166-90c709865d83\", \"type\": \"evaluations\", \"attributes\": {\"action\": \"DENY\", \"is_blocking_enabled\": true, \"reason\": \"Destructive tool call detected.\", \"sds_findings\": [], \"tag_probs\": {\"authority-override\": 0.0, \"data-exfiltration\": 0.0, \"denial-of-service-tool-call\": 0.0, \"destructive-tool-call\": 0.97, \"indirect-prompt-injection\": 0.0, \"instruction-override\": 0.0, \"jailbreak\": 0.0, \"obfuscation\": 0.0, \"role-play\": 0.0, \"security-exploit\": 0.0, \"system-prompt-extraction\": 0.0}, \"tags\": [\"destructive-tool-call\"], \"extra_tags\": [{\"key\": \"ai_guard.evaluate_id\", \"value\": \"55e3505d-93eb-5171-8166-90c709865d83\"}, {\"key\": \"ai_guard.policy_name\", \"value\": \"destructive-actions\"}], \"extra_metastruct\": [{\"key\": \"policy_details\", \"value\": {\"rule\": \"no-recursive-delete\", \"severity\": \"high\"}}]}}}" } } diff --git a/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_2b336647.json b/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_2b336647.json index 5f4665efd88..d9b44d35d42 100644 --- a/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_2b336647.json +++ b/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_2b336647.json @@ -15,8 +15,8 @@ }, "headers": { "content-type": "application/vnd.api+json", - "content-length": "823" + "content-length": "612" }, - "body": "{\"data\": {\"id\": \"9c9036fc-d7a1-5d5e-bcb3-7d62cc9fa218\", \"type\": \"evaluations\", \"attributes\": {\"action\": \"ALLOW\", \"is_blocking_enabled\": true, \"reason\": \"No rule match.\", \"sds_findings\": [], \"tag_probs\": {\"authority-override\": 0.0, \"data-exfiltration\": 0.0, \"denial-of-service-tool-call\": 0.0, \"destructive-tool-call\": 0.0, \"indirect-prompt-injection\": 0.0, \"instruction-override\": 0.0, \"jailbreak\": 0.0, \"obfuscation\": 0.0, \"role-play\": 0.0, \"security-exploit\": 0.0, \"system-prompt-extraction\": 0.0}, \"tags\": [], \"extra_tags\": [{\"key\": \"ai_guard.evaluate_id\", \"value\": \"9c9036fc-d7a1-5d5e-bcb3-7d62cc9fa218\"}, {\"key\": \"ai_guard.policy_name\", \"value\": \"elections-2026\"}], \"extra_metastruct\": [{\"key\": \"evaluate_id\", \"value\": \"9c9036fc-d7a1-5d5e-bcb3-7d62cc9fa218\"}, {\"key\": \"policy\", \"value\": {\"name\": \"elections-2026\"}}]}}}" + "body": "{\"data\": {\"id\": \"9c9036fc-d7a1-5d5e-bcb3-7d62cc9fa218\", \"type\": \"evaluations\", \"attributes\": {\"action\": \"ALLOW\", \"is_blocking_enabled\": true, \"reason\": \"No rule match.\", \"sds_findings\": [], \"tag_probs\": {\"authority-override\": 0.0, \"data-exfiltration\": 0.0, \"denial-of-service-tool-call\": 0.0, \"destructive-tool-call\": 0.0, \"indirect-prompt-injection\": 0.0, \"instruction-override\": 0.0, \"jailbreak\": 0.0, \"obfuscation\": 0.0, \"role-play\": 0.0, \"security-exploit\": 0.0, \"system-prompt-extraction\": 0.0}, \"tags\": [], \"extra_tags\": [{\"key\": \"ai_guard.evaluate_id\", \"value\": \"9c9036fc-d7a1-5d5e-bcb3-7d62cc9fa218\"}]}}}" } } diff --git a/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_827d1449.json b/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_827d1449.json index ddb025e8df5..3ee1eee7dc9 100644 --- a/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_827d1449.json +++ b/utils/build/docker/vcr/cassettes/aiguard/aiguard_evaluate_post_827d1449.json @@ -15,8 +15,8 @@ }, "headers": { "content-type": "application/vnd.api+json", - "content-length": "904" + "content-length": "775" }, - "body": "{\"data\": {\"id\": \"73606e72-26b6-52f8-8652-7f321db98dda\", \"type\": \"evaluations\", \"attributes\": {\"action\": \"ALLOW\", \"is_blocking_enabled\": true, \"reason\": \"No rule match.\", \"sds_findings\": [], \"tag_probs\": {\"authority-override\": 0.0, \"data-exfiltration\": 0.0, \"denial-of-service-tool-call\": 0.0, \"destructive-tool-call\": 0.0, \"indirect-prompt-injection\": 0.0, \"instruction-override\": 0.0, \"jailbreak\": 0.0, \"obfuscation\": 0.0, \"role-play\": 0.0, \"security-exploit\": 0.0, \"system-prompt-extraction\": 0.0}, \"tags\": [], \"extra_metastruct\": [{\"key\": \"request_context\", \"value\": {\"desk\": \"metro\", \"region\": \"us-east\", \"priority\": 2}}, {\"key\": \"matched_rules\", \"value\": [\"style-guide\", \"embargo-check\"]}, {\"key\": \"policy_rules\", \"value\": [{\"id\": \"sourcing\", \"enabled\": true}, {\"id\": \"attribution\", \"enabled\": false}]}, {\"key\": \"policy_score\", \"value\": 0.25}, {\"key\": \"policy_name\", \"value\": \"newsroom-default\"}]}}}" + "body": "{\"data\": {\"id\": \"73606e72-26b6-52f8-8652-7f321db98dda\", \"type\": \"evaluations\", \"attributes\": {\"action\": \"ALLOW\", \"is_blocking_enabled\": true, \"reason\": \"No rule match.\", \"sds_findings\": [], \"tag_probs\": {\"authority-override\": 0.0, \"data-exfiltration\": 0.0, \"denial-of-service-tool-call\": 0.0, \"destructive-tool-call\": 0.0, \"indirect-prompt-injection\": 0.0, \"instruction-override\": 0.0, \"jailbreak\": 0.0, \"obfuscation\": 0.0, \"role-play\": 0.0, \"security-exploit\": 0.0, \"system-prompt-extraction\": 0.0}, \"tags\": [], \"extra_metastruct\": [{\"key\": \"policy_match\", \"value\": {\"rule\": \"credit_card\", \"categories\": [\"pii\", \"financial\"]}}, {\"key\": \"matched_rules\", \"value\": [{\"id\": \"r1\", \"action\": \"DENY\"}, {\"id\": \"r2\", \"action\": \"ALLOW\"}]}, {\"key\": \"policy_version\", \"value\": \"v3\"}]}}}" } } diff --git a/utils/scripts/gen_extra_tags_cassettes.py b/utils/scripts/gen_extra_tags_cassettes.py index 22bd856f86d..3a05d8e2df2 100755 --- a/utils/scripts/gen_extra_tags_cassettes.py +++ b/utils/scripts/gen_extra_tags_cassettes.py @@ -12,9 +12,9 @@ struct under the key exactly as returned. The backend owns the ai_guard prefix and the namespace, so tracers never add the prefix or filter -keys. The first entry the backend sends is the evaluation ID (the same value as data.id), as -ai_guard.evaluate_id in extra_tags and as evaluate_id in extra_metastruct. A malformed field or -entry is skipped and never changes the evaluation. +keys. The backend sends the evaluation ID (the same value as data.id) only in extra_tags, as +ai_guard.evaluate_id; extra_metastruct is empty, and so omitted, in today's responses. A malformed +field or entry is skipped and never changes the evaluation. The backend does not return these attributes in the recorded cassettes, so this script writes them by hand, reusing the request hashing and rendering of gen_redaction_cassettes.py, and writes @@ -60,12 +60,9 @@ def _kv(**entries: object) -> list[dict]: return [{"key": key, "value": value} for key, value in entries.items()] -def _evaluate_id_entries(name: str) -> tuple[list[dict], list[dict]]: - evaluation_id = _evaluation_id(name) - return ( - [{"key": "ai_guard.evaluate_id", "value": evaluation_id}], - [{"key": "evaluate_id", "value": evaluation_id}], - ) +def _evaluate_id_tag(name: str) -> list[dict]: + """The extra_tags entry carrying the evaluation ID, the only place the backend sends it.""" + return [{"key": "ai_guard.evaluate_id", "value": _evaluation_id(name)}] # Every scenario produces one cassette and one sidecar entry: @@ -84,24 +81,25 @@ def _evaluate_id_entries(name: str) -> tuple[list[dict], list[dict]]: "tags": [], "extra_tags": _kv(**{"ai_guard.policy_name": "newsroom-default", "ai_guard.policy_revision": "7"}), }, + # The illustrative extra_metastruct from the backend contract: an object, an array of objects and + # a scalar. No null anywhere and no boolean directly inside an array, shapes the backend avoids + # and that some tracers' meta struct encoders drop. "EXTRA_METASTRUCT": { "messages": [{"role": "user", "content": "Draft a headline for the city council budget story."}], "action": "ALLOW", "tags": [], "extra_metastruct": _kv( - request_context={"desk": "metro", "region": "us-east", "priority": 2}, - matched_rules=["style-guide", "embargo-check"], - policy_rules=[{"id": "sourcing", "enabled": True}, {"id": "attribution", "enabled": False}], - policy_score=0.25, - policy_name="newsroom-default", + policy_match={"rule": "credit_card", "categories": ["pii", "financial"]}, + matched_rules=[{"id": "r1", "action": "DENY"}, {"id": "r2", "action": "ALLOW"}], + policy_version="v3", ), }, + # Mirrors today's backend response: the evaluation ID in extra_tags only, no extra_metastruct. "EVALUATE_ID": { "messages": [{"role": "user", "content": "Which sources did the election explainer cite?"}], "action": "ALLOW", "tags": [], - "extra_tags": _evaluate_id_entries("EVALUATE_ID")[0] + _kv(**{"ai_guard.policy_name": "elections-2026"}), - "extra_metastruct": _evaluate_id_entries("EVALUATE_ID")[1] + _kv(policy={"name": "elections-2026"}), + "extra_tags": _evaluate_id_tag("EVALUATE_ID"), }, # The backend guarantees every extra_tags key is in the ai_guard namespace; the tracer must not # double check it. A key outside the namespace still lands verbatim, which is what proves the @@ -128,9 +126,8 @@ def _evaluate_id_entries(name: str) -> tuple[list[dict], list[dict]]: ], "action": "DENY", "tags": ["destructive-tool-call"], - "extra_tags": _evaluate_id_entries("ON_DENY")[0] + _kv(**{"ai_guard.policy_name": "destructive-actions"}), - "extra_metastruct": _evaluate_id_entries("ON_DENY")[1] - + _kv(policy_details={"rule": "no-recursive-delete", "severity": "high"}), + "extra_tags": _evaluate_id_tag("ON_DENY") + _kv(**{"ai_guard.policy_name": "destructive-actions"}), + "extra_metastruct": _kv(policy_details={"rule": "no-recursive-delete", "severity": "high"}), }, # extra_tags in the map shape of the first draft of the contract: ignored as a whole, while the # well-formed extra_metastruct next to it is still applied. @@ -155,8 +152,9 @@ def _evaluate_id_entries(name: str) -> tuple[list[dict], list[dict]]: "skipped_tags": [], }, # Each malformed entry is skipped on its own: the well-formed entries around it still land. - # extra_tags only carries strings, so a number or an object there is skipped too, and never - # rerouted to the meta struct. + # extra_tags only carries strings, so a number, a boolean or an object there is skipped too, and + # never rerouted to the meta struct. A null extra_metastruct value is skipped, and a non-string or + # empty key is never stringified or kept. "MALFORMED_ENTRIES": { "messages": [{"role": "user", "content": "When does the weekend magazine go to print?"}], "action": "ALLOW", @@ -169,6 +167,7 @@ def _evaluate_id_entries(name: str) -> tuple[list[dict], list[dict]]: {"key": "", "value": "empty key"}, {"key": "ai_guard.policy_revision", "value": 7}, {"key": "ai_guard.policy_details", "value": {"rule": "embargo-check"}}, + {"key": "ai_guard.policy_enforced", "value": True}, {"key": "ai_guard.policy_name", "value": "newsroom-default"}, ], "extra_metastruct": [ @@ -176,11 +175,12 @@ def _evaluate_id_entries(name: str) -> tuple[list[dict], list[dict]]: {"value": {"name": "missing key"}}, {"key": 1, "value": {"name": "integer key"}}, {"key": "", "value": {"name": "empty key"}}, + {"key": "policy_note", "value": None}, {"key": "policy", "value": {"name": "newsroom-default"}}, ], "expected_tags": {"ai_guard.policy_name": "newsroom-default"}, "expected_metastruct": {"policy": {"name": "newsroom-default"}}, - "skipped_tags": ["ai_guard.policy_revision", "ai_guard.policy_details"], + "skipped_tags": ["ai_guard.policy_revision", "ai_guard.policy_details", "ai_guard.policy_enforced", "1", ""], }, }