Rule engine: read retrieval documents that arrive as JSON strings

audits/agentic/rule_engine.py

@@ -144,6 +144,11 @@ def _units(kind: str, trajectory: AgentTrajectory) -> list[tuple[Any, str]]:
                 except json.JSONDecodeError:
                     docs = []
             for doc in docs if isinstance(docs, list) else []:
+                if isinstance(doc, str):
+                    try:
+                        doc = json.loads(doc)
+                    except json.JSONDecodeError:
+                        continue
                 if isinstance(doc, dict):
                     out.append((doc, step.span_id))
         return out

audits/agentic/semantic_judge.py

@@ -60,11 +60,26 @@ def _compact_result(result: Any) -> str:
     return _bounded(json.dumps(_compact(result), ensure_ascii=False, default=str), _MAX_RESULT_CHARS)
 
 
+def _document(item: Any) -> Any:
+    """A retrieval document, parsed when it arrives as a JSON object or list; any other string stays a name."""
+    if isinstance(item, str):
+        try:
+            parsed = json.loads(item)
+        except json.JSONDecodeError:
+            return item
+        if isinstance(parsed, (dict, list)):
+            return parsed
+    return item
+
+
 def _retrieved_sources(trajectory: AgentTrajectory) -> list[str]:
     """Sources of every document the trace shows, whichever step returned it."""
     sources: set[str] = set()
     for step in trajectory.steps:
-        sources.update(_source_values(step.attributes.get("gen_ai.retrieval.documents")))
+        documents = step.attributes.get("gen_ai.retrieval.documents")
+        if isinstance(documents, list):
+            documents = [_document(item) for item in documents]
+        sources.update(_source_values(documents))
         if step.kind == "tool":
             sources.update(_captured_tool_sources(step))
     return sorted(sources)

audits/agentic/tests/test_rule_engine.py

@@ -99,3 +99,28 @@ def test_all_check_with_nothing_to_check_is_inconclusive_not_pass():
     rule = {"type": "check", "select": {"kind": "tool", "where": [
         {"field": "tool.name", "op": "equals", "value": "lookup"}]}, "check": "all"}
     assert evaluate_check(0, rule, normalize([_answer("no tools")])).status == "INCONCLUSIVE"
+
+
+def _retrieval(documents):
+    return {"span_id": "r1", "name": "retrieval", "attributes": {
+        "gen_ai.operation.name": "retrieval", "gen_ai.retrieval.documents": documents}}
+
+
+def test_retrieval_documents_sent_as_json_strings_are_read():
+    # Open WebUI's instrumentation exports each document as its own JSON string, without a source.
+    owui = normalize([_retrieval([
+        json.dumps({"document_id": "doc-1", "file_id": "f-1", "chunk_id": "c-1", "rank": 1, "score": 0.9,
+                    "content": "Returns in 30 days."}),
+        "not json",
+    ])])
+    count = {"type": "check", "select": {"kind": "retrieval_document", "where": []}, "check": "count",
+             "expect": {"op": "equals", "value": 1}}
+    text = {"type": "check", "select": {"kind": "retrieval_document", "where": [
+        {"field": "retrieval.text", "op": "contains", "value": "30 days"}]}, "check": "exists"}
+    named = {"type": "check", "select": {"kind": "retrieval_document", "where": [
+        {"field": "retrieval.source", "op": "equals", "value": "customer_faq.md"}]}, "check": "exists"}
+    assert evaluate_check(0, count, owui).status == "PASS"
+    assert evaluate_check(1, text, owui).status == "PASS"
+    assert evaluate_check(2, named, owui).status == "INCONCLUSIVE"
+    sourced = normalize([_retrieval([json.dumps({"source": "customer_faq.md", "content": "Returns in 30 days."})])])
+    assert evaluate_check(2, named, sourced).status == "PASS"

audits/agentic/tests/test_semantic_judge_execution.py

@@ -137,3 +137,15 @@ def test_packet_keeps_every_retrieved_source_and_drops_noise():
     result = packet["tool_steps"][0]["result"]
     assert "score" not in result
     assert len(result) <= 1700
+
+
+def test_packet_reads_retrieval_documents_sent_as_json_strings():
+    # Open WebUI's instrumentation exports each document as its own JSON string.
+    trajectory = AgentTrajectory([TrajectoryStep("r1", "retrieval", "retrieval", attributes={
+        "gen_ai.retrieval.documents": [
+            json.dumps({"document_id": "doc-1", "file_id": "f-1", "chunk_id": "c-1", "rank": 1, "score": 0.9}),
+            json.dumps({"source": "customer_faq.md", "content": "Returns in 30 days."}),
+            "shipping.md",
+        ]})])
+    packet = build_evidence_packet(trajectory)
+    assert packet["retrieval_source_ids"] == ["customer_faq.md", "f-1", "shipping.md"]