chore(llm): switch default LLM from gpt-4o/gpt-4o-mini to gpt-5.6-sol

.github/workflows/issue-formatter.yml

@@ -27,7 +27,7 @@ jobs:
                 'Authorization': `Bearer ${process.env.OPENAI_API_KEY}`
               },
               body: JSON.stringify({
-                model: 'gpt-4o-mini',
+                model: 'gpt-5.6-luna',
                 messages: [{
                   role: 'user',
                   content: `Rewrite this GitHub issue body using the exact format below. Keep it short, crisp and easy to understand. Do NOT add any information that wasn't in the original. Output ONLY the formatted markdown, nothing else.
@@ -79,7 +79,7 @@ jobs:
                 'Authorization': `Bearer ${process.env.OPENAI_API_KEY}`
               },
               body: JSON.stringify({
-                model: 'gpt-4o-mini',
+                model: 'gpt-5.6-sol',
                 messages: [{
                   role: 'user',
                   content: `Based on the issue body below, generate a short title in this exact format:

backend/app/core/config.py

@@ -60,8 +60,8 @@ class Settings(BaseSettings):
     ALLOWED_IPS: Annotated[list[str] | str, BeforeValidator(parse_ip_list)] = []
     CORE_DIR: ClassVar[Path] = Path(__file__).resolve().parent
     OPENAI_API_KEY: str | None = None
-    ANSWER_RELEVANCE_LLM_MODEL: str = "gpt-4o-mini"
-    DEFAULT_LLM_CALLABLE: str = "gpt-4o-mini"
+    ANSWER_RELEVANCE_LLM_MODEL: str = "gpt-5.6-sol"
+    DEFAULT_LLM_CALLABLE: str = "gpt-5.6-sol"
     TOPIC_RELEVANCE_LLM_THRESHOLD: int = 2
 
     SLUR_LIST_FILENAME: ClassVar[str] = "curated_slurlist_hi_en.csv"

backend/app/core/validators/README.md

@@ -319,7 +319,7 @@ Parameters / customization:
 
 - `metrics: dict` (required) — metric name-to-description mapping passed to the LLM judge
 - `max_score: int` (required) — maximum score per metric; used to define the scoring scale
-- `llm_callable: str` (required) — model identifier passed to LiteLLM (e.g. `gpt-4o-mini`, `gpt-4o`)
+- `llm_callable: str` (required) — model identifier passed to LiteLLM (e.g. `gpt-5.6-sol`)
 - `on_fail`
 
 Notes / limitations:
@@ -359,7 +359,7 @@ Parameters / customization:
 
 - `topic_relevance_config_id: UUID` (required at runtime; resolves configuration and prompt version from tenant config)
 - `prompt_schema_version: int` (optional; defaults to `1`)
-- `llm_callable: str` (default: `gpt-4o-mini`) — the model identifier passed to Guardrails' LLMCritic to perform the scope evaluation. This must be a model string supported by LiteLLM (e.g. `gpt-4o-mini`, `gpt-4o`). It controls which LLM is used to score whether the input is within the allowed topic scope; changing it affects cost, latency, and scoring quality.
+- `llm_callable: str` (default: `gpt-5.6-sol`) — the model identifier passed to Guardrails' LLMCritic to perform the scope evaluation. This must be a model string supported by LiteLLM (e.g. `gpt-5.6-sol`). It controls which LLM is used to score whether the input is within the allowed topic scope; changing it affects cost, latency, and scoring quality.
 - `on_fail`
 
 Notes / limitations:
@@ -387,10 +387,28 @@ Parameters / customization:
 - `configuration: str` — topic text. Inline, or resolved from `topic_relevance_config_id`.
 - `prompt_schema_version: int` (default `2`) — `1` = `configuration` lists allowed topics, `2` = forbidden topics, `3` = both (forbidden checked first). Resolved from the stored config when `topic_relevance_config_id` is used.
 - `threshold: int` (1–3, default `2`) — `3` also fails ambiguous messages.
-- `llm_callable: str` (default `gpt-4o-mini`)
+- `llm_callable: str` (default `gpt-5.6-sol`)
 - `fallback_message: str` (optional) — with `on_fail=rephrase`, returned as `safe_text` instead of the generic rephrase text; `rephrase_needed` stays `true`. The same LLM call translates it into the user's language (`prompts/topic_relevance_llm/fallback_translation.md`); if no translation comes back, the original text is used. Adds ~30–60 output tokens per call when set.
 - `on_fail` — use `rephrase`. `fix` returns an empty string, since this validator has no fix value.
 
+LLM call:
+
+- `max_tokens=1024` (2048 when `fallback_message` is set). Reasoning models count hidden reasoning tokens against this limit, so it's sized well above observed use and only stops runaway calls.
+- `reasoning_effort="medium"` is sent when the model supports reasoning; other models don't receive it.
+
+Evaluation (live run, October 2026):
+
+- 100 rows sampled from the education and healthcare topic relevance datasets (`backend/app/evaluation/datasets/topic_relevance/`): 50 per domain, 32 out of scope. Prompt schema v2, threshold 2, `max_tokens=1024`.
+
+| Model | Accuracy | Out-of-scope caught | In-scope wrongly blocked | F1 | Latency (typical / slowest 5%) |
+|---|---|---|---|---|---|
+| `gpt-5.6-sol` (`reasoning_effort="medium"`) | 0.89 | 25 / 32 | 4 / 68 | 0.82 | 2.7s / 4.1s |
+| `gpt-4o-mini` | 0.89 | 21 / 32 | 0 / 68 | 0.79 | 1.3s / 1.6s |
+
+- `gpt-5.6-sol` catches more out-of-scope messages but blocks some in-scope ones. One example is "where can I get my baby's vaccination near my village?" on the healthcare config.
+- No empty, cut-off or unparseable replies in 300 `gpt-5.6-sol` calls. The highest usage was 263 output tokens.
+- `gpt-5.6-sol` is about 2x slower per call, and lowering `reasoning_effort` barely changes that. Expect roughly 2.5–4s per call.
+
 ### 7) LlamaGuard 7B Validator (`llamaguard_7b`)
 
 Code:
@@ -538,7 +556,7 @@ Recommendation:
 
 Parameters / customization:
 
-- `llm_callable: str` (default: `gpt-4o-mini`) — model identifier passed to LiteLLM for the YES/NO evaluation
+- `llm_callable: str` (default: `gpt-5.6-sol`) — model identifier passed to LiteLLM for the YES/NO evaluation
 - `prompt_template: str` (optional) — inline prompt with `{query}` and `{answer}` placeholders
 - `custom_prompt_id: UUID` (optional) — reference to a tenant-stored prompt config; resolved to `prompt_template` before execution
 - `on_fail`
@@ -557,6 +575,8 @@ Notes / limitations:
 - **Requires `OPENAI_API_KEY` to be set in environment variables.**
 - `input` to the guardrail endpoint must be a JSON string: `{"query": "...", "answer": "..."}`. Both fields must be non-empty.
 - LLM-judge responses can vary; YES/NO parsing uses prefix matching.
+- The LLM call uses `max_tokens=512` so reasoning models (e.g. `gpt-5.6-sol`) have room for hidden reasoning tokens, and sends `reasoning_effort="medium"` when the model supports reasoning. An empty reply fails as an unexpected response.
+- Evaluation (live run, October 2026, 100 query/answer pairs): `gpt-5.6-sol` scored 0.91–0.92 accuracy against 0.84 for `gpt-4o-mini`. It's about 2x slower (1.2–1.4s typical, 2.7–2.8s for the slowest 5%). Neither model passed a wrong answer; every error was a correct answer rejected by the strict default prompt. Full results: `docs/validators/answer-relevance.md#evaluation-results`.
 - `on_fail=fix` has no programmatic fix for irrelevant answers — `safe_text` will be `""` and `metadata.reason` will identify this validator.
 - If `custom_prompt_id` is deleted after being referenced, the guardrail will return a 404 at resolution time.
 

backend/app/core/validators/answer_relevance_custom_llm.py

@@ -18,7 +18,10 @@
     ANSWER_RELEVANCE_NOT_RELEVANT_ERROR,
     ANSWER_RELEVANCE_UNEXPECTED_RESPONSE_TEMPLATE,
 )
-from app.core.validators.llm_utils import traced_completion as completion
+from app.core.validators.llm_utils import (
+    supports_reasoning,
+    traced_completion as completion,
+)
 
 DEFAULT_PROMPT_TEMPLATE = (
     "Query: {query}\n"
@@ -27,6 +30,11 @@
     "Answer only YES or NO."
 )
 
+# Room for a one-word YES/NO plus the hidden reasoning tokens that reasoning
+# models (e.g. gpt-5.6-sol) count against the same completion budget.
+_MAX_TOKENS = 512
+_REASONING_EFFORT = "medium"
+
 
 @register_validator(name="answer-relevance-custom-llm", data_type="string")
 class AnswerRelevanceCustomLLM(Validator):
@@ -62,13 +70,17 @@ def _validate(self, value: str, metadata: dict | None = None) -> ValidationResul
                 )
             )
 
+        completion_kwargs = {
+            "model": self.llm_callable,
+            "messages": [{"role": "user", "content": prompt}],
+            "max_tokens": _MAX_TOKENS,
+        }
+        if supports_reasoning(self.llm_callable):
+            completion_kwargs["reasoning_effort"] = _REASONING_EFFORT
+
         try:
-            response = completion(
-                model=self.llm_callable,
-                messages=[{"role": "user", "content": prompt}],
-                max_tokens=10,
-            )
-            response_text = response.choices[0].message.content.strip().upper()
+            response = completion(**completion_kwargs)
+            response_text = (response.choices[0].message.content or "").strip().upper()
         except Exception as e:
             return FailResult(
                 error_message=ANSWER_RELEVANCE_LLM_CALL_FAILED_TEMPLATE.format(error=e)

backend/app/core/validators/llm_utils.py

@@ -1,4 +1,5 @@
 from litellm import completion, get_supported_openai_params
+from litellm import supports_reasoning as _litellm_supports_reasoning
 
 # Passed to litellm/OpenAI to force a strict JSON object response.
 JSON_OBJECT_RESPONSE_FORMAT = {"type": "json_object"}
@@ -19,8 +20,7 @@ def traced_completion(**kwargs):
 # Models known to support JSON-object response_format that litellm may not list yet.
 _KNOWN_JSON_CAPABLE_MODELS = frozenset(
     {
-        "gpt-4o-mini",
-        "gpt-4o",
+        "gpt-5.6-sol",
         "gpt-5-mini",
         "gpt-5.4-mini",
         "gpt-5.4-nano",
@@ -42,3 +42,15 @@ def supports_response_format(model: str) -> bool:
         return "response_format" in (get_supported_openai_params(model=model) or [])
     except Exception:
         return False
+
+
+def supports_reasoning(model: str) -> bool:
+    """Return True if the given model is a reasoning model (accepts ``reasoning_effort``).
+
+    Reasoning models count hidden reasoning tokens against the completion budget,
+    so callers use this to set a low effort and leave room for the visible answer.
+    """
+    try:
+        return bool(_litellm_supports_reasoning(model=model))
+    except Exception:
+        return False

backend/app/core/validators/topic_relevance_llm.py

@@ -13,12 +13,12 @@
     Validator,
     register_validator,
 )
-from litellm import supports_reasoning as _litellm_supports_reasoning
 
 from app.core.config import settings
 from app.core.constants import EMPTY_MESSAGE_ERROR, TOPIC_OUT_OF_SCOPE_ERROR
 from app.core.validators.llm_utils import (
     JSON_OBJECT_RESPONSE_FORMAT,
+    supports_reasoning,
     supports_response_format,
     traced_completion as completion,
 )
@@ -47,13 +47,10 @@ def _extract_first_json_object(text: str) -> dict:
 )
 # Reasoning models (e.g. gpt-5.6-sol) accept reasoning_effort; non-reasoning models reject it.
 _REASONING_EFFORT = "medium"
-
-
-def supports_reasoning(model: str) -> bool:
-    try:
-        return bool(_litellm_supports_reasoning(model=model))
-    except Exception:
-        return False
+# Ceiling on output tokens, hidden reasoning included. Sized well above observed
+# gpt-5.6-sol usage at medium effort (max ~260) so it only stops runaway calls;
+# doubled when localized_fallback is requested.
+_MAX_TOKENS = 1024
 
 
 @lru_cache(maxsize=8)
@@ -106,6 +103,7 @@ def __init__(
         self._invalid_config_reason: Optional[str] = None
         self._system_prompt: Optional[str] = None
         self._supports_response_format: bool = False
+        self._max_tokens = _MAX_TOKENS * 2 if fallback_message else _MAX_TOKENS
 
         if not system_prompt or not system_prompt.strip():
             self._invalid_config_reason = "system_prompt is blank or missing"
@@ -143,6 +141,7 @@ def _validate(
                     {"role": "system", "content": self._system_prompt},
                     {"role": "user", "content": value},
                 ],
+                "max_tokens": self._max_tokens,
             }
             if supports_reasoning(self.llm_callable):
                 kwargs["reasoning_effort"] = _REASONING_EFFORT

backend/app/evaluation/README.md

@@ -234,7 +234,7 @@ python3 app/evaluation/topic_relevance/run.py --backend topic_relevance_llm
 
 `topic_relevance` (the `LLMCritic`-based validator) is only imported when that backend actually runs, so `--backend topic_relevance_llm` works without installing the `llm_critic` hub validator.
 
-> **Note:** Requires `OPENAI_API_KEY` to be set. Uses `gpt-4o-mini` by default (`DEFAULT_CONFIG` in the script).
+> **Note:** Requires `OPENAI_API_KEY` to be set. Uses `gpt-5.6-sol` by default (`settings.DEFAULT_LLM_CALLABLE`).
 
 ---
 

backend/app/tests/test_llm_validators.py

@@ -20,7 +20,7 @@
 _SAMPLE_TOPIC_CONFIG = dict(
     type="topic_relevance",
     configuration="Only answer about cooking.",
-    llm_callable="gpt-4o-mini",
+    llm_callable="gpt-5.6-sol",
 )
 
 _TOPIC_RELEVANCE_SETTINGS_PATH = (
@@ -70,7 +70,7 @@ def test_topic_relevance_blank_config_returns_fail_result():
 _SAMPLE_LLM_TOPIC_CONFIG = dict(
     type="topic_relevance_llm",
     configuration="Only answer about cooking.",
-    llm_callable="gpt-4o-mini",
+    llm_callable="gpt-5.6-sol",
 )
 
 _TOPIC_RELEVANCE_LLM_SETTINGS_PATH = (
@@ -146,7 +146,7 @@ def test_topic_relevance_llm_custom_threshold_forwarded_to_validator():
         "quality": {"description": "Is the response high quality?", "threshold": 2}
     },
     max_score=3,
-    llm_callable="gpt-4o-mini",
+    llm_callable="gpt-5.6-sol",
 )
 
 
@@ -273,7 +273,7 @@ def test_answer_relevance_build_passes_inline_prompt_template():
 
 def test_answer_relevance_build_passes_llm_callable():
     config = AnswerRelevanceCustomLLMSafetyValidatorConfig(
-        **{**_SAMPLE_ANSWER_RELEVANCE_CONFIG, "llm_callable": "gpt-4o"}
+        **{**_SAMPLE_ANSWER_RELEVANCE_CONFIG, "llm_callable": "openai/gpt-5.6-sol"}
     )
 
     with patch(_ANSWER_RELEVANCE_SETTINGS_PATH) as mock_settings, patch(
@@ -284,4 +284,4 @@ def test_answer_relevance_build_passes_llm_callable():
         config.build()
 
     _, kwargs = mock_validator.call_args
-    assert kwargs["llm_callable"] == "gpt-4o"
+    assert kwargs["llm_callable"] == "openai/gpt-5.6-sol"

backend/app/tests/validators/test_answer_relevance_custom_llm.py

@@ -220,7 +220,7 @@ def test_fails_on_unexpected_llm_response(validator):
 
 def test_llm_callable_is_forwarded():
     v = AnswerRelevanceCustomLLM(
-        llm_callable="gpt-4o",
+        llm_callable="openai/gpt-5.6-sol",
         input=QUERY,
         output=ANSWER_RELEVANT,
     )
@@ -231,4 +231,19 @@ def test_llm_callable_is_forwarded():
         mock_llm.return_value = _make_llm_response("YES")
         v._validate(ANSWER_RELEVANT)
 
-        assert mock_llm.call_args.kwargs["model"] == "gpt-4o"
+        assert mock_llm.call_args.kwargs["model"] == "openai/gpt-5.6-sol"
+
+
+# ---------------------------------------------------------------------------
+# Empty LLM reply
+# ---------------------------------------------------------------------------
+
+
+def test_empty_content_returns_unexpected_response_fail(validator):
+    with patch(
+        "app.core.validators.answer_relevance_custom_llm.completion"
+    ) as mock_llm:
+        mock_llm.return_value = _make_llm_response(None)
+        result = validator._validate(ANSWER_RELEVANT)
+
+    assert isinstance(result, FailResult)

backend/app/tests/validators/test_topic_relevance_llm.py

@@ -457,13 +457,21 @@ def test_fallback_instructions_added_to_system_prompt():
     assert FALLBACK in validator._system_prompt
 
 
-def test_no_max_tokens_cap_is_sent():
+def test_max_tokens_cap_is_sent(validator):
+    with patch("app.core.validators.topic_relevance_llm.completion") as mock_llm:
+        mock_llm.return_value = _make_llm_response('{"scope_violation": 3}')
+        validator._validate("How do I make pasta?")
+
+    assert mock_llm.call_args.kwargs["max_tokens"] == 1024
+
+
+def test_fallback_doubles_max_tokens_cap():
     validator = _fallback_validator()
     with patch("app.core.validators.topic_relevance_llm.completion") as mock_llm:
         mock_llm.return_value = _make_llm_response('{"scope_violation": 3}')
         validator._validate("How do I make pasta?")
 
-    assert "max_tokens" not in mock_llm.call_args.kwargs
+    assert mock_llm.call_args.kwargs["max_tokens"] == 2048
 
 
 def test_reasoning_model_gets_medium_effort():