From 0f6d81d9c8339012179f4ab1dc44183756d65ba4 Mon Sep 17 00:00:00 2001 From: itsskofficial Date: Sat, 10 Oct 2026 21:45:52 +0530 Subject: [PATCH 1/2] fix(llm): show known provider failures as plain sentences (#280) Rate limits (OpenRouter's free models per day and per minute), no credits, a rejected key, a model OpenRouter keeps for coding tools, an unknown or undownloaded model, and a provider that is down, slow or unreachable now reach chat, tasks, the model check-up and embeddings as one sentence that says what to do. The provider's raw reply only goes to the log. Tests serve error bodies recorded from the real providers to the real LiteLLM. --- CHANGELOG.md | 5 + docs/API.md | 8 ++ sentient/agent/loop.py | 12 +- sentient/gateway/routes/tasks.py | 4 +- sentient/llm/checkup.py | 4 +- sentient/llm/errors.py | 107 ++++++++++++++ sentient/llm/provider.py | 49 ++++++- sentient/tasks/service.py | 9 +- tests/test_provider_errors.py | 235 +++++++++++++++++++++++++++++++ 9 files changed, 417 insertions(+), 16 deletions(-) create mode 100644 sentient/llm/errors.py create mode 100644 tests/test_provider_errors.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 67bd2cc9..84eda10a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -183,6 +183,11 @@ and versions follow [Semantic Versioning](https://semver.org/). - A task that repeats the same failing step three times now pauses as stuck and asks what to do, instead of failing. ### Fixed +- When a cloud model says no, chat and tasks now say why in a plain sentence and what to do, instead of showing the + provider's raw reply. For example: "OpenRouter's free models have reached today's limit. Add credits on + openrouter.ai, wait until tomorrow, or pick another model in Settings > Models." This covers rate limits, an + account out of credits, a key that was mistyped or revoked, a free model OpenRouter keeps for coding tools, a model + that doesn't exist or isn't downloaded, and a provider that is down or too slow. The raw reply stays in the log. - Turning on "Use my Claude Code for chats" now shows the Claude Code version straight away, instead of sometimes showing nothing until you came back to the page. Settings also says that its replies take a few seconds to start and arrive in larger pieces than with your other models. diff --git a/docs/API.md b/docs/API.md index f86eee9e..651d1786 100644 --- a/docs/API.md +++ b/docs/API.md @@ -43,6 +43,14 @@ All carry `session_id` and `turn_id`. | `done` | `content` (final text), `message_id`, `cancelled?`, `memory_sources: [MemorySource]` (what this reply had in mind, section 2; `[]` when none), `dropped?: string[]` (section 17: messages queued behind a stopped reply, never sent). A stopped reply's `done` (`cancelled: true`) carries the kept message's `message_id` and its `memory_sources` too (none when it was stopped before it started) | | `approval.ack` | `approval_id`, `resolved` | +**Model errors** (#280). When a provider turns a request down for a known reason, `error.message` (and a task's +`error`, section 4) is one plain sentence that says what to do, e.g. `OpenRouter's free models have reached today's +limit. Add credits on openrouter.ai, wait until tomorrow, or pick another model in Settings > Models.` This covers rate +limits (OpenRouter's free models per day and per minute), no credits (402), a key the provider rejects (401), a model +the provider keeps for coding tools, a model it doesn't have (an Ollama model that isn't downloaded), and a provider +that is down, slow or unreachable. The provider's raw reply only goes to the engine log; other failures keep their +text. + **Context meter** (#131). After every model call `usage` says how full the model's context is: `context_used` is the prompt (the larger of what the provider reported and a local count with LiteLLM's bundled tokenizer, because Ollama reports only the part of a prompt it had not cached) plus the reply, `context_length` what the model reads at once in diff --git a/sentient/agent/loop.py b/sentient/agent/loop.py index 243ffabd..ef84849e 100644 --- a/sentient/agent/loop.py +++ b/sentient/agent/loop.py @@ -46,6 +46,7 @@ from sentient.config.schema import SentientConfig from sentient.files.extract import extract_text, is_image from sentient.llm import claude_code +from sentient.llm.errors import is_tool_support_error from sentient.llm.events import ( AgentEvent, ApprovalRequest, @@ -61,7 +62,13 @@ tool_progress_event, ) from sentient.llm.meter import measure -from sentient.llm.provider import LLMProvider, ProviderError, StreamChunk, ToolCall +from sentient.llm.provider import ( + LLMProvider, + PlainProviderError, + ProviderError, + StreamChunk, + ToolCall, +) from sentient.memory import review as memory_review from sentient.memory.facts import FactMemory from sentient.memory.sources import MemorySources @@ -214,8 +221,7 @@ def history_to_openai(rows: list[dict]) -> list[dict]: def is_tool_format_error(exc: Exception) -> bool: """True when the provider rejected the request because the model cannot do tool calling (e.g. an old Ollama model template emitting malformed tool calls).""" - m = str(exc).lower() - return "does not support tools" in m or "invalid character" in m or ("tool" in m and "pars" in m) + return not isinstance(exc, PlainProviderError) and is_tool_support_error(str(exc)) def _json_safe(value: Any) -> str: diff --git a/sentient/gateway/routes/tasks.py b/sentient/gateway/routes/tasks.py index 0a2eeb7b..8811124c 100644 --- a/sentient/gateway/routes/tasks.py +++ b/sentient/gateway/routes/tasks.py @@ -9,7 +9,7 @@ from pydantic import BaseModel, Field from sentient.gateway.deps import AUTH, get_core -from sentient.llm.provider import ModelRefused, ProviderError +from sentient.llm.provider import PlainProviderError, ProviderError from sentient.tasks.service import TaskConflict, TaskNotFound router = APIRouter(prefix="/api/tasks", tags=["tasks"], dependencies=AUTH) @@ -47,7 +47,7 @@ async def _guard[T](awaitable: Awaitable[T]) -> T: raise HTTPException(status_code=404, detail="Task not found") from exc except TaskConflict as exc: raise HTTPException(status_code=409, detail=str(exc)) from exc - except ModelRefused as exc: # a model that is set up can't do this job: say why and what to change + except PlainProviderError as exc: # a refused job, no credits, a bad key...: say why and what to change raise HTTPException(status_code=503, detail=str(exc)) from exc except ProviderError as exc: raise HTTPException(status_code=503, detail=f"The AI model is unavailable: {exc}") from exc diff --git a/sentient/llm/checkup.py b/sentient/llm/checkup.py index a9b77d7c..dafca633 100644 --- a/sentient/llm/checkup.py +++ b/sentient/llm/checkup.py @@ -21,7 +21,7 @@ from sentient import secrets from sentient.config.schema import ModelRoles, SentientConfig from sentient.llm import chatgpt, claude_code -from sentient.llm.provider import CLAUDE_CODE, ToolCall, provider_config +from sentient.llm.provider import CLAUDE_CODE, PlainProviderError, ToolCall, provider_config ROLES = ("primary", "fast", "planner", "executor", "vision", "voice", "embedding") TOOL_ROLES = {"primary", "fast", "executor", "vision", "voice"} # roles that run the agent loop with tools @@ -178,6 +178,8 @@ def _failure_fix(self, exc: BaseException) -> str: if isinstance(exc, TimeoutError): return (f"No reply within {int(self.run.timeout_s)} seconds. The model may be too big for this " "computer: try a smaller model, or a shorter context length.") + if isinstance(exc, PlainProviderError): + return "" # the reason already says what to do msg = str(exc).lower() if any(k in msg for k in ("401", "403", "api key", "api_key", "authentication", "unauthorized")): return f"{self.label} turned the key down. Check your {self.label} key under Providers." diff --git a/sentient/llm/errors.py b/sentient/llm/errors.py new file mode 100644 index 00000000..dc75f082 --- /dev/null +++ b/sentient/llm/errors.py @@ -0,0 +1,107 @@ +"""Plain sentences for the ways a model provider says no (issue #280). + +LiteLLM's errors carry the provider's raw reply (``litellm.RateLimitError: ... OpenrouterException - {"error": ...``). +``plain_error`` turns the failures people actually hit into one sentence that says what to do. The raw text only +goes to the log; anything unknown returns None and keeps its own message. +""" + +from __future__ import annotations + +SETTINGS = "Settings > Models" + +LABELS = { + "openrouter": "OpenRouter", "anthropic": "Anthropic", "openai": "OpenAI", "gemini": "Google Gemini", + "vertex_ai": "Google Vertex AI", "groq": "Groq", "mistral": "Mistral", "deepseek": "DeepSeek", "xai": "xAI", + "together_ai": "Together AI", "fireworks_ai": "Fireworks AI", "cohere": "Cohere", "perplexity": "Perplexity", + "nous": "Nous Portal", "azure": "Azure OpenAI", "bedrock": "Amazon Bedrock", "ollama": "Ollama", + "ollama_chat": "Ollama", "lm_studio": "LM Studio", "chatgpt": "ChatGPT", +} +# where each provider sells credits, for "out of credits" +BILLING = { + "openrouter": "openrouter.ai/settings/credits", "anthropic": "console.anthropic.com", + "openai": "platform.openai.com", "deepseek": "platform.deepseek.com", "xai": "console.x.ai", + "mistral": "console.mistral.ai", "nous": "portal.nousresearch.com", +} +LOCAL = {"ollama", "ollama_chat", "lm_studio", "llamafile", "vllm", "hosted_vllm"} + +CREDIT_HINTS = ( + "requires more credits", "insufficient credits", "insufficient_quota", "exceeded your current quota", + "credit balance is too low", "insufficient balance", "out of credits", "payment required", +) +KEY_HINTS = ("invalid api key", "invalid x-api-key", "incorrect api key", "api key not valid", "invalid_api_key", + "user not found", "no auth credentials") +NOT_FOUND_HINTS = ("not a valid model id", "model_not_found", "does not exist", "not found, try pulling it first", + "unknown model", "no such model") +UNREACHABLE_HINTS = ("cannot connect to host", "all connection attempts failed", "connection refused", + "refused the network connection", "connecterror", "getaddrinfo failed", "name or service not known", "nodename nor servname") + + +def _parts(model: str) -> tuple[str, str]: + prefix, _, name = model.partition("/") + return (prefix, name) if name else ("", model) + + +def _label(prefix: str) -> str: + return LABELS.get(prefix) or (prefix.replace("_", " ").title() if prefix else "The model's provider") + + +def _status(exc: BaseException) -> int | None: + code = getattr(exc, "status_code", None) + return code if isinstance(code, int) else None + + +def is_tool_support_error(text: str) -> bool: + """The provider turned the request down because the model can't call tools (the agent loop then answers + without tools, so these keep their own message).""" + m = text.lower() + return ("does not support tools" in m or "support tool use" in m or "invalid character" in m + or ("tool" in m and "pars" in m)) + + +def plain_error(exc: BaseException, model: str) -> str | None: + """A plain sentence for a known provider failure of ``model`` that says what to do, or None when unknown. + + Only LiteLLM's own errors are translated: Sentient's errors (Claude Code, the ChatGPT plan) are already plain. + """ + if not type(exc).__module__.startswith("litellm"): + return None + text = str(exc) + m = text.lower() + if is_tool_support_error(m): + return None + prefix, name = _parts(model) + label, status, kind = _label(prefix), _status(exc), type(exc).__name__ + pick = f"pick another model in {SETTINGS}" + + if "agentic harness" in m: + return (f"{label} only lets coding tools use {name}, not assistants like Sentient. " + f"Please {pick}.") + if "free-models-per-day" in m: + return (f"{label}'s free models have reached today's limit. Add credits on openrouter.ai, wait until " + f"tomorrow, or {pick}.") + if "free-models-per-min" in m: + return f"{label}'s free models allow only a few requests a minute. Wait a minute and try again, or {pick}." + if status == 402 or any(h in m for h in CREDIT_HINTS): + where = f" on {BILLING[prefix]}" if prefix in BILLING else "" + return f"Your {label} account is out of credits. Add credits{where}, or {pick}." + if "no endpoints found matching your data policy" in m: + return (f"Your {label} privacy settings don't allow any provider of {name}. Change them on " + f"openrouter.ai/settings/privacy, or {pick}.") + if status == 401 or kind == "AuthenticationError" or any(h in m for h in KEY_HINTS): + return f"{label} didn't accept your key. It may be mistyped or revoked: add a new one in {SETTINGS}." + if status == 404 or kind == "NotFoundError" or any(h in m for h in NOT_FOUND_HINTS): + if prefix in {"ollama", "ollama_chat"}: + return f"{name} isn't downloaded in Ollama. Download it in {SETTINGS}, or pick another model there." + return f"{label} doesn't have a model called {name}. Check the name, or {pick}." + if status == 429 or kind == "RateLimitError": + return f"{label} is getting too many requests right now. Wait a minute and try again, or {pick}." + if status == 408 or kind == "Timeout": + return f"{label} took too long to answer. Try again in a moment, or {pick}." + if any(h in m for h in UNREACHABLE_HINTS): + if prefix in LOCAL: + return f"Sentient couldn't reach {label}. Make sure it's running, then try again." + return f"Sentient couldn't reach {label}. Check your internet connection and try again." + if (kind in {"InternalServerError", "ServiceUnavailableError", "BadGatewayError"} + or (kind != "APIConnectionError" and status is not None and status >= 500) or "overloaded" in m): + return f"{label} is having trouble right now. Try again in a few minutes, or {pick}." + return None diff --git a/sentient/llm/provider.py b/sentient/llm/provider.py index 76c28751..a56fba1e 100644 --- a/sentient/llm/provider.py +++ b/sentient/llm/provider.py @@ -19,6 +19,9 @@ from sentient import secrets from sentient.config.schema import ProviderConfig, SentientConfig +from sentient.llm.chatgpt import ChatGPTError +from sentient.llm.errors import plain_error +from sentient.llm.responses import ResponsesError log = logging.getLogger(__name__) @@ -27,15 +30,39 @@ class ProviderError(RuntimeError): pass -class ModelRefused(ProviderError): +class PlainProviderError(ProviderError): + """The message is a plain sentence that says what to do, for the user as is (the raw error is only logged).""" + + +class ModelRefused(PlainProviderError): """A model that is set up can't do this job (Claude Code in the background, for one). The message is a plain sentence that says what to change, for the user as is.""" +class ProviderFailed(PlainProviderError): + """A provider turned the request down for a known reason: out of credits, a rate limit, a bad key... (#280).""" + + +def _failure(exc: Exception, model: str) -> Exception: + """What a failed model leaves for the user: a plain error for a known provider failure, else ``exc`` itself. + Claude Code and ChatGPT plan errors are written for the user already.""" + if isinstance(exc, PlainProviderError): + return exc + if isinstance(exc, ResponsesError | ChatGPTError) or ( + _provider_prefix(model) == CLAUDE_CODE and type(exc) is ProviderError + ): + return ProviderFailed(str(exc)) + plain = plain_error(exc, model) + return ProviderFailed(plain) if plain else exc + + def _all_failed(role: str, errors: list[Exception]) -> ProviderError: - """The error for a role whose every model failed. When every model refused the job, say why and what to do.""" + """The error for a role whose every model failed: the last model's plain reason when Sentient knows it (when + every model refused the job, why and what to do), else the general error.""" if errors and all(isinstance(e, ModelRefused) for e in errors): return ModelRefused(str(errors[0])) + if errors and isinstance(errors[-1], PlainProviderError): + return ProviderFailed(str(errors[-1])) return ProviderError(f"All models failed for role '{role}': {errors[-1] if errors else None}") @@ -332,10 +359,13 @@ async def stream( ) return except Exception as exc: - errors.append(exc) + failure = _failure(exc, model) + errors.append(failure) log.warning("model %s failed for role %s: %s", model, role, exc) if emitted: # part of a reply already reached the user; switching models would duplicate it + if isinstance(failure, PlainProviderError): + raise ProviderFailed(str(failure)) from exc raise ProviderError(f"{model} stopped mid-reply: {exc}") from exc continue raise _all_failed(role, errors) @@ -397,7 +427,7 @@ async def complete_text(self, role: str, messages: list[dict], *, model: str | N text = resp.choices[0].message.content or "" return re.sub(r".*?", "", text, flags=re.DOTALL).strip() except Exception as exc: - errors.append(exc) + errors.append(_failure(exc, model)) log.warning("model %s failed for role %s: %s", model, role, exc) raise _all_failed(role, errors) @@ -421,7 +451,7 @@ async def complete_json(self, role: str, messages: list[dict], *, model: str | N text = resp.choices[0].message.content or "" return parse_json_loose(text) except Exception as exc: - errors.append(exc) + errors.append(_failure(exc, model)) log.warning("model %s failed for role %s: %s", model, role, exc) raise _all_failed(role, errors) @@ -438,7 +468,14 @@ async def embed(self, texts: list[str], *, model: str | None = None) -> list[lis raise ProviderError("ChatGPT plans don't include embedding models. Pick a local or API embedding model.") kwargs = self._kwargs_for(model) kwargs.pop("timeout", None) - resp = await litellm.aembedding(model=litellm_model(model), input=texts, **kwargs) + try: + resp = await litellm.aembedding(model=litellm_model(model), input=texts, **kwargs) + except Exception as exc: + plain = plain_error(exc, model) + if plain is None: + raise + log.warning("embedding model %s failed: %s", model, exc) + raise ProviderFailed(plain) from exc return [d["embedding"] for d in resp.data] diff --git a/sentient/tasks/service.py b/sentient/tasks/service.py index f5f070fd..7cfe77ec 100644 --- a/sentient/tasks/service.py +++ b/sentient/tasks/service.py @@ -30,7 +30,7 @@ from datetime import UTC, datetime, timedelta from typing import Any -from sentient.llm.provider import ModelRefused, ProviderError +from sentient.llm.provider import PlainProviderError, ProviderError from sentient.services import Service, cancel_tasks from sentient.tasks import ask, catchup, executor, limits, scripts, stuck, swarm from sentient.tasks.delivery import from_stored, stored @@ -80,9 +80,10 @@ def _provider_down(exc: ProviderError, *, detail: bool = False) -> str: - """What a task shows when its model failed: the reason itself when a model refused the job (it says what to - change), else the general sentence, with the error when ``detail``.""" - if isinstance(exc, ModelRefused): + """What a task shows when its model failed: the reason itself when Sentient knows it (a model that refused the + job, a provider out of credits or over its limit: it says what to do), else the general sentence, with the error + when ``detail``.""" + if isinstance(exc, PlainProviderError): return str(exc) return f"{PROVIDER_DOWN} ({exc})" if detail else PROVIDER_DOWN diff --git a/tests/test_provider_errors.py b/tests/test_provider_errors.py new file mode 100644 index 00000000..a45b9b44 --- /dev/null +++ b/tests/test_provider_errors.py @@ -0,0 +1,235 @@ +"""Known provider failures reach the user as plain sentences that say what to do; the raw text only goes to the +log (#280). + +The error bodies below were recorded from the real providers on 2026-10-10 (user and request ids replaced). respx +serves them to the real LiteLLM, so the exceptions are the ones people get. The 5xx bodies follow the providers' +documented shapes. +""" + +from __future__ import annotations + +import json +import logging + +import httpx +import litellm +import pytest +import respx + +from sentient.app import SentientApp +from sentient.config.schema import SentientConfig +from sentient.llm.errors import plain_error +from sentient.llm.events import Done, Error +from sentient.llm.provider import LiteLLMProvider, PlainProviderError, ProviderError, ProviderFailed + +OPENROUTER = "https://openrouter.ai/api/v1/chat/completions" +ANTHROPIC = "https://api.anthropic.com/v1/messages" +OPENAI = "https://api.openai.com/v1/chat/completions" +OLLAMA = "http://localhost:11434/api/chat" +FREE = "openrouter/nvidia/nemotron-3-super-120b-a12b:free" + +FREE_PER_DAY = {"error": { + "message": "Rate limit exceeded: free-models-per-day. Add 10 credits to unlock 1000 free model requests per day", + "code": 429, "metadata": { + "headers": {"X-RateLimit-Limit": "50", "X-RateLimit-Remaining": "0", "X-RateLimit-Reset": "1791676800000"}, + "limit_source": "openrouter_free_tier_daily", + "remedy_hint": "Wait for the daily reset (see X-RateLimit-Reset), or purchase credits to raise your " + "free-model daily limit.", "provider_name": None}}, "user_id": "user_test"} +AGENTIC_ONLY = {"error": { + "message": "thinkingmachines/inkling:free is only available on agentic harnesses. Try plugging it into a coding " + "agent or productivity app listed on https://openrouter.ai/apps", + "code": 403, "metadata": {"routing_funnel": [{"step": "Initial Endpoints", "endpoint_count": 1}], + "failed_routing_step": "Gate Free Endpoints by Agentic Harness"}}} +NO_CREDITS = {"error": { + "message": "This request requires more credits, or fewer max_tokens. You requested up to 32000 tokens, but can " + "only afford 398. To increase, visit https://openrouter.ai/settings/credits and upgrade to a paid " + "account", "code": 402, "metadata": { + "limit_source": "openrouter_credits", + "remedy_hint": "Add credits at https://openrouter.ai/settings/credits, or lower max_tokens / prompt size to " + "fit your remaining balance.", "provider_name": None}}, "user_id": "user_test"} +NOT_A_MODEL = {"error": {"message": "nosuchlab/no-such-model-9 is not a valid model ID", "code": 400}, + "user_id": "user_test"} +REVOKED_KEY = {"error": {"message": "User not found.", "code": 401}} +ANTHROPIC_BAD_KEY = {"type": "error", "error": {"type": "authentication_error", "message": "invalid x-api-key"}, + "request_id": "req_test"} +OPENAI_BAD_KEY = {"error": { + "message": "Incorrect API key provided: sk-proj-************************************0000. You can find your API " + "key at https://platform.openai.com/account/api-keys.", + "type": "invalid_request_error", "param": None, "code": "invalid_api_key"}} +OLLAMA_NOT_PULLED = {"error": 'model "no-such-model:1b" not found, try pulling it first'} +OPENROUTER_502 = {"error": {"message": "Provider returned error", "code": 502, + "metadata": {"raw": "upstream connect error", "provider_name": "Chutes"}}} +OPENROUTER_503 = {"error": {"message": "No instances available", "code": 503}} +ANTHROPIC_OVERLOADED = {"type": "error", "error": {"type": "overloaded_error", "message": "Overloaded"}} +NO_TOOL_ENDPOINTS = {"error": {"message": "No endpoints found that support tool use. To learn more about provider " + "routing, visit: https://openrouter.ai/docs/provider-routing", "code": 404}} + +PICK = "pick another model in Settings > Models." +CASES = [ + ("free_per_day", FREE, OPENROUTER, 429, FREE_PER_DAY, + "OpenRouter's free models have reached today's limit. Add credits on openrouter.ai, wait until tomorrow, or " + + PICK), + ("agentic_only", "openrouter/thinkingmachines/inkling:free", OPENROUTER, 403, AGENTIC_ONLY, + "OpenRouter only lets coding tools use thinkingmachines/inkling:free, not assistants like Sentient. Please " + + PICK), + ("no_credits", "openrouter/anthropic/claude-opus-4.1", OPENROUTER, 402, NO_CREDITS, + "Your OpenRouter account is out of credits. Add credits on openrouter.ai/settings/credits, or " + PICK), + ("not_a_model", "openrouter/nosuchlab/no-such-model-9", OPENROUTER, 400, NOT_A_MODEL, + "OpenRouter doesn't have a model called nosuchlab/no-such-model-9. Check the name, or " + PICK), + ("revoked_key", "openrouter/openai/gpt-4o-mini", OPENROUTER, 401, REVOKED_KEY, + "OpenRouter didn't accept your key. It may be mistyped or revoked: add a new one in Settings > Models."), + ("anthropic_bad_key", "anthropic/claude-haiku-4-5", ANTHROPIC, 401, ANTHROPIC_BAD_KEY, + "Anthropic didn't accept your key. It may be mistyped or revoked: add a new one in Settings > Models."), + ("openai_bad_key", "openai/gpt-4o-mini", OPENAI, 401, OPENAI_BAD_KEY, + "OpenAI didn't accept your key. It may be mistyped or revoked: add a new one in Settings > Models."), + ("ollama_not_pulled", "ollama_chat/no-such-model:1b", OLLAMA, 404, OLLAMA_NOT_PULLED, + "no-such-model:1b isn't downloaded in Ollama. Download it in Settings > Models, or pick another model there."), + ("openrouter_502", "openrouter/some/model", OPENROUTER, 502, OPENROUTER_502, + "OpenRouter is having trouble right now. Try again in a few minutes, or " + PICK), + ("openrouter_503", "openrouter/some/model", OPENROUTER, 503, OPENROUTER_503, + "OpenRouter is having trouble right now. Try again in a few minutes, or " + PICK), + ("anthropic_overloaded", "anthropic/claude-haiku-4-5", ANTHROPIC, 529, ANTHROPIC_OVERLOADED, + "Anthropic is having trouble right now. Try again in a few minutes, or " + PICK), +] +RAW = ("litellm", "Exception", "{", "user_test") # what a user must never see + + +@pytest.fixture +def http(monkeypatch): + """LiteLLM over plain httpx (respx can't see its aiohttp transport), with a stored key for every provider.""" + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr("sentient.llm.provider.secrets.get_secret", lambda *a, **k: "test-key") + litellm.in_memory_llm_clients_cache.flush_cache() + with respx.mock(assert_all_called=False) as router: + router.post("http://localhost:11434/api/show").mock(return_value=httpx.Response(404, json={})) + yield router + litellm.in_memory_llm_clients_cache.flush_cache() + + +def provider(model: str) -> LiteLLMProvider: + config = SentientConfig() + for role in ("primary", "fast", "planner", "executor"): + setattr(config.models.roles, role, model) + config.models.fallbacks = {} + return LiteLLMProvider(config) + + +def assert_plain(message: str, expected: str) -> None: + assert message == expected + assert not any(raw in message for raw in RAW) + + +@pytest.mark.parametrize(("case", "model", "url", "status", "body", "expected"), CASES, ids=[c[0] for c in CASES]) +async def test_known_failures_become_plain_sentences(http, caplog, case, model, url, status, body, expected): + http.post(url).mock(return_value=httpx.Response(status, json=body)) + llm = provider(model) + with caplog.at_level(logging.WARNING, logger="sentient.llm.provider"), pytest.raises(ProviderFailed) as streamed: + [c async for c in llm.stream("primary", [{"role": "user", "content": "hi"}])] + assert_plain(str(streamed.value), expected) + raw = body["error"]["message"] if isinstance(body["error"], dict) else body["error"] + assert raw.split(".")[0][:40] in caplog.text.replace('\\"', '"') # the raw reply stays in the log + + with pytest.raises(ProviderFailed) as planned: # background jobs (task planning, memory) say the same + await llm.complete_json("planner", [{"role": "user", "content": "plan"}]) + assert_plain(str(planned.value), expected) + + +async def test_unreachable_and_slow_providers(http): + http.post(OLLAMA).mock(side_effect=httpx.ConnectError("All connection attempts failed")) + http.post(OPENROUTER).mock(side_effect=httpx.ReadTimeout("timed out")) + with pytest.raises(ProviderFailed) as down: + await provider("ollama_chat/qwen3:8b").complete_text("primary", [{"role": "user", "content": "hi"}]) + assert_plain(str(down.value), "Sentient couldn't reach Ollama. Make sure it's running, then try again.") + with pytest.raises(ProviderFailed) as slow: + await provider(FREE).complete_text("primary", [{"role": "user", "content": "hi"}]) + assert_plain(str(slow.value), "OpenRouter took too long to answer. Try again in a moment, or " + PICK) + + # what LiteLLM's default aiohttp transport says when Ollama isn't running (recorded) + refused = litellm.APIConnectionError( + message="Ollama_chatException - Cannot connect to host 127.0.0.1:11434 ssl:default " + "[The remote computer refused the network connection]", llm_provider="ollama_chat", model="qwen3:8b") + assert plain_error(refused, "ollama_chat/qwen3:8b") == str(down.value) + offline = litellm.APIConnectionError(message="OpenrouterException - Cannot connect to host openrouter.ai:443", + llm_provider="openrouter", model="x") + assert plain_error(offline, FREE) == ( + "Sentient couldn't reach OpenRouter. Check your internet connection and try again.") + + +async def test_unknown_failures_and_tool_refusals_keep_their_own_text(http): + from sentient.agent.loop import is_tool_format_error + + http.post(OPENROUTER).mock(return_value=httpx.Response(404, json=NO_TOOL_ENDPOINTS)) + with pytest.raises(ProviderError) as no_tools: + [c async for c in provider(FREE).stream("primary", [{"role": "user", "content": "hi"}], tools=[{ + "type": "function", "function": {"name": "t", "parameters": {"type": "object", "properties": {}}}}])] + assert not isinstance(no_tools.value, PlainProviderError) + assert is_tool_format_error(no_tools.value) # chat answers without tools instead of failing + + odd = litellm.BadRequestError(message="OpenrouterException - something new", model="x", llm_provider="openrouter") + assert plain_error(odd, FREE) is None + assert plain_error(ValueError("Model did not return JSON: 'x'"), FREE) is None # Sentient's own errors + assert not is_tool_format_error(ProviderFailed("Tools are parsed... by nobody")) + + +async def test_a_backup_model_reports_its_own_reason(http): + http.post(OPENROUTER).mock(return_value=httpx.Response(429, json=FREE_PER_DAY)) + http.post(OLLAMA).mock(side_effect=httpx.ConnectError("All connection attempts failed")) + llm = provider(FREE) + llm.config.models.fallbacks = {"primary": ["ollama_chat/qwen3:8b"]} + with pytest.raises(ProviderFailed, match="couldn't reach Ollama"): + [c async for c in llm.stream("primary", [{"role": "user", "content": "hi"}])] + + +async def test_a_reply_cut_off_midway_says_why(monkeypatch): + async def acompletion(**kwargs): + async def chunks(): + from types import SimpleNamespace + + yield SimpleNamespace(choices=[SimpleNamespace(delta=SimpleNamespace(content="Hel", reasoning_content=None))]) + raise litellm.ServiceUnavailableError(message="OpenrouterException - " + json.dumps(OPENROUTER_503), + llm_provider="openrouter", model="some/model") + + return chunks() + + monkeypatch.setattr(litellm, "acompletion", acompletion) + monkeypatch.setattr("sentient.llm.provider.secrets.get_secret", lambda *a, **k: None) + got = [] + with pytest.raises(ProviderFailed) as err: + async for c in provider("openrouter/some/model").stream("primary", [{"role": "user", "content": "hi"}]): + got.append(c.text) + assert got == ["Hel"] + assert_plain(str(err.value), "OpenRouter is having trouble right now. Try again in a few minutes, or " + PICK) + + +async def test_embeddings_say_it_plainly_too(http): + http.post("http://localhost:11434/api/embed").mock(return_value=httpx.Response(404, json={ + "error": 'model "nomic-embed-text" not found, try pulling it first'})) + with pytest.raises(ProviderFailed, match="nomic-embed-text isn't downloaded in Ollama"): + await provider(FREE).embed(["hello"], model="ollama/nomic-embed-text") + + +async def test_chat_and_tasks_show_the_plain_sentence(http, config, isolated_home): + """End to end in the engine: the chat's error event and a task's error, with the real provider class.""" + http.post(OPENROUTER).mock(return_value=httpx.Response(429, json=FREE_PER_DAY)) + for role in ("primary", "fast", "planner", "executor"): + setattr(config.models.roles, role, FREE) + config.models.fallbacks = {} + expected = ("OpenRouter's free models have reached today's limit. Add credits on openrouter.ai, wait until " + "tomorrow, or " + PICK) + app = SentientApp(config, llm=LiteLLMProvider(config), db_path=isolated_home / "e.db", enable_background=False) + await app.start() + try: + sid = await app.store.create_session(channel="cli") + events = [ev async for ev in app.agent.run_turn(sid, "hi", channel="cli")] + errors = [e for e in events if isinstance(e, Error)] + assert len(errors) == 1 and not errors[0].recoverable + assert_plain(errors[0].message, expected) + assert not any(isinstance(e, Done) and "litellm" in (e.content or "") for e in events) + + task = await app.tasks.create_task("Look up tomorrow's weather in Pune") + await app.tasks.drain() + task = await app.tasks.get(task["task_id"]) + assert task["status"] == "error" + assert_plain(task["error"], expected) + finally: + await app.stop() From 7005f1315f9adc163930c05d758669346a17a29d Mon Sep 17 00:00:00 2001 From: itsskofficial Date: Sat, 10 Oct 2026 21:53:30 +0530 Subject: [PATCH 2/2] fix(llm): keep plan and Claude Code tool refusals for the answer-without-tools fallback --- sentient/llm/provider.py | 6 +++--- tests/test_provider_errors.py | 7 +++++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/sentient/llm/provider.py b/sentient/llm/provider.py index a56fba1e..b54936f7 100644 --- a/sentient/llm/provider.py +++ b/sentient/llm/provider.py @@ -20,7 +20,7 @@ from sentient import secrets from sentient.config.schema import ProviderConfig, SentientConfig from sentient.llm.chatgpt import ChatGPTError -from sentient.llm.errors import plain_error +from sentient.llm.errors import is_tool_support_error, plain_error from sentient.llm.responses import ResponsesError log = logging.getLogger(__name__) @@ -48,9 +48,9 @@ def _failure(exc: Exception, model: str) -> Exception: Claude Code and ChatGPT plan errors are written for the user already.""" if isinstance(exc, PlainProviderError): return exc - if isinstance(exc, ResponsesError | ChatGPTError) or ( + if (isinstance(exc, ResponsesError | ChatGPTError) or ( _provider_prefix(model) == CLAUDE_CODE and type(exc) is ProviderError - ): + )) and not is_tool_support_error(str(exc)): # a tool refusal stays as is: chat answers without tools return ProviderFailed(str(exc)) plain = plain_error(exc, model) return ProviderFailed(plain) if plain else exc diff --git a/tests/test_provider_errors.py b/tests/test_provider_errors.py index a45b9b44..db9d787e 100644 --- a/tests/test_provider_errors.py +++ b/tests/test_provider_errors.py @@ -165,6 +165,13 @@ async def test_unknown_failures_and_tool_refusals_keep_their_own_text(http): assert not isinstance(no_tools.value, PlainProviderError) assert is_tool_format_error(no_tools.value) # chat answers without tools instead of failing + from sentient.llm.provider import _failure + from sentient.llm.responses import ResponsesError + + no_tools_plan = ResponsesError("This model does not support tools.", 400) + assert _failure(no_tools_plan, "chatgpt/gpt-test") is no_tools_plan # left for the loop to answer without tools + assert isinstance(_failure(ResponsesError("Usage limit reached.", 429), "chatgpt/gpt-test"), ProviderFailed) + odd = litellm.BadRequestError(message="OpenrouterException - something new", model="x", llm_provider="openrouter") assert plain_error(odd, FREE) is None assert plain_error(ValueError("Model did not return JSON: 'x'"), FREE) is None # Sentient's own errors