diff --git a/CHANGELOG.md b/CHANGELOG.md index ef8bea6b..5f1fab0a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -189,6 +189,11 @@ and versions follow [Semantic Versioning](https://semver.org/). - A task that repeats the same failing step three times now pauses as stuck and asks what to do, instead of failing. ### Fixed +- When a cloud model says no, chat and tasks now say why in a plain sentence and what to do, instead of showing the + provider's raw reply. For example: "OpenRouter's free models have reached today's limit. Add credits on + openrouter.ai, wait until tomorrow, or pick another model in Settings > Models." This covers rate limits, an + account out of credits, a key that was mistyped or revoked, a free model OpenRouter keeps for coding tools, a model + that doesn't exist or isn't downloaded, and a provider that is down or too slow. The raw reply stays in the log. - One long tool result no longer fills a local model's whole context. A result may now take about a quarter of what the model reads at once (Settings > General > Conversations), so on an 8,192-token model a 13,000-character Composio search no longer triggers "This chat is getting long" after a single step. The model is told plainly when a result diff --git a/docs/API.md b/docs/API.md index 9a9b68d7..bcf21d1f 100644 --- a/docs/API.md +++ b/docs/API.md @@ -43,6 +43,14 @@ All carry `session_id` and `turn_id`. | `done` | `content` (final text), `message_id`, `cancelled?`, `memory_sources: [MemorySource]` (what this reply had in mind, section 2; `[]` when none), `dropped?: string[]` (section 17: messages queued behind a stopped reply, never sent). A stopped reply's `done` (`cancelled: true`) carries the kept message's `message_id` and its `memory_sources` too (none when it was stopped before it started) | | `approval.ack` | `approval_id`, `resolved` | +**Model errors** (#280). When a provider turns a request down for a known reason, `error.message` (and a task's +`error`, section 4) is one plain sentence that says what to do, e.g. `OpenRouter's free models have reached today's +limit. Add credits on openrouter.ai, wait until tomorrow, or pick another model in Settings > Models.` This covers rate +limits (OpenRouter's free models per day and per minute), no credits (402), a key the provider rejects (401), a model +the provider keeps for coding tools, a model it doesn't have (an Ollama model that isn't downloaded), and a provider +that is down, slow or unreachable. The provider's raw reply only goes to the engine log; other failures keep their +text. + **Context meter** (#131). After every model call `usage` says how full the model's context is: `context_used` is the prompt (the larger of what the provider reported and a local count with LiteLLM's bundled tokenizer, because Ollama reports only the part of a prompt it had not cached) plus the reply, `context_length` what the model reads at once in diff --git a/sentient/agent/loop.py b/sentient/agent/loop.py index 95ed3242..263f8fbc 100644 --- a/sentient/agent/loop.py +++ b/sentient/agent/loop.py @@ -46,6 +46,7 @@ from sentient.config.schema import SentientConfig from sentient.files.extract import extract_text, is_image from sentient.llm import claude_code +from sentient.llm.errors import is_tool_support_error from sentient.llm.events import ( AgentEvent, ApprovalRequest, @@ -62,7 +63,13 @@ ) from sentient.llm.jobs import as_kind, detached from sentient.llm.meter import measure -from sentient.llm.provider import LLMProvider, ProviderError, StreamChunk, ToolCall +from sentient.llm.provider import ( + LLMProvider, + PlainProviderError, + ProviderError, + StreamChunk, + ToolCall, +) from sentient.memory import review as memory_review from sentient.memory.facts import FactMemory from sentient.memory.sources import MemorySources @@ -219,8 +226,7 @@ def history_to_openai(rows: list[dict]) -> list[dict]: def is_tool_format_error(exc: Exception) -> bool: """True when the provider rejected the request because the model cannot do tool calling (e.g. an old Ollama model template emitting malformed tool calls).""" - m = str(exc).lower() - return "does not support tools" in m or "invalid character" in m or ("tool" in m and "pars" in m) + return not isinstance(exc, PlainProviderError) and is_tool_support_error(str(exc)) def _json_safe(value: Any) -> str: diff --git a/sentient/gateway/routes/tasks.py b/sentient/gateway/routes/tasks.py index 0a2eeb7b..8811124c 100644 --- a/sentient/gateway/routes/tasks.py +++ b/sentient/gateway/routes/tasks.py @@ -9,7 +9,7 @@ from pydantic import BaseModel, Field from sentient.gateway.deps import AUTH, get_core -from sentient.llm.provider import ModelRefused, ProviderError +from sentient.llm.provider import PlainProviderError, ProviderError from sentient.tasks.service import TaskConflict, TaskNotFound router = APIRouter(prefix="/api/tasks", tags=["tasks"], dependencies=AUTH) @@ -47,7 +47,7 @@ async def _guard[T](awaitable: Awaitable[T]) -> T: raise HTTPException(status_code=404, detail="Task not found") from exc except TaskConflict as exc: raise HTTPException(status_code=409, detail=str(exc)) from exc - except ModelRefused as exc: # a model that is set up can't do this job: say why and what to change + except PlainProviderError as exc: # a refused job, no credits, a bad key...: say why and what to change raise HTTPException(status_code=503, detail=str(exc)) from exc except ProviderError as exc: raise HTTPException(status_code=503, detail=f"The AI model is unavailable: {exc}") from exc diff --git a/sentient/llm/checkup.py b/sentient/llm/checkup.py index a9b77d7c..dafca633 100644 --- a/sentient/llm/checkup.py +++ b/sentient/llm/checkup.py @@ -21,7 +21,7 @@ from sentient import secrets from sentient.config.schema import ModelRoles, SentientConfig from sentient.llm import chatgpt, claude_code -from sentient.llm.provider import CLAUDE_CODE, ToolCall, provider_config +from sentient.llm.provider import CLAUDE_CODE, PlainProviderError, ToolCall, provider_config ROLES = ("primary", "fast", "planner", "executor", "vision", "voice", "embedding") TOOL_ROLES = {"primary", "fast", "executor", "vision", "voice"} # roles that run the agent loop with tools @@ -178,6 +178,8 @@ def _failure_fix(self, exc: BaseException) -> str: if isinstance(exc, TimeoutError): return (f"No reply within {int(self.run.timeout_s)} seconds. The model may be too big for this " "computer: try a smaller model, or a shorter context length.") + if isinstance(exc, PlainProviderError): + return "" # the reason already says what to do msg = str(exc).lower() if any(k in msg for k in ("401", "403", "api key", "api_key", "authentication", "unauthorized")): return f"{self.label} turned the key down. Check your {self.label} key under Providers." diff --git a/sentient/llm/errors.py b/sentient/llm/errors.py new file mode 100644 index 00000000..dc75f082 --- /dev/null +++ b/sentient/llm/errors.py @@ -0,0 +1,107 @@ +"""Plain sentences for the ways a model provider says no (issue #280). + +LiteLLM's errors carry the provider's raw reply (``litellm.RateLimitError: ... OpenrouterException - {"error": ...``). +``plain_error`` turns the failures people actually hit into one sentence that says what to do. The raw text only +goes to the log; anything unknown returns None and keeps its own message. +""" + +from __future__ import annotations + +SETTINGS = "Settings > Models" + +LABELS = { + "openrouter": "OpenRouter", "anthropic": "Anthropic", "openai": "OpenAI", "gemini": "Google Gemini", + "vertex_ai": "Google Vertex AI", "groq": "Groq", "mistral": "Mistral", "deepseek": "DeepSeek", "xai": "xAI", + "together_ai": "Together AI", "fireworks_ai": "Fireworks AI", "cohere": "Cohere", "perplexity": "Perplexity", + "nous": "Nous Portal", "azure": "Azure OpenAI", "bedrock": "Amazon Bedrock", "ollama": "Ollama", + "ollama_chat": "Ollama", "lm_studio": "LM Studio", "chatgpt": "ChatGPT", +} +# where each provider sells credits, for "out of credits" +BILLING = { + "openrouter": "openrouter.ai/settings/credits", "anthropic": "console.anthropic.com", + "openai": "platform.openai.com", "deepseek": "platform.deepseek.com", "xai": "console.x.ai", + "mistral": "console.mistral.ai", "nous": "portal.nousresearch.com", +} +LOCAL = {"ollama", "ollama_chat", "lm_studio", "llamafile", "vllm", "hosted_vllm"} + +CREDIT_HINTS = ( + "requires more credits", "insufficient credits", "insufficient_quota", "exceeded your current quota", + "credit balance is too low", "insufficient balance", "out of credits", "payment required", +) +KEY_HINTS = ("invalid api key", "invalid x-api-key", "incorrect api key", "api key not valid", "invalid_api_key", + "user not found", "no auth credentials") +NOT_FOUND_HINTS = ("not a valid model id", "model_not_found", "does not exist", "not found, try pulling it first", + "unknown model", "no such model") +UNREACHABLE_HINTS = ("cannot connect to host", "all connection attempts failed", "connection refused", + "refused the network connection", "connecterror", "getaddrinfo failed", "name or service not known", "nodename nor servname") + + +def _parts(model: str) -> tuple[str, str]: + prefix, _, name = model.partition("/") + return (prefix, name) if name else ("", model) + + +def _label(prefix: str) -> str: + return LABELS.get(prefix) or (prefix.replace("_", " ").title() if prefix else "The model's provider") + + +def _status(exc: BaseException) -> int | None: + code = getattr(exc, "status_code", None) + return code if isinstance(code, int) else None + + +def is_tool_support_error(text: str) -> bool: + """The provider turned the request down because the model can't call tools (the agent loop then answers + without tools, so these keep their own message).""" + m = text.lower() + return ("does not support tools" in m or "support tool use" in m or "invalid character" in m + or ("tool" in m and "pars" in m)) + + +def plain_error(exc: BaseException, model: str) -> str | None: + """A plain sentence for a known provider failure of ``model`` that says what to do, or None when unknown. + + Only LiteLLM's own errors are translated: Sentient's errors (Claude Code, the ChatGPT plan) are already plain. + """ + if not type(exc).__module__.startswith("litellm"): + return None + text = str(exc) + m = text.lower() + if is_tool_support_error(m): + return None + prefix, name = _parts(model) + label, status, kind = _label(prefix), _status(exc), type(exc).__name__ + pick = f"pick another model in {SETTINGS}" + + if "agentic harness" in m: + return (f"{label} only lets coding tools use {name}, not assistants like Sentient. " + f"Please {pick}.") + if "free-models-per-day" in m: + return (f"{label}'s free models have reached today's limit. Add credits on openrouter.ai, wait until " + f"tomorrow, or {pick}.") + if "free-models-per-min" in m: + return f"{label}'s free models allow only a few requests a minute. Wait a minute and try again, or {pick}." + if status == 402 or any(h in m for h in CREDIT_HINTS): + where = f" on {BILLING[prefix]}" if prefix in BILLING else "" + return f"Your {label} account is out of credits. Add credits{where}, or {pick}." + if "no endpoints found matching your data policy" in m: + return (f"Your {label} privacy settings don't allow any provider of {name}. Change them on " + f"openrouter.ai/settings/privacy, or {pick}.") + if status == 401 or kind == "AuthenticationError" or any(h in m for h in KEY_HINTS): + return f"{label} didn't accept your key. It may be mistyped or revoked: add a new one in {SETTINGS}." + if status == 404 or kind == "NotFoundError" or any(h in m for h in NOT_FOUND_HINTS): + if prefix in {"ollama", "ollama_chat"}: + return f"{name} isn't downloaded in Ollama. Download it in {SETTINGS}, or pick another model there." + return f"{label} doesn't have a model called {name}. Check the name, or {pick}." + if status == 429 or kind == "RateLimitError": + return f"{label} is getting too many requests right now. Wait a minute and try again, or {pick}." + if status == 408 or kind == "Timeout": + return f"{label} took too long to answer. Try again in a moment, or {pick}." + if any(h in m for h in UNREACHABLE_HINTS): + if prefix in LOCAL: + return f"Sentient couldn't reach {label}. Make sure it's running, then try again." + return f"Sentient couldn't reach {label}. Check your internet connection and try again." + if (kind in {"InternalServerError", "ServiceUnavailableError", "BadGatewayError"} + or (kind != "APIConnectionError" and status is not None and status >= 500) or "overloaded" in m): + return f"{label} is having trouble right now. Try again in a few minutes, or {pick}." + return None diff --git a/sentient/llm/provider.py b/sentient/llm/provider.py index 1fac2c2d..bf6b4f28 100644 --- a/sentient/llm/provider.py +++ b/sentient/llm/provider.py @@ -20,7 +20,10 @@ from sentient import secrets from sentient.config.schema import ProviderConfig, SentientConfig +from sentient.llm.chatgpt import ChatGPTError +from sentient.llm.errors import is_tool_support_error, plain_error from sentient.llm.jobs import ModelJobs +from sentient.llm.responses import ResponsesError log = logging.getLogger(__name__) @@ -29,15 +32,39 @@ class ProviderError(RuntimeError): pass -class ModelRefused(ProviderError): +class PlainProviderError(ProviderError): + """The message is a plain sentence that says what to do, for the user as is (the raw error is only logged).""" + + +class ModelRefused(PlainProviderError): """A model that is set up can't do this job (Claude Code in the background, for one). The message is a plain sentence that says what to change, for the user as is.""" +class ProviderFailed(PlainProviderError): + """A provider turned the request down for a known reason: out of credits, a rate limit, a bad key... (#280).""" + + +def _failure(exc: Exception, model: str) -> Exception: + """What a failed model leaves for the user: a plain error for a known provider failure, else ``exc`` itself. + Claude Code and ChatGPT plan errors are written for the user already.""" + if isinstance(exc, PlainProviderError): + return exc + if (isinstance(exc, ResponsesError | ChatGPTError) or ( + _provider_prefix(model) == CLAUDE_CODE and type(exc) is ProviderError + )) and not is_tool_support_error(str(exc)): # a tool refusal stays as is: chat answers without tools + return ProviderFailed(str(exc)) + plain = plain_error(exc, model) + return ProviderFailed(plain) if plain else exc + + def _all_failed(role: str, errors: list[Exception]) -> ProviderError: - """The error for a role whose every model failed. When every model refused the job, say why and what to do.""" + """The error for a role whose every model failed: the last model's plain reason when Sentient knows it (when + every model refused the job, why and what to do), else the general error.""" if errors and all(isinstance(e, ModelRefused) for e in errors): return ModelRefused(str(errors[0])) + if errors and isinstance(errors[-1], PlainProviderError): + return ProviderFailed(str(errors[-1])) return ProviderError(f"All models failed for role '{role}': {errors[-1] if errors else None}") @@ -348,10 +375,13 @@ async def stream( ) return except Exception as exc: - errors.append(exc) + failure = _failure(exc, model) + errors.append(failure) log.warning("model %s failed for role %s: %s", model, role, exc) if emitted: # part of a reply already reached the user; switching models would duplicate it + if isinstance(failure, PlainProviderError): + raise ProviderFailed(str(failure)) from exc raise ProviderError(f"{model} stopped mid-reply: {exc}") from exc continue raise _all_failed(role, errors) @@ -414,7 +444,7 @@ async def complete_text(self, role: str, messages: list[dict], *, model: str | N text = resp.choices[0].message.content or "" return re.sub(r".*?", "", text, flags=re.DOTALL).strip() except Exception as exc: - errors.append(exc) + errors.append(_failure(exc, model)) log.warning("model %s failed for role %s: %s", model, role, exc) raise _all_failed(role, errors) @@ -439,7 +469,7 @@ async def complete_json(self, role: str, messages: list[dict], *, model: str | N text = resp.choices[0].message.content or "" return parse_json_loose(text) except Exception as exc: - errors.append(exc) + errors.append(_failure(exc, model)) log.warning("model %s failed for role %s: %s", model, role, exc) raise _all_failed(role, errors) @@ -456,8 +486,15 @@ async def embed(self, texts: list[str], *, model: str | None = None) -> list[lis raise ProviderError("ChatGPT plans don't include embedding models. Pick a local or API embedding model.") kwargs = self._kwargs_for(model) kwargs.pop("timeout", None) - async with self.jobs.slot(model): - resp = await litellm.aembedding(model=litellm_model(model), input=texts, **kwargs) + try: + async with self.jobs.slot(model): + resp = await litellm.aembedding(model=litellm_model(model), input=texts, **kwargs) + except Exception as exc: + plain = plain_error(exc, model) + if plain is None: + raise + log.warning("embedding model %s failed: %s", model, exc) + raise ProviderFailed(plain) from exc return [d["embedding"] for d in resp.data] diff --git a/sentient/tasks/service.py b/sentient/tasks/service.py index db04f496..a19ae5b4 100644 --- a/sentient/tasks/service.py +++ b/sentient/tasks/service.py @@ -31,7 +31,7 @@ from typing import Any from sentient.llm.jobs import detached -from sentient.llm.provider import ModelRefused, ProviderError +from sentient.llm.provider import PlainProviderError, ProviderError from sentient.services import Service, cancel_tasks from sentient.tasks import ask, catchup, executor, limits, scripts, stuck, swarm from sentient.tasks.delivery import from_stored, stored @@ -81,9 +81,10 @@ def _provider_down(exc: ProviderError, *, detail: bool = False) -> str: - """What a task shows when its model failed: the reason itself when a model refused the job (it says what to - change), else the general sentence, with the error when ``detail``.""" - if isinstance(exc, ModelRefused): + """What a task shows when its model failed: the reason itself when Sentient knows it (a model that refused the + job, a provider out of credits or over its limit: it says what to do), else the general sentence, with the error + when ``detail``.""" + if isinstance(exc, PlainProviderError): return str(exc) return f"{PROVIDER_DOWN} ({exc})" if detail else PROVIDER_DOWN diff --git a/tests/test_provider_errors.py b/tests/test_provider_errors.py new file mode 100644 index 00000000..db9d787e --- /dev/null +++ b/tests/test_provider_errors.py @@ -0,0 +1,242 @@ +"""Known provider failures reach the user as plain sentences that say what to do; the raw text only goes to the +log (#280). + +The error bodies below were recorded from the real providers on 2026-10-10 (user and request ids replaced). respx +serves them to the real LiteLLM, so the exceptions are the ones people get. The 5xx bodies follow the providers' +documented shapes. +""" + +from __future__ import annotations + +import json +import logging + +import httpx +import litellm +import pytest +import respx + +from sentient.app import SentientApp +from sentient.config.schema import SentientConfig +from sentient.llm.errors import plain_error +from sentient.llm.events import Done, Error +from sentient.llm.provider import LiteLLMProvider, PlainProviderError, ProviderError, ProviderFailed + +OPENROUTER = "https://openrouter.ai/api/v1/chat/completions" +ANTHROPIC = "https://api.anthropic.com/v1/messages" +OPENAI = "https://api.openai.com/v1/chat/completions" +OLLAMA = "http://localhost:11434/api/chat" +FREE = "openrouter/nvidia/nemotron-3-super-120b-a12b:free" + +FREE_PER_DAY = {"error": { + "message": "Rate limit exceeded: free-models-per-day. Add 10 credits to unlock 1000 free model requests per day", + "code": 429, "metadata": { + "headers": {"X-RateLimit-Limit": "50", "X-RateLimit-Remaining": "0", "X-RateLimit-Reset": "1791676800000"}, + "limit_source": "openrouter_free_tier_daily", + "remedy_hint": "Wait for the daily reset (see X-RateLimit-Reset), or purchase credits to raise your " + "free-model daily limit.", "provider_name": None}}, "user_id": "user_test"} +AGENTIC_ONLY = {"error": { + "message": "thinkingmachines/inkling:free is only available on agentic harnesses. Try plugging it into a coding " + "agent or productivity app listed on https://openrouter.ai/apps", + "code": 403, "metadata": {"routing_funnel": [{"step": "Initial Endpoints", "endpoint_count": 1}], + "failed_routing_step": "Gate Free Endpoints by Agentic Harness"}}} +NO_CREDITS = {"error": { + "message": "This request requires more credits, or fewer max_tokens. You requested up to 32000 tokens, but can " + "only afford 398. To increase, visit https://openrouter.ai/settings/credits and upgrade to a paid " + "account", "code": 402, "metadata": { + "limit_source": "openrouter_credits", + "remedy_hint": "Add credits at https://openrouter.ai/settings/credits, or lower max_tokens / prompt size to " + "fit your remaining balance.", "provider_name": None}}, "user_id": "user_test"} +NOT_A_MODEL = {"error": {"message": "nosuchlab/no-such-model-9 is not a valid model ID", "code": 400}, + "user_id": "user_test"} +REVOKED_KEY = {"error": {"message": "User not found.", "code": 401}} +ANTHROPIC_BAD_KEY = {"type": "error", "error": {"type": "authentication_error", "message": "invalid x-api-key"}, + "request_id": "req_test"} +OPENAI_BAD_KEY = {"error": { + "message": "Incorrect API key provided: sk-proj-************************************0000. You can find your API " + "key at https://platform.openai.com/account/api-keys.", + "type": "invalid_request_error", "param": None, "code": "invalid_api_key"}} +OLLAMA_NOT_PULLED = {"error": 'model "no-such-model:1b" not found, try pulling it first'} +OPENROUTER_502 = {"error": {"message": "Provider returned error", "code": 502, + "metadata": {"raw": "upstream connect error", "provider_name": "Chutes"}}} +OPENROUTER_503 = {"error": {"message": "No instances available", "code": 503}} +ANTHROPIC_OVERLOADED = {"type": "error", "error": {"type": "overloaded_error", "message": "Overloaded"}} +NO_TOOL_ENDPOINTS = {"error": {"message": "No endpoints found that support tool use. To learn more about provider " + "routing, visit: https://openrouter.ai/docs/provider-routing", "code": 404}} + +PICK = "pick another model in Settings > Models." +CASES = [ + ("free_per_day", FREE, OPENROUTER, 429, FREE_PER_DAY, + "OpenRouter's free models have reached today's limit. Add credits on openrouter.ai, wait until tomorrow, or " + + PICK), + ("agentic_only", "openrouter/thinkingmachines/inkling:free", OPENROUTER, 403, AGENTIC_ONLY, + "OpenRouter only lets coding tools use thinkingmachines/inkling:free, not assistants like Sentient. Please " + + PICK), + ("no_credits", "openrouter/anthropic/claude-opus-4.1", OPENROUTER, 402, NO_CREDITS, + "Your OpenRouter account is out of credits. Add credits on openrouter.ai/settings/credits, or " + PICK), + ("not_a_model", "openrouter/nosuchlab/no-such-model-9", OPENROUTER, 400, NOT_A_MODEL, + "OpenRouter doesn't have a model called nosuchlab/no-such-model-9. Check the name, or " + PICK), + ("revoked_key", "openrouter/openai/gpt-4o-mini", OPENROUTER, 401, REVOKED_KEY, + "OpenRouter didn't accept your key. It may be mistyped or revoked: add a new one in Settings > Models."), + ("anthropic_bad_key", "anthropic/claude-haiku-4-5", ANTHROPIC, 401, ANTHROPIC_BAD_KEY, + "Anthropic didn't accept your key. It may be mistyped or revoked: add a new one in Settings > Models."), + ("openai_bad_key", "openai/gpt-4o-mini", OPENAI, 401, OPENAI_BAD_KEY, + "OpenAI didn't accept your key. It may be mistyped or revoked: add a new one in Settings > Models."), + ("ollama_not_pulled", "ollama_chat/no-such-model:1b", OLLAMA, 404, OLLAMA_NOT_PULLED, + "no-such-model:1b isn't downloaded in Ollama. Download it in Settings > Models, or pick another model there."), + ("openrouter_502", "openrouter/some/model", OPENROUTER, 502, OPENROUTER_502, + "OpenRouter is having trouble right now. Try again in a few minutes, or " + PICK), + ("openrouter_503", "openrouter/some/model", OPENROUTER, 503, OPENROUTER_503, + "OpenRouter is having trouble right now. Try again in a few minutes, or " + PICK), + ("anthropic_overloaded", "anthropic/claude-haiku-4-5", ANTHROPIC, 529, ANTHROPIC_OVERLOADED, + "Anthropic is having trouble right now. Try again in a few minutes, or " + PICK), +] +RAW = ("litellm", "Exception", "{", "user_test") # what a user must never see + + +@pytest.fixture +def http(monkeypatch): + """LiteLLM over plain httpx (respx can't see its aiohttp transport), with a stored key for every provider.""" + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr("sentient.llm.provider.secrets.get_secret", lambda *a, **k: "test-key") + litellm.in_memory_llm_clients_cache.flush_cache() + with respx.mock(assert_all_called=False) as router: + router.post("http://localhost:11434/api/show").mock(return_value=httpx.Response(404, json={})) + yield router + litellm.in_memory_llm_clients_cache.flush_cache() + + +def provider(model: str) -> LiteLLMProvider: + config = SentientConfig() + for role in ("primary", "fast", "planner", "executor"): + setattr(config.models.roles, role, model) + config.models.fallbacks = {} + return LiteLLMProvider(config) + + +def assert_plain(message: str, expected: str) -> None: + assert message == expected + assert not any(raw in message for raw in RAW) + + +@pytest.mark.parametrize(("case", "model", "url", "status", "body", "expected"), CASES, ids=[c[0] for c in CASES]) +async def test_known_failures_become_plain_sentences(http, caplog, case, model, url, status, body, expected): + http.post(url).mock(return_value=httpx.Response(status, json=body)) + llm = provider(model) + with caplog.at_level(logging.WARNING, logger="sentient.llm.provider"), pytest.raises(ProviderFailed) as streamed: + [c async for c in llm.stream("primary", [{"role": "user", "content": "hi"}])] + assert_plain(str(streamed.value), expected) + raw = body["error"]["message"] if isinstance(body["error"], dict) else body["error"] + assert raw.split(".")[0][:40] in caplog.text.replace('\\"', '"') # the raw reply stays in the log + + with pytest.raises(ProviderFailed) as planned: # background jobs (task planning, memory) say the same + await llm.complete_json("planner", [{"role": "user", "content": "plan"}]) + assert_plain(str(planned.value), expected) + + +async def test_unreachable_and_slow_providers(http): + http.post(OLLAMA).mock(side_effect=httpx.ConnectError("All connection attempts failed")) + http.post(OPENROUTER).mock(side_effect=httpx.ReadTimeout("timed out")) + with pytest.raises(ProviderFailed) as down: + await provider("ollama_chat/qwen3:8b").complete_text("primary", [{"role": "user", "content": "hi"}]) + assert_plain(str(down.value), "Sentient couldn't reach Ollama. Make sure it's running, then try again.") + with pytest.raises(ProviderFailed) as slow: + await provider(FREE).complete_text("primary", [{"role": "user", "content": "hi"}]) + assert_plain(str(slow.value), "OpenRouter took too long to answer. Try again in a moment, or " + PICK) + + # what LiteLLM's default aiohttp transport says when Ollama isn't running (recorded) + refused = litellm.APIConnectionError( + message="Ollama_chatException - Cannot connect to host 127.0.0.1:11434 ssl:default " + "[The remote computer refused the network connection]", llm_provider="ollama_chat", model="qwen3:8b") + assert plain_error(refused, "ollama_chat/qwen3:8b") == str(down.value) + offline = litellm.APIConnectionError(message="OpenrouterException - Cannot connect to host openrouter.ai:443", + llm_provider="openrouter", model="x") + assert plain_error(offline, FREE) == ( + "Sentient couldn't reach OpenRouter. Check your internet connection and try again.") + + +async def test_unknown_failures_and_tool_refusals_keep_their_own_text(http): + from sentient.agent.loop import is_tool_format_error + + http.post(OPENROUTER).mock(return_value=httpx.Response(404, json=NO_TOOL_ENDPOINTS)) + with pytest.raises(ProviderError) as no_tools: + [c async for c in provider(FREE).stream("primary", [{"role": "user", "content": "hi"}], tools=[{ + "type": "function", "function": {"name": "t", "parameters": {"type": "object", "properties": {}}}}])] + assert not isinstance(no_tools.value, PlainProviderError) + assert is_tool_format_error(no_tools.value) # chat answers without tools instead of failing + + from sentient.llm.provider import _failure + from sentient.llm.responses import ResponsesError + + no_tools_plan = ResponsesError("This model does not support tools.", 400) + assert _failure(no_tools_plan, "chatgpt/gpt-test") is no_tools_plan # left for the loop to answer without tools + assert isinstance(_failure(ResponsesError("Usage limit reached.", 429), "chatgpt/gpt-test"), ProviderFailed) + + odd = litellm.BadRequestError(message="OpenrouterException - something new", model="x", llm_provider="openrouter") + assert plain_error(odd, FREE) is None + assert plain_error(ValueError("Model did not return JSON: 'x'"), FREE) is None # Sentient's own errors + assert not is_tool_format_error(ProviderFailed("Tools are parsed... by nobody")) + + +async def test_a_backup_model_reports_its_own_reason(http): + http.post(OPENROUTER).mock(return_value=httpx.Response(429, json=FREE_PER_DAY)) + http.post(OLLAMA).mock(side_effect=httpx.ConnectError("All connection attempts failed")) + llm = provider(FREE) + llm.config.models.fallbacks = {"primary": ["ollama_chat/qwen3:8b"]} + with pytest.raises(ProviderFailed, match="couldn't reach Ollama"): + [c async for c in llm.stream("primary", [{"role": "user", "content": "hi"}])] + + +async def test_a_reply_cut_off_midway_says_why(monkeypatch): + async def acompletion(**kwargs): + async def chunks(): + from types import SimpleNamespace + + yield SimpleNamespace(choices=[SimpleNamespace(delta=SimpleNamespace(content="Hel", reasoning_content=None))]) + raise litellm.ServiceUnavailableError(message="OpenrouterException - " + json.dumps(OPENROUTER_503), + llm_provider="openrouter", model="some/model") + + return chunks() + + monkeypatch.setattr(litellm, "acompletion", acompletion) + monkeypatch.setattr("sentient.llm.provider.secrets.get_secret", lambda *a, **k: None) + got = [] + with pytest.raises(ProviderFailed) as err: + async for c in provider("openrouter/some/model").stream("primary", [{"role": "user", "content": "hi"}]): + got.append(c.text) + assert got == ["Hel"] + assert_plain(str(err.value), "OpenRouter is having trouble right now. Try again in a few minutes, or " + PICK) + + +async def test_embeddings_say_it_plainly_too(http): + http.post("http://localhost:11434/api/embed").mock(return_value=httpx.Response(404, json={ + "error": 'model "nomic-embed-text" not found, try pulling it first'})) + with pytest.raises(ProviderFailed, match="nomic-embed-text isn't downloaded in Ollama"): + await provider(FREE).embed(["hello"], model="ollama/nomic-embed-text") + + +async def test_chat_and_tasks_show_the_plain_sentence(http, config, isolated_home): + """End to end in the engine: the chat's error event and a task's error, with the real provider class.""" + http.post(OPENROUTER).mock(return_value=httpx.Response(429, json=FREE_PER_DAY)) + for role in ("primary", "fast", "planner", "executor"): + setattr(config.models.roles, role, FREE) + config.models.fallbacks = {} + expected = ("OpenRouter's free models have reached today's limit. Add credits on openrouter.ai, wait until " + "tomorrow, or " + PICK) + app = SentientApp(config, llm=LiteLLMProvider(config), db_path=isolated_home / "e.db", enable_background=False) + await app.start() + try: + sid = await app.store.create_session(channel="cli") + events = [ev async for ev in app.agent.run_turn(sid, "hi", channel="cli")] + errors = [e for e in events if isinstance(e, Error)] + assert len(errors) == 1 and not errors[0].recoverable + assert_plain(errors[0].message, expected) + assert not any(isinstance(e, Done) and "litellm" in (e.content or "") for e in events) + + task = await app.tasks.create_task("Look up tomorrow's weather in Pune") + await app.tasks.drain() + task = await app.tasks.get(task["task_id"]) + assert task["status"] == "error" + assert_plain(task["error"], expected) + finally: + await app.stop()