Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 32 additions & 0 deletions docs/cookbooks/20-judgments.md
Original file line number Diff line number Diff line change
Expand Up @@ -99,3 +99,35 @@ df["quality"] = pd.Categorical(df.quality, categories=range(5), ordered=True)
- Pictures: `Message.user(image(path="door.jpg"))` makes the image Jev's state; `Message.user(["Listing: a red mug.", image(path="a.png"), image(path="b.jpg")])` sends the parts in order, as an array (a question can say "the second image" or point at `` `[2]` ``). PNG and JPEG bytes only: a URL, a file id, another format, audio or video is refused before the wire. lm15 sends TypeSafe's `TypeSafe-Danger-Zone-Images: true` header for you; TypeSafe calls images beta, so check answers on pictures you know. `lm15-contract/changes/2026-10-14-jev-images.md`.
- Bounding boxes: `lm15-contract/changes/2026-10-15-bounding-box.md`.
- Contract: `lm15-contract/changes/2026-09-17-judgments.md`, MAP-14.


## OpenAI Decisions

Select `openai-decisions:gpt-6-luna` to send judgments to `/v1/decisions`
using `OPENAI_API_KEY`. The ordinary `openai:` route still uses Responses.
Boolean judgments become predicates; choices and ordered levels retain their
probability distributions. `response.expected(name)` reads an ordered score's
expected value; `response.data[name]` is its most likely level.

```python
from lm15 import LMRouter, Request, Message, Config, judgments, yes_no

with LMRouter() as router:
result = router.complete(Request(
model="openai-decisions:gpt-6-luna",
messages=(Message.user("The package arrived damaged."),),
config=Config(response_format=judgments(damaged=yes_no("Does the message report damage?"))),
))
print(result.probabilities["damaged"])
```

`AsyncLMRouter` uses the same codec. Text, JSON data, and image user inputs are
supported, together with text conversation history. Streaming, tool-call
history, bounding boxes, free-form outputs, generation controls, and explicit
`Config.cache` controls are rejected. The endpoint's implicit prompt caching
still applies; native cache options can be supplied through `Config.extensions`.
Refusals retain other answers and usage, set `finish_reason="content_filter"`,
and name refused questions in `provider_data["decisions"]["refusals"]`.
The original reply is retained under `provider_data["openai-decisions"]`.

See the [OpenAI Decisions guide](https://developers.openai.com/api/docs/guides/decisions).
4 changes: 3 additions & 1 deletion lm15/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,7 @@
OpenAICodexLM,
XaiLM,
TypeSafeLM,
OpenAIDecisionsLM,
)
from .providers.openai_chat import request_from_openai_chat, response_from_openai_chat
from .protocols import ProviderLM
Expand All @@ -194,6 +195,7 @@
AsyncOpenAICodexLM,
AsyncXaiLM,
AsyncTypeSafeLM,
AsyncOpenAIDecisionsLM,
)

# ── Stream assembly ──────────────────────────────────────────────────
Expand Down Expand Up @@ -269,7 +271,7 @@
"Auth", "AsyncAuth", "BoundClient", "TerminalUI", "providers", "connect",
# providers
"bounding_box", "choice", "judgments", "score", "yes_no", "data",
"OpenAILM", "OpenAIChatLM", "AnthropicLM", "GeminiLM", "ClaudeCodeLM", "OpenAICodexLM", "XaiLM", "TypeSafeLM", "AsyncTypeSafeLM",
"OpenAILM", "OpenAIChatLM", "AnthropicLM", "GeminiLM", "ClaudeCodeLM", "OpenAICodexLM", "XaiLM", "TypeSafeLM", "AsyncTypeSafeLM", "OpenAIDecisionsLM", "AsyncOpenAIDecisionsLM",
"ProviderLM",
# the Chat Completions dialect read in: a request body -> Request (MAP-12), a response body -> Response
"request_from_openai_chat", "response_from_openai_chat",
Expand Down
7 changes: 7 additions & 0 deletions lm15/access.py
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,13 @@
base_url=DEFAULT_CODEX_BASE_URL,
)

OPENAI_DECISIONS_API = AccessPolicy(
provider="openai-decisions",
supports=EndpointSupport(complete=True, stream=False),
auth_modes=("bearer",),
env_keys=("OPENAI_API_KEY",),
)

OPENAI_CHAT_API = AccessPolicy(
provider="openai-chat",
supports=EndpointSupport(complete=True, stream=True, models=True),
Expand Down
4 changes: 4 additions & 0 deletions lm15/providers/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
AsyncOpenAILM,
AsyncTransport,
AsyncTypeSafeLM,
AsyncOpenAIDecisionsLM,
AsyncXaiLM,
)
from .base import BaseProviderLM, Credential, HttpResponse, ProviderDialect, SyncTransport, resolve_credential
Expand All @@ -18,6 +19,7 @@
from .openai_chat import OpenAIChatLM
from .openai_codex import OpenAICodexLM
from .typesafe import TypeSafeLM
from .openai_decisions import OpenAIDecisionsLM
from .xai import XaiLM

__all__ = [
Expand All @@ -30,6 +32,8 @@
"XaiLM",
"AsyncXaiLM",
"TypeSafeLM",
"OpenAIDecisionsLM",
"AsyncOpenAIDecisionsLM",
"AsyncTypeSafeLM",
"AsyncOpenAILM",
"AsyncOpenAIChatLM",
Expand Down
34 changes: 34 additions & 0 deletions lm15/providers/async_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@
OpenAICodexLM,
)
from .typesafe import TypeSafeLM
from .openai_decisions import OpenAIDecisionsLM
from .xai import DEFAULT_XAI_BASE_URL, XaiLM

_T = TypeVar("_T")
Expand Down Expand Up @@ -1068,3 +1069,36 @@ def __post_init__(self) -> None:
clock=self.clock,
)
self._mirror_binding()


@dataclass
class AsyncOpenAIDecisionsLM(AsyncBaseProviderLM):
"""Async mirror of :class:`OpenAIDecisionsLM` (composition; the inner adapter
cannot reach the network)."""

api_key: Credential | None = field(default=None, repr=False)
transport: AsyncTransport = field(default_factory=default_async_transport)
base_url: str = "https://api.openai.com/v1"
access: ProviderManifest = field(default=OpenAIDecisionsLM.manifest, repr=False)
credentials_path: "str | os.PathLike[str] | None" = field(default=None, repr=False)
settings: "Mapping[str, str] | None" = None
clock: "Callable[[], datetime] | None" = field(default=None, repr=False)
adaptations: AdaptationPolicy = field(default="note", kw_only=True)

provider: str = field(default="openai-decisions", init=False)
manifest: ClassVar[ProviderManifest] = OpenAIDecisionsLM.manifest

_inner: OpenAIDecisionsLM = field(init=False, repr=False, compare=False)

def __post_init__(self) -> None:
self._inner = OpenAIDecisionsLM(
adaptations=self.adaptations,
api_key=self.api_key,
transport=_ForbiddenTransport(),
base_url=self.base_url,
access=self.access,
credentials_path=self.credentials_path,
settings=self.settings,
clock=self.clock,
)
self._mirror_binding()
238 changes: 238 additions & 0 deletions lm15/providers/openai_decisions.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,238 @@
"""OpenAI Decisions: typed judgments, not text generation.

Wire contract: https://developers.openai.com/api/docs/guides/decisions
The Responses codec supplies conversation input; decisions have their own
questions, answers, capabilities, and endpoint. No SDK dependency is needed.
"""

from __future__ import annotations

import json
from dataclasses import dataclass, field, fields
from typing import Any, ClassVar

from ..access import OPENAI_DECISIONS_API
from ..adaptation import adapt
from ..errors import ProviderError, UnsupportedFeatureError
from ..features import ProviderManifest
from ..judgments import non_judgment_properties, request_judgments
from ..types import Config, DataPart, ImagePart, Message, Part, RefusalPart, Request, Response, TextPart
from .base import HttpResponse
from .common import parts_to_text, usage_from_responses
from .openai import OpenAILM


def _text(value: Any) -> str:
return value if isinstance(value, str) else json.dumps(value, ensure_ascii=False, allow_nan=False)


@dataclass
class OpenAIDecisionsLM(OpenAILM):
provider: str = field(default="openai-decisions", init=False)
manifest: ClassVar[ProviderManifest] = OPENAI_DECISIONS_API

def __post_init__(self):
self._compat_base = None
super().__post_init__()

def _refuse(self, feature: str, reason: str) -> UnsupportedFeatureError:
return UnsupportedFeatureError(f"OpenAI Decisions: {reason}", provider=self.provider, feature=feature)

def _questions(self, request: Request) -> list[dict[str, Any]]:
found = request_judgments(request)
fmt = request.config.response_format or {}
if not found or non_judgment_properties(fmt.get("schema"), found):
raise self._refuse("response_format", "declare only boolean, choice, or ordered-score judgments")
if len(found) > 400:
raise self._refuse("questions", "at most 400 questions are supported")
overrides = (request.config.extensions or {}).get("questions", {})
if not isinstance(overrides, dict) or overrides.keys() - found.keys():
raise self._refuse("questions", "question overrides must name declared judgments")
questions = []
for name, judgment in found.items():
if judgment.kind == "box":
raise self._refuse("response_format", "bounding boxes are not supported")
kind = {"boolean": "predicate", "choice": "choice", "ordered": "score"}[judgment.kind]
override = overrides.get(name, {})
if not isinstance(override, dict) or override.keys() - {"type", "instructions", "criteria"}:
raise self._refuse("questions", f"invalid question override for {name!r}")
if override.get("type", kind) not in ({"noul", "predicate"} if kind == "predicate" else {kind}):
raise self._refuse("questions", f"override changes the answer type for {name!r}")
instruction = override.get("instructions", judgment.instruction or name)
question: dict[str, Any] = {"name": name, "type": kind, "instructions": _text(instruction)}
criteria: Any = override.get("criteria")
if kind == "predicate":
if criteria is not None:
question["instructions"] = _text({"instructions": instruction, "criteria": criteria})
elif kind == "choice":
if not 2 <= len(judgment.keys) <= 255:
raise self._refuse("questions", "choices require 2–255 options")
criteria = criteria if criteria is not None else judgment.descriptions
if not isinstance(criteria, dict) or set(criteria) != set(judgment.keys):
raise self._refuse("questions", f"choice criteria must preserve the keys of {name!r}")
question["choices"] = [
{"value": key, **({"description": _text(criteria[key])} if criteria[key] is not None else {})}
for key in judgment.keys
]
else:
criteria = (
criteria if criteria is not None else [judgment.descriptions.get(k) or k for k in judgment.keys]
)
if (
not isinstance(criteria, list)
or len(criteria) != len(judgment.keys)
or not 2 <= len(criteria) <= 10
):
raise self._refuse("questions", "score criteria must preserve the declared 2–10 levels")
question["levels"] = [
{"label": key, "description": _text(desc)} for key, desc in zip(judgment.keys, criteria)
]
questions.append(question)
return questions

def _payload(self, request: Request, stream: bool) -> dict[str, Any]:
if stream:
raise self._refuse("stream", "streaming is not supported")
if request.tools:
raise self._refuse("tools", "decisions do not invoke tools")
cfg = request.config
defaults = Config()
for item in fields(cfg):
value = getattr(cfg, item.name)
if item.name in {"response_format", "extensions", "probabilities", "user_id"} or value == getattr(defaults, item.name):
continue
if item.name in {"temperature", "top_p", "top_k", "seed", "frequency_penalty", "presence_penalty"}:
adapt(f"config.{item.name}", "dropped", "MAP-13: decisions score fixed alternatives rather than sampling text",
asked=value, provider=self.provider)
else:
raise self._refuse(f"config.{item.name}", f"MAP-13: {item.name} cannot be mapped safely to Decisions")
extensions = cfg.extensions or {}
extra = extensions.keys() - {"questions", "prompt_cache_key", "prompt_cache_options", "safety_identifier"}
if extra:
raise self._refuse("config.extensions", f"unsupported options: {sorted(extra)}")
# Admit only content this endpoint can read. Never let the broader
# Responses codec silently drop or send unsupported media.
for message in request.messages:
for part in message.parts:
allowed = (
(TextPart, DataPart, ImagePart)
if message.role == "user"
else (TextPart, RefusalPart)
if message.role == "assistant"
else (TextPart,)
)
if not isinstance(part, allowed) or message.role == "tool":
raise self._refuse("messages", "use text/JSON/images in user messages and text in other roles")
inputs = self._build_input(request.messages, self._compat(request))
if request.system:
system = request.system if isinstance(request.system, str) else parts_to_text(request.system)
inputs.insert(0, {"role": "developer", "content": system})
payload: dict[str, Any] = {"model": request.model, "input": inputs, "questions": self._questions(request)}
if cfg.user_id is not None:
payload["safety_identifier"] = cfg.user_id
payload.update({key: value for key, value in extensions.items() if key != "questions"})
return payload

def build_request(self, request: Request, stream: bool):
request = self._wire_request(request)
return self._emit(
method="POST",
url=f"{self.base_url.rstrip('/')}/decisions",
endpoint="decisions",
model=request.model,
headers=self._headers(),
payload=self._payload(request, stream),
)

def parse_response(self, request: Request, response: HttpResponse) -> Response:
request = self._wire_request(request)
data = response.json()
questions = self._questions(request)

def invalid(detail):
return ProviderError(
f"Malformed OpenAI Decisions reply: {detail}",
provider=self.provider,
status=response.status,
request_id=response.header("x-request-id"),
)

def probability(value):
if isinstance(value, bool) or not isinstance(value, (int, float)) or not 0 <= value <= 1:
raise invalid("probabilities must be finite numbers in [0, 1]")
return float(value)

if not isinstance(data, dict) or not isinstance(data.get("answers"), list):
raise invalid("expected an answers array")
raw_answers = data["answers"]
if len(raw_answers) != len(questions):
raise invalid("answer count differs from question count")
values, distributions, evidence, refusals = {}, {}, {}, []
for question, answer in zip(questions, raw_answers):
name, kind = question["name"], question["type"]
if not isinstance(answer, dict) or answer.get("name") != name:
raise invalid("answers must match question names and order")
if answer.get("type") == "refusal":
refusals.append(name)
continue
if answer.get("type") != kind:
raise invalid(f"wrong answer type for {name!r}")
if kind == "predicate":
p = probability(answer.get("probability"))
values[name] = p >= 0.5
distributions[name] = {"true": p, "false": 1 - p}
evidence[name] = {"noul": p}
continue
raw = answer.get("probabilities")
expected = (
[x["value"] for x in question["choices"]] if kind == "choice" else list(range(len(question["levels"])))
)
if not isinstance(raw, list) or len(raw) != len(expected):
raise invalid(f"incomplete distribution for {name!r}")
probs = {}
for entry in raw:
if not isinstance(entry, dict):
raise invalid("expected probability objects")
key = entry.get("value")
if type(key) is not type(expected[0]) or key not in expected or str(key) in probs:
raise invalid(f"invalid or duplicate distribution key for {name!r}")
if kind == "score" and entry.get("label") != question["levels"][key]["label"]:
raise invalid(f"invalid score label for {name!r}")
probs[str(key)] = probability(entry.get("probability"))
confidence = probability(answer.get("confidence"))
if kind == "choice":
value = answer.get("choice")
if type(value) is not str or value not in expected:
raise invalid(f"invalid selected choice for {name!r}")
else:
score = answer.get("score")
if (
isinstance(score, bool)
or not isinstance(score, (int, float))
or not 0 <= score <= len(expected) - 1
):
raise invalid(f"invalid score for {name!r}")
value = int(max(probs, key=lambda key: probs[key]))
values[name], distributions[name] = value, probs
evidence[name] = {"probabilities": probs, "confidence": confidence}
try:
parts: list[Part] = [
DataPart(
value=values,
probabilities=distributions or None,
method="provider_classification" if distributions else None,
)
]
if refusals:
parts.append(RefusalPart(text="Decisions refused: " + ", ".join(refusals)))
provider_data: dict[str, Any] = {self.provider: data, "decisions": {"answers": evidence, "refusals": refusals}}
return Response(
id=response.header("x-request-id"),
model=data.get("model", request.model),
message=Message(role="assistant", parts=tuple(parts)),
finish_reason="content_filter" if refusals else "stop",
usage=usage_from_responses(data.get("usage")),
provider_data=provider_data,
)
except (TypeError, ValueError, OverflowError) as error:
raise invalid(str(error)) from error
Loading