diff --git a/src/lib/api-config.ts b/src/lib/api-config.ts index 83bce92ad8..c18b023ce1 100644 --- a/src/lib/api-config.ts +++ b/src/lib/api-config.ts @@ -7,11 +7,11 @@ export const standardHeaders = () => ({ accept: "application/json", }) -const COPILOT_VERSION = "0.26.7" +const COPILOT_VERSION = "0.48.1" const EDITOR_PLUGIN_VERSION = `copilot-chat/${COPILOT_VERSION}` const USER_AGENT = `GitHubCopilotChat/${COPILOT_VERSION}` -const API_VERSION = "2025-04-01" +const API_VERSION = "2026-01-09" export const copilotBaseUrl = (state: State) => state.accountType === "individual" ? diff --git a/src/lib/state.ts b/src/lib/state.ts index 5ba4dc1d1c..cd3f76778a 100644 --- a/src/lib/state.ts +++ b/src/lib/state.ts @@ -1,3 +1,4 @@ +import type { ModelSession } from "~/services/copilot/create-model-session" import type { ModelsResponse } from "~/services/copilot/get-models" export interface State { @@ -6,6 +7,8 @@ export interface State { accountType: string models?: ModelsResponse + modelSession?: ModelSession + proxyBaseUrl?: string vsCodeVersion?: string manualApprove: boolean diff --git a/src/lib/tokenizer.ts b/src/lib/tokenizer.ts index 8c3eda7368..eef9fb14c0 100644 --- a/src/lib/tokenizer.ts +++ b/src/lib/tokenizer.ts @@ -143,7 +143,7 @@ const getEncodeChatFunction = async (encoding: string): Promise => { * Get tokenizer type from model information */ export const getTokenizerFromModel = (model: Model): string => { - return model.capabilities.tokenizer || "o200k_base" + return model.capabilities?.tokenizer ?? "o200k_base" } /** diff --git a/src/routes/chat-completions/handler.ts b/src/routes/chat-completions/handler.ts index 04a5ae9edd..8219b23306 100644 --- a/src/routes/chat-completions/handler.ts +++ b/src/routes/chat-completions/handler.ts @@ -39,10 +39,16 @@ export async function handleCompletion(c: Context) { if (state.manualApprove) await awaitApproval() - if (isNullish(payload.max_tokens)) { + // An explicitly-null max_completion_tokens still counts as "present": + // forwarding it together with an injected max_tokens would recreate the + // both-parameters rejection this guard exists to prevent. + if ( + isNullish(payload.max_tokens) + && payload.max_completion_tokens === undefined + ) { payload = { ...payload, - max_tokens: selectedModel?.capabilities.limits.max_output_tokens, + max_tokens: selectedModel?.capabilities?.limits.max_output_tokens, } consola.debug("Set max_tokens to:", JSON.stringify(payload.max_tokens)) } diff --git a/src/routes/completions/route.ts b/src/routes/completions/route.ts new file mode 100644 index 0000000000..2eac777bb1 --- /dev/null +++ b/src/routes/completions/route.ts @@ -0,0 +1,101 @@ +import { Hono } from "hono" +import { streamSSE } from "hono/streaming" + +import { awaitApproval } from "~/lib/approval" +import { forwardError } from "~/lib/error" +import { checkRateLimit } from "~/lib/rate-limit" +import { state } from "~/lib/state" +import { + chatResultToCompletionResult, + completionPayloadToChatPayload, + streamChatAsCompletionChunks, +} from "~/services/copilot/completions-adapter" +import { createChatCompletions } from "~/services/copilot/create-chat-completions" +import { + aggregateProxyCompletionsStream, + createProxyCompletions, + type CompletionsPayload, +} from "~/services/copilot/create-proxy-completions" + +export const completionsRoutes = new Hono() + +completionsRoutes.post("/", async (c) => { + try { + // same request guards as the chat route: rate limit + manual approval + await checkRateLimit(state) + if (state.manualApprove) await awaitApproval() + + const body = (await c.req.json()) as unknown + if (typeof body !== "object" || body === null) { + return c.json({ error: "model is required" }, 400) + } + const payload = body as CompletionsPayload + if (typeof payload.model !== "string" || payload.model.length === 0) { + return c.json({ error: "model is required" }, 400) + } + + // legacy fill-in-the-middle engines (gpt-41-copilot, ...) live behind + // the traditional proxy completions API; everything else is translated + // through Chat Completions so all models are reachable here + if (payload.model.endsWith("-copilot")) { + if (payload.stream) { + const response = await createProxyCompletions(payload) + const body = response.body + if (!body) { + return c.json({ error: "Empty response body" }, 502) + } + return c.body(body, 200, { + "content-type": "text/event-stream", + "cache-control": "no-cache", + }) + } + + const result = await aggregateProxyCompletionsStream(payload) + return c.json(result) + } + + // array prompts are independent prompts (or FIM prefix/suffix pairs) in + // the legacy contract and cannot be translated to a single chat turn + if (Array.isArray(payload.prompt)) { + return c.json( + { + error: + "Array prompts are only supported for -copilot engine models; chat-model translation accepts a single string prompt", + }, + 400, + ) + } + + // FIM suffix has no chat-completions equivalent; reject explicitly + // instead of silently dropping it + if (payload.suffix !== undefined && payload.suffix !== null) { + return c.json( + { error: "suffix is only supported for -copilot engine models" }, + 400, + ) + } + + const chatPayload = completionPayloadToChatPayload(payload) + + if (payload.stream) { + const chatStream = (await createChatCompletions({ + ...chatPayload, + stream: true, + })) as AsyncIterable<{ data?: string | null }> + + return streamSSE(c, async (stream) => { + for await (const event of streamChatAsCompletionChunks(chatStream)) { + await stream.writeSSE({ data: event.data }) + } + }) + } + + const chatResult = await createChatCompletions(chatPayload) + const result = chatResultToCompletionResult( + chatResult as Parameters[0], + ) + return c.json(result) + } catch (error) { + return await forwardError(c, error) + } +}) diff --git a/src/routes/embeddings/route.ts b/src/routes/embeddings/route.ts index 4c4fc7b8a3..4f1ce9eb9a 100644 --- a/src/routes/embeddings/route.ts +++ b/src/routes/embeddings/route.ts @@ -1,6 +1,9 @@ import { Hono } from "hono" +import { awaitApproval } from "~/lib/approval" import { forwardError } from "~/lib/error" +import { checkRateLimit } from "~/lib/rate-limit" +import { state } from "~/lib/state" import { createEmbeddings, type EmbeddingRequest, @@ -10,6 +13,11 @@ export const embeddingRoutes = new Hono() embeddingRoutes.post("/", async (c) => { try { + // same request guards as the other generation routes: embeddings + // calls must not bypass --rate-limit / --manual safeguards + await checkRateLimit(state) + if (state.manualApprove) await awaitApproval() + const paylod = await c.req.json() const response = await createEmbeddings(paylod) diff --git a/src/routes/responses/route.ts b/src/routes/responses/route.ts new file mode 100644 index 0000000000..7eadfae92c --- /dev/null +++ b/src/routes/responses/route.ts @@ -0,0 +1,86 @@ +import { Hono } from "hono" +import { streamSSE } from "hono/streaming" + +import { awaitApproval } from "~/lib/approval" +import { forwardError } from "~/lib/error" +import { checkRateLimit } from "~/lib/rate-limit" +import { state } from "~/lib/state" +import { createChatCompletions } from "~/services/copilot/create-chat-completions" +import { + createResponses, + type ResponsesPayload, +} from "~/services/copilot/create-responses" +import { + chatResultToResponsesResult, + isResponsesCapableModel, + responsesPayloadHints, + responsesPayloadToChatPayload, + streamChatAsResponsesEvents, +} from "~/services/copilot/responses-adapter" + +export const responseRoutes = new Hono() + +responseRoutes.post("/", async (c) => { + try { + // same request guards as the chat route: rate limit + manual approval + await checkRateLimit(state) + if (state.manualApprove) await awaitApproval() + + const body = (await c.req.json()) as unknown + if (typeof body !== "object" || body === null) { + return c.json({ error: "model is required" }, 400) + } + const payload = body as Record + if (typeof payload.model !== "string" || payload.model.length === 0) { + return c.json({ error: "model is required" }, 400) + } + const model = payload.model + + // responses-native models stream straight through + if (isResponsesCapableModel(model)) { + const response = await createResponses( + payload as ResponsesPayload, + responsesPayloadHints(payload), + ) + const body = response.body + if (!body) { + return c.json({ error: "Empty response body" }, 502) + } + + const contentType = + response.headers.get("content-type") ?? "application/json" + + return c.body(body, 200, { + "content-type": contentType, + "cache-control": "no-cache", + }) + } + + // chat-only models are served by translating through Chat Completions + const chatPayload = responsesPayloadToChatPayload(payload) + + if (payload.stream === true) { + const chatStream = (await createChatCompletions({ + ...chatPayload, + stream: true, + })) as AsyncIterable<{ data?: string | null }> + + return streamSSE(c, async (stream) => { + for await (const event of streamChatAsResponsesEvents( + chatStream, + model, + )) { + await stream.writeSSE({ data: event.data }) + } + }) + } + + const chatResult = await createChatCompletions(chatPayload) + const responsesResult = chatResultToResponsesResult( + chatResult as Parameters[0], + ) + return c.json(responsesResult) + } catch (error) { + return await forwardError(c, error) + } +}) diff --git a/src/server.ts b/src/server.ts index 462a278f38..8102162a38 100644 --- a/src/server.ts +++ b/src/server.ts @@ -3,9 +3,11 @@ import { cors } from "hono/cors" import { logger } from "hono/logger" import { completionRoutes } from "./routes/chat-completions/route" +import { completionsRoutes } from "./routes/completions/route" import { embeddingRoutes } from "./routes/embeddings/route" import { messageRoutes } from "./routes/messages/route" import { modelRoutes } from "./routes/models/route" +import { responseRoutes } from "./routes/responses/route" import { tokenRoute } from "./routes/token/route" import { usageRoute } from "./routes/usage/route" @@ -17,15 +19,19 @@ server.use(cors()) server.get("/", (c) => c.text("Server running")) server.route("/chat/completions", completionRoutes) +server.route("/completions", completionsRoutes) server.route("/models", modelRoutes) server.route("/embeddings", embeddingRoutes) +server.route("/responses", responseRoutes) server.route("/usage", usageRoute) server.route("/token", tokenRoute) // Compatibility with tools that expect v1/ prefix server.route("/v1/chat/completions", completionRoutes) +server.route("/v1/completions", completionsRoutes) server.route("/v1/models", modelRoutes) server.route("/v1/embeddings", embeddingRoutes) +server.route("/v1/responses", responseRoutes) // Anthropic compatible endpoints server.route("/v1/messages", messageRoutes) diff --git a/src/services/copilot/completions-adapter.ts b/src/services/copilot/completions-adapter.ts new file mode 100644 index 0000000000..701f2f446e --- /dev/null +++ b/src/services/copilot/completions-adapter.ts @@ -0,0 +1,96 @@ +import type { + CompletionsPayload, + ProxyCompletionResult, +} from "~/services/copilot/create-proxy-completions" + +import { isNullish } from "~/lib/utils" +import { + type ChatCompletionChunk, + type ChatCompletionsPayload, + type ChatCompletionResponse, +} from "~/services/copilot/create-chat-completions" + +/* + * Legacy completions surface for chat-capable models: translates the old + * prompt-based /v1/completions format through Chat Completions, so every + * usable model is reachable from legacy clients too. + */ + +export const completionPayloadToChatPayload = ( + payload: CompletionsPayload, +): ChatCompletionsPayload => { + // Array prompts mean independent prompts (or FIM prefix/suffix pairs) in + // the legacy contract and cannot be represented as a single chat message; + // the route rejects them before we get here. + if (typeof payload.prompt !== "string") { + throw new TypeError("chat translation requires a single string prompt") + } + + const result: ChatCompletionsPayload = { + model: payload.model, + messages: [{ role: "user", content: payload.prompt }], + } + + if (!isNullish(payload.max_tokens)) result.max_tokens = payload.max_tokens + if (!isNullish(payload.temperature)) { + result.temperature = payload.temperature + } + if (!isNullish(payload.top_p)) result.top_p = payload.top_p + if (!isNullish(payload.stop)) result.stop = payload.stop + if (!isNullish(payload.n)) result.n = payload.n + + return result +} + +export const chatResultToCompletionResult = ( + chat: ChatCompletionResponse, +): ProxyCompletionResult => ({ + id: chat.id, + object: "text_completion", + created: chat.created, + model: chat.model, + choices: chat.choices.map((choice) => ({ + text: choice.message.content ?? "", + index: choice.index, + finish_reason: choice.finish_reason, + logprobs: null, + })), + usage: + chat.usage ? + { + prompt_tokens: chat.usage.prompt_tokens, + completion_tokens: chat.usage.completion_tokens, + total_tokens: chat.usage.total_tokens, + } + : undefined, +}) + +export async function* streamChatAsCompletionChunks( + chatStream: AsyncIterable<{ data?: string | null }>, +): AsyncGenerator<{ data: string }> { + for await (const event of chatStream) { + if (!event.data) continue + // skip the non-JSON SSE terminator line + if (!event.data.startsWith("{")) break + const chunk = JSON.parse(event.data) as ChatCompletionChunk + if (chunk.choices.length === 0) continue + + yield { + data: JSON.stringify({ + id: chunk.id, + object: "text_completion", + created: chunk.created, + model: chunk.model, + choices: chunk.choices.map((choice) => ({ + text: choice.delta.content ?? "", + index: choice.index, + finish_reason: choice.finish_reason, + logprobs: null, + })), + }), + } + } + + // emit the conventional SSE terminator line + yield { data: "[DO" + "NE]" } +} diff --git a/src/services/copilot/create-chat-completions.ts b/src/services/copilot/create-chat-completions.ts index 8534151daa..724685d5ad 100644 --- a/src/services/copilot/create-chat-completions.ts +++ b/src/services/copilot/create-chat-completions.ts @@ -4,6 +4,15 @@ import { events } from "fetch-event-stream" import { copilotHeaders, copilotBaseUrl } from "~/lib/api-config" import { HTTPError } from "~/lib/error" import { state } from "~/lib/state" +import { modelSessionHeaders } from "~/services/copilot/create-model-session" +import { createResponses } from "~/services/copilot/create-responses" +import { + chatPayloadToResponsesPayload, + isResponsesOnlyModel, + responsesResultToChatCompletion, + streamResponsesAsChatChunks, + type ResponsesResult, +} from "~/services/copilot/responses-adapter" export const createChatCompletions = async ( payload: ChatCompletionsPayload, @@ -22,10 +31,35 @@ export const createChatCompletions = async ( ["assistant", "tool"].includes(msg.role), ) + // Models that only serve the Responses API (gpt-5.3-codex, gpt-5.4-mini, + // ...) are handled via protocol translation so clients can keep using the + // Chat Completions format + if (isResponsesOnlyModel(payload.model)) { + const responsesPayload = chatPayloadToResponsesPayload(payload) + const upstream = await createResponses( + responsesPayload as Parameters[0], + { vision: enableVision, initiator: isAgentCall ? "agent" : "user" }, + ) + + if (payload.stream) { + return streamResponsesAsChatChunks(upstream, payload.model, events) + } + + return responsesResultToChatCompletion( + (await upstream.json()) as ResponsesResult, + payload.model, + ) + } + + // Some models (e.g. Claude on limited SKUs) are only served through the + // auto-mode session flow and require the Copilot-Session-Token header + const sessionHeaders = await modelSessionHeaders(payload.model) + // Build headers and add X-Initiator const headers: Record = { ...copilotHeaders(state, enableVision), "X-Initiator": isAgentCall ? "agent" : "user", + ...sessionHeaders, } const response = await fetch(`${copilotBaseUrl(state)}/chat/completions`, { @@ -130,6 +164,7 @@ export interface ChatCompletionsPayload { temperature?: number | null top_p?: number | null max_tokens?: number | null + max_completion_tokens?: number | null stop?: string | Array | null n?: number | null stream?: boolean | null diff --git a/src/services/copilot/create-model-session.ts b/src/services/copilot/create-model-session.ts new file mode 100644 index 0000000000..a3c8f9c927 --- /dev/null +++ b/src/services/copilot/create-model-session.ts @@ -0,0 +1,130 @@ +import consola from "consola" + +import { copilotBaseUrl, copilotHeaders } from "~/lib/api-config" +import { state } from "~/lib/state" + +export interface ModelSession { + token: string + availableModels: Array + expiresAt: number + /** + * The copilot token this session was created with. Sessions are scoped to + * the issuing token, so a session must be re-created when the copilot + * token rotates underneath it. + */ + createdWithToken?: string +} + +interface ModelSessionResponse { + available_models?: Array + selected_model: string + session_token: string +} + +// Upstream session tokens live ~1h; refresh a bit early to be safe +const SESSION_TTL_MS = 55 * 60 * 1000 + +// When session creation fails (e.g. the account/SKU does not support it), +// back off for a while instead of retrying on every single request +const FAILURE_BACKOFF_MS = 5 * 60 * 1000 + +let retryAfter = 0 +let pending: Promise | undefined + +// The session's available_models pool is the intersection of the requested +// hints and what the account is entitled to. Hinting every known model +// maximizes the pool (the plain "auto" rotation is only a subset), unlocking +// e.g. the mai-code-1-flash-* variants that auto mode never includes. +const modelHints = (): Array => { + const hints = state.models?.data.map((entry) => entry.id) ?? [] + return hints.length > 0 ? hints : ["auto"] +} + +/** + * GitHub serves some models (e.g. Claude on limited SKUs) only through the + * auto-mode model session flow: POST /models/session returns a short-lived + * session token that must be attached as `Copilot-Session-Token` on chat, + * responses and messages calls for the models it lists as available. + * + * Best-effort: when session creation fails, requests are still sent without + * the header so base models keep working. Concurrent callers share one + * in-flight request; failures are cached for FAILURE_BACKOFF_MS. + */ +export const ensureModelSession = async (): Promise< + ModelSession | undefined +> => { + if (!state.copilotToken) return + + const current = state.modelSession + if ( + current + && current.expiresAt > Date.now() + // sessions pre-seeded by tests may not record the issuing token; only + // treat a session as stale when a recorded token no longer matches + && (current.createdWithToken === undefined + || current.createdWithToken === state.copilotToken) + ) { + return current + } + // monotonic clock: wall-clock jumps must not extend or skip the backoff + if (performance.now() < retryAfter) return undefined + + pending ??= createSession().finally(() => { + pending = undefined + }) + return pending +} + +const createSession = async (): Promise => { + try { + const response = await fetch(`${copilotBaseUrl(state)}/models/session`, { + method: "POST", + headers: copilotHeaders(state), + body: JSON.stringify({ auto_mode: { model_hints: modelHints() } }), + signal: AbortSignal.timeout(10_000), + }) + + if (!response.ok) { + consola.warn("Model session creation failed with status", response.status) + retryAfter = performance.now() + FAILURE_BACKOFF_MS + return undefined + } + + const data = (await response.json()) as ModelSessionResponse + const session: ModelSession = { + token: data.session_token, + availableModels: data.available_models ?? [], + expiresAt: Date.now() + SESSION_TTL_MS, + createdWithToken: state.copilotToken, + } + + state.modelSession = session + consola.info( + "Model session created, unlocked models:", + session.availableModels.join(", ") || "(none)", + ) + return session + } catch (error) { + consola.warn("Model session creation errored:", error) + retryAfter = performance.now() + FAILURE_BACKOFF_MS + return undefined + } +} + +/** Test-only: clears module-level backoff and in-flight state between tests. */ +export const __resetModelSessionForTests = (): void => { + retryAfter = 0 + pending = undefined +} + +/** + * Returns the `Copilot-Session-Token` header when the given model requires + * the auto-mode session flow, otherwise an empty header map. + */ +export const modelSessionHeaders = async ( + model: string, +): Promise> => { + const session = await ensureModelSession() + if (!session?.availableModels.includes(model)) return {} + return { "Copilot-Session-Token": session.token } +} diff --git a/src/services/copilot/create-proxy-completions.ts b/src/services/copilot/create-proxy-completions.ts new file mode 100644 index 0000000000..811592da7e --- /dev/null +++ b/src/services/copilot/create-proxy-completions.ts @@ -0,0 +1,172 @@ +import consola from "consola" +import { events } from "fetch-event-stream" + +import { copilotHeaders } from "~/lib/api-config" +import { HTTPError } from "~/lib/error" +import { state } from "~/lib/state" +import { getCopilotUsage } from "~/services/github/get-copilot-usage" + +const DEFAULT_PROXY_BASE_URL = "https://copilot-proxy.githubusercontent.com" + +/** + * Legacy "-copilot" engine models (e.g. `gpt-41-copilot` used for inline + * fill-in-the-middle suggestions) are not served by the CAPI chat endpoints; + * they live behind the traditional Copilot proxy completions API: + * + * POST {proxyBase}/v1/engines/{model}/completions (stream ONLY) + * + * The proxy rejects non-streaming requests, so `stream: true` is always + * forced upstream and aggregated for clients that ask for a full response. + */ +export const createProxyCompletions = async (payload: CompletionsPayload) => { + if (!state.copilotToken) throw new Error("Copilot token not found") + + const base = await ensureProxyBaseUrl() + const response = await fetch( + `${base}/v1/engines/${encodeURIComponent(payload.model)}/completions`, + { + method: "POST", + headers: copilotHeaders(state), + body: JSON.stringify({ ...payload, stream: true }), + }, + ) + + if (!response.ok) { + throw new HTTPError("Failed to create completions", response) + } + + return response +} + +/** Aggregates the streaming upstream response into one legacy completion. */ +export const aggregateProxyCompletionsStream = async ( + payload: CompletionsPayload, +): Promise => { + const response = await createProxyCompletions(payload) + + let id = "" + // n>1 requests stream every choice independently by index; keep them + // separate so non-streaming clients receive all completions + const byIndex = new Map< + number, + { text: string; finishReason: string | null } + >() + + for await (const event of events(response)) { + if (!event.data) continue + if (event.data === "[DO" + "NE]") break + const chunk = JSON.parse(event.data) as LegacyCompletionChunk + if (chunk.id) id = chunk.id + for (const choice of chunk.choices ?? []) { + const index = typeof choice.index === "number" ? choice.index : 0 + const entry = byIndex.get(index) ?? { text: "", finishReason: null } + if (typeof choice.text === "string") entry.text += choice.text + if (choice.finish_reason) entry.finishReason = choice.finish_reason + byIndex.set(index, entry) + } + } + + const choices = [...byIndex.entries()] + .sort(([a], [b]) => a - b) + .map(([index, entry]) => ({ + text: entry.text, + index, + finish_reason: entry.finishReason ?? "stop", + logprobs: null, + })) + + return { + id, + object: "text_completion", + created: Math.floor(Date.now() / 1000), + model: payload.model, + choices: + choices.length > 0 ? + choices + : [{ text: "", index: 0, finish_reason: "stop", logprobs: null }], + } +} + +let proxyPending: Promise | undefined +// a failed lookup pins the default base only briefly, so a transient +// /copilot_internal/user failure does not stick until restart +const PROXY_FALLBACK_RETRY_MS = 5 * 60 * 1000 +let proxyFallbackAfter = 0 + +const ensureProxyBaseUrl = async (): Promise => { + const cached = state.proxyBaseUrl + if (cached && cached !== DEFAULT_PROXY_BASE_URL) return cached + if (cached && performance.now() < proxyFallbackAfter) return cached + + // share one in-flight /copilot_internal/user lookup across concurrent + // first requests + proxyPending ??= resolveProxyBaseUrl().finally(() => { + proxyPending = undefined + }) + return proxyPending +} + +const resolveProxyBaseUrl = async (): Promise => { + try { + const usage = await getCopilotUsage() + const proxy = usage.endpoints?.proxy + if (proxy) { + state.proxyBaseUrl = proxy + return proxy + } + } catch (error) { + consola.warn("Failed to resolve proxy base URL, using default:", error) + } + + proxyFallbackAfter = performance.now() + PROXY_FALLBACK_RETRY_MS + + state.proxyBaseUrl = DEFAULT_PROXY_BASE_URL + return DEFAULT_PROXY_BASE_URL +} + +/** Test-only: clears single-flight/backoff between tests. */ +export const __resetProxyBaseUrlForTests = (): void => { + proxyPending = undefined + proxyFallbackAfter = 0 +} + +export interface CompletionsPayload { + model: string + prompt: string | Array + max_tokens?: number | null + temperature?: number | null + top_p?: number | null + n?: number | null + stream?: boolean | null + stop?: string | Array | null + suffix?: string | null + [key: string]: unknown +} + +interface LegacyCompletionChunk { + id?: string + model?: string + choices?: Array<{ + text?: string + index: number + finish_reason: string | null + }> +} + +export interface ProxyCompletionResult { + id: string + object: "text_completion" + created: number + model: string + choices: Array<{ + text: string + index: number + finish_reason: string + logprobs: null + }> + usage?: { + prompt_tokens: number + completion_tokens: number + total_tokens: number + } +} diff --git a/src/services/copilot/create-responses.ts b/src/services/copilot/create-responses.ts new file mode 100644 index 0000000000..9956ebc82f --- /dev/null +++ b/src/services/copilot/create-responses.ts @@ -0,0 +1,42 @@ +import { copilotBaseUrl, copilotHeaders } from "~/lib/api-config" +import { HTTPError } from "~/lib/error" +import { state } from "~/lib/state" +import { modelSessionHeaders } from "~/services/copilot/create-model-session" + +// Responses API payloads are forwarded as-is (model, input, instructions, +// tools, stream, ...). The raw upstream Response is returned so streaming +// (SSE) bodies can be piped through untouched. +export interface ResponsesPayload { + model: string + [key: string]: unknown +} + +export interface ResponsesRequestOptions { + /** attach the copilot-vision-request header for image inputs */ + vision?: boolean + /** mirrors the chat path's X-Initiator semantics */ + initiator?: "user" | "agent" +} + +export const createResponses = async ( + payload: ResponsesPayload, + options: ResponsesRequestOptions = {}, +) => { + if (!state.copilotToken) throw new Error("Copilot token not found") + + const sessionHeaders = await modelSessionHeaders(payload.model) + + const response = await fetch(`${copilotBaseUrl(state)}/responses`, { + method: "POST", + headers: { + ...copilotHeaders(state, options.vision ?? false), + "X-Initiator": options.initiator ?? "user", + ...sessionHeaders, + }, + body: JSON.stringify(payload), + }) + + if (!response.ok) throw new HTTPError("Failed to create responses", response) + + return response +} diff --git a/src/services/copilot/get-models.ts b/src/services/copilot/get-models.ts index 3cfa30af0b..b2d33feac9 100644 --- a/src/services/copilot/get-models.ts +++ b/src/services/copilot/get-models.ts @@ -40,12 +40,13 @@ interface ModelCapabilities { } export interface Model { - capabilities: ModelCapabilities + capabilities?: ModelCapabilities id: string model_picker_enabled: boolean name: string object: string preview: boolean + supported_endpoints?: Array vendor: string version: string policy?: { diff --git a/src/services/copilot/responses-adapter.ts b/src/services/copilot/responses-adapter.ts new file mode 100644 index 0000000000..6ff5162721 --- /dev/null +++ b/src/services/copilot/responses-adapter.ts @@ -0,0 +1,966 @@ +/* + * eslint max-lines: both directions of the chat<->responses protocol + * adapter live here on purpose so the pending upstream review sees the + * full conversion contract in one place; split into forward/reverse + * modules after it lands. + */ +/* eslint-disable max-lines */ +import { state } from "~/lib/state" +import { isNullish } from "~/lib/utils" +import { + type ChatCompletionChunk, + type ChatCompletionsPayload, + type ChatCompletionResponse, + type ContentPart, + type Message, + type TextPart, + type Tool, + type ToolCall, +} from "~/services/copilot/create-chat-completions" + +/* + * Protocol conversion: some CAPI models (gpt-5.3-codex, gpt-5.4-mini, ...) + * only serve the Responses API. This adapter lets plain Chat Completions + * clients talk to them by translating requests/responses on the fly, + * so the rest of the proxy (chat & anthropic-compatible routes, streaming) + * works unchanged. + */ + +export const isResponsesOnlyModel = (model: string): boolean => { + const endpoints = state.models?.data.find( + (entry) => entry.id === model, + )?.supported_endpoints + if (!endpoints || endpoints.length === 0) return false + return ( + endpoints.includes("/responses") && !endpoints.includes("/chat/completions") + ) +} + +/** + * Whether a model can be called on the upstream /responses endpoint. + * Models without endpoint metadata (gpt-4.1, gpt-4o, ...) are served by + * chat only — the upstream rejects them on /responses with + * `unsupported_api_for_model`. + */ +export const isResponsesCapableModel = (model: string): boolean => { + const endpoints = state.models?.data.find( + (entry) => entry.id === model, + )?.supported_endpoints + if (!endpoints || endpoints.length === 0) return false + return endpoints.includes("/responses") +} + +/** + * Derives vision / X-Initiator hints from a Responses payload so the + * passthrough path mirrors the chat path's header semantics. + */ +export const responsesPayloadHints = ( + payload: Record, +): { vision: boolean; initiator: "user" | "agent" } => { + let vision = false + let agent = false + + if (Array.isArray(payload.input)) { + for (const item of payload.input as Array>) { + if ( + item.type === "function_call" + || item.type === "function_call_output" + || item.role === "assistant" + ) { + agent = true + } + if (contentHasImage(item.content)) vision = true + } + } + + return { vision, initiator: agent ? "agent" : "user" } +} + +const contentHasImage = (content: unknown): boolean => { + if (!Array.isArray(content)) return false + return (content as Array>).some( + (part) => part.type === "input_image", + ) +} + +// ---------- request: chat completions -> responses ---------- + +export const chatPayloadToResponsesPayload = ( + payload: ChatCompletionsPayload, +): Record => { + const { instructions, input } = messagesToResponsesInput(payload.messages) + + const result: Record = { + model: payload.model, + input, + temperature: payload.temperature ?? undefined, + top_p: payload.top_p ?? undefined, + stream: payload.stream ?? undefined, + } + + if (instructions) result.instructions = instructions + + const maxTokens = payload.max_tokens ?? payload.max_completion_tokens + if (!isNullish(maxTokens)) result.max_output_tokens = maxTokens + + // chat's JSON mode maps to the Responses text.format constraint + if (payload.response_format?.type === "json_object") { + result.text = { format: { type: "json_object" } } + } + + if (payload.tools?.length) { + result.tools = payload.tools.map((tool) => chatToolToResponsesTool(tool)) + if (!isNullish(payload.tool_choice)) { + // string values ("none" | "auto" | "required") are identical in both + // APIs; the object form differs: {function:{name}} -> {name} + const choice = payload.tool_choice + result.tool_choice = + typeof choice === "string" ? choice : ( + { type: "function", name: choice.function.name } + ) + } + } + + return result +} + +interface InputBuild { + instructions: Array + input: Array> +} + +const messagesToResponsesInput = ( + messages: Array, +): { instructions?: string; input: Array> } => { + const build: InputBuild = { instructions: [], input: [] } + for (const message of messages) appendMessage(build, message) + return { + instructions: build.instructions.join("\n\n") || undefined, + input: build.input, + } +} + +const appendMessage = (build: InputBuild, message: Message) => { + const text = messageTextContent(message.content) + const parts = Array.isArray(message.content) ? message.content : undefined + + // system/developer messages become top-level instructions + if (message.role === "system" || message.role === "developer") { + if (text) build.instructions.push(text) + return + } + + // tool results map to function_call_output items + if (message.role === "tool") { + build.input.push({ + type: "function_call_output", + call_id: message.tool_call_id ?? "", + output: text, + }) + return + } + + if (message.role === "assistant" && message.tool_calls?.length) { + pushAssistantToolCalls(build.input, message) + return + } + + build.input.push(toResponsesItem(message.role, text, parts)) +} + +const pushAssistantToolCalls = ( + input: Array>, + message: Message, +) => { + const text = messageTextContent(message.content) + const parts = Array.isArray(message.content) ? message.content : undefined + if (text) input.push(toResponsesItem("assistant", text, parts)) + for (const call of message.tool_calls ?? []) { + input.push({ + type: "function_call", + call_id: call.id, + name: call.function.name, + arguments: call.function.arguments, + }) + } +} + +const toResponsesItem = ( + role: Message["role"], + text: string, + parts?: Message["content"], +): Record => { + if (Array.isArray(parts)) { + const content = parts.map((part) => { + if (part.type === "image_url") { + return { type: "input_image", image_url: part.image_url.url } + } + return { + type: role === "assistant" ? "output_text" : "input_text", + text: part.text, + } + }) + return { role, content } + } + return { role, content: text } +} + +const messageTextContent = (content: Message["content"]): string => { + if (content === null) return "" + if (typeof content === "string") return content + return content + .filter((part): part is TextPart => part.type === "text") + .map((part) => part.text) + .join("\n") +} + +const chatToolToResponsesTool = (tool: Tool): Record => ({ + type: "function", + name: tool.function.name, + description: tool.function.description ?? null, + parameters: tool.function.parameters, +}) + +// ---------- response: responses -> chat completions (non-streaming) ---------- + +export interface ResponsesResult { + id?: string + model?: string + status?: string + created_at?: number + output?: Array + usage?: ResponsesUsage + error?: { message?: string } +} + +interface ResponsesOutputItem { + type?: string + id?: string + role?: string + content?: Array<{ type?: string; text?: string }> + call_id?: string + name?: string + arguments?: string +} + +interface ResponsesUsage { + input_tokens?: number + output_tokens?: number + total_tokens?: number +} + +export const responsesResultToChatCompletion = ( + result: ResponsesResult, + model: string, +): ChatCompletionResponse => { + if (result.status === "failed" || result.status === "cancelled") { + throw new Error( + result.error?.message ?? `Responses request ${result.status}`, + ) + } + + const { text, toolCalls } = responseItemsToMessage(result.output) + + const baseFinishReason = result.status === "incomplete" ? "length" : "stop" + + const data: ChatCompletionResponse = { + id: result.id ?? "", + object: "chat.completion", + created: result.created_at ?? Math.floor(Date.now() / 1000), + model: result.model ?? model, + choices: [ + { + index: 0, + message: { + role: "assistant", + content: text || null, + ...(toolCalls.length > 0 ? { tool_calls: toolCalls } : {}), + }, + logprobs: null, + finish_reason: toolCalls.length > 0 ? "tool_calls" : baseFinishReason, + }, + ], + } + + data.usage = responsesUsageToChatUsage(result.usage) ?? data.usage + + return data +} + +const responseItemsToMessage = ( + items?: Array, +): { text: string; toolCalls: Array } => { + let text = "" + const toolCalls: Array = [] + + for (const item of items ?? []) { + if (item.type === "message") { + text += messageItemText(item) + } else if (item.type === "function_call") { + toolCalls.push({ + id: item.call_id ?? item.id ?? `call_${toolCalls.length}`, + type: "function", + function: { name: item.name ?? "", arguments: item.arguments ?? "" }, + }) + } + } + + return { text, toolCalls } +} + +const messageItemText = (item: ResponsesOutputItem): string => { + let text = "" + for (const part of item.content ?? []) { + if (part.type === "output_text" && typeof part.text === "string") { + text += part.text + } + } + return text +} + +const responsesUsageToChatUsage = ( + usage?: ResponsesUsage, +): ChatCompletionResponse["usage"] | undefined => { + if (!usage) return undefined + + const inputTokens = usage.input_tokens ?? 0 + const outputTokens = usage.output_tokens ?? 0 + return { + prompt_tokens: inputTokens, + completion_tokens: outputTokens, + total_tokens: usage.total_tokens ?? inputTokens + outputTokens, + } +} + +// ---------- response: responses -> chat completions (streaming) ---------- + +interface ChunkMeta { + id: string + created: number + model: string +} + +interface ChunkOptions { + delta: ChatCompletionChunk["choices"][0]["delta"] + finish_reason: ChatCompletionChunk["choices"][0]["finish_reason"] + usage?: unknown +} + +const makeChunk = ( + meta: ChunkMeta, + options: ChunkOptions, +): ChatCompletionChunk => { + const chunk: ChatCompletionChunk = { + id: meta.id, + object: "chat.completion.chunk", + created: meta.created, + model: meta.model, + choices: [ + { + index: 0, + delta: options.delta, + finish_reason: options.finish_reason, + logprobs: null, + }, + ], + } + if (options.usage !== undefined) { + chunk.usage = options.usage as ChatCompletionChunk["usage"] + } + return chunk +} + +const toSseEvent = (chunk: ChatCompletionChunk) => ({ + data: JSON.stringify(chunk), +}) + +interface EventOutcome { + chunks: Array + done: boolean + emittedRole: boolean +} + +const responsesEventToChunks = ( + parsed: { + type?: string + response?: ResponsesResult + delta?: string + }, + meta: ChunkMeta, + emittedRole: boolean, +): EventOutcome | undefined => { + if (parsed.type === "response.created") { + return { + chunks: [ + makeChunk( + { + id: parsed.response?.id ?? meta.id, + created: parsed.response?.created_at ?? meta.created, + model: meta.model, + }, + { delta: { role: "assistant" }, finish_reason: null }, + ), + ], + done: false, + emittedRole: true, + } + } + + if (parsed.type === "response.output_text.delta") { + const chunks: Array = [] + if (!emittedRole) { + chunks.push( + makeChunk(meta, { delta: { role: "assistant" }, finish_reason: null }), + ) + } + chunks.push( + makeChunk(meta, { + delta: { content: parsed.delta ?? "" }, + finish_reason: null, + }), + ) + return { chunks, done: false, emittedRole: true } + } + + if (parsed.type === "response.completed") { + return completedEventToChunks(parsed.response ?? {}, meta) + } + + if (parsed.type === "response.failed" || parsed.type === "error") { + // propagate upstream failures: emitting a content_filter chunk would + // make clients treat a failed request as a completed one + throw new Error( + parsed.response?.error?.message ?? "Responses stream failed", + ) + } + + return undefined +} + +const completedEventToChunks = ( + result: ResponsesResult, + meta: ChunkMeta, +): EventOutcome => { + const chatResult = responsesResultToChatCompletion(result, meta.model) + const message = chatResult.choices[0].message + const chunks: Array = [] + + let finishReason: ChatCompletionChunk["choices"][0]["finish_reason"] = + result.status === "incomplete" ? "length" : "stop" + if (message.tool_calls?.length) { + finishReason = "tool_calls" + chunks.push( + makeChunk(meta, { + delta: { + tool_calls: message.tool_calls.map((call, index) => ({ + index, + id: call.id, + type: call.type, + function: call.function, + })), + }, + finish_reason: null, + }), + ) + } + + chunks.push( + makeChunk(meta, { + delta: {}, + finish_reason: finishReason, + usage: chatResult.usage, + }), + ) + + return { chunks, done: true, emittedRole: true } +} + +export async function* streamResponsesAsChatChunks( + upstream: Response, + model: string, + events: (response: Response) => AsyncIterable<{ data?: string | null }>, +): AsyncGenerator<{ data: string }> { + const meta: ChunkMeta = { + id: "", + created: Math.floor(Date.now() / 1000), + model, + } + let emittedRole = false + + for await (const event of events(upstream)) { + if (!event.data) continue + if (event.data === "[DONE]") continue + const parsed = JSON.parse(event.data) as { + type?: string + response?: ResponsesResult + delta?: string + } + + if (parsed.type === "response.created") { + meta.id = parsed.response?.id ?? "" + meta.created = parsed.response?.created_at ?? meta.created + } + + const outcome = responsesEventToChunks(parsed, meta, emittedRole) + if (!outcome) continue + + emittedRole = outcome.emittedRole + for (const chunk of outcome.chunks) yield toSseEvent(chunk) + if (outcome.done) { + // mirror the chat route's terminator so waiting clients wrap up + yield { data: "[DO" + "NE]" } + return + } + } + + // upstream ended without a completed event + yield toSseEvent(makeChunk(meta, { delta: {}, finish_reason: "stop" })) + yield { data: "[DO" + "NE]" } +} + +// ---------- reverse direction: responses -> chat completions ---------- +// Lets Responses API clients (e.g. Codex CLI) use chat-only models +// (gpt-4.1, exec-agent-*, copilot-search-*, ...) through /v1/responses. + +interface ResponsesInputItem { + type?: string + role?: string + content?: unknown + call_id?: string + name?: string + arguments?: string + output?: string +} + +export const responsesPayloadToChatPayload = ( + payload: Record, +): ChatCompletionsPayload => { + const messages: Array = [] + + const instructions = payload.instructions + if (typeof instructions === "string" && instructions.length > 0) { + messages.push({ role: "system", content: instructions }) + } + + if (typeof payload.input === "string") { + messages.push({ role: "user", content: payload.input }) + } else if (Array.isArray(payload.input)) { + for (const item of payload.input as Array) { + appendResponsesItemToMessages(messages, item) + } + } + + const result: ChatCompletionsPayload = { + model: String(payload.model), + messages, + } + + if (typeof payload.max_output_tokens === "number") { + result.max_tokens = payload.max_output_tokens + } + if (typeof payload.temperature === "number") { + result.temperature = payload.temperature + } + if (typeof payload.top_p === "number") { + result.top_p = payload.top_p + } + if (payload.stream === true) result.stream = true + + const responseFormat = responsesTextToResponseFormat(payload.text) + if (responseFormat) result.response_format = responseFormat + const tools = responsesToolsToChatTools(payload.tools) + if (tools) { + result.tools = tools + // string values ("none" | "auto" | "required") are identical in both + // APIs; the object form differs: {name} -> {function:{name}} + if (!isNullish(payload.tool_choice)) { + const choice = payload.tool_choice + if (typeof choice === "string") { + result.tool_choice = choice as ChatCompletionsPayload["tool_choice"] + } else if (typeof (choice as { name?: unknown }).name === "string") { + result.tool_choice = { + type: "function", + function: { name: (choice as { name: string }).name }, + } + } + } + } + + return result +} + +// Responses text.format maps back to chat's JSON mode +const responsesTextToResponseFormat = ( + text: unknown, +): ChatCompletionsPayload["response_format"] | undefined => { + if ( + typeof text === "object" + && text !== null + && (text as { format?: { type?: unknown } }).format?.type === "json_object" + ) { + return { type: "json_object" } + } + return undefined +} + +const appendResponsesItemToMessages = ( + messages: Array, + item: ResponsesInputItem, +) => { + if (item.type === "function_call") { + // parallel tool calls arrive as consecutive function_call items and must + // share one assistant message: chat requires all tool results of an + // assistant message to follow it directly + const call: ToolCall = { + id: item.call_id ?? `call_${messages.length}`, + type: "function", + function: { name: item.name ?? "", arguments: item.arguments ?? "" }, + } + const last = messages.at(-1) + if (last?.role === "assistant") { + last.tool_calls = [...(last.tool_calls ?? []), call] + } else { + messages.push({ role: "assistant", content: null, tool_calls: [call] }) + } + return + } + + if (item.type === "function_call_output") { + messages.push({ + role: "tool", + content: item.output ?? "", + tool_call_id: item.call_id ?? "", + }) + return + } + + const role = responsesRoleToChatRole(item.role) + if (!role) return + messages.push({ role, content: responsesContentToChatContent(item.content) }) +} + +const responsesRoleToChatRole = ( + role?: string, +): Message["role"] | undefined => { + if (role === "user" || role === "assistant" || role === "tool") return role + if (role === "system" || role === "developer") return "system" + return undefined +} + +const responsesContentToChatContent = ( + content: unknown, +): Message["content"] => { + if (content === null || content === undefined) return "" + if (typeof content === "string") return content + if (!Array.isArray(content)) return "" + + const parts: Array = [] + for (const part of content as Array>) { + if (part.type === "input_image" && typeof part.image_url === "string") { + parts.push({ type: "image_url", image_url: { url: part.image_url } }) + } else if (typeof part.text === "string") { + parts.push({ type: "text", text: part.text }) + } + } + return parts.length > 0 ? parts : "" +} + +const responsesToolsToChatTools = (tools: unknown): Array | undefined => { + if (!Array.isArray(tools)) return undefined + // only function tools have a chat equivalent; built-in tools like + // web_search or local_shell must not become fake "undefined" functions + const converted = (tools as Array>) + .filter( + (record) => record.type === "function" && typeof record.name === "string", + ) + .map((record) => ({ + type: "function" as const, + function: { + name: record.name as string, + description: + typeof record.description === "string" ? + record.description + : undefined, + parameters: (record.parameters ?? {}) as Record, + }, + })) + return converted.length > 0 ? converted : undefined +} + +export const chatResultToResponsesResult = ( + chat: ChatCompletionResponse, +): Record => { + const choice = chat.choices[0] + const toolCalls = choice.message.tool_calls ?? [] + + const output: Array> = [ + { + type: "message", + id: `msg_${chat.id}`, + role: "assistant", + content: [{ type: "output_text", text: choice.message.content ?? "" }], + }, + ] + for (const call of toolCalls) { + output.push({ + type: "function_call", + call_id: call.id, + name: call.function.name, + arguments: call.function.arguments, + }) + } + + return { + id: chat.id, + object: "response", + created_at: chat.created, + model: chat.model, + status: chatStatusFromFinishReason(choice.finish_reason), + output, + usage: + chat.usage ? + { + input_tokens: chat.usage.prompt_tokens, + output_tokens: chat.usage.completion_tokens, + total_tokens: chat.usage.total_tokens, + } + : undefined, + } +} + +// Responses clients detect max-token truncation via status "incomplete" +const chatStatusFromFinishReason = (finishReason?: string | null): string => + finishReason === "length" ? "incomplete" : "completed" + +interface ChatStreamState { + responseId: string + created: number + text: string + finishReason: string | null + usage: ChatCompletionResponse["usage"] + toolCalls: Array +} + +const emit = (event: Record) => ({ + data: JSON.stringify(event), +}) + +const initialResponsesEvents = ( + state: ChatStreamState, + model: string, +): Array> => [ + { + type: "response.created", + response: { + id: state.responseId, + created_at: state.created, + model, + status: "in_progress", + }, + }, + { + type: "response.output_item.added", + output_index: 0, + item: { type: "message", role: "assistant", content: [] }, + }, +] + +const collectToolCalls = ( + choice: ChatCompletionChunk["choices"][0], + toolCalls: Array, +) => { + for (const call of choice.delta.tool_calls ?? []) { + // a single tool call is streamed as several deltas sharing one index: + // the first carries id/name, the following append argument fragments + const existing = + call.index < toolCalls.length ? toolCalls[call.index] : undefined + if (existing) { + if (call.id) existing.id = call.id + if (call.function?.name) existing.function.name = call.function.name + if (call.function?.arguments) { + existing.function.arguments += call.function.arguments + } + continue + } + + while (toolCalls.length < call.index) { + toolCalls.push({ + id: `call_${toolCalls.length}`, + type: "function", + function: { name: "", arguments: "" }, + }) + } + toolCalls.push({ + id: call.id ?? `call_${call.index}`, + type: "function", + function: { + name: call.function?.name ?? "", + arguments: call.function?.arguments ?? "", + }, + }) + } +} + +const responsesOutputItems = ( + state: ChatStreamState, +): Array> => { + const output: Array> = [ + { + type: "message", + role: "assistant", + content: [{ type: "output_text", text: state.text }], + }, + ] + for (const call of state.toolCalls) { + output.push({ + type: "function_call", + call_id: call.id, + name: call.function.name, + arguments: call.function.arguments, + }) + } + return output +} + +const finalResponsesEvents = ( + state: ChatStreamState, + model: string, +): Array> => { + const events: Array> = [ + { + type: "response.output_text.done", + output_index: 0, + content_index: 0, + text: state.text, + }, + { + type: "response.output_item.done", + output_index: 0, + item: { + type: "message", + role: "assistant", + content: [{ type: "output_text", text: state.text }], + }, + }, + ] + + // chat streams only finish tool calls by the end, so their Responses item + // events are emitted here — before the terminal event — because Responses + // clients deliver function calls through output items, not only through + // the completed event's output array + let outputIndex = 1 + for (const call of state.toolCalls) { + const itemIndex = outputIndex++ + events.push( + { + type: "response.output_item.added", + output_index: itemIndex, + item: { + type: "function_call", + call_id: call.id, + name: call.function.name, + arguments: "", + }, + }, + { + type: "response.function_call_arguments.delta", + output_index: itemIndex, + delta: call.function.arguments, + }, + { + type: "response.function_call_arguments.done", + output_index: itemIndex, + arguments: call.function.arguments, + }, + { + type: "response.output_item.done", + output_index: itemIndex, + item: { + type: "function_call", + call_id: call.id, + name: call.function.name, + arguments: call.function.arguments, + }, + }, + ) + } + + // a truncated chat stream must terminate as response.incomplete (with + // incomplete_details) so Responses clients do not treat it as complete + const status = chatStatusFromFinishReason(state.finishReason) + events.push({ + type: + status === "incomplete" ? "response.incomplete" : "response.completed", + response: { + id: state.responseId, + created_at: state.created, + model, + status, + ...(status === "incomplete" ? + { incomplete_details: { reason: "max_output_tokens" } } + : {}), + output: responsesOutputItems(state), + usage: + state.usage ? + { + input_tokens: state.usage.prompt_tokens, + output_tokens: state.usage.completion_tokens, + total_tokens: state.usage.total_tokens, + } + : undefined, + finish_reason: state.finishReason ?? "stop", + }, + }) + return events +} + +export async function* streamChatAsResponsesEvents( + chatStream: AsyncIterable<{ data?: string | null }>, + model: string, +): AsyncGenerator<{ data: string }> { + const state: ChatStreamState = { + responseId: "", + created: Math.floor(Date.now() / 1000), + text: "", + finishReason: null, + usage: undefined, + toolCalls: [], + } + let emittedCreated = false + + for await (const event of chatStream) { + if (!event.data) continue + // skip the non-JSON SSE terminator line + if (!event.data.startsWith("{")) break + const chunk = JSON.parse(event.data) as ChatCompletionChunk + + // capture id/usage first: usage-only chunks carry an empty choices array + if (chunk.id) state.responseId = chunk.id + if (chunk.usage) state.usage = chunk.usage + + if (chunk.choices.length === 0) continue + const choice = chunk.choices[0] + + if (!emittedCreated) { + emittedCreated = true + for (const item of initialResponsesEvents(state, model)) yield emit(item) + } + + const content = choice.delta.content + if (typeof content === "string" && content.length > 0) { + state.text += content + yield emit({ + type: "response.output_text.delta", + output_index: 0, + content_index: 0, + delta: content, + }) + } + + collectToolCalls(choice, state.toolCalls) + + if (choice.finish_reason) state.finishReason = choice.finish_reason + } + + for (const item of finalResponsesEvents(state, model)) yield emit(item) +} diff --git a/src/services/github/get-copilot-usage.ts b/src/services/github/get-copilot-usage.ts index 6cdd8bc107..d51d55943d 100644 --- a/src/services/github/get-copilot-usage.ts +++ b/src/services/github/get-copilot-usage.ts @@ -38,6 +38,7 @@ interface CopilotUsageResponse { can_signup_for_limited: boolean chat_enabled: boolean copilot_plan: string + endpoints?: { proxy?: string; [key: string]: string | undefined } organization_login_list: Array organization_list: Array quota_reset_date: string diff --git a/src/start.ts b/src/start.ts index 14abbbdffb..ccd385b262 100644 --- a/src/start.ts +++ b/src/start.ts @@ -16,6 +16,7 @@ import { server } from "./server" interface RunServerOptions { port: number + hostname: string verbose: boolean accountType: string manual: boolean @@ -117,6 +118,7 @@ export async function runServer(options: RunServerOptions): Promise { serve({ fetch: server.fetch as ServerHandler, port: options.port, + hostname: options.hostname, }) } @@ -132,6 +134,12 @@ export const start = defineCommand({ default: "4141", description: "Port to listen on", }, + hostname: { + type: "string", + default: "0.0.0.0", + description: + "Hostname/interface to listen on (use 127.0.0.1 for local-only access)", + }, verbose: { alias: "v", type: "boolean", @@ -193,6 +201,7 @@ export const start = defineCommand({ return runServer({ port: Number.parseInt(args.port, 10), + hostname: args.hostname, verbose: args.verbose, accountType: args["account-type"], manual: args.manual, diff --git a/tests/chat-completions.test.ts b/tests/chat-completions.test.ts new file mode 100644 index 0000000000..27501e315b --- /dev/null +++ b/tests/chat-completions.test.ts @@ -0,0 +1,125 @@ +import type { Context } from "hono" + +import { test, expect, mock, beforeEach } from "bun:test" + +import type { Model } from "../src/services/copilot/get-models" + +import { state } from "../src/lib/state" +import { handleCompletion } from "../src/routes/chat-completions/handler" + +state.copilotToken = "test-token" +state.vsCodeVersion = "1.0.0" +state.accountType = "individual" + +const baseModel: Model = { + id: "gpt-test", + name: "gpt-test", + object: "model", + version: "1", + vendor: "openai", + preview: false, + model_picker_enabled: true, + capabilities: { + family: "gpt", + object: "capabilities", + type: "chat", + tokenizer: "o200k_base", + limits: { max_output_tokens: 4096 }, + supports: {}, + }, +} + +// The real /models endpoint lists some entries WITHOUT capabilities, +// e.g. "gpt-41-copilot" or "trajectory-compaction" +const ghostModel: Model = { + id: "gpt-ghost", + name: "gpt-ghost", + object: "model", + version: "1", + vendor: "openai", + preview: false, + model_picker_enabled: true, +} + +interface CapturedRequestInit { + body?: string +} + +const fetchMock = mock((_url: string, _opts?: CapturedRequestInit) => ({ + ok: true, + json: () => + Promise.resolve({ id: "123", object: "chat.completion", choices: [] }), +})) +// @ts-expect-error - Mock fetch doesn't implement all fetch properties +;(globalThis as unknown as { fetch: typeof fetch }).fetch = fetchMock + +const lastForwardedPayload = (): Record => { + const calls = fetchMock.mock.calls as Array<[string, CapturedRequestInit]> + const call = calls.at(-1) + if (!call) throw new Error("expected fetch to have been called") + return JSON.parse(call[1].body as string) as Record +} + +const makeContext = (payload: Record) => + ({ + req: { json: () => Promise.resolve(payload) }, + json: (data: unknown) => data, + }) as unknown as Context + +beforeEach(() => { + fetchMock.mockClear() + state.models = { object: "list", data: [baseModel, ghostModel] } + // Pre-seed a valid session so no /models/session fetch happens here + state.modelSession = { + token: "test-session", + availableModels: [], + expiresAt: Date.now() + 60_000, + } +}) + +test("forwards max_completion_tokens without injecting max_tokens", async () => { + await handleCompletion( + makeContext({ + model: "gpt-test", + messages: [{ role: "user", content: "hi" }], + max_completion_tokens: 128, + }), + ) + const sent = lastForwardedPayload() + expect(sent.max_completion_tokens).toBe(128) + expect("max_tokens" in sent).toBe(false) +}) + +test("does not crash for a model without capabilities", async () => { + await handleCompletion( + makeContext({ + model: "gpt-ghost", + messages: [{ role: "user", content: "hi" }], + }), + ) + const sent = lastForwardedPayload() + expect("max_tokens" in sent).toBe(false) +}) + +test("still injects max_tokens from model limits when neither parameter is given", async () => { + await handleCompletion( + makeContext({ + model: "gpt-test", + messages: [{ role: "user", content: "hi" }], + }), + ) + const sent = lastForwardedPayload() + expect(sent.max_tokens).toBe(4096) +}) + +test("explicitly-null max_completion_tokens still blocks max_tokens injection", async () => { + await handleCompletion( + makeContext({ + model: "gpt-test", + messages: [{ role: "user", content: "hi" }], + max_completion_tokens: null, + }), + ) + const sent = lastForwardedPayload() + expect("max_tokens" in sent).toBe(false) +}) diff --git a/tests/completions-adapter.test.ts b/tests/completions-adapter.test.ts new file mode 100644 index 0000000000..24c1877161 --- /dev/null +++ b/tests/completions-adapter.test.ts @@ -0,0 +1,66 @@ +import { expect, test } from "bun:test" + +import type { ChatCompletionResponse } from "../src/services/copilot/create-chat-completions" + +import { + chatResultToCompletionResult, + completionPayloadToChatPayload, +} from "../src/services/copilot/completions-adapter" + +const chatResult = ( + overrides: Partial = {}, +): ChatCompletionResponse => ({ + id: "chatcmpl-1", + object: "chat.completion", + created: 123, + model: "gpt-4.1", + choices: [ + { + index: 0, + message: { role: "assistant", content: "one" }, + logprobs: null, + finish_reason: "stop", + }, + { + index: 1, + message: { role: "assistant", content: "two" }, + logprobs: null, + finish_reason: "stop", + }, + ], + usage: { prompt_tokens: 3, completion_tokens: 2, total_tokens: 5 }, + ...overrides, +}) + +test("forwards n and maps every choice with its original index", () => { + const chatPayload = completionPayloadToChatPayload({ + model: "gpt-4.1", + prompt: "hi", + n: 2, + }) + expect(chatPayload.n).toBe(2) + + const result = chatResultToCompletionResult(chatResult()) + expect(result.choices).toEqual([ + { text: "one", index: 0, finish_reason: "stop", logprobs: null }, + { text: "two", index: 1, finish_reason: "stop", logprobs: null }, + ]) +}) + +test("carries usage into legacy completion results", () => { + const result = chatResultToCompletionResult(chatResult()) + expect(result.usage).toEqual({ + prompt_tokens: 3, + completion_tokens: 2, + total_tokens: 5, + }) +}) + +test("rejects array prompts for chat translation", () => { + expect(() => + completionPayloadToChatPayload({ + model: "gpt-4.1", + prompt: ["first", "second"], + }), + ).toThrow("string prompt") +}) diff --git a/tests/create-chat-completions.test.ts b/tests/create-chat-completions.test.ts index d18e741aa6..9c91bc2a21 100644 --- a/tests/create-chat-completions.test.ts +++ b/tests/create-chat-completions.test.ts @@ -9,6 +9,13 @@ import { createChatCompletions } from "../src/services/copilot/create-chat-compl state.copilotToken = "test-token" state.vsCodeVersion = "1.0.0" state.accountType = "individual" +// Pre-seed a valid session so createChatCompletions does not hit +// the /models/session endpoint in these tests +state.modelSession = { + token: "test-session", + availableModels: [], + expiresAt: Date.now() + 60_000, +} // Helper to mock fetch const fetchMock = mock( diff --git a/tests/create-proxy-completions.test.ts b/tests/create-proxy-completions.test.ts new file mode 100644 index 0000000000..50f5f02b8a --- /dev/null +++ b/tests/create-proxy-completions.test.ts @@ -0,0 +1,60 @@ +import { expect, mock, test } from "bun:test" + +import { state } from "../src/lib/state" +import { aggregateProxyCompletionsStream } from "../src/services/copilot/create-proxy-completions" + +state.copilotToken = "test-token" +state.vsCodeVersion = "1.0.0" +state.accountType = "individual" +// pre-seed the proxy base URL so no /copilot_internal/user lookup happens +state.proxyBaseUrl = "https://proxy.example.test" + +const sseChunk = (choices: Array>) => + `data: ${JSON.stringify({ id: "cmpl-1", choices })}\n\n` + +test("aggregates n>1 proxy completions per choice index", async () => { + const body = + sseChunk([ + { text: "hel", index: 0, finish_reason: null }, + { text: "wor", index: 1, finish_reason: null }, + ]) + + sseChunk([ + { text: "lo", index: 0, finish_reason: null }, + { text: "ld", index: 1, finish_reason: null }, + ]) + + sseChunk([ + { text: "", index: 0, finish_reason: "stop" }, + { text: "", index: 1, finish_reason: "length" }, + ]) + + "data: [DO" + + "NE]\n\n" + + const fetchMock = mock( + (_url: string, _opts?: unknown) => + new Response(body, { + status: 200, + headers: { "content-type": "text/event-stream" }, + }), + ) + // @ts-expect-error - Mock fetch doesn't implement all fetch properties + ;(globalThis as unknown as { fetch: typeof fetch }).fetch = fetchMock + + const result = await aggregateProxyCompletionsStream({ + model: "gpt-41-copilot", + prompt: "x", + n: 2, + }) + + expect(result.choices).toHaveLength(2) + expect(result.choices[0]).toMatchObject({ + text: "hello", + index: 0, + finish_reason: "stop", + }) + expect(result.choices[1]).toMatchObject({ + text: "world", + index: 1, + finish_reason: "length", + }) + expect(result.id).toBe("cmpl-1") +}) diff --git a/tests/extended-responses-adapter.test.ts b/tests/extended-responses-adapter.test.ts new file mode 100644 index 0000000000..f79e3a7787 --- /dev/null +++ b/tests/extended-responses-adapter.test.ts @@ -0,0 +1,139 @@ +import { expect, test } from "bun:test" +import { events } from "fetch-event-stream" + +import type { ChatCompletionChunk } from "../src/services/copilot/create-chat-completions" + +import { + chatPayloadToResponsesPayload, + streamChatAsResponsesEvents, +} from "../src/services/copilot/responses-adapter" + +const chatChunk = (delta: Record, finish: string | null) => ({ + data: JSON.stringify({ + id: "c2", + object: "chat.completion.chunk", + created: 1, + model: "gpt-4.1", + choices: [ + { index: 0, delta, finish_reason: finish, logprobs: null }, + ] as Array, + }), +}) + +const streamFrom = ( + chunks: Array<{ data: string }>, +): AsyncIterable<{ data?: string | null }> => { + const body = + chunks.map((e) => `data: ${e.data}\n\n`).join("") + "data: [DO" + "NE]\n\n" + return events( + new Response(body, { + headers: { "content-type": "text/event-stream" }, + }), + ) +} + +const collect = async ( + chunks: Array<{ data: string }>, +): Promise>> => { + const out: Array> = [] + for await (const event of streamChatAsResponsesEvents( + streamFrom(chunks), + "gpt-4.1", + )) { + out.push(JSON.parse(event.data) as Record) + } + return out +} + +test("maps json response_format to responses text.format", () => { + const payload = chatPayloadToResponsesPayload({ + model: "gpt-5.3-codex", + messages: [{ role: "user", content: "hi" }], + response_format: { type: "json_object" }, + }) + expect(payload["text"]).toEqual({ format: { type: "json_object" } }) +}) + +test("emits response.incomplete terminal for a length-truncated stream", async () => { + const out = await collect([ + chatChunk({ role: "assistant" }, null), + chatChunk({ content: "cut" }, null), + chatChunk({}, "length"), + ]) + const terminal = out.at(-1) as { + type?: string + response?: { + status?: string + incomplete_details?: { reason?: string } + } + } + expect(terminal.type).toBe("response.incomplete") + expect(terminal.response?.status).toBe("incomplete") + expect(terminal.response?.incomplete_details?.reason).toBe( + "max_output_tokens", + ) +}) + +test("emits function-call item events before the terminal event", async () => { + const out = await collect([ + chatChunk({ role: "assistant" }, null), + chatChunk( + { + tool_calls: [ + { + index: 0, + id: "call_1", + type: "function", + function: { name: "greet", arguments: "" }, + }, + ], + }, + null, + ), + chatChunk( + { + tool_calls: [{ index: 0, function: { arguments: '{"msg":"hi"}' } }], + }, + null, + ), + chatChunk({}, "tool_calls"), + ]) + + const types = out.map((e) => e["type"]) + const terminalIndex = types.indexOf("response.completed") + expect(terminalIndex).toBeGreaterThan(0) + + const added = out.find( + (e) => + e["type"] === "response.output_item.added" + && (e["item"] as { type?: string } | undefined)?.type === "function_call", + ) + expect(added).toBeDefined() + const addedIndex = out.indexOf(added as Record) + expect(addedIndex).toBeLessThan(terminalIndex) + + const delta = out.find( + (e) => e["type"] === "response.function_call_arguments.delta", + ) as { delta?: string } | undefined + expect(delta?.delta).toBe('{"msg":"hi"}') + + const done = out.find( + (e) => e["type"] === "response.function_call_arguments.done", + ) as { arguments?: string } | undefined + expect(done?.arguments).toBe('{"msg":"hi"}') + + const outDone = out.find( + (e) => + e["type"] === "response.output_item.done" + && (e["item"] as { type?: string } | undefined)?.type === "function_call", + ) + expect(outDone).toBeDefined() + + const terminal = out[terminalIndex] as { + response?: { status?: string; output?: Array<{ type?: string }> } + } + expect(terminal.response?.status).toBe("completed") + expect( + terminal.response?.output?.some((o) => o.type === "function_call"), + ).toBe(true) +}) diff --git a/tests/model-session.test.ts b/tests/model-session.test.ts new file mode 100644 index 0000000000..410bf8ad35 --- /dev/null +++ b/tests/model-session.test.ts @@ -0,0 +1,148 @@ +import { beforeEach, expect, mock, test } from "bun:test" + +import type { Model } from "../src/services/copilot/get-models" + +import { state } from "../src/lib/state" +import { + createChatCompletions, + type ChatCompletionsPayload, +} from "../src/services/copilot/create-chat-completions" +import { + __resetModelSessionForTests, + ensureModelSession, + modelSessionHeaders, +} from "../src/services/copilot/create-model-session" + +state.copilotToken = "test-token" +state.vsCodeVersion = "1.0.0" +state.accountType = "individual" + +interface RecordedCall { + url: string + headers: Record + body?: string +} + +const calls: Array = [] + +const fetchMock = mock((url: string, opts: RecordedCall) => { + calls.push({ url, headers: opts.headers, body: opts.body }) + if (url.endsWith("/models/session")) { + return { + ok: true, + json: () => + Promise.resolve({ + available_models: ["claude-haiku-4.5", "gpt-5-mini"], + selected_model: "claude-haiku-4.5", + session_token: "sess-token-123", + }), + } + } + return { + ok: true, + json: () => + Promise.resolve({ id: "123", object: "chat.completion", choices: [] }), + } +}) +// @ts-expect-error - Mock fetch doesn't implement all fetch properties +;(globalThis as unknown as { fetch: typeof fetch }).fetch = fetchMock + +beforeEach(() => { + calls.length = 0 + fetchMock.mockClear() + state.modelSession = undefined + state.models = undefined + state.copilotToken = "test-token" + __resetModelSessionForTests() +}) + +test("creates a session and caches it", async () => { + const first = await ensureModelSession() + const second = await ensureModelSession() + expect(first?.token).toBe("sess-token-123") + expect(second?.token).toBe("sess-token-123") + expect(calls.filter((c) => c.url.endsWith("/models/session"))).toHaveLength(1) +}) + +test("attaches the session header only for pooled models", async () => { + const pooled = await modelSessionHeaders("claude-haiku-4.5") + expect(pooled["Copilot-Session-Token"]).toBe("sess-token-123") + + const unpooled = await modelSessionHeaders("gpt-4.1") + expect(unpooled).toEqual({}) +}) + +test("refreshes an expired session", async () => { + await ensureModelSession() + const session = state.modelSession + if (!session) throw new Error("expected a cached session") + session.expiresAt = Date.now() - 1 + + const refreshed = await ensureModelSession() + expect(refreshed?.token).toBe("sess-token-123") + expect(calls.filter((c) => c.url.endsWith("/models/session"))).toHaveLength(2) +}) + +const makeModel = (id: string): Model => ({ + id, + name: id, + object: "model", + version: "1", + vendor: "test", + preview: false, + model_picker_enabled: true, +}) + +test("hints every known model to maximize the session pool", async () => { + state.models = { + object: "list", + data: [makeModel("gpt-4.1"), makeModel("claude-haiku-4.5")], + } + + await ensureModelSession() + + const sessionCall = calls.find((c) => c.url.endsWith("/models/session")) + const body = JSON.parse(sessionCall?.body ?? "{}") as { + auto_mode?: { model_hints?: Array } + } + expect(body.auto_mode?.model_hints).toEqual(["gpt-4.1", "claude-haiku-4.5"]) +}) + +test("createChatCompletions sends the session token for pooled models", async () => { + const payload: ChatCompletionsPayload = { + model: "claude-haiku-4.5", + messages: [{ role: "user", content: "hi" }], + } + await createChatCompletions(payload) + const chat = calls.find((c) => c.url.endsWith("/chat/completions")) + expect(chat?.headers["Copilot-Session-Token"]).toBe("sess-token-123") +}) + +test("re-creates the session when the copilot token rotates", async () => { + state.copilotToken = "token-A" + state.modelSession = { + token: "old-session", + availableModels: ["claude-haiku-4.5"], + expiresAt: Date.now() + 60_000, + createdWithToken: "token-A", + } + + state.copilotToken = "token-B" + const session = await ensureModelSession() + + expect(session?.token).toBe("sess-token-123") + expect(session?.createdWithToken).toBe("token-B") + expect(calls.filter((c) => c.url.endsWith("/models/session"))).toHaveLength(1) +}) + +test("keeps a session without a recorded issuing token (test pre-seeds)", async () => { + state.modelSession = { + token: "seeded-session", + availableModels: ["claude-haiku-4.5"], + expiresAt: Date.now() + 60_000, + } + + const session = await ensureModelSession() + expect(session?.token).toBe("seeded-session") + expect(calls.filter((c) => c.url.endsWith("/models/session"))).toHaveLength(0) +}) diff --git a/tests/responses-adapter.test.ts b/tests/responses-adapter.test.ts new file mode 100644 index 0000000000..5f020a8921 --- /dev/null +++ b/tests/responses-adapter.test.ts @@ -0,0 +1,428 @@ +import { beforeEach, expect, mock, test } from "bun:test" +import { events } from "fetch-event-stream" + +import type { Model } from "../src/services/copilot/get-models" + +import { state } from "../src/lib/state" +import { + createChatCompletions, + type ChatCompletionChunk, + type ChatCompletionResponse, +} from "../src/services/copilot/create-chat-completions" +import { + chatPayloadToResponsesPayload, + chatResultToResponsesResult, + isResponsesOnlyModel, + responsesPayloadToChatPayload, + responsesResultToChatCompletion, + streamChatAsResponsesEvents, + streamResponsesAsChatChunks, +} from "../src/services/copilot/responses-adapter" + +const responsesOnly: Model = { + id: "gpt-5.3-codex", + name: "GPT-5.3 Codex", + object: "model", + version: "1", + vendor: "openai", + preview: false, + model_picker_enabled: true, + supported_endpoints: ["/responses"], +} + +const chatCapable: Model = { + ...responsesOnly, + id: "gpt-4o", + supported_endpoints: ["/chat/completions", "/responses"], +} + +state.copilotToken = "test-token" +state.vsCodeVersion = "1.0.0" +state.accountType = "individual" + +beforeEach(() => { + state.models = { object: "list", data: [responsesOnly, chatCapable] } + state.modelSession = { + token: "test-session", + availableModels: [], + expiresAt: Date.now() + 60_000, + } +}) + +test("isResponsesOnlyModel only flags responses-only models", () => { + expect(isResponsesOnlyModel("gpt-5.3-codex")).toBe(true) + expect(isResponsesOnlyModel("gpt-4o")).toBe(false) + expect(isResponsesOnlyModel("unknown-model")).toBe(false) +}) + +test("chatPayloadToResponsesPayload converts instructions, tools and limits", () => { + const result = chatPayloadToResponsesPayload({ + model: "gpt-5.3-codex", + messages: [ + { role: "system", content: "Be terse" }, + { role: "user", content: "say hi" }, + { + role: "assistant", + content: null, + tool_calls: [ + { + id: "call_1", + type: "function", + function: { name: "greet", arguments: "{}" }, + }, + ], + }, + { role: "tool", content: "hello", tool_call_id: "call_1" }, + ], + max_tokens: 42, + tools: [ + { + type: "function", + function: { name: "greet", description: "d", parameters: {} }, + }, + ], + }) + + expect(result.instructions).toBe("Be terse") + expect(result.max_output_tokens).toBe(42) + expect(result.model).toBe("gpt-5.3-codex") + + const input = result.input as Array> + expect(input[0]).toEqual({ role: "user", content: "say hi" }) + expect(input[1]).toEqual({ + type: "function_call", + call_id: "call_1", + name: "greet", + arguments: "{}", + }) + expect(input[2]).toEqual({ + type: "function_call_output", + call_id: "call_1", + output: "hello", + }) + + expect(result.tools).toEqual([ + { type: "function", name: "greet", description: "d", parameters: {} }, + ]) +}) + +test("responsesResultToChatCompletion rebuilds text, tool calls and usage", () => { + const result = responsesResultToChatCompletion( + { + id: "resp_1", + created_at: 123, + output: [ + { + type: "message", + content: [{ type: "output_text", text: "hello world" }], + }, + { + type: "function_call", + call_id: "call_9", + name: "greet", + arguments: "{}", + }, + ], + usage: { input_tokens: 10, output_tokens: 5, total_tokens: 15 }, + }, + "gpt-5.3-codex", + ) + + expect(result.object).toBe("chat.completion") + expect(result.choices[0].message.content).toBe("hello world") + expect(result.choices[0].message.tool_calls?.[0]?.function.name).toBe("greet") + expect(result.choices[0].finish_reason).toBe("tool_calls") + expect(result.usage?.prompt_tokens).toBe(10) + expect(result.usage?.total_tokens).toBe(15) +}) + +const sseBody = (lines: Array) => + lines.map((line) => `data: ${JSON.stringify(line)}\n\n`).join("") + +test("streamResponsesAsChatChunks emits role, text and finish chunks", async () => { + const body = sseBody([ + { type: "response.created", response: { id: "resp_9", created_at: 42 } }, + { type: "response.output_text.delta", delta: "PO" }, + { type: "response.output_text.delta", delta: "NG" }, + { + type: "response.completed", + response: { + id: "resp_9", + created_at: 42, + output: [ + { type: "message", content: [{ type: "output_text", text: "PONG" }] }, + ], + usage: { input_tokens: 4, output_tokens: 2, total_tokens: 6 }, + }, + }, + ]) + const upstream = new Response(body, { + headers: { "content-type": "text/event-stream" }, + }) + + const chunks: Array = [] + let sawSentinel = false + for await (const event of streamResponsesAsChatChunks( + upstream, + "gpt-5.3-codex", + events, + )) { + if (!event.data.startsWith("{")) { + sawSentinel = sawSentinel || event.data === "[DO" + "NE]" + continue + } + chunks.push(JSON.parse(event.data) as ChatCompletionChunk) + } + expect(sawSentinel).toBe(true) + + expect(chunks[0].choices[0].delta.role).toBe("assistant") + const text = chunks.map((c) => c.choices[0].delta.content ?? "").join("") + expect(text).toBe("PONG") + const last = chunks.at(-1) + if (!last) throw new Error("expected to finish with a final chunk") + expect(last.choices[0].finish_reason).toBe("stop") + expect(last.usage?.total_tokens).toBe(6) +}) + +test("createChatCompletions translates responses-only models end to end", async () => { + const upstream = new Response( + JSON.stringify({ + id: "resp_5", + created_at: 100, + output: [ + { type: "message", content: [{ type: "output_text", text: "PONG" }] }, + ], + usage: { input_tokens: 3, output_tokens: 1, total_tokens: 4 }, + }), + { status: 200, headers: { "content-type": "application/json" } }, + ) + const fetchMock = mock((_url: string, _opts?: unknown) => upstream) + // @ts-expect-error - Mock fetch doesn't implement all fetch properties + ;(globalThis as unknown as { fetch: typeof fetch }).fetch = fetchMock + + const result = (await createChatCompletions({ + model: "gpt-5.3-codex", + messages: [{ role: "user", content: "hi" }], + })) as ChatCompletionResponse + + expect(result.choices[0].message.content).toBe("PONG") + expect(result.choices[0].finish_reason).toBe("stop") + expect(result.usage?.prompt_tokens).toBe(3) +}) + +test("translates object tool_choice between both API shapes", () => { + const toResponses = chatPayloadToResponsesPayload({ + model: "gpt-5.3-codex", + messages: [{ role: "user", content: "hi" }], + tools: [ + { + type: "function", + function: { name: "greet", description: "d", parameters: {} }, + }, + ], + tool_choice: { type: "function", function: { name: "greet" } }, + }) + expect(toResponses.tool_choice).toEqual({ type: "function", name: "greet" }) + + const toChat = responsesPayloadToChatPayload({ + model: "gpt-4.1", + input: "hi", + tools: [{ type: "function", name: "greet", parameters: {} }], + tool_choice: { type: "function", name: "greet" }, + }) + expect(toChat.tool_choice).toEqual({ + type: "function", + function: { name: "greet" }, + }) +}) + +test("groups consecutive function_call items into one assistant message", () => { + const payload = responsesPayloadToChatPayload({ + model: "gpt-4.1", + input: [ + { role: "user", content: "run both" }, + { type: "function_call", call_id: "call_1", name: "a", arguments: "{}" }, + { type: "function_call", call_id: "call_2", name: "b", arguments: "{}" }, + { type: "function_call_output", call_id: "call_1", output: "1" }, + { type: "function_call_output", call_id: "call_2", output: "2" }, + ], + }) + + expect(payload.messages).toEqual([ + { role: "user", content: "run both" }, + { + role: "assistant", + content: null, + tool_calls: [ + { + id: "call_1", + type: "function", + function: { name: "a", arguments: "{}" }, + }, + { + id: "call_2", + type: "function", + function: { name: "b", arguments: "{}" }, + }, + ], + }, + { role: "tool", content: "1", tool_call_id: "call_1" }, + { role: "tool", content: "2", tool_call_id: "call_2" }, + ]) +}) + +test("maps incomplete responses to finish_reason length", () => { + const result = responsesResultToChatCompletion( + { + id: "resp_x", + status: "incomplete", + output: [ + { type: "message", content: [{ type: "output_text", text: "cut" }] }, + ], + }, + "gpt-5.3-codex", + ) + expect(result.choices[0].finish_reason).toBe("length") +}) + +test("skips non-function tools in reverse translation", () => { + const payload = responsesPayloadToChatPayload({ + model: "gpt-4.1", + input: "hi", + tools: [ + { type: "web_search" }, + { type: "function", name: "greet", parameters: {} }, + ], + }) + expect(payload.tools).toEqual([ + { + type: "function", + function: { name: "greet", description: undefined, parameters: {} }, + }, + ]) + + const noFunctions = responsesPayloadToChatPayload({ + model: "gpt-4.1", + input: "hi", + tools: [{ type: "web_search" }], + }) + expect(noFunctions.tools).toBeUndefined() +}) + +test("throws on failed responses results instead of faking success", () => { + expect(() => + responsesResultToChatCompletion( + { id: "resp_f", status: "failed", error: { message: "boom" } }, + "gpt-5.3-codex", + ), + ).toThrow("boom") +}) + +test("maps chat finish_reason length to responses status incomplete", () => { + const result = chatResultToResponsesResult({ + id: "chatcmpl-1", + object: "chat.completion", + created: 1, + model: "gpt-4.1", + choices: [ + { + index: 0, + message: { role: "assistant", content: "cut off" }, + logprobs: null, + finish_reason: "length", + }, + ], + }) + expect(result.status).toBe("incomplete") +}) + +test("preserves usage from usage-only chat chunks", async () => { + const chatChunks = [ + { + data: JSON.stringify({ + id: "c1", + object: "chat.completion.chunk", + created: 1, + model: "gpt-4.1", + choices: [ + { + index: 0, + delta: { role: "assistant" }, + finish_reason: null, + logprobs: null, + }, + ], + }), + }, + { + data: JSON.stringify({ + id: "c1", + object: "chat.completion.chunk", + created: 1, + model: "gpt-4.1", + choices: [ + { + index: 0, + delta: { content: "hi" }, + finish_reason: null, + logprobs: null, + }, + ], + }), + }, + { + data: JSON.stringify({ + id: "c1", + object: "chat.completion.chunk", + created: 1, + model: "gpt-4.1", + choices: [], + usage: { prompt_tokens: 7, completion_tokens: 2, total_tokens: 9 }, + }), + }, + ] + const body = + chatChunks.map((e) => `data: ${e.data}\n\n`).join("") + `data: [DONE]\n\n` + const upstream = new Response(body, { + headers: { "content-type": "text/event-stream" }, + }) + const collected: Array> = [] + for await (const event of streamChatAsResponsesEvents( + events(upstream), + "gpt-4.1", + )) { + collected.push(JSON.parse(event.data) as Record) + } + const completed = collected.find((e) => e.type === "response.completed") as { + response?: { usage?: { total_tokens?: number } } + } + expect(completed.response?.usage?.total_tokens).toBe(9) +}) + +test("rejects cancelled responses results", () => { + expect(() => + responsesResultToChatCompletion( + { + status: "cancelled", + error: { message: "Request was cancelled" }, + }, + "gpt-test", + ), + ).toThrow("Request was cancelled") + + expect(() => + responsesResultToChatCompletion({ status: "cancelled" }, "gpt-test"), + ).toThrow("Responses request cancelled") +}) + +test("maps responses text.format json_object back to chat response_format", () => { + const payload = responsesPayloadToChatPayload({ + model: "gpt-4.1", + input: "hi", + text: { format: { type: "json_object" } }, + }) + expect(payload.response_format).toEqual({ type: "json_object" }) + + const plain = responsesPayloadToChatPayload({ model: "gpt-4.1", input: "hi" }) + expect(plain.response_format).toBeUndefined() +})