diff --git a/docs/1_EXTENSION.md b/docs/1_EXTENSION.md index 3d3419d..016a8d2 100644 --- a/docs/1_EXTENSION.md +++ b/docs/1_EXTENSION.md @@ -339,6 +339,15 @@ string is treated as unset. | `SNAPSHOT_NAME` | `config.ts` | Snapshot-name override | | `REPOS` | None | Repository-list override | +The Render provider also applies internal transport budgets: SDK calls and +file transfers default to 30 seconds, individual control requests and +workspace lookup are bounded by 30 seconds, and termination gets an +independent 10 seconds. Long-running exec streams use the agent or build +operation deadline. Cleanup can extend a provisioning deadline by up to its +own budget. If a create's ID arrives late, cleanup still runs; when no ID +arrives, a `sandbox_create_unknown` log records that the provider-side +lifetime remains the backstop. These budgets are not environment settings. + ### Snapshot cron | Variable | Default | Purpose | diff --git a/docs/2_ARCHITECTURE.md b/docs/2_ARCHITECTURE.md index f3d1ec2..adc6f64 100644 --- a/docs/2_ARCHITECTURE.md +++ b/docs/2_ARCHITECTURE.md @@ -157,6 +157,29 @@ wraps it with `timeout`, the host stops waiting after a grace period, Render terminates the Sandbox after `SANDBOX_TIMEOUT_SECONDS`, and the workflow task has an outer computed deadline. +The Render provider supplies the public Sandbox SDK with its own typed HTTP +transport. It carries each operation's cancellation signal into control and +connect-token requests, including response-body reads. By default, an SDK +call gets 30 seconds for pacing, retries, token acquisition, and any file +transfer; individual control requests also have a 30-second backstop. Exec +streams retain the workload deadline. Automatic workspace lookup has a +30-second budget across all pages and body reads. + +Termination has an independent 10-second budget, including pacing, retries, +and HTTP/body reads. Cleanup failure is logged and does not indefinitely hold +an already computed answer. A create is retried only after a known rejection +such as HTTP 429. If a canceled create later returns an ID, an observer tries +to terminate it under a fresh cleanup budget. If a network failure or abort +prevents receiving an ID, the remote outcome remains unknown: the API cannot +reconcile that resource, so its mandatory provider lifetime is the backstop. +The provider logs this as `sandbox_create_unknown`. + +The adapter uses the SDK's public `SandboxesClient` constructor, keeps SDK +error decoding, and does not replace global `fetch` or access private SDK +fields. The pinned SDK 1.3.0 omits signals during token acquisition; this is +also visible in the [SDK 1.5.0 token implementation](https://github.com/render-oss/sdk/blob/9f6c31fcfe33a9b6dcebe53c563a2e790578fb9b/typescript/src/experimental/sandboxes/index.ts#L828-L848). +The provider transport closes that gap without an SDK upgrade. + ### 4. Open and verify an investigation Sandbox `investigateTask` in [`src/tasks.ts`](../src/tasks.ts): diff --git a/docs/3_TAKEAWAYS.md b/docs/3_TAKEAWAYS.md index 3f42213..d9ee8c0 100644 --- a/docs/3_TAKEAWAYS.md +++ b/docs/3_TAKEAWAYS.md @@ -197,7 +197,12 @@ Apply deadlines at several layers: - workflow task timeout. Cleanup should be idempotent and live in `finally`. A provider-side timeout is -still necessary for the gap where the host dies before cleanup. +still necessary for the gap where the host dies before cleanup. Give cleanup +its own finite budget so a hung terminate request cannot hold a completed +answer indefinitely. Carry operation deadlines through SDK token requests +and response-body reads as well as the visible command stream. Cancellation +of a create can leave an unknown remote outcome: observe late returned IDs +for cleanup and retain a provider lifetime backstop when no ID arrives. Implementation: [`sandbox/run-investigation.sh`](../sandbox/run-investigation.sh), [`src/sandbox/render/sandbox.ts`](../src/sandbox/render/sandbox.ts), and diff --git a/package.json b/package.json index a539789..40bbe6c 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "@renderinc/sdk": "1.3.0", "esbuild": "^0.28.2", "hono": "^4.13.0", + "openapi-fetch": "^0.15.2", "tar": "^7.5.1", "tsx": "^4.23.15", "typebox": "1.3.7", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 847c9ed..2051d76 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -26,6 +26,9 @@ importers: hono: specifier: ^4.13.0 version: 4.13.13 + openapi-fetch: + specifier: ^0.15.2 + version: 0.15.2 tar: specifier: ^7.5.1 version: 7.5.22 diff --git a/src/sandbox/render/client.ts b/src/sandbox/render/client.ts index 988fe95..e7eba1a 100644 --- a/src/sandbox/render/client.ts +++ b/src/sandbox/render/client.ts @@ -1,8 +1,9 @@ import { setTimeout as sleep } from "node:timers/promises"; -import { ClientError, Render, ServerError } from "@renderinc/sdk"; -import type { SandboxesClient, SandboxUploadContentType } from "@renderinc/sdk/experimental"; +import { ClientError, ServerError } from "@renderinc/sdk"; +import type { SandboxesClient, SandboxExecEvent, SandboxUploadContentType } from "@renderinc/sdk/experimental"; import type { Log } from "../../log.ts"; import type { RenderConfig } from "./config.ts"; +import { abortable, createSandboxApi, deadline, REQUEST_TIMEOUT_MS, withSandboxSignal } from "./transport.ts"; /** The subset of the SDK's SandboxesClient used here; tests substitute a fake. */ export type SandboxApi = Pick & { @@ -12,6 +13,10 @@ export type SandboxApi = Pick = {}): SandboxClient { + const t = { ...DEFAULT_TIMINGS, ...timings }; + return new SandboxClient(createSandboxApi(cfg, t.requestTimeoutMs), cfg, log, t); } /** Paces every API call and retries the given statuses (429 by default, which is never processed server-side). */ - async call(fn: () => Promise, opts: { retryOn?: readonly number[]; attempts?: number; signal?: AbortSignal } = {}): Promise { + async call(fn: () => Promise, opts: { retryOn?: readonly number[]; attempts?: number; signal?: AbortSignal; timeoutMs?: number; onLateResult?: (value: T) => Promise } = {}): Promise { + const signal = deadline(opts.signal, opts.timeoutMs ?? this.t.requestTimeoutMs); + signal.throwIfAborted(); const retryOn = opts.retryOn ?? [429]; const attempts = opts.attempts ?? 5; for (let attempt = 1; ; attempt++) { + signal.throwIfAborted(); const now = Date.now(); const at = Math.max(now, this.#nextRequestAt); this.#nextRequestAt = at + this.cfg.minRequestIntervalMs; - if (at > now) await sleep(at - now, undefined, { signal: opts.signal }); + if (at > now) await sleep(at - now, undefined, { signal }); + signal.throwIfAborted(); try { - return await fn(); + return await abortable(withSandboxSignal(signal, fn), signal, opts.onLateResult); } catch (err) { const status = err instanceof ClientError || err instanceof ServerError ? err.statusCode : undefined; - if (attempt >= attempts || status === undefined || !retryOn.includes(status) || opts.signal?.aborted) throw err; - await sleep(this.t.pollMs, undefined, { signal: opts.signal }); + if (attempt >= attempts || status === undefined || !retryOn.includes(status) || signal.aborted) throw err; + await sleep(this.t.pollMs, undefined, { signal }); } } } + /** Never retry an uncertain create. A late returned ID still gets independent cleanup. */ + async create(input: Parameters[0], opts: { signal?: AbortSignal; timeoutMs?: number } = {}) { + let requested = false; + try { + return await this.call(() => { + requested = true; + return this.api.create({ ...input, timeoutSeconds: input?.timeoutSeconds ?? this.cfg.investigationTimeoutSeconds }).catch((err: unknown) => { + if (err instanceof ClientError) requested = false; // A rejected request created no resource. + throw err; + }); + }, { ...opts, onLateResult: ({ id }) => this.terminateQuietly(id) }); + } catch (err) { + if (requested && !(err instanceof ClientError)) { + this.log.warn("sandbox create outcome unknown; provider lifetime timeout remains the backstop", { event: "sandbox_create_unknown" }); + } + throw err; + } + } + async exec(id: string, command: string, opts: ExecOptions): Promise<{ exitCode: number; stdout: string; stderr: string }> { const timeout = AbortSignal.timeout(opts.timeoutMs); const signal = opts.signal ? AbortSignal.any([opts.signal, timeout]) : timeout; @@ -95,7 +124,7 @@ export class SandboxClient { let stderr = ""; try { const stream = await this.call(() => this.api.exec(id, command, undefined, signal), { ...retry, signal }); - for await (const ev of stream) { + for await (const ev of this.events(stream, signal)) { if (ev.type === "exit") return { exitCode: ev.exit_code, stdout, stderr }; if (ev.stream === "stdout") stdout = (stdout + ev.data).slice(-OUTPUT_TAIL); else stderr = (stderr + ev.data).slice(-OUTPUT_TAIL); @@ -107,6 +136,19 @@ export class SandboxClient { throw new Error(`exec in ${id} ended without an exit event: ${command}`); } + async *events(stream: AsyncGenerator, signal: AbortSignal) { + try { + for (;;) { + signal.throwIfAborted(); + const next = await abortable(stream.next(), signal); + if (next.done) return; + yield next.value; + } + } finally { + void stream.return(undefined).catch(() => {}); + } + } + /** Like `exec`, but throws on a non-zero exit. */ async check(id: string, command: string, opts: ExecOptions): Promise { const res = await this.exec(id, command, opts); @@ -119,16 +161,19 @@ export class SandboxClient { data: string | Buffer, opts: { contentType?: SandboxUploadContentType; signal?: AbortSignal } = {}, ): Promise { - await this.call(() => this.api.upload(id, path, data, undefined, opts), { retryOn: TRANSIENT, signal: opts.signal }); + const signal = deadline(opts.signal, this.t.requestTimeoutMs); + await this.call(() => this.api.upload(id, path, data, undefined, { ...opts, signal }), { retryOn: TRANSIENT, signal }); } async download(id: string, path: string, signal?: AbortSignal): Promise { + signal = deadline(signal, this.t.requestTimeoutMs); const file = await this.call(() => this.api.download(id, path, undefined, signal), { retryOn: TRANSIENT, signal }); return file.data.toString("utf8"); } async waitRunning(id: string, { timeoutMs = this.t.bootTimeoutMs, signal }: { timeoutMs?: number; signal?: AbortSignal } = {}): Promise { const until = Date.now() + timeoutMs; + signal = deadline(signal, timeoutMs); for (;;) { const { status } = await this.call(() => this.api.get(id), { retryOn: READY_RETRY, signal }); if (status === "running") return; @@ -141,7 +186,7 @@ export class SandboxClient { /** Idempotent: a sandbox that no longer exists counts as terminated. */ async terminate(id: string): Promise { try { - await this.call(() => this.api.terminate(id), { retryOn: TRANSIENT }); + await this.call(() => this.api.terminate(id), { retryOn: TRANSIENT, timeoutMs: this.t.cleanupTimeoutMs }); } catch (err) { if (err instanceof ClientError && err.statusCode === 404) return; throw err; diff --git a/src/sandbox/render/config.ts b/src/sandbox/render/config.ts index 98009e9..5c35e96 100644 --- a/src/sandbox/render/config.ts +++ b/src/sandbox/render/config.ts @@ -5,6 +5,7 @@ import config from "../../../config.ts"; import { DEFAULT_CREDENTIALS, type HostCredentials } from "../../bot-config.ts"; import { present } from "../../util.ts"; import { REPO_ROOT } from "../interface.ts"; +import { abortable, REQUEST_TIMEOUT_MS } from "./transport.ts"; export type NetworkPolicy = SandboxNetworkPolicy; @@ -87,16 +88,18 @@ type Owner = { id: string; name: string; type: string }; * * API: GET /v1/owners — https://api-docs.render.com/reference/list-owners */ -export async function resolveWorkspaceId(apiKey: string, fetcher: typeof fetch = fetch): Promise<`tea-${string}`> { +export async function resolveWorkspaceId(apiKey: string, fetcher: typeof fetch = fetch, signal = AbortSignal.timeout(REQUEST_TIMEOUT_MS)): Promise<`tea-${string}`> { const owners: Owner[] = []; for (let cursor: string | undefined; ; ) { + signal.throwIfAborted(); const params = new URLSearchParams({ limit: "100" }); if (cursor) params.set("cursor", cursor); - const res = await fetcher(`https://api.render.com/v1/owners?${params}`, { + const res = await abortable(fetcher(`https://api.render.com/v1/owners?${params}`, { headers: { authorization: `Bearer ${apiKey}`, accept: "application/json" }, - }); + signal, + }), signal); if (!res.ok) throw new Error(`Render API /v1/owners failed: ${res.status}`); - const page = (await res.json()) as { owner: Owner; cursor: string }[]; + const page = (await abortable(res.json(), signal)) as { owner: Owner; cursor: string }[]; owners.push(...page.map((item) => item.owner)); cursor = page.at(-1)?.cursor; if (page.length < 100 || !cursor) break; diff --git a/src/sandbox/render/sandbox.ts b/src/sandbox/render/sandbox.ts index d82b8ca..fa1be46 100644 --- a/src/sandbox/render/sandbox.ts +++ b/src/sandbox/render/sandbox.ts @@ -91,14 +91,13 @@ export async function openRenderSandbox(client: SandboxClient, snapshotName: str if (policy.type === "allow-list" && !policy.rules.length) { throw new Error("cannot create an investigation sandbox with allow-list policy and no egress rules"); } - // SDK create and get take no signal: one of those requests in flight can finish past the deadline. const signal = AbortSignal.timeout(timeoutMs); const timedOut = (cause: unknown) => new Error(`could not open a sandbox from ${JSON.stringify(snapshotName)} within ${timeoutMs}ms (SANDBOX_PROVISION_TIMEOUT_MS)`, { cause }); let created: Awaited>; try { - created = await client.call( - () => client.api.create({ snapshotName, networkPolicy: policy, timeoutSeconds: client.cfg.investigationTimeoutSeconds }), + created = await client.create( + { snapshotName, networkPolicy: policy, timeoutSeconds: client.cfg.investigationTimeoutSeconds }, { signal }, ); } catch (err) { @@ -158,6 +157,7 @@ export async function runInvestigation(client: SandboxClient, id: string, opts: const runDir = `/tmp/run-${randomUUID()}`; const deadline = AbortSignal.timeout(opts.timeoutMs + client.t.investigationGraceMs); const signal = opts.signal ? AbortSignal.any([opts.signal, deadline]) : deadline; + signal.throwIfAborted(); let exitCode: number | null = null; let stderrTail = ""; let streaming = false; @@ -168,7 +168,7 @@ export async function runInvestigation(client: SandboxClient, id: string, opts: const command = `bash ${SANDBOX_APP}/run-investigation.sh ${runDir} ${Math.ceil(opts.timeoutMs / 1000)}`; const stream = await client.call(() => client.api.exec(id, command, undefined, signal), { retryOn: TRANSIENT, signal }); streaming = true; - for await (const ev of stream) { + for await (const ev of client.events(stream, signal)) { if (ev.type === "exit") { exitCode = ev.exit_code; break; diff --git a/src/sandbox/render/snapshot.ts b/src/sandbox/render/snapshot.ts index deca4a7..f6409f9 100644 --- a/src/sandbox/render/snapshot.ts +++ b/src/sandbox/render/snapshot.ts @@ -107,8 +107,9 @@ export async function buildSnapshot( const t = client.t; const prepare = { timeoutMs: t.prepareTimeoutMs }; const boot = { timeoutMs: t.bootTimeoutMs }; - const { id } = await client.call(() => - client.api.create({ timeoutSeconds: client.cfg.builderTimeoutSeconds, networkPolicy: { type: "allow-all" } }), + const { id } = await client.create( + { timeoutSeconds: client.cfg.builderTimeoutSeconds, networkPolicy: { type: "allow-all" } }, + { timeoutMs: t.bootTimeoutMs }, ); try { await client.waitRunning(id); @@ -236,13 +237,14 @@ function lockdownCommand(nodePath: string): string { } async function createSnapshot(client: SandboxClient, sandboxId: string, name: string): Promise { - let snap = await client.call(() => client.api.snapshots.create({ sandboxId, kind: "filesystem", name })); + const signal = AbortSignal.timeout(client.t.snapshotTimeoutMs); + let snap = await client.call(() => client.api.snapshots.create({ sandboxId, kind: "filesystem", name }), { signal }); const until = Date.now() + client.t.snapshotTimeoutMs; while (snap.status === "creating") { if (Date.now() > until) throw new Error(`snapshot ${snap.id} still creating after ${client.t.snapshotTimeoutMs}ms`); - await sleep(client.t.pollMs); + await sleep(client.t.pollMs, undefined, { signal }); const { id, sandboxGroupId } = snap; - snap = await client.call(() => client.api.snapshots.get({ sandboxGroupId, snapshotId: id }), { retryOn: [429, 503] }); + snap = await client.call(() => client.api.snapshots.get({ sandboxGroupId, snapshotId: id }), { retryOn: [429, 503], signal }); } if (snap.status !== "available") throw new Error(`snapshot ${snap.id} ${snap.status}: ${snap.error ?? "no error given"}`); return snap; diff --git a/src/sandbox/render/transport.ts b/src/sandbox/render/transport.ts new file mode 100644 index 0000000..1ad4ecd --- /dev/null +++ b/src/sandbox/render/transport.ts @@ -0,0 +1,55 @@ +import { AsyncLocalStorage } from "node:async_hooks"; +import { getUserAgent } from "@renderinc/sdk"; +import { SandboxesClient } from "@renderinc/sdk/experimental"; +import createClient, { type Client } from "openapi-fetch"; +import type { RenderConfig } from "./config.ts"; + +type ApiClient = ConstructorParameters[0]; +type ApiPaths = ApiClient extends Client ? Paths : never; +const operation = new AsyncLocalStorage(); +export const REQUEST_TIMEOUT_MS = 30_000; + +export function deadline(signal: AbortSignal | undefined, timeoutMs: number): AbortSignal { + const timeout = AbortSignal.timeout(timeoutMs); + return signal ? AbortSignal.any([signal, timeout]) : timeout; +} + +/** Scopes SDK calls whose public signatures do not accept a control-plane signal. */ +export function withSandboxSignal(signal: AbortSignal, fn: () => Promise): Promise { + signal.throwIfAborted(); + return operation.run(signal, fn); +} + +/** Retains a late-result observer even after the caller stops waiting. */ +export function abortable(promise: Promise, signal: AbortSignal, onLateResult?: (value: T) => Promise): Promise { + return new Promise((resolve, reject) => { + let aborted = signal.aborted; + const onAbort = () => { aborted = true; reject(signal.reason); }; + if (aborted) reject(signal.reason); + else signal.addEventListener("abort", onAbort, { once: true }); + promise.then((value) => { + signal.removeEventListener("abort", onAbort); + if (aborted) { + if (onLateResult) void Promise.resolve().then(() => onLateResult(value)).catch(() => {}); + } + else resolve(value); + }, (err: unknown) => { + signal.removeEventListener("abort", onAbort); + if (!aborted) reject(err); + }); + }); +} + +export function createSandboxApi(cfg: RenderConfig, requestTimeoutMs: number): SandboxesClient { + const api = createClient({ + // Workflows local-dev variables must never redirect Sandbox credentials to its local server. + baseUrl: "https://api.render.com/v1", + headers: { Authorization: `Bearer ${cfg.apiKey}`, "User-Agent": getUserAgent() }, + fetch: (request) => { + const signal = deadline(operation.getStore(), requestTimeoutMs); + signal.throwIfAborted(); + return fetch(new Request(request, { signal: AbortSignal.any([request.signal, signal]) })); + }, + }); + return new SandboxesClient(api, cfg.workspaceId, process.env.RENDER_REGION); +} diff --git a/test/sandbox-render.test.ts b/test/sandbox-render.test.ts index 5d34d71..1c4485d 100644 --- a/test/sandbox-render.test.ts +++ b/test/sandbox-render.test.ts @@ -386,6 +386,14 @@ describe("runInvestigation", () => { const res = await runInvestigation(client(), "sbx-run", { input: runnerInput(LAYOUT), timeoutMs: 60_000 }); expect(res).toMatchObject({ exitCode: null, aborted: false, streamLost: true }); }); + + it("starts no setup, credential upload, or output reads for a pre-aborted investigation", async () => { + await expect(runInvestigation(client(), "sbx-test", { + input: runnerInput(LAYOUT), modelApiKey: MODEL_KEY, timeoutMs: 1000, + signal: AbortSignal.abort(new Error("already canceled")), + })).rejects.toThrow("already canceled"); + expect(fake.calls).toEqual([]); + }); }); describe("configureSandboxes", () => { diff --git a/test/sandbox-transport.test.ts b/test/sandbox-transport.test.ts new file mode 100644 index 0000000..ad699e6 --- /dev/null +++ b/test/sandbox-transport.test.ts @@ -0,0 +1,305 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { createServer, type ServerResponse } from "node:http"; +import { setTimeout as sleep } from "node:timers/promises"; +import { ClientError, ServerError, getUserAgent } from "@renderinc/sdk"; +import { createLog } from "../src/log.ts"; +import { SandboxClient, type RenderTimings } from "../src/sandbox/render/client.ts"; +import { resolveWorkspaceId, type RenderConfig } from "../src/sandbox/render/config.ts"; +import { openRenderSandbox } from "../src/sandbox/render/sandbox.ts"; +import { investigateTask } from "../src/tasks.ts"; +import { loadConfig } from "../src/workflow.ts"; +import { FakeSandboxes } from "./fakes.ts"; + +const nativeFetch = globalThis.fetch; +const cfg: RenderConfig = { + apiKey: "synthetic-render-key", workspaceId: "tea-test", hostCredentials: {}, + networkPolicy: { type: "allow-all" }, investigationTimeoutSeconds: 1800, + builderTimeoutSeconds: 3600, minRequestIntervalMs: 0, sandboxDir: "sandbox", +}; +const sandbox = { id: "sbx-test", status: "running", plan: "starter", networkPolicy: { type: "allow-all" }, region: "oregon", timeoutSeconds: 1800, createdAt: "" }; +const json = (res: ServerResponse, value: unknown) => { + res.setHeader("content-type", "application/json"); + res.end(JSON.stringify(value)); +}; +let dispose = async () => {}; +afterEach(async () => { vi.unstubAllGlobals(); vi.unstubAllEnvs(); await dispose(); }); + +async function fixture(controlDelayMs = 180, opts: { + handle?: (path: string, res: ServerResponse, count: number) => boolean; + timings?: Partial; +} = {}) { + const requests: Array<{ path: string; headers: Record; body: string }> = []; + const aborted: string[] = []; + const timers = new Set>(); + let origin: string; + const server = createServer(async (req, res) => { + const chunks: Buffer[] = []; + for await (const chunk of req) chunks.push(Buffer.from(chunk)); + const path = req.url!; + requests.push({ path, headers: req.headers, body: Buffer.concat(chunks).toString() }); + res.on("close", () => { if (!res.writableEnded) aborted.push(path); }); + if (opts.handle?.(path, res, requests.length)) return; + const respond = () => { + if (path.includes("/token")) { + const kind = path.includes("/runs/") ? "exec" : path.includes("/download/") ? "download" : "upload"; + json(res, { uri: `${origin}/data/${kind}`, method: kind === "download" ? "GET" : "POST", token: "synthetic-connect-token" }); + } else if (path.startsWith("/v1/")) json(res, sandbox); + else if (path === "/data/exec") res.end('event: output\ndata: {"stream":"stdout","data":"ok"}\n\nevent: exit\ndata: {"exit_code":0}\n\n'); + else if (path === "/data/download") res.end("downloaded"); + else res.end(); + }; + if (path.startsWith("/v1/")) { + const timer = setTimeout(() => { timers.delete(timer); respond(); }, controlDelayMs); + timers.add(timer); + } else respond(); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("fixture did not bind"); + origin = `http://127.0.0.1:${address.port}`; + dispose = async () => { + for (const timer of timers) clearTimeout(timer); + server.closeAllConnections(); + await new Promise((resolve) => server.close(() => resolve())); + }; + vi.stubGlobal("fetch", (input: string | URL | Request, init?: RequestInit) => { + const request = new Request(input, init); + const url = new URL(request.url); + return nativeFetch(url.hostname === "api.render.com" ? new Request(`${origin}${url.pathname}${url.search}`, request) : request); + }); + const logs: Record[] = []; + const client = SandboxClient.fromConfig(cfg, createLog({}, (line) => void logs.push(JSON.parse(line))), { requestTimeoutMs: 100, cleanupTimeoutMs: 40, pollMs: 1, ...opts.timings }); + return { client, requests, aborted, logs }; +} + +describe("real SDK control-plane deadlines", () => { + it.each(["exec", "upload", "download", "create", "get", "terminate"] as const)("bounds %s before the delayed control response", async (method) => { + const { client, requests, aborted } = await fixture(); + const signal = AbortSignal.timeout(30); + const started = performance.now(); + const operation = method === "exec" ? client.exec("sbx-test", "true", { timeoutMs: 30 }) + : method === "upload" ? client.upload("sbx-test", "/input", "input", { signal }) + : method === "download" ? client.download("sbx-test", "/output", signal) + : client.call(() => method === "create" ? client.api.create({ timeoutSeconds: 1800 }) + : method === "get" ? client.api.get("sbx-test") : client.api.terminate("sbx-test"), { signal }); + await expect(operation).rejects.toThrow(); + expect(performance.now() - started).toBeLessThan(130); + await sleep(10); + expect(aborted.length).toBeGreaterThan(0); + expect(requests.some(({ path }) => path.startsWith("/data/"))).toBe(false); + }); +}); + + +describe("SDK transport lifecycle", () => { + it("bounds control response-body reads and aborts their socket", async () => { + const { client, aborted } = await fixture(0, { handle: (_path, res) => { + res.writeHead(200, { "content-type": "application/json" }); + res.write("{"); + return true; + } }); + const started = performance.now(); + await expect(client.call(() => client.api.get("sbx-test"), { signal: AbortSignal.timeout(30) })).rejects.toThrow(); + expect(performance.now() - started).toBeLessThan(130); + await sleep(10); + expect(aborted).toHaveLength(1); + }); + + it("retains successful exec/transfers plus SDK auth, workspace, user-agent and local-dev isolation", async () => { + vi.stubEnv("RENDER_USE_LOCAL_DEV", "true"); + vi.stubEnv("RENDER_LOCAL_DEV_URL", "http://127.0.0.1:1"); + const { client, requests } = await fixture(0); + expect(await client.exec("sbx-test", "true", { timeoutMs: 100 })).toEqual({ stdout: "ok", stderr: "", exitCode: 0 }); + await client.upload("sbx-test", "/input", "input"); + expect(await client.download("sbx-test", "/output")).toBe("downloaded"); + await client.create({ networkPolicy: cfg.networkPolicy }); + const control = requests.filter(({ path }) => path.startsWith("/v1/")); + expect(control.every(({ headers }) => headers.authorization === `Bearer ${cfg.apiKey}` && headers["user-agent"] === getUserAgent())).toBe(true); + expect(control.filter(({ path }) => path.includes("/token")).every(({ path }) => new URL(path, "http://test").searchParams.get("ownerId") === cfg.workspaceId)).toBe(true); + expect(JSON.parse(control.at(-1)!.body)).toMatchObject({ ownerId: cfg.workspaceId, timeoutSeconds: 1800 }); + expect(requests.find(({ path }) => path === "/data/upload")?.body).toBe("input"); + }); + + it("uses the independent cleanup budget for terminate and terminateQuietly", async () => { + const { client, aborted, logs } = await fixture(); + const started = performance.now(); + await expect(client.terminate("sbx-test")).rejects.toThrow(); + await expect(client.terminateQuietly("sbx-test")).resolves.toBeUndefined(); + expect(performance.now() - started).toBeLessThan(180); + await sleep(10); + expect(aborted).toHaveLength(2); + expect(logs).toContainEqual(expect.objectContaining({ msg: "could not terminate sandbox" })); + }); + + it("cleans a known ID after provisioning expires without waiting indefinitely for cleanup", async () => { + const { client, requests } = await fixture(180, { handle: (path, res) => { + if (path === "/v1/sandboxes") { json(res, sandbox); return true; } + return false; + } }); + const started = performance.now(); + await expect(openRenderSandbox(client, "test-snapshot", { timeoutMs: 30 })).rejects.toThrow("within 30ms"); + expect(performance.now() - started).toBeLessThan(160); + expect(requests.map(({ path }) => path.split("?")[0])).toEqual(["/v1/sandboxes", "/v1/sandboxes/sbx-test", "/v1/sandboxes/sbx-test/terminate"]); + }); + + it("keeps a late-ID observer and gives its cleanup a fresh signal", async () => { + const { client, requests } = await fixture(0); + const routedFetch = globalThis.fetch; + let createSignal: AbortSignal | undefined; + vi.stubGlobal("fetch", async (input: string | URL | Request, init?: RequestInit) => { + const request = new Request(input, init); + if (new URL(request.url).pathname === "/v1/sandboxes") { + createSignal = request.signal; + await sleep(80); // Deliberately emulate a transport which returns after cancellation. + return Response.json(sandbox); + } + expect(request.signal.aborted).toBe(false); + return routedFetch(request); + }); + await expect(client.create({}, { signal: AbortSignal.timeout(20) })).rejects.toThrow(); + await sleep(100); + expect(createSignal?.aborted).toBe(true); + expect(requests.map(({ path }) => path.split("?")[0])).toEqual(["/v1/sandboxes/sbx-test/terminate"]); + }); + + it("does not retry an unknown create outcome and retains its provider timeout backstop", async () => { + const { client, requests, logs } = await fixture(); + await expect(client.create({}, { signal: AbortSignal.timeout(30) })).rejects.toThrow(); + expect(requests).toHaveLength(1); + expect(JSON.parse(requests[0]!.body).timeoutSeconds).toBe(1800); + expect(logs).toContainEqual(expect.objectContaining({ event: "sandbox_create_unknown" })); + }); + + it("retries known create rejection but preserves SDK error types for other failures", async () => { + const { client, requests } = await fixture(0, { handle: (_path, res, count) => { + if (count === 1) { res.statusCode = 429; json(res, { message: "try later" }); return true; } + if (count === 3) { res.statusCode = 503; json(res, { message: "unavailable" }); return true; } + if (count === 4) { res.statusCode = 404; json(res, { message: "not found" }); return true; } + return false; + } }); + expect((await client.create({})).id).toBe("sbx-test"); + expect(requests).toHaveLength(2); + await expect(client.create({})).rejects.toBeInstanceOf(ServerError); + expect(requests).toHaveLength(3); + await expect(client.call(() => client.api.get("missing"), { retryOn: [] })).rejects.toBeInstanceOf(ClientError); + expect(requests).toHaveLength(4); + }); + + it("prevents pre-aborted work and cancels a paced call before sending it", async () => { + const { client, requests } = await fixture(0); + const signal = AbortSignal.abort(new Error("already canceled")); + await expect(client.create({}, { signal })).rejects.toThrow("already canceled"); + await expect(client.exec("sbx-test", "true", { timeoutMs: 100, signal })).rejects.toThrow("already canceled"); + await expect(client.upload("sbx-test", "/input", "input", { signal })).rejects.toThrow("already canceled"); + await expect(client.download("sbx-test", "/output", signal)).rejects.toThrow("already canceled"); + expect(requests).toEqual([]); + client.cfg.minRequestIntervalMs = 50; + try { + await client.call(() => client.api.get("sbx-test")); + await expect(client.call(() => client.api.get("sbx-test"), { signal: AbortSignal.timeout(10) })).rejects.toThrow(); + await sleep(60); + expect(requests).toHaveLength(1); + } finally { client.cfg.minRequestIntervalMs = 0; } + }); + + it("keeps concurrent request signals separate", async () => { + const { client, requests } = await fixture(50); + const short = client.call(() => client.api.get("short"), { signal: AbortSignal.timeout(20) }); + const long = client.call(() => client.api.get("long"), { signal: AbortSignal.timeout(100) }); + await expect(short).rejects.toThrow(); + await expect(long).resolves.toMatchObject({ id: "sbx-test" }); + expect(requests).toHaveLength(2); + }); + + + it("returns an already computed investigation result after cleanup times out", async () => { + const { client, requests } = await fixture(); + const sandboxes = new FakeSandboxes().script({ events: [{ + type: "result", at: new Date().toISOString(), + result: { status: "needs_clarification", question: "Which version?", why_needed: "It changed" }, + }] }); + const started = performance.now(); + const result = await investigateTask({ + config: loadConfig(), log: createLog({}, () => {}), + configureSandboxes: async () => async (name, options) => { + const opened = await sandboxes.open(name, options); + return { ...opened, close: () => client.terminate(opened.id) }; + }, + }, { trace_id: "test", transcript: [{ role: "user", text: "How does it work?" }] }); + expect(result).toMatchObject({ kind: "result", result: { status: "needs_clarification" } }); + expect(performance.now() - started).toBeLessThan(160); + expect(requests).toHaveLength(1); + }); + + it.each(["exec", "upload", "download"] as const)("retains the %s data-plane deadline after token acquisition", async (method) => { + const { client, aborted } = await fixture(0, { handle: (path, res) => { + if (!path.startsWith("/data/")) return false; + if (method !== "upload") { res.writeHead(200); res.write("partial"); } + return true; + } }); + const signal = AbortSignal.timeout(30); + const started = performance.now(); + const operation = method === "exec" ? client.exec("sbx-test", "true", { timeoutMs: 30 }) + : method === "upload" ? client.upload("sbx-test", "/input", "input", { signal }) + : client.download("sbx-test", "/output", signal); + await expect(operation).rejects.toThrow(); + expect(performance.now() - started).toBeLessThan(130); + await sleep(10); + expect(aborted.some((path) => path.startsWith("/data/"))).toBe(true); + }); + + it("bounds snapshot control requests and default waits without a supplied operation signal", async () => { + const { client, aborted } = await fixture(180, { timings: { requestTimeoutMs: 35 } }); + const started = performance.now(); + await expect(client.call(() => client.api.snapshots.get({ sandboxGroupId: "sbg-test", snapshotId: "snp-test" }))).rejects.toThrow(); + expect(performance.now() - started).toBeLessThan(130); + await sleep(10); + expect(aborted).toHaveLength(1); + }); + + it.each(["exec", "upload", "download"] as const)("starts no %s transfer after a late token response", async (method) => { + const { client, requests } = await fixture(70); + const routedFetch = globalThis.fetch; + vi.stubGlobal("fetch", (input: string | URL | Request, init?: RequestInit) => { + const request = new Request(input, init); + // Deliberately ignore cancellation of token acquisition; the SDK data-plane signal must still stop work. + return routedFetch(new URL(request.url).pathname.endsWith("/token") + ? new Request(request, { signal: new AbortController().signal }) : request); + }); + const signal = AbortSignal.timeout(20); + const operation = method === "exec" ? client.exec("sbx-test", "true", { timeoutMs: 20 }) + : method === "upload" ? client.upload("sbx-test", "/input", "input", { signal }) + : client.download("sbx-test", "/output", signal); + await expect(operation).rejects.toThrow(); + await sleep(100); + expect(requests).toHaveLength(1); + expect(requests[0]!.path).toContain("/token"); + }); + + it("cancels retry backoff without issuing another request", async () => { + const { client, requests } = await fixture(0, { timings: { pollMs: 60 }, handle: (_path, res) => { + res.statusCode = 503; json(res, { message: "unavailable" }); return true; + } }); + await expect(client.call(() => client.api.get("sbx-test"), { retryOn: [503], signal: AbortSignal.timeout(30) })).rejects.toThrow(); + await sleep(70); + expect(requests).toHaveLength(1); + }); + + it("also bounds automatic workspace lookup, including a stalled response body", async () => { + let bodyStarted = false; + const { aborted } = await fixture(0, { handle: (_path, res) => { + res.writeHead(200, { "content-type": "application/json" }); + res.write("["); + bodyStarted = true; + return true; + } }); + const controller = new AbortController(); + const lookup = resolveWorkspaceId(cfg.apiKey, fetch, controller.signal); + await vi.waitFor(() => expect(bodyStarted).toBe(true), { timeout: 1000, interval: 5 }); + const started = performance.now(); + controller.abort(new Error("workspace lookup canceled")); + await expect(lookup).rejects.toThrow("workspace lookup canceled"); + expect(performance.now() - started).toBeLessThan(500); + await vi.waitFor(() => expect(aborted).toHaveLength(1), { timeout: 1000, interval: 5 }); + }); +});