Bound Sandbox SDK requests and cleanup waits

docs/1_EXTENSION.md

@@ -339,6 +339,15 @@ string is treated as unset.
 | `SNAPSHOT_NAME` | `config.ts` | Snapshot-name override |
 | `REPOS` | None | Repository-list override |
 
+The Render provider also applies internal transport budgets: SDK calls and
+file transfers default to 30 seconds, individual control requests and
+workspace lookup are bounded by 30 seconds, and termination gets an
+independent 10 seconds. Long-running exec streams use the agent or build
+operation deadline. Cleanup can extend a provisioning deadline by up to its
+own budget. If a create's ID arrives late, cleanup still runs; when no ID
+arrives, a `sandbox_create_unknown` log records that the provider-side
+lifetime remains the backstop. These budgets are not environment settings.
+
 ### Snapshot cron
 
 | Variable | Default | Purpose |

docs/2_ARCHITECTURE.md

@@ -157,6 +157,29 @@ wraps it with `timeout`, the host stops waiting after a grace period, Render
 terminates the Sandbox after `SANDBOX_TIMEOUT_SECONDS`, and the workflow task
 has an outer computed deadline.
 
+The Render provider supplies the public Sandbox SDK with its own typed HTTP
+transport. It carries each operation's cancellation signal into control and
+connect-token requests, including response-body reads. By default, an SDK
+call gets 30 seconds for pacing, retries, token acquisition, and any file
+transfer; individual control requests also have a 30-second backstop. Exec
+streams retain the workload deadline. Automatic workspace lookup has a
+30-second budget across all pages and body reads.
+
+Termination has an independent 10-second budget, including pacing, retries,
+and HTTP/body reads. Cleanup failure is logged and does not indefinitely hold
+an already computed answer. A create is retried only after a known rejection
+such as HTTP 429. If a canceled create later returns an ID, an observer tries
+to terminate it under a fresh cleanup budget. If a network failure or abort
+prevents receiving an ID, the remote outcome remains unknown: the API cannot
+reconcile that resource, so its mandatory provider lifetime is the backstop.
+The provider logs this as `sandbox_create_unknown`.
+
+The adapter uses the SDK's public `SandboxesClient` constructor, keeps SDK
+error decoding, and does not replace global `fetch` or access private SDK
+fields. The pinned SDK 1.3.0 omits signals during token acquisition; this is
+also visible in the [SDK 1.5.0 token implementation](https://github.com/render-oss/sdk/blob/9f6c31fcfe33a9b6dcebe53c563a2e790578fb9b/typescript/src/experimental/sandboxes/index.ts#L828-L848).
+The provider transport closes that gap without an SDK upgrade.
+
 ### 4. Open and verify an investigation Sandbox
 
 `investigateTask` in [`src/tasks.ts`](../src/tasks.ts):

docs/3_TAKEAWAYS.md

@@ -197,7 +197,12 @@ Apply deadlines at several layers:
 - workflow task timeout.
 
 Cleanup should be idempotent and live in `finally`. A provider-side timeout is
-still necessary for the gap where the host dies before cleanup.
+still necessary for the gap where the host dies before cleanup. Give cleanup
+its own finite budget so a hung terminate request cannot hold a completed
+answer indefinitely. Carry operation deadlines through SDK token requests
+and response-body reads as well as the visible command stream. Cancellation
+of a create can leave an unknown remote outcome: observe late returned IDs
+for cleanup and retain a provider lifetime backstop when no ID arrives.
 
 Implementation: [`sandbox/run-investigation.sh`](../sandbox/run-investigation.sh),
 [`src/sandbox/render/sandbox.ts`](../src/sandbox/render/sandbox.ts), and

package.json

@@ -26,6 +26,7 @@
     "@renderinc/sdk": "1.3.0",
     "esbuild": "^0.28.2",
     "hono": "^4.13.0",
+    "openapi-fetch": "^0.15.2",
     "tar": "^7.5.1",
     "tsx": "^4.23.15",
     "typebox": "1.3.7",

src/sandbox/render/client.ts

@@ -1,8 +1,9 @@
 import { setTimeout as sleep } from "node:timers/promises";
-import { ClientError, Render, ServerError } from "@renderinc/sdk";
-import type { SandboxesClient, SandboxUploadContentType } from "@renderinc/sdk/experimental";
+import { ClientError, ServerError } from "@renderinc/sdk";
+import type { SandboxesClient, SandboxExecEvent, SandboxUploadContentType } from "@renderinc/sdk/experimental";
 import type { Log } from "../../log.ts";
 import type { RenderConfig } from "./config.ts";
+import { abortable, createSandboxApi, deadline, REQUEST_TIMEOUT_MS, withSandboxSignal } from "./transport.ts";
 
 /** The subset of the SDK's SandboxesClient used here; tests substitute a fake. */
 export type SandboxApi = Pick<SandboxesClient, "create" | "get" | "exec" | "upload" | "download" | "terminate"> & {
@@ -12,6 +13,10 @@ export type SandboxApi = Pick<SandboxesClient, "create" | "get" | "exec" | "uplo
 export type RenderTimings = {
   /** Minimum gap between get()/snapshot polls (rate limit: 100 req/min per workspace). */
   pollMs: number;
+  /** Default call budget (pacing/retries included) and individual control-request backstop. */
+  requestTimeoutMs: number;
+  /** Independent termination budget, including pacing, retries, HTTP and response-body reads. */
+  cleanupTimeoutMs: number;
   bootTimeoutMs: number;
   prepareTimeoutMs: number;
   snapshotTimeoutMs: number;
@@ -22,6 +27,8 @@ export type RenderTimings = {
 
 const DEFAULT_TIMINGS: RenderTimings = {
   pollMs: 3_000,
+  requestTimeoutMs: REQUEST_TIMEOUT_MS,
+  cleanupTimeoutMs: 10_000,
   bootTimeoutMs: 5 * 60_000,
   prepareTimeoutMs: 60 * 60_000,
   snapshotTimeoutMs: 20 * 60_000,
@@ -44,7 +51,7 @@ type ExecOptions = { timeoutMs: number; waitReady?: boolean; signal?: AbortSigna
  * - pacing: the Sandbox API allows 100 requests/min per workspace and the SDK neither paces nor retries 429s;
  * - retries on 429/503, which the SDK surfaces as errors;
  * - `waitRunning`: `create` returns while the sandbox is still `creating`, and there is no wait helper;
- * - `exec` collection: the SDK returns the raw event stream; `exec` here gathers output and the exit code.
+ * - `exec` collection: the SDK returns decoded stream events; `exec` here gathers output and the exit code.
  * Drop each piece when the SDK grows the equivalent.
  */
 export class SandboxClient {
@@ -60,31 +67,53 @@ export class SandboxClient {
     this.t = { ...DEFAULT_TIMINGS, ...timings };
   }
 
-  static fromConfig(cfg: RenderConfig, log: Log): SandboxClient {
-    // useLocalDev: false keeps sandbox calls on api.render.com even when RENDER_USE_LOCAL_DEV is set for workflows.
-    const render = new Render({ useLocalDev: false, token: cfg.apiKey, ownerId: cfg.workspaceId });
-    return new SandboxClient(render.experimental.sandboxes, cfg, log);
+  static fromConfig(cfg: RenderConfig, log: Log, timings: Partial<RenderTimings> = {}): SandboxClient {
+    const t = { ...DEFAULT_TIMINGS, ...timings };
+    return new SandboxClient(createSandboxApi(cfg, t.requestTimeoutMs), cfg, log, t);
   }
 
   /** Paces every API call and retries the given statuses (429 by default, which is never processed server-side). */
-  async call<T>(fn: () => Promise<T>, opts: { retryOn?: readonly number[]; attempts?: number; signal?: AbortSignal } = {}): Promise<T> {
+  async call<T>(fn: () => Promise<T>, opts: { retryOn?: readonly number[]; attempts?: number; signal?: AbortSignal; timeoutMs?: number; onLateResult?: (value: T) => Promise<void> } = {}): Promise<T> {
+    const signal = deadline(opts.signal, opts.timeoutMs ?? this.t.requestTimeoutMs);
+    signal.throwIfAborted();
     const retryOn = opts.retryOn ?? [429];
     const attempts = opts.attempts ?? 5;
     for (let attempt = 1; ; attempt++) {
+      signal.throwIfAborted();
       const now = Date.now();
       const at = Math.max(now, this.#nextRequestAt);
       this.#nextRequestAt = at + this.cfg.minRequestIntervalMs;
-      if (at > now) await sleep(at - now, undefined, { signal: opts.signal });
+      if (at > now) await sleep(at - now, undefined, { signal });
+      signal.throwIfAborted();
       try {
-        return await fn();
+        return await abortable(withSandboxSignal(signal, fn), signal, opts.onLateResult);
       } catch (err) {
         const status = err instanceof ClientError || err instanceof ServerError ? err.statusCode : undefined;
-        if (attempt >= attempts || status === undefined || !retryOn.includes(status) || opts.signal?.aborted) throw err;
-        await sleep(this.t.pollMs, undefined, { signal: opts.signal });
+        if (attempt >= attempts || status === undefined || !retryOn.includes(status) || signal.aborted) throw err;
+        await sleep(this.t.pollMs, undefined, { signal });
       }
     }
   }
 
+  /** Never retry an uncertain create. A late returned ID still gets independent cleanup. */
+  async create(input: Parameters<SandboxApi["create"]>[0], opts: { signal?: AbortSignal; timeoutMs?: number } = {}) {
+    let requested = false;
+    try {
+      return await this.call(() => {
+        requested = true;
+        return this.api.create({ ...input, timeoutSeconds: input?.timeoutSeconds ?? this.cfg.investigationTimeoutSeconds }).catch((err: unknown) => {
+          if (err instanceof ClientError) requested = false; // A rejected request created no resource.
+          throw err;
+        });
+      }, { ...opts, onLateResult: ({ id }) => this.terminateQuietly(id) });
+    } catch (err) {
+      if (requested && !(err instanceof ClientError)) {
+        this.log.warn("sandbox create outcome unknown; provider lifetime timeout remains the backstop", { event: "sandbox_create_unknown" });
+      }
+      throw err;
+    }
+  }
+
   async exec(id: string, command: string, opts: ExecOptions): Promise<{ exitCode: number; stdout: string; stderr: string }> {
     const timeout = AbortSignal.timeout(opts.timeoutMs);
     const signal = opts.signal ? AbortSignal.any([opts.signal, timeout]) : timeout;
@@ -95,7 +124,7 @@ export class SandboxClient {
     let stderr = "";
     try {
       const stream = await this.call(() => this.api.exec(id, command, undefined, signal), { ...retry, signal });
-      for await (const ev of stream) {
+      for await (const ev of this.events(stream, signal)) {
         if (ev.type === "exit") return { exitCode: ev.exit_code, stdout, stderr };
         if (ev.stream === "stdout") stdout = (stdout + ev.data).slice(-OUTPUT_TAIL);
         else stderr = (stderr + ev.data).slice(-OUTPUT_TAIL);
@@ -107,6 +136,19 @@ export class SandboxClient {
     throw new Error(`exec in ${id} ended without an exit event: ${command}`);
   }
 
+  async *events(stream: AsyncGenerator<SandboxExecEvent>, signal: AbortSignal) {
+    try {
+      for (;;) {
+        signal.throwIfAborted();
+        const next = await abortable(stream.next(), signal);
+        if (next.done) return;
+        yield next.value;
+      }
+    } finally {
+      void stream.return(undefined).catch(() => {});
+    }
+  }
+
   /** Like `exec`, but throws on a non-zero exit. */
   async check(id: string, command: string, opts: ExecOptions): Promise<void> {
     const res = await this.exec(id, command, opts);
@@ -119,16 +161,19 @@ export class SandboxClient {
     data: string | Buffer,
     opts: { contentType?: SandboxUploadContentType; signal?: AbortSignal } = {},
   ): Promise<void> {
-    await this.call(() => this.api.upload(id, path, data, undefined, opts), { retryOn: TRANSIENT, signal: opts.signal });
+    const signal = deadline(opts.signal, this.t.requestTimeoutMs);
+    await this.call(() => this.api.upload(id, path, data, undefined, { ...opts, signal }), { retryOn: TRANSIENT, signal });
   }
 
   async download(id: string, path: string, signal?: AbortSignal): Promise<string> {
+    signal = deadline(signal, this.t.requestTimeoutMs);
     const file = await this.call(() => this.api.download(id, path, undefined, signal), { retryOn: TRANSIENT, signal });
     return file.data.toString("utf8");
   }
 
   async waitRunning(id: string, { timeoutMs = this.t.bootTimeoutMs, signal }: { timeoutMs?: number; signal?: AbortSignal } = {}): Promise<void> {
     const until = Date.now() + timeoutMs;
+    signal = deadline(signal, timeoutMs);
     for (;;) {
       const { status } = await this.call(() => this.api.get(id), { retryOn: READY_RETRY, signal });
       if (status === "running") return;
@@ -141,7 +186,7 @@ export class SandboxClient {
   /** Idempotent: a sandbox that no longer exists counts as terminated. */
   async terminate(id: string): Promise<void> {
     try {
-      await this.call(() => this.api.terminate(id), { retryOn: TRANSIENT });
+      await this.call(() => this.api.terminate(id), { retryOn: TRANSIENT, timeoutMs: this.t.cleanupTimeoutMs });
     } catch (err) {
       if (err instanceof ClientError && err.statusCode === 404) return;
       throw err;

src/sandbox/render/config.ts

@@ -5,6 +5,7 @@ import config from "../../../config.ts";
 import { DEFAULT_CREDENTIALS, type HostCredentials } from "../../bot-config.ts";
 import { present } from "../../util.ts";
 import { REPO_ROOT } from "../interface.ts";
+import { abortable, REQUEST_TIMEOUT_MS } from "./transport.ts";
 
 export type NetworkPolicy = SandboxNetworkPolicy;
 
@@ -87,16 +88,18 @@ type Owner = { id: string; name: string; type: string };
  *
  * API: GET /v1/owners — https://api-docs.render.com/reference/list-owners
  */
-export async function resolveWorkspaceId(apiKey: string, fetcher: typeof fetch = fetch): Promise<`tea-${string}`> {
+export async function resolveWorkspaceId(apiKey: string, fetcher: typeof fetch = fetch, signal = AbortSignal.timeout(REQUEST_TIMEOUT_MS)): Promise<`tea-${string}`> {
   const owners: Owner[] = [];
   for (let cursor: string | undefined; ; ) {
+    signal.throwIfAborted();
     const params = new URLSearchParams({ limit: "100" });
     if (cursor) params.set("cursor", cursor);
-    const res = await fetcher(`https://api.render.com/v1/owners?${params}`, {
+    const res = await abortable(fetcher(`https://api.render.com/v1/owners?${params}`, {
       headers: { authorization: `Bearer ${apiKey}`, accept: "application/json" },
-    });
+      signal,
+    }), signal);
     if (!res.ok) throw new Error(`Render API /v1/owners failed: ${res.status}`);
-    const page = (await res.json()) as { owner: Owner; cursor: string }[];
+    const page = (await abortable(res.json(), signal)) as { owner: Owner; cursor: string }[];
     owners.push(...page.map((item) => item.owner));
     cursor = page.at(-1)?.cursor;
     if (page.length < 100 || !cursor) break;

src/sandbox/render/sandbox.ts

@@ -91,14 +91,13 @@ export async function openRenderSandbox(client: SandboxClient, snapshotName: str
   if (policy.type === "allow-list" && !policy.rules.length) {
     throw new Error("cannot create an investigation sandbox with allow-list policy and no egress rules");
   }
-  // SDK create and get take no signal: one of those requests in flight can finish past the deadline.
   const signal = AbortSignal.timeout(timeoutMs);
   const timedOut = (cause: unknown) =>
     new Error(`could not open a sandbox from ${JSON.stringify(snapshotName)} within ${timeoutMs}ms (SANDBOX_PROVISION_TIMEOUT_MS)`, { cause });
   let created: Awaited<ReturnType<SandboxApi["create"]>>;
   try {
-    created = await client.call(
-      () => client.api.create({ snapshotName, networkPolicy: policy, timeoutSeconds: client.cfg.investigationTimeoutSeconds }),
+    created = await client.create(
+      { snapshotName, networkPolicy: policy, timeoutSeconds: client.cfg.investigationTimeoutSeconds },
       { signal },
     );
   } catch (err) {
@@ -158,6 +157,7 @@ export async function runInvestigation(client: SandboxClient, id: string, opts:
   const runDir = `/tmp/run-${randomUUID()}`;
   const deadline = AbortSignal.timeout(opts.timeoutMs + client.t.investigationGraceMs);
   const signal = opts.signal ? AbortSignal.any([opts.signal, deadline]) : deadline;
+  signal.throwIfAborted();
   let exitCode: number | null = null;
   let stderrTail = "";
   let streaming = false;
@@ -168,7 +168,7 @@ export async function runInvestigation(client: SandboxClient, id: string, opts:
     const command = `bash ${SANDBOX_APP}/run-investigation.sh ${runDir} ${Math.ceil(opts.timeoutMs / 1000)}`;
     const stream = await client.call(() => client.api.exec(id, command, undefined, signal), { retryOn: TRANSIENT, signal });
     streaming = true;
-    for await (const ev of stream) {
+    for await (const ev of client.events(stream, signal)) {
       if (ev.type === "exit") {
         exitCode = ev.exit_code;
         break;

src/sandbox/render/snapshot.ts

@@ -107,8 +107,9 @@ export async function buildSnapshot(
   const t = client.t;
   const prepare = { timeoutMs: t.prepareTimeoutMs };
   const boot = { timeoutMs: t.bootTimeoutMs };
-  const { id } = await client.call(() =>
-    client.api.create({ timeoutSeconds: client.cfg.builderTimeoutSeconds, networkPolicy: { type: "allow-all" } }),
+  const { id } = await client.create(
+    { timeoutSeconds: client.cfg.builderTimeoutSeconds, networkPolicy: { type: "allow-all" } },
+    { timeoutMs: t.bootTimeoutMs },
   );
   try {
     await client.waitRunning(id);
@@ -236,13 +237,14 @@ function lockdownCommand(nodePath: string): string {
 }
 
 async function createSnapshot(client: SandboxClient, sandboxId: string, name: string): Promise<SandboxSnapshot> {
-  let snap = await client.call(() => client.api.snapshots.create({ sandboxId, kind: "filesystem", name }));
+  const signal = AbortSignal.timeout(client.t.snapshotTimeoutMs);
+  let snap = await client.call(() => client.api.snapshots.create({ sandboxId, kind: "filesystem", name }), { signal });
   const until = Date.now() + client.t.snapshotTimeoutMs;
   while (snap.status === "creating") {
     if (Date.now() > until) throw new Error(`snapshot ${snap.id} still creating after ${client.t.snapshotTimeoutMs}ms`);
-    await sleep(client.t.pollMs);
+    await sleep(client.t.pollMs, undefined, { signal });
     const { id, sandboxGroupId } = snap;
-    snap = await client.call(() => client.api.snapshots.get({ sandboxGroupId, snapshotId: id }), { retryOn: [429, 503] });
+    snap = await client.call(() => client.api.snapshots.get({ sandboxGroupId, snapshotId: id }), { retryOn: [429, 503], signal });
   }
   if (snap.status !== "available") throw new Error(`snapshot ${snap.id} ${snap.status}: ${snap.error ?? "no error given"}`);
   return snap;