From dda0d932b52491a244ddedeb4ec4e7025ba401ef Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 20 Sep 2026 11:12:50 +0800 Subject: [PATCH 01/53] fix(store/s3): report a meta without an export as not found Delete keeps meta.json until the exports are gone, so a failed export cleanup leaves a record with a meta and no export. The dir backend answers store.ErrNotFound for that record and resolveGolden cold-boots; s3 answered a plain error and every claim of the template key failed until someone retried the delete. All four Fetch callers already map ErrNotFound to their unknown-record outcome. --- sandboxd/store/s3/s3.go | 2 +- sandboxd/store/s3/s3_test.go | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/sandboxd/store/s3/s3.go b/sandboxd/store/s3/s3.go index edaeb713..71edd48b 100644 --- a/sandboxd/store/s3/s3.go +++ b/sandboxd/store/s3/s3.go @@ -331,7 +331,7 @@ func (s *Store) populate(ctx context.Context, id string, meta []byte, gen string } } if len(keys) == 0 { - return fmt.Errorf("record %s has no export", id) + return store.ErrNotFound } g, gctx := errgroup.WithContext(ctx) g.SetLimit(fetchConcurrency) diff --git a/sandboxd/store/s3/s3_test.go b/sandboxd/store/s3/s3_test.go index e5d17a4a..ff19362b 100644 --- a/sandboxd/store/s3/s3_test.go +++ b/sandboxd/store/s3/s3_test.go @@ -6,6 +6,7 @@ import ( "context" "crypto/sha256" "encoding/xml" + "errors" "fmt" "io" "maps" @@ -137,6 +138,17 @@ func TestFetchLegacyExportLayout(t *testing.T) { } } +func TestFetchReportsMetaWithoutExportAsNotFound(t *testing.T) { + const id = "ck_00000000000000cc" + fake := &fakeS3{objects: map[string][]byte{ + "ck/" + id + "/" + store.MetaFile: []byte(`{"id":"` + id + `"}`), + }} + st := newTestStore(t, fake) + if _, _, _, err := st.Fetch(t.Context(), id); !errors.Is(err, store.ErrNotFound) { + t.Fatalf("Fetch = %v, want store.ErrNotFound", err) + } +} + func TestDigestReaderIgnoresReadBoundaries(t *testing.T) { data := bytes.Repeat([]byte{'x'}, int(store.DigestChunkSize)) data = append(data, 'y') From 6aeece94212a1c7401885fcd7d6776c5be5477a2 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 20 Sep 2026 11:15:25 +0800 Subject: [PATCH 02/53] fix(mcp): cap the logs tool's output like exec and read_file toolLogs collected the guest's replay in a bare strings.Builder. silkd bounds a log ring at 256 KiB, so only a replaced agent streams more, but that agent runs in the guest and the cap on exec and read_file exists for the same reason: guest-fed output must not grow this process. --- mcp/server_test.go | 14 ++++++++++++++ mcp/tools.go | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/mcp/server_test.go b/mcp/server_test.go index 0a17a606..6a399231 100644 --- a/mcp/server_test.go +++ b/mcp/server_test.go @@ -111,6 +111,20 @@ func TestReadFileStopsAtTheCap(t *testing.T) { } } +func TestLogsKeepsAtMostTheCap(t *testing.T) { + chunk := `{"type":"stdout","data":"` + base64.StdEncoding.EncodeToString(bytes.Repeat([]byte("x"), 256<<10)) + `"}` + "\n" + srv := newTestServer(t, agentRoute(t, func(string) (string, bool) { + return strings.Repeat(chunk, 8) + `{"type":"done"}` + "\n", false + })) + replies := serveLines(t, srv, + `{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"create_sandbox","arguments":{}}}`, + `{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"logs","arguments":{"sandbox_id":"sb_1","pid":7}}}`, + ) + if n := strings.Count(toolText(t, replies[2]), "x"); n != execOutputCap { + t.Errorf("logs kept %d bytes of a 2 MiB replay, want the %d cap", n, execOutputCap) + } +} + func TestTrackBoxAfterCloseReleasesTheClaim(t *testing.T) { released := make(chan string, 1) node := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/mcp/tools.go b/mcp/tools.go index fcc0eabc..c2abb9c0 100644 --- a/mcp/tools.go +++ b/mcp/tools.go @@ -252,7 +252,7 @@ func toolLogs(ctx context.Context, s *server, raw json.RawMessage) (string, erro if err != nil { return "", err } - var stdout, stderr strings.Builder + var stdout, stderr cappedOutput code, exited, err := sb.Logs(ctx, args.PID, &stdout, &stderr) if err != nil { return "", err From a881cbcd7ebb31e9a94852c49d27c7fd491dfd39 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 20 Sep 2026 11:16:50 +0800 Subject: [PATCH 03/53] fix(server): audit the buffered exec frame without its delimiter handleExec appended the newline before it handed the frame to Audit, so an exec frame of exactly AuditLineCap bytes was journaled as oversized while the relay tap, which strips the delimiter, records the same frame in full. --- sandboxd/server/exec.go | 2 +- sandboxd/server/exec_test.go | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/sandboxd/server/exec.go b/sandboxd/server/exec.go index 4af9280d..39e85c1a 100644 --- a/sandboxd/server/exec.go +++ b/sandboxd/server/exec.go @@ -68,10 +68,10 @@ func (s *Server) handleExec(w http.ResponseWriter, r *http.Request) { } defer closeOnCancel(ctx, guest)() frame, _ := wire.EncodeRequest(wire.Exec{Argv: req.Argv, Cwd: req.Cwd, Env: req.Env}) - frame = append(frame, '\n') if s.mgr.AuditEnabled() { s.mgr.Audit(ctx, id, frame) } + frame = append(frame, '\n') stdinClose, _ := wire.EncodeRequest(wire.StdinClose{}) frame = append(frame, stdinClose...) frame = append(frame, '\n') diff --git a/sandboxd/server/exec_test.go b/sandboxd/server/exec_test.go index bece9766..2c9ed76e 100644 --- a/sandboxd/server/exec_test.go +++ b/sandboxd/server/exec_test.go @@ -58,6 +58,28 @@ func TestExecReturnsBufferedOutput(t *testing.T) { } } +func TestExecAuditsTheFrameWithoutItsDelimiter(t *testing.T) { + ts, srv := newRelayServer(t, func(conn net.Conn) { + defer conn.Close() + r := bufio.NewReader(conn) + for range 2 { + if _, err := r.ReadBytes('\n'); err != nil { + return + } + } + _, _ = io.WriteString(conn, `{"type":"started","pid":7}`+"\n"+`{"type":"exit","code":0}`+"\n") + }) + audited := make(chan []byte, 1) + srv.mgr.(*fakeManager).audited = func(_ string, line []byte) { audited <- bytes.Clone(line) } + if status, body := postExec(t, ts, `{"argv":["true"]}`); status != http.StatusOK { + t.Fatalf("status %d, want 200: %s", status, body) + } + want, _ := wire.EncodeRequest(wire.Exec{Argv: []string{"true"}}) + if got := <-audited; !bytes.Equal(got, want) { + t.Errorf("audited %q, want the bare frame %q", got, want) + } +} + func TestExecMapsSilkdErrors(t *testing.T) { for _, tt := range []struct { kind string From e26f7d528ef97da01472cb773927245728a78a62 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 20 Sep 2026 11:16:50 +0800 Subject: [PATCH 04/53] fix(pool): log a failed template listing at startup NewManager dropped the Metas error: the template set stayed empty for the life of the process and the node gossiped no promoted template, with nothing in the log. Warn and carry on, as the startup archive listing in reconcile already does. --- sandboxd/pool/pool.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index eb5acee0..0f2dc68e 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -404,12 +404,14 @@ func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *eg } m.ckptTTL = time.Duration(cfg.CheckpointTTLHours) * time.Hour m.tplSet = map[string]string{} - if metas, listErr := m.tpls.Metas(ctx); listErr == nil { - for _, raw := range metas { - var rec templateRecord - if json.Unmarshal(raw, &rec) == nil && rec.ID != "" { - m.tplSet[rec.ID] = rec.Tenant - } + metas, listErr := m.tpls.Metas(ctx) + if listErr != nil { + log.WithFunc("pool.NewManager").Warnf(ctx, "list templates skipped: %v", listErr) + } + for _, raw := range metas { + var rec templateRecord + if json.Unmarshal(raw, &rec) == nil && rec.ID != "" { + m.tplSet[rec.ID] = rec.Tenant } } usage, err := newJournal(filepath.Join(cfg.DataDir, "usage.jsonl")) From f099f3b1d754227b5d41b8d9b56a8d3c7916a900 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 20 Sep 2026 11:18:02 +0800 Subject: [PATCH 05/53] fix(sdk/go): keep the relay connection when a stream open is refused DialPort, Watch and OpenPty closed the connection when the guest answered the open with an error frame; every other verb parks it through lease.done. silkd returns to its serve loop after each of those error frames, so a DialPort poll on a port that is not listening yet paid a dial, an upgrade and a TLS handshake per attempt. --- sdk/go/keepalive_test.go | 17 +++++++++++++++++ sdk/go/port.go | 3 +-- sdk/go/pty.go | 3 +-- sdk/go/watch.go | 3 +-- 4 files changed, 20 insertions(+), 6 deletions(-) diff --git a/sdk/go/keepalive_test.go b/sdk/go/keepalive_test.go index 78436354..c574988d 100644 --- a/sdk/go/keepalive_test.go +++ b/sdk/go/keepalive_test.go @@ -39,6 +39,23 @@ func TestCallsShareOneConnection(t *testing.T) { } } +func TestRefusedStreamOpenKeepsItsConnection(t *testing.T) { + var upgrades atomic.Int32 + fake := silkdtest.NewFake(t.TempDir()) + sb := testSandbox(t, newAgentServer(t, func(c net.Conn) { + upgrades.Add(1) + fake.ServeConn(c) + })) + for range 3 { + if _, err := sb.Watch(t.Context(), "/missing", false); err == nil { + t.Fatal("watch on a missing path succeeded") + } + } + if got := upgrades.Load(); got != 1 { + t.Errorf("upgrades = %d, want 1: an error frame ends the RPC, not the connection", got) + } +} + func TestOldDaemonDialsPerCall(t *testing.T) { var upgrades atomic.Int32 sb := testSandbox(t, newAgentServer(t, func(c net.Conn) { diff --git a/sdk/go/port.go b/sdk/go/port.go index 5af3042d..00375db5 100644 --- a/sdk/go/port.go +++ b/sdk/go/port.go @@ -138,8 +138,7 @@ func (s *Sandbox) openStream(ctx context.Context, req wire.Request) (*PortConn, return nil, err } if _, err = expect[wire.Ready](ctx, conn); err != nil { - l.close() - return nil, err + return nil, l.done(err) } pr, pw := io.Pipe() p := &PortConn{conn: conn, stop: l.close, out: pr} diff --git a/sdk/go/pty.go b/sdk/go/pty.go index b04db459..117ab49c 100644 --- a/sdk/go/pty.go +++ b/sdk/go/pty.go @@ -104,8 +104,7 @@ func (s *Sandbox) OpenPty(ctx context.Context, opts PtyOpts) (*Pty, error) { } started, err := expect[wire.Started](ctx, conn) if err != nil { - l.close() - return nil, err + return nil, l.done(err) } pr, pw := io.Pipe() diff --git a/sdk/go/watch.go b/sdk/go/watch.go index 8d5e1b50..33be6a1f 100644 --- a/sdk/go/watch.go +++ b/sdk/go/watch.go @@ -87,8 +87,7 @@ func (s *Sandbox) Watch(ctx context.Context, path string, recursive bool) (*Watc return nil, err } if _, err = expect[wire.Ready](ctx, conn); err != nil { - l.close() - return nil, err + return nil, l.done(err) } w := &Watcher{ events: make(chan wire.Event, 16), From c0e0b56fbac104befe788128fef39c82473ced66 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 20 Sep 2026 11:23:35 +0800 Subject: [PATCH 06/53] fix(sdk/python): report a guest frame without its field as a ProtocolError The stdio pump and the data drain indexed frame["data"] and frame["code"] directly, so a guest that omits the field raised a bare KeyError outside the SandboxError hierarchy callers catch. Every sibling reader already goes through _need. --- sdk/python/cocoonsandbox/sandbox.py | 8 ++++---- sdk/python/tests/test_proc.py | 9 +++++++++ 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/sdk/python/cocoonsandbox/sandbox.py b/sdk/python/cocoonsandbox/sandbox.py index 3e9c8e19..d3446bf4 100644 --- a/sdk/python/cocoonsandbox/sandbox.py +++ b/sdk/python/cocoonsandbox/sandbox.py @@ -639,11 +639,11 @@ def _pump_stdio( for frame in conn.recv_until("exit", "done"): t = frame["type"] if t == "stdout" and on_stdout: - on_stdout(frame["data"]) + on_stdout(_need(frame, "data")) elif t == "stderr" and on_stderr: - on_stderr(frame["data"]) + on_stderr(_need(frame, "data")) elif t == "exit": - return cast(int, frame["code"]) + return cast(int, _need(frame, "code")) return None @@ -672,5 +672,5 @@ def _drain_data(conn: Conn) -> bytes: chunks = [] for frame in conn.recv_until("done"): if frame["type"] == "data": - chunks.append(frame["data"]) + chunks.append(_need(frame, "data")) return b"".join(chunks) diff --git a/sdk/python/tests/test_proc.py b/sdk/python/tests/test_proc.py index 440d99b9..ad5f7eaa 100644 --- a/sdk/python/tests/test_proc.py +++ b/sdk/python/tests/test_proc.py @@ -3,8 +3,10 @@ import threading +import pytest from conftest import sandbox_at +from cocoonsandbox import ProtocolError from cocoonsandbox.frames import FS_CHUNK @@ -43,6 +45,13 @@ def test_attach_returns_exit_code(monkeypatch): assert out == [b"late"] +@pytest.mark.parametrize("frame", [{"type": "stdout"}, {"type": "stderr"}, {"type": "exit"}]) +def test_attach_reports_a_frame_without_its_field_as_a_protocol_error(monkeypatch, frame): + sb, _ = fake_sandbox(monkeypatch, [frame]) + with pytest.raises(ProtocolError): + sb.attach(41, on_stdout=bytes, on_stderr=bytes) + + def test_run_pumps_stdin_while_reading_output(monkeypatch): sb = sandbox_at("127.0.0.1:1", keep_alive=0) blocking = BlockingStdinConn([{"type": "exit", "code": 0}], buffer_frames=1) From b421b0d7ea6476d889c41c30a9ccfbf437164a7e Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 20 Sep 2026 11:25:56 +0800 Subject: [PATCH 07/53] fix(sdk/python): retire proxy_port's accept thread when the listener closes close() from another thread does not wake a blocked accept(2) on Linux, so every proxy_port call left its accept thread parked for the life of the process; macOS does wake it, which kept the suite green. The loop now accepts under a poll timeout and leaves on the closed socket's OSError. Measured in python:3.11-slim and python:3.9-slim: five closed listeners left 5 threads before, 0 after. --- sdk/python/cocoonsandbox/sandbox.py | 13 ++++++++++--- sdk/python/tests/test_proxy.py | 11 +++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/sdk/python/cocoonsandbox/sandbox.py b/sdk/python/cocoonsandbox/sandbox.py index d3446bf4..b63085cd 100644 --- a/sdk/python/cocoonsandbox/sandbox.py +++ b/sdk/python/cocoonsandbox/sandbox.py @@ -18,6 +18,8 @@ if TYPE_CHECKING: from .client import Client +_ACCEPT_POLL_SECONDS = 1.0 + class Sandbox: """One claimed microVM.""" @@ -425,10 +427,15 @@ def _open_stream(self, op: str, expect: str = "ready", **fields: object) -> tupl return conn, frame def _proxy_accept_loop(self, listener: socket.socket, port: int) -> None: - with contextlib.suppress(OSError): - while True: + listener.settimeout(_ACCEPT_POLL_SECONDS) + while True: + try: local, _ = listener.accept() - threading.Thread(target=self._proxy_conn, args=(local, port), daemon=True).start() + except socket.timeout: + continue + except OSError: + return + threading.Thread(target=self._proxy_conn, args=(local, port), daemon=True).start() def _proxy_conn(self, local: socket.socket, port: int) -> None: try: diff --git a/sdk/python/tests/test_proxy.py b/sdk/python/tests/test_proxy.py index 06f7f25b..03ee24cf 100644 --- a/sdk/python/tests/test_proxy.py +++ b/sdk/python/tests/test_proxy.py @@ -3,6 +3,7 @@ can't see it, only running the accept loop does.""" import socket +import threading import time from conftest import sandbox_at @@ -86,3 +87,13 @@ def test_proxy_port_ends_the_local_connection_when_the_guest_port_is_closed(monk c.close() finally: listener.close() + + +def test_proxy_port_retires_its_accept_thread_when_the_listener_closes(): + sb = sandbox_at("127.0.0.1:1", keep_alive=0) + known = set(threading.enumerate()) + listener = sb.proxy_port("127.0.0.1:0", 80) + (accept_loop,) = set(threading.enumerate()) - known + listener.close() + accept_loop.join(5) + assert not accept_loop.is_alive() From b114a4cd2f03daff438720962f21fbdc1c828a18 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 20 Sep 2026 11:26:44 +0800 Subject: [PATCH 08/53] fix(sdk/openai): keep one sandbox handle per session _sandbox() built a fresh Client and Sandbox for every operation, so the SDK's keep-alive pool never served a second call: each operation paid a relay upgrade and an info probe, then parked its connection for the 30 s keep-alive window under its own timer, holding the sandbox's idle clock. The session state is set once at construction; delete releases through the same handle, which drains its pool. --- sdk/openai/cocoonsandbox_openai/adapter.py | 9 ++++++--- sdk/openai/tests/test_adapter.py | 11 +++++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/sdk/openai/cocoonsandbox_openai/adapter.py b/sdk/openai/cocoonsandbox_openai/adapter.py index 21834ede..8801311b 100644 --- a/sdk/openai/cocoonsandbox_openai/adapter.py +++ b/sdk/openai/cocoonsandbox_openai/adapter.py @@ -53,6 +53,7 @@ class CocoonSandboxSession(BaseSandboxSession): def __init__(self, *, state: CocoonSandboxSessionState) -> None: self.state = state self._proxies: list[socket.socket] = [] + self._sb: Sandbox | None = None @classmethod def from_state(cls, state: CocoonSandboxSessionState) -> CocoonSandboxSession: @@ -120,9 +121,11 @@ async def _shutdown_backend(self) -> None: self._proxies.clear() def _sandbox(self) -> Sandbox: - s = self.state - client = Client(s.addr, api_token=s.api_token) - return Sandbox(client=client, id=s.sandbox_id, token=s.sandbox_token, owner=s.owner or s.addr) + if self._sb is None: + s = self.state + client = Client(s.addr, api_token=s.api_token) + self._sb = Sandbox(client=client, id=s.sandbox_id, token=s.sandbox_token, owner=s.owner or s.addr) + return self._sb def _abs(self, path: Path | str) -> Path: path = Path(path) diff --git a/sdk/openai/tests/test_adapter.py b/sdk/openai/tests/test_adapter.py index 146738ed..48ed3b91 100644 --- a/sdk/openai/tests/test_adapter.py +++ b/sdk/openai/tests/test_adapter.py @@ -66,6 +66,17 @@ async def go(): asyncio.run(go()) +def test_operations_share_one_sandbox_handle(node): + async def go(): + client = CocoonSandboxClient() + session = await client.create(options=CocoonSandboxClientOptions(addr=node, template="rt:24.04")) + inner = session._inner + assert inner._sandbox() is inner._sandbox() + await client.delete(session) + + asyncio.run(go()) + + def test_exec_maps_stdio_and_exit(node, monkeypatch): class FakeSandbox: def __init__(self, **kw): From d3cbb620e1b452e86ea0a60c515be959945fadae Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 20 Sep 2026 11:28:02 +0800 Subject: [PATCH 09/53] fix(sdk/langchain): bound every tool's first claim by the call budget The claim deadline from #212 reached sandbox_exec only. write_file, read_file and list_dir claimed with no deadline, so a run whose first tool call is a file write still walked a black-holed redirect at one client timeout per hop. The toolkit's sandbox() now defaults the claim deadline to CALL_TIMEOUT from the call. --- sdk/langchain/cocoonsandbox_langchain/toolkit.py | 4 ++-- sdk/langchain/tests/test_toolkit.py | 12 ++++++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/sdk/langchain/cocoonsandbox_langchain/toolkit.py b/sdk/langchain/cocoonsandbox_langchain/toolkit.py index 048eb918..eff0b6bf 100644 --- a/sdk/langchain/cocoonsandbox_langchain/toolkit.py +++ b/sdk/langchain/cocoonsandbox_langchain/toolkit.py @@ -108,12 +108,12 @@ def close(self) -> None: sb.close() def sandbox(self, deadline: float | None = None) -> Sandbox: - """The claimed sandbox, claiming (or branching) on first use within the caller's deadline.""" + """The claimed sandbox, claiming (or branching) on first use by deadline, else within CALL_TIMEOUT.""" with self._lock: if self._closed: raise RuntimeError("toolkit is closed") if self._sb is None: - self._sb = self._claim(deadline) + self._sb = self._claim(time.monotonic() + CALL_TIMEOUT if deadline is None else deadline) return self._sb def _claim(self, deadline: float | None = None) -> Sandbox: diff --git a/sdk/langchain/tests/test_toolkit.py b/sdk/langchain/tests/test_toolkit.py index 59cdcb85..714dc93e 100644 --- a/sdk/langchain/tests/test_toolkit.py +++ b/sdk/langchain/tests/test_toolkit.py @@ -43,7 +43,15 @@ def test_file_tools_round_trip(monkeypatch): assert "a.txt" in list_dir.invoke({"path": "/w"}) -def test_exec_claims_inside_the_call_budget(monkeypatch): +@pytest.mark.parametrize( + ("name", "args"), + [ + ("sandbox_exec", {"command": "echo hi"}), + ("sandbox_write_file", {"path": "/w/a.txt", "content": "body"}), + ("sandbox_list_dir", {"path": "/w"}), + ], +) +def test_first_tool_call_claims_inside_the_call_budget(monkeypatch, name, args): kit = CocoonToolkit("127.0.0.1:1") seen = [] @@ -52,7 +60,7 @@ def claim(deadline=None): return FakeSandbox() monkeypatch.setattr(kit, "_claim", claim) - kit.get_tools()[0].invoke({"command": "echo hi"}) + next(t for t in kit.get_tools() if t.name == name).invoke(args) assert len(seen) == 1 and seen[0] is not None assert 0 < seen[0] - time.monotonic() <= CALL_TIMEOUT, seen[0] From de375229804b6c2bd1b0815f51f4f7f055d57376 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 20 Sep 2026 11:28:45 +0800 Subject: [PATCH 10/53] fix(sdk/langchain): return SDK failures to the model as tool errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit handle_tool_error=True makes LangChain catch ToolException only, and the toolkit raised one for a missing read_file path alone. Every other failure — write_file into a missing directory, an expired claim, a guest error — escaped tool.invoke() and ended the agent run. _tool now maps SandboxError to ToolException in one place, which also replaces read_file's special case. docs/langchain.md states this and the claim budget every tool's first call runs under. --- docs/langchain.md | 5 +++++ .../cocoonsandbox_langchain/toolkit.py | 19 ++++++++++--------- sdk/langchain/tests/test_toolkit.py | 11 +++++++++++ 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/docs/langchain.md b/docs/langchain.md index dcb29525..901626a1 100644 --- a/docs/langchain.md +++ b/docs/langchain.md @@ -25,6 +25,11 @@ schemas, sync-native with `asyncio.to_thread` async bridges): | `sandbox_read_file` | read a text file | | `sandbox_list_dir` | list a directory as JSON | +A failed call — a missing path, a guest error, an expired or unreachable +sandbox — comes back to the model as the tool's error text, not as an +exception out of the agent run. Every tool's first call claims the sandbox +inside the same 5-minute budget. + **Branching**: `CocoonToolkit(..., from_checkpoint="ck_...")` claims the sandbox from a [checkpoint](sdk-python.md#checkpoints--branching-and-time-travel)'s captured moment instead of a clean template — agents start from prepared diff --git a/sdk/langchain/cocoonsandbox_langchain/toolkit.py b/sdk/langchain/cocoonsandbox_langchain/toolkit.py index eff0b6bf..b30bbb9f 100644 --- a/sdk/langchain/cocoonsandbox_langchain/toolkit.py +++ b/sdk/langchain/cocoonsandbox_langchain/toolkit.py @@ -11,7 +11,7 @@ import time from collections.abc import Callable -from cocoonsandbox import Client, Sandbox, SilkdError +from cocoonsandbox import Client, Sandbox, SandboxError from langchain_core.tools import StructuredTool, ToolException from pydantic import BaseModel, Field @@ -122,11 +122,17 @@ def _claim(self, deadline: float | None = None) -> Sandbox: return self._client.new(self._template, net=self._net, ttl_seconds=self._ttl, deadline=deadline) def _tool(self, name: str, description: str, schema: type[BaseModel], func: Callable[..., str]) -> StructuredTool: + def run(**kwargs: object) -> str: + try: + return func(**kwargs) + except SandboxError as exc: + raise ToolException(str(exc)) from exc + async def arun(**kwargs): - return await asyncio.to_thread(func, **kwargs) + return await asyncio.to_thread(run, **kwargs) return StructuredTool.from_function( - func=func, coroutine=arun, name=name, description=description, args_schema=schema, handle_tool_error=True + func=run, coroutine=arun, name=name, description=description, args_schema=schema, handle_tool_error=True ) def _exec(self, command: str, cwd: str = "") -> str: @@ -158,12 +164,7 @@ def _write_file(self, path: str, content: str) -> str: return f"wrote {path}" def _read_file(self, path: str) -> str: - try: - return self.sandbox().read_file(path).decode(errors="replace") - except SilkdError as exc: - if exc.kind != "not_found": - raise - raise ToolException(f"{path}: no such file") from None + return self.sandbox().read_file(path).decode(errors="replace") def _list_dir(self, path: str) -> str: return json.dumps(self.sandbox().list_dir(path)) diff --git a/sdk/langchain/tests/test_toolkit.py b/sdk/langchain/tests/test_toolkit.py index 714dc93e..d7bc31a7 100644 --- a/sdk/langchain/tests/test_toolkit.py +++ b/sdk/langchain/tests/test_toolkit.py @@ -75,6 +75,17 @@ def claim(deadline=None): assert kit.get_tools()[0].invoke({"command": "echo hi"}) == f"cut off after {CALL_TIMEOUT}s" +def test_a_guest_error_comes_back_as_a_tool_error(monkeypatch): + kit, fake = hooked(monkeypatch) + + def refused(path, data): + raise SilkdError("not_found", "no such directory") + + fake.write_file = refused + tool = next(t for t in kit.get_tools() if t.name == "sandbox_write_file") + assert "no such directory" in tool.invoke({"path": "/nope/a.txt", "content": "body"}) + + def test_close_releases_once(monkeypatch): kit, fake = hooked(monkeypatch) kit.get_tools()[0].invoke({"command": "x"}) From a76156793a11ab8276ad79b0a4a2cdf3792803a5 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 20 Sep 2026 11:30:14 +0800 Subject: [PATCH 11/53] fix(ci): lock the released silkd build and run the Go gate on its own config release.yml built the amd64 silkd asset without --locked while the arm64 leg and every image build use it, so one release could ship two dependency sets. sandboxd.yml did not trigger on Makefile or .golangci.yml, the two files that define its lint and test commands. The golangci-lint install ran under the host's go: on a host older than go.mod's directive the fresh linter refuses the config, so the install now pins GOTOOLCHAIN to that directive. --- .github/workflows/release.yml | 2 +- .github/workflows/sandboxd.yml | 4 ++++ Makefile | 3 ++- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f1e6d8a1..5bd00b78 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -25,7 +25,7 @@ jobs: - name: Build silkd (musl static) run: | sudo apt-get update && sudo apt-get install -y --no-install-recommends musl-tools - (cd silkd && cargo build --release --target x86_64-unknown-linux-musl) + (cd silkd && cargo build --release --locked --target x86_64-unknown-linux-musl) strip silkd/target/x86_64-unknown-linux-musl/release/silkd mkdir -p dist-extra tar -C silkd/target/x86_64-unknown-linux-musl/release \ diff --git a/.github/workflows/sandboxd.yml b/.github/workflows/sandboxd.yml index cb05e862..1f9466c4 100644 --- a/.github/workflows/sandboxd.yml +++ b/.github/workflows/sandboxd.yml @@ -13,6 +13,8 @@ on: - "mcp/**" - "e2e/**" - "protocol/**" + - "Makefile" + - ".golangci.yml" - ".github/workflows/sandboxd.yml" pull_request: paths: @@ -22,6 +24,8 @@ on: - "mcp/**" - "e2e/**" - "protocol/**" + - "Makefile" + - ".golangci.yml" - ".github/workflows/sandboxd.yml" workflow_dispatch: diff --git a/Makefile b/Makefile index ae7b48e6..d55553c3 100644 --- a/Makefile +++ b/Makefile @@ -22,6 +22,7 @@ $(LOCALBIN): ## Tool versions GOLANGCILINT_VERSION ?= v2.13.2 +GO_VERSION := $(shell sed -n 's/^go //p' sandboxd/go.mod) GOLANGCILINT_ROOT := $(LOCALBIN)/golangci-lint-$(GOLANGCILINT_VERSION) GOLANGCILINT := $(GOLANGCILINT_ROOT)/golangci-lint @@ -32,7 +33,7 @@ GOLANGCILINT := $(GOLANGCILINT_ROOT)/golangci-lint .PHONY: golangci-lint golangci-lint: $(GOLANGCILINT) $(GOLANGCILINT): - GOBIN=$(GOLANGCILINT_ROOT) go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@$(GOLANGCILINT_VERSION) + GOBIN=$(GOLANGCILINT_ROOT) GOTOOLCHAIN=go$(GO_VERSION) go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@$(GOLANGCILINT_VERSION) cloc: ## Count lines of code excluding tests (requires cloc) cloc --exclude-dir=target,dist,node_modules --exclude-ext=json \ From 4feb01a2fec9b9fb4099e224bb0e5c6b9f7f4877 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 20 Sep 2026 11:30:14 +0800 Subject: [PATCH 12/53] fix(images): track the released silkd in the android default, assert the pty symbol The android flavor's SILKD_IMAGE default stayed at 0.1.0 through twelve releases; CI overrides it, a local build does not. The kernel's post-olddefconfig assertion list did not cover UNIX98_PTYS, which silkd's pty RPC needs and which sits under EXPERT with VT and LEGACY_PTYS off. --- boot/Dockerfile | 2 +- os-image/android/15/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/boot/Dockerfile b/boot/Dockerfile index 4699732c..1720e9c3 100644 --- a/boot/Dockerfile +++ b/boot/Dockerfile @@ -55,7 +55,7 @@ RUN set -e; \ make olddefconfig; \ for s in $arch_asserts DEVTMPFS BLK_DEV_INITRD \ VIRTIO_PCI VIRTIO_BLK VIRTIO_NET \ - VIRTIO_CONSOLE VIRTIO_VSOCKETS \ + VIRTIO_CONSOLE VIRTIO_VSOCKETS UNIX98_PTYS \ EROFS_FS EROFS_FS_ZIP OVERLAY_FS EXT4_FS \ NAMESPACES USER_NS SECCOMP_FILTER \ BRIDGE VETH BRIDGE_NETFILTER NF_NAT NFT_COMPAT \ diff --git a/os-image/android/15/Dockerfile b/os-image/android/15/Dockerfile index 05d07785..b958ed3b 100644 --- a/os-image/android/15/Dockerfile +++ b/os-image/android/15/Dockerfile @@ -9,7 +9,7 @@ ARG ANDROID_BASE_IMAGE=ghcr.io/cocoonstack/cocoon/android:15.0 # silkd carrier image (built from silkd/Dockerfile); CI passes the # version-pinned tag. -ARG SILKD_IMAGE=ghcr.io/cocoonstack/sandbox/silkd:0.1.0 +ARG SILKD_IMAGE=ghcr.io/cocoonstack/sandbox/silkd:0.1.12 FROM ${SILKD_IMAGE} AS silkd From f7f8d7d1748eb3664dce7286c48d67d4ea17109c Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 20 Sep 2026 11:30:14 +0800 Subject: [PATCH 13/53] fix(e2e): fail loudly instead of passing on an unmet premise The egress, intercept and socks scripts waited for a warm pool and fell through on timeout, so the smoke then passed at the cold tier; a kit without jq takes exactly that path. They now exit the way sandboxd-e2e.sh does. meshsmoke printed PASS with both owner assertions skipped when -peer was empty; it now requires the flag. --- e2e/cmd/meshsmoke/main.go | 8 ++++++-- scripts/egress-e2e.sh | 3 ++- scripts/intercept-e2e.sh | 3 ++- scripts/socks-e2e.sh | 3 ++- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/e2e/cmd/meshsmoke/main.go b/e2e/cmd/meshsmoke/main.go index 2807ee62..be00c5af 100644 --- a/e2e/cmd/meshsmoke/main.go +++ b/e2e/cmd/meshsmoke/main.go @@ -7,6 +7,7 @@ package main import ( "context" + "errors" "flag" "fmt" "os" @@ -32,6 +33,9 @@ func main() { } func run(addr, peer, token, template string) error { + if peer == "" { + return errors.New("peer is required") + } ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second) defer cancel() @@ -40,7 +44,7 @@ func run(addr, peer, token, template string) error { return fmt.Errorf("via %s: %w", addr, err) } defer func() { _ = sb.Close() }() - if peer != "" && sb.Owner() != peer { + if sb.Owner() != peer { return fmt.Errorf("claim owner %s, want redirect to %s", sb.Owner(), peer) } fmt.Printf(" redirect: claim entered at %s, owned by %s\n", addr, sb.Owner()) @@ -76,7 +80,7 @@ func run(addr, peer, token, template string) error { if got := strings.TrimSpace(out); got != "mesh-marker" { return fmt.Errorf("marker %q, want mesh-marker (clone not from the promoted golden)", got) } - if peer != "" && clone.Owner() != peer { + if clone.Owner() != peer { return fmt.Errorf("clone owner %s, want %s (template owner)", clone.Owner(), peer) } fmt.Printf(" name-based claim: clone on %s carries the marker\n", clone.Owner()) diff --git a/scripts/egress-e2e.sh b/scripts/egress-e2e.sh index efab59d8..1a26b33c 100755 --- a/scripts/egress-e2e.sh +++ b/scripts/egress-e2e.sh @@ -81,9 +81,10 @@ echo "== start sandboxd ($NET-lane pool + egress policy, reach=$REACH)" DAEMON_PID=$! for _ in $(seq 1 40); do curl -sf "http://$ADDR/healthz" >/dev/null && break; sleep 0.5; done # Wait for the warm pool so the claim is a warm hit (proxy armed at claim). -for _ in $(seq 1 120); do +for i in $(seq 1 120); do curl -sf -H "Authorization: Bearer $TOKEN" "http://$ADDR/v1/info" 2>/dev/null | jq -e '.pools[0].warm >= 1' >/dev/null 2>&1 && break + [[ $i == 120 ]] && { echo "pool never became warm"; exit 1; } sleep 1 done diff --git a/scripts/intercept-e2e.sh b/scripts/intercept-e2e.sh index e0f64ef3..01b6bbdd 100755 --- a/scripts/intercept-e2e.sh +++ b/scripts/intercept-e2e.sh @@ -73,9 +73,10 @@ echo "== start sandboxd (intercept pool; golden build bakes the cluster root via "$DATA/sandboxd" -config "$DATA/config.json" >>"$DATA/daemon.log" 2>&1 & DAEMON_PID=$! for _ in $(seq 1 40); do curl -sf "http://$ADDR/healthz" >/dev/null 2>&1 && break; sleep 0.5; done -for _ in $(seq 1 300); do +for i in $(seq 1 300); do curl -sf -H "Authorization: Bearer $TOKEN" "http://$ADDR/v1/info" 2>/dev/null | jq -e '.pools[0].warm >= 1' >/dev/null 2>&1 && break + [[ $i == 300 ]] && { echo "pool never became warm"; exit 1; } sleep 1 done diff --git a/scripts/socks-e2e.sh b/scripts/socks-e2e.sh index b9b9ab14..90762044 100755 --- a/scripts/socks-e2e.sh +++ b/scripts/socks-e2e.sh @@ -70,9 +70,10 @@ echo "== start sandboxd (small pool opted into SOCKS5, medium pool without a pol "$DATA/sandboxd" -config "$DATA/config.json" >>"$DATA/daemon.log" 2>&1 & DAEMON_PID=$! for _ in $(seq 1 40); do curl -sf "http://$ADDR/healthz" >/dev/null && break; sleep 0.5; done -for _ in $(seq 1 180); do +for i in $(seq 1 180); do curl -sf -H "Authorization: Bearer $TOKEN" "http://$ADDR/v1/info" 2>/dev/null | jq -e '(.pools | length) == 3 and all(.pools[]; .warm >= 1)' >/dev/null 2>&1 && break + [[ $i == 180 ]] && { echo "pools never became warm"; exit 1; } sleep 1 done From 02fbbbbfe3db0cb3e10d1b701b160bd38608eeb4 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 20 Sep 2026 11:30:50 +0800 Subject: [PATCH 14/53] review: godoc on the func it names, annotated constructor, tests ahead of helpers runBounded's godoc sat on destroyAll (/code comment budget: a comment describes the declaration it sits on). Template.__init__ lacked its return annotation (/code-py item 4). The langchain suite's last test followed its helpers. --- sandboxd/pool/refill.go | 2 +- sdk/langchain/tests/test_toolkit.py | 22 +++++++++++----------- sdk/python/cocoonsandbox/template.py | 2 +- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/sandboxd/pool/refill.go b/sandboxd/pool/refill.go index 2f77ab76..b25f9ae7 100644 --- a/sandboxd/pool/refill.go +++ b/sandboxd/pool/refill.go @@ -469,11 +469,11 @@ func (m *Manager) findVM(ctx context.Context, name string) (types.VMRecord, bool return vms[i], true, nil } -// runBounded fans f over n items on the refill semaphore, sharing the node-wide budget. func (m *Manager) destroyAll(ctx context.Context, names []string) *sync.WaitGroup { return m.runBounded(ctx, len(names), func(ctx context.Context, i int) { m.destroy(ctx, names[i]) }) } +// runBounded fans f over n items on the refill semaphore, sharing the node-wide budget. func (m *Manager) runBounded(ctx context.Context, n int, f func(context.Context, int)) *sync.WaitGroup { var wg sync.WaitGroup for i := range n { diff --git a/sdk/langchain/tests/test_toolkit.py b/sdk/langchain/tests/test_toolkit.py index d7bc31a7..decc12ca 100644 --- a/sdk/langchain/tests/test_toolkit.py +++ b/sdk/langchain/tests/test_toolkit.py @@ -101,6 +101,17 @@ def test_use_after_close_raises(monkeypatch): kit.sandbox() +def test_read_file_reports_a_missing_path_as_a_tool_error(monkeypatch): + kit, fake = hooked(monkeypatch) + + def missing(path): + raise SilkdError("not_found", "no such file") + + fake.read_file = missing + tool = next(t for t in kit.get_tools() if t.name == "sandbox_read_file") + assert "no such file" in tool.invoke({"path": "/nope"}) + + class FakeSandbox: def __init__(self): self.closed = 0 @@ -136,14 +147,3 @@ def hooked(monkeypatch): fake = FakeSandbox() monkeypatch.setattr(kit, "_claim", lambda deadline=None: fake) return kit, fake - - -def test_read_file_reports_a_missing_path_as_a_tool_error(monkeypatch): - kit, fake = hooked(monkeypatch) - - def missing(path): - raise SilkdError("not_found", "no such file") - - fake.read_file = missing - tool = next(t for t in kit.get_tools() if t.name == "sandbox_read_file") - assert "no such file" in tool.invoke({"path": "/nope"}) diff --git a/sdk/python/cocoonsandbox/template.py b/sdk/python/cocoonsandbox/template.py index 7573ce29..a7261c54 100644 --- a/sdk/python/cocoonsandbox/template.py +++ b/sdk/python/cocoonsandbox/template.py @@ -14,7 +14,7 @@ class Template: """A promoted template on its owner node.""" - def __init__(self, client: Client, addr: str, name: str, net: str, size: str, content_digest: str = ""): + def __init__(self, client: Client, addr: str, name: str, net: str, size: str, content_digest: str = "") -> None: self._client = client self._addr = addr self.name = name From a824782a69cb6bf9aacfd1d28b157447a3e3c2ed Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 20 Sep 2026 11:31:06 +0800 Subject: [PATCH 15/53] docs: the sandbox index fields and the intercepting proxy's idle ceiling docs/sdk.md and docs/sdk-python.md listed four of the eight SandboxSummary fields as if complete; both now point at GET /v1/sandboxes, which owns the list. docs/egress.md gave 64 idle upstream connections per sandbox; an intercepting pool's proxy holds a second transport of the same size. --- docs/egress.md | 5 +++-- docs/sdk-python.md | 5 +++-- docs/sdk.md | 5 +++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/docs/egress.md b/docs/egress.md index 78851cba..0955201d 100644 --- a/docs/egress.md +++ b/docs/egress.md @@ -44,8 +44,9 @@ supported*), no authentication (the socket path already carries the sandbox's identity), and a `DOMAINNAME` destination is resolved host-side, so the guest still needs no resolver. Each door serves at most 256 connections per sandbox at a time; later ones sit unserved in the socket backlog until one closes, and -the proxy keeps at most 64 idle upstream connections per sandbox, so a guest -cannot turn its proxy into a host descriptor sink. +the proxy keeps at most 64 idle upstream connections per sandbox — 64 more for +decrypted traffic when the pool intercepts — so a guest cannot turn its proxy +into a host descriptor sink. A SOCKS5 tunnel takes the decision an HTTP `CONNECT` to the same host takes, through the same code: a rule with a nonempty `methods` list that omits diff --git a/docs/sdk-python.md b/docs/sdk-python.md index cb2641cd..ecc1d41c 100644 --- a/docs/sdk-python.md +++ b/docs/sdk-python.md @@ -459,14 +459,15 @@ pty.close() ```python sb = client.new("rt:24.04", claim_ref="ns/workload") -client.sandboxes() # id, key, deadline, claim_ref — never tokens +client.sandboxes() # one summary dict per live claim — never tokens client.drain() # cordon: refuse new claims, run leases out client.uncordon() client.attach(owner_addr, id, token) # bind a known handle, no lookup round-trip ``` `sandboxes()` is scoped to the calling token, so a tenant sees only its own -claims. `drain()` leaves live claims alone — poll `info()` until `claimed` is +claims; the fields are those of [`GET /v1/sandboxes`](sandboxd-api.md#get-v1sandboxes). +`drain()` leaves live claims alone — poll `info()` until `claimed` is zero. ## Errors diff --git a/docs/sdk.md b/docs/sdk.md index 17fb3cf7..b3eee6de 100644 --- a/docs/sdk.md +++ b/docs/sdk.md @@ -591,7 +591,7 @@ apiserver claims under: ```go sb, _ := client.New(ctx, "rt:24.04", sandbox.WithClaimRef("ns/workload")) -list, _ := client.Sandboxes(ctx) // id, key, deadline, claim_ref — never tokens +list, _ := client.Sandboxes(ctx) // one SandboxSummary per live claim — never tokens info, _ := client.Drain(ctx) // cordon: refuse new claims, run leases out info, _ = client.Uncordon(ctx) info, _ = client.SetPools(ctx, pools) // retune warm targets without a restart @@ -600,7 +600,8 @@ sb = client.Attach(ownerAddr, id, token) // bind a known handle, no lookup roun ``` `Sandboxes` is scoped to the calling token, so a tenant sees only its own -claims. `Drain` leaves live claims alone — poll `Info` until `Claimed` is zero. +claims; the fields are those of [`GET /v1/sandboxes`](sandboxd-api.md#get-v1sandboxes). +`Drain` leaves live claims alone — poll `Info` until `Claimed` is zero. ## Error handling From fc3f2f0734428ea280bc970e17b614c282b0d264 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 20 Sep 2026 11:32:30 +0800 Subject: [PATCH 16/53] fix(sdk/python): keep the relay connection when a stream open is refused The Python twin of the Go SDK fix: _open_stream closed the connection on a guest error frame while _lease parks it, so a dial_port poll on a port that is not listening yet re-dialed and re-upgraded on every attempt. --- sdk/python/cocoonsandbox/sandbox.py | 3 +++ sdk/python/tests/test_keepalive.py | 14 ++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/sdk/python/cocoonsandbox/sandbox.py b/sdk/python/cocoonsandbox/sandbox.py index b63085cd..3ffcc9d0 100644 --- a/sdk/python/cocoonsandbox/sandbox.py +++ b/sdk/python/cocoonsandbox/sandbox.py @@ -421,6 +421,9 @@ def _open_stream(self, op: str, expect: str = "ready", **fields: object) -> tupl try: conn.send(op, **fields) frame = _expect(conn, expect) + except SilkdError: + self._park(conn) + raise except BaseException: conn.close() raise diff --git a/sdk/python/tests/test_keepalive.py b/sdk/python/tests/test_keepalive.py index 8e57fd4c..37e53e6f 100644 --- a/sdk/python/tests/test_keepalive.py +++ b/sdk/python/tests/test_keepalive.py @@ -10,6 +10,7 @@ import pytest from conftest import accept_upgrade, sandbox_at, wait_until +from cocoonsandbox import SilkdError from cocoonsandbox.frames import KEEP_ALIVE_PROTO INPUT_OPS = ("stdin", "stdin_close", "data", "data_end") @@ -28,6 +29,19 @@ def test_calls_share_one_connection(): assert agent.upgrades == 1 +def test_a_refused_stream_open_keeps_its_connection(): + agent = FakeAgent() + sb = sandbox_at(agent.addr) + try: + for _ in range(3): + with pytest.raises(SilkdError): + sb.watch("/missing") + finally: + sb._pool.drain() + agent.stop() + assert agent.upgrades == 1 + + def test_old_daemon_dials_per_call(): agent = FakeAgent(proto=1) sb = sandbox_at(agent.addr) From dcbfd6f977796f6a6a5c2c8ae6a6321476680e12 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 20 Sep 2026 12:11:59 +0800 Subject: [PATCH 17/53] fix(pool): let a warm 0 pool under warm_max grow, and return to empty The target is ceil(rate x lead x safety). The lead is measured only by a refill and a refill needs a target above zero, so a pool configured warm 0 with a warm_max stayed at zero under any load: every claim took the provision path. With no measured lead, demand now asks for one VM, whose refill measures it. The ceiling of any positive rate is 1 and the rate only decays exponentially, so an empty-floor pool would then hold one idle VM until the float underflowed, about twelve hours; under one arrival per decay period now counts as silence. Both branches are the identity for a floor of one or more. --- sandboxd/pool/watermark.go | 6 +++++ sandboxd/pool/watermark_test.go | 39 +++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/sandboxd/pool/watermark.go b/sandboxd/pool/watermark.go index 71fd8088..e37199cb 100644 --- a/sandboxd/pool/watermark.go +++ b/sandboxd/pool/watermark.go @@ -49,6 +49,12 @@ func (p *pool) effectiveTarget(now time.Time) int { if !p.lastArrival.IsZero() { rate *= math.Exp(-now.Sub(p.lastArrival).Seconds() / rateDecayTau.Seconds()) } + if rate*rateDecayTau.Seconds() < 1 { + return p.floor + } + if p.lead == 0 { + return max(p.floor, 1) + } dynamic := int(math.Ceil(rate * p.lead.Seconds() * leadSafety)) return max(p.floor, min(dynamic, p.warmMax)) } diff --git a/sandboxd/pool/watermark_test.go b/sandboxd/pool/watermark_test.go index 7375923f..c88a9b31 100644 --- a/sandboxd/pool/watermark_test.go +++ b/sandboxd/pool/watermark_test.go @@ -76,6 +76,45 @@ func TestEffectiveTargetOffWithoutWarmMax(t *testing.T) { } } +func TestEffectiveTargetGrowsAnEmptyPoolAndReturnsItToEmpty(t *testing.T) { + now := time.Now() + p := &pool{key: types.PoolKey{}, warmMax: 8} + for i := range 60 { + p.noteArrival(now.Add(time.Duration(i) * 100 * time.Millisecond)) + } + busy := now.Add(6 * time.Second) + if got := p.effectiveTarget(busy); got != 1 { + t.Fatalf("target %d under demand with no measured lead, want 1 so a refill can measure it", got) + } + p.noteLead(500 * time.Millisecond) + if got := p.effectiveTarget(busy); got != 8 { + t.Errorf("target %d at 10/s with a 500 ms lead, want warmMax 8", got) + } + if got := p.effectiveTarget(busy.Add(10 * time.Minute)); got != 0 { + t.Errorf("target %d after ten silent minutes, want the empty floor", got) + } +} + +func TestRefillGrowsAnEmptyPoolOnDemand(t *testing.T) { + eng := newFakeEngine() + m := newTestManager(t, eng, config.PoolSpec{PoolKey: testKey, WarmMax: 4}) + p := m.pools[testKey] + p.goldenDir = "/goldens/x" + start := time.Now().Add(-3 * time.Second) + m.mu.Lock() + for i := range 30 { + p.noteArrival(start.Add(time.Duration(i) * 100 * time.Millisecond)) + } + m.mu.Unlock() + + m.refillOnce(t.Context()) + waitFor(t, func() bool { + m.mu.Lock() + defer m.mu.Unlock() + return len(p.warm) > 0 && p.lead > 0 + }) +} + func TestNoteLeadConverges(t *testing.T) { p := &pool{} p.noteLead(time.Second) From 80222bb1ababc2c3b5b783ab3c6ef012d5f5ec0d Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 20 Sep 2026 12:13:44 +0800 Subject: [PATCH 18/53] fix(pool): close an archived claim's usage interval when retention purges it The purge arm of the reap journaled archive_delete and nothing else, so the claim never got the release or reap that docs/sandboxd-api.md folds compute seconds over, and reaps_total missed it. Release of an archived claim already emits archive_delete then release; the purge now emits archive_delete then reap. --- sandboxd/pool/archive_test.go | 6 +++++- sandboxd/pool/claim.go | 2 ++ sandboxd/pool/telemetry_test.go | 38 ++++++++++++++++++--------------- 3 files changed, 28 insertions(+), 18 deletions(-) diff --git a/sandboxd/pool/archive_test.go b/sandboxd/pool/archive_test.go index d610382d..a05bfb4c 100644 --- a/sandboxd/pool/archive_test.go +++ b/sandboxd/pool/archive_test.go @@ -6,6 +6,7 @@ import ( "os" "path/filepath" "slices" + "strings" "testing" "testing/synctest" "time" @@ -333,10 +334,13 @@ func TestArchiveRetentionPurge(t *testing.T) { m.reapOnce(t.Context()) waitFor(t, func() bool { return archivedCount(m) == 0 }) - waitFor(t, func() bool { return m.Counters().ArchiveDeletes > 0 }) + waitFor(t, func() bool { return m.Counters().Reaps > 0 }) if ckExists(t, m, ck) { t.Error("purge did not delete the archived checkpoint") } + if got := usageEventsOf(t, m, sb.ID); !strings.HasSuffix(got, "archive,archive_delete,reap") { + t.Errorf("events %s, want the purge to close the claim with a reap", got) + } m.mu.Lock() live := m.tenantLive["acme"] diff --git a/sandboxd/pool/claim.go b/sandboxd/pool/claim.go index f593243e..181bae8a 100644 --- a/sandboxd/pool/claim.go +++ b/sandboxd/pool/claim.go @@ -450,6 +450,8 @@ func (m *Manager) reapOnce(ctx context.Context) { m.disarmEgress(v.id, true) m.purgeArchiveCk(ctx, v.id, v.ck, v.tenant) m.untrack(m.pendingCks, v.ck) + m.counters.reaps.Add(1) + m.recordUsage(ctx, usageEvent{Event: "reap", ID: v.id}) logger.Infof(ctx, "purged archived sandbox %s", v.id) case reapArchive: logSweepResult(ctx, logger, m.archive(ctx, v.sb), "archived expired sandbox "+v.id, "archive expired sandbox "+v.id) diff --git a/sandboxd/pool/telemetry_test.go b/sandboxd/pool/telemetry_test.go index 5095580c..e512d2fe 100644 --- a/sandboxd/pool/telemetry_test.go +++ b/sandboxd/pool/telemetry_test.go @@ -27,23 +27,8 @@ func TestUsageJournalRecordsLifecycle(t *testing.T) { t.Fatalf("Release: %v", err) } - raw, err := os.ReadFile(filepath.Join(m.dataDir, "usage.jsonl")) - if err != nil { - t.Fatalf("read journal: %v", err) - } - var events []string - for line := range strings.SplitSeq(strings.TrimSpace(string(raw)), "\n") { - var ev usageEvent - if err := json.Unmarshal([]byte(line), &ev); err != nil { - t.Fatalf("bad journal line %q: %v", line, err) - } - if ev.ID == sb.ID { - events = append(events, ev.Event) - } - } - want := []string{"claim", "hibernate", "wake", "release"} - if strings.Join(events, ",") != strings.Join(want, ",") { - t.Errorf("events %v, want %v", events, want) + if got, want := usageEventsOf(t, m, sb.ID), "claim,hibernate,wake,release"; got != want { + t.Errorf("events %s, want %s", got, want) } c := m.Counters() @@ -216,3 +201,22 @@ func TestPreviewDialWritesAuditEvent(t *testing.T) { t.Errorf("audit event %+v", ev) } } + +func usageEventsOf(t *testing.T, m *Manager, id string) string { + t.Helper() + raw, err := os.ReadFile(filepath.Join(m.dataDir, "usage.jsonl")) + if err != nil { + t.Fatalf("read journal: %v", err) + } + var events []string + for line := range strings.SplitSeq(strings.TrimSpace(string(raw)), "\n") { + var ev usageEvent + if err := json.Unmarshal([]byte(line), &ev); err != nil { + t.Fatalf("bad journal line %q: %v", line, err) + } + if ev.ID == id { + events = append(events, ev.Event) + } + } + return strings.Join(events, ",") +} From 537fa83ce607947d2635a6d31022729c35063c5a Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 20 Sep 2026 12:13:44 +0800 Subject: [PATCH 19/53] fix(pool): journal a hibernate that reconcile adopts A daemon killed between the journaled hibernate intent and the usage event restarts, adopts the claim as hibernated, and later wakes it: the journal then held a wake with no hibernate and the whole hibernated span folded into billable compute. settlePendingSnap already journals the adoption it makes in-process; reconcile now does the same, after it drops the manager mutex. --- sandboxd/pool/reconcile.go | 5 +++++ sandboxd/pool/reconcile_test.go | 27 +++++++++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/sandboxd/pool/reconcile.go b/sandboxd/pool/reconcile.go index 8d830012..3ee41aed 100644 --- a/sandboxd/pool/reconcile.go +++ b/sandboxd/pool/reconcile.go @@ -36,6 +36,7 @@ func (m *Manager) Reconcile(ctx context.Context) error { owned := map[string]bool{} referenced := map[string]bool{} + var adopted []*types.Sandbox m.mu.Lock() for id, sb := range claims { rec, ok := live[sb.VMName] @@ -55,6 +56,7 @@ func (m *Manager) Reconcile(ctx context.Context) error { case ok && sb.PendingSnap != "" && (snapsErr != nil || slices.Contains(snaps, sb.PendingSnap)): // the journaled intent names the wake image, so adopt it when the snapshot is there sb.HibernateSnap, sb.PendingSnap = sb.PendingSnap, "" + adopted = append(adopted, sb) default: continue } @@ -69,6 +71,9 @@ func (m *Manager) Reconcile(ctx context.Context) error { m.adoptGolden(p) } m.mu.Unlock() + for _, sb := range adopted { + m.recordHibernate(ctx, sb) + } // a crash mid-export leaves a golden *.tmp or a fork-* staging dir nothing in this life reuses tmps, _ := filepath.Glob(filepath.Join(m.goldensDir(), "*.tmp")) forks, _ := filepath.Glob(filepath.Join(m.dataDir, "fork-*")) diff --git a/sandboxd/pool/reconcile_test.go b/sandboxd/pool/reconcile_test.go index 799cc3e1..66f66a61 100644 --- a/sandboxd/pool/reconcile_test.go +++ b/sandboxd/pool/reconcile_test.go @@ -6,12 +6,39 @@ import ( "os" "path/filepath" "slices" + "strings" "testing" + "github.com/cocoonstack/sandbox/sandboxd/config" "github.com/cocoonstack/sandbox/sandboxd/engine" "github.com/cocoonstack/sandbox/sandboxd/types" ) +func TestReconcileJournalsAnAdoptedHibernate(t *testing.T) { + eng := newFakeEngine() + dir := t.TempDir() + m := newTestManagerAt(t, eng, dir, config.PoolSpec{PoolKey: testKey, Warm: 1}) + sb := mustClaim(t, m, testKey) + snap := hibernatePrefix + strings.TrimPrefix(sb.VMName, vmPrefix) + "-abc" + if err := eng.Hibernate(t.Context(), sb.VMName, snap); err != nil { + t.Fatalf("engine hibernate: %v", err) + } + if err := m.store.commit(m.setPendingSnap(sb, snap)); err != nil { + t.Fatalf("journal intent: %v", err) + } + + restarted := newTestManagerAt(t, eng, dir, config.PoolSpec{PoolKey: testKey, Warm: 1}) + if err := restarted.Reconcile(t.Context()); err != nil { + t.Fatalf("reconcile: %v", err) + } + if _, _, err := restarted.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { + t.Fatalf("wake: %v", err) + } + if got, want := usageEventsOf(t, restarted, sb.ID), "claim,hibernate,wake"; got != want { + t.Errorf("events %s, want %s", got, want) + } +} + func TestReconcileStaleCreateSweep(t *testing.T) { tests := []staleCreateCase{ {"collected frees the record", engine.StaleCreateCollected, nil, false, false, false}, From bf38fd9561cb0c6859ffc4045f6ec70361dc2520 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 20 Sep 2026 12:15:15 +0800 Subject: [PATCH 20/53] fix(store/s3): do not tie a shared download to the caller that started it Fetch ran populate inside singleflight.Do under the first caller's request context. When that client hung up the download was cancelled and discarded, and every concurrent brancher of the same checkpoint got its context canceled back as a 500. A download that outlasts the client timeout never completed at all: each retry restarted from zero and died at the same point. The flight now runs detached under its own 30 minute budget, the shape healCheckpoint already uses. --- sandboxd/store/s3/s3.go | 6 ++++- sandboxd/store/s3/s3_test.go | 44 +++++++++++++++++++++++++++++++++++- 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/sandboxd/store/s3/s3.go b/sandboxd/store/s3/s3.go index 71edd48b..56e3e2d9 100644 --- a/sandboxd/store/s3/s3.go +++ b/sandboxd/store/s3/s3.go @@ -15,6 +15,7 @@ import ( "regexp" "slices" "strings" + "time" "github.com/aws/aws-sdk-go-v2/aws" awsconfig "github.com/aws/aws-sdk-go-v2/config" @@ -36,6 +37,7 @@ const ( metaReadConcurrency = 8 publishConcurrency = 4 // files in parallel; each already multiparts internally fetchConcurrency = 4 + fetchBudget = 30 * time.Minute deleteBatch = 1000 // DeleteObjects caps one request at 1000 keys ) @@ -126,7 +128,9 @@ func (s *Store) Fetch(ctx context.Context, id string) (string, []byte, string, e return export, meta, digest, nil } _, err, _ = s.fetches.Do(gen, func() (any, error) { - return nil, s.populate(ctx, id, meta, gen) + fctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), fetchBudget) + defer cancel() + return nil, s.populate(fctx, id, meta, gen) }) if err != nil { return "", nil, "", err diff --git a/sandboxd/store/s3/s3_test.go b/sandboxd/store/s3/s3_test.go index ff19362b..dcb861bf 100644 --- a/sandboxd/store/s3/s3_test.go +++ b/sandboxd/store/s3/s3_test.go @@ -19,6 +19,7 @@ import ( "strings" "sync" "testing" + "time" "github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager" @@ -149,6 +150,38 @@ func TestFetchReportsMetaWithoutExportAsNotFound(t *testing.T) { } } +func TestFetchOutlivesTheCallerThatStartedIt(t *testing.T) { + const id = "ck_00000000000000dd" + metaKey := "ck/" + id + "/" + store.MetaFile + fake := &fakeS3{objects: map[string][]byte{}, exportGate: make(chan struct{}), exportEntered: make(chan struct{}, 1)} + st := newTestStore(t, fake) + publishRecord(t, st, id, []byte(`{"id":"`+id+`"}`), "payload") + fake.resetRequests() + + first, hangUp := context.WithCancel(t.Context()) + started := make(chan error, 1) + go func() { + _, _, _, err := st.Fetch(first, id) + started <- err + }() + <-fake.exportEntered + joined := make(chan error, 1) + go func() { + _, _, _, err := st.Fetch(t.Context(), id) + joined <- err + }() + for fake.requestCount(http.MethodGet, metaKey) < 2 { + time.Sleep(time.Millisecond) + } + time.Sleep(50 * time.Millisecond) + hangUp() + close(fake.exportGate) + if err := <-joined; err != nil { + t.Fatalf("second caller's Fetch = %v after the first caller hung up, want the shared download", err) + } + <-started +} + func TestDigestReaderIgnoresReadBoundaries(t *testing.T) { data := bytes.Repeat([]byte{'x'}, int(store.DigestChunkSize)) data = append(data, 'y') @@ -493,12 +526,21 @@ type fakeS3 struct { deleteBatches [][]string failList bool failDelete bool + exportGate chan struct{} + exportEntered chan struct{} } func (f *fakeS3) ServeHTTP(w http.ResponseWriter, r *http.Request) { + key := strings.TrimPrefix(r.URL.Path, "/testbucket/") + if f.exportGate != nil && r.Method == http.MethodGet && strings.Contains(key, "/"+store.ExportDir) { + select { + case f.exportEntered <- struct{}{}: + default: + } + <-f.exportGate + } f.mu.Lock() defer f.mu.Unlock() - key := strings.TrimPrefix(r.URL.Path, "/testbucket/") if f.requests == nil { f.requests = map[fakeRequest]int{} } From c48415de42c0511373d75badce669a421be1a00b Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 20 Sep 2026 12:16:11 +0800 Subject: [PATCH 21/53] fix(pool): run the store generation sweep off the housekeeping loop sweepStoreGenerations was the one long task Run still called inline in its select; the TTL sweep, the archive-delete retries and every engine fan-out already run in their own goroutines. On a shared FUSE checkpoint root the hourly walk held refill, reap and the claim path's refill kick for its whole duration. --- sandboxd/pool/checkpoint_test.go | 47 ++++++++++++++++++++++++++++++++ sandboxd/pool/pool.go | 2 +- 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/sandboxd/pool/checkpoint_test.go b/sandboxd/pool/checkpoint_test.go index 810fd5f7..d39d2a0b 100644 --- a/sandboxd/pool/checkpoint_test.go +++ b/sandboxd/pool/checkpoint_test.go @@ -8,6 +8,7 @@ import ( "path/filepath" "strings" "testing" + "testing/synctest" "time" "github.com/cocoonstack/sandbox/sandboxd/config" @@ -151,6 +152,37 @@ func TestRunRefillsWhileTheRetentionSweepBlocks(t *testing.T) { } } +func TestRunRefillsWhileTheGenerationSweepBlocks(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + eng := newFakeEngine() + m := newTestManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 1}) + m.pools[testKey].goldenDir = "/goldens/x" + stall := &stallingSweepStore{Store: m.ckpts, entered: make(chan struct{}, 1), release: make(chan struct{})} + m.ckpts = stall + + ctx, cancel := context.WithCancel(t.Context()) + done := make(chan struct{}) + go func() { + m.Run(ctx) + close(done) + }() + defer func() { + close(stall.release) + cancel() + <-done + }() + time.Sleep(time.Hour + time.Second) + <-stall.entered + + mustClaim(t, m, testKey) + waitFor(t, func() bool { + m.mu.Lock() + defer m.mu.Unlock() + return len(m.pools[testKey].warm) > 0 + }) + }) +} + func TestCheckpointTenantIsolation(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) @@ -256,6 +288,21 @@ type stallingMetasStore struct { release chan struct{} } +type stallingSweepStore struct { + store.Store + entered chan struct{} + release chan struct{} +} + +func (s *stallingSweepStore) SweepGenerations() error { + select { + case s.entered <- struct{}{}: + default: + } + <-s.release + return s.Store.SweepGenerations() +} + func (s *stallingMetasStore) Metas(ctx context.Context) ([][]byte, error) { select { case s.entered <- struct{}{}: diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index 0f2dc68e..e93795dd 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -503,7 +503,7 @@ func (m *Manager) Run(ctx context.Context) { m.shrinkOnce(ctx) go m.retryArchiveDeletes(ctx) case <-storeSweep.C: - m.sweepStoreGenerations(ctx) + go m.sweepStoreGenerations(ctx) go m.sweepExpiredCheckpoints(ctx) } } From d77b7649b1730e25636746917db429864d3f4a8e Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 20 Sep 2026 12:16:29 +0800 Subject: [PATCH 22/53] fix(packaging): stop sandboxd without signalling its service cgroup docs/deploy.md promises that stopping sandboxd leaves VMs alive, but the unit left KillMode at control-group, so the promise held only for VMMs that cocoon had moved into a scope of their own. A host survey found one VMM, up for days, still inside the sandboxd service cgroup, where a systemctl stop would have killed it. KillMode=process signals the daemon alone, as libvirtd, containerd and dockerd do. --- docs/deploy.md | 6 +++++- packaging/sandboxd.service | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/deploy.md b/docs/deploy.md index 9a32394b..7825ed38 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -423,6 +423,7 @@ After=network-online.target [Service] ExecStart=/usr/local/bin/sandboxd -config /etc/sandboxd/config.json Restart=on-failure +KillMode=process Environment=SANDBOXD_LOG_LEVEL=info [Install] @@ -435,7 +436,10 @@ degradations. `journalctl` answers an empty range with `-- No entries --`, so a check that counts lines reads one problem where there are none — count records (`-o json | wc -l`) or test the output for emptiness. -Stopping sandboxd leaves VMs alive; the next start reconciles them. Claimed +Stopping sandboxd leaves VMs alive; the next start reconciles them. The unit's +`KillMode=process` is what guarantees it: systemd signals sandboxd alone, so a +VMM that cocoon could not move into its own scope, and any `cocoon` call still +in flight, outlive the stop instead of dying with the service's cgroup. Claimed sandboxes are reaped when their TTL expires (default 5m, capped at 24h). To empty a node for maintenance, cordon it first: diff --git a/packaging/sandboxd.service b/packaging/sandboxd.service index 20c56166..d4f8b5c5 100644 --- a/packaging/sandboxd.service +++ b/packaging/sandboxd.service @@ -6,6 +6,7 @@ After=network-online.target [Service] ExecStart=/usr/local/bin/sandboxd -config /etc/sandboxd/config.json Restart=on-failure +KillMode=process Environment=SANDBOXD_LOG_LEVEL=info [Install] From 7da27f730d02b835323fcedc7dd19d9e9769713a Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 20 Sep 2026 12:17:48 +0800 Subject: [PATCH 23/53] fix(sdk/langchain, sdk/openai): default the claim to a one hour lease Both adapters sent no ttl_seconds, so the node applied its 5 minute default and nothing renews a lease: a run that thought for five minutes between two tool calls lost its sandbox, and every later call failed for the rest of the run. The MCP server already defaults to an hour for this reason (mcp/tools.go:113). An explicit ttl_seconds is unchanged. --- docs/langchain.md | 4 ++++ docs/openai-adapter.md | 3 ++- .../cocoonsandbox_langchain/toolkit.py | 2 +- sdk/langchain/tests/test_toolkit.py | 13 +++++++++++++ sdk/openai/cocoonsandbox_openai/adapter.py | 2 +- sdk/openai/tests/test_adapter.py | 17 ++++++++++++++++- 6 files changed, 37 insertions(+), 4 deletions(-) diff --git a/docs/langchain.md b/docs/langchain.md index 901626a1..66549dd2 100644 --- a/docs/langchain.md +++ b/docs/langchain.md @@ -25,6 +25,10 @@ schemas, sync-native with `asyncio.to_thread` async bridges): | `sandbox_read_file` | read a text file | | `sandbox_list_dir` | list a directory as JSON | +The claim's lease is `ttl_seconds`, one hour by default: nothing renews a +lease, and an agent run outlives the node's 5-minute default. Once the lease +ends the sandbox is gone and every later tool call reports it. + A failed call — a missing path, a guest error, an expired or unreachable sandbox — comes back to the model as the tool's error text, not as an exception out of the agent run. Every tool's first call claims the sandbox diff --git a/docs/openai-adapter.md b/docs/openai-adapter.md index 8522b00b..506c6a1e 100644 --- a/docs/openai-adapter.md +++ b/docs/openai-adapter.md @@ -41,7 +41,8 @@ pair over the sync [Python SDK](sdk-python.md), bridged with `CocoonSandboxClientOptions` carries the node address, api token, template ref, network lane (empty for the node's default, or `none`/`egress`), and -TTL. The session state is +`ttl_seconds`, which defaults to one hour: nothing renews a lease, and a +session outlives the node's 5-minute default. The session state is JSON-serializable, so a run can be resumed against the same sandbox after a process restart. Requires Python 3.10+ (the Agents SDK floor); the underlying `cocoonsandbox` stays 3.9+. diff --git a/sdk/langchain/cocoonsandbox_langchain/toolkit.py b/sdk/langchain/cocoonsandbox_langchain/toolkit.py index b30bbb9f..e3ff3282 100644 --- a/sdk/langchain/cocoonsandbox_langchain/toolkit.py +++ b/sdk/langchain/cocoonsandbox_langchain/toolkit.py @@ -42,7 +42,7 @@ def __init__( api_token: str = "", template: str = "rt:24.04", net: str = "", - ttl_seconds: int = 0, + ttl_seconds: int = 3600, from_checkpoint: str = "", ) -> None: self._client = Client(addr, api_token=api_token, timeout=CALL_TIMEOUT) diff --git a/sdk/langchain/tests/test_toolkit.py b/sdk/langchain/tests/test_toolkit.py index decc12ca..0e1b8b87 100644 --- a/sdk/langchain/tests/test_toolkit.py +++ b/sdk/langchain/tests/test_toolkit.py @@ -86,6 +86,19 @@ def refused(path, data): assert "no such directory" in tool.invoke({"path": "/nope/a.txt", "content": "body"}) +def test_default_lease_outlives_an_agent_run(monkeypatch): + kit = CocoonToolkit("127.0.0.1:1") + seen = {} + + def new(template, **kwargs): + seen.update(kwargs) + return FakeSandbox() + + monkeypatch.setattr(kit._client, "new", new) + kit.get_tools()[0].invoke({"command": "echo hi"}) + assert seen["ttl_seconds"] == 3600, seen + + def test_close_releases_once(monkeypatch): kit, fake = hooked(monkeypatch) kit.get_tools()[0].invoke({"command": "x"}) diff --git a/sdk/openai/cocoonsandbox_openai/adapter.py b/sdk/openai/cocoonsandbox_openai/adapter.py index 8801311b..b5efff73 100644 --- a/sdk/openai/cocoonsandbox_openai/adapter.py +++ b/sdk/openai/cocoonsandbox_openai/adapter.py @@ -31,7 +31,7 @@ class CocoonSandboxClientOptions(BaseSandboxClientOptions): api_token: str = "" template: str = "rt:24.04" net: str = "" - ttl_seconds: int = 0 + ttl_seconds: int = 3600 class CocoonSandboxSessionState(SandboxSessionState): diff --git a/sdk/openai/tests/test_adapter.py b/sdk/openai/tests/test_adapter.py index 48ed3b91..87ff58a4 100644 --- a/sdk/openai/tests/test_adapter.py +++ b/sdk/openai/tests/test_adapter.py @@ -14,12 +14,15 @@ from cocoonsandbox import ProtocolError, SilkdError from cocoonsandbox_openai import CocoonSandboxClient, CocoonSandboxClientOptions, CocoonSandboxSessionState +CLAIMS: list = [] + class FakeNode(BaseHTTPRequestHandler): def do_POST(self): length = int(self.headers.get("Content-Length") or 0) - _ = self.rfile.read(length) + body = self.rfile.read(length) if self.path == "/v1/claim": + self.server.claims.append(json.loads(body)) self._reply(200, {"id": "sb_1", "token": "tok", "owner_addr": self.headers["Host"]}) elif self.path.endswith("/release"): self._reply(204, None) @@ -44,6 +47,8 @@ def log_message(self, *args): @pytest.fixture def node(): server = HTTPServer(("127.0.0.1", 0), FakeNode) + server.claims = CLAIMS + CLAIMS.clear() threading.Thread(target=server.serve_forever, daemon=True).start() yield f"127.0.0.1:{server.server_port}" server.shutdown() @@ -66,6 +71,16 @@ async def go(): asyncio.run(go()) +def test_default_lease_outlives_an_agent_run(node): + async def go(): + client = CocoonSandboxClient() + session = await client.create(options=CocoonSandboxClientOptions(addr=node)) + await client.delete(session) + + asyncio.run(go()) + assert CLAIMS[0]["ttl_seconds"] == 3600, CLAIMS + + def test_operations_share_one_sandbox_handle(node): async def go(): client = CocoonSandboxClient() From a38e75d291834e45196c257583760e85f7a1d005 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 20 Sep 2026 12:19:34 +0800 Subject: [PATCH 24/53] docs: state what the code does where the pages said otherwise silkd.md, sdk.md, sdk-python.md: a dropped connection kills a foreground exec (only detached processes and sessions survive); exec with a session uses argv alone, sends no started frame and merges stderr; a command's environment is cleared to PATH, TERM and the lane's proxy variables; an exited detached process leaves the table after 5 minutes; after the child exits an undrained client has 2 seconds before the tail is dropped; one oversized match ends a find; replace is atomic per file, not per list. sdk-python.md: only run and exec take a timeout, unlike the Go SDK's contexts; Template.new takes no claim_ref. mcp.md: logs forgets an exited process after 5 minutes. sandboxd-api.md, security.md: preview has no operator path, unlike fork, promote and checkpoint; an archived claim kept forever has no lease to clamp a preview URL to. deploy.md: a volume claim refused at max_claims answers 429 with no redirect. egress.md: an absolute-form https request on the forward door injects without intercept; an egress pool's warm VMs and golden build sit on the bridge unlocked until claim. --- docs/deploy.md | 2 +- docs/egress.md | 9 +++++++-- docs/mcp.md | 2 +- docs/sandboxd-api.md | 11 ++++++++--- docs/sdk-python.md | 22 +++++++++++++++++----- docs/sdk.md | 15 +++++++++++++-- docs/security.md | 5 +++-- docs/silkd.md | 18 +++++++++++------- 8 files changed, 61 insertions(+), 23 deletions(-) diff --git a/docs/deploy.md b/docs/deploy.md index 7825ed38..9f7de59d 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -120,7 +120,7 @@ sandboxd reads one JSON file (`-config`, default | `checkpoint_peer_heal` | false | on a cluster, lets a node pull a checkpoint it lacks from a peer — found via a live probe, not gossip — rather than failing the branch; see [placement lifecycle](cluster.md#checkpoints-on-a-cluster). Three requirements, all enforced at config load: a nonempty `api_token` (the blob transfer between peers authenticates with it; without one the raw record stream would be open), `mesh.cluster_key` set (the pull presents the fleet `api_token` to an address learned from the peer probe, so the gossip layer carrying that address must itself be authenticated), and `checkpoint_ttl_hours` nonzero (a replica a delete broadcast missed becomes eligible for expiry after it, and its next successful hourly sweep removes it — so it is the finite eligibility point, not an exact ceiling). A shared checkpoint store (`checkpoint_store` kind `s3`) ignores this setting — every node already resolves every checkpoint directly, so there is nothing to heal | | `warm_max` (pool entry) | 0 (static) | turns on the demand-adaptive watermark for that pool: the warm target rises from `warm` toward `warm_max` while claims arrive faster than the measured provision lead covers, and decays back over ~a minute of silence. The pool follows the target down: warm VMs above it are destroyed once the count has stayed above it for a full decay period, so a burst does not leave the pool parked at its high-water mark | | `warmup` (pool entry) | unset | argv run in the golden VM after readiness and before its snapshot, so the files it touches are page-cache-resident in every clone, and again in every clone before it joins the warm pool, so those pages are already faulted into the restored VM when the first command runs — e.g. `["node", "-e", "0"]` on a Node flavor. It runs under the engine's 2-minute command timeout in silkd's base environment (`PATH`, `TERM`, and the guest image's proxy variables wherever nothing routes directly — the none lane and the locked bridge egress lane — with the proxy not yet serving, since a door pre-bound at refill only starts serving at claim); a non-zero exit or a timeout fails the golden build, so the pool stays unfilled until the config is fixed. Config-owned like `egress`: `PUT /v1/pools` rejects it, and a golden built with a different warmup is rebuilt | -| `max_claims` | 0 (unlimited) | node-wide cap on live claims; claim/fork/branch requests beyond it answer 429 with the pool state unharmed (on a cluster, normal warm-candidate placement applies, with volume claims limited to candidates holding every requested volume) | +| `max_claims` | 0 (unlimited) | node-wide cap on live claims; claim/fork/branch requests beyond it answer 429 with the pool state unharmed (on a cluster a non-volume claim tries a warm-peer redirect first; a volume claim answers the 429 with no redirect) | | `audit_log` | false | append every relayed frame that opens an RPC — its op + addressing fields, never payloads — to `/audit.jsonl`; continuation `data`, `data_end`, `stdin`, and `stdin_close` frames pass through unrecorded. The file is size-rotated with one `.1` backup. Records are `{t, id, op}` plus whichever addressing fields the op carries (`argv`, `path`, `dest`, `from`, `to`, `url`, `session`, `port`), plus `method` (`GET`, `CONNECT`, `SOCKS5`, …), `decision` and `secret` (the ref name, never its value) on `egress` records; preview accesses record as op `preview`, one per request. An opening frame whose first line exceeds 4 KiB records as op `oversized` with no addressing fields | | `idle_hibernate_seconds` | 0 (off) | node-wide idle policy for unpooled claims (template/checkpoint claims): a none-lane claim is hibernated once it has had no open data-plane connection (relay, buffered exec, preview) and no egress request in flight for this long; the clock restarts when the last connection closes, so a long command is never cut short. The next call that reaches the guest wakes it transparently. Per-pool `idle_hibernate_seconds` does the same for that pool's claims; pooled keys ignore the node-wide value, and egress pools reject it because they cannot resume safely. Opt in deliberately: a wake costs latency and the snapshot, so callers with their own idle logic must not pay twice | | `archive_after_seconds` | 0 (off) | tier below hibernation: a hibernated claim idle this long is checkpointed to the store and its local VM dropped, freeing the node entirely; the next call that reaches the guest restores it transparently (a checkpoint restore's latency) with a fresh server-default 5m lease. Requires `idle_hibernate_seconds > 0` and must exceed it. Node-wide for unpooled keys; per-pool overrides for that pool | diff --git a/docs/egress.md b/docs/egress.md index 0955201d..3f8b89d8 100644 --- a/docs/egress.md +++ b/docs/egress.md @@ -73,7 +73,9 @@ curl --socks5-hostname 127.0.0.1:1080 imaps://imap.example.com/ # allowed: {"s A `net:"egress"` guest owns a real NIC on the host bridge and reaches the same proxy over the same vsock path. To stop it bypassing the proxy, sandboxd locks -the NIC at claim: an nftables netdev table (`sandbox_egress_`) with an +the NIC at claim — until then a warm VM of the pool, and the pool's golden +build, sit on the bridge unlocked, running only the image's own services, never +tenant code: an nftables netdev table (`sandbox_egress_`) with an ingress hook on the guest's tap drops every guest-initiated packet except IPv4 broadcast DHCP, so the guest's only routed egress is the audited vsock proxy. The lock is fail-closed — a claim whose NIC cannot be locked is rejected, not handed @@ -200,7 +202,10 @@ woken sandbox binds at arm time. request inside an intercepted tunnel. Exact ports only; `0` and repeats are rejected at load. - `secret`: injects the named registered secret's header. A guest-supplied value - for the same header is overwritten. On HTTPS the injection needs `intercept`. + for the same header is overwritten. Inside an HTTPS `CONNECT` tunnel the + injection needs `intercept`; an absolute-form `https://` request on the + forward door needs none — the proxy dials the origin over verified TLS + itself, checks the method, and injects. - `intercept`: terminate a matched HTTPS CONNECT so the request is filtered by method and the secret injected (see below). Only a pool rule may set it. - No policy on a claim ⇒ no egress at all (the proxy is not started). diff --git a/docs/mcp.md b/docs/mcp.md index fbaa39b2..1ab4d70c 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -31,7 +31,7 @@ Every tool call is capped at 5 minutes. | `exec` | run a shell command to completion; returns stdout, stderr and the exit code, each stream keeping its first 1 MiB with `truncated` set past that; a cut-off or dropped run returns the output collected so far next to an `error` field; a hibernated sandbox wakes transparently | | `spawn` | start a command detached and return its pid; output goes to a 256 KiB ring buffer that `logs` replays | | `ps` | list tracked processes (exec, spawn, pty) with state, exit code and start time | -| `logs` | replay up to 256 KiB of a tracked process's newest whole stdout/stderr chunks (+ exit code once ended) | +| `logs` | replay up to 256 KiB of a tracked process's newest whole stdout/stderr chunks (+ exit code once ended); an exited process is forgotten after 5 minutes | | `kill` | signal a tracked process (0 = SIGKILL); an exited process is a no-op success | | `write_file` / `read_file` / `list_dir` | atomic whole-file write (parent must exist); whole-file text read of a regular file up to 1 MiB (a larger file, a directory or a device is an error, invalid UTF-8 replaced, missing path is an error); one-level listing of `{name, kind, size}` entries | | `fork` | clone into N children (1 to the node's `max_fork_count`, default 16) carrying exact memory + disk state, all-or-nothing; the parent keeps running and each child lives one hour | diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index eb6f6dce..f83e2fd3 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -351,11 +351,16 @@ fresh info payload. ## POST /v1/sandboxes/{id}/preview -Auth: like fork — node `api_token` in the header, the sandbox's own token -in the body. Mints a signed URL serving a +Auth: node `api_token` in the header, the sandbox's own token in the body. +Unlike fork, promote and checkpoint — where the root token with an empty body +token acts by id as the operator — preview has no operator path: the body +token is required. Mints a signed URL serving a guest HTTP port from a browser: body `{"token": "...", "port": 8080, "ttl_seconds": 0}` → `{"url": "http:///p//"}`. -The URL's life is clamped to the claim's remaining lease. 501 when the node +The URL's life is clamped to the claim's remaining lease; an archived claim +kept forever (`archive_delete_after_seconds: 0`) has no lease, so there the +requested `ttl_seconds` stands and releasing the sandbox is what ends the URL. +501 when the node has no `preview_listen`. The signed token embeds the sandbox id, port, and owner `advertise_addr`, so any node's preview listener can serve it (forwarding to the owner's main listener) and a released sandbox's URL simply stops diff --git a/docs/sdk-python.md b/docs/sdk-python.md index ecc1d41c..f46bef3c 100644 --- a/docs/sdk-python.md +++ b/docs/sdk-python.md @@ -11,9 +11,9 @@ with client.new("ghcr.io/cocoonstack/sandbox/rt:24.04") as sb: `pip install cocoonstack-sandbox` — stdlib-only, no dependencies, and synchronous by design (agent frameworks that need async wrap calls in `asyncio.to_thread`, exactly like the [OpenAI adapter](openai-adapter.md) -does). It matches the [Go SDK](sdk.md) guest and data-plane surface, with two +does). It matches the [Go SDK](sdk.md) guest and data-plane surface, with three exceptions: operator pool retuning (`SetPools`/`SetPoolsCluster`) remains -Go-only, and file and tar payloads are whole `bytes` values rather than +Go-only, `Template.new` takes no `claim_ref`, and file and tar payloads are whole `bytes` values rather than streams (`read_file`/`pull` return them, `write_file`/`push` take them; the wire is chunked, the caller's copy is not). Wire fidelity is pinned by the shared protocol fixture corpus that the Rust guest, Go, and @@ -75,8 +75,10 @@ client = Client("10.0.0.5:7777", api_token="...", timeout=120.0) tenant token (resource-creating verbs only; operator surfaces answer it 403). On a cluster every node shares the root token and the same tenants set. `timeout` bounds every control-plane request and the data-plane dial -and upgrade; a guest stream then lives until the guest ends it, as in the -Go SDK. A caller with a wall clock of its own passes `deadline` to a claim +and upgrade; a guest stream then lives until the guest ends it. Unlike the +Go SDK, where a canceled context closes any call, only `run`/`exec` take a +`timeout` here: every other data-plane call blocks until the guest answers +or the connection drops. A caller with a wall clock of its own passes `deadline` to a claim (below) so the redirect walk cannot outlive it. **Clusters need nothing extra**: dial any node. On a warm miss the entry @@ -333,7 +335,11 @@ its end the connection is cut, which makes silkd kill the command, and `exec` returns stdout and raises `ExitError` on a non-zero exit — carrying `code`, `stderr`, and the `stdout` produced before it failed. `run` streams raw bytes through the callbacks (chunk boundaries may split multi-byte sequences) and returns the exit code. `user` de-escalates -inside the guest; `session=` routes the command into a persistent session. +inside the guest; `session=` routes the command into a persistent session, +where only `argv` applies — the session owns the cwd, environment and user, +and stderr arrives merged into stdout. A command's environment is `PATH`, +`TERM` and the lane's proxy variables plus `env`, not the image's `ENV`. A +dropped connection kills a foreground command; `spawn` outlives it. ## Background processes @@ -348,6 +354,8 @@ sb.kill(pid) # default SIGKILL `spawn` starts the command detached with a bounded output ring; `logs` replays it, `attach` follows live output until exit (replay and live stream hand off atomically). Killing an already-exited process is a no-op success. +An exited process leaves the table after 5 minutes; from then on its pid +raises `SilkdError` `not_found`. ## Sessions @@ -408,6 +416,10 @@ results = sb.replace(["/work/main.py"], r"foo", "bar") ``` Patterns are regular expressions evaluated in the guest — no shell quoting. +`replace` is atomic per file, not per list: it raises at the first missing, +unreadable or non-UTF-8 path and the files before it stay rewritten, so pass it +paths a `find` just returned. A `find` fails as a whole when one match's frame +would pass the 8 MiB cap. ## Watching diff --git a/docs/sdk.md b/docs/sdk.md index b3eee6de..1e156daf 100644 --- a/docs/sdk.md +++ b/docs/sdk.md @@ -446,7 +446,13 @@ code, err := sb.Run(ctx, sandbox.Cmd{ `Cmd` fields: `Argv` (required), `Cwd`, `Env`, `User` (de-escalation inside the guest), `Session` (run inside a persistent session, below), `Stdin` (nil closes the child's stdin immediately; do not share one blocking reader -across Runs), `Stdout`/`Stderr` (nil discards). +across Runs), `Stdout`/`Stderr` (nil discards). With `Session` set only +`Argv` applies: the session owns the cwd, environment and user, its commands +read `/dev/null`, and stderr arrives merged into stdout. A command's +environment is `PATH`, `TERM` and the lane's proxy variables plus `Env` — not +the image's `ENV`; `HOME` and `USER` are set only with `User`. A dropped +connection kills a foreground command; use `Spawn` for work that must outlive +it. Non-zero exits surface as `*sandbox.ExitError{Code, Stderr}` from `Exec` (alongside partial stdout); `Run` returns the code directly. @@ -467,7 +473,8 @@ the process has ended (`exited=false` while running); `Attach` follows live output until exit — the replay and the live stream hand off atomically, so no chunk is lost or doubled between them. Killing an already-exited process is a no-op success (its OS pid may be recycled; silkd never signals a -reaped child). +reaped child). An exited process leaves the table after 5 minutes; from then +on its pid answers `not_found` to every verb. ## Sessions @@ -532,6 +539,10 @@ results, err := sb.Replace(ctx, []string{"/work/main.go"}, `foo`, "bar") ``` Patterns are regular expressions evaluated in the guest — no shell quoting. +`Replace` is atomic per file, not per list: it fails at the first missing, +unreadable or non-UTF-8 path and the files before it stay rewritten, so pass it +paths a `Find` just returned. A `Find` fails as a whole when one match's frame +would pass the 8 MiB cap. ## Watching diff --git a/docs/security.md b/docs/security.md index b9ecd315..ff5fda0b 100644 --- a/docs/security.md +++ b/docs/security.md @@ -77,8 +77,9 @@ the root `api_token` (operator surfaces, full access), tenant tokens (resource-creating verbs, everything stamped and quota'd per tenant), and per-sandbox tokens (that sandbox only — holding a handle amplifies to nothing node-level). Two capability tokens ride on top: preview URLs are -HMAC-signed, expire with the claim's lease, and die with the sandbox (no -revocation list to leak); a checkpoint id is the unguessable capability to +HMAC-signed, expire with the claim's lease (an archived claim kept forever +has none, so its URL runs for the TTL it was minted with), and die with the +sandbox (no revocation list to leak); a checkpoint id is the unguessable capability to branch it. On a cluster, deleting a checkpoint does not revoke that capability fleet-wide the instant it runs: the delete is best-effort — broadcast to every peer the node currently sees — so a peer that is offline diff --git a/docs/silkd.md b/docs/silkd.md index a8a7d1f5..b70fa660 100644 --- a/docs/silkd.md +++ b/docs/silkd.md @@ -25,9 +25,11 @@ on a daemon that keeps the connection. Binary payloads ride base64 in `data` fields. Frames are capped at 8 MiB; requests carry `"v": 1` and unknown fields are ignored (the forward-compatibility story). -Sessions and processes are server-side state addressed by id — a dropped -connection loses nothing (`attach` resumes). The connection-bound verbs are -`fs_watch`, `pty_open`, `lsp_request`, and `port_forward`. +Sessions and detached processes are server-side state addressed by id — a +dropped connection loses nothing (`attach` resumes). A foreground `exec` is +bound to its connection: a drop kills the child and frees its pid. The other +connection-bound verbs are `fs_watch`, `pty_open`, `lsp_request`, and +`port_forward`. The authoritative wire contract is the shared fixture corpus in `protocol/wire/fixtures/v1`, round-tripped by the Rust, Go, and Python test @@ -37,12 +39,12 @@ suites — a frame only one side can parse fails CI. | group | ops | response flow | |---|---|---| -| exec | `exec {argv, cwd?, env?, user?, detach?, session?}` | `started{pid}` → `stdout/stderr{data}`… → `exit{code}`; the client may stream `stdin{data}` / `stdin_close`. `detach` returns after `started`; the process keeps a bounded output ring for later `logs`/`attach`; not combinable with `session` | +| exec | `exec {argv, cwd?, env?, user?, detach?, session?}` | `started{pid}` → `stdout/stderr{data}`… → `exit{code}`; the client may stream `stdin{data}` / `stdin_close`. `detach` returns after `started`; the process keeps a bounded output ring for later `logs`/`attach` and leaves the table 5 minutes after it exits, after which its pid answers `not_found`; not combinable with `session`. With `session` only `argv` is used — `cwd`, `env`, `user` and `stdin` frames are ignored (the session owns them, and its commands read `/dev/null`) — no `started` is sent, and stderr arrives merged into `stdout`. A child's environment is cleared to `PATH`, `TERM` and the lane's proxy variables plus `env`; `HOME` and `USER` are set only with `user` | | procs | `ps` / `kill {pid, signal?}` / `attach {pid}` / `logs {pid}` | `ps` answers one `procs{procs}` frame, `kill` a bare `done`; `logs` and `attach` stream `stdout/stderr{data}`… then `exit{code}` (`logs` closes with `done`, and so does `attach` when the process is gone before its exit). Handles are guest pids; any connection can list, signal, replay, or re-attach live | | sessions | `session_create {id?, cwd?, env?}` / `session_list` / `session_rm {id}` | `session_created{id}` / `sessions{sessions}` / `done`. A session is a real persistent bash; `exec` with `session` runs inside it. Idle sessions are reaped after 30 minutes | | fs | `fs_write {path, mode?}` (+`data`/`data_end` frames) / `fs_read` / `fs_list` / `fs_stat` / `fs_mkdir {parents?}` / `fs_rm {recursive?}` / `fs_rename {from, to}` | `fs_read` streams `data{data}`… → `done` for a regular file and answers `bad_request` for a directory, device or FIFO, `fs_list` streams 4096-entry `entries{entries}` batches → `done`, `fs_stat` answers one `stat{info}`, and the mutating verbs terminate with `done`. Streaming runs both directions; write commits atomically via temp+rename and inherits an overwritten file's mode | | tree | `fs_push {dest}` (+tar as `data` frames) / `fs_pull {path}` | whole trees as tar streams through the guest tar: `fs_pull` streams `data{data}`… → `done`, `fs_push` terminates with `done` | -| search | `fs_find {path, pattern, glob?}` → `match{file, line, content}`… → `done` / `fs_replace {files, pattern, replacement}` → `replaced{file, replacements}`… → `done` | regex as data, no shell quoting; `glob` is anchored `*`/`?` wildcards over file names; find skips binary and >8 MiB files, and replace skips the same 8 MiB bound with a zero count | +| search | `fs_find {path, pattern, glob?}` → `match{file, line, content}`… → `done` / `fs_replace {files, pattern, replacement}` → `replaced{file, replacements}`… → `done` | regex as data, no shell quoting; `glob` is anchored `*`/`?` wildcards over file names; find skips binary and >8 MiB files, and replace skips the same 8 MiB bound with a zero count. One match whose frame would pass the 8 MiB cap ends the find with an `error`. Replace is atomic per file, not per list: it stops with an `error` at the first missing, unreadable or non-UTF-8 file, and the files before it stay rewritten | | watch | `fs_watch {path, recursive?}` | `ready` once armed (events after it are guaranteed captured), then `event{kind, path}` until the client disconnects; watcher or delivery-queue overflow errors arrive as a terminal `error` instead of silently losing events | | pty | `pty_open {cols, rows, cwd?, env?, user?}` / `pty_resize {pid, cols, rows}` | `pty_open` answers `started{pid}`, then a shell under a pseudo-terminal: output as `stdout` frames, input as `stdin` frames, `exit{code}` terminal; `pty_resize` answers `done`. PTYs register in the proc table like any exec | | git | `git_clone {url, path, branch?, depth?, auth?}` / `git_status {path}` / `git_add {path, files}` / `git_commit {path, message, author}` / `git_push {path, auth?}` / `git_pull {path, auth?}` / `git_branch {path, action, name?}` | `git_status_result` (porcelain v2), `git_commit_result{hash}`, `git_branches{current, branches}` for `git_branch {action: "list"}`; every other verb terminates with `done`. A status past about 1 MiB of entries carries `truncated: true` with the head of the list, so one frame never exceeds the cap. `auth` is injected as an in-memory header, never written to guest disk | @@ -85,8 +87,10 @@ own routed network never gets them. ## Limits - 8 MiB frame cap (a malformed peer cannot OOM the daemon) -- foreground exec output is backpressured: a slow client paces the child - rather than losing output; detached observers ride a best-effort broadcast +- foreground exec output is backpressured while the child runs: a slow client + paces it rather than losing output. Once the child exits, the client has 2 + seconds to drain what is still queued (up to 8 MiB) before `exit` is sent + and the rest is dropped. Detached observers ride a best-effort broadcast - no fsync durability on writes (guest page cache semantics) - `fs_push` is network-failure atomic: the stream extracts into a staging dir inside `dest` and merges (local renames) only after the tar terminates From f3f1a3cc6e6f167ab2a9fabf5f38f24cbd953a61 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 20 Sep 2026 12:20:38 +0800 Subject: [PATCH 25/53] review: keep a test fixture's type and its method together The generation-sweep test's stallingSweepStore landed between stallingMetasStore and its Metas method (/code item 1: a type is immediately followed by its complete method set). --- sandboxd/pool/checkpoint_test.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/sandboxd/pool/checkpoint_test.go b/sandboxd/pool/checkpoint_test.go index d39d2a0b..15042051 100644 --- a/sandboxd/pool/checkpoint_test.go +++ b/sandboxd/pool/checkpoint_test.go @@ -288,6 +288,15 @@ type stallingMetasStore struct { release chan struct{} } +func (s *stallingMetasStore) Metas(ctx context.Context) ([][]byte, error) { + select { + case s.entered <- struct{}{}: + default: + } + <-s.release + return s.Store.Metas(ctx) +} + type stallingSweepStore struct { store.Store entered chan struct{} @@ -302,12 +311,3 @@ func (s *stallingSweepStore) SweepGenerations() error { <-s.release return s.Store.SweepGenerations() } - -func (s *stallingMetasStore) Metas(ctx context.Context) ([][]byte, error) { - select { - case s.entered <- struct{}{}: - default: - } - <-s.release - return s.Store.Metas(ctx) -} From f895a307fc815b144406da43c8d72d1664e5253f Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 20 Sep 2026 12:42:02 +0800 Subject: [PATCH 26/53] fix(pool): keep the empty-floor cutoff off a pool whose lead sizes it dcbfd6f returned the floor whenever the decayed rate fell under one arrival per decay period, and claimed that was the identity for a floor of one or more. It is not: warm 1, warm_max 3, 0.01/s and a two minute lead target 3, and the cutoff made it 1. The cutoff and the unmeasured lead bootstrap now only ever turn a dynamic target of 1 into 0 or of 0 into 1, which max(floor, ...) absorbs for any floor of one or more. --- sandboxd/pool/watermark.go | 10 +++++----- sandboxd/pool/watermark_test.go | 8 ++++++++ 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/sandboxd/pool/watermark.go b/sandboxd/pool/watermark.go index e37199cb..50411a19 100644 --- a/sandboxd/pool/watermark.go +++ b/sandboxd/pool/watermark.go @@ -49,13 +49,13 @@ func (p *pool) effectiveTarget(now time.Time) int { if !p.lastArrival.IsZero() { rate *= math.Exp(-now.Sub(p.lastArrival).Seconds() / rateDecayTau.Seconds()) } - if rate*rateDecayTau.Seconds() < 1 { - return p.floor + dynamic := int(math.Ceil(rate * p.lead.Seconds() * leadSafety)) + if p.lead == 0 && rate > 0 { + dynamic = 1 } - if p.lead == 0 { - return max(p.floor, 1) + if dynamic == 1 && rate*rateDecayTau.Seconds() < 1 { + dynamic = 0 } - dynamic := int(math.Ceil(rate * p.lead.Seconds() * leadSafety)) return max(p.floor, min(dynamic, p.warmMax)) } diff --git a/sandboxd/pool/watermark_test.go b/sandboxd/pool/watermark_test.go index c88a9b31..5808c45f 100644 --- a/sandboxd/pool/watermark_test.go +++ b/sandboxd/pool/watermark_test.go @@ -95,6 +95,14 @@ func TestEffectiveTargetGrowsAnEmptyPoolAndReturnsItToEmpty(t *testing.T) { } } +func TestEffectiveTargetKeepsASlowPoolAheadOfSparseDemand(t *testing.T) { + now := time.Now() + p := &pool{key: types.PoolKey{}, floor: 1, warmMax: 3, lead: 2 * time.Minute, rate: 0.01, lastArrival: now} + if got := p.effectiveTarget(now); got != 3 { + t.Errorf("target %d at 0.01/s with a two minute lead, want 3: the empty-floor cutoff must not reach a sized pool", got) + } +} + func TestRefillGrowsAnEmptyPoolOnDemand(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng, config.PoolSpec{PoolKey: testKey, WarmMax: 4}) From 18449ef7f5098b318743581ee6a5bfa23ed577a1 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 20 Sep 2026 12:42:02 +0800 Subject: [PATCH 27/53] fix(store/s3): let a cancelled caller leave a stalled shared download bf38fd9 detached the flight but kept the synchronous Do, so a caller whose client hung up stayed blocked, holding its record read lock, until the download ended or its 30 minute budget ran out. DoChan plus a select on the caller's context is the shape healCheckpoint uses. --- sandboxd/store/s3/s3.go | 11 ++++++++--- sandboxd/store/s3/s3_test.go | 10 +++++++++- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/sandboxd/store/s3/s3.go b/sandboxd/store/s3/s3.go index 56e3e2d9..255d3c9a 100644 --- a/sandboxd/store/s3/s3.go +++ b/sandboxd/store/s3/s3.go @@ -127,13 +127,18 @@ func (s *Store) Fetch(ctx context.Context, id string) (string, []byte, string, e if _, statErr := os.Stat(export); statErr == nil { return export, meta, digest, nil } - _, err, _ = s.fetches.Do(gen, func() (any, error) { + flight := s.fetches.DoChan(gen, func() (any, error) { fctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), fetchBudget) defer cancel() return nil, s.populate(fctx, id, meta, gen) }) - if err != nil { - return "", nil, "", err + select { + case res := <-flight: + if res.Err != nil { + return "", nil, "", res.Err + } + case <-ctx.Done(): + return "", nil, "", ctx.Err() } return export, meta, digest, nil } diff --git a/sandboxd/store/s3/s3_test.go b/sandboxd/store/s3/s3_test.go index dcb861bf..d8a37f01 100644 --- a/sandboxd/store/s3/s3_test.go +++ b/sandboxd/store/s3/s3_test.go @@ -175,11 +175,19 @@ func TestFetchOutlivesTheCallerThatStartedIt(t *testing.T) { } time.Sleep(50 * time.Millisecond) hangUp() + select { + case err := <-started: + if !errors.Is(err, context.Canceled) { + t.Fatalf("first caller's Fetch = %v, want its own cancellation", err) + } + case <-time.After(5 * time.Second): + close(fake.exportGate) + t.Fatal("first caller's Fetch still blocked on the stalled download after it hung up") + } close(fake.exportGate) if err := <-joined; err != nil { t.Fatalf("second caller's Fetch = %v after the first caller hung up, want the shared download", err) } - <-started } func TestDigestReaderIgnoresReadBoundaries(t *testing.T) { From 904d046c595a00562b2ab40349cd065dfe2195b0 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 20 Sep 2026 12:42:02 +0800 Subject: [PATCH 28/53] fix(pool): run one store generation sweep at a time Moving the sweep off the housekeeping loop let an hourly tick start a second walk while one on a wedged mount was still blocked. Every other sweep this loop starts is single-flight through an atomic flag. --- sandboxd/pool/pool.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index e93795dd..6b314aa5 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -302,6 +302,7 @@ type Manager struct { tpls store.Store ckptTTL time.Duration ckptSweeping atomic.Bool + genSweeping atomic.Bool // tplSet caches each template id against its owning tenant ("" = operator). tplMu sync.Mutex @@ -594,6 +595,10 @@ func (m *Manager) WithPeerDelete(fn PeerDeleteFunc) { } func (m *Manager) sweepStoreGenerations(ctx context.Context) { + if !m.genSweeping.CompareAndSwap(false, true) { + return + } + defer m.genSweeping.Store(false) logger := log.WithFunc("pool.sweepStoreGenerations") if err := m.ckpts.SweepGenerations(); err != nil { logger.Error(ctx, err, "sweep checkpoint generations") From 9b2c554dc63ddc6fff54d8b3f15fe2b00ed1fed0 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 20 Sep 2026 13:52:59 +0800 Subject: [PATCH 29/53] build: list zerolog as a direct sandboxd dependency log.go has imported it directly since the journald priority prefix landed; go mod tidy moves it out of the indirect block. The other four modules are already tidy. --- sandboxd/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sandboxd/go.mod b/sandboxd/go.mod index b7cd0038..6dfd0785 100644 --- a/sandboxd/go.mod +++ b/sandboxd/go.mod @@ -12,6 +12,7 @@ require ( github.com/google/nftables v0.3.0 github.com/hashicorp/memberlist v0.6.0 github.com/projecteru2/core v0.1.5 + github.com/rs/zerolog v1.35.1 golang.org/x/net v0.58.0 golang.org/x/sync v0.22.0 golang.org/x/sys v0.48.0 @@ -57,7 +58,6 @@ require ( github.com/miekg/dns v1.1.72 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/rogpeppe/go-internal v1.16.0 // indirect - github.com/rs/zerolog v1.35.1 // indirect github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect github.com/vishvananda/netns v0.0.5 // indirect golang.org/x/mod v0.40.0 // indirect From a33b56a8a3b0891f7873c2f50713657926db3b12 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 20 Sep 2026 13:52:59 +0800 Subject: [PATCH 30/53] fix(pool): keep one claims.json retrier while the write keeps failing recommit started a detached retry per failed persist. On a data_dir that stays unwritable the reap tick alone rolls back and calls it every 5 seconds, so the daemon grew one goroutine and one error line per tick with no client activity. commit always writes the current full state, so a single retrier converges every outstanding change. --- sandboxd/pool/claim.go | 4 ++++ sandboxd/pool/claims_test.go | 27 +++++++++++++++++++++++++++ sandboxd/pool/pool.go | 1 + 3 files changed, 32 insertions(+) diff --git a/sandboxd/pool/claim.go b/sandboxd/pool/claim.go index 181bae8a..37d013e8 100644 --- a/sandboxd/pool/claim.go +++ b/sandboxd/pool/claim.go @@ -479,7 +479,11 @@ func (m *Manager) purgeArchiveCk(ctx context.Context, id, ck, tenant string) { // recommit re-persists in the background until success or a newer durable write; detached. func (m *Manager) recommit(ctx context.Context, snap claimSnapshot) { + if !m.recommitting.CompareAndSwap(false, true) { + return + } go func() { + defer m.recommitting.Store(false) backoff := recommitBackoff for { err := m.store.commit(snap) diff --git a/sandboxd/pool/claims_test.go b/sandboxd/pool/claims_test.go index 72f3d744..fd775957 100644 --- a/sandboxd/pool/claims_test.go +++ b/sandboxd/pool/claims_test.go @@ -4,6 +4,7 @@ import ( "os" "path/filepath" "reflect" + "runtime" "slices" "sync" "testing" @@ -12,6 +13,32 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) +func TestRecommitKeepsOneRetrierWhileWritesFail(t *testing.T) { + m := newTestManager(t, newFakeEngine()) + sb := mustClaim(t, m, testKey) + dir := filepath.Join(t.TempDir(), "absent") + m.store.path = filepath.Join(dir, "claims.json") + + before := runtime.NumGoroutine() + for range 50 { + m.mu.Lock() + snap := m.store.set(sb) + m.mu.Unlock() + m.recommit(t.Context(), snap) + } + if grown := runtime.NumGoroutine() - before; grown > 2 { + t.Fatalf("%d goroutines for 50 failed persists, want one retrier", grown) + } + + if err := os.MkdirAll(dir, 0o750); err != nil { + t.Fatalf("mkdir: %v", err) + } + waitFor(t, func() bool { + _, err := os.Stat(m.store.path) + return err == nil && !m.recommitting.Load() + }) +} + func TestStoreRoundTrip(t *testing.T) { s := newClaimStore(t.TempDir()) claims := map[string]*types.Sandbox{ diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index 6b314aa5..7e9b79bf 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -303,6 +303,7 @@ type Manager struct { ckptTTL time.Duration ckptSweeping atomic.Bool genSweeping atomic.Bool + recommitting atomic.Bool // tplSet caches each template id against its owning tenant ("" = operator). tplMu sync.Mutex From a6eda20e4d48e833650cad5ae192c30813d830c5 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 20 Sep 2026 13:54:38 +0800 Subject: [PATCH 31/53] fix(types, server): saturate wire seconds instead of wrapping them ttl_seconds and the buffered exec's timeout_seconds were multiplied by time.Second unbounded. A large value wraps negative: a ttl_seconds in [9223372037, 2^63) read as unset and got the 5 minute default instead of the documented 24h cap, and a timeout_seconds such as 10000000000 woke a hibernated sandbox and then answered 504 without running the command. --- sandboxd/server/exec.go | 3 ++- sandboxd/server/exec_test.go | 17 +++++++++++++++++ sandboxd/types/api.go | 8 +++++++- sandboxd/types/types_test.go | 20 +++++++++++++++++++- 4 files changed, 45 insertions(+), 3 deletions(-) diff --git a/sandboxd/server/exec.go b/sandboxd/server/exec.go index 39e85c1a..e5b976de 100644 --- a/sandboxd/server/exec.go +++ b/sandboxd/server/exec.go @@ -13,6 +13,7 @@ import ( "github.com/projecteru2/core/log" "github.com/cocoonstack/sandbox/protocol/wire" + "github.com/cocoonstack/sandbox/sandboxd/types" ) const ( @@ -63,7 +64,7 @@ func (s *Server) handleExec(w http.ResponseWriter, r *http.Request) { defer done() if req.TimeoutSeconds > 0 { var cancel context.CancelFunc - ctx, cancel = context.WithTimeout(ctx, time.Duration(req.TimeoutSeconds)*time.Second) + ctx, cancel = context.WithTimeout(ctx, types.Seconds(req.TimeoutSeconds)) defer cancel() } defer closeOnCancel(ctx, guest)() diff --git a/sandboxd/server/exec_test.go b/sandboxd/server/exec_test.go index 2c9ed76e..e54ae03f 100644 --- a/sandboxd/server/exec_test.go +++ b/sandboxd/server/exec_test.go @@ -80,6 +80,23 @@ func TestExecAuditsTheFrameWithoutItsDelimiter(t *testing.T) { } } +func TestExecRunsUnderAnOverflowingTimeout(t *testing.T) { + ts, _ := newRelayServer(t, func(conn net.Conn) { + defer conn.Close() + r := bufio.NewReader(conn) + for range 2 { + if _, err := r.ReadBytes('\n'); err != nil { + return + } + } + _, _ = io.WriteString(conn, `{"type":"started","pid":7}`+"\n"+`{"type":"exit","code":0}`+"\n") + }) + status, body := postExec(t, ts, `{"argv":["true"],"timeout_seconds":10000000000}`) + if status != http.StatusOK { + t.Fatalf("status %d, want 200: a huge timeout_seconds must not wrap into an expired deadline: %s", status, body) + } +} + func TestExecMapsSilkdErrors(t *testing.T) { for _, tt := range []struct { kind string diff --git a/sandboxd/types/api.go b/sandboxd/types/api.go index 2b3ca854..8a98f8ff 100644 --- a/sandboxd/types/api.go +++ b/sandboxd/types/api.go @@ -1,6 +1,7 @@ package types import ( + "math" "time" ) @@ -11,7 +12,7 @@ type TTLField struct { // TTL converts the wire seconds to a duration. func (f TTLField) TTL() time.Duration { - return time.Duration(f.TTLSeconds) * time.Second + return Seconds(f.TTLSeconds) } // ClaimRequest is the wire body of POST /v1/claim. @@ -127,3 +128,8 @@ type PreviewRequest struct { type PreviewResponse struct { URL string `json:"url"` } + +// Seconds converts wire seconds to a duration and saturates where the multiply would overflow. +func Seconds(n int) time.Duration { + return time.Duration(min(int64(n), math.MaxInt64/int64(time.Second))) * time.Second +} diff --git a/sandboxd/types/types_test.go b/sandboxd/types/types_test.go index 0234958f..2e4c9a6b 100644 --- a/sandboxd/types/types_test.go +++ b/sandboxd/types/types_test.go @@ -1,6 +1,24 @@ package types -import "testing" +import ( + "math" + "testing" + "time" +) + +func TestSecondsSaturatesInsteadOfWrapping(t *testing.T) { + for _, n := range []int{9223372037, 10_000_000_000, math.MaxInt64} { + if got := Seconds(n); got <= 24*time.Hour { + t.Errorf("Seconds(%d) = %v, want a duration past any cap, never a wrapped one", n, got) + } + } + if got := (TTLField{TTLSeconds: 9223372037}).TTL(); got <= 24*time.Hour { + t.Errorf("TTL() = %v for a huge ttl_seconds, want it to reach the 24h clamp", got) + } + if got := Seconds(90); got != 90*time.Second { + t.Errorf("Seconds(90) = %v", got) + } +} func TestValidateSizes(t *testing.T) { for _, tt := range []struct { From d369b3e81d90ebf284f6cffb3ea958c4371d61fd Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 20 Sep 2026 13:54:53 +0800 Subject: [PATCH 32/53] perf(engine): size the preview port reader from the frame it reads portReadBuf was 64 KiB under a comment saying it fits one silkd data frame, but a port_forward frame carries BULK_CHUNK = 256 KiB and is 349,554 bytes on the wire, so every frame took bufio's fragment path. The constant now derives from wire.BulkChunk. Interleaved A/B (ABBA x3, 2000 frames per run): 9 allocations per frame to 1, median 75 ms to 47 ms. Preview path only; nothing on the claim or SDK relay path reads here. --- sandboxd/engine/portconn.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sandboxd/engine/portconn.go b/sandboxd/engine/portconn.go index 02fdaefc..660e82a8 100644 --- a/sandboxd/engine/portconn.go +++ b/sandboxd/engine/portconn.go @@ -13,7 +13,7 @@ import ( ) // portReadBuf fits silkd's data frames in one buffered read. -const portReadBuf = 64 << 10 +const portReadBuf = wire.BulkChunk*4/3 + 64 var portDataHead = []byte(`{"type":"data","data":"`) From e4255e8b817ec6304cbfdf697f5c1aacd0a4a54f Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 20 Sep 2026 13:59:02 +0800 Subject: [PATCH 33/53] fix(sdk/python): raise a typed error when a send fails Conn.recv wraps a socket failure in ProtocolError; Conn.send let a bare BrokenPipeError or ConnectionResetError out, the one SDK failure outside the SandboxError hierarchy. A write into a relay the node had just closed therefore escaped every caller that catches SandboxError, including the LangChain toolkit's tool-error mapping. The two places that tolerated a failed send now name the typed error. --- sdk/python/cocoonsandbox/conn.py | 5 ++++- sdk/python/cocoonsandbox/sandbox.py | 4 ++-- sdk/python/tests/test_hardening.py | 11 +++++++++++ 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/sdk/python/cocoonsandbox/conn.py b/sdk/python/cocoonsandbox/conn.py index d3a0fb21..9bd2c36e 100644 --- a/sdk/python/cocoonsandbox/conn.py +++ b/sdk/python/cocoonsandbox/conn.py @@ -40,7 +40,10 @@ def __init__(self, sock: socket.socket, reader: BinaryIO) -> None: self._reader = reader def send(self, op: str, **fields: object) -> None: - self._sock.sendall(encode_request(op, **fields)) + try: + self._sock.sendall(encode_request(op, **fields)) + except OSError as exc: + raise ProtocolError(f"write failed: {exc}") from exc def settimeout(self, timeout: float | None) -> None: self._sock.settimeout(timeout) diff --git a/sdk/python/cocoonsandbox/sandbox.py b/sdk/python/cocoonsandbox/sandbox.py index 3ffcc9d0..f3a3f21b 100644 --- a/sdk/python/cocoonsandbox/sandbox.py +++ b/sdk/python/cocoonsandbox/sandbox.py @@ -122,7 +122,7 @@ def run( if pump is not None: pump.start() else: - with contextlib.suppress(OSError): + with contextlib.suppress(ProtocolError): conn.send("stdin_close") code = _pump_stdio(conn, on_stdout, on_stderr) except (ProtocolError, OSError): @@ -460,7 +460,7 @@ def pump_out() -> None: pump = threading.Thread(target=pump_out, daemon=True) pump.start() try: - with contextlib.suppress(OSError): + with contextlib.suppress(SandboxError, OSError): while True: chunk = local.recv(FS_CHUNK) if not chunk: diff --git a/sdk/python/tests/test_hardening.py b/sdk/python/tests/test_hardening.py index 7d280b4b..c2a97b2f 100644 --- a/sdk/python/tests/test_hardening.py +++ b/sdk/python/tests/test_hardening.py @@ -26,6 +26,17 @@ def test_dial_agent_wraps_refused_connection(dead_addr): dial_agent(dead_addr, "sb_1", "tok", 0.5) +def test_send_on_a_dead_socket_raises_a_typed_error(): + ours, theirs = socket.socketpair() + theirs.close() + conn = Conn(ours, ours.makefile("rb")) + try: + with pytest.raises(ProtocolError): + conn.send("info") + finally: + conn.close() + + def test_watcher_propagates_silkd_error(): client_sock, guest_sock = socket.socketpair() guest_sock.sendall(json.dumps({"type": "error", "kind": "not_found", "message": "gone"}).encode() + b"\n") From b49bd8ba5f65674aa65df09424256a07fec07820 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 20 Sep 2026 13:59:02 +0800 Subject: [PATCH 34/53] fix(ci): run the shell gate when the Makefile changes shell.yml's only step is `make sh-lint`; a761567 added the Makefile to the Go workflow's paths and missed this one. --- .github/workflows/shell.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/shell.yml b/.github/workflows/shell.yml index 1b57d767..76999ce5 100644 --- a/.github/workflows/shell.yml +++ b/.github/workflows/shell.yml @@ -7,10 +7,12 @@ on: branches: [main] paths: - "**.sh" + - "Makefile" - ".github/workflows/shell.yml" pull_request: paths: - "**.sh" + - "Makefile" - ".github/workflows/shell.yml" workflow_dispatch: From 0e5754907672064092773eb7b0e2c144d07ba061 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 20 Sep 2026 14:00:04 +0800 Subject: [PATCH 35/53] docs: correct this round's own sentences and six more promises Corrects two sentences earlier commits on this branch added: a dropped connection kills a foreground command only once it next writes output (a silent one runs on; silkd notices a hang-up through a failed write), and on a claim with no lease ttl_seconds 0 mints a one hour preview URL. deploy.md: the fuller config's idle_hibernate_seconds 300 equalled the 5 minute lease a woken archive gets, so that sandbox could never hibernate again before the reap destroyed it (now 120, with the constraint stated); the minimal config's policy-less egress pool has no network. sandboxd-api.md, egress.md: PUT /v1/pools decides which policy layers a key has, and live claims follow at their next wake or restart; require_promoted belongs to volume claims only. mcp.md: ending a session releases its sandboxes. silkd.md: fs_push creates dest before the stream. langchain.md: from_checkpoint pins template and net. sdk.md: Close and Stop answer not_found for a session or server already gone. --- docs/deploy.md | 9 +++++++-- docs/egress.md | 5 ++++- docs/langchain.md | 3 ++- docs/mcp.md | 5 ++++- docs/sandboxd-api.md | 17 ++++++++++++----- docs/sdk-python.md | 4 +++- docs/sdk.md | 10 +++++++--- docs/silkd.md | 10 +++++++--- 8 files changed, 46 insertions(+), 17 deletions(-) diff --git a/docs/deploy.md b/docs/deploy.md index 9f7de59d..62fd318b 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -91,6 +91,11 @@ sandboxd reads one JSON file (`-config`, default } ``` +As written the `egress` pool's guests reach nothing: on a bridge lane the NIC +is locked default-deny and the proxy door is bound only for a claim that has a +policy, so give the pool (or its tenants) an [`egress`](egress.md) block before +expecting traffic. + | field | default | meaning | |---|---|---| | `listen` | `:7777` | control- and data-plane HTTP listener | @@ -123,7 +128,7 @@ sandboxd reads one JSON file (`-config`, default | `max_claims` | 0 (unlimited) | node-wide cap on live claims; claim/fork/branch requests beyond it answer 429 with the pool state unharmed (on a cluster a non-volume claim tries a warm-peer redirect first; a volume claim answers the 429 with no redirect) | | `audit_log` | false | append every relayed frame that opens an RPC — its op + addressing fields, never payloads — to `/audit.jsonl`; continuation `data`, `data_end`, `stdin`, and `stdin_close` frames pass through unrecorded. The file is size-rotated with one `.1` backup. Records are `{t, id, op}` plus whichever addressing fields the op carries (`argv`, `path`, `dest`, `from`, `to`, `url`, `session`, `port`), plus `method` (`GET`, `CONNECT`, `SOCKS5`, …), `decision` and `secret` (the ref name, never its value) on `egress` records; preview accesses record as op `preview`, one per request. An opening frame whose first line exceeds 4 KiB records as op `oversized` with no addressing fields | | `idle_hibernate_seconds` | 0 (off) | node-wide idle policy for unpooled claims (template/checkpoint claims): a none-lane claim is hibernated once it has had no open data-plane connection (relay, buffered exec, preview) and no egress request in flight for this long; the clock restarts when the last connection closes, so a long command is never cut short. The next call that reaches the guest wakes it transparently. Per-pool `idle_hibernate_seconds` does the same for that pool's claims; pooled keys ignore the node-wide value, and egress pools reject it because they cannot resume safely. Opt in deliberately: a wake costs latency and the snapshot, so callers with their own idle logic must not pay twice | -| `archive_after_seconds` | 0 (off) | tier below hibernation: a hibernated claim idle this long is checkpointed to the store and its local VM dropped, freeing the node entirely; the next call that reaches the guest restores it transparently (a checkpoint restore's latency) with a fresh server-default 5m lease. Requires `idle_hibernate_seconds > 0` and must exceed it. Node-wide for unpooled keys; per-pool overrides for that pool | +| `archive_after_seconds` | 0 (off) | tier below hibernation: a hibernated claim idle this long is checkpointed to the store and its local VM dropped, freeing the node entirely; the next call that reaches the guest restores it transparently (a checkpoint restore's latency) with a fresh server-default 5m lease — whatever TTL the claim first asked for, and nothing renews it. Keep `idle_hibernate_seconds` under those 5 minutes: a woken sandbox that idles can then hibernate and be archived again, where at 300 or more the lease runs out first and the reap destroys it, its archive already consumed by the wake. A sandbox still in use when the lease ends is destroyed the same way. Requires `idle_hibernate_seconds > 0` and must exceed it. Node-wide for unpooled keys; per-pool overrides for that pool | | `archive_delete_after_seconds` | 0 (keep) | purge an archived claim's store checkpoint this long after it was archived, reclaiming storage; the claim is then gone for good. On archive this retention window replaces the live claim deadline; 0 clears the deadline so the archive is kept forever. Same node-wide/per-pool split | | `mesh` | unset | join a cluster ([Clusters](cluster.md)); unset = single node | | `pools[]` | — | warm pools, keyed by `(template, net, size)`. `warm` defaults to 4 only when both `warm` and `warm_max` are omitted; an explicit `warm: 0` is kept, so `warm: 0` under a `warm_max` starts the pool empty and grows it on demand, and `warm: 0` alone registers the key with no warm VMs (`PUT /v1/pools` never defaults `warm`); `net` is `none` or `egress`; `size` is a tier, below. Retune online without a restart via [`PUT /v1/pools`](sandboxd-api.md#put-v1pools) — omitted pools drain. This is the **first-boot seed**: once a node takes a `PUT /v1/pools`, the applied set persists to `/pools.json` and overrides this section on every later boot (a startup log notes it); delete `pools.json` to return to config-owned pools. Egress stays config-owned either way. See [state ownership](cluster.md#state-ownership) | @@ -299,7 +304,7 @@ here validates on load: "max_claims": 200, "audit_log": true, - "idle_hibernate_seconds": 300, + "idle_hibernate_seconds": 120, "archive_after_seconds": 3600, "archive_delete_after_seconds": 604800, diff --git a/docs/egress.md b/docs/egress.md index 3f8b89d8..1bceb094 100644 --- a/docs/egress.md +++ b/docs/egress.md @@ -157,7 +157,10 @@ and the secret injection that rides it — is always an explicit act. Root claims have no tenant layer and take the pool's policy whole; a key with no configured pool — a promoted template, or an image cold-booted on demand — has no pool layer, so a tenant's claim of one takes the tenant's policy alone (a -root claim of one stays denied). Secrets are registered separately and +root claim of one stays denied). The layers are read against the pool set of +the moment, not pinned at claim time: after a `PUT /v1/pools` adds or drops a +pool, a live claim of that key follows the change at its next wake or the next +restart. Secrets are registered separately and referenced by name — the value comes from the environment, never the config file. diff --git a/docs/langchain.md b/docs/langchain.md index 66549dd2..37973763 100644 --- a/docs/langchain.md +++ b/docs/langchain.md @@ -34,7 +34,8 @@ sandbox — comes back to the model as the tool's error text, not as an exception out of the agent run. Every tool's first call claims the sandbox inside the same 5-minute budget. -**Branching**: `CocoonToolkit(..., from_checkpoint="ck_...")` claims the +**Branching**: `CocoonToolkit(..., from_checkpoint="ck_...")` — the checkpoint +pins the template, lane and size, so `template` and `net` are ignored — claims the sandbox from a [checkpoint](sdk-python.md#checkpoints--branching-and-time-travel)'s captured moment instead of a clean template — agents start from prepared state (dependencies installed, repo cloned) in milliseconds, and every diff --git a/docs/mcp.md b/docs/mcp.md index 1ab4d70c..724a2941 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -44,7 +44,10 @@ Every tool call is capped at 5 minutes. | `node_info` | warm pools, live claims, drain state, capacity and mesh peers | Sandbox handles (and their tokens) are held by the server process for the -session. Checkpoints outlive sessions: `branch_checkpoint` accepts any known +session, and released with it: when the MCP client disconnects or the server +is signalled, it destroys every sandbox that session claimed, whatever +`ttl_seconds` asked for. `checkpoint` first if the state must survive. +Checkpoints outlive sessions: `branch_checkpoint` accepts any known id without a listing round-trip. If the connected node does not hold it, the claim follows a live owner probe and redirect, or heals the checkpoint locally when peer healing is enabled. `delete_checkpoint` is different: it acts on the diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index f83e2fd3..6a75bfb4 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -40,9 +40,10 @@ Auth: `Authorization: Bearer ` (when configured). - `claim_ref` is an optional opaque caller reference echoed by the scoped sandbox index; the aggregated apiserver uses `/` - `no_redirect` is set by the SDK when retrying at a redirect target -- `require_promoted` is an internal redirect field. When a redirect response - sets it, copy it into the `no_redirect` retry so the target cannot cold-boot - a promoted template name if its gossip view is stale +- `require_promoted` is an internal redirect field of volume claims, the only + path that sets or reads it. When a redirect response sets it, copy it into + the `no_redirect` retry so the target cannot cold-boot a promoted template + name if its gossip view is stale - `volumes` is an ordered list of at most eight unique catalog names. `mount` defaults to `/volumes/`; a custom value must be absolute and clean, outside the guest OS tree, unique, and non-nesting within the request. @@ -318,7 +319,12 @@ protocol: a node answering a `no_redirect` delete speaks only for itself. ## PUT /v1/pools Auth: root only (tenant tokens get 403). Replaces the node's desired warm -targets online — no restart, live claims untouched: +targets online — no restart, and no live claim's VM is touched. One thing does +follow the pool set on a node with egress policies: whether a key is pooled +decides which [policy layers](egress.md) apply to it, and a live claim picks +that up at its next wake or the next restart — a key that gains a pool with no +`egress` block loses tenant egress, a key that loses such a pool falls back to +the tenant's policy alone: ```json {"pools": [{"template": "base:24.04", "net": "none", "size": "small", @@ -359,7 +365,8 @@ guest HTTP port from a browser: body `{"token": "...", "port": 8080, "ttl_seconds": 0}` → `{"url": "http:///p//"}`. The URL's life is clamped to the claim's remaining lease; an archived claim kept forever (`archive_delete_after_seconds: 0`) has no lease, so there the -requested `ttl_seconds` stands and releasing the sandbox is what ends the URL. +requested `ttl_seconds` stands unclamped and `ttl_seconds: 0` mints a one-hour +URL; releasing the sandbox ends it early either way. 501 when the node has no `preview_listen`. The signed token embeds the sandbox id, port, and owner `advertise_addr`, so any node's preview listener can serve it (forwarding diff --git a/docs/sdk-python.md b/docs/sdk-python.md index f46bef3c..7739c865 100644 --- a/docs/sdk-python.md +++ b/docs/sdk-python.md @@ -339,7 +339,9 @@ inside the guest; `session=` routes the command into a persistent session, where only `argv` applies — the session owns the cwd, environment and user, and stderr arrives merged into stdout. A command's environment is `PATH`, `TERM` and the lane's proxy variables plus `env`, not the image's `ENV`. A -dropped connection kills a foreground command; `spawn` outlives it. +dropped connection — a `run(timeout=)` cut included — kills a foreground command +only once it next writes output; a silent one runs on in the guest. `spawn` +outlives the connection by design; `ps` then `kill` stops one still running. ## Background processes diff --git a/docs/sdk.md b/docs/sdk.md index 1e156daf..12a39be3 100644 --- a/docs/sdk.md +++ b/docs/sdk.md @@ -451,8 +451,10 @@ across Runs), `Stdout`/`Stderr` (nil discards). With `Session` set only read `/dev/null`, and stderr arrives merged into stdout. A command's environment is `PATH`, `TERM` and the lane's proxy variables plus `Env` — not the image's `ENV`; `HOME` and `USER` are set only with `User`. A dropped -connection kills a foreground command; use `Spawn` for work that must outlive -it. +connection — a canceled ctx included — kills a foreground command only once it +next writes output; a silent one (`sleep`, a quiet build) runs on in the guest. +Use `Spawn` for work that must outlive the connection; to stop a command that +is still running after a drop, find its pid with `Ps` and `Kill` it. Non-zero exits surface as `*sandbox.ExitError{Code, Stderr}` from `Exec` (alongside partial stdout); `Run` returns the code directly. @@ -492,7 +494,9 @@ err = sess.Close(ctx) ids, err := sb.Sessions(ctx) // live session ids ``` -Idle sessions are reaped guest-side after 30 minutes. +Idle sessions are reaped guest-side after 30 minutes. `Close` on a session +that is already gone — and `Lsp.Stop` on a server that already exited — answers +silkd's `not_found`; ignore it in a deferred cleanup. ## Files diff --git a/docs/silkd.md b/docs/silkd.md index b70fa660..828e2ada 100644 --- a/docs/silkd.md +++ b/docs/silkd.md @@ -27,7 +27,9 @@ fields are ignored (the forward-compatibility story). Sessions and detached processes are server-side state addressed by id — a dropped connection loses nothing (`attach` resumes). A foreground `exec` is -bound to its connection: a drop kills the child and frees its pid. The other +bound to its connection, but silkd learns of a drop only when a write to the +client fails: a child that is producing output is killed and its pid freed, a +silent one runs to completion. The other connection-bound verbs are `fs_watch`, `pty_open`, `lsp_request`, and `port_forward`. @@ -94,5 +96,7 @@ own routed network never gets them. - no fsync durability on writes (guest page cache semantics) - `fs_push` is network-failure atomic: the stream extracts into a staging dir inside `dest` and merges (local renames) only after the tar terminates - cleanly, so a truncated stream or dropped connection leaves `dest` - untouched; the residual crash window is the merge itself, microseconds + cleanly, so a truncated stream or dropped connection leaves `dest`'s + contents untouched; `dest` itself and any missing parent are created before + the stream starts and stay behind, empty, on failure. The residual crash + window is the merge itself, microseconds From 868089e6aac5d3ad9e9a64a3c50b6aaa9f5b4bb5 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 20 Sep 2026 14:07:23 +0800 Subject: [PATCH 36/53] fix(pool): drain the latest claims sequence before the retrier lets go MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit a33b56a made recommit single-flight, which dropped a recommit that found the retrier busy. If the retrier had already cloned the state for its write — commitWake's temporary record, say — and the rollback's recommit arrived before it finished, claims.json kept the stale record with no one left to replace it; an unclean restart then lost the archived claim. The retrier now clears its flag, re-reads the sequence and carries on while a newer one is unwritten. TestRecommitDrainsAChangeItWasTooBusyToSee holds the first write on a FIFO to make the interleaving deterministic. --- sandboxd/pool/claim.go | 7 +++++-- sandboxd/pool/claims.go | 6 ++++++ sandboxd/pool/claims_test.go | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 2 deletions(-) diff --git a/sandboxd/pool/claim.go b/sandboxd/pool/claim.go index 37d013e8..56965f1c 100644 --- a/sandboxd/pool/claim.go +++ b/sandboxd/pool/claim.go @@ -483,12 +483,15 @@ func (m *Manager) recommit(ctx context.Context, snap claimSnapshot) { return } go func() { - defer m.recommitting.Store(false) backoff := recommitBackoff for { err := m.store.commit(snap) if err == nil { - return + m.recommitting.Store(false) + if snap = m.store.mark(); !m.store.pending(snap) || !m.recommitting.CompareAndSwap(false, true) { + return + } + continue } log.WithFunc("pool.recommit").Error(ctx, err, "persist claims") time.Sleep(backoff) diff --git a/sandboxd/pool/claims.go b/sandboxd/pool/claims.go index f68931c7..39e6edad 100644 --- a/sandboxd/pool/claims.go +++ b/sandboxd/pool/claims.go @@ -98,6 +98,12 @@ func (s *claimStore) mark() claimSnapshot { return claimSnapshot{seq: s.seq} } +func (s *claimStore) pending(snap claimSnapshot) bool { + s.mu.Lock() + defer s.mu.Unlock() + return snap.seq > s.written +} + // reset rebuilds the whole projection; the startup path, before contention exists. func (s *claimStore) reset(claims map[string]*types.Sandbox) claimSnapshot { s.mu.Lock() diff --git a/sandboxd/pool/claims_test.go b/sandboxd/pool/claims_test.go index fd775957..b3564467 100644 --- a/sandboxd/pool/claims_test.go +++ b/sandboxd/pool/claims_test.go @@ -1,12 +1,14 @@ package pool import ( + "io" "os" "path/filepath" "reflect" "runtime" "slices" "sync" + "syscall" "testing" "time" @@ -39,6 +41,39 @@ func TestRecommitKeepsOneRetrierWhileWritesFail(t *testing.T) { }) } +func TestRecommitDrainsAChangeItWasTooBusyToSee(t *testing.T) { + m := newTestManager(t, newFakeEngine()) + sb := mustClaim(t, m, testKey) + m.store.path = filepath.Join(t.TempDir(), "claims.json") + fifo := m.store.path + ".tmp" + if err := syscall.Mkfifo(fifo, 0o600); err != nil { + t.Fatalf("mkfifo: %v", err) + } + change := func() claimSnapshot { + m.mu.Lock() + defer m.mu.Unlock() + return m.store.set(sb) + } + + m.recommit(t.Context(), change()) + waitFor(t, func() bool { + if m.store.writeMu.TryLock() { + m.store.writeMu.Unlock() + return false + } + return true + }) + m.recommit(t.Context(), change()) + + r, err := os.Open(fifo) + if err != nil { + t.Fatalf("open fifo: %v", err) + } + _, _ = io.Copy(io.Discard, r) + _ = r.Close() + waitFor(t, func() bool { return !m.recommitting.Load() && !m.store.pending(m.store.mark()) }) +} + func TestStoreRoundTrip(t *testing.T) { s := newClaimStore(t.TempDir()) claims := map[string]*types.Sandbox{ From 88234dd5344c55911ccc69a99a2407feee68d058 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 20 Sep 2026 14:44:30 +0800 Subject: [PATCH 37/53] fix(pool): a woken archive gets the lease its claim asked for MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The wake stamped a fresh server-default 5 minute lease whatever TTL the claim was made with, and nothing renews a lease, so a sandbox archived after an hour of idling and woken by its next call was destroyed five minutes later — mid-use if still busy — with its archive checkpoint already consumed. The claim record now keeps the lease it asked for and a wake grants that again; records written before the field still get the default. --- sandboxd/pool/archive.go | 2 +- sandboxd/pool/archive_test.go | 19 +++++++++++++++++++ sandboxd/pool/claim.go | 1 + sandboxd/pool/claims.go | 4 +++- sandboxd/types/types.go | 2 ++ 5 files changed, 26 insertions(+), 2 deletions(-) diff --git a/sandboxd/pool/archive.go b/sandboxd/pool/archive.go index 4eb4d89e..85673208 100644 --- a/sandboxd/pool/archive.go +++ b/sandboxd/pool/archive.go @@ -218,7 +218,7 @@ func (m *Manager) commitWake(ctx context.Context, sb *types.Sandbox, vmName, soc } ck, deadline := sb.ArchiveCk, sb.Deadline sb.VMName, sb.VsockSocket, sb.ArchiveCk = vmName, sock, "" - sb.Deadline = time.Now().Add(clampTTL(0)) // a woken sandbox is a fresh lease + sb.Deadline = time.Now().Add(clampTTL(types.Seconds(sb.LeaseSeconds))) // a woken sandbox is a fresh lease js := m.store.set(sb) m.mu.Unlock() if err := m.store.commit(js); err != nil { diff --git a/sandboxd/pool/archive_test.go b/sandboxd/pool/archive_test.go index a05bfb4c..2f0b0449 100644 --- a/sandboxd/pool/archive_test.go +++ b/sandboxd/pool/archive_test.go @@ -351,6 +351,25 @@ func TestArchiveRetentionPurge(t *testing.T) { }) } +func TestArchiveWakeGrantsTheLeaseTheClaimAskedFor(t *testing.T) { + eng := newFakeEngine() + m := newTestManager(t, eng, archivePool(3600)) + sb, err := m.ClaimProvision(t.Context(), testKey, 2*time.Hour, "acme", "", nil) + if err != nil { + t.Fatalf("claim: %v", err) + } + mustArchive(t, m, sb) + if _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { + t.Fatalf("wake: %v", err) + } + m.mu.Lock() + lease := time.Until(sb.Deadline) + m.mu.Unlock() + if lease < time.Hour || lease > 2*time.Hour { + t.Errorf("lease after the wake = %v, want the 2h the claim asked for", lease) + } +} + func TestReleaseArchivedDeletesCheckpoint(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng, archivePool(3600)) diff --git a/sandboxd/pool/claim.go b/sandboxd/pool/claim.go index 56965f1c..d86a6ab6 100644 --- a/sandboxd/pool/claim.go +++ b/sandboxd/pool/claim.go @@ -567,6 +567,7 @@ func stampIdentity(sb *types.Sandbox, ttl time.Duration) { sb.ID = "sb_" + randHex(8) sb.Token = randHex(16) sb.Deadline = time.Now().Add(ttl) + sb.LeaseSeconds = int(ttl / time.Second) } func clampTTL(ttl time.Duration) time.Duration { diff --git a/sandboxd/pool/claims.go b/sandboxd/pool/claims.go index 39e6edad..0d5967fd 100644 --- a/sandboxd/pool/claims.go +++ b/sandboxd/pool/claims.go @@ -27,6 +27,7 @@ type claimDTO struct { Key types.PoolKey `json:"key"` Token string `json:"token,omitempty"` Deadline time.Time `json:"deadline,omitzero"` + LeaseSeconds int `json:"lease_seconds,omitzero"` Tenant string `json:"tenant,omitempty"` ClaimRef string `json:"claim_ref,omitempty"` Volumes []types.Volume `json:"volumes,omitempty"` @@ -158,7 +159,8 @@ func (s *claimStore) synced() bool { func dtoOf(sb *types.Sandbox) claimDTO { return claimDTO{ ID: sb.ID, VMName: sb.VMName, Key: sb.Key, Token: sb.Token, - Deadline: sb.Deadline, Tenant: sb.Tenant, ClaimRef: sb.ClaimRef, + Deadline: sb.Deadline, LeaseSeconds: sb.LeaseSeconds, + Tenant: sb.Tenant, ClaimRef: sb.ClaimRef, Volumes: slices.Clone(sb.Volumes), VsockSocket: sb.VsockSocket, TAP: sb.TAP, HibernateSnap: sb.HibernateSnap, PendingSnap: sb.PendingSnap, ArchiveCk: sb.ArchiveCk, FromCheckpoint: sb.FromCheckpoint, diff --git a/sandboxd/types/types.go b/sandboxd/types/types.go index d70334aa..f0bff8d2 100644 --- a/sandboxd/types/types.go +++ b/sandboxd/types/types.go @@ -142,6 +142,8 @@ type Sandbox struct { Token string `json:"token,omitempty"` Deadline time.Time `json:"deadline,omitzero"` + // LeaseSeconds is the lease the claim asked for; a wake from the archive grants it again. + LeaseSeconds int `json:"lease_seconds,omitzero"` // Tenant names the owning tenant; empty means the operator claimed it. Tenant string `json:"tenant,omitempty"` From abc7d02983d11ed1d67a9a6ada19189977a32c33 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 20 Sep 2026 14:44:48 +0800 Subject: [PATCH 38/53] fix(pool): pin a claim's egress policy layer when it is made effectivePolicy asked whether the key was pooled at every arm, so a PUT /v1/pools that added a policy-less pool for a key tenants already held claims on took their egress away at the next wake or restart, and one that dropped such a pool widened the same claims to the tenant's whole allow-list. The claim record now carries the layer decided at claim time; a record written before the field keeps resolving against the live pool set, so an upgrade neither widens nor narrows a claim. --- sandboxd/pool/claim.go | 4 ++++ sandboxd/pool/claims.go | 33 +++++++++++++++++---------------- sandboxd/pool/egress.go | 6 +++++- sandboxd/pool/egress_test.go | 30 ++++++++++++++++++++++++++++++ sandboxd/types/types.go | 8 ++++++++ 5 files changed, 64 insertions(+), 17 deletions(-) diff --git a/sandboxd/pool/claim.go b/sandboxd/pool/claim.go index d86a6ab6..c2813720 100644 --- a/sandboxd/pool/claim.go +++ b/sandboxd/pool/claim.go @@ -290,6 +290,10 @@ func (m *Manager) finalizeBatch(ctx context.Context, sbs []*types.Sandbox, ttl t return quotaErr } for _, sb := range sbs { + sb.Layer = types.LayerUnpooled + if _, pooled := m.activePool(sb.Key); pooled { + sb.Layer = types.LayerPooled + } m.claimed[sb.ID] = sb m.tenantDelta(sb.Tenant, 1) } diff --git a/sandboxd/pool/claims.go b/sandboxd/pool/claims.go index 0d5967fd..1e672e36 100644 --- a/sandboxd/pool/claims.go +++ b/sandboxd/pool/claims.go @@ -22,21 +22,22 @@ type claimSnapshot struct { // claimDTO is the persisted projection of a Sandbox, copied so commit marshals off m.mu. type claimDTO struct { - ID string `json:"id"` - VMName string `json:"vm_name"` - Key types.PoolKey `json:"key"` - Token string `json:"token,omitempty"` - Deadline time.Time `json:"deadline,omitzero"` - LeaseSeconds int `json:"lease_seconds,omitzero"` - Tenant string `json:"tenant,omitempty"` - ClaimRef string `json:"claim_ref,omitempty"` - Volumes []types.Volume `json:"volumes,omitempty"` - VsockSocket string `json:"vsock_socket,omitempty"` - TAP string `json:"tap,omitempty"` - HibernateSnap string `json:"hibernate_snap,omitempty"` - PendingSnap string `json:"pending_snap,omitempty"` - ArchiveCk string `json:"archive_ck,omitempty"` - FromCheckpoint string `json:"from_checkpoint,omitempty"` + ID string `json:"id"` + VMName string `json:"vm_name"` + Key types.PoolKey `json:"key"` + Token string `json:"token,omitempty"` + Deadline time.Time `json:"deadline,omitzero"` + LeaseSeconds int `json:"lease_seconds,omitzero"` + Layer types.PolicyLayer `json:"policy_layer,omitempty"` + Tenant string `json:"tenant,omitempty"` + ClaimRef string `json:"claim_ref,omitempty"` + Volumes []types.Volume `json:"volumes,omitempty"` + VsockSocket string `json:"vsock_socket,omitempty"` + TAP string `json:"tap,omitempty"` + HibernateSnap string `json:"hibernate_snap,omitempty"` + PendingSnap string `json:"pending_snap,omitempty"` + ArchiveCk string `json:"archive_ck,omitempty"` + FromCheckpoint string `json:"from_checkpoint,omitempty"` } // claimStore persists claimed sandboxes across daemon restarts; warm VMs are not persisted. @@ -159,7 +160,7 @@ func (s *claimStore) synced() bool { func dtoOf(sb *types.Sandbox) claimDTO { return claimDTO{ ID: sb.ID, VMName: sb.VMName, Key: sb.Key, Token: sb.Token, - Deadline: sb.Deadline, LeaseSeconds: sb.LeaseSeconds, + Deadline: sb.Deadline, LeaseSeconds: sb.LeaseSeconds, Layer: sb.Layer, Tenant: sb.Tenant, ClaimRef: sb.ClaimRef, Volumes: slices.Clone(sb.Volumes), VsockSocket: sb.VsockSocket, TAP: sb.TAP, HibernateSnap: sb.HibernateSnap, PendingSnap: sb.PendingSnap, diff --git a/sandboxd/pool/egress.go b/sandboxd/pool/egress.go index 356a44e8..fa06ed4e 100644 --- a/sandboxd/pool/egress.go +++ b/sandboxd/pool/egress.go @@ -311,7 +311,11 @@ func (m *Manager) effectivePolicy(sb *types.Sandbox) (egress.Evaluator, bool) { poolPol := m.poolEgress[sb.Key] tenantPol := m.tenantEgress[sb.Tenant] if poolPol == nil { - if _, pooled := m.activePool(sb.Key); pooled || sb.Tenant == "" || tenantPol == nil { + pooled := sb.Layer == types.LayerPooled + if sb.Layer == "" { + _, pooled = m.activePool(sb.Key) + } + if pooled || sb.Tenant == "" || tenantPol == nil { return nil, false } return *tenantPol, true diff --git a/sandboxd/pool/egress_test.go b/sandboxd/pool/egress_test.go index f05a02dc..022c6610 100644 --- a/sandboxd/pool/egress_test.go +++ b/sandboxd/pool/egress_test.go @@ -211,6 +211,36 @@ func TestEgressLaneWakeFailsClosed(t *testing.T) { } } +func TestEffectivePolicyKeepsTheLayerPinnedAtClaim(t *testing.T) { + m := newTestManager(t, newFakeEngine()) + m.tenantEgress = map[string]*egress.Policy{"acme": {Allow: []egress.Rule{{Host: "b.test"}}}} + m.poolEgress = map[types.PoolKey]*egress.Policy{} + + m.pools = map[types.PoolKey]*pool{testKey: newPool(testKey)} + if _, ok := m.effectivePolicy(&types.Sandbox{Key: testKey, Tenant: "acme", Layer: types.LayerUnpooled}); !ok { + t.Error("a claim made on an unpooled key lost its tenant policy when the key gained a pool") + } + m.pools = map[types.PoolKey]*pool{} + if _, ok := m.effectivePolicy(&types.Sandbox{Key: testKey, Tenant: "acme", Layer: types.LayerPooled}); ok { + t.Error("a claim made on a policy-less pool gained the tenant policy when the pool was dropped") + } +} + +func TestClaimRecordsItsPolicyLayer(t *testing.T) { + m := newTestManager(t, newFakeEngine(), config.PoolSpec{PoolKey: testKey, Warm: 1}) + if sb := mustClaim(t, m, testKey); sb.Layer != types.LayerPooled { + t.Errorf("pooled claim layer %q, want %q", sb.Layer, types.LayerPooled) + } + unpooled := types.PoolKey{Template: "promoted-name", Net: types.NetNone, Size: types.SizeSmall} + sb, err := m.ClaimProvision(t.Context(), unpooled, time.Hour, "acme", "", nil) + if err != nil { + t.Fatalf("claim: %v", err) + } + if sb.Layer != types.LayerUnpooled { + t.Errorf("unpooled claim layer %q, want %q", sb.Layer, types.LayerUnpooled) + } +} + func TestEffectivePolicyComposition(t *testing.T) { m := newTestManager(t, newFakeEngine()) both := &egress.Policy{Allow: []egress.Rule{{Host: "a.test"}, {Host: "b.test"}}} diff --git a/sandboxd/types/types.go b/sandboxd/types/types.go index f0bff8d2..b763f2a2 100644 --- a/sandboxd/types/types.go +++ b/sandboxd/types/types.go @@ -26,6 +26,9 @@ const ( SizeXLarge Size = "xlarge" Size2XLarge Size = "2xlarge" + LayerPooled PolicyLayer = "pooled" + LayerUnpooled PolicyLayer = "unpooled" + RestoreCopy RestoreMode = "copy" RestoreOnDemand RestoreMode = "ondemand" RestoreMmap RestoreMode = "mmap" @@ -64,6 +67,9 @@ var ( // NetShape selects whether the Cloud Hypervisor guest has a NIC. type NetShape string +// PolicyLayer pins, at claim time, whether the key had a pool layer in its egress policy. +type PolicyLayer string + // RestoreMode selects cocoon's clone memory-restore strategy; empty is cocoon's default. type RestoreMode string @@ -144,6 +150,8 @@ type Sandbox struct { Deadline time.Time `json:"deadline,omitzero"` // LeaseSeconds is the lease the claim asked for; a wake from the archive grants it again. LeaseSeconds int `json:"lease_seconds,omitzero"` + // Layer is empty on a record older than the field, which resolves against the live pool set. + Layer PolicyLayer `json:"policy_layer,omitempty"` // Tenant names the owning tenant; empty means the operator claimed it. Tenant string `json:"tenant,omitempty"` From df530ce7839882ce3adc1f74e3958b8b33eb0a32 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 20 Sep 2026 15:04:58 +0800 Subject: [PATCH 39/53] fix(sdk): kill the command a canceled Run or a run timeout leaves behind MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The guest kills a foreground exec whose connection dropped only when a write to the client fails, so a silent command (sleep, a quiet build) outlived every canceled ctx and every run(timeout=) cut. Run and run now keep the pid the started frame carried and, when the ctx or the timeout ends the stream, send it a kill over a connection of their own, bounded to 5 s and best effort, before returning the cancel — the shape of sandboxd's killExec for buffered exec. A session command reports no pid and is untouched. Hot path: one ctx.Err check on the success path, no extra RPC. --- sdk/go/proc.go | 2 +- sdk/go/sandbox.go | 10 +++++-- sdk/go/sandbox_test.go | 45 +++++++++++++++++++++++++++++ sdk/go/utils.go | 19 ++++++------ sdk/python/cocoonsandbox/sandbox.py | 28 +++++++++++++++--- sdk/python/tests/test_stream.py | 12 ++++++-- 6 files changed, 97 insertions(+), 19 deletions(-) diff --git a/sdk/go/proc.go b/sdk/go/proc.go index b9ca12f9..30b28ff6 100644 --- a/sdk/go/proc.go +++ b/sdk/go/proc.go @@ -68,7 +68,7 @@ func (s *Sandbox) drainProc(ctx context.Context, req wire.Request, stdout, stder return 0, false, err } defer l.close() - code, exited, err := pumpStdio(ctx, conn, stdout, stderr) + _, code, exited, err := pumpStdio(ctx, conn, stdout, stderr) // logs closes with done after the exit frame if _, replay := req.(*wire.Logs); replay && exited && err == nil { err = terminalErr(ctx, conn) diff --git a/sdk/go/sandbox.go b/sdk/go/sandbox.go index 495c8932..b3f72d93 100644 --- a/sdk/go/sandbox.go +++ b/sdk/go/sandbox.go @@ -16,6 +16,7 @@ import ( const ( releaseTimeout = 30 * time.Second + killWait = 5 * time.Second stdinChunk = 32 * 1024 ) @@ -100,7 +101,7 @@ func (s *Sandbox) Exec(ctx context.Context, argv ...string) (string, error) { } // Run executes cmd in the sandbox, streaming stdio over one relayed silkd -// connection, and returns the exit code. +// connection, and returns the exit code; a ctx that ends first kills the command. func (s *Sandbox) Run(ctx context.Context, cmd Cmd) (int, error) { if len(cmd.Argv) == 0 { return 0, fmt.Errorf("empty argv") @@ -123,7 +124,7 @@ func (s *Sandbox) Run(ctx context.Context, cmd Cmd) (int, error) { }() } - code, exited, err := pumpStdio(ctx, conn, cmd.Stdout, cmd.Stderr) + pid, code, exited, err := pumpStdio(ctx, conn, cmd.Stdout, cmd.Stderr) // a pump still parked in Read would write into the next RPC, so its connection is not kept select { case <-stdinDone: @@ -131,6 +132,11 @@ func (s *Sandbox) Run(ctx context.Context, cmd Cmd) (int, error) { default: } if err != nil { + if ctx.Err() != nil && pid != 0 { + kctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), killWait) + _ = s.Kill(kctx, pid, 0) + cancel() + } return 0, err } if !exited { diff --git a/sdk/go/sandbox_test.go b/sdk/go/sandbox_test.go index 1defa563..6aa5d8c6 100644 --- a/sdk/go/sandbox_test.go +++ b/sdk/go/sandbox_test.go @@ -1,6 +1,8 @@ package sandbox import ( + "bufio" + "bytes" "context" "errors" "io" @@ -77,6 +79,49 @@ func TestRunContextCancel(t *testing.T) { } } +func TestRunCancelKillsTheCommandItStarted(t *testing.T) { + killed := make(chan uint32, 1) + sb := testSandbox(t, newAgentServer(t, func(conn net.Conn) { + defer conn.Close() + r := bufio.NewReader(conn) + for { + line, err := r.ReadBytes('\n') + if err != nil { + return + } + req, err := wire.DecodeRequest(bytes.TrimSpace(line)) + if err != nil { + return + } + switch req := req.(type) { + case *wire.Info: + _, _ = io.WriteString(conn, `{"type":"info","version":"test","proto":2}`+"\n") + case *wire.Exec: + _, _ = io.WriteString(conn, `{"type":"started","pid":7}`+"\n") + _, _ = io.Copy(io.Discard, r) + return + case *wire.Kill: + killed <- req.PID + _, _ = io.WriteString(conn, `{"type":"done"}`+"\n") + } + } + })) + + ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond) + defer cancel() + if _, err := sb.Run(ctx, Cmd{Argv: []string{"sleep", "300"}}); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("got %v, want deadline exceeded", err) + } + select { + case pid := <-killed: + if pid != 7 { + t.Errorf("killed pid %d, want 7", pid) + } + default: + t.Fatal("no kill reached the guest before Run returned") + } +} + func TestUpgradeKeepsCoalescedBytes(t *testing.T) { mux := http.NewServeMux() mux.HandleFunc("GET /v1/sandboxes/{id}/agent", func(w http.ResponseWriter, r *http.Request) { diff --git a/sdk/go/utils.go b/sdk/go/utils.go index e7b16a68..8c11cb33 100644 --- a/sdk/go/utils.go +++ b/sdk/go/utils.go @@ -71,33 +71,34 @@ func (s *Sandbox) downloadRPC(ctx context.Context, req wire.Request, sink func([ return l.done(drainData(ctx, conn, sink)) } -// pumpStdio copies stdout/stderr frames to the writers until the terminal frame: exit carries the code, done means the stream ended without one. -func pumpStdio(ctx context.Context, conn *silkd.Conn, stdout, stderr io.Writer) (code int32, exited bool, err error) { +// pumpStdio copies stdout/stderr frames to the writers until the terminal frame: exit carries the code, done means the stream ended without one; pid is the one started reported, zero before it. +func pumpStdio(ctx context.Context, conn *silkd.Conn, stdout, stderr io.Writer) (pid uint32, code int32, exited bool, err error) { stdout = cmp.Or(stdout, io.Discard) stderr = cmp.Or(stderr, io.Discard) for { resp, err := recv(ctx, conn) if err != nil { - return 0, false, err + return pid, 0, false, err } switch resp := resp.(type) { case *wire.Started: + pid = resp.PID case *wire.Stdout: if _, err := stdout.Write(resp.Data); err != nil { - return 0, false, err + return pid, 0, false, err } case *wire.Stderr: if _, err := stderr.Write(resp.Data); err != nil { - return 0, false, err + return pid, 0, false, err } case *wire.Exit: - return resp.Code, true, nil + return pid, resp.Code, true, nil case *wire.Done: - return 0, false, nil + return pid, 0, false, nil case *wire.ErrorResp: - return 0, false, resp + return pid, 0, false, resp default: - return 0, false, unexpected(resp) + return pid, 0, false, unexpected(resp) } } } diff --git a/sdk/python/cocoonsandbox/sandbox.py b/sdk/python/cocoonsandbox/sandbox.py index f3a3f21b..757e5d06 100644 --- a/sdk/python/cocoonsandbox/sandbox.py +++ b/sdk/python/cocoonsandbox/sandbox.py @@ -19,6 +19,7 @@ from .client import Client _ACCEPT_POLL_SECONDS = 1.0 +_KILL_WAIT_SECONDS = 5.0 class Sandbox: @@ -95,11 +96,12 @@ def run( on_stderr: Callable[[bytes], object] | None = None, timeout: float | None = None, ) -> int: - """Streams raw output bytes and returns the exit code; timeout bounds the entire call.""" + """Streams raw output bytes and returns the exit code; timeout bounds the call and kills what it cuts.""" if timeout is not None and timeout <= 0: raise ValueError("timeout must be positive") deadline = None if timeout is None else time.monotonic() + timeout expired = threading.Event() + started: list[int] = [] try: conn = self._connect(deadline) except (ProtocolError, TimeoutError): @@ -124,9 +126,11 @@ def run( else: with contextlib.suppress(ProtocolError): conn.send("stdin_close") - code = _pump_stdio(conn, on_stdout, on_stderr) + code = _pump_stdio(conn, on_stdout, on_stderr, started.append) except (ProtocolError, OSError): if expired.is_set(): + if started: + self._kill_after_cut(started[0]) raise SandboxTimeout(f"command did not finish within {timeout}s") from None raise except SilkdError: @@ -497,6 +501,17 @@ def _drain_proc( _expect(conn, "done") return code + def _kill_after_cut(self, pid: int) -> None: + with contextlib.suppress(SandboxError, OSError): + conn = self._connect(time.monotonic() + _KILL_WAIT_SECONDS) + with self._lease(conn): + conn.settimeout(_KILL_WAIT_SECONDS) + try: + conn.send("kill", pid=pid) + _expect(conn, "done") + finally: + conn.settimeout(None) + class Session(_Closeable): """A persistent shell inside the sandbox, addressed by id.""" @@ -644,11 +659,16 @@ def _feed_stdin(conn: Conn, stdin: bytes) -> None: def _pump_stdio( - conn: Conn, on_stdout: Callable[[bytes], object] | None, on_stderr: Callable[[bytes], object] | None + conn: Conn, + on_stdout: Callable[[bytes], object] | None, + on_stderr: Callable[[bytes], object] | None, + on_started: Callable[[int], object] | None = None, ) -> int | None: for frame in conn.recv_until("exit", "done"): t = frame["type"] - if t == "stdout" and on_stdout: + if t == "started" and on_started: + on_started(cast(int, _need(frame, "pid"))) + elif t == "stdout" and on_stdout: on_stdout(_need(frame, "data")) elif t == "stderr" and on_stderr: on_stderr(_need(frame, "data")) diff --git a/sdk/python/tests/test_stream.py b/sdk/python/tests/test_stream.py index fb9e966c..7e1aab91 100644 --- a/sdk/python/tests/test_stream.py +++ b/sdk/python/tests/test_stream.py @@ -49,11 +49,15 @@ def serve_port_forward(server: socket.socket, quiet: float, ops: list[str]) -> N conn.close() -def serve_started_then_hang(server: socket.socket, quiet: float) -> None: +def serve_started_then_hang(server: socket.socket, quiet: float, kills: list[dict]) -> None: conn, _ = server.accept() reader = accept_upgrade(conn) reader.readline() conn.sendall(b'{"type":"started","pid":7}\n') + killer, _ = server.accept() + kills.append(json.loads(accept_upgrade(killer).readline())) + killer.sendall(b'{"type":"done"}\n') + killer.close() time.sleep(quiet) conn.close() @@ -80,10 +84,11 @@ def test_port_stream_outlives_the_client_timeout(): assert ops == ["port_forward"] -def test_run_timeout_cuts_a_silent_command(): +def test_run_timeout_cuts_a_silent_command_and_kills_it(): server = socket.create_server(("127.0.0.1", 0)) addr = f"127.0.0.1:{server.getsockname()[1]}" - threading.Thread(target=serve_started_then_hang, args=(server, 5 * TIMEOUT), daemon=True).start() + kills: list[dict] = [] + threading.Thread(target=serve_started_then_hang, args=(server, 5 * TIMEOUT, kills), daemon=True).start() sb = legacy_sandbox(addr) started = time.monotonic() try: @@ -92,6 +97,7 @@ def test_run_timeout_cuts_a_silent_command(): finally: server.close() assert time.monotonic() - started < 3 * TIMEOUT + assert kills == [{"v": 1, "op": "kill", "pid": 7}], kills def test_run_timeout_cuts_a_blocked_exec_send(monkeypatch): From 2e6d75912d743d0d917bbf9e3b3a0423f2ce6f0d Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 20 Sep 2026 15:04:58 +0800 Subject: [PATCH 40/53] docs: lease, policy layer and cancel semantics as the code has them A wake grants the lease the claim was made with, a claim's policy layer is settled when it is made, a session command is the session's and Session.Close ends it, and a canceled Run or a run timeout kills the command it cuts. --- docs/deploy.md | 2 +- docs/egress.md | 8 ++++---- docs/langchain.md | 2 +- docs/mcp.md | 3 ++- docs/openai-adapter.md | 2 +- docs/sandboxd-api.md | 15 ++++++++------- docs/sdk-python.md | 17 ++++++++++++----- docs/sdk.md | 18 ++++++++++++------ docs/silkd.md | 3 ++- 9 files changed, 43 insertions(+), 27 deletions(-) diff --git a/docs/deploy.md b/docs/deploy.md index 62fd318b..6a676b38 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -128,7 +128,7 @@ expecting traffic. | `max_claims` | 0 (unlimited) | node-wide cap on live claims; claim/fork/branch requests beyond it answer 429 with the pool state unharmed (on a cluster a non-volume claim tries a warm-peer redirect first; a volume claim answers the 429 with no redirect) | | `audit_log` | false | append every relayed frame that opens an RPC — its op + addressing fields, never payloads — to `/audit.jsonl`; continuation `data`, `data_end`, `stdin`, and `stdin_close` frames pass through unrecorded. The file is size-rotated with one `.1` backup. Records are `{t, id, op}` plus whichever addressing fields the op carries (`argv`, `path`, `dest`, `from`, `to`, `url`, `session`, `port`), plus `method` (`GET`, `CONNECT`, `SOCKS5`, …), `decision` and `secret` (the ref name, never its value) on `egress` records; preview accesses record as op `preview`, one per request. An opening frame whose first line exceeds 4 KiB records as op `oversized` with no addressing fields | | `idle_hibernate_seconds` | 0 (off) | node-wide idle policy for unpooled claims (template/checkpoint claims): a none-lane claim is hibernated once it has had no open data-plane connection (relay, buffered exec, preview) and no egress request in flight for this long; the clock restarts when the last connection closes, so a long command is never cut short. The next call that reaches the guest wakes it transparently. Per-pool `idle_hibernate_seconds` does the same for that pool's claims; pooled keys ignore the node-wide value, and egress pools reject it because they cannot resume safely. Opt in deliberately: a wake costs latency and the snapshot, so callers with their own idle logic must not pay twice | -| `archive_after_seconds` | 0 (off) | tier below hibernation: a hibernated claim idle this long is checkpointed to the store and its local VM dropped, freeing the node entirely; the next call that reaches the guest restores it transparently (a checkpoint restore's latency) with a fresh server-default 5m lease — whatever TTL the claim first asked for, and nothing renews it. Keep `idle_hibernate_seconds` under those 5 minutes: a woken sandbox that idles can then hibernate and be archived again, where at 300 or more the lease runs out first and the reap destroys it, its archive already consumed by the wake. A sandbox still in use when the lease ends is destroyed the same way. Requires `idle_hibernate_seconds > 0` and must exceed it. Node-wide for unpooled keys; per-pool overrides for that pool | +| `archive_after_seconds` | 0 (off) | tier below hibernation: a hibernated claim idle this long is checkpointed to the store and its local VM dropped, freeing the node entirely; the next call that reaches the guest restores it transparently (a checkpoint restore's latency) with a fresh lease of the TTL the claim was made with (the server default of 5m when it asked for none); nothing renews a lease. Keep `idle_hibernate_seconds` under the leases your claims use, or a woken sandbox that idles again is destroyed by the reap before it can hibernate, its archive already consumed by the wake; one still in use when its lease ends is destroyed the same way. Requires `idle_hibernate_seconds > 0` and must exceed it. Node-wide for unpooled keys; per-pool overrides for that pool | | `archive_delete_after_seconds` | 0 (keep) | purge an archived claim's store checkpoint this long after it was archived, reclaiming storage; the claim is then gone for good. On archive this retention window replaces the live claim deadline; 0 clears the deadline so the archive is kept forever. Same node-wide/per-pool split | | `mesh` | unset | join a cluster ([Clusters](cluster.md)); unset = single node | | `pools[]` | — | warm pools, keyed by `(template, net, size)`. `warm` defaults to 4 only when both `warm` and `warm_max` are omitted; an explicit `warm: 0` is kept, so `warm: 0` under a `warm_max` starts the pool empty and grows it on demand, and `warm: 0` alone registers the key with no warm VMs (`PUT /v1/pools` never defaults `warm`); `net` is `none` or `egress`; `size` is a tier, below. Retune online without a restart via [`PUT /v1/pools`](sandboxd-api.md#put-v1pools) — omitted pools drain. This is the **first-boot seed**: once a node takes a `PUT /v1/pools`, the applied set persists to `/pools.json` and overrides this section on every later boot (a startup log notes it); delete `pools.json` to return to config-owned pools. Egress stays config-owned either way. See [state ownership](cluster.md#state-ownership) | diff --git a/docs/egress.md b/docs/egress.md index 1bceb094..19bb1c82 100644 --- a/docs/egress.md +++ b/docs/egress.md @@ -157,10 +157,10 @@ and the secret injection that rides it — is always an explicit act. Root claims have no tenant layer and take the pool's policy whole; a key with no configured pool — a promoted template, or an image cold-booted on demand — has no pool layer, so a tenant's claim of one takes the tenant's policy alone (a -root claim of one stays denied). The layers are read against the pool set of -the moment, not pinned at claim time: after a `PUT /v1/pools` adds or drops a -pool, a live claim of that key follows the change at its next wake or the next -restart. Secrets are registered separately and +root claim of one stays denied). Which layers a claim has is settled when it +is made and kept for its life: a `PUT /v1/pools` that adds or drops a pool +changes the claims made after it, never a live one (a record from before this +rule resolves against the live pool set). Secrets are registered separately and referenced by name — the value comes from the environment, never the config file. diff --git a/docs/langchain.md b/docs/langchain.md index 37973763..0d512429 100644 --- a/docs/langchain.md +++ b/docs/langchain.md @@ -20,7 +20,7 @@ schemas, sync-native with `asyncio.to_thread` async bridges): | tool | what it does | |---|---| -| `sandbox_exec` | run a shell command, cut off after 5 minutes with the reply saying so; the budget is one wall clock over the claim, the dial and the command, so a first call that waits on a slow or redirecting cluster still answers inside it; stdout/stderr/exit code; disk state persists across calls | +| `sandbox_exec` | run a shell command, cut off and killed after 5 minutes with the reply saying so; the budget is one wall clock over the claim, the dial and the command, so a first call that waits on a slow or redirecting cluster still answers inside it; stdout/stderr/exit code; disk state persists across calls | | `sandbox_write_file` | write a text file (atomic on the guest; the parent directory must exist) | | `sandbox_read_file` | read a text file | | `sandbox_list_dir` | list a directory as JSON | diff --git a/docs/mcp.md b/docs/mcp.md index 724a2941..5988f317 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -23,7 +23,8 @@ sandbox-mcp .` ## Tools -Every tool call is capped at 5 minutes. +Every tool call is capped at 5 minutes; an `exec` that hits the cap is killed +in the guest. | tool | what it does | |---|---| diff --git a/docs/openai-adapter.md b/docs/openai-adapter.md index 506c6a1e..8c4e27f3 100644 --- a/docs/openai-adapter.md +++ b/docs/openai-adapter.md @@ -32,7 +32,7 @@ pair over the sync [Python SDK](sdk-python.md), bridged with | SDK surface | cocoon | |---|---| | `create` | `Client.new` — one claimed sandbox per session | -| session `exec` | `Sandbox.run` (stdout/stderr/exit); a per-call `timeout` is `run`'s wall clock and surfaces as `TimeoutError` | +| session `exec` | `Sandbox.run` (stdout/stderr/exit); a per-call `timeout` is `run`'s wall clock: it kills the command in the guest and surfaces as `TimeoutError` | | `read` / `write` | `Sandbox.read_file` / `write_file`; a missing path on `read` → `FileNotFoundError`; a per-call `user` raises `NotImplementedError` (fs verbs carry no user on the wire) | | `persist_workspace` / `hydrate_workspace` | `Sandbox.pull` / `push` (tar) | | exposed port | `Sandbox.proxy_port` | diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 6a75bfb4..a96fcae0 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -233,7 +233,8 @@ otherwise only a side effect of the next agent access, so this is the explicit form for warming a sandbox ahead of use. Idempotent on one already running. A hibernated sandbox retains its existing deadline. An archived sandbox instead uses `archive_delete_after_seconds` as its retention deadline -while stored, and waking it starts a fresh server-default 5m lease. 204 on +while stored, and waking it starts a fresh lease of the TTL the claim was made +with (the server default when it asked for none). 204 on success, 404 unknown id or wrong token. ## POST /v1/sandboxes/{id}/fork @@ -319,12 +320,12 @@ protocol: a node answering a `no_redirect` delete speaks only for itself. ## PUT /v1/pools Auth: root only (tenant tokens get 403). Replaces the node's desired warm -targets online — no restart, and no live claim's VM is touched. One thing does -follow the pool set on a node with egress policies: whether a key is pooled -decides which [policy layers](egress.md) apply to it, and a live claim picks -that up at its next wake or the next restart — a key that gains a pool with no -`egress` block loses tenant egress, a key that loses such a pool falls back to -the tenant's policy alone: +targets online — no restart, and no live claim's VM is touched. On a node with egress policies, +whether a key is pooled decides which [policy layers](egress.md) a claim of it +gets; that is settled when the claim is made, so adding or dropping a pool here +changes only claims made afterwards — a key that gains a pool with no `egress` +block leaves new tenant claims without egress, a key that loses such a pool +gives new tenant claims the tenant's policy alone: ```json {"pools": [{"template": "base:24.04", "net": "none", "size": "small", diff --git a/docs/sdk-python.md b/docs/sdk-python.md index 7739c865..b77ce0db 100644 --- a/docs/sdk-python.md +++ b/docs/sdk-python.md @@ -191,7 +191,8 @@ gets one connection per call as before. If that deployment also enables `archive_after_seconds`, archiving replaces the original claim deadline with the archive-retention deadline (or no deadline when archives are kept forever). Waking an archive starts a fresh -server-default 5m lease. The existing handle's `sb.deadline` remains the value +lease of the TTL the claim was made with (the server default when it asked +for none). The existing handle's `sb.deadline` remains the value returned when that handle was created; call `client.sandboxes()` to read the current server deadline after an archive/wake transition. @@ -338,10 +339,16 @@ split multi-byte sequences) and returns the exit code. `user` de-escalates inside the guest; `session=` routes the command into a persistent session, where only `argv` applies — the session owns the cwd, environment and user, and stderr arrives merged into stdout. A command's environment is `PATH`, -`TERM` and the lane's proxy variables plus `env`, not the image's `ENV`. A -dropped connection — a `run(timeout=)` cut included — kills a foreground command -only once it next writes output; a silent one runs on in the guest. `spawn` -outlives the connection by design; `ps` then `kill` stops one still running. +`TERM` and the lane's proxy variables plus `env`, not the image's `ENV`. The +guest kills a foreground command whose connection dropped only once it next +writes output, so a `run(timeout=)` cut does not leave that to the drop: once +the guest has reported the command's pid, the cut sends that pid a `kill` over +a connection of its own (best effort, bounded to 5 s) before `SandboxTimeout` +is raised. A client that only drops the connection leaves a silent command +running in the guest; `ps` then `kill` stops it. `spawn` outlives the +connection by design. A command run through a session is the session's: it +reports no pid, so no cut kills it and it is not in `ps`; it runs to +completion, and the session's `close()` is what ends the session. ## Background processes diff --git a/docs/sdk.md b/docs/sdk.md index 12a39be3..4ff1c300 100644 --- a/docs/sdk.md +++ b/docs/sdk.md @@ -287,7 +287,8 @@ egress request) is not swept; the idle clock restarts when that connection ends. If that deployment also enables `archive_after_seconds`, archiving replaces the original claim deadline with the archive-retention deadline (or no deadline when archives are kept forever). Waking an archive starts a fresh -server-default 5m lease. The existing handle's `Sandbox.Deadline` is the value +lease of the TTL the claim was made with (the server default when it asked +for none). The existing handle's `Sandbox.Deadline` is the value returned when that handle was created; call `Client.Sandboxes` to read the current server deadline after an archive/wake transition. @@ -450,11 +451,16 @@ across Runs), `Stdout`/`Stderr` (nil discards). With `Session` set only `Argv` applies: the session owns the cwd, environment and user, its commands read `/dev/null`, and stderr arrives merged into stdout. A command's environment is `PATH`, `TERM` and the lane's proxy variables plus `Env` — not -the image's `ENV`; `HOME` and `USER` are set only with `User`. A dropped -connection — a canceled ctx included — kills a foreground command only once it -next writes output; a silent one (`sleep`, a quiet build) runs on in the guest. -Use `Spawn` for work that must outlive the connection; to stop a command that -is still running after a drop, find its pid with `Ps` and `Kill` it. +the image's `ENV`; `HOME` and `USER` are set only with `User`. The guest kills +a foreground command whose connection dropped only once it next writes output, +so `Run` does not leave that to the drop: when its ctx ends after the guest +reported the command's pid, `Run` sends that pid a `kill` over a connection of +its own (best effort, bounded to 5 s) before returning the ctx error. A client +that only drops the connection leaves a silent command (`sleep`, a quiet build) +running in the guest; find its pid with `Ps` and `Kill` it. Use `Spawn` for +work that must outlive the connection. A command run inside a `Session` is the +session's: it reports no pid, so no cancel kills it and it does not appear in +`Ps`; it runs to completion, and `Session.Close` is what ends the session. Non-zero exits surface as `*sandbox.ExitError{Code, Stderr}` from `Exec` (alongside partial stdout); `Run` returns the code directly. diff --git a/docs/silkd.md b/docs/silkd.md index 828e2ada..145fbdec 100644 --- a/docs/silkd.md +++ b/docs/silkd.md @@ -29,7 +29,8 @@ Sessions and detached processes are server-side state addressed by id — a dropped connection loses nothing (`attach` resumes). A foreground `exec` is bound to its connection, but silkd learns of a drop only when a write to the client fails: a child that is producing output is killed and its pid freed, a -silent one runs to completion. The other +silent one runs to completion — which is why the SDKs' `Run`/`run` answer a +canceled ctx or a timeout with a `kill` of the pid `started` reported. The other connection-bound verbs are `fs_watch`, `pty_open`, `lsp_request`, and `port_forward`. From 019241b0cca9a87b447828fdce1eecd565cceeab Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 20 Sep 2026 15:16:19 +0800 Subject: [PATCH 41/53] fix(sdk/python): bound the kill a cut sends by one absolute deadline A socket timeout bounds each read, not the reply: a guest trickling the kill's done frame a byte at a time kept run() past its timeout for as long as it liked. The cleanup now arms the same watchdog run() uses, which aborts the connection at deadline whatever arrives, and keeps only the last started pid instead of every started frame a guest cares to send. --- sdk/python/cocoonsandbox/sandbox.py | 23 ++++++++++--------- sdk/python/tests/test_stream.py | 34 +++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 12 deletions(-) diff --git a/sdk/python/cocoonsandbox/sandbox.py b/sdk/python/cocoonsandbox/sandbox.py index 757e5d06..5b3d2a14 100644 --- a/sdk/python/cocoonsandbox/sandbox.py +++ b/sdk/python/cocoonsandbox/sandbox.py @@ -6,6 +6,7 @@ import socket import threading import time +from collections import deque from collections.abc import Callable, Iterator from typing import TYPE_CHECKING, Any, cast @@ -101,7 +102,7 @@ def run( raise ValueError("timeout must be positive") deadline = None if timeout is None else time.monotonic() + timeout expired = threading.Event() - started: list[int] = [] + started: deque[int] = deque(maxlen=1) try: conn = self._connect(deadline) except (ProtocolError, TimeoutError): @@ -109,7 +110,7 @@ def run( raise SandboxTimeout(f"command did not finish within {timeout}s") from None raise with self._lease(conn): - watchdog = _arm_watchdog(conn, deadline, expired) + watchdog = None if deadline is None else _arm_watchdog(conn, deadline, expired) pump = threading.Thread(target=_feed_stdin, args=(conn, stdin), daemon=True) if stdin else None try: conn.send( @@ -502,15 +503,16 @@ def _drain_proc( return code def _kill_after_cut(self, pid: int) -> None: + deadline = time.monotonic() + _KILL_WAIT_SECONDS with contextlib.suppress(SandboxError, OSError): - conn = self._connect(time.monotonic() + _KILL_WAIT_SECONDS) - with self._lease(conn): - conn.settimeout(_KILL_WAIT_SECONDS) - try: + conn = self._connect(deadline) + watchdog = _arm_watchdog(conn, deadline, threading.Event()) + try: + with self._lease(conn): conn.send("kill", pid=pid) _expect(conn, "done") - finally: - conn.settimeout(None) + finally: + watchdog.cancel() class Session(_Closeable): @@ -637,10 +639,7 @@ def _send_chunks(conn: Conn, data: bytes, op: str = "data", chunk: int = FS_CHUN conn.send(op, data=view[off : off + chunk]) -def _arm_watchdog(conn: Conn, deadline: float | None, expired: threading.Event) -> threading.Timer | None: - if deadline is None: - return None - +def _arm_watchdog(conn: Conn, deadline: float, expired: threading.Event) -> threading.Timer: def cut() -> None: expired.set() conn.abort() diff --git a/sdk/python/tests/test_stream.py b/sdk/python/tests/test_stream.py index 7e1aab91..c9e281bf 100644 --- a/sdk/python/tests/test_stream.py +++ b/sdk/python/tests/test_stream.py @@ -1,6 +1,7 @@ """The client timeout bounds the dial and the upgrade, not the frames that follow: a guest stream outlives it.""" +import contextlib import json import socket import threading @@ -10,6 +11,7 @@ from conftest import accept_upgrade, sandbox_at from cocoonsandbox import Sandbox +from cocoonsandbox import sandbox as sandbox_module TIMEOUT = 0.2 @@ -62,6 +64,21 @@ def serve_started_then_hang(server: socket.socket, quiet: float, kills: list[dic conn.close() +def serve_started_then_trickle_the_kill_reply(server: socket.socket, kills: list[dict]) -> None: + conn, _ = server.accept() + reader = accept_upgrade(conn) + reader.readline() + conn.sendall(b'{"type":"started","pid":7}\n') + killer, _ = server.accept() + kills.append(json.loads(accept_upgrade(killer).readline())) + with contextlib.suppress(OSError): + for byte in b'{"type":"done"}\n': + time.sleep(0.75 * TIMEOUT) + killer.sendall(bytes([byte])) + killer.close() + conn.close() + + def serve_silence(server: socket.socket) -> None: conn, _ = server.accept() conn.recv(4096) @@ -100,6 +117,23 @@ def test_run_timeout_cuts_a_silent_command_and_kills_it(): assert kills == [{"v": 1, "op": "kill", "pid": 7}], kills +def test_run_timeout_bounds_the_kill_it_sends(monkeypatch): + monkeypatch.setattr(sandbox_module, "_KILL_WAIT_SECONDS", TIMEOUT) + server = socket.create_server(("127.0.0.1", 0)) + addr = f"127.0.0.1:{server.getsockname()[1]}" + kills: list[dict] = [] + threading.Thread(target=serve_started_then_trickle_the_kill_reply, args=(server, kills), daemon=True).start() + sb = legacy_sandbox(addr) + started = time.monotonic() + try: + with pytest.raises(TimeoutError): + sb.run(["sleep", "9"], timeout=TIMEOUT) + finally: + server.close() + assert time.monotonic() - started < 4 * TIMEOUT + assert kills == [{"v": 1, "op": "kill", "pid": 7}], kills + + def test_run_timeout_cuts_a_blocked_exec_send(monkeypatch): sb = legacy_sandbox("127.0.0.1:1") conn = BlockedSendConn() From 51e6d90591abad7dfcf015ac94a32e30f22e8aec Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 20 Sep 2026 15:16:19 +0800 Subject: [PATCH 42/53] docs: the exec cap may hold a reply up to 5 s while the kill lands --- docs/langchain.md | 2 +- docs/mcp.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/langchain.md b/docs/langchain.md index 0d512429..fa59dd3a 100644 --- a/docs/langchain.md +++ b/docs/langchain.md @@ -20,7 +20,7 @@ schemas, sync-native with `asyncio.to_thread` async bridges): | tool | what it does | |---|---| -| `sandbox_exec` | run a shell command, cut off and killed after 5 minutes with the reply saying so; the budget is one wall clock over the claim, the dial and the command, so a first call that waits on a slow or redirecting cluster still answers inside it; stdout/stderr/exit code; disk state persists across calls | +| `sandbox_exec` | run a shell command, cut off after 5 minutes with the reply saying so (the cut also kills the command in the guest, which can hold the reply up to 5 s more); the budget is one wall clock over the claim, the dial and the command, so a first call that waits on a slow or redirecting cluster still answers inside it; stdout/stderr/exit code; disk state persists across calls | | `sandbox_write_file` | write a text file (atomic on the guest; the parent directory must exist) | | `sandbox_read_file` | read a text file | | `sandbox_list_dir` | list a directory as JSON | diff --git a/docs/mcp.md b/docs/mcp.md index 5988f317..a63593ad 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -24,7 +24,7 @@ sandbox-mcp .` ## Tools Every tool call is capped at 5 minutes; an `exec` that hits the cap is killed -in the guest. +in the guest, which can hold its reply up to 5 s past the cap. | tool | what it does | |---|---| From 865cb4713310019bb785feaa48c628648ae0fae5 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 20 Sep 2026 15:25:52 +0800 Subject: [PATCH 43/53] fix(types): clamp a negative wire seconds count to the server default Seconds saturated only the positive half: min(n, MaxInt64/1e9) returns a negative n unchanged, so ttl_seconds -10000000000 still wrapped into a positive duration and clampTTL granted the 24 h cap where 0, the server default, was meant. A small negative already meant "default" through clampTTL's <= 0 branch; the low clamp keeps that and closes the wrap. --- sandboxd/types/api.go | 4 ++-- sandboxd/types/types_test.go | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/sandboxd/types/api.go b/sandboxd/types/api.go index 8a98f8ff..58d72545 100644 --- a/sandboxd/types/api.go +++ b/sandboxd/types/api.go @@ -129,7 +129,7 @@ type PreviewResponse struct { URL string `json:"url"` } -// Seconds converts wire seconds to a duration and saturates where the multiply would overflow. +// Seconds converts wire seconds to a duration: a negative count is zero and a huge one saturates instead of wrapping. func Seconds(n int) time.Duration { - return time.Duration(min(int64(n), math.MaxInt64/int64(time.Second))) * time.Second + return time.Duration(min(max(int64(n), 0), math.MaxInt64/int64(time.Second))) * time.Second } diff --git a/sandboxd/types/types_test.go b/sandboxd/types/types_test.go index 2e4c9a6b..d4a541ed 100644 --- a/sandboxd/types/types_test.go +++ b/sandboxd/types/types_test.go @@ -15,6 +15,11 @@ func TestSecondsSaturatesInsteadOfWrapping(t *testing.T) { if got := (TTLField{TTLSeconds: 9223372037}).TTL(); got <= 24*time.Hour { t.Errorf("TTL() = %v for a huge ttl_seconds, want it to reach the 24h clamp", got) } + for _, n := range []int{-1, -10_000_000_000, math.MinInt64} { + if got := Seconds(n); got != 0 { + t.Errorf("Seconds(%d) = %v, want 0 (the server default), never a wrapped positive", n, got) + } + } if got := Seconds(90); got != 90*time.Second { t.Errorf("Seconds(90) = %v", got) } From 9ed5cd68c99897778e3fdd876213629fc2053271 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 20 Sep 2026 15:25:53 +0800 Subject: [PATCH 44/53] fix(sdk/python): cancel the kill's watchdog before its connection is parked The watchdog was cancelled outside the lease, so a timer firing between the park and the cancel shut down a connection already in the pool. The lease now wraps the whole cleanup: the cancel runs before the park, and a watchdog that fails to start leaves the connection to the lease's close. --- sdk/python/cocoonsandbox/sandbox.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/sdk/python/cocoonsandbox/sandbox.py b/sdk/python/cocoonsandbox/sandbox.py index 5b3d2a14..29ea2434 100644 --- a/sdk/python/cocoonsandbox/sandbox.py +++ b/sdk/python/cocoonsandbox/sandbox.py @@ -504,13 +504,11 @@ def _drain_proc( def _kill_after_cut(self, pid: int) -> None: deadline = time.monotonic() + _KILL_WAIT_SECONDS - with contextlib.suppress(SandboxError, OSError): - conn = self._connect(deadline) + with contextlib.suppress(SandboxError, OSError), self._lease(self._connect(deadline)) as conn: watchdog = _arm_watchdog(conn, deadline, threading.Event()) try: - with self._lease(conn): - conn.send("kill", pid=pid) - _expect(conn, "done") + conn.send("kill", pid=pid) + _expect(conn, "done") finally: watchdog.cancel() From bfa382388bc2fd20dc96c972b0a4ee566edf5c90 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 20 Sep 2026 15:25:53 +0800 Subject: [PATCH 45/53] review: say what recommit and LeaseSeconds do now recommit is one retrier per manager that writes the current state until nothing newer is pending, not a per-call background persist; LeaseSeconds is the lease the claim was granted, the clamped value, not the one it asked for. The test name follows the field. --- sandboxd/pool/archive_test.go | 2 +- sandboxd/pool/claim.go | 2 +- sandboxd/types/types.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sandboxd/pool/archive_test.go b/sandboxd/pool/archive_test.go index 2f0b0449..cc8768a2 100644 --- a/sandboxd/pool/archive_test.go +++ b/sandboxd/pool/archive_test.go @@ -351,7 +351,7 @@ func TestArchiveRetentionPurge(t *testing.T) { }) } -func TestArchiveWakeGrantsTheLeaseTheClaimAskedFor(t *testing.T) { +func TestArchiveWakeRegrantsTheClaimsLease(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng, archivePool(3600)) sb, err := m.ClaimProvision(t.Context(), testKey, 2*time.Hour, "acme", "", nil) diff --git a/sandboxd/pool/claim.go b/sandboxd/pool/claim.go index c2813720..389c528c 100644 --- a/sandboxd/pool/claim.go +++ b/sandboxd/pool/claim.go @@ -481,7 +481,7 @@ func (m *Manager) purgeArchiveCk(ctx context.Context, id, ck, tenant string) { m.recordUsage(ctx, usageEvent{Event: "archive_delete", ID: id, Reference: ck, Tenant: tenant}) } -// recommit re-persists in the background until success or a newer durable write; detached. +// recommit keeps claims.json catching up with the store: one retrier at a time writes the current state until it is durable and nothing newer is pending. func (m *Manager) recommit(ctx context.Context, snap claimSnapshot) { if !m.recommitting.CompareAndSwap(false, true) { return diff --git a/sandboxd/types/types.go b/sandboxd/types/types.go index b763f2a2..696c3a5c 100644 --- a/sandboxd/types/types.go +++ b/sandboxd/types/types.go @@ -148,7 +148,7 @@ type Sandbox struct { Token string `json:"token,omitempty"` Deadline time.Time `json:"deadline,omitzero"` - // LeaseSeconds is the lease the claim asked for; a wake from the archive grants it again. + // LeaseSeconds is the lease the claim was granted; a wake from the archive grants it again. LeaseSeconds int `json:"lease_seconds,omitzero"` // Layer is empty on a record older than the field, which resolves against the live pool set. Layer PolicyLayer `json:"policy_layer,omitempty"` From f10b9b39e1feb43105f88cb4b44aa64c6cb27a8e Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 20 Sep 2026 15:25:53 +0800 Subject: [PATCH 46/53] test(engine): pin the port reader to one bulk data frame --- sandboxd/engine/portconn_test.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/sandboxd/engine/portconn_test.go b/sandboxd/engine/portconn_test.go index 759488de..0d8cda57 100644 --- a/sandboxd/engine/portconn_test.go +++ b/sandboxd/engine/portconn_test.go @@ -10,6 +10,8 @@ import ( "os" "path/filepath" "testing" + + "github.com/cocoonstack/sandbox/protocol/wire" ) func TestGuestPortConnFramesBothWays(t *testing.T) { @@ -100,3 +102,13 @@ func TestGuestPortConnRoundTripsFixtures(t *testing.T) { t.Errorf("write frame %q, want fixture %q", line[:n], want) } } + +func TestPortReadBufHoldsOneBulkFrame(t *testing.T) { + frame, err := wire.EncodeResponse(wire.DataResp{Data: make([]byte, wire.BulkChunk)}) + if err != nil { + t.Fatal(err) + } + if n := len(frame) + 1; n > portReadBuf { + t.Errorf("a full bulk data frame is %d bytes with its delimiter, portReadBuf %d cannot take it in one read", n, portReadBuf) + } +} From 03bb9b3a108dca6ecca8baabd509a95f7baaabd1 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 20 Sep 2026 15:25:53 +0800 Subject: [PATCH 47/53] docs: a wake regrants the lease the claim was granted --- docs/deploy.md | 2 +- docs/sandboxd-api.md | 3 +-- docs/sdk-python.md | 2 +- docs/sdk.md | 2 +- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/deploy.md b/docs/deploy.md index 6a676b38..c6867380 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -128,7 +128,7 @@ expecting traffic. | `max_claims` | 0 (unlimited) | node-wide cap on live claims; claim/fork/branch requests beyond it answer 429 with the pool state unharmed (on a cluster a non-volume claim tries a warm-peer redirect first; a volume claim answers the 429 with no redirect) | | `audit_log` | false | append every relayed frame that opens an RPC — its op + addressing fields, never payloads — to `/audit.jsonl`; continuation `data`, `data_end`, `stdin`, and `stdin_close` frames pass through unrecorded. The file is size-rotated with one `.1` backup. Records are `{t, id, op}` plus whichever addressing fields the op carries (`argv`, `path`, `dest`, `from`, `to`, `url`, `session`, `port`), plus `method` (`GET`, `CONNECT`, `SOCKS5`, …), `decision` and `secret` (the ref name, never its value) on `egress` records; preview accesses record as op `preview`, one per request. An opening frame whose first line exceeds 4 KiB records as op `oversized` with no addressing fields | | `idle_hibernate_seconds` | 0 (off) | node-wide idle policy for unpooled claims (template/checkpoint claims): a none-lane claim is hibernated once it has had no open data-plane connection (relay, buffered exec, preview) and no egress request in flight for this long; the clock restarts when the last connection closes, so a long command is never cut short. The next call that reaches the guest wakes it transparently. Per-pool `idle_hibernate_seconds` does the same for that pool's claims; pooled keys ignore the node-wide value, and egress pools reject it because they cannot resume safely. Opt in deliberately: a wake costs latency and the snapshot, so callers with their own idle logic must not pay twice | -| `archive_after_seconds` | 0 (off) | tier below hibernation: a hibernated claim idle this long is checkpointed to the store and its local VM dropped, freeing the node entirely; the next call that reaches the guest restores it transparently (a checkpoint restore's latency) with a fresh lease of the TTL the claim was made with (the server default of 5m when it asked for none); nothing renews a lease. Keep `idle_hibernate_seconds` under the leases your claims use, or a woken sandbox that idles again is destroyed by the reap before it can hibernate, its archive already consumed by the wake; one still in use when its lease ends is destroyed the same way. Requires `idle_hibernate_seconds > 0` and must exceed it. Node-wide for unpooled keys; per-pool overrides for that pool | +| `archive_after_seconds` | 0 (off) | tier below hibernation: a hibernated claim idle this long is checkpointed to the store and its local VM dropped, freeing the node entirely; the next call that reaches the guest restores it transparently (a checkpoint restore's latency) with a fresh lease of the length the claim was granted (the server default of 5m when it asked for none); nothing renews a lease. Keep `idle_hibernate_seconds` under the leases your claims use, or a woken sandbox that idles again is destroyed by the reap before it can hibernate, its archive already consumed by the wake; one still in use when its lease ends is destroyed the same way. Requires `idle_hibernate_seconds > 0` and must exceed it. Node-wide for unpooled keys; per-pool overrides for that pool | | `archive_delete_after_seconds` | 0 (keep) | purge an archived claim's store checkpoint this long after it was archived, reclaiming storage; the claim is then gone for good. On archive this retention window replaces the live claim deadline; 0 clears the deadline so the archive is kept forever. Same node-wide/per-pool split | | `mesh` | unset | join a cluster ([Clusters](cluster.md)); unset = single node | | `pools[]` | — | warm pools, keyed by `(template, net, size)`. `warm` defaults to 4 only when both `warm` and `warm_max` are omitted; an explicit `warm: 0` is kept, so `warm: 0` under a `warm_max` starts the pool empty and grows it on demand, and `warm: 0` alone registers the key with no warm VMs (`PUT /v1/pools` never defaults `warm`); `net` is `none` or `egress`; `size` is a tier, below. Retune online without a restart via [`PUT /v1/pools`](sandboxd-api.md#put-v1pools) — omitted pools drain. This is the **first-boot seed**: once a node takes a `PUT /v1/pools`, the applied set persists to `/pools.json` and overrides this section on every later boot (a startup log notes it); delete `pools.json` to return to config-owned pools. Egress stays config-owned either way. See [state ownership](cluster.md#state-ownership) | diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index a96fcae0..28cc2b9f 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -233,8 +233,7 @@ otherwise only a side effect of the next agent access, so this is the explicit form for warming a sandbox ahead of use. Idempotent on one already running. A hibernated sandbox retains its existing deadline. An archived sandbox instead uses `archive_delete_after_seconds` as its retention deadline -while stored, and waking it starts a fresh lease of the TTL the claim was made -with (the server default when it asked for none). 204 on +while stored, and waking it starts a fresh lease of the length the claim was granted (the server default when it asked for none). 204 on success, 404 unknown id or wrong token. ## POST /v1/sandboxes/{id}/fork diff --git a/docs/sdk-python.md b/docs/sdk-python.md index b77ce0db..ec92e265 100644 --- a/docs/sdk-python.md +++ b/docs/sdk-python.md @@ -191,7 +191,7 @@ gets one connection per call as before. If that deployment also enables `archive_after_seconds`, archiving replaces the original claim deadline with the archive-retention deadline (or no deadline when archives are kept forever). Waking an archive starts a fresh -lease of the TTL the claim was made with (the server default when it asked +lease of the length the claim was granted (the server default when it asked for none). The existing handle's `sb.deadline` remains the value returned when that handle was created; call `client.sandboxes()` to read the current server deadline after an archive/wake transition. diff --git a/docs/sdk.md b/docs/sdk.md index 4ff1c300..1b5e5242 100644 --- a/docs/sdk.md +++ b/docs/sdk.md @@ -287,7 +287,7 @@ egress request) is not swept; the idle clock restarts when that connection ends. If that deployment also enables `archive_after_seconds`, archiving replaces the original claim deadline with the archive-retention deadline (or no deadline when archives are kept forever). Waking an archive starts a fresh -lease of the TTL the claim was made with (the server default when it asked +lease of the length the claim was granted (the server default when it asked for none). The existing handle's `Sandbox.Deadline` is the value returned when that handle was created; call `Client.Sandboxes` to read the current server deadline after an archive/wake transition. From 584e2fb741acf7db5f3a7dcc14727e26f617cd5e Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 20 Sep 2026 15:46:28 +0800 Subject: [PATCH 48/53] cut: inline the redirect walk DeleteTemplate alone used --- sdk/go/client.go | 33 ++++++++------------------------- 1 file changed, 8 insertions(+), 25 deletions(-) diff --git a/sdk/go/client.go b/sdk/go/client.go index 64177090..67303c22 100644 --- a/sdk/go/client.go +++ b/sdk/go/client.go @@ -215,13 +215,15 @@ func (c *Client) DeleteTemplate(ctx context.Context, template string, opts ...Op return err } u.Set(noRedirectQueryParam, "1") - if tryErr := tryEach(redirect, func(addr string) error { - _, retryErr := c.deleteTemplates(ctx, addr, u) - return retryErr - }, retryMiss); tryErr != nil { - return fmt.Errorf("delete template at owner: %w", tryErr) + for _, addr := range redirect { + if _, err = c.deleteTemplates(ctx, addr, u); err == nil { + return nil + } + if he, ok := errors.AsType[*APIError](err); ok && he.Status != http.StatusNotFound { + break + } } - return nil + return fmt.Errorf("delete template at owner: %w", err) } func (c *Client) ownerAt(ctx context.Context, addr, id, token string) (string, error) { @@ -333,25 +335,6 @@ func doNoContent(ctx context.Context, c *Client, method, addr, path string, body return nil } -func tryEach(candidates []string, call func(addr string) error, retry func(error) bool) error { - var lastErr error - for _, addr := range candidates { - lastErr = call(addr) - if lastErr == nil { - return nil - } - if !retry(lastErr) { - return lastErr - } - } - return lastErr -} - -func retryMiss(err error) bool { - he, ok := errors.AsType[*APIError](err) - return !ok || he.Status == http.StatusNotFound -} - func retryTransient(err error) bool { he, ok := errors.AsType[*APIError](err) if !ok { From 8bcdbdd3b934817b49d33eb82f8abedc765417f0 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 20 Sep 2026 15:46:28 +0800 Subject: [PATCH 49/53] cut: one routable-host check in config --- sandboxd/config/config.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sandboxd/config/config.go b/sandboxd/config/config.go index 6d3b65a1..27d587b2 100644 --- a/sandboxd/config/config.go +++ b/sandboxd/config/config.go @@ -423,7 +423,7 @@ func (c *Config) validateClientAdvertise() error { (u.Path != "" && u.Path != "/") || strings.ContainsAny(c.ClientAdvertise, "?#") { return fmt.Errorf("client_advertise must be an http or https origin") } - if ip, _ := netip.ParseAddr(u.Hostname()); ip.IsUnspecified() { + if !routableHost(u.Hostname()) { return fmt.Errorf("client_advertise must name a routable host") } if port := u.Port(); port != "" { @@ -450,7 +450,7 @@ func (c *Config) validateMesh() error { if err != nil { return fmt.Errorf("advertise_addr: %w", err) } - if ip, _ := netip.ParseAddr(host); host == "" || ip.IsUnspecified() { + if !routableHost(host) { return fmt.Errorf("advertise_addr %q is gossiped to peers and must name a routable host", c.AdvertiseAddr) } return nil @@ -618,10 +618,10 @@ func namesHost(addr string) bool { addr = "http://" + addr } u, err := url.Parse(addr) - return err == nil && u.Hostname() != "" && !isUnspecifiedHost(u.Hostname()) + return err == nil && routableHost(u.Hostname()) } -func isUnspecifiedHost(host string) bool { +func routableHost(host string) bool { ip, _ := netip.ParseAddr(host) - return ip.IsUnspecified() + return host != "" && !ip.IsUnspecified() } From d5ff5e065098a946c0a96af1fbb64f83577ddcc7 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 20 Sep 2026 15:46:28 +0800 Subject: [PATCH 50/53] cut: drop the pre-generation record layout readers Nothing has written the flat /export layout or a .old sibling since generations landed: every publish renames /export to its generation dir. The dir Fetch, sweep and touch fallbacks, the s3 prefix fallback, the .old removals and the four tests that planted the old layout go with it. Records published before generations are no longer readable; there is no compatibility window to keep. --- sandboxd/store/dir/dir.go | 29 +++--------- sandboxd/store/dir/dir_test.go | 82 ---------------------------------- sandboxd/store/s3/s3.go | 7 --- sandboxd/store/s3/s3_test.go | 17 ------- sandboxd/store/store.go | 2 +- 5 files changed, 8 insertions(+), 129 deletions(-) diff --git a/sandboxd/store/dir/dir.go b/sandboxd/store/dir/dir.go index b7883e45..87366ca7 100644 --- a/sandboxd/store/dir/dir.go +++ b/sandboxd/store/dir/dir.go @@ -15,7 +15,6 @@ import ( "path/filepath" "regexp" "runtime" - "slices" "strings" "time" @@ -26,7 +25,6 @@ import ( ) const ( - oldSuffix = ".old" // pre-generation crash artifacts, reclaimed at Delete digestPrefix = "digest-" digestWorkerLimit = 8 digestReadBufferSize = 128 << 10 @@ -87,15 +85,10 @@ func (d *Store) Fetch(ctx context.Context, id string) (string, []byte, string, e return "", nil, "", err } dir := filepath.Join(d.root, id, store.ExportGen(meta)) - if _, statErr := os.Stat(dir); errors.Is(statErr, fs.ErrNotExist) { - dir = filepath.Join(d.root, id, store.ExportDir) - switch _, legacyErr := os.Stat(dir); { - case errors.Is(legacyErr, fs.ErrNotExist): - return "", nil, "", store.ErrNotFound - case legacyErr != nil: - return "", nil, "", legacyErr - } - } else if statErr != nil { + switch _, statErr := os.Stat(dir); { + case errors.Is(statErr, fs.ErrNotExist): + return "", nil, "", store.ErrNotFound + case statErr != nil: return "", nil, "", statErr } digest, err := os.ReadFile(filepath.Join(d.root, id, digestName(meta))) //nolint:gosec // id pinned by the instance idRe @@ -139,7 +132,7 @@ func (d *Store) Delete(_ context.Context, id string) error { final := filepath.Join(d.root, id) entries, err := os.ReadDir(final) if errors.Is(err, fs.ErrNotExist) { - return os.RemoveAll(final + oldSuffix) + return nil } if err != nil { return err @@ -152,7 +145,7 @@ func (d *Store) Delete(_ context.Context, id string) error { return err } } - return errors.Join(os.RemoveAll(final), os.RemoveAll(final+oldSuffix)) + return os.RemoveAll(final) } func (d *Store) SweepStaging() error { @@ -277,15 +270,10 @@ func (d *Store) sweepGenerations(id string) (err error) { } current := store.ExportGen(meta) currentDigest := digestName(meta) - hasCurrent := slices.ContainsFunc(entries, func(e fs.DirEntry) bool { return e.Name() == current }) for _, e := range entries { - name := e.Name() - if name == store.MetaFile || name == current || name == currentDigest { + if name := e.Name(); name == store.MetaFile || name == current || name == currentDigest { continue } - if name == store.ExportDir && !hasCurrent { - continue // the legacy flat layout still backs the current meta - } if removeErr := removeAgedEntry(final, e); removeErr != nil { return removeErr } @@ -303,9 +291,6 @@ func touchCurrentGeneration(final string, now time.Time) error { } current := filepath.Join(final, store.ExportGen(meta)) exportErr := os.Chtimes(current, now, now) //nolint:gosec // current is a hash-derived name under our root - if errors.Is(exportErr, fs.ErrNotExist) { - exportErr = os.Chtimes(filepath.Join(final, store.ExportDir), now, now) - } if errors.Is(exportErr, fs.ErrNotExist) { return nil } diff --git a/sandboxd/store/dir/dir_test.go b/sandboxd/store/dir/dir_test.go index d119e3db..328e0342 100644 --- a/sandboxd/store/dir/dir_test.go +++ b/sandboxd/store/dir/dir_test.go @@ -333,33 +333,6 @@ func TestPublishRetriesAfterExpiredInstall(t *testing.T) { } } -func TestFetchLegacyFlatLayout(t *testing.T) { - const id = "ck_00000000000000aa" - root := t.TempDir() - st, err := New(root, store.CheckpointIDRe) - if err != nil { - t.Fatalf("New: %v", err) - } - seedRecord(t, filepath.Join(root, id), "legacy") - - dir, meta, digest, err := st.Fetch(t.Context(), id) - if err != nil { - t.Fatalf("fetch legacy: %v", err) - } - if string(meta) != metaJSON("legacy") || digest != "" || dir != filepath.Join(root, id, store.ExportDir) { - t.Fatalf("legacy fetch = %q %q %q, want the flat export dir without digest", dir, meta, digest) - } - - mustPublish(t, st, id, "modern") - dir, meta, digest, err = st.Fetch(t.Context(), id) - if err != nil { - t.Fatalf("fetch after re-publish: %v", err) - } - if string(meta) != metaJSON("modern") || digest != "" || dir == filepath.Join(root, id, store.ExportDir) { - t.Fatalf("re-published fetch = %q %q %q, want a generation dir without digest", dir, meta, digest) - } -} - func TestPublishSweepsGenerationsBySupersessionAge(t *testing.T) { const id = "ck_00000000000000aa" root := t.TempDir() @@ -441,37 +414,6 @@ func TestSweepSparesPublishingGeneration(t *testing.T) { } } -func TestSweepSparesLegacyFallback(t *testing.T) { - const id = "ck_00000000000000aa" - root := t.TempDir() - st, err := New(root, store.CheckpointIDRe) - if err != nil { - t.Fatalf("New: %v", err) - } - seedRecord(t, filepath.Join(root, id), "legacy") - backdate(t, filepath.Join(root, id, store.MetaFile)) - - if err := st.SweepStaging(); err != nil { - t.Fatalf("sweep: %v", err) - } - if _, _, _, err := st.Fetch(t.Context(), id); err != nil { - t.Fatalf("legacy record swept while still current: %v", err) - } - - backdate(t, filepath.Join(root, id, store.ExportDir)) - mustPublish(t, st, id, "modern") - if _, err := os.Stat(filepath.Join(root, id, store.ExportDir)); err != nil { - t.Fatalf("legacy export reclaimed inside the grace: %v", err) - } - backdate(t, filepath.Join(root, id, store.ExportDir)) - if err := st.SweepStaging(); err != nil { - t.Fatalf("sweep after re-publish: %v", err) - } - if _, err := os.Stat(filepath.Join(root, id, store.ExportDir)); !os.IsNotExist(err) { - t.Fatalf("superseded legacy export survived the graced sweep: %v", err) - } -} - func TestSweepReclaimsOnlyAgedUncommittedGenerations(t *testing.T) { const ( orphanID = "ck_00000000000000aa" @@ -591,30 +533,6 @@ func TestSweepGenerationsToleratesConcurrentDelete(t *testing.T) { } } -func TestDeleteRemovesLegacyResidue(t *testing.T) { - const id = "ck_00000000000000aa" - root := t.TempDir() - st, err := New(root, store.CheckpointIDRe) - if err != nil { - t.Fatalf("New: %v", err) - } - seedRecord(t, filepath.Join(root, id), id) - seedRecord(t, filepath.Join(root, id+oldSuffix), "stale") - - if err := st.Delete(t.Context(), id); err != nil { - t.Fatalf("Delete: %v", err) - } - if _, err := st.ReadMeta(t.Context(), id); !errors.Is(err, store.ErrNotFound) { - t.Errorf("ReadMeta after Delete: %v, want store.ErrNotFound", err) - } - if _, err := os.Stat(filepath.Join(root, id+oldSuffix)); !os.IsNotExist(err) { - t.Errorf("legacy .old residue survived Delete: %v", err) - } - if err := st.Delete(t.Context(), id); err != nil { - t.Errorf("Delete of a missing record: %v, want nil", err) - } -} - func TestMetasSurfacesReadError(t *testing.T) { const id = "ck_00000000000000aa" root := t.TempDir() diff --git a/sandboxd/store/s3/s3.go b/sandboxd/store/s3/s3.go index 255d3c9a..44fa2f25 100644 --- a/sandboxd/store/s3/s3.go +++ b/sandboxd/store/s3/s3.go @@ -332,13 +332,6 @@ func (s *Store) populate(ctx context.Context, id string, meta []byte, gen string if err != nil { return err } - if len(keys) == 0 { - // records published before per-generation prefixes. - exportPrefix = s.key(id, store.ExportDir) + "/" - if keys, err = s.list(ctx, exportPrefix); err != nil { - return err - } - } if len(keys) == 0 { return store.ErrNotFound } diff --git a/sandboxd/store/s3/s3_test.go b/sandboxd/store/s3/s3_test.go index d8a37f01..58202742 100644 --- a/sandboxd/store/s3/s3_test.go +++ b/sandboxd/store/s3/s3_test.go @@ -122,23 +122,6 @@ func TestDeleteRetainsMetaUntilExportsAreGone(t *testing.T) { } } -func TestFetchLegacyExportLayout(t *testing.T) { - const id = "ck_00000000000000aa" - fake := &fakeS3{objects: map[string][]byte{ - "ck/" + id + "/" + store.MetaFile: []byte(`{"id":"` + id + `"}`), - "ck/" + id + "/" + store.ExportDir + "/disk.img": []byte("legacy-bytes"), - }} - st := newTestStore(t, fake) - dir, _, _, err := st.Fetch(t.Context(), id) - if err != nil { - t.Fatalf("Fetch: %v", err) - } - got, err := os.ReadFile(filepath.Join(dir, "disk.img")) - if err != nil || string(got) != "legacy-bytes" { - t.Fatalf("fetched legacy export: %q, %v", got, err) - } -} - func TestFetchReportsMetaWithoutExportAsNotFound(t *testing.T) { const id = "ck_00000000000000cc" fake := &fakeS3{objects: map[string][]byte{ diff --git a/sandboxd/store/store.go b/sandboxd/store/store.go index 72a2a095..d25867b8 100644 --- a/sandboxd/store/store.go +++ b/sandboxd/store/store.go @@ -17,7 +17,7 @@ import ( ) const ( - // ExportDir is the snapshot export under a record: /export. + // ExportDir is the snapshot export inside a staging dir; publish renames it to its ExportGen. ExportDir = "export" // MetaFile is the record's metadata, written last so a lister never sees a half-published record. MetaFile = "meta.json" From a50f82b223c87be83d57171427f2cda9d513f026 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 20 Sep 2026 15:46:28 +0800 Subject: [PATCH 51/53] cut: one stub server per status in the peer probe tests --- sandboxd/store/peer/probe_test.go | 85 ++++++++++--------------------- 1 file changed, 27 insertions(+), 58 deletions(-) diff --git a/sandboxd/store/peer/probe_test.go b/sandboxd/store/peer/probe_test.go index f76fe3ac..9462c936 100644 --- a/sandboxd/store/peer/probe_test.go +++ b/sandboxd/store/peer/probe_test.go @@ -10,14 +10,8 @@ import ( ) func TestOwnersReturnsOnly200(t *testing.T) { - ok := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusOK) - })) - defer ok.Close() - miss := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusNotFound) - })) - defer miss.Close() + ok := statusServer(t, http.StatusOK) + miss := statusServer(t, http.StatusNotFound) p := &HTTPProber{Peers: func() []string { return []string{ok.URL, miss.URL} }} owners := p.Owners(t.Context(), testID) @@ -27,10 +21,7 @@ func TestOwnersReturnsOnly200(t *testing.T) { } func TestOwnersAllMissIsEmpty(t *testing.T) { - miss := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusNotFound) - })) - defer miss.Close() + miss := statusServer(t, http.StatusNotFound) p := &HTTPProber{Peers: func() []string { return []string{miss.URL} }} if owners := p.Owners(t.Context(), testID); len(owners) != 0 { @@ -39,16 +30,8 @@ func TestOwnersAllMissIsEmpty(t *testing.T) { } func TestOwnersHungPeerExcludedWithoutBlockingOthers(t *testing.T) { - block := make(chan struct{}) - hung := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) { - <-block - })) - defer hung.Close() - defer close(block) - ok := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusOK) - })) - defer ok.Close() + hung := hungServer(t) + ok := statusServer(t, http.StatusOK) start := time.Now() p := &HTTPProber{Peers: func() []string { return []string{hung.URL, ok.URL} }} @@ -62,16 +45,8 @@ func TestOwnersHungPeerExcludedWithoutBlockingOthers(t *testing.T) { } func TestOwnersReturnsPromptlyWithOneOwner(t *testing.T) { - block := make(chan struct{}) - hung := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) { - <-block - })) - defer hung.Close() - defer close(block) - ok := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusOK) - })) - defer ok.Close() + hung := hungServer(t) + ok := statusServer(t, http.StatusOK) p := &HTTPProber{Peers: func() []string { return []string{hung.URL, ok.URL} }} start := time.Now() @@ -138,11 +113,7 @@ func TestOwnersDedupsAddrs(t *testing.T) { func TestOwnersCapsAtThree(t *testing.T) { var addrs []string for range 4 { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusOK) - })) - defer srv.Close() - addrs = append(addrs, srv.URL) + addrs = append(addrs, statusServer(t, http.StatusOK).URL) } p := &HTTPProber{Peers: func() []string { return addrs }} @@ -154,19 +125,9 @@ func TestOwnersCapsAtThree(t *testing.T) { func TestOwnersCancelsStragglersOnceCapReached(t *testing.T) { var addrs []string for range maxRedirectOwners { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusOK) - })) - defer srv.Close() - addrs = append(addrs, srv.URL) + addrs = append(addrs, statusServer(t, http.StatusOK).URL) } - block := make(chan struct{}) - hung := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) { - <-block - })) - defer hung.Close() - defer close(block) - addrs = append(addrs, hung.URL) + addrs = append(addrs, hungServer(t).URL) start := time.Now() p := &HTTPProber{Peers: func() []string { return addrs }} @@ -183,11 +144,7 @@ func TestHealOwnersReturnsMoreThanRedirectCap(t *testing.T) { const want = maxRedirectOwners + 2 var addrs []string for range want { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusOK) - })) - defer srv.Close() - addrs = append(addrs, srv.URL) + addrs = append(addrs, statusServer(t, http.StatusOK).URL) } p := &HTTPProber{Peers: func() []string { return addrs }} @@ -197,10 +154,7 @@ func TestHealOwnersReturnsMoreThanRedirectCap(t *testing.T) { } func TestHealOwnersIsExhaustiveDespiteAFastOwner(t *testing.T) { - fast := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusOK) - })) - defer fast.Close() + fast := statusServer(t, http.StatusOK) slow := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { time.Sleep(2 * probeGrace) w.WriteHeader(http.StatusOK) @@ -352,3 +306,18 @@ func TestVerifyProbeMACRejectsOutsideWindow(t *testing.T) { t.Error("VerifyProbeMAC accepted a bucket two steps away, want rejected") } } + +func statusServer(t *testing.T, code int) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(code) })) + t.Cleanup(srv.Close) + return srv +} + +func hungServer(t *testing.T) *httptest.Server { + t.Helper() + block := make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { <-block })) + t.Cleanup(func() { close(block); srv.Close() }) + return srv +} From 5454578a54a8c919223fcb2ca6b67bde313850d2 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 20 Sep 2026 15:46:28 +0800 Subject: [PATCH 52/53] cut: one attach rendezvous fixture in the volume tests --- sandboxd/pool/volume_attach_test.go | 21 +++++---------------- sandboxd/pool/volume_test.go | 11 ++++++++--- 2 files changed, 13 insertions(+), 19 deletions(-) diff --git a/sandboxd/pool/volume_attach_test.go b/sandboxd/pool/volume_attach_test.go index acf48feb..ee5deb60 100644 --- a/sandboxd/pool/volume_attach_test.go +++ b/sandboxd/pool/volume_attach_test.go @@ -4,7 +4,6 @@ import ( "errors" "slices" "strings" - "sync" "testing" "github.com/cocoonstack/sandbox/sandboxd/config" @@ -73,9 +72,7 @@ func TestAttachOnlyClaimAttachesEveryVolumeConcurrently(t *testing.T) { {Name: "dataset", AttachOnly: true}, {Name: "cache", Mode: types.VolumeModeRW, AttachOnly: true}, } - var attaches sync.WaitGroup - attaches.Add(len(requested)) - eng.attachRendezvous = &attaches + rendezvousAttaches(eng, len(requested)) sb, err := m.ClaimProvision(t.Context(), testKey, 0, "", "", requested) if err != nil { @@ -104,9 +101,7 @@ func TestClaimProvisionAttachFailureKeepsHoldsUntilRemoval(t *testing.T) { eng := newFakeEngine() eng.diskAttachErrFor = "broken" eng.removeStall = make(chan struct{}) - var attaches sync.WaitGroup - attaches.Add(2) - eng.attachRendezvous = &attaches + attaches := rendezvousAttaches(eng, 2) m := newVolumeManager(t, eng, []config.VolumeSpec{ {Name: "scratch", Path: scratch, Writable: true}, {Name: "broken", Path: broken}, @@ -170,9 +165,7 @@ func TestClaimWarmAttachFailureKeepsHoldsUntilRemoval(t *testing.T) { eng.diskAttachErrFor = "broken" eng.removeErrFor = "sbx-warm" eng.vms["sbx-warm"] = "/vsock/warm" - var attaches sync.WaitGroup - attaches.Add(2) - eng.attachRendezvous = &attaches + rendezvousAttaches(eng, 2) m := newVolumePoolManager(t, eng, t.TempDir(), []config.VolumeSpec{ {Name: "scratch", Path: scratch, Writable: true}, {Name: "broken", Path: broken}, @@ -216,9 +209,7 @@ func TestFinalizeQuotaFailureKeepsHoldsUntilRemoval(t *testing.T) { eng := newFakeEngine() m := newVolumeManager(t, eng, []config.VolumeSpec{{Name: "scratch", Path: scratch, Writable: true}}) m.maxClaims = 1 - var attaches sync.WaitGroup - attaches.Add(2) - eng.attachRendezvous = &attaches + attaches := rendezvousAttaches(eng, 2) writable := []types.Volume{{Name: "scratch", Mode: types.VolumeModeRW}} claimErr := make(chan error, 1) @@ -291,9 +282,7 @@ func TestFinalizeTenantQuotaFailureQuiescesAndUncountsTenant(t *testing.T) { eng := newFakeEngine() m := newVolumeManager(t, eng, []config.VolumeSpec{{Name: "scratch", Path: scratch, Writable: true}}) m.tenantMax = map[string]int{"acme": 1} - var attaches sync.WaitGroup - attaches.Add(2) - eng.attachRendezvous = &attaches + attaches := rendezvousAttaches(eng, 2) claimErr := make(chan error, 1) go func() { diff --git a/sandboxd/pool/volume_test.go b/sandboxd/pool/volume_test.go index 9a9e7740..8484e4d1 100644 --- a/sandboxd/pool/volume_test.go +++ b/sandboxd/pool/volume_test.go @@ -97,9 +97,7 @@ func TestClaimProvisionBringsMixedVolumesUpConcurrently(t *testing.T) { {Name: "dataset", Mount: "/volumes/dataset"}, {Name: "cache", Mount: "/cache", Mode: types.VolumeModeRW}, } - var attaches sync.WaitGroup - attaches.Add(len(requested)) - eng.attachRendezvous = &attaches + rendezvousAttaches(eng, len(requested)) sb, err := m.ClaimProvision(t.Context(), testKey, 0, "", "", requested) if err != nil { @@ -586,6 +584,13 @@ func newVolumeManager(t *testing.T, eng *fakeEngine, volumes []config.VolumeSpec return newVolumeManagerAt(t, eng, t.TempDir(), volumes) } +func rendezvousAttaches(eng *fakeEngine, n int) *sync.WaitGroup { + var attaches sync.WaitGroup + attaches.Add(n) + eng.attachRendezvous = &attaches + return &attaches +} + func newVolumePoolManager(t *testing.T, eng *fakeEngine, dataDir string, volumes []config.VolumeSpec) *Manager { t.Helper() m, err := NewManager(t.Context(), &config.Config{ From e1a05f373f63a3aa990267b800e49f982fd73405 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 20 Sep 2026 16:10:25 +0800 Subject: [PATCH 53/53] fix(sdk/python): raise typed errors from the agent handshake The connect and TLS phases already turned an OSError into ProtocolError, but the upgrade exchange (send, status line, headers) let a socket timeout or a connection reset out raw, so a node that accepts TCP and stalls the 101 escaped the adapters' SandboxError handling and broke the agent run. Every handshake failure now maps through one helper: a timeout is SandboxTimeout, anything else ProtocolError; a SandboxTimeout raised by remaining_timeout inside the exchange passes through unwrapped. docs/deploy.md: the egress pool example said the pool "or its tenants" needs an egress block; a tenant's claim of a pooled key needs both blocks and gets their intersection, so the sentence now says so. --- docs/deploy.md | 5 +++-- sdk/langchain/tests/test_toolkit.py | 25 ++++++++++++++++++++++++- sdk/python/cocoonsandbox/conn.py | 16 ++++++++++++---- sdk/python/tests/test_endpoint.py | 4 ++-- sdk/python/tests/test_stream.py | 10 ++++++++-- 5 files changed, 49 insertions(+), 11 deletions(-) diff --git a/docs/deploy.md b/docs/deploy.md index c6867380..099b0a26 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -93,8 +93,9 @@ sandboxd reads one JSON file (`-config`, default As written the `egress` pool's guests reach nothing: on a bridge lane the NIC is locked default-deny and the proxy door is bound only for a claim that has a -policy, so give the pool (or its tenants) an [`egress`](egress.md) block before -expecting traffic. +policy. Give the pool an [`egress`](egress.md) block before expecting traffic; +a tenant's claim of it (`acme` above) needs the tenant's own block as well and +gets the intersection of the two. | field | default | meaning | |---|---|---| diff --git a/sdk/langchain/tests/test_toolkit.py b/sdk/langchain/tests/test_toolkit.py index 0e1b8b87..761137d0 100644 --- a/sdk/langchain/tests/test_toolkit.py +++ b/sdk/langchain/tests/test_toolkit.py @@ -2,10 +2,12 @@ shaping, lazy claim, and double-close safety — no real node.""" import asyncio +import socket +import threading import time import pytest -from cocoonsandbox import SilkdError +from cocoonsandbox import Sandbox, SilkdError from cocoonsandbox_langchain import CocoonToolkit from cocoonsandbox_langchain.toolkit import CALL_TIMEOUT @@ -125,6 +127,20 @@ def missing(path): assert "no such file" in tool.invoke({"path": "/nope"}) +def test_a_stalled_agent_upgrade_comes_back_as_a_tool_error(): + server = socket.create_server(("127.0.0.1", 0)) + addr = f"127.0.0.1:{server.getsockname()[1]}" + threading.Thread(target=serve_silence, args=(server,), daemon=True).start() + kit = CocoonToolkit(addr) + kit._client.timeout = 0.2 + kit._sb = Sandbox(kit._client, "sb_1", "tok", addr) + tool = next(t for t in kit.get_tools() if t.name == "sandbox_read_file") + try: + assert "timed out" in tool.invoke({"path": "/etc/hostname"}) + finally: + server.close() + + class FakeSandbox: def __init__(self): self.closed = 0 @@ -160,3 +176,10 @@ def hooked(monkeypatch): fake = FakeSandbox() monkeypatch.setattr(kit, "_claim", lambda deadline=None: fake) return kit, fake + + +def serve_silence(server: socket.socket) -> None: + conn, _ = server.accept() + conn.recv(4096) + time.sleep(1) + conn.close() diff --git a/sdk/python/cocoonsandbox/conn.py b/sdk/python/cocoonsandbox/conn.py index 9bd2c36e..a8018a2d 100644 --- a/sdk/python/cocoonsandbox/conn.py +++ b/sdk/python/cocoonsandbox/conn.py @@ -12,7 +12,7 @@ from collections.abc import Iterator from typing import Any, BinaryIO, Protocol, TypeVar -from .errors import APIError, ProtocolError, SandboxTimeout, SilkdError +from .errors import APIError, ProtocolError, SandboxError, SandboxTimeout, SilkdError from .frames import MAX_FRAME, decode_response, encode_request KEEP_ALIVE_CONNS = 8 @@ -168,7 +168,7 @@ def dial_agent( try: sock = socket.create_connection((host, port), timeout=remaining_timeout(timeout, deadline)) except OSError as exc: - raise ProtocolError(f"dial {addr}: {exc}") from exc + raise _handshake_error(f"dial {addr}", exc) from exc sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) reader = None try: @@ -179,7 +179,7 @@ def dial_agent( try: sock.do_handshake() except OSError as exc: - raise ProtocolError(f"tls handshake {addr}: {exc}") from exc + raise _handshake_error(f"tls handshake {addr}", exc) from exc request = ( f"GET /v1/sandboxes/{sandbox_id}/agent HTTP/1.1\r\n" f"Host: {endpoint.netloc}\r\n" @@ -216,10 +216,12 @@ def dial_agent( remaining_timeout(timeout, deadline) sock.settimeout(None) return Conn(sock, reader) - except Exception: + except Exception as exc: if reader is not None: reader.close() sock.close() + if isinstance(exc, OSError) and not isinstance(exc, SandboxError): + raise _handshake_error(f"agent upgrade {addr}", exc) from exc raise @@ -231,3 +233,9 @@ def remaining_timeout(timeout: float, deadline: float | None, what: str = "agent if remaining <= 0: raise SandboxTimeout(f"{what} timed out") return min(timeout, remaining) + + +def _handshake_error(what: str, exc: OSError) -> SandboxError: + if isinstance(exc, (TimeoutError, socket.timeout)): + return SandboxTimeout(f"{what} timed out") + return ProtocolError(f"{what}: {exc}") diff --git a/sdk/python/tests/test_endpoint.py b/sdk/python/tests/test_endpoint.py index 1a20ffad..89869730 100644 --- a/sdk/python/tests/test_endpoint.py +++ b/sdk/python/tests/test_endpoint.py @@ -3,7 +3,7 @@ import pytest -from cocoonsandbox import Client, ProtocolError +from cocoonsandbox import Client, SandboxTimeout from cocoonsandbox.conn import dial_agent from cocoonsandbox.endpoint import _endpoint_url @@ -58,7 +58,7 @@ def stall() -> None: thread = threading.Thread(target=stall, daemon=True) thread.start() - with pytest.raises(ProtocolError): + with pytest.raises(SandboxTimeout): dial_agent(f"https://127.0.0.1:{server.getsockname()[1]}", "sb_1", "token", 0.05) assert done.wait(5), "timed-out handshake kept its socket open" thread.join() diff --git a/sdk/python/tests/test_stream.py b/sdk/python/tests/test_stream.py index c9e281bf..042448bf 100644 --- a/sdk/python/tests/test_stream.py +++ b/sdk/python/tests/test_stream.py @@ -10,7 +10,8 @@ import pytest from conftest import accept_upgrade, sandbox_at -from cocoonsandbox import Sandbox +from cocoonsandbox import ProtocolError, Sandbox, SandboxTimeout +from cocoonsandbox import conn as conn_module from cocoonsandbox import sandbox as sandbox_module TIMEOUT = 0.2 @@ -156,8 +157,13 @@ def test_dial_is_still_bounded_by_the_client_timeout(): sb = legacy_sandbox(addr) started = time.monotonic() try: - with pytest.raises(OSError): + with pytest.raises(SandboxTimeout): sb.dial_port(5000) finally: server.close() assert time.monotonic() - started < 3 * TIMEOUT + + +@pytest.mark.parametrize(("exc", "want"), [(socket.timeout(), SandboxTimeout), (ConnectionResetError(), ProtocolError)]) +def test_handshake_failures_are_typed(exc, want): + assert isinstance(conn_module._handshake_error("agent upgrade", exc), want)